From 3619d2f383f65108dfd33686119f675aaeea54b7 Mon Sep 17 00:00:00 2001 From: "Matthias J. Sax" Date: Fri, 8 Mar 2019 12:31:34 -0800 Subject: [PATCH 0001/1071] MINOR: cleanup deprectaion annotations (#6290) If deprecated interface methods are inherited, the @Deprication tag should be used (instead on suppressing the deprecation warning). Reviewers: Guozhang Wang , John Roesler , Bill Bejeck --- .../kafka/streams/kstream/JoinWindows.java | 14 +--- .../kafka/streams/kstream/SessionWindows.java | 2 - .../kafka/streams/kstream/TimeWindows.java | 14 ++-- .../streams/kstream/UnlimitedWindows.java | 9 +-- .../apache/kafka/streams/kstream/Windows.java | 3 +- .../kstream/internals/SessionWindow.java | 2 - .../internals/AbstractProcessorContext.java | 4 -- .../ForwardingDisabledProcessorContext.java | 6 +- .../internals/GlobalProcessorContextImpl.java | 4 +- .../internals/InternalProcessorContext.java | 3 + .../internals/ProcessorContextImpl.java | 36 +++++----- .../internals/StandbyContextImpl.java | 8 +-- .../apache/kafka/streams/state/Stores.java | 14 ++-- .../kafka/streams/state/WindowStore.java | 15 +++-- .../state/internals/CachingWindowStore.java | 6 +- .../ChangeLoggingWindowBytesStore.java | 6 +- .../CompositeReadOnlyWindowStore.java | 66 ++++++++++++------- .../state/internals/MeteredWindowStore.java | 6 +- .../state/internals/RocksDBWindowStore.java | 6 +- .../AbstractProcessorContextTest.java | 20 ++++-- ...orwardingDisabledProcessorContextTest.java | 2 + .../GlobalProcessorContextImplTest.java | 2 + .../internals/ProcessorTopologyTest.java | 4 +- .../internals/RecordDeserializerTest.java | 3 +- .../test/InternalMockProcessorContext.java | 14 ++-- .../test/MockInternalProcessorContext.java | 8 +-- .../kafka/test/NoOpProcessorContext.java | 22 +++---- .../processor/MockProcessorContext.java | 16 +++-- 28 files changed, 165 insertions(+), 150 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/JoinWindows.java b/streams/src/main/java/org/apache/kafka/streams/kstream/JoinWindows.java index 219489f9bc4e4..6331877f790ea 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/JoinWindows.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/JoinWindows.java @@ -92,7 +92,7 @@ private JoinWindows(final long beforeMs, this.maintainDurationMs = maintainDurationMs; } - @SuppressWarnings("deprecation") // removing segments from Windows will fix this + @Deprecated // removing segments from Windows will fix this private JoinWindows(final long beforeMs, final long afterMs, final long graceMs, @@ -131,7 +131,6 @@ public static JoinWindows of(final long timeDifferenceMs) throws IllegalArgument * @param timeDifference join window interval * @throws IllegalArgumentException if {@code timeDifference} is negative or can't be represented as {@code long milliseconds} */ - @SuppressWarnings("deprecation") public static JoinWindows of(final Duration timeDifference) throws IllegalArgumentException { final String msgPrefix = prepareMillisCheckFailMsgPrefix(timeDifference, "timeDifference"); return of(ApiUtils.validateMillisecondDuration(timeDifference, msgPrefix)); @@ -148,7 +147,6 @@ public static JoinWindows of(final Duration timeDifference) throws IllegalArgume * @throws IllegalArgumentException if the resulting window size is negative * @deprecated Use {@link #before(Duration)} instead. */ - @SuppressWarnings("deprecation") // removing segments from Windows will fix this @Deprecated public JoinWindows before(final long timeDifferenceMs) throws IllegalArgumentException { return new JoinWindows(timeDifferenceMs, afterMs, graceMs, maintainDurationMs, segments); @@ -164,7 +162,6 @@ public JoinWindows before(final long timeDifferenceMs) throws IllegalArgumentExc * @param timeDifference relative window start time * @throws IllegalArgumentException if the resulting window size is negative or {@code timeDifference} can't be represented as {@code long milliseconds} */ - @SuppressWarnings("deprecation") // removing segments from Windows will fix this public JoinWindows before(final Duration timeDifference) throws IllegalArgumentException { final String msgPrefix = prepareMillisCheckFailMsgPrefix(timeDifference, "timeDifference"); return before(ApiUtils.validateMillisecondDuration(timeDifference, msgPrefix)); @@ -181,7 +178,6 @@ public JoinWindows before(final Duration timeDifference) throws IllegalArgumentE * @throws IllegalArgumentException if the resulting window size is negative * @deprecated Use {@link #after(Duration)} instead */ - @SuppressWarnings("deprecation") // removing segments from Windows will fix this @Deprecated public JoinWindows after(final long timeDifferenceMs) throws IllegalArgumentException { return new JoinWindows(beforeMs, timeDifferenceMs, graceMs, maintainDurationMs, segments); @@ -197,7 +193,6 @@ public JoinWindows after(final long timeDifferenceMs) throws IllegalArgumentExce * @param timeDifference relative window end time * @throws IllegalArgumentException if the resulting window size is negative or {@code timeDifference} can't be represented as {@code long milliseconds} */ - @SuppressWarnings("deprecation") // removing segments from Windows will fix this public JoinWindows after(final Duration timeDifference) throws IllegalArgumentException { final String msgPrefix = prepareMillisCheckFailMsgPrefix(timeDifference, "timeDifference"); return after(ApiUtils.validateMillisecondDuration(timeDifference, msgPrefix)); @@ -239,7 +234,6 @@ public JoinWindows grace(final Duration afterWindowEnd) throws IllegalArgumentEx return new JoinWindows(beforeMs, afterMs, afterWindowEndMs, maintainDurationMs, segments); } - @SuppressWarnings("deprecation") // continuing to support Windows#maintainMs/segmentInterval in fallback mode @Override public long gracePeriodMs() { // NOTE: in the future, when we remove maintainMs, @@ -254,7 +248,6 @@ public long gracePeriodMs() { * @throws IllegalArgumentException if {@code durationMs} is smaller than the window size * @deprecated since 2.1. Use {@link Materialized#withRetention(Duration)} instead. */ - @SuppressWarnings("deprecation") @Override @Deprecated public JoinWindows until(final long durationMs) throws IllegalArgumentException { @@ -272,14 +265,13 @@ public JoinWindows until(final long durationMs) throws IllegalArgumentException * @return the window maintain duration * @deprecated since 2.1. This function should not be used anymore as retention period can be specified via {@link Materialized#withRetention(Duration)}. */ - @SuppressWarnings({"deprecation", "deprecatedMemberStillInUse"}) @Override @Deprecated public long maintainMs() { return Math.max(maintainDurationMs, size()); } - @SuppressWarnings({"deprecation", "NonFinalFieldReferenceInEquals"}) // removing segments from Windows will fix this + @SuppressWarnings("deprecation") // removing segments from Windows will fix this @Override public boolean equals(final Object o) { if (this == o) { @@ -296,7 +288,7 @@ public boolean equals(final Object o) { graceMs == that.graceMs; } - @SuppressWarnings({"deprecation", "NonFinalFieldReferencedInHashCode"}) // removing segments from Windows will fix this + @SuppressWarnings("deprecation") // removing segments from Windows will fix this @Override public int hashCode() { return Objects.hash(beforeMs, afterMs, graceMs, maintainDurationMs, segments); diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindows.java b/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindows.java index 9c77fa535de9c..c0153a30fbe94 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindows.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindows.java @@ -108,7 +108,6 @@ public static SessionWindows with(final long inactivityGapMs) { * * @throws IllegalArgumentException if {@code inactivityGap} is zero or negative or can't be represented as {@code long milliseconds} */ - @SuppressWarnings("deprecation") public static SessionWindows with(final Duration inactivityGap) { final String msgPrefix = prepareMillisCheckFailMsgPrefix(inactivityGap, "inactivityGap"); return with(ApiUtils.validateMillisecondDuration(inactivityGap, msgPrefix)); @@ -163,7 +162,6 @@ public SessionWindows grace(final Duration afterWindowEnd) throws IllegalArgumen @SuppressWarnings("deprecation") // continuing to support Windows#maintainMs/segmentInterval in fallback mode public long gracePeriodMs() { - // NOTE: in the future, when we remove maintainMs, // we should default the grace period to 24h to maintain the default behavior, // or we can default to (24h - gapMs) if you want to be super accurate. diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindows.java b/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindows.java index 03203f04ed087..a87dbf366bb2d 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindows.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindows.java @@ -79,7 +79,6 @@ private TimeWindows(final long sizeMs, final long advanceMs, final long graceMs, } /** Private constructor for preserving segments. Can be removed along with Windows.segments. **/ - @SuppressWarnings("DeprecatedIsStillUsed") @Deprecated private TimeWindows(final long sizeMs, final long advanceMs, @@ -127,7 +126,7 @@ public static TimeWindows of(final long sizeMs) throws IllegalArgumentException * @return a new window definition with default maintain duration of 1 day * @throws IllegalArgumentException if the specified window size is zero or negative or can't be represented as {@code long milliseconds} */ - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // removing #of(final long sizeMs) will fix this public static TimeWindows of(final Duration size) throws IllegalArgumentException { final String msgPrefix = prepareMillisCheckFailMsgPrefix(size, "size"); return of(ApiUtils.validateMillisecondDuration(size, msgPrefix)); @@ -145,7 +144,6 @@ public static TimeWindows of(final Duration size) throws IllegalArgumentExceptio * @throws IllegalArgumentException if the advance interval is negative, zero, or larger than the window size * @deprecated Use {@link #advanceBy(Duration)} instead */ - @SuppressWarnings("deprecation") // will be fixed when we remove segments from Windows @Deprecated public TimeWindows advanceBy(final long advanceMs) { if (advanceMs <= 0 || advanceMs > sizeMs) { @@ -166,7 +164,7 @@ public TimeWindows advanceBy(final long advanceMs) { * @return a new window definition with default maintain duration of 1 day * @throws IllegalArgumentException if the advance interval is negative, zero, or larger than the window size */ - @SuppressWarnings("deprecation") // will be fixed when we remove segments from Windows + @SuppressWarnings("deprecation") // removing #advanceBy(final long advanceMs) will fix this public TimeWindows advanceBy(final Duration advance) { final String msgPrefix = prepareMillisCheckFailMsgPrefix(advance, "advance"); return advanceBy(ApiUtils.validateMillisecondDuration(advance, msgPrefix)); @@ -227,7 +225,6 @@ public long gracePeriodMs() { * @deprecated since 2.1. Use {@link Materialized#retention} or directly configure the retention in a store supplier * and use {@link Materialized#as(WindowBytesStoreSupplier)}. */ - @SuppressWarnings("deprecation") @Override @Deprecated public TimeWindows until(final long durationMs) throws IllegalArgumentException { @@ -245,14 +242,13 @@ public TimeWindows until(final long durationMs) throws IllegalArgumentException * @return the window maintain duration * @deprecated since 2.1. Use {@link Materialized#retention} instead. */ - @SuppressWarnings({"DeprecatedIsStillUsed", "deprecation"}) @Override @Deprecated public long maintainMs() { return Math.max(maintainDurationMs, sizeMs); } - @SuppressWarnings({"deprecation", "NonFinalFieldReferenceInEquals"}) // removing segments from Windows will fix this + @SuppressWarnings("deprecation") // removing segments from Windows will fix this @Override public boolean equals(final Object o) { if (this == o) { @@ -269,13 +265,13 @@ public boolean equals(final Object o) { graceMs == that.graceMs; } - @SuppressWarnings({"deprecation", "NonFinalFieldReferencedInHashCode"}) // removing segments from Windows will fix this + @SuppressWarnings("deprecation") // removing segments from Windows will fix this @Override public int hashCode() { return Objects.hash(maintainDurationMs, segments, sizeMs, advanceMs, graceMs); } - @SuppressWarnings({"deprecation"}) // removing segments from Windows will fix this + @SuppressWarnings("deprecation") // removing segments from Windows will fix this @Override public String toString() { return "TimeWindows{" + diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/UnlimitedWindows.java b/streams/src/main/java/org/apache/kafka/streams/kstream/UnlimitedWindows.java index e1894ba088b9d..f8ec6eeb2db4b 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/UnlimitedWindows.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/UnlimitedWindows.java @@ -84,7 +84,6 @@ public UnlimitedWindows startOn(final long startMs) throws IllegalArgumentExcept * @return a new unlimited window that starts at {@code start} * @throws IllegalArgumentException if the start time is negative or can't be represented as {@code long milliseconds} */ - @SuppressWarnings("deprecation") public UnlimitedWindows startOn(final Instant start) throws IllegalArgumentException { final String msgPrefix = prepareMillisCheckFailMsgPrefix(start, "start"); return startOn(ApiUtils.validateMillisecondInstant(start, msgPrefix)); @@ -120,7 +119,6 @@ public long size() { * @throws IllegalArgumentException on every invocation. * @deprecated since 2.1. */ - @SuppressWarnings("deprecation") @Override @Deprecated public UnlimitedWindows until(final long durationMs) { @@ -134,7 +132,6 @@ public UnlimitedWindows until(final long durationMs) { * @return the window retention time that is {@link Long#MAX_VALUE} * @deprecated since 2.1. Use {@link Materialized#retention} instead. */ - @SuppressWarnings("deprecation") @Override @Deprecated public long maintainMs() { @@ -146,7 +143,7 @@ public long gracePeriodMs() { return 0L; } - @SuppressWarnings({"deprecation", "NonFinalFieldReferenceInEquals"}) // removing segments from Windows will fix this + @SuppressWarnings("deprecation") // removing segments from Windows will fix this @Override public boolean equals(final Object o) { if (this == o) { @@ -159,13 +156,13 @@ public boolean equals(final Object o) { return startMs == that.startMs && segments == that.segments; } - @SuppressWarnings({"deprecation", "NonFinalFieldReferencedInHashCode"}) // removing segments from Windows will fix this + @SuppressWarnings("deprecation") // removing segments from Windows will fix this @Override public int hashCode() { return Objects.hash(startMs, segments); } - @SuppressWarnings({"deprecation"}) // removing segments from Windows will fix this + @SuppressWarnings("deprecation") // removing segments from Windows will fix this @Override public String toString() { return "UnlimitedWindows{" + diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Windows.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Windows.java index feaee1e13362f..e122b4a5dcaab 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/Windows.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Windows.java @@ -46,7 +46,7 @@ public abstract class Windows { protected Windows() {} - @SuppressWarnings("deprecation") // remove this constructor when we remove segments. + @Deprecated // remove this constructor when we remove segments. Windows(final int segments) { this.segments = segments; } @@ -77,7 +77,6 @@ public Windows until(final long durationMs) throws IllegalArgumentException { * @return the window maintain duration * @deprecated since 2.1. Use {@link Materialized#retention} instead. */ - @SuppressWarnings("DeprecatedIsStillUsed") @Deprecated public long maintainMs() { return maintainDurationMs; diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionWindow.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionWindow.java index 8111cdf35b237..3057e32780d84 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionWindow.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionWindow.java @@ -16,7 +16,6 @@ */ package org.apache.kafka.streams.kstream.internals; -import org.apache.kafka.common.annotation.InterfaceStability; import org.apache.kafka.streams.kstream.Window; /** @@ -29,7 +28,6 @@ * @see org.apache.kafka.streams.kstream.SessionWindows * @see org.apache.kafka.streams.processor.TimestampExtractor */ -@InterfaceStability.Unstable public final class SessionWindow extends Window { /** diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractProcessorContext.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractProcessorContext.java index af8b073092b98..ef1799bf0f0fa 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractProcessorContext.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractProcessorContext.java @@ -127,7 +127,6 @@ public int partition() { if (recordContext == null) { throw new IllegalStateException("This should not happen as partition() should only be called while a record is processed"); } - return recordContext.partition(); } @@ -139,7 +138,6 @@ public long offset() { if (recordContext == null) { throw new IllegalStateException("This should not happen as offset() should only be called while a record is processed"); } - return recordContext.offset(); } @@ -148,7 +146,6 @@ public Headers headers() { if (recordContext == null) { throw new IllegalStateException("This should not happen as headers() should only be called while a record is processed"); } - return recordContext.headers(); } @@ -160,7 +157,6 @@ public long timestamp() { if (recordContext == null) { throw new IllegalStateException("This should not happen as timestamp() should only be called while a record is processed"); } - return recordContext.timestamp(); } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ForwardingDisabledProcessorContext.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ForwardingDisabledProcessorContext.java index 0ef70b770305e..ba3936897f9a7 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ForwardingDisabledProcessorContext.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ForwardingDisabledProcessorContext.java @@ -16,7 +16,6 @@ */ package org.apache.kafka.streams.processor.internals; -import java.time.Duration; import org.apache.kafka.common.header.Headers; import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.streams.StreamsMetrics; @@ -31,6 +30,7 @@ import org.apache.kafka.streams.processor.To; import java.io.File; +import java.time.Duration; import java.util.Map; import java.util.Objects; @@ -110,14 +110,14 @@ public void forward(final K key, final V value, final To to) { throw new StreamsException("ProcessorContext#forward() not supported."); } - @SuppressWarnings("deprecation") @Override + @Deprecated public void forward(final K key, final V value, final int childIndex) { throw new StreamsException("ProcessorContext#forward() not supported."); } - @SuppressWarnings("deprecation") @Override + @Deprecated public void forward(final K key, final V value, final String childName) { throw new StreamsException("ProcessorContext#forward() not supported."); } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalProcessorContextImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalProcessorContextImpl.java index 900cc71e7f6e4..0693ef780bbc5 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalProcessorContextImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalProcessorContextImpl.java @@ -88,8 +88,8 @@ public void forward(final K key, final V value, final To to) { /** * @throws UnsupportedOperationException on every invocation */ - @SuppressWarnings("deprecation") @Override + @Deprecated public void forward(final K key, final V value, final int childIndex) { throw new UnsupportedOperationException("this should not happen: forward() not supported in global processor context."); } @@ -97,8 +97,8 @@ public void forward(final K key, final V value, final int childIndex) { /** * @throws UnsupportedOperationException on every invocation */ - @SuppressWarnings("deprecation") @Override + @Deprecated public void forward(final K key, final V value, final String childName) { throw new UnsupportedOperationException("this should not happen: forward() not supported in global processor context."); } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalProcessorContext.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalProcessorContext.java index 0f67dff9fa22c..2a1d05e2807db 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalProcessorContext.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalProcessorContext.java @@ -47,6 +47,9 @@ public interface InternalProcessorContext extends ProcessorContext { */ void setCurrentNode(ProcessorNode currentNode); + /** + * Get the current {@link ProcessorNode} + */ ProcessorNode currentNode(); /** diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextImpl.java index 764d50c2ac1e4..2afd5e9a687e0 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextImpl.java @@ -128,8 +128,9 @@ public void forward(final K key, forward(key, value, SEND_TO_ALL); } - @SuppressWarnings({"unchecked", "deprecation"}) + @SuppressWarnings("unchecked") @Override + @Deprecated public void forward(final K key, final V value, final int childIndex) { @@ -139,8 +140,9 @@ public void forward(final K key, To.child(((List) currentNode().children()).get(childIndex).name())); } - @SuppressWarnings({"unchecked", "deprecation"}) + @SuppressWarnings("unchecked") @Override + @Deprecated public void forward(final K key, final V value, final String childName) { @@ -192,16 +194,16 @@ public void commit() { @Override @Deprecated - public Cancellable schedule(final long interval, + public Cancellable schedule(final long intervalMs, final PunctuationType type, final Punctuator callback) { - if (interval < 1) { + if (intervalMs < 1) { throw new IllegalArgumentException("The minimum supported scheduling interval is 1 millisecond."); } - return task.schedule(interval, type, callback); + return task.schedule(intervalMs, type, callback); } - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // removing #schedule(final long intervalMs,...) will fix this @Override public Cancellable schedule(final Duration interval, final PunctuationType type, @@ -315,16 +317,16 @@ public V fetch(final K key, return wrapped().fetch(key, time); } - @Deprecated @Override + @Deprecated public WindowStoreIterator fetch(final K key, final long timeFrom, final long timeTo) { return wrapped().fetch(key, timeFrom, timeTo); } - @Deprecated @Override + @Deprecated public KeyValueIterator, V> fetch(final K from, final K to, final long timeFrom, @@ -337,8 +339,8 @@ public KeyValueIterator, V> all() { return wrapped().all(); } - @Deprecated @Override + @Deprecated public KeyValueIterator, V> fetchAll(final long timeFrom, final long timeTo) { return wrapped().fetchAll(timeFrom, timeTo); @@ -505,7 +507,7 @@ public V fetch(final K key, return wrapped().fetch(key, time); } - @Deprecated + @SuppressWarnings("deprecation") // note, this method must be kept if super#fetch(...) is removed @Override public WindowStoreIterator fetch(final K key, final long timeFrom, @@ -513,7 +515,7 @@ public WindowStoreIterator fetch(final K key, return wrapped().fetch(key, timeFrom, timeTo); } - @Deprecated + @SuppressWarnings("deprecation") // note, this method must be kept if super#fetch(...) is removed @Override public KeyValueIterator, V> fetch(final K from, final K to, @@ -522,17 +524,17 @@ public KeyValueIterator, V> fetch(final K from, return wrapped().fetch(from, to, timeFrom, timeTo); } - @Override - public KeyValueIterator, V> all() { - return wrapped().all(); - } - - @Deprecated + @SuppressWarnings("deprecation") // note, this method must be kept if super#fetch(...) is removed @Override public KeyValueIterator, V> fetchAll(final long timeFrom, final long timeTo) { return wrapped().fetchAll(timeFrom, timeTo); } + + @Override + public KeyValueIterator, V> all() { + return wrapped().all(); + } } private static class TimestampedWindowStoreReadWriteDecorator diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyContextImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyContextImpl.java index ee693739dc7aa..49dc5f63ddf78 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyContextImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyContextImpl.java @@ -16,7 +16,6 @@ */ package org.apache.kafka.streams.processor.internals; -import java.time.Duration; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.header.Headers; @@ -33,6 +32,7 @@ import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl; import org.apache.kafka.streams.state.internals.ThreadCache; +import java.time.Duration; import java.util.Collections; import java.util.Map; @@ -161,8 +161,8 @@ public void forward(final K key, final V value, final To to) { /** * @throws UnsupportedOperationException on every invocation */ - @SuppressWarnings("deprecation") @Override + @Deprecated public void forward(final K key, final V value, final int childIndex) { throw new UnsupportedOperationException("this should not happen: forward() not supported in standby tasks."); } @@ -170,8 +170,8 @@ public void forward(final K key, final V value, final int childIndex) { /** * @throws UnsupportedOperationException on every invocation */ - @SuppressWarnings("deprecation") @Override + @Deprecated public void forward(final K key, final V value, final String childName) { throw new UnsupportedOperationException("this should not happen: forward() not supported in standby tasks."); } @@ -188,7 +188,7 @@ public void commit() { * @throws UnsupportedOperationException on every invocation */ @Override - @SuppressWarnings("deprecation") + @Deprecated public Cancellable schedule(final long interval, final PunctuationType type, final Punctuator callback) { throw new UnsupportedOperationException("this should not happen: schedule() not supported in standby tasks."); } diff --git a/streams/src/main/java/org/apache/kafka/streams/state/Stores.java b/streams/src/main/java/org/apache/kafka/streams/state/Stores.java index 113e53141be33..ac2a02378d891 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/Stores.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/Stores.java @@ -196,7 +196,7 @@ public static WindowBytesStoreSupplier inMemoryWindowStore(final String name, * @return an instance of {@link WindowBytesStoreSupplier} * @deprecated since 2.1 Use {@link Stores#persistentWindowStore(String, Duration, Duration, boolean)} instead */ - @Deprecated + @Deprecated // continuing to support Windows#maintainMs/segmentInterval in fallback mode public static WindowBytesStoreSupplier persistentWindowStore(final String name, final long retentionPeriod, final int numSegments, @@ -271,21 +271,21 @@ private static WindowBytesStoreSupplier persistentWindowStore(final String name, /** * Create a persistent {@link SessionBytesStoreSupplier}. * @param name name of the store (cannot be {@code null}) - * @param retentionPeriod length ot time to retain data in the store (cannot be negative) + * @param retentionPeriodMs length ot time to retain data in the store (cannot be negative) * Note that the retention period must be at least long enough to contain the * windowed data's entire life cycle, from window-start through window-end, * and for the entire grace period. * @return an instance of a {@link SessionBytesStoreSupplier} * @deprecated since 2.1 Use {@link Stores#persistentSessionStore(String, Duration)} instead */ - @Deprecated + @Deprecated // continuing to support Windows#maintainMs/segmentInterval in fallback mode public static SessionBytesStoreSupplier persistentSessionStore(final String name, - final long retentionPeriod) { + final long retentionPeriodMs) { Objects.requireNonNull(name, "name cannot be null"); - if (retentionPeriod < 0) { + if (retentionPeriodMs < 0) { throw new IllegalArgumentException("retentionPeriod cannot be negative"); } - return new RocksDbSessionBytesStoreSupplier(name, retentionPeriod); + return new RocksDbSessionBytesStoreSupplier(name, retentionPeriodMs); } /** @@ -297,7 +297,7 @@ public static SessionBytesStoreSupplier persistentSessionStore(final String name * and for the entire grace period. * @return an instance of a {@link SessionBytesStoreSupplier} */ - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // removing #persistentSessionStore(String name, long retentionPeriodMs) will fix this public static SessionBytesStoreSupplier persistentSessionStore(final String name, final Duration retentionPeriod) { final String msgPrefix = prepareMillisCheckFailMsgPrefix(retentionPeriod, "retentionPeriod"); diff --git a/streams/src/main/java/org/apache/kafka/streams/state/WindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/WindowStore.java index f7eb37eb6fd43..83a0ee1d2836e 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/WindowStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/WindowStore.java @@ -91,11 +91,13 @@ public interface WindowStore extends StateStore, ReadOnlyWindowStore * @throws InvalidStateStoreException if the store is not initialized * @throws NullPointerException if the given key is {@code null} */ - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // note, this method must be kept if super#fetch(...) is removed WindowStoreIterator fetch(K key, long timeFrom, long timeTo); @Override - default WindowStoreIterator fetch(final K key, final Instant from, final Instant to) { + default WindowStoreIterator fetch(final K key, + final Instant from, + final Instant to) { return fetch( key, ApiUtils.validateMillisecondInstant(from, prepareMillisCheckFailMsgPrefix(from, "from")), @@ -115,11 +117,14 @@ default WindowStoreIterator fetch(final K key, final Instant from, final Inst * @throws InvalidStateStoreException if the store is not initialized * @throws NullPointerException if one of the given keys is {@code null} */ - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // note, this method must be kept if super#fetch(...) is removed KeyValueIterator, V> fetch(K from, K to, long timeFrom, long timeTo); @Override - default KeyValueIterator, V> fetch(final K from, final K to, final Instant fromTime, final Instant toTime) { + default KeyValueIterator, V> fetch(final K from, + final K to, + final Instant fromTime, + final Instant toTime) { return fetch( from, to, @@ -135,7 +140,7 @@ default KeyValueIterator, V> fetch(final K from, final K to, final I * @return an iterator over windowed key-value pairs {@code , value>} * @throws InvalidStateStoreException if the store is not initialized */ - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // note, this method must be kept if super#fetchAll(...) is removed KeyValueIterator, V> fetchAll(long timeFrom, long timeTo); @Override diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java index 0a869da9d57d9..0edd8f265b4a4 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java @@ -165,7 +165,7 @@ public byte[] fetch(final Bytes key, } } - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // note, this method must be kept if super#fetch(...) is removed @Override public synchronized WindowStoreIterator fetch(final Bytes key, final long timeFrom, @@ -190,7 +190,7 @@ public synchronized WindowStoreIterator fetch(final Bytes key, return new MergedSortedCacheWindowStoreIterator(filteredCacheIterator, underlyingIterator); } - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // note, this method must be kept if super#fetch(...) is removed @Override public KeyValueIterator, byte[]> fetch(final Bytes from, final Bytes to, @@ -221,7 +221,7 @@ public KeyValueIterator, byte[]> fetch(final Bytes from, ); } - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // note, this method must be kept if super#fetchAll(...) is removed @Override public KeyValueIterator, byte[]> fetchAll(final long timeFrom, final long timeTo) { diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStore.java index ef5a4c7f63999..c58e9f09e59f2 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStore.java @@ -65,7 +65,7 @@ public byte[] fetch(final Bytes key, return wrapped().fetch(key, timestamp); } - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // note, this method must be kept if super#fetch(...) is removed @Override public WindowStoreIterator fetch(final Bytes key, final long from, @@ -73,7 +73,7 @@ public WindowStoreIterator fetch(final Bytes key, return wrapped().fetch(key, from, to); } - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // note, this method must be kept if super#fetch(...) is removed @Override public KeyValueIterator, byte[]> fetch(final Bytes keyFrom, final Bytes keyTo, @@ -87,7 +87,7 @@ public KeyValueIterator, byte[]> all() { return wrapped().all(); } - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // note, this method must be kept if super#fetchAll(...) is removed @Override public KeyValueIterator, byte[]> fetchAll(final long timeFrom, final long timeTo) { diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlyWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlyWindowStore.java index 09080859a5fd1..fbfc7a02e9c3f 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlyWindowStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlyWindowStore.java @@ -69,7 +69,9 @@ public V fetch(final K key, final long time) { @Override @Deprecated - public WindowStoreIterator fetch(final K key, final long timeFrom, final long timeTo) { + public WindowStoreIterator fetch(final K key, + final long timeFrom, + final long timeTo) { Objects.requireNonNull(key, "key can't be null"); final List> stores = provider.stores(storeName, windowStoreType); for (final ReadOnlyWindowStore windowStore : stores) { @@ -89,29 +91,39 @@ public WindowStoreIterator fetch(final K key, final long timeFrom, final long return KeyValueIterators.emptyWindowStoreIterator(); } - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // removing fetch(K from, long from, long to) will fix this @Override - public WindowStoreIterator fetch(final K key, final Instant from, final Instant to) throws IllegalArgumentException { + public WindowStoreIterator fetch(final K key, + final Instant from, + final Instant to) throws IllegalArgumentException { return fetch( key, ApiUtils.validateMillisecondInstant(from, prepareMillisCheckFailMsgPrefix(from, "from")), ApiUtils.validateMillisecondInstant(to, prepareMillisCheckFailMsgPrefix(to, "to"))); } - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // removing fetch(K from, K to, long from, long to) will fix this @Override - public KeyValueIterator, V> fetch(final K from, final K to, final long timeFrom, final long timeTo) { + public KeyValueIterator, V> fetch(final K from, + final K to, + final long timeFrom, + final long timeTo) { Objects.requireNonNull(from, "from can't be null"); Objects.requireNonNull(to, "to can't be null"); - final NextIteratorFunction, V, ReadOnlyWindowStore> nextIteratorFunction = store -> store.fetch(from, to, timeFrom, timeTo); - return new DelegatingPeekingKeyValueIterator<>(storeName, - new CompositeKeyValueIterator<>( - provider.stores(storeName, windowStoreType).iterator(), - nextIteratorFunction)); + final NextIteratorFunction, V, ReadOnlyWindowStore> nextIteratorFunction = + store -> store.fetch(from, to, timeFrom, timeTo); + return new DelegatingPeekingKeyValueIterator<>( + storeName, + new CompositeKeyValueIterator<>( + provider.stores(storeName, windowStoreType).iterator(), + nextIteratorFunction)); } @Override - public KeyValueIterator, V> fetch(final K from, final K to, final Instant fromTime, final Instant toTime) throws IllegalArgumentException { + public KeyValueIterator, V> fetch(final K from, + final K to, + final Instant fromTime, + final Instant toTime) throws IllegalArgumentException { return fetch( from, to, @@ -121,26 +133,32 @@ public KeyValueIterator, V> fetch(final K from, final K to, final In @Override public KeyValueIterator, V> all() { - final NextIteratorFunction, V, ReadOnlyWindowStore> nextIteratorFunction = ReadOnlyWindowStore::all; - return new DelegatingPeekingKeyValueIterator<>(storeName, - new CompositeKeyValueIterator<>( - provider.stores(storeName, windowStoreType).iterator(), - nextIteratorFunction)); + final NextIteratorFunction, V, ReadOnlyWindowStore> nextIteratorFunction = + ReadOnlyWindowStore::all; + return new DelegatingPeekingKeyValueIterator<>( + storeName, + new CompositeKeyValueIterator<>( + provider.stores(storeName, windowStoreType).iterator(), + nextIteratorFunction)); } @Override @Deprecated - public KeyValueIterator, V> fetchAll(final long timeFrom, final long timeTo) { - final NextIteratorFunction, V, ReadOnlyWindowStore> nextIteratorFunction = store -> store.fetchAll(timeFrom, timeTo); - return new DelegatingPeekingKeyValueIterator<>(storeName, - new CompositeKeyValueIterator<>( - provider.stores(storeName, windowStoreType).iterator(), - nextIteratorFunction)); + public KeyValueIterator, V> fetchAll(final long timeFrom, + final long timeTo) { + final NextIteratorFunction, V, ReadOnlyWindowStore> nextIteratorFunction = + store -> store.fetchAll(timeFrom, timeTo); + return new DelegatingPeekingKeyValueIterator<>( + storeName, + new CompositeKeyValueIterator<>( + provider.stores(storeName, windowStoreType).iterator(), + nextIteratorFunction)); } - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // removing fetchAll(long from, long to) will fix this @Override - public KeyValueIterator, V> fetchAll(final Instant from, final Instant to) throws IllegalArgumentException { + public KeyValueIterator, V> fetchAll(final Instant from, + final Instant to) throws IllegalArgumentException { return fetchAll( ApiUtils.validateMillisecondInstant(from, prepareMillisCheckFailMsgPrefix(from, "from")), ApiUtils.validateMillisecondInstant(to, prepareMillisCheckFailMsgPrefix(to, "to"))); diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java index 681b210b4ad74..6d2eaab3d196c 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java @@ -159,7 +159,7 @@ public V fetch(final K key, } } - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // note, this method must be kept if super#fetch(...) is removed @Override public WindowStoreIterator fetch(final K key, final long timeFrom, @@ -171,7 +171,7 @@ public WindowStoreIterator fetch(final K key, time); } - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // note, this method must be kept if super#fetchAll(...) is removed @Override public KeyValueIterator, V> fetch(final K from, final K to, @@ -185,7 +185,7 @@ public KeyValueIterator, V> fetch(final K from, time); } - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // note, this method must be kept if super#fetch(...) is removed @Override public KeyValueIterator, V> fetchAll(final long timeFrom, final long timeTo) { diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStore.java index e621290f20605..3b634ebea200a 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStore.java @@ -69,14 +69,14 @@ public byte[] fetch(final Bytes key, final long timestamp) { return bytesValue; } - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // note, this method must be kept if super#fetch(...) is removed @Override public WindowStoreIterator fetch(final Bytes key, final long timeFrom, final long timeTo) { final KeyValueIterator bytesIterator = wrapped().fetch(key, timeFrom, timeTo); return new WindowStoreIteratorWrapper(bytesIterator, windowSize).valuesIterator(); } - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // note, this method must be kept if super#fetch(...) is removed @Override public KeyValueIterator, byte[]> fetch(final Bytes from, final Bytes to, @@ -92,7 +92,7 @@ public KeyValueIterator, byte[]> all() { return new WindowStoreIteratorWrapper(bytesIterator, windowSize).keyValueIterator(); } - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // note, this method must be kept if super#fetchAll(...) is removed @Override public KeyValueIterator, byte[]> fetchAll(final long timeFrom, final long timeTo) { final KeyValueIterator bytesIterator = wrapped().fetchAll(timeFrom, timeTo); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractProcessorContextTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractProcessorContextTest.java index 8afd30298b54c..1548e7ef0dd15 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractProcessorContextTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractProcessorContextTest.java @@ -16,7 +16,6 @@ */ package org.apache.kafka.streams.processor.internals; -import java.time.Duration; import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.Headers; import org.apache.kafka.common.header.internals.RecordHeader; @@ -36,6 +35,7 @@ import org.junit.Before; import org.junit.Test; +import java.time.Duration; import java.util.Properties; import static org.apache.kafka.test.StreamsTestUtils.getStreamsConfig; @@ -171,12 +171,16 @@ public void shouldThrowIllegalStateExceptionOnHeadersIfNoRecordContext() { @SuppressWarnings("unchecked") @Test public void appConfigsShouldReturnParsedValues() { - assertThat((Class) context.appConfigs().get(StreamsConfig.ROCKSDB_CONFIG_SETTER_CLASS_CONFIG), equalTo(RocksDBConfigSetter.class)); + assertThat( + context.appConfigs().get(StreamsConfig.ROCKSDB_CONFIG_SETTER_CLASS_CONFIG), + equalTo(RocksDBConfigSetter.class)); } @Test public void appConfigsShouldReturnUnrecognizedValues() { - assertThat((String) context.appConfigs().get("user.supplied.config"), equalTo("user-suppplied-value")); + assertThat( + context.appConfigs().get("user.supplied.config"), + equalTo("user-suppplied-value")); } @@ -198,9 +202,11 @@ public StateStore getStateStore(final String name) { return null; } - @SuppressWarnings("deprecation") @Override - public Cancellable schedule(final long interval, final PunctuationType type, final Punctuator callback) { + @Deprecated + public Cancellable schedule(final long interval, + final PunctuationType type, + final Punctuator callback) { return null; } @@ -217,12 +223,12 @@ public void forward(final K key, final V value) {} @Override public void forward(final K key, final V value, final To to) {} - @SuppressWarnings("deprecation") @Override + @Deprecated public void forward(final K key, final V value, final int childIndex) {} - @SuppressWarnings("deprecation") @Override + @Deprecated public void forward(final K key, final V value, final String childName) {} @Override diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ForwardingDisabledProcessorContextTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ForwardingDisabledProcessorContextTest.java index 03e79b77c89b2..c6b2cbe06fbfc 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ForwardingDisabledProcessorContextTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ForwardingDisabledProcessorContextTest.java @@ -47,11 +47,13 @@ public void shouldThrowOnForwardWithTo() { context.forward("key", "value", To.all()); } + @SuppressWarnings("deprecation") // need to test deprecated code until removed @Test(expected = StreamsException.class) public void shouldThrowOnForwardWithChildIndex() { context.forward("key", "value", 1); } + @SuppressWarnings("deprecation") // need to test deprecated code until removed @Test(expected = StreamsException.class) public void shouldThrowOnForwardWithChildName() { context.forward("key", "value", "child1"); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalProcessorContextImplTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalProcessorContextImplTest.java index deb14e9302941..4153ccacbe378 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalProcessorContextImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalProcessorContextImplTest.java @@ -106,11 +106,13 @@ public void shouldFailToForwardUsingToParameter() { globalContext.forward(null, null, To.all()); } + @SuppressWarnings("deprecation") // need to test deprecated code until removed @Test(expected = UnsupportedOperationException.class) public void shouldNotSupportForwardingViaChildIndex() { globalContext.forward(null, null, 0); } + @SuppressWarnings("deprecation") // need to test deprecated code until removed @Test(expected = UnsupportedOperationException.class) public void shouldNotSupportForwardingViaChildName() { globalContext.forward(null, null, "processorName"); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java index 6a7bd027bf8c6..1e3fad39d7cb3 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java @@ -611,7 +611,7 @@ protected static class MultiplexingProcessor extends AbstractProcessor { final boolean valueThrowsException, final Object key, final Object value) { - super("", Collections.emptyList(), null, null); + super("", Collections.emptyList(), null, null); this.keyThrowsException = keyThrowsException; this.valueThrowsException = valueThrowsException; this.key = key; diff --git a/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java b/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java index c9255ce626a19..4b9267959ac8b 100644 --- a/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java +++ b/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java @@ -159,7 +159,6 @@ public void setValueSerde(final Serde valSerde) { this.valSerde = valSerde; } - // serdes will override whatever specified in the configs @Override public Serde keySerde() { return keySerde; @@ -179,7 +178,6 @@ public File stateDir() { if (stateDir == null) { throw new UnsupportedOperationException("State directory not specified"); } - return stateDir; } @@ -195,8 +193,8 @@ public StateStore getStateStore(final String name) { return storeMap.get(name); } - @SuppressWarnings("deprecation") @Override + @Deprecated public Cancellable schedule(final long interval, final PunctuationType type, final Punctuator callback) { throw new UnsupportedOperationException("schedule() not supported."); } @@ -209,22 +207,24 @@ public Cancellable schedule(final Duration interval, } @Override - public void commit() { } + public void commit() {} - @Override @SuppressWarnings("unchecked") + @Override public void forward(final K key, final V value) { forward(key, value, To.all()); } + @SuppressWarnings("unchecked") @Override - @SuppressWarnings({"unchecked", "deprecation"}) + @Deprecated public void forward(final K key, final V value, final int childIndex) { forward(key, value, To.child(((List) currentNode().children()).get(childIndex).name())); } + @SuppressWarnings("unchecked") @Override - @SuppressWarnings({"unchecked", "deprecation"}) + @Deprecated public void forward(final K key, final V value, final String childName) { forward(key, value, To.child(childName)); } diff --git a/streams/src/test/java/org/apache/kafka/test/MockInternalProcessorContext.java b/streams/src/test/java/org/apache/kafka/test/MockInternalProcessorContext.java index 62a8491626594..5ae97c96e1875 100644 --- a/streams/src/test/java/org/apache/kafka/test/MockInternalProcessorContext.java +++ b/streams/src/test/java/org/apache/kafka/test/MockInternalProcessorContext.java @@ -63,12 +63,8 @@ public ThreadCache getCache() { } @Override - public void initialize() { - - } + public void initialize() {} @Override - public void uninitialize() { - - } + public void uninitialize() {} } \ No newline at end of file diff --git a/streams/src/test/java/org/apache/kafka/test/NoOpProcessorContext.java b/streams/src/test/java/org/apache/kafka/test/NoOpProcessorContext.java index c7c834340929e..77dd4186e9ec5 100644 --- a/streams/src/test/java/org/apache/kafka/test/NoOpProcessorContext.java +++ b/streams/src/test/java/org/apache/kafka/test/NoOpProcessorContext.java @@ -16,7 +16,6 @@ */ package org.apache.kafka.test; -import java.time.Duration; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.processor.Cancellable; @@ -29,19 +28,21 @@ import org.apache.kafka.streams.processor.internals.AbstractProcessorContext; import org.apache.kafka.streams.processor.internals.MockStreamsMetrics; +import java.time.Duration; import java.util.HashMap; import java.util.Map; import java.util.Properties; public class NoOpProcessorContext extends AbstractProcessorContext { public boolean initialized; + @SuppressWarnings("WeakerAccess") public Map forwardedValues = new HashMap<>(); public NoOpProcessorContext() { super(new TaskId(1, 1), streamsConfig(), new MockStreamsMetrics(new Metrics()), null, null); } - static StreamsConfig streamsConfig() { + private static StreamsConfig streamsConfig() { final Properties props = new Properties(); props.put(StreamsConfig.APPLICATION_ID_CONFIG, "appId"); props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "boot"); @@ -53,9 +54,11 @@ public StateStore getStateStore(final String name) { return null; } - @SuppressWarnings("deprecation") @Override - public Cancellable schedule(final long interval, final PunctuationType type, final Punctuator callback) { + @Deprecated + public Cancellable schedule(final long interval, + final PunctuationType type, + final Punctuator callback) { return null; } @@ -76,21 +79,20 @@ public void forward(final K key, final V value, final To to) { forwardedValues.put(key, value); } - @SuppressWarnings("deprecation") @Override + @Deprecated public void forward(final K key, final V value, final int childIndex) { forward(key, value); } - @SuppressWarnings("deprecation") @Override + @Deprecated public void forward(final K key, final V value, final String childName) { forward(key, value); } @Override - public void commit() { - } + public void commit() {} @Override public void initialize() { @@ -99,7 +101,5 @@ public void initialize() { @Override public void register(final StateStore store, - final StateRestoreCallback stateRestoreCallback) { - // no-op - } + final StateRestoreCallback stateRestoreCallback) {} } diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java index 7b4c58b674936..34a7ed92c688b 100644 --- a/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java +++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java @@ -275,7 +275,11 @@ public StreamsMetrics metrics() { * @param timestamp A record timestamp */ @SuppressWarnings({"WeakerAccess", "unused"}) - public void setRecordMetadata(final String topic, final int partition, final long offset, final Headers headers, final long timestamp) { + public void setRecordMetadata(final String topic, + final int partition, + final long offset, + final Headers headers, + final long timestamp) { this.topic = topic; this.partition = partition; this.offset = offset; @@ -390,7 +394,9 @@ public StateStore getStateStore(final String name) { @Override @Deprecated - public Cancellable schedule(final long intervalMs, final PunctuationType type, final Punctuator callback) { + public Cancellable schedule(final long intervalMs, + final PunctuationType type, + final Punctuator callback) { final CapturedPunctuator capturedPunctuator = new CapturedPunctuator(intervalMs, type, callback); punctuators.add(capturedPunctuator); @@ -398,7 +404,7 @@ public Cancellable schedule(final long intervalMs, final PunctuationType type, f return capturedPunctuator::cancel; } - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // removing #schedule(final long intervalMs,...) will fix this @Override public Cancellable schedule(final Duration interval, final PunctuationType type, @@ -433,8 +439,8 @@ public void forward(final K key, final V value, final To to) { ); } - @SuppressWarnings("deprecation") @Override + @Deprecated public void forward(final K key, final V value, final int childIndex) { throw new UnsupportedOperationException( "Forwarding to a child by index is deprecated. " + @@ -442,8 +448,8 @@ public void forward(final K key, final V value, final int childIndex) { ); } - @SuppressWarnings("deprecation") @Override + @Deprecated public void forward(final K key, final V value, final String childName) { throw new UnsupportedOperationException( "Forwarding to a child by name is deprecated. " + From 09f1009d246b82475bbf018d187fff5a3e035539 Mon Sep 17 00:00:00 2001 From: "Matthias J. Sax" Date: Fri, 8 Mar 2019 19:24:26 -0800 Subject: [PATCH 0002/1071] KAFKA-8065: restore original input record timestamp in forward() (#6393) Reviewers: Bill Bejeck , John Roesler , Guozhang Wang --- .../internals/ProcessorContextImpl.java | 12 ++-- .../internals/ProcessorTopologyTest.java | 55 +++++++++++++++++++ 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextImpl.java index 2afd5e9a687e0..c10ea091e447f 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextImpl.java @@ -154,12 +154,15 @@ public void forward(final K key, public void forward(final K key, final V value, final To to) { - toInternal.update(to); - if (toInternal.hasTimestamp()) { - recordContext.setTimestamp(toInternal.timestamp()); - } final ProcessorNode previousNode = currentNode(); + final long currentTimestamp = recordContext.timestamp; + try { + toInternal.update(to); + if (toInternal.hasTimestamp()) { + recordContext.setTimestamp(toInternal.timestamp()); + } + final String sendTo = toInternal.child(); if (sendTo == null) { final List> children = (List>) currentNode().children(); @@ -175,6 +178,7 @@ public void forward(final K key, forward(child, key, value); } } finally { + recordContext.timestamp = currentTimestamp; setCurrentNode(previousNode); } } diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java index 1e3fad39d7cb3..76252c1581e10 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java @@ -348,6 +348,32 @@ public void shouldConsiderModifiedTimeStamps() { assertNextOutputRecord(OUTPUT_TOPIC_1, "key3", "value3", partition, 40L); } + @Test + public void shouldConsiderModifiedTimeStampsForMultipleProcessors() { + final int partition = 10; + driver = new TopologyTestDriver(createMultiProcessorTimestampTopology(partition), props); + + driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1", 10L)); + assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1", partition, 10L); + assertNextOutputRecord(OUTPUT_TOPIC_2, "key1", "value1", partition, 20L); + assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1", partition, 15L); + assertNextOutputRecord(OUTPUT_TOPIC_2, "key1", "value1", partition, 20L); + assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1", partition, 12L); + assertNextOutputRecord(OUTPUT_TOPIC_2, "key1", "value1", partition, 22L); + assertNoOutputRecord(OUTPUT_TOPIC_1); + assertNoOutputRecord(OUTPUT_TOPIC_2); + + driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2", 20L)); + assertNextOutputRecord(OUTPUT_TOPIC_1, "key2", "value2", partition, 20L); + assertNextOutputRecord(OUTPUT_TOPIC_2, "key2", "value2", partition, 30L); + assertNextOutputRecord(OUTPUT_TOPIC_1, "key2", "value2", partition, 25L); + assertNextOutputRecord(OUTPUT_TOPIC_2, "key2", "value2", partition, 30L); + assertNextOutputRecord(OUTPUT_TOPIC_1, "key2", "value2", partition, 22L); + assertNextOutputRecord(OUTPUT_TOPIC_2, "key2", "value2", partition, 32L); + assertNoOutputRecord(OUTPUT_TOPIC_1); + assertNoOutputRecord(OUTPUT_TOPIC_2); + } + @Test public void shouldConsiderHeaders() { final int partition = 10; @@ -489,6 +515,16 @@ private Topology createTimestampTopology(final int partition) { .addSink("sink", OUTPUT_TOPIC_1, constantPartitioner(partition), "processor"); } + private Topology createMultiProcessorTimestampTopology(final int partition) { + return topology + .addSource("source", STRING_DESERIALIZER, STRING_DESERIALIZER, INPUT_TOPIC_1) + .addProcessor("processor", define(new FanOutTimestampProcessor("child1", "child2")), "source") + .addProcessor("child1", define(new ForwardingProcessor()), "processor") + .addProcessor("child2", define(new TimestampProcessor()), "processor") + .addSink("sink1", OUTPUT_TOPIC_1, constantPartitioner(partition), "child1") + .addSink("sink2", OUTPUT_TOPIC_2, constantPartitioner(partition), "child2"); + } + private Topology createMultiplexingTopology() { return topology .addSource("source", STRING_DESERIALIZER, STRING_DESERIALIZER, INPUT_TOPIC_1) @@ -582,6 +618,25 @@ public void process(final String key, final String value) { } } + protected static class FanOutTimestampProcessor extends AbstractProcessor { + private final String firstChild; + private final String secondChild; + + FanOutTimestampProcessor(final String firstChild, + final String secondChild) { + this.firstChild = firstChild; + this.secondChild = secondChild; + } + + @Override + public void process(final String key, final String value) { + context().forward(key, value); + context().forward(key, value, To.child(firstChild).withTimestamp(context().timestamp() + 5)); + context().forward(key, value, To.child(secondChild)); + context().forward(key, value, To.all().withTimestamp(context().timestamp() + 2)); + } + } + protected static class AddHeaderProcessor extends AbstractProcessor { @Override public void process(final String key, final String value) { From da5f353401baaae3da9bc23690c1c9cbb28ecd56 Mon Sep 17 00:00:00 2001 From: Rajini Sivaram Date: Sat, 9 Mar 2019 08:09:56 +0000 Subject: [PATCH 0003/1071] KAFKA-7288; Fix check in SelectorTest to wait for no buffered bytes (#6415) Reviewers: Ismael Juma --- .../test/java/org/apache/kafka/common/network/SelectorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java b/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java index 9d48ab5264967..7cb89c28de25e 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java @@ -493,7 +493,7 @@ private KafkaChannel createConnectionWithStagedReceives(int maxStagedReceives) t do { selector.poll(1000); } while (selector.completedReceives().isEmpty()); - } while (selector.numStagedReceives(channel) == 0 && !channel.hasBytesBuffered() && --retries > 0); + } while ((selector.numStagedReceives(channel) == 0 || channel.hasBytesBuffered()) && --retries > 0); assertTrue("No staged receives after 100 attempts", selector.numStagedReceives(channel) > 0); // We want to return without any bytes buffered to ensure that channel will be closed after idle time assertFalse("Channel has bytes buffered", channel.hasBytesBuffered()); From 0ef2b4cccea4196256ea28b5429ce95e2507928b Mon Sep 17 00:00:00 2001 From: "Matthias J. Sax" Date: Sat, 9 Mar 2019 05:09:07 -0800 Subject: [PATCH 0004/1071] MINOR: fix Scala compiler warning (#6417) Reviewers: Guozhang Wang , Bill Bejeck --- .../scala/org/apache/kafka/streams/scala/StreamsBuilder.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/StreamsBuilder.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/StreamsBuilder.scala index 4a1df92e2e8a8..1fcba488fc319 100644 --- a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/StreamsBuilder.scala +++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/StreamsBuilder.scala @@ -27,7 +27,6 @@ import org.apache.kafka.streams.state.StoreBuilder import org.apache.kafka.streams.{Topology, StreamsBuilder => StreamsBuilderJ} import org.apache.kafka.streams.scala.kstream._ import ImplicitConversions._ -import org.apache.kafka.streams.errors.TopologyException import scala.collection.JavaConverters._ @@ -163,7 +162,7 @@ class StreamsBuilder(inner: StreamsBuilderJ = new StreamsBuilderJ) { * * @param builder the builder used to obtain this state store `StateStore` instance * @return the underlying Java abstraction `StreamsBuilder` after adding the `StateStore` - * @throws TopologyException if state store supplier is already added + * @throws org.apache.kafka.streams.errors.TopologyException if state store supplier is already added * @see `org.apache.kafka.streams.StreamsBuilder#addStateStore` */ def addStateStore(builder: StoreBuilder[_ <: StateStore]): StreamsBuilderJ = inner.addStateStore(builder) From 65aea1f362fb6562c39df34b4d5c014f544b45ec Mon Sep 17 00:00:00 2001 From: Suman BN Date: Sun, 10 Mar 2019 12:19:43 +0530 Subject: [PATCH 0005/1071] MINOR: Print usage when parse fails during console producer *Handle OptionException while parsing options when using console producer and print usage before die.* Author: Suman BN Reviewers: Manikumar Reddy Closes #6386 from sumannewton/console-producer-parse-printusage --- .../src/main/scala/kafka/tools/ConsoleProducer.scala | 12 +++++++++++- .../scala/unit/kafka/tools/ConsoleProducerTest.scala | 9 +++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/kafka/tools/ConsoleProducer.scala b/core/src/main/scala/kafka/tools/ConsoleProducer.scala index 829e2717813bb..e6f526b3ea66c 100644 --- a/core/src/main/scala/kafka/tools/ConsoleProducer.scala +++ b/core/src/main/scala/kafka/tools/ConsoleProducer.scala @@ -21,6 +21,7 @@ import java.io._ import java.nio.charset.StandardCharsets import java.util.Properties +import joptsimple.{OptionException, OptionParser, OptionSet} import kafka.common._ import kafka.message._ import kafka.utils.Implicits._ @@ -213,7 +214,7 @@ object ConsoleProducer { .describedAs("config file") .ofType(classOf[String]) - options = parser.parse(args : _*) + options = tryParse(parser, args) CommandLineUtils.printHelpAndExitIfNeeded(this, "This tool helps to read data from standard input and publish it to Kafka.") CommandLineUtils.checkRequiredArgs(parser, options, topicOpt, brokerListOpt) @@ -232,6 +233,15 @@ object ConsoleProducer { val readerClass = options.valueOf(messageReaderOpt) val cmdLineProps = CommandLineUtils.parseKeyValueArgs(options.valuesOf(propertyOpt).asScala) val extraProducerProps = CommandLineUtils.parseKeyValueArgs(options.valuesOf(producerPropertyOpt).asScala) + + def tryParse(parser: OptionParser, args: Array[String]): OptionSet = { + try + parser.parse(args: _*) + catch { + case e: OptionException => + CommandLineUtils.printUsageAndDie(parser, e.getMessage) + } + } } class LineMessageReader extends MessageReader { diff --git a/core/src/test/scala/unit/kafka/tools/ConsoleProducerTest.scala b/core/src/test/scala/unit/kafka/tools/ConsoleProducerTest.scala index e69a5e67ae820..ed2044eeab6f3 100644 --- a/core/src/test/scala/unit/kafka/tools/ConsoleProducerTest.scala +++ b/core/src/test/scala/unit/kafka/tools/ConsoleProducerTest.scala @@ -23,6 +23,7 @@ import ConsoleProducer.LineMessageReader import org.apache.kafka.clients.producer.ProducerConfig import org.junit.{Assert, Test} import Assert.assertEquals +import kafka.utils.Exit class ConsoleProducerTest { @@ -50,13 +51,13 @@ class ConsoleProducerTest { producerConfig.getList(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG)) } - @Test + @Test(expected = classOf[IllegalArgumentException]) def testInvalidConfigs() { + Exit.setExitProcedure((_, message) => throw new IllegalArgumentException(message.orNull)) try { new ConsoleProducer.ProducerConfig(invalidArgs) - Assert.fail("Should have thrown an UnrecognizedOptionException") - } catch { - case _: joptsimple.OptionException => // expected exception + } finally { + Exit.resetExitProcedure() } } From a42f16f980cba86a8889be8b7499437ecbc2cd42 Mon Sep 17 00:00:00 2001 From: Manikumar Reddy Date: Sun, 10 Mar 2019 17:30:16 +0530 Subject: [PATCH 0006/1071] KAFKA-7922: Return authorized operations in Metadata request response (KIP-430 Part-2) - Use automatic RPC generation in Metadata Request/Response classes - https://cwiki.apache.org/confluence/display/KAFKA/KIP-430+-+Return+Authorized+Operations+in+Describe+Responses Author: Manikumar Reddy Reviewers: Rajini Sivaram Closes #6352 from omkreddy/KIP-430-METADATA --- .../admin/ConsumerGroupDescription.java | 2 +- .../clients/admin/DescribeClusterOptions.java | 10 + .../clients/admin/DescribeClusterResult.java | 14 +- .../clients/admin/DescribeTopicsOptions.java | 11 + .../kafka/clients/admin/KafkaAdminClient.java | 35 +- .../kafka/clients/admin/TopicDescription.java | 36 +- .../clients/consumer/internals/Fetcher.java | 2 +- .../apache/kafka/common/protocol/ApiKeys.java | 6 +- .../common/requests/AbstractResponse.java | 2 +- .../common/requests/MetadataRequest.java | 193 +++---- .../common/requests/MetadataResponse.java | 470 ++++++------------ .../kafka/common/requests/RequestUtils.java | 5 + .../common/message/MetadataRequest.json | 9 +- .../common/message/MetadataResponse.json | 13 +- .../apache/kafka/clients/MetadataTest.java | 2 +- .../org/apache/kafka/clients/MockClient.java | 2 +- .../clients/admin/KafkaAdminClientTest.java | 18 +- .../kafka/clients/admin/MockAdminClient.java | 10 +- .../clients/consumer/KafkaConsumerTest.java | 2 +- .../internals/ConsumerCoordinatorTest.java | 2 +- .../internals/ConsumerMetadataTest.java | 6 +- .../consumer/internals/FetcherTest.java | 4 +- .../clients/producer/KafkaProducerTest.java | 2 +- .../kafka/common/message/MessageTest.java | 2 + .../common/requests/MetadataRequestTest.java | 19 +- .../common/requests/RequestResponseTest.java | 2 +- .../java/org/apache/kafka/test/TestUtils.java | 2 +- .../main/scala/kafka/server/KafkaApis.scala | 27 +- .../api/AdminClientIntegrationTest.scala | 43 +- .../DescribeAuthorizedOperationsTest.scala | 80 ++- .../SaslSslAdminClientIntegrationTest.scala | 8 +- .../kafka/server/MetadataRequestTest.scala | 28 +- .../unit/kafka/server/RequestQuotaTest.scala | 3 +- .../internals/InternalTopicManagerTest.java | 6 +- .../kafka/trogdor/common/WorkerUtilsTest.java | 9 +- 35 files changed, 559 insertions(+), 526 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupDescription.java b/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupDescription.java index 8dd6018427fba..7320f6568150b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupDescription.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupDescription.java @@ -38,7 +38,7 @@ public class ConsumerGroupDescription { private final String partitionAssignor; private final ConsumerGroupState state; private final Node coordinator; - private Set authorizedOperations; + private final Set authorizedOperations; public ConsumerGroupDescription(String groupId, boolean isSimpleConsumerGroup, diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClusterOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClusterOptions.java index 92640fd54ea42..abde15492059a 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClusterOptions.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClusterOptions.java @@ -27,6 +27,8 @@ @InterfaceStability.Evolving public class DescribeClusterOptions extends AbstractOptions { + private boolean includeAuthorizedOperations; + /** * Set the request timeout in milliseconds for this operation or {@code null} if the default request timeout for the * AdminClient should be used. @@ -38,4 +40,12 @@ public DescribeClusterOptions timeoutMs(Integer timeoutMs) { return this; } + public DescribeClusterOptions includeAuthorizedOperations(boolean includeAuthorizedOperations) { + this.includeAuthorizedOperations = includeAuthorizedOperations; + return this; + } + + public boolean includeAuthorizedOperations() { + return includeAuthorizedOperations; + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClusterResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClusterResult.java index 7d3ffc66fb550..23f876a0519d0 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClusterResult.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClusterResult.java @@ -19,9 +19,11 @@ import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.Node; +import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.annotation.InterfaceStability; import java.util.Collection; +import java.util.Set; /** * The result of the {@link KafkaAdminClient#describeCluster()} call. @@ -33,13 +35,16 @@ public class DescribeClusterResult { private final KafkaFuture> nodes; private final KafkaFuture controller; private final KafkaFuture clusterId; + private final KafkaFuture> authorizedOperations; DescribeClusterResult(KafkaFuture> nodes, KafkaFuture controller, - KafkaFuture clusterId) { + KafkaFuture clusterId, + KafkaFuture> authorizedOperations) { this.nodes = nodes; this.controller = controller; this.clusterId = clusterId; + this.authorizedOperations = authorizedOperations; } /** @@ -64,4 +69,11 @@ public KafkaFuture controller() { public KafkaFuture clusterId() { return clusterId; } + + /** + * Returns a future which yields authorized operations. + */ + public KafkaFuture> authorizedOperations() { + return authorizedOperations; + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeTopicsOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeTopicsOptions.java index cc3d420e134cd..9e7d9da6f152b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeTopicsOptions.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeTopicsOptions.java @@ -29,6 +29,8 @@ @InterfaceStability.Evolving public class DescribeTopicsOptions extends AbstractOptions { + private boolean includeAuthorizedOperations; + /** * Set the request timeout in milliseconds for this operation or {@code null} if the default request timeout for the * AdminClient should be used. @@ -40,4 +42,13 @@ public DescribeTopicsOptions timeoutMs(Integer timeoutMs) { return this; } + public DescribeTopicsOptions includeAuthorizedOperations(boolean includeAuthorizedOperations) { + this.includeAuthorizedOperations = includeAuthorizedOperations; + return this; + } + + public boolean includeAuthorizedOperations() { + return includeAuthorizedOperations; + } + } diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java index 95337d093553f..606c8165e6cb5 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java @@ -66,6 +66,7 @@ import org.apache.kafka.common.message.DescribeGroupsRequestData; import org.apache.kafka.common.message.DescribeGroupsResponseData.DescribedGroup; import org.apache.kafka.common.message.DescribeGroupsResponseData.DescribedGroupMember; +import org.apache.kafka.common.message.MetadataRequestData; import org.apache.kafka.common.metrics.JmxReporter; import org.apache.kafka.common.metrics.MetricConfig; import org.apache.kafka.common.metrics.Metrics; @@ -157,6 +158,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; +import static org.apache.kafka.common.requests.MetadataRequest.convertToMetadataRequestTopic; import static org.apache.kafka.common.utils.Utils.closeQuietly; /** @@ -1222,7 +1224,9 @@ public AbstractRequest.Builder createRequest(int timeoutMs) { // Since this only requests node information, it's safe to pass true // for allowAutoTopicCreation (and it simplifies communication with // older brokers) - return new MetadataRequest.Builder(Collections.emptyList(), true); + return new MetadataRequest.Builder(new MetadataRequestData() + .setTopics(Collections.emptyList()) + .setAllowAutoTopicCreation(true)); } @Override @@ -1462,7 +1466,10 @@ public DescribeTopicsResult describeTopics(final Collection topicNames, @Override AbstractRequest.Builder createRequest(int timeoutMs) { if (supportsDisablingTopicCreation) - return new MetadataRequest.Builder(topicNamesList, false); + return new MetadataRequest.Builder(new MetadataRequestData() + .setTopics(convertToMetadataRequestTopic(topicNamesList)) + .setAllowAutoTopicCreation(false) + .setIncludeTopicAuthorizedOperations(options.includeAuthorizedOperations())); else return MetadataRequest.Builder.allTopics(); } @@ -1495,7 +1502,8 @@ void handleResponse(AbstractResponse abstractResponse) { partitions.add(topicPartitionInfo); } partitions.sort(Comparator.comparingInt(TopicPartitionInfo::partition)); - TopicDescription topicDescription = new TopicDescription(topicName, isInternal, partitions); + TopicDescription topicDescription = new TopicDescription(topicName, isInternal, partitions, + validAclOperations(response.data().topics().find(topicName).topicAuthorizedOperations())); future.complete(topicDescription); } } @@ -1531,6 +1539,8 @@ public DescribeClusterResult describeCluster(DescribeClusterOptions options) { final KafkaFutureImpl> describeClusterFuture = new KafkaFutureImpl<>(); final KafkaFutureImpl controllerFuture = new KafkaFutureImpl<>(); final KafkaFutureImpl clusterIdFuture = new KafkaFutureImpl<>(); + final KafkaFutureImpl> authorizedOperationsFuture = new KafkaFutureImpl<>(); + final long now = time.milliseconds(); runnable.call(new Call("listNodes", calcDeadlineMs(now, options.timeoutMs()), new LeastLoadedNodeProvider()) { @@ -1539,7 +1549,10 @@ public DescribeClusterResult describeCluster(DescribeClusterOptions options) { AbstractRequest.Builder createRequest(int timeoutMs) { // Since this only requests node information, it's safe to pass true for allowAutoTopicCreation (and it // simplifies communication with older brokers) - return new MetadataRequest.Builder(Collections.emptyList(), true); + return new MetadataRequest.Builder(new MetadataRequestData() + .setTopics(Collections.emptyList()) + .setAllowAutoTopicCreation(true) + .setIncludeClusterAuthorizedOperations(options.includeAuthorizedOperations())); } @Override @@ -1548,6 +1561,8 @@ void handleResponse(AbstractResponse abstractResponse) { describeClusterFuture.complete(response.brokers()); controllerFuture.complete(controller(response)); clusterIdFuture.complete(response.clusterId()); + authorizedOperationsFuture.complete( + validAclOperations(response.data().clusterAuthorizedOperations())); } private Node controller(MetadataResponse response) { @@ -1561,10 +1576,12 @@ void handleFailure(Throwable throwable) { describeClusterFuture.completeExceptionally(throwable); controllerFuture.completeExceptionally(throwable); clusterIdFuture.completeExceptionally(throwable); + authorizedOperationsFuture.completeExceptionally(throwable); } }, now); - return new DescribeClusterResult(describeClusterFuture, controllerFuture, clusterIdFuture); + return new DescribeClusterResult(describeClusterFuture, controllerFuture, clusterIdFuture, + authorizedOperationsFuture); } @Override @@ -2179,7 +2196,9 @@ public DeleteRecordsResult deleteRecords(final Map(topics), false); + return new MetadataRequest.Builder(new MetadataRequestData() + .setTopics(convertToMetadataRequestTopic(topics)) + .setAllowAutoTopicCreation(false)); } @Override @@ -2583,7 +2602,9 @@ public ListConsumerGroupsResult listConsumerGroups(ListConsumerGroupsOptions opt runnable.call(new Call("findAllBrokers", deadline, new LeastLoadedNodeProvider()) { @Override AbstractRequest.Builder createRequest(int timeoutMs) { - return new MetadataRequest.Builder(Collections.emptyList(), true); + return new MetadataRequest.Builder(new MetadataRequestData() + .setTopics(Collections.emptyList()) + .setAllowAutoTopicCreation(true)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/TopicDescription.java b/clients/src/main/java/org/apache/kafka/clients/admin/TopicDescription.java index 4e3e59a30fd19..daadac00940e0 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/TopicDescription.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/TopicDescription.java @@ -18,9 +18,12 @@ package org.apache.kafka.clients.admin; import org.apache.kafka.common.TopicPartitionInfo; +import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.utils.Utils; import java.util.List; +import java.util.Objects; +import java.util.Set; /** * A detailed description of a single topic in the cluster. @@ -29,25 +32,22 @@ public class TopicDescription { private final String name; private final boolean internal; private final List partitions; + private Set authorizedOperations; @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - - TopicDescription that = (TopicDescription) o; - - if (internal != that.internal) return false; - if (name != null ? !name.equals(that.name) : that.name != null) return false; - return partitions != null ? partitions.equals(that.partitions) : that.partitions == null; + final TopicDescription that = (TopicDescription) o; + return internal == that.internal && + Objects.equals(name, that.name) && + Objects.equals(partitions, that.partitions) && + Objects.equals(authorizedOperations, that.authorizedOperations); } @Override public int hashCode() { - int result = name != null ? name.hashCode() : 0; - result = 31 * result + (internal ? 1 : 0); - result = 31 * result + (partitions != null ? partitions.hashCode() : 0); - return result; + return Objects.hash(name, internal, partitions, authorizedOperations); } /** @@ -57,11 +57,14 @@ public int hashCode() { * @param internal Whether the topic is internal to Kafka * @param partitions A list of partitions where the index represents the partition id and the element contains * leadership and replica information for that partition. + * @param authorizedOperations authorized operations for this topic */ - public TopicDescription(String name, boolean internal, List partitions) { + public TopicDescription(String name, boolean internal, List partitions, + Set authorizedOperations) { this.name = name; this.internal = internal; this.partitions = partitions; + this.authorizedOperations = authorizedOperations; } /** @@ -87,9 +90,16 @@ public List partitions() { return partitions; } + /** + * authorized operations for this topic + */ + public Set authorizedOperations() { + return authorizedOperations; + } + @Override public String toString() { return "(name=" + name + ", internal=" + internal + ", partitions=" + - Utils.join(partitions, ",") + ")"; + Utils.join(partitions, ",") + ", authorizedOperations=" + authorizedOperations + ")"; } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index 9009ffe092a9f..8ac5730110c51 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -286,7 +286,7 @@ public Map> getAllTopicMetadata(Timer timer) { */ public Map> getTopicMetadata(MetadataRequest.Builder request, Timer timer) { // Save the round trip if no topics are requested. - if (!request.isAllTopics() && request.topics().isEmpty()) + if (!request.isAllTopics() && request.emptyTopicList()) return Collections.emptyMap(); do { diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java index 0a19939164703..c23aa7ef5aa4b 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java @@ -24,6 +24,8 @@ import org.apache.kafka.common.message.ElectPreferredLeadersResponseData; import org.apache.kafka.common.message.LeaveGroupRequestData; import org.apache.kafka.common.message.LeaveGroupResponseData; +import org.apache.kafka.common.message.MetadataRequestData; +import org.apache.kafka.common.message.MetadataResponseData; import org.apache.kafka.common.message.SaslAuthenticateRequestData; import org.apache.kafka.common.message.SaslAuthenticateResponseData; import org.apache.kafka.common.message.SaslHandshakeRequestData; @@ -87,8 +89,6 @@ import org.apache.kafka.common.requests.ListGroupsResponse; import org.apache.kafka.common.requests.ListOffsetRequest; import org.apache.kafka.common.requests.ListOffsetResponse; -import org.apache.kafka.common.requests.MetadataRequest; -import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.requests.OffsetCommitRequest; import org.apache.kafka.common.requests.OffsetCommitResponse; import org.apache.kafka.common.requests.OffsetFetchRequest; @@ -124,7 +124,7 @@ public enum ApiKeys { PRODUCE(0, "Produce", ProduceRequest.schemaVersions(), ProduceResponse.schemaVersions()), FETCH(1, "Fetch", FetchRequest.schemaVersions(), FetchResponse.schemaVersions()), LIST_OFFSETS(2, "ListOffsets", ListOffsetRequest.schemaVersions(), ListOffsetResponse.schemaVersions()), - METADATA(3, "Metadata", MetadataRequest.schemaVersions(), MetadataResponse.schemaVersions()), + METADATA(3, "Metadata", MetadataRequestData.SCHEMAS, MetadataResponseData.SCHEMAS), LEADER_AND_ISR(4, "LeaderAndIsr", true, LeaderAndIsrRequest.schemaVersions(), LeaderAndIsrResponse.schemaVersions()), STOP_REPLICA(5, "StopReplica", true, StopReplicaRequest.schemaVersions(), StopReplicaResponse.schemaVersions()), UPDATE_METADATA(6, "UpdateMetadata", true, UpdateMetadataRequest.schemaVersions(), diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java index 712d732de77b7..1d3fd771203ea 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java @@ -77,7 +77,7 @@ public static AbstractResponse parseResponse(ApiKeys apiKey, Struct struct, shor case LIST_OFFSETS: return new ListOffsetResponse(struct); case METADATA: - return new MetadataResponse(struct); + return new MetadataResponse(struct, version); case OFFSET_COMMIT: return new OffsetCommitResponse(struct); case OFFSET_FETCH: diff --git a/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java index 3f12f1de9a87a..7f5a5440b2c50 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java @@ -17,159 +17,116 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.errors.UnsupportedVersionException; +import org.apache.kafka.common.message.MetadataRequestData; +import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; +import org.apache.kafka.common.message.MetadataResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.ArrayOf; -import org.apache.kafka.common.protocol.types.Field; -import org.apache.kafka.common.protocol.types.Schema; import org.apache.kafka.common.protocol.types.Struct; -import org.apache.kafka.common.utils.Utils; import java.nio.ByteBuffer; -import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; - -import static org.apache.kafka.common.protocol.types.Type.STRING; +import java.util.stream.Collectors; public class MetadataRequest extends AbstractRequest { - private static final String TOPICS_KEY_NAME = "topics"; - - private static final Schema METADATA_REQUEST_V0 = new Schema( - new Field(TOPICS_KEY_NAME, new ArrayOf(STRING), "An array of topics to fetch metadata for. If no topics are specified fetch metadata for all topics.")); - - private static final Schema METADATA_REQUEST_V1 = new Schema( - new Field(TOPICS_KEY_NAME, ArrayOf.nullable(STRING), "An array of topics to fetch metadata for. If the topics array is null fetch metadata for all topics.")); - - /* The v2 metadata request is the same as v1. An additional field for cluster id has been added to the v2 metadata response */ - private static final Schema METADATA_REQUEST_V2 = METADATA_REQUEST_V1; - - /* The v3 metadata request is the same as v1 and v2. An additional field for throttle time has been added to the v3 metadata response */ - private static final Schema METADATA_REQUEST_V3 = METADATA_REQUEST_V2; - - /* The v4 metadata request has an additional field for allowing auto topic creation. The response is the same as v3. */ - private static final Field.Bool ALLOW_AUTO_TOPIC_CREATION = new Field.Bool("allow_auto_topic_creation", - "If this and the broker config auto.create.topics.enable are true, topics that " + - "don't exist will be created by the broker. Otherwise, no topics will be created by the broker."); - - private static final Schema METADATA_REQUEST_V4 = new Schema( - new Field(TOPICS_KEY_NAME, ArrayOf.nullable(STRING), "An array of topics to fetch metadata for. " + - "If the topics array is null fetch metadata for all topics."), - ALLOW_AUTO_TOPIC_CREATION); - - /* The v5 metadata request is the same as v4. An additional field for offline_replicas has been added to the v5 metadata response */ - private static final Schema METADATA_REQUEST_V5 = METADATA_REQUEST_V4; + public static class Builder extends AbstractRequest.Builder { + private static final MetadataRequestData ALL_TOPICS_REQUEST_DATA = new MetadataRequestData(). + setTopics(null).setAllowAutoTopicCreation(true); - /** - * The version number is bumped to indicate that on quota violation brokers send out responses before throttling. - */ - private static final Schema METADATA_REQUEST_V6 = METADATA_REQUEST_V5; + private final MetadataRequestData data; - /** - * Bumped for the addition of the current leader epoch in the metadata response. - */ - private static final Schema METADATA_REQUEST_V7 = METADATA_REQUEST_V6; + public Builder(MetadataRequestData data) { + super(ApiKeys.METADATA); + this.data = data; + } - public static Schema[] schemaVersions() { - return new Schema[] {METADATA_REQUEST_V0, METADATA_REQUEST_V1, METADATA_REQUEST_V2, METADATA_REQUEST_V3, - METADATA_REQUEST_V4, METADATA_REQUEST_V5, METADATA_REQUEST_V6, METADATA_REQUEST_V7}; - } + public Builder(List topics, boolean allowAutoTopicCreation, short version) { + super(ApiKeys.METADATA, version); + MetadataRequestData data = new MetadataRequestData(); + if (topics == null) + data.setTopics(null); + else { + topics.forEach(topic -> data.topics().add(new MetadataRequestTopic().setName(topic))); + } - public static class Builder extends AbstractRequest.Builder { - private static final List ALL_TOPICS = null; + data.setAllowAutoTopicCreation(allowAutoTopicCreation); + this.data = data; + } - // The list of topics, or null if we want to request metadata about all topics. - private final List topics; - private final boolean allowAutoTopicCreation; + public Builder(List topics, boolean allowAutoTopicCreation) { + this(topics, allowAutoTopicCreation, ApiKeys.METADATA.latestVersion()); + } public static Builder allTopics() { // This never causes auto-creation, but we set the boolean to true because that is the default value when // deserializing V2 and older. This way, the value is consistent after serialization and deserialization. - return new Builder(ALL_TOPICS, true); + return new Builder(ALL_TOPICS_REQUEST_DATA); } - public Builder(List topics, boolean allowAutoTopicCreation) { - super(ApiKeys.METADATA); - this.topics = topics; - this.allowAutoTopicCreation = allowAutoTopicCreation; + public boolean emptyTopicList() { + return data.topics().isEmpty(); } - public List topics() { - return this.topics; + public boolean isAllTopics() { + return data.topics() == null; } - public boolean isAllTopics() { - return this.topics == ALL_TOPICS; + public List topics() { + return data.topics() + .stream() + .map(MetadataRequestTopic::name) + .collect(Collectors.toList()); } @Override public MetadataRequest build(short version) { if (version < 1) throw new UnsupportedVersionException("MetadataRequest versions older than 1 are not supported."); - if (!allowAutoTopicCreation && version < 4) + if (!data.allowAutoTopicCreation() && version < 4) throw new UnsupportedVersionException("MetadataRequest versions older than 4 don't support the " + "allowAutoTopicCreation field"); - return new MetadataRequest(this.topics, allowAutoTopicCreation, version); + return new MetadataRequest(data, version); } @Override public String toString() { - StringBuilder bld = new StringBuilder(); - bld.append("(type=MetadataRequest"). - append(", topics="); - if (topics == null) { - bld.append(""); - } else { - bld.append(Utils.join(topics, ",")); - } - bld.append(")"); - return bld.toString(); + return data.toString(); } } - private final List topics; - private final boolean allowAutoTopicCreation; + private final MetadataRequestData data; + private final short version; - /** - * In v0 null is not allowed and an empty list indicates requesting all topics. - * Note: modern clients do not support sending v0 requests. - * In v1 null indicates requesting all topics, and an empty list indicates requesting no topics. - */ - public MetadataRequest(List topics, boolean allowAutoTopicCreation, short version) { + public MetadataRequest(MetadataRequestData data, short version) { super(ApiKeys.METADATA, version); - this.topics = topics; - this.allowAutoTopicCreation = allowAutoTopicCreation; + this.data = data; + this.version = version; } public MetadataRequest(Struct struct, short version) { super(ApiKeys.METADATA, version); - Object[] topicArray = struct.getArray(TOPICS_KEY_NAME); - if (topicArray != null) { - if (topicArray.length == 0 && version == 0) { - topics = null; - } else { - topics = new ArrayList<>(); - for (Object topicObj: topicArray) { - topics.add((String) topicObj); - } - } - } else { - topics = null; - } + this.data = new MetadataRequestData(struct, version); + this.version = version; + } - allowAutoTopicCreation = struct.getOrElse(ALLOW_AUTO_TOPIC_CREATION, true); + public MetadataRequestData data() { + return data; } @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { - List topicMetadatas = new ArrayList<>(); Errors error = Errors.forException(e); - List partitions = Collections.emptyList(); - - if (topics != null) { - for (String topic : topics) - topicMetadatas.add(new MetadataResponse.TopicMetadata(error, topic, false, partitions)); + MetadataResponseData responseData = new MetadataResponseData(); + if (topics() != null) { + for (String topic :topics()) + responseData.topics().add(new MetadataResponseData.MetadataResponseTopic() + .setName(topic) + .setErrorCode(error.code()) + .setIsInternal(false) + .setPartitions(Collections.emptyList())); } short versionId = version(); @@ -177,13 +134,15 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { case 0: case 1: case 2: - return new MetadataResponse(Collections.emptyList(), null, MetadataResponse.NO_CONTROLLER_ID, topicMetadatas); + return new MetadataResponse(responseData); case 3: case 4: case 5: case 6: case 7: - return new MetadataResponse(throttleTimeMs, Collections.emptyList(), null, MetadataResponse.NO_CONTROLLER_ID, topicMetadatas); + case 8: + responseData.setThrottleTimeMs(throttleTimeMs); + return new MetadataResponse(responseData); default: throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d", versionId, this.getClass().getSimpleName(), ApiKeys.METADATA.latestVersion())); @@ -191,29 +150,37 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public boolean isAllTopics() { - return topics == null; + return (data.topics() == null) || + (data.topics().isEmpty() && version == 0); //In version 0, an empty topic list indicates + // "request metadata for all topics." } public List topics() { - return topics; + if (isAllTopics()) //In version 0, we return null for empty topic list + return null; + else + return data.topics() + .stream() + .map(MetadataRequestTopic::name) + .collect(Collectors.toList()); } public boolean allowAutoTopicCreation() { - return allowAutoTopicCreation; + return data.allowAutoTopicCreation(); } public static MetadataRequest parse(ByteBuffer buffer, short version) { return new MetadataRequest(ApiKeys.METADATA.parseRequest(version, buffer), version); } + public static List convertToMetadataRequestTopic(final Collection topics) { + return topics.stream().map(topic -> new MetadataRequestTopic() + .setName(topic)) + .collect(Collectors.toList()); + } + @Override protected Struct toStruct() { - Struct struct = new Struct(ApiKeys.METADATA.requestSchema(version())); - if (topics == null) - struct.set(TOPICS_KEY_NAME, null); - else - struct.set(TOPICS_KEY_NAME, topics.toArray()); - struct.setIfExists(ALLOW_AUTO_TOPIC_CREATION, allowAutoTopicCreation); - return struct; + return data.toStruct(version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java index f90876fd261bb..3455d5b69c5cd 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java @@ -19,11 +19,14 @@ import org.apache.kafka.common.Cluster; import org.apache.kafka.common.Node; import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.message.MetadataResponseData; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseBroker; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Field; -import org.apache.kafka.common.protocol.types.Schema; import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.utils.Utils; import java.nio.ByteBuffer; @@ -33,15 +36,10 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; - -import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE; -import static org.apache.kafka.common.protocol.CommonFields.LEADER_EPOCH; -import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID; -import static org.apache.kafka.common.protocol.CommonFields.THROTTLE_TIME_MS; -import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME; -import static org.apache.kafka.common.protocol.types.Type.INT32; +import java.util.stream.Collectors; /** * Possible topic-level error codes: @@ -57,239 +55,37 @@ public class MetadataResponse extends AbstractResponse { public static final int NO_CONTROLLER_ID = -1; - private static final Field.ComplexArray BROKERS = new Field.ComplexArray("brokers", - "Host and port information for all brokers."); - private static final Field.ComplexArray TOPIC_METADATA = new Field.ComplexArray("topic_metadata", - "Metadata for requested topics"); - - // cluster level fields - private static final Field.NullableStr CLUSTER_ID = new Field.NullableStr("cluster_id", - "The cluster id that this broker belongs to."); - private static final Field.Int32 CONTROLLER_ID = new Field.Int32("controller_id", - "The broker id of the controller broker."); - - // broker level fields - private static final Field.Int32 NODE_ID = new Field.Int32("node_id", "The broker id."); - private static final Field.Str HOST = new Field.Str("host", "The hostname of the broker."); - private static final Field.Int32 PORT = new Field.Int32("port", "The port on which the broker accepts requests."); - private static final Field.NullableStr RACK = new Field.NullableStr("rack", "The rack of the broker."); - - // topic level fields - private static final Field.ComplexArray PARTITION_METADATA = new Field.ComplexArray("partition_metadata", - "Metadata for each partition of the topic."); - private static final Field.Bool IS_INTERNAL = new Field.Bool("is_internal", - "Indicates if the topic is considered a Kafka internal topic"); - - // partition level fields - private static final Field.Int32 LEADER = new Field.Int32("leader", - "The id of the broker acting as leader for this partition."); - private static final Field.Array REPLICAS = new Field.Array("replicas", INT32, - "The set of all nodes that host this partition."); - private static final Field.Array ISR = new Field.Array("isr", INT32, - "The set of nodes that are in sync with the leader for this partition."); - private static final Field.Array OFFLINE_REPLICAS = new Field.Array("offline_replicas", INT32, - "The set of offline replicas of this partition."); - - private static final Field METADATA_BROKER_V0 = BROKERS.withFields( - NODE_ID, - HOST, - PORT); - - private static final Field PARTITION_METADATA_V0 = PARTITION_METADATA.withFields( - ERROR_CODE, - PARTITION_ID, - LEADER, - REPLICAS, - ISR); - - private static final Field TOPIC_METADATA_V0 = TOPIC_METADATA.withFields( - ERROR_CODE, - TOPIC_NAME, - PARTITION_METADATA_V0); - - private static final Schema METADATA_RESPONSE_V0 = new Schema( - METADATA_BROKER_V0, - TOPIC_METADATA_V0); - - // V1 adds fields for the rack of each broker, the controller id, and whether or not the topic is internal - private static final Field METADATA_BROKER_V1 = BROKERS.withFields( - NODE_ID, - HOST, - PORT, - RACK); - - private static final Field TOPIC_METADATA_V1 = TOPIC_METADATA.withFields( - ERROR_CODE, - TOPIC_NAME, - IS_INTERNAL, - PARTITION_METADATA_V0); - - private static final Schema METADATA_RESPONSE_V1 = new Schema( - METADATA_BROKER_V1, - CONTROLLER_ID, - TOPIC_METADATA_V1); - - // V2 added a field for the cluster id - private static final Schema METADATA_RESPONSE_V2 = new Schema( - METADATA_BROKER_V1, - CLUSTER_ID, - CONTROLLER_ID, - TOPIC_METADATA_V1); - - // V3 adds the throttle time to the response - private static final Schema METADATA_RESPONSE_V3 = new Schema( - THROTTLE_TIME_MS, - METADATA_BROKER_V1, - CLUSTER_ID, - CONTROLLER_ID, - TOPIC_METADATA_V1); - - private static final Schema METADATA_RESPONSE_V4 = METADATA_RESPONSE_V3; - - // V5 added a per-partition offline_replicas field. This field specifies the list of replicas that are offline. - private static final Field PARTITION_METADATA_V5 = PARTITION_METADATA.withFields( - ERROR_CODE, - PARTITION_ID, - LEADER, - REPLICAS, - ISR, - OFFLINE_REPLICAS); - - private static final Field TOPIC_METADATA_V5 = TOPIC_METADATA.withFields( - ERROR_CODE, - TOPIC_NAME, - IS_INTERNAL, - PARTITION_METADATA_V5); - - private static final Schema METADATA_RESPONSE_V5 = new Schema( - THROTTLE_TIME_MS, - METADATA_BROKER_V1, - CLUSTER_ID, - CONTROLLER_ID, - TOPIC_METADATA_V5); - - // V6 bump used to indicate that on quota violation brokers send out responses before throttling. - private static final Schema METADATA_RESPONSE_V6 = METADATA_RESPONSE_V5; - - // V7 adds the leader epoch to the partition metadata - private static final Field PARTITION_METADATA_V7 = PARTITION_METADATA.withFields( - ERROR_CODE, - PARTITION_ID, - LEADER, - LEADER_EPOCH, - REPLICAS, - ISR, - OFFLINE_REPLICAS); - - private static final Field TOPIC_METADATA_V7 = TOPIC_METADATA.withFields( - ERROR_CODE, - TOPIC_NAME, - IS_INTERNAL, - PARTITION_METADATA_V7); - - private static final Schema METADATA_RESPONSE_V7 = new Schema( - THROTTLE_TIME_MS, - METADATA_BROKER_V1, - CLUSTER_ID, - CONTROLLER_ID, - TOPIC_METADATA_V7); - - public static Schema[] schemaVersions() { - return new Schema[] {METADATA_RESPONSE_V0, METADATA_RESPONSE_V1, METADATA_RESPONSE_V2, METADATA_RESPONSE_V3, - METADATA_RESPONSE_V4, METADATA_RESPONSE_V5, METADATA_RESPONSE_V6, METADATA_RESPONSE_V7}; - } - - private final int throttleTimeMs; - private final Collection brokers; - private final Node controller; - private final List topicMetadata; - private final String clusterId; + private MetadataResponseData data; - /** - * Constructor for all versions. - */ - public MetadataResponse(List brokers, String clusterId, int controllerId, List topicMetadata) { - this(DEFAULT_THROTTLE_TIME, brokers, clusterId, controllerId, topicMetadata); + public MetadataResponse(MetadataResponseData data) { + this.data = data; } - public MetadataResponse(int throttleTimeMs, List brokers, String clusterId, int controllerId, List topicMetadata) { - this.throttleTimeMs = throttleTimeMs; - this.brokers = brokers; - this.controller = getControllerNode(controllerId, brokers); - this.topicMetadata = topicMetadata; - this.clusterId = clusterId; + private Map brokersMap() { + return data.brokers().stream().collect( + Collectors.toMap(MetadataResponseBroker::nodeId, b -> new Node(b.nodeId(), b.host(), b.port(), b.rack()))); } - public MetadataResponse(Struct struct) { - this.throttleTimeMs = struct.getOrElse(THROTTLE_TIME_MS, DEFAULT_THROTTLE_TIME); - Map brokers = new HashMap<>(); - Object[] brokerStructs = struct.get(BROKERS); - for (Object brokerStruct : brokerStructs) { - Struct broker = (Struct) brokerStruct; - int nodeId = broker.get(NODE_ID); - String host = broker.get(HOST); - int port = broker.get(PORT); - // This field only exists in v1+ - // When we can't know if a rack exists in a v0 response we default to null - String rack = broker.getOrElse(RACK, null); - brokers.put(nodeId, new Node(nodeId, host, port, rack)); - } - - // This field only exists in v1+ - // When we can't know the controller id in a v0 response we default to NO_CONTROLLER_ID - int controllerId = struct.getOrElse(CONTROLLER_ID, NO_CONTROLLER_ID); - - // This field only exists in v2+ - this.clusterId = struct.getOrElse(CLUSTER_ID, null); - - List topicMetadata = new ArrayList<>(); - Object[] topicInfos = struct.get(TOPIC_METADATA); - for (Object topicInfoObj : topicInfos) { - Struct topicInfo = (Struct) topicInfoObj; - Errors topicError = Errors.forCode(topicInfo.get(ERROR_CODE)); - String topic = topicInfo.get(TOPIC_NAME); - // This field only exists in v1+ - // When we can't know if a topic is internal or not in a v0 response we default to false - boolean isInternal = topicInfo.getOrElse(IS_INTERNAL, false); - List partitionMetadata = new ArrayList<>(); - - Object[] partitionInfos = topicInfo.get(PARTITION_METADATA); - for (Object partitionInfoObj : partitionInfos) { - Struct partitionInfo = (Struct) partitionInfoObj; - Errors partitionError = Errors.forCode(partitionInfo.get(ERROR_CODE)); - int partition = partitionInfo.get(PARTITION_ID); - int leader = partitionInfo.get(LEADER); - Optional leaderEpoch = RequestUtils.getLeaderEpoch(partitionInfo, LEADER_EPOCH); - Node leaderNode = leader == -1 ? null : brokers.get(leader); - - Object[] replicas = partitionInfo.get(REPLICAS); - List replicaNodes = convertToNodes(brokers, replicas); - - Object[] isr = partitionInfo.get(ISR); - List isrNodes = convertToNodes(brokers, isr); - - Object[] offlineReplicas = partitionInfo.getOrEmpty(OFFLINE_REPLICAS); - List offlineNodes = convertToNodes(brokers, offlineReplicas); - - partitionMetadata.add(new PartitionMetadata(partitionError, partition, leaderNode, leaderEpoch, - replicaNodes, isrNodes, offlineNodes)); - } + public MetadataResponse(Struct struct, short version) { + this(new MetadataResponseData(struct, version)); + } - topicMetadata.add(new TopicMetadata(topicError, topic, isInternal, partitionMetadata)); - } + @Override + protected Struct toStruct(short version) { + return data.toStruct(version); + } - this.brokers = brokers.values(); - this.controller = getControllerNode(controllerId, brokers.values()); - this.topicMetadata = topicMetadata; + public MetadataResponseData data() { + return data; } - private List convertToNodes(Map brokers, Object[] brokerIds) { - List nodes = new ArrayList<>(brokerIds.length); - for (Object brokerId : brokerIds) + private List convertToNodes(Map brokers, List brokerIds) { + List nodes = new ArrayList<>(brokerIds.size()); + for (Integer brokerId : brokerIds) if (brokers.containsKey(brokerId)) nodes.add(brokers.get(brokerId)); else - nodes.add(new Node((int) brokerId, "", -1)); + nodes.add(new Node(brokerId, "", -1)); return nodes; } @@ -303,7 +99,7 @@ private Node getControllerNode(int controllerId, Collection brokers) { @Override public int throttleTimeMs() { - return throttleTimeMs; + return data.throttleTimeMs(); } /** @@ -312,9 +108,9 @@ public int throttleTimeMs() { */ public Map errors() { Map errors = new HashMap<>(); - for (TopicMetadata metadata : topicMetadata) { - if (metadata.error != Errors.NONE) - errors.put(metadata.topic(), metadata.error); + for (MetadataResponseTopic metadata : data.topics()) { + if (metadata.errorCode() != Errors.NONE.code()) + errors.put(metadata.name(), Errors.forCode(metadata.errorCode())); } return errors; } @@ -322,8 +118,8 @@ public Map errors() { @Override public Map errorCounts() { Map errorCounts = new HashMap<>(); - for (TopicMetadata metadata : topicMetadata) - updateErrorCounts(errorCounts, metadata.error); + for (MetadataResponseTopic metadata : data.topics()) + updateErrorCounts(errorCounts, Errors.forCode(metadata.errorCode())); return errorCounts; } @@ -332,9 +128,9 @@ public Map errorCounts() { */ public Set topicsByError(Errors error) { Set errorTopics = new HashSet<>(); - for (TopicMetadata metadata : topicMetadata) { - if (metadata.error == error) - errorTopics.add(metadata.topic()); + for (MetadataResponseTopic metadata : data.topics()) { + if (metadata.errorCode() == error.code()) + errorTopics.add(metadata.name()); } return errorTopics; } @@ -346,7 +142,7 @@ public Set topicsByError(Errors error) { public Cluster cluster() { Set internalTopics = new HashSet<>(); List partitions = new ArrayList<>(); - for (TopicMetadata metadata : topicMetadata) { + for (TopicMetadata metadata : topicMetadata()) { if (metadata.error == Errors.NONE) { if (metadata.isInternal) @@ -356,8 +152,8 @@ public Cluster cluster() { } } } - return new Cluster(this.clusterId, this.brokers, partitions, topicsByError(Errors.TOPIC_AUTHORIZATION_FAILED), - topicsByError(Errors.INVALID_TOPIC_EXCEPTION), internalTopics, this.controller); + return new Cluster(data.clusterId(), brokersMap().values(), partitions, topicsByError(Errors.TOPIC_AUTHORIZATION_FAILED), + topicsByError(Errors.INVALID_TOPIC_EXCEPTION), internalTopics, controller()); } /** @@ -379,7 +175,7 @@ public static PartitionInfo partitionMetaToInfo(String topic, PartitionMetadata * @return the brokers */ public Collection brokers() { - return brokers; + return new ArrayList<>(brokersMap().values()); } /** @@ -387,7 +183,30 @@ public Collection brokers() { * @return the topicMetadata */ public Collection topicMetadata() { - return topicMetadata; + List topicMetadataList = new ArrayList<>(); + for (MetadataResponseTopic topicMetadata : data.topics()) { + Errors topicError = Errors.forCode(topicMetadata.errorCode()); + String topic = topicMetadata.name(); + boolean isInternal = topicMetadata.isInternal(); + List partitionMetadataList = new ArrayList<>(); + + for (MetadataResponsePartition partitionMetadata : topicMetadata.partitions()) { + Errors partitionError = Errors.forCode(partitionMetadata.errorCode()); + int partitionIndex = partitionMetadata.partitionIndex(); + int leader = partitionMetadata.leaderId(); + Optional leaderEpoch = RequestUtils.getLeaderEpoch(partitionMetadata.leaderEpoch()); + Node leaderNode = leader == -1 ? null : brokersMap().get(leader); + List replicaNodes = convertToNodes(brokersMap(), partitionMetadata.replicaNodes()); + List isrNodes = convertToNodes(brokersMap(), partitionMetadata.isrNodes()); + List offlineNodes = convertToNodes(brokersMap(), partitionMetadata.offlineReplicas()); + partitionMetadataList.add(new PartitionMetadata(partitionError, partitionIndex, leaderNode, leaderEpoch, + replicaNodes, isrNodes, offlineNodes)); + } + + topicMetadataList.add(new TopicMetadata(topicError, topic, isInternal, partitionMetadataList, + topicMetadata.topicAuthorizedOperations())); + } + return topicMetadataList; } /** @@ -395,7 +214,7 @@ public Collection topicMetadata() { * @return the controller node or null if it doesn't exist */ public Node controller() { - return controller; + return getControllerNode(data.controllerId(), brokers()); } /** @@ -403,11 +222,11 @@ public Node controller() { * @return cluster identifier if it is present in the response, null otherwise. */ public String clusterId() { - return this.clusterId; + return this.data.clusterId(); } public static MetadataResponse parse(ByteBuffer buffer, short version) { - return new MetadataResponse(ApiKeys.METADATA.parseResponse(version, buffer)); + return new MetadataResponse(ApiKeys.METADATA.responseSchema(version).read(buffer), version); } public static class TopicMetadata { @@ -415,15 +234,25 @@ public static class TopicMetadata { private final String topic; private final boolean isInternal; private final List partitionMetadata; + private int authorizedOperations; public TopicMetadata(Errors error, String topic, boolean isInternal, - List partitionMetadata) { + List partitionMetadata, + int authorizedOperations) { this.error = error; this.topic = topic; this.isInternal = isInternal; this.partitionMetadata = partitionMetadata; + this.authorizedOperations = authorizedOperations; + } + + public TopicMetadata(Errors error, + String topic, + boolean isInternal, + List partitionMetadata) { + this(error, topic, isInternal, partitionMetadata, 0); } public Errors error() { @@ -442,13 +271,40 @@ public List partitionMetadata() { return partitionMetadata; } + public void authorizedOperations(int authorizedOperations) { + this.authorizedOperations = authorizedOperations; + } + + public int authorizedOperations() { + return authorizedOperations; + } + + @Override + public boolean equals(final Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final TopicMetadata that = (TopicMetadata) o; + return isInternal == that.isInternal && + error == that.error && + Objects.equals(topic, that.topic) && + Objects.equals(partitionMetadata, that.partitionMetadata) && + Objects.equals(authorizedOperations, that.authorizedOperations); + } + + @Override + public int hashCode() { + return Objects.hash(error, topic, isInternal, partitionMetadata, authorizedOperations); + } + @Override public String toString() { - return "(type=TopicMetadata" + - ", error=" + error + - ", topic=" + topic + - ", isInternal=" + isInternal + - ", partitionMetadata=" + partitionMetadata + ')'; + return "TopicMetadata{" + + "error=" + error + + ", topic='" + topic + '\'' + + ", isInternal=" + isInternal + + ", partitionMetadata=" + partitionMetadata + + ", authorizedOperations=" + authorizedOperations + + '}'; } } @@ -523,68 +379,54 @@ public String toString() { } } - @Override - protected Struct toStruct(short version) { - Struct struct = new Struct(ApiKeys.METADATA.responseSchema(version)); - struct.setIfExists(THROTTLE_TIME_MS, throttleTimeMs); - List brokerArray = new ArrayList<>(); - for (Node node : brokers) { - Struct broker = struct.instance(BROKERS); - broker.set(NODE_ID, node.id()); - broker.set(HOST, node.host()); - broker.set(PORT, node.port()); - // This field only exists in v1+ - broker.setIfExists(RACK, node.rack()); - brokerArray.add(broker); - } - struct.set(BROKERS, brokerArray.toArray()); - - // This field only exists in v1+ - struct.setIfExists(CONTROLLER_ID, controller == null ? NO_CONTROLLER_ID : controller.id()); - - // This field only exists in v2+ - struct.setIfExists(CLUSTER_ID, clusterId); - - List topicMetadataArray = new ArrayList<>(topicMetadata.size()); - for (TopicMetadata metadata : topicMetadata) { - Struct topicData = struct.instance(TOPIC_METADATA); - topicData.set(TOPIC_NAME, metadata.topic); - topicData.set(ERROR_CODE, metadata.error.code()); - // This field only exists in v1+ - topicData.setIfExists(IS_INTERNAL, metadata.isInternal()); - - List partitionMetadataArray = new ArrayList<>(metadata.partitionMetadata.size()); - for (PartitionMetadata partitionMetadata : metadata.partitionMetadata()) { - Struct partitionData = topicData.instance(PARTITION_METADATA); - partitionData.set(ERROR_CODE, partitionMetadata.error.code()); - partitionData.set(PARTITION_ID, partitionMetadata.partition); - partitionData.set(LEADER, partitionMetadata.leaderId()); - - // Leader epoch exists in v7 forward - RequestUtils.setLeaderEpochIfExists(partitionData, LEADER_EPOCH, partitionMetadata.leaderEpoch); - - ArrayList replicas = new ArrayList<>(partitionMetadata.replicas.size()); - for (Node node : partitionMetadata.replicas) - replicas.add(node.id()); - partitionData.set(REPLICAS, replicas.toArray()); - ArrayList isr = new ArrayList<>(partitionMetadata.isr.size()); - for (Node node : partitionMetadata.isr) - isr.add(node.id()); - partitionData.set(ISR, isr.toArray()); - if (partitionData.hasField(OFFLINE_REPLICAS)) { - ArrayList offlineReplicas = new ArrayList<>(partitionMetadata.offlineReplicas.size()); - for (Node node : partitionMetadata.offlineReplicas) - offlineReplicas.add(node.id()); - partitionData.set(OFFLINE_REPLICAS, offlineReplicas.toArray()); - } - partitionMetadataArray.add(partitionData); - + public static MetadataResponse prepareResponse(int throttleTimeMs, List brokers, String clusterId, + int controllerId, List topicMetadataList, + int clusterAuthorizedOperations) { + MetadataResponseData responseData = new MetadataResponseData(); + responseData.setThrottleTimeMs(throttleTimeMs); + brokers.forEach(broker -> { + responseData.brokers().add(new MetadataResponseBroker() + .setNodeId(broker.id()) + .setHost(broker.host()) + .setPort(broker.port()) + .setRack(broker.rack())); + }); + + responseData.setClusterId(clusterId); + responseData.setControllerId(controllerId); + responseData.setClusterAuthorizedOperations(clusterAuthorizedOperations); + + topicMetadataList.forEach(topicMetadata -> { + MetadataResponseTopic metadataResponseTopic = new MetadataResponseTopic(); + metadataResponseTopic + .setErrorCode(topicMetadata.error.code()) + .setName(topicMetadata.topic) + .setIsInternal(topicMetadata.isInternal) + .setTopicAuthorizedOperations(topicMetadata.authorizedOperations); + + for (PartitionMetadata partitionMetadata : topicMetadata.partitionMetadata) { + metadataResponseTopic.partitions().add(new MetadataResponsePartition() + .setErrorCode(partitionMetadata.error.code()) + .setPartitionIndex(partitionMetadata.partition) + .setLeaderId(partitionMetadata.leader == null ? -1 : partitionMetadata.leader.id()) + .setLeaderEpoch(partitionMetadata.leaderEpoch().orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)) + .setReplicaNodes(partitionMetadata.replicas.stream().map(Node::id).collect(Collectors.toList())) + .setIsrNodes(partitionMetadata.isr.stream().map(Node::id).collect(Collectors.toList())) + .setOfflineReplicas(partitionMetadata.offlineReplicas.stream().map(Node::id).collect(Collectors.toList()))); } - topicData.set(PARTITION_METADATA, partitionMetadataArray.toArray()); - topicMetadataArray.add(topicData); - } - struct.set(TOPIC_METADATA, topicMetadataArray.toArray()); - return struct; + responseData.topics().add(metadataResponseTopic); + }); + return new MetadataResponse(responseData); + } + + public static MetadataResponse prepareResponse(int throttleTimeMs, List brokers, String clusterId, + int controllerId, List topicMetadataList) { + return prepareResponse(throttleTimeMs, brokers, clusterId, controllerId, topicMetadataList, 0); + } + + public static MetadataResponse prepareResponse(List brokers, String clusterId, int controllerId, + List topicMetadata) { + return prepareResponse(AbstractResponse.DEFAULT_THROTTLE_TIME, brokers, clusterId, controllerId, topicMetadata); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java index 24c2fbe441661..b4a2420c46a8a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java @@ -117,4 +117,9 @@ static Optional getLeaderEpoch(Struct struct, Field.Int32 leaderEpochFi return leaderEpochOpt; } + static Optional getLeaderEpoch(int leaderEpoch) { + Optional leaderEpochOpt = leaderEpoch == RecordBatch.NO_PARTITION_LEADER_EPOCH ? + Optional.empty() : Optional.of(leaderEpoch); + return leaderEpochOpt; + } } diff --git a/clients/src/main/resources/common/message/MetadataRequest.json b/clients/src/main/resources/common/message/MetadataRequest.json index 74f3fab5cd585..8848ac1bc251f 100644 --- a/clients/src/main/resources/common/message/MetadataRequest.json +++ b/clients/src/main/resources/common/message/MetadataRequest.json @@ -17,7 +17,7 @@ "apiKey": 3, "type": "request", "name": "MetadataRequest", - "validVersions": "0-7", + "validVersions": "0-8", "fields": [ // In version 0, an empty array indicates "request metadata for all topics." In version 1 and // higher, an empty array indicates "request metadata for no topics," and a null array is used to @@ -26,12 +26,17 @@ // Version 2 and 3 are the same as version 1. // // Version 4 adds AllowAutoTopicCreation. + // Starting in version 8, authorized operations can be requested for cluster and topic resource. { "name": "Topics", "type": "[]MetadataRequestTopic", "versions": "0+", "nullableVersions": "1+", "about": "The topics to fetch metadata for.", "fields": [ { "name": "Name", "type": "string", "versions": "0+", "about": "The topic name." } ]}, { "name": "AllowAutoTopicCreation", "type": "bool", "versions": "4+", "default": "true", "ignorable": false, - "about": "If this is true, the broker may auto-create topics that we requested which do not already exist, if it is configured to do so." } + "about": "If this is true, the broker may auto-create topics that we requested which do not already exist, if it is configured to do so." }, + { "name": "IncludeClusterAuthorizedOperations", "type": "bool", "versions": "8+", + "about": "Whether to include cluster authorized operations." }, + { "name": "IncludeTopicAuthorizedOperations", "type": "bool", "versions": "8+", + "about": "Whether to include topic authorized operations." } ] } diff --git a/clients/src/main/resources/common/message/MetadataResponse.json b/clients/src/main/resources/common/message/MetadataResponse.json index e58a720c23e42..2d248ab5daf4e 100644 --- a/clients/src/main/resources/common/message/MetadataResponse.json +++ b/clients/src/main/resources/common/message/MetadataResponse.json @@ -32,7 +32,8 @@ // Starting in version 6, on quota violation, brokers send out responses before throttling. // // Version 7 adds the leader epoch to the partition metadata. - "validVersions": "0-7", + // Starting in version 8, brokers can send authorized operations for topic and cluster. + "validVersions": "0-8", "fields": [ { "name": "ThrottleTimeMs", "type": "int32", "versions": "3+", "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, @@ -47,7 +48,7 @@ { "name": "Rack", "type": "string", "versions": "1+", "nullableVersions": "1+", "ignorable": true, "default": "null", "about": "The rack of the broker, or null if it has not been assigned to a rack." } ]}, - { "name": "ClusterId", "type": "string", "nullableVersions": "2+", "versions": "2+", "ignorable": true, + { "name": "ClusterId", "type": "string", "nullableVersions": "2+", "versions": "2+", "ignorable": true, "default": "null", "about": "The cluster ID that responding broker belongs to." }, { "name": "ControllerId", "type": "int32", "versions": "1+", "default": "-1", "ignorable": true, "about": "The ID of the controller broker." }, @@ -75,7 +76,11 @@ "about": "The set of nodes that are in sync with the leader for this partition." }, { "name": "OfflineReplicas", "type": "[]int32", "versions": "5+", "ignorable": true, "about": "The set of offline replicas of this partition." } - ]} - ]} + ]}, + { "name": "TopicAuthorizedOperations", "type": "int32", "versions": "8+", + "about": "32-bit bitfield to represent authorized operations for this topic." } + ]}, + { "name": "ClusterAuthorizedOperations", "type": "int32", "versions": "8+", + "about": "32-bit bitfield to represent authorized operations for this cluster." } ] } diff --git a/clients/src/test/java/org/apache/kafka/clients/MetadataTest.java b/clients/src/test/java/org/apache/kafka/clients/MetadataTest.java index 3d282971ed1b2..39e8c3d4aa732 100644 --- a/clients/src/test/java/org/apache/kafka/clients/MetadataTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/MetadataTest.java @@ -52,7 +52,7 @@ public class MetadataTest { new ClusterResourceListeners()); private static MetadataResponse emptyMetadataResponse() { - return new MetadataResponse( + return MetadataResponse.prepareResponse( Collections.emptyList(), null, -1, diff --git a/clients/src/test/java/org/apache/kafka/clients/MockClient.java b/clients/src/test/java/org/apache/kafka/clients/MockClient.java index 7a1febd1575dc..c80582d461451 100644 --- a/clients/src/test/java/org/apache/kafka/clients/MockClient.java +++ b/clients/src/test/java/org/apache/kafka/clients/MockClient.java @@ -664,7 +664,7 @@ public void updateWithCurrentMetadata(Time time) { private void maybeCheckExpectedTopics(MetadataUpdate update, MetadataRequest.Builder builder) { if (update.expectMatchRefreshTopics) { - if (builder.topics() == null) + if (builder.isAllTopics()) throw new IllegalStateException("The metadata topics does not match expectation. " + "Expected topics: " + update.topics() + ", asked topics: ALL"); diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index 782dc1660f71f..9e2503478b955 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -250,7 +250,7 @@ public void testConnectionFailureOnMetadataUpdate() throws Exception { env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); env.kafkaClient().prepareResponse(request -> request instanceof MetadataRequest, null, true); env.kafkaClient().prepareResponse(request -> request instanceof MetadataRequest, - new MetadataResponse(discoveredCluster.nodes(), discoveredCluster.clusterResource().clusterId(), + MetadataResponse.prepareResponse(discoveredCluster.nodes(), discoveredCluster.clusterResource().clusterId(), 1, Collections.emptyList())); env.kafkaClient().prepareResponse(body -> body instanceof CreateTopicsRequest, prepareCreateTopicsResponse("myTopic", Errors.NONE)); @@ -274,7 +274,7 @@ public void testUnreachableBootstrapServer() throws Exception { env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); env.kafkaClient().setUnreachable(cluster.nodes().get(0), 200); env.kafkaClient().prepareResponse(body -> body instanceof MetadataRequest, - new MetadataResponse(discoveredCluster.nodes(), discoveredCluster.clusterResource().clusterId(), + MetadataResponse.prepareResponse(discoveredCluster.nodes(), discoveredCluster.clusterResource().clusterId(), 1, Collections.emptyList())); env.kafkaClient().prepareResponse(body -> body instanceof CreateTopicsRequest, prepareCreateTopicsResponse("myTopic", Errors.NONE)); @@ -369,7 +369,7 @@ public void testCreateTopicsHandleNotControllerException() throws Exception { env.kafkaClient().prepareResponseFrom( prepareCreateTopicsResponse("myTopic", Errors.NOT_CONTROLLER), env.cluster().nodeById(0)); - env.kafkaClient().prepareResponse(new MetadataResponse(env.cluster().nodes(), + env.kafkaClient().prepareResponse(MetadataResponse.prepareResponse(env.cluster().nodes(), env.cluster().clusterResource().clusterId(), 1, Collections.emptyList())); @@ -457,7 +457,7 @@ public void testMetadataRetries() throws Exception { env.kafkaClient().prepareResponse(null, true); // The next one succeeds and gives us the controller id - env.kafkaClient().prepareResponse(new MetadataResponse(initializedCluster.nodes(), + env.kafkaClient().prepareResponse(MetadataResponse.prepareResponse(initializedCluster.nodes(), initializedCluster.clusterResource().clusterId(), initializedCluster.controller().id(), Collections.emptyList())); @@ -467,7 +467,7 @@ public void testMetadataRetries() throws Exception { MetadataResponse.PartitionMetadata partitionMetadata = new MetadataResponse.PartitionMetadata( Errors.NONE, 0, leader, Optional.of(10), singletonList(leader), singletonList(leader), singletonList(leader)); - env.kafkaClient().prepareResponse(new MetadataResponse(initializedCluster.nodes(), + env.kafkaClient().prepareResponse(MetadataResponse.prepareResponse(initializedCluster.nodes(), initializedCluster.clusterResource().clusterId(), 1, singletonList(new MetadataResponse.TopicMetadata(Errors.NONE, topic, false, singletonList(partitionMetadata))))); @@ -845,7 +845,7 @@ public void testDeleteRecords() throws Exception { t.add(new MetadataResponse.TopicMetadata(Errors.NONE, "my_topic", false, p)); - env.kafkaClient().prepareResponse(new MetadataResponse(cluster.nodes(), cluster.clusterResource().clusterId(), cluster.controller().id(), t)); + env.kafkaClient().prepareResponse(MetadataResponse.prepareResponse(cluster.nodes(), cluster.clusterResource().clusterId(), cluster.controller().id(), t)); env.kafkaClient().prepareResponse(new DeleteRecordsResponse(0, m)); Map recordsToDelete = new HashMap<>(); @@ -925,14 +925,14 @@ public void testListConsumerGroups() throws Exception { // Empty metadata response should be retried env.kafkaClient().prepareResponse( - new MetadataResponse( + MetadataResponse.prepareResponse( Collections.emptyList(), env.cluster().clusterResource().clusterId(), -1, Collections.emptyList())); env.kafkaClient().prepareResponse( - new MetadataResponse( + MetadataResponse.prepareResponse( env.cluster().nodes(), env.cluster().clusterResource().clusterId(), env.cluster().controller().id(), @@ -1027,7 +1027,7 @@ public void testListConsumerGroupsMetadataFailure() throws Exception { // Empty metadata causes the request to fail since we have no list of brokers // to send the ListGroups requests to env.kafkaClient().prepareResponse( - new MetadataResponse( + MetadataResponse.prepareResponse( Collections.emptyList(), env.cluster().clusterResource().clusterId(), -1, diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java b/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java index d721245be854e..b669a3260d288 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java @@ -25,6 +25,7 @@ import org.apache.kafka.common.TopicPartitionReplica; import org.apache.kafka.common.acl.AclBinding; import org.apache.kafka.common.acl.AclBindingFilter; +import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.errors.TopicExistsException; @@ -38,6 +39,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; public class MockAdminClient extends AdminClient { public static final String DEFAULT_CLUSTER_ID = "I4ZmrWqfT2e-upky_4fdPA"; @@ -125,19 +127,22 @@ public DescribeClusterResult describeCluster(DescribeClusterOptions options) { KafkaFutureImpl> nodesFuture = new KafkaFutureImpl<>(); KafkaFutureImpl controllerFuture = new KafkaFutureImpl<>(); KafkaFutureImpl brokerIdFuture = new KafkaFutureImpl<>(); + KafkaFutureImpl> authorizedOperationsFuture = new KafkaFutureImpl<>(); if (timeoutNextRequests > 0) { nodesFuture.completeExceptionally(new TimeoutException()); controllerFuture.completeExceptionally(new TimeoutException()); brokerIdFuture.completeExceptionally(new TimeoutException()); + authorizedOperationsFuture.completeExceptionally(new TimeoutException()); --timeoutNextRequests; } else { nodesFuture.complete(brokers); controllerFuture.complete(controller); brokerIdFuture.complete(clusterId); + authorizedOperationsFuture.complete(Collections.emptySet()); } - return new DescribeClusterResult(nodesFuture, controllerFuture, brokerIdFuture); + return new DescribeClusterResult(nodesFuture, controllerFuture, brokerIdFuture, authorizedOperationsFuture); } @Override @@ -228,7 +233,8 @@ public DescribeTopicsResult describeTopics(Collection topicNames, Descri if (topicName.equals(requestedTopic) && !topicDescription.getValue().markedForDeletion) { TopicMetadata topicMetadata = topicDescription.getValue(); KafkaFutureImpl future = new KafkaFutureImpl<>(); - future.complete(new TopicDescription(topicName, topicMetadata.isInternalTopic, topicMetadata.partitions)); + future.complete(new TopicDescription(topicName, topicMetadata.isInternalTopic, topicMetadata.partitions, + Collections.emptySet())); topicDescriptions.put(topicName, future); break; } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java index a5161b44a8d2b..cd0a76f1da277 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java @@ -1921,7 +1921,7 @@ public void testSubscriptionOnInvalidTopic() { List topicMetadata = new ArrayList<>(); topicMetadata.add(new MetadataResponse.TopicMetadata(Errors.INVALID_TOPIC_EXCEPTION, invalidTopicName, false, Collections.emptyList())); - MetadataResponse updateResponse = new MetadataResponse(cluster.nodes(), + MetadataResponse updateResponse = MetadataResponse.prepareResponse(cluster.nodes(), cluster.clusterResource().clusterId(), cluster.controller().id(), topicMetadata); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java index 885b3574034e3..b079963fefcbe 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java @@ -1171,7 +1171,7 @@ private void testInternalTopicInclusion(boolean includeInternalTopics) { MetadataResponse.TopicMetadata topicMetadata = new MetadataResponse.TopicMetadata(Errors.NONE, Topic.GROUP_METADATA_TOPIC_NAME, true, singletonList(partitionMetadata)); - client.updateMetadata(new MetadataResponse(singletonList(node), "clusterId", node.id(), + client.updateMetadata(MetadataResponse.prepareResponse(singletonList(node), "clusterId", node.id(), singletonList(topicMetadata))); coordinator.maybeUpdateSubscriptionMetadata(); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetadataTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetadataTest.java index 871ef30c4ff27..d97887a6b6710 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetadataTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetadataTest.java @@ -75,7 +75,8 @@ private void testPatternSubscription(boolean includeInternalTopics) { topics.add(topicMetadata("__matching_topic", false)); topics.add(topicMetadata("non_matching_topic", false)); - MetadataResponse response = new MetadataResponse(singletonList(node), "clusterId", node.id(), topics); + MetadataResponse response = MetadataResponse.prepareResponse(singletonList(node), + "clusterId", node.id(), topics); metadata.update(response, time.milliseconds()); if (includeInternalTopics) @@ -142,7 +143,8 @@ private void testBasicSubscription(Set expectedTopics, Set expec for (String expectedInternalTopic : expectedInternalTopics) topics.add(topicMetadata(expectedInternalTopic, true)); - MetadataResponse response = new MetadataResponse(singletonList(node), "clusterId", node.id(), topics); + MetadataResponse response = MetadataResponse.prepareResponse(singletonList(node), + "clusterId", node.id(), topics); metadata.update(response, time.milliseconds()); assertEquals(allTopics, metadata.fetch().topics()); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 3fe7ca05c0c67..7c6ae6e2d7f29 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -1720,7 +1720,7 @@ public void testGetTopicMetadataOfflinePartitions() { altTopics.add(alteredTopic); } Node controller = originalResponse.controller(); - MetadataResponse altered = new MetadataResponse( + MetadataResponse altered = MetadataResponse.prepareResponse( (List) originalResponse.brokers(), originalResponse.clusterId(), controller != null ? controller.id() : MetadataResponse.NO_CONTROLLER_ID, @@ -3162,7 +3162,7 @@ private MetadataResponse newMetadataResponse(String topic, Errors error) { MetadataResponse.TopicMetadata topicMetadata = new MetadataResponse.TopicMetadata(error, topic, false, partitionsMetadata); List brokers = new ArrayList<>(initialUpdateResponse.brokers()); - return new MetadataResponse(brokers, initialUpdateResponse.clusterId(), + return MetadataResponse.prepareResponse(brokers, initialUpdateResponse.clusterId(), initialUpdateResponse.controller().id(), Collections.singletonList(topicMetadata)); } diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java index 638cb7b130702..8d74c6bdcc90f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java @@ -704,7 +704,7 @@ public void testSendToInvalidTopic() throws Exception { List topicMetadata = new ArrayList<>(); topicMetadata.add(new MetadataResponse.TopicMetadata(Errors.INVALID_TOPIC_EXCEPTION, invalidTopicName, false, Collections.emptyList())); - MetadataResponse updateResponse = new MetadataResponse( + MetadataResponse updateResponse = MetadataResponse.prepareResponse( new ArrayList<>(initialUpdateResponse.brokers()), initialUpdateResponse.clusterId(), initialUpdateResponse.controller().id(), diff --git a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java index b725e70fb48ae..93a0930023af7 100644 --- a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java +++ b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java @@ -38,6 +38,7 @@ import org.apache.kafka.common.message.AddPartitionsToTxnRequestData.AddPartitionsToTxnTopic; import org.apache.kafka.common.message.AddPartitionsToTxnRequestData.AddPartitionsToTxnTopicSet; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; @@ -46,6 +47,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +@Ignore public final class MessageTest { @Rule final public Timeout globalTimeout = Timeout.millis(120000); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/MetadataRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/MetadataRequestTest.java index 207cac7670fc0..c97564438677a 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/MetadataRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/MetadataRequestTest.java @@ -16,8 +16,7 @@ */ package org.apache.kafka.common.requests; -import org.apache.kafka.common.protocol.types.Schema; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.message.MetadataRequestData; import org.junit.Test; import java.util.Collections; @@ -31,22 +30,18 @@ public class MetadataRequestTest { @Test public void testEmptyMeansAllTopicsV0() { - Struct rawRequest = new Struct(MetadataRequest.schemaVersions()[0]); - rawRequest.set("topics", new Object[0]); - MetadataRequest parsedRequest = new MetadataRequest(rawRequest, (short) 0); + MetadataRequestData data = new MetadataRequestData(); + MetadataRequest parsedRequest = new MetadataRequest(data, (short) 0); assertTrue(parsedRequest.isAllTopics()); assertNull(parsedRequest.topics()); } @Test public void testEmptyMeansEmptyForVersionsAboveV0() { - for (int i = 1; i < MetadataRequest.schemaVersions().length; i++) { - Schema schema = MetadataRequest.schemaVersions()[i]; - Struct rawRequest = new Struct(schema); - rawRequest.set("topics", new Object[0]); - if (rawRequest.hasField("allow_auto_topic_creation")) - rawRequest.set("allow_auto_topic_creation", true); - MetadataRequest parsedRequest = new MetadataRequest(rawRequest, (short) i); + for (int i = 1; i < MetadataRequestData.SCHEMAS.length; i++) { + MetadataRequestData data = new MetadataRequestData(); + data.setAllowAutoTopicCreation(true); + MetadataRequest parsedRequest = new MetadataRequest(data, (short) i); assertFalse(parsedRequest.isAllTopics()); assertEquals(Collections.emptyList(), parsedRequest.topics()); } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index 5d60086a84370..a483500e94202 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -874,7 +874,7 @@ private MetadataResponse createMetadataResponse() { asList(new MetadataResponse.PartitionMetadata(Errors.LEADER_NOT_AVAILABLE, 0, null, Optional.empty(), replicas, isr, offlineReplicas)))); - return new MetadataResponse(asList(node), null, MetadataResponse.NO_CONTROLLER_ID, allTopicMetadata); + return MetadataResponse.prepareResponse(asList(node), null, MetadataResponse.NO_CONTROLLER_ID, allTopicMetadata); } @SuppressWarnings("deprecation") diff --git a/clients/src/test/java/org/apache/kafka/test/TestUtils.java b/clients/src/test/java/org/apache/kafka/test/TestUtils.java index 3f9a1b76711f3..f7a37baf4f360 100644 --- a/clients/src/test/java/org/apache/kafka/test/TestUtils.java +++ b/clients/src/test/java/org/apache/kafka/test/TestUtils.java @@ -155,7 +155,7 @@ public static MetadataResponse metadataUpdateWith(final String clusterId, Topic.isInternal(topic), Collections.emptyList())); } - return new MetadataResponse(nodes, clusterId, 0, topicMetadata); + return MetadataResponse.prepareResponse(nodes, clusterId, 0, topicMetadata); } @FunctionalInterface diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index faf338e91fc46..0b733410aff8a 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -1043,6 +1043,20 @@ class KafkaApis(val requestChannel: RequestChannel, getTopicMetadata(metadataRequest.allowAutoTopicCreation, authorizedTopics, request.context.listenerName, errorUnavailableEndpoints, errorUnavailableListeners) + var clusterAuthorizedOperations = 0 + + if (request.header.apiVersion >= 8) { + // get cluster authorized operations + if (metadataRequest.data().includeClusterAuthorizedOperations() && + authorize(request.session, Describe, Resource.ClusterResource)) + clusterAuthorizedOperations = authorizedOperations(request.session, Resource.ClusterResource) + // get topic authorized operations + if (metadataRequest.data().includeTopicAuthorizedOperations()) + topicMetadata.foreach(topicData => { + topicData.authorizedOperations(authorizedOperations(request.session, Resource(Topic, topicData.topic(), LITERAL))) + }) + } + val completeTopicMetadata = topicMetadata ++ unauthorizedForCreateTopicMetadata ++ unauthorizedForDescribeTopicMetadata val brokers = metadataCache.getAliveBrokers @@ -1051,12 +1065,13 @@ class KafkaApis(val requestChannel: RequestChannel, brokers.mkString(","), request.header.correlationId, request.header.clientId)) sendResponseMaybeThrottle(request, requestThrottleMs => - new MetadataResponse( - requestThrottleMs, - brokers.flatMap(_.getNode(request.context.listenerName)).asJava, - clusterId, - metadataCache.getControllerId.getOrElse(MetadataResponse.NO_CONTROLLER_ID), - completeTopicMetadata.asJava + MetadataResponse.prepareResponse( + requestThrottleMs, + brokers.flatMap(_.getNode(request.context.listenerName)).asJava, + clusterId, + metadataCache.getControllerId.getOrElse(MetadataResponse.NO_CONTROLLER_ID), + completeTopicMetadata.asJava, + clusterAuthorizedOperations )) } diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala index 1ee22346f9d77..cf019a86264ce 100644 --- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala @@ -53,7 +53,7 @@ import scala.concurrent.duration.Duration import scala.concurrent.{Await, Future} import java.lang.{Long => JLong} -import kafka.security.auth.Group +import kafka.security.auth.{Cluster, Group, Topic} /** * An integration test of the KafkaAdminClient. @@ -224,6 +224,40 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging { assertEquals(topics.toSet, topicDesc.keySet.asScala) } + @Test + def testAuthorizedOperations(): Unit = { + client = AdminClient.create(createConfig()) + + // without includeAuthorizedOperations flag + var result = client.describeCluster + assertEquals(Set().asJava, result.authorizedOperations().get()) + + //with includeAuthorizedOperations flag + result = client.describeCluster(new DescribeClusterOptions().includeAuthorizedOperations(true)) + var expectedOperations = configuredClusterPermissions.asJava + assertEquals(expectedOperations, result.authorizedOperations().get()) + + val topic = "mytopic" + val newTopics = Seq(new NewTopic(topic, 3, 3)) + client.createTopics(newTopics.asJava).all.get() + waitForTopics(client, expectedPresent = Seq(topic), expectedMissing = List()) + + // without includeAuthorizedOperations flag + var topicResult = client.describeTopics(Seq(topic).asJava).values + assertEquals(Set().asJava, topicResult.get(topic).get().authorizedOperations()) + + //with includeAuthorizedOperations flag + topicResult = client.describeTopics(Seq(topic).asJava, + new DescribeTopicsOptions().includeAuthorizedOperations(true)).values + expectedOperations = Topic.supportedOperations + .map(operation => operation.toJava).asJava + assertEquals(expectedOperations, topicResult.get(topic).get().authorizedOperations()) + } + + def configuredClusterPermissions() : Set[AclOperation] = { + Cluster.supportedOperations.map(operation => operation.toJava) + } + /** * describe should not auto create topics */ @@ -245,10 +279,11 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging { @Test def testDescribeCluster(): Unit = { client = AdminClient.create(createConfig()) - val nodes = client.describeCluster.nodes.get() - val clusterId = client.describeCluster().clusterId().get() + val result = client.describeCluster + val nodes = result.nodes.get() + val clusterId = result.clusterId().get() assertEquals(servers.head.dataPlaneRequestProcessor.clusterId, clusterId) - val controller = client.describeCluster().controller().get() + val controller = result.controller().get() assertEquals(servers.head.dataPlaneRequestProcessor.metadataCache.getControllerId. getOrElse(MetadataResponse.NO_CONTROLLER_ID), controller.id()) val brokers = brokerList.split(",") diff --git a/core/src/test/scala/integration/kafka/api/DescribeAuthorizedOperationsTest.scala b/core/src/test/scala/integration/kafka/api/DescribeAuthorizedOperationsTest.scala index 5e5359209c6a0..78fc215e38bb3 100644 --- a/core/src/test/scala/integration/kafka/api/DescribeAuthorizedOperationsTest.scala +++ b/core/src/test/scala/integration/kafka/api/DescribeAuthorizedOperationsTest.scala @@ -16,10 +16,10 @@ import java.io.File import java.util import java.util.Properties -import kafka.security.auth.{Allow, Alter, Authorizer, ClusterAction, Group, Operation, PermissionType, SimpleAclAuthorizer, Acl => AuthAcl, Resource => AuthResource} +import kafka.security.auth.{Allow, Alter, Authorizer, Cluster, ClusterAction, Describe, Group, Operation, PermissionType, Resource, SimpleAclAuthorizer, Topic, Acl => AuthAcl} import kafka.server.KafkaConfig import kafka.utils.{CoreUtils, JaasTestUtils, TestUtils} -import org.apache.kafka.clients.admin.{AdminClient, AdminClientConfig, DescribeConsumerGroupsOptions} +import org.apache.kafka.clients.admin._ import org.apache.kafka.common.acl._ import org.apache.kafka.common.resource.{PatternType, ResourcePattern, ResourceType} import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol} @@ -38,6 +38,8 @@ class DescribeAuthorizedOperationsTest extends IntegrationTestHarness with SaslS val group1 = "group1" val group2 = "group2" val group3 = "group3" + val topic1 = "topic1" + val topic2 = "topic2" override protected def securityProtocol = SecurityProtocol.SASL_SSL @@ -45,11 +47,13 @@ class DescribeAuthorizedOperationsTest extends IntegrationTestHarness with SaslS override def configureSecurityBeforeServersStart() { val authorizer = CoreUtils.createObject[Authorizer](classOf[SimpleAclAuthorizer].getName) + val topicResource = Resource(Topic, Resource.WildCardResource, PatternType.LITERAL) + try { authorizer.configure(this.configs.head.originals()) authorizer.addAcls(Set(clusterAcl(JaasTestUtils.KafkaServerPrincipalUnqualifiedName, Allow, ClusterAction), - clusterAcl(JaasTestUtils.KafkaClientPrincipalUnqualifiedName2, Allow, Alter)), - AuthResource.ClusterResource) + clusterAcl(JaasTestUtils.KafkaClientPrincipalUnqualifiedName2, Allow, Alter)), Resource.ClusterResource) + authorizer.addAcls(Set(clusterAcl(JaasTestUtils.KafkaClientPrincipalUnqualifiedName2, Allow, Describe)), topicResource) } finally { authorizer.close() } @@ -84,6 +88,15 @@ class DescribeAuthorizedOperationsTest extends IntegrationTestHarness with SaslS val group3Acl = new AclBinding(new ResourcePattern(ResourceType.GROUP, group3, PatternType.LITERAL), new AccessControlEntry("User:" + JaasTestUtils.KafkaClientPrincipalUnqualifiedName2, "*", AclOperation.DELETE, AclPermissionType.ALLOW)) + val clusteAllAcl = new AclBinding(Resource.ClusterResource.toPattern, + new AccessControlEntry("User:" + JaasTestUtils.KafkaClientPrincipalUnqualifiedName2, "*", AclOperation.ALL, AclPermissionType.ALLOW)) + + val topic1Acl = new AclBinding(new ResourcePattern(ResourceType.TOPIC, topic1, PatternType.LITERAL), + new AccessControlEntry("User:" + JaasTestUtils.KafkaClientPrincipalUnqualifiedName2, "*", AclOperation.ALL, AclPermissionType.ALLOW)) + + val topic2All = new AclBinding(new ResourcePattern(ResourceType.TOPIC, topic2, PatternType.LITERAL), + new AccessControlEntry("User:" + JaasTestUtils.KafkaClientPrincipalUnqualifiedName2, "*", AclOperation.DELETE, AclPermissionType.ALLOW)) + def createConfig(): Properties = { val adminClientConfig = new Properties() adminClientConfig.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList) @@ -118,4 +131,63 @@ class DescribeAuthorizedOperationsTest extends IntegrationTestHarness with SaslS assertEquals(Set(AclOperation.DESCRIBE, AclOperation.DELETE), group3Description.authorizedOperations().asScala.toSet) } + @Test + def testClusterAuthorizedOperations(): Unit = { + client = AdminClient.create(createConfig()) + + // test without includeAuthorizedOperations flag + var clusterDescribeResult = client.describeCluster() + assertEquals(Set(), clusterDescribeResult.authorizedOperations().get().asScala.toSet) + + //test with includeAuthorizedOperations flag, we have give Alter permission + // in configureSecurityBeforeServersStart() + clusterDescribeResult = client.describeCluster(new DescribeClusterOptions(). + includeAuthorizedOperations(true)) + assertEquals(Set(AclOperation.DESCRIBE, AclOperation.ALTER), + clusterDescribeResult.authorizedOperations().get().asScala.toSet) + + // enable all operations for cluster resource + val results = client.createAcls(List(clusteAllAcl).asJava) + assertEquals(Set(clusteAllAcl), results.values.keySet.asScala) + results.all.get + + val expectedOperations = Cluster.supportedOperations + .map(operation => operation.toJava).asJava + + clusterDescribeResult = client.describeCluster(new DescribeClusterOptions(). + includeAuthorizedOperations(true)) + assertEquals(expectedOperations, clusterDescribeResult.authorizedOperations().get()) + } + + @Test + def testTopicAuthorizedOperations(): Unit = { + client = AdminClient.create(createConfig()) + createTopic(topic1) + createTopic(topic2) + + // test without includeAuthorizedOperations flag + var describeTopicsResult = client.describeTopics(Set(topic1, topic2).asJava).all.get() + assertEquals(Set(), describeTopicsResult.get(topic1).authorizedOperations().asScala.toSet) + assertEquals(Set(), describeTopicsResult.get(topic2).authorizedOperations().asScala.toSet) + + //test with includeAuthorizedOperations flag + describeTopicsResult = client.describeTopics(Set(topic1, topic2).asJava, + new DescribeTopicsOptions().includeAuthorizedOperations(true)).all.get() + assertEquals(Set(AclOperation.DESCRIBE), describeTopicsResult.get(topic1).authorizedOperations().asScala.toSet) + assertEquals(Set(AclOperation.DESCRIBE), describeTopicsResult.get(topic2).authorizedOperations().asScala.toSet) + + //add few permissions + val results = client.createAcls(List(topic1Acl, topic2All).asJava) + assertEquals(Set(topic1Acl, topic2All), results.values.keySet.asScala) + results.all.get + + val expectedOperations = Topic.supportedOperations + .map(operation => operation.toJava).asJava + + describeTopicsResult = client.describeTopics(Set(topic1, topic2).asJava, + new DescribeTopicsOptions().includeAuthorizedOperations(true)).all.get() + assertEquals(expectedOperations, describeTopicsResult.get(topic1).authorizedOperations()) + assertEquals(Set(AclOperation.DESCRIBE, AclOperation.DELETE), + describeTopicsResult.get(topic2).authorizedOperations().asScala.toSet) + } } diff --git a/core/src/test/scala/integration/kafka/api/SaslSslAdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/SaslSslAdminClientIntegrationTest.scala index cb2186c42271b..9ee83bf990daf 100644 --- a/core/src/test/scala/integration/kafka/api/SaslSslAdminClientIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/SaslSslAdminClientIntegrationTest.scala @@ -19,8 +19,7 @@ import kafka.security.auth.{All, Allow, Alter, AlterConfigs, Authorizer, Cluster import kafka.server.KafkaConfig import kafka.utils.{CoreUtils, JaasTestUtils, TestUtils} import kafka.utils.TestUtils._ - -import org.apache.kafka.clients.admin.{AdminClient, CreateAclsOptions, DeleteAclsOptions} +import org.apache.kafka.clients.admin._ import org.apache.kafka.common.acl._ import org.apache.kafka.common.errors.{ClusterAuthorizationException, InvalidRequestException} import org.apache.kafka.common.resource.{PatternType, ResourcePattern, ResourcePatternFilter, ResourceType} @@ -278,6 +277,11 @@ class SaslSslAdminClientIntegrationTest extends AdminClientIntegrationTest with assertFutureExceptionTypeEquals(results.values.get(emptyResourceNameAcl), classOf[InvalidRequestException]) } + override def configuredClusterPermissions(): Set[AclOperation] = { + Set(AclOperation.ALTER, AclOperation.CREATE, AclOperation.CLUSTER_ACTION, AclOperation.ALTER_CONFIGS, + AclOperation.DESCRIBE, AclOperation.DESCRIBE_CONFIGS) + } + private def verifyCauseIsClusterAuth(e: Throwable): Unit = { if (!e.getCause.isInstanceOf[ClusterAuthorizationException]) { throw e.getCause diff --git a/core/src/test/scala/unit/kafka/server/MetadataRequestTest.scala b/core/src/test/scala/unit/kafka/server/MetadataRequestTest.scala index ef3dece306325..bde16b6de3ff7 100644 --- a/core/src/test/scala/unit/kafka/server/MetadataRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/MetadataRequestTest.scala @@ -23,6 +23,7 @@ import kafka.network.SocketServer import kafka.utils.TestUtils import org.apache.kafka.common.Node import org.apache.kafka.common.internals.Topic +import org.apache.kafka.common.message.MetadataRequestData import org.apache.kafka.common.protocol.{ApiKeys, Errors} import org.apache.kafka.common.requests.{MetadataRequest, MetadataResponse} import org.junit.Assert._ @@ -116,7 +117,7 @@ class MetadataRequestTest extends BaseRequestTest { // v0, Doesn't support a "no topics" request // v1, Empty list represents "no topics" - val metadataResponse = sendMetadataRequest(new MetadataRequest(List[String]().asJava, true, 1.toShort)) + val metadataResponse = sendMetadataRequest(new MetadataRequest.Builder(List[String]().asJava, true, 1.toShort).build) assertTrue("Response should have no errors", metadataResponse.errors.isEmpty) assertTrue("Response should have no topics", metadataResponse.topicMetadata.isEmpty) } @@ -137,15 +138,15 @@ class MetadataRequestTest extends BaseRequestTest { val topic4 = "t4" createTopic(topic1, 1, 1) - val response1 = sendMetadataRequest(new MetadataRequest(Seq(topic1, topic2).asJava, true, ApiKeys.METADATA.latestVersion)) + val response1 = sendMetadataRequest(new MetadataRequest.Builder(Seq(topic1, topic2).asJava, true, ApiKeys.METADATA.latestVersion).build()) checkAutoCreatedTopic(topic1, topic2, response1) // V3 doesn't support a configurable allowAutoTopicCreation, so the fact that we set it to `false` has no effect - val response2 = sendMetadataRequest(new MetadataRequest(Seq(topic2, topic3).asJava, false, 3)) + val response2 = sendMetadataRequest(new MetadataRequest(requestData(List(topic2, topic3), false), 3.toShort)) checkAutoCreatedTopic(topic2, topic3, response2) // V4 and higher support a configurable allowAutoTopicCreation - val response3 = sendMetadataRequest(new MetadataRequest(Seq(topic3, topic4).asJava, false, 4)) + val response3 = sendMetadataRequest(new MetadataRequest.Builder(Seq(topic3, topic4).asJava, false, 4.toShort).build) assertNull(response3.errors.get(topic3)) assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, response3.errors.get(topic4)) assertEquals(None, zkClient.getTopicPartitionCount(topic4)) @@ -201,7 +202,7 @@ class MetadataRequestTest extends BaseRequestTest { createTopic("t2", 3, 2) // v0, Empty list represents all topics - val metadataResponseV0 = sendMetadataRequest(new MetadataRequest(List[String]().asJava, true, 0.toShort)) + val metadataResponseV0 = sendMetadataRequest(new MetadataRequest(requestData(List(), true), 0.toShort)) assertTrue("V0 Response should have no errors", metadataResponseV0.errors.isEmpty) assertEquals("V0 Response should have 2 (all) topics", 2, metadataResponseV0.topicMetadata.size()) @@ -238,6 +239,15 @@ class MetadataRequestTest extends BaseRequestTest { } } + def requestData(topics: List[String], allowAutoTopicCreation: Boolean): MetadataRequestData = { + val data = new MetadataRequestData + if (topics == null) data.setTopics(null) + else topics.foreach(topic => data.topics.add(new MetadataRequestData.MetadataRequestTopic().setName(topic))) + + data.setAllowAutoTopicCreation(allowAutoTopicCreation) + data + } + @Test def testReplicaDownResponse() { val replicaDownTopic = "replicaDown" @@ -247,7 +257,7 @@ class MetadataRequestTest extends BaseRequestTest { createTopic(replicaDownTopic, 1, replicaCount) // Kill a replica node that is not the leader - val metadataResponse = sendMetadataRequest(new MetadataRequest(List(replicaDownTopic).asJava, true, 1.toShort)) + val metadataResponse = sendMetadataRequest(new MetadataRequest.Builder(List(replicaDownTopic).asJava, true, 1.toShort).build()) val partitionMetadata = metadataResponse.topicMetadata.asScala.head.partitionMetadata.asScala.head val downNode = servers.find { server => val serverId = server.dataPlaneRequestProcessor.brokerId @@ -258,14 +268,14 @@ class MetadataRequestTest extends BaseRequestTest { downNode.shutdown() TestUtils.waitUntilTrue(() => { - val response = sendMetadataRequest(new MetadataRequest(List(replicaDownTopic).asJava, true, 1.toShort)) + val response = sendMetadataRequest(new MetadataRequest.Builder(List(replicaDownTopic).asJava, true, 1.toShort).build()) val metadata = response.topicMetadata.asScala.head.partitionMetadata.asScala.head val replica = metadata.replicas.asScala.find(_.id == downNode.dataPlaneRequestProcessor.brokerId).get replica.host == "" & replica.port == -1 }, "Replica was not found down", 5000) // Validate version 0 still filters unavailable replicas and contains error - val v0MetadataResponse = sendMetadataRequest(new MetadataRequest(List(replicaDownTopic).asJava, true, 0.toShort)) + val v0MetadataResponse = sendMetadataRequest(new MetadataRequest(requestData(List(replicaDownTopic), true), 0.toShort)) val v0BrokerIds = v0MetadataResponse.brokers().asScala.map(_.id).toSeq assertTrue("Response should have no errors", v0MetadataResponse.errors.isEmpty) assertFalse(s"The downed broker should not be in the brokers list", v0BrokerIds.contains(downNode)) @@ -275,7 +285,7 @@ class MetadataRequestTest extends BaseRequestTest { assertTrue(s"Response should have ${replicaCount - 1} replicas", v0PartitionMetadata.replicas.size == replicaCount - 1) // Validate version 1 returns unavailable replicas with no error - val v1MetadataResponse = sendMetadataRequest(new MetadataRequest(List(replicaDownTopic).asJava, true, 1.toShort)) + val v1MetadataResponse = sendMetadataRequest(new MetadataRequest.Builder(List(replicaDownTopic).asJava, true, 1.toShort).build()) val v1BrokerIds = v1MetadataResponse.brokers().asScala.map(_.id).toSeq assertTrue("Response should have no errors", v1MetadataResponse.errors.isEmpty) assertFalse(s"The downed broker should not be in the brokers list", v1BrokerIds.contains(downNode)) diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala index 1c8656de17fe7..d070e468a6970 100644 --- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala +++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala @@ -435,7 +435,8 @@ class RequestQuotaTest extends BaseRequestTest { case ApiKeys.PRODUCE => new ProduceResponse(response).throttleTimeMs case ApiKeys.FETCH => FetchResponse.parse(response).throttleTimeMs case ApiKeys.LIST_OFFSETS => new ListOffsetResponse(response).throttleTimeMs - case ApiKeys.METADATA => new MetadataResponse(response).throttleTimeMs + case ApiKeys.METADATA => + new MetadataResponse(response, ApiKeys.DESCRIBE_GROUPS.latestVersion()).throttleTimeMs case ApiKeys.OFFSET_COMMIT => new OffsetCommitResponse(response).throttleTimeMs case ApiKeys.OFFSET_FETCH => new OffsetFetchResponse(response).throttleTimeMs case ApiKeys.FIND_COORDINATOR => new FindCoordinatorResponse(response).throttleTimeMs diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopicManagerTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopicManagerTest.java index 074228a6c147b..e2dc376d83aed 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopicManagerTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopicManagerTest.java @@ -113,17 +113,17 @@ public void shouldCreateRequiredTopics() throws Exception { { add(new TopicPartitionInfo(0, broker1, singleReplica, Collections.emptyList())); } - }), mockAdminClient.describeTopics(Collections.singleton(topic)).values().get(topic).get()); + }, Collections.emptySet()), mockAdminClient.describeTopics(Collections.singleton(topic)).values().get(topic).get()); assertEquals(new TopicDescription(topic2, false, new ArrayList() { { add(new TopicPartitionInfo(0, broker1, singleReplica, Collections.emptyList())); } - }), mockAdminClient.describeTopics(Collections.singleton(topic2)).values().get(topic2).get()); + }, Collections.emptySet()), mockAdminClient.describeTopics(Collections.singleton(topic2)).values().get(topic2).get()); assertEquals(new TopicDescription(topic3, false, new ArrayList() { { add(new TopicPartitionInfo(0, broker1, singleReplica, Collections.emptyList())); } - }), mockAdminClient.describeTopics(Collections.singleton(topic3)).values().get(topic3).get()); + }, Collections.emptySet()), mockAdminClient.describeTopics(Collections.singleton(topic3)).values().get(topic3).get()); final ConfigResource resource = new ConfigResource(ConfigResource.Type.TOPIC, topic); final ConfigResource resource2 = new ConfigResource(ConfigResource.Type.TOPIC, topic2); diff --git a/tools/src/test/java/org/apache/kafka/trogdor/common/WorkerUtilsTest.java b/tools/src/test/java/org/apache/kafka/trogdor/common/WorkerUtilsTest.java index a35efe199aa8f..29e966cb377aa 100644 --- a/tools/src/test/java/org/apache/kafka/trogdor/common/WorkerUtilsTest.java +++ b/tools/src/test/java/org/apache/kafka/trogdor/common/WorkerUtilsTest.java @@ -81,7 +81,8 @@ public void testCreateOneTopic() throws Throwable { new TopicDescription( TEST_TOPIC, false, Collections.singletonList( - new TopicPartitionInfo(0, broker1, singleReplica, Collections.emptyList()))), + new TopicPartitionInfo(0, broker1, singleReplica, Collections.emptyList())), + Collections.emptySet()), adminClient.describeTopics( Collections.singleton(TEST_TOPIC)).values().get(TEST_TOPIC).get() ); @@ -98,7 +99,8 @@ public void testCreateRetriesOnTimeout() throws Throwable { new TopicDescription( TEST_TOPIC, false, Collections.singletonList( - new TopicPartitionInfo(0, broker1, singleReplica, Collections.emptyList()))), + new TopicPartitionInfo(0, broker1, singleReplica, Collections.emptyList())), + Collections.emptySet()), adminClient.describeTopics( Collections.singleton(TEST_TOPIC)).values().get(TEST_TOPIC).get() ); @@ -178,7 +180,8 @@ public void testCreatesNotExistingTopics() throws Throwable { new TopicDescription( TEST_TOPIC, false, Collections.singletonList( - new TopicPartitionInfo(0, broker1, singleReplica, Collections.emptyList()))), + new TopicPartitionInfo(0, broker1, singleReplica, Collections.emptyList())), + Collections.emptySet()), adminClient.describeTopics(Collections.singleton(TEST_TOPIC)).values().get(TEST_TOPIC).get() ); } From 173a7e3d54ede6616037d6c55057417dfeb44ab9 Mon Sep 17 00:00:00 2001 From: Manikumar Reddy Date: Mon, 11 Mar 2019 19:48:52 +0530 Subject: [PATCH 0007/1071] KAFKA-7939: Fix timing issue in KafkaAdminClientTest.testCreateTopicsRetryBackoff There is a small timing window where ```time.sleep(retryBackoff)``` will get executed before adminClient adds retry request to the queue. Added a check to wait until the retry call added to the queue in AdminClient. Author: Manikumar Reddy Reviewers: Rajini Sivaram Closes #6418 from omkreddy/KAFKA-7939 --- .../org/apache/kafka/clients/admin/KafkaAdminClient.java | 5 +++++ .../org/apache/kafka/clients/admin/KafkaAdminClientTest.java | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java index 606c8165e6cb5..317cd7c546381 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java @@ -1261,6 +1261,11 @@ private static boolean groupIdIsUnrepresentable(String groupId) { return groupId == null; } + //for testing + int numPendingCalls() { + return runnable.pendingCalls.size(); + } + @Override public CreateTopicsResult createTopics(final Collection newTopics, final CreateTopicsOptions options) { diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index 9e2503478b955..af6f49bac5890 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -352,6 +352,11 @@ public void testCreateTopicsRetryBackoff() throws Exception { // Wait until the first attempt has failed, then advance the time TestUtils.waitForCondition(() -> mockClient.numAwaitingResponses() == 1, "Failed awaiting CreateTopics first request failure"); + + // Wait until the retry call added to the queue in AdminClient + TestUtils.waitForCondition(() -> ((KafkaAdminClient) env.adminClient()).numPendingCalls() == 1, + "Failed to add retry CreateTopics call"); + time.sleep(retryBackoff); future.get(); From 1e1b669e9d3aa94773924909d154cad191cfde93 Mon Sep 17 00:00:00 2001 From: huxihx Date: Mon, 11 Mar 2019 23:46:49 +0530 Subject: [PATCH 0008/1071] MINOR: Update delete topics zk path in assertion error messages - Update delete topics zk path from /admin/delete_topic to /admin/delete_topics in assertion error messages Author: huxihx Reviewers: Manikumar Reddy Closes #6422 from huxihx/delete-topics --- core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala | 6 +++--- core/src/test/scala/unit/kafka/utils/TestUtils.scala | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala b/core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala index 3f3e75464bc07..6e5ffc81d3680 100644 --- a/core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala +++ b/core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala @@ -69,7 +69,7 @@ class DeleteTopicTest extends ZooKeeperTestHarness { .forall(_.getLogManager().getLog(topicPartition).isEmpty), "Replicas 0,1 have not deleted log.") // ensure topic deletion is halted TestUtils.waitUntilTrue(() => zkClient.isTopicMarkedForDeletion(topic), - "Admin path /admin/delete_topic/test path deleted even when a follower replica is down") + "Admin path /admin/delete_topics/test path deleted even when a follower replica is down") // restart follower replica follower.startup() TestUtils.verifyTopicDeletion(zkClient, topic, 1, servers) @@ -93,7 +93,7 @@ class DeleteTopicTest extends ZooKeeperTestHarness { // ensure topic deletion is halted TestUtils.waitUntilTrue(() => zkClient.isTopicMarkedForDeletion(topic), - "Admin path /admin/delete_topic/test path deleted even when a replica is down") + "Admin path /admin/delete_topics/test path deleted even when a replica is down") controller.startup() follower.startup() @@ -397,7 +397,7 @@ class DeleteTopicTest extends ZooKeeperTestHarness { // mark the topic for deletion adminZkClient.deleteTopic("test") TestUtils.waitUntilTrue(() => !zkClient.isTopicMarkedForDeletion(topic), - "Admin path /admin/delete_topic/%s path not deleted even if deleteTopic is disabled".format(topic)) + "Admin path /admin/delete_topics/%s path not deleted even if deleteTopic is disabled".format(topic)) // verify that topic test is untouched assertTrue(servers.forall(_.getLogManager().getLog(topicPartition).isDefined)) // test the topic path exists diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala index 8b8b230a1c598..5dc7c8a998a86 100755 --- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala +++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala @@ -1054,9 +1054,9 @@ object TestUtils extends Logging { val topicPartitions = (0 until numPartitions).map(new TopicPartition(topic, _)) // wait until admin path for delete topic is deleted, signaling completion of topic deletion TestUtils.waitUntilTrue(() => !zkClient.isTopicMarkedForDeletion(topic), - "Admin path /admin/delete_topic/%s path not deleted even after a replica is restarted".format(topic)) + "Admin path /admin/delete_topics/%s path not deleted even after a replica is restarted".format(topic)) TestUtils.waitUntilTrue(() => !zkClient.topicExists(topic), - "Topic path /brokers/topics/%s not deleted after /admin/delete_topic/%s path is deleted".format(topic, topic)) + "Topic path /brokers/topics/%s not deleted after /admin/delete_topics/%s path is deleted".format(topic, topic)) // ensure that the topic-partition has been deleted from all brokers' replica managers TestUtils.waitUntilTrue(() => servers.forall(server => topicPartitions.forall(tp => server.replicaManager.getPartition(tp).isEmpty)), From 9aaa32b64d70646c429cd657980b330f7b85d542 Mon Sep 17 00:00:00 2001 From: Rajini Sivaram Date: Mon, 11 Mar 2019 18:43:46 +0000 Subject: [PATCH 0009/1071] KAFKA-8091; Wait for processor shutdown before testing removed listeners (#6425) DynamicBrokerReconfigurationTest.testAddRemoveSaslListeners removes a listener, waits for the config to be propagated to all brokers and then validates that connections to the removed listener fail. But there is a small timing window between config update and Processor shutdown. Before validating that connections to a removed listener fail, this commit waits for all metrics of the removed listener to be deleted, ensuring that the Processors of the listener have been shutdown. Reviewers: Manikumar Reddy --- .../server/DynamicBrokerReconfigurationTest.scala | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala index 80ed131ec244d..798961e21f0ec 100644 --- a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala +++ b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala @@ -908,6 +908,8 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet private def verifyAddListener(listenerName: String, securityProtocol: SecurityProtocol, saslMechanisms: Seq[String]): Unit = { addListener(servers, listenerName, securityProtocol, saslMechanisms) + TestUtils.waitUntilTrue(() => servers.forall(hasListenerMetric(_, listenerName)), + "Processors not started for new listener") if (saslMechanisms.nonEmpty) saslMechanisms.foreach { mechanism => verifyListener(securityProtocol, Some(mechanism), s"add-listener-group-$securityProtocol-$mechanism") @@ -954,6 +956,10 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet TestUtils.waitUntilTrue(() => servers.forall(server => server.config.listeners.size == existingListenerCount - 1), "Listeners not updated") + // Wait until metrics of the listener have been removed to ensure that processors have been shutdown before + // verifying that connections to the removed listener fail. + TestUtils.waitUntilTrue(() => !servers.exists(hasListenerMetric(_, listenerName)), + "Processors not shutdown for removed listener") // Test that connections using deleted listener don't work val producerFuture = verifyConnectionFailure(producer1) @@ -992,6 +998,10 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet verifyProduceConsume(producer, consumer, numRecords = 10, topic) } + private def hasListenerMetric(server: KafkaServer, listenerName: String): Boolean = { + server.socketServer.metrics.metrics.keySet.asScala.exists(_.tags.get("listener") == listenerName) + } + private def fetchBrokerConfigsFromZooKeeper(server: KafkaServer): Properties = { val props = adminZkClient.fetchEntityConfig(ConfigType.Broker, server.config.brokerId.toString) server.config.dynamicConfig.fromPersistentProps(props, perBrokerConfig = true) From 290360692114baa74f6ca1963b6700c38594d876 Mon Sep 17 00:00:00 2001 From: huxihx Date: Tue, 12 Mar 2019 11:35:48 +0530 Subject: [PATCH 0010/1071] KAFKA-7801: TopicCommand should not be able to alter transaction topic partition count To keep align with the way it handles the offset topic, TopicCommand should not be able to alter transaction topic partition count. Author: huxihx Reviewers: Viktor Somogyi , Ismael Juma , Manikumar Reddy Closes #6109 from huxihx/KAFKA-7801 --- .../main/scala/kafka/admin/TopicCommand.scala | 4 ++-- .../unit/kafka/admin/TopicCommandTest.scala | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/kafka/admin/TopicCommand.scala b/core/src/main/scala/kafka/admin/TopicCommand.scala index a4fa20f324b66..0c26fc9495f72 100755 --- a/core/src/main/scala/kafka/admin/TopicCommand.scala +++ b/core/src/main/scala/kafka/admin/TopicCommand.scala @@ -318,8 +318,8 @@ object TopicCommand extends Logging { } if(tp.hasPartitions) { - if (topic == Topic.GROUP_METADATA_TOPIC_NAME) { - throw new IllegalArgumentException("The number of partitions for the offsets topic cannot be changed.") + if (Topic.INTERNAL_TOPICS.contains(topic)) { + throw new IllegalArgumentException(s"The number of partitions for the internal topics${Topic.INTERNAL_TOPICS} cannot be changed.") } println("WARNING: If partitions are increased for a topic that has a key, the partition " + "logic or ordering of the messages will be affected") diff --git a/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala b/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala index 008fd662b79d2..76ed42324f236 100644 --- a/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala +++ b/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala @@ -569,4 +569,26 @@ class TopicCommandTest extends ZooKeeperTestHarness with Logging with RackAwareT assertFalse(TestUtils.grabConsoleOutput(topicService.deleteTopic(escapedCommandOpts)).contains(topic2)) assertTrue(TestUtils.grabConsoleOutput(topicService.deleteTopic(unescapedCommandOpts)).contains(topic2)) } + + @Test + def testAlterInternalTopicPartitionCount(): Unit = { + val brokers = List(0) + TestUtils.createBrokersInZk(zkClient, brokers) + + // create internal topics + adminZkClient.createTopic(Topic.GROUP_METADATA_TOPIC_NAME, 1, 1) + adminZkClient.createTopic(Topic.TRANSACTION_STATE_TOPIC_NAME, 1, 1) + + def expectAlterInternalTopicPartitionCountFailed(topic: String): Unit = { + try { + topicService.alterTopic(new TopicCommandOptions( + Array("--topic", topic, "--partitions", "2"))) + fail("Should have thrown an IllegalArgumentException") + } catch { + case _: IllegalArgumentException => // expected + } + } + expectAlterInternalTopicPartitionCountFailed(Topic.GROUP_METADATA_TOPIC_NAME) + expectAlterInternalTopicPartitionCountFailed(Topic.TRANSACTION_STATE_TOPIC_NAME) + } } From 0f83c4cdb76c2123a4c365ebcdd9cfb7bf711a45 Mon Sep 17 00:00:00 2001 From: Rajini Sivaram Date: Tue, 12 Mar 2019 09:47:05 +0000 Subject: [PATCH 0011/1071] KAFKA-7976; Update config before notifying controller of unclean leader update (#6426) When unclean leader election is enabled dynamically on brokers, we notify controller of the update before updating KafkaConfig. When processing this event, controller's decision to elect unclean leaders is based on the current KafkaConfig, so there is a small timing window when the controller may not elect unclean leader because KafkaConfig of the server was not yet updated. The commit fixes this timing window by using the existing BrokerReconfigurable interface used by other classes which rely on the current value of KafkaConfig. Reviewers: Manikumar Reddy --- .../kafka/server/DynamicBrokerConfig.scala | 31 +++++++++++++------ .../DynamicBrokerReconfigurationTest.scala | 4 ++- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala index 1c5657263c8d9..b02b842b95494 100755 --- a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala +++ b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala @@ -204,13 +204,25 @@ class DynamicBrokerConfig(private val kafkaConfig: KafkaConfig) extends Logging brokerReconfigurables.clear() } + /** + * Add reconfigurables to be notified when a dynamic broker config is updated. + * + * `Reconfigurable` is the public API used by configurable plugins like metrics reporter + * and quota callbacks. These are reconfigured before `KafkaConfig` is updated so that + * the update can be aborted if `reconfigure()` fails with an exception. + * + * `BrokerReconfigurable` is used for internal reconfigurable classes. These are + * reconfigured after `KafkaConfig` is updated so that they can access `KafkaConfig` + * directly. They are provided both old and new configs. + */ def addReconfigurables(kafkaServer: KafkaServer): Unit = { + addReconfigurable(new DynamicMetricsReporters(kafkaConfig.brokerId, kafkaServer)) + addReconfigurable(new DynamicClientQuotaCallback(kafkaConfig.brokerId, kafkaServer)) + addBrokerReconfigurable(new DynamicThreadPool(kafkaServer)) if (kafkaServer.logManager.cleaner != null) addBrokerReconfigurable(kafkaServer.logManager.cleaner) - addReconfigurable(new DynamicLogConfig(kafkaServer.logManager, kafkaServer)) - addReconfigurable(new DynamicMetricsReporters(kafkaConfig.brokerId, kafkaServer)) - addReconfigurable(new DynamicClientQuotaCallback(kafkaConfig.brokerId, kafkaServer)) + addBrokerReconfigurable(new DynamicLogConfig(kafkaServer.logManager, kafkaServer)) addBrokerReconfigurable(new DynamicListenerConfig(kafkaServer)) addBrokerReconfigurable(new DynamicConnectionQuota(kafkaServer)) } @@ -565,25 +577,24 @@ object DynamicLogConfig { val ReconfigurableConfigs = LogConfig.TopicConfigSynonyms.values.toSet -- ExcludedConfigs val KafkaConfigToLogConfigName = LogConfig.TopicConfigSynonyms.map { case (k, v) => (v, k) } } -class DynamicLogConfig(logManager: LogManager, server: KafkaServer) extends Reconfigurable with Logging { - override def configure(configs: util.Map[String, _]): Unit = {} +class DynamicLogConfig(logManager: LogManager, server: KafkaServer) extends BrokerReconfigurable with Logging { - override def reconfigurableConfigs(): util.Set[String] = { - DynamicLogConfig.ReconfigurableConfigs.asJava + override def reconfigurableConfigs: Set[String] = { + DynamicLogConfig.ReconfigurableConfigs } - override def validateReconfiguration(configs: util.Map[String, _]): Unit = { + override def validateReconfiguration(newConfig: KafkaConfig): Unit = { // For update of topic config overrides, only config names and types are validated // Names and types have already been validated. For consistency with topic config // validation, no additional validation is performed. } - override def reconfigure(configs: util.Map[String, _]): Unit = { + override def reconfigure(oldConfig: KafkaConfig, newConfig: KafkaConfig): Unit = { val currentLogConfig = logManager.currentDefaultConfig val origUncleanLeaderElectionEnable = logManager.currentDefaultConfig.uncleanLeaderElectionEnable val newBrokerDefaults = new util.HashMap[String, Object](currentLogConfig.originals) - configs.asScala.filterKeys(DynamicLogConfig.ReconfigurableConfigs.contains).foreach { case (k, v) => + newConfig.valuesFromThisConfig.asScala.filterKeys(DynamicLogConfig.ReconfigurableConfigs.contains).foreach { case (k, v) => if (v != null) { DynamicLogConfig.KafkaConfigToLogConfigName.get(k).foreach { configName => newBrokerDefaults.put(configName, v.asInstanceOf[AnyRef]) diff --git a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala index 798961e21f0ec..c28fcea15cb38 100644 --- a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala +++ b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala @@ -391,7 +391,9 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet // Verify that configs of existing logs have been updated val newLogConfig = LogConfig(KafkaServer.copyKafkaConfigToLog(servers.head.config)) - assertEquals(newLogConfig, servers.head.logManager.currentDefaultConfig) + TestUtils.waitUntilTrue(() => servers.head.logManager.currentDefaultConfig == newLogConfig, + "Config not updated in LogManager") + val log = servers.head.logManager.getLog(new TopicPartition(topic, 0)).getOrElse(throw new IllegalStateException("Log not found")) TestUtils.waitUntilTrue(() => log.config.segmentSize == 4000, "Existing topic config using defaults not updated") props.asScala.foreach { case (k, v) => From 9a384daf0eafb26e2e445c62659be78956c6e1c9 Mon Sep 17 00:00:00 2001 From: Stanislav Kozlovski Date: Tue, 12 Mar 2019 17:31:50 +0200 Subject: [PATCH 0012/1071] MINOR: Change Trogdor agent's cleanup executor to a cached thread pool (#6309) It is best to use a growing thread pool for worker cleanups. This lets us ensure that we close workers as fast as possible and not get slowed down on blocking cleanups. Reviewers: Colin McCabe , Jason Gustafson --- .../org/apache/kafka/trogdor/agent/WorkerManager.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/src/main/java/org/apache/kafka/trogdor/agent/WorkerManager.java b/tools/src/main/java/org/apache/kafka/trogdor/agent/WorkerManager.java index ba0c3b5c1b4a6..5ef9299476ca9 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/agent/WorkerManager.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/agent/WorkerManager.java @@ -43,6 +43,7 @@ import java.util.TreeMap; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; @@ -85,7 +86,7 @@ public final class WorkerManager { /** * An ExecutorService used to clean up TaskWorkers. */ - private final ScheduledExecutorService workerCleanupExecutor; + private final ExecutorService workerCleanupExecutor; /** * An ExecutorService to help with shutting down. @@ -161,7 +162,7 @@ synchronized void waitForQuiescence() throws InterruptedException { this.workers = new HashMap<>(); this.stateChangeExecutor = Executors.newSingleThreadScheduledExecutor( ThreadUtils.createThreadFactory("WorkerManagerStateThread", false)); - this.workerCleanupExecutor = Executors.newScheduledThreadPool(1, + this.workerCleanupExecutor = Executors.newCachedThreadPool( ThreadUtils.createThreadFactory("WorkerCleanupThread%d", false)); this.shutdownExecutor = Executors.newScheduledThreadPool(0, ThreadUtils.createThreadFactory("WorkerManagerShutdownThread%d", false)); @@ -289,13 +290,13 @@ void transitionToRunning() { Math.max(0, spec.endMs() - time.milliseconds())); } - void transitionToStopping() { + Future transitionToStopping() { state = State.STOPPING; if (timeoutFuture != null) { timeoutFuture.cancel(false); timeoutFuture = null; } - workerCleanupExecutor.submit(new HaltWorker(this)); + return workerCleanupExecutor.submit(new HaltWorker(this)); } void transitionToDone() { From ab7ea07f5e57ec405dc7fddce95de7c639a2fd6e Mon Sep 17 00:00:00 2001 From: "Matthias J. Sax" Date: Tue, 12 Mar 2019 09:28:14 -0700 Subject: [PATCH 0013/1071] KAFKA-3522: add missing guards for TimestampedXxxStore (#6356) Reviewers: John Roesler , Bill Bejeck --- .../internals/ProcessorContextImpl.java | 27 ++++- .../GlobalProcessorContextImplTest.java | 106 ++++++++++++++++-- .../internals/ProcessorContextImplTest.java | 95 ++++++++++++++++ 3 files changed, 215 insertions(+), 13 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextImpl.java index c10ea091e447f..5f32a3b7a7a8c 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextImpl.java @@ -32,6 +32,7 @@ import org.apache.kafka.streams.state.KeyValueIterator; import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.streams.state.SessionStore; +import org.apache.kafka.streams.state.TimestampedKeyValueStore; import org.apache.kafka.streams.state.TimestampedWindowStore; import org.apache.kafka.streams.state.ValueAndTimestamp; import org.apache.kafka.streams.state.WindowStore; @@ -84,7 +85,9 @@ public StateStore getStateStore(final String name) { final StateStore global = stateManager.getGlobalStore(name); if (global != null) { - if (global instanceof KeyValueStore) { + if (global instanceof TimestampedKeyValueStore) { + return new TimestampedKeyValueStoreReadOnlyDecorator((TimestampedKeyValueStore) global); + } else if (global instanceof KeyValueStore) { return new KeyValueStoreReadOnlyDecorator((KeyValueStore) global); } else if (global instanceof TimestampedWindowStore) { return new TimestampedWindowStoreReadOnlyDecorator((TimestampedWindowStore) global); @@ -108,7 +111,9 @@ public StateStore getStateStore(final String name) { } final StateStore store = stateManager.getStore(name); - if (store instanceof KeyValueStore) { + if (store instanceof TimestampedKeyValueStore) { + return new TimestampedKeyValueStoreReadWriteDecorator((TimestampedKeyValueStore) store); + } else if (store instanceof KeyValueStore) { return new KeyValueStoreReadWriteDecorator((KeyValueStore) store); } else if (store instanceof TimestampedWindowStore) { return new TimestampedWindowStoreReadWriteDecorator((TimestampedWindowStore) store); @@ -294,6 +299,15 @@ public V delete(final K key) { } } + private static class TimestampedKeyValueStoreReadOnlyDecorator + extends KeyValueStoreReadOnlyDecorator> + implements TimestampedKeyValueStore { + + private TimestampedKeyValueStoreReadOnlyDecorator(final TimestampedKeyValueStore inner) { + super(inner); + } + } + private static class WindowStoreReadOnlyDecorator extends StateStoreReadOnlyDecorator, K, V> implements WindowStore { @@ -484,6 +498,15 @@ public V delete(final K key) { } } + private static class TimestampedKeyValueStoreReadWriteDecorator + extends KeyValueStoreReadWriteDecorator> + implements TimestampedKeyValueStore { + + private TimestampedKeyValueStoreReadWriteDecorator(final TimestampedKeyValueStore inner) { + super(inner); + } + } + static class WindowStoreReadWriteDecorator extends StateStoreReadWriteDecorator, K, V> implements WindowStore { diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalProcessorContextImplTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalProcessorContextImplTest.java index 4153ccacbe378..b36557cda22fd 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalProcessorContextImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalProcessorContextImplTest.java @@ -21,6 +21,10 @@ import org.apache.kafka.streams.processor.StateStore; import org.apache.kafka.streams.processor.To; import org.apache.kafka.streams.state.KeyValueStore; +import org.apache.kafka.streams.state.SessionStore; +import org.apache.kafka.streams.state.TimestampedKeyValueStore; +import org.apache.kafka.streams.state.TimestampedWindowStore; +import org.apache.kafka.streams.state.WindowStore; import org.hamcrest.core.IsInstanceOf; import org.junit.Before; import org.junit.Test; @@ -39,6 +43,11 @@ public class GlobalProcessorContextImplTest { private static final String GLOBAL_STORE_NAME = "global-store"; + private static final String GLOBAL_KEY_VALUE_STORE_NAME = "global-key-value-store"; + private static final String GLOBAL_TIMESTAMPED_KEY_VALUE_STORE_NAME = "global-timestamped-key-value-store"; + private static final String GLOBAL_WINDOW_STORE_NAME = "global-window-store"; + private static final String GLOBAL_TIMESTAMPED_WINDOW_STORE_NAME = "global-timestamped-window-store"; + private static final String GLOBAL_SESSION_STORE_NAME = "global-session-store"; private static final String UNKNOWN_STORE = "unknown-store"; private static final String CHILD_PROCESSOR = "child"; @@ -56,7 +65,12 @@ public void setup() { replay(streamsConfig); final StateManager stateManager = mock(StateManager.class); - expect(stateManager.getGlobalStore(GLOBAL_STORE_NAME)).andReturn(mock(KeyValueStore.class)); + expect(stateManager.getGlobalStore(GLOBAL_STORE_NAME)).andReturn(mock(StateStore.class)); + expect(stateManager.getGlobalStore(GLOBAL_KEY_VALUE_STORE_NAME)).andReturn(mock(KeyValueStore.class)); + expect(stateManager.getGlobalStore(GLOBAL_TIMESTAMPED_KEY_VALUE_STORE_NAME)).andReturn(mock(TimestampedKeyValueStore.class)); + expect(stateManager.getGlobalStore(GLOBAL_WINDOW_STORE_NAME)).andReturn(mock(WindowStore.class)); + expect(stateManager.getGlobalStore(GLOBAL_TIMESTAMPED_WINDOW_STORE_NAME)).andReturn(mock(TimestampedWindowStore.class)); + expect(stateManager.getGlobalStore(GLOBAL_SESSION_STORE_NAME)).andReturn(mock(SessionStore.class)); expect(stateManager.getGlobalStore(UNKNOWN_STORE)).andReturn(null); replay(stateManager); @@ -86,7 +100,7 @@ public void setup() { @Test public void shouldReturnGlobalOrNullStore() { - assertThat(globalContext.getStateStore(GLOBAL_STORE_NAME), new IsInstanceOf(KeyValueStore.class)); + assertThat(globalContext.getStateStore(GLOBAL_STORE_NAME), new IsInstanceOf(StateStore.class)); assertNull(globalContext.getStateStore(UNKNOWN_STORE)); } @@ -135,22 +149,92 @@ public void shouldNotAllowToSchedulePunctuations() { } @Test - public void shouldNotAllowInit() { - final StateStore store = globalContext.getStateStore(GLOBAL_STORE_NAME); + public void shouldNotAllowInitForKeyValueStore() { + final StateStore store = globalContext.getStateStore(GLOBAL_KEY_VALUE_STORE_NAME); try { store.init(null, null); fail("Should have thrown UnsupportedOperationException."); - } catch (final UnsupportedOperationException expected) { - } + } catch (final UnsupportedOperationException expected) { } } @Test - public void shouldNotAllowClose() { - final StateStore store = globalContext.getStateStore(GLOBAL_STORE_NAME); + public void shouldNotAllowInitForTimestampedKeyValueStore() { + final StateStore store = globalContext.getStateStore(GLOBAL_TIMESTAMPED_KEY_VALUE_STORE_NAME); + try { + store.init(null, null); + fail("Should have thrown UnsupportedOperationException."); + } catch (final UnsupportedOperationException expected) { } + } + + @Test + public void shouldNotAllowInitForWindowStore() { + final StateStore store = globalContext.getStateStore(GLOBAL_WINDOW_STORE_NAME); + try { + store.init(null, null); + fail("Should have thrown UnsupportedOperationException."); + } catch (final UnsupportedOperationException expected) { } + } + + @Test + public void shouldNotAllowInitForTimestampedWindowStore() { + final StateStore store = globalContext.getStateStore(GLOBAL_TIMESTAMPED_WINDOW_STORE_NAME); + try { + store.init(null, null); + fail("Should have thrown UnsupportedOperationException."); + } catch (final UnsupportedOperationException expected) { } + } + + @Test + public void shouldNotAllowInitForSessionStore() { + final StateStore store = globalContext.getStateStore(GLOBAL_SESSION_STORE_NAME); + try { + store.init(null, null); + fail("Should have thrown UnsupportedOperationException."); + } catch (final UnsupportedOperationException expected) { } + } + + @Test + public void shouldNotAllowCloseForKeyValueStore() { + final StateStore store = globalContext.getStateStore(GLOBAL_KEY_VALUE_STORE_NAME); + try { + store.close(); + fail("Should have thrown UnsupportedOperationException."); + } catch (final UnsupportedOperationException expected) { } + } + + @Test + public void shouldNotAllowCloseForTimestampedKeyValueStore() { + final StateStore store = globalContext.getStateStore(GLOBAL_TIMESTAMPED_KEY_VALUE_STORE_NAME); + try { + store.close(); + fail("Should have thrown UnsupportedOperationException."); + } catch (final UnsupportedOperationException expected) { } + } + + @Test + public void shouldNotAllowCloseForWindowStore() { + final StateStore store = globalContext.getStateStore(GLOBAL_WINDOW_STORE_NAME); + try { + store.close(); + fail("Should have thrown UnsupportedOperationException."); + } catch (final UnsupportedOperationException expected) { } + } + + @Test + public void shouldNotAllowCloseForTimestampedWindowStore() { + final StateStore store = globalContext.getStateStore(GLOBAL_TIMESTAMPED_WINDOW_STORE_NAME); + try { + store.close(); + fail("Should have thrown UnsupportedOperationException."); + } catch (final UnsupportedOperationException expected) { } + } + + @Test + public void shouldNotAllowCloseForSessionStore() { + final StateStore store = globalContext.getStateStore(GLOBAL_SESSION_STORE_NAME); try { store.close(); fail("Should have thrown UnsupportedOperationException."); - } catch (final UnsupportedOperationException expected) { - } + } catch (final UnsupportedOperationException expected) { } } -} +} \ No newline at end of file diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorContextImplTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorContextImplTest.java index 9b36ec7c8e8d4..fe4d948803b04 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorContextImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorContextImplTest.java @@ -27,6 +27,7 @@ import org.apache.kafka.streams.state.KeyValueIterator; import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.streams.state.SessionStore; +import org.apache.kafka.streams.state.TimestampedKeyValueStore; import org.apache.kafka.streams.state.TimestampedWindowStore; import org.apache.kafka.streams.state.ValueAndTimestamp; import org.apache.kafka.streams.state.WindowStore; @@ -70,7 +71,9 @@ public class ProcessorContextImplTest { private boolean removeExecuted; private KeyValueIterator rangeIter; + private KeyValueIterator> timestampedRangeIter; private KeyValueIterator allIter; + private KeyValueIterator> timestampedAllIter; private final List, Long>> iters = new ArrayList<>(7); private final List, ValueAndTimestamp>> timestampedIters = new ArrayList<>(7); @@ -86,7 +89,9 @@ public void setup() { removeExecuted = false; rangeIter = mock(KeyValueIterator.class); + timestampedRangeIter = mock(KeyValueIterator.class); allIter = mock(KeyValueIterator.class); + timestampedAllIter = mock(KeyValueIterator.class); windowStoreIter = mock(WindowStoreIterator.class); for (int i = 0; i < 7; i++) { @@ -103,12 +108,14 @@ public void setup() { final ProcessorStateManager stateManager = mock(ProcessorStateManager.class); expect(stateManager.getGlobalStore("GlobalKeyValueStore")).andReturn(keyValueStoreMock()); + expect(stateManager.getGlobalStore("GlobalTimestampedKeyValueStore")).andReturn(timestampedKeyValueStoreMock()); expect(stateManager.getGlobalStore("GlobalWindowStore")).andReturn(windowStoreMock()); expect(stateManager.getGlobalStore("GlobalTimestampedWindowStore")).andReturn(timestampedWindowStoreMock()); expect(stateManager.getGlobalStore("GlobalSessionStore")).andReturn(sessionStoreMock()); expect(stateManager.getGlobalStore(anyString())).andReturn(null); expect(stateManager.getStore("LocalKeyValueStore")).andReturn(keyValueStoreMock()); + expect(stateManager.getStore("LocalTimestampedKeyValueStore")).andReturn(timestampedKeyValueStoreMock()); expect(stateManager.getStore("LocalWindowStore")).andReturn(windowStoreMock()); expect(stateManager.getStore("LocalTimestampedWindowStore")).andReturn(timestampedWindowStoreMock()); expect(stateManager.getStore("LocalSessionStore")).andReturn(sessionStoreMock()); @@ -128,6 +135,7 @@ public void setup() { context.setCurrentNode(new ProcessorNode("fake", null, new HashSet<>(asList( "LocalKeyValueStore", + "LocalTimestampedKeyValueStore", "LocalWindowStore", "LocalTimestampedWindowStore", "LocalSessionStore")))); @@ -151,6 +159,24 @@ public void globalKeyValueStoreShouldBeReadOnly() { }); } + @Test + public void globalTimestampedKeyValueStoreShouldBeReadOnly() { + doTest("GlobalTimestampedKeyValueStore", (Consumer>) store -> { + verifyStoreCannotBeInitializedOrClosed(store); + + checkThrowsUnsupportedOperation(store::flush, "flush()"); + checkThrowsUnsupportedOperation(() -> store.put("1", ValueAndTimestamp.make(1L, 2L)), "put()"); + checkThrowsUnsupportedOperation(() -> store.putIfAbsent("1", ValueAndTimestamp.make(1L, 2L)), "putIfAbsent()"); + checkThrowsUnsupportedOperation(() -> store.putAll(Collections.emptyList()), "putAll()"); + checkThrowsUnsupportedOperation(() -> store.delete("1"), "delete()"); + + assertEquals(VALUE_AND_TIMESTAMP, store.get(KEY)); + assertEquals(timestampedRangeIter, store.range("one", "two")); + assertEquals(timestampedAllIter, store.all()); + assertEquals(VALUE, store.approximateNumEntries()); + }); + } + @Test public void globalWindowStoreShouldBeReadOnly() { doTest("GlobalWindowStore", (Consumer>) store -> { @@ -228,6 +254,33 @@ public void localKeyValueStoreShouldNotAllowInitOrClose() { }); } + @Test + public void localTimestampedKeyValueStoreShouldNotAllowInitOrClose() { + doTest("LocalTimestampedKeyValueStore", (Consumer>) store -> { + verifyStoreCannotBeInitializedOrClosed(store); + + store.flush(); + assertTrue(flushExecuted); + + store.put("1", ValueAndTimestamp.make(1L, 2L)); + assertTrue(putExecuted); + + store.putIfAbsent("1", ValueAndTimestamp.make(1L, 2L)); + assertTrue(putIfAbsentExecuted); + + store.putAll(Collections.emptyList()); + assertTrue(putAllExecuted); + + store.delete("1"); + assertTrue(deleteExecuted); + + assertEquals(VALUE_AND_TIMESTAMP, store.get(KEY)); + assertEquals(timestampedRangeIter, store.range("one", "two")); + assertEquals(timestampedAllIter, store.all()); + assertEquals(VALUE, store.approximateNumEntries()); + }); + } + @Test public void localWindowStoreShouldNotAllowInitOrClose() { doTest("LocalWindowStore", (Consumer>) store -> { @@ -332,6 +385,48 @@ private KeyValueStore keyValueStoreMock() { return keyValueStoreMock; } + @SuppressWarnings("unchecked") + private TimestampedKeyValueStore timestampedKeyValueStoreMock() { + final TimestampedKeyValueStore timestampedKeyValueStoreMock = mock(TimestampedKeyValueStore.class); + + initStateStoreMock(timestampedKeyValueStoreMock); + + expect(timestampedKeyValueStoreMock.get(KEY)).andReturn(VALUE_AND_TIMESTAMP); + expect(timestampedKeyValueStoreMock.approximateNumEntries()).andReturn(VALUE); + + expect(timestampedKeyValueStoreMock.range("one", "two")).andReturn(timestampedRangeIter); + expect(timestampedKeyValueStoreMock.all()).andReturn(timestampedAllIter); + + + timestampedKeyValueStoreMock.put(anyString(), anyObject(ValueAndTimestamp.class)); + expectLastCall().andAnswer(() -> { + putExecuted = true; + return null; + }); + + timestampedKeyValueStoreMock.putIfAbsent(anyString(), anyObject(ValueAndTimestamp.class)); + expectLastCall().andAnswer(() -> { + putIfAbsentExecuted = true; + return null; + }); + + timestampedKeyValueStoreMock.putAll(anyObject(List.class)); + expectLastCall().andAnswer(() -> { + putAllExecuted = true; + return null; + }); + + timestampedKeyValueStoreMock.delete(anyString()); + expectLastCall().andAnswer(() -> { + deleteExecuted = true; + return null; + }); + + replay(timestampedKeyValueStoreMock); + + return timestampedKeyValueStoreMock; + } + @SuppressWarnings("unchecked") private WindowStore windowStoreMock() { final WindowStore windowStore = mock(WindowStore.class); From 8e975400711b0ea64bf4a00c8c551e448ab48416 Mon Sep 17 00:00:00 2001 From: John Roesler Date: Tue, 12 Mar 2019 11:53:29 -0500 Subject: [PATCH 0014/1071] KAFKA-7944: Improve Suppress test coverage (#6382) * add a normal windowed suppress with short windows and a short grace period * improve the smoke test so that it actually verifies the intended conditions See https://issues.apache.org/jira/browse/KAFKA-7944 Reviewers: Bill Bejeck , Guozhang Wang --- .../SmokeTestDriverIntegrationTest.java | 7 +- .../SuppressionDurabilityIntegrationTest.java | 14 +- .../kafka/streams/tests/SmokeTestClient.java | 35 +-- .../kafka/streams/tests/SmokeTestDriver.java | 212 ++++++++++-------- .../kafka/streams/tests/StreamsSmokeTest.java | 21 +- tests/kafkatest/services/streams.py | 4 + .../tests/streams/streams_bounce_test.py | 75 ------- .../tests/streams/streams_smoke_test.py | 104 ++++++--- 8 files changed, 251 insertions(+), 221 deletions(-) delete mode 100644 tests/kafkatest/tests/streams/streams_bounce_test.py diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/SmokeTestDriverIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/SmokeTestDriverIntegrationTest.java index 82f86c28b1dfb..7b896ecf847e2 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/SmokeTestDriverIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/SmokeTestDriverIntegrationTest.java @@ -18,12 +18,14 @@ import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster; +import org.apache.kafka.streams.integration.utils.IntegrationTestUtils; import org.apache.kafka.streams.tests.SmokeTestClient; import org.apache.kafka.streams.tests.SmokeTestDriver; import org.junit.Assert; import org.junit.ClassRule; import org.junit.Test; +import java.time.Duration; import java.util.ArrayList; import java.util.Map; import java.util.Properties; @@ -53,7 +55,8 @@ private Driver(final String bootstrapServers, final int numKeys, final int maxRe @Override public void run() { try { - final Map> allData = generate(bootstrapServers, numKeys, maxRecordsPerKey, true); + final Map> allData = + generate(bootstrapServers, numKeys, maxRecordsPerKey, Duration.ofSeconds(20)); result = verify(bootstrapServers, allData, maxRecordsPerKey); } catch (final Exception ex) { @@ -76,7 +79,7 @@ public void shouldWorkWithRebalance() throws InterruptedException { int numClientsCreated = 0; final ArrayList clients = new ArrayList<>(); - CLUSTER.createTopics(SmokeTestDriver.topics()); + IntegrationTestUtils.cleanStateBeforeTest(CLUSTER, SmokeTestDriver.topics()); final String bootstrapServers = CLUSTER.bootstrapServers(); final Driver driver = new Driver(bootstrapServers, 10, 1000); diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/SuppressionDurabilityIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/SuppressionDurabilityIntegrationTest.java index 6f759bc82fb2f..3bb1131c1637c 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/SuppressionDurabilityIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/SuppressionDurabilityIntegrationTest.java @@ -49,7 +49,6 @@ import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; -import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Locale; @@ -88,13 +87,16 @@ public class SuppressionDurabilityIntegrationTest { private static final int COMMIT_INTERVAL = 100; private final boolean eosEnabled; - public SuppressionDurabilityIntegrationTest(final boolean eosEnabled) { - this.eosEnabled = eosEnabled; - } - @Parameters(name = "{index}: eosEnabled={0}") public static Collection parameters() { - return Arrays.asList(new Object[] {false}, new Object[] {true}); + return asList( + new Object[] {false}, + new Object[] {true} + ); + } + + public SuppressionDurabilityIntegrationTest(final boolean eosEnabled) { + this.eosEnabled = eosEnabled; } private KTable buildCountsTable(final String input, final StreamsBuilder builder) { diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestClient.java b/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestClient.java index 3b76fcab1807a..b81c0a0b58508 100644 --- a/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestClient.java +++ b/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestClient.java @@ -32,7 +32,7 @@ import org.apache.kafka.streams.kstream.KTable; import org.apache.kafka.streams.kstream.Materialized; import org.apache.kafka.streams.kstream.Produced; -import org.apache.kafka.streams.kstream.Suppressed; +import org.apache.kafka.streams.kstream.Suppressed.BufferConfig; import org.apache.kafka.streams.kstream.TimeWindows; import org.apache.kafka.streams.kstream.Windowed; import org.apache.kafka.streams.state.Stores; @@ -40,8 +40,11 @@ import org.apache.kafka.test.TestUtils; import java.time.Duration; +import java.time.Instant; import java.util.Properties; +import static org.apache.kafka.streams.kstream.Suppressed.untilWindowCloses; + public class SmokeTestClient extends SmokeTestUtil { private final String name; @@ -113,7 +116,7 @@ private KafkaStreams createKafkaStreams(final Properties props) { final Topology build = getTopology(); final KafkaStreams streamsClient = new KafkaStreams(build, getStreamsConfig(props)); streamsClient.setStateListener((newState, oldState) -> { - System.out.printf("%s: %s -> %s%n", name, oldState, newState); + System.out.printf("%s %s: %s -> %s%n", name, Instant.now(), oldState, newState); if (oldState == KafkaStreams.State.REBALANCING && newState == KafkaStreams.State.RUNNING) { started = true; } @@ -149,24 +152,22 @@ public Topology getTopology() { .withRetention(Duration.ofHours(25)) ); - minAggregation - .toStream() - .filterNot((k, v) -> k.key().equals("flush")) - .map((key, value) -> new KeyValue<>(key.toString(), value)) - .to("min-raw", Produced.with(stringSerde, intSerde)); + streamify(minAggregation, "min-raw"); - minAggregation - .suppress(Suppressed.untilWindowCloses(Suppressed.BufferConfig.unbounded())) - .toStream() - .filterNot((k, v) -> k.key().equals("flush")) - .map((key, value) -> new KeyValue<>(key.toString(), value)) - .to("min-suppressed", Produced.with(stringSerde, intSerde)); + streamify(minAggregation.suppress(untilWindowCloses(BufferConfig.unbounded())), "min-suppressed"); minAggregation .toStream(new Unwindow<>()) .filterNot((k, v) -> k.equals("flush")) .to("min", Produced.with(stringSerde, intSerde)); + final KTable, Integer> smallWindowSum = groupedData + .windowedBy(TimeWindows.of(Duration.ofSeconds(2)).advanceBy(Duration.ofSeconds(1)).grace(Duration.ofSeconds(30))) + .reduce((l, r) -> l + r); + + streamify(smallWindowSum, "sws-raw"); + streamify(smallWindowSum.suppress(untilWindowCloses(BufferConfig.unbounded())), "sws-suppressed"); + final KTable minTable = builder.table( "min", Consumed.with(stringSerde, intSerde), @@ -250,4 +251,12 @@ public Topology getTopology() { return builder.build(); } + + private static void streamify(final KTable, Integer> windowedTable, final String topic) { + windowedTable + .toStream() + .filterNot((k, v) -> k.key().equals("flush")) + .map((key, value) -> new KeyValue<>(key.toString(), value)) + .to(topic, Produced.with(stringSerde, intSerde)); + } } diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestDriver.java b/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestDriver.java index b7f9d7f5e06b7..98e6e8fc97824 100644 --- a/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestDriver.java +++ b/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestDriver.java @@ -60,13 +60,14 @@ import static org.apache.kafka.common.utils.Utils.mkEntry; public class SmokeTestDriver extends SmokeTestUtil { - private static final String[] TOPICS = new String[] { + private static final String[] TOPICS = { "data", "echo", "max", "min", "min-suppressed", "min-raw", "dif", "sum", + "sws-raw", "sws-suppressed", "cnt", "avg", "tagg" @@ -80,18 +81,18 @@ private static class ValueList { private int index; ValueList(final int min, final int max) { - this.key = min + "-" + max; + key = min + "-" + max; - this.values = new int[max - min + 1]; - for (int i = 0; i < this.values.length; i++) { - this.values[i] = min + i; + values = new int[max - min + 1]; + for (int i = 0; i < values.length; i++) { + values[i] = min + i; } // We want to randomize the order of data to test not completely predictable processing order // However, values are also use as a timestamp of the record. (TODO: separate data and timestamp) // We keep some correlation of time and order. Thus, the shuffling is done with a sliding window - shuffle(this.values, 10); + shuffle(values, 10); - this.index = 0; + index = 0; } int next() { @@ -103,45 +104,25 @@ public static String[] topics() { return Arrays.copyOf(TOPICS, TOPICS.length); } - public static Map> generate(final String kafka, - final int numKeys, - final int maxRecordsPerKey, - final boolean autoTerminate) { - final Properties producerProps = new Properties(); - producerProps.put(ProducerConfig.CLIENT_ID_CONFIG, "SmokeTest"); - producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka); - producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); - producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); - producerProps.put(ProducerConfig.ACKS_CONFIG, "all"); - - final KafkaProducer producer = new KafkaProducer<>(producerProps); + static void generatePerpetually(final String kafka, + final int numKeys, + final int maxRecordsPerKey) { + final Properties producerProps = generatorProperties(kafka); int numRecordsProduced = 0; - final Map> allData = new HashMap<>(); final ValueList[] data = new ValueList[numKeys]; for (int i = 0; i < numKeys; i++) { data[i] = new ValueList(i, i + maxRecordsPerKey - 1); - allData.put(data[i].key, new HashSet<>()); - } - final Random rand = new Random(); - - int remaining = 1; // dummy value must be positive if is false - if (autoTerminate) { - remaining = data.length; } - List> needRetry = new ArrayList<>(); - - while (remaining > 0) { - final int index = autoTerminate ? rand.nextInt(remaining) : rand.nextInt(numKeys); - final String key = data[index].key; - final int value = data[index].next(); + final Random rand = new Random(); - if (autoTerminate && value < 0) { - remaining--; - data[index] = data[remaining]; - } else { + try (final KafkaProducer producer = new KafkaProducer<>(producerProps)) { + while (true) { + final int index = rand.nextInt(numKeys); + final String key = data[index].key; + final int value = data[index].next(); final ProducerRecord record = new ProducerRecord<>( @@ -150,51 +131,112 @@ public static Map> generate(final String kafka, intSerde.serializer().serialize("", value) ); - producer.send(record, new TestCallback(record, needRetry)); + producer.send(record); numRecordsProduced++; - allData.get(key).add(value); if (numRecordsProduced % 100 == 0) { - System.out.println(numRecordsProduced + " records produced"); + System.out.println(Instant.now() + " " + numRecordsProduced + " records produced"); } Utils.sleep(2); } } - producer.flush(); - - int remainingRetries = 5; - while (!needRetry.isEmpty()) { - final List> needRetry2 = new ArrayList<>(); - for (final ProducerRecord record : needRetry) { - System.out.println("retry producing " + stringSerde.deserializer().deserialize("", record.key())); - producer.send(record, new TestCallback(record, needRetry2)); + } + + public static Map> generate(final String kafka, + final int numKeys, + final int maxRecordsPerKey, + final Duration timeToSpend) { + final Properties producerProps = generatorProperties(kafka); + + + int numRecordsProduced = 0; + + final Map> allData = new HashMap<>(); + final ValueList[] data = new ValueList[numKeys]; + for (int i = 0; i < numKeys; i++) { + data[i] = new ValueList(i, i + maxRecordsPerKey - 1); + allData.put(data[i].key, new HashSet<>()); + } + final Random rand = new Random(); + + int remaining = data.length; + + final long recordPauseTime = timeToSpend.toMillis() / numKeys / maxRecordsPerKey; + + List> needRetry = new ArrayList<>(); + + try (final KafkaProducer producer = new KafkaProducer<>(producerProps)) { + while (remaining > 0) { + final int index = rand.nextInt(remaining); + final String key = data[index].key; + final int value = data[index].next(); + + if (value < 0) { + remaining--; + data[index] = data[remaining]; + } else { + + final ProducerRecord record = + new ProducerRecord<>( + "data", + stringSerde.serializer().serialize("", key), + intSerde.serializer().serialize("", value) + ); + + producer.send(record, new TestCallback(record, needRetry)); + + numRecordsProduced++; + allData.get(key).add(value); + if (numRecordsProduced % 100 == 0) { + System.out.println(Instant.now() + " " + numRecordsProduced + " records produced"); + } + Utils.sleep(Math.max(recordPauseTime, 2)); + } } producer.flush(); - needRetry = needRetry2; - if (--remainingRetries == 0 && !needRetry.isEmpty()) { - System.err.println("Failed to produce all records after multiple retries"); - Exit.exit(1); + int remainingRetries = 5; + while (!needRetry.isEmpty()) { + final List> needRetry2 = new ArrayList<>(); + for (final ProducerRecord record : needRetry) { + System.out.println("retry producing " + stringSerde.deserializer().deserialize("", record.key())); + producer.send(record, new TestCallback(record, needRetry2)); + } + producer.flush(); + needRetry = needRetry2; + + if (--remainingRetries == 0 && !needRetry.isEmpty()) { + System.err.println("Failed to produce all records after multiple retries"); + Exit.exit(1); + } } - } - // now that we've sent everything, we'll send some final records with a timestamp high enough to flush out - // all suppressed records. - final List partitions = producer.partitionsFor("data"); - for (final PartitionInfo partition : partitions) { - producer.send(new ProducerRecord<>( - partition.topic(), - partition.partition(), - System.currentTimeMillis() + Duration.ofDays(2).toMillis(), - stringSerde.serializer().serialize("", "flush"), - intSerde.serializer().serialize("", 0) - )); + // now that we've sent everything, we'll send some final records with a timestamp high enough to flush out + // all suppressed records. + final List partitions = producer.partitionsFor("data"); + for (final PartitionInfo partition : partitions) { + producer.send(new ProducerRecord<>( + partition.topic(), + partition.partition(), + System.currentTimeMillis() + Duration.ofDays(2).toMillis(), + stringSerde.serializer().serialize("", "flush"), + intSerde.serializer().serialize("", 0) + )); + } } - - producer.close(); return Collections.unmodifiableMap(allData); } + private static Properties generatorProperties(final String kafka) { + final Properties producerProps = new Properties(); + producerProps.put(ProducerConfig.CLIENT_ID_CONFIG, "SmokeTest"); + producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka); + producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + producerProps.put(ProducerConfig.ACKS_CONFIG, "all"); + return producerProps; + } + private static class TestCallback implements Callback { private final ProducerRecord originalRecord; private final List> needRetry; @@ -232,12 +274,6 @@ private static void shuffle(final int[] data, @SuppressWarnings("SameParameterVa } public static class NumberDeserializer implements Deserializer { - - @Override - public void configure(final Map configs, final boolean isKey) { - - } - @Override public Number deserialize(final String topic, final byte[] data) { final Number value; @@ -247,6 +283,8 @@ public Number deserialize(final String topic, final byte[] data) { case "min": case "min-raw": case "min-suppressed": + case "sws-raw": + case "sws-suppressed": case "max": case "dif": value = intSerde.deserializer().deserialize(topic, data); @@ -264,11 +302,6 @@ public Number deserialize(final String topic, final byte[] data) { } return value; } - - @Override - public void close() { - - } } public static VerificationResult verify(final String kafka, @@ -279,6 +312,7 @@ public static VerificationResult verify(final String kafka, props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, NumberDeserializer.class); + props.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed"); final KafkaConsumer consumer = new KafkaConsumer<>(props); final List partitions = getAllPartitions(consumer, TOPICS); @@ -406,7 +440,12 @@ private static VerificationResult verifyAll(final Map> inpu boolean pass; try (final PrintStream resultStream = new PrintStream(byteArrayOutputStream)) { pass = verifyTAgg(resultStream, inputs, events.get("tagg")); - pass &= verifySuppressed(resultStream, "min-suppressed", inputs, events, SmokeTestDriver::getMin); + pass &= verifySuppressed(resultStream, "min-suppressed", events); + pass &= verify(resultStream, "min-suppressed", inputs, events, windowedKey -> { + final String unwindowedKey = windowedKey.substring(1, windowedKey.length() - 1).replaceAll("@.*", ""); + return getMin(unwindowedKey); + }); + pass &= verifySuppressed(resultStream, "sws-suppressed", events); pass &= verify(resultStream, "min", inputs, events, SmokeTestDriver::getMin); pass &= verify(resultStream, "max", inputs, events, SmokeTestDriver::getMax); pass &= verify(resultStream, "dif", inputs, events, key -> getMax(key).intValue() - getMin(key).intValue()); @@ -457,9 +496,7 @@ private static boolean verify(final PrintStream resultStream, private static boolean verifySuppressed(final PrintStream resultStream, @SuppressWarnings("SameParameterValue") final String topic, - final Map> inputs, - final Map>>> events, - final Function getMin) { + final Map>>> events) { resultStream.println("verifying suppressed " + topic); final Map>> topicEvents = events.getOrDefault(topic, emptyMap()); for (final Map.Entry>> entry : topicEvents.entrySet()) { @@ -476,14 +513,11 @@ private static boolean verifySuppressed(final PrintStream resultStream, return false; } } - return verify(resultStream, topic, inputs, events, windowedKey -> { - final String unwindowedKey = windowedKey.substring(1, windowedKey.length() - 1).replaceAll("@.*", ""); - return getMin.apply(unwindowedKey); - }); + return true; } private static String indent(@SuppressWarnings("SameParameterValue") final String prefix, - final LinkedList> list) { + final Iterable> list) { final StringBuilder stringBuilder = new StringBuilder(); for (final ConsumerRecord record : list) { stringBuilder.append(prefix).append(record).append('\n'); @@ -494,13 +528,13 @@ private static String indent(@SuppressWarnings("SameParameterValue") final Strin private static Long getSum(final String key) { final int min = getMin(key).intValue(); final int max = getMax(key).intValue(); - return ((long) min + (long) max) * (max - min + 1L) / 2L; + return ((long) min + max) * (max - min + 1L) / 2L; } private static Double getAvg(final String key) { final int min = getMin(key).intValue(); final int max = getMax(key).intValue(); - return ((long) min + (long) max) / 2.0; + return ((long) min + max) / 2.0; } @@ -554,7 +588,7 @@ private static Number getMax(final String key) { } private static List getAllPartitions(final KafkaConsumer consumer, final String... topics) { - final ArrayList partitions = new ArrayList<>(); + final List partitions = new ArrayList<>(); for (final String topic : topics) { for (final PartitionInfo info : consumer.partitionsFor(topic)) { diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsSmokeTest.java b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsSmokeTest.java index 4c2f6d20d8459..00de2669983b7 100644 --- a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsSmokeTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsSmokeTest.java @@ -20,11 +20,15 @@ import org.apache.kafka.streams.StreamsConfig; import java.io.IOException; +import java.time.Duration; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.UUID; +import static org.apache.kafka.streams.tests.SmokeTestDriver.generate; +import static org.apache.kafka.streams.tests.SmokeTestDriver.generatePerpetually; + public class StreamsSmokeTest { /** @@ -62,16 +66,23 @@ public static void main(final String[] args) throws InterruptedException, IOExce final int numKeys = 10; final int maxRecordsPerKey = 500; if (disableAutoTerminate) { - SmokeTestDriver.generate(kafka, numKeys, maxRecordsPerKey, false); + generatePerpetually(kafka, numKeys, maxRecordsPerKey); } else { - final Map> allData = SmokeTestDriver.generate(kafka, numKeys, maxRecordsPerKey, true); + // slow down data production to span 30 seconds so that system tests have time to + // do their bounces, etc. + final Map> allData = + generate(kafka, numKeys, maxRecordsPerKey, Duration.ofSeconds(30)); SmokeTestDriver.verify(kafka, allData, maxRecordsPerKey); } break; case "process": - // this starts a KafkaStreams client - final SmokeTestClient client = new SmokeTestClient(UUID.randomUUID().toString()); - client.start(streamsProperties); + // this starts the stream processing app + new SmokeTestClient(UUID.randomUUID().toString()).start(streamsProperties); + break; + case "process-eos": + // this starts the stream processing app with EOS + streamsProperties.setProperty(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE); + new SmokeTestClient(UUID.randomUUID().toString()).start(streamsProperties); break; case "close-deadlock-test": final ShutdownDeadlockTest test = new ShutdownDeadlockTest(kafka); diff --git a/tests/kafkatest/services/streams.py b/tests/kafkatest/services/streams.py index 771b67f511a8f..b606267009d14 100644 --- a/tests/kafkatest/services/streams.py +++ b/tests/kafkatest/services/streams.py @@ -353,6 +353,10 @@ class StreamsSmokeTestJobRunnerService(StreamsSmokeTestBaseService): def __init__(self, test_context, kafka): super(StreamsSmokeTestJobRunnerService, self).__init__(test_context, kafka, "process") +class StreamsSmokeTestEOSJobRunnerService(StreamsSmokeTestBaseService): + def __init__(self, test_context, kafka): + super(StreamsSmokeTestEOSJobRunnerService, self).__init__(test_context, kafka, "process-eos") + class StreamsEosTestDriverService(StreamsEosTestBaseService): def __init__(self, test_context, kafka): diff --git a/tests/kafkatest/tests/streams/streams_bounce_test.py b/tests/kafkatest/tests/streams/streams_bounce_test.py deleted file mode 100644 index 7ac79396381b0..0000000000000 --- a/tests/kafkatest/tests/streams/streams_bounce_test.py +++ /dev/null @@ -1,75 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ducktape.mark.resource import cluster - -from kafkatest.tests.kafka_test import KafkaTest -from kafkatest.services.streams import StreamsSmokeTestDriverService, StreamsSmokeTestJobRunnerService -import time - - -class StreamsBounceTest(KafkaTest): - """ - Simple test of Kafka Streams. - """ - - def __init__(self, test_context): - super(StreamsBounceTest, self).__init__(test_context, num_zk=1, num_brokers=3, topics={ - 'echo' : { 'partitions': 5, 'replication-factor': 2 }, - 'data' : { 'partitions': 5, 'replication-factor': 2 }, - 'min' : { 'partitions': 5, 'replication-factor': 2 }, - 'max' : { 'partitions': 5, 'replication-factor': 2 }, - 'sum' : { 'partitions': 5, 'replication-factor': 2 }, - 'dif' : { 'partitions': 5, 'replication-factor': 2 }, - 'cnt' : { 'partitions': 5, 'replication-factor': 2 }, - 'avg' : { 'partitions': 5, 'replication-factor': 2 }, - 'wcnt' : { 'partitions': 5, 'replication-factor': 2 }, - 'tagg' : { 'partitions': 5, 'replication-factor': 2 } - }) - - self.driver = StreamsSmokeTestDriverService(test_context, self.kafka) - self.processor1 = StreamsSmokeTestJobRunnerService(test_context, self.kafka) - - @cluster(num_nodes=6) - def test_bounce(self): - """ - Start a smoke test client, then abort (kill -9) and restart it a few times. - Ensure that all records are delivered. - """ - - self.driver.start() - - self.processor1.start() - - time.sleep(15) - - self.processor1.abortThenRestart() - - time.sleep(15) - - # enable this after we add change log partition replicas - #self.kafka.signal_leader("data") - - #time.sleep(15); - - self.processor1.abortThenRestart() - - self.driver.wait() - self.driver.stop() - - self.processor1.stop() - - node = self.driver.node - node.account.ssh("grep ALL-RECORDS-DELIVERED %s" % self.driver.STDOUT_FILE, allow_fail=False) diff --git a/tests/kafkatest/tests/streams/streams_smoke_test.py b/tests/kafkatest/tests/streams/streams_smoke_test.py index 496c495827551..094869be422f5 100644 --- a/tests/kafkatest/tests/streams/streams_smoke_test.py +++ b/tests/kafkatest/tests/streams/streams_smoke_test.py @@ -14,11 +14,11 @@ # limitations under the License. +from ducktape.mark import matrix from ducktape.mark.resource import cluster from kafkatest.tests.kafka_test import KafkaTest -from kafkatest.services.streams import StreamsSmokeTestDriverService, StreamsSmokeTestJobRunnerService -import time +from kafkatest.services.streams import StreamsSmokeTestDriverService, StreamsSmokeTestJobRunnerService, StreamsSmokeTestEOSJobRunnerService class StreamsSmokeTest(KafkaTest): @@ -31,8 +31,12 @@ def __init__(self, test_context): 'echo' : { 'partitions': 5, 'replication-factor': 1 }, 'data' : { 'partitions': 5, 'replication-factor': 1 }, 'min' : { 'partitions': 5, 'replication-factor': 1 }, + 'min-suppressed' : { 'partitions': 5, 'replication-factor': 1 }, + 'min-raw' : { 'partitions': 5, 'replication-factor': 1 }, 'max' : { 'partitions': 5, 'replication-factor': 1 }, 'sum' : { 'partitions': 5, 'replication-factor': 1 }, + 'sws-raw' : { 'partitions': 5, 'replication-factor': 1 }, + 'sws-suppressed' : { 'partitions': 5, 'replication-factor': 1 }, 'dif' : { 'partitions': 5, 'replication-factor': 1 }, 'cnt' : { 'partitions': 5, 'replication-factor': 1 }, 'avg' : { 'partitions': 5, 'replication-factor': 1 }, @@ -40,39 +44,77 @@ def __init__(self, test_context): 'tagg' : { 'partitions': 5, 'replication-factor': 1 } }) + self.test_context = test_context self.driver = StreamsSmokeTestDriverService(test_context, self.kafka) - self.processor1 = StreamsSmokeTestJobRunnerService(test_context, self.kafka) - self.processor2 = StreamsSmokeTestJobRunnerService(test_context, self.kafka) - self.processor3 = StreamsSmokeTestJobRunnerService(test_context, self.kafka) - self.processor4 = StreamsSmokeTestJobRunnerService(test_context, self.kafka) - @cluster(num_nodes=9) - def test_streams(self): - """ - Start a few smoke test clients, then repeat start a new one, stop (cleanly) running one a few times. - Ensure that all results (stats on values computed by Kafka Streams) are correct. - """ - - self.driver.start() - - self.processor1.start() - self.processor2.start() - - time.sleep(15) - - self.processor3.start() - self.processor1.stop() - - time.sleep(15) - - self.processor4.start() + @cluster(num_nodes=8) + @matrix(eos=[True, False], crash=[True, False]) + def test_streams(self, eos, crash): + # + if eos: + processor1 = StreamsSmokeTestEOSJobRunnerService(self.test_context, self.kafka) + processor2 = StreamsSmokeTestEOSJobRunnerService(self.test_context, self.kafka) + processor3 = StreamsSmokeTestEOSJobRunnerService(self.test_context, self.kafka) + else: + processor1 = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka) + processor2 = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka) + processor3 = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka) + + + + with processor1.node.account.monitor_log(processor1.STDOUT_FILE) as monitor1: + processor1.start() + monitor1.wait_until('REBALANCING -> RUNNING', + timeout_sec=60, + err_msg="Never saw 'REBALANCING -> RUNNING' message " + str(processor1.node.account) + ) + + self.driver.start() + + monitor1.wait_until('processed', + timeout_sec=30, + err_msg="Didn't see any processing messages " + str(processor1.node.account) + ) + + # make sure we're not already done processing (which would invalidate the test) + self.driver.node.account.ssh("! grep 'Result Verification' %s" % self.driver.STDOUT_FILE, allow_fail=False) + + processor1.stop_nodes(not crash) + + with processor2.node.account.monitor_log(processor2.STDOUT_FILE) as monitor2: + processor2.start() + monitor2.wait_until('REBALANCING -> RUNNING', + timeout_sec=120, + err_msg="Never saw 'REBALANCING -> RUNNING' message " + str(processor2.node.account) + ) + monitor2.wait_until('processed', + timeout_sec=30, + err_msg="Didn't see any processing messages " + str(processor2.node.account) + ) + + # make sure we're not already done processing (which would invalidate the test) + self.driver.node.account.ssh("! grep 'Result Verification' %s" % self.driver.STDOUT_FILE, allow_fail=False) + + processor2.stop_nodes(not crash) + + with processor3.node.account.monitor_log(processor3.STDOUT_FILE) as monitor3: + processor3.start() + monitor3.wait_until('REBALANCING -> RUNNING', + timeout_sec=120, + err_msg="Never saw 'REBALANCING -> RUNNING' message " + str(processor3.node.account) + ) + # there should still be some data left for this processor to work on. + monitor3.wait_until('processed', + timeout_sec=30, + err_msg="Didn't see any processing messages " + str(processor3.node.account) + ) self.driver.wait() self.driver.stop() - self.processor2.stop() - self.processor3.stop() - self.processor4.stop() + processor3.stop() - node = self.driver.node - node.account.ssh("grep SUCCESS %s" % self.driver.STDOUT_FILE, allow_fail=False) + if crash and not eos: + self.driver.node.account.ssh("grep -E 'SUCCESS|PROCESSED-MORE-THAN-GENERATED' %s" % self.driver.STDOUT_FILE, allow_fail=False) + else: + self.driver.node.account.ssh("grep SUCCESS %s" % self.driver.STDOUT_FILE, allow_fail=False) From 2aca6241624f6b924b3e0164a9b7d021d80096b6 Mon Sep 17 00:00:00 2001 From: cadonna Date: Tue, 12 Mar 2019 19:05:53 +0100 Subject: [PATCH 0015/1071] MINOR: Avoid double null check in KStream#transform() (#6429) Reviewers: A. Sophie Blee-Goldman , Matthias J. Sax --- .../kstream/internals/KStreamImpl.java | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index 0eda64ff8abca..856536cc78831 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -439,17 +439,8 @@ private void to(final TopicNameExtractor topicExtractor, final ProducedInt builder.addGraphNode(this.streamsGraphNode, sinkNode); } - @Override - public KStream transform(final TransformerSupplier> transformerSupplier, - final String... stateStoreNames) { - Objects.requireNonNull(transformerSupplier, "transformerSupplier can't be null"); - return flatTransform(new TransformerSupplierAdapter<>(transformerSupplier), stateStoreNames); - } - - @Override - public KStream flatTransform(final TransformerSupplier>> transformerSupplier, - final String... stateStoreNames) { - Objects.requireNonNull(transformerSupplier, "transformerSupplier can't be null"); + private KStream doFlatTransform(final TransformerSupplier>> transformerSupplier, + final String... stateStoreNames) { final String name = builder.newProcessorName(TRANSFORM_NAME); final StatefulProcessorNode transformNode = new StatefulProcessorNode<>( name, @@ -464,6 +455,20 @@ public KStream flatTransform(final TransformerSupplier(name, null, null, sourceNodes, true, transformNode, builder); } + @Override + public KStream transform(final TransformerSupplier> transformerSupplier, + final String... stateStoreNames) { + Objects.requireNonNull(transformerSupplier, "transformerSupplier can't be null"); + return doFlatTransform(new TransformerSupplierAdapter<>(transformerSupplier), stateStoreNames); + } + + @Override + public KStream flatTransform(final TransformerSupplier>> transformerSupplier, + final String... stateStoreNames) { + Objects.requireNonNull(transformerSupplier, "transformerSupplier can't be null"); + return doFlatTransform(transformerSupplier, stateStoreNames); + } + @Override public KStream transformValues(final ValueTransformerSupplier valueTransformerSupplier, final String... stateStoreNames) { From 9ecadc4df474d9cfbfda3256f01eba1423cf5902 Mon Sep 17 00:00:00 2001 From: Bill Bejeck Date: Tue, 12 Mar 2019 16:35:25 -0400 Subject: [PATCH 0016/1071] MINOR: Use Java 8 lambdas in KStreamImplTest (#6430) Just a minor cleanup to use Java 8 lambdas vs anonymous classes in this test. I ran all tests in the streams test suite Reviewers: Matthias J. Sax , Guozhang Wang --- .../scala/kafka/tools/StreamsResetter.java | 2 +- .../kstream/internals/KStreamImplTest.java | 73 ++++--------------- 2 files changed, 14 insertions(+), 61 deletions(-) diff --git a/core/src/main/scala/kafka/tools/StreamsResetter.java b/core/src/main/scala/kafka/tools/StreamsResetter.java index 3666f6740e37b..71529f8ca4133 100644 --- a/core/src/main/scala/kafka/tools/StreamsResetter.java +++ b/core/src/main/scala/kafka/tools/StreamsResetter.java @@ -257,7 +257,7 @@ private void parseArguments(final String[] args) throws IOException { CommandLineUtils.printUsageAndDie(optionParser, "Only one of --dry-run and --execute can be specified"); } - final scala.collection.immutable.HashSet> allScenarioOptions = new scala.collection.immutable.HashSet>(); + final scala.collection.immutable.HashSet> allScenarioOptions = new scala.collection.immutable.HashSet<>(); allScenarioOptions.$plus(toOffsetOption); allScenarioOptions.$plus(toDatetimeOption); allScenarioOptions.$plus(byDurationOption); diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java index 76a01cde53c90..bd2ab5ba7489a 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java @@ -104,78 +104,31 @@ public void testNumProcesses() { final KStream source2 = builder.stream(Arrays.asList("topic-3", "topic-4"), stringConsumed); - final KStream stream1 = - source1.filter(new Predicate() { - @Override - public boolean test(final String key, final String value) { - return true; - } - }).filterNot(new Predicate() { - @Override - public boolean test(final String key, final String value) { - return false; - } - }); + final KStream stream1 = source1.filter((key, value) -> true) + .filterNot((key, value) -> false); - final KStream stream2 = stream1.mapValues(new ValueMapper() { - @Override - public Integer apply(final String value) { - return new Integer(value); - } - }); + final KStream stream2 = stream1.mapValues(Integer::new); - final KStream stream3 = source2.flatMapValues(new ValueMapper>() { - @Override - public Iterable apply(final String value) { - return Collections.singletonList(new Integer(value)); - } - }); + final KStream stream3 = source2.flatMapValues((ValueMapper>) + value -> Collections.singletonList(new Integer(value))); final KStream[] streams2 = stream2.branch( - new Predicate() { - @Override - public boolean test(final String key, final Integer value) { - return (value % 2) == 0; - } - }, - new Predicate() { - @Override - public boolean test(final String key, final Integer value) { - return true; - } - } + (key, value) -> (value % 2) == 0, + (key, value) -> true ); final KStream[] streams3 = stream3.branch( - new Predicate() { - @Override - public boolean test(final String key, final Integer value) { - return (value % 2) == 0; - } - }, - new Predicate() { - @Override - public boolean test(final String key, final Integer value) { - return true; - } - } + (key, value) -> (value % 2) == 0, + (key, value) -> true ); final int anyWindowSize = 1; final Joined joined = Joined.with(Serdes.String(), Serdes.Integer(), Serdes.Integer()); - final KStream stream4 = streams2[0].join(streams3[0], new ValueJoiner() { - @Override - public Integer apply(final Integer value1, final Integer value2) { - return value1 + value2; - } - }, JoinWindows.of(ofMillis(anyWindowSize)), joined); + final KStream stream4 = streams2[0].join(streams3[0], + (value1, value2) -> value1 + value2, JoinWindows.of(ofMillis(anyWindowSize)), joined); - streams2[1].join(streams3[1], new ValueJoiner() { - @Override - public Integer apply(final Integer value1, final Integer value2) { - return value1 + value2; - } - }, JoinWindows.of(ofMillis(anyWindowSize)), joined); + streams2[1].join(streams3[1], (value1, value2) -> value1 + value2, + JoinWindows.of(ofMillis(anyWindowSize)), joined); stream4.to("topic-5"); From fd79dd060812af0531428cb2f7fd4aa74ea8124a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Armando=20Garc=C3=ADa=20Sancio?= Date: Tue, 12 Mar 2019 15:24:24 -0700 Subject: [PATCH 0017/1071] MINOR: Better messaging for invalid fetch response (#6427) Users have reported (KAFKA-7565) that when consumer poll wake up is used, it is possible to receive fetch responses that don't match the copied topic partitions collection for the session when the fetch request was created. This commit improves the error handling here by throwing an IllegalStateException instead of a NullPointerException. And by generating a message for the exception that includes a bit of more information. Reviewers: Jason Gustafson --- .../clients/consumer/internals/Fetcher.java | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index 8ac5730110c51..64bc92130c7e2 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -68,6 +68,7 @@ import org.apache.kafka.common.utils.Timer; import org.apache.kafka.common.utils.Utils; import org.slf4j.Logger; +import org.slf4j.helpers.MessageFormatter; import java.io.Closeable; import java.nio.ByteBuffer; @@ -241,13 +242,30 @@ public void onSuccess(ClientResponse resp) { for (Map.Entry> entry : response.responseData().entrySet()) { TopicPartition partition = entry.getKey(); - long fetchOffset = data.sessionPartitions().get(partition).fetchOffset; - FetchResponse.PartitionData fetchData = entry.getValue(); - - log.debug("Fetch {} at offset {} for partition {} returned fetch data {}", - isolationLevel, fetchOffset, partition, fetchData); - completedFetches.add(new CompletedFetch(partition, fetchOffset, fetchData, metricAggregator, - resp.requestHeader().apiVersion())); + FetchRequest.PartitionData requestData = data.sessionPartitions().get(partition); + if (requestData == null) { + String message; + if (data.metadata().isFull()) { + message = MessageFormatter.arrayFormat( + "Response for missing full request partition: partition={}; metadata={}", + new Object[]{partition, data.metadata()}).getMessage(); + } else { + message = MessageFormatter.arrayFormat( + "Response for missing session request partition: partition={}; metadata={}; toSend={}; toForget={}", + new Object[]{partition, data.metadata(), data.toSend(), data.toForget()}).getMessage(); + } + + // Received fetch response for missing session partition + throw new IllegalStateException(message); + } else { + long fetchOffset = requestData.fetchOffset; + FetchResponse.PartitionData fetchData = entry.getValue(); + + log.debug("Fetch {} at offset {} for partition {} returned fetch data {}", + isolationLevel, fetchOffset, partition, fetchData); + completedFetches.add(new CompletedFetch(partition, fetchOffset, fetchData, metricAggregator, + resp.requestHeader().apiVersion())); + } } sensors.fetchLatency.record(resp.requestLatencyMs()); From a648ef0b44e9c3efbee3d66fc7dae85fea24a74b Mon Sep 17 00:00:00 2001 From: "Matthias J. Sax" Date: Wed, 13 Mar 2019 09:16:48 -0700 Subject: [PATCH 0018/1071] MINOR: Fix JavaDocs warnings (#6435) Fix JavaDocs Warning Reviewers: uozhang Wang , Bill Bejeck --- .../java/org/apache/kafka/common/config/ConfigDef.java | 4 ++-- .../OAuthBearerExtensionsValidatorCallback.java | 2 +- .../main/java/org/apache/kafka/streams/KafkaStreams.java | 8 ++++---- .../main/java/org/apache/kafka/streams/StreamsConfig.java | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java b/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java index 7669f1dd99942..1259882a89ce4 100644 --- a/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java +++ b/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java @@ -53,7 +53,7 @@ * defs.define("config_with_validator", Type.INT, 42, Range.atLeast(0), "Configuration with user provided validator."); * defs.define("config_with_dependents", Type.INT, "Configuration with dependents.", "group", 1, "Config With Dependents", Arrays.asList("config_with_default","config_with_validator")); * - * Map<String, String> props = new HashMap<>(); + * Map<String, String> props = new HashMap<>(); * props.put("config_with_default", "some value"); * props.put("config_with_dependents", "some other value"); * @@ -1133,7 +1133,7 @@ private void addColumnValue(StringBuilder builder, String value) { * If dynamicUpdateModes is non-empty, a "Dynamic Update Mode" column * will be included n the table with the value of the update mode. Default * mode is "read-only". - * @param dynamicUpdateModes Config name -> update mode mapping + * @param dynamicUpdateModes Config name -> update mode mapping */ public String toHtmlTable(Map dynamicUpdateModes) { boolean hasUpdateModes = !dynamicUpdateModes.isEmpty(); diff --git a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/OAuthBearerExtensionsValidatorCallback.java b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/OAuthBearerExtensionsValidatorCallback.java index 97ac4d96838f5..eab208baa6f2b 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/OAuthBearerExtensionsValidatorCallback.java +++ b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/OAuthBearerExtensionsValidatorCallback.java @@ -76,7 +76,7 @@ public Map validatedExtensions() { } /** - * @return An immutable {@link Map} consisting of the name->error messages of extensions which failed validation + * @return An immutable {@link Map} consisting of the name->error messages of extensions which failed validation */ public Map invalidExtensions() { return Collections.unmodifiableMap(invalidExtensions); diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java index 2d1aa79bc9a9f..4f15deadbc784 100644 --- a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java +++ b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java @@ -155,23 +155,23 @@ public class KafkaStreams implements AutoCloseable { * * 
      *                 +--------------+
-     *         +<----- | Created (0)  |
+     *         +<----- | Created (0)  |
      *         |       +-----+--------+
      *         |             |
      *         |             v
      *         |       +----+--+------+
      *         |       | Re-          |
-     *         +<----- | Balancing (1)| -------->+
+     *         +<----- | Balancing (1)| -------->+
      *         |       +-----+-+------+          |
      *         |             | ^                 |
      *         |             v |                 |
      *         |       +--------------+          v
-     *         |       | Running (2)  | -------->+
+     *         |       | Running (2)  | -------->+
      *         |       +------+-------+          |
      *         |              |                  |
      *         |              v                  |
      *         |       +------+-------+     +----+-------+
-     *         +-----> | Pending      |<--- | Error (5)  |
+     *         +-----> | Pending      |<--- | Error (5)  |
      *                 | Shutdown (3) |     +------------+
      *                 +------+-------+
      *                        |
diff --git a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
index b756ba2d7f6bd..2ba73128528f5 100644
--- a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
+++ b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
@@ -102,7 +102,7 @@
  * When increasing {@link ProducerConfig#MAX_BLOCK_MS_CONFIG} to be more resilient to non-available brokers you should also
  * increase {@link ConsumerConfig#MAX_POLL_INTERVAL_MS_CONFIG} using the following guidance:
  * 
- *     max.poll.interval.ms > max.block.ms
+ *     max.poll.interval.ms > max.block.ms
  * 

  *
  *

From d436f32e6b15b9ff8bc4a4ab0ce6dd0ec5f14a8d Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Thu, 14 Mar 2019 19:29:33 +0000
Subject: [PATCH 0019/1071] KAFKA-8091; Remove unsafe produce from dynamic
 listener update test (#6443)

Reviewers: Manikumar Reddy 
---
 .../kafka/server/DynamicBrokerReconfigurationTest.scala         | 2 --
 1 file changed, 2 deletions(-)

diff --git a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
index c28fcea15cb38..d1427b2356c42 100644
--- a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
+++ b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
@@ -935,8 +935,6 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
       .autoOffsetReset("latest")
       .build()
     verifyProduceConsume(producer1, consumer1, numRecords = 10, topic)
-    // send another message to check consumer later
-    producer1.send(new ProducerRecord(topic, "key", "value")).get(1, TimeUnit.SECONDS)
 
     val config = servers.head.config
     val existingListenerCount = config.listeners.size

From 3ec2ca5e334ef42683263ed50c718c62917046af Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Thu, 14 Mar 2019 16:18:52 -0700
Subject: [PATCH 0020/1071] KAFKA-7730; Limit number of active connections per
 listener in brokers (KIP-402)

Adds a new listener config `max.connections` to limit the number of active connections on each listener. The config may be prefixed with listener prefix. This limit may be dynamically reconfigured without restarting the broker.

This is one of the PRs for KIP-402 (https://cwiki.apache.org/confluence/display/KAFKA/KIP-402%3A+Improve+fairness+in+SocketServer+processors). Note that this is currently built on top of PR #6022

Author: Rajini Sivaram 

Reviewers: Gwen Shapira 

Closes #6034 from rajinisivaram/KAFKA-7730-max-connections
---
 .../apache/kafka/common/network/Selector.java |  22 ++
 .../kafka/common/network/SelectorTest.java    |  30 +++
 .../scala/kafka/network/SocketServer.scala    | 209 +++++++++++++++--
 .../kafka/server/DynamicBrokerConfig.scala    |  40 ++--
 .../main/scala/kafka/server/KafkaConfig.scala |  19 +-
 .../network/DynamicConnectionQuotaTest.scala  | 217 +++++++++++++-----
 .../server/DynamicBrokerConfigTest.scala      |   6 +
 7 files changed, 441 insertions(+), 102 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selector.java b/clients/src/main/java/org/apache/kafka/common/network/Selector.java
index e431e2788d858..b3497973afd0a 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/Selector.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/Selector.java
@@ -919,6 +919,28 @@ public KafkaChannel closingChannel(String id) {
         return closingChannels.get(id);
     }
 
+    /**
+     * Returns the lowest priority channel chosen using the following sequence:
+     *   1) If one or more channels are in closing state, return any one of them
+     *   2) If idle expiry manager is enabled, return the least recently updated channel
+     *   3) Otherwise return any of the channels
+     *
+     * This method is used to close a channel to accommodate a new channel on the inter-broker listener
+     * when broker-wide `max.connections` limit is enabled.
+     */
+    public KafkaChannel lowestPriorityChannel() {
+        KafkaChannel channel = null;
+        if (!closingChannels.isEmpty()) {
+            channel = closingChannels.values().iterator().next();
+        } else if (idleExpiryManager != null && !idleExpiryManager.lruConnections.isEmpty()) {
+            String channelId = idleExpiryManager.lruConnections.keySet().iterator().next();
+            channel = channel(channelId);
+        } else if (!channels.isEmpty()) {
+            channel = channels.values().iterator().next();
+        }
+        return channel;
+    }
+
     /**
      * Get the channel associated with selectionKey
      */
diff --git a/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java b/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java
index 7cb89c28de25e..8cbada750c26f 100644
--- a/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java
@@ -688,6 +688,36 @@ public void testInboundConnectionsCountInConnectionCreationMetric() throws Excep
         assertEquals((double) conns, getMetric("connection-count").metricValue());
     }
 
+    @SuppressWarnings("unchecked")
+    @Test
+    public void testLowestPriorityChannel() throws Exception {
+        int conns = 5;
+        InetSocketAddress addr = new InetSocketAddress("localhost", server.port);
+        for (int i = 0; i < conns; i++) {
+            connect(String.valueOf(i), addr);
+        }
+        assertNotNull(selector.lowestPriorityChannel());
+        for (int i = conns - 1; i >= 0; i--) {
+            if (i != 2)
+              assertEquals("", blockingRequest(String.valueOf(i), ""));
+            time.sleep(10);
+        }
+        assertEquals("2", selector.lowestPriorityChannel().id());
+
+        Field field = Selector.class.getDeclaredField("closingChannels");
+        field.setAccessible(true);
+        Map closingChannels = (Map) field.get(selector);
+        closingChannels.put("3", selector.channel("3"));
+        assertEquals("3", selector.lowestPriorityChannel().id());
+        closingChannels.remove("3");
+
+        for (int i = 0; i < conns; i++) {
+            selector.close(String.valueOf(i));
+        }
+        assertNull(selector.lowestPriorityChannel());
+    }
+
+
     private String blockingRequest(String node, String s) throws IOException {
         selector.send(createSend(node, s));
         selector.poll(1000L);
diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala
index d8c4a74cdd25c..e06cee75ffe14 100644
--- a/core/src/main/scala/kafka/network/SocketServer.scala
+++ b/core/src/main/scala/kafka/network/SocketServer.scala
@@ -21,6 +21,7 @@ import java.io.IOException
 import java.net._
 import java.nio.channels._
 import java.nio.channels.{Selector => NSelector}
+import java.util
 import java.util.concurrent._
 import java.util.concurrent.atomic._
 import java.util.function.Supplier
@@ -32,15 +33,16 @@ import kafka.network.RequestChannel.{CloseConnectionResponse, EndThrottlingRespo
 import kafka.network.Processor._
 import kafka.network.SocketServer._
 import kafka.security.CredentialProvider
-import kafka.server.KafkaConfig
+import kafka.server.{BrokerReconfigurable, KafkaConfig}
 import kafka.utils._
+import org.apache.kafka.common.config.ConfigException
 import org.apache.kafka.common.{KafkaException, Reconfigurable}
 import org.apache.kafka.common.memory.{MemoryPool, SimpleMemoryPool}
 import org.apache.kafka.common.metrics._
 import org.apache.kafka.common.metrics.stats.Meter
 import org.apache.kafka.common.metrics.stats.Total
 import org.apache.kafka.common.network.KafkaChannel.ChannelMuteEvent
-import org.apache.kafka.common.network.{ChannelBuilder, ChannelBuilders, KafkaChannel, ListenerName, Selectable, Send, Selector => KSelector}
+import org.apache.kafka.common.network.{ChannelBuilder, ChannelBuilders, KafkaChannel, ListenerName, ListenerReconfigurable, Selectable, Send, Selector => KSelector}
 import org.apache.kafka.common.protocol.ApiKeys
 import org.apache.kafka.common.requests.{RequestContext, RequestHeader}
 import org.apache.kafka.common.security.auth.SecurityProtocol
@@ -70,7 +72,11 @@ import scala.util.control.ControlThrowable
  *      Acceptor has 1 Processor thread that has its own selector and read requests from the socket.
  *      1 Handler thread that handles requests and produce responses back to the processor thread for writing.
  */
-class SocketServer(val config: KafkaConfig, val metrics: Metrics, val time: Time, val credentialProvider: CredentialProvider) extends Logging with KafkaMetricsGroup {
+class SocketServer(val config: KafkaConfig,
+                   val metrics: Metrics,
+                   val time: Time,
+                   val credentialProvider: CredentialProvider)
+  extends Logging with KafkaMetricsGroup with BrokerReconfigurable {
 
   private val maxQueuedRequests = config.queuedMaxRequests
 
@@ -109,7 +115,7 @@ class SocketServer(val config: KafkaConfig, val metrics: Metrics, val time: Time
    */
   def startup(startupProcessors: Boolean = true) {
     this.synchronized {
-      connectionQuotas = new ConnectionQuotas(config.maxConnectionsPerIp, config.maxConnectionsPerIpOverrides)
+      connectionQuotas = new ConnectionQuotas(config, time)
       createControlPlaneAcceptorAndProcessor(config.controlPlaneListener)
       createDataPlaneAcceptorsAndProcessors(config.numNetworkThreads, config.dataPlaneListeners)
       if (startupProcessors) {
@@ -212,6 +218,7 @@ class SocketServer(val config: KafkaConfig, val metrics: Metrics, val time: Time
   private def createDataPlaneAcceptorsAndProcessors(dataProcessorsPerListener: Int,
                                                     endpoints: Seq[EndPoint]): Unit = synchronized {
     endpoints.foreach { endpoint =>
+      connectionQuotas.addListener(config, endpoint.listenerName)
       val dataPlaneAcceptor = createAcceptor(endpoint, DataPlaneMetricPrefix)
       addDataPlaneProcessors(dataPlaneAcceptor, endpoint, dataProcessorsPerListener)
       KafkaThread.nonDaemon(s"data-plane-kafka-socket-acceptor-${endpoint.listenerName}-${endpoint.securityProtocol}-${endpoint.port}", dataPlaneAcceptor).start()
@@ -223,6 +230,7 @@ class SocketServer(val config: KafkaConfig, val metrics: Metrics, val time: Time
 
   private def createControlPlaneAcceptorAndProcessor(endpointOpt: Option[EndPoint]): Unit = synchronized {
     endpointOpt.foreach { endpoint =>
+      connectionQuotas.addListener(config, endpoint.listenerName)
       val controlPlaneAcceptor = createAcceptor(endpoint, ControlPlaneMetricPrefix)
       val controlPlaneProcessor = newProcessor(nextProcessorId, controlPlaneRequestChannelOpt.get, connectionQuotas, endpoint.listenerName, endpoint.securityProtocol, memoryPool)
       controlPlaneAcceptorOpt = Some(controlPlaneAcceptor)
@@ -324,18 +332,33 @@ class SocketServer(val config: KafkaConfig, val metrics: Metrics, val time: Time
   def removeListeners(listenersRemoved: Seq[EndPoint]): Unit = synchronized {
     info(s"Removing data-plane listeners for endpoints $listenersRemoved")
     listenersRemoved.foreach { endpoint =>
+      connectionQuotas.removeListener(config, endpoint.listenerName)
       dataPlaneAcceptors.asScala.remove(endpoint).foreach(_.shutdown())
     }
   }
 
-  def updateMaxConnectionsPerIp(maxConnectionsPerIp: Int): Unit = {
-    info(s"Updating maxConnectionsPerIp: $maxConnectionsPerIp")
-    connectionQuotas.updateMaxConnectionsPerIp(maxConnectionsPerIp)
+  override def reconfigurableConfigs: Set[String] = SocketServer.ReconfigurableConfigs
+
+  override def validateReconfiguration(newConfig: KafkaConfig): Unit = {
+
   }
 
-  def updateMaxConnectionsPerIpOverride(maxConnectionsPerIpOverrides: Map[String, Int]): Unit = {
-    info(s"Updating maxConnectionsPerIpOverrides: ${maxConnectionsPerIpOverrides.map { case (k, v) => s"$k=$v" }.mkString(",")}")
-    connectionQuotas.updateMaxConnectionsPerIpOverride(maxConnectionsPerIpOverrides)
+  override def reconfigure(oldConfig: KafkaConfig, newConfig: KafkaConfig): Unit = {
+    val maxConnectionsPerIp = newConfig.maxConnectionsPerIp
+    if (maxConnectionsPerIp != oldConfig.maxConnectionsPerIp) {
+      info(s"Updating maxConnectionsPerIp: $maxConnectionsPerIp")
+      connectionQuotas.updateMaxConnectionsPerIp(maxConnectionsPerIp)
+    }
+    val maxConnectionsPerIpOverrides = newConfig.maxConnectionsPerIpOverrides
+    if (maxConnectionsPerIpOverrides != oldConfig.maxConnectionsPerIpOverrides) {
+      info(s"Updating maxConnectionsPerIpOverrides: ${maxConnectionsPerIpOverrides.map { case (k, v) => s"$k=$v" }.mkString(",")}")
+      connectionQuotas.updateMaxConnectionsPerIpOverride(maxConnectionsPerIpOverrides)
+    }
+    val maxConnections = newConfig.maxConnections
+    if (maxConnections != oldConfig.maxConnections) {
+      info(s"Updating broker-wide maxConnections: $maxConnections")
+      connectionQuotas.updateBrokerMaxConnections(maxConnections)
+    }
   }
 
   // `protected` for test usage
@@ -373,6 +396,13 @@ object SocketServer {
   val ControlPlaneThreadPrefix = "control-plane"
   val DataPlaneMetricPrefix = ""
   val ControlPlaneMetricPrefix = "ControlPlane"
+
+  val ReconfigurableConfigs = Set(
+    KafkaConfig.MaxConnectionsPerIpProp,
+    KafkaConfig.MaxConnectionsPerIpOverridesProp,
+    KafkaConfig.MaxConnectionsProp)
+
+  val ListenerReconfigurableConfigs = Set(KafkaConfig.MaxConnectionsProp)
 }
 
 /**
@@ -427,10 +457,10 @@ private[kafka] abstract class AbstractServerThread(connectionQuotas: ConnectionQ
   /**
    * Close `channel` and decrement the connection count.
    */
-  def close(channel: SocketChannel): Unit = {
+  def close(listenerName: ListenerName, channel: SocketChannel): Unit = {
     if (channel != null) {
       debug(s"Closing connection from ${channel.socket.getRemoteSocketAddress()}")
-      connectionQuotas.dec(channel.socket.getInetAddress)
+      connectionQuotas.dec(listenerName, channel.socket.getInetAddress)
       CoreUtils.swallow(channel.socket().close(), this, Level.ERROR)
       CoreUtils.swallow(channel.close(), this, Level.ERROR)
     }
@@ -500,6 +530,7 @@ private[kafka] class Acceptor(val endPoint: EndPoint,
       var currentProcessorIndex = 0
       while (isRunning) {
         try {
+
           val ready = nioSelector.select(500)
           if (ready > 0) {
             val keys = nioSelector.selectedKeys()
@@ -508,6 +539,7 @@ private[kafka] class Acceptor(val endPoint: EndPoint,
               try {
                 val key = iter.next
                 iter.remove()
+
                 if (key.isAcceptable) {
                   accept(key).foreach { socketChannel =>
 
@@ -582,7 +614,7 @@ private[kafka] class Acceptor(val endPoint: EndPoint,
     val serverSocketChannel = key.channel().asInstanceOf[ServerSocketChannel]
     val socketChannel = serverSocketChannel.accept()
     try {
-      connectionQuotas.inc(socketChannel.socket().getInetAddress)
+      connectionQuotas.inc(endPoint.listenerName, socketChannel.socket.getInetAddress, blockedPercentMeter)
       socketChannel.configureBlocking(false)
       socketChannel.socket().setTcpNoDelay(true)
       socketChannel.socket().setKeepAlive(true)
@@ -592,7 +624,7 @@ private[kafka] class Acceptor(val endPoint: EndPoint,
     } catch {
       case e: TooManyConnectionsException =>
         info(s"Rejected connection from ${e.ip}, address already has the configured maximum of ${e.count} connections.")
-        close(socketChannel)
+        close(endPoint.listenerName, socketChannel)
         None
     }
   }
@@ -731,6 +763,7 @@ private[kafka] class Processor(val id: Int,
           processCompletedReceives()
           processCompletedSends()
           processDisconnected()
+          closeExcessConnections()
         } catch {
           // We catch all the throwables here to prevent the processor thread from exiting. We do this because
           // letting a processor exit might cause a bigger impact on the broker. This behavior might need to be
@@ -911,13 +944,21 @@ private[kafka] class Processor(val id: Int,
         }.remoteHost
         inflightResponses.remove(connectionId).foreach(updateRequestMetrics)
         // the channel has been closed by the selector but the quotas still need to be updated
-        connectionQuotas.dec(InetAddress.getByName(remoteHost))
+        connectionQuotas.dec(listenerName, InetAddress.getByName(remoteHost))
       } catch {
         case e: Throwable => processException(s"Exception while processing disconnection of $connectionId", e)
       }
     }
   }
 
+  private def closeExcessConnections(): Unit = {
+    if (connectionQuotas.maxConnectionsExceeded(listenerName)) {
+      val channel = selector.lowestPriorityChannel()
+      if (channel != null)
+        close(channel.id)
+    }
+  }
+
   /**
    * Close the connection identified by `connectionId` and decrement the connection count.
    * The channel will be immediately removed from the selector's `channels` or `closingChannels`
@@ -930,7 +971,7 @@ private[kafka] class Processor(val id: Int,
       debug(s"Closing selector connection $connectionId")
       val address = channel.socketAddress
       if (address != null)
-        connectionQuotas.dec(address)
+        connectionQuotas.dec(listenerName, address)
       selector.close(connectionId)
 
       inflightResponses.remove(connectionId).foreach(response => updateRequestMetrics(response))
@@ -977,7 +1018,7 @@ private[kafka] class Processor(val id: Int,
         case e: Throwable =>
           val remoteAddress = channel.socket.getRemoteSocketAddress
           // need to close the channel here to avoid a socket leak.
-          close(channel)
+          close(listenerName, channel)
           processException(s"Processor $id closed connection from $remoteAddress", e)
       }
     }
@@ -1058,31 +1099,72 @@ private[kafka] class Processor(val id: Int,
 
 }
 
-class ConnectionQuotas(val defaultMax: Int, overrideQuotas: Map[String, Int]) {
+class ConnectionQuotas(config: KafkaConfig, time: Time) extends Logging {
 
-  @volatile private var defaultMaxConnectionsPerIp = defaultMax
-  @volatile private var maxConnectionsPerIpOverrides = overrideQuotas.map { case (host, count) => (InetAddress.getByName(host), count) }
+  @volatile private var defaultMaxConnectionsPerIp: Int = config.maxConnectionsPerIp
+  @volatile private var maxConnectionsPerIpOverrides = config.maxConnectionsPerIpOverrides.map { case (host, count) => (InetAddress.getByName(host), count) }
+  @volatile private var brokerMaxConnections = config.maxConnections
   private val counts = mutable.Map[InetAddress, Int]()
 
-  def inc(address: InetAddress) {
+  // Listener counts and configs are synchronized on `counts`
+  private val listenerCounts = mutable.Map[ListenerName, Int]()
+  private val maxConnectionsPerListener = mutable.Map[ListenerName, ListenerConnectionQuota]()
+  @volatile private var totalCount = 0
+
+  def inc(listenerName: ListenerName, address: InetAddress, acceptorBlockedPercentMeter: com.yammer.metrics.core.Meter) {
     counts.synchronized {
+      waitForConnectionSlot(listenerName, acceptorBlockedPercentMeter)
+
       val count = counts.getOrElseUpdate(address, 0)
       counts.put(address, count + 1)
+      totalCount += 1
+      if (listenerCounts.contains(listenerName)) {
+        listenerCounts.put(listenerName, listenerCounts(listenerName) + 1)
+      }
       val max = maxConnectionsPerIpOverrides.getOrElse(address, defaultMaxConnectionsPerIp)
       if (count >= max)
         throw new TooManyConnectionsException(address, max)
     }
   }
 
-  def updateMaxConnectionsPerIp(maxConnectionsPerIp: Int): Unit = {
+  private[network] def updateMaxConnectionsPerIp(maxConnectionsPerIp: Int): Unit = {
     defaultMaxConnectionsPerIp = maxConnectionsPerIp
   }
 
-  def updateMaxConnectionsPerIpOverride(overrideQuotas: Map[String, Int]): Unit = {
+  private[network] def updateMaxConnectionsPerIpOverride(overrideQuotas: Map[String, Int]): Unit = {
     maxConnectionsPerIpOverrides = overrideQuotas.map { case (host, count) => (InetAddress.getByName(host), count) }
   }
 
-  def dec(address: InetAddress) {
+  private[network] def updateBrokerMaxConnections(maxConnections: Int): Unit = {
+    counts.synchronized {
+      brokerMaxConnections = maxConnections
+      counts.notifyAll()
+    }
+  }
+
+  private[network] def addListener(config: KafkaConfig, listenerName: ListenerName): Unit = {
+    counts.synchronized {
+      if (!maxConnectionsPerListener.contains(listenerName)) {
+        val newListenerQuota = new ListenerConnectionQuota(counts, listenerName)
+        maxConnectionsPerListener.put(listenerName, newListenerQuota)
+        listenerCounts.put(listenerName, 0)
+        config.addReconfigurable(newListenerQuota)
+      }
+      counts.notifyAll()
+    }
+  }
+
+  private[network] def removeListener(config: KafkaConfig, listenerName: ListenerName): Unit = {
+    counts.synchronized {
+      maxConnectionsPerListener.remove(listenerName).foreach { listenerQuota =>
+        listenerCounts.remove(listenerName)
+        counts.notifyAll() // wake up any waiting acceptors to close cleanly
+        config.removeReconfigurable(listenerQuota)
+      }
+    }
+  }
+
+  def dec(listenerName: ListenerName, address: InetAddress) {
     counts.synchronized {
       val count = counts.getOrElse(address,
         throw new IllegalArgumentException(s"Attempted to decrease connection count for address with no connections, address: $address"))
@@ -1090,6 +1172,19 @@ class ConnectionQuotas(val defaultMax: Int, overrideQuotas: Map[String, Int]) {
         counts.remove(address)
       else
         counts.put(address, count - 1)
+
+      if (totalCount <= 0)
+        error(s"Attempted to decrease total connection count for broker with no connections")
+      totalCount -= 1
+
+      if (maxConnectionsPerListener.contains(listenerName)) {
+        val listenerCount = listenerCounts(listenerName)
+        if (listenerCount == 0)
+          error(s"Attempted to decrease connection count for listener $listenerName with no connections")
+        else
+          listenerCounts.put(listenerName, listenerCount - 1)
+      }
+      counts.notifyAll() // wake up any acceptors waiting to process a new connection since listener connection limit was reached
     }
   }
 
@@ -1097,6 +1192,72 @@ class ConnectionQuotas(val defaultMax: Int, overrideQuotas: Map[String, Int]) {
     counts.getOrElse(address, 0)
   }
 
+  private def waitForConnectionSlot(listenerName: ListenerName,
+                                    acceptorBlockedPercentMeter: com.yammer.metrics.core.Meter): Unit = {
+    counts.synchronized {
+      if (!connectionSlotAvailable(listenerName)) {
+        val startNs = time.nanoseconds
+        do {
+          counts.wait()
+        } while (!connectionSlotAvailable(listenerName))
+        acceptorBlockedPercentMeter.mark(time.nanoseconds - startNs)
+      }
+    }
+  }
+
+  // This is invoked in every poll iteration and we close one LRU connection in an iteration
+  // if necessary
+  def maxConnectionsExceeded(listenerName: ListenerName): Boolean = {
+    totalCount > brokerMaxConnections && !protectedListener(listenerName)
+  }
+
+  private def connectionSlotAvailable(listenerName: ListenerName): Boolean = {
+    if (listenerCounts(listenerName) >= maxListenerConnections(listenerName))
+      false
+    else if (protectedListener(listenerName))
+      true
+    else
+      totalCount < brokerMaxConnections
+  }
+
+  private def protectedListener(listenerName: ListenerName): Boolean =
+    config.interBrokerListenerName == listenerName && config.listeners.size > 1
+
+  private def maxListenerConnections(listenerName: ListenerName): Int =
+    maxConnectionsPerListener.get(listenerName).map(_.maxConnections).getOrElse(Int.MaxValue)
+
+  class ListenerConnectionQuota(lock: Object, listener: ListenerName) extends ListenerReconfigurable {
+    @volatile private var _maxConnections = Int.MaxValue
+    private val listenerPropName = s"${listener.configPrefix}${KafkaConfig.MaxConnectionsProp}"
+    def maxConnections: Int = _maxConnections
+
+    override def listenerName(): ListenerName = listener
+
+    override def configure(configs: util.Map[String, _]): Unit = {
+      _maxConnections = maxConnections(configs)
+    }
+
+    override def reconfigurableConfigs(): util.Set[String] = {
+      SocketServer.ListenerReconfigurableConfigs.asJava
+    }
+
+    override def validateReconfiguration(configs: util.Map[String, _]): Unit = {
+      val value = maxConnections(configs)
+      if (value <= 0)
+        throw new ConfigException("Invalid max.connections $listenerMax")
+    }
+
+    override def reconfigure(configs: util.Map[String, _]): Unit = {
+      lock.synchronized {
+        _maxConnections = maxConnections(configs)
+        lock.notifyAll()
+      }
+    }
+
+    private def maxConnections(configs: util.Map[String, _]): Int = {
+      Option(configs.get(KafkaConfig.MaxConnectionsProp)).map(_.toString.toInt).getOrElse(Int.MaxValue)
+    }
+  }
 }
 
 class TooManyConnectionsException(val ip: InetAddress, val count: Int) extends KafkaException(s"Too many connections from $ip (maximum = $count)")
diff --git a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
index b02b842b95494..4f27226265e69 100755
--- a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
+++ b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
@@ -23,6 +23,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock
 
 import kafka.cluster.EndPoint
 import kafka.log.{LogCleaner, LogConfig, LogManager}
+import kafka.network.SocketServer
 import kafka.server.DynamicBrokerConfig._
 import kafka.utils.{CoreUtils, Logging, PasswordEncoder}
 import kafka.zk.{AdminZkClient, KafkaZkClient}
@@ -81,10 +82,11 @@ object DynamicBrokerConfig {
     DynamicThreadPool.ReconfigurableConfigs ++
     Set(KafkaConfig.MetricReporterClassesProp) ++
     DynamicListenerConfig.ReconfigurableConfigs ++
-    DynamicConnectionQuota.ReconfigurableConfigs
+    SocketServer.ReconfigurableConfigs
 
+  private val ClusterLevelListenerConfigs = Set(KafkaConfig.MaxConnectionsProp)
   private val PerBrokerConfigs = DynamicSecurityConfigs  ++
-    DynamicListenerConfig.ReconfigurableConfigs
+    DynamicListenerConfig.ReconfigurableConfigs -- ClusterLevelListenerConfigs
   private val ListenerMechanismConfigs = Set(KafkaConfig.SaslJaasConfigProp)
 
   private val ReloadableFileConfigs = Set(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)
@@ -135,7 +137,13 @@ object DynamicBrokerConfig {
 
   private def perBrokerConfigs(props: Properties): Set[String] = {
     val configNames = props.asScala.keySet
-    configNames.intersect(PerBrokerConfigs) ++ configNames.filter(ListenerConfigRegex.findFirstIn(_).nonEmpty)
+    def perBrokerListenerConfig(name: String): Boolean = {
+      name match {
+        case ListenerConfigRegex(baseName) => !ClusterLevelListenerConfigs.contains(baseName)
+        case _ => false
+      }
+    }
+    configNames.intersect(PerBrokerConfigs) ++ configNames.filter(perBrokerListenerConfig)
   }
 
   private def nonDynamicConfigs(props: Properties): Set[String] = {
@@ -224,7 +232,7 @@ class DynamicBrokerConfig(private val kafkaConfig: KafkaConfig) extends Logging
       addBrokerReconfigurable(kafkaServer.logManager.cleaner)
     addBrokerReconfigurable(new DynamicLogConfig(kafkaServer.logManager, kafkaServer))
     addBrokerReconfigurable(new DynamicListenerConfig(kafkaServer))
-    addBrokerReconfigurable(new DynamicConnectionQuota(kafkaServer))
+    addBrokerReconfigurable(kafkaServer.socketServer)
   }
 
   def addReconfigurable(reconfigurable: Reconfigurable): Unit = CoreUtils.inWriteLock(lock) {
@@ -789,7 +797,10 @@ object DynamicListenerConfig {
     KafkaConfig.SaslLoginRefreshWindowFactorProp,
     KafkaConfig.SaslLoginRefreshWindowJitterProp,
     KafkaConfig.SaslLoginRefreshMinPeriodSecondsProp,
-    KafkaConfig.SaslLoginRefreshBufferSecondsProp
+    KafkaConfig.SaslLoginRefreshBufferSecondsProp,
+
+    // Connection limit configs
+    KafkaConfig.MaxConnectionsProp
   )
 }
 
@@ -884,24 +895,5 @@ class DynamicListenerConfig(server: KafkaServer) extends BrokerReconfigurable wi
 
 }
 
-object DynamicConnectionQuota {
-  val ReconfigurableConfigs = Set(KafkaConfig.MaxConnectionsPerIpProp, KafkaConfig.MaxConnectionsPerIpOverridesProp)
-}
-
-class DynamicConnectionQuota(server: KafkaServer) extends BrokerReconfigurable {
-
-  override def reconfigurableConfigs: Set[String] = {
-    DynamicConnectionQuota.ReconfigurableConfigs
-  }
-
-  override def validateReconfiguration(newConfig: KafkaConfig): Unit = {
-  }
-
-  override def reconfigure(oldConfig: KafkaConfig, newConfig: KafkaConfig): Unit = {
-    server.socketServer.updateMaxConnectionsPerIpOverride(newConfig.maxConnectionsPerIpOverrides)
 
-    if (newConfig.maxConnectionsPerIp != oldConfig.maxConnectionsPerIp)
-      server.socketServer.updateMaxConnectionsPerIp(newConfig.maxConnectionsPerIp)
-  }
-}
 
diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index 02e171527e501..8ab7b43755074 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -75,6 +75,7 @@ object Defaults {
   val SocketRequestMaxBytes: Int = 100 * 1024 * 1024
   val MaxConnectionsPerIp: Int = Int.MaxValue
   val MaxConnectionsPerIpOverrides: String = ""
+  val MaxConnections: Int = Int.MaxValue
   val ConnectionsMaxIdleMs = 10 * 60 * 1000L
   val RequestTimeoutMs = 30000
   val FailedAuthenticationDelayMs = 100
@@ -293,6 +294,7 @@ object KafkaConfig {
   val SocketRequestMaxBytesProp = "socket.request.max.bytes"
   val MaxConnectionsPerIpProp = "max.connections.per.ip"
   val MaxConnectionsPerIpOverridesProp = "max.connections.per.ip.overrides"
+  val MaxConnectionsProp = "max.connections"
   val ConnectionsMaxIdleMsProp = "connections.max.idle.ms"
   val FailedAuthenticationDelayMsProp = "connection.failed.authentication.delay.ms"
   /***************** rack configuration *************/
@@ -570,8 +572,15 @@ object KafkaConfig {
   val SocketReceiveBufferBytesDoc = "The SO_RCVBUF buffer of the socket server sockets. If the value is -1, the OS default will be used."
   val SocketRequestMaxBytesDoc = "The maximum number of bytes in a socket request"
   val MaxConnectionsPerIpDoc = "The maximum number of connections we allow from each ip address. This can be set to 0 if there are overrides " +
-    "configured using " + MaxConnectionsPerIpOverridesProp + " property"
-  val MaxConnectionsPerIpOverridesDoc = "A comma-separated list of per-ip or hostname overrides to the default maximum number of connections. An example value is \"hostName:100,127.0.0.1:200\""
+    s"configured using $MaxConnectionsPerIpOverridesProp property. New connections from the ip address are dropped if the limit is reached."
+  val MaxConnectionsPerIpOverridesDoc = "A comma-separated list of per-ip or hostname overrides to the default maximum number of connections. " +
+    "An example value is \"hostName:100,127.0.0.1:200\""
+  val MaxConnectionsDoc = "The maximum number of connections we allow in the broker at any time. This limit is applied in addition " +
+    s"to any per-ip limits configured using $MaxConnectionsPerIpProp. Listener-level limits may also be configured by prefixing the " +
+    s"config name with the listener prefix, for example, listener.name.internal.$MaxConnectionsProp. Broker-wide limit " +
+    "should be configured based on broker capacity while listener limits should be configured based on application requirements. " +
+    "New connections are blocked if either the listener or broker limit is reached. Connections on the inter-broker listener are " +
+    "permitted even if broker-wide limit is reached. The least recently used connection on another listener will be closed in this case."
   val ConnectionsMaxIdleMsDoc = "Idle connections timeout: the server socket processor threads close the connections that idle more than this"
   val FailedAuthenticationDelayMsDoc = "Connection close delay on failed authentication: this is the time (in milliseconds) by which connection close will be delayed on authentication failure. " +
     s"This must be configured to be less than $ConnectionsMaxIdleMsProp to prevent connection timeout."
@@ -872,6 +881,7 @@ object KafkaConfig {
       .define(SocketRequestMaxBytesProp, INT, Defaults.SocketRequestMaxBytes, atLeast(1), HIGH, SocketRequestMaxBytesDoc)
       .define(MaxConnectionsPerIpProp, INT, Defaults.MaxConnectionsPerIp, atLeast(0), MEDIUM, MaxConnectionsPerIpDoc)
       .define(MaxConnectionsPerIpOverridesProp, STRING, Defaults.MaxConnectionsPerIpOverrides, MEDIUM, MaxConnectionsPerIpOverridesDoc)
+      .define(MaxConnectionsProp, INT, Defaults.MaxConnections, atLeast(0), MEDIUM, MaxConnectionsDoc)
       .define(ConnectionsMaxIdleMsProp, LONG, Defaults.ConnectionsMaxIdleMs, MEDIUM, ConnectionsMaxIdleMsDoc)
       .define(FailedAuthenticationDelayMsProp, INT, Defaults.FailedAuthenticationDelayMs, atLeast(0), LOW, FailedAuthenticationDelayMsDoc)
 
@@ -1163,6 +1173,7 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO
   val maxConnectionsPerIp = getInt(KafkaConfig.MaxConnectionsPerIpProp)
   val maxConnectionsPerIpOverrides: Map[String, Int] =
     getMap(KafkaConfig.MaxConnectionsPerIpOverridesProp, getString(KafkaConfig.MaxConnectionsPerIpOverridesProp)).map { case (k, v) => (k, v.toInt)}
+  def maxConnections = getInt(KafkaConfig.MaxConnectionsProp)
   val connectionsMaxIdleMs = getLong(KafkaConfig.ConnectionsMaxIdleMsProp)
   val failedAuthenticationDelayMs = getInt(KafkaConfig.FailedAuthenticationDelayMsProp)
 
@@ -1331,6 +1342,10 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO
     dynamicConfig.addReconfigurable(reconfigurable)
   }
 
+  def removeReconfigurable(reconfigurable: Reconfigurable): Unit = {
+    dynamicConfig.removeReconfigurable(reconfigurable)
+  }
+
   def logRetentionTimeMillis: Long = {
     val millisInMinute = 60L * 1000L
     val millisInHour = 60L * millisInMinute
diff --git a/core/src/test/scala/integration/kafka/network/DynamicConnectionQuotaTest.scala b/core/src/test/scala/integration/kafka/network/DynamicConnectionQuotaTest.scala
index 78d9af7bdfca3..514d879a1d966 100644
--- a/core/src/test/scala/integration/kafka/network/DynamicConnectionQuotaTest.scala
+++ b/core/src/test/scala/integration/kafka/network/DynamicConnectionQuotaTest.scala
@@ -21,18 +21,19 @@ package kafka.network
 import java.io.IOException
 import java.net.{InetAddress, Socket}
 import java.util.Properties
+import java.util.concurrent._
 
 import kafka.server.{BaseRequestTest, KafkaConfig}
-import kafka.utils.TestUtils
+import kafka.utils.{CoreUtils, TestUtils}
 import org.apache.kafka.clients.admin.{AdminClient, AdminClientConfig}
-import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.{KafkaException, TopicPartition}
 import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
 import org.apache.kafka.common.record.{CompressionType, MemoryRecords, SimpleRecord}
 import org.apache.kafka.common.requests.{ProduceRequest, ProduceResponse}
 import org.apache.kafka.common.security.auth.SecurityProtocol
-import org.junit.Assert.assertEquals
-import org.junit.{Before, Test}
+import org.junit.Assert._
+import org.junit.{After, Before, Test}
 
 import scala.collection.JavaConverters._
 
@@ -41,6 +42,9 @@ class DynamicConnectionQuotaTest extends BaseRequestTest {
   override def numBrokers = 1
 
   val topic = "test"
+  val listener = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)
+  val localAddress = InetAddress.getByName("127.0.0.1")
+  var executor: ExecutorService = _
 
   @Before
   override def setUp(): Unit = {
@@ -48,76 +52,138 @@ class DynamicConnectionQuotaTest extends BaseRequestTest {
     TestUtils.createTopic(zkClient, topic, numBrokers, numBrokers, servers)
   }
 
-  @Test
-  def testDynamicConnectionQuota(): Unit = {
-    def connect(socketServer: SocketServer, protocol: SecurityProtocol = SecurityProtocol.PLAINTEXT, localAddr: InetAddress = null) = {
-      new Socket("localhost", socketServer.boundPort(ListenerName.forSecurityProtocol(protocol)), localAddr, 0)
+  @After
+  override def tearDown(): Unit = {
+    try {
+      if (executor != null) {
+        executor.shutdownNow()
+        assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS))
+      }
+    } finally {
+      super.tearDown()
     }
+  }
 
-    val socketServer = servers.head.socketServer
-    val localAddress = InetAddress.getByName("127.0.0.1")
-    def connectionCount = socketServer.connectionCount(localAddress)
+  override protected def propertyOverrides(properties: Properties): Unit = {
+    super.propertyOverrides(properties)
+  }
+
+  @Test
+  def testDynamicConnectionQuota(): Unit = {
     val initialConnectionCount = connectionCount
     val maxConnectionsPerIP = 5
 
+    def connectAndVerify: Unit = {
+      val socket = connect()
+      try {
+        sendAndReceive(produceRequest, ApiKeys.PRODUCE, socket)
+      } finally {
+        socket.close()
+      }
+    }
+
     val props = new Properties
     props.put(KafkaConfig.MaxConnectionsPerIpProp, maxConnectionsPerIP.toString)
     reconfigureServers(props, perBrokerConfig = false, (KafkaConfig.MaxConnectionsPerIpProp, maxConnectionsPerIP.toString))
 
-    //wait for adminClient connections to close
-    TestUtils.waitUntilTrue(() => initialConnectionCount == connectionCount, "Connection count mismatch")
-
-    //create connections up to maxConnectionsPerIP - 1, leave space for one connection
-    var conns = (connectionCount until (maxConnectionsPerIP - 1)).map(_ => connect(socketServer))
-
-    // produce should succeed
-    var produceResponse = sendProduceRequest()
-    assertEquals(1, produceResponse.responses.size)
-    val (tp, partitionResponse) = produceResponse.responses.asScala.head
-    assertEquals(Errors.NONE, partitionResponse.error)
-
-    TestUtils.waitUntilTrue(() => connectionCount == (maxConnectionsPerIP - 1), "produce request connection is not closed")
-    conns = conns :+ connect(socketServer)
-    // now try one more (should fail)
-    intercept[IOException](sendProduceRequest())
-
-    conns.foreach(conn => conn.close())
-    TestUtils.waitUntilTrue(() => initialConnectionCount == connectionCount, "Connection count mismatch")
+    verifyMaxConnections(maxConnectionsPerIP, () => connectAndVerify)
 
     // Increase MaxConnectionsPerIpOverrides for localhost to 7
     val maxConnectionsPerIPOverride = 7
     props.put(KafkaConfig.MaxConnectionsPerIpOverridesProp, s"localhost:$maxConnectionsPerIPOverride")
     reconfigureServers(props, perBrokerConfig = false, (KafkaConfig.MaxConnectionsPerIpOverridesProp, s"localhost:$maxConnectionsPerIPOverride"))
 
-    //wait for adminClient connections to close
-    TestUtils.waitUntilTrue(() => initialConnectionCount == connectionCount, "Connection count mismatch")
+    verifyMaxConnections(maxConnectionsPerIPOverride, () => connectAndVerify)
+  }
 
-    //create connections up to maxConnectionsPerIPOverride - 1, leave space for one connection
-    conns = (connectionCount until maxConnectionsPerIPOverride - 1).map(_ => connect(socketServer))
+  @Test
+  def testDynamicListenerConnectionQuota(): Unit = {
+    val socketServer = servers.head.socketServer
+    val initialConnectionCount = connectionCount
 
-    // send should succeed
-    produceResponse = sendProduceRequest()
-    assertEquals(1, produceResponse.responses.size)
-    val (tp1, partitionResponse1) = produceResponse.responses.asScala.head
-    assertEquals(Errors.NONE, partitionResponse1.error)
+    def connectAndVerify(): Unit = {
+      val socket = connect("PLAINTEXT")
+      socket.setSoTimeout(1000)
+      try {
+        val response = sendAndReceive(produceRequest, ApiKeys.PRODUCE, socket)
+        assertEquals(0, response.remaining)
+      } finally {
+        socket.close()
+      }
+    }
 
-    TestUtils.waitUntilTrue(() => connectionCount == (maxConnectionsPerIPOverride - 1), "produce request connection is not closed")
-    conns = conns :+ connect(socketServer)
-    // now try one more (should fail)
-    intercept[IOException](sendProduceRequest())
+    // Reduce total broker MaxConnections to 5 at the cluster level
+    val props = new Properties
+    props.put(KafkaConfig.MaxConnectionsProp, "5")
+    reconfigureServers(props, perBrokerConfig = false, (KafkaConfig.MaxConnectionsProp, "5"))
+    verifyMaxConnections(5, () => connectAndVerify)
 
-    //close one connection
-    conns.head.close()
-    TestUtils.waitUntilTrue(() => connectionCount == (maxConnectionsPerIPOverride - 1), "connection is not closed")
-    // send should succeed
-    sendProduceRequest()
+    // Create another listener and verify listener connection limit of 5 for each listener
+    val newListeners = "PLAINTEXT://localhost:0,INTERNAL://localhost:0"
+    props.put(KafkaConfig.ListenersProp, newListeners)
+    props.put(KafkaConfig.ListenerSecurityProtocolMapProp, "PLAINTEXT:PLAINTEXT,INTERNAL:PLAINTEXT")
+    props.put(KafkaConfig.MaxConnectionsProp, "10")
+    props.put("listener.name.internal.max.connections", "5")
+    props.put("listener.name.plaintext.max.connections", "5")
+    reconfigureServers(props, perBrokerConfig = true, (KafkaConfig.ListenersProp, newListeners))
+    waitForListener("INTERNAL")
+
+    var conns = (connectionCount until 5).map(_ => connect("PLAINTEXT"))
+    conns ++= (5 until 10).map(_ => connect("INTERNAL"))
+    conns.foreach(verifyConnection)
+    conns.foreach(_.close())
+    TestUtils.waitUntilTrue(() => initialConnectionCount == connectionCount, "Connections not closed")
+
+    // Increase MaxConnections for PLAINTEXT listener to 7 at the broker level
+    val maxConnectionsPlaintext = 7
+    val listenerProp = s"${listener.configPrefix}${KafkaConfig.MaxConnectionsProp}"
+    props.put(listenerProp, maxConnectionsPlaintext.toString)
+    reconfigureServers(props, perBrokerConfig = true, (listenerProp, maxConnectionsPlaintext.toString))
+    verifyMaxConnections(maxConnectionsPlaintext, () => connectAndVerify)
+
+    // Verify that connection blocked on the limit connects successfully when an existing connection is closed
+    val plaintextConnections = (connectionCount until maxConnectionsPlaintext).map(_ => connect("PLAINTEXT"))
+    executor = Executors.newSingleThreadExecutor
+    val future = executor.submit(CoreUtils.runnable { createAndVerifyConnection() })
+    Thread.sleep(100)
+    assertFalse(future.isDone)
+    plaintextConnections.head.close()
+    future.get(30, TimeUnit.SECONDS)
+    plaintextConnections.foreach(_.close())
+    TestUtils.waitUntilTrue(() => initialConnectionCount == connectionCount, "Connections not closed")
+
+    // Verify that connections on inter-broker listener succeed even if broker max connections has been
+    // reached by closing connections on another listener
+    var plaintextConns = (connectionCount until 5).map(_ => connect("PLAINTEXT"))
+    val internalConns = (5 until 10).map(_ => connect("INTERNAL"))
+    plaintextConns.foreach(verifyConnection)
+    internalConns.foreach(verifyConnection)
+    plaintextConns ++= (0 until 2).map(_ => connect("PLAINTEXT"))
+    TestUtils.waitUntilTrue(() => connectionCount <= 10, "Internal connections not closed")
+    plaintextConns.foreach(verifyConnection)
+    intercept[IOException](internalConns.foreach { socket => sendAndReceive(produceRequest, ApiKeys.PRODUCE, socket) })
+    plaintextConns.foreach(_.close())
+    internalConns.foreach(_.close())
+    TestUtils.waitUntilTrue(() => initialConnectionCount == connectionCount, "Connections not closed")
   }
 
   private def reconfigureServers(newProps: Properties, perBrokerConfig: Boolean, aPropToVerify: (String, String)): Unit = {
+    val initialConnectionCount = connectionCount
     val adminClient = createAdminClient()
     TestUtils.alterConfigs(servers, adminClient, newProps, perBrokerConfig).all.get()
     waitForConfigOnServer(aPropToVerify._1, aPropToVerify._2)
     adminClient.close()
+    TestUtils.waitUntilTrue(() => initialConnectionCount == connectionCount, "Admin client connection not closed")
+  }
+
+  private def waitForListener(listenerName: String): Unit = {
+    TestUtils.retry(maxWaitMs = 10000) {
+      try {
+        assertTrue(servers.head.socketServer.boundPort(ListenerName.normalised(listenerName)) > 0)
+      } catch {
+        case e: KafkaException => throw new AssertionError(e)
+      }
+    }
   }
 
   private def createAdminClient(): AdminClient = {
@@ -135,12 +201,59 @@ class DynamicConnectionQuotaTest extends BaseRequestTest {
     }
   }
 
-  private def sendProduceRequest(): ProduceResponse = {
+  private def produceRequest: ProduceRequest = {
     val topicPartition = new TopicPartition(topic, 0)
     val memoryRecords = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord(System.currentTimeMillis(), "key".getBytes, "value".getBytes))
     val partitionRecords = Map(topicPartition -> memoryRecords)
-    val request = ProduceRequest.Builder.forCurrentMagic(-1, 3000, partitionRecords.asJava).build()
-    val response = connectAndSend(request, ApiKeys.PRODUCE, servers.head.socketServer)
-    ProduceResponse.parse(response, request.version)
+    ProduceRequest.Builder.forCurrentMagic(-1, 3000, partitionRecords.asJava).build()
+  }
+
+  def connectionCount: Int = servers.head.socketServer.connectionCount(localAddress)
+
+  def connect(listener: String): Socket = {
+    val listenerName = ListenerName.normalised(listener)
+    new Socket("localhost", servers.head.socketServer.boundPort(listenerName))
+  }
+
+  private def createAndVerifyConnection(listener: String = "PLAINTEXT"): Unit = {
+    val socket = connect(listener)
+    try {
+      verifyConnection(socket)
+    } finally {
+      socket.close()
+    }
+  }
+
+  private def verifyConnection(socket: Socket): Unit = {
+    val request = produceRequest
+    val response = sendAndReceive(request, ApiKeys.PRODUCE, socket)
+    val produceResponse = ProduceResponse.parse(response, request.version)
+    assertEquals(1, produceResponse.responses.size)
+    val (_, partitionResponse) = produceResponse.responses.asScala.head
+    assertEquals(Errors.NONE, partitionResponse.error)
+  }
+
+  private def verifyMaxConnections(maxConnections: Int, connectWithFailure: () => Unit): Unit = {
+    val initialConnectionCount = connectionCount
+
+    //create connections up to maxConnectionsPerIP - 1, leave space for one connection
+    var conns = (connectionCount until (maxConnections - 1)).map(_ => connect("PLAINTEXT"))
+
+    // produce should succeed on a new connection
+    createAndVerifyConnection()
+
+    TestUtils.waitUntilTrue(() => connectionCount == (maxConnections - 1), "produce request connection is not closed")
+    conns = conns :+ connect("PLAINTEXT")
+
+    // now try one more (should fail)
+    intercept[IOException](connectWithFailure.apply())
+
+    //close one connection
+    conns.head.close()
+    TestUtils.waitUntilTrue(() => connectionCount == (maxConnections - 1), "connection is not closed")
+    createAndVerifyConnection()
+
+    conns.foreach(_.close())
+    TestUtils.waitUntilTrue(() => initialConnectionCount == connectionCount, "Connections not closed")
   }
 }
diff --git a/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala b/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala
index 45ef18f5187f0..5d20da64cb661 100755
--- a/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala
+++ b/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala
@@ -186,6 +186,12 @@ class DynamicBrokerConfigTest extends JUnitSuite {
     //test invalid address
     verifyConfigUpdate(KafkaConfig.MaxConnectionsPerIpOverridesProp, "hostName#:100", perBrokerConfig = true,
       expectFailure = true)
+
+    verifyConfigUpdate(KafkaConfig.MaxConnectionsProp, "100", perBrokerConfig = true, expectFailure = false)
+    verifyConfigUpdate(KafkaConfig.MaxConnectionsProp, "100", perBrokerConfig = false, expectFailure = false)
+    val listenerMaxConnectionsProp = s"listener.name.external.${KafkaConfig.MaxConnectionsProp}"
+    verifyConfigUpdate(listenerMaxConnectionsProp, "10", perBrokerConfig = true, expectFailure = false)
+    verifyConfigUpdate(listenerMaxConnectionsProp, "10", perBrokerConfig = false, expectFailure = false)
   }
 
   private def verifyConfigUpdate(name: String, value: Object, perBrokerConfig: Boolean, expectFailure: Boolean) {

From 4ca8e40e2f4ab415657d1e07ee868448802f4565 Mon Sep 17 00:00:00 2001
From: Lee Dongjin 
Date: Fri, 15 Mar 2019 10:42:07 +0900
Subject: [PATCH 0021/1071] KAFKA-7502: Cleanup KTable materialization logic in
 a single place (#6174)

This is a draft cleanup for KAFKA-7502. Here is the details:

* Make KTableKTableJoinNode abstract, and define its child classes ([NonMaterialized,Materialized]KTableKTableJoinNode) instead: now, all materialization-related routines are separated into the other classes.

* KTableKTableJoinNodeBuilder#build now instantiates [NonMaterialized,Materialized]KTableKTableJoinNode classes instead of KTableKTableJoinNode.

Reviewers: Guozhang Wang ,  Bill Bejeck 
---
 .../streams/kstream/internals/KTableImpl.java | 106 +++++------
 .../internals/KTableKTableJoinMerger.java     |  17 +-
 .../internals/graph/KTableKTableJoinNode.java | 172 +++++++++++-------
 3 files changed, 170 insertions(+), 125 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
index 68f940c905e6f..d97213671e149 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
@@ -43,6 +43,7 @@
 import org.apache.kafka.streams.kstream.internals.suppress.SuppressedInternal;
 import org.apache.kafka.streams.processor.ProcessorSupplier;
 import org.apache.kafka.streams.state.KeyValueStore;
+import org.apache.kafka.streams.state.StoreBuilder;
 import org.apache.kafka.streams.state.internals.InMemoryTimeOrderedKeyValueBuffer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -465,28 +466,9 @@ private  KTable doJoin(final KTable other,
                                           final boolean rightOuter) {
         Objects.requireNonNull(other, "other can't be null");
         Objects.requireNonNull(joiner, "joiner can't be null");
-        final String internalQueryableName = materializedInternal == null ? null : materializedInternal.storeName();
-        final String joinMergeName = builder.newProcessorName(MERGE_NAME);
 
-        return buildJoin(
-            (AbstractStream) other,
-            joiner,
-            leftOuter,
-            rightOuter,
-            joinMergeName,
-            internalQueryableName,
-            materializedInternal
-        );
-    }
-
-    private  KTable buildJoin(final AbstractStream other,
-                                           final ValueJoiner joiner,
-                                           final boolean leftOuter,
-                                           final boolean rightOuter,
-                                           final String joinMergeName,
-                                           final String internalQueryableName,
-                                           final MaterializedInternal> materializedInternal) {
-        final Set allSourceNodes = ensureJoinableWith(other);
+        final String joinMergeName = builder.newProcessorName(MERGE_NAME);
+        final Set allSourceNodes = ensureJoinableWith((AbstractStream) other);
 
         if (leftOuter) {
             enableSendingOldValues();
@@ -495,57 +477,67 @@ private  KTable buildJoin(final AbstractStream other,
             ((KTableImpl) other).enableSendingOldValues();
         }
 
-        final String joinThisName = builder.newProcessorName(JOINTHIS_NAME);
-        final String joinOtherName = builder.newProcessorName(JOINOTHER_NAME);
-
-
-        final KTableKTableAbstractJoin joinThis;
-        final KTableKTableAbstractJoin joinOther;
+        final KTableKTableAbstractJoin joinThis;
+        final KTableKTableAbstractJoin joinOther;
 
         if (!leftOuter) { // inner
-            joinThis = new KTableKTableInnerJoin<>(this, (KTableImpl) other, joiner);
-            joinOther = new KTableKTableInnerJoin<>((KTableImpl) other, this, reverseJoiner(joiner));
+            joinThis = new KTableKTableInnerJoin<>(this, (KTableImpl) other, joiner);
+            joinOther = new KTableKTableInnerJoin<>((KTableImpl) other, this, reverseJoiner(joiner));
         } else if (!rightOuter) { // left
-            joinThis = new KTableKTableLeftJoin<>(this, (KTableImpl) other, joiner);
-            joinOther = new KTableKTableRightJoin<>((KTableImpl) other, this, reverseJoiner(joiner));
+            joinThis = new KTableKTableLeftJoin<>(this, (KTableImpl) other, joiner);
+            joinOther = new KTableKTableRightJoin<>((KTableImpl) other, this, reverseJoiner(joiner));
         } else { // outer
-            joinThis = new KTableKTableOuterJoin<>(this, (KTableImpl) other, joiner);
-            joinOther = new KTableKTableOuterJoin<>((KTableImpl) other, this, reverseJoiner(joiner));
+            joinThis = new KTableKTableOuterJoin<>(this, (KTableImpl) other, joiner);
+            joinOther = new KTableKTableOuterJoin<>((KTableImpl) other, this, reverseJoiner(joiner));
         }
 
-        final KTableKTableJoinMerger joinMerge = new KTableKTableJoinMerger<>(joinThis, joinOther, internalQueryableName);
+        final String joinThisName = builder.newProcessorName(JOINTHIS_NAME);
+        final String joinOtherName = builder.newProcessorName(JOINOTHER_NAME);
+
+        final ProcessorParameters> joinThisProcessorParameters = new ProcessorParameters<>(joinThis, joinThisName);
+        final ProcessorParameters> joinOtherProcessorParameters = new ProcessorParameters<>(joinOther, joinOtherName);
 
-        final KTableKTableJoinNode.KTableKTableJoinNodeBuilder kTableJoinNodeBuilder = KTableKTableJoinNode.kTableKTableJoinNodeBuilder();
+        final Serde keySerde;
+        final Serde valueSerde;
+        final String queryableStoreName;
+        final StoreBuilder> storeBuilder;
 
-        // only materialize if specified in Materialized
         if (materializedInternal != null) {
-            kTableJoinNodeBuilder.withMaterializedInternal(materializedInternal);
+            keySerde = materializedInternal.keySerde() != null ? materializedInternal.keySerde() : this.keySerde;
+            valueSerde = materializedInternal.valueSerde();
+            queryableStoreName = materializedInternal.storeName();
+            storeBuilder = new KeyValueStoreMaterializer<>(materializedInternal).materialize();
+        } else {
+            keySerde = this.keySerde;
+            valueSerde = null;
+            queryableStoreName = null;
+            storeBuilder = null;
         }
-        kTableJoinNodeBuilder.withNodeName(joinMergeName);
 
-        final ProcessorParameters> joinThisProcessorParameters = new ProcessorParameters<>(joinThis, joinThisName);
-        final ProcessorParameters> joinOtherProcessorParameters = new ProcessorParameters<>(joinOther, joinOtherName);
-        final ProcessorParameters> joinMergeProcessorParameters = new ProcessorParameters<>(joinMerge, joinMergeName);
-
-        kTableJoinNodeBuilder.withJoinMergeProcessorParameters(joinMergeProcessorParameters)
-                             .withJoinOtherProcessorParameters(joinOtherProcessorParameters)
-                             .withJoinThisProcessorParameters(joinThisProcessorParameters)
-                             .withJoinThisStoreNames(valueGetterSupplier().storeNames())
-                             .withJoinOtherStoreNames(((KTableImpl) other).valueGetterSupplier().storeNames())
-                             .withOtherJoinSideNodeName(((KTableImpl) other).name)
-                             .withThisJoinSideNodeName(name);
-
-        final KTableKTableJoinNode kTableKTableJoinNode = kTableJoinNodeBuilder.build();
+        final KTableKTableJoinNode kTableKTableJoinNode =
+            KTableKTableJoinNode.kTableKTableJoinNodeBuilder()
+                .withNodeName(joinMergeName)
+                .withJoinThisProcessorParameters(joinThisProcessorParameters)
+                .withJoinOtherProcessorParameters(joinOtherProcessorParameters)
+                .withThisJoinSideNodeName(name)
+                .withOtherJoinSideNodeName(((KTableImpl) other).name)
+                .withJoinThisStoreNames(valueGetterSupplier().storeNames())
+                .withJoinOtherStoreNames(((KTableImpl) other).valueGetterSupplier().storeNames())
+                .withKeySerde(keySerde)
+                .withValueSerde(valueSerde)
+                .withQueryableStoreName(queryableStoreName)
+                .withStoreBuilder(storeBuilder)
+                .build();
         builder.addGraphNode(this.streamsGraphNode, kTableKTableJoinNode);
 
         // we can inherit parent key serde if user do not provide specific overrides
-        return new KTableImpl, R>(
-            joinMergeName,
-            materializedInternal != null && materializedInternal.keySerde() != null ? materializedInternal.keySerde() : keySerde,
-            materializedInternal != null ? materializedInternal.valueSerde() : null,
+        return new KTableImpl, VR>(
+            kTableKTableJoinNode.nodeName(),
+            kTableKTableJoinNode.keySerde(),
+            kTableKTableJoinNode.valueSerde(),
             allSourceNodes,
-            internalQueryableName,
-            joinMerge,
+            kTableKTableJoinNode.queryableStoreName(),
+            kTableKTableJoinNode.joinMerger(),
             kTableKTableJoinNode,
             builder
         );
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableJoinMerger.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableJoinMerger.java
index 78c1dc6f48001..de3804246124c 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableJoinMerger.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableJoinMerger.java
@@ -25,7 +25,7 @@
 import java.util.HashSet;
 import java.util.Set;
 
-class KTableKTableJoinMerger implements KTableProcessorSupplier {
+public class KTableKTableJoinMerger implements KTableProcessorSupplier {
 
     private final KTableProcessorSupplier parent1;
     private final KTableProcessorSupplier parent2;
@@ -40,6 +40,10 @@ class KTableKTableJoinMerger implements KTableProcessorSupplier {
         this.queryableName = queryableName;
     }
 
+    public String getQueryableName() {
+        return queryableName;
+    }
+
     @Override
     public Processor> get() {
         return new KTableKTableJoinMergeProcessor();
@@ -78,6 +82,17 @@ public void enableSendingOldValues() {
         sendOldValues = true;
     }
 
+    public static  KTableKTableJoinMerger of(final KTableProcessorSupplier parent1,
+                                                         final KTableProcessorSupplier parent2) {
+        return of(parent1, parent2, null);
+    }
+
+    public static  KTableKTableJoinMerger of(final KTableProcessorSupplier parent1,
+                                                         final KTableProcessorSupplier parent2,
+                                                         final String queryableName) {
+        return new KTableKTableJoinMerger<>(parent1, parent2, queryableName);
+    }
+
     private class KTableKTableJoinMergeProcessor extends AbstractProcessor> {
         private KeyValueStore store;
         private TupleForwarder tupleForwarder;
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/KTableKTableJoinNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/KTableKTableJoinNode.java
index aeda0d9e19d13..03bdda0b2d844 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/KTableKTableJoinNode.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/KTableKTableJoinNode.java
@@ -17,11 +17,10 @@
 
 package org.apache.kafka.streams.kstream.internals.graph;
 
-import org.apache.kafka.common.utils.Bytes;
-import org.apache.kafka.streams.kstream.ValueJoiner;
+import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.streams.kstream.internals.Change;
-import org.apache.kafka.streams.kstream.internals.KeyValueStoreMaterializer;
-import org.apache.kafka.streams.kstream.internals.MaterializedInternal;
+import org.apache.kafka.streams.kstream.internals.KTableKTableJoinMerger;
+import org.apache.kafka.streams.kstream.internals.KTableProcessorSupplier;
 import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder;
 import org.apache.kafka.streams.state.KeyValueStore;
 import org.apache.kafka.streams.state.StoreBuilder;
@@ -33,32 +32,64 @@
  */
 public class KTableKTableJoinNode extends BaseJoinProcessorNode, Change, Change> {
 
+    private final Serde keySerde;
+    private final Serde valueSerde;
     private final String[] joinThisStoreNames;
     private final String[] joinOtherStoreNames;
-    private final MaterializedInternal> materializedInternal;
+    private final StoreBuilder> storeBuilder;
 
     KTableKTableJoinNode(final String nodeName,
-                         final ValueJoiner, ? super Change, ? extends Change> valueJoiner,
                          final ProcessorParameters> joinThisProcessorParameters,
                          final ProcessorParameters> joinOtherProcessorParameters,
                          final ProcessorParameters> joinMergeProcessorParameters,
-                         final MaterializedInternal> materializedInternal,
                          final String thisJoinSide,
                          final String otherJoinSide,
+                         final Serde keySerde,
+                         final Serde valueSerde,
                          final String[] joinThisStoreNames,
-                         final String[] joinOtherStoreNames) {
+                         final String[] joinOtherStoreNames,
+                         final StoreBuilder> storeBuilder) {
 
         super(nodeName,
-              valueJoiner,
-              joinThisProcessorParameters,
-              joinOtherProcessorParameters,
-              joinMergeProcessorParameters,
-              thisJoinSide,
-              otherJoinSide);
-
+            null,
+            joinThisProcessorParameters,
+            joinOtherProcessorParameters,
+            joinMergeProcessorParameters,
+            thisJoinSide,
+            otherJoinSide);
+
+        this.keySerde = keySerde;
+        this.valueSerde = valueSerde;
         this.joinThisStoreNames = joinThisStoreNames;
         this.joinOtherStoreNames = joinOtherStoreNames;
-        this.materializedInternal = materializedInternal;
+        this.storeBuilder = storeBuilder;
+    }
+
+    public Serde keySerde() {
+        return keySerde;
+    }
+
+    public Serde valueSerde() {
+        return valueSerde;
+    }
+
+    public String[] joinThisStoreNames() {
+        return joinThisStoreNames;
+    }
+
+    public String[] joinOtherStoreNames() {
+        return joinOtherStoreNames;
+    }
+
+    public String queryableStoreName() {
+        return ((KTableKTableJoinMerger) mergeProcessorParameters().processorSupplier()).getQueryableName();
+    }
+
+    /**
+     * The supplier which provides processor with KTable-KTable join merge functionality.
+     */
+    public KTableKTableJoinMerger joinMerger() {
+        return (KTableKTableJoinMerger) mergeProcessorParameters().processorSupplier();
     }
 
     @Override
@@ -68,26 +99,24 @@ public void writeToTopology(final InternalTopologyBuilder topologyBuilder) {
         final String mergeProcessorName = mergeProcessorParameters().processorName();
 
         topologyBuilder.addProcessor(thisProcessorName,
-                                     thisProcessorParameters().processorSupplier(),
-                                     thisJoinSideNodeName());
+            thisProcessorParameters().processorSupplier(),
+            thisJoinSideNodeName());
 
         topologyBuilder.addProcessor(otherProcessorName,
-                                     otherProcessorParameters().processorSupplier(),
-                                     otherJoinSideNodeName());
+            otherProcessorParameters().processorSupplier(),
+            otherJoinSideNodeName());
 
         topologyBuilder.addProcessor(mergeProcessorName,
-                                     mergeProcessorParameters().processorSupplier(),
-                                     thisProcessorName,
-                                     otherProcessorName);
+            mergeProcessorParameters().processorSupplier(),
+            thisProcessorName,
+            otherProcessorName);
 
         topologyBuilder.connectProcessorAndStateStores(thisProcessorName,
-                                                       joinOtherStoreNames);
+            joinOtherStoreNames);
         topologyBuilder.connectProcessorAndStateStores(otherProcessorName,
-                                                       joinThisStoreNames);
+            joinThisStoreNames);
 
-        if (materializedInternal != null) {
-            final StoreBuilder> storeBuilder =
-                new KeyValueStoreMaterializer<>(materializedInternal).materialize();
+        if (storeBuilder != null) {
             topologyBuilder.addStateStore(storeBuilder, mergeProcessorName);
         }
     }
@@ -95,10 +124,9 @@ public void writeToTopology(final InternalTopologyBuilder topologyBuilder) {
     @Override
     public String toString() {
         return "KTableKTableJoinNode{" +
-               "joinThisStoreNames=" + Arrays.toString(joinThisStoreNames) +
-               ", joinOtherStoreNames=" + Arrays.toString(joinOtherStoreNames) +
-               ", materializedInternal=" + materializedInternal +
-               "} " + super.toString();
+            "joinThisStoreNames=" + Arrays.toString(joinThisStoreNames()) +
+            ", joinOtherStoreNames=" + Arrays.toString(joinOtherStoreNames()) +
+            "} " + super.toString();
     }
 
     public static  KTableKTableJoinNodeBuilder kTableKTableJoinNodeBuilder() {
@@ -106,23 +134,23 @@ public static  KTableKTableJoinNodeBuilder kTableK
     }
 
     public static final class KTableKTableJoinNodeBuilder {
-
         private String nodeName;
-        private String[] joinThisStoreNames;
         private ProcessorParameters> joinThisProcessorParameters;
-        private String[] joinOtherStoreNames;
-        private MaterializedInternal> materializedInternal;
         private ProcessorParameters> joinOtherProcessorParameters;
-        private ProcessorParameters> joinMergeProcessorParameters;
-        private ValueJoiner, ? super Change, ? extends Change> valueJoiner;
         private String thisJoinSide;
         private String otherJoinSide;
+        private Serde keySerde;
+        private Serde valueSerde;
+        private String[] joinThisStoreNames;
+        private String[] joinOtherStoreNames;
+        private String queryableStoreName;
+        private StoreBuilder> storeBuilder;
 
         private KTableKTableJoinNodeBuilder() {
         }
 
-        public KTableKTableJoinNodeBuilder withJoinThisStoreNames(final String[] joinThisStoreNames) {
-            this.joinThisStoreNames = joinThisStoreNames;
+        public KTableKTableJoinNodeBuilder withNodeName(final String nodeName) {
+            this.nodeName = nodeName;
             return this;
         }
 
@@ -131,59 +159,69 @@ public KTableKTableJoinNodeBuilder withJoinThisProcessorParameter
             return this;
         }
 
-        public KTableKTableJoinNodeBuilder withNodeName(final String nodeName) {
-            this.nodeName = nodeName;
+        public KTableKTableJoinNodeBuilder withJoinOtherProcessorParameters(final ProcessorParameters> joinOtherProcessorParameters) {
+            this.joinOtherProcessorParameters = joinOtherProcessorParameters;
             return this;
         }
 
-        public KTableKTableJoinNodeBuilder withJoinOtherStoreNames(final String[] joinOtherStoreNames) {
-            this.joinOtherStoreNames = joinOtherStoreNames;
+        public KTableKTableJoinNodeBuilder withThisJoinSideNodeName(final String thisJoinSide) {
+            this.thisJoinSide = thisJoinSide;
             return this;
         }
 
-        public KTableKTableJoinNodeBuilder withJoinOtherProcessorParameters(final ProcessorParameters> joinOtherProcessorParameters) {
-            this.joinOtherProcessorParameters = joinOtherProcessorParameters;
+        public KTableKTableJoinNodeBuilder withOtherJoinSideNodeName(final String otherJoinSide) {
+            this.otherJoinSide = otherJoinSide;
             return this;
         }
 
-        public KTableKTableJoinNodeBuilder withJoinMergeProcessorParameters(final ProcessorParameters> joinMergeProcessorParameters) {
-            this.joinMergeProcessorParameters = joinMergeProcessorParameters;
+        public KTableKTableJoinNodeBuilder withKeySerde(final Serde keySerde) {
+            this.keySerde = keySerde;
             return this;
         }
 
-        public KTableKTableJoinNodeBuilder withValueJoiner(final ValueJoiner, ? super Change, ? extends Change> valueJoiner) {
-            this.valueJoiner = valueJoiner;
+        public KTableKTableJoinNodeBuilder withValueSerde(final Serde valueSerde) {
+            this.valueSerde = valueSerde;
             return this;
         }
 
-        public KTableKTableJoinNodeBuilder withThisJoinSideNodeName(final String thisJoinSide) {
-            this.thisJoinSide = thisJoinSide;
+        public KTableKTableJoinNodeBuilder withJoinThisStoreNames(final String[] joinThisStoreNames) {
+            this.joinThisStoreNames = joinThisStoreNames;
             return this;
         }
 
-        public KTableKTableJoinNodeBuilder withOtherJoinSideNodeName(final String otherJoinSide) {
-            this.otherJoinSide = otherJoinSide;
+        public KTableKTableJoinNodeBuilder withJoinOtherStoreNames(final String[] joinOtherStoreNames) {
+            this.joinOtherStoreNames = joinOtherStoreNames;
             return this;
         }
 
-        public KTableKTableJoinNodeBuilder withMaterializedInternal(
-                final MaterializedInternal> materializedInternal) {
-            this.materializedInternal = materializedInternal;
+        public KTableKTableJoinNodeBuilder withQueryableStoreName(final String queryableStoreName) {
+            this.queryableStoreName = queryableStoreName;
             return this;
         }
 
-        public KTableKTableJoinNode build() {
+        public KTableKTableJoinNodeBuilder withStoreBuilder(final StoreBuilder> storeBuilder) {
+            this.storeBuilder = storeBuilder;
+            return this;
+        }
 
+        @SuppressWarnings("unchecked")
+        public KTableKTableJoinNode build() {
             return new KTableKTableJoinNode<>(nodeName,
-                                              valueJoiner,
-                                              joinThisProcessorParameters,
-                                              joinOtherProcessorParameters,
-                                              joinMergeProcessorParameters,
-                                              materializedInternal,
-                                              thisJoinSide,
-                                              otherJoinSide,
-                                              joinThisStoreNames,
-                                              joinOtherStoreNames);
+                joinThisProcessorParameters,
+                joinOtherProcessorParameters,
+                new ProcessorParameters<>(
+                    KTableKTableJoinMerger.of(
+                        (KTableProcessorSupplier) (joinThisProcessorParameters.processorSupplier()),
+                        (KTableProcessorSupplier) (joinOtherProcessorParameters.processorSupplier()),
+                        queryableStoreName),
+                    nodeName),
+                thisJoinSide,
+                otherJoinSide,
+                keySerde,
+                valueSerde,
+                joinThisStoreNames,
+                joinOtherStoreNames,
+                storeBuilder);
         }
     }
 }

From ee3b9c57fd4ba37aa34396da44ed04e55b224c18 Mon Sep 17 00:00:00 2001
From: huxi 
Date: Fri, 15 Mar 2019 14:54:42 +0800
Subject: [PATCH 0022/1071] MINOR: Fix typos in LogValidator (#6449)

---
 core/src/main/scala/kafka/log/LogValidator.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/kafka/log/LogValidator.scala b/core/src/main/scala/kafka/log/LogValidator.scala
index f7c7d04688560..8bf215ef79844 100644
--- a/core/src/main/scala/kafka/log/LogValidator.scala
+++ b/core/src/main/scala/kafka/log/LogValidator.scala
@@ -341,7 +341,7 @@ private[kafka] object LogValidator extends Logging {
                                            isTransactional: Boolean,
                                            partitionLeaderEpoch: Int,
                                            isFromClient: Boolean,
-                                           uncompresssedSizeInBytes: Int): ValidationAndOffsetAssignResult = {
+                                           uncompressedSizeInBytes: Int): ValidationAndOffsetAssignResult = {
     val startNanos = time.nanoseconds
     val estimatedSize = AbstractRecords.estimateSizeInBytes(magic, offsetCounter.value, compressionType,
       validatedRecords.asJava)
@@ -362,7 +362,7 @@ private[kafka] object LogValidator extends Logging {
     // message format V0 or if the inner offsets are not consecutive. This is OK since the impact is the same: we have
     // to rebuild the records (including recompression if enabled).
     val conversionCount = builder.numRecords
-    val recordConversionStats = new RecordConversionStats(uncompresssedSizeInBytes + builder.uncompressedBytesWritten,
+    val recordConversionStats = new RecordConversionStats(uncompressedSizeInBytes + builder.uncompressedBytesWritten,
       conversionCount, time.nanoseconds - startNanos)
 
     ValidationAndOffsetAssignResult(

From 853f24a4a18b26b4e4cc03555673ee951067fa6e Mon Sep 17 00:00:00 2001
From: Massimo Siani 
Date: Fri, 15 Mar 2019 15:56:48 +0100
Subject: [PATCH 0023/1071] KAFKA-7027: Add an overload build method in scala
 (#6373)

The Java API can pass a Properties object to StreamsBuilder#build, to allow, e.g., topology optimization, while the Scala API does not yet. The latter only delegates the work to the underlying Java implementation.

Reviewers: John Roesler ,  Bill Bejeck 
---
 .../kafka/streams/scala/StreamsBuilder.scala  |  11 ++
 .../kafka/streams/scala/TopologyTest.scala    | 175 +++++++++++++++++-
 2 files changed, 184 insertions(+), 2 deletions(-)

diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/StreamsBuilder.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/StreamsBuilder.scala
index 1fcba488fc319..9c4e65adc08d4 100644
--- a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/StreamsBuilder.scala
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/StreamsBuilder.scala
@@ -19,6 +19,7 @@
  */
 package org.apache.kafka.streams.scala
 
+import java.util.Properties
 import java.util.regex.Pattern
 
 import org.apache.kafka.streams.kstream.GlobalKTable
@@ -183,4 +184,14 @@ class StreamsBuilder(inner: StreamsBuilderJ = new StreamsBuilderJ) {
     inner.addGlobalStore(storeBuilder, topic, consumed, stateUpdateSupplier)
 
   def build(): Topology = inner.build()
+
+  /**
+   * Returns the `Topology` that represents the specified processing logic and accepts
+   * a `Properties` instance used to indicate whether to optimize topology or not.
+   *
+   * @param props the `Properties` used for building possibly optimized topology
+   * @return the `Topology` that represents the specified processing logic
+   * @see `org.apache.kafka.streams.StreamsBuilder#build`
+   */
+  def build(props: Properties): Topology = inner.build(props)
 }
diff --git a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/TopologyTest.scala b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/TopologyTest.scala
index 6035dd0db37be..3917552ccab70 100644
--- a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/TopologyTest.scala
+++ b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/TopologyTest.scala
@@ -19,20 +19,31 @@
  */
 package org.apache.kafka.streams.scala
 
+import java.time.Duration
+import java.util
+import java.util.{Locale, Properties}
 import java.util.regex.Pattern
 
+import org.apache.kafka.common.serialization.{Serdes => SerdesJ}
 import org.apache.kafka.streams.kstream.{
+  Aggregator,
+  ForeachAction,
+  Initializer,
+  JoinWindows,
   KeyValueMapper,
+  Predicate,
   Reducer,
   Transformer,
   TransformerSupplier,
   ValueJoiner,
   ValueMapper,
+  Joined => JoinedJ,
   KGroupedStream => KGroupedStreamJ,
   KStream => KStreamJ,
-  KTable => KTableJ
+  KTable => KTableJ,
+  Materialized => MaterializedJ
 }
-import org.apache.kafka.streams.processor.ProcessorContext
+import org.apache.kafka.streams.processor.{AbstractProcessor, ProcessorContext, ProcessorSupplier}
 import org.apache.kafka.streams.scala.ImplicitConversions._
 import org.apache.kafka.streams.scala.Serdes._
 import org.apache.kafka.streams.scala.kstream._
@@ -268,4 +279,164 @@ class TopologyTest extends JUnitSuite {
     // should match
     assertEquals(getTopologyScala, getTopologyJava)
   }
+
+  @Test def shouldBuildIdenticalTopologyInJavaNScalaProperties(): Unit = {
+
+    val props = new Properties()
+    props.put(StreamsConfig.TOPOLOGY_OPTIMIZATION, StreamsConfig.OPTIMIZE)
+
+    val propsNoOptimization = new Properties()
+    propsNoOptimization.put(StreamsConfig.TOPOLOGY_OPTIMIZATION, StreamsConfig.NO_OPTIMIZATION)
+
+    val AGGREGATION_TOPIC = "aggregationTopic"
+    val REDUCE_TOPIC = "reduceTopic"
+    val JOINED_TOPIC = "joinedTopic"
+
+    // build the Scala topology
+    def getTopologyScala: StreamsBuilder = {
+
+      val aggregator = (_: String, v: String, agg: Int) => agg + v.length
+      val reducer = (v1: String, v2: String) => v1 + ":" + v2
+      val processorValueCollector: util.List[String] = new util.ArrayList[String]
+
+      val builder: StreamsBuilder = new StreamsBuilder
+
+      val sourceStream: KStream[String, String] =
+        builder.stream(inputTopic)(Consumed.`with`(Serdes.String, Serdes.String))
+
+      val mappedStream: KStream[String, String] =
+        sourceStream.map((k: String, v: String) => (k.toUpperCase(Locale.getDefault), v))
+      mappedStream
+        .filter((k: String, _: String) => k == "B")
+        .mapValues((v: String) => v.toUpperCase(Locale.getDefault))
+        .process(() => new SimpleProcessor(processorValueCollector))
+
+      val stream2 = mappedStream.groupByKey
+        .aggregate(0)(aggregator)(Materialized.`with`(Serdes.String, Serdes.Integer))
+        .toStream
+      stream2.to(AGGREGATION_TOPIC)(Produced.`with`(Serdes.String, Serdes.Integer))
+
+      // adding operators for case where the repartition node is further downstream
+      val stream3 = mappedStream
+        .filter((_: String, _: String) => true)
+        .peek((k: String, v: String) => System.out.println(k + ":" + v))
+        .groupByKey
+        .reduce(reducer)(Materialized.`with`(Serdes.String, Serdes.String))
+        .toStream
+      stream3.to(REDUCE_TOPIC)(Produced.`with`(Serdes.String, Serdes.String))
+
+      mappedStream
+        .filter((k: String, _: String) => k == "A")
+        .join(stream2)((v1: String, v2: Int) => v1 + ":" + v2.toString, JoinWindows.of(Duration.ofMillis(5000)))(
+          Joined.`with`(Serdes.String, Serdes.String, Serdes.Integer)
+        )
+        .to(JOINED_TOPIC)
+
+      mappedStream
+        .filter((k: String, _: String) => k == "A")
+        .join(stream3)((v1: String, v2: String) => v1 + ":" + v2.toString, JoinWindows.of(Duration.ofMillis(5000)))(
+          Joined.`with`(Serdes.String, Serdes.String, Serdes.String)
+        )
+        .to(JOINED_TOPIC)
+
+      builder
+    }
+
+    // build the Java topology
+    def getTopologyJava: StreamsBuilderJ = {
+
+      val keyValueMapper: KeyValueMapper[String, String, KeyValue[String, String]] =
+        new KeyValueMapper[String, String, KeyValue[String, String]] {
+          override def apply(key: String, value: String): KeyValue[String, String] =
+            KeyValue.pair(key.toUpperCase(Locale.getDefault), value)
+        }
+      val initializer: Initializer[Integer] = new Initializer[Integer] {
+        override def apply(): Integer = 0
+      }
+      val aggregator: Aggregator[String, String, Integer] = new Aggregator[String, String, Integer] {
+        override def apply(key: String, value: String, aggregate: Integer): Integer = aggregate + value.length
+      }
+      val reducer: Reducer[String] = new Reducer[String] {
+        override def apply(v1: String, v2: String): String = v1 + ":" + v2
+      }
+      val valueMapper: ValueMapper[String, String] = new ValueMapper[String, String] {
+        override def apply(v: String): String = v.toUpperCase(Locale.getDefault)
+      }
+      val processorValueCollector = new util.ArrayList[String]
+      val processorSupplier: ProcessorSupplier[String, String] = new ProcessorSupplier[String, String] {
+        override def get() = new SimpleProcessor(processorValueCollector)
+      }
+      val valueJoiner2: ValueJoiner[String, Integer, String] = new ValueJoiner[String, Integer, String] {
+        override def apply(value1: String, value2: Integer): String = value1 + ":" + value2.toString
+      }
+      val valueJoiner3: ValueJoiner[String, String, String] = new ValueJoiner[String, String, String] {
+        override def apply(value1: String, value2: String): String = value1 + ":" + value2.toString
+      }
+
+      val builder = new StreamsBuilderJ
+
+      val sourceStream = builder.stream(inputTopic, Consumed.`with`(Serdes.String, Serdes.String))
+
+      val mappedStream: KStreamJ[String, String] =
+        sourceStream.map(keyValueMapper)
+      mappedStream
+        .filter(new Predicate[String, String] {
+          override def test(key: String, value: String): Boolean = key == "B"
+        })
+        .mapValues[String](valueMapper)
+        .process(processorSupplier)
+
+      val stream2 = mappedStream.groupByKey
+        .aggregate(initializer, aggregator, MaterializedJ.`with`(Serdes.String, SerdesJ.Integer))
+        .toStream
+      stream2.to(AGGREGATION_TOPIC, Produced.`with`(Serdes.String, SerdesJ.Integer))
+
+      // adding operators for case where the repartition node is further downstream
+      val stream3 = mappedStream
+        .filter(new Predicate[String, String] {
+          override def test(k: String, v: String) = true
+        })
+        .peek(new ForeachAction[String, String] {
+          override def apply(k: String, v: String) = System.out.println(k + ":" + v)
+        })
+        .groupByKey
+        .reduce(reducer, MaterializedJ.`with`(Serdes.String, Serdes.String))
+        .toStream
+      stream3.to(REDUCE_TOPIC, Produced.`with`(Serdes.String, Serdes.String))
+
+      mappedStream
+        .filter(new Predicate[String, String] {
+          override def test(key: String, value: String): Boolean = key == "A"
+        })
+        .join(stream2,
+              valueJoiner2,
+              JoinWindows.of(Duration.ofMillis(5000)),
+              JoinedJ.`with`(Serdes.String, Serdes.String, SerdesJ.Integer))
+        .to(JOINED_TOPIC)
+
+      mappedStream
+        .filter(new Predicate[String, String] {
+          override def test(key: String, value: String): Boolean = key == "A"
+        })
+        .join(stream3,
+              valueJoiner3,
+              JoinWindows.of(Duration.ofMillis(5000)),
+              JoinedJ.`with`(Serdes.String, Serdes.String, SerdesJ.String))
+        .to(JOINED_TOPIC)
+
+      builder
+    }
+
+    assertNotEquals(getTopologyScala.build(props).describe.toString,
+                    getTopologyScala.build(propsNoOptimization).describe.toString)
+    assertEquals(getTopologyScala.build(propsNoOptimization).describe.toString,
+                 getTopologyJava.build(propsNoOptimization).describe.toString)
+    assertEquals(getTopologyScala.build(props).describe.toString, getTopologyJava.build(props).describe.toString)
+  }
+
+  private class SimpleProcessor private[TopologyTest] (val valueList: util.List[String])
+      extends AbstractProcessor[String, String] {
+    override def process(key: String, value: String): Unit =
+      valueList.add(value)
+  }
 }

From a6691fb79e2c55b325513710c5e6e34e8dd4961e Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Fri, 15 Mar 2019 15:36:22 +0000
Subject: [PATCH 0024/1071] KAFKA-8091; Use commitSync to check connection
 failure in listener update test (#6450)

The use of consumer.poll() made the test flaky since in some cases, it doesn't wait for coordinator connection.

Reviewers: Manikumar Reddy 
---
 .../kafka/server/DynamicBrokerReconfigurationTest.scala         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
index d1427b2356c42..c14f41ec70690 100644
--- a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
+++ b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
@@ -1315,7 +1315,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
     executors += executor
     val future = executor.submit(new Runnable() {
       def run() {
-        assertEquals(0, consumer.poll(100).count)
+        consumer.commitSync()
       }
     })
     verifyTimeout(future)

From f20f3c1a97b8a31a2f211cd66506fb823f420c55 Mon Sep 17 00:00:00 2001
From: Stanislav Kozlovski 
Date: Fri, 15 Mar 2019 18:53:21 +0200
Subject: [PATCH 0025/1071] MINOR: Update Trogdor ConnectionStressWorker status
 at the end of execution (#6445)

Reviewers: Colin P. McCabe 
---
 tests/spec/connection_stress_test.json        | 29 ++++++++++++++
 .../workload/ConnectionStressWorker.java      | 38 +++++++++++--------
 2 files changed, 51 insertions(+), 16 deletions(-)
 create mode 100644 tests/spec/connection_stress_test.json

diff --git a/tests/spec/connection_stress_test.json b/tests/spec/connection_stress_test.json
new file mode 100644
index 0000000000000..7b6698544f388
--- /dev/null
+++ b/tests/spec/connection_stress_test.json
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// An example task specification for running a connection stress test in Trogdor.
+// See TROGDOR.md for details.
+//
+
+{
+  "class": "org.apache.kafka.trogdor.workload.ConnectionStressSpec",
+  "durationMs": 60000,
+  "clientNode": "node0",
+  "bootstrapServers": "localhost:9092",
+  "targetConnectionsPerSec": 100,
+  "numThreads": 10,
+  "action": "CONNECT"
+}
diff --git a/tools/src/main/java/org/apache/kafka/trogdor/workload/ConnectionStressWorker.java b/tools/src/main/java/org/apache/kafka/trogdor/workload/ConnectionStressWorker.java
index d85effc2833cb..9a9439ad39061 100644
--- a/tools/src/main/java/org/apache/kafka/trogdor/workload/ConnectionStressWorker.java
+++ b/tools/src/main/java/org/apache/kafka/trogdor/workload/ConnectionStressWorker.java
@@ -99,12 +99,14 @@ public void start(Platform platform, WorkerStatusTracker status,
         log.info("{}: Activating ConnectionStressWorker with {}", id, spec);
         this.doneFuture = doneFuture;
         this.status = status;
-        this.totalConnections = 0;
-        this.totalFailedConnections  = 0;
-        this.startTimeMs = TIME.milliseconds();
+        synchronized (ConnectionStressWorker.this) {
+            this.totalConnections = 0;
+            this.totalFailedConnections = 0;
+            this.nextReportTime = 0;
+            this.startTimeMs = TIME.milliseconds();
+        }
         this.throttle = new ConnectStressThrottle(WorkerUtils.
             perSecToPerPeriod(spec.targetConnectionsPerSec(), THROTTLE_PERIOD_MS));
-        this.nextReportTime = 0;
         this.workerExecutor = Executors.newFixedThreadPool(spec.numThreads(),
             ThreadUtils.createThreadFactory("ConnectionStressWorkerThread%d", false));
         for (int i = 0; i < spec.numThreads(); i++) {
@@ -112,6 +114,17 @@ public void start(Platform platform, WorkerStatusTracker status,
         }
     }
 
+    /**
+     * Update the worker's status and next status report time.
+     */
+    private synchronized void updateStatus(long lastTimeMs) {
+        status.update(JsonUtil.JSON_SERDE.valueToTree(
+                new StatusData(totalConnections,
+                        totalFailedConnections,
+                        (totalConnections * 1000.0) / (lastTimeMs - startTimeMs))));
+        nextReportTime = lastTimeMs + REPORT_INTERVAL_MS;
+    }
+
     private static class ConnectStressThrottle extends Throttle {
         ConnectStressThrottle(int maxPerPeriod) {
             super(maxPerPeriod, THROTTLE_PERIOD_MS);
@@ -130,10 +143,7 @@ public void run() {
                         conf.getList(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG),
                         conf.getString(AdminClientConfig.CLIENT_DNS_LOOKUP_CONFIG));
                 ManualMetadataUpdater updater = new ManualMetadataUpdater(Cluster.bootstrap(addresses).nodes());
-                while (true) {
-                    if (doneFuture.isDone()) {
-                        break;
-                    }
+                while (!doneFuture.isDone()) {
                     throttle.increment();
                     long lastTimeMs = throttle.lastTimeMs();
                     boolean success = false;
@@ -150,13 +160,8 @@ public void run() {
                         if (!success) {
                             totalFailedConnections++;
                         }
-                        if (lastTimeMs > nextReportTime) {
-                            status.update(JsonUtil.JSON_SERDE.valueToTree(
-                                new StatusData(totalConnections,
-                                    totalFailedConnections,
-                                    (totalConnections * 1000.0) / (lastTimeMs - startTimeMs))));
-                            nextReportTime = lastTimeMs + REPORT_INTERVAL_MS;
-                        }
+                        if (lastTimeMs > nextReportTime)
+                            updateStatus(lastTimeMs);
                     }
                 }
             } catch (Exception e) {
@@ -165,7 +170,7 @@ public void run() {
         }
 
         private boolean attemptConnection(AdminClientConfig conf,
-                                          ManualMetadataUpdater updater) throws Exception {
+                                          ManualMetadataUpdater updater) {
             try {
                 List nodes = updater.fetchNodes();
                 Node targetNode = nodes.get(ThreadLocalRandom.current().nextInt(nodes.size()));
@@ -250,6 +255,7 @@ public void stop(Platform platform) throws Exception {
         doneFuture.complete("");
         workerExecutor.shutdownNow();
         workerExecutor.awaitTermination(1, TimeUnit.DAYS);
+        updateStatus(throttle.lastTimeMs());
         this.workerExecutor = null;
         this.status = null;
     }

From e1961b8298c6d7ec028cd95721f4b498689c506e Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Fri, 15 Mar 2019 19:47:40 -0700
Subject: [PATCH 0026/1071] MINOR: Update code to not use deprecated methods
 (#6434)

Reviewers: Bill Bejeck , John Roesler , Colin P. McCabe 
---
 core/src/main/scala/kafka/tools/MirrorMaker.scala   |  4 ++--
 .../kafka/api/AdminClientIntegrationTest.scala      | 10 +++++-----
 .../kafka/api/BaseProducerSendTest.scala            | 13 +++++++------
 .../scala/integration/kafka/api/BaseQuotaTest.scala |  7 ++++---
 .../integration/kafka/api/ConsumerBounceTest.scala  | 11 ++++++-----
 .../kafka/api/IntegrationTestHarness.scala          |  2 +-
 .../integration/kafka/api/TransactionsTest.scala    |  3 ++-
 .../server/DynamicBrokerReconfigurationTest.scala   |  4 ++--
 8 files changed, 29 insertions(+), 25 deletions(-)

diff --git a/core/src/main/scala/kafka/tools/MirrorMaker.scala b/core/src/main/scala/kafka/tools/MirrorMaker.scala
index 823f0fd283845..fda1812719a98 100755
--- a/core/src/main/scala/kafka/tools/MirrorMaker.scala
+++ b/core/src/main/scala/kafka/tools/MirrorMaker.scala
@@ -324,7 +324,7 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
         // uncommitted record since last poll. Using one second as poll's timeout ensures that
         // offsetCommitIntervalMs, of value greater than 1 second, does not see delays in offset
         // commit.
-        recordIter = consumer.poll(Duration.ofSeconds(1)).iterator
+        recordIter = consumer.poll(Duration.ofSeconds(1L)).iterator
         if (!recordIter.hasNext)
           throw new NoRecordsException
       }
@@ -387,7 +387,7 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
     }
 
     def close(timeout: Long) {
-      this.producer.close(timeout, TimeUnit.MILLISECONDS)
+      this.producer.close(Duration.ofMillis(timeout))
     }
   }
 
diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index cf019a86264ce..027266d84e497 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -16,7 +16,7 @@
  */
 package kafka.api
 
-import java.util
+import java.{time, util}
 import java.util.{Collections, Properties}
 import java.util.Arrays.asList
 import java.util.concurrent.{ExecutionException, TimeUnit}
@@ -1066,11 +1066,11 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     val topics = Seq("mytopic", "mytopic2")
     val newTopics = topics.map(new NewTopic(_, 1, 1))
     val future = client.createTopics(newTopics.asJava, new CreateTopicsOptions().validateOnly(true)).all()
-    client.close(2, TimeUnit.HOURS)
+    client.close(time.Duration.ofHours(2))
     val future2 = client.createTopics(newTopics.asJava, new CreateTopicsOptions().validateOnly(true)).all()
     assertFutureExceptionTypeEquals(future2, classOf[TimeoutException])
     future.get
-    client.close(30, TimeUnit.MINUTES) // multiple close-with-timeout should have no effect
+    client.close(time.Duration.ofMinutes(30)) // multiple close-with-timeout should have no effect
   }
 
   /**
@@ -1086,7 +1086,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     // cancelled by the close operation.
     val future = client.createTopics(Seq("mytopic", "mytopic2").map(new NewTopic(_, 1, 1)).asJava,
       new CreateTopicsOptions().timeoutMs(900000)).all()
-    client.close(0, TimeUnit.MILLISECONDS)
+    client.close(time.Duration.ZERO)
     assertFutureExceptionTypeEquals(future, classOf[TimeoutException])
   }
 
@@ -1164,7 +1164,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
           override def run {
             consumer.subscribe(Collections.singleton(testTopicName))
             while (true) {
-              consumer.poll(5000)
+              consumer.poll(time.Duration.ofSeconds(5L))
               consumer.commitSync()
             }
           }
diff --git a/core/src/test/scala/integration/kafka/api/BaseProducerSendTest.scala b/core/src/test/scala/integration/kafka/api/BaseProducerSendTest.scala
index 9d454e93bf2b7..2ce16d2c97217 100644
--- a/core/src/test/scala/integration/kafka/api/BaseProducerSendTest.scala
+++ b/core/src/test/scala/integration/kafka/api/BaseProducerSendTest.scala
@@ -17,6 +17,7 @@
 
 package kafka.api
 
+import java.time.Duration
 import java.nio.charset.StandardCharsets
 import java.util.Properties
 import java.util.concurrent.TimeUnit
@@ -193,7 +194,7 @@ abstract class BaseProducerSendTest extends KafkaServerTestHarness {
           s"value$i".getBytes(StandardCharsets.UTF_8))
         producer.send(record)
       }
-      producer.close(timeoutMs, TimeUnit.MILLISECONDS)
+      producer.close(Duration.ofMillis(timeoutMs))
       val lastOffset = futures.foldLeft(0) { (offset, future) =>
         val recordMetadata = future.get
         assertEquals(topic, recordMetadata.topic)
@@ -248,7 +249,7 @@ abstract class BaseProducerSendTest extends KafkaServerTestHarness {
           s"value$i".getBytes(StandardCharsets.UTF_8))
         (record, producer.send(record, callback))
       }
-      producer.close(20000L, TimeUnit.MILLISECONDS)
+      producer.close(Duration.ofSeconds(20L))
       recordAndFutures.foreach { case (record, future) =>
         val recordMetadata = future.get
         if (timestampType == TimestampType.LOG_APPEND_TIME)
@@ -445,7 +446,7 @@ abstract class BaseProducerSendTest extends KafkaServerTestHarness {
       val producer = createProducer(brokerList, lingerMs = Int.MaxValue)
       val responses = (0 until numRecords) map (_ => producer.send(record0))
       assertTrue("No request is complete.", responses.forall(!_.isDone()))
-      producer.close(0, TimeUnit.MILLISECONDS)
+      producer.close(Duration.ZERO)
       responses.foreach { future =>
         try {
           future.get()
@@ -454,7 +455,7 @@ abstract class BaseProducerSendTest extends KafkaServerTestHarness {
           case e: ExecutionException => assertEquals(classOf[KafkaException], e.getCause.getClass)
         }
       }
-      assertEquals("Fetch response should have no message returned.", 0, consumer.poll(50).count)
+      assertEquals("Fetch response should have no message returned.", 0, consumer.poll(Duration.ofMillis(50L)).count)
     }
   }
 
@@ -476,9 +477,9 @@ abstract class BaseProducerSendTest extends KafkaServerTestHarness {
         if (sendRecords)
           (0 until numRecords) foreach (_ => producer.send(record))
         // The close call will be called by all the message callbacks. This tests idempotence of the close call.
-        producer.close(0, TimeUnit.MILLISECONDS)
+        producer.close(Duration.ZERO)
         // Test close with non zero timeout. Should not block at all.
-        producer.close(Long.MaxValue, TimeUnit.MICROSECONDS)
+        producer.close()
       }
     }
     for (i <- 0 until 50) {
diff --git a/core/src/test/scala/integration/kafka/api/BaseQuotaTest.scala b/core/src/test/scala/integration/kafka/api/BaseQuotaTest.scala
index b28a40f890e00..4b278f08bb095 100644
--- a/core/src/test/scala/integration/kafka/api/BaseQuotaTest.scala
+++ b/core/src/test/scala/integration/kafka/api/BaseQuotaTest.scala
@@ -14,6 +14,7 @@
 
 package kafka.api
 
+import java.time.Duration
 import java.util.{Collections, HashMap, Properties}
 
 import kafka.api.QuotaTestClients._
@@ -140,7 +141,7 @@ abstract class BaseQuotaTest extends IntegrationTestHarness {
     val endTimeMs = System.currentTimeMillis + 10000
     var throttled = false
     while ((!throttled || quotaTestClients.exemptRequestMetric == null) && System.currentTimeMillis < endTimeMs) {
-      consumer.poll(100)
+      consumer.poll(Duration.ofMillis(100L))
       val throttleMetric = quotaTestClients.throttleMetric(QuotaType.Request, consumerClientId)
       throttled = throttleMetric != null && metricValue(throttleMetric) > 0
     }
@@ -197,7 +198,7 @@ abstract class QuotaTestClients(topic: String,
     var numConsumed = 0
     var throttled = false
     do {
-      numConsumed += consumer.poll(100).count
+      numConsumed += consumer.poll(Duration.ofMillis(100L)).count
       val metric = throttleMetric(QuotaType.Fetch, consumerClientId)
       throttled = metric != null && metricValue(metric) > 0
     }  while (numConsumed < maxRecords && !throttled)
@@ -206,7 +207,7 @@ abstract class QuotaTestClients(topic: String,
     if (throttled && numConsumed < maxRecords && waitForRequestCompletion) {
       val minRecords = numConsumed + 1
       while (numConsumed < minRecords)
-        numConsumed += consumer.poll(100).count
+        numConsumed += consumer.poll(Duration.ofMillis(100L)).count
     }
     numConsumed
   }
diff --git a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
index e535104306fa4..1a1f37ee5c522 100644
--- a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
+++ b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
@@ -178,7 +178,7 @@ class ConsumerBounceTest extends BaseRequestTest with Logging {
     executor.schedule(new Runnable {
         def run() = createTopic(newtopic, numPartitions = numBrokers, replicationFactor = numBrokers)
       }, 2, TimeUnit.SECONDS)
-    consumer.poll(0)
+    consumer.poll(time.Duration.ZERO)
 
     val producer = createProducer()
 
@@ -481,14 +481,15 @@ class ConsumerBounceTest extends BaseRequestTest with Logging {
               revokeSemaphore.foreach(s => s.release())
             }
           })
-          consumer.poll(0)
+        // requires to used deprecated `poll(long)` to trigger metadata update
+          consumer.poll(0L)
         }, 0)
     }
 
     def waitForRebalance(timeoutMs: Long, future: Future[Any], otherConsumers: KafkaConsumer[Array[Byte], Array[Byte]]*) {
       val startMs = System.currentTimeMillis
       while (System.currentTimeMillis < startMs + timeoutMs && !future.isDone)
-          otherConsumers.foreach(consumer => consumer.poll(100))
+          otherConsumers.foreach(consumer => consumer.poll(time.Duration.ofMillis(100L)))
       assertTrue("Rebalance did not complete in time", future.isDone)
     }
 
@@ -569,7 +570,7 @@ class ConsumerBounceTest extends BaseRequestTest with Logging {
       val closeGraceTimeMs = 2000
       val startNanos = System.nanoTime
       info("Closing consumer with timeout " + closeTimeoutMs + " ms.")
-      consumer.close(closeTimeoutMs, TimeUnit.MILLISECONDS)
+      consumer.close(time.Duration.ofMillis(closeTimeoutMs))
       val timeTakenMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime - startNanos)
       maxCloseTimeMs.foreach { ms =>
         assertTrue("Close took too long " + timeTakenMs, timeTakenMs < ms + closeGraceTimeMs)
@@ -592,7 +593,7 @@ class ConsumerBounceTest extends BaseRequestTest with Logging {
       }
       def onPartitionsRevoked(partitions: Collection[TopicPartition]) {
       }})
-    consumer.poll(3000)
+    consumer.poll(time.Duration.ofSeconds(3L))
     assertTrue("Assignment did not complete on time", assignSemaphore.tryAcquire(1, TimeUnit.SECONDS))
     if (committedRecords > 0)
       assertEquals(committedRecords, consumer.committed(tp).offset)
diff --git a/core/src/test/scala/integration/kafka/api/IntegrationTestHarness.scala b/core/src/test/scala/integration/kafka/api/IntegrationTestHarness.scala
index 5a2000557424f..640244da950ce 100644
--- a/core/src/test/scala/integration/kafka/api/IntegrationTestHarness.scala
+++ b/core/src/test/scala/integration/kafka/api/IntegrationTestHarness.scala
@@ -126,7 +126,7 @@ abstract class IntegrationTestHarness extends KafkaServerTestHarness {
 
   @After
   override def tearDown() {
-    producers.foreach(_.close(0, TimeUnit.MILLISECONDS))
+    producers.foreach(_.close(Duration.ZERO))
     consumers.foreach(_.wakeup())
     consumers.foreach(_.close(Duration.ZERO))
     producers.clear()
diff --git a/core/src/test/scala/integration/kafka/api/TransactionsTest.scala b/core/src/test/scala/integration/kafka/api/TransactionsTest.scala
index 34dea70ca8c6e..e3bfdc0c6a3d0 100644
--- a/core/src/test/scala/integration/kafka/api/TransactionsTest.scala
+++ b/core/src/test/scala/integration/kafka/api/TransactionsTest.scala
@@ -18,6 +18,7 @@
 package kafka.api
 
 import java.lang.{Long => JLong}
+import java.time.Duration
 import java.util.{Optional, Properties}
 import java.util.concurrent.TimeUnit
 
@@ -578,7 +579,7 @@ class TransactionsTest extends KafkaServerTestHarness {
     try {
       producer.commitTransaction()
     } finally {
-      producer.close(0, TimeUnit.MILLISECONDS)
+      producer.close(Duration.ZERO)
     }
   }
 
diff --git a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
index c14f41ec70690..e8aa081925c26 100644
--- a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
+++ b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
@@ -146,7 +146,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
     clientThreads.foreach(_.initiateShutdown())
     clientThreads.foreach(_.join(5 * 1000))
     executors.foreach(_.shutdownNow())
-    producers.foreach(_.close(0, TimeUnit.MILLISECONDS))
+    producers.foreach(_.close(Duration.ZERO))
     consumers.foreach(_.close(Duration.ofMillis(0)))
     adminClients.foreach(_.close())
     TestUtils.shutdownServers(servers)
@@ -1470,7 +1470,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
     override def doWork(): Unit = {
       try {
         while (isRunning || (lastReceived != producerThread.lastSent && System.currentTimeMillis < endTimeMs)) {
-          val records = consumer.poll(50)
+          val records = consumer.poll(Duration.ofMillis(50L))
           received += records.count
           if (!records.isEmpty) {
             lastBatch = records

From fd5c0849e119fa19b393c9900584a6ff8abe3a79 Mon Sep 17 00:00:00 2001
From: Kristian Aurlien 
Date: Sat, 16 Mar 2019 03:51:20 +0100
Subject: [PATCH 0027/1071] KAFKA-7855: Kafka Streams Maven Archetype
 quickstart fails to compile out of the box (#6194)

Reviewers: Guozhang Wang , Matthias J. Sax 
---
 .../archetype-resources/src/main/java/LineSplit.java        | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/streams/quickstart/java/src/main/resources/archetype-resources/src/main/java/LineSplit.java b/streams/quickstart/java/src/main/resources/archetype-resources/src/main/java/LineSplit.java
index bbf54e6a8cb79..d712a8392b764 100644
--- a/streams/quickstart/java/src/main/resources/archetype-resources/src/main/java/LineSplit.java
+++ b/streams/quickstart/java/src/main/resources/archetype-resources/src/main/java/LineSplit.java
@@ -44,9 +44,9 @@ public static void main(String[] args) throws Exception {
 
         final StreamsBuilder builder = new StreamsBuilder();
 
-        builder.stream("streams-plaintext-input")
-               .flatMapValues(value -> Arrays.asList(value.split("\\W+")))
-               .to("streams-linesplit-output");
+        builder.stream("streams-plaintext-input")
+            .flatMapValues(value -> Arrays.asList(value.split("\\W+")))
+            .to("streams-linesplit-output");
 
         final Topology topology = builder.build();
         final KafkaStreams streams = new KafkaStreams(topology, props);

From 20e3a3035b02388d46a4d7bf93370329aa405956 Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Sat, 16 Mar 2019 11:00:03 +0000
Subject: [PATCH 0028/1071] KAFKA-8111; Set min and max versions for Metadata
 requests (#6451)

Reviewers: Manikumar Reddy 
---
 .../kafka/common/requests/MetadataRequest.java | 10 +++++++---
 .../common/requests/MetadataRequestTest.java   | 18 ++++++++++++++++++
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java
index 7f5a5440b2c50..cbfca4bbdf6f2 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java
@@ -43,8 +43,12 @@ public Builder(MetadataRequestData data) {
             this.data = data;
         }
 
-        public Builder(List topics, boolean allowAutoTopicCreation, short version) {
-            super(ApiKeys.METADATA, version);
+        public Builder(List topics, boolean allowAutoTopicCreation, short allowedVersion) {
+            this(topics, allowAutoTopicCreation, allowedVersion, allowedVersion);
+        }
+
+        public Builder(List topics, boolean allowAutoTopicCreation, short minVersion, short maxVersion) {
+            super(ApiKeys.METADATA, minVersion, maxVersion);
             MetadataRequestData data = new MetadataRequestData();
             if (topics == null)
                 data.setTopics(null);
@@ -57,7 +61,7 @@ public Builder(List topics, boolean allowAutoTopicCreation, short versio
         }
 
         public Builder(List topics, boolean allowAutoTopicCreation) {
-            this(topics, allowAutoTopicCreation, ApiKeys.METADATA.latestVersion());
+            this(topics, allowAutoTopicCreation, ApiKeys.METADATA.oldestVersion(),  ApiKeys.METADATA.latestVersion());
         }
 
         public static Builder allTopics() {
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/MetadataRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/MetadataRequestTest.java
index c97564438677a..31e22625ce8d1 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/MetadataRequestTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/MetadataRequestTest.java
@@ -17,6 +17,7 @@
 package org.apache.kafka.common.requests;
 
 import org.apache.kafka.common.message.MetadataRequestData;
+import org.apache.kafka.common.protocol.ApiKeys;
 import org.junit.Test;
 
 import java.util.Collections;
@@ -47,4 +48,21 @@ public void testEmptyMeansEmptyForVersionsAboveV0() {
         }
     }
 
+    @Test
+    public void testMetadataRequestVersion() {
+        MetadataRequest.Builder builder = new MetadataRequest.Builder(Collections.singletonList("topic"), false);
+        assertEquals(ApiKeys.METADATA.oldestVersion(), builder.oldestAllowedVersion());
+        assertEquals(ApiKeys.METADATA.latestVersion(), builder.latestAllowedVersion());
+
+        short version = 5;
+        MetadataRequest.Builder builder2 = new MetadataRequest.Builder(Collections.singletonList("topic"), false, version);
+        assertEquals(version, builder2.oldestAllowedVersion());
+        assertEquals(version, builder2.latestAllowedVersion());
+
+        short minVersion = 1;
+        short maxVersion = 6;
+        MetadataRequest.Builder builder3 = new MetadataRequest.Builder(Collections.singletonList("topic"), false, minVersion, maxVersion);
+        assertEquals(minVersion, builder3.oldestAllowedVersion());
+        assertEquals(maxVersion, builder3.latestAllowedVersion());
+    }
 }

From 74e755fdb1ea98c2f90f2127160f905ad3c00486 Mon Sep 17 00:00:00 2001
From: Manikumar Reddy 
Date: Sat, 16 Mar 2019 16:33:35 +0530
Subject: [PATCH 0029/1071] KAFKA-8114: Wait for SCRAM credential propagation
 in DelegationTokenEndToEndAuthorizationTest (#6452)

Reviewers: Rajini Sivaram 
---
 .../api/DelegationTokenEndToEndAuthorizationTest.scala   | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/core/src/test/scala/integration/kafka/api/DelegationTokenEndToEndAuthorizationTest.scala b/core/src/test/scala/integration/kafka/api/DelegationTokenEndToEndAuthorizationTest.scala
index 29c7507449be3..b72cf3c08aba9 100644
--- a/core/src/test/scala/integration/kafka/api/DelegationTokenEndToEndAuthorizationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/DelegationTokenEndToEndAuthorizationTest.scala
@@ -24,6 +24,7 @@ import kafka.zk.ConfigEntityChangeNotificationZNode
 import org.apache.kafka.clients.admin.{AdminClient, AdminClientConfig}
 import org.apache.kafka.common.config.SaslConfigs
 import org.apache.kafka.common.security.auth.SecurityProtocol
+import org.apache.kafka.common.security.scram.ScramCredential
 import org.apache.kafka.common.security.scram.internals.ScramMechanism
 import org.apache.kafka.common.security.token.delegation.DelegationToken
 import org.junit.Before
@@ -58,6 +59,7 @@ class DelegationTokenEndToEndAuthorizationTest extends EndToEndAuthorizationTest
 
     // create scram credential for user "scram-user"
     createScramCredentials(zkConnect, clientPrincipal, clientPassword)
+    waitForScramCredentials(clientPrincipal)
 
     //create a token with "scram-user" credentials
     val token = createDelegationToken()
@@ -68,6 +70,13 @@ class DelegationTokenEndToEndAuthorizationTest extends EndToEndAuthorizationTest
     consumerConfig.put(SaslConfigs.SASL_JAAS_CONFIG, clientLoginContext)
   }
 
+  private def waitForScramCredentials(clientPrincipal: String): Unit = {
+    servers.foreach { server =>
+      val cache = server.credentialProvider.credentialCache.cache(kafkaClientSaslMechanism, classOf[ScramCredential])
+      TestUtils.waitUntilTrue(() => cache.get(clientPrincipal) != null, s"SCRAM credentials not created for $clientPrincipal")
+    }
+  }
+
   @Before
   override def setUp() {
     startSasl(jaasSections(kafkaServerSaslMechanisms, Option(kafkaClientSaslMechanism), Both))

From 938580ff6c5c27b1d7a3baf9cc09029ef3c2eb68 Mon Sep 17 00:00:00 2001
From: huxihx 
Date: Sun, 17 Mar 2019 18:50:55 -0700
Subject: [PATCH 0030/1071] KAFKA-7813: JmxTool throws NPE when --object-name
 is omitted

https://issues.apache.org/jira/browse/KAFKA-7813

Running the JMX tool without --object-name parameter, results in a NullPointerException.

*More detailed description of your change,
if necessary. The PR title and PR message become
the squashed commit message, so use a separate
comment to ping reviewers.*

*Summary of testing strategy (including rationale)
for the feature or bug fix. Unit and/or integration
tests are expected for any behaviour change and
system tests should be considered for larger changes.*

Author: huxihx 

Reviewers: Ewen Cheslack-Postava 

Closes #6139 from huxihx/KAFKA-7813
---
 core/src/main/scala/kafka/tools/JmxTool.scala | 22 +++++++++++++------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/core/src/main/scala/kafka/tools/JmxTool.scala b/core/src/main/scala/kafka/tools/JmxTool.scala
index c5303a9d96123..9451cc2983b8c 100644
--- a/core/src/main/scala/kafka/tools/JmxTool.scala
+++ b/core/src/main/scala/kafka/tools/JmxTool.scala
@@ -18,7 +18,7 @@
  */
 package kafka.tools
 
-import java.util.Date
+import java.util.{Date, Objects}
 import java.text.SimpleDateFormat
 import javax.management._
 import javax.management.remote._
@@ -28,7 +28,7 @@ import joptsimple.OptionParser
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 import scala.math._
-import kafka.utils.{CommandLineUtils , Exit, Logging}
+import kafka.utils.{CommandLineUtils, Exit, Logging}
 
 
 /**
@@ -140,7 +140,7 @@ object JmxTool extends Logging {
       else
         List(null)
 
-    val hasPatternQueries = queries.exists((name: ObjectName) => name.isPattern)
+    val hasPatternQueries = queries.filterNot(Objects.isNull).exists((name: ObjectName) => name.isPattern)
 
     var names: Iterable[ObjectName] = null
     def namesSet = Option(names).toSet.flatten
@@ -165,12 +165,20 @@ object JmxTool extends Logging {
     }
 
     val numExpectedAttributes: Map[ObjectName, Int] =
-      if (attributesWhitelistExists)
-        queries.map((_, attributesWhitelist.get.length)).toMap
-      else {
-        names.map{(name: ObjectName) =>
+      if (!attributesWhitelistExists)
+        names.map{name: ObjectName =>
           val mbean = mbsc.getMBeanInfo(name)
           (name, mbsc.getAttributes(name, mbean.getAttributes.map(_.getName)).size)}.toMap
+      else {
+        if (!hasPatternQueries)
+          names.map{name: ObjectName =>
+            val mbean = mbsc.getMBeanInfo(name)
+            val attributes = mbsc.getAttributes(name, mbean.getAttributes.map(_.getName))
+            val expectedAttributes = attributes.asScala.asInstanceOf[mutable.Buffer[Attribute]]
+              .filter(attr => attributesWhitelist.get.contains(attr.getName))
+            (name, expectedAttributes.size)}.toMap.filter(_._2 > 0)
+        else
+          queries.map((_, attributesWhitelist.get.length)).toMap
       }
 
     if(numExpectedAttributes.isEmpty) {

From 6279b03813af58411951a94cfa3509f6ef14bec0 Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Mon, 18 Mar 2019 08:47:28 +0000
Subject: [PATCH 0031/1071] KAFKA-8118; Ensure ZK clients are closed in tests,
 fix verification (#6456)

We verify that ZK clients are closed in tests since these can affect subsequent tests and that makes it hard to debug test failures. But because of changes to ZooKeeper client, we were checking the wrong thread name. The thread name used now is -EventThread where creatorThreadName varies depending on the test. Fixed ZooKeeperTestHarness to check this format and fixed tests which were leaving ZK clients behind. Also added a test to make sure we can detect changes to the thread name when we update ZK clients in future.

Reviewers: Ismael Juma , Manikumar Reddy 
---
 .../unit/kafka/zk/ZooKeeperTestHarness.scala  |  4 +--
 .../kafka/zookeeper/ZooKeeperClientTest.scala | 30 ++++++++++++-------
 2 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala b/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala
index ebb5fb74dac45..5a62464b917e5 100755
--- a/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala
+++ b/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala
@@ -84,7 +84,7 @@ abstract class ZooKeeperTestHarness extends JUnitSuite with Logging {
 }
 
 object ZooKeeperTestHarness {
-  val ZkClientEventThreadPrefix = "ZkClient-EventThread"
+  val ZkClientEventThreadSuffix = "-EventThread"
 
   // Threads which may cause transient failures in subsequent tests if not shutdown.
   // These include threads which make connections to brokers and may cause issues
@@ -94,7 +94,7 @@ object ZooKeeperTestHarness {
                                   KafkaProducer.NETWORK_THREAD_PREFIX,
                                   AdminClientUnitTestEnv.kafkaAdminClientNetworkThreadPrefix(),
                                   AbstractCoordinator.HEARTBEAT_THREAD_PREFIX,
-                                  ZkClientEventThreadPrefix)
+                                  ZkClientEventThreadSuffix)
 
   /**
    * Verify that a previous test that doesn't use ZooKeeperTestHarness hasn't left behind an unexpected thread.
diff --git a/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala b/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala
index fd3f59cefc2ef..e943348180f35 100644
--- a/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala
+++ b/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala
@@ -82,8 +82,16 @@ class ZooKeeperClientTest extends ZooKeeperTestHarness {
 
   @Test
   def testConnection(): Unit = {
-    new ZooKeeperClient(zkConnect, zkSessionTimeout, zkConnectionTimeout, Int.MaxValue, time, "testMetricGroup",
-      "testMetricType").close()
+    val client = new ZooKeeperClient(zkConnect, zkSessionTimeout, zkConnectionTimeout, Int.MaxValue, time, "testMetricGroup",
+      "testMetricType")
+    try {
+      // Verify ZooKeeper event thread name. This is used in ZooKeeperTestHarness to verify that tests have closed ZK clients
+      val threads = Thread.getAllStackTraces.keySet.asScala.map(_.getName)
+      assertTrue(s"ZooKeeperClient event thread not found, threads=$threads",
+        threads.exists(_.contains(ZooKeeperTestHarness.ZkClientEventThreadSuffix)))
+    } finally {
+      client.close()
+    }
   }
 
   @Test
@@ -327,14 +335,15 @@ class ZooKeeperClientTest extends ZooKeeperTestHarness {
       }
     }
 
-    val client = new ZooKeeperClient(zkConnect, zkSessionTimeout, zkConnectionTimeout, Int.MaxValue, time,
+    zooKeeperClient.close()
+    zooKeeperClient = new ZooKeeperClient(zkConnect, zkSessionTimeout, zkConnectionTimeout, Int.MaxValue, time,
       "testMetricGroup", "testMetricType")
-    client.registerStateChangeHandler(stateChangeHandler)
+    zooKeeperClient.registerStateChangeHandler(stateChangeHandler)
 
     val requestThread = new Thread() {
       override def run(): Unit = {
         try
-          client.handleRequest(CreateRequest(mockPath, Array.empty[Byte],
+          zooKeeperClient.handleRequest(CreateRequest(mockPath, Array.empty[Byte],
             ZooDefs.Ids.OPEN_ACL_UNSAFE.asScala, CreateMode.PERSISTENT))
         finally
           latch.countDown()
@@ -343,7 +352,7 @@ class ZooKeeperClientTest extends ZooKeeperTestHarness {
 
     val reinitializeThread = new Thread() {
       override def run(): Unit = {
-        client.forceReinitialize()
+        zooKeeperClient.forceReinitialize()
       }
     }
 
@@ -375,12 +384,13 @@ class ZooKeeperClientTest extends ZooKeeperTestHarness {
       }
     }
 
-    val client = new ZooKeeperClient(zkConnect, zkSessionTimeout, zkConnectionTimeout, Int.MaxValue, time,
+    zooKeeperClient.close()
+    zooKeeperClient = new ZooKeeperClient(zkConnect, zkSessionTimeout, zkConnectionTimeout, Int.MaxValue, time,
       "testMetricGroup", "testMetricType")
-    client.registerStateChangeHandler(faultyHandler)
-    client.registerStateChangeHandler(goodHandler)
+    zooKeeperClient.registerStateChangeHandler(faultyHandler)
+    zooKeeperClient.registerStateChangeHandler(goodHandler)
 
-    client.forceReinitialize()
+    zooKeeperClient.forceReinitialize()
 
     assertEquals(1, goodCalls.get)
 

From ca6ac9393bae9161465275cb8f836bc987a5098d Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Mon, 18 Mar 2019 08:51:50 +0000
Subject: [PATCH 0032/1071] MINOR: Retain public constructors of classes from
 public API (#6455)

TopicDescription and ConsumerGroupDescription in org.apache.kafka.clients.admin. are part of the public API, so we should retain the existing public constructor. Changed the new constructor with authorized operations to be package-private to avoid maintaining more public constructors since we only expect admin client to use this.

Reviewers: Manikumar Reddy 
---
 .../clients/admin/ConsumerGroupDescription.java   |  9 +++++++++
 .../kafka/clients/admin/TopicDescription.java     | 15 ++++++++++++++-
 .../internals/InternalTopicManagerTest.java       |  6 +++---
 .../kafka/trogdor/common/WorkerUtilsTest.java     |  9 +++------
 4 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupDescription.java b/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupDescription.java
index 7320f6568150b..52f23ed116ba9 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupDescription.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupDescription.java
@@ -41,6 +41,15 @@ public class ConsumerGroupDescription {
     private final Set authorizedOperations;
 
     public ConsumerGroupDescription(String groupId,
+                                    boolean isSimpleConsumerGroup,
+                                    Collection members,
+                                    String partitionAssignor,
+                                    ConsumerGroupState state,
+                                    Node coordinator) {
+        this(groupId, isSimpleConsumerGroup, members, partitionAssignor, state, coordinator, Collections.emptySet());
+    }
+
+    ConsumerGroupDescription(String groupId,
                                     boolean isSimpleConsumerGroup,
                                     Collection members,
                                     String partitionAssignor,
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/TopicDescription.java b/clients/src/main/java/org/apache/kafka/clients/admin/TopicDescription.java
index daadac00940e0..c6d44e88cabc0 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/TopicDescription.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/TopicDescription.java
@@ -21,6 +21,7 @@
 import org.apache.kafka.common.acl.AclOperation;
 import org.apache.kafka.common.utils.Utils;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
@@ -50,6 +51,18 @@ public int hashCode() {
         return Objects.hash(name, internal, partitions, authorizedOperations);
     }
 
+    /**
+     * Create an instance with the specified parameters.
+     *
+     * @param name The topic name
+     * @param internal Whether the topic is internal to Kafka
+     * @param partitions A list of partitions where the index represents the partition id and the element contains
+     *                   leadership and replica information for that partition.
+     */
+    public TopicDescription(String name, boolean internal, List partitions) {
+        this(name, internal, partitions, Collections.emptySet());
+    }
+
     /**
      * Create an instance with the specified parameters.
      *
@@ -59,7 +72,7 @@ public int hashCode() {
      *                   leadership and replica information for that partition.
      * @param authorizedOperations authorized operations for this topic
      */
-    public TopicDescription(String name, boolean internal, List partitions,
+    TopicDescription(String name, boolean internal, List partitions,
                             Set authorizedOperations) {
         this.name = name;
         this.internal = internal;
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopicManagerTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopicManagerTest.java
index e2dc376d83aed..074228a6c147b 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopicManagerTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopicManagerTest.java
@@ -113,17 +113,17 @@ public void shouldCreateRequiredTopics() throws Exception {
             {
                 add(new TopicPartitionInfo(0, broker1, singleReplica, Collections.emptyList()));
             }
-        }, Collections.emptySet()), mockAdminClient.describeTopics(Collections.singleton(topic)).values().get(topic).get());
+        }), mockAdminClient.describeTopics(Collections.singleton(topic)).values().get(topic).get());
         assertEquals(new TopicDescription(topic2, false, new ArrayList() {
             {
                 add(new TopicPartitionInfo(0, broker1, singleReplica, Collections.emptyList()));
             }
-        }, Collections.emptySet()), mockAdminClient.describeTopics(Collections.singleton(topic2)).values().get(topic2).get());
+        }), mockAdminClient.describeTopics(Collections.singleton(topic2)).values().get(topic2).get());
         assertEquals(new TopicDescription(topic3, false, new ArrayList() {
             {
                 add(new TopicPartitionInfo(0, broker1, singleReplica, Collections.emptyList()));
             }
-        }, Collections.emptySet()), mockAdminClient.describeTopics(Collections.singleton(topic3)).values().get(topic3).get());
+        }), mockAdminClient.describeTopics(Collections.singleton(topic3)).values().get(topic3).get());
 
         final ConfigResource resource = new ConfigResource(ConfigResource.Type.TOPIC, topic);
         final ConfigResource resource2 = new ConfigResource(ConfigResource.Type.TOPIC, topic2);
diff --git a/tools/src/test/java/org/apache/kafka/trogdor/common/WorkerUtilsTest.java b/tools/src/test/java/org/apache/kafka/trogdor/common/WorkerUtilsTest.java
index 29e966cb377aa..a35efe199aa8f 100644
--- a/tools/src/test/java/org/apache/kafka/trogdor/common/WorkerUtilsTest.java
+++ b/tools/src/test/java/org/apache/kafka/trogdor/common/WorkerUtilsTest.java
@@ -81,8 +81,7 @@ public void testCreateOneTopic() throws Throwable {
             new TopicDescription(
                 TEST_TOPIC, false,
                 Collections.singletonList(
-                    new TopicPartitionInfo(0, broker1, singleReplica, Collections.emptyList())),
-                    Collections.emptySet()),
+                    new TopicPartitionInfo(0, broker1, singleReplica, Collections.emptyList()))),
             adminClient.describeTopics(
                 Collections.singleton(TEST_TOPIC)).values().get(TEST_TOPIC).get()
         );
@@ -99,8 +98,7 @@ public void testCreateRetriesOnTimeout() throws Throwable {
             new TopicDescription(
                 TEST_TOPIC, false,
                 Collections.singletonList(
-                    new TopicPartitionInfo(0, broker1, singleReplica, Collections.emptyList())),
-                    Collections.emptySet()),
+                    new TopicPartitionInfo(0, broker1, singleReplica, Collections.emptyList()))),
             adminClient.describeTopics(
                 Collections.singleton(TEST_TOPIC)).values().get(TEST_TOPIC).get()
         );
@@ -180,8 +178,7 @@ public void testCreatesNotExistingTopics() throws Throwable {
             new TopicDescription(
                 TEST_TOPIC, false,
                 Collections.singletonList(
-                    new TopicPartitionInfo(0, broker1, singleReplica, Collections.emptyList())),
-                    Collections.emptySet()),
+                    new TopicPartitionInfo(0, broker1, singleReplica, Collections.emptyList()))),
             adminClient.describeTopics(Collections.singleton(TEST_TOPIC)).values().get(TEST_TOPIC).get()
         );
     }

From 5192956581dad7eeb7796d51d85195102f151ab0 Mon Sep 17 00:00:00 2001
From: Anna Povzner 
Date: Mon, 18 Mar 2019 09:48:49 -0700
Subject: [PATCH 0033/1071] MINOR: Improve verification in flaky
 testPartitionReassignmentDuringDeleteTopic (#6460)

Reviewers: Rajini Sivaram 
---
 core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala b/core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala
index 6e5ffc81d3680..400920eeae9fd 100644
--- a/core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala
@@ -138,8 +138,11 @@ class DeleteTopicTest extends ZooKeeperTestHarness {
     }, "Partition reassignment shouldn't complete.")
     val controllerId = zkClient.getControllerId.getOrElse(fail("Controller doesn't exist"))
     val controller = servers.filter(s => s.config.brokerId == controllerId).head
-    assertFalse("Partition reassignment should fail",
-      controller.kafkaController.controllerContext.partitionsBeingReassigned.contains(topicPartition))
+
+    // partitionsBeingReassigned is updated after re-assignment znode is removed, so wait again
+    TestUtils.waitUntilTrue(() => {
+      !controller.kafkaController.controllerContext.partitionsBeingReassigned.contains(topicPartition)
+    }, "Partition should be removed from partitionsBeingReassigned.")
     val assignedReplicas = zkClient.getReplicasForPartition(new TopicPartition(topic, 0))
     assertEquals("Partition should not be reassigned to 0, 1, 2", oldAssignedReplicas, assignedReplicas)
     follower.startup()

From 810bc69b7a474297aa345613cf18cb8bfb115603 Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Mon, 18 Mar 2019 18:07:24 +0000
Subject: [PATCH 0034/1071] KAFKA-8121; Shutdown ZK client expiry handler
 earlier during close (#6462)

Shutdown session expiry thread prior to closing ZooKeeper client to ensure that new clients are not created by the expiry thread and left active when returning from ZooKeeperClient.close().

Reviewers: Ismael Juma 
---
 .../src/main/scala/kafka/zookeeper/ZooKeeperClient.scala | 9 ++++++---
 .../scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala | 3 +++
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/core/src/main/scala/kafka/zookeeper/ZooKeeperClient.scala b/core/src/main/scala/kafka/zookeeper/ZooKeeperClient.scala
index 4d5ef96f4659b..ad4da8b5385a4 100755
--- a/core/src/main/scala/kafka/zookeeper/ZooKeeperClient.scala
+++ b/core/src/main/scala/kafka/zookeeper/ZooKeeperClient.scala
@@ -320,6 +320,12 @@ class ZooKeeperClient(connectString: String,
 
   def close(): Unit = {
     info("Closing.")
+
+    // Shutdown scheduler outside of lock to avoid deadlock if scheduler
+    // is waiting for lock to process session expiry. Close expiry thread
+    // first to ensure that new clients are not created during close().
+    expiryScheduler.shutdown()
+
     inWriteLock(initializationLock) {
       zNodeChangeHandlers.clear()
       zNodeChildChangeHandlers.clear()
@@ -327,9 +333,6 @@ class ZooKeeperClient(connectString: String,
       zooKeeper.close()
       metricNames.foreach(removeMetric(_))
     }
-    // Shutdown scheduler outside of lock to avoid deadlock if scheduler
-    // is waiting for lock to process session expiry
-    expiryScheduler.shutdown()
     info("Closed.")
   }
 
diff --git a/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala b/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala
index e943348180f35..a4b63e07ad2b8 100644
--- a/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala
+++ b/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala
@@ -43,6 +43,7 @@ class ZooKeeperClientTest extends ZooKeeperTestHarness {
 
   @Before
   override def setUp() {
+    ZooKeeperTestHarness.verifyNoUnexpectedThreads("@Before")
     cleanMetricsRegistry()
     super.setUp()
     zooKeeperClient = new ZooKeeperClient(zkConnect, zkSessionTimeout, zkConnectionTimeout, zkMaxInFlightRequests,
@@ -55,6 +56,7 @@ class ZooKeeperClientTest extends ZooKeeperTestHarness {
       zooKeeperClient.close()
     super.tearDown()
     System.clearProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM)
+    ZooKeeperTestHarness.verifyNoUnexpectedThreads("@After")
   }
 
   @Test
@@ -596,6 +598,7 @@ class ZooKeeperClientTest extends ZooKeeperTestHarness {
         }
       })
       assertFalse("Close completed without shutting down expiry scheduler gracefully", closeFuture.isDone)
+      assertTrue(zooKeeperClient.currentZooKeeper.getState.isAlive) // Client should be closed after expiry handler
       semaphore.release()
       closeFuture.get(10, TimeUnit.SECONDS)
       assertFalse("Expiry executor not shutdown", zooKeeperClient.expiryScheduler.isStarted)

From 8406f3624d8f99b614eb7171b71fae8b0e663dcb Mon Sep 17 00:00:00 2001
From: Boyang Chen 
Date: Mon, 18 Mar 2019 13:26:09 -0700
Subject: [PATCH 0035/1071] KAFKA-7858: Automatically generate JoinGroup
 request/response

Reviewers: Colin P. McCabe 
---
 .../internals/AbstractCoordinator.java        |  34 +--
 .../internals/ConsumerCoordinator.java        |  23 +-
 .../apache/kafka/common/protocol/ApiKeys.java |   6 +-
 .../common/requests/AbstractResponse.java     |   2 +-
 .../common/requests/JoinGroupRequest.java     | 252 +++---------------
 .../common/requests/JoinGroupResponse.java    | 197 ++------------
 .../common/message/JoinGroupResponse.json     |   2 +-
 .../clients/consumer/KafkaConsumerTest.java   |  37 ++-
 .../internals/AbstractCoordinatorTest.java    |  32 ++-
 .../internals/ConsumerCoordinatorTest.java    |  79 +++---
 .../common/requests/RequestResponseTest.java  |  56 +++-
 .../distributed/WorkerCoordinator.java        |  18 +-
 .../distributed/WorkerCoordinatorTest.java    |  85 ++++--
 .../main/scala/kafka/server/KafkaApis.scala   |  63 +++--
 .../kafka/api/AuthorizerIntegrationTest.scala |  20 +-
 .../unit/kafka/server/RequestQuotaTest.scala  |  20 +-
 16 files changed, 393 insertions(+), 533 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index fd7431b01501c..4ff4e19590aed 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -29,6 +29,8 @@
 import org.apache.kafka.common.errors.RebalanceInProgressException;
 import org.apache.kafka.common.errors.RetriableException;
 import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.message.JoinGroupRequestData;
+import org.apache.kafka.common.message.JoinGroupResponseData;
 import org.apache.kafka.common.message.LeaveGroupRequestData;
 import org.apache.kafka.common.metrics.Measurable;
 import org.apache.kafka.common.metrics.MetricConfig;
@@ -44,7 +46,6 @@
 import org.apache.kafka.common.requests.HeartbeatRequest;
 import org.apache.kafka.common.requests.HeartbeatResponse;
 import org.apache.kafka.common.requests.JoinGroupRequest;
-import org.apache.kafka.common.requests.JoinGroupRequest.ProtocolMetadata;
 import org.apache.kafka.common.requests.JoinGroupResponse;
 import org.apache.kafka.common.requests.LeaveGroupRequest;
 import org.apache.kafka.common.requests.LeaveGroupResponse;
@@ -86,7 +87,7 @@
  *
  * To leverage this protocol, an implementation must define the format of metadata provided by each
  * member for group registration in {@link #metadata()} and the format of the state assignment provided
- * by the leader in {@link #performAssignment(String, String, Map)} and becomes available to members in
+ * by the leader in {@link #performAssignment(String, String, List)} and becomes available to members in
  * {@link #onJoinComplete(int, String, String, ByteBuffer)}.
  *
  * Note on locking: this class shares state between the caller and a background thread which is
@@ -183,7 +184,7 @@ public AbstractCoordinator(LogContext logContext,
      * on the preference).
      * @return Non-empty map of supported protocols and metadata
      */
-    protected abstract List metadata();
+    protected abstract JoinGroupRequestData.JoinGroupRequestProtocolSet metadata();
 
     /**
      * Invoked prior to each group join or rejoin. This is typically used to perform any
@@ -202,7 +203,7 @@ public AbstractCoordinator(LogContext logContext,
      */
     protected abstract Map performAssignment(String leaderId,
                                                                  String protocol,
-                                                                 Map allMemberMetadata);
+                                                                 List allMemberMetadata);
 
     /**
      * Invoked when a group member has successfully joined a group. If this call fails with an exception,
@@ -476,7 +477,7 @@ public void onFailure(RuntimeException e) {
 
     /**
      * Join the group and return the assignment for the next generation. This function handles both
-     * JoinGroup and SyncGroup, delegating to {@link #performAssignment(String, String, Map)} if
+     * JoinGroup and SyncGroup, delegating to {@link #performAssignment(String, String, List)} if
      * elected leader by the coordinator.
      *
      * NOTE: This is visible only for testing
@@ -490,11 +491,14 @@ RequestFuture sendJoinGroupRequest() {
         // send a join group request to the coordinator
         log.info("(Re-)joining group");
         JoinGroupRequest.Builder requestBuilder = new JoinGroupRequest.Builder(
-                groupId,
-                this.sessionTimeoutMs,
-                this.generation.memberId,
-                protocolType(),
-                metadata()).setRebalanceTimeout(this.rebalanceTimeoutMs);
+                new JoinGroupRequestData()
+                        .setGroupId(groupId)
+                        .setSessionTimeoutMs(this.sessionTimeoutMs)
+                        .setMemberId(this.generation.memberId)
+                        .setProtocolType(protocolType())
+                        .setProtocols(metadata())
+                        .setRebalanceTimeoutMs(this.rebalanceTimeoutMs)
+        );
 
         log.debug("Sending JoinGroup ({}) to coordinator {}", requestBuilder, this.coordinator);
 
@@ -520,8 +524,8 @@ public void handle(JoinGroupResponse joinResponse, RequestFuture fut
                         // the group. In this case, we do not want to continue with the sync group.
                         future.raise(new UnjoinedGroupException());
                     } else {
-                        AbstractCoordinator.this.generation = new Generation(joinResponse.generationId(),
-                                joinResponse.memberId(), joinResponse.groupProtocol());
+                        AbstractCoordinator.this.generation = new Generation(joinResponse.data().generationId(),
+                                joinResponse.data().memberId(), joinResponse.data().protocolName());
                         if (joinResponse.isLeader()) {
                             onJoinLeader(joinResponse).chain(future);
                         } else {
@@ -562,7 +566,7 @@ public void handle(JoinGroupResponse joinResponse, RequestFuture fut
                 // and send another join group request in next cycle.
                 synchronized (AbstractCoordinator.this) {
                     AbstractCoordinator.this.generation = new Generation(OffsetCommitRequest.DEFAULT_GENERATION_ID,
-                        joinResponse.memberId(), null);
+                        joinResponse.data().memberId(), null);
                     AbstractCoordinator.this.rejoinNeeded = true;
                     AbstractCoordinator.this.state = MemberState.UNJOINED;
                 }
@@ -587,8 +591,8 @@ private RequestFuture onJoinFollower() {
     private RequestFuture onJoinLeader(JoinGroupResponse joinResponse) {
         try {
             // perform the leader synchronization and send back the assignment for the group
-            Map groupAssignment = performAssignment(joinResponse.leaderId(), joinResponse.groupProtocol(),
-                    joinResponse.members());
+            Map groupAssignment = performAssignment(joinResponse.data().leader(), joinResponse.data().protocolName(),
+                    joinResponse.data().members());
 
             SyncGroupRequest.Builder requestBuilder =
                     new SyncGroupRequest.Builder(groupId, generation.generationId, generation.memberId, groupAssignment);
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 79907074af27b..2b949a37deb21 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -33,6 +33,8 @@
 import org.apache.kafka.common.errors.TimeoutException;
 import org.apache.kafka.common.errors.TopicAuthorizationException;
 import org.apache.kafka.common.errors.WakeupException;
+import org.apache.kafka.common.message.JoinGroupRequestData;
+import org.apache.kafka.common.message.JoinGroupResponseData;
 import org.apache.kafka.common.metrics.Measurable;
 import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
@@ -40,7 +42,6 @@
 import org.apache.kafka.common.metrics.stats.Avg;
 import org.apache.kafka.common.metrics.stats.Max;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.requests.JoinGroupRequest.ProtocolMetadata;
 import org.apache.kafka.common.requests.OffsetCommitRequest;
 import org.apache.kafka.common.requests.OffsetCommitResponse;
 import org.apache.kafka.common.requests.OffsetFetchRequest;
@@ -166,16 +167,20 @@ public String protocolType() {
     }
 
     @Override
-    protected List metadata() {
+    protected JoinGroupRequestData.JoinGroupRequestProtocolSet metadata() {
         log.debug("Joining group with current subscription: {}", subscriptions.subscription());
         this.joinedSubscription = subscriptions.subscription();
-        List metadataList = new ArrayList<>();
+        JoinGroupRequestData.JoinGroupRequestProtocolSet protocolSet = new JoinGroupRequestData.JoinGroupRequestProtocolSet();
+
         for (PartitionAssignor assignor : assignors) {
             Subscription subscription = assignor.subscription(joinedSubscription);
             ByteBuffer metadata = ConsumerProtocol.serializeSubscription(subscription);
-            metadataList.add(new ProtocolMetadata(assignor.name(), metadata));
+
+            protocolSet.add(new JoinGroupRequestData.JoinGroupRequestProtocol()
+                    .setName(assignor.name())
+                    .setMetadata(metadata.array()));
         }
-        return metadataList;
+        return protocolSet;
     }
 
     public void updatePatternSubscription(Cluster cluster) {
@@ -385,16 +390,16 @@ private void updateGroupSubscription(Set topics) {
     @Override
     protected Map performAssignment(String leaderId,
                                                         String assignmentStrategy,
-                                                        Map allSubscriptions) {
+                                                        List allSubscriptions) {
         PartitionAssignor assignor = lookupAssignor(assignmentStrategy);
         if (assignor == null)
             throw new IllegalStateException("Coordinator selected invalid assignment protocol: " + assignmentStrategy);
 
         Set allSubscribedTopics = new HashSet<>();
         Map subscriptions = new HashMap<>();
-        for (Map.Entry subscriptionEntry : allSubscriptions.entrySet()) {
-            Subscription subscription = ConsumerProtocol.deserializeSubscription(subscriptionEntry.getValue());
-            subscriptions.put(subscriptionEntry.getKey(), subscription);
+        for (JoinGroupResponseData.JoinGroupResponseMember memberSubScription : allSubscriptions) {
+            Subscription subscription = ConsumerProtocol.deserializeSubscription(ByteBuffer.wrap(memberSubScription.metadata()));
+            subscriptions.put(memberSubScription.memberId(), subscription);
             allSubscribedTopics.addAll(subscription.topics());
         }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
index c23aa7ef5aa4b..3f8d80df5a71f 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
@@ -22,6 +22,8 @@
 import org.apache.kafka.common.message.DescribeGroupsResponseData;
 import org.apache.kafka.common.message.ElectPreferredLeadersRequestData;
 import org.apache.kafka.common.message.ElectPreferredLeadersResponseData;
+import org.apache.kafka.common.message.JoinGroupRequestData;
+import org.apache.kafka.common.message.JoinGroupResponseData;
 import org.apache.kafka.common.message.LeaveGroupRequestData;
 import org.apache.kafka.common.message.LeaveGroupResponseData;
 import org.apache.kafka.common.message.MetadataRequestData;
@@ -81,8 +83,6 @@
 import org.apache.kafka.common.requests.HeartbeatResponse;
 import org.apache.kafka.common.requests.InitProducerIdRequest;
 import org.apache.kafka.common.requests.InitProducerIdResponse;
-import org.apache.kafka.common.requests.JoinGroupRequest;
-import org.apache.kafka.common.requests.JoinGroupResponse;
 import org.apache.kafka.common.requests.LeaderAndIsrRequest;
 import org.apache.kafka.common.requests.LeaderAndIsrResponse;
 import org.apache.kafka.common.requests.ListGroupsRequest;
@@ -135,7 +135,7 @@ public enum ApiKeys {
     OFFSET_FETCH(9, "OffsetFetch", OffsetFetchRequest.schemaVersions(), OffsetFetchResponse.schemaVersions()),
     FIND_COORDINATOR(10, "FindCoordinator", FindCoordinatorRequest.schemaVersions(),
             FindCoordinatorResponse.schemaVersions()),
-    JOIN_GROUP(11, "JoinGroup", JoinGroupRequest.schemaVersions(), JoinGroupResponse.schemaVersions()),
+    JOIN_GROUP(11, "JoinGroup", JoinGroupRequestData.SCHEMAS, JoinGroupResponseData.SCHEMAS),
     HEARTBEAT(12, "Heartbeat", HeartbeatRequest.schemaVersions(), HeartbeatResponse.schemaVersions()),
     LEAVE_GROUP(13, "LeaveGroup", LeaveGroupRequestData.SCHEMAS, LeaveGroupResponseData.SCHEMAS),
     SYNC_GROUP(14, "SyncGroup", SyncGroupRequest.schemaVersions(), SyncGroupResponse.schemaVersions()),
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
index 1d3fd771203ea..eadd3024769fc 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
@@ -85,7 +85,7 @@ public static AbstractResponse parseResponse(ApiKeys apiKey, Struct struct, shor
             case FIND_COORDINATOR:
                 return new FindCoordinatorResponse(struct);
             case JOIN_GROUP:
-                return new JoinGroupResponse(struct);
+                return new JoinGroupResponse(struct, version);
             case HEARTBEAT:
                 return new HeartbeatResponse(struct);
             case LEAVE_GROUP:
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java
index 236d6848f53aa..2f4617213a1d9 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java
@@ -16,188 +16,56 @@
  */
 package org.apache.kafka.common.requests;
 
+import org.apache.kafka.common.message.JoinGroupRequestData;
+import org.apache.kafka.common.message.JoinGroupResponseData;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.ArrayOf;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
-import org.apache.kafka.common.utils.Utils;
 
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.List;
-
-import static org.apache.kafka.common.protocol.CommonFields.GROUP_ID;
-import static org.apache.kafka.common.protocol.CommonFields.MEMBER_ID;
-import static org.apache.kafka.common.protocol.types.Type.BYTES;
-import static org.apache.kafka.common.protocol.types.Type.INT32;
-import static org.apache.kafka.common.protocol.types.Type.STRING;
 
 public class JoinGroupRequest extends AbstractRequest {
-    private static final String SESSION_TIMEOUT_KEY_NAME = "session_timeout";
-    private static final String REBALANCE_TIMEOUT_KEY_NAME = "rebalance_timeout";
-    private static final String PROTOCOL_TYPE_KEY_NAME = "protocol_type";
-    private static final String GROUP_PROTOCOLS_KEY_NAME = "group_protocols";
-    private static final String PROTOCOL_NAME_KEY_NAME = "protocol_name";
-    private static final String PROTOCOL_METADATA_KEY_NAME = "protocol_metadata";
-
-    /* Join group api */
-    private static final Schema JOIN_GROUP_REQUEST_PROTOCOL_V0 = new Schema(
-            new Field(PROTOCOL_NAME_KEY_NAME, STRING),
-            new Field(PROTOCOL_METADATA_KEY_NAME, BYTES));
-
-    private static final Schema JOIN_GROUP_REQUEST_V0 = new Schema(
-            GROUP_ID,
-            new Field(SESSION_TIMEOUT_KEY_NAME, INT32, "The coordinator considers the consumer dead if it receives " +
-                    "no heartbeat after this timeout in ms."),
-            MEMBER_ID,
-            new Field(PROTOCOL_TYPE_KEY_NAME, STRING, "Unique name for class of protocols implemented by group"),
-            new Field(GROUP_PROTOCOLS_KEY_NAME, new ArrayOf(JOIN_GROUP_REQUEST_PROTOCOL_V0), "List of protocols " +
-                    "that the member supports"));
-
-    private static final Schema JOIN_GROUP_REQUEST_V1 = new Schema(
-            GROUP_ID,
-            new Field(SESSION_TIMEOUT_KEY_NAME, INT32, "The coordinator considers the consumer dead if it receives no " +
-                    "heartbeat after this timeout in ms."),
-            new Field(REBALANCE_TIMEOUT_KEY_NAME, INT32, "The maximum time that the coordinator will wait for each " +
-                    "member to rejoin when rebalancing the group"),
-            MEMBER_ID,
-            new Field(PROTOCOL_TYPE_KEY_NAME, STRING, "Unique name for class of protocols implemented by group"),
-            new Field(GROUP_PROTOCOLS_KEY_NAME, new ArrayOf(JOIN_GROUP_REQUEST_PROTOCOL_V0), "List of protocols " +
-                    "that the member supports"));
-
-    /* v2 request is the same as v1. Throttle time has been added to response */
-    private static final Schema JOIN_GROUP_REQUEST_V2 = JOIN_GROUP_REQUEST_V1;
-
-    /**
-     * The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
-     */
-    private static final Schema JOIN_GROUP_REQUEST_V3 = JOIN_GROUP_REQUEST_V2;
-
-    /**
-     * The version number is bumped to indicate that client needs to issue a second join group request under first try
-     * with UNKNOWN_MEMBER_ID.
-     */
-    private static final Schema JOIN_GROUP_REQUEST_V4 = JOIN_GROUP_REQUEST_V3;
-
-    public static Schema[] schemaVersions() {
-        return new Schema[] {JOIN_GROUP_REQUEST_V0, JOIN_GROUP_REQUEST_V1, JOIN_GROUP_REQUEST_V2,
-            JOIN_GROUP_REQUEST_V3, JOIN_GROUP_REQUEST_V4};
-    }
-
-    public static final String UNKNOWN_MEMBER_ID = "";
-
-    private final String groupId;
-    private final int sessionTimeout;
-    private final int rebalanceTimeout;
-    private final String memberId;
-    private final String protocolType;
-    private final List groupProtocols;
-
-    public static class ProtocolMetadata {
-        private final String name;
-        private final ByteBuffer metadata;
-
-        public ProtocolMetadata(String name, ByteBuffer metadata) {
-            this.name = name;
-            this.metadata = metadata;
-        }
-
-        public String name() {
-            return name;
-        }
-
-        public ByteBuffer metadata() {
-            return metadata;
-        }
-    }
 
     public static class Builder extends AbstractRequest.Builder {
-        private final String groupId;
-        private final int sessionTimeout;
-        private final String memberId;
-        private final String protocolType;
-        private final List groupProtocols;
-        private int rebalanceTimeout = 0;
 
-        public Builder(String groupId, int sessionTimeout, String memberId,
-                       String protocolType, List groupProtocols) {
-            super(ApiKeys.JOIN_GROUP);
-            this.groupId = groupId;
-            this.sessionTimeout = sessionTimeout;
-            this.rebalanceTimeout = sessionTimeout;
-            this.memberId = memberId;
-            this.protocolType = protocolType;
-            this.groupProtocols = groupProtocols;
-        }
+        private final JoinGroupRequestData data;
 
-        public Builder setRebalanceTimeout(int rebalanceTimeout) {
-            this.rebalanceTimeout = rebalanceTimeout;
-            return this;
+        public Builder(JoinGroupRequestData data) {
+            super(ApiKeys.JOIN_GROUP);
+            this.data = data;
         }
 
         @Override
         public JoinGroupRequest build(short version) {
-            if (version < 1) {
-                // v0 had no rebalance timeout but used session timeout implicitly
-                rebalanceTimeout = sessionTimeout;
-            }
-            return new JoinGroupRequest(version, groupId, sessionTimeout,
-                    rebalanceTimeout, memberId, protocolType, groupProtocols);
+            return new JoinGroupRequest(data, version);
         }
 
         @Override
         public String toString() {
-            StringBuilder bld = new StringBuilder();
-            bld.append("(type: JoinGroupRequest").
-                append(", groupId=").append(groupId).
-                append(", sessionTimeout=").append(sessionTimeout).
-                append(", rebalanceTimeout=").append(rebalanceTimeout).
-                append(", memberId=").append(memberId).
-                append(", protocolType=").append(protocolType).
-                append(", groupProtocols=").append(Utils.join(groupProtocols, ", ")).
-                append(")");
-            return bld.toString();
+            return data.toString();
         }
     }
 
-    private JoinGroupRequest(short version, String groupId, int sessionTimeout,
-            int rebalanceTimeout, String memberId, String protocolType,
-            List groupProtocols) {
-        super(ApiKeys.JOIN_GROUP, version);
-        this.groupId = groupId;
-        this.sessionTimeout = sessionTimeout;
-        this.rebalanceTimeout = rebalanceTimeout;
-        this.memberId = memberId;
-        this.protocolType = protocolType;
-        this.groupProtocols = groupProtocols;
-    }
-
-    public JoinGroupRequest(Struct struct, short versionId) {
-        super(ApiKeys.JOIN_GROUP, versionId);
+    private final JoinGroupRequestData data;
+    private final short version;
 
-        groupId = struct.get(GROUP_ID);
-        sessionTimeout = struct.getInt(SESSION_TIMEOUT_KEY_NAME);
+    public static final String UNKNOWN_MEMBER_ID = "";
 
-        if (struct.hasField(REBALANCE_TIMEOUT_KEY_NAME))
-            // rebalance timeout is added in v1
-            rebalanceTimeout = struct.getInt(REBALANCE_TIMEOUT_KEY_NAME);
-        else
-            // v0 had no rebalance timeout but used session timeout implicitly
-            rebalanceTimeout = sessionTimeout;
+    public JoinGroupRequest(JoinGroupRequestData data, short version) {
+        super(ApiKeys.JOIN_GROUP, version);
+        this.data = data;
+        this.version = version;
+    }
 
-        memberId = struct.get(MEMBER_ID);
-        protocolType = struct.getString(PROTOCOL_TYPE_KEY_NAME);
+    public JoinGroupRequest(Struct struct, short version) {
+        super(ApiKeys.JOIN_GROUP, version);
+        this.data = new JoinGroupRequestData(struct, version);
+        this.version = version;
+    }
 
-        groupProtocols = new ArrayList<>();
-        for (Object groupProtocolObj : struct.getArray(GROUP_PROTOCOLS_KEY_NAME)) {
-            Struct groupProtocolStruct = (Struct) groupProtocolObj;
-            String name = groupProtocolStruct.getString(PROTOCOL_NAME_KEY_NAME);
-            ByteBuffer metadata = groupProtocolStruct.getBytes(PROTOCOL_METADATA_KEY_NAME);
-            groupProtocols.add(new ProtocolMetadata(name, metadata));
-        }
+    public JoinGroupRequestData data() {
+        return data;
     }
 
     @Override
@@ -207,77 +75,39 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
             case 0:
             case 1:
                 return new JoinGroupResponse(
-                    Errors.forException(e),
-                    JoinGroupResponse.UNKNOWN_GENERATION_ID,
-                    JoinGroupResponse.UNKNOWN_PROTOCOL,
-                    JoinGroupResponse.UNKNOWN_MEMBER_ID, // memberId
-                    JoinGroupResponse.UNKNOWN_MEMBER_ID, // leaderId
-                    Collections.emptyMap());
+                        new JoinGroupResponseData()
+                                .setErrorCode(Errors.forException(e).code())
+                                .setGenerationId(JoinGroupResponse.UNKNOWN_GENERATION_ID)
+                                .setProtocolName(JoinGroupResponse.UNKNOWN_PROTOCOL)
+                                .setLeader(JoinGroupResponse.UNKNOWN_MEMBER_ID)
+                                .setMemberId(JoinGroupResponse.UNKNOWN_MEMBER_ID)
+                                .setMembers(Collections.emptyList())
+                );
             case 2:
             case 3:
             case 4:
                 return new JoinGroupResponse(
-                    throttleTimeMs,
-                    Errors.forException(e),
-                    JoinGroupResponse.UNKNOWN_GENERATION_ID,
-                    JoinGroupResponse.UNKNOWN_PROTOCOL,
-                    JoinGroupResponse.UNKNOWN_MEMBER_ID, // memberId
-                    JoinGroupResponse.UNKNOWN_MEMBER_ID, // leaderId
-                    Collections.emptyMap());
-
+                        new JoinGroupResponseData()
+                                .setThrottleTimeMs(throttleTimeMs)
+                                .setErrorCode(Errors.forException(e).code())
+                                .setGenerationId(JoinGroupResponse.UNKNOWN_GENERATION_ID)
+                                .setProtocolName(JoinGroupResponse.UNKNOWN_PROTOCOL)
+                                .setLeader(JoinGroupResponse.UNKNOWN_MEMBER_ID)
+                                .setMemberId(JoinGroupResponse.UNKNOWN_MEMBER_ID)
+                                .setMembers(Collections.emptyList())
+                );
             default:
                 throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
                         versionId, this.getClass().getSimpleName(), ApiKeys.JOIN_GROUP.latestVersion()));
         }
     }
 
-    public String groupId() {
-        return groupId;
-    }
-
-    public int sessionTimeout() {
-        return sessionTimeout;
-    }
-
-    public int rebalanceTimeout() {
-        return rebalanceTimeout;
-    }
-
-    public String memberId() {
-        return memberId;
-    }
-
-    public List groupProtocols() {
-        return groupProtocols;
-    }
-
-    public String protocolType() {
-        return protocolType;
-    }
-
     public static JoinGroupRequest parse(ByteBuffer buffer, short version) {
         return new JoinGroupRequest(ApiKeys.JOIN_GROUP.parseRequest(version, buffer), version);
     }
 
     @Override
     protected Struct toStruct() {
-        short version = version();
-        Struct struct = new Struct(ApiKeys.JOIN_GROUP.requestSchema(version));
-        struct.set(GROUP_ID, groupId);
-        struct.set(SESSION_TIMEOUT_KEY_NAME, sessionTimeout);
-        if (version >= 1) {
-            struct.set(REBALANCE_TIMEOUT_KEY_NAME, rebalanceTimeout);
-        }
-        struct.set(MEMBER_ID, memberId);
-        struct.set(PROTOCOL_TYPE_KEY_NAME, protocolType);
-        List groupProtocolsList = new ArrayList<>(groupProtocols.size());
-        for (ProtocolMetadata protocol : groupProtocols) {
-            Struct protocolStruct = struct.instance(GROUP_PROTOCOLS_KEY_NAME);
-            protocolStruct.set(PROTOCOL_NAME_KEY_NAME, protocol.name);
-            protocolStruct.set(PROTOCOL_METADATA_KEY_NAME, protocol.metadata);
-            groupProtocolsList.add(protocolStruct);
-        }
-        struct.set(GROUP_PROTOCOLS_KEY_NAME, groupProtocolsList.toArray());
-        return struct;
+        return data.toStruct(version);
     }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java
index 55cb97f01c512..0e1644e465923 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java
@@ -16,217 +16,70 @@
  */
 package org.apache.kafka.common.requests;
 
+import org.apache.kafka.common.message.JoinGroupResponseData;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.ArrayOf;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
-import org.apache.kafka.common.utils.Utils;
 
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
+import java.util.Collections;
 import java.util.Map;
 
-import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
-import static org.apache.kafka.common.protocol.CommonFields.GENERATION_ID;
-import static org.apache.kafka.common.protocol.CommonFields.MEMBER_ID;
-import static org.apache.kafka.common.protocol.CommonFields.THROTTLE_TIME_MS;
-import static org.apache.kafka.common.protocol.types.Type.BYTES;
-import static org.apache.kafka.common.protocol.types.Type.STRING;
-
 public class JoinGroupResponse extends AbstractResponse {
 
-    private static final String GROUP_PROTOCOL_KEY_NAME = "group_protocol";
-    private static final String LEADER_ID_KEY_NAME = "leader_id";
-    private static final String MEMBERS_KEY_NAME = "members";
-
-    private static final String MEMBER_METADATA_KEY_NAME = "member_metadata";
-
-    private static final Schema JOIN_GROUP_RESPONSE_MEMBER_V0 = new Schema(
-            MEMBER_ID,
-            new Field(MEMBER_METADATA_KEY_NAME, BYTES));
-
-    private static final Schema JOIN_GROUP_RESPONSE_V0 = new Schema(
-            ERROR_CODE,
-            GENERATION_ID,
-            new Field(GROUP_PROTOCOL_KEY_NAME, STRING, "The group protocol selected by the coordinator"),
-            new Field(LEADER_ID_KEY_NAME, STRING, "The leader of the group"),
-            MEMBER_ID,
-            new Field(MEMBERS_KEY_NAME, new ArrayOf(JOIN_GROUP_RESPONSE_MEMBER_V0)));
-
-    private static final Schema JOIN_GROUP_RESPONSE_V1 = JOIN_GROUP_RESPONSE_V0;
-
-    private static final Schema JOIN_GROUP_RESPONSE_V2 = new Schema(
-            THROTTLE_TIME_MS,
-            ERROR_CODE,
-            GENERATION_ID,
-            new Field(GROUP_PROTOCOL_KEY_NAME, STRING, "The group protocol selected by the coordinator"),
-            new Field(LEADER_ID_KEY_NAME, STRING, "The leader of the group"),
-            MEMBER_ID,
-            new Field(MEMBERS_KEY_NAME, new ArrayOf(JOIN_GROUP_RESPONSE_MEMBER_V0)));
-
-    /**
-     * The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
-     */
-    private static final Schema JOIN_GROUP_RESPONSE_V3 = JOIN_GROUP_RESPONSE_V2;
-
-    /**
-     * The version number is bumped to indicate that client needs to issue a second join group request under first try
-     * with UNKNOWN_MEMBER_ID.
-     */
-    private static final Schema JOIN_GROUP_RESPONSE_V4 = JOIN_GROUP_RESPONSE_V3;
-
-    public static Schema[] schemaVersions() {
-        return new Schema[] {JOIN_GROUP_RESPONSE_V0, JOIN_GROUP_RESPONSE_V1, JOIN_GROUP_RESPONSE_V2,
-            JOIN_GROUP_RESPONSE_V3, JOIN_GROUP_RESPONSE_V4};
-    }
+    private final JoinGroupResponseData data;
 
     public static final String UNKNOWN_PROTOCOL = "";
     public static final int UNKNOWN_GENERATION_ID = -1;
     public static final String UNKNOWN_MEMBER_ID = "";
 
-    /**
-     * Possible error codes:
-     *
-     * COORDINATOR_LOAD_IN_PROGRESS (14)
-     * GROUP_COORDINATOR_NOT_AVAILABLE (15)
-     * NOT_COORDINATOR (16)
-     * INCONSISTENT_GROUP_PROTOCOL (23)
-     * UNKNOWN_MEMBER_ID (25)
-     * INVALID_SESSION_TIMEOUT (26)
-     * GROUP_AUTHORIZATION_FAILED (30)
-     * MEMBER_ID_REQUIRED (79)
-     */
-
-    private final int throttleTimeMs;
-    private final Errors error;
-    private final int generationId;
-    private final String groupProtocol;
-    private final String memberId;
-    private final String leaderId;
-    private final Map members;
-
-    public JoinGroupResponse(Errors error,
-                             int generationId,
-                             String groupProtocol,
-                             String memberId,
-                             String leaderId,
-                             Map groupMembers) {
-        this(DEFAULT_THROTTLE_TIME, error, generationId, groupProtocol, memberId, leaderId, groupMembers);
-    }
-
-    public JoinGroupResponse(int throttleTimeMs,
-            Errors error,
-            int generationId,
-            String groupProtocol,
-            String memberId,
-            String leaderId,
-            Map groupMembers) {
-        this.throttleTimeMs = throttleTimeMs;
-        this.error = error;
-        this.generationId = generationId;
-        this.groupProtocol = groupProtocol;
-        this.memberId = memberId;
-        this.leaderId = leaderId;
-        this.members = groupMembers;
+    public JoinGroupResponse(JoinGroupResponseData data) {
+        this.data = data;
     }
 
     public JoinGroupResponse(Struct struct) {
-        this.throttleTimeMs = struct.getOrElse(THROTTLE_TIME_MS, DEFAULT_THROTTLE_TIME);
-        members = new HashMap<>();
-
-        for (Object memberDataObj : struct.getArray(MEMBERS_KEY_NAME)) {
-            Struct memberData = (Struct) memberDataObj;
-            String memberId = memberData.get(MEMBER_ID);
-            ByteBuffer memberMetadata = memberData.getBytes(MEMBER_METADATA_KEY_NAME);
-            members.put(memberId, memberMetadata);
-        }
-        error = Errors.forCode(struct.get(ERROR_CODE));
-        generationId = struct.get(GENERATION_ID);
-        groupProtocol = struct.getString(GROUP_PROTOCOL_KEY_NAME);
-        memberId = struct.get(MEMBER_ID);
-        leaderId = struct.getString(LEADER_ID_KEY_NAME);
-    }
-
-    @Override
-    public int throttleTimeMs() {
-        return throttleTimeMs;
-    }
-
-    public Errors error() {
-        return error;
+        short latestVersion = (short) (JoinGroupResponseData.SCHEMAS.length - 1);
+        this.data = new JoinGroupResponseData(struct, latestVersion);
     }
 
-    @Override
-    public Map errorCounts() {
-        return errorCounts(error);
-    }
-
-    public int generationId() {
-        return generationId;
+    public JoinGroupResponse(Struct struct, short version) {
+        this.data = new JoinGroupResponseData(struct, version);
     }
 
-    public String groupProtocol() {
-        return groupProtocol;
+    public JoinGroupResponseData data() {
+        return data;
     }
 
-    public String memberId() {
-        return memberId;
+    public boolean isLeader() {
+        return data.memberId().equals(data.leader());
     }
 
-    public String leaderId() {
-        return leaderId;
+    @Override
+    public int throttleTimeMs() {
+        return data.throttleTimeMs();
     }
 
-    public boolean isLeader() {
-        return memberId.equals(leaderId);
+    public Errors error() {
+        return Errors.forCode(data.errorCode());
     }
 
-    public Map members() {
-        return members;
+    @Override
+    public Map errorCounts() {
+        return Collections.singletonMap(Errors.forCode(data.errorCode()), 1);
     }
 
-    public static JoinGroupResponse parse(ByteBuffer buffer, short version) {
-        return new JoinGroupResponse(ApiKeys.JOIN_GROUP.parseResponse(version, buffer));
+    public static JoinGroupResponse parse(ByteBuffer buffer, short versionId) {
+        return new JoinGroupResponse(ApiKeys.JOIN_GROUP.parseResponse(versionId, buffer), versionId);
     }
 
     @Override
     protected Struct toStruct(short version) {
-        Struct struct = new Struct(ApiKeys.JOIN_GROUP.responseSchema(version));
-        struct.setIfExists(THROTTLE_TIME_MS, throttleTimeMs);
-
-        struct.set(ERROR_CODE, error.code());
-        struct.set(GENERATION_ID, generationId);
-        struct.set(GROUP_PROTOCOL_KEY_NAME, groupProtocol);
-        struct.set(MEMBER_ID, memberId);
-        struct.set(LEADER_ID_KEY_NAME, leaderId);
-
-        List memberArray = new ArrayList<>();
-        for (Map.Entry entries : members.entrySet()) {
-            Struct memberData = struct.instance(MEMBERS_KEY_NAME);
-            memberData.set(MEMBER_ID, entries.getKey());
-            memberData.set(MEMBER_METADATA_KEY_NAME, entries.getValue());
-            memberArray.add(memberData);
-        }
-        struct.set(MEMBERS_KEY_NAME, memberArray.toArray());
-
-        return struct;
+        return data.toStruct(version);
     }
 
     @Override
     public String toString() {
-        return "JoinGroupResponse" +
-            "(throttleTimeMs=" + throttleTimeMs +
-            ", error=" + error +
-            ", generationId=" + generationId +
-            ", groupProtocol=" + groupProtocol +
-            ", memberId=" + memberId +
-            ", leaderId=" + leaderId +
-            ", members=" + ((members == null) ? "null" :
-                Utils.join(members.keySet(), ",")) + ")";
+        return data.toString();
     }
 
     @Override
diff --git a/clients/src/main/resources/common/message/JoinGroupResponse.json b/clients/src/main/resources/common/message/JoinGroupResponse.json
index fb36bb076afe4..a1c6a70810612 100644
--- a/clients/src/main/resources/common/message/JoinGroupResponse.json
+++ b/clients/src/main/resources/common/message/JoinGroupResponse.json
@@ -28,7 +28,7 @@
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
     { "name": "ErrorCode", "type": "int16", "versions": "0+",
       "about": "The error code, or 0 if there was no error." },
-    { "name": "GenerationId", "type": "int32", "versions": "0+",
+    { "name": "GenerationId", "type": "int32", "versions": "0+", "default": "-1",
       "about": "The generation ID of the group." },
     { "name": "ProtocolName", "type": "string", "versions": "0+",
       "about": "The group protocol selected by the coordinator." },
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
index cd0a76f1da277..e17a6efcd9f7a 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
@@ -39,6 +39,8 @@
 import org.apache.kafka.common.errors.InvalidGroupIdException;
 import org.apache.kafka.common.errors.InvalidTopicException;
 import org.apache.kafka.common.errors.WakeupException;
+import org.apache.kafka.common.message.JoinGroupRequestData;
+import org.apache.kafka.common.message.JoinGroupResponseData;
 import org.apache.kafka.common.message.LeaveGroupResponseData;
 import org.apache.kafka.common.internals.ClusterResourceListeners;
 import org.apache.kafka.common.metrics.Metrics;
@@ -89,6 +91,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -1420,8 +1423,20 @@ public boolean matches(AbstractRequest body) {
         final ByteBuffer byteBuffer = ConsumerProtocol.serializeSubscription(new PartitionAssignor.Subscription(singletonList(topic)));
 
         // This member becomes the leader
-        final JoinGroupResponse leaderResponse = new JoinGroupResponse(Errors.NONE, 1, assignor.name(), "memberId", "memberId",
-                Collections.singletonMap("memberId", byteBuffer));
+        String memberId = "memberId";
+        final JoinGroupResponse leaderResponse = new JoinGroupResponse(
+                new JoinGroupResponseData()
+                        .setErrorCode(Errors.NONE.code())
+                        .setGenerationId(1).setProtocolName(assignor.name())
+                        .setLeader(memberId).setMemberId(memberId)
+                        .setMembers(Collections.singletonList(
+                                new JoinGroupResponseData.JoinGroupResponseMember()
+                                        .setMemberId(memberId)
+                                        .setMetadata(byteBuffer.array())
+                                )
+                        )
+        );
+
         client.prepareResponseFrom(leaderResponse, coordinator);
 
         // sync group fails due to disconnect
@@ -1635,7 +1650,12 @@ private Node prepareRebalance(MockClient client, Node node, final Set su
             @Override
             public boolean matches(AbstractRequest body) {
                 JoinGroupRequest joinGroupRequest = (JoinGroupRequest) body;
-                PartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(joinGroupRequest.groupProtocols().get(0).metadata());
+                Iterator protocolIterator =
+                        joinGroupRequest.data().protocols().iterator();
+                assertTrue(protocolIterator.hasNext());
+
+                ByteBuffer protocolMetadata = ByteBuffer.wrap(protocolIterator.next().metadata());
+                PartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(protocolMetadata);
                 return subscribedTopics.equals(new HashSet<>(subscription.topics()));
             }
         }, joinGroupFollowerResponse(assignor, 1, "memberId", "leaderId", Errors.NONE), coordinator);
@@ -1707,8 +1727,15 @@ private OffsetCommitResponse offsetCommitResponse(Map re
     }
 
     private JoinGroupResponse joinGroupFollowerResponse(PartitionAssignor assignor, int generationId, String memberId, String leaderId, Errors error) {
-        return new JoinGroupResponse(error, generationId, assignor.name(), memberId, leaderId,
-                Collections.emptyMap());
+        return new JoinGroupResponse(
+                new JoinGroupResponseData()
+                        .setErrorCode(error.code())
+                        .setGenerationId(generationId)
+                        .setProtocolName(assignor.name())
+                        .setLeader(leaderId)
+                        .setMemberId(memberId)
+                        .setMembers(Collections.emptyList())
+        );
     }
 
     private SyncGroupResponse syncGroupResponse(List partitions, Errors error) {
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
index 9a68db706b4df..dc8ea4642d54a 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
@@ -22,6 +22,8 @@
 import org.apache.kafka.common.errors.AuthenticationException;
 import org.apache.kafka.common.errors.WakeupException;
 import org.apache.kafka.common.internals.ClusterResourceListeners;
+import org.apache.kafka.common.message.JoinGroupRequestData;
+import org.apache.kafka.common.message.JoinGroupResponseData;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.requests.AbstractRequest;
@@ -215,7 +217,7 @@ public boolean matches(AbstractRequest body) {
                     return false;
                 }
                 JoinGroupRequest joinGroupRequest = (JoinGroupRequest) body;
-                if (!joinGroupRequest.memberId().equals(memberId)) {
+                if (!joinGroupRequest.data().memberId().equals(memberId)) {
                     return false;
                 }
                 return true;
@@ -691,8 +693,15 @@ private HeartbeatResponse heartbeatResponse(Errors error) {
     }
 
     private JoinGroupResponse joinGroupFollowerResponse(int generationId, String memberId, String leaderId, Errors error) {
-        return new JoinGroupResponse(error, generationId, "dummy-subprotocol", memberId, leaderId,
-                Collections.emptyMap());
+        return new JoinGroupResponse(
+                new JoinGroupResponseData()
+                        .setErrorCode(error.code())
+                        .setGenerationId(generationId)
+                        .setProtocolName("dummy-subprotocol")
+                        .setMemberId(memberId)
+                        .setLeader(leaderId)
+                        .setMembers(Collections.emptyList())
+        );
     }
 
     private JoinGroupResponse joinGroupResponse(Errors error) {
@@ -725,15 +734,22 @@ protected String protocolType() {
         }
 
         @Override
-        protected List metadata() {
-            return Collections.singletonList(new JoinGroupRequest.ProtocolMetadata("dummy-subprotocol", EMPTY_DATA));
+        protected JoinGroupRequestData.JoinGroupRequestProtocolSet metadata() {
+            return new JoinGroupRequestData.JoinGroupRequestProtocolSet(
+                    Collections.singleton(new JoinGroupRequestData.JoinGroupRequestProtocol()
+                            .setName("dummy-subprotocol")
+                            .setMetadata(EMPTY_DATA.array())).iterator()
+            );
         }
 
         @Override
-        protected Map performAssignment(String leaderId, String protocol, Map allMemberMetadata) {
+        protected Map performAssignment(String leaderId,
+                                                            String protocol,
+                                                            List allMemberMetadata) {
             Map assignment = new HashMap<>();
-            for (Map.Entry metadata : allMemberMetadata.entrySet())
-                assignment.put(metadata.getKey(), EMPTY_DATA);
+            for (JoinGroupResponseData.JoinGroupResponseMember member : allMemberMetadata) {
+                assignment.put(member.memberId(), EMPTY_DATA);
+            }
             return assignment;
         }
 
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index b079963fefcbe..49bbcec206b0a 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -23,9 +23,7 @@
 import org.apache.kafka.clients.consumer.OffsetAndMetadata;
 import org.apache.kafka.clients.consumer.OffsetCommitCallback;
 import org.apache.kafka.clients.consumer.OffsetResetStrategy;
-import org.apache.kafka.clients.consumer.RangeAssignor;
 import org.apache.kafka.clients.consumer.RetriableCommitFailedException;
-import org.apache.kafka.clients.consumer.RoundRobinAssignor;
 import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.Metric;
 import org.apache.kafka.common.MetricName;
@@ -39,6 +37,8 @@
 import org.apache.kafka.common.errors.WakeupException;
 import org.apache.kafka.common.internals.ClusterResourceListeners;
 import org.apache.kafka.common.internals.Topic;
+import org.apache.kafka.common.message.JoinGroupRequestData;
+import org.apache.kafka.common.message.JoinGroupResponseData;
 import org.apache.kafka.common.message.LeaveGroupResponseData;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.protocol.Errors;
@@ -46,7 +46,6 @@
 import org.apache.kafka.common.requests.FindCoordinatorResponse;
 import org.apache.kafka.common.requests.HeartbeatResponse;
 import org.apache.kafka.common.requests.JoinGroupRequest;
-import org.apache.kafka.common.requests.JoinGroupRequest.ProtocolMetadata;
 import org.apache.kafka.common.requests.JoinGroupResponse;
 import org.apache.kafka.common.requests.LeaveGroupRequest;
 import org.apache.kafka.common.requests.LeaveGroupResponse;
@@ -71,6 +70,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -584,9 +584,14 @@ public boolean matches(AbstractRequest body) {
             @Override
             public boolean matches(AbstractRequest body) {
                 JoinGroupRequest join = (JoinGroupRequest) body;
-                ProtocolMetadata protocolMetadata = join.groupProtocols().iterator().next();
-                PartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(protocolMetadata.metadata());
-                protocolMetadata.metadata().rewind();
+                Iterator protocolIterator =
+                        join.data().protocols().iterator();
+                assertTrue(protocolIterator.hasNext());
+                JoinGroupRequestData.JoinGroupRequestProtocol protocolMetadata = protocolIterator.next();
+
+                ByteBuffer metadata = ByteBuffer.wrap(protocolMetadata.metadata());
+                PartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(metadata);
+                metadata.rewind();
                 return subscription.topics().containsAll(updatedSubscriptionSet);
             }
         }, joinGroupLeaderResponse(2, consumerId, updatedSubscriptions, Errors.NONE));
@@ -895,7 +900,7 @@ public void testUnknownMemberIdOnSyncGroup() {
             @Override
             public boolean matches(AbstractRequest body) {
                 JoinGroupRequest joinRequest = (JoinGroupRequest) body;
-                return joinRequest.memberId().equals(JoinGroupRequest.UNKNOWN_MEMBER_ID);
+                return joinRequest.data().memberId().equals(JoinGroupRequest.UNKNOWN_MEMBER_ID);
             }
         }, joinGroupFollowerResponse(2, consumerId, "leader", Errors.NONE));
         client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE));
@@ -947,7 +952,7 @@ public void testIllegalGenerationOnSyncGroup() {
             @Override
             public boolean matches(AbstractRequest body) {
                 JoinGroupRequest joinRequest = (JoinGroupRequest) body;
-                return joinRequest.memberId().equals(JoinGroupRequest.UNKNOWN_MEMBER_ID);
+                return joinRequest.data().memberId().equals(JoinGroupRequest.UNKNOWN_MEMBER_ID);
             }
         }, joinGroupFollowerResponse(2, consumerId, "leader", Errors.NONE));
         client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE));
@@ -1479,7 +1484,8 @@ public void testCommitAfterLeaveGroup() {
         joinAsFollowerAndReceiveAssignment("consumer", coordinator, singletonList(t1p));
 
         // now switch to manual assignment
-        client.prepareResponse(new LeaveGroupResponse(new LeaveGroupResponseData().setErrorCode(Errors.NONE.code())));
+        client.prepareResponse(new LeaveGroupResponse(new LeaveGroupResponseData()
+                .setErrorCode(Errors.NONE.code())));
         subscriptions.unsubscribe();
         coordinator.maybeLeaveGroup();
         subscriptions.assignFromUser(singleton(t1p));
@@ -1850,30 +1856,6 @@ public void testAuthenticationFailureInEnsureActiveGroup() {
         }
     }
 
-    @Test
-    public void testProtocolMetadataOrder() {
-        RoundRobinAssignor roundRobin = new RoundRobinAssignor();
-        RangeAssignor range = new RangeAssignor();
-
-        try (Metrics metrics = new Metrics(time)) {
-            ConsumerCoordinator coordinator = buildCoordinator(metrics, Arrays.asList(roundRobin, range),
-                    false, true);
-            List metadata = coordinator.metadata();
-            assertEquals(2, metadata.size());
-            assertEquals(roundRobin.name(), metadata.get(0).name());
-            assertEquals(range.name(), metadata.get(1).name());
-        }
-
-        try (Metrics metrics = new Metrics(time)) {
-            ConsumerCoordinator coordinator = buildCoordinator(metrics, Arrays.asList(range, roundRobin),
-                    false, true);
-            List metadata = coordinator.metadata();
-            assertEquals(2, metadata.size());
-            assertEquals(range.name(), metadata.get(0).name());
-            assertEquals(roundRobin.name(), metadata.get(1).name());
-        }
-    }
-
     @Test
     public void testThreadSafeAssignedPartitionsMetric() throws Exception {
         // Get the assigned-partitions metric
@@ -2131,7 +2113,8 @@ public boolean matches(AbstractRequest body) {
                 LeaveGroupRequest leaveRequest = (LeaveGroupRequest) body;
                 return leaveRequest.data().groupId().equals(groupId);
             }
-        }, new LeaveGroupResponse(new LeaveGroupResponseData().setErrorCode(Errors.NONE.code())));
+        }, new LeaveGroupResponse(new LeaveGroupResponseData()
+                .setErrorCode(Errors.NONE.code())));
 
         coordinator.close();
         assertTrue("Commit not requested", commitRequested.get());
@@ -2174,18 +2157,36 @@ private JoinGroupResponse joinGroupLeaderResponse(int generationId,
                                                       String memberId,
                                                       Map> subscriptions,
                                                       Errors error) {
-        Map metadata = new HashMap<>();
+        List metadata = new ArrayList<>();
         for (Map.Entry> subscriptionEntry : subscriptions.entrySet()) {
             PartitionAssignor.Subscription subscription = new PartitionAssignor.Subscription(subscriptionEntry.getValue());
             ByteBuffer buf = ConsumerProtocol.serializeSubscription(subscription);
-            metadata.put(subscriptionEntry.getKey(), buf);
+            metadata.add(new JoinGroupResponseData.JoinGroupResponseMember()
+                    .setMemberId(subscriptionEntry.getKey())
+                    .setMetadata(buf.array()));
         }
-        return new JoinGroupResponse(error, generationId, partitionAssignor.name(), memberId, memberId, metadata);
+
+        return new JoinGroupResponse(
+                new JoinGroupResponseData()
+                        .setErrorCode(error.code())
+                        .setGenerationId(generationId)
+                        .setProtocolName(partitionAssignor.name())
+                        .setLeader(memberId)
+                        .setMemberId(memberId)
+                        .setMembers(Collections.emptyList())
+        );
     }
 
     private JoinGroupResponse joinGroupFollowerResponse(int generationId, String memberId, String leaderId, Errors error) {
-        return new JoinGroupResponse(error, generationId, partitionAssignor.name(), memberId, leaderId,
-                Collections.emptyMap());
+        return new JoinGroupResponse(
+                new JoinGroupResponseData()
+                        .setErrorCode(error.code())
+                        .setGenerationId(generationId)
+                        .setProtocolName(partitionAssignor.name())
+                        .setLeader(leaderId)
+                        .setMemberId(memberId)
+                        .setMembers(Collections.emptyList())
+        );
     }
 
     private SyncGroupResponse syncGroupResponse(List partitions, Errors error) {
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index a483500e94202..5690743967715 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -45,6 +45,8 @@
 import org.apache.kafka.common.message.ElectPreferredLeadersResponseData;
 import org.apache.kafka.common.message.ElectPreferredLeadersResponseData.PartitionResult;
 import org.apache.kafka.common.message.ElectPreferredLeadersResponseData.ReplicaElectionResult;
+import org.apache.kafka.common.message.JoinGroupRequestData;
+import org.apache.kafka.common.message.JoinGroupResponseData;
 import org.apache.kafka.common.message.LeaveGroupRequestData;
 import org.apache.kafka.common.message.LeaveGroupResponseData;
 import org.apache.kafka.common.message.SaslAuthenticateRequestData;
@@ -648,7 +650,7 @@ public void testJoinGroupRequestVersion0RebalanceTimeout() {
         final short version = 0;
         JoinGroupRequest jgr = createJoinGroupRequest(version);
         JoinGroupRequest jgr2 = new JoinGroupRequest(jgr.toStruct(), version);
-        assertEquals(jgr2.rebalanceTimeout(), jgr.rebalanceTimeout());
+        assertEquals(jgr2.data().rebalanceTimeoutMs(), jgr.data().rebalanceTimeoutMs());
     }
 
     @Test
@@ -742,23 +744,53 @@ private HeartbeatResponse createHeartBeatResponse() {
     }
 
     private JoinGroupRequest createJoinGroupRequest(int version) {
-        ByteBuffer metadata = ByteBuffer.wrap(new byte[] {});
-        List protocols = new ArrayList<>();
-        protocols.add(new JoinGroupRequest.ProtocolMetadata("consumer-range", metadata));
+        JoinGroupRequestData.JoinGroupRequestProtocolSet protocols = new JoinGroupRequestData.JoinGroupRequestProtocolSet(
+                Collections.singleton(
+                new JoinGroupRequestData.JoinGroupRequestProtocol()
+                        .setName("consumer-range")
+                        .setMetadata(new byte[0])).iterator()
+        );
         if (version == 0) {
-            return new JoinGroupRequest.Builder("group1", 30000, "consumer1", "consumer", protocols).
-                    build((short) version);
+            return new JoinGroupRequest.Builder(
+                    new JoinGroupRequestData()
+                            .setGroupId("group1")
+                            .setSessionTimeoutMs(30000)
+                            .setMemberId("consumer1")
+                            .setProtocolType("consumer")
+                            .setProtocols(protocols))
+                    .build((short) version);
         } else {
-            return new JoinGroupRequest.Builder("group1", 10000, "consumer1", "consumer", protocols).
-                    setRebalanceTimeout(60000).build();
+            return new JoinGroupRequest.Builder(
+                    new JoinGroupRequestData()
+                            .setGroupId("group1")
+                            .setSessionTimeoutMs(30000)
+                            .setMemberId("consumer1")
+                            .setProtocolType("consumer")
+                            .setProtocols(protocols)
+                            .setRebalanceTimeoutMs(60000)) // v1 and above contains rebalance timeout
+                    .build((short) version);
         }
     }
 
     private JoinGroupResponse createJoinGroupResponse() {
-        Map members = new HashMap<>();
-        members.put("consumer1", ByteBuffer.wrap(new byte[]{}));
-        members.put("consumer2", ByteBuffer.wrap(new byte[]{}));
-        return new JoinGroupResponse(Errors.NONE, 1, "range", "consumer1", "leader", members);
+        List members = Arrays.asList(
+                new JoinGroupResponseData.JoinGroupResponseMember()
+                        .setMemberId("consumer1")
+                        .setMetadata(new byte[0]),
+                new JoinGroupResponseData.JoinGroupResponseMember()
+                        .setMemberId("consumer2")
+                        .setMetadata(new byte[0])
+        );
+
+        return new JoinGroupResponse(
+                new JoinGroupResponseData()
+                        .setErrorCode(Errors.NONE.code())
+                        .setGenerationId(1)
+                        .setProtocolName("range")
+                        .setLeader("leader")
+                        .setMemberId("consumer1")
+                        .setMembers(members)
+        );
     }
 
     private ListGroupsRequest createListGroupsRequest() {
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java
index 103a323eba063..fa89a9def403e 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java
@@ -18,11 +18,12 @@
 
 import org.apache.kafka.clients.consumer.internals.AbstractCoordinator;
 import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient;
+import org.apache.kafka.common.message.JoinGroupRequestData;
+import org.apache.kafka.common.message.JoinGroupResponseData;
 import org.apache.kafka.common.metrics.Measurable;
 import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.requests.JoinGroupRequest;
-import org.apache.kafka.common.requests.JoinGroupRequest.ProtocolMetadata;
 import org.apache.kafka.common.utils.CircularIterator;
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.common.utils.Time;
@@ -144,11 +145,16 @@ public void poll(long timeout) {
     }
 
     @Override
-    public List metadata() {
+    public JoinGroupRequestData.JoinGroupRequestProtocolSet metadata() {
         configSnapshot = configStorage.snapshot();
         ConnectProtocol.WorkerState workerState = new ConnectProtocol.WorkerState(restUrl, configSnapshot.offset());
         ByteBuffer metadata = ConnectProtocol.serializeMetadata(workerState);
-        return Collections.singletonList(new ProtocolMetadata(DEFAULT_SUBPROTOCOL, metadata));
+        return new JoinGroupRequestData.JoinGroupRequestProtocolSet(
+                Collections.singleton(new JoinGroupRequestData.JoinGroupRequestProtocol()
+                        .setName(DEFAULT_SUBPROTOCOL)
+                        .setMetadata(metadata.array()))
+                        .iterator()
+        );
     }
 
     @Override
@@ -163,12 +169,12 @@ protected void onJoinComplete(int generation, String memberId, String protocol,
     }
 
     @Override
-    protected Map performAssignment(String leaderId, String protocol, Map allMemberMetadata) {
+    protected Map performAssignment(String leaderId, String protocol, List allMemberMetadata) {
         log.debug("Performing task assignment");
 
         Map memberConfigs = new HashMap<>();
-        for (Map.Entry entry : allMemberMetadata.entrySet())
-            memberConfigs.put(entry.getKey(), ConnectProtocol.deserializeMetadata(entry.getValue()));
+        for (JoinGroupResponseData.JoinGroupResponseMember memberMetadata : allMemberMetadata)
+            memberConfigs.put(memberMetadata.memberId(), ConnectProtocol.deserializeMetadata(ByteBuffer.wrap(memberMetadata.metadata())));
 
         long maxOffset = findMaxMemberConfigOffset(memberConfigs);
         Long leaderOffset = ensureLeaderConfig(maxOffset);
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorTest.java
index 68d236f14aeb7..c78c7a4756cb4 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorTest.java
@@ -21,11 +21,12 @@
 import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient;
 import org.apache.kafka.common.Node;
 import org.apache.kafka.common.internals.ClusterResourceListeners;
+import org.apache.kafka.common.message.JoinGroupRequestData;
+import org.apache.kafka.common.message.JoinGroupResponseData;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.requests.AbstractRequest;
 import org.apache.kafka.common.requests.FindCoordinatorResponse;
-import org.apache.kafka.common.requests.JoinGroupRequest.ProtocolMetadata;
 import org.apache.kafka.common.requests.JoinGroupResponse;
 import org.apache.kafka.common.requests.SyncGroupRequest;
 import org.apache.kafka.common.requests.SyncGroupResponse;
@@ -44,15 +45,18 @@
 import org.powermock.reflect.Whitebox;
 
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 public class WorkerCoordinatorTest {
 
@@ -186,12 +190,15 @@ public void testMetadata() {
 
         PowerMock.replayAll();
 
-        List serialized = coordinator.metadata();
+        JoinGroupRequestData.JoinGroupRequestProtocolSet serialized = coordinator.metadata();
         assertEquals(1, serialized.size());
 
-        ProtocolMetadata defaultMetadata = serialized.get(0);
+        Iterator protocolIterator = serialized.iterator();
+        assertTrue(protocolIterator.hasNext());
+        JoinGroupRequestData.JoinGroupRequestProtocol defaultMetadata = protocolIterator.next();
         assertEquals(WorkerCoordinator.DEFAULT_SUBPROTOCOL, defaultMetadata.name());
-        ConnectProtocol.WorkerState state = ConnectProtocol.deserializeMetadata(defaultMetadata.metadata());
+        ConnectProtocol.WorkerState state = ConnectProtocol.deserializeMetadata(
+                ByteBuffer.wrap(defaultMetadata.metadata()));
         assertEquals(1, state.offset());
 
         PowerMock.verifyAll();
@@ -364,11 +371,17 @@ public void testLeaderPerformAssignment1() throws Exception {
         // Prime the current configuration state
         coordinator.metadata();
 
-        Map configs = new HashMap<>();
         // Mark everyone as in sync with configState1
-        configs.put("leader", ConnectProtocol.serializeMetadata(new ConnectProtocol.WorkerState(LEADER_URL, 1L)));
-        configs.put("member", ConnectProtocol.serializeMetadata(new ConnectProtocol.WorkerState(MEMBER_URL, 1L)));
-        Map result = Whitebox.invokeMethod(coordinator, "performAssignment", "leader", WorkerCoordinator.DEFAULT_SUBPROTOCOL, configs);
+        List responseMembers = new ArrayList<>();
+        responseMembers.add(new JoinGroupResponseData.JoinGroupResponseMember()
+                .setMemberId("leader")
+                .setMetadata(ConnectProtocol.serializeMetadata(new ConnectProtocol.WorkerState(LEADER_URL, 1L)).array())
+        );
+        responseMembers.add(new JoinGroupResponseData.JoinGroupResponseMember()
+                .setMemberId("member")
+                .setMetadata(ConnectProtocol.serializeMetadata(new ConnectProtocol.WorkerState(MEMBER_URL, 1L)).array())
+        );
+        Map result = Whitebox.invokeMethod(coordinator, "performAssignment", "leader", WorkerCoordinator.DEFAULT_SUBPROTOCOL, responseMembers);
 
         // configState1 has 1 connector, 1 task
         ConnectProtocol.Assignment leaderAssignment = ConnectProtocol.deserializeAssignment(result.get("leader"));
@@ -400,11 +413,18 @@ public void testLeaderPerformAssignment2() throws Exception {
         // Prime the current configuration state
         coordinator.metadata();
 
-        Map configs = new HashMap<>();
         // Mark everyone as in sync with configState1
-        configs.put("leader", ConnectProtocol.serializeMetadata(new ConnectProtocol.WorkerState(LEADER_URL, 1L)));
-        configs.put("member", ConnectProtocol.serializeMetadata(new ConnectProtocol.WorkerState(MEMBER_URL, 1L)));
-        Map result = Whitebox.invokeMethod(coordinator, "performAssignment", "leader", WorkerCoordinator.DEFAULT_SUBPROTOCOL, configs);
+        List responseMembers = new ArrayList<>();
+        responseMembers.add(new JoinGroupResponseData.JoinGroupResponseMember()
+                .setMemberId("leader")
+                .setMetadata(ConnectProtocol.serializeMetadata(new ConnectProtocol.WorkerState(LEADER_URL, 1L)).array())
+        );
+        responseMembers.add(new JoinGroupResponseData.JoinGroupResponseMember()
+                .setMemberId("member")
+                .setMetadata(ConnectProtocol.serializeMetadata(new ConnectProtocol.WorkerState(MEMBER_URL, 1L)).array())
+        );
+
+        Map result = Whitebox.invokeMethod(coordinator, "performAssignment", "leader", WorkerCoordinator.DEFAULT_SUBPROTOCOL, responseMembers);
 
         // configState2 has 2 connector, 3 tasks and should trigger round robin assignment
         ConnectProtocol.Assignment leaderAssignment = ConnectProtocol.deserializeAssignment(result.get("leader"));
@@ -436,11 +456,18 @@ public void testLeaderPerformAssignmentSingleTaskConnectors() throws Exception {
         // Prime the current configuration state
         coordinator.metadata();
 
-        Map configs = new HashMap<>();
         // Mark everyone as in sync with configState1
-        configs.put("leader", ConnectProtocol.serializeMetadata(new ConnectProtocol.WorkerState(LEADER_URL, 1L)));
-        configs.put("member", ConnectProtocol.serializeMetadata(new ConnectProtocol.WorkerState(MEMBER_URL, 1L)));
-        Map result = Whitebox.invokeMethod(coordinator, "performAssignment", "leader", WorkerCoordinator.DEFAULT_SUBPROTOCOL, configs);
+        List responseMembers = new ArrayList<>();
+        responseMembers.add(new JoinGroupResponseData.JoinGroupResponseMember()
+                .setMemberId("leader")
+                .setMetadata(ConnectProtocol.serializeMetadata(new ConnectProtocol.WorkerState(LEADER_URL, 1L)).array())
+        );
+        responseMembers.add(new JoinGroupResponseData.JoinGroupResponseMember()
+                .setMemberId("member")
+                .setMetadata(ConnectProtocol.serializeMetadata(new ConnectProtocol.WorkerState(MEMBER_URL, 1L)).array())
+        );
+
+        Map result = Whitebox.invokeMethod(coordinator, "performAssignment", "leader", WorkerCoordinator.DEFAULT_SUBPROTOCOL, responseMembers);
 
         // Round robin assignment when there are the same number of connectors and tasks should result in each being
         // evenly distributed across the workers, i.e. round robin assignment of connectors first, then followed by tasks
@@ -468,20 +495,36 @@ private FindCoordinatorResponse groupCoordinatorResponse(Node node, Errors error
 
     private JoinGroupResponse joinGroupLeaderResponse(int generationId, String memberId,
                                            Map configOffsets, Errors error) {
-        Map metadata = new HashMap<>();
+        List metadata = new ArrayList<>();
         for (Map.Entry configStateEntry : configOffsets.entrySet()) {
             // We need a member URL, but it doesn't matter for the purposes of this test. Just set it to the member ID
             String memberUrl = configStateEntry.getKey();
             long configOffset = configStateEntry.getValue();
             ByteBuffer buf = ConnectProtocol.serializeMetadata(new ConnectProtocol.WorkerState(memberUrl, configOffset));
-            metadata.put(configStateEntry.getKey(), buf);
+            metadata.add(new JoinGroupResponseData.JoinGroupResponseMember()
+                    .setMemberId(configStateEntry.getKey())
+                    .setMetadata(buf.array())
+            );
         }
-        return new JoinGroupResponse(error, generationId, WorkerCoordinator.DEFAULT_SUBPROTOCOL, memberId, memberId, metadata);
+        return new JoinGroupResponse(
+                new JoinGroupResponseData().setErrorCode(error.code())
+                .setGenerationId(generationId)
+                .setProtocolName(WorkerCoordinator.DEFAULT_SUBPROTOCOL)
+                .setLeader(memberId)
+                .setMemberId(memberId)
+                .setMembers(metadata)
+        );
     }
 
     private JoinGroupResponse joinGroupFollowerResponse(int generationId, String memberId, String leaderId, Errors error) {
-        return new JoinGroupResponse(error, generationId, WorkerCoordinator.DEFAULT_SUBPROTOCOL, memberId, leaderId,
-                Collections.emptyMap());
+        return new JoinGroupResponse(
+                new JoinGroupResponseData().setErrorCode(error.code())
+                        .setGenerationId(generationId)
+                        .setProtocolName(WorkerCoordinator.DEFAULT_SUBPROTOCOL)
+                        .setLeader(leaderId)
+                        .setMemberId(memberId)
+                        .setMembers(Collections.emptyList())
+        );
     }
 
     private SyncGroupResponse syncGroupResponse(short assignmentError, String leader, long configOffset, List connectorIds,
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 0b733410aff8a..68d0823eb341b 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -45,9 +45,11 @@ import org.apache.kafka.common.errors._
 import org.apache.kafka.common.internals.FatalExitError
 import org.apache.kafka.common.internals.Topic.{GROUP_METADATA_TOPIC_NAME, TRANSACTION_STATE_TOPIC_NAME, isInternal}
 import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic
-import org.apache.kafka.common.message.{CreateTopicsResponseData, DescribeGroupsResponseData}
+import org.apache.kafka.common.message.CreateTopicsResponseData
 import org.apache.kafka.common.message.CreateTopicsResponseData.{CreatableTopicResult, CreatableTopicResultSet}
+import org.apache.kafka.common.message.DescribeGroupsResponseData
 import org.apache.kafka.common.message.ElectPreferredLeadersResponseData
+import org.apache.kafka.common.message.JoinGroupResponseData
 import org.apache.kafka.common.message.LeaveGroupResponseData
 import org.apache.kafka.common.message.SaslAuthenticateResponseData
 import org.apache.kafka.common.message.SaslHandshakeResponseData
@@ -1282,10 +1284,23 @@ class KafkaApis(val requestChannel: RequestChannel,
 
     // the callback for sending a join-group response
     def sendResponseCallback(joinResult: JoinGroupResult) {
-      val members = joinResult.members map { case (memberId, metadataArray) => (memberId, ByteBuffer.wrap(metadataArray)) }
+      val members = joinResult.members map { case (memberId, metadataArray) =>
+        new JoinGroupResponseData.JoinGroupResponseMember()
+          .setMemberId(memberId)
+          .setMetadata(metadataArray)
+      }
+
       def createResponse(requestThrottleMs: Int): AbstractResponse = {
-        val responseBody = new JoinGroupResponse(requestThrottleMs, joinResult.error, joinResult.generationId,
-          joinResult.subProtocol, joinResult.memberId, joinResult.leaderId, members.asJava)
+        val responseBody = new JoinGroupResponse(
+          new JoinGroupResponseData()
+            .setThrottleTimeMs(requestThrottleMs)
+            .setErrorCode(joinResult.error.code())
+            .setGenerationId(joinResult.generationId)
+            .setProtocolName(joinResult.subProtocol)
+            .setLeader(joinResult.leaderId)
+            .setMemberId(joinResult.memberId)
+            .setMembers(members.toSeq.asJava)
+        )
 
         trace("Sending join group response %s for correlation id %d to client %s."
           .format(responseBody, request.header.correlationId, request.header.clientId))
@@ -1294,33 +1309,35 @@ class KafkaApis(val requestChannel: RequestChannel,
       sendResponseMaybeThrottle(request, createResponse)
     }
 
-    if (!authorize(request.session, Read, Resource(Group, joinGroupRequest.groupId(), LITERAL))) {
+    if (!authorize(request.session, Read, Resource(Group, joinGroupRequest.data().groupId(), LITERAL))) {
       sendResponseMaybeThrottle(request, requestThrottleMs =>
         new JoinGroupResponse(
-          requestThrottleMs,
-          Errors.GROUP_AUTHORIZATION_FAILED,
-          JoinGroupResponse.UNKNOWN_GENERATION_ID,
-          JoinGroupResponse.UNKNOWN_PROTOCOL,
-          JoinGroupResponse.UNKNOWN_MEMBER_ID, // memberId
-          JoinGroupResponse.UNKNOWN_MEMBER_ID, // leaderId
-          Collections.emptyMap())
+          new JoinGroupResponseData()
+            .setThrottleTimeMs(requestThrottleMs)
+            .setErrorCode(Errors.GROUP_AUTHORIZATION_FAILED.code())
+            .setGenerationId(JoinGroupResponse.UNKNOWN_GENERATION_ID)
+            .setProtocolName(JoinGroupResponse.UNKNOWN_PROTOCOL)
+            .setLeader(JoinGroupResponse.UNKNOWN_MEMBER_ID)
+            .setMemberId(JoinGroupResponse.UNKNOWN_MEMBER_ID)
+            .setMembers(Collections.emptyList())
+        )
       )
     } else {
       // Only return MEMBER_ID_REQUIRED error if joinGroupRequest version is >= 4
       val requireKnownMemberId = joinGroupRequest.version >= 4
 
       // let the coordinator handle join-group
-      val protocols = joinGroupRequest.groupProtocols().asScala.map(protocol =>
-        (protocol.name, Utils.toArray(protocol.metadata))).toList
+      val protocols = joinGroupRequest.data().protocols().asScala.map(protocol =>
+        (protocol.name, protocol.metadata)).toList
       groupCoordinator.handleJoinGroup(
-        joinGroupRequest.groupId,
-        joinGroupRequest.memberId,
+        joinGroupRequest.data().groupId,
+        joinGroupRequest.data().memberId,
         requireKnownMemberId,
         request.header.clientId,
         request.session.clientAddress.toString,
-        joinGroupRequest.rebalanceTimeout,
-        joinGroupRequest.sessionTimeout,
-        joinGroupRequest.protocolType,
+        joinGroupRequest.data().rebalanceTimeoutMs,
+        joinGroupRequest.data().sessionTimeoutMs,
+        joinGroupRequest.data().protocolType,
         protocols,
         sendResponseCallback)
     }
@@ -1396,9 +1413,10 @@ class KafkaApis(val requestChannel: RequestChannel,
     def sendResponseCallback(error: Errors) {
       def createResponse(requestThrottleMs: Int): AbstractResponse = {
         val response = new LeaveGroupResponse(new LeaveGroupResponseData()
-          .setThrottleTimeMs(requestThrottleMs).setErrorCode(error.code()))
+          .setThrottleTimeMs(requestThrottleMs)
+          .setErrorCode(error.code()))
         trace("Sending leave group response %s for correlation id %d to client %s."
-                    .format(response, request.header.correlationId, request.header.clientId))
+          .format(response, request.header.correlationId, request.header.clientId))
         response
       }
       sendResponseMaybeThrottle(request, createResponse)
@@ -1407,7 +1425,8 @@ class KafkaApis(val requestChannel: RequestChannel,
     if (!authorize(request.session, Read, Resource(Group, leaveGroupRequest.data().groupId(), LITERAL))) {
       sendResponseMaybeThrottle(request, requestThrottleMs =>
         new LeaveGroupResponse(new LeaveGroupResponseData()
-          .setThrottleTimeMs(requestThrottleMs).setErrorCode(Errors.GROUP_AUTHORIZATION_FAILED.code())))
+          .setThrottleTimeMs(requestThrottleMs)
+          .setErrorCode(Errors.GROUP_AUTHORIZATION_FAILED.code())))
     } else {
       // let the coordinator to handle leave-group
       groupCoordinator.handleLeaveGroup(
diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
index 044f5952670f5..30cc16181fbaf 100644
--- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
@@ -33,7 +33,7 @@ import org.apache.kafka.common.acl.{AccessControlEntry, AccessControlEntryFilter
 import org.apache.kafka.common.config.ConfigResource
 import org.apache.kafka.common.errors._
 import org.apache.kafka.common.internals.Topic.GROUP_METADATA_TOPIC_NAME
-import org.apache.kafka.common.message.{CreateTopicsRequestData, DescribeGroupsRequestData, LeaveGroupRequestData}
+import org.apache.kafka.common.message.{CreateTopicsRequestData, DescribeGroupsRequestData, JoinGroupRequestData, LeaveGroupRequestData}
 import org.apache.kafka.common.message.CreateTopicsRequestData.{CreatableTopic, CreatableTopicSet}
 import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
@@ -310,9 +310,21 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   private def createJoinGroupRequest = {
-    new JoinGroupRequest.Builder(group, 10000, "", "consumer",
-      List( new JoinGroupRequest.ProtocolMetadata("consumer-range",ByteBuffer.wrap("test".getBytes()))).asJava)
-      .setRebalanceTimeout(60000).build()
+    val protocolSet = new JoinGroupRequestData.JoinGroupRequestProtocolSet(
+      Collections.singletonList(new JoinGroupRequestData.JoinGroupRequestProtocol()
+        .setName("consumer-range")
+        .setMetadata("test".getBytes())
+    ).iterator())
+
+    new JoinGroupRequest.Builder(
+      new JoinGroupRequestData()
+        .setGroupId(group)
+        .setSessionTimeoutMs(10000)
+        .setMemberId(JoinGroupRequest.UNKNOWN_MEMBER_ID)
+        .setProtocolType("consumer")
+        .setProtocols(protocolSet)
+        .setRebalanceTimeoutMs(60000)
+    ).build()
   }
 
   private def createSyncGroupRequest = {
diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
index d070e468a6970..32541d3fcc288 100644
--- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
+++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
@@ -24,7 +24,7 @@ import kafka.security.auth._
 import kafka.utils.TestUtils
 import org.apache.kafka.common.acl.{AccessControlEntry, AccessControlEntryFilter, AclBinding, AclBindingFilter, AclOperation, AclPermissionType}
 import org.apache.kafka.common.config.ConfigResource
-import org.apache.kafka.common.message.{CreateTopicsRequestData, DescribeGroupsRequestData, ElectPreferredLeadersRequestData, LeaveGroupRequestData}
+import org.apache.kafka.common.message.{CreateTopicsRequestData, DescribeGroupsRequestData, ElectPreferredLeadersRequestData, LeaveGroupRequestData, JoinGroupRequestData}
 import org.apache.kafka.common.resource.{PatternType, ResourcePattern, ResourcePatternFilter, ResourceType => AdminResourceType}
 import org.apache.kafka.common.{Node, TopicPartition}
 import org.apache.kafka.common.message.CreateTopicsRequestData.{CreatableTopic, CreatableTopicSet}
@@ -254,9 +254,21 @@ class RequestQuotaTest extends BaseRequestTest {
           new FindCoordinatorRequest.Builder(FindCoordinatorRequest.CoordinatorType.GROUP, "test-group")
 
         case ApiKeys.JOIN_GROUP =>
-          new JoinGroupRequest.Builder("test-join-group", 200, "", "consumer",
-            List(new JoinGroupRequest.ProtocolMetadata("consumer-range", ByteBuffer.wrap("test".getBytes()))).asJava)
-           .setRebalanceTimeout(100)
+          new JoinGroupRequest.Builder(
+            new JoinGroupRequestData()
+              .setGroupId("test-join-group")
+              .setSessionTimeoutMs(200)
+              .setMemberId(JoinGroupRequest.UNKNOWN_MEMBER_ID)
+              .setProtocolType("consumer")
+              .setProtocols(
+                new JoinGroupRequestData.JoinGroupRequestProtocolSet(
+                  Collections.singletonList(new JoinGroupRequestData.JoinGroupRequestProtocol()
+                    .setName("consumer-range")
+                    .setMetadata("test".getBytes())).iterator()
+                )
+              )
+              .setRebalanceTimeoutMs(100)
+          )
 
         case ApiKeys.HEARTBEAT =>
           new HeartbeatRequest.Builder("test-group", 1, "")

From 70ddd8af71938b4f5f6d1bb3df6243ef13359bcf Mon Sep 17 00:00:00 2001
From: Bob Barrett 
Date: Tue, 19 Mar 2019 02:00:01 -0400
Subject: [PATCH 0036/1071] MINOR: Improve logging around index files (#6385)

This patch adds additional DEBUG statements in AbstractIndex.scala, OffsetIndex.scala, and TimeIndex.scala. It also changes the logging on append from DEBUG to TRACE to make DEBUG logging less disruptive, and it ensures that exceptions raised from index classes include file/offset information.

Reviewers: Jason Gustafson 
---
 .../main/scala/kafka/log/AbstractIndex.scala    |  3 +++
 core/src/main/scala/kafka/log/OffsetIndex.scala | 15 +++++++++------
 core/src/main/scala/kafka/log/TimeIndex.scala   | 17 +++++++++++------
 .../main/scala/kafka/log/TransactionIndex.scala | 13 +++++++------
 4 files changed, 30 insertions(+), 18 deletions(-)

diff --git a/core/src/main/scala/kafka/log/AbstractIndex.scala b/core/src/main/scala/kafka/log/AbstractIndex.scala
index c69e78344dc67..05237c44193fc 100644
--- a/core/src/main/scala/kafka/log/AbstractIndex.scala
+++ b/core/src/main/scala/kafka/log/AbstractIndex.scala
@@ -175,6 +175,7 @@ abstract class AbstractIndex[K, V](@volatile var file: File, val baseOffset: Lon
       val roundedNewSize = roundDownToExactMultiple(newSize, entrySize)
 
       if (_length == roundedNewSize) {
+        debug(s"Index ${file.getAbsolutePath} was not resized because it already has size $roundedNewSize")
         false
       } else {
         val raf = new RandomAccessFile(file, "rw")
@@ -189,6 +190,8 @@ abstract class AbstractIndex[K, V](@volatile var file: File, val baseOffset: Lon
           mmap = raf.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, roundedNewSize)
           _maxEntries = mmap.limit() / entrySize
           mmap.position(position)
+          debug(s"Resized ${file.getAbsolutePath} to $roundedNewSize, position is ${mmap.position()} " +
+            s"and limit is ${mmap.limit()}")
           true
         } finally {
           CoreUtils.swallow(raf.close(), AbstractIndex)
diff --git a/core/src/main/scala/kafka/log/OffsetIndex.scala b/core/src/main/scala/kafka/log/OffsetIndex.scala
index 876895cd5cce4..d043d0a50271a 100755
--- a/core/src/main/scala/kafka/log/OffsetIndex.scala
+++ b/core/src/main/scala/kafka/log/OffsetIndex.scala
@@ -59,8 +59,8 @@ class OffsetIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writabl
   /* the last offset in the index */
   private[this] var _lastOffset = lastEntry.offset
 
-  debug("Loaded index file %s with maxEntries = %d, maxIndexSize = %d, entries = %d, lastOffset = %d, file position = %d"
-    .format(file.getAbsolutePath, maxEntries, maxIndexSize, _entries, _lastOffset, mmap.position()))
+  debug(s"Loaded index file ${file.getAbsolutePath} with maxEntries = $maxEntries, " +
+    s"maxIndexSize = $maxIndexSize, entries = ${_entries}, lastOffset = ${_lastOffset}, file position = ${mmap.position()}")
 
   /**
    * The last entry in the index
@@ -128,7 +128,8 @@ class OffsetIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writabl
   def entry(n: Int): OffsetPosition = {
     maybeLock(lock) {
       if (n >= _entries)
-        throw new IllegalArgumentException(s"Attempt to fetch the ${n}th entry from an index of size ${_entries}.")
+        throw new IllegalArgumentException(s"Attempt to fetch the ${n}th entry from index ${file.getAbsolutePath}, " +
+          s"which has size ${_entries}.")
       parseEntry(mmap, n)
     }
   }
@@ -141,15 +142,15 @@ class OffsetIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writabl
     inLock(lock) {
       require(!isFull, "Attempt to append to a full index (size = " + _entries + ").")
       if (_entries == 0 || offset > _lastOffset) {
-        debug("Adding index entry %d => %d to %s.".format(offset, position, file.getName))
+        trace(s"Adding index entry $offset => $position to ${file.getAbsolutePath}")
         mmap.putInt(relativeOffset(offset))
         mmap.putInt(position)
         _entries += 1
         _lastOffset = offset
         require(_entries * entrySize == mmap.position(), entries + " entries but file position in index is " + mmap.position() + ".")
       } else {
-        throw new InvalidOffsetException("Attempt to append an offset (%d) to position %d no larger than the last offset appended (%d) to %s."
-          .format(offset, entries, _lastOffset, file.getAbsolutePath))
+        throw new InvalidOffsetException(s"Attempt to append an offset ($offset) to position $entries no larger than" +
+          s" the last offset appended (${_lastOffset}) to ${file.getAbsolutePath}.")
       }
     }
   }
@@ -185,6 +186,8 @@ class OffsetIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writabl
       _entries = entries
       mmap.position(_entries * entrySize)
       _lastOffset = lastEntry.offset
+      debug(s"Truncated index ${file.getAbsolutePath} to $entries entries;" +
+        s" position is now ${mmap.position()} and last offset is now ${_lastOffset}")
     }
   }
 
diff --git a/core/src/main/scala/kafka/log/TimeIndex.scala b/core/src/main/scala/kafka/log/TimeIndex.scala
index f5c4d90aac5ad..f9bc929178ad4 100644
--- a/core/src/main/scala/kafka/log/TimeIndex.scala
+++ b/core/src/main/scala/kafka/log/TimeIndex.scala
@@ -58,6 +58,9 @@ class TimeIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writable:
 
   override def entrySize = 12
 
+  debug(s"Loaded index file ${file.getAbsolutePath} with maxEntries = $maxEntries, maxIndexSize = $maxIndexSize," +
+    s" entries = ${_entries}, lastOffset = ${_lastEntry}, file position = ${mmap.position()}")
+
   // We override the full check to reserve the last time index entry slot for the on roll call.
   override def isFull: Boolean = entries >= maxEntries - 1
 
@@ -87,7 +90,8 @@ class TimeIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writable:
   def entry(n: Int): TimestampOffset = {
     maybeLock(lock) {
       if(n >= _entries)
-        throw new IllegalArgumentException("Attempt to fetch the %dth entry from a time index of size %d.".format(n, _entries))
+        throw new IllegalArgumentException(s"Attempt to fetch the ${n}th entry from  time index ${file.getAbsolutePath} " +
+          s"which has size ${_entries}.")
       parseEntry(mmap, n)
     }
   }
@@ -117,16 +121,16 @@ class TimeIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writable:
       // 1. A log segment is closed.
       // 2. LogSegment.onBecomeInactiveSegment() is called when an active log segment is rolled.
       if (_entries != 0 && offset < lastEntry.offset)
-        throw new InvalidOffsetException("Attempt to append an offset (%d) to slot %d no larger than the last offset appended (%d) to %s."
-          .format(offset, _entries, lastEntry.offset, file.getAbsolutePath))
+        throw new InvalidOffsetException(s"Attempt to append an offset ($offset) to slot ${_entries} no larger than" +
+          s" the last offset appended (${lastEntry.offset}) to ${file.getAbsolutePath}.")
       if (_entries != 0 && timestamp < lastEntry.timestamp)
-        throw new IllegalStateException("Attempt to append a timestamp (%d) to slot %d no larger than the last timestamp appended (%d) to %s."
-            .format(timestamp, _entries, lastEntry.timestamp, file.getAbsolutePath))
+        throw new IllegalStateException(s"Attempt to append a timestamp ($timestamp) to slot ${_entries} no larger" +
+          s" than the last timestamp appended (${lastEntry.timestamp}) to ${file.getAbsolutePath}.")
       // We only append to the time index when the timestamp is greater than the last inserted timestamp.
       // If all the messages are in message format v0, the timestamp will always be NoTimestamp. In that case, the time
       // index will be empty.
       if (timestamp > lastEntry.timestamp) {
-        debug("Adding index entry %d => %d to %s.".format(timestamp, offset, file.getName))
+        trace(s"Adding index entry $timestamp => $offset to ${file.getAbsolutePath}.")
         mmap.putLong(timestamp)
         mmap.putInt(relativeOffset(offset))
         _entries += 1
@@ -200,6 +204,7 @@ class TimeIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writable:
       _entries = entries
       mmap.position(_entries * entrySize)
       _lastEntry = lastEntryFromIndexFile
+      debug(s"Truncated index ${file.getAbsolutePath} to $entries entries; position is now ${mmap.position()} and last entry is now ${_lastEntry}")
     }
   }
 
diff --git a/core/src/main/scala/kafka/log/TransactionIndex.scala b/core/src/main/scala/kafka/log/TransactionIndex.scala
index da7fce815dcf9..e730fdba985d1 100644
--- a/core/src/main/scala/kafka/log/TransactionIndex.scala
+++ b/core/src/main/scala/kafka/log/TransactionIndex.scala
@@ -53,7 +53,8 @@ class TransactionIndex(val startOffset: Long, @volatile var file: File) extends
   def append(abortedTxn: AbortedTxn): Unit = {
     lastOffset.foreach { offset =>
       if (offset >= abortedTxn.lastOffset)
-        throw new IllegalArgumentException("The last offset of appended transactions must increase sequentially")
+        throw new IllegalArgumentException(s"The last offset of appended transactions must increase sequentially, but " +
+          s"${abortedTxn.lastOffset} is not greater than current last offset $offset of index ${file.getAbsolutePath}")
     }
     lastOffset = Some(abortedTxn.lastOffset)
     Utils.writeFully(channel, abortedTxn.buffer.duplicate())
@@ -138,8 +139,8 @@ class TransactionIndex(val startOffset: Long, @volatile var file: File) extends
 
               val abortedTxn = new AbortedTxn(buffer)
               if (abortedTxn.version > AbortedTxn.CurrentVersion)
-                throw new KafkaException(s"Unexpected aborted transaction version ${abortedTxn.version}, " +
-                  s"current version is ${AbortedTxn.CurrentVersion}")
+                throw new KafkaException(s"Unexpected aborted transaction version ${abortedTxn.version} " +
+                  s"in transaction index ${file.getAbsolutePath}, current version is ${AbortedTxn.CurrentVersion}")
               val nextEntry = (abortedTxn, position)
               position += AbortedTxn.TotalSize
               nextEntry
@@ -147,7 +148,7 @@ class TransactionIndex(val startOffset: Long, @volatile var file: File) extends
               case e: IOException =>
                 // We received an unexpected error reading from the index file. We propagate this as an
                 // UNKNOWN error to the consumer, which will cause it to retry the fetch.
-                throw new KafkaException(s"Failed to read from the transaction index $file", e)
+                throw new KafkaException(s"Failed to read from the transaction index ${file.getAbsolutePath}", e)
             }
           }
         }
@@ -187,8 +188,8 @@ class TransactionIndex(val startOffset: Long, @volatile var file: File) extends
     val buffer = ByteBuffer.allocate(AbortedTxn.TotalSize)
     for ((abortedTxn, _) <- iterator(() => buffer)) {
       if (abortedTxn.lastOffset < startOffset)
-        throw new CorruptIndexException(s"Last offset of aborted transaction $abortedTxn is less than start offset " +
-          s"$startOffset")
+        throw new CorruptIndexException(s"Last offset of aborted transaction $abortedTxn in index " +
+          s"${file.getAbsolutePath} is less than start offset $startOffset")
     }
   }
 

From 6d649f503a964ed9612e79ef2d9e55e26240fbc3 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Tue, 19 Mar 2019 07:12:49 -0700
Subject: [PATCH 0037/1071] KAFKA-8062: Do not remore StateListener when
 shutting down stream thread (#6468)

In a previous commit #6091, we've fixed a couple of edge cases and hence do not need to remove state listener anymore (before that we removed the state listener intentionally to avoid some race conditions, which has been gone for now).

Reviewers: Matthias J. Sax ,   Bill Bejeck 
---
 .../apache/kafka/streams/KafkaStreams.java    |  4 +-
 .../processor/internals/StreamThread.java     |  1 -
 .../kafka/streams/KafkaStreamsTest.java       | 72 +++++++++++++++++--
 3 files changed, 69 insertions(+), 8 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
index 4f15deadbc784..315a6bbaa9209 100644
--- a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
+++ b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
@@ -191,7 +191,7 @@ public class KafkaStreams implements AutoCloseable {
      *   the instance will be in the ERROR state. The user will need to close it.
      */
     public enum State {
-        CREATED(1, 3), REBALANCING(2, 3, 5), RUNNING(1, 3, 5), PENDING_SHUTDOWN(4), NOT_RUNNING, ERROR(3, 5);
+        CREATED(1, 3), REBALANCING(2, 3, 5), RUNNING(1, 3, 5), PENDING_SHUTDOWN(4), NOT_RUNNING, ERROR(3);
 
         private final Set validTransitions = new HashSet<>();
 
@@ -857,7 +857,6 @@ private boolean close(final long timeoutMs) {
                 // notify all the threads to stop; avoid deadlocks by stopping any
                 // further state reports from the thread since we're shutting down
                 for (final StreamThread thread : threads) {
-                    thread.setStateListener(null);
                     thread.shutdown();
                 }
 
@@ -872,7 +871,6 @@ private boolean close(final long timeoutMs) {
                 }
 
                 if (globalStreamThread != null) {
-                    globalStreamThread.setStateListener(null);
                     globalStreamThread.shutdown();
                 }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index 71df0f963daf1..1bd09e468f223 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -269,7 +269,6 @@ public void onPartitionsAssigned(final Collection assignment) {
             if (streamThread.assignmentErrorCode.get() == StreamsPartitionAssignor.Error.INCOMPLETE_SOURCE_TOPIC_METADATA.code()) {
                 log.error("Received error code {} - shutdown", streamThread.assignmentErrorCode.get());
                 streamThread.shutdown();
-                streamThread.setStateListener(null);
                 return;
             }
             final long start = time.milliseconds();
diff --git a/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java b/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java
index 6b8b5b529d134..3e55f2922cd6f 100644
--- a/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java
@@ -140,14 +140,14 @@ public void testInvalidSocketReceiveBufferSize() {
     }
 
     @Test
-    public void testStateCloseAfterCreate() {
+    public void stateShouldTransitToNotRunningIfCloseRightAfterCreated() {
         globalStreams.close();
 
         Assert.assertEquals(KafkaStreams.State.NOT_RUNNING, globalStreams.state());
     }
 
     @Test
-    public void testStateOneThreadDeadButRebalanceFinish() throws InterruptedException {
+    public void stateShouldTransitToRunningIfNonDeadThreadsBackToRunning() throws InterruptedException {
         final StateListenerStub stateListener = new StateListenerStub();
         globalStreams.setStateListener(stateListener);
 
@@ -171,7 +171,7 @@ public void testStateOneThreadDeadButRebalanceFinish() throws InterruptedExcepti
         Assert.assertEquals(3, stateListener.numChanges);
         Assert.assertEquals(KafkaStreams.State.REBALANCING, globalStreams.state());
 
-        for (final StreamThread thread: globalStreams.threads) {
+        for (final StreamThread thread : globalStreams.threads) {
             thread.stateListener().onChange(
                 thread,
                 StreamThread.State.PARTITIONS_ASSIGNED,
@@ -194,7 +194,7 @@ public void testStateOneThreadDeadButRebalanceFinish() throws InterruptedExcepti
         Assert.assertEquals(3, stateListener.numChanges);
         Assert.assertEquals(KafkaStreams.State.REBALANCING, globalStreams.state());
 
-        for (final StreamThread thread: globalStreams.threads) {
+        for (final StreamThread thread : globalStreams.threads) {
             if (thread != globalStreams.threads[NUM_THREADS - 1]) {
                 thread.stateListener().onChange(
                     thread,
@@ -214,6 +214,70 @@ public void testStateOneThreadDeadButRebalanceFinish() throws InterruptedExcepti
         Assert.assertEquals(KafkaStreams.State.NOT_RUNNING, globalStreams.state());
     }
 
+    @Test
+    public void stateShouldTransitToErrorIfAllThreadsDead() throws InterruptedException {
+        final StateListenerStub stateListener = new StateListenerStub();
+        globalStreams.setStateListener(stateListener);
+
+        Assert.assertEquals(0, stateListener.numChanges);
+        Assert.assertEquals(KafkaStreams.State.CREATED, globalStreams.state());
+
+        globalStreams.start();
+
+        TestUtils.waitForCondition(
+            () -> stateListener.numChanges == 2,
+            "Streams never started.");
+        Assert.assertEquals(KafkaStreams.State.RUNNING, globalStreams.state());
+
+        for (final StreamThread thread : globalStreams.threads) {
+            thread.stateListener().onChange(
+                thread,
+                StreamThread.State.PARTITIONS_REVOKED,
+                StreamThread.State.RUNNING);
+        }
+
+        Assert.assertEquals(3, stateListener.numChanges);
+        Assert.assertEquals(KafkaStreams.State.REBALANCING, globalStreams.state());
+
+        globalStreams.threads[NUM_THREADS - 1].stateListener().onChange(
+            globalStreams.threads[NUM_THREADS - 1],
+            StreamThread.State.PENDING_SHUTDOWN,
+            StreamThread.State.PARTITIONS_REVOKED);
+
+        globalStreams.threads[NUM_THREADS - 1].stateListener().onChange(
+            globalStreams.threads[NUM_THREADS - 1],
+            StreamThread.State.DEAD,
+            StreamThread.State.PENDING_SHUTDOWN);
+
+        Assert.assertEquals(3, stateListener.numChanges);
+        Assert.assertEquals(KafkaStreams.State.REBALANCING, globalStreams.state());
+
+        for (final StreamThread thread : globalStreams.threads) {
+            if (thread != globalStreams.threads[NUM_THREADS - 1]) {
+                thread.stateListener().onChange(
+                    thread,
+                    StreamThread.State.PENDING_SHUTDOWN,
+                    StreamThread.State.PARTITIONS_REVOKED);
+
+                thread.stateListener().onChange(
+                    thread,
+                    StreamThread.State.DEAD,
+                    StreamThread.State.PENDING_SHUTDOWN);
+            }
+        }
+
+        Assert.assertEquals(4, stateListener.numChanges);
+        Assert.assertEquals(KafkaStreams.State.ERROR, globalStreams.state());
+
+        globalStreams.close();
+
+        // the state should not stuck with ERROR, but transit to NOT_RUNNING in the end
+        TestUtils.waitForCondition(
+            () -> stateListener.numChanges == 6,
+            "Streams never closed.");
+        Assert.assertEquals(KafkaStreams.State.NOT_RUNNING, globalStreams.state());
+    }
+
     @Test
     public void shouldCleanupResourcesOnCloseWithoutPreviousStart() throws Exception {
         builder.globalTable("anyTopic");

From 538bd7eddf13897245524f015e3207affb03fcdc Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Tue, 19 Mar 2019 08:51:10 -0700
Subject: [PATCH 0038/1071] KAFKA-8094: Iterating over cache with get(key) is
 inefficient (#6433)

Use concurrent data structure for the underlying cache in NamedCache, and iterate over it with subMap instead of many calls to get()

Reviewers: Guozhang Wang , Bill Bejeck 
---
 .../streams/state/internals/NamedCache.java   | 18 +++++--------
 .../streams/state/internals/ThreadCache.java  | 24 ++++++++---------
 .../state/internals/NamedCacheTest.java       | 27 -------------------
 .../state/internals/ThreadCacheTest.java      |  3 ++-
 4 files changed, 21 insertions(+), 51 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
index 3ce7cbe427e78..0201f20f9fbf4 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
@@ -16,6 +16,8 @@
  */
 package org.apache.kafka.streams.state.internals;
 
+import java.util.NavigableMap;
+import java.util.concurrent.ConcurrentSkipListMap;
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
@@ -33,13 +35,11 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
 
 class NamedCache {
     private static final Logger log = LoggerFactory.getLogger(NamedCache.class);
     private final String name;
-    private final TreeMap cache = new TreeMap<>();
+    private final NavigableMap cache = new ConcurrentSkipListMap<>();
     private final Set dirtyKeys = new LinkedHashSet<>();
     private ThreadCache.DirtyEntryFlushListener listener;
     private LRUNode tail;
@@ -266,16 +266,12 @@ public long size() {
         return cache.size();
     }
 
-    synchronized Iterator keyRange(final Bytes from, final Bytes to) {
-        return keySetIterator(cache.navigableKeySet().subSet(from, true, to, true));
+    synchronized Iterator> subMapIterator(final Bytes from, final Bytes to) {
+        return cache.subMap(from, true, to, true).entrySet().iterator();
     }
 
-    private Iterator keySetIterator(final Set keySet) {
-        return new TreeSet<>(keySet).iterator();
-    }
-
-    synchronized Iterator allKeys() {
-        return keySetIterator(cache.navigableKeySet());
+    synchronized Iterator> allIterator() {
+        return cache.entrySet().iterator();
     }
 
     synchronized LRUCacheEntry first() {
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java
index 941b52215242e..0db6c78a5772e 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java
@@ -20,6 +20,7 @@
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.state.internals.NamedCache.LRUNode;
 import org.slf4j.Logger;
 
 import java.util.Collections;
@@ -180,17 +181,17 @@ public LRUCacheEntry delete(final String namespace, final Bytes key) {
     public MemoryLRUCacheBytesIterator range(final String namespace, final Bytes from, final Bytes to) {
         final NamedCache cache = getCache(namespace);
         if (cache == null) {
-            return new MemoryLRUCacheBytesIterator(Collections.emptyIterator(), new NamedCache(namespace, this.metrics));
+            return new MemoryLRUCacheBytesIterator(Collections.emptyIterator());
         }
-        return new MemoryLRUCacheBytesIterator(cache.keyRange(from, to), cache);
+        return new MemoryLRUCacheBytesIterator(cache.subMapIterator(from, to));
     }
 
     public MemoryLRUCacheBytesIterator all(final String namespace) {
         final NamedCache cache = getCache(namespace);
         if (cache == null) {
-            return new MemoryLRUCacheBytesIterator(Collections.emptyIterator(), new NamedCache(namespace, this.metrics));
+            return new MemoryLRUCacheBytesIterator(Collections.emptyIterator());
         }
-        return new MemoryLRUCacheBytesIterator(cache.allKeys(), cache);
+        return new MemoryLRUCacheBytesIterator(cache.allIterator());
     }
     
     public long size() {
@@ -260,13 +261,11 @@ private synchronized NamedCache getOrCreateCache(final String name) {
     }
 
     static class MemoryLRUCacheBytesIterator implements PeekingKeyValueIterator {
-        private final Iterator keys;
-        private final NamedCache cache;
+        private final Iterator> underlying;
         private KeyValue nextEntry;
 
-        MemoryLRUCacheBytesIterator(final Iterator keys, final NamedCache cache) {
-            this.keys = keys;
-            this.cache = cache;
+        MemoryLRUCacheBytesIterator(final Iterator> underlying) {
+            this.underlying = underlying;
         }
 
         public Bytes peekNextKey() {
@@ -290,7 +289,7 @@ public boolean hasNext() {
                 return true;
             }
 
-            while (keys.hasNext() && nextEntry == null) {
+            while (underlying.hasNext() && nextEntry == null) {
                 internalNext();
             }
 
@@ -308,8 +307,9 @@ public KeyValue next() {
         }
 
         private void internalNext() {
-            final Bytes cacheKey = keys.next();
-            final LRUCacheEntry entry = cache.get(cacheKey);
+            final Map.Entry mapEntry = underlying.next();
+            final Bytes cacheKey = mapEntry.getKey();
+            final LRUCacheEntry entry = mapEntry.getValue().entry();
             if (entry == null) {
                 return;
             }
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/NamedCacheTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/NamedCacheTest.java
index 394feed256af9..6c82209c33aca 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/NamedCacheTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/NamedCacheTest.java
@@ -32,7 +32,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -40,7 +39,6 @@
 import static org.apache.kafka.test.StreamsTestUtils.getMetricByNameFilterByTags;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertSame;
@@ -208,31 +206,6 @@ public void apply(final List dirty) {
         assertEquals(cache.flushes(), 1);
     }
 
-    @Test
-    public void shouldGetRangeIteratorOverKeys() {
-        cache.put(Bytes.wrap(new byte[]{0}), new LRUCacheEntry(new byte[]{10}, headers, true, 0, 0, 0, ""));
-        cache.put(Bytes.wrap(new byte[]{1}), new LRUCacheEntry(new byte[]{20}));
-        cache.put(Bytes.wrap(new byte[]{2}), new LRUCacheEntry(new byte[]{30}, null, true, 0, 0, 0, ""));
-
-        final Iterator iterator = cache.keyRange(Bytes.wrap(new byte[]{1}), Bytes.wrap(new byte[]{2}));
-        assertEquals(Bytes.wrap(new byte[]{1}), iterator.next());
-        assertEquals(Bytes.wrap(new byte[]{2}), iterator.next());
-        assertFalse(iterator.hasNext());
-    }
-
-    @Test
-    public void shouldGetIteratorOverAllKeys() {
-        cache.put(Bytes.wrap(new byte[]{0}), new LRUCacheEntry(new byte[]{10}, headers, true, 0, 0, 0, ""));
-        cache.put(Bytes.wrap(new byte[]{1}), new LRUCacheEntry(new byte[]{20}));
-        cache.put(Bytes.wrap(new byte[]{2}), new LRUCacheEntry(new byte[]{30}, null, true, 0, 0, 0, ""));
-
-        final Iterator iterator = cache.allKeys();
-        assertEquals(Bytes.wrap(new byte[]{0}), iterator.next());
-        assertEquals(Bytes.wrap(new byte[]{1}), iterator.next());
-        assertEquals(Bytes.wrap(new byte[]{2}), iterator.next());
-        assertFalse(iterator.hasNext());
-    }
-
     @Test
     public void shouldNotThrowNullPointerWhenCacheIsEmptyAndEvictionCalled() {
         cache.evict();
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/ThreadCacheTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/ThreadCacheTest.java
index a7a64c423b213..5882ee4b94a6f 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/ThreadCacheTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/ThreadCacheTest.java
@@ -310,10 +310,11 @@ public void apply(final List dirty) {
         }
         assertEquals(5, cache.size());
 
-        final ThreadCache.MemoryLRUCacheBytesIterator range = cache.range(namespace, Bytes.wrap(new byte[]{0}), Bytes.wrap(new byte[]{5}));
         // should evict byte[] {0}
         cache.put(namespace, Bytes.wrap(new byte[]{6}), dirtyEntry(new byte[]{6}));
 
+        final ThreadCache.MemoryLRUCacheBytesIterator range = cache.range(namespace, Bytes.wrap(new byte[]{0}), Bytes.wrap(new byte[]{5}));
+
         assertEquals(Bytes.wrap(new byte[]{1}), range.peekNextKey());
     }
 

From 5092b26393de731e8f826fde86b7ea97c9aaba35 Mon Sep 17 00:00:00 2001
From: Victoria Bialas 
Date: Tue, 19 Mar 2019 15:01:59 -0700
Subject: [PATCH 0039/1071] MINOR: update docs JSON serde links (#6465)

Reviewers: Joel Mamill , Matthias J. Sax 
---
 docs/streams/developer-guide/datatypes.html | 23 +++++++++------------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/docs/streams/developer-guide/datatypes.html b/docs/streams/developer-guide/datatypes.html
index 6c7869c3ba143..ca17c0b63627e 100644
--- a/docs/streams/developer-guide/datatypes.html
+++ b/docs/streams/developer-guide/datatypes.html
@@ -112,7 +112,7 @@ 
Primitive and basic types</dependency>
-

This artifact provides the following serde implementations under the package org.apache.kafka.common.serialization, which you can leverage when e.g., defining default serializers in your Streams configuration.

+

This artifact provides the following serde implementations under the package org.apache.kafka.common.serialization, which you can leverage when e.g., defining default serializers in your Streams configuration.

@@ -149,20 +149,17 @@

Primitive and basic types

Tip

-

Bytes is a wrapper for Java’s byte[] (byte array) that supports proper equality and ordering semantics. You may want to consider using Bytes instead of byte[] in your applications.

+

Bytes is a wrapper for Java’s byte[] (byte array) that supports proper equality and ordering semantics. You may want to consider using Bytes instead of byte[] in your applications.

JSON

-

The code examples of Kafka Streams also include a basic serde implementation for JSON:

+

The Kafka Streams code examples also include a basic serde implementation for JSON:

-

You can construct a unified JSON serde from the JsonPOJOSerializer and JsonPOJODeserializer via - Serdes.serdeFrom(<serializerInstance>, <deserializerInstance>). The - PageViewTypedDemo - example demonstrates how to use this JSON serde.

+

As shown in the example, you can use JSONSerdes inner classes Serdes.serdeFrom(<serializerInstance>, <deserializerInstance>) to construct JSON compatible serializers and deserializers. +

Implementing custom SerDes

@@ -170,13 +167,13 @@

JSON
  1. Write a serializer for your data type T by implementing - org.apache.kafka.common.serialization.Serializer.
    2. + org.apache.kafka.common.serialization.Serializer.
  2. Write a deserializer for T by implementing - org.apache.kafka.common.serialization.Deserializer.
    3. + org.apache.kafka.common.serialization.Deserializer.
  3. Write a serde for T by implementing - org.apache.kafka.common.serialization.Serde, + org.apache.kafka.common.serialization.Serde, which you either do manually (see existing SerDes in the previous section) or by leveraging helper functions in - Serdes + Serdes such as Serdes.serdeFrom(Serializer<T>, Deserializer<T>). Note that you will need to implement your own class (that has no generic types) if you want to use your custom serde in the configuration provided to KafkaStreams. If your serde class has generic types or you use Serdes.serdeFrom(Serializer<T>, Deserializer<T>), you can pass your serde only From 70ee72491fead119b18384e06cb93e66f81799b3 Mon Sep 17 00:00:00 2001 From: Victoria Bialas Date: Tue, 19 Mar 2019 16:33:41 -0700 Subject: [PATCH 0040/1071] MINOR: updated names for deprecated streams constants (#6466) * updated names for deprecated streams constants * add DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG in place of deprecated Reviewers: Jim Galasyn , Matthias J. Sax --- docs/streams/developer-guide/config-streams.html | 2 +- docs/streams/developer-guide/datatypes.html | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/streams/developer-guide/config-streams.html b/docs/streams/developer-guide/config-streams.html index 4b893ee7d1209..6e8b711500ea8 100644 --- a/docs/streams/developer-guide/config-streams.html +++ b/docs/streams/developer-guide/config-streams.html @@ -560,7 +560,7 @@

    state.dirimport org.apache.kafka.streams.StreamsConfig; Properties streamsConfiguration = new Properties(); -streamsConfiguration.put(StreamsConfig.TIMESTAMP_EXTRACTOR_CLASS_CONFIG, MyEventTimeExtractor.class); +streamsConfiguration.put(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, MyEventTimeExtractor.class);

diff --git a/docs/streams/developer-guide/datatypes.html b/docs/streams/developer-guide/datatypes.html index ca17c0b63627e..84868ba55f330 100644 --- a/docs/streams/developer-guide/datatypes.html +++ b/docs/streams/developer-guide/datatypes.html @@ -61,9 +61,9 @@

Configuring SerDesProperties settings = new Properties(); // Default serde for keys of data records (here: built-in serde for String type) -settings.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName()); +settings.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName()); // Default serde for values of data records (here: built-in serde for Long type) -settings.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.Long().getClass().getName()); +settings.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.Long().getClass().getName()); From b5ce093a24d0fe212b5d1374330fc720b4913bee Mon Sep 17 00:00:00 2001 From: "Matthias J. Sax" Date: Tue, 19 Mar 2019 17:27:32 -0700 Subject: [PATCH 0041/1071] MINOR: capture result timestamps in Kafka Streams DSL tests (#6447) Reviewers: Bill Bejeck , Guozhang Wang --- .../kafka/streams/StreamsBuilderTest.java | 14 +- .../kstream/internals/KStreamFlatMapTest.java | 21 +- .../internals/KStreamFlatMapValuesTest.java | 30 ++- .../KStreamGlobalKTableJoinTest.java | 36 ++-- .../KStreamGlobalKTableLeftJoinTest.java | 42 ++-- .../kstream/internals/KStreamImplTest.java | 79 ++++--- .../internals/KStreamKStreamJoinTest.java | 194 +++++------------- .../internals/KStreamKStreamLeftJoinTest.java | 79 ++++--- .../internals/KStreamKTableJoinTest.java | 41 ++-- .../internals/KStreamKTableLeftJoinTest.java | 44 ++-- .../kstream/internals/KStreamMapTest.java | 10 +- .../internals/KStreamMapValuesTest.java | 31 +-- .../internals/KStreamSelectKeyTest.java | 36 +--- .../internals/KStreamTransformTest.java | 93 ++++----- .../internals/KStreamTransformValuesTest.java | 83 ++++---- .../internals/KStreamWindowAggregateTest.java | 113 +++++----- .../internals/KTableAggregateTest.java | 57 +++-- .../kstream/internals/KTableFilterTest.java | 52 ++--- .../kstream/internals/KTableImplTest.java | 25 +-- .../internals/KTableKTableInnerJoinTest.java | 28 ++- .../internals/KTableKTableLeftJoinTest.java | 38 ++-- .../internals/KTableKTableOuterJoinTest.java | 43 ++-- .../kstream/internals/KTableMapKeysTest.java | 20 +- .../internals/KTableMapValuesTest.java | 39 +--- .../kstream/internals/KTableSourceTest.java | 50 ++--- .../internals/KTableTransformValuesTest.java | 49 ++--- .../SessionWindowedKStreamImplTest.java | 112 +++++----- .../TimeWindowedKStreamImplTest.java | 14 +- .../org/apache/kafka/test/MockProcessor.java | 7 +- 29 files changed, 594 insertions(+), 886 deletions(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java index 9e88a87b40201..b51eac8b9a697 100644 --- a/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java @@ -232,14 +232,14 @@ public void shouldProcessingFromSinkTopic() { source.process(processorSupplier); final ConsumerRecordFactory recordFactory = - new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer()); + new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L); try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { driver.pipeInput(recordFactory.create("topic-source", "A", "aa")); } // no exception was thrown - assertEquals(Collections.singletonList("A:aa"), processorSupplier.theCapturedProcessor().processed); + assertEquals(Collections.singletonList("A:aa (ts: 0)"), processorSupplier.theCapturedProcessor().processed); } @Test @@ -254,14 +254,14 @@ public void shouldProcessViaThroughTopic() { through.process(throughProcessorSupplier); final ConsumerRecordFactory recordFactory = - new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer()); + new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L); try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { driver.pipeInput(recordFactory.create("topic-source", "A", "aa")); } - assertEquals(Collections.singletonList("A:aa"), sourceProcessorSupplier.theCapturedProcessor().processed); - assertEquals(Collections.singletonList("A:aa"), throughProcessorSupplier.theCapturedProcessor().processed); + assertEquals(Collections.singletonList("A:aa (ts: 0)"), sourceProcessorSupplier.theCapturedProcessor().processed); + assertEquals(Collections.singletonList("A:aa (ts: 0)"), throughProcessorSupplier.theCapturedProcessor().processed); } @Test @@ -277,7 +277,7 @@ public void shouldMergeStreams() { merged.process(processorSupplier); final ConsumerRecordFactory recordFactory = - new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer()); + new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L); try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { driver.pipeInput(recordFactory.create(topic1, "A", "aa")); @@ -286,7 +286,7 @@ public void shouldMergeStreams() { driver.pipeInput(recordFactory.create(topic1, "D", "dd")); } - assertEquals(asList("A:aa", "B:bb", "C:cc", "D:dd"), processorSupplier.theCapturedProcessor().processed); + assertEquals(asList("A:aa (ts: 0)", "B:bb (ts: 0)", "C:cc (ts: 0)", "D:dd (ts: 0)"), processorSupplier.theCapturedProcessor().processed); } @Test diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapTest.java index 7feb18ffb2396..af8ec0879b0cf 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapTest.java @@ -36,25 +36,22 @@ import static org.junit.Assert.assertEquals; public class KStreamFlatMapTest { - - private String topicName = "topic"; - private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer()); + private final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer(), 0L); private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String()); @Test public void testFlatMap() { final StreamsBuilder builder = new StreamsBuilder(); + final String topicName = "topic"; final KeyValueMapper>> mapper = - new KeyValueMapper>>() { - @Override - public Iterable> apply(final Number key, final Object value) { - final ArrayList> result = new ArrayList<>(); - for (int i = 0; i < key.intValue(); i++) { - result.add(KeyValue.pair(Integer.toString(key.intValue() * 10 + i), value.toString())); - } - return result; + (key, value) -> { + final ArrayList> result = new ArrayList<>(); + for (int i = 0; i < key.intValue(); i++) { + result.add(KeyValue.pair(Integer.toString(key.intValue() * 10 + i), value.toString())); } + return result; }; final int[] expectedKeys = {0, 1, 2, 3}; @@ -74,7 +71,7 @@ public Iterable> apply(final Number key, final Object v assertEquals(6, supplier.theCapturedProcessor().processed.size()); - final String[] expected = {"10:V1", "20:V2", "21:V2", "30:V3", "31:V3", "32:V3"}; + final String[] expected = {"10:V1 (ts: 0)", "20:V2 (ts: 0)", "21:V2 (ts: 0)", "30:V3 (ts: 0)", "31:V3 (ts: 0)", "32:V3 (ts: 0)"}; for (int i = 0; i < expected.length; i++) { assertEquals(expected[i], supplier.theCapturedProcessor().processed.get(i)); diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapValuesTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapValuesTest.java index 0c4495e0a1a24..07437aa2509b2 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapValuesTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapValuesTest.java @@ -35,9 +35,9 @@ import static org.junit.Assert.assertArrayEquals; public class KStreamFlatMapValuesTest { - - private String topicName = "topic"; - private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new IntegerSerializer()); + private final String topicName = "topic"; + private final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(new IntegerSerializer(), new IntegerSerializer(), 0L); private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String()); @Test @@ -45,14 +45,11 @@ public void testFlatMapValues() { final StreamsBuilder builder = new StreamsBuilder(); final ValueMapper> mapper = - new ValueMapper>() { - @Override - public Iterable apply(final Number value) { - final ArrayList result = new ArrayList(); - result.add("v" + value); - result.add("V" + value); - return result; - } + value -> { + final ArrayList result = new ArrayList<>(); + result.add("v" + value); + result.add("V" + value); + return result; }; final int[] expectedKeys = {0, 1, 2, 3}; @@ -68,7 +65,7 @@ public Iterable apply(final Number value) { } } - final String[] expected = {"0:v0", "0:V0", "1:v1", "1:V1", "2:v2", "2:V2", "3:v3", "3:V3"}; + final String[] expected = {"0:v0 (ts: 0)", "0:V0 (ts: 0)", "1:v1 (ts: 0)", "1:V1 (ts: 0)", "2:v2 (ts: 0)", "2:V2 (ts: 0)", "3:v3 (ts: 0)", "3:V3 (ts: 0)"}; assertArrayEquals(expected, supplier.theCapturedProcessor().processed.toArray()); } @@ -79,15 +76,12 @@ public void testFlatMapValuesWithKeys() { final StreamsBuilder builder = new StreamsBuilder(); final ValueMapperWithKey> mapper = - new ValueMapperWithKey>() { - @Override - public Iterable apply(final Integer readOnlyKey, final Number value) { + (readOnlyKey, value) -> { final ArrayList result = new ArrayList<>(); result.add("v" + value); result.add("k" + readOnlyKey); return result; - } - }; + }; final int[] expectedKeys = {0, 1, 2, 3}; @@ -103,7 +97,7 @@ public Iterable apply(final Integer readOnlyKey, final Number value) { } } - final String[] expected = {"0:v0", "0:k0", "1:v1", "1:k1", "2:v2", "2:k2", "3:v3", "3:k3"}; + final String[] expected = {"0:v0 (ts: 0)", "0:k0 (ts: 0)", "1:v1 (ts: 0)", "1:k1 (ts: 0)", "2:v2 (ts: 0)", "2:k2 (ts: 0)", "3:v3 (ts: 0)", "3:k3 (ts: 0)"}; assertArrayEquals(expected, supplier.theCapturedProcessor().processed.toArray()); } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableJoinTest.java index a68583ec0451a..f98df4c11f812 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableJoinTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableJoinTest.java @@ -42,7 +42,6 @@ import static org.junit.Assert.assertEquals; public class KStreamGlobalKTableJoinTest { - private final String streamTopic = "streamTopic"; private final String globalTableTopic = "globalTableTopic"; private TopologyTestDriver driver; @@ -63,14 +62,11 @@ public void setUp() { final Consumed tableConsumed = Consumed.with(Serdes.String(), Serdes.String()); stream = builder.stream(streamTopic, streamConsumed); table = builder.globalTable(globalTableTopic, tableConsumed); - keyMapper = new KeyValueMapper() { - @Override - public String apply(final Integer key, final String value) { - final String[] tokens = value.split(","); - // Value is comma delimited. If second token is present, it's the key to the global ktable. - // If not present, use null to indicate no match - return tokens.length > 1 ? tokens[1] : null; - } + keyMapper = (key, value) -> { + final String[] tokens = value.split(","); + // Value is comma delimited. If second token is present, it's the key to the global ktable. + // If not present, use null to indicate no match + return tokens.length > 1 ? tokens[1] : null; }; stream.join(table, keyMapper, MockValueJoiner.TOSTRING_JOINER).process(supplier); @@ -86,7 +82,8 @@ public void cleanup() { } private void pushToStream(final int messageCount, final String valuePrefix, final boolean includeForeignKey) { - final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer()); + final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer(), 0L); for (int i = 0; i < messageCount; i++) { String value = valuePrefix + expectedKeys[i]; if (includeForeignKey) { @@ -97,14 +94,16 @@ private void pushToStream(final int messageCount, final String valuePrefix, fina } private void pushToGlobalTable(final int messageCount, final String valuePrefix) { - final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer()); + final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer()); for (int i = 0; i < messageCount; i++) { driver.pipeInput(recordFactory.create(globalTableTopic, "FKey" + expectedKeys[i], valuePrefix + expectedKeys[i])); } } private void pushNullValueToGlobalTable(final int messageCount) { - final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer()); + final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer()); for (int i = 0; i < messageCount; i++) { driver.pipeInput(recordFactory.create(globalTableTopic, "FKey" + expectedKeys[i], (String) null)); } @@ -112,7 +111,8 @@ private void pushNullValueToGlobalTable(final int messageCount) { @Test public void shouldNotRequireCopartitioning() { - final Collection> copartitionGroups = TopologyWrapper.getInternalTopologyBuilder(builder.build()).copartitionGroups(); + final Collection> copartitionGroups = + TopologyWrapper.getInternalTopologyBuilder(builder.build()).copartitionGroups(); assertEquals("KStream-GlobalKTable joins do not need to be co-partitioned", 0, copartitionGroups.size()); } @@ -142,7 +142,7 @@ public void shouldNotJoinOnGlobalTableUpdates() { // push all four items to the primary stream. this should produce two items. pushToStream(4, "X", true); - processor.checkAndClearProcessResult("0:X0,FKey0+Y0", "1:X1,FKey1+Y1"); + processor.checkAndClearProcessResult("0:X0,FKey0+Y0 (ts: 0)", "1:X1,FKey1+Y1 (ts: 0)"); // push all items to the globalTable. this should not produce any item @@ -152,7 +152,7 @@ public void shouldNotJoinOnGlobalTableUpdates() { // push all four items to the primary stream. this should produce four items. pushToStream(4, "X", true); - processor.checkAndClearProcessResult("0:X0,FKey0+YY0", "1:X1,FKey1+YY1", "2:X2,FKey2+YY2", "3:X3,FKey3+YY3"); + processor.checkAndClearProcessResult("0:X0,FKey0+YY0 (ts: 0)", "1:X1,FKey1+YY1 (ts: 0)", "2:X2,FKey2+YY2 (ts: 0)", "3:X3,FKey3+YY3 (ts: 0)"); // push all items to the globalTable. this should not produce any item @@ -171,7 +171,7 @@ public void shouldJoinOnlyIfMatchFoundOnStreamUpdates() { // push all four items to the primary stream. this should produce two items. pushToStream(4, "X", true); - processor.checkAndClearProcessResult("0:X0,FKey0+Y0", "1:X1,FKey1+Y1"); + processor.checkAndClearProcessResult("0:X0,FKey0+Y0 (ts: 0)", "1:X1,FKey1+Y1 (ts: 0)"); } @@ -186,7 +186,7 @@ public void shouldClearGlobalTableEntryOnNullValueUpdates() { // push all four items to the primary stream. this should produce four items. pushToStream(4, "X", true); - processor.checkAndClearProcessResult("0:X0,FKey0+Y0", "1:X1,FKey1+Y1", "2:X2,FKey2+Y2", "3:X3,FKey3+Y3"); + processor.checkAndClearProcessResult("0:X0,FKey0+Y0 (ts: 0)", "1:X1,FKey1+Y1 (ts: 0)", "2:X2,FKey2+Y2 (ts: 0)", "3:X3,FKey3+Y3 (ts: 0)"); // push two items with null to the globalTable as deletes. this should not produce any item. @@ -196,7 +196,7 @@ public void shouldClearGlobalTableEntryOnNullValueUpdates() { // push all four items to the primary stream. this should produce two items. pushToStream(4, "XX", true); - processor.checkAndClearProcessResult("2:XX2,FKey2+Y2", "3:XX3,FKey3+Y3"); + processor.checkAndClearProcessResult("2:XX2,FKey2+Y2 (ts: 0)", "3:XX3,FKey3+Y3 (ts: 0)"); } @Test diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableLeftJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableLeftJoinTest.java index aebc2a53cd67b..b8925bc435bc8 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableLeftJoinTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableLeftJoinTest.java @@ -42,7 +42,6 @@ import static org.junit.Assert.assertEquals; public class KStreamGlobalKTableLeftJoinTest { - final private String streamTopic = "streamTopic"; final private String globalTableTopic = "globalTableTopic"; @@ -65,14 +64,11 @@ public void setUp() { final Consumed tableConsumed = Consumed.with(Serdes.String(), Serdes.String()); stream = builder.stream(streamTopic, streamConsumed); table = builder.globalTable(globalTableTopic, tableConsumed); - keyMapper = new KeyValueMapper() { - @Override - public String apply(final Integer key, final String value) { - final String[] tokens = value.split(","); - // Value is comma delimited. If second token is present, it's the key to the global ktable. - // If not present, use null to indicate no match - return tokens.length > 1 ? tokens[1] : null; - } + keyMapper = (key, value) -> { + final String[] tokens = value.split(","); + // Value is comma delimited. If second token is present, it's the key to the global ktable. + // If not present, use null to indicate no match + return tokens.length > 1 ? tokens[1] : null; }; stream.leftJoin(table, keyMapper, MockValueJoiner.TOSTRING_JOINER).process(supplier); @@ -88,7 +84,8 @@ public void cleanup() { } private void pushToStream(final int messageCount, final String valuePrefix, final boolean includeForeignKey) { - final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer()); + final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer(), 0L); for (int i = 0; i < messageCount; i++) { String value = valuePrefix + expectedKeys[i]; if (includeForeignKey) { @@ -99,14 +96,16 @@ private void pushToStream(final int messageCount, final String valuePrefix, fina } private void pushToGlobalTable(final int messageCount, final String valuePrefix) { - final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer()); + final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L); for (int i = 0; i < messageCount; i++) { driver.pipeInput(recordFactory.create(globalTableTopic, "FKey" + expectedKeys[i], valuePrefix + expectedKeys[i])); } } private void pushNullValueToGlobalTable(final int messageCount) { - final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer()); + final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L); for (int i = 0; i < messageCount; i++) { driver.pipeInput(recordFactory.create(globalTableTopic, "FKey" + expectedKeys[i], (String) null)); } @@ -114,7 +113,8 @@ private void pushNullValueToGlobalTable(final int messageCount) { @Test public void shouldNotRequireCopartitioning() { - final Collection> copartitionGroups = TopologyWrapper.getInternalTopologyBuilder(builder.build()).copartitionGroups(); + final Collection> copartitionGroups = + TopologyWrapper.getInternalTopologyBuilder(builder.build()).copartitionGroups(); assertEquals("KStream-GlobalKTable joins do not need to be co-partitioned", 0, copartitionGroups.size()); } @@ -125,7 +125,7 @@ public void shouldNotJoinWithEmptyGlobalTableOnStreamUpdates() { // push two items to the primary stream. the globalTable is empty pushToStream(2, "X", true); - processor.checkAndClearProcessResult("0:X0,FKey0+null", "1:X1,FKey1+null"); + processor.checkAndClearProcessResult("0:X0,FKey0+null (ts: 0)", "1:X1,FKey1+null (ts: 0)"); } @Test @@ -134,7 +134,7 @@ public void shouldNotJoinOnGlobalTableUpdates() { // push two items to the primary stream. the globalTable is empty pushToStream(2, "X", true); - processor.checkAndClearProcessResult("0:X0,FKey0+null", "1:X1,FKey1+null"); + processor.checkAndClearProcessResult("0:X0,FKey0+null (ts: 0)", "1:X1,FKey1+null (ts: 0)"); // push two items to the globalTable. this should not produce any item. @@ -144,7 +144,7 @@ public void shouldNotJoinOnGlobalTableUpdates() { // push all four items to the primary stream. this should produce four items. pushToStream(4, "X", true); - processor.checkAndClearProcessResult("0:X0,FKey0+Y0", "1:X1,FKey1+Y1", "2:X2,FKey2+null", "3:X3,FKey3+null"); + processor.checkAndClearProcessResult("0:X0,FKey0+Y0 (ts: 0)", "1:X1,FKey1+Y1 (ts: 0)", "2:X2,FKey2+null (ts: 0)", "3:X3,FKey3+null (ts: 0)"); // push all items to the globalTable. this should not produce any item @@ -154,7 +154,7 @@ public void shouldNotJoinOnGlobalTableUpdates() { // push all four items to the primary stream. this should produce four items. pushToStream(4, "X", true); - processor.checkAndClearProcessResult("0:X0,FKey0+YY0", "1:X1,FKey1+YY1", "2:X2,FKey2+YY2", "3:X3,FKey3+YY3"); + processor.checkAndClearProcessResult("0:X0,FKey0+YY0 (ts: 0)", "1:X1,FKey1+YY1 (ts: 0)", "2:X2,FKey2+YY2 (ts: 0)", "3:X3,FKey3+YY3 (ts: 0)"); // push all items to the globalTable. this should not produce any item @@ -173,7 +173,7 @@ public void shouldJoinRegardlessIfMatchFoundOnStreamUpdates() { // push all four items to the primary stream. this should produce four items. pushToStream(4, "X", true); - processor.checkAndClearProcessResult("0:X0,FKey0+Y0", "1:X1,FKey1+Y1", "2:X2,FKey2+null", "3:X3,FKey3+null"); + processor.checkAndClearProcessResult("0:X0,FKey0+Y0 (ts: 0)", "1:X1,FKey1+Y1 (ts: 0)", "2:X2,FKey2+null (ts: 0)", "3:X3,FKey3+null (ts: 0)"); } @@ -188,7 +188,7 @@ public void shouldClearGlobalTableEntryOnNullValueUpdates() { // push all four items to the primary stream. this should produce four items. pushToStream(4, "X", true); - processor.checkAndClearProcessResult("0:X0,FKey0+Y0", "1:X1,FKey1+Y1", "2:X2,FKey2+Y2", "3:X3,FKey3+Y3"); + processor.checkAndClearProcessResult("0:X0,FKey0+Y0 (ts: 0)", "1:X1,FKey1+Y1 (ts: 0)", "2:X2,FKey2+Y2 (ts: 0)", "3:X3,FKey3+Y3 (ts: 0)"); // push two items with null to the globalTable as deletes. this should not produce any item. @@ -198,7 +198,7 @@ public void shouldClearGlobalTableEntryOnNullValueUpdates() { // push all four items to the primary stream. this should produce four items. pushToStream(4, "XX", true); - processor.checkAndClearProcessResult("0:XX0,FKey0+null", "1:XX1,FKey1+null", "2:XX2,FKey2+Y2", "3:XX3,FKey3+Y3"); + processor.checkAndClearProcessResult("0:XX0,FKey0+null (ts: 0)", "1:XX1,FKey1+null (ts: 0)", "2:XX2,FKey2+Y2 (ts: 0)", "3:XX3,FKey3+Y3 (ts: 0)"); } @Test @@ -213,7 +213,7 @@ public void shouldJoinOnNullKeyMapperValues() { // this should produce four items. pushToStream(4, "XXX", false); - processor.checkAndClearProcessResult("0:XXX0+null", "1:XXX1+null", "2:XXX2+null", "3:XXX3+null"); + processor.checkAndClearProcessResult("0:XXX0+null (ts: 0)", "1:XXX1+null (ts: 0)", "2:XXX2+null (ts: 0)", "3:XXX3+null (ts: 0)"); } } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java index bd2ab5ba7489a..3b450b15fa66e 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java @@ -85,10 +85,11 @@ public class KStreamImplTest { private KStream testStream; private StreamsBuilder builder; - private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer()); + private final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L); private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String()); - private Serde mySerde = new Serdes.StringSerde(); + private final Serde mySerde = new Serdes.StringSerde(); @Before public void before() { @@ -132,7 +133,7 @@ public void testNumProcesses() { stream4.to("topic-5"); - streams2[1].through("topic-6").process(new MockProcessorSupplier()); + streams2[1].through("topic-6").process(new MockProcessorSupplier<>()); assertEquals(2 + // sources 2 + // stream1 @@ -225,41 +226,41 @@ public void close() {} assertEquals(((AbstractStream) stream1.groupByKey(Grouped.with(mySerde, mySerde))).keySerde(), mySerde); assertEquals(((AbstractStream) stream1.groupByKey(Grouped.with(mySerde, mySerde))).valueSerde(), mySerde); - assertEquals(((AbstractStream) stream1.groupBy(selector)).keySerde(), null); + assertNull(((AbstractStream) stream1.groupBy(selector)).keySerde()); assertEquals(((AbstractStream) stream1.groupBy(selector)).valueSerde(), consumedInternal.valueSerde()); assertEquals(((AbstractStream) stream1.groupBy(selector, Grouped.with(mySerde, mySerde))).keySerde(), mySerde); assertEquals(((AbstractStream) stream1.groupBy(selector, Grouped.with(mySerde, mySerde))).valueSerde(), mySerde); - assertEquals(((AbstractStream) stream1.join(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)))).keySerde(), null); - assertEquals(((AbstractStream) stream1.join(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)))).valueSerde(), null); + assertNull(((AbstractStream) stream1.join(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)))).keySerde()); + assertNull(((AbstractStream) stream1.join(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)))).valueSerde()); assertEquals(((AbstractStream) stream1.join(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)), Joined.with(mySerde, mySerde, mySerde))).keySerde(), mySerde); assertNull(((AbstractStream) stream1.join(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)), Joined.with(mySerde, mySerde, mySerde))).valueSerde()); - assertEquals(((AbstractStream) stream1.leftJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)))).keySerde(), null); - assertEquals(((AbstractStream) stream1.leftJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)))).valueSerde(), null); + assertNull(((AbstractStream) stream1.leftJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)))).keySerde()); + assertNull(((AbstractStream) stream1.leftJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)))).valueSerde()); assertEquals(((AbstractStream) stream1.leftJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)), Joined.with(mySerde, mySerde, mySerde))).keySerde(), mySerde); assertNull(((AbstractStream) stream1.leftJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)), Joined.with(mySerde, mySerde, mySerde))).valueSerde()); - assertEquals(((AbstractStream) stream1.outerJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)))).keySerde(), null); - assertEquals(((AbstractStream) stream1.outerJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)))).valueSerde(), null); + assertNull(((AbstractStream) stream1.outerJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)))).keySerde()); + assertNull(((AbstractStream) stream1.outerJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)))).valueSerde()); assertEquals(((AbstractStream) stream1.outerJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)), Joined.with(mySerde, mySerde, mySerde))).keySerde(), mySerde); assertNull(((AbstractStream) stream1.outerJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)), Joined.with(mySerde, mySerde, mySerde))).valueSerde()); assertEquals(((AbstractStream) stream1.join(table1, joiner)).keySerde(), consumedInternal.keySerde()); - assertEquals(((AbstractStream) stream1.join(table1, joiner)).valueSerde(), null); + assertNull(((AbstractStream) stream1.join(table1, joiner)).valueSerde()); assertEquals(((AbstractStream) stream1.join(table1, joiner, Joined.with(mySerde, mySerde, mySerde))).keySerde(), mySerde); - assertEquals(((AbstractStream) stream1.join(table1, joiner, Joined.with(mySerde, mySerde, mySerde))).valueSerde(), null); + assertNull(((AbstractStream) stream1.join(table1, joiner, Joined.with(mySerde, mySerde, mySerde))).valueSerde()); assertEquals(((AbstractStream) stream1.leftJoin(table1, joiner)).keySerde(), consumedInternal.keySerde()); - assertEquals(((AbstractStream) stream1.leftJoin(table1, joiner)).valueSerde(), null); + assertNull(((AbstractStream) stream1.leftJoin(table1, joiner)).valueSerde()); assertEquals(((AbstractStream) stream1.leftJoin(table1, joiner, Joined.with(mySerde, mySerde, mySerde))).keySerde(), mySerde); - assertEquals(((AbstractStream) stream1.leftJoin(table1, joiner, Joined.with(mySerde, mySerde, mySerde))).valueSerde(), null); + assertNull(((AbstractStream) stream1.leftJoin(table1, joiner, Joined.with(mySerde, mySerde, mySerde))).valueSerde()); assertEquals(((AbstractStream) stream1.join(table2, selector, joiner)).keySerde(), consumedInternal.keySerde()); - assertEquals(((AbstractStream) stream1.join(table2, selector, joiner)).valueSerde(), null); + assertNull(((AbstractStream) stream1.join(table2, selector, joiner)).valueSerde()); assertEquals(((AbstractStream) stream1.leftJoin(table2, selector, joiner)).keySerde(), consumedInternal.keySerde()); - assertEquals(((AbstractStream) stream1.leftJoin(table2, selector, joiner)).valueSerde(), null); + assertNull(((AbstractStream) stream1.leftJoin(table2, selector, joiner)).valueSerde()); } @Test @@ -273,10 +274,10 @@ public void shouldUseRecordMetadataTimestampExtractorWithThrough() { final ProcessorTopology processorTopology = TopologyWrapper.getInternalTopologyBuilder(builder.build()).setApplicationId("X").build(null); assertThat(processorTopology.source("topic-6").getTimestampExtractor(), instanceOf(FailOnInvalidTimestamp.class)); - assertEquals(processorTopology.source("topic-4").getTimestampExtractor(), null); - assertEquals(processorTopology.source("topic-3").getTimestampExtractor(), null); - assertEquals(processorTopology.source("topic-2").getTimestampExtractor(), null); - assertEquals(processorTopology.source("topic-1").getTimestampExtractor(), null); + assertNull(processorTopology.source("topic-4").getTimestampExtractor()); + assertNull(processorTopology.source("topic-3").getTimestampExtractor()); + assertNull(processorTopology.source("topic-2").getTimestampExtractor()); + assertNull(processorTopology.source("topic-1").getTimestampExtractor()); } @Test @@ -289,7 +290,7 @@ public void shouldSendDataThroughTopicUsingProduced() { try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { driver.pipeInput(recordFactory.create(input, "a", "b")); } - assertThat(processorSupplier.theCapturedProcessor().processed, equalTo(Collections.singletonList("a:b"))); + assertThat(processorSupplier.theCapturedProcessor().processed, equalTo(Collections.singletonList("a:b (ts: 0)"))); } @Test @@ -303,7 +304,7 @@ public void shouldSendDataToTopicUsingProduced() { try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { driver.pipeInput(recordFactory.create(input, "e", "f")); } - assertThat(processorSupplier.theCapturedProcessor().processed, equalTo(Collections.singletonList("e:f"))); + assertThat(processorSupplier.theCapturedProcessor().processed, equalTo(Collections.singletonList("e:f (ts: 0)"))); } @Test @@ -322,8 +323,8 @@ public void shouldSendDataToDynamicTopics() { driver.pipeInput(recordFactory.create(input, "b", "v1")); } final List> mockProcessors = processorSupplier.capturedProcessors(2); - assertThat(mockProcessors.get(0).processed, equalTo(asList("a:v1", "a:v2"))); - assertThat(mockProcessors.get(1).processed, equalTo(Collections.singletonList("b:v1"))); + assertThat(mockProcessors.get(0).processed, equalTo(asList("a:v1 (ts: 0)", "a:v2 (ts: 0)"))); + assertThat(mockProcessors.get(1).processed, equalTo(Collections.singletonList("b:v1 (ts: 0)"))); } @SuppressWarnings("deprecation") // specifically testing the deprecated variant @@ -334,12 +335,7 @@ public void shouldUseRecordMetadataTimestampExtractorWhenInternalRepartitioningT final ValueJoiner valueJoiner = MockValueJoiner.instance(":"); final long windowSize = TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS); final KStream stream = kStream - .map(new KeyValueMapper>() { - @Override - public KeyValue apply(final String key, final String value) { - return KeyValue.pair(value, value); - } - }); + .map((key, value) -> KeyValue.pair(value, value)); stream.join(kStream, valueJoiner, JoinWindows.of(ofMillis(windowSize)).grace(ofMillis(3 * windowSize)), @@ -368,12 +364,7 @@ public void shouldUseRecordMetadataTimestampExtractorWhenInternalRepartitioningT final ValueJoiner valueJoiner = MockValueJoiner.instance(":"); final long windowSize = TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS); final KStream stream = kStream - .map(new KeyValueMapper>() { - @Override - public KeyValue apply(final String key, final String value) { - return KeyValue.pair(value, value); - } - }); + .map((key, value) -> KeyValue.pair(value, value)); stream.join( kStream, valueJoiner, @@ -559,7 +550,7 @@ public void shouldNotAllowNullActionOnForEach() { @Test(expected = NullPointerException.class) public void shouldNotAllowNullTableOnJoinWithGlobalTable() { testStream.join((GlobalKTable) null, - MockMapper.selectValueMapper(), + MockMapper.selectValueMapper(), MockValueJoiner.TOSTRING_JOINER); } @@ -573,14 +564,14 @@ public void shouldNotAllowNullMapperOnJoinWithGlobalTable() { @Test(expected = NullPointerException.class) public void shouldNotAllowNullJoinerOnJoinWithGlobalTable() { testStream.join(builder.globalTable("global", stringConsumed), - MockMapper.selectValueMapper(), + MockMapper.selectValueMapper(), null); } @Test(expected = NullPointerException.class) public void shouldNotAllowNullTableOnJLeftJoinWithGlobalTable() { testStream.leftJoin((GlobalKTable) null, - MockMapper.selectValueMapper(), + MockMapper.selectValueMapper(), MockValueJoiner.TOSTRING_JOINER); } @@ -594,7 +585,7 @@ public void shouldNotAllowNullMapperOnLeftJoinWithGlobalTable() { @Test(expected = NullPointerException.class) public void shouldNotAllowNullJoinerOnLeftJoinWithGlobalTable() { testStream.leftJoin(builder.globalTable("global", stringConsumed), - MockMapper.selectValueMapper(), + MockMapper.selectValueMapper(), null); } @@ -667,7 +658,7 @@ public void shouldMergeTwoStreams() { driver.pipeInput(recordFactory.create(topic1, "D", "dd")); } - assertEquals(asList("A:aa", "B:bb", "C:cc", "D:dd"), processorSupplier.theCapturedProcessor().processed); + assertEquals(asList("A:aa (ts: 0)", "B:bb (ts: 0)", "C:cc (ts: 0)", "D:dd (ts: 0)"), processorSupplier.theCapturedProcessor().processed); } @Test @@ -696,7 +687,7 @@ public void shouldMergeMultipleStreams() { driver.pipeInput(recordFactory.create(topic1, "H", "hh")); } - assertEquals(asList("A:aa", "B:bb", "C:cc", "D:dd", "E:ee", "F:ff", "G:gg", "H:hh"), + assertEquals(asList("A:aa (ts: 0)", "B:bb (ts: 0)", "C:cc (ts: 0)", "D:dd (ts: 0)", "E:ee (ts: 0)", "F:ff (ts: 0)", "G:gg (ts: 0)", "H:hh (ts: 0)"), processorSupplier.theCapturedProcessor().processed); } @@ -714,7 +705,7 @@ public void shouldProcessFromSourceThatMatchPattern() { driver.pipeInput(recordFactory.create("topic-7", "E", "ee")); } - assertEquals(asList("A:aa", "B:bb", "C:cc", "D:dd", "E:ee"), + assertEquals(asList("A:aa (ts: 0)", "B:bb (ts: 0)", "C:cc (ts: 0)", "D:dd (ts: 0)", "E:ee (ts: 0)"), processorSupplier.theCapturedProcessor().processed); } @@ -737,7 +728,7 @@ public void shouldProcessFromSourcesThatMatchMultiplePattern() { driver.pipeInput(recordFactory.create(topic3, "E", "ee")); } - assertEquals(asList("A:aa", "B:bb", "C:cc", "D:dd", "E:ee"), + assertEquals(asList("A:aa (ts: 0)", "B:bb (ts: 0)", "C:cc (ts: 0)", "D:dd (ts: 0)", "E:ee (ts: 0)"), processorSupplier.theCapturedProcessor().processed); } } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoinTest.java index 71367b24f3dd7..baac74e6c3eb7 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoinTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoinTest.java @@ -47,13 +47,12 @@ import static org.junit.Assert.assertEquals; public class KStreamKStreamJoinTest { - final private String topic1 = "topic1"; final private String topic2 = "topic2"; private final Consumed consumed = Consumed.with(Serdes.Integer(), Serdes.String()); private final ConsumerRecordFactory recordFactory = - new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer()); + new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer(), 0L); private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String()); @Test @@ -109,7 +108,6 @@ public void testJoin() { assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next()); try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { - final MockProcessor processor = supplier.theCapturedProcessor(); // push two items to the primary stream. the other window is empty @@ -117,11 +115,9 @@ public void testJoin() { // w2 = {} // --> w1 = { 0:X0, 1:X1 } // w2 = {} - for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i])); } - processor.checkAndClearProcessResult(); // push two items to the other stream. this should produce two items. @@ -129,60 +125,50 @@ public void testJoin() { // w2 = {} // --> w1 = { 0:X0, 1:X1 } // w2 = { 0:Y0, 1:Y1 } - for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i])); } - - processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1"); + processor.checkAndClearProcessResult("0:X0+Y0 (ts: 0)", "1:X1+Y1 (ts: 0)"); // push all four items to the primary stream. this should produce two items. // w1 = { 0:X0, 1:X1 } // w2 = { 0:Y0, 1:Y1 } // --> w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2, 3:X3 } // w2 = { 0:Y0, 1:Y1 } - for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "X" + expectedKey)); } - - processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1"); + processor.checkAndClearProcessResult("0:X0+Y0 (ts: 0)", "1:X1+Y1 (ts: 0)"); // push all items to the other stream. this should produce six items. // w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2, 3:X3 } // w2 = { 0:Y0, 1:Y1 } // --> w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2, 3:X3 } // w2 = { 0:Y0, 1:Y1, 0:YY0, 1:YY1, 2:YY2, 3:YY3 } - for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey)); } - - processor.checkAndClearProcessResult("0:X0+YY0", "0:X0+YY0", "1:X1+YY1", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3"); + processor.checkAndClearProcessResult("0:X0+YY0 (ts: 0)", "0:X0+YY0 (ts: 0)", "1:X1+YY1 (ts: 0)", "1:X1+YY1 (ts: 0)", "2:X2+YY2 (ts: 0)", "3:X3+YY3 (ts: 0)"); // push all four items to the primary stream. this should produce six items. // w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2, 3:X3 } - // w2 = { 0:Y0, 1:Y1, 0:YY0, 0:YY0, 1:YY1, 2:YY2, 3:YY3 + // w2 = { 0:Y0, 1:Y1, 0:YY0, 0:YY0, 1:YY1, 2:YY2, 3:YY3 } // --> w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2, 3:X3, 0:XX0, 1:XX1, 2:XX2, 3:XX3 } // w2 = { 0:Y0, 1:Y1, 0:YY0, 1:YY1, 2:YY2, 3:YY3 } - for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey)); } - - processor.checkAndClearProcessResult("0:XX0+Y0", "0:XX0+YY0", "1:XX1+Y1", "1:XX1+YY1", "2:XX2+YY2", "3:XX3+YY3"); + processor.checkAndClearProcessResult("0:XX0+Y0 (ts: 0)", "0:XX0+YY0 (ts: 0)", "1:XX1+Y1 (ts: 0)", "1:XX1+YY1 (ts: 0)", "2:XX2+YY2 (ts: 0)", "3:XX3+YY3 (ts: 0)"); // push two items to the other stream. this should produce six item. // w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2, 3:X3, 0:XX0, 1:XX1, 2:XX2, 3:XX3 } - // w2 = { 0:Y0, 1:Y1, 0:YY0, 0:YY0, 1:YY1, 2:YY2, 3:YY3 + // w2 = { 0:Y0, 1:Y1, 0:YY0, 0:YY0, 1:YY1, 2:YY2, 3:YY3 } // --> w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2, 3:X3, 0:XX0, 1:XX1, 2:XX2, 3:XX3 } // w2 = { 0:Y0, 1:Y1, 0:YY0, 1:YY1, 2:YY2, 3:YY3, 0:YYY0, 1:YYY1 } - for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "YYY" + expectedKeys[i])); } - - processor.checkAndClearProcessResult("0:X0+YYY0", "0:X0+YYY0", "0:XX0+YYY0", "1:X1+YYY1", "1:X1+YYY1", "1:XX1+YYY1"); + processor.checkAndClearProcessResult("0:X0+YYY0 (ts: 0)", "0:X0+YYY0 (ts: 0)", "0:XX0+YYY0 (ts: 0)", "1:X1+YYY1 (ts: 0)", "1:X1+YYY1 (ts: 0)", "1:XX1+YYY1 (ts: 0)"); } } @@ -211,8 +197,7 @@ public void testOuterJoin() { assertEquals(1, copartitionGroups.size()); assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next()); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { - + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { final MockProcessor processor = supplier.theCapturedProcessor(); // push two items to the primary stream. the other window is empty.this should produce two items @@ -220,79 +205,65 @@ public void testOuterJoin() { // w2 = {} // --> w1 = { 0:X0, 1:X1 } // w2 = {} - for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i])); } - - processor.checkAndClearProcessResult("0:X0+null", "1:X1+null"); + processor.checkAndClearProcessResult("0:X0+null (ts: 0)", "1:X1+null (ts: 0)"); // push two items to the other stream. this should produce two items. // w1 = { 0:X0, 1:X1 } // w2 = {} // --> w1 = { 0:X0, 1:X1 } // w2 = { 0:Y0, 1:Y1 } - for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i])); } - - processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1"); + processor.checkAndClearProcessResult("0:X0+Y0 (ts: 0)", "1:X1+Y1 (ts: 0)"); // push all four items to the primary stream. this should produce four items. // w1 = { 0:X0, 1:X1 } // w2 = { 0:Y0, 1:Y1 } // --> w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2, 3:X3 } // w2 = { 0:Y0, 1:Y1 } - for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "X" + expectedKey)); } - - processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1", "2:X2+null", "3:X3+null"); + processor.checkAndClearProcessResult("0:X0+Y0 (ts: 0)", "1:X1+Y1 (ts: 0)", "2:X2+null (ts: 0)", "3:X3+null (ts: 0)"); // push all items to the other stream. this should produce six items. // w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2, 3:X3 } // w2 = { 0:Y0, 1:Y1 } // --> w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2, 3:X3 } // w2 = { 0:Y0, 1:Y1, 0:YY0, 0:YY0, 1:YY1, 2:YY2, 3:YY3 } - for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey)); } - - processor.checkAndClearProcessResult("0:X0+YY0", "0:X0+YY0", "1:X1+YY1", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3"); + processor.checkAndClearProcessResult("0:X0+YY0 (ts: 0)", "0:X0+YY0 (ts: 0)", "1:X1+YY1 (ts: 0)", "1:X1+YY1 (ts: 0)", "2:X2+YY2 (ts: 0)", "3:X3+YY3 (ts: 0)"); // push all four items to the primary stream. this should produce six items. // w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2, 3:X3 } - // w2 = { 0:Y0, 1:Y1, 0:YY0, 0:YY0, 1:YY1, 2:YY2, 3:YY3 + // w2 = { 0:Y0, 1:Y1, 0:YY0, 0:YY0, 1:YY1, 2:YY2, 3:YY3 } // --> w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2, 3:X3, 0:XX0, 1:XX1, 2:XX2, 3:XX3 } // w2 = { 0:Y0, 1:Y1, 0:YY0, 0:YY0, 1:YY1, 2:YY2, 3:YY3 } - for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey)); } - - processor.checkAndClearProcessResult("0:XX0+Y0", "0:XX0+YY0", "1:XX1+Y1", "1:XX1+YY1", "2:XX2+YY2", "3:XX3+YY3"); + processor.checkAndClearProcessResult("0:XX0+Y0 (ts: 0)", "0:XX0+YY0 (ts: 0)", "1:XX1+Y1 (ts: 0)", "1:XX1+YY1 (ts: 0)", "2:XX2+YY2 (ts: 0)", "3:XX3+YY3 (ts: 0)"); // push two items to the other stream. this should produce six item. // w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2, 3:X3, 0:XX0, 1:XX1, 2:XX2, 3:XX3 } - // w2 = { 0:Y0, 1:Y1, 0:YY0, 0:YY0, 1:YY1, 2:YY2, 3:YY3 + // w2 = { 0:Y0, 1:Y1, 0:YY0, 0:YY0, 1:YY1, 2:YY2, 3:YY3 } // --> w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2, 3:X3, 0:XX0, 1:XX1, 2:XX2, 3:XX3 } // w2 = { 0:Y0, 1:Y1, 0:YY0, 0:YY0, 1:YY1, 2:YY2, 3:YY3, 0:YYY0, 1:YYY1 } - for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "YYY" + expectedKeys[i])); } - - processor.checkAndClearProcessResult("0:X0+YYY0", "0:X0+YYY0", "0:XX0+YYY0", "1:X1+YYY1", "1:X1+YYY1", "1:XX1+YYY1"); + processor.checkAndClearProcessResult("0:X0+YYY0 (ts: 0)", "0:X0+YYY0 (ts: 0)", "0:XX0+YYY0 (ts: 0)", "1:X1+YYY1 (ts: 0)", "1:X1+YYY1 (ts: 0)", "1:XX1+YYY1 (ts: 0)"); } } @Test public void testWindowing() { - long time = 0L; - final StreamsBuilder builder = new StreamsBuilder(); final int[] expectedKeys = new int[]{0, 1, 2, 3}; @@ -317,7 +288,9 @@ public void testWindowing() { assertEquals(1, copartitionGroups.size()); assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next()); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, time)) { + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { + final MockProcessor processor = supplier.theCapturedProcessor(); + long time = 0L; // push two items to the primary stream. the other window is empty. this should produce no items. // w1 = {} @@ -327,9 +300,6 @@ public void testWindowing() { for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i], time)); } - - final MockProcessor processor = supplier.theCapturedProcessor(); - processor.checkAndClearProcessResult(); // push two items to the other stream. this should produce two items. @@ -341,8 +311,7 @@ public void testWindowing() { for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i], time)); } - - processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1"); + processor.checkAndClearProcessResult("0:X0+Y0 (ts: 0)", "1:X1+Y1 (ts: 0)"); // clear logically time = 1000L; @@ -353,168 +322,141 @@ public void testWindowing() { // gradually expires items in w1 // w1 = { 0:X0, 1:X1, 2:X2, 3:X3 } - time += 100L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3"); + processor.checkAndClearProcessResult("0:X0+YY0 (ts: 1100)", "1:X1+YY1 (ts: 1100)", "2:X2+YY2 (ts: 1100)", "3:X3+YY3 (ts: 1100)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("1:X1+YY1", "2:X2+YY2", "3:X3+YY3"); + processor.checkAndClearProcessResult("1:X1+YY1 (ts: 1101)", "2:X2+YY2 (ts: 1101)", "3:X3+YY3 (ts: 1101)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("2:X2+YY2", "3:X3+YY3"); + processor.checkAndClearProcessResult("2:X2+YY2 (ts: 1102)", "3:X3+YY3 (ts: 1102)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("3:X3+YY3"); + processor.checkAndClearProcessResult("3:X3+YY3 (ts: 1103)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - processor.checkAndClearProcessResult(); // go back to the time before expiration - time = 1000L - 100L - 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - processor.checkAndClearProcessResult(); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("0:X0+YY0"); + processor.checkAndClearProcessResult("0:X0+YY0 (ts: 900)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1"); + processor.checkAndClearProcessResult("0:X0+YY0 (ts: 901)", "1:X1+YY1 (ts: 901)"); time += 1; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2"); + processor.checkAndClearProcessResult("0:X0+YY0 (ts: 902)", "1:X1+YY1 (ts: 902)", "2:X2+YY2 (ts: 902)"); time += 1; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3"); + processor.checkAndClearProcessResult("0:X0+YY0 (ts: 903)", "1:X1+YY1 (ts: 903)", "2:X2+YY2 (ts: 903)", "3:X3+YY3 (ts: 903)"); // clear (logically) time = 2000L; for (int i = 0; i < expectedKeys.length; i++) { driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i], time + i)); } - processor.checkAndClearProcessResult(); // gradually expires items in w2 // w2 = { 0:Y0, 1:Y1, 2:Y2, 3:Y3 } - time = 2000L + 100L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time)); } - - processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+Y1", "2:XX2+Y2", "3:XX3+Y3"); + processor.checkAndClearProcessResult("0:XX0+Y0 (ts: 2100)", "1:XX1+Y1 (ts: 2100)", "2:XX2+Y2 (ts: 2100)", "3:XX3+Y3 (ts: 2100)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time)); } - - processor.checkAndClearProcessResult("1:XX1+Y1", "2:XX2+Y2", "3:XX3+Y3"); + processor.checkAndClearProcessResult("1:XX1+Y1 (ts: 2101)", "2:XX2+Y2 (ts: 2101)", "3:XX3+Y3 (ts: 2101)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time)); } - - processor.checkAndClearProcessResult("2:XX2+Y2", "3:XX3+Y3"); + processor.checkAndClearProcessResult("2:XX2+Y2 (ts: 2102)", "3:XX3+Y3 (ts: 2102)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time)); } - - processor.checkAndClearProcessResult("3:XX3+Y3"); + processor.checkAndClearProcessResult("3:XX3+Y3 (ts: 2103)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time)); } - processor.checkAndClearProcessResult(); // go back to the time before expiration - time = 2000L - 100L - 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time)); } - processor.checkAndClearProcessResult(); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time)); } - - processor.checkAndClearProcessResult("0:XX0+Y0"); + processor.checkAndClearProcessResult("0:XX0+Y0 (ts: 1900)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time)); } - - processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+Y1"); + processor.checkAndClearProcessResult("0:XX0+Y0 (ts: 1901)", "1:XX1+Y1 (ts: 1901)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time)); } - - processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+Y1", "2:XX2+Y2"); + processor.checkAndClearProcessResult("0:XX0+Y0 (ts: 1902)", "1:XX1+Y1 (ts: 1902)", "2:XX2+Y2 (ts: 1902)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time)); } - - processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+Y1", "2:XX2+Y2", "3:XX3+Y3"); + processor.checkAndClearProcessResult("0:XX0+Y0 (ts: 1903)", "1:XX1+Y1 (ts: 1903)", "2:XX2+Y2 (ts: 1903)", "3:XX3+Y3 (ts: 1903)"); } } @Test public void testAsymmetricWindowingAfter() { - long time = 1000L; - final StreamsBuilder builder = new StreamsBuilder(); final int[] expectedKeys = new int[]{0, 1, 2, 3}; @@ -541,9 +483,9 @@ public void testAsymmetricWindowingAfter() { assertEquals(1, copartitionGroups.size()); assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next()); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, time)) { - + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { final MockProcessor processor = supplier.theCapturedProcessor(); + long time = 1000L; for (int i = 0; i < expectedKeys.length; i++) { driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i], time + i)); @@ -554,78 +496,66 @@ public void testAsymmetricWindowingAfter() { for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - processor.checkAndClearProcessResult(); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("0:X0+YY0"); + processor.checkAndClearProcessResult("0:X0+YY0 (ts: 1000)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1"); + processor.checkAndClearProcessResult("0:X0+YY0 (ts: 1001)", "1:X1+YY1 (ts: 1001)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2"); + processor.checkAndClearProcessResult("0:X0+YY0 (ts: 1002)", "1:X1+YY1 (ts: 1002)", "2:X2+YY2 (ts: 1002)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3"); + processor.checkAndClearProcessResult("0:X0+YY0 (ts: 1003)", "1:X1+YY1 (ts: 1003)", "2:X2+YY2 (ts: 1003)", "3:X3+YY3 (ts: 1003)"); time = 1000 + 100L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3"); + processor.checkAndClearProcessResult("0:X0+YY0 (ts: 1100)", "1:X1+YY1 (ts: 1100)", "2:X2+YY2 (ts: 1100)", "3:X3+YY3 (ts: 1100)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("1:X1+YY1", "2:X2+YY2", "3:X3+YY3"); + processor.checkAndClearProcessResult("1:X1+YY1 (ts: 1101)", "2:X2+YY2 (ts: 1101)", "3:X3+YY3 (ts: 1101)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("2:X2+YY2", "3:X3+YY3"); + processor.checkAndClearProcessResult("2:X2+YY2 (ts: 1102)", "3:X3+YY3 (ts: 1102)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("3:X3+YY3"); + processor.checkAndClearProcessResult("3:X3+YY3 (ts: 1103)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - processor.checkAndClearProcessResult(); } } @Test public void testAsymmetricWindowingBefore() { - long time = 1000L; - final StreamsBuilder builder = new StreamsBuilder(); final int[] expectedKeys = new int[]{0, 1, 2, 3}; @@ -651,9 +581,9 @@ public void testAsymmetricWindowingBefore() { assertEquals(1, copartitionGroups.size()); assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next()); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, time)) { - + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { final MockProcessor processor = supplier.theCapturedProcessor(); + long time = 1000L; for (int i = 0; i < expectedKeys.length; i++) { driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i], time + i)); @@ -664,70 +594,60 @@ public void testAsymmetricWindowingBefore() { for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - processor.checkAndClearProcessResult(); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("0:X0+YY0"); + processor.checkAndClearProcessResult("0:X0+YY0 (ts: 900)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1"); + processor.checkAndClearProcessResult("0:X0+YY0 (ts: 901)", "1:X1+YY1 (ts: 901)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2"); + processor.checkAndClearProcessResult("0:X0+YY0 (ts: 902)", "1:X1+YY1 (ts: 902)", "2:X2+YY2 (ts: 902)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3"); + processor.checkAndClearProcessResult("0:X0+YY0 (ts: 903)", "1:X1+YY1 (ts: 903)", "2:X2+YY2 (ts: 903)", "3:X3+YY3 (ts: 903)"); time = 1000L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3"); + processor.checkAndClearProcessResult("0:X0+YY0 (ts: 1000)", "1:X1+YY1 (ts: 1000)", "2:X2+YY2 (ts: 1000)", "3:X3+YY3 (ts: 1000)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("1:X1+YY1", "2:X2+YY2", "3:X3+YY3"); + processor.checkAndClearProcessResult("1:X1+YY1 (ts: 1001)", "2:X2+YY2 (ts: 1001)", "3:X3+YY3 (ts: 1001)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("2:X2+YY2", "3:X3+YY3"); + processor.checkAndClearProcessResult("2:X2+YY2 (ts: 1002)", "3:X3+YY3 (ts: 1002)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - - processor.checkAndClearProcessResult("3:X3+YY3"); + processor.checkAndClearProcessResult("3:X3+YY3 (ts: 1003)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time)); } - processor.checkAndClearProcessResult(); } } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamLeftJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamLeftJoinTest.java index b019411e3ff84..863b790399e49 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamLeftJoinTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamLeftJoinTest.java @@ -43,12 +43,12 @@ import static org.junit.Assert.assertEquals; public class KStreamKStreamLeftJoinTest { - final private String topic1 = "topic1"; final private String topic2 = "topic2"; private final Consumed consumed = Consumed.with(Serdes.Integer(), Serdes.String()); - private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer()); + private final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer(), 0L); private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String()); @Test @@ -64,19 +64,20 @@ public void testLeftJoin() { stream1 = builder.stream(topic1, consumed); stream2 = builder.stream(topic2, consumed); - joined = stream1.leftJoin(stream2, - MockValueJoiner.TOSTRING_JOINER, - JoinWindows.of(ofMillis(100)), - Joined.with(Serdes.Integer(), Serdes.String(), Serdes.String())); + joined = stream1.leftJoin( + stream2, + MockValueJoiner.TOSTRING_JOINER, + JoinWindows.of(ofMillis(100)), + Joined.with(Serdes.Integer(), Serdes.String(), Serdes.String())); joined.process(supplier); - final Collection> copartitionGroups = TopologyWrapper.getInternalTopologyBuilder(builder.build()).copartitionGroups(); + final Collection> copartitionGroups = + TopologyWrapper.getInternalTopologyBuilder(builder.build()).copartitionGroups(); assertEquals(1, copartitionGroups.size()); assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next()); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { - + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { final MockProcessor processor = supplier.theCapturedProcessor(); // push two items to the primary stream. the other window is empty @@ -84,56 +85,50 @@ public void testLeftJoin() { // w2 {} // --> w1 = { 0:X0, 1:X1 } // --> w2 = {} - for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i])); } - processor.checkAndClearProcessResult("0:X0+null", "1:X1+null"); + processor.checkAndClearProcessResult("0:X0+null (ts: 0)", "1:X1+null (ts: 0)"); // push two items to the other stream. this should produce two items. // w1 = { 0:X0, 1:X1 } // w2 {} // --> w1 = { 0:X0, 1:X1 } // --> w2 = { 0:Y0, 1:Y1 } - for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i])); } - - processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1"); + processor.checkAndClearProcessResult("0:X0+Y0 (ts: 0)", "1:X1+Y1 (ts: 0)"); // push three items to the primary stream. this should produce four items. // w1 = { 0:X0, 1:X1 } // w2 = { 0:Y0, 1:Y1 } // --> w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2 } // --> w2 = { 0:Y0, 1:Y1 } - for (int i = 0; i < 3; i++) { driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i])); } - processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1", "2:X2+null"); + processor.checkAndClearProcessResult("0:X0+Y0 (ts: 0)", "1:X1+Y1 (ts: 0)", "2:X2+null (ts: 0)"); // push all items to the other stream. this should produce 5 items // w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2 } // w2 = { 0:Y0, 1:Y1 } // --> w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2 } // --> w2 = { 0:Y0, 1:Y1, 0:YY0, 1:YY1, 2:YY2, 3:YY3} - for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey)); } - processor.checkAndClearProcessResult("0:X0+YY0", "0:X0+YY0", "1:X1+YY1", "1:X1+YY1", "2:X2+YY2"); + processor.checkAndClearProcessResult("0:X0+YY0 (ts: 0)", "0:X0+YY0 (ts: 0)", "1:X1+YY1 (ts: 0)", "1:X1+YY1 (ts: 0)", "2:X2+YY2 (ts: 0)"); // push all four items to the primary stream. this should produce six items. // w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2 } // w2 = { 0:Y0, 1:Y1, 0:YY0, 1:YY1, 2:YY2, 3:YY3} // --> w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2, 0:XX0, 1:XX1, 2:XX2, 3:XX3 } // --> w2 = { 0:Y0, 1:Y1, 0:YY0, 1:YY1, 2:YY2, 3:YY3} - for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey)); } - processor.checkAndClearProcessResult("0:XX0+Y0", "0:XX0+YY0", "1:XX1+Y1", "1:XX1+YY1", "2:XX2+YY2", "3:XX3+YY3"); + processor.checkAndClearProcessResult("0:XX0+Y0 (ts: 0)", "0:XX0+YY0 (ts: 0)", "1:XX1+Y1 (ts: 0)", "1:XX1+YY1 (ts: 0)", "2:XX2+YY2 (ts: 0)", "3:XX3+YY3 (ts: 0)"); } } @@ -141,7 +136,6 @@ public void testLeftJoin() { public void testWindowing() { final StreamsBuilder builder = new StreamsBuilder(); final int[] expectedKeys = new int[]{0, 1, 2, 3}; - long time = 0L; final KStream stream1; final KStream stream2; @@ -150,42 +144,42 @@ public void testWindowing() { stream1 = builder.stream(topic1, consumed); stream2 = builder.stream(topic2, consumed); - joined = stream1.leftJoin(stream2, - MockValueJoiner.TOSTRING_JOINER, - JoinWindows.of(ofMillis(100)), - Joined.with(Serdes.Integer(), Serdes.String(), Serdes.String())); + joined = stream1.leftJoin( + stream2, + MockValueJoiner.TOSTRING_JOINER, + JoinWindows.of(ofMillis(100)), + Joined.with(Serdes.Integer(), Serdes.String(), Serdes.String())); joined.process(supplier); - final Collection> copartitionGroups = TopologyWrapper.getInternalTopologyBuilder(builder.build()).copartitionGroups(); + final Collection> copartitionGroups = + TopologyWrapper.getInternalTopologyBuilder(builder.build()).copartitionGroups(); assertEquals(1, copartitionGroups.size()); assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next()); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, time)) { - + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { final MockProcessor processor = supplier.theCapturedProcessor(); + long time = 0L; // push two items to the primary stream. the other window is empty. this should produce two items // w1 = {} // w2 = {} // --> w1 = { 0:X0, 1:X1 } // --> w2 = {} - for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i], time)); } - processor.checkAndClearProcessResult("0:X0+null", "1:X1+null"); + processor.checkAndClearProcessResult("0:X0+null (ts: 0)", "1:X1+null (ts: 0)"); // push two items to the other stream. this should produce no items. // w1 = { 0:X0, 1:X1 } // w2 = {} // --> w1 = { 0:X0, 1:X1 } // --> w2 = { 0:Y0, 1:Y1 } - for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i], time)); } - processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1"); + processor.checkAndClearProcessResult("0:X0+Y0 (ts: 0)", "1:X1+Y1 (ts: 0)"); // clear logically time = 1000L; @@ -205,36 +199,35 @@ public void testWindowing() { // w2 = {} // --> w1 = {} // --> w2 = { 0:Y0, 1:Y1, 2:Y2, 3:Y3 } - time = 1000L + 100L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time)); } - processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+Y1", "2:XX2+Y2", "3:XX3+Y3"); + processor.checkAndClearProcessResult("0:XX0+Y0 (ts: 1100)", "1:XX1+Y1 (ts: 1100)", "2:XX2+Y2 (ts: 1100)", "3:XX3+Y3 (ts: 1100)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time)); } - processor.checkAndClearProcessResult("0:XX0+null", "1:XX1+Y1", "2:XX2+Y2", "3:XX3+Y3"); + processor.checkAndClearProcessResult("0:XX0+null (ts: 1101)", "1:XX1+Y1 (ts: 1101)", "2:XX2+Y2 (ts: 1101)", "3:XX3+Y3 (ts: 1101)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time)); } - processor.checkAndClearProcessResult("0:XX0+null", "1:XX1+null", "2:XX2+Y2", "3:XX3+Y3"); + processor.checkAndClearProcessResult("0:XX0+null (ts: 1102)", "1:XX1+null (ts: 1102)", "2:XX2+Y2 (ts: 1102)", "3:XX3+Y3 (ts: 1102)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time)); } - processor.checkAndClearProcessResult("0:XX0+null", "1:XX1+null", "2:XX2+null", "3:XX3+Y3"); + processor.checkAndClearProcessResult("0:XX0+null (ts: 1103)", "1:XX1+null (ts: 1103)", "2:XX2+null (ts: 1103)", "3:XX3+Y3 (ts: 1103)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time)); } - processor.checkAndClearProcessResult("0:XX0+null", "1:XX1+null", "2:XX2+null", "3:XX3+null"); + processor.checkAndClearProcessResult("0:XX0+null (ts: 1104)", "1:XX1+null (ts: 1104)", "2:XX2+null (ts: 1104)", "3:XX3+null (ts: 1104)"); // go back to the time before expiration @@ -242,31 +235,31 @@ public void testWindowing() { for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time)); } - processor.checkAndClearProcessResult("0:XX0+null", "1:XX1+null", "2:XX2+null", "3:XX3+null"); + processor.checkAndClearProcessResult("0:XX0+null (ts: 899)", "1:XX1+null (ts: 899)", "2:XX2+null (ts: 899)", "3:XX3+null (ts: 899)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time)); } - processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+null", "2:XX2+null", "3:XX3+null"); + processor.checkAndClearProcessResult("0:XX0+Y0 (ts: 900)", "1:XX1+null (ts: 900)", "2:XX2+null (ts: 900)", "3:XX3+null (ts: 900)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time)); } - processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+Y1", "2:XX2+null", "3:XX3+null"); + processor.checkAndClearProcessResult("0:XX0+Y0 (ts: 901)", "1:XX1+Y1 (ts: 901)", "2:XX2+null (ts: 901)", "3:XX3+null (ts: 901)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time)); } - processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+Y1", "2:XX2+Y2", "3:XX3+null"); + processor.checkAndClearProcessResult("0:XX0+Y0 (ts: 902)", "1:XX1+Y1 (ts: 902)", "2:XX2+Y2 (ts: 902)", "3:XX3+null (ts: 902)"); time += 1L; for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time)); } - processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+Y1", "2:XX2+Y2", "3:XX3+Y3"); + processor.checkAndClearProcessResult("0:XX0+Y0 (ts: 903)", "1:XX1+Y1 (ts: 903)", "2:XX2+Y2 (ts: 903)", "3:XX3+Y3 (ts: 903)"); } } } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableJoinTest.java index eb6536d37a6bc..ef36b6edb75b5 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableJoinTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableJoinTest.java @@ -47,11 +47,11 @@ import static org.junit.Assert.assertEquals; public class KStreamKTableJoinTest { - private final String streamTopic = "streamTopic"; private final String tableTopic = "tableTopic"; - private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer()); + private final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer(), 0L); private final int[] expectedKeys = {0, 1, 2, 3}; @@ -73,7 +73,7 @@ public void setUp() { stream.join(table, MockValueJoiner.TOSTRING_JOINER).process(supplier); final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String()); - driver = new TopologyTestDriver(builder.build(), props, 0L); + driver = new TopologyTestDriver(builder.build(), props); processor = supplier.theCapturedProcessor(); } @@ -97,14 +97,14 @@ private void pushToTable(final int messageCount, final String valuePrefix) { private void pushNullValueToTable() { for (int i = 0; i < 2; i++) { - driver.pipeInput(recordFactory.create(tableTopic, expectedKeys[i], (String) null)); + driver.pipeInput(recordFactory.create(tableTopic, expectedKeys[i], null)); } } @Test public void shouldRequireCopartitionedStreams() { - - final Collection> copartitionGroups = TopologyWrapper.getInternalTopologyBuilder(builder.build()).copartitionGroups(); + final Collection> copartitionGroups = + TopologyWrapper.getInternalTopologyBuilder(builder.build()).copartitionGroups(); assertEquals(1, copartitionGroups.size()); assertEquals(new HashSet<>(Arrays.asList(streamTopic, tableTopic)), copartitionGroups.iterator().next()); @@ -112,84 +112,67 @@ public void shouldRequireCopartitionedStreams() { @Test public void shouldNotJoinWithEmptyTableOnStreamUpdates() { - // push two items to the primary stream. the table is empty - pushToStream(2, "X"); processor.checkAndClearProcessResult(); } @Test public void shouldNotJoinOnTableUpdates() { - // push two items to the primary stream. the table is empty - pushToStream(2, "X"); processor.checkAndClearProcessResult(); // push two items to the table. this should not produce any item. - pushToTable(2, "Y"); processor.checkAndClearProcessResult(); // push all four items to the primary stream. this should produce two items. - pushToStream(4, "X"); - processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1"); + processor.checkAndClearProcessResult("0:X0+Y0 (ts: 0)", "1:X1+Y1 (ts: 0)"); // push all items to the table. this should not produce any item - pushToTable(4, "YY"); processor.checkAndClearProcessResult(); // push all four items to the primary stream. this should produce four items. - pushToStream(4, "X"); - processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3"); + processor.checkAndClearProcessResult("0:X0+YY0 (ts: 0)", "1:X1+YY1 (ts: 0)", "2:X2+YY2 (ts: 0)", "3:X3+YY3 (ts: 0)"); // push all items to the table. this should not produce any item - pushToTable(4, "YYY"); processor.checkAndClearProcessResult(); } @Test public void shouldJoinOnlyIfMatchFoundOnStreamUpdates() { - // push two items to the table. this should not produce any item. - pushToTable(2, "Y"); processor.checkAndClearProcessResult(); // push all four items to the primary stream. this should produce two items. - pushToStream(4, "X"); - processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1"); + processor.checkAndClearProcessResult("0:X0+Y0 (ts: 0)", "1:X1+Y1 (ts: 0)"); } @Test public void shouldClearTableEntryOnNullValueUpdates() { - // push all four items to the table. this should not produce any item. - pushToTable(4, "Y"); processor.checkAndClearProcessResult(); // push all four items to the primary stream. this should produce four items. - pushToStream(4, "X"); - processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1", "2:X2+Y2", "3:X3+Y3"); + processor.checkAndClearProcessResult("0:X0+Y0 (ts: 0)", "1:X1+Y1 (ts: 0)", "2:X2+Y2 (ts: 0)", "3:X3+Y3 (ts: 0)"); // push two items with null to the table as deletes. this should not produce any item. - pushNullValueToTable(); processor.checkAndClearProcessResult(); // push all four items to the primary stream. this should produce two items. - pushToStream(4, "XX"); - processor.checkAndClearProcessResult("2:XX2+Y2", "3:XX3+Y3"); + processor.checkAndClearProcessResult("2:XX2+Y2 (ts: 0)", "3:XX3+Y3 (ts: 0)"); } @Test @@ -205,7 +188,7 @@ public void shouldLogAndMeterWhenSkippingNullLeftKey() { @Test public void shouldLogAndMeterWhenSkippingNullLeftValue() { final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(); - driver.pipeInput(recordFactory.create(streamTopic, 1, (String) null)); + driver.pipeInput(recordFactory.create(streamTopic, 1, null)); LogCaptureAppender.unregister(appender); assertEquals(1.0, getMetricByName(driver.metrics(), "skipped-records-total", "stream-metrics").metricValue()); diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableLeftJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableLeftJoinTest.java index 8c57038b042ba..e04da173328c0 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableLeftJoinTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableLeftJoinTest.java @@ -43,11 +43,11 @@ import static org.junit.Assert.assertEquals; public class KStreamKTableLeftJoinTest { - final private String streamTopic = "streamTopic"; final private String tableTopic = "tableTopic"; - private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer()); + private final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer(), 0L); private TopologyTestDriver driver; private MockProcessor processor; @@ -58,7 +58,6 @@ public class KStreamKTableLeftJoinTest { public void setUp() { builder = new StreamsBuilder(); - final KStream stream; final KTable table; @@ -69,7 +68,7 @@ public void setUp() { stream.leftJoin(table, MockValueJoiner.TOSTRING_JOINER).process(supplier); final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String()); - driver = new TopologyTestDriver(builder.build(), props, 0L); + driver = new TopologyTestDriver(builder.build(), props); processor = supplier.theCapturedProcessor(); } @@ -93,14 +92,14 @@ private void pushToTable(final int messageCount, final String valuePrefix) { private void pushNullValueToTable(final int messageCount) { for (int i = 0; i < messageCount; i++) { - driver.pipeInput(recordFactory.create(tableTopic, expectedKeys[i], (String) null)); + driver.pipeInput(recordFactory.create(tableTopic, expectedKeys[i], null)); } } @Test public void shouldRequireCopartitionedStreams() { - - final Collection> copartitionGroups = TopologyWrapper.getInternalTopologyBuilder(builder.build()).copartitionGroups(); + final Collection> copartitionGroups = + TopologyWrapper.getInternalTopologyBuilder(builder.build()).copartitionGroups(); assertEquals(1, copartitionGroups.size()); assertEquals(new HashSet<>(Arrays.asList(streamTopic, tableTopic)), copartitionGroups.iterator().next()); @@ -108,84 +107,67 @@ public void shouldRequireCopartitionedStreams() { @Test public void shouldJoinWithEmptyTableOnStreamUpdates() { - // push two items to the primary stream. the table is empty - pushToStream(2, "X"); - processor.checkAndClearProcessResult("0:X0+null", "1:X1+null"); + processor.checkAndClearProcessResult("0:X0+null (ts: 0)", "1:X1+null (ts: 0)"); } @Test public void shouldNotJoinOnTableUpdates() { - // push two items to the primary stream. the table is empty - pushToStream(2, "X"); - processor.checkAndClearProcessResult("0:X0+null", "1:X1+null"); + processor.checkAndClearProcessResult("0:X0+null (ts: 0)", "1:X1+null (ts: 0)"); // push two items to the table. this should not produce any item. - pushToTable(2, "Y"); processor.checkAndClearProcessResult(); // push all four items to the primary stream. this should produce four items. - pushToStream(4, "X"); - processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1", "2:X2+null", "3:X3+null"); + processor.checkAndClearProcessResult("0:X0+Y0 (ts: 0)", "1:X1+Y1 (ts: 0)", "2:X2+null (ts: 0)", "3:X3+null (ts: 0)"); // push all items to the table. this should not produce any item - pushToTable(4, "YY"); processor.checkAndClearProcessResult(); // push all four items to the primary stream. this should produce four items. - pushToStream(4, "X"); - processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3"); + processor.checkAndClearProcessResult("0:X0+YY0 (ts: 0)", "1:X1+YY1 (ts: 0)", "2:X2+YY2 (ts: 0)", "3:X3+YY3 (ts: 0)"); // push all items to the table. this should not produce any item - pushToTable(4, "YYY"); processor.checkAndClearProcessResult(); } @Test public void shouldJoinRegardlessIfMatchFoundOnStreamUpdates() { - // push two items to the table. this should not produce any item. - pushToTable(2, "Y"); processor.checkAndClearProcessResult(); // push all four items to the primary stream. this should produce four items. - pushToStream(4, "X"); - processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1", "2:X2+null", "3:X3+null"); + processor.checkAndClearProcessResult("0:X0+Y0 (ts: 0)", "1:X1+Y1 (ts: 0)", "2:X2+null (ts: 0)", "3:X3+null (ts: 0)"); } @Test public void shouldClearTableEntryOnNullValueUpdates() { - // push all four items to the table. this should not produce any item. - pushToTable(4, "Y"); processor.checkAndClearProcessResult(); // push all four items to the primary stream. this should produce four items. - pushToStream(4, "X"); - processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1", "2:X2+Y2", "3:X3+Y3"); + processor.checkAndClearProcessResult("0:X0+Y0 (ts: 0)", "1:X1+Y1 (ts: 0)", "2:X2+Y2 (ts: 0)", "3:X3+Y3 (ts: 0)"); // push two items with null to the table as deletes. this should not produce any item. - pushNullValueToTable(2); processor.checkAndClearProcessResult(); // push all four items to the primary stream. this should produce four items. - pushToStream(4, "XX"); - processor.checkAndClearProcessResult("0:XX0+null", "1:XX1+null", "2:XX2+Y2", "3:XX3+Y3"); + processor.checkAndClearProcessResult("0:XX0+null (ts: 0)", "1:XX1+null (ts: 0)", "2:XX2+Y2 (ts: 0)", "3:XX3+Y3 (ts: 0)"); } } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapTest.java index 10a27b9f2bbcd..7ea570d5dc4f6 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapTest.java @@ -34,14 +34,14 @@ import static org.junit.Assert.assertEquals; public class KStreamMapTest { - - private String topicName = "topic"; - private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer()); + private final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer(), 0L); private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String()); @Test public void testMap() { final StreamsBuilder builder = new StreamsBuilder(); + final String topicName = "topic"; final int[] expectedKeys = new int[]{0, 1, 2, 3}; final MockProcessorSupplier supplier = new MockProcessorSupplier<>(); @@ -54,10 +54,8 @@ public void testMap() { } } + final String[] expected = new String[]{"V0:0 (ts: 0)", "V1:1 (ts: 0)", "V2:2 (ts: 0)", "V3:3 (ts: 0)"}; assertEquals(4, supplier.theCapturedProcessor().processed.size()); - - final String[] expected = new String[]{"V0:0", "V1:1", "V2:2", "V3:3"}; - for (int i = 0; i < expected.length; i++) { assertEquals(expected[i], supplier.theCapturedProcessor().processed.get(i)); } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapValuesTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapValuesTest.java index 67891be18a59f..ab9e68e64a5f0 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapValuesTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapValuesTest.java @@ -19,11 +19,10 @@ import org.apache.kafka.common.serialization.IntegerSerializer; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.serialization.StringSerializer; -import org.apache.kafka.streams.kstream.Consumed; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.TopologyTestDriver; +import org.apache.kafka.streams.kstream.Consumed; import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.kstream.ValueMapperWithKey; import org.apache.kafka.streams.test.ConsumerRecordFactory; import org.apache.kafka.test.MockProcessorSupplier; @@ -35,35 +34,27 @@ import static org.junit.Assert.assertArrayEquals; public class KStreamMapValuesTest { - - private String topicName = "topic"; + private final String topicName = "topic"; private final MockProcessorSupplier supplier = new MockProcessorSupplier<>(); - private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer()); + private final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer(), 0L); private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String()); @Test public void testFlatMapValues() { final StreamsBuilder builder = new StreamsBuilder(); - final ValueMapper mapper = - new ValueMapper() { - @Override - public Integer apply(final CharSequence value) { - return value.length(); - } - }; - final int[] expectedKeys = {1, 10, 100, 1000}; final KStream stream = builder.stream(topicName, Consumed.with(Serdes.Integer(), Serdes.String())); - stream.mapValues(mapper).process(supplier); + stream.mapValues(CharSequence::length).process(supplier); try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topicName, expectedKey, Integer.toString(expectedKey))); } } - final String[] expected = {"1:1", "10:2", "100:3", "1000:4"}; + final String[] expected = {"1:1 (ts: 0)", "10:2 (ts: 0)", "100:3 (ts: 0)", "1000:4 (ts: 0)"}; assertArrayEquals(expected, supplier.theCapturedProcessor().processed.toArray()); } @@ -72,13 +63,7 @@ public Integer apply(final CharSequence value) { public void testMapValuesWithKeys() { final StreamsBuilder builder = new StreamsBuilder(); - final ValueMapperWithKey mapper = - new ValueMapperWithKey() { - @Override - public Integer apply(final Integer readOnlyKey, final CharSequence value) { - return value.length() + readOnlyKey; - } - }; + final ValueMapperWithKey mapper = (readOnlyKey, value) -> value.length() + readOnlyKey; final int[] expectedKeys = {1, 10, 100, 1000}; @@ -90,7 +75,7 @@ public Integer apply(final Integer readOnlyKey, final CharSequence value) { driver.pipeInput(recordFactory.create(topicName, expectedKey, Integer.toString(expectedKey))); } } - final String[] expected = {"1:2", "10:12", "100:103", "1000:1004"}; + final String[] expected = {"1:2 (ts: 0)", "10:12 (ts: 0)", "100:103 (ts: 0)", "1000:1004 (ts: 0)"}; assertArrayEquals(expected, supplier.theCapturedProcessor().processed.toArray()); } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSelectKeyTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSelectKeyTest.java index 5f804d45cf468..ea20022c052c4 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSelectKeyTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSelectKeyTest.java @@ -19,12 +19,10 @@ import org.apache.kafka.common.serialization.IntegerSerializer; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.serialization.StringSerializer; -import org.apache.kafka.streams.kstream.Consumed; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.TopologyTestDriver; -import org.apache.kafka.streams.kstream.ForeachAction; +import org.apache.kafka.streams.kstream.Consumed; import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.KeyValueMapper; import org.apache.kafka.streams.test.ConsumerRecordFactory; import org.apache.kafka.test.MockProcessorSupplier; import org.apache.kafka.test.StreamsTestUtils; @@ -37,10 +35,9 @@ import static org.junit.Assert.assertEquals; public class KStreamSelectKeyTest { - - private String topicName = "topic_key_select"; - - private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(topicName, new StringSerializer(), new IntegerSerializer()); + private final String topicName = "topic_key_select"; + private final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(topicName, new StringSerializer(), new IntegerSerializer(), 0L); private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.Integer()); @Test @@ -52,22 +49,13 @@ public void testSelectKey() { keyMap.put(2, "TWO"); keyMap.put(3, "THREE"); - - final KeyValueMapper selector = new KeyValueMapper() { - @Override - public String apply(final Object key, final Number value) { - return keyMap.get(value); - } - }; - - final String[] expected = new String[]{"ONE:1", "TWO:2", "THREE:3"}; + final String[] expected = new String[]{"ONE:1 (ts: 0)", "TWO:2 (ts: 0)", "THREE:3 (ts: 0)"}; final int[] expectedValues = new int[]{1, 2, 3}; - final KStream stream = builder.stream(topicName, Consumed.with(Serdes.String(), Serdes.Integer())); - + final KStream stream = + builder.stream(topicName, Consumed.with(Serdes.String(), Serdes.Integer())); final MockProcessorSupplier supplier = new MockProcessorSupplier<>(); - - stream.selectKey(selector).process(supplier); + stream.selectKey((key, value) -> keyMap.get(value)).process(supplier); try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { for (final int expectedValue : expectedValues) { @@ -76,7 +64,6 @@ public String apply(final Object key, final Number value) { } assertEquals(3, supplier.theCapturedProcessor().processed.size()); - for (int i = 0; i < expected.length; i++) { assertEquals(expected[i], supplier.theCapturedProcessor().processed.get(i)); } @@ -85,13 +72,8 @@ public String apply(final Object key, final Number value) { @Test public void testTypeVariance() { - final ForeachAction consume = new ForeachAction() { - @Override - public void apply(final Number key, final Object value) {} - }; - new StreamsBuilder() .stream("empty") - .foreach(consume); + .foreach((key, value) -> { }); } } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformTest.java index 0fb9a5da41ef1..de60fb902f747 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformTest.java @@ -16,34 +16,32 @@ */ package org.apache.kafka.streams.kstream.internals; -import java.time.Duration; import org.apache.kafka.common.serialization.IntegerSerializer; import org.apache.kafka.common.serialization.Serdes; -import org.apache.kafka.streams.kstream.Consumed; import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.TopologyTestDriver; +import org.apache.kafka.streams.kstream.Consumed; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.Transformer; import org.apache.kafka.streams.kstream.TransformerSupplier; import org.apache.kafka.streams.processor.ProcessorContext; import org.apache.kafka.streams.processor.PunctuationType; -import org.apache.kafka.streams.processor.Punctuator; import org.apache.kafka.streams.test.ConsumerRecordFactory; import org.apache.kafka.test.MockProcessorSupplier; import org.apache.kafka.test.StreamsTestUtils; import org.junit.Rule; import org.junit.Test; +import java.time.Duration; import java.util.Properties; import static org.junit.Assert.assertEquals; public class KStreamTransformTest { - - private String topicName = "topic"; - - private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new IntegerSerializer()); + private final String topicName = "topic"; + private final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(new IntegerSerializer(), new IntegerSerializer(), 0L); private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.Integer()); @SuppressWarnings("deprecation") @@ -54,26 +52,22 @@ public class KStreamTransformTest { public void testTransform() { final StreamsBuilder builder = new StreamsBuilder(); - final TransformerSupplier> transformerSupplier = new TransformerSupplier>() { - public Transformer> get() { - return new Transformer>() { + final TransformerSupplier> transformerSupplier = + () -> new Transformer>() { + private int total = 0; - private int total = 0; + @Override + public void init(final ProcessorContext context) {} - @Override - public void init(final ProcessorContext context) {} + @Override + public KeyValue transform(final Number key, final Number value) { + total += value.intValue(); + return KeyValue.pair(key.intValue() * 2, total); + } - @Override - public KeyValue transform(final Number key, final Number value) { - total += value.intValue(); - return KeyValue.pair(key.intValue() * 2, total); - } - - @Override - public void close() {} - }; - } - }; + @Override + public void close() {} + }; final int[] expectedKeys = {1, 10, 100, 1000}; @@ -94,7 +88,7 @@ public void close() {} //String[] expected = {"2:10", "20:110", "200:1110", "2000:11110", "-1:2", "-1:3"}; - final String[] expected = {"2:10", "20:110", "200:1110", "2000:11110"}; + final String[] expected = {"2:10 (ts: 0)", "20:110 (ts: 0)", "200:1110 (ts: 0)", "2000:11110 (ts: 0)"}; for (int i = 0; i < expected.length; i++) { assertEquals(expected[i], processor.theCapturedProcessor().processed.get(i)); @@ -105,34 +99,27 @@ public void close() {} public void testTransformWithNewDriverAndPunctuator() { final StreamsBuilder builder = new StreamsBuilder(); - final TransformerSupplier> transformerSupplier = new TransformerSupplier>() { - public Transformer> get() { - return new Transformer>() { - - private int total = 0; - - @Override - public void init(final ProcessorContext context) { - context.schedule(Duration.ofMillis(1), PunctuationType.WALL_CLOCK_TIME, new Punctuator() { - @Override - public void punctuate(final long timestamp) { - context.forward(-1, (int) timestamp); - } - }); - } - - @Override - public KeyValue transform(final Number key, final Number value) { - total += value.intValue(); - return KeyValue.pair(key.intValue() * 2, total); - } - - @Override - public void close() {} - }; - } - }; + final TransformerSupplier> transformerSupplier = + () -> new Transformer>() { + private int total = 0; + + @Override + public void init(final ProcessorContext context) { + context.schedule( + Duration.ofMillis(1), + PunctuationType.WALL_CLOCK_TIME, + timestamp -> context.forward(-1, (int) timestamp)); + } + + @Override + public KeyValue transform(final Number key, final Number value) { + total += value.intValue(); + return KeyValue.pair(key.intValue() * 2, total); + } + @Override + public void close() {} + }; final int[] expectedKeys = {1, 10, 100, 1000}; @@ -153,7 +140,7 @@ public void close() {} assertEquals(6, processor.theCapturedProcessor().processed.size()); - final String[] expected = {"2:10", "20:110", "200:1110", "2000:11110", "-1:2", "-1:3"}; + final String[] expected = {"2:10 (ts: 0)", "20:110 (ts: 0)", "200:1110 (ts: 0)", "2000:11110 (ts: 0)", "-1:2 (ts: 2)", "-1:3 (ts: 3)"}; for (int i = 0; i < expected.length; i++) { assertEquals(expected[i], processor.theCapturedProcessor().processed.get(i)); diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValuesTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValuesTest.java index 94d06eb2dcd37..a3ce8303acc1a 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValuesTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValuesTest.java @@ -47,10 +47,10 @@ @RunWith(EasyMockRunner.class) public class KStreamTransformValuesTest { - - private String topicName = "topic"; + private final String topicName = "topic"; private final MockProcessorSupplier supplier = new MockProcessorSupplier<>(); - private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new IntegerSerializer()); + private final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(new IntegerSerializer(), new IntegerSerializer(), 0L); private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.Integer()); @Mock(MockType.NICE) private ProcessorContext context; @@ -60,27 +60,20 @@ public void testTransform() { final StreamsBuilder builder = new StreamsBuilder(); final ValueTransformerSupplier valueTransformerSupplier = - new ValueTransformerSupplier() { - public ValueTransformer get() { - return new ValueTransformer() { - - private int total = 0; - - @Override - public void init(final ProcessorContext context) { - } - - @Override - public Integer transform(final Number value) { - total += value.intValue(); - return total; - } - - @Override - public void close() { - } - }; + () -> new ValueTransformer() { + private int total = 0; + + @Override + public void init(final ProcessorContext context) {} + + @Override + public Integer transform(final Number value) { + total += value.intValue(); + return total; } + + @Override + public void close() {} }; final int[] expectedKeys = {1, 10, 100, 1000}; @@ -89,12 +82,12 @@ public void close() { stream = builder.stream(topicName, Consumed.with(Serdes.Integer(), Serdes.Integer())); stream.transformValues(valueTransformerSupplier).process(supplier); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topicName, expectedKey, expectedKey * 10, 0L)); } } - final String[] expected = {"1:10", "10:110", "100:1110", "1000:11110"}; + final String[] expected = {"1:10 (ts: 0)", "10:110 (ts: 0)", "100:1110 (ts: 0)", "1000:11110 (ts: 0)"}; assertArrayEquals(expected, supplier.theCapturedProcessor().processed.toArray()); } @@ -104,27 +97,21 @@ public void testTransformWithKey() { final StreamsBuilder builder = new StreamsBuilder(); final ValueTransformerWithKeySupplier valueTransformerSupplier = - new ValueTransformerWithKeySupplier() { - public ValueTransformerWithKey get() { - return new ValueTransformerWithKey() { - private int total = 0; - @Override - public void init(final ProcessorContext context) { - - } - @Override - public Integer transform(final Integer readOnlyKey, final Number value) { - total += value.intValue() + readOnlyKey; - return total; - } - - @Override - public void close() { - - } - }; - } - }; + () -> new ValueTransformerWithKey() { + private int total = 0; + + @Override + public void init(final ProcessorContext context) {} + + @Override + public Integer transform(final Integer readOnlyKey, final Number value) { + total += value.intValue() + readOnlyKey; + return total; + } + + @Override + public void close() {} + }; final int[] expectedKeys = {1, 10, 100, 1000}; @@ -132,12 +119,12 @@ public void close() { stream = builder.stream(topicName, Consumed.with(Serdes.Integer(), Serdes.Integer())); stream.transformValues(valueTransformerSupplier).process(supplier); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topicName, expectedKey, expectedKey * 10, 0L)); } } - final String[] expected = {"1:11", "10:121", "100:1221", "1000:12221"}; + final String[] expected = {"1:11 (ts: 0)", "10:121 (ts: 0)", "100:1221 (ts: 0)", "1000:12221 (ts: 0)"}; assertArrayEquals(expected, supplier.theCapturedProcessor().processed.toArray()); } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java index 068062beb211b..d8e7cafbbcafb 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java @@ -61,8 +61,8 @@ import static org.junit.Assert.assertEquals; public class KStreamWindowAggregateTest { - - private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer()); + private final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer()); private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String()); @Test @@ -79,7 +79,7 @@ public void testAggBasic() { final MockProcessorSupplier, String> supplier = new MockProcessorSupplier<>(); table2.toStream().process(supplier); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { driver.pipeInput(recordFactory.create(topic1, "A", "1", 0L)); driver.pipeInput(recordFactory.create(topic1, "B", "2", 1L)); driver.pipeInput(recordFactory.create(topic1, "C", "3", 2L)); @@ -100,23 +100,23 @@ public void testAggBasic() { assertEquals( asList( - "[A@0/10]:0+1", - "[B@0/10]:0+2", - "[C@0/10]:0+3", - "[D@0/10]:0+4", - "[A@0/10]:0+1+1", - - "[A@0/10]:0+1+1+1", "[A@5/15]:0+1", - "[B@0/10]:0+2+2", "[B@5/15]:0+2", - "[D@0/10]:0+4+4", "[D@5/15]:0+4", - "[B@0/10]:0+2+2+2", "[B@5/15]:0+2+2", - "[C@0/10]:0+3+3", "[C@5/15]:0+3", - - "[A@5/15]:0+1+1", "[A@10/20]:0+1", - "[B@5/15]:0+2+2+2", "[B@10/20]:0+2", - "[D@5/15]:0+4+4", "[D@10/20]:0+4", - "[B@5/15]:0+2+2+2+2", "[B@10/20]:0+2+2", - "[C@5/15]:0+3+3", "[C@10/20]:0+3" + "[A@0/10]:0+1 (ts: 0)", + "[B@0/10]:0+2 (ts: 1)", + "[C@0/10]:0+3 (ts: 2)", + "[D@0/10]:0+4 (ts: 3)", + "[A@0/10]:0+1+1 (ts: 4)", + + "[A@0/10]:0+1+1+1 (ts: 5)", "[A@5/15]:0+1 (ts: 5)", + "[B@0/10]:0+2+2 (ts: 6)", "[B@5/15]:0+2 (ts: 6)", + "[D@0/10]:0+4+4 (ts: 7)", "[D@5/15]:0+4 (ts: 7)", + "[B@0/10]:0+2+2+2 (ts: 8)", "[B@5/15]:0+2+2 (ts: 8)", + "[C@0/10]:0+3+3 (ts: 9)", "[C@5/15]:0+3 (ts: 9)", + + "[A@5/15]:0+1+1 (ts: 10)", "[A@10/20]:0+1 (ts: 10)", + "[B@5/15]:0+2+2+2 (ts: 11)", "[B@10/20]:0+2 (ts: 11)", + "[D@5/15]:0+4+4 (ts: 12)", "[D@10/20]:0+4 (ts: 12)", + "[B@5/15]:0+2+2+2+2 (ts: 13)", "[B@10/20]:0+2+2 (ts: 13)", + "[C@5/15]:0+3+3 (ts: 14)", "[C@10/20]:0+3 (ts: 14)" ), supplier.theCapturedProcessor().processed ); @@ -142,13 +142,11 @@ public void testJoin() { .groupByKey(Grouped.with(Serdes.String(), Serdes.String())) .windowedBy(TimeWindows.of(ofMillis(10)).advanceBy(ofMillis(5))) .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as("topic2-Canonized").withValueSerde(Serdes.String())); - table2.toStream().process(supplier); - table1.join(table2, (p1, p2) -> p1 + "%" + p2).toStream().process(supplier); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { driver.pipeInput(recordFactory.create(topic1, "A", "1", 0L)); driver.pipeInput(recordFactory.create(topic1, "B", "2", 1L)); driver.pipeInput(recordFactory.create(topic1, "C", "3", 2L)); @@ -158,11 +156,11 @@ public void testJoin() { final List, String>> processors = supplier.capturedProcessors(3); processors.get(0).checkAndClearProcessResult( - "[A@0/10]:0+1", - "[B@0/10]:0+2", - "[C@0/10]:0+3", - "[D@0/10]:0+4", - "[A@0/10]:0+1+1" + "[A@0/10]:0+1 (ts: 0)", + "[B@0/10]:0+2 (ts: 1)", + "[C@0/10]:0+3 (ts: 2)", + "[D@0/10]:0+4 (ts: 3)", + "[A@0/10]:0+1+1 (ts: 4)" ); processors.get(1).checkAndClearProcessResult(); processors.get(2).checkAndClearProcessResult(); @@ -174,11 +172,11 @@ public void testJoin() { driver.pipeInput(recordFactory.create(topic1, "C", "3", 9L)); processors.get(0).checkAndClearProcessResult( - "[A@0/10]:0+1+1+1", "[A@5/15]:0+1", - "[B@0/10]:0+2+2", "[B@5/15]:0+2", - "[D@0/10]:0+4+4", "[D@5/15]:0+4", - "[B@0/10]:0+2+2+2", "[B@5/15]:0+2+2", - "[C@0/10]:0+3+3", "[C@5/15]:0+3" + "[A@0/10]:0+1+1+1 (ts: 5)", "[A@5/15]:0+1 (ts: 5)", + "[B@0/10]:0+2+2 (ts: 6)", "[B@5/15]:0+2 (ts: 6)", + "[D@0/10]:0+4+4 (ts: 7)", "[D@5/15]:0+4 (ts: 7)", + "[B@0/10]:0+2+2+2 (ts: 8)", "[B@5/15]:0+2+2 (ts: 8)", + "[C@0/10]:0+3+3 (ts: 9)", "[C@5/15]:0+3 (ts: 9)" ); processors.get(1).checkAndClearProcessResult(); processors.get(2).checkAndClearProcessResult(); @@ -191,18 +189,18 @@ public void testJoin() { processors.get(0).checkAndClearProcessResult(); processors.get(1).checkAndClearProcessResult( - "[A@0/10]:0+a", - "[B@0/10]:0+b", - "[C@0/10]:0+c", - "[D@0/10]:0+d", - "[A@0/10]:0+a+a" + "[A@0/10]:0+a (ts: 0)", + "[B@0/10]:0+b (ts: 1)", + "[C@0/10]:0+c (ts: 2)", + "[D@0/10]:0+d (ts: 3)", + "[A@0/10]:0+a+a (ts: 4)" ); processors.get(2).checkAndClearProcessResult( - "[A@0/10]:0+1+1+1%0+a", - "[B@0/10]:0+2+2+2%0+b", - "[C@0/10]:0+3+3%0+c", - "[D@0/10]:0+4+4%0+d", - "[A@0/10]:0+1+1+1%0+a+a"); + "[A@0/10]:0+1+1+1%0+a (ts: 0)", + "[B@0/10]:0+2+2+2%0+b (ts: 1)", + "[C@0/10]:0+3+3%0+c (ts: 2)", + "[D@0/10]:0+4+4%0+d (ts: 3)", + "[A@0/10]:0+1+1+1%0+a+a (ts: 4)"); driver.pipeInput(recordFactory.create(topic2, "A", "a", 5L)); driver.pipeInput(recordFactory.create(topic2, "B", "b", 6L)); @@ -212,18 +210,18 @@ public void testJoin() { processors.get(0).checkAndClearProcessResult(); processors.get(1).checkAndClearProcessResult( - "[A@0/10]:0+a+a+a", "[A@5/15]:0+a", - "[B@0/10]:0+b+b", "[B@5/15]:0+b", - "[D@0/10]:0+d+d", "[D@5/15]:0+d", - "[B@0/10]:0+b+b+b", "[B@5/15]:0+b+b", - "[C@0/10]:0+c+c", "[C@5/15]:0+c" + "[A@0/10]:0+a+a+a (ts: 5)", "[A@5/15]:0+a (ts: 5)", + "[B@0/10]:0+b+b (ts: 6)", "[B@5/15]:0+b (ts: 6)", + "[D@0/10]:0+d+d (ts: 7)", "[D@5/15]:0+d (ts: 7)", + "[B@0/10]:0+b+b+b (ts: 8)", "[B@5/15]:0+b+b (ts: 8)", + "[C@0/10]:0+c+c (ts: 9)", "[C@5/15]:0+c (ts: 9)" ); processors.get(2).checkAndClearProcessResult( - "[A@0/10]:0+1+1+1%0+a+a+a", "[A@5/15]:0+1%0+a", - "[B@0/10]:0+2+2+2%0+b+b", "[B@5/15]:0+2+2%0+b", - "[D@0/10]:0+4+4%0+d+d", "[D@5/15]:0+4%0+d", - "[B@0/10]:0+2+2+2%0+b+b+b", "[B@5/15]:0+2+2%0+b+b", - "[C@0/10]:0+3+3%0+c+c", "[C@5/15]:0+3%0+c" + "[A@0/10]:0+1+1+1%0+a+a+a (ts: 5)", "[A@5/15]:0+1%0+a (ts: 5)", + "[B@0/10]:0+2+2+2%0+b+b (ts: 6)", "[B@5/15]:0+2+2%0+b (ts: 6)", + "[D@0/10]:0+4+4%0+d+d (ts: 7)", "[D@5/15]:0+4%0+d (ts: 7)", + "[B@0/10]:0+2+2+2%0+b+b+b (ts: 8)", "[B@5/15]:0+2+2%0+b+b (ts: 8)", + "[C@0/10]:0+3+3%0+c+c (ts: 9)", "[C@5/15]:0+3%0+c (ts: 9)" ); } } @@ -244,7 +242,7 @@ public void shouldLogAndMeterWhenSkippingNullKey() { ); final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { driver.pipeInput(recordFactory.create(topic, null, "1")); LogCaptureAppender.unregister(appender); @@ -273,7 +271,7 @@ public void shouldLogAndMeterWhenSkippingExpiredWindow() { LogCaptureAppender.setClassLoggerToDebug(KStreamWindowAggregate.class); final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { driver.pipeInput(recordFactory.create(topic, "k", "100", 100L)); driver.pipeInput(recordFactory.create(topic, "k", "0", 0L)); driver.pipeInput(recordFactory.create(topic, "k", "1", 1L)); @@ -328,7 +326,7 @@ public void shouldLogAndMeterWhenSkippingExpiredWindowByGrace() { LogCaptureAppender.setClassLoggerToDebug(KStreamWindowAggregate.class); final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { driver.pipeInput(recordFactory.create(topic, "k", "100", 200L)); driver.pipeInput(recordFactory.create(topic, "k", "0", 100L)); driver.pipeInput(recordFactory.create(topic, "k", "1", 101L)); @@ -370,10 +368,8 @@ private void assertLatenessMetrics(final TopologyTestDriver driver, mkEntry("processor-node-id", "KSTREAM-AGGREGATE-0000000001") ) ); - assertThat(driver.metrics().get(dropMetric).metricValue(), dropTotal); - final MetricName dropRate = new MetricName( "late-record-drop-rate", "stream-processor-node-metrics", @@ -384,7 +380,6 @@ private void assertLatenessMetrics(final TopologyTestDriver driver, mkEntry("processor-node-id", "KSTREAM-AGGREGATE-0000000001") ) ); - assertThat(driver.metrics().get(dropRate).metricValue(), not(0.0)); final MetricName latenessMaxMetric = new MetricName( diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableAggregateTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableAggregateTest.java index 2e4f318cc4112..87360a18581b8 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableAggregateTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableAggregateTest.java @@ -47,7 +47,6 @@ @SuppressWarnings("deprecation") public class KTableAggregateTest { - private final Serde stringSerde = Serdes.String(); private final Consumed consumed = Consumed.with(stringSerde, stringSerde); private final Grouped stringSerialzied = Grouped.with(stringSerde, stringSerde); @@ -105,14 +104,14 @@ public void testAggBasic() { assertEquals( asList( - "A:0+1", - "B:0+2", - "A:0+1-1+3", - "B:0+2-2+4", - "C:0+5", - "D:0+6", - "B:0+2-2+4-4+7", - "C:0+5-5+8"), + "A:0+1 (ts: 0)", + "B:0+2 (ts: 0)", + "A:0+1-1+3 (ts: 0)", + "B:0+2-2+4 (ts: 0)", + "C:0+5 (ts: 0)", + "D:0+6 (ts: 0)", + "B:0+2-2+4-4+7 (ts: 0)", + "C:0+5-5+8 (ts: 0)"), supplier.theCapturedProcessor().processed); } @@ -142,10 +141,9 @@ public void testAggCoalesced() { driver.process(topic1, "A", "4"); driver.flushState(); - assertEquals(Collections.singletonList("A:0+4"), supplier.theCapturedProcessor().processed); + assertEquals(Collections.singletonList("A:0+4 (ts: 0)"), supplier.theCapturedProcessor().processed); } - @Test public void testAggRepartition() { final StreamsBuilder builder = new StreamsBuilder(); @@ -195,14 +193,14 @@ public void testAggRepartition() { assertEquals( asList( - "1:0+1", - "1:0+1-1", - "1:0+1-1+1", - "2:0+2", + "1:0+1 (ts: 0)", + "1:0+1-1 (ts: 0)", + "1:0+1-1+1 (ts: 0)", + "2:0+2 (ts: 0)", //noop - "2:0+2-2", "4:0+4", + "2:0+2-2 (ts: 0)", "4:0+4 (ts: 0)", //noop - "4:0+4-4", "7:0+7"), + "4:0+4-4 (ts: 0)", "7:0+7 (ts: 0)"), supplier.theCapturedProcessor().processed); } @@ -225,11 +223,11 @@ private void testCountHelper(final StreamsBuilder builder, assertEquals( asList( - "green:1", - "green:2", - "green:1", "blue:1", - "yellow:1", - "green:2"), + "green:1 (ts: 0)", + "green:2 (ts: 0)", + "green:1 (ts: 0)", "blue:1 (ts: 0)", + "yellow:1 (ts: 0)", + "green:2 (ts: 0)"), supplier.theCapturedProcessor().processed); } @@ -289,9 +287,9 @@ public void testCountCoalesced() { assertEquals( asList( - "blue:1", - "yellow:1", - "green:2"), + "blue:1 (ts: 0)", + "yellow:1 (ts: 0)", + "green:2 (ts: 0)"), proc.processed); } @@ -332,10 +330,10 @@ public void testRemoveOldBeforeAddNew() { assertEquals( asList( - "1:1", - "1:12", - "1:2", - "1:2"), + "1:1 (ts: 0)", + "1:12 (ts: 0)", + "1:2 (ts: 0)", + "1:2 (ts: 0)"), proc.processed); } @@ -379,5 +377,4 @@ public void shouldForwardToCorrectProcessorNodeWhenMultiCacheEvictions() { driver.process("tableOne", "1", "5"); assertEquals(Long.valueOf(4L), reduceResults.get("2")); } - } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java index 1c4ba46c65164..5904cd328f9b1 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java @@ -47,9 +47,9 @@ @SuppressWarnings("unchecked") public class KTableFilterTest { - private final Consumed consumed = Consumed.with(Serdes.String(), Serdes.Integer()); - private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new IntegerSerializer()); + private final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(new StringSerializer(), new IntegerSerializer(), 0L); private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.Integer()); @Before @@ -79,14 +79,13 @@ private void doTestKTable(final StreamsBuilder builder, final List> processors = supplier.capturedProcessors(2); - processors.get(0).checkAndClearProcessResult("A:null", "B:2", "C:null", "D:4", "A:null", "B:null"); - processors.get(1).checkAndClearProcessResult("A:1", "B:null", "C:3", "D:null", "A:null", "B:null"); + processors.get(0).checkAndClearProcessResult("A:null (ts: 0)", "B:2 (ts: 0)", "C:null (ts: 0)", "D:4 (ts: 0)", "A:null (ts: 0)", "B:null (ts: 0)"); + processors.get(1).checkAndClearProcessResult("A:1 (ts: 0)", "B:null (ts: 0)", "C:3 (ts: 0)", "D:null (ts: 0)", "A:null (ts: 0)", "B:null (ts: 0)"); } @Test public void shouldPassThroughWithoutMaterialization() { final StreamsBuilder builder = new StreamsBuilder(); - final String topic1 = "topic1"; final KTable table1 = builder.table(topic1, consumed); @@ -103,7 +102,6 @@ public void shouldPassThroughWithoutMaterialization() { @Test public void shouldPassThroughOnMaterialization() { final StreamsBuilder builder = new StreamsBuilder(); - final String topic1 = "topic1"; final KTable table1 = builder.table(topic1, consumed); @@ -132,7 +130,6 @@ private void doTestValueGetter(final StreamsBuilder builder, topologyBuilder.connectProcessorAndStateStores(table3.name, getterSupplier3.storeNames()); try (final TopologyTestDriverWrapper driver = new TopologyTestDriverWrapper(topology, props)) { - final KTableValueGetter getter2 = getterSupplier2.get(); final KTableValueGetter getter3 = getterSupplier3.get(); @@ -188,7 +185,6 @@ private void doTestValueGetter(final StreamsBuilder builder, @Test public void shouldGetValuesOnMaterialization() { final StreamsBuilder builder = new StreamsBuilder(); - final String topic1 = "topic1"; final KTableImpl table1 = @@ -225,25 +221,25 @@ private void doTestNotSendingOldValue(final StreamsBuilder builder, final List> processors = supplier.capturedProcessors(2); - processors.get(0).checkAndClearProcessResult("A:(1<-null)", "B:(1<-null)", "C:(1<-null)"); - processors.get(1).checkAndClearProcessResult("A:(null<-null)", "B:(null<-null)", "C:(null<-null)"); + processors.get(0).checkAndClearProcessResult("A:(1<-null) (ts: 0)", "B:(1<-null) (ts: 0)", "C:(1<-null) (ts: 0)"); + processors.get(1).checkAndClearProcessResult("A:(null<-null) (ts: 0)", "B:(null<-null) (ts: 0)", "C:(null<-null) (ts: 0)"); driver.pipeInput(recordFactory.create(topic1, "A", 2)); driver.pipeInput(recordFactory.create(topic1, "B", 2)); - processors.get(0).checkAndClearProcessResult("A:(2<-null)", "B:(2<-null)"); - processors.get(1).checkAndClearProcessResult("A:(2<-null)", "B:(2<-null)"); + processors.get(0).checkAndClearProcessResult("A:(2<-null) (ts: 0)", "B:(2<-null) (ts: 0)"); + processors.get(1).checkAndClearProcessResult("A:(2<-null) (ts: 0)", "B:(2<-null) (ts: 0)"); driver.pipeInput(recordFactory.create(topic1, "A", 3)); - processors.get(0).checkAndClearProcessResult("A:(3<-null)"); - processors.get(1).checkAndClearProcessResult("A:(null<-null)"); + processors.get(0).checkAndClearProcessResult("A:(3<-null) (ts: 0)"); + processors.get(1).checkAndClearProcessResult("A:(null<-null) (ts: 0)"); driver.pipeInput(recordFactory.create(topic1, "A", null)); driver.pipeInput(recordFactory.create(topic1, "B", null)); - processors.get(0).checkAndClearProcessResult("A:(null<-null)", "B:(null<-null)"); - processors.get(1).checkAndClearProcessResult("A:(null<-null)", "B:(null<-null)"); + processors.get(0).checkAndClearProcessResult("A:(null<-null) (ts: 0)", "B:(null<-null) (ts: 0)"); + processors.get(1).checkAndClearProcessResult("A:(null<-null) (ts: 0)", "B:(null<-null) (ts: 0)"); } } @@ -251,7 +247,6 @@ private void doTestNotSendingOldValue(final StreamsBuilder builder, @Test public void shouldNotSendOldValuesWithoutMaterialization() { final StreamsBuilder builder = new StreamsBuilder(); - final String topic1 = "topic1"; final KTableImpl table1 = @@ -264,7 +259,6 @@ public void shouldNotSendOldValuesWithoutMaterialization() { @Test public void shouldNotSendOldValuesOnMaterialization() { final StreamsBuilder builder = new StreamsBuilder(); - final String topic1 = "topic1"; final KTableImpl table1 = @@ -288,39 +282,37 @@ private void doTestSendingOldValue(final StreamsBuilder builder, topology.addProcessor("proc2", supplier, table2.name); try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) { - driver.pipeInput(recordFactory.create(topic1, "A", 1)); driver.pipeInput(recordFactory.create(topic1, "B", 1)); driver.pipeInput(recordFactory.create(topic1, "C", 1)); final List> processors = supplier.capturedProcessors(2); - processors.get(0).checkAndClearProcessResult("A:(1<-null)", "B:(1<-null)", "C:(1<-null)"); + processors.get(0).checkAndClearProcessResult("A:(1<-null) (ts: 0)", "B:(1<-null) (ts: 0)", "C:(1<-null) (ts: 0)"); processors.get(1).checkEmptyAndClearProcessResult(); driver.pipeInput(recordFactory.create(topic1, "A", 2)); driver.pipeInput(recordFactory.create(topic1, "B", 2)); - processors.get(0).checkAndClearProcessResult("A:(2<-1)", "B:(2<-1)"); - processors.get(1).checkAndClearProcessResult("A:(2<-null)", "B:(2<-null)"); + processors.get(0).checkAndClearProcessResult("A:(2<-1) (ts: 0)", "B:(2<-1) (ts: 0)"); + processors.get(1).checkAndClearProcessResult("A:(2<-null) (ts: 0)", "B:(2<-null) (ts: 0)"); driver.pipeInput(recordFactory.create(topic1, "A", 3)); - processors.get(0).checkAndClearProcessResult("A:(3<-2)"); - processors.get(1).checkAndClearProcessResult("A:(null<-2)"); + processors.get(0).checkAndClearProcessResult("A:(3<-2) (ts: 0)"); + processors.get(1).checkAndClearProcessResult("A:(null<-2) (ts: 0)"); driver.pipeInput(recordFactory.create(topic1, "A", null)); driver.pipeInput(recordFactory.create(topic1, "B", null)); - processors.get(0).checkAndClearProcessResult("A:(null<-3)", "B:(null<-2)"); - processors.get(1).checkAndClearProcessResult("B:(null<-2)"); + processors.get(0).checkAndClearProcessResult("A:(null<-3) (ts: 0)", "B:(null<-2) (ts: 0)"); + processors.get(1).checkAndClearProcessResult("B:(null<-2) (ts: 0)"); } } @Test public void shouldSendOldValuesWhenEnabledWithoutMaterialization() { final StreamsBuilder builder = new StreamsBuilder(); - final String topic1 = "topic1"; final KTableImpl table1 = @@ -334,7 +326,6 @@ public void shouldSendOldValuesWhenEnabledWithoutMaterialization() { @Test public void shouldSendOldValuesWhenEnabledOnMaterialization() { final StreamsBuilder builder = new StreamsBuilder(); - final String topic1 = "topic1"; final KTableImpl table1 = @@ -355,7 +346,8 @@ private void doTestSkipNullOnMaterialization(final StreamsBuilder builder, topology.addProcessor("proc1", supplier, table1.name); topology.addProcessor("proc2", supplier, table2.name); - final ConsumerRecordFactory stringRecordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer()); + final ConsumerRecordFactory stringRecordFactory = + new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L); try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) { driver.pipeInput(stringRecordFactory.create(topic1, "A", "reject")); @@ -364,7 +356,7 @@ private void doTestSkipNullOnMaterialization(final StreamsBuilder builder, } final List> processors = supplier.capturedProcessors(2); - processors.get(0).checkAndClearProcessResult("A:(reject<-null)", "B:(reject<-null)", "C:(reject<-null)"); + processors.get(0).checkAndClearProcessResult("A:(reject<-null) (ts: 0)", "B:(reject<-null) (ts: 0)", "C:(reject<-null) (ts: 0)"); processors.get(1).checkEmptyAndClearProcessResult(); } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java index c8aef07543ca5..81ba31aa6ddca 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java @@ -65,13 +65,12 @@ @SuppressWarnings("unchecked") public class KTableImplTest { - private final Consumed stringConsumed = Consumed.with(Serdes.String(), Serdes.String()); private final Consumed consumed = Consumed.with(Serdes.String(), Serdes.String()); private final Produced produced = Produced.with(Serdes.String(), Serdes.String()); private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String()); private final ConsumerRecordFactory recordFactory = - new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer()); + new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L); private final Serde mySerde = new Serdes.StringSerde(); private KTable table; @@ -94,16 +93,13 @@ public void testKTable() { table1.toStream().process(supplier); final KTable table2 = table1.mapValues(Integer::new); - table2.toStream().process(supplier); final KTable table3 = table2.filter((key, value) -> (value % 2) == 0); - table3.toStream().process(supplier); - table1.toStream().to(topic2, produced); - final KTable table4 = builder.table(topic2, consumed); + final KTable table4 = builder.table(topic2, consumed); table4.toStream().process(supplier); try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { @@ -114,10 +110,10 @@ public void testKTable() { } final List> processors = supplier.capturedProcessors(4); - assertEquals(asList("A:01", "B:02", "C:03", "D:04"), processors.get(0).processed); - assertEquals(asList("A:1", "B:2", "C:3", "D:4"), processors.get(1).processed); - assertEquals(asList("A:null", "B:2", "C:null", "D:4"), processors.get(2).processed); - assertEquals(asList("A:01", "B:02", "C:03", "D:04"), processors.get(3).processed); + assertEquals(asList("A:01 (ts: 0)", "B:02 (ts: 0)", "C:03 (ts: 0)", "D:04 (ts: 0)"), processors.get(0).processed); + assertEquals(asList("A:1 (ts: 0)", "B:2 (ts: 0)", "C:3 (ts: 0)", "D:4 (ts: 0)"), processors.get(1).processed); + assertEquals(asList("A:null (ts: 0)", "B:2 (ts: 0)", "C:null (ts: 0)", "D:4 (ts: 0)"), processors.get(2).processed); + assertEquals(asList("A:01 (ts: 0)", "B:02 (ts: 0)", "C:03 (ts: 0)", "D:04 (ts: 0)"), processors.get(3).processed); } @Test @@ -246,11 +242,10 @@ public void close() {} @Test public void testStateStoreLazyEval() { + final StreamsBuilder builder = new StreamsBuilder(); final String topic1 = "topic1"; final String topic2 = "topic2"; - final StreamsBuilder builder = new StreamsBuilder(); - final KTableImpl table1 = (KTableImpl) builder.table(topic1, consumed); builder.table(topic2, consumed); @@ -266,11 +261,10 @@ public void testStateStoreLazyEval() { @Test public void testStateStore() { + final StreamsBuilder builder = new StreamsBuilder(); final String topic1 = "topic1"; final String topic2 = "topic2"; - final StreamsBuilder builder = new StreamsBuilder(); - final KTableImpl table1 = (KTableImpl) builder.table(topic1, consumed); final KTableImpl table2 = @@ -301,11 +295,10 @@ private void assertTopologyContainsProcessor(final Topology topology, final Stri @Test public void shouldCreateSourceAndSinkNodesForRepartitioningTopic() throws Exception { + final StreamsBuilder builder = new StreamsBuilder(); final String topic1 = "topic1"; final String storeName1 = "storeName1"; - final StreamsBuilder builder = new StreamsBuilder(); - final KTableImpl table1 = (KTableImpl) builder.table( topic1, diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableInnerJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableInnerJoinTest.java index 5d1f5c3d8c8c8..f02d5ddc3e353 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableInnerJoinTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableInnerJoinTest.java @@ -56,8 +56,10 @@ public class KTableKTableInnerJoinTest { final private String output = "output"; private final Consumed consumed = Consumed.with(Serdes.Integer(), Serdes.String()); - private final Materialized> materialized = Materialized.with(Serdes.Integer(), Serdes.String()); - private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(Serdes.Integer().serializer(), Serdes.String().serializer()); + private final Materialized> materialized = + Materialized.with(Serdes.Integer(), Serdes.String()); + private final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(Serdes.Integer().serializer(), Serdes.String().serializer(), 0L); private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String()); @Test @@ -161,8 +163,7 @@ private void doTestNotSendingOldValues(final StreamsBuilder builder, final MockProcessorSupplier supplier, final KTable joined) { - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { - + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { final MockProcessor proc = supplier.theCapturedProcessor(); assertFalse(((KTableImpl) table1).sendingOldValueEnabled()); @@ -179,50 +180,48 @@ private void doTestNotSendingOldValues(final StreamsBuilder builder, for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i])); } - proc.checkAndClearProcessResult("0:(X0+Y0<-null)", "1:(X1+Y1<-null)"); + proc.checkAndClearProcessResult("0:(X0+Y0<-null) (ts: 0)", "1:(X1+Y1<-null) (ts: 0)"); // push all four items to the primary stream. this should produce two items. for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey)); } - proc.checkAndClearProcessResult("0:(XX0+Y0<-null)", "1:(XX1+Y1<-null)"); + proc.checkAndClearProcessResult("0:(XX0+Y0<-null) (ts: 0)", "1:(XX1+Y1<-null) (ts: 0)"); // push all items to the other stream. this should produce four items. for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey)); } - proc.checkAndClearProcessResult("0:(XX0+YY0<-null)", "1:(XX1+YY1<-null)", "2:(XX2+YY2<-null)", "3:(XX3+YY3<-null)"); + proc.checkAndClearProcessResult("0:(XX0+YY0<-null) (ts: 0)", "1:(XX1+YY1<-null) (ts: 0)", "2:(XX2+YY2<-null) (ts: 0)", "3:(XX3+YY3<-null) (ts: 0)"); // push all four items to the primary stream. this should produce four items. - for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "X" + expectedKey)); } - proc.checkAndClearProcessResult("0:(X0+YY0<-null)", "1:(X1+YY1<-null)", "2:(X2+YY2<-null)", "3:(X3+YY3<-null)"); + proc.checkAndClearProcessResult("0:(X0+YY0<-null) (ts: 0)", "1:(X1+YY1<-null) (ts: 0)", "2:(X2+YY2<-null) (ts: 0)", "3:(X3+YY3<-null) (ts: 0)"); // push two items with null to the other stream as deletes. this should produce two item. for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], null)); } - proc.checkAndClearProcessResult("0:(null<-null)", "1:(null<-null)"); + proc.checkAndClearProcessResult("0:(null<-null) (ts: 0)", "1:(null<-null) (ts: 0)"); // push all four items to the primary stream. this should produce two items. for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey)); } - proc.checkAndClearProcessResult("2:(XX2+YY2<-null)", "3:(XX3+YY3<-null)"); + proc.checkAndClearProcessResult("2:(XX2+YY2<-null) (ts: 0)", "3:(XX3+YY3<-null) (ts: 0)"); } } private void doTestJoin(final StreamsBuilder builder, final int[] expectedKeys) { - - final Collection> copartitionGroups = TopologyWrapper.getInternalTopologyBuilder(builder.build()).copartitionGroups(); + final Collection> copartitionGroups = + TopologyWrapper.getInternalTopologyBuilder(builder.build()).copartitionGroups(); assertEquals(1, copartitionGroups.size()); assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next()); try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { - // push two items to the primary stream. the other table is empty for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i])); @@ -288,7 +287,6 @@ private void doTestJoin(final StreamsBuilder builder, final int[] expectedKeys) driver.pipeInput(recordFactory.create(topic1, null, "XX" + 1)); assertNull(driver.readOutput(output)); } - } private void assertOutputKeyValue(final TopologyTestDriver driver, diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoinTest.java index be43e5ea8f2c4..3ec03f40a6894 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoinTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoinTest.java @@ -58,13 +58,13 @@ import static org.junit.Assert.assertTrue; public class KTableKTableLeftJoinTest { - final private String topic1 = "topic1"; final private String topic2 = "topic2"; final private String output = "output"; private final Consumed consumed = Consumed.with(Serdes.Integer(), Serdes.String()); - private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(Serdes.Integer().serializer(), Serdes.String().serializer()); + private final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(Serdes.Integer().serializer(), Serdes.String().serializer(), 0L); private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String()); @Test @@ -78,13 +78,13 @@ public void testJoin() { final KTable joined = table1.leftJoin(table2, MockValueJoiner.TOSTRING_JOINER); joined.toStream().to(output); - final Collection> copartitionGroups = TopologyWrapper.getInternalTopologyBuilder(builder.build()).copartitionGroups(); + final Collection> copartitionGroups = + TopologyWrapper.getInternalTopologyBuilder(builder.build()).copartitionGroups(); assertEquals(1, copartitionGroups.size()); assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next()); try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { - // push two items to the primary stream. the other table is empty for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i])); @@ -174,7 +174,6 @@ public void testNotSendingOldValue() { final Topology topology = builder.build().addProcessor("proc", supplier, ((KTableImpl) joined).name); try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) { - final MockProcessor proc = supplier.theCapturedProcessor(); assertTrue(((KTableImpl) table1).sendingOldValueEnabled()); @@ -185,43 +184,43 @@ public void testNotSendingOldValue() { for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i])); } - proc.checkAndClearProcessResult("0:(X0+null<-null)", "1:(X1+null<-null)"); + proc.checkAndClearProcessResult("0:(X0+null<-null) (ts: 0)", "1:(X1+null<-null) (ts: 0)"); // push two items to the other stream. this should produce two items. for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i])); } - proc.checkAndClearProcessResult("0:(X0+Y0<-null)", "1:(X1+Y1<-null)"); + proc.checkAndClearProcessResult("0:(X0+Y0<-null) (ts: 0)", "1:(X1+Y1<-null) (ts: 0)"); // push all four items to the primary stream. this should produce four items. for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "X" + expectedKey)); } - proc.checkAndClearProcessResult("0:(X0+Y0<-null)", "1:(X1+Y1<-null)", "2:(X2+null<-null)", "3:(X3+null<-null)"); + proc.checkAndClearProcessResult("0:(X0+Y0<-null) (ts: 0)", "1:(X1+Y1<-null) (ts: 0)", "2:(X2+null<-null) (ts: 0)", "3:(X3+null<-null) (ts: 0)"); // push all items to the other stream. this should produce four items. for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey)); } - proc.checkAndClearProcessResult("0:(X0+YY0<-null)", "1:(X1+YY1<-null)", "2:(X2+YY2<-null)", "3:(X3+YY3<-null)"); + proc.checkAndClearProcessResult("0:(X0+YY0<-null) (ts: 0)", "1:(X1+YY1<-null) (ts: 0)", "2:(X2+YY2<-null) (ts: 0)", "3:(X3+YY3<-null) (ts: 0)"); // push all four items to the primary stream. this should produce four items. for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "X" + expectedKey)); } - proc.checkAndClearProcessResult("0:(X0+YY0<-null)", "1:(X1+YY1<-null)", "2:(X2+YY2<-null)", "3:(X3+YY3<-null)"); + proc.checkAndClearProcessResult("0:(X0+YY0<-null) (ts: 0)", "1:(X1+YY1<-null) (ts: 0)", "2:(X2+YY2<-null) (ts: 0)", "3:(X3+YY3<-null) (ts: 0)"); // push two items with null to the other stream as deletes. this should produce two item. for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], null)); } - proc.checkAndClearProcessResult("0:(X0+null<-null)", "1:(X1+null<-null)"); + proc.checkAndClearProcessResult("0:(X0+null<-null) (ts: 0)", "1:(X1+null<-null) (ts: 0)"); // push all four items to the primary stream. this should produce four items. for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey)); } - proc.checkAndClearProcessResult("0:(XX0+null<-null)", "1:(XX1+null<-null)", "2:(XX2+YY2<-null)", "3:(XX3+YY3<-null)"); + proc.checkAndClearProcessResult("0:(XX0+null<-null) (ts: 0)", "1:(XX1+null<-null) (ts: 0)", "2:(XX2+YY2<-null) (ts: 0)", "3:(XX3+YY3<-null) (ts: 0)"); } } @@ -246,7 +245,6 @@ public void testSendingOldValue() { final Topology topology = builder.build().addProcessor("proc", supplier, ((KTableImpl) joined).name); try (final TopologyTestDriver driver = new TopologyTestDriverWrapper(topology, props)) { - final MockProcessor proc = supplier.theCapturedProcessor(); assertTrue(((KTableImpl) table1).sendingOldValueEnabled()); @@ -257,43 +255,43 @@ public void testSendingOldValue() { for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i])); } - proc.checkAndClearProcessResult("0:(X0+null<-null)", "1:(X1+null<-null)"); + proc.checkAndClearProcessResult("0:(X0+null<-null) (ts: 0)", "1:(X1+null<-null) (ts: 0)"); // push two items to the other stream. this should produce two items. for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i])); } - proc.checkAndClearProcessResult("0:(X0+Y0<-X0+null)", "1:(X1+Y1<-X1+null)"); + proc.checkAndClearProcessResult("0:(X0+Y0<-X0+null) (ts: 0)", "1:(X1+Y1<-X1+null) (ts: 0)"); // push all four items to the primary stream. this should produce four items. for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "X" + expectedKey)); } - proc.checkAndClearProcessResult("0:(X0+Y0<-X0+Y0)", "1:(X1+Y1<-X1+Y1)", "2:(X2+null<-null)", "3:(X3+null<-null)"); + proc.checkAndClearProcessResult("0:(X0+Y0<-X0+Y0) (ts: 0)", "1:(X1+Y1<-X1+Y1) (ts: 0)", "2:(X2+null<-null) (ts: 0)", "3:(X3+null<-null) (ts: 0)"); // push all items to the other stream. this should produce four items. for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey)); } - proc.checkAndClearProcessResult("0:(X0+YY0<-X0+Y0)", "1:(X1+YY1<-X1+Y1)", "2:(X2+YY2<-X2+null)", "3:(X3+YY3<-X3+null)"); + proc.checkAndClearProcessResult("0:(X0+YY0<-X0+Y0) (ts: 0)", "1:(X1+YY1<-X1+Y1) (ts: 0)", "2:(X2+YY2<-X2+null) (ts: 0)", "3:(X3+YY3<-X3+null) (ts: 0)"); // push all four items to the primary stream. this should produce four items. for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "X" + expectedKey)); } - proc.checkAndClearProcessResult("0:(X0+YY0<-X0+YY0)", "1:(X1+YY1<-X1+YY1)", "2:(X2+YY2<-X2+YY2)", "3:(X3+YY3<-X3+YY3)"); + proc.checkAndClearProcessResult("0:(X0+YY0<-X0+YY0) (ts: 0)", "1:(X1+YY1<-X1+YY1) (ts: 0)", "2:(X2+YY2<-X2+YY2) (ts: 0)", "3:(X3+YY3<-X3+YY3) (ts: 0)"); // push two items with null to the other stream as deletes. this should produce two item. for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], null)); } - proc.checkAndClearProcessResult("0:(X0+null<-X0+YY0)", "1:(X1+null<-X1+YY1)"); + proc.checkAndClearProcessResult("0:(X0+null<-X0+YY0) (ts: 0)", "1:(X1+null<-X1+YY1) (ts: 0)"); // push all four items to the primary stream. this should produce four items. for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey)); } - proc.checkAndClearProcessResult("0:(XX0+null<-X0+null)", "1:(XX1+null<-X1+null)", "2:(XX2+YY2<-X2+YY2)", "3:(XX3+YY3<-X3+YY3)"); + proc.checkAndClearProcessResult("0:(XX0+null<-X0+null) (ts: 0)", "1:(XX1+null<-X1+null) (ts: 0)", "2:(XX2+YY2<-X2+YY2) (ts: 0)", "3:(XX3+YY3<-X3+YY3) (ts: 0)"); } } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableOuterJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableOuterJoinTest.java index 092b5a14b8f42..93a055d875f29 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableOuterJoinTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableOuterJoinTest.java @@ -49,13 +49,13 @@ import static org.junit.Assert.assertTrue; public class KTableKTableOuterJoinTest { - final private String topic1 = "topic1"; final private String topic2 = "topic2"; final private String output = "output"; private final Consumed consumed = Consumed.with(Serdes.Integer(), Serdes.String()); - private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(Serdes.Integer().serializer(), Serdes.String().serializer()); + private final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(Serdes.Integer().serializer(), Serdes.String().serializer(), 0L); private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String()); @Test @@ -73,13 +73,13 @@ public void testJoin() { joined = table1.outerJoin(table2, MockValueJoiner.TOSTRING_JOINER); joined.toStream().to(output); - final Collection> copartitionGroups = TopologyWrapper.getInternalTopologyBuilder(builder.build()).copartitionGroups(); + final Collection> copartitionGroups = + TopologyWrapper.getInternalTopologyBuilder(builder.build()).copartitionGroups(); assertEquals(1, copartitionGroups.size()); assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next()); try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { - // push two items to the primary stream. the other table is empty for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i])); @@ -175,8 +175,8 @@ public void testNotSendingOldValue() { supplier = new MockProcessorSupplier<>(); final Topology topology = builder.build().addProcessor("proc", supplier, ((KTableImpl) joined).name); - try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) { + try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) { final MockProcessor proc = supplier.theCapturedProcessor(); assertTrue(((KTableImpl) table1).sendingOldValueEnabled()); @@ -187,49 +187,49 @@ public void testNotSendingOldValue() { for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i])); } - proc.checkAndClearProcessResult("0:(X0+null<-null)", "1:(X1+null<-null)"); + proc.checkAndClearProcessResult("0:(X0+null<-null) (ts: 0)", "1:(X1+null<-null) (ts: 0)"); // push two items to the other stream. this should produce two items. for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i])); } - proc.checkAndClearProcessResult("0:(X0+Y0<-null)", "1:(X1+Y1<-null)"); + proc.checkAndClearProcessResult("0:(X0+Y0<-null) (ts: 0)", "1:(X1+Y1<-null) (ts: 0)"); // push all four items to the primary stream. this should produce four items. for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "X" + expectedKey)); } - proc.checkAndClearProcessResult("0:(X0+Y0<-null)", "1:(X1+Y1<-null)", "2:(X2+null<-null)", "3:(X3+null<-null)"); + proc.checkAndClearProcessResult("0:(X0+Y0<-null) (ts: 0)", "1:(X1+Y1<-null) (ts: 0)", "2:(X2+null<-null) (ts: 0)", "3:(X3+null<-null) (ts: 0)"); // push all items to the other stream. this should produce four items. for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey)); } - proc.checkAndClearProcessResult("0:(X0+YY0<-null)", "1:(X1+YY1<-null)", "2:(X2+YY2<-null)", "3:(X3+YY3<-null)"); + proc.checkAndClearProcessResult("0:(X0+YY0<-null) (ts: 0)", "1:(X1+YY1<-null) (ts: 0)", "2:(X2+YY2<-null) (ts: 0)", "3:(X3+YY3<-null) (ts: 0)"); // push all four items to the primary stream. this should produce four items. for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "X" + expectedKey)); } - proc.checkAndClearProcessResult("0:(X0+YY0<-null)", "1:(X1+YY1<-null)", "2:(X2+YY2<-null)", "3:(X3+YY3<-null)"); + proc.checkAndClearProcessResult("0:(X0+YY0<-null) (ts: 0)", "1:(X1+YY1<-null) (ts: 0)", "2:(X2+YY2<-null) (ts: 0)", "3:(X3+YY3<-null) (ts: 0)"); // push two items with null to the other stream as deletes. this should produce two item. for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], null)); } - proc.checkAndClearProcessResult("0:(X0+null<-null)", "1:(X1+null<-null)"); + proc.checkAndClearProcessResult("0:(X0+null<-null) (ts: 0)", "1:(X1+null<-null) (ts: 0)"); // push all four items to the primary stream. this should produce four items. for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey)); } - proc.checkAndClearProcessResult("0:(XX0+null<-null)", "1:(XX1+null<-null)", "2:(XX2+YY2<-null)", "3:(XX3+YY3<-null)"); + proc.checkAndClearProcessResult("0:(XX0+null<-null) (ts: 0)", "1:(XX1+null<-null) (ts: 0)", "2:(XX2+YY2<-null) (ts: 0)", "3:(XX3+YY3<-null) (ts: 0)"); // push middle two items to the primary stream with null. this should produce two items. for (int i = 1; i < 3; i++) { driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], null)); } - proc.checkAndClearProcessResult("1:(null<-null)", "2:(null+YY2<-null)"); + proc.checkAndClearProcessResult("1:(null<-null) (ts: 0)", "2:(null+YY2<-null) (ts: 0)"); } } @@ -254,7 +254,6 @@ public void testSendingOldValue() { final Topology topology = builder.build().addProcessor("proc", supplier, ((KTableImpl) joined).name); try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) { - final MockProcessor proc = supplier.theCapturedProcessor(); assertTrue(((KTableImpl) table1).sendingOldValueEnabled()); @@ -265,49 +264,49 @@ public void testSendingOldValue() { for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i])); } - proc.checkAndClearProcessResult("0:(X0+null<-null)", "1:(X1+null<-null)"); + proc.checkAndClearProcessResult("0:(X0+null<-null) (ts: 0)", "1:(X1+null<-null) (ts: 0)"); // push two items to the other stream. this should produce two items. for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i])); } - proc.checkAndClearProcessResult("0:(X0+Y0<-X0+null)", "1:(X1+Y1<-X1+null)"); + proc.checkAndClearProcessResult("0:(X0+Y0<-X0+null) (ts: 0)", "1:(X1+Y1<-X1+null) (ts: 0)"); // push all four items to the primary stream. this should produce four items. for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "X" + expectedKey)); } - proc.checkAndClearProcessResult("0:(X0+Y0<-X0+Y0)", "1:(X1+Y1<-X1+Y1)", "2:(X2+null<-null)", "3:(X3+null<-null)"); + proc.checkAndClearProcessResult("0:(X0+Y0<-X0+Y0) (ts: 0)", "1:(X1+Y1<-X1+Y1) (ts: 0)", "2:(X2+null<-null) (ts: 0)", "3:(X3+null<-null) (ts: 0)"); // push all items to the other stream. this should produce four items. for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey)); } - proc.checkAndClearProcessResult("0:(X0+YY0<-X0+Y0)", "1:(X1+YY1<-X1+Y1)", "2:(X2+YY2<-X2+null)", "3:(X3+YY3<-X3+null)"); + proc.checkAndClearProcessResult("0:(X0+YY0<-X0+Y0) (ts: 0)", "1:(X1+YY1<-X1+Y1) (ts: 0)", "2:(X2+YY2<-X2+null) (ts: 0)", "3:(X3+YY3<-X3+null) (ts: 0)"); // push all four items to the primary stream. this should produce four items. for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "X" + expectedKey)); } - proc.checkAndClearProcessResult("0:(X0+YY0<-X0+YY0)", "1:(X1+YY1<-X1+YY1)", "2:(X2+YY2<-X2+YY2)", "3:(X3+YY3<-X3+YY3)"); + proc.checkAndClearProcessResult("0:(X0+YY0<-X0+YY0) (ts: 0)", "1:(X1+YY1<-X1+YY1) (ts: 0)", "2:(X2+YY2<-X2+YY2) (ts: 0)", "3:(X3+YY3<-X3+YY3) (ts: 0)"); // push two items with null to the other stream as deletes. this should produce two item. for (int i = 0; i < 2; i++) { driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], null)); } - proc.checkAndClearProcessResult("0:(X0+null<-X0+YY0)", "1:(X1+null<-X1+YY1)"); + proc.checkAndClearProcessResult("0:(X0+null<-X0+YY0) (ts: 0)", "1:(X1+null<-X1+YY1) (ts: 0)"); // push all four items to the primary stream. this should produce four items. for (final int expectedKey : expectedKeys) { driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey)); } - proc.checkAndClearProcessResult("0:(XX0+null<-X0+null)", "1:(XX1+null<-X1+null)", "2:(XX2+YY2<-X2+YY2)", "3:(XX3+YY3<-X3+YY3)"); + proc.checkAndClearProcessResult("0:(XX0+null<-X0+null) (ts: 0)", "1:(XX1+null<-X1+null) (ts: 0)", "2:(XX2+YY2<-X2+YY2) (ts: 0)", "3:(XX3+YY3<-X3+YY3) (ts: 0)"); // push middle two items to the primary stream with null. this should produce two items. for (int i = 1; i < 3; i++) { driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], null)); } - proc.checkAndClearProcessResult("1:(null<-XX1+null)", "2:(null+YY2<-XX2+YY2)"); + proc.checkAndClearProcessResult("1:(null<-XX1+null) (ts: 0)", "2:(null+YY2<-XX2+YY2) (ts: 0)"); } } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableMapKeysTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableMapKeysTest.java index ce7de16501c01..5a008d5e5051c 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableMapKeysTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableMapKeysTest.java @@ -16,7 +16,6 @@ */ package org.apache.kafka.streams.kstream.internals; - import org.apache.kafka.common.serialization.IntegerSerializer; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.serialization.StringSerializer; @@ -25,7 +24,6 @@ import org.apache.kafka.streams.kstream.Consumed; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.KTable; -import org.apache.kafka.streams.kstream.KeyValueMapper; import org.apache.kafka.streams.test.ConsumerRecordFactory; import org.apache.kafka.test.MockProcessorSupplier; import org.apache.kafka.test.StreamsTestUtils; @@ -38,14 +36,13 @@ import static org.junit.Assert.assertEquals; public class KTableMapKeysTest { - - private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer()); + private final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer(), 0L); private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String()); @Test public void testMapKeysConvertingToStream() { final StreamsBuilder builder = new StreamsBuilder(); - final String topic1 = "topic_map_keys"; final KTable table1 = builder.table(topic1, Consumed.with(Serdes.Integer(), Serdes.String())); @@ -55,21 +52,13 @@ public void testMapKeysConvertingToStream() { keyMap.put(2, "TWO"); keyMap.put(3, "THREE"); - final KeyValueMapper keyMapper = new KeyValueMapper() { - @Override - public String apply(final Integer key, final String value) { - return keyMap.get(key); - } - }; + final KStream convertedStream = table1.toStream((key, value) -> keyMap.get(key)); - final KStream convertedStream = table1.toStream(keyMapper); - - final String[] expected = new String[]{"ONE:V_ONE", "TWO:V_TWO", "THREE:V_THREE"}; + final String[] expected = new String[]{"ONE:V_ONE (ts: 0)", "TWO:V_TWO (ts: 0)", "THREE:V_THREE (ts: 0)"}; final int[] originalKeys = new int[]{1, 2, 3}; final String[] values = new String[]{"V_ONE", "V_TWO", "V_THREE"}; final MockProcessorSupplier supplier = new MockProcessorSupplier<>(); - convertedStream.process(supplier); try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { @@ -79,7 +68,6 @@ public String apply(final Integer key, final String value) { } assertEquals(3, supplier.theCapturedProcessor().processed.size()); - for (int i = 0; i < expected.length; i++) { assertEquals(expected[i], supplier.theCapturedProcessor().processed.get(i)); } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableMapValuesTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableMapValuesTest.java index b16391571c9cf..b472b2d4897f5 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableMapValuesTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableMapValuesTest.java @@ -45,10 +45,9 @@ @SuppressWarnings("unchecked") public class KTableMapValuesTest { - private final Consumed consumed = Consumed.with(Serdes.String(), Serdes.String()); private final ConsumerRecordFactory recordFactory = - new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer()); + new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L); private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String()); private void doTestKTable(final StreamsBuilder builder, @@ -59,14 +58,13 @@ private void doTestKTable(final StreamsBuilder builder, driver.pipeInput(recordFactory.create(topic1, "B", "2")); driver.pipeInput(recordFactory.create(topic1, "C", "3")); driver.pipeInput(recordFactory.create(topic1, "D", "4")); - assertEquals(asList("A:1", "B:2", "C:3", "D:4"), supplier.theCapturedProcessor().processed); + assertEquals(asList("A:1 (ts: 0)", "B:2 (ts: 0)", "C:3 (ts: 0)", "D:4 (ts: 0)"), supplier.theCapturedProcessor().processed); } } @Test public void testKTable() { final StreamsBuilder builder = new StreamsBuilder(); - final String topic1 = "topic1"; final KTable table1 = builder.table(topic1, consumed); @@ -81,7 +79,6 @@ public void testKTable() { @Test public void testQueryableKTable() { final StreamsBuilder builder = new StreamsBuilder(); - final String topic1 = "topic1"; final KTable table1 = builder.table(topic1, consumed); @@ -166,7 +163,6 @@ private void doTestValueGetter(final StreamsBuilder builder, @Test public void testQueryableValueGetter() { final StreamsBuilder builder = new StreamsBuilder(); - final String topic1 = "topic1"; final String storeName2 = "store2"; final String storeName3 = "store3"; @@ -196,7 +192,6 @@ public void testQueryableValueGetter() { @Test public void testNotSendingOldValue() { final StreamsBuilder builder = new StreamsBuilder(); - final String topic1 = "topic1"; final KTableImpl table1 = @@ -205,11 +200,9 @@ public void testNotSendingOldValue() { (KTableImpl) table1.mapValues(Integer::new); final MockProcessorSupplier supplier = new MockProcessorSupplier<>(); - final Topology topology = builder.build().addProcessor("proc", supplier, table2.name); try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) { - final MockProcessor proc = supplier.theCapturedProcessor(); assertFalse(table1.sendingOldValueEnabled()); @@ -218,43 +211,35 @@ public void testNotSendingOldValue() { driver.pipeInput(recordFactory.create(topic1, "A", "01")); driver.pipeInput(recordFactory.create(topic1, "B", "01")); driver.pipeInput(recordFactory.create(topic1, "C", "01")); - - proc.checkAndClearProcessResult("A:(1<-null)", "B:(1<-null)", "C:(1<-null)"); + proc.checkAndClearProcessResult("A:(1<-null) (ts: 0)", "B:(1<-null) (ts: 0)", "C:(1<-null) (ts: 0)"); driver.pipeInput(recordFactory.create(topic1, "A", "02")); driver.pipeInput(recordFactory.create(topic1, "B", "02")); - - proc.checkAndClearProcessResult("A:(2<-null)", "B:(2<-null)"); + proc.checkAndClearProcessResult("A:(2<-null) (ts: 0)", "B:(2<-null) (ts: 0)"); driver.pipeInput(recordFactory.create(topic1, "A", "03")); - - proc.checkAndClearProcessResult("A:(3<-null)"); + proc.checkAndClearProcessResult("A:(3<-null) (ts: 0)"); driver.pipeInput(recordFactory.create(topic1, "A", (String) null)); - - proc.checkAndClearProcessResult("A:(null<-null)"); + proc.checkAndClearProcessResult("A:(null<-null) (ts: 0)"); } } @Test public void testSendingOldValue() { final StreamsBuilder builder = new StreamsBuilder(); - final String topic1 = "topic1"; final KTableImpl table1 = (KTableImpl) builder.table(topic1, consumed); final KTableImpl table2 = (KTableImpl) table1.mapValues(Integer::new); - table2.enableSendingOldValues(); final MockProcessorSupplier supplier = new MockProcessorSupplier<>(); - builder.build().addProcessor("proc", supplier, table2.name); try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { - final MockProcessor proc = supplier.theCapturedProcessor(); assertTrue(table1.sendingOldValueEnabled()); @@ -263,21 +248,17 @@ public void testSendingOldValue() { driver.pipeInput(recordFactory.create(topic1, "A", "01")); driver.pipeInput(recordFactory.create(topic1, "B", "01")); driver.pipeInput(recordFactory.create(topic1, "C", "01")); - - proc.checkAndClearProcessResult("A:(1<-null)", "B:(1<-null)", "C:(1<-null)"); + proc.checkAndClearProcessResult("A:(1<-null) (ts: 0)", "B:(1<-null) (ts: 0)", "C:(1<-null) (ts: 0)"); driver.pipeInput(recordFactory.create(topic1, "A", "02")); driver.pipeInput(recordFactory.create(topic1, "B", "02")); - - proc.checkAndClearProcessResult("A:(2<-1)", "B:(2<-1)"); + proc.checkAndClearProcessResult("A:(2<-1) (ts: 0)", "B:(2<-1) (ts: 0)"); driver.pipeInput(recordFactory.create(topic1, "A", "03")); - - proc.checkAndClearProcessResult("A:(3<-2)"); + proc.checkAndClearProcessResult("A:(3<-2) (ts: 0)"); driver.pipeInput(recordFactory.create(topic1, "A", (String) null)); - - proc.checkAndClearProcessResult("A:(null<-3)"); + proc.checkAndClearProcessResult("A:(null<-3) (ts: 0)"); } } } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableSourceTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableSourceTest.java index ec739e07c3ad6..aa0c8d72c51a3 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableSourceTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableSourceTest.java @@ -46,15 +46,14 @@ import static org.junit.Assert.assertTrue; public class KTableSourceTest { - private final Consumed stringConsumed = Consumed.with(Serdes.String(), Serdes.String()); - private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer()); + private final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L); private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String()); @Test public void testKTable() { final StreamsBuilder builder = new StreamsBuilder(); - final String topic1 = "topic1"; final KTable table1 = builder.table(topic1, Consumed.with(Serdes.String(), Serdes.Integer())); @@ -62,7 +61,8 @@ public void testKTable() { final MockProcessorSupplier supplier = new MockProcessorSupplier<>(); table1.toStream().process(supplier); - final ConsumerRecordFactory integerFactory = new ConsumerRecordFactory<>(new StringSerializer(), new IntegerSerializer()); + final ConsumerRecordFactory integerFactory = + new ConsumerRecordFactory<>(new StringSerializer(), new IntegerSerializer(), 0L); try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { driver.pipeInput(integerFactory.create(topic1, "A", 1)); driver.pipeInput(integerFactory.create(topic1, "B", 2)); @@ -72,7 +72,9 @@ public void testKTable() { driver.pipeInput(integerFactory.create(topic1, "B", null)); } - assertEquals(asList("A:1", "B:2", "C:3", "D:4", "A:null", "B:null"), supplier.theCapturedProcessor().processed); + assertEquals( + asList("A:1 (ts: 0)", "B:2 (ts: 0)", "C:3 (ts: 0)", "D:4 (ts: 0)", "A:null (ts: 0)", "B:null (ts: 0)"), + supplier.theCapturedProcessor().processed); } @Test @@ -94,14 +96,13 @@ public void kTableShouldLogAndMeterOnSkippedRecords() { @Test public void testValueGetter() { final StreamsBuilder builder = new StreamsBuilder(); - final String topic1 = "topic1"; @SuppressWarnings("unchecked") - final KTableImpl table1 = (KTableImpl) builder.table(topic1, stringConsumed, Materialized.as("store")); + final KTableImpl table1 = + (KTableImpl) builder.table(topic1, stringConsumed, Materialized.as("store")); final Topology topology = builder.build(); - final KTableValueGetterSupplier getterSupplier1 = table1.valueGetterSupplier(); final InternalTopologyBuilder topologyBuilder = TopologyWrapper.getInternalTopologyBuilder(topology); @@ -139,88 +140,71 @@ public void testValueGetter() { assertNull(getter1.get("B")); assertEquals("01", getter1.get("C")); } - } @Test public void testNotSendingOldValue() { final StreamsBuilder builder = new StreamsBuilder(); - final String topic1 = "topic1"; @SuppressWarnings("unchecked") final KTableImpl table1 = (KTableImpl) builder.table(topic1, stringConsumed); final MockProcessorSupplier supplier = new MockProcessorSupplier<>(); - final Topology topology = builder.build().addProcessor("proc1", supplier, table1.name); try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) { - final MockProcessor proc1 = supplier.theCapturedProcessor(); driver.pipeInput(recordFactory.create(topic1, "A", "01")); driver.pipeInput(recordFactory.create(topic1, "B", "01")); driver.pipeInput(recordFactory.create(topic1, "C", "01")); - - proc1.checkAndClearProcessResult("A:(01<-null)", "B:(01<-null)", "C:(01<-null)"); + proc1.checkAndClearProcessResult("A:(01<-null) (ts: 0)", "B:(01<-null) (ts: 0)", "C:(01<-null) (ts: 0)"); driver.pipeInput(recordFactory.create(topic1, "A", "02")); driver.pipeInput(recordFactory.create(topic1, "B", "02")); - - proc1.checkAndClearProcessResult("A:(02<-null)", "B:(02<-null)"); + proc1.checkAndClearProcessResult("A:(02<-null) (ts: 0)", "B:(02<-null) (ts: 0)"); driver.pipeInput(recordFactory.create(topic1, "A", "03")); - - proc1.checkAndClearProcessResult("A:(03<-null)"); + proc1.checkAndClearProcessResult("A:(03<-null) (ts: 0)"); driver.pipeInput(recordFactory.create(topic1, "A", (String) null)); driver.pipeInput(recordFactory.create(topic1, "B", (String) null)); - - proc1.checkAndClearProcessResult("A:(null<-null)", "B:(null<-null)"); + proc1.checkAndClearProcessResult("A:(null<-null) (ts: 0)", "B:(null<-null) (ts: 0)"); } } @Test public void testSendingOldValue() { final StreamsBuilder builder = new StreamsBuilder(); - final String topic1 = "topic1"; @SuppressWarnings("unchecked") final KTableImpl table1 = (KTableImpl) builder.table(topic1, stringConsumed); - table1.enableSendingOldValues(); - assertTrue(table1.sendingOldValueEnabled()); final MockProcessorSupplier supplier = new MockProcessorSupplier<>(); - final Topology topology = builder.build().addProcessor("proc1", supplier, table1.name); try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) { - final MockProcessor proc1 = supplier.theCapturedProcessor(); driver.pipeInput(recordFactory.create(topic1, "A", "01")); driver.pipeInput(recordFactory.create(topic1, "B", "01")); driver.pipeInput(recordFactory.create(topic1, "C", "01")); - - proc1.checkAndClearProcessResult("A:(01<-null)", "B:(01<-null)", "C:(01<-null)"); + proc1.checkAndClearProcessResult("A:(01<-null) (ts: 0)", "B:(01<-null) (ts: 0)", "C:(01<-null) (ts: 0)"); driver.pipeInput(recordFactory.create(topic1, "A", "02")); driver.pipeInput(recordFactory.create(topic1, "B", "02")); - - proc1.checkAndClearProcessResult("A:(02<-01)", "B:(02<-01)"); + proc1.checkAndClearProcessResult("A:(02<-01) (ts: 0)", "B:(02<-01) (ts: 0)"); driver.pipeInput(recordFactory.create(topic1, "A", "03")); - - proc1.checkAndClearProcessResult("A:(03<-02)"); + proc1.checkAndClearProcessResult("A:(03<-02) (ts: 0)"); driver.pipeInput(recordFactory.create(topic1, "A", (String) null)); driver.pipeInput(recordFactory.create(topic1, "B", (String) null)); - - proc1.checkAndClearProcessResult("A:(null<-03)", "B:(null<-02)"); + proc1.checkAndClearProcessResult("A:(null<-03) (ts: 0)", "B:(null<-02) (ts: 0)"); } } } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableTransformValuesTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableTransformValuesTest.java index 7da5077221045..ed6b64952534f 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableTransformValuesTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableTransformValuesTest.java @@ -76,7 +76,7 @@ public class KTableTransformValuesTest { private static final Consumed CONSUMED = Consumed.with(Serdes.String(), Serdes.String()); private final ConsumerRecordFactory recordFactory = - new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer()); + new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L); private TopologyTestDriver driver; private MockProcessorSupplier capture; @@ -325,11 +325,11 @@ public void shouldTransformValuesWithKey() { driver = new TopologyTestDriver(builder.build(), props()); - driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "a", 0L)); - driver.pipeInput(recordFactory.create(INPUT_TOPIC, "B", "b", 0L)); - driver.pipeInput(recordFactory.create(INPUT_TOPIC, "D", (String) null, 0L)); + driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "a")); + driver.pipeInput(recordFactory.create(INPUT_TOPIC, "B", "b")); + driver.pipeInput(recordFactory.create(INPUT_TOPIC, "D", (String) null)); - assertThat(output(), hasItems("A:A->a!", "B:B->b!", "D:D->null!")); + assertThat(output(), hasItems("A:A->a! (ts: 0)", "B:B->b! (ts: 0)", "D:D->null! (ts: 0)")); assertNull("Store should not be materialized", driver.getKeyValueStore(QUERYABLE_NAME)); } @@ -349,11 +349,11 @@ public void shouldTransformValuesWithKeyAndMaterialize() { driver = new TopologyTestDriver(builder.build(), props()); - driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "a", 0L)); - driver.pipeInput(recordFactory.create(INPUT_TOPIC, "B", "b", 0L)); - driver.pipeInput(recordFactory.create(INPUT_TOPIC, "C", (String) null, 0L)); + driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "a")); + driver.pipeInput(recordFactory.create(INPUT_TOPIC, "B", "b")); + driver.pipeInput(recordFactory.create(INPUT_TOPIC, "C", (String) null)); - assertThat(output(), hasItems("A:A->a!", "B:B->b!", "C:C->null!")); + assertThat(output(), hasItems("A:A->a! (ts: 0)", "B:B->b! (ts: 0)", "C:C->null! (ts: 0)")); final KeyValueStore keyValueStore = driver.getKeyValueStore(QUERYABLE_NAME); assertThat(keyValueStore.get("A"), is("A->a!")); @@ -378,11 +378,11 @@ public void shouldCalculateCorrectOldValuesIfMaterializedEvenIfStateful() { driver = new TopologyTestDriver(builder.build(), props()); - driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "ignore", 0L)); - driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "ignored", 0L)); - driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "ignored", 0L)); + driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "ignore")); + driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "ignored")); + driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "ignored")); - assertThat(output(), hasItems("A:1", "A:0", "A:2", "A:0", "A:3")); + assertThat(output(), hasItems("A:1 (ts: 0)", "A:0 (ts: 0)", "A:2 (ts: 0)", "A:0 (ts: 0)", "A:3 (ts: 0)")); final KeyValueStore keyValueStore = driver.getKeyValueStore(QUERYABLE_NAME); assertThat(keyValueStore.get("A"), is(3)); @@ -401,11 +401,11 @@ public void shouldCalculateCorrectOldValuesIfNotStatefulEvenIfNotMaterialized() driver = new TopologyTestDriver(builder.build(), props()); - driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "a", 0L)); - driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "aa", 0L)); - driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "aaa", 0L)); + driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "a")); + driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "aa")); + driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "aaa")); - assertThat(output(), hasItems("A:1", "A:0", "A:2", "A:0", "A:3")); + assertThat(output(), hasItems("A:1 (ts: 0)", "A:0 (ts: 0)", "A:2 (ts: 0)", "A:0 (ts: 0)", "A:3 (ts: 0)")); } private ArrayList output() { @@ -480,8 +480,7 @@ public String transform(final Object readOnlyKey, final String value) { } @Override - public void close() { - } + public void close() {} } private static class NullSupplier implements ValueTransformerWithKeySupplier { @@ -502,8 +501,7 @@ private static class StatefulTransformer implements ValueTransformerWithKey { @@ -524,8 +521,7 @@ public ValueTransformerWithKey get() { private static class StatelessTransformer implements ValueTransformerWithKey { @Override - public void init(final ProcessorContext context) { - } + public void init(final ProcessorContext context) {} @Override public Integer transform(final String readOnlyKey, final String value) { @@ -533,7 +529,6 @@ public Integer transform(final String readOnlyKey, final String value) { } @Override - public void close() { - } + public void close() {} } } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImplTest.java index 57d7b149a8814..b2c4ec88bf29b 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImplTest.java @@ -24,7 +24,6 @@ import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.TopologyTestDriver; import org.apache.kafka.streams.kstream.Consumed; -import org.apache.kafka.streams.kstream.ForeachAction; import org.apache.kafka.streams.kstream.Grouped; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.Materialized; @@ -52,18 +51,12 @@ import static org.hamcrest.MatcherAssert.assertThat; public class SessionWindowedKStreamImplTest { - private static final String TOPIC = "input"; private final StreamsBuilder builder = new StreamsBuilder(); - private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer()); + private final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer()); private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String()); - - private final Merger sessionMerger = new Merger() { - @Override - public String apply(final String aggKey, final String aggOne, final String aggTwo) { - return aggOne + "+" + aggTwo; - } - }; + private final Merger sessionMerger = (aggKey, aggOne, aggTwo) -> aggOne + "+" + aggTwo; private SessionWindowedKStream stream; @Before @@ -78,14 +71,9 @@ public void shouldCountSessionWindowed() { final Map, Long> results = new HashMap<>(); stream.count() .toStream() - .foreach(new ForeachAction, Long>() { - @Override - public void apply(final Windowed key, final Long value) { - results.put(key, value); - } - }); + .foreach(results::put); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { processData(driver); } assertThat(results.get(new Windowed<>("1", new SessionWindow(10, 15))), equalTo(2L)); @@ -98,14 +86,9 @@ public void shouldReduceWindowed() { final Map, String> results = new HashMap<>(); stream.reduce(MockReducer.STRING_ADDER) .toStream() - .foreach(new ForeachAction, String>() { - @Override - public void apply(final Windowed key, final String value) { - results.put(key, value); - } - }); + .foreach(results::put); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { processData(driver); } assertThat(results.get(new Windowed<>("1", new SessionWindow(10, 15))), equalTo("1+2")); @@ -119,15 +102,10 @@ public void shouldAggregateSessionWindowed() { stream.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, sessionMerger, - Materialized.>with(Serdes.String(), Serdes.String())) + Materialized.with(Serdes.String(), Serdes.String())) .toStream() - .foreach(new ForeachAction, String>() { - @Override - public void apply(final Windowed key, final String value) { - results.put(key, value); - } - }); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { + .foreach(results::put); + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { processData(driver); } assertThat(results.get(new Windowed<>("1", new SessionWindow(10, 15))), equalTo("0+0+1+2")); @@ -137,13 +115,15 @@ public void apply(final Windowed key, final String value) { @Test public void shouldMaterializeCount() { - stream.count(Materialized.>as("count-store")); + stream.count(Materialized.as("count-store")); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { processData(driver); final SessionStore store = driver.getSessionStore("count-store"); final List, Long>> data = StreamsTestUtils.toList(store.fetch("1", "2")); - assertThat(data, equalTo(Arrays.asList( + assertThat( + data, + equalTo(Arrays.asList( KeyValue.pair(new Windowed<>("1", new SessionWindow(10, 15)), 2L), KeyValue.pair(new Windowed<>("1", new SessionWindow(600, 600)), 1L), KeyValue.pair(new Windowed<>("2", new SessionWindow(600, 600)), 1L)))); @@ -152,14 +132,16 @@ public void shouldMaterializeCount() { @Test public void shouldMaterializeReduced() { - stream.reduce(MockReducer.STRING_ADDER, Materialized.>as("reduced")); + stream.reduce(MockReducer.STRING_ADDER, Materialized.as("reduced")); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { processData(driver); final SessionStore sessionStore = driver.getSessionStore("reduced"); final List, String>> data = StreamsTestUtils.toList(sessionStore.fetch("1", "2")); - assertThat(data, equalTo(Arrays.asList( + assertThat( + data, + equalTo(Arrays.asList( KeyValue.pair(new Windowed<>("1", new SessionWindow(10, 15)), "1+2"), KeyValue.pair(new Windowed<>("1", new SessionWindow(600, 600)), "3"), KeyValue.pair(new Windowed<>("2", new SessionWindow(600, 600)), "1")))); @@ -168,16 +150,19 @@ public void shouldMaterializeReduced() { @Test public void shouldMaterializeAggregated() { - stream.aggregate(MockInitializer.STRING_INIT, - MockAggregator.TOSTRING_ADDER, - sessionMerger, - Materialized.>as("aggregated").withValueSerde(Serdes.String())); + stream.aggregate( + MockInitializer.STRING_INIT, + MockAggregator.TOSTRING_ADDER, + sessionMerger, + Materialized.>as("aggregated").withValueSerde(Serdes.String())); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { processData(driver); final SessionStore sessionStore = driver.getSessionStore("aggregated"); final List, String>> data = StreamsTestUtils.toList(sessionStore.fetch("1", "2")); - assertThat(data, equalTo(Arrays.asList( + assertThat( + data, + equalTo(Arrays.asList( KeyValue.pair(new Windowed<>("1", new SessionWindow(10, 15)), "0+0+1+2"), KeyValue.pair(new Windowed<>("1", new SessionWindow(600, 600)), "0+3"), KeyValue.pair(new Windowed<>("2", new SessionWindow(600, 600)), "0+1")))); @@ -206,41 +191,44 @@ public void shouldThrowNullPointerOnReduceIfReducerIsNull() { @Test(expected = NullPointerException.class) public void shouldThrowNullPointerOnMaterializedAggregateIfInitializerIsNull() { - stream.aggregate(null, - MockAggregator.TOSTRING_ADDER, - sessionMerger, - Materialized.>as("store")); + stream.aggregate( + null, + MockAggregator.TOSTRING_ADDER, + sessionMerger, + Materialized.as("store")); } @Test(expected = NullPointerException.class) public void shouldThrowNullPointerOnMaterializedAggregateIfAggregatorIsNull() { - stream.aggregate(MockInitializer.STRING_INIT, - null, - sessionMerger, - Materialized.>as("store")); + stream.aggregate( + MockInitializer.STRING_INIT, + null, + sessionMerger, + Materialized.as("store")); } @Test(expected = NullPointerException.class) public void shouldThrowNullPointerOnMaterializedAggregateIfMergerIsNull() { - stream.aggregate(MockInitializer.STRING_INIT, - MockAggregator.TOSTRING_ADDER, - null, - Materialized.>as("store")); + stream.aggregate( + MockInitializer.STRING_INIT, + MockAggregator.TOSTRING_ADDER, + null, + Materialized.as("store")); } @SuppressWarnings("unchecked") @Test(expected = NullPointerException.class) public void shouldThrowNullPointerOnMaterializedAggregateIfMaterializedIsNull() { - stream.aggregate(MockInitializer.STRING_INIT, - MockAggregator.TOSTRING_ADDER, - sessionMerger, - (Materialized) null); + stream.aggregate( + MockInitializer.STRING_INIT, + MockAggregator.TOSTRING_ADDER, + sessionMerger, + (Materialized) null); } @Test(expected = NullPointerException.class) public void shouldThrowNullPointerOnMaterializedReduceIfReducerIsNull() { - stream.reduce(null, - Materialized.>as("store")); + stream.reduce(null, Materialized.as("store")); } @Test(expected = NullPointerException.class) diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImplTest.java index e0707ced8be2f..743d710093445 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImplTest.java @@ -51,7 +51,6 @@ import static org.hamcrest.MatcherAssert.assertThat; public class TimeWindowedKStreamImplTest { - private static final String TOPIC = "input"; private final StreamsBuilder builder = new StreamsBuilder(); private final ConsumerRecordFactory recordFactory = @@ -75,7 +74,7 @@ public void shouldCountWindowed() { .toStream() .foreach(results::put); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { processData(driver); } assertThat(results.get(new Windowed<>("1", new TimeWindow(0, 500))), equalTo(2L)); @@ -83,7 +82,6 @@ public void shouldCountWindowed() { assertThat(results.get(new Windowed<>("1", new TimeWindow(500, 1000))), equalTo(1L)); } - @Test public void shouldReduceWindowed() { final Map, String> results = new HashMap<>(); @@ -92,7 +90,7 @@ public void shouldReduceWindowed() { .toStream() .foreach(results::put); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { processData(driver); } assertThat(results.get(new Windowed<>("1", new TimeWindow(0, 500))), equalTo("1+2")); @@ -111,7 +109,7 @@ public void shouldAggregateWindowed() { .toStream() .foreach(results::put); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { processData(driver); } assertThat(results.get(new Windowed<>("1", new TimeWindow(0, 500))), equalTo("0+1+2")); @@ -126,7 +124,7 @@ public void shouldMaterializeCount() { .withKeySerde(Serdes.String()) .withValueSerde(Serdes.Long())); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { processData(driver); final WindowStore windowStore = driver.getWindowStore("count-store"); final List, Long>> data = @@ -147,7 +145,7 @@ public void shouldMaterializeReduced() { .withKeySerde(Serdes.String()) .withValueSerde(Serdes.String())); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { processData(driver); final WindowStore windowStore = driver.getWindowStore("reduced"); final List, String>> data = @@ -169,7 +167,7 @@ public void shouldMaterializeAggregated() { .withKeySerde(Serdes.String()) .withValueSerde(Serdes.String())); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) { + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { processData(driver); final WindowStore windowStore = driver.getWindowStore("aggregated"); final List, String>> data = diff --git a/streams/src/test/java/org/apache/kafka/test/MockProcessor.java b/streams/src/test/java/org/apache/kafka/test/MockProcessor.java index b3050f41bc29f..e000b8271ba0b 100644 --- a/streams/src/test/java/org/apache/kafka/test/MockProcessor.java +++ b/streams/src/test/java/org/apache/kafka/test/MockProcessor.java @@ -77,8 +77,11 @@ public void init(final ProcessorContext context) { public void process(final K key, final V value) { processedKeys.add(key); processedValues.add(value); - processed.add((key == null ? "null" : key) + ":" + - (value == null ? "null" : value)); + processed.add( + (key == null ? "null" : key) + + ":" + (value == null ? "null" : value) + + " (ts: " + context().timestamp() + ")" + ); if (commitRequested) { context().commit(); From fa57eb065d032c225f63a0b2ca3f050e728c2235 Mon Sep 17 00:00:00 2001 From: Florian Hussonnois Date: Wed, 20 Mar 2019 03:27:03 +0100 Subject: [PATCH 0042/1071] KAFKA-6958: Add new NamedOperation interface to enforce consistency in naming operations (#6409) Sub-task required to allow to define custom processor names with KStreams DSL(KIP-307) : - add new public interface NamedOperation - deprecate methods Joined.as() and Joined.name() - update Suppredded interface to extend NamedOperation Reviewers: Matthias J. Sax , John Roesler , Bill Bejeck --- .../apache/kafka/streams/kstream/Joined.java | 40 ++++++++++++++--- .../kafka/streams/kstream/NamedOperation.java | 32 +++++++++++++ .../kafka/streams/kstream/Suppressed.java | 3 +- .../kstream/internals/JoinedInternal.java | 45 +++++++++++++++++++ .../kstream/internals/KStreamImpl.java | 15 +++++-- 5 files changed, 124 insertions(+), 11 deletions(-) create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/NamedOperation.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/JoinedInternal.java diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Joined.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Joined.java index aa29c6805b381..1343487c9e28b 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/Joined.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Joined.java @@ -22,13 +22,12 @@ * The {@code Joined} class represents optional params that can be passed to * {@link KStream#join}, {@link KStream#leftJoin}, and {@link KStream#outerJoin} operations. */ -public class Joined { - - private final Serde keySerde; - private final Serde valueSerde; - private final Serde otherValueSerde; - private final String name; +public class Joined implements NamedOperation> { + protected final Serde keySerde; + protected final Serde valueSerde; + protected final Serde otherValueSerde; + protected final String name; private Joined(final Serde keySerde, final Serde valueSerde, @@ -40,6 +39,10 @@ private Joined(final Serde keySerde, this.name = name; } + protected Joined(final Joined joined) { + this(joined.keySerde, joined.valueSerde, joined.otherValueSerde, joined.name); + } + /** * Create an instance of {@code Joined} with key, value, and otherValue {@link Serde} instances. * {@code null} values are accepted and will be replaced by the default serdes as defined in config. @@ -135,11 +138,30 @@ public static Joined otherValueSerde(final Serde otherV * @param value type * @param other value type * @return new {@code Joined} instance configured with the name + * + * @deprecated use {@link #as(String)} instead */ + @Deprecated public static Joined named(final String name) { return new Joined<>(null, null, null, name); } + /** + * Create an instance of {@code Joined} with base name for all components of the join, this may + * include any repartition topics created to complete the join. + * + * @param name the name used as the base for naming components of the join including any + * repartition topics + * @param key type + * @param value type + * @param other value type + * @return new {@code Joined} instance configured with the name + * + */ + public static Joined as(final String name) { + return new Joined<>(null, null, null, name); + } + /** * Set the key {@link Serde} to be used. Null values are accepted and will be replaced by the default @@ -182,6 +204,7 @@ public Joined withOtherValueSerde(final Serde otherValueSerde) { * repartition topics * @return new {@code Joined} instance configured with the {@code name} */ + @Override public Joined withName(final String name) { return new Joined<>(keySerde, valueSerde, otherValueSerde, name); } @@ -198,7 +221,12 @@ public Serde otherValueSerde() { return otherValueSerde; } + /** + * @deprecated this method will be removed in a in a future release + */ + @Deprecated public String name() { return name; } + } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/NamedOperation.java b/streams/src/main/java/org/apache/kafka/streams/kstream/NamedOperation.java new file mode 100644 index 0000000000000..9a2c40b168b75 --- /dev/null +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/NamedOperation.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.kstream; + +/** + * Default interface which can be used to personalized the named of operations, internal topics or store. + */ +interface NamedOperation> { + + /** + * Sets the name to be used for an operation. + * + * @param name the name to use. + * @return an instance of {@link NamedOperation} + */ + T withName(final String name); + +} diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Suppressed.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Suppressed.java index 68541012ece9c..b5d79371747e6 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/Suppressed.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Suppressed.java @@ -23,7 +23,7 @@ import java.time.Duration; -public interface Suppressed { +public interface Suppressed extends NamedOperation> { /** * Marker interface for a buffer configuration that is "strict" in the sense that it will strictly @@ -163,5 +163,6 @@ static Suppressed untilTimeLimit(final Duration timeToWaitForMoreEvents, * @param name The name to be used for the suppression node and changelog topic * @return The same configuration with the addition of the given {@code name}. */ + @Override Suppressed withName(final String name); } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/JoinedInternal.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/JoinedInternal.java new file mode 100644 index 0000000000000..99f7a0fd740cb --- /dev/null +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/JoinedInternal.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.kstream.internals; + +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.kstream.Joined; + +public class JoinedInternal extends Joined { + + JoinedInternal(final Joined joined) { + super(joined); + } + + public Serde keySerde() { + return keySerde; + } + + public Serde valueSerde() { + return valueSerde; + } + + public Serde otherValueSerde() { + return otherValueSerde; + } + + @Override // TODO remove annotation when super.name() is removed + @SuppressWarnings("deprecation") // this method should not be removed if super.name() is removed + public String name() { + return name; + } +} diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index 856536cc78831..41260c5a0162f 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -567,13 +567,15 @@ private KStream doJoin(final KStream other, KStreamImpl joinThis = this; KStreamImpl joinOther = (KStreamImpl) other; + final JoinedInternal joinedInternal = new JoinedInternal<>(joined); + final String name = joinedInternal.name(); if (joinThis.repartitionRequired) { - final String leftJoinRepartitionTopicName = joined.name() != null ? joined.name() + "-left" : joinThis.name; + final String leftJoinRepartitionTopicName = name != null ? name + "-left" : joinThis.name; joinThis = joinThis.repartitionForJoin(leftJoinRepartitionTopicName, joined.keySerde(), joined.valueSerde()); } if (joinOther.repartitionRequired) { - final String rightJoinRepartitionTopicName = joined.name() != null ? joined.name() + "-right" : joinOther.name; + final String rightJoinRepartitionTopicName = name != null ? name + "-right" : joinOther.name; joinOther = joinOther.repartitionForJoin(rightJoinRepartitionTopicName, joined.keySerde(), joined.otherValueSerde()); } @@ -679,9 +681,12 @@ public KStream join(final KTable other, Objects.requireNonNull(other, "other can't be null"); Objects.requireNonNull(joiner, "joiner can't be null"); Objects.requireNonNull(joined, "joined can't be null"); + + final JoinedInternal joinedInternal = new JoinedInternal<>(joined); + final String name = joinedInternal.name(); if (repartitionRequired) { final KStreamImpl thisStreamRepartitioned = repartitionForJoin( - joined.name() != null ? joined.name() : name, + name != null ? name : this.name, joined.keySerde(), joined.valueSerde() ); @@ -703,9 +708,11 @@ public KStream leftJoin(final KTable other, Objects.requireNonNull(other, "other can't be null"); Objects.requireNonNull(joiner, "joiner can't be null"); Objects.requireNonNull(joined, "joined can't be null"); + final JoinedInternal joinedInternal = new JoinedInternal<>(joined); + final String internalName = joinedInternal.name(); if (repartitionRequired) { final KStreamImpl thisStreamRepartitioned = repartitionForJoin( - joined.name() != null ? joined.name() : name, + internalName != null ? internalName : name, joined.keySerde(), joined.valueSerde() ); From f6f8da70713be6486b9ca085f4130043a3b9e9aa Mon Sep 17 00:00:00 2001 From: huxihx Date: Wed, 20 Mar 2019 10:47:34 +0530 Subject: [PATCH 0043/1071] KAFKA-8098: Fix Flaky Test testConsumerGroups - The flaky failure is caused by the fact that the main thread sometimes issues DescribeConsumerGroup request before the consumer assignment takes effect. Added a latch to make sure such situation is not going to happen. Author: huxihx Author: huxi Author: Manikumar Reddy Reviewers: Manikumar Reddy Closes #6441 from huxihx/KAFKA-8098 --- .../kafka/api/AdminClientIntegrationTest.scala | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala index 027266d84e497..5c72cbf65323a 100644 --- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala @@ -19,7 +19,7 @@ package kafka.api import java.{time, util} import java.util.{Collections, Properties} import java.util.Arrays.asList -import java.util.concurrent.{ExecutionException, TimeUnit} +import java.util.concurrent.{CountDownLatch, ExecutionException, TimeUnit} import java.io.File import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger} @@ -52,6 +52,7 @@ import kafka.zk.KafkaZkClient import scala.concurrent.duration.Duration import scala.concurrent.{Await, Future} import java.lang.{Long => JLong} +import java.time.{Duration => JDuration} import kafka.security.auth.{Cluster, Group, Topic} @@ -1158,19 +1159,28 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging { newConsumerConfig.setProperty(ConsumerConfig.GROUP_ID_CONFIG, testGroupId) newConsumerConfig.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, testClientId) val consumer = createConsumer(configOverrides = newConsumerConfig) + val latch = new CountDownLatch(1) try { // Start a consumer in a thread that will subscribe to a new group. val consumerThread = new Thread { override def run { consumer.subscribe(Collections.singleton(testTopicName)) - while (true) { - consumer.poll(time.Duration.ofSeconds(5L)) - consumer.commitSync() + + try { + while (true) { + consumer.poll(JDuration.ofSeconds(5)) + if (!consumer.assignment.isEmpty && latch.getCount > 0L) + latch.countDown() + consumer.commitSync() + } + } catch { + case _: InterruptException => // Suppress the output to stderr } } } try { consumerThread.start + assertTrue(latch.await(30000, TimeUnit.MILLISECONDS)) // Test that we can list the new group. TestUtils.waitUntilTrue(() => { val matching = client.listConsumerGroups.all.get().asScala.filter(_.groupId == testGroupId) From 58d057296ae66cdf439dff19e73d7d6497dffc37 Mon Sep 17 00:00:00 2001 From: Anna Povzner Date: Thu, 21 Mar 2019 03:15:43 -0700 Subject: [PATCH 0044/1071] KAFKA-7989: RequestQuotaTest should wait for quota config change before running tests (#6482) Reviewers: Rajini Sivaram --- .../scala/unit/kafka/server/RequestQuotaTest.scala | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala index 32541d3fcc288..22967e256c2cc 100644 --- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala +++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala @@ -102,10 +102,14 @@ class RequestQuotaTest extends BaseRequestTest { quotaProps.put(DynamicConfig.Client.RequestPercentageOverrideProp, "0.01") adminZkClient.changeClientIdConfig(Sanitizer.sanitize(smallQuotaConsumerClientId), quotaProps) - TestUtils.retry(10000) { + TestUtils.retry(20000) { val quotaManager = servers.head.dataPlaneRequestProcessor.quotas.request assertEquals(s"Default request quota not set", Quota.upperBound(0.01), quotaManager.quota("some-user", "some-client")) assertEquals(s"Request quota override not set", Quota.upperBound(2000), quotaManager.quota("some-user", unthrottledClientId)) + val produceQuotaManager = servers.head.dataPlaneRequestProcessor.quotas.produce + assertEquals(s"Produce quota override not set", Quota.upperBound(1), produceQuotaManager.quota("some-user", smallQuotaProducerClientId)) + val consumeQuotaManager = servers.head.dataPlaneRequestProcessor.quotas.fetch + assertEquals(s"Consume quota override not set", Quota.upperBound(1), consumeQuotaManager.quota("some-user", smallQuotaConsumerClientId)) } } @@ -416,7 +420,10 @@ class RequestQuotaTest extends BaseRequestTest { override def toString: String = { val requestTime = requestTimeMetricValue(clientId) val throttleTime = throttleTimeMetricValue(clientId) - s"Client $clientId apiKey $apiKey requests $correlationId requestTime $requestTime throttleTime $throttleTime" + val produceThrottleTime = throttleTimeMetricValueForQuotaType(clientId, QuotaType.Produce) + val consumeThrottleTime = throttleTimeMetricValueForQuotaType(clientId, QuotaType.Fetch) + s"Client $clientId apiKey $apiKey requests $correlationId requestTime $requestTime " + + s"throttleTime $throttleTime produceThrottleTime $produceThrottleTime consumeThrottleTime $consumeThrottleTime" } } @@ -519,7 +526,7 @@ class RequestQuotaTest extends BaseRequestTest { val throttled = smallQuotaConsumerClient.runUntil(response => responseThrottleTime(apiKey, response) > 0) assertTrue(s"Response not throttled: $smallQuotaConsumerClientId", throttled) - assertTrue(s"Throttle time metrics for consumer quota not updated: $smallQuotaConsumerClientId", + assertTrue(s"Throttle time metrics for consumer quota not updated: $smallQuotaConsumerClient", throttleTimeMetricValueForQuotaType(smallQuotaConsumerClientId, QuotaType.Fetch) > 0) assertTrue(s"Throttle time metrics for request quota updated: $smallQuotaConsumerClient", throttleTimeMetricValueForQuotaType(smallQuotaConsumerClientId, QuotaType.Request).isNaN) From 62171781396b613b6be8e13a2541ab0895b9bb6b Mon Sep 17 00:00:00 2001 From: Stanislav Kozlovski Date: Thu, 21 Mar 2019 19:03:09 +0200 Subject: [PATCH 0045/1071] KAFKA-7819: Improve RoundTripWorker (#6187) RoundTripWorker to should use a long field for maxMessages rather than an int. The consumer group used should unique as well. Reviewers: Colin P. McCabe --- .../trogdor/workload/RoundTripWorker.java | 90 ++++++++++++------- .../workload/RoundTripWorkloadSpec.java | 6 +- 2 files changed, 62 insertions(+), 34 deletions(-) diff --git a/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorker.java b/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorker.java index b22292adf7f9f..d08d807454424 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorker.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorker.java @@ -57,11 +57,13 @@ import java.util.Map; import java.util.Properties; import java.util.TreeSet; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; public class RoundTripWorker implements TaskWorker { private static final int THROTTLE_PERIOD_MS = 100; @@ -82,6 +84,10 @@ public class RoundTripWorker implements TaskWorker { private final AtomicBoolean running = new AtomicBoolean(false); + private final Lock lock = new ReentrantLock(); + + private final Condition unackedSendsAreZero = lock.newCondition(); + private ScheduledExecutorService executor; private WorkerStatusTracker status; @@ -92,7 +98,7 @@ public class RoundTripWorker implements TaskWorker { private KafkaConsumer consumer; - private CountDownLatch unackedSends; + private Long unackedSends; private ToSendTracker toSendTracker; @@ -114,7 +120,7 @@ public void start(Platform platform, WorkerStatusTracker status, this.doneFuture = doneFuture; this.producer = null; this.consumer = null; - this.unackedSends = new CountDownLatch(spec.maxMessages()); + this.unackedSends = spec.maxMessages(); executor.submit(new Prepare()); } @@ -157,29 +163,29 @@ public void run() { } private static class ToSendTrackerResult { - final int index; + final long index; final boolean firstSend; - ToSendTrackerResult(int index, boolean firstSend) { + ToSendTrackerResult(long index, boolean firstSend) { this.index = index; this.firstSend = firstSend; } } private static class ToSendTracker { - private final int maxMessages; - private final List failed = new ArrayList<>(); - private int frontier = 0; + private final long maxMessages; + private final List failed = new ArrayList<>(); + private long frontier = 0; - ToSendTracker(int maxMessages) { + ToSendTracker(long maxMessages) { this.maxMessages = maxMessages; } - synchronized void addFailed(int index) { + synchronized void addFailed(long index) { failed.add(index); } - synchronized int frontier() { + synchronized long frontier() { return frontier; } @@ -232,7 +238,7 @@ public void run() { break; } throttle.increment(); - final int messageIndex = result.index; + final long messageIndex = result.index; if (result.firstSend) { toReceiveTracker.addPending(messageIndex); uniqueMessagesSent++; @@ -248,7 +254,14 @@ public void run() { spec.valueGenerator().generate(messageIndex)); producer.send(record, (metadata, exception) -> { if (exception == null) { - unackedSends.countDown(); + try { + lock.lock(); + unackedSends -= 1; + if (unackedSends <= 0) + unackedSendsAreZero.signalAll(); + } finally { + lock.unlock(); + } } else { log.info("{}: Got exception when sending message {}: {}", id, messageIndex, exception.getMessage()); @@ -259,23 +272,28 @@ public void run() { } catch (Throwable e) { WorkerUtils.abort(log, "ProducerRunnable", e, doneFuture); } finally { - log.info("{}: ProducerRunnable is exiting. messagesSent={}; uniqueMessagesSent={}; " + - "ackedSends={}.", id, messagesSent, uniqueMessagesSent, - spec.maxMessages() - unackedSends.getCount()); + try { + lock.lock(); + log.info("{}: ProducerRunnable is exiting. messagesSent={}; uniqueMessagesSent={}; " + + "ackedSends={}/{}.", id, messagesSent, uniqueMessagesSent, + spec.maxMessages() - unackedSends, spec.maxMessages()); + } finally { + lock.unlock(); + } } } } private class ToReceiveTracker { - private final TreeSet pending = new TreeSet<>(); + private final TreeSet pending = new TreeSet<>(); - private int totalReceived = 0; + private long totalReceived = 0; - synchronized void addPending(int messageIndex) { + synchronized void addPending(long messageIndex) { pending.add(messageIndex); } - synchronized boolean removePending(int messageIndex) { + synchronized boolean removePending(long messageIndex) { if (pending.remove(messageIndex)) { totalReceived++; return true; @@ -284,18 +302,18 @@ synchronized boolean removePending(int messageIndex) { } } - synchronized int totalReceived() { + synchronized long totalReceived() { return totalReceived; } void log() { - int numToReceive; - List list = new ArrayList<>(LOG_NUM_MESSAGES); + long numToReceive; + List list = new ArrayList<>(LOG_NUM_MESSAGES); synchronized (this) { numToReceive = pending.size(); - for (Iterator iter = pending.iterator(); + for (Iterator iter = pending.iterator(); iter.hasNext() && (list.size() < LOG_NUM_MESSAGES); ) { - Integer i = iter.next(); + Long i = iter.next(); list.add(i); } } @@ -311,7 +329,7 @@ class ConsumerRunnable implements Runnable { this.props = new Properties(); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, spec.bootstrapServers()); props.put(ConsumerConfig.CLIENT_ID_CONFIG, "consumer." + id); - props.put(ConsumerConfig.GROUP_ID_CONFIG, "round-trip-consumer-group-1"); + props.put(ConsumerConfig.GROUP_ID_CONFIG, "round-trip-consumer-group-" + id); props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); props.put(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, 105000); props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, 100000); @@ -341,9 +359,16 @@ public void run() { if (toReceiveTracker.removePending(messageIndex)) { uniqueMessagesReceived++; if (uniqueMessagesReceived >= spec.maxMessages()) { - log.info("{}: Consumer received the full count of {} unique messages. " + - "Waiting for all sends to be acked...", id, spec.maxMessages()); - unackedSends.await(); + try { + lock.lock(); + log.info("{}: Consumer received the full count of {} unique messages. " + + "Waiting for all {} sends to be acked...", id, spec.maxMessages(), unackedSends); + while (unackedSends > 0) + unackedSendsAreZero.await(); + } finally { + lock.unlock(); + } + log.info("{}: all sends have been acked.", id); new StatusUpdater().update(); doneFuture.complete(""); @@ -360,6 +385,8 @@ public void run() { log.debug("{}: Consumer got WakeupException", id, e); } catch (TimeoutException e) { log.debug("{}: Consumer got TimeoutException", id, e); + } finally { + lock.unlock(); } } } catch (Throwable e) { @@ -415,9 +442,9 @@ public long totalReceived() { @Override public void stop(Platform platform) throws Exception { if (!running.compareAndSet(true, false)) { - throw new IllegalStateException("ProduceBenchWorker is not running."); + throw new IllegalStateException("RoundTripWorker is not running."); } - log.info("{}: Deactivating RoundTripWorkloadWorker.", id); + log.info("{}: Deactivating RoundTripWorker.", id); doneFuture.complete(""); executor.shutdownNow(); executor.awaitTermination(1, TimeUnit.DAYS); @@ -428,5 +455,6 @@ public void stop(Platform platform) throws Exception { this.unackedSends = null; this.executor = null; this.doneFuture = null; + log.info("{}: Deactivated RoundTripWorker.", id); } } diff --git a/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorkloadSpec.java b/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorkloadSpec.java index 42e09ee798f3a..fd30e8eb78b34 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorkloadSpec.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorkloadSpec.java @@ -36,7 +36,7 @@ public class RoundTripWorkloadSpec extends TaskSpec { private final int targetMessagesPerSec; private final PayloadGenerator valueGenerator; private final TopicsSpec activeTopics; - private final int maxMessages; + private final long maxMessages; private final Map commonClientConf; private final Map producerConf; private final Map consumerConf; @@ -54,7 +54,7 @@ public RoundTripWorkloadSpec(@JsonProperty("startMs") long startMs, @JsonProperty("targetMessagesPerSec") int targetMessagesPerSec, @JsonProperty("valueGenerator") PayloadGenerator valueGenerator, @JsonProperty("activeTopics") TopicsSpec activeTopics, - @JsonProperty("maxMessages") int maxMessages) { + @JsonProperty("maxMessages") long maxMessages) { super(startMs, durationMs); this.clientNode = clientNode == null ? "" : clientNode; this.bootstrapServers = bootstrapServers == null ? "" : bootstrapServers; @@ -96,7 +96,7 @@ public PayloadGenerator valueGenerator() { } @JsonProperty - public int maxMessages() { + public long maxMessages() { return maxMessages; } From 4cae4523fc04597814d85c1484ec67d8cadc2256 Mon Sep 17 00:00:00 2001 From: "Matthias J. Sax" Date: Thu, 21 Mar 2019 12:51:18 -0700 Subject: [PATCH 0046/1071] MINOR: Add verification step for Streams archetype to Jenkins build (#6431) Updates ./jenkins.sh to build stream archetype and install it in local maven cache. Afterward, archetype is used to create a new maven project and maven project is compiled for verification. Reviewers: Guozhang Wang , John Roesler , Bill Bejeck --- docs/streams/tutorial.html | 8 ++++++-- jenkins.sh | 42 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/docs/streams/tutorial.html b/docs/streams/tutorial.html index 0006e3ef53515..c5db2803871cc 100644 --- a/docs/streams/tutorial.html +++ b/docs/streams/tutorial.html @@ -74,8 +74,12 @@

Setting up a Maven

- The pom.xml file included in the project already has the Streams dependency defined, - and there are already several example programs written with Streams library under src/main/java. + The pom.xml file included in the project already has the Streams dependency defined. + Note, that the generated pom.xml targets Java 8, and does not work with higher Java versions. +

+ +

+ There are already several example programs written with Streams library under src/main/java. Since we are going to start writing such programs from scratch, we can now delete these examples:

diff --git a/jenkins.sh b/jenkins.sh index 5d549fe074346..6f94134982634 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -28,3 +28,45 @@ ./gradlew unitTest integrationTest \ --profile --no-daemon --continue -PtestLoggingEvents=started,passed,skipped,failed "$@" \ || { echo 'Test steps failed'; exit 1; } + +# Verify that Kafka Streams archetype compiles +if [ $JAVA_HOME = "/home/jenkins/tools/java/latest11" ] ; then + echo "Skipping Kafka Streams archetype test for Java 11" + exit 0 +fi + +./gradlew streams:install clients:install connect:json:install connect:api:install \ + || { echo 'Could not install kafka-streams.jar (and dependencies) locally`'; exit 1; } + +version=`grep "^version=" gradle.properties | cut -d= -f 2` \ + || { echo 'Could not get version from `gradle.properties`'; exit 1; } + +cd streams/quickstart \ + || { echo 'Could not change into directory `streams/quickstart`'; exit 1; } + +# variable $MAVEN_LATEST__HOME is provided by Jenkins (see build configuration) +mvn=$MAVEN_LATEST__HOME/bin/mvn + +$mvn clean install -Dgpg.skip \ + || { echo 'Could not `mvn install` streams quickstart archetype'; exit 1; } + +mkdir test-streams-archetype && cd test-streams-archetype \ + || { echo 'Could not create test directory for stream quickstart archetype'; exit 1; } + +echo "Y" | $mvn archetype:generate \ + -DarchetypeCatalog=local \ + -DarchetypeGroupId=org.apache.kafka \ + -DarchetypeArtifactId=streams-quickstart-java \ + -DarchetypeVersion=$version \ + -DgroupId=streams.examples \ + -DartifactId=streams.examples \ + -Dversion=0.1 \ + -Dpackage=myapps \ + || { echo 'Could not create new project using streams quickstart archetype'; exit 1; } + +cd streams.examples \ + || { echo 'Could not change into directory `streams.examples`'; exit 1; } + +$mvn compile \ + || { echo 'Could not compile streams quickstart archetype project'; exit 1; } + From 3124b070666d3b8cdc0e96067e69b8e7a6245742 Mon Sep 17 00:00:00 2001 From: khairy Date: Fri, 22 Mar 2019 01:21:55 +0100 Subject: [PATCH 0047/1071] KAFKA-7243: Add unit integration tests to validate metrics in Kafka Streams (#6080) The goal of this task is to implement an integration test for the kafka stream metrics. We have to check 2 things: 1. After streams application are started, all metrics from different levels (thread, task, processor, store, cache) are correctly created and displaying recorded values. 2. When streams application are shutdown, all metrics are correctly de-registered and removed. Reviewers: John Roesler , Guozhang Wang --- .../org/apache/kafka/test/TestCondition.java | 2 +- .../integration/MetricsIntegrationTest.java | 532 ++++++++++++++++++ 2 files changed, 533 insertions(+), 1 deletion(-) create mode 100644 streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java diff --git a/clients/src/test/java/org/apache/kafka/test/TestCondition.java b/clients/src/test/java/org/apache/kafka/test/TestCondition.java index e7b78cf71e6a7..b153c75938b34 100644 --- a/clients/src/test/java/org/apache/kafka/test/TestCondition.java +++ b/clients/src/test/java/org/apache/kafka/test/TestCondition.java @@ -23,5 +23,5 @@ @FunctionalInterface public interface TestCondition { - boolean conditionMet(); + boolean conditionMet() throws InterruptedException; } diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java new file mode 100644 index 0000000000000..3ce31cb328955 --- /dev/null +++ b/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java @@ -0,0 +1,532 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.integration; + +import org.apache.kafka.common.Metric; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsBuilder; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster; +import org.apache.kafka.streams.integration.utils.IntegrationTestUtils; +import org.apache.kafka.streams.kstream.KGroupedStream; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Consumed; +import org.apache.kafka.streams.kstream.Produced; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.SessionWindows; +import org.apache.kafka.streams.kstream.TimeWindows; +import org.apache.kafka.streams.state.SessionStore; +import org.apache.kafka.streams.state.Stores; +import org.apache.kafka.streams.state.WindowStore; +import org.apache.kafka.test.IntegrationTest; +import org.apache.kafka.test.TestUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.After; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +@SuppressWarnings("unchecked") +@Category({IntegrationTest.class}) +public class MetricsIntegrationTest { + + private static final int NUM_BROKERS = 1; + + @ClassRule + public static final EmbeddedKafkaCluster CLUSTER = + new EmbeddedKafkaCluster(NUM_BROKERS); + + // Metric group + private static final String STREAM_THREAD_NODE_METRICS = "stream-metrics"; + private static final String STREAM_TASK_NODE_METRICS = "stream-task-metrics"; + private static final String STREAM_PROCESSOR_NODE_METRICS = "stream-processor-node-metrics"; + private static final String STREAM_CACHE_NODE_METRICS = "stream-record-cache-metrics"; + private static final String STREAM_STORE_IN_MEMORY_STATE_METRICS = "stream-in-memory-state-metrics"; + private static final String STREAM_STORE_IN_MEMORY_LRU_STATE_METRICS = "stream-in-memory-lru-state-metrics"; + private static final String STREAM_STORE_ROCKSDB_STATE_METRICS = "stream-rocksdb-state-metrics"; + private static final String STREAM_STORE_WINDOW_ROCKSDB_STATE_METRICS = "stream-rocksdb-window-state-metrics"; + private static final String STREAM_STORE_SESSION_ROCKSDB_STATE_METRICS = "stream-rocksdb-session-state-metrics"; + + // Metrics name + private static final String PUT_LATENCY_AVG = "put-latency-avg"; + private static final String PUT_LATENCY_MAX = "put-latency-max"; + private static final String PUT_IF_ABSENT_LATENCY_AVG = "put-if-absent-latency-avg"; + private static final String PUT_IF_ABSENT_LATENCY_MAX = "put-if-absent-latency-max"; + private static final String GET_LATENCY_AVG = "get-latency-avg"; + private static final String GET_LATENCY_MAX = "get-latency-max"; + private static final String DELETE_LATENCY_AVG = "delete-latency-avg"; + private static final String DELETE_LATENCY_MAX = "delete-latency-max"; + private static final String PUT_ALL_LATENCY_AVG = "put-all-latency-avg"; + private static final String PUT_ALL_LATENCY_MAX = "put-all-latency-max"; + private static final String ALL_LATENCY_AVG = "all-latency-avg"; + private static final String ALL_LATENCY_MAX = "all-latency-max"; + private static final String RANGE_LATENCY_AVG = "range-latency-avg"; + private static final String RANGE_LATENCY_MAX = "range-latency-max"; + private static final String FLUSH_LATENCY_AVG = "flush-latency-avg"; + private static final String FLUSH_LATENCY_MAX = "flush-latency-max"; + private static final String RESTORE_LATENCY_AVG = "restore-latency-avg"; + private static final String RESTORE_LATENCY_MAX = "restore-latency-max"; + private static final String PUT_RATE = "put-rate"; + private static final String PUT_TOTAL = "put-total"; + private static final String PUT_IF_ABSENT_RATE = "put-if-absent-rate"; + private static final String PUT_IF_ABSENT_TOTAL = "put-if-absent-total"; + private static final String GET_RATE = "get-rate"; + private static final String DELETE_RATE = "delete-rate"; + private static final String DELETE_TOTAL = "delete-total"; + private static final String PUT_ALL_RATE = "put-all-rate"; + private static final String PUT_ALL_TOTAL = "put-all-total"; + private static final String ALL_RATE = "all-rate"; + private static final String ALL_TOTAL = "all-total"; + private static final String RANGE_RATE = "range-rate"; + private static final String RANGE_TOTAL = "range-total"; + private static final String FLUSH_RATE = "flush-rate"; + private static final String FLUSH_TOTAL = "flush-total"; + private static final String RESTORE_RATE = "restore-rate"; + private static final String RESTORE_TOTAL = "restore-total"; + private static final String PROCESS_LATENCY_AVG = "process-latency-avg"; + private static final String PROCESS_LATENCY_MAX = "process-latency-max"; + private static final String PUNCTUATE_LATENCY_AVG = "punctuate-latency-avg"; + private static final String PUNCTUATE_LATENCY_MAX = "punctuate-latency-max"; + private static final String CREATE_LATENCY_AVG = "create-latency-avg"; + private static final String CREATE_LATENCY_MAX = "create-latency-max"; + private static final String DESTROY_LATENCY_AVG = "destroy-latency-avg"; + private static final String DESTROY_LATENCY_MAX = "destroy-latency-max"; + private static final String PROCESS_RATE = "process-rate"; + private static final String PROCESS_TOTAL = "process-total"; + private static final String PUNCTUATE_RATE = "punctuate-rate"; + private static final String PUNCTUATE_TOTAL = "punctuate-total"; + private static final String CREATE_RATE = "create-rate"; + private static final String CREATE_TOTAL = "create-total"; + private static final String DESTROY_RATE = "destroy-rate"; + private static final String DESTROY_TOTAL = "destroy-total"; + private static final String FORWARD_TOTAL = "forward-total"; + private static final String STREAM_STRING = "stream"; + private static final String COMMIT_LATENCY_AVG = "commit-latency-avg"; + private static final String COMMIT_LATENCY_MAX = "commit-latency-max"; + private static final String POLL_LATENCY_AVG = "poll-latency-avg"; + private static final String POLL_LATENCY_MAX = "poll-latency-max"; + private static final String COMMIT_RATE = "commit-rate"; + private static final String COMMIT_TOTAL = "commit-total"; + private static final String POLL_RATE = "poll-rate"; + private static final String POLL_TOTAL = "poll-total"; + private static final String TASK_CREATED_RATE = "task-created-rate"; + private static final String TASK_CREATED_TOTAL = "task-created-total"; + private static final String TASK_CLOSED_RATE = "task-closed-rate"; + private static final String TASK_CLOSED_TOTAL = "task-closed-total"; + private static final String SKIPPED_RECORDS_RATE = "skipped-records-rate"; + private static final String SKIPPED_RECORDS_TOTAL = "skipped-records-total"; + private static final String RECORD_LATENESS_AVG = "record-lateness-avg"; + private static final String RECORD_LATENESS_MAX = "record-lateness-max"; + private static final String HIT_RATIO_AVG = "hitRatio-avg"; + private static final String HIT_RATIO_MIN = "hitRatio-min"; + private static final String HIT_RATIO_MAX = "hitRatio-max"; + + // stores name + private static final String TIME_WINDOWED_AGGREGATED_STREAM_STORE = "time-windowed-aggregated-stream-store"; + private static final String SESSION_AGGREGATED_STREAM_STORE = "session-aggregated-stream-store"; + private static final String MY_STORE_IN_MEMORY = "myStoreInMemory"; + private static final String MY_STORE_PERSISTENT_KEY_VALUE = "myStorePersistentKeyValue"; + private static final String MY_STORE_LRU_MAP = "myStoreLruMap"; + + private StreamsBuilder builder; + private Properties streamsConfiguration; + private KafkaStreams kafkaStreams; + + // topic names + private static final String STREAM_INPUT = "STREAM_INPUT"; + private static final String STREAM_OUTPUT_1 = "STREAM_OUTPUT_1"; + private static final String STREAM_OUTPUT_2 = "STREAM_OUTPUT_2"; + private static final String STREAM_OUTPUT_3 = "STREAM_OUTPUT_3"; + private static final String STREAM_OUTPUT_4 = "STREAM_OUTPUT_4"; + + private KStream stream; + private KStream stream2; + + private final String appId = "stream-metrics-test"; + + @Before + public void before() throws InterruptedException { + builder = new StreamsBuilder(); + CLUSTER.createTopics(STREAM_INPUT, STREAM_OUTPUT_1, STREAM_OUTPUT_2, STREAM_OUTPUT_3, STREAM_OUTPUT_4); + streamsConfiguration = new Properties(); + streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, appId); + streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()); + streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass()); + streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass()); + streamsConfiguration.put(StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG, Sensor.RecordingLevel.DEBUG.name); + streamsConfiguration.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 10 * 1024 * 1024L); + } + + @After + public void after() throws InterruptedException { + CLUSTER.deleteTopics(STREAM_INPUT, STREAM_OUTPUT_1, STREAM_OUTPUT_2, STREAM_OUTPUT_3, STREAM_OUTPUT_4); + } + + private void startApplication() { + kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration); + kafkaStreams.start(); + } + + private void closeApplication() throws Exception { + kafkaStreams.close(); + kafkaStreams.cleanUp(); + IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration); + } + + private void checkMetricDeregistration() throws InterruptedException { + TestUtils.waitForCondition(() -> { + final List listMetricAfterClosingApp = new ArrayList(kafkaStreams.metrics().values()).stream().filter(m -> m.metricName().group().contains(STREAM_STRING)).collect(Collectors.toList()); + return listMetricAfterClosingApp.size() == 0; + }, 10000, "de-registration of metrics"); + } + + @Test + public void testStreamMetric() throws Exception { + final StringBuilder errorMessage = new StringBuilder(); + stream = builder.stream(STREAM_INPUT, Consumed.with(Serdes.Integer(), Serdes.String())); + stream.to(STREAM_OUTPUT_1, Produced.with(Serdes.Integer(), Serdes.String())); + builder.table(STREAM_OUTPUT_1, Materialized.as(Stores.inMemoryKeyValueStore(MY_STORE_IN_MEMORY)).withCachingEnabled()) + .toStream() + .to(STREAM_OUTPUT_2); + builder.table(STREAM_OUTPUT_2, Materialized.as(Stores.persistentKeyValueStore(MY_STORE_PERSISTENT_KEY_VALUE)).withCachingEnabled()) + .toStream() + .to(STREAM_OUTPUT_3); + builder.table(STREAM_OUTPUT_3, Materialized.as(Stores.lruMap(MY_STORE_LRU_MAP, 10000)).withCachingEnabled()) + .toStream() + .to(STREAM_OUTPUT_4); + + startApplication(); + + // metric level : Thread + TestUtils.waitForCondition(() -> testThreadMetric(errorMessage), 10000, () -> "testThreadMetric -> " + errorMessage.toString()); + + // metric level : Task + TestUtils.waitForCondition(() -> testTaskMetric(errorMessage), 10000, () -> "testTaskMetric -> " + errorMessage.toString()); + + // metric level : Processor + TestUtils.waitForCondition(() -> testProcessorMetric(errorMessage), 10000, () -> "testProcessorMetric -> " + errorMessage.toString()); + + // metric level : Store (in-memory-state, in-memory-lru-state, rocksdb-state) + TestUtils.waitForCondition(() -> testStoreMetricKeyValueByType(STREAM_STORE_IN_MEMORY_STATE_METRICS, errorMessage), 10000, () -> "testStoreMetricKeyValueByType:" + STREAM_STORE_IN_MEMORY_STATE_METRICS + " -> " + errorMessage.toString()); + TestUtils.waitForCondition(() -> testStoreMetricKeyValueByType(STREAM_STORE_IN_MEMORY_LRU_STATE_METRICS, errorMessage), 10000, () -> "testStoreMetricKeyValueByType:" + STREAM_STORE_IN_MEMORY_LRU_STATE_METRICS + " -> " + errorMessage.toString()); + TestUtils.waitForCondition(() -> testStoreMetricKeyValueByType(STREAM_STORE_ROCKSDB_STATE_METRICS, errorMessage), 10000, () -> "testStoreMetricKeyValueByType:" + STREAM_STORE_ROCKSDB_STATE_METRICS + " -> " + errorMessage.toString()); + + //metric level : Cache + TestUtils.waitForCondition(() -> testCacheMetric(errorMessage), 10000, () -> "testCacheMetric -> " + errorMessage.toString()); + + closeApplication(); + + // check all metrics de-registered + checkMetricDeregistration(); + } + + @Test + public void testStreamMetricOfWindowStore() throws Exception { + final StringBuilder errorMessage = new StringBuilder(); + stream2 = builder.stream(STREAM_INPUT, Consumed.with(Serdes.Integer(), Serdes.String())); + final KGroupedStream groupedStream = stream2.groupByKey(); + groupedStream.windowedBy(TimeWindows.of(Duration.ofMillis(50))) + .aggregate(() -> 0L, (aggKey, newValue, aggValue) -> aggValue, + Materialized.>as(TIME_WINDOWED_AGGREGATED_STREAM_STORE) + .withValueSerde(Serdes.Long())); + + startApplication(); + + // metric level : Store (window) + TestUtils.waitForCondition(() -> testStoreMetricWindow(errorMessage), 10000, () -> "testStoreMetricWindow -> " + errorMessage.toString()); + + closeApplication(); + + // check all metrics de-registered + checkMetricDeregistration(); + } + + @Test + public void testStreamMetricOfSessionStore() throws Exception { + final StringBuilder errorMessage = new StringBuilder(); + stream2 = builder.stream(STREAM_INPUT, Consumed.with(Serdes.Integer(), Serdes.String())); + final KGroupedStream groupedStream = stream2.groupByKey(); + groupedStream.windowedBy(SessionWindows.with(Duration.ofMillis(50))) + .aggregate(() -> 0L, (aggKey, newValue, aggValue) -> aggValue, (aggKey, leftAggValue, rightAggValue) -> leftAggValue, + Materialized.>as(SESSION_AGGREGATED_STREAM_STORE) + .withValueSerde(Serdes.Long())); + + startApplication(); + + // metric level : Store (session) + TestUtils.waitForCondition(() -> testStoreMetricSession(errorMessage), 10000, () -> "testStoreMetricSession -> " + errorMessage.toString()); + + closeApplication(); + + // check all metrics de-registered + checkMetricDeregistration(); + } + + private boolean testThreadMetric(final StringBuilder errorMessage) { + errorMessage.setLength(0); + try { + final List listMetricThread = new ArrayList(kafkaStreams.metrics().values()).stream().filter(m -> m.metricName().group().equals(STREAM_THREAD_NODE_METRICS)).collect(Collectors.toList()); + testMetricByName(listMetricThread, COMMIT_LATENCY_AVG, 1); + testMetricByName(listMetricThread, COMMIT_LATENCY_MAX, 1); + testMetricByName(listMetricThread, POLL_LATENCY_AVG, 1); + testMetricByName(listMetricThread, POLL_LATENCY_MAX, 1); + testMetricByName(listMetricThread, PROCESS_LATENCY_AVG, 1); + testMetricByName(listMetricThread, PROCESS_LATENCY_MAX, 1); + testMetricByName(listMetricThread, PUNCTUATE_LATENCY_AVG, 1); + testMetricByName(listMetricThread, PUNCTUATE_LATENCY_MAX, 1); + testMetricByName(listMetricThread, COMMIT_RATE, 1); + testMetricByName(listMetricThread, COMMIT_TOTAL, 1); + testMetricByName(listMetricThread, POLL_RATE, 1); + testMetricByName(listMetricThread, POLL_TOTAL, 1); + testMetricByName(listMetricThread, PROCESS_RATE, 1); + testMetricByName(listMetricThread, PROCESS_TOTAL, 1); + testMetricByName(listMetricThread, PUNCTUATE_RATE, 1); + testMetricByName(listMetricThread, PUNCTUATE_TOTAL, 1); + testMetricByName(listMetricThread, TASK_CREATED_RATE, 1); + testMetricByName(listMetricThread, TASK_CREATED_TOTAL, 1); + testMetricByName(listMetricThread, TASK_CLOSED_RATE, 1); + testMetricByName(listMetricThread, TASK_CLOSED_TOTAL, 1); + testMetricByName(listMetricThread, SKIPPED_RECORDS_RATE, 1); + testMetricByName(listMetricThread, SKIPPED_RECORDS_TOTAL, 1); + return true; + } catch (final Throwable e) { + errorMessage.append(e.getMessage()); + return false; + } + } + + private boolean testTaskMetric(final StringBuilder errorMessage) { + errorMessage.setLength(0); + try { + final List listMetricTask = new ArrayList(kafkaStreams.metrics().values()).stream().filter(m -> m.metricName().group().equals(STREAM_TASK_NODE_METRICS)).collect(Collectors.toList()); + testMetricByName(listMetricTask, COMMIT_LATENCY_AVG, 5); + testMetricByName(listMetricTask, COMMIT_LATENCY_MAX, 5); + testMetricByName(listMetricTask, COMMIT_RATE, 5); + testMetricByName(listMetricTask, COMMIT_TOTAL, 5); + testMetricByName(listMetricTask, RECORD_LATENESS_AVG, 4); + testMetricByName(listMetricTask, RECORD_LATENESS_MAX, 4); + return true; + } catch (final Throwable e) { + errorMessage.append(e.getMessage()); + return false; + } + } + + private boolean testProcessorMetric(final StringBuilder errorMessage) { + errorMessage.setLength(0); + try { + final List listMetricProcessor = new ArrayList(kafkaStreams.metrics().values()).stream().filter(m -> m.metricName().group().equals(STREAM_PROCESSOR_NODE_METRICS)).collect(Collectors.toList()); + testMetricByName(listMetricProcessor, PROCESS_LATENCY_AVG, 18); + testMetricByName(listMetricProcessor, PROCESS_LATENCY_MAX, 18); + testMetricByName(listMetricProcessor, PUNCTUATE_LATENCY_AVG, 18); + testMetricByName(listMetricProcessor, PUNCTUATE_LATENCY_MAX, 18); + testMetricByName(listMetricProcessor, CREATE_LATENCY_AVG, 18); + testMetricByName(listMetricProcessor, CREATE_LATENCY_MAX, 18); + testMetricByName(listMetricProcessor, DESTROY_LATENCY_AVG, 18); + testMetricByName(listMetricProcessor, DESTROY_LATENCY_MAX, 18); + testMetricByName(listMetricProcessor, PROCESS_RATE, 18); + testMetricByName(listMetricProcessor, PROCESS_TOTAL, 18); + testMetricByName(listMetricProcessor, PUNCTUATE_RATE, 18); + testMetricByName(listMetricProcessor, PUNCTUATE_TOTAL, 18); + testMetricByName(listMetricProcessor, CREATE_RATE, 18); + testMetricByName(listMetricProcessor, CREATE_TOTAL, 18); + testMetricByName(listMetricProcessor, DESTROY_RATE, 18); + testMetricByName(listMetricProcessor, DESTROY_TOTAL, 18); + testMetricByName(listMetricProcessor, FORWARD_TOTAL, 18); + return true; + } catch (final Throwable e) { + errorMessage.append(e.getMessage()); + return false; + } + } + + private boolean testStoreMetricWindow(final StringBuilder errorMessage) { + errorMessage.setLength(0); + try { + final List listMetricStore = new ArrayList(kafkaStreams.metrics().values()).stream() + .filter(m -> m.metricName().group().equals(STREAM_STORE_WINDOW_ROCKSDB_STATE_METRICS)) + .collect(Collectors.toList()); + testMetricByName(listMetricStore, PUT_LATENCY_AVG, 2); + testMetricByName(listMetricStore, PUT_LATENCY_MAX, 2); + testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_AVG, 0); + testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_MAX, 0); + testMetricByName(listMetricStore, GET_LATENCY_AVG, 0); + testMetricByName(listMetricStore, GET_LATENCY_MAX, 0); + testMetricByName(listMetricStore, DELETE_LATENCY_AVG, 0); + testMetricByName(listMetricStore, DELETE_LATENCY_MAX, 0); + testMetricByName(listMetricStore, PUT_ALL_LATENCY_AVG, 0); + testMetricByName(listMetricStore, PUT_ALL_LATENCY_MAX, 0); + testMetricByName(listMetricStore, ALL_LATENCY_AVG, 0); + testMetricByName(listMetricStore, ALL_LATENCY_MAX, 0); + testMetricByName(listMetricStore, RANGE_LATENCY_AVG, 0); + testMetricByName(listMetricStore, RANGE_LATENCY_MAX, 0); + testMetricByName(listMetricStore, FLUSH_LATENCY_AVG, 2); + testMetricByName(listMetricStore, FLUSH_LATENCY_MAX, 2); + testMetricByName(listMetricStore, RESTORE_LATENCY_AVG, 2); + testMetricByName(listMetricStore, RESTORE_LATENCY_MAX, 2); + testMetricByName(listMetricStore, PUT_RATE, 2); + testMetricByName(listMetricStore, PUT_TOTAL, 2); + testMetricByName(listMetricStore, PUT_IF_ABSENT_RATE, 0); + testMetricByName(listMetricStore, PUT_IF_ABSENT_TOTAL, 0); + testMetricByName(listMetricStore, GET_RATE, 0); + testMetricByName(listMetricStore, DELETE_RATE, 0); + testMetricByName(listMetricStore, DELETE_TOTAL, 0); + testMetricByName(listMetricStore, PUT_ALL_RATE, 0); + testMetricByName(listMetricStore, PUT_ALL_TOTAL, 0); + testMetricByName(listMetricStore, ALL_RATE, 0); + testMetricByName(listMetricStore, ALL_TOTAL, 0); + testMetricByName(listMetricStore, RANGE_RATE, 0); + testMetricByName(listMetricStore, RANGE_TOTAL, 0); + testMetricByName(listMetricStore, FLUSH_RATE, 2); + testMetricByName(listMetricStore, FLUSH_TOTAL, 2); + testMetricByName(listMetricStore, RESTORE_RATE, 2); + testMetricByName(listMetricStore, RESTORE_TOTAL, 2); + return true; + } catch (final Throwable e) { + errorMessage.append(e.getMessage()); + return false; + } + } + + private boolean testStoreMetricSession(final StringBuilder errorMessage) { + errorMessage.setLength(0); + try { + final List listMetricStore = new ArrayList(kafkaStreams.metrics().values()).stream() + .filter(m -> m.metricName().group().equals(STREAM_STORE_SESSION_ROCKSDB_STATE_METRICS)) + .collect(Collectors.toList()); + testMetricByName(listMetricStore, PUT_LATENCY_AVG, 2); + testMetricByName(listMetricStore, PUT_LATENCY_MAX, 2); + testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_AVG, 0); + testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_MAX, 0); + testMetricByName(listMetricStore, GET_LATENCY_AVG, 0); + testMetricByName(listMetricStore, GET_LATENCY_MAX, 0); + testMetricByName(listMetricStore, DELETE_LATENCY_AVG, 0); + testMetricByName(listMetricStore, DELETE_LATENCY_MAX, 0); + testMetricByName(listMetricStore, PUT_ALL_LATENCY_AVG, 0); + testMetricByName(listMetricStore, PUT_ALL_LATENCY_MAX, 0); + testMetricByName(listMetricStore, ALL_LATENCY_AVG, 0); + testMetricByName(listMetricStore, ALL_LATENCY_MAX, 0); + testMetricByName(listMetricStore, RANGE_LATENCY_AVG, 0); + testMetricByName(listMetricStore, RANGE_LATENCY_MAX, 0); + testMetricByName(listMetricStore, FLUSH_LATENCY_AVG, 2); + testMetricByName(listMetricStore, FLUSH_LATENCY_MAX, 2); + testMetricByName(listMetricStore, RESTORE_LATENCY_AVG, 2); + testMetricByName(listMetricStore, RESTORE_LATENCY_MAX, 2); + testMetricByName(listMetricStore, PUT_RATE, 2); + testMetricByName(listMetricStore, PUT_TOTAL, 2); + testMetricByName(listMetricStore, PUT_IF_ABSENT_RATE, 0); + testMetricByName(listMetricStore, PUT_IF_ABSENT_TOTAL, 0); + testMetricByName(listMetricStore, GET_RATE, 0); + testMetricByName(listMetricStore, DELETE_RATE, 0); + testMetricByName(listMetricStore, DELETE_TOTAL, 0); + testMetricByName(listMetricStore, PUT_ALL_RATE, 0); + testMetricByName(listMetricStore, PUT_ALL_TOTAL, 0); + testMetricByName(listMetricStore, ALL_RATE, 0); + testMetricByName(listMetricStore, ALL_TOTAL, 0); + testMetricByName(listMetricStore, RANGE_RATE, 0); + testMetricByName(listMetricStore, RANGE_TOTAL, 0); + testMetricByName(listMetricStore, FLUSH_RATE, 2); + testMetricByName(listMetricStore, FLUSH_TOTAL, 2); + testMetricByName(listMetricStore, RESTORE_RATE, 2); + testMetricByName(listMetricStore, RESTORE_TOTAL, 2); + return true; + } catch (final Throwable e) { + errorMessage.append(e.getMessage()); + return false; + } + } + + private boolean testStoreMetricKeyValueByType(final String storeType, final StringBuilder errorMessage) { + errorMessage.setLength(0); + try { + final List listMetricStore = new ArrayList(kafkaStreams.metrics().values()).stream() + .filter(m -> m.metricName().group().equals(storeType)) + .collect(Collectors.toList()); + testMetricByName(listMetricStore, PUT_LATENCY_AVG, 2); + testMetricByName(listMetricStore, PUT_LATENCY_MAX, 2); + testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_AVG, 2); + testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_MAX, 2); + testMetricByName(listMetricStore, GET_LATENCY_AVG, 2); + testMetricByName(listMetricStore, GET_LATENCY_MAX, 2); + testMetricByName(listMetricStore, DELETE_LATENCY_AVG, 2); + testMetricByName(listMetricStore, DELETE_LATENCY_MAX, 2); + testMetricByName(listMetricStore, PUT_ALL_LATENCY_AVG, 2); + testMetricByName(listMetricStore, PUT_ALL_LATENCY_MAX, 2); + testMetricByName(listMetricStore, ALL_LATENCY_AVG, 2); + testMetricByName(listMetricStore, ALL_LATENCY_MAX, 2); + testMetricByName(listMetricStore, RANGE_LATENCY_AVG, 2); + testMetricByName(listMetricStore, RANGE_LATENCY_MAX, 2); + testMetricByName(listMetricStore, FLUSH_LATENCY_AVG, 2); + testMetricByName(listMetricStore, FLUSH_LATENCY_MAX, 2); + testMetricByName(listMetricStore, RESTORE_LATENCY_AVG, 2); + testMetricByName(listMetricStore, RESTORE_LATENCY_MAX, 2); + testMetricByName(listMetricStore, PUT_RATE, 2); + testMetricByName(listMetricStore, PUT_TOTAL, 2); + testMetricByName(listMetricStore, PUT_IF_ABSENT_RATE, 2); + testMetricByName(listMetricStore, PUT_IF_ABSENT_TOTAL, 2); + testMetricByName(listMetricStore, GET_RATE, 2); + testMetricByName(listMetricStore, DELETE_RATE, 2); + testMetricByName(listMetricStore, DELETE_TOTAL, 2); + testMetricByName(listMetricStore, PUT_ALL_RATE, 2); + testMetricByName(listMetricStore, PUT_ALL_TOTAL, 2); + testMetricByName(listMetricStore, ALL_RATE, 2); + testMetricByName(listMetricStore, ALL_TOTAL, 2); + testMetricByName(listMetricStore, RANGE_RATE, 2); + testMetricByName(listMetricStore, RANGE_TOTAL, 2); + testMetricByName(listMetricStore, FLUSH_RATE, 2); + testMetricByName(listMetricStore, FLUSH_TOTAL, 2); + testMetricByName(listMetricStore, RESTORE_RATE, 2); + testMetricByName(listMetricStore, RESTORE_TOTAL, 2); + return true; + } catch (final Throwable e) { + errorMessage.append(e.getMessage()); + return false; + } + } + + private boolean testCacheMetric(final StringBuilder errorMessage) { + errorMessage.setLength(0); + try { + final List listMetricCache = new ArrayList(kafkaStreams.metrics().values()).stream().filter(m -> m.metricName().group().equals(STREAM_CACHE_NODE_METRICS)).collect(Collectors.toList()); + testMetricByName(listMetricCache, HIT_RATIO_AVG, 6); + testMetricByName(listMetricCache, HIT_RATIO_MIN, 6); + testMetricByName(listMetricCache, HIT_RATIO_MAX, 6); + return true; + } catch (final Throwable e) { + errorMessage.append(e.getMessage()); + return false; + } + } + + private void testMetricByName(final List listMetric, final String metricName, final int numMetric) { + final List metrics = listMetric.stream().filter(m -> m.metricName().name().equals(metricName)).collect(Collectors.toList()); + Assert.assertEquals("Size of metrics of type:'" + metricName + "' must be equal to:" + numMetric + " but it's equal to " + metrics.size(), numMetric, metrics.size()); + for (final Metric m : metrics) { + Assert.assertNotNull("Metric:'" + m.metricName() + "' must be not null", m.metricValue()); + } + } +} \ No newline at end of file From 1f692bdf53af4a80b7fd256de4e94ff1d17fc861 Mon Sep 17 00:00:00 2001 From: "Matthias J. Sax" Date: Thu, 21 Mar 2019 23:53:56 -0700 Subject: [PATCH 0048/1071] KAFKA-8142: Fix NPE for nulls in Headers (#6484) Reviewers: Bill Bejeck , Guozhang Wang --- .../header/internals/RecordHeaders.java | 12 ++- .../header/internals/RecordHeadersTest.java | 21 ++-- .../internals/ProcessorRecordContext.java | 5 +- .../internals/ProcessorRecordContextTest.java | 98 +++++++++++++++++++ 4 files changed, 122 insertions(+), 14 deletions(-) create mode 100644 streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorRecordContextTest.java diff --git a/clients/src/main/java/org/apache/kafka/common/header/internals/RecordHeaders.java b/clients/src/main/java/org/apache/kafka/common/header/internals/RecordHeaders.java index 577e758348d7a..5801bed99cd8c 100644 --- a/clients/src/main/java/org/apache/kafka/common/header/internals/RecordHeaders.java +++ b/clients/src/main/java/org/apache/kafka/common/header/internals/RecordHeaders.java @@ -16,16 +16,17 @@ */ package org.apache.kafka.common.header.internals; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.Headers; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.utils.AbstractIterator; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Iterator; import java.util.List; - -import org.apache.kafka.common.header.Header; -import org.apache.kafka.common.header.Headers; -import org.apache.kafka.common.record.Record; -import org.apache.kafka.common.utils.AbstractIterator; +import java.util.Objects; public class RecordHeaders implements Headers { @@ -61,6 +62,7 @@ public RecordHeaders(Iterable
headers) { @Override public Headers add(Header header) throws IllegalStateException { + Objects.requireNonNull(header, "Header cannot be null."); canWrite(); headers.add(header); return this; diff --git a/clients/src/test/java/org/apache/kafka/common/header/internals/RecordHeadersTest.java b/clients/src/test/java/org/apache/kafka/common/header/internals/RecordHeadersTest.java index 39c1c9c9a7bce..5b9f95ea91f18 100644 --- a/clients/src/test/java/org/apache/kafka/common/header/internals/RecordHeadersTest.java +++ b/clients/src/test/java/org/apache/kafka/common/header/internals/RecordHeadersTest.java @@ -16,19 +16,19 @@ */ package org.apache.kafka.common.header.internals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.Headers; +import org.junit.Test; import java.io.IOException; import java.util.Arrays; import java.util.Iterator; -import org.apache.kafka.common.header.Header; -import org.apache.kafka.common.header.Headers; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class RecordHeadersTest { @@ -206,6 +206,11 @@ public void testNew() throws IOException { assertEquals(2, getCount(newHeaders)); } + @Test(expected = NullPointerException.class) + public void shouldThrowNpeWhenAddingNullHeader() { + new RecordHeaders().add(null); + } + private int getCount(Headers headers) { int count = 0; Iterator
headerIterator = headers.iterator(); diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorRecordContext.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorRecordContext.java index 7e1466eb32d09..00012746bd02d 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorRecordContext.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorRecordContext.java @@ -90,7 +90,10 @@ public long sizeBytes() { if (headers != null) { for (final Header header : headers) { size += header.key().toCharArray().length; - size += header.value().length; + final byte[] value = header.value(); + if (value != null) { + size += value.length; + } } } return size; diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorRecordContextTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorRecordContextTest.java new file mode 100644 index 0000000000000..1ea646fce2f3e --- /dev/null +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorRecordContextTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.processor.internals; + +import org.apache.kafka.common.header.Headers; +import org.apache.kafka.common.header.internals.RecordHeaders; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class ProcessorRecordContextTest { + // timestamp + offset + partition: 8 + 8 + 4 + private final static long MIN_SIZE = 20L; + + @Test + public void shouldEstimateNullTopicAndNullHeadersAsZeroLength() { + final Headers headers = new RecordHeaders(); + final ProcessorRecordContext context = new ProcessorRecordContext( + 42L, + 73L, + 0, + null, + null + ); + + assertEquals(MIN_SIZE, context.sizeBytes()); + } + + @Test + public void shouldEstimateEmptyHeaderAsZeroLength() { + final ProcessorRecordContext context = new ProcessorRecordContext( + 42L, + 73L, + 0, + null, + new RecordHeaders() + ); + + assertEquals(MIN_SIZE, context.sizeBytes()); + } + + @Test + public void shouldEstimateTopicLength() { + final ProcessorRecordContext context = new ProcessorRecordContext( + 42L, + 73L, + 0, + "topic", + null + ); + + assertEquals(MIN_SIZE + 5L, context.sizeBytes()); + } + + @Test + public void shouldEstimateHeadersLength() { + final Headers headers = new RecordHeaders(); + headers.add("header-key", "header-value".getBytes()); + final ProcessorRecordContext context = new ProcessorRecordContext( + 42L, + 73L, + 0, + null, + headers + ); + + assertEquals(MIN_SIZE + 10L + 12L, context.sizeBytes()); + } + + @Test + public void shouldEstimateNullValueInHeaderAsZero() { + final Headers headers = new RecordHeaders(); + headers.add("header-key", null); + final ProcessorRecordContext context = new ProcessorRecordContext( + 42L, + 73L, + 0, + null, + headers + ); + + assertEquals(MIN_SIZE + 10L, context.sizeBytes()); + } +} From 1deb072f56ac07b3233089ccaab8271d75f483c5 Mon Sep 17 00:00:00 2001 From: Radai Rosenblatt Date: Fri, 22 Mar 2019 01:32:45 -0700 Subject: [PATCH 0049/1071] MINOR: Avoid unnecessary collection copy in MetadataCache (#6397) `map` was being used to convert `Iterable[Integer]` to `Iterable[Int`]. That operation represented 11% of total CPU time measured under load for us. We also expect a positive impact on GC. Reviewers: Joel Koshy , Ismael Juma --- core/src/main/scala/kafka/server/MetadataCache.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/kafka/server/MetadataCache.scala b/core/src/main/scala/kafka/server/MetadataCache.scala index ec5a2b9ed38b1..647ccb13b04a4 100755 --- a/core/src/main/scala/kafka/server/MetadataCache.scala +++ b/core/src/main/scala/kafka/server/MetadataCache.scala @@ -51,9 +51,10 @@ class MetadataCache(brokerId: Int) extends Logging { private val stateChangeLogger = new StateChangeLogger(brokerId, inControllerContext = false, None) // This method is the main hotspot when it comes to the performance of metadata requests, - // we should be careful about adding additional logic here. + // we should be careful about adding additional logic here. Relatedly, `brokers` is + // `Iterable[Integer]` instead of `Iterable[Int]` to avoid a collection copy. // filterUnavailableEndpoints exists to support v0 MetadataResponses - private def getEndpoints(snapshot: MetadataSnapshot, brokers: Iterable[Int], listenerName: ListenerName, filterUnavailableEndpoints: Boolean): Seq[Node] = { + private def getEndpoints(snapshot: MetadataSnapshot, brokers: Iterable[java.lang.Integer], listenerName: ListenerName, filterUnavailableEndpoints: Boolean): Seq[Node] = { val result = new mutable.ArrayBuffer[Node](math.min(snapshot.aliveBrokers.size, brokers.size)) brokers.foreach { brokerId => val endpoint = getAliveEndpoint(snapshot, brokerId, listenerName) match { @@ -76,9 +77,9 @@ class MetadataCache(brokerId: Int) extends Logging { val leaderBrokerId = partitionState.basePartitionState.leader val leaderEpoch = partitionState.basePartitionState.leaderEpoch val maybeLeader = getAliveEndpoint(snapshot, leaderBrokerId, listenerName) - val replicas = partitionState.basePartitionState.replicas.asScala.map(_.toInt) + val replicas = partitionState.basePartitionState.replicas.asScala val replicaInfo = getEndpoints(snapshot, replicas, listenerName, errorUnavailableEndpoints) - val offlineReplicaInfo = getEndpoints(snapshot, partitionState.offlineReplicas.asScala.map(_.toInt), listenerName, errorUnavailableEndpoints) + val offlineReplicaInfo = getEndpoints(snapshot, partitionState.offlineReplicas.asScala, listenerName, errorUnavailableEndpoints) maybeLeader match { case None => @@ -94,7 +95,7 @@ class MetadataCache(brokerId: Int) extends Logging { offlineReplicaInfo.asJava) case Some(leader) => - val isr = partitionState.basePartitionState.isr.asScala.map(_.toInt) + val isr = partitionState.basePartitionState.isr.asScala val isrInfo = getEndpoints(snapshot, isr, listenerName, errorUnavailableEndpoints) if (replicaInfo.size < replicas.size) { From 1acae2a67c8fce071bfe7a373187bdff209f1705 Mon Sep 17 00:00:00 2001 From: Bill Bejeck Date: Fri, 22 Mar 2019 09:27:58 -0400 Subject: [PATCH 0050/1071] MINOR: Clean up ThreadCacheTest (#6485) Minor clean up ofThreadCacheTest Reviewers: Guozhang Wang , Matthias J. Sax --- .../state/internals/ThreadCacheTest.java | 96 +++++-------------- 1 file changed, 22 insertions(+), 74 deletions(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/ThreadCacheTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/ThreadCacheTest.java index 5882ee4b94a6f..c9c5789516232 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/ThreadCacheTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/ThreadCacheTest.java @@ -146,7 +146,7 @@ static int memoryCacheEntrySize(final byte[] key, final byte[] value, final Stri } @Test - public void evict() throws IOException { + public void evict() { final List> received = new ArrayList<>(); final List> expected = Collections.singletonList( new KeyValue<>("K1", "V1")); @@ -161,14 +161,10 @@ public void evict() throws IOException { final ThreadCache cache = new ThreadCache(logContext, memoryCacheEntrySize(kv.key.getBytes(), kv.value.getBytes(), ""), new MockStreamsMetrics(new Metrics())); - cache.addDirtyEntryFlushListener(namespace, new ThreadCache.DirtyEntryFlushListener() { - @Override - public void apply(final List dirty) { - for (final ThreadCache.DirtyEntry dirtyEntry : dirty) { - received.add(new KeyValue<>(dirtyEntry.key().toString(), new String(dirtyEntry.newValue()))); - } + cache.addDirtyEntryFlushListener(namespace, dirty -> { + for (final ThreadCache.DirtyEntry dirtyEntry : dirty) { + received.add(new KeyValue<>(dirtyEntry.key().toString(), new String(dirtyEntry.newValue()))); } - }); for (final KeyValue kvToInsert : toInsert) { @@ -200,12 +196,7 @@ public void shouldNotFlushAfterDelete() { final Bytes key = Bytes.wrap(new byte[]{0}); final ThreadCache cache = new ThreadCache(logContext, 10000L, new MockStreamsMetrics(new Metrics())); final List received = new ArrayList<>(); - cache.addDirtyEntryFlushListener(namespace, new ThreadCache.DirtyEntryFlushListener() { - @Override - public void apply(final List dirty) { - received.addAll(dirty); - } - }); + cache.addDirtyEntryFlushListener(namespace, received::addAll); cache.put(namespace, key, dirtyEntry(key.get())); assertEquals(key.get(), cache.delete(namespace, key).value()); @@ -298,12 +289,8 @@ public void shouldPeekAndIterateOverRange() { public void shouldSkipEntriesWhereValueHasBeenEvictedFromCache() { final int entrySize = memoryCacheEntrySize(new byte[1], new byte[1], ""); final ThreadCache cache = new ThreadCache(logContext, entrySize * 5, new MockStreamsMetrics(new Metrics())); - cache.addDirtyEntryFlushListener(namespace, new ThreadCache.DirtyEntryFlushListener() { - @Override - public void apply(final List dirty) { + cache.addDirtyEntryFlushListener(namespace, dirty -> { }); - } - }); final byte[][] bytes = {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}}; for (int i = 0; i < 5; i++) { cache.put(namespace, Bytes.wrap(bytes[i]), dirtyEntry(bytes[i])); @@ -322,12 +309,9 @@ public void apply(final List dirty) { public void shouldFlushDirtyEntriesForNamespace() { final ThreadCache cache = new ThreadCache(logContext, 100000, new MockStreamsMetrics(new Metrics())); final List received = new ArrayList<>(); - cache.addDirtyEntryFlushListener(namespace1, new ThreadCache.DirtyEntryFlushListener() { - @Override - public void apply(final List dirty) { - for (final ThreadCache.DirtyEntry dirtyEntry : dirty) { - received.add(dirtyEntry.key().get()); - } + cache.addDirtyEntryFlushListener(namespace1, dirty -> { + for (final ThreadCache.DirtyEntry dirtyEntry : dirty) { + received.add(dirtyEntry.key().get()); } }); final List expected = Arrays.asList(new byte[]{0}, new byte[]{1}, new byte[]{2}); @@ -344,12 +328,9 @@ public void apply(final List dirty) { public void shouldNotFlushCleanEntriesForNamespace() { final ThreadCache cache = new ThreadCache(logContext, 100000, new MockStreamsMetrics(new Metrics())); final List received = new ArrayList<>(); - cache.addDirtyEntryFlushListener(namespace1, new ThreadCache.DirtyEntryFlushListener() { - @Override - public void apply(final List dirty) { - for (final ThreadCache.DirtyEntry dirtyEntry : dirty) { - received.add(dirtyEntry.key().get()); - } + cache.addDirtyEntryFlushListener(namespace1, dirty -> { + for (final ThreadCache.DirtyEntry dirtyEntry : dirty) { + received.add(dirtyEntry.key().get()); } }); final List toInsert = Arrays.asList(new byte[]{0}, new byte[]{1}, new byte[]{2}); @@ -366,12 +347,7 @@ public void apply(final List dirty) { private void shouldEvictImmediatelyIfCacheSizeIsZeroOrVerySmall(final ThreadCache cache) { final List received = new ArrayList<>(); - cache.addDirtyEntryFlushListener(namespace, new ThreadCache.DirtyEntryFlushListener() { - @Override - public void apply(final List dirty) { - received.addAll(dirty); - } - }); + cache.addDirtyEntryFlushListener(namespace, received::addAll); cache.put(namespace, Bytes.wrap(new byte[]{0}), dirtyEntry(new byte[]{0})); assertEquals(1, received.size()); @@ -396,12 +372,7 @@ public void shouldEvictImmediatelyIfCacheSizeIsZero() { public void shouldEvictAfterPutAll() { final List received = new ArrayList<>(); final ThreadCache cache = new ThreadCache(logContext, 1, new MockStreamsMetrics(new Metrics())); - cache.addDirtyEntryFlushListener(namespace, new ThreadCache.DirtyEntryFlushListener() { - @Override - public void apply(final List dirty) { - received.addAll(dirty); - } - }); + cache.addDirtyEntryFlushListener(namespace, received::addAll); cache.putAll(namespace, Arrays.asList(KeyValue.pair(Bytes.wrap(new byte[]{0}), dirtyEntry(new byte[]{5})), KeyValue.pair(Bytes.wrap(new byte[]{1}), dirtyEntry(new byte[]{6})))); @@ -425,12 +396,7 @@ public void shouldPutAll() { public void shouldNotForwardCleanEntryOnEviction() { final ThreadCache cache = new ThreadCache(logContext, 0, new MockStreamsMetrics(new Metrics())); final List received = new ArrayList<>(); - cache.addDirtyEntryFlushListener(namespace, new ThreadCache.DirtyEntryFlushListener() { - @Override - public void apply(final List dirty) { - received.addAll(dirty); - } - }); + cache.addDirtyEntryFlushListener(namespace, received::addAll); cache.put(namespace, Bytes.wrap(new byte[]{1}), cleanEntry(new byte[]{0})); assertEquals(0, received.size()); } @@ -448,12 +414,7 @@ public void shouldPutIfAbsent() { public void shouldEvictAfterPutIfAbsent() { final List received = new ArrayList<>(); final ThreadCache cache = new ThreadCache(logContext, 1, new MockStreamsMetrics(new Metrics())); - cache.addDirtyEntryFlushListener(namespace, new ThreadCache.DirtyEntryFlushListener() { - @Override - public void apply(final List dirty) { - received.addAll(dirty); - } - }); + cache.addDirtyEntryFlushListener(namespace, received::addAll); cache.putIfAbsent(namespace, Bytes.wrap(new byte[]{0}), dirtyEntry(new byte[]{5})); cache.putIfAbsent(namespace, Bytes.wrap(new byte[]{1}), dirtyEntry(new byte[]{6})); @@ -468,26 +429,13 @@ public void shouldNotLoopForEverWhenEvictingAndCurrentCacheIsEmpty() { final int maxCacheSizeInBytes = 100; final ThreadCache threadCache = new ThreadCache(logContext, maxCacheSizeInBytes, new MockStreamsMetrics(new Metrics())); // trigger a put into another cache on eviction from "name" - threadCache.addDirtyEntryFlushListener(namespace, new ThreadCache.DirtyEntryFlushListener() { - @Override - public void apply(final List dirty) { - // put an item into an empty cache when the total cache size - // is already > than maxCacheSizeBytes - threadCache.put(namespace1, Bytes.wrap(new byte[]{0}), dirtyEntry(new byte[2])); - } - }); - threadCache.addDirtyEntryFlushListener(namespace1, new ThreadCache.DirtyEntryFlushListener() { - @Override - public void apply(final List dirty) { - // - } - }); - threadCache.addDirtyEntryFlushListener(namespace2, new ThreadCache.DirtyEntryFlushListener() { - @Override - public void apply(final List dirty) { - - } + threadCache.addDirtyEntryFlushListener(namespace, dirty -> { + // put an item into an empty cache when the total cache size + // is already > than maxCacheSizeBytes + threadCache.put(namespace1, Bytes.wrap(new byte[]{0}), dirtyEntry(new byte[2])); }); + threadCache.addDirtyEntryFlushListener(namespace1, dirty -> { }); + threadCache.addDirtyEntryFlushListener(namespace2, dirty -> { }); threadCache.put(namespace2, Bytes.wrap(new byte[]{1}), dirtyEntry(new byte[1])); threadCache.put(namespace, Bytes.wrap(new byte[]{1}), dirtyEntry(new byte[1])); From 860e9579994aa00dbbbeadb3ab10eec0cb854ddf Mon Sep 17 00:00:00 2001 From: Brian Bushree Date: Fri, 22 Mar 2019 11:50:00 -0700 Subject: [PATCH 0051/1071] MINOR: list-topics should not require topic param `kafka.list_topics(...)` should not require a topic parameter Author: Brian Bushree Reviewers: Ewen Cheslack-Postava Closes #6367 from brianbushree/list-topics-no-topic --- tests/kafkatest/services/kafka/kafka.py | 2 +- tests/kafkatest/tests/streams/streams_broker_bounce_test.py | 2 +- tests/kafkatest/tests/streams/streams_upgrade_test.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/kafkatest/services/kafka/kafka.py b/tests/kafkatest/services/kafka/kafka.py index a59bb71f62fd7..72d84a0ee734c 100644 --- a/tests/kafkatest/services/kafka/kafka.py +++ b/tests/kafkatest/services/kafka/kafka.py @@ -387,7 +387,7 @@ def describe_topic(self, topic, node=None): output += line return output - def list_topics(self, topic, node=None): + def list_topics(self, topic=None, node=None): if node is None: node = self.nodes[0] cmd = "%s --zookeeper %s --list" % \ diff --git a/tests/kafkatest/tests/streams/streams_broker_bounce_test.py b/tests/kafkatest/tests/streams/streams_broker_bounce_test.py index 7859d699ee58e..3d6572dec6b27 100644 --- a/tests/kafkatest/tests/streams/streams_broker_bounce_test.py +++ b/tests/kafkatest/tests/streams/streams_broker_bounce_test.py @@ -139,7 +139,7 @@ def confirm_topics_on_all_brokers(self, expected_topic_set): # need to iterate over topic_list_generator as kafka.list_topics() # returns a python generator so values are fetched lazily # so we can't just compare directly we must iterate over what's returned - topic_list_generator = self.kafka.list_topics("placeholder", node) + topic_list_generator = self.kafka.list_topics(node=node) for topic in topic_list_generator: if topic in expected_topic_set: match_count += 1 diff --git a/tests/kafkatest/tests/streams/streams_upgrade_test.py b/tests/kafkatest/tests/streams/streams_upgrade_test.py index 6c2bca94b1a18..91f9052467479 100644 --- a/tests/kafkatest/tests/streams/streams_upgrade_test.py +++ b/tests/kafkatest/tests/streams/streams_upgrade_test.py @@ -642,7 +642,7 @@ def confirm_topics_on_all_brokers(self, expected_topic_set): # need to iterate over topic_list_generator as kafka.list_topics() # returns a python generator so values are fetched lazily # so we can't just compare directly we must iterate over what's returned - topic_list_generator = self.kafka.list_topics("placeholder", node) + topic_list_generator = self.kafka.list_topics(node=node) for topic in topic_list_generator: if topic in expected_topic_set: match_count += 1 From f304667df089b9e39850fe401ddfd0c457b1bb67 Mon Sep 17 00:00:00 2001 From: Colin Patrick McCabe Date: Sat, 23 Mar 2019 12:14:35 -0700 Subject: [PATCH 0052/1071] MINOR: fix message protocol help text for ElectPreferredLeadersResult (#6479) Reviewers: Jun Rao --- .../kafka/clients/admin/ElectPreferredLeadersResult.java | 8 +++++--- .../common/message/ElectPreferredLeadersResponse.json | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java index c76336a7cbc0a..963c5f1ad2924 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java @@ -62,10 +62,12 @@ public void accept(Map topicPartitions, Throwable thro result.completeExceptionally(new UnknownTopicOrPartitionException( "Preferred leader election for partition \"" + partition + "\" was not attempted")); + } else if (partitions == null && topicPartitions.isEmpty()) { + // If partitions is null, we requested information about all partitions. In + // that case, if topicPartitions is empty, that indicates a + // CLUSTER_AUTHORIZATION_FAILED error. + result.completeExceptionally(Errors.CLUSTER_AUTHORIZATION_FAILED.exception()); } else { - if (partitions == null && topicPartitions.isEmpty()) { - result.completeExceptionally(Errors.CLUSTER_AUTHORIZATION_FAILED.exception()); - } ApiException exception = topicPartitions.get(partition).exception(); if (exception == null) { result.complete(null); diff --git a/clients/src/main/resources/common/message/ElectPreferredLeadersResponse.json b/clients/src/main/resources/common/message/ElectPreferredLeadersResponse.json index f34599cf03f39..491bd03284451 100644 --- a/clients/src/main/resources/common/message/ElectPreferredLeadersResponse.json +++ b/clients/src/main/resources/common/message/ElectPreferredLeadersResponse.json @@ -22,7 +22,7 @@ { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, { "name": "ReplicaElectionResults", "type": "[]ReplicaElectionResult", "versions": "0+", - "about": "The error code, or 0 if there was no error.", "fields": [ + "about": "The election results, or an empty array if the requester did not have permission and the request asks for all partitions.", "fields": [ { "name": "Topic", "type": "string", "versions": "0+", "about": "The topic name" }, { "name": "PartitionResult", "type": "[]PartitionResult", "versions": "0+", @@ -36,4 +36,4 @@ ]} ]} ] -} \ No newline at end of file +} From 82615256ffcb97b74eee0a81850421a3c0e14a68 Mon Sep 17 00:00:00 2001 From: "Matthias J. Sax" Date: Sat, 23 Mar 2019 15:48:24 -0700 Subject: [PATCH 0053/1071] MINOR: add MacOS requirement to Streams docs *More detailed description of your change, if necessary. The PR title and PR message become the squashed commit message, so use a separate comment to ping reviewers.* *Summary of testing strategy (including rationale) for the feature or bug fix. Unit and/or integration tests are expected for any behaviour change and system tests should be considered for larger changes.* Author: Matthias J. Sax Reviewers: Bill Bejeck Closes #6490 from mjsax/minor-streams-docs-rocksdb --- docs/streams/upgrade-guide.html | 2 ++ docs/upgrade.html | 1 + 2 files changed, 3 insertions(+) diff --git a/docs/streams/upgrade-guide.html b/docs/streams/upgrade-guide.html index 9071dc2eefa15..96893818aae1d 100644 --- a/docs/streams/upgrade-guide.html +++ b/docs/streams/upgrade-guide.html @@ -58,6 +58,8 @@

Upgrade Guide and API Changes

For Kafka Streams 0.10.0, broker version 0.10.0 or higher is required.

+

Since 2.2.0 release, Kafka Streams depends on a RocksDBs version that requires MacOS 10.13 or higher.

+

Another important thing to keep in mind: in deprecated KStreamBuilder class, when a KTable is created from a source topic via KStreamBuilder.table(), its materialized state store will reuse the source topic as its changelog topic for restoring, and will disable logging to avoid appending new updates to the source topic; in the StreamsBuilder class introduced in 1.0, this behavior was changed diff --git a/docs/upgrade.html b/docs/upgrade.html index 2ba7fd3915737..86bf5dd3b5ad9 100644 --- a/docs/upgrade.html +++ b/docs/upgrade.html @@ -28,6 +28,7 @@

Notable changes in 2 KIP-289.
  • The bin/kafka-topics.sh command line tool is now able to connect directly to brokers with --bootstrap-server instead of zookeeper. The old --zookeeper option is still available for now. Please read KIP-377 for more information.
    • +
  • Kafka Streams depends on a newer version of RocksDBs that requires MacOS 10.13 or higher.

    • Upgrading from 0.8.x, 0.9.x, 0.10.0.x, 0.10.1.x, 0.10.2.x, 0.11.0.x, 1.0.x, 1.1.x, or 2.0.0 to 2.1.0

    From c74acb24eb1da5d16cc2721a63931cd87f79ec66 Mon Sep 17 00:00:00 2001 From: Bill Bejeck Date: Sun, 24 Mar 2019 12:47:58 -0400 Subject: [PATCH 0054/1071] MINOR: Remove line for testing repartition topic name (#6488) With KIP-307 joined.name() is deprecated plus we don't need to test for repartition topic names. Reviewers: Matthias J. Sax --- .../org/apache/kafka/streams/scala/kstream/JoinedTest.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/kstream/JoinedTest.scala b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/kstream/JoinedTest.scala index 9a96a811c31c3..f523e2057e502 100644 --- a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/kstream/JoinedTest.scala +++ b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/kstream/JoinedTest.scala @@ -42,6 +42,5 @@ class JoinedTest extends FlatSpec with Matchers { joined.keySerde.getClass shouldBe Serdes.String.getClass joined.valueSerde.getClass shouldBe Serdes.Long.getClass joined.otherValueSerde.getClass shouldBe Serdes.Integer.getClass - joined.name() shouldBe repartitionTopicName } } From e4cad353124478fd4034f18d2292ba62fb702924 Mon Sep 17 00:00:00 2001 From: Konstantine Karantasis Date: Mon, 25 Mar 2019 07:29:33 -0700 Subject: [PATCH 0055/1071] KAFKA-8014: Extend Connect integration tests to add and remove workers dynamically (#6342) Extend Connect's integration test framework to add or remove workers to EmbeddedConnectCluster, and choosing whether to fail the test on ungraceful service shutdown. Also added more JavaDoc and other minor improvements. Author: Konstantine Karantasis Reviewers: Arjun Satish , Randall Hauch Closes #6342 from kkonstantine/KAFKA-8014 --- checkstyle/import-control.xml | 1 + .../runtime/rest/entities/ServerInfo.java | 14 +- .../ConnectWorkerIntegrationTest.java | 177 +++++++++++++++ .../connect/integration/ConnectorHandle.java | 106 ++++++++- .../ErrorHandlingIntegrationTest.java | 2 +- .../ExampleConnectIntegrationTest.java | 69 +++++- .../integration/MonitorableSinkConnector.java | 44 +++- .../MonitorableSourceConnector.java | 160 +++++++++++++ .../kafka/connect/integration/TaskHandle.java | 98 +++++++- .../util/clusters/EmbeddedConnectCluster.java | 214 ++++++++++++++++-- .../clusters/UngracefulShutdownException.java | 38 ++++ .../connect/util/clusters/WorkerHandle.java | 103 +++++++++ 12 files changed, 962 insertions(+), 64 deletions(-) create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/UngracefulShutdownException.java create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/WorkerHandle.java diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index ffc9bf9b18822..f751aadbcae1a 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -381,6 +381,7 @@ + diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/entities/ServerInfo.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/entities/ServerInfo.java index a12751c7ae91e..e5c55533a31ab 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/entities/ServerInfo.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/entities/ServerInfo.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.connect.runtime.rest.entities; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.kafka.common.utils.AppInfoParser; @@ -24,12 +25,19 @@ public class ServerInfo { private final String commit; private final String kafkaClusterId; - public ServerInfo(String kafkaClusterId) { - this.version = AppInfoParser.getVersion(); - this.commit = AppInfoParser.getCommitId(); + @JsonCreator + private ServerInfo(@JsonProperty("version") String version, + @JsonProperty("commit") String commit, + @JsonProperty("kafka_cluster_id") String kafkaClusterId) { + this.version = version; + this.commit = commit; this.kafkaClusterId = kafkaClusterId; } + public ServerInfo(String kafkaClusterId) { + this(AppInfoParser.getVersion(), AppInfoParser.getCommitId(), kafkaClusterId); + } + @JsonProperty public String version() { return version; diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java new file mode 100644 index 0000000000000..09363cdd3afaf --- /dev/null +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.connect.integration; + +import org.apache.kafka.connect.runtime.AbstractStatus; +import org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo; +import org.apache.kafka.connect.storage.StringConverter; +import org.apache.kafka.connect.util.clusters.EmbeddedConnectCluster; +import org.apache.kafka.connect.util.clusters.WorkerHandle; +import org.apache.kafka.test.IntegrationTest; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; + +import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLASS_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.KEY_CONVERTER_CLASS_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.TASKS_MAX_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.VALUE_CONVERTER_CLASS_CONFIG; +import static org.apache.kafka.connect.runtime.WorkerConfig.OFFSET_COMMIT_INTERVAL_MS_CONFIG; +import static org.apache.kafka.test.TestUtils.waitForCondition; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Test simple operations on the workers of a Connect cluster. + */ +@Category(IntegrationTest.class) +public class ConnectWorkerIntegrationTest { + private static final Logger log = LoggerFactory.getLogger(ConnectWorkerIntegrationTest.class); + + private static final int NUM_TOPIC_PARTITIONS = 3; + private static final int CONNECTOR_SETUP_DURATION_MS = 15_000; + private static final int WORKER_SETUP_DURATION_MS = 20_000; + private static final int OFFSET_COMMIT_INTERVAL_MS = 30_000; + private static final int NUM_WORKERS = 3; + private static final String CONNECTOR_NAME = "simple-source"; + + private EmbeddedConnectCluster connect; + + @Before + public void setup() throws IOException { + // setup Connect worker properties + Map exampleWorkerProps = new HashMap<>(); + exampleWorkerProps.put(OFFSET_COMMIT_INTERVAL_MS_CONFIG, String.valueOf(OFFSET_COMMIT_INTERVAL_MS)); + + // setup Kafka broker properties + Properties exampleBrokerProps = new Properties(); + exampleBrokerProps.put("auto.create.topics.enable", String.valueOf(false)); + + // build a Connect cluster backed by Kafka and Zk + connect = new EmbeddedConnectCluster.Builder() + .name("connect-cluster") + .numWorkers(NUM_WORKERS) + .workerProps(exampleWorkerProps) + .brokerProps(exampleBrokerProps) + .maskExitProcedures(true) // true is the default, setting here as example + .build(); + + // start the clusters + connect.start(); + } + + @After + public void close() { + // stop all Connect, Kafka and Zk threads. + connect.stop(); + } + + /** + * Simple test case to add and then remove a worker from the embedded Connect cluster while + * running a simple source connector. + */ + @Test + public void testAddAndRemoveWorker() throws Exception { + int numTasks = 4; + // create test topic + connect.kafka().createTopic("test-topic", NUM_TOPIC_PARTITIONS); + + // setup up props for the sink connector + Map props = new HashMap<>(); + props.put(CONNECTOR_CLASS_CONFIG, MonitorableSourceConnector.class.getSimpleName()); + props.put(TASKS_MAX_CONFIG, String.valueOf(numTasks)); + props.put("throughput", String.valueOf(1)); + props.put("messages.per.poll", String.valueOf(10)); + props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + + waitForCondition(() -> assertWorkersUp(NUM_WORKERS).orElse(false), + WORKER_SETUP_DURATION_MS, "Initial group of workers did not start in time."); + + // start a source connector + connect.configureConnector(CONNECTOR_NAME, props); + + waitForCondition(() -> assertConnectorAndTasksRunning(CONNECTOR_NAME, numTasks).orElse(false), + CONNECTOR_SETUP_DURATION_MS, "Connector tasks did not start in time."); + + WorkerHandle extraWorker = connect.addWorker(); + + waitForCondition(() -> assertWorkersUp(NUM_WORKERS + 1).orElse(false), + WORKER_SETUP_DURATION_MS, "Expanded group of workers did not start in time."); + + waitForCondition(() -> assertConnectorAndTasksRunning(CONNECTOR_NAME, numTasks).orElse(false), + CONNECTOR_SETUP_DURATION_MS, "Connector tasks are not all in running state."); + + Set workers = connect.activeWorkers(); + assertTrue(workers.contains(extraWorker)); + + connect.removeWorker(extraWorker); + + waitForCondition(() -> assertWorkersUp(NUM_WORKERS).orElse(false) && !assertWorkersUp(NUM_WORKERS + 1).orElse(false), + WORKER_SETUP_DURATION_MS, "Group of workers did not shrink in time."); + + workers = connect.activeWorkers(); + assertFalse(workers.contains(extraWorker)); + } + + /** + * Confirm that the requested number of workers is up and running. + * + * @param numWorkers the number of online workers + * @return true if at least {@code numWorkers} are up; false otherwise + */ + private Optional assertWorkersUp(int numWorkers) { + try { + int numUp = connect.activeWorkers().size(); + return Optional.of(numUp >= numWorkers); + } catch (Exception e) { + log.error("Could not check active workers.", e); + return Optional.empty(); + } + } + + /** + * Confirm that a connector with an exact number of tasks is running. + * + * @param connectorName the connector + * @param numTasks the expected number of tasks + * @return true if the connector and tasks are in RUNNING state; false otherwise + */ + private Optional assertConnectorAndTasksRunning(String connectorName, int numTasks) { + try { + ConnectorStateInfo info = connect.connectorStatus(connectorName); + boolean result = info != null + && info.tasks().size() == numTasks + && info.connector().state().equals(AbstractStatus.State.RUNNING.toString()) + && info.tasks().stream().allMatch(s -> s.state().equals(AbstractStatus.State.RUNNING.toString())); + return Optional.of(result); + } catch (Exception e) { + log.error("Could not check connector state info.", e); + return Optional.empty(); + } + } +} diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorHandle.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorHandle.java index e59691b843d09..0df0f8cad21bc 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorHandle.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorHandle.java @@ -25,6 +25,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.stream.IntStream; /** * A handle to a connector executing in a Connect cluster. @@ -37,7 +38,9 @@ public class ConnectorHandle { private final Map taskHandles = new ConcurrentHashMap<>(); private CountDownLatch recordsRemainingLatch; + private CountDownLatch recordsToCommitLatch; private int expectedRecords = -1; + private int expectedCommits = -1; public ConnectorHandle(String connectorName) { this.connectorName = connectorName; @@ -54,6 +57,20 @@ public TaskHandle taskHandle(String taskId) { return taskHandles.computeIfAbsent(taskId, k -> new TaskHandle(this, taskId)); } + /** + * Get the connector's name corresponding to this handle. + * + * @return the connector's name + */ + public String name() { + return connectorName; + } + + /** + * Get the list of tasks handles monitored by this connector handle. + * + * @return the task handle list + */ public Collection tasks() { return taskHandles.values(); } @@ -69,13 +86,23 @@ public void deleteTask(String taskId) { } /** - * Set the number of expected records for this task. + * Set the number of expected records for this connector. + * + * @param expected number of records + */ + public void expectedRecords(int expected) { + expectedRecords = expected; + recordsRemainingLatch = new CountDownLatch(expected); + } + + /** + * Set the number of expected commits performed by this connector. * - * @param expectedRecords number of records + * @param expected number of commits */ - public void expectedRecords(int expectedRecords) { - this.expectedRecords = expectedRecords; - this.recordsRemainingLatch = new CountDownLatch(expectedRecords); + public void expectedCommits(int expected) { + expectedCommits = expected; + recordsToCommitLatch = new CountDownLatch(expected); } /** @@ -88,25 +115,80 @@ public void record() { } /** - * Wait for this task to receive the expected number of records. + * Record arrival of a batch of messages at the connector. * - * @param consumeMaxDurationMs max duration to wait for records + * @param batchSize the number of messages + */ + public void record(int batchSize) { + if (recordsRemainingLatch != null) { + IntStream.range(0, batchSize).forEach(i -> recordsRemainingLatch.countDown()); + } + } + + /** + * Record a message commit from the connector. + */ + public void commit() { + if (recordsToCommitLatch != null) { + recordsToCommitLatch.countDown(); + } + } + + /** + * Record commit on a batch of messages from the connector. + * + * @param batchSize the number of messages + */ + public void commit(int batchSize) { + if (recordsToCommitLatch != null) { + IntStream.range(0, batchSize).forEach(i -> recordsToCommitLatch.countDown()); + } + } + + /** + * Wait for this connector to meet the expected number of records as defined by {@code + * expectedRecords}. + * + * @param timeout max duration to wait for records * @throws InterruptedException if another threads interrupts this one while waiting for records */ - public void awaitRecords(int consumeMaxDurationMs) throws InterruptedException { + public void awaitRecords(int timeout) throws InterruptedException { if (recordsRemainingLatch == null || expectedRecords < 0) { - throw new IllegalStateException("expectedRecords() was not set for this task?"); + throw new IllegalStateException("expectedRecords() was not set for this connector?"); } - if (!recordsRemainingLatch.await(consumeMaxDurationMs, TimeUnit.MILLISECONDS)) { - String msg = String.format("Insufficient records seen by connector %s in %d millis. Records expected=%d, actual=%d", + if (!recordsRemainingLatch.await(timeout, TimeUnit.MILLISECONDS)) { + String msg = String.format( + "Insufficient records seen by connector %s in %d millis. Records expected=%d, actual=%d", connectorName, - consumeMaxDurationMs, + timeout, expectedRecords, expectedRecords - recordsRemainingLatch.getCount()); throw new DataException(msg); } } + /** + * Wait for this connector to meet the expected number of commits as defined by {@code + * expectedCommits}. + * + * @param timeout duration to wait for commits + * @throws InterruptedException if another threads interrupts this one while waiting for commits + */ + public void awaitCommits(int timeout) throws InterruptedException { + if (recordsToCommitLatch == null || expectedCommits < 0) { + throw new IllegalStateException("expectedCommits() was not set for this connector?"); + } + if (!recordsToCommitLatch.await(timeout, TimeUnit.MILLISECONDS)) { + String msg = String.format( + "Insufficient records committed by connector %s in %d millis. Records expected=%d, actual=%d", + connectorName, + timeout, + expectedCommits, + expectedCommits - recordsToCommitLatch.getCount()); + throw new DataException(msg); + } + } + @Override public String toString() { return "ConnectorHandle{" + diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ErrorHandlingIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ErrorHandlingIntegrationTest.java index 5f7cfc93082ea..67bcc748c6509 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ErrorHandlingIntegrationTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ErrorHandlingIntegrationTest.java @@ -106,7 +106,7 @@ public void testSkipRetryAndDLQWithHeaders() throws Exception { // setup connector config Map props = new HashMap<>(); - props.put(CONNECTOR_CLASS_CONFIG, "MonitorableSink"); + props.put(CONNECTOR_CLASS_CONFIG, MonitorableSinkConnector.class.getSimpleName()); props.put(TASKS_MAX_CONFIG, String.valueOf(NUM_TASKS)); props.put(TOPICS_CONFIG, "test-topic"); props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExampleConnectIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExampleConnectIntegrationTest.java index 0648e9ff59ac3..224d6ac9867e8 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExampleConnectIntegrationTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExampleConnectIntegrationTest.java @@ -40,6 +40,7 @@ import static org.apache.kafka.connect.runtime.WorkerConfig.OFFSET_COMMIT_INTERVAL_MS_CONFIG; import static org.apache.kafka.test.TestUtils.waitForCondition; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * An example integration test that demonstrates how to setup an integration test for Connect. @@ -54,9 +55,10 @@ public class ExampleConnectIntegrationTest { private static final int NUM_RECORDS_PRODUCED = 2000; private static final int NUM_TOPIC_PARTITIONS = 3; - private static final int CONSUME_MAX_DURATION_MS = 5000; + private static final int RECORD_TRANSFER_DURATION_MS = 5000; private static final int CONNECTOR_SETUP_DURATION_MS = 15000; private static final int NUM_TASKS = 3; + private static final int NUM_WORKERS = 3; private static final String CONNECTOR_NAME = "simple-conn"; private EmbeddedConnectCluster connect; @@ -66,16 +68,16 @@ public class ExampleConnectIntegrationTest { public void setup() throws IOException { // setup Connect worker properties Map exampleWorkerProps = new HashMap<>(); - exampleWorkerProps.put(OFFSET_COMMIT_INTERVAL_MS_CONFIG, "30000"); + exampleWorkerProps.put(OFFSET_COMMIT_INTERVAL_MS_CONFIG, String.valueOf(5_000)); // setup Kafka broker properties Properties exampleBrokerProps = new Properties(); exampleBrokerProps.put("auto.create.topics.enable", "false"); - // build a Connect cluster backed by Kakfa and Zk + // build a Connect cluster backed by Kafka and Zk connect = new EmbeddedConnectCluster.Builder() - .name("example-cluster") - .numWorkers(3) + .name("connect-cluster") + .numWorkers(NUM_WORKERS) .numBrokers(1) .workerProps(exampleWorkerProps) .brokerProps(exampleBrokerProps) @@ -93,7 +95,7 @@ public void close() { // delete connector handle RuntimeHandles.get().deleteConnector(CONNECTOR_NAME); - // stop all Connect, Kakfa and Zk threads. + // stop all Connect, Kafka and Zk threads. connect.stop(); } @@ -102,13 +104,13 @@ public void close() { * records, and start up a sink connector which will consume these records. */ @Test - public void testProduceConsumeConnector() throws Exception { + public void testSinkConnector() throws Exception { // create test topic connect.kafka().createTopic("test-topic", NUM_TOPIC_PARTITIONS); // setup up props for the sink connector Map props = new HashMap<>(); - props.put(CONNECTOR_CLASS_CONFIG, "MonitorableSink"); + props.put(CONNECTOR_CLASS_CONFIG, MonitorableSinkConnector.class.getSimpleName()); props.put(TASKS_MAX_CONFIG, String.valueOf(NUM_TASKS)); props.put(TOPICS_CONFIG, "test-topic"); props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); @@ -117,6 +119,9 @@ public void testProduceConsumeConnector() throws Exception { // expect all records to be consumed by the connector connectorHandle.expectedRecords(NUM_RECORDS_PRODUCED); + // expect all records to be consumed by the connector + connectorHandle.expectedCommits(NUM_RECORDS_PRODUCED); + // start a sink connector connect.configureConnector(CONNECTOR_NAME, props); @@ -131,10 +136,54 @@ public void testProduceConsumeConnector() throws Exception { // consume all records from the source topic or fail, to ensure that they were correctly produced. assertEquals("Unexpected number of records consumed", NUM_RECORDS_PRODUCED, - connect.kafka().consume(NUM_RECORDS_PRODUCED, CONSUME_MAX_DURATION_MS, "test-topic").count()); + connect.kafka().consume(NUM_RECORDS_PRODUCED, RECORD_TRANSFER_DURATION_MS, "test-topic").count()); // wait for the connector tasks to consume all records. - connectorHandle.awaitRecords(CONSUME_MAX_DURATION_MS); + connectorHandle.awaitRecords(RECORD_TRANSFER_DURATION_MS); + + // wait for the connector tasks to commit all records. + connectorHandle.awaitCommits(CONNECTOR_SETUP_DURATION_MS); + + // delete connector + connect.deleteConnector(CONNECTOR_NAME); + } + + /** + * Simple test case to configure and execute an embedded Connect cluster. The test will produce and consume + * records, and start up a sink connector which will consume these records. + */ + @Test + public void testSourceConnector() throws Exception { + // create test topic + connect.kafka().createTopic("test-topic", NUM_TOPIC_PARTITIONS); + + // setup up props for the sink connector + Map props = new HashMap<>(); + props.put(CONNECTOR_CLASS_CONFIG, MonitorableSourceConnector.class.getSimpleName()); + props.put(TASKS_MAX_CONFIG, String.valueOf(NUM_TASKS)); + props.put("topic", "test-topic"); + props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + + // expect all records to be produced by the connector + connectorHandle.expectedRecords(NUM_RECORDS_PRODUCED); + + // expect all records to be produced by the connector + connectorHandle.expectedCommits(NUM_RECORDS_PRODUCED); + + // start a source connector + connect.configureConnector(CONNECTOR_NAME, props); + + // wait for the connector tasks to produce enough records + connectorHandle.awaitRecords(RECORD_TRANSFER_DURATION_MS); + + // wait for the connector tasks to commit enough records + connectorHandle.awaitCommits(CONNECTOR_SETUP_DURATION_MS); + + // consume all records from the source topic or fail, to ensure that they were correctly produced + int recordNum = connect.kafka().consume(NUM_RECORDS_PRODUCED, RECORD_TRANSFER_DURATION_MS, "test-topic").count(); + assertTrue("Not enough records produced by source connector. Expected at least: " + NUM_RECORDS_PRODUCED + " + but got " + recordNum, + recordNum >= NUM_RECORDS_PRODUCED); // delete connector connect.deleteConnector(CONNECTOR_NAME); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSinkConnector.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSinkConnector.java index 23a8d99e84edc..06145de977a3a 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSinkConnector.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSinkConnector.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.connect.integration; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.connect.connector.Task; @@ -28,11 +29,15 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; /** - * A connector to be used in integration tests. This class provides methods to find task instances + * A sink connector that is used in Apache Kafka integration tests to verify the behavior of the + * Connect framework, but that can be used in other integration tests as a simple connector that + * consumes and counts records. This class provides methods to find task instances * which are initiated by the embedded connector, and wait for them to consume a desired number of * messages. */ @@ -41,10 +46,12 @@ public class MonitorableSinkConnector extends TestSinkConnector { private static final Logger log = LoggerFactory.getLogger(MonitorableSinkConnector.class); private String connectorName; + private Map commonConfigs; @Override public void start(Map props) { connectorName = props.get("name"); + commonConfigs = props; log.info("Starting connector {}", props.get("name")); } @@ -57,7 +64,7 @@ public Class taskClass() { public List> taskConfigs(int maxTasks) { List> configs = new ArrayList<>(); for (int i = 0; i < maxTasks; i++) { - Map config = new HashMap<>(); + Map config = new HashMap<>(commonConfigs); config.put("connector.name", connectorName); config.put("task.id", connectorName + "-" + i); configs.add(config); @@ -79,6 +86,15 @@ public static class MonitorableSinkTask extends SinkTask { private String connectorName; private String taskId; private TaskHandle taskHandle; + private Set assignments; + private Map committedOffsets; + private Map> cachedTopicPartitions; + + public MonitorableSinkTask() { + this.assignments = new HashSet<>(); + this.committedOffsets = new HashMap<>(); + this.cachedTopicPartitions = new HashMap<>(); + } @Override public String version() { @@ -96,7 +112,7 @@ public void start(Map props) { @Override public void open(Collection partitions) { log.debug("Opening {} partitions", partitions.size()); - super.open(partitions); + assignments.addAll(partitions); taskHandle.partitionsAssigned(partitions.size()); } @@ -104,10 +120,32 @@ public void open(Collection partitions) { public void put(Collection records) { for (SinkRecord rec : records) { taskHandle.record(); + TopicPartition tp = cachedTopicPartitions + .computeIfAbsent(rec.topic(), v -> new HashMap<>()) + .computeIfAbsent(rec.kafkaPartition(), v -> new TopicPartition(rec.topic(), rec.kafkaPartition())); + committedOffsets.put(tp, committedOffsets.getOrDefault(tp, 0L) + 1); log.trace("Task {} obtained record (key='{}' value='{}')", taskId, rec.key(), rec.value()); } } + @Override + public Map preCommit(Map offsets) { + for (TopicPartition tp : assignments) { + Long recordsSinceLastCommit = committedOffsets.get(tp); + if (recordsSinceLastCommit == null) { + log.warn("preCommit was called with topic-partition {} that is not included " + + "in the assignments of this task {}", tp, assignments); + } else { + taskHandle.commit(recordsSinceLastCommit.intValue()); + log.error("Forwarding to framework request to commit additional {} for {}", + recordsSinceLastCommit, tp); + taskHandle.commit((int) (long) recordsSinceLastCommit); + committedOffsets.put(tp, 0L); + } + } + return offsets; + } + @Override public void stop() { } diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java new file mode 100644 index 0000000000000..8bc895368f474 --- /dev/null +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.connect.integration; + +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.connect.connector.Task; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.runtime.TestSourceConnector; +import org.apache.kafka.connect.source.SourceRecord; +import org.apache.kafka.connect.source.SourceTask; +import org.apache.kafka.tools.ThroughputThrottler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.LongStream; + +/** + * A source connector that is used in Apache Kafka integration tests to verify the behavior of + * the Connect framework, but that can be used in other integration tests as a simple connector + * that generates records of a fixed structure. The rate of record production can be adjusted + * through the configs 'throughput' and 'messages.per.poll' + */ +public class MonitorableSourceConnector extends TestSourceConnector { + private static final Logger log = LoggerFactory.getLogger(MonitorableSourceConnector.class); + + private String connectorName; + private ConnectorHandle connectorHandle; + private Map commonConfigs; + + @Override + public void start(Map props) { + connectorHandle = RuntimeHandles.get().connectorHandle(props.get("name")); + connectorName = connectorHandle.name(); + commonConfigs = props; + log.info("Started {} connector {}", this.getClass().getSimpleName(), connectorName); + } + + @Override + public Class taskClass() { + return MonitorableSourceTask.class; + } + + @Override + public List> taskConfigs(int maxTasks) { + List> configs = new ArrayList<>(); + for (int i = 0; i < maxTasks; i++) { + Map config = new HashMap<>(commonConfigs); + config.put("connector.name", connectorName); + config.put("task.id", connectorName + "-" + i); + configs.add(config); + } + return configs; + } + + @Override + public void stop() { + log.info("Stopped {} connector {}", this.getClass().getSimpleName(), connectorName); + } + + @Override + public ConfigDef config() { + log.info("Configured {} connector {}", this.getClass().getSimpleName(), connectorName); + return new ConfigDef(); + } + + public static class MonitorableSourceTask extends SourceTask { + private String connectorName; + private String taskId; + private String topicName; + private TaskHandle taskHandle; + private volatile boolean stopped; + private long startingSeqno; + private long seqno; + private long throughput; + private int batchSize; + private ThroughputThrottler throttler; + + @Override + public String version() { + return "unknown"; + } + + @Override + public void start(Map props) { + taskId = props.get("task.id"); + connectorName = props.get("connector.name"); + topicName = props.getOrDefault("topic", "sequential-topic"); + throughput = Long.valueOf(props.getOrDefault("throughput", "-1")); + batchSize = Integer.valueOf(props.getOrDefault("messages.per.poll", "1")); + taskHandle = RuntimeHandles.get().connectorHandle(connectorName).taskHandle(taskId); + Map offset = Optional.ofNullable( + context.offsetStorageReader().offset(Collections.singletonMap("task.id", taskId))) + .orElse(Collections.emptyMap()); + startingSeqno = Optional.ofNullable((Long) offset.get("saved")).orElse(0L); + log.info("Started {} task {}", this.getClass().getSimpleName(), taskId); + throttler = new ThroughputThrottler(throughput, System.currentTimeMillis()); + } + + @Override + public List poll() { + if (!stopped) { + if (throttler.shouldThrottle(seqno - startingSeqno, System.currentTimeMillis())) { + throttler.throttle(); + } + taskHandle.record(batchSize); + return LongStream.range(0, batchSize) + .mapToObj(i -> new SourceRecord( + Collections.singletonMap("task.id", taskId), + Collections.singletonMap("saved", ++seqno), + topicName, + null, + Schema.STRING_SCHEMA, + "key-" + taskId + "-" + seqno, + Schema.STRING_SCHEMA, + "value-" + taskId + "-" + seqno)) + .collect(Collectors.toList()); + } + return null; + } + + @Override + public void commit() { + log.info("Task {} committing offsets", taskId); + //TODO: save progress outside the offset topic, potentially in the task handle + } + + @Override + public void commitRecord(SourceRecord record) { + log.trace("Committing record: {}", record); + taskHandle.commit(); + } + + @Override + public void stop() { + log.info("Stopped {} task {}", this.getClass().getSimpleName(), taskId); + stopped = true; + } + } +} diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/TaskHandle.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/TaskHandle.java index de3d9240d1be7..6081ea3cdcd7c 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/TaskHandle.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/TaskHandle.java @@ -23,6 +23,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.IntStream; /** * A handle to an executing task in a worker. Use this class to record progress, for example: number of records seen @@ -37,7 +38,9 @@ public class TaskHandle { private final AtomicInteger partitionsAssigned = new AtomicInteger(0); private CountDownLatch recordsRemainingLatch; + private CountDownLatch recordsToCommitLatch; private int expectedRecords = -1; + private int expectedCommits = -1; public TaskHandle(ConnectorHandle connectorHandle, String taskId) { log.info("Created task {} for connector {}", taskId, connectorHandle); @@ -46,7 +49,7 @@ public TaskHandle(ConnectorHandle connectorHandle, String taskId) { } /** - * Record a message arrival at the task. + * Record a message arrival at the task and the connector overall. */ public void record() { if (recordsRemainingLatch != null) { @@ -55,14 +58,58 @@ public void record() { connectorHandle.record(); } + /** + * Record arrival of a batch of messages at the task and the connector overall. + * + * @param batchSize the number of messages + */ + public void record(int batchSize) { + if (recordsRemainingLatch != null) { + IntStream.range(0, batchSize).forEach(i -> recordsRemainingLatch.countDown()); + } + connectorHandle.record(batchSize); + } + + /** + * Record a message commit from the task and the connector overall. + */ + public void commit() { + if (recordsToCommitLatch != null) { + recordsToCommitLatch.countDown(); + } + connectorHandle.commit(); + } + + /** + * Record commit on a batch of messages from the task and the connector overall. + * + * @param batchSize the number of messages + */ + public void commit(int batchSize) { + if (recordsToCommitLatch != null) { + IntStream.range(0, batchSize).forEach(i -> recordsToCommitLatch.countDown()); + } + connectorHandle.commit(batchSize); + } + /** * Set the number of expected records for this task. * - * @param expectedRecords number of records + * @param expected number of records + */ + public void expectedRecords(int expected) { + expectedRecords = expected; + recordsRemainingLatch = new CountDownLatch(expected); + } + + /** + * Set the number of expected record commits performed by this task. + * + * @param expected number of commits */ - public void expectedRecords(int expectedRecords) { - this.expectedRecords = expectedRecords; - this.recordsRemainingLatch = new CountDownLatch(expectedRecords); + public void expectedCommits(int expected) { + expectedRecords = expected; + recordsToCommitLatch = new CountDownLatch(expected); } /** @@ -82,24 +129,51 @@ public int partitionsAssigned() { } /** - * Wait for this task to receive the expected number of records. + * Wait for this task to meet the expected number of records as defined by {@code + * expectedRecords}. * - * @param consumeMaxDurationMs max duration to wait for records + * @param timeout duration to wait for records * @throws InterruptedException if another threads interrupts this one while waiting for records */ - public void awaitRecords(int consumeMaxDurationMs) throws InterruptedException { + public void awaitRecords(int timeout) throws InterruptedException { if (recordsRemainingLatch == null) { throw new IllegalStateException("Illegal state encountered. expectedRecords() was not set for this task?"); } - if (!recordsRemainingLatch.await(consumeMaxDurationMs, TimeUnit.MILLISECONDS)) { - String msg = String.format("Insufficient records seen by task %s in %d millis. Records expected=%d, actual=%d", + if (!recordsRemainingLatch.await(timeout, TimeUnit.MILLISECONDS)) { + String msg = String.format( + "Insufficient records seen by task %s in %d millis. Records expected=%d, actual=%d", taskId, - consumeMaxDurationMs, + timeout, expectedRecords, expectedRecords - recordsRemainingLatch.getCount()); throw new DataException(msg); } - log.debug("Task {} saw {} records, expected {} records", taskId, expectedRecords - recordsRemainingLatch.getCount(), expectedRecords); + log.debug("Task {} saw {} records, expected {} records", + taskId, expectedRecords - recordsRemainingLatch.getCount(), expectedRecords); + } + + /** + * Wait for this task to meet the expected number of commits as defined by {@code + * expectedCommits}. + * + * @param timeout duration to wait for commits + * @throws InterruptedException if another threads interrupts this one while waiting for commits + */ + public void awaitCommits(int timeout) throws InterruptedException { + if (recordsToCommitLatch == null) { + throw new IllegalStateException("Illegal state encountered. expectedRecords() was not set for this task?"); + } + if (!recordsToCommitLatch.await(timeout, TimeUnit.MILLISECONDS)) { + String msg = String.format( + "Insufficient records seen by task %s in %d millis. Records expected=%d, actual=%d", + taskId, + timeout, + expectedCommits, + expectedCommits - recordsToCommitLatch.getCount()); + throw new DataException(msg); + } + log.debug("Task {} saw {} records, expected {} records", + taskId, expectedCommits - recordsToCommitLatch.getCount(), expectedCommits); } @Override diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java index b660a1dfa5b94..590649b04296c 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java @@ -17,10 +17,10 @@ package org.apache.kafka.connect.util.clusters; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.kafka.connect.cli.ConnectDistributed; +import org.apache.kafka.common.utils.Exit; import org.apache.kafka.connect.errors.ConnectException; -import org.apache.kafka.connect.runtime.Connect; import org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo; +import org.apache.kafka.connect.runtime.rest.entities.ServerInfo; import org.apache.kafka.connect.runtime.rest.errors.ConnectRestException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,10 +32,17 @@ import java.io.OutputStreamWriter; import java.net.HttpURLConnection; import java.net.URL; +import java.util.Collection; import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.Map; +import java.util.Objects; import java.util.Properties; +import java.util.Set; import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_ID_CONFIG; import static org.apache.kafka.connect.runtime.ConnectorConfig.KEY_CONVERTER_CLASS_CONFIG; @@ -52,7 +59,8 @@ /** * Start an embedded connect worker. Internally, this class will spin up a Kafka and Zk cluster, setup any tmp - * directories and clean up them on them. + * directories and clean up them on them. Methods on the same {@code EmbeddedConnectCluster} are + * not guaranteed to be thread-safe. */ public class EmbeddedConnectCluster { @@ -63,24 +71,64 @@ public class EmbeddedConnectCluster { private static final Properties DEFAULT_BROKER_CONFIG = new Properties(); private static final String REST_HOST_NAME = "localhost"; - private final Connect[] connectCluster; + private static final String DEFAULT_WORKER_NAME_PREFIX = "connect-worker-"; + + private final Set connectCluster; private final EmbeddedKafkaCluster kafkaCluster; private final Map workerProps; private final String connectClusterName; private final int numBrokers; - - private EmbeddedConnectCluster(String name, Map workerProps, int numWorkers, int numBrokers, Properties brokerProps) { + private final int numInitialWorkers; + private final boolean maskExitProcedures; + private final String workerNamePrefix; + private final AtomicInteger nextWorkerId = new AtomicInteger(0); + + private EmbeddedConnectCluster(String name, Map workerProps, int numWorkers, + int numBrokers, Properties brokerProps, + boolean maskExitProcedures) { this.workerProps = workerProps; this.connectClusterName = name; this.numBrokers = numBrokers; this.kafkaCluster = new EmbeddedKafkaCluster(numBrokers, brokerProps); - this.connectCluster = new Connect[numWorkers]; + this.connectCluster = new LinkedHashSet<>(); + this.numInitialWorkers = numWorkers; + this.maskExitProcedures = maskExitProcedures; + // leaving non-configurable for now + this.workerNamePrefix = DEFAULT_WORKER_NAME_PREFIX; } + /** + * A more graceful way to handle abnormal exit of services in integration tests. + */ + public Exit.Procedure exitProcedure = (code, message) -> { + if (code != 0) { + String exitMessage = "Abrupt service exit with code " + code + " and message " + message; + log.warn(exitMessage); + throw new UngracefulShutdownException(exitMessage); + } + Exit.exit(0, message); + }; + + /** + * A more graceful way to handle abnormal halt of services in integration tests. + */ + public Exit.Procedure haltProcedure = (code, message) -> { + if (code != 0) { + String haltMessage = "Abrupt service halt with code " + code + " and message " + message; + log.warn(haltMessage); + throw new UngracefulShutdownException(haltMessage); + } + Exit.halt(0, message); + }; + /** * Start the connect cluster and the embedded Kafka and Zookeeper cluster. */ public void start() throws IOException { + if (maskExitProcedures) { + Exit.setExitProcedure(exitProcedure); + Exit.setHaltProcedure(haltProcedure); + } kafkaCluster.before(); startConnect(); } @@ -90,26 +138,74 @@ public void start() throws IOException { * Clean up any temp directories created locally. */ public void stop() { - for (Connect worker : this.connectCluster) { - try { - worker.stop(); - } catch (Exception e) { - log.error("Could not stop connect", e); - throw new RuntimeException("Could not stop worker", e); - } - } - + connectCluster.forEach(this::stopWorker); try { kafkaCluster.after(); + } catch (UngracefulShutdownException e) { + log.warn("Kafka did not shutdown gracefully"); } catch (Exception e) { log.error("Could not stop kafka", e); throw new RuntimeException("Could not stop brokers", e); + } finally { + Exit.resetExitProcedure(); + Exit.resetHaltProcedure(); + } + } + + /** + * Provision and start an additional worker to the Connect cluster. + * + * @return the worker handle of the worker that was provisioned + */ + public WorkerHandle addWorker() { + WorkerHandle worker = WorkerHandle.start(workerNamePrefix + nextWorkerId.getAndIncrement(), workerProps); + connectCluster.add(worker); + log.info("Started worker {}", worker); + return worker; + } + + /** + * Decommission one of the workers from this Connect cluster. Which worker is removed is + * implementation dependent and selection is not guaranteed to be consistent. Use this method + * when you don't care which worker stops. + * + * @see #removeWorker(WorkerHandle) + */ + public void removeWorker() { + WorkerHandle toRemove = null; + for (Iterator it = connectCluster.iterator(); it.hasNext(); toRemove = it.next()) { + } + removeWorker(toRemove); + } + + /** + * Decommission a specific worker from this Connect cluster. + * + * @param worker the handle of the worker to remove from the cluster + */ + public void removeWorker(WorkerHandle worker) { + if (connectCluster.isEmpty()) { + throw new IllegalStateException("Cannot remove worker. Cluster is empty"); + } + stopWorker(worker); + connectCluster.remove(worker); + } + + private void stopWorker(WorkerHandle worker) { + try { + log.info("Stopping worker {}", worker); + worker.stop(); + } catch (UngracefulShutdownException e) { + log.warn("Worker {} did not shutdown gracefully", worker); + } catch (Exception e) { + log.error("Could not stop connect", e); + throw new RuntimeException("Could not stop worker", e); } } @SuppressWarnings("deprecation") public void startConnect() { - log.info("Starting Connect cluster with {} workers. clusterName {}", connectCluster.length, connectClusterName); + log.info("Starting Connect cluster '{}' with {} workers", connectClusterName, numInitialWorkers); workerProps.put(BOOTSTRAP_SERVERS_CONFIG, kafka().bootstrapServers()); workerProps.put(REST_HOST_NAME_CONFIG, REST_HOST_NAME); @@ -126,11 +222,41 @@ public void startConnect() { putIfAbsent(workerProps, KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter"); putIfAbsent(workerProps, VALUE_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter"); - for (int i = 0; i < connectCluster.length; i++) { - connectCluster[i] = new ConnectDistributed().startConnect(workerProps); + for (int i = 0; i < numInitialWorkers; i++) { + addWorker(); } } + /** + * Get the workers that are up and running. + * + * @return the list of handles of the online workers + */ + public Set activeWorkers() { + ObjectMapper mapper = new ObjectMapper(); + return connectCluster.stream() + .filter(w -> { + try { + mapper.readerFor(ServerInfo.class) + .readValue(executeGet(w.url().toString())); + return true; + } catch (IOException e) { + // Worker failed to respond. Consider it's offline + return false; + } + }) + .collect(Collectors.toSet()); + } + + /** + * Get the provisioned workers. + * + * @return the list of handles of the provisioned workers + */ + public Set workers() { + return new LinkedHashSet<>(connectCluster); + } + /** * Configure a connector. If the connector does not already exist, a new one will be created and * the given configuration will be applied to it. @@ -170,6 +296,24 @@ public void deleteConnector(String connName) throws IOException { } } + /** + * Get the connector names of the connectors currently running on this cluster. + * + * @return the list of connector names + * @throws ConnectRestException if the HTTP request to the REST API failed with a valid status code. + * @throws ConnectException for any other error. + */ + public Collection connectors() { + ObjectMapper mapper = new ObjectMapper(); + try { + String url = endpointForResource("connectors"); + return mapper.readerFor(Collection.class).readValue(executeGet(url)); + } catch (IOException e) { + log.error("Could not read connector list", e); + throw new ConnectException("Could not read connector list", e); + } + } + /** * Get the status for a connector running in this cluster. * @@ -180,8 +324,8 @@ public void deleteConnector(String connName) throws IOException { */ public ConnectorStateInfo connectorStatus(String connectorName) { ObjectMapper mapper = new ObjectMapper(); - String url = endpointForResource(String.format("connectors/%s/status", connectorName)); try { + String url = endpointForResource(String.format("connectors/%s/status", connectorName)); return mapper.readerFor(ConnectorStateInfo.class).readValue(executeGet(url)); } catch (IOException e) { log.error("Could not read connector state", e); @@ -189,8 +333,13 @@ public ConnectorStateInfo connectorStatus(String connectorName) { } } - private String endpointForResource(String resource) { - String url = String.valueOf(connectCluster[0].restUrl()); + private String endpointForResource(String resource) throws IOException { + String url = connectCluster.stream() + .map(WorkerHandle::url) + .filter(Objects::nonNull) + .findFirst() + .orElseThrow(() -> new IOException("Connect workers have not been provisioned")) + .toString(); return url + resource; } @@ -270,6 +419,7 @@ public static class Builder { private int numWorkers = DEFAULT_NUM_WORKERS; private int numBrokers = DEFAULT_NUM_BROKERS; private Properties brokerProps = DEFAULT_BROKER_CONFIG; + private boolean maskExitProcedures = false; public Builder name(String name) { this.name = name; @@ -296,8 +446,26 @@ public Builder brokerProps(Properties brokerProps) { return this; } + /** + * In the event of ungraceful shutdown, embedded clusters call exit or halt with non-zero + * exit statuses. Exiting with a non-zero status forces a test to fail and is hard to + * handle. Because graceful exit is usually not required during a test and because + * depending on such an exit increases flakiness, this setting allows masking + * exit and halt procedures by using a runtime exception instead. Customization of the + * exit and halt procedures is possible through {@code exitProcedure} and {@code + * haltProcedure} respectively. + * + * @param mask if false, exit and halt procedures remain unchanged; true is the default. + * @return the builder for this cluster + */ + public Builder maskExitProcedures(boolean mask) { + this.maskExitProcedures = mask; + return this; + } + public EmbeddedConnectCluster build() { - return new EmbeddedConnectCluster(name, workerProps, numWorkers, numBrokers, brokerProps); + return new EmbeddedConnectCluster(name, workerProps, numWorkers, numBrokers, + brokerProps, maskExitProcedures); } } diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/UngracefulShutdownException.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/UngracefulShutdownException.java new file mode 100644 index 0000000000000..2f2b030efa1b2 --- /dev/null +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/UngracefulShutdownException.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.connect.util.clusters; + +import org.apache.kafka.common.KafkaException; + +/** + * An exception that can be used from within an {@code Exit.Procedure} to mask exit or halt calls + * and signify that the service terminated abruptly. It's intended to be used only from within + * integration tests. + */ +public class UngracefulShutdownException extends KafkaException { + public UngracefulShutdownException(String s) { + super(s); + } + + public UngracefulShutdownException(String s, Throwable throwable) { + super(s, throwable); + } + + public UngracefulShutdownException(Throwable throwable) { + super(throwable); + } +} diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/WorkerHandle.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/WorkerHandle.java new file mode 100644 index 0000000000000..7113f52a19b1f --- /dev/null +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/WorkerHandle.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.connect.util.clusters; + +import org.apache.kafka.connect.cli.ConnectDistributed; +import org.apache.kafka.connect.runtime.Connect; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URI; +import java.util.Map; +import java.util.Objects; + +/** + * A handle to a worker executing in a Connect cluster. + */ +public class WorkerHandle { + private static final Logger log = LoggerFactory.getLogger(WorkerHandle.class); + + private final String workerName; + private final Connect worker; + + protected WorkerHandle(String workerName, Connect worker) { + this.workerName = workerName; + this.worker = worker; + } + + /** + * Create and start a new worker with the given properties. + * + * @param name a name for this worker + * @param workerProperties the worker properties + * @return the worker's handle + */ + public static WorkerHandle start(String name, Map workerProperties) { + return new WorkerHandle(name, new ConnectDistributed().startConnect(workerProperties)); + } + + /** + * Stop this worker. + */ + public void stop() { + worker.stop(); + } + + /** + * Get the workers's name corresponding to this handle. + * + * @return the worker's name + */ + public String name() { + return workerName; + } + + /** + * Get the workers's url that accepts requests to its REST endpoint. + * + * @return the worker's url + */ + public URI url() { + return worker.restUrl(); + } + + @Override + public String toString() { + return "WorkerHandle{" + + "workerName='" + workerName + '\'' + + "workerURL='" + worker.restUrl() + '\'' + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof WorkerHandle)) { + return false; + } + WorkerHandle that = (WorkerHandle) o; + return Objects.equals(workerName, that.workerName) && + Objects.equals(worker, that.worker); + } + + @Override + public int hashCode() { + return Objects.hash(workerName, worker); + } +} From e0d028bf6cbf140c72706247c40bded7bfabcb0c Mon Sep 17 00:00:00 2001 From: Colin Patrick McCabe Date: Mon, 25 Mar 2019 09:43:44 -0700 Subject: [PATCH 0056/1071] KAFKA-8150: Fix bugs in handling null arrays in generated RPC code (#6489) ToString functions must not get a NullPointException. read() functions must properly translate a negative array length to a null field. Reviewers: Manikumar Reddy --- .../apache/kafka/common/message/MessageTest.java | 7 +++++-- .../kafka/message/MessageDataGenerator.java | 15 ++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java index 93a0930023af7..d573b3be31e13 100644 --- a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java +++ b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java @@ -38,7 +38,6 @@ import org.apache.kafka.common.message.AddPartitionsToTxnRequestData.AddPartitionsToTxnTopic; import org.apache.kafka.common.message.AddPartitionsToTxnRequestData.AddPartitionsToTxnTopicSet; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; @@ -47,7 +46,6 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -@Ignore public final class MessageTest { @Rule final public Timeout globalTimeout = Timeout.millis(120000); @@ -87,6 +85,11 @@ public void testRoundTrips() throws Exception { setHostFilter(null). setOperation((byte) 0). setPermissionType((byte) 0), (short) 0); + testMessageRoundTrips(new MetadataRequestData(). + setTopics(null). + setAllowAutoTopicCreation(false). + setIncludeClusterAuthorizedOperations(false). + setIncludeTopicAuthorizedOperations(false)); } private void testMessageRoundTrips(Message message) throws Exception { diff --git a/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java b/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java index 76029f4c1baf9..c8e70bba1e673 100644 --- a/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java +++ b/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java @@ -416,9 +416,8 @@ private void generateFieldReader(FieldSpec field, Versions curVersions) { buffer.printf("int arrayLength = readable.readInt();%n"); buffer.printf("if (arrayLength < 0) {%n"); buffer.incrementIndent(); - buffer.printf("this.%s.clear(%s);%n", - field.camelCaseName(), - hasKeys ? "0" : ""); + buffer.printf("this.%s = null;%n", + field.camelCaseName()); buffer.decrementIndent(); buffer.printf("} else {%n"); buffer.incrementIndent(); @@ -1069,8 +1068,14 @@ private void generateFieldToString(String prefix, FieldSpec field) { prefix, field.camelCaseName(), field.camelCaseName()); } else if (field.type().isArray()) { headerGenerator.addImport(MessageGenerator.MESSAGE_UTIL_CLASS); - buffer.printf("+ \"%s%s=\" + MessageUtil.deepToString(%s.iterator())%n", - prefix, field.camelCaseName(), field.camelCaseName()); + if (field.nullableVersions().empty()) { + buffer.printf("+ \"%s%s=\" + MessageUtil.deepToString(%s.iterator())%n", + prefix, field.camelCaseName(), field.camelCaseName()); + } else { + buffer.printf("+ \"%s%s=\" + ((%s == null) ? \"null\" : " + + "MessageUtil.deepToString(%s.iterator()))%n", + prefix, field.camelCaseName(), field.camelCaseName(), field.camelCaseName()); + } } else { throw new RuntimeException("Unsupported field type " + field.type()); } From 1baba1b347a9db51514777dba07263e285e8237b Mon Sep 17 00:00:00 2001 From: pierDipi <33736985+pierDipi@users.noreply.github.com> Date: Mon, 25 Mar 2019 17:54:17 +0100 Subject: [PATCH 0057/1071] MINOR: Fix misspelling in protocol documentation Reviewers: Colin P. McCabe --- .../java/org/apache/kafka/common/protocol/CommonFields.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/CommonFields.java b/clients/src/main/java/org/apache/kafka/common/protocol/CommonFields.java index 708500c65e6ce..5fdc37f044eb2 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/CommonFields.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/CommonFields.java @@ -51,7 +51,7 @@ public class CommonFields { public static final Field.Int8 RESOURCE_TYPE = new Field.Int8("resource_type", "The resource type"); public static final Field.Str RESOURCE_NAME = new Field.Str("resource_name", "The resource name"); public static final Field.NullableStr RESOURCE_NAME_FILTER = new Field.NullableStr("resource_name", "The resource name filter"); - public static final Field.Int8 RESOURCE_PATTERN_TYPE = new Field.Int8("resource_pattten_type", "The resource pattern type", PatternType.LITERAL.code()); + public static final Field.Int8 RESOURCE_PATTERN_TYPE = new Field.Int8("resource_pattern_type", "The resource pattern type", PatternType.LITERAL.code()); public static final Field.Int8 RESOURCE_PATTERN_TYPE_FILTER = new Field.Int8("resource_pattern_type_filter", "The resource pattern type filter", PatternType.LITERAL.code()); public static final Field.Str PRINCIPAL = new Field.Str("principal", "The ACL principal"); public static final Field.NullableStr PRINCIPAL_FILTER = new Field.NullableStr("principal", "The ACL principal filter"); From 0d55f0f3ec8f97bc250b325481f6f2fa70f52a5c Mon Sep 17 00:00:00 2001 From: Stanislav Kozlovski Date: Mon, 25 Mar 2019 09:58:11 -0700 Subject: [PATCH 0058/1071] KAFKA-8102: Add an interval-based Trogdor transaction generator (#6444) This patch adds a TimeIntervalTransactionsGenerator class which enables the Trogdor ProduceBench worker to commit transactions based on a configurable millisecond time interval. Also, we now handle 409 create task responses in the coordinator command-line client by printing a more informative message Reviewers: Colin P. McCabe --- TROGDOR.md | 1 + tests/spec/transactional-produce-bench.json | 2 +- .../coordinator/CoordinatorClient.java | 12 +++- .../trogdor/coordinator/TaskManager.java | 1 + .../TimeIntervalTransactionsGenerator.java | 67 +++++++++++++++++++ .../workload/TransactionGenerator.java | 1 + ...TimeIntervalTransactionsGeneratorTest.java | 42 ++++++++++++ 7 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 tools/src/main/java/org/apache/kafka/trogdor/workload/TimeIntervalTransactionsGenerator.java create mode 100644 tools/src/test/java/org/apache/kafka/trogdor/workload/TimeIntervalTransactionsGeneratorTest.java diff --git a/TROGDOR.md b/TROGDOR.md index b551773ab98ab..8b446b3eeb0f4 100644 --- a/TROGDOR.md +++ b/TROGDOR.md @@ -105,6 +105,7 @@ Trogdor can run several workloads. Workloads perform operations on the cluster ### ProduceBench ProduceBench starts a Kafka producer on a single agent node, producing to several partitions. The workload measures the average produce latency, as well as the median, 95th percentile, and 99th percentile latency. +It can be configured to use a transactional producer which can commit transactions based on a set time interval or number of messages. ### RoundTripWorkload RoundTripWorkload tests both production and consumption. The workload starts a Kafka producer and consumer on a single node. The consumer will read back the messages that were produced by the producer. diff --git a/tests/spec/transactional-produce-bench.json b/tests/spec/transactional-produce-bench.json index 40f008b1e4519..bf1b6ca0ba137 100644 --- a/tests/spec/transactional-produce-bench.json +++ b/tests/spec/transactional-produce-bench.json @@ -15,7 +15,7 @@ // // An example task specification for running a transactional producer benchmark -in Trogdor. See TROGDOR.md for details. +// in Trogdor. See TROGDOR.md for details. // { diff --git a/tools/src/main/java/org/apache/kafka/trogdor/coordinator/CoordinatorClient.java b/tools/src/main/java/org/apache/kafka/trogdor/coordinator/CoordinatorClient.java index 476c32d8b8433..ba40e7bc0d6bb 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/coordinator/CoordinatorClient.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/coordinator/CoordinatorClient.java @@ -44,6 +44,7 @@ import org.apache.kafka.trogdor.rest.TaskState; import org.apache.kafka.trogdor.rest.TasksResponse; import org.apache.kafka.trogdor.task.TaskSpec; +import org.apache.kafka.trogdor.rest.RequestConflictException; import org.apache.kafka.trogdor.rest.UptimeResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -426,8 +427,15 @@ public static void main(String[] args) throws Exception { TaskSpec taskSpec = JsonUtil. objectFromCommandLineArgument(res.getString("taskSpec"), TaskSpec.class); CreateTaskRequest req = new CreateTaskRequest(taskId, taskSpec); - client.createTask(req); - System.out.printf("Sent CreateTaskRequest for task %s.%n", req.id()); + try { + client.createTask(req); + System.out.printf("Sent CreateTaskRequest for task %s.%n", req.id()); + } catch (RequestConflictException rce) { + System.out.printf("CreateTaskRequest for task %s got a 409 status code - " + + "a task with the same ID but a different specification already exists.%nException: %s%n", + req.id(), rce.getMessage()); + Exit.exit(1); + } break; } case "stopTask": { diff --git a/tools/src/main/java/org/apache/kafka/trogdor/coordinator/TaskManager.java b/tools/src/main/java/org/apache/kafka/trogdor/coordinator/TaskManager.java index 941656e36186e..60e2b1e8e9f95 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/coordinator/TaskManager.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/coordinator/TaskManager.java @@ -302,6 +302,7 @@ TreeMap activeWorkerIds() { * * @param id The ID of the task to create. * @param spec The specification of the task to create. + * @throws RequestConflictException - if a task with the same ID but different spec exists */ public void createTask(final String id, TaskSpec spec) throws Throwable { diff --git a/tools/src/main/java/org/apache/kafka/trogdor/workload/TimeIntervalTransactionsGenerator.java b/tools/src/main/java/org/apache/kafka/trogdor/workload/TimeIntervalTransactionsGenerator.java new file mode 100644 index 0000000000000..8d5f05b1ee001 --- /dev/null +++ b/tools/src/main/java/org/apache/kafka/trogdor/workload/TimeIntervalTransactionsGenerator.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.trogdor.workload; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.kafka.common.utils.Time; + +/** + * A transactions generator where we commit a transaction every N milliseconds + */ +public class TimeIntervalTransactionsGenerator implements TransactionGenerator { + + private static final long NULL_START_MS = -1; + + private final Time time; + private final int intervalMs; + + private long lastTransactionStartMs = NULL_START_MS; + + @JsonCreator + public TimeIntervalTransactionsGenerator(@JsonProperty("transactionIntervalMs") int intervalMs) { + this(intervalMs, Time.SYSTEM); + } + + TimeIntervalTransactionsGenerator(@JsonProperty("transactionIntervalMs") int intervalMs, + Time time) { + if (intervalMs < 1) { + throw new IllegalArgumentException("Cannot have a negative interval"); + } + this.time = time; + this.intervalMs = intervalMs; + } + + @JsonProperty + public int transactionIntervalMs() { + return intervalMs; + } + + @Override + public synchronized TransactionAction nextAction() { + if (lastTransactionStartMs == NULL_START_MS) { + lastTransactionStartMs = time.milliseconds(); + return TransactionAction.BEGIN_TRANSACTION; + } + if (time.milliseconds() - lastTransactionStartMs >= intervalMs) { + lastTransactionStartMs = NULL_START_MS; + return TransactionAction.COMMIT_TRANSACTION; + } + + return TransactionAction.NO_OP; + } +} diff --git a/tools/src/main/java/org/apache/kafka/trogdor/workload/TransactionGenerator.java b/tools/src/main/java/org/apache/kafka/trogdor/workload/TransactionGenerator.java index 5ec47ec91c167..b2e8add8bef07 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/workload/TransactionGenerator.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/workload/TransactionGenerator.java @@ -27,6 +27,7 @@ property = "type") @JsonSubTypes(value = { @JsonSubTypes.Type(value = UniformTransactionsGenerator.class, name = "uniform"), + @JsonSubTypes.Type(value = TimeIntervalTransactionsGenerator.class, name = "interval"), }) public interface TransactionGenerator { enum TransactionAction { diff --git a/tools/src/test/java/org/apache/kafka/trogdor/workload/TimeIntervalTransactionsGeneratorTest.java b/tools/src/test/java/org/apache/kafka/trogdor/workload/TimeIntervalTransactionsGeneratorTest.java new file mode 100644 index 0000000000000..29ed3c8d936b7 --- /dev/null +++ b/tools/src/test/java/org/apache/kafka/trogdor/workload/TimeIntervalTransactionsGeneratorTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.trogdor.workload; + +import org.apache.kafka.common.utils.MockTime; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TimeIntervalTransactionsGeneratorTest { + @Test + public void testCommitsTransactionAfterIntervalPasses() { + MockTime time = new MockTime(); + TimeIntervalTransactionsGenerator generator = new TimeIntervalTransactionsGenerator(100, time); + + assertEquals(100, generator.transactionIntervalMs()); + assertEquals(TransactionGenerator.TransactionAction.BEGIN_TRANSACTION, generator.nextAction()); + assertEquals(TransactionGenerator.TransactionAction.NO_OP, generator.nextAction()); + time.sleep(50); + assertEquals(TransactionGenerator.TransactionAction.NO_OP, generator.nextAction()); + time.sleep(49); + assertEquals(TransactionGenerator.TransactionAction.NO_OP, generator.nextAction()); + time.sleep(1); + assertEquals(TransactionGenerator.TransactionAction.COMMIT_TRANSACTION, generator.nextAction()); + assertEquals(TransactionGenerator.TransactionAction.BEGIN_TRANSACTION, generator.nextAction()); + } +} From 13e265ab3dfd69cdc3709b6f871418bcf1a2221f Mon Sep 17 00:00:00 2001 From: Ivan Yurchenko Date: Tue, 26 Mar 2019 03:50:12 +0200 Subject: [PATCH 0059/1071] KAFKA-7986: Distinguish logging from different ZooKeeperClient instances (#6493) A broken can have more than one instance of ZooKeeperClient. For example, SimpleAclAuthorizer creates a separate ZooKeeperClient instance when configured. This commit makes it possible to optionally specify the name for the ZooKeeperClient instance. The name is specified only for a broker's ZooKeeperClient instances, but not for commands' and tests'. Reviewers: Jun Rao --- .../security/auth/SimpleAclAuthorizer.scala | 2 +- .../main/scala/kafka/server/KafkaServer.scala | 2 +- .../main/scala/kafka/zk/KafkaZkClient.scala | 5 +++-- .../kafka/zookeeper/ZooKeeperClient.scala | 20 +++++++++++++++++-- 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala b/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala index 8a0b4a072e40a..e39babf892485 100644 --- a/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala +++ b/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala @@ -98,7 +98,7 @@ class SimpleAclAuthorizer extends Authorizer with Logging { val time = Time.SYSTEM zkClient = KafkaZkClient(zkUrl, kafkaConfig.zkEnableSecureAcls, zkSessionTimeOutMs, zkConnectionTimeoutMs, - zkMaxInFlightRequests, time, "kafka.security", "SimpleAclAuthorizer") + zkMaxInFlightRequests, time, "kafka.security", "SimpleAclAuthorizer", name=Some("Simple ACL authorizer")) zkClient.createAclPaths() extendedAclSupport = kafkaConfig.interBrokerProtocolVersion >= KAFKA_2_0_IV1 diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala index 8fc5197b7e19d..b5ee8cc7907e1 100755 --- a/core/src/main/scala/kafka/server/KafkaServer.scala +++ b/core/src/main/scala/kafka/server/KafkaServer.scala @@ -359,7 +359,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP def createZkClient(zkConnect: String, isSecure: Boolean) = KafkaZkClient(zkConnect, isSecure, config.zkSessionTimeoutMs, config.zkConnectionTimeoutMs, - config.zkMaxInFlightRequests, time) + config.zkMaxInFlightRequests, time, name = Some("Kafka server")) val chrootIndex = config.zkConnect.indexOf("/") val chrootOption = { diff --git a/core/src/main/scala/kafka/zk/KafkaZkClient.scala b/core/src/main/scala/kafka/zk/KafkaZkClient.scala index 6d8d50443e9cd..782ec2ab99045 100644 --- a/core/src/main/scala/kafka/zk/KafkaZkClient.scala +++ b/core/src/main/scala/kafka/zk/KafkaZkClient.scala @@ -1820,9 +1820,10 @@ object KafkaZkClient { maxInFlightRequests: Int, time: Time, metricGroup: String = "kafka.server", - metricType: String = "SessionExpireListener") = { + metricType: String = "SessionExpireListener", + name: Option[String] = None) = { val zooKeeperClient = new ZooKeeperClient(connectString, sessionTimeoutMs, connectionTimeoutMs, maxInFlightRequests, - time, metricGroup, metricType) + time, metricGroup, metricType, name) new KafkaZkClient(zooKeeperClient, isSecure, time) } diff --git a/core/src/main/scala/kafka/zookeeper/ZooKeeperClient.scala b/core/src/main/scala/kafka/zookeeper/ZooKeeperClient.scala index ad4da8b5385a4..c193ff2fbf2e9 100755 --- a/core/src/main/scala/kafka/zookeeper/ZooKeeperClient.scala +++ b/core/src/main/scala/kafka/zookeeper/ZooKeeperClient.scala @@ -44,6 +44,7 @@ import scala.collection.mutable.Set * @param sessionTimeoutMs session timeout in milliseconds * @param connectionTimeoutMs connection timeout in milliseconds * @param maxInFlightRequests maximum number of unacknowledged requests the client will send before blocking. + * @param name name of the client instance */ class ZooKeeperClient(connectString: String, sessionTimeoutMs: Int, @@ -51,8 +52,23 @@ class ZooKeeperClient(connectString: String, maxInFlightRequests: Int, time: Time, metricGroup: String, - metricType: String) extends Logging with KafkaMetricsGroup { - this.logIdent = "[ZooKeeperClient] " + metricType: String, + name: Option[String]) extends Logging with KafkaMetricsGroup { + + def this(connectString: String, + sessionTimeoutMs: Int, + connectionTimeoutMs: Int, + maxInFlightRequests: Int, + time: Time, + metricGroup: String, + metricType: String) = { + this(connectString, sessionTimeoutMs, connectionTimeoutMs, maxInFlightRequests, time, metricGroup, metricType, None) + } + + this.logIdent = name match { + case Some(n) => s"[ZooKeeperClient $n] " + case _ => "[ZooKeeperClient] " + } private val initializationLock = new ReentrantReadWriteLock() private val isConnectedOrExpiredLock = new ReentrantLock() private val isConnectedOrExpiredCondition = isConnectedOrExpiredLock.newCondition() From f47b8493bce314e166e27c9c7a9fabb8ba650c0f Mon Sep 17 00:00:00 2001 From: "Matthias J. Sax" Date: Tue, 26 Mar 2019 09:41:29 -0700 Subject: [PATCH 0060/1071] MINOR: Add 2.2.0 upgrade instructions (#6501) Reviewers: Jason Gustafson --- docs/upgrade.html | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/docs/upgrade.html b/docs/upgrade.html index 86bf5dd3b5ad9..1828ab7813991 100644 --- a/docs/upgrade.html +++ b/docs/upgrade.html @@ -20,6 +20,47 @@ \ No newline at end of file + From 9f1ce60a9c0eae99388504349870018f0dd55afa Mon Sep 17 00:00:00 2001 From: Lifei Chen Date: Thu, 30 May 2019 23:33:37 +0800 Subject: [PATCH 0307/1071] KAFKA-8187: Add wait time for other thread in the same jvm to free the locks (#6818) Fix KAFKA-8187: State store record loss across multiple reassignments when using standby tasks. Do not let the thread to transit to RUNNING until all tasks (including standby tasks) are ready. Reviewers: Guozhang Wang , Bill Bejeck --- .../processor/internals/TaskManager.java | 2 +- .../processor/internals/TaskManagerTest.java | 108 ++++++++++-------- 2 files changed, 64 insertions(+), 46 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java index 455d226d69ce7..86a8e4937b965 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java @@ -334,7 +334,7 @@ boolean updateNewAndRestoringTasks() { log.trace("Resuming partitions {}", assignment); consumer.resume(assignment); assignStandbyPartitions(); - return true; + return standby.allTasksRunning(); } return false; } diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java index f71d7a134eb44..fcf275b104d02 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java @@ -263,7 +263,7 @@ public void shouldCloseStandbyUnAssignedSuspendedTasksWhenCreatingNewTasks() { @Test public void shouldAddNonResumedActiveTasks() { mockSingleActiveTask(); - EasyMock.expect(active.maybeResumeSuspendedTask(taskId0, taskId0Partitions)).andReturn(false); + expect(active.maybeResumeSuspendedTask(taskId0, taskId0Partitions)).andReturn(false); active.addNewTask(EasyMock.same(streamTask)); replay(); @@ -276,7 +276,7 @@ public void shouldAddNonResumedActiveTasks() { @Test public void shouldNotAddResumedActiveTasks() { checkOrder(active, true); - EasyMock.expect(active.maybeResumeSuspendedTask(taskId0, taskId0Partitions)).andReturn(true); + expect(active.maybeResumeSuspendedTask(taskId0, taskId0Partitions)).andReturn(true); replay(); taskManager.setAssignmentMetadata(taskId0Assignment, Collections.>emptyMap()); @@ -289,7 +289,7 @@ public void shouldNotAddResumedActiveTasks() { @Test public void shouldAddNonResumedStandbyTasks() { mockStandbyTaskExpectations(); - EasyMock.expect(standby.maybeResumeSuspendedTask(taskId0, taskId0Partitions)).andReturn(false); + expect(standby.maybeResumeSuspendedTask(taskId0, taskId0Partitions)).andReturn(false); standby.addNewTask(EasyMock.same(standbyTask)); replay(); @@ -302,7 +302,7 @@ public void shouldAddNonResumedStandbyTasks() { @Test public void shouldNotAddResumedStandbyTasks() { checkOrder(active, true); - EasyMock.expect(standby.maybeResumeSuspendedTask(taskId0, taskId0Partitions)).andReturn(true); + expect(standby.maybeResumeSuspendedTask(taskId0, taskId0Partitions)).andReturn(true); replay(); taskManager.setAssignmentMetadata(Collections.>emptyMap(), taskId0Assignment); @@ -316,7 +316,7 @@ public void shouldNotAddResumedStandbyTasks() { public void shouldPauseActivePartitions() { mockSingleActiveTask(); consumer.pause(taskId0Partitions); - EasyMock.expectLastCall(); + expectLastCall(); replay(); taskManager.setAssignmentMetadata(taskId0Assignment, Collections.>emptyMap()); @@ -326,7 +326,7 @@ public void shouldPauseActivePartitions() { @Test public void shouldSuspendActiveTasks() { - EasyMock.expect(active.suspend()).andReturn(null); + expect(active.suspend()).andReturn(null); replay(); taskManager.suspendTasksAndState(); @@ -335,7 +335,7 @@ public void shouldSuspendActiveTasks() { @Test public void shouldSuspendStandbyTasks() { - EasyMock.expect(standby.suspend()).andReturn(null); + expect(standby.suspend()).andReturn(null); replay(); taskManager.suspendTasksAndState(); @@ -345,7 +345,7 @@ public void shouldSuspendStandbyTasks() { @Test public void shouldUnassignChangelogPartitionsOnSuspend() { restoreConsumer.unsubscribe(); - EasyMock.expectLastCall(); + expectLastCall(); replay(); taskManager.suspendTasksAndState(); @@ -354,9 +354,9 @@ public void shouldUnassignChangelogPartitionsOnSuspend() { @Test public void shouldThrowStreamsExceptionAtEndIfExceptionDuringSuspend() { - EasyMock.expect(active.suspend()).andReturn(new RuntimeException("")); - EasyMock.expect(standby.suspend()).andReturn(new RuntimeException("")); - EasyMock.expectLastCall(); + expect(active.suspend()).andReturn(new RuntimeException("")); + expect(standby.suspend()).andReturn(new RuntimeException("")); + expectLastCall(); restoreConsumer.unsubscribe(); replay(); @@ -372,7 +372,7 @@ public void shouldThrowStreamsExceptionAtEndIfExceptionDuringSuspend() { @Test public void shouldCloseActiveTasksOnShutdown() { active.close(true); - EasyMock.expectLastCall(); + expectLastCall(); replay(); taskManager.shutdown(true); @@ -382,7 +382,7 @@ public void shouldCloseActiveTasksOnShutdown() { @Test public void shouldCloseStandbyTasksOnShutdown() { standby.close(false); - EasyMock.expectLastCall(); + expectLastCall(); replay(); taskManager.shutdown(false); @@ -392,7 +392,7 @@ public void shouldCloseStandbyTasksOnShutdown() { @Test public void shouldUnassignChangelogPartitionsOnShutdown() { restoreConsumer.unsubscribe(); - EasyMock.expectLastCall(); + expectLastCall(); replay(); taskManager.shutdown(true); @@ -402,7 +402,7 @@ public void shouldUnassignChangelogPartitionsOnShutdown() { @Test public void shouldInitializeNewActiveTasks() { active.updateRestored(EasyMock.>anyObject()); - EasyMock.expectLastCall(); + expectLastCall(); replay(); taskManager.updateNewAndRestoringTasks(); @@ -412,7 +412,7 @@ public void shouldInitializeNewActiveTasks() { @Test public void shouldInitializeNewStandbyTasks() { active.updateRestored(EasyMock.>anyObject()); - EasyMock.expectLastCall(); + expectLastCall(); replay(); taskManager.updateNewAndRestoringTasks(); @@ -421,9 +421,9 @@ public void shouldInitializeNewStandbyTasks() { @Test public void shouldRestoreStateFromChangeLogReader() { - EasyMock.expect(changeLogReader.restore(active)).andReturn(taskId0Partitions); + expect(changeLogReader.restore(active)).andReturn(taskId0Partitions); active.updateRestored(taskId0Partitions); - EasyMock.expectLastCall(); + expectLastCall(); replay(); taskManager.updateNewAndRestoringTasks(); @@ -432,13 +432,13 @@ public void shouldRestoreStateFromChangeLogReader() { @Test public void shouldResumeRestoredPartitions() { - EasyMock.expect(changeLogReader.restore(active)).andReturn(taskId0Partitions); - EasyMock.expect(active.allTasksRunning()).andReturn(true); - EasyMock.expect(consumer.assignment()).andReturn(taskId0Partitions); - EasyMock.expect(standby.running()).andReturn(Collections.emptySet()); + expect(changeLogReader.restore(active)).andReturn(taskId0Partitions); + expect(active.allTasksRunning()).andReturn(true); + expect(consumer.assignment()).andReturn(taskId0Partitions); + expect(standby.running()).andReturn(Collections.emptySet()); consumer.resume(taskId0Partitions); - EasyMock.expectLastCall(); + expectLastCall(); replay(); taskManager.updateNewAndRestoringTasks(); @@ -450,13 +450,31 @@ public void shouldAssignStandbyPartitionsWhenAllActiveTasksAreRunning() { mockAssignStandbyPartitions(1L); replay(); - assertTrue(taskManager.updateNewAndRestoringTasks()); + taskManager.updateNewAndRestoringTasks(); verify(restoreConsumer); } + @Test + public void shouldReturnTrueWhenActiveAndStandbyTasksAreRunning() { + mockAssignStandbyPartitions(1L); + expect(standby.allTasksRunning()).andReturn(true); + replay(); + + assertTrue(taskManager.updateNewAndRestoringTasks()); + } + + @Test + public void shouldReturnFalseWhenOnlyActiveTasksAreRunning() { + mockAssignStandbyPartitions(1L); + expect(standby.allTasksRunning()).andReturn(false); + replay(); + + assertFalse(taskManager.updateNewAndRestoringTasks()); + } + @Test public void shouldReturnFalseWhenThereAreStillNonRunningTasks() { - EasyMock.expect(active.allTasksRunning()).andReturn(false); + expect(active.allTasksRunning()).andReturn(false); replay(); assertFalse(taskManager.updateNewAndRestoringTasks()); @@ -466,7 +484,7 @@ public void shouldReturnFalseWhenThereAreStillNonRunningTasks() { public void shouldSeekToCheckpointedOffsetOnStandbyPartitionsWhenOffsetGreaterThanEqualTo0() { mockAssignStandbyPartitions(1L); restoreConsumer.seek(t1p0, 1L); - EasyMock.expectLastCall(); + expectLastCall(); replay(); taskManager.updateNewAndRestoringTasks(); @@ -477,7 +495,7 @@ public void shouldSeekToCheckpointedOffsetOnStandbyPartitionsWhenOffsetGreaterTh public void shouldSeekToBeginningIfOffsetIsLessThan0() { mockAssignStandbyPartitions(-1L); restoreConsumer.seekToBeginning(taskId0Partitions); - EasyMock.expectLastCall(); + expectLastCall(); replay(); taskManager.updateNewAndRestoringTasks(); @@ -486,8 +504,8 @@ public void shouldSeekToBeginningIfOffsetIsLessThan0() { @Test public void shouldCommitActiveAndStandbyTasks() { - EasyMock.expect(active.commit()).andReturn(1); - EasyMock.expect(standby.commit()).andReturn(2); + expect(active.commit()).andReturn(1); + expect(standby.commit()).andReturn(2); replay(); @@ -500,7 +518,7 @@ public void shouldPropagateExceptionFromActiveCommit() { // upgrade to strict mock to ensure no calls checkOrder(standby, true); active.commit(); - EasyMock.expectLastCall().andThrow(new RuntimeException("")); + expectLastCall().andThrow(new RuntimeException("")); replay(); try { @@ -514,7 +532,7 @@ public void shouldPropagateExceptionFromActiveCommit() { @Test public void shouldPropagateExceptionFromStandbyCommit() { - EasyMock.expect(standby.commit()).andThrow(new RuntimeException("")); + expect(standby.commit()).andThrow(new RuntimeException("")); replay(); try { @@ -534,8 +552,8 @@ public void shouldSendPurgeData() { futureDeletedRecords.complete(null); - EasyMock.expect(active.recordsToDelete()).andReturn(Collections.singletonMap(t1p1, 5L)).times(2); - EasyMock.expect(adminClient.deleteRecords(recordsToDelete)).andReturn(deleteRecordsResult).times(2); + expect(active.recordsToDelete()).andReturn(Collections.singletonMap(t1p1, 5L)).times(2); + expect(adminClient.deleteRecords(recordsToDelete)).andReturn(deleteRecordsResult).times(2); replay(); taskManager.maybePurgeCommitedRecords(); @@ -549,8 +567,8 @@ public void shouldNotSendPurgeDataIfPreviousNotDone() { final Map recordsToDelete = Collections.singletonMap(t1p1, RecordsToDelete.beforeOffset(5L)); final DeleteRecordsResult deleteRecordsResult = new DeleteRecordsResult(Collections.singletonMap(t1p1, futureDeletedRecords)); - EasyMock.expect(active.recordsToDelete()).andReturn(Collections.singletonMap(t1p1, 5L)).once(); - EasyMock.expect(adminClient.deleteRecords(recordsToDelete)).andReturn(deleteRecordsResult).once(); + expect(active.recordsToDelete()).andReturn(Collections.singletonMap(t1p1, 5L)).once(); + expect(adminClient.deleteRecords(recordsToDelete)).andReturn(deleteRecordsResult).once(); replay(); taskManager.maybePurgeCommitedRecords(); @@ -567,8 +585,8 @@ public void shouldIgnorePurgeDataErrors() { futureDeletedRecords.completeExceptionally(new Exception("KABOOM!")); - EasyMock.expect(active.recordsToDelete()).andReturn(Collections.singletonMap(t1p1, 5L)).times(2); - EasyMock.expect(adminClient.deleteRecords(recordsToDelete)).andReturn(deleteRecordsResult).times(2); + expect(active.recordsToDelete()).andReturn(Collections.singletonMap(t1p1, 5L)).times(2); + expect(adminClient.deleteRecords(recordsToDelete)).andReturn(deleteRecordsResult).times(2); replay(); taskManager.maybePurgeCommitedRecords(); @@ -578,7 +596,7 @@ public void shouldIgnorePurgeDataErrors() { @Test public void shouldMaybeCommitActiveTasks() { - EasyMock.expect(active.maybeCommitPerUserRequested()).andReturn(5); + expect(active.maybeCommitPerUserRequested()).andReturn(5); replay(); assertThat(taskManager.maybeCommitActiveTasksPerUserRequested(), equalTo(5)); @@ -587,7 +605,7 @@ public void shouldMaybeCommitActiveTasks() { @Test public void shouldProcessActiveTasks() { - EasyMock.expect(active.process(0L)).andReturn(10); + expect(active.process(0L)).andReturn(10); replay(); assertThat(taskManager.process(0L), equalTo(10)); @@ -596,7 +614,7 @@ public void shouldProcessActiveTasks() { @Test public void shouldPunctuateActiveTasks() { - EasyMock.expect(active.punctuate()).andReturn(20); + expect(active.punctuate()).andReturn(20); replay(); assertThat(taskManager.punctuate(), equalTo(20)); @@ -605,7 +623,7 @@ public void shouldPunctuateActiveTasks() { @Test public void shouldNotResumeConsumptionUntilAllStoresRestored() { - EasyMock.expect(active.allTasksRunning()).andReturn(false); + expect(active.allTasksRunning()).andReturn(false); final Consumer consumer = EasyMock.createStrictMock(Consumer.class); taskManager.setConsumer(consumer); EasyMock.replay(active, consumer); @@ -637,12 +655,12 @@ public void shouldUpdateTasksFromPartitionAssignment() { private void mockAssignStandbyPartitions(final long offset) { final StandbyTask task = EasyMock.createNiceMock(StandbyTask.class); - EasyMock.expect(active.allTasksRunning()).andReturn(true); - EasyMock.expect(standby.running()).andReturn(Collections.singletonList(task)); - EasyMock.expect(task.checkpointedOffsets()).andReturn(Collections.singletonMap(t1p0, offset)); + expect(active.allTasksRunning()).andReturn(true); + expect(standby.running()).andReturn(Collections.singletonList(task)); + expect(task.checkpointedOffsets()).andReturn(Collections.singletonMap(t1p0, offset)); restoreConsumer.assign(taskId0Partitions); - EasyMock.expectLastCall(); + expectLastCall(); EasyMock.replay(task); } From fd9a20e4167c51c6645c55ed98800b768518c863 Mon Sep 17 00:00:00 2001 From: Jason Gustafson Date: Thu, 30 May 2019 08:50:45 -0700 Subject: [PATCH 0308/1071] KAFKA-8429; Handle offset change when OffsetForLeaderEpoch inflight (#6811) It is possible for the offset of a partition to be changed while we are in the middle of validation. If the OffsetForLeaderEpoch request is in-flight and the offset changes, we need to redo the validation after it returns. We had a check for this situation previously, but it was only checking if the current leader epoch had changed. This patch fixes this and moves the validation in `SubscriptionState` where it can be protected with a lock. Additionally, this patch adds test cases for the SubscriptionState validation API. We fix a small bug handling broker downgrades. Basically we should skip validation if the latest metadata does not include leader epoch information. Reviewers: David Arthur --- .../kafka/clients/consumer/KafkaConsumer.java | 4 +- .../internals/ConsumerCoordinator.java | 4 +- .../clients/consumer/internals/Fetcher.java | 55 +--- .../consumer/internals/SubscriptionState.java | 98 ++++--- .../consumer/internals/FetcherTest.java | 56 +++- .../internals/SubscriptionStateTest.java | 240 +++++++++++++++++- 6 files changed, 368 insertions(+), 89 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index 9c1a45ca5b125..01b8989e7c338 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -1547,7 +1547,7 @@ public void seek(TopicPartition partition, long offset) { offset, Optional.empty(), // This will ensure we skip validation this.metadata.leaderAndEpoch(partition)); - this.subscriptions.seek(partition, newPosition); + this.subscriptions.seekUnvalidated(partition, newPosition); } finally { release(); } @@ -1583,7 +1583,7 @@ public void seek(TopicPartition partition, OffsetAndMetadata offsetAndMetadata) offsetAndMetadata.leaderEpoch(), currentLeaderAndEpoch); this.updateLastSeenEpochIfNewer(partition, offsetAndMetadata); - this.subscriptions.seekAndValidate(partition, newPosition); + this.subscriptions.seekUnvalidated(partition, newPosition); } finally { release(); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index f9655714cd17b..5d39da5d4b525 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -511,11 +511,11 @@ public boolean refreshCommittedOffsetsIfNeeded(Timer timer) { final ConsumerMetadata.LeaderAndEpoch leaderAndEpoch = metadata.leaderAndEpoch(tp); final SubscriptionState.FetchPosition position = new SubscriptionState.FetchPosition( offsetAndMetadata.offset(), offsetAndMetadata.leaderEpoch(), - new ConsumerMetadata.LeaderAndEpoch(leaderAndEpoch.leader, Optional.empty())); + leaderAndEpoch); log.info("Setting offset for partition {} to the committed offset {}", tp, position); entry.getValue().leaderEpoch().ifPresent(epoch -> this.metadata.updateLastSeenEpochIfNewer(entry.getKey(), epoch)); - this.subscriptions.seekAndValidate(tp, position); + this.subscriptions.seekUnvalidated(tp, position); } return true; } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index d839791fa07cd..e638963e9c835 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -19,7 +19,6 @@ import org.apache.kafka.clients.ApiVersions; import org.apache.kafka.clients.ClientResponse; import org.apache.kafka.clients.FetchSessionHandler; -import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.MetadataCache; import org.apache.kafka.clients.NodeApiVersions; import org.apache.kafka.clients.StaleMetadataException; @@ -466,7 +465,7 @@ public void validateOffsetsIfNeeded() { // Validate each partition against the current leader and epoch subscriptions.assignedPartitions().forEach(topicPartition -> { ConsumerMetadata.LeaderAndEpoch leaderAndEpoch = metadata.leaderAndEpoch(topicPartition); - subscriptions.maybeValidatePosition(topicPartition, leaderAndEpoch); + subscriptions.maybeValidatePositionForCurrentLeader(topicPartition, leaderAndEpoch); }); // Collect positions needing validation, with backoff @@ -677,7 +676,7 @@ private void resetOffsetIfNeeded(TopicPartition partition, OffsetResetStrategy r SubscriptionState.FetchPosition position = new SubscriptionState.FetchPosition( offsetData.offset, offsetData.leaderEpoch, metadata.leaderAndEpoch(partition)); offsetData.leaderEpoch.ifPresent(epoch -> metadata.updateLastSeenEpochIfNewer(partition, epoch)); - subscriptions.maybeSeek(partition, position.offset, requestedResetStrategy); + subscriptions.maybeSeekUnvalidated(partition, position.offset, requestedResetStrategy); } private void resetOffsetsAsync(Map partitionResetTimestamps) { @@ -735,16 +734,12 @@ private void validateOffsetsAsync(Map> regrouped = regroupFetchPositionsByLeader(partitionsToValidate); - regrouped.forEach((node, dataMap) -> { + regrouped.forEach((node, fetchPostitions) -> { if (node.isEmpty()) { metadata.requestUpdate(); return; } - final Map cachedLeaderAndEpochs = partitionsToValidate.entrySet() - .stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().currentLeader)); - NodeApiVersions nodeApiVersions = apiVersions.get(node.idString()); if (nodeApiVersions == null) { client.tryConnect(node); @@ -754,14 +749,14 @@ private void validateOffsetsAsync(Map future = offsetsForLeaderEpochClient.sendAsyncRequest(node, partitionsToValidate); future.addListener(new RequestFutureListener() { @@ -777,34 +772,12 @@ public void onSuccess(OffsetsForLeaderEpochClient.OffsetForEpochResult offsetsRe // for the partition. If so, it means we have experienced log truncation and need to reposition // that partition's offset. offsetsResult.endOffsets().forEach((respTopicPartition, respEndOffset) -> { - if (!subscriptions.isAssigned(respTopicPartition)) { - log.debug("Ignoring OffsetsForLeader response for partition {} which is not currently assigned.", respTopicPartition); - return; - } - - if (subscriptions.awaitingValidation(respTopicPartition)) { - SubscriptionState.FetchPosition currentPosition = subscriptions.position(respTopicPartition); - Metadata.LeaderAndEpoch currentLeader = currentPosition.currentLeader; - if (!currentLeader.equals(cachedLeaderAndEpochs.get(respTopicPartition))) { - return; - } - - if (respEndOffset.endOffset() < currentPosition.offset) { - if (subscriptions.hasDefaultOffsetResetPolicy()) { - SubscriptionState.FetchPosition newPosition = new SubscriptionState.FetchPosition( - respEndOffset.endOffset(), Optional.of(respEndOffset.leaderEpoch()), currentLeader); - log.info("Truncation detected for partition {}, resetting offset to {}", respTopicPartition, newPosition); - subscriptions.seek(respTopicPartition, newPosition); - } else { - log.warn("Truncation detected for partition {}, but no reset policy is set", respTopicPartition); - truncationWithoutResetPolicy.put(respTopicPartition, new OffsetAndMetadata( - respEndOffset.endOffset(), Optional.of(respEndOffset.leaderEpoch()), null)); - } - } else { - // Offset is fine, clear the validation state - subscriptions.completeValidation(respTopicPartition); - } - } + SubscriptionState.FetchPosition requestPosition = fetchPostitions.get(respTopicPartition); + Optional divergentOffsetOpt = subscriptions.maybeCompleteValidation( + respTopicPartition, requestPosition, respEndOffset); + divergentOffsetOpt.ifPresent(divergentOffset -> { + truncationWithoutResetPolicy.put(respTopicPartition, divergentOffset); + }); }); if (!truncationWithoutResetPolicy.isEmpty()) { @@ -814,7 +787,7 @@ public void onSuccess(OffsetsForLeaderEpochClient.OffsetForEpochResult offsetsRe @Override public void onFailure(RuntimeException e) { - subscriptions.requestFailed(dataMap.keySet(), time.milliseconds() + retryBackoffMs); + subscriptions.requestFailed(fetchPostitions.keySet(), time.milliseconds() + retryBackoffMs); metadata.requestUpdate(); if (!(e instanceof RetriableException) && !cachedOffsetForLeaderException.compareAndSet(null, e)) { @@ -1084,7 +1057,7 @@ private Map prepareFetchRequests() { // Ensure the position has an up-to-date leader subscriptions.assignedPartitions().forEach( - tp -> subscriptions.maybeValidatePosition(tp, metadata.leaderAndEpoch(tp))); + tp -> subscriptions.maybeValidatePositionForCurrentLeader(tp, metadata.leaderAndEpoch(tp))); long currentTimeMs = time.milliseconds(); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java index 640ed5595ce97..4a29ffc3e3aec 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java @@ -24,6 +24,7 @@ import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.internals.PartitionStates; +import org.apache.kafka.common.requests.EpochEndOffset; import org.apache.kafka.common.requests.IsolationLevel; import org.apache.kafka.common.utils.LogContext; import org.slf4j.Logger; @@ -50,7 +51,7 @@ * or with {@link #assignFromSubscribed(Collection)} (automatic assignment from subscription). * * Once assigned, the partition is not considered "fetchable" until its initial position has - * been set with {@link #seek(TopicPartition, FetchPosition)}. Fetchable partitions track a fetch + * been set with {@link #seekValidated(TopicPartition, FetchPosition)}. Fetchable partitions track a fetch * position which is used to set the offset of the next fetch, and a consumed position * which is the last offset that has been returned to the user. You can suspend fetching * from a partition through {@link #pause(TopicPartition)} without affecting the fetched/consumed @@ -315,7 +316,7 @@ public synchronized Set subscription() { return Collections.emptySet(); } - public Set pausedPartitions() { + public synchronized Set pausedPartitions() { return collectPartitions(TopicPartitionState::isPaused, Collectors.toSet()); } @@ -349,19 +350,19 @@ private TopicPartitionState assignedStateOrNull(TopicPartition tp) { return this.assignment.stateValue(tp); } - public synchronized void seek(TopicPartition tp, FetchPosition position) { - assignedState(tp).seek(position); + public synchronized void seekValidated(TopicPartition tp, FetchPosition position) { + assignedState(tp).seekValidated(position); } - public synchronized void seekAndValidate(TopicPartition tp, FetchPosition position) { - assignedState(tp).seekAndValidate(position); + public void seek(TopicPartition tp, long offset) { + seekValidated(tp, new FetchPosition(offset)); } - public void seek(TopicPartition tp, long offset) { - seek(tp, new FetchPosition(offset)); + public void seekUnvalidated(TopicPartition tp, FetchPosition position) { + assignedState(tp).seekUnvalidated(position); } - synchronized void maybeSeek(TopicPartition tp, long offset, OffsetResetStrategy requestedResetStrategy) { + synchronized void maybeSeekUnvalidated(TopicPartition tp, long offset, OffsetResetStrategy requestedResetStrategy) { TopicPartitionState state = assignedStateOrNull(tp); if (state == null) { log.debug("Skipping reset of partition {} since it is no longer assigned", tp); @@ -371,7 +372,7 @@ synchronized void maybeSeek(TopicPartition tp, long offset, OffsetResetStrategy log.debug("Skipping reset of partition {} since an alternative reset has been requested", tp); } else { log.info("Resetting offset for partition {} to offset {}.", tp, offset); - state.seek(new FetchPosition(offset)); + state.seekUnvalidated(new FetchPosition(offset)); } } @@ -405,23 +406,64 @@ public synchronized void position(TopicPartition tp, FetchPosition position) { assignedState(tp).position(position); } - synchronized boolean maybeValidatePosition(TopicPartition tp, Metadata.LeaderAndEpoch leaderAndEpoch) { + public synchronized boolean maybeValidatePositionForCurrentLeader(TopicPartition tp, Metadata.LeaderAndEpoch leaderAndEpoch) { return assignedState(tp).maybeValidatePosition(leaderAndEpoch); } - synchronized boolean awaitingValidation(TopicPartition tp) { + /** + * Attempt to complete validation with the end offset returned from the OffsetForLeaderEpoch request. + * @return The diverging offset if truncation was detected and no reset policy is defined. + */ + public synchronized Optional maybeCompleteValidation(TopicPartition tp, + FetchPosition requestPosition, + EpochEndOffset epochEndOffset) { + TopicPartitionState state = assignedStateOrNull(tp); + if (state == null) { + log.debug("Skipping completed validation for partition {} which is not currently assigned.", tp); + } else if (!state.awaitingValidation()) { + log.debug("Skipping completed validation for partition {} which is no longer expecting validation.", tp); + } else { + SubscriptionState.FetchPosition currentPosition = state.position; + if (!currentPosition.equals(requestPosition)) { + log.debug("Skipping completed validation for partition {} since the current position {} " + + "no longer matches the position {} when the request was sent", + tp, currentPosition, requestPosition); + } else if (epochEndOffset.endOffset() < currentPosition.offset) { + if (hasDefaultOffsetResetPolicy()) { + SubscriptionState.FetchPosition newPosition = new SubscriptionState.FetchPosition( + epochEndOffset.endOffset(), Optional.of(epochEndOffset.leaderEpoch()), + currentPosition.currentLeader); + log.info("Truncation detected for partition {} at offset {}, resetting offset to " + + "the first offset known to diverge {}", tp, currentPosition, newPosition); + state.seekValidated(newPosition); + } else { + log.warn("Truncation detected for partition {} at offset {} (the end offset from the " + + "broker is {}), but no reset policy is set", + tp, currentPosition, epochEndOffset); + return Optional.of(new OffsetAndMetadata(epochEndOffset.endOffset(), + Optional.of(epochEndOffset.leaderEpoch()), null)); + } + } else { + state.completeValidation(); + } + } + + return Optional.empty(); + } + + public synchronized boolean awaitingValidation(TopicPartition tp) { return assignedState(tp).awaitingValidation(); } public synchronized void completeValidation(TopicPartition tp) { - assignedState(tp).validate(); + assignedState(tp).completeValidation(); } public synchronized FetchPosition validPosition(TopicPartition tp) { return assignedState(tp).validPosition(); } - synchronized public FetchPosition position(TopicPartition tp) { + public synchronized FetchPosition position(TopicPartition tp) { return assignedState(tp).position; } @@ -531,11 +573,11 @@ public synchronized boolean hasAllFetchPositions() { return assignment.stream().allMatch(state -> state.value().hasValidPosition()); } - Set missingFetchPositions() { + public synchronized Set missingFetchPositions() { return collectPartitions(state -> !state.hasPosition(), Collectors.toSet()); } - private synchronized > T collectPartitions(Predicate filter, Collector collector) { + private > T collectPartitions(Predicate filter, Collector collector) { return assignment.stream() .filter(state -> filter.test(state.value())) .map(PartitionStates.PartitionState::topicPartition) @@ -560,12 +602,12 @@ public synchronized void resetMissingPositions() { throw new NoOffsetForPartitionException(partitionsWithNoOffsets); } - Set partitionsNeedingReset(long nowMs) { + public synchronized Set partitionsNeedingReset(long nowMs) { return collectPartitions(state -> state.awaitingReset() && !state.awaitingRetryBackoff(nowMs), Collectors.toSet()); } - Set partitionsNeedingValidation(long nowMs) { + public synchronized Set partitionsNeedingValidation(long nowMs) { return collectPartitions(state -> state.awaitingValidation() && !state.awaitingRetryBackoff(nowMs), Collectors.toSet()); } @@ -695,7 +737,7 @@ private boolean maybeValidatePosition(Metadata.LeaderAndEpoch currentLeaderAndEp return false; } - if (position != null && !position.safeToFetchFrom(currentLeaderAndEpoch)) { + if (position != null && !position.currentLeader.equals(currentLeaderAndEpoch)) { FetchPosition newPosition = new FetchPosition(position.offset, position.offsetEpoch, currentLeaderAndEpoch); validatePosition(newPosition); preferredReadReplica = null; @@ -704,7 +746,7 @@ private boolean maybeValidatePosition(Metadata.LeaderAndEpoch currentLeaderAndEp } private void validatePosition(FetchPosition position) { - if (position.offsetEpoch.isPresent()) { + if (position.offsetEpoch.isPresent() && position.currentLeader.epoch.isPresent()) { transitionState(FetchStates.AWAIT_VALIDATION, () -> { this.position = position; this.nextRetryTimeMs = null; @@ -721,7 +763,7 @@ private void validatePosition(FetchPosition position) { /** * Clear the awaiting validation state and enter fetching. */ - private void validate() { + private void completeValidation() { if (hasPosition()) { transitionState(FetchStates.FETCHING, () -> { this.nextRetryTimeMs = null; @@ -761,7 +803,7 @@ private boolean isPaused() { return paused; } - private void seek(FetchPosition position) { + private void seekValidated(FetchPosition position) { transitionState(FetchStates.FETCHING, () -> { this.position = position; this.resetStrategy = null; @@ -769,8 +811,8 @@ private void seek(FetchPosition position) { }); } - private void seekAndValidate(FetchPosition fetchPosition) { - seek(fetchPosition); + private void seekUnvalidated(FetchPosition fetchPosition) { + seekValidated(fetchPosition); validatePosition(fetchPosition); } @@ -934,14 +976,6 @@ public FetchPosition(long offset, Optional offsetEpoch, Metadata.Leader this.currentLeader = Objects.requireNonNull(currentLeader); } - /** - * Test if it is "safe" to fetch from a given leader and epoch. This effectively is testing if - * {@link Metadata.LeaderAndEpoch} known to the subscription is equal to the one supplied by the caller. - */ - boolean safeToFetchFrom(Metadata.LeaderAndEpoch leaderAndEpoch) { - return !currentLeader.leader.isEmpty() && currentLeader.equals(leaderAndEpoch); - } - @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 1e64e3ba10015..44c00c48a3ee6 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -75,6 +75,7 @@ import org.apache.kafka.common.requests.ListOffsetResponse; import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.OffsetsForLeaderEpochRequest; import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse; import org.apache.kafka.common.requests.ResponseHeader; import org.apache.kafka.common.serialization.ByteArrayDeserializer; @@ -1504,7 +1505,7 @@ public void testEarlierOffsetResetArrivesLate() throws InterruptedException { Object result = invocation.callRealMethod(); latchEarliestDone.countDown(); return result; - }).when(subscriptions).maybeSeek(tp0, 0L, OffsetResetStrategy.EARLIEST); + }).when(subscriptions).maybeSeekUnvalidated(tp0, 0L, OffsetResetStrategy.EARLIEST); es.submit(() -> { subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.EARLIEST); @@ -2795,8 +2796,8 @@ public void testConsumingViaIncrementalFetchRequests() { List> records; assignFromUser(new HashSet<>(Arrays.asList(tp0, tp1))); - subscriptions.seek(tp0, new SubscriptionState.FetchPosition(0, Optional.empty(), metadata.leaderAndEpoch(tp0))); - subscriptions.seek(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.leaderAndEpoch(tp1))); + subscriptions.seekValidated(tp0, new SubscriptionState.FetchPosition(0, Optional.empty(), metadata.leaderAndEpoch(tp0))); + subscriptions.seekValidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.leaderAndEpoch(tp1))); // Fetch some records and establish an incremental fetch session. LinkedHashMap> partitions1 = new LinkedHashMap<>(); @@ -3276,7 +3277,7 @@ public void testOffsetValidationAwaitsNodeApiVersion() { // Seek with a position and leader+epoch Metadata.LeaderAndEpoch leaderAndEpoch = new Metadata.LeaderAndEpoch( metadata.leaderAndEpoch(tp0).leader, Optional.of(epochOne)); - subscriptions.seekAndValidate(tp0, new SubscriptionState.FetchPosition(20L, Optional.of(epochOne), leaderAndEpoch)); + subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(20L, Optional.of(epochOne), leaderAndEpoch)); assertFalse(client.isConnected(node.idString())); assertTrue(subscriptions.awaitingValidation(tp0)); @@ -3325,7 +3326,7 @@ public void testOffsetValidationSkippedForOldBroker() { // Seek with a position and leader+epoch Metadata.LeaderAndEpoch leaderAndEpoch = new Metadata.LeaderAndEpoch( metadata.leaderAndEpoch(tp0).leader, Optional.of(epochOne)); - subscriptions.seek(tp0, new SubscriptionState.FetchPosition(0, Optional.of(epochOne), leaderAndEpoch)); + subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(0, Optional.of(epochOne), leaderAndEpoch)); // Update metadata to epoch=2, enter validation metadata.update(TestUtils.metadataUpdateWith("dummy", 1, @@ -3336,6 +3337,45 @@ public void testOffsetValidationSkippedForOldBroker() { assertFalse(subscriptions.awaitingValidation(tp0)); } + @Test + public void testOffsetValidationHandlesSeekWithInflightOffsetForLeaderRequest() { + buildFetcher(); + assignFromUser(singleton(tp0)); + + Map partitionCounts = new HashMap<>(); + partitionCounts.put(tp0.topic(), 4); + + final int epochOne = 1; + + metadata.update(TestUtils.metadataUpdateWith("dummy", 1, Collections.emptyMap(), partitionCounts, tp -> epochOne), 0L); + + // Offset validation requires OffsetForLeaderEpoch request v3 or higher + Node node = metadata.fetch().nodes().get(0); + apiVersions.update(node.idString(), NodeApiVersions.create()); + + Metadata.LeaderAndEpoch leaderAndEpoch = new Metadata.LeaderAndEpoch(metadata.leaderAndEpoch(tp0).leader, Optional.of(epochOne)); + subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(0, Optional.of(epochOne), leaderAndEpoch)); + + fetcher.validateOffsetsIfNeeded(); + consumerClient.poll(time.timer(Duration.ZERO)); + assertTrue(subscriptions.awaitingValidation(tp0)); + assertTrue(client.hasInFlightRequests()); + + // While the OffsetForLeaderEpoch request is in-flight, we seek to a different offset. + subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(5, Optional.of(epochOne), leaderAndEpoch)); + assertTrue(subscriptions.awaitingValidation(tp0)); + + client.respond(request -> { + OffsetsForLeaderEpochRequest epochRequest = (OffsetsForLeaderEpochRequest) request; + OffsetsForLeaderEpochRequest.PartitionData partitionData = epochRequest.epochsByTopicPartition().get(tp0); + return partitionData.currentLeaderEpoch.equals(Optional.of(epochOne)) && partitionData.leaderEpoch == epochOne; + }, new OffsetsForLeaderEpochResponse(singletonMap(tp0, new EpochEndOffset(0, 0L)))); + consumerClient.poll(time.timer(Duration.ZERO)); + + // The response should be ignored since we were validating a different position. + assertTrue(subscriptions.awaitingValidation(tp0)); + } + @Test public void testOffsetValidationFencing() { buildFetcher(); @@ -3357,7 +3397,7 @@ public void testOffsetValidationFencing() { // Seek with a position and leader+epoch Metadata.LeaderAndEpoch leaderAndEpoch = new Metadata.LeaderAndEpoch(metadata.leaderAndEpoch(tp0).leader, Optional.of(epochOne)); - subscriptions.seek(tp0, new SubscriptionState.FetchPosition(0, Optional.of(epochOne), leaderAndEpoch)); + subscriptions.seekValidated(tp0, new SubscriptionState.FetchPosition(0, Optional.of(epochOne), leaderAndEpoch)); // Update metadata to epoch=2, enter validation metadata.update(TestUtils.metadataUpdateWith("dummy", 1, Collections.emptyMap(), partitionCounts, tp -> epochTwo), 0L); @@ -3371,7 +3411,7 @@ public void testOffsetValidationFencing() { Optional.of(epochTwo), new Metadata.LeaderAndEpoch(leaderAndEpoch.leader, Optional.of(epochTwo))); subscriptions.position(tp0, nextPosition); - subscriptions.maybeValidatePosition(tp0, new Metadata.LeaderAndEpoch(leaderAndEpoch.leader, Optional.of(epochThree))); + subscriptions.maybeValidatePositionForCurrentLeader(tp0, new Metadata.LeaderAndEpoch(leaderAndEpoch.leader, Optional.of(epochThree))); // Prepare offset list response from async validation with epoch=2 Map endOffsetMap = new HashMap<>(); @@ -3427,7 +3467,7 @@ public void testTruncationDetected() { // Seek Metadata.LeaderAndEpoch leaderAndEpoch = new Metadata.LeaderAndEpoch(metadata.leaderAndEpoch(tp0).leader, Optional.of(1)); - subscriptions.seek(tp0, new SubscriptionState.FetchPosition(0, Optional.of(1), leaderAndEpoch)); + subscriptions.seekValidated(tp0, new SubscriptionState.FetchPosition(0, Optional.of(1), leaderAndEpoch)); // Check for truncation, this should cause tp0 to go into validation fetcher.validateOffsetsIfNeeded(); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java index 528c2b997dd4c..484b9de0a9bce 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java @@ -18,9 +18,11 @@ import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.OffsetResetStrategy; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.requests.EpochEndOffset; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.test.TestUtils; @@ -42,9 +44,7 @@ public class SubscriptionStateTest { - private final SubscriptionState state = new SubscriptionState( - new LogContext(), - OffsetResetStrategy.EARLIEST); + private SubscriptionState state = new SubscriptionState(new LogContext(), OffsetResetStrategy.EARLIEST); private final String topic = "test"; private final String topic1 = "test1"; private final TopicPartition tp0 = new TopicPartition(topic, 0); @@ -340,13 +340,245 @@ public void testPreferredReadReplicaLease() { assertFalse(state.preferredReadReplica(tp0, 31L).isPresent()); } + @Test + public void testSeekUnvalidatedWithNoOffsetEpoch() { + Node broker1 = new Node(1, "localhost", 9092); + state.assignFromUser(Collections.singleton(tp0)); + + // Seek with no offset epoch requires no validation no matter what the current leader is + state.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(0L, Optional.empty(), + new Metadata.LeaderAndEpoch(broker1, Optional.of(5)))); + assertTrue(state.hasValidPosition(tp0)); + assertFalse(state.awaitingValidation(tp0)); + + assertFalse(state.maybeValidatePositionForCurrentLeader(tp0, new Metadata.LeaderAndEpoch(broker1, Optional.empty()))); + assertTrue(state.hasValidPosition(tp0)); + assertFalse(state.awaitingValidation(tp0)); + + assertFalse(state.maybeValidatePositionForCurrentLeader(tp0, new Metadata.LeaderAndEpoch(broker1, Optional.of(10)))); + assertTrue(state.hasValidPosition(tp0)); + assertFalse(state.awaitingValidation(tp0)); + } + + @Test + public void testSeekUnvalidatedWithNoEpochClearsAwaitingValidation() { + Node broker1 = new Node(1, "localhost", 9092); + state.assignFromUser(Collections.singleton(tp0)); + + // Seek with no offset epoch requires no validation no matter what the current leader is + state.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(0L, Optional.of(2), + new Metadata.LeaderAndEpoch(broker1, Optional.of(5)))); + assertFalse(state.hasValidPosition(tp0)); + assertTrue(state.awaitingValidation(tp0)); + + state.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(0L, Optional.empty(), + new Metadata.LeaderAndEpoch(broker1, Optional.of(5)))); + assertTrue(state.hasValidPosition(tp0)); + assertFalse(state.awaitingValidation(tp0)); + } + + @Test + public void testSeekUnvalidatedWithOffsetEpoch() { + Node broker1 = new Node(1, "localhost", 9092); + state.assignFromUser(Collections.singleton(tp0)); + + state.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(0L, Optional.of(2), + new Metadata.LeaderAndEpoch(broker1, Optional.of(5)))); + assertFalse(state.hasValidPosition(tp0)); + assertTrue(state.awaitingValidation(tp0)); + + // Update using the current leader and epoch + assertTrue(state.maybeValidatePositionForCurrentLeader(tp0, new Metadata.LeaderAndEpoch(broker1, Optional.of(5)))); + assertFalse(state.hasValidPosition(tp0)); + assertTrue(state.awaitingValidation(tp0)); + + // Update with a newer leader and epoch + assertTrue(state.maybeValidatePositionForCurrentLeader(tp0, new Metadata.LeaderAndEpoch(broker1, Optional.of(15)))); + assertFalse(state.hasValidPosition(tp0)); + assertTrue(state.awaitingValidation(tp0)); + + // If the updated leader has no epoch information, then skip validation and begin fetching + assertFalse(state.maybeValidatePositionForCurrentLeader(tp0, new Metadata.LeaderAndEpoch(broker1, Optional.empty()))); + assertTrue(state.hasValidPosition(tp0)); + assertFalse(state.awaitingValidation(tp0)); + } + + @Test + public void testSeekValidatedShouldClearAwaitingValidation() { + Node broker1 = new Node(1, "localhost", 9092); + state.assignFromUser(Collections.singleton(tp0)); + + state.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(10L, Optional.of(5), + new Metadata.LeaderAndEpoch(broker1, Optional.of(10)))); + assertFalse(state.hasValidPosition(tp0)); + assertTrue(state.awaitingValidation(tp0)); + assertEquals(10L, state.position(tp0).offset); + + state.seekValidated(tp0, new SubscriptionState.FetchPosition(8L, Optional.of(4), + new Metadata.LeaderAndEpoch(broker1, Optional.of(10)))); + assertTrue(state.hasValidPosition(tp0)); + assertFalse(state.awaitingValidation(tp0)); + assertEquals(8L, state.position(tp0).offset); + } + + @Test + public void testCompleteValidationShouldClearAwaitingValidation() { + Node broker1 = new Node(1, "localhost", 9092); + state.assignFromUser(Collections.singleton(tp0)); + + state.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(10L, Optional.of(5), + new Metadata.LeaderAndEpoch(broker1, Optional.of(10)))); + assertFalse(state.hasValidPosition(tp0)); + assertTrue(state.awaitingValidation(tp0)); + assertEquals(10L, state.position(tp0).offset); + + state.completeValidation(tp0); + assertTrue(state.hasValidPosition(tp0)); + assertFalse(state.awaitingValidation(tp0)); + assertEquals(10L, state.position(tp0).offset); + } + + @Test + public void testOffsetResetWhileAwaitingValidation() { + Node broker1 = new Node(1, "localhost", 9092); + state.assignFromUser(Collections.singleton(tp0)); + + state.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(10L, Optional.of(5), + new Metadata.LeaderAndEpoch(broker1, Optional.of(10)))); + assertTrue(state.awaitingValidation(tp0)); + + state.requestOffsetReset(tp0, OffsetResetStrategy.EARLIEST); + assertFalse(state.awaitingValidation(tp0)); + assertTrue(state.isOffsetResetNeeded(tp0)); + } + + @Test + public void testMaybeCompleteValidation() { + Node broker1 = new Node(1, "localhost", 9092); + state.assignFromUser(Collections.singleton(tp0)); + + int currentEpoch = 10; + long initialOffset = 10L; + int initialOffsetEpoch = 5; + + SubscriptionState.FetchPosition initialPosition = new SubscriptionState.FetchPosition(initialOffset, + Optional.of(initialOffsetEpoch), new Metadata.LeaderAndEpoch(broker1, Optional.of(currentEpoch))); + state.seekUnvalidated(tp0, initialPosition); + assertTrue(state.awaitingValidation(tp0)); + + Optional divergentOffsetMetadataOpt = state.maybeCompleteValidation(tp0, initialPosition, + new EpochEndOffset(initialOffsetEpoch, initialOffset + 5)); + assertEquals(Optional.empty(), divergentOffsetMetadataOpt); + assertFalse(state.awaitingValidation(tp0)); + assertEquals(initialPosition, state.position(tp0)); + } + + @Test + public void testMaybeCompleteValidationAfterPositionChange() { + Node broker1 = new Node(1, "localhost", 9092); + state.assignFromUser(Collections.singleton(tp0)); + + int currentEpoch = 10; + long initialOffset = 10L; + int initialOffsetEpoch = 5; + long updateOffset = 20L; + int updateOffsetEpoch = 8; + + SubscriptionState.FetchPosition initialPosition = new SubscriptionState.FetchPosition(initialOffset, + Optional.of(initialOffsetEpoch), new Metadata.LeaderAndEpoch(broker1, Optional.of(currentEpoch))); + state.seekUnvalidated(tp0, initialPosition); + assertTrue(state.awaitingValidation(tp0)); + + SubscriptionState.FetchPosition updatePosition = new SubscriptionState.FetchPosition(updateOffset, + Optional.of(updateOffsetEpoch), new Metadata.LeaderAndEpoch(broker1, Optional.of(currentEpoch))); + state.seekUnvalidated(tp0, updatePosition); + + Optional divergentOffsetMetadataOpt = state.maybeCompleteValidation(tp0, initialPosition, + new EpochEndOffset(initialOffsetEpoch, initialOffset + 5)); + assertEquals(Optional.empty(), divergentOffsetMetadataOpt); + assertTrue(state.awaitingValidation(tp0)); + assertEquals(updatePosition, state.position(tp0)); + } + + @Test + public void testMaybeCompleteValidationAfterOffsetReset() { + Node broker1 = new Node(1, "localhost", 9092); + state.assignFromUser(Collections.singleton(tp0)); + + int currentEpoch = 10; + long initialOffset = 10L; + int initialOffsetEpoch = 5; + + SubscriptionState.FetchPosition initialPosition = new SubscriptionState.FetchPosition(initialOffset, + Optional.of(initialOffsetEpoch), new Metadata.LeaderAndEpoch(broker1, Optional.of(currentEpoch))); + state.seekUnvalidated(tp0, initialPosition); + assertTrue(state.awaitingValidation(tp0)); + + state.requestOffsetReset(tp0); + + Optional divergentOffsetMetadataOpt = state.maybeCompleteValidation(tp0, initialPosition, + new EpochEndOffset(initialOffsetEpoch, initialOffset + 5)); + assertEquals(Optional.empty(), divergentOffsetMetadataOpt); + assertFalse(state.awaitingValidation(tp0)); + assertTrue(state.isOffsetResetNeeded(tp0)); + } + + @Test + public void testTruncationDetectionWithResetPolicy() { + Node broker1 = new Node(1, "localhost", 9092); + state.assignFromUser(Collections.singleton(tp0)); + + int currentEpoch = 10; + long initialOffset = 10L; + int initialOffsetEpoch = 5; + long divergentOffset = 5L; + int divergentOffsetEpoch = 7; + + SubscriptionState.FetchPosition initialPosition = new SubscriptionState.FetchPosition(initialOffset, + Optional.of(initialOffsetEpoch), new Metadata.LeaderAndEpoch(broker1, Optional.of(currentEpoch))); + state.seekUnvalidated(tp0, initialPosition); + assertTrue(state.awaitingValidation(tp0)); + + Optional divergentOffsetMetadata = state.maybeCompleteValidation(tp0, initialPosition, + new EpochEndOffset(divergentOffsetEpoch, divergentOffset)); + assertEquals(Optional.empty(), divergentOffsetMetadata); + assertFalse(state.awaitingValidation(tp0)); + + SubscriptionState.FetchPosition updatedPosition = new SubscriptionState.FetchPosition(divergentOffset, + Optional.of(divergentOffsetEpoch), new Metadata.LeaderAndEpoch(broker1, Optional.of(currentEpoch))); + assertEquals(updatedPosition, state.position(tp0)); + } + + @Test + public void testTruncationDetectionWithoutResetPolicy() { + Node broker1 = new Node(1, "localhost", 9092); + state = new SubscriptionState(new LogContext(), OffsetResetStrategy.NONE); + state.assignFromUser(Collections.singleton(tp0)); + + int currentEpoch = 10; + long initialOffset = 10L; + int initialOffsetEpoch = 5; + long divergentOffset = 5L; + int divergentOffsetEpoch = 7; + + SubscriptionState.FetchPosition initialPosition = new SubscriptionState.FetchPosition(initialOffset, + Optional.of(initialOffsetEpoch), new Metadata.LeaderAndEpoch(broker1, Optional.of(currentEpoch))); + state.seekUnvalidated(tp0, initialPosition); + assertTrue(state.awaitingValidation(tp0)); + + Optional divergentOffsetMetadata = state.maybeCompleteValidation(tp0, initialPosition, + new EpochEndOffset(divergentOffsetEpoch, divergentOffset)); + assertEquals(Optional.of(new OffsetAndMetadata(divergentOffset, Optional.of(divergentOffsetEpoch), "")), + divergentOffsetMetadata); + assertTrue(state.awaitingValidation(tp0)); + } + private static class MockRebalanceListener implements ConsumerRebalanceListener { public Collection revoked; public Collection assigned; public int revokedCount = 0; public int assignedCount = 0; - @Override public void onPartitionsAssigned(Collection partitions) { this.assigned = partitions; From 77e6e8ec054608a30626271b4952b63294a93c3b Mon Sep 17 00:00:00 2001 From: "Matthias J. Sax" Date: Thu, 30 May 2019 09:46:12 -0700 Subject: [PATCH 0309/1071] KAFKA-6455: Update integration tests to verify result timestamps (#6751) Reviewers: Bill Bejeck , John Roesler --- .../kstream/internals/KTableReduce.java | 3 +- .../AbstractJoinIntegrationTest.java | 68 ++- .../AbstractResetIntegrationTest.java | 36 +- .../FineGrainedAutoResetIntegrationTest.java | 7 +- .../GlobalKTableIntegrationTest.java | 113 +++-- ...StreamAggregationDedupIntegrationTest.java | 63 ++- .../KStreamAggregationIntegrationTest.java | 227 +++++---- .../StreamStreamJoinIntegrationTest.java | 349 +++++++++---- .../StreamTableJoinIntegrationTest.java | 72 +-- .../TableTableJoinIntegrationTest.java | 479 +++++++++++------- .../utils/IntegrationTestUtils.java | 91 +++- .../kafka/test/MockProcessorSupplier.java | 6 +- 12 files changed, 922 insertions(+), 592 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableReduce.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableReduce.java index 1b1a2bf3420d8..171912fd23ac9 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableReduce.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableReduce.java @@ -80,7 +80,7 @@ public void process(final K key, final Change value) { final ValueAndTimestamp oldAggAndTimestamp = store.get(key); final V oldAgg = getValueOrNull(oldAggAndTimestamp); final V intermediateAgg; - long newTimestamp = context().timestamp(); + long newTimestamp; // first try to remove the old value if (value.oldValue != null && oldAgg != null) { @@ -88,6 +88,7 @@ public void process(final K key, final Change value) { newTimestamp = Math.max(context().timestamp(), oldAggAndTimestamp.timestamp()); } else { intermediateAgg = oldAgg; + newTimestamp = context().timestamp(); } // then try to add the new value diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/AbstractJoinIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/AbstractJoinIntegrationTest.java index 7fa108d59a65b..daea1d732ae0b 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/AbstractJoinIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/AbstractJoinIntegrationTest.java @@ -27,6 +27,7 @@ import org.apache.kafka.common.serialization.StringSerializer; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.KeyValueTimestamp; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster; @@ -35,6 +36,7 @@ import org.apache.kafka.streams.state.KeyValueIterator; import org.apache.kafka.streams.state.QueryableStoreTypes; import org.apache.kafka.streams.state.ReadOnlyKeyValueStore; +import org.apache.kafka.streams.state.ValueAndTimestamp; import org.apache.kafka.test.IntegrationTest; import org.apache.kafka.test.TestUtils; import org.junit.After; @@ -51,6 +53,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Properties; import java.util.concurrent.atomic.AtomicBoolean; @@ -86,7 +89,7 @@ public static Collection data() { static final String INPUT_TOPIC_RIGHT = "inputTopicRight"; static final String INPUT_TOPIC_LEFT = "inputTopicLeft"; static final String OUTPUT_TOPIC = "outputTopic"; - private final long anyUniqueKey = 0L; + static final long ANY_UNIQUE_KEY = 0L; private final static Properties PRODUCER_CONFIG = new Properties(); private final static Properties RESULT_CONSUMER_CONFIG = new Properties(); @@ -163,13 +166,13 @@ public void cleanup() throws InterruptedException { CLUSTER.deleteAllTopicsAndWait(120000); } - private void checkResult(final String outputTopic, final List expectedResult) throws InterruptedException { - final List result = IntegrationTestUtils.waitUntilMinValuesRecordsReceived(RESULT_CONSUMER_CONFIG, outputTopic, expectedResult.size(), 30 * 1000L); - assertThat(result, is(expectedResult)); + private void checkResult(final String outputTopic, final List> expectedResult) throws InterruptedException { + IntegrationTestUtils.verifyKeyValueTimestamps(RESULT_CONSUMER_CONFIG, outputTopic, expectedResult); } - private void checkResult(final String outputTopic, final String expectedFinalResult, final int expectedTotalNumRecords) throws InterruptedException { - final List result = IntegrationTestUtils.waitUntilMinValuesRecordsReceived(RESULT_CONSUMER_CONFIG, outputTopic, expectedTotalNumRecords, 30 * 1000L); + private void checkResult(final String outputTopic, final KeyValueTimestamp expectedFinalResult, final int expectedTotalNumRecords) throws InterruptedException { + final List> result = + IntegrationTestUtils.waitUntilMinKeyValueWithTimestampRecordsReceived(RESULT_CONSUMER_CONFIG, outputTopic, expectedTotalNumRecords, 30 * 1000L); assertThat(result.get(result.size() - 1), is(expectedFinalResult)); } @@ -177,7 +180,7 @@ private void checkResult(final String outputTopic, final String expectedFinalRes * Runs the actual test. Checks the result after each input record to ensure fixed processing order. * If an input tuple does not trigger any result, "expectedResult" should contain a "null" entry */ - void runTest(final List> expectedResult) throws Exception { + void runTest(final List>> expectedResult) throws Exception { runTest(expectedResult, null); } @@ -186,28 +189,34 @@ void runTest(final List> expectedResult) throws Exception { * Runs the actual test. Checks the result after each input record to ensure fixed processing order. * If an input tuple does not trigger any result, "expectedResult" should contain a "null" entry */ - void runTest(final List> expectedResult, final String storeName) throws Exception { + void runTest(final List>> expectedResult, final String storeName) throws Exception { assert expectedResult.size() == input.size(); IntegrationTestUtils.purgeLocalStreamsState(STREAMS_CONFIG); streams = new KafkaStreams(builder.build(), STREAMS_CONFIG); - String expectedFinalResult = null; + KeyValueTimestamp expectedFinalResult = null; try { streams.start(); - long ts = System.currentTimeMillis(); + final long firstTimestamp = System.currentTimeMillis(); + long ts = firstTimestamp; - final Iterator> resultIterator = expectedResult.iterator(); + final Iterator>> resultIterator = expectedResult.iterator(); for (final Input singleInput : input) { producer.send(new ProducerRecord<>(singleInput.topic, null, ++ts, singleInput.record.key, singleInput.record.value)).get(); - final List expected = resultIterator.next(); + final List> expected = resultIterator.next(); if (expected != null) { - checkResult(OUTPUT_TOPIC, expected); - expectedFinalResult = expected.get(expected.size() - 1); + final List> updatedExpected = new LinkedList<>(); + for (final KeyValueTimestamp record : expected) { + updatedExpected.add(new KeyValueTimestamp<>(record.key(), record.value(), firstTimestamp + record.timestamp())); + } + + checkResult(OUTPUT_TOPIC, updatedExpected); + expectedFinalResult = updatedExpected.get(expected.size() - 1); } } @@ -222,21 +231,22 @@ void runTest(final List> expectedResult, final String storeName) th /* * Runs the actual test. Checks the final result only after expected number of records have been consumed. */ - void runTest(final String expectedFinalResult) throws Exception { + void runTest(final KeyValueTimestamp expectedFinalResult) throws Exception { runTest(expectedFinalResult, null); } /* * Runs the actual test. Checks the final result only after expected number of records have been consumed. */ - void runTest(final String expectedFinalResult, final String storeName) throws Exception { + void runTest(final KeyValueTimestamp expectedFinalResult, final String storeName) throws Exception { IntegrationTestUtils.purgeLocalStreamsState(STREAMS_CONFIG); streams = new KafkaStreams(builder.build(), STREAMS_CONFIG); try { streams.start(); - long ts = System.currentTimeMillis(); + final long firstTimestamp = System.currentTimeMillis(); + long ts = firstTimestamp; for (final Input singleInput : input) { producer.send(new ProducerRecord<>(singleInput.topic, null, ++ts, singleInput.record.key, singleInput.record.value)).get(); @@ -244,10 +254,15 @@ void runTest(final String expectedFinalResult, final String storeName) throws Ex TestUtils.waitForCondition(() -> finalResultReached.get(), "Never received expected final result."); - checkResult(OUTPUT_TOPIC, expectedFinalResult, numRecordsExpected); + final KeyValueTimestamp updatedExpectedFinalResult = + new KeyValueTimestamp<>( + expectedFinalResult.key(), + expectedFinalResult.value(), + firstTimestamp + expectedFinalResult.timestamp()); + checkResult(OUTPUT_TOPIC, updatedExpectedFinalResult, numRecordsExpected); if (storeName != null) { - checkQueryableStore(storeName, expectedFinalResult); + checkQueryableStore(storeName, updatedExpectedFinalResult); } } finally { streams.close(); @@ -257,15 +272,16 @@ void runTest(final String expectedFinalResult, final String storeName) throws Ex /* * Checks the embedded queryable state store snapshot */ - private void checkQueryableStore(final String queryableName, final String expectedFinalResult) { - final ReadOnlyKeyValueStore store = streams.store(queryableName, QueryableStoreTypes.keyValueStore()); + private void checkQueryableStore(final String queryableName, final KeyValueTimestamp expectedFinalResult) { + final ReadOnlyKeyValueStore> store = streams.store(queryableName, QueryableStoreTypes.timestampedKeyValueStore()); - final KeyValueIterator all = store.all(); - final KeyValue onlyEntry = all.next(); + final KeyValueIterator> all = store.all(); + final KeyValue> onlyEntry = all.next(); try { - assertThat(onlyEntry.key, is(anyUniqueKey)); - assertThat(onlyEntry.value, is(expectedFinalResult)); + assertThat(onlyEntry.key, is(expectedFinalResult.key())); + assertThat(onlyEntry.value.value(), is(expectedFinalResult.value())); + assertThat(onlyEntry.value.timestamp(), is(expectedFinalResult.timestamp())); assertThat(all.hasNext(), is(false)); } finally { all.close(); @@ -278,7 +294,7 @@ private final class Input { Input(final String topic, final V value) { this.topic = topic; - record = KeyValue.pair(anyUniqueKey, value); + record = KeyValue.pair(ANY_UNIQUE_KEY, value); } } } diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/AbstractResetIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/AbstractResetIntegrationTest.java index c9ae1bbcb8522..000f2993c739a 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/AbstractResetIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/AbstractResetIntegrationTest.java @@ -16,7 +16,7 @@ */ package org.apache.kafka.streams.integration; -import java.time.Duration; +import kafka.tools.StreamsResetter; import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.ConsumerGroupDescription; @@ -39,10 +39,8 @@ import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster; import org.apache.kafka.streams.integration.utils.IntegrationTestUtils; import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.KeyValueMapper; import org.apache.kafka.streams.kstream.Produced; import org.apache.kafka.streams.kstream.TimeWindows; -import org.apache.kafka.streams.kstream.Windowed; import org.apache.kafka.test.IntegrationTest; import org.apache.kafka.test.TestCondition; import org.apache.kafka.test.TestUtils; @@ -56,6 +54,7 @@ import java.io.File; import java.io.FileWriter; import java.text.SimpleDateFormat; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; @@ -65,8 +64,6 @@ import java.util.Properties; import java.util.concurrent.ExecutionException; -import kafka.tools.StreamsResetter; - import static java.time.Duration.ofMillis; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; @@ -223,7 +220,7 @@ private void add10InputElements() throws java.util.concurrent.ExecutionException } } - void shouldNotAllowToResetWhileStreamsIsRunning() throws Exception { + void shouldNotAllowToResetWhileStreamsIsRunning() { appID = testId + "-not-reset-during-runtime"; final String[] parameters = new String[] { "--application-id", appID, @@ -390,7 +387,6 @@ void testReprocessingFromFileAfterResetWithoutIntermediateUserTopic() throws Exc final File resetFile = File.createTempFile("reset", ".csv"); try (final BufferedWriter writer = new BufferedWriter(new FileWriter(resetFile))) { writer.write(INPUT_TOPIC + ",0,1"); - writer.close(); } streams = new KafkaStreams(setupTopologyWithoutIntermediateUserTopic(), streamsConfig); @@ -434,7 +430,6 @@ void testReprocessingFromDateTimeAfterResetWithoutIntermediateUserTopic() throws final File resetFile = File.createTempFile("reset", ".csv"); try (final BufferedWriter writer = new BufferedWriter(new FileWriter(resetFile))) { writer.write(INPUT_TOPIC + ",0,1"); - writer.close(); } streams = new KafkaStreams(setupTopologyWithoutIntermediateUserTopic(), streamsConfig); @@ -482,7 +477,6 @@ void testReprocessingByDurationAfterResetWithoutIntermediateUserTopic() throws E final File resetFile = File.createTempFile("reset", ".csv"); try (final BufferedWriter writer = new BufferedWriter(new FileWriter(resetFile))) { writer.write(INPUT_TOPIC + ",0,1"); - writer.close(); } streams = new KafkaStreams(setupTopologyWithoutIntermediateUserTopic(), streamsConfig); @@ -514,12 +508,7 @@ private Topology setupTopologyWithIntermediateUserTopic(final String outputTopic final KStream input = builder.stream(INPUT_TOPIC); // use map to trigger internal re-partitioning before groupByKey - input.map(new KeyValueMapper>() { - @Override - public KeyValue apply(final Long key, final String value) { - return new KeyValue<>(key, value); - } - }) + input.map(KeyValue::new) .groupByKey() .count() .toStream() @@ -530,12 +519,7 @@ public KeyValue apply(final Long key, final String value) { .windowedBy(TimeWindows.of(ofMillis(35)).advanceBy(ofMillis(10))) .count() .toStream() - .map(new KeyValueMapper, Long, KeyValue>() { - @Override - public KeyValue apply(final Windowed key, final Long value) { - return new KeyValue<>(key.window().start() + key.window().end(), value); - } - }) + .map((key, value) -> new KeyValue<>(key.window().start() + key.window().end(), value)) .to(outputTopic2, Produced.with(Serdes.Long(), Serdes.Long())); return builder.build(); @@ -547,12 +531,8 @@ private Topology setupTopologyWithoutIntermediateUserTopic() { final KStream input = builder.stream(INPUT_TOPIC); // use map to trigger internal re-partitioning before groupByKey - input.map(new KeyValueMapper>() { - @Override - public KeyValue apply(final Long key, final String value) { - return new KeyValue<>(key, key); - } - }).to(OUTPUT_TOPIC, Produced.with(Serdes.Long(), Serdes.Long())); + input.map((key, value) -> new KeyValue<>(key, key)) + .to(OUTPUT_TOPIC, Produced.with(Serdes.Long(), Serdes.Long())); return builder.build(); } @@ -590,7 +570,7 @@ private void cleanGlobal(final boolean withIntermediateTopics, parameterList.add(resetScenarioArg); } - final String[] parameters = parameterList.toArray(new String[parameterList.size()]); + final String[] parameters = parameterList.toArray(new String[0]); final Properties cleanUpConfig = new Properties(); cleanUpConfig.put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, 100); diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/FineGrainedAutoResetIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/FineGrainedAutoResetIntegrationTest.java index 87d6a1693090a..45ea71cf9f534 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/FineGrainedAutoResetIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/FineGrainedAutoResetIntegrationTest.java @@ -17,6 +17,7 @@ package org.apache.kafka.streams.integration; +import kafka.utils.MockTime; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; @@ -55,8 +56,6 @@ import java.util.Properties; import java.util.regex.Pattern; -import kafka.utils.MockTime; - import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; @@ -191,8 +190,8 @@ private void shouldOnlyReadForEarliest( final StreamsBuilder builder = new StreamsBuilder(); - final KStream pattern1Stream = builder.stream(Pattern.compile("topic-\\d" + topicSuffix), Consumed.with(Topology.AutoOffsetReset.EARLIEST)); - final KStream pattern2Stream = builder.stream(Pattern.compile("topic-[A-D]" + topicSuffix), Consumed.with(Topology.AutoOffsetReset.LATEST)); + final KStream pattern1Stream = builder.stream(Pattern.compile("topic-\\d" + topicSuffix), Consumed.with(Topology.AutoOffsetReset.EARLIEST)); + final KStream pattern2Stream = builder.stream(Pattern.compile("topic-[A-D]" + topicSuffix), Consumed.with(Topology.AutoOffsetReset.LATEST)); final KStream namedTopicsStream = builder.stream(Arrays.asList(topicY, topicZ)); pattern1Stream.to(outputTopic, Produced.with(stringSerde, stringSerde)); diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/GlobalKTableIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/GlobalKTableIntegrationTest.java index d3e0d245693a3..0a9148d61fb10 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/GlobalKTableIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/GlobalKTableIntegrationTest.java @@ -29,7 +29,6 @@ import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster; import org.apache.kafka.streams.integration.utils.IntegrationTestUtils; import org.apache.kafka.streams.kstream.Consumed; -import org.apache.kafka.streams.kstream.ForeachAction; import org.apache.kafka.streams.kstream.GlobalKTable; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.KeyValueMapper; @@ -39,7 +38,9 @@ import org.apache.kafka.streams.state.QueryableStoreTypes; import org.apache.kafka.streams.state.ReadOnlyKeyValueStore; import org.apache.kafka.streams.state.Stores; +import org.apache.kafka.streams.state.ValueAndTimestamp; import org.apache.kafka.test.IntegrationTest; +import org.apache.kafka.test.MockProcessorSupplier; import org.apache.kafka.test.TestUtils; import org.junit.After; import org.junit.Before; @@ -68,7 +69,6 @@ public class GlobalKTableIntegrationTest { private final KeyValueMapper keyMapper = (key, value) -> value; private final ValueJoiner joiner = (value1, value2) -> value1 + "+" + value2; private final String globalStore = "globalStore"; - private final Map results = new HashMap<>(); private StreamsBuilder builder; private Properties streamsConfiguration; private KafkaStreams kafkaStreams; @@ -76,7 +76,7 @@ public class GlobalKTableIntegrationTest { private String streamTopic; private GlobalKTable globalTable; private KStream stream; - private ForeachAction foreachAction; + private MockProcessorSupplier supplier; @Before public void before() throws Exception { @@ -96,7 +96,7 @@ public void before() throws Exception { .withValueSerde(Serdes.String())); final Consumed stringLongConsumed = Consumed.with(Serdes.String(), Serdes.Long()); stream = builder.stream(streamTopic, stringLongConsumed); - foreachAction = results::put; + supplier = new MockProcessorSupplier<>(); } @After @@ -110,24 +110,34 @@ public void whenShuttingDown() throws Exception { @Test public void shouldKStreamGlobalKTableLeftJoin() throws Exception { final KStream streamTableJoin = stream.leftJoin(globalTable, keyMapper, joiner); - streamTableJoin.foreach(foreachAction); + streamTableJoin.process(supplier); produceInitialGlobalTableValues(); startStreams(); + long firstTimestamp = mockTime.milliseconds(); produceTopicValues(streamTopic); - final Map expected = new HashMap<>(); - expected.put("a", "1+A"); - expected.put("b", "2+B"); - expected.put("c", "3+C"); - expected.put("d", "4+D"); - expected.put("e", "5+null"); + final Map> expected = new HashMap<>(); + expected.put("a", ValueAndTimestamp.make("1+A", firstTimestamp)); + expected.put("b", ValueAndTimestamp.make("2+B", firstTimestamp + 1L)); + expected.put("c", ValueAndTimestamp.make("3+C", firstTimestamp + 2L)); + expected.put("d", ValueAndTimestamp.make("4+D", firstTimestamp + 3L)); + expected.put("e", ValueAndTimestamp.make("5+null", firstTimestamp + 4L)); TestUtils.waitForCondition( - () -> results.equals(expected), + () -> { + if (supplier.capturedProcessorsCount() < 2) { + return false; + } + final Map> result = new HashMap<>(); + result.putAll(supplier.capturedProcessors(2).get(0).lastValueAndTimestampPerKey); + result.putAll(supplier.capturedProcessors(2).get(1).lastValueAndTimestampPerKey); + return result.equals(expected); + }, 30000L, "waiting for initial values"); + firstTimestamp = mockTime.milliseconds(); produceGlobalTableValues(); final ReadOnlyKeyValueStore replicatedStore = @@ -138,16 +148,29 @@ public void shouldKStreamGlobalKTableLeftJoin() throws Exception { 30000, "waiting for data in replicated store"); + final ReadOnlyKeyValueStore> replicatedStoreWithTimestamp = + kafkaStreams.store(globalStore, QueryableStoreTypes.timestampedKeyValueStore()); + assertThat(replicatedStoreWithTimestamp.get(5L), equalTo(ValueAndTimestamp.make("J", firstTimestamp + 4L))); + + firstTimestamp = mockTime.milliseconds(); produceTopicValues(streamTopic); - expected.put("a", "1+F"); - expected.put("b", "2+G"); - expected.put("c", "3+H"); - expected.put("d", "4+I"); - expected.put("e", "5+J"); + expected.put("a", ValueAndTimestamp.make("1+F", firstTimestamp)); + expected.put("b", ValueAndTimestamp.make("2+G", firstTimestamp + 1L)); + expected.put("c", ValueAndTimestamp.make("3+H", firstTimestamp + 2L)); + expected.put("d", ValueAndTimestamp.make("4+I", firstTimestamp + 3L)); + expected.put("e", ValueAndTimestamp.make("5+J", firstTimestamp + 4L)); TestUtils.waitForCondition( - () -> results.equals(expected), + () -> { + if (supplier.capturedProcessorsCount() < 2) { + return false; + } + final Map> result = new HashMap<>(); + result.putAll(supplier.capturedProcessors(2).get(0).lastValueAndTimestampPerKey); + result.putAll(supplier.capturedProcessors(2).get(1).lastValueAndTimestampPerKey); + return result.equals(expected); + }, 30000L, "waiting for final values"); } @@ -155,23 +178,33 @@ public void shouldKStreamGlobalKTableLeftJoin() throws Exception { @Test public void shouldKStreamGlobalKTableJoin() throws Exception { final KStream streamTableJoin = stream.join(globalTable, keyMapper, joiner); - streamTableJoin.foreach(foreachAction); + streamTableJoin.process(supplier); produceInitialGlobalTableValues(); startStreams(); + long firstTimestamp = mockTime.milliseconds(); produceTopicValues(streamTopic); - final Map expected = new HashMap<>(); - expected.put("a", "1+A"); - expected.put("b", "2+B"); - expected.put("c", "3+C"); - expected.put("d", "4+D"); + final Map> expected = new HashMap<>(); + expected.put("a", ValueAndTimestamp.make("1+A", firstTimestamp)); + expected.put("b", ValueAndTimestamp.make("2+B", firstTimestamp + 1L)); + expected.put("c", ValueAndTimestamp.make("3+C", firstTimestamp + 2L)); + expected.put("d", ValueAndTimestamp.make("4+D", firstTimestamp + 3L)); TestUtils.waitForCondition( - () -> results.equals(expected), + () -> { + if (supplier.capturedProcessorsCount() < 2) { + return false; + } + final Map> result = new HashMap<>(); + result.putAll(supplier.capturedProcessors(2).get(0).lastValueAndTimestampPerKey); + result.putAll(supplier.capturedProcessors(2).get(1).lastValueAndTimestampPerKey); + return result.equals(expected); + }, 30000L, "waiting for initial values"); + firstTimestamp = mockTime.milliseconds(); produceGlobalTableValues(); final ReadOnlyKeyValueStore replicatedStore = @@ -182,16 +215,29 @@ public void shouldKStreamGlobalKTableJoin() throws Exception { 30000, "waiting for data in replicated store"); + final ReadOnlyKeyValueStore> replicatedStoreWithTimestamp = + kafkaStreams.store(globalStore, QueryableStoreTypes.timestampedKeyValueStore()); + assertThat(replicatedStoreWithTimestamp.get(5L), equalTo(ValueAndTimestamp.make("J", firstTimestamp + 4L))); + + firstTimestamp = mockTime.milliseconds(); produceTopicValues(streamTopic); - expected.put("a", "1+F"); - expected.put("b", "2+G"); - expected.put("c", "3+H"); - expected.put("d", "4+I"); - expected.put("e", "5+J"); + expected.put("a", ValueAndTimestamp.make("1+F", firstTimestamp)); + expected.put("b", ValueAndTimestamp.make("2+G", firstTimestamp + 1L)); + expected.put("c", ValueAndTimestamp.make("3+H", firstTimestamp + 2L)); + expected.put("d", ValueAndTimestamp.make("4+I", firstTimestamp + 3L)); + expected.put("e", ValueAndTimestamp.make("5+J", firstTimestamp + 4L)); TestUtils.waitForCondition( - () -> results.equals(expected), + () -> { + if (supplier.capturedProcessorsCount() < 2) { + return false; + } + final Map> result = new HashMap<>(); + result.putAll(supplier.capturedProcessors(2).get(0).lastValueAndTimestampPerKey); + result.putAll(supplier.capturedProcessors(2).get(1).lastValueAndTimestampPerKey); + return result.equals(expected); + }, 30000L, "waiting for final values"); } @@ -209,11 +255,16 @@ public void shouldRestoreGlobalInMemoryKTableOnRestart() throws Exception { startStreams(); ReadOnlyKeyValueStore store = kafkaStreams.store(globalStore, QueryableStoreTypes.keyValueStore()); assertThat(store.approximateNumEntries(), equalTo(4L)); + ReadOnlyKeyValueStore> timestampedStore = + kafkaStreams.store(globalStore, QueryableStoreTypes.timestampedKeyValueStore()); + assertThat(timestampedStore.approximateNumEntries(), equalTo(4L)); kafkaStreams.close(); startStreams(); store = kafkaStreams.store(globalStore, QueryableStoreTypes.keyValueStore()); assertThat(store.approximateNumEntries(), equalTo(4L)); + timestampedStore = kafkaStreams.store(globalStore, QueryableStoreTypes.timestampedKeyValueStore()); + assertThat(timestampedStore.approximateNumEntries(), equalTo(4L)); } private void createTopics() throws Exception { diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationDedupIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationDedupIntegrationTest.java index 14346cf195e07..61f6356853a3a 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationDedupIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationDedupIntegrationTest.java @@ -26,6 +26,7 @@ import org.apache.kafka.common.serialization.StringSerializer; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.KeyValueTimestamp; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster; @@ -66,8 +67,7 @@ public class KStreamAggregationDedupIntegrationTest { private static final long COMMIT_INTERVAL_MS = 300L; @ClassRule - public static final EmbeddedKafkaCluster CLUSTER = - new EmbeddedKafkaCluster(NUM_BROKERS); + public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(NUM_BROKERS); private final MockTime mockTime = CLUSTER.time; private static volatile AtomicInteger testNo = new AtomicInteger(0); @@ -119,17 +119,18 @@ public void shouldReduce() throws Exception { startStreams(); - produceMessages(System.currentTimeMillis()); + final long timestamp = System.currentTimeMillis(); + produceMessages(timestamp); validateReceivedMessages( new StringDeserializer(), new StringDeserializer(), Arrays.asList( - KeyValue.pair("A", "A:A"), - KeyValue.pair("B", "B:B"), - KeyValue.pair("C", "C:C"), - KeyValue.pair("D", "D:D"), - KeyValue.pair("E", "E:E"))); + new KeyValueTimestamp<>("A", "A:A", timestamp), + new KeyValueTimestamp<>("B", "B:B", timestamp), + new KeyValueTimestamp<>("C", "C:C", timestamp), + new KeyValueTimestamp<>("D", "D:D", timestamp), + new KeyValueTimestamp<>("E", "E:E", timestamp))); } @Test @@ -155,16 +156,16 @@ public void shouldReduceWindowed() throws Exception { new StringDeserializer(), new StringDeserializer(), Arrays.asList( - new KeyValue<>("A@" + firstBatchWindow, "A"), - new KeyValue<>("A@" + secondBatchWindow, "A:A"), - new KeyValue<>("B@" + firstBatchWindow, "B"), - new KeyValue<>("B@" + secondBatchWindow, "B:B"), - new KeyValue<>("C@" + firstBatchWindow, "C"), - new KeyValue<>("C@" + secondBatchWindow, "C:C"), - new KeyValue<>("D@" + firstBatchWindow, "D"), - new KeyValue<>("D@" + secondBatchWindow, "D:D"), - new KeyValue<>("E@" + firstBatchWindow, "E"), - new KeyValue<>("E@" + secondBatchWindow, "E:E") + new KeyValueTimestamp<>("A@" + firstBatchWindow, "A", firstBatchTimestamp), + new KeyValueTimestamp<>("A@" + secondBatchWindow, "A:A", secondBatchTimestamp), + new KeyValueTimestamp<>("B@" + firstBatchWindow, "B", firstBatchTimestamp), + new KeyValueTimestamp<>("B@" + secondBatchWindow, "B:B", secondBatchTimestamp), + new KeyValueTimestamp<>("C@" + firstBatchWindow, "C", firstBatchTimestamp), + new KeyValueTimestamp<>("C@" + secondBatchWindow, "C:C", secondBatchTimestamp), + new KeyValueTimestamp<>("D@" + firstBatchWindow, "D", firstBatchTimestamp), + new KeyValueTimestamp<>("D@" + secondBatchWindow, "D:D", secondBatchTimestamp), + new KeyValueTimestamp<>("E@" + firstBatchWindow, "E", firstBatchTimestamp), + new KeyValueTimestamp<>("E@" + secondBatchWindow, "E:E", secondBatchTimestamp) ) ); } @@ -189,11 +190,11 @@ public void shouldGroupByKey() throws Exception { new StringDeserializer(), new LongDeserializer(), Arrays.asList( - KeyValue.pair("1@" + window, 2L), - KeyValue.pair("2@" + window, 2L), - KeyValue.pair("3@" + window, 2L), - KeyValue.pair("4@" + window, 2L), - KeyValue.pair("5@" + window, 2L) + new KeyValueTimestamp<>("1@" + window, 2L, timestamp), + new KeyValueTimestamp<>("2@" + window, 2L, timestamp), + new KeyValueTimestamp<>("3@" + window, 2L, timestamp), + new KeyValueTimestamp<>("4@" + window, 2L, timestamp), + new KeyValueTimestamp<>("5@" + window, 2L, timestamp) ) ); } @@ -232,20 +233,16 @@ private void startStreams() { private void validateReceivedMessages(final Deserializer keyDeserializer, final Deserializer valueDeserializer, - final List> expectedRecords) + final List> expectedRecords) throws InterruptedException { final Properties consumerProperties = new Properties(); - consumerProperties - .setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()); - consumerProperties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "kgroupedstream-test-" + - testNo); + consumerProperties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()); + consumerProperties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "kgroupedstream-test-" + testNo); consumerProperties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - consumerProperties.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - keyDeserializer.getClass().getName()); - consumerProperties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - valueDeserializer.getClass().getName()); + consumerProperties.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, keyDeserializer.getClass().getName()); + consumerProperties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, valueDeserializer.getClass().getName()); - IntegrationTestUtils.waitUntilFinalKeyValueRecordsReceived( + IntegrationTestUtils.waitUntilFinalKeyValueTimestampRecordsReceived( consumerProperties, outputTopic, expectedRecords); diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java index 2f06af740c003..dcf72a501ec1a 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java @@ -29,6 +29,7 @@ import org.apache.kafka.common.serialization.StringSerializer; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.KeyValueTimestamp; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster; @@ -98,8 +99,7 @@ public class KStreamAggregationIntegrationTest { private static final int NUM_BROKERS = 1; @ClassRule - public static final EmbeddedKafkaCluster CLUSTER = - new EmbeddedKafkaCluster(NUM_BROKERS); + public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(NUM_BROKERS); private static volatile AtomicInteger testNo = new AtomicInteger(0); private final MockTime mockTime = CLUSTER.time; @@ -122,8 +122,7 @@ public void before() throws InterruptedException { streamsConfiguration = new Properties(); final String applicationId = "kgrouped-stream-test-" + testNo.incrementAndGet(); streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, applicationId); - streamsConfiguration - .put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()); + streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()); streamsConfiguration.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath()); streamsConfiguration.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0); @@ -160,30 +159,35 @@ public void shouldReduce() throws Exception { produceMessages(mockTime.milliseconds()); - final List> results = receiveMessages( + final List> results = receiveMessages( new StringDeserializer(), new StringDeserializer(), 10); results.sort(KStreamAggregationIntegrationTest::compare); - assertThat(results, is(Arrays.asList(KeyValue.pair("A", "A"), - KeyValue.pair("A", "A:A"), - KeyValue.pair("B", "B"), - KeyValue.pair("B", "B:B"), - KeyValue.pair("C", "C"), - KeyValue.pair("C", "C:C"), - KeyValue.pair("D", "D"), - KeyValue.pair("D", "D:D"), - KeyValue.pair("E", "E"), - KeyValue.pair("E", "E:E")))); + assertThat(results, is(Arrays.asList( + new KeyValueTimestamp("A", "A", mockTime.milliseconds()), + new KeyValueTimestamp("A", "A:A", mockTime.milliseconds()), + new KeyValueTimestamp("B", "B", mockTime.milliseconds()), + new KeyValueTimestamp("B", "B:B", mockTime.milliseconds()), + new KeyValueTimestamp("C", "C", mockTime.milliseconds()), + new KeyValueTimestamp("C", "C:C", mockTime.milliseconds()), + new KeyValueTimestamp("D", "D", mockTime.milliseconds()), + new KeyValueTimestamp("D", "D:D", mockTime.milliseconds()), + new KeyValueTimestamp("E", "E", mockTime.milliseconds()), + new KeyValueTimestamp("E", "E:E", mockTime.milliseconds())))); } - private static int compare(final KeyValue o1, - final KeyValue o2) { - final int keyComparison = o1.key.compareTo(o2.key); + private static int compare(final KeyValueTimestamp o1, + final KeyValueTimestamp o2) { + final int keyComparison = o1.key().compareTo(o2.key()); if (keyComparison == 0) { - return o1.value.compareTo(o2.value); + final int valueComparison = o1.value().compareTo(o2.value()); + if (valueComparison == 0) { + return Long.compare(o1.timestamp(), o2.timestamp()); + } + return valueComparison; } return keyComparison; } @@ -206,7 +210,7 @@ public void shouldReduceWindowed() throws Exception { startStreams(); - final List, String>> windowedOutput = receiveMessages( + final List, String>> windowedOutput = receiveMessages( new TimeWindowedDeserializer<>(), new StringDeserializer(), String.class, @@ -218,44 +222,45 @@ public void shouldReduceWindowed() throws Exception { new StringDeserializer(), String.class, 15, - false); + true); - final Comparator, String>> - comparator = - Comparator.comparing((KeyValue, String> o) -> o.key.key()).thenComparing(o -> o.value); + final Comparator, String>> comparator = + Comparator.comparing((KeyValueTimestamp, String> o) -> o.key().key()) + .thenComparing(KeyValueTimestamp::value); windowedOutput.sort(comparator); final long firstBatchWindow = firstBatchTimestamp / 500 * 500; final long secondBatchWindow = secondBatchTimestamp / 500 * 500; - final List, String>> expectResult = Arrays.asList( - new KeyValue<>(new Windowed<>("A", new TimeWindow(firstBatchWindow, Long.MAX_VALUE)), "A"), - new KeyValue<>(new Windowed<>("A", new TimeWindow(secondBatchWindow, Long.MAX_VALUE)), "A"), - new KeyValue<>(new Windowed<>("A", new TimeWindow(secondBatchWindow, Long.MAX_VALUE)), "A:A"), - new KeyValue<>(new Windowed<>("B", new TimeWindow(firstBatchWindow, Long.MAX_VALUE)), "B"), - new KeyValue<>(new Windowed<>("B", new TimeWindow(secondBatchWindow, Long.MAX_VALUE)), "B"), - new KeyValue<>(new Windowed<>("B", new TimeWindow(secondBatchWindow, Long.MAX_VALUE)), "B:B"), - new KeyValue<>(new Windowed<>("C", new TimeWindow(firstBatchWindow, Long.MAX_VALUE)), "C"), - new KeyValue<>(new Windowed<>("C", new TimeWindow(secondBatchWindow, Long.MAX_VALUE)), "C"), - new KeyValue<>(new Windowed<>("C", new TimeWindow(secondBatchWindow, Long.MAX_VALUE)), "C:C"), - new KeyValue<>(new Windowed<>("D", new TimeWindow(firstBatchWindow, Long.MAX_VALUE)), "D"), - new KeyValue<>(new Windowed<>("D", new TimeWindow(secondBatchWindow, Long.MAX_VALUE)), "D"), - new KeyValue<>(new Windowed<>("D", new TimeWindow(secondBatchWindow, Long.MAX_VALUE)), "D:D"), - new KeyValue<>(new Windowed<>("E", new TimeWindow(firstBatchWindow, Long.MAX_VALUE)), "E"), - new KeyValue<>(new Windowed<>("E", new TimeWindow(secondBatchWindow, Long.MAX_VALUE)), "E"), - new KeyValue<>(new Windowed<>("E", new TimeWindow(secondBatchWindow, Long.MAX_VALUE)), "E:E") + final List, String>> expectResult = Arrays.asList( + new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(firstBatchWindow, Long.MAX_VALUE)), "A", firstBatchTimestamp), + new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(secondBatchWindow, Long.MAX_VALUE)), "A", secondBatchTimestamp), + new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(secondBatchWindow, Long.MAX_VALUE)), "A:A", secondBatchTimestamp), + new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(firstBatchWindow, Long.MAX_VALUE)), "B", firstBatchTimestamp), + new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(secondBatchWindow, Long.MAX_VALUE)), "B", secondBatchTimestamp), + new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(secondBatchWindow, Long.MAX_VALUE)), "B:B", secondBatchTimestamp), + new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(firstBatchWindow, Long.MAX_VALUE)), "C", firstBatchTimestamp), + new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(secondBatchWindow, Long.MAX_VALUE)), "C", secondBatchTimestamp), + new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(secondBatchWindow, Long.MAX_VALUE)), "C:C", secondBatchTimestamp), + new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(firstBatchWindow, Long.MAX_VALUE)), "D", firstBatchTimestamp), + new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(secondBatchWindow, Long.MAX_VALUE)), "D", secondBatchTimestamp), + new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(secondBatchWindow, Long.MAX_VALUE)), "D:D", secondBatchTimestamp), + new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(firstBatchWindow, Long.MAX_VALUE)), "E", firstBatchTimestamp), + new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(secondBatchWindow, Long.MAX_VALUE)), "E", secondBatchTimestamp), + new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(secondBatchWindow, Long.MAX_VALUE)), "E:E", secondBatchTimestamp) ); assertThat(windowedOutput, is(expectResult)); final Set expectResultString = new HashSet<>(expectResult.size()); - for (final KeyValue, String> eachRecord: expectResult) { - expectResultString.add(eachRecord.toString()); + for (final KeyValueTimestamp, String> eachRecord: expectResult) { + expectResultString.add("CreateTime:" + eachRecord.timestamp() + ", " + + eachRecord.key() + ", " + eachRecord.value()); } // check every message is contained in the expect result final String[] allRecords = resultFromConsoleConsumer.split("\n"); for (final String record: allRecords) { - assertTrue(expectResultString.contains("KeyValue(" + record + ")")); + assertTrue(expectResultString.contains(record)); } } @@ -273,7 +278,7 @@ public void shouldAggregate() throws Exception { produceMessages(mockTime.milliseconds()); - final List> results = receiveMessages( + final List> results = receiveMessages( new StringDeserializer(), new IntegerDeserializer(), 10); @@ -281,16 +286,16 @@ public void shouldAggregate() throws Exception { results.sort(KStreamAggregationIntegrationTest::compare); assertThat(results, is(Arrays.asList( - KeyValue.pair("A", 1), - KeyValue.pair("A", 2), - KeyValue.pair("B", 1), - KeyValue.pair("B", 2), - KeyValue.pair("C", 1), - KeyValue.pair("C", 2), - KeyValue.pair("D", 1), - KeyValue.pair("D", 2), - KeyValue.pair("E", 1), - KeyValue.pair("E", 2) + new KeyValueTimestamp("A", 1, mockTime.milliseconds()), + new KeyValueTimestamp("A", 2, mockTime.milliseconds()), + new KeyValueTimestamp("B", 1, mockTime.milliseconds()), + new KeyValueTimestamp("B", 2, mockTime.milliseconds()), + new KeyValueTimestamp("C", 1, mockTime.milliseconds()), + new KeyValueTimestamp("C", 2, mockTime.milliseconds()), + new KeyValueTimestamp("D", 1, mockTime.milliseconds()), + new KeyValueTimestamp("D", 2, mockTime.milliseconds()), + new KeyValueTimestamp("E", 1, mockTime.milliseconds()), + new KeyValueTimestamp("E", 2, mockTime.milliseconds()) ))); } @@ -315,7 +320,7 @@ public void shouldAggregateWindowed() throws Exception { startStreams(); - final List, KeyValue>> windowedMessages = receiveMessagesWithTimestamp( + final List, Integer>> windowedMessages = receiveMessagesWithTimestamp( new TimeWindowedDeserializer<>(), new IntegerDeserializer(), String.class, @@ -329,37 +334,36 @@ public void shouldAggregateWindowed() throws Exception { 15, true); - final Comparator, KeyValue>> - comparator = - Comparator.comparing((KeyValue, KeyValue> o) -> o.key.key()).thenComparingInt(o -> o.value.key); - + final Comparator, Integer>> comparator = + Comparator.comparing((KeyValueTimestamp, Integer> o) -> o.key().key()) + .thenComparingInt(KeyValueTimestamp::value); windowedMessages.sort(comparator); final long firstWindow = firstTimestamp / 500 * 500; final long secondWindow = secondTimestamp / 500 * 500; - final List, KeyValue>> expectResult = Arrays.asList( - new KeyValue<>(new Windowed<>("A", new TimeWindow(firstWindow, Long.MAX_VALUE)), KeyValue.pair(1, firstTimestamp)), - new KeyValue<>(new Windowed<>("A", new TimeWindow(secondWindow, Long.MAX_VALUE)), KeyValue.pair(1, secondTimestamp)), - new KeyValue<>(new Windowed<>("A", new TimeWindow(secondWindow, Long.MAX_VALUE)), KeyValue.pair(2, secondTimestamp)), - new KeyValue<>(new Windowed<>("B", new TimeWindow(firstWindow, Long.MAX_VALUE)), KeyValue.pair(1, firstTimestamp)), - new KeyValue<>(new Windowed<>("B", new TimeWindow(secondWindow, Long.MAX_VALUE)), KeyValue.pair(1, secondTimestamp)), - new KeyValue<>(new Windowed<>("B", new TimeWindow(secondWindow, Long.MAX_VALUE)), KeyValue.pair(2, secondTimestamp)), - new KeyValue<>(new Windowed<>("C", new TimeWindow(firstWindow, Long.MAX_VALUE)), KeyValue.pair(1, firstTimestamp)), - new KeyValue<>(new Windowed<>("C", new TimeWindow(secondWindow, Long.MAX_VALUE)), KeyValue.pair(1, secondTimestamp)), - new KeyValue<>(new Windowed<>("C", new TimeWindow(secondWindow, Long.MAX_VALUE)), KeyValue.pair(2, secondTimestamp)), - new KeyValue<>(new Windowed<>("D", new TimeWindow(firstWindow, Long.MAX_VALUE)), KeyValue.pair(1, firstTimestamp)), - new KeyValue<>(new Windowed<>("D", new TimeWindow(secondWindow, Long.MAX_VALUE)), KeyValue.pair(1, secondTimestamp)), - new KeyValue<>(new Windowed<>("D", new TimeWindow(secondWindow, Long.MAX_VALUE)), KeyValue.pair(2, secondTimestamp)), - new KeyValue<>(new Windowed<>("E", new TimeWindow(firstWindow, Long.MAX_VALUE)), KeyValue.pair(1, firstTimestamp)), - new KeyValue<>(new Windowed<>("E", new TimeWindow(secondWindow, Long.MAX_VALUE)), KeyValue.pair(1, secondTimestamp)), - new KeyValue<>(new Windowed<>("E", new TimeWindow(secondWindow, Long.MAX_VALUE)), KeyValue.pair(2, secondTimestamp))); + final List, Integer>> expectResult = Arrays.asList( + new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(firstWindow, Long.MAX_VALUE)), 1, firstTimestamp), + new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(secondWindow, Long.MAX_VALUE)), 1, secondTimestamp), + new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(secondWindow, Long.MAX_VALUE)), 2, secondTimestamp), + new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(firstWindow, Long.MAX_VALUE)), 1, firstTimestamp), + new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(secondWindow, Long.MAX_VALUE)), 1, secondTimestamp), + new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(secondWindow, Long.MAX_VALUE)), 2, secondTimestamp), + new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(firstWindow, Long.MAX_VALUE)), 1, firstTimestamp), + new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(secondWindow, Long.MAX_VALUE)), 1, secondTimestamp), + new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(secondWindow, Long.MAX_VALUE)), 2, secondTimestamp), + new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(firstWindow, Long.MAX_VALUE)), 1, firstTimestamp), + new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(secondWindow, Long.MAX_VALUE)), 1, secondTimestamp), + new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(secondWindow, Long.MAX_VALUE)), 2, secondTimestamp), + new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(firstWindow, Long.MAX_VALUE)), 1, firstTimestamp), + new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(secondWindow, Long.MAX_VALUE)), 1, secondTimestamp), + new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(secondWindow, Long.MAX_VALUE)), 2, secondTimestamp)); assertThat(windowedMessages, is(expectResult)); final Set expectResultString = new HashSet<>(expectResult.size()); - for (final KeyValue, KeyValue> eachRecord: expectResult) { - expectResultString.add("CreateTime:" + eachRecord.value.value + ", " + eachRecord.key.toString() + ", " + eachRecord.value.key); + for (final KeyValueTimestamp, Integer> eachRecord: expectResult) { + expectResultString.add("CreateTime:" + eachRecord.timestamp() + ", " + eachRecord.key() + ", " + eachRecord.value()); } // check every message is contained in the expect result @@ -375,23 +379,23 @@ private void shouldCountHelper() throws Exception { produceMessages(mockTime.milliseconds()); - final List> results = receiveMessages( + final List> results = receiveMessages( new StringDeserializer(), new LongDeserializer(), 10); results.sort(KStreamAggregationIntegrationTest::compare); assertThat(results, is(Arrays.asList( - KeyValue.pair("A", 1L), - KeyValue.pair("A", 2L), - KeyValue.pair("B", 1L), - KeyValue.pair("B", 2L), - KeyValue.pair("C", 1L), - KeyValue.pair("C", 2L), - KeyValue.pair("D", 1L), - KeyValue.pair("D", 2L), - KeyValue.pair("E", 1L), - KeyValue.pair("E", 2L) + new KeyValueTimestamp("A", 1L, mockTime.milliseconds()), + new KeyValueTimestamp("A", 2L, mockTime.milliseconds()), + new KeyValueTimestamp("B", 1L, mockTime.milliseconds()), + new KeyValueTimestamp("B", 2L, mockTime.milliseconds()), + new KeyValueTimestamp("C", 1L, mockTime.milliseconds()), + new KeyValueTimestamp("C", 2L, mockTime.milliseconds()), + new KeyValueTimestamp("D", 1L, mockTime.milliseconds()), + new KeyValueTimestamp("D", 2L, mockTime.milliseconds()), + new KeyValueTimestamp("E", 1L, mockTime.milliseconds()), + new KeyValueTimestamp("E", 2L, mockTime.milliseconds()) ))); } @@ -430,7 +434,7 @@ public void shouldGroupByKey() throws Exception { startStreams(); - final List> results = receiveMessages( + final List> results = receiveMessages( new StringDeserializer(), new LongDeserializer(), 10); @@ -438,18 +442,17 @@ public void shouldGroupByKey() throws Exception { final long window = timestamp / 500 * 500; assertThat(results, is(Arrays.asList( - KeyValue.pair("1@" + window, 1L), - KeyValue.pair("1@" + window, 2L), - KeyValue.pair("2@" + window, 1L), - KeyValue.pair("2@" + window, 2L), - KeyValue.pair("3@" + window, 1L), - KeyValue.pair("3@" + window, 2L), - KeyValue.pair("4@" + window, 1L), - KeyValue.pair("4@" + window, 2L), - KeyValue.pair("5@" + window, 1L), - KeyValue.pair("5@" + window, 2L) + new KeyValueTimestamp("1@" + window, 1L, timestamp), + new KeyValueTimestamp("1@" + window, 2L, timestamp), + new KeyValueTimestamp("2@" + window, 1L, timestamp), + new KeyValueTimestamp("2@" + window, 2L, timestamp), + new KeyValueTimestamp("3@" + window, 1L, timestamp), + new KeyValueTimestamp("3@" + window, 2L, timestamp), + new KeyValueTimestamp("4@" + window, 1L, timestamp), + new KeyValueTimestamp("4@" + window, 2L, timestamp), + new KeyValueTimestamp("5@" + window, 1L, timestamp), + new KeyValueTimestamp("5@" + window, 2L, timestamp) ))); - } @Test @@ -796,17 +799,17 @@ private void startStreams() { kafkaStreams.start(); } - private List> receiveMessages(final Deserializer keyDeserializer, - final Deserializer valueDeserializer, - final int numMessages) + private List> receiveMessages(final Deserializer keyDeserializer, + final Deserializer valueDeserializer, + final int numMessages) throws InterruptedException { return receiveMessages(keyDeserializer, valueDeserializer, null, numMessages); } - private List> receiveMessages(final Deserializer keyDeserializer, - final Deserializer valueDeserializer, - final Class innerClass, - final int numMessages) throws InterruptedException { + private List> receiveMessages(final Deserializer keyDeserializer, + final Deserializer valueDeserializer, + final Class innerClass, + final int numMessages) throws InterruptedException { final Properties consumerProperties = new Properties(); consumerProperties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()); consumerProperties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "kgroupedstream-test-" + testNo); @@ -817,17 +820,17 @@ private List> receiveMessages(final Deserializer keyDes consumerProperties.setProperty(StreamsConfig.DEFAULT_WINDOWED_KEY_SERDE_INNER_CLASS, Serdes.serdeFrom(innerClass).getClass().getName()); } - return IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived( + return IntegrationTestUtils.waitUntilMinKeyValueWithTimestampRecordsReceived( consumerProperties, outputTopic, numMessages, 60 * 1000); } - private List>> receiveMessagesWithTimestamp(final Deserializer keyDeserializer, - final Deserializer valueDeserializer, - final Class innerClass, - final int numMessages) throws InterruptedException { + private List> receiveMessagesWithTimestamp(final Deserializer keyDeserializer, + final Deserializer valueDeserializer, + final Class innerClass, + final int numMessages) throws InterruptedException { final Properties consumerProperties = new Properties(); consumerProperties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()); consumerProperties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "kgroupedstream-test-" + testNo); diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/StreamStreamJoinIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/StreamStreamJoinIntegrationTest.java index 646185ebb93c6..4be14c22a98f6 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/StreamStreamJoinIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/StreamStreamJoinIntegrationTest.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.streams.integration; +import org.apache.kafka.streams.KeyValueTimestamp; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.kstream.JoinWindows; @@ -62,22 +63,36 @@ public void prepareTopology() throws InterruptedException { public void testInner() throws Exception { STREAMS_CONFIG.put(StreamsConfig.APPLICATION_ID_CONFIG, appID + "-inner"); - final List> expectedResult = Arrays.asList( + final List>> expectedResult = Arrays.asList( null, null, null, - Collections.singletonList("A-a"), - Collections.singletonList("B-a"), - Arrays.asList("A-b", "B-b"), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a", 4L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a", 5L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-b", 6L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b", 6L)), null, null, - Arrays.asList("C-a", "C-b"), - Arrays.asList("A-c", "B-c", "C-c"), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-a", 9L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-b", 9L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c", 10L)), null, null, null, - Arrays.asList("A-d", "B-d", "C-d"), - Arrays.asList("D-a", "D-b", "D-c", "D-d") + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-d", 14L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-a", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-b", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-c", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d", 15L)) ); leftStream.join(rightStream, valueJoiner, JoinWindows.of(ofSeconds(10))).to(OUTPUT_TOPIC); @@ -89,27 +104,41 @@ public void testInner() throws Exception { public void testInnerRepartitioned() throws Exception { STREAMS_CONFIG.put(StreamsConfig.APPLICATION_ID_CONFIG, appID + "-inner-repartitioned"); - final List> expectedResult = Arrays.asList( - null, - null, - null, - Collections.singletonList("A-a"), - Collections.singletonList("B-a"), - Arrays.asList("A-b", "B-b"), - null, - null, - Arrays.asList("C-a", "C-b"), - Arrays.asList("A-c", "B-c", "C-c"), - null, - null, - null, - Arrays.asList("A-d", "B-d", "C-d"), - Arrays.asList("D-a", "D-b", "D-c", "D-d") + final List>> expectedResult = Arrays.asList( + null, + null, + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a", 4L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a", 5L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-b", 6L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b", 6L)), + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-a", 9L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-b", 9L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c", 10L)), + null, + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-d", 14L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-a", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-b", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-c", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d", 15L)) ); - leftStream.map(MockMapper.noOpKeyValueMapper()) - .join(rightStream.flatMap(MockMapper.noOpFlatKeyValueMapper()) - .selectKey(MockMapper.selectKeyKeyValueMapper()), + leftStream.map(MockMapper.noOpKeyValueMapper()) + .join(rightStream.flatMap(MockMapper.noOpFlatKeyValueMapper()) + .selectKey(MockMapper.selectKeyKeyValueMapper()), valueJoiner, JoinWindows.of(ofSeconds(10))).to(OUTPUT_TOPIC); runTest(expectedResult); @@ -119,22 +148,36 @@ public void testInnerRepartitioned() throws Exception { public void testLeft() throws Exception { STREAMS_CONFIG.put(StreamsConfig.APPLICATION_ID_CONFIG, appID + "-left"); - final List> expectedResult = Arrays.asList( + final List>> expectedResult = Arrays.asList( null, null, - Collections.singletonList("A-null"), - Collections.singletonList("A-a"), - Collections.singletonList("B-a"), - Arrays.asList("A-b", "B-b"), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-null", 3L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a", 4L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a", 5L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-b", 6L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b", 6L)), null, null, - Arrays.asList("C-a", "C-b"), - Arrays.asList("A-c", "B-c", "C-c"), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-a", 9L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-b", 9L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c", 10L)), null, null, null, - Arrays.asList("A-d", "B-d", "C-d"), - Arrays.asList("D-a", "D-b", "D-c", "D-d") + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-d", 14L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-a", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-b", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-c", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d", 15L)) ); leftStream.leftJoin(rightStream, valueJoiner, JoinWindows.of(ofSeconds(10))).to(OUTPUT_TOPIC); @@ -146,27 +189,41 @@ public void testLeft() throws Exception { public void testLeftRepartitioned() throws Exception { STREAMS_CONFIG.put(StreamsConfig.APPLICATION_ID_CONFIG, appID + "-left-repartitioned"); - final List> expectedResult = Arrays.asList( - null, - null, - Collections.singletonList("A-null"), - Collections.singletonList("A-a"), - Collections.singletonList("B-a"), - Arrays.asList("A-b", "B-b"), - null, - null, - Arrays.asList("C-a", "C-b"), - Arrays.asList("A-c", "B-c", "C-c"), - null, - null, - null, - Arrays.asList("A-d", "B-d", "C-d"), - Arrays.asList("D-a", "D-b", "D-c", "D-d") + final List>> expectedResult = Arrays.asList( + null, + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-null", 3L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a", 4L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a", 5L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-b", 6L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b", 6L)), + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-a", 9L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-b", 9L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c", 10L)), + null, + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-d", 14L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-a", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-b", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-c", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d", 15L)) ); - leftStream.map(MockMapper.noOpKeyValueMapper()) - .leftJoin(rightStream.flatMap(MockMapper.noOpFlatKeyValueMapper()) - .selectKey(MockMapper.selectKeyKeyValueMapper()), + leftStream.map(MockMapper.noOpKeyValueMapper()) + .leftJoin(rightStream.flatMap(MockMapper.noOpFlatKeyValueMapper()) + .selectKey(MockMapper.selectKeyKeyValueMapper()), valueJoiner, JoinWindows.of(ofSeconds(10))).to(OUTPUT_TOPIC); runTest(expectedResult); @@ -176,22 +233,36 @@ public void testLeftRepartitioned() throws Exception { public void testOuter() throws Exception { STREAMS_CONFIG.put(StreamsConfig.APPLICATION_ID_CONFIG, appID + "-outer"); - final List> expectedResult = Arrays.asList( + final List>> expectedResult = Arrays.asList( null, null, - Collections.singletonList("A-null"), - Collections.singletonList("A-a"), - Collections.singletonList("B-a"), - Arrays.asList("A-b", "B-b"), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-null", 3L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a", 4L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a", 5L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-b", 6L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b", 6L)), null, null, - Arrays.asList("C-a", "C-b"), - Arrays.asList("A-c", "B-c", "C-c"), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-a", 9L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-b", 9L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c", 10L)), null, null, null, - Arrays.asList("A-d", "B-d", "C-d"), - Arrays.asList("D-a", "D-b", "D-c", "D-d") + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-d", 14L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-a", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-b", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-c", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d", 15L)) ); leftStream.outerJoin(rightStream, valueJoiner, JoinWindows.of(ofSeconds(10))).to(OUTPUT_TOPIC); @@ -203,27 +274,41 @@ public void testOuter() throws Exception { public void testOuterRepartitioned() throws Exception { STREAMS_CONFIG.put(StreamsConfig.APPLICATION_ID_CONFIG, appID + "-outer"); - final List> expectedResult = Arrays.asList( - null, - null, - Collections.singletonList("A-null"), - Collections.singletonList("A-a"), - Collections.singletonList("B-a"), - Arrays.asList("A-b", "B-b"), - null, - null, - Arrays.asList("C-a", "C-b"), - Arrays.asList("A-c", "B-c", "C-c"), - null, - null, - null, - Arrays.asList("A-d", "B-d", "C-d"), - Arrays.asList("D-a", "D-b", "D-c", "D-d") + final List>> expectedResult = Arrays.asList( + null, + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-null", 3L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a", 4L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a", 5L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-b", 6L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b", 6L)), + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-a", 9L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-b", 9L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c", 10L)), + null, + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-d", 14L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-a", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-b", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-c", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d", 15L)) ); - leftStream.map(MockMapper.noOpKeyValueMapper()) - .outerJoin(rightStream.flatMap(MockMapper.noOpFlatKeyValueMapper()) - .selectKey(MockMapper.selectKeyKeyValueMapper()), + leftStream.map(MockMapper.noOpKeyValueMapper()) + .outerJoin(rightStream.flatMap(MockMapper.noOpFlatKeyValueMapper()) + .selectKey(MockMapper.selectKeyKeyValueMapper()), valueJoiner, JoinWindows.of(ofSeconds(10))).to(OUTPUT_TOPIC); runTest(expectedResult); @@ -233,26 +318,84 @@ public void testOuterRepartitioned() throws Exception { public void testMultiInner() throws Exception { STREAMS_CONFIG.put(StreamsConfig.APPLICATION_ID_CONFIG, appID + "-multi-inner"); - final List> expectedResult = Arrays.asList( - null, - null, - null, - Collections.singletonList("A-a-a"), - Collections.singletonList("B-a-a"), - Arrays.asList("A-b-a", "B-b-a", "A-a-b", "B-a-b", "A-b-b", "B-b-b"), - null, - null, - Arrays.asList("C-a-a", "C-a-b", "C-b-a", "C-b-b"), - Arrays.asList("A-c-a", "A-c-b", "B-c-a", "B-c-b", "C-c-a", "C-c-b", "A-a-c", "B-a-c", - "A-b-c", "B-b-c", "C-a-c", "C-b-c", "A-c-c", "B-c-c", "C-c-c"), - null, - null, - null, - Arrays.asList("A-d-a", "A-d-b", "A-d-c", "B-d-a", "B-d-b", "B-d-c", "C-d-a", "C-d-b", "C-d-c", - "A-a-d", "B-a-d", "A-b-d", "B-b-d", "C-a-d", "C-b-d", "A-c-d", "B-c-d", "C-c-d", - "A-d-d", "B-d-d", "C-d-d"), - Arrays.asList("D-a-a", "D-a-b", "D-a-c", "D-a-d", "D-b-a", "D-b-b", "D-b-c", "D-b-d", "D-c-a", - "D-c-b", "D-c-c", "D-c-d", "D-d-a", "D-d-b", "D-d-c", "D-d-d") + final List>> expectedResult = Arrays.asList( + null, + null, + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-a", 4L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a-a", 5L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-b-a", 6L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-a", 6L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-b", 6L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a-b", 6L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-b-b", 6L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-b", 6L)), + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-a-a", 9L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-a-b", 9L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-b-a", 9L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-b-b", 9L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-c-a", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-c-b", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-c-a", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-c-b", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-a", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-b", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-b-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-a-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-b-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-c-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-c-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-c", 10L)), + null, + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-d-a", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-d-b", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-d-c", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-d-a", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-d-b", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-d-c", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-d-a", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-d-b", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-d-c", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-b-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-a-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-b-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-c-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-c-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-d-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-d-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-d-d", 14L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-a-a", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-a-b", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-a-c", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-a-d", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-b-a", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-b-b", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-b-c", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-b-d", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-c-a", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-c-b", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-c-c", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-c-d", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d-a", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d-b", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d-c", 15L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d-d", 15L)) ); leftStream.join(rightStream, valueJoiner, JoinWindows.of(ofSeconds(10))) diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/StreamTableJoinIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/StreamTableJoinIntegrationTest.java index ca78d025addf7..772c91d7b36af 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/StreamTableJoinIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/StreamTableJoinIntegrationTest.java @@ -17,6 +17,7 @@ package org.apache.kafka.streams.integration; import org.apache.kafka.streams.KafkaStreamsWrapper; +import org.apache.kafka.streams.KeyValueTimestamp; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.integration.utils.IntegrationTestUtils; @@ -34,8 +35,7 @@ import java.util.Collections; import java.util.List; -import static org.junit.Assert.assertEquals; - +import static org.junit.Assert.assertTrue; /** * Tests all available joins of Kafka Streams DSL. @@ -82,30 +82,30 @@ public void testShouldAutoShutdownOnIncompleteMetadata() throws InterruptedExcep TestUtils.waitForCondition(listener::revokedToPendingShutdownSeen, "Did not seen thread state transited to PENDING_SHUTDOWN"); streams.close(); - assertEquals(listener.createdToRevokedSeen(), true); - assertEquals(listener.revokedToPendingShutdownSeen(), true); + assertTrue(listener.createdToRevokedSeen()); + assertTrue(listener.revokedToPendingShutdownSeen()); } @Test public void testInner() throws Exception { STREAMS_CONFIG.put(StreamsConfig.APPLICATION_ID_CONFIG, appID + "-inner"); - final List> expectedResult = Arrays.asList( - null, - null, - null, - null, - Collections.singletonList("B-a"), - null, - null, - null, - null, - null, - null, - null, - null, - null, - Collections.singletonList("D-d") + final List>> expectedResult = Arrays.asList( + null, + null, + null, + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a", 5L)), + null, + null, + null, + null, + null, + null, + null, + null, + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d", 15L)) ); leftStream.join(rightTable, valueJoiner).to(OUTPUT_TOPIC); @@ -117,22 +117,22 @@ public void testInner() throws Exception { public void testLeft() throws Exception { STREAMS_CONFIG.put(StreamsConfig.APPLICATION_ID_CONFIG, appID + "-left"); - final List> expectedResult = Arrays.asList( - null, - null, - Collections.singletonList("A-null"), - null, - Collections.singletonList("B-a"), - null, - null, - null, - Collections.singletonList("C-null"), - null, - null, - null, - null, - null, - Collections.singletonList("D-d") + final List>> expectedResult = Arrays.asList( + null, + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-null", 3L)), + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a", 5L)), + null, + null, + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-null", 9L)), + null, + null, + null, + null, + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d", 15L)) ); leftStream.leftJoin(rightTable, valueJoiner).to(OUTPUT_TOPIC); diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/TableTableJoinIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/TableTableJoinIntegrationTest.java index 73d1e3d315af5..2b685f9e63fe2 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/TableTableJoinIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/TableTableJoinIntegrationTest.java @@ -18,6 +18,7 @@ import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.KeyValueTimestamp; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.kstream.ForeachAction; @@ -59,8 +60,8 @@ public void prepareTopology() throws InterruptedException { rightTable = builder.table(INPUT_TOPIC_RIGHT, Materialized.>as("right").withLoggingDisabled()); } - final private String expectedFinalJoinResult = "D-d"; - final private String expectedFinalMultiJoinResult = "D-d-d"; + final private KeyValueTimestamp expectedFinalJoinResult = new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d", 15L); + final private KeyValueTimestamp expectedFinalMultiJoinResult = new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d-d", 15L); final private String storeName = appID + "-store"; private Materialized> materialized = Materialized.>as(storeName) @@ -70,7 +71,7 @@ public void prepareTopology() throws InterruptedException { .withLoggingDisabled(); final private class CountingPeek implements ForeachAction { - final private String expected; + final private KeyValueTimestamp expected; CountingPeek(final boolean multiJoin) { this.expected = multiJoin ? expectedFinalMultiJoinResult : expectedFinalJoinResult; @@ -79,7 +80,7 @@ final private class CountingPeek implements ForeachAction { @Override public void apply(final Long key, final String value) { numRecordsExpected++; - if (expected.equals(value)) { + if (expected.value().equals(value)) { final boolean ret = finalResultReached.compareAndSet(false, true); if (!ret) { @@ -98,22 +99,22 @@ public void testInner() throws Exception { leftTable.join(rightTable, valueJoiner, materialized).toStream().peek(new CountingPeek(false)).to(OUTPUT_TOPIC); runTest(expectedFinalJoinResult, storeName); } else { - final List> expectedResult = Arrays.asList( - null, - null, - null, - Collections.singletonList("A-a"), - Collections.singletonList("B-a"), - Collections.singletonList("B-b"), - Collections.singletonList((String) null), - null, - null, - Collections.singletonList("C-c"), - Collections.singletonList((String) null), - null, - null, - null, - Collections.singletonList("D-d") + final List>> expectedResult = Arrays.asList( + null, + null, + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a", 4L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a", 5L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b", 6L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 7L)), + null, + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c", 10L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 11L)), + null, + null, + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d", 15L)) ); leftTable.join(rightTable, valueJoiner, materialized).toStream().to(OUTPUT_TOPIC); @@ -129,22 +130,22 @@ public void testLeft() throws Exception { leftTable.leftJoin(rightTable, valueJoiner, materialized).toStream().peek(new CountingPeek(false)).to(OUTPUT_TOPIC); runTest(expectedFinalJoinResult, storeName); } else { - final List> expectedResult = Arrays.asList( - null, - null, - Collections.singletonList("A-null"), - Collections.singletonList("A-a"), - Collections.singletonList("B-a"), - Collections.singletonList("B-b"), - Collections.singletonList((String) null), - null, - Collections.singletonList("C-null"), - Collections.singletonList("C-c"), - Collections.singletonList("C-null"), - Collections.singletonList((String) null), - null, - null, - Collections.singletonList("D-d") + final List>> expectedResult = Arrays.asList( + null, + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-null", 3L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a", 4L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a", 5L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b", 6L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 7L)), + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-null", 9L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c", 10L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-null", 11L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 12L)), + null, + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d", 15L)) ); leftTable.leftJoin(rightTable, valueJoiner, materialized).toStream().to(OUTPUT_TOPIC); @@ -160,22 +161,22 @@ public void testOuter() throws Exception { leftTable.outerJoin(rightTable, valueJoiner, materialized).toStream().peek(new CountingPeek(false)).to(OUTPUT_TOPIC); runTest(expectedFinalJoinResult, storeName); } else { - final List> expectedResult = Arrays.asList( - null, - null, - Collections.singletonList("A-null"), - Collections.singletonList("A-a"), - Collections.singletonList("B-a"), - Collections.singletonList("B-b"), - Collections.singletonList("null-b"), - Collections.singletonList((String) null), - Collections.singletonList("C-null"), - Collections.singletonList("C-c"), - Collections.singletonList("C-null"), - Collections.singletonList((String) null), - null, - Collections.singletonList("null-d"), - Collections.singletonList("D-d") + final List>> expectedResult = Arrays.asList( + null, + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-null", 3L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a", 4L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a", 5L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b", 6L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "null-b", 7L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 8L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-null", 9L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c", 10L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-null", 11L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 12L)), + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "null-d", 14L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d", 15L)) ); leftTable.outerJoin(rightTable, valueJoiner, materialized).toStream().to(OUTPUT_TOPIC); @@ -197,22 +198,29 @@ public void testInnerInner() throws Exception { } else { // FIXME: the duplicate below for all the multi-joins // are due to KAFKA-6443, should be updated once it is fixed. - final List> expectedResult = Arrays.asList( - null, - null, - null, - Arrays.asList("A-a-a", "A-a-a"), - Collections.singletonList("B-a-a"), - Arrays.asList("B-b-b", "B-b-b"), - Collections.singletonList((String) null), - null, - null, - Arrays.asList("C-c-c", "C-c-c"), - null, - null, - null, - null, - Collections.singletonList("D-d-d") + final List>> expectedResult = Arrays.asList( + null, + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-a", 4L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-a", 4L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a-a", 5L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-b", 6L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-b", 6L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 7L)), + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-c", 10L)), + null, // correct would be -> new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 11L) + // we don't get correct value, because of self-join of `rightTable` + null, + null, + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d-d", 15L)) ); leftTable.join(rightTable, valueJoiner) @@ -235,22 +243,28 @@ public void testInnerLeft() throws Exception { .to(OUTPUT_TOPIC); runTest(expectedFinalMultiJoinResult, storeName); } else { - final List> expectedResult = Arrays.asList( - null, - null, - null, - Arrays.asList("A-a-a", "A-a-a"), - Collections.singletonList("B-a-a"), - Arrays.asList("B-b-b", "B-b-b"), - Collections.singletonList((String) null), - null, - null, - Arrays.asList("C-c-c", "C-c-c"), - Collections.singletonList((String) null), - null, - null, - null, - Collections.singletonList("D-d-d") + final List>> expectedResult = Arrays.asList( + null, + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-a", 4L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-a", 4L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a-a", 5L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-b", 6L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-b", 6L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 7L)), + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-c", 10L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 11L)), + null, + null, + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d-d", 15L)) ); leftTable.join(rightTable, valueJoiner) @@ -274,22 +288,33 @@ public void testInnerOuter() throws Exception { .to(OUTPUT_TOPIC); runTest(expectedFinalMultiJoinResult, storeName); } else { - final List> expectedResult = Arrays.asList( - null, - null, - null, - Arrays.asList("A-a-a", "A-a-a"), - Collections.singletonList("B-a-a"), - Arrays.asList("B-b-b", "B-b-b"), - Collections.singletonList("null-b"), - Collections.singletonList((String) null), - null, - Arrays.asList("C-c-c", "C-c-c"), - Arrays.asList((String) null, null), - null, - null, - null, - Arrays.asList("null-d", "D-d-d") + final List>> expectedResult = Arrays.asList( + null, + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-a", 4L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-a", 4L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a-a", 5L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-b", 6L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-b", 6L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "null-b", 7L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 8L)), + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-c", 10L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 11L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 11L)), + null, + null, + null, + Arrays.asList( + // incorrect result `null-d` is caused by self-join of `rightTable` + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "null-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d-d", 15L)) ); leftTable.join(rightTable, valueJoiner) @@ -312,22 +337,28 @@ public void testLeftInner() throws Exception { .to(OUTPUT_TOPIC); runTest(expectedFinalMultiJoinResult, storeName); } else { - final List> expectedResult = Arrays.asList( - null, - null, - null, - Arrays.asList("A-a-a", "A-a-a"), - Collections.singletonList("B-a-a"), - Arrays.asList("B-b-b", "B-b-b"), - Collections.singletonList((String) null), - null, - null, - Arrays.asList("C-c-c", "C-c-c"), - Collections.singletonList((String) null), - null, - null, - null, - Collections.singletonList("D-d-d") + final List>> expectedResult = Arrays.asList( + null, + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-a", 4L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-a", 4L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a-a", 5L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-b", 6L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-b", 6L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 7L)), + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-c", 10L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 11L)), + null, + null, + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d-d", 15L)) ); leftTable.leftJoin(rightTable, valueJoiner) @@ -351,22 +382,32 @@ public void testLeftLeft() throws Exception { .to(OUTPUT_TOPIC); runTest(expectedFinalMultiJoinResult, storeName); } else { - final List> expectedResult = Arrays.asList( - null, - null, - null, - Arrays.asList("A-null-null", "A-a-a", "A-a-a"), - Collections.singletonList("B-a-a"), - Arrays.asList("B-b-b", "B-b-b"), - Collections.singletonList((String) null), - null, - null, - Arrays.asList("C-null-null", "C-c-c", "C-c-c"), - Arrays.asList("C-null-null", "C-null-null"), - Collections.singletonList((String) null), - null, - null, - Collections.singletonList("D-d-d") + final List>> expectedResult = Arrays.asList( + null, + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-null-null", 3L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-a", 4L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-a", 4L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a-a", 5L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-b", 6L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-b", 6L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 7L)), + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-null-null", 9L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-c", 10L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-null-null", 11L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-null-null", 11L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 12L)), + null, + null, + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d-d", 15L)) ); leftTable.leftJoin(rightTable, valueJoiner) @@ -390,22 +431,34 @@ public void testLeftOuter() throws Exception { .to(OUTPUT_TOPIC); runTest(expectedFinalMultiJoinResult, storeName); } else { - final List> expectedResult = Arrays.asList( - null, - null, - null, - Arrays.asList("A-null-null", "A-a-a", "A-a-a"), - Collections.singletonList("B-a-a"), - Arrays.asList("B-b-b", "B-b-b"), - Collections.singletonList("null-b"), - Collections.singletonList((String) null), - null, - Arrays.asList("C-null-null", "C-c-c", "C-c-c"), - Arrays.asList("C-null-null", "C-null-null"), - Collections.singletonList((String) null), - null, - null, - Arrays.asList("null-d", "D-d-d") + final List>> expectedResult = Arrays.asList( + null, + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-null-null", 3L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-a", 4L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-a", 4L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a-a", 5L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-b", 6L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-b", 6L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "null-b", 7L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 8L)), + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-null-null", 9L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-c", 10L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-null-null", 11L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-null-null", 11L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 12L)), + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "null-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d-d", 15L)) ); leftTable.leftJoin(rightTable, valueJoiner) @@ -428,22 +481,30 @@ public void testOuterInner() throws Exception { .to(OUTPUT_TOPIC); runTest(expectedFinalMultiJoinResult, storeName); } else { - final List> expectedResult = Arrays.asList( - null, - null, - null, - Arrays.asList("A-a-a", "A-a-a"), - Collections.singletonList("B-a-a"), - Arrays.asList("B-b-b", "B-b-b"), - Collections.singletonList("null-b-b"), - null, - null, - Arrays.asList("C-c-c", "C-c-c"), - Collections.singletonList((String) null), - null, - null, - Arrays.asList("null-d-d", "null-d-d"), - Collections.singletonList("D-d-d") + final List>> expectedResult = Arrays.asList( + null, + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-a", 4L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-a", 4L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a-a", 5L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-b", 6L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-b", 6L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "null-b-b", 7L)), + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-c", 10L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 11L)), + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "null-d-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "null-d-d", 14L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d-d", 15L)) ); leftTable.outerJoin(rightTable, valueJoiner) @@ -467,22 +528,34 @@ public void testOuterLeft() throws Exception { .to(OUTPUT_TOPIC); runTest(expectedFinalMultiJoinResult, storeName); } else { - final List> expectedResult = Arrays.asList( - null, - null, - null, - Arrays.asList("A-null-null", "A-a-a", "A-a-a"), - Collections.singletonList("B-a-a"), - Arrays.asList("B-b-b", "B-b-b"), - Collections.singletonList("null-b-b"), - Collections.singletonList((String) null), - null, - Arrays.asList("C-null-null", "C-c-c", "C-c-c"), - Arrays.asList("C-null-null", "C-null-null"), - Collections.singletonList((String) null), - null, - Arrays.asList("null-d-d", "null-d-d"), - Collections.singletonList("D-d-d") + final List>> expectedResult = Arrays.asList( + null, + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-null-null", 3L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-a", 4L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-a", 4L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a-a", 5L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-b", 6L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-b", 6L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "null-b-b", 7L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 8L)), + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-null-null", 9L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-c", 10L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-null-null", 11L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-null-null", 11L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 12L)), + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "null-d-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "null-d-d", 14L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d-d", 15L)) ); leftTable.outerJoin(rightTable, valueJoiner) @@ -506,22 +579,36 @@ public void testOuterOuter() throws Exception { .to(OUTPUT_TOPIC); runTest(expectedFinalMultiJoinResult, storeName); } else { - final List> expectedResult = Arrays.asList( - null, - null, - null, - Arrays.asList("A-null-null", "A-a-a", "A-a-a"), - Collections.singletonList("B-a-a"), - Arrays.asList("B-b-b", "B-b-b"), - Collections.singletonList("null-b-b"), - Arrays.asList((String) null, null), - null, - Arrays.asList("C-null-null", "C-c-c", "C-c-c"), - Arrays.asList("C-null-null", "C-null-null"), - Collections.singletonList((String) null), - null, - null, - Arrays.asList("null-d-d", "null-d-d", "D-d-d") + final List>> expectedResult = Arrays.asList( + null, + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-null-null", 3L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-a", 4L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-a-a", 4L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a-a", 5L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-b", 6L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-b-b", 6L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "null-b-b", 7L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 8L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 8L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-null-null", 9L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-c", 10L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-c-c", 10L)), + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-null-null", 11L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-null-null", 11L)), + Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, null, 12L)), + null, + null, + Arrays.asList( + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "null-d-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "null-d-d", 14L), + new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d-d", 15L)) ); leftTable.outerJoin(rightTable, valueJoiner) diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java b/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java index e6cf85d83b0db..46515aa84690c 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java @@ -241,12 +241,14 @@ public static void produceKeyValuesSynchronouslyWithTimestamp(final Strin * @param Key type of the data records * @param Value type of the data records */ + @SuppressWarnings("WeakerAccess") public static void produceKeyValuesSynchronouslyWithTimestamp(final String topic, final Collection> records, final Properties producerConfig, final Long timestamp, final boolean enableTransactions) - throws ExecutionException, InterruptedException { + throws ExecutionException, InterruptedException { + produceKeyValuesSynchronouslyWithTimestamp(topic, records, producerConfig, null, timestamp, enableTransactions); } @@ -260,13 +262,15 @@ public static void produceKeyValuesSynchronouslyWithTimestamp(final Strin * @param Key type of the data records * @param Value type of the data records */ + @SuppressWarnings("WeakerAccess") public static void produceKeyValuesSynchronouslyWithTimestamp(final String topic, final Collection> records, final Properties producerConfig, final Headers headers, final Long timestamp, final boolean enableTransactions) - throws ExecutionException, InterruptedException { + throws ExecutionException, InterruptedException { + try (final Producer producer = new KafkaProducer<>(producerConfig)) { if (enableTransactions) { producer.initTransactions(); @@ -369,6 +373,7 @@ public static void produceValuesSynchronously(final String topic, * @param enableTransactions Send messages in a transaction * @param Value type of the data records */ + @SuppressWarnings("WeakerAccess") public static void produceValuesSynchronously(final String topic, final Collection records, final Properties producerConfig, @@ -427,6 +432,7 @@ public static void waitForCompletion(final KafkaStreams streams, * @param Value type of the data records * @return All the records consumed, or null if no records are consumed */ + @SuppressWarnings("WeakerAccess") public static List> waitUntilMinRecordsReceived(final Properties consumerConfig, final String topic, final int expectedNumRecords) throws InterruptedException { @@ -444,6 +450,7 @@ public static List> waitUntilMinRecordsReceived(fina * @param Value type of the data records * @return All the records consumed, or null if no records are consumed */ + @SuppressWarnings("WeakerAccess") public static List> waitUntilMinRecordsReceived(final Properties consumerConfig, final String topic, final int expectedNumRecords, @@ -519,14 +526,14 @@ public static List> waitUntilMinKeyValueRecordsReceived(fi * @param Key type of the data records * @param Value type of the data records */ - public static List>> waitUntilMinKeyValueWithTimestampRecordsReceived(final Properties consumerConfig, + public static List> waitUntilMinKeyValueWithTimestampRecordsReceived(final Properties consumerConfig, final String topic, final int expectedNumRecords, final long waitTime) throws InterruptedException { - final List>> accumData = new ArrayList<>(); + final List> accumData = new ArrayList<>(); try (final Consumer consumer = createConsumer(consumerConfig)) { final TestCondition valuesRead = () -> { - final List>> readData = + final List> readData = readKeyValuesWithTimestamp(topic, consumer, waitTime, expectedNumRecords); accumData.addAll(readData); return accumData.size() >= expectedNumRecords; @@ -553,6 +560,22 @@ public static List> waitUntilFinalKeyValueRecordsReceived( return waitUntilFinalKeyValueRecordsReceived(consumerConfig, topic, expectedRecords, DEFAULT_TIMEOUT); } + /** + * Wait until final key-value mappings have been consumed. + * + * @param consumerConfig Kafka Consumer configuration + * @param topic Kafka topic to consume from + * @param expectedRecords Expected key-value mappings + * @param Key type of the data records + * @param Value type of the data records + * @return All the mappings consumed, or null if no records are consumed + */ + public static List> waitUntilFinalKeyValueTimestampRecordsReceived(final Properties consumerConfig, + final String topic, + final List> expectedRecords) throws InterruptedException { + return waitUntilFinalKeyValueRecordsReceived(consumerConfig, topic, expectedRecords, DEFAULT_TIMEOUT, true); + } + /** * Wait until final key-value mappings have been consumed. * @@ -564,28 +587,53 @@ public static List> waitUntilFinalKeyValueRecordsReceived( * @param Value type of the data records * @return All the mappings consumed, or null if no records are consumed */ + @SuppressWarnings("WeakerAccess") public static List> waitUntilFinalKeyValueRecordsReceived(final Properties consumerConfig, final String topic, final List> expectedRecords, final long waitTime) throws InterruptedException { - final List> accumData = new ArrayList<>(); + return waitUntilFinalKeyValueRecordsReceived(consumerConfig, topic, expectedRecords, waitTime, false); + } + + public static List> waitUntilFinalKeyValueTimestampRecordsReceived(final Properties consumerConfig, + final String topic, + final List> expectedRecords, + final long waitTime) throws InterruptedException { + return waitUntilFinalKeyValueRecordsReceived(consumerConfig, topic, expectedRecords, waitTime, true); + } + + @SuppressWarnings("unchecked") + private static List waitUntilFinalKeyValueRecordsReceived(final Properties consumerConfig, + final String topic, + final List expectedRecords, + final long waitTime, + final boolean withTimestamp) throws InterruptedException { + final List accumData = new ArrayList<>(); try (final Consumer consumer = createConsumer(consumerConfig)) { final TestCondition valuesRead = () -> { - final List> readData = - readKeyValues(topic, consumer, waitTime, expectedRecords.size()); + final List readData; + if (withTimestamp) { + readData = (List) readKeyValuesWithTimestamp(topic, consumer, waitTime, expectedRecords.size()); + } else { + readData = (List) readKeyValues(topic, consumer, waitTime, expectedRecords.size()); + } accumData.addAll(readData); // filter out all intermediate records we don't want - final List> accumulatedActual = accumData.stream().filter(expectedRecords::contains).collect(Collectors.toList()); + final List accumulatedActual = accumData.stream().filter(expectedRecords::contains).collect(Collectors.toList()); // still need to check that for each key, the ordering is expected - final Map>> finalAccumData = new HashMap<>(); - for (final KeyValue kv : accumulatedActual) { - finalAccumData.computeIfAbsent(kv.key, key -> new ArrayList<>()).add(kv); + final Map> finalAccumData = new HashMap<>(); + for (final T kv : accumulatedActual) { + finalAccumData.computeIfAbsent( + (K) (withTimestamp ? ((KeyValueTimestamp) kv).key() : ((KeyValue) kv).key), + key -> new ArrayList<>()).add(kv); } - final Map>> finalExpected = new HashMap<>(); - for (final KeyValue kv : expectedRecords) { - finalExpected.computeIfAbsent(kv.key, key -> new ArrayList<>()).add(kv); + final Map> finalExpected = new HashMap<>(); + for (final T kv : expectedRecords) { + finalExpected.computeIfAbsent( + (K) (withTimestamp ? ((KeyValueTimestamp) kv).key() : ((KeyValue) kv).key), + key -> new ArrayList<>()).add(kv); } // returns true only if the remaining records in both lists are the same and in the same order @@ -643,6 +691,7 @@ public static List waitUntilMinValuesRecordsReceived(final Properties con return accumData; } + @SuppressWarnings("WeakerAccess") public static void waitForTopicPartitions(final List servers, final List partitions, final long timeout) throws InterruptedException { @@ -863,14 +912,14 @@ private static List> readKeyValues(final String topic, * @param maxMessages Maximum number of messages to read via the consumer * @return The KeyValue elements retrieved via the consumer */ - private static List>> readKeyValuesWithTimestamp(final String topic, - final Consumer consumer, - final long waitTime, - final int maxMessages) { - final List>> consumedValues = new ArrayList<>(); + private static List> readKeyValuesWithTimestamp(final String topic, + final Consumer consumer, + final long waitTime, + final int maxMessages) { + final List> consumedValues = new ArrayList<>(); final List> records = readRecords(topic, consumer, waitTime, maxMessages); for (final ConsumerRecord record : records) { - consumedValues.add(new KeyValue<>(record.key(), KeyValue.pair(record.value(), record.timestamp()))); + consumedValues.add(new KeyValueTimestamp<>(record.key(), record.value(), record.timestamp())); } return consumedValues; } diff --git a/streams/src/test/java/org/apache/kafka/test/MockProcessorSupplier.java b/streams/src/test/java/org/apache/kafka/test/MockProcessorSupplier.java index 0bd22d482f2d5..a6959e196281f 100644 --- a/streams/src/test/java/org/apache/kafka/test/MockProcessorSupplier.java +++ b/streams/src/test/java/org/apache/kafka/test/MockProcessorSupplier.java @@ -56,7 +56,11 @@ public MockProcessor theCapturedProcessor() { return capturedProcessors(1).get(0); } - // get the captured processors with the expected number + public int capturedProcessorsCount() { + return processors.size(); + } + + // get the captured processors with the expected number public List> capturedProcessors(final int expectedNumberOfProcessors) { assertEquals(expectedNumberOfProcessors, processors.size()); From 77a9a108ff1dff861ecc8dacc7220acde061658c Mon Sep 17 00:00:00 2001 From: Alex Diachenko Date: Thu, 30 May 2019 12:01:00 -0700 Subject: [PATCH 0310/1071] KAFKA-8418: Wait until REST resources are loaded when starting a Connect Worker. (#6840) Author: Alex Diachenko Reviewers: Arjun Satish , Konstantine Karantasis , Randall Hauch --- tests/kafkatest/services/connect.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/kafkatest/services/connect.py b/tests/kafkatest/services/connect.py index afd2bffabbd83..96e0d54c87a89 100644 --- a/tests/kafkatest/services/connect.py +++ b/tests/kafkatest/services/connect.py @@ -20,7 +20,6 @@ import time import requests -from ducktape.cluster.remoteaccount import RemoteCommandError from ducktape.errors import DucktapeError from ducktape.services.service import Service from ducktape.utils.util import wait_until @@ -107,12 +106,12 @@ def set_external_configs(self, external_config_template_func): def listening(self, node): try: - cmd = "nc -z %s %s" % (node.account.hostname, self.CONNECT_REST_PORT) - node.account.ssh_output(cmd, allow_fail=False) - self.logger.debug("Connect worker started accepting connections at: '%s:%s')", node.account.hostname, + self.list_connectors(node) + self.logger.debug("Connect worker started serving REST at: '%s:%s')", node.account.hostname, self.CONNECT_REST_PORT) return True - except (RemoteCommandError, ValueError) as e: + except requests.exceptions.ConnectionError: + self.logger.debug("REST resources are not loaded yet") return False def start(self, mode=STARTUP_MODE_LISTEN): From 55bfea13789c3898ce4c9259a5fad74192c1bfb0 Mon Sep 17 00:00:00 2001 From: "Matthias J. Sax" Date: Thu, 30 May 2019 12:50:30 -0700 Subject: [PATCH 0311/1071] KAFKA-8155: Add 2.1.1 release to system tests (#6596) Reviewers: Bill Bejeck , John Roesler , Guozhang Wang --- gradle/dependencies.gradle | 2 +- .../streams/processor/internals/StreamsPartitionAssignor.java | 4 ++-- tests/docker/Dockerfile | 3 +-- tests/kafkatest/version.py | 1 + 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 85a1607ff198c..d58d067efd96f 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -78,7 +78,7 @@ versions += [ kafka_10: "1.0.2", kafka_11: "1.1.1", kafka_20: "2.0.1", - kafka_21: "2.1.0", + kafka_21: "2.1.1", lz4: "1.6.0", mavenArtifact: "3.6.1", metrics: "2.2.0", diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java index 3f3c1233da006..33cce13faeb42 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java @@ -799,7 +799,7 @@ List> interleaveTasksByGroupId(final Collection taskIds, fi @Override public void onAssignment(final Assignment assignment) { final List partitions = new ArrayList<>(assignment.partitions()); - Collections.sort(partitions, PARTITION_COMPARATOR); + partitions.sort(PARTITION_COMPARATOR); final AssignmentInfo info = AssignmentInfo.decode(assignment.userData()); if (info.errCode() != Error.NONE.code) { @@ -1009,7 +1009,7 @@ void validate(final Set copartitionGroup, if (numPartitions == UNKNOWN) { numPartitions = partitions; } else if (numPartitions != partitions) { - final String[] topics = copartitionGroup.toArray(new String[copartitionGroup.size()]); + final String[] topics = copartitionGroup.toArray(new String[0]); Arrays.sort(topics); throw new org.apache.kafka.streams.errors.TopologyException(String.format("%sTopics not co-partitioned: [%s]", logPrefix, Utils.join(Arrays.asList(topics), ","))); } diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index 41a2336c3bc04..665dc270405cf 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -53,7 +53,6 @@ RUN mkdir -p "/opt/kafka-0.11.0.3" && chmod a+rw /opt/kafka-0.11.0.3 && curl -s RUN mkdir -p "/opt/kafka-1.0.2" && chmod a+rw /opt/kafka-1.0.2 && curl -s "$KAFKA_MIRROR/kafka_2.11-1.0.2.tgz" | tar xz --strip-components=1 -C "/opt/kafka-1.0.2" RUN mkdir -p "/opt/kafka-1.1.1" && chmod a+rw /opt/kafka-1.1.1 && curl -s "$KAFKA_MIRROR/kafka_2.11-1.1.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-1.1.1" RUN mkdir -p "/opt/kafka-2.0.1" && chmod a+rw /opt/kafka-2.0.1 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.0.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.0.1" -RUN mkdir -p "/opt/kafka-2.1.0" && chmod a+rw /opt/kafka-2.1.0 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.1.0.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.1.0" RUN mkdir -p "/opt/kafka-2.1.1" && chmod a+rw /opt/kafka-2.1.1 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.1.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.1.1" RUN mkdir -p "/opt/kafka-2.2.0" && chmod a+rw /opt/kafka-2.2.0 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.2.0.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.2.0" @@ -65,7 +64,7 @@ RUN curl -s "$KAFKA_MIRROR/kafka-streams-0.11.0.3-test.jar" -o /opt/kafka-0.11.0 RUN curl -s "$KAFKA_MIRROR/kafka-streams-1.0.2-test.jar" -o /opt/kafka-1.0.2/libs/kafka-streams-1.0.2-test.jar RUN curl -s "$KAFKA_MIRROR/kafka-streams-1.1.1-test.jar" -o /opt/kafka-1.1.1/libs/kafka-streams-1.1.1-test.jar RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.0.1-test.jar" -o /opt/kafka-2.0.1/libs/kafka-streams-2.0.1-test.jar -RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.1.0-test.jar" -o /opt/kafka-2.1.0/libs/kafka-streams-2.1.0-test.jar +RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.1.1-test.jar" -o /opt/kafka-2.1.1/libs/kafka-streams-2.1.1-test.jar # The version of Kibosh to use for testing. # If you update this, also update vagrant/base.sy diff --git a/tests/kafkatest/version.py b/tests/kafkatest/version.py index e5aed5a4d3b31..403da03b2f19f 100644 --- a/tests/kafkatest/version.py +++ b/tests/kafkatest/version.py @@ -119,5 +119,6 @@ def get_version(node=None): V_2_1_1 = KafkaVersion("2.1.1") LATEST_2_1 = V_2_1_1 +# 2.2.x versions V_2_2_0 = KafkaVersion("2.2.0") LATEST_2_2 = V_2_2_0 From a4c0b1841a9fe69d040128129293db201f81fe4a Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Thu, 30 May 2019 14:58:53 -0700 Subject: [PATCH 0312/1071] KAFKA-8389: Remove redundant bookkeeping from MockProcessor (#6761) Remove processedKeys / processedValues / processedWithTimestamps as they are covered with processed already. Reviewers: Matthias J. Sax , John Roesler , Boyang Chen --- .../internals/KGroupedStreamImplTest.java | 28 ++++++++-------- .../org/apache/kafka/test/MockProcessor.java | 32 +++++-------------- 2 files changed, 22 insertions(+), 38 deletions(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java index 0c850e711b269..b19fd55bfad9d 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java @@ -22,7 +22,6 @@ import org.apache.kafka.common.serialization.StringSerializer; import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.streams.KeyValue; -import org.apache.kafka.streams.KeyValueTimestamp; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.TopologyTestDriver; import org.apache.kafka.streams.errors.TopologyException; @@ -43,6 +42,7 @@ import org.apache.kafka.streams.test.ConsumerRecordFactory; import org.apache.kafka.test.MockAggregator; import org.apache.kafka.test.MockInitializer; +import org.apache.kafka.test.MockProcessor; import org.apache.kafka.test.MockProcessorSupplier; import org.apache.kafka.test.MockReducer; import org.apache.kafka.test.StreamsTestUtils; @@ -575,19 +575,19 @@ private void doCountWindowed(final MockProcessorSupplier, Long driver.pipeInput(recordFactory.create(TOPIC, "2", "B", 500L)); driver.pipeInput(recordFactory.create(TOPIC, "3", "B", 100L)); } - assertThat(supplier.theCapturedProcessor().processedWithTimestamps, equalTo(Arrays.asList( - new KeyValueTimestamp<>(new Windowed<>("1", new TimeWindow(0L, 500L)), 1L, 0L), - new KeyValueTimestamp<>(new Windowed<>("1", new TimeWindow(0L, 500L)), 2L, 499L), - new KeyValueTimestamp<>(new Windowed<>("1", new TimeWindow(0L, 500L)), 3L, 499L), - new KeyValueTimestamp<>(new Windowed<>("2", new TimeWindow(0L, 500L)), 1L, 0L), - new KeyValueTimestamp<>(new Windowed<>("2", new TimeWindow(0L, 500L)), 2L, 100L), - new KeyValueTimestamp<>(new Windowed<>("2", new TimeWindow(0L, 500L)), 3L, 200L), - new KeyValueTimestamp<>(new Windowed<>("3", new TimeWindow(0L, 500L)), 1L, 1L), - new KeyValueTimestamp<>(new Windowed<>("1", new TimeWindow(500L, 1000L)), 1L, 500L), - new KeyValueTimestamp<>(new Windowed<>("1", new TimeWindow(500L, 1000L)), 2L, 500L), - new KeyValueTimestamp<>(new Windowed<>("2", new TimeWindow(500L, 1000L)), 1L, 500L), - new KeyValueTimestamp<>(new Windowed<>("2", new TimeWindow(500L, 1000L)), 2L, 500L), - new KeyValueTimestamp<>(new Windowed<>("3", new TimeWindow(0L, 500L)), 2L, 100L) + assertThat(supplier.theCapturedProcessor().processed, equalTo(Arrays.asList( + MockProcessor.makeRecord(new Windowed<>("1", new TimeWindow(0L, 500L)), 1L, 0L), + MockProcessor.makeRecord(new Windowed<>("1", new TimeWindow(0L, 500L)), 2L, 499L), + MockProcessor.makeRecord(new Windowed<>("1", new TimeWindow(0L, 500L)), 3L, 499L), + MockProcessor.makeRecord(new Windowed<>("2", new TimeWindow(0L, 500L)), 1L, 0L), + MockProcessor.makeRecord(new Windowed<>("2", new TimeWindow(0L, 500L)), 2L, 100L), + MockProcessor.makeRecord(new Windowed<>("2", new TimeWindow(0L, 500L)), 3L, 200L), + MockProcessor.makeRecord(new Windowed<>("3", new TimeWindow(0L, 500L)), 1L, 1L), + MockProcessor.makeRecord(new Windowed<>("1", new TimeWindow(500L, 1000L)), 1L, 500L), + MockProcessor.makeRecord(new Windowed<>("1", new TimeWindow(500L, 1000L)), 2L, 500L), + MockProcessor.makeRecord(new Windowed<>("2", new TimeWindow(500L, 1000L)), 1L, 500L), + MockProcessor.makeRecord(new Windowed<>("2", new TimeWindow(500L, 1000L)), 2L, 500L), + MockProcessor.makeRecord(new Windowed<>("3", new TimeWindow(0L, 500L)), 2L, 100L) ))); } diff --git a/streams/src/test/java/org/apache/kafka/test/MockProcessor.java b/streams/src/test/java/org/apache/kafka/test/MockProcessor.java index 0c7a01575f714..f1e0597540159 100644 --- a/streams/src/test/java/org/apache/kafka/test/MockProcessor.java +++ b/streams/src/test/java/org/apache/kafka/test/MockProcessor.java @@ -16,7 +16,6 @@ */ package org.apache.kafka.test; -import org.apache.kafka.streams.KeyValueTimestamp; import org.apache.kafka.streams.processor.AbstractProcessor; import org.apache.kafka.streams.processor.Cancellable; import org.apache.kafka.streams.processor.ProcessorContext; @@ -30,13 +29,9 @@ import static org.junit.Assert.assertEquals; -@SuppressWarnings("WeakerAccess") public class MockProcessor extends AbstractProcessor { public final ArrayList processed = new ArrayList<>(); - public final ArrayList processedKeys = new ArrayList<>(); - public final ArrayList processedValues = new ArrayList<>(); - public final ArrayList processedWithTimestamps = new ArrayList<>(); public final Map> lastValueAndTimestampPerKey = new HashMap<>(); public final ArrayList punctuatedStreamTime = new ArrayList<>(); @@ -81,19 +76,13 @@ public void init(final ProcessorContext context) { @Override public void process(final K key, final V value) { - processedKeys.add(key); - processedValues.add(value); - processedWithTimestamps.add(new KeyValueTimestamp<>(key, value, context().timestamp())); if (value != null) { lastValueAndTimestampPerKey.put(key, ValueAndTimestamp.make(value, context().timestamp())); } else { lastValueAndTimestampPerKey.remove(key); } - processed.add( - (key == null ? "null" : key) + - ":" + (value == null ? "null" : value) + - " (ts: " + context().timestamp() + ")" - ); + + processed.add(makeRecord(key, value, context().timestamp())); if (commitRequested) { context().commit(); @@ -101,24 +90,19 @@ public void process(final K key, final V value) { } } - public void checkAndClearProcessResult(final String... expected) { - assertEquals("the number of outputs:" + processed, expected.length, processed.size()); - for (int i = 0; i < expected.length; i++) { - assertEquals("output[" + i + "]:", expected[i], processed.get(i)); - } - - processed.clear(); - processedWithTimestamps.clear(); + public static String makeRecord(final Object key, final Object value, final long timestamp) { + return (key == null ? "null" : key) + + ":" + (value == null ? "null" : value) + + " (ts: " + timestamp + ")"; } - public void checkAndClearProcessResult(final KeyValueTimestamp... expected) { + public void checkAndClearProcessResult(final String... expected) { assertEquals("the number of outputs:" + processed, expected.length, processed.size()); for (int i = 0; i < expected.length; i++) { - assertEquals("output[" + i + "]:", expected[i], processedWithTimestamps.get(i)); + assertEquals("output[" + i + "]:", expected[i], processed.get(i)); } processed.clear(); - processedWithTimestamps.clear(); } public void requestCommit() { From efa6410611aa0862065ad804323c280a4d8a372d Mon Sep 17 00:00:00 2001 From: John Roesler Date: Thu, 30 May 2019 17:42:04 -0500 Subject: [PATCH 0313/1071] KAFKA-8199: Implement ValueGetter for Suppress (#6781) See also #6684 KTable processors must be supplied with a KTableProcessorSupplier, which in turn requires implementing a ValueGetter, for use with joins and groupings. For suppression, a correct view only includes the previously emitted values (not the currently buffered ones), so this change also involves pushing the Change value type into the suppression buffer's interface, so that it can get the prior value upon first buffering (which is also the previously emitted value). Reviewers: A. Sophie Blee-Goldman , Guozhang Wang --- .../kstream/internals/FullChangeSerde.java | 19 +- .../streams/kstream/internals/KTableImpl.java | 17 +- .../KTableSourceValueGetterSupplier.java | 2 +- .../suppress/KTableSuppressProcessor.java | 133 -------- .../KTableSuppressProcessorSupplier.java | 214 +++++++++++++ .../streams/state/internals/BufferKey.java | 72 +++++ .../streams/state/internals/BufferValue.java | 98 ++++++ .../state/internals/ContextualRecord.java | 6 +- .../InMemoryTimeOrderedKeyValueBuffer.java | 235 +++++++------- .../kafka/streams/state/internals/Maybe.java | 89 ++++++ .../internals/TimeOrderedKeyValueBuffer.java | 13 +- .../internals/SuppressScenarioTest.java | 198 ++++++++++++ .../KTableSuppressProcessorMetricsTest.java | 25 +- .../suppress/KTableSuppressProcessorTest.java | 71 ++--- .../internals/suppress/SuppressSuite.java | 50 +++ .../streams/state/internals/MaybeTest.java | 79 +++++ .../TimeOrderedKeyValueBufferTest.java | 300 +++++++++++++++--- 17 files changed, 1265 insertions(+), 356 deletions(-) delete mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessor.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorSupplier.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/state/internals/BufferKey.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/state/internals/BufferValue.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/state/internals/Maybe.java create mode 100644 streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/SuppressSuite.java create mode 100644 streams/src/test/java/org/apache/kafka/streams/state/internals/MaybeTest.java diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/FullChangeSerde.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/FullChangeSerde.java index f06a42851f943..30d55beacaa7b 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/FullChangeSerde.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/FullChangeSerde.java @@ -43,6 +43,10 @@ private FullChangeSerde(final Serde inner) { this.inner = requireNonNull(inner); } + public Serde innerSerde() { + return inner; + } + @Override public void configure(final Map configs, final boolean isKey) { inner.configure(configs, isKey); @@ -110,11 +114,7 @@ public Change deserialize(final String topic, final byte[] data) { } final ByteBuffer buffer = ByteBuffer.wrap(data); - final int oldSize = buffer.getInt(); - final byte[] oldBytes = oldSize == -1 ? null : new byte[oldSize]; - if (oldBytes != null) { - buffer.get(oldBytes); - } + final byte[] oldBytes = extractOldValuePart(buffer); final T oldValue = oldBytes == null ? null : innerDeserializer.deserialize(topic, oldBytes); final int newSize = buffer.getInt(); @@ -132,4 +132,13 @@ public void close() { } }; } + + public static byte[] extractOldValuePart(final ByteBuffer buffer) { + final int oldSize = buffer.getInt(); + final byte[] oldBytes = oldSize == -1 ? null : new byte[oldSize]; + if (oldBytes != null) { + buffer.get(oldBytes); + } + return oldBytes; + } } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java index 3a4994f109470..666a109bbc012 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java @@ -38,7 +38,7 @@ import org.apache.kafka.streams.kstream.internals.graph.StreamsGraphNode; import org.apache.kafka.streams.kstream.internals.graph.TableProcessorNode; import org.apache.kafka.streams.kstream.internals.suppress.FinalResultsSuppressionBuilder; -import org.apache.kafka.streams.kstream.internals.suppress.KTableSuppressProcessor; +import org.apache.kafka.streams.kstream.internals.suppress.KTableSuppressProcessorSupplier; import org.apache.kafka.streams.kstream.internals.suppress.NamedSuppressed; import org.apache.kafka.streams.kstream.internals.suppress.SuppressedInternal; import org.apache.kafka.streams.processor.ProcessorSupplier; @@ -391,7 +391,7 @@ public KStream toStream(final KeyValueMapper suppress(final Suppressed suppressed) { final String name; if (suppressed instanceof NamedSuppressed) { - final String givenName = ((NamedSuppressed) suppressed).name(); + final String givenName = ((NamedSuppressed) suppressed).name(); name = givenName != null ? givenName : builder.newProcessorName(SUPPRESS_NAME); } else { throw new IllegalArgumentException("Custom subclasses of Suppressed are not supported."); @@ -402,14 +402,17 @@ public KTable suppress(final Suppressed suppressed) { final String storeName = suppressedInternal.name() != null ? suppressedInternal.name() + "-store" : builder.newStoreName(SUPPRESS_NAME); - final ProcessorSupplier> suppressionSupplier = - () -> new KTableSuppressProcessor<>(suppressedInternal, storeName); + final ProcessorSupplier> suppressionSupplier = new KTableSuppressProcessorSupplier<>( + suppressedInternal, + storeName, + this + ); final ProcessorGraphNode> node = new StatefulProcessorNode<>( name, new ProcessorParameters<>(suppressionSupplier, name), - new InMemoryTimeOrderedKeyValueBuffer.Builder<>(storeName, keySerde, FullChangeSerde.castOrWrap(valSerde)) + new InMemoryTimeOrderedKeyValueBuffer.Builder<>(storeName, keySerde, valSerde) ); builder.addGraphNode(streamsGraphNode, node); @@ -624,7 +627,7 @@ public KGroupedTable groupBy(final KeyValueMapper valueGetterSupplier() { + public KTableValueGetterSupplier valueGetterSupplier() { if (processorSupplier instanceof KTableSource) { final KTableSource source = (KTableSource) processorSupplier; // whenever a source ktable is required for getter, it should be materialized @@ -638,7 +641,7 @@ KTableValueGetterSupplier valueGetterSupplier() { } @SuppressWarnings("unchecked") - void enableSendingOldValues() { + public void enableSendingOldValues() { if (!sendOldValues) { if (processorSupplier instanceof KTableSource) { final KTableSource source = (KTableSource) processorSupplier; diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableSourceValueGetterSupplier.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableSourceValueGetterSupplier.java index 21731e906b491..7083b88cc5471 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableSourceValueGetterSupplier.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableSourceValueGetterSupplier.java @@ -37,7 +37,7 @@ public String[] storeNames() { } private class KTableSourceValueGetter implements KTableValueGetter { - TimestampedKeyValueStore store = null; + private TimestampedKeyValueStore store = null; @SuppressWarnings("unchecked") public void init(final ProcessorContext context) { diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessor.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessor.java deleted file mode 100644 index 7184d7abbd115..0000000000000 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessor.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.streams.kstream.internals.suppress; - -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.streams.errors.StreamsException; -import org.apache.kafka.streams.kstream.internals.Change; -import org.apache.kafka.streams.kstream.internals.FullChangeSerde; -import org.apache.kafka.streams.kstream.internals.metrics.Sensors; -import org.apache.kafka.streams.kstream.internals.suppress.TimeDefinitions.TimeDefinition; -import org.apache.kafka.streams.processor.Processor; -import org.apache.kafka.streams.processor.ProcessorContext; -import org.apache.kafka.streams.processor.internals.InternalProcessorContext; -import org.apache.kafka.streams.processor.internals.ProcessorRecordContext; -import org.apache.kafka.streams.state.internals.TimeOrderedKeyValueBuffer; - -import static java.util.Objects.requireNonNull; - -public class KTableSuppressProcessor implements Processor> { - private final long maxRecords; - private final long maxBytes; - private final long suppressDurationMillis; - private final TimeDefinition bufferTimeDefinition; - private final BufferFullStrategy bufferFullStrategy; - private final boolean safeToDropTombstones; - private final String storeName; - - private TimeOrderedKeyValueBuffer> buffer; - private InternalProcessorContext internalProcessorContext; - private Sensor suppressionEmitSensor; - private long observedStreamTime = ConsumerRecord.NO_TIMESTAMP; - - public KTableSuppressProcessor(final SuppressedInternal suppress, final String storeName) { - this.storeName = storeName; - requireNonNull(suppress); - maxRecords = suppress.bufferConfig().maxRecords(); - maxBytes = suppress.bufferConfig().maxBytes(); - suppressDurationMillis = suppress.timeToWaitForMoreEvents().toMillis(); - bufferTimeDefinition = suppress.timeDefinition(); - bufferFullStrategy = suppress.bufferConfig().bufferFullStrategy(); - safeToDropTombstones = suppress.safeToDropTombstones(); - } - - @SuppressWarnings("unchecked") - @Override - public void init(final ProcessorContext context) { - internalProcessorContext = (InternalProcessorContext) context; - suppressionEmitSensor = Sensors.suppressionEmitSensor(internalProcessorContext); - - buffer = requireNonNull((TimeOrderedKeyValueBuffer>) context.getStateStore(storeName)); - buffer.setSerdesIfNull((Serde) context.keySerde(), FullChangeSerde.castOrWrap((Serde) context.valueSerde())); - } - - @Override - public void process(final K key, final Change value) { - observedStreamTime = Math.max(observedStreamTime, internalProcessorContext.timestamp()); - buffer(key, value); - enforceConstraints(); - } - - private void buffer(final K key, final Change value) { - final long bufferTime = bufferTimeDefinition.time(internalProcessorContext, key); - buffer.put(bufferTime, key, value, internalProcessorContext.recordContext()); - } - - private void enforceConstraints() { - final long streamTime = observedStreamTime; - final long expiryTime = streamTime - suppressDurationMillis; - - buffer.evictWhile(() -> buffer.minTimestamp() <= expiryTime, this::emit); - - if (overCapacity()) { - switch (bufferFullStrategy) { - case EMIT: - buffer.evictWhile(this::overCapacity, this::emit); - return; - case SHUT_DOWN: - throw new StreamsException(String.format( - "%s buffer exceeded its max capacity. Currently [%d/%d] records and [%d/%d] bytes.", - internalProcessorContext.currentNode().name(), - buffer.numRecords(), maxRecords, - buffer.bufferSize(), maxBytes - )); - default: - throw new UnsupportedOperationException( - "The bufferFullStrategy [" + bufferFullStrategy + - "] is not implemented. This is a bug in Kafka Streams." - ); - } - } - } - - private boolean overCapacity() { - return buffer.numRecords() > maxRecords || buffer.bufferSize() > maxBytes; - } - - private void emit(final TimeOrderedKeyValueBuffer.Eviction> toEmit) { - if (shouldForward(toEmit.value())) { - final ProcessorRecordContext prevRecordContext = internalProcessorContext.recordContext(); - internalProcessorContext.setRecordContext(toEmit.recordContext()); - try { - internalProcessorContext.forward(toEmit.key(), toEmit.value()); - suppressionEmitSensor.record(); - } finally { - internalProcessorContext.setRecordContext(prevRecordContext); - } - } - } - - private boolean shouldForward(final Change value) { - return value.newValue != null || !safeToDropTombstones; - } - - @Override - public void close() { - } -} diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorSupplier.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorSupplier.java new file mode 100644 index 0000000000000..58e38312a7988 --- /dev/null +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorSupplier.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.kstream.internals.suppress; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.errors.StreamsException; +import org.apache.kafka.streams.kstream.internals.Change; +import org.apache.kafka.streams.kstream.internals.KTableImpl; +import org.apache.kafka.streams.kstream.internals.KTableProcessorSupplier; +import org.apache.kafka.streams.kstream.internals.KTableValueGetter; +import org.apache.kafka.streams.kstream.internals.KTableValueGetterSupplier; +import org.apache.kafka.streams.kstream.internals.metrics.Sensors; +import org.apache.kafka.streams.kstream.internals.suppress.TimeDefinitions.TimeDefinition; +import org.apache.kafka.streams.processor.Processor; +import org.apache.kafka.streams.processor.ProcessorContext; +import org.apache.kafka.streams.processor.internals.InternalProcessorContext; +import org.apache.kafka.streams.processor.internals.ProcessorRecordContext; +import org.apache.kafka.streams.state.ValueAndTimestamp; +import org.apache.kafka.streams.state.internals.Maybe; +import org.apache.kafka.streams.state.internals.TimeOrderedKeyValueBuffer; + +import static java.util.Objects.requireNonNull; + +public class KTableSuppressProcessorSupplier implements KTableProcessorSupplier { + private final SuppressedInternal suppress; + private final String storeName; + private final KTableImpl parentKTable; + + public KTableSuppressProcessorSupplier(final SuppressedInternal suppress, + final String storeName, + final KTableImpl parentKTable) { + this.suppress = suppress; + this.storeName = storeName; + this.parentKTable = parentKTable; + // The suppress buffer requires seeing the old values, to support the prior value view. + parentKTable.enableSendingOldValues(); + } + + @Override + public Processor> get() { + return new KTableSuppressProcessor<>(suppress, storeName); + } + + @Override + public KTableValueGetterSupplier view() { + final KTableValueGetterSupplier parentValueGetterSupplier = parentKTable.valueGetterSupplier(); + return new KTableValueGetterSupplier() { + + @Override + public KTableValueGetter get() { + final KTableValueGetter parentGetter = parentValueGetterSupplier.get(); + return new KTableValueGetter() { + private TimeOrderedKeyValueBuffer buffer; + + @SuppressWarnings("unchecked") + @Override + public void init(final ProcessorContext context) { + parentGetter.init(context); + // the main processor is responsible for the buffer's lifecycle + buffer = requireNonNull((TimeOrderedKeyValueBuffer) context.getStateStore(storeName)); + } + + @Override + public ValueAndTimestamp get(final K key) { + final Maybe> maybeValue = buffer.priorValueForBuffered(key); + if (maybeValue.isDefined()) { + return maybeValue.getNullableValue(); + } else { + // not buffered, so the suppressed view is equal to the parent view + return parentGetter.get(key); + } + } + + @Override + public void close() { + parentGetter.close(); + // the main processor is responsible for the buffer's lifecycle + } + }; + } + + @Override + public String[] storeNames() { + final String[] parentStores = parentValueGetterSupplier.storeNames(); + final String[] stores = new String[1 + parentStores.length]; + System.arraycopy(parentStores, 0, stores, 1, parentStores.length); + stores[0] = storeName; + return stores; + } + }; + } + + @Override + public void enableSendingOldValues() { + parentKTable.enableSendingOldValues(); + } + + private static final class KTableSuppressProcessor implements Processor> { + private final long maxRecords; + private final long maxBytes; + private final long suppressDurationMillis; + private final TimeDefinition bufferTimeDefinition; + private final BufferFullStrategy bufferFullStrategy; + private final boolean safeToDropTombstones; + private final String storeName; + + private TimeOrderedKeyValueBuffer buffer; + private InternalProcessorContext internalProcessorContext; + private Sensor suppressionEmitSensor; + private long observedStreamTime = ConsumerRecord.NO_TIMESTAMP; + + private KTableSuppressProcessor(final SuppressedInternal suppress, final String storeName) { + this.storeName = storeName; + requireNonNull(suppress); + maxRecords = suppress.bufferConfig().maxRecords(); + maxBytes = suppress.bufferConfig().maxBytes(); + suppressDurationMillis = suppress.timeToWaitForMoreEvents().toMillis(); + bufferTimeDefinition = suppress.timeDefinition(); + bufferFullStrategy = suppress.bufferConfig().bufferFullStrategy(); + safeToDropTombstones = suppress.safeToDropTombstones(); + } + + @SuppressWarnings("unchecked") + @Override + public void init(final ProcessorContext context) { + internalProcessorContext = (InternalProcessorContext) context; + suppressionEmitSensor = Sensors.suppressionEmitSensor(internalProcessorContext); + + buffer = requireNonNull((TimeOrderedKeyValueBuffer) context.getStateStore(storeName)); + buffer.setSerdesIfNull((Serde) context.keySerde(), (Serde) context.valueSerde()); + } + + @Override + public void process(final K key, final Change value) { + observedStreamTime = Math.max(observedStreamTime, internalProcessorContext.timestamp()); + buffer(key, value); + enforceConstraints(); + } + + private void buffer(final K key, final Change value) { + final long bufferTime = bufferTimeDefinition.time(internalProcessorContext, key); + + buffer.put(bufferTime, key, value, internalProcessorContext.recordContext()); + } + + private void enforceConstraints() { + final long streamTime = observedStreamTime; + final long expiryTime = streamTime - suppressDurationMillis; + + buffer.evictWhile(() -> buffer.minTimestamp() <= expiryTime, this::emit); + + if (overCapacity()) { + switch (bufferFullStrategy) { + case EMIT: + buffer.evictWhile(this::overCapacity, this::emit); + return; + case SHUT_DOWN: + throw new StreamsException(String.format( + "%s buffer exceeded its max capacity. Currently [%d/%d] records and [%d/%d] bytes.", + internalProcessorContext.currentNode().name(), + buffer.numRecords(), maxRecords, + buffer.bufferSize(), maxBytes + )); + default: + throw new UnsupportedOperationException( + "The bufferFullStrategy [" + bufferFullStrategy + + "] is not implemented. This is a bug in Kafka Streams." + ); + } + } + } + + private boolean overCapacity() { + return buffer.numRecords() > maxRecords || buffer.bufferSize() > maxBytes; + } + + private void emit(final TimeOrderedKeyValueBuffer.Eviction toEmit) { + if (shouldForward(toEmit.value())) { + final ProcessorRecordContext prevRecordContext = internalProcessorContext.recordContext(); + internalProcessorContext.setRecordContext(toEmit.recordContext()); + try { + internalProcessorContext.forward(toEmit.key(), toEmit.value()); + suppressionEmitSensor.record(); + } finally { + internalProcessorContext.setRecordContext(prevRecordContext); + } + } + } + + private boolean shouldForward(final Change value) { + return value.newValue != null || !safeToDropTombstones; + } + + @Override + public void close() { + } + } +} diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/BufferKey.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/BufferKey.java new file mode 100644 index 0000000000000..9a13aa0ca958c --- /dev/null +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/BufferKey.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.state.internals; + +import org.apache.kafka.common.utils.Bytes; + +import java.util.Objects; + +public final class BufferKey implements Comparable { + private final long time; + private final Bytes key; + + BufferKey(final long time, final Bytes key) { + this.time = time; + this.key = key; + } + + long time() { + return time; + } + + Bytes key() { + return key; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final BufferKey bufferKey = (BufferKey) o; + return time == bufferKey.time && + Objects.equals(key, bufferKey.key); + } + + @Override + public int hashCode() { + return Objects.hash(time, key); + } + + @Override + public int compareTo(final BufferKey o) { + // ordering of keys within a time uses hashCode. + final int timeComparison = Long.compare(time, o.time); + return timeComparison == 0 ? key.compareTo(o.key) : timeComparison; + } + + @Override + public String toString() { + return "BufferKey{" + + "key=" + key + + ", time=" + time + + '}'; + } +} diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/BufferValue.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/BufferValue.java new file mode 100644 index 0000000000000..816894ec1690b --- /dev/null +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/BufferValue.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.state.internals; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Objects; + +public final class BufferValue { + private final ContextualRecord record; + private final byte[] priorValue; + + BufferValue(final ContextualRecord record, + final byte[] priorValue) { + this.record = record; + this.priorValue = priorValue; + } + + ContextualRecord record() { + return record; + } + + byte[] priorValue() { + return priorValue; + } + + static BufferValue deserialize(final ByteBuffer buffer) { + final ContextualRecord record = ContextualRecord.deserialize(buffer); + + final int priorValueLength = buffer.getInt(); + if (priorValueLength == -1) { + return new BufferValue(record, null); + } else { + final byte[] priorValue = new byte[priorValueLength]; + buffer.get(priorValue); + return new BufferValue(record, priorValue); + } + } + + ByteBuffer serialize(final int endPadding) { + + final int sizeOfPriorValueLength = Integer.BYTES; + final int sizeOfPriorValue = priorValue == null ? 0 : priorValue.length; + + final ByteBuffer buffer = record.serialize(sizeOfPriorValueLength + sizeOfPriorValue + endPadding); + + if (priorValue == null) { + buffer.putInt(-1); + } else { + buffer.putInt(priorValue.length); + buffer.put(priorValue); + } + + return buffer; + } + + long sizeBytes() { + return (priorValue == null ? 0 : priorValue.length) + record.sizeBytes(); + } + + @Override + public boolean equals(final Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final BufferValue that = (BufferValue) o; + return Objects.equals(record, that.record) && + Arrays.equals(priorValue, that.priorValue); + } + + @Override + public int hashCode() { + int result = Objects.hash(record); + result = 31 * result + Arrays.hashCode(priorValue); + return result; + } + + @Override + public String toString() { + return "BufferValue{" + + "record=" + record + + ", priorValue=" + Arrays.toString(priorValue) + + '}'; + } +} diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/ContextualRecord.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/ContextualRecord.java index 3893b35f921be..3cd2c37358404 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/ContextualRecord.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/ContextualRecord.java @@ -43,13 +43,13 @@ long sizeBytes() { return (value == null ? 0 : value.length) + recordContext.sizeBytes(); } - byte[] serialize() { + ByteBuffer serialize(final int endPadding) { final byte[] serializedContext = recordContext.serialize(); final int sizeOfContext = serializedContext.length; final int sizeOfValueLength = Integer.BYTES; final int sizeOfValue = value == null ? 0 : value.length; - final ByteBuffer buffer = ByteBuffer.allocate(sizeOfContext + sizeOfValueLength + sizeOfValue); + final ByteBuffer buffer = ByteBuffer.allocate(sizeOfContext + sizeOfValueLength + sizeOfValue + endPadding); buffer.put(serializedContext); if (value == null) { @@ -59,7 +59,7 @@ byte[] serialize() { buffer.put(value); } - return buffer.array(); + return buffer; } static ContextualRecord deserialize(final ByteBuffer buffer) { diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryTimeOrderedKeyValueBuffer.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryTimeOrderedKeyValueBuffer.java index e11df7caf38f0..6c5022f080ac2 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryTimeOrderedKeyValueBuffer.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryTimeOrderedKeyValueBuffer.java @@ -25,6 +25,8 @@ import org.apache.kafka.common.serialization.BytesSerializer; import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.kstream.internals.Change; +import org.apache.kafka.streams.kstream.internals.FullChangeSerde; import org.apache.kafka.streams.processor.ProcessorContext; import org.apache.kafka.streams.processor.StateStore; import org.apache.kafka.streams.processor.internals.InternalProcessorContext; @@ -32,7 +34,9 @@ import org.apache.kafka.streams.processor.internals.ProcessorStateManager; import org.apache.kafka.streams.processor.internals.RecordBatchingStateRestoreCallback; import org.apache.kafka.streams.processor.internals.RecordCollector; +import org.apache.kafka.streams.processor.internals.RecordQueue; import org.apache.kafka.streams.state.StoreBuilder; +import org.apache.kafka.streams.state.ValueAndTimestamp; import org.apache.kafka.streams.state.internals.metrics.Sensors; import java.nio.ByteBuffer; @@ -42,7 +46,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.Map; -import java.util.Objects; +import java.util.NoSuchElementException; import java.util.Set; import java.util.TreeMap; import java.util.function.Consumer; @@ -55,16 +59,18 @@ public final class InMemoryTimeOrderedKeyValueBuffer implements TimeOrdere private static final ByteArraySerializer VALUE_SERIALIZER = new ByteArraySerializer(); private static final RecordHeaders V_1_CHANGELOG_HEADERS = new RecordHeaders(new Header[] {new RecordHeader("v", new byte[] {(byte) 1})}); + private static final RecordHeaders V_2_CHANGELOG_HEADERS = + new RecordHeaders(new Header[] {new RecordHeader("v", new byte[] {(byte) 2})}); private final Map index = new HashMap<>(); - private final TreeMap sortedMap = new TreeMap<>(); + private final TreeMap sortedMap = new TreeMap<>(); private final Set dirtyKeys = new HashSet<>(); private final String storeName; private final boolean loggingEnabled; private Serde keySerde; - private Serde valueSerde; + private FullChangeSerde valueSerde; private long memBufferSize = 0L; private long minTimestamp = Long.MAX_VALUE; @@ -77,7 +83,7 @@ public final class InMemoryTimeOrderedKeyValueBuffer implements TimeOrdere private int partition; - public static class Builder implements StoreBuilder { + public static class Builder implements StoreBuilder> { private final String storeName; private final Serde keySerde; @@ -94,11 +100,11 @@ public Builder(final String storeName, final Serde keySerde, final Serde v * As of 2.1, there's no way for users to directly interact with the buffer, * so this method is implemented solely to be called by Streams (which * it will do based on the {@code cache.max.bytes.buffering} config. - * + *

    * It's currently a no-op. */ @Override - public StoreBuilder withCachingEnabled() { + public StoreBuilder> withCachingEnabled() { return this; } @@ -106,21 +112,21 @@ public StoreBuilder withCachingEnabled() { * As of 2.1, there's no way for users to directly interact with the buffer, * so this method is implemented solely to be called by Streams (which * it will do based on the {@code cache.max.bytes.buffering} config. - * + *

    * It's currently a no-op. */ @Override - public StoreBuilder withCachingDisabled() { + public StoreBuilder> withCachingDisabled() { return this; } @Override - public StoreBuilder withLoggingEnabled(final Map config) { + public StoreBuilder> withLoggingEnabled(final Map config) { throw new UnsupportedOperationException(); } @Override - public StoreBuilder withLoggingDisabled() { + public StoreBuilder> withLoggingDisabled() { loggingEnabled = false; return this; } @@ -146,49 +152,6 @@ public String name() { } } - private static final class BufferKey implements Comparable { - private final long time; - private final Bytes key; - - private BufferKey(final long time, final Bytes key) { - this.time = time; - this.key = key; - } - - @Override - public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - final BufferKey bufferKey = (BufferKey) o; - return time == bufferKey.time && - Objects.equals(key, bufferKey.key); - } - - @Override - public int hashCode() { - return Objects.hash(time, key); - } - - @Override - public int compareTo(final BufferKey o) { - // ordering of keys within a time uses hashCode. - final int timeComparison = Long.compare(time, o.time); - return timeComparison == 0 ? key.compareTo(o.key) : timeComparison; - } - - @Override - public String toString() { - return "BufferKey{" + - "key=" + key + - ", time=" + time + - '}'; - } - } - private InMemoryTimeOrderedKeyValueBuffer(final String storeName, final boolean loggingEnabled, final Serde keySerde, @@ -196,7 +159,7 @@ private InMemoryTimeOrderedKeyValueBuffer(final String storeName, this.storeName = storeName; this.loggingEnabled = loggingEnabled; this.keySerde = keySerde; - this.valueSerde = valueSerde; + this.valueSerde = FullChangeSerde.castOrWrap(valueSerde); } @Override @@ -213,7 +176,7 @@ public boolean persistent() { @Override public void setSerdesIfNull(final Serde keySerde, final Serde valueSerde) { this.keySerde = this.keySerde == null ? keySerde : this.keySerde; - this.valueSerde = this.valueSerde == null ? valueSerde : this.valueSerde; + this.valueSerde = this.valueSerde == null ? FullChangeSerde.castOrWrap(valueSerde) : this.valueSerde; } @Override @@ -261,7 +224,7 @@ public void flush() { // The record was evicted from the buffer. Send a tombstone. logTombstone(key); } else { - final ContextualRecord value = sortedMap.get(bufferKey); + final BufferValue value = sortedMap.get(bufferKey); logValue(key, bufferKey, value); } @@ -270,22 +233,17 @@ public void flush() { } } - private void logValue(final Bytes key, final BufferKey bufferKey, final ContextualRecord value) { - final byte[] serializedContextualRecord = value.serialize(); + private void logValue(final Bytes key, final BufferKey bufferKey, final BufferValue value) { final int sizeOfBufferTime = Long.BYTES; - final int sizeOfContextualRecord = serializedContextualRecord.length; - - final byte[] timeAndContextualRecord = ByteBuffer.wrap(new byte[sizeOfBufferTime + sizeOfContextualRecord]) - .putLong(bufferKey.time) - .put(serializedContextualRecord) - .array(); + final ByteBuffer buffer = value.serialize(sizeOfBufferTime); + buffer.putLong(bufferKey.time()); collector.send( changelogTopic, key, - timeAndContextualRecord, - V_1_CHANGELOG_HEADERS, + buffer.array(), + V_2_CHANGELOG_HEADERS, partition, null, KEY_SERIALIZER, @@ -312,12 +270,12 @@ private void restoreBatch(final Collection> batch // This was a tombstone. Delete the record. final BufferKey bufferKey = index.remove(key); if (bufferKey != null) { - final ContextualRecord removed = sortedMap.remove(bufferKey); + final BufferValue removed = sortedMap.remove(bufferKey); if (removed != null) { - memBufferSize -= computeRecordSize(bufferKey.key, removed); + memBufferSize -= computeRecordSize(bufferKey.key(), removed); } - if (bufferKey.time == minTimestamp) { - minTimestamp = sortedMap.isEmpty() ? Long.MAX_VALUE : sortedMap.firstKey().time; + if (bufferKey.time() == minTimestamp) { + minTimestamp = sortedMap.isEmpty() ? Long.MAX_VALUE : sortedMap.firstKey().time(); } } @@ -331,33 +289,46 @@ private void restoreBatch(final Collection> batch ); } } else { - final ByteBuffer timeAndValue = ByteBuffer.wrap(record.value()); - final long time = timeAndValue.getLong(); - final byte[] value = new byte[record.value().length - 8]; - timeAndValue.get(value); if (record.headers().lastHeader("v") == null) { + // in this case, the changelog value is just the serialized record value + final ByteBuffer timeAndValue = ByteBuffer.wrap(record.value()); + final long time = timeAndValue.getLong(); + final byte[] changelogValue = new byte[record.value().length - 8]; + timeAndValue.get(changelogValue); + cleanPut( time, key, - new ContextualRecord( - value, - new ProcessorRecordContext( - record.timestamp(), - record.offset(), - record.partition(), - record.topic(), - record.headers() - ) + new BufferValue( + new ContextualRecord( + changelogValue, + new ProcessorRecordContext( + record.timestamp(), + record.offset(), + record.partition(), + record.topic(), + record.headers() + ) + ), + inferPriorValue(key, changelogValue) ) ); } else if (V_1_CHANGELOG_HEADERS.lastHeader("v").equals(record.headers().lastHeader("v"))) { - final ContextualRecord contextualRecord = ContextualRecord.deserialize(ByteBuffer.wrap(value)); - - cleanPut( - time, - key, - contextualRecord - ); + // in this case, the changelog value is a serialized ContextualRecord + final ByteBuffer timeAndValue = ByteBuffer.wrap(record.value()); + final long time = timeAndValue.getLong(); + final byte[] changelogValue = new byte[record.value().length - 8]; + timeAndValue.get(changelogValue); + + final ContextualRecord contextualRecord = ContextualRecord.deserialize(ByteBuffer.wrap(changelogValue)); + cleanPut(time, key, new BufferValue(contextualRecord, inferPriorValue(key, contextualRecord.value()))); + } else if (V_2_CHANGELOG_HEADERS.lastHeader("v").equals(record.headers().lastHeader("v"))) { + // in this case, the changelog value is a serialized BufferValue + + final ByteBuffer valueAndTime = ByteBuffer.wrap(record.value()); + final BufferValue bufferValue = BufferValue.deserialize(valueAndTime); + final long time = valueAndTime.getLong(); + cleanPut(time, key, bufferValue); } else { throw new IllegalArgumentException("Restoring apparently invalid changelog record: " + record); } @@ -375,42 +346,50 @@ private void restoreBatch(final Collection> batch updateBufferMetrics(); } + private byte[] inferPriorValue(final Bytes key, final byte[] serializedChange) { + return index.containsKey(key) + ? internalPriorValueForBuffered(key) + : FullChangeSerde.extractOldValuePart(ByteBuffer.wrap(serializedChange)); + } + @Override public void evictWhile(final Supplier predicate, final Consumer> callback) { - final Iterator> delegate = sortedMap.entrySet().iterator(); + final Iterator> delegate = sortedMap.entrySet().iterator(); int evictions = 0; if (predicate.get()) { - Map.Entry next = null; + Map.Entry next = null; if (delegate.hasNext()) { next = delegate.next(); } // predicate being true means we read one record, call the callback, and then remove it while (next != null && predicate.get()) { - if (next.getKey().time != minTimestamp) { + if (next.getKey().time() != minTimestamp) { throw new IllegalStateException( "minTimestamp [" + minTimestamp + "] did not match the actual min timestamp [" + - next.getKey().time + "]" + next.getKey().time() + "]" ); } - final K key = keySerde.deserializer().deserialize(changelogTopic, next.getKey().key.get()); - final V value = valueSerde.deserializer().deserialize(changelogTopic, next.getValue().value()); - callback.accept(new Eviction<>(key, value, next.getValue().recordContext())); + final K key = keySerde.deserializer().deserialize(changelogTopic, next.getKey().key().get()); + final BufferValue bufferValue = next.getValue(); + final ContextualRecord record = bufferValue.record(); + final Change value = valueSerde.deserializer().deserialize(changelogTopic, record.value()); + callback.accept(new Eviction<>(key, value, record.recordContext())); delegate.remove(); - index.remove(next.getKey().key); + index.remove(next.getKey().key()); - dirtyKeys.add(next.getKey().key); + dirtyKeys.add(next.getKey().key()); - memBufferSize -= computeRecordSize(next.getKey().key, next.getValue()); + memBufferSize -= computeRecordSize(next.getKey().key(), bufferValue); // peek at the next record so we can update the minTimestamp if (delegate.hasNext()) { next = delegate.next(); - minTimestamp = next == null ? Long.MAX_VALUE : next.getKey().time; + minTimestamp = next == null ? Long.MAX_VALUE : next.getKey().time(); } else { next = null; minTimestamp = Long.MAX_VALUE; @@ -424,10 +403,40 @@ public void evictWhile(final Supplier predicate, } } + @Override + public Maybe> priorValueForBuffered(final K key) { + final Bytes serializedKey = Bytes.wrap(keySerde.serializer().serialize(changelogTopic, key)); + if (index.containsKey(serializedKey)) { + final byte[] serializedValue = internalPriorValueForBuffered(serializedKey); + + final V deserializedValue = valueSerde.innerSerde().deserializer().deserialize( + changelogTopic, + serializedValue + ); + + // it's unfortunately not possible to know this, unless we materialize the suppressed result, since our only + // knowledge of the prior value is what the upstream processor sends us as the "old value" when we first + // buffer something. + return Maybe.defined(ValueAndTimestamp.make(deserializedValue, RecordQueue.UNKNOWN)); + } else { + return Maybe.undefined(); + } + } + + private byte[] internalPriorValueForBuffered(final Bytes key) { + final BufferKey bufferKey = index.get(key); + if (bufferKey == null) { + throw new NoSuchElementException("Key [" + key + "] is not in the buffer."); + } else { + final BufferValue bufferValue = sortedMap.get(bufferKey); + return bufferValue.priorValue(); + } + } + @Override public void put(final long time, final K key, - final V value, + final Change value, final ProcessorRecordContext recordContext) { requireNonNull(value, "value cannot be null"); requireNonNull(recordContext, "recordContext cannot be null"); @@ -435,12 +444,26 @@ public void put(final long time, final Bytes serializedKey = Bytes.wrap(keySerde.serializer().serialize(changelogTopic, key)); final byte[] serializedValue = valueSerde.serializer().serialize(changelogTopic, value); - cleanPut(time, serializedKey, new ContextualRecord(serializedValue, recordContext)); + final BufferValue buffered = getBuffered(serializedKey); + final byte[] serializedPriorValue; + if (buffered == null) { + final V priorValue = value.oldValue; + serializedPriorValue = valueSerde.innerSerde().serializer().serialize(changelogTopic, priorValue); + } else { + serializedPriorValue = buffered.priorValue(); + } + + cleanPut(time, serializedKey, new BufferValue(new ContextualRecord(serializedValue, recordContext), serializedPriorValue)); dirtyKeys.add(serializedKey); updateBufferMetrics(); } - private void cleanPut(final long time, final Bytes key, final ContextualRecord value) { + private BufferValue getBuffered(final Bytes key) { + final BufferKey bufferKey = index.get(key); + return bufferKey == null ? null : sortedMap.get(bufferKey); + } + + private void cleanPut(final long time, final Bytes key, final BufferValue value) { // non-resetting semantics: // if there was a previous version of the same record, // then insert the new record in the same place in the priority queue @@ -453,7 +476,7 @@ private void cleanPut(final long time, final Bytes key, final ContextualRecord v minTimestamp = Math.min(minTimestamp, time); memBufferSize += computeRecordSize(key, value); } else { - final ContextualRecord removedValue = sortedMap.put(previousKey, value); + final BufferValue removedValue = sortedMap.put(previousKey, value); memBufferSize = memBufferSize + computeRecordSize(key, value) @@ -476,7 +499,7 @@ public long minTimestamp() { return minTimestamp; } - private static long computeRecordSize(final Bytes key, final ContextualRecord value) { + private static long computeRecordSize(final Bytes key, final BufferValue value) { long size = 0L; size += 8; // buffer time size += key.get().length; diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/Maybe.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/Maybe.java new file mode 100644 index 0000000000000..c292c17011d34 --- /dev/null +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/Maybe.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.state.internals; + +import java.util.NoSuchElementException; +import java.util.Objects; + +/** + * A container that may be empty, may contain null, or may contain a value. + * Distinct from {@link java.util.Optional}, since Optional cannot contain null. + * + * @param + */ +public final class Maybe { + private final T nullableValue; + private final boolean defined; + + public static Maybe defined(final T nullableValue) { + return new Maybe<>(nullableValue); + } + + public static Maybe undefined() { + return new Maybe<>(); + } + + private Maybe(final T nullableValue) { + this.nullableValue = nullableValue; + defined = true; + } + + private Maybe() { + nullableValue = null; + defined = false; + } + + public T getNullableValue() { + if (defined) { + return nullableValue; + } else { + throw new NoSuchElementException(); + } + } + + public boolean isDefined() { + return defined; + } + + @Override + public boolean equals(final Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final Maybe maybe = (Maybe) o; + + // All undefined maybes are equal + // All defined null maybes are equal + return defined == maybe.defined && + (!defined || Objects.equals(nullableValue, maybe.nullableValue)); + } + + @Override + public int hashCode() { + // Since all undefined maybes are equal, we can hard-code their hashCode to -1. + // Since all defined null maybes are equal, we can hard-code their hashCode to 0. + return defined ? nullableValue == null ? 0 : nullableValue.hashCode() : -1; + } + + @Override + public String toString() { + if (defined) { + return "DefinedMaybe{" + String.valueOf(nullableValue) + "}"; + } else { + return "UndefinedMaybe{}"; + } + } +} diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/TimeOrderedKeyValueBuffer.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/TimeOrderedKeyValueBuffer.java index ffa1f49c4bca7..3aabbaad54de1 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/TimeOrderedKeyValueBuffer.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/TimeOrderedKeyValueBuffer.java @@ -17,20 +17,23 @@ package org.apache.kafka.streams.state.internals; import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.kstream.internals.Change; import org.apache.kafka.streams.processor.StateStore; import org.apache.kafka.streams.processor.internals.ProcessorRecordContext; +import org.apache.kafka.streams.state.ValueAndTimestamp; import java.util.Objects; import java.util.function.Consumer; import java.util.function.Supplier; public interface TimeOrderedKeyValueBuffer extends StateStore { + final class Eviction { private final K key; - private final V value; + private final Change value; private final ProcessorRecordContext recordContext; - Eviction(final K key, final V value, final ProcessorRecordContext recordContext) { + Eviction(final K key, final Change value, final ProcessorRecordContext recordContext) { this.key = key; this.value = value; this.recordContext = recordContext; @@ -40,7 +43,7 @@ public K key() { return key; } - public V value() { + public Change value() { return value; } @@ -73,7 +76,9 @@ public int hashCode() { void evictWhile(final Supplier predicate, final Consumer> callback); - void put(long time, K key, V value, ProcessorRecordContext recordContext); + Maybe> priorValueForBuffered(K key); + + void put(long time, K key, Change value, ProcessorRecordContext recordContext); int numRecords(); diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SuppressScenarioTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SuppressScenarioTest.java index 43ad42ad01370..3c7fd1e2c2e64 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SuppressScenarioTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SuppressScenarioTest.java @@ -517,6 +517,204 @@ public void shouldSupportFinalResultsForSessionWindows() { } } + @Test + public void shouldWorkBeforeGroupBy() { + final StreamsBuilder builder = new StreamsBuilder(); + + builder + .table("topic", Consumed.with(Serdes.String(), Serdes.String())) + .suppress(untilTimeLimit(ofMillis(10), unbounded())) + .groupBy(KeyValue::pair, Grouped.with(Serdes.String(), Serdes.String())) + .count() + .toStream() + .to("output", Produced.with(Serdes.String(), Serdes.Long())); + + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), config)) { + final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(STRING_SERIALIZER, STRING_SERIALIZER); + + driver.pipeInput(recordFactory.create("topic", "A", "a", 0L)); + driver.pipeInput(recordFactory.create("topic", "tick", "tick", 10L)); + + verify( + drainProducerRecords(driver, "output", STRING_DESERIALIZER, LONG_DESERIALIZER), + singletonList(new KeyValueTimestamp<>("A", 1L, 0L)) + ); + } + } + + @Test + public void shouldWorkBeforeJoinRight() { + final StreamsBuilder builder = new StreamsBuilder(); + + final KTable left = builder + .table("left", Consumed.with(Serdes.String(), Serdes.String())); + + final KTable right = builder + .table("right", Consumed.with(Serdes.String(), Serdes.String())) + .suppress(untilTimeLimit(ofMillis(10), unbounded())); + + left + .outerJoin(right, (l, r) -> String.format("(%s,%s)", l, r)) + .toStream() + .to("output", Produced.with(Serdes.String(), Serdes.String())); + + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), config)) { + final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(STRING_SERIALIZER, STRING_SERIALIZER); + + driver.pipeInput(recordFactory.create("right", "B", "1", 0L)); + driver.pipeInput(recordFactory.create("right", "A", "1", 0L)); + // buffered, no output + verify( + drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER), + emptyList() + ); + + + driver.pipeInput(recordFactory.create("right", "tick", "tick", 10L)); + // flush buffer + verify( + drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER), + asList( + new KeyValueTimestamp<>("A", "(null,1)", 0L), + new KeyValueTimestamp<>("B", "(null,1)", 0L) + ) + ); + + + driver.pipeInput(recordFactory.create("right", "A", "2", 11L)); + // buffered, no output + verify( + drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER), + emptyList() + ); + + + driver.pipeInput(recordFactory.create("left", "A", "a", 12L)); + // should join with previously emitted right side + verify( + drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER), + singletonList(new KeyValueTimestamp<>("A", "(a,1)", 12L)) + ); + + + driver.pipeInput(recordFactory.create("left", "B", "b", 12L)); + // should view through to the parent KTable, since B is no longer buffered + verify( + drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER), + singletonList(new KeyValueTimestamp<>("B", "(b,1)", 12L)) + ); + + + driver.pipeInput(recordFactory.create("left", "A", "b", 13L)); + // should join with previously emitted right side + verify( + drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER), + singletonList(new KeyValueTimestamp<>("A", "(b,1)", 13L)) + ); + + + driver.pipeInput(recordFactory.create("right", "tick", "tick", 21L)); + verify( + drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER), + asList( + new KeyValueTimestamp<>("tick", "(null,tick)", 21), // just a testing artifact + new KeyValueTimestamp<>("A", "(b,2)", 13L) + ) + ); + } + + } + + + @Test + public void shouldWorkBeforeJoinLeft() { + final StreamsBuilder builder = new StreamsBuilder(); + + final KTable left = builder + .table("left", Consumed.with(Serdes.String(), Serdes.String())) + .suppress(untilTimeLimit(ofMillis(10), unbounded())); + + final KTable right = builder + .table("right", Consumed.with(Serdes.String(), Serdes.String())); + + left + .outerJoin(right, (l, r) -> String.format("(%s,%s)", l, r)) + .toStream() + .to("output", Produced.with(Serdes.String(), Serdes.String())); + + final Topology topology = builder.build(); + System.out.println(topology.describe()); + try (final TopologyTestDriver driver = new TopologyTestDriver(topology, config)) { + final ConsumerRecordFactory recordFactory = + new ConsumerRecordFactory<>(STRING_SERIALIZER, STRING_SERIALIZER); + + driver.pipeInput(recordFactory.create("left", "B", "1", 0L)); + driver.pipeInput(recordFactory.create("left", "A", "1", 0L)); + // buffered, no output + verify( + drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER), + emptyList() + ); + + + driver.pipeInput(recordFactory.create("left", "tick", "tick", 10L)); + // flush buffer + verify( + drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER), + asList( + new KeyValueTimestamp<>("A", "(1,null)", 0L), + new KeyValueTimestamp<>("B", "(1,null)", 0L) + ) + ); + + + driver.pipeInput(recordFactory.create("left", "A", "2", 11L)); + // buffered, no output + verify( + drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER), + emptyList() + ); + + + driver.pipeInput(recordFactory.create("right", "A", "a", 12L)); + // should join with previously emitted left side + verify( + drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER), + singletonList(new KeyValueTimestamp<>("A", "(1,a)", 12L)) + ); + + + driver.pipeInput(recordFactory.create("right", "B", "b", 12L)); + // should view through to the parent KTable, since B is no longer buffered + verify( + drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER), + singletonList(new KeyValueTimestamp<>("B", "(1,b)", 12L)) + ); + + + driver.pipeInput(recordFactory.create("right", "A", "b", 13L)); + // should join with previously emitted left side + verify( + drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER), + singletonList(new KeyValueTimestamp<>("A", "(1,b)", 13L)) + ); + + + driver.pipeInput(recordFactory.create("left", "tick", "tick", 21L)); + verify( + drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER), + asList( + new KeyValueTimestamp<>("tick", "(tick,null)", 21), // just a testing artifact + new KeyValueTimestamp<>("A", "(2,b)", 13L) + ) + ); + } + + } + + private static void verify(final List> results, final List> expectedResults) { if (results.size() != expectedResults.size()) { diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorMetricsTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorMetricsTest.java index 62ae3bfc41a37..96ee73553a40e 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorMetricsTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorMetricsTest.java @@ -22,10 +22,13 @@ import org.apache.kafka.streams.kstream.Suppressed; import org.apache.kafka.streams.kstream.internals.Change; import org.apache.kafka.streams.kstream.internals.FullChangeSerde; +import org.apache.kafka.streams.kstream.internals.KTableImpl; +import org.apache.kafka.streams.processor.Processor; import org.apache.kafka.streams.processor.StateStore; import org.apache.kafka.streams.processor.internals.ProcessorNode; import org.apache.kafka.streams.state.internals.InMemoryTimeOrderedKeyValueBuffer; import org.apache.kafka.test.MockInternalProcessorContext; +import org.easymock.EasyMock; import org.hamcrest.Matcher; import org.junit.Test; @@ -141,11 +144,13 @@ public void shouldRecordMetrics() { .withLoggingDisabled() .build(); - final KTableSuppressProcessor processor = - new KTableSuppressProcessor<>( + final KTableImpl mock = EasyMock.mock(KTableImpl.class); + final Processor> processor = + new KTableSuppressProcessorSupplier<>( (SuppressedInternal) Suppressed.untilTimeLimit(Duration.ofDays(100), maxRecords(1)), - storeName - ); + storeName, + mock + ).get(); final MockInternalProcessorContext context = new MockInternalProcessorContext(); context.setCurrentNode(new ProcessorNode("testNode")); @@ -164,9 +169,9 @@ public void shouldRecordMetrics() { verifyMetric(metrics, EVICTION_RATE_METRIC, is(0.0)); verifyMetric(metrics, EVICTION_TOTAL_METRIC, is(0.0)); - verifyMetric(metrics, BUFFER_SIZE_AVG_METRIC, is(25.5)); - verifyMetric(metrics, BUFFER_SIZE_CURRENT_METRIC, is(51.0)); - verifyMetric(metrics, BUFFER_SIZE_MAX_METRIC, is(51.0)); + verifyMetric(metrics, BUFFER_SIZE_AVG_METRIC, is(29.5)); + verifyMetric(metrics, BUFFER_SIZE_CURRENT_METRIC, is(59.0)); + verifyMetric(metrics, BUFFER_SIZE_MAX_METRIC, is(59.0)); verifyMetric(metrics, BUFFER_COUNT_AVG_METRIC, is(0.5)); verifyMetric(metrics, BUFFER_COUNT_CURRENT_METRIC, is(1.0)); verifyMetric(metrics, BUFFER_COUNT_MAX_METRIC, is(1.0)); @@ -180,9 +185,9 @@ public void shouldRecordMetrics() { verifyMetric(metrics, EVICTION_RATE_METRIC, greaterThan(0.0)); verifyMetric(metrics, EVICTION_TOTAL_METRIC, is(1.0)); - verifyMetric(metrics, BUFFER_SIZE_AVG_METRIC, is(49.0)); - verifyMetric(metrics, BUFFER_SIZE_CURRENT_METRIC, is(47.0)); - verifyMetric(metrics, BUFFER_SIZE_MAX_METRIC, is(98.0)); + verifyMetric(metrics, BUFFER_SIZE_AVG_METRIC, is(57.0)); + verifyMetric(metrics, BUFFER_SIZE_CURRENT_METRIC, is(55.0)); + verifyMetric(metrics, BUFFER_SIZE_MAX_METRIC, is(114.0)); verifyMetric(metrics, BUFFER_COUNT_AVG_METRIC, is(1.0)); verifyMetric(metrics, BUFFER_COUNT_CURRENT_METRIC, is(1.0)); verifyMetric(metrics, BUFFER_COUNT_MAX_METRIC, is(2.0)); diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorTest.java index 6c10d91cf90ca..d8cb858f5f91e 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorTest.java @@ -26,13 +26,16 @@ import org.apache.kafka.streams.kstream.Windowed; import org.apache.kafka.streams.kstream.internals.Change; import org.apache.kafka.streams.kstream.internals.FullChangeSerde; +import org.apache.kafka.streams.kstream.internals.KTableImpl; import org.apache.kafka.streams.kstream.internals.SessionWindow; import org.apache.kafka.streams.kstream.internals.TimeWindow; import org.apache.kafka.streams.processor.MockProcessorContext; +import org.apache.kafka.streams.processor.Processor; import org.apache.kafka.streams.processor.StateStore; import org.apache.kafka.streams.processor.internals.ProcessorNode; import org.apache.kafka.streams.state.internals.InMemoryTimeOrderedKeyValueBuffer; import org.apache.kafka.test.MockInternalProcessorContext; +import org.easymock.EasyMock; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; import org.hamcrest.Matcher; @@ -62,7 +65,7 @@ public class KTableSuppressProcessorTest { private static final Change ARBITRARY_CHANGE = new Change<>(7L, 14L); private static class Harness { - private final KTableSuppressProcessor processor; + private final Processor> processor; private final MockInternalProcessorContext context; @@ -76,8 +79,9 @@ private static class Harness { .withLoggingDisabled() .build(); - final KTableSuppressProcessor processor = - new KTableSuppressProcessor<>((SuppressedInternal) suppressed, storeName); + final KTableImpl parent = EasyMock.mock(KTableImpl.class); + final Processor> processor = + new KTableSuppressProcessorSupplier<>((SuppressedInternal) suppressed, storeName, parent).get(); final MockInternalProcessorContext context = new MockInternalProcessorContext(); context.setCurrentNode(new ProcessorNode("testNode")); @@ -95,13 +99,12 @@ public void zeroTimeLimitShouldImmediatelyEmit() { final Harness harness = new Harness<>(untilTimeLimit(ZERO, unbounded()), String(), Long()); final MockInternalProcessorContext context = harness.context; - final KTableSuppressProcessor processor = harness.processor; final long timestamp = ARBITRARY_LONG; context.setRecordMetadata("", 0, 0L, null, timestamp); final String key = "hey"; final Change value = ARBITRARY_CHANGE; - processor.process(key, value); + harness.processor.process(key, value); assertThat(context.forwarded(), hasSize(1)); final MockProcessorContext.CapturedForward capturedForward = context.forwarded().get(0); @@ -114,13 +117,12 @@ public void windowedZeroTimeLimitShouldImmediatelyEmit() { final Harness, Long> harness = new Harness<>(untilTimeLimit(ZERO, unbounded()), timeWindowedSerdeFrom(String.class, 100L), Long()); final MockInternalProcessorContext context = harness.context; - final KTableSuppressProcessor, Long> processor = harness.processor; final long timestamp = ARBITRARY_LONG; context.setRecordMetadata("", 0, 0L, null, timestamp); final Windowed key = new Windowed<>("hey", new TimeWindow(0L, 100L)); final Change value = ARBITRARY_CHANGE; - processor.process(key, value); + harness.processor.process(key, value); assertThat(context.forwarded(), hasSize(1)); final MockProcessorContext.CapturedForward capturedForward = context.forwarded().get(0); @@ -133,17 +135,16 @@ public void intermediateSuppressionShouldBufferAndEmitLater() { final Harness harness = new Harness<>(untilTimeLimit(ofMillis(1), unbounded()), String(), Long()); final MockInternalProcessorContext context = harness.context; - final KTableSuppressProcessor processor = harness.processor; final long timestamp = 0L; context.setRecordMetadata("topic", 0, 0, null, timestamp); final String key = "hey"; final Change value = new Change<>(null, 1L); - processor.process(key, value); + harness.processor.process(key, value); assertThat(context.forwarded(), hasSize(0)); context.setRecordMetadata("topic", 0, 1, null, 1L); - processor.process("tick", new Change<>(null, null)); + harness.processor.process("tick", new Change<>(null, null)); assertThat(context.forwarded(), hasSize(1)); final MockProcessorContext.CapturedForward capturedForward = context.forwarded().get(0); @@ -156,7 +157,6 @@ public void finalResultsSuppressionShouldBufferAndEmitAtGraceExpiration() { final Harness, Long> harness = new Harness<>(finalResults(ofMillis(1L)), timeWindowedSerdeFrom(String.class, 1L), Long()); final MockInternalProcessorContext context = harness.context; - final KTableSuppressProcessor, Long> processor = harness.processor; final long windowStart = 99L; final long recordTime = 99L; @@ -164,7 +164,7 @@ public void finalResultsSuppressionShouldBufferAndEmitAtGraceExpiration() { context.setRecordMetadata("topic", 0, 0, null, recordTime); final Windowed key = new Windowed<>("hey", new TimeWindow(windowStart, windowEnd)); final Change value = ARBITRARY_CHANGE; - processor.process(key, value); + harness.processor.process(key, value); assertThat(context.forwarded(), hasSize(0)); // although the stream time is now 100, we have to wait 1 ms after the window *end* before we @@ -173,7 +173,7 @@ public void finalResultsSuppressionShouldBufferAndEmitAtGraceExpiration() { final long recordTime2 = 100L; final long windowEnd2 = 101L; context.setRecordMetadata("topic", 0, 1, null, recordTime2); - processor.process(new Windowed<>("dummyKey1", new TimeWindow(windowStart2, windowEnd2)), ARBITRARY_CHANGE); + harness.processor.process(new Windowed<>("dummyKey1", new TimeWindow(windowStart2, windowEnd2)), ARBITRARY_CHANGE); assertThat(context.forwarded(), hasSize(0)); // ok, now it's time to emit "hey" @@ -181,7 +181,7 @@ public void finalResultsSuppressionShouldBufferAndEmitAtGraceExpiration() { final long recordTime3 = 101L; final long windowEnd3 = 102L; context.setRecordMetadata("topic", 0, 1, null, recordTime3); - processor.process(new Windowed<>("dummyKey2", new TimeWindow(windowStart3, windowEnd3)), ARBITRARY_CHANGE); + harness.processor.process(new Windowed<>("dummyKey2", new TimeWindow(windowStart3, windowEnd3)), ARBITRARY_CHANGE); assertThat(context.forwarded(), hasSize(1)); final MockProcessorContext.CapturedForward capturedForward = context.forwarded().get(0); @@ -199,7 +199,6 @@ public void finalResultsWithZeroGraceShouldStillBufferUntilTheWindowEnd() { final Harness, Long> harness = new Harness<>(finalResults(ofMillis(0L)), timeWindowedSerdeFrom(String.class, 100L), Long()); final MockInternalProcessorContext context = harness.context; - final KTableSuppressProcessor, Long> processor = harness.processor; // note the record is in the past, but the window end is in the future, so we still have to buffer, // even though the grace period is 0. @@ -208,11 +207,11 @@ public void finalResultsWithZeroGraceShouldStillBufferUntilTheWindowEnd() { context.setRecordMetadata("", 0, 0L, null, timestamp); final Windowed key = new Windowed<>("hey", new TimeWindow(0, windowEnd)); final Change value = ARBITRARY_CHANGE; - processor.process(key, value); + harness.processor.process(key, value); assertThat(context.forwarded(), hasSize(0)); context.setRecordMetadata("", 0, 1L, null, windowEnd); - processor.process(new Windowed<>("dummyKey", new TimeWindow(windowEnd, windowEnd + 100L)), ARBITRARY_CHANGE); + harness.processor.process(new Windowed<>("dummyKey", new TimeWindow(windowEnd, windowEnd + 100L)), ARBITRARY_CHANGE); assertThat(context.forwarded(), hasSize(1)); final MockProcessorContext.CapturedForward capturedForward = context.forwarded().get(0); @@ -225,13 +224,12 @@ public void finalResultsWithZeroGraceAtWindowEndShouldImmediatelyEmit() { final Harness, Long> harness = new Harness<>(finalResults(ofMillis(0L)), timeWindowedSerdeFrom(String.class, 100L), Long()); final MockInternalProcessorContext context = harness.context; - final KTableSuppressProcessor, Long> processor = harness.processor; final long timestamp = 100L; context.setRecordMetadata("", 0, 0L, null, timestamp); final Windowed key = new Windowed<>("hey", new TimeWindow(0, 100L)); final Change value = ARBITRARY_CHANGE; - processor.process(key, value); + harness.processor.process(key, value); assertThat(context.forwarded(), hasSize(1)); final MockProcessorContext.CapturedForward capturedForward = context.forwarded().get(0); @@ -248,13 +246,12 @@ public void finalResultsShouldDropTombstonesForTimeWindows() { final Harness, Long> harness = new Harness<>(finalResults(ofMillis(0L)), timeWindowedSerdeFrom(String.class, 100L), Long()); final MockInternalProcessorContext context = harness.context; - final KTableSuppressProcessor, Long> processor = harness.processor; final long timestamp = 100L; context.setRecordMetadata("", 0, 0L, null, timestamp); final Windowed key = new Windowed<>("hey", new TimeWindow(0, 100L)); final Change value = new Change<>(null, ARBITRARY_LONG); - processor.process(key, value); + harness.processor.process(key, value); assertThat(context.forwarded(), hasSize(0)); } @@ -269,13 +266,12 @@ public void finalResultsShouldDropTombstonesForSessionWindows() { final Harness, Long> harness = new Harness<>(finalResults(ofMillis(0L)), sessionWindowedSerdeFrom(String.class), Long()); final MockInternalProcessorContext context = harness.context; - final KTableSuppressProcessor, Long> processor = harness.processor; final long timestamp = 100L; context.setRecordMetadata("", 0, 0L, null, timestamp); final Windowed key = new Windowed<>("hey", new SessionWindow(0L, 0L)); final Change value = new Change<>(null, ARBITRARY_LONG); - processor.process(key, value); + harness.processor.process(key, value); assertThat(context.forwarded(), hasSize(0)); } @@ -289,13 +285,12 @@ public void suppressShouldNotDropTombstonesForTimeWindows() { final Harness, Long> harness = new Harness<>(untilTimeLimit(ofMillis(0), maxRecords(0)), timeWindowedSerdeFrom(String.class, 100L), Long()); final MockInternalProcessorContext context = harness.context; - final KTableSuppressProcessor, Long> processor = harness.processor; final long timestamp = 100L; context.setRecordMetadata("", 0, 0L, null, timestamp); final Windowed key = new Windowed<>("hey", new TimeWindow(0L, 100L)); final Change value = new Change<>(null, ARBITRARY_LONG); - processor.process(key, value); + harness.processor.process(key, value); assertThat(context.forwarded(), hasSize(1)); final MockProcessorContext.CapturedForward capturedForward = context.forwarded().get(0); @@ -313,13 +308,12 @@ public void suppressShouldNotDropTombstonesForSessionWindows() { final Harness, Long> harness = new Harness<>(untilTimeLimit(ofMillis(0), maxRecords(0)), sessionWindowedSerdeFrom(String.class), Long()); final MockInternalProcessorContext context = harness.context; - final KTableSuppressProcessor, Long> processor = harness.processor; final long timestamp = 100L; context.setRecordMetadata("", 0, 0L, null, timestamp); final Windowed key = new Windowed<>("hey", new SessionWindow(0L, 0L)); final Change value = new Change<>(null, ARBITRARY_LONG); - processor.process(key, value); + harness.processor.process(key, value); assertThat(context.forwarded(), hasSize(1)); final MockProcessorContext.CapturedForward capturedForward = context.forwarded().get(0); @@ -337,13 +331,12 @@ public void suppressShouldNotDropTombstonesForKTable() { final Harness harness = new Harness<>(untilTimeLimit(ofMillis(0), maxRecords(0)), String(), Long()); final MockInternalProcessorContext context = harness.context; - final KTableSuppressProcessor processor = harness.processor; final long timestamp = 100L; context.setRecordMetadata("", 0, 0L, null, timestamp); final String key = "hey"; final Change value = new Change<>(null, ARBITRARY_LONG); - processor.process(key, value); + harness.processor.process(key, value); assertThat(context.forwarded(), hasSize(1)); final MockProcessorContext.CapturedForward capturedForward = context.forwarded().get(0); @@ -356,16 +349,15 @@ public void suppressShouldEmitWhenOverRecordCapacity() { final Harness harness = new Harness<>(untilTimeLimit(Duration.ofDays(100), maxRecords(1)), String(), Long()); final MockInternalProcessorContext context = harness.context; - final KTableSuppressProcessor processor = harness.processor; final long timestamp = 100L; context.setRecordMetadata("", 0, 0L, null, timestamp); final String key = "hey"; final Change value = new Change<>(null, ARBITRARY_LONG); - processor.process(key, value); + harness.processor.process(key, value); context.setRecordMetadata("", 0, 1L, null, timestamp + 1); - processor.process("dummyKey", value); + harness.processor.process("dummyKey", value); assertThat(context.forwarded(), hasSize(1)); final MockProcessorContext.CapturedForward capturedForward = context.forwarded().get(0); @@ -378,16 +370,15 @@ public void suppressShouldEmitWhenOverByteCapacity() { final Harness harness = new Harness<>(untilTimeLimit(Duration.ofDays(100), maxBytes(60L)), String(), Long()); final MockInternalProcessorContext context = harness.context; - final KTableSuppressProcessor processor = harness.processor; final long timestamp = 100L; context.setRecordMetadata("", 0, 0L, null, timestamp); final String key = "hey"; final Change value = new Change<>(null, ARBITRARY_LONG); - processor.process(key, value); + harness.processor.process(key, value); context.setRecordMetadata("", 0, 1L, null, timestamp + 1); - processor.process("dummyKey", value); + harness.processor.process("dummyKey", value); assertThat(context.forwarded(), hasSize(1)); final MockProcessorContext.CapturedForward capturedForward = context.forwarded().get(0); @@ -400,18 +391,17 @@ public void suppressShouldShutDownWhenOverRecordCapacity() { final Harness harness = new Harness<>(untilTimeLimit(Duration.ofDays(100), maxRecords(1).shutDownWhenFull()), String(), Long()); final MockInternalProcessorContext context = harness.context; - final KTableSuppressProcessor processor = harness.processor; final long timestamp = 100L; context.setRecordMetadata("", 0, 0L, null, timestamp); context.setCurrentNode(new ProcessorNode("testNode")); final String key = "hey"; final Change value = new Change<>(null, ARBITRARY_LONG); - processor.process(key, value); + harness.processor.process(key, value); context.setRecordMetadata("", 0, 1L, null, timestamp); try { - processor.process("dummyKey", value); + harness.processor.process("dummyKey", value); fail("expected an exception"); } catch (final StreamsException e) { assertThat(e.getMessage(), containsString("buffer exceeded its max capacity")); @@ -423,18 +413,17 @@ public void suppressShouldShutDownWhenOverByteCapacity() { final Harness harness = new Harness<>(untilTimeLimit(Duration.ofDays(100), maxBytes(60L).shutDownWhenFull()), String(), Long()); final MockInternalProcessorContext context = harness.context; - final KTableSuppressProcessor processor = harness.processor; final long timestamp = 100L; context.setRecordMetadata("", 0, 0L, null, timestamp); context.setCurrentNode(new ProcessorNode("testNode")); final String key = "hey"; final Change value = new Change<>(null, ARBITRARY_LONG); - processor.process(key, value); + harness.processor.process(key, value); context.setRecordMetadata("", 0, 1L, null, timestamp); try { - processor.process("dummyKey", value); + harness.processor.process("dummyKey", value); fail("expected an exception"); } catch (final StreamsException e) { assertThat(e.getMessage(), containsString("buffer exceeded its max capacity")); diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/SuppressSuite.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/SuppressSuite.java new file mode 100644 index 0000000000000..3aef6d09b0920 --- /dev/null +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/SuppressSuite.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.kstream.internals.suppress; + +import org.apache.kafka.streams.integration.SuppressionDurabilityIntegrationTest; +import org.apache.kafka.streams.integration.SuppressionIntegrationTest; +import org.apache.kafka.streams.kstream.SuppressedTest; +import org.apache.kafka.streams.kstream.internals.SuppressScenarioTest; +import org.apache.kafka.streams.kstream.internals.SuppressTopologyTest; +import org.apache.kafka.streams.state.internals.InMemoryTimeOrderedKeyValueBufferTest; +import org.apache.kafka.streams.state.internals.TimeOrderedKeyValueBufferTest; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +/** + * This suite runs all the tests related to the Suppression feature. + * + * It can be used from an IDE to selectively just run these tests when developing code related to Suppress. + * + * If desired, it can also be added to a Gradle build task, although this isn't strictly necessary, since all + * these tests are already included in the `:streams:test` task. + */ +@RunWith(Suite.class) +@Suite.SuiteClasses({ + KTableSuppressProcessorMetricsTest.class, + KTableSuppressProcessorTest.class, + SuppressScenarioTest.class, + SuppressTopologyTest.class, + SuppressedTest.class, + SuppressionIntegrationTest.class, + SuppressionDurabilityIntegrationTest.class, + InMemoryTimeOrderedKeyValueBufferTest.class, + TimeOrderedKeyValueBufferTest.class +}) +public class SuppressSuite { +} diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MaybeTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MaybeTest.java new file mode 100644 index 0000000000000..7a29e1374fc3a --- /dev/null +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MaybeTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.state.internals; + +import org.junit.Test; + +import java.util.NoSuchElementException; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.fail; + +public class MaybeTest { + @Test + public void shouldReturnDefinedValue() { + assertThat(Maybe.defined(null).getNullableValue(), nullValue()); + assertThat(Maybe.defined("ASDF").getNullableValue(), is("ASDF")); + } + + @Test + public void shouldAnswerIsDefined() { + assertThat(Maybe.defined(null).isDefined(), is(true)); + assertThat(Maybe.defined("ASDF").isDefined(), is(true)); + assertThat(Maybe.undefined().isDefined(), is(false)); + } + + @Test + public void shouldThrowOnGetUndefinedValue() { + final Maybe undefined = Maybe.undefined(); + try { + undefined.getNullableValue(); + fail(); + } catch (final NoSuchElementException e) { + // no assertion necessary + } + } + + @Test + public void shouldUpholdEqualityCorrectness() { + assertThat(Maybe.undefined().equals(Maybe.undefined()), is(true)); + assertThat(Maybe.defined(null).equals(Maybe.defined(null)), is(true)); + assertThat(Maybe.defined("q").equals(Maybe.defined("q")), is(true)); + + assertThat(Maybe.undefined().equals(Maybe.defined(null)), is(false)); + assertThat(Maybe.undefined().equals(Maybe.defined("x")), is(false)); + + assertThat(Maybe.defined(null).equals(Maybe.undefined()), is(false)); + assertThat(Maybe.defined(null).equals(Maybe.defined("x")), is(false)); + + assertThat(Maybe.defined("a").equals(Maybe.undefined()), is(false)); + assertThat(Maybe.defined("a").equals(Maybe.defined(null)), is(false)); + assertThat(Maybe.defined("a").equals(Maybe.defined("b")), is(false)); + } + + @Test + public void shouldUpholdHashCodeCorrectness() { + // This specifies the current implementation, which is simpler to write than an exhaustive test. + // As long as this implementation doesn't change, then the equals/hashcode contract is upheld. + + assertThat(Maybe.undefined().hashCode(), is(-1)); + assertThat(Maybe.defined(null).hashCode(), is(0)); + assertThat(Maybe.defined("a").hashCode(), is("a".hashCode())); + } +} diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/TimeOrderedKeyValueBufferTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/TimeOrderedKeyValueBufferTest.java index 6ae36d4d911e3..941832bff3a85 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/TimeOrderedKeyValueBufferTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/TimeOrderedKeyValueBufferTest.java @@ -23,12 +23,16 @@ import org.apache.kafka.common.header.internals.RecordHeaders; import org.apache.kafka.common.record.TimestampType; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.Serializer; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.kstream.internals.Change; +import org.apache.kafka.streams.kstream.internals.FullChangeSerde; import org.apache.kafka.streams.processor.TaskId; import org.apache.kafka.streams.processor.internals.ProcessorRecordContext; import org.apache.kafka.streams.processor.internals.RecordBatchingStateRestoreCallback; +import org.apache.kafka.streams.state.ValueAndTimestamp; import org.apache.kafka.streams.state.internals.TimeOrderedKeyValueBuffer.Eviction; import org.apache.kafka.test.MockInternalProcessorContext; import org.apache.kafka.test.MockInternalProcessorContext.MockRecordCollector; @@ -57,8 +61,8 @@ @RunWith(Parameterized.class) public class TimeOrderedKeyValueBufferTest> { - private static final RecordHeaders V_1_CHANGELOG_HEADERS = - new RecordHeaders(new Header[] {new RecordHeader("v", new byte[] {(byte) 1})}); + private static final RecordHeaders V_2_CHANGELOG_HEADERS = + new RecordHeaders(new Header[] {new RecordHeader("v", new byte[] {(byte) 2})}); private static final String APP_ID = "test-app"; private final Function bufferSupplier; @@ -129,10 +133,7 @@ public void shouldRejectNullValues() { final MockInternalProcessorContext context = makeContext(); buffer.init(context, buffer); try { - buffer.put(0, "asdf", - null, - getContext(0) - ); + buffer.put(0, "asdf", null, getContext(0)); fail("expected an exception"); } catch (final NullPointerException expected) { // expected @@ -163,7 +164,9 @@ public void shouldRespectEvictionPredicate() { final List> evicted = new LinkedList<>(); buffer.evictWhile(() -> buffer.numRecords() > 1, evicted::add); assertThat(buffer.numRecords(), is(1)); - assertThat(evicted, is(singletonList(new Eviction<>("asdf", "eyt", getContext(0L))))); + assertThat(evicted, is(singletonList( + new Eviction<>("asdf", new Change<>("eyt", null), getContext(0L)) + ))); cleanup(context, buffer); } @@ -187,11 +190,11 @@ public void shouldTrackSize() { final MockInternalProcessorContext context = makeContext(); buffer.init(context, buffer); putRecord(buffer, context, 0L, 0L, "asdf", "23roni"); - assertThat(buffer.bufferSize(), is(43L)); + assertThat(buffer.bufferSize(), is(51L)); putRecord(buffer, context, 1L, 0L, "asdf", "3l"); - assertThat(buffer.bufferSize(), is(39L)); + assertThat(buffer.bufferSize(), is(47L)); putRecord(buffer, context, 0L, 0L, "zxcv", "qfowin"); - assertThat(buffer.bufferSize(), is(82L)); + assertThat(buffer.bufferSize(), is(98L)); cleanup(context, buffer); } @@ -215,12 +218,12 @@ public void shouldEvictOldestAndUpdateSizeAndCountAndMinTimestamp() { putRecord(buffer, context, 1L, 0L, "zxcv", "o23i4"); assertThat(buffer.numRecords(), is(1)); - assertThat(buffer.bufferSize(), is(42L)); + assertThat(buffer.bufferSize(), is(50L)); assertThat(buffer.minTimestamp(), is(1L)); putRecord(buffer, context, 0L, 0L, "asdf", "3ng"); assertThat(buffer.numRecords(), is(2)); - assertThat(buffer.bufferSize(), is(82L)); + assertThat(buffer.bufferSize(), is(98L)); assertThat(buffer.minTimestamp(), is(0L)); final AtomicInteger callbackCount = new AtomicInteger(0); @@ -229,14 +232,14 @@ public void shouldEvictOldestAndUpdateSizeAndCountAndMinTimestamp() { case 1: { assertThat(kv.key(), is("asdf")); assertThat(buffer.numRecords(), is(2)); - assertThat(buffer.bufferSize(), is(82L)); + assertThat(buffer.bufferSize(), is(98L)); assertThat(buffer.minTimestamp(), is(0L)); break; } case 2: { assertThat(kv.key(), is("zxcv")); assertThat(buffer.numRecords(), is(1)); - assertThat(buffer.bufferSize(), is(42L)); + assertThat(buffer.bufferSize(), is(50L)); assertThat(buffer.minTimestamp(), is(1L)); break; } @@ -253,6 +256,29 @@ public void shouldEvictOldestAndUpdateSizeAndCountAndMinTimestamp() { cleanup(context, buffer); } + @Test + public void shouldReturnUndefinedOnPriorValueForNotBufferedKey() { + final TimeOrderedKeyValueBuffer buffer = bufferSupplier.apply(testName); + final MockInternalProcessorContext context = makeContext(); + buffer.init(context, buffer); + + assertThat(buffer.priorValueForBuffered("ASDF"), is(Maybe.undefined())); + } + + @Test + public void shouldReturnPriorValueForBufferedKey() { + final TimeOrderedKeyValueBuffer buffer = bufferSupplier.apply(testName); + final MockInternalProcessorContext context = makeContext(); + buffer.init(context, buffer); + + final ProcessorRecordContext recordContext = getContext(0L); + context.setRecordContext(recordContext); + buffer.put(1L, "A", new Change<>("new-value", "old-value"), recordContext); + buffer.put(1L, "B", new Change<>("new-value", null), recordContext); + assertThat(buffer.priorValueForBuffered("A"), is(Maybe.defined(ValueAndTimestamp.make("old-value", -1)))); + assertThat(buffer.priorValueForBuffered("B"), is(Maybe.defined(null))); + } + @Test public void shouldFlush() { final TimeOrderedKeyValueBuffer buffer = bufferSupplier.apply(testName); @@ -272,19 +298,19 @@ public void shouldFlush() { // which we can't compare for equality using ProducerRecord. // As a workaround, I'm deserializing them and shoving them in a KeyValue, just for ease of testing. - final List>> collected = + final List>> collected = ((MockRecordCollector) context.recordCollector()) .collected() .stream() .map(pr -> { - final KeyValue niceValue; + final KeyValue niceValue; if (pr.value() == null) { niceValue = null; } else { - final byte[] timestampAndValue = pr.value(); - final ByteBuffer wrap = ByteBuffer.wrap(timestampAndValue); - final long timestamp = wrap.getLong(); - final ContextualRecord contextualRecord = ContextualRecord.deserialize(wrap); + final byte[] serializedValue = pr.value(); + final ByteBuffer valueBuffer = ByteBuffer.wrap(serializedValue); + final BufferValue contextualRecord = BufferValue.deserialize(valueBuffer); + final long timestamp = valueBuffer.getLong(); niceValue = new KeyValue<>(timestamp, contextualRecord); } @@ -309,15 +335,15 @@ public void shouldFlush() { 0, null, "zxcv", - new KeyValue<>(1L, getRecord("3gon4i", 1)), - V_1_CHANGELOG_HEADERS + new KeyValue<>(1L, getBufferValue("3gon4i", 1)), + V_2_CHANGELOG_HEADERS ), new ProducerRecord<>(APP_ID + "-" + testName + "-changelog", 0, null, "asdf", - new KeyValue<>(2L, getRecord("2093j", 0)), - V_1_CHANGELOG_HEADERS + new KeyValue<>(2L, getBufferValue("2093j", 0)), + V_2_CHANGELOG_HEADERS ) ))); @@ -335,6 +361,12 @@ public void shouldRestoreOldFormat() { context.setRecordContext(new ProcessorRecordContext(0, 0, 0, "", null)); + final Serializer> serializer = FullChangeSerde.castOrWrap(Serdes.String()).serializer(); + + final byte[] todeleteValue = serializer.serialize(null, new Change<>("doomed", null)); + final byte[] asdfValue = serializer.serialize(null, new Change<>("qwer", null)); + final byte[] zxcvValue1 = serializer.serialize(null, new Change<>("eo4im", "previous")); + final byte[] zxcvValue2 = serializer.serialize(null, new Change<>("next", "eo4im")); stateRestoreCallback.restoreBatch(asList( new ConsumerRecord<>("changelog-topic", 0, @@ -345,7 +377,7 @@ public void shouldRestoreOldFormat() { -1, -1, "todelete".getBytes(UTF_8), - ByteBuffer.allocate(Long.BYTES + 6).putLong(0L).put("doomed".getBytes(UTF_8)).array()), + ByteBuffer.allocate(Long.BYTES + todeleteValue.length).putLong(0L).put(todeleteValue).array()), new ConsumerRecord<>("changelog-topic", 0, 1, @@ -355,7 +387,7 @@ public void shouldRestoreOldFormat() { -1, -1, "asdf".getBytes(UTF_8), - ByteBuffer.allocate(Long.BYTES + 4).putLong(2L).put("qwer".getBytes(UTF_8)).array()), + ByteBuffer.allocate(Long.BYTES + asdfValue.length).putLong(2L).put(asdfValue).array()), new ConsumerRecord<>("changelog-topic", 0, 2, @@ -365,12 +397,22 @@ public void shouldRestoreOldFormat() { -1, -1, "zxcv".getBytes(UTF_8), - ByteBuffer.allocate(Long.BYTES + 5).putLong(1L).put("3o4im".getBytes(UTF_8)).array()) + ByteBuffer.allocate(Long.BYTES + zxcvValue1.length).putLong(1L).put(zxcvValue1).array()), + new ConsumerRecord<>("changelog-topic", + 0, + 3, + 3, + TimestampType.CREATE_TIME, + -1, + -1, + -1, + "zxcv".getBytes(UTF_8), + ByteBuffer.allocate(Long.BYTES + zxcvValue2.length).putLong(1L).put(zxcvValue2).array()) )); assertThat(buffer.numRecords(), is(3)); assertThat(buffer.minTimestamp(), is(0L)); - assertThat(buffer.bufferSize(), is(160L)); + assertThat(buffer.bufferSize(), is(196L)); stateRestoreCallback.restoreBatch(singletonList( new ConsumerRecord<>("changelog-topic", @@ -387,7 +429,11 @@ public void shouldRestoreOldFormat() { assertThat(buffer.numRecords(), is(2)); assertThat(buffer.minTimestamp(), is(1L)); - assertThat(buffer.bufferSize(), is(103L)); + assertThat(buffer.bufferSize(), is(131L)); + + assertThat(buffer.priorValueForBuffered("todelete"), is(Maybe.undefined())); + assertThat(buffer.priorValueForBuffered("asdf"), is(Maybe.defined(null))); + assertThat(buffer.priorValueForBuffered("zxcv"), is(Maybe.defined(ValueAndTimestamp.make("previous", -1)))); // flush the buffer into a list in buffer order so we can make assertions about the contents. @@ -405,11 +451,11 @@ public void shouldRestoreOldFormat() { assertThat(evicted, is(asList( new Eviction<>( "zxcv", - "3o4im", - new ProcessorRecordContext(2L, 2, 0, "changelog-topic", new RecordHeaders())), + new Change<>("next", "eo4im"), + new ProcessorRecordContext(3L, 3, 0, "changelog-topic", new RecordHeaders())), new Eviction<>( "asdf", - "qwer", + new Change<>("qwer", null), new ProcessorRecordContext(1L, 1, 0, "changelog-topic", new RecordHeaders())) ))); @@ -417,7 +463,7 @@ public void shouldRestoreOldFormat() { } @Test - public void shouldRestoreNewFormat() { + public void shouldRestoreV1Format() { final TimeOrderedKeyValueBuffer buffer = bufferSupplier.apply(testName); final MockInternalProcessorContext context = makeContext(); buffer.init(context, buffer); @@ -429,9 +475,18 @@ public void shouldRestoreNewFormat() { final RecordHeaders v1FlagHeaders = new RecordHeaders(new Header[] {new RecordHeader("v", new byte[] {(byte) 1})}); - final byte[] todeleteValue = getRecord("doomed", 0).serialize(); - final byte[] asdfValue = getRecord("qwer", 1).serialize(); - final byte[] zxcvValue = getRecord("3o4im", 2).serialize(); + final byte[] todeleteValue = getContextualRecord("doomed", 0).serialize(0).array(); + final byte[] asdfValue = getContextualRecord("qwer", 1).serialize(0).array(); + final FullChangeSerde fullChangeSerde = FullChangeSerde.castOrWrap(Serdes.String()); + final byte[] zxcvValue1 = new ContextualRecord( + fullChangeSerde.serializer().serialize(null, new Change<>("3o4im", "previous")), + getContext(2L) + ).serialize(0).array(); + final FullChangeSerde fullChangeSerde1 = FullChangeSerde.castOrWrap(Serdes.String()); + final byte[] zxcvValue2 = new ContextualRecord( + fullChangeSerde1.serializer().serialize(null, new Change<>("next", "3o4im")), + getContext(3L) + ).serialize(0).array(); stateRestoreCallback.restoreBatch(asList( new ConsumerRecord<>("changelog-topic", 0, @@ -464,13 +519,24 @@ public void shouldRestoreNewFormat() { -1, -1, "zxcv".getBytes(UTF_8), - ByteBuffer.allocate(Long.BYTES + zxcvValue.length).putLong(1L).put(zxcvValue).array(), + ByteBuffer.allocate(Long.BYTES + zxcvValue1.length).putLong(1L).put(zxcvValue1).array(), + v1FlagHeaders), + new ConsumerRecord<>("changelog-topic", + 0, + 3, + 100, + TimestampType.CREATE_TIME, + -1L, + -1, + -1, + "zxcv".getBytes(UTF_8), + ByteBuffer.allocate(Long.BYTES + zxcvValue2.length).putLong(1L).put(zxcvValue2).array(), v1FlagHeaders) )); assertThat(buffer.numRecords(), is(3)); assertThat(buffer.minTimestamp(), is(0L)); - assertThat(buffer.bufferSize(), is(130L)); + assertThat(buffer.bufferSize(), is(166L)); stateRestoreCallback.restoreBatch(singletonList( new ConsumerRecord<>("changelog-topic", @@ -487,7 +553,11 @@ public void shouldRestoreNewFormat() { assertThat(buffer.numRecords(), is(2)); assertThat(buffer.minTimestamp(), is(1L)); - assertThat(buffer.bufferSize(), is(83L)); + assertThat(buffer.bufferSize(), is(111L)); + + assertThat(buffer.priorValueForBuffered("todelete"), is(Maybe.undefined())); + assertThat(buffer.priorValueForBuffered("asdf"), is(Maybe.defined(null))); + assertThat(buffer.priorValueForBuffered("zxcv"), is(Maybe.defined(ValueAndTimestamp.make("previous", -1)))); // flush the buffer into a list in buffer order so we can make assertions about the contents. @@ -505,11 +575,143 @@ public void shouldRestoreNewFormat() { assertThat(evicted, is(asList( new Eviction<>( "zxcv", - "3o4im", - getContext(2L)), + new Change<>("next", "3o4im"), + getContext(3L)), new Eviction<>( "asdf", - "qwer", + new Change<>("qwer", null), + getContext(1L) + )))); + + cleanup(context, buffer); + } + + @Test + public void shouldRestoreV2Format() { + final TimeOrderedKeyValueBuffer buffer = bufferSupplier.apply(testName); + final MockInternalProcessorContext context = makeContext(); + buffer.init(context, buffer); + + final RecordBatchingStateRestoreCallback stateRestoreCallback = + (RecordBatchingStateRestoreCallback) context.stateRestoreCallback(testName); + + context.setRecordContext(new ProcessorRecordContext(0, 0, 0, "", null)); + + final RecordHeaders v2FlagHeaders = new RecordHeaders(new Header[] {new RecordHeader("v", new byte[] {(byte) 2})}); + + final byte[] todeleteValue = getBufferValue("doomed", 0).serialize(0).array(); + final byte[] asdfValue = getBufferValue("qwer", 1).serialize(0).array(); + final FullChangeSerde fullChangeSerde = FullChangeSerde.castOrWrap(Serdes.String()); + final byte[] zxcvValue1 = + new BufferValue( + new ContextualRecord( + fullChangeSerde.serializer().serialize(null, new Change<>("3o4im", "IGNORED")), + getContext(2L) + ), + Serdes.String().serializer().serialize(null, "previous") + ).serialize(0).array(); + final FullChangeSerde fullChangeSerde1 = FullChangeSerde.castOrWrap(Serdes.String()); + final byte[] zxcvValue2 = + new BufferValue( + new ContextualRecord( + fullChangeSerde1.serializer().serialize(null, new Change<>("next", "3o4im")), + getContext(3L) + ), + Serdes.String().serializer().serialize(null, "previous") + ).serialize(0).array(); + stateRestoreCallback.restoreBatch(asList( + new ConsumerRecord<>("changelog-topic", + 0, + 0, + 999, + TimestampType.CREATE_TIME, + -1L, + -1, + -1, + "todelete".getBytes(UTF_8), + ByteBuffer.allocate(Long.BYTES + todeleteValue.length).put(todeleteValue).putLong(0L).array(), + v2FlagHeaders), + new ConsumerRecord<>("changelog-topic", + 0, + 1, + 9999, + TimestampType.CREATE_TIME, + -1L, + -1, + -1, + "asdf".getBytes(UTF_8), + ByteBuffer.allocate(Long.BYTES + asdfValue.length).put(asdfValue).putLong(2L).array(), + v2FlagHeaders), + new ConsumerRecord<>("changelog-topic", + 0, + 2, + 99, + TimestampType.CREATE_TIME, + -1L, + -1, + -1, + "zxcv".getBytes(UTF_8), + ByteBuffer.allocate(Long.BYTES + zxcvValue1.length).put(zxcvValue1).putLong(1L).array(), + v2FlagHeaders), + new ConsumerRecord<>("changelog-topic", + 0, + 2, + 100, + TimestampType.CREATE_TIME, + -1L, + -1, + -1, + "zxcv".getBytes(UTF_8), + ByteBuffer.allocate(Long.BYTES + zxcvValue2.length).put(zxcvValue2).putLong(1L).array(), + v2FlagHeaders) + )); + + assertThat(buffer.numRecords(), is(3)); + assertThat(buffer.minTimestamp(), is(0L)); + assertThat(buffer.bufferSize(), is(166L)); + + stateRestoreCallback.restoreBatch(singletonList( + new ConsumerRecord<>("changelog-topic", + 0, + 3, + 3, + TimestampType.CREATE_TIME, + -1L, + -1, + -1, + "todelete".getBytes(UTF_8), + null) + )); + + assertThat(buffer.numRecords(), is(2)); + assertThat(buffer.minTimestamp(), is(1L)); + assertThat(buffer.bufferSize(), is(111L)); + + assertThat(buffer.priorValueForBuffered("todelete"), is(Maybe.undefined())); + assertThat(buffer.priorValueForBuffered("asdf"), is(Maybe.defined(null))); + assertThat(buffer.priorValueForBuffered("zxcv"), is(Maybe.defined(ValueAndTimestamp.make("previous", -1)))); + + // flush the buffer into a list in buffer order so we can make assertions about the contents. + + final List> evicted = new LinkedList<>(); + buffer.evictWhile(() -> true, evicted::add); + + // Several things to note: + // * The buffered records are ordered according to their buffer time (serialized in the value of the changelog) + // * The record timestamps are properly restored, and not conflated with the record's buffer time. + // * The keys and values are properly restored + // * The record topic is set to the original input topic, *not* the changelog topic + // * The record offset preserves the original input record's offset, *not* the offset of the changelog record + + + assertThat(evicted, is(asList( + new Eviction<>( + "zxcv", + new Change<>("next", "3o4im"), + getContext(3L)), + new Eviction<>( + "asdf", + new Change<>("qwer", null), getContext(1L) )))); @@ -529,7 +731,7 @@ public void shouldNotRestoreUnrecognizedVersionRecord() { final RecordHeaders unknownFlagHeaders = new RecordHeaders(new Header[] {new RecordHeader("v", new byte[] {(byte) -1})}); - final byte[] todeleteValue = getRecord("doomed", 0).serialize(); + final byte[] todeleteValue = getBufferValue("doomed", 0).serialize(0).array(); try { stateRestoreCallback.restoreBatch(singletonList( new ConsumerRecord<>("changelog-topic", @@ -560,12 +762,18 @@ private static void putRecord(final TimeOrderedKeyValueBuffer bu final String value) { final ProcessorRecordContext recordContext = getContext(recordTimestamp); context.setRecordContext(recordContext); - buffer.put(streamTime, key, value, recordContext); + buffer.put(streamTime, key, new Change<>(value, null), recordContext); + } + + private static BufferValue getBufferValue(final String value, final long timestamp) { + final ContextualRecord contextualRecord = getContextualRecord(value, timestamp); + return new BufferValue(contextualRecord, null); } - private static ContextualRecord getRecord(final String value, final long timestamp) { + private static ContextualRecord getContextualRecord(final String value, final long timestamp) { + final FullChangeSerde fullChangeSerde = FullChangeSerde.castOrWrap(Serdes.String()); return new ContextualRecord( - value.getBytes(UTF_8), + fullChangeSerde.serializer().serialize(null, new Change<>(value, null)), getContext(timestamp) ); } From 579567559915e750d305d51a10d9c6797b6eb2aa Mon Sep 17 00:00:00 2001 From: Lifei Chen Date: Fri, 31 May 2019 07:46:41 +0800 Subject: [PATCH 0314/1071] MINOR:Replace duplicated code with common function in utils (#6819) Reviewers: Ivan Yurchenko , Matthias J. Sax --- .../java/org/apache/kafka/common/utils/SystemTime.java | 7 +------ .../streams/processor/internals/InternalTopicManager.java | 7 +------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/utils/SystemTime.java b/clients/src/main/java/org/apache/kafka/common/utils/SystemTime.java index 9ef096f27d227..31919a22155b6 100644 --- a/clients/src/main/java/org/apache/kafka/common/utils/SystemTime.java +++ b/clients/src/main/java/org/apache/kafka/common/utils/SystemTime.java @@ -38,12 +38,7 @@ public long nanoseconds() { @Override public void sleep(long ms) { - try { - Thread.sleep(ms); - } catch (InterruptedException e) { - // just wake up early - Thread.currentThread().interrupt(); - } + Utils.sleep(ms); } @Override diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java index f9fe042348200..3cb06f69e0e05 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java @@ -155,12 +155,7 @@ public void makeReady(final Map topics) { if (!topicsNotReady.isEmpty()) { log.info("Topics {} can not be made ready with {} retries left", topicsNotReady, retries); - try { - Thread.sleep(retryBackOffMs); - } catch (final InterruptedException e) { - // this is okay, we just wake up early - Thread.currentThread().interrupt(); - } + Utils.sleep(retryBackOffMs); remainingRetries--; } From 78c55c8d66f5570d975caa53a9751b126ca10538 Mon Sep 17 00:00:00 2001 From: Florian Hussonnois Date: Fri, 31 May 2019 15:29:43 +0200 Subject: [PATCH 0315/1071] KAFKA-6958: Overload KStream methods to allow to name operation name using the new Named class (#6411) Sub-task required to allow to define custom processor names with KStreams DSL(KIP-307) : - overload methods for stateless operations to accept a Named parameter (filter, filterNot, map, mapValues, foreach, peek, branch, transform, transformValue, flatTransform) - overload process method to accept a Named parameter - overload join/leftJoin/outerJoin methods Reviewers: John Roesler , Boyang Chen , Bill Bejeck --- .../apache/kafka/streams/kstream/KStream.java | 1229 +++++++++++++++-- .../kstream/internals/KStreamImpl.java | 374 +++-- .../kstream/internals/NamedInternal.java | 19 +- .../kafka/streams/StreamsBuilderTest.java | 245 +++- .../kstream/RepartitionTopicNamingTest.java | 66 +- 5 files changed, 1694 insertions(+), 239 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java index cd64f757bde11..8313addb6ff68 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java @@ -66,6 +66,19 @@ public interface KStream { */ KStream filter(final Predicate predicate); + /** + * Create a new {@code KStream} that consists of all records of this stream which satisfy the given predicate. + * All records that do not satisfy the predicate are dropped. + * This is a stateless record-by-record operation. + * + * @param predicate a filter {@link Predicate} that is applied to each record + * @param named a {@link Named} config used to name the processor in the topology + * @return a {@code KStream} that contains only those records that satisfy the given predicate + * @see #filterNot(Predicate) + */ + KStream filter(final Predicate predicate, final Named named); + + /** * Create a new {@code KStream} that consists all records of this stream which do not satisfy the given * predicate. @@ -78,6 +91,19 @@ public interface KStream { */ KStream filterNot(final Predicate predicate); + /** + * Create a new {@code KStream} that consists all records of this stream which do not satisfy the given + * predicate. + * All records that do satisfy the predicate are dropped. + * This is a stateless record-by-record operation. + * + * @param predicate a filter {@link Predicate} that is applied to each record + * @param named a {@link Named} config used to name the processor in the topology + * @return a {@code KStream} that contains only those records that do not satisfy the given predicate + * @see #filter(Predicate) + */ + KStream filterNot(final Predicate predicate, final Named named); + /** * Set a new key (with possibly new type) for each input record. * The provided {@link KeyValueMapper} is applied to each input record and computes a new key for it. @@ -110,6 +136,40 @@ public interface KStream { */ KStream selectKey(final KeyValueMapper mapper); + /** + * Set a new key (with possibly new type) for each input record. + * The provided {@link KeyValueMapper} is applied to each input record and computes a new key for it. + * Thus, an input record {@code } can be transformed into an output record {@code }. + * This is a stateless record-by-record operation. + *

    + * For example, you can use this transformation to set a key for a key-less input record {@code } by + * extracting a key from the value within your {@link KeyValueMapper}. The example below computes the new key as the + * length of the value string. + * 

    {@code
+     * KStream keyLessStream = builder.stream("key-less-topic");
+     * KStream keyedStream = keyLessStream.selectKey(new KeyValueMapper {
+     *     Integer apply(Byte[] key, String value) {
+     *         return value.length();
+     *     }
+     * });
+     * }
    + * Setting a new key might result in an internal data redistribution if a key based operator (like an aggregation or + * join) is applied to the result {@code KStream}. + * + * @param mapper a {@link KeyValueMapper} that computes a new key for each record + * @param named a {@link Named} config used to name the processor in the topology + * @param the new key type of the result stream + * @return a {@code KStream} that contains records with new key (possibly of different type) and unmodified value + * @see #map(KeyValueMapper) + * @see #flatMap(KeyValueMapper) + * @see #mapValues(ValueMapper) + * @see #mapValues(ValueMapperWithKey) + * @see #flatMapValues(ValueMapper) + * @see #flatMapValues(ValueMapperWithKey) + */ + KStream selectKey(final KeyValueMapper mapper, + final Named named); + /** * Transform each record of the input stream into a new record in the output stream (both key and value type can be * altered arbitrarily). @@ -148,6 +208,46 @@ public interface KStream { */ KStream map(final KeyValueMapper> mapper); + /** + * Transform each record of the input stream into a new record in the output stream (both key and value type can be + * altered arbitrarily). + * The provided {@link KeyValueMapper} is applied to each input record and computes a new output record. + * Thus, an input record {@code } can be transformed into an output record {@code }. + * This is a stateless record-by-record operation (cf. {@link #transform(TransformerSupplier, String...)} for + * stateful record transformation). + *

    + * The example below normalizes the String key to upper-case letters and counts the number of token of the value string. + * 

    {@code
+     * KStream inputStream = builder.stream("topic");
+     * KStream outputStream = inputStream.map(new KeyValueMapper> {
+     *     KeyValue apply(String key, String value) {
+     *         return new KeyValue<>(key.toUpperCase(), value.split(" ").length);
+     *     }
+     * });
+     * }
    + * The provided {@link KeyValueMapper} must return a {@link KeyValue} type and must not return {@code null}. + *

    + * Mapping records might result in an internal data redistribution if a key based operator (like an aggregation or + * join) is applied to the result {@code KStream}. (cf. {@link #mapValues(ValueMapper)}) + * + * @param mapper a {@link KeyValueMapper} that computes a new output record + * @param named a {@link Named} config used to name the processor in the topology + * @param the key type of the result stream + * @param the value type of the result stream + * @return a {@code KStream} that contains records with new key and value (possibly both of different type) + * @see #selectKey(KeyValueMapper) + * @see #flatMap(KeyValueMapper) + * @see #mapValues(ValueMapper) + * @see #mapValues(ValueMapperWithKey) + * @see #flatMapValues(ValueMapper) + * @see #flatMapValues(ValueMapperWithKey) + * @see #transform(TransformerSupplier, String...) + * @see #transformValues(ValueTransformerSupplier, String...) + * @see #transformValues(ValueTransformerWithKeySupplier, String...) + */ + KStream map(final KeyValueMapper> mapper, + final Named named); + /** * Transform the value of each input record into a new value (with possible new type) of the output record. * The provided {@link ValueMapper} is applied to each input record value and computes a new value for it. @@ -182,6 +282,43 @@ public interface KStream { */ KStream mapValues(final ValueMapper mapper); + + /** + * Transform the value of each input record into a new value (with possible new type) of the output record. + * The provided {@link ValueMapper} is applied to each input record value and computes a new value for it. + * Thus, an input record {@code } can be transformed into an output record {@code }. + * This is a stateless record-by-record operation (cf. + * {@link #transformValues(ValueTransformerSupplier, String...)} for stateful value transformation). + *

    + * The example below counts the number of token of the value string. + * 

    {@code
+     * KStream inputStream = builder.stream("topic");
+     * KStream outputStream = inputStream.mapValues(new ValueMapper {
+     *     Integer apply(String value) {
+     *         return value.split(" ").length;
+     *     }
+     * });
+     * }
    + * Setting a new value preserves data co-location with respect to the key. + * Thus, no internal data redistribution is required if a key based operator (like an aggregation or join) + * is applied to the result {@code KStream}. (cf. {@link #map(KeyValueMapper)}) + * + * @param mapper a {@link ValueMapper} that computes a new output value + * @param named a {@link Named} config used to name the processor in the topology + * @param the value type of the result stream + * @return a {@code KStream} that contains records with unmodified key and new values (possibly of different type) + * @see #selectKey(KeyValueMapper) + * @see #map(KeyValueMapper) + * @see #flatMap(KeyValueMapper) + * @see #flatMapValues(ValueMapper) + * @see #flatMapValues(ValueMapperWithKey) + * @see #transform(TransformerSupplier, String...) + * @see #transformValues(ValueTransformerSupplier, String...) + * @see #transformValues(ValueTransformerWithKeySupplier, String...) + */ + KStream mapValues(final ValueMapper mapper, + final Named named); + /** * Transform the value of each input record into a new value (with possible new type) of the output record. * The provided {@link ValueMapperWithKey} is applied to each input record value and computes a new value for it. @@ -217,6 +354,43 @@ public interface KStream { */ KStream mapValues(final ValueMapperWithKey mapper); + /** + * Transform the value of each input record into a new value (with possible new type) of the output record. + * The provided {@link ValueMapperWithKey} is applied to each input record value and computes a new value for it. + * Thus, an input record {@code } can be transformed into an output record {@code }. + * This is a stateless record-by-record operation (cf. + * {@link #transformValues(ValueTransformerWithKeySupplier, String...)} for stateful value transformation). + *

    + * The example below counts the number of tokens of key and value strings. + * 

    {@code
+     * KStream inputStream = builder.stream("topic");
+     * KStream outputStream = inputStream.mapValues(new ValueMapperWithKey {
+     *     Integer apply(String readOnlyKey, String value) {
+     *         return readOnlyKey.split(" ").length + value.split(" ").length;
+     *     }
+     * });
+     * }
    + * Note that the key is read-only and should not be modified, as this can lead to corrupt partitioning. + * So, setting a new value preserves data co-location with respect to the key. + * Thus, no internal data redistribution is required if a key based operator (like an aggregation or join) + * is applied to the result {@code KStream}. (cf. {@link #map(KeyValueMapper)}) + * + * @param mapper a {@link ValueMapperWithKey} that computes a new output value + * @param named a {@link Named} config used to name the processor in the topology + * @param the value type of the result stream + * @return a {@code KStream} that contains records with unmodified key and new values (possibly of different type) + * @see #selectKey(KeyValueMapper) + * @see #map(KeyValueMapper) + * @see #flatMap(KeyValueMapper) + * @see #flatMapValues(ValueMapper) + * @see #flatMapValues(ValueMapperWithKey) + * @see #transform(TransformerSupplier, String...) + * @see #transformValues(ValueTransformerSupplier, String...) + * @see #transformValues(ValueTransformerWithKeySupplier, String...) + */ + KStream mapValues(final ValueMapperWithKey mapper, + final Named named); + /** * Transform each record of the input stream into zero or more records in the output stream (both key and value type * can be altered arbitrarily). @@ -268,6 +442,59 @@ public interface KStream { */ KStream flatMap(final KeyValueMapper>> mapper); + /** + * Transform each record of the input stream into zero or more records in the output stream (both key and value type + * can be altered arbitrarily). + * The provided {@link KeyValueMapper} is applied to each input record and computes zero or more output records. + * Thus, an input record {@code } can be transformed into output records {@code , , ...}. + * This is a stateless record-by-record operation (cf. {@link #transform(TransformerSupplier, String...)} for + * stateful record transformation). + *

    + * The example below splits input records {@code } containing sentences as values into their words + * and emit a record {@code } for each word. + * 

    {@code
+     * KStream inputStream = builder.stream("topic");
+     * KStream outputStream = inputStream.flatMap(
+     *     new KeyValueMapper>> {
+     *         Iterable> apply(byte[] key, String value) {
+     *             String[] tokens = value.split(" ");
+     *             List> result = new ArrayList<>(tokens.length);
+     *
+     *             for(String token : tokens) {
+     *                 result.add(new KeyValue<>(token, 1));
+     *             }
+     *
+     *             return result;
+     *         }
+     *     });
+     * }
    + * The provided {@link KeyValueMapper} must return an {@link Iterable} (e.g., any {@link java.util.Collection} type) + * and the return value must not be {@code null}. + *

    + * Flat-mapping records might result in an internal data redistribution if a key based operator (like an aggregation + * or join) is applied to the result {@code KStream}. (cf. {@link #flatMapValues(ValueMapper)}) + * + * @param mapper a {@link KeyValueMapper} that computes the new output records + * @param named a {@link Named} config used to name the processor in the topology + * @param the key type of the result stream + * @param the value type of the result stream + * @return a {@code KStream} that contains more or less records with new key and value (possibly of different type) + * @see #selectKey(KeyValueMapper) + * @see #map(KeyValueMapper) + * @see #mapValues(ValueMapper) + * @see #mapValues(ValueMapperWithKey) + * @see #flatMapValues(ValueMapper) + * @see #flatMapValues(ValueMapperWithKey) + * @see #transform(TransformerSupplier, String...) + * @see #flatTransform(TransformerSupplier, String...) + * @see #transformValues(ValueTransformerSupplier, String...) + * @see #transformValues(ValueTransformerWithKeySupplier, String...) + * @see #flatTransformValues(ValueTransformerSupplier, String...) + * @see #flatTransformValues(ValueTransformerWithKeySupplier, String...) + */ + KStream flatMap(final KeyValueMapper>> mapper, + final Named named); + /** * Create a new {@code KStream} by transforming the value of each record in this stream into zero or more values * with the same key in the new stream. @@ -311,6 +538,50 @@ public interface KStream { */ KStream flatMapValues(final ValueMapper> mapper); + /** + * Create a new {@code KStream} by transforming the value of each record in this stream into zero or more values + * with the same key in the new stream. + * Transform the value of each input record into zero or more records with the same (unmodified) key in the output + * stream (value type can be altered arbitrarily). + * The provided {@link ValueMapper} is applied to each input record and computes zero or more output values. + * Thus, an input record {@code } can be transformed into output records {@code , , ...}. + * This is a stateless record-by-record operation (cf. {@link #transformValues(ValueTransformerSupplier, String...)} + * for stateful value transformation). + *

    + * The example below splits input records {@code } containing sentences as values into their words. + * 

    {@code
+     * KStream inputStream = builder.stream("topic");
+     * KStream outputStream = inputStream.flatMapValues(new ValueMapper> {
+     *     Iterable apply(String value) {
+     *         return Arrays.asList(value.split(" "));
+     *     }
+     * });
+     * }
    + * The provided {@link ValueMapper} must return an {@link Iterable} (e.g., any {@link java.util.Collection} type) + * and the return value must not be {@code null}. + *

    + * Splitting a record into multiple records with the same key preserves data co-location with respect to the key. + * Thus, no internal data redistribution is required if a key based operator (like an aggregation or join) + * is applied to the result {@code KStream}. (cf. {@link #flatMap(KeyValueMapper)}) + * + * @param mapper a {@link ValueMapper} the computes the new output values + * @param named a {@link Named} config used to name the processor in the topology + * @param the value type of the result stream + * @return a {@code KStream} that contains more or less records with unmodified keys and new values of different type + * @see #selectKey(KeyValueMapper) + * @see #map(KeyValueMapper) + * @see #flatMap(KeyValueMapper) + * @see #mapValues(ValueMapper) + * @see #mapValues(ValueMapperWithKey) + * @see #transform(TransformerSupplier, String...) + * @see #flatTransform(TransformerSupplier, String...) + * @see #transformValues(ValueTransformerSupplier, String...) + * @see #transformValues(ValueTransformerWithKeySupplier, String...) + * @see #flatTransformValues(ValueTransformerSupplier, String...) + * @see #flatTransformValues(ValueTransformerWithKeySupplier, String...) + */ + KStream flatMapValues(final ValueMapper> mapper, + final Named named); /** * Create a new {@code KStream} by transforming the value of each record in this stream into zero or more values * with the same key in the new stream. @@ -361,13 +632,63 @@ public interface KStream { KStream flatMapValues(final ValueMapperWithKey> mapper); /** - * Print the records of this KStream using the options provided by {@link Printed} - * Note that this is mainly for debugging/testing purposes, and it will try to flush on each record print. - * It SHOULD NOT be used for production usage if performance requirements are concerned. - * - * @param printed options for printing - */ - void print(final Printed printed); + * Create a new {@code KStream} by transforming the value of each record in this stream into zero or more values + * with the same key in the new stream. + * Transform the value of each input record into zero or more records with the same (unmodified) key in the output + * stream (value type can be altered arbitrarily). + * The provided {@link ValueMapperWithKey} is applied to each input record and computes zero or more output values. + * Thus, an input record {@code } can be transformed into output records {@code , , ...}. + * This is a stateless record-by-record operation (cf. {@link #transformValues(ValueTransformerWithKeySupplier, String...)} + * for stateful value transformation). + *

    + * The example below splits input records {@code }, with key=1, containing sentences as values + * into their words. + * 

    {@code
+     * KStream inputStream = builder.stream("topic");
+     * KStream outputStream = inputStream.flatMapValues(new ValueMapper> {
+     *     Iterable apply(Integer readOnlyKey, String value) {
+     *         if(readOnlyKey == 1) {
+     *             return Arrays.asList(value.split(" "));
+     *         } else {
+     *             return Arrays.asList(value);
+     *         }
+     *     }
+     * });
+     * }
    + * The provided {@link ValueMapperWithKey} must return an {@link Iterable} (e.g., any {@link java.util.Collection} type) + * and the return value must not be {@code null}. + *

    + * Note that the key is read-only and should not be modified, as this can lead to corrupt partitioning. + * So, splitting a record into multiple records with the same key preserves data co-location with respect to the key. + * Thus, no internal data redistribution is required if a key based operator (like an aggregation or join) + * is applied to the result {@code KStream}. (cf. {@link #flatMap(KeyValueMapper)}) + * + * @param mapper a {@link ValueMapperWithKey} the computes the new output values + * @param named a {@link Named} config used to name the processor in the topology + * @param the value type of the result stream + * @return a {@code KStream} that contains more or less records with unmodified keys and new values of different type + * @see #selectKey(KeyValueMapper) + * @see #map(KeyValueMapper) + * @see #flatMap(KeyValueMapper) + * @see #mapValues(ValueMapper) + * @see #mapValues(ValueMapperWithKey) + * @see #transform(TransformerSupplier, String...) + * @see #flatTransform(TransformerSupplier, String...) + * @see #transformValues(ValueTransformerSupplier, String...) + * @see #transformValues(ValueTransformerWithKeySupplier, String...) + * @see #flatTransformValues(ValueTransformerSupplier, String...) + * @see #flatTransformValues(ValueTransformerWithKeySupplier, String...) + */ + KStream flatMapValues(final ValueMapperWithKey> mapper, + final Named named); + /** + * Print the records of this KStream using the options provided by {@link Printed} + * Note that this is mainly for debugging/testing purposes, and it will try to flush on each record print. + * It SHOULD NOT be used for production usage if performance requirements are concerned. + * + * @param printed options for printing + */ + void print(final Printed printed); /** * Perform an action on each record of {@code KStream}. @@ -379,6 +700,17 @@ public interface KStream { */ void foreach(final ForeachAction action); + /** + * Perform an action on each record of {@code KStream}. + * This is a stateless record-by-record operation (cf. {@link #process(ProcessorSupplier, String...)}). + * Note that this is a terminal operation that returns void. + * + * @param action an action to perform on each record + * @param named a {@link Named} config used to name the processor in the topology + * @see #process(ProcessorSupplier, String...) + */ + void foreach(final ForeachAction action, final Named named); + /** * Perform an action on each record of {@code KStream}. * This is a stateless record-by-record operation (cf. {@link #process(ProcessorSupplier, String...)}). @@ -394,6 +726,22 @@ public interface KStream { */ KStream peek(final ForeachAction action); + /** + * Perform an action on each record of {@code KStream}. + * This is a stateless record-by-record operation (cf. {@link #process(ProcessorSupplier, String...)}). + *

    + * Peek is a non-terminal operation that triggers a side effect (such as logging or statistics collection) + * and returns an unchanged stream. + *

    + * Note that since this operation is stateless, it may execute multiple times for a single record in failure cases. + * + * @param action an action to perform on each record + * @param named a {@link Named} config used to name the processor in the topology + * @see #process(ProcessorSupplier, String...) + * @return itself + */ + KStream peek(final ForeachAction action, final Named named); + /** * Creates an array of {@code KStream} from this stream by branching the records in the original stream based on * the supplied predicates. @@ -410,6 +758,23 @@ public interface KStream { @SuppressWarnings("unchecked") KStream[] branch(final Predicate... predicates); + /** + * Creates an array of {@code KStream} from this stream by branching the records in the original stream based on + * the supplied predicates. + * Each record is evaluated against the supplied predicates, and predicates are evaluated in order. + * Each stream in the result array corresponds position-wise (index) to the predicate in the supplied predicates. + * The branching happens on first-match: A record in the original stream is assigned to the corresponding result + * stream for the first predicate that evaluates to true, and is assigned to this stream only. + * A record will be dropped if none of the predicates evaluate to true. + * This is a stateless record-by-record operation. + * + * @param named a {@link Named} config used to name the processor in the topology + * @param predicates the ordered list of {@link Predicate} instances + * @return multiple distinct substreams of this {@code KStream} + */ + @SuppressWarnings("unchecked") + KStream[] branch(final Named named, final Predicate... predicates); + /** * Merge this stream and the given stream into one larger stream. *

    @@ -423,6 +788,20 @@ public interface KStream { */ KStream merge(final KStream stream); + /** + * Merge this stream and the given stream into one larger stream. + *

    + * There is no ordering guarantee between records from this {@code KStream} and records from + * the provided {@code KStream} in the merged stream. + * Relative order is preserved within each input stream though (ie, records within one input + * stream are processed in order). + * + * @param stream a stream which is to be merged into this stream + * @param named a {@link Named} config used to name the processor in the topology + * @return a merged stream containing all records from this and the provided {@code KStream} + */ + KStream merge(final KStream stream, final Named named); + /** * Materialize this stream to a topic and creates a new {@code KStream} from the topic using default serializers, * deserializers, and producer's {@link DefaultPartitioner}. @@ -589,6 +968,99 @@ void to(final TopicNameExtractor topicExtractor, KStream transform(final TransformerSupplier> transformerSupplier, final String... stateStoreNames); + /** + * Transform each record of the input stream into zero or one record in the output stream (both key and value type + * can be altered arbitrarily). + * A {@link Transformer} (provided by the given {@link TransformerSupplier}) is applied to each input record and + * returns zero or one output record. + * Thus, an input record {@code } can be transformed into an output record {@code }. + * This is a stateful record-by-record operation (cf. {@link #map(KeyValueMapper) map()}). + * Furthermore, via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) Punctuator#punctuate()}, + * the processing progress can be observed and additional periodic actions can be performed. + *

    + * In order to assign a state, the state must be created and registered beforehand (it's not required to connect + * global state stores; read-only access to global state stores is available by default): + * 

    {@code
+     * // create store
+     * StoreBuilder> keyValueStoreBuilder =
+     *         Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore("myTransformState"),
+     *                 Serdes.String(),
+     *                 Serdes.String());
+     * // register store
+     * builder.addStateStore(keyValueStoreBuilder);
+     *
+     * KStream outputStream = inputStream.transform(new TransformerSupplier() { ... }, "myTransformState");
+     * }
    + * Within the {@link Transformer}, the state is obtained via the {@link ProcessorContext}. + * To trigger periodic actions via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) punctuate()}, + * a schedule must be registered. + * The {@link Transformer} must return a {@link KeyValue} type in {@link Transformer#transform(Object, Object) + * transform()}. + * The return value of {@link Transformer#transform(Object, Object) Transformer#transform()} may be {@code null}, + * in which case no record is emitted. + * 
    {@code
+     * new TransformerSupplier() {
+     *     Transformer get() {
+     *         return new Transformer() {
+     *             private ProcessorContext context;
+     *             private StateStore state;
+     *
+     *             void init(ProcessorContext context) {
+     *                 this.context = context;
+     *                 this.state = context.getStateStore("myTransformState");
+     *                 // punctuate each second; can access this.state
+     *                 context.schedule(Duration.ofSeconds(1), PunctuationType.WALL_CLOCK_TIME, new Punctuator(..));
+     *             }
+     *
+     *             KeyValue transform(K key, V value) {
+     *                 // can access this.state
+     *                 return new KeyValue(key, value); // can emit a single value via return -- can also be null
+     *             }
+     *
+     *             void close() {
+     *                 // can access this.state
+     *             }
+     *         }
+     *     }
+     * }
+     * }
    + * Even if any upstream operation was key-changing, no auto-repartition is triggered. + * If repartitioning is required, a call to {@link #through(String) through()} should be performed before + * {@code transform()}. + *

    + * Transforming records might result in an internal data redistribution if a key based operator (like an aggregation + * or join) is applied to the result {@code KStream}. + * (cf. {@link #transformValues(ValueTransformerSupplier, String...) transformValues()} ) + *

    + * Note that it is possible to emit multiple records for each input record by using + * {@link ProcessorContext#forward(Object, Object) context#forward()} in + * {@link Transformer#transform(Object, Object) Transformer#transform()} and + * {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) Punctuator#punctuate()}. + * Be aware that a mismatch between the types of the emitted records and the type of the stream would only be + * detected at runtime. + * To ensure type-safety at compile-time, {@link ProcessorContext#forward(Object, Object) context#forward()} should + * not be used in {@link Transformer#transform(Object, Object) Transformer#transform()} and + * {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) Punctuator#punctuate()}. + * If in {@link Transformer#transform(Object, Object) Transformer#transform()} multiple records need to be emitted + * for each input record, it is recommended to use {@link #flatTransform(TransformerSupplier, String...) + * flatTransform()}. + * + * @param transformerSupplier an instance of {@link TransformerSupplier} that generates a {@link Transformer} + * @param named a {@link Named} config used to name the processor in the topology + * @param stateStoreNames the names of the state stores used by the processor + * @param the key type of the new stream + * @param the value type of the new stream + * @return a {@code KStream} that contains more or less records with new key and value (possibly of different type) + * @see #map(KeyValueMapper) + * @see #flatTransform(TransformerSupplier, String...) + * @see #transformValues(ValueTransformerSupplier, String...) + * @see #transformValues(ValueTransformerWithKeySupplier, String...) + * @see #process(ProcessorSupplier, String...) + */ + KStream transform(final TransformerSupplier> transformerSupplier, + final Named named, + final String... stateStoreNames); + /** * Transform each record of the input stream into zero or more records in the output stream (both key and value type * can be altered arbitrarily). @@ -682,52 +1154,465 @@ KStream flatTransform(final TransformerSupplier} can be transformed into an output record {@code }. - * This is a stateful record-by-record operation (cf. {@link #mapValues(ValueMapper)}). - * Furthermore, via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long)} the processing progress - * can be observed and additional periodic actions can be performed. + * Transform each record of the input stream into zero or more records in the output stream (both key and value type + * can be altered arbitrarily). + * A {@link Transformer} (provided by the given {@link TransformerSupplier}) is applied to each input record and + * returns zero or more output records. + * Thus, an input record {@code } can be transformed into output records {@code , , ...}. + * This is a stateful record-by-record operation (cf. {@link #flatMap(KeyValueMapper) flatMap()} for stateless + * record transformation). + * Furthermore, via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) Punctuator#punctuate()} + * the processing progress can be observed and additional periodic actions can be performed. *

    - * In order to assign a state store, the state store must be created and registered beforehand (it's not required to - * connect global state stores; read-only access to global state stores is available by default): + * In order to assign a state, the state must be created and registered beforehand (it's not required to connect + * global state stores; read-only access to global state stores is available by default): * 

    {@code
      * // create store
      * StoreBuilder> keyValueStoreBuilder =
-     *         Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore("myValueTransformState"),
+     *         Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore("myTransformState"),
      *                 Serdes.String(),
      *                 Serdes.String());
      * // register store
      * builder.addStateStore(keyValueStoreBuilder);
      *
-     * KStream outputStream = inputStream.transformValues(new ValueTransformerSupplier() { ... }, "myValueTransformState");
+     * KStream outputStream = inputStream.flatTransform(new TransformerSupplier() { ... }, "myTransformState");
      * }
    - * Within the {@link ValueTransformer}, the state store is obtained via the {@link ProcessorContext}. - * To trigger periodic actions via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) punctuate()}, - * a schedule must be registered. - * The {@link ValueTransformer} must return the new value in {@link ValueTransformer#transform(Object) transform()}. - * If the return value of {@link ValueTransformer#transform(Object) ValueTransformer#transform()} is {@code null}, - * no records are emitted. - * In contrast to {@link #transform(TransformerSupplier, String...) transform()}, no additional {@link KeyValue} - * pairs can be emitted via {@link ProcessorContext#forward(Object, Object) ProcessorContext.forward()}. - * A {@link org.apache.kafka.streams.errors.StreamsException} is thrown if the {@link ValueTransformer} tries to - * emit a {@link KeyValue} pair. + * Within the {@link Transformer}, the state is obtained via the {@link ProcessorContext}. + * To trigger periodic actions via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) + * punctuate()}, a schedule must be registered. + * The {@link Transformer} must return an {@link java.lang.Iterable} type (e.g., any {@link java.util.Collection} + * type) in {@link Transformer#transform(Object, Object) transform()}. + * The return value of {@link Transformer#transform(Object, Object) Transformer#transform()} may be {@code null}, + * which is equal to returning an empty {@link java.lang.Iterable Iterable}, i.e., no records are emitted. * 
    {@code
-     * new ValueTransformerSupplier() {
-     *     ValueTransformer get() {
-     *         return new ValueTransformer() {
+     * new TransformerSupplier() {
+     *     Transformer get() {
+     *         return new Transformer() {
+     *             private ProcessorContext context;
      *             private StateStore state;
      *
      *             void init(ProcessorContext context) {
-     *                 this.state = context.getStateStore("myValueTransformState");
-     *                 // punctuate each second, can access this.state
+     *                 this.context = context;
+     *                 this.state = context.getStateStore("myTransformState");
+     *                 // punctuate each second; can access this.state
      *                 context.schedule(Duration.ofSeconds(1), PunctuationType.WALL_CLOCK_TIME, new Punctuator(..));
      *             }
      *
-     *             NewValueType transform(V value) {
+     *             Iterable transform(K key, V value) {
      *                 // can access this.state
-     *                 return new NewValueType(); // or null
+     *                 List result = new ArrayList<>();
+     *                 for (int i = 0; i < 3; i++) {
+     *                     result.add(new KeyValue(key, value));
+     *                 }
+     *                 return result; // emits a list of key-value pairs via return
+     *             }
+     *
+     *             void close() {
+     *                 // can access this.state
+     *             }
+     *         }
+     *     }
+     * }
+     * }
    + * Even if any upstream operation was key-changing, no auto-repartition is triggered. + * If repartitioning is required, a call to {@link #through(String) through()} should be performed before + * {@code flatTransform()}. + *

    + * Transforming records might result in an internal data redistribution if a key based operator (like an aggregation + * or join) is applied to the result {@code KStream}. + * (cf. {@link #transformValues(ValueTransformerSupplier, String...) transformValues()}) + *

    + * Note that it is possible to emit records by using {@link ProcessorContext#forward(Object, Object) + * context#forward()} in {@link Transformer#transform(Object, Object) Transformer#transform()} and + * {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) Punctuator#punctuate()}. + * Be aware that a mismatch between the types of the emitted records and the type of the stream would only be + * detected at runtime. + * To ensure type-safety at compile-time, {@link ProcessorContext#forward(Object, Object) context#forward()} should + * not be used in {@link Transformer#transform(Object, Object) Transformer#transform()} and + * {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) Punctuator#punctuate()}. + * + * @param transformerSupplier an instance of {@link TransformerSupplier} that generates a {@link Transformer} + * @param named a {@link Named} config used to name the processor in the topology + * @param stateStoreNames the names of the state stores used by the processor + * @param the key type of the new stream + * @param the value type of the new stream + * @return a {@code KStream} that contains more or less records with new key and value (possibly of different type) + * @see #flatMap(KeyValueMapper) + * @see #transform(TransformerSupplier, String...) + * @see #transformValues(ValueTransformerSupplier, String...) + * @see #transformValues(ValueTransformerWithKeySupplier, String...) + * @see #process(ProcessorSupplier, String...) + */ + KStream flatTransform(final TransformerSupplier>> transformerSupplier, + final Named named, + final String... stateStoreNames); + + /** + * Transform the value of each input record into a new value (with possibly a new type) of the output record. + * A {@link ValueTransformer} (provided by the given {@link ValueTransformerSupplier}) is applied to each input + * record value and computes a new value for it. + * Thus, an input record {@code } can be transformed into an output record {@code }. + * This is a stateful record-by-record operation (cf. {@link #mapValues(ValueMapper)}). + * Furthermore, via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long)} the processing progress + * can be observed and additional periodic actions can be performed. + *

    + * In order to assign a state store, the state store must be created and registered beforehand (it's not required to + * connect global state stores; read-only access to global state stores is available by default): + * 

    {@code
+     * // create store
+     * StoreBuilder> keyValueStoreBuilder =
+     *         Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore("myValueTransformState"),
+     *                 Serdes.String(),
+     *                 Serdes.String());
+     * // register store
+     * builder.addStateStore(keyValueStoreBuilder);
+     *
+     * KStream outputStream = inputStream.transformValues(new ValueTransformerSupplier() { ... }, "myValueTransformState");
+     * }
    + * Within the {@link ValueTransformer}, the state store is obtained via the {@link ProcessorContext}. + * To trigger periodic actions via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) punctuate()}, + * a schedule must be registered. + * The {@link ValueTransformer} must return the new value in {@link ValueTransformer#transform(Object) transform()}. + * If the return value of {@link ValueTransformer#transform(Object) ValueTransformer#transform()} is {@code null}, + * no records are emitted. + * In contrast to {@link #transform(TransformerSupplier, String...) transform()}, no additional {@link KeyValue} + * pairs can be emitted via {@link ProcessorContext#forward(Object, Object) ProcessorContext.forward()}. + * A {@link org.apache.kafka.streams.errors.StreamsException} is thrown if the {@link ValueTransformer} tries to + * emit a {@link KeyValue} pair. + * 
    {@code
+     * new ValueTransformerSupplier() {
+     *     ValueTransformer get() {
+     *         return new ValueTransformer() {
+     *             private StateStore state;
+     *
+     *             void init(ProcessorContext context) {
+     *                 this.state = context.getStateStore("myValueTransformState");
+     *                 // punctuate each second, can access this.state
+     *                 context.schedule(Duration.ofSeconds(1), PunctuationType.WALL_CLOCK_TIME, new Punctuator(..));
+     *             }
+     *
+     *             NewValueType transform(V value) {
+     *                 // can access this.state
+     *                 return new NewValueType(); // or null
+     *             }
+     *
+     *             void close() {
+     *                 // can access this.state
+     *             }
+     *         }
+     *     }
+     * }
+     * }
    + * Even if any upstream operation was key-changing, no auto-repartition is triggered. + * If repartitioning is required, a call to {@link #through(String) through()} should be performed before + * {@code transformValues()}. + *

    + * Setting a new value preserves data co-location with respect to the key. + * Thus, no internal data redistribution is required if a key based operator (like an aggregation or join) + * is applied to the result {@code KStream}. (cf. {@link #transform(TransformerSupplier, String...)}) + * + * @param valueTransformerSupplier a instance of {@link ValueTransformerSupplier} that generates a + * {@link ValueTransformer} + * @param stateStoreNames the names of the state stores used by the processor + * @param the value type of the result stream + * @return a {@code KStream} that contains records with unmodified key and new values (possibly of different type) + * @see #mapValues(ValueMapper) + * @see #mapValues(ValueMapperWithKey) + * @see #transform(TransformerSupplier, String...) + */ + KStream transformValues(final ValueTransformerSupplier valueTransformerSupplier, + final String... stateStoreNames); + /** + * Transform the value of each input record into a new value (with possibly a new type) of the output record. + * A {@link ValueTransformer} (provided by the given {@link ValueTransformerSupplier}) is applied to each input + * record value and computes a new value for it. + * Thus, an input record {@code } can be transformed into an output record {@code }. + * This is a stateful record-by-record operation (cf. {@link #mapValues(ValueMapper)}). + * Furthermore, via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long)} the processing progress + * can be observed and additional periodic actions can be performed. + *

    + * In order to assign a state store, the state store must be created and registered beforehand (it's not required to + * connect global state stores; read-only access to global state stores is available by default): + * 

    {@code
+     * // create store
+     * StoreBuilder> keyValueStoreBuilder =
+     *         Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore("myValueTransformState"),
+     *                 Serdes.String(),
+     *                 Serdes.String());
+     * // register store
+     * builder.addStateStore(keyValueStoreBuilder);
+     *
+     * KStream outputStream = inputStream.transformValues(new ValueTransformerSupplier() { ... }, "myValueTransformState");
+     * }
    + * Within the {@link ValueTransformer}, the state store is obtained via the {@link ProcessorContext}. + * To trigger periodic actions via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) punctuate()}, + * a schedule must be registered. + * The {@link ValueTransformer} must return the new value in {@link ValueTransformer#transform(Object) transform()}. + * If the return value of {@link ValueTransformer#transform(Object) ValueTransformer#transform()} is {@code null}, no + * records are emitted. + * In contrast to {@link #transform(TransformerSupplier, String...) transform()}, no additional {@link KeyValue} + * pairs can be emitted via {@link ProcessorContext#forward(Object, Object) ProcessorContext.forward()}. + * A {@link org.apache.kafka.streams.errors.StreamsException} is thrown if the {@link ValueTransformer} tries to + * emit a {@link KeyValue} pair. + * 
    {@code
+     * new ValueTransformerSupplier() {
+     *     ValueTransformer get() {
+     *         return new ValueTransformer() {
+     *             private StateStore state;
+     *
+     *             void init(ProcessorContext context) {
+     *                 this.state = context.getStateStore("myValueTransformState");
+     *                 // punctuate each second, can access this.state
+     *                 context.schedule(Duration.ofSeconds(1), PunctuationType.WALL_CLOCK_TIME, new Punctuator(..));
+     *             }
+     *
+     *             NewValueType transform(V value) {
+     *                 // can access this.state
+     *                 return new NewValueType(); // or null
+     *             }
+     *
+     *             void close() {
+     *                 // can access this.state
+     *             }
+     *         }
+     *     }
+     * }
+     * }
    + * Even if any upstream operation was key-changing, no auto-repartition is triggered. + * If repartitioning is required, a call to {@link #through(String) through()} should be performed before + * {@code transformValues()}. + *

    + * Setting a new value preserves data co-location with respect to the key. + * Thus, no internal data redistribution is required if a key based operator (like an aggregation or join) + * is applied to the result {@code KStream}. (cf. {@link #transform(TransformerSupplier, String...)}) + * + * @param valueTransformerSupplier a instance of {@link ValueTransformerSupplier} that generates a + * {@link ValueTransformer} + * @param named a {@link Named} config used to name the processor in the topology + * @param stateStoreNames the names of the state stores used by the processor + * @param the value type of the result stream + * @return a {@code KStream} that contains records with unmodified key and new values (possibly of different type) + * @see #mapValues(ValueMapper) + * @see #mapValues(ValueMapperWithKey) + * @see #transform(TransformerSupplier, String...) + */ + KStream transformValues(final ValueTransformerSupplier valueTransformerSupplier, + final Named named, + final String... stateStoreNames); + + /** + * Transform the value of each input record into a new value (with possibly a new type) of the output record. + * A {@link ValueTransformerWithKey} (provided by the given {@link ValueTransformerWithKeySupplier}) is applied to + * each input record value and computes a new value for it. + * Thus, an input record {@code } can be transformed into an output record {@code }. + * This is a stateful record-by-record operation (cf. {@link #mapValues(ValueMapperWithKey)}). + * Furthermore, via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long)} the processing progress + * can be observed and additional periodic actions can be performed. + *

    + * In order to assign a state store, the state store must be created and registered beforehand (it's not required to + * connect global state stores; read-only access to global state stores is available by default): + * 

    {@code
+     * // create store
+     * StoreBuilder> keyValueStoreBuilder =
+     *         Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore("myValueTransformState"),
+     *                 Serdes.String(),
+     *                 Serdes.String());
+     * // register store
+     * builder.addStateStore(keyValueStoreBuilder);
+     *
+     * KStream outputStream = inputStream.transformValues(new ValueTransformerWithKeySupplier() { ... }, "myValueTransformState");
+     * }
    + * Within the {@link ValueTransformerWithKey}, the state store is obtained via the {@link ProcessorContext}. + * To trigger periodic actions via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) punctuate()}, + * a schedule must be registered. + * The {@link ValueTransformerWithKey} must return the new value in + * {@link ValueTransformerWithKey#transform(Object, Object) transform()}. + * If the return value of {@link ValueTransformerWithKey#transform(Object, Object) ValueTransformerWithKey#transform()} + * is {@code null}, no records are emitted. + * In contrast to {@link #transform(TransformerSupplier, String...) transform()} and + * {@link #flatTransform(TransformerSupplier, String...) flatTransform()}, no additional {@link KeyValue} pairs + * can be emitted via {@link ProcessorContext#forward(Object, Object) ProcessorContext.forward()}. + * A {@link org.apache.kafka.streams.errors.StreamsException} is thrown if the {@link ValueTransformerWithKey} tries + * to emit a {@link KeyValue} pair. + * 
    {@code
+     * new ValueTransformerWithKeySupplier() {
+     *     ValueTransformerWithKey get() {
+     *         return new ValueTransformerWithKey() {
+     *             private StateStore state;
+     *
+     *             void init(ProcessorContext context) {
+     *                 this.state = context.getStateStore("myValueTransformState");
+     *                 // punctuate each second, can access this.state
+     *                 context.schedule(Duration.ofSeconds(1), PunctuationType.WALL_CLOCK_TIME, new Punctuator(..));
+     *             }
+     *
+     *             NewValueType transform(K readOnlyKey, V value) {
+     *                 // can access this.state and use read-only key
+     *                 return new NewValueType(readOnlyKey); // or null
+     *             }
+     *
+     *             void close() {
+     *                 // can access this.state
+     *             }
+     *         }
+     *     }
+     * }
+     * }
    + * Even if any upstream operation was key-changing, no auto-repartition is triggered. + * If repartitioning is required, a call to {@link #through(String) through()} should be performed before + * {@code transformValues()}. + *

    + * Note that the key is read-only and should not be modified, as this can lead to corrupt partitioning. + * So, setting a new value preserves data co-location with respect to the key. + * Thus, no internal data redistribution is required if a key based operator (like an aggregation or join) + * is applied to the result {@code KStream}. (cf. {@link #transform(TransformerSupplier, String...)}) + * + * @param valueTransformerSupplier a instance of {@link ValueTransformerWithKeySupplier} that generates a + * {@link ValueTransformerWithKey} + * @param stateStoreNames the names of the state stores used by the processor + * @param the value type of the result stream + * @return a {@code KStream} that contains records with unmodified key and new values (possibly of different type) + * @see #mapValues(ValueMapper) + * @see #mapValues(ValueMapperWithKey) + * @see #transform(TransformerSupplier, String...) + */ + KStream transformValues(final ValueTransformerWithKeySupplier valueTransformerSupplier, + final String... stateStoreNames); + + /** + * Transform the value of each input record into a new value (with possibly a new type) of the output record. + * A {@link ValueTransformerWithKey} (provided by the given {@link ValueTransformerWithKeySupplier}) is applied to + * each input record value and computes a new value for it. + * Thus, an input record {@code } can be transformed into an output record {@code }. + * This is a stateful record-by-record operation (cf. {@link #mapValues(ValueMapperWithKey)}). + * Furthermore, via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long)} the processing progress + * can be observed and additional periodic actions can be performed. + *

    + * In order to assign a state store, the state store must be created and registered beforehand (it's not required to + * connect global state stores; read-only access to global state stores is available by default): + * 

    {@code
+     * // create store
+     * StoreBuilder> keyValueStoreBuilder =
+     *         Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore("myValueTransformState"),
+     *                 Serdes.String(),
+     *                 Serdes.String());
+     * // register store
+     * builder.addStateStore(keyValueStoreBuilder);
+     *
+     * KStream outputStream = inputStream.transformValues(new ValueTransformerWithKeySupplier() { ... }, "myValueTransformState");
+     * }
    + * Within the {@link ValueTransformerWithKey}, the state store is obtained via the {@link ProcessorContext}. + * To trigger periodic actions via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) punctuate()}, + * a schedule must be registered. + * The {@link ValueTransformerWithKey} must return the new value in + * {@link ValueTransformerWithKey#transform(Object, Object) transform()}. + * If the return value of {@link ValueTransformerWithKey#transform(Object, Object) ValueTransformerWithKey#transform()} + * is {@code null}, no records are emitted. + * In contrast to {@link #transform(TransformerSupplier, String...) transform()} and + * {@link #flatTransform(TransformerSupplier, String...) flatTransform()}, no additional {@link KeyValue} pairs + * can be emitted via {@link ProcessorContext#forward(Object, Object) ProcessorContext.forward()}. + * A {@link org.apache.kafka.streams.errors.StreamsException} is thrown if the {@link ValueTransformerWithKey} tries + * to emit a {@link KeyValue} pair. + * 
    {@code
+     * new ValueTransformerWithKeySupplier() {
+     *     ValueTransformerWithKey get() {
+     *         return new ValueTransformerWithKey() {
+     *             private StateStore state;
+     *
+     *             void init(ProcessorContext context) {
+     *                 this.state = context.getStateStore("myValueTransformState");
+     *                 // punctuate each second, can access this.state
+     *                 context.schedule(Duration.ofSeconds(1), PunctuationType.WALL_CLOCK_TIME, new Punctuator(..));
+     *             }
+     *
+     *             NewValueType transform(K readOnlyKey, V value) {
+     *                 // can access this.state and use read-only key
+     *                 return new NewValueType(readOnlyKey); // or null
+     *             }
+     *
+     *             void close() {
+     *                 // can access this.state
+     *             }
+     *         }
+     *     }
+     * }
+     * }
    + * Even if any upstream operation was key-changing, no auto-repartition is triggered. + * If repartitioning is required, a call to {@link #through(String) through()} should be performed before + * {@code transformValues()}. + *

    + * Note that the key is read-only and should not be modified, as this can lead to corrupt partitioning. + * So, setting a new value preserves data co-location with respect to the key. + * Thus, no internal data redistribution is required if a key based operator (like an aggregation or join) + * is applied to the result {@code KStream}. (cf. {@link #transform(TransformerSupplier, String...)}) + * + * @param valueTransformerSupplier a instance of {@link ValueTransformerWithKeySupplier} that generates a + * {@link ValueTransformerWithKey} + * @param named a {@link Named} config used to name the processor in the topology + * @param stateStoreNames the names of the state stores used by the processor + * @param the value type of the result stream + * @return a {@code KStream} that contains records with unmodified key and new values (possibly of different type) + * @see #mapValues(ValueMapper) + * @see #mapValues(ValueMapperWithKey) + * @see #transform(TransformerSupplier, String...) + */ + KStream transformValues(final ValueTransformerWithKeySupplier valueTransformerSupplier, + final Named named, + final String... stateStoreNames); + /** + * Transform the value of each input record into zero or more new values (with possibly a new + * type) and emit for each new value a record with the same key of the input record and the value. + * A {@link ValueTransformer} (provided by the given {@link ValueTransformerSupplier}) is applied to each input + * record value and computes zero or more new values. + * Thus, an input record {@code } can be transformed into output records {@code , , ...}. + * This is a stateful record-by-record operation (cf. {@link #mapValues(ValueMapper) mapValues()}). + * Furthermore, via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) Punctuator#punctuate()} + * the processing progress can be observed and additional periodic actions can be performed. + *

    + * In order to assign a state store, the state store must be created and registered beforehand: + * 

    {@code
+     * // create store
+     * StoreBuilder> keyValueStoreBuilder =
+     *         Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore("myValueTransformState"),
+     *                 Serdes.String(),
+     *                 Serdes.String());
+     * // register store
+     * builder.addStateStore(keyValueStoreBuilder);
+     *
+     * KStream outputStream = inputStream.flatTransformValues(new ValueTransformerSupplier() { ... }, "myValueTransformState");
+     * }
    + * Within the {@link ValueTransformer}, the state store is obtained via the {@link ProcessorContext}. + * To trigger periodic actions via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) punctuate()}, + * a schedule must be registered. + * The {@link ValueTransformer} must return an {@link java.lang.Iterable} type (e.g., any + * {@link java.util.Collection} type) in {@link ValueTransformer#transform(Object) + * transform()}. + * If the return value of {@link ValueTransformer#transform(Object) ValueTransformer#transform()} is an empty + * {@link java.lang.Iterable Iterable} or {@code null}, no records are emitted. + * In contrast to {@link #transform(TransformerSupplier, String...) transform()} and + * {@link #flatTransform(TransformerSupplier, String...) flatTransform()}, no additional {@link KeyValue} pairs + * can be emitted via {@link ProcessorContext#forward(Object, Object) ProcessorContext.forward()}. + * A {@link org.apache.kafka.streams.errors.StreamsException} is thrown if the {@link ValueTransformer} tries to + * emit a {@link KeyValue} pair. + * 
    {@code
+     * new ValueTransformerSupplier() {
+     *     ValueTransformer get() {
+     *         return new ValueTransformer() {
+     *             private StateStore state;
+     *
+     *             void init(ProcessorContext context) {
+     *                 this.state = context.getStateStore("myValueTransformState");
+     *                 // punctuate each second, can access this.state
+     *                 context.schedule(Duration.ofSeconds(1), PunctuationType.WALL_CLOCK_TIME, new Punctuator(..));
+     *             }
+     *
+     *             Iterable transform(V value) {
+     *                 // can access this.state
+     *                 List result = new ArrayList<>();
+     *                 for (int i = 0; i < 3; i++) {
+     *                     result.add(new NewValueType(value));
+     *                 }
+     *                 return result; // values
      *             }
      *
      *             void close() {
@@ -739,35 +1624,38 @@  KStream flatTransform(final TransformerSupplier
      * Even if any upstream operation was key-changing, no auto-repartition is triggered.
      * If repartitioning is required, a call to {@link #through(String) through()} should be performed before
-     * {@code transformValues()}.
+     * {@code flatTransformValues()}.
      * 
    
      * Setting a new value preserves data co-location with respect to the key.
      * Thus, no internal data redistribution is required if a key based operator (like an aggregation or join)
-     * is applied to the result {@code KStream}. (cf. {@link #transform(TransformerSupplier, String...)})
+     * is applied to the result {@code KStream}. (cf. {@link #flatTransform(TransformerSupplier, String...)
+     * flatTransform()})
      *
-     * @param valueTransformerSupplier a instance of {@link ValueTransformerSupplier} that generates a
+     * @param valueTransformerSupplier an instance of {@link ValueTransformerSupplier} that generates a
      *                                 {@link ValueTransformer}
      * @param stateStoreNames          the names of the state stores used by the processor
      * @param                      the value type of the result stream
-     * @return a {@code KStream} that contains records with unmodified key and new values (possibly of different type)
+     * @return a {@code KStream} that contains more or less records with unmodified key and new values (possibly of
+     * different type)
      * @see #mapValues(ValueMapper)
      * @see #mapValues(ValueMapperWithKey)
      * @see #transform(TransformerSupplier, String...)
+     * @see #flatTransform(TransformerSupplier, String...)
      */
-     KStream transformValues(final ValueTransformerSupplier valueTransformerSupplier,
-                                        final String... stateStoreNames);
+     KStream flatTransformValues(final ValueTransformerSupplier> valueTransformerSupplier,
+                                            final String... stateStoreNames);
 
     /**
-     * Transform the value of each input record into a new value (with possibly a new type) of the output record.
-     * A {@link ValueTransformerWithKey} (provided by the given {@link ValueTransformerWithKeySupplier}) is applied to
-     * each input record value and computes a new value for it.
-     * Thus, an input record {@code } can be transformed into an output record {@code }.
-     * This is a stateful record-by-record operation (cf. {@link #mapValues(ValueMapperWithKey)}).
-     * Furthermore, via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long)} the processing progress
-     * can be observed and additional periodic actions can be performed.
+     * Transform the value of each input record into zero or more new values (with possibly a new
+     * type) and emit for each new value a record with the same key of the input record and the value.
+     * A {@link ValueTransformer} (provided by the given {@link ValueTransformerSupplier}) is applied to each input
+     * record value and computes zero or more new values.
+     * Thus, an input record {@code } can be transformed into output records {@code , , ...}.
+     * This is a stateful record-by-record operation (cf. {@link #mapValues(ValueMapper) mapValues()}).
+     * Furthermore, via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) Punctuator#punctuate()}
+     * the processing progress can be observed and additional periodic actions can be performed.
      * 
    
-     * In order to assign a state store, the state store must be created and registered beforehand (it's not required to
-     * connect global state stores; read-only access to global state stores is available by default):
+     * In order to assign a state store, the state store must be created and registered beforehand:
      * 
    {@code
      * // create store
      * StoreBuilder> keyValueStoreBuilder =
@@ -777,24 +1665,25 @@  KStream transformValues(final ValueTransformerSupplier
-     * Within the {@link ValueTransformerWithKey}, the state store is obtained via the {@link ProcessorContext}.
+     * Within the {@link ValueTransformer}, the state store is obtained via the {@link ProcessorContext}.
      * To trigger periodic actions via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) punctuate()},
      * a schedule must be registered.
-     * The {@link ValueTransformerWithKey} must return the new value in
-     * {@link ValueTransformerWithKey#transform(Object, Object) transform()}.
-     * If the return value of {@link ValueTransformerWithKey#transform(Object, Object) ValueTransformerWithKey#transform()}
-     * is {@code null}, no records are emitted.
+     * The {@link ValueTransformer} must return an {@link java.lang.Iterable} type (e.g., any
+     * {@link java.util.Collection} type) in {@link ValueTransformer#transform(Object)
+     * transform()}.
+     * If the return value of {@link ValueTransformer#transform(Object) ValueTransformer#transform()} is an empty
+     * {@link java.lang.Iterable Iterable} or {@code null}, no records are emitted.
      * In contrast to {@link #transform(TransformerSupplier, String...) transform()} and
      * {@link #flatTransform(TransformerSupplier, String...) flatTransform()}, no additional {@link KeyValue} pairs
      * can be emitted via {@link ProcessorContext#forward(Object, Object) ProcessorContext.forward()}.
-     * A {@link org.apache.kafka.streams.errors.StreamsException} is thrown if the {@link ValueTransformerWithKey} tries
-     * to emit a {@link KeyValue} pair.
+     * A {@link org.apache.kafka.streams.errors.StreamsException} is thrown if the {@link ValueTransformer} tries to
+     * emit a {@link KeyValue} pair.
      * 
    {@code
-     * new ValueTransformerWithKeySupplier() {
-     *     ValueTransformerWithKey get() {
-     *         return new ValueTransformerWithKey() {
+     * new ValueTransformerSupplier() {
+     *     ValueTransformer get() {
+     *         return new ValueTransformer() {
      *             private StateStore state;
      *
      *             void init(ProcessorContext context) {
@@ -803,9 +1692,13 @@  KStream transformValues(final ValueTransformerSupplier transform(V value) {
+     *                 // can access this.state
+     *                 List result = new ArrayList<>();
+     *                 for (int i = 0; i < 3; i++) {
+     *                     result.add(new NewValueType(value));
+     *                 }
+     *                 return result; // values
      *             }
      *
      *             void close() {
@@ -817,34 +1710,38 @@  KStream transformValues(final ValueTransformerSupplier
      * Even if any upstream operation was key-changing, no auto-repartition is triggered.
      * If repartitioning is required, a call to {@link #through(String) through()} should be performed before
-     * {@code transformValues()}.
+     * {@code flatTransformValues()}.
      * 
    
-     * Note that the key is read-only and should not be modified, as this can lead to corrupt partitioning.
-     * So, setting a new value preserves data co-location with respect to the key.
+     * Setting a new value preserves data co-location with respect to the key.
      * Thus, no internal data redistribution is required if a key based operator (like an aggregation or join)
-     * is applied to the result {@code KStream}. (cf. {@link #transform(TransformerSupplier, String...)})
+     * is applied to the result {@code KStream}. (cf. {@link #flatTransform(TransformerSupplier, String...)
+     * flatTransform()})
      *
-     * @param valueTransformerSupplier a instance of {@link ValueTransformerWithKeySupplier} that generates a
-     *                                 {@link ValueTransformerWithKey}
+     * @param valueTransformerSupplier an instance of {@link ValueTransformerSupplier} that generates a
+     *                                 {@link ValueTransformer}
+     * @param named                    a {@link Named} config used to name the processor in the topology
      * @param stateStoreNames          the names of the state stores used by the processor
      * @param                      the value type of the result stream
-     * @return a {@code KStream} that contains records with unmodified key and new values (possibly of different type)
+     * @return a {@code KStream} that contains more or less records with unmodified key and new values (possibly of
+     * different type)
      * @see #mapValues(ValueMapper)
      * @see #mapValues(ValueMapperWithKey)
      * @see #transform(TransformerSupplier, String...)
+     * @see #flatTransform(TransformerSupplier, String...)
      */
-     KStream transformValues(final ValueTransformerWithKeySupplier valueTransformerSupplier,
-                                        final String... stateStoreNames);
+     KStream flatTransformValues(final ValueTransformerSupplier> valueTransformerSupplier,
+                                            final Named named,
+                                            final String... stateStoreNames);
 
     /**
      * Transform the value of each input record into zero or more new values (with possibly a new
      * type) and emit for each new value a record with the same key of the input record and the value.
-     * A {@link ValueTransformer} (provided by the given {@link ValueTransformerSupplier}) is applied to each input
-     * record value and computes zero or more new values.
+     * A {@link ValueTransformerWithKey} (provided by the given {@link ValueTransformerWithKeySupplier}) is applied to
+     * each input record value and computes zero or more new values.
      * Thus, an input record {@code } can be transformed into output records {@code , , ...}.
-     * This is a stateful record-by-record operation (cf. {@link #mapValues(ValueMapper) mapValues()}).
-     * Furthermore, via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) Punctuator#punctuate()}
-     * the processing progress can be observed and additional periodic actions can be performed.
+     * This is a stateful record-by-record operation (cf. {@link #flatMapValues(ValueMapperWithKey) flatMapValues()}).
+     * Furthermore, via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long)} the processing progress can
+     * be observed and additional periodic actions can be performed.
      * 
    
      * In order to assign a state store, the state store must be created and registered beforehand:
      * 
    {@code
@@ -856,25 +1753,25 @@  KStream transformValues(final ValueTransformerWithKeySupplier
-     * Within the {@link ValueTransformer}, the state store is obtained via the {@link ProcessorContext}.
+     * Within the {@link ValueTransformerWithKey}, the state store is obtained via the {@link ProcessorContext}.
      * To trigger periodic actions via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) punctuate()},
      * a schedule must be registered.
-     * The {@link ValueTransformer} must return an {@link java.lang.Iterable} type (e.g., any
-     * {@link java.util.Collection} type) in {@link ValueTransformer#transform(Object)
+     * The {@link ValueTransformerWithKey} must return an {@link java.lang.Iterable} type (e.g., any
+     * {@link java.util.Collection} type) in {@link ValueTransformerWithKey#transform(Object, Object)
      * transform()}.
-     * If the return value of {@link ValueTransformer#transform(Object) ValueTransformer#transform()} is an empty
-     * {@link java.lang.Iterable Iterable} or {@code null}, no records are emitted.
+     * If the return value of {@link ValueTransformerWithKey#transform(Object, Object) ValueTransformerWithKey#transform()}
+     * is an empty {@link java.lang.Iterable Iterable} or {@code null}, no records are emitted.
      * In contrast to {@link #transform(TransformerSupplier, String...) transform()} and
      * {@link #flatTransform(TransformerSupplier, String...) flatTransform()}, no additional {@link KeyValue} pairs
      * can be emitted via {@link ProcessorContext#forward(Object, Object) ProcessorContext.forward()}.
-     * A {@link org.apache.kafka.streams.errors.StreamsException} is thrown if the {@link ValueTransformer} tries to
-     * emit a {@link KeyValue} pair.
+     * A {@link org.apache.kafka.streams.errors.StreamsException} is thrown if the {@link ValueTransformerWithKey} tries
+     * to emit a {@link KeyValue} pair.
      * 
    {@code
-     * new ValueTransformerSupplier() {
-     *     ValueTransformer get() {
-     *         return new ValueTransformer() {
+     * new ValueTransformerWithKeySupplier() {
+     *     ValueTransformerWithKey get() {
+     *         return new ValueTransformerWithKey() {
      *             private StateStore state;
      *
      *             void init(ProcessorContext context) {
@@ -883,11 +1780,11 @@  KStream transformValues(final ValueTransformerWithKeySupplier transform(V value) {
-     *                 // can access this.state
+     *             Iterable transform(K readOnlyKey, V value) {
+     *                 // can access this.state and use read-only key
      *                 List result = new ArrayList<>();
      *                 for (int i = 0; i < 3; i++) {
-     *                     result.add(new NewValueType(value));
+     *                     result.add(new NewValueType(readOnlyKey));
      *                 }
      *                 return result; // values
      *             }
@@ -903,13 +1800,14 @@  KStream transformValues(final ValueTransformerWithKeySupplier
-     * Setting a new value preserves data co-location with respect to the key.
+     * Note that the key is read-only and should not be modified, as this can lead to corrupt partitioning.
+     * So, setting a new value preserves data co-location with respect to the key.
      * Thus, no internal data redistribution is required if a key based operator (like an aggregation or join)
      * is applied to the result {@code KStream}. (cf. {@link #flatTransform(TransformerSupplier, String...)
      * flatTransform()})
      *
-     * @param valueTransformerSupplier an instance of {@link ValueTransformerSupplier} that generates a
-     *                                 {@link ValueTransformer}
+     * @param valueTransformerSupplier a instance of {@link ValueTransformerWithKeySupplier} that generates a
+     *                                 {@link ValueTransformerWithKey}
      * @param stateStoreNames          the names of the state stores used by the processor
      * @param                      the value type of the result stream
      * @return a {@code KStream} that contains more or less records with unmodified key and new values (possibly of
@@ -919,7 +1817,7 @@  KStream transformValues(final ValueTransformerWithKeySupplier KStream flatTransformValues(final ValueTransformerSupplier> valueTransformerSupplier,
+     KStream flatTransformValues(final ValueTransformerWithKeySupplier> valueTransformerSupplier,
                                             final String... stateStoreNames);
 
     /**
@@ -929,8 +1827,8 @@  KStream flatTransformValues(final ValueTransformerSupplier} can be transformed into output records {@code , , ...}.
      * This is a stateful record-by-record operation (cf. {@link #flatMapValues(ValueMapperWithKey) flatMapValues()}).
-     * Furthermore, via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) punctuate()} the processing
-     * progress can be observed and additional periodic actions can be performed.
+     * Furthermore, via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long)} the processing progress can
+     * be observed and additional periodic actions can be performed.
      * 
    
      * In order to assign a state store, the state store must be created and registered beforehand:
      * 
    {@code
@@ -997,6 +1895,7 @@  KStream flatTransformValues(final ValueTransformerSupplier                     the value type of the result stream
      * @return a {@code KStream} that contains more or less records with unmodified key and new values (possibly of
@@ -1007,8 +1906,67 @@  KStream flatTransformValues(final ValueTransformerSupplier KStream flatTransformValues(final ValueTransformerWithKeySupplier> valueTransformerSupplier,
+                                            final Named named,
                                             final String... stateStoreNames);
 
+    /**
+     * Process all records in this stream, one record at a time, by applying a {@link Processor} (provided by the given
+     * {@link ProcessorSupplier}).
+     * This is a stateful record-by-record operation (cf. {@link #foreach(ForeachAction)}).
+     * Furthermore, via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long)} the processing progress
+     * can be observed and additional periodic actions can be performed.
+     * Note that this is a terminal operation that returns void.
+     * 
    
+     * In order to assign a state, the state must be created and registered beforehand (it's not required to connect
+     * global state stores; read-only access to global state stores is available by default):
+     * 
    {@code
+     * // create store
+     * StoreBuilder> keyValueStoreBuilder =
+     *         Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore("myProcessorState"),
+     *                 Serdes.String(),
+     *                 Serdes.String());
+     * // register store
+     * builder.addStateStore(keyValueStoreBuilder);
+     *
+     * inputStream.process(new ProcessorSupplier() { ... }, "myProcessorState");
+     * }
    
+     * Within the {@link Processor}, the state is obtained via the
+     * {@link ProcessorContext}.
+     * To trigger periodic actions via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) punctuate()},
+     * a schedule must be registered.
+     * 
    {@code
+     * new ProcessorSupplier() {
+     *     Processor get() {
+     *         return new Processor() {
+     *             private StateStore state;
+     *
+     *             void init(ProcessorContext context) {
+     *                 this.state = context.getStateStore("myProcessorState");
+     *                 // punctuate each second, can access this.state
+     *                 context.schedule(Duration.ofSeconds(1), PunctuationType.WALL_CLOCK_TIME, new Punctuator(..));
+     *             }
+     *
+     *             void process(K key, V value) {
+     *                 // can access this.state
+     *             }
+     *
+     *             void close() {
+     *                 // can access this.state
+     *             }
+     *         }
+     *     }
+     * }
+     * }
    
+     * Even if any upstream operation was key-changing, no auto-repartition is triggered.
+     * If repartitioning is required, a call to {@link #through(String)} should be performed before {@code transform()}.
+     *
+     * @param processorSupplier a instance of {@link ProcessorSupplier} that generates a {@link Processor}
+     * @param stateStoreNames   the names of the state store used by the processor
+     * @see #foreach(ForeachAction)
+     * @see #transform(TransformerSupplier, String...)
+     */
+    void process(final ProcessorSupplier processorSupplier,
+                 final String... stateStoreNames);
 
     /**
      * Process all records in this stream, one record at a time, by applying a {@link Processor} (provided by the given
@@ -1062,11 +2020,13 @@  KStream flatTransformValues(final ValueTransformerWithKeySupplier processorSupplier,
+                 final Named named,
                  final String... stateStoreNames);
 
     /**
@@ -2082,6 +3042,40 @@  KStream join(final GlobalKTable globalKTable,
                                      final KeyValueMapper keyValueMapper,
                                      final ValueJoiner joiner);
 
+    /**
+     * Join records of this stream with {@link GlobalKTable}'s records using non-windowed inner equi join.
+     * The join is a primary key table lookup join with join attribute
+     * {@code keyValueMapper.map(stream.keyValue) == table.key}.
+     * "Table lookup join" means, that results are only computed if {@code KStream} records are processed.
+     * This is done by performing a lookup for matching records in the current internal {@link GlobalKTable}
+     * state.
+     * In contrast, processing {@link GlobalKTable} input records will only update the internal {@link GlobalKTable}
+     * state and will not produce any result records.
+     * 
    
+     * For each {@code KStream} record that finds a corresponding record in {@link GlobalKTable} the provided
+     * {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record.
+     * The key of the result record is the same as the key of this {@code KStream}.
+     * If a {@code KStream} input record key or value is {@code null} the record will not be included in the join
+     * operation and thus no output record will be added to the resulting {@code KStream}.
+     * If {@code keyValueMapper} returns {@code null} implying no match exists, no output record will be added to the
+     * resulting {@code KStream}.
+     *
+     * @param globalKTable   the {@link GlobalKTable} to be joined with this stream
+     * @param keyValueMapper instance of {@link KeyValueMapper} used to map from the (key, value) of this stream
+     *                       to the key of the {@link GlobalKTable}
+     * @param joiner         a {@link ValueJoiner} that computes the join result for a pair of matching records
+     * @param named          a {@link Named} config used to name the processor in the topology
+     * @param            the key type of {@link GlobalKTable}
+     * @param            the value type of the {@link GlobalKTable}
+     * @param            the value type of the resulting {@code KStream}
+     * @return a {@code KStream} that contains join-records for each key and values computed by the given
+     * {@link ValueJoiner}, one output for each input {@code KStream} record
+     * @see #leftJoin(GlobalKTable, KeyValueMapper, ValueJoiner)
+     */
+     KStream join(final GlobalKTable globalKTable,
+                                     final KeyValueMapper keyValueMapper,
+                                     final ValueJoiner joiner,
+                                     final Named named);
     /**
      * Join records of this stream with {@link GlobalKTable}'s records using non-windowed left equi join.
      * In contrast to {@link #join(GlobalKTable, KeyValueMapper, ValueJoiner) inner-join}, all records from this stream
@@ -2118,4 +3112,43 @@  KStream join(final GlobalKTable globalKTable,
      KStream leftJoin(final GlobalKTable globalKTable,
                                          final KeyValueMapper keyValueMapper,
                                          final ValueJoiner valueJoiner);
+
+    /**
+     * Join records of this stream with {@link GlobalKTable}'s records using non-windowed left equi join.
+     * In contrast to {@link #join(GlobalKTable, KeyValueMapper, ValueJoiner) inner-join}, all records from this stream
+     * will produce an output record (cf. below).
+     * The join is a primary key table lookup join with join attribute
+     * {@code keyValueMapper.map(stream.keyValue) == table.key}.
+     * "Table lookup join" means, that results are only computed if {@code KStream} records are processed.
+     * This is done by performing a lookup for matching records in the current internal {@link GlobalKTable}
+     * state.
+     * In contrast, processing {@link GlobalKTable} input records will only update the internal {@link GlobalKTable}
+     * state and will not produce any result records.
+     * 
    
+     * For each {@code KStream} record whether or not it finds a corresponding record in {@link GlobalKTable} the
+     * provided {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record.
+     * The key of the result record is the same as this {@code KStream}.
+     * If a {@code KStream} input record key or value is {@code null} the record will not be included in the join
+     * operation and thus no output record will be added to the resulting {@code KStream}.
+     * If {@code keyValueMapper} returns {@code null} implying no match exists, a {@code null} value will be
+     * provided to {@link ValueJoiner}.
+     * If no {@link GlobalKTable} record was found during lookup, a {@code null} value will be provided to
+     * {@link ValueJoiner}.
+     *
+     * @param globalKTable   the {@link GlobalKTable} to be joined with this stream
+     * @param keyValueMapper instance of {@link KeyValueMapper} used to map from the (key, value) of this stream
+     *                       to the key of the {@link GlobalKTable}
+     * @param valueJoiner    a {@link ValueJoiner} that computes the join result for a pair of matching records
+     * @param named          a {@link Named} config used to name the processor in the topology
+     * @param            the key type of {@link GlobalKTable}
+     * @param            the value type of the {@link GlobalKTable}
+     * @param            the value type of the resulting {@code KStream}
+     * @return a {@code KStream} that contains join-records for each key and values computed by the given
+     * {@link ValueJoiner}, one output for each input {@code KStream} record
+     * @see #join(GlobalKTable, KeyValueMapper, ValueJoiner)
+     */
+     KStream leftJoin(final GlobalKTable globalKTable,
+                                         final KeyValueMapper keyValueMapper,
+                                         final ValueJoiner valueJoiner,
+                                         final Named named);
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java
index 67729eceaa51f..9d76033e82f3d 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java
@@ -27,6 +27,7 @@
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KTable;
 import org.apache.kafka.streams.kstream.KeyValueMapper;
+import org.apache.kafka.streams.kstream.Named;
 import org.apache.kafka.streams.kstream.Predicate;
 import org.apache.kafka.streams.kstream.Printed;
 import org.apache.kafka.streams.kstream.Produced;
@@ -128,47 +129,64 @@ public class KStreamImpl extends AbstractStream implements KStream filter(final Predicate predicate) {
-        Objects.requireNonNull(predicate, "predicate can't be null");
-        final String name = builder.newProcessorName(FILTER_NAME);
-
+        return filter(predicate, NamedInternal.empty());
+    }
 
+    @Override
+    public KStream filter(final Predicate predicate, final Named named) {
+        Objects.requireNonNull(predicate, "predicate can't be null");
+        Objects.requireNonNull(named, "named can't be null");
+        final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, FILTER_NAME);
         final ProcessorParameters processorParameters = new ProcessorParameters<>(new KStreamFilter<>(predicate, false), name);
         final ProcessorGraphNode filterProcessorNode = new ProcessorGraphNode<>(name, processorParameters);
         builder.addGraphNode(this.streamsGraphNode, filterProcessorNode);
 
-        return new KStreamImpl<>(name,
-                                 keySerde,
-                                 valSerde,
-                                 sourceNodes,
-                                 repartitionRequired,
-                                 filterProcessorNode,
-                                 builder);
+        return new KStreamImpl<>(
+                name,
+                keySerde,
+                valSerde,
+                sourceNodes,
+                repartitionRequired,
+                filterProcessorNode,
+                builder);
     }
 
     @Override
     public KStream filterNot(final Predicate predicate) {
-        Objects.requireNonNull(predicate, "predicate can't be null");
-        final String name = builder.newProcessorName(FILTER_NAME);
+        return filterNot(predicate, NamedInternal.empty());
+    }
 
+    @Override
+    public KStream filterNot(final Predicate predicate, final Named named) {
+        Objects.requireNonNull(predicate, "predicate can't be null");
+        Objects.requireNonNull(named, "named can't be null");
+        final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, FILTER_NAME);
         final ProcessorParameters processorParameters = new ProcessorParameters<>(new KStreamFilter<>(predicate, true), name);
         final ProcessorGraphNode filterNotProcessorNode = new ProcessorGraphNode<>(name, processorParameters);
 
         builder.addGraphNode(this.streamsGraphNode, filterNotProcessorNode);
 
-        return new KStreamImpl<>(name,
-                                 keySerde,
-                                 valSerde,
-                                 sourceNodes,
-                                 repartitionRequired,
-                                 filterNotProcessorNode,
-                                 builder);
+        return new KStreamImpl<>(
+                name,
+                keySerde,
+                valSerde,
+                sourceNodes,
+                repartitionRequired,
+                filterNotProcessorNode,
+                builder);
     }
 
     @Override
     public  KStream selectKey(final KeyValueMapper mapper) {
+        return selectKey(mapper, NamedInternal.empty());
+    }
+
+    @Override
+    public  KStream selectKey(final KeyValueMapper mapper, final Named named) {
         Objects.requireNonNull(mapper, "mapper can't be null");
+        Objects.requireNonNull(named, "named can't be null");
 
-        final ProcessorGraphNode selectKeyProcessorNode = internalSelectKey(mapper, NamedInternal.empty());
+        final ProcessorGraphNode selectKeyProcessorNode = internalSelectKey(mapper, new NamedInternal(named));
 
         selectKeyProcessorNode.keyChangingOperation(true);
         builder.addGraphNode(this.streamsGraphNode, selectKeyProcessorNode);
@@ -177,7 +195,6 @@ public  KStream selectKey(final KeyValueMapper(selectKeyProcessorNode.nodeName(), null, valSerde, sourceNodes, true, selectKeyProcessorNode, builder);
     }
 
-
     private  ProcessorGraphNode internalSelectKey(final KeyValueMapper mapper,
                                                             final NamedInternal named) {
         final String name = named.orElseGenerateWithPrefix(builder, KEY_SELECT_NAME);
@@ -190,9 +207,14 @@ private  ProcessorGraphNode internalSelectKey(final KeyValueMapper KStream map(final KeyValueMapper> mapper) {
-        Objects.requireNonNull(mapper, "mapper can't be null");
-        final String name = builder.newProcessorName(MAP_NAME);
+        return map(mapper, NamedInternal.empty());
+    }
 
+    @Override
+    public  KStream map(final KeyValueMapper> mapper, final Named named) {
+        Objects.requireNonNull(mapper, "mapper can't be null");
+        Objects.requireNonNull(named, "named can't be null");
+        final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, MAP_NAME);
         final ProcessorParameters processorParameters = new ProcessorParameters<>(new KStreamMap<>(mapper), name);
 
         final ProcessorGraphNode mapProcessorNode = new ProcessorGraphNode<>(name, processorParameters);
@@ -201,26 +223,36 @@ public  KStream map(final KeyValueMapper(name,
-                                 null,
-                                 null,
-                                 sourceNodes,
-                                 true,
-                                 mapProcessorNode,
-                                 builder);
+        return new KStreamImpl<>(
+                name,
+                null,
+                null,
+                sourceNodes,
+                true,
+                mapProcessorNode,
+                builder);
     }
 
-
     @Override
     public  KStream mapValues(final ValueMapper mapper) {
         return mapValues(withKey(mapper));
     }
 
+    @Override
+    public  KStream mapValues(final ValueMapper mapper, final Named named) {
+        return mapValues(withKey(mapper), named);
+    }
+
     @Override
     public  KStream mapValues(final ValueMapperWithKey mapper) {
-        Objects.requireNonNull(mapper, "mapper can't be null");
-        final String name = builder.newProcessorName(MAPVALUES_NAME);
+        return mapValues(mapper, NamedInternal.empty());
+    }
 
+    @Override
+    public  KStream mapValues(final ValueMapperWithKey mapper, final Named named) {
+        Objects.requireNonNull(mapper, "mapper can't be null");
+        Objects.requireNonNull(mapper, "named can't be null");
+        final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, MAPVALUES_NAME);
         final ProcessorParameters processorParameters = new ProcessorParameters<>(new KStreamMapValues<>(mapper), name);
         final ProcessorGraphNode mapValuesProcessorNode = new ProcessorGraphNode<>(name, processorParameters);
 
@@ -228,13 +260,14 @@ public  KStream mapValues(final ValueMapperWithKey(name,
-                                 keySerde,
-                                 null,
-                                 sourceNodes,
-                                 repartitionRequired,
-                                 mapValuesProcessorNode,
-                                 builder);
+        return new KStreamImpl<>(
+                name,
+                keySerde,
+                null,
+                sourceNodes,
+                repartitionRequired,
+                mapValuesProcessorNode,
+                builder);
     }
 
     @Override
@@ -250,9 +283,15 @@ public void print(final Printed printed) {
 
     @Override
     public  KStream flatMap(final KeyValueMapper>> mapper) {
-        Objects.requireNonNull(mapper, "mapper can't be null");
-        final String name = builder.newProcessorName(FLATMAP_NAME);
+        return flatMap(mapper, NamedInternal.empty());
+    }
 
+    @Override
+    public  KStream flatMap(final KeyValueMapper>> mapper,
+                                            final Named named) {
+        Objects.requireNonNull(mapper, "mapper can't be null");
+        Objects.requireNonNull(named, "named can't be null");
+        final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, FLATMAP_NAME);
         final ProcessorParameters processorParameters = new ProcessorParameters<>(new KStreamFlatMap<>(mapper), name);
         final ProcessorGraphNode flatMapNode = new ProcessorGraphNode<>(name, processorParameters);
         flatMapNode.keyChangingOperation(true);
@@ -260,13 +299,7 @@ public  KStream flatMap(final KeyValueMapper(name,
-                                 null,
-                                 null,
-                                 sourceNodes,
-                                 true,
-                                 flatMapNode,
-                                 builder);
+        return new KStreamImpl<>(name, null, null, sourceNodes, true, flatMapNode, builder);
     }
 
     @Override
@@ -274,11 +307,23 @@ public  KStream flatMapValues(final ValueMapper KStream flatMapValues(final ValueMapper> mapper,
+                                             final Named named) {
+        return flatMapValues(withKey(mapper), named);
+    }
+
     @Override
     public  KStream flatMapValues(final ValueMapperWithKey> mapper) {
-        Objects.requireNonNull(mapper, "mapper can't be null");
-        final String name = builder.newProcessorName(FLATMAPVALUES_NAME);
+        return flatMapValues(mapper, NamedInternal.empty());
+    }
 
+    @Override
+    public  KStream flatMapValues(final ValueMapperWithKey> mapper,
+                                             final Named named) {
+        Objects.requireNonNull(mapper, "mapper can't be null");
+        Objects.requireNonNull(named, "named can't be null");
+        final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, FLATMAPVALUES_NAME);
         final ProcessorParameters processorParameters = new ProcessorParameters<>(new KStreamFlatMapValues<>(mapper), name);
         final ProcessorGraphNode flatMapValuesNode = new ProcessorGraphNode<>(name, processorParameters);
 
@@ -292,6 +337,19 @@ public  KStream flatMapValues(final ValueMapperWithKey[] branch(final Predicate... predicates) {
+        return doBranch(NamedInternal.empty(), predicates);
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public KStream[] branch(final Named name, final Predicate... predicates) {
+        Objects.requireNonNull(name, "name can't be null");
+        return doBranch(new NamedInternal(name), predicates);
+    }
+
+    @SuppressWarnings("unchecked")
+    private KStream[] doBranch(final NamedInternal named,
+                                     final Predicate... predicates) {
         if (predicates.length == 0) {
             throw new IllegalArgumentException("you must provide at least one predicate");
         }
@@ -299,11 +357,11 @@ public KStream[] branch(final Predicate... predicate
             Objects.requireNonNull(predicate, "predicates can't have null values");
         }
 
-        final String branchName = builder.newProcessorName(BRANCH_NAME);
+        final String branchName = named.orElseGenerateWithPrefix(builder, BRANCH_NAME);
 
         final String[] childNames = new String[predicates.length];
         for (int i = 0; i < predicates.length; i++) {
-            childNames[i] = builder.newProcessorName(BRANCHCHILD_NAME);
+            childNames[i] = named.suffixWithOrElseGet("-predicate-" + i, builder, BRANCHCHILD_NAME);
         }
 
         final ProcessorParameters processorParameters = new ProcessorParameters<>(new KStreamBranch(predicates.clone(), childNames), branchName);
@@ -326,13 +384,20 @@ public KStream[] branch(final Predicate... predicate
     @Override
     public KStream merge(final KStream stream) {
         Objects.requireNonNull(stream);
-        return merge(builder, stream);
+        return merge(builder, stream, NamedInternal.empty());
+    }
+
+    @Override
+    public KStream merge(final KStream stream, final Named processorName) {
+        Objects.requireNonNull(stream);
+        return merge(builder, stream, new NamedInternal(processorName));
     }
 
     private KStream merge(final InternalStreamsBuilder builder,
-                                final KStream stream) {
+                                final KStream stream,
+                                final NamedInternal processorName) {
         final KStreamImpl streamImpl = (KStreamImpl) stream;
-        final String name = builder.newProcessorName(MERGE_NAME);
+        final String name = processorName.orElseGenerateWithPrefix(builder, MERGE_NAME);
         final Set allSourceNodes = new HashSet<>();
 
         final boolean requireRepartitioning = streamImpl.repartitionRequired || repartitionRequired;
@@ -341,9 +406,7 @@ private KStream merge(final InternalStreamsBuilder builder,
 
         final ProcessorParameters processorParameters = new ProcessorParameters<>(new KStreamPassThrough<>(), name);
 
-
         final ProcessorGraphNode mergeNode = new ProcessorGraphNode<>(name, processorParameters);
-
         mergeNode.setMergeNode(true);
         builder.addGraphNode(Arrays.asList(this.streamsGraphNode, streamImpl.streamsGraphNode), mergeNode);
 
@@ -353,12 +416,16 @@ private KStream merge(final InternalStreamsBuilder builder,
 
     @Override
     public void foreach(final ForeachAction action) {
-        Objects.requireNonNull(action, "action can't be null");
-        final String name = builder.newProcessorName(FOREACH_NAME);
+        foreach(action, NamedInternal.empty());
+    }
 
+    @Override
+    public void foreach(final ForeachAction action, final Named named) {
+        Objects.requireNonNull(action, "action can't be null");
+        final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, FOREACH_NAME);
         final ProcessorParameters processorParameters = new ProcessorParameters<>(
-            new KStreamPeek<>(action, false),
-            name
+                new KStreamPeek<>(action, false),
+                name
         );
 
         final ProcessorGraphNode foreachNode = new ProcessorGraphNode<>(name, processorParameters);
@@ -367,12 +434,18 @@ public void foreach(final ForeachAction action) {
 
     @Override
     public KStream peek(final ForeachAction action) {
+        return peek(action, NamedInternal.empty());
+    }
+
+    @Override
+    public KStream peek(final ForeachAction action, final Named named) {
         Objects.requireNonNull(action, "action can't be null");
-        final String name = builder.newProcessorName(PEEK_NAME);
+        Objects.requireNonNull(named, "named can't be null");
+        final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, PEEK_NAME);
 
         final ProcessorParameters processorParameters = new ProcessorParameters<>(
-            new KStreamPeek<>(action, true),
-            name
+                new KStreamPeek<>(action, true),
+                name
         );
 
         final ProcessorGraphNode peekNode = new ProcessorGraphNode<>(name, processorParameters);
@@ -459,56 +532,89 @@ private void to(final TopicNameExtractor topicExtractor, final ProducedInt
         builder.addGraphNode(this.streamsGraphNode, sinkNode);
     }
 
-    private  KStream doFlatTransform(final TransformerSupplier>> transformerSupplier,
-                                                     final String... stateStoreNames) {
+    @Override
+    public  KStream transform(final TransformerSupplier> transformerSupplier,
+                                              final String... stateStoreNames) {
+        Objects.requireNonNull(transformerSupplier, "transformerSupplier can't be null");
         final String name = builder.newProcessorName(TRANSFORM_NAME);
-        final StatefulProcessorNode transformNode = new StatefulProcessorNode<>(
-            name,
-            new ProcessorParameters<>(new KStreamFlatTransform<>(transformerSupplier), name),
-            stateStoreNames
-        );
-
-        transformNode.keyChangingOperation(true);
-        builder.addGraphNode(this.streamsGraphNode, transformNode);
-
-        // cannot inherit key and value serde
-        return new KStreamImpl<>(name, null, null, sourceNodes, true, transformNode, builder);
+        return flatTransform(new TransformerSupplierAdapter<>(transformerSupplier), Named.as(name), stateStoreNames);
     }
 
     @Override
     public  KStream transform(final TransformerSupplier> transformerSupplier,
+                                              final Named named,
                                               final String... stateStoreNames) {
         Objects.requireNonNull(transformerSupplier, "transformerSupplier can't be null");
-        return doFlatTransform(new TransformerSupplierAdapter<>(transformerSupplier), stateStoreNames);
+        return flatTransform(new TransformerSupplierAdapter<>(transformerSupplier), named, stateStoreNames);
+    }
+
+    @Override
+    public  KStream flatTransform(final TransformerSupplier>> transformerSupplier,
+                                                  final String... stateStoreNames) {
+        Objects.requireNonNull(transformerSupplier, "transformerSupplier can't be null");
+        final String name = builder.newProcessorName(TRANSFORM_NAME);
+        return flatTransform(transformerSupplier, Named.as(name), stateStoreNames);
     }
 
     @Override
     public  KStream flatTransform(final TransformerSupplier>> transformerSupplier,
+                                                  final Named named,
                                                   final String... stateStoreNames) {
         Objects.requireNonNull(transformerSupplier, "transformerSupplier can't be null");
-        return doFlatTransform(transformerSupplier, stateStoreNames);
+        Objects.requireNonNull(named, "named can't be null");
+
+        final String name = new NamedInternal(named).name();
+        final StatefulProcessorNode transformNode = new StatefulProcessorNode<>(
+                name,
+                new ProcessorParameters<>(new KStreamFlatTransform<>(transformerSupplier), name),
+                stateStoreNames
+        );
+
+        transformNode.keyChangingOperation(true);
+        builder.addGraphNode(streamsGraphNode, transformNode);
+
+        // cannot inherit key and value serde
+        return new KStreamImpl<>(name, null, null, sourceNodes, true, transformNode, builder);
     }
 
     @Override
     public  KStream transformValues(final ValueTransformerSupplier valueTransformerSupplier,
                                                final String... stateStoreNames) {
         Objects.requireNonNull(valueTransformerSupplier, "valueTransformerSupplier can't be null");
+        return doTransformValues(toValueTransformerWithKeySupplier(valueTransformerSupplier), NamedInternal.empty(), stateStoreNames);
+    }
 
-        return doTransformValues(toValueTransformerWithKeySupplier(valueTransformerSupplier), stateStoreNames);
+    @Override
+    public  KStream transformValues(final ValueTransformerSupplier valueTransformerSupplier,
+                                               final Named named,
+                                               final String... stateStoreNames) {
+        Objects.requireNonNull(valueTransformerSupplier, "valueTransformSupplier can't be null");
+        Objects.requireNonNull(named, "named can't be null");
+        return doTransformValues(toValueTransformerWithKeySupplier(valueTransformerSupplier),
+                new NamedInternal(named), stateStoreNames);
     }
 
     @Override
     public  KStream transformValues(final ValueTransformerWithKeySupplier valueTransformerSupplier,
                                                final String... stateStoreNames) {
         Objects.requireNonNull(valueTransformerSupplier, "valueTransformerSupplier can't be null");
+        return doTransformValues(valueTransformerSupplier, NamedInternal.empty(), stateStoreNames);
+    }
 
-        return doTransformValues(valueTransformerSupplier, stateStoreNames);
+    @Override
+    public  KStream transformValues(final ValueTransformerWithKeySupplier valueTransformerSupplier,
+                                               final Named named,
+                                               final String... stateStoreNames) {
+        Objects.requireNonNull(valueTransformerSupplier, "valueTransformSupplier can't be null");
+        Objects.requireNonNull(named, "named can't be null");
+        return doTransformValues(valueTransformerSupplier, new NamedInternal(named), stateStoreNames);
     }
 
     private  KStream doTransformValues(final ValueTransformerWithKeySupplier valueTransformerWithKeySupplier,
+                                                  final NamedInternal named,
                                                   final String... stateStoreNames) {
-        final String name = builder.newProcessorName(TRANSFORMVALUES_NAME);
 
+        final String name = named.orElseGenerateWithPrefix(builder, TRANSFORMVALUES_NAME);
         final StatefulProcessorNode transformNode = new StatefulProcessorNode<>(
             name,
             new ProcessorParameters<>(new KStreamTransformValues<>(valueTransformerWithKeySupplier), name),
@@ -527,20 +633,39 @@ public  KStream flatTransformValues(final ValueTransformerSupplier KStream flatTransformValues(final ValueTransformerSupplier> valueTransformerSupplier,
+                                                   final Named named,
+                                                   final String... stateStoreNames) {
+        Objects.requireNonNull(valueTransformerSupplier, "valueTransformerSupplier can't be null");
+
+        return doFlatTransformValues(toValueTransformerWithKeySupplier(valueTransformerSupplier), named, stateStoreNames);
+    }
+
+    @Override
+    public  KStream flatTransformValues(final ValueTransformerWithKeySupplier> valueTransformerSupplier,
+                                                   final String... stateStoreNames) {
+        Objects.requireNonNull(valueTransformerSupplier, "valueTransformerSupplier can't be null");
+
+        return doFlatTransformValues(valueTransformerSupplier, NamedInternal.empty(), stateStoreNames);
     }
 
     @Override
     public  KStream flatTransformValues(final ValueTransformerWithKeySupplier> valueTransformerSupplier,
+                                                   final Named named,
                                                    final String... stateStoreNames) {
         Objects.requireNonNull(valueTransformerSupplier, "valueTransformerSupplier can't be null");
 
-        return doFlatTransformValues(valueTransformerSupplier, stateStoreNames);
+        return doFlatTransformValues(valueTransformerSupplier, named, stateStoreNames);
     }
 
     private  KStream doFlatTransformValues(final ValueTransformerWithKeySupplier> valueTransformerWithKeySupplier,
+                                                      final Named named,
                                                       final String... stateStoreNames) {
-        final String name = builder.newProcessorName(TRANSFORMVALUES_NAME);
+        final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, TRANSFORMVALUES_NAME);
 
         final StatefulProcessorNode transformNode = new StatefulProcessorNode<>(
             name,
@@ -561,11 +686,21 @@ public void process(final ProcessorSupplier processorSuppl
 
         Objects.requireNonNull(processorSupplier, "ProcessSupplier cant' be null");
         final String name = builder.newProcessorName(PROCESSOR_NAME);
+        process(processorSupplier, Named.as(name), stateStoreNames);
+    }
+
+    @Override
+    public void process(final ProcessorSupplier processorSupplier,
+                        final Named named,
+                        final String... stateStoreNames) {
+        Objects.requireNonNull(processorSupplier, "ProcessSupplier cant' be null");
+        Objects.requireNonNull(named, "named cant' be null");
 
+        final String name = new NamedInternal(named).name();
         final StatefulProcessorNode processNode = new StatefulProcessorNode<>(
-            name,
-            new ProcessorParameters<>(processorSupplier, name),
-            stateStoreNames
+                name,
+                new ProcessorParameters<>(processorSupplier, name),
+                stateStoreNames
         );
 
         builder.addGraphNode(this.streamsGraphNode, processNode);
@@ -621,14 +756,16 @@ private  KStream doJoin(final KStream other,
         KStreamImpl joinOther = (KStreamImpl) other;
 
         final JoinedInternal joinedInternal = new JoinedInternal<>(joined);
-        final String name = joinedInternal.name();
+        final NamedInternal name = new NamedInternal(joinedInternal.name());
         if (joinThis.repartitionRequired) {
-            final String leftJoinRepartitionTopicName = name != null ? name + "-left" : joinThis.name;
+            final String joinThisName = joinThis.name;
+            final String leftJoinRepartitionTopicName = name.suffixWithOrElseGet("-left", joinThisName);
             joinThis = joinThis.repartitionForJoin(leftJoinRepartitionTopicName, joined.keySerde(), joined.valueSerde());
         }
 
         if (joinOther.repartitionRequired) {
-            final String rightJoinRepartitionTopicName = name != null ? name + "-right" : joinOther.name;
+            final String joinOtherName = joinOther.name;
+            final String rightJoinRepartitionTopicName = name.suffixWithOrElseGet("-right", joinOtherName);
             joinOther = joinOther.repartitionForJoin(rightJoinRepartitionTopicName, joined.keySerde(), joined.otherValueSerde());
         }
 
@@ -779,26 +916,45 @@ public  KStream leftJoin(final KTable other,
     public  KStream join(final GlobalKTable globalTable,
                                             final KeyValueMapper keyMapper,
                                             final ValueJoiner joiner) {
-        return globalTableJoin(globalTable, keyMapper, joiner, false);
+        return globalTableJoin(globalTable, keyMapper, joiner, false, NamedInternal.empty());
+    }
+
+    @Override
+    public  KStream join(final GlobalKTable globalTable,
+                                            final KeyValueMapper keyMapper,
+                                            final ValueJoiner joiner,
+                                            final Named named) {
+        return globalTableJoin(globalTable, keyMapper, joiner, false, named);
     }
 
     @Override
     public  KStream leftJoin(final GlobalKTable globalTable,
                                                 final KeyValueMapper keyMapper,
                                                 final ValueJoiner joiner) {
-        return globalTableJoin(globalTable, keyMapper, joiner, true);
+        return globalTableJoin(globalTable, keyMapper, joiner, true, NamedInternal.empty());
     }
 
+    @Override
+    public  KStream leftJoin(final GlobalKTable globalTable,
+                                                final KeyValueMapper keyMapper,
+                                                final ValueJoiner joiner,
+                                                final Named named) {
+        return globalTableJoin(globalTable, keyMapper, joiner, true, named);
+    }
+
+
     private  KStream globalTableJoin(final GlobalKTable globalTable,
                                                         final KeyValueMapper keyMapper,
                                                         final ValueJoiner joiner,
-                                                        final boolean leftJoin) {
+                                                        final boolean leftJoin,
+                                                        final Named named) {
         Objects.requireNonNull(globalTable, "globalTable can't be null");
         Objects.requireNonNull(keyMapper, "keyMapper can't be null");
         Objects.requireNonNull(joiner, "joiner can't be null");
+        Objects.requireNonNull(named, "named can't be null");
 
         final KTableValueGetterSupplier valueGetterSupplier = ((GlobalKTableImpl) globalTable).valueGetterSupplier();
-        final String name = builder.newProcessorName(LEFTJOIN_NAME);
+        final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, LEFTJOIN_NAME);
 
         final ProcessorSupplier processorSupplier = new KStreamGlobalKTableJoin<>(
             valueGetterSupplier,
@@ -828,7 +984,10 @@ private  KStream doStreamTableJoin(final KTable other,
 
         final Set allSourceNodes = ensureJoinableWith((AbstractStream) other);
 
-        final String name = builder.newProcessorName(leftJoin ? LEFTJOIN_NAME : JOIN_NAME);
+        final JoinedInternal joinedInternal = new JoinedInternal<>(joined);
+        final NamedInternal renamed = new NamedInternal(joinedInternal.name());
+
+        final String name = renamed.orElseGenerateWithPrefix(builder, leftJoin ? LEFTJOIN_NAME : JOIN_NAME);
         final ProcessorSupplier processorSupplier = new KStreamKTableJoin<>(
             ((KTableImpl) other).valueGetterSupplier(),
             joiner,
@@ -945,12 +1104,23 @@ public  KStream join(final KStream lhs,
                                                    final ValueJoiner joiner,
                                                    final JoinWindows windows,
                                                    final Joined joined) {
-            final String thisWindowStreamName = builder.newProcessorName(WINDOWED_NAME);
-            final String otherWindowStreamName = builder.newProcessorName(WINDOWED_NAME);
-            final String joinThisName = rightOuter ? builder.newProcessorName(OUTERTHIS_NAME) : builder.newProcessorName(JOINTHIS_NAME);
-            final String joinOtherName = leftOuter ? builder.newProcessorName(OUTEROTHER_NAME) : builder.newProcessorName(JOINOTHER_NAME);
-            final String joinMergeName = builder.newProcessorName(MERGE_NAME);
 
+            final JoinedInternal  joinedInternal = new JoinedInternal<>(joined);
+            final NamedInternal renamed = new NamedInternal(joinedInternal.name());
+
+            final String thisWindowStreamName =  renamed.suffixWithOrElseGet(
+                    "-this-windowed", builder, WINDOWED_NAME);
+            final String otherWindowStreamName = renamed.suffixWithOrElseGet(
+                    "-other-windowed", builder, WINDOWED_NAME);
+
+            final String joinThisName = rightOuter ?
+                    renamed.suffixWithOrElseGet("-outer-this-join", builder, OUTERTHIS_NAME)
+                    : renamed.suffixWithOrElseGet("-this-join", builder, JOINTHIS_NAME);
+            final String joinOtherName = leftOuter ?
+                    renamed.suffixWithOrElseGet("-outer-other-join", builder, OUTEROTHER_NAME)
+                    : renamed.suffixWithOrElseGet("-other-join", builder, JOINOTHER_NAME);
+            final String joinMergeName = renamed.suffixWithOrElseGet(
+                    "-merge", builder, MERGE_NAME);
             final StreamsGraphNode thisStreamsGraphNode = ((AbstractStream) lhs).streamsGraphNode;
             final StreamsGraphNode otherStreamsGraphNode = ((AbstractStream) other).streamsGraphNode;
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/NamedInternal.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/NamedInternal.java
index d478e9b6f64b0..532928a2d7e67 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/NamedInternal.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/NamedInternal.java
@@ -21,13 +21,22 @@
 public class NamedInternal extends Named {
 
     public static NamedInternal empty() {
-        return new NamedInternal(null);
+        return new NamedInternal((String) null);
     }
 
     public static NamedInternal with(final String name) {
         return new NamedInternal(name);
     }
 
+    /**
+     * Creates a new {@link NamedInternal} instance.
+     *
+     * @param internal  the internal name.
+     */
+    NamedInternal(final Named internal) {
+        super(internal);
+    }
+
     /**
      * Creates a new {@link NamedInternal} instance.
      *
@@ -48,6 +57,14 @@ public String name() {
     public NamedInternal withName(final String name) {
         return new NamedInternal(name);
     }
+    
+    String suffixWithOrElseGet(final String suffix, final String other) {
+        if (name != null) {
+            return name + suffix;
+        } else {
+            return other;
+        }
+    }
 
     String suffixWithOrElseGet(final String suffix, final InternalNameProvider provider, final String prefix) {
         // We actually do not need to generate processor names for operation if a name is specified.
diff --git a/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java
index 93d444bb5a32f..49477b0fb1136 100644
--- a/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java
@@ -24,10 +24,16 @@
 import org.apache.kafka.streams.errors.TopologyException;
 import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.ForeachAction;
+import org.apache.kafka.streams.kstream.JoinWindows;
+import org.apache.kafka.streams.kstream.Joined;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KTable;
 import org.apache.kafka.streams.kstream.Materialized;
+import org.apache.kafka.streams.kstream.Named;
+import org.apache.kafka.streams.kstream.Printed;
 import org.apache.kafka.streams.kstream.Produced;
+import org.apache.kafka.streams.kstream.ValueTransformer;
+import org.apache.kafka.streams.kstream.ValueTransformerWithKey;
 import org.apache.kafka.streams.processor.StateStore;
 import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder;
 import org.apache.kafka.streams.processor.internals.ProcessorNode;
@@ -39,9 +45,9 @@
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockValueJoiner;
 import org.apache.kafka.test.StreamsTestUtils;
-import org.junit.Assert;
 import org.junit.Test;
 
+import java.time.Duration;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -59,6 +65,8 @@ public class StreamsBuilderTest {
 
     private static final String STREAM_TOPIC     = "stream-topic";
 
+    private static final String STREAM_OPERATION_NAME = "stream-operation";
+
     private static final String STREAM_TOPIC_TWO = "stream-topic-two";
 
     private static final String TABLE_TOPIC      = "table-topic";
@@ -470,16 +478,243 @@ public void shouldUseSpecifiedNameForSinkProcessor() {
         assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", expected, "KSTREAM-SINK-0000000002");
     }
 
-    private void assertSpecifiedNameForOperation(final ProcessorTopology topology, final String... expected) {
+    @Test
+    public void shouldUseSpecifiedNameForMapOperation() {
+        builder.stream(STREAM_TOPIC).map(KeyValue::pair, Named.as(STREAM_OPERATION_NAME));
+        builder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
+    }
+
+    @Test
+    public void shouldUseSpecifiedNameForMapValuesOperation() {
+        builder.stream(STREAM_TOPIC).mapValues(v -> v, Named.as(STREAM_OPERATION_NAME));
+        builder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
+    }
+
+    @Test
+    public void shouldUseSpecifiedNameForMapValuesWithKeyOperation() {
+        builder.stream(STREAM_TOPIC).mapValues((k, v) -> v, Named.as(STREAM_OPERATION_NAME));
+        builder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
+    }
+
+    @Test
+    public void shouldUseSpecifiedNameForFilterOperation() {
+        builder.stream(STREAM_TOPIC).filter((k, v) -> true, Named.as(STREAM_OPERATION_NAME));
+        builder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
+    }
+
+    @Test
+    public void shouldUseSpecifiedNameForForEachOperation() {
+        builder.stream(STREAM_TOPIC).foreach((k, v) -> { }, Named.as(STREAM_OPERATION_NAME));
+        builder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
+    }
+
+    @Test
+    public void shouldUseSpecifiedNameForTransform() {
+        builder.stream(STREAM_TOPIC).transform(() -> null, Named.as(STREAM_OPERATION_NAME));
+        builder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void shouldUseSpecifiedNameForTransformValues() {
+        builder.stream(STREAM_TOPIC).transformValues(() -> (ValueTransformer) null, Named.as(STREAM_OPERATION_NAME));
+        builder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
+    }
+
+    @Test
+    @SuppressWarnings({"unchecked", "rawtypes"})
+    public void shouldUseSpecifiedNameForTransformValuesWithKey() {
+        builder.stream(STREAM_TOPIC).transformValues(() -> (ValueTransformerWithKey) null, Named.as(STREAM_OPERATION_NAME));
+        builder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
+    }
+
+    @Test
+    @SuppressWarnings({"unchecked", "rawtypes"})
+    public void shouldUseSpecifiedNameForBranchOperation() {
+        builder.stream(STREAM_TOPIC)
+                .branch(Named.as("branch-processor"), (k, v) -> true, (k, v) -> false);
+
+        builder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertSpecifiedNameForOperation(topology,
+                "KSTREAM-SOURCE-0000000000",
+                "branch-processor",
+                "branch-processor-predicate-0",
+                "branch-processor-predicate-1");
+    }
+
+    @Test
+    public void shouldUseSpecifiedNameForJoinOperationBetweenKStreamAndKTable() {
+        final KStream streamOne = builder.stream(STREAM_TOPIC);
+        final KTable streamTwo = builder.table("table-topic");
+        streamOne.join(streamTwo, (value1, value2) -> value1, Joined.as(STREAM_OPERATION_NAME));
+        builder.build();
+
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertSpecifiedNameForOperation(topology,
+                "KSTREAM-SOURCE-0000000000",
+                "KSTREAM-SOURCE-0000000002",
+                "KTABLE-SOURCE-0000000003",
+                STREAM_OPERATION_NAME);
+    }
+
+    @Test
+    public void shouldUseSpecifiedNameForLeftJoinOperationBetweenKStreamAndKTable() {
+        final KStream streamOne = builder.stream(STREAM_TOPIC);
+        final KTable streamTwo = builder.table(STREAM_TOPIC_TWO);
+        streamOne.leftJoin(streamTwo, (value1, value2) -> value1, Joined.as(STREAM_OPERATION_NAME));
+        builder.build();
+
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertSpecifiedNameForOperation(topology,
+                "KSTREAM-SOURCE-0000000000",
+                "KSTREAM-SOURCE-0000000002",
+                "KTABLE-SOURCE-0000000003",
+                STREAM_OPERATION_NAME);
+    }
+
+    @Test
+    public void shouldUseSpecifiedNameForLeftJoinOperationBetweenKStreamAndKStream() {
+        final KStream streamOne = builder.stream(STREAM_TOPIC);
+        final KStream streamTwo = builder.stream(STREAM_TOPIC_TWO);
+
+        streamOne.leftJoin(streamTwo, (value1, value2) -> value1, JoinWindows.of(Duration.ofHours(1)), Joined.as(STREAM_OPERATION_NAME));
+        builder.build();
+
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertSpecifiedNameForStateStore(topology.stateStores(),
+                STREAM_OPERATION_NAME + "-this-join-store", STREAM_OPERATION_NAME + "-outer-other-join-store"
+        );
+        assertSpecifiedNameForOperation(topology,
+                "KSTREAM-SOURCE-0000000000",
+                "KSTREAM-SOURCE-0000000001",
+                STREAM_OPERATION_NAME + "-this-windowed",
+                STREAM_OPERATION_NAME + "-other-windowed",
+                STREAM_OPERATION_NAME + "-this-join",
+                STREAM_OPERATION_NAME + "-outer-other-join",
+                STREAM_OPERATION_NAME + "-merge");
+    }
+
+    @Test
+    public void shouldUseSpecifiedNameForJoinOperationBetweenKStreamAndKStream() {
+        final KStream streamOne = builder.stream(STREAM_TOPIC);
+        final KStream streamTwo = builder.stream(STREAM_TOPIC_TWO);
+
+        streamOne.join(streamTwo, (value1, value2) -> value1, JoinWindows.of(Duration.ofHours(1)), Joined.as(STREAM_OPERATION_NAME));
+        builder.build();
+
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertSpecifiedNameForStateStore(topology.stateStores(),
+                STREAM_OPERATION_NAME + "-this-join-store",
+                STREAM_OPERATION_NAME + "-other-join-store"
+        );
+        assertSpecifiedNameForOperation(topology,
+                "KSTREAM-SOURCE-0000000000",
+                "KSTREAM-SOURCE-0000000001",
+                STREAM_OPERATION_NAME + "-this-windowed",
+                STREAM_OPERATION_NAME + "-other-windowed",
+                STREAM_OPERATION_NAME + "-this-join",
+                STREAM_OPERATION_NAME + "-other-join",
+                STREAM_OPERATION_NAME + "-merge");
+    }
+
+    @Test
+    public void shouldUseSpecifiedNameForOuterJoinOperationBetweenKStreamAndKStream() {
+        final KStream streamOne = builder.stream(STREAM_TOPIC);
+        final KStream streamTwo = builder.stream(STREAM_TOPIC_TWO);
+
+        streamOne.outerJoin(streamTwo, (value1, value2) -> value1, JoinWindows.of(Duration.ofHours(1)), Joined.as(STREAM_OPERATION_NAME));
+        builder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertSpecifiedNameForStateStore(topology.stateStores(),
+                STREAM_OPERATION_NAME + "-outer-this-join-store",
+                STREAM_OPERATION_NAME + "-outer-other-join-store");
+        assertSpecifiedNameForOperation(topology,
+                "KSTREAM-SOURCE-0000000000",
+                "KSTREAM-SOURCE-0000000001",
+                STREAM_OPERATION_NAME + "-this-windowed",
+                STREAM_OPERATION_NAME + "-other-windowed",
+                STREAM_OPERATION_NAME + "-outer-this-join",
+                STREAM_OPERATION_NAME + "-outer-other-join",
+                STREAM_OPERATION_NAME + "-merge");
+
+    }
+
+    @Test
+    public void shouldUseSpecifiedNameForMergeOperation() {
+        final String topic1 = "topic-1";
+        final String topic2 = "topic-2";
+
+        final KStream source1 = builder.stream(topic1);
+        final KStream source2 = builder.stream(topic2);
+        source1.merge(source2, Named.as("merge-processor"));
+        builder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", "KSTREAM-SOURCE-0000000001", "merge-processor");
+    }
+
+    @Test
+    public void shouldUseSpecifiedNameForProcessOperation() {
+        builder.stream(STREAM_TOPIC)
+                .process(() -> null, Named.as("test-processor"));
+
+        builder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", "test-processor");
+    }
+
+    @Test
+    public void shouldUseSpecifiedNameForPrintOperation() {
+        builder.stream(STREAM_TOPIC).print(Printed.toSysOut().withName("print-processor"));
+        builder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", "print-processor");
+    }
+
+    @Test
+    @SuppressWarnings({"unchecked", "rawtypes"})
+    public void shouldUseSpecifiedNameForFlatTransformValueOperation() {
+        builder.stream(STREAM_TOPIC).flatTransformValues(() -> (ValueTransformer) null, Named.as(STREAM_OPERATION_NAME));
+        builder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
+    }
+
+    @Test
+    @SuppressWarnings({"unchecked", "rawtypes"})
+    public void shouldUseSpecifiedNameForFlatTransformValueWithKeyOperation() {
+        builder.stream(STREAM_TOPIC).flatTransformValues(() -> (ValueTransformerWithKey) null, Named.as(STREAM_OPERATION_NAME));
+        builder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
+    }
+
+    private static void assertSpecifiedNameForOperation(final ProcessorTopology topology, final String... expected) {
         final List processors = topology.processors();
-        Assert.assertEquals("Invalid number of expected processors", expected.length, processors.size());
+        assertEquals("Invalid number of expected processors", expected.length, processors.size());
         for (int i = 0; i < expected.length; i++) {
             assertEquals(expected[i], processors.get(i).name());
         }
     }
 
-    private void assertSpecifiedNameForStateStore(final List stores, final String... expected) {
-        Assert.assertEquals("Invalid number of expected state stores", expected.length, stores.size());
+    private static void assertSpecifiedNameForStateStore(final List stores, final String... expected) {
+        assertEquals("Invalid number of expected state stores", expected.length, stores.size());
         for (int i = 0; i < expected.length; i++) {
             assertEquals(expected[i], stores.get(i).name());
         }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/RepartitionTopicNamingTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/RepartitionTopicNamingTest.java
index 3c7e8c0809dfc..e621ffc337294 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/RepartitionTopicNamingTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/RepartitionTopicNamingTest.java
@@ -550,44 +550,44 @@ public void process(final String key, final String value) {
             "      --> KTABLE-TOSTREAM-0000000011\n" +
             "      <-- KSTREAM-SOURCE-0000000041\n" +
             "    Processor: KTABLE-TOSTREAM-0000000011 (stores: [])\n" +
-            "      --> KSTREAM-SINK-0000000012, KSTREAM-WINDOWED-0000000034\n" +
+            "      --> joined-stream-other-windowed, KSTREAM-SINK-0000000012\n" +
             "      <-- KSTREAM-AGGREGATE-0000000007\n" +
             "    Processor: KSTREAM-FILTER-0000000020 (stores: [])\n" +
             "      --> KSTREAM-PEEK-0000000021\n" +
             "      <-- KSTREAM-SOURCE-0000000041\n" +
             "    Processor: KSTREAM-FILTER-0000000029 (stores: [])\n" +
-            "      --> KSTREAM-WINDOWED-0000000033\n" +
+            "      --> joined-stream-this-windowed\n" +
             "      <-- KSTREAM-SOURCE-0000000041\n" +
             "    Processor: KSTREAM-PEEK-0000000021 (stores: [])\n" +
             "      --> KSTREAM-REDUCE-0000000023\n" +
             "      <-- KSTREAM-FILTER-0000000020\n" +
-            "    Processor: KSTREAM-WINDOWED-0000000033 (stores: [KSTREAM-JOINTHIS-0000000035-store])\n" +
-            "      --> KSTREAM-JOINTHIS-0000000035\n" +
-            "      <-- KSTREAM-FILTER-0000000029\n" +
-            "    Processor: KSTREAM-WINDOWED-0000000034 (stores: [KSTREAM-JOINOTHER-0000000036-store])\n" +
-            "      --> KSTREAM-JOINOTHER-0000000036\n" +
+            "    Processor: joined-stream-other-windowed (stores: [joined-stream-other-join-store])\n" +
+            "      --> joined-stream-other-join\n" +
             "      <-- KTABLE-TOSTREAM-0000000011\n" +
+            "    Processor: joined-stream-this-windowed (stores: [joined-stream-this-join-store])\n" +
+            "      --> joined-stream-this-join\n" +
+            "      <-- KSTREAM-FILTER-0000000029\n" +
             "    Processor: KSTREAM-AGGREGATE-0000000014 (stores: [KSTREAM-AGGREGATE-STATE-STORE-0000000013])\n" +
             "      --> KTABLE-TOSTREAM-0000000018\n" +
             "      <-- KSTREAM-SOURCE-0000000041\n" +
-            "    Processor: KSTREAM-JOINOTHER-0000000036 (stores: [KSTREAM-JOINTHIS-0000000035-store])\n" +
-            "      --> KSTREAM-MERGE-0000000037\n" +
-            "      <-- KSTREAM-WINDOWED-0000000034\n" +
-            "    Processor: KSTREAM-JOINTHIS-0000000035 (stores: [KSTREAM-JOINOTHER-0000000036-store])\n" +
-            "      --> KSTREAM-MERGE-0000000037\n" +
-            "      <-- KSTREAM-WINDOWED-0000000033\n" +
             "    Processor: KSTREAM-REDUCE-0000000023 (stores: [KSTREAM-REDUCE-STATE-STORE-0000000022])\n" +
             "      --> KTABLE-TOSTREAM-0000000027\n" +
             "      <-- KSTREAM-PEEK-0000000021\n" +
-            "    Processor: KSTREAM-MERGE-0000000037 (stores: [])\n" +
-            "      --> KSTREAM-SINK-0000000038\n" +
-            "      <-- KSTREAM-JOINTHIS-0000000035, KSTREAM-JOINOTHER-0000000036\n" +
+            "    Processor: joined-stream-other-join (stores: [joined-stream-this-join-store])\n" +
+            "      --> joined-stream-merge\n" +
+            "      <-- joined-stream-other-windowed\n" +
+            "    Processor: joined-stream-this-join (stores: [joined-stream-other-join-store])\n" +
+            "      --> joined-stream-merge\n" +
+            "      <-- joined-stream-this-windowed\n" +
             "    Processor: KTABLE-TOSTREAM-0000000018 (stores: [])\n" +
             "      --> KSTREAM-SINK-0000000019\n" +
             "      <-- KSTREAM-AGGREGATE-0000000014\n" +
             "    Processor: KTABLE-TOSTREAM-0000000027 (stores: [])\n" +
             "      --> KSTREAM-SINK-0000000028\n" +
             "      <-- KSTREAM-REDUCE-0000000023\n" +
+            "    Processor: joined-stream-merge (stores: [])\n" +
+            "      --> KSTREAM-SINK-0000000038\n" +
+            "      <-- joined-stream-this-join, joined-stream-other-join\n" +
             "    Sink: KSTREAM-SINK-0000000012 (topic: outputTopic_0)\n" +
             "      <-- KTABLE-TOSTREAM-0000000011\n" +
             "    Sink: KSTREAM-SINK-0000000019 (topic: outputTopic_1)\n" +
@@ -595,7 +595,7 @@ public void process(final String key, final String value) {
             "    Sink: KSTREAM-SINK-0000000028 (topic: outputTopic_2)\n" +
             "      <-- KTABLE-TOSTREAM-0000000027\n" +
             "    Sink: KSTREAM-SINK-0000000038 (topic: outputTopicForJoin)\n" +
-            "      <-- KSTREAM-MERGE-0000000037\n\n";
+            "      <-- joined-stream-merge\n\n";
 
 
     private static final String EXPECTED_UNOPTIMIZED_TOPOLOGY = "Topologies:\n" +
@@ -651,29 +651,29 @@ public void process(final String key, final String value) {
             "      --> KTABLE-TOSTREAM-0000000011\n" +
             "      <-- KSTREAM-SOURCE-0000000010\n" +
             "    Processor: KTABLE-TOSTREAM-0000000011 (stores: [])\n" +
-            "      --> KSTREAM-SINK-0000000012, KSTREAM-WINDOWED-0000000034\n" +
+            "      --> KSTREAM-SINK-0000000012, joined-stream-other-windowed\n" +
             "      <-- KSTREAM-AGGREGATE-0000000007\n" +
             "    Source: KSTREAM-SOURCE-0000000032 (topics: [joined-stream-left-repartition])\n" +
-            "      --> KSTREAM-WINDOWED-0000000033\n" +
-            "    Processor: KSTREAM-WINDOWED-0000000033 (stores: [KSTREAM-JOINTHIS-0000000035-store])\n" +
-            "      --> KSTREAM-JOINTHIS-0000000035\n" +
-            "      <-- KSTREAM-SOURCE-0000000032\n" +
-            "    Processor: KSTREAM-WINDOWED-0000000034 (stores: [KSTREAM-JOINOTHER-0000000036-store])\n" +
-            "      --> KSTREAM-JOINOTHER-0000000036\n" +
+            "      --> joined-stream-this-windowed\n" +
+            "    Processor: joined-stream-other-windowed (stores: [joined-stream-other-join-store])\n" +
+            "      --> joined-stream-other-join\n" +
             "      <-- KTABLE-TOSTREAM-0000000011\n" +
-            "    Processor: KSTREAM-JOINOTHER-0000000036 (stores: [KSTREAM-JOINTHIS-0000000035-store])\n" +
-            "      --> KSTREAM-MERGE-0000000037\n" +
-            "      <-- KSTREAM-WINDOWED-0000000034\n" +
-            "    Processor: KSTREAM-JOINTHIS-0000000035 (stores: [KSTREAM-JOINOTHER-0000000036-store])\n" +
-            "      --> KSTREAM-MERGE-0000000037\n" +
-            "      <-- KSTREAM-WINDOWED-0000000033\n" +
-            "    Processor: KSTREAM-MERGE-0000000037 (stores: [])\n" +
+            "    Processor: joined-stream-this-windowed (stores: [joined-stream-this-join-store])\n" +
+            "      --> joined-stream-this-join\n" +
+            "      <-- KSTREAM-SOURCE-0000000032\n" +
+            "    Processor: joined-stream-other-join (stores: [joined-stream-this-join-store])\n" +
+            "      --> joined-stream-merge\n" +
+            "      <-- joined-stream-other-windowed\n" +
+            "    Processor: joined-stream-this-join (stores: [joined-stream-other-join-store])\n" +
+            "      --> joined-stream-merge\n" +
+            "      <-- joined-stream-this-windowed\n" +
+            "    Processor: joined-stream-merge (stores: [])\n" +
             "      --> KSTREAM-SINK-0000000038\n" +
-            "      <-- KSTREAM-JOINTHIS-0000000035, KSTREAM-JOINOTHER-0000000036\n" +
+            "      <-- joined-stream-this-join, joined-stream-other-join\n" +
             "    Sink: KSTREAM-SINK-0000000012 (topic: outputTopic_0)\n" +
             "      <-- KTABLE-TOSTREAM-0000000011\n" +
             "    Sink: KSTREAM-SINK-0000000038 (topic: outputTopicForJoin)\n" +
-            "      <-- KSTREAM-MERGE-0000000037\n" +
+            "      <-- joined-stream-merge\n" +
             "\n" +
             "  Sub-topology: 2\n" +
             "    Source: KSTREAM-SOURCE-0000000017 (topics: [aggregate-stream-repartition])\n" +

From 529a5a502e5fc4d05536d23df0bd75973affff01 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Fri, 31 May 2019 17:02:40 -0700
Subject: [PATCH 0316/1071] MINOR: Reordering the props modification with
 configs construction

Reviewers: Randall Hauch , Matthias J. Sax ,  Bill Bejeck 
---
 .../apache/kafka/streams/StreamsConfigTest.java   | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java b/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
index 5f053bca1ca62..aa3860e8bc250 100644
--- a/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
@@ -152,6 +152,7 @@ public void consumerConfigShouldContainAdminClientConfigsForRetriesAndRetryBackO
     public void testGetMainConsumerConfigsWithMainConsumerOverridenPrefix() {
         props.put(StreamsConfig.consumerPrefix(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), "5");
         props.put(StreamsConfig.mainConsumerPrefix(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), "50");
+        final StreamsConfig streamsConfig = new StreamsConfig(props);
         final Map returnedProps = streamsConfig.getMainConsumerConfigs(groupId, clientId, threadIdx);
         assertEquals("50", returnedProps.get(ConsumerConfig.MAX_POLL_RECORDS_CONFIG));
     }
@@ -215,24 +216,24 @@ public void shouldSupportPrefixedRestoreConsumerConfigs() {
 
     @Test
     public void shouldSupportPrefixedPropertiesThatAreNotPartOfConsumerConfig() {
-        final StreamsConfig streamsConfig = new StreamsConfig(props);
         props.put(consumerPrefix("interceptor.statsd.host"), "host");
+        final StreamsConfig streamsConfig = new StreamsConfig(props);
         final Map consumerConfigs = streamsConfig.getMainConsumerConfigs(groupId, clientId, threadIdx);
         assertEquals("host", consumerConfigs.get("interceptor.statsd.host"));
     }
 
     @Test
     public void shouldSupportPrefixedPropertiesThatAreNotPartOfRestoreConsumerConfig() {
-        final StreamsConfig streamsConfig = new StreamsConfig(props);
         props.put(consumerPrefix("interceptor.statsd.host"), "host");
+        final StreamsConfig streamsConfig = new StreamsConfig(props);
         final Map consumerConfigs = streamsConfig.getRestoreConsumerConfigs(clientId);
         assertEquals("host", consumerConfigs.get("interceptor.statsd.host"));
     }
 
     @Test
     public void shouldSupportPrefixedPropertiesThatAreNotPartOfProducerConfig() {
-        final StreamsConfig streamsConfig = new StreamsConfig(props);
         props.put(producerPrefix("interceptor.statsd.host"), "host");
+        final StreamsConfig streamsConfig = new StreamsConfig(props);
         final Map producerConfigs = streamsConfig.getProducerConfigs(clientId);
         assertEquals("host", producerConfigs.get("interceptor.statsd.host"));
     }
@@ -277,8 +278,8 @@ public void shouldSupportNonPrefixedProducerConfigs() {
 
     @Test
     public void shouldForwardCustomConfigsWithNoPrefixToAllClients() {
-        final StreamsConfig streamsConfig = new StreamsConfig(props);
         props.put("custom.property.host", "host");
+        final StreamsConfig streamsConfig = new StreamsConfig(props);
         final Map consumerConfigs = streamsConfig.getMainConsumerConfigs(groupId, clientId, threadIdx);
         final Map restoreConsumerConfigs = streamsConfig.getRestoreConsumerConfigs(clientId);
         final Map producerConfigs = streamsConfig.getProducerConfigs(clientId);
@@ -291,11 +292,11 @@ public void shouldForwardCustomConfigsWithNoPrefixToAllClients() {
 
     @Test
     public void shouldOverrideNonPrefixedCustomConfigsWithPrefixedConfigs() {
-        final StreamsConfig streamsConfig = new StreamsConfig(props);
         props.put("custom.property.host", "host0");
         props.put(consumerPrefix("custom.property.host"), "host1");
         props.put(producerPrefix("custom.property.host"), "host2");
         props.put(adminClientPrefix("custom.property.host"), "host3");
+        final StreamsConfig streamsConfig = new StreamsConfig(props);
         final Map consumerConfigs = streamsConfig.getMainConsumerConfigs(groupId, clientId, threadIdx);
         final Map restoreConsumerConfigs = streamsConfig.getRestoreConsumerConfigs(clientId);
         final Map producerConfigs = streamsConfig.getProducerConfigs(clientId);
@@ -374,6 +375,7 @@ public void shouldResetToDefaultIfRestoreConsumerAutoCommitIsOverridden() {
     public void testGetRestoreConsumerConfigsWithRestoreConsumerOverridenPrefix() {
         props.put(StreamsConfig.consumerPrefix(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), "5");
         props.put(StreamsConfig.restoreConsumerPrefix(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), "50");
+        final StreamsConfig streamsConfig = new StreamsConfig(props);
         final Map returnedProps = streamsConfig.getRestoreConsumerConfigs(clientId);
         assertEquals("50", returnedProps.get(ConsumerConfig.MAX_POLL_RECORDS_CONFIG));
     }
@@ -395,8 +397,8 @@ public void shouldSupportPrefixedGlobalConsumerConfigs() {
 
     @Test
     public void shouldSupportPrefixedPropertiesThatAreNotPartOfGlobalConsumerConfig() {
-        final StreamsConfig streamsConfig = new StreamsConfig(props);
         props.put(consumerPrefix("interceptor.statsd.host"), "host");
+        final StreamsConfig streamsConfig = new StreamsConfig(props);
         final Map consumerConfigs = streamsConfig.getGlobalConsumerConfigs(clientId);
         assertEquals("host", consumerConfigs.get("interceptor.statsd.host"));
     }
@@ -421,6 +423,7 @@ public void shouldResetToDefaultIfGlobalConsumerAutoCommitIsOverridden() {
     public void testGetGlobalConsumerConfigsWithGlobalConsumerOverridenPrefix() {
         props.put(StreamsConfig.consumerPrefix(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), "5");
         props.put(StreamsConfig.globalConsumerPrefix(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), "50");
+        final StreamsConfig streamsConfig = new StreamsConfig(props);
         final Map returnedProps = streamsConfig.getGlobalConsumerConfigs(clientId);
         assertEquals("50", returnedProps.get(ConsumerConfig.MAX_POLL_RECORDS_CONFIG));
     }

From 87d493f07219a215816783557a5981a74f192f06 Mon Sep 17 00:00:00 2001
From: Boyang Chen 
Date: Sat, 1 Jun 2019 19:12:58 -0700
Subject: [PATCH 0317/1071] KAFKA-8446: Kafka Streams restoration crashes with
 NPE when the record value is null (#6842)

When the restored record value is null, we are in danger of NPE during restoration phase.

Reviewers: Bill Bejeck , Guozhang Wang , Matthias J. Sax 
---
 .../state/internals/RecordConverters.java     |  11 +-
 .../StateRestorationIntegrationTest.java      | 122 ++++++++++++++++++
 .../state/internals/RecordConvertersTest.java |  49 +++++++
 3 files changed, 177 insertions(+), 5 deletions(-)
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/integration/StateRestorationIntegrationTest.java
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/state/internals/RecordConvertersTest.java

diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RecordConverters.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RecordConverters.java
index f65cc322bb009..8305d5216afc3 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RecordConverters.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RecordConverters.java
@@ -27,6 +27,11 @@ public final class RecordConverters {
     private static final RecordConverter RAW_TO_TIMESTAMED_INSTANCE = record -> {
         final byte[] rawValue = record.value();
         final long timestamp = record.timestamp();
+        final byte[] recordValue = rawValue == null ? null :
+            ByteBuffer.allocate(8 + rawValue.length)
+                .putLong(timestamp)
+                .put(rawValue)
+                .array();
         return new ConsumerRecord<>(
             record.topic(),
             record.partition(),
@@ -37,11 +42,7 @@ public final class RecordConverters {
             record.serializedKeySize(),
             record.serializedValueSize(),
             record.key(),
-            ByteBuffer
-                .allocate(8 + rawValue.length)
-                .putLong(timestamp)
-                .put(rawValue)
-                .array(),
+            recordValue,
             record.headers(),
             record.leaderEpoch()
         );
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/StateRestorationIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/StateRestorationIntegrationTest.java
new file mode 100644
index 0000000000000..e22ff4f76597d
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/StateRestorationIntegrationTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.integration;
+
+import kafka.utils.MockTime;
+import org.apache.kafka.common.serialization.BytesDeserializer;
+import org.apache.kafka.common.serialization.BytesSerializer;
+import org.apache.kafka.common.serialization.IntegerDeserializer;
+import org.apache.kafka.common.serialization.IntegerSerializer;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
+import org.apache.kafka.streams.integration.utils.IntegrationTestUtils;
+import org.apache.kafka.streams.kstream.Materialized;
+import org.apache.kafka.streams.state.Stores;
+import org.apache.kafka.test.IntegrationTest;
+import org.apache.kafka.test.StreamsTestUtils;
+import org.apache.kafka.test.TestUtils;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.ExecutionException;
+
+@Category({IntegrationTest.class})
+public class StateRestorationIntegrationTest {
+    private StreamsBuilder builder = new StreamsBuilder();
+
+    private static final String APPLICATION_ID = "restoration-test-app";
+    private static final String STATE_STORE_NAME = "stateStore";
+    private static final String INPUT_TOPIC = "input";
+    private static final String OUTPUT_TOPIC = "output";
+
+    private Properties streamsConfiguration;
+
+    @ClassRule
+    public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(1);
+    private final MockTime mockTime = CLUSTER.time;
+
+    @Before
+    public void setUp() throws Exception {
+        final Properties props = new Properties();
+
+        streamsConfiguration = StreamsTestUtils.getStreamsConfig(
+                APPLICATION_ID,
+                CLUSTER.bootstrapServers(),
+                Serdes.Integer().getClass().getName(),
+                Serdes.ByteArray().getClass().getName(),
+                props);
+
+        CLUSTER.createTopics(INPUT_TOPIC);
+        CLUSTER.createTopics(OUTPUT_TOPIC);
+
+        IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration);
+    }
+
+    @Test
+    public void shouldRestoreNullRecord() throws InterruptedException, ExecutionException {
+        builder.table(INPUT_TOPIC, Materialized.as(
+                Stores.persistentTimestampedKeyValueStore(STATE_STORE_NAME))
+                .withKeySerde(Serdes.Integer())
+                .withValueSerde(Serdes.Bytes())
+                .withCachingDisabled()).toStream().to(OUTPUT_TOPIC);
+
+        final Properties producerConfig = TestUtils.producerConfig(
+                CLUSTER.bootstrapServers(), IntegerSerializer.class, BytesSerializer.class);
+
+        final List> initialKeyValues = Arrays.asList(
+                KeyValue.pair(3, new Bytes(new byte[]{3})),
+                KeyValue.pair(3, null),
+                KeyValue.pair(1, new Bytes(new byte[]{1})));
+
+        IntegrationTestUtils.produceKeyValuesSynchronously(
+                INPUT_TOPIC, initialKeyValues, producerConfig, mockTime);
+
+        KafkaStreams streams = new KafkaStreams(builder.build(streamsConfiguration), streamsConfiguration);
+        streams.start();
+
+        final Properties consumerConfig = TestUtils.consumerConfig(
+                CLUSTER.bootstrapServers(), IntegerDeserializer.class, BytesDeserializer.class);
+
+        IntegrationTestUtils.waitUntilFinalKeyValueRecordsReceived(
+                consumerConfig, OUTPUT_TOPIC, initialKeyValues);
+
+        // wipe out state store to trigger restore process on restart
+        streams.close();
+        streams.cleanUp();
+
+        // Restart the stream instance. There should not be exception handling the null value within changelog topic.
+        final List> newKeyValues =
+                Collections.singletonList(KeyValue.pair(2, new Bytes(new byte[3])));
+        IntegrationTestUtils.produceKeyValuesSynchronously(
+                INPUT_TOPIC, newKeyValues, producerConfig, mockTime);
+        streams = new KafkaStreams(builder.build(streamsConfiguration), streamsConfiguration);
+        streams.start();
+        IntegrationTestUtils.waitUntilFinalKeyValueRecordsReceived(
+                consumerConfig, OUTPUT_TOPIC, newKeyValues);
+        streams.close();
+    }
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RecordConvertersTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RecordConvertersTest.java
new file mode 100644
index 0000000000000..bacbacd2fed01
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RecordConvertersTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.common.record.TimestampType;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+
+import static org.apache.kafka.streams.state.internals.RecordConverters.rawValueToTimestampedValue;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertNull;
+
+public class RecordConvertersTest {
+
+    private final RecordConverter timestampedValueConverter = rawValueToTimestampedValue();
+
+    @Test
+    public void shouldPreserveNullValueOnConversion() {
+        final ConsumerRecord nullValueRecord = new ConsumerRecord<>("", 0, 0L, new byte[0], null);
+        assertNull(timestampedValueConverter.convert(nullValueRecord).value());
+    }
+
+    @Test
+    public void shouldAddTimestampToValueOnConversionWhenValueIsNotNull() {
+        final long timestamp = 10L;
+        final byte[] value = new byte[1];
+        final ConsumerRecord inputRecord = new ConsumerRecord<>(
+                "topic", 1, 0, timestamp, TimestampType.CREATE_TIME, 0L, 0, 0, new byte[0], value);
+        final byte[] expectedValue = ByteBuffer.allocate(9).putLong(timestamp).put(value).array();
+        final byte[] actualValue = timestampedValueConverter.convert(inputRecord).value();
+        assertArrayEquals(expectedValue, actualValue);
+    }
+}

From 17345b3be527a3e4808440fceb1179a1b9f9e684 Mon Sep 17 00:00:00 2001
From: Konstantine Karantasis 
Date: Sun, 2 Jun 2019 10:19:19 -0700
Subject: [PATCH 0318/1071] KAFKA-8463: Fix redundant reassignment of tasks
 when leader worker leaves (#6859)

Author: Konstantine Karantasis 
Reviewer: Randall Hauch 
---
 .../IncrementalCooperativeAssignor.java       |   4 +-
 .../IncrementalCooperativeAssignorTest.java   | 103 ++++++++++++++++--
 2 files changed, 94 insertions(+), 13 deletions(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java
index ae36837912f31..8e56ca8120f7e 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java
@@ -195,8 +195,8 @@ protected Map performTaskAssignment(String leaderId, long ma
         log.debug("Lost assignments: {}", lostAssignments);
 
         // Derived set: The set of new connectors-and-tasks is a derived set from the set
-        // difference of configured - previous
-        ConnectorsAndTasks newSubmissions = diff(configured, previousAssignment);
+        // difference of configured - previous - active
+        ConnectorsAndTasks newSubmissions = diff(configured, previousAssignment, activeAssignments);
         log.debug("New assignments: {}", newSubmissions);
 
         // A collection of the complete assignment
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java
index 71ccefdf52c44..7085a7f15d429 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java
@@ -120,6 +120,7 @@ public void testTaskAssignmentWhenWorkerJoins() {
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(2, 8, 0, 0, "worker1");
 
         // Second assignment with a second worker joining and all connectors running on previous worker
@@ -131,6 +132,7 @@ public void testTaskAssignmentWhenWorkerJoins() {
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(0, 0, 1, 4, "worker1", "worker2");
 
         // Third assignment after revocations
@@ -141,6 +143,7 @@ public void testTaskAssignmentWhenWorkerJoins() {
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(1, 4, 0, 0, "worker1", "worker2");
 
         // A fourth rebalance should not change assignments
@@ -151,6 +154,7 @@ public void testTaskAssignmentWhenWorkerJoins() {
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(0, 0, 0, 0, "worker1", "worker2");
 
         verify(coordinator, times(rebalanceNum)).configSnapshot();
@@ -173,6 +177,7 @@ public void testTaskAssignmentWhenWorkerLeavesPermanently() {
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(2, 8, 0, 0, "worker1", "worker2");
 
         // Second assignment with only one worker remaining in the group. The worker that left the
@@ -186,6 +191,7 @@ public void testTaskAssignmentWhenWorkerLeavesPermanently() {
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(rebalanceDelay, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(0, 0, 0, 0, "worker1");
 
         time.sleep(rebalanceDelay / 2);
@@ -199,6 +205,7 @@ public void testTaskAssignmentWhenWorkerLeavesPermanently() {
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(rebalanceDelay / 2, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(0, 0, 0, 0, "worker1");
 
         time.sleep(rebalanceDelay / 2 + 1);
@@ -211,6 +218,7 @@ public void testTaskAssignmentWhenWorkerLeavesPermanently() {
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(1, 4, 0, 0, "worker1");
 
         verify(coordinator, times(rebalanceNum)).configSnapshot();
@@ -233,6 +241,7 @@ public void testTaskAssignmentWhenWorkerBounces() {
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(2, 8, 0, 0, "worker1", "worker2");
 
         // Second assignment with only one worker remaining in the group. The worker that left the
@@ -246,6 +255,7 @@ public void testTaskAssignmentWhenWorkerBounces() {
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(rebalanceDelay, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(0, 0, 0, 0, "worker1");
 
         time.sleep(rebalanceDelay / 2);
@@ -259,6 +269,7 @@ public void testTaskAssignmentWhenWorkerBounces() {
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(rebalanceDelay / 2, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(0, 0, 0, 0, "worker1");
 
         time.sleep(rebalanceDelay / 4);
@@ -273,6 +284,7 @@ public void testTaskAssignmentWhenWorkerBounces() {
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(rebalanceDelay / 4, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(0, 0, 0, 0, "worker1", "worker2");
 
         time.sleep(rebalanceDelay / 4);
@@ -286,6 +298,7 @@ public void testTaskAssignmentWhenWorkerBounces() {
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(1, 4, 0, 0, "worker1", "worker2");
 
         verify(coordinator, times(rebalanceNum)).configSnapshot();
@@ -301,16 +314,18 @@ public void testTaskAssignmentWhenLeaderLeavesPermanently() {
         when(coordinator.configSnapshot()).thenReturn(configState);
         doReturn(Collections.EMPTY_MAP).when(assignor).serializeAssignments(assignmentsCapture.capture());
 
-        // First assignment with 2 workers and 2 connectors configured but not yet assigned
+        // First assignment with 3 workers and 2 connectors configured but not yet assigned
         memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, null));
+        memberConfigs.put("worker3", new ExtendedWorkerState(leaderUrl, offset, null));
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
-        assertAssignment(2, 8, 0, 0, "worker1", "worker2");
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
+        assertAssignment(2, 8, 0, 0, "worker1", "worker2", "worker3");
 
-        // Second assignment with only one worker remaining in the group. The worker that left the
+        // Second assignment with two workers remaining in the group. The worker that left the
         // group was the leader. The new leader has no previous assignments and is not tracking a
         // delay upon a leader's exit
         applyAssignments(returnedAssignments);
@@ -328,7 +343,8 @@ public void testTaskAssignmentWhenLeaderLeavesPermanently() {
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
-        assertAssignment(1, 4, 0, 0, "worker2");
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
+        assertAssignment(1, 3, 0, 0, "worker2", "worker3");
 
         // Third (incidental) assignment with still only one worker in the group.
         applyAssignments(returnedAssignments);
@@ -337,7 +353,8 @@ public void testTaskAssignmentWhenLeaderLeavesPermanently() {
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
-        assertAssignment(0, 0, 0, 0, "worker2");
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
+        assertAssignment(0, 0, 0, 0, "worker2", "worker3");
 
         verify(coordinator, times(rebalanceNum)).configSnapshot();
         verify(coordinator, times(rebalanceNum)).leaderState(any());
@@ -352,16 +369,18 @@ public void testTaskAssignmentWhenLeaderBounces() {
         when(coordinator.configSnapshot()).thenReturn(configState);
         doReturn(Collections.EMPTY_MAP).when(assignor).serializeAssignments(assignmentsCapture.capture());
 
-        // First assignment with 2 workers and 2 connectors configured but not yet assigned
+        // First assignment with 3 workers and 2 connectors configured but not yet assigned
         memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, null));
+        memberConfigs.put("worker3", new ExtendedWorkerState(leaderUrl, offset, null));
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
-        assertAssignment(2, 8, 0, 0, "worker1", "worker2");
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
+        assertAssignment(2, 8, 0, 0, "worker1", "worker2", "worker3");
 
-        // Second assignment with only one worker remaining in the group. The worker that left the
+        // Second assignment with two workers remaining in the group. The worker that left the
         // group was the leader. The new leader has no previous assignments and is not tracking a
         // delay upon a leader's exit
         applyAssignments(returnedAssignments);
@@ -379,7 +398,8 @@ public void testTaskAssignmentWhenLeaderBounces() {
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
-        assertAssignment(1, 4, 0, 0, "worker2");
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
+        assertAssignment(1, 3, 0, 0, "worker2", "worker3");
 
         // Third assignment with the previous leader returning as a follower. In this case, the
         // arrival of the previous leader is treated as an arrival of a new worker. Reassignment
@@ -391,7 +411,8 @@ public void testTaskAssignmentWhenLeaderBounces() {
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
-        assertAssignment(0, 0, 1, 4, "worker1", "worker2");
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
+        assertAssignment(0, 0, 0, 2, "worker1", "worker2", "worker3");
 
         // Fourth assignment after revocations
         applyAssignments(returnedAssignments);
@@ -401,7 +422,8 @@ public void testTaskAssignmentWhenLeaderBounces() {
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
-        assertAssignment(1, 4, 0, 0, "worker1", "worker2");
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
+        assertAssignment(0, 2, 0, 0, "worker1", "worker2", "worker3");
 
         verify(coordinator, times(rebalanceNum)).configSnapshot();
         verify(coordinator, times(rebalanceNum)).leaderState(any());
@@ -429,6 +451,7 @@ public void testTaskAssignmentWhenFirstAssignmentAttemptFails() {
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
         // This was the assignment that should have been sent, but didn't make it after all the way
         assertDelay(0, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(2, 8, 0, 0, "worker1", "worker2");
 
         // Second assignment happens with members returning the same assignments (memberConfigs)
@@ -441,6 +464,7 @@ public void testTaskAssignmentWhenFirstAssignmentAttemptFails() {
         returnedAssignments = assignmentsCapture.getValue();
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
         assertDelay(rebalanceDelay, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(0, 0, 0, 0, "worker1", "worker2");
 
         time.sleep(rebalanceDelay / 2);
@@ -454,6 +478,7 @@ public void testTaskAssignmentWhenFirstAssignmentAttemptFails() {
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(rebalanceDelay / 2, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(0, 0, 0, 0, "worker1", "worker2");
 
         time.sleep(rebalanceDelay / 2 + 1);
@@ -466,6 +491,7 @@ public void testTaskAssignmentWhenFirstAssignmentAttemptFails() {
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(2, 8, 0, 0, "worker1", "worker2");
 
         verify(coordinator, times(rebalanceNum)).configSnapshot();
@@ -488,6 +514,7 @@ public void testTaskAssignmentWhenSubsequentAssignmentAttemptFails() {
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(2, 8, 0, 0, "worker1", "worker2");
 
         when(coordinator.configSnapshot()).thenReturn(configState);
@@ -509,6 +536,7 @@ public void testTaskAssignmentWhenSubsequentAssignmentAttemptFails() {
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
         // This was the assignment that should have been sent, but didn't make it after all the way
         assertDelay(0, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(0, 0, 0, 2, "worker1", "worker2", "worker3");
 
         // Third assignment happens with members returning the same assignments (memberConfigs)
@@ -519,6 +547,7 @@ public void testTaskAssignmentWhenSubsequentAssignmentAttemptFails() {
         returnedAssignments = assignmentsCapture.getValue();
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
         assertDelay(0, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(0, 0, 0, 2, "worker1", "worker2", "worker3");
 
         verify(coordinator, times(rebalanceNum)).configSnapshot();
@@ -538,6 +567,7 @@ public void testTaskAssignmentWhenConnectorsAreDeleted() {
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(3, 12, 0, 0, "worker1", "worker2");
 
         // Second assignment with an updated config state that reflects removal of a connector
@@ -550,6 +580,7 @@ public void testTaskAssignmentWhenConnectorsAreDeleted() {
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(0, 0, 1, 4, "worker1", "worker2");
 
         verify(coordinator, times(rebalanceNum)).configSnapshot();
@@ -1076,4 +1107,54 @@ private void assertDelay(int expectedDelay, Map newA
                 .forEach(a -> assertEquals(
                         "Wrong rebalance delay in " + a, expectedDelay, a.delay()));
     }
+
+    private void assertNoReassignments(Map existingAssignments,
+                                       Map newAssignments) {
+        assertNoDuplicateInAssignment(existingAssignments);
+        assertNoDuplicateInAssignment(newAssignments);
+
+        List existingConnectors = existingAssignments.values().stream()
+                .flatMap(a -> a.assignment().connectors().stream())
+                .collect(Collectors.toList());
+        List newConnectors = newAssignments.values().stream()
+                .flatMap(a -> a.assignment().connectors().stream())
+                .collect(Collectors.toList());
+
+        List existingTasks = existingAssignments.values().stream()
+                .flatMap(a -> a.assignment().tasks().stream())
+                .collect(Collectors.toList());
+
+        List newTasks = newAssignments.values().stream()
+                .flatMap(a -> a.assignment().tasks().stream())
+                .collect(Collectors.toList());
+
+        existingConnectors.retainAll(newConnectors);
+        assertThat("Found connectors in new assignment that already exist in current assignment",
+                Collections.emptyList(),
+                is(existingConnectors));
+        existingTasks.retainAll(newTasks);
+        assertThat("Found tasks in new assignment that already exist in current assignment",
+                Collections.emptyList(),
+                is(existingConnectors));
+    }
+
+    private void assertNoDuplicateInAssignment(Map existingAssignment) {
+        List existingConnectors = existingAssignment.values().stream()
+                .flatMap(a -> a.assignment().connectors().stream())
+                .collect(Collectors.toList());
+        Set existingUniqueConnectors = new HashSet<>(existingConnectors);
+        existingConnectors.removeAll(existingUniqueConnectors);
+        assertThat("Connectors should be unique in assignments but duplicates where found",
+                Collections.emptyList(),
+                is(existingConnectors));
+
+        List existingTasks = existingAssignment.values().stream()
+                .flatMap(a -> a.assignment().tasks().stream())
+                .collect(Collectors.toList());
+        Set existingUniqueTasks = new HashSet<>(existingTasks);
+        existingTasks.removeAll(existingUniqueTasks);
+        assertThat("Tasks should be unique in assignments but duplicates where found",
+                Collections.emptyList(),
+                is(existingTasks));
+    }
 }

From 2c810e4afb1b41ec1f8565d9a830d479b29dc708 Mon Sep 17 00:00:00 2001
From: tadsul <43974298+tadsul@users.noreply.github.com>
Date: Mon, 3 Jun 2019 04:43:11 -0700
Subject: [PATCH 0319/1071] KAFKA-8425: Fix for correctly handling immutable
 maps (KIP-421 bug) (#6795)

Since the originals map passed to AbstractConfig constructor may be immutable, avoid updating this map while resolving indirect config variables. Instead a new ResolvingMap instance is now used to store resolved configs.

Reviewers: Randall Hauch , Boyang Chen , Rajini Sivaram 
---
 .../kafka/common/config/AbstractConfig.java   | 37 +++++++++++++++++--
 .../common/config/AbstractConfigTest.java     | 14 +++++++
 gradle/spotbugs-exclude.xml                   |  7 ++++
 3 files changed, 54 insertions(+), 4 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java b/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java
index 555b634ec251e..13865d0fca8b4 100644
--- a/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java
+++ b/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java
@@ -103,7 +103,6 @@ public AbstractConfig(ConfigDef definition, Map originals,  Map
                 throw new ConfigException(entry.getKey().toString(), entry.getValue(), "Key must be a string.");
 
         this.originals = resolveConfigVariables(configProviderProps, (Map) originals);
-
         this.values = definition.parse(this.originals);
         Map configUpdates = postProcessParsedConfig(Collections.unmodifiableMap(this.values));
         for (Map.Entry update : configUpdates.entrySet()) {
@@ -459,10 +458,11 @@ private Map extractPotentialVariables(Map  configMap) {
     private  Map resolveConfigVariables(Map configProviderProps, Map originals) {
         Map providerConfigString;
         Map configProperties;
-
+        Map resolvedOriginals = new HashMap<>();
         // As variable configs are strings, parse the originals and obtain the potential variable configs.
         Map indirectVariables = extractPotentialVariables(originals);
 
+        resolvedOriginals.putAll(originals);
         if (configProviderProps == null || configProviderProps.isEmpty()) {
             providerConfigString = indirectVariables;
             configProperties = originals;
@@ -475,10 +475,12 @@ private Map extractPotentialVariables(Map  configMap) {
         if (!providers.isEmpty()) {
             ConfigTransformer configTransformer = new ConfigTransformer(providers);
             ConfigTransformerResult result = configTransformer.transform(indirectVariables);
-            originals.putAll(result.data());
+            if (!result.data().isEmpty()) {
+                resolvedOriginals.putAll(result.data());
+            }
         }
 
-        return originals;
+        return new ResolvingMap<>(resolvedOriginals, originals);
     }
 
     private Map configProviderProperties(String configProviderPrefix, Map providerConfigProperties) {
@@ -585,4 +587,31 @@ public V get(Object key) {
             return super.get(key);
         }
     }
+
+    /**
+     * ResolvingMap keeps a track of the original map instance and the resolved configs.
+     * The originals are tracked in a separate nested map and may be a `RecordingMap`; thus
+     * any access to a value for a key needs to be recorded on the originals map.
+     * The resolved configs are kept in the inherited map and are therefore mutable, though any
+     * mutations are not applied to the originals.
+     */
+    private static class ResolvingMap extends HashMap {
+
+        private final Map originals;
+
+        ResolvingMap(Map resolved, Map originals) {
+            super(resolved);
+            this.originals = Collections.unmodifiableMap(originals);
+        }
+
+        @Override
+        public V get(Object key) {
+            if (key instanceof String && originals.containsKey(key)) {
+                // Intentionally ignore the result; call just to mark the original entry as used
+                originals.get(key);
+            }
+            // But always use the resolved entry
+            return super.get(key);
+        }
+    }
 }
diff --git a/clients/src/test/java/org/apache/kafka/common/config/AbstractConfigTest.java b/clients/src/test/java/org/apache/kafka/common/config/AbstractConfigTest.java
index a007fd3707eb4..33aaac51b3756 100644
--- a/clients/src/test/java/org/apache/kafka/common/config/AbstractConfigTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/config/AbstractConfigTest.java
@@ -363,6 +363,20 @@ public void testConfigProvidersPropsAsParam() {
         assertEquals(config.originals().get("sasl.kerberos.password"), "randomPassword");
     }
 
+    @Test
+    public void testImmutableOriginalsWithConfigProvidersProps() {
+        // Test Case: Valid Test Case for ConfigProviders as a separate variable
+        Properties providers = new Properties();
+        providers.put("config.providers", "file");
+        providers.put("config.providers.file.class", "org.apache.kafka.common.config.provider.MockFileConfigProvider");
+        Properties props = new Properties();
+        props.put("sasl.kerberos.key", "${file:/usr/kerberos:key}");
+        Map immutableMap = Collections.unmodifiableMap(props);
+        Map provMap = convertPropertiesToMap(providers);
+        TestIndirectConfigResolution config = new TestIndirectConfigResolution(immutableMap, provMap);
+        assertEquals(config.originals().get("sasl.kerberos.key"), "testKey");
+    }
+
     @Test
     public void testAutoConfigResolutionWithMultipleConfigProviders() {
         // Test Case: Valid Test Case With Multiple ConfigProviders as a separate variable
diff --git a/gradle/spotbugs-exclude.xml b/gradle/spotbugs-exclude.xml
index eeda703145709..6f91df0b2dcce 100644
--- a/gradle/spotbugs-exclude.xml
+++ b/gradle/spotbugs-exclude.xml
@@ -334,4 +334,11 @@ For a detailed description of spotbugs bug categories, see https://spotbugs.read
 
     
 
+    
+        
+        
+        
+        
+    
+
 

From b042b36674f0bd8046f0229c96e0b8cf99554b2b Mon Sep 17 00:00:00 2001
From: tadsul <43974298+tadsul@users.noreply.github.com>
Date: Mon, 3 Jun 2019 04:56:31 -0700
Subject: [PATCH 0320/1071] KAFKA-8426; Fix for keeping the ConfigProvider
 configs consistent with KIP-297 (#6750)

According to KIP-297 a parameter is passed to ConfigProvider with syntax "config.providers.{name}.param.{param-name}". Currently AbstractConfig allows parameters of the format "config.providers.{name}.{param-name}". With this fix AbstractConfig will be consistent with KIP-297 syntax.

Reviewers: Robert Yokota , Rajini Sivaram 
---
 .../kafka/common/config/AbstractConfig.java   | 14 +++++++-
 .../common/config/AbstractConfigTest.java     | 32 +++++++++++++++----
 .../provider/MockVaultConfigProvider.java     | 18 ++++++++++-
 3 files changed, 55 insertions(+), 9 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java b/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java
index 13865d0fca8b4..a48856ffb53c6 100644
--- a/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java
+++ b/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java
@@ -55,6 +55,8 @@ public class AbstractConfig {
 
     private static final String CONFIG_PROVIDERS_CONFIG = "config.providers";
 
+    private static final String CONFIG_PROVIDERS_PARAM = ".param.";
+
     /**
      * Construct a configuration with a ConfigDef and the configuration properties, which can include properties
      * for zero or more {@link ConfigProvider} that will be used to resolve variables in configuration property
@@ -494,6 +496,16 @@ private Map configProviderProperties(String configProviderPrefix
         return result;
     }
 
+    /**
+     * Instantiates and configures the ConfigProviders. The config providers configs are defined as follows:
+     * config.providers : A comma-separated list of names for providers.
+     * config.providers.{name}.class : The Java class name for a provider.
+     * config.providers.{name}.param.{param-name} : A parameter to be passed to the above Java class on initialization.
+     * returns a map of config provider name and its instance.
+     * @param indirectConfigs The map of potential variable configs
+     * @param providerConfigProperties The map of config provider configs
+     * @return map map of config provider name and its instance.
+     */
     private Map instantiateConfigProviders(Map indirectConfigs, Map providerConfigProperties) {
         final String configProviders = indirectConfigs.get(CONFIG_PROVIDERS_CONFIG);
 
@@ -513,7 +525,7 @@ private Map instantiateConfigProviders(Map configProviderInstances = new HashMap<>();
         for (Map.Entry entry : providerMap.entrySet()) {
             try {
-                String prefix = CONFIG_PROVIDERS_CONFIG + "." + entry.getKey() + ".";
+                String prefix = CONFIG_PROVIDERS_CONFIG + "." + entry.getKey() + CONFIG_PROVIDERS_PARAM;
                 Map configProperties = configProviderProperties(prefix, providerConfigProperties);
                 ConfigProvider provider = Utils.newInstance(entry.getValue(), ConfigProvider.class);
                 provider.configure(configProperties);
diff --git a/clients/src/test/java/org/apache/kafka/common/config/AbstractConfigTest.java b/clients/src/test/java/org/apache/kafka/common/config/AbstractConfigTest.java
index 33aaac51b3756..f6864754ca79b 100644
--- a/clients/src/test/java/org/apache/kafka/common/config/AbstractConfigTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/config/AbstractConfigTest.java
@@ -24,6 +24,8 @@
 import org.apache.kafka.common.metrics.JmxReporter;
 import org.apache.kafka.common.metrics.MetricsReporter;
 import org.apache.kafka.common.security.TestSecurityConfig;
+import org.apache.kafka.common.config.provider.MockVaultConfigProvider;
+import org.apache.kafka.common.config.provider.MockFileConfigProvider;
 import org.junit.Test;
 
 import java.util.Arrays;
@@ -337,7 +339,7 @@ public void testOriginalsWithConfigProvidersProps() {
 
         // Test Case: Valid Test Case for ConfigProviders as part of config.properties
         props.put("config.providers", "file");
-        props.put("config.providers.file.class", "org.apache.kafka.common.config.provider.MockFileConfigProvider");
+        props.put("config.providers.file.class", MockFileConfigProvider.class.getName());
         props.put("prefix.ssl.truststore.location.number", 5);
         props.put("sasl.kerberos.service.name", "service name");
         props.put("sasl.kerberos.key", "${file:/usr/kerberos:key}");
@@ -354,7 +356,7 @@ public void testConfigProvidersPropsAsParam() {
         // Test Case: Valid Test Case for ConfigProviders as a separate variable
         Properties providers = new Properties();
         providers.put("config.providers", "file");
-        providers.put("config.providers.file.class", "org.apache.kafka.common.config.provider.MockFileConfigProvider");
+        providers.put("config.providers.file.class", MockFileConfigProvider.class.getName());
         Properties props = new Properties();
         props.put("sasl.kerberos.key", "${file:/usr/kerberos:key}");
         props.put("sasl.kerberos.password", "${file:/usr/kerberos:password}");
@@ -382,8 +384,8 @@ public void testAutoConfigResolutionWithMultipleConfigProviders() {
         // Test Case: Valid Test Case With Multiple ConfigProviders as a separate variable
         Properties providers = new Properties();
         providers.put("config.providers", "file,vault");
-        providers.put("config.providers.file.class", "org.apache.kafka.common.config.provider.MockFileConfigProvider");
-        providers.put("config.providers.vault.class", "org.apache.kafka.common.config.provider.MockVaultConfigProvider");
+        providers.put("config.providers.file.class", MockFileConfigProvider.class.getName());
+        providers.put("config.providers.vault.class", MockVaultConfigProvider.class.getName());
         Properties props = new Properties();
         props.put("sasl.kerberos.key", "${file:/usr/kerberos:key}");
         props.put("sasl.kerberos.password", "${file:/usr/kerberos:password}");
@@ -426,7 +428,7 @@ public void testAutoConfigResolutionWithMissingConfigKey() {
         // Test Case: Config Provider fails to resolve the config (key not present)
         Properties props = new Properties();
         props.put("config.providers", "test");
-        props.put("config.providers.test.class", "org.apache.kafka.common.config.provider.MockFileConfigProvider");
+        props.put("config.providers.test.class", MockFileConfigProvider.class.getName());
         props.put("random", "${test:/foo/bar/testpath:random}");
         TestIndirectConfigResolution config = new TestIndirectConfigResolution(props);
         assertEquals(config.originals().get("random"), "${test:/foo/bar/testpath:random}");
@@ -437,17 +439,33 @@ public void testAutoConfigResolutionWithDuplicateConfigProvider() {
         // Test Case: If ConfigProvider is provided in both originals and provider. Only the ones in provider should be used.
         Properties providers = new Properties();
         providers.put("config.providers", "test");
-        providers.put("config.providers.test.class", "org.apache.kafka.common.config.provider.MockFileConfigProvider");
+        providers.put("config.providers.test.class", MockVaultConfigProvider.class.getName());
 
         Properties props = new Properties();
         props.put("sasl.kerberos.key", "${file:/usr/kerberos:key}");
         props.put("config.providers", "file");
-        props.put("config.providers.file.class", "org.apache.kafka.common.config.provider.MockFileConfigProvider");
+        props.put("config.providers.file.class", MockVaultConfigProvider.class.getName());
 
         TestIndirectConfigResolution config = new TestIndirectConfigResolution(props, convertPropertiesToMap(providers));
         assertEquals(config.originals().get("sasl.kerberos.key"), "${file:/usr/kerberos:key}");
     }
 
+    @Test
+    public void testConfigProviderConfigurationWithConfigParams() {
+        // Test Case: Valid Test Case With Multiple ConfigProviders as a separate variable
+        Properties providers = new Properties();
+        providers.put("config.providers", "vault");
+        providers.put("config.providers.vault.class", MockVaultConfigProvider.class.getName());
+        providers.put("config.providers.vault.param.key", "randomKey");
+        providers.put("config.providers.vault.param.location", "/usr/vault");
+        Properties props = new Properties();
+        props.put("sasl.truststore.location", "${vault:/usr/truststore:truststoreKey}");
+        props.put("sasl.truststore.password", "${vault:/usr/truststore:truststorePassword}");
+        props.put("sasl.truststore.location", "${vault:/usr/truststore:truststoreLocation}");
+        TestIndirectConfigResolution config = new TestIndirectConfigResolution(props, convertPropertiesToMap(providers));
+        assertEquals(config.originals().get("sasl.truststore.location"), "/usr/vault");
+    }
+
     private static class TestIndirectConfigResolution extends AbstractConfig {
 
         private static final ConfigDef CONFIG;
diff --git a/clients/src/test/java/org/apache/kafka/common/config/provider/MockVaultConfigProvider.java b/clients/src/test/java/org/apache/kafka/common/config/provider/MockVaultConfigProvider.java
index dbfe15813941c..c741798a72972 100644
--- a/clients/src/test/java/org/apache/kafka/common/config/provider/MockVaultConfigProvider.java
+++ b/clients/src/test/java/org/apache/kafka/common/config/provider/MockVaultConfigProvider.java
@@ -19,11 +19,27 @@
 import java.io.IOException;
 import java.io.Reader;
 import java.io.StringReader;
+import java.util.Map;
 
 public class MockVaultConfigProvider extends FileConfigProvider {
 
+    Map vaultConfigs;
+    private boolean configured = false;
+    private static final String LOCATION = "location";
+
     @Override
     protected Reader reader(String path) throws IOException {
-        return new StringReader("truststoreKey=testTruststoreKey\ntruststorePassword=randomtruststorePassword");
+        String vaultLocation = (String) vaultConfigs.get(LOCATION);
+        return new StringReader("truststoreKey=testTruststoreKey\ntruststorePassword=randomtruststorePassword\n" + "truststoreLocation=" + vaultLocation + "\n");
+    }
+
+    @Override
+    public void configure(Map configs) {
+        this.vaultConfigs = configs;
+        configured = true;
+    }
+
+    public boolean configured() {
+        return configured;
     }
 }

From 3c7c988e3903d87ae1072fa9be5f349e5912d98e Mon Sep 17 00:00:00 2001
From: Konstantine Karantasis 
Date: Mon, 3 Jun 2019 09:13:40 -0700
Subject: [PATCH 0321/1071] KAFKA-8449: Restart tasks on reconfiguration under
 incremental cooperative rebalancing (#6850)

Restart task on reconfiguration under incremental cooperative rebalancing, and keep execution paths separate for config updates between eager and cooperative. Include the group generation in the log message when the worker receives its assignment.

Author: Konstantine Karantasis 
Reviewer: Randall Hauch 
---
 .../distributed/DistributedHerder.java        | 203 ++++++++++++++----
 .../MonitorableSourceConnector.java           |   5 +-
 ...alanceSourceConnectorsIntegrationTest.java |  63 +++++-
 3 files changed, 220 insertions(+), 51 deletions(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
index 585836e557744..52709f7856fbb 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
@@ -80,6 +80,8 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
 
 import static org.apache.kafka.connect.runtime.distributed.ConnectProtocol.CONNECT_PROTOCOL_V0;
 
@@ -141,6 +143,7 @@ public class DistributedHerder extends AbstractHerder implements Runnable {
     // and the from other nodes are safe to process
     private boolean rebalanceResolved;
     private ExtendedAssignment runningAssignment = ExtendedAssignment.empty();
+    private Set tasksToRestart = new HashSet<>();
     private ExtendedAssignment assignment;
     private boolean canReadConfigs;
     private ClusterConfigState configState;
@@ -151,6 +154,7 @@ public class DistributedHerder extends AbstractHerder implements Runnable {
     // Config updates can be collected and applied together when possible. Also, we need to take care to rebalance when
     // needed (e.g. task reconfiguration, which requires everyone to coordinate offset commits).
     private Set connectorConfigUpdates = new HashSet<>();
+    private Set taskConfigUpdates = new HashSet<>();
     // Similarly collect target state changes (when observed by the config storage listener) for handling in the
     // herder's main thread.
     private Set connectorTargetStateChanges = new HashSet<>();
@@ -304,51 +308,47 @@ public void tick() {
         }
 
         // Process any configuration updates
-        Set connectorConfigUpdatesCopy = null;
-        Set connectorTargetStateChangesCopy = null;
-        synchronized (this) {
-            if (needsReconfigRebalance || !connectorConfigUpdates.isEmpty() || !connectorTargetStateChanges.isEmpty()) {
-                // Connector reconfigs only need local updates since there is no coordination between workers required.
-                // However, if connectors were added or removed, work needs to be rebalanced since we have more work
-                // items to distribute among workers.
-                configState = configBackingStore.snapshot();
-
-                if (needsReconfigRebalance) {
-                    // Task reconfigs require a rebalance. Request the rebalance, clean out state, and then restart
-                    // this loop, which will then ensure the rebalance occurs without any other requests being
-                    // processed until it completes.
-                    member.requestRejoin();
-                    // Any connector config updates or target state changes will be addressed during the rebalance too
-                    connectorConfigUpdates.clear();
-                    connectorTargetStateChanges.clear();
-                    needsReconfigRebalance = false;
-                    log.debug("Requesting rebalance due to reconfiguration of tasks (needsReconfigRebalance: {})",
-                            needsReconfigRebalance);
-                    return;
-                } else {
-                    if (!connectorConfigUpdates.isEmpty()) {
-                        // We can't start/stop while locked since starting connectors can cause task updates that will
-                        // require writing configs, which in turn make callbacks into this class from another thread that
-                        // require acquiring a lock. This leads to deadlock. Instead, just copy the info we need and process
-                        // the updates after unlocking.
-                        connectorConfigUpdatesCopy = connectorConfigUpdates;
-                        connectorConfigUpdates = new HashSet<>();
-                    }
+        AtomicReference> connectorConfigUpdatesCopy = new AtomicReference<>();
+        AtomicReference> connectorTargetStateChangesCopy = new AtomicReference<>();
+        AtomicReference> taskConfigUpdatesCopy = new AtomicReference<>();
+
+        boolean shouldReturn;
+        if (member.currentProtocolVersion() == CONNECT_PROTOCOL_V0) {
+            shouldReturn = updateConfigsWithEager(connectorConfigUpdatesCopy,
+                    connectorTargetStateChangesCopy);
+            // With eager protocol we should return immediately if needsReconfigRebalance has
+            // been set to retain the old workflow
+            if (shouldReturn) {
+                return;
+            }
 
-                    if (!connectorTargetStateChanges.isEmpty()) {
-                        // Similarly for target state changes which can cause connectors to be restarted
-                        connectorTargetStateChangesCopy = connectorTargetStateChanges;
-                        connectorTargetStateChanges = new HashSet<>();
-                    }
-                }
+            if (connectorConfigUpdatesCopy.get() != null) {
+                processConnectorConfigUpdates(connectorConfigUpdatesCopy.get());
             }
-        }
 
-        if (connectorConfigUpdatesCopy != null)
-            processConnectorConfigUpdates(connectorConfigUpdatesCopy);
+            if (connectorTargetStateChangesCopy.get() != null) {
+                processTargetStateChanges(connectorTargetStateChangesCopy.get());
+            }
+        } else {
+            shouldReturn = updateConfigsWithIncrementalCooperative(connectorConfigUpdatesCopy,
+                    connectorTargetStateChangesCopy, taskConfigUpdatesCopy);
+
+            if (connectorConfigUpdatesCopy.get() != null) {
+                processConnectorConfigUpdates(connectorConfigUpdatesCopy.get());
+            }
+
+            if (connectorTargetStateChangesCopy.get() != null) {
+                processTargetStateChanges(connectorTargetStateChangesCopy.get());
+            }
 
-        if (connectorTargetStateChangesCopy != null)
-            processTargetStateChanges(connectorTargetStateChangesCopy);
+            if (taskConfigUpdatesCopy.get() != null) {
+                processTaskConfigUpdatesWithIncrementalCooperative(taskConfigUpdatesCopy.get());
+            }
+
+            if (shouldReturn) {
+                return;
+            }
+        }
 
         // Let the group take any actions it needs to
         try {
@@ -360,6 +360,95 @@ public void tick() {
         }
     }
 
+    private synchronized boolean updateConfigsWithEager(AtomicReference> connectorConfigUpdatesCopy,
+                                                        AtomicReference> connectorTargetStateChangesCopy) {
+        // This branch is here to avoid creating a snapshot if not needed
+        if (needsReconfigRebalance
+                || !connectorConfigUpdates.isEmpty()
+                || !connectorTargetStateChanges.isEmpty()) {
+            // Connector reconfigs only need local updates since there is no coordination between workers required.
+            // However, if connectors were added or removed, work needs to be rebalanced since we have more work
+            // items to distribute among workers.
+            configState = configBackingStore.snapshot();
+
+            if (needsReconfigRebalance) {
+                // Task reconfigs require a rebalance. Request the rebalance, clean out state, and then restart
+                // this loop, which will then ensure the rebalance occurs without any other requests being
+                // processed until it completes.
+                log.debug("Requesting rebalance due to reconfiguration of tasks (needsReconfigRebalance: {})",
+                        needsReconfigRebalance);
+                member.requestRejoin();
+                needsReconfigRebalance = false;
+                // Any connector config updates or target state changes will be addressed during the rebalance too
+                connectorConfigUpdates.clear();
+                connectorTargetStateChanges.clear();
+                return true;
+            } else {
+                if (!connectorConfigUpdates.isEmpty()) {
+                    // We can't start/stop while locked since starting connectors can cause task updates that will
+                    // require writing configs, which in turn make callbacks into this class from another thread that
+                    // require acquiring a lock. This leads to deadlock. Instead, just copy the info we need and process
+                    // the updates after unlocking.
+                    connectorConfigUpdatesCopy.set(connectorConfigUpdates);
+                    connectorConfigUpdates = new HashSet<>();
+                }
+
+                if (!connectorTargetStateChanges.isEmpty()) {
+                    // Similarly for target state changes which can cause connectors to be restarted
+                    connectorTargetStateChangesCopy.set(connectorTargetStateChanges);
+                    connectorTargetStateChanges = new HashSet<>();
+                }
+            }
+        }
+        return false;
+    }
+
+    private synchronized boolean updateConfigsWithIncrementalCooperative(AtomicReference> connectorConfigUpdatesCopy,
+                                                                         AtomicReference> connectorTargetStateChangesCopy,
+                                                                         AtomicReference> taskConfigUpdatesCopy) {
+        boolean retValue = false;
+        // This branch is here to avoid creating a snapshot if not needed
+        if (needsReconfigRebalance
+                || !connectorConfigUpdates.isEmpty()
+                || !connectorTargetStateChanges.isEmpty()
+                || !taskConfigUpdates.isEmpty()) {
+            // Connector reconfigs only need local updates since there is no coordination between workers required.
+            // However, if connectors were added or removed, work needs to be rebalanced since we have more work
+            // items to distribute among workers.
+            configState = configBackingStore.snapshot();
+
+            if (needsReconfigRebalance) {
+                log.debug("Requesting rebalance due to reconfiguration of tasks (needsReconfigRebalance: {})",
+                        needsReconfigRebalance);
+                member.requestRejoin();
+                needsReconfigRebalance = false;
+                retValue = true;
+            }
+
+            if (!connectorConfigUpdates.isEmpty()) {
+                // We can't start/stop while locked since starting connectors can cause task updates that will
+                // require writing configs, which in turn make callbacks into this class from another thread that
+                // require acquiring a lock. This leads to deadlock. Instead, just copy the info we need and process
+                // the updates after unlocking.
+                connectorConfigUpdatesCopy.set(connectorConfigUpdates);
+                connectorConfigUpdates = new HashSet<>();
+            }
+
+            if (!connectorTargetStateChanges.isEmpty()) {
+                // Similarly for target state changes which can cause connectors to be restarted
+                connectorTargetStateChangesCopy.set(connectorTargetStateChanges);
+                connectorTargetStateChanges = new HashSet<>();
+            }
+
+            if (!taskConfigUpdates.isEmpty()) {
+                // Similarly for task config updates
+                taskConfigUpdatesCopy.set(taskConfigUpdates);
+                taskConfigUpdates = new HashSet<>();
+            }
+        }
+        return retValue;
+    }
+
     private void processConnectorConfigUpdates(Set connectorConfigUpdates) {
         // If we only have connector config updates, we can just bounce the updated connectors that are
         // currently assigned to this worker.
@@ -396,6 +485,21 @@ private void processTargetStateChanges(Set connectorTargetStateChanges)
         }
     }
 
+    private void processTaskConfigUpdatesWithIncrementalCooperative(Set taskConfigUpdates) {
+        Set localTasks = assignment == null
+                                          ? Collections.emptySet()
+                                          : new HashSet<>(assignment.tasks());
+        Set connectorsWhoseTasksToStop = taskConfigUpdates.stream()
+                .map(ConnectorTaskId::connector).collect(Collectors.toSet());
+
+        List tasksToStop = localTasks.stream()
+                .filter(taskId -> connectorsWhoseTasksToStop.contains(taskId.connector()))
+                .collect(Collectors.toList());
+        log.info("Handling task config update by restarting tasks {}", tasksToStop);
+        worker.stopAndAwaitTasks(tasksToStop);
+        tasksToRestart.addAll(tasksToStop);
+    }
+
     // public for testing
     public void halt() {
         synchronized (this) {
@@ -900,6 +1004,12 @@ private void startWork() {
             callables.add(getConnectorStartingCallable(connectorName));
         }
 
+        // These tasks have been stopped by this worker due to task reconfiguration. In order to
+        // restart them, they are removed just before the overall task startup from the set of
+        // currently running tasks. Therefore, they'll be restarted only if they are included in
+        // the assignment that was just received after rebalancing.
+        runningAssignment.tasks().removeAll(tasksToRestart);
+        tasksToRestart.clear();
         for (ConnectorTaskId taskId : assignmentDifference(assignment.tasks(), runningAssignment.tasks())) {
             callables.add(getTaskStartingCallable(taskId));
         }
@@ -1172,12 +1282,17 @@ public void onConnectorConfigUpdate(String connector) {
         public void onTaskConfigUpdate(Collection tasks) {
             log.info("Tasks {} configs updated", tasks);
 
-            // Stage the update and wake up the work thread. No need to record the set of tasks here because task reconfigs
-            // always need a rebalance to ensure offsets get committed.
+            // Stage the update and wake up the work thread.
+            // The set of tasks is recorder for incremental cooperative rebalancing, in which
+            // tasks don't get restarted unless they are balanced between workers.
+            // With eager rebalancing there's no need to record the set of tasks because task reconfigs
+            // always need a rebalance to ensure offsets get committed. In eager rebalancing the
+            // recorded set of tasks remains unused.
             // TODO: As an optimization, some task config updates could avoid a rebalance. In particular, single-task
             // connectors clearly don't need any coordination.
             synchronized (DistributedHerder.this) {
                 needsReconfigRebalance = true;
+                taskConfigUpdates.addAll(tasks);
             }
             member.wakeup();
         }
@@ -1279,7 +1394,7 @@ public void onAssigned(ExtendedAssignment assignment, int generation) {
             // catch up (or backoff if we fail) not executed in a callback, and so we'll be able to invoke other
             // group membership actions (e.g., we may need to explicitly leave the group if we cannot handle the
             // assigned tasks).
-            log.info("Joined group and got assignment: {}", assignment);
+            log.info("Joined group at generation {} and got assignment: {}", generation, assignment);
             synchronized (DistributedHerder.this) {
                 DistributedHerder.this.assignment = assignment;
                 DistributedHerder.this.generation = generation;
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java
index 8bc895368f474..2ca7698381fec 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java
@@ -44,6 +44,7 @@
 public class MonitorableSourceConnector extends TestSourceConnector {
     private static final Logger log = LoggerFactory.getLogger(MonitorableSourceConnector.class);
 
+    public static final String TOPIC_CONFIG = "topic";
     private String connectorName;
     private ConnectorHandle connectorHandle;
     private Map commonConfigs;
@@ -105,7 +106,7 @@ public String version() {
         public void start(Map props) {
             taskId = props.get("task.id");
             connectorName = props.get("connector.name");
-            topicName = props.getOrDefault("topic", "sequential-topic");
+            topicName = props.getOrDefault(TOPIC_CONFIG, "sequential-topic");
             throughput = Long.valueOf(props.getOrDefault("throughput", "-1"));
             batchSize = Integer.valueOf(props.getOrDefault("messages.per.poll", "1"));
             taskHandle = RuntimeHandles.get().connectorHandle(connectorName).taskHandle(taskId);
@@ -113,7 +114,7 @@ public void start(Map props) {
                     context.offsetStorageReader().offset(Collections.singletonMap("task.id", taskId)))
                     .orElse(Collections.emptyMap());
             startingSeqno = Optional.ofNullable((Long) offset.get("saved")).orElse(0L);
-            log.info("Started {} task {}", this.getClass().getSimpleName(), taskId);
+            log.info("Started {} task {} with properties {}", this.getClass().getSimpleName(), taskId, props);
             throttler = new ThroughputThrottler(throughput, System.currentTimeMillis());
         }
 
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java
index b0125b25b4806..d3cc8dbfe58dc 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java
@@ -39,17 +39,18 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import static org.apache.kafka.connect.integration.MonitorableSourceConnector.TOPIC_CONFIG;
 import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLASS_CONFIG;
 import static org.apache.kafka.connect.runtime.ConnectorConfig.KEY_CONVERTER_CLASS_CONFIG;
 import static org.apache.kafka.connect.runtime.ConnectorConfig.TASKS_MAX_CONFIG;
 import static org.apache.kafka.connect.runtime.ConnectorConfig.VALUE_CONVERTER_CLASS_CONFIG;
-import static org.apache.kafka.connect.runtime.SinkConnectorConfig.TOPICS_CONFIG;
 import static org.apache.kafka.connect.runtime.WorkerConfig.OFFSET_COMMIT_INTERVAL_MS_CONFIG;
 import static org.apache.kafka.connect.runtime.distributed.ConnectProtocolCompatibility.COMPATIBLE;
 import static org.apache.kafka.connect.runtime.distributed.DistributedConfig.CONNECT_PROTOCOL_CONFIG;
 import static org.apache.kafka.test.TestUtils.waitForCondition;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
 
 /**
  * Integration tests for incremental cooperative rebalancing between Connect workers
@@ -109,7 +110,7 @@ public void testStartTwoConnectors() throws Exception {
         props.put(TASKS_MAX_CONFIG, String.valueOf(NUM_TASKS));
         props.put("throughput", String.valueOf(1));
         props.put("messages.per.poll", String.valueOf(10));
-        props.put(TOPICS_CONFIG, TOPIC_NAME);
+        props.put(TOPIC_CONFIG, TOPIC_NAME);
         props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
         props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
 
@@ -129,6 +130,58 @@ public void testStartTwoConnectors() throws Exception {
                 CONNECTOR_SETUP_DURATION_MS, "Connector tasks did not start in time.");
     }
 
+    @Test
+    public void testReconfigConnector() throws Exception {
+        ConnectorHandle connectorHandle = RuntimeHandles.get().connectorHandle(CONNECTOR_NAME);
+
+        // create test topic
+        String anotherTopic = "another-topic";
+        connect.kafka().createTopic(TOPIC_NAME, NUM_TOPIC_PARTITIONS);
+        connect.kafka().createTopic(anotherTopic, NUM_TOPIC_PARTITIONS);
+
+        // setup up props for the source connector
+        Map props = new HashMap<>();
+        props.put(CONNECTOR_CLASS_CONFIG, MonitorableSourceConnector.class.getSimpleName());
+        props.put(TASKS_MAX_CONFIG, String.valueOf(NUM_TASKS));
+        props.put("throughput", String.valueOf(1));
+        props.put("messages.per.poll", String.valueOf(10));
+        props.put(TOPIC_CONFIG, TOPIC_NAME);
+        props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
+        props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
+
+        // start a source connector
+        connect.configureConnector(CONNECTOR_NAME, props);
+
+        waitForCondition(() -> this.assertConnectorAndTasksRunning(CONNECTOR_NAME, NUM_TASKS).orElse(false),
+                CONNECTOR_SETUP_DURATION_MS, "Connector tasks did not start in time.");
+
+        int numRecordsProduced = 100;
+        int recordTransferDurationMs = 5000;
+
+        // consume all records from the source topic or fail, to ensure that they were correctly produced
+        int recordNum = connect.kafka().consume(numRecordsProduced, recordTransferDurationMs, TOPIC_NAME).count();
+        assertTrue("Not enough records produced by source connector. Expected at least: " + numRecordsProduced + " + but got " + recordNum,
+                recordNum >= numRecordsProduced);
+
+        // Reconfigure the source connector by changing the Kafka topic used as output
+        props.put(TOPIC_CONFIG, anotherTopic);
+        connect.configureConnector(CONNECTOR_NAME, props);
+
+        waitForCondition(() -> this.assertConnectorAndTasksRunning(CONNECTOR_NAME, NUM_TASKS).orElse(false),
+                CONNECTOR_SETUP_DURATION_MS, "Connector tasks did not start in time.");
+
+        // expect all records to be produced by the connector
+        connectorHandle.expectedRecords(numRecordsProduced);
+
+        // expect all records to be produced by the connector
+        connectorHandle.expectedCommits(numRecordsProduced);
+
+        // consume all records from the source topic or fail, to ensure that they were correctly produced
+        recordNum = connect.kafka().consume(numRecordsProduced, recordTransferDurationMs, anotherTopic).count();
+        assertTrue("Not enough records produced by source connector. Expected at least: " + numRecordsProduced + " + but got " + recordNum,
+                recordNum >= numRecordsProduced);
+    }
+
     @Test
     public void testDeleteConnector() throws Exception {
         // create test topic
@@ -140,7 +193,7 @@ public void testDeleteConnector() throws Exception {
         props.put(TASKS_MAX_CONFIG, String.valueOf(NUM_TASKS));
         props.put("throughput", String.valueOf(1));
         props.put("messages.per.poll", String.valueOf(10));
-        props.put(TOPICS_CONFIG, TOPIC_NAME);
+        props.put(TOPIC_CONFIG, TOPIC_NAME);
         props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
         props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
 
@@ -181,7 +234,7 @@ public void testAddingWorker() throws Exception {
         props.put(TASKS_MAX_CONFIG, String.valueOf(NUM_TASKS));
         props.put("throughput", String.valueOf(1));
         props.put("messages.per.poll", String.valueOf(10));
-        props.put(TOPICS_CONFIG, TOPIC_NAME);
+        props.put(TOPIC_CONFIG, TOPIC_NAME);
         props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
         props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
 
@@ -224,7 +277,7 @@ public void testRemovingWorker() throws Exception {
         props.put(TASKS_MAX_CONFIG, String.valueOf(NUM_TASKS));
         props.put("throughput", String.valueOf(1));
         props.put("messages.per.poll", String.valueOf(10));
-        props.put(TOPICS_CONFIG, TOPIC_NAME);
+        props.put(TOPIC_CONFIG, TOPIC_NAME);
         props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
         props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
 

From ce008e72de6ce67a8e9bb3d5d78fb0bb3849a38a Mon Sep 17 00:00:00 2001
From: Randall Hauch 
Date: Mon, 3 Jun 2019 12:34:55 -0500
Subject: [PATCH 0322/1071] KAFKA-8475: Temporarily restore
 SslFactory.sslContext() helper

Temporarily restore the SslFactory.sslContext() function, which some connectors use. This function is not a public API and it will be removed eventually. For now, we will mark it as deprecated.
---
 .../org/apache/kafka/common/security/ssl/SslFactory.java    | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/clients/src/main/java/org/apache/kafka/common/security/ssl/SslFactory.java b/clients/src/main/java/org/apache/kafka/common/security/ssl/SslFactory.java
index 882b63d4cc22d..4d942303a4685 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/ssl/SslFactory.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/ssl/SslFactory.java
@@ -26,6 +26,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLEngineResult;
 import javax.net.ssl.SSLException;
@@ -168,6 +169,11 @@ public SSLEngine createSslEngine(String peerHost, int peerPort) {
         return sslEngineBuilder.createSslEngine(mode, peerHost, peerPort, endpointIdentification);
     }
 
+    @Deprecated
+    public SSLContext sslContext() {
+        return sslEngineBuilder.sslContext();
+    }
+
     public SslEngineBuilder sslEngineBuilder() {
         return sslEngineBuilder;
     }

From 45fae339373fe4bd90fb320695267b8340094fa3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Armando=20Garc=C3=ADa=20Sancio?=
 
Date: Mon, 3 Jun 2019 13:16:38 -0700
Subject: [PATCH 0323/1071] KAFKA-8383; Integration tests for unclean
 `electLeaders` (#6857)

Reviewers: Jason Gustafson 
---
 .../api/AdminClientIntegrationTest.scala      | 346 +++++++++++++++---
 .../UncleanLeaderElectionTest.scala           |   2 +-
 .../unit/kafka/server/LogDirFailureTest.scala |   1 -
 .../unit/kafka/server/LogRecoveryTest.scala   |   1 -
 4 files changed, 294 insertions(+), 56 deletions(-)

diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index 66689e43c6dc3..41451374d6ff1 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -16,22 +16,22 @@
  */
 package kafka.api
 
-import java.{time, util}
-import java.util.{Collections, Properties}
-import java.util.Arrays.asList
-import java.util.concurrent.{CountDownLatch, ExecutionException, TimeUnit}
 import java.io.File
+import java.lang.{Long => JLong}
+import java.time.{Duration => JDuration}
+import java.util.Arrays.asList
 import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger}
-
-import org.apache.kafka.clients.admin.KafkaAdminClientTest
-import org.apache.kafka.common.utils.{Time, Utils}
+import java.util.concurrent.{CountDownLatch, ExecutionException, TimeUnit}
+import java.util.{Collections, Properties}
+import java.{time, util}
 import kafka.log.LogConfig
+import kafka.security.auth.{Cluster, Group, Topic}
 import kafka.server.{Defaults, KafkaConfig, KafkaServer}
-import org.apache.kafka.clients.admin._
-import kafka.utils.{Logging, TestUtils}
-import kafka.utils.TestUtils._
 import kafka.utils.Implicits._
-import org.apache.kafka.clients.admin.NewTopic
+import kafka.utils.TestUtils._
+import kafka.utils.{Logging, TestUtils}
+import kafka.zk.KafkaZkClient
+import org.apache.kafka.clients.admin._
 import org.apache.kafka.clients.consumer.{ConsumerConfig, KafkaConsumer}
 import org.apache.kafka.clients.producer.KafkaProducer
 import org.apache.kafka.clients.producer.ProducerRecord
@@ -42,23 +42,17 @@ import org.apache.kafka.common.TopicPartitionReplica
 import org.apache.kafka.common.acl._
 import org.apache.kafka.common.config.ConfigResource
 import org.apache.kafka.common.errors._
-import org.junit.{After, Before, Rule, Test}
 import org.apache.kafka.common.requests.{DeleteRecordsRequest, MetadataResponse}
 import org.apache.kafka.common.resource.{PatternType, ResourcePattern, ResourceType}
-import org.junit.rules.Timeout
+import org.apache.kafka.common.utils.{Time, Utils}
 import org.junit.Assert._
+import org.junit.rules.Timeout
+import org.junit.{After, Before, Rule, Test}
 import org.scalatest.Assertions.intercept
-
-import scala.util.Random
 import scala.collection.JavaConverters._
-import kafka.zk.KafkaZkClient
-
 import scala.concurrent.duration.Duration
 import scala.concurrent.{Await, Future}
-import java.lang.{Long => JLong}
-import java.time.{Duration => JDuration}
-
-import kafka.security.auth.{Cluster, Group, Topic}
+import scala.util.Random
 
 /**
  * An integration test of the KafkaAdminClient.
@@ -1271,22 +1265,15 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     val partition2 = new TopicPartition("elect-preferred-leaders-topic-2", 0)
     TestUtils.createTopic(zkClient, partition2.topic, Map[Int, Seq[Int]](partition2.partition -> prefer0), servers)
 
-    def currentLeader(topicPartition: TopicPartition) =
-      client.describeTopics(asList(topicPartition.topic)).values.get(topicPartition.topic).
-        get.partitions.get(topicPartition.partition).leader.id
-
     def preferredLeader(topicPartition: TopicPartition) =
       client.describeTopics(asList(topicPartition.topic)).values.get(topicPartition.topic).
         get.partitions.get(topicPartition.partition).replicas.get(0).id
 
-    def waitForLeaderToBecome(topicPartition: TopicPartition, leader: Int) =
-      TestUtils.waitUntilTrue(() => currentLeader(topicPartition) == leader, s"Expected leader to become $leader", 10000)
-
     /** Changes the preferred leader without changing the current leader. */
     def changePreferredLeader(newAssignment: Seq[Int]) = {
       val preferred = newAssignment.head
-      val prior1 = currentLeader(partition1)
-      val prior2 = currentLeader(partition2)
+      val prior1 = currentLeader(client, partition1).get
+      val prior2 = currentLeader(client, partition2).get
 
       var m = Map.empty[TopicPartition, Seq[Int]]
 
@@ -1301,26 +1288,26 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
         s"Expected preferred leader to become $preferred, but is ${preferredLeader(partition1)} and ${preferredLeader(partition2)}",
         10000)
       // Check the leader hasn't moved
-      assertEquals(prior1, currentLeader(partition1))
-      assertEquals(prior2, currentLeader(partition2))
+      assertEquals(Some(prior1), currentLeader(client, partition1))
+      assertEquals(Some(prior2), currentLeader(client, partition2))
     }
 
     // Check current leaders are 0
-    assertEquals(0, currentLeader(partition1))
-    assertEquals(0, currentLeader(partition2))
+    assertEquals(Some(0), currentLeader(client, partition1))
+    assertEquals(Some(0), currentLeader(client, partition2))
 
     // Noop election
     var electResult = client.electLeaders(ElectionType.PREFERRED, Set(partition1).asJava)
     var exception = electResult.partitions.get.get(partition1).get
     assertEquals(classOf[ElectionNotNeededException], exception.getClass)
     assertEquals("Leader election not needed for topic partition", exception.getMessage)
-    assertEquals(0, currentLeader(partition1))
+    assertEquals(Some(0), currentLeader(client, partition1))
 
     // Noop election with null partitions
     electResult = client.electLeaders(ElectionType.PREFERRED, null)
     assertTrue(electResult.partitions.get.isEmpty)
-    assertEquals(0, currentLeader(partition1))
-    assertEquals(0, currentLeader(partition2))
+    assertEquals(Some(0), currentLeader(client, partition1))
+    assertEquals(Some(0), currentLeader(client, partition2))
 
     // Now change the preferred leader to 1
     changePreferredLeader(prefer1)
@@ -1329,17 +1316,17 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     electResult = client.electLeaders(ElectionType.PREFERRED, Set(partition1).asJava)
     assertEquals(Set(partition1).asJava, electResult.partitions.get.keySet)
     assertFalse(electResult.partitions.get.get(partition1).isPresent)
-    waitForLeaderToBecome(partition1, 1)
+    waitForLeaderToBecome(client, partition1, Some(1))
 
     // topic 2 unchanged
     assertFalse(electResult.partitions.get.containsKey(partition2))
-    assertEquals(0, currentLeader(partition2))
+    assertEquals(Some(0), currentLeader(client, partition2))
 
     // meaningful election with null partitions
     electResult = client.electLeaders(ElectionType.PREFERRED, null)
     assertEquals(Set(partition2), electResult.partitions.get.keySet.asScala)
     assertFalse(electResult.partitions.get.get(partition2).isPresent)
-    waitForLeaderToBecome(partition2, 1)
+    waitForLeaderToBecome(client, partition2, Some(1))
 
     // unknown topic
     val unknownPartition = new TopicPartition("topic-does-not-exist", 0)
@@ -1348,8 +1335,8 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     exception = electResult.partitions.get.get(unknownPartition).get
     assertEquals(classOf[UnknownTopicOrPartitionException], exception.getClass)
     assertEquals("The partition does not exist.", exception.getMessage)
-    assertEquals(1, currentLeader(partition1))
-    assertEquals(1, currentLeader(partition2))
+    assertEquals(Some(1), currentLeader(client, partition1))
+    assertEquals(Some(1), currentLeader(client, partition2))
 
     // Now change the preferred leader to 2
     changePreferredLeader(prefer2)
@@ -1357,8 +1344,8 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     // mixed results
     electResult = client.electLeaders(ElectionType.PREFERRED, Set(unknownPartition, partition1).asJava)
     assertEquals(Set(unknownPartition, partition1).asJava, electResult.partitions.get.keySet)
-    waitForLeaderToBecome(partition1, 2)
-    assertEquals(1, currentLeader(partition2))
+    waitForLeaderToBecome(client, partition1, Some(2))
+    assertEquals(Some(1), currentLeader(client, partition2))
     exception = electResult.partitions.get.get(unknownPartition).get
     assertEquals(classOf[UnknownTopicOrPartitionException], exception.getClass)
     assertEquals("The partition does not exist.", exception.getMessage)
@@ -1367,17 +1354,13 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     electResult = client.electLeaders(ElectionType.PREFERRED, Set(partition2).asJava)
     assertEquals(Set(partition2).asJava, electResult.partitions.get.keySet)
     assertFalse(electResult.partitions.get.get(partition2).isPresent)
-    waitForLeaderToBecome(partition2, 2)
+    waitForLeaderToBecome(client, partition2, Some(2))
 
     // Now change the preferred leader to 1
     changePreferredLeader(prefer1)
     // but shut it down...
     servers(1).shutdown()
-    waitUntilTrue (() => {
-      val description = client.describeTopics(Set(partition1.topic, partition2.topic).asJava).all.get
-      val isr = description.asScala.values.flatMap(_.partitions.asScala.flatMap(_.isr.asScala))
-      !isr.exists(_.id == 1)
-    }, "Expect broker 1 to no longer be in any ISR")
+    waitForBrokerOutOfIsr(client, Set(partition1, partition2), 1)
 
     // ... now what happens if we try to elect the preferred leader and it's down?
     val shortTimeout = new ElectLeadersOptions().timeoutMs(10000)
@@ -1387,7 +1370,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     assertEquals(classOf[PreferredLeaderNotAvailableException], exception.getClass)
     assertTrue(s"Wrong message ${exception.getMessage}", exception.getMessage.contains(
       "Failed to elect leader for partition elect-preferred-leaders-topic-1-0 under strategy PreferredReplicaPartitionLeaderElectionStrategy"))
-    assertEquals(2, currentLeader(partition1))
+    assertEquals(Some(2), currentLeader(client, partition1))
 
     // preferred leader unavailable with null argument
     electResult = client.electLeaders(ElectionType.PREFERRED, null, shortTimeout)
@@ -1402,8 +1385,235 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     assertTrue(s"Wrong message ${exception.getMessage}", exception.getMessage.contains(
       "Failed to elect leader for partition elect-preferred-leaders-topic-2-0 under strategy PreferredReplicaPartitionLeaderElectionStrategy"))
 
-    assertEquals(2, currentLeader(partition1))
-    assertEquals(2, currentLeader(partition2))
+    assertEquals(Some(2), currentLeader(client, partition1))
+    assertEquals(Some(2), currentLeader(client, partition2))
+  }
+
+  @Test
+  def testElectUncleanLeadersForOnePartition(): Unit = {
+    // Case: unclean leader election with one topic partition
+    client = AdminClient.create(createConfig)
+
+    val broker1 = 1
+    val broker2 = 2
+    val assignment1 = Seq(broker1, broker2)
+
+    val partition1 = new TopicPartition("unclean-test-topic-1", 0)
+    TestUtils.createTopic(zkClient, partition1.topic, Map[Int, Seq[Int]](partition1.partition -> assignment1), servers)
+
+    waitForLeaderToBecome(client, partition1, Option(broker1))
+
+    servers(broker2).shutdown()
+    waitForBrokerOutOfIsr(client, Set(partition1), broker2)
+    servers(broker1).shutdown()
+    waitForLeaderToBecome(client, partition1, None)
+    servers(broker2).startup()
+
+    val electResult = client.electLeaders(ElectionType.UNCLEAN, Set(partition1).asJava)
+    assertFalse(electResult.partitions.get.get(partition1).isPresent)
+    assertEquals(Option(broker2), currentLeader(client, partition1))
+  }
+
+  @Test
+  def testElectUncleanLeadersForManyPartitions(): Unit = {
+    // Case: unclean leader election with many topic partitions
+    client = AdminClient.create(createConfig)
+
+    val broker1 = 1
+    val broker2 = 2
+    val assignment1 = Seq(broker1, broker2)
+    val assignment2 = Seq(broker1, broker2)
+
+    val topic = "unclean-test-topic-1"
+    val partition1 = new TopicPartition(topic, 0)
+    val partition2 = new TopicPartition(topic, 1)
+
+    TestUtils.createTopic(
+      zkClient,
+      topic,
+      Map(partition1.partition -> assignment1, partition2.partition -> assignment2),
+      servers
+    )
+
+    waitForLeaderToBecome(client, partition1, Option(broker1))
+    waitForLeaderToBecome(client, partition2, Option(broker1))
+
+    servers(broker2).shutdown()
+    waitForBrokerOutOfIsr(client, Set(partition1, partition2), broker2)
+    servers(broker1).shutdown()
+    waitForLeaderToBecome(client, partition1, None)
+    waitForLeaderToBecome(client, partition2, None)
+    servers(broker2).startup()
+
+    val electResult = client.electLeaders(ElectionType.UNCLEAN, Set(partition1, partition2).asJava)
+    assertFalse(electResult.partitions.get.get(partition1).isPresent)
+    assertFalse(electResult.partitions.get.get(partition2).isPresent)
+    assertEquals(Option(broker2), currentLeader(client, partition1))
+    assertEquals(Option(broker2), currentLeader(client, partition2))
+  }
+
+  @Test
+  def testElectUncleanLeadersForAllPartitions(): Unit = {
+    // Case: noop unclean leader election and valid unclean leader election for all partitions
+    client = AdminClient.create(createConfig)
+
+    val broker1 = 1
+    val broker2 = 2
+    val broker3 = 0
+    val assignment1 = Seq(broker1, broker2)
+    val assignment2 = Seq(broker1, broker3)
+
+    val topic = "unclean-test-topic-1"
+    val partition1 = new TopicPartition(topic, 0)
+    val partition2 = new TopicPartition(topic, 1)
+
+    TestUtils.createTopic(
+      zkClient,
+      topic,
+      Map(partition1.partition -> assignment1, partition2.partition -> assignment2),
+      servers
+    )
+
+    waitForLeaderToBecome(client, partition1, Option(broker1))
+    waitForLeaderToBecome(client, partition2, Option(broker1))
+
+    servers(broker2).shutdown()
+    waitForBrokerOutOfIsr(client, Set(partition1), broker2)
+    servers(broker1).shutdown()
+    waitForLeaderToBecome(client, partition1, None)
+    waitForLeaderToBecome(client, partition2, Some(broker3))
+    servers(broker2).startup()
+
+    val electResult = client.electLeaders(ElectionType.UNCLEAN, null)
+    assertFalse(electResult.partitions.get.get(partition1).isPresent)
+    assertFalse(electResult.partitions.get.containsKey(partition2))
+    assertEquals(Option(broker2), currentLeader(client, partition1))
+    assertEquals(Option(broker3), currentLeader(client, partition2))
+  }
+
+  @Test
+  def testElectUncleanLeadersForUnknownPartitions(): Unit = {
+    // Case: unclean leader election for unknown topic
+    client = AdminClient.create(createConfig)
+
+    val broker1 = 1
+    val broker2 = 2
+    val assignment1 = Seq(broker1, broker2)
+
+    val topic = "unclean-test-topic-1"
+    val unknownPartition = new TopicPartition(topic, 1)
+    val unknownTopic = new TopicPartition("unknown-topic", 0)
+
+    TestUtils.createTopic(
+      zkClient,
+      topic,
+      Map(0 -> assignment1),
+      servers
+    )
+
+    waitForLeaderToBecome(client, new TopicPartition(topic, 0), Option(broker1))
+
+    val electResult = client.electLeaders(ElectionType.UNCLEAN, Set(unknownPartition, unknownTopic).asJava)
+    assertTrue(electResult.partitions.get.get(unknownPartition).get.isInstanceOf[UnknownTopicOrPartitionException])
+    assertTrue(electResult.partitions.get.get(unknownTopic).get.isInstanceOf[UnknownTopicOrPartitionException])
+  }
+
+  @Test
+  def testElectUncleanLeadersWhenNoLiveBrokers(): Unit = {
+    // Case: unclean leader election with no live brokers
+    client = AdminClient.create(createConfig)
+
+    val broker1 = 1
+    val broker2 = 2
+    val assignment1 = Seq(broker1, broker2)
+
+    val topic = "unclean-test-topic-1"
+    val partition1 = new TopicPartition(topic, 0)
+
+    TestUtils.createTopic(
+      zkClient,
+      topic,
+      Map(partition1.partition -> assignment1),
+      servers
+    )
+
+    waitForLeaderToBecome(client, partition1, Option(broker1))
+
+    servers(broker2).shutdown()
+    waitForBrokerOutOfIsr(client, Set(partition1), broker2)
+    servers(broker1).shutdown()
+    waitForLeaderToBecome(client, partition1, None)
+
+    val electResult = client.electLeaders(ElectionType.UNCLEAN, Set(partition1).asJava)
+    assertTrue(electResult.partitions.get.get(partition1).get.isInstanceOf[EligibleLeadersNotAvailableException])
+  }
+
+  @Test
+  def testElectUncleanLeadersNoop(): Unit = {
+    // Case: noop unclean leader election with explicit topic partitions
+    client = AdminClient.create(createConfig)
+
+    val broker1 = 1
+    val broker2 = 2
+    val assignment1 = Seq(broker1, broker2)
+
+    val topic = "unclean-test-topic-1"
+    val partition1 = new TopicPartition(topic, 0)
+
+    TestUtils.createTopic(
+      zkClient,
+      topic,
+      Map(partition1.partition -> assignment1),
+      servers
+    )
+
+    waitForLeaderToBecome(client, partition1, Option(broker1))
+
+    servers(broker1).shutdown()
+    waitForLeaderToBecome(client, partition1, Some(broker2))
+    servers(broker1).startup()
+
+    val electResult = client.electLeaders(ElectionType.UNCLEAN, Set(partition1).asJava)
+    assertTrue(electResult.partitions.get.get(partition1).get.isInstanceOf[ElectionNotNeededException])
+  }
+
+  @Test
+  def testElectUncleanLeadersAndNoop(): Unit = {
+    // Case: one noop unclean leader election and one valid unclean leader election
+    client = AdminClient.create(createConfig)
+
+    val broker1 = 1
+    val broker2 = 2
+    val broker3 = 0
+    val assignment1 = Seq(broker1, broker2)
+    val assignment2 = Seq(broker1, broker3)
+
+    val topic = "unclean-test-topic-1"
+    val partition1 = new TopicPartition(topic, 0)
+    val partition2 = new TopicPartition(topic, 1)
+
+    TestUtils.createTopic(
+      zkClient,
+      topic,
+      Map(partition1.partition -> assignment1, partition2.partition -> assignment2),
+      servers
+    )
+
+    waitForLeaderToBecome(client, partition1, Option(broker1))
+    waitForLeaderToBecome(client, partition2, Option(broker1))
+
+    servers(broker2).shutdown()
+    waitForBrokerOutOfIsr(client, Set(partition1), broker2)
+    servers(broker1).shutdown()
+    waitForLeaderToBecome(client, partition1, None)
+    waitForLeaderToBecome(client, partition2, Some(broker3))
+    servers(broker2).startup()
+
+    val electResult = client.electLeaders(ElectionType.UNCLEAN, Set(partition1, partition2).asJava)
+    assertFalse(electResult.partitions.get.get(partition1).isPresent)
+    assertTrue(electResult.partitions.get.get(partition2).get.isInstanceOf[ElectionNotNeededException])
+    assertEquals(Option(broker2), currentLeader(client, partition1))
+    assertEquals(Option(broker3), currentLeader(client, partition2))
   }
 
   @Test
@@ -1724,4 +1934,34 @@ object AdminClientIntegrationTest {
     assertEquals(Defaults.CompressionType.toString, configs.get(brokerResource).get(KafkaConfig.CompressionTypeProp).value)
   }
 
+  def currentLeader(client: AdminClient, topicPartition: TopicPartition): Option[Int] = {
+    Option(
+      client
+        .describeTopics(asList(topicPartition.topic))
+        .all
+        .get
+        .get(topicPartition.topic)
+        .partitions
+        .get(topicPartition.partition)
+        .leader
+    ).map(_.id)
+  }
+
+  def waitForLeaderToBecome(client: AdminClient, topicPartition: TopicPartition, leader: Option[Int]): Unit = {
+    TestUtils.waitUntilTrue(
+      () => currentLeader(client, topicPartition) == leader,
+      s"Expected leader to become $leader", 10000
+    )
+  }
+
+  def waitForBrokerOutOfIsr(client: AdminClient, partitions: Set[TopicPartition], brokerId: Int): Unit = {
+    TestUtils.waitUntilTrue(
+      () => {
+        val description = client.describeTopics(partitions.map(_.topic).asJava).all.get.asScala
+        val isr = description.values.flatMap(_.partitions.asScala.flatMap(_.isr.asScala))
+        isr.forall(_.id != brokerId)
+      },
+      s"Expect broker $brokerId to no longer be in any ISR for $partitions"
+    )
+  }
 }
diff --git a/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala b/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala
index 29747615d2d23..c0c8c95b65959 100755
--- a/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala
+++ b/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala
@@ -18,7 +18,7 @@
 package kafka.integration
 
 import org.apache.kafka.common.config.ConfigException
-import org.junit.{After, Before, Ignore, Test}
+import org.junit.{After, Before, Test}
 
 import scala.util.Random
 import scala.collection.JavaConverters._
diff --git a/core/src/test/scala/unit/kafka/server/LogDirFailureTest.scala b/core/src/test/scala/unit/kafka/server/LogDirFailureTest.scala
index 6b69c41fed4f4..f83562c2c1e76 100644
--- a/core/src/test/scala/unit/kafka/server/LogDirFailureTest.scala
+++ b/core/src/test/scala/unit/kafka/server/LogDirFailureTest.scala
@@ -22,7 +22,6 @@ import java.util.concurrent.{ExecutionException, TimeUnit}
 
 import kafka.server.LogDirFailureTest._
 import kafka.api.IntegrationTestHarness
-import kafka.cluster.Partition
 import kafka.controller.{OfflineReplica, PartitionAndReplica}
 import kafka.utils.{CoreUtils, Exit, TestUtils}
 import org.apache.kafka.clients.consumer.KafkaConsumer
diff --git a/core/src/test/scala/unit/kafka/server/LogRecoveryTest.scala b/core/src/test/scala/unit/kafka/server/LogRecoveryTest.scala
index 780d18917b99d..6ab31385bc08d 100755
--- a/core/src/test/scala/unit/kafka/server/LogRecoveryTest.scala
+++ b/core/src/test/scala/unit/kafka/server/LogRecoveryTest.scala
@@ -23,7 +23,6 @@ import TestUtils._
 import kafka.zk.ZooKeeperTestHarness
 import java.io.File
 
-import kafka.cluster.Partition
 import kafka.server.checkpoints.OffsetCheckpointFile
 import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord}
 import org.apache.kafka.common.TopicPartition

From 17712b96c86d6c2cce695531d7e5f2e3d5718b21 Mon Sep 17 00:00:00 2001
From: cadonna 
Date: Mon, 3 Jun 2019 23:31:19 +0200
Subject: [PATCH 0324/1071] KAFKA-6819: Pt. 1 - Refactor thread-level Streams
 metrics (#6631)

* StreamsMetricsImpl wraps the Kafka Streams' metrics registry and provides logic to create
and register sensors and their corresponding metrics. An example for such logic can be found in
threadLevelSensor(). Furthermore, StreamsMetricsmpl keeps track of the sensors on the
different levels of an application, i.e., thread, task, etc., and provides logic to remove sensors per
level, e.g., removeAllThreadLevelSensors(). There is one StreamsMetricsImpl object per
application instance.
* ThreadMetrics contains only static methods that specify all built-in thread-level sensors and
metrics and provide logic to register and retrieve those thread-level sensors, e.g., commitSensor().
* From anywhere inside the code base with access to StreamsMetricsImpl, thread-level sensors can be accessed by using ThreadMetrics.
* ThreadsMetrics does not inherit from StreamsMetricsImpl anymore.

Reviewers: A. Sophie Blee-Goldman , John Roesler , Guozhang Wang 
---
 build.gradle                                  |   2 +
 .../kstream/internals/KStreamAggregate.java   |   6 +-
 .../kstream/internals/KStreamKStreamJoin.java |   7 +-
 .../internals/KStreamKTableJoinProcessor.java |   7 +-
 .../kstream/internals/KStreamReduce.java      |   6 +-
 .../KStreamSessionWindowAggregate.java        |   5 +-
 .../internals/KStreamWindowAggregate.java     |   9 +-
 .../internals/KTableKTableInnerJoin.java      |   6 +-
 .../internals/KTableKTableLeftJoin.java       |   6 +-
 .../internals/KTableKTableOuterJoin.java      |   6 +-
 .../internals/KTableKTableRightJoin.java      |   6 +-
 .../kstream/internals/KTableSource.java       |   6 +-
 .../internals/GlobalStateUpdateTask.java      |   3 +-
 .../processor/internals/RecordQueue.java      |  16 +-
 .../processor/internals/StreamTask.java       |  44 +---
 .../processor/internals/StreamThread.java     | 110 +++-----
 .../internals/metrics/StreamsMetricsImpl.java | 112 ++++++--
 .../internals/metrics/ThreadMetrics.java      | 179 +++++++++++++
 .../integration/MetricsIntegrationTest.java   |   6 +-
 ...amSessionWindowAggregateProcessorTest.java |   3 +
 .../processor/internals/StreamTaskTest.java   |  20 +-
 .../processor/internals/StreamThreadTest.java | 108 +++++---
 .../metrics/StreamsMetricsImplTest.java       |  39 ++-
 .../internals/metrics/ThreadMetricsTest.java  | 244 ++++++++++++++++++
 .../StreamThreadStateStoreProviderTest.java   |   3 +-
 .../kafka/streams/TopologyTestDriver.java     |  25 +-
 .../processor/MockProcessorContext.java       |   8 +-
 27 files changed, 773 insertions(+), 219 deletions(-)
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetrics.java
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetricsTest.java

diff --git a/build.gradle b/build.gradle
index 07200a80b5fed..516a06cfed2b1 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1120,6 +1120,8 @@ project(':streams') {
     testCompile libs.log4j
     testCompile libs.junit
     testCompile libs.easymock
+    testCompile libs.powermockJunit4
+    testCompile libs.powermockEasymock
     testCompile libs.bcpkix
     testCompile libs.hamcrest
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamAggregate.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamAggregate.java
index 8fd4f5b4b7d48..ca7266f5aba35 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamAggregate.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamAggregate.java
@@ -16,12 +16,14 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.streams.kstream.Aggregator;
 import org.apache.kafka.streams.kstream.Initializer;
 import org.apache.kafka.streams.processor.AbstractProcessor;
 import org.apache.kafka.streams.processor.Processor;
 import org.apache.kafka.streams.processor.ProcessorContext;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.apache.kafka.streams.state.TimestampedKeyValueStore;
 import org.apache.kafka.streams.state.ValueAndTimestamp;
 import org.slf4j.Logger;
@@ -57,6 +59,7 @@ public void enableSendingOldValues() {
     private class KStreamAggregateProcessor extends AbstractProcessor {
         private TimestampedKeyValueStore store;
         private StreamsMetricsImpl metrics;
+        private Sensor skippedRecordsSensor;
         private TimestampedTupleForwarder tupleForwarder;
 
         @SuppressWarnings("unchecked")
@@ -64,6 +67,7 @@ private class KStreamAggregateProcessor extends AbstractProcessor {
         public void init(final ProcessorContext context) {
             super.init(context);
             metrics = (StreamsMetricsImpl) context.metrics();
+            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(metrics);
             store = (TimestampedKeyValueStore) context.getStateStore(storeName);
             tupleForwarder = new TimestampedTupleForwarder<>(
                 store,
@@ -80,7 +84,7 @@ public void process(final K key, final V value) {
                     "Skipping record due to null key or value. key=[{}] value=[{}] topic=[{}] partition=[{}] offset=[{}]",
                     key, value, context().topic(), context().partition(), context().offset()
                 );
-                metrics.skippedRecordsSensor().record();
+                skippedRecordsSensor.record();
                 return;
             }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoin.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoin.java
index 17d7837a03f42..97debbc802195 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoin.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoin.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.kstream.ValueJoiner;
 import org.apache.kafka.streams.processor.AbstractProcessor;
@@ -24,12 +25,12 @@
 import org.apache.kafka.streams.processor.ProcessorSupplier;
 import org.apache.kafka.streams.processor.To;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.apache.kafka.streams.state.WindowStore;
 import org.apache.kafka.streams.state.WindowStoreIterator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-
 class KStreamKStreamJoin implements ProcessorSupplier {
     private static final Logger LOG = LoggerFactory.getLogger(KStreamKStreamJoin.class);
 
@@ -57,12 +58,14 @@ private class KStreamKStreamJoinProcessor extends AbstractProcessor {
 
         private WindowStore otherWindow;
         private StreamsMetricsImpl metrics;
+        private Sensor skippedRecordsSensor;
 
         @SuppressWarnings("unchecked")
         @Override
         public void init(final ProcessorContext context) {
             super.init(context);
             metrics = (StreamsMetricsImpl) context.metrics();
+            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(metrics);
 
             otherWindow = (WindowStore) context.getStateStore(otherWindowName);
         }
@@ -81,7 +84,7 @@ public void process(final K key, final V1 value) {
                     "Skipping record due to null key or value. key=[{}] value=[{}] topic=[{}] partition=[{}] offset=[{}]",
                     key, value, context().topic(), context().partition(), context().offset()
                 );
-                metrics.skippedRecordsSensor().record();
+                skippedRecordsSensor.record();
                 return;
             }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamKTableJoinProcessor.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamKTableJoinProcessor.java
index dcf0799c190fc..92fd4d52e5ae2 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamKTableJoinProcessor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamKTableJoinProcessor.java
@@ -16,11 +16,13 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.streams.kstream.KeyValueMapper;
 import org.apache.kafka.streams.kstream.ValueJoiner;
 import org.apache.kafka.streams.processor.AbstractProcessor;
 import org.apache.kafka.streams.processor.ProcessorContext;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -34,6 +36,7 @@ class KStreamKTableJoinProcessor extends AbstractProcessor joiner;
     private final boolean leftJoin;
     private StreamsMetricsImpl metrics;
+    private Sensor skippedRecordsSensor;
 
     KStreamKTableJoinProcessor(final KTableValueGetter valueGetter,
                                final KeyValueMapper keyMapper,
@@ -49,6 +52,8 @@ class KStreamKTableJoinProcessor extends AbstractProcessor {
         private TimestampedKeyValueStore store;
         private TimestampedTupleForwarder tupleForwarder;
         private StreamsMetricsImpl metrics;
+        private Sensor skippedRecordsSensor;
 
         @SuppressWarnings("unchecked")
         @Override
         public void init(final ProcessorContext context) {
             super.init(context);
             metrics = (StreamsMetricsImpl) context.metrics();
+            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(metrics);
             store = (TimestampedKeyValueStore) context.getStateStore(storeName);
             tupleForwarder = new TimestampedTupleForwarder<>(
                 store,
@@ -78,7 +82,7 @@ public void process(final K key, final V value) {
                     "Skipping record due to null key or value. key=[{}] value=[{}] topic=[{}] partition=[{}] offset=[{}]",
                     key, value, context().topic(), context().partition(), context().offset()
                 );
-                metrics.skippedRecordsSensor().record();
+                skippedRecordsSensor.record();
                 return;
             }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregate.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregate.java
index fdbf475c9f2ff..cb117b70ec92c 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregate.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregate.java
@@ -30,6 +30,7 @@
 import org.apache.kafka.streams.processor.ProcessorContext;
 import org.apache.kafka.streams.processor.internals.InternalProcessorContext;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.apache.kafka.streams.state.KeyValueIterator;
 import org.apache.kafka.streams.state.SessionStore;
 import org.apache.kafka.streams.state.ValueAndTimestamp;
@@ -83,6 +84,7 @@ private class KStreamSessionWindowAggregateProcessor extends AbstractProcessor) context.getStateStore(storeName);
             tupleForwarder = new SessionTupleForwarder<>(store, context, new SessionCacheFlushListener<>(context), sendOldValues);
@@ -106,7 +109,7 @@ public void process(final K key, final V value) {
                     "Skipping record due to null key. value=[{}] topic=[{}] partition=[{}] offset=[{}]",
                     value, context().topic(), context().partition(), context().offset()
                 );
-                metrics.skippedRecordsSensor().record();
+                skippedRecordsSensor.record();
                 return;
             }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregate.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregate.java
index 3458ca02e1cae..2983a3a9c7ea9 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregate.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregate.java
@@ -29,6 +29,7 @@
 import org.apache.kafka.streams.processor.ProcessorContext;
 import org.apache.kafka.streams.processor.internals.InternalProcessorContext;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.apache.kafka.streams.state.TimestampedWindowStore;
 import org.apache.kafka.streams.state.ValueAndTimestamp;
 import org.slf4j.Logger;
@@ -79,6 +80,7 @@ private class KStreamWindowAggregateProcessor extends AbstractProcessor {
         private StreamsMetricsImpl metrics;
         private InternalProcessorContext internalProcessorContext;
         private Sensor lateRecordDropSensor;
+        private Sensor skippedRecordsSensor;
         private long observedStreamTime = ConsumerRecord.NO_TIMESTAMP;
 
         @SuppressWarnings("unchecked")
@@ -86,8 +88,11 @@ private class KStreamWindowAggregateProcessor extends AbstractProcessor {
         public void init(final ProcessorContext context) {
             super.init(context);
             internalProcessorContext = (InternalProcessorContext) context;
-            metrics = (StreamsMetricsImpl) context.metrics();
+
+            metrics = internalProcessorContext.metrics();
+
             lateRecordDropSensor = Sensors.lateRecordDropSensor(internalProcessorContext);
+            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(metrics);
             windowStore = (TimestampedWindowStore) context.getStateStore(storeName);
             tupleForwarder = new TimestampedTupleForwarder<>(
                 windowStore,
@@ -103,7 +108,7 @@ public void process(final K key, final V value) {
                     "Skipping record due to null key. value=[{}] topic=[{}] partition=[{}] offset=[{}]",
                     value, context().topic(), context().partition(), context().offset()
                 );
-                metrics.skippedRecordsSensor().record();
+                skippedRecordsSensor.record();
                 return;
             }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableInnerJoin.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableInnerJoin.java
index 2d4cbc8415c9a..005ea809b3fe7 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableInnerJoin.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableInnerJoin.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.streams.kstream.KeyValueMapper;
 import org.apache.kafka.streams.kstream.ValueJoiner;
 import org.apache.kafka.streams.processor.AbstractProcessor;
@@ -23,6 +24,7 @@
 import org.apache.kafka.streams.processor.ProcessorContext;
 import org.apache.kafka.streams.processor.To;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.apache.kafka.streams.state.ValueAndTimestamp;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -66,6 +68,7 @@ private class KTableKTableJoinProcessor extends AbstractProcessor>
 
         private final KTableValueGetter valueGetter;
         private StreamsMetricsImpl metrics;
+        private Sensor skippedRecordsSensor;
 
         KTableKTableJoinProcessor(final KTableValueGetter valueGetter) {
             this.valueGetter = valueGetter;
@@ -75,6 +78,7 @@ private class KTableKTableJoinProcessor extends AbstractProcessor>
         public void init(final ProcessorContext context) {
             super.init(context);
             metrics = (StreamsMetricsImpl) context.metrics();
+            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(metrics);
             valueGetter.init(context);
         }
 
@@ -86,7 +90,7 @@ public void process(final K key, final Change change) {
                     "Skipping record due to null key. change=[{}] topic=[{}] partition=[{}] offset=[{}]",
                     change, context().topic(), context().partition(), context().offset()
                 );
-                metrics.skippedRecordsSensor().record();
+                skippedRecordsSensor.record();
                 return;
             }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoin.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoin.java
index 03e59d929ba4a..4bd6af99d6db5 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoin.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoin.java
@@ -16,12 +16,14 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.streams.kstream.ValueJoiner;
 import org.apache.kafka.streams.processor.AbstractProcessor;
 import org.apache.kafka.streams.processor.Processor;
 import org.apache.kafka.streams.processor.ProcessorContext;
 import org.apache.kafka.streams.processor.To;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.apache.kafka.streams.state.ValueAndTimestamp;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -65,6 +67,7 @@ private class KTableKTableLeftJoinProcessor extends AbstractProcessor valueGetter;
         private StreamsMetricsImpl metrics;
+        private Sensor skippedRecordsSensor;
 
         KTableKTableLeftJoinProcessor(final KTableValueGetter valueGetter) {
             this.valueGetter = valueGetter;
@@ -74,6 +77,7 @@ private class KTableKTableLeftJoinProcessor extends AbstractProcessor change) {
                     "Skipping record due to null key. change=[{}] topic=[{}] partition=[{}] offset=[{}]",
                     change, context().topic(), context().partition(), context().offset()
                 );
-                metrics.skippedRecordsSensor().record();
+                skippedRecordsSensor.record();
                 return;
             }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableOuterJoin.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableOuterJoin.java
index d600718f78c7a..794f21229adc4 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableOuterJoin.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableOuterJoin.java
@@ -16,12 +16,14 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.streams.kstream.ValueJoiner;
 import org.apache.kafka.streams.processor.AbstractProcessor;
 import org.apache.kafka.streams.processor.Processor;
 import org.apache.kafka.streams.processor.ProcessorContext;
 import org.apache.kafka.streams.processor.To;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.apache.kafka.streams.state.ValueAndTimestamp;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -64,6 +66,7 @@ private class KTableKTableOuterJoinProcessor extends AbstractProcessor valueGetter;
         private StreamsMetricsImpl metrics;
+        private Sensor skippedRecordsSensor;
 
         KTableKTableOuterJoinProcessor(final KTableValueGetter valueGetter) {
             this.valueGetter = valueGetter;
@@ -73,6 +76,7 @@ private class KTableKTableOuterJoinProcessor extends AbstractProcessor change) {
                     "Skipping record due to null key. change=[{}] topic=[{}] partition=[{}] offset=[{}]",
                     change, context().topic(), context().partition(), context().offset()
                 );
-                metrics.skippedRecordsSensor().record();
+                skippedRecordsSensor.record();
                 return;
             }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableRightJoin.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableRightJoin.java
index fbddd9b3565c9..981b5bae6e9b4 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableRightJoin.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableRightJoin.java
@@ -16,12 +16,14 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.streams.kstream.ValueJoiner;
 import org.apache.kafka.streams.processor.AbstractProcessor;
 import org.apache.kafka.streams.processor.Processor;
 import org.apache.kafka.streams.processor.ProcessorContext;
 import org.apache.kafka.streams.processor.To;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.apache.kafka.streams.state.ValueAndTimestamp;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -63,6 +65,7 @@ private class KTableKTableRightJoinProcessor extends AbstractProcessor valueGetter;
         private StreamsMetricsImpl metrics;
+        private Sensor skippedRecordsSensor;
 
         KTableKTableRightJoinProcessor(final KTableValueGetter valueGetter) {
             this.valueGetter = valueGetter;
@@ -72,6 +75,7 @@ private class KTableKTableRightJoinProcessor extends AbstractProcessor change) {
                     "Skipping record due to null key. change=[{}] topic=[{}] partition=[{}] offset=[{}]",
                     change, context().topic(), context().partition(), context().offset()
                 );
-                metrics.skippedRecordsSensor().record();
+                skippedRecordsSensor.record();
                 return;
             }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableSource.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableSource.java
index 864cf51c5b9d0..bee89b35f0035 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableSource.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableSource.java
@@ -16,11 +16,13 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.streams.processor.AbstractProcessor;
 import org.apache.kafka.streams.processor.Processor;
 import org.apache.kafka.streams.processor.ProcessorContext;
 import org.apache.kafka.streams.processor.ProcessorSupplier;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.apache.kafka.streams.state.TimestampedKeyValueStore;
 import org.apache.kafka.streams.state.ValueAndTimestamp;
 import org.slf4j.Logger;
@@ -70,12 +72,14 @@ private class KTableSourceProcessor extends AbstractProcessor {
         private TimestampedKeyValueStore store;
         private TimestampedTupleForwarder tupleForwarder;
         private StreamsMetricsImpl metrics;
+        private Sensor skippedRecordsSensor;
 
         @SuppressWarnings("unchecked")
         @Override
         public void init(final ProcessorContext context) {
             super.init(context);
             metrics = (StreamsMetricsImpl) context.metrics();
+            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(metrics);
             if (queryableName != null) {
                 store = (TimestampedKeyValueStore) context.getStateStore(queryableName);
                 tupleForwarder = new TimestampedTupleForwarder<>(
@@ -94,7 +98,7 @@ public void process(final K key, final V value) {
                     "Skipping record due to null key. topic=[{}] partition=[{}] offset=[{}]",
                     context().topic(), context().partition(), context().offset()
                 );
-                metrics.skippedRecordsSensor().record();
+                skippedRecordsSensor.record();
                 return;
             }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStateUpdateTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStateUpdateTask.java
index 29c861e4291ce..d6f60e351334e 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStateUpdateTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStateUpdateTask.java
@@ -21,6 +21,7 @@
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.streams.errors.DeserializationExceptionHandler;
 import org.apache.kafka.streams.errors.StreamsException;
+import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -69,7 +70,7 @@ public Map initialize() {
                     source,
                     deserializationExceptionHandler,
                     logContext,
-                    processorContext.metrics().skippedRecordsSensor()
+                    ThreadMetrics.skipRecordSensor(processorContext.metrics())
                 )
             );
         }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordQueue.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordQueue.java
index 86f5be383cd8d..6f3e70b9922fc 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordQueue.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordQueue.java
@@ -18,17 +18,17 @@
 
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.streams.errors.DeserializationExceptionHandler;
 import org.apache.kafka.streams.errors.StreamsException;
 import org.apache.kafka.streams.processor.ProcessorContext;
 import org.apache.kafka.streams.processor.TimestampExtractor;
-import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.slf4j.Logger;
 
 import java.util.ArrayDeque;
 
-
 /**
  * RecordQueue is a FIFO queue of {@link StampedRecord} (ConsumerRecord + timestamp). It also keeps track of the
  * partition timestamp defined as the minimum timestamp of records in its queue; in addition, its partition
@@ -48,6 +48,8 @@ public class RecordQueue {
 
     private StampedRecord headRecord = null;
 
+    private Sensor skipRecordsSensor;
+
     RecordQueue(final TopicPartition partition,
                 final SourceNode source,
                 final TimestampExtractor timestampExtractor,
@@ -58,13 +60,14 @@ public class RecordQueue {
         this.partition = partition;
         this.fifoQueue = new ArrayDeque<>();
         this.timestampExtractor = timestampExtractor;
-        this.recordDeserializer = new RecordDeserializer(
+        this.processorContext = processorContext;
+        skipRecordsSensor = ThreadMetrics.skipRecordSensor(processorContext.metrics());
+        recordDeserializer = new RecordDeserializer(
             source,
             deserializationExceptionHandler,
             logContext,
-            processorContext.metrics().skippedRecordsSensor()
+            skipRecordsSensor
         );
-        this.processorContext = processorContext;
         this.log = logContext.logger(RecordQueue.class);
     }
 
@@ -180,7 +183,8 @@ private void updateHead() {
                         "Skipping record due to negative extracted timestamp. topic=[{}] partition=[{}] offset=[{}] extractedTimestamp=[{}] extractor=[{}]",
                         deserialized.topic(), deserialized.partition(), deserialized.offset(), timestamp, timestampExtractor.getClass().getCanonicalName()
                 );
-                ((StreamsMetricsImpl) processorContext.metrics()).skippedRecordsSensor().record();
+
+                skipRecordsSensor.record();
                 continue;
             }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
index 80653c5e5d44b..4fd57ba3db898 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
@@ -45,6 +45,7 @@
 import org.apache.kafka.streams.processor.TimestampExtractor;
 import org.apache.kafka.streams.processor.internals.metrics.CumulativeCount;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.apache.kafka.streams.state.internals.ThreadCache;
 
 import java.io.IOException;
@@ -78,7 +79,7 @@ public class StreamTask extends AbstractTask implements ProcessorNodePunctuator
     private final PunctuationQueue systemTimePunctuationQueue;
     private final ProducerSupplier producerSupplier;
 
-    private Sensor closeSensor;
+    private Sensor closeTaskSensor;
     private long idleStartTime;
     private Producer producer;
     private boolean commitRequested = false;
@@ -96,24 +97,7 @@ protected static final class TaskMetrics {
             final String group = "stream-task-metrics";
 
             // first add the global operation metrics if not yet, with the global tags only
-            final Map allTagMap = metrics.tagMap("task-id", "all");
-            final Sensor parent = metrics.threadLevelSensor("commit", Sensor.RecordingLevel.DEBUG);
-            parent.add(
-                new MetricName("commit-latency-avg", group, "The average latency of commit operation.", allTagMap),
-                new Avg()
-            );
-            parent.add(
-                new MetricName("commit-latency-max", group, "The max latency of commit operation.", allTagMap),
-                new Max()
-            );
-            parent.add(
-                new MetricName("commit-rate", group, "The average number of occurrence of commit operation per second.", allTagMap),
-                new Rate(TimeUnit.SECONDS, new Count())
-            );
-            parent.add(
-                new MetricName("commit-total", group, "The total number of occurrence of commit operations.", allTagMap),
-                new CumulativeCount()
-            );
+            final Sensor parent = ThreadMetrics.commitOverTasksSensor(metrics);
 
             // add the operation metrics with additional tags
             final Map tagMap = metrics.tagMap("task-id", taskName);
@@ -167,9 +151,8 @@ public StreamTask(final TaskId id,
                       final StateDirectory stateDirectory,
                       final ThreadCache cache,
                       final Time time,
-                      final ProducerSupplier producerSupplier,
-                      final Sensor closeSensor) {
-        this(id, partitions, topology, consumer, changelogReader, config, metrics, stateDirectory, cache, time, producerSupplier, null, closeSensor);
+                      final ProducerSupplier producerSupplier) {
+        this(id, partitions, topology, consumer, changelogReader, config, metrics, stateDirectory, cache, time, producerSupplier, null);
     }
 
     public StreamTask(final TaskId id,
@@ -178,20 +161,20 @@ public StreamTask(final TaskId id,
                       final Consumer consumer,
                       final ChangelogReader changelogReader,
                       final StreamsConfig config,
-                      final StreamsMetricsImpl metrics,
+                      final StreamsMetricsImpl streamsMetrics,
                       final StateDirectory stateDirectory,
                       final ThreadCache cache,
                       final Time time,
                       final ProducerSupplier producerSupplier,
-                      final RecordCollector recordCollector,
-                      final Sensor closeSensor) {
+                      final RecordCollector recordCollector) {
         super(id, partitions, topology, consumer, changelogReader, false, stateDirectory, config);
 
         this.time = time;
         this.producerSupplier = producerSupplier;
         this.producer = producerSupplier.get();
-        this.closeSensor = closeSensor;
-        this.taskMetrics = new TaskMetrics(id, metrics);
+        this.taskMetrics = new TaskMetrics(id, streamsMetrics);
+
+        closeTaskSensor = ThreadMetrics.closeTaskSensor(streamsMetrics);
 
         final ProductionExceptionHandler productionExceptionHandler = config.defaultProductionExceptionHandler();
 
@@ -200,8 +183,7 @@ public StreamTask(final TaskId id,
                 id.toString(),
                 logContext,
                 productionExceptionHandler,
-                metrics.skippedRecordsSensor()
-            );
+                ThreadMetrics.skipRecordSensor(streamsMetrics));
         } else {
             this.recordCollector = recordCollector;
         }
@@ -220,7 +202,7 @@ public StreamTask(final TaskId id,
         final Map partitionQueues = new HashMap<>();
 
         // initialize the topology with its own context
-        final ProcessorContextImpl processorContextImpl = new ProcessorContextImpl(id, this, config, this.recordCollector, stateMgr, metrics, cache);
+        final ProcessorContextImpl processorContextImpl = new ProcessorContextImpl(id, this, config, this.recordCollector, stateMgr, streamsMetrics, cache);
         processorContext = processorContextImpl;
 
         final TimestampExtractor defaultTimestampExtractor = config.defaultTimestampExtractor();
@@ -691,7 +673,7 @@ public void closeSuspended(final boolean clean,
         partitionGroup.close();
         taskMetrics.removeAllSensors();
 
-        closeSensor.record();
+        closeTaskSensor.record();
 
         if (firstException != null) {
             throw firstException;
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index 419e181196365..4dc1bde9f3a63 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -31,9 +31,6 @@
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
-import org.apache.kafka.common.metrics.stats.Count;
-import org.apache.kafka.common.metrics.stats.Rate;
-import org.apache.kafka.common.metrics.stats.Total;
 import org.apache.kafka.common.serialization.ByteArrayDeserializer;
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.common.utils.Time;
@@ -45,8 +42,8 @@
 import org.apache.kafka.streams.processor.TaskId;
 import org.apache.kafka.streams.processor.TaskMetadata;
 import org.apache.kafka.streams.processor.ThreadMetadata;
-import org.apache.kafka.streams.processor.internals.metrics.CumulativeCount;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.apache.kafka.streams.state.internals.ThreadCache;
 import org.slf4j.Logger;
 
@@ -62,7 +59,6 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import static java.util.Collections.singleton;
@@ -340,7 +336,7 @@ static abstract class AbstractTaskCreator {
         final String applicationId;
         final InternalTopologyBuilder builder;
         final StreamsConfig config;
-        final StreamsMetricsThreadImpl streamsMetrics;
+        final StreamsMetricsImpl streamsMetrics;
         final StateDirectory stateDirectory;
         final ChangelogReader storeChangelogReader;
         final Time time;
@@ -349,7 +345,7 @@ static abstract class AbstractTaskCreator {
 
         AbstractTaskCreator(final InternalTopologyBuilder builder,
                             final StreamsConfig config,
-                            final StreamsMetricsThreadImpl streamsMetrics,
+                            final StreamsMetricsImpl streamsMetrics,
                             final StateDirectory stateDirectory,
                             final ChangelogReader storeChangelogReader,
                             final Time time,
@@ -398,10 +394,11 @@ static class TaskCreator extends AbstractTaskCreator {
         private final KafkaClientSupplier clientSupplier;
         private final String threadClientId;
         private final Producer threadProducer;
+        private final Sensor createTaskSensor;
 
         TaskCreator(final InternalTopologyBuilder builder,
                     final StreamsConfig config,
-                    final StreamsMetricsThreadImpl streamsMetrics,
+                    final StreamsMetricsImpl streamsMetrics,
                     final StateDirectory stateDirectory,
                     final ChangelogReader storeChangelogReader,
                     final ThreadCache cache,
@@ -422,13 +419,14 @@ static class TaskCreator extends AbstractTaskCreator {
             this.clientSupplier = clientSupplier;
             this.threadProducer = threadProducer;
             this.threadClientId = threadClientId;
+            createTaskSensor = ThreadMetrics.createTaskSensor(streamsMetrics);
         }
 
         @Override
         StreamTask createTask(final Consumer consumer,
                               final TaskId taskId,
                               final Set partitions) {
-            streamsMetrics.taskCreatedSensor.record();
+            createTaskSensor.record();
 
             return new StreamTask(
                 taskId,
@@ -441,8 +439,7 @@ StreamTask createTask(final Consumer consumer,
                 stateDirectory,
                 cache,
                 time,
-                () -> createProducer(taskId),
-                streamsMetrics.taskClosedSensor);
+                () -> createProducer(taskId));
         }
 
         private Producer createProducer(final TaskId id) {
@@ -470,9 +467,11 @@ public void close() {
     }
 
     static class StandbyTaskCreator extends AbstractTaskCreator {
+        private final Sensor createTaskSensor;
+
         StandbyTaskCreator(final InternalTopologyBuilder builder,
                            final StreamsConfig config,
-                           final StreamsMetricsThreadImpl streamsMetrics,
+                           final StreamsMetricsImpl streamsMetrics,
                            final StateDirectory stateDirectory,
                            final ChangelogReader storeChangelogReader,
                            final Time time,
@@ -485,13 +484,14 @@ static class StandbyTaskCreator extends AbstractTaskCreator {
                 storeChangelogReader,
                 time,
                 log);
+            createTaskSensor = ThreadMetrics.createTaskSensor(streamsMetrics);
         }
 
         @Override
         StandbyTask createTask(final Consumer consumer,
                                final TaskId taskId,
                                final Set partitions) {
-            streamsMetrics.taskCreatedSensor.record();
+            createTaskSensor.record();
 
             final ProcessorTopology topology = builder.build(taskId.topicGroupId);
 
@@ -516,47 +516,6 @@ StandbyTask createTask(final Consumer consumer,
         }
     }
 
-    static class StreamsMetricsThreadImpl extends StreamsMetricsImpl {
-
-        private final Sensor commitTimeSensor;
-        private final Sensor pollTimeSensor;
-        private final Sensor processTimeSensor;
-        private final Sensor punctuateTimeSensor;
-        private final Sensor taskCreatedSensor;
-        private final Sensor taskClosedSensor;
-
-        StreamsMetricsThreadImpl(final Metrics metrics, final String threadName) {
-            super(metrics, threadName);
-            final String group = "stream-metrics";
-
-            commitTimeSensor = threadLevelSensor("commit-latency", Sensor.RecordingLevel.INFO);
-            addAvgMaxLatency(commitTimeSensor, group, tagMap(), "commit");
-            addInvocationRateAndCount(commitTimeSensor, group, tagMap(), "commit");
-
-            pollTimeSensor = threadLevelSensor("poll-latency", Sensor.RecordingLevel.INFO);
-            addAvgMaxLatency(pollTimeSensor, group, tagMap(), "poll");
-            // can't use addInvocationRateAndCount due to non-standard description string
-            pollTimeSensor.add(metrics.metricName("poll-rate", group, "The average per-second number of record-poll calls", tagMap()), new Rate(TimeUnit.SECONDS, new Count()));
-            pollTimeSensor.add(metrics.metricName("poll-total", group, "The total number of record-poll calls", tagMap()), new CumulativeCount());
-
-            processTimeSensor = threadLevelSensor("process-latency", Sensor.RecordingLevel.INFO);
-            addAvgMaxLatency(processTimeSensor, group, tagMap(), "process");
-            addInvocationRateAndCount(processTimeSensor, group, tagMap(), "process");
-
-            punctuateTimeSensor = threadLevelSensor("punctuate-latency", Sensor.RecordingLevel.INFO);
-            addAvgMaxLatency(punctuateTimeSensor, group, tagMap(), "punctuate");
-            addInvocationRateAndCount(punctuateTimeSensor, group, tagMap(), "punctuate");
-
-            taskCreatedSensor = threadLevelSensor("task-created", Sensor.RecordingLevel.INFO);
-            taskCreatedSensor.add(metrics.metricName("task-created-rate", "stream-metrics", "The average per-second number of newly created tasks", tagMap()), new Rate(TimeUnit.SECONDS, new Count()));
-            taskCreatedSensor.add(metrics.metricName("task-created-total", "stream-metrics", "The total number of newly created tasks", tagMap()), new Total());
-
-            taskClosedSensor = threadLevelSensor("task-closed", Sensor.RecordingLevel.INFO);
-            taskClosedSensor.add(metrics.metricName("task-closed-rate", group, "The average per-second number of closed tasks", tagMap()), new Rate(TimeUnit.SECONDS, new Count()));
-            taskClosedSensor.add(metrics.metricName("task-closed-total", group, "The total number of closed tasks", tagMap()), new Total());
-        }
-    }
-
     private final Time time;
     private final Logger log;
     private final String logPrefix;
@@ -566,9 +525,14 @@ static class StreamsMetricsThreadImpl extends StreamsMetricsImpl {
     private final int maxPollTimeMs;
     private final String originalReset;
     private final TaskManager taskManager;
-    private final StreamsMetricsThreadImpl streamsMetrics;
     private final AtomicInteger assignmentErrorCode;
 
+    private final StreamsMetricsImpl streamsMetrics;
+    private final Sensor commitSensor;
+    private final Sensor pollSensor;
+    private final Sensor punctuateSensor;
+    private final Sensor processSensor;
+
     private long now;
     private long lastPollMs;
     private long lastCommitMs;
@@ -620,10 +584,7 @@ public static StreamThread create(final InternalTopologyBuilder builder,
             threadProducer = clientSupplier.getProducer(producerConfigs);
         }
 
-        final StreamsMetricsThreadImpl streamsMetrics = new StreamsMetricsThreadImpl(
-            metrics,
-            threadClientId
-        );
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, threadClientId);
 
         final ThreadCache cache = new ThreadCache(logContext, cacheSizeBytes, streamsMetrics);
 
@@ -697,7 +658,7 @@ public StreamThread(final Time time,
                         final Consumer consumer,
                         final String originalReset,
                         final TaskManager taskManager,
-                        final StreamsMetricsThreadImpl streamsMetrics,
+                        final StreamsMetricsImpl streamsMetrics,
                         final InternalTopologyBuilder builder,
                         final String threadClientId,
                         final LogContext logContext,
@@ -707,9 +668,24 @@ public StreamThread(final Time time,
         this.stateLock = new Object();
         this.standbyRecords = new HashMap<>();
 
+        this.streamsMetrics = streamsMetrics;
+        this.commitSensor = ThreadMetrics.commitSensor(streamsMetrics);
+        this.pollSensor = ThreadMetrics.pollSensor(streamsMetrics);
+        this.processSensor = ThreadMetrics.processSensor(streamsMetrics);
+        this.punctuateSensor = ThreadMetrics.punctuateSensor(streamsMetrics);
+
+        // The following sensors are created here but their references are not stored in this object, since within
+        // this object they are not recorded. The sensors are created here so that the stream threads starts with all
+        // its metrics initialised. Otherwise, those sensors would have been created during processing, which could
+        // lead to missing metrics. For instance, if no task were created, the metrics for created and closed
+        // tasks would never be added to the metrics.
+        ThreadMetrics.createTaskSensor(streamsMetrics);
+        ThreadMetrics.closeTaskSensor(streamsMetrics);
+        ThreadMetrics.skipRecordSensor(streamsMetrics);
+        ThreadMetrics.commitOverTasksSensor(streamsMetrics);
+
         this.time = time;
         this.builder = builder;
-        this.streamsMetrics = streamsMetrics;
         this.logPrefix = logContext.logPrefix();
         this.log = logContext.logger(StreamThread.class);
         this.rebalanceListener = new RebalanceListener(time, taskManager, this, this.log);
@@ -857,7 +833,7 @@ void runOnce() {
         final long pollLatency = advanceNowAndComputeLatency();
 
         if (records != null && !records.isEmpty()) {
-            streamsMetrics.pollTimeSensor.record(pollLatency, now);
+            pollSensor.record(pollLatency, now);
             addRecordsToTasks(records);
         }
 
@@ -891,14 +867,14 @@ void runOnce() {
 
                     if (processed > 0) {
                         final long processLatency = advanceNowAndComputeLatency();
-                        streamsMetrics.processTimeSensor.record(processLatency / (double) processed, now);
+                        processSensor.record(processLatency / (double) processed, now);
 
                         // commit any tasks that have requested a commit
                         final int committed = taskManager.maybeCommitActiveTasksPerUserRequested();
 
                         if (committed > 0) {
                             final long commitLatency = advanceNowAndComputeLatency();
-                            streamsMetrics.commitTimeSensor.record(commitLatency / (double) committed, now);
+                            commitSensor.record(commitLatency / (double) committed, now);
                         }
                     } else {
                         // if there is no records to be processed, exit immediately
@@ -1031,7 +1007,7 @@ private boolean maybePunctuate() {
         final int punctuated = taskManager.punctuate();
         if (punctuated > 0) {
             final long punctuateLatency = advanceNowAndComputeLatency();
-            streamsMetrics.punctuateTimeSensor.record(punctuateLatency / (double) punctuated, now);
+            punctuateSensor.record(punctuateLatency / (double) punctuated, now);
         }
 
         return punctuated > 0;
@@ -1057,7 +1033,7 @@ boolean maybeCommit() {
             committed += taskManager.commitAll();
             if (committed > 0) {
                 final long intervalCommitLatency = advanceNowAndComputeLatency();
-                streamsMetrics.commitTimeSensor.record(intervalCommitLatency / (double) committed, now);
+                commitSensor.record(intervalCommitLatency / (double) committed, now);
 
                 // try to purge the committed records for repartition topics if possible
                 taskManager.maybePurgeCommitedRecords();
@@ -1074,7 +1050,7 @@ boolean maybeCommit() {
             final int commitPerRequested = taskManager.maybeCommitActiveTasksPerUserRequested();
             if (commitPerRequested > 0) {
                 final long requestCommitLatency = advanceNowAndComputeLatency();
-                streamsMetrics.commitTimeSensor.record(requestCommitLatency / (double) committed, now);
+                commitSensor.record(requestCommitLatency / (double) committed, now);
                 committed += commitPerRequested;
             }
         }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
index 0a47fce8101fd..46b5669d6011c 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
@@ -20,11 +20,11 @@
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.metrics.Sensor.RecordingLevel;
 import org.apache.kafka.common.metrics.stats.Avg;
 import org.apache.kafka.common.metrics.stats.Count;
 import org.apache.kafka.common.metrics.stats.Max;
 import org.apache.kafka.common.metrics.stats.Rate;
-import org.apache.kafka.common.metrics.stats.Total;
 import org.apache.kafka.streams.StreamsMetrics;
 
 import java.util.Arrays;
@@ -40,7 +40,6 @@
 public class StreamsMetricsImpl implements StreamsMetrics {
     private final Metrics metrics;
     private final Map parentSensors;
-    private final Sensor skippedRecordsSensor;
     private final String threadName;
 
     private final Deque threadLevelSensors = new LinkedList<>();
@@ -52,6 +51,20 @@ public class StreamsMetricsImpl implements StreamsMetrics {
     private static final String SENSOR_PREFIX_DELIMITER = ".";
     private static final String SENSOR_NAME_DELIMITER = ".s.";
 
+    public static final String THREAD_ID_TAG = "client-id";
+    public static final String TASK_ID_TAG = "task-id";
+
+    public static final String ALL_TASKS = "all";
+
+    public static final String LATENCY_SUFFIX = "-latency";
+    public static final String AVG_SUFFIX = "-avg";
+    public static final String MAX_SUFFIX = "-max";
+    public static final String RATE_SUFFIX = "-rate";
+    public static final String TOTAL_SUFFIX = "-total";
+
+    public static final String THREAD_LEVEL_GROUP = "stream-metrics";
+    public static final String TASK_LEVEL_GROUP = "stream-task-metrics";
+
     public static final String PROCESSOR_NODE_METRICS_GROUP = "stream-processor-node-metrics";
     public static final String PROCESSOR_NODE_ID_TAG = "processor-node-id";
 
@@ -60,30 +73,47 @@ public class StreamsMetricsImpl implements StreamsMetrics {
 
     public StreamsMetricsImpl(final Metrics metrics, final String threadName) {
         Objects.requireNonNull(metrics, "Metrics cannot be null");
-        this.threadName = threadName;
-
         this.metrics = metrics;
+        this.threadName = threadName;
 
         this.parentSensors = new HashMap<>();
-
-        final String group = "stream-metrics";
-        skippedRecordsSensor = threadLevelSensor("skipped-records", Sensor.RecordingLevel.INFO);
-        skippedRecordsSensor.add(new MetricName("skipped-records-rate", group, "The average per-second number of skipped records", tagMap()), new Rate(TimeUnit.SECONDS, new Count()));
-        skippedRecordsSensor.add(new MetricName("skipped-records-total", group, "The total number of skipped records", tagMap()), new Total());
     }
 
     public final Sensor threadLevelSensor(final String sensorName,
-                                          final Sensor.RecordingLevel recordingLevel,
+                                          final RecordingLevel recordingLevel,
                                           final Sensor... parents) {
         synchronized (threadLevelSensors) {
             final String fullSensorName = threadSensorPrefix() + SENSOR_NAME_DELIMITER + sensorName;
             final Sensor sensor = metrics.sensor(fullSensorName, recordingLevel, parents);
             threadLevelSensors.push(fullSensorName);
-
             return sensor;
         }
     }
 
+    private String threadSensorPrefix() {
+        return "internal" + SENSOR_PREFIX_DELIMITER + threadName;
+    }
+
+    public Map threadLevelTagMap() {
+        final Map tagMap = new LinkedHashMap<>();
+        tagMap.put(THREAD_ID_TAG, threadName);
+        return tagMap;
+    }
+
+    public Map threadLevelTagMap(final String... tags) {
+        final Map tagMap = threadLevelTagMap();
+        if (tags != null) {
+            if ((tags.length % 2) != 0) {
+                throw new IllegalArgumentException("Tags needs to be specified in key-value pairs");
+            }
+
+            for (int i = 0; i < tags.length; i += 2) {
+                tagMap.put(tags[i], tags[i + 1]);
+            }
+        }
+        return tagMap;
+    }
+
     public final void removeAllThreadLevelSensors() {
         synchronized (threadLevelSensors) {
             while (!threadLevelSensors.isEmpty()) {
@@ -92,13 +122,9 @@ public final void removeAllThreadLevelSensors() {
         }
     }
 
-    private String threadSensorPrefix() {
-        return "internal" + SENSOR_PREFIX_DELIMITER + threadName;
-    }
-
     public final Sensor taskLevelSensor(final String taskName,
                                         final String sensorName,
-                                        final Sensor.RecordingLevel recordingLevel,
+                                        final RecordingLevel recordingLevel,
                                         final Sensor... parents) {
         final String key = taskSensorPrefix(taskName);
         synchronized (taskLevelSensors) {
@@ -235,10 +261,6 @@ private String storeSensorPrefix(final String taskName, final String storeName)
         return taskSensorPrefix(taskName) + SENSOR_PREFIX_DELIMITER + "store" + SENSOR_PREFIX_DELIMITER + storeName;
     }
 
-    public final Sensor skippedRecordsSensor() {
-        return skippedRecordsSensor;
-    }
-
     @Override
     public Sensor addSensor(final String name, final Sensor.RecordingLevel recordingLevel) {
         return metrics.sensor(name, recordingLevel);
@@ -357,6 +379,28 @@ private String externalParentSensorName(final String operationName) {
     }
 
 
+    public static void addAvgAndMax(final Sensor sensor,
+                                    final String group,
+                                    final Map tags,
+                                    final String operation) {
+        sensor.add(
+            new MetricName(
+                operation + AVG_SUFFIX,
+                group,
+                "The average value of " + operation + ".",
+                tags),
+            new Avg()
+        );
+        sensor.add(
+            new MetricName(
+                operation + MAX_SUFFIX,
+                group,
+                "The max value of " + operation + ".",
+                tags),
+            new Max()
+        );
+    }
+
     public static void addAvgMaxLatency(final Sensor sensor,
                                         final String group,
                                         final Map tags,
@@ -382,27 +426,41 @@ public static void addAvgMaxLatency(final Sensor sensor,
     public static void addInvocationRateAndCount(final Sensor sensor,
                                                  final String group,
                                                  final Map tags,
-                                                 final String operation) {
+                                                 final String operation,
+                                                 final String descriptionOfInvocation,
+                                                 final String descriptionOfRate) {
         sensor.add(
             new MetricName(
-                operation + "-rate",
+                operation + TOTAL_SUFFIX,
                 group,
-                "The average number of occurrence of " + operation + " operation per second.",
+                descriptionOfInvocation,
                 tags
             ),
-            new Rate(TimeUnit.SECONDS, new Count())
+            new CumulativeCount()
         );
         sensor.add(
             new MetricName(
-                operation + "-total",
+                operation + RATE_SUFFIX,
                 group,
-                "The total number of occurrence of " + operation + " operations.",
+                descriptionOfRate,
                 tags
             ),
-            new CumulativeCount()
+            new Rate(TimeUnit.SECONDS, new Count())
         );
     }
 
+    public static void addInvocationRateAndCount(final Sensor sensor,
+                                                 final String group,
+                                                 final Map tags,
+                                                 final String operation) {
+        addInvocationRateAndCount(sensor,
+                                  group,
+                                  tags,
+                                  operation,
+                                  "The total number of " + operation,
+                                  "The average per-second number of " + operation);
+    }
+
     /**
      * Deletes a sensor and its parents, if any
      */
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetrics.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetrics.java
new file mode 100644
index 0000000000000..e177667b35f57
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetrics.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.metrics;
+
+import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.metrics.Sensor.RecordingLevel;
+
+import java.util.Map;
+
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.ALL_TASKS;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.LATENCY_SUFFIX;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.TASK_ID_TAG;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.TASK_LEVEL_GROUP;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.THREAD_LEVEL_GROUP;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addAvgAndMax;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addInvocationRateAndCount;
+
+public class ThreadMetrics {
+    private ThreadMetrics() {}
+
+    private static final String COMMIT = "commit";
+    private static final String POLL = "poll";
+    private static final String PROCESS = "process";
+    private static final String PUNCTUATE = "punctuate";
+    private static final String CREATE_TASK = "task-created";
+    private static final String CLOSE_TASK = "task-closed";
+    private static final String SKIP_RECORD = "skipped-records";
+
+    private static final String TOTAL_DESCRIPTION = "The total number of ";
+    private static final String RATE_DESCRIPTION = "The average per-second number of ";
+    private static final String COMMIT_DESCRIPTION = "commit calls";
+    private static final String COMMIT_TOTAL_DESCRIPTION = TOTAL_DESCRIPTION + COMMIT_DESCRIPTION;
+    private static final String COMMIT_RATE_DESCRIPTION = RATE_DESCRIPTION + COMMIT_DESCRIPTION;
+    private static final String CREATE_TASK_DESCRIPTION = "newly created tasks";
+    private static final String CREATE_TASK_TOTAL_DESCRIPTION = TOTAL_DESCRIPTION + CREATE_TASK_DESCRIPTION;
+    private static final String CREATE_TASK_RATE_DESCRIPTION = RATE_DESCRIPTION + CREATE_TASK_DESCRIPTION;
+    private static final String CLOSE_TASK_DESCRIPTION = "closed tasks";
+    private static final String CLOSE_TASK_TOTAL_DESCRIPTION = TOTAL_DESCRIPTION + CLOSE_TASK_DESCRIPTION;
+    private static final String CLOSE_TASK_RATE_DESCRIPTION = RATE_DESCRIPTION + CLOSE_TASK_DESCRIPTION;
+    private static final String POLL_DESCRIPTION = "poll calls";
+    private static final String POLL_TOTAL_DESCRIPTION = TOTAL_DESCRIPTION + POLL_DESCRIPTION;
+    private static final String POLL_RATE_DESCRIPTION = RATE_DESCRIPTION + POLL_DESCRIPTION;
+    private static final String PROCESS_DESCRIPTION = "process calls";
+    private static final String PROCESS_TOTAL_DESCRIPTION = TOTAL_DESCRIPTION + PROCESS_DESCRIPTION;
+    private static final String PROCESS_RATE_DESCRIPTION = RATE_DESCRIPTION + PROCESS_DESCRIPTION;
+    private static final String PUNCTUATE_DESCRIPTION = "punctuate calls";
+    private static final String PUNCTUATE_TOTAL_DESCRIPTION = TOTAL_DESCRIPTION + PUNCTUATE_DESCRIPTION;
+    private static final String PUNCTUATE_RATE_DESCRIPTION = RATE_DESCRIPTION + PUNCTUATE_DESCRIPTION;
+    private static final String SKIP_RECORDS_DESCRIPTION = "skipped records";
+    private static final String SKIP_RECORD_TOTAL_DESCRIPTION = TOTAL_DESCRIPTION + SKIP_RECORDS_DESCRIPTION;
+    private static final String SKIP_RECORD_RATE_DESCRIPTION = RATE_DESCRIPTION + SKIP_RECORDS_DESCRIPTION;
+    private static final String COMMIT_OVER_TASKS_DESCRIPTION = "commit calls over all tasks";
+    private static final String COMMIT_OVER_TASKS_TOTAL_DESCRIPTION = TOTAL_DESCRIPTION + COMMIT_OVER_TASKS_DESCRIPTION;
+    private static final String COMMIT_OVER_TASKS_RATE_DESCRIPTION = RATE_DESCRIPTION + COMMIT_OVER_TASKS_DESCRIPTION;
+
+    private static final String COMMIT_LATENCY = COMMIT + LATENCY_SUFFIX;
+    private static final String POLL_LATENCY = POLL + LATENCY_SUFFIX;
+    private static final String PROCESS_LATENCY = PROCESS + LATENCY_SUFFIX;
+    private static final String PUNCTUATE_LATENCY = PUNCTUATE + LATENCY_SUFFIX;
+
+    public static Sensor createTaskSensor(final StreamsMetricsImpl streamsMetrics) {
+        final Sensor createTaskSensor = streamsMetrics.threadLevelSensor(CREATE_TASK, RecordingLevel.INFO);
+        addInvocationRateAndCount(createTaskSensor,
+                                  THREAD_LEVEL_GROUP,
+                                  streamsMetrics.threadLevelTagMap(),
+                                  CREATE_TASK,
+                                  CREATE_TASK_TOTAL_DESCRIPTION,
+                                  CREATE_TASK_RATE_DESCRIPTION);
+        return createTaskSensor;
+    }
+
+    public static Sensor closeTaskSensor(final StreamsMetricsImpl streamsMetrics) {
+        final Sensor closeTaskSensor = streamsMetrics.threadLevelSensor(CLOSE_TASK, RecordingLevel.INFO);
+        addInvocationRateAndCount(closeTaskSensor,
+                                  THREAD_LEVEL_GROUP,
+                                  streamsMetrics.threadLevelTagMap(),
+                                  CLOSE_TASK,
+                                  CLOSE_TASK_TOTAL_DESCRIPTION,
+                                  CLOSE_TASK_RATE_DESCRIPTION);
+        return closeTaskSensor;
+    }
+
+    public static Sensor commitSensor(final StreamsMetricsImpl streamsMetrics) {
+        final Sensor commitSensor = streamsMetrics.threadLevelSensor(COMMIT, Sensor.RecordingLevel.INFO);
+        final Map tagMap = streamsMetrics.threadLevelTagMap();
+        addAvgAndMax(commitSensor, THREAD_LEVEL_GROUP, tagMap, COMMIT_LATENCY);
+        addInvocationRateAndCount(commitSensor,
+                                  THREAD_LEVEL_GROUP,
+                                  tagMap,
+                                  COMMIT,
+                                  COMMIT_TOTAL_DESCRIPTION,
+                                  COMMIT_RATE_DESCRIPTION);
+        return commitSensor;
+    }
+
+    public static Sensor pollSensor(final StreamsMetricsImpl streamsMetrics) {
+        final Sensor pollSensor = streamsMetrics.threadLevelSensor(POLL, Sensor.RecordingLevel.INFO);
+        final Map tagMap = streamsMetrics.threadLevelTagMap();
+        addAvgAndMax(pollSensor, THREAD_LEVEL_GROUP, tagMap, POLL_LATENCY);
+        addInvocationRateAndCount(pollSensor,
+                                  THREAD_LEVEL_GROUP,
+                                  tagMap,
+                                  POLL,
+                                  POLL_TOTAL_DESCRIPTION,
+                                  POLL_RATE_DESCRIPTION);
+        return pollSensor;
+    }
+
+    public static Sensor processSensor(final StreamsMetricsImpl streamsMetrics) {
+        final Sensor processSensor = streamsMetrics.threadLevelSensor(PROCESS, Sensor.RecordingLevel.INFO);
+        final Map tagMap = streamsMetrics.threadLevelTagMap();
+        addAvgAndMax(processSensor, THREAD_LEVEL_GROUP, tagMap, PROCESS_LATENCY);
+        addInvocationRateAndCount(processSensor,
+                                  THREAD_LEVEL_GROUP,
+                                  tagMap,
+                                  PROCESS,
+                                  PROCESS_TOTAL_DESCRIPTION,
+                                  PROCESS_RATE_DESCRIPTION);
+
+        return processSensor;
+    }
+
+    public static Sensor punctuateSensor(final StreamsMetricsImpl streamsMetrics) {
+        final Sensor punctuateSensor = streamsMetrics.threadLevelSensor(PUNCTUATE, Sensor.RecordingLevel.INFO);
+        final Map tagMap = streamsMetrics.threadLevelTagMap();
+        addAvgAndMax(punctuateSensor, THREAD_LEVEL_GROUP, tagMap, PUNCTUATE_LATENCY);
+        addInvocationRateAndCount(punctuateSensor,
+                                  THREAD_LEVEL_GROUP,
+                                  tagMap,
+                                  PUNCTUATE,
+                                  PUNCTUATE_TOTAL_DESCRIPTION,
+                                  PUNCTUATE_RATE_DESCRIPTION);
+
+        return punctuateSensor;
+    }
+
+    public static Sensor skipRecordSensor(final StreamsMetricsImpl streamsMetrics) {
+        final Sensor skippedRecordsSensor = streamsMetrics.threadLevelSensor(SKIP_RECORD, Sensor.RecordingLevel.INFO);
+        addInvocationRateAndCount(skippedRecordsSensor,
+                                  THREAD_LEVEL_GROUP,
+                                  streamsMetrics.threadLevelTagMap(),
+                                  SKIP_RECORD,
+                                  SKIP_RECORD_TOTAL_DESCRIPTION,
+                                  SKIP_RECORD_RATE_DESCRIPTION);
+
+        return skippedRecordsSensor;
+    }
+
+    public static Sensor commitOverTasksSensor(final StreamsMetricsImpl streamsMetrics) {
+        final Sensor commitOverTasksSensor = streamsMetrics.threadLevelSensor(COMMIT, Sensor.RecordingLevel.DEBUG);
+        final Map tagMap = streamsMetrics.threadLevelTagMap(TASK_ID_TAG, ALL_TASKS);
+        addAvgAndMax(commitOverTasksSensor,
+                     TASK_LEVEL_GROUP,
+                     tagMap,
+                     COMMIT_LATENCY);
+        addInvocationRateAndCount(commitOverTasksSensor,
+                                  TASK_LEVEL_GROUP,
+                                  tagMap,
+                                  COMMIT,
+                                  COMMIT_OVER_TASKS_TOTAL_DESCRIPTION,
+                                  COMMIT_OVER_TASKS_RATE_DESCRIPTION);
+
+        return commitOverTasksSensor;
+    }
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
index 3f6c80691f0e6..bc4d89535e058 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
@@ -25,11 +25,11 @@
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
 import org.apache.kafka.streams.integration.utils.IntegrationTestUtils;
+import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.KGroupedStream;
+import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.Materialized;
-import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.Produced;
-import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.SessionWindows;
 import org.apache.kafka.streams.kstream.TimeWindows;
 import org.apache.kafka.streams.state.SessionStore;
@@ -37,9 +37,9 @@
 import org.apache.kafka.streams.state.WindowStore;
 import org.apache.kafka.test.IntegrationTest;
 import org.apache.kafka.test.TestUtils;
+import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
-import org.junit.After;
 import org.junit.ClassRule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregateProcessorTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregateProcessorTest.java
index 0dfd8ad45f38d..5b207372964fc 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregateProcessorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregateProcessorTest.java
@@ -32,6 +32,7 @@
 import org.apache.kafka.streams.processor.To;
 import org.apache.kafka.streams.processor.internals.MockStreamsMetrics;
 import org.apache.kafka.streams.processor.internals.ProcessorRecordContext;
+import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.apache.kafka.streams.processor.internals.ToInternal;
 import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender;
 import org.apache.kafka.streams.state.KeyValueIterator;
@@ -92,6 +93,8 @@ public void initializeStore() {
         final File stateDir = TestUtils.tempDirectory();
         metrics = new Metrics();
         final MockStreamsMetrics metrics = new MockStreamsMetrics(KStreamSessionWindowAggregateProcessorTest.this.metrics);
+        ThreadMetrics.skipRecordSensor(metrics);
+
         context = new InternalMockProcessorContext(
             stateDir,
             Serdes.String(),
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
index ccd94de34c2da..ab9a47e92afd8 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
@@ -246,7 +246,6 @@ public void initTransactions() {
                         throw new TimeoutException("test");
                     }
                 },
-                null,
                 null
             );
             fail("Expected an exception");
@@ -301,7 +300,6 @@ public void initTransactions() {
                     }
                 }
             },
-            null,
             null
         );
         testTask.initializeTopology();
@@ -851,8 +849,7 @@ public void shouldFlushRecordCollectorOnFlushState() {
                 public void flush() {
                     flushed.set(true);
                 }
-            },
-            metrics.sensor("dummy"));
+            });
         streamTask.flushState();
         assertTrue(flushed.get());
     }
@@ -1427,8 +1424,7 @@ public void shouldReturnOffsetsForRepartitionTopicsForPurging() {
             stateDirectory,
             null,
             time,
-            () -> producer = new MockProducer<>(false, bytesSerializer, bytesSerializer),
-            metrics.sensor("dummy"));
+            () -> producer = new MockProducer<>(false, bytesSerializer, bytesSerializer));
         task.initializeStateStores();
         task.initializeTopology();
 
@@ -1498,8 +1494,7 @@ private StreamTask createStatefulTask(final StreamsConfig config, final boolean
             stateDirectory,
             null,
             time,
-            () -> producer = new MockProducer<>(false, bytesSerializer, bytesSerializer),
-            metrics.sensor("dummy"));
+            () -> producer = new MockProducer<>(false, bytesSerializer, bytesSerializer));
     }
 
     private StreamTask createStatefulTaskThatThrowsExceptionOnClose() {
@@ -1520,8 +1515,7 @@ private StreamTask createStatefulTaskThatThrowsExceptionOnClose() {
             stateDirectory,
             null,
             time,
-            () -> producer = new MockProducer<>(false, bytesSerializer, bytesSerializer),
-            metrics.sensor("dummy"));
+            () -> producer = new MockProducer<>(false, bytesSerializer, bytesSerializer));
     }
 
     private StreamTask createStatelessTask(final StreamsConfig streamsConfig) {
@@ -1546,8 +1540,7 @@ private StreamTask createStatelessTask(final StreamsConfig streamsConfig) {
             stateDirectory,
             null,
             time,
-            () -> producer = new MockProducer<>(false, bytesSerializer, bytesSerializer),
-            metrics.sensor("dummy"));
+            () -> producer = new MockProducer<>(false, bytesSerializer, bytesSerializer));
     }
 
     // this task will throw exception when processing (on partition2), flushing, suspending and closing
@@ -1573,8 +1566,7 @@ private StreamTask createTaskThatThrowsException(final boolean enableEos) {
             stateDirectory,
             null,
             time,
-            () -> producer = new MockProducer<>(false, bytesSerializer, bytesSerializer),
-            metrics.sensor("dummy")) {
+            () -> producer = new MockProducer<>(false, bytesSerializer, bytesSerializer)) {
             @Override
             protected void flushState() {
                 throw new RuntimeException("KABOOM!");
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
index 1de39d2274bb5..7c17ca7a45c1e 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
@@ -56,6 +56,7 @@
 import org.apache.kafka.streams.processor.TaskId;
 import org.apache.kafka.streams.processor.TaskMetadata;
 import org.apache.kafka.streams.processor.ThreadMetadata;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender;
 import org.apache.kafka.streams.state.KeyValueStore;
 import org.apache.kafka.streams.state.internals.OffsetCheckpoint;
@@ -254,35 +255,70 @@ public void testMetricsCreatedAtStartup() {
         final StreamThread thread = createStreamThread(clientId, config, false);
         final String defaultGroupName = "stream-metrics";
         final Map defaultTags = Collections.singletonMap("client-id", thread.getName());
-
-        assertNotNull(metrics.metrics().get(metrics.metricName("commit-latency-avg", defaultGroupName, "The average commit time in ms", defaultTags)));
-        assertNotNull(metrics.metrics().get(metrics.metricName("commit-latency-max", defaultGroupName, "The maximum commit time in ms", defaultTags)));
-        assertNotNull(metrics.metrics().get(metrics.metricName("commit-rate", defaultGroupName, "The average per-second number of commit calls", defaultTags)));
-        assertNotNull(metrics.metrics().get(metrics.metricName("commit-total", defaultGroupName, "The total number of commit calls", defaultTags)));
-        assertNotNull(metrics.metrics().get(metrics.metricName("poll-latency-avg", defaultGroupName, "The average poll time in ms", defaultTags)));
-        assertNotNull(metrics.metrics().get(metrics.metricName("poll-latency-max", defaultGroupName, "The maximum poll time in ms", defaultTags)));
-        assertNotNull(metrics.metrics().get(metrics.metricName("poll-rate", defaultGroupName, "The average per-second number of record-poll calls", defaultTags)));
-        assertNotNull(metrics.metrics().get(metrics.metricName("poll-total", defaultGroupName, "The total number of record-poll calls", defaultTags)));
-        assertNotNull(metrics.metrics().get(metrics.metricName("process-latency-avg", defaultGroupName, "The average process time in ms", defaultTags)));
-        assertNotNull(metrics.metrics().get(metrics.metricName("process-latency-max", defaultGroupName, "The maximum process time in ms", defaultTags)));
-        assertNotNull(metrics.metrics().get(metrics.metricName("process-rate", defaultGroupName, "The average per-second number of process calls", defaultTags)));
-        assertNotNull(metrics.metrics().get(metrics.metricName("process-total", defaultGroupName, "The total number of process calls", defaultTags)));
-        assertNotNull(metrics.metrics().get(metrics.metricName("punctuate-latency-avg", defaultGroupName, "The average punctuate time in ms", defaultTags)));
-        assertNotNull(metrics.metrics().get(metrics.metricName("punctuate-latency-max", defaultGroupName, "The maximum punctuate time in ms", defaultTags)));
-        assertNotNull(metrics.metrics().get(metrics.metricName("punctuate-rate", defaultGroupName, "The average per-second number of punctuate calls", defaultTags)));
-        assertNotNull(metrics.metrics().get(metrics.metricName("punctuate-total", defaultGroupName, "The total number of punctuate calls", defaultTags)));
-        assertNotNull(metrics.metrics().get(metrics.metricName("task-created-rate", defaultGroupName, "The average per-second number of newly created tasks", defaultTags)));
-        assertNotNull(metrics.metrics().get(metrics.metricName("task-created-total", defaultGroupName, "The total number of newly created tasks", defaultTags)));
-        assertNotNull(metrics.metrics().get(metrics.metricName("task-closed-rate", defaultGroupName, "The average per-second number of closed tasks", defaultTags)));
-        assertNotNull(metrics.metrics().get(metrics.metricName("task-closed-total", defaultGroupName, "The total number of closed tasks", defaultTags)));
-        assertNotNull(metrics.metrics().get(metrics.metricName("skipped-records-rate", defaultGroupName, "The average per-second number of skipped records.", defaultTags)));
-        assertNotNull(metrics.metrics().get(metrics.metricName("skipped-records-total", defaultGroupName, "The total number of skipped records.", defaultTags)));
+        final String descriptionIsNotVerified = "";
+
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "commit-latency-avg", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "commit-latency-max", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "commit-rate", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "commit-total", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "poll-latency-avg", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "poll-latency-max", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "poll-rate", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "poll-total", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "process-latency-avg", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "process-latency-max", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "process-rate", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "process-total", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "punctuate-latency-avg", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "punctuate-latency-max", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "punctuate-rate", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "punctuate-total", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "task-created-rate", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "task-created-total", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "task-closed-rate", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "task-closed-total", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "skipped-records-rate", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "skipped-records-total", defaultGroupName, descriptionIsNotVerified, defaultTags)));
+
+        final String taskGroupName = "stream-task-metrics";
+        final Map taskTags =
+            mkMap(mkEntry("task-id", "all"), mkEntry("client-id", thread.getName()));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "commit-latency-avg", taskGroupName, descriptionIsNotVerified, taskTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "commit-latency-max", taskGroupName, descriptionIsNotVerified, taskTags)));
+        assertNotNull(metrics.metrics().get(metrics.metricName(
+            "commit-rate", taskGroupName, descriptionIsNotVerified, taskTags)));
 
         final JmxReporter reporter = new JmxReporter("kafka.streams");
         metrics.addReporter(reporter);
         assertEquals(clientId + "-StreamThread-1", thread.getName());
         assertTrue(reporter.containsMbean(String.format("kafka.streams:type=%s,client-id=%s",
-                defaultGroupName, thread.getName())));
+                   defaultGroupName, 
+                   thread.getName())));
+        assertTrue(reporter.containsMbean("kafka.streams:type=stream-task-metrics,client-id=" + thread.getName() + ",task-id=all"));
     }
 
     @Test
@@ -296,8 +332,7 @@ public void shouldNotCommitBeforeTheCommitInterval() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
         final TaskManager taskManager = mockTaskManagerCommit(consumer, 1, 1);
 
-        final StreamThread.StreamsMetricsThreadImpl streamsMetrics
-            = new StreamThread.StreamsMetricsThreadImpl(metrics, "");
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -423,8 +458,7 @@ public void shouldNotCauseExceptionIfNothingCommitted() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
         final TaskManager taskManager = mockTaskManagerCommit(consumer, 1, 0);
 
-        final StreamThread.StreamsMetricsThreadImpl streamsMetrics
-            = new StreamThread.StreamsMetricsThreadImpl(metrics, "");
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -459,8 +493,7 @@ public void shouldCommitAfterTheCommitInterval() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
         final TaskManager taskManager = mockTaskManagerCommit(consumer, 2, 1);
 
-        final StreamThread.StreamsMetricsThreadImpl streamsMetrics
-            = new StreamThread.StreamsMetricsThreadImpl(metrics, "");
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -610,8 +643,7 @@ public void shouldShutdownTaskManagerOnClose() {
         EasyMock.expectLastCall();
         EasyMock.replay(taskManager, consumer);
 
-        final StreamThread.StreamsMetricsThreadImpl streamsMetrics
-            = new StreamThread.StreamsMetricsThreadImpl(metrics, "");
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -644,8 +676,7 @@ public void shouldShutdownTaskManagerOnCloseWithoutStart() {
         EasyMock.expectLastCall();
         EasyMock.replay(taskManager, consumer);
 
-        final StreamThread.StreamsMetricsThreadImpl streamsMetrics
-            = new StreamThread.StreamsMetricsThreadImpl(metrics, "");
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -672,8 +703,7 @@ public void shouldOnlyShutdownOnce() {
         EasyMock.expectLastCall();
         EasyMock.replay(taskManager, consumer);
 
-        final StreamThread.StreamsMetricsThreadImpl streamsMetrics
-            = new StreamThread.StreamsMetricsThreadImpl(metrics, "");
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -1449,8 +1479,7 @@ public void producerMetricsVerificationWithoutEOS() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
         final TaskManager taskManager = mockTaskManagerCommit(consumer, 1, 0);
 
-        final StreamThread.StreamsMetricsThreadImpl streamsMetrics
-            = new StreamThread.StreamsMetricsThreadImpl(metrics, "");
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId);
         final StreamThread thread = new StreamThread(
                 mockTime,
                 config,
@@ -1489,8 +1518,7 @@ public void adminClientMetricsVerification() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
         final TaskManager taskManager = EasyMock.createNiceMock(TaskManager.class);
 
-        final StreamThread.StreamsMetricsThreadImpl streamsMetrics
-            = new StreamThread.StreamsMetricsThreadImpl(metrics, "");
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId);
         final StreamThread thread = new StreamThread(
                 mockTime,
                 config,
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java
index e1d6f0c086cbf..f381a58c1233e 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java
@@ -22,7 +22,10 @@
 import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.metrics.Sensor.RecordingLevel;
 import org.apache.kafka.common.utils.MockTime;
+import org.easymock.EasyMock;
+import org.easymock.EasyMockSupport;
 import org.junit.Test;
 
 import java.util.Collections;
@@ -38,8 +41,34 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 
-public class StreamsMetricsImplTest {
+public class StreamsMetricsImplTest extends EasyMockSupport {
+
+    private final static String SENSOR_PREFIX_DELIMITER = ".";
+    private final static String SENSOR_NAME_DELIMITER = ".s.";
+    private final static String INTERNAL_PREFIX = "internal";
+
+    @Test
+    public void shouldGetThreadLevelSensor() {
+        final Metrics metrics = mock(Metrics.class);
+        final String threadName = "thread1";
+        final String sensorName = "sensor1";
+        final String expectedFullSensorName =
+            INTERNAL_PREFIX + SENSOR_PREFIX_DELIMITER + threadName + SENSOR_NAME_DELIMITER + sensorName;
+        final RecordingLevel recordingLevel = RecordingLevel.DEBUG;
+        final Sensor[] parents = {};
+        EasyMock.expect(metrics.sensor(expectedFullSensorName, recordingLevel, parents)).andReturn(null);
+
+        replayAll();
+
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, threadName);
+        final Sensor sensor = streamsMetrics.threadLevelSensor(sensorName, recordingLevel);
+
+        verifyAll();
+
+        assertNull(sensor);
+    }
 
     @Test(expected = NullPointerException.class)
     public void testNullMetrics() {
@@ -92,13 +121,13 @@ public void testMutiLevelSensorRemoval() {
 
         final Sensor parent1 = metrics.taskLevelSensor(taskName, operation, Sensor.RecordingLevel.DEBUG);
         addAvgMaxLatency(parent1, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation);
-        addInvocationRateAndCount(parent1, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation);
+        addInvocationRateAndCount(parent1, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation, "", "");
 
         final int numberOfTaskMetrics = registry.metrics().size();
 
         final Sensor sensor1 = metrics.nodeLevelSensor(taskName, processorNodeName, operation, Sensor.RecordingLevel.DEBUG, parent1);
         addAvgMaxLatency(sensor1, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation);
-        addInvocationRateAndCount(sensor1, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation);
+        addInvocationRateAndCount(sensor1, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation, "", "");
 
         assertThat(registry.metrics().size(), greaterThan(numberOfTaskMetrics));
 
@@ -108,13 +137,13 @@ public void testMutiLevelSensorRemoval() {
 
         final Sensor parent2 = metrics.taskLevelSensor(taskName, operation, Sensor.RecordingLevel.DEBUG);
         addAvgMaxLatency(parent2, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation);
-        addInvocationRateAndCount(parent2, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation);
+        addInvocationRateAndCount(parent2, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation, "", "");
 
         assertThat(registry.metrics().size(), equalTo(numberOfTaskMetrics));
 
         final Sensor sensor2 = metrics.nodeLevelSensor(taskName, processorNodeName, operation, Sensor.RecordingLevel.DEBUG, parent2);
         addAvgMaxLatency(sensor2, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation);
-        addInvocationRateAndCount(sensor2, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation);
+        addInvocationRateAndCount(sensor2, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation, "", "");
 
         assertThat(registry.metrics().size(), greaterThan(numberOfTaskMetrics));
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetricsTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetricsTest.java
new file mode 100644
index 0000000000000..89395d9c51a8c
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetricsTest.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.metrics;
+
+import org.apache.kafka.common.metrics.Metrics;
+import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.metrics.Sensor.RecordingLevel;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.util.Collections;
+import java.util.Map;
+
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.ALL_TASKS;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.TASK_ID_TAG;
+import static org.easymock.EasyMock.expect;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.powermock.api.easymock.PowerMock.createStrictMock;
+import static org.powermock.api.easymock.PowerMock.mockStatic;
+import static org.powermock.api.easymock.PowerMock.replay;
+import static org.powermock.api.easymock.PowerMock.replayAll;
+import static org.powermock.api.easymock.PowerMock.verify;
+import static org.powermock.api.easymock.PowerMock.verifyAll;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(StreamsMetricsImpl.class)
+public class ThreadMetricsTest {
+
+    private static final String THREAD_LEVEL_GROUP = "stream-metrics";
+    private static final String TASK_LEVEL_GROUP = "stream-task-metrics";
+
+    private final Metrics dummyMetrics = new Metrics();
+    private final Sensor dummySensor = dummyMetrics.sensor("dummy");
+    private final StreamsMetricsImpl streamsMetrics = createStrictMock(StreamsMetricsImpl.class);
+    private final Map dummyTagMap = Collections.singletonMap("hello", "world");
+
+    @Test
+    public void shouldGetCreateTaskSensor() {
+        final String operation = "task-created";
+        final String totalDescription = "The total number of newly created tasks";
+        final String rateDescription = "The average per-second number of newly created tasks";
+        mockStatic(StreamsMetricsImpl.class);
+        expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.INFO)).andReturn(dummySensor);
+        expect(streamsMetrics.threadLevelTagMap()).andReturn(dummyTagMap);
+        StreamsMetricsImpl.addInvocationRateAndCount(
+            dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
+
+        replayAll();
+        replay(StreamsMetricsImpl.class);
+
+        final Sensor sensor = ThreadMetrics.createTaskSensor(streamsMetrics);
+
+        verifyAll();
+        verify(StreamsMetricsImpl.class);
+
+        assertThat(sensor, is(dummySensor));
+    }
+
+    @Test
+    public void shouldGetCloseTaskSensor() {
+        final String operation = "task-closed";
+        final String totalDescription = "The total number of closed tasks";
+        final String rateDescription = "The average per-second number of closed tasks";
+        mockStatic(StreamsMetricsImpl.class);
+        expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.INFO)).andReturn(dummySensor);
+        expect(streamsMetrics.threadLevelTagMap()).andReturn(dummyTagMap);
+        StreamsMetricsImpl.addInvocationRateAndCount(
+            dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
+
+        replayAll();
+        replay(StreamsMetricsImpl.class);
+
+        final Sensor sensor = ThreadMetrics.closeTaskSensor(streamsMetrics);
+
+        verifyAll();
+        verify(StreamsMetricsImpl.class);
+
+        assertThat(sensor, is(dummySensor));
+    }
+
+    @Test
+    public void shouldGetCommitSensor() {
+        final String operation = "commit";
+        final String operationLatency = operation + StreamsMetricsImpl.LATENCY_SUFFIX;
+        final String totalDescription = "The total number of commit calls";
+        final String rateDescription = "The average per-second number of commit calls";
+        mockStatic(StreamsMetricsImpl.class);
+        expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.INFO)).andReturn(dummySensor);
+        expect(streamsMetrics.threadLevelTagMap()).andReturn(dummyTagMap);
+        StreamsMetricsImpl.addInvocationRateAndCount(
+            dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
+        StreamsMetricsImpl.addAvgAndMax(
+            dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operationLatency);
+
+        replayAll();
+        replay(StreamsMetricsImpl.class);
+
+        final Sensor sensor = ThreadMetrics.commitSensor(streamsMetrics);
+
+        verifyAll();
+        verify(StreamsMetricsImpl.class);
+
+        assertThat(sensor, is(dummySensor));
+    }
+
+    @Test
+    public void shouldGetPollSensor() {
+        final String operation = "poll";
+        final String operationLatency = operation + StreamsMetricsImpl.LATENCY_SUFFIX;
+        final String totalDescription = "The total number of poll calls";
+        final String rateDescription = "The average per-second number of poll calls";
+        mockStatic(StreamsMetricsImpl.class);
+        expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.INFO)).andReturn(dummySensor);
+        expect(streamsMetrics.threadLevelTagMap()).andReturn(dummyTagMap);
+        StreamsMetricsImpl.addInvocationRateAndCount(
+            dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
+        StreamsMetricsImpl.addAvgAndMax(
+            dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operationLatency);
+
+        replayAll();
+        replay(StreamsMetricsImpl.class);
+
+        final Sensor sensor = ThreadMetrics.pollSensor(streamsMetrics);
+
+        verifyAll();
+        verify(StreamsMetricsImpl.class);
+
+        assertThat(sensor, is(dummySensor));
+    }
+
+    @Test
+    public void shouldGetProcessSensor() {
+        final String operation = "process";
+        final String operationLatency = operation + StreamsMetricsImpl.LATENCY_SUFFIX;
+        final String totalDescription = "The total number of process calls";
+        final String rateDescription = "The average per-second number of process calls";
+        mockStatic(StreamsMetricsImpl.class);
+        expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.INFO)).andReturn(dummySensor);
+        expect(streamsMetrics.threadLevelTagMap()).andReturn(dummyTagMap);
+        StreamsMetricsImpl.addInvocationRateAndCount(
+            dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
+        StreamsMetricsImpl.addAvgAndMax(
+            dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operationLatency);
+
+        replayAll();
+        replay(StreamsMetricsImpl.class);
+
+        final Sensor sensor = ThreadMetrics.processSensor(streamsMetrics);
+
+        verifyAll();
+        verify(StreamsMetricsImpl.class);
+
+        assertThat(sensor, is(dummySensor));
+    }
+
+    @Test
+    public void shouldGetPunctuateSensor() {
+        final String operation = "punctuate";
+        final String operationLatency = operation + StreamsMetricsImpl.LATENCY_SUFFIX;
+        final String totalDescription = "The total number of punctuate calls";
+        final String rateDescription = "The average per-second number of punctuate calls";
+        mockStatic(StreamsMetricsImpl.class);
+        expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.INFO)).andReturn(dummySensor);
+        expect(streamsMetrics.threadLevelTagMap()).andReturn(dummyTagMap);
+        StreamsMetricsImpl.addInvocationRateAndCount(
+            dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
+        StreamsMetricsImpl.addAvgAndMax(
+            dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operationLatency);
+
+        replayAll();
+        replay(StreamsMetricsImpl.class);
+
+        final Sensor sensor = ThreadMetrics.punctuateSensor(streamsMetrics);
+
+        verifyAll();
+        verify(StreamsMetricsImpl.class);
+
+        assertThat(sensor, is(dummySensor));
+    }
+
+    @Test
+    public void shouldGetSkipRecordSensor() {
+        final String operation = "skipped-records";
+        final String totalDescription = "The total number of skipped records";
+        final String rateDescription = "The average per-second number of skipped records";
+        mockStatic(StreamsMetricsImpl.class);
+        expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.INFO)).andReturn(dummySensor);
+        expect(streamsMetrics.threadLevelTagMap()).andReturn(dummyTagMap);
+        StreamsMetricsImpl.addInvocationRateAndCount(
+            dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
+
+        replayAll();
+        replay(StreamsMetricsImpl.class);
+
+        final Sensor sensor = ThreadMetrics.skipRecordSensor(streamsMetrics);
+
+        verifyAll();
+        verify(StreamsMetricsImpl.class);
+
+        assertThat(sensor, is(dummySensor));
+    }
+
+    @Test
+    public void shouldGetCommitOverTasksSensor() {
+        final String operation = "commit";
+        final String operationLatency = operation + StreamsMetricsImpl.LATENCY_SUFFIX;
+        final String totalDescription = "The total number of commit calls over all tasks";
+        final String rateDescription = "The average per-second number of commit calls over all tasks";
+        mockStatic(StreamsMetricsImpl.class);
+        expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.DEBUG)).andReturn(dummySensor);
+        expect(streamsMetrics.threadLevelTagMap(TASK_ID_TAG, ALL_TASKS)).andReturn(dummyTagMap);
+        StreamsMetricsImpl.addInvocationRateAndCount(
+            dummySensor, TASK_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
+        StreamsMetricsImpl.addAvgAndMax(
+            dummySensor, TASK_LEVEL_GROUP, dummyTagMap, operationLatency);
+
+        replayAll();
+        replay(StreamsMetricsImpl.class);
+
+        final Sensor sensor = ThreadMetrics.commitOverTasksSensor(streamsMetrics);
+
+        verifyAll();
+        verify(StreamsMetricsImpl.class);
+
+        assertThat(sensor, is(dummySensor));
+    }
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/StreamThreadStateStoreProviderTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/StreamThreadStateStoreProviderTest.java
index da2d46d6360e3..f48c31c1d91c2 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/StreamThreadStateStoreProviderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/StreamThreadStateStoreProviderTest.java
@@ -317,8 +317,7 @@ private StreamTask createStreamsTask(final StreamsConfig streamsConfig,
             stateDirectory,
             null,
             new MockTime(),
-            () -> clientSupplier.getProducer(new HashMap<>()),
-            metrics.sensor("dummy")) {
+            () -> clientSupplier.getProducer(new HashMap<>())) {
             @Override
             protected void updateOffsetLimits() {}
         };
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
index 38da0d8250907..48c038c2d5f65 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
@@ -32,6 +32,9 @@
 import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.metrics.stats.Count;
+import org.apache.kafka.common.metrics.stats.Rate;
+import org.apache.kafka.common.metrics.stats.Total;
 import org.apache.kafka.common.record.TimestampType;
 import org.apache.kafka.common.serialization.ByteArraySerializer;
 import org.apache.kafka.common.serialization.Deserializer;
@@ -277,10 +280,21 @@ public List partitionsFor(final String topic) {
             .timeWindow(streamsConfig.getLong(StreamsConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS);
 
         metrics = new Metrics(metricConfig, mockWallClockTime);
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(
-            metrics,
-            "topology-test-driver-virtual-thread"
-        );
+
+        final String threadName = "topology-test-driver-virtual-thread";
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, threadName);
+        final Sensor skippedRecordsSensor = streamsMetrics.threadLevelSensor("skipped-records", Sensor.RecordingLevel.INFO);
+        final String threadLevelGroup = "stream-metrics";
+        skippedRecordsSensor.add(new MetricName("skipped-records-rate",
+                                                threadLevelGroup,
+                                                "The average per-second number of skipped records",
+                                                streamsMetrics.tagMap()),
+                                 new Rate(TimeUnit.SECONDS, new Count()));
+        skippedRecordsSensor.add(new MetricName("skipped-records-total",
+                                                threadLevelGroup,
+                                                "The total number of skipped records",
+                                                streamsMetrics.tagMap()),
+                                 new Total());
         final ThreadCache cache = new ThreadCache(
             new LogContext("topology-test-driver "),
             Math.max(0, streamsConfig.getLong(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG)),
@@ -360,8 +374,7 @@ public void onRestoreEnd(final TopicPartition topicPartition, final String store
                 stateDirectory,
                 cache,
                 mockWallClockTime,
-                () -> producer,
-                metrics.sensor("dummy"));
+                () -> producer);
             task.initializeStateStores();
             task.initializeTopology();
             ((InternalProcessorContext) task.context()).setRecordContext(new ProcessorRecordContext(
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java
index 34a7ed92c688b..ed0274649bf23 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java
@@ -33,6 +33,7 @@
 import org.apache.kafka.streams.kstream.ValueTransformer;
 import org.apache.kafka.streams.processor.internals.RecordCollector;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.apache.kafka.streams.state.internals.InMemoryKeyValueStore;
 
 import java.io.File;
@@ -214,10 +215,9 @@ public MockProcessorContext(final Properties config, final TaskId taskId, final
         this.stateDir = stateDir;
         final MetricConfig metricConfig = new MetricConfig();
         metricConfig.recordLevel(Sensor.RecordingLevel.DEBUG);
-        this.metrics = new StreamsMetricsImpl(
-            new Metrics(metricConfig),
-            "mock-processor-context-virtual-thread"
-        );
+        final String threadName = "mock-processor-context-virtual-thread";
+        this.metrics = new StreamsMetricsImpl(new Metrics(metricConfig), threadName);
+        ThreadMetrics.skipRecordSensor(metrics);
     }
 
     @Override

From 55d07e717ec3d2a0b79a690d3a3fdf912cf01a26 Mon Sep 17 00:00:00 2001
From: Konstantine Karantasis 
Date: Mon, 3 Jun 2019 14:50:03 -0700
Subject: [PATCH 0325/1071] KAFKA-8473: Adjust Connect system tests for
 incremental cooperative rebalancing (#6872)

Author: Konstantine Karantasis 
Reviewer: Randall Hauch 
---
 .../tests/connect/connect_distributed_test.py | 74 +++++++++++++------
 .../tests/connect/connect_rest_test.py        |  7 +-
 .../templates/connect-distributed.properties  |  3 +
 3 files changed, 61 insertions(+), 23 deletions(-)

diff --git a/tests/kafkatest/tests/connect/connect_distributed_test.py b/tests/kafkatest/tests/connect/connect_distributed_test.py
index a27b54d6f89ee..cd1a15dc723cd 100644
--- a/tests/kafkatest/tests/connect/connect_distributed_test.py
+++ b/tests/kafkatest/tests/connect/connect_distributed_test.py
@@ -53,6 +53,8 @@ class ConnectDistributedTest(Test):
     STATUS_TOPIC = "connect-status"
     STATUS_REPLICATION_FACTOR = "1"
     STATUS_PARTITIONS = "1"
+    SCHEDULED_REBALANCE_MAX_DELAY_MS = "60000"
+    CONNECT_PROTOCOL="compatible"
 
     # Since tasks can be assigned to any node and we're testing with files, we need to make sure the content is the same
     # across all nodes.
@@ -155,7 +157,9 @@ def task_is_running(self, connector, task_id, node=None):
         return self._task_has_state(task_id, status, 'RUNNING')
 
     @cluster(num_nodes=5)
-    def test_restart_failed_connector(self):
+    @matrix(connect_protocol=['compatible', 'eager'])
+    def test_restart_failed_connector(self, connect_protocol):
+        self.CONNECT_PROTOCOL = connect_protocol
         self.setup_services()
         self.cc.set_configs(lambda node: self.render("connect-distributed.properties", node=node))
         self.cc.start()
@@ -172,8 +176,9 @@ def test_restart_failed_connector(self):
                    err_msg="Failed to see connector transition to the RUNNING state")
 
     @cluster(num_nodes=5)
-    @matrix(connector_type=["source", "sink"])
-    def test_restart_failed_task(self, connector_type):
+    @matrix(connector_type=['source', 'sink'], connect_protocol=['compatible', 'eager'])
+    def test_restart_failed_task(self, connector_type, connect_protocol):
+        self.CONNECT_PROTOCOL = connect_protocol
         self.setup_services()
         self.cc.set_configs(lambda node: self.render("connect-distributed.properties", node=node))
         self.cc.start()
@@ -196,12 +201,14 @@ def test_restart_failed_task(self, connector_type):
                    err_msg="Failed to see task transition to the RUNNING state")
 
     @cluster(num_nodes=5)
-    def test_pause_and_resume_source(self):
+    @matrix(connect_protocol=['compatible', 'eager'])
+    def test_pause_and_resume_source(self, connect_protocol):
         """
         Verify that source connectors stop producing records when paused and begin again after
         being resumed.
         """
 
+        self.CONNECT_PROTOCOL = connect_protocol
         self.setup_services()
         self.cc.set_configs(lambda node: self.render("connect-distributed.properties", node=node))
         self.cc.start()
@@ -235,12 +242,14 @@ def test_pause_and_resume_source(self):
                    err_msg="Failed to produce messages after resuming source connector")
 
     @cluster(num_nodes=5)
-    def test_pause_and_resume_sink(self):
+    @matrix(connect_protocol=['compatible', 'eager'])
+    def test_pause_and_resume_sink(self, connect_protocol):
         """
         Verify that sink connectors stop consuming records when paused and begin again after
         being resumed.
         """
 
+        self.CONNECT_PROTOCOL = connect_protocol
         self.setup_services()
         self.cc.set_configs(lambda node: self.render("connect-distributed.properties", node=node))
         self.cc.start()
@@ -281,11 +290,13 @@ def test_pause_and_resume_sink(self):
                    err_msg="Failed to consume messages after resuming sink connector")
 
     @cluster(num_nodes=5)
-    def test_pause_state_persistent(self):
+    @matrix(connect_protocol=['compatible', 'eager'])
+    def test_pause_state_persistent(self, connect_protocol):
         """
         Verify that paused state is preserved after a cluster restart.
         """
 
+        self.CONNECT_PROTOCOL = connect_protocol
         self.setup_services()
         self.cc.set_configs(lambda node: self.render("connect-distributed.properties", node=node))
         self.cc.start()
@@ -300,21 +311,25 @@ def test_pause_state_persistent(self):
 
         self.cc.restart()
 
+        if connect_protocol == 'compatible':
+            timeout_sec = 120
+        else:
+            timeout_sec = 30
+
         # we should still be paused after restarting
         for node in self.cc.nodes:
-            wait_until(lambda: self.is_paused(self.source, node), timeout_sec=30,
+            wait_until(lambda: self.is_paused(self.source, node), timeout_sec=timeout_sec,
                        err_msg="Failed to see connector startup in PAUSED state")
 
-    @cluster(num_nodes=5)
-    @parametrize(security_protocol=SecurityConfig.PLAINTEXT)
     @cluster(num_nodes=6)
-    @parametrize(security_protocol=SecurityConfig.SASL_SSL)
-    def test_file_source_and_sink(self, security_protocol):
+    @matrix(security_protocol=[SecurityConfig.PLAINTEXT, SecurityConfig.SASL_SSL], connect_protocol=['compatible', 'eager'])
+    def test_file_source_and_sink(self, security_protocol, connect_protocol):
         """
         Tests that a basic file connector works across clean rolling bounces. This validates that the connector is
         correctly created, tasks instantiated, and as nodes restart the work is rebalanced across nodes.
         """
 
+        self.CONNECT_PROTOCOL = connect_protocol
         self.setup_services(security_protocol=security_protocol)
         self.cc.set_configs(lambda node: self.render("connect-distributed.properties", node=node))
 
@@ -335,19 +350,25 @@ def test_file_source_and_sink(self, security_protocol):
         # only processing new data.
         self.cc.restart()
 
+        if connect_protocol == 'compatible':
+            timeout_sec = 150
+        else:
+            timeout_sec = 70
+
         for node in self.cc.nodes:
             node.account.ssh("echo -e -n " + repr(self.SECOND_INPUTS) + " >> " + self.INPUT_FILE)
-        wait_until(lambda: self._validate_file_output(self.FIRST_INPUT_LIST + self.SECOND_INPUT_LIST), timeout_sec=70, err_msg="Sink output file never converged to the same state as the input file")
+        wait_until(lambda: self._validate_file_output(self.FIRST_INPUT_LIST + self.SECOND_INPUT_LIST), timeout_sec=timeout_sec, err_msg="Sink output file never converged to the same state as the input file")
 
     @cluster(num_nodes=6)
-    @matrix(clean=[True, False])
-    def test_bounce(self, clean):
+    @matrix(clean=[True, False], connect_protocol=['compatible', 'eager'])
+    def test_bounce(self, clean, connect_protocol):
         """
         Validates that source and sink tasks that run continuously and produce a predictable sequence of messages
         run correctly and deliver messages exactly once when Kafka Connect workers undergo clean rolling bounces.
         """
         num_tasks = 3
 
+        self.CONNECT_PROTOCOL = connect_protocol
         self.setup_services()
         self.cc.set_configs(lambda node: self.render("connect-distributed.properties", node=node))
         self.cc.start()
@@ -449,7 +470,9 @@ def test_bounce(self, clean):
         assert success, "Found validation errors:\n" + "\n  ".join(errors)
 
     @cluster(num_nodes=6)
-    def test_transformations(self):
+    @matrix(connect_protocol=['compatible', 'eager'])
+    def test_transformations(self, connect_protocol):
+        self.CONNECT_PROTOCOL = connect_protocol
         self.setup_services(timestamp_type='CreateTime')
         self.cc.set_configs(lambda node: self.render("connect-distributed.properties", node=node))
         self.cc.start()
@@ -504,18 +527,25 @@ def test_transformations(self):
             assert obj['payload'][ts_fieldname] == ts
 
     @cluster(num_nodes=5)
-    @parametrize(broker_version=str(DEV_BRANCH), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT)
-    @parametrize(broker_version=str(LATEST_0_11_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT)
-    @parametrize(broker_version=str(LATEST_0_10_2), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT)
-    @parametrize(broker_version=str(LATEST_0_10_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT)
-    @parametrize(broker_version=str(LATEST_0_10_0), auto_create_topics=True, security_protocol=SecurityConfig.PLAINTEXT)
-    @parametrize(broker_version=str(LATEST_0_9), auto_create_topics=True, security_protocol=SecurityConfig.PLAINTEXT)
-    def test_broker_compatibility(self, broker_version, auto_create_topics, security_protocol):
+    @parametrize(broker_version=str(DEV_BRANCH), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible')
+    @parametrize(broker_version=str(LATEST_0_11_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible')
+    @parametrize(broker_version=str(LATEST_0_10_2), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible')
+    @parametrize(broker_version=str(LATEST_0_10_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible')
+    @parametrize(broker_version=str(LATEST_0_10_0), auto_create_topics=True, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible')
+    @parametrize(broker_version=str(LATEST_0_9), auto_create_topics=True, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible')
+    @parametrize(broker_version=str(DEV_BRANCH), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager')
+    @parametrize(broker_version=str(LATEST_0_11_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager')
+    @parametrize(broker_version=str(LATEST_0_10_2), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager')
+    @parametrize(broker_version=str(LATEST_0_10_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager')
+    @parametrize(broker_version=str(LATEST_0_10_0), auto_create_topics=True, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager')
+    @parametrize(broker_version=str(LATEST_0_9), auto_create_topics=True, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager')
+    def test_broker_compatibility(self, broker_version, auto_create_topics, security_protocol, connect_protocol):
         """
         Verify that Connect will start up with various broker versions with various configurations. 
         When Connect distributed starts up, it either creates internal topics (v0.10.1.0 and after) 
         or relies upon the broker to auto-create the topics (v0.10.0.x and before).
         """
+        self.CONNECT_PROTOCOL = connect_protocol
         self.setup_services(broker_version=broker_version, auto_create_topics=auto_create_topics, security_protocol=security_protocol)
         self.cc.set_configs(lambda node: self.render("connect-distributed.properties", node=node))
 
diff --git a/tests/kafkatest/tests/connect/connect_rest_test.py b/tests/kafkatest/tests/connect/connect_rest_test.py
index c13515bea066c..13dcf046d4075 100644
--- a/tests/kafkatest/tests/connect/connect_rest_test.py
+++ b/tests/kafkatest/tests/connect/connect_rest_test.py
@@ -16,6 +16,7 @@
 from kafkatest.tests.kafka_test import KafkaTest
 from kafkatest.services.connect import ConnectDistributedService, ConnectRestError, ConnectServiceBase
 from ducktape.utils.util import wait_until
+from ducktape.mark import matrix
 from ducktape.mark.resource import cluster
 from ducktape.cluster.remoteaccount import RemoteCommandError
 
@@ -65,6 +66,8 @@ class ConnectRestApiTest(KafkaTest):
 
     SCHEMA = {"type": "string", "optional": False}
 
+    CONNECT_PROTOCOL="compatible"
+
     def __init__(self, test_context):
         super(ConnectRestApiTest, self).__init__(test_context, num_zk=1, num_brokers=1, topics={
             'test': {'partitions': 1, 'replication-factor': 1}
@@ -73,11 +76,13 @@ def __init__(self, test_context):
         self.cc = ConnectDistributedService(test_context, 2, self.kafka, [self.INPUT_FILE, self.INPUT_FILE2, self.OUTPUT_FILE])
 
     @cluster(num_nodes=4)
-    def test_rest_api(self):
+    @matrix(connect_protocol=['compatible', 'eager'])
+    def test_rest_api(self, connect_protocol):
         # Template parameters
         self.key_converter = "org.apache.kafka.connect.json.JsonConverter"
         self.value_converter = "org.apache.kafka.connect.json.JsonConverter"
         self.schemas = True
+        self.CONNECT_PROTOCOL = connect_protocol
 
         self.cc.set_configs(lambda node: self.render("connect-distributed.properties", node=node))
         self.cc.set_external_configs(lambda node: self.render("connect-file-external.properties", node=node))
diff --git a/tests/kafkatest/tests/connect/templates/connect-distributed.properties b/tests/kafkatest/tests/connect/templates/connect-distributed.properties
index ca8c4f84efb3e..e31a54a27a14f 100644
--- a/tests/kafkatest/tests/connect/templates/connect-distributed.properties
+++ b/tests/kafkatest/tests/connect/templates/connect-distributed.properties
@@ -20,6 +20,9 @@ bootstrap.servers={{ kafka.bootstrap_servers(kafka.security_config.security_prot
 
 group.id={{ group|default("connect-cluster") }}
 
+connect.protocol={{ CONNECT_PROTOCOL|default("compatible") }}
+scheduled.rebalance.max.delay.ms={{ SCHEDULED_REBALANCE_MAX_DELAY_MS|default(60000) }}
+
 key.converter={{ key_converter|default("org.apache.kafka.connect.json.JsonConverter") }}
 value.converter={{ value_converter|default("org.apache.kafka.connect.json.JsonConverter") }}
 {% if key_converter is not defined or key_converter.endswith("JsonConverter") %}

From 573152dfa8087e40ee69b27fb9fb8f45d3825eb6 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Mon, 3 Jun 2019 16:56:28 -0700
Subject: [PATCH 0326/1071] HOTFIX: Allow multi-batches for old format and no
 compression (#6871)

Reviewers: Jason Gustafson 
---
 .../kafka/common/record/AbstractRecords.java  |  9 +++
 .../main/scala/kafka/log/LogValidator.scala   | 77 ++++++++++---------
 .../unit/kafka/log/LogValidatorTest.scala     | 44 ++++++-----
 3 files changed, 74 insertions(+), 56 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/record/AbstractRecords.java b/clients/src/main/java/org/apache/kafka/common/record/AbstractRecords.java
index 1994a71968a1c..411e647d50b16 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/AbstractRecords.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/AbstractRecords.java
@@ -43,6 +43,15 @@ public boolean hasCompatibleMagic(byte magic) {
         return true;
     }
 
+    public boolean firstBatchHasCompatibleMagic(byte magic) {
+        Iterator iterator = batches().iterator();
+
+        if (!iterator.hasNext())
+            return true;
+
+        return iterator.next().magic() <= magic;
+    }
+
     /**
      * Get an iterator over the deep records.
      * @return An iterator over the records
diff --git a/core/src/main/scala/kafka/log/LogValidator.scala b/core/src/main/scala/kafka/log/LogValidator.scala
index 77a4b2bc3cf41..d10eed8af7dd4 100644
--- a/core/src/main/scala/kafka/log/LogValidator.scala
+++ b/core/src/main/scala/kafka/log/LogValidator.scala
@@ -74,21 +74,18 @@ private[kafka] object LogValidator extends Logging {
     }
   }
 
-  private[kafka] def validateOneBatchRecords(records: MemoryRecords): RecordBatch = {
-    // Assume there's only one batch with compressed memory records; otherwise, return InvalidRecordException
+  private[kafka] def validateOneBatchRecords(records: MemoryRecords) {
     val batchIterator = records.batches.iterator
 
     if (!batchIterator.hasNext) {
       throw new InvalidRecordException("Compressed outer record has no batches at all")
     }
 
-    val batch = batchIterator.next()
+    batchIterator.next()
 
     if (batchIterator.hasNext) {
       throw new InvalidRecordException("Compressed outer record has more than one batch")
     }
-
-    batch
   }
 
   private def validateBatch(batch: RecordBatch, isFromClient: Boolean, toMagic: Byte): Unit = {
@@ -197,11 +194,12 @@ private[kafka] object LogValidator extends Logging {
     var offsetOfMaxTimestamp = -1L
     val initialOffset = offsetCounter.value
 
-    for (batch <- records.batches.asScala) {
-      if (batch.magic() >= RecordBatch.MAGIC_VALUE_V2) {
-        validateOneBatchRecords(records)
-      }
+    if (!records.firstBatchHasCompatibleMagic(RecordBatch.MAGIC_VALUE_V1)) {
+      // for v2 and beyond, we should check there's only one batch.
+      validateOneBatchRecords(records)
+    }
 
+    for (batch <- records.batches.asScala) {
       validateBatch(batch, isFromClient, magic)
 
       var maxBatchTimestamp = RecordBatch.NO_TIMESTAMP
@@ -280,38 +278,47 @@ private[kafka] object LogValidator extends Logging {
 
     var uncompressedSizeInBytes = 0
 
-    val batch = validateOneBatchRecords(records)
+    // Assume there's only one batch with compressed memory records; otherwise, return InvalidRecordException
+    // One exception though is that with format smaller than v2, if sourceCodec is noCompression, then each batch is actually
+    // a single record so we'd need to special handle it by creating a single wrapper batch that includes all the records
+    if (sourceCodec != NoCompressionCodec || !records.firstBatchHasCompatibleMagic(RecordBatch.MAGIC_VALUE_V1)) {
+      validateOneBatchRecords(records)
+    }
+
+    val batches = records.batches.asScala
 
-    validateBatch(batch, isFromClient, toMagic)
-    uncompressedSizeInBytes += AbstractRecords.recordBatchHeaderSizeInBytes(toMagic, batch.compressionType())
+    for (batch <- batches) {
+      validateBatch(batch, isFromClient, toMagic)
+      uncompressedSizeInBytes += AbstractRecords.recordBatchHeaderSizeInBytes(toMagic, batch.compressionType())
 
-    // Do not compress control records unless they are written compressed
-    if (sourceCodec == NoCompressionCodec && batch.isControlBatch)
-      inPlaceAssignment = true
+      // Do not compress control records unless they are written compressed
+      if (sourceCodec == NoCompressionCodec && batch.isControlBatch)
+        inPlaceAssignment = true
 
-    for (record <- batch.asScala) {
-      if (sourceCodec != NoCompressionCodec && record.isCompressed)
-        throw new InvalidRecordException("Compressed outer record should not have an inner record with a " +
-          s"compression attribute set: $record")
-      if (targetCodec == ZStdCompressionCodec && interBrokerProtocolVersion < KAFKA_2_1_IV0)
-        throw new UnsupportedCompressionTypeException("Produce requests to inter.broker.protocol.version < 2.1 broker " + "are not allowed to use ZStandard compression")
-      validateRecord(batch, record, now, timestampType, timestampDiffMaxMs, compactedTopic)
+      for (record <- batch.asScala) {
+        if (sourceCodec != NoCompressionCodec && record.isCompressed)
+          throw new InvalidRecordException("Compressed outer record should not have an inner record with a " +
+            s"compression attribute set: $record")
+        if (targetCodec == ZStdCompressionCodec && interBrokerProtocolVersion < KAFKA_2_1_IV0)
+          throw new UnsupportedCompressionTypeException("Produce requests to inter.broker.protocol.version < 2.1 broker " + "are not allowed to use ZStandard compression")
+        validateRecord(batch, record, now, timestampType, timestampDiffMaxMs, compactedTopic)
 
-      uncompressedSizeInBytes += record.sizeInBytes()
-      if (batch.magic > RecordBatch.MAGIC_VALUE_V0 && toMagic > RecordBatch.MAGIC_VALUE_V0) {
-        // Check if we need to overwrite offset
-        // No in place assignment situation 3
-        if (record.offset != expectedInnerOffset.getAndIncrement())
-          inPlaceAssignment = false
-        if (record.timestamp > maxTimestamp)
-          maxTimestamp = record.timestamp
-      }
+        uncompressedSizeInBytes += record.sizeInBytes()
+        if (batch.magic > RecordBatch.MAGIC_VALUE_V0 && toMagic > RecordBatch.MAGIC_VALUE_V0) {
+          // Check if we need to overwrite offset
+          // No in place assignment situation 3
+          if (record.offset != expectedInnerOffset.getAndIncrement())
+            inPlaceAssignment = false
+          if (record.timestamp > maxTimestamp)
+            maxTimestamp = record.timestamp
+        }
 
-      // No in place assignment situation 4
-      if (!record.hasMagic(toMagic))
-        inPlaceAssignment = false
+        // No in place assignment situation 4
+        if (!record.hasMagic(toMagic))
+          inPlaceAssignment = false
 
-      validatedRecords += record
+        validatedRecords += record
+      }
     }
 
     if (!inPlaceAssignment) {
diff --git a/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala b/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
index d306b162804cb..324314f151e04 100644
--- a/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
@@ -37,40 +37,42 @@ class LogValidatorTest {
   val time = Time.SYSTEM
 
   @Test
-  def testOnlyOneBatchCompressedV0(): Unit = {
-    checkOnlyOneBatchCompressed(RecordBatch.MAGIC_VALUE_V0, CompressionType.GZIP)
+  def testOnlyOneBatch(): Unit = {
+    checkOnlyOneBatch(RecordBatch.MAGIC_VALUE_V0, CompressionType.GZIP, CompressionType.GZIP)
+    checkOnlyOneBatch(RecordBatch.MAGIC_VALUE_V1, CompressionType.GZIP, CompressionType.GZIP)
+    checkOnlyOneBatch(RecordBatch.MAGIC_VALUE_V2, CompressionType.GZIP, CompressionType.GZIP)
+    checkOnlyOneBatch(RecordBatch.MAGIC_VALUE_V0, CompressionType.GZIP, CompressionType.NONE)
+    checkOnlyOneBatch(RecordBatch.MAGIC_VALUE_V1, CompressionType.GZIP, CompressionType.NONE)
+    checkOnlyOneBatch(RecordBatch.MAGIC_VALUE_V2, CompressionType.GZIP, CompressionType.NONE)
+    checkOnlyOneBatch(RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, CompressionType.NONE)
+    checkOnlyOneBatch(RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, CompressionType.GZIP)
   }
 
   @Test
-  def testOnlyOneBatchCompressedV1(): Unit = {
-    checkOnlyOneBatchCompressed(RecordBatch.MAGIC_VALUE_V1, CompressionType.GZIP)
+  def testAllowMultiBatch(): Unit = {
+    checkAllowMultiBatch(RecordBatch.MAGIC_VALUE_V0, CompressionType.NONE, CompressionType.NONE)
+    checkAllowMultiBatch(RecordBatch.MAGIC_VALUE_V1, CompressionType.NONE, CompressionType.NONE)
+    checkAllowMultiBatch(RecordBatch.MAGIC_VALUE_V0, CompressionType.NONE, CompressionType.GZIP)
+    checkAllowMultiBatch(RecordBatch.MAGIC_VALUE_V1, CompressionType.NONE, CompressionType.GZIP)
   }
 
-  @Test
-  def testOnlyOneBatchCompressedV2(): Unit = {
-    checkOnlyOneBatchCompressed(RecordBatch.MAGIC_VALUE_V2, CompressionType.GZIP)
-  }
-
-  @Test
-  def testOnlyOneBatchUncompressedV2(): Unit = {
-    checkOnlyOneBatchCompressed(RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE)
-  }
-
-  private def checkOnlyOneBatchCompressed(magic: Byte, compressionType: CompressionType) {
-    validateMessages(createRecords(magic, 0L, compressionType), magic, compressionType)
-
+  private def checkOnlyOneBatch(magic: Byte, sourceCompressionType: CompressionType, targetCompressionType: CompressionType) {
     assertThrows[InvalidRecordException] {
-      validateMessages(createTwoBatchedRecords(magic, 0L, compressionType), magic, compressionType)
+      validateMessages(createTwoBatchedRecords(magic, 0L, sourceCompressionType), magic, sourceCompressionType, targetCompressionType)
     }
   }
 
-  private def validateMessages(records: MemoryRecords, magic: Byte, compressionType: CompressionType): Unit = {
+  private def checkAllowMultiBatch(magic: Byte, sourceCompressionType: CompressionType, targetCompressionType: CompressionType) {
+    validateMessages(createTwoBatchedRecords(magic, 0L, sourceCompressionType), magic, sourceCompressionType, targetCompressionType)
+  }
+
+  private def validateMessages(records: MemoryRecords, magic: Byte, sourceCompressionType: CompressionType, targetCompressionType: CompressionType): Unit = {
     LogValidator.validateMessagesAndAssignOffsets(records,
       new LongRef(0L),
       time,
       now = 0L,
-      CompressionCodec.getCompressionCodec(compressionType.name),
-      CompressionCodec.getCompressionCodec(compressionType.name),
+      CompressionCodec.getCompressionCodec(sourceCompressionType.name),
+      CompressionCodec.getCompressionCodec(targetCompressionType.name),
       compactedTopic = false,
       magic,
       TimestampType.CREATE_TIME,

From 1a3fe9aa52555eb24ce692963e4461d6f05b771d Mon Sep 17 00:00:00 2001
From: Hai-Dang Dam 
Date: Mon, 3 Jun 2019 19:06:00 -0700
Subject: [PATCH 0327/1071] KAFKA-8404: Add HttpHeader to RestClient HTTP
 Request and Connector REST API (#6791)

When Connect forwards a REST request from one worker to another, the Authorization header was not forwarded. This commit changes the Connect framework to add include the authorization header when forwarding requests to other workers.

Author: Hai-Dang Dam 
Reviewers: Robert Yokota , Randall Hauch 
---
 .../auth/extension/JaasBasicAuthFilter.java   |  16 ++-
 .../extension/JaasBasicAuthFilterTest.java    |  34 ++++-
 .../distributed/DistributedHerder.java        |   2 +-
 .../connect/runtime/rest/RestClient.java      |  21 ++-
 .../rest/resources/ConnectorsResource.java    |  45 ++++---
 .../ConnectorPluginsResourceTest.java         |   3 +-
 .../resources/ConnectorsResourceTest.java     | 122 ++++++++++++------
 7 files changed, 175 insertions(+), 68 deletions(-)

diff --git a/connect/basic-auth-extension/src/main/java/org/apache/kafka/connect/rest/basic/auth/extension/JaasBasicAuthFilter.java b/connect/basic-auth-extension/src/main/java/org/apache/kafka/connect/rest/basic/auth/extension/JaasBasicAuthFilter.java
index 6167434b98031..d5b15c6c65a5f 100644
--- a/connect/basic-auth-extension/src/main/java/org/apache/kafka/connect/rest/basic/auth/extension/JaasBasicAuthFilter.java
+++ b/connect/basic-auth-extension/src/main/java/org/apache/kafka/connect/rest/basic/auth/extension/JaasBasicAuthFilter.java
@@ -17,6 +17,8 @@
 
 package org.apache.kafka.connect.rest.basic.auth.extension;
 
+import java.util.regex.Pattern;
+import javax.ws.rs.HttpMethod;
 import org.apache.kafka.common.config.ConfigException;
 
 import java.io.IOException;
@@ -35,18 +37,18 @@
 import javax.ws.rs.core.Response;
 
 public class JaasBasicAuthFilter implements ContainerRequestFilter {
-
     private static final String CONNECT_LOGIN_MODULE = "KafkaConnect";
     static final String AUTHORIZATION = "Authorization";
-
+    private static final Pattern TASK_REQUEST_PATTERN = Pattern.compile("/?connectors/([^/]+)/tasks/?");
     @Override
     public void filter(ContainerRequestContext requestContext) throws IOException {
-
         try {
-            LoginContext loginContext =
-                new LoginContext(CONNECT_LOGIN_MODULE, new BasicAuthCallBackHandler(
-                    requestContext.getHeaderString(AUTHORIZATION)));
-            loginContext.login();
+            if (!(requestContext.getMethod().equals(HttpMethod.POST) && TASK_REQUEST_PATTERN.matcher(requestContext.getUriInfo().getPath()).matches())) {
+                LoginContext loginContext =
+                    new LoginContext(CONNECT_LOGIN_MODULE, new BasicAuthCallBackHandler(
+                        requestContext.getHeaderString(AUTHORIZATION)));
+                loginContext.login();
+            }
         } catch (LoginException | ConfigException e) {
             requestContext.abortWith(
                 Response.status(Response.Status.UNAUTHORIZED)
diff --git a/connect/basic-auth-extension/src/test/java/org/apache/kafka/connect/rest/basic/auth/extension/JaasBasicAuthFilterTest.java b/connect/basic-auth-extension/src/test/java/org/apache/kafka/connect/rest/basic/auth/extension/JaasBasicAuthFilterTest.java
index 835bef0f37c14..fe5f8b9de5e4e 100644
--- a/connect/basic-auth-extension/src/test/java/org/apache/kafka/connect/rest/basic/auth/extension/JaasBasicAuthFilterTest.java
+++ b/connect/basic-auth-extension/src/test/java/org/apache/kafka/connect/rest/basic/auth/extension/JaasBasicAuthFilterTest.java
@@ -17,12 +17,15 @@
 
 package org.apache.kafka.connect.rest.basic.auth.extension;
 
+import javax.ws.rs.HttpMethod;
+import javax.ws.rs.core.UriInfo;
 import org.apache.kafka.common.security.JaasUtils;
 import org.easymock.EasyMock;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.powermock.api.easymock.PowerMock;
 import org.powermock.api.easymock.annotation.MockStrict;
 import org.powermock.core.classloader.annotations.PowerMockIgnore;
 import org.powermock.modules.junit4.PowerMockRunner;
@@ -52,6 +55,9 @@ public class JaasBasicAuthFilterTest {
     private String previousJaasConfig;
     private Configuration previousConfiguration;
 
+    @MockStrict
+    private UriInfo uriInfo;
+
     @Before
     public void setup() {
         EasyMock.reset(requestContext);
@@ -137,7 +143,34 @@ public void testNoFileOption() throws IOException {
         jaasBasicAuthFilter.filter(requestContext);
     }
 
+    @Test
+    public void testPostWithoutAppropriateCredential() throws IOException {
+        EasyMock.expect(requestContext.getMethod()).andReturn(HttpMethod.POST);
+        EasyMock.expect(requestContext.getUriInfo()).andReturn(uriInfo);
+        EasyMock.expect(uriInfo.getPath()).andReturn("connectors/connName/tasks");
+
+        PowerMock.replayAll();
+        jaasBasicAuthFilter.filter(requestContext);
+        EasyMock.verify(requestContext);
+    }
+
+    @Test
+    public void testPostNotChangingConnectorTask() throws IOException {
+        EasyMock.expect(requestContext.getMethod()).andReturn(HttpMethod.POST);
+        EasyMock.expect(requestContext.getUriInfo()).andReturn(uriInfo);
+        EasyMock.expect(uriInfo.getPath()).andReturn("local:randomport/connectors/connName");
+        String authHeader = "Basic" + Base64.getEncoder().encodeToString(("user" + ":" + "password").getBytes());
+        EasyMock.expect(requestContext.getHeaderString(JaasBasicAuthFilter.AUTHORIZATION))
+            .andReturn(authHeader);
+        requestContext.abortWith(EasyMock.anyObject(Response.class));
+        EasyMock.expectLastCall();
+        PowerMock.replayAll();
+        jaasBasicAuthFilter.filter(requestContext);
+        EasyMock.verify(requestContext);
+    }
+
     private void setMock(String authorization, String username, String password, boolean exceptionCase) {
+        EasyMock.expect(requestContext.getMethod()).andReturn(HttpMethod.GET);
         String authHeader = authorization + " " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes());
         EasyMock.expect(requestContext.getHeaderString(JaasBasicAuthFilter.AUTHORIZATION))
             .andReturn(authHeader);
@@ -152,7 +185,6 @@ private void setupJaasConfig(String loginModule, String credentialFilePath, bool
         File jaasConfigFile = File.createTempFile("ks-jaas-", ".conf");
         jaasConfigFile.deleteOnExit();
         previousJaasConfig = System.setProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM, jaasConfigFile.getPath());
-
         List lines;
         lines = new ArrayList<>();
         lines.add(loginModule + " { org.apache.kafka.connect.rest.basic.auth.extension.PropertyFileLoginModule required ");
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
index 52709f7856fbb..8e59d87153a33 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
@@ -1202,7 +1202,7 @@ public void run() {
                                     return;
                                 }
                                 String reconfigUrl = RestServer.urlJoin(leaderUrl, "/connectors/" + connName + "/tasks");
-                                RestClient.httpRequest(reconfigUrl, "POST", rawTaskProps, null, config);
+                                RestClient.httpRequest(reconfigUrl, "POST", null, rawTaskProps, null, config);
                                 cb.onCompletion(null, null);
                             } catch (ConnectException e) {
                                 log.error("Request to leader to reconfigure connector tasks failed", e);
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestClient.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestClient.java
index c1b6036b28563..7ef1c78cf0031 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestClient.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestClient.java
@@ -19,6 +19,7 @@
 
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import javax.ws.rs.core.HttpHeaders;
 import org.apache.kafka.connect.runtime.WorkerConfig;
 import org.apache.kafka.connect.runtime.rest.entities.ErrorMessage;
 import org.apache.kafka.connect.runtime.rest.errors.ConnectRestException;
@@ -50,12 +51,13 @@ public class RestClient {
      *
      * @param url             HTTP connection will be established with this url.
      * @param method          HTTP method ("GET", "POST", "PUT", etc.)
+     * @param headers         HTTP headers from REST endpoint
      * @param requestBodyData Object to serialize as JSON and send in the request body.
      * @param responseFormat  Expected format of the response to the HTTP request.
      * @param              The type of the deserialized response to the HTTP request.
      * @return The deserialized response to the HTTP request, or null if no data is expected.
      */
-    public static  HttpResponse httpRequest(String url, String method, Object requestBodyData,
+    public static  HttpResponse httpRequest(String url, String method, HttpHeaders headers, Object requestBodyData,
                                                   TypeReference responseFormat, WorkerConfig config) {
         HttpClient client;
 
@@ -82,6 +84,8 @@ public static  HttpResponse httpRequest(String url, String method, Object
             req.method(method);
             req.accept("application/json");
             req.agent("kafka-connect");
+            addHeadersToRequest(headers, req);
+
             if (serializedBody != null) {
                 req.content(new StringContentProvider(serializedBody, StandardCharsets.UTF_8), "application/json");
             }
@@ -116,6 +120,21 @@ public static  HttpResponse httpRequest(String url, String method, Object
         }
     }
 
+
+    /**
+     * Extract headers from REST call and add to client request
+     * @param headers         Headers from REST endpoint
+     * @param req             The client request to modify
+     */
+    private static void addHeadersToRequest(HttpHeaders headers, Request req) {
+        if (headers != null) {
+            String credentialAuthorization = headers.getHeaderString(HttpHeaders.AUTHORIZATION);
+            if (credentialAuthorization != null) {
+                req.header(HttpHeaders.AUTHORIZATION, credentialAuthorization);
+            }
+        }
+    }
+
     /**
      * Convert response parameters from Jetty format (HttpFields)
      * @param httpFields
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResource.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResource.java
index 61cf5da83a190..96a4f51eec312 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResource.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResource.java
@@ -18,6 +18,7 @@
 
 import com.fasterxml.jackson.core.type.TypeReference;
 
+import javax.ws.rs.core.HttpHeaders;
 import org.apache.kafka.connect.errors.NotFoundException;
 import org.apache.kafka.connect.runtime.ConnectorConfig;
 import org.apache.kafka.connect.runtime.Herder;
@@ -85,7 +86,8 @@ public ConnectorsResource(Herder herder, WorkerConfig config) {
     @GET
     @Path("/")
     public Response listConnectors(
-        final @Context UriInfo uriInfo
+        final @Context UriInfo uriInfo,
+        final @Context HttpHeaders headers
     ) throws Throwable {
         if (uriInfo.getQueryParameters().containsKey("expand")) {
             Map> out = new HashMap<>();
@@ -121,6 +123,7 @@ public Response listConnectors(
     @POST
     @Path("/")
     public Response createConnector(final @QueryParam("forward") Boolean forward,
+                                    final @Context HttpHeaders headers,
                                     final CreateConnectorRequest createRequest) throws Throwable {
         // Trim leading and trailing whitespaces from the connector name, replace null with empty string
         // if no name element present to keep validation within validator (NonEmptyStringWithoutControlChars
@@ -132,7 +135,7 @@ public Response createConnector(final @QueryParam("forward") Boolean forward,
 
         FutureCallback> cb = new FutureCallback<>();
         herder.putConnectorConfig(name, configs, false, cb);
-        Herder.Created info = completeOrForwardRequest(cb, "/connectors", "POST", createRequest,
+        Herder.Created info = completeOrForwardRequest(cb, "/connectors", "POST", headers, createRequest,
                 new TypeReference() { }, new CreatedConnectorInfoTranslator(), forward);
 
         URI location = UriBuilder.fromUri("/connectors").path(name).build();
@@ -142,19 +145,21 @@ public Response createConnector(final @QueryParam("forward") Boolean forward,
     @GET
     @Path("/{connector}")
     public ConnectorInfo getConnector(final @PathParam("connector") String connector,
+                                      final @Context HttpHeaders headers,
                                       final @QueryParam("forward") Boolean forward) throws Throwable {
         FutureCallback cb = new FutureCallback<>();
         herder.connectorInfo(connector, cb);
-        return completeOrForwardRequest(cb, "/connectors/" + connector, "GET", null, forward);
+        return completeOrForwardRequest(cb, "/connectors/" + connector, "GET", headers, null, forward);
     }
 
     @GET
     @Path("/{connector}/config")
     public Map getConnectorConfig(final @PathParam("connector") String connector,
+                                                  final @Context HttpHeaders headers,
                                                   final @QueryParam("forward") Boolean forward) throws Throwable {
         FutureCallback> cb = new FutureCallback<>();
         herder.connectorConfig(connector, cb);
-        return completeOrForwardRequest(cb, "/connectors/" + connector + "/config", "GET", null, forward);
+        return completeOrForwardRequest(cb, "/connectors/" + connector + "/config", "GET", headers, null, forward);
     }
 
     @GET
@@ -166,6 +171,7 @@ public ConnectorStateInfo getConnectorStatus(final @PathParam("connector") Strin
     @PUT
     @Path("/{connector}/config")
     public Response putConnectorConfig(final @PathParam("connector") String connector,
+                                       final @Context HttpHeaders headers,
                                        final @QueryParam("forward") Boolean forward,
                                        final Map connectorConfig) throws Throwable {
         FutureCallback> cb = new FutureCallback<>();
@@ -173,7 +179,7 @@ public Response putConnectorConfig(final @PathParam("connector") String connecto
 
         herder.putConnectorConfig(connector, connectorConfig, true, cb);
         Herder.Created createdInfo = completeOrForwardRequest(cb, "/connectors/" + connector + "/config",
-                "PUT", connectorConfig, new TypeReference() { }, new CreatedConnectorInfoTranslator(), forward);
+                "PUT", headers, connectorConfig, new TypeReference() { }, new CreatedConnectorInfoTranslator(), forward);
         Response.ResponseBuilder response;
         if (createdInfo.created()) {
             URI location = UriBuilder.fromUri("/connectors").path(connector).build();
@@ -187,15 +193,16 @@ public Response putConnectorConfig(final @PathParam("connector") String connecto
     @POST
     @Path("/{connector}/restart")
     public void restartConnector(final @PathParam("connector") String connector,
+                                 final @Context HttpHeaders headers,
                                  final @QueryParam("forward") Boolean forward) throws Throwable {
         FutureCallback cb = new FutureCallback<>();
         herder.restartConnector(connector, cb);
-        completeOrForwardRequest(cb, "/connectors/" + connector + "/restart", "POST", null, forward);
+        completeOrForwardRequest(cb, "/connectors/" + connector + "/restart", "POST", headers, null, forward);
     }
 
     @PUT
     @Path("/{connector}/pause")
-    public Response pauseConnector(@PathParam("connector") String connector) {
+    public Response pauseConnector(@PathParam("connector") String connector, final @Context HttpHeaders headers) {
         herder.pauseConnector(connector);
         return Response.accepted().build();
     }
@@ -210,26 +217,29 @@ public Response resumeConnector(@PathParam("connector") String connector) {
     @GET
     @Path("/{connector}/tasks")
     public List getTaskConfigs(final @PathParam("connector") String connector,
+                                         final @Context HttpHeaders headers,
                                          final @QueryParam("forward") Boolean forward) throws Throwable {
         FutureCallback> cb = new FutureCallback<>();
         herder.taskConfigs(connector, cb);
-        return completeOrForwardRequest(cb, "/connectors/" + connector + "/tasks", "GET", null, new TypeReference>() {
+        return completeOrForwardRequest(cb, "/connectors/" + connector + "/tasks", "GET", headers, null, new TypeReference>() {
         }, forward);
     }
 
     @POST
     @Path("/{connector}/tasks")
     public void putTaskConfigs(final @PathParam("connector") String connector,
+                               final @Context HttpHeaders headers,
                                final @QueryParam("forward") Boolean forward,
                                final List> taskConfigs) throws Throwable {
         FutureCallback cb = new FutureCallback<>();
         herder.putTaskConfigs(connector, taskConfigs, cb);
-        completeOrForwardRequest(cb, "/connectors/" + connector + "/tasks", "POST", taskConfigs, forward);
+        completeOrForwardRequest(cb, "/connectors/" + connector + "/tasks", "POST", headers, taskConfigs, forward);
     }
 
     @GET
     @Path("/{connector}/tasks/{task}/status")
     public ConnectorStateInfo.TaskState getTaskStatus(final @PathParam("connector") String connector,
+                                                      final @Context HttpHeaders headers,
                                                       final @PathParam("task") Integer task) throws Throwable {
         return herder.taskStatus(new ConnectorTaskId(connector, task));
     }
@@ -238,20 +248,22 @@ public ConnectorStateInfo.TaskState getTaskStatus(final @PathParam("connector")
     @Path("/{connector}/tasks/{task}/restart")
     public void restartTask(final @PathParam("connector") String connector,
                             final @PathParam("task") Integer task,
+                            final @Context HttpHeaders headers,
                             final @QueryParam("forward") Boolean forward) throws Throwable {
         FutureCallback cb = new FutureCallback<>();
         ConnectorTaskId taskId = new ConnectorTaskId(connector, task);
         herder.restartTask(taskId, cb);
-        completeOrForwardRequest(cb, "/connectors/" + connector + "/tasks/" + task + "/restart", "POST", null, forward);
+        completeOrForwardRequest(cb, "/connectors/" + connector + "/tasks/" + task + "/restart", "POST", headers, null, forward);
     }
 
     @DELETE
     @Path("/{connector}")
     public void destroyConnector(final @PathParam("connector") String connector,
+                                 final @Context HttpHeaders headers,
                                  final @QueryParam("forward") Boolean forward) throws Throwable {
         FutureCallback> cb = new FutureCallback<>();
         herder.deleteConnectorConfig(connector, cb);
-        completeOrForwardRequest(cb, "/connectors/" + connector, "DELETE", null, forward);
+        completeOrForwardRequest(cb, "/connectors/" + connector, "DELETE", headers, null, forward);
     }
 
     // Check whether the connector name from the url matches the one (if there is one) provided in the connectorconfig
@@ -271,6 +283,7 @@ private void checkAndPutConnectorConfigName(String connectorName, Map T completeOrForwardRequest(FutureCallback cb,
                                               String path,
                                               String method,
+                                              HttpHeaders headers,
                                               Object body,
                                               TypeReference resultType,
                                               Translator translator,
@@ -293,7 +306,7 @@ private  T completeOrForwardRequest(FutureCallback cb,
                             .build()
                             .toString();
                     log.debug("Forwarding request {} {} {}", forwardUrl, method, body);
-                    return translator.translate(RestClient.httpRequest(forwardUrl, method, body, resultType, config));
+                    return translator.translate(RestClient.httpRequest(forwardUrl, method, headers, body, resultType, config));
                 } else {
                     // we should find the right target for the query within two hops, so if
                     // we don't, it probably means that a rebalance has taken place.
@@ -315,14 +328,14 @@ private  T completeOrForwardRequest(FutureCallback cb,
         }
     }
 
-    private  T completeOrForwardRequest(FutureCallback cb, String path, String method, Object body,
+    private  T completeOrForwardRequest(FutureCallback cb, String path, String method, HttpHeaders headers, Object body,
                                            TypeReference resultType, Boolean forward) throws Throwable {
-        return completeOrForwardRequest(cb, path, method, body, resultType, new IdentityTranslator(), forward);
+        return completeOrForwardRequest(cb, path, method, headers, body, resultType, new IdentityTranslator(), forward);
     }
 
-    private  T completeOrForwardRequest(FutureCallback cb, String path, String method,
+    private  T completeOrForwardRequest(FutureCallback cb, String path, String method, HttpHeaders headers,
                                            Object body, Boolean forward) throws Throwable {
-        return completeOrForwardRequest(cb, path, method, body, null, new IdentityTranslator(), forward);
+        return completeOrForwardRequest(cb, path, method, headers, body, null, new IdentityTranslator(), forward);
     }
 
     private interface Translator {
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorPluginsResourceTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorPluginsResourceTest.java
index a3aee6a407d2e..2882355eccbec 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorPluginsResourceTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorPluginsResourceTest.java
@@ -18,6 +18,7 @@
 
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import javax.ws.rs.core.HttpHeaders;
 import org.apache.kafka.common.config.Config;
 import org.apache.kafka.common.config.ConfigDef;
 import org.apache.kafka.common.config.ConfigDef.Importance;
@@ -182,7 +183,7 @@ public class ConnectorPluginsResourceTest {
     @Before
     public void setUp() throws Exception {
         PowerMock.mockStatic(RestClient.class,
-                RestClient.class.getMethod("httpRequest", String.class, String.class, Object.class, TypeReference.class, WorkerConfig.class));
+                RestClient.class.getMethod("httpRequest", String.class, String.class, HttpHeaders.class, Object.class, TypeReference.class, WorkerConfig.class));
 
         plugins = PowerMock.createMock(Plugins.class);
         herder = PowerMock.createMock(AbstractHerder.class);
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java
index 5dc7f1ee687b9..5490df03afb6d 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java
@@ -18,6 +18,7 @@
 
 import com.fasterxml.jackson.core.type.TypeReference;
 
+import javax.ws.rs.core.HttpHeaders;
 import org.apache.kafka.connect.errors.AlreadyExistsException;
 import org.apache.kafka.connect.errors.NotFoundException;
 import org.apache.kafka.connect.runtime.ConnectorConfig;
@@ -79,6 +80,7 @@ public class ConnectorsResourceTest {
     private static final String CONNECTOR_NAME_PADDING_WHITESPACES = "   " + CONNECTOR_NAME + "  \n  ";
     private static final Boolean FORWARD = true;
     private static final Map CONNECTOR_CONFIG_SPECIAL_CHARS = new HashMap<>();
+    private static final HttpHeaders NULL_HEADERS = null;
     static {
         CONNECTOR_CONFIG_SPECIAL_CHARS.put("name", CONNECTOR_NAME_SPECIAL_CHARS);
         CONNECTOR_CONFIG_SPECIAL_CHARS.put("sample_config", "test_config");
@@ -130,7 +132,7 @@ public class ConnectorsResourceTest {
     @Before
     public void setUp() throws NoSuchMethodException {
         PowerMock.mockStatic(RestClient.class,
-                RestClient.class.getMethod("httpRequest", String.class, String.class, Object.class, TypeReference.class, WorkerConfig.class));
+                RestClient.class.getMethod("httpRequest", String.class, String.class, HttpHeaders.class, Object.class, TypeReference.class, WorkerConfig.class));
         connectorsResource = new ConnectorsResource(herder, null);
         forward = EasyMock.mock(UriInfo.class);
         MultivaluedMap queryParams = new MultivaluedHashMap<>();
@@ -151,7 +153,7 @@ public void testListConnectors() throws Throwable {
 
         PowerMock.replayAll();
 
-        Collection connectors = (Collection) connectorsResource.listConnectors(forward).getEntity();
+        Collection connectors = (Collection) connectorsResource.listConnectors(forward, NULL_HEADERS).getEntity();
         // Ordering isn't guaranteed, compare sets
         assertEquals(new HashSet<>(Arrays.asList(CONNECTOR_NAME, CONNECTOR2_NAME)), new HashSet<>(connectors));
 
@@ -174,7 +176,7 @@ public void testExpandConnectorsStatus() throws Throwable {
 
         PowerMock.replayAll();
 
-        Map> expanded = (Map>) connectorsResource.listConnectors(forward).getEntity();
+        Map> expanded = (Map>) connectorsResource.listConnectors(forward, NULL_HEADERS).getEntity();
         // Ordering isn't guaranteed, compare sets
         assertEquals(new HashSet<>(Arrays.asList(CONNECTOR_NAME, CONNECTOR2_NAME)), expanded.keySet());
         assertEquals(connector2, expanded.get(CONNECTOR2_NAME).get("status"));
@@ -198,7 +200,7 @@ public void testExpandConnectorsInfo() throws Throwable {
 
         PowerMock.replayAll();
 
-        Map> expanded = (Map>) connectorsResource.listConnectors(forward).getEntity();
+        Map> expanded = (Map>) connectorsResource.listConnectors(forward, NULL_HEADERS).getEntity();
         // Ordering isn't guaranteed, compare sets
         assertEquals(new HashSet<>(Arrays.asList(CONNECTOR_NAME, CONNECTOR2_NAME)), expanded.keySet());
         assertEquals(connector2, expanded.get(CONNECTOR2_NAME).get("info"));
@@ -226,7 +228,7 @@ public void testFullExpandConnectors() throws Throwable {
 
         PowerMock.replayAll();
 
-        Map> expanded = (Map>) connectorsResource.listConnectors(forward).getEntity();
+        Map> expanded = (Map>) connectorsResource.listConnectors(forward, NULL_HEADERS).getEntity();
         // Ordering isn't guaranteed, compare sets
         assertEquals(new HashSet<>(Arrays.asList(CONNECTOR_NAME, CONNECTOR2_NAME)), expanded.keySet());
         assertEquals(connectorInfo2, expanded.get(CONNECTOR2_NAME).get("info"));
@@ -252,7 +254,7 @@ public void testExpandConnectorsWithConnectorNotFound() throws Throwable {
 
         PowerMock.replayAll();
 
-        Map> expanded = (Map>) connectorsResource.listConnectors(forward).getEntity();
+        Map> expanded = (Map>) connectorsResource.listConnectors(forward, NULL_HEADERS).getEntity();
         // Ordering isn't guaranteed, compare sets
         assertEquals(Collections.singleton(CONNECTOR2_NAME), expanded.keySet());
         assertEquals(connector2, expanded.get(CONNECTOR2_NAME).get("status"));
@@ -271,7 +273,7 @@ public void testCreateConnector() throws Throwable {
 
         PowerMock.replayAll();
 
-        connectorsResource.createConnector(FORWARD, body);
+        connectorsResource.createConnector(FORWARD, NULL_HEADERS, body);
 
         PowerMock.verifyAll();
     }
@@ -284,19 +286,57 @@ public void testCreateConnectorNotLeader() throws Throwable {
         herder.putConnectorConfig(EasyMock.eq(CONNECTOR_NAME), EasyMock.eq(body.config()), EasyMock.eq(false), EasyMock.capture(cb));
         expectAndCallbackNotLeaderException(cb);
         // Should forward request
-        EasyMock.expect(RestClient.httpRequest(EasyMock.eq("http://leader:8083/connectors?forward=false"), EasyMock.eq("POST"), EasyMock.eq(body), EasyMock.anyObject(), EasyMock.anyObject(WorkerConfig.class)))
+        EasyMock.expect(RestClient.httpRequest(EasyMock.eq("http://leader:8083/connectors?forward=false"), EasyMock.eq("POST"), EasyMock.isNull(), EasyMock.eq(body), EasyMock.anyObject(), EasyMock.anyObject(WorkerConfig.class)))
                 .andReturn(new RestClient.HttpResponse<>(201, new HashMap(), new ConnectorInfo(CONNECTOR_NAME, CONNECTOR_CONFIG, CONNECTOR_TASK_NAMES,
                     ConnectorType.SOURCE)));
 
         PowerMock.replayAll();
 
-        connectorsResource.createConnector(FORWARD, body);
+        connectorsResource.createConnector(FORWARD, NULL_HEADERS, body);
 
         PowerMock.verifyAll();
 
 
     }
 
+    @Test
+    public void testCreateConnectorWithHeaderAuthorization() throws Throwable {
+        CreateConnectorRequest body = new CreateConnectorRequest(CONNECTOR_NAME, Collections.singletonMap(ConnectorConfig.NAME_CONFIG, CONNECTOR_NAME));
+        final Capture>> cb = Capture.newInstance();
+        HttpHeaders httpHeaders = EasyMock.mock(HttpHeaders.class);
+        EasyMock.expect(httpHeaders.getHeaderString("Authorization")).andReturn("Basic YWxhZGRpbjpvcGVuc2VzYW1l").times(1);
+        EasyMock.replay(httpHeaders);
+        herder.putConnectorConfig(EasyMock.eq(CONNECTOR_NAME), EasyMock.eq(body.config()), EasyMock.eq(false), EasyMock.capture(cb));
+        expectAndCallbackResult(cb, new Herder.Created<>(true, new ConnectorInfo(CONNECTOR_NAME, CONNECTOR_CONFIG,
+            CONNECTOR_TASK_NAMES, ConnectorType.SOURCE)));
+
+        PowerMock.replayAll();
+
+        connectorsResource.createConnector(FORWARD, httpHeaders, body);
+
+        PowerMock.verifyAll();
+    }
+
+
+
+    @Test
+    public void testCreateConnectorWithoutHeaderAuthorization() throws Throwable {
+        CreateConnectorRequest body = new CreateConnectorRequest(CONNECTOR_NAME, Collections.singletonMap(ConnectorConfig.NAME_CONFIG, CONNECTOR_NAME));
+        final Capture>> cb = Capture.newInstance();
+        HttpHeaders httpHeaders = EasyMock.mock(HttpHeaders.class);
+        EasyMock.expect(httpHeaders.getHeaderString("Authorization")).andReturn(null).times(1);
+        EasyMock.replay(httpHeaders);
+        herder.putConnectorConfig(EasyMock.eq(CONNECTOR_NAME), EasyMock.eq(body.config()), EasyMock.eq(false), EasyMock.capture(cb));
+        expectAndCallbackResult(cb, new Herder.Created<>(true, new ConnectorInfo(CONNECTOR_NAME, CONNECTOR_CONFIG,
+            CONNECTOR_TASK_NAMES, ConnectorType.SOURCE)));
+
+        PowerMock.replayAll();
+
+        connectorsResource.createConnector(FORWARD, httpHeaders, body);
+
+        PowerMock.verifyAll();
+    }
+
     @Test(expected = AlreadyExistsException.class)
     public void testCreateConnectorExists() throws Throwable {
         CreateConnectorRequest body = new CreateConnectorRequest(CONNECTOR_NAME, Collections.singletonMap(ConnectorConfig.NAME_CONFIG, CONNECTOR_NAME));
@@ -307,7 +347,7 @@ public void testCreateConnectorExists() throws Throwable {
 
         PowerMock.replayAll();
 
-        connectorsResource.createConnector(FORWARD, body);
+        connectorsResource.createConnector(FORWARD, NULL_HEADERS, body);
 
         PowerMock.verifyAll();
     }
@@ -326,7 +366,7 @@ public void testCreateConnectorNameTrimWhitespaces() throws Throwable {
 
         PowerMock.replayAll();
 
-        connectorsResource.createConnector(FORWARD, bodyIn);
+        connectorsResource.createConnector(FORWARD, NULL_HEADERS, bodyIn);
 
         PowerMock.verifyAll();
     }
@@ -345,7 +385,7 @@ public void testCreateConnectorNameAllWhitespaces() throws Throwable {
 
         PowerMock.replayAll();
 
-        connectorsResource.createConnector(FORWARD, bodyIn);
+        connectorsResource.createConnector(FORWARD, NULL_HEADERS, bodyIn);
 
         PowerMock.verifyAll();
     }
@@ -364,7 +404,7 @@ public void testCreateConnectorNoName() throws Throwable {
 
         PowerMock.replayAll();
 
-        connectorsResource.createConnector(FORWARD, bodyIn);
+        connectorsResource.createConnector(FORWARD, NULL_HEADERS, bodyIn);
 
         PowerMock.verifyAll();
     }
@@ -377,7 +417,7 @@ public void testDeleteConnector() throws Throwable {
 
         PowerMock.replayAll();
 
-        connectorsResource.destroyConnector(CONNECTOR_NAME, FORWARD);
+        connectorsResource.destroyConnector(CONNECTOR_NAME, NULL_HEADERS, FORWARD);
 
         PowerMock.verifyAll();
     }
@@ -388,12 +428,12 @@ public void testDeleteConnectorNotLeader() throws Throwable {
         herder.deleteConnectorConfig(EasyMock.eq(CONNECTOR_NAME), EasyMock.capture(cb));
         expectAndCallbackNotLeaderException(cb);
         // Should forward request
-        EasyMock.expect(RestClient.httpRequest("http://leader:8083/connectors/" + CONNECTOR_NAME + "?forward=false", "DELETE", null, null, null))
+        EasyMock.expect(RestClient.httpRequest("http://leader:8083/connectors/" + CONNECTOR_NAME + "?forward=false", "DELETE", NULL_HEADERS, null, null, null))
                 .andReturn(new RestClient.HttpResponse<>(204, new HashMap(), null));
 
         PowerMock.replayAll();
 
-        connectorsResource.destroyConnector(CONNECTOR_NAME, FORWARD);
+        connectorsResource.destroyConnector(CONNECTOR_NAME, NULL_HEADERS, FORWARD);
 
         PowerMock.verifyAll();
     }
@@ -407,7 +447,7 @@ public void testDeleteConnectorNotFound() throws Throwable {
 
         PowerMock.replayAll();
 
-        connectorsResource.destroyConnector(CONNECTOR_NAME, FORWARD);
+        connectorsResource.destroyConnector(CONNECTOR_NAME, NULL_HEADERS, FORWARD);
 
         PowerMock.verifyAll();
     }
@@ -421,7 +461,7 @@ public void testGetConnector() throws Throwable {
 
         PowerMock.replayAll();
 
-        ConnectorInfo connInfo = connectorsResource.getConnector(CONNECTOR_NAME, FORWARD);
+        ConnectorInfo connInfo = connectorsResource.getConnector(CONNECTOR_NAME, NULL_HEADERS, FORWARD);
         assertEquals(new ConnectorInfo(CONNECTOR_NAME, CONNECTOR_CONFIG, CONNECTOR_TASK_NAMES, ConnectorType.SOURCE),
             connInfo);
 
@@ -436,7 +476,7 @@ public void testGetConnectorConfig() throws Throwable {
 
         PowerMock.replayAll();
 
-        Map connConfig = connectorsResource.getConnectorConfig(CONNECTOR_NAME, FORWARD);
+        Map connConfig = connectorsResource.getConnectorConfig(CONNECTOR_NAME, NULL_HEADERS, FORWARD);
         assertEquals(CONNECTOR_CONFIG, connConfig);
 
         PowerMock.verifyAll();
@@ -450,7 +490,7 @@ public void testGetConnectorConfigConnectorNotFound() throws Throwable {
 
         PowerMock.replayAll();
 
-        connectorsResource.getConnectorConfig(CONNECTOR_NAME, FORWARD);
+        connectorsResource.getConnectorConfig(CONNECTOR_NAME, NULL_HEADERS, FORWARD);
 
         PowerMock.verifyAll();
     }
@@ -464,7 +504,7 @@ public void testPutConnectorConfig() throws Throwable {
 
         PowerMock.replayAll();
 
-        connectorsResource.putConnectorConfig(CONNECTOR_NAME, FORWARD, CONNECTOR_CONFIG);
+        connectorsResource.putConnectorConfig(CONNECTOR_NAME, NULL_HEADERS, FORWARD, CONNECTOR_CONFIG);
 
         PowerMock.verifyAll();
     }
@@ -480,7 +520,7 @@ public void testCreateConnectorWithSpecialCharsInName() throws Throwable {
 
         PowerMock.replayAll();
 
-        String rspLocation = connectorsResource.createConnector(FORWARD, body).getLocation().toString();
+        String rspLocation = connectorsResource.createConnector(FORWARD, NULL_HEADERS, body).getLocation().toString();
         String decoded = new URI(rspLocation).getPath();
         Assert.assertEquals("/connectors/" + CONNECTOR_NAME_SPECIAL_CHARS, decoded);
 
@@ -498,7 +538,7 @@ public void testCreateConnectorWithControlSequenceInName() throws Throwable {
 
         PowerMock.replayAll();
 
-        String rspLocation = connectorsResource.createConnector(FORWARD, body).getLocation().toString();
+        String rspLocation = connectorsResource.createConnector(FORWARD, NULL_HEADERS, body).getLocation().toString();
         String decoded = new URI(rspLocation).getPath();
         Assert.assertEquals("/connectors/" + CONNECTOR_NAME_CONTROL_SEQUENCES1, decoded);
 
@@ -515,7 +555,7 @@ public void testPutConnectorConfigWithSpecialCharsInName() throws Throwable {
 
         PowerMock.replayAll();
 
-        String rspLocation = connectorsResource.putConnectorConfig(CONNECTOR_NAME_SPECIAL_CHARS, FORWARD, CONNECTOR_CONFIG_SPECIAL_CHARS).getLocation().toString();
+        String rspLocation = connectorsResource.putConnectorConfig(CONNECTOR_NAME_SPECIAL_CHARS, NULL_HEADERS, FORWARD, CONNECTOR_CONFIG_SPECIAL_CHARS).getLocation().toString();
         String decoded = new URI(rspLocation).getPath();
         Assert.assertEquals("/connectors/" + CONNECTOR_NAME_SPECIAL_CHARS, decoded);
 
@@ -532,7 +572,7 @@ public void testPutConnectorConfigWithControlSequenceInName() throws Throwable {
 
         PowerMock.replayAll();
 
-        String rspLocation = connectorsResource.putConnectorConfig(CONNECTOR_NAME_CONTROL_SEQUENCES1, FORWARD, CONNECTOR_CONFIG_CONTROL_SEQUENCES).getLocation().toString();
+        String rspLocation = connectorsResource.putConnectorConfig(CONNECTOR_NAME_CONTROL_SEQUENCES1, NULL_HEADERS, FORWARD, CONNECTOR_CONFIG_CONTROL_SEQUENCES).getLocation().toString();
         String decoded = new URI(rspLocation).getPath();
         Assert.assertEquals("/connectors/" + CONNECTOR_NAME_CONTROL_SEQUENCES1, decoded);
 
@@ -543,7 +583,7 @@ public void testPutConnectorConfigWithControlSequenceInName() throws Throwable {
     public void testPutConnectorConfigNameMismatch() throws Throwable {
         Map connConfig = new HashMap<>(CONNECTOR_CONFIG);
         connConfig.put(ConnectorConfig.NAME_CONFIG, "mismatched-name");
-        connectorsResource.putConnectorConfig(CONNECTOR_NAME, FORWARD, connConfig);
+        connectorsResource.putConnectorConfig(CONNECTOR_NAME, NULL_HEADERS, FORWARD, connConfig);
     }
 
     @Test(expected = BadRequestException.class)
@@ -551,7 +591,7 @@ public void testCreateConnectorConfigNameMismatch() throws Throwable {
         Map connConfig = new HashMap<>();
         connConfig.put(ConnectorConfig.NAME_CONFIG, "mismatched-name");
         CreateConnectorRequest request = new CreateConnectorRequest(CONNECTOR_NAME, connConfig);
-        connectorsResource.createConnector(FORWARD, request);
+        connectorsResource.createConnector(FORWARD, NULL_HEADERS, request);
     }
 
     @Test
@@ -562,7 +602,7 @@ public void testGetConnectorTaskConfigs() throws Throwable {
 
         PowerMock.replayAll();
 
-        List taskInfos = connectorsResource.getTaskConfigs(CONNECTOR_NAME, FORWARD);
+        List taskInfos = connectorsResource.getTaskConfigs(CONNECTOR_NAME, NULL_HEADERS, FORWARD);
         assertEquals(TASK_INFOS, taskInfos);
 
         PowerMock.verifyAll();
@@ -576,7 +616,7 @@ public void testGetConnectorTaskConfigsConnectorNotFound() throws Throwable {
 
         PowerMock.replayAll();
 
-        connectorsResource.getTaskConfigs(CONNECTOR_NAME, FORWARD);
+        connectorsResource.getTaskConfigs(CONNECTOR_NAME, NULL_HEADERS, FORWARD);
 
         PowerMock.verifyAll();
     }
@@ -589,7 +629,7 @@ public void testPutConnectorTaskConfigs() throws Throwable {
 
         PowerMock.replayAll();
 
-        connectorsResource.putTaskConfigs(CONNECTOR_NAME, FORWARD, TASK_CONFIGS);
+        connectorsResource.putTaskConfigs(CONNECTOR_NAME, NULL_HEADERS, FORWARD, TASK_CONFIGS);
 
         PowerMock.verifyAll();
     }
@@ -602,7 +642,7 @@ public void testPutConnectorTaskConfigsConnectorNotFound() throws Throwable {
 
         PowerMock.replayAll();
 
-        connectorsResource.putTaskConfigs(CONNECTOR_NAME, FORWARD, TASK_CONFIGS);
+        connectorsResource.putTaskConfigs(CONNECTOR_NAME, NULL_HEADERS, FORWARD, TASK_CONFIGS);
 
         PowerMock.verifyAll();
     }
@@ -615,7 +655,7 @@ public void testRestartConnectorNotFound() throws Throwable {
 
         PowerMock.replayAll();
 
-        connectorsResource.restartConnector(CONNECTOR_NAME, FORWARD);
+        connectorsResource.restartConnector(CONNECTOR_NAME, NULL_HEADERS, FORWARD);
 
         PowerMock.verifyAll();
     }
@@ -627,12 +667,12 @@ public void testRestartConnectorLeaderRedirect() throws Throwable {
         expectAndCallbackNotLeaderException(cb);
 
         EasyMock.expect(RestClient.httpRequest(EasyMock.eq("http://leader:8083/connectors/" + CONNECTOR_NAME + "/restart?forward=true"),
-                EasyMock.eq("POST"), EasyMock.isNull(), EasyMock.anyObject(), EasyMock.anyObject(WorkerConfig.class)))
+                EasyMock.eq("POST"), EasyMock.isNull(), EasyMock.isNull(), EasyMock.anyObject(), EasyMock.anyObject(WorkerConfig.class)))
                 .andReturn(new RestClient.HttpResponse<>(202, new HashMap(), null));
 
         PowerMock.replayAll();
 
-        connectorsResource.restartConnector(CONNECTOR_NAME, null);
+        connectorsResource.restartConnector(CONNECTOR_NAME, NULL_HEADERS, null);
 
         PowerMock.verifyAll();
     }
@@ -645,12 +685,12 @@ public void testRestartConnectorOwnerRedirect() throws Throwable {
         expectAndCallbackException(cb, new NotAssignedException("not owner test", ownerUrl));
 
         EasyMock.expect(RestClient.httpRequest(EasyMock.eq("http://owner:8083/connectors/" + CONNECTOR_NAME + "/restart?forward=false"),
-                EasyMock.eq("POST"), EasyMock.isNull(), EasyMock.anyObject(), EasyMock.anyObject(WorkerConfig.class)))
+                EasyMock.eq("POST"), EasyMock.isNull(), EasyMock.isNull(), EasyMock.anyObject(), EasyMock.anyObject(WorkerConfig.class)))
                 .andReturn(new RestClient.HttpResponse<>(202, new HashMap(), null));
 
         PowerMock.replayAll();
 
-        connectorsResource.restartConnector(CONNECTOR_NAME, true);
+        connectorsResource.restartConnector(CONNECTOR_NAME, NULL_HEADERS, true);
 
         PowerMock.verifyAll();
     }
@@ -664,7 +704,7 @@ public void testRestartTaskNotFound() throws Throwable {
 
         PowerMock.replayAll();
 
-        connectorsResource.restartTask(CONNECTOR_NAME, 0, FORWARD);
+        connectorsResource.restartTask(CONNECTOR_NAME, 0, NULL_HEADERS, FORWARD);
 
         PowerMock.verifyAll();
     }
@@ -678,12 +718,12 @@ public void testRestartTaskLeaderRedirect() throws Throwable {
         expectAndCallbackNotLeaderException(cb);
 
         EasyMock.expect(RestClient.httpRequest(EasyMock.eq("http://leader:8083/connectors/" + CONNECTOR_NAME + "/tasks/0/restart?forward=true"),
-                EasyMock.eq("POST"), EasyMock.isNull(), EasyMock.anyObject(), EasyMock.anyObject(WorkerConfig.class)))
+                EasyMock.eq("POST"), EasyMock.isNull(), EasyMock.isNull(), EasyMock.anyObject(), EasyMock.anyObject(WorkerConfig.class)))
                 .andReturn(new RestClient.HttpResponse<>(202, new HashMap(), null));
 
         PowerMock.replayAll();
 
-        connectorsResource.restartTask(CONNECTOR_NAME, 0, null);
+        connectorsResource.restartTask(CONNECTOR_NAME, 0, NULL_HEADERS, null);
 
         PowerMock.verifyAll();
     }
@@ -698,12 +738,12 @@ public void testRestartTaskOwnerRedirect() throws Throwable {
         expectAndCallbackException(cb, new NotAssignedException("not owner test", ownerUrl));
 
         EasyMock.expect(RestClient.httpRequest(EasyMock.eq("http://owner:8083/connectors/" + CONNECTOR_NAME + "/tasks/0/restart?forward=false"),
-                EasyMock.eq("POST"), EasyMock.isNull(), EasyMock.anyObject(), EasyMock.anyObject(WorkerConfig.class)))
+                EasyMock.eq("POST"), EasyMock.isNull(), EasyMock.isNull(), EasyMock.anyObject(), EasyMock.anyObject(WorkerConfig.class)))
                 .andReturn(new RestClient.HttpResponse<>(202, new HashMap(), null));
 
         PowerMock.replayAll();
 
-        connectorsResource.restartTask(CONNECTOR_NAME, 0, true);
+        connectorsResource.restartTask(CONNECTOR_NAME, 0, NULL_HEADERS, true);
 
         PowerMock.verifyAll();
     }

From ba3dc494371145e8ad35d6b85f45b8fe1e44c21f Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Mon, 3 Jun 2019 21:09:58 -0700
Subject: [PATCH 0328/1071] KAFKA-8155: Add 2.2.0 release to system tests
 (#6597)

Reviewers: Bill Bejeck , Boyang Chen , Bruno Cadonna , Guozhang Wang 
---
 build.gradle                                  | 12 +++
 gradle/dependencies.gradle                    |  2 +
 settings.gradle                               |  1 +
 .../streams/tests/StreamsUpgradeTest.java     |  5 +-
 .../streams/tests/StreamsUpgradeTest.java     |  5 +-
 .../streams/tests/StreamsUpgradeTest.java     |  5 +-
 .../streams/tests/StreamsUpgradeTest.java     |  5 +-
 .../streams/tests/StreamsUpgradeTest.java     |  3 +-
 .../streams/tests/StreamsUpgradeTest.java     |  3 +-
 .../streams/tests/StreamsUpgradeTest.java     | 90 +++++++++++++++++++
 tests/docker/Dockerfile                       |  1 +
 tests/kafkatest/services/streams.py           |  5 +-
 .../tests/streams/streams_upgrade_test.py     |  6 +-
 13 files changed, 119 insertions(+), 24 deletions(-)
 create mode 100644 streams/upgrade-system-tests-22/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java

diff --git a/build.gradle b/build.gradle
index 516a06cfed2b1..42cacb0be02c8 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1363,6 +1363,18 @@ project(':streams:upgrade-system-tests-21') {
   }
 }
 
+project(':streams:upgrade-system-tests-22') {
+  archivesBaseName = "kafka-streams-upgrade-system-tests-22"
+
+  dependencies {
+    testCompile libs.kafkaStreams_22
+  }
+
+  systemTestLibs {
+    dependsOn testJar
+  }
+}
+
 project(':jmh-benchmarks') {
 
   apply plugin: 'com.github.johnrengelman.shadow'
diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle
index d58d067efd96f..c912deff660de 100644
--- a/gradle/dependencies.gradle
+++ b/gradle/dependencies.gradle
@@ -79,6 +79,7 @@ versions += [
   kafka_11: "1.1.1",
   kafka_20: "2.0.1",
   kafka_21: "2.1.1",
+  kafka_22: "2.2.0",
   lz4: "1.6.0",
   mavenArtifact: "3.6.1",
   metrics: "2.2.0",
@@ -143,6 +144,7 @@ libs += [
   kafkaStreams_11: "org.apache.kafka:kafka-streams:$versions.kafka_11",
   kafkaStreams_20: "org.apache.kafka:kafka-streams:$versions.kafka_20",
   kafkaStreams_21: "org.apache.kafka:kafka-streams:$versions.kafka_21",
+  kafkaStreams_22: "org.apache.kafka:kafka-streams:$versions.kafka_22",
   log4j: "log4j:log4j:$versions.log4j",
   lz4: "org.lz4:lz4-java:$versions.lz4",
   metrics: "com.yammer.metrics:metrics-core:$versions.metrics",
diff --git a/settings.gradle b/settings.gradle
index b2814b1844dc0..c7e51e2300db7 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -37,4 +37,5 @@ include 'clients',
     'streams:upgrade-system-tests-11',
     'streams:upgrade-system-tests-20',
     'streams:upgrade-system-tests-21',
+    'streams:upgrade-system-tests-22',
     'tools'
diff --git a/streams/upgrade-system-tests-0102/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java b/streams/upgrade-system-tests-0102/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
index c113bd40f9049..fa855521a64fb 100644
--- a/streams/upgrade-system-tests-0102/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
+++ b/streams/upgrade-system-tests-0102/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
@@ -30,13 +30,10 @@
 
 public class StreamsUpgradeTest {
 
-    /**
-     * This test cannot be executed, as long as Kafka 0.10.2.2 is not released
-     */
     @SuppressWarnings("unchecked")
     public static void main(final String[] args) throws Exception {
         if (args.length < 2) {
-            System.err.println("StreamsUpgradeTest requires three argument (kafka-url, properties-file) but only " + args.length + " provided: "
+            System.err.println("StreamsUpgradeTest requires two argument (kafka-url, properties-file) but only " + args.length + " provided: "
                 + (args.length > 0 ? args[0] : ""));
         }
         final String kafka = args[0];
diff --git a/streams/upgrade-system-tests-0110/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java b/streams/upgrade-system-tests-0110/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
index 7df30a7251aeb..efb96ff5f0da7 100644
--- a/streams/upgrade-system-tests-0110/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
+++ b/streams/upgrade-system-tests-0110/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
@@ -30,13 +30,10 @@
 
 public class StreamsUpgradeTest {
 
-    /**
-     * This test cannot be executed, as long as Kafka 0.11.0.3 is not released
-     */
     @SuppressWarnings("unchecked")
     public static void main(final String[] args) throws Exception {
         if (args.length < 2) {
-            System.err.println("StreamsUpgradeTest requires three argument (kafka-url, properties-file) but only " + args.length + " provided: "
+            System.err.println("StreamsUpgradeTest requires two argument (kafka-url, properties-file) but only " + args.length + " provided: "
                 + (args.length > 0 ? args[0] : ""));
         }
         final String kafka = args[0];
diff --git a/streams/upgrade-system-tests-10/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java b/streams/upgrade-system-tests-10/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
index 163b94b57f301..6f63f8dd7558e 100644
--- a/streams/upgrade-system-tests-10/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
+++ b/streams/upgrade-system-tests-10/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
@@ -30,13 +30,10 @@
 
 public class StreamsUpgradeTest {
 
-    /**
-     * This test cannot be executed, as long as Kafka 1.0.2 is not released
-     */
     @SuppressWarnings("unchecked")
     public static void main(final String[] args) throws Exception {
         if (args.length < 2) {
-            System.err.println("StreamsUpgradeTest requires three argument (kafka-url, properties-file) but only " + args.length + " provided: "
+            System.err.println("StreamsUpgradeTest requires two argument (kafka-url, properties-file) but only " + args.length + " provided: "
                 + (args.length > 0 ? args[0] : ""));
         }
         final String kafka = args[0];
diff --git a/streams/upgrade-system-tests-11/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java b/streams/upgrade-system-tests-11/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
index 5f4c2187af064..b5759f56a5a28 100644
--- a/streams/upgrade-system-tests-11/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
+++ b/streams/upgrade-system-tests-11/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
@@ -30,13 +30,10 @@
 
 public class StreamsUpgradeTest {
 
-    /**
-     * This test cannot be executed, as long as Kafka 1.1.1 is not released
-     */
     @SuppressWarnings("unchecked")
     public static void main(final String[] args) throws Exception {
         if (args.length < 2) {
-            System.err.println("StreamsUpgradeTest requires three argument (kafka-url, properties-file) but only " + args.length + " provided: "
+            System.err.println("StreamsUpgradeTest requires two argument (kafka-url, properties-file) but only " + args.length + " provided: "
                 + (args.length > 0 ? args[0] : ""));
         }
         final String kafka = args[0];
diff --git a/streams/upgrade-system-tests-20/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java b/streams/upgrade-system-tests-20/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
index 0b4939c99f4cf..d963e4a1f65c1 100644
--- a/streams/upgrade-system-tests-20/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
+++ b/streams/upgrade-system-tests-20/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
@@ -29,11 +29,10 @@
 
 public class StreamsUpgradeTest {
 
-
     @SuppressWarnings("unchecked")
     public static void main(final String[] args) throws Exception {
         if (args.length < 2) {
-            System.err.println("StreamsUpgradeTest requires three argument (kafka-url, properties-file) but only " + args.length + " provided: "
+            System.err.println("StreamsUpgradeTest requires two argument (kafka-url, properties-file) but only " + args.length + " provided: "
                 + (args.length > 0 ? args[0] : ""));
         }
         final String kafka = args[0];
diff --git a/streams/upgrade-system-tests-21/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java b/streams/upgrade-system-tests-21/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
index 797c1e83f9693..6e409a00658b0 100644
--- a/streams/upgrade-system-tests-21/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
+++ b/streams/upgrade-system-tests-21/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
@@ -29,11 +29,10 @@
 
 public class StreamsUpgradeTest {
 
-
     @SuppressWarnings("unchecked")
     public static void main(final String[] args) throws Exception {
         if (args.length < 2) {
-            System.err.println("StreamsUpgradeTest requires three argument (kafka-url, properties-file) but only " + args.length + " provided: "
+            System.err.println("StreamsUpgradeTest requires two argument (kafka-url, properties-file) but only " + args.length + " provided: "
                 + (args.length > 0 ? args[0] : ""));
         }
         final String kafka = args[0];
diff --git a/streams/upgrade-system-tests-22/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java b/streams/upgrade-system-tests-22/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
new file mode 100644
index 0000000000000..7ff4d815406e1
--- /dev/null
+++ b/streams/upgrade-system-tests-22/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.tests;
+
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.kstream.KStream;
+import org.apache.kafka.streams.processor.AbstractProcessor;
+import org.apache.kafka.streams.processor.ProcessorContext;
+import org.apache.kafka.streams.processor.ProcessorSupplier;
+
+import java.util.Properties;
+
+public class StreamsUpgradeTest {
+
+    @SuppressWarnings("unchecked")
+    public static void main(final String[] args) throws Exception {
+        if (args.length < 2) {
+            System.err.println("StreamsUpgradeTest requires two argument (kafka-url, properties-file) but only " + args.length + " provided: "
+                + (args.length > 0 ? args[0] : ""));
+        }
+        final String kafka = args[0];
+        final String propFileName = args.length > 1 ? args[1] : null;
+
+        final Properties streamsProperties = Utils.loadProps(propFileName);
+
+        System.out.println("StreamsTest instance started (StreamsUpgradeTest v2.2)");
+        System.out.println("kafka=" + kafka);
+        System.out.println("props=" + streamsProperties);
+
+        final StreamsBuilder builder = new StreamsBuilder();
+        final KStream dataStream = builder.stream("data");
+        dataStream.process(printProcessorSupplier());
+        dataStream.to("echo");
+
+        final Properties config = new Properties();
+        config.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "StreamsUpgradeTest");
+        config.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, kafka);
+        config.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000);
+        config.putAll(streamsProperties);
+
+        final KafkaStreams streams = new KafkaStreams(builder.build(), config);
+        streams.start();
+
+        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            streams.close();
+            System.out.println("UPGRADE-TEST-CLIENT-CLOSED");
+            System.out.flush();
+        }));
+    }
+
+    private static  ProcessorSupplier printProcessorSupplier() {
+        return () -> new AbstractProcessor() {
+            private int numRecordsProcessed = 0;
+
+            @Override
+            public void init(final ProcessorContext context) {
+                System.out.println("[2.2] initializing processor: topic=data taskId=" + context.taskId());
+                numRecordsProcessed = 0;
+            }
+
+            @Override
+            public void process(final K key, final V value) {
+                numRecordsProcessed++;
+                if (numRecordsProcessed % 100 == 0) {
+                    System.out.println("processed " + numRecordsProcessed + " records from topic=data");
+                }
+            }
+
+            @Override
+            public void close() {}
+        };
+    }
+}
diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile
index 665dc270405cf..82188231e3e84 100644
--- a/tests/docker/Dockerfile
+++ b/tests/docker/Dockerfile
@@ -65,6 +65,7 @@ RUN curl -s "$KAFKA_MIRROR/kafka-streams-1.0.2-test.jar" -o /opt/kafka-1.0.2/lib
 RUN curl -s "$KAFKA_MIRROR/kafka-streams-1.1.1-test.jar" -o /opt/kafka-1.1.1/libs/kafka-streams-1.1.1-test.jar
 RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.0.1-test.jar" -o /opt/kafka-2.0.1/libs/kafka-streams-2.0.1-test.jar
 RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.1.1-test.jar" -o /opt/kafka-2.1.1/libs/kafka-streams-2.1.1-test.jar
+RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.2.0-test.jar" -o /opt/kafka-2.2.0/libs/kafka-streams-2.2.0-test.jar
 
 # The version of Kibosh to use for testing.
 # If you update this, also update vagrant/base.sy
diff --git a/tests/kafkatest/services/streams.py b/tests/kafkatest/services/streams.py
index b606267009d14..5e2c2e9e00c70 100644
--- a/tests/kafkatest/services/streams.py
+++ b/tests/kafkatest/services/streams.py
@@ -21,7 +21,8 @@
 from kafkatest.directory_layout.kafka_path import KafkaPathResolverMixin
 from kafkatest.services.kafka import KafkaConfig
 from kafkatest.services.monitor.jmx import JmxMixin
-from kafkatest.version import LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, LATEST_2_0, LATEST_2_1
+from kafkatest.version import LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1,\
+    LATEST_2_0, LATEST_2_1, LATEST_2_2
 
 STATE_DIR = "state.dir"
 
@@ -487,7 +488,7 @@ def start_cmd(self, node):
         args = self.args.copy()
         if self.KAFKA_STREAMS_VERSION in [str(LATEST_0_10_0), str(LATEST_0_10_1), str(LATEST_0_10_2),
                                           str(LATEST_0_11_0), str(LATEST_1_0), str(LATEST_1_1),
-                                          str(LATEST_2_0), str(LATEST_2_1)]:
+                                          str(LATEST_2_0), str(LATEST_2_1), str(LATEST_2_2)]:
             args['kafka'] = self.kafka.bootstrap_servers()
         else:
             args['kafka'] = ""
diff --git a/tests/kafkatest/tests/streams/streams_upgrade_test.py b/tests/kafkatest/tests/streams/streams_upgrade_test.py
index d7a454859667a..37ab770befed1 100644
--- a/tests/kafkatest/tests/streams/streams_upgrade_test.py
+++ b/tests/kafkatest/tests/streams/streams_upgrade_test.py
@@ -24,7 +24,7 @@
     StreamsUpgradeTestJobRunnerService
 from kafkatest.services.zookeeper import ZookeeperService
 from kafkatest.version import LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, \
-    LATEST_2_0, LATEST_2_1, DEV_BRANCH, DEV_VERSION, KafkaVersion
+    LATEST_2_0, LATEST_2_1, LATEST_2_2, DEV_BRANCH, DEV_VERSION, KafkaVersion
 
 # broker 0.10.0 is not compatible with newer Kafka Streams versions
 broker_upgrade_versions = [str(LATEST_0_10_1), str(LATEST_0_10_2), str(LATEST_0_11_0), str(LATEST_1_0), str(LATEST_1_1), str(LATEST_2_0), str(LATEST_2_1), str(DEV_BRANCH)]
@@ -34,7 +34,9 @@
 # once 0.10.1.2 is available backward_compatible_metadata_2_versions
 # can be replaced with metadata_2_versions
 backward_compatible_metadata_2_versions = [str(LATEST_0_10_2), str(LATEST_0_11_0), str(LATEST_1_0), str(LATEST_1_1)]
-metadata_3_or_higher_versions = [str(LATEST_2_0), str(LATEST_2_1), str(DEV_VERSION)]
+# If we add a new version below, we also need to add this version to `streams.py`:
+# -> class `StreamsUpgradeTestJobRunnerService`, method `start_cmd`, variable `KAFKA_STREAMS_VERSION`
+metadata_3_or_higher_versions = [str(LATEST_2_0), str(LATEST_2_1), str(LATEST_2_2), str(DEV_VERSION)]
 
 """
 After each release one should first check that the released version has been uploaded to 

From 055c9c7bd6780436d201f5101df464814ba940e2 Mon Sep 17 00:00:00 2001
From: Boyang Chen 
Date: Tue, 4 Jun 2019 12:14:35 -0700
Subject: [PATCH 0329/1071] KAFKA 8311: better handle timeout exception on
 Stream thread (#6662)

The goals for this small diff are:

1. Give user guidance if they want to relax commit timeout threshold
2. Indicate the code path where timeout exception was caught

Reviewers: John Roesler , Guozhang Wang 
---
 .../java/org/apache/kafka/clients/consumer/KafkaConsumer.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index 01b8989e7c338..5be065d5f2621 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -1770,7 +1770,8 @@ public OffsetAndMetadata committed(TopicPartition partition, final Duration time
                     Collections.singleton(partition), time.timer(timeout));
             if (offsets == null) {
                 throw new TimeoutException("Timeout of " + timeout.toMillis() + "ms expired before the last " +
-                        "committed offset for partition " + partition + " could be determined");
+                        "committed offset for partition " + partition + " could be determined. Try tuning default.api.timeout.ms " +
+                        "larger to relax the threshold.");
             } else {
                 offsets.forEach(this::updateLastSeenEpochIfNewer);
                 return offsets.get(partition);

From e6563aab722b35c4984b77e9eee42a1904cd1ea6 Mon Sep 17 00:00:00 2001
From: "Colin P. Mccabe" 
Date: Tue, 4 Jun 2019 14:14:32 -0700
Subject: [PATCH 0330/1071] KAFKA-4893; Fix deletion and moving of topics with
 long names

Author: Colin P. Mccabe 

Reviewers: Gwen Shapira, David Arthur, James Cheng, Vahid Hashemian

Closes #6869 from cmccabe/KAFKA-4893
---
 core/src/main/scala/kafka/log/Log.scala       | 15 ++++++++++-----
 .../scala/kafka/server/ReplicaManager.scala   |  8 +++++++-
 .../api/AdminClientIntegrationTest.scala      | 18 ++++++++++++++++++
 .../scala/unit/kafka/log/LogManagerTest.scala |  9 ++++++++-
 .../test/scala/unit/kafka/log/LogTest.scala   | 19 ++++++++++++++++++-
 5 files changed, 61 insertions(+), 8 deletions(-)

diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index 56b2969e2bf2f..9ab6fda30924f 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -2180,8 +2180,8 @@ object Log {
   /** a directory that is used for future partition */
   val FutureDirSuffix = "-future"
 
-  private val DeleteDirPattern = Pattern.compile(s"^(\\S+)-(\\S+)\\.(\\S+)$DeleteDirSuffix")
-  private val FutureDirPattern = Pattern.compile(s"^(\\S+)-(\\S+)\\.(\\S+)$FutureDirSuffix")
+  private[log] val DeleteDirPattern = Pattern.compile(s"^(\\S+)-(\\S+)\\.(\\S+)$DeleteDirSuffix")
+  private[log] val FutureDirPattern = Pattern.compile(s"^(\\S+)-(\\S+)\\.(\\S+)$FutureDirSuffix")
 
   val UnknownLogStartOffset = -1L
 
@@ -2227,11 +2227,16 @@ object Log {
     new File(dir, filenamePrefixFromOffset(offset) + LogFileSuffix + suffix)
 
   /**
-   * Return a directory name to rename the log directory to for async deletion. The name will be in the following
-   * format: topic-partition.uniqueId-delete where topic, partition and uniqueId are variables.
+   * Return a directory name to rename the log directory to for async deletion.
+   * The name will be in the following format: "topic-partitionId.uniqueId-delete".
+   * If the topic name is too long, it will be truncated to prevent the total name
+   * from exceeding 255 characters.
    */
   def logDeleteDirName(topicPartition: TopicPartition): String = {
-    logDirNameWithSuffix(topicPartition, DeleteDirSuffix)
+    val uniqueId = java.util.UUID.randomUUID.toString.replaceAll("-", "")
+    val suffix = s"-${topicPartition.partition()}.${uniqueId}${DeleteDirSuffix}"
+    val prefixLength = Math.min(topicPartition.topic().size, 255 - suffix.size)
+    s"${topicPartition.topic().substring(0, prefixLength)}${suffix}"
   }
 
   /**
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index 8cfa247f39f1f..e9ab7386101e8 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -578,6 +578,11 @@ class ReplicaManager(val config: KafkaConfig,
     replicaStateChangeLock synchronized {
       partitionDirs.map { case (topicPartition, destinationDir) =>
         try {
+          /* If the topic name is exceptionally long, we can't support altering the log directory.
+           * See KAFKA-4893 for details.
+           * TODO: fix this by implementing topic IDs. */
+          if (Log.logFutureDirName(topicPartition).size > 255)
+            throw new InvalidTopicException("The topic name is too long.")
           if (!logManager.isLogDirOnline(destinationDir))
             throw new KafkaStorageException(s"Log directory $destinationDir is offline")
 
@@ -621,7 +626,8 @@ class ReplicaManager(val config: KafkaConfig,
 
           (topicPartition, Errors.NONE)
         } catch {
-          case e@(_: LogDirNotFoundException |
+          case e@(_: InvalidTopicException |
+                  _: LogDirNotFoundException |
                   _: ReplicaNotAvailableException |
                   _: KafkaStorageException) =>
             (topicPartition, Errors.forException(e))
diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index 41451374d6ff1..88f10ff51720c 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -1793,6 +1793,24 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     assertFutureExceptionTypeEquals(alterResult.values().get(topic1Resource), classOf[InvalidRequestException],
       Some("Invalid config value for resource"))
   }
+
+  @Test
+  def testLongTopicNames(): Unit = {
+    val client = AdminClient.create(createConfig)
+    val longTopicName = String.join("", Collections.nCopies(249, "x"));
+    val invalidTopicName = String.join("", Collections.nCopies(250, "x"));
+    val newTopics2 = Seq(new NewTopic(invalidTopicName, 3, 3),
+                         new NewTopic(longTopicName, 3, 3))
+    val results = client.createTopics(newTopics2.asJava).values()
+    assertTrue(results.containsKey(longTopicName))
+    results.get(longTopicName).get()
+    assertTrue(results.containsKey(invalidTopicName))
+    assertFutureExceptionTypeEquals(results.get(invalidTopicName), classOf[InvalidTopicException])
+    assertFutureExceptionTypeEquals(client.alterReplicaLogDirs(
+      Map(new TopicPartitionReplica(longTopicName, 0, 0) -> servers(0).config.logDirs(0)).asJava).all(),
+      classOf[InvalidTopicException])
+    client.close()
+  }
 }
 
 object AdminClientIntegrationTest {
diff --git a/core/src/test/scala/unit/kafka/log/LogManagerTest.scala b/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
index b3ecd23a9cf58..3df09e7a54a94 100755
--- a/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
@@ -18,7 +18,7 @@
 package kafka.log
 
 import java.io._
-import java.util.Properties
+import java.util.{Collections, Properties}
 
 import kafka.server.FetchDataInfo
 import kafka.server.checkpoints.OffsetCheckpointFile
@@ -380,6 +380,13 @@ class LogManagerTest {
     assertFalse("Logs not deleted", logManager.hasLogsToBeDeleted)
   }
 
+  @Test
+  def testCreateAndDeleteOverlyLongTopic(): Unit = {
+    val invalidTopicName = String.join("", Collections.nCopies(253, "x"));
+    val log = logManager.getOrCreateLog(new TopicPartition(invalidTopicName, 0), logConfig)
+    logManager.asyncDelete(new TopicPartition(invalidTopicName, 0))
+  }
+
   @Test
   def testCheckpointForOnlyAffectedLogs() {
     val tps = Seq(
diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala
index 8635969108682..dc9a423e34556 100755
--- a/core/src/test/scala/unit/kafka/log/LogTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogTest.scala
@@ -20,7 +20,8 @@ package kafka.log
 import java.io._
 import java.nio.ByteBuffer
 import java.nio.file.{Files, Paths}
-import java.util.{Optional, Properties}
+import java.util.regex.Pattern
+import java.util.{Collections, Optional, Properties}
 
 import kafka.api.{ApiVersion, KAFKA_0_11_0_IV0}
 import kafka.common.{OffsetsOutOfOrderException, UnexpectedAppendOffsetException}
@@ -74,6 +75,22 @@ class LogTest {
     }
   }
 
+  @Test
+  def testLogDeleteDirName(): Unit = {
+    val name1 = Log.logDeleteDirName(new TopicPartition("foo", 3))
+    assertTrue(name1.length <= 255)
+    assertTrue(Pattern.compile("foo-3\\.[0-9a-z]{32}-delete").matcher(name1).matches())
+    assertTrue(Log.DeleteDirPattern.matcher(name1).matches())
+    assertFalse(Log.FutureDirPattern.matcher(name1).matches())
+    val name2 = Log.logDeleteDirName(
+      new TopicPartition("n" + String.join("", Collections.nCopies(248, "o")), 5))
+    System.out.println("name2 = " + name2)
+    assertEquals(255, name2.length)
+    assertTrue(Pattern.compile("n[o]{212}-5\\.[0-9a-z]{32}-delete").matcher(name2).matches())
+    assertTrue(Log.DeleteDirPattern.matcher(name2).matches())
+    assertFalse(Log.FutureDirPattern.matcher(name2).matches())
+  }
+
   @Test
   def testOffsetFromFile() {
     val offset = 23423423L

From 264d1d8a8b7ede0fb8e3595d0153893966bb73b8 Mon Sep 17 00:00:00 2001
From: David Arthur 
Date: Tue, 4 Jun 2019 17:49:30 -0400
Subject: [PATCH 0331/1071] Improve logging in the consumer for epoch updates
 (#6879)

---
 .../src/main/java/org/apache/kafka/clients/Metadata.java   | 2 +-
 .../main/java/org/apache/kafka/clients/MetadataCache.java  | 5 ++++-
 .../apache/kafka/clients/consumer/internals/Fetcher.java   | 7 ++++---
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/Metadata.java b/clients/src/main/java/org/apache/kafka/clients/Metadata.java
index f991fa64016ee..94e4eb3298772 100644
--- a/clients/src/main/java/org/apache/kafka/clients/Metadata.java
+++ b/clients/src/main/java/org/apache/kafka/clients/Metadata.java
@@ -177,7 +177,7 @@ private synchronized boolean updateLastSeenEpoch(TopicPartition topicPartition,
             }
             return true;
         } else {
-            log.debug("Not replacing existing epoch {} with new epoch {}", oldEpoch, epoch);
+            log.debug("Not replacing existing epoch {} with new epoch {} for partition {}", oldEpoch, epoch, topicPartition);
             return false;
         }
     }
diff --git a/clients/src/main/java/org/apache/kafka/clients/MetadataCache.java b/clients/src/main/java/org/apache/kafka/clients/MetadataCache.java
index b928b8e0ef731..b5c9de638a735 100644
--- a/clients/src/main/java/org/apache/kafka/clients/MetadataCache.java
+++ b/clients/src/main/java/org/apache/kafka/clients/MetadataCache.java
@@ -137,7 +137,10 @@ static MetadataCache empty() {
     @Override
     public String toString() {
         return "MetadataCache{" +
-                "cluster=" + cluster() +
+                "clusterId='" + clusterId + '\'' +
+                ", nodes=" + nodes +
+                ", partitions=" + metadataByPartition.values() +
+                ", controller=" + controller +
                 '}';
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
index e638963e9c835..cff6e30bc93a5 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
@@ -861,13 +861,14 @@ private Map> groupLis
         final Map partitionDataMap = new HashMap<>();
         for (Map.Entry entry: timestampsToSearch.entrySet()) {
             TopicPartition tp  = entry.getKey();
+            Long offset = entry.getValue();
             Optional currentInfo = metadata.partitionInfoIfCurrent(tp);
             if (!currentInfo.isPresent()) {
-                log.debug("Leader for partition {} is unknown for fetching offset", tp);
+                log.debug("Leader for partition {} is unknown for fetching offset {}", tp, offset);
                 metadata.requestUpdate();
                 partitionsToRetry.add(tp);
             } else if (currentInfo.get().partitionInfo().leader() == null) {
-                log.debug("Leader for partition {} is unavailable for fetching offset", tp);
+                log.debug("Leader for partition {} is unavailable for fetching offset {}", tp, offset);
                 metadata.requestUpdate();
                 partitionsToRetry.add(tp);
             } else if (client.isUnavailable(currentInfo.get().partitionInfo().leader())) {
@@ -881,7 +882,7 @@ private Map> groupLis
                 partitionsToRetry.add(tp);
             } else {
                 partitionDataMap.put(tp,
-                        new ListOffsetRequest.PartitionData(entry.getValue(), Optional.of(currentInfo.get().epoch())));
+                        new ListOffsetRequest.PartitionData(offset, Optional.of(currentInfo.get().epoch())));
             }
         }
         return regroupPartitionMapByNode(partitionDataMap);

From b6d9e152234b14031ff1851b659b7bdf209d4710 Mon Sep 17 00:00:00 2001
From: Colin Patrick McCabe 
Date: Tue, 4 Jun 2019 15:38:18 -0700
Subject: [PATCH 0332/1071] MINOR: Update docs to say 2.3 (#6881)

Jason Gustafson 
---
 docs/documentation.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/documentation.html b/docs/documentation.html
index 83c029bb3513b..8d790c1660ba2 100644
--- a/docs/documentation.html
+++ b/docs/documentation.html
@@ -26,8 +26,8 @@
 	
    
 		
     
    Documentation
    
-    
    Kafka 2.2 Documentation
    
-    Prior releases: 0.7.x, 0.8.0, 0.8.1.X, 0.8.2.X, 0.9.0.X, 0.10.0.X, 0.10.1.X, 0.10.2.X, 0.11.0.X, 1.0.X, 1.1.X, 2.0.X, 2.1.X.
+    
    Kafka 2.3 Documentation
    
+    Prior releases: 0.7.x, 0.8.0, 0.8.1.X, 0.8.2.X, 0.9.0.X, 0.10.0.X, 0.10.1.X, 0.10.2.X, 0.11.0.X, 1.0.X, 1.1.X, 2.0.X, 2.1.X, 2.2.X.
 
     
 

From 8e161580b859b2fcd54c59625e232b99f3bb48d0 Mon Sep 17 00:00:00 2001
From: Almog Gavra 
Date: Wed, 5 Jun 2019 14:10:00 -0700
Subject: [PATCH 0333/1071] KAFKA-8305; Support default partitions &
 replication factor in AdminClient#createTopic (KIP-464) (#6728)

This commit makes three changes:
- Adds a constructor for NewTopic(String, Optional, Optional)
which allows users to specify Optional.empty() for numPartitions or
replicationFactor in order to use the broker default.
- Changes AdminManager to accept -1 as valid options for replication
factor and numPartitions (resolving to broker defaults).
- Makes --partitions and --replication-factor optional arguments when creating
topics using kafka-topics.sh.
- Adds a dependency on scalaJava8Compat library to make it simpler to
convert Scala Option to Java Optional

Reviewers: Ismael Juma , Ryanne Dolan , Jason Gustafson 
---
 build.gradle                                  |   2 +
 .../apache/kafka/clients/admin/NewTopic.java  |  34 ++++--
 .../common/requests/CreateTopicsRequest.java  |  22 +++-
 .../common/message/CreateTopicsRequest.json   |   7 +-
 .../common/message/CreateTopicsResponse.json  |   3 +-
 .../common/requests/RequestResponseTest.java  |  24 +++++
 .../main/scala/kafka/admin/TopicCommand.scala |  28 +++--
 .../scala/kafka/server/AdminManager.scala     |  16 ++-
 .../api/AdminClientIntegrationTest.scala      |  36 ++++---
 .../kafka/api/ConsumerTopicCreationTest.scala |   2 +-
 .../kafka/tools/LogCompactionTester.scala     |   2 +-
 .../TopicCommandWithAdminClientTest.scala     | 101 +++++++++++++-----
 .../AbstractCreateTopicsRequestTest.scala     |  14 ++-
 .../server/CreateTopicsRequestTest.scala      |  11 +-
 .../CreateTopicsRequestWithPolicyTest.scala   |   7 +-
 .../scala/unit/kafka/utils/TestUtils.scala    |  14 ++-
 docs/upgrade.html                             |   1 +
 gradle/dependencies.gradle                    |   2 +
 18 files changed, 248 insertions(+), 78 deletions(-)

diff --git a/build.gradle b/build.gradle
index 42cacb0be02c8..d1a2a9da52ba7 100644
--- a/build.gradle
+++ b/build.gradle
@@ -939,6 +939,8 @@ project(':clients') {
     compile libs.lz4
     compile libs.snappy
     compile libs.slf4jApi
+    compile libs.scalaJava8Compat
+
     compileOnly libs.jacksonDatabind // for SASL/OAUTHBEARER bearer token parsing
     compileOnly libs.jacksonJDK8Datatypes
 
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/NewTopic.java b/clients/src/main/java/org/apache/kafka/clients/admin/NewTopic.java
index 5b1bd32f89acb..66585eac32685 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/NewTopic.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/NewTopic.java
@@ -17,6 +17,7 @@
 
 package org.apache.kafka.clients.admin;
 
+import java.util.Optional;
 import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignment;
 import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic;
 import org.apache.kafka.common.message.CreateTopicsRequestData.CreateableTopicConfig;
@@ -31,9 +32,13 @@
  * A new topic to be created via {@link AdminClient#createTopics(Collection)}.
  */
 public class NewTopic {
+
+    private static final int NO_PARTITIONS = -1;
+    private static final short NO_REPLICATION_FACTOR = -1;
+
     private final String name;
-    private final int numPartitions;
-    private final short replicationFactor;
+    private final Optional numPartitions;
+    private final Optional replicationFactor;
     private final Map> replicasAssignments;
     private Map configs = null;
 
@@ -41,6 +46,15 @@ public class NewTopic {
      * A new topic with the specified replication factor and number of partitions.
      */
     public NewTopic(String name, int numPartitions, short replicationFactor) {
+        this(name, Optional.of(numPartitions), Optional.of(replicationFactor));
+    }
+
+    /**
+     * A new topic that optionally defaults {@code numPartitions} and {@code replicationFactor} to
+     * the broker configurations for {@code num.partitions} and {@code default.replication.factor}
+     * respectively.
+     */
+    public NewTopic(String name, Optional numPartitions, Optional replicationFactor) {
         this.name = name;
         this.numPartitions = numPartitions;
         this.replicationFactor = replicationFactor;
@@ -56,8 +70,8 @@ public NewTopic(String name, int numPartitions, short replicationFactor) {
      */
     public NewTopic(String name, Map> replicasAssignments) {
         this.name = name;
-        this.numPartitions = -1;
-        this.replicationFactor = -1;
+        this.numPartitions = Optional.empty();
+        this.replicationFactor = Optional.empty();
         this.replicasAssignments = Collections.unmodifiableMap(replicasAssignments);
     }
 
@@ -72,14 +86,14 @@ public String name() {
      * The number of partitions for the new topic or -1 if a replica assignment has been specified.
      */
     public int numPartitions() {
-        return numPartitions;
+        return numPartitions.orElse(NO_PARTITIONS);
     }
 
     /**
      * The replication factor for the new topic or -1 if a replica assignment has been specified.
      */
     public short replicationFactor() {
-        return replicationFactor;
+        return replicationFactor.orElse(NO_REPLICATION_FACTOR);
     }
 
     /**
@@ -111,8 +125,8 @@ public Map configs() {
     CreatableTopic convertToCreatableTopic() {
         CreatableTopic creatableTopic = new CreatableTopic().
             setName(name).
-            setNumPartitions(numPartitions).
-            setReplicationFactor(replicationFactor);
+            setNumPartitions(numPartitions.orElse(NO_PARTITIONS)).
+            setReplicationFactor(replicationFactor.orElse(NO_REPLICATION_FACTOR));
         if (replicasAssignments != null) {
             for (Entry> entry : replicasAssignments.entrySet()) {
                 creatableTopic.assignments().add(
@@ -136,8 +150,8 @@ CreatableTopic convertToCreatableTopic() {
     public String toString() {
         StringBuilder bld = new StringBuilder();
         bld.append("(name=").append(name).
-                append(", numPartitions=").append(numPartitions).
-                append(", replicationFactor=").append(replicationFactor).
+                append(", numPartitions=").append(numPartitions.map(String::valueOf).orElse("default")).
+                append(", replicationFactor=").append(replicationFactor.map(String::valueOf).orElse("default")).
                 append(", replicasAssignments=").append(replicasAssignments).
                 append(", configs=").append(configs).
                 append(")");
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java
index a2cd17d9c7ad6..dd26e5642632e 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java
@@ -16,6 +16,9 @@
  */
 package org.apache.kafka.common.requests;
 
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.stream.Collectors;
 import org.apache.kafka.common.errors.UnsupportedVersionException;
 import org.apache.kafka.common.message.CreateTopicsRequestData;
 import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic;
@@ -24,8 +27,6 @@
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.types.Struct;
 
-import java.nio.ByteBuffer;
-
 public class CreateTopicsRequest extends AbstractRequest {
     public static class Builder extends AbstractRequest.Builder {
         private final CreateTopicsRequestData data;
@@ -40,6 +41,23 @@ public CreateTopicsRequest build(short version) {
             if (data.validateOnly() && version == 0)
                 throw new UnsupportedVersionException("validateOnly is not supported in version 0 of " +
                         "CreateTopicsRequest");
+
+            final List topicsWithDefaults = data.topics()
+                .stream()
+                .filter(topic -> topic.assignments().isEmpty())
+                .filter(topic ->
+                    topic.numPartitions() == CreateTopicsRequest.NO_NUM_PARTITIONS
+                        || topic.replicationFactor() == CreateTopicsRequest.NO_REPLICATION_FACTOR)
+                .map(CreatableTopic::name)
+                .collect(Collectors.toList());
+
+            if (!topicsWithDefaults.isEmpty() && version < 4) {
+                throw new UnsupportedVersionException("Creating topics with default "
+                    + "partitions/replication factor are only supported in CreateTopicRequest "
+                    + "version 4+. The following topics need values for partitions and replicas: "
+                    + topicsWithDefaults);
+            }
+
             return new CreateTopicsRequest(data, version);
         }
 
diff --git a/clients/src/main/resources/common/message/CreateTopicsRequest.json b/clients/src/main/resources/common/message/CreateTopicsRequest.json
index 842fb204ceff7..d2285668f4cba 100644
--- a/clients/src/main/resources/common/message/CreateTopicsRequest.json
+++ b/clients/src/main/resources/common/message/CreateTopicsRequest.json
@@ -18,16 +18,17 @@
   "type": "request",
   "name": "CreateTopicsRequest",
   // Version 1 adds validateOnly.
-  "validVersions": "0-3",
+  // Version 4 makes partitions/replicationFactor optional even when assignments are not present (KIP-464)
+  "validVersions": "0-4",
   "fields": [
     { "name": "Topics", "type": "[]CreatableTopic", "versions": "0+",
       "about": "The topics to create.", "fields": [
       { "name": "Name", "type": "string", "versions": "0+", "mapKey": true, "entityType": "topicName",
         "about": "The topic name." },
       { "name": "NumPartitions", "type": "int32", "versions": "0+",
-        "about": "The number of partitions to create in the topic, or -1 if we are specifying a manual partition assignment." },
+        "about": "The number of partitions to create in the topic, or -1 if we are either specifying a manual partition assignment or using the default partitions." },
       { "name": "ReplicationFactor", "type": "int16", "versions": "0+",
-        "about": "The number of replicas to create for each partition in the topic, or -1 if we are specifying a manual partition assignment." },
+        "about": "The number of replicas to create for each partition in the topic, or -1 if we are either specifying a manual partition assignment or using the default replication factor." },
       { "name": "Assignments", "type": "[]CreatableReplicaAssignment", "versions": "0+",
         "about": "The manual partition assignment, or the empty array if we are using automatic assignment.", "fields": [
         { "name": "PartitionIndex", "type": "int32", "versions": "0+", "mapKey": true,
diff --git a/clients/src/main/resources/common/message/CreateTopicsResponse.json b/clients/src/main/resources/common/message/CreateTopicsResponse.json
index 864e5fa36510f..d56e642061e43 100644
--- a/clients/src/main/resources/common/message/CreateTopicsResponse.json
+++ b/clients/src/main/resources/common/message/CreateTopicsResponse.json
@@ -20,7 +20,8 @@
   // Version 1 adds a per-topic error message string.
   // Version 2 adds the throttle time.
   // Starting in version 3, on quota violation, brokers send out responses before throttling.
-  "validVersions": "0-3",
+  // Version 4 makes partitions/replicationFactor optional even when assignments are not present (KIP-464)
+  "validVersions": "0-4",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "2+", "ignorable": true,
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index e06202bb2f683..11d85d6d7cf8a 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -82,6 +82,7 @@
 import org.apache.kafka.common.requests.CreateAclsRequest.AclCreation;
 import org.apache.kafka.common.requests.CreateAclsResponse.AclCreationResponse;
 import org.apache.kafka.common.requests.CreatePartitionsRequest.PartitionDetails;
+import org.apache.kafka.common.requests.CreateTopicsRequest.Builder;
 import org.apache.kafka.common.requests.DeleteAclsResponse.AclDeletionResult;
 import org.apache.kafka.common.requests.DeleteAclsResponse.AclFilterResponse;
 import org.apache.kafka.common.requests.FindCoordinatorRequest.CoordinatorType;
@@ -120,6 +121,7 @@
 import static org.apache.kafka.test.TestUtils.toBuffer;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -644,6 +646,28 @@ public void testCreateTopicRequestV0FailsIfValidateOnly() {
         createCreateTopicRequest(0, true);
     }
 
+    @Test
+    public void testCreateTopicRequestV3FailsIfNoPartitionsOrReplicas() {
+        final UnsupportedVersionException exception = assertThrows(
+            UnsupportedVersionException.class, () -> {
+                CreateTopicsRequestData data = new CreateTopicsRequestData()
+                    .setTimeoutMs(123)
+                    .setValidateOnly(false);
+                data.topics().add(new CreatableTopic().
+                    setName("foo").
+                    setNumPartitions(CreateTopicsRequest.NO_NUM_PARTITIONS).
+                    setReplicationFactor((short) 1));
+                data.topics().add(new CreatableTopic().
+                    setName("bar").
+                    setNumPartitions(1).
+                    setReplicationFactor(CreateTopicsRequest.NO_REPLICATION_FACTOR));
+
+                new Builder(data).build((short) 3);
+            });
+        assertTrue(exception.getMessage().contains("supported in CreateTopicRequest version 4+"));
+        assertTrue(exception.getMessage().contains("[foo, bar]"));
+    }
+
     @Test
     public void testFetchRequestMaxBytesOldVersions() throws Exception {
         final short version = 1;
diff --git a/core/src/main/scala/kafka/admin/TopicCommand.scala b/core/src/main/scala/kafka/admin/TopicCommand.scala
index 996f73bbfb05f..5042fa27fca20 100755
--- a/core/src/main/scala/kafka/admin/TopicCommand.scala
+++ b/core/src/main/scala/kafka/admin/TopicCommand.scala
@@ -30,8 +30,8 @@ import kafka.zk.{AdminZkClient, KafkaZkClient}
 import org.apache.kafka.clients.CommonClientConfigs
 import org.apache.kafka.clients.admin.{ListTopicsOptions, NewPartitions, NewTopic, AdminClient => JAdminClient}
 import org.apache.kafka.common.TopicPartition
-import org.apache.kafka.common.config.{ConfigResource, TopicConfig}
 import org.apache.kafka.common.config.ConfigResource.Type
+import org.apache.kafka.common.config.{ConfigResource, TopicConfig}
 import org.apache.kafka.common.errors.{InvalidTopicException, TopicExistsException}
 import org.apache.kafka.common.internals.Topic
 import org.apache.kafka.common.security.JaasUtils
@@ -40,6 +40,7 @@ import org.apache.zookeeper.KeeperException.NodeExistsException
 
 import scala.collection.JavaConverters._
 import scala.collection._
+import scala.compat.java8.OptionConverters._
 import scala.io.StdIn
 
 object TopicCommand extends Logging {
@@ -82,7 +83,7 @@ object TopicCommand extends Logging {
   class CommandTopicPartition(opts: TopicCommandOptions) {
     val name: String = opts.topic.get
     val partitions: Option[Integer] = opts.partitions
-    val replicationFactor: Integer = opts.replicationFactor.getOrElse(-1)
+    val replicationFactor: Option[Integer] = opts.replicationFactor
     val replicaAssignment: Option[Map[Int, List[Int]]] = opts.replicaAssignment
     val configsToAdd: Properties = parseTopicConfigsToBeAdded(opts)
     val configsToDelete: Seq[String] = parseTopicConfigsToBeDeleted(opts)
@@ -172,14 +173,21 @@ object TopicCommand extends Logging {
   case class AdminClientTopicService private (adminClient: JAdminClient) extends TopicService {
 
     override def createTopic(topic: CommandTopicPartition): Unit = {
-      if (topic.replicationFactor > Short.MaxValue)
-        throw new IllegalArgumentException(s"The replication factor's maximum value must be smaller or equal to ${Short.MaxValue}")
+      if (topic.replicationFactor.exists(rf => rf > Short.MaxValue || rf < 1))
+        throw new IllegalArgumentException(s"The replication factor must be between 1 and ${Short.MaxValue} inclusive")
+      if (topic.partitions.exists(partitions => partitions < 1))
+        throw new IllegalArgumentException(s"The partitions must be greater than 0")
 
       if (!adminClient.listTopics().names().get().contains(topic.name)) {
         val newTopic = if (topic.hasReplicaAssignment)
           new NewTopic(topic.name, asJavaReplicaReassignment(topic.replicaAssignment.get))
-        else
-          new NewTopic(topic.name, topic.partitions.get, topic.replicationFactor.shortValue())
+        else {
+          new NewTopic(
+            topic.name,
+            topic.partitions.asJava,
+            topic.replicationFactor.map(_.toShort).map(Short.box).asJava)
+        }
+
         val configsMap = topic.configsToAdd.stringPropertyNames()
           .asScala
           .map(name => name -> topic.configsToAdd.getProperty(name))
@@ -289,7 +297,7 @@ object TopicCommand extends Logging {
         if (topic.hasReplicaAssignment)
           adminZkClient.createTopicWithAssignment(topic.name, topic.configsToAdd, topic.replicaAssignment.get)
         else
-          adminZkClient.createTopic(topic.name, topic.partitions.get, topic.replicationFactor, topic.configsToAdd, topic.rackAwareMode)
+          adminZkClient.createTopic(topic.name, topic.partitions.get, topic.replicationFactor.get, topic.configsToAdd, topic.rackAwareMode)
         println(s"Created topic ${topic.name}.")
       } catch  {
         case e: TopicExistsException => if (!topic.ifTopicDoesntExist()) throw e
@@ -538,11 +546,11 @@ object TopicCommand extends Logging {
                            .describedAs("name")
                            .ofType(classOf[String])
     private val partitionsOpt = parser.accepts("partitions", "The number of partitions for the topic being created or " +
-      "altered (WARNING: If partitions are increased for a topic that has a key, the partition logic or ordering of the messages will be affected")
+      "altered (WARNING: If partitions are increased for a topic that has a key, the partition logic or ordering of the messages will be affected). If not supplied for create, defaults to the cluster default.")
                            .withRequiredArg
                            .describedAs("# of partitions")
                            .ofType(classOf[java.lang.Integer])
-    private val replicationFactorOpt = parser.accepts("replication-factor", "The replication factor for each partition in the topic being created.")
+    private val replicationFactorOpt = parser.accepts("replication-factor", "The replication factor for each partition in the topic being created. If not supplied, defaults to the cluster default.")
                            .withRequiredArg
                            .describedAs("replication factor")
                            .ofType(classOf[java.lang.Integer])
@@ -633,7 +641,7 @@ object TopicCommand extends Logging {
         CommandLineUtils.checkRequiredArgs(parser, options, topicOpt)
       if (!has(listOpt) && !has(describeOpt))
         CommandLineUtils.checkRequiredArgs(parser, options, topicOpt)
-      if (has(createOpt) && !has(replicaAssignmentOpt))
+      if (has(createOpt) && !has(replicaAssignmentOpt) && has(zkConnectOpt))
         CommandLineUtils.checkRequiredArgs(parser, options, partitionsOpt, replicationFactorOpt)
       if (has(bootstrapServerOpt) && has(alterOpt)) {
         CommandLineUtils.checkInvalidArgsSet(parser, options, Set(bootstrapServerOpt, configOpt), Set(alterOpt))
diff --git a/core/src/main/scala/kafka/server/AdminManager.scala b/core/src/main/scala/kafka/server/AdminManager.scala
index 85d272c4e593d..cfa599da34541 100644
--- a/core/src/main/scala/kafka/server/AdminManager.scala
+++ b/core/src/main/scala/kafka/server/AdminManager.scala
@@ -62,6 +62,9 @@ class AdminManager(val config: KafkaConfig,
 
   def hasDelayedTopicOperations = topicPurgatory.numDelayed != 0
 
+  private val defaultNumPartitions = config.numPartitions.intValue()
+  private val defaultReplicationFactor = config.defaultReplicationFactor.shortValue()
+
   /**
     * Try to complete delayed topic operations with the request key
     */
@@ -95,8 +98,15 @@ class AdminManager(val config: KafkaConfig,
           throw new InvalidRequestException("Both numPartitions or replicationFactor and replicasAssignments were set. " +
             "Both cannot be used at the same time.")
         }
+
+        val resolvedNumPartitions = if (topic.numPartitions == NO_NUM_PARTITIONS)
+          defaultNumPartitions else topic.numPartitions
+        val resolvedReplicationFactor = if (topic.replicationFactor == NO_REPLICATION_FACTOR)
+          defaultReplicationFactor else topic.replicationFactor
+
         val assignments = if (topic.assignments().isEmpty) {
-          AdminUtils.assignReplicasToBrokers(brokers, topic.numPartitions, topic.replicationFactor)
+          AdminUtils.assignReplicasToBrokers(
+            brokers, resolvedNumPartitions, resolvedReplicationFactor)
         } else {
           val assignments = new mutable.HashMap[Int, Seq[Int]]
           // Note: we don't check that replicaAssignment contains unknown brokers - unlike in add-partitions case,
@@ -115,9 +125,9 @@ class AdminManager(val config: KafkaConfig,
 
             // Use `null` for unset fields in the public API
             val numPartitions: java.lang.Integer =
-              if (topic.numPartitions == NO_NUM_PARTITIONS) null else topic.numPartitions
+              if (topic.assignments().isEmpty) resolvedNumPartitions else null
             val replicationFactor: java.lang.Short =
-              if (topic.replicationFactor == NO_REPLICATION_FACTOR) null else topic.replicationFactor
+              if (topic.assignments().isEmpty) resolvedReplicationFactor else null
             val javaAssignments = if (topic.assignments().isEmpty) {
               null
             } else {
diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index 88f10ff51720c..d622a7963de10 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -50,10 +50,10 @@ import org.junit.rules.Timeout
 import org.junit.{After, Before, Rule, Test}
 import org.scalatest.Assertions.intercept
 import scala.collection.JavaConverters._
+import scala.compat.java8.OptionConverters._
 import scala.concurrent.duration.Duration
 import scala.concurrent.{Await, Future}
 import scala.util.Random
-
 /**
  * An integration test of the KafkaAdminClient.
  *
@@ -150,10 +150,11 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
   @Test
   def testCreateDeleteTopics(): Unit = {
     client = AdminClient.create(createConfig())
-    val topics = Seq("mytopic", "mytopic2")
+    val topics = Seq("mytopic", "mytopic2", "mytopic3")
     val newTopics = Seq(
       new NewTopic("mytopic", Map((0: Integer) -> Seq[Integer](1, 2).asJava, (1: Integer) -> Seq[Integer](2, 0).asJava).asJava),
-      new NewTopic("mytopic2", 3, 3)
+      new NewTopic("mytopic2", 3, 3.toShort),
+      new NewTopic("mytopic3", Option.empty[Integer].asJava, Option.empty[java.lang.Short].asJava)
     )
     client.createTopics(newTopics.asJava, new CreateTopicsOptions().validateOnly(true)).all.get()
     waitForTopics(client, List(), topics)
@@ -166,6 +167,8 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     assertFutureExceptionTypeEquals(results.get("mytopic"), classOf[TopicExistsException])
     assertTrue(results.containsKey("mytopic2"))
     assertFutureExceptionTypeEquals(results.get("mytopic2"), classOf[TopicExistsException])
+    assertTrue(results.containsKey("mytopic3"))
+    assertFutureExceptionTypeEquals(results.get("mytopic3"), classOf[TopicExistsException])
 
     val topicToDescription = client.describeTopics(topics.asJava).all.get()
     assertEquals(topics.toSet, topicToDescription.keySet.asScala)
@@ -204,6 +207,11 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
       assertTrue(partition.replicas.contains(partition.leader))
     }
 
+    val topic3 = topicToDescription.get("mytopic3")
+    assertEquals("mytopic3", topic3.name)
+    assertEquals(configs.head.numPartitions, topic3.partitions.size)
+    assertEquals(configs.head.defaultReplicationFactor, topic3.partitions.get(0).replicas().size())
+
     client.deleteTopics(topics.asJava).all.get()
     waitForTopics(client, List(), topics)
   }
@@ -212,7 +220,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
   def testMetadataRefresh(): Unit = {
     client = AdminClient.create(createConfig())
     val topics = Seq("mytopic")
-    val newTopics = Seq(new NewTopic("mytopic", 3, 3))
+    val newTopics = Seq(new NewTopic("mytopic", 3, 3.toShort))
     client.createTopics(newTopics.asJava).all.get()
     waitForTopics(client, expectedPresent = topics, expectedMissing = List())
 
@@ -237,7 +245,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     assertEquals(expectedOperations, result.authorizedOperations().get())
 
     val topic = "mytopic"
-    val newTopics = Seq(new NewTopic(topic, 3, 3))
+    val newTopics = Seq(new NewTopic(topic, 3, 3.toShort))
     client.createTopics(newTopics.asJava).all.get()
     waitForTopics(client, expectedPresent = Seq(topic), expectedMissing = List())
 
@@ -265,7 +273,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     client = AdminClient.create(createConfig())
 
     val existingTopic = "existing-topic"
-    client.createTopics(Seq(existingTopic).map(new NewTopic(_, 1, 1)).asJava).all.get()
+    client.createTopics(Seq(existingTopic).map(new NewTopic(_, 1, 1.toShort)).asJava).all.get()
     waitForTopics(client, Seq(existingTopic), List())
 
     val nonExistingTopic = "non-existing"
@@ -1063,7 +1071,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
   def testDelayedClose(): Unit = {
     client = AdminClient.create(createConfig())
     val topics = Seq("mytopic", "mytopic2")
-    val newTopics = topics.map(new NewTopic(_, 1, 1))
+    val newTopics = topics.map(new NewTopic(_, 1, 1.toShort))
     val future = client.createTopics(newTopics.asJava, new CreateTopicsOptions().validateOnly(true)).all()
     client.close(time.Duration.ofHours(2))
     val future2 = client.createTopics(newTopics.asJava, new CreateTopicsOptions().validateOnly(true)).all()
@@ -1083,7 +1091,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     client = AdminClient.create(config)
     // Because the bootstrap servers are set up incorrectly, this call will not complete, but must be
     // cancelled by the close operation.
-    val future = client.createTopics(Seq("mytopic", "mytopic2").map(new NewTopic(_, 1, 1)).asJava,
+    val future = client.createTopics(Seq("mytopic", "mytopic2").map(new NewTopic(_, 1, 1.toShort)).asJava,
       new CreateTopicsOptions().timeoutMs(900000)).all()
     client.close(time.Duration.ZERO)
     assertFutureExceptionTypeEquals(future, classOf[TimeoutException])
@@ -1100,7 +1108,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     config.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, "0")
     client = AdminClient.create(config)
     val startTimeMs = Time.SYSTEM.milliseconds()
-    val future = client.createTopics(Seq("mytopic", "mytopic2").map(new NewTopic(_, 1, 1)).asJava,
+    val future = client.createTopics(Seq("mytopic", "mytopic2").map(new NewTopic(_, 1, 1.toShort)).asJava,
       new CreateTopicsOptions().timeoutMs(2)).all()
     assertFutureExceptionTypeEquals(future, classOf[TimeoutException])
     val endTimeMs = Time.SYSTEM.milliseconds()
@@ -1116,10 +1124,10 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     config.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, "100000000")
     val factory = new KafkaAdminClientTest.FailureInjectingTimeoutProcessorFactory()
     client = KafkaAdminClientTest.createInternal(new AdminClientConfig(config), factory)
-    val future = client.createTopics(Seq("mytopic", "mytopic2").map(new NewTopic(_, 1, 1)).asJava,
+    val future = client.createTopics(Seq("mytopic", "mytopic2").map(new NewTopic(_, 1, 1.toShort)).asJava,
         new CreateTopicsOptions().validateOnly(true)).all()
     assertFutureExceptionTypeEquals(future, classOf[TimeoutException])
-    val future2 = client.createTopics(Seq("mytopic3", "mytopic4").map(new NewTopic(_, 1, 1)).asJava,
+    val future2 = client.createTopics(Seq("mytopic3", "mytopic4").map(new NewTopic(_, 1, 1.toShort)).asJava,
       new CreateTopicsOptions().validateOnly(true)).all()
     future2.get
     assertEquals(1, factory.failuresInjected)
@@ -1141,7 +1149,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
       val testTopicName = "test_topic"
       val testNumPartitions = 2
       client.createTopics(Collections.singleton(
-        new NewTopic(testTopicName, testNumPartitions, 1))).all().get()
+        new NewTopic(testTopicName, testNumPartitions, 1.toShort))).all().get()
       waitForTopics(client, List(testTopicName), List())
 
       val producer = createProducer()
@@ -1799,8 +1807,8 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     val client = AdminClient.create(createConfig)
     val longTopicName = String.join("", Collections.nCopies(249, "x"));
     val invalidTopicName = String.join("", Collections.nCopies(250, "x"));
-    val newTopics2 = Seq(new NewTopic(invalidTopicName, 3, 3),
-                         new NewTopic(longTopicName, 3, 3))
+    val newTopics2 = Seq(new NewTopic(invalidTopicName, 3, 3.toShort),
+                         new NewTopic(longTopicName, 3, 3.toShort))
     val results = client.createTopics(newTopics2.asJava).values()
     assertTrue(results.containsKey(longTopicName))
     results.get(longTopicName).get()
diff --git a/core/src/test/scala/integration/kafka/api/ConsumerTopicCreationTest.scala b/core/src/test/scala/integration/kafka/api/ConsumerTopicCreationTest.scala
index c145b24416fdb..f447a56f50db9 100644
--- a/core/src/test/scala/integration/kafka/api/ConsumerTopicCreationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/ConsumerTopicCreationTest.scala
@@ -67,7 +67,7 @@ class ConsumerTopicCreationTest(brokerAutoTopicCreationEnable: JBoolean, consume
     val record = new ProducerRecord(topic_1, 0, "key".getBytes, "value".getBytes)
 
     // create `topic_1` and produce a record to it
-    adminClient.createTopics(Collections.singleton(new NewTopic(topic_1, 1, 1))).all.get
+    adminClient.createTopics(Collections.singleton(new NewTopic(topic_1, 1, 1.toShort))).all.get
     producer.send(record).get
 
     consumer.subscribe(util.Arrays.asList(topic_1, topic_2))
diff --git a/core/src/test/scala/kafka/tools/LogCompactionTester.scala b/core/src/test/scala/kafka/tools/LogCompactionTester.scala
index 3690856cd90ab..4360b2b266819 100755
--- a/core/src/test/scala/kafka/tools/LogCompactionTester.scala
+++ b/core/src/test/scala/kafka/tools/LogCompactionTester.scala
@@ -142,7 +142,7 @@ object LogCompactionTester {
 
     try {
       val topicConfigs = Map(TopicConfig.CLEANUP_POLICY_CONFIG -> TopicConfig.CLEANUP_POLICY_COMPACT)
-      val newTopics = topics.map(name => new NewTopic(name, 1, 1).configs(topicConfigs.asJava)).asJava
+      val newTopics = topics.map(name => new NewTopic(name, 1, 1.toShort).configs(topicConfigs.asJava)).asJava
       adminClient.createTopics(newTopics).all.get
 
       var pendingTopics: Seq[String] = Seq()
diff --git a/core/src/test/scala/unit/kafka/admin/TopicCommandWithAdminClientTest.scala b/core/src/test/scala/unit/kafka/admin/TopicCommandWithAdminClientTest.scala
index caa7a3be98e28..5db4309d64afa 100644
--- a/core/src/test/scala/unit/kafka/admin/TopicCommandWithAdminClientTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/TopicCommandWithAdminClientTest.scala
@@ -32,8 +32,8 @@ import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.security.auth.SecurityProtocol
 import org.junit.Assert.{assertEquals, assertFalse, assertTrue}
-import org.junit.{After, Before, Rule, Test}
 import org.junit.rules.TestName
+import org.junit.{After, Before, Rule, Test}
 import org.scalatest.Assertions.{fail, intercept}
 
 import scala.collection.JavaConverters._
@@ -49,8 +49,13 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
   override def generateConfigs: Seq[KafkaConfig] = TestUtils.createBrokerConfigs(
     numConfigs = 6,
     zkConnect = zkConnect,
-    rackInfo = Map(0 -> "rack1", 1 -> "rack2", 2 -> "rack2", 3 -> "rack1", 4 -> "rack3", 5 -> "rack3"
-    )).map(KafkaConfig.fromProps)
+    rackInfo = Map(0 -> "rack1", 1 -> "rack2", 2 -> "rack2", 3 -> "rack1", 4 -> "rack3", 5 -> "rack3"),
+    numPartitions = numPartitions,
+    defaultReplicationFactor = defaultReplicationFactor
+    ).map(KafkaConfig.fromProps)
+
+  private var numPartitions = 1
+  private var defaultReplicationFactor = 1.toShort
 
   private var topicService: AdminClientTopicService = _
   private var adminClient: JAdminClient = _
@@ -155,6 +160,52 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
     adminClient.listTopics().names().get().contains(testTopicName)
   }
 
+  @Test
+  def testCreateWithDefaults(): Unit = {
+    createAndWaitTopic(new TopicCommandOptions(
+      Array("--topic", testTopicName)))
+
+    val partitions = adminClient
+      .describeTopics(Collections.singletonList(testTopicName))
+      .all()
+      .get()
+      .get(testTopicName)
+      .partitions()
+    assertEquals(partitions.size(), numPartitions)
+    assertEquals(partitions.get(0).replicas().size(), defaultReplicationFactor)
+  }
+
+  @Test
+  def testCreateWithDefaultReplication(): Unit = {
+    createAndWaitTopic(new TopicCommandOptions(
+      Array("--topic", testTopicName, "--partitions", "2")))
+
+    val partitions = adminClient
+      .describeTopics(Collections.singletonList(testTopicName))
+      .all()
+      .get()
+      .get(testTopicName)
+      .partitions()
+    assertEquals(partitions.size(), 2)
+    assertEquals(partitions.get(0).replicas().size(), defaultReplicationFactor)
+  }
+
+  @Test
+  def testCreateWithDefaultPartitions(): Unit = {
+    createAndWaitTopic(new TopicCommandOptions(
+      Array("--topic", testTopicName, "--replication-factor", "2")))
+
+    val partitions = adminClient
+      .describeTopics(Collections.singletonList(testTopicName))
+      .all()
+      .get()
+      .get(testTopicName)
+      .partitions()
+
+    assertEquals(partitions.size(), numPartitions)
+    assertEquals(partitions.get(0).replicas().size(), 2)
+  }
+
   @Test
   def testCreateWithConfigs(): Unit = {
     val configResource = new ConfigResource(ConfigResource.Type.TOPIC, testTopicName)
@@ -211,7 +262,7 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
 
   @Test
   def testCreateWithNegativeReplicationFactor(): Unit = {
-    intercept[ExecutionException] {
+    intercept[IllegalArgumentException] {
       topicService.createTopic(new TopicCommandOptions(
         Array("--partitions", "2", "--replication-factor", "-1", "--topic", testTopicName)))
     }
@@ -241,17 +292,17 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
 
   @Test
   def testCreateWithNegativePartitionCount(): Unit = {
-    intercept[ExecutionException] {
+    intercept[IllegalArgumentException] {
       topicService.createTopic(new TopicCommandOptions(
         Array("--partitions", "-1", "--replication-factor", "1", "--topic", testTopicName)))
     }
   }
 
   @Test
-  def testCreateWithUnspecifiedPartitionCount(): Unit = {
-    assertExitCode(1,
-      () => topicService.createTopic(new TopicCommandOptions(
-        Array("--replication-factor", "1", "--topic", testTopicName))))
+  def testCreateWithUnspecifiedReplicationFactorAndPartitionsWithZkClient(): Unit = {
+    assertExitCode(1, () =>
+      new TopicCommandOptions(Array("--create", "--zookeeper", "zk", "--topic", testTopicName)).checkArgs()
+    )
   }
 
   @Test
@@ -281,9 +332,9 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
     val topic2 = "kafka.testTopic2"
     val topic3 = "oooof.testTopic1"
     adminClient.createTopics(
-      List(new NewTopic(topic1, 2, 2),
-        new NewTopic(topic2, 2, 2),
-        new NewTopic(topic3, 2, 2)).asJavaCollection)
+      List(new NewTopic(topic1, 2, 2.toShort),
+        new NewTopic(topic2, 2, 2.toShort),
+        new NewTopic(topic3, 2, 2.toShort)).asJavaCollection)
       .all().get()
     waitForTopicCreated(topic1)
     waitForTopicCreated(topic2)
@@ -301,8 +352,8 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
   def testListTopicsWithExcludeInternal(): Unit = {
     val topic1 = "kafka.testTopic1"
     adminClient.createTopics(
-      List(new NewTopic(topic1, 2, 2),
-        new NewTopic(Topic.GROUP_METADATA_TOPIC_NAME, 2, 2)).asJavaCollection)
+      List(new NewTopic(topic1, 2, 2.toShort),
+        new NewTopic(Topic.GROUP_METADATA_TOPIC_NAME, 2, 2.toShort)).asJavaCollection)
       .all().get()
     waitForTopicCreated(topic1)
 
@@ -316,7 +367,7 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
   @Test
   def testAlterPartitionCount(): Unit = {
     adminClient.createTopics(
-      List(new NewTopic(testTopicName, 2, 2)).asJavaCollection).all().get()
+      List(new NewTopic(testTopicName, 2, 2.toShort)).asJavaCollection).all().get()
     waitForTopicCreated(testTopicName)
 
     topicService.alterTopic(new TopicCommandOptions(
@@ -329,7 +380,7 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
   @Test
   def testAlterAssignment(): Unit = {
     adminClient.createTopics(
-      Collections.singletonList(new NewTopic(testTopicName, 2, 2))).all().get()
+      Collections.singletonList(new NewTopic(testTopicName, 2, 2.toShort))).all().get()
     waitForTopicCreated(testTopicName)
 
     topicService.alterTopic(new TopicCommandOptions(
@@ -343,7 +394,7 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
   @Test
   def testAlterAssignmentWithMoreAssignmentThanPartitions(): Unit = {
     adminClient.createTopics(
-      List(new NewTopic(testTopicName, 2, 2)).asJavaCollection).all().get()
+      List(new NewTopic(testTopicName, 2, 2.toShort)).asJavaCollection).all().get()
     waitForTopicCreated(testTopicName)
 
     intercept[ExecutionException] {
@@ -355,7 +406,7 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
   @Test
   def testAlterAssignmentWithMorePartitionsThanAssignment(): Unit = {
     adminClient.createTopics(
-      List(new NewTopic(testTopicName, 2, 2)).asJavaCollection).all().get()
+      List(new NewTopic(testTopicName, 2, 2.toShort)).asJavaCollection).all().get()
     waitForTopicCreated(testTopicName)
 
     intercept[ExecutionException] {
@@ -502,7 +553,7 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
   @Test
   def testDescribe(): Unit = {
     adminClient.createTopics(
-      Collections.singletonList(new NewTopic(testTopicName, 2, 2))).all().get()
+      Collections.singletonList(new NewTopic(testTopicName, 2, 2.toShort))).all().get()
     waitForTopicCreated(testTopicName)
 
     val output = TestUtils.grabConsoleOutput(
@@ -515,7 +566,7 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
   @Test
   def testDescribeUnavailablePartitions(): Unit = {
     adminClient.createTopics(
-      Collections.singletonList(new NewTopic(testTopicName, 6, 1))).all().get()
+      Collections.singletonList(new NewTopic(testTopicName, 6, 1.toShort))).all().get()
     waitForTopicCreated(testTopicName)
 
     try {
@@ -559,7 +610,7 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
   @Test
   def testDescribeUnderReplicatedPartitions(): Unit = {
     adminClient.createTopics(
-      Collections.singletonList(new NewTopic(testTopicName, 1, 6))).all().get()
+      Collections.singletonList(new NewTopic(testTopicName, 1, 6.toShort))).all().get()
     waitForTopicCreated(testTopicName)
 
     try {
@@ -581,7 +632,7 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
     configMap.put(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "6")
 
     adminClient.createTopics(
-      Collections.singletonList(new NewTopic(testTopicName, 1, 6).configs(configMap))).all().get()
+      Collections.singletonList(new NewTopic(testTopicName, 1, 6.toShort).configs(configMap))).all().get()
     waitForTopicCreated(testTopicName)
 
     try {
@@ -603,7 +654,7 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
     configMap.put(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "4")
 
     adminClient.createTopics(
-      Collections.singletonList(new NewTopic(testTopicName, 1, 6).configs(configMap))).all().get()
+      Collections.singletonList(new NewTopic(testTopicName, 1, 6.toShort).configs(configMap))).all().get()
     waitForTopicCreated(testTopicName)
 
     try {
@@ -640,8 +691,8 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
 
     adminClient.createTopics(
       java.util.Arrays.asList(
-        new NewTopic(underMinIsrTopic, 1, 6).configs(configMap),
-        new NewTopic(notUnderMinIsrTopic, 1, 6),
+        new NewTopic(underMinIsrTopic, 1, 6.toShort).configs(configMap),
+        new NewTopic(notUnderMinIsrTopic, 1, 6.toShort),
         new NewTopic(offlineTopic, Collections.singletonMap(0, Collections.singletonList(0))),
         new NewTopic(fullyReplicatedTopic, Collections.singletonMap(0, java.util.Arrays.asList(1, 2, 3))))).all().get()
 
diff --git a/core/src/test/scala/unit/kafka/server/AbstractCreateTopicsRequestTest.scala b/core/src/test/scala/unit/kafka/server/AbstractCreateTopicsRequestTest.scala
index 514e7aedc470f..d0dbd8e5ab9c4 100644
--- a/core/src/test/scala/unit/kafka/server/AbstractCreateTopicsRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/AbstractCreateTopicsRequestTest.scala
@@ -124,8 +124,18 @@ class AbstractCreateTopicsRequestTest extends BaseRequestTest {
         else {
           assertNotNull("The topic should be created", metadataForTopic)
           assertEquals(Errors.NONE, metadataForTopic.error)
-          assertEquals("The topic should have the correct number of partitions", partitions, metadataForTopic.partitionMetadata.size)
-          assertEquals("The topic should have the correct replication factor", replication, metadataForTopic.partitionMetadata.asScala.head.replicas.size)
+          if (partitions == -1) {
+            assertEquals("The topic should have the default number of partitions", configs.head.numPartitions, metadataForTopic.partitionMetadata.size)
+          } else {
+            assertEquals("The topic should have the correct number of partitions", partitions, metadataForTopic.partitionMetadata.size)
+          }
+
+          if (replication == -1) {
+            assertEquals("The topic should have the default replication factor",
+              configs.head.defaultReplicationFactor, metadataForTopic.partitionMetadata.asScala.head.replicas.size)
+          } else {
+            assertEquals("The topic should have the correct replication factor", replication, metadataForTopic.partitionMetadata.asScala.head.replicas.size)
+          }
         }
       }
 
diff --git a/core/src/test/scala/unit/kafka/server/CreateTopicsRequestTest.scala b/core/src/test/scala/unit/kafka/server/CreateTopicsRequestTest.scala
index 709b3c977c033..6d1b771a78a8d 100644
--- a/core/src/test/scala/unit/kafka/server/CreateTopicsRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/CreateTopicsRequestTest.scala
@@ -43,6 +43,13 @@ class CreateTopicsRequestTest extends AbstractCreateTopicsRequestTest {
       topicReq("topic10", numPartitions = 5, replicationFactor = 2),
       topicReq("topic11", assignment = Map(0 -> List(0, 1), 1 -> List(1, 0), 2 -> List(1, 2)))),
       validateOnly = true))
+    // Defaults
+    validateValidCreateTopicsRequests(topicsReq(Seq(
+      topicReq("topic12", replicationFactor = -1, numPartitions = -1))))
+    validateValidCreateTopicsRequests(topicsReq(Seq(
+      topicReq("topic13", replicationFactor = 2, numPartitions = -1))))
+    validateValidCreateTopicsRequests(topicsReq(Seq(
+      topicReq("topic14", replicationFactor = -1, numPartitions = 2))))
   }
 
   @Test
@@ -52,7 +59,7 @@ class CreateTopicsRequestTest extends AbstractCreateTopicsRequestTest {
     // Basic
     validateErrorCreateTopicsRequests(topicsReq(Seq(topicReq(existingTopic))),
       Map(existingTopic -> error(Errors.TOPIC_ALREADY_EXISTS, Some("Topic 'existing-topic' already exists."))))
-    validateErrorCreateTopicsRequests(topicsReq(Seq(topicReq("error-partitions", numPartitions = -1))),
+    validateErrorCreateTopicsRequests(topicsReq(Seq(topicReq("error-partitions", numPartitions = -2))),
       Map("error-partitions" -> error(Errors.INVALID_PARTITIONS)), checkErrorMessage = false)
     validateErrorCreateTopicsRequests(topicsReq(Seq(topicReq("error-replication",
       replicationFactor = brokerCount + 1))),
@@ -70,7 +77,7 @@ class CreateTopicsRequestTest extends AbstractCreateTopicsRequestTest {
     // Partial
     validateErrorCreateTopicsRequests(topicsReq(Seq(
       topicReq(existingTopic),
-      topicReq("partial-partitions", numPartitions = -1),
+      topicReq("partial-partitions", numPartitions = -2),
       topicReq("partial-replication", replicationFactor=brokerCount + 1),
       topicReq("partial-assignment", assignment=Map(0 -> List(0, 1), 1 -> List(0))),
       topicReq("partial-none"))),
diff --git a/core/src/test/scala/unit/kafka/server/CreateTopicsRequestWithPolicyTest.scala b/core/src/test/scala/unit/kafka/server/CreateTopicsRequestWithPolicyTest.scala
index 0395484cf3d22..74d9892e1ba55 100644
--- a/core/src/test/scala/unit/kafka/server/CreateTopicsRequestWithPolicyTest.scala
+++ b/core/src/test/scala/unit/kafka/server/CreateTopicsRequestWithPolicyTest.scala
@@ -99,9 +99,14 @@ class CreateTopicsRequestWithPolicyTest extends AbstractCreateTopicsRequestTest
         Some("Replication factor: 4 larger than available brokers: 3."))))
 
     validateErrorCreateTopicsRequests(topicsReq(Seq(topicReq("error-replication2",
-      numPartitions = 10, replicationFactor = -1)), validateOnly = true),
+      numPartitions = 10, replicationFactor = -2)), validateOnly = true),
       Map("error-replication2" -> error(Errors.INVALID_REPLICATION_FACTOR,
         Some("Replication factor must be larger than 0."))))
+
+    validateErrorCreateTopicsRequests(topicsReq(Seq(topicReq("error-partitions",
+      numPartitions = -2, replicationFactor = 1)), validateOnly = true),
+      Map("error-partitions" -> error(Errors.INVALID_PARTITIONS,
+        Some("Number of partitions must be larger than 0."))))
   }
 
 }
diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index ea8d2b3e549fa..3f9c8c3cfb25a 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -173,11 +173,14 @@ object TestUtils extends Logging {
     enableSaslSsl: Boolean = false,
     rackInfo: Map[Int, String] = Map(),
     logDirCount: Int = 1,
-    enableToken: Boolean = false): Seq[Properties] = {
+    enableToken: Boolean = false,
+    numPartitions: Int = 1,
+    defaultReplicationFactor: Short = 1): Seq[Properties] = {
     (0 until numConfigs).map { node =>
       createBrokerConfig(node, zkConnect, enableControlledShutdown, enableDeleteTopic, RandomPort,
         interBrokerSecurityProtocol, trustStoreFile, saslProperties, enablePlaintext = enablePlaintext, enableSsl = enableSsl,
-        enableSaslPlaintext = enableSaslPlaintext, enableSaslSsl = enableSaslSsl, rack = rackInfo.get(node), logDirCount = logDirCount, enableToken = enableToken)
+        enableSaslPlaintext = enableSaslPlaintext, enableSaslSsl = enableSaslSsl, rack = rackInfo.get(node), logDirCount = logDirCount, enableToken = enableToken,
+        numPartitions = numPartitions, defaultReplicationFactor = defaultReplicationFactor)
     }
   }
 
@@ -229,7 +232,9 @@ object TestUtils extends Logging {
                          saslSslPort: Int = RandomPort,
                          rack: Option[String] = None,
                          logDirCount: Int = 1,
-                         enableToken: Boolean = false): Properties = {
+                         enableToken: Boolean = false,
+                         numPartitions: Int = 1,
+                         defaultReplicationFactor: Short = 1): Properties = {
     def shouldEnable(protocol: SecurityProtocol) = interBrokerSecurityProtocol.fold(false)(_ == protocol)
 
     val protocolAndPorts = ArrayBuffer[(SecurityProtocol, Int)]()
@@ -289,6 +294,9 @@ object TestUtils extends Logging {
     if (enableToken)
       props.put(KafkaConfig.DelegationTokenMasterKeyProp, "masterkey")
 
+    props.put(KafkaConfig.NumPartitionsProp, numPartitions.toString)
+    props.put(KafkaConfig.DefaultReplicationFactorProp, defaultReplicationFactor.toString)
+
     props
   }
 
diff --git a/docs/upgrade.html b/docs/upgrade.html
index 3c737ec55b676..4f483e6b4f084 100644
--- a/docs/upgrade.html
+++ b/docs/upgrade.html
@@ -75,6 +75,7 @@ 
    Notable changes in 2
 
      
     
    • The bin/kafka-preferred-replica-election.sh command line tool has been deprecated. It has been replaced by bin/kafka-leader-election.sh.
      • 
     
    • The methods electPreferredLeaders in the Java AdminClient class have been deprecated in favor of the methods electLeaders.
      • 
+    
    • Scala code leveraging the NewTopic(String, int, short) constructor with literal values will need to explicitly call toShort on the second literal.
      • 
 
    
 
 
    Notable changes in 2.3.0
    
diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle
index c912deff660de..ba26d6cf3422e 100644
--- a/gradle/dependencies.gradle
+++ b/gradle/dependencies.gradle
@@ -90,6 +90,7 @@ versions += [
   rocksDB: "5.18.3",
   scalafmt: "1.5.1",
   scalatest: "3.0.7",
+  scalaJava8Compat : "0.9.0",
   scoverage: "1.3.1",
   scoveragePlugin: "2.5.0",
   shadowPlugin: "4.0.4",
@@ -157,6 +158,7 @@ libs += [
   scalaLogging: "com.typesafe.scala-logging:scala-logging_$versions.baseScala:$versions.scalaLogging",
   scalaReflect: "org.scala-lang:scala-reflect:$versions.scala",
   scalatest: "org.scalatest:scalatest_$versions.baseScala:$versions.scalatest",
+  scalaJava8Compat: "org.scala-lang.modules:scala-java8-compat_$versions.baseScala:$versions.scalaJava8Compat",
   scoveragePlugin: "org.scoverage:scalac-scoverage-plugin_$versions.baseScala:$versions.scoverage",
   scoverageRuntime: "org.scoverage:scalac-scoverage-runtime_$versions.baseScala:$versions.scoverage",
   slf4jApi: "org.slf4j:slf4j-api:$versions.slf4j",

From fb1f74958d1431af7c45a4d499b3b6ffef0bf70e Mon Sep 17 00:00:00 2001
From: Boyang Chen 
Date: Wed, 5 Jun 2019 14:20:04 -0700
Subject: [PATCH 0334/1071] KAFKA-8386; Use COORDINATOR_NOT_AVAILABLE error
 when group is Dead (#6762)

The Dead state in the coordinator is used for groups which are either pending deletion or migration to a new coordinator. Currently requests received while in this state result in an UNKNOWN_MEMBER_ID which causes consumers to reset the memberId. This is a problem for KIP-345 since it can cause an older member to fence a newer member. This patch changes the error code returned in this state to COORDINATOR_NOT_AVAILABLE, which causes the consumer to rediscover the coordinator, but not reset the memberId.

Reviewers: Guozhang Wang , Jason Gustafson 
---
 .../coordinator/group/GroupCoordinator.scala  | 28 ++++------
 .../group/GroupCoordinatorTest.scala          | 56 ++++++++++++++++---
 2 files changed, 61 insertions(+), 23 deletions(-)

diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
index daf38f35da14d..5d40e9bd51966 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
@@ -170,8 +170,8 @@ class GroupCoordinator(val brokerId: Int,
         // if the group is marked as dead, it means some other thread has just removed the group
         // from the coordinator metadata; it is likely that the group has migrated to some other
         // coordinator OR the group is in a transient unstable phase. Let the member retry
-        // joining without the specified member id.
-        responseCallback(joinError(JoinGroupRequest.UNKNOWN_MEMBER_ID, Errors.UNKNOWN_MEMBER_ID))
+        // finding the correct coordinator and rejoin.
+        responseCallback(joinError(JoinGroupRequest.UNKNOWN_MEMBER_ID, Errors.COORDINATOR_NOT_AVAILABLE))
       } else if (!group.supportsProtocols(protocolType, MemberMetadata.plainProtocolSet(protocols))) {
         responseCallback(joinError(JoinGroupRequest.UNKNOWN_MEMBER_ID, Errors.INCONSISTENT_GROUP_PROTOCOL))
       } else {
@@ -235,8 +235,8 @@ class GroupCoordinator(val brokerId: Int,
         // if the group is marked as dead, it means some other thread has just removed the group
         // from the coordinator metadata; this is likely that the group has migrated to some other
         // coordinator OR the group is in a transient unstable phase. Let the member retry
-        // joining without the specified member id.
-        responseCallback(joinError(memberId, Errors.UNKNOWN_MEMBER_ID))
+        // finding the correct coordinator and rejoin.
+        responseCallback(joinError(memberId, Errors.COORDINATOR_NOT_AVAILABLE))
       } else if (!group.supportsProtocols(protocolType, MemberMetadata.plainProtocolSet(protocols))) {
         responseCallback(joinError(memberId, Errors.INCONSISTENT_GROUP_PROTOCOL))
       } else if (group.isPendingMember(memberId)) {
@@ -351,8 +351,8 @@ class GroupCoordinator(val brokerId: Int,
         // if the group is marked as dead, it means some other thread has just removed the group
         // from the coordinator metadata; this is likely that the group has migrated to some other
         // coordinator OR the group is in a transient unstable phase. Let the member retry
-        // joining without the specified member id.
-        responseCallback(Array.empty, Errors.UNKNOWN_MEMBER_ID)
+        // finding the correct coordinator and rejoin.
+        responseCallback(Array.empty, Errors.COORDINATOR_NOT_AVAILABLE)
       } else if (group.isStaticMemberFenced(memberId, groupInstanceId)) {
         responseCallback(Array.empty, Errors.FENCED_INSTANCE_ID)
       } else if (!group.has(memberId)) {
@@ -417,16 +417,12 @@ class GroupCoordinator(val brokerId: Int,
 
     groupManager.getGroup(groupId) match {
       case None =>
-        // if the group is marked as dead, it means some other thread has just removed the group
-        // from the coordinator metadata; it is likely that the group has migrated to some other
-        // coordinator OR the group is in a transient unstable phase. Let the consumer to retry
-        // joining without specified consumer id,
         responseCallback(Errors.UNKNOWN_MEMBER_ID)
 
       case Some(group) =>
         group.inLock {
           if (group.is(Dead)) {
-            responseCallback(Errors.UNKNOWN_MEMBER_ID)
+            responseCallback(Errors.COORDINATOR_NOT_AVAILABLE)
           } else if (group.isPendingMember(memberId)) {
             // if a pending member is leaving, it needs to be removed from the pending list, heartbeat cancelled
             // and if necessary, prompt a JoinGroup completion.
@@ -509,10 +505,10 @@ class GroupCoordinator(val brokerId: Int,
       case Some(group) => group.inLock {
         if (group.is(Dead)) {
           // if the group is marked as dead, it means some other thread has just removed the group
-          // from the coordinator metadata; it is likely that the group has migrated to some other
+          // from the coordinator metadata; this is likely that the group has migrated to some other
           // coordinator OR the group is in a transient unstable phase. Let the member retry
-          // joining without the specified member id.
-          responseCallback(Errors.UNKNOWN_MEMBER_ID)
+          // finding the correct coordinator and rejoin.
+          responseCallback(Errors.COORDINATOR_NOT_AVAILABLE)
         } else if (group.isStaticMemberFenced(memberId, groupInstanceId)) {
           responseCallback(Errors.FENCED_INSTANCE_ID)
         } else if (!group.has(memberId)) {
@@ -609,8 +605,8 @@ class GroupCoordinator(val brokerId: Int,
         // if the group is marked as dead, it means some other thread has just removed the group
         // from the coordinator metadata; it is likely that the group has migrated to some other
         // coordinator OR the group is in a transient unstable phase. Let the member retry
-        // joining without the specified member id.
-        responseCallback(offsetMetadata.mapValues(_ => Errors.UNKNOWN_MEMBER_ID))
+        // finding the correct coordinator and rejoin.
+        responseCallback(offsetMetadata.mapValues(_ => Errors.COORDINATOR_NOT_AVAILABLE))
       } else if (group.isStaticMemberFenced(memberId, groupInstanceId)) {
         responseCallback(offsetMetadata.mapValues(_ => Errors.FENCED_INSTANCE_ID))
       } else if ((generationId < 0 && group.is(Empty)) || (producerId != NO_PRODUCER_ID)) {
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
index 770868c9a6700..5a1b20abbb58b 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
@@ -386,7 +386,18 @@ class GroupCoordinatorTest {
 
     groupCoordinator.groupManager.addGroup(new GroupMetadata(deadGroupId, Dead, new MockTime()))
     val joinGroupResult = dynamicJoinGroup(deadGroupId, memberId, protocolType, protocols)
-    assertEquals(Errors.UNKNOWN_MEMBER_ID, joinGroupResult.error)
+    assertEquals(Errors.COORDINATOR_NOT_AVAILABLE, joinGroupResult.error)
+  }
+
+  @Test
+  def testSyncDeadGroup() {
+    val memberId = "memberId"
+
+    val deadGroupId = "deadGroupId"
+
+    groupCoordinator.groupManager.addGroup(new GroupMetadata(deadGroupId, Dead, new MockTime()))
+    val syncGroupResult = syncGroupFollower(deadGroupId, 1, memberId)
+    assertEquals(Errors.COORDINATOR_NOT_AVAILABLE, syncGroupResult._2)
   }
 
   @Test
@@ -507,7 +518,7 @@ class GroupCoordinatorTest {
 
     EasyMock.reset(replicaManager)
     val joinGroupResult = staticJoinGroup(groupId, rebalanceResult.leaderId, leaderInstanceId, protocolType, protocols, clockAdvance = 1)
-    assertEquals(Errors.UNKNOWN_MEMBER_ID, joinGroupResult.error)
+    assertEquals(Errors.COORDINATOR_NOT_AVAILABLE, joinGroupResult.error)
   }
 
   @Test
@@ -715,6 +726,19 @@ class GroupCoordinatorTest {
     assertEquals(Errors.FENCED_INSTANCE_ID, invalidHeartbeatResult)
   }
 
+  @Test
+  def testOffsetCommitDeadGroup() {
+    val memberId = "memberId"
+
+    val deadGroupId = "deadGroupId"
+    val tp = new TopicPartition("topic", 0)
+    val offset = offsetAndMetadata(0)
+
+    groupCoordinator.groupManager.addGroup(new GroupMetadata(deadGroupId, Dead, new MockTime()))
+    val offsetCommitResult = commitOffsets(deadGroupId, memberId, 1, Map(tp -> offset))
+    assertEquals(Errors.COORDINATOR_NOT_AVAILABLE, offsetCommitResult(tp))
+  }
+
   @Test
   def staticMemberCommitOffsetWithInvalidMemberId() {
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
@@ -999,11 +1023,21 @@ class GroupCoordinatorTest {
 
   @Test
   def testHeartbeatUnknownGroup() {
-
     val heartbeatResult = heartbeat(groupId, memberId, -1)
     assertEquals(Errors.UNKNOWN_MEMBER_ID, heartbeatResult)
   }
 
+  @Test
+  def testheartbeatDeadGroup() {
+    val memberId = "memberId"
+
+    val deadGroupId = "deadGroupId"
+
+    groupCoordinator.groupManager.addGroup(new GroupMetadata(deadGroupId, Dead, new MockTime()))
+    val heartbeatResult = heartbeat(deadGroupId, memberId, 1)
+    assertEquals(Errors.COORDINATOR_NOT_AVAILABLE, heartbeatResult)
+  }
+
   @Test
   def testHeartbeatUnknownConsumerExistingGroup() {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
@@ -1304,9 +1338,7 @@ class GroupCoordinatorTest {
 
   @Test
   def testSyncGroupFromUnknownGroup() {
-    val generation = 1
-
-    val syncGroupResult = syncGroupFollower(groupId, generation, memberId)
+    val syncGroupResult = syncGroupFollower(groupId, 1, memberId)
     assertEquals(Errors.UNKNOWN_MEMBER_ID, syncGroupResult._2)
   }
 
@@ -2179,11 +2211,21 @@ class GroupCoordinatorTest {
 
   @Test
   def testLeaveGroupUnknownGroup() {
-
     val leaveGroupResult = leaveGroup(groupId, memberId)
     assertEquals(Errors.UNKNOWN_MEMBER_ID, leaveGroupResult)
   }
 
+  @Test
+  def testLeaveDeadGroup() {
+    val memberId = "memberId"
+
+    val deadGroupId = "deadGroupId"
+
+    groupCoordinator.groupManager.addGroup(new GroupMetadata(deadGroupId, Dead, new MockTime()))
+    val leaveGroupResult = leaveGroup(deadGroupId, memberId)
+    assertEquals(Errors.COORDINATOR_NOT_AVAILABLE, leaveGroupResult)
+  }
+
   @Test
   def testLeaveGroupUnknownConsumerExistingGroup() {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID

From 0e95c9f3a829110f0cd8c3695f40ba47f146fef7 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Wed, 5 Jun 2019 14:36:43 -0700
Subject: [PATCH 0335/1071] KAFKA-8400; Do not update follower replica state if
 the log read failed (#6814)

This patch checks for errors handling a fetch request before updating follower state. Previously we were unsafely passing the failed `LogReadResult` with most fields set to -1 into `Replica` to update follower state. Additionally, this patch attempts to improve the test coverage for ISR shrinking and expansion logic in `Partition`.

Reviewers: Guozhang Wang 
---
 .../main/scala/kafka/cluster/Partition.scala  | 135 +++---
 .../main/scala/kafka/cluster/Replica.scala    |  74 +--
 core/src/main/scala/kafka/log/Log.scala       |   2 +-
 .../kafka/server/LogOffsetMetadata.scala      |   6 +-
 .../server/ReplicaAlterLogDirsThread.scala    |   2 +-
 .../kafka/server/ReplicaFetcherThread.scala   |   6 +-
 .../scala/kafka/server/ReplicaManager.scala   |  98 ++--
 .../kafka/api/ConsumerBounceTest.scala        |   2 +-
 .../admin/ReassignPartitionsClusterTest.scala |   4 +-
 .../unit/kafka/cluster/PartitionTest.scala    | 443 +++++++++++++++---
 .../unit/kafka/cluster/ReplicaTest.scala      |  10 +-
 .../server/HighwatermarkPersistenceTest.scala |  22 +-
 .../unit/kafka/server/ISRExpirationTest.scala |  81 ++--
 .../unit/kafka/server/LogRecoveryTest.scala   |  10 +-
 .../server/ReplicaFetcherThreadTest.scala     |  20 +-
 .../server/ReplicaManagerQuotasTest.scala     |   6 +-
 .../kafka/server/ReplicaManagerTest.scala     |  92 +++-
 .../unit/kafka/server/SimpleFetchTest.scala   |  15 +-
 18 files changed, 726 insertions(+), 302 deletions(-)

diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala
index a6cce324748d2..22d9dffccd106 100755
--- a/core/src/main/scala/kafka/cluster/Partition.scala
+++ b/core/src/main/scala/kafka/cluster/Partition.scala
@@ -243,7 +243,7 @@ class Partition(val topicPartition: TopicPartition,
     new Gauge[Long] {
       def value = {
         leaderReplicaIfLocal.map { replica =>
-          replica.highWatermark.messageOffset - replica.lastStableOffset.messageOffset
+          replica.highWatermark - replica.lastStableOffset
         }.getOrElse(0)
       }
     },
@@ -527,11 +527,18 @@ class Partition(val topicPartition: TopicPartition,
 
       if (isNewLeader) {
         // construct the high watermark metadata for the new leader replica
-        leaderReplica.convertHWToLocalOffsetMetadata()
+        leaderReplica.maybeFetchHighWatermarkOffsetMetadata()
         // mark local replica as the leader after converting hw
         leaderReplicaIdOpt = Some(localBrokerId)
         // reset log end offset for remote replicas
-        assignedReplicas.filter(_.brokerId != localBrokerId).foreach(_.updateLogReadResult(LogReadResult.UnknownLogReadResult))
+        assignedReplicas.filter(_.brokerId != localBrokerId).foreach { replica =>
+          replica.updateFetchState(
+            followerFetchOffsetMetadata = LogOffsetMetadata.UnknownOffsetMetadata,
+            followerStartOffset = Log.UnknownOffset,
+            followerFetchTimeMs = 0L,
+            leaderEndOffset = Log.UnknownOffset
+          )
+        }
       }
       // we may need to increment high watermark since ISR could be down to 1
       (maybeIncrementLeaderHW(leaderReplica), isNewLeader)
@@ -581,28 +588,43 @@ class Partition(val topicPartition: TopicPartition,
    * Update the follower's state in the leader based on the last fetch request. See
    * [[kafka.cluster.Replica#updateLogReadResult]] for details.
    *
-   * @return true if the leader's log start offset or high watermark have been updated
+   * @return true if the follower's fetch state was updated, false if the followerId is not recognized
    */
-  def updateReplicaLogReadResult(replica: Replica, logReadResult: LogReadResult): Boolean = {
-    val replicaId = replica.brokerId
-    // No need to calculate low watermark if there is no delayed DeleteRecordsRequest
-    val oldLeaderLW = if (delayedOperations.numDelayedDelete > 0) lowWatermarkIfLeader else -1L
-    replica.updateLogReadResult(logReadResult)
-    val newLeaderLW = if (delayedOperations.numDelayedDelete > 0) lowWatermarkIfLeader else -1L
-    // check if the LW of the partition has incremented
-    // since the replica's logStartOffset may have incremented
-    val leaderLWIncremented = newLeaderLW > oldLeaderLW
-    // check if we need to expand ISR to include this replica
-    // if it is not in the ISR yet
-    val leaderHWIncremented = maybeExpandIsr(replicaId, logReadResult)
-
-    val result = leaderLWIncremented || leaderHWIncremented
-    // some delayed operations may be unblocked after HW or LW changed
-    if (result)
-      tryCompleteDelayedRequests()
+  def updateFollowerFetchState(followerId: Int,
+                               followerFetchOffsetMetadata: LogOffsetMetadata,
+                               followerStartOffset: Long,
+                               followerFetchTimeMs: Long,
+                               leaderEndOffset: Long): Boolean = {
+
+    getReplica(followerId) match {
+      case Some(followerReplica) =>
+        // No need to calculate low watermark if there is no delayed DeleteRecordsRequest
+        val oldLeaderLW = if (delayedOperations.numDelayedDelete > 0) lowWatermarkIfLeader else -1L
+        followerReplica.updateFetchState(
+          followerFetchOffsetMetadata,
+          followerStartOffset,
+          followerFetchTimeMs,
+          leaderEndOffset)
+        val newLeaderLW = if (delayedOperations.numDelayedDelete > 0) lowWatermarkIfLeader else -1L
+        // check if the LW of the partition has incremented
+        // since the replica's logStartOffset may have incremented
+        val leaderLWIncremented = newLeaderLW > oldLeaderLW
+        // check if we need to expand ISR to include this replica
+        // if it is not in the ISR yet
+        val followerFetchOffset = followerFetchOffsetMetadata.messageOffset
+        val leaderHWIncremented = maybeExpandIsr(followerReplica, followerFetchTimeMs)
+
+        // some delayed operations may be unblocked after HW or LW changed
+        if (leaderLWIncremented || leaderHWIncremented)
+          tryCompleteDelayedRequests()
+
+        debug(s"Recorded replica $followerId log end offset (LEO) position " +
+          s"$followerFetchOffset and log start offset $followerStartOffset.")
+        true
 
-    debug(s"Recorded replica $replicaId log end offset (LEO) position ${logReadResult.info.fetchOffsetMetadata.messageOffset}.")
-    result
+      case None =>
+        false
+    }
   }
 
   /**
@@ -621,19 +643,15 @@ class Partition(val topicPartition: TopicPartition,
    *
    * @return true if the high watermark has been updated
    */
-  def maybeExpandIsr(replicaId: Int, logReadResult: LogReadResult): Boolean = {
+  private def maybeExpandIsr(followerReplica: Replica, followerFetchTimeMs: Long): Boolean = {
     inWriteLock(leaderIsrUpdateLock) {
       // check if this replica needs to be added to the ISR
       leaderReplicaIfLocal match {
         case Some(leaderReplica) =>
-          val replica = getReplica(replicaId).get
-          val leaderHW = leaderReplica.highWatermark
-          val fetchOffset = logReadResult.info.fetchOffsetMetadata.messageOffset
-          if (!inSyncReplicas.contains(replica) &&
-             assignedReplicas.map(_.brokerId).contains(replicaId) &&
-             replica.logEndOffsetMetadata.offsetDiff(leaderHW) >= 0 &&
-             leaderEpochStartOffsetOpt.exists(fetchOffset >= _)) {
-            val newInSyncReplicas = inSyncReplicas + replica
+          val leaderHighwatermark = leaderReplica.highWatermark
+          if (!inSyncReplicas.contains(followerReplica) && isFollowerInSync(followerReplica, leaderHighwatermark)) {
+            val newInSyncReplicas = inSyncReplicas + followerReplica
+
             info(s"Expanding ISR from ${inSyncReplicas.map(_.brokerId).mkString(",")} " +
               s"to ${newInSyncReplicas.map(_.brokerId).mkString(",")}")
 
@@ -642,12 +660,17 @@ class Partition(val topicPartition: TopicPartition,
           }
           // check if the HW of the partition can now be incremented
           // since the replica may already be in the ISR and its LEO has just incremented
-          maybeIncrementLeaderHW(leaderReplica, logReadResult.fetchTimeMs)
+          maybeIncrementLeaderHW(leaderReplica, followerFetchTimeMs)
         case None => false // nothing to do if no longer leader
       }
     }
   }
 
+  private def isFollowerInSync(followerReplica: Replica, highWatermark: Long): Boolean = {
+    val followerEndOffset = followerReplica.logEndOffset
+    followerEndOffset >= highWatermark && leaderEpochStartOffsetOpt.exists(followerEndOffset >= _)
+  }
+
   /*
    * Returns a tuple where the first element is a boolean indicating whether enough replicas reached `requiredOffset`
    * and the second element is an error (which would be `Errors.NONE` for no error).
@@ -672,7 +695,7 @@ class Partition(val topicPartition: TopicPartition,
         }
 
         val minIsr = leaderReplica.log.get.config.minInSyncReplicas
-        if (leaderReplica.highWatermark.messageOffset >= requiredOffset) {
+        if (leaderReplica.highWatermark >= requiredOffset) {
           /*
            * The topic may be configured not to accept messages if there are not enough replicas in ISR
            * in this scenario the request was already appended locally and then added to the purgatory before the ISR was shrunk
@@ -711,18 +734,18 @@ class Partition(val topicPartition: TopicPartition,
       curTime - replica.lastCaughtUpTimeMs <= replicaLagTimeMaxMs || inSyncReplicas.contains(replica)
     }.map(_.logEndOffsetMetadata)
     val newHighWatermark = allLogEndOffsets.min(new LogOffsetMetadata.OffsetOrdering)
-    val oldHighWatermark = leaderReplica.highWatermark
+    val oldHighWatermark = leaderReplica.highWatermarkMetadata
 
     // Ensure that the high watermark increases monotonically. We also update the high watermark when the new
     // offset metadata is on a newer segment, which occurs whenever the log is rolled to a new segment.
     if (oldHighWatermark.messageOffset < newHighWatermark.messageOffset ||
       (oldHighWatermark.messageOffset == newHighWatermark.messageOffset && oldHighWatermark.onOlderSegment(newHighWatermark))) {
-      leaderReplica.highWatermark = newHighWatermark
+      leaderReplica.highWatermarkMetadata = newHighWatermark
       debug(s"High watermark updated to $newHighWatermark")
       true
     } else {
       def logEndOffsetString(r: Replica) = s"replica ${r.brokerId}: ${r.logEndOffsetMetadata}"
-      debug(s"Skipping update high watermark since new hw $newHighWatermark is not larger than old hw $oldHighWatermark. " +
+      trace(s"Skipping update high watermark since new hw $newHighWatermark is not larger than old hw $oldHighWatermark. " +
         s"All current LEOs are ${assignedReplicas.map(logEndOffsetString)}")
       false
     }
@@ -747,7 +770,7 @@ class Partition(val topicPartition: TopicPartition,
    */
   private def tryCompleteDelayedRequests(): Unit = delayedOperations.checkAndCompleteAll()
 
-  def maybeShrinkIsr(replicaMaxLagTimeMs: Long) {
+  def maybeShrinkIsr(replicaMaxLagTimeMs: Long): Unit = {
     val leaderHWIncremented = inWriteLock(leaderIsrUpdateLock) {
       leaderReplicaIfLocal match {
         case Some(leaderReplica) =>
@@ -758,7 +781,7 @@ class Partition(val topicPartition: TopicPartition,
             info("Shrinking ISR from %s to %s. Leader: (highWatermark: %d, endOffset: %d). Out of sync replicas: %s."
               .format(inSyncReplicas.map(_.brokerId).mkString(","),
                 newInSyncReplicas.map(_.brokerId).mkString(","),
-                leaderReplica.highWatermark.messageOffset,
+                leaderReplica.highWatermarkMetadata.messageOffset,
                 leaderReplica.logEndOffset,
                 outOfSyncReplicas.map { replica =>
                   s"(brokerId: ${replica.brokerId}, endOffset: ${replica.logEndOffset})"
@@ -784,6 +807,14 @@ class Partition(val topicPartition: TopicPartition,
       tryCompleteDelayedRequests()
   }
 
+  private def isFollowerOutOfSync(followerReplica: Replica,
+                                  leaderEndOffset: Long,
+                                  currentTimeMs: Long,
+                                  maxLagMs: Long): Boolean = {
+    followerReplica.logEndOffset != leaderEndOffset &&
+      (currentTimeMs - followerReplica.lastCaughtUpTimeMs) > maxLagMs
+  }
+
   def getOutOfSyncReplicas(leaderReplica: Replica, maxLagMs: Long): Set[Replica] = {
     /**
      * If the follower already has the same leo as the leader, it will not be considered as out-of-sync,
@@ -798,13 +829,9 @@ class Partition(val topicPartition: TopicPartition,
      *
      **/
     val candidateReplicas = inSyncReplicas - leaderReplica
-
-    val laggingReplicas = candidateReplicas.filter(r =>
-      r.logEndOffset != leaderReplica.logEndOffset && (time.milliseconds - r.lastCaughtUpTimeMs) > maxLagMs)
-    if (laggingReplicas.nonEmpty)
-      debug("Lagging replicas are %s".format(laggingReplicas.map(_.brokerId).mkString(",")))
-
-    laggingReplicas
+    val currentTimeMs = time.milliseconds()
+    val leaderEndOffset = leaderReplica.logEndOffset
+    candidateReplicas.filter(r => isFollowerOutOfSync(r, leaderEndOffset, currentTimeMs, maxLagMs))
   }
 
   private def doAppendRecordsToFollowerOrFutureReplica(records: MemoryRecords, isFuture: Boolean): Option[LogAppendInfo] = {
@@ -903,10 +930,10 @@ class Partition(val topicPartition: TopicPartition,
      * where data gets appended to the log immediately after the replica has consumed from it
      * This can cause a replica to always be out of sync.
      */
-    val initialHighWatermark = localReplica.highWatermark.messageOffset
+    val initialHighWatermark = localReplica.highWatermark
     val initialLogStartOffset = localReplica.logStartOffset
     val initialLogEndOffset = localReplica.logEndOffset
-    val initialLastStableOffset = localReplica.lastStableOffset.messageOffset
+    val initialLastStableOffset = localReplica.lastStableOffset
 
     val maxOffsetOpt = fetchIsolation match {
       case FetchLogEnd => None
@@ -940,8 +967,8 @@ class Partition(val topicPartition: TopicPartition,
     val localReplica = localReplicaWithEpochOrException(currentLeaderEpoch, fetchOnlyFromLeader)
 
     val lastFetchableOffset = isolationLevel match {
-      case Some(IsolationLevel.READ_COMMITTED) => localReplica.lastStableOffset.messageOffset
-      case Some(IsolationLevel.READ_UNCOMMITTED) => localReplica.highWatermark.messageOffset
+      case Some(IsolationLevel.READ_COMMITTED) => localReplica.lastStableOffset
+      case Some(IsolationLevel.READ_UNCOMMITTED) => localReplica.highWatermark
       case None => localReplica.logEndOffset
     }
 
@@ -954,10 +981,10 @@ class Partition(val topicPartition: TopicPartition,
     // Only consider throwing an error if we get a client request (isolationLevel is defined) and the start offset
     // is lagging behind the high watermark
     val maybeOffsetsError: Option[ApiException] = leaderEpochStartOffsetOpt
-      .filter(epochStart => isolationLevel.isDefined && epochStart > localReplica.highWatermark.messageOffset)
+      .filter(epochStart => isolationLevel.isDefined && epochStart > localReplica.highWatermark)
       .map(epochStart => Errors.OFFSET_NOT_AVAILABLE.exception(s"Failed to fetch offsets for " +
         s"partition $topicPartition with leader $epochLogString as this partition's " +
-        s"high watermark (${localReplica.highWatermark.messageOffset}) is lagging behind the " +
+        s"high watermark (${localReplica.highWatermark}) is lagging behind the " +
         s"start offset from the beginning of this epoch ($epochStart)."))
 
     def getOffsetByTimestamp: Option[TimestampAndOffset] = {
@@ -1011,7 +1038,7 @@ class Partition(val topicPartition: TopicPartition,
     if (!isFromConsumer) {
       allOffsets
     } else {
-      val hw = localReplica.highWatermark.messageOffset
+      val hw = localReplica.highWatermark
       if (allOffsets.exists(_ > hw))
         hw +: allOffsets.dropWhile(_ > hw)
       else
@@ -1038,7 +1065,7 @@ class Partition(val topicPartition: TopicPartition,
           throw new PolicyViolationException(s"Records of partition $topicPartition can not be deleted due to the configured policy")
 
         val convertedOffset = if (offset == DeleteRecordsRequest.HIGH_WATERMARK)
-          leaderReplica.highWatermark.messageOffset
+          leaderReplica.highWatermark
         else
           offset
 
diff --git a/core/src/main/scala/kafka/cluster/Replica.scala b/core/src/main/scala/kafka/cluster/Replica.scala
index 32055d7a5e30d..831233ea08132 100644
--- a/core/src/main/scala/kafka/cluster/Replica.scala
+++ b/core/src/main/scala/kafka/cluster/Replica.scala
@@ -19,7 +19,7 @@ package kafka.cluster
 
 import kafka.log.{Log, LogOffsetSnapshot}
 import kafka.utils.Logging
-import kafka.server.{LogOffsetMetadata, LogReadResult, OffsetAndEpoch}
+import kafka.server.{LogOffsetMetadata, OffsetAndEpoch}
 import org.apache.kafka.common.{KafkaException, TopicPartition}
 import org.apache.kafka.common.errors.OffsetOutOfRangeException
 import org.apache.kafka.common.utils.Time
@@ -30,13 +30,13 @@ class Replica(val brokerId: Int,
               initialHighWatermarkValue: Long = 0L,
               @volatile var log: Option[Log] = None) extends Logging {
   // the high watermark offset value, in non-leader replicas only its message offsets are kept
-  @volatile private[this] var highWatermarkMetadata = new LogOffsetMetadata(initialHighWatermarkValue)
+  @volatile private[this] var _highWatermarkMetadata = new LogOffsetMetadata(initialHighWatermarkValue)
   // the log end offset value, kept in all replicas;
   // for local replica it is the log's end offset, for remote replicas its value is only updated by follower fetch
   @volatile private[this] var _logEndOffsetMetadata = LogOffsetMetadata.UnknownOffsetMetadata
   // the log start offset value, kept in all replicas;
   // for local replica it is the log's start offset, for remote replicas its value is only updated by follower fetch
-  @volatile private[this] var _logStartOffset = Log.UnknownLogStartOffset
+  @volatile private[this] var _logStartOffset = Log.UnknownOffset
 
   // The log end offset value at the time the leader received the last FetchRequest from this follower
   // This is used to determine the lastCaughtUpTimeMs of the follower
@@ -69,16 +69,19 @@ class Replica(val brokerId: Int,
    * fetch request is always smaller than the leader's LEO, which can happen if small produce requests are received at
    * high frequency.
    */
-  def updateLogReadResult(logReadResult: LogReadResult) {
-    if (logReadResult.info.fetchOffsetMetadata.messageOffset >= logReadResult.leaderLogEndOffset)
-      _lastCaughtUpTimeMs = math.max(_lastCaughtUpTimeMs, logReadResult.fetchTimeMs)
-    else if (logReadResult.info.fetchOffsetMetadata.messageOffset >= lastFetchLeaderLogEndOffset)
+  def updateFetchState(followerFetchOffsetMetadata: LogOffsetMetadata,
+                       followerStartOffset: Long,
+                       followerFetchTimeMs: Long,
+                       leaderEndOffset: Long): Unit = {
+    if (followerFetchOffsetMetadata.messageOffset >= leaderEndOffset)
+      _lastCaughtUpTimeMs = math.max(_lastCaughtUpTimeMs, followerFetchTimeMs)
+    else if (followerFetchOffsetMetadata.messageOffset >= lastFetchLeaderLogEndOffset)
       _lastCaughtUpTimeMs = math.max(_lastCaughtUpTimeMs, lastFetchTimeMs)
 
-    logStartOffset = logReadResult.followerLogStartOffset
-    logEndOffsetMetadata = logReadResult.info.fetchOffsetMetadata
-    lastFetchLeaderLogEndOffset = logReadResult.leaderLogEndOffset
-    lastFetchTimeMs = logReadResult.fetchTimeMs
+    logStartOffset = followerStartOffset
+    logEndOffsetMetadata = followerFetchOffsetMetadata
+    lastFetchLeaderLogEndOffset = leaderEndOffset
+    lastFetchTimeMs = followerFetchTimeMs
   }
 
   def resetLastCaughtUpTime(curLeaderLogEndOffset: Long, curTimeMs: Long, lastCaughtUpTimeMs: Long) {
@@ -127,9 +130,9 @@ class Replica(val brokerId: Int,
    */
   def maybeIncrementLogStartOffset(newLogStartOffset: Long) {
     if (isLocal) {
-      if (newLogStartOffset > highWatermark.messageOffset)
+      if (newLogStartOffset > highWatermark)
         throw new OffsetOutOfRangeException(s"Cannot increment the log start offset to $newLogStartOffset of partition $topicPartition " +
-          s"since it is larger than the high watermark ${highWatermark.messageOffset}")
+          s"since it is larger than the high watermark $highWatermark")
       log.get.maybeIncrementLogStartOffset(newLogStartOffset)
     } else {
       throw new KafkaException(s"Should not try to delete records on partition $topicPartition's non-local replica $brokerId")
@@ -152,20 +155,26 @@ class Replica(val brokerId: Int,
     else
       _logStartOffset
 
-  def highWatermark_=(newHighWatermark: LogOffsetMetadata) {
+  def highWatermarkMetadata_=(newHighWatermarkMetadata: LogOffsetMetadata) {
     if (isLocal) {
-      if (newHighWatermark.messageOffset < 0)
+      if (newHighWatermarkMetadata.messageOffset < 0)
         throw new IllegalArgumentException("High watermark offset should be non-negative")
 
-      highWatermarkMetadata = newHighWatermark
-      log.foreach(_.onHighWatermarkIncremented(newHighWatermark.messageOffset))
-      trace(s"Setting high watermark for replica $brokerId partition $topicPartition to [$newHighWatermark]")
+      _highWatermarkMetadata = newHighWatermarkMetadata
+      log.foreach(_.onHighWatermarkIncremented(newHighWatermarkMetadata.messageOffset))
+      trace(s"Setting high watermark for replica $brokerId partition $topicPartition to [$newHighWatermarkMetadata]")
     } else {
       throw new KafkaException(s"Should not set high watermark on partition $topicPartition's non-local replica $brokerId")
     }
   }
 
-  def highWatermark: LogOffsetMetadata = highWatermarkMetadata
+  def highWatermark_=(newHighWatermark: Long): Unit = {
+    highWatermarkMetadata = LogOffsetMetadata(newHighWatermark)
+  }
+
+  def highWatermarkMetadata: LogOffsetMetadata = _highWatermarkMetadata
+
+  def highWatermark: Long = _highWatermarkMetadata.messageOffset
 
   /**
    * The last stable offset (LSO) is defined as the first offset such that all lower offsets have been "decided."
@@ -174,30 +183,33 @@ class Replica(val brokerId: Int,
    * to the high watermark if there are no transactional messages in the log. Note also that the LSO cannot advance
    * beyond the high watermark.
    */
-  def lastStableOffset: LogOffsetMetadata = {
+  def lastStableOffsetMetadata: LogOffsetMetadata = {
     log.map { log =>
       log.firstUnstableOffset match {
-        case Some(offsetMetadata) if offsetMetadata.messageOffset < highWatermark.messageOffset => offsetMetadata
-        case _ => highWatermark
+        case Some(offsetMetadata) if offsetMetadata.messageOffset < highWatermark => offsetMetadata
+        case _ => highWatermarkMetadata
       }
     }.getOrElse(throw new KafkaException(s"Cannot fetch last stable offset on partition $topicPartition's " +
       s"non-local replica $brokerId"))
   }
 
+  def lastStableOffset: Long = lastStableOffsetMetadata.messageOffset
+
   /*
    * Convert hw to local offset metadata by reading the log at the hw offset.
    * If the hw offset is out of range, return the first offset of the first log segment as the offset metadata.
    */
-  def convertHWToLocalOffsetMetadata() {
-    if (isLocal) {
-      highWatermarkMetadata = log.get.convertToOffsetMetadata(highWatermarkMetadata.messageOffset).getOrElse {
+  def maybeFetchHighWatermarkOffsetMetadata(): Unit = {
+    if (!isLocal)
+      throw new KafkaException(s"Should not construct complete high watermark on partition $topicPartition's non-local replica $brokerId")
+
+    if (highWatermarkMetadata.messageOffsetOnly) {
+      highWatermarkMetadata = log.get.convertToOffsetMetadata(highWatermark).getOrElse {
         log.get.convertToOffsetMetadata(logStartOffset).getOrElse {
           val firstSegmentOffset = log.get.logSegments.head.baseOffset
           new LogOffsetMetadata(firstSegmentOffset, firstSegmentOffset, 0)
         }
       }
-    } else {
-      throw new KafkaException(s"Should not construct complete high watermark on partition $topicPartition's non-local replica $brokerId")
     }
   }
 
@@ -205,8 +217,8 @@ class Replica(val brokerId: Int,
     LogOffsetSnapshot(
       logStartOffset = logStartOffset,
       logEndOffset = logEndOffsetMetadata,
-      highWatermark =  highWatermark,
-      lastStableOffset = lastStableOffset)
+      highWatermark =  highWatermarkMetadata,
+      lastStableOffset = lastStableOffsetMetadata)
   }
 
   override def equals(that: Any): Boolean = that match {
@@ -224,8 +236,8 @@ class Replica(val brokerId: Int,
     replicaString.append(s", isLocal=$isLocal")
     replicaString.append(s", lastCaughtUpTimeMs=$lastCaughtUpTimeMs")
     if (isLocal) {
-      replicaString.append(s", highWatermark=$highWatermark")
-      replicaString.append(s", lastStableOffset=$lastStableOffset")
+      replicaString.append(s", highWatermark=$highWatermarkMetadata")
+      replicaString.append(s", lastStableOffset=$lastStableOffsetMetadata")
     }
     replicaString.append(")")
     replicaString.toString
diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index 9ab6fda30924f..38c36179d95a0 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -2183,7 +2183,7 @@ object Log {
   private[log] val DeleteDirPattern = Pattern.compile(s"^(\\S+)-(\\S+)\\.(\\S+)$DeleteDirSuffix")
   private[log] val FutureDirPattern = Pattern.compile(s"^(\\S+)-(\\S+)\\.(\\S+)$FutureDirSuffix")
 
-  val UnknownLogStartOffset = -1L
+  val UnknownOffset = -1L
 
   def apply(dir: File,
             config: LogConfig,
diff --git a/core/src/main/scala/kafka/server/LogOffsetMetadata.scala b/core/src/main/scala/kafka/server/LogOffsetMetadata.scala
index effbaa04ea081..67afac69e378d 100644
--- a/core/src/main/scala/kafka/server/LogOffsetMetadata.scala
+++ b/core/src/main/scala/kafka/server/LogOffsetMetadata.scala
@@ -17,11 +17,11 @@
 
 package kafka.server
 
+import kafka.log.Log
 import org.apache.kafka.common.KafkaException
 
 object LogOffsetMetadata {
   val UnknownOffsetMetadata = new LogOffsetMetadata(-1, 0, 0)
-  val UnknownSegBaseOffset = -1L
   val UnknownFilePosition = -1
 
   class OffsetOrdering extends Ordering[LogOffsetMetadata] {
@@ -39,7 +39,7 @@ object LogOffsetMetadata {
  *  3. the physical position on the located segment
  */
 case class LogOffsetMetadata(messageOffset: Long,
-                             segmentBaseOffset: Long = LogOffsetMetadata.UnknownSegBaseOffset,
+                             segmentBaseOffset: Long = Log.UnknownOffset,
                              relativePositionInSegment: Int = LogOffsetMetadata.UnknownFilePosition) {
 
   // check if this offset is already on an older segment compared with the given offset
@@ -76,7 +76,7 @@ case class LogOffsetMetadata(messageOffset: Long,
 
   // decide if the offset metadata only contains message offset info
   def messageOffsetOnly: Boolean = {
-    segmentBaseOffset == LogOffsetMetadata.UnknownSegBaseOffset && relativePositionInSegment == LogOffsetMetadata.UnknownFilePosition
+    segmentBaseOffset == Log.UnknownOffset && relativePositionInSegment == LogOffsetMetadata.UnknownFilePosition
   }
 
   override def toString = s"(offset=$messageOffset segment=[$segmentBaseOffset:$relativePositionInSegment])"
diff --git a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
index 0622b303498d2..8b45501bd6f64 100644
--- a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
+++ b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
@@ -111,7 +111,7 @@ class ReplicaAlterLogDirsThread(name: String,
 
     val logAppendInfo = partition.appendRecordsToFollowerOrFutureReplica(records, isFuture = true)
     val futureReplicaHighWatermark = futureReplica.logEndOffset.min(partitionData.highWatermark)
-    futureReplica.highWatermark = new LogOffsetMetadata(futureReplicaHighWatermark)
+    futureReplica.highWatermark = futureReplicaHighWatermark
     futureReplica.maybeIncrementLogStartOffset(partitionData.logStartOffset)
 
     if (partition.maybeReplaceCurrentWithFutureReplica())
diff --git a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
index b1b5dd0551dc2..947e16aad58e5 100644
--- a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
+++ b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
@@ -168,7 +168,7 @@ class ReplicaFetcherThread(name: String,
     // for the follower replica, we do not need to keep
     // its segment base offset the physical position,
     // these values will be computed upon making the leader
-    replica.highWatermark = new LogOffsetMetadata(followerHighWatermark)
+    replica.highWatermark = followerHighWatermark
     replica.maybeIncrementLogStartOffset(leaderLogStartOffset)
     if (isTraceEnabled)
       trace(s"Follower set replica high watermark for partition $topicPartition to $followerHighWatermark")
@@ -282,9 +282,9 @@ class ReplicaFetcherThread(name: String,
 
     partition.truncateTo(offsetTruncationState.offset, isFuture = false)
 
-    if (offsetTruncationState.offset < replica.highWatermark.messageOffset)
+    if (offsetTruncationState.offset < replica.highWatermark)
       warn(s"Truncating $tp to offset ${offsetTruncationState.offset} below high watermark " +
-        s"${replica.highWatermark.messageOffset}")
+        s"${replica.highWatermark}")
 
     // mark the future replica for truncation only when we do last truncation
     if (offsetTruncationState.truncationCompleted)
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index e9ab7386101e8..8b383c2b342bf 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -108,16 +108,6 @@ case class FetchPartitionData(error: Errors = Errors.NONE,
                               lastStableOffset: Option[Long],
                               abortedTransactions: Option[List[AbortedTransaction]])
 
-object LogReadResult {
-  val UnknownLogReadResult = LogReadResult(info = FetchDataInfo(LogOffsetMetadata.UnknownOffsetMetadata, MemoryRecords.EMPTY),
-                                           highWatermark = -1L,
-                                           leaderLogStartOffset = -1L,
-                                           leaderLogEndOffset = -1L,
-                                           followerLogStartOffset = -1L,
-                                           fetchTimeMs = -1L,
-                                           readSize = -1,
-                                           lastStableOffset = None)
-}
 
 /**
  * Trait to represent the state of hosted partitions. We create a concrete (active) Partition
@@ -620,7 +610,7 @@ class ReplicaManager(val config: KafkaConfig,
             logManager.abortAndPauseCleaning(topicPartition)
 
             val initialFetchState = InitialFetchState(BrokerEndPoint(config.brokerId, "localhost", -1),
-              partition.getLeaderEpoch, futureReplica.highWatermark.messageOffset)
+              partition.getLeaderEpoch, futureReplica.highWatermark)
             replicaAlterLogDirsManager.addFetcherForPartitions(Map(topicPartition -> initialFetchState))
           }
 
@@ -685,7 +675,7 @@ class ReplicaManager(val config: KafkaConfig,
         if (isFuture)
           replica.logEndOffset - logEndOffset
         else
-          math.max(replica.highWatermark.messageOffset - logEndOffset, 0)
+          math.max(replica.highWatermark - logEndOffset, 0)
       case None =>
         // return -1L to indicate that the LEO lag is not available if the replica is not created or is offline
         DescribeLogDirsResponse.INVALID_OFFSET_LAG
@@ -849,7 +839,7 @@ class ReplicaManager(val config: KafkaConfig,
         hardMaxBytesLimit = hardMaxBytesLimit,
         readPartitionInfo = fetchInfos,
         quota = quota)
-      if (isFromFollower) updateFollowerLogReadResults(replicaId, result)
+      if (isFromFollower) updateFollowerFetchState(replicaId, result)
       else result
     }
 
@@ -967,14 +957,14 @@ class ReplicaManager(val config: KafkaConfig,
                  _: KafkaStorageException |
                  _: OffsetOutOfRangeException) =>
           LogReadResult(info = FetchDataInfo(LogOffsetMetadata.UnknownOffsetMetadata, MemoryRecords.EMPTY),
-                        highWatermark = -1L,
-                        leaderLogStartOffset = -1L,
-                        leaderLogEndOffset = -1L,
-                        followerLogStartOffset = -1L,
-                        fetchTimeMs = -1L,
-                        readSize = 0,
-                        lastStableOffset = None,
-                        exception = Some(e))
+            highWatermark = Log.UnknownOffset,
+            leaderLogStartOffset = Log.UnknownOffset,
+            leaderLogEndOffset = Log.UnknownOffset,
+            followerLogStartOffset = Log.UnknownOffset,
+            fetchTimeMs = -1L,
+            readSize = 0,
+            lastStableOffset = None,
+            exception = Some(e))
         case e: Throwable =>
           brokerTopicStats.topicStats(tp.topic).failedFetchRequestRate.mark()
           brokerTopicStats.allTopicsStats.failedFetchRequestRate.mark()
@@ -984,14 +974,14 @@ class ReplicaManager(val config: KafkaConfig,
             s"on partition $tp: $fetchInfo", e)
 
           LogReadResult(info = FetchDataInfo(LogOffsetMetadata.UnknownOffsetMetadata, MemoryRecords.EMPTY),
-                        highWatermark = -1L,
-                        leaderLogStartOffset = -1L,
-                        leaderLogEndOffset = -1L,
-                        followerLogStartOffset = -1L,
-                        fetchTimeMs = -1L,
-                        readSize = 0,
-                        lastStableOffset = None,
-                        exception = Some(e))
+            highWatermark = Log.UnknownOffset,
+            leaderLogStartOffset = Log.UnknownOffset,
+            leaderLogEndOffset = Log.UnknownOffset,
+            followerLogStartOffset = Log.UnknownOffset,
+            fetchTimeMs = -1L,
+            readSize = 0,
+            lastStableOffset = None,
+            exception = Some(e))
       }
     }
 
@@ -1161,7 +1151,7 @@ class ReplicaManager(val config: KafkaConfig,
               logManager.abortAndPauseCleaning(topicPartition)
 
               futureReplicasAndInitialOffset.put(topicPartition, InitialFetchState(leader,
-                partition.getLeaderEpoch, replica.highWatermark.messageOffset))
+                partition.getLeaderEpoch, replica.highWatermark))
             }
           }
         }
@@ -1356,7 +1346,7 @@ class ReplicaManager(val config: KafkaConfig,
         val partitionsToMakeFollowerWithLeaderAndOffset = partitionsToMakeFollower.map { partition =>
           val leader = metadataCache.getAliveBrokers.find(_.id == partition.leaderReplicaIdOpt.get).get
             .brokerEndPoint(config.interBrokerListenerName)
-          val fetchOffset = partition.localReplicaOrException.highWatermark.messageOffset
+          val fetchOffset = partition.localReplicaOrException.highWatermark
           partition.topicPartition -> InitialFetchState(leader, partition.getLeaderEpoch, fetchOffset)
        }.toMap
 
@@ -1394,33 +1384,41 @@ class ReplicaManager(val config: KafkaConfig,
   }
 
   /**
-   * Update the follower's fetch state in the leader based on the last fetch request and update `readResult`,
-   * if the follower replica is not recognized to be one of the assigned replicas. Do not update
-   * `readResult` otherwise, so that log start/end offset and high watermark is consistent with
+   * Update the follower's fetch state on the leader based on the last fetch request and update `readResult`.
+   * If the follower replica is not recognized to be one of the assigned replicas, do not update
+   * `readResult` so that log start/end offset and high watermark is consistent with
    * records in fetch response. Log start/end offset and high watermark may change not only due to
    * this fetch request, e.g., rolling new log segment and removing old log segment may move log
    * start offset further than the last offset in the fetched records. The followers will get the
    * updated leader's state in the next fetch response.
    */
-  private def updateFollowerLogReadResults(replicaId: Int,
-                                           readResults: Seq[(TopicPartition, LogReadResult)]): Seq[(TopicPartition, LogReadResult)] = {
-    debug(s"Recording follower broker $replicaId log end offsets: $readResults")
+  private def updateFollowerFetchState(followerId: Int,
+                                       readResults: Seq[(TopicPartition, LogReadResult)]): Seq[(TopicPartition, LogReadResult)] = {
     readResults.map { case (topicPartition, readResult) =>
-      var updatedReadResult = readResult
-      nonOfflinePartition(topicPartition) match {
-        case Some(partition) =>
-          partition.getReplica(replicaId) match {
-            case Some(replica) =>
-              partition.updateReplicaLogReadResult(replica, readResult)
-            case None =>
-              warn(s"Leader $localBrokerId failed to record follower $replicaId's position " +
+      val updatedReadResult = if (readResult.error != Errors.NONE) {
+        debug(s"Skipping update of fetch state for follower $followerId since the " +
+          s"log read returned error ${readResult.error}")
+        readResult
+      } else {
+        nonOfflinePartition(topicPartition) match {
+          case Some(partition) =>
+            if (partition.updateFollowerFetchState(followerId,
+              followerFetchOffsetMetadata = readResult.info.fetchOffsetMetadata,
+              followerStartOffset = readResult.followerLogStartOffset,
+              followerFetchTimeMs = readResult.fetchTimeMs,
+              leaderEndOffset = readResult.leaderLogEndOffset)) {
+              readResult
+            } else {
+              warn(s"Leader $localBrokerId failed to record follower $followerId's position " +
                 s"${readResult.info.fetchOffsetMetadata.messageOffset} since the replica is not recognized to be " +
                 s"one of the assigned replicas ${partition.assignedReplicas.map(_.brokerId).mkString(",")} " +
                 s"for partition $topicPartition. Empty records will be returned for this partition.")
-              updatedReadResult = readResult.withEmptyFetchInfo
-          }
-        case None =>
-          warn(s"While recording the replica LEO, the partition $topicPartition hasn't been created.")
+              readResult.withEmptyFetchInfo
+            }
+          case None =>
+            warn(s"While recording the replica LEO, the partition $topicPartition hasn't been created.")
+            readResult
+        }
       }
       topicPartition -> updatedReadResult
     }
@@ -1442,7 +1440,7 @@ class ReplicaManager(val config: KafkaConfig,
     }.filter(_.log.isDefined).toBuffer
     val replicasByDir = replicas.groupBy(_.log.get.dir.getParent)
     for ((dir, reps) <- replicasByDir) {
-      val hwms = reps.map(r => r.topicPartition -> r.highWatermark.messageOffset).toMap
+      val hwms = reps.map(r => r.topicPartition -> r.highWatermark).toMap
       try {
         highWatermarkCheckpoints.get(dir).foreach(_.write(hwms))
       } catch {
diff --git a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
index ae6fc00f1a63a..16a117be4e811 100644
--- a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
+++ b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
@@ -132,7 +132,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
 
     // wait until all the followers have synced the last HW with leader
     TestUtils.waitUntilTrue(() => servers.forall(server =>
-      server.replicaManager.localReplica(tp).get.highWatermark.messageOffset == numRecords
+      server.replicaManager.localReplica(tp).get.highWatermark == numRecords
     ), "Failed to update high watermark for followers after timeout")
 
     val scheduler = new BounceBrokerScheduler(numIters)
diff --git a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
index c5763adf59ad9..d604cd0b7950b 100644
--- a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
@@ -104,10 +104,10 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     )
 
     assertEquals(100, newLeaderServer.replicaManager.localReplicaOrException(topicPartition)
-      .highWatermark.messageOffset)
+      .highWatermark)
     val newFollowerServer = servers.find(_.config.brokerId == 102).get
     TestUtils.waitUntilTrue(() => newFollowerServer.replicaManager.localReplicaOrException(topicPartition)
-      .highWatermark.messageOffset == 100,
+      .highWatermark == 100,
       "partition follower's highWatermark should be 100")
   }
 
diff --git a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
index 2b12a04cf4d4b..650dce441f2d2 100644
--- a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
+++ b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
@@ -487,24 +487,20 @@ class PartitionTest {
 
     // after makeLeader(() call, partition should know about all the replicas
     val leaderReplica = partition.getReplica(leader).get
-    val follower1Replica = partition.getReplica(follower1).get
-    val follower2Replica = partition.getReplica(follower2).get
 
     // append records with initial leader epoch
     partition.appendRecordsToLeader(batch1, isFromClient = true)
     partition.appendRecordsToLeader(batch2, isFromClient = true)
-    assertEquals("Expected leader's HW not move", leaderReplica.logStartOffset, leaderReplica.highWatermark.messageOffset)
+    assertEquals("Expected leader's HW not move", leaderReplica.logStartOffset, leaderReplica.highWatermark)
 
     // let the follower in ISR move leader's HW to move further but below LEO
-    def readResult(fetchInfo: FetchDataInfo, leaderReplica: Replica): LogReadResult = {
-      LogReadResult(info = fetchInfo,
-        highWatermark = leaderReplica.highWatermark.messageOffset,
-        leaderLogStartOffset = leaderReplica.logStartOffset,
-        leaderLogEndOffset = leaderReplica.logEndOffset,
-        followerLogStartOffset = 0,
-        fetchTimeMs = time.milliseconds,
-        readSize = 10240,
-        lastStableOffset = None)
+    def updateFollowerFetchState(followerId: Int, fetchOffsetMetadata: LogOffsetMetadata): Unit = {
+      partition.updateFollowerFetchState(
+        followerId,
+        followerFetchOffsetMetadata = fetchOffsetMetadata,
+        followerStartOffset = 0L,
+        followerFetchTimeMs = time.milliseconds(),
+        leaderEndOffset = leaderReplica.logEndOffset)
     }
 
     def fetchOffsetsForTimestamp(timestamp: Long, isolation: Option[IsolationLevel]): Either[ApiException, Option[TimestampAndOffset]] = {
@@ -524,19 +520,14 @@ class PartitionTest {
       List(leader, follower2, follower1), 1)))
       .thenReturn(Some(2))
 
-    // Update follower 1
-    partition.updateReplicaLogReadResult(
-      follower1Replica, readResult(FetchDataInfo(LogOffsetMetadata(0), batch1), leaderReplica))
-    partition.updateReplicaLogReadResult(
-      follower1Replica, readResult(FetchDataInfo(LogOffsetMetadata(2), batch2), leaderReplica))
+    updateFollowerFetchState(follower1, LogOffsetMetadata(0))
+    updateFollowerFetchState(follower1, LogOffsetMetadata(2))
 
-    partition.updateReplicaLogReadResult(
-      follower2Replica, readResult(FetchDataInfo(LogOffsetMetadata(0), batch1), leaderReplica))
-    partition.updateReplicaLogReadResult(
-      follower2Replica, readResult(FetchDataInfo(LogOffsetMetadata(2), batch2), leaderReplica))
+    updateFollowerFetchState(follower2, LogOffsetMetadata(0))
+    updateFollowerFetchState(follower2, LogOffsetMetadata(2))
 
     // At this point, the leader has gotten 5 writes, but followers have only fetched two
-    assertEquals(2, partition.localReplica.get.highWatermark.messageOffset)
+    assertEquals(2, partition.localReplica.get.highWatermark)
 
     // Get the LEO
     fetchOffsetsForTimestamp(ListOffsetRequest.LATEST_TIMESTAMP, None) match {
@@ -609,10 +600,8 @@ class PartitionTest {
       .thenReturn(Some(2))
 
     // Next fetch from replicas, HW is moved up to 5 (ahead of the LEO)
-    partition.updateReplicaLogReadResult(
-      follower1Replica, readResult(FetchDataInfo(LogOffsetMetadata(5), MemoryRecords.EMPTY), leaderReplica))
-    partition.updateReplicaLogReadResult(
-      follower2Replica, readResult(FetchDataInfo(LogOffsetMetadata(5), MemoryRecords.EMPTY), leaderReplica))
+    updateFollowerFetchState(follower1, LogOffsetMetadata(5))
+    updateFollowerFetchState(follower2, LogOffsetMetadata(5))
 
     // Error goes away
     fetchOffsetsForTimestamp(ListOffsetRequest.LATEST_TIMESTAMP, Some(IsolationLevel.READ_UNCOMMITTED)) match {
@@ -747,7 +736,7 @@ class PartitionTest {
     assertEquals(0L, fetchLatestOffset(isolationLevel = Some(IsolationLevel.READ_UNCOMMITTED)).offset)
     assertEquals(0L, fetchLatestOffset(isolationLevel = Some(IsolationLevel.READ_COMMITTED)).offset)
 
-    replica.highWatermark = LogOffsetMetadata(1L)
+    replica.highWatermark = 1L
 
     assertEquals(3L, fetchLatestOffset(isolationLevel = None).offset)
     assertEquals(1L, fetchLatestOffset(isolationLevel = Some(IsolationLevel.READ_UNCOMMITTED)).offset)
@@ -822,30 +811,25 @@ class PartitionTest {
 
     // after makeLeader(() call, partition should know about all the replicas
     val leaderReplica = partition.getReplica(leader).get
-    val follower1Replica = partition.getReplica(follower1).get
-    val follower2Replica = partition.getReplica(follower2).get
 
     // append records with initial leader epoch
     val lastOffsetOfFirstBatch = partition.appendRecordsToLeader(batch1, isFromClient = true).lastOffset
     partition.appendRecordsToLeader(batch2, isFromClient = true)
-    assertEquals("Expected leader's HW not move", leaderReplica.logStartOffset, leaderReplica.highWatermark.messageOffset)
+    assertEquals("Expected leader's HW not move", leaderReplica.logStartOffset, leaderReplica.highWatermark)
 
     // let the follower in ISR move leader's HW to move further but below LEO
-    def readResult(fetchInfo: FetchDataInfo, leaderReplica: Replica): LogReadResult = {
-      LogReadResult(info = fetchInfo,
-                    highWatermark = leaderReplica.highWatermark.messageOffset,
-                    leaderLogStartOffset = leaderReplica.logStartOffset,
-                    leaderLogEndOffset = leaderReplica.logEndOffset,
-                    followerLogStartOffset = 0,
-                    fetchTimeMs = time.milliseconds,
-                    readSize = 10240,
-                    lastStableOffset = None)
+    def updateFollowerFetchState(followerId: Int, fetchOffsetMetadata: LogOffsetMetadata): Unit = {
+      partition.updateFollowerFetchState(
+        followerId,
+        followerFetchOffsetMetadata = fetchOffsetMetadata,
+        followerStartOffset = 0L,
+        followerFetchTimeMs = time.milliseconds(),
+        leaderEndOffset = leaderReplica.logEndOffset)
     }
-    partition.updateReplicaLogReadResult(
-      follower2Replica, readResult(FetchDataInfo(LogOffsetMetadata(0), batch1), leaderReplica))
-    partition.updateReplicaLogReadResult(
-      follower2Replica, readResult(FetchDataInfo(LogOffsetMetadata(lastOffsetOfFirstBatch), batch2), leaderReplica))
-    assertEquals("Expected leader's HW", lastOffsetOfFirstBatch, leaderReplica.highWatermark.messageOffset)
+
+    updateFollowerFetchState(follower2, LogOffsetMetadata(0))
+    updateFollowerFetchState(follower2, LogOffsetMetadata(lastOffsetOfFirstBatch))
+    assertEquals("Expected leader's HW", lastOffsetOfFirstBatch, leaderReplica.highWatermark)
 
     // current leader becomes follower and then leader again (without any new records appended)
     val followerState = new LeaderAndIsrRequest.PartitionState(controllerEpoch, follower2, leaderEpoch + 1, isr, 1,
@@ -862,18 +846,15 @@ class PartitionTest {
     partition.appendRecordsToLeader(batch3, isFromClient = true)
 
     // fetch from follower not in ISR from log start offset should not add this follower to ISR
-    partition.updateReplicaLogReadResult(follower1Replica,
-                                         readResult(FetchDataInfo(LogOffsetMetadata(0), batch1), leaderReplica))
-    partition.updateReplicaLogReadResult(follower1Replica,
-                                         readResult(FetchDataInfo(LogOffsetMetadata(lastOffsetOfFirstBatch), batch2), leaderReplica))
+    updateFollowerFetchState(follower1, LogOffsetMetadata(0))
+    updateFollowerFetchState(follower1, LogOffsetMetadata(lastOffsetOfFirstBatch))
     assertEquals("ISR", Set[Integer](leader, follower2), partition.inSyncReplicas.map(_.brokerId))
 
     // fetch from the follower not in ISR from start offset of the current leader epoch should
     // add this follower to ISR
     when(stateStore.expandIsr(controllerEpoch, new LeaderAndIsr(leader, leaderEpoch + 2,
       List(leader, follower2, follower1), 1))).thenReturn(Some(2))
-    partition.updateReplicaLogReadResult(follower1Replica,
-      readResult(FetchDataInfo(LogOffsetMetadata(currentLeaderEpochStartOffset), batch3), leaderReplica))
+    updateFollowerFetchState(follower1, LogOffsetMetadata(currentLeaderEpochStartOffset))
     assertEquals("ISR", Set[Integer](leader, follower1, follower2), partition.inSyncReplicas.map(_.brokerId))
   }
 
@@ -1012,19 +993,351 @@ class PartitionTest {
     assertTrue(partition.isAtMinIsr)
   }
 
+  @Test
+  def testUpdateFollowerFetchState(): Unit = {
+    val log = logManager.getOrCreateLog(topicPartition, logConfig)
+    seedLogData(log, numRecords = 6, leaderEpoch = 4)
+
+    val controllerId = 0
+    val controllerEpoch = 0
+    val leaderEpoch = 5
+    val remoteBrokerId = brokerId + 1
+    val replicas = List[Integer](brokerId, remoteBrokerId).asJava
+    val isr = replicas
+
+    doNothing().when(delayedOperations).checkAndCompleteFetch()
+
+    partition.getOrCreateReplica(brokerId, isNew = false, offsetCheckpoints)
+
+    val initializeTimeMs = time.milliseconds()
+    assertTrue("Expected become leader transition to succeed",
+      partition.makeLeader(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
+        leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
+
+    val remoteReplica = partition.getReplica(remoteBrokerId).get
+    assertEquals(initializeTimeMs, remoteReplica.lastCaughtUpTimeMs)
+    assertEquals(LogOffsetMetadata.UnknownOffsetMetadata.messageOffset, remoteReplica.logEndOffset)
+    assertEquals(Log.UnknownOffset, remoteReplica.logStartOffset)
+
+    time.sleep(500)
+
+    partition.updateFollowerFetchState(remoteBrokerId,
+      followerFetchOffsetMetadata = LogOffsetMetadata(3),
+      followerStartOffset = 0L,
+      followerFetchTimeMs = time.milliseconds(),
+      leaderEndOffset = 6L)
+
+    assertEquals(initializeTimeMs, remoteReplica.lastCaughtUpTimeMs)
+    assertEquals(3L, remoteReplica.logEndOffset)
+    assertEquals(0L, remoteReplica.logStartOffset)
+
+    time.sleep(500)
+
+    partition.updateFollowerFetchState(remoteBrokerId,
+      followerFetchOffsetMetadata = LogOffsetMetadata(6L),
+      followerStartOffset = 0L,
+      followerFetchTimeMs = time.milliseconds(),
+      leaderEndOffset = 6L)
+
+    assertEquals(time.milliseconds(), remoteReplica.lastCaughtUpTimeMs)
+    assertEquals(6L, remoteReplica.logEndOffset)
+    assertEquals(0L, remoteReplica.logStartOffset)
+  }
+
+  @Test
+  def testIsrExpansion(): Unit = {
+    val log = logManager.getOrCreateLog(topicPartition, logConfig)
+    seedLogData(log, numRecords = 10, leaderEpoch = 4)
+
+    val controllerId = 0
+    val controllerEpoch = 0
+    val leaderEpoch = 5
+    val remoteBrokerId = brokerId + 1
+    val replicas = List[Integer](brokerId, remoteBrokerId).asJava
+    val isr = List[Integer](brokerId).asJava
+
+    doNothing().when(delayedOperations).checkAndCompleteFetch()
+
+    partition.getOrCreateReplica(brokerId, isNew = false, offsetCheckpoints)
+    assertTrue("Expected become leader transition to succeed",
+      partition.makeLeader(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
+        leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
+    assertEquals(Set(brokerId), partition.inSyncReplicas.map(_.brokerId))
+
+    val remoteReplica = partition.getReplica(remoteBrokerId).get
+    assertEquals(LogOffsetMetadata.UnknownOffsetMetadata.messageOffset, remoteReplica.logEndOffset)
+    assertEquals(Log.UnknownOffset, remoteReplica.logStartOffset)
+
+    partition.updateFollowerFetchState(remoteBrokerId,
+      followerFetchOffsetMetadata = LogOffsetMetadata(3),
+      followerStartOffset = 0L,
+      followerFetchTimeMs = time.milliseconds(),
+      leaderEndOffset = 6L)
+
+    assertEquals(Set(brokerId), partition.inSyncReplicas.map(_.brokerId))
+    assertEquals(3L, remoteReplica.logEndOffset)
+    assertEquals(0L, remoteReplica.logStartOffset)
+
+    // The next update should bring the follower back into the ISR
+    val updatedLeaderAndIsr = LeaderAndIsr(
+      leader = brokerId,
+      leaderEpoch = leaderEpoch,
+      isr = List(brokerId, remoteBrokerId),
+      zkVersion = 1)
+    when(stateStore.expandIsr(controllerEpoch, updatedLeaderAndIsr)).thenReturn(Some(2))
+
+    partition.updateFollowerFetchState(remoteBrokerId,
+      followerFetchOffsetMetadata = LogOffsetMetadata(10),
+      followerStartOffset = 0L,
+      followerFetchTimeMs = time.milliseconds(),
+      leaderEndOffset = 6L)
+
+    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas.map(_.brokerId))
+    assertEquals(10L, remoteReplica.logEndOffset)
+    assertEquals(0L, remoteReplica.logStartOffset)
+  }
+
+  @Test
+  def testIsrNotExpandedIfUpdateFails(): Unit = {
+    val log = logManager.getOrCreateLog(topicPartition, logConfig)
+    seedLogData(log, numRecords = 10, leaderEpoch = 4)
+
+    val controllerId = 0
+    val controllerEpoch = 0
+    val leaderEpoch = 5
+    val remoteBrokerId = brokerId + 1
+    val replicas = List[Integer](brokerId, remoteBrokerId).asJava
+    val isr = List[Integer](brokerId).asJava
+
+    doNothing().when(delayedOperations).checkAndCompleteFetch()
+
+    partition.getOrCreateReplica(brokerId, isNew = false, offsetCheckpoints)
+    assertTrue("Expected become leader transition to succeed",
+      partition.makeLeader(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
+        leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
+    assertEquals(Set(brokerId), partition.inSyncReplicas.map(_.brokerId))
+
+    val remoteReplica = partition.getReplica(remoteBrokerId).get
+    assertEquals(LogOffsetMetadata.UnknownOffsetMetadata.messageOffset, remoteReplica.logEndOffset)
+    assertEquals(Log.UnknownOffset, remoteReplica.logStartOffset)
+
+    // Mock the expected ISR update failure
+    val updatedLeaderAndIsr = LeaderAndIsr(
+      leader = brokerId,
+      leaderEpoch = leaderEpoch,
+      isr = List(brokerId, remoteBrokerId),
+      zkVersion = 1)
+    when(stateStore.expandIsr(controllerEpoch, updatedLeaderAndIsr)).thenReturn(None)
+
+    partition.updateFollowerFetchState(remoteBrokerId,
+      followerFetchOffsetMetadata = LogOffsetMetadata(10),
+      followerStartOffset = 0L,
+      followerFetchTimeMs = time.milliseconds(),
+      leaderEndOffset = 10L)
+
+    // Follower state is updated, but the ISR has not expanded
+    assertEquals(Set(brokerId), partition.inSyncReplicas.map(_.brokerId))
+    assertEquals(10L, remoteReplica.logEndOffset)
+    assertEquals(0L, remoteReplica.logStartOffset)
+  }
+
+  @Test
+  def testMaybeShrinkIsr(): Unit = {
+    val log = logManager.getOrCreateLog(topicPartition, logConfig)
+    seedLogData(log, numRecords = 10, leaderEpoch = 4)
+
+    val controllerId = 0
+    val controllerEpoch = 0
+    val leaderEpoch = 5
+    val remoteBrokerId = brokerId + 1
+    val replicas = List[Integer](brokerId, remoteBrokerId).asJava
+    val isr = List[Integer](brokerId, remoteBrokerId).asJava
+
+    doNothing().when(delayedOperations).checkAndCompleteFetch()
+
+    val initializeTimeMs = time.milliseconds()
+    partition.getOrCreateReplica(brokerId, isNew = false, offsetCheckpoints)
+    assertTrue("Expected become leader transition to succeed",
+      partition.makeLeader(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
+        leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
+    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas.map(_.brokerId))
+    assertEquals(0L, partition.localReplicaOrException.highWatermark)
+
+    val remoteReplica = partition.getReplica(remoteBrokerId).get
+    assertEquals(initializeTimeMs, remoteReplica.lastCaughtUpTimeMs)
+    assertEquals(LogOffsetMetadata.UnknownOffsetMetadata.messageOffset, remoteReplica.logEndOffset)
+    assertEquals(Log.UnknownOffset, remoteReplica.logStartOffset)
+
+    // On initialization, the replica is considered caught up and should not be removed
+    partition.maybeShrinkIsr(10000)
+    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas.map(_.brokerId))
+
+    // If enough time passes without a fetch update, the ISR should shrink
+    time.sleep(10001)
+    val updatedLeaderAndIsr = LeaderAndIsr(
+      leader = brokerId,
+      leaderEpoch = leaderEpoch,
+      isr = List(brokerId),
+      zkVersion = 1)
+    when(stateStore.shrinkIsr(controllerEpoch, updatedLeaderAndIsr)).thenReturn(Some(2))
+
+    partition.maybeShrinkIsr(10000)
+    assertEquals(Set(brokerId), partition.inSyncReplicas.map(_.brokerId))
+    assertEquals(10L, partition.localReplicaOrException.highWatermark)
+  }
+
+  @Test
+  def testShouldNotShrinkIsrIfPreviousFetchIsCaughtUp(): Unit = {
+    val log = logManager.getOrCreateLog(topicPartition, logConfig)
+    seedLogData(log, numRecords = 10, leaderEpoch = 4)
+
+    val controllerId = 0
+    val controllerEpoch = 0
+    val leaderEpoch = 5
+    val remoteBrokerId = brokerId + 1
+    val replicas = List[Integer](brokerId, remoteBrokerId).asJava
+    val isr = List[Integer](brokerId, remoteBrokerId).asJava
+
+    doNothing().when(delayedOperations).checkAndCompleteFetch()
+
+    val initializeTimeMs = time.milliseconds()
+    partition.getOrCreateReplica(brokerId, isNew = false, offsetCheckpoints)
+    assertTrue("Expected become leader transition to succeed",
+      partition.makeLeader(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
+        leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
+    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas.map(_.brokerId))
+    assertEquals(0L, partition.localReplicaOrException.highWatermark)
+
+    val remoteReplica = partition.getReplica(remoteBrokerId).get
+    assertEquals(initializeTimeMs, remoteReplica.lastCaughtUpTimeMs)
+    assertEquals(LogOffsetMetadata.UnknownOffsetMetadata.messageOffset, remoteReplica.logEndOffset)
+    assertEquals(Log.UnknownOffset, remoteReplica.logStartOffset)
+
+    // There is a short delay before the first fetch. The follower is not yet caught up to the log end.
+    time.sleep(5000)
+    val firstFetchTimeMs = time.milliseconds()
+    partition.updateFollowerFetchState(remoteBrokerId,
+      followerFetchOffsetMetadata = LogOffsetMetadata(5),
+      followerStartOffset = 0L,
+      followerFetchTimeMs = firstFetchTimeMs,
+      leaderEndOffset = 10L)
+    assertEquals(initializeTimeMs, remoteReplica.lastCaughtUpTimeMs)
+    assertEquals(5L, partition.localReplicaOrException.highWatermark)
+    assertEquals(5L, remoteReplica.logEndOffset)
+    assertEquals(0L, remoteReplica.logStartOffset)
+
+    // Some new data is appended, but the follower catches up to the old end offset.
+    // The total elapsed time from initialization is larger than the max allowed replica lag.
+    time.sleep(5001)
+    seedLogData(log, numRecords = 5, leaderEpoch = leaderEpoch)
+    partition.updateFollowerFetchState(remoteBrokerId,
+      followerFetchOffsetMetadata = LogOffsetMetadata(10),
+      followerStartOffset = 0L,
+      followerFetchTimeMs = time.milliseconds(),
+      leaderEndOffset = 15L)
+    assertEquals(firstFetchTimeMs, remoteReplica.lastCaughtUpTimeMs)
+    assertEquals(10L, partition.localReplicaOrException.highWatermark)
+    assertEquals(10L, remoteReplica.logEndOffset)
+    assertEquals(0L, remoteReplica.logStartOffset)
+
+    // The ISR should not be shrunk because the follower has caught up with the leader at the
+    // time of the first fetch.
+    partition.maybeShrinkIsr(10000)
+    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas.map(_.brokerId))
+  }
+
+  @Test
+  def testShouldNotShrinkIsrIfFollowerCaughtUpToLogEnd(): Unit = {
+    val log = logManager.getOrCreateLog(topicPartition, logConfig)
+    seedLogData(log, numRecords = 10, leaderEpoch = 4)
+
+    val controllerId = 0
+    val controllerEpoch = 0
+    val leaderEpoch = 5
+    val remoteBrokerId = brokerId + 1
+    val replicas = List[Integer](brokerId, remoteBrokerId).asJava
+    val isr = List[Integer](brokerId, remoteBrokerId).asJava
+
+    doNothing().when(delayedOperations).checkAndCompleteFetch()
+
+    val initializeTimeMs = time.milliseconds()
+    partition.getOrCreateReplica(brokerId, isNew = false, offsetCheckpoints)
+    assertTrue("Expected become leader transition to succeed",
+      partition.makeLeader(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
+        leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
+    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas.map(_.brokerId))
+    assertEquals(0L, partition.localReplicaOrException.highWatermark)
+
+    val remoteReplica = partition.getReplica(remoteBrokerId).get
+    assertEquals(initializeTimeMs, remoteReplica.lastCaughtUpTimeMs)
+    assertEquals(LogOffsetMetadata.UnknownOffsetMetadata.messageOffset, remoteReplica.logEndOffset)
+    assertEquals(Log.UnknownOffset, remoteReplica.logStartOffset)
+
+    // The follower catches up to the log end immediately.
+    partition.updateFollowerFetchState(remoteBrokerId,
+      followerFetchOffsetMetadata = LogOffsetMetadata(10),
+      followerStartOffset = 0L,
+      followerFetchTimeMs = time.milliseconds(),
+      leaderEndOffset = 10L)
+    assertEquals(initializeTimeMs, remoteReplica.lastCaughtUpTimeMs)
+    assertEquals(10L, partition.localReplicaOrException.highWatermark)
+    assertEquals(10L, remoteReplica.logEndOffset)
+    assertEquals(0L, remoteReplica.logStartOffset)
+
+    // Sleep longer than the max allowed follower lag
+    time.sleep(10001)
+
+    // The ISR should not be shrunk because the follower is caught up to the leader's log end
+    partition.maybeShrinkIsr(10000)
+    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas.map(_.brokerId))
+  }
+
+  @Test
+  def testIsrNotShrunkIfUpdateFails(): Unit = {
+    val log = logManager.getOrCreateLog(topicPartition, logConfig)
+    seedLogData(log, numRecords = 10, leaderEpoch = 4)
+
+    val controllerId = 0
+    val controllerEpoch = 0
+    val leaderEpoch = 5
+    val remoteBrokerId = brokerId + 1
+    val replicas = List[Integer](brokerId, remoteBrokerId).asJava
+    val isr = List[Integer](brokerId, remoteBrokerId).asJava
+
+    doNothing().when(delayedOperations).checkAndCompleteFetch()
+
+    val initializeTimeMs = time.milliseconds()
+    partition.getOrCreateReplica(brokerId, isNew = false, offsetCheckpoints)
+    assertTrue("Expected become leader transition to succeed",
+      partition.makeLeader(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
+        leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
+    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas.map(_.brokerId))
+    assertEquals(0L, partition.localReplicaOrException.highWatermark)
+
+    val remoteReplica = partition.getReplica(remoteBrokerId).get
+    assertEquals(initializeTimeMs, remoteReplica.lastCaughtUpTimeMs)
+    assertEquals(LogOffsetMetadata.UnknownOffsetMetadata.messageOffset, remoteReplica.logEndOffset)
+    assertEquals(Log.UnknownOffset, remoteReplica.logStartOffset)
+
+    time.sleep(10001)
+
+    // Mock the expected ISR update failure
+    val updatedLeaderAndIsr = LeaderAndIsr(
+      leader = brokerId,
+      leaderEpoch = leaderEpoch,
+      isr = List(brokerId),
+      zkVersion = 1)
+    when(stateStore.shrinkIsr(controllerEpoch, updatedLeaderAndIsr)).thenReturn(None)
+
+    partition.maybeShrinkIsr(10000)
+    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas.map(_.brokerId))
+    assertEquals(0L, partition.localReplicaOrException.highWatermark)
+  }
+
   @Test
   def testUseCheckpointToInitializeHighWatermark(): Unit = {
     val log = logManager.getOrCreateLog(topicPartition, logConfig)
-    log.appendAsLeader(MemoryRecords.withRecords(0L, CompressionType.NONE, 0,
-      new SimpleRecord("k1".getBytes, "v1".getBytes),
-      new SimpleRecord("k2".getBytes, "v2".getBytes),
-      new SimpleRecord("k3".getBytes, "v3".getBytes),
-      new SimpleRecord("k4".getBytes, "v4".getBytes)
-    ), leaderEpoch = 0)
-    log.appendAsLeader(MemoryRecords.withRecords(0L, CompressionType.NONE, 5,
-      new SimpleRecord("k5".getBytes, "v5".getBytes),
-      new SimpleRecord("k5".getBytes, "v5".getBytes)
-    ), leaderEpoch = 5)
+    seedLogData(log, numRecords = 6, leaderEpoch = 5)
 
     when(offsetCheckpoints.fetch(logDir1.getAbsolutePath, topicPartition))
       .thenReturn(Some(4L))
@@ -1035,7 +1348,7 @@ class PartitionTest {
     val leaderState = new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
       6, replicas, 1, replicas, false)
     partition.makeLeader(controllerId, leaderState, 0, offsetCheckpoints)
-    assertEquals(4, partition.localReplicaOrException.highWatermark.messageOffset)
+    assertEquals(4, partition.localReplicaOrException.highWatermark)
   }
 
   @Test
@@ -1061,4 +1374,12 @@ class PartitionTest {
     assertEquals(Set(), Metrics.defaultRegistry().allMetrics().asScala.keySet.filter(_.getType == "Partition"))
   }
 
+  private def seedLogData(log: Log, numRecords: Int, leaderEpoch: Int): Unit = {
+    for (i <- 0 until numRecords) {
+      val records = MemoryRecords.withRecords(0L, CompressionType.NONE, leaderEpoch,
+        new SimpleRecord(s"k$i".getBytes, s"v$i".getBytes))
+      log.appendAsLeader(records, leaderEpoch)
+    }
+  }
+
 }
diff --git a/core/src/test/scala/unit/kafka/cluster/ReplicaTest.scala b/core/src/test/scala/unit/kafka/cluster/ReplicaTest.scala
index d63901a3c3462..bacdc8151bd37 100644
--- a/core/src/test/scala/unit/kafka/cluster/ReplicaTest.scala
+++ b/core/src/test/scala/unit/kafka/cluster/ReplicaTest.scala
@@ -76,7 +76,7 @@ class ReplicaTest {
       initialHighWatermarkValue = initialHighWatermark,
       log = Some(log))
 
-    assertEquals(initialHighWatermark, replica.highWatermark.messageOffset)
+    assertEquals(initialHighWatermark, replica.highWatermark)
 
     val expiredTimestamp = time.milliseconds() - 1000
     for (i <- 0 until 100) {
@@ -100,13 +100,13 @@ class ReplicaTest {
 
     // ensure we have at least a few segments so the test case is not trivial
     assertTrue(log.numberOfSegments > 5)
-    assertEquals(0L, replica.highWatermark.messageOffset)
+    assertEquals(0L, replica.highWatermark)
     assertEquals(0L, replica.logStartOffset)
     assertEquals(100L, replica.logEndOffset)
 
     for (hw <- 0 to 100) {
-      replica.highWatermark = new LogOffsetMetadata(hw)
-      assertEquals(hw, replica.highWatermark.messageOffset)
+      replica.highWatermark = hw
+      assertEquals(hw, replica.highWatermark)
       log.deleteOldSegments()
       assertTrue(replica.logStartOffset <= hw)
 
@@ -134,7 +134,7 @@ class ReplicaTest {
       log.appendAsLeader(records, leaderEpoch = 0)
     }
 
-    replica.highWatermark = new LogOffsetMetadata(25L)
+    replica.highWatermark = 25L
     replica.maybeIncrementLogStartOffset(26L)
   }
 
diff --git a/core/src/test/scala/unit/kafka/server/HighwatermarkPersistenceTest.scala b/core/src/test/scala/unit/kafka/server/HighwatermarkPersistenceTest.scala
index 3da22bb871409..61c521b153d11 100755
--- a/core/src/test/scala/unit/kafka/server/HighwatermarkPersistenceTest.scala
+++ b/core/src/test/scala/unit/kafka/server/HighwatermarkPersistenceTest.scala
@@ -81,12 +81,12 @@ class HighwatermarkPersistenceTest {
       partition0.addReplicaIfNotExists(followerReplicaPartition0)
       replicaManager.checkpointHighWatermarks()
       fooPartition0Hw = hwmFor(replicaManager, topic, 0)
-      assertEquals(leaderReplicaPartition0.highWatermark.messageOffset, fooPartition0Hw)
+      assertEquals(leaderReplicaPartition0.highWatermark, fooPartition0Hw)
       // set the high watermark for local replica
-      partition0.localReplica.get.highWatermark = new LogOffsetMetadata(5L)
+      partition0.localReplica.get.highWatermark = 5L
       replicaManager.checkpointHighWatermarks()
       fooPartition0Hw = hwmFor(replicaManager, topic, 0)
-      assertEquals(leaderReplicaPartition0.highWatermark.messageOffset, fooPartition0Hw)
+      assertEquals(leaderReplicaPartition0.highWatermark, fooPartition0Hw)
       EasyMock.verify(zkClient)
     } finally {
       // shutdown the replica manager upon test completion
@@ -125,12 +125,12 @@ class HighwatermarkPersistenceTest {
       topic1Partition0.addReplicaIfNotExists(leaderReplicaTopic1Partition0)
       replicaManager.checkpointHighWatermarks()
       topic1Partition0Hw = hwmFor(replicaManager, topic1, 0)
-      assertEquals(leaderReplicaTopic1Partition0.highWatermark.messageOffset, topic1Partition0Hw)
+      assertEquals(leaderReplicaTopic1Partition0.highWatermark, topic1Partition0Hw)
       // set the high watermark for local replica
-      topic1Partition0.localReplica.get.highWatermark = new LogOffsetMetadata(5L)
+      topic1Partition0.localReplica.get.highWatermark = 5L
       replicaManager.checkpointHighWatermarks()
       topic1Partition0Hw = hwmFor(replicaManager, topic1, 0)
-      assertEquals(5L, leaderReplicaTopic1Partition0.highWatermark.messageOffset)
+      assertEquals(5L, leaderReplicaTopic1Partition0.highWatermark)
       assertEquals(5L, topic1Partition0Hw)
       // add another partition and set highwatermark
       val t2p0 = new TopicPartition(topic2, 0)
@@ -142,13 +142,13 @@ class HighwatermarkPersistenceTest {
       topic2Partition0.addReplicaIfNotExists(leaderReplicaTopic2Partition0)
       replicaManager.checkpointHighWatermarks()
       var topic2Partition0Hw = hwmFor(replicaManager, topic2, 0)
-      assertEquals(leaderReplicaTopic2Partition0.highWatermark.messageOffset, topic2Partition0Hw)
+      assertEquals(leaderReplicaTopic2Partition0.highWatermark, topic2Partition0Hw)
       // set the highwatermark for local replica
-      topic2Partition0.localReplica.get.highWatermark = new LogOffsetMetadata(15L)
-      assertEquals(15L, leaderReplicaTopic2Partition0.highWatermark.messageOffset)
+      topic2Partition0.localReplica.get.highWatermark = 15L
+      assertEquals(15L, leaderReplicaTopic2Partition0.highWatermark)
       // change the highwatermark for topic1
-      topic1Partition0.localReplica.get.highWatermark = new LogOffsetMetadata(10L)
-      assertEquals(10L, leaderReplicaTopic1Partition0.highWatermark.messageOffset)
+      topic1Partition0.localReplica.get.highWatermark = 10L
+      assertEquals(10L, leaderReplicaTopic1Partition0.highWatermark)
       replicaManager.checkpointHighWatermarks()
       // verify checkpointed hw for topic 2
       topic2Partition0Hw = hwmFor(replicaManager, topic2, 0)
diff --git a/core/src/test/scala/unit/kafka/server/ISRExpirationTest.scala b/core/src/test/scala/unit/kafka/server/ISRExpirationTest.scala
index 1dd4b24c49994..0ee0fa164d9d2 100644
--- a/core/src/test/scala/unit/kafka/server/ISRExpirationTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ISRExpirationTest.scala
@@ -25,7 +25,6 @@ import kafka.log.{Log, LogManager}
 import kafka.utils._
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.metrics.Metrics
-import org.apache.kafka.common.record.MemoryRecords
 import org.apache.kafka.common.utils.Time
 import org.easymock.EasyMock
 import org.junit.Assert._
@@ -82,14 +81,11 @@ class IsrExpirationTest {
 
     // let the follower catch up to the Leader logEndOffset - 1
     for (replica <- partition0.assignedReplicas - leaderReplica)
-      replica.updateLogReadResult(new LogReadResult(info = FetchDataInfo(new LogOffsetMetadata(leaderLogEndOffset - 1), MemoryRecords.EMPTY),
-                                                    highWatermark = leaderLogEndOffset - 1,
-                                                    leaderLogStartOffset = 0L,
-                                                    leaderLogEndOffset = leaderLogEndOffset,
-                                                    followerLogStartOffset = 0L,
-                                                    fetchTimeMs = time.milliseconds,
-                                                    readSize = -1,
-                                                    lastStableOffset = None))
+      replica.updateFetchState(
+        followerFetchOffsetMetadata = new LogOffsetMetadata(leaderLogEndOffset - 1),
+        followerStartOffset = 0L,
+        followerFetchTimeMs= time.milliseconds,
+        leaderEndOffset = leaderLogEndOffset)
     var partition0OSR = partition0.getOutOfSyncReplicas(leaderReplica, configs.head.replicaLagTimeMaxMs)
     assertEquals("No replica should be out of sync", Set.empty[Int], partition0OSR.map(_.brokerId))
 
@@ -137,14 +133,11 @@ class IsrExpirationTest {
 
     // Make the remote replica not read to the end of log. It should be not be out of sync for at least 100 ms
     for (replica <- partition0.assignedReplicas - leaderReplica)
-      replica.updateLogReadResult(new LogReadResult(info = FetchDataInfo(new LogOffsetMetadata(leaderLogEndOffset - 2), MemoryRecords.EMPTY),
-                                                    highWatermark = leaderLogEndOffset - 2,
-                                                    leaderLogStartOffset = 0L,
-                                                    leaderLogEndOffset = leaderLogEndOffset,
-                                                    followerLogStartOffset = 0L,
-                                                    fetchTimeMs = time.milliseconds,
-                                                    readSize = -1,
-                                                    lastStableOffset = None))
+      replica.updateFetchState(
+        followerFetchOffsetMetadata = new LogOffsetMetadata(leaderLogEndOffset - 2),
+        followerStartOffset = 0L,
+        followerFetchTimeMs= time.milliseconds,
+        leaderEndOffset = leaderLogEndOffset)
 
     // Simulate 2 fetch requests spanning more than 100 ms which do not read to the end of the log.
     // The replicas will no longer be in ISR. We do 2 fetches because we want to simulate the case where the replica is lagging but is not stuck
@@ -154,14 +147,11 @@ class IsrExpirationTest {
     time.sleep(75)
 
     (partition0.assignedReplicas - leaderReplica).foreach { r =>
-      r.updateLogReadResult(new LogReadResult(info = FetchDataInfo(new LogOffsetMetadata(leaderLogEndOffset - 1), MemoryRecords.EMPTY),
-                            highWatermark = leaderLogEndOffset - 1,
-                            leaderLogStartOffset = 0L,
-                            leaderLogEndOffset = leaderLogEndOffset,
-                            followerLogStartOffset = 0L,
-                            fetchTimeMs = time.milliseconds,
-                            readSize = -1,
-                            lastStableOffset = None))
+      r.updateFetchState(
+        followerFetchOffsetMetadata = new LogOffsetMetadata(leaderLogEndOffset - 1),
+        followerStartOffset = 0L,
+        followerFetchTimeMs= time.milliseconds,
+        leaderEndOffset = leaderLogEndOffset)
     }
     partition0OSR = partition0.getOutOfSyncReplicas(leaderReplica, configs.head.replicaLagTimeMaxMs)
     assertEquals("No replica should be out of sync", Set.empty[Int], partition0OSR.map(_.brokerId))
@@ -174,14 +164,11 @@ class IsrExpirationTest {
 
     // Now actually make a fetch to the end of the log. The replicas should be back in ISR
     (partition0.assignedReplicas - leaderReplica).foreach { r =>
-      r.updateLogReadResult(new LogReadResult(info = FetchDataInfo(new LogOffsetMetadata(leaderLogEndOffset), MemoryRecords.EMPTY),
-                            highWatermark = leaderLogEndOffset,
-                            leaderLogStartOffset = 0L,
-                            leaderLogEndOffset = leaderLogEndOffset,
-                            followerLogStartOffset = 0L,
-                            fetchTimeMs = time.milliseconds,
-                            readSize = -1,
-                            lastStableOffset = None))
+      r.updateFetchState(
+        followerFetchOffsetMetadata = new LogOffsetMetadata(leaderLogEndOffset),
+        followerStartOffset = 0L,
+        followerFetchTimeMs= time.milliseconds,
+        leaderEndOffset = leaderLogEndOffset)
     }
     partition0OSR = partition0.getOutOfSyncReplicas(leaderReplica, configs.head.replicaLagTimeMaxMs)
     assertEquals("No replica should be out of sync", Set.empty[Int], partition0OSR.map(_.brokerId))
@@ -203,14 +190,12 @@ class IsrExpirationTest {
 
     // let the follower catch up to the Leader logEndOffset
     for (replica <- partition0.assignedReplicas - leaderReplica)
-      replica.updateLogReadResult(new LogReadResult(info = FetchDataInfo(new LogOffsetMetadata(leaderLogEndOffset), MemoryRecords.EMPTY),
-        highWatermark = leaderLogEndOffset,
-        leaderLogStartOffset = 0L,
-        leaderLogEndOffset = leaderLogEndOffset,
-        followerLogStartOffset = 0L,
-        fetchTimeMs = time.milliseconds,
-        readSize = -1,
-        lastStableOffset = None))
+      replica.updateFetchState(
+        followerFetchOffsetMetadata = new LogOffsetMetadata(leaderLogEndOffset),
+        followerStartOffset = 0L,
+        followerFetchTimeMs= time.milliseconds,
+        leaderEndOffset = leaderLogEndOffset)
+
     var partition0OSR = partition0.getOutOfSyncReplicas(leaderReplica, configs.head.replicaLagTimeMaxMs)
     assertEquals("No replica should be out of sync", Set.empty[Int], partition0OSR.map(_.brokerId))
 
@@ -236,14 +221,12 @@ class IsrExpirationTest {
     partition.inSyncReplicas = allReplicas.toSet
     // set lastCaughtUpTime to current time
     for (replica <- partition.assignedReplicas - leaderReplica)
-      replica.updateLogReadResult(new LogReadResult(info = FetchDataInfo(new LogOffsetMetadata(0L), MemoryRecords.EMPTY),
-                                                    highWatermark = 0L,
-                                                    leaderLogStartOffset = 0L,
-                                                    leaderLogEndOffset = 0L,
-                                                    followerLogStartOffset = 0L,
-                                                    fetchTimeMs = time.milliseconds,
-                                                    readSize = -1,
-                                                    lastStableOffset = None))
+      replica.updateFetchState(
+        followerFetchOffsetMetadata = new LogOffsetMetadata(0L),
+        followerStartOffset = 0L,
+        followerFetchTimeMs= time.milliseconds,
+        leaderEndOffset = 0L)
+
     // set the leader and its hw and the hw update time
     partition.leaderReplicaIdOpt = Some(leaderId)
     partition
diff --git a/core/src/test/scala/unit/kafka/server/LogRecoveryTest.scala b/core/src/test/scala/unit/kafka/server/LogRecoveryTest.scala
index 6ab31385bc08d..e868f6c4a8b19 100755
--- a/core/src/test/scala/unit/kafka/server/LogRecoveryTest.scala
+++ b/core/src/test/scala/unit/kafka/server/LogRecoveryTest.scala
@@ -104,7 +104,7 @@ class LogRecoveryTest extends ZooKeeperTestHarness {
 
     // give some time for the follower 1 to record leader HW
     TestUtils.waitUntilTrue(() =>
-      server2.replicaManager.localReplica(topicPartition).get.highWatermark.messageOffset == numMessages,
+      server2.replicaManager.localReplica(topicPartition).get.highWatermark == numMessages,
       "Failed to update high watermark for follower after timeout")
 
     servers.foreach(_.replicaManager.checkpointHighWatermarks())
@@ -166,7 +166,7 @@ class LogRecoveryTest extends ZooKeeperTestHarness {
 
     // give some time for follower 1 to record leader HW of 60
     TestUtils.waitUntilTrue(() =>
-      server2.replicaManager.localReplica(topicPartition).get.highWatermark.messageOffset == hw,
+      server2.replicaManager.localReplica(topicPartition).get.highWatermark == hw,
       "Failed to update high watermark for follower after timeout")
     // shutdown the servers to allow the hw to be checkpointed
     servers.foreach(_.shutdown())
@@ -180,7 +180,7 @@ class LogRecoveryTest extends ZooKeeperTestHarness {
     val hw = 20L
     // give some time for follower 1 to record leader HW of 600
     TestUtils.waitUntilTrue(() =>
-      server2.replicaManager.localReplica(topicPartition).get.highWatermark.messageOffset == hw,
+      server2.replicaManager.localReplica(topicPartition).get.highWatermark == hw,
       "Failed to update high watermark for follower after timeout")
     // shutdown the servers to allow the hw to be checkpointed
     servers.foreach(_.shutdown())
@@ -199,7 +199,7 @@ class LogRecoveryTest extends ZooKeeperTestHarness {
 
     // allow some time for the follower to get the leader HW
     TestUtils.waitUntilTrue(() =>
-      server2.replicaManager.localReplica(topicPartition).get.highWatermark.messageOffset == hw,
+      server2.replicaManager.localReplica(topicPartition).get.highWatermark == hw,
       "Failed to update high watermark for follower after timeout")
     // kill the server hosting the preferred replica
     server1.shutdown()
@@ -230,7 +230,7 @@ class LogRecoveryTest extends ZooKeeperTestHarness {
       "Failed to create replica in follower after timeout")
     // allow some time for the follower to get the leader HW
     TestUtils.waitUntilTrue(() =>
-      server1.replicaManager.localReplica(topicPartition).get.highWatermark.messageOffset == hw,
+      server1.replicaManager.localReplica(topicPartition).get.highWatermark == hw,
       "Failed to update high watermark for follower after timeout")
     // shutdown the servers to allow the hw to be checkpointed
     servers.foreach(_.shutdown())
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala
index a51641adf5c57..5400f8698243d 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala
@@ -87,7 +87,7 @@ class ReplicaFetcherThreadTest {
 
     //Stubs
     expect(replica.logEndOffset).andReturn(0).anyTimes()
-    expect(replica.highWatermark).andReturn(new LogOffsetMetadata(0)).anyTimes()
+    expect(replica.highWatermark).andReturn(0L).anyTimes()
     expect(replica.latestEpoch).andReturn(Some(leaderEpoch)).once()
     expect(replica.latestEpoch).andReturn(Some(leaderEpoch)).once()
     expect(replica.latestEpoch).andReturn(None).once()  // t2p1 doesnt support epochs
@@ -218,7 +218,7 @@ class ReplicaFetcherThreadTest {
 
     //Stubs
     expect(replica.logEndOffset).andReturn(0).anyTimes()
-    expect(replica.highWatermark).andReturn(new LogOffsetMetadata(0)).anyTimes()
+    expect(replica.highWatermark).andReturn(0L).anyTimes()
     expect(replica.latestEpoch).andReturn(Some(leaderEpoch)).anyTimes()
     expect(replica.endOffsetForEpoch(leaderEpoch)).andReturn(
       Some(OffsetAndEpoch(0, leaderEpoch))).anyTimes()
@@ -280,7 +280,7 @@ class ReplicaFetcherThreadTest {
     //Stubs
     expect(partition.truncateTo(capture(truncateToCapture), anyBoolean())).anyTimes()
     expect(replica.logEndOffset).andReturn(initialLEO).anyTimes()
-    expect(replica.highWatermark).andReturn(new LogOffsetMetadata(initialLEO - 1)).anyTimes()
+    expect(replica.highWatermark).andReturn(initialLEO - 1).anyTimes()
     expect(replica.latestEpoch).andReturn(Some(leaderEpoch)).anyTimes()
     expect(replica.endOffsetForEpoch(leaderEpoch)).andReturn(
       Some(OffsetAndEpoch(initialLEO, leaderEpoch))).anyTimes()
@@ -329,7 +329,7 @@ class ReplicaFetcherThreadTest {
     //Stubs
     expect(partition.truncateTo(capture(truncateToCapture), anyBoolean())).anyTimes()
     expect(replica.logEndOffset).andReturn(initialLEO).anyTimes()
-    expect(replica.highWatermark).andReturn(new LogOffsetMetadata(initialLEO - 3)).anyTimes()
+    expect(replica.highWatermark).andReturn(initialLEO - 3).anyTimes()
     expect(replica.latestEpoch).andReturn(Some(leaderEpochAtFollower)).anyTimes()
     expect(replica.endOffsetForEpoch(leaderEpochAtLeader)).andReturn(None).anyTimes()
     expect(replicaManager.logManager).andReturn(logManager).anyTimes()
@@ -379,7 +379,7 @@ class ReplicaFetcherThreadTest {
     // Stubs
     expect(partition.truncateTo(capture(truncateToCapture), anyBoolean())).anyTimes()
     expect(replica.logEndOffset).andReturn(initialLEO).anyTimes()
-    expect(replica.highWatermark).andReturn(new LogOffsetMetadata(initialLEO - 2)).anyTimes()
+    expect(replica.highWatermark).andReturn(initialLEO - 2).anyTimes()
     expect(replica.latestEpoch).andReturn(Some(5)).anyTimes()
     expect(replica.endOffsetForEpoch(4)).andReturn(
       Some(OffsetAndEpoch(120, 3))).anyTimes()
@@ -450,7 +450,7 @@ class ReplicaFetcherThreadTest {
     // Stubs
     expect(partition.truncateTo(capture(truncateToCapture), anyBoolean())).anyTimes()
     expect(replica.logEndOffset).andReturn(initialLEO).anyTimes()
-    expect(replica.highWatermark).andReturn(new LogOffsetMetadata(initialLEO - 2)).anyTimes()
+    expect(replica.highWatermark).andReturn(initialLEO - 2).anyTimes()
     expect(replica.latestEpoch).andReturn(Some(5)).anyTimes()
     expect(replica.endOffsetForEpoch(4)).andReturn(
       Some(OffsetAndEpoch(120, 3))).anyTimes()
@@ -512,7 +512,7 @@ class ReplicaFetcherThreadTest {
     //Stubs
     expect(partition.truncateTo(capture(truncated), anyBoolean())).anyTimes()
     expect(replica.logEndOffset).andReturn(initialLeo).anyTimes()
-    expect(replica.highWatermark).andReturn(new LogOffsetMetadata(initialFetchOffset)).anyTimes()
+    expect(replica.highWatermark).andReturn(initialFetchOffset).anyTimes()
     expect(replica.latestEpoch).andReturn(Some(5))
     expect(replicaManager.logManager).andReturn(logManager).anyTimes()
     expect(replicaManager.replicaAlterLogDirsManager).andReturn(replicaAlterLogDirsManager).anyTimes()
@@ -554,7 +554,7 @@ class ReplicaFetcherThreadTest {
     val initialLeo = 300
 
     //Stubs
-    expect(replica.highWatermark).andReturn(new LogOffsetMetadata(highWaterMark)).anyTimes()
+    expect(replica.highWatermark).andReturn(highWaterMark).anyTimes()
     expect(partition.truncateTo(capture(truncated), anyBoolean())).anyTimes()
     expect(replica.logEndOffset).andReturn(initialLeo).anyTimes()
     expect(replica.latestEpoch).andReturn(Some(leaderEpoch)).anyTimes()
@@ -611,7 +611,7 @@ class ReplicaFetcherThreadTest {
     //Stub return values
     expect(partition.truncateTo(0L, false)).times(2)
     expect(replica.logEndOffset).andReturn(0).anyTimes()
-    expect(replica.highWatermark).andReturn(new LogOffsetMetadata(0)).anyTimes()
+    expect(replica.highWatermark).andReturn(0).anyTimes()
     expect(replica.latestEpoch).andReturn(Some(leaderEpoch)).anyTimes()
     expect(replica.endOffsetForEpoch(leaderEpoch)).andReturn(
       Some(OffsetAndEpoch(0, leaderEpoch))).anyTimes()
@@ -662,7 +662,7 @@ class ReplicaFetcherThreadTest {
     //Stub return values
     expect(partition.truncateTo(capture(truncateToCapture), anyBoolean())).once
     expect(replica.logEndOffset).andReturn(initialLEO).anyTimes()
-    expect(replica.highWatermark).andReturn(new LogOffsetMetadata(initialLEO - 2)).anyTimes()
+    expect(replica.highWatermark).andReturn(initialLEO - 2).anyTimes()
     expect(replica.latestEpoch).andReturn(Some(5)).anyTimes()
     expect(replica.endOffsetForEpoch(5)).andReturn(Some(OffsetAndEpoch(initialLEO, 5))).anyTimes()
     expect(replicaManager.logManager).andReturn(logManager).anyTimes()
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala
index c2d92df458145..d298003674c36 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala
@@ -239,17 +239,17 @@ class ReplicaManagerQuotasTest {
     for ((p, _) <- fetchInfo) {
       val partition = replicaManager.createPartition(p)
       val leaderReplica = new Replica(configs.head.brokerId, p, time, 0, Some(log))
-      leaderReplica.highWatermark = new LogOffsetMetadata(5)
+      leaderReplica.highWatermark = 5
       partition.leaderReplicaIdOpt = Some(leaderReplica.brokerId)
       val followerReplica = new Replica(configs.last.brokerId, p, time, 0, Some(log))
       val allReplicas = Set(leaderReplica, followerReplica)
       allReplicas.foreach(partition.addReplicaIfNotExists)
       if (bothReplicasInSync) {
         partition.inSyncReplicas = allReplicas
-        followerReplica.highWatermark = new LogOffsetMetadata(5)
+        followerReplica.highWatermark = 5
       } else {
         partition.inSyncReplicas = Set(leaderReplica)
-        followerReplica.highWatermark = new LogOffsetMetadata(0)
+        followerReplica.highWatermark = 0
       }
     }
   }
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index 7c14cd26a6fae..59248f0d02af9 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -34,7 +34,7 @@ import org.I0Itec.zkclient.ZkClient
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
 import org.apache.kafka.common.record._
-import org.apache.kafka.common.requests.{EpochEndOffset, IsolationLevel, LeaderAndIsrRequest}
+import org.apache.kafka.common.requests.{EpochEndOffset, FetchRequest, IsolationLevel, LeaderAndIsrRequest}
 import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse
 import org.apache.kafka.common.requests.FetchRequest.PartitionData
 import org.apache.kafka.common.requests.FetchResponse.AbortedTransaction
@@ -454,6 +454,92 @@ class ReplicaManagerTest {
     }
   }
 
+  @Test
+  def testFollowerStateNotUpdatedIfLogReadFails(): Unit = {
+    val maxFetchBytes = 1024 * 1024
+    val aliveBrokersIds = Seq(0, 1)
+    val leaderEpoch = 5
+    val replicaManager = setupReplicaManagerWithMockedPurgatories(new MockTimer, aliveBrokersIds)
+    try {
+      val tp = new TopicPartition(topic, 0)
+      val replicas = aliveBrokersIds.toList.map(Int.box).asJava
+
+      // Broker 0 becomes leader of the partition
+      val leaderAndIsrPartitionState = new LeaderAndIsrRequest.PartitionState(0, 0, leaderEpoch,
+        replicas, 0, replicas, true)
+      val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+        Map(tp -> leaderAndIsrPartitionState).asJava,
+        Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
+      val leaderAndIsrResponse = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest, (_, _) => ())
+      assertEquals(Errors.NONE, leaderAndIsrResponse.error)
+
+      // Follower replica state is initialized, but initial state is not known
+      assertTrue(replicaManager.nonOfflinePartition(tp).isDefined)
+      val partition = replicaManager.nonOfflinePartition(tp).get
+
+      assertTrue(partition.getReplica(1).isDefined)
+      val followerReplica = partition.getReplica(1).get
+      assertEquals(None, followerReplica.log)
+      assertEquals(-1L, followerReplica.logStartOffset)
+      assertEquals(-1L, followerReplica.logEndOffset)
+
+      // Leader appends some data
+      for (i <- 1 to 5) {
+        appendRecords(replicaManager, tp, TestUtils.singletonRecords(s"message $i".getBytes)).onFire { response =>
+          assertEquals(Errors.NONE, response.error)
+        }
+      }
+
+      // We receive one valid request from the follower and replica state is updated
+      var successfulFetch: Option[FetchPartitionData] = None
+      def callback(response: Seq[(TopicPartition, FetchPartitionData)]): Unit = {
+        successfulFetch = response.headOption.filter(_._1 == tp).map(_._2)
+      }
+
+      val validFetchPartitionData = new FetchRequest.PartitionData(0L, 0L, maxFetchBytes,
+        Optional.of(leaderEpoch))
+
+      replicaManager.fetchMessages(
+        timeout = 0L,
+        replicaId = 1,
+        fetchMinBytes = 1,
+        fetchMaxBytes = maxFetchBytes,
+        hardMaxBytesLimit = false,
+        fetchInfos = Seq(tp -> validFetchPartitionData),
+        isolationLevel = IsolationLevel.READ_UNCOMMITTED,
+        responseCallback = callback
+      )
+
+      assertTrue(successfulFetch.isDefined)
+      assertEquals(0L, followerReplica.logStartOffset)
+      assertEquals(0L, followerReplica.logEndOffset)
+
+
+      // Next we receive an invalid request with a higher fetch offset, but an old epoch.
+      // We expect that the replica state does not get updated.
+      val invalidFetchPartitionData = new FetchRequest.PartitionData(3L, 0L, maxFetchBytes,
+        Optional.of(leaderEpoch - 1))
+
+      replicaManager.fetchMessages(
+        timeout = 0L,
+        replicaId = 1,
+        fetchMinBytes = 1,
+        fetchMaxBytes = maxFetchBytes,
+        hardMaxBytesLimit = false,
+        fetchInfos = Seq(tp -> invalidFetchPartitionData),
+        isolationLevel = IsolationLevel.READ_UNCOMMITTED,
+        responseCallback = callback
+      )
+
+      assertTrue(successfulFetch.isDefined)
+      assertEquals(0L, followerReplica.logStartOffset)
+      assertEquals(0L, followerReplica.logEndOffset)
+
+    } finally {
+      replicaManager.shutdown(checkpointHW = false)
+    }
+  }
+
   /**
    * If a follower sends a fetch request for 2 partitions and it's no longer the follower for one of them, the other
    * partition should not be affected.
@@ -525,12 +611,12 @@ class ReplicaManagerTest {
       )
       val tp0Replica = replicaManager.localReplica(tp0)
       assertTrue(tp0Replica.isDefined)
-      assertEquals("hw should be incremented", 1, tp0Replica.get.highWatermark.messageOffset)
+      assertEquals("hw should be incremented", 1, tp0Replica.get.highWatermark)
 
       replicaManager.localReplica(tp1)
       val tp1Replica = replicaManager.localReplica(tp1)
       assertTrue(tp1Replica.isDefined)
-      assertEquals("hw should not be incremented", 0, tp1Replica.get.highWatermark.messageOffset)
+      assertEquals("hw should not be incremented", 0, tp1Replica.get.highWatermark)
 
     } finally {
       replicaManager.shutdown(checkpointHW = false)
diff --git a/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala b/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala
index 41c6b3e2ae257..e7e6e8fab6609 100644
--- a/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala
+++ b/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala
@@ -121,20 +121,17 @@ class SimpleFetchTest {
 
     // create the leader replica with the local log
     val leaderReplica = new Replica(configs.head.brokerId, partition.topicPartition, time, 0, Some(log))
-    leaderReplica.highWatermark = new LogOffsetMetadata(partitionHW)
+    leaderReplica.highWatermark = partitionHW
     partition.leaderReplicaIdOpt = Some(leaderReplica.brokerId)
 
     // create the follower replica with defined log end offset
     val followerReplica= new Replica(configs(1).brokerId, partition.topicPartition, time)
     val leo = new LogOffsetMetadata(followerLEO, 0L, followerLEO.toInt)
-    followerReplica.updateLogReadResult(new LogReadResult(info = FetchDataInfo(leo, MemoryRecords.EMPTY),
-                                                          highWatermark = leo.messageOffset,
-                                                          leaderLogStartOffset = 0L,
-                                                          leaderLogEndOffset = leo.messageOffset,
-                                                          followerLogStartOffset = 0L,
-                                                          fetchTimeMs = time.milliseconds,
-                                                          readSize = -1,
-                                                          lastStableOffset = None))
+    followerReplica.updateFetchState(
+      followerFetchOffsetMetadata = leo,
+      followerStartOffset = 0L,
+      followerFetchTimeMs= time.milliseconds,
+      leaderEndOffset = leo.messageOffset)
 
     // add both of them to ISR
     val allReplicas = List(leaderReplica, followerReplica)

From 59d3a56740a8c668ff8151b88d00c7adc371a50c Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Thu, 6 Jun 2019 06:58:58 -0700
Subject: [PATCH 0336/1071] Minor: Replace InternalTopicMetadata with
 InternalTopicConfig (#6886)

Quick tech debt cleanup. For some reason StreamsPartitionAssignor uses an InternalTopicMetadata class which wraps an InternalTopicConfig object along with the number of partitions. But InternalTopicConfig already has a numPartitions field, so we should just use it directly instead.

Reviewers: Guozhang Wang , Bruno Cadonna ,  Bill Bejeck 
---
 .../internals/InternalTopicConfig.java        |  6 +-
 .../internals/StreamsPartitionAssignor.java   | 66 +++++++------------
 .../CopartitionedTopicsValidatorTest.java     | 47 +++++++------
 3 files changed, 46 insertions(+), 73 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java
index aa565e4aa5dbe..9005311c7af31 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java
@@ -26,10 +26,11 @@
  * the internal topics we create for change-logs and repartitioning etc.
  */
 public abstract class InternalTopicConfig {
+
     final String name;
     final Map topicConfigs;
 
-    private int numberOfPartitions = -1;
+    private int numberOfPartitions = StreamsPartitionAssignor.UNKNOWN;
 
     InternalTopicConfig(final String name, final Map topicConfigs) {
         Objects.requireNonNull(name, "name can't be null");
@@ -53,9 +54,6 @@ public String name() {
     }
 
     public int numberOfPartitions() {
-        if (numberOfPartitions == -1) {
-            throw new IllegalStateException("Number of partitions not specified.");
-        }
         return numberOfPartitions;
     }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
index 33cce13faeb42..714bb0ea21041 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
@@ -58,7 +58,7 @@
 
 public class StreamsPartitionAssignor implements PartitionAssignor, Configurable {
 
-    private final static int UNKNOWN = -1;
+    final static int UNKNOWN = -1;
     private final static int VERSION_ONE = 1;
     private final static int VERSION_TWO = 2;
     private final static int VERSION_THREE = 3;
@@ -174,25 +174,6 @@ public String toString() {
         }
     }
 
-    static class InternalTopicMetadata {
-        public final InternalTopicConfig config;
-
-        public int numPartitions;
-
-        InternalTopicMetadata(final InternalTopicConfig config) {
-            this.config = config;
-            this.numPartitions = UNKNOWN;
-        }
-
-        @Override
-        public String toString() {
-            return "InternalTopicMetadata(" +
-                    "config=" + config +
-                    ", numPartitions=" + numPartitions +
-                    ")";
-        }
-    }
-
     private static final class InternalStreamsConfig extends StreamsConfig {
         private InternalStreamsConfig(final Map props) {
             super(props, false);
@@ -458,7 +439,7 @@ public Map assign(final Cluster metadata,
         // the maximum of the depending sub-topologies source topics' number of partitions
         final Map topicGroups = taskManager.builder().topicGroups();
 
-        final Map repartitionTopicMetadata = new HashMap<>();
+        final Map repartitionTopicMetadata = new HashMap<>();
         for (final InternalTopologyBuilder.TopicsInfo topicsInfo : topicGroups.values()) {
             for (final String topic : topicsInfo.sourceTopics) {
                 if (!topicsInfo.repartitionSourceTopics.keySet().contains(topic) &&
@@ -469,7 +450,7 @@ public Map assign(final Cluster metadata,
                 }
             }
             for (final InternalTopicConfig topic: topicsInfo.repartitionSourceTopics.values()) {
-                repartitionTopicMetadata.put(topic.name(), new InternalTopicMetadata(topic));
+                repartitionTopicMetadata.put(topic.name(), topic);
             }
         }
 
@@ -479,10 +460,10 @@ public Map assign(final Cluster metadata,
 
             for (final InternalTopologyBuilder.TopicsInfo topicsInfo : topicGroups.values()) {
                 for (final String topicName : topicsInfo.repartitionSourceTopics.keySet()) {
-                    int numPartitions = repartitionTopicMetadata.get(topicName).numPartitions;
+                    int numPartitions = repartitionTopicMetadata.get(topicName).numberOfPartitions();
 
-                    // try set the number of partitions for this repartition topic if it is not set yet
                     if (numPartitions == UNKNOWN) {
+                        // try set the number of partitions for this repartition topic if it is not set yet
                         for (final InternalTopologyBuilder.TopicsInfo otherTopicsInfo : topicGroups.values()) {
                             final Set otherSinkTopics = otherTopicsInfo.sinkTopics;
 
@@ -494,7 +475,7 @@ public Map assign(final Cluster metadata,
                                     // It is possible the sourceTopic is another internal topic, i.e,
                                     // map().join().join(map())
                                     if (repartitionTopicMetadata.containsKey(sourceTopicName)) {
-                                        numPartitionsCandidate = repartitionTopicMetadata.get(sourceTopicName).numPartitions;
+                                        numPartitionsCandidate = repartitionTopicMetadata.get(sourceTopicName).numberOfPartitions();
                                     } else {
                                         numPartitionsCandidate = metadata.partitionCountForTopic(sourceTopicName);
                                     }
@@ -510,7 +491,7 @@ public Map assign(final Cluster metadata,
                         if (numPartitions == UNKNOWN) {
                             numPartitionsNeeded = true;
                         } else {
-                            repartitionTopicMetadata.get(topicName).numPartitions = numPartitions;
+                            repartitionTopicMetadata.get(topicName).setNumberOfPartitions(numPartitions);
                         }
                     }
                 }
@@ -530,9 +511,9 @@ public Map assign(final Cluster metadata,
         // augment the metadata with the newly computed number of partitions for all the
         // repartition source topics
         final Map allRepartitionTopicPartitions = new HashMap<>();
-        for (final Map.Entry entry : repartitionTopicMetadata.entrySet()) {
+        for (final Map.Entry entry : repartitionTopicMetadata.entrySet()) {
             final String topic = entry.getKey();
-            final int numPartitions = entry.getValue().numPartitions;
+            final int numPartitions = entry.getValue().numberOfPartitions();
 
             for (int partition = 0; partition < numPartitions; partition++) {
                 allRepartitionTopicPartitions.put(new TopicPartition(topic, partition),
@@ -591,7 +572,7 @@ public Map assign(final Cluster metadata,
         }
 
         // add tasks to state change log topic subscribers
-        final Map changelogTopicMetadata = new HashMap<>();
+        final Map changelogTopicMetadata = new HashMap<>();
         for (final Map.Entry entry : topicGroups.entrySet()) {
             final int topicGroupId = entry.getKey();
             final Map stateChangelogTopics = entry.getValue().stateChangelogTopics;
@@ -605,10 +586,9 @@ public Map assign(final Cluster metadata,
                             numPartitions = task.partition + 1;
                         }
                     }
-                    final InternalTopicMetadata topicMetadata = new InternalTopicMetadata(topicConfig);
-                    topicMetadata.numPartitions = numPartitions;
+                    topicConfig.setNumberOfPartitions(numPartitions);
 
-                    changelogTopicMetadata.put(topicConfig.name(), topicMetadata);
+                    changelogTopicMetadata.put(topicConfig.name(), topicConfig);
                 } else {
                     log.debug("No tasks found for topic group {}", topicGroupId);
                 }
@@ -949,17 +929,15 @@ protected void processLatestVersionAssignment(final AssignmentInfo info,
      *
      * @param topicPartitions Map that contains the topic names to be created with the number of partitions
      */
-    private void prepareTopic(final Map topicPartitions) {
+    private void prepareTopic(final Map topicPartitions) {
         log.debug("Starting to validate internal topics {} in partition assignor.", topicPartitions);
 
         // first construct the topics to make ready
         final Map topicsToMakeReady = new HashMap<>();
 
-        for (final InternalTopicMetadata metadata : topicPartitions.values()) {
-            final InternalTopicConfig topic = metadata.config;
-            final int numPartitions = metadata.numPartitions;
-
-            if (numPartitions < 0) {
+        for (final InternalTopicConfig topic : topicPartitions.values()) {
+            final int numPartitions = topic.numberOfPartitions();
+            if (numPartitions == UNKNOWN) {
                 throw new StreamsException(String.format("%sTopic [%s] number of partitions not defined", logPrefix, topic.name()));
             }
 
@@ -975,7 +953,7 @@ private void prepareTopic(final Map topicPartitio
     }
 
     private void ensureCopartitioning(final Collection> copartitionGroups,
-                                      final Map allRepartitionTopicsNumPartitions,
+                                      final Map allRepartitionTopicsNumPartitions,
                                       final Cluster metadata) {
         for (final Set copartitionGroup : copartitionGroups) {
             copartitionedTopicsValidator.validate(copartitionGroup, allRepartitionTopicsNumPartitions, metadata);
@@ -993,7 +971,7 @@ static class CopartitionedTopicsValidator {
         }
 
         void validate(final Set copartitionGroup,
-                      final Map allRepartitionTopicsNumPartitions,
+                      final Map allRepartitionTopicsNumPartitions,
                       final Cluster metadata) {
             int numPartitions = UNKNOWN;
 
@@ -1019,9 +997,9 @@ void validate(final Set copartitionGroup,
             // if all topics for this co-partition group is repartition topics,
             // then set the number of partitions to be the maximum of the number of partitions.
             if (numPartitions == UNKNOWN) {
-                for (final Map.Entry entry: allRepartitionTopicsNumPartitions.entrySet()) {
+                for (final Map.Entry entry: allRepartitionTopicsNumPartitions.entrySet()) {
                     if (copartitionGroup.contains(entry.getKey())) {
-                        final int partitions = entry.getValue().numPartitions;
+                        final int partitions = entry.getValue().numberOfPartitions();
                         if (partitions > numPartitions) {
                             numPartitions = partitions;
                         }
@@ -1029,9 +1007,9 @@ void validate(final Set copartitionGroup,
                 }
             }
             // enforce co-partitioning restrictions to repartition topics by updating their number of partitions
-            for (final Map.Entry entry : allRepartitionTopicsNumPartitions.entrySet()) {
+            for (final Map.Entry entry : allRepartitionTopicsNumPartitions.entrySet()) {
                 if (copartitionGroup.contains(entry.getKey())) {
-                    entry.getValue().numPartitions = numPartitions;
+                    entry.getValue().setNumberOfPartitions(numPartitions);
                 }
             }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsValidatorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsValidatorTest.java
index d6119fc0e913c..a272aa5407b79 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsValidatorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsValidatorTest.java
@@ -72,49 +72,46 @@ public void shouldThrowTopologyBuilderExceptionIfPartitionCountsForCoPartitioned
 
     @Test
     public void shouldEnforceCopartitioningOnRepartitionTopics() {
-        final StreamsPartitionAssignor.InternalTopicMetadata metadata = createTopicMetadata("repartitioned", 10);
+        final InternalTopicConfig config = createTopicConfig("repartitioned", 10);
 
-        validator.validate(Utils.mkSet("first", "second", metadata.config.name()),
-                           Collections.singletonMap(metadata.config.name(),
-                                                    metadata),
+        validator.validate(Utils.mkSet("first", "second", config.name()),
+                           Collections.singletonMap(config.name(), config),
                            cluster.withPartitions(partitions));
 
-        assertThat(metadata.numPartitions, equalTo(2));
+        assertThat(config.numberOfPartitions(), equalTo(2));
     }
 
 
     @Test
     public void shouldSetNumPartitionsToMaximumPartitionsWhenAllTopicsAreRepartitionTopics() {
-        final StreamsPartitionAssignor.InternalTopicMetadata one = createTopicMetadata("one", 1);
-        final StreamsPartitionAssignor.InternalTopicMetadata two = createTopicMetadata("two", 15);
-        final StreamsPartitionAssignor.InternalTopicMetadata three = createTopicMetadata("three", 5);
-        final Map repartitionTopicConfig = new HashMap<>();
-
-        repartitionTopicConfig.put(one.config.name(), one);
-        repartitionTopicConfig.put(two.config.name(), two);
-        repartitionTopicConfig.put(three.config.name(), three);
-
-        validator.validate(Utils.mkSet(one.config.name(),
-                                       two.config.name(),
-                                       three.config.name()),
+        final InternalTopicConfig one = createTopicConfig("one", 1);
+        final InternalTopicConfig two = createTopicConfig("two", 15);
+        final InternalTopicConfig three = createTopicConfig("three", 5);
+        final Map repartitionTopicConfig = new HashMap<>();
+
+        repartitionTopicConfig.put(one.name(), one);
+        repartitionTopicConfig.put(two.name(), two);
+        repartitionTopicConfig.put(three.name(), three);
+
+        validator.validate(Utils.mkSet(one.name(),
+                                       two.name(),
+                                       three.name()),
                            repartitionTopicConfig,
                            cluster
         );
 
-        assertThat(one.numPartitions, equalTo(15));
-        assertThat(two.numPartitions, equalTo(15));
-        assertThat(three.numPartitions, equalTo(15));
+        assertThat(one.numberOfPartitions(), equalTo(15));
+        assertThat(two.numberOfPartitions(), equalTo(15));
+        assertThat(three.numberOfPartitions(), equalTo(15));
     }
 
-    private StreamsPartitionAssignor.InternalTopicMetadata createTopicMetadata(final String repartitionTopic,
+    private InternalTopicConfig createTopicConfig(final String repartitionTopic,
                                                                                final int partitions) {
         final InternalTopicConfig repartitionTopicConfig =
             new RepartitionTopicConfig(repartitionTopic, Collections.emptyMap());
 
-        final StreamsPartitionAssignor.InternalTopicMetadata metadata =
-            new StreamsPartitionAssignor.InternalTopicMetadata(repartitionTopicConfig);
-        metadata.numPartitions = partitions;
-        return metadata;
+        repartitionTopicConfig.setNumberOfPartitions(partitions);
+        return repartitionTopicConfig;
     }
 
 }
\ No newline at end of file

From 58aa04f91e9cd8203cd58f972d042d068727d256 Mon Sep 17 00:00:00 2001
From: Stanislav Kozlovski 
Date: Thu, 6 Jun 2019 20:04:05 +0300
Subject: [PATCH 0337/1071] MINOR: Improve Trogdor external command worker docs
 (#6438)

Reviewers: Colin McCabe , Xi Yang 
---
 TROGDOR.md                                    | 23 ++++++++++++++++++-
 tests/bin/external_trogdor_command_example.py | 13 +++++++----
 .../workload/ExternalCommandWorker.java       |  2 +-
 .../workload/ExternalCommandWorkerTest.java   | 10 ++------
 4 files changed, 33 insertions(+), 15 deletions(-)

diff --git a/TROGDOR.md b/TROGDOR.md
index 49c69428db378..ad8d8af797e02 100644
--- a/TROGDOR.md
+++ b/TROGDOR.md
@@ -16,7 +16,7 @@ Running Kafka:
 
     > ./bin/kafka-server-start.sh ./config/server.properties &> /tmp/kafka.log &
 
-Then, we want to run a Trogdor Agent, plus a Trogdor broker.
+Then, we want to run a Trogdor Agent, plus a Trogdor Coordinator.
 
 To run the Trogdor Agent:
 
@@ -125,6 +125,27 @@ ProcessStopFault stops a process by sending it a SIGSTOP signal.  When the fault
 ### NetworkPartitionFault
 NetworkPartitionFault sets up an artificial network partition between one or more sets of nodes.  Currently, this is implemented using iptables.  The iptables rules are set up on the outbound traffic from the affected nodes.  Therefore, the affected nodes should still be reachable from outside the cluster.
 
+External Processes
+========================================
+Trogdor supports running arbitrary commands in external processes. This is a generic way to run any configurable command in the Trogdor framework - be it a Python program, bash script, docker image, etc.
+
+### ExternalCommandWorker
+ExternalCommandWorker starts an external command defined by the ExternalCommandSpec. It essentially allows you to run any command on any Trogdor agent node.
+The worker communicates with the external process via its stdin, stdout and stderr in a JSON protocol. It uses stdout for any actionable communication and only logs what it sees in stderr.
+On startup the worker will first send a message describing the workload to the external process in this format:
+```
+{"id":, "workload":}
+```
+and will then listen for messages from the external process, again in a JSON format.
+Said JSON can contain the following fields:
+- status: If the object contains this field, the status of the worker will be set to the given value.
+- error: If the object contains this field, the error of the worker will be set to the given value. Once an error occurs, the external process will be terminated.
+- log: If the object contains this field, a log message will be issued with this text.
+An example:
+```json
+{"log": "Finished successfully.", "status": {"p99ProduceLatency": "100ms", "messagesSent": 10000}}
+```
+
 Exec Mode
 ========================================
 Sometimes, you just want to run a test quickly on a single node.  In this case, you can use "exec mode."  This mode allows you to run a single Trogdor Agent without a Coordinator.
diff --git a/tests/bin/external_trogdor_command_example.py b/tests/bin/external_trogdor_command_example.py
index 0e53557ca93c5..1254b8283e33e 100755
--- a/tests/bin/external_trogdor_command_example.py
+++ b/tests/bin/external_trogdor_command_example.py
@@ -20,7 +20,7 @@
 
 #
 # This is an example of an external script which can be run through Trogdor's
-# ExternalCommandWorker.
+# ExternalCommandWorker. It sleeps for the given amount of time expressed by the delayMs field in the ExternalCommandSpec
 #
 
 if __name__ == '__main__':
@@ -28,11 +28,14 @@
     line = sys.stdin.readline()
     start_message = json.loads(line)
     workload = start_message["workload"]
-    print("Starting external_trogdor_command_example with task id %s, workload %s" \
-        % (start_message["id"], workload))
+    print("Starting external_trogdor_command_example with task id %s, workload %s"
+          % (start_message["id"], workload))
     sys.stdout.flush()
-   `print(json.dumps({"status": "running"}))`
+
+    # pretend to start some workload
+    print(json.dumps({"status": "running"}))
     sys.stdout.flush()
     time.sleep(0.001 * workload["delayMs"])
-   `print(json.dumps({"status": "exiting after %s delayMs" % workload["delayMs"]}))`
+
+    print(json.dumps({"status": "exiting after %s delayMs" % workload["delayMs"]}))
     sys.stdout.flush()
diff --git a/tools/src/main/java/org/apache/kafka/trogdor/workload/ExternalCommandWorker.java b/tools/src/main/java/org/apache/kafka/trogdor/workload/ExternalCommandWorker.java
index 6f5799fb606fc..9db7f18e24722 100644
--- a/tools/src/main/java/org/apache/kafka/trogdor/workload/ExternalCommandWorker.java
+++ b/tools/src/main/java/org/apache/kafka/trogdor/workload/ExternalCommandWorker.java
@@ -389,7 +389,7 @@ public void stop(Platform platform) throws Exception {
             spec.shutdownGracePeriodMs().get() : DEFAULT_SHUTDOWN_GRACE_PERIOD_MS;
         if (!executor.awaitTermination(shutdownGracePeriodMs, TimeUnit.MILLISECONDS)) {
             terminatorActionQueue.add(TerminatorAction.DESTROY_FORCIBLY);
-            executor.awaitTermination(1000, TimeUnit.DAYS);
+            executor.awaitTermination(1, TimeUnit.DAYS);
         }
         this.status = null;
         this.doneFuture = null;
diff --git a/tools/src/test/java/org/apache/kafka/trogdor/workload/ExternalCommandWorkerTest.java b/tools/src/test/java/org/apache/kafka/trogdor/workload/ExternalCommandWorkerTest.java
index 9800b137c326f..73ac77c9f7977 100644
--- a/tools/src/test/java/org/apache/kafka/trogdor/workload/ExternalCommandWorkerTest.java
+++ b/tools/src/test/java/org/apache/kafka/trogdor/workload/ExternalCommandWorkerTest.java
@@ -17,7 +17,6 @@
 
 package org.apache.kafka.trogdor.workload;
 
-import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.IntNode;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -109,7 +108,7 @@ public void testProcessWithFailedExit() throws Exception {
     }
 
     /**
-     * Test attempting to run an exeutable which doesn't exist.
+     * Test attempting to run an executable which doesn't exist.
      * We use a path which starts with /dev/null, since that should never be a
      * directory in UNIX.
      */
@@ -172,12 +171,7 @@ public void testProcessForceKillTimeout() throws Exception {
                 }
             }
             CompletableFuture statusFuture = new CompletableFuture<>();
-            final WorkerStatusTracker statusTracker = new WorkerStatusTracker() {
-                @Override
-                public void update(JsonNode status) {
-                    statusFuture .complete(status.textValue().toString());
-                }
-            };
+            final WorkerStatusTracker statusTracker = status -> statusFuture .complete(status.textValue());
             ExternalCommandWorker worker = new ExternalCommandWorkerBuilder("testForceKillTask").
                 shutdownGracePeriodMs(1).
                 command("bash", tempFile.getAbsolutePath()).

From 30adf2a096a7c879497b435b1704620e86d9e71e Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Thu, 6 Jun 2019 11:29:39 -0700
Subject: [PATCH 0338/1071] HOTFIX: Close unused ColumnFamilyHandle (#6893)

In RocksDBTimestampedStore#openRocksDB we try to open a db with two column families. If this succeeds but the first column family is empty (db.newIterator.seekToFirst.isValid() == false) we never actually close its ColumnFamilyHandle

Reviewers: Matthias J. Sax , Guozhang Wang 
---
 .../kafka/streams/state/internals/RocksDBTimestampedStore.java   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStore.java
index 05db0eadd041c..74f091936d28a 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStore.java
@@ -81,6 +81,7 @@ void openRocksDB(final DBOptions dbOptions,
             } else {
                 log.info("Opening store {} in regular mode", name);
                 dbAccessor = new SingleColumnFamilyAccessor(columnFamilies.get(1));
+                noTimestampColumnFamily.close();
             }
             noTimestampsIter.close();
         } catch (final RocksDBException e) {

From 562424cee87d8f87ae94d88cf5863c9f5db6dab5 Mon Sep 17 00:00:00 2001
From: Wennn 
Date: Fri, 7 Jun 2019 07:18:50 +0800
Subject: [PATCH 0339/1071] MINOR: Remove redundant semicolons in `KafkaApis`
 imports (#6889)

Reviewers: Jason Gustafson 
---
 core/src/main/scala/kafka/server/KafkaApis.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 8b419022d6288..0a1f25122b900 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -53,8 +53,8 @@ import org.apache.kafka.common.message.CreateTopicsResponseData.{CreatableTopicR
 import org.apache.kafka.common.message.DeleteTopicsResponseData
 import org.apache.kafka.common.message.DeleteTopicsResponseData.{DeletableTopicResult, DeletableTopicResultCollection}
 import org.apache.kafka.common.message.DescribeGroupsResponseData
-import org.apache.kafka.common.message.ElectLeadersResponseData.PartitionResult;
-import org.apache.kafka.common.message.ElectLeadersResponseData.ReplicaElectionResult;
+import org.apache.kafka.common.message.ElectLeadersResponseData.PartitionResult
+import org.apache.kafka.common.message.ElectLeadersResponseData.ReplicaElectionResult
 import org.apache.kafka.common.message.FindCoordinatorResponseData
 import org.apache.kafka.common.message.HeartbeatResponseData
 import org.apache.kafka.common.message.InitProducerIdResponseData

From 72c3b88ed5066d71eac7be393b94e1f64b631b1f Mon Sep 17 00:00:00 2001
From: Boyang Chen 
Date: Thu, 6 Jun 2019 17:32:13 -0700
Subject: [PATCH 0340/1071] MINOR: Fixed compiler warnings in LogManagerTest
 (#6897)

Reviewers: Jason Gustafson , Guozhang Wang 
---
 core/src/test/scala/unit/kafka/log/LogManagerTest.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/test/scala/unit/kafka/log/LogManagerTest.scala b/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
index 3df09e7a54a94..ed1824df04d2a 100755
--- a/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
@@ -382,8 +382,8 @@ class LogManagerTest {
 
   @Test
   def testCreateAndDeleteOverlyLongTopic(): Unit = {
-    val invalidTopicName = String.join("", Collections.nCopies(253, "x"));
-    val log = logManager.getOrCreateLog(new TopicPartition(invalidTopicName, 0), logConfig)
+    val invalidTopicName = String.join("", Collections.nCopies(253, "x"))
+    logManager.getOrCreateLog(new TopicPartition(invalidTopicName, 0), logConfig)
     logManager.asyncDelete(new TopicPartition(invalidTopicName, 0))
   }
 

From 28eccc3ca3fe591c06d6067f08f60ce71f2a1e3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Armando=20Garc=C3=ADa=20Sancio?=
 
Date: Thu, 6 Jun 2019 21:36:33 -0700
Subject: [PATCH 0341/1071] KAFKA-8384; Leader election command integration
 tests (#6880)

This patch adds test cases for the leader election command added in KIP-460.

Reviewers: Vikas Singh, David Arthur , Jason Gustafson 
---
 .../kafka/admin/LeaderElectionCommand.scala   |  62 ++-
 .../admin/LeaderElectionCommandTest.scala     | 373 ++++++++++++++++++
 ...rredReplicaLeaderElectionCommandTest.scala |   8 +-
 .../scala/unit/kafka/utils/TestUtils.scala    |  15 +
 4 files changed, 447 insertions(+), 11 deletions(-)
 create mode 100644 core/src/test/scala/unit/kafka/admin/LeaderElectionCommandTest.scala

diff --git a/core/src/main/scala/kafka/admin/LeaderElectionCommand.scala b/core/src/main/scala/kafka/admin/LeaderElectionCommand.scala
index 58a55e49dc02f..3d177959ad1cd 100644
--- a/core/src/main/scala/kafka/admin/LeaderElectionCommand.scala
+++ b/core/src/main/scala/kafka/admin/LeaderElectionCommand.scala
@@ -49,6 +49,8 @@ object LeaderElectionCommand extends Logging {
       "This tool attempts to elect a new leader for a set of topic partitions. The type of elections supported are preferred replicas and unclean replicas."
     )
 
+    validate(commandOptions)
+
     val electionType = commandOptions.options.valueOf(commandOptions.electionType)
 
     val jsonFileTopicPartitions = Option(commandOptions.options.valueOf(commandOptions.pathToJsonFile)).map { path  =>
@@ -64,7 +66,8 @@ object LeaderElectionCommand extends Logging {
     }
 
     /* Note: No need to look at --all-topic-partitions as we want this to be None if it is use.
-     * Jopt-Simple should be validating that this option required if the --topic and --path-to-json-file
+     * The validate function should be checking that this option is required if the --topic and --path-to-json-file
+     * are not specified.
      */
     val topicPartitions = jsonFileTopicPartitions.orElse(singleTopicPartition)
 
@@ -107,7 +110,7 @@ object LeaderElectionCommand extends Logging {
               )
             }
             partitions.toSet
-          case None => throw new AdminOperationException("Replica election data is missing \"partition\" field")
+          case None => throw new AdminOperationException("Replica election data is missing \"partitions\" field")
         }
       case None => throw new AdminOperationException("Replica election data is empty")
     }
@@ -175,6 +178,55 @@ object LeaderElectionCommand extends Logging {
       throw rootException
     }
   }
+
+  private[this] def validate(commandOptions: LeaderElectionCommandOptions): Unit = {
+    // required options: --bootstrap-server and --election-type
+    var missingOptions = List.empty[String]
+    if (!commandOptions.options.has(commandOptions.bootstrapServer)) {
+      missingOptions = commandOptions.bootstrapServer.options().get(0) :: missingOptions
+    }
+
+    if (!commandOptions.options.has(commandOptions.electionType)) {
+      missingOptions = commandOptions.electionType.options().get(0) :: missingOptions
+    }
+
+    if (missingOptions.nonEmpty) {
+      throw new AdminCommandFailedException(s"Missing required option(s): ${missingOptions.mkString(", ")}")
+    }
+
+    // One and only one is required: --topic, --all-topic-partitions or --path-to-json-file
+    val mutuallyExclusiveOptions = Seq(
+      commandOptions.topic,
+      commandOptions.allTopicPartitions,
+      commandOptions.pathToJsonFile
+    )
+
+    mutuallyExclusiveOptions.count(commandOptions.options.has) match {
+      case 1 => // This is the only correct configuration, don't throw an exception
+      case _ =>
+        throw new AdminCommandFailedException(
+          "One and only one of the following options is required: " +
+          s"${mutuallyExclusiveOptions.map(_.options.get(0)).mkString(", ")}"
+        )
+    }
+
+    // --partition if and only if --topic is used
+    (
+      commandOptions.options.has(commandOptions.topic),
+      commandOptions.options.has(commandOptions.partition)
+    ) match {
+      case (true, false) =>
+        throw new AdminCommandFailedException(
+          s"Missing required option(s): ${commandOptions.partition.options.get(0)}"
+        )
+      case (false, true) =>
+        throw new AdminCommandFailedException(
+          s"Option ${commandOptions.partition.options.get(0)} is only allowed if " +
+          s"${commandOptions.topic.options.get(0)} is used"
+        )
+      case _ => // Ignore; we have a valid configuration
+    }
+  }
 }
 
 private final class LeaderElectionCommandOptions(args: Array[String]) extends CommandDefaultOptions(args) {
@@ -183,7 +235,6 @@ private final class LeaderElectionCommandOptions(args: Array[String]) extends Co
       "bootstrap-server",
       "A hostname and port for the broker to connect to, in the form host:port. Multiple comma separated URLs can be given. REQUIRED.")
     .withRequiredArg
-    .required
     .describedAs("host:port")
     .ofType(classOf[String])
   val adminClientConfig = parser
@@ -206,15 +257,14 @@ private final class LeaderElectionCommandOptions(args: Array[String]) extends Co
     .accepts(
       "topic",
       "Name of topic for which to perform an election. Not allowed if --path-to-json-file or --all-topic-partitions is specified.")
-    .availableUnless("path-to-json-file")
     .withRequiredArg
     .describedAs("topic name")
     .ofType(classOf[String])
+
   val partition = parser
     .accepts(
       "partition",
       "Partition id for which to perform an election. REQUIRED if --topic is specified.")
-    .requiredIf("topic")
     .withRequiredArg
     .describedAs("partition id")
     .ofType(classOf[Integer])
@@ -223,14 +273,12 @@ private final class LeaderElectionCommandOptions(args: Array[String]) extends Co
     .accepts(
       "all-topic-partitions",
       "Perform election on all of the eligible topic partitions based on the type of election (see the --election-type flag). Not allowed if --topic or --path-to-json-file is specified.")
-    .requiredUnless("path-to-json-file", "topic")
 
   val electionType = parser
     .accepts(
       "election-type",
       "Type of election to attempt. Possible values are \"preferred\" for preferred leader election or \"unclean\" for unclean leader election. If preferred election is selection, the election is only performed if the current leader is not the preferred leader for the topic partition. If unclean election is selected, the election is only performed if there are no leader for the topic partition. REQUIRED.")
     .withRequiredArg
-    .required
     .describedAs("election type")
     .withValuesConvertedBy(ElectionTypeConverter)
 
diff --git a/core/src/test/scala/unit/kafka/admin/LeaderElectionCommandTest.scala b/core/src/test/scala/unit/kafka/admin/LeaderElectionCommandTest.scala
new file mode 100644
index 0000000000000..6f87687602e72
--- /dev/null
+++ b/core/src/test/scala/unit/kafka/admin/LeaderElectionCommandTest.scala
@@ -0,0 +1,373 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package kafka.admin
+
+import java.io.File
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Path
+import kafka.common.AdminCommandFailedException
+import kafka.server.KafkaConfig
+import kafka.server.KafkaServer
+import kafka.utils.TestUtils
+import kafka.zk.ZooKeeperTestHarness
+import org.apache.kafka.clients.admin.AdminClientConfig
+import org.apache.kafka.clients.admin.{AdminClient => JAdminClient}
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.errors.TimeoutException
+import org.apache.kafka.common.errors.UnknownTopicOrPartitionException
+import org.apache.kafka.common.network.ListenerName
+import org.junit.After
+import org.junit.Assert._
+import org.junit.Before
+import org.junit.Test
+import scala.collection.JavaConverters._
+import scala.concurrent.duration._
+
+final class LeaderElectionCommandTest extends ZooKeeperTestHarness {
+  import LeaderElectionCommandTest._
+
+  var servers = Seq.empty[KafkaServer]
+  val broker1 = 0
+  val broker2 = 1
+  val broker3 = 2
+
+  @Before
+  override def setUp() {
+    super.setUp()
+
+    val brokerConfigs = TestUtils.createBrokerConfigs(3, zkConnect, false)
+    servers = brokerConfigs.map { config =>
+      config.setProperty("auto.leader.rebalance.enable", "false")
+      config.setProperty("controlled.shutdown.enable", "true")
+      config.setProperty("controlled.shutdown.max.retries", "1")
+      config.setProperty("controlled.shutdown.retry.backoff.ms", "1000")
+      TestUtils.createServer(KafkaConfig.fromProps(config))
+    }
+  }
+
+  @After
+  override def tearDown() {
+    TestUtils.shutdownServers(servers)
+
+    super.tearDown()
+  }
+
+  @Test
+  def testAllTopicPartition(): Unit = {
+    TestUtils.resource(JAdminClient.create(createConfig(servers).asJava)) { client =>
+      val topic = "unclean-topic"
+      val partition = 0
+      val assignment = Seq(broker2, broker3)
+
+      TestUtils.createTopic(zkClient, topic, Map(partition -> assignment), servers)
+
+      val topicPartition = new TopicPartition(topic, partition)
+
+      waitForLeaderToBecome(client, topicPartition, Option(broker2))
+
+      servers(broker3).shutdown()
+      waitForBrokerOutOfIsr(client, Set(topicPartition), broker3)
+      servers(broker2).shutdown()
+      waitForLeaderToBecome(client, topicPartition, None)
+      servers(broker3).startup()
+
+      LeaderElectionCommand.main(
+        Array(
+          "--bootstrap-server", bootstrapServers(servers),
+          "--election-type", "unclean",
+          "--all-topic-partitions"
+        )
+      )
+
+      assertEquals(Option(broker3), currentLeader(client, topicPartition))
+    }
+  }
+
+  @Test
+  def testTopicPartition(): Unit = {
+    TestUtils.resource(JAdminClient.create(createConfig(servers).asJava)) { client =>
+      val topic = "unclean-topic"
+      val partition = 0
+      val assignment = Seq(broker2, broker3)
+
+      TestUtils.createTopic(zkClient, topic, Map(partition -> assignment), servers)
+
+      val topicPartition = new TopicPartition(topic, partition)
+
+      waitForLeaderToBecome(client, topicPartition, Option(broker2))
+
+      servers(broker3).shutdown()
+      waitForBrokerOutOfIsr(client, Set(topicPartition), broker3)
+      servers(broker2).shutdown()
+      waitForLeaderToBecome(client, topicPartition, None)
+      servers(broker3).startup()
+
+      LeaderElectionCommand.main(
+        Array(
+          "--bootstrap-server", bootstrapServers(servers),
+          "--election-type", "unclean",
+          "--topic", topic,
+          "--partition", partition.toString
+        )
+      )
+
+      assertEquals(Option(broker3), currentLeader(client, topicPartition))
+    }
+  }
+
+  @Test
+  def testPathToJsonFile(): Unit = {
+    TestUtils.resource(JAdminClient.create(createConfig(servers).asJava)) { client =>
+      val topic = "unclean-topic"
+      val partition = 0
+      val assignment = Seq(broker2, broker3)
+
+      TestUtils.createTopic(zkClient, topic, Map(partition -> assignment), servers)
+
+      val topicPartition = new TopicPartition(topic, partition)
+
+      waitForLeaderToBecome(client, topicPartition, Option(broker2))
+
+      servers(broker3).shutdown()
+      waitForBrokerOutOfIsr(client, Set(topicPartition), broker3)
+      servers(broker2).shutdown()
+      waitForLeaderToBecome(client, topicPartition, None)
+      servers(broker3).startup()
+
+      val topicPartitionPath = tempTopicPartitionFile(Set(topicPartition))
+
+      LeaderElectionCommand.main(
+        Array(
+          "--bootstrap-server", bootstrapServers(servers),
+          "--election-type", "unclean",
+          "--path-to-json-file", topicPartitionPath.toString
+        )
+      )
+
+      assertEquals(Option(broker3), currentLeader(client, topicPartition))
+    }
+  }
+
+  @Test
+  def testPreferredReplicaElection(): Unit = {
+    TestUtils.resource(JAdminClient.create(createConfig(servers).asJava)) { client =>
+      val topic = "unclean-topic"
+      val partition = 0
+      val assignment = Seq(broker2, broker3)
+
+      TestUtils.createTopic(zkClient, topic, Map(partition -> assignment), servers)
+
+      val topicPartition = new TopicPartition(topic, partition)
+
+      waitForLeaderToBecome(client, topicPartition, Option(broker2))
+
+      servers(broker2).shutdown()
+      waitForLeaderToBecome(client, topicPartition, Some(broker3))
+      servers(broker2).startup()
+      waitForBrokerInIsr(client, Set(topicPartition), broker2)
+
+      LeaderElectionCommand.main(
+        Array(
+          "--bootstrap-server", bootstrapServers(servers),
+          "--election-type", "preferred",
+          "--all-topic-partitions"
+        )
+      )
+
+      assertEquals(Option(broker2), currentLeader(client, topicPartition))
+    }
+  }
+
+  @Test
+  def testTopicWithoutPartition(): Unit = {
+    try {
+      LeaderElectionCommand.main(
+        Array(
+          "--bootstrap-server", bootstrapServers(servers),
+          "--election-type", "unclean",
+          "--topic", "some-topic"
+        )
+      )
+      fail()
+    } catch {
+      case e: Throwable =>
+        assertTrue(e.getMessage.startsWith("Missing required option(s)"))
+        assertTrue(e.getMessage.contains(" partition"))
+    }
+  }
+
+  @Test
+  def testPartitionWithoutTopic(): Unit = {
+    try {
+      LeaderElectionCommand.main(
+        Array(
+          "--bootstrap-server", bootstrapServers(servers),
+          "--election-type", "unclean",
+          "--all-topic-partitions",
+          "--partition", "0"
+        )
+      )
+      fail()
+    } catch {
+      case e: Throwable =>
+        assertEquals("Option partition is only allowed if topic is used", e.getMessage)
+    }
+  }
+
+  @Test
+  def testTopicDoesNotExist(): Unit = {
+    try {
+      LeaderElectionCommand.main(
+        Array(
+          "--bootstrap-server", bootstrapServers(servers),
+          "--election-type", "preferred",
+          "--topic", "unknown-topic-name",
+          "--partition", "0"
+        )
+      )
+      fail()
+    } catch {
+      case e: AdminCommandFailedException =>
+        assertTrue(e.getSuppressed()(0).isInstanceOf[UnknownTopicOrPartitionException])
+    }
+  }
+
+  @Test
+  def testMissingElectionType(): Unit = {
+    try {
+      LeaderElectionCommand.main(
+        Array(
+          "--bootstrap-server", bootstrapServers(servers),
+          "--topic", "some-topic",
+          "--partition", "0"
+        )
+      )
+      fail()
+    } catch {
+      case e: Throwable =>
+        assertTrue(e.getMessage.startsWith("Missing required option(s)"))
+        assertTrue(e.getMessage.contains(" election-type"))
+    }
+  }
+
+  @Test
+  def testMissingTopicPartitionSelection(): Unit = {
+    try {
+      LeaderElectionCommand.main(
+        Array(
+          "--bootstrap-server", bootstrapServers(servers),
+          "--election-type", "preferrred"
+        )
+      )
+      fail()
+    } catch {
+      case e: Throwable =>
+        assertTrue(e.getMessage.startsWith("One and only one of the following options is required: "))
+        assertTrue(e.getMessage.contains(" all-topic-partitions"))
+        assertTrue(e.getMessage.contains(" topic"))
+        assertTrue(e.getMessage.contains(" path-to-json-file"))
+    }
+  }
+
+  @Test
+  def testInvalidBroker(): Unit = {
+    try {
+      LeaderElectionCommand.run(
+        Array(
+          "--bootstrap-server", "example.com:1234",
+          "--election-type", "unclean",
+          "--all-topic-partitions"
+        ),
+        1.seconds
+      )
+      fail()
+    } catch {
+      case e: AdminCommandFailedException =>
+        assertTrue(e.getCause.isInstanceOf[TimeoutException])
+    }
+  }
+}
+
+object LeaderElectionCommandTest {
+  def createConfig(servers: Seq[KafkaServer]): Map[String, Object] = {
+    Map(
+      AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG -> bootstrapServers(servers),
+      AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG -> "20000"
+    )
+  }
+
+  def bootstrapServers(servers: Seq[KafkaServer]): String = {
+    servers.map { server =>
+      val port = server.socketServer.boundPort(ListenerName.normalised("PLAINTEXT"))
+      s"localhost:$port"
+    }.headOption.mkString(",")
+  }
+
+  def currentLeader(client: JAdminClient, topicPartition: TopicPartition): Option[Int] = {
+    Option(
+      client
+        .describeTopics(List(topicPartition.topic).asJava)
+        .all
+        .get
+        .get(topicPartition.topic)
+        .partitions
+        .get(topicPartition.partition)
+        .leader
+    ).map(_.id)
+  }
+
+  def waitForLeaderToBecome(client: JAdminClient, topicPartition: TopicPartition, leader: Option[Int]): Unit = {
+    TestUtils.waitUntilTrue(
+      () => currentLeader(client, topicPartition) == leader,
+      s"Expected leader to become $leader", 10000
+    )
+  }
+
+  def waitForBrokerOutOfIsr(client: JAdminClient, partitions: Set[TopicPartition], brokerId: Int): Unit = {
+    TestUtils.waitUntilTrue(
+      () => {
+        val description = client.describeTopics(partitions.map(_.topic).asJava).all.get.asScala
+        val isr = description.values.flatMap(_.partitions.asScala.flatMap(_.isr.asScala))
+        isr.forall(_.id != brokerId)
+      },
+      s"Expect broker $brokerId to no longer be in any ISR for $partitions"
+    )
+  }
+
+  def waitForBrokerInIsr(client: JAdminClient, partitions: Set[TopicPartition], brokerId: Int): Unit = {
+    TestUtils.waitUntilTrue(
+      () => {
+        val description = client.describeTopics(partitions.map(_.topic).asJava).all.get.asScala
+        val isr = description.values.flatMap(_.partitions.asScala.flatMap(_.isr.asScala))
+        isr.exists(_.id == brokerId)
+      },
+      s"Expect broker $brokerId to no longer be in any ISR for $partitions"
+    )
+  }
+
+  def tempTopicPartitionFile(partitions: Set[TopicPartition]): Path = {
+    val file = File.createTempFile("leader-election-command", ".json")
+    file.deleteOnExit()
+
+    val jsonString = TestUtils.stringifyTopicPartitions(partitions)
+
+    Files.write(file.toPath, jsonString.getBytes(StandardCharsets.UTF_8))
+
+    file.toPath
+  }
+}
diff --git a/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala b/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala
index 7ee2c59b98baa..03ffd2f4f8dd3 100644
--- a/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala
@@ -21,11 +21,11 @@ import java.nio.charset.StandardCharsets
 import java.nio.file.{Files, Paths}
 import java.util.Properties
 
-import kafka.common.{AdminCommandFailedException, TopicAndPartition}
+import kafka.common.AdminCommandFailedException
 import kafka.network.RequestChannel
 import kafka.security.auth._
 import kafka.server.{KafkaConfig, KafkaServer}
-import kafka.utils.{Logging, TestUtils, ZkUtils}
+import kafka.utils.{Logging, TestUtils}
 import kafka.zk.ZooKeeperTestHarness
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.ClusterAuthorizationException
@@ -152,10 +152,10 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
     assertEquals(testPartitionPreferredLeader, getLeader(testPartition))
   }
 
-  private def toJsonFile(partitions: scala.collection.Set[TopicPartition]): File = {
+  private def toJsonFile(partitions: Set[TopicPartition]): File = {
     val jsonFile = File.createTempFile("preferredreplicaelection", ".js")
     jsonFile.deleteOnExit()
-    val jsonString = ZkUtils.preferredReplicaLeaderElectionZkData(partitions.map(new TopicAndPartition(_)))
+    val jsonString = TestUtils.stringifyTopicPartitions(partitions)
     debug("Using json: "+jsonString)
     Files.write(Paths.get(jsonFile.getAbsolutePath), jsonString.getBytes(StandardCharsets.UTF_8))
     jsonFile
diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index 3f9c8c3cfb25a..ca62a7953c5e9 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -1498,4 +1498,19 @@ object TestUtils extends Logging {
       Metrics.defaultRegistry.removeMetric(metricName)
   }
 
+  def stringifyTopicPartitions(partitions: Set[TopicPartition]): String = {
+    Json.legacyEncodeAsString(
+      Map(
+        "partitions" -> partitions.map(tp => Map("topic" -> tp.topic, "partition" -> tp.partition))
+      )
+    )
+  }
+
+  def resource[R <: AutoCloseable, A](resource: R)(func: R => A): A = {
+    try {
+      func(resource)
+    } finally {
+      resource.close()
+    }
+  }
 }

From 060701fb6f000046e19ccb94a5e366377d0a2599 Mon Sep 17 00:00:00 2001
From: Manikumar Reddy 
Date: Fri, 7 Jun 2019 12:15:03 +0530
Subject: [PATCH 0342/1071] KAFKA-8461: Wait for follower to join the ISR in
 testUncleanLeaderElectionDisabled Test

Author: Manikumar Reddy 

Reviewers: Jason Gustafson , Boyang Chen 

Closes #6887 from omkreddy/unclean-leader
---
 .../integration/UncleanLeaderElectionTest.scala  | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala b/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala
index c0c8c95b65959..64d717424a143 100755
--- a/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala
+++ b/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala
@@ -219,16 +219,16 @@ class UncleanLeaderElectionTest extends ZooKeeperTestHarness {
     assertEquals(List("first"), consumeAllMessages(topic, 1))
 
     // shutdown follower server
-    servers.filter(server => server.config.brokerId == followerId).map(server => shutdownServer(server))
+    servers.filter(server => server.config.brokerId == followerId).foreach(server => shutdownServer(server))
 
     produceMessage(servers, topic, "second")
     assertEquals(List("first", "second"), consumeAllMessages(topic, 2))
 
     //remove any previous unclean election metric
-    servers.map(server => server.kafkaController.controllerContext.stats.removeMetric("UncleanLeaderElectionsPerSec"))
+    servers.foreach(server => server.kafkaController.controllerContext.stats.removeMetric("UncleanLeaderElectionsPerSec"))
 
     // shutdown leader and then restart follower
-    servers.filter(server => server.config.brokerId == leaderId).map(server => shutdownServer(server))
+    servers.filter(server => server.config.brokerId == leaderId).foreach(server => shutdownServer(server))
     val followerServer = servers.find(_.config.brokerId == followerId).get
     followerServer.startup()
 
@@ -247,13 +247,17 @@ class UncleanLeaderElectionTest extends ZooKeeperTestHarness {
     assertEquals(List.empty[String], consumeAllMessages(topic, 0))
 
     // restart leader temporarily to send a successfully replicated message
-    servers.filter(server => server.config.brokerId == leaderId).map(server => server.startup())
+    servers.filter(server => server.config.brokerId == leaderId).foreach(server => server.startup())
     waitUntilLeaderIsElectedOrChanged(zkClient, topic, partitionId, newLeaderOpt = Some(leaderId))
 
     produceMessage(servers, topic, "third")
-    waitUntilMetadataIsPropagated(servers, topic, partitionId)
-    servers.filter(server => server.config.brokerId == leaderId).map(server => shutdownServer(server))
+    //make sure follower server joins the ISR
+    TestUtils.waitUntilTrue(() => {
+      val partitionInfoOpt = followerServer.metadataCache.getPartitionInfo(topic, partitionId)
+      partitionInfoOpt.isDefined && partitionInfoOpt.get.basePartitionState.isr.contains(followerId)
+    }, "Inconsistent metadata after first server startup")
 
+    servers.filter(server => server.config.brokerId == leaderId).foreach(server => shutdownServer(server))
     // verify clean leader transition to ISR follower
     waitUntilLeaderIsElectedOrChanged(zkClient, topic, partitionId, newLeaderOpt = Some(followerId))
 

From 07dd7db324ac623190e48720ce446d82320edc36 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Fri, 7 Jun 2019 00:48:18 -0700
Subject: [PATCH 0343/1071] MINOR: Increase timeout for flaky
 ResetConsumerGroupOffsetTest (#6900)

Reviewers: Manikumar Reddy 
---
 .../admin/ResetConsumerGroupOffsetTest.scala  | 104 +++++++++---------
 1 file changed, 49 insertions(+), 55 deletions(-)

diff --git a/core/src/test/scala/unit/kafka/admin/ResetConsumerGroupOffsetTest.scala b/core/src/test/scala/unit/kafka/admin/ResetConsumerGroupOffsetTest.scala
index baf1d05801aa3..8b1bad66da954 100644
--- a/core/src/test/scala/unit/kafka/admin/ResetConsumerGroupOffsetTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ResetConsumerGroupOffsetTest.scala
@@ -22,6 +22,7 @@ import kafka.server.KafkaConfig
 import kafka.utils.TestUtils
 import org.apache.kafka.clients.producer.ProducerRecord
 import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.test
 import org.junit.Assert._
 import org.junit.Test
 
@@ -81,11 +82,29 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
       .map(KafkaConfig.fromProps(_, overridingProps))
   }
 
+  private def basicArgs: Array[String] = {
+    Array("--reset-offsets",
+      "--bootstrap-server", brokerList,
+      "--timeout", test.TestUtils.DEFAULT_MAX_WAIT_MS.toString)
+  }
+
+  private def buildArgsForGroups(groups: Seq[String], args: String*): Array[String] = {
+    val groupArgs = groups.flatMap(group => Seq("--group", group)).toArray
+    basicArgs ++ groupArgs ++ args
+  }
+
+  private def buildArgsForGroup(group: String, args: String*): Array[String] = {
+    buildArgsForGroups(Seq(group), args: _*)
+  }
+
+  private def buildArgsForAllGroups(args: String*): Array[String] = {
+    basicArgs ++ Array("--all-groups") ++ args
+  }
+
   @Test
   def testResetOffsetsNotExistingGroup() {
     val group = "missing.group"
-    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
-      "--to-current", "--execute")
+    val args = buildArgsForGroup(group, "--all-topics", "--to-current", "--execute")
     val consumerGroupCommand = getConsumerGroupService(args)
     // Make sure we got a coordinator
     TestUtils.waitUntilTrue(() => {
@@ -99,8 +118,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
   @Test
   def testResetOffsetsExistingTopic(): Unit = {
     val group = "new.group"
-    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--topic", topic,
-      "--to-offset", "50")
+    val args = buildArgsForGroup(group, "--topic", topic, "--to-offset", "50")
     produceMessages(topic, 100)
     resetAndAssertOffsets(args, expectedOffset = 50, dryRun = true)
     resetAndAssertOffsets(args ++ Array("--dry-run"), expectedOffset = 50, dryRun = true)
@@ -110,16 +128,15 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
   @Test
   def testResetOffsetsExistingTopicSelectedGroups(): Unit = {
     produceMessages(topic, 100)
-    val groups = (
+    val groups =
       for (id <- 1 to 3) yield {
         val group = this.group + id
         val executor = addConsumerGroupExecutor(numConsumers = 1, topic = topic, group = group)
         awaitConsumerProgress(count = 100L, group = group)
         executor.shutdown()
-        Array("--group", group)
-      }).toArray.flatten
-    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--topic", topic,
-      "--to-offset", "50") ++ groups
+        group
+      }
+    val args = buildArgsForGroups(groups,"--topic", topic, "--to-offset", "50")
     resetAndAssertOffsets(args, expectedOffset = 50, dryRun = true)
     resetAndAssertOffsets(args ++ Array("--dry-run"), expectedOffset = 50, dryRun = true)
     resetAndAssertOffsets(args ++ Array("--execute"), expectedOffset = 50)
@@ -127,8 +144,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
 
   @Test
   def testResetOffsetsExistingTopicAllGroups(): Unit = {
-    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--all-groups", "--topic", topic,
-      "--to-offset", "50")
+    val args = buildArgsForAllGroups("--topic", topic, "--to-offset", "50")
     produceMessages(topic, 100)
     for (group <- 1 to 3 map (group + _)) {
       val executor = addConsumerGroupExecutor(numConsumers = 1, topic = topic, group = group)
@@ -142,8 +158,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
 
   @Test
   def testResetOffsetsAllTopicsAllGroups(): Unit = {
-    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--all-groups", "--all-topics",
-      "--to-offset", "50")
+    val args = buildArgsForAllGroups("--all-topics", "--to-offset", "50")
     val topics = 1 to 3 map (topic + _)
     val groups = 1 to 3 map (group + _)
     topics foreach (topic => produceMessages(topic, 100))
@@ -172,8 +187,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
     awaitConsumerProgress(count = 100L)
     executor.shutdown()
 
-    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
-      "--to-datetime", format.format(calendar.getTime), "--execute")
+    val args = buildArgsForGroup(group, "--all-topics", "--to-datetime", format.format(calendar.getTime), "--execute")
     resetAndAssertOffsets(args, expectedOffset = 0)
   }
 
@@ -189,39 +203,34 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
     awaitConsumerProgress(count = 100L)
     executor.shutdown()
 
-    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
-      "--to-datetime", format.format(checkpoint), "--execute")
+    val args = buildArgsForGroup(group, "--all-topics", "--to-datetime", format.format(checkpoint), "--execute")
     resetAndAssertOffsets(args, expectedOffset = 50)
   }
 
   @Test
   def testResetOffsetsByDuration() {
-    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
-      "--by-duration", "PT1M", "--execute")
+    val args = buildArgsForGroup(group, "--all-topics", "--by-duration", "PT1M", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     resetAndAssertOffsets(args, expectedOffset = 0)
   }
 
   @Test
   def testResetOffsetsByDurationToEarliest() {
-    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
-      "--by-duration", "PT0.1S", "--execute")
+    val args = buildArgsForGroup(group, "--all-topics", "--by-duration", "PT0.1S", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     resetAndAssertOffsets(args, expectedOffset = 100)
   }
 
   @Test
   def testResetOffsetsToEarliest() {
-    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
-      "--to-earliest", "--execute")
+    val args = buildArgsForGroup(group, "--all-topics", "--to-earliest", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     resetAndAssertOffsets(args, expectedOffset = 0)
   }
 
   @Test
   def testResetOffsetsToLatest() {
-    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
-      "--to-latest", "--execute")
+    val args = buildArgsForGroup(group, "--all-topics", "--to-latest", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     produceMessages(topic, 100)
     resetAndAssertOffsets(args, expectedOffset = 200)
@@ -229,8 +238,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
 
   @Test
   def testResetOffsetsToCurrentOffset() {
-    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
-      "--to-current", "--execute")
+    val args = buildArgsForGroup(group, "--all-topics", "--to-current", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     produceMessages(topic, 100)
     resetAndAssertOffsets(args, expectedOffset = 100)
@@ -238,16 +246,14 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
 
   @Test
   def testResetOffsetsToSpecificOffset() {
-    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
-      "--to-offset", "1", "--execute")
+    val args = buildArgsForGroup(group, "--all-topics", "--to-offset", "1", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     resetAndAssertOffsets(args, expectedOffset = 1)
   }
 
   @Test
   def testResetOffsetsShiftPlus() {
-    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
-      "--shift-by", "50", "--execute")
+    val args = buildArgsForGroup(group, "--all-topics", "--shift-by", "50", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     produceMessages(topic, 100)
     resetAndAssertOffsets(args, expectedOffset = 150)
@@ -255,8 +261,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
 
   @Test
   def testResetOffsetsShiftMinus() {
-    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
-      "--shift-by", "-50", "--execute")
+    val args = buildArgsForGroup(group, "--all-topics", "--shift-by", "-50", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     produceMessages(topic, 100)
     resetAndAssertOffsets(args, expectedOffset = 50)
@@ -264,8 +269,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
 
   @Test
   def testResetOffsetsShiftByLowerThanEarliest() {
-    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
-      "--shift-by", "-150", "--execute")
+    val args = buildArgsForGroup(group, "--all-topics", "--shift-by", "-150", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     produceMessages(topic, 100)
     resetAndAssertOffsets(args, expectedOffset = 0)
@@ -273,8 +277,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
 
   @Test
   def testResetOffsetsShiftByHigherThanLatest() {
-    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
-      "--shift-by", "150", "--execute")
+    val args = buildArgsForGroup(group, "--all-topics", "--shift-by", "150", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     produceMessages(topic, 100)
     resetAndAssertOffsets(args, expectedOffset = 200)
@@ -282,8 +285,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
 
   @Test
   def testResetOffsetsToEarliestOnOneTopic() {
-    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--topic", topic,
-      "--to-earliest", "--execute")
+    val args = buildArgsForGroup(group, "--topic", topic, "--to-earliest", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     resetAndAssertOffsets(args, expectedOffset = 0)
   }
@@ -293,8 +295,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
     val topic = "bar"
     createTopic(topic, 2, 1)
 
-    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--topic",
-      s"$topic:1", "--to-earliest", "--execute")
+    val args = buildArgsForGroup(group, "--topic", s"$topic:1", "--to-earliest", "--execute")
     val consumerGroupCommand = getConsumerGroupService(args)
 
     produceConsumeAndShutdown(topic, group, totalMessages = 100, numConsumers = 2)
@@ -315,8 +316,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
     createTopic(topic1, 1, 1)
     createTopic(topic2, 1, 1)
 
-    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--topic", topic1,
-      "--topic", topic2, "--to-earliest", "--execute")
+    val args = buildArgsForGroup(group, "--topic", topic1, "--topic", topic2, "--to-earliest", "--execute")
     val consumerGroupCommand = getConsumerGroupService(args)
 
     produceConsumeAndShutdown(topic1, group, 100, 1)
@@ -342,8 +342,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
     createTopic(topic1, 2, 1)
     createTopic(topic2, 2, 1)
 
-    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--topic",
-      s"$topic1:1", "--topic", s"$topic2:1", "--to-earliest", "--execute")
+    val args = buildArgsForGroup(group, "--topic", s"$topic1:1", "--topic", s"$topic2:1", "--to-earliest", "--execute")
     val consumerGroupCommand = getConsumerGroupService(args)
 
     produceConsumeAndShutdown(topic1, group, 100, 2)
@@ -372,8 +371,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
     val tp1 = new TopicPartition(topic, 1)
     createTopic(topic, 2, 1)
 
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
-      "--to-offset", "2", "--export")
+    val cgcArgs = buildArgsForGroup(group, "--all-topics", "--to-offset", "2", "--export")
     val consumerGroupCommand = getConsumerGroupService(cgcArgs)
 
     produceConsumeAndShutdown(topic = topic, group = group, totalMessages = 100, numConsumers = 2)
@@ -387,8 +385,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
     bw.close()
     assertEquals(Map(tp0 -> 2L, tp1 -> 2L), exportedOffsets(group).mapValues(_.offset))
 
-    val cgcArgsExec = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
-      "--from-file", file.getCanonicalPath, "--dry-run")
+    val cgcArgsExec = buildArgsForGroup(group, "--all-topics", "--from-file", file.getCanonicalPath, "--dry-run")
     val consumerGroupCommandExec = getConsumerGroupService(cgcArgsExec)
     val importedOffsets = consumerGroupCommandExec.resetOffsets()
     assertEquals(Map(tp0 -> 2L, tp1 -> 2L), importedOffsets(group).mapValues(_.offset))
@@ -411,8 +408,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
     createTopic(topic1, 2, 1)
     createTopic(topic2, 2, 1)
 
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group1, "--group", group2, "--all-topics",
-      "--to-offset", "2", "--export")
+    val cgcArgs = buildArgsForGroups(Seq(group1, group2), "--all-topics", "--to-offset", "2", "--export")
     val consumerGroupCommand = getConsumerGroupService(cgcArgs)
 
     produceConsumeAndShutdown(topic = topic1, group = group1, totalMessages = 100, numConsumers = 2)
@@ -429,16 +425,14 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
     assertEquals(Map(t2p0 -> 2L, t2p1 -> 2L), exportedOffsets(group2).mapValues(_.offset))
 
     // Multiple --group's offset import
-    val cgcArgsExec = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group1, "--group", group2, "--all-topics",
-      "--from-file", file.getCanonicalPath, "--dry-run")
+    val cgcArgsExec = buildArgsForGroups(Seq(group1, group2), "--all-topics", "--from-file", file.getCanonicalPath, "--dry-run")
     val consumerGroupCommandExec = getConsumerGroupService(cgcArgsExec)
     val importedOffsets = consumerGroupCommandExec.resetOffsets()
     assertEquals(Map(t1p0 -> 2L, t1p1 -> 2L), importedOffsets(group1).mapValues(_.offset))
     assertEquals(Map(t2p0 -> 2L, t2p1 -> 2L), importedOffsets(group2).mapValues(_.offset))
 
     // Single --group offset import using "group,topic,partition,offset" csv format
-    val cgcArgsExec2 = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group1, "--all-topics",
-      "--from-file", file.getCanonicalPath, "--dry-run")
+    val cgcArgsExec2 = buildArgsForGroup(group1, "--all-topics", "--from-file", file.getCanonicalPath, "--dry-run")
     val consumerGroupCommandExec2 = getConsumerGroupService(cgcArgsExec2)
     val importedOffsets2 = consumerGroupCommandExec2.resetOffsets()
     assertEquals(Map(t1p0 -> 2L, t1p1 -> 2L), importedOffsets2(group1).mapValues(_.offset))

From bb8de0b8c5f98f7a9d6b5ae7342ba7a0e1af8868 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Fri, 7 Jun 2019 12:51:51 -0700
Subject: [PATCH 0344/1071] KAFKA-8003; Fix flaky
 testFencingOnTransactionExpiration

We see this failure from time to time:
```
java.lang.AssertionError: expected:<1> but was:<0>
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:633)
	at kafka.api.TransactionsTest.testFencingOnTransactionExpiration(TransactionsTest.scala:512)
```
The cause is probably that we are using `consumeRecordsFor` which has no expectation on the number of records to fetch and a timeout of just 1s. This patch changes the code to use `consumeRecords` and the default 15s timeout.

Note we have also fixed a bug in the test case itself, which was using the wrong topic for the second write, which meant it could never have failed in the anticipated way anyway.

Author: Jason Gustafson 

Reviewers: Gwen Shapira

Closes #6905 from hachikuji/fix-flaky-transaction-test
---
 .../scala/integration/kafka/api/TransactionsTest.scala | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/core/src/test/scala/integration/kafka/api/TransactionsTest.scala b/core/src/test/scala/integration/kafka/api/TransactionsTest.scala
index 375adaaed4c79..13ddd921003ad 100644
--- a/core/src/test/scala/integration/kafka/api/TransactionsTest.scala
+++ b/core/src/test/scala/integration/kafka/api/TransactionsTest.scala
@@ -497,7 +497,7 @@ class TransactionsTest extends KafkaServerTestHarness {
 
     try {
       // Now that the transaction has expired, the second send should fail with a ProducerFencedException.
-      producer.send(TestUtils.producerRecordWithExpectedTransactionStatus(topic2, "2", "2", willBeCommitted = false)).get()
+      producer.send(TestUtils.producerRecordWithExpectedTransactionStatus(topic1, "2", "2", willBeCommitted = false)).get()
       fail("should have raised a ProducerFencedException since the transaction has expired")
     } catch {
       case _: ProducerFencedException =>
@@ -506,9 +506,13 @@ class TransactionsTest extends KafkaServerTestHarness {
     }
 
     // Verify that the first message was aborted and the second one was never written at all.
-    val nonTransactionalConsumer = nonTransactionalConsumers(0)
+    val nonTransactionalConsumer = nonTransactionalConsumers.head
     nonTransactionalConsumer.subscribe(List(topic1).asJava)
-    val records = TestUtils.consumeRecordsFor(nonTransactionalConsumer, 1000)
+
+    // Attempt to consume the one written record. We should not see the second. The
+    // assertion does not strictly guarantee that the record wasn't written, but the
+    // data is small enough that had it been written, it would have been in the first fetch.
+    val records = TestUtils.consumeRecords(nonTransactionalConsumer, numRecords = 1)
     assertEquals(1, records.size)
     assertEquals("1", TestUtils.recordValueAsString(records.head))
 

From cca05cace4105c829f303c13eed8ace2efd7fa0c Mon Sep 17 00:00:00 2001
From: Boyang Chen 
Date: Fri, 7 Jun 2019 13:52:12 -0700
Subject: [PATCH 0345/1071] KAFKA-8331: stream static membership system test
 (#6877)

As title suggested, we boost 3 stream instances stream job with one minute session timeout, and once the group is stable, doing couple of rolling bounces for the entire cluster. Every rejoin based on restart should have no generation bump on the client side.

Reviewers: Guozhang Wang ,  Bill Bejeck 
---
 .../internals/AbstractCoordinator.java        |   5 +
 .../internals/ConsumerCoordinator.java        |   3 +-
 .../distributed/WorkerCoordinator.java        |   1 +
 .../streams/tests/StaticMemberTestClient.java |  84 ++++++++++++++
 .../tests/StreamsNamedRepartitionTest.java    |   2 -
 tests/kafkatest/services/consumer_property.py |  21 ++++
 tests/kafkatest/services/streams.py           |  22 ++++
 tests/kafkatest/services/streams_property.py  |   4 -
 .../streams_named_repartition_topic_test.py   |  35 +-----
 .../tests/streams/streams_optimized_test.py   |  21 +---
 .../streams/streams_static_membership_test.py | 108 ++++++++++++++++++
 .../tests/streams/streams_upgrade_test.py     |  24 ++--
 .../kafkatest/tests/streams/utils/__init__.py |  16 +++
 tests/kafkatest/tests/streams/utils/util.py   |  36 ++++++
 14 files changed, 316 insertions(+), 66 deletions(-)
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/tests/StaticMemberTestClient.java
 create mode 100644 tests/kafkatest/services/consumer_property.py
 create mode 100644 tests/kafkatest/tests/streams/streams_static_membership_test.py
 create mode 100644 tests/kafkatest/tests/streams/utils/__init__.py
 create mode 100644 tests/kafkatest/tests/streams/utils/util.py

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index 54678f7ada3eb..73563fd96b86b 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -802,6 +802,11 @@ protected synchronized Generation generation() {
         return generation;
     }
 
+    protected synchronized String memberId() {
+        return generation == null ? JoinGroupRequest.UNKNOWN_MEMBER_ID :
+                generation.memberId;
+    }
+
     /**
      * Check whether given generation id is matching the record within current generation.
      * Only using in unit tests.
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 5d39da5d4b525..64bf17d96f213 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -585,7 +585,8 @@ public void close(final Timer timer) {
     // visible for testing
     void invokeCompletedOffsetCommitCallbacks() {
         if (asyncCommitFenced.get()) {
-            throw new FencedInstanceIdException("Get fenced exception for group.instance.id " + groupInstanceId.orElse("unset_instance_id"));
+            throw new FencedInstanceIdException("Get fenced exception for group.instance.id: " +
+                    groupInstanceId.orElse("unset_instance_id") + ", current member.id is " + memberId());
         }
         while (true) {
             OffsetCommitCompletion completion = completedOffsetCommits.poll();
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java
index fd7c7a429f26a..230a2725ae46e 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java
@@ -227,6 +227,7 @@ protected boolean rejoinNeededOrPending() {
         return super.rejoinNeededOrPending() || (assignmentSnapshot == null || assignmentSnapshot.failed()) || rejoinRequested;
     }
 
+    @Override
     public String memberId() {
         Generation generation = generation();
         if (generation != null)
diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/StaticMemberTestClient.java b/streams/src/test/java/org/apache/kafka/streams/tests/StaticMemberTestClient.java
new file mode 100644
index 0000000000000..96ccad44de72d
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/StaticMemberTestClient.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.tests;
+
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.kstream.KStream;
+
+import java.util.Objects;
+import java.util.Properties;
+
+public class StaticMemberTestClient {
+
+    private static String testName = "StaticMemberTestClient";
+
+    @SuppressWarnings("unchecked")
+    public static void main(final String[] args) throws Exception {
+        if (args.length < 1) {
+            System.err.println(testName + " requires one argument (properties-file) but none provided: ");
+        }
+
+        System.out.println("StreamsTest instance started");
+
+        final String propFileName = args[0];
+
+        final Properties streamsProperties = Utils.loadProps(propFileName);
+
+        final String groupInstanceId = Objects.requireNonNull(streamsProperties.getProperty(ConsumerConfig.GROUP_INSTANCE_ID_CONFIG));
+
+        System.out.println(testName + " instance started with group.instance.id " + groupInstanceId);
+        System.out.println("props=" + streamsProperties);
+        System.out.flush();
+
+        final StreamsBuilder builder = new StreamsBuilder();
+        final String inputTopic = (String) (Objects.requireNonNull(streamsProperties.remove("input.topic")));
+
+        final KStream dataStream = builder.stream(inputTopic);
+        dataStream.peek((k, v) ->  System.out.println(String.format("PROCESSED key=%s value=%s", k, v)));
+
+        final Properties config = new Properties();
+        config.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, testName);
+        config.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000);
+
+        config.putAll(streamsProperties);
+
+        final KafkaStreams streams = new KafkaStreams(builder.build(), config);
+        streams.setStateListener((newState, oldState) -> {
+            if (oldState == KafkaStreams.State.REBALANCING && newState == KafkaStreams.State.RUNNING) {
+                System.out.println("REBALANCING -> RUNNING");
+                System.out.flush();
+            }
+        });
+
+        streams.start();
+
+        Runtime.getRuntime().addShutdownHook(new Thread() {
+            @Override
+            public void run() {
+                System.out.println("closing Kafka Streams instance");
+                System.out.flush();
+                streams.close();
+                System.out.println("Static membership test closed");
+                System.out.flush();
+            }
+        });
+    }
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsNamedRepartitionTest.java b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsNamedRepartitionTest.java
index 911716f348ec1..c408d9f3da592 100644
--- a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsNamedRepartitionTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsNamedRepartitionTest.java
@@ -117,7 +117,5 @@ public static void main(final String[] args) throws Exception {
             System.out.println("NAMED_REPARTITION_TEST Streams Stopped");
             System.out.flush();
         }));
-
     }
-
 }
diff --git a/tests/kafkatest/services/consumer_property.py b/tests/kafkatest/services/consumer_property.py
new file mode 100644
index 0000000000000..0a9756a65ca11
--- /dev/null
+++ b/tests/kafkatest/services/consumer_property.py
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Define Consumer configuration property names here.
+"""
+
+GROUP_INSTANCE_ID = "group.instance.id"
+SESSION_TIMEOUT_MS = "session.timeout.ms"
diff --git a/tests/kafkatest/services/streams.py b/tests/kafkatest/services/streams.py
index 5e2c2e9e00c70..70564b90f383d 100644
--- a/tests/kafkatest/services/streams.py
+++ b/tests/kafkatest/services/streams.py
@@ -16,6 +16,7 @@
 import os.path
 import signal
 import streams_property
+import consumer_property
 from ducktape.services.service import Service
 from ducktape.utils.util import wait_until
 from kafkatest.directory_layout.kafka_path import KafkaPathResolverMixin
@@ -534,3 +535,24 @@ def prop_file(self):
 
         cfg = KafkaConfig(**properties)
         return cfg.render()
+
+class StaticMemberTestService(StreamsTestBaseService):
+    def __init__(self, test_context, kafka, group_instance_id, num_threads):
+        super(StaticMemberTestService, self).__init__(test_context,
+                                                      kafka,
+                                                      "org.apache.kafka.streams.tests.StaticMemberTestClient",
+                                                      "")
+        self.INPUT_TOPIC = None
+        self.GROUP_INSTANCE_ID = group_instance_id
+        self.NUM_THREADS = num_threads
+    def prop_file(self):
+        properties = {streams_property.STATE_DIR: self.PERSISTENT_ROOT,
+                      streams_property.KAFKA_SERVERS: self.kafka.bootstrap_servers(),
+                      streams_property.NUM_THREADS: self.NUM_THREADS,
+                      consumer_property.GROUP_INSTANCE_ID: self.GROUP_INSTANCE_ID,
+                      consumer_property.SESSION_TIMEOUT_MS: 60000}
+
+        properties['input.topic'] = self.INPUT_TOPIC
+
+        cfg = KafkaConfig(**properties)
+        return cfg.render()
diff --git a/tests/kafkatest/services/streams_property.py b/tests/kafkatest/services/streams_property.py
index 054ea64625041..99f0ece5598d5 100644
--- a/tests/kafkatest/services/streams_property.py
+++ b/tests/kafkatest/services/streams_property.py
@@ -20,7 +20,3 @@
 STATE_DIR = "state.dir"
 KAFKA_SERVERS = "bootstrap.servers"
 NUM_THREADS = "num.stream.threads"
-
-
-
-
diff --git a/tests/kafkatest/tests/streams/streams_named_repartition_topic_test.py b/tests/kafkatest/tests/streams/streams_named_repartition_topic_test.py
index b9894eef7a6c8..1fcdd5fbe1166 100644
--- a/tests/kafkatest/tests/streams/streams_named_repartition_topic_test.py
+++ b/tests/kafkatest/tests/streams/streams_named_repartition_topic_test.py
@@ -13,14 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import time
 from ducktape.tests.test import Test
-from ducktape.utils.util import wait_until
 from kafkatest.services.kafka import KafkaService
 from kafkatest.services.streams import StreamsNamedRepartitionTopicService
 from kafkatest.services.verifiable_producer import VerifiableProducer
 from kafkatest.services.zookeeper import ZookeeperService
-
+from kafkatest.tests.streams.utils import verify_stopped, stop_processors, verify_running
 
 class StreamsNamedRepartitionTopicTest(Test):
     """
@@ -32,6 +30,7 @@ class StreamsNamedRepartitionTopicTest(Test):
     input_topic = 'inputTopic'
     aggregation_topic = 'aggregationTopic'
     pattern = 'AGGREGATED'
+    stopped_message = 'NAMED_REPARTITION_TEST Streams Stopped'
 
     def __init__(self, test_context):
         super(StreamsNamedRepartitionTopicTest, self).__init__(test_context)
@@ -66,43 +65,25 @@ def test_upgrade_topology_with_named_repartition_topic(self):
         for processor in processors:
             processor.CLEAN_NODE_ENABLED = False
             self.set_topics(processor)
-            self.verify_running(processor, 'REBALANCING -> RUNNING')
+            verify_running(processor, 'REBALANCING -> RUNNING')
 
         self.verify_processing(processors)
 
         # do rolling upgrade
         for processor in processors:
-            self.verify_stopped(processor)
+            verify_stopped(processor, self.stopped_message)
             #  will tell app to add operations before repartition topic
             processor.ADD_ADDITIONAL_OPS = 'true'
-            self.verify_running(processor, 'UPDATED Topology')
+            verify_running(processor, 'UPDATED Topology')
 
         self.verify_processing(processors)
 
-        self.stop_processors(processors)
+        stop_processors(processors, self.stopped_message)
 
         self.producer.stop()
         self.kafka.stop()
         self.zookeeper.stop()
 
-    @staticmethod
-    def verify_running(processor, message):
-        node = processor.node
-        with node.account.monitor_log(processor.STDOUT_FILE) as monitor:
-            processor.start()
-            monitor.wait_until(message,
-                               timeout_sec=60,
-                               err_msg="Never saw '%s' message " % message + str(processor.node.account))
-
-    @staticmethod
-    def verify_stopped(processor):
-        node = processor.node
-        with node.account.monitor_log(processor.STDOUT_FILE) as monitor:
-            processor.stop()
-            monitor.wait_until('NAMED_REPARTITION_TEST Streams Stopped',
-                               timeout_sec=60,
-                               err_msg="'NAMED_REPARTITION_TEST Streams Stopped' message" + str(processor.node.account))
-
     def verify_processing(self, processors):
         for processor in processors:
             with processor.node.account.monitor_log(processor.STDOUT_FILE) as monitor:
@@ -110,10 +91,6 @@ def verify_processing(self, processors):
                                    timeout_sec=60,
                                    err_msg="Never saw processing of %s " % self.pattern + str(processor.node.account))
 
-    def stop_processors(self, processors):
-        for processor in processors:
-            self.verify_stopped(processor)
-
     def set_topics(self, processor):
         processor.INPUT_TOPIC = self.input_topic
         processor.AGGREGATION_TOPIC = self.aggregation_topic
diff --git a/tests/kafkatest/tests/streams/streams_optimized_test.py b/tests/kafkatest/tests/streams/streams_optimized_test.py
index 31efc0d6b8f19..ecd84c2b2e085 100644
--- a/tests/kafkatest/tests/streams/streams_optimized_test.py
+++ b/tests/kafkatest/tests/streams/streams_optimized_test.py
@@ -15,12 +15,11 @@
 
 import time
 from ducktape.tests.test import Test
-from ducktape.utils.util import wait_until
 from kafkatest.services.kafka import KafkaService
 from kafkatest.services.streams import StreamsOptimizedUpgradeTestService
 from kafkatest.services.verifiable_producer import VerifiableProducer
 from kafkatest.services.zookeeper import ZookeeperService
-
+from kafkatest.tests.streams.utils import stop_processors
 
 class StreamsOptimizedTest(Test):
     """
@@ -33,6 +32,7 @@ class StreamsOptimizedTest(Test):
     reduce_topic = 'reduceTopic'
     join_topic = 'joinTopic'
     operation_pattern = 'AGGREGATED\|REDUCED\|JOINED'
+    stopped_message = 'OPTIMIZE_TEST Streams Stopped'
 
     def __init__(self, test_context):
         super(StreamsOptimizedTest, self).__init__(test_context)
@@ -75,7 +75,7 @@ def test_upgrade_optimized_topology(self):
 
         self.verify_processing(processors, verify_individual_operations=False)
 
-        self.stop_processors(processors)
+        stop_processors(processors, self.stopped_message)
 
         # start again with topology optimized
         for processor in processors:
@@ -84,7 +84,7 @@ def test_upgrade_optimized_topology(self):
 
         self.verify_processing(processors, verify_individual_operations=True)
 
-        self.stop_processors(processors)
+        stop_processors(processors, self.stopped_message)
 
         self.producer.stop()
         self.kafka.stop()
@@ -100,15 +100,6 @@ def verify_running_repartition_topic_count(processor, repartition_topic_count):
                                err_msg="Never saw 'REBALANCING -> RUNNING with REPARTITION TOPIC COUNT=%s' message "
                                        % repartition_topic_count + str(processor.node.account))
 
-    @staticmethod
-    def verify_stopped(processor):
-        node = processor.node
-        with node.account.monitor_log(processor.STDOUT_FILE) as monitor:
-            processor.stop()
-            monitor.wait_until('OPTIMIZE_TEST Streams Stopped',
-                               timeout_sec=60,
-                               err_msg="'OPTIMIZE_TEST Streams Stopped' message" + str(processor.node.account))
-
     def verify_processing(self, processors, verify_individual_operations):
         for processor in processors:
             if not self.all_source_subtopology_tasks(processor):
@@ -139,10 +130,6 @@ def all_source_subtopology_tasks(self, processor):
 
         return False
 
-    def stop_processors(self, processors):
-        for processor in processors:
-            self.verify_stopped(processor)
-
     def set_topics(self, processor):
         processor.INPUT_TOPIC = self.input_topic
         processor.AGGREGATION_TOPIC = self.aggregation_topic
diff --git a/tests/kafkatest/tests/streams/streams_static_membership_test.py b/tests/kafkatest/tests/streams/streams_static_membership_test.py
new file mode 100644
index 0000000000000..a466ea86648a6
--- /dev/null
+++ b/tests/kafkatest/tests/streams/streams_static_membership_test.py
@@ -0,0 +1,108 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ducktape.tests.test import Test
+from kafkatest.services.kafka import KafkaService
+from kafkatest.services.streams import StaticMemberTestService
+from kafkatest.services.verifiable_producer import VerifiableProducer
+from kafkatest.services.zookeeper import ZookeeperService
+from kafkatest.tests.streams.utils import verify_stopped, stop_processors, verify_running, extract_generation_from_logs
+
+class StreamsStaticMembershipTest(Test):
+    """
+    Tests using static membership when broker points to minimum supported
+    version (2.3) or higher.
+    """
+
+    input_topic = 'inputTopic'
+    pattern = 'PROCESSED'
+    running_message = 'REBALANCING -> RUNNING'
+    stopped_message = 'Static membership test closed'
+
+    def __init__(self, test_context):
+        super(StreamsStaticMembershipTest, self).__init__(test_context)
+        self.topics = {
+            self.input_topic: {'partitions': 18},
+        }
+
+        self.zookeeper = ZookeeperService(self.test_context, num_nodes=1)
+        self.kafka = KafkaService(self.test_context, num_nodes=3,
+                                  zk=self.zookeeper, topics=self.topics)
+
+        self.producer = VerifiableProducer(self.test_context,
+                                           1,
+                                           self.kafka,
+                                           self.input_topic,
+                                           throughput=1000,
+                                           acks=1)
+
+    def test_rolling_bounces_will_not_trigger_rebalance_under_static_membership(self):
+        self.zookeeper.start()
+        self.kafka.start()
+
+        numThreads = 3
+        processor1 = StaticMemberTestService(self.test_context, self.kafka, "consumer-A", numThreads)
+        processor2 = StaticMemberTestService(self.test_context, self.kafka, "consumer-B", numThreads)
+        processor3 = StaticMemberTestService(self.test_context, self.kafka, "consumer-C", numThreads)
+
+        processors = [processor1, processor2, processor3]
+
+        self.producer.start()
+
+        for processor in processors:
+            processor.CLEAN_NODE_ENABLED = False
+            self.set_topics(processor)
+            verify_running(processor, self.running_message)
+
+        self.verify_processing(processors)
+
+        # do several rolling bounces
+        num_bounces = 3
+        for i in range(0, num_bounces):
+            for processor in processors:
+                verify_stopped(processor, self.stopped_message)
+                verify_running(processor, self.running_message)
+
+        stable_generation = -1
+        for processor in processors:
+            generations = extract_generation_from_logs(processor)
+            num_bounce_generations = num_bounces * numThreads
+            assert num_bounce_generations <= len(generations), \
+                "Smaller than minimum expected %d generation messages, actual %d" % (num_bounce_generations, len(generations))
+
+            for generation in generations[-num_bounce_generations:]:
+                generation = int(generation)
+                if stable_generation == -1:
+                    stable_generation = generation
+                assert stable_generation == generation, \
+                    "Stream rolling bounce have caused unexpected generation bump %d" % generation
+
+        self.verify_processing(processors)
+
+        stop_processors(processors, self.stopped_message)
+
+        self.producer.stop()
+        self.kafka.stop()
+        self.zookeeper.stop()
+
+    def verify_processing(self, processors):
+        for processor in processors:
+            with processor.node.account.monitor_log(processor.STDOUT_FILE) as monitor:
+                monitor.wait_until(self.pattern,
+                                   timeout_sec=60,
+                                   err_msg="Never saw processing of %s " % self.pattern + str(processor.node.account))
+
+    def set_topics(self, processor):
+        processor.INPUT_TOPIC = self.input_topic
diff --git a/tests/kafkatest/tests/streams/streams_upgrade_test.py b/tests/kafkatest/tests/streams/streams_upgrade_test.py
index 37ab770befed1..6a6a8bc4f2b03 100644
--- a/tests/kafkatest/tests/streams/streams_upgrade_test.py
+++ b/tests/kafkatest/tests/streams/streams_upgrade_test.py
@@ -23,6 +23,7 @@
 from kafkatest.services.streams import StreamsSmokeTestDriverService, StreamsSmokeTestJobRunnerService, \
     StreamsUpgradeTestJobRunnerService
 from kafkatest.services.zookeeper import ZookeeperService
+from kafkatest.tests.streams.utils import extract_generation_from_logs
 from kafkatest.version import LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, \
     LATEST_2_0, LATEST_2_1, LATEST_2_2, DEV_BRANCH, DEV_VERSION, KafkaVersion
 
@@ -45,7 +46,7 @@
 curl https://s3-us-west-2.amazonaws.com/kafka-packages/kafka_$scala_version-$version.tgz to download the jar
 and if it is not uploaded yet, ping the dev@kafka mailing list to request it being uploaded.
 
-This test needs to get updated, but this requires several steps
+This test needs to get updated, but this requires several steps,
 which are outlined here:
 
 1. Update all relevant versions in tests/kafkatest/version.py this will include adding a new version for the new
@@ -57,17 +58,17 @@
    during the system test run.
    
 3. Update the vagrant/bash.sh file to include all new versions, including the newly released version
-   and all point releases for existing releases.  You only need to list the latest version in 
+   and all point releases for existing releases. You only need to list the latest version in 
    this file.
    
 4. Then update all relevant versions in the tests/docker/Dockerfile
 
-5. Add a new "upgrade-system-tests-XXXX module under streams.  You can probably just copy the 
-   latest system test module from the last release.  Just make sure to update the systout print
-   statement in StreamsUpgradeTest to the version for the release.  After you add the new module
+5. Add a new upgrade-system-tests-XXXX module under streams. You can probably just copy the 
+   latest system test module from the last release. Just make sure to update the systout print
+   statement in StreamsUpgradeTest to the version for the release. After you add the new module
    you'll need to update settings.gradle file to include the name of the module you just created
-   for gradle to recognize the newly added module
-   
+   for gradle to recognize the newly added module.
+
 6. Then you'll need to update any version changes in gradle/dependencies.gradle
 
 """
@@ -598,9 +599,9 @@ def do_rolling_bounce(self, processor, counter, current_generation):
                     retries = 0
 
                     while retries < 10:
-                        processor_found = self.extract_generation_from_logs(processor)
-                        first_other_processor_found = self.extract_generation_from_logs(first_other_processor)
-                        second_other_processor_found = self.extract_generation_from_logs(second_other_processor)
+                        processor_found = extract_generation_from_logs(processor)
+                        first_other_processor_found = extract_generation_from_logs(first_other_processor)
+                        second_other_processor_found = extract_generation_from_logs(second_other_processor)
 
                         if len(processor_found) > 0 and len(first_other_processor_found) > 0 and len(second_other_processor_found) > 0:
                             self.logger.info("processor: " + str(processor_found))
@@ -632,9 +633,6 @@ def do_rolling_bounce(self, processor, counter, current_generation):
 
         return current_generation
 
-    def extract_generation_from_logs(self, processor):
-        return list(processor.node.account.ssh_capture("grep \"Successfully joined group with generation\" %s| awk \'{for(i=1;i<=NF;i++) {if ($i == \"generation\") beginning=i+1; if($i== \"(org.apache.kafka.clients.consumer.internals.AbstractCoordinator)\") ending=i }; for (j=beginning;j
Date: Fri, 7 Jun 2019 14:23:49 -0700
Subject: [PATCH 0346/1071] KAFKA-8499: ensure java is in PATH for ducker
 system tests (#6898)

Reviewers: Colin P. McCabe 
---
 tests/docker/Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile
index 82188231e3e84..2aa259a324794 100644
--- a/tests/docker/Dockerfile
+++ b/tests/docker/Dockerfile
@@ -40,6 +40,7 @@ RUN pip install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddres
 COPY ./ssh-config /root/.ssh/config
 # NOTE: The paramiko library supports the PEM-format private key, but does not support the RFC4716 format.
 RUN ssh-keygen -m PEM -q -t rsa -N '' -f /root/.ssh/id_rsa && cp -f /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
+RUN echo 'PermitUserEnvironment yes' >> /etc/ssh/sshd_config
 
 # Install binary test dependencies.
 # we use the same versions as in vagrant/base.sh
@@ -76,7 +77,7 @@ RUN apt-get install fuse
 RUN cd /opt && git clone -q  https://github.com/confluentinc/kibosh.git && cd "/opt/kibosh" && git reset --hard $KIBOSH_VERSION && mkdir "/opt/kibosh/build" && cd "/opt/kibosh/build" && ../configure && make -j 2
 
 # Set up the ducker user.
-RUN useradd -ms /bin/bash ducker && mkdir -p /home/ducker/ && rsync -aiq /root/.ssh/ /home/ducker/.ssh && chown -R ducker /home/ducker/ /mnt/ /var/log/ && echo 'ducker ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
+RUN useradd -ms /bin/bash ducker && mkdir -p /home/ducker/ && rsync -aiq /root/.ssh/ /home/ducker/.ssh && chown -R ducker /home/ducker/ /mnt/ /var/log/ && echo "PATH=$(runuser -l ducker -c 'echo $PATH'):$JAVA_HOME/bin" >> /home/ducker/.ssh/environment && echo 'PATH=$PATH:'"$JAVA_HOME/bin" >> /home/ducker/.profile && echo 'ducker ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
 USER ducker
 
 CMD sudo service ssh start && tail -f /dev/null

From c7c310beff23423c484b3a86aa284141a190d581 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Fri, 7 Jun 2019 16:53:50 -0700
Subject: [PATCH 0347/1071] MINOR: Lower producer throughput in flaky upgrade
 system test

We see the upgrade test failing from time to time. I looked into it and found that the root cause is basically that the test throughput can be too high for the 0.9 producer to make progress. Eventually it reaches a point where it has a huge backlog of timed out requests in the accumulator which all have to be expired. We see a long run of messages like this in the output:

```
{"exception":"class org.apache.kafka.common.errors.TimeoutException","time_ms":1559907386132,"name":"producer_send_error","topic":"test_topic","message":"Batch Expired","class":"class org.apache.kafka.tools.VerifiableProducer","value":"335160","key":null}
{"exception":"class org.apache.kafka.common.errors.TimeoutException","time_ms":1559907386132,"name":"producer_send_error","topic":"test_topic","message":"Batch Expired","class":"class org.apache.kafka.tools.VerifiableProducer","value":"335163","key":null}
{"exception":"class org.apache.kafka.common.errors.TimeoutException","time_ms":1559907386133,"name":"producer_send_error","topic":"test_topic","message":"Batch Expired","class":"class org.apache.kafka.tools.VerifiableProducer","value":"335166","key":null}
{"exception":"class org.apache.kafka.common.errors.TimeoutException","time_ms":1559907386133,"name":"producer_send_error","topic":"test_topic","message":"Batch Expired","class":"class org.apache.kafka.tools.VerifiableProducer","value":"335169","key":null}
```
This can continue for a long time (I have observed up to 1 min) and prevents the producer from successfully writing any new data. While it is busy expiring the batches, no data is getting delivered to the consumer, which causes it to eventually raise a timeout.
```
kafka.consumer.ConsumerTimeoutException
at kafka.consumer.NewShinyConsumer.receive(BaseConsumer.scala:50)
at kafka.tools.ConsoleConsumer$.process(ConsoleConsumer.scala:109)
at kafka.tools.ConsoleConsumer$.run(ConsoleConsumer.scala:69)
at kafka.tools.ConsoleConsumer$.main(ConsoleConsumer.scala:47)
at kafka.tools.ConsoleConsumer.main(ConsoleConsumer.scala)
```
The fix here is to reduce the throughput, which seems reasonable since the purpose of the test is to verify the upgrade, which does not demand heavy load. Note that I investigated several failing instances of this test going back to 1.0 and saw a similar pattern, so there does not appear to be a regression.

Author: Jason Gustafson 

Reviewers: Gwen Shapira

Closes #6907 from hachikuji/lower-throughput-for-upgrade-test
---
 tests/kafkatest/tests/core/upgrade_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/kafkatest/tests/core/upgrade_test.py b/tests/kafkatest/tests/core/upgrade_test.py
index 80f18fff2c7b0..1cda4e772b101 100644
--- a/tests/kafkatest/tests/core/upgrade_test.py
+++ b/tests/kafkatest/tests/core/upgrade_test.py
@@ -36,7 +36,7 @@ def setUp(self):
         self.zk.start()
 
         # Producer and consumer
-        self.producer_throughput = 10000
+        self.producer_throughput = 1000
         self.num_producers = 1
         self.num_consumers = 1
 

From 2feb44ebc89d36b682396ca00134440818fa8d8c Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Fri, 7 Jun 2019 16:56:21 -0700
Subject: [PATCH 0348/1071] MINOR: Fix race condition on shutdown of verifiable
 producer

We've seen `ReplicaVerificationToolTest.test_replica_lags` fail occasionally due to errors such as the following:
```
RemoteCommandError: ubuntuworker7: Command 'kill -15 2896' returned non-zero exit status 1. Remote error message: bash: line 0: kill: (2896) - No such process
```
The problem seems to be a shutdown race condition when using `max_messages` with the producer. The process may already be gone which will cause the signal to fail.

Author: Jason Gustafson 

Reviewers: Gwen Shapira

Closes #6906 from hachikuji/fix-failing-replicat-verification-test
---
 tests/kafkatest/services/verifiable_producer.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/kafkatest/services/verifiable_producer.py b/tests/kafkatest/services/verifiable_producer.py
index 3322d1698e46c..893baa46c807f 100644
--- a/tests/kafkatest/services/verifiable_producer.py
+++ b/tests/kafkatest/services/verifiable_producer.py
@@ -278,7 +278,11 @@ def each_produced_at_least(self, count):
             return True
 
     def stop_node(self, node):
-        self.kill_node(node, clean_shutdown=True, allow_fail=False)
+        # There is a race condition on shutdown if using `max_messages` since the
+        # VerifiableProducer will shutdown automatically when all messages have been
+        # written. In this case, the process will be gone and the signal will fail.
+        allow_fail = self.max_messages > 0
+        self.kill_node(node, clean_shutdown=True, allow_fail=allow_fail)
 
         stopped = self.wait_node(node, timeout_sec=self.stop_timeout_sec)
         assert stopped, "Node %s: did not stop within the specified timeout of %s seconds" % \

From b9c55351fd2e7c47dd1abd6c7a47f7fff8bb0157 Mon Sep 17 00:00:00 2001
From: Victoria Bialas 
Date: Mon, 10 Jun 2019 10:09:07 -0700
Subject: [PATCH 0349/1071] KAFKA-7315 DOCS update TOC internal links serdes
 all versions (#6875)

Reviewers: Joel Hamill , Jim Galasyn , Matthias J. Sax 
---
 docs/streams/developer-guide/datatypes.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/streams/developer-guide/datatypes.html b/docs/streams/developer-guide/datatypes.html
index 83159e8a86a3b..ecdb48c732439 100644
--- a/docs/streams/developer-guide/datatypes.html
+++ b/docs/streams/developer-guide/datatypes.html
@@ -48,7 +48,7 @@
           
           
  • Kafka Streams DSL for Scala Implicit SerDes
    • 
       

From 1e5227c230c20af4a8182c6324ffbb481712b718 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Mon, 10 Jun 2019 12:20:51 -0700
Subject: [PATCH 0350/1071] MINOR: Fix transient failure in
 PreferredReplicaLeaderElectionCommandTest (#6908)

We have seen this failing recently due to the follower error:
```
java.util.NoSuchElementException: None.get
	at scala.None$.get(Option.scala:366)
	at scala.None$.get(Option.scala:364)
	at kafka.admin.PreferredReplicaLeaderElectionCommandTest.getLeader(PreferredReplicaLeaderElectionCommandTest.scala:101)
	at kafka.admin.PreferredReplicaLeaderElectionCommandTest.testNoopElection(PreferredReplicaLeaderElectionCommandTest.scala:240)
```
We need to wait for the leader to be available.

Reviewers: David Arthur 
---
 ...rredReplicaLeaderElectionCommandTest.scala | 54 ++++++++++---------
 .../scala/unit/kafka/utils/TestUtils.scala    | 49 +++++++++++------
 2 files changed, 62 insertions(+), 41 deletions(-)

diff --git a/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala b/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala
index 03ffd2f4f8dd3..38f2430d783ae 100644
--- a/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala
@@ -33,6 +33,7 @@ import org.apache.kafka.common.errors.PreferredLeaderNotAvailableException
 import org.apache.kafka.common.errors.TimeoutException
 import org.apache.kafka.common.errors.UnknownTopicOrPartitionException
 import org.apache.kafka.common.network.ListenerName
+import org.apache.kafka.test
 import org.junit.Assert._
 import org.junit.{After, Test}
 
@@ -63,9 +64,9 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
     // create brokers
     servers = brokerConfigs.map(b => TestUtils.createServer(KafkaConfig.fromProps(b)))
     // create the topic
-    partitionsAndAssignments.foreach { case (tp, assigment) =>
-      zkClient.createTopicAssignment(tp.topic(),
-      Map(tp -> assigment))
+    partitionsAndAssignments.foreach { case (tp, assignment) =>
+      zkClient.createTopicAssignment(tp.topic,
+      Map(tp -> assignment))
     }
     // wait until replica log is created on every broker
     TestUtils.waitUntilTrue(
@@ -97,8 +98,11 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
     servers.find(p => p.kafkaController.isActive)
   }
 
-  private def getLeader(topicPartition: TopicPartition) = {
-    servers(0).metadataCache.getPartitionInfo(topicPartition.topic(), topicPartition.partition()).get.basePartitionState.leader
+  private def awaitLeader(topicPartition: TopicPartition, timeoutMs: Long = test.TestUtils.DEFAULT_MAX_WAIT_MS): Int = {
+    TestUtils.awaitValue(() => {
+      servers.head.metadataCache.getPartitionInfo(topicPartition.topic, topicPartition.partition)
+          .map(_.basePartitionState.leader)
+    }, s"Timed out waiting to find current leader of $topicPartition", timeoutMs)
   }
 
   private def bootstrapServer(broker: Int = 0): String = {
@@ -118,11 +122,11 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
     createTestTopicAndCluster(testPartitionAndAssignment)
     bounceServer(testPartitionPreferredLeader, testPartition)
     // Check the leader for the partition is not the preferred one
-    assertNotEquals(testPartitionPreferredLeader, getLeader(testPartition))
+    assertNotEquals(testPartitionPreferredLeader, awaitLeader(testPartition))
     PreferredReplicaLeaderElectionCommand.run(Array(
       "--bootstrap-server", s"${bootstrapServer(1)},${bootstrapServer(0)}"))
     // Check the leader for the partition IS the preferred one
-    assertEquals(testPartitionPreferredLeader, getLeader(testPartition))
+    assertEquals(testPartitionPreferredLeader, awaitLeader(testPartition))
   }
 
   /** Test the case when an invalid broker is given for --bootstrap-broker */
@@ -145,11 +149,11 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
     createTestTopicAndCluster(testPartitionAndAssignment)
     bounceServer(testPartitionPreferredLeader, testPartition)
     // Check the leader for the partition is not the preferred one
-    assertNotEquals(testPartitionPreferredLeader, getLeader(testPartition))
+    assertNotEquals(testPartitionPreferredLeader, awaitLeader(testPartition))
     PreferredReplicaLeaderElectionCommand.run(Array(
       "--bootstrap-server", bootstrapServer()))
     // Check the leader for the partition IS the preferred one
-    assertEquals(testPartitionPreferredLeader, getLeader(testPartition))
+    assertEquals(testPartitionPreferredLeader, awaitLeader(testPartition))
   }
 
   private def toJsonFile(partitions: Set[TopicPartition]): File = {
@@ -167,7 +171,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
     createTestTopicAndCluster(testPartitionAndAssignment)
     bounceServer(testPartitionPreferredLeader, testPartition)
     // Check the leader for the partition is not the preferred one
-    assertNotEquals(testPartitionPreferredLeader, getLeader(testPartition))
+    assertNotEquals(testPartitionPreferredLeader, awaitLeader(testPartition))
     val jsonFile = toJsonFile(testPartitionAndAssignment.keySet)
     try {
       PreferredReplicaLeaderElectionCommand.run(Array(
@@ -177,7 +181,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
       jsonFile.delete()
     }
     // Check the leader for the partition IS the preferred one
-    assertEquals(testPartitionPreferredLeader, getLeader(testPartition))
+    assertEquals(testPartitionPreferredLeader, awaitLeader(testPartition))
   }
 
   /** Test the case where a topic does not exist */
@@ -217,8 +221,8 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
     createTestTopicAndCluster(testPartitionAndAssignment)
     bounceServer(testPartitionPreferredLeader, testPartitionA)
     // Check the leader for the partition is not the preferred one
-    assertNotEquals(testPartitionPreferredLeader, getLeader(testPartitionA))
-    assertNotEquals(testPartitionPreferredLeader, getLeader(testPartitionB))
+    assertNotEquals(testPartitionPreferredLeader, awaitLeader(testPartitionA))
+    assertNotEquals(testPartitionPreferredLeader, awaitLeader(testPartitionB))
     val jsonFile = toJsonFile(testPartitionAndAssignment.keySet)
     try {
       PreferredReplicaLeaderElectionCommand.run(Array(
@@ -228,16 +232,16 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
       jsonFile.delete()
     }
     // Check the leader for the partition IS the preferred one
-    assertEquals(testPartitionPreferredLeader, getLeader(testPartitionA))
-    assertEquals(testPartitionPreferredLeader, getLeader(testPartitionB))
+    assertEquals(testPartitionPreferredLeader, awaitLeader(testPartitionA))
+    assertEquals(testPartitionPreferredLeader, awaitLeader(testPartitionB))
   }
 
   /** What happens when the preferred replica is already the leader? */
   @Test
   def testNoopElection() {
     createTestTopicAndCluster(testPartitionAndAssignment)
-    // Don't bounce the server. Doublec heck the leader for the partition is the preferred one
-    assertEquals(testPartitionPreferredLeader, getLeader(testPartition))
+    // Don't bounce the server. Doublecheck the leader for the partition is the preferred one
+    assertEquals(testPartitionPreferredLeader, awaitLeader(testPartition))
     val jsonFile = toJsonFile(testPartitionAndAssignment.keySet)
     try {
       // Now do the election, even though the preferred replica is *already* the leader
@@ -245,7 +249,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
         "--bootstrap-server", bootstrapServer(),
         "--path-to-json-file", jsonFile.getAbsolutePath))
       // Check the leader for the partition still is the preferred one
-      assertEquals(testPartitionPreferredLeader, getLeader(testPartition))
+      assertEquals(testPartitionPreferredLeader, awaitLeader(testPartition))
     } finally {
       jsonFile.delete()
     }
@@ -257,7 +261,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
     createTestTopicAndCluster(testPartitionAndAssignment)
     bounceServer(testPartitionPreferredLeader, testPartition)
     // Check the leader for the partition is not the preferred one
-    val leader = getLeader(testPartition)
+    val leader = awaitLeader(testPartition)
     assertNotEquals(testPartitionPreferredLeader, leader)
     // Now kill the preferred one
     servers(testPartitionPreferredLeader).shutdown()
@@ -275,7 +279,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
         assertTrue(suppressed.isInstanceOf[PreferredLeaderNotAvailableException])
         assertTrue(suppressed.getMessage, suppressed.getMessage.contains("Failed to elect leader for partition test-0 under strategy PreferredReplicaPartitionLeaderElectionStrategy"))
         // Check we still have the same leader
-        assertEquals(leader, getLeader(testPartition))
+        assertEquals(leader, awaitLeader(testPartition))
     } finally {
       jsonFile.delete()
     }
@@ -287,7 +291,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
     createTestTopicAndCluster(testPartitionAndAssignment)
     bounceServer(testPartitionPreferredLeader, testPartition)
     // Check the leader for the partition is not the preferred one
-    val leader = getLeader(testPartition)
+    val leader = awaitLeader(testPartition)
     assertNotEquals(testPartitionPreferredLeader, leader)
     // Now kill the controller just before we trigger the election
     val controller = getController().get.config.brokerId
@@ -303,7 +307,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
       case e: AdminCommandFailedException =>
         assertEquals("Timeout waiting for election results", e.getMessage)
         // Check we still have the same leader
-        assertEquals(leader, getLeader(testPartition))
+        assertEquals(leader, awaitLeader(testPartition))
     } finally {
       jsonFile.delete()
     }
@@ -315,10 +319,10 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
     createTestTopicAndCluster(testPartitionAndAssignment, Some(classOf[PreferredReplicaLeaderElectionCommandTestAuthorizer].getName))
     bounceServer(testPartitionPreferredLeader, testPartition)
     // Check the leader for the partition is not the preferred one
-    val leader = getLeader(testPartition)
+    val leader = awaitLeader(testPartition)
     assertNotEquals(testPartitionPreferredLeader, leader)
     // Check the leader for the partition is not the preferred one
-    assertNotEquals(testPartitionPreferredLeader, getLeader(testPartition))
+    assertNotEquals(testPartitionPreferredLeader, awaitLeader(testPartition))
     val jsonFile = toJsonFile(testPartitionAndAssignment.keySet)
     try {
       PreferredReplicaLeaderElectionCommand.run(Array(
@@ -330,7 +334,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
         assertEquals("Not authorized to perform leader election", e.getMessage)
         assertTrue(e.getCause().isInstanceOf[ClusterAuthorizationException])
         // Check we still have the same leader
-        assertEquals(leader, getLeader(testPartition))
+        assertEquals(leader, awaitLeader(testPartition))
     } finally {
       jsonFile.delete()
     }
diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index ca62a7953c5e9..5c9284ff6b8b7 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -313,7 +313,7 @@ object TestUtils extends Logging {
                   topicConfig: Properties = new Properties): scala.collection.immutable.Map[Int, Int] = {
     val adminZkClient = new AdminZkClient(zkClient)
     // create topic
-    TestUtils.waitUntilTrue( () => {
+    waitUntilTrue( () => {
       var hasSessionExpirationException = false
       try {
         adminZkClient.createTopic(topic, numPartitions, replicationFactor, topicConfig)
@@ -355,7 +355,7 @@ object TestUtils extends Logging {
                   topicConfig: Properties): scala.collection.immutable.Map[Int, Int] = {
     val adminZkClient = new AdminZkClient(zkClient)
     // create topic
-    TestUtils.waitUntilTrue( () => {
+    waitUntilTrue( () => {
       var hasSessionExpirationException = false
       try {
         adminZkClient.createTopicWithAssignment(topic, topicConfig, partitionReplicaAssignment)
@@ -776,6 +776,23 @@ object TestUtils extends Logging {
     }, msg = msg, pause = 0L, waitTimeMs = waitTimeMs)
   }
 
+  /**
+   * Wait for the presence of an optional value.
+   *
+   * @param func The function defining the optional value
+   * @param msg Error message in the case that the value never appears
+   * @param waitTimeMs Maximum time to wait
+   * @return The unwrapped value returned by the function
+   */
+  def awaitValue[T](func: () => Option[T], msg: => String, waitTimeMs: Long = JTestUtils.DEFAULT_MAX_WAIT_MS): T = {
+    var value: Option[T] = None
+    waitUntilTrue(() => {
+      value = func()
+      value.isDefined
+    }, msg, waitTimeMs)
+    value.get
+  }
+
   /**
     *  Wait until the given condition is true or throw an exception if the given wait time elapses.
     *
@@ -865,7 +882,7 @@ object TestUtils extends Logging {
   def waitUntilBrokerMetadataIsPropagated(servers: Seq[KafkaServer],
                                           timeout: Long = JTestUtils.DEFAULT_MAX_WAIT_MS): Unit = {
     val expectedBrokerIds = servers.map(_.config.brokerId).toSet
-    TestUtils.waitUntilTrue(() => servers.forall(server =>
+    waitUntilTrue(() => servers.forall(server =>
       expectedBrokerIds == server.dataPlaneRequestProcessor.metadataCache.getAliveBrokers.map(_.id).toSet
     ), "Timed out waiting for broker metadata to propagate to all servers", timeout)
   }
@@ -883,7 +900,7 @@ object TestUtils extends Logging {
   def waitUntilMetadataIsPropagated(servers: Seq[KafkaServer], topic: String, partition: Int,
                                     timeout: Long = JTestUtils.DEFAULT_MAX_WAIT_MS): Int = {
     var leader: Int = -1
-    TestUtils.waitUntilTrue(() =>
+    waitUntilTrue(() =>
       servers.foldLeft(true) {
         (result, server) =>
           val partitionStateOpt = server.dataPlaneRequestProcessor.metadataCache.getPartitionInfo(topic, partition)
@@ -916,7 +933,7 @@ object TestUtils extends Logging {
       }.map(_.config.brokerId)
     }
 
-    TestUtils.waitUntilTrue(() => newLeaderExists.isDefined,
+    waitUntilTrue(() => newLeaderExists.isDefined,
       s"Did not observe leader change for partition $tp after $timeout ms", waitTimeMs = timeout)
 
     newLeaderExists.get
@@ -931,7 +948,7 @@ object TestUtils extends Logging {
       }.map(_.config.brokerId)
     }
 
-    TestUtils.waitUntilTrue(() => leaderIfExists.isDefined,
+    waitUntilTrue(() => leaderIfExists.isDefined,
       s"Partition $tp leaders not made yet after $timeout ms", waitTimeMs = timeout)
 
     leaderIfExists.get
@@ -964,18 +981,18 @@ object TestUtils extends Logging {
   def ensureNoUnderReplicatedPartitions(zkClient: KafkaZkClient, topic: String, partitionToBeReassigned: Int, assignedReplicas: Seq[Int],
                                                 servers: Seq[KafkaServer]) {
     val topicPartition = new TopicPartition(topic, partitionToBeReassigned)
-    TestUtils.waitUntilTrue(() => {
+    waitUntilTrue(() => {
         val inSyncReplicas = zkClient.getInSyncReplicasForPartition(topicPartition)
         inSyncReplicas.get.size == assignedReplicas.size
       },
       "Reassigned partition [%s,%d] is under replicated".format(topic, partitionToBeReassigned))
     var leader: Option[Int] = None
-    TestUtils.waitUntilTrue(() => {
+    waitUntilTrue(() => {
         leader = zkClient.getLeaderForPartition(topicPartition)
         leader.isDefined
       },
       "Reassigned partition [%s,%d] is unavailable".format(topic, partitionToBeReassigned))
-    TestUtils.waitUntilTrue(() => {
+    waitUntilTrue(() => {
         val leaderBroker = servers.filter(s => s.config.brokerId == leader.get).head
         leaderBroker.replicaManager.underReplicatedPartitionCount == 0
       },
@@ -1059,33 +1076,33 @@ object TestUtils extends Logging {
   def verifyTopicDeletion(zkClient: KafkaZkClient, topic: String, numPartitions: Int, servers: Seq[KafkaServer]) {
     val topicPartitions = (0 until numPartitions).map(new TopicPartition(topic, _))
     // wait until admin path for delete topic is deleted, signaling completion of topic deletion
-    TestUtils.waitUntilTrue(() => !zkClient.isTopicMarkedForDeletion(topic),
+    waitUntilTrue(() => !zkClient.isTopicMarkedForDeletion(topic),
       "Admin path /admin/delete_topics/%s path not deleted even after a replica is restarted".format(topic))
-    TestUtils.waitUntilTrue(() => !zkClient.topicExists(topic),
+    waitUntilTrue(() => !zkClient.topicExists(topic),
       "Topic path /brokers/topics/%s not deleted after /admin/delete_topics/%s path is deleted".format(topic, topic))
     // ensure that the topic-partition has been deleted from all brokers' replica managers
-    TestUtils.waitUntilTrue(() =>
+    waitUntilTrue(() =>
       servers.forall(server => topicPartitions.forall(tp => server.replicaManager.nonOfflinePartition(tp).isEmpty)),
       "Replica manager's should have deleted all of this topic's partitions")
     // ensure that logs from all replicas are deleted if delete topic is marked successful in ZooKeeper
     assertTrue("Replica logs not deleted after delete topic is complete",
       servers.forall(server => topicPartitions.forall(tp => server.getLogManager.getLog(tp).isEmpty)))
     // ensure that topic is removed from all cleaner offsets
-    TestUtils.waitUntilTrue(() => servers.forall(server => topicPartitions.forall { tp =>
+    waitUntilTrue(() => servers.forall(server => topicPartitions.forall { tp =>
       val checkpoints = server.getLogManager.liveLogDirs.map { logDir =>
         new OffsetCheckpointFile(new File(logDir, "cleaner-offset-checkpoint")).read()
       }
       checkpoints.forall(checkpointsPerLogDir => !checkpointsPerLogDir.contains(tp))
     }), "Cleaner offset for deleted partition should have been removed")
     import scala.collection.JavaConverters._
-    TestUtils.waitUntilTrue(() => servers.forall(server =>
+    waitUntilTrue(() => servers.forall(server =>
       server.config.logDirs.forall { logDir =>
         topicPartitions.forall { tp =>
           !new File(logDir, tp.topic + "-" + tp.partition).exists()
         }
       }
     ), "Failed to soft-delete the data to a delete directory")
-    TestUtils.waitUntilTrue(() => servers.forall(server =>
+    waitUntilTrue(() => servers.forall(server =>
       server.config.logDirs.forall { logDir =>
         topicPartitions.forall { tp =>
           !java.util.Arrays.asList(new File(logDir).list()).asScala.exists { partitionDirectoryName =>
@@ -1145,7 +1162,7 @@ object TestUtils extends Logging {
   def waitAndVerifyAcls(expected: Set[Acl], authorizer: Authorizer, resource: Resource) = {
     val newLine = scala.util.Properties.lineSeparator
 
-    TestUtils.waitUntilTrue(() => authorizer.getAcls(resource) == expected,
+    waitUntilTrue(() => authorizer.getAcls(resource) == expected,
       s"expected acls:${expected.mkString(newLine + "\t", newLine + "\t", newLine)}" +
         s"but got:${authorizer.getAcls(resource).mkString(newLine + "\t", newLine + "\t", newLine)}", waitTimeMs = JTestUtils.DEFAULT_MAX_WAIT_MS)
   }

From 0864779ce2a0454b2924567189e2f690d92498a6 Mon Sep 17 00:00:00 2001
From: Bill Bejeck 
Date: Mon, 10 Jun 2019 18:13:30 -0400
Subject: [PATCH 0351/1071] MINOR: Increase timeouts to 30 seconds (#6852)

The ResetIntegrationTest has experienced several failures and it seems the current timeout of 10 seconds may not be enough time

Reviewers: Matthias J. Sax , Boyang Chen 
---
 .../kafka/streams/integration/AbstractResetIntegrationTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/AbstractResetIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/AbstractResetIntegrationTest.java
index 000f2993c739a..d8c0570465309 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/AbstractResetIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/AbstractResetIntegrationTest.java
@@ -168,7 +168,7 @@ private void prepareConfigs() {
 
     private static final long STREAMS_CONSUMER_TIMEOUT = 2000L;
     private static final long CLEANUP_CONSUMER_TIMEOUT = 2000L;
-    private static final int TIMEOUT_MULTIPLIER = 5;
+    private static final int TIMEOUT_MULTIPLIER = 15;
 
     private class ConsumerGroupInactiveCondition implements TestCondition {
         @Override

From 783ab74793cc7e541e6b8ed4e1c545bf5dcff959 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Mon, 10 Jun 2019 18:59:59 -0700
Subject: [PATCH 0352/1071] KAFKA-8333; Load high watermark checkpoint lazily
 when initializing replicas (#6800)

Currently we load the high watermark checkpoint separately for every replica that we load. This patch makes this loading logic lazy and caches the loaded map while a LeaderAndIsr request is being handled.

Reviewers: Jun Rao 
---
 .../scala/kafka/server/ReplicaManager.scala   |  6 ++--
 .../checkpoints/OffsetCheckpointFile.scala    | 23 +++++++++---
 .../kafka/server/ReplicaManagerTest.scala     | 20 +++++------
 .../OffsetCheckpointFileTest.scala            | 36 +++++++++++++++++++
 4 files changed, 67 insertions(+), 18 deletions(-)

diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index 8b383c2b342bf..64b6beed4edea 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -29,7 +29,7 @@ import kafka.controller.{KafkaController, StateChangeLogger}
 import kafka.log._
 import kafka.metrics.KafkaMetricsGroup
 import kafka.server.QuotaFactory.{QuotaManagers, UnboundedQuota}
-import kafka.server.checkpoints.{OffsetCheckpointFile, OffsetCheckpoints, SimpleOffsetCheckpoints}
+import kafka.server.checkpoints.{LazyOffsetCheckpoints, OffsetCheckpointFile, OffsetCheckpoints}
 import kafka.utils._
 import kafka.zk.KafkaZkClient
 import org.apache.kafka.common.ElectionType
@@ -604,7 +604,7 @@ class ReplicaManager(val config: KafkaConfig,
           //   start ReplicaAlterDirThread to move data of this partition from the current log to the future log
           // - Otherwise, return KafkaStorageException. We do not create the future log while there is offline log directory
           //   so that we can avoid creating future log for the same partition in multiple log directories.
-          val highWatermarkCheckpoints = new SimpleOffsetCheckpoints(this.highWatermarkCheckpoints)
+          val highWatermarkCheckpoints = new LazyOffsetCheckpoints(this.highWatermarkCheckpoints)
           if (partition.maybeCreateFutureReplica(destinationDir, highWatermarkCheckpoints)) {
             val futureReplica = futureLocalReplicaOrException(topicPartition)
             logManager.abortAndPauseCleaning(topicPartition)
@@ -1109,7 +1109,7 @@ class ReplicaManager(val config: KafkaConfig,
         }
         val partitionsToBeFollower = partitionState -- partitionsTobeLeader.keys
 
-        val highWatermarkCheckpoints = new SimpleOffsetCheckpoints(this.highWatermarkCheckpoints)
+        val highWatermarkCheckpoints = new LazyOffsetCheckpoints(this.highWatermarkCheckpoints)
         val partitionsBecomeLeader = if (partitionsTobeLeader.nonEmpty)
           makeLeaders(controllerId, controllerEpoch, partitionsTobeLeader, correlationId, responseMap,
             highWatermarkCheckpoints)
diff --git a/core/src/main/scala/kafka/server/checkpoints/OffsetCheckpointFile.scala b/core/src/main/scala/kafka/server/checkpoints/OffsetCheckpointFile.scala
index 715f42fa30f41..69e62d277a2be 100644
--- a/core/src/main/scala/kafka/server/checkpoints/OffsetCheckpointFile.scala
+++ b/core/src/main/scala/kafka/server/checkpoints/OffsetCheckpointFile.scala
@@ -66,13 +66,26 @@ trait OffsetCheckpoints {
   def fetch(logDir: String, topicPartition: TopicPartition): Option[Long]
 }
 
-class SimpleOffsetCheckpoints(checkpointFilesByLogDir: Map[String, OffsetCheckpointFile])
-  extends OffsetCheckpoints {
+/**
+ * Loads checkpoint files on demand and caches the offsets for reuse.
+ */
+class LazyOffsetCheckpoints(checkpointsByLogDir: Map[String, OffsetCheckpointFile]) extends OffsetCheckpoints {
+  private val lazyCheckpointsByLogDir = checkpointsByLogDir.map { case (logDir, checkpointFile) =>
+    logDir -> new LazyOffsetCheckpointMap(checkpointFile)
+  }.toMap
 
   override def fetch(logDir: String, topicPartition: TopicPartition): Option[Long] = {
-    val checkpoint = checkpointFilesByLogDir(logDir)
-    val offsetMap = checkpoint.read()
-    offsetMap.get(topicPartition)
+    val offsetCheckpointFile = lazyCheckpointsByLogDir.getOrElse(logDir,
+      throw new IllegalArgumentException(s"No checkpoint file for log dir $logDir"))
+    offsetCheckpointFile.fetch(topicPartition)
+  }
+}
+
+class LazyOffsetCheckpointMap(checkpoint: OffsetCheckpointFile) {
+  private lazy val offsets: Map[TopicPartition, Long] = checkpoint.read()
+
+  def fetch(topicPartition: TopicPartition): Option[Long] = {
+    offsets.get(topicPartition)
   }
 
 }
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index 59248f0d02af9..64a29ed387708 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -26,7 +26,7 @@ import kafka.log.{Log, LogConfig, LogManager, ProducerStateManager}
 import kafka.utils.{MockScheduler, MockTime, TestUtils}
 import TestUtils.createBroker
 import kafka.cluster.BrokerEndPoint
-import kafka.server.checkpoints.SimpleOffsetCheckpoints
+import kafka.server.checkpoints.LazyOffsetCheckpoints
 import kafka.server.epoch.util.ReplicaFetcherMockBlockingSend
 import kafka.utils.timer.MockTimer
 import kafka.zk.KafkaZkClient
@@ -87,7 +87,7 @@ class ReplicaManagerTest {
       new MetadataCache(config.brokerId), new LogDirFailureChannel(config.logDirs.size))
     try {
       val partition = rm.createPartition(new TopicPartition(topic, 1))
-      partition.getOrCreateReplica(1, isNew = false, new SimpleOffsetCheckpoints(rm.highWatermarkCheckpoints))
+      partition.getOrCreateReplica(1, isNew = false, new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints))
       rm.checkpointHighWatermarks()
     } finally {
       // shutdown the replica manager upon test completion
@@ -106,7 +106,7 @@ class ReplicaManagerTest {
       new MetadataCache(config.brokerId), new LogDirFailureChannel(config.logDirs.size))
     try {
       val partition = rm.createPartition(new TopicPartition(topic, 1))
-      partition.getOrCreateReplica(1, isNew = false, new SimpleOffsetCheckpoints(rm.highWatermarkCheckpoints))
+      partition.getOrCreateReplica(1, isNew = false, new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints))
       rm.checkpointHighWatermarks()
     } finally {
       // shutdown the replica manager upon test completion
@@ -160,7 +160,7 @@ class ReplicaManagerTest {
       val brokerList = Seq[Integer](0, 1).asJava
 
       val partition = rm.createPartition(new TopicPartition(topic, 0))
-      partition.getOrCreateReplica(0, isNew = false, new SimpleOffsetCheckpoints(rm.highWatermarkCheckpoints))
+      partition.getOrCreateReplica(0, isNew = false, new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints))
       // Make this replica the leader.
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
         collection.immutable.Map(new TopicPartition(topic, 0) ->
@@ -204,7 +204,7 @@ class ReplicaManagerTest {
       val brokerList = Seq[Integer](0, 1).asJava
 
       val partition = replicaManager.createPartition(new TopicPartition(topic, 0))
-      partition.getOrCreateReplica(0, isNew = false, new SimpleOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
+      partition.getOrCreateReplica(0, isNew = false, new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
 
       // Make this replica the leader.
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
@@ -255,7 +255,7 @@ class ReplicaManagerTest {
       val brokerList = Seq[Integer](0, 1).asJava
 
       val partition = replicaManager.createPartition(new TopicPartition(topic, 0))
-      partition.getOrCreateReplica(0, isNew = false, new SimpleOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
+      partition.getOrCreateReplica(0, isNew = false, new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
 
       // Make this replica the leader.
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
@@ -351,7 +351,7 @@ class ReplicaManagerTest {
     try {
       val brokerList = Seq[Integer](0, 1).asJava
       val partition = replicaManager.createPartition(new TopicPartition(topic, 0))
-      partition.getOrCreateReplica(0, isNew = false, new SimpleOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
+      partition.getOrCreateReplica(0, isNew = false, new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
 
       // Make this replica the leader.
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
@@ -417,7 +417,7 @@ class ReplicaManagerTest {
       val brokerList = Seq[Integer](0, 1, 2).asJava
 
       val partition = rm.createPartition(new TopicPartition(topic, 0))
-      partition.getOrCreateReplica(0, isNew = false, new SimpleOffsetCheckpoints(rm.highWatermarkCheckpoints))
+      partition.getOrCreateReplica(0, isNew = false, new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints))
 
       // Make this replica the leader.
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
@@ -552,7 +552,7 @@ class ReplicaManagerTest {
       // Create 2 partitions, assign replica 0 as the leader for both a different follower (1 and 2) for each
       val tp0 = new TopicPartition(topic, 0)
       val tp1 = new TopicPartition(topic, 1)
-      val offsetCheckpoints = new SimpleOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
+      val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
       replicaManager.createPartition(tp0).getOrCreateReplica(0, isNew = false, offsetCheckpoints)
       replicaManager.createPartition(tp1).getOrCreateReplica(0, isNew = false, offsetCheckpoints)
       val partition0Replicas = Seq[Integer](0, 1).asJava
@@ -645,7 +645,7 @@ class ReplicaManagerTest {
 
     // Initialize partition state to follower, with leader = 1, leaderEpoch = 1
     val partition = replicaManager.createPartition(new TopicPartition(topic, topicPartition))
-    val offsetCheckpoints = new SimpleOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
+    val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
     partition.getOrCreateReplica(followerBrokerId, isNew = false, offsetCheckpoints)
     partition.makeFollower(controllerId,
       leaderAndIsrPartitionState(leaderEpoch, leaderBrokerId, aliveBrokerIds),
diff --git a/core/src/test/scala/unit/kafka/server/checkpoints/OffsetCheckpointFileTest.scala b/core/src/test/scala/unit/kafka/server/checkpoints/OffsetCheckpointFileTest.scala
index 2d20674026c8e..99a40b607d6f5 100644
--- a/core/src/test/scala/unit/kafka/server/checkpoints/OffsetCheckpointFileTest.scala
+++ b/core/src/test/scala/unit/kafka/server/checkpoints/OffsetCheckpointFileTest.scala
@@ -22,6 +22,8 @@ import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.KafkaStorageException
 import org.junit.Assert._
 import org.junit.Test
+import org.mockito.Mockito
+import org.scalatest.Assertions.assertThrows
 
 import scala.collection.Map
 
@@ -98,4 +100,38 @@ class OffsetCheckpointFileTest extends Logging {
     new OffsetCheckpointFile(checkpointFile.file, logDirFailureChannel).read()
   }
 
+  @Test
+  def testLazyOffsetCheckpoint(): Unit = {
+    val logDir = "/tmp/kafka-logs"
+    val mockCheckpointFile = Mockito.mock(classOf[OffsetCheckpointFile])
+
+    val lazyCheckpoints = new LazyOffsetCheckpoints(Map(logDir -> mockCheckpointFile))
+    Mockito.verify(mockCheckpointFile, Mockito.never()).read()
+
+    val partition0 = new TopicPartition("foo", 0)
+    val partition1 = new TopicPartition("foo", 1)
+    val partition2 = new TopicPartition("foo", 2)
+
+    Mockito.when(mockCheckpointFile.read()).thenReturn(Map(
+      partition0 -> 1000L,
+      partition1 -> 2000L
+    ))
+
+    assertEquals(Some(1000L), lazyCheckpoints.fetch(logDir, partition0))
+    assertEquals(Some(2000L), lazyCheckpoints.fetch(logDir, partition1))
+    assertEquals(None, lazyCheckpoints.fetch(logDir, partition2))
+
+    Mockito.verify(mockCheckpointFile, Mockito.times(1)).read()
+  }
+
+  @Test
+  def testLazyOffsetCheckpointFileInvalidLogDir(): Unit = {
+    val logDir = "/tmp/kafka-logs"
+    val mockCheckpointFile = Mockito.mock(classOf[OffsetCheckpointFile])
+    val lazyCheckpoints = new LazyOffsetCheckpoints(Map(logDir -> mockCheckpointFile))
+    assertThrows[IllegalArgumentException] {
+      lazyCheckpoints.fetch("/invalid/kafka-logs", new TopicPartition("foo", 0))
+    }
+  }
+
 }

From 1669b773ba31f7bb3395ec536be7c17bc98ab9fc Mon Sep 17 00:00:00 2001
From: Carlos Manuel Duclos Vergara 
Date: Tue, 11 Jun 2019 16:04:44 +0200
Subject: [PATCH 0353/1071] KAFKA-8501: Removing key and value from exception
 message (#6904)

Messages containing key and value were moved to the TRACE logging level, however the exception is still adding the key and value.
This commits remove the key and value from StreamsException.

Reviewers: Bill Bejeck 
---
 .../streams/processor/internals/RecordCollectorImpl.java  | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordCollectorImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordCollectorImpl.java
index 2e9ead8bb0293..4edbc3d701f00 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordCollectorImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordCollectorImpl.java
@@ -58,7 +58,7 @@ public class RecordCollectorImpl implements RecordCollector {
     private final static String LOG_MESSAGE = "Error sending record to topic {} due to {}; " +
         "No more records will be sent and no more offsets will be recorded for this task. " +
         "Enable TRACE logging to view failed record key and value.";
-    private final static String EXCEPTION_MESSAGE = "%sAbort sending since %s with a previous record (key %s value %s timestamp %d) to topic %s due to %s";
+    private final static String EXCEPTION_MESSAGE = "%sAbort sending since %s with a previous record (timestamp %d) to topic %s due to %s";
     private final static String PARAMETER_HINT = "\nYou can increase producer parameter `retries` and `retry.backoff.ms` to avoid this error.";
     private volatile KafkaException sendException;
 
@@ -139,8 +139,6 @@ private  void recordSendError(
                 errorMessage,
                 logPrefix,
                 "an error caught",
-                key,
-                value,
                 timestamp,
                 topic,
                 exception.toString()
@@ -187,8 +185,6 @@ public void onCompletion(final RecordMetadata metadata,
                                         EXCEPTION_MESSAGE,
                                         logPrefix,
                                         "producer got fenced",
-                                        key,
-                                        value,
                                         timestamp,
                                         topic,
                                         exception.toString()
@@ -245,8 +241,6 @@ public void onCompletion(final RecordMetadata metadata,
                         EXCEPTION_MESSAGE,
                         logPrefix,
                         "an error caught",
-                        key,
-                        value,
                         timestamp,
                         topic,
                         uncaughtException.toString()

From bebcbe3a049f78c4184404f2dfb8b4150233856e Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Tue, 11 Jun 2019 09:48:43 -0700
Subject: [PATCH 0354/1071] KAFKA-8487: Only request re-join on
 REBALANCE_IN_PROGRESS in CommitOffsetResponse (#6894)

Plus some minor cleanups on AbstractCoordinator.

Reviewers: Boyang Chen , Jason Gustafson 
---
 .../internals/AbstractCoordinator.java        | 10 +--
 .../internals/ConsumerCoordinator.java        | 16 +++-
 .../internals/ConsumerCoordinatorTest.java    | 34 +++++++-
 .../coordinator/group/GroupCoordinator.scala  | 84 ++++++++++++-------
 .../group/GroupCoordinatorTest.scala          | 46 ++++++++--
 5 files changed, 141 insertions(+), 49 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index 73563fd96b86b..30277b3819b5c 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -555,7 +555,7 @@ public void handle(JoinGroupResponse joinResponse, RequestFuture fut
                 // reset the member id and retry immediately
                 resetGeneration();
                 log.debug("Attempt to join group failed due to unknown member id.");
-                future.raise(Errors.UNKNOWN_MEMBER_ID);
+                future.raise(error);
             } else if (error == Errors.COORDINATOR_NOT_AVAILABLE
                     || error == Errors.NOT_COORDINATOR) {
                 // re-discover the coordinator and retry with backoff
@@ -592,7 +592,7 @@ public void handle(JoinGroupResponse joinResponse, RequestFuture fut
                     AbstractCoordinator.this.rejoinNeeded = true;
                     AbstractCoordinator.this.state = MemberState.UNJOINED;
                 }
-                future.raise(Errors.MEMBER_ID_REQUIRED);
+                future.raise(error);
             } else {
                 // unexpected error, throw the exception
                 log.error("Attempt to join group failed due to unexpected error: {}", error.message());
@@ -940,18 +940,18 @@ public void handle(HeartbeatResponse heartbeatResponse, RequestFuture futu
             } else if (error == Errors.REBALANCE_IN_PROGRESS) {
                 log.info("Attempt to heartbeat failed since group is rebalancing");
                 requestRejoin();
-                future.raise(Errors.REBALANCE_IN_PROGRESS);
+                future.raise(error);
             } else if (error == Errors.ILLEGAL_GENERATION) {
                 log.info("Attempt to heartbeat failed since generation {} is not current", generation.generationId);
                 resetGeneration();
-                future.raise(Errors.ILLEGAL_GENERATION);
+                future.raise(error);
             } else if (error == Errors.FENCED_INSTANCE_ID) {
                 log.error("Received fatal exception: group.instance.id gets fenced");
                 future.raise(error);
             } else if (error == Errors.UNKNOWN_MEMBER_ID) {
                 log.info("Attempt to heartbeat failed for since member id {} is not valid.", generation.memberId);
                 resetGeneration();
-                future.raise(Errors.UNKNOWN_MEMBER_ID);
+                future.raise(error);
             } else if (error == Errors.GROUP_AUTHORIZATION_FAILED) {
                 future.raise(new GroupAuthorizationException(groupId));
             } else {
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 64bf17d96f213..bacb96001bcbe 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -881,10 +881,20 @@ public void handle(OffsetCommitResponse commitResponse, RequestFuture futu
                             log.error("Received fatal exception: group.instance.id gets fenced");
                             future.raise(error);
                             return;
+                        } else if (error == Errors.REBALANCE_IN_PROGRESS) {
+                            /* Consumer never tries to commit offset in between join-group and sync-group,
+                             * and hence on broker-side it is not expected to see a commit offset request
+                             * during CompletingRebalance phase; if it ever happens then broker would return
+                             * this error. In this case we should just treat as a fatal CommitFailed exception.
+                             * However, we do not need to reset generations and just request re-join, such that
+                             * if the caller decides to proceed and poll, it would still try to proceed and re-join normally.
+                             */
+                            requestRejoin();
+                            future.raise(new CommitFailedException());
+                            return;
                         } else if (error == Errors.UNKNOWN_MEMBER_ID
-                                || error == Errors.ILLEGAL_GENERATION
-                                || error == Errors.REBALANCE_IN_PROGRESS) {
-                            // need to re-join group
+                                || error == Errors.ILLEGAL_GENERATION) {
+                            // need to reset generation and re-join group
                             resetGeneration();
                             future.raise(new CommitFailedException());
                             return;
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index 6f62bbf89a03d..9595639a48c69 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -1678,15 +1678,43 @@ public void testCommitOffsetUnknownMemberId() {
                 new OffsetAndMetadata(100L, "metadata")), time.timer(Long.MAX_VALUE));
     }
 
-    @Test(expected = CommitFailedException.class)
+    @Test
     public void testCommitOffsetRebalanceInProgress() {
         // we cannot retry if a rebalance occurs before the commit completed
+        final String consumerId = "leader";
+
+        subscriptions.subscribe(singleton(topic1), rebalanceListener);
+
+        // ensure metadata is up-to-date for leader
+        client.updateMetadata(metadataResponse);
+
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
 
+        // normal join group
+        Map> memberSubscriptions = singletonMap(consumerId, singletonList(topic1));
+        partitionAssignor.prepare(singletonMap(consumerId, singletonList(t1p)));
+
+        client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE));
+        client.prepareResponse(body -> {
+            SyncGroupRequest sync = (SyncGroupRequest) body;
+            return sync.data.memberId().equals(consumerId) &&
+                sync.data.generationId() == 1 &&
+                sync.groupAssignments().containsKey(consumerId);
+        }, syncGroupResponse(singletonList(t1p), Errors.NONE));
+        coordinator.poll(time.timer(Long.MAX_VALUE));
+
+        AbstractCoordinator.Generation expectedGeneration = new AbstractCoordinator.Generation(1, consumerId, partitionAssignor.name());
+        assertFalse(coordinator.rejoinNeededOrPending());
+        assertEquals(expectedGeneration, coordinator.generation());
+
         prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.REBALANCE_IN_PROGRESS);
-        coordinator.commitOffsetsSync(singletonMap(t1p,
-                new OffsetAndMetadata(100L, "metadata")), time.timer(Long.MAX_VALUE));
+
+        assertThrows(CommitFailedException.class, () -> coordinator.commitOffsetsSync(singletonMap(t1p,
+            new OffsetAndMetadata(100L, "metadata")), time.timer(Long.MAX_VALUE)));
+
+        assertTrue(coordinator.rejoinNeededOrPending());
+        assertEquals(expectedGeneration, coordinator.generation());
     }
 
     @Test(expected = KafkaException.class)
diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
index 5d40e9bd51966..e601d34d218b4 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
@@ -180,38 +180,48 @@ class GroupCoordinator(val brokerId: Int,
         if (group.hasStaticMember(groupInstanceId)) {
           val oldMemberId = group.getStaticMemberId(groupInstanceId)
 
-          if (group.is(Stable)) {
-            info(s"Static member $groupInstanceId with unknown member id rejoins, assigning new member id $newMemberId, while " +
-              s"old member $oldMemberId will be removed. No rebalance will be triggered.")
-
-            val oldMember = group.replaceGroupInstance(oldMemberId, newMemberId, groupInstanceId)
-
-            // Heartbeat of old member id will expire without affection since the group no longer contains that member id.
-            // New heartbeat shall be scheduled with new member id.
-            completeAndScheduleNextHeartbeatExpiration(group, oldMember)
-
-            responseCallback(JoinGroupResult(
-              members = if (group.isLeader(newMemberId)) {
-                group.currentMemberMetadata
-              } else {
-                List.empty
-              },
-              memberId = newMemberId,
-              generationId = group.generationId,
-              subProtocol = group.protocolOrNull,
-              leaderId = group.leaderOrNull,
-              error = Errors.NONE))
-          } else {
-            val knownStaticMember = group.get(oldMemberId)
-            updateMemberAndRebalance(group, knownStaticMember, protocols, responseCallback)
+          group.currentState match {
+            case Stable =>
+              info(s"Static member $groupInstanceId with unknown member id rejoins group ${group.groupId} " +
+                s"in ${group.currentState} state. Assigning new member id $newMemberId, while old member $oldMemberId " +
+                "will be removed. No rebalance will be triggered.")
+
+              val oldMember = group.replaceGroupInstance(oldMemberId, newMemberId, groupInstanceId)
+
+              // Heartbeat of old member id will expire without affection since the group no longer contains that member id.
+              // New heartbeat shall be scheduled with new member id.
+              completeAndScheduleNextHeartbeatExpiration(group, oldMember)
+
+              responseCallback(JoinGroupResult(
+                members = if (group.isLeader(newMemberId)) {
+                  group.currentMemberMetadata
+                } else {
+                  List.empty
+                },
+                memberId = newMemberId,
+                generationId = group.generationId,
+                subProtocol = group.protocolOrNull,
+                leaderId = group.leaderOrNull,
+                error = Errors.NONE))
+
+            case _ =>
+              info(s"Static member $groupInstanceId with unkonwn member id rejoins group ${group.groupId} " +
+                s"in ${group.currentState} state. Update its membership with the pre-registered old member id $oldMemberId.")
+
+              val knownStaticMember = group.get(oldMemberId)
+              updateMemberAndRebalance(group, knownStaticMember, protocols, responseCallback)
           }
         } else if (requireKnownMemberId) {
             // If member id required (dynamic membership), register the member in the pending member list
             // and send back a response to call for another join group request with allocated member id.
-            group.addPendingMember(newMemberId)
-            addPendingMemberExpiration(group, newMemberId, sessionTimeoutMs)
-            responseCallback(joinError(newMemberId, Errors.MEMBER_ID_REQUIRED))
+          debug(s"Dynamic member with unknown member id rejoins group ${group.groupId} in " +
+              s"${group.currentState} state. Created a new member id $newMemberId and request the member to rejoin with this id.")
+          group.addPendingMember(newMemberId)
+          addPendingMemberExpiration(group, newMemberId, sessionTimeoutMs)
+          responseCallback(joinError(newMemberId, Errors.MEMBER_ID_REQUIRED))
         } else {
+          debug(s"Dynamic member with unknown member id rejoins group ${group.groupId} in " +
+            s"${group.currentState} state. Created a new member id $newMemberId for this member and add to the group.")
           addMemberAndRebalance(rebalanceTimeoutMs, sessionTimeoutMs, newMemberId, groupInstanceId,
             clientId, clientHost, protocolType, protocols, group, responseCallback)
 
@@ -613,16 +623,26 @@ class GroupCoordinator(val brokerId: Int,
         // The group is only using Kafka to store offsets.
         // Also, for transactional offset commits we don't need to validate group membership and the generation.
         groupManager.storeOffsets(group, memberId, offsetMetadata, responseCallback, producerId, producerEpoch)
-      } else if (group.is(CompletingRebalance)) {
-        responseCallback(offsetMetadata.mapValues(_ => Errors.REBALANCE_IN_PROGRESS))
       } else if (!group.has(memberId)) {
         responseCallback(offsetMetadata.mapValues(_ => Errors.UNKNOWN_MEMBER_ID))
       } else if (generationId != group.generationId) {
         responseCallback(offsetMetadata.mapValues(_ => Errors.ILLEGAL_GENERATION))
       } else {
-        val member = group.get(memberId)
-        completeAndScheduleNextHeartbeatExpiration(group, member)
-        groupManager.storeOffsets(group, memberId, offsetMetadata, responseCallback)
+        group.currentState match {
+          case Stable | PreparingRebalance =>
+            // During PreparingRebalance phase, we still allow a commit request since we rely
+            // on heartbeat response to eventually notify the rebalance in progress signal to the consumer
+            val member = group.get(memberId)
+            completeAndScheduleNextHeartbeatExpiration(group, member)
+            groupManager.storeOffsets(group, memberId, offsetMetadata, responseCallback)
+
+          case CompletingRebalance =>
+            // We should not receive a commit request if the group has not completed rebalance;
+            // but since the consumer's member.id and generation is valid, it means it has received
+            // the latest group generation information from the JoinResponse.
+            // So let's return a REBALANCE_IN_PROGRESS to let consumer handle it gracefully.
+            responseCallback(offsetMetadata.mapValues(_ => Errors.REBALANCE_IN_PROGRESS))
+        }
       }
     }
   }
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
index 5a1b20abbb58b..cd1ebc50428e9 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
@@ -1176,25 +1176,25 @@ class GroupCoordinatorTest {
 
     val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols,
       rebalanceTimeout = sessionTimeout, sessionTimeout = sessionTimeout)
-    val assignedConsumerId = joinGroupResult.memberId
+    val assignedMemberId = joinGroupResult.memberId
     val generationId = joinGroupResult.generationId
     val joinGroupError = joinGroupResult.error
     assertEquals(Errors.NONE, joinGroupError)
 
     EasyMock.reset(replicaManager)
-    val (_, syncGroupError) = syncGroupLeader(groupId, generationId, assignedConsumerId, Map(assignedConsumerId -> Array[Byte]()))
+    val (_, syncGroupError) = syncGroupLeader(groupId, generationId, assignedMemberId, Map(assignedMemberId -> Array[Byte]()))
     assertEquals(Errors.NONE, syncGroupError)
 
     timer.advanceClock(sessionTimeout / 2)
 
     EasyMock.reset(replicaManager)
-    val commitOffsetResult = commitOffsets(groupId, assignedConsumerId, generationId, Map(tp -> offset))
+    val commitOffsetResult = commitOffsets(groupId, assignedMemberId, generationId, Map(tp -> offset))
     assertEquals(Errors.NONE, commitOffsetResult(tp))
 
     timer.advanceClock(sessionTimeout / 2 + 100)
 
     EasyMock.reset(replicaManager)
-    val heartbeatResult = heartbeat(groupId, assignedConsumerId, 1)
+    val heartbeatResult = heartbeat(groupId, assignedMemberId, 1)
     assertEquals(Errors.NONE, heartbeatResult)
   }
 
@@ -2158,6 +2158,40 @@ class GroupCoordinatorTest {
     assertEquals(Errors.REBALANCE_IN_PROGRESS, commitOffsetResult(tp))
   }
 
+  @Test
+  def testCommitOffsetInCompletingRebalanceFromUnknownMemberId() {
+    val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
+    val tp = new TopicPartition("topic", 0)
+    val offset = offsetAndMetadata(0)
+
+    val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols)
+    val assignedMemberId = joinGroupResult.memberId
+    val generationId = joinGroupResult.generationId
+    val joinGroupError = joinGroupResult.error
+    assertEquals(Errors.NONE, joinGroupError)
+
+    EasyMock.reset(replicaManager)
+    val commitOffsetResult = commitOffsets(groupId, memberId, generationId, Map(tp -> offset))
+    assertEquals(Errors.UNKNOWN_MEMBER_ID, commitOffsetResult(tp))
+  }
+
+  @Test
+  def testCommitOffsetInCompletingRebalanceFromIllegalGeneration() {
+    val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
+    val tp = new TopicPartition("topic", 0)
+    val offset = offsetAndMetadata(0)
+
+    val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols)
+    val assignedMemberId = joinGroupResult.memberId
+    val generationId = joinGroupResult.generationId
+    val joinGroupError = joinGroupResult.error
+    assertEquals(Errors.NONE, joinGroupError)
+
+    EasyMock.reset(replicaManager)
+    val commitOffsetResult = commitOffsets(groupId, assignedMemberId, generationId + 1, Map(tp -> offset))
+    assertEquals(Errors.ILLEGAL_GENERATION, commitOffsetResult(tp))
+  }
+
   @Test
   def testHeartbeatDuringRebalanceCausesRebalanceInProgress() {
     // First start up a group (with a slightly larger timeout to give us time to heartbeat when the rebalance starts)
@@ -2666,7 +2700,7 @@ class GroupCoordinatorTest {
   }
 
   private def commitOffsets(groupId: String,
-                            consumerId: String,
+                            memberId: String,
                             generationId: Int,
                             offsets: Map[TopicPartition, OffsetAndMetadata],
                             groupInstanceId: Option[String] = None): CommitOffsetCallbackParams = {
@@ -2692,7 +2726,7 @@ class GroupCoordinatorTest {
     EasyMock.expect(replicaManager.getMagic(EasyMock.anyObject())).andReturn(Some(RecordBatch.MAGIC_VALUE_V1)).anyTimes()
     EasyMock.replay(replicaManager)
 
-    groupCoordinator.handleCommitOffsets(groupId, consumerId, groupInstanceId, generationId, offsets, responseCallback)
+    groupCoordinator.handleCommitOffsets(groupId, memberId, groupInstanceId, generationId, offsets, responseCallback)
     Await.result(responseFuture, Duration(40, TimeUnit.MILLISECONDS))
   }
 

From cb2c4c3d7aa854a8ce209145ab85851a8b4ffe85 Mon Sep 17 00:00:00 2001
From: Gardner Vickers 
Date: Tue, 11 Jun 2019 22:34:12 -0700
Subject: [PATCH 0355/1071] MINOR: Remove integration package prefix from
 ConsumerTopicCreationTest (#6916)

Reviewers: Manikumar Reddy 
---
 .../scala/integration/kafka/api/ConsumerTopicCreationTest.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/scala/integration/kafka/api/ConsumerTopicCreationTest.scala b/core/src/test/scala/integration/kafka/api/ConsumerTopicCreationTest.scala
index f447a56f50db9..55671bab20d5f 100644
--- a/core/src/test/scala/integration/kafka/api/ConsumerTopicCreationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/ConsumerTopicCreationTest.scala
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package integration.kafka.api
+package kafka.api
 
 import java.lang.{Boolean => JBoolean}
 import java.time.Duration

From 388889fbfe6e586355fd186572016d8c199b4f55 Mon Sep 17 00:00:00 2001
From: Almog Gavra 
Date: Wed, 12 Jun 2019 06:25:33 -0700
Subject: [PATCH 0356/1071] KAFKA-8514: Move the scala-java8-compat dependency
 from clients to core (#6910)

`clients` is a Java-only module and should not have any Scala dependency.

Reviewers: Ismael Juma 
---
 build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index d1a2a9da52ba7..ce72a0e81f728 100644
--- a/build.gradle
+++ b/build.gradle
@@ -660,6 +660,7 @@ project(':core') {
     compile libs.jacksonJDK8Datatypes
     compile libs.joptSimple
     compile libs.metrics
+    compile libs.scalaJava8Compat
     compile libs.scalaLibrary
     // only needed transitively, but set it explicitly to ensure it has the same version as scala-library
     compile libs.scalaReflect
@@ -939,7 +940,6 @@ project(':clients') {
     compile libs.lz4
     compile libs.snappy
     compile libs.slf4jApi
-    compile libs.scalaJava8Compat
 
     compileOnly libs.jacksonDatabind // for SASL/OAUTHBEARER bearer token parsing
     compileOnly libs.jacksonJDK8Datatypes

From e2c15e0eeb8e853aead9ef5e307930fd781b38e4 Mon Sep 17 00:00:00 2001
From: highluck 
Date: Wed, 12 Jun 2019 22:33:03 +0900
Subject: [PATCH 0357/1071] MINOR: Remove uncommitted code (#6919)

---
 .../main/java/org/apache/kafka/clients/MetadataCache.java  | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/MetadataCache.java b/clients/src/main/java/org/apache/kafka/clients/MetadataCache.java
index b5c9de638a735..e58da12127365 100644
--- a/clients/src/main/java/org/apache/kafka/clients/MetadataCache.java
+++ b/clients/src/main/java/org/apache/kafka/clients/MetadataCache.java
@@ -97,13 +97,6 @@ Optional getPartitionInfo(TopicPartition topicPartition)
         return Optional.ofNullable(metadataByPartition.get(topicPartition));
     }
 
-    synchronized void retainTopics(Collection topics) {
-        metadataByPartition.entrySet().removeIf(entry -> !topics.contains(entry.getKey().topic()));
-        unauthorizedTopics.retainAll(topics);
-        invalidTopics.retainAll(topics);
-        computeClusterView();
-    }
-
     Cluster cluster() {
         if (clusterInstance == null) {
             throw new IllegalStateException("Cached Cluster instance should not be null, but was.");

From e981b82601e9967d3a2492d2cebb45369407b398 Mon Sep 17 00:00:00 2001
From: Boyang Chen 
Date: Wed, 12 Jun 2019 08:41:58 -0700
Subject: [PATCH 0358/1071] KAFKA-8500; Static member rejoin should always
 update member.id (#6899)

This PR fixes a bug in static group membership. Previously we limit the `member.id` replacement in JoinGroup to only cases when the group is in Stable. This is error-prone and could potentially allow duplicate consumers reading from the same topic. For example, imagine a case where two unknown members join in the `PrepareRebalance` stage at the same time.

The PR fixes the following things:

1. Replace `member.id` at any time we see a known static member rejoins group with unknown member.id
2. Immediately fence any ongoing join/sync group callback to early terminate the duplicate member.
3. Clearly handle Dead/Empty cases as exceptional.
4. Return old leader id upon static member leader rejoin to avoid trivial member assignment being triggered.

Reviewers: Guozhang Wang , Jason Gustafson 
---
 .../coordinator/group/GroupCoordinator.scala  | 110 ++++++-----
 .../coordinator/group/GroupMetadata.scala     |  34 +++-
 .../coordinator/group/MemberMetadata.scala    |   3 +-
 .../main/scala/kafka/server/KafkaApis.scala   |  12 +-
 .../GroupCoordinatorConcurrencyTest.scala     |   6 +-
 .../group/GroupCoordinatorTest.scala          | 173 ++++++++++++++++--
 .../coordinator/group/GroupMetadataTest.scala |  95 +++++++++-
 tests/kafkatest/tests/client/consumer_test.py |   6 +
 8 files changed, 349 insertions(+), 90 deletions(-)

diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
index e601d34d218b4..a9ac67ff2a18c 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
@@ -58,7 +58,7 @@ class GroupCoordinator(val brokerId: Int,
   import GroupCoordinator._
 
   type JoinCallback = JoinGroupResult => Unit
-  type SyncCallback = (Array[Byte], Errors) => Unit
+  type SyncCallback = SyncGroupResult => Unit
 
   this.logIdent = "[GroupCoordinator " + brokerId + "]: "
 
@@ -179,37 +179,38 @@ class GroupCoordinator(val brokerId: Int,
 
         if (group.hasStaticMember(groupInstanceId)) {
           val oldMemberId = group.getStaticMemberId(groupInstanceId)
+          info(s"Static member $groupInstanceId with unknown member id rejoins, assigning new member id $newMemberId, while " +
+            s"old member $oldMemberId will be removed.")
+
+          val currentLeader = group.leaderOrNull
+          val member = group.replaceGroupInstance(oldMemberId, newMemberId, groupInstanceId)
+          // Heartbeat of old member id will expire without effect since the group no longer contains that member id.
+          // New heartbeat shall be scheduled with new member id.
+          completeAndScheduleNextHeartbeatExpiration(group, member)
+
+          val knownStaticMember = group.get(newMemberId)
+          group.updateMember(knownStaticMember, protocols, responseCallback)
 
           group.currentState match {
-            case Stable =>
-              info(s"Static member $groupInstanceId with unknown member id rejoins group ${group.groupId} " +
-                s"in ${group.currentState} state. Assigning new member id $newMemberId, while old member $oldMemberId " +
-                "will be removed. No rebalance will be triggered.")
-
-              val oldMember = group.replaceGroupInstance(oldMemberId, newMemberId, groupInstanceId)
-
-              // Heartbeat of old member id will expire without affection since the group no longer contains that member id.
-              // New heartbeat shall be scheduled with new member id.
-              completeAndScheduleNextHeartbeatExpiration(group, oldMember)
-
-              responseCallback(JoinGroupResult(
-                members = if (group.isLeader(newMemberId)) {
-                  group.currentMemberMetadata
-                } else {
-                  List.empty
-                },
+            case Stable | CompletingRebalance =>
+              info(s"Static member joins during ${group.currentState} stage will not trigger rebalance.")
+              group.maybeInvokeJoinCallback(member, JoinGroupResult(
+                members = List.empty,
                 memberId = newMemberId,
                 generationId = group.generationId,
                 subProtocol = group.protocolOrNull,
-                leaderId = group.leaderOrNull,
+                // We want to avoid current leader performing trivial assignment while the group
+                // is in stable/awaiting sync stage, because the new assignment in leader's next sync call
+                // won't be broadcast by a stable/awaiting sync group. This could be guaranteed by
+                // always returning the old leader id so that the current leader won't assume itself
+                // as a leader based on the returned message, since the new member.id won't match
+                // returned leader id, therefore no assignment will be performed.
+                leaderId = currentLeader,
                 error = Errors.NONE))
-
-            case _ =>
-              info(s"Static member $groupInstanceId with unkonwn member id rejoins group ${group.groupId} " +
-                s"in ${group.currentState} state. Update its membership with the pre-registered old member id $oldMemberId.")
-
-              val knownStaticMember = group.get(oldMemberId)
-              updateMemberAndRebalance(group, knownStaticMember, protocols, responseCallback)
+            case Empty | Dead =>
+              throw new IllegalStateException(s"Group ${group.groupId} was not supposed to be " +
+                s"in the state ${group.currentState} when the unknown static member $groupInstanceId rejoins.")
+            case PreparingRebalance =>
           }
         } else if (requireKnownMemberId) {
             // If member id required (dynamic membership), register the member in the pending member list
@@ -224,7 +225,6 @@ class GroupCoordinator(val brokerId: Int,
             s"${group.currentState} state. Created a new member id $newMemberId for this member and add to the group.")
           addMemberAndRebalance(rebalanceTimeoutMs, sessionTimeoutMs, newMemberId, groupInstanceId,
             clientId, clientHost, protocolType, protocols, group, responseCallback)
-
         }
       }
     }
@@ -338,13 +338,13 @@ class GroupCoordinator(val brokerId: Int,
         // group will need to start over at JoinGroup. By returning rebalance in progress, the consumer
         // will attempt to rejoin without needing to rediscover the coordinator. Note that we cannot
         // return COORDINATOR_LOAD_IN_PROGRESS since older clients do not expect the error.
-        responseCallback(Array.empty, Errors.REBALANCE_IN_PROGRESS)
+        responseCallback(SyncGroupResult(Array.empty, Errors.REBALANCE_IN_PROGRESS))
 
-      case Some(error) => responseCallback(Array.empty, error)
+      case Some(error) => responseCallback(SyncGroupResult(Array.empty, error))
 
       case None =>
         groupManager.getGroup(groupId) match {
-          case None => responseCallback(Array.empty, Errors.UNKNOWN_MEMBER_ID)
+          case None => responseCallback(SyncGroupResult(Array.empty, Errors.UNKNOWN_MEMBER_ID))
           case Some(group) => doSyncGroup(group, generation, memberId, groupInstanceId, groupAssignment, responseCallback)
         }
     }
@@ -362,20 +362,20 @@ class GroupCoordinator(val brokerId: Int,
         // from the coordinator metadata; this is likely that the group has migrated to some other
         // coordinator OR the group is in a transient unstable phase. Let the member retry
         // finding the correct coordinator and rejoin.
-        responseCallback(Array.empty, Errors.COORDINATOR_NOT_AVAILABLE)
+        responseCallback(SyncGroupResult(Array.empty, Errors.COORDINATOR_NOT_AVAILABLE))
       } else if (group.isStaticMemberFenced(memberId, groupInstanceId)) {
-        responseCallback(Array.empty, Errors.FENCED_INSTANCE_ID)
+        responseCallback(SyncGroupResult(Array.empty, Errors.FENCED_INSTANCE_ID))
       } else if (!group.has(memberId)) {
-        responseCallback(Array.empty, Errors.UNKNOWN_MEMBER_ID)
+        responseCallback(SyncGroupResult(Array.empty, Errors.UNKNOWN_MEMBER_ID))
       } else if (generationId != group.generationId) {
-        responseCallback(Array.empty, Errors.ILLEGAL_GENERATION)
+        responseCallback(SyncGroupResult(Array.empty, Errors.ILLEGAL_GENERATION))
       } else {
         group.currentState match {
           case Empty =>
-            responseCallback(Array.empty, Errors.UNKNOWN_MEMBER_ID)
+            responseCallback(SyncGroupResult(Array.empty, Errors.UNKNOWN_MEMBER_ID))
 
           case PreparingRebalance =>
-            responseCallback(Array.empty, Errors.REBALANCE_IN_PROGRESS)
+            responseCallback(SyncGroupResult(Array.empty, Errors.REBALANCE_IN_PROGRESS))
 
           case CompletingRebalance =>
             group.get(memberId).awaitingSyncCallback = responseCallback
@@ -409,7 +409,7 @@ class GroupCoordinator(val brokerId: Int,
           case Stable =>
             // if the group is stable, we just return the current assignment
             val memberMetadata = group.get(memberId)
-            responseCallback(memberMetadata.assignment, Errors.NONE)
+            responseCallback(SyncGroupResult(memberMetadata.assignment, Errors.NONE))
             completeAndScheduleNextHeartbeatExpiration(group, group.get(memberId))
 
           case Dead =>
@@ -476,7 +476,7 @@ class GroupCoordinator(val brokerId: Int,
                   case Empty =>
                     group.transitionTo(Dead)
                     groupsEligibleForDeletion :+= group
-                  case _ =>
+                  case Stable | PreparingRebalance | CompletingRebalance =>
                     groupErrors += groupId -> Errors.NON_EMPTY_GROUP
                 }
               }
@@ -734,10 +734,7 @@ class GroupCoordinator(val brokerId: Int,
 
         case Stable | CompletingRebalance =>
           for (member <- group.allMemberMetadata) {
-            if (member.awaitingSyncCallback != null) {
-              member.awaitingSyncCallback(Array.empty[Byte], Errors.NOT_COORDINATOR)
-              member.awaitingSyncCallback = null
-            }
+            group.maybeInvokeSyncCallback(member, SyncGroupResult(Array.empty, Errors.NOT_COORDINATOR))
             heartbeatPurgatory.checkAndComplete(MemberKey(member.groupId, member.memberId))
           }
       }
@@ -772,16 +769,13 @@ class GroupCoordinator(val brokerId: Int,
 
   private def resetAndPropagateAssignmentError(group: GroupMetadata, error: Errors) {
     assert(group.is(CompletingRebalance))
-    group.allMemberMetadata.foreach(_.assignment = Array.empty[Byte])
+    group.allMemberMetadata.foreach(_.assignment = Array.empty)
     propagateAssignment(group, error)
   }
 
   private def propagateAssignment(group: GroupMetadata, error: Errors) {
     for (member <- group.allMemberMetadata) {
-      if (member.awaitingSyncCallback != null) {
-        member.awaitingSyncCallback(member.assignment, error)
-        member.awaitingSyncCallback = null
-
+      if (group.maybeInvokeSyncCallback(member, SyncGroupResult(member.assignment, error))) {
         // reset the session timeout for members after propagating the member's assignment.
         // This is because if any member's session expired while we were still awaiting either
         // the leader sync group or the storage callback, its expiration will be ignored and no
@@ -791,16 +785,6 @@ class GroupCoordinator(val brokerId: Int,
     }
   }
 
-  private def joinError(memberId: String, error: Errors): JoinGroupResult = {
-    JoinGroupResult(
-      members = List.empty,
-      memberId = memberId,
-      generationId = GroupCoordinator.NoGeneration,
-      subProtocol = GroupCoordinator.NoProtocol,
-      leaderId = GroupCoordinator.NoLeader,
-      error = error)
-  }
-
   /**
    * Complete existing DelayedHeartbeats for the given member and schedule the next one
    */
@@ -1110,6 +1094,15 @@ object GroupCoordinator {
     new GroupCoordinator(config.brokerId, groupConfig, offsetConfig, groupMetadataManager, heartbeatPurgatory, joinPurgatory, time)
   }
 
+  def joinError(memberId: String, error: Errors): JoinGroupResult = {
+    JoinGroupResult(
+      members = List.empty,
+      memberId = memberId,
+      generationId = GroupCoordinator.NoGeneration,
+      subProtocol = GroupCoordinator.NoProtocol,
+      leaderId = GroupCoordinator.NoLeader,
+      error = error)
+  }
 }
 
 case class GroupConfig(groupMinSessionTimeoutMs: Int,
@@ -1123,3 +1116,6 @@ case class JoinGroupResult(members: List[JoinGroupResponseMember],
                            subProtocol: String,
                            leaderId: String,
                            error: Errors)
+
+case class SyncGroupResult(memberAssignment: Array[Byte],
+                           error: Errors)
diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala
index 21aae4265479c..58a68a2c18b16 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala
@@ -23,6 +23,7 @@ import kafka.common.OffsetAndMetadata
 import kafka.utils.{CoreUtils, Logging, nonthreadsafe}
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.message.JoinGroupResponseData.JoinGroupResponseMember
+import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.utils.Time
 
 import scala.collection.{Seq, immutable, mutable}
@@ -162,7 +163,7 @@ case class GroupSummary(state: String,
   * being materialized.
   */
 case class CommitRecordMetadataAndOffset(appendedBatchOffset: Option[Long], offsetAndMetadata: OffsetAndMetadata) {
-  def olderThan(that: CommitRecordMetadataAndOffset) : Boolean = appendedBatchOffset.get < that.appendedBatchOffset.get
+  def olderThan(that: CommitRecordMetadataAndOffset): Boolean = appendedBatchOffset.get < that.appendedBatchOffset.get
 }
 
 /**
@@ -290,6 +291,19 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
     val oldMember = members.remove(oldMemberId)
       .getOrElse(throw new IllegalArgumentException(s"Cannot replace non-existing member id $oldMemberId"))
 
+    // Fence potential duplicate member immediately if someone awaits join/sync callback.
+    maybeInvokeJoinCallback(oldMember, JoinGroupResult(
+      members = List.empty,
+      memberId = oldMemberId,
+      generationId = GroupCoordinator.NoGeneration,
+      subProtocol = GroupCoordinator.NoProtocol,
+      leaderId = GroupCoordinator.NoLeader,
+      error = Errors.FENCED_INSTANCE_ID))
+
+    maybeInvokeSyncCallback(oldMember, SyncGroupResult(
+      Array.empty, Errors.FENCED_INSTANCE_ID
+    ))
+
     oldMember.memberId = newMemberId
     members.put(newMemberId, oldMember)
 
@@ -425,7 +439,7 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
   }
 
   def maybeInvokeJoinCallback(member: MemberMetadata,
-                              joinGroupResult: JoinGroupResult) : Unit = {
+                              joinGroupResult: JoinGroupResult): Unit = {
     if (member.isAwaitingJoin) {
       member.awaitingJoinCallback(joinGroupResult)
       member.awaitingJoinCallback = null
@@ -433,6 +447,20 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
     }
   }
 
+  /**
+    * @return true if a sync callback actually performs.
+    */
+  def maybeInvokeSyncCallback(member: MemberMetadata,
+                              syncGroupResult: SyncGroupResult): Boolean = {
+    if (member.isAwaitingSync) {
+      member.awaitingSyncCallback(syncGroupResult)
+      member.awaitingSyncCallback = null
+      true
+    } else {
+      false
+    }
+  }
+
   def initNextGeneration() = {
     if (members.nonEmpty) {
       generationId += 1
@@ -600,7 +628,7 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
     }.toMap
   }
 
-  def removeExpiredOffsets(currentTimestamp: Long, offsetRetentionMs: Long) : Map[TopicPartition, OffsetAndMetadata] = {
+  def removeExpiredOffsets(currentTimestamp: Long, offsetRetentionMs: Long): Map[TopicPartition, OffsetAndMetadata] = {
 
     def getExpiredOffsets(baseTimestamp: CommitRecordMetadataAndOffset => Long): Map[TopicPartition, OffsetAndMetadata] = {
       offsets.filter {
diff --git a/core/src/main/scala/kafka/coordinator/group/MemberMetadata.scala b/core/src/main/scala/kafka/coordinator/group/MemberMetadata.scala
index a090d97728aee..83ff70971d377 100644
--- a/core/src/main/scala/kafka/coordinator/group/MemberMetadata.scala
+++ b/core/src/main/scala/kafka/coordinator/group/MemberMetadata.scala
@@ -66,13 +66,14 @@ private[group] class MemberMetadata(var memberId: String,
 
   var assignment: Array[Byte] = Array.empty[Byte]
   var awaitingJoinCallback: JoinGroupResult => Unit = null
-  var awaitingSyncCallback: (Array[Byte], Errors) => Unit = null
+  var awaitingSyncCallback: SyncGroupResult => Unit = null
   var latestHeartbeat: Long = -1
   var isLeaving: Boolean = false
   var isNew: Boolean = false
   val isStaticMember: Boolean = groupInstanceId.isDefined
 
   def isAwaitingJoin = awaitingJoinCallback != null
+  def isAwaitingSync = awaitingSyncCallback != null
 
   /**
    * Get metadata corresponding to the provided protocol.
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 0a1f25122b900..fa45cf5308a84 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -31,7 +31,7 @@ import kafka.api.{ApiVersion, KAFKA_0_11_0_IV0, KAFKA_2_3_IV0}
 import kafka.cluster.Partition
 import kafka.common.OffsetAndMetadata
 import kafka.controller.KafkaController
-import kafka.coordinator.group.{GroupCoordinator, JoinGroupResult}
+import kafka.coordinator.group.{GroupCoordinator, JoinGroupResult, SyncGroupResult}
 import kafka.coordinator.transaction.{InitProducerIdResult, TransactionCoordinator}
 import kafka.message.ZStdCompressionCodec
 import kafka.network.RequestChannel
@@ -1408,12 +1408,12 @@ class KafkaApis(val requestChannel: RequestChannel,
   def handleSyncGroupRequest(request: RequestChannel.Request) {
     val syncGroupRequest = request.body[SyncGroupRequest]
 
-    def sendResponseCallback(memberState: Array[Byte], error: Errors) {
+    def sendResponseCallback(syncGroupResult: SyncGroupResult) {
       sendResponseMaybeThrottle(request, requestThrottleMs =>
         new SyncGroupResponse(
           new SyncGroupResponseData()
-            .setErrorCode(error.code)
-            .setAssignment(memberState)
+            .setErrorCode(syncGroupResult.error.code)
+            .setAssignment(syncGroupResult.memberAssignment)
             .setThrottleTimeMs(requestThrottleMs)
         ))
     }
@@ -1422,9 +1422,9 @@ class KafkaApis(val requestChannel: RequestChannel,
       // Only enable static membership when IBP >= 2.3, because it is not safe for the broker to use the static member logic
       // until we are sure that all brokers support it. If static group being loaded by an older coordinator, it will discard
       // the group.instance.id field, so static members could accidentally become "dynamic", which leads to wrong states.
-      sendResponseCallback(Array[Byte](), Errors.UNSUPPORTED_VERSION)
+      sendResponseCallback(SyncGroupResult(Array[Byte](), Errors.UNSUPPORTED_VERSION))
     } else if (!authorize(request.session, Read, Resource(Group, syncGroupRequest.data.groupId, LITERAL))) {
-      sendResponseCallback(Array[Byte](), Errors.GROUP_AUTHORIZATION_FAILED)
+      sendResponseCallback(SyncGroupResult(Array[Byte](), Errors.GROUP_AUTHORIZATION_FAILED))
     } else {
       val assignmentMap = immutable.Map.newBuilder[String, Array[Byte]]
       syncGroupRequest.data.assignments.asScala.foreach { assignment =>
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorConcurrencyTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorConcurrencyTest.scala
index 3da5a0cf7aa87..1cee665f1c893 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorConcurrencyTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorConcurrencyTest.scala
@@ -173,8 +173,8 @@ class GroupCoordinatorConcurrencyTest extends AbstractCoordinatorConcurrencyTest
 
   class SyncGroupOperation extends GroupOperation[SyncGroupCallbackParams, SyncGroupCallback] {
     override def responseCallback(responsePromise: Promise[SyncGroupCallbackParams]): SyncGroupCallback = {
-      val callback: SyncGroupCallback = (assignment, error) =>
-        responsePromise.success((assignment, error))
+      val callback: SyncGroupCallback = syncGroupResult =>
+        responsePromise.success(syncGroupResult.memberAssignment, syncGroupResult.error)
       callback
     }
     override def runWithCallback(member: GroupMember, responseCallback: SyncGroupCallback): Unit = {
@@ -280,7 +280,7 @@ object GroupCoordinatorConcurrencyTest {
 
   type JoinGroupCallback = JoinGroupResult => Unit
   type SyncGroupCallbackParams = (Array[Byte], Errors)
-  type SyncGroupCallback = (Array[Byte], Errors) => Unit
+  type SyncGroupCallback = SyncGroupResult => Unit
   type HeartbeatCallbackParams = Errors
   type HeartbeatCallback = Errors => Unit
   type CommitOffsetCallbackParams = Map[TopicPartition, Errors]
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
index cd1ebc50428e9..c12e19d1f5c84 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
@@ -47,7 +47,7 @@ import scala.concurrent.{Await, Future, Promise, TimeoutException}
 class GroupCoordinatorTest {
   type JoinGroupCallback = JoinGroupResult => Unit
   type SyncGroupCallbackParams = (Array[Byte], Errors)
-  type SyncGroupCallback = (Array[Byte], Errors) => Unit
+  type SyncGroupCallback = SyncGroupResult => Unit
   type HeartbeatCallbackParams = Errors
   type HeartbeatCallback = Errors => Unit
   type CommitOffsetCallbackParams = Map[TopicPartition, Errors]
@@ -59,7 +59,7 @@ class GroupCoordinatorTest {
   val ClientHost = "localhost"
   val GroupMinSessionTimeout = 10
   val GroupMaxSessionTimeout = 10 * 60 * 1000
-  val GroupMaxSize = 3
+  val GroupMaxSize = 4
   val DefaultRebalanceTimeout = 500
   val DefaultSessionTimeout = 500
   val GroupInitialRebalanceDelay = 50
@@ -146,7 +146,7 @@ class GroupCoordinatorTest {
     // SyncGroup
     var syncGroupResponse: Option[Errors] = None
     groupCoordinator.handleSyncGroup(otherGroupId, 1, memberId, None, Map.empty[String, Array[Byte]],
-      (_, error)=> syncGroupResponse = Some(error))
+      syncGroupResult => syncGroupResponse = Some(syncGroupResult.error))
     assertEquals(Some(Errors.REBALANCE_IN_PROGRESS), syncGroupResponse)
 
     // OffsetCommit
@@ -438,6 +438,151 @@ class GroupCoordinatorTest {
     assertEquals(Errors.FENCED_INSTANCE_ID, joinGroupResult.error)
   }
 
+  @Test
+  def staticMemberFenceDuplicateRejoinedFollower() {
+    val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
+
+    EasyMock.reset(replicaManager)
+    // A third member joins will trigger rebalance.
+    sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols)
+    timer.advanceClock(1)
+    assertTrue(getGroup(groupId).is(PreparingRebalance))
+
+    EasyMock.reset(replicaManager)
+    timer.advanceClock(1)
+    // Old follower rejoins group will be matching current member.id.
+    val oldFollowerJoinGroupFuture =
+      sendJoinGroup(groupId, rebalanceResult.followerId, protocolType, protocols, groupInstanceId = followerInstanceId)
+
+    EasyMock.reset(replicaManager)
+    timer.advanceClock(1)
+    // Duplicate follower joins group with unknown member id will trigger member.id replacement.
+    val duplicateFollowerJoinFuture =
+      sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols, groupInstanceId = followerInstanceId)
+
+    timer.advanceClock(1)
+    // Old member shall be fenced immediately upon duplicate follower joins.
+    val oldFollowerJoinGroupResult = Await.result(oldFollowerJoinGroupFuture, Duration(1, TimeUnit.MILLISECONDS))
+    checkJoinGroupResult(oldFollowerJoinGroupResult,
+      Errors.FENCED_INSTANCE_ID,
+      -1,
+      Set.empty,
+      groupId,
+      PreparingRebalance)
+    verifyDelayedTaskNotCompleted(duplicateFollowerJoinFuture)
+  }
+
+  @Test
+  def staticMemberFenceDuplicateSyncingFollowerAfterMemberIdChanged() {
+    val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
+
+    EasyMock.reset(replicaManager)
+    // Known leader rejoins will trigger rebalance.
+    val leaderJoinGroupFuture =
+      sendJoinGroup(groupId, rebalanceResult.leaderId, protocolType, protocols, groupInstanceId = leaderInstanceId)
+    timer.advanceClock(1)
+    assertTrue(getGroup(groupId).is(PreparingRebalance))
+
+    EasyMock.reset(replicaManager)
+    timer.advanceClock(1)
+    // Old follower rejoins group will match current member.id.
+    val oldFollowerJoinGroupFuture =
+      sendJoinGroup(groupId, rebalanceResult.followerId, protocolType, protocols, groupInstanceId = followerInstanceId)
+
+    timer.advanceClock(1)
+    val leaderJoinGroupResult = Await.result(leaderJoinGroupFuture, Duration(1, TimeUnit.MILLISECONDS))
+    checkJoinGroupResult(leaderJoinGroupResult,
+      Errors.NONE,
+      rebalanceResult.generation + 1,
+      Set(leaderInstanceId, followerInstanceId),
+      groupId,
+      CompletingRebalance)
+    assertEquals(leaderJoinGroupResult.leaderId, leaderJoinGroupResult.memberId)
+    assertEquals(rebalanceResult.leaderId, leaderJoinGroupResult.leaderId)
+
+    // Old member shall be getting a successful join group response.
+    val oldFollowerJoinGroupResult = Await.result(oldFollowerJoinGroupFuture, Duration(1, TimeUnit.MILLISECONDS))
+    checkJoinGroupResult(oldFollowerJoinGroupResult,
+      Errors.NONE,
+      rebalanceResult.generation + 1,
+      Set.empty,
+      groupId,
+      CompletingRebalance,
+      expectedLeaderId = leaderJoinGroupResult.memberId)
+
+    EasyMock.reset(replicaManager)
+    val oldFollowerSyncGroupFuture = sendSyncGroupFollower(groupId, oldFollowerJoinGroupResult.generationId,
+      oldFollowerJoinGroupResult.memberId, followerInstanceId)
+
+    // Duplicate follower joins group with unknown member id will trigger member.id replacement.
+    EasyMock.reset(replicaManager)
+    val duplicateFollowerJoinFuture =
+      sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols, groupInstanceId = followerInstanceId)
+    timer.advanceClock(1)
+
+    // Old follower sync callback will return fenced exception while broker replaces the member identity.
+    val oldFollowerSyncGroupResult = Await.result(oldFollowerSyncGroupFuture, Duration(1, TimeUnit.MILLISECONDS))
+    assertEquals(oldFollowerSyncGroupResult._2, Errors.FENCED_INSTANCE_ID)
+
+    // Duplicate follower will get the same response as old follower.
+    val duplicateFollowerJoinGroupResult = Await.result(duplicateFollowerJoinFuture, Duration(1, TimeUnit.MILLISECONDS))
+    checkJoinGroupResult(duplicateFollowerJoinGroupResult,
+      Errors.NONE,
+      rebalanceResult.generation + 1,
+      Set.empty,
+      groupId,
+      CompletingRebalance,
+      expectedLeaderId = leaderJoinGroupResult.memberId)
+  }
+
+  @Test
+  def staticMemberFenceDuplicateRejoiningFollowerAfterMemberIdChanged() {
+    val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
+
+    EasyMock.reset(replicaManager)
+    // Known leader rejoins will trigger rebalance.
+    val leaderJoinGroupFuture =
+      sendJoinGroup(groupId, rebalanceResult.leaderId, protocolType, protocols, groupInstanceId = leaderInstanceId)
+    timer.advanceClock(1)
+    assertTrue(getGroup(groupId).is(PreparingRebalance))
+
+    EasyMock.reset(replicaManager)
+    // Duplicate follower joins group will trigger member.id replacement.
+    val duplicateFollowerJoinGroupFuture =
+      sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols, groupInstanceId = followerInstanceId)
+
+    EasyMock.reset(replicaManager)
+    timer.advanceClock(1)
+    // Old follower rejoins group will fail because member.id already updated.
+    val oldFollowerJoinGroupFuture =
+      sendJoinGroup(groupId, rebalanceResult.followerId, protocolType, protocols, groupInstanceId = followerInstanceId)
+
+    val leaderRejoinGroupResult = Await.result(leaderJoinGroupFuture, Duration(1, TimeUnit.MILLISECONDS))
+    checkJoinGroupResult(leaderRejoinGroupResult,
+      Errors.NONE,
+      rebalanceResult.generation + 1,
+      Set(leaderInstanceId, followerInstanceId),
+      groupId,
+      CompletingRebalance)
+
+    val duplicateFollowerJoinGroupResult = Await.result(duplicateFollowerJoinGroupFuture, Duration(1, TimeUnit.MILLISECONDS))
+    checkJoinGroupResult(duplicateFollowerJoinGroupResult,
+      Errors.NONE,
+      rebalanceResult.generation + 1,
+      Set.empty,
+      groupId,
+      CompletingRebalance)
+    assertNotEquals(rebalanceResult.followerId, duplicateFollowerJoinGroupResult.memberId)
+
+    val oldFollowerJoinGroupResult = Await.result(oldFollowerJoinGroupFuture, Duration(1, TimeUnit.MILLISECONDS))
+    checkJoinGroupResult(oldFollowerJoinGroupResult,
+      Errors.FENCED_INSTANCE_ID,
+      -1,
+      Set.empty,
+      groupId,
+      CompletingRebalance)
+  }
+
   @Test
   def staticMemberRejoinWithKnownMemberId() {
     var joinGroupResult = staticJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, groupInstanceId, protocolType, protocols)
@@ -464,31 +609,31 @@ class GroupCoordinatorTest {
   def staticMemberRejoinWithLeaderIdAndUnknownMemberId() {
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
 
-    // A static leader rejoin with unknown id will not trigger rebalance.
+    // A static leader rejoin with unknown id will not trigger rebalance, and no assignment will be returned.
     EasyMock.reset(replicaManager)
     val joinGroupResult = staticJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, leaderInstanceId, protocolType, protocolSuperset, clockAdvance = 1)
 
     checkJoinGroupResult(joinGroupResult,
       Errors.NONE,
       rebalanceResult.generation, // The group should be at the same generation
-      Set(leaderInstanceId, followerInstanceId),
+      Set.empty,
       groupId,
-      Stable)
+      Stable,
+      rebalanceResult.leaderId)
 
     EasyMock.reset(replicaManager)
     val oldLeaderJoinGroupResult = staticJoinGroup(groupId, rebalanceResult.leaderId, leaderInstanceId, protocolType, protocolSuperset, clockAdvance = 1)
     assertEquals(Errors.FENCED_INSTANCE_ID, oldLeaderJoinGroupResult.error)
 
     EasyMock.reset(replicaManager)
-    assertNotEquals(rebalanceResult.leaderId, joinGroupResult.leaderId)
     // Old leader will get fenced.
     val oldLeaderSyncGroupResult = syncGroupLeader(groupId, rebalanceResult.generation, rebalanceResult.leaderId, Map.empty, leaderInstanceId)
     assertEquals(Errors.FENCED_INSTANCE_ID, oldLeaderSyncGroupResult._2)
 
+    // Calling sync on old leader.id will fail because that leader.id is no longer valid and replaced.
     EasyMock.reset(replicaManager)
     val newLeaderSyncGroupResult = syncGroupLeader(groupId, rebalanceResult.generation, joinGroupResult.leaderId, Map.empty)
-    assertEquals(Errors.NONE, newLeaderSyncGroupResult._2)
-    assertEquals(rebalanceResult.leaderAssignment, newLeaderSyncGroupResult._1)
+    assertEquals(Errors.UNKNOWN_MEMBER_ID, newLeaderSyncGroupResult._2)
   }
 
   @Test
@@ -599,7 +744,7 @@ class GroupCoordinatorTest {
     val leaderRejoinGroupFuture = sendJoinGroup(groupId, rebalanceResult.leaderId, protocolType, protocolSuperset, leaderInstanceId)
     // Rebalance complete immediately after follower rejoin.
     EasyMock.reset(replicaManager)
-    val followerRejoinWithFuture = sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocolSuperset, followerInstanceId)
+    val followerRejoinWithFuture = sendJoinGroup(groupId, rebalanceResult.followerId, protocolType, protocolSuperset, followerInstanceId)
 
     timer.advanceClock(1)
 
@@ -656,6 +801,8 @@ class GroupCoordinatorTest {
       groupId,
       Stable)
 
+    assertNotEquals(rebalanceResult.followerId, joinGroupResult.memberId)
+
     EasyMock.reset(replicaManager)
     val syncGroupResult = syncGroupFollower(groupId, rebalanceResult.generation, joinGroupResult.memberId)
     assertEquals(Errors.NONE, syncGroupResult._2)
@@ -1000,7 +1147,7 @@ class GroupCoordinatorTest {
                                    expectedGroupState: GroupState,
                                    expectedLeaderId: String = JoinGroupRequest.UNKNOWN_MEMBER_ID,
                                    expectedMemberId: String = JoinGroupRequest.UNKNOWN_MEMBER_ID) {
-    assertEquals(Errors.NONE, joinGroupResult.error)
+    assertEquals(expectedError, joinGroupResult.error)
     assertEquals(expectedGeneration, joinGroupResult.generationId)
     assertEquals(expectedGroupInstanceIds.size, joinGroupResult.members.size)
     val resultedGroupInstanceIds = joinGroupResult.members.map(member => Some(member.groupInstanceId())).toSet
@@ -2547,8 +2694,8 @@ class GroupCoordinatorTest {
   private def setupSyncGroupCallback: (Future[SyncGroupCallbackParams], SyncGroupCallback) = {
     val responsePromise = Promise[SyncGroupCallbackParams]
     val responseFuture = responsePromise.future
-    val responseCallback: SyncGroupCallback = (assignment, error) =>
-      responsePromise.success((assignment, error))
+    val responseCallback: SyncGroupCallback = syncGroupResult =>
+      responsePromise.success(syncGroupResult.memberAssignment, syncGroupResult.error)
     (responseFuture, responseCallback)
   }
 
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala
index a3a9008b8933e..177bef726eb9f 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala
@@ -19,6 +19,7 @@ package kafka.coordinator.group
 
 import kafka.common.OffsetAndMetadata
 import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.utils.Time
 import org.junit.Assert._
 import org.junit.{Before, Test}
@@ -30,16 +31,20 @@ class GroupMetadataTest {
   private val protocolType = "consumer"
   private val groupId = "groupId"
   private val groupInstanceId = Some("groupInstanceId")
+  private val memberId = "memberId"
   private val clientId = "clientId"
   private val clientHost = "clientHost"
   private val rebalanceTimeoutMs = 60000
   private val sessionTimeoutMs = 10000
 
   private var group: GroupMetadata = null
+  private var member: MemberMetadata = null
 
   @Before
   def setUp() {
     group = new GroupMetadata("groupId", Empty, Time.SYSTEM)
+    member = new MemberMetadata(memberId, groupId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs,
+      protocolType, List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte])))
   }
 
   @Test
@@ -240,10 +245,6 @@ class GroupMetadataTest {
     // by default, the group supports everything
     assertTrue(group.supportsProtocols(protocolType, Set("roundrobin", "range")))
 
-    val memberId = "memberId"
-    val member = new MemberMetadata(memberId, groupId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs,
-      sessionTimeoutMs, protocolType, List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte])))
-
     group.add(member)
     group.transitionTo(PreparingRebalance)
     assertTrue(group.supportsProtocols(protocolType, Set("roundrobin", "foo")))
@@ -263,9 +264,7 @@ class GroupMetadataTest {
 
   @Test
   def testInitNextGeneration() {
-    val memberId = "memberId"
-    val member = new MemberMetadata(memberId, groupId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs,
-      protocolType, List(("roundrobin", Array.empty[Byte])))
+    member.supportedProtocols = List(("roundrobin", Array.empty[Byte]))
 
     group.transitionTo(PreparingRebalance)
     group.add(member, _ => ())
@@ -465,6 +464,88 @@ class GroupMetadataTest {
     assertFalse(group.hasPendingOffsetCommitsFromProducer(producerId))
   }
 
+  @Test(expected = classOf[IllegalArgumentException])
+  def testReplaceGroupInstanceWithEmptyGroupInstanceId(): Unit = {
+    group.add(member)
+    group.addStaticMember(groupInstanceId, memberId)
+    assertTrue(group.isLeader(memberId))
+    assertEquals(memberId, group.getStaticMemberId(groupInstanceId))
+
+    val newMemberId = "newMemberId"
+    group.replaceGroupInstance(memberId, newMemberId, Option.empty)
+  }
+
+  @Test(expected = classOf[IllegalArgumentException])
+  def testReplaceGroupInstanceWithNonExistingMember(): Unit = {
+    val newMemberId = "newMemberId"
+    group.replaceGroupInstance(memberId, newMemberId, groupInstanceId)
+  }
+
+  @Test
+  def testReplaceGroupInstance(): Unit = {
+    var joinAwaitingMemberFenced = false
+    group.add(member, joinGroupResult => {
+      joinAwaitingMemberFenced = joinGroupResult.error == Errors.FENCED_INSTANCE_ID
+    })
+    var syncAwaitingMemberFenced = false
+    member.awaitingSyncCallback = syncGroupResult => {
+      syncAwaitingMemberFenced = syncGroupResult.error == Errors.FENCED_INSTANCE_ID
+    }
+    group.addStaticMember(groupInstanceId, memberId)
+    assertTrue(group.isLeader(memberId))
+    assertEquals(memberId, group.getStaticMemberId(groupInstanceId))
+
+    val newMemberId = "newMemberId"
+    group.replaceGroupInstance(memberId, newMemberId, groupInstanceId)
+    assertTrue(group.isLeader(newMemberId))
+    assertEquals(newMemberId, group.getStaticMemberId(groupInstanceId))
+    assertTrue(joinAwaitingMemberFenced)
+    assertTrue(syncAwaitingMemberFenced)
+    assertFalse(member.isAwaitingJoin)
+    assertFalse(member.isAwaitingSync)
+  }
+
+  @Test
+  def testInvokeJoinCallback(): Unit = {
+    var invoked = false
+    group.add(member, _ => {
+      invoked = true
+    })
+
+    assertTrue(group.hasAllMembersJoined)
+    group.maybeInvokeJoinCallback(member, GroupCoordinator.joinError(member.memberId, Errors.NONE))
+    assertTrue(invoked)
+    assertFalse(member.isAwaitingJoin)
+  }
+
+  @Test
+  def testNotInvokeJoinCallback(): Unit = {
+    group.add(member)
+
+    assertFalse(member.isAwaitingJoin)
+    group.maybeInvokeJoinCallback(member, GroupCoordinator.joinError(member.memberId, Errors.NONE))
+    assertFalse(member.isAwaitingJoin)
+  }
+
+  @Test
+  def testInvokeSyncCallback(): Unit = {
+    group.add(member)
+    member.awaitingSyncCallback = _ => {}
+
+    val invoked = group.maybeInvokeSyncCallback(member, SyncGroupResult(Array.empty, Errors.NONE))
+    assertTrue(invoked)
+    assertFalse(member.isAwaitingSync)
+  }
+
+  @Test
+  def testNotInvokeSyncCallback(): Unit = {
+    group.add(member)
+
+    val invoked = group.maybeInvokeSyncCallback(member, SyncGroupResult(Array.empty, Errors.NONE))
+    assertFalse(invoked)
+    assertFalse(member.isAwaitingSync)
+  }
+
   private def assertState(group: GroupMetadata, targetState: GroupState) {
     val states: Set[GroupState] = Set(Stable, PreparingRebalance, CompletingRebalance, Dead)
     val otherStates = states - targetState
diff --git a/tests/kafkatest/tests/client/consumer_test.py b/tests/kafkatest/tests/client/consumer_test.py
index be15e6a9706c6..131123f55ff3c 100644
--- a/tests/kafkatest/tests/client/consumer_test.py
+++ b/tests/kafkatest/tests/client/consumer_test.py
@@ -265,6 +265,12 @@ def test_fencing_static_consumer(self, num_conflict_consumers, fencing_stage):
                                "normal consumer group and %d from conflict consumer group" % \
                                (len(consumer.nodes), len(consumer.joined_nodes()), len(conflict_consumer.joined_nodes()))
                        )
+            wait_until(lambda: len(consumer.dead_nodes()) + len(conflict_consumer.dead_nodes()) == len(conflict_consumer.nodes),
+                       timeout_sec=self.session_timeout_sec,
+                       err_msg="Timed out waiting for fenced consumers to die, expected total %d dead, but only see %d dead in"
+                               "normal consumer group and %d dead in conflict consumer group" % \
+                               (len(conflict_consumer.nodes), len(consumer.dead_nodes()), len(conflict_consumer.dead_nodes()))
+                       )
 
     @cluster(num_nodes=7)
     @matrix(clean_shutdown=[True], enable_autocommit=[True, False])

From 1cc0b5eb81b838c4ffb6ff2c2d3f16b7b3754a01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Armando=20Garc=C3=ADa=20Sancio?=
 
Date: Wed, 12 Jun 2019 09:38:21 -0700
Subject: [PATCH 0359/1071] MINOR: Seal the HostedPartition enumeration (#6917)

Makes HostedPartition a sealed trait and make all of the match cases explicit.

Reviewers: Vikas Singh , Jason Gustafson 
---
 .../main/scala/kafka/cluster/Partition.scala  |  2 +-
 .../kafka/server/DelayedDeleteRecords.scala   |  5 +-
 .../scala/kafka/server/DelayedProduce.scala   |  5 +-
 .../scala/kafka/server/ReplicaManager.scala   | 52 ++++++++++---------
 .../group/GroupCoordinatorTest.scala          |  6 +--
 .../group/GroupMetadataManagerTest.scala      | 22 ++++----
 6 files changed, 46 insertions(+), 46 deletions(-)

diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala
index 22d9dffccd106..251c57021c3db 100755
--- a/core/src/main/scala/kafka/cluster/Partition.scala
+++ b/core/src/main/scala/kafka/cluster/Partition.scala
@@ -165,7 +165,7 @@ class Partition(val topicPartition: TopicPartition,
                 private val stateStore: PartitionStateStore,
                 private val delayedOperations: DelayedOperations,
                 private val metadataCache: MetadataCache,
-                private val logManager: LogManager) extends HostedPartition with Logging with KafkaMetricsGroup {
+                private val logManager: LogManager) extends Logging with KafkaMetricsGroup {
 
   def topic: String = topicPartition.topic
   def partitionId: Int = topicPartition.partition
diff --git a/core/src/main/scala/kafka/server/DelayedDeleteRecords.scala b/core/src/main/scala/kafka/server/DelayedDeleteRecords.scala
index dac9f7927ec77..236c8d103fd17 100644
--- a/core/src/main/scala/kafka/server/DelayedDeleteRecords.scala
+++ b/core/src/main/scala/kafka/server/DelayedDeleteRecords.scala
@@ -20,10 +20,9 @@ package kafka.server
 
 import java.util.concurrent.TimeUnit
 
-import kafka.cluster.Partition
 import kafka.metrics.KafkaMetricsGroup
-import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.requests.DeleteRecordsResponse
 
 import scala.collection._
@@ -74,7 +73,7 @@ class DelayedDeleteRecords(delayMs: Long,
       // skip those partitions that have already been satisfied
       if (status.acksPending) {
         val (lowWatermarkReached, error, lw) = replicaManager.getPartition(topicPartition) match {
-          case partition: Partition =>
+          case HostedPartition.Online(partition) =>
             partition.leaderReplicaIfLocal match {
               case Some(_) =>
                 val leaderLW = partition.lowWatermarkIfLeader
diff --git a/core/src/main/scala/kafka/server/DelayedProduce.scala b/core/src/main/scala/kafka/server/DelayedProduce.scala
index 1570d4baf2827..f1d1407ff8bec 100644
--- a/core/src/main/scala/kafka/server/DelayedProduce.scala
+++ b/core/src/main/scala/kafka/server/DelayedProduce.scala
@@ -22,11 +22,10 @@ import java.util.concurrent.TimeUnit
 import java.util.concurrent.locks.Lock
 
 import com.yammer.metrics.core.Meter
-import kafka.cluster.Partition
 import kafka.metrics.KafkaMetricsGroup
 import kafka.utils.Pool
-import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse
 
 import scala.collection._
@@ -88,7 +87,7 @@ class DelayedProduce(delayMs: Long,
       // skip those partitions that have already been satisfied
       if (status.acksPending) {
         val (hasEnough, error) = replicaManager.getPartition(topicPartition) match {
-          case partition: Partition  =>
+          case HostedPartition.Online(partition) =>
             partition.checkEnoughReplicasReachOffset(status.requiredOffset)
 
           case HostedPartition.Offline =>
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index 64b6beed4edea..7ded9854dbbb6 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -114,21 +114,24 @@ case class FetchPartitionData(error: Errors = Errors.NONE,
  * instance when the broker receives a LeaderAndIsr request from the controller indicating
  * that it should be either a leader or follower of a partition.
  */
-trait HostedPartition
-
+sealed trait HostedPartition
 object HostedPartition {
   /**
    * This broker does not have any state for this partition locally.
    */
-  object None extends HostedPartition
+  final object None extends HostedPartition
+
+  /**
+   * This broker hosts the partition and it is online.
+   */
+  final case class Online(partition: Partition) extends HostedPartition
 
   /**
    * This broker hosts the partition, but it is in an offline log directory.
    */
-  object Offline extends HostedPartition
+  final object Offline extends HostedPartition
 }
 
-
 object ReplicaManager {
   val HighWatermarkFilename = "replication-offset-checkpoint"
   val IsrChangePropagationBlackOut = 5000L
@@ -183,8 +186,9 @@ class ReplicaManager(val config: KafkaConfig,
   /* epoch of the controller that last changed the leader */
   @volatile var controllerEpoch: Int = KafkaController.InitialControllerEpoch
   private val localBrokerId = config.brokerId
-  private val allPartitions = new Pool[TopicPartition, HostedPartition](valueFactory = Some(tp =>
-    Partition(tp, time, this)))
+  private val allPartitions = new Pool[TopicPartition, HostedPartition](
+    valueFactory = Some(tp => HostedPartition.Online(Partition(tp, time, this)))
+  )
   private val replicaStateChangeLock = new Object
   val replicaFetcherManager = createReplicaFetcherManager(metrics, time, threadNamePrefix, quotaManagers.follower)
   val replicaAlterLogDirsManager = createReplicaAlterLogDirsManager(quotaManagers.alterLogDirs, brokerTopicStats)
@@ -320,8 +324,8 @@ class ReplicaManager(val config: KafkaConfig,
 
   private def maybeRemoveTopicMetrics(topic: String): Unit = {
     val topicHasOnlinePartition = allPartitions.values.exists {
-      case partition: Partition => topic == partition.topic
-      case _ => false
+      case HostedPartition.Online(partition) => topic == partition.topic
+      case HostedPartition.None | HostedPartition.Offline => false
     }
     if (!topicHasOnlinePartition)
       brokerTopicStats.removeMetrics(topic)
@@ -335,8 +339,8 @@ class ReplicaManager(val config: KafkaConfig,
         case HostedPartition.Offline =>
           throw new KafkaStorageException(s"Partition $topicPartition is on an offline disk")
 
-        case removedPartition: Partition =>
-          if (allPartitions.remove(topicPartition, removedPartition)) {
+        case hostedPartition @ HostedPartition.Online(removedPartition) =>
+          if (allPartitions.remove(topicPartition, hostedPartition)) {
             maybeRemoveTopicMetrics(topicPartition.topic)
             // this will delete the local log. This call may throw exception if the log is on offline directory
             removedPartition.delete()
@@ -393,14 +397,14 @@ class ReplicaManager(val config: KafkaConfig,
   // Visible for testing
   def createPartition(topicPartition: TopicPartition): Partition = {
     val partition = Partition(topicPartition, time, this)
-    allPartitions.put(topicPartition, partition)
+    allPartitions.put(topicPartition, HostedPartition.Online(partition))
     partition
   }
 
   def nonOfflinePartition(topicPartition: TopicPartition): Option[Partition] = {
     getPartition(topicPartition) match {
-      case partition: Partition => Some(partition)
-      case _ => None
+      case HostedPartition.Online(partition) => Some(partition)
+      case HostedPartition.None | HostedPartition.Offline => None
     }
   }
 
@@ -408,8 +412,8 @@ class ReplicaManager(val config: KafkaConfig,
   // the iterator has been constructed could still be returned by this iterator.
   private def nonOfflinePartitionsIterator: Iterator[Partition] = {
     allPartitions.values.iterator.flatMap {
-      case p: Partition => Some(p)
-      case _ => None
+      case HostedPartition.Online(partition) => Some(partition)
+      case HostedPartition.None | HostedPartition.Offline => None
     }
   }
 
@@ -419,7 +423,7 @@ class ReplicaManager(val config: KafkaConfig,
 
   def getPartitionOrException(topicPartition: TopicPartition, expectLeader: Boolean): Partition = {
     getPartition(topicPartition) match {
-      case partition: Partition =>
+      case HostedPartition.Online(partition) =>
         partition
 
       case HostedPartition.Offline =>
@@ -577,7 +581,7 @@ class ReplicaManager(val config: KafkaConfig,
             throw new KafkaStorageException(s"Log directory $destinationDir is offline")
 
           getPartition(topicPartition) match {
-            case partition: Partition =>
+            case HostedPartition.Online(partition) =>
               // Stop current replica movement if the destinationDir is different from the existing destination log directory
               if (partition.futureReplicaDirChanged(destinationDir)) {
                 replicaAlterLogDirsManager.removeFetcherForPartitions(Set(topicPartition))
@@ -586,7 +590,7 @@ class ReplicaManager(val config: KafkaConfig,
             case HostedPartition.Offline =>
               throw new KafkaStorageException(s"Partition $topicPartition is offline")
 
-            case _ => // Do nothing
+            case HostedPartition.None => // Do nothing
           }
 
           // If the log for this partition has not been created yet:
@@ -776,8 +780,8 @@ class ReplicaManager(val config: KafkaConfig,
             (topicPartition, LogAppendResult(LogAppendInfo.UnknownLogAppendInfo, Some(e)))
           case t: Throwable =>
             val logStartOffset = getPartition(topicPartition) match {
-              case partition: Partition => partition.logStartOffset
-              case _ => -1L
+              case HostedPartition.Online(partition) => partition.logStartOffset
+              case HostedPartition.None | HostedPartition.Offline => -1L
             }
             brokerTopicStats.topicStats(topicPartition.topic).failedProduceRequestRate.mark()
             brokerTopicStats.allTopicsStats.failedProduceRequestRate.mark()
@@ -1064,11 +1068,11 @@ class ReplicaManager(val config: KafkaConfig,
               responseMap.put(topicPartition, Errors.KAFKA_STORAGE_ERROR)
               None
 
-            case partition: Partition => Some(partition)
+            case HostedPartition.Online(partition) => Some(partition)
 
             case HostedPartition.None =>
               val partition = Partition(topicPartition, time, this)
-              allPartitions.putIfNotExists(topicPartition, partition)
+              allPartitions.putIfNotExists(topicPartition, HostedPartition.Online(partition))
               newPartitions.add(partition)
               Some(partition)
           }
@@ -1534,7 +1538,7 @@ class ReplicaManager(val config: KafkaConfig,
   def lastOffsetForLeaderEpoch(requestedEpochInfo: Map[TopicPartition, OffsetsForLeaderEpochRequest.PartitionData]): Map[TopicPartition, EpochEndOffset] = {
     requestedEpochInfo.map { case (tp, partitionData) =>
       val epochEndOffset = getPartition(tp) match {
-        case partition: Partition =>
+        case HostedPartition.Online(partition) =>
           partition.lastOffsetForLeaderEpoch(partitionData.currentLeaderEpoch, partitionData.leaderEpoch,
             fetchOnlyFromLeader = true)
 
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
index c12e19d1f5c84..d5d33fb7aac5a 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
@@ -2017,7 +2017,7 @@ class GroupCoordinatorTest {
 
     EasyMock.reset(replicaManager)
     EasyMock.expect(replicaManager.getMagic(EasyMock.anyObject())).andStubReturn(Some(RecordBatch.CURRENT_MAGIC_VALUE))
-    EasyMock.expect(replicaManager.getPartition(groupTopicPartition)).andStubReturn(partition)
+    EasyMock.expect(replicaManager.getPartition(groupTopicPartition)).andStubReturn(HostedPartition.Online(partition))
     EasyMock.expect(replicaManager.nonOfflinePartition(groupTopicPartition)).andStubReturn(Some(partition))
     EasyMock.replay(replicaManager, partition)
 
@@ -2558,7 +2558,7 @@ class GroupCoordinatorTest {
 
     EasyMock.reset(replicaManager)
     EasyMock.expect(replicaManager.getMagic(EasyMock.anyObject())).andStubReturn(Some(RecordBatch.CURRENT_MAGIC_VALUE))
-    EasyMock.expect(replicaManager.getPartition(groupTopicPartition)).andStubReturn(partition)
+    EasyMock.expect(replicaManager.getPartition(groupTopicPartition)).andStubReturn(HostedPartition.Online(partition))
     EasyMock.expect(replicaManager.nonOfflinePartition(groupTopicPartition)).andStubReturn(Some(partition))
     EasyMock.replay(replicaManager, partition)
 
@@ -2599,7 +2599,7 @@ class GroupCoordinatorTest {
 
     EasyMock.reset(replicaManager)
     EasyMock.expect(replicaManager.getMagic(EasyMock.anyObject())).andStubReturn(Some(RecordBatch.CURRENT_MAGIC_VALUE))
-    EasyMock.expect(replicaManager.getPartition(groupTopicPartition)).andStubReturn(partition)
+    EasyMock.expect(replicaManager.getPartition(groupTopicPartition)).andStubReturn(HostedPartition.Online(partition))
     EasyMock.expect(replicaManager.nonOfflinePartition(groupTopicPartition)).andStubReturn(Some(partition))
     EasyMock.replay(replicaManager, partition)
 
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
index 0487178c474bd..f9b571e68c2a8 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
@@ -17,15 +17,24 @@
 
 package kafka.coordinator.group
 
+import com.yammer.metrics.Metrics
+import com.yammer.metrics.core.Gauge
+import java.nio.ByteBuffer
+import java.util.Collections
+import java.util.Optional
+import java.util.concurrent.locks.ReentrantLock
 import kafka.api._
 import kafka.cluster.Partition
 import kafka.common.OffsetAndMetadata
 import kafka.log.{Log, LogAppendInfo}
 import kafka.server.{FetchDataInfo, KafkaConfig, LogOffsetMetadata, ReplicaManager}
+import kafka.server.HostedPartition
 import kafka.utils.{KafkaScheduler, MockTime, TestUtils}
+import kafka.zk.KafkaZkClient
 import org.apache.kafka.clients.consumer.internals.ConsumerProtocol
 import org.apache.kafka.clients.consumer.internals.PartitionAssignor.Subscription
 import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.internals.Topic
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.record._
 import org.apache.kafka.common.requests.OffsetFetchResponse
@@ -34,19 +43,8 @@ import org.easymock.{Capture, EasyMock, IAnswer}
 import org.junit.Assert.{assertEquals, assertFalse, assertNull, assertTrue}
 import org.junit.{Before, Test}
 import org.scalatest.Assertions.fail
-import java.nio.ByteBuffer
-import java.util.Collections
-import java.util.Optional
-
-import com.yammer.metrics.Metrics
-import com.yammer.metrics.core.Gauge
-import org.apache.kafka.common.internals.Topic
-
 import scala.collection.JavaConverters._
 import scala.collection._
-import java.util.concurrent.locks.ReentrantLock
-
-import kafka.zk.KafkaZkClient
 
 class GroupMetadataManagerTest {
 
@@ -2025,7 +2023,7 @@ class GroupMetadataManagerTest {
   }
 
   private def mockGetPartition(): Unit = {
-    EasyMock.expect(replicaManager.getPartition(groupTopicPartition)).andStubReturn(partition)
+    EasyMock.expect(replicaManager.getPartition(groupTopicPartition)).andStubReturn(HostedPartition.Online(partition))
     EasyMock.expect(replicaManager.nonOfflinePartition(groupTopicPartition)).andStubReturn(Some(partition))
   }
 

From af2801031c455f86fb771d9f36c313f99459cd1a Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Wed, 12 Jun 2019 12:54:17 -0700
Subject: [PATCH 0360/1071] KAFKA-8483/KAFKA-8484; Ensure safe handling of
 producerId resets (#6883)

The idempotent producer attempts to detect spurious UNKNOWN_PRODUCER_ID errors and handle them by reassigning sequence numbers to the inflight batches. The inflight batches are tracked in a PriorityQueue. The problem is that the reassignment of sequence numbers depends on the iteration order of PriorityQueue, which does not guarantee any ordering. So this can result in sequence numbers being assigned in the wrong order.  This patch fixes the problem by using a sorted set instead of a priority queue so that the iteration order preserves the sequence order. Note that resetting sequence numbers is an exceptional case.

This patch also fixes KAFKA-8484, which can cause an IllegalStateException when the producerId is reset while there are pending produce requests inflight. The solution is to ensure that sequence numbers are only reset if the producerId of a failed batch corresponds to the current producerId.

Reviewers: Guozhang Wang 
---
 .../producer/internals/RecordAccumulator.java |   2 +-
 .../clients/producer/internals/Sender.java    |  53 +---
 .../internals/TransactionManager.java         | 152 ++++++---
 .../internals/TransactionManagerTest.java     | 288 ++++++++++++++++--
 4 files changed, 399 insertions(+), 96 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
index e6b29f3c11ca4..91fb8c92c2e0a 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
@@ -558,7 +558,7 @@ private List drainBatchesForOneNode(Cluster cluster, Node node, i
                     if (shouldStopDrainBatchesForPartition(first, tp))
                         break;
 
-                    boolean isTransactional = transactionManager != null ? transactionManager.isTransactional() : false;
+                    boolean isTransactional = transactionManager != null && transactionManager.isTransactional();
                     ProducerIdAndEpoch producerIdAndEpoch =
                         transactionManager != null ? transactionManager.producerIdAndEpoch() : null;
                     ProducerBatch batch = deque.pollFirst();
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java
index cd1712aedc240..121ddb25595fa 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.clients.producer.internals;
 
-import java.util.ArrayList;
 import org.apache.kafka.clients.ApiVersions;
 import org.apache.kafka.clients.ClientRequest;
 import org.apache.kafka.clients.ClientResponse;
@@ -33,11 +32,9 @@
 import org.apache.kafka.common.errors.ClusterAuthorizationException;
 import org.apache.kafka.common.errors.InvalidMetadataException;
 import org.apache.kafka.common.errors.OutOfOrderSequenceException;
-import org.apache.kafka.common.errors.ProducerFencedException;
 import org.apache.kafka.common.errors.RetriableException;
 import org.apache.kafka.common.errors.TimeoutException;
 import org.apache.kafka.common.errors.TopicAuthorizationException;
-import org.apache.kafka.common.errors.TransactionalIdAuthorizationException;
 import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
 import org.apache.kafka.common.errors.UnsupportedVersionException;
 import org.apache.kafka.common.message.InitProducerIdRequestData;
@@ -60,6 +57,7 @@
 import org.slf4j.Logger;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -295,9 +293,7 @@ public void run() {
     void runOnce() {
         if (transactionManager != null) {
             try {
-                if (transactionManager.shouldResetProducerStateAfterResolvingSequences())
-                    // Check if the previous run expired batches which requires a reset of the producer state.
-                    transactionManager.resetProducerId();
+                transactionManager.resetProducerIdIfNeeded();
 
                 if (!transactionManager.isTransactional()) {
                     // this is an idempotent producer, so make sure we have a producer id
@@ -694,16 +690,7 @@ private void reenqueueBatch(ProducerBatch batch, long currentTimeMs) {
 
     private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionResponse response) {
         if (transactionManager != null) {
-            if (transactionManager.hasProducerIdAndEpoch(batch.producerId(), batch.producerEpoch())) {
-                transactionManager
-                    .maybeUpdateLastAckedSequence(batch.topicPartition, batch.baseSequence() + batch.recordCount - 1);
-                log.debug("ProducerId: {}; Set last ack'd sequence number for topic-partition {} to {}",
-                    batch.producerId(),
-                    batch.topicPartition,
-                    transactionManager.lastAckedSequence(batch.topicPartition).orElse(-1));
-            }
-            transactionManager.updateLastAckedOffset(response, batch);
-            transactionManager.removeInFlightBatch(batch);
+            transactionManager.handleCompletedBatch(batch, response);
         }
 
         if (batch.done(response.baseOffset, response.logAppendTime, null)) {
@@ -712,36 +699,20 @@ private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionRespons
         }
     }
 
-    private void failBatch(ProducerBatch batch, ProduceResponse.PartitionResponse response, RuntimeException exception,
+    private void failBatch(ProducerBatch batch,
+                           ProduceResponse.PartitionResponse response,
+                           RuntimeException exception,
                            boolean adjustSequenceNumbers) {
         failBatch(batch, response.baseOffset, response.logAppendTime, exception, adjustSequenceNumbers);
     }
 
-    private void failBatch(ProducerBatch batch, long baseOffset, long logAppendTime, RuntimeException exception,
-        boolean adjustSequenceNumbers) {
+    private void failBatch(ProducerBatch batch,
+                           long baseOffset,
+                           long logAppendTime,
+                           RuntimeException exception,
+                           boolean adjustSequenceNumbers) {
         if (transactionManager != null) {
-            if (exception instanceof OutOfOrderSequenceException
-                    && !transactionManager.isTransactional()
-                    && transactionManager.hasProducerId(batch.producerId())) {
-                log.error("The broker returned {} for topic-partition " +
-                            "{} at offset {}. This indicates data loss on the broker, and should be investigated.",
-                        exception, batch.topicPartition, baseOffset);
-
-                // Reset the transaction state since we have hit an irrecoverable exception and cannot make any guarantees
-                // about the previously committed message. Note that this will discard the producer id and sequence
-                // numbers for all existing partitions.
-                transactionManager.resetProducerId();
-            } else if (exception instanceof ClusterAuthorizationException
-                    || exception instanceof TransactionalIdAuthorizationException
-                    || exception instanceof ProducerFencedException
-                    || exception instanceof UnsupportedVersionException) {
-                transactionManager.transitionToFatalError(exception);
-            } else if (transactionManager.isTransactional()) {
-                transactionManager.transitionToAbortableError(exception);
-            }
-            transactionManager.removeInFlightBatch(batch);
-            if (adjustSequenceNumbers)
-                transactionManager.adjustSequencesDueToFailedBatch(batch);
+            transactionManager.handleFailedBatch(batch, exception, adjustSequenceNumbers);
         }
 
         this.sensors.recordErrors(batch.topicPartition.topic(), batch.recordCount);
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java
index 9ed0dde416b3a..182c92c900175 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java
@@ -23,9 +23,13 @@
 import org.apache.kafka.common.Node;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.errors.AuthenticationException;
+import org.apache.kafka.common.errors.ClusterAuthorizationException;
 import org.apache.kafka.common.errors.GroupAuthorizationException;
+import org.apache.kafka.common.errors.OutOfOrderSequenceException;
 import org.apache.kafka.common.errors.ProducerFencedException;
 import org.apache.kafka.common.errors.TopicAuthorizationException;
+import org.apache.kafka.common.errors.TransactionalIdAuthorizationException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
 import org.apache.kafka.common.message.FindCoordinatorRequestData;
 import org.apache.kafka.common.message.InitProducerIdRequestData;
 import org.apache.kafka.common.protocol.Errors;
@@ -62,6 +66,10 @@
 import java.util.OptionalLong;
 import java.util.PriorityQueue;
 import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
 import java.util.function.Supplier;
 
 import static org.apache.kafka.common.record.RecordBatch.NO_PRODUCER_EPOCH;
@@ -133,7 +141,7 @@ private static class TopicPartitionEntry {
         // we continue to order batches by the sequence numbers even when the responses come back out of order during
         // leader failover. We add a batch to the queue when it is drained, and remove it when the batch completes
         // (either successfully or through a fatal failure).
-        private PriorityQueue inflightBatchesBySequence;
+        private SortedSet inflightBatchesBySequence;
 
         // We keep track of the last acknowledged offset on a per partition basis in order to disambiguate UnknownProducer
         // responses which are due to the retention period elapsing, and those which are due to actual lost data.
@@ -143,8 +151,18 @@ private static class TopicPartitionEntry {
             this.nextSequence = 0;
             this.lastAckedSequence = NO_LAST_ACKED_SEQUENCE_NUMBER;
             this.lastAckedOffset = ProduceResponse.INVALID_OFFSET;
-            this.inflightBatchesBySequence = new PriorityQueue<>(5, Comparator.comparingInt(ProducerBatch::baseSequence));
+            this.inflightBatchesBySequence = new TreeSet<>(Comparator.comparingInt(ProducerBatch::baseSequence));
         }
+
+        public void resetSequenceNumbers(Consumer resetSequence) {
+            TreeSet newInflights = new TreeSet<>(Comparator.comparingInt(ProducerBatch::baseSequence));
+            for (ProducerBatch inflightBatch : inflightBatchesBySequence) {
+                resetSequence.accept(inflightBatch);
+                newInflights.add(inflightBatch);
+            }
+            inflightBatchesBySequence = newInflights;
+        }
+
     }
 
     private final TopicPartitionBookkeeper topicPartitionBookkeeper;
@@ -459,6 +477,12 @@ synchronized void resetProducerId() {
         this.partitionsWithUnresolvedSequences.clear();
     }
 
+    synchronized void resetProducerIdIfNeeded() {
+        if (shouldResetProducerStateAfterResolvingSequences())
+            // Check if the previous run expired batches which requires a reset of the producer state.
+            resetProducerId();
+    }
+
     /**
      * Returns the next sequence number to be written to the given TopicPartition.
      */
@@ -479,7 +503,7 @@ synchronized void incrementSequenceNumber(TopicPartition topicPartition, int inc
     synchronized void addInFlightBatch(ProducerBatch batch) {
         if (!batch.hasSequence())
             throw new IllegalStateException("Can't track batch for partition " + batch.topicPartition + " when sequence is not set.");
-        topicPartitionBookkeeper.getPartition(batch.topicPartition).inflightBatchesBySequence.offer(batch);
+        topicPartitionBookkeeper.getPartition(batch.topicPartition).inflightBatchesBySequence.add(batch);
     }
 
     /**
@@ -493,26 +517,25 @@ synchronized int firstInFlightSequence(TopicPartition topicPartition) {
         if (!hasInflightBatches(topicPartition))
             return RecordBatch.NO_SEQUENCE;
 
-        ProducerBatch first = topicPartitionBookkeeper.getPartition(topicPartition).inflightBatchesBySequence.peek();
-        if (first == null)
+        SortedSet inflightBatches = topicPartitionBookkeeper.getPartition(topicPartition).inflightBatchesBySequence;
+        if (inflightBatches.isEmpty())
             return RecordBatch.NO_SEQUENCE;
-
-        return first.baseSequence();
+        else
+            return inflightBatches.first().baseSequence();
     }
 
     synchronized ProducerBatch nextBatchBySequence(TopicPartition topicPartition) {
-        PriorityQueue queue = topicPartitionBookkeeper.getPartition(topicPartition).inflightBatchesBySequence;
-        return queue.peek();
+        SortedSet queue = topicPartitionBookkeeper.getPartition(topicPartition).inflightBatchesBySequence;
+        return queue.isEmpty() ? null : queue.first();
     }
 
     synchronized void removeInFlightBatch(ProducerBatch batch) {
         if (hasInflightBatches(batch.topicPartition)) {
-            PriorityQueue queue = topicPartitionBookkeeper.getPartition(batch.topicPartition).inflightBatchesBySequence;
-            queue.remove(batch);
+            topicPartitionBookkeeper.getPartition(batch.topicPartition).inflightBatchesBySequence.remove(batch);
         }
     }
 
-    synchronized void maybeUpdateLastAckedSequence(TopicPartition topicPartition, int sequence) {
+    private void maybeUpdateLastAckedSequence(TopicPartition topicPartition, int sequence) {
         if (sequence > lastAckedSequence(topicPartition).orElse(NO_LAST_ACKED_SEQUENCE_NUMBER))
             topicPartitionBookkeeper.getPartition(topicPartition).lastAckedSequence = sequence;
     }
@@ -525,7 +548,7 @@ synchronized OptionalLong lastAckedOffset(TopicPartition topicPartition) {
         return topicPartitionBookkeeper.lastAckedOffset(topicPartition);
     }
 
-    synchronized void updateLastAckedOffset(ProduceResponse.PartitionResponse response, ProducerBatch batch) {
+    private void updateLastAckedOffset(ProduceResponse.PartitionResponse response, ProducerBatch batch) {
         if (response.baseOffset == ProduceResponse.INVALID_OFFSET)
             return;
         long lastOffset = response.baseOffset + batch.recordCount - 1;
@@ -543,12 +566,66 @@ synchronized void updateLastAckedOffset(ProduceResponse.PartitionResponse respon
         }
     }
 
+    public synchronized void handleCompletedBatch(ProducerBatch batch, ProduceResponse.PartitionResponse response) {
+        if (!hasProducerIdAndEpoch(batch.producerId(), batch.producerEpoch())) {
+            log.debug("Ignoring completed batch {} with producer id {}, epoch {}, and sequence number {} " +
+                            "since the producerId has been reset internally", batch, batch.producerId(),
+                    batch.producerEpoch(), batch.baseSequence());
+            return;
+        }
+
+        maybeUpdateLastAckedSequence(batch.topicPartition, batch.baseSequence() + batch.recordCount - 1);
+        log.debug("ProducerId: {}; Set last ack'd sequence number for topic-partition {} to {}",
+                batch.producerId(),
+                batch.topicPartition,
+                lastAckedSequence(batch.topicPartition).orElse(-1));
+
+        updateLastAckedOffset(response, batch);
+        removeInFlightBatch(batch);
+    }
+
+    private void maybeTransitionToErrorState(RuntimeException exception) {
+        if (exception instanceof ClusterAuthorizationException
+                || exception instanceof TransactionalIdAuthorizationException
+                || exception instanceof ProducerFencedException
+                || exception instanceof UnsupportedVersionException) {
+            transitionToFatalError(exception);
+        } else if (isTransactional()) {
+            transitionToAbortableError(exception);
+        }
+    }
+
+    public synchronized void handleFailedBatch(ProducerBatch batch, RuntimeException exception, boolean adjustSequenceNumbers) {
+        maybeTransitionToErrorState(exception);
+
+        if (!hasProducerIdAndEpoch(batch.producerId(), batch.producerEpoch())) {
+            log.debug("Ignoring failed batch {} with producer id {}, epoch {}, and sequence number {} " +
+                    "since the producerId has been reset internally", batch, batch.producerId(),
+                    batch.producerEpoch(), batch.baseSequence(), exception);
+            return;
+        }
+
+        if (exception instanceof OutOfOrderSequenceException && !isTransactional()) {
+            log.error("The broker returned {} for topic-partition {} with producerId {}, epoch {}, and sequence number {}",
+                    exception, batch.topicPartition, batch.producerId(), batch.producerEpoch(), batch.baseSequence());
+
+            // Reset the producerId since we have hit an irrecoverable exception and cannot make any guarantees
+            // about the previously committed message. Note that this will discard the producer id and sequence
+            // numbers for all existing partitions.
+            resetProducerId();
+        } else {
+            removeInFlightBatch(batch);
+            if (adjustSequenceNumbers)
+                adjustSequencesDueToFailedBatch(batch);
+        }
+    }
+
     // If a batch is failed fatally, the sequence numbers for future batches bound for the partition must be adjusted
     // so that they don't fail with the OutOfOrderSequenceException.
     //
     // This method must only be called when we know that the batch is question has been unequivocally failed by the broker,
     // ie. it has received a confirmed fatal status code like 'Message Too Large' or something similar.
-    synchronized void adjustSequencesDueToFailedBatch(ProducerBatch batch) {
+    private void adjustSequencesDueToFailedBatch(ProducerBatch batch) {
         if (!topicPartitionBookkeeper.contains(batch.topicPartition))
             // Sequence numbers are not being tracked for this partition. This could happen if the producer id was just
             // reset due to a previous OutOfOrderSequenceException.
@@ -558,38 +635,39 @@ synchronized void adjustSequencesDueToFailedBatch(ProducerBatch batch) {
         int currentSequence = sequenceNumber(batch.topicPartition);
         currentSequence -= batch.recordCount;
         if (currentSequence < 0)
-            throw new IllegalStateException("Sequence number for partition " + batch.topicPartition + " is going to become negative : " + currentSequence);
+            throw new IllegalStateException("Sequence number for partition " + batch.topicPartition + " is going to become negative: " + currentSequence);
 
         setNextSequence(batch.topicPartition, currentSequence);
 
-        for (ProducerBatch inFlightBatch : topicPartitionBookkeeper.getPartition(batch.topicPartition).inflightBatchesBySequence) {
+        topicPartitionBookkeeper.getPartition(batch.topicPartition).resetSequenceNumbers(inFlightBatch -> {
             if (inFlightBatch.baseSequence() < batch.baseSequence())
-                continue;
+                return;
+
             int newSequence = inFlightBatch.baseSequence() - batch.recordCount;
             if (newSequence < 0)
                 throw new IllegalStateException("Sequence number for batch with sequence " + inFlightBatch.baseSequence()
-                        + " for partition " + batch.topicPartition + " is going to become negative :" + newSequence);
+                        + " for partition " + batch.topicPartition + " is going to become negative: " + newSequence);
 
             log.info("Resetting sequence number of batch with current sequence {} for partition {} to {}", inFlightBatch.baseSequence(), batch.topicPartition, newSequence);
             inFlightBatch.resetProducerState(new ProducerIdAndEpoch(inFlightBatch.producerId(), inFlightBatch.producerEpoch()), newSequence, inFlightBatch.isTransactional());
-        }
+
+        });
     }
 
-    private synchronized void startSequencesAtBeginning(TopicPartition topicPartition) {
-        int sequence = 0;
-        for (ProducerBatch inFlightBatch : topicPartitionBookkeeper.getPartition(topicPartition).inflightBatchesBySequence) {
+    private void startSequencesAtBeginning(TopicPartition topicPartition) {
+        final AtomicInteger sequence = new AtomicInteger(0);
+        topicPartitionBookkeeper.getPartition(topicPartition).resetSequenceNumbers(inFlightBatch -> {
             log.info("Resetting sequence number of batch with current sequence {} for partition {} to {}",
-                    inFlightBatch.baseSequence(), inFlightBatch.topicPartition, sequence);
+                    inFlightBatch.baseSequence(), inFlightBatch.topicPartition, sequence.get());
             inFlightBatch.resetProducerState(new ProducerIdAndEpoch(inFlightBatch.producerId(),
-                    inFlightBatch.producerEpoch()), sequence, inFlightBatch.isTransactional());
-
-            sequence += inFlightBatch.recordCount;
-        }
-        setNextSequence(topicPartition, sequence);
+                    inFlightBatch.producerEpoch()), sequence.get(), inFlightBatch.isTransactional());
+            sequence.getAndAdd(inFlightBatch.recordCount);
+        });
+        setNextSequence(topicPartition, sequence.get());
         topicPartitionBookkeeper.getPartition(topicPartition).lastAckedSequence = NO_LAST_ACKED_SEQUENCE_NUMBER;
     }
 
-    private synchronized boolean hasInflightBatches(TopicPartition topicPartition) {
+    private boolean hasInflightBatches(TopicPartition topicPartition) {
         return topicPartitionBookkeeper.contains(topicPartition)
                 && !topicPartitionBookkeeper.getPartition(topicPartition).inflightBatchesBySequence.isEmpty();
     }
@@ -609,7 +687,7 @@ synchronized void markSequenceUnresolved(TopicPartition topicPartition) {
 
     // Checks if there are any partitions with unresolved partitions which may now be resolved. Returns true if
     // the producer id needs a reset, false otherwise.
-    synchronized boolean shouldResetProducerStateAfterResolvingSequences() {
+    private boolean shouldResetProducerStateAfterResolvingSequences() {
         if (isTransactional())
             // We should not reset producer state if we are transactional. We will transition to a fatal error instead.
             return false;
@@ -634,11 +712,11 @@ synchronized boolean shouldResetProducerStateAfterResolvingSequences() {
         return false;
     }
 
-    private synchronized boolean isNextSequence(TopicPartition topicPartition, int sequence) {
+    private boolean isNextSequence(TopicPartition topicPartition, int sequence) {
         return sequence - lastAckedSequence(topicPartition).orElse(NO_LAST_ACKED_SEQUENCE_NUMBER) == 1;
     }
 
-    private synchronized void setNextSequence(TopicPartition topicPartition, int sequence) {
+    private void setNextSequence(TopicPartition topicPartition, int sequence) {
         topicPartitionBookkeeper.getPartition(topicPartition).nextSequence = sequence;
     }
 
@@ -755,7 +833,7 @@ synchronized boolean hasOngoingTransaction() {
     }
 
     synchronized boolean canRetry(ProduceResponse.PartitionResponse response, ProducerBatch batch) {
-        if (!hasProducerId(batch.producerId()))
+        if (!hasProducerIdAndEpoch(batch.producerId(), batch.producerEpoch()))
             return false;
 
         Errors error = response.error;
@@ -807,7 +885,7 @@ private void transitionTo(State target) {
         transitionTo(target, null);
     }
 
-    private synchronized void transitionTo(State target, RuntimeException error) {
+    private void transitionTo(State target, RuntimeException error) {
         if (!currentState.isTransitionValid(currentState, target)) {
             String idString = transactionalId == null ?  "" : "TransactionalId " + transactionalId + ": ";
             throw new KafkaException(idString + "Invalid transition attempted from state "
@@ -865,7 +943,7 @@ private void enqueueRequest(TxnRequestHandler requestHandler) {
         pendingRequests.add(requestHandler);
     }
 
-    private synchronized void lookupCoordinator(FindCoordinatorRequest.CoordinatorType type, String coordinatorKey) {
+    private void lookupCoordinator(FindCoordinatorRequest.CoordinatorType type, String coordinatorKey) {
         switch (type) {
             case GROUP:
                 consumerGroupCoordinator = null;
@@ -884,7 +962,7 @@ private synchronized void lookupCoordinator(FindCoordinatorRequest.CoordinatorTy
         enqueueRequest(new FindCoordinatorHandler(builder));
     }
 
-    private synchronized void completeTransaction() {
+    private void completeTransaction() {
         transitionTo(State.READY);
         lastError = null;
         transactionStarted = false;
@@ -893,7 +971,7 @@ private synchronized void completeTransaction() {
         partitionsInTransaction.clear();
     }
 
-    private synchronized TxnRequestHandler addPartitionsToTransactionHandler() {
+    private TxnRequestHandler addPartitionsToTransactionHandler() {
         pendingPartitionsInTransaction.addAll(newPartitionsInTransaction);
         newPartitionsInTransaction.clear();
         AddPartitionsToTxnRequest.Builder builder = new AddPartitionsToTxnRequest.Builder(transactionalId,
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java
index eceb8dfca3972..03b13d3fdf05b 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java
@@ -33,6 +33,7 @@
 import org.apache.kafka.common.errors.TransactionalIdAuthorizationException;
 import org.apache.kafka.common.errors.UnsupportedForMessageFormatException;
 import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.header.Header;
 import org.apache.kafka.common.internals.ClusterResourceListeners;
 import org.apache.kafka.common.message.InitProducerIdResponseData;
 import org.apache.kafka.common.metrics.MetricConfig;
@@ -40,9 +41,11 @@
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.record.CompressionType;
 import org.apache.kafka.common.record.MemoryRecords;
+import org.apache.kafka.common.record.MemoryRecordsBuilder;
 import org.apache.kafka.common.record.MutableRecordBatch;
 import org.apache.kafka.common.record.Record;
 import org.apache.kafka.common.record.RecordBatch;
+import org.apache.kafka.common.record.TimestampType;
 import org.apache.kafka.common.requests.AddOffsetsToTxnRequest;
 import org.apache.kafka.common.requests.AddOffsetsToTxnResponse;
 import org.apache.kafka.common.requests.AddPartitionsToTxnRequest;
@@ -65,6 +68,7 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -73,6 +77,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.OptionalInt;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
@@ -85,8 +90,8 @@
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 public class TransactionManagerTest {
@@ -581,6 +586,164 @@ public void testDefaultSequenceNumber() {
         assertEquals((int) transactionManager.sequenceNumber(tp0), 3);
     }
 
+    @Test
+    public void testResetSequenceNumbersAfterUnknownProducerId() {
+        final long producerId = 13131L;
+        final short epoch = 1;
+        ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(producerId, epoch);
+
+        TransactionManager transactionManager = new TransactionManager();
+        transactionManager.setProducerIdAndEpoch(producerIdAndEpoch);
+
+        ProducerBatch b1 = writeIdempotentBatchWithValue(transactionManager, tp0, "1");
+        ProducerBatch b2 = writeIdempotentBatchWithValue(transactionManager, tp0, "2");
+        ProducerBatch b3 = writeIdempotentBatchWithValue(transactionManager, tp0, "3");
+        ProducerBatch b4 = writeIdempotentBatchWithValue(transactionManager, tp0, "4");
+        ProducerBatch b5 = writeIdempotentBatchWithValue(transactionManager, tp0, "5");
+        assertEquals(5, transactionManager.sequenceNumber(tp0).intValue());
+
+        // First batch succeeds
+        long b1AppendTime = time.milliseconds();
+        ProduceResponse.PartitionResponse b1Response = new ProduceResponse.PartitionResponse(
+                Errors.NONE, 500L, b1AppendTime, 0L);
+        b1.done(500L, b1AppendTime, null);
+        transactionManager.handleCompletedBatch(b1, b1Response);
+
+        // Retention caused log start offset to jump forward. We set sequence numbers back to 0
+        ProduceResponse.PartitionResponse b2Response = new ProduceResponse.PartitionResponse(
+                Errors.UNKNOWN_PRODUCER_ID, -1, -1, 600L);
+        assertTrue(transactionManager.canRetry(b2Response, b2));
+        assertEquals(4, transactionManager.sequenceNumber(tp0).intValue());
+        assertEquals(0, b2.baseSequence());
+        assertEquals(1, b3.baseSequence());
+        assertEquals(2, b4.baseSequence());
+        assertEquals(3, b5.baseSequence());
+    }
+
+    @Test
+    public void testAdjustSequenceNumbersAfterFatalError() {
+        final long producerId = 13131L;
+        final short epoch = 1;
+        ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(producerId, epoch);
+
+        TransactionManager transactionManager = new TransactionManager();
+        transactionManager.setProducerIdAndEpoch(producerIdAndEpoch);
+
+        ProducerBatch b1 = writeIdempotentBatchWithValue(transactionManager, tp0, "1");
+        ProducerBatch b2 = writeIdempotentBatchWithValue(transactionManager, tp0, "2");
+        ProducerBatch b3 = writeIdempotentBatchWithValue(transactionManager, tp0, "3");
+        ProducerBatch b4 = writeIdempotentBatchWithValue(transactionManager, tp0, "4");
+        ProducerBatch b5 = writeIdempotentBatchWithValue(transactionManager, tp0, "5");
+        assertEquals(5, transactionManager.sequenceNumber(tp0).intValue());
+
+        // First batch succeeds
+        long b1AppendTime = time.milliseconds();
+        ProduceResponse.PartitionResponse b1Response = new ProduceResponse.PartitionResponse(
+                Errors.NONE, 500L, b1AppendTime, 0L);
+        b1.done(500L, b1AppendTime, null);
+        transactionManager.handleCompletedBatch(b1, b1Response);
+
+        // Second batch fails with a fatal error. Sequence numbers are adjusted by one for remaining
+        // inflight batches.
+        ProduceResponse.PartitionResponse b2Response = new ProduceResponse.PartitionResponse(
+                Errors.MESSAGE_TOO_LARGE, -1, -1, 0L);
+        assertFalse(transactionManager.canRetry(b2Response, b2));
+
+        b2.done(-1L, -1L, Errors.MESSAGE_TOO_LARGE.exception());
+        transactionManager.handleFailedBatch(b2, Errors.MESSAGE_TOO_LARGE.exception(), true);
+        assertEquals(4, transactionManager.sequenceNumber(tp0).intValue());
+        assertEquals(1, b3.baseSequence());
+        assertEquals(2, b4.baseSequence());
+        assertEquals(3, b5.baseSequence());
+
+        // The remaining batches are doomed to fail, but they can be retried. Expected
+        // sequence numbers should remain the same.
+        ProduceResponse.PartitionResponse b3Response = new ProduceResponse.PartitionResponse(
+                Errors.OUT_OF_ORDER_SEQUENCE_NUMBER, -1, -1, 0L);
+        assertTrue(transactionManager.canRetry(b3Response, b3));
+        assertEquals(4, transactionManager.sequenceNumber(tp0).intValue());
+        assertEquals(1, b3.baseSequence());
+        assertEquals(2, b4.baseSequence());
+        assertEquals(3, b5.baseSequence());
+    }
+
+    @Test
+    public void testBatchFailureAfterProducerReset() {
+        // This tests a scenario where the producerId is reset while pending requests are still inflight.
+        // The returned responses should not update internal state.
+
+        final long producerId = 13131L;
+        final short epoch = 1;
+        ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(producerId, epoch);
+        TransactionManager transactionManager = new TransactionManager();
+        transactionManager.setProducerIdAndEpoch(producerIdAndEpoch);
+
+        ProducerBatch b1 = writeIdempotentBatchWithValue(transactionManager, tp0, "1");
+
+        ProducerIdAndEpoch updatedProducerIdAndEpoch = new ProducerIdAndEpoch(producerId + 1, epoch);
+        transactionManager.resetProducerId();
+        transactionManager.setProducerIdAndEpoch(updatedProducerIdAndEpoch);
+
+        ProducerBatch b2 = writeIdempotentBatchWithValue(transactionManager, tp0, "2");
+        assertEquals(1, transactionManager.sequenceNumber(tp0).intValue());
+
+        ProduceResponse.PartitionResponse b1Response = new ProduceResponse.PartitionResponse(
+                Errors.UNKNOWN_PRODUCER_ID, -1, -1, 400L);
+        assertFalse(transactionManager.canRetry(b1Response, b1));
+        transactionManager.handleFailedBatch(b1, Errors.UNKNOWN_PRODUCER_ID.exception(), true);
+
+        assertEquals(1, transactionManager.sequenceNumber(tp0).intValue());
+        assertEquals(b2, transactionManager.nextBatchBySequence(tp0));
+    }
+
+    @Test
+    public void testBatchCompletedAfterProducerReset() {
+        final long producerId = 13131L;
+        final short epoch = 1;
+        ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(producerId, epoch);
+        TransactionManager transactionManager = new TransactionManager();
+        transactionManager.setProducerIdAndEpoch(producerIdAndEpoch);
+
+        ProducerBatch b1 = writeIdempotentBatchWithValue(transactionManager, tp0, "1");
+
+        // The producerId might be reset due to a failure on another partition
+        ProducerIdAndEpoch updatedProducerIdAndEpoch = new ProducerIdAndEpoch(producerId + 1, epoch);
+        transactionManager.resetProducerId();
+        transactionManager.setProducerIdAndEpoch(updatedProducerIdAndEpoch);
+
+        ProducerBatch b2 = writeIdempotentBatchWithValue(transactionManager, tp0, "2");
+        assertEquals(1, transactionManager.sequenceNumber(tp0).intValue());
+
+        // If the request returns successfully, we should ignore the response and not update any state
+        ProduceResponse.PartitionResponse b1Response = new ProduceResponse.PartitionResponse(
+                Errors.NONE, 500L, time.milliseconds(), 0L);
+        transactionManager.handleCompletedBatch(b1, b1Response);
+
+        assertEquals(1, transactionManager.sequenceNumber(tp0).intValue());
+        assertEquals(b2, transactionManager.nextBatchBySequence(tp0));
+    }
+
+    private ProducerBatch writeIdempotentBatchWithValue(TransactionManager manager,
+                                                        TopicPartition tp,
+                                                        String value) {
+        int seq = manager.sequenceNumber(tp);
+        manager.incrementSequenceNumber(tp, 1);
+        ProducerBatch batch = batchWithValue(tp, value);
+        batch.setProducerState(manager.producerIdAndEpoch(), seq, false);
+        manager.addInFlightBatch(batch);
+        batch.close();
+        return batch;
+    }
+
+    private ProducerBatch batchWithValue(TopicPartition tp, String value) {
+        MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(64),
+                CompressionType.NONE, TimestampType.CREATE_TIME, 0L);
+        long currentTimeMs = time.milliseconds();
+        ProducerBatch batch = new ProducerBatch(tp, builder, currentTimeMs);
+        batch.tryAppend(currentTimeMs, new byte[0], value.getBytes(), new Header[0], null, currentTimeMs);
+        return batch;
+    }
+
     @Test
     public void testSequenceNumberOverflow() {
         TransactionManager transactionManager = new TransactionManager();
@@ -2272,30 +2435,121 @@ public void testTransitionToFatalErrorWhenRetriedBatchIsExpired() throws Interru
     }
 
     @Test
-    public void testShouldResetProducerStateAfterResolvingSequences() {
-        // Create a TransactionManager without a transactionalId to test
-        // shouldResetProducerStateAfterResolvingSequences.
+    public void testResetProducerIdAfterWithoutPendingInflightRequests() {
         TransactionManager manager = new TransactionManager(logContext, null, transactionTimeoutMs,
             DEFAULT_RETRY_BACKOFF_MS);
-        assertFalse(manager.shouldResetProducerStateAfterResolvingSequences());
+        long producerId = 15L;
+        short epoch = 5;
+        ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(producerId, epoch);
+        manager.setProducerIdAndEpoch(producerIdAndEpoch);
+
+        // Nothing to resolve, so no reset is needed
+        manager.resetProducerIdIfNeeded();
+        assertEquals(producerIdAndEpoch, manager.producerIdAndEpoch());
+
         TopicPartition tp0 = new TopicPartition("foo", 0);
-        TopicPartition tp1 = new TopicPartition("foo", 1);
         assertEquals(Integer.valueOf(0), manager.sequenceNumber(tp0));
-        assertEquals(Integer.valueOf(0), manager.sequenceNumber(tp1));
 
-        manager.incrementSequenceNumber(tp0, 1);
-        manager.incrementSequenceNumber(tp1, 1);
-        manager.maybeUpdateLastAckedSequence(tp0, 0);
-        manager.maybeUpdateLastAckedSequence(tp1, 0);
+        ProducerBatch b1 = writeIdempotentBatchWithValue(manager, tp0, "1");
+        assertEquals(Integer.valueOf(1), manager.sequenceNumber(tp0));
+        manager.handleCompletedBatch(b1, new ProduceResponse.PartitionResponse(
+                Errors.NONE, 500L, time.milliseconds(), 0L));
+        assertEquals(OptionalInt.of(0), manager.lastAckedSequence(tp0));
+
+        // Marking sequence numbers unresolved without inflight requests is basically a no-op.
+        manager.markSequenceUnresolved(tp0);
+        manager.resetProducerIdIfNeeded();
+        assertEquals(producerIdAndEpoch, manager.producerIdAndEpoch());
+        assertFalse(manager.hasUnresolvedSequences());
+
+        // We have a new batch which fails with a timeout
+        ProducerBatch b2 = writeIdempotentBatchWithValue(manager, tp0, "2");
+        assertEquals(Integer.valueOf(2), manager.sequenceNumber(tp0));
+        manager.markSequenceUnresolved(tp0);
+        manager.handleFailedBatch(b2, new TimeoutException(), false);
+        assertTrue(manager.hasUnresolvedSequences());
+
+        // We only had one inflight batch, so we should be able to clear the unresolved status
+        // and reset the producerId
+        manager.resetProducerIdIfNeeded();
+        assertFalse(manager.hasUnresolvedSequences());
+        assertFalse(manager.hasProducerId());
+    }
+
+    @Test
+    public void testNoProducerIdResetAfterLastInFlightBatchSucceeds() {
+        TransactionManager manager = new TransactionManager(logContext, null, transactionTimeoutMs,
+                DEFAULT_RETRY_BACKOFF_MS);
+        long producerId = 15L;
+        short epoch = 5;
+        ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(producerId, epoch);
+        manager.setProducerIdAndEpoch(producerIdAndEpoch);
+
+        TopicPartition tp0 = new TopicPartition("foo", 0);
+        ProducerBatch b1 = writeIdempotentBatchWithValue(manager, tp0, "1");
+        ProducerBatch b2 = writeIdempotentBatchWithValue(manager, tp0, "2");
+        ProducerBatch b3 = writeIdempotentBatchWithValue(manager, tp0, "3");
+        assertEquals(3, manager.sequenceNumber(tp0).intValue());
+
+        // The first batch fails with a timeout
         manager.markSequenceUnresolved(tp0);
-        manager.markSequenceUnresolved(tp1);
-        assertFalse(manager.shouldResetProducerStateAfterResolvingSequences());
+        manager.handleFailedBatch(b1, new TimeoutException(), false);
+        assertTrue(manager.hasUnresolvedSequences());
+
+        // The reset should not occur until sequence numbers have been resolved
+        manager.resetProducerIdIfNeeded();
+        assertEquals(producerIdAndEpoch, manager.producerIdAndEpoch());
+        assertTrue(manager.hasUnresolvedSequences());
+
+        // The second batch fails as well with a timeout
+        manager.handleFailedBatch(b2, new TimeoutException(), false);
+        manager.resetProducerIdIfNeeded();
+        assertEquals(producerIdAndEpoch, manager.producerIdAndEpoch());
+        assertTrue(manager.hasUnresolvedSequences());
+
+        // The third batch succeeds, which should resolve the sequence number without
+        // requiring a producerId reset.
+        manager.handleCompletedBatch(b3, new ProduceResponse.PartitionResponse(
+                Errors.NONE, 500L, time.milliseconds(), 0L));
+        manager.resetProducerIdIfNeeded();
+        assertEquals(producerIdAndEpoch, manager.producerIdAndEpoch());
+        assertFalse(manager.hasUnresolvedSequences());
+        assertEquals(3, manager.sequenceNumber(tp0).intValue());
+    }
+
+    @Test
+    public void testProducerIdResetAfterLastInFlightBatchFails() {
+        TransactionManager manager = new TransactionManager(logContext, null, transactionTimeoutMs,
+                DEFAULT_RETRY_BACKOFF_MS);
+        long producerId = 15L;
+        short epoch = 5;
+        ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(producerId, epoch);
+        manager.setProducerIdAndEpoch(producerIdAndEpoch);
+
+        TopicPartition tp0 = new TopicPartition("foo", 0);
+        ProducerBatch b1 = writeIdempotentBatchWithValue(manager, tp0, "1");
+        ProducerBatch b2 = writeIdempotentBatchWithValue(manager, tp0, "2");
+        ProducerBatch b3 = writeIdempotentBatchWithValue(manager, tp0, "3");
+        assertEquals(Integer.valueOf(3), manager.sequenceNumber(tp0));
 
-        manager.maybeUpdateLastAckedSequence(tp0, 5);
-        manager.incrementSequenceNumber(tp0, 1);
+        // The first batch fails with a timeout
         manager.markSequenceUnresolved(tp0);
-        manager.markSequenceUnresolved(tp1);
-        assertTrue(manager.shouldResetProducerStateAfterResolvingSequences());
+        manager.handleFailedBatch(b1, new TimeoutException(), false);
+        assertTrue(manager.hasUnresolvedSequences());
+
+        // The second batch succeeds, but sequence numbers are still not resolved
+        manager.handleCompletedBatch(b2, new ProduceResponse.PartitionResponse(
+                Errors.NONE, 500L, time.milliseconds(), 0L));
+        manager.resetProducerIdIfNeeded();
+        assertEquals(producerIdAndEpoch, manager.producerIdAndEpoch());
+        assertTrue(manager.hasUnresolvedSequences());
+
+        // When the last inflight batch fails, we have to reset the producerId
+        manager.handleFailedBatch(b3, new TimeoutException(), false);
+        manager.resetProducerIdIfNeeded();
+        assertFalse(manager.hasProducerId());
+        assertFalse(manager.hasUnresolvedSequences());
+        assertEquals(0, manager.sequenceNumber(tp0).intValue());
     }
 
     @Test

From df9ea618a397955806fc7f1fba7003492046c77a Mon Sep 17 00:00:00 2001
From: cadonna 
Date: Wed, 12 Jun 2019 23:10:39 +0200
Subject: [PATCH 0361/1071] KAFKA-8262, KAFKA-8263: Fix flaky test
 `MetricsIntegrationTest` (#6922)

- Timeout occurred due to initial slow rebalancing.
- Added code to wait until `KafkaStreams` instance is in state RUNNING to check registration of metrics and in state NOT_RUNNING to check deregistration of metrics.
- I removed all other wait conditions, because they are not needed if `KafkaStreams` instance is in the right state.

Reviewers: Guozhang Wang 
---
 .../integration/MetricsIntegrationTest.java   | 456 ++++++++----------
 1 file changed, 205 insertions(+), 251 deletions(-)

diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
index bc4d89535e058..a3c5b8753c206 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
@@ -21,6 +21,7 @@
 import org.apache.kafka.common.serialization.Serdes;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.KafkaStreams.State;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
@@ -50,6 +51,9 @@
 import java.util.Properties;
 import java.util.stream.Collectors;
 
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
 @SuppressWarnings("unchecked")
 @Category({IntegrationTest.class})
 public class MetricsIntegrationTest {
@@ -187,27 +191,35 @@ public void after() throws InterruptedException {
         CLUSTER.deleteTopics(STREAM_INPUT, STREAM_OUTPUT_1, STREAM_OUTPUT_2, STREAM_OUTPUT_3, STREAM_OUTPUT_4);
     }
 
-    private void startApplication() {
+    private void startApplication() throws InterruptedException {
         kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration);
         kafkaStreams.start();
+        final long timeout = 60000;
+        TestUtils.waitForCondition(
+            () -> kafkaStreams.state() == State.RUNNING,
+            timeout,
+            () -> "Kafka Streams application did not reach state RUNNING in " + timeout + " ms");
     }
 
     private void closeApplication() throws Exception {
         kafkaStreams.close();
         kafkaStreams.cleanUp();
         IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration);
+        final long timeout = 60000;
+        TestUtils.waitForCondition(
+            () -> kafkaStreams.state() == State.NOT_RUNNING,
+            timeout,
+            () -> "Kafka Streams application did not reach state NOT_RUNNING in " + timeout + " ms");
     }
 
-    private void checkMetricDeregistration() throws InterruptedException {
-        TestUtils.waitForCondition(() -> {
-            final List listMetricAfterClosingApp = new ArrayList(kafkaStreams.metrics().values()).stream().filter(m -> m.metricName().group().contains(STREAM_STRING)).collect(Collectors.toList());
-            return listMetricAfterClosingApp.size() == 0;
-        }, 10000, "de-registration of metrics");
+    private void checkMetricDeregistration() {
+        final List listMetricAfterClosingApp = new ArrayList(kafkaStreams.metrics().values()).stream()
+            .filter(m -> m.metricName().group().contains(STREAM_STRING)).collect(Collectors.toList());
+        assertThat(listMetricAfterClosingApp.size(), is(0));
     }
 
     @Test
     public void testStreamMetric() throws Exception {
-        final StringBuilder errorMessage = new StringBuilder();
         stream = builder.stream(STREAM_INPUT, Consumed.with(Serdes.Integer(), Serdes.String()));
         stream.to(STREAM_OUTPUT_1, Produced.with(Serdes.Integer(), Serdes.String()));
         builder.table(STREAM_OUTPUT_1, Materialized.as(Stores.inMemoryKeyValueStore(MY_STORE_IN_MEMORY)).withCachingEnabled())
@@ -222,22 +234,13 @@ public void testStreamMetric() throws Exception {
 
         startApplication();
 
-        // metric level : Thread
-        TestUtils.waitForCondition(() -> testThreadMetric(errorMessage), 10000, () -> "testThreadMetric -> " + errorMessage.toString());
-
-        // metric level : Task
-        TestUtils.waitForCondition(() -> testTaskMetric(errorMessage), 10000, () -> "testTaskMetric -> " + errorMessage.toString());
-
-        // metric level : Processor
-        TestUtils.waitForCondition(() -> testProcessorMetric(errorMessage), 10000, () -> "testProcessorMetric -> " + errorMessage.toString());
-
-        // metric level : Store (in-memory-state, in-memory-lru-state, rocksdb-state)
-        TestUtils.waitForCondition(() -> testStoreMetricKeyValueByType(STREAM_STORE_IN_MEMORY_STATE_METRICS, errorMessage), 10000, () -> "testStoreMetricKeyValueByType:" + STREAM_STORE_IN_MEMORY_STATE_METRICS + " -> " + errorMessage.toString());
-        TestUtils.waitForCondition(() -> testStoreMetricKeyValueByType(STREAM_STORE_IN_MEMORY_LRU_STATE_METRICS, errorMessage), 10000, () -> "testStoreMetricKeyValueByType:" + STREAM_STORE_IN_MEMORY_LRU_STATE_METRICS + " -> " + errorMessage.toString());
-        TestUtils.waitForCondition(() -> testStoreMetricKeyValueByType(STREAM_STORE_ROCKSDB_STATE_METRICS, errorMessage), 10000, () -> "testStoreMetricKeyValueByType:" + STREAM_STORE_ROCKSDB_STATE_METRICS + " -> " + errorMessage.toString());
-
-        //metric level : Cache
-        TestUtils.waitForCondition(() -> testCacheMetric(errorMessage), 10000, () -> "testCacheMetric -> " + errorMessage.toString());
+        checkThreadLevelMetrics();
+        checkTaskLevelMetrics();
+        checkProcessorLevelMetrics();
+        checkKeyValueStoreMetricsByType(STREAM_STORE_IN_MEMORY_STATE_METRICS);
+        checkKeyValueStoreMetricsByType(STREAM_STORE_IN_MEMORY_LRU_STATE_METRICS);
+        checkKeyValueStoreMetricsByType(STREAM_STORE_ROCKSDB_STATE_METRICS);
+        checkCacheMetrics();
 
         closeApplication();
 
@@ -247,7 +250,6 @@ public void testStreamMetric() throws Exception {
 
     @Test
     public void testStreamMetricOfWindowStore() throws Exception {
-        final StringBuilder errorMessage = new StringBuilder();
         stream2 = builder.stream(STREAM_INPUT, Consumed.with(Serdes.Integer(), Serdes.String()));
         final KGroupedStream groupedStream = stream2.groupByKey();
         groupedStream.windowedBy(TimeWindows.of(Duration.ofMillis(50)))
@@ -257,8 +259,7 @@ public void testStreamMetricOfWindowStore() throws Exception {
 
         startApplication();
 
-        // metric level : Store (window)
-        TestUtils.waitForCondition(() -> testStoreMetricWindow(errorMessage), 10000, () -> "testStoreMetricWindow -> " + errorMessage.toString());
+        checkWindowStoreMetrics();
 
         closeApplication();
 
@@ -268,7 +269,6 @@ public void testStreamMetricOfWindowStore() throws Exception {
 
     @Test
     public void testStreamMetricOfSessionStore() throws Exception {
-        final StringBuilder errorMessage = new StringBuilder();
         stream2 = builder.stream(STREAM_INPUT, Consumed.with(Serdes.Integer(), Serdes.String()));
         final KGroupedStream groupedStream = stream2.groupByKey();
         groupedStream.windowedBy(SessionWindows.with(Duration.ofMillis(50)))
@@ -278,8 +278,7 @@ public void testStreamMetricOfSessionStore() throws Exception {
 
         startApplication();
 
-        // metric level : Store (session)
-        TestUtils.waitForCondition(() -> testStoreMetricSession(errorMessage), 10000, () -> "testStoreMetricSession -> " + errorMessage.toString());
+        checkSessionStoreMetrics();
 
         closeApplication();
 
@@ -287,240 +286,195 @@ public void testStreamMetricOfSessionStore() throws Exception {
         checkMetricDeregistration();
     }
 
-    private boolean testThreadMetric(final StringBuilder errorMessage) {
-        errorMessage.setLength(0);
-        try {
-            final List listMetricThread = new ArrayList(kafkaStreams.metrics().values()).stream().filter(m -> m.metricName().group().equals(STREAM_THREAD_NODE_METRICS)).collect(Collectors.toList());
-            testMetricByName(listMetricThread, COMMIT_LATENCY_AVG, 1);
-            testMetricByName(listMetricThread, COMMIT_LATENCY_MAX, 1);
-            testMetricByName(listMetricThread, POLL_LATENCY_AVG, 1);
-            testMetricByName(listMetricThread, POLL_LATENCY_MAX, 1);
-            testMetricByName(listMetricThread, PROCESS_LATENCY_AVG, 1);
-            testMetricByName(listMetricThread, PROCESS_LATENCY_MAX, 1);
-            testMetricByName(listMetricThread, PUNCTUATE_LATENCY_AVG, 1);
-            testMetricByName(listMetricThread, PUNCTUATE_LATENCY_MAX, 1);
-            testMetricByName(listMetricThread, COMMIT_RATE, 1);
-            testMetricByName(listMetricThread, COMMIT_TOTAL, 1);
-            testMetricByName(listMetricThread, POLL_RATE, 1);
-            testMetricByName(listMetricThread, POLL_TOTAL, 1);
-            testMetricByName(listMetricThread, PROCESS_RATE, 1);
-            testMetricByName(listMetricThread, PROCESS_TOTAL, 1);
-            testMetricByName(listMetricThread, PUNCTUATE_RATE, 1);
-            testMetricByName(listMetricThread, PUNCTUATE_TOTAL, 1);
-            testMetricByName(listMetricThread, TASK_CREATED_RATE, 1);
-            testMetricByName(listMetricThread, TASK_CREATED_TOTAL, 1);
-            testMetricByName(listMetricThread, TASK_CLOSED_RATE, 1);
-            testMetricByName(listMetricThread, TASK_CLOSED_TOTAL, 1);
-            testMetricByName(listMetricThread, SKIPPED_RECORDS_RATE, 1);
-            testMetricByName(listMetricThread, SKIPPED_RECORDS_TOTAL, 1);
-            return true;
-        } catch (final Throwable e) {
-            errorMessage.append(e.getMessage());
-            return false;
-        }
+    private void checkTaskLevelMetrics() {
+        final List listMetricTask = new ArrayList(kafkaStreams.metrics().values()).stream()
+            .filter(m -> m.metricName().group().equals(STREAM_TASK_NODE_METRICS)).collect(Collectors.toList());
+        testMetricByName(listMetricTask, COMMIT_LATENCY_AVG, 5);
+        testMetricByName(listMetricTask, COMMIT_LATENCY_MAX, 5);
+        testMetricByName(listMetricTask, COMMIT_RATE, 5);
+        testMetricByName(listMetricTask, COMMIT_TOTAL, 5);
+        testMetricByName(listMetricTask, RECORD_LATENESS_AVG, 4);
+        testMetricByName(listMetricTask, RECORD_LATENESS_MAX, 4);
     }
 
-    private boolean testTaskMetric(final StringBuilder errorMessage) {
-        errorMessage.setLength(0);
-        try {
-            final List listMetricTask = new ArrayList(kafkaStreams.metrics().values()).stream().filter(m -> m.metricName().group().equals(STREAM_TASK_NODE_METRICS)).collect(Collectors.toList());
-            testMetricByName(listMetricTask, COMMIT_LATENCY_AVG, 5);
-            testMetricByName(listMetricTask, COMMIT_LATENCY_MAX, 5);
-            testMetricByName(listMetricTask, COMMIT_RATE, 5);
-            testMetricByName(listMetricTask, COMMIT_TOTAL, 5);
-            testMetricByName(listMetricTask, RECORD_LATENESS_AVG, 4);
-            testMetricByName(listMetricTask, RECORD_LATENESS_MAX, 4);
-            return true;
-        } catch (final Throwable e) {
-            errorMessage.append(e.getMessage());
-            return false;
-        }
+    private void checkThreadLevelMetrics() {
+        final List listMetricThread = new ArrayList(kafkaStreams.metrics().values()).stream()
+            .filter(m -> m.metricName().group().equals(STREAM_THREAD_NODE_METRICS)).collect(Collectors.toList());
+        testMetricByName(listMetricThread, COMMIT_LATENCY_AVG, 1);
+        testMetricByName(listMetricThread, COMMIT_LATENCY_MAX, 1);
+        testMetricByName(listMetricThread, POLL_LATENCY_AVG, 1);
+        testMetricByName(listMetricThread, POLL_LATENCY_MAX, 1);
+        testMetricByName(listMetricThread, PROCESS_LATENCY_AVG, 1);
+        testMetricByName(listMetricThread, PROCESS_LATENCY_MAX, 1);
+        testMetricByName(listMetricThread, PUNCTUATE_LATENCY_AVG, 1);
+        testMetricByName(listMetricThread, PUNCTUATE_LATENCY_MAX, 1);
+        testMetricByName(listMetricThread, COMMIT_RATE, 1);
+        testMetricByName(listMetricThread, COMMIT_TOTAL, 1);
+        testMetricByName(listMetricThread, POLL_RATE, 1);
+        testMetricByName(listMetricThread, POLL_TOTAL, 1);
+        testMetricByName(listMetricThread, PROCESS_RATE, 1);
+        testMetricByName(listMetricThread, PROCESS_TOTAL, 1);
+        testMetricByName(listMetricThread, PUNCTUATE_RATE, 1);
+        testMetricByName(listMetricThread, PUNCTUATE_TOTAL, 1);
+        testMetricByName(listMetricThread, TASK_CREATED_RATE, 1);
+        testMetricByName(listMetricThread, TASK_CREATED_TOTAL, 1);
+        testMetricByName(listMetricThread, TASK_CLOSED_RATE, 1);
+        testMetricByName(listMetricThread, TASK_CLOSED_TOTAL, 1);
+        testMetricByName(listMetricThread, SKIPPED_RECORDS_RATE, 1);
+        testMetricByName(listMetricThread, SKIPPED_RECORDS_TOTAL, 1);
     }
 
-    private boolean testProcessorMetric(final StringBuilder errorMessage) {
-        errorMessage.setLength(0);
-        try {
-            final List listMetricProcessor = new ArrayList(kafkaStreams.metrics().values()).stream().filter(m -> m.metricName().group().equals(STREAM_PROCESSOR_NODE_METRICS)).collect(Collectors.toList());
-            testMetricByName(listMetricProcessor, PROCESS_LATENCY_AVG, 18);
-            testMetricByName(listMetricProcessor, PROCESS_LATENCY_MAX, 18);
-            testMetricByName(listMetricProcessor, PUNCTUATE_LATENCY_AVG, 18);
-            testMetricByName(listMetricProcessor, PUNCTUATE_LATENCY_MAX, 18);
-            testMetricByName(listMetricProcessor, CREATE_LATENCY_AVG, 18);
-            testMetricByName(listMetricProcessor, CREATE_LATENCY_MAX, 18);
-            testMetricByName(listMetricProcessor, DESTROY_LATENCY_AVG, 18);
-            testMetricByName(listMetricProcessor, DESTROY_LATENCY_MAX, 18);
-            testMetricByName(listMetricProcessor, PROCESS_RATE, 18);
-            testMetricByName(listMetricProcessor, PROCESS_TOTAL, 18);
-            testMetricByName(listMetricProcessor, PUNCTUATE_RATE, 18);
-            testMetricByName(listMetricProcessor, PUNCTUATE_TOTAL, 18);
-            testMetricByName(listMetricProcessor, CREATE_RATE, 18);
-            testMetricByName(listMetricProcessor, CREATE_TOTAL, 18);
-            testMetricByName(listMetricProcessor, DESTROY_RATE, 18);
-            testMetricByName(listMetricProcessor, DESTROY_TOTAL, 18);
-            testMetricByName(listMetricProcessor, FORWARD_TOTAL, 18);
-            return true;
-        } catch (final Throwable e) {
-            errorMessage.append(e.getMessage());
-            return false;
-        }
+    private void checkProcessorLevelMetrics() {
+        final List listMetricProcessor = new ArrayList(kafkaStreams.metrics().values()).stream()
+            .filter(m -> m.metricName().group().equals(STREAM_PROCESSOR_NODE_METRICS)).collect(Collectors.toList());
+        testMetricByName(listMetricProcessor, PROCESS_LATENCY_AVG, 18);
+        testMetricByName(listMetricProcessor, PROCESS_LATENCY_MAX, 18);
+        testMetricByName(listMetricProcessor, PUNCTUATE_LATENCY_AVG, 18);
+        testMetricByName(listMetricProcessor, PUNCTUATE_LATENCY_MAX, 18);
+        testMetricByName(listMetricProcessor, CREATE_LATENCY_AVG, 18);
+        testMetricByName(listMetricProcessor, CREATE_LATENCY_MAX, 18);
+        testMetricByName(listMetricProcessor, DESTROY_LATENCY_AVG, 18);
+        testMetricByName(listMetricProcessor, DESTROY_LATENCY_MAX, 18);
+        testMetricByName(listMetricProcessor, PROCESS_RATE, 18);
+        testMetricByName(listMetricProcessor, PROCESS_TOTAL, 18);
+        testMetricByName(listMetricProcessor, PUNCTUATE_RATE, 18);
+        testMetricByName(listMetricProcessor, PUNCTUATE_TOTAL, 18);
+        testMetricByName(listMetricProcessor, CREATE_RATE, 18);
+        testMetricByName(listMetricProcessor, CREATE_TOTAL, 18);
+        testMetricByName(listMetricProcessor, DESTROY_RATE, 18);
+        testMetricByName(listMetricProcessor, DESTROY_TOTAL, 18);
+        testMetricByName(listMetricProcessor, FORWARD_TOTAL, 18);
     }
 
-    private boolean testStoreMetricWindow(final StringBuilder errorMessage) {
-        errorMessage.setLength(0);
-        try {
-            final List listMetricStore = new ArrayList(kafkaStreams.metrics().values()).stream()
-                    .filter(m -> m.metricName().group().equals(STREAM_STORE_WINDOW_ROCKSDB_STATE_METRICS))
-                    .collect(Collectors.toList());
-            testMetricByName(listMetricStore, PUT_LATENCY_AVG, 2);
-            testMetricByName(listMetricStore, PUT_LATENCY_MAX, 2);
-            testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_AVG, 0);
-            testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_MAX, 0);
-            testMetricByName(listMetricStore, GET_LATENCY_AVG, 0);
-            testMetricByName(listMetricStore, GET_LATENCY_MAX, 0);
-            testMetricByName(listMetricStore, DELETE_LATENCY_AVG, 0);
-            testMetricByName(listMetricStore, DELETE_LATENCY_MAX, 0);
-            testMetricByName(listMetricStore, PUT_ALL_LATENCY_AVG, 0);
-            testMetricByName(listMetricStore, PUT_ALL_LATENCY_MAX, 0);
-            testMetricByName(listMetricStore, ALL_LATENCY_AVG, 0);
-            testMetricByName(listMetricStore, ALL_LATENCY_MAX, 0);
-            testMetricByName(listMetricStore, RANGE_LATENCY_AVG, 0);
-            testMetricByName(listMetricStore, RANGE_LATENCY_MAX, 0);
-            testMetricByName(listMetricStore, FLUSH_LATENCY_AVG, 2);
-            testMetricByName(listMetricStore, FLUSH_LATENCY_MAX, 2);
-            testMetricByName(listMetricStore, RESTORE_LATENCY_AVG, 2);
-            testMetricByName(listMetricStore, RESTORE_LATENCY_MAX, 2);
-            testMetricByName(listMetricStore, PUT_RATE, 2);
-            testMetricByName(listMetricStore, PUT_TOTAL, 2);
-            testMetricByName(listMetricStore, PUT_IF_ABSENT_RATE, 0);
-            testMetricByName(listMetricStore, PUT_IF_ABSENT_TOTAL, 0);
-            testMetricByName(listMetricStore, GET_RATE, 0);
-            testMetricByName(listMetricStore, DELETE_RATE, 0);
-            testMetricByName(listMetricStore, DELETE_TOTAL, 0);
-            testMetricByName(listMetricStore, PUT_ALL_RATE, 0);
-            testMetricByName(listMetricStore, PUT_ALL_TOTAL, 0);
-            testMetricByName(listMetricStore, ALL_RATE, 0);
-            testMetricByName(listMetricStore, ALL_TOTAL, 0);
-            testMetricByName(listMetricStore, RANGE_RATE, 0);
-            testMetricByName(listMetricStore, RANGE_TOTAL, 0);
-            testMetricByName(listMetricStore, FLUSH_RATE, 2);
-            testMetricByName(listMetricStore, FLUSH_TOTAL, 2);
-            testMetricByName(listMetricStore, RESTORE_RATE, 2);
-            testMetricByName(listMetricStore, RESTORE_TOTAL, 2);
-            return true;
-        } catch (final Throwable e) {
-            errorMessage.append(e.getMessage());
-            return false;
-        }
+    private void checkKeyValueStoreMetricsByType(final String storeType) {
+        final List listMetricStore = new ArrayList(kafkaStreams.metrics().values()).stream()
+            .filter(m -> m.metricName().group().equals(storeType))
+            .collect(Collectors.toList());
+        testMetricByName(listMetricStore, PUT_LATENCY_AVG, 2);
+        testMetricByName(listMetricStore, PUT_LATENCY_MAX, 2);
+        testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_AVG, 2);
+        testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_MAX, 2);
+        testMetricByName(listMetricStore, GET_LATENCY_AVG, 2);
+        testMetricByName(listMetricStore, GET_LATENCY_MAX, 2);
+        testMetricByName(listMetricStore, DELETE_LATENCY_AVG, 2);
+        testMetricByName(listMetricStore, DELETE_LATENCY_MAX, 2);
+        testMetricByName(listMetricStore, PUT_ALL_LATENCY_AVG, 2);
+        testMetricByName(listMetricStore, PUT_ALL_LATENCY_MAX, 2);
+        testMetricByName(listMetricStore, ALL_LATENCY_AVG, 2);
+        testMetricByName(listMetricStore, ALL_LATENCY_MAX, 2);
+        testMetricByName(listMetricStore, RANGE_LATENCY_AVG, 2);
+        testMetricByName(listMetricStore, RANGE_LATENCY_MAX, 2);
+        testMetricByName(listMetricStore, FLUSH_LATENCY_AVG, 2);
+        testMetricByName(listMetricStore, FLUSH_LATENCY_MAX, 2);
+        testMetricByName(listMetricStore, RESTORE_LATENCY_AVG, 2);
+        testMetricByName(listMetricStore, RESTORE_LATENCY_MAX, 2);
+        testMetricByName(listMetricStore, PUT_RATE, 2);
+        testMetricByName(listMetricStore, PUT_TOTAL, 2);
+        testMetricByName(listMetricStore, PUT_IF_ABSENT_RATE, 2);
+        testMetricByName(listMetricStore, PUT_IF_ABSENT_TOTAL, 2);
+        testMetricByName(listMetricStore, GET_RATE, 2);
+        testMetricByName(listMetricStore, DELETE_RATE, 2);
+        testMetricByName(listMetricStore, DELETE_TOTAL, 2);
+        testMetricByName(listMetricStore, PUT_ALL_RATE, 2);
+        testMetricByName(listMetricStore, PUT_ALL_TOTAL, 2);
+        testMetricByName(listMetricStore, ALL_RATE, 2);
+        testMetricByName(listMetricStore, ALL_TOTAL, 2);
+        testMetricByName(listMetricStore, RANGE_RATE, 2);
+        testMetricByName(listMetricStore, RANGE_TOTAL, 2);
+        testMetricByName(listMetricStore, FLUSH_RATE, 2);
+        testMetricByName(listMetricStore, FLUSH_TOTAL, 2);
+        testMetricByName(listMetricStore, RESTORE_RATE, 2);
+        testMetricByName(listMetricStore, RESTORE_TOTAL, 2);
     }
 
-    private boolean testStoreMetricSession(final StringBuilder errorMessage) {
-        errorMessage.setLength(0);
-        try {
-            final List listMetricStore = new ArrayList(kafkaStreams.metrics().values()).stream()
-                    .filter(m -> m.metricName().group().equals(STREAM_STORE_SESSION_ROCKSDB_STATE_METRICS))
-                    .collect(Collectors.toList());
-            testMetricByName(listMetricStore, PUT_LATENCY_AVG, 2);
-            testMetricByName(listMetricStore, PUT_LATENCY_MAX, 2);
-            testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_AVG, 0);
-            testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_MAX, 0);
-            testMetricByName(listMetricStore, GET_LATENCY_AVG, 0);
-            testMetricByName(listMetricStore, GET_LATENCY_MAX, 0);
-            testMetricByName(listMetricStore, DELETE_LATENCY_AVG, 0);
-            testMetricByName(listMetricStore, DELETE_LATENCY_MAX, 0);
-            testMetricByName(listMetricStore, PUT_ALL_LATENCY_AVG, 0);
-            testMetricByName(listMetricStore, PUT_ALL_LATENCY_MAX, 0);
-            testMetricByName(listMetricStore, ALL_LATENCY_AVG, 0);
-            testMetricByName(listMetricStore, ALL_LATENCY_MAX, 0);
-            testMetricByName(listMetricStore, RANGE_LATENCY_AVG, 0);
-            testMetricByName(listMetricStore, RANGE_LATENCY_MAX, 0);
-            testMetricByName(listMetricStore, FLUSH_LATENCY_AVG, 2);
-            testMetricByName(listMetricStore, FLUSH_LATENCY_MAX, 2);
-            testMetricByName(listMetricStore, RESTORE_LATENCY_AVG, 2);
-            testMetricByName(listMetricStore, RESTORE_LATENCY_MAX, 2);
-            testMetricByName(listMetricStore, PUT_RATE, 2);
-            testMetricByName(listMetricStore, PUT_TOTAL, 2);
-            testMetricByName(listMetricStore, PUT_IF_ABSENT_RATE, 0);
-            testMetricByName(listMetricStore, PUT_IF_ABSENT_TOTAL, 0);
-            testMetricByName(listMetricStore, GET_RATE, 0);
-            testMetricByName(listMetricStore, DELETE_RATE, 0);
-            testMetricByName(listMetricStore, DELETE_TOTAL, 0);
-            testMetricByName(listMetricStore, PUT_ALL_RATE, 0);
-            testMetricByName(listMetricStore, PUT_ALL_TOTAL, 0);
-            testMetricByName(listMetricStore, ALL_RATE, 0);
-            testMetricByName(listMetricStore, ALL_TOTAL, 0);
-            testMetricByName(listMetricStore, RANGE_RATE, 0);
-            testMetricByName(listMetricStore, RANGE_TOTAL, 0);
-            testMetricByName(listMetricStore, FLUSH_RATE, 2);
-            testMetricByName(listMetricStore, FLUSH_TOTAL, 2);
-            testMetricByName(listMetricStore, RESTORE_RATE, 2);
-            testMetricByName(listMetricStore, RESTORE_TOTAL, 2);
-            return true;
-        } catch (final Throwable e) {
-            errorMessage.append(e.getMessage());
-            return false;
-        }
+    private void checkCacheMetrics() {
+        final List listMetricCache = new ArrayList(kafkaStreams.metrics().values()).stream()
+            .filter(m -> m.metricName().group().equals(STREAM_CACHE_NODE_METRICS)).collect(Collectors.toList());
+        testMetricByName(listMetricCache, HIT_RATIO_AVG, 6);
+        testMetricByName(listMetricCache, HIT_RATIO_MIN, 6);
+        testMetricByName(listMetricCache, HIT_RATIO_MAX, 6);
     }
 
-    private boolean testStoreMetricKeyValueByType(final String storeType, final StringBuilder errorMessage) {
-        errorMessage.setLength(0);
-        try {
-            final List listMetricStore = new ArrayList(kafkaStreams.metrics().values()).stream()
-                    .filter(m -> m.metricName().group().equals(storeType))
-                    .collect(Collectors.toList());
-            testMetricByName(listMetricStore, PUT_LATENCY_AVG, 2);
-            testMetricByName(listMetricStore, PUT_LATENCY_MAX, 2);
-            testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_AVG, 2);
-            testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_MAX, 2);
-            testMetricByName(listMetricStore, GET_LATENCY_AVG, 2);
-            testMetricByName(listMetricStore, GET_LATENCY_MAX, 2);
-            testMetricByName(listMetricStore, DELETE_LATENCY_AVG, 2);
-            testMetricByName(listMetricStore, DELETE_LATENCY_MAX, 2);
-            testMetricByName(listMetricStore, PUT_ALL_LATENCY_AVG, 2);
-            testMetricByName(listMetricStore, PUT_ALL_LATENCY_MAX, 2);
-            testMetricByName(listMetricStore, ALL_LATENCY_AVG, 2);
-            testMetricByName(listMetricStore, ALL_LATENCY_MAX, 2);
-            testMetricByName(listMetricStore, RANGE_LATENCY_AVG, 2);
-            testMetricByName(listMetricStore, RANGE_LATENCY_MAX, 2);
-            testMetricByName(listMetricStore, FLUSH_LATENCY_AVG, 2);
-            testMetricByName(listMetricStore, FLUSH_LATENCY_MAX, 2);
-            testMetricByName(listMetricStore, RESTORE_LATENCY_AVG, 2);
-            testMetricByName(listMetricStore, RESTORE_LATENCY_MAX, 2);
-            testMetricByName(listMetricStore, PUT_RATE, 2);
-            testMetricByName(listMetricStore, PUT_TOTAL, 2);
-            testMetricByName(listMetricStore, PUT_IF_ABSENT_RATE, 2);
-            testMetricByName(listMetricStore, PUT_IF_ABSENT_TOTAL, 2);
-            testMetricByName(listMetricStore, GET_RATE, 2);
-            testMetricByName(listMetricStore, DELETE_RATE, 2);
-            testMetricByName(listMetricStore, DELETE_TOTAL, 2);
-            testMetricByName(listMetricStore, PUT_ALL_RATE, 2);
-            testMetricByName(listMetricStore, PUT_ALL_TOTAL, 2);
-            testMetricByName(listMetricStore, ALL_RATE, 2);
-            testMetricByName(listMetricStore, ALL_TOTAL, 2);
-            testMetricByName(listMetricStore, RANGE_RATE, 2);
-            testMetricByName(listMetricStore, RANGE_TOTAL, 2);
-            testMetricByName(listMetricStore, FLUSH_RATE, 2);
-            testMetricByName(listMetricStore, FLUSH_TOTAL, 2);
-            testMetricByName(listMetricStore, RESTORE_RATE, 2);
-            testMetricByName(listMetricStore, RESTORE_TOTAL, 2);
-            return true;
-        } catch (final Throwable e) {
-            errorMessage.append(e.getMessage());
-            return false;
-        }
+    private void checkWindowStoreMetrics() {
+        final List listMetricStore = new ArrayList(kafkaStreams.metrics().values()).stream()
+            .filter(m -> m.metricName().group().equals(STREAM_STORE_WINDOW_ROCKSDB_STATE_METRICS))
+            .collect(Collectors.toList());
+        testMetricByName(listMetricStore, PUT_LATENCY_AVG, 2);
+        testMetricByName(listMetricStore, PUT_LATENCY_MAX, 2);
+        testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_AVG, 0);
+        testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_MAX, 0);
+        testMetricByName(listMetricStore, GET_LATENCY_AVG, 0);
+        testMetricByName(listMetricStore, GET_LATENCY_MAX, 0);
+        testMetricByName(listMetricStore, DELETE_LATENCY_AVG, 0);
+        testMetricByName(listMetricStore, DELETE_LATENCY_MAX, 0);
+        testMetricByName(listMetricStore, PUT_ALL_LATENCY_AVG, 0);
+        testMetricByName(listMetricStore, PUT_ALL_LATENCY_MAX, 0);
+        testMetricByName(listMetricStore, ALL_LATENCY_AVG, 0);
+        testMetricByName(listMetricStore, ALL_LATENCY_MAX, 0);
+        testMetricByName(listMetricStore, RANGE_LATENCY_AVG, 0);
+        testMetricByName(listMetricStore, RANGE_LATENCY_MAX, 0);
+        testMetricByName(listMetricStore, FLUSH_LATENCY_AVG, 2);
+        testMetricByName(listMetricStore, FLUSH_LATENCY_MAX, 2);
+        testMetricByName(listMetricStore, RESTORE_LATENCY_AVG, 2);
+        testMetricByName(listMetricStore, RESTORE_LATENCY_MAX, 2);
+        testMetricByName(listMetricStore, PUT_RATE, 2);
+        testMetricByName(listMetricStore, PUT_TOTAL, 2);
+        testMetricByName(listMetricStore, PUT_IF_ABSENT_RATE, 0);
+        testMetricByName(listMetricStore, PUT_IF_ABSENT_TOTAL, 0);
+        testMetricByName(listMetricStore, GET_RATE, 0);
+        testMetricByName(listMetricStore, DELETE_RATE, 0);
+        testMetricByName(listMetricStore, DELETE_TOTAL, 0);
+        testMetricByName(listMetricStore, PUT_ALL_RATE, 0);
+        testMetricByName(listMetricStore, PUT_ALL_TOTAL, 0);
+        testMetricByName(listMetricStore, ALL_RATE, 0);
+        testMetricByName(listMetricStore, ALL_TOTAL, 0);
+        testMetricByName(listMetricStore, RANGE_RATE, 0);
+        testMetricByName(listMetricStore, RANGE_TOTAL, 0);
+        testMetricByName(listMetricStore, FLUSH_RATE, 2);
+        testMetricByName(listMetricStore, FLUSH_TOTAL, 2);
+        testMetricByName(listMetricStore, RESTORE_RATE, 2);
+        testMetricByName(listMetricStore, RESTORE_TOTAL, 2);
     }
 
-    private boolean testCacheMetric(final StringBuilder errorMessage) {
-        errorMessage.setLength(0);
-        try {
-            final List listMetricCache = new ArrayList(kafkaStreams.metrics().values()).stream().filter(m -> m.metricName().group().equals(STREAM_CACHE_NODE_METRICS)).collect(Collectors.toList());
-            testMetricByName(listMetricCache, HIT_RATIO_AVG, 6);
-            testMetricByName(listMetricCache, HIT_RATIO_MIN, 6);
-            testMetricByName(listMetricCache, HIT_RATIO_MAX, 6);
-            return true;
-        } catch (final Throwable e) {
-            errorMessage.append(e.getMessage());
-            return false;
-        }
+    private void checkSessionStoreMetrics() {
+        final List listMetricStore = new ArrayList(kafkaStreams.metrics().values()).stream()
+            .filter(m -> m.metricName().group().equals(STREAM_STORE_SESSION_ROCKSDB_STATE_METRICS))
+            .collect(Collectors.toList());
+        testMetricByName(listMetricStore, PUT_LATENCY_AVG, 2);
+        testMetricByName(listMetricStore, PUT_LATENCY_MAX, 2);
+        testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_AVG, 0);
+        testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_MAX, 0);
+        testMetricByName(listMetricStore, GET_LATENCY_AVG, 0);
+        testMetricByName(listMetricStore, GET_LATENCY_MAX, 0);
+        testMetricByName(listMetricStore, DELETE_LATENCY_AVG, 0);
+        testMetricByName(listMetricStore, DELETE_LATENCY_MAX, 0);
+        testMetricByName(listMetricStore, PUT_ALL_LATENCY_AVG, 0);
+        testMetricByName(listMetricStore, PUT_ALL_LATENCY_MAX, 0);
+        testMetricByName(listMetricStore, ALL_LATENCY_AVG, 0);
+        testMetricByName(listMetricStore, ALL_LATENCY_MAX, 0);
+        testMetricByName(listMetricStore, RANGE_LATENCY_AVG, 0);
+        testMetricByName(listMetricStore, RANGE_LATENCY_MAX, 0);
+        testMetricByName(listMetricStore, FLUSH_LATENCY_AVG, 2);
+        testMetricByName(listMetricStore, FLUSH_LATENCY_MAX, 2);
+        testMetricByName(listMetricStore, RESTORE_LATENCY_AVG, 2);
+        testMetricByName(listMetricStore, RESTORE_LATENCY_MAX, 2);
+        testMetricByName(listMetricStore, PUT_RATE, 2);
+        testMetricByName(listMetricStore, PUT_TOTAL, 2);
+        testMetricByName(listMetricStore, PUT_IF_ABSENT_RATE, 0);
+        testMetricByName(listMetricStore, PUT_IF_ABSENT_TOTAL, 0);
+        testMetricByName(listMetricStore, GET_RATE, 0);
+        testMetricByName(listMetricStore, DELETE_RATE, 0);
+        testMetricByName(listMetricStore, DELETE_TOTAL, 0);
+        testMetricByName(listMetricStore, PUT_ALL_RATE, 0);
+        testMetricByName(listMetricStore, PUT_ALL_TOTAL, 0);
+        testMetricByName(listMetricStore, ALL_RATE, 0);
+        testMetricByName(listMetricStore, ALL_TOTAL, 0);
+        testMetricByName(listMetricStore, RANGE_RATE, 0);
+        testMetricByName(listMetricStore, RANGE_TOTAL, 0);
+        testMetricByName(listMetricStore, FLUSH_RATE, 2);
+        testMetricByName(listMetricStore, FLUSH_TOTAL, 2);
+        testMetricByName(listMetricStore, RESTORE_RATE, 2);
+        testMetricByName(listMetricStore, RESTORE_TOTAL, 2);
     }
 
     private void testMetricByName(final List listMetric, final String metricName, final int numMetric) {

From f396372fb894b43c4ca85f1a71bd1b1750ca3e05 Mon Sep 17 00:00:00 2001
From: Boyang Chen 
Date: Wed, 12 Jun 2019 15:38:06 -0700
Subject: [PATCH 0362/1071] MINOR: add group coordinator test coverage (#6926)

Some edge cases are not currently being tested. Add more tests to cover.

Reviewers: Guozhang Wang 
---
 .../group/GroupCoordinatorTest.scala          | 36 ++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
index d5d33fb7aac5a..4cd91dd6fccfe 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
@@ -933,9 +933,28 @@ class GroupCoordinatorTest {
     assertTrue(message.contains(followerInstanceId.get))
   }
 
+  @Test
+  def staticMemberReJoinWithIllegalStateAsUnknownMember() {
+    staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
+    val group = groupCoordinator.groupManager.getGroup(groupId).get
+    group.transitionTo(PreparingRebalance)
+    group.transitionTo(Empty)
+
+    EasyMock.reset(replicaManager)
+
+    // Illegal state exception shall trigger since follower id resides in pending member bucket.
+    val expectedException = intercept[IllegalStateException] {
+      staticJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, followerInstanceId, protocolType, protocolSuperset, clockAdvance = 1)
+    }
+
+    val message = expectedException.getMessage
+    assertTrue(message.contains(group.groupId))
+    assertTrue(message.contains(followerInstanceId.get))
+  }
+
   @Test
   def staticMemberReJoinWithIllegalArgumentAsMissingOldMember() {
-    val _ = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
+    staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
     val group = groupCoordinator.groupManager.getGroup(groupId).get
     val invalidMemberId = "invalid_member_id"
     group.addStaticMember(followerInstanceId, invalidMemberId)
@@ -1185,6 +1204,21 @@ class GroupCoordinatorTest {
     assertEquals(Errors.COORDINATOR_NOT_AVAILABLE, heartbeatResult)
   }
 
+  @Test
+  def testheartbeatEmptyGroup() {
+    val memberId = "memberId"
+
+    val group = new GroupMetadata(groupId, Empty, new MockTime())
+    val member = new MemberMetadata(memberId, groupId, groupInstanceId,
+      ClientId, ClientHost, DefaultRebalanceTimeout, DefaultSessionTimeout,
+      protocolType, List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte])))
+
+    group.add(member)
+    groupCoordinator.groupManager.addGroup(group)
+    val heartbeatResult = heartbeat(groupId, memberId, 0)
+    assertEquals(Errors.UNKNOWN_MEMBER_ID, heartbeatResult)
+  }
+
   @Test
   def testHeartbeatUnknownConsumerExistingGroup() {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID

From e54ab292e7e2fd1d18e82387a586969ba57eb7ea Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Thu, 13 Jun 2019 08:48:15 -0500
Subject: [PATCH 0363/1071] KAFKA-8452: Compressed BufferValue (#6848)

De-duplicate the common case in which the prior value is the same as the old value.

Reviewers: Sophie Blee-Goldman ,  Bill Bejeck 
---
 .../streams/kstream/internals/Change.java     |   2 +-
 .../kstream/internals/FullChangeSerde.java    | 153 ++++++-------
 .../internals/ProcessorRecordContext.java     |   2 +-
 .../streams/state/internals/BufferValue.java  | 137 +++++++++---
 .../state/internals/ContextualRecord.java     |   4 +-
 .../InMemoryTimeOrderedKeyValueBuffer.java    |  71 +++---
 .../state/internals/LRUCacheEntry.java        |   2 +-
 .../internals/FullChangeSerdeTest.java        | 132 +++---------
 .../KTableSuppressProcessorMetricsTest.java   |  15 +-
 .../suppress/KTableSuppressProcessorTest.java |   3 +-
 .../internals/suppress/SuppressSuite.java     |  14 +-
 .../internals/ProcessorRecordContextTest.java |  10 +-
 .../state/internals/BufferValueTest.java      | 203 ++++++++++++++++++
 .../TimeOrderedKeyValueBufferTest.java        |  78 +++----
 14 files changed, 518 insertions(+), 308 deletions(-)
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/state/internals/BufferValueTest.java

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/Change.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/Change.java
index c9a18de0de7ff..f28a16d964b99 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/Change.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/Change.java
@@ -30,7 +30,7 @@ public Change(final T newValue, final T oldValue) {
 
     @Override
     public String toString() {
-        return "(" + newValue + "<-" + oldValue + ")";
+        return "(" + String.valueOf(newValue) + "<-" + String.valueOf(oldValue) + ")";
     }
 
     @Override
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/FullChangeSerde.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/FullChangeSerde.java
index 30d55beacaa7b..f28f9e7ef3996 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/FullChangeSerde.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/FullChangeSerde.java
@@ -21,19 +21,15 @@
 import org.apache.kafka.common.serialization.Serializer;
 
 import java.nio.ByteBuffer;
-import java.util.Map;
 
 import static java.util.Objects.requireNonNull;
 
-public final class FullChangeSerde implements Serde> {
+public final class FullChangeSerde {
     private final Serde inner;
 
-    @SuppressWarnings("unchecked")
-    public static  FullChangeSerde castOrWrap(final Serde serde) {
+    public static  FullChangeSerde wrap(final Serde serde) {
         if (serde == null) {
             return null;
-        } else if (serde instanceof FullChangeSerde) {
-            return (FullChangeSerde) serde;
         } else {
             return new FullChangeSerde<>(serde);
         }
@@ -47,98 +43,81 @@ public Serde innerSerde() {
         return inner;
     }
 
-    @Override
-    public void configure(final Map configs, final boolean isKey) {
-        inner.configure(configs, isKey);
+    public Change serializeParts(final String topic, final Change data) {
+        if (data == null) {
+            return null;
+        }
+        final Serializer innerSerializer = innerSerde().serializer();
+        final byte[] oldBytes = data.oldValue == null ? null : innerSerializer.serialize(topic, data.oldValue);
+        final byte[] newBytes = data.newValue == null ? null : innerSerializer.serialize(topic, data.newValue);
+        return new Change<>(newBytes, oldBytes);
     }
 
-    @Override
-    public void close() {
-        inner.close();
-    }
 
-    @Override
-    public Serializer> serializer() {
-        final Serializer innerSerializer = inner.serializer();
-
-        return new Serializer>() {
-            @Override
-            public void configure(final Map configs, final boolean isKey) {
-                innerSerializer.configure(configs, isKey);
-            }
-
-            @Override
-            public byte[] serialize(final String topic, final Change data) {
-                if (data == null) {
-                    return null;
-                }
-                final byte[] oldBytes = data.oldValue == null ? null : innerSerializer.serialize(topic, data.oldValue);
-                final int oldSize = oldBytes == null ? -1 : oldBytes.length;
-                final byte[] newBytes = data.newValue == null ? null : innerSerializer.serialize(topic, data.newValue);
-                final int newSize = newBytes == null ? -1 : newBytes.length;
-
-                final ByteBuffer buffer = ByteBuffer.wrap(
-                    new byte[4 + (oldSize == -1 ? 0 : oldSize) + 4 + (newSize == -1 ? 0 : newSize)]
-                );
-                buffer.putInt(oldSize);
-                if (oldBytes != null) {
-                    buffer.put(oldBytes);
-                }
-                buffer.putInt(newSize);
-                if (newBytes != null) {
-                    buffer.put(newBytes);
-                }
-                return buffer.array();
-            }
-
-            @Override
-            public void close() {
-                innerSerializer.close();
-            }
-        };
+    public Change deserializeParts(final String topic, final Change serialChange) {
+        if (serialChange == null) {
+            return null;
+        }
+        final Deserializer innerDeserializer = innerSerde().deserializer();
+
+        final T oldValue =
+            serialChange.oldValue == null ? null : innerDeserializer.deserialize(topic, serialChange.oldValue);
+        final T newValue =
+            serialChange.newValue == null ? null : innerDeserializer.deserialize(topic, serialChange.newValue);
+
+        return new Change<>(newValue, oldValue);
     }
 
-    @Override
-    public Deserializer> deserializer() {
-        final Deserializer innerDeserializer = inner.deserializer();
-        return new Deserializer>() {
-            @Override
-            public void configure(final Map configs, final boolean isKey) {
-                innerDeserializer.configure(configs, isKey);
-            }
-
-            @Override
-            public Change deserialize(final String topic, final byte[] data) {
-                if (data == null) {
-                    return null;
-                }
-                final ByteBuffer buffer = ByteBuffer.wrap(data);
-
-                final byte[] oldBytes = extractOldValuePart(buffer);
-                final T oldValue = oldBytes == null ? null : innerDeserializer.deserialize(topic, oldBytes);
-
-                final int newSize = buffer.getInt();
-                final byte[] newBytes = newSize == -1 ? null : new byte[newSize];
-                if (newBytes != null) {
-                    buffer.get(newBytes);
-                }
-                final T newValue = newBytes == null ? null : innerDeserializer.deserialize(topic, newBytes);
-                return new Change<>(newValue, oldValue);
-            }
-
-            @Override
-            public void close() {
-                innerDeserializer.close();
-            }
-        };
+    /**
+     * We used to serialize a Change into a single byte[]. Now, we don't anymore, but we still keep this logic here
+     * so that we can produce the legacy format to test that we can still deserialize it.
+     */
+    public static byte[] composeLegacyFormat(final Change serialChange) {
+        if (serialChange == null) {
+            return null;
+        }
+
+        final int oldSize = serialChange.oldValue == null ? -1 : serialChange.oldValue.length;
+        final int newSize = serialChange.newValue == null ? -1 : serialChange.newValue.length;
+
+        final ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES * 2 + Math.max(0, oldSize) + Math.max(0, newSize));
+
+
+        buffer.putInt(oldSize);
+        if (serialChange.oldValue != null) {
+            buffer.put(serialChange.oldValue);
+        }
+
+        buffer.putInt(newSize);
+        if (serialChange.newValue != null) {
+            buffer.put(serialChange.newValue);
+        }
+        return buffer.array();
     }
 
-    public static byte[] extractOldValuePart(final ByteBuffer buffer) {
+    /**
+     * We used to serialize a Change into a single byte[]. Now, we don't anymore, but we still
+     * need to be able to read it (so that we can load the state store from previously-written changelog records).
+     */
+    public static Change decomposeLegacyFormat(final byte[] data) {
+        if (data == null) {
+            return null;
+        }
+        final ByteBuffer buffer = ByteBuffer.wrap(data);
+
         final int oldSize = buffer.getInt();
         final byte[] oldBytes = oldSize == -1 ? null : new byte[oldSize];
         if (oldBytes != null) {
             buffer.get(oldBytes);
         }
-        return oldBytes;
+
+        final int newSize = buffer.getInt();
+        final byte[] newBytes = newSize == -1 ? null : new byte[newSize];
+        if (newBytes != null) {
+            buffer.get(newBytes);
+        }
+
+        return new Change<>(newBytes, oldBytes);
     }
+
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorRecordContext.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorRecordContext.java
index 1b2248274b6eb..5662417a21460 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorRecordContext.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorRecordContext.java
@@ -73,7 +73,7 @@ public Headers headers() {
         return headers;
     }
 
-    public long sizeBytes() {
+    public long residentMemorySizeEstimate() {
         long size = 0;
         size += Long.BYTES; // value.context.timestamp
         size += Long.BYTES; // value.context.offset
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/BufferValue.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/BufferValue.java
index 816894ec1690b..f1990c71256e2 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/BufferValue.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/BufferValue.java
@@ -16,60 +16,133 @@
  */
 package org.apache.kafka.streams.state.internals;
 
+import org.apache.kafka.streams.processor.internals.ProcessorRecordContext;
+
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.Objects;
 
 public final class BufferValue {
-    private final ContextualRecord record;
+    private static final int NULL_VALUE_SENTINEL = -1;
+    private static final int OLD_PREV_DUPLICATE_VALUE_SENTINEL = -2;
     private final byte[] priorValue;
+    private final byte[] oldValue;
+    private final byte[] newValue;
+    private final ProcessorRecordContext recordContext;
+
+    BufferValue(final byte[] priorValue, final byte[] oldValue, final byte[] newValue, final ProcessorRecordContext recordContext) {
+        this.oldValue = oldValue;
+        this.newValue = newValue;
+        this.recordContext = recordContext;
+
+        // This de-duplicates the prior and old references.
+        // If they were already the same reference, the comparison is trivially fast, so we don't specifically check
+        // for that case.
+        if (Arrays.equals(priorValue, oldValue)) {
+            this.priorValue = oldValue;
+        } else {
+            this.priorValue = priorValue;
+        }
+    }
 
-    BufferValue(final ContextualRecord record,
-                final byte[] priorValue) {
-        this.record = record;
-        this.priorValue = priorValue;
+    byte[] priorValue() {
+        return priorValue;
     }
 
-    ContextualRecord record() {
-        return record;
+    byte[] oldValue() {
+        return oldValue;
     }
 
-    byte[] priorValue() {
-        return priorValue;
+    byte[] newValue() {
+        return newValue;
+    }
+
+    ProcessorRecordContext context() {
+        return recordContext;
     }
 
     static BufferValue deserialize(final ByteBuffer buffer) {
-        final ContextualRecord record = ContextualRecord.deserialize(buffer);
+        final ProcessorRecordContext context = ProcessorRecordContext.deserialize(buffer);
+
+        final byte[] priorValue = extractValue(buffer);
+
+        final byte[] oldValue;
+        final int oldValueLength = buffer.getInt();
+        if (oldValueLength == NULL_VALUE_SENTINEL) {
+            oldValue = null;
+        } else if (oldValueLength == OLD_PREV_DUPLICATE_VALUE_SENTINEL) {
+            oldValue = priorValue;
+        } else {
+            oldValue = new byte[oldValueLength];
+            buffer.get(oldValue);
+        }
+
+        final byte[] newValue = extractValue(buffer);
+
+        return new BufferValue(priorValue, oldValue, newValue, context);
+    }
 
-        final int priorValueLength = buffer.getInt();
-        if (priorValueLength == -1) {
-            return new BufferValue(record, null);
+    private static byte[] extractValue(final ByteBuffer buffer) {
+        final int valueLength = buffer.getInt();
+        if (valueLength == NULL_VALUE_SENTINEL) {
+            return null;
         } else {
-            final byte[] priorValue = new byte[priorValueLength];
-            buffer.get(priorValue);
-            return new BufferValue(record, priorValue);
+            final byte[] value = new byte[valueLength];
+            buffer.get(value);
+            return value;
         }
     }
 
     ByteBuffer serialize(final int endPadding) {
 
-        final int sizeOfPriorValueLength = Integer.BYTES;
+        final int sizeOfValueLength = Integer.BYTES;
+
         final int sizeOfPriorValue = priorValue == null ? 0 : priorValue.length;
+        final int sizeOfOldValue = oldValue == null || priorValue == oldValue ? 0 : oldValue.length;
+        final int sizeOfNewValue = newValue == null ? 0 : newValue.length;
+
+        final byte[] serializedContext = recordContext.serialize();
+
+        final ByteBuffer buffer = ByteBuffer.allocate(
+            serializedContext.length
+                + sizeOfValueLength + sizeOfPriorValue
+                + sizeOfValueLength + sizeOfOldValue
+                + sizeOfValueLength + sizeOfNewValue
+                + endPadding
+        );
 
-        final ByteBuffer buffer = record.serialize(sizeOfPriorValueLength + sizeOfPriorValue + endPadding);
+        buffer.put(serializedContext);
 
-        if (priorValue == null) {
-            buffer.putInt(-1);
+        addValue(buffer, priorValue);
+
+        if (oldValue == null) {
+            buffer.putInt(NULL_VALUE_SENTINEL);
+        } else if (priorValue == oldValue) {
+            buffer.putInt(OLD_PREV_DUPLICATE_VALUE_SENTINEL);
         } else {
-            buffer.putInt(priorValue.length);
-            buffer.put(priorValue);
+            buffer.putInt(sizeOfOldValue);
+            buffer.put(oldValue);
         }
 
+        addValue(buffer, newValue);
+
         return buffer;
     }
 
-    long sizeBytes() {
-        return (priorValue == null ? 0 : priorValue.length) + record.sizeBytes();
+    private static void addValue(final ByteBuffer buffer, final byte[] value) {
+        if (value == null) {
+            buffer.putInt(NULL_VALUE_SENTINEL);
+        } else {
+            buffer.putInt(value.length);
+            buffer.put(value);
+        }
+    }
+
+    long residentMemorySizeEstimate() {
+        return (priorValue == null ? 0 : priorValue.length)
+            + (oldValue == null || priorValue == oldValue ? 0 : oldValue.length)
+            + (newValue == null ? 0 : newValue.length)
+            + recordContext.residentMemorySizeEstimate();
     }
 
     @Override
@@ -77,22 +150,28 @@ public boolean equals(final Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         final BufferValue that = (BufferValue) o;
-        return Objects.equals(record, that.record) &&
-            Arrays.equals(priorValue, that.priorValue);
+        return Arrays.equals(priorValue, that.priorValue) &&
+            Arrays.equals(oldValue, that.oldValue) &&
+            Arrays.equals(newValue, that.newValue) &&
+            Objects.equals(recordContext, that.recordContext);
     }
 
     @Override
     public int hashCode() {
-        int result = Objects.hash(record);
+        int result = Objects.hash(recordContext);
         result = 31 * result + Arrays.hashCode(priorValue);
+        result = 31 * result + Arrays.hashCode(oldValue);
+        result = 31 * result + Arrays.hashCode(newValue);
         return result;
     }
 
     @Override
     public String toString() {
         return "BufferValue{" +
-            "record=" + record +
-            ", priorValue=" + Arrays.toString(priorValue) +
+            "priorValue=" + Arrays.toString(priorValue) +
+            ", oldValue=" + Arrays.toString(oldValue) +
+            ", newValue=" + Arrays.toString(newValue) +
+            ", recordContext=" + recordContext +
             '}';
     }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/ContextualRecord.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/ContextualRecord.java
index 3cd2c37358404..3c24f521c23ed 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/ContextualRecord.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/ContextualRecord.java
@@ -39,8 +39,8 @@ public byte[] value() {
         return value;
     }
 
-    long sizeBytes() {
-        return (value == null ? 0 : value.length) + recordContext.sizeBytes();
+    long residentMemorySizeEstimate() {
+        return (value == null ? 0 : value.length) + recordContext.residentMemorySizeEstimate();
     }
 
     ByteBuffer serialize(final int endPadding) {
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryTimeOrderedKeyValueBuffer.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryTimeOrderedKeyValueBuffer.java
index 6c5022f080ac2..6c6ef36792270 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryTimeOrderedKeyValueBuffer.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryTimeOrderedKeyValueBuffer.java
@@ -159,7 +159,7 @@ private InMemoryTimeOrderedKeyValueBuffer(final String storeName,
         this.storeName = storeName;
         this.loggingEnabled = loggingEnabled;
         this.keySerde = keySerde;
-        this.valueSerde = FullChangeSerde.castOrWrap(valueSerde);
+        this.valueSerde = FullChangeSerde.wrap(valueSerde);
     }
 
     @Override
@@ -176,7 +176,7 @@ public boolean persistent() {
     @Override
     public void setSerdesIfNull(final Serde keySerde, final Serde valueSerde) {
         this.keySerde = this.keySerde == null ? keySerde : this.keySerde;
-        this.valueSerde = this.valueSerde == null ? FullChangeSerde.castOrWrap(valueSerde) : this.valueSerde;
+        this.valueSerde = this.valueSerde == null ? FullChangeSerde.wrap(valueSerde) : this.valueSerde;
     }
 
     @Override
@@ -296,21 +296,26 @@ private void restoreBatch(final Collection> batch
                     final byte[] changelogValue = new byte[record.value().length - 8];
                     timeAndValue.get(changelogValue);
 
+                    final Change change = requireNonNull(FullChangeSerde.decomposeLegacyFormat(changelogValue));
+
+                    final ProcessorRecordContext recordContext = new ProcessorRecordContext(
+                        record.timestamp(),
+                        record.offset(),
+                        record.partition(),
+                        record.topic(),
+                        record.headers()
+                    );
+
                     cleanPut(
                         time,
                         key,
                         new BufferValue(
-                            new ContextualRecord(
-                                changelogValue,
-                                new ProcessorRecordContext(
-                                    record.timestamp(),
-                                    record.offset(),
-                                    record.partition(),
-                                    record.topic(),
-                                    record.headers()
-                                )
-                            ),
-                            inferPriorValue(key, changelogValue)
+                            index.containsKey(key)
+                                ? internalPriorValueForBuffered(key)
+                                : change.oldValue,
+                            change.oldValue,
+                            change.newValue,
+                            recordContext
                         )
                     );
                 } else if (V_1_CHANGELOG_HEADERS.lastHeader("v").equals(record.headers().lastHeader("v"))) {
@@ -321,7 +326,20 @@ private void restoreBatch(final Collection> batch
                     timeAndValue.get(changelogValue);
 
                     final ContextualRecord contextualRecord = ContextualRecord.deserialize(ByteBuffer.wrap(changelogValue));
-                    cleanPut(time, key, new BufferValue(contextualRecord, inferPriorValue(key, contextualRecord.value())));
+                    final Change change = requireNonNull(FullChangeSerde.decomposeLegacyFormat(contextualRecord.value()));
+
+                    cleanPut(
+                        time,
+                        key,
+                        new BufferValue(
+                            index.containsKey(key)
+                                ? internalPriorValueForBuffered(key)
+                                : change.oldValue,
+                            change.oldValue,
+                            change.newValue,
+                            contextualRecord.recordContext()
+                        )
+                    );
                 } else if (V_2_CHANGELOG_HEADERS.lastHeader("v").equals(record.headers().lastHeader("v"))) {
                     // in this case, the changelog value is a serialized BufferValue
 
@@ -346,13 +364,6 @@ private void restoreBatch(final Collection> batch
         updateBufferMetrics();
     }
 
-    private byte[] inferPriorValue(final Bytes key, final byte[] serializedChange) {
-        return index.containsKey(key)
-            ? internalPriorValueForBuffered(key)
-            : FullChangeSerde.extractOldValuePart(ByteBuffer.wrap(serializedChange));
-    }
-
-
     @Override
     public void evictWhile(final Supplier predicate,
                            final Consumer> callback) {
@@ -375,9 +386,11 @@ public void evictWhile(final Supplier predicate,
                 }
                 final K key = keySerde.deserializer().deserialize(changelogTopic, next.getKey().key().get());
                 final BufferValue bufferValue = next.getValue();
-                final ContextualRecord record = bufferValue.record();
-                final Change value = valueSerde.deserializer().deserialize(changelogTopic, record.value());
-                callback.accept(new Eviction<>(key, value, record.recordContext()));
+                final Change value = valueSerde.deserializeParts(
+                    changelogTopic,
+                    new Change<>(bufferValue.newValue(), bufferValue.oldValue())
+                );
+                callback.accept(new Eviction<>(key, value, bufferValue.context()));
 
                 delegate.remove();
                 index.remove(next.getKey().key());
@@ -442,7 +455,7 @@ public void put(final long time,
         requireNonNull(recordContext, "recordContext cannot be null");
 
         final Bytes serializedKey = Bytes.wrap(keySerde.serializer().serialize(changelogTopic, key));
-        final byte[] serializedValue = valueSerde.serializer().serialize(changelogTopic, value);
+        final Change serialChange = valueSerde.serializeParts(changelogTopic, value);
 
         final BufferValue buffered = getBuffered(serializedKey);
         final byte[] serializedPriorValue;
@@ -453,7 +466,11 @@ public void put(final long time,
             serializedPriorValue = buffered.priorValue();
         }
 
-        cleanPut(time, serializedKey, new BufferValue(new ContextualRecord(serializedValue, recordContext), serializedPriorValue));
+        cleanPut(
+            time,
+            serializedKey,
+            new BufferValue(serializedPriorValue, serialChange.oldValue, serialChange.newValue, recordContext)
+        );
         dirtyKeys.add(serializedKey);
         updateBufferMetrics();
     }
@@ -504,7 +521,7 @@ private static long computeRecordSize(final Bytes key, final BufferValue value)
         size += 8; // buffer time
         size += key.get().length;
         if (value != null) {
-            size += value.sizeBytes();
+            size += value.residentMemorySizeEstimate();
         }
         return size;
     }
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/LRUCacheEntry.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/LRUCacheEntry.java
index f454862f2a8ee..0f1a1acc3ec7b 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/LRUCacheEntry.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/LRUCacheEntry.java
@@ -50,7 +50,7 @@ class LRUCacheEntry {
 
         this.isDirty = isDirty;
         this.sizeBytes = 1 + // isDirty
-            record.sizeBytes();
+            record.residentMemorySizeEstimate();
     }
 
     void markClean() {
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/FullChangeSerdeTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/FullChangeSerdeTest.java
index ddba05e06414f..97e6c0697ea07 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/FullChangeSerdeTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/FullChangeSerdeTest.java
@@ -16,146 +16,74 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
-import org.apache.kafka.common.serialization.Deserializer;
-import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
-import org.apache.kafka.common.serialization.Serializer;
-import org.easymock.EasyMock;
 import org.junit.Test;
 
-import static java.util.Collections.emptyMap;
 import static org.hamcrest.CoreMatchers.nullValue;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.core.Is.is;
 
 public class FullChangeSerdeTest {
-    private final FullChangeSerde serde = FullChangeSerde.castOrWrap(Serdes.String());
+    private final FullChangeSerde serde = FullChangeSerde.wrap(Serdes.String());
 
     @Test
     public void shouldRoundTripNull() {
-        final byte[] serialized = serde.serializer().serialize(null, null);
-        assertThat(
-            serde.deserializer().deserialize(null, serialized),
-            nullValue()
-        );
+        assertThat(serde.serializeParts(null, null), nullValue());
+        assertThat(FullChangeSerde.composeLegacyFormat(null), nullValue());
+        assertThat(FullChangeSerde.decomposeLegacyFormat(null), nullValue());
+        assertThat(serde.deserializeParts(null, null), nullValue());
     }
 
 
     @Test
     public void shouldRoundTripNullChange() {
-        final byte[] serialized = serde.serializer().serialize(null, new Change<>(null, null));
         assertThat(
-            serde.deserializer().deserialize(null, serialized),
-            is(new Change<>(null, null))
+            serde.serializeParts(null, new Change<>(null, null)),
+            is(new Change(null, null))
+        );
+
+        assertThat(
+            serde.deserializeParts(null, new Change<>(null, null)),
+            is(new Change(null, null))
+        );
+
+        final byte[] legacyFormat = FullChangeSerde.composeLegacyFormat(new Change<>(null, null));
+        assertThat(
+            FullChangeSerde.decomposeLegacyFormat(legacyFormat),
+            is(new Change(null, null))
         );
     }
 
     @Test
     public void shouldRoundTripOldNull() {
-        final byte[] serialized = serde.serializer().serialize(null, new Change<>("new", null));
+        final Change serialized = serde.serializeParts(null, new Change<>("new", null));
+        final byte[] legacyFormat = FullChangeSerde.composeLegacyFormat(serialized);
+        final Change decomposedLegacyFormat = FullChangeSerde.decomposeLegacyFormat(legacyFormat);
         assertThat(
-            serde.deserializer().deserialize(null, serialized),
+            serde.deserializeParts(null, decomposedLegacyFormat),
             is(new Change<>("new", null))
         );
     }
 
     @Test
     public void shouldRoundTripNewNull() {
-        final byte[] serialized = serde.serializer().serialize(null, new Change<>(null, "old"));
+        final Change serialized = serde.serializeParts(null, new Change<>(null, "old"));
+        final byte[] legacyFormat = FullChangeSerde.composeLegacyFormat(serialized);
+        final Change decomposedLegacyFormat = FullChangeSerde.decomposeLegacyFormat(legacyFormat);
         assertThat(
-            serde.deserializer().deserialize(null, serialized),
+            serde.deserializeParts(null, decomposedLegacyFormat),
             is(new Change<>(null, "old"))
         );
     }
 
     @Test
     public void shouldRoundTripChange() {
-        final byte[] serialized = serde.serializer().serialize(null, new Change<>("new", "old"));
+        final Change serialized = serde.serializeParts(null, new Change<>("new", "old"));
+        final byte[] legacyFormat = FullChangeSerde.composeLegacyFormat(serialized);
+        final Change decomposedLegacyFormat = FullChangeSerde.decomposeLegacyFormat(legacyFormat);
         assertThat(
-            serde.deserializer().deserialize(null, serialized),
+            serde.deserializeParts(null, decomposedLegacyFormat),
             is(new Change<>("new", "old"))
         );
     }
-
-    @Test
-    public void shouldConfigureSerde() {
-        final Serde mock = EasyMock.mock(Serde.class);
-        mock.configure(emptyMap(), false);
-        EasyMock.expectLastCall();
-        EasyMock.replay(mock);
-        final FullChangeSerde serde = FullChangeSerde.castOrWrap(mock);
-        serde.configure(emptyMap(), false);
-        EasyMock.verify(mock);
-    }
-
-    @Test
-    public void shouldCloseSerde() {
-        final Serde mock = EasyMock.mock(Serde.class);
-        mock.close();
-        EasyMock.expectLastCall();
-        EasyMock.replay(mock);
-        final FullChangeSerde serde = FullChangeSerde.castOrWrap(mock);
-        serde.close();
-        EasyMock.verify(mock);
-    }
-
-    @Test
-    public void shouldConfigureSerializer() {
-        final Serde mockSerde = EasyMock.mock(Serde.class);
-        final Serializer mockSerializer = EasyMock.mock(Serializer.class);
-        EasyMock.expect(mockSerde.serializer()).andReturn(mockSerializer);
-        EasyMock.replay(mockSerde);
-        mockSerializer.configure(emptyMap(), false);
-        EasyMock.expectLastCall();
-        EasyMock.replay(mockSerializer);
-        final Serializer> serializer = FullChangeSerde.castOrWrap(mockSerde).serializer();
-        serializer.configure(emptyMap(), false);
-        EasyMock.verify(mockSerde);
-        EasyMock.verify(mockSerializer);
-    }
-
-    @Test
-    public void shouldCloseSerializer() {
-        final Serde mockSerde = EasyMock.mock(Serde.class);
-        final Serializer mockSerializer = EasyMock.mock(Serializer.class);
-        EasyMock.expect(mockSerde.serializer()).andReturn(mockSerializer);
-        EasyMock.replay(mockSerde);
-        mockSerializer.close();
-        EasyMock.expectLastCall();
-        EasyMock.replay(mockSerializer);
-        final Serializer> serializer = FullChangeSerde.castOrWrap(mockSerde).serializer();
-        serializer.close();
-        EasyMock.verify(mockSerde);
-        EasyMock.verify(mockSerializer);
-    }
-
-    @Test
-    public void shouldConfigureDeserializer() {
-        final Serde mockSerde = EasyMock.mock(Serde.class);
-        final Deserializer mockDeserializer = EasyMock.mock(Deserializer.class);
-        EasyMock.expect(mockSerde.deserializer()).andReturn(mockDeserializer);
-        EasyMock.replay(mockSerde);
-        mockDeserializer.configure(emptyMap(), false);
-        EasyMock.expectLastCall();
-        EasyMock.replay(mockDeserializer);
-        final Deserializer> serializer = FullChangeSerde.castOrWrap(mockSerde).deserializer();
-        serializer.configure(emptyMap(), false);
-        EasyMock.verify(mockSerde);
-        EasyMock.verify(mockDeserializer);
-    }
-
-    @Test
-    public void shouldCloseDeserializer() {
-        final Serde mockSerde = EasyMock.mock(Serde.class);
-        final Deserializer mockDeserializer = EasyMock.mock(Deserializer.class);
-        EasyMock.expect(mockSerde.deserializer()).andReturn(mockDeserializer);
-        EasyMock.replay(mockSerde);
-        mockDeserializer.close();
-        EasyMock.expectLastCall();
-        EasyMock.replay(mockDeserializer);
-        final Deserializer> serializer = FullChangeSerde.castOrWrap(mockSerde).deserializer();
-        serializer.close();
-        EasyMock.verify(mockSerde);
-        EasyMock.verify(mockDeserializer);
-    }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorMetricsTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorMetricsTest.java
index 96ee73553a40e..ef46663309099 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorMetricsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorMetricsTest.java
@@ -21,7 +21,6 @@
 import org.apache.kafka.common.serialization.Serdes;
 import org.apache.kafka.streams.kstream.Suppressed;
 import org.apache.kafka.streams.kstream.internals.Change;
-import org.apache.kafka.streams.kstream.internals.FullChangeSerde;
 import org.apache.kafka.streams.kstream.internals.KTableImpl;
 import org.apache.kafka.streams.processor.Processor;
 import org.apache.kafka.streams.processor.StateStore;
@@ -139,7 +138,7 @@ public void shouldRecordMetrics() {
 
         final StateStore buffer = new InMemoryTimeOrderedKeyValueBuffer.Builder<>(
             storeName, Serdes.String(),
-            FullChangeSerde.castOrWrap(Serdes.Long())
+            Serdes.Long()
         )
             .withLoggingDisabled()
             .build();
@@ -169,9 +168,9 @@ public void shouldRecordMetrics() {
 
             verifyMetric(metrics, EVICTION_RATE_METRIC, is(0.0));
             verifyMetric(metrics, EVICTION_TOTAL_METRIC, is(0.0));
-            verifyMetric(metrics, BUFFER_SIZE_AVG_METRIC, is(29.5));
-            verifyMetric(metrics, BUFFER_SIZE_CURRENT_METRIC, is(59.0));
-            verifyMetric(metrics, BUFFER_SIZE_MAX_METRIC, is(59.0));
+            verifyMetric(metrics, BUFFER_SIZE_AVG_METRIC, is(21.5));
+            verifyMetric(metrics, BUFFER_SIZE_CURRENT_METRIC, is(43.0));
+            verifyMetric(metrics, BUFFER_SIZE_MAX_METRIC, is(43.0));
             verifyMetric(metrics, BUFFER_COUNT_AVG_METRIC, is(0.5));
             verifyMetric(metrics, BUFFER_COUNT_CURRENT_METRIC, is(1.0));
             verifyMetric(metrics, BUFFER_COUNT_MAX_METRIC, is(1.0));
@@ -185,9 +184,9 @@ public void shouldRecordMetrics() {
 
             verifyMetric(metrics, EVICTION_RATE_METRIC, greaterThan(0.0));
             verifyMetric(metrics, EVICTION_TOTAL_METRIC, is(1.0));
-            verifyMetric(metrics, BUFFER_SIZE_AVG_METRIC, is(57.0));
-            verifyMetric(metrics, BUFFER_SIZE_CURRENT_METRIC, is(55.0));
-            verifyMetric(metrics, BUFFER_SIZE_MAX_METRIC, is(114.0));
+            verifyMetric(metrics, BUFFER_SIZE_AVG_METRIC, is(41.0));
+            verifyMetric(metrics, BUFFER_SIZE_CURRENT_METRIC, is(39.0));
+            verifyMetric(metrics, BUFFER_SIZE_MAX_METRIC, is(82.0));
             verifyMetric(metrics, BUFFER_COUNT_AVG_METRIC, is(1.0));
             verifyMetric(metrics, BUFFER_COUNT_CURRENT_METRIC, is(1.0));
             verifyMetric(metrics, BUFFER_COUNT_MAX_METRIC, is(2.0));
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorTest.java
index d8cb858f5f91e..1d1d6fb2ab0c8 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorTest.java
@@ -25,7 +25,6 @@
 import org.apache.kafka.streams.kstream.TimeWindowedSerializer;
 import org.apache.kafka.streams.kstream.Windowed;
 import org.apache.kafka.streams.kstream.internals.Change;
-import org.apache.kafka.streams.kstream.internals.FullChangeSerde;
 import org.apache.kafka.streams.kstream.internals.KTableImpl;
 import org.apache.kafka.streams.kstream.internals.SessionWindow;
 import org.apache.kafka.streams.kstream.internals.TimeWindow;
@@ -75,7 +74,7 @@ private static class Harness {
 
             final String storeName = "test-store";
 
-            final StateStore buffer = new InMemoryTimeOrderedKeyValueBuffer.Builder<>(storeName, keySerde, FullChangeSerde.castOrWrap(valueSerde))
+            final StateStore buffer = new InMemoryTimeOrderedKeyValueBuffer.Builder<>(storeName, keySerde, valueSerde)
                 .withLoggingDisabled()
                 .build();
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/SuppressSuite.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/SuppressSuite.java
index 3aef6d09b0920..a323b9b25bde4 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/SuppressSuite.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/SuppressSuite.java
@@ -19,8 +19,10 @@
 import org.apache.kafka.streams.integration.SuppressionDurabilityIntegrationTest;
 import org.apache.kafka.streams.integration.SuppressionIntegrationTest;
 import org.apache.kafka.streams.kstream.SuppressedTest;
+import org.apache.kafka.streams.kstream.internals.FullChangeSerdeTest;
 import org.apache.kafka.streams.kstream.internals.SuppressScenarioTest;
 import org.apache.kafka.streams.kstream.internals.SuppressTopologyTest;
+import org.apache.kafka.streams.state.internals.BufferValueTest;
 import org.apache.kafka.streams.state.internals.InMemoryTimeOrderedKeyValueBufferTest;
 import org.apache.kafka.streams.state.internals.TimeOrderedKeyValueBufferTest;
 import org.junit.runner.RunWith;
@@ -30,21 +32,25 @@
  * This suite runs all the tests related to the Suppression feature.
  *
  * It can be used from an IDE to selectively just run these tests when developing code related to Suppress.
- * 
+ *
  * If desired, it can also be added to a Gradle build task, although this isn't strictly necessary, since all
  * these tests are already included in the `:streams:test` task.
  */
 @RunWith(Suite.class)
 @Suite.SuiteClasses({
+    BufferValueTest.class,
     KTableSuppressProcessorMetricsTest.class,
     KTableSuppressProcessorTest.class,
     SuppressScenarioTest.class,
     SuppressTopologyTest.class,
     SuppressedTest.class,
-    SuppressionIntegrationTest.class,
-    SuppressionDurabilityIntegrationTest.class,
     InMemoryTimeOrderedKeyValueBufferTest.class,
-    TimeOrderedKeyValueBufferTest.class
+    TimeOrderedKeyValueBufferTest.class,
+    FullChangeSerdeTest.class,
+    SuppressionIntegrationTest.class,
+    SuppressionDurabilityIntegrationTest.class
 })
 public class SuppressSuite {
 }
+
+
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorRecordContextTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorRecordContextTest.java
index 1ea646fce2f3e..83ab1279d0dfb 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorRecordContextTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorRecordContextTest.java
@@ -37,7 +37,7 @@ public void shouldEstimateNullTopicAndNullHeadersAsZeroLength() {
             null
         );
 
-        assertEquals(MIN_SIZE, context.sizeBytes());
+        assertEquals(MIN_SIZE, context.residentMemorySizeEstimate());
     }
 
     @Test
@@ -50,7 +50,7 @@ public void shouldEstimateEmptyHeaderAsZeroLength() {
             new RecordHeaders()
         );
 
-        assertEquals(MIN_SIZE, context.sizeBytes());
+        assertEquals(MIN_SIZE, context.residentMemorySizeEstimate());
     }
 
     @Test
@@ -63,7 +63,7 @@ public void shouldEstimateTopicLength() {
             null
         );
 
-        assertEquals(MIN_SIZE + 5L, context.sizeBytes());
+        assertEquals(MIN_SIZE + 5L, context.residentMemorySizeEstimate());
     }
 
     @Test
@@ -78,7 +78,7 @@ public void shouldEstimateHeadersLength() {
             headers
         );
 
-        assertEquals(MIN_SIZE + 10L + 12L, context.sizeBytes());
+        assertEquals(MIN_SIZE + 10L + 12L, context.residentMemorySizeEstimate());
     }
 
     @Test
@@ -93,6 +93,6 @@ public void shouldEstimateNullValueInHeaderAsZero() {
             headers
         );
 
-        assertEquals(MIN_SIZE + 10L, context.sizeBytes());
+        assertEquals(MIN_SIZE + 10L, context.residentMemorySizeEstimate());
     }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/BufferValueTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/BufferValueTest.java
new file mode 100644
index 0000000000000..d663461a8cf9b
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/BufferValueTest.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.streams.processor.internals.ProcessorRecordContext;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+
+public class BufferValueTest {
+    @Test
+    public void shouldDeduplicateNullValues() {
+        final BufferValue bufferValue = new BufferValue(null, null, null, null);
+        assertSame(bufferValue.priorValue(), bufferValue.oldValue());
+    }
+
+    @Test
+    public void shouldDeduplicateIndenticalValues() {
+        final byte[] bytes = {(byte) 0};
+        final BufferValue bufferValue = new BufferValue(bytes, bytes, null, null);
+        assertSame(bufferValue.priorValue(), bufferValue.oldValue());
+    }
+
+    @Test
+    public void shouldDeduplicateEqualValues() {
+        final BufferValue bufferValue = new BufferValue(new byte[] {(byte) 0}, new byte[] {(byte) 0}, null, null);
+        assertSame(bufferValue.priorValue(), bufferValue.oldValue());
+    }
+
+    @Test
+    public void shouldStoreDifferentValues() {
+        final byte[] priorValue = {(byte) 0};
+        final byte[] oldValue = {(byte) 1};
+        final BufferValue bufferValue = new BufferValue(priorValue, oldValue, null, null);
+        assertSame(priorValue, bufferValue.priorValue());
+        assertSame(oldValue, bufferValue.oldValue());
+    }
+
+    @Test
+    public void shouldStoreDifferentValuesWithPriorNull() {
+        final byte[] priorValue = null;
+        final byte[] oldValue = {(byte) 1};
+        final BufferValue bufferValue = new BufferValue(priorValue, oldValue, null, null);
+        assertNull(bufferValue.priorValue());
+        assertSame(oldValue, bufferValue.oldValue());
+    }
+
+    @Test
+    public void shouldStoreDifferentValuesWithOldNull() {
+        final byte[] priorValue = {(byte) 0};
+        final byte[] oldValue = null;
+        final BufferValue bufferValue = new BufferValue(priorValue, oldValue, null, null);
+        assertSame(priorValue, bufferValue.priorValue());
+        assertNull(bufferValue.oldValue());
+    }
+
+    @Test
+    public void shouldAccountForDeduplicationInSizeEstimate() {
+        final ProcessorRecordContext context = new ProcessorRecordContext(0L, 0L, 0, "topic", null);
+        assertEquals(25L, new BufferValue(null, null, null, context).residentMemorySizeEstimate());
+        assertEquals(26L, new BufferValue(new byte[] {(byte) 0}, null, null, context).residentMemorySizeEstimate());
+        assertEquals(26L, new BufferValue(null, new byte[] {(byte) 0}, null, context).residentMemorySizeEstimate());
+        assertEquals(26L, new BufferValue(new byte[] {(byte) 0}, new byte[] {(byte) 0}, null, context).residentMemorySizeEstimate());
+        assertEquals(27L, new BufferValue(new byte[] {(byte) 0}, new byte[] {(byte) 1}, null, context).residentMemorySizeEstimate());
+
+        // new value should get counted, but doesn't get deduplicated
+        assertEquals(28L, new BufferValue(new byte[] {(byte) 0}, new byte[] {(byte) 1}, new byte[] {(byte) 0}, context).residentMemorySizeEstimate());
+    }
+
+    @Test
+    public void shouldSerializeNulls() {
+        final ProcessorRecordContext context = new ProcessorRecordContext(0L, 0L, 0, "topic", null);
+        final byte[] serializedContext = context.serialize();
+        final byte[] bytes = new BufferValue(null, null, null, context).serialize(0).array();
+        final byte[] withoutContext = Arrays.copyOfRange(bytes, serializedContext.length, bytes.length);
+
+        assertThat(withoutContext, is(ByteBuffer.allocate(Integer.BYTES * 3).putInt(-1).putInt(-1).putInt(-1).array()));
+    }
+
+    @Test
+    public void shouldSerializePrior() {
+        final ProcessorRecordContext context = new ProcessorRecordContext(0L, 0L, 0, "topic", null);
+        final byte[] serializedContext = context.serialize();
+        final byte[] priorValue = {(byte) 5};
+        final byte[] bytes = new BufferValue(priorValue, null, null, context).serialize(0).array();
+        final byte[] withoutContext = Arrays.copyOfRange(bytes, serializedContext.length, bytes.length);
+
+        assertThat(withoutContext, is(ByteBuffer.allocate(Integer.BYTES * 3 + 1).putInt(1).put(priorValue).putInt(-1).putInt(-1).array()));
+    }
+
+    @Test
+    public void shouldSerializeOld() {
+        final ProcessorRecordContext context = new ProcessorRecordContext(0L, 0L, 0, "topic", null);
+        final byte[] serializedContext = context.serialize();
+        final byte[] oldValue = {(byte) 5};
+        final byte[] bytes = new BufferValue(null, oldValue, null, context).serialize(0).array();
+        final byte[] withoutContext = Arrays.copyOfRange(bytes, serializedContext.length, bytes.length);
+
+        assertThat(withoutContext, is(ByteBuffer.allocate(Integer.BYTES * 3 + 1).putInt(-1).putInt(1).put(oldValue).putInt(-1).array()));
+    }
+
+    @Test
+    public void shouldSerializeNew() {
+        final ProcessorRecordContext context = new ProcessorRecordContext(0L, 0L, 0, "topic", null);
+        final byte[] serializedContext = context.serialize();
+        final byte[] newValue = {(byte) 5};
+        final byte[] bytes = new BufferValue(null, null, newValue, context).serialize(0).array();
+        final byte[] withoutContext = Arrays.copyOfRange(bytes, serializedContext.length, bytes.length);
+
+        assertThat(withoutContext, is(ByteBuffer.allocate(Integer.BYTES * 3 + 1).putInt(-1).putInt(-1).putInt(1).put(newValue).array()));
+    }
+
+    @Test
+    public void shouldCompactDuplicates() {
+        final ProcessorRecordContext context = new ProcessorRecordContext(0L, 0L, 0, "topic", null);
+        final byte[] serializedContext = context.serialize();
+        final byte[] duplicate = {(byte) 5};
+        final byte[] bytes = new BufferValue(duplicate, duplicate, null, context).serialize(0).array();
+        final byte[] withoutContext = Arrays.copyOfRange(bytes, serializedContext.length, bytes.length);
+
+        assertThat(withoutContext, is(ByteBuffer.allocate(Integer.BYTES * 3 + 1).putInt(1).put(duplicate).putInt(-2).putInt(-1).array()));
+    }
+
+    @Test
+    public void shouldDeserializePrior() {
+        final ProcessorRecordContext context = new ProcessorRecordContext(0L, 0L, 0, "topic", null);
+        final byte[] serializedContext = context.serialize();
+        final byte[] priorValue = {(byte) 5};
+        final ByteBuffer serialValue =
+            ByteBuffer
+                .allocate(serializedContext.length + Integer.BYTES * 3 + priorValue.length)
+                .put(serializedContext).putInt(1).put(priorValue).putInt(-1).putInt(-1);
+        serialValue.position(0);
+
+        final BufferValue deserialize = BufferValue.deserialize(serialValue);
+        assertThat(deserialize, is(new BufferValue(priorValue, null, null, context)));
+    }
+
+    @Test
+    public void shouldDeserializeOld() {
+        final ProcessorRecordContext context = new ProcessorRecordContext(0L, 0L, 0, "topic", null);
+        final byte[] serializedContext = context.serialize();
+        final byte[] oldValue = {(byte) 5};
+        final ByteBuffer serialValue =
+            ByteBuffer
+                .allocate(serializedContext.length + Integer.BYTES * 3 + oldValue.length)
+                .put(serializedContext).putInt(-1).putInt(1).put(oldValue).putInt(-1);
+        serialValue.position(0);
+
+        assertThat(BufferValue.deserialize(serialValue), is(new BufferValue(null, oldValue, null, context)));
+    }
+
+    @Test
+    public void shouldDeserializeNew() {
+        final ProcessorRecordContext context = new ProcessorRecordContext(0L, 0L, 0, "topic", null);
+        final byte[] serializedContext = context.serialize();
+        final byte[] newValue = {(byte) 5};
+        final ByteBuffer serialValue =
+            ByteBuffer
+                .allocate(serializedContext.length + Integer.BYTES * 3 + newValue.length)
+                .put(serializedContext).putInt(-1).putInt(-1).putInt(1).put(newValue);
+        serialValue.position(0);
+
+        assertThat(BufferValue.deserialize(serialValue), is(new BufferValue(null, null, newValue, context)));
+    }
+
+    @Test
+    public void shouldDeserializeCompactedDuplicates() {
+        final ProcessorRecordContext context = new ProcessorRecordContext(0L, 0L, 0, "topic", null);
+        final byte[] serializedContext = context.serialize();
+        final byte[] duplicate = {(byte) 5};
+        final ByteBuffer serialValue =
+            ByteBuffer
+                .allocate(serializedContext.length + Integer.BYTES * 3 + duplicate.length)
+                .put(serializedContext).putInt(1).put(duplicate).putInt(-2).putInt(-1);
+        serialValue.position(0);
+
+        final BufferValue bufferValue = BufferValue.deserialize(serialValue);
+        assertThat(bufferValue, is(new BufferValue(duplicate, duplicate, null, context)));
+        assertSame(bufferValue.priorValue(), bufferValue.oldValue());
+    }
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/TimeOrderedKeyValueBufferTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/TimeOrderedKeyValueBufferTest.java
index 941832bff3a85..5c9cbf91769df 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/TimeOrderedKeyValueBufferTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/TimeOrderedKeyValueBufferTest.java
@@ -23,7 +23,6 @@
 import org.apache.kafka.common.header.internals.RecordHeaders;
 import org.apache.kafka.common.record.TimestampType;
 import org.apache.kafka.common.serialization.Serdes;
-import org.apache.kafka.common.serialization.Serializer;
 import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsConfig;
@@ -190,11 +189,11 @@ public void shouldTrackSize() {
         final MockInternalProcessorContext context = makeContext();
         buffer.init(context, buffer);
         putRecord(buffer, context, 0L, 0L, "asdf", "23roni");
-        assertThat(buffer.bufferSize(), is(51L));
+        assertThat(buffer.bufferSize(), is(43L));
         putRecord(buffer, context, 1L, 0L, "asdf", "3l");
-        assertThat(buffer.bufferSize(), is(47L));
+        assertThat(buffer.bufferSize(), is(39L));
         putRecord(buffer, context, 0L, 0L, "zxcv", "qfowin");
-        assertThat(buffer.bufferSize(), is(98L));
+        assertThat(buffer.bufferSize(), is(82L));
         cleanup(context, buffer);
     }
 
@@ -218,12 +217,12 @@ public void shouldEvictOldestAndUpdateSizeAndCountAndMinTimestamp() {
 
         putRecord(buffer, context, 1L, 0L, "zxcv", "o23i4");
         assertThat(buffer.numRecords(), is(1));
-        assertThat(buffer.bufferSize(), is(50L));
+        assertThat(buffer.bufferSize(), is(42L));
         assertThat(buffer.minTimestamp(), is(1L));
 
         putRecord(buffer, context, 0L, 0L, "asdf", "3ng");
         assertThat(buffer.numRecords(), is(2));
-        assertThat(buffer.bufferSize(), is(98L));
+        assertThat(buffer.bufferSize(), is(82L));
         assertThat(buffer.minTimestamp(), is(0L));
 
         final AtomicInteger callbackCount = new AtomicInteger(0);
@@ -232,14 +231,14 @@ public void shouldEvictOldestAndUpdateSizeAndCountAndMinTimestamp() {
                 case 1: {
                     assertThat(kv.key(), is("asdf"));
                     assertThat(buffer.numRecords(), is(2));
-                    assertThat(buffer.bufferSize(), is(98L));
+                    assertThat(buffer.bufferSize(), is(82L));
                     assertThat(buffer.minTimestamp(), is(0L));
                     break;
                 }
                 case 2: {
                     assertThat(kv.key(), is("zxcv"));
                     assertThat(buffer.numRecords(), is(1));
-                    assertThat(buffer.bufferSize(), is(50L));
+                    assertThat(buffer.bufferSize(), is(42L));
                     assertThat(buffer.minTimestamp(), is(1L));
                     break;
                 }
@@ -361,12 +360,12 @@ public void shouldRestoreOldFormat() {
 
         context.setRecordContext(new ProcessorRecordContext(0, 0, 0, "", null));
 
-        final Serializer> serializer = FullChangeSerde.castOrWrap(Serdes.String()).serializer();
+        final FullChangeSerde serializer = FullChangeSerde.wrap(Serdes.String());
 
-        final byte[] todeleteValue = serializer.serialize(null, new Change<>("doomed", null));
-        final byte[] asdfValue = serializer.serialize(null, new Change<>("qwer", null));
-        final byte[] zxcvValue1 = serializer.serialize(null, new Change<>("eo4im", "previous"));
-        final byte[] zxcvValue2 = serializer.serialize(null, new Change<>("next", "eo4im"));
+        final byte[] todeleteValue = FullChangeSerde.composeLegacyFormat(serializer.serializeParts(null, new Change<>("doomed", null)));
+        final byte[] asdfValue = FullChangeSerde.composeLegacyFormat(serializer.serializeParts(null, new Change<>("qwer", null)));
+        final byte[] zxcvValue1 = FullChangeSerde.composeLegacyFormat(serializer.serializeParts(null, new Change<>("eo4im", "previous")));
+        final byte[] zxcvValue2 = FullChangeSerde.composeLegacyFormat(serializer.serializeParts(null, new Change<>("next", "eo4im")));
         stateRestoreCallback.restoreBatch(asList(
             new ConsumerRecord<>("changelog-topic",
                                  0,
@@ -412,7 +411,7 @@ public void shouldRestoreOldFormat() {
 
         assertThat(buffer.numRecords(), is(3));
         assertThat(buffer.minTimestamp(), is(0L));
-        assertThat(buffer.bufferSize(), is(196L));
+        assertThat(buffer.bufferSize(), is(172L));
 
         stateRestoreCallback.restoreBatch(singletonList(
             new ConsumerRecord<>("changelog-topic",
@@ -429,7 +428,7 @@ public void shouldRestoreOldFormat() {
 
         assertThat(buffer.numRecords(), is(2));
         assertThat(buffer.minTimestamp(), is(1L));
-        assertThat(buffer.bufferSize(), is(131L));
+        assertThat(buffer.bufferSize(), is(115L));
 
         assertThat(buffer.priorValueForBuffered("todelete"), is(Maybe.undefined()));
         assertThat(buffer.priorValueForBuffered("asdf"), is(Maybe.defined(null)));
@@ -477,14 +476,13 @@ public void shouldRestoreV1Format() {
 
         final byte[] todeleteValue = getContextualRecord("doomed", 0).serialize(0).array();
         final byte[] asdfValue = getContextualRecord("qwer", 1).serialize(0).array();
-        final FullChangeSerde fullChangeSerde = FullChangeSerde.castOrWrap(Serdes.String());
+        final FullChangeSerde fullChangeSerde = FullChangeSerde.wrap(Serdes.String());
         final byte[] zxcvValue1 = new ContextualRecord(
-            fullChangeSerde.serializer().serialize(null, new Change<>("3o4im", "previous")),
+            FullChangeSerde.composeLegacyFormat(fullChangeSerde.serializeParts(null, new Change<>("3o4im", "previous"))),
             getContext(2L)
         ).serialize(0).array();
-        final FullChangeSerde fullChangeSerde1 = FullChangeSerde.castOrWrap(Serdes.String());
         final byte[] zxcvValue2 = new ContextualRecord(
-            fullChangeSerde1.serializer().serialize(null, new Change<>("next", "3o4im")),
+            FullChangeSerde.composeLegacyFormat(fullChangeSerde.serializeParts(null, new Change<>("next", "3o4im"))),
             getContext(3L)
         ).serialize(0).array();
         stateRestoreCallback.restoreBatch(asList(
@@ -536,7 +534,7 @@ public void shouldRestoreV1Format() {
 
         assertThat(buffer.numRecords(), is(3));
         assertThat(buffer.minTimestamp(), is(0L));
-        assertThat(buffer.bufferSize(), is(166L));
+        assertThat(buffer.bufferSize(), is(142L));
 
         stateRestoreCallback.restoreBatch(singletonList(
             new ConsumerRecord<>("changelog-topic",
@@ -553,7 +551,7 @@ public void shouldRestoreV1Format() {
 
         assertThat(buffer.numRecords(), is(2));
         assertThat(buffer.minTimestamp(), is(1L));
-        assertThat(buffer.bufferSize(), is(111L));
+        assertThat(buffer.bufferSize(), is(95L));
 
         assertThat(buffer.priorValueForBuffered("todelete"), is(Maybe.undefined()));
         assertThat(buffer.priorValueForBuffered("asdf"), is(Maybe.defined(null)));
@@ -601,23 +599,21 @@ public void shouldRestoreV2Format() {
 
         final byte[] todeleteValue = getBufferValue("doomed", 0).serialize(0).array();
         final byte[] asdfValue = getBufferValue("qwer", 1).serialize(0).array();
-        final FullChangeSerde fullChangeSerde = FullChangeSerde.castOrWrap(Serdes.String());
+        final FullChangeSerde fullChangeSerde = FullChangeSerde.wrap(Serdes.String());
         final byte[] zxcvValue1 =
             new BufferValue(
-                new ContextualRecord(
-                    fullChangeSerde.serializer().serialize(null, new Change<>("3o4im", "IGNORED")),
-                    getContext(2L)
-                ),
-                Serdes.String().serializer().serialize(null, "previous")
+                Serdes.String().serializer().serialize(null, "previous"),
+                Serdes.String().serializer().serialize(null, "IGNORED"),
+                Serdes.String().serializer().serialize(null, "3o4im"),
+                getContext(2L)
             ).serialize(0).array();
-        final FullChangeSerde fullChangeSerde1 = FullChangeSerde.castOrWrap(Serdes.String());
+        final FullChangeSerde fullChangeSerde1 = FullChangeSerde.wrap(Serdes.String());
         final byte[] zxcvValue2 =
             new BufferValue(
-                new ContextualRecord(
-                    fullChangeSerde1.serializer().serialize(null, new Change<>("next", "3o4im")),
-                    getContext(3L)
-                ),
-                Serdes.String().serializer().serialize(null, "previous")
+                Serdes.String().serializer().serialize(null, "previous"),
+                Serdes.String().serializer().serialize(null, "3o4im"),
+                Serdes.String().serializer().serialize(null, "next"),
+                getContext(3L)
             ).serialize(0).array();
         stateRestoreCallback.restoreBatch(asList(
             new ConsumerRecord<>("changelog-topic",
@@ -668,7 +664,7 @@ public void shouldRestoreV2Format() {
 
         assertThat(buffer.numRecords(), is(3));
         assertThat(buffer.minTimestamp(), is(0L));
-        assertThat(buffer.bufferSize(), is(166L));
+        assertThat(buffer.bufferSize(), is(142L));
 
         stateRestoreCallback.restoreBatch(singletonList(
             new ConsumerRecord<>("changelog-topic",
@@ -685,7 +681,7 @@ public void shouldRestoreV2Format() {
 
         assertThat(buffer.numRecords(), is(2));
         assertThat(buffer.minTimestamp(), is(1L));
-        assertThat(buffer.bufferSize(), is(111L));
+        assertThat(buffer.bufferSize(), is(95L));
 
         assertThat(buffer.priorValueForBuffered("todelete"), is(Maybe.undefined()));
         assertThat(buffer.priorValueForBuffered("asdf"), is(Maybe.defined(null)));
@@ -766,14 +762,18 @@ private static void putRecord(final TimeOrderedKeyValueBuffer bu
     }
 
     private static BufferValue getBufferValue(final String value, final long timestamp) {
-        final ContextualRecord contextualRecord = getContextualRecord(value, timestamp);
-        return new BufferValue(contextualRecord, null);
+        return new BufferValue(
+            null,
+            null,
+            Serdes.String().serializer().serialize(null, value),
+            getContext(timestamp)
+        );
     }
 
     private static ContextualRecord getContextualRecord(final String value, final long timestamp) {
-        final FullChangeSerde fullChangeSerde = FullChangeSerde.castOrWrap(Serdes.String());
+        final FullChangeSerde fullChangeSerde = FullChangeSerde.wrap(Serdes.String());
         return new ContextualRecord(
-            fullChangeSerde.serializer().serialize(null, new Change<>(value, null)),
+            FullChangeSerde.composeLegacyFormat(fullChangeSerde.serializeParts(null, new Change<>(value, null))),
             getContext(timestamp)
         );
     }

From 48b65ac617a17cb5570e152256c1073619cfaabe Mon Sep 17 00:00:00 2001
From: cadonna 
Date: Thu, 13 Jun 2019 23:57:26 +0200
Subject: [PATCH 0364/1071] MINOR: Refactor `MetricsIntegrationTest` (#6930)

Reviewers: Sophie Blee-Goldman ,  Bill Bejeck 
---
 .../integration/MetricsIntegrationTest.java   | 433 +++++++++---------
 1 file changed, 216 insertions(+), 217 deletions(-)

diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
index a3c5b8753c206..0f727d39a5505 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
@@ -27,8 +27,6 @@
 import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
 import org.apache.kafka.streams.integration.utils.IntegrationTestUtils;
 import org.apache.kafka.streams.kstream.Consumed;
-import org.apache.kafka.streams.kstream.KGroupedStream;
-import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.Materialized;
 import org.apache.kafka.streams.kstream.Produced;
 import org.apache.kafka.streams.kstream.SessionWindows;
@@ -61,8 +59,7 @@ public class MetricsIntegrationTest {
     private static final int NUM_BROKERS = 1;
 
     @ClassRule
-    public static final EmbeddedKafkaCluster CLUSTER =
-            new EmbeddedKafkaCluster(NUM_BROKERS);
+    public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(NUM_BROKERS);
 
     // Metric group
     private static final String STREAM_THREAD_NODE_METRICS = "stream-metrics";
@@ -156,10 +153,6 @@ public class MetricsIntegrationTest {
     private static final String MY_STORE_PERSISTENT_KEY_VALUE = "myStorePersistentKeyValue";
     private static final String MY_STORE_LRU_MAP = "myStoreLruMap";
 
-    private StreamsBuilder builder;
-    private Properties streamsConfiguration;
-    private KafkaStreams kafkaStreams;
-
     // topic names
     private static final String STREAM_INPUT = "STREAM_INPUT";
     private static final String STREAM_OUTPUT_1 = "STREAM_OUTPUT_1";
@@ -167,17 +160,16 @@ public class MetricsIntegrationTest {
     private static final String STREAM_OUTPUT_3 = "STREAM_OUTPUT_3";
     private static final String STREAM_OUTPUT_4 = "STREAM_OUTPUT_4";
 
-    private KStream stream;
-    private KStream stream2;
-
-    private final String appId = "stream-metrics-test";
+    private StreamsBuilder builder;
+    private Properties streamsConfiguration;
+    private KafkaStreams kafkaStreams;
 
     @Before
     public void before() throws InterruptedException {
         builder = new StreamsBuilder();
         CLUSTER.createTopics(STREAM_INPUT, STREAM_OUTPUT_1, STREAM_OUTPUT_2, STREAM_OUTPUT_3, STREAM_OUTPUT_4);
         streamsConfiguration = new Properties();
-        streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, appId);
+        streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, "stream-metrics-test");
         streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
         streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass());
         streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
@@ -212,26 +204,22 @@ private void closeApplication() throws Exception {
             () -> "Kafka Streams application did not reach state NOT_RUNNING in " + timeout + " ms");
     }
 
-    private void checkMetricDeregistration() {
-        final List listMetricAfterClosingApp = new ArrayList(kafkaStreams.metrics().values()).stream()
-            .filter(m -> m.metricName().group().contains(STREAM_STRING)).collect(Collectors.toList());
-        assertThat(listMetricAfterClosingApp.size(), is(0));
-    }
-
     @Test
-    public void testStreamMetric() throws Exception {
-        stream = builder.stream(STREAM_INPUT, Consumed.with(Serdes.Integer(), Serdes.String()));
-        stream.to(STREAM_OUTPUT_1, Produced.with(Serdes.Integer(), Serdes.String()));
-        builder.table(STREAM_OUTPUT_1, Materialized.as(Stores.inMemoryKeyValueStore(MY_STORE_IN_MEMORY)).withCachingEnabled())
-                .toStream()
-                .to(STREAM_OUTPUT_2);
-        builder.table(STREAM_OUTPUT_2, Materialized.as(Stores.persistentKeyValueStore(MY_STORE_PERSISTENT_KEY_VALUE)).withCachingEnabled())
-                .toStream()
-                .to(STREAM_OUTPUT_3);
-        builder.table(STREAM_OUTPUT_3, Materialized.as(Stores.lruMap(MY_STORE_LRU_MAP, 10000)).withCachingEnabled())
-                .toStream()
-                .to(STREAM_OUTPUT_4);
-
+    public void shouldAddMetricsOnAllLevels() throws Exception {
+        builder.stream(STREAM_INPUT, Consumed.with(Serdes.Integer(), Serdes.String()))
+            .to(STREAM_OUTPUT_1, Produced.with(Serdes.Integer(), Serdes.String()));
+        builder.table(STREAM_OUTPUT_1,
+                      Materialized.as(Stores.inMemoryKeyValueStore(MY_STORE_IN_MEMORY)).withCachingEnabled())
+            .toStream()
+            .to(STREAM_OUTPUT_2);
+        builder.table(STREAM_OUTPUT_2,
+                      Materialized.as(Stores.persistentKeyValueStore(MY_STORE_PERSISTENT_KEY_VALUE)).withCachingEnabled())
+            .toStream()
+            .to(STREAM_OUTPUT_3);
+        builder.table(STREAM_OUTPUT_3,
+                      Materialized.as(Stores.lruMap(MY_STORE_LRU_MAP, 10000)).withCachingEnabled())
+            .toStream()
+            .to(STREAM_OUTPUT_4);
         startApplication();
 
         checkThreadLevelMetrics();
@@ -244,241 +232,252 @@ public void testStreamMetric() throws Exception {
 
         closeApplication();
 
-        // check all metrics de-registered
-        checkMetricDeregistration();
+        checkMetricsDeregistration();
     }
 
     @Test
-    public void testStreamMetricOfWindowStore() throws Exception {
-        stream2 = builder.stream(STREAM_INPUT, Consumed.with(Serdes.Integer(), Serdes.String()));
-        final KGroupedStream groupedStream = stream2.groupByKey();
-        groupedStream.windowedBy(TimeWindows.of(Duration.ofMillis(50)))
-                .aggregate(() -> 0L, (aggKey, newValue, aggValue) -> aggValue,
-                        Materialized.>as(TIME_WINDOWED_AGGREGATED_STREAM_STORE)
-                                .withValueSerde(Serdes.Long()));
-
+    public void shouldAddMetricsForWindowStore() throws Exception {
+        builder.stream(STREAM_INPUT, Consumed.with(Serdes.Integer(), Serdes.String()))
+            .groupByKey()
+            .windowedBy(TimeWindows.of(Duration.ofMillis(50)))
+            .aggregate(() -> 0L,
+                (aggKey, newValue, aggValue) -> aggValue,
+                Materialized.>as(TIME_WINDOWED_AGGREGATED_STREAM_STORE)
+                    .withValueSerde(Serdes.Long()));
         startApplication();
 
         checkWindowStoreMetrics();
 
         closeApplication();
 
-        // check all metrics de-registered
-        checkMetricDeregistration();
+        checkMetricsDeregistration();
     }
 
     @Test
-    public void testStreamMetricOfSessionStore() throws Exception {
-        stream2 = builder.stream(STREAM_INPUT, Consumed.with(Serdes.Integer(), Serdes.String()));
-        final KGroupedStream groupedStream = stream2.groupByKey();
-        groupedStream.windowedBy(SessionWindows.with(Duration.ofMillis(50)))
-                .aggregate(() -> 0L, (aggKey, newValue, aggValue) -> aggValue, (aggKey, leftAggValue, rightAggValue) -> leftAggValue,
-                        Materialized.>as(SESSION_AGGREGATED_STREAM_STORE)
-                                .withValueSerde(Serdes.Long()));
-
+    public void shouldAddMetricsForSessionStore() throws Exception {
+        builder.stream(STREAM_INPUT, Consumed.with(Serdes.Integer(), Serdes.String()))
+            .groupByKey()
+            .windowedBy(SessionWindows.with(Duration.ofMillis(50)))
+            .aggregate(() -> 0L,
+                (aggKey, newValue, aggValue) -> aggValue,
+                (aggKey, leftAggValue, rightAggValue) -> leftAggValue,
+                Materialized.>as(SESSION_AGGREGATED_STREAM_STORE)
+                    .withValueSerde(Serdes.Long()));
         startApplication();
 
         checkSessionStoreMetrics();
 
         closeApplication();
 
-        // check all metrics de-registered
-        checkMetricDeregistration();
-    }
-
-    private void checkTaskLevelMetrics() {
-        final List listMetricTask = new ArrayList(kafkaStreams.metrics().values()).stream()
-            .filter(m -> m.metricName().group().equals(STREAM_TASK_NODE_METRICS)).collect(Collectors.toList());
-        testMetricByName(listMetricTask, COMMIT_LATENCY_AVG, 5);
-        testMetricByName(listMetricTask, COMMIT_LATENCY_MAX, 5);
-        testMetricByName(listMetricTask, COMMIT_RATE, 5);
-        testMetricByName(listMetricTask, COMMIT_TOTAL, 5);
-        testMetricByName(listMetricTask, RECORD_LATENESS_AVG, 4);
-        testMetricByName(listMetricTask, RECORD_LATENESS_MAX, 4);
+        checkMetricsDeregistration();
     }
 
     private void checkThreadLevelMetrics() {
         final List listMetricThread = new ArrayList(kafkaStreams.metrics().values()).stream()
-            .filter(m -> m.metricName().group().equals(STREAM_THREAD_NODE_METRICS)).collect(Collectors.toList());
-        testMetricByName(listMetricThread, COMMIT_LATENCY_AVG, 1);
-        testMetricByName(listMetricThread, COMMIT_LATENCY_MAX, 1);
-        testMetricByName(listMetricThread, POLL_LATENCY_AVG, 1);
-        testMetricByName(listMetricThread, POLL_LATENCY_MAX, 1);
-        testMetricByName(listMetricThread, PROCESS_LATENCY_AVG, 1);
-        testMetricByName(listMetricThread, PROCESS_LATENCY_MAX, 1);
-        testMetricByName(listMetricThread, PUNCTUATE_LATENCY_AVG, 1);
-        testMetricByName(listMetricThread, PUNCTUATE_LATENCY_MAX, 1);
-        testMetricByName(listMetricThread, COMMIT_RATE, 1);
-        testMetricByName(listMetricThread, COMMIT_TOTAL, 1);
-        testMetricByName(listMetricThread, POLL_RATE, 1);
-        testMetricByName(listMetricThread, POLL_TOTAL, 1);
-        testMetricByName(listMetricThread, PROCESS_RATE, 1);
-        testMetricByName(listMetricThread, PROCESS_TOTAL, 1);
-        testMetricByName(listMetricThread, PUNCTUATE_RATE, 1);
-        testMetricByName(listMetricThread, PUNCTUATE_TOTAL, 1);
-        testMetricByName(listMetricThread, TASK_CREATED_RATE, 1);
-        testMetricByName(listMetricThread, TASK_CREATED_TOTAL, 1);
-        testMetricByName(listMetricThread, TASK_CLOSED_RATE, 1);
-        testMetricByName(listMetricThread, TASK_CLOSED_TOTAL, 1);
-        testMetricByName(listMetricThread, SKIPPED_RECORDS_RATE, 1);
-        testMetricByName(listMetricThread, SKIPPED_RECORDS_TOTAL, 1);
+            .filter(m -> m.metricName().group().equals(STREAM_THREAD_NODE_METRICS))
+            .collect(Collectors.toList());
+        checkMetricByName(listMetricThread, COMMIT_LATENCY_AVG, 1);
+        checkMetricByName(listMetricThread, COMMIT_LATENCY_MAX, 1);
+        checkMetricByName(listMetricThread, POLL_LATENCY_AVG, 1);
+        checkMetricByName(listMetricThread, POLL_LATENCY_MAX, 1);
+        checkMetricByName(listMetricThread, PROCESS_LATENCY_AVG, 1);
+        checkMetricByName(listMetricThread, PROCESS_LATENCY_MAX, 1);
+        checkMetricByName(listMetricThread, PUNCTUATE_LATENCY_AVG, 1);
+        checkMetricByName(listMetricThread, PUNCTUATE_LATENCY_MAX, 1);
+        checkMetricByName(listMetricThread, COMMIT_RATE, 1);
+        checkMetricByName(listMetricThread, COMMIT_TOTAL, 1);
+        checkMetricByName(listMetricThread, POLL_RATE, 1);
+        checkMetricByName(listMetricThread, POLL_TOTAL, 1);
+        checkMetricByName(listMetricThread, PROCESS_RATE, 1);
+        checkMetricByName(listMetricThread, PROCESS_TOTAL, 1);
+        checkMetricByName(listMetricThread, PUNCTUATE_RATE, 1);
+        checkMetricByName(listMetricThread, PUNCTUATE_TOTAL, 1);
+        checkMetricByName(listMetricThread, TASK_CREATED_RATE, 1);
+        checkMetricByName(listMetricThread, TASK_CREATED_TOTAL, 1);
+        checkMetricByName(listMetricThread, TASK_CLOSED_RATE, 1);
+        checkMetricByName(listMetricThread, TASK_CLOSED_TOTAL, 1);
+        checkMetricByName(listMetricThread, SKIPPED_RECORDS_RATE, 1);
+        checkMetricByName(listMetricThread, SKIPPED_RECORDS_TOTAL, 1);
+    }
+
+    private void checkTaskLevelMetrics() {
+        final List listMetricTask = new ArrayList(kafkaStreams.metrics().values()).stream()
+            .filter(m -> m.metricName().group().equals(STREAM_TASK_NODE_METRICS))
+            .collect(Collectors.toList());
+        checkMetricByName(listMetricTask, COMMIT_LATENCY_AVG, 5);
+        checkMetricByName(listMetricTask, COMMIT_LATENCY_MAX, 5);
+        checkMetricByName(listMetricTask, COMMIT_RATE, 5);
+        checkMetricByName(listMetricTask, COMMIT_TOTAL, 5);
+        checkMetricByName(listMetricTask, RECORD_LATENESS_AVG, 4);
+        checkMetricByName(listMetricTask, RECORD_LATENESS_MAX, 4);
     }
 
     private void checkProcessorLevelMetrics() {
         final List listMetricProcessor = new ArrayList(kafkaStreams.metrics().values()).stream()
-            .filter(m -> m.metricName().group().equals(STREAM_PROCESSOR_NODE_METRICS)).collect(Collectors.toList());
-        testMetricByName(listMetricProcessor, PROCESS_LATENCY_AVG, 18);
-        testMetricByName(listMetricProcessor, PROCESS_LATENCY_MAX, 18);
-        testMetricByName(listMetricProcessor, PUNCTUATE_LATENCY_AVG, 18);
-        testMetricByName(listMetricProcessor, PUNCTUATE_LATENCY_MAX, 18);
-        testMetricByName(listMetricProcessor, CREATE_LATENCY_AVG, 18);
-        testMetricByName(listMetricProcessor, CREATE_LATENCY_MAX, 18);
-        testMetricByName(listMetricProcessor, DESTROY_LATENCY_AVG, 18);
-        testMetricByName(listMetricProcessor, DESTROY_LATENCY_MAX, 18);
-        testMetricByName(listMetricProcessor, PROCESS_RATE, 18);
-        testMetricByName(listMetricProcessor, PROCESS_TOTAL, 18);
-        testMetricByName(listMetricProcessor, PUNCTUATE_RATE, 18);
-        testMetricByName(listMetricProcessor, PUNCTUATE_TOTAL, 18);
-        testMetricByName(listMetricProcessor, CREATE_RATE, 18);
-        testMetricByName(listMetricProcessor, CREATE_TOTAL, 18);
-        testMetricByName(listMetricProcessor, DESTROY_RATE, 18);
-        testMetricByName(listMetricProcessor, DESTROY_TOTAL, 18);
-        testMetricByName(listMetricProcessor, FORWARD_TOTAL, 18);
+            .filter(m -> m.metricName().group().equals(STREAM_PROCESSOR_NODE_METRICS))
+            .collect(Collectors.toList());
+        checkMetricByName(listMetricProcessor, PROCESS_LATENCY_AVG, 18);
+        checkMetricByName(listMetricProcessor, PROCESS_LATENCY_MAX, 18);
+        checkMetricByName(listMetricProcessor, PUNCTUATE_LATENCY_AVG, 18);
+        checkMetricByName(listMetricProcessor, PUNCTUATE_LATENCY_MAX, 18);
+        checkMetricByName(listMetricProcessor, CREATE_LATENCY_AVG, 18);
+        checkMetricByName(listMetricProcessor, CREATE_LATENCY_MAX, 18);
+        checkMetricByName(listMetricProcessor, DESTROY_LATENCY_AVG, 18);
+        checkMetricByName(listMetricProcessor, DESTROY_LATENCY_MAX, 18);
+        checkMetricByName(listMetricProcessor, PROCESS_RATE, 18);
+        checkMetricByName(listMetricProcessor, PROCESS_TOTAL, 18);
+        checkMetricByName(listMetricProcessor, PUNCTUATE_RATE, 18);
+        checkMetricByName(listMetricProcessor, PUNCTUATE_TOTAL, 18);
+        checkMetricByName(listMetricProcessor, CREATE_RATE, 18);
+        checkMetricByName(listMetricProcessor, CREATE_TOTAL, 18);
+        checkMetricByName(listMetricProcessor, DESTROY_RATE, 18);
+        checkMetricByName(listMetricProcessor, DESTROY_TOTAL, 18);
+        checkMetricByName(listMetricProcessor, FORWARD_TOTAL, 18);
     }
 
     private void checkKeyValueStoreMetricsByType(final String storeType) {
         final List listMetricStore = new ArrayList(kafkaStreams.metrics().values()).stream()
             .filter(m -> m.metricName().group().equals(storeType))
             .collect(Collectors.toList());
-        testMetricByName(listMetricStore, PUT_LATENCY_AVG, 2);
-        testMetricByName(listMetricStore, PUT_LATENCY_MAX, 2);
-        testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_AVG, 2);
-        testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_MAX, 2);
-        testMetricByName(listMetricStore, GET_LATENCY_AVG, 2);
-        testMetricByName(listMetricStore, GET_LATENCY_MAX, 2);
-        testMetricByName(listMetricStore, DELETE_LATENCY_AVG, 2);
-        testMetricByName(listMetricStore, DELETE_LATENCY_MAX, 2);
-        testMetricByName(listMetricStore, PUT_ALL_LATENCY_AVG, 2);
-        testMetricByName(listMetricStore, PUT_ALL_LATENCY_MAX, 2);
-        testMetricByName(listMetricStore, ALL_LATENCY_AVG, 2);
-        testMetricByName(listMetricStore, ALL_LATENCY_MAX, 2);
-        testMetricByName(listMetricStore, RANGE_LATENCY_AVG, 2);
-        testMetricByName(listMetricStore, RANGE_LATENCY_MAX, 2);
-        testMetricByName(listMetricStore, FLUSH_LATENCY_AVG, 2);
-        testMetricByName(listMetricStore, FLUSH_LATENCY_MAX, 2);
-        testMetricByName(listMetricStore, RESTORE_LATENCY_AVG, 2);
-        testMetricByName(listMetricStore, RESTORE_LATENCY_MAX, 2);
-        testMetricByName(listMetricStore, PUT_RATE, 2);
-        testMetricByName(listMetricStore, PUT_TOTAL, 2);
-        testMetricByName(listMetricStore, PUT_IF_ABSENT_RATE, 2);
-        testMetricByName(listMetricStore, PUT_IF_ABSENT_TOTAL, 2);
-        testMetricByName(listMetricStore, GET_RATE, 2);
-        testMetricByName(listMetricStore, DELETE_RATE, 2);
-        testMetricByName(listMetricStore, DELETE_TOTAL, 2);
-        testMetricByName(listMetricStore, PUT_ALL_RATE, 2);
-        testMetricByName(listMetricStore, PUT_ALL_TOTAL, 2);
-        testMetricByName(listMetricStore, ALL_RATE, 2);
-        testMetricByName(listMetricStore, ALL_TOTAL, 2);
-        testMetricByName(listMetricStore, RANGE_RATE, 2);
-        testMetricByName(listMetricStore, RANGE_TOTAL, 2);
-        testMetricByName(listMetricStore, FLUSH_RATE, 2);
-        testMetricByName(listMetricStore, FLUSH_TOTAL, 2);
-        testMetricByName(listMetricStore, RESTORE_RATE, 2);
-        testMetricByName(listMetricStore, RESTORE_TOTAL, 2);
+        checkMetricByName(listMetricStore, PUT_LATENCY_AVG, 2);
+        checkMetricByName(listMetricStore, PUT_LATENCY_MAX, 2);
+        checkMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_AVG, 2);
+        checkMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_MAX, 2);
+        checkMetricByName(listMetricStore, GET_LATENCY_AVG, 2);
+        checkMetricByName(listMetricStore, GET_LATENCY_MAX, 2);
+        checkMetricByName(listMetricStore, DELETE_LATENCY_AVG, 2);
+        checkMetricByName(listMetricStore, DELETE_LATENCY_MAX, 2);
+        checkMetricByName(listMetricStore, PUT_ALL_LATENCY_AVG, 2);
+        checkMetricByName(listMetricStore, PUT_ALL_LATENCY_MAX, 2);
+        checkMetricByName(listMetricStore, ALL_LATENCY_AVG, 2);
+        checkMetricByName(listMetricStore, ALL_LATENCY_MAX, 2);
+        checkMetricByName(listMetricStore, RANGE_LATENCY_AVG, 2);
+        checkMetricByName(listMetricStore, RANGE_LATENCY_MAX, 2);
+        checkMetricByName(listMetricStore, FLUSH_LATENCY_AVG, 2);
+        checkMetricByName(listMetricStore, FLUSH_LATENCY_MAX, 2);
+        checkMetricByName(listMetricStore, RESTORE_LATENCY_AVG, 2);
+        checkMetricByName(listMetricStore, RESTORE_LATENCY_MAX, 2);
+        checkMetricByName(listMetricStore, PUT_RATE, 2);
+        checkMetricByName(listMetricStore, PUT_TOTAL, 2);
+        checkMetricByName(listMetricStore, PUT_IF_ABSENT_RATE, 2);
+        checkMetricByName(listMetricStore, PUT_IF_ABSENT_TOTAL, 2);
+        checkMetricByName(listMetricStore, GET_RATE, 2);
+        checkMetricByName(listMetricStore, DELETE_RATE, 2);
+        checkMetricByName(listMetricStore, DELETE_TOTAL, 2);
+        checkMetricByName(listMetricStore, PUT_ALL_RATE, 2);
+        checkMetricByName(listMetricStore, PUT_ALL_TOTAL, 2);
+        checkMetricByName(listMetricStore, ALL_RATE, 2);
+        checkMetricByName(listMetricStore, ALL_TOTAL, 2);
+        checkMetricByName(listMetricStore, RANGE_RATE, 2);
+        checkMetricByName(listMetricStore, RANGE_TOTAL, 2);
+        checkMetricByName(listMetricStore, FLUSH_RATE, 2);
+        checkMetricByName(listMetricStore, FLUSH_TOTAL, 2);
+        checkMetricByName(listMetricStore, RESTORE_RATE, 2);
+        checkMetricByName(listMetricStore, RESTORE_TOTAL, 2);
+    }
+
+    private void checkMetricsDeregistration() {
+        final List listMetricAfterClosingApp = new ArrayList(kafkaStreams.metrics().values()).stream()
+            .filter(m -> m.metricName().group().contains(STREAM_STRING))
+            .collect(Collectors.toList());
+        assertThat(listMetricAfterClosingApp.size(), is(0));
     }
 
     private void checkCacheMetrics() {
         final List listMetricCache = new ArrayList(kafkaStreams.metrics().values()).stream()
-            .filter(m -> m.metricName().group().equals(STREAM_CACHE_NODE_METRICS)).collect(Collectors.toList());
-        testMetricByName(listMetricCache, HIT_RATIO_AVG, 6);
-        testMetricByName(listMetricCache, HIT_RATIO_MIN, 6);
-        testMetricByName(listMetricCache, HIT_RATIO_MAX, 6);
+            .filter(m -> m.metricName().group().equals(STREAM_CACHE_NODE_METRICS))
+            .collect(Collectors.toList());
+        checkMetricByName(listMetricCache, HIT_RATIO_AVG, 6);
+        checkMetricByName(listMetricCache, HIT_RATIO_MIN, 6);
+        checkMetricByName(listMetricCache, HIT_RATIO_MAX, 6);
     }
 
     private void checkWindowStoreMetrics() {
         final List listMetricStore = new ArrayList(kafkaStreams.metrics().values()).stream()
             .filter(m -> m.metricName().group().equals(STREAM_STORE_WINDOW_ROCKSDB_STATE_METRICS))
             .collect(Collectors.toList());
-        testMetricByName(listMetricStore, PUT_LATENCY_AVG, 2);
-        testMetricByName(listMetricStore, PUT_LATENCY_MAX, 2);
-        testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_AVG, 0);
-        testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_MAX, 0);
-        testMetricByName(listMetricStore, GET_LATENCY_AVG, 0);
-        testMetricByName(listMetricStore, GET_LATENCY_MAX, 0);
-        testMetricByName(listMetricStore, DELETE_LATENCY_AVG, 0);
-        testMetricByName(listMetricStore, DELETE_LATENCY_MAX, 0);
-        testMetricByName(listMetricStore, PUT_ALL_LATENCY_AVG, 0);
-        testMetricByName(listMetricStore, PUT_ALL_LATENCY_MAX, 0);
-        testMetricByName(listMetricStore, ALL_LATENCY_AVG, 0);
-        testMetricByName(listMetricStore, ALL_LATENCY_MAX, 0);
-        testMetricByName(listMetricStore, RANGE_LATENCY_AVG, 0);
-        testMetricByName(listMetricStore, RANGE_LATENCY_MAX, 0);
-        testMetricByName(listMetricStore, FLUSH_LATENCY_AVG, 2);
-        testMetricByName(listMetricStore, FLUSH_LATENCY_MAX, 2);
-        testMetricByName(listMetricStore, RESTORE_LATENCY_AVG, 2);
-        testMetricByName(listMetricStore, RESTORE_LATENCY_MAX, 2);
-        testMetricByName(listMetricStore, PUT_RATE, 2);
-        testMetricByName(listMetricStore, PUT_TOTAL, 2);
-        testMetricByName(listMetricStore, PUT_IF_ABSENT_RATE, 0);
-        testMetricByName(listMetricStore, PUT_IF_ABSENT_TOTAL, 0);
-        testMetricByName(listMetricStore, GET_RATE, 0);
-        testMetricByName(listMetricStore, DELETE_RATE, 0);
-        testMetricByName(listMetricStore, DELETE_TOTAL, 0);
-        testMetricByName(listMetricStore, PUT_ALL_RATE, 0);
-        testMetricByName(listMetricStore, PUT_ALL_TOTAL, 0);
-        testMetricByName(listMetricStore, ALL_RATE, 0);
-        testMetricByName(listMetricStore, ALL_TOTAL, 0);
-        testMetricByName(listMetricStore, RANGE_RATE, 0);
-        testMetricByName(listMetricStore, RANGE_TOTAL, 0);
-        testMetricByName(listMetricStore, FLUSH_RATE, 2);
-        testMetricByName(listMetricStore, FLUSH_TOTAL, 2);
-        testMetricByName(listMetricStore, RESTORE_RATE, 2);
-        testMetricByName(listMetricStore, RESTORE_TOTAL, 2);
+        checkMetricByName(listMetricStore, PUT_LATENCY_AVG, 2);
+        checkMetricByName(listMetricStore, PUT_LATENCY_MAX, 2);
+        checkMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_AVG, 0);
+        checkMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_MAX, 0);
+        checkMetricByName(listMetricStore, GET_LATENCY_AVG, 0);
+        checkMetricByName(listMetricStore, GET_LATENCY_MAX, 0);
+        checkMetricByName(listMetricStore, DELETE_LATENCY_AVG, 0);
+        checkMetricByName(listMetricStore, DELETE_LATENCY_MAX, 0);
+        checkMetricByName(listMetricStore, PUT_ALL_LATENCY_AVG, 0);
+        checkMetricByName(listMetricStore, PUT_ALL_LATENCY_MAX, 0);
+        checkMetricByName(listMetricStore, ALL_LATENCY_AVG, 0);
+        checkMetricByName(listMetricStore, ALL_LATENCY_MAX, 0);
+        checkMetricByName(listMetricStore, RANGE_LATENCY_AVG, 0);
+        checkMetricByName(listMetricStore, RANGE_LATENCY_MAX, 0);
+        checkMetricByName(listMetricStore, FLUSH_LATENCY_AVG, 2);
+        checkMetricByName(listMetricStore, FLUSH_LATENCY_MAX, 2);
+        checkMetricByName(listMetricStore, RESTORE_LATENCY_AVG, 2);
+        checkMetricByName(listMetricStore, RESTORE_LATENCY_MAX, 2);
+        checkMetricByName(listMetricStore, PUT_RATE, 2);
+        checkMetricByName(listMetricStore, PUT_TOTAL, 2);
+        checkMetricByName(listMetricStore, PUT_IF_ABSENT_RATE, 0);
+        checkMetricByName(listMetricStore, PUT_IF_ABSENT_TOTAL, 0);
+        checkMetricByName(listMetricStore, GET_RATE, 0);
+        checkMetricByName(listMetricStore, DELETE_RATE, 0);
+        checkMetricByName(listMetricStore, DELETE_TOTAL, 0);
+        checkMetricByName(listMetricStore, PUT_ALL_RATE, 0);
+        checkMetricByName(listMetricStore, PUT_ALL_TOTAL, 0);
+        checkMetricByName(listMetricStore, ALL_RATE, 0);
+        checkMetricByName(listMetricStore, ALL_TOTAL, 0);
+        checkMetricByName(listMetricStore, RANGE_RATE, 0);
+        checkMetricByName(listMetricStore, RANGE_TOTAL, 0);
+        checkMetricByName(listMetricStore, FLUSH_RATE, 2);
+        checkMetricByName(listMetricStore, FLUSH_TOTAL, 2);
+        checkMetricByName(listMetricStore, RESTORE_RATE, 2);
+        checkMetricByName(listMetricStore, RESTORE_TOTAL, 2);
     }
 
     private void checkSessionStoreMetrics() {
         final List listMetricStore = new ArrayList(kafkaStreams.metrics().values()).stream()
             .filter(m -> m.metricName().group().equals(STREAM_STORE_SESSION_ROCKSDB_STATE_METRICS))
             .collect(Collectors.toList());
-        testMetricByName(listMetricStore, PUT_LATENCY_AVG, 2);
-        testMetricByName(listMetricStore, PUT_LATENCY_MAX, 2);
-        testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_AVG, 0);
-        testMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_MAX, 0);
-        testMetricByName(listMetricStore, GET_LATENCY_AVG, 0);
-        testMetricByName(listMetricStore, GET_LATENCY_MAX, 0);
-        testMetricByName(listMetricStore, DELETE_LATENCY_AVG, 0);
-        testMetricByName(listMetricStore, DELETE_LATENCY_MAX, 0);
-        testMetricByName(listMetricStore, PUT_ALL_LATENCY_AVG, 0);
-        testMetricByName(listMetricStore, PUT_ALL_LATENCY_MAX, 0);
-        testMetricByName(listMetricStore, ALL_LATENCY_AVG, 0);
-        testMetricByName(listMetricStore, ALL_LATENCY_MAX, 0);
-        testMetricByName(listMetricStore, RANGE_LATENCY_AVG, 0);
-        testMetricByName(listMetricStore, RANGE_LATENCY_MAX, 0);
-        testMetricByName(listMetricStore, FLUSH_LATENCY_AVG, 2);
-        testMetricByName(listMetricStore, FLUSH_LATENCY_MAX, 2);
-        testMetricByName(listMetricStore, RESTORE_LATENCY_AVG, 2);
-        testMetricByName(listMetricStore, RESTORE_LATENCY_MAX, 2);
-        testMetricByName(listMetricStore, PUT_RATE, 2);
-        testMetricByName(listMetricStore, PUT_TOTAL, 2);
-        testMetricByName(listMetricStore, PUT_IF_ABSENT_RATE, 0);
-        testMetricByName(listMetricStore, PUT_IF_ABSENT_TOTAL, 0);
-        testMetricByName(listMetricStore, GET_RATE, 0);
-        testMetricByName(listMetricStore, DELETE_RATE, 0);
-        testMetricByName(listMetricStore, DELETE_TOTAL, 0);
-        testMetricByName(listMetricStore, PUT_ALL_RATE, 0);
-        testMetricByName(listMetricStore, PUT_ALL_TOTAL, 0);
-        testMetricByName(listMetricStore, ALL_RATE, 0);
-        testMetricByName(listMetricStore, ALL_TOTAL, 0);
-        testMetricByName(listMetricStore, RANGE_RATE, 0);
-        testMetricByName(listMetricStore, RANGE_TOTAL, 0);
-        testMetricByName(listMetricStore, FLUSH_RATE, 2);
-        testMetricByName(listMetricStore, FLUSH_TOTAL, 2);
-        testMetricByName(listMetricStore, RESTORE_RATE, 2);
-        testMetricByName(listMetricStore, RESTORE_TOTAL, 2);
+        checkMetricByName(listMetricStore, PUT_LATENCY_AVG, 2);
+        checkMetricByName(listMetricStore, PUT_LATENCY_MAX, 2);
+        checkMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_AVG, 0);
+        checkMetricByName(listMetricStore, PUT_IF_ABSENT_LATENCY_MAX, 0);
+        checkMetricByName(listMetricStore, GET_LATENCY_AVG, 0);
+        checkMetricByName(listMetricStore, GET_LATENCY_MAX, 0);
+        checkMetricByName(listMetricStore, DELETE_LATENCY_AVG, 0);
+        checkMetricByName(listMetricStore, DELETE_LATENCY_MAX, 0);
+        checkMetricByName(listMetricStore, PUT_ALL_LATENCY_AVG, 0);
+        checkMetricByName(listMetricStore, PUT_ALL_LATENCY_MAX, 0);
+        checkMetricByName(listMetricStore, ALL_LATENCY_AVG, 0);
+        checkMetricByName(listMetricStore, ALL_LATENCY_MAX, 0);
+        checkMetricByName(listMetricStore, RANGE_LATENCY_AVG, 0);
+        checkMetricByName(listMetricStore, RANGE_LATENCY_MAX, 0);
+        checkMetricByName(listMetricStore, FLUSH_LATENCY_AVG, 2);
+        checkMetricByName(listMetricStore, FLUSH_LATENCY_MAX, 2);
+        checkMetricByName(listMetricStore, RESTORE_LATENCY_AVG, 2);
+        checkMetricByName(listMetricStore, RESTORE_LATENCY_MAX, 2);
+        checkMetricByName(listMetricStore, PUT_RATE, 2);
+        checkMetricByName(listMetricStore, PUT_TOTAL, 2);
+        checkMetricByName(listMetricStore, PUT_IF_ABSENT_RATE, 0);
+        checkMetricByName(listMetricStore, PUT_IF_ABSENT_TOTAL, 0);
+        checkMetricByName(listMetricStore, GET_RATE, 0);
+        checkMetricByName(listMetricStore, DELETE_RATE, 0);
+        checkMetricByName(listMetricStore, DELETE_TOTAL, 0);
+        checkMetricByName(listMetricStore, PUT_ALL_RATE, 0);
+        checkMetricByName(listMetricStore, PUT_ALL_TOTAL, 0);
+        checkMetricByName(listMetricStore, ALL_RATE, 0);
+        checkMetricByName(listMetricStore, ALL_TOTAL, 0);
+        checkMetricByName(listMetricStore, RANGE_RATE, 0);
+        checkMetricByName(listMetricStore, RANGE_TOTAL, 0);
+        checkMetricByName(listMetricStore, FLUSH_RATE, 2);
+        checkMetricByName(listMetricStore, FLUSH_TOTAL, 2);
+        checkMetricByName(listMetricStore, RESTORE_RATE, 2);
+        checkMetricByName(listMetricStore, RESTORE_TOTAL, 2);
     }
 
-    private void testMetricByName(final List listMetric, final String metricName, final int numMetric) {
-        final List metrics = listMetric.stream().filter(m -> m.metricName().name().equals(metricName)).collect(Collectors.toList());
+    private void checkMetricByName(final List listMetric, final String metricName, final int numMetric) {
+        final List metrics = listMetric.stream()
+            .filter(m -> m.metricName().name().equals(metricName))
+            .collect(Collectors.toList());
         Assert.assertEquals("Size of metrics of type:'" + metricName + "' must be equal to:" + numMetric + " but it's equal to " + metrics.size(), numMetric, metrics.size());
         for (final Metric m : metrics) {
             Assert.assertNotNull("Metric:'" + m.metricName() + "' must be not null", m.metricValue());

From 9e60f4cd17a5574a52755c58c78bf7291440a79b Mon Sep 17 00:00:00 2001
From: Stanislav Kozlovski 
Date: Fri, 14 Jun 2019 16:21:38 +0300
Subject: [PATCH 0365/1071] MINOR: Add system tests README link from main
 README (#6939)

---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index 0d443495ee4d1..3f24e1f370edc 100644
--- a/README.md
+++ b/README.md
@@ -200,6 +200,10 @@ The following options should be set with a `-P` switch, for example `./gradlew -
 * `testLoggingEvents`: unit test events to be logged, separated by comma. For example `./gradlew -PtestLoggingEvents=started,passed,skipped,failed test`.
 * `xmlSpotBugsReport`: enable XML reports for spotBugs. This also disables HTML reports as only one can be enabled at a time.
 
+### Running system tests ###
+
+See [tests/README.md](tests/README.md).
+
 ### Running in Vagrant ###
 
 See [vagrant/README.md](vagrant/README.md).

From 8dd4fb5ebeccbb427ec5d0d92d4d5281ba9291f6 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Fri, 14 Jun 2019 08:16:11 -0700
Subject: [PATCH 0366/1071] KAFKA-8530; Check for topic authorization errors in
 OffsetFetch response (#6928)

The OffsetFetch requires Topic Describe permission. If a client does not have this, we return TOPIC_AUTHORIZATION_FAILED at the partition level. Currently the consumer does not handle this error explicitly, but raises it as a generic `KafkaException`. For consistency with other APIs and to fix transient test failures in `PlaintextEndToEndAuthorizationTest`, we should raise `TopicAuthorizationFailedException` instead.

Reviewers: Ismael Juma 
---
 .../consumer/internals/ConsumerCoordinator.java  | 15 +++++++++++++--
 .../common/requests/OffsetFetchResponse.java     | 11 ++++++-----
 .../internals/ConsumerCoordinatorTest.java       | 16 ++++++++++++++++
 3 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index bacb96001bcbe..fbf5db42af3e7 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -959,6 +959,7 @@ public void handle(OffsetFetchResponse response, RequestFuture unauthorizedTopics = null;
             Map offsets = new HashMap<>(response.responseData().size());
             for (Map.Entry entry : response.responseData().entrySet()) {
                 TopicPartition tp = entry.getKey();
@@ -969,11 +970,17 @@ public void handle(OffsetFetchResponse response, RequestFuture();
+                        }
+                        unauthorizedTopics.add(tp.topic());
                     } else {
                         future.raise(new KafkaException("Unexpected error in fetch offset response for partition " +
                             tp + ": " + error.message()));
+                        return;
                     }
-                    return;
                 } else if (data.offset >= 0) {
                     // record the position with the offset (-1 indicates no committed offset to fetch)
                     offsets.put(tp, new OffsetAndMetadata(data.offset, data.leaderEpoch, data.metadata));
@@ -982,7 +989,11 @@ public void handle(OffsetFetchResponse response, RequestFuture
+                coordinator.fetchCommittedOffsets(singleton(t1p), time.timer(Long.MAX_VALUE)));
+
+        assertEquals(singleton(topic1), exception.unauthorizedTopics());
+    }
+
     @Test
     public void testRefreshOffsetLoadInProgress() {
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));

From 35814298e1fa2840f89ef1f40163b2480da28a2b Mon Sep 17 00:00:00 2001
From: wenhoujx 
Date: Fri, 14 Jun 2019 12:24:27 -0400
Subject: [PATCH 0367/1071] KAFKA-8488: Reduce logging-related string
 allocation in FetchSessionHandler

Reviewers: Colin P. McCabe , Ismael Juma 
---
 .../kafka/clients/FetchSessionHandler.java       | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java b/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java
index 30ae65f2309c4..5b402bf5b3b2a 100644
--- a/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java
+++ b/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java
@@ -196,8 +196,10 @@ public void add(TopicPartition topicPartition, PartitionData data) {
 
         public FetchRequestData build() {
             if (nextMetadata.isFull()) {
-                log.debug("Built full fetch {} for node {} with {}.",
-                    nextMetadata, node, partitionsToLogString(next.keySet()));
+                if (log.isDebugEnabled()) {
+                    log.debug("Built full fetch {} for node {} with {}.",
+                              nextMetadata, node, partitionsToLogString(next.keySet()));
+                }
                 sessionPartitions = next;
                 next = null;
                 Map toSend =
@@ -247,10 +249,12 @@ public FetchRequestData build() {
                 sessionPartitions.put(topicPartition, nextData);
                 added.add(topicPartition);
             }
-            log.debug("Built incremental fetch {} for node {}. Added {}, altered {}, removed {} " +
-                    "out of {}", nextMetadata, node, partitionsToLogString(added),
-                    partitionsToLogString(altered), partitionsToLogString(removed),
-                    partitionsToLogString(sessionPartitions.keySet()));
+            if (log.isDebugEnabled()) {
+                log.debug("Built incremental fetch {} for node {}. Added {}, altered {}, removed {} " +
+                          "out of {}", nextMetadata, node, partitionsToLogString(added),
+                          partitionsToLogString(altered), partitionsToLogString(removed),
+                          partitionsToLogString(sessionPartitions.keySet()));
+            }
             Map toSend =
                 Collections.unmodifiableMap(new LinkedHashMap<>(next));
             Map curSessionPartitions =

From 2ef02f111e3a1640d3e92aa29dae5677d0eefe18 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Fri, 14 Jun 2019 15:51:33 -0700
Subject: [PATCH 0368/1071] KAFKA-8179: Part I, Bump up consumer protocol to v2
 (#6528)

1. Add new fields of subscription / assignment and bump up consumer protocol to v2.
2. Update tests to make sure old versioned protocol can be successfully deserialized, and new versioned protocol can be deserialized by old byte code.

Reviewers: Boyang Chen , Sophie Blee-Goldman ,  Bill Bejeck 
---
 .../consumer/internals/ConsumerProtocol.java  | 246 +++++++++++++++---
 .../consumer/internals/PartitionAssignor.java | 117 ++++++++-
 .../org/apache/kafka/common/utils/Bytes.java  |   2 +-
 .../internals/ConsumerProtocolTest.java       | 123 +++++++--
 4 files changed, 426 insertions(+), 62 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java
index 7bef8f7ff748a..b4ad4514eb60b 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java
@@ -30,24 +30,32 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
+
 /**
  * ConsumerProtocol contains the schemas for consumer subscriptions and assignments for use with
- * Kafka's generalized group management protocol. Below is the version 0 format:
+ * Kafka's generalized group management protocol. Below is the version 1 format:
  *
  * 
      * Subscription => Version Topics
  *   Version    => Int16
  *   Topics     => [String]
  *   UserData   => Bytes
+ *   OwnedPartitions    => [Topic Partitions]
+ *     Topic            => String
+ *     Partitions       => [int32]
  *
  * Assignment => Version TopicPartitions
- *   Version         => int16
- *   TopicPartitions => [Topic Partitions]
- *     Topic         => String
- *     Partitions    => [int32]
- *   UserData        => Bytes
+ *   Version            => int16
+ *   AssignedPartitions => [Topic Partitions]
+ *     Topic            => String
+ *     Partitions       => [int32]
+ *   UserData           => Bytes
+ *   ErrorCode          => [int16]
  * 
    
  *
+ * Older versioned formats can be inferred by reading the code below.
+ *
  * The current implementation assumes that future versions will not break compatibility. When
  * it encounters a newer version, it parses it using the current format. This basically means
  * that new versions cannot remove or reorder any of the existing fields.
@@ -60,29 +68,73 @@ public class ConsumerProtocol {
     public static final String TOPICS_KEY_NAME = "topics";
     public static final String TOPIC_KEY_NAME = "topic";
     public static final String PARTITIONS_KEY_NAME = "partitions";
+    public static final String OWNED_PARTITIONS_KEY_NAME = "owned_partitions";
     public static final String TOPIC_PARTITIONS_KEY_NAME = "topic_partitions";
     public static final String USER_DATA_KEY_NAME = "user_data";
 
     public static final short CONSUMER_PROTOCOL_V0 = 0;
+    public static final short CONSUMER_PROTOCOL_V1 = 1;
+
     public static final Schema CONSUMER_PROTOCOL_HEADER_SCHEMA = new Schema(
             new Field(VERSION_KEY_NAME, Type.INT16));
     private static final Struct CONSUMER_PROTOCOL_HEADER_V0 = new Struct(CONSUMER_PROTOCOL_HEADER_SCHEMA)
             .set(VERSION_KEY_NAME, CONSUMER_PROTOCOL_V0);
+    private static final Struct CONSUMER_PROTOCOL_HEADER_V1 = new Struct(CONSUMER_PROTOCOL_HEADER_SCHEMA)
+            .set(VERSION_KEY_NAME, CONSUMER_PROTOCOL_V1);
+
+    public static final Schema TOPIC_ASSIGNMENT_V0 = new Schema(
+        new Field(TOPIC_KEY_NAME, Type.STRING),
+        new Field(PARTITIONS_KEY_NAME, new ArrayOf(Type.INT32)));
 
     public static final Schema SUBSCRIPTION_V0 = new Schema(
             new Field(TOPICS_KEY_NAME, new ArrayOf(Type.STRING)),
             new Field(USER_DATA_KEY_NAME, Type.NULLABLE_BYTES));
-    public static final Schema TOPIC_ASSIGNMENT_V0 = new Schema(
-            new Field(TOPIC_KEY_NAME, Type.STRING),
-            new Field(PARTITIONS_KEY_NAME, new ArrayOf(Type.INT32)));
+
+    public static final Schema SUBSCRIPTION_V1 = new Schema(
+        new Field(TOPICS_KEY_NAME, new ArrayOf(Type.STRING)),
+        new Field(USER_DATA_KEY_NAME, Type.NULLABLE_BYTES),
+        new Field(OWNED_PARTITIONS_KEY_NAME, new ArrayOf(TOPIC_ASSIGNMENT_V0)));
+
     public static final Schema ASSIGNMENT_V0 = new Schema(
             new Field(TOPIC_PARTITIONS_KEY_NAME, new ArrayOf(TOPIC_ASSIGNMENT_V0)),
             new Field(USER_DATA_KEY_NAME, Type.NULLABLE_BYTES));
 
-    public static ByteBuffer serializeSubscription(PartitionAssignor.Subscription subscription) {
+    public static final Schema ASSIGNMENT_V1 = new Schema(
+        new Field(TOPIC_PARTITIONS_KEY_NAME, new ArrayOf(TOPIC_ASSIGNMENT_V0)),
+        new Field(USER_DATA_KEY_NAME, Type.NULLABLE_BYTES),
+        ERROR_CODE);
+
+    public enum Errors {
+        NONE(0),
+        NEED_REJOIN(1);
+
+        private final short code;
+
+        Errors(final int code) {
+            this.code = (short) code;
+        }
+
+        public short code() {
+            return code;
+        }
+
+        public static Errors fromCode(final short code) {
+            switch (code) {
+                case 0:
+                    return NONE;
+                case 1:
+                    return NEED_REJOIN;
+                default:
+                    throw new IllegalArgumentException("Unknown error code: " + code);
+            }
+        }
+    }
+
+    public static ByteBuffer serializeSubscriptionV0(PartitionAssignor.Subscription subscription) {
         Struct struct = new Struct(SUBSCRIPTION_V0);
         struct.set(USER_DATA_KEY_NAME, subscription.userData());
         struct.set(TOPICS_KEY_NAME, subscription.topics().toArray());
+
         ByteBuffer buffer = ByteBuffer.allocate(CONSUMER_PROTOCOL_HEADER_V0.sizeOf() + SUBSCRIPTION_V0.sizeOf(struct));
         CONSUMER_PROTOCOL_HEADER_V0.writeTo(buffer);
         SUBSCRIPTION_V0.write(buffer, struct);
@@ -90,37 +142,91 @@ public static ByteBuffer serializeSubscription(PartitionAssignor.Subscription su
         return buffer;
     }
 
-    public static PartitionAssignor.Subscription deserializeSubscription(ByteBuffer buffer) {
-        Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer);
-        Short version = header.getShort(VERSION_KEY_NAME);
-        checkVersionCompatibility(version);
+    public static ByteBuffer serializeSubscriptionV1(PartitionAssignor.Subscription subscription) {
+        Struct struct = new Struct(SUBSCRIPTION_V1);
+        struct.set(USER_DATA_KEY_NAME, subscription.userData());
+        struct.set(TOPICS_KEY_NAME, subscription.topics().toArray());
+        List topicAssignments = new ArrayList<>();
+        Map> partitionsByTopic = CollectionUtils.groupPartitionsByTopic(subscription.ownedPartitions());
+        for (Map.Entry> topicEntry : partitionsByTopic.entrySet()) {
+            Struct topicAssignment = new Struct(TOPIC_ASSIGNMENT_V0);
+            topicAssignment.set(TOPIC_KEY_NAME, topicEntry.getKey());
+            topicAssignment.set(PARTITIONS_KEY_NAME, topicEntry.getValue().toArray());
+            topicAssignments.add(topicAssignment);
+        }
+        struct.set(OWNED_PARTITIONS_KEY_NAME, topicAssignments.toArray());
+
+        ByteBuffer buffer = ByteBuffer.allocate(CONSUMER_PROTOCOL_HEADER_V1.sizeOf() + SUBSCRIPTION_V1.sizeOf(struct));
+        CONSUMER_PROTOCOL_HEADER_V1.writeTo(buffer);
+        SUBSCRIPTION_V1.write(buffer, struct);
+        buffer.flip();
+        return buffer;
+    }
+
+    public static ByteBuffer serializeSubscription(PartitionAssignor.Subscription subscription) {
+        switch (subscription.version()) {
+            case CONSUMER_PROTOCOL_V0:
+                return serializeSubscriptionV0(subscription);
+
+            case CONSUMER_PROTOCOL_V1:
+                return serializeSubscriptionV1(subscription);
+
+            default:
+                // for any versions higher than known, try to serialize it as V1
+                return serializeSubscriptionV1(subscription);
+        }
+    }
+
+    public static PartitionAssignor.Subscription deserializeSubscriptionV0(ByteBuffer buffer) {
         Struct struct = SUBSCRIPTION_V0.read(buffer);
         ByteBuffer userData = struct.getBytes(USER_DATA_KEY_NAME);
         List topics = new ArrayList<>();
         for (Object topicObj : struct.getArray(TOPICS_KEY_NAME))
             topics.add((String) topicObj);
-        return new PartitionAssignor.Subscription(topics, userData);
+
+        return new PartitionAssignor.Subscription(CONSUMER_PROTOCOL_V0, topics, userData);
     }
 
-    public static PartitionAssignor.Assignment deserializeAssignment(ByteBuffer buffer) {
-        Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer);
-        Short version = header.getShort(VERSION_KEY_NAME);
-        checkVersionCompatibility(version);
-        Struct struct = ASSIGNMENT_V0.read(buffer);
+    public static PartitionAssignor.Subscription deserializeSubscriptionV1(ByteBuffer buffer) {
+        Struct struct = SUBSCRIPTION_V1.read(buffer);
         ByteBuffer userData = struct.getBytes(USER_DATA_KEY_NAME);
-        List partitions = new ArrayList<>();
-        for (Object structObj : struct.getArray(TOPIC_PARTITIONS_KEY_NAME)) {
+        List topics = new ArrayList<>();
+        for (Object topicObj : struct.getArray(TOPICS_KEY_NAME))
+            topics.add((String) topicObj);
+
+        List ownedPartitions = new ArrayList<>();
+        for (Object structObj : struct.getArray(OWNED_PARTITIONS_KEY_NAME)) {
             Struct assignment = (Struct) structObj;
             String topic = assignment.getString(TOPIC_KEY_NAME);
             for (Object partitionObj : assignment.getArray(PARTITIONS_KEY_NAME)) {
-                Integer partition = (Integer) partitionObj;
-                partitions.add(new TopicPartition(topic, partition));
+                ownedPartitions.add(new TopicPartition(topic, (Integer) partitionObj));
             }
         }
-        return new PartitionAssignor.Assignment(partitions, userData);
+
+        return new PartitionAssignor.Subscription(CONSUMER_PROTOCOL_V1, topics, userData, ownedPartitions);
     }
 
-    public static ByteBuffer serializeAssignment(PartitionAssignor.Assignment assignment) {
+    public static PartitionAssignor.Subscription deserializeSubscription(ByteBuffer buffer) {
+        Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer);
+        Short version = header.getShort(VERSION_KEY_NAME);
+
+        if (version < CONSUMER_PROTOCOL_V0)
+            throw new SchemaException("Unsupported subscription version: " + version);
+
+        switch (version) {
+            case CONSUMER_PROTOCOL_V0:
+                return deserializeSubscriptionV0(buffer);
+
+            case CONSUMER_PROTOCOL_V1:
+                return deserializeSubscriptionV1(buffer);
+
+            // assume all higher versions can be parsed as V1
+            default:
+                return deserializeSubscriptionV1(buffer);
+        }
+    }
+
+    public static ByteBuffer serializeAssignmentV0(PartitionAssignor.Assignment assignment) {
         Struct struct = new Struct(ASSIGNMENT_V0);
         struct.set(USER_DATA_KEY_NAME, assignment.userData());
         List topicAssignments = new ArrayList<>();
@@ -132,6 +238,7 @@ public static ByteBuffer serializeAssignment(PartitionAssignor.Assignment assign
             topicAssignments.add(topicAssignment);
         }
         struct.set(TOPIC_PARTITIONS_KEY_NAME, topicAssignments.toArray());
+
         ByteBuffer buffer = ByteBuffer.allocate(CONSUMER_PROTOCOL_HEADER_V0.sizeOf() + ASSIGNMENT_V0.sizeOf(struct));
         CONSUMER_PROTOCOL_HEADER_V0.writeTo(buffer);
         ASSIGNMENT_V0.write(buffer, struct);
@@ -139,12 +246,89 @@ public static ByteBuffer serializeAssignment(PartitionAssignor.Assignment assign
         return buffer;
     }
 
-    private static void checkVersionCompatibility(short version) {
-        // check for invalid versions
-        if (version < CONSUMER_PROTOCOL_V0)
-            throw new SchemaException("Unsupported subscription version: " + version);
+    public static ByteBuffer serializeAssignmentV1(PartitionAssignor.Assignment assignment) {
+        Struct struct = new Struct(ASSIGNMENT_V1);
+        struct.set(USER_DATA_KEY_NAME, assignment.userData());
+        List topicAssignments = new ArrayList<>();
+        Map> partitionsByTopic = CollectionUtils.groupPartitionsByTopic(assignment.partitions());
+        for (Map.Entry> topicEntry : partitionsByTopic.entrySet()) {
+            Struct topicAssignment = new Struct(TOPIC_ASSIGNMENT_V0);
+            topicAssignment.set(TOPIC_KEY_NAME, topicEntry.getKey());
+            topicAssignment.set(PARTITIONS_KEY_NAME, topicEntry.getValue().toArray());
+            topicAssignments.add(topicAssignment);
+        }
+        struct.set(TOPIC_PARTITIONS_KEY_NAME, topicAssignments.toArray());
+        struct.set(ERROR_CODE.name, assignment.error().code);
 
-        // otherwise, assume versions can be parsed as V0
+        ByteBuffer buffer = ByteBuffer.allocate(CONSUMER_PROTOCOL_HEADER_V1.sizeOf() + ASSIGNMENT_V1.sizeOf(struct));
+        CONSUMER_PROTOCOL_HEADER_V1.writeTo(buffer);
+        ASSIGNMENT_V1.write(buffer, struct);
+        buffer.flip();
+        return buffer;
     }
 
+    public static ByteBuffer serializeAssignment(PartitionAssignor.Assignment assignment) {
+        switch (assignment.version()) {
+            case CONSUMER_PROTOCOL_V0:
+                return serializeAssignmentV0(assignment);
+
+            case CONSUMER_PROTOCOL_V1:
+                return serializeAssignmentV1(assignment);
+
+            default:
+                // for any versions higher than known, try to serialize it as V1
+                return serializeAssignmentV1(assignment);
+        }
+    }
+
+    public static PartitionAssignor.Assignment deserializeAssignmentV0(ByteBuffer buffer) {
+        Struct struct = ASSIGNMENT_V0.read(buffer);
+        ByteBuffer userData = struct.getBytes(USER_DATA_KEY_NAME);
+        List partitions = new ArrayList<>();
+        for (Object structObj : struct.getArray(TOPIC_PARTITIONS_KEY_NAME)) {
+            Struct assignment = (Struct) structObj;
+            String topic = assignment.getString(TOPIC_KEY_NAME);
+            for (Object partitionObj : assignment.getArray(PARTITIONS_KEY_NAME)) {
+                partitions.add(new TopicPartition(topic, (Integer) partitionObj));
+            }
+        }
+        return new PartitionAssignor.Assignment(CONSUMER_PROTOCOL_V0, partitions, userData);
+    }
+
+    public static PartitionAssignor.Assignment deserializeAssignmentV1(ByteBuffer buffer) {
+        Struct struct = ASSIGNMENT_V1.read(buffer);
+        ByteBuffer userData = struct.getBytes(USER_DATA_KEY_NAME);
+        List partitions = new ArrayList<>();
+        for (Object structObj : struct.getArray(TOPIC_PARTITIONS_KEY_NAME)) {
+            Struct assignment = (Struct) structObj;
+            String topic = assignment.getString(TOPIC_KEY_NAME);
+            for (Object partitionObj : assignment.getArray(PARTITIONS_KEY_NAME)) {
+                partitions.add(new TopicPartition(topic, (Integer) partitionObj));
+            }
+        }
+
+        Errors error = Errors.fromCode(struct.get(ERROR_CODE));
+
+        return new PartitionAssignor.Assignment(CONSUMER_PROTOCOL_V1, partitions, userData, error);
+    }
+
+    public static PartitionAssignor.Assignment deserializeAssignment(ByteBuffer buffer) {
+        Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer);
+        Short version = header.getShort(VERSION_KEY_NAME);
+
+        if (version < CONSUMER_PROTOCOL_V0)
+            throw new SchemaException("Unsupported assignment version: " + version);
+
+        switch (version) {
+            case CONSUMER_PROTOCOL_V0:
+                return deserializeAssignmentV0(buffer);
+
+            case CONSUMER_PROTOCOL_V1:
+                return deserializeAssignmentV1(buffer);
+
+            default:
+                // assume all higher versions can be parsed as V1
+                return deserializeAssignmentV1(buffer);
+        }
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
index 43fdaf3b01cd3..5c76fd66dfef0 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
@@ -18,12 +18,17 @@
 
 import org.apache.kafka.common.Cluster;
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.protocol.types.SchemaException;
 
 import java.nio.ByteBuffer;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.CONSUMER_PROTOCOL_V0;
+import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.CONSUMER_PROTOCOL_V1;
+
 /**
  * This interface is used to define custom partition assignment for use in
  * {@link org.apache.kafka.clients.consumer.KafkaConsumer}. Members of the consumer group subscribe
@@ -73,6 +78,21 @@ default void onAssignment(Assignment assignment, int generation) {
         onAssignment(assignment);
     }
 
+    /**
+     * Indicate which rebalance protocol this assignor can would work with;
+     * By default it should always work with {@link RebalanceProtocol#EAGER}.
+     */
+    default List supportedProtocols() {
+        return Collections.singletonList(RebalanceProtocol.EAGER);
+    }
+
+    /**
+     * Return the version of the assignor which indicate how the user metadata encodings
+     * and the assignment algorithm gets evolved.
+     */
+    default short version() {
+        return (short) 0;
+    }
 
     /**
      * Unique name for this assignor (e.g. "range" or "roundrobin" or "sticky")
@@ -80,23 +100,78 @@ default void onAssignment(Assignment assignment, int generation) {
      */
     String name();
 
+    enum RebalanceProtocol {
+        EAGER((byte) 0), COOPERATIVE((byte) 1);
+
+        private final byte id;
+
+        RebalanceProtocol(byte id) {
+            this.id = id;
+        }
+
+        public byte id() {
+            return id;
+        }
+
+        public static RebalanceProtocol forId(byte id) {
+            switch (id) {
+                case 0:
+                    return EAGER;
+                case 1:
+                    return COOPERATIVE;
+                default:
+                    throw new IllegalArgumentException("Unknown rebalance protocol id: " + id);
+            }
+        }
+    }
+
     class Subscription {
+        private final Short version;
         private final List topics;
         private final ByteBuffer userData;
+        private final List ownedPartitions;
 
-        public Subscription(List topics, ByteBuffer userData) {
+        Subscription(Short version, List topics, ByteBuffer userData, List ownedPartitions) {
+            this.version = version;
             this.topics = topics;
             this.userData = userData;
+            this.ownedPartitions = ownedPartitions;
+
+            if (version < CONSUMER_PROTOCOL_V0)
+                throw new SchemaException("Unsupported subscription version: " + version);
+
+            if (version < CONSUMER_PROTOCOL_V1 && !ownedPartitions.isEmpty())
+                throw new IllegalArgumentException("Subscription version smaller than 1 should not have owned partitions");
+        }
+
+        Subscription(Short version, List topics, ByteBuffer userData) {
+            this(version, topics, userData, Collections.emptyList());
+        }
+
+        public Subscription(List topics, ByteBuffer userData, List ownedPartitions) {
+            this(CONSUMER_PROTOCOL_V1, topics, userData, ownedPartitions);
+        }
+
+        public Subscription(List topics, ByteBuffer userData) {
+            this(CONSUMER_PROTOCOL_V1, topics, userData);
         }
 
         public Subscription(List topics) {
             this(topics, ByteBuffer.wrap(new byte[0]));
         }
 
+        Short version() {
+            return version;
+        }
+
         public List topics() {
             return topics;
         }
 
+        public List ownedPartitions() {
+            return ownedPartitions;
+        }
+
         public ByteBuffer userData() {
             return userData;
         }
@@ -104,28 +179,60 @@ public ByteBuffer userData() {
         @Override
         public String toString() {
             return "Subscription(" +
-                    "topics=" + topics +
+                    "version=" + version +
+                    ", topics=" + topics +
+                    ", ownedPartitions=" + ownedPartitions +
                     ')';
         }
     }
 
     class Assignment {
+        private final Short version;
         private final List partitions;
         private final ByteBuffer userData;
+        private final ConsumerProtocol.Errors error;
 
-        public Assignment(List partitions, ByteBuffer userData) {
+        Assignment(Short version, List partitions, ByteBuffer userData, ConsumerProtocol.Errors error) {
+            this.version = version;
             this.partitions = partitions;
             this.userData = userData;
+            this.error = error;
+
+            if (version < CONSUMER_PROTOCOL_V0)
+                throw new SchemaException("Unsupported subscription version: " + version);
+
+            if (version < CONSUMER_PROTOCOL_V1 && error != ConsumerProtocol.Errors.NONE)
+                throw new IllegalArgumentException("Assignment version smaller than 1 should not have error code.");
+        }
+
+        Assignment(Short version, List partitions, ByteBuffer userData) {
+            this(version, partitions, userData, ConsumerProtocol.Errors.NONE);
+        }
+
+        public Assignment(List partitions, ByteBuffer userData, ConsumerProtocol.Errors error) {
+            this(CONSUMER_PROTOCOL_V1, partitions, userData, error);
+        }
+
+        public Assignment(List partitions, ByteBuffer userData) {
+            this(CONSUMER_PROTOCOL_V1, partitions, userData);
         }
 
         public Assignment(List partitions) {
             this(partitions, ByteBuffer.wrap(new byte[0]));
         }
 
+        Short version() {
+            return version;
+        }
+
         public List partitions() {
             return partitions;
         }
 
+        public ConsumerProtocol.Errors error() {
+            return error;
+        }
+
         public ByteBuffer userData() {
             return userData;
         }
@@ -133,7 +240,9 @@ public ByteBuffer userData() {
         @Override
         public String toString() {
             return "Assignment(" +
-                    "partitions=" + partitions +
+                    "version=" + version +
+                    ", partitions=" + partitions +
+                    ", error=" + error +
                     ')';
         }
     }
diff --git a/clients/src/main/java/org/apache/kafka/common/utils/Bytes.java b/clients/src/main/java/org/apache/kafka/common/utils/Bytes.java
index 19cd711928fc8..d90a44131a6dc 100644
--- a/clients/src/main/java/org/apache/kafka/common/utils/Bytes.java
+++ b/clients/src/main/java/org/apache/kafka/common/utils/Bytes.java
@@ -141,7 +141,7 @@ private static String toString(final byte[] b, int off, int len) {
     }
 
     /**
-     * A byte array comparator based on lexicograpic ordering.
+     * A byte array comparator based on lexicographic ordering.
      */
     public final static ByteArrayComparator BYTES_LEXICO_COMPARATOR = new LexicographicByteArrayComparator();
 
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java
index 37d105cf1cc3a..8a8ba0a82e199 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java
@@ -16,6 +16,8 @@
  */
 package org.apache.kafka.clients.consumer.internals;
 
+import org.apache.kafka.clients.consumer.internals.ConsumerProtocol.Errors;
+import org.apache.kafka.clients.consumer.internals.PartitionAssignor.Assignment;
 import org.apache.kafka.clients.consumer.internals.PartitionAssignor.Subscription;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.protocol.types.ArrayOf;
@@ -28,21 +30,35 @@
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.CONSUMER_PROTOCOL_HEADER_SCHEMA;
+import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.OWNED_PARTITIONS_KEY_NAME;
+import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.TOPICS_KEY_NAME;
+import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.TOPIC_ASSIGNMENT_V0;
+import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.TOPIC_PARTITIONS_KEY_NAME;
+import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.USER_DATA_KEY_NAME;
+import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.VERSION_KEY_NAME;
+import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
 
 public class ConsumerProtocolTest {
 
+    private final TopicPartition tp1 = new TopicPartition("foo", 1);
+    private final TopicPartition tp2 = new TopicPartition("bar", 2);
+
     @Test
     public void serializeDeserializeMetadata() {
         Subscription subscription = new Subscription(Arrays.asList("foo", "bar"));
         ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription);
         Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer);
         assertEquals(subscription.topics(), parsedSubscription.topics());
+        assertEquals(0, parsedSubscription.userData().limit());
     }
 
     @Test
@@ -51,26 +67,53 @@ public void serializeDeserializeNullSubscriptionUserData() {
         ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription);
         Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer);
         assertEquals(subscription.topics(), parsedSubscription.topics());
-        assertNull(subscription.userData());
+        assertNull(parsedSubscription.userData());
+    }
+
+    @Test
+    public void deserializeOldSubscriptionVersion() {
+        Subscription subscription = new Subscription((short) 0, Arrays.asList("foo", "bar"), null);
+        ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription);
+        Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer);
+        assertEquals(parsedSubscription.topics(), parsedSubscription.topics());
+        assertNull(parsedSubscription.userData());
+        assertTrue(parsedSubscription.ownedPartitions().isEmpty());
+    }
+
+    @Test
+    public void deserializeNewSubscriptionWithOldVersion() {
+        Subscription subscription = new Subscription((short) 1, Arrays.asList("foo", "bar"), null, Collections.singletonList(tp2));
+        ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription);
+        // ignore the version assuming it is the old byte code, as it will blindly deserialize as V0
+        Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer);
+        header.getShort(VERSION_KEY_NAME);
+        Subscription parsedSubscription = ConsumerProtocol.deserializeSubscriptionV0(buffer);
+        assertEquals(subscription.topics(), parsedSubscription.topics());
+        assertNull(parsedSubscription.userData());
+        assertTrue(parsedSubscription.ownedPartitions().isEmpty());
     }
 
     @Test
-    public void deserializeNewSubscriptionVersion() {
+    public void deserializeFutureSubscriptionVersion() {
         // verify that a new version which adds a field is still parseable
         short version = 100;
 
         Schema subscriptionSchemaV100 = new Schema(
-                new Field(ConsumerProtocol.TOPICS_KEY_NAME, new ArrayOf(Type.STRING)),
-                new Field(ConsumerProtocol.USER_DATA_KEY_NAME, Type.BYTES),
+                new Field(TOPICS_KEY_NAME, new ArrayOf(Type.STRING)),
+                new Field(USER_DATA_KEY_NAME, Type.BYTES),
+                new Field(OWNED_PARTITIONS_KEY_NAME, new ArrayOf(TOPIC_ASSIGNMENT_V0)),
                 new Field("foo", Type.STRING));
 
         Struct subscriptionV100 = new Struct(subscriptionSchemaV100);
-        subscriptionV100.set(ConsumerProtocol.TOPICS_KEY_NAME, new Object[]{"topic"});
-        subscriptionV100.set(ConsumerProtocol.USER_DATA_KEY_NAME, ByteBuffer.wrap(new byte[0]));
+        subscriptionV100.set(TOPICS_KEY_NAME, new Object[]{"topic"});
+        subscriptionV100.set(USER_DATA_KEY_NAME, ByteBuffer.wrap(new byte[0]));
+        subscriptionV100.set(OWNED_PARTITIONS_KEY_NAME, new Object[]{new Struct(TOPIC_ASSIGNMENT_V0)
+            .set(ConsumerProtocol.TOPIC_KEY_NAME, tp2.topic())
+            .set(ConsumerProtocol.PARTITIONS_KEY_NAME, new Object[]{tp2.partition()})});
         subscriptionV100.set("foo", "bar");
 
-        Struct headerV100 = new Struct(ConsumerProtocol.CONSUMER_PROTOCOL_HEADER_SCHEMA);
-        headerV100.set(ConsumerProtocol.VERSION_KEY_NAME, version);
+        Struct headerV100 = new Struct(CONSUMER_PROTOCOL_HEADER_SCHEMA);
+        headerV100.set(VERSION_KEY_NAME, version);
 
         ByteBuffer buffer = ByteBuffer.allocate(subscriptionV100.sizeOf() + headerV100.sizeOf());
         headerV100.writeTo(buffer);
@@ -79,46 +122,73 @@ public void deserializeNewSubscriptionVersion() {
         buffer.flip();
 
         Subscription subscription = ConsumerProtocol.deserializeSubscription(buffer);
-        assertEquals(Arrays.asList("topic"), subscription.topics());
+        assertEquals(Collections.singletonList("topic"), subscription.topics());
+        assertEquals(Collections.singletonList(tp2), subscription.ownedPartitions());
     }
 
     @Test
     public void serializeDeserializeAssignment() {
-        List partitions = Arrays.asList(new TopicPartition("foo", 0), new TopicPartition("bar", 2));
-        ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new PartitionAssignor.Assignment(partitions));
-        PartitionAssignor.Assignment parsedAssignment = ConsumerProtocol.deserializeAssignment(buffer);
+        List partitions = Arrays.asList(tp1, tp2);
+        ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new Assignment(partitions));
+        Assignment parsedAssignment = ConsumerProtocol.deserializeAssignment(buffer);
         assertEquals(toSet(partitions), toSet(parsedAssignment.partitions()));
+        assertEquals(0, parsedAssignment.userData().limit());
     }
 
     @Test
     public void deserializeNullAssignmentUserData() {
-        List partitions = Arrays.asList(new TopicPartition("foo", 0), new TopicPartition("bar", 2));
-        ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new PartitionAssignor.Assignment(partitions, null));
-        PartitionAssignor.Assignment parsedAssignment = ConsumerProtocol.deserializeAssignment(buffer);
+        List partitions = Arrays.asList(tp1, tp2);
+        ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new Assignment(partitions, null));
+        Assignment parsedAssignment = ConsumerProtocol.deserializeAssignment(buffer);
+        assertEquals(toSet(partitions), toSet(parsedAssignment.partitions()));
+        assertNull(parsedAssignment.userData());
+    }
+
+    @Test
+    public void deserializeOldAssignmentVersion() {
+        List partitions = Arrays.asList(tp1, tp2);
+        ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new Assignment((short) 0, partitions, null));
+        Assignment parsedAssignment = ConsumerProtocol.deserializeAssignment(buffer);
+        assertEquals(toSet(partitions), toSet(parsedAssignment.partitions()));
+        assertNull(parsedAssignment.userData());
+        assertEquals(Errors.NONE, parsedAssignment.error());
+    }
+
+    @Test
+    public void deserializeNewAssignmentWithOldVersion() {
+        List partitions = Collections.singletonList(tp1);
+        ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new Assignment((short) 1, partitions, null, Errors.NEED_REJOIN));
+        // ignore the version assuming it is the old byte code, as it will blindly deserialize as 0
+        Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer);
+        header.getShort(VERSION_KEY_NAME);
+        Assignment parsedAssignment = ConsumerProtocol.deserializeAssignmentV0(buffer);
         assertEquals(toSet(partitions), toSet(parsedAssignment.partitions()));
         assertNull(parsedAssignment.userData());
+        assertEquals(Errors.NONE, parsedAssignment.error());
     }
 
     @Test
-    public void deserializeNewAssignmentVersion() {
+    public void deserializeFutureAssignmentVersion() {
         // verify that a new version which adds a field is still parseable
         short version = 100;
 
         Schema assignmentSchemaV100 = new Schema(
-                new Field(ConsumerProtocol.TOPIC_PARTITIONS_KEY_NAME, new ArrayOf(ConsumerProtocol.TOPIC_ASSIGNMENT_V0)),
-                new Field(ConsumerProtocol.USER_DATA_KEY_NAME, Type.BYTES),
+                new Field(TOPIC_PARTITIONS_KEY_NAME, new ArrayOf(TOPIC_ASSIGNMENT_V0)),
+                new Field(USER_DATA_KEY_NAME, Type.BYTES),
+                ERROR_CODE,
                 new Field("foo", Type.STRING));
 
         Struct assignmentV100 = new Struct(assignmentSchemaV100);
-        assignmentV100.set(ConsumerProtocol.TOPIC_PARTITIONS_KEY_NAME,
-                new Object[]{new Struct(ConsumerProtocol.TOPIC_ASSIGNMENT_V0)
-                        .set(ConsumerProtocol.TOPIC_KEY_NAME, "foo")
-                        .set(ConsumerProtocol.PARTITIONS_KEY_NAME, new Object[]{1})});
-        assignmentV100.set(ConsumerProtocol.USER_DATA_KEY_NAME, ByteBuffer.wrap(new byte[0]));
+        assignmentV100.set(TOPIC_PARTITIONS_KEY_NAME,
+                new Object[]{new Struct(TOPIC_ASSIGNMENT_V0)
+                        .set(ConsumerProtocol.TOPIC_KEY_NAME, tp1.topic())
+                        .set(ConsumerProtocol.PARTITIONS_KEY_NAME, new Object[]{tp1.partition()})});
+        assignmentV100.set(USER_DATA_KEY_NAME, ByteBuffer.wrap(new byte[0]));
+        assignmentV100.set(ERROR_CODE.name, Errors.NEED_REJOIN.code());
         assignmentV100.set("foo", "bar");
 
-        Struct headerV100 = new Struct(ConsumerProtocol.CONSUMER_PROTOCOL_HEADER_SCHEMA);
-        headerV100.set(ConsumerProtocol.VERSION_KEY_NAME, version);
+        Struct headerV100 = new Struct(CONSUMER_PROTOCOL_HEADER_SCHEMA);
+        headerV100.set(VERSION_KEY_NAME, version);
 
         ByteBuffer buffer = ByteBuffer.allocate(assignmentV100.sizeOf() + headerV100.sizeOf());
         headerV100.writeTo(buffer);
@@ -127,7 +197,8 @@ public void deserializeNewAssignmentVersion() {
         buffer.flip();
 
         PartitionAssignor.Assignment assignment = ConsumerProtocol.deserializeAssignment(buffer);
-        assertEquals(toSet(Arrays.asList(new TopicPartition("foo", 1))), toSet(assignment.partitions()));
+        assertEquals(toSet(Collections.singletonList(tp1)), toSet(assignment.partitions()));
+        assertEquals(Errors.NEED_REJOIN, assignment.error());
     }
 
     private static  Set toSet(Collection collection) {

From e047864f30fa47c6bcb2a0e6d9da86fa053fe6f6 Mon Sep 17 00:00:00 2001
From: "Colin P. Mccabe" 
Date: Sat, 15 Jun 2019 19:21:10 -0700
Subject: [PATCH 0369/1071] MINOR: fix some warnings in the broker

Author: Colin P. Mccabe 

Reviewers: Gwen Shapira

Closes #6942 from cmccabe/fix-scala-warnings
---
 clients/src/main/java/org/apache/kafka/common/Cluster.java    | 2 +-
 .../main/scala/kafka/coordinator/group/GroupCoordinator.scala | 3 +++
 .../main/scala/kafka/coordinator/group/MemberMetadata.scala   | 1 -
 core/src/main/scala/kafka/log/LogCleaner.scala                | 2 +-
 .../integration/kafka/api/ConsumerTopicCreationTest.scala     | 1 -
 .../unit/kafka/admin/TopicCommandWithAdminClientTest.scala    | 4 ++--
 core/src/test/scala/unit/kafka/cluster/ReplicaTest.scala      | 2 +-
 .../unit/kafka/coordinator/group/GroupCoordinatorTest.scala   | 1 -
 core/src/test/scala/unit/kafka/server/KafkaApisTest.scala     | 2 +-
 9 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/Cluster.java b/clients/src/main/java/org/apache/kafka/common/Cluster.java
index 0b01d22b2751c..e69be4282eb74 100644
--- a/clients/src/main/java/org/apache/kafka/common/Cluster.java
+++ b/clients/src/main/java/org/apache/kafka/common/Cluster.java
@@ -189,7 +189,7 @@ public Node nodeById(int id) {
      * Get the node by node id if the replica for the given partition is online
      * @param partition
      * @param id
-     * @return
+     * @return the node
      */
     public Optional nodeIfOnline(TopicPartition partition, int id) {
         Node node = nodeById(id);
diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
index a9ac67ff2a18c..0bd1572566345 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
@@ -642,6 +642,9 @@ class GroupCoordinator(val brokerId: Int,
             // the latest group generation information from the JoinResponse.
             // So let's return a REBALANCE_IN_PROGRESS to let consumer handle it gracefully.
             responseCallback(offsetMetadata.mapValues(_ => Errors.REBALANCE_IN_PROGRESS))
+
+          case _ =>
+            throw new RuntimeException(s"Logic error: unexpected group state ${group.currentState}")
         }
       }
     }
diff --git a/core/src/main/scala/kafka/coordinator/group/MemberMetadata.scala b/core/src/main/scala/kafka/coordinator/group/MemberMetadata.scala
index 83ff70971d377..1014f40205dc6 100644
--- a/core/src/main/scala/kafka/coordinator/group/MemberMetadata.scala
+++ b/core/src/main/scala/kafka/coordinator/group/MemberMetadata.scala
@@ -20,7 +20,6 @@ package kafka.coordinator.group
 import java.util
 
 import kafka.utils.nonthreadsafe
-import org.apache.kafka.common.protocol.Errors
 
 case class MemberSummary(memberId: String,
                          groupInstanceId: Option[String],
diff --git a/core/src/main/scala/kafka/log/LogCleaner.scala b/core/src/main/scala/kafka/log/LogCleaner.scala
index 3180f3db9bf9f..cdeb4e6428582 100644
--- a/core/src/main/scala/kafka/log/LogCleaner.scala
+++ b/core/src/main/scala/kafka/log/LogCleaner.scala
@@ -1061,7 +1061,7 @@ private[log] class CleanedTransactionMetadata {
   private val ongoingCommittedTxns = mutable.Set.empty[Long]
   private val ongoingAbortedTxns = mutable.Map.empty[Long, AbortedTransactionMetadata]
   // Minheap of aborted transactions sorted by the transaction first offset
-  private var abortedTransactions = mutable.PriorityQueue.empty[AbortedTxn](new Ordering[AbortedTxn] {
+  private val abortedTransactions = mutable.PriorityQueue.empty[AbortedTxn](new Ordering[AbortedTxn] {
     override def compare(x: AbortedTxn, y: AbortedTxn): Int = x.firstOffset compare y.firstOffset
   }.reverse)
 
diff --git a/core/src/test/scala/integration/kafka/api/ConsumerTopicCreationTest.scala b/core/src/test/scala/integration/kafka/api/ConsumerTopicCreationTest.scala
index 55671bab20d5f..a126c8397cbf5 100644
--- a/core/src/test/scala/integration/kafka/api/ConsumerTopicCreationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/ConsumerTopicCreationTest.scala
@@ -22,7 +22,6 @@ import java.time.Duration
 import java.util
 import java.util.Collections
 
-import kafka.api.IntegrationTestHarness
 import kafka.server.KafkaConfig
 import kafka.utils.TestUtils
 import org.apache.kafka.clients.admin.NewTopic
diff --git a/core/src/test/scala/unit/kafka/admin/TopicCommandWithAdminClientTest.scala b/core/src/test/scala/unit/kafka/admin/TopicCommandWithAdminClientTest.scala
index 5db4309d64afa..de7fd6b4387ac 100644
--- a/core/src/test/scala/unit/kafka/admin/TopicCommandWithAdminClientTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/TopicCommandWithAdminClientTest.scala
@@ -54,8 +54,8 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
     defaultReplicationFactor = defaultReplicationFactor
     ).map(KafkaConfig.fromProps)
 
-  private var numPartitions = 1
-  private var defaultReplicationFactor = 1.toShort
+  private val numPartitions = 1
+  private val defaultReplicationFactor = 1.toShort
 
   private var topicService: AdminClientTopicService = _
   private var adminClient: JAdminClient = _
diff --git a/core/src/test/scala/unit/kafka/cluster/ReplicaTest.scala b/core/src/test/scala/unit/kafka/cluster/ReplicaTest.scala
index bacdc8151bd37..64e298fa1eaf6 100644
--- a/core/src/test/scala/unit/kafka/cluster/ReplicaTest.scala
+++ b/core/src/test/scala/unit/kafka/cluster/ReplicaTest.scala
@@ -19,7 +19,7 @@ package kafka.cluster
 import java.util.Properties
 
 import kafka.log.{Log, LogConfig, LogManager}
-import kafka.server.{BrokerTopicStats, LogDirFailureChannel, LogOffsetMetadata}
+import kafka.server.{BrokerTopicStats, LogDirFailureChannel}
 import kafka.utils.{MockTime, TestUtils}
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.OffsetOutOfRangeException
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
index 4cd91dd6fccfe..2cf3e5db4093a 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
@@ -2346,7 +2346,6 @@ class GroupCoordinatorTest {
     val offset = offsetAndMetadata(0)
 
     val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols)
-    val assignedMemberId = joinGroupResult.memberId
     val generationId = joinGroupResult.generationId
     val joinGroupError = joinGroupResult.error
     assertEquals(Errors.NONE, joinGroupError)
diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
index f2cdd42f79ac2..aac7ad1a56d8a 100644
--- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
@@ -510,7 +510,7 @@ class KafkaApisTest {
   }
 
   @Test
-  def testJoinGroupProtocolsOrder: Unit = {
+  def testJoinGroupProtocolsOrder(): Unit = {
     val protocols = List(
       new JoinGroupRequestProtocol().setName("first").setMetadata("first".getBytes()),
       new JoinGroupRequestProtocol().setName("second").setMetadata("second".getBytes())

From c758122ce59674ec3e33618d896e4e5cdbb45e87 Mon Sep 17 00:00:00 2001
From: Vahid Hashemian 
Date: Sat, 15 Jun 2019 21:00:09 -0700
Subject: [PATCH 0370/1071] MINOR: Fix expected output in Streams quickstart

Include the topic config `segment.bytes`.

Author: Vahid Hashemian 

Reviewers: Gwen Shapira

Closes #6945 from vahidhashemian/minor/update_streams_quickstart_doc
---
 docs/streams/quickstart.html | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/streams/quickstart.html b/docs/streams/quickstart.html
index c89f4054ae2dc..4c0d0c92b732f 100644
--- a/docs/streams/quickstart.html
+++ b/docs/streams/quickstart.html
@@ -165,10 +165,10 @@ 
    Step 3
 
     > bin/kafka-topics.sh --bootstrap-server localhost:9092 --describe
 
-Topic:streams-plaintext-input	PartitionCount:1	ReplicationFactor:1	Configs:
-    Topic: streams-plaintext-input	Partition: 0	Leader: 0	Replicas: 0	Isr: 0
-Topic:streams-wordcount-output	PartitionCount:1	ReplicationFactor:1	Configs:cleanup.policy=compact
+Topic:streams-wordcount-output	PartitionCount:1	ReplicationFactor:1	Configs:cleanup.policy=compact,segment.bytes=1073741824
 	Topic: streams-wordcount-output	Partition: 0	Leader: 0	Replicas: 0	Isr: 0
+Topic:streams-plaintext-input	PartitionCount:1	ReplicationFactor:1	Configs:segment.bytes=1073741824
+	Topic: streams-plaintext-input	Partition: 0	Leader: 0	Replicas: 0	Isr: 0
 
    
 
 
    Step 4: Start the Wordcount Application
    

From 57baa4079d9fc14103411f790b9a025c9f2146a4 Mon Sep 17 00:00:00 2001
From: Vikas Singh <50422828+soondenana@users.noreply.github.com>
Date: Mon, 17 Jun 2019 08:52:37 -0700
Subject: [PATCH 0371/1071] KAFKA-8457; Move `Log' reference from `Replica`
 into `Partition` (#6841)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A `Partition` object contain one or many `Replica` objects. These replica
objects in turn can have the "log" if the replica corresponds to the
local node. All the code in Partition or ReplicaManager peek into
replica object to fetch the log if they need to operate on that. As
replica object can represent a local replica or a remote one, this
lead to a bunch of "if-else" code in log fetch and offset update code.

NOTE: In addition to a "log" that is in use during normal operation, if
an alter log directory command is issued, we also create a future log
object. This object catches up with local log and then we switch the log
directory. So temporarily a Partition can have two local logs. Before
this change both logs are inside replica objects.

This change is an attempt to untangle this relationship. In particular
it moves `Log` from `Replica` into `Partition`. So a partition contains
a local log to which all writes go and possibly a "future log" if the
partition is being moved between directories. Additionally, it maintains
a list of remote replicas for offset and "caught up time" data that it uses
for replication protocol.

Reviewers: José Armando García Sancio , Jason Gustafson 
---
 .../main/scala/kafka/cluster/Partition.scala  | 521 +++++++++---------
 .../main/scala/kafka/cluster/Replica.scala    | 190 +------
 core/src/main/scala/kafka/log/Log.scala       |  95 +++-
 .../src/main/scala/kafka/log/LogManager.scala |   1 +
 .../src/main/scala/kafka/log/LogSegment.scala |   2 +-
 .../kafka/server/AbstractFetcherThread.scala  |   2 +-
 .../kafka/server/DelayedDeleteRecords.scala   |   2 +-
 .../kafka/server/LogOffsetMetadata.scala      |   2 +-
 .../server/ReplicaAlterLogDirsThread.scala    |  19 +-
 .../kafka/server/ReplicaFetcherThread.scala   |  30 +-
 .../scala/kafka/server/ReplicaManager.scala   |  88 ++-
 .../api/AdminClientIntegrationTest.scala      |  18 +-
 .../kafka/api/ConsumerBounceTest.scala        |   2 +-
 .../admin/ReassignPartitionsClusterTest.scala |   6 +-
 .../unit/kafka/cluster/PartitionTest.scala    | 164 +++---
 .../unit/kafka/cluster/ReplicaTest.scala      |  38 +-
 .../kafka/log/LogCleanerManagerTest.scala     |   8 +-
 ...gCleanerParameterizedIntegrationTest.scala |   4 +-
 .../scala/unit/kafka/log/LogCleanerTest.scala |   5 +-
 .../scala/unit/kafka/log/LogManagerTest.scala |   8 +-
 .../test/scala/unit/kafka/log/LogTest.scala   |  57 +-
 .../kafka/log/ProducerStateManagerTest.scala  |   4 +-
 .../server/HighwatermarkPersistenceTest.scala |  34 +-
 .../unit/kafka/server/ISRExpirationTest.scala |  83 ++-
 .../unit/kafka/server/LogDirFailureTest.scala |   8 +-
 .../unit/kafka/server/LogOffsetTest.scala     |   3 +-
 .../unit/kafka/server/LogRecoveryTest.scala   |  12 +-
 .../ReplicaAlterLogDirsThreadTest.scala       | 147 +++--
 .../server/ReplicaFetcherThreadTest.scala     | 185 ++++---
 .../server/ReplicaManagerQuotasTest.scala     |  32 +-
 .../kafka/server/ReplicaManagerTest.scala     |  50 +-
 .../unit/kafka/server/SimpleFetchTest.scala   |  22 +-
 ...venReplicationProtocolAcceptanceTest.scala |   2 +-
 .../epoch/LeaderEpochIntegrationTest.scala    |   2 +-
 .../epoch/OffsetsForLeaderEpochTest.scala     |   4 +-
 .../scala/unit/kafka/utils/TestUtils.scala    |   6 +-
 36 files changed, 907 insertions(+), 949 deletions(-)

diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala
index 251c57021c3db..544fc0c170933 100755
--- a/core/src/main/scala/kafka/cluster/Partition.scala
+++ b/core/src/main/scala/kafka/cluster/Partition.scala
@@ -170,8 +170,7 @@ class Partition(val topicPartition: TopicPartition,
   def topic: String = topicPartition.topic
   def partitionId: Int = topicPartition.partition
 
-  // allReplicasMap includes both assigned replicas and the future replica if there is ongoing replica movement
-  private val allReplicasMap = new Pool[Int, Replica]
+  private val remoteReplicasMap = new Pool[Int, Replica]
   // The read lock is only required when multiple reads are executed and needs to be in a consistent manner
   private val leaderIsrUpdateLock = new ReentrantReadWriteLock
   private var zkVersion: Int = LeaderAndIsr.initialZKVersion
@@ -180,7 +179,18 @@ class Partition(val topicPartition: TopicPartition,
   // defined when this broker is leader for partition
   @volatile private var leaderEpochStartOffsetOpt: Option[Long] = None
   @volatile var leaderReplicaIdOpt: Option[Int] = None
-  @volatile var inSyncReplicas: Set[Replica] = Set.empty[Replica]
+  @volatile var inSyncReplicas: Set[Int] = Set.empty[Int]
+  // Includes all valid broker ids (@see Request::isValidBrokerId) that contain logs. Doesn't contain
+  // future log replica id
+  @volatile var allReplicaIds: scala.collection.mutable.Set[Int] = scala.collection.mutable.Set(localBrokerId)
+
+  // Logs belonging to this partition. Majority of time it will be only one log, but if log directory
+  // is getting changed (as a result of ReplicaAlterLogDirs command), we may have two logs until copy
+  // completes and a switch to new location is performed.
+  // log and futureLog variables defined below are used to capture this
+  @volatile var log: Option[Log] = None
+  // If ReplicaAlterLogDir command is in progress, this is future location of the log
+  @volatile var futureLog: Option[Log] = None
 
   /* Epoch of the controller that last changed the leader. This needs to be initialized correctly upon broker startup.
    * One way of doing that is through the controller's start replica state change command. When a new broker starts up
@@ -190,13 +200,11 @@ class Partition(val topicPartition: TopicPartition,
   private var controllerEpoch: Int = KafkaController.InitialControllerEpoch
   this.logIdent = s"[Partition $topicPartition broker=$localBrokerId] "
 
-  private def isReplicaLocal(replicaId: Int): Boolean = replicaId == localBrokerId || replicaId == Request.FutureLocalReplicaId
-
   private val tags = Map("topic" -> topic, "partition" -> partitionId.toString)
 
   newGauge("UnderReplicated",
     new Gauge[Int] {
-      def value = {
+      def value: Int = {
         if (isUnderReplicated) 1 else 0
       }
     },
@@ -205,8 +213,8 @@ class Partition(val topicPartition: TopicPartition,
 
   newGauge("InSyncReplicasCount",
     new Gauge[Int] {
-      def value = {
-        if (isLeaderReplicaLocal) inSyncReplicas.size else 0
+      def value: Int = {
+        if (isLeader) inSyncReplicas.size else 0
       }
     },
     tags
@@ -214,7 +222,7 @@ class Partition(val topicPartition: TopicPartition,
 
   newGauge("UnderMinIsr",
     new Gauge[Int] {
-      def value = {
+      def value: Int = {
         if (isUnderMinIsr) 1 else 0
       }
     },
@@ -223,7 +231,7 @@ class Partition(val topicPartition: TopicPartition,
 
   newGauge("AtMinIsr",
     new Gauge[Int] {
-      def value = {
+      def value: Int = {
         if (isAtMinIsr) 1 else 0
       }
     },
@@ -232,8 +240,8 @@ class Partition(val topicPartition: TopicPartition,
 
   newGauge("ReplicasCount",
     new Gauge[Int] {
-      def value = {
-        if (isLeaderReplicaLocal) assignedReplicas.size else 0
+      def value: Int = {
+        if (isLeader) allReplicaIds.size else 0
       }
     },
     tags
@@ -241,36 +249,22 @@ class Partition(val topicPartition: TopicPartition,
 
   newGauge("LastStableOffsetLag",
     new Gauge[Long] {
-      def value = {
-        leaderReplicaIfLocal.map { replica =>
-          replica.highWatermark - replica.lastStableOffset
-        }.getOrElse(0)
+      def value: Long = {
+        log.map(_.lastStableOffsetLag).getOrElse(0)
       }
     },
     tags
   )
 
-  private def isLeaderReplicaLocal: Boolean = leaderReplicaIfLocal.isDefined
-
   def isUnderReplicated: Boolean =
-    isLeaderReplicaLocal && inSyncReplicas.size < assignedReplicas.size
+    isLeader && inSyncReplicas.size < allReplicaIds.size
 
   def isUnderMinIsr: Boolean = {
-    leaderReplicaIfLocal match {
-      case Some(leaderReplica) =>
-        inSyncReplicas.size < leaderReplica.log.get.config.minInSyncReplicas
-      case None =>
-        false
-    }
+    leaderLogIfLocal.exists { inSyncReplicas.size < _.config.minInSyncReplicas }
   }
 
   def isAtMinIsr: Boolean = {
-    leaderReplicaIfLocal match {
-      case Some(leaderReplica) =>
-        inSyncReplicas.size == leaderReplica.log.get.config.minInSyncReplicas
-      case None =>
-        false
-    }
+    leaderLogIfLocal.exists { inSyncReplicas.size == _.config.minInSyncReplicas }
   }
 
   /**
@@ -286,43 +280,72 @@ class Partition(val topicPartition: TopicPartition,
     // current replica and the existence of the future replica, no other thread can update the log directory of the
     // current replica or remove the future replica.
     inWriteLock(leaderIsrUpdateLock) {
-      val currentReplica = localReplicaOrException
-      val currentLog = currentReplica.log.get
-      if (currentLog.dir.getParent == logDir)
+      val currentLogDir = localLogOrException.dir.getParent
+      if (currentLogDir == logDir) {
+        info(s"Current log directory $currentLogDir is same as requested log dir $logDir. " +
+          s"Skipping future replica creation.")
         false
-      else {
-        futureLocalReplica match {
-          case Some(replica) =>
-            val futureReplicaLogDir = replica.log.get.dir.getParent
-            if (futureReplicaLogDir != logDir)
-              throw new IllegalStateException(s"The future log dir $futureReplicaLogDir of $topicPartition is " +
+      } else {
+        futureLog match {
+          case Some(partitionFutureLog) =>
+            val futureLogDir = partitionFutureLog.dir.getParent
+            if (futureLogDir != logDir)
+              throw new IllegalStateException(s"The future log dir $futureLogDir of $topicPartition is " +
                 s"different from the requested log dir $logDir")
             false
           case None =>
-            getOrCreateReplica(Request.FutureLocalReplicaId, isNew = false, highWatermarkCheckpoints)
+            createLogIfNotExists(Request.FutureLocalReplicaId, isNew = false, isFutureReplica = true, highWatermarkCheckpoints)
             true
         }
       }
     }
   }
 
-  def getOrCreateReplica(replicaId: Int, isNew: Boolean, offsetCheckpoints: OffsetCheckpoints): Replica = {
-    allReplicasMap.getAndMaybePut(replicaId, {
-      if (isReplicaLocal(replicaId)) {
-        val props = stateStore.fetchTopicConfig()
-        val config = LogConfig.fromProps(logManager.currentDefaultConfig.originals, props)
-        val log = logManager.getOrCreateLog(topicPartition, config, isNew, replicaId == Request.FutureLocalReplicaId)
-        val checkpointHighWatermark = offsetCheckpoints.fetch(log.dir.getParent, topicPartition).getOrElse {
-          info(s"No checkpointed highwatermark is found for partition $topicPartition")
-          0L
-        }
-        val initialHighWatermark = math.min(checkpointHighWatermark, log.logEndOffset)
-        new Replica(replicaId, topicPartition, time, initialHighWatermark, Some(log))
-      } else new Replica(replicaId, topicPartition, time)
-    })
+  /**
+   * Creates a remote replica and puts that in a map. A future invocation to create
+   * replica with same id will return previously created object.
+   */
+  def getOrCreateReplica(replicaId: Int): Replica = {
+    require(replicaId != localBrokerId, s"Cannot create replica for local broker: $replicaId")
+    val newReplica = remoteReplicasMap.getAndMaybePut(replicaId, new Replica(replicaId, topicPartition))
+    allReplicaIds.add(replicaId)
+    require(remoteReplicasMap.size + 1 == allReplicaIds.size,
+      s"Invalid state. All Replica Ids: $allReplicaIds, remote replica ids: ${remoteReplicasMap.keys}")
+    newReplica
   }
 
-  def getReplica(replicaId: Int): Option[Replica] = Option(allReplicasMap.get(replicaId))
+  def createLogIfNotExists(replicaId: Int, isNew: Boolean, isFutureReplica: Boolean, offsetCheckpoints: OffsetCheckpoints): Unit = {
+    isFutureReplica match {
+      case true if futureLog.isEmpty =>
+        val log = createLog(replicaId, isNew, isFutureReplica, offsetCheckpoints)
+        this.futureLog = Option(log)
+      case false if log.isEmpty =>
+        val log = createLog(replicaId, isNew, isFutureReplica, offsetCheckpoints)
+        this.log = Option(log)
+      case _ => trace(s"${if (isFutureReplica) "Future Log" else "Log"} already exists.")
+    }
+  }
+
+  private def createLog(replicaId: Int, isNew: Boolean, isFutureReplica: Boolean, offsetCheckpoints: OffsetCheckpoints): Log = {
+    val props = stateStore.fetchTopicConfig()
+    val config = LogConfig.fromProps(logManager.currentDefaultConfig.originals, props)
+    val log = logManager.getOrCreateLog(topicPartition, config, isNew, isFutureReplica)
+    val checkpointHighWatermark = offsetCheckpoints.fetch(log.dir.getParent, topicPartition).getOrElse {
+      info(s"No checkpointed highwatermark is found for partition $topicPartition")
+      0L
+    }
+    val initialHighWatermark = math.min(checkpointHighWatermark, log.logEndOffset)
+    log.highWatermarkMetadata = LogOffsetMetadata(initialHighWatermark)
+
+    info(s"Log loaded for partition $topicPartition with initial high watermark $initialHighWatermark")
+    log
+  }
+
+  def getReplica(replicaId: Int): Option[Replica] = Option(remoteReplicasMap.get(replicaId))
+
+  private def getReplicaOrException(replicaId: Int): Replica = getReplica(replicaId).getOrElse{
+    throw new ReplicaNotAvailableException(s"Replica with id $replicaId is not available on broker $localBrokerId")
+  }
 
   private def checkCurrentLeaderEpoch(remoteLeaderEpochOpt: Optional[Integer]): Errors = {
     if (!remoteLeaderEpochOpt.isPresent) {
@@ -339,22 +362,21 @@ class Partition(val topicPartition: TopicPartition,
     }
   }
 
-  private def getLocalReplica(replicaId: Int,
-                              currentLeaderEpoch: Optional[Integer],
-                              requireLeader: Boolean): Either[Replica, Errors] = {
+  private def getLocalLog(currentLeaderEpoch: Optional[Integer],
+                          requireLeader: Boolean): Either[Log, Errors] = {
     checkCurrentLeaderEpoch(currentLeaderEpoch) match {
       case Errors.NONE =>
-        if (requireLeader && !leaderReplicaIdOpt.contains(localBrokerId)) {
+        if (requireLeader && !isLeader) {
           Right(Errors.NOT_LEADER_FOR_PARTITION)
         } else {
-          val replica = allReplicasMap.get(replicaId)
-          if (replica == null) {
-            if (requireLeader)
-              Right(Errors.NOT_LEADER_FOR_PARTITION)
-            else
-              Right(Errors.REPLICA_NOT_AVAILABLE)
-          } else {
-            Left(replica)
+          log match {
+            case Some(partitionLog) =>
+              Left(partitionLog)
+            case _ =>
+              if (requireLeader)
+                Right(Errors.NOT_LEADER_FOR_PARTITION)
+              else
+                Right(Errors.REPLICA_NOT_AVAILABLE)
           }
         }
       case error =>
@@ -362,68 +384,68 @@ class Partition(val topicPartition: TopicPartition,
     }
   }
 
-  def localReplica: Option[Replica] = getReplica(localBrokerId)
-
-  def localReplicaOrException: Replica = localReplica.getOrElse {
-    throw new ReplicaNotAvailableException(s"Replica for partition $topicPartition is not available " +
+  def localLogOrException: Log = log.getOrElse {
+    throw new ReplicaNotAvailableException(s"Log for partition $topicPartition is not available " +
       s"on broker $localBrokerId")
   }
 
-  def futureLocalReplica: Option[Replica] = getReplica(Request.FutureLocalReplicaId)
-
-  def futureLocalReplicaOrException: Replica = futureLocalReplica.getOrElse {
-    throw new ReplicaNotAvailableException(s"Future replica for partition $topicPartition is not available " +
+  def futureLocalLogOrException: Log = futureLog.getOrElse {
+    throw new ReplicaNotAvailableException(s"Future log for partition $topicPartition is not available " +
       s"on broker $localBrokerId")
   }
 
-  def leaderReplicaIfLocal: Option[Replica] = {
-    if (leaderReplicaIdOpt.contains(localBrokerId))
-      localReplica
-    else
-      None
+  def leaderLogIfLocal: Option[Log] = {
+    log.filter(_ => isLeader)
   }
 
-  private def localReplicaWithEpochOrException(currentLeaderEpoch: Optional[Integer],
-                                               requireLeader: Boolean): Replica = {
-    getLocalReplica(localBrokerId, currentLeaderEpoch, requireLeader) match {
-      case Left(replica) => replica
+  /**
+   * Returns true if this node is currently leader for the Partition.
+   */
+  def isLeader: Boolean = leaderReplicaIdOpt.contains(localBrokerId)
+
+  private def localLogWithEpochOrException(currentLeaderEpoch: Optional[Integer],
+                                           requireLeader: Boolean): Log = {
+    getLocalLog(currentLeaderEpoch, requireLeader) match {
+      case Left(localLog) => localLog
       case Right(error) =>
-        throw error.exception(s"Failed to find ${if (requireLeader) "leader " else ""} for " +
+        throw error.exception(s"Failed to find ${if (requireLeader) "leader " else ""} log for " +
           s"partition $topicPartition with leader epoch $currentLeaderEpoch. The current leader " +
           s"is $leaderReplicaIdOpt and the current epoch $leaderEpoch")
     }
   }
 
-  // Visible for testing
-  def addReplicaIfNotExists(replica: Replica): Replica =
-    allReplicasMap.putIfNotExists(replica.brokerId, replica)
+  // Visible for testing -- Only used in tests to add replica to existing partition
+  def addReplicaIfNotExists(replica: Replica): Replica = {
+    allReplicaIds.add(replica.brokerId)
+    remoteReplicasMap.getAndMaybePut(replica.brokerId, replica)
+  }
 
-  def assignedReplicas: Set[Replica] =
-    allReplicasMap.values.filter(replica => Request.isValidBrokerId(replica.brokerId)).toSet
+  // Visible for testing -- Used by unit tests to set log for this partition
+  def setLog(log: Log, isFutureLog: Boolean): Unit = {
+    if (isFutureLog)
+      futureLog = Some(log)
+    else
+      this.log = Some(log)
+  }
 
-  def allReplicas: Set[Replica] =
-    allReplicasMap.values.toSet
+  def remoteReplicas: Set[Replica] =
+    remoteReplicasMap.values.toSet
 
   private def removeReplica(replicaId: Int) {
-    allReplicasMap.remove(replicaId)
+    require(replicaId != localBrokerId, s"Cannot remove replica for local broker: $replicaId")
+    allReplicaIds.remove(replicaId)
+    remoteReplicasMap.remove(replicaId)
   }
 
   def futureReplicaDirChanged(newDestinationDir: String): Boolean = {
     inReadLock(leaderIsrUpdateLock) {
-      futureLocalReplica match {
-        case Some(replica) =>
-          if (replica.log.get.dir.getParent != newDestinationDir)
-            true
-          else
-            false
-        case None => false
-      }
+      futureLog.exists(_.dir.getParent != newDestinationDir)
     }
   }
 
   def removeFutureLocalReplica(deleteFromLogDir: Boolean = true) {
     inWriteLock(leaderIsrUpdateLock) {
-      allReplicasMap.remove(Request.FutureLocalReplicaId)
+      futureLog = None
       if (deleteFromLogDir)
         logManager.asyncDelete(topicPartition, isFuture = true)
     }
@@ -433,25 +455,25 @@ class Partition(val topicPartition: TopicPartition,
   // Only ReplicaAlterDirThread will call this method and ReplicaAlterDirThread should remove the partition
   // from its partitionStates if this method returns true
   def maybeReplaceCurrentWithFutureReplica(): Boolean = {
-    val replica = localReplicaOrException
-    val futureReplicaLEO = futureLocalReplica.map(_.logEndOffset)
-    if (futureReplicaLEO.contains(replica.logEndOffset)) {
+    val localReplicaLEO = localLogOrException.logEndOffset
+    val futureReplicaLEO = futureLog.map(_.logEndOffset)
+    if (futureReplicaLEO.contains(localReplicaLEO)) {
       // The write lock is needed to make sure that while ReplicaAlterDirThread checks the LEO of the
       // current replica, no other thread can update LEO of the current replica via log truncation or log append operation.
       inWriteLock(leaderIsrUpdateLock) {
-        futureLocalReplica match {
-          case Some(futureReplica) =>
-            if (replica.logEndOffset == futureReplica.logEndOffset) {
+        futureLog match {
+          case Some(futurePartitionLog) =>
+            if (log.exists(_.logEndOffset == futurePartitionLog.logEndOffset)) {
               logManager.replaceCurrentWithFutureLog(topicPartition)
-              replica.log = futureReplica.log
-              futureReplica.log = None
-              allReplicasMap.remove(Request.FutureLocalReplicaId)
+              log = futureLog
+              removeFutureLocalReplica(false)
               true
             } else false
           case None =>
             // Future replica is removed by a non-ReplicaAlterLogDirsThread before this method is called
             // In this case the partition should have been removed from state of the ReplicaAlterLogDirsThread
-            // Return false so that ReplicaAlterLogDirsThread does not have to remove this partition from the state again to avoid race condition
+            // Return false so that ReplicaAlterLogDirsThread does not have to remove this partition from the
+            // state again to avoid race condition
             false
         }
       }
@@ -461,8 +483,11 @@ class Partition(val topicPartition: TopicPartition,
   def delete() {
     // need to hold the lock to prevent appendMessagesToLeader() from hitting I/O exceptions due to log being deleted
     inWriteLock(leaderIsrUpdateLock) {
-      allReplicasMap.clear()
-      inSyncReplicas = Set.empty[Replica]
+      remoteReplicasMap.clear()
+      allReplicaIds = scala.collection.mutable.Set(localBrokerId)
+      log = None
+      futureLog = None
+      inSyncReplicas = Set.empty[Int]
       leaderReplicaIdOpt = None
       leaderEpochStartOffsetOpt = None
       Partition.removeMetrics(topicPartition)
@@ -484,21 +509,23 @@ class Partition(val topicPartition: TopicPartition,
                  correlationId: Int,
                  highWatermarkCheckpoints: OffsetCheckpoints): Boolean = {
     val (leaderHWIncremented, isNewLeader) = inWriteLock(leaderIsrUpdateLock) {
-      val newAssignedReplicas = partitionStateInfo.basePartitionState.replicas.asScala.map(_.toInt)
       // record the epoch of the controller that made the leadership decision. This is useful while updating the isr
       // to maintain the decision maker controller's epoch in the zookeeper path
       controllerEpoch = partitionStateInfo.basePartitionState.controllerEpoch
       // add replicas that are new
-      val newInSyncReplicas = partitionStateInfo.basePartitionState.isr.asScala.map {
-        id => getOrCreateReplica(id, partitionStateInfo.isNew, highWatermarkCheckpoints)
-      }.toSet
+      val newInSyncReplicas = partitionStateInfo.basePartitionState.isr.asScala.map(_.toInt)
+      newInSyncReplicas.filter(_ != localBrokerId).foreach(getOrCreateReplica)
+      inSyncReplicas = newInSyncReplicas.toSet
+
       // remove assigned replicas that have been removed by the controller
-      (assignedReplicas.map(_.brokerId) -- newAssignedReplicas).foreach(removeReplica)
-      inSyncReplicas = newInSyncReplicas
-      newAssignedReplicas.foreach(id => getOrCreateReplica(id, partitionStateInfo.isNew, highWatermarkCheckpoints))
+      val newAssignedReplicas = partitionStateInfo.basePartitionState.replicas.asScala.map(_.toInt)
+      (remoteReplicasMap.keys -- newAssignedReplicas).foreach(removeReplica)
+      newAssignedReplicas.filter(_ != localBrokerId).foreach(getOrCreateReplica)
 
-      val leaderReplica = localReplicaOrException
-      val leaderEpochStartOffset = leaderReplica.logEndOffset
+      createLogIfNotExists(localBrokerId, partitionStateInfo.isNew, isFutureReplica = false, highWatermarkCheckpoints)
+
+      val leaderLog = localLogOrException
+      val leaderEpochStartOffset = leaderLog.logEndOffset
       info(s"$topicPartition starts at Leader Epoch ${partitionStateInfo.basePartitionState.leaderEpoch} from " +
         s"offset $leaderEpochStartOffset. Previous Leader Epoch was: $leaderEpoch")
 
@@ -512,27 +539,24 @@ class Partition(val topicPartition: TopicPartition,
       // to ensure that these followers can truncate to the right offset, we must cache the new
       // leader epoch and the start offset since it should be larger than any epoch that a follower
       // would try to query.
-      leaderReplica.log.foreach { log =>
-        log.maybeAssignEpochStartOffset(leaderEpoch, leaderEpochStartOffset)
-      }
+      leaderLog.maybeAssignEpochStartOffset(leaderEpoch, leaderEpochStartOffset)
 
-      val isNewLeader = !leaderReplicaIdOpt.contains(localBrokerId)
-      val curLeaderLogEndOffset = leaderReplica.logEndOffset
+      val isNewLeader = !isLeader
+      val curLeaderLogEndOffset = leaderLog.logEndOffset
       val curTimeMs = time.milliseconds
       // initialize lastCaughtUpTime of replicas as well as their lastFetchTimeMs and lastFetchLeaderLogEndOffset.
-      (assignedReplicas - leaderReplica).foreach { replica =>
-        val lastCaughtUpTimeMs = if (inSyncReplicas.contains(replica)) curTimeMs else 0L
+      remoteReplicas.foreach { replica =>
+        val lastCaughtUpTimeMs = if (inSyncReplicas.contains(replica.brokerId)) curTimeMs else 0L
         replica.resetLastCaughtUpTime(curLeaderLogEndOffset, curTimeMs, lastCaughtUpTimeMs)
       }
 
       if (isNewLeader) {
         // construct the high watermark metadata for the new leader replica
-        leaderReplica.maybeFetchHighWatermarkOffsetMetadata()
+        leaderLog.maybeFetchHighWatermarkOffsetMetadata()
         // mark local replica as the leader after converting hw
         leaderReplicaIdOpt = Some(localBrokerId)
         // reset log end offset for remote replicas
-        assignedReplicas.filter(_.brokerId != localBrokerId).foreach { replica =>
-          replica.updateFetchState(
+        remoteReplicas.foreach { _.updateFetchState(
             followerFetchOffsetMetadata = LogOffsetMetadata.UnknownOffsetMetadata,
             followerStartOffset = Log.UnknownOffset,
             followerFetchTimeMs = 0L,
@@ -541,7 +565,7 @@ class Partition(val topicPartition: TopicPartition,
         }
       }
       // we may need to increment high watermark since ISR could be down to 1
-      (maybeIncrementLeaderHW(leaderReplica), isNewLeader)
+      (maybeIncrementLeaderHW(leaderLog), isNewLeader)
     }
     // some delayed operations may be unblocked after HW changed
     if (leaderHWIncremented)
@@ -567,10 +591,12 @@ class Partition(val topicPartition: TopicPartition,
       // to maintain the decision maker controller's epoch in the zookeeper path
       controllerEpoch = partitionStateInfo.basePartitionState.controllerEpoch
       // add replicas that are new
-      newAssignedReplicas.foreach(id => getOrCreateReplica(id, partitionStateInfo.isNew, highWatermarkCheckpoints))
+      newAssignedReplicas.filter(_ != localBrokerId).foreach(id => getOrCreateReplica(id))
+      createLogIfNotExists(localBrokerId, partitionStateInfo.isNew, isFutureReplica = false, highWatermarkCheckpoints)
+
       // remove assigned replicas that have been removed by the controller
-      (assignedReplicas.map(_.brokerId) -- newAssignedReplicas).foreach(removeReplica)
-      inSyncReplicas = Set.empty[Replica]
+      (remoteReplicasMap.keys -- newAssignedReplicas).foreach(removeReplica)
+      inSyncReplicas = Set.empty[Int]
       leaderEpoch = partitionStateInfo.basePartitionState.leaderEpoch
       leaderEpochStartOffsetOpt = None
       zkVersion = partitionStateInfo.basePartitionState.zkVersion
@@ -646,21 +672,20 @@ class Partition(val topicPartition: TopicPartition,
   private def maybeExpandIsr(followerReplica: Replica, followerFetchTimeMs: Long): Boolean = {
     inWriteLock(leaderIsrUpdateLock) {
       // check if this replica needs to be added to the ISR
-      leaderReplicaIfLocal match {
-        case Some(leaderReplica) =>
-          val leaderHighwatermark = leaderReplica.highWatermark
-          if (!inSyncReplicas.contains(followerReplica) && isFollowerInSync(followerReplica, leaderHighwatermark)) {
-            val newInSyncReplicas = inSyncReplicas + followerReplica
-
-            info(s"Expanding ISR from ${inSyncReplicas.map(_.brokerId).mkString(",")} " +
-              s"to ${newInSyncReplicas.map(_.brokerId).mkString(",")}")
+      leaderLogIfLocal match {
+        case Some(leaderLog) =>
+          val leaderHighwatermark = leaderLog.highWatermark
+          if (!inSyncReplicas.contains(followerReplica.brokerId) && isFollowerInSync(followerReplica, leaderHighwatermark)) {
+            val newInSyncReplicas = inSyncReplicas + followerReplica.brokerId
+            info(s"Expanding ISR from ${inSyncReplicas.mkString(",")} " +
+              s"to ${newInSyncReplicas.mkString(",")}")
 
             // update ISR in ZK and cache
             expandIsr(newInSyncReplicas)
           }
           // check if the HW of the partition can now be incremented
           // since the replica may already be in the ISR and its LEO has just incremented
-          maybeIncrementLeaderHW(leaderReplica, followerFetchTimeMs)
+          maybeIncrementLeaderHW(leaderLog, followerFetchTimeMs)
         case None => false // nothing to do if no longer leader
       }
     }
@@ -680,22 +705,28 @@ class Partition(val topicPartition: TopicPartition,
    * produce request.
    */
   def checkEnoughReplicasReachOffset(requiredOffset: Long): (Boolean, Errors) = {
-    leaderReplicaIfLocal match {
-      case Some(leaderReplica) =>
+    leaderLogIfLocal match {
+      case Some(leaderLog) =>
         // keep the current immutable replica list reference
         val curInSyncReplicas = inSyncReplicas
 
         if (isTraceEnabled) {
-          def logEndOffsetString(r: Replica) = s"broker ${r.brokerId}: ${r.logEndOffset}"
-          val (ackedReplicas, awaitingReplicas) = curInSyncReplicas.partition { replica =>
-            replica.logEndOffset >= requiredOffset
+          def logEndOffsetString: ((Int, Long)) => String = {
+            case (brokerId, logEndOffset) => s"broker $brokerId: $logEndOffset"
           }
-          trace(s"Progress awaiting ISR acks for offset $requiredOffset: acked: ${ackedReplicas.map(logEndOffsetString)}, " +
+
+          val curInSyncReplicaObjects = (curInSyncReplicas - localBrokerId).map(getReplicaOrException)
+          val replicaInfo = curInSyncReplicaObjects.map(replica => (replica.brokerId, replica.logEndOffset))
+          val localLogInfo = (localBrokerId, localLogOrException.logEndOffset)
+          val (ackedReplicas, awaitingReplicas) = (replicaInfo + localLogInfo).partition { _._2 >= requiredOffset}
+
+          trace(s"Progress awaiting ISR acks for offset $requiredOffset: " +
+            s"acked: ${ackedReplicas.map(logEndOffsetString)}, " +
             s"awaiting ${awaitingReplicas.map(logEndOffsetString)}")
         }
 
-        val minIsr = leaderReplica.log.get.config.minInSyncReplicas
-        if (leaderReplica.highWatermark >= requiredOffset) {
+        val minIsr = leaderLog.config.minInSyncReplicas
+        if (leaderLog.highWatermark >= requiredOffset) {
           /*
            * The topic may be configured not to accept messages if there are not enough replicas in ISR
            * in this scenario the request was already appended locally and then added to the purgatory before the ISR was shrunk
@@ -729,24 +760,29 @@ class Partition(val topicPartition: TopicPartition,
    * Note There is no need to acquire the leaderIsrUpdate lock here
    * since all callers of this private API acquire that lock
    */
-  private def maybeIncrementLeaderHW(leaderReplica: Replica, curTime: Long = time.milliseconds): Boolean = {
-    val allLogEndOffsets = assignedReplicas.filter { replica =>
-      curTime - replica.lastCaughtUpTimeMs <= replicaLagTimeMaxMs || inSyncReplicas.contains(replica)
+  private def maybeIncrementLeaderHW(leaderLog: Log, curTime: Long = time.milliseconds): Boolean = {
+    val replicaLogEndOffsets = remoteReplicas.filter { replica =>
+      curTime - replica.lastCaughtUpTimeMs <= replicaLagTimeMaxMs || inSyncReplicas.contains(replica.brokerId)
     }.map(_.logEndOffsetMetadata)
-    val newHighWatermark = allLogEndOffsets.min(new LogOffsetMetadata.OffsetOrdering)
-    val oldHighWatermark = leaderReplica.highWatermarkMetadata
+    val newHighWatermark = (replicaLogEndOffsets + leaderLog.logEndOffsetMetadata).min(new LogOffsetMetadata.OffsetOrdering)
+    val oldHighWatermark = leaderLog.highWatermarkMetadata
 
     // Ensure that the high watermark increases monotonically. We also update the high watermark when the new
     // offset metadata is on a newer segment, which occurs whenever the log is rolled to a new segment.
     if (oldHighWatermark.messageOffset < newHighWatermark.messageOffset ||
       (oldHighWatermark.messageOffset == newHighWatermark.messageOffset && oldHighWatermark.onOlderSegment(newHighWatermark))) {
-      leaderReplica.highWatermarkMetadata = newHighWatermark
+      leaderLog.highWatermarkMetadata = newHighWatermark
       debug(s"High watermark updated to $newHighWatermark")
       true
     } else {
-      def logEndOffsetString(r: Replica) = s"replica ${r.brokerId}: ${r.logEndOffsetMetadata}"
+      def logEndOffsetString: ((Int, LogOffsetMetadata)) => String = {
+        case (brokerId, logEndOffsetMetadata) => s"replica $brokerId: $logEndOffsetMetadata"
+      }
+
+      val replicaInfo = remoteReplicas.map(replica => (replica.brokerId, replica.logEndOffsetMetadata))
+      val localLogInfo = (localBrokerId, localLogOrException.logEndOffsetMetadata)
       trace(s"Skipping update high watermark since new hw $newHighWatermark is not larger than old hw $oldHighWatermark. " +
-        s"All current LEOs are ${assignedReplicas.map(logEndOffsetString)}")
+        s"All current LEOs are ${(replicaInfo + localLogInfo).map(logEndOffsetString)}")
       false
     }
   }
@@ -757,12 +793,18 @@ class Partition(val topicPartition: TopicPartition,
    * Low watermark will increase when the leader broker receives either FetchRequest or DeleteRecordsRequest.
    */
   def lowWatermarkIfLeader: Long = {
-    if (!isLeaderReplicaLocal)
+    if (!isLeader)
       throw new NotLeaderForPartitionException(s"Leader not local for partition $topicPartition on broker $localBrokerId")
-    val logStartOffsets = allReplicas.collect {
-      case replica if metadataCache.isBrokerAlive(replica.brokerId) || replica.brokerId == Request.FutureLocalReplicaId => replica.logStartOffset
+    val logStartOffsets = remoteReplicas.collect {
+      case replica if metadataCache.isBrokerAlive(replica.brokerId) => replica.logStartOffset
+    } + localLogOrException.logStartOffset
+
+    futureLog match {
+      case Some(partitionFutureLog) =>
+        CoreUtils.min(logStartOffsets + partitionFutureLog.logStartOffset, 0L)
+      case None =>
+        CoreUtils.min(logStartOffsets, 0L)
     }
-    CoreUtils.min(logStartOffsets, 0L)
   }
 
   /**
@@ -772,19 +814,19 @@ class Partition(val topicPartition: TopicPartition,
 
   def maybeShrinkIsr(replicaMaxLagTimeMs: Long): Unit = {
     val leaderHWIncremented = inWriteLock(leaderIsrUpdateLock) {
-      leaderReplicaIfLocal match {
-        case Some(leaderReplica) =>
-          val outOfSyncReplicas = getOutOfSyncReplicas(leaderReplica, replicaMaxLagTimeMs)
+      leaderLogIfLocal match {
+        case Some(leaderLog) =>
+          val outOfSyncReplicas = getOutOfSyncReplicas(replicaMaxLagTimeMs)
           if (outOfSyncReplicas.nonEmpty) {
             val newInSyncReplicas = inSyncReplicas -- outOfSyncReplicas
             assert(newInSyncReplicas.nonEmpty)
             info("Shrinking ISR from %s to %s. Leader: (highWatermark: %d, endOffset: %d). Out of sync replicas: %s."
-              .format(inSyncReplicas.map(_.brokerId).mkString(","),
-                newInSyncReplicas.map(_.brokerId).mkString(","),
-                leaderReplica.highWatermarkMetadata.messageOffset,
-                leaderReplica.logEndOffset,
-                outOfSyncReplicas.map { replica =>
-                  s"(brokerId: ${replica.brokerId}, endOffset: ${replica.logEndOffset})"
+              .format(inSyncReplicas.mkString(","),
+                newInSyncReplicas.mkString(","),
+                leaderLog.highWatermark,
+                leaderLog.logEndOffset,
+                outOfSyncReplicas.map { replicaId =>
+                  s"(brokerId: $replicaId, endOffset: ${getReplicaOrException(replicaId).logEndOffset})"
                 }.mkString(" ")
               )
             )
@@ -793,7 +835,7 @@ class Partition(val topicPartition: TopicPartition,
             shrinkIsr(newInSyncReplicas)
 
             // we may need to increment high watermark since ISR could be down to 1
-            maybeIncrementLeaderHW(leaderReplica)
+            maybeIncrementLeaderHW(leaderLog)
           } else {
             false
           }
@@ -807,15 +849,16 @@ class Partition(val topicPartition: TopicPartition,
       tryCompleteDelayedRequests()
   }
 
-  private def isFollowerOutOfSync(followerReplica: Replica,
+  private def isFollowerOutOfSync(replicaId: Int,
                                   leaderEndOffset: Long,
                                   currentTimeMs: Long,
                                   maxLagMs: Long): Boolean = {
+    val followerReplica = getReplicaOrException(replicaId)
     followerReplica.logEndOffset != leaderEndOffset &&
       (currentTimeMs - followerReplica.lastCaughtUpTimeMs) > maxLagMs
   }
 
-  def getOutOfSyncReplicas(leaderReplica: Replica, maxLagMs: Long): Set[Replica] = {
+  def getOutOfSyncReplicas(maxLagMs: Long): Set[Int] = {
     /**
      * If the follower already has the same leo as the leader, it will not be considered as out-of-sync,
      * otherwise there are two cases that will be handled here -
@@ -828,10 +871,10 @@ class Partition(val topicPartition: TopicPartition,
      * is violated, that replica is considered to be out of sync
      *
      **/
-    val candidateReplicas = inSyncReplicas - leaderReplica
+    val candidateReplicas = inSyncReplicas - localBrokerId
     val currentTimeMs = time.milliseconds()
-    val leaderEndOffset = leaderReplica.logEndOffset
-    candidateReplicas.filter(r => isFollowerOutOfSync(r, leaderEndOffset, currentTimeMs, maxLagMs))
+    val leaderEndOffset = localLogOrException.logEndOffset
+    candidateReplicas.filter(replicaId => isFollowerOutOfSync(replicaId, leaderEndOffset, currentTimeMs, maxLagMs))
   }
 
   private def doAppendRecordsToFollowerOrFutureReplica(records: MemoryRecords, isFuture: Boolean): Option[LogAppendInfo] = {
@@ -841,13 +884,11 @@ class Partition(val topicPartition: TopicPartition,
       if (isFuture) {
         // Note the replica may be undefined if it is removed by a non-ReplicaAlterLogDirsThread before
         // this method is called
-        futureLocalReplica.map { replica =>
-          replica.log.get.appendAsFollower(records)
-        }
+        futureLog.map { _.appendAsFollower(records) }
       } else {
         // The read lock is needed to prevent the follower replica from being updated while ReplicaAlterDirThread
         // is executing maybeDeleteAndSwapFutureReplica() to replace follower replica with the future replica.
-        Some(localReplicaOrException.log.get.appendAsFollower(records))
+        Some(localLogOrException.appendAsFollower(records))
       }
     }
   }
@@ -857,9 +898,9 @@ class Partition(val topicPartition: TopicPartition,
       doAppendRecordsToFollowerOrFutureReplica(records, isFuture)
     } catch {
       case e: UnexpectedAppendOffsetException =>
-        val replica = if (isFuture) futureLocalReplicaOrException else localReplicaOrException
-        val logEndOffset = replica.logEndOffset
-        if (logEndOffset == replica.logStartOffset &&
+        val log = if (isFuture) futureLocalLogOrException else localLogOrException
+        val logEndOffset = log.logEndOffset
+        if (logEndOffset == log.logStartOffset &&
             e.firstOffset < logEndOffset && e.lastOffset >= logEndOffset) {
           // This may happen if the log start offset on the leader (or current replica) falls in
           // the middle of the batch due to delete records request and the follower tries to
@@ -869,7 +910,7 @@ class Partition(val topicPartition: TopicPartition,
           // (base offset of the batch), which will move recoveryPoint backwards, so we will need
           // to checkpoint the new recovery point before we append
           val replicaName = if (isFuture) "future replica" else "follower"
-          info(s"Unexpected offset in append to $topicPartition. First offset ${e.firstOffset} is less than log start offset ${replica.logStartOffset}." +
+          info(s"Unexpected offset in append to $topicPartition. First offset ${e.firstOffset} is less than log start offset ${log.logStartOffset}." +
                s" Since this is the first record to be appended to the $replicaName's log, will start the log from offset ${e.firstOffset}.")
           truncateFullyAndStartAt(e.firstOffset, isFuture)
           doAppendRecordsToFollowerOrFutureReplica(records, isFuture)
@@ -880,23 +921,22 @@ class Partition(val topicPartition: TopicPartition,
 
   def appendRecordsToLeader(records: MemoryRecords, isFromClient: Boolean, requiredAcks: Int = 0): LogAppendInfo = {
     val (info, leaderHWIncremented) = inReadLock(leaderIsrUpdateLock) {
-      leaderReplicaIfLocal match {
-        case Some(leaderReplica) =>
-          val log = leaderReplica.log.get
-          val minIsr = log.config.minInSyncReplicas
+      leaderLogIfLocal match {
+        case Some(leaderLog) =>
+          val minIsr = leaderLog.config.minInSyncReplicas
           val inSyncSize = inSyncReplicas.size
 
           // Avoid writing to leader if there are not enough insync replicas to make it safe
           if (inSyncSize < minIsr && requiredAcks == -1) {
-            throw new NotEnoughReplicasException(s"The size of the current ISR ${inSyncReplicas.map(_.brokerId)} " +
+            throw new NotEnoughReplicasException(s"The size of the current ISR $inSyncReplicas " +
               s"is insufficient to satisfy the min.isr requirement of $minIsr for partition $topicPartition")
           }
 
-          val info = log.appendAsLeader(records, leaderEpoch = this.leaderEpoch, isFromClient,
+          val info = leaderLog.appendAsLeader(records, leaderEpoch = this.leaderEpoch, isFromClient,
             interBrokerProtocolVersion)
 
           // we may need to increment high watermark since ISR could be down to 1
-          (info, maybeIncrementLeaderHW(leaderReplica))
+          (info, maybeIncrementLeaderHW(leaderLog))
 
         case None =>
           throw new NotLeaderForPartitionException("Leader not local for partition %s on broker %d"
@@ -922,7 +962,7 @@ class Partition(val topicPartition: TopicPartition,
                   fetchOnlyFromLeader: Boolean,
                   minOneMessage: Boolean): LogReadInfo = inReadLock(leaderIsrUpdateLock) {
     // decide whether to only fetch from leader
-    val localReplica = localReplicaWithEpochOrException(currentLeaderEpoch, fetchOnlyFromLeader)
+    val localLog = localLogWithEpochOrException(currentLeaderEpoch, fetchOnlyFromLeader)
 
     /* Read the LogOffsetMetadata prior to performing the read from the log.
      * We use the LogOffsetMetadata to determine if a particular replica is in-sync or not.
@@ -930,10 +970,10 @@ class Partition(val topicPartition: TopicPartition,
      * where data gets appended to the log immediately after the replica has consumed from it
      * This can cause a replica to always be out of sync.
      */
-    val initialHighWatermark = localReplica.highWatermark
-    val initialLogStartOffset = localReplica.logStartOffset
-    val initialLogEndOffset = localReplica.logEndOffset
-    val initialLastStableOffset = localReplica.lastStableOffset
+    val initialHighWatermark = localLog.highWatermark
+    val initialLogStartOffset = localLog.logStartOffset
+    val initialLogEndOffset = localLog.logEndOffset
+    val initialLastStableOffset = localLog.lastStableOffset
 
     val maxOffsetOpt = fetchIsolation match {
       case FetchLogEnd => None
@@ -941,16 +981,9 @@ class Partition(val topicPartition: TopicPartition,
       case FetchTxnCommitted => Some(initialLastStableOffset)
     }
 
-    val fetchedData = localReplica.log match {
-      case Some(log) =>
-        log.read(fetchOffset, maxBytes, maxOffsetOpt, minOneMessage,
+    val fetchedData = localLog.read(fetchOffset, maxBytes, maxOffsetOpt, minOneMessage,
           includeAbortedTxns = fetchIsolation == FetchTxnCommitted)
 
-      case None =>
-        error(s"Leader does not have a local log")
-        FetchDataInfo(LogOffsetMetadata.UnknownOffsetMetadata, MemoryRecords.EMPTY)
-    }
-
     LogReadInfo(
       fetchedData = fetchedData,
       highWatermark = initialHighWatermark,
@@ -964,12 +997,12 @@ class Partition(val topicPartition: TopicPartition,
                               currentLeaderEpoch: Optional[Integer],
                               fetchOnlyFromLeader: Boolean): Option[TimestampAndOffset] = inReadLock(leaderIsrUpdateLock) {
     // decide whether to only fetch from leader
-    val localReplica = localReplicaWithEpochOrException(currentLeaderEpoch, fetchOnlyFromLeader)
+    val localLog = localLogWithEpochOrException(currentLeaderEpoch, fetchOnlyFromLeader)
 
     val lastFetchableOffset = isolationLevel match {
-      case Some(IsolationLevel.READ_COMMITTED) => localReplica.lastStableOffset
-      case Some(IsolationLevel.READ_UNCOMMITTED) => localReplica.highWatermark
-      case None => localReplica.logEndOffset
+      case Some(IsolationLevel.READ_COMMITTED) => localLog.lastStableOffset
+      case Some(IsolationLevel.READ_UNCOMMITTED) => localLog.highWatermark
+      case None => localLog.logEndOffset
     }
 
     val epochLogString = if(currentLeaderEpoch.isPresent) {
@@ -981,10 +1014,10 @@ class Partition(val topicPartition: TopicPartition,
     // Only consider throwing an error if we get a client request (isolationLevel is defined) and the start offset
     // is lagging behind the high watermark
     val maybeOffsetsError: Option[ApiException] = leaderEpochStartOffsetOpt
-      .filter(epochStart => isolationLevel.isDefined && epochStart > localReplica.highWatermark)
+      .filter(epochStart => isolationLevel.isDefined && epochStart > localLog.highWatermark)
       .map(epochStart => Errors.OFFSET_NOT_AVAILABLE.exception(s"Failed to fetch offsets for " +
         s"partition $topicPartition with leader $epochLogString as this partition's " +
-        s"high watermark (${localReplica.highWatermark}) is lagging behind the " +
+        s"high watermark (${localLog.highWatermark}) is lagging behind the " +
         s"start offset from the beginning of this epoch ($epochStart)."))
 
     def getOffsetByTimestamp: Option[TimestampAndOffset] = {
@@ -1008,14 +1041,14 @@ class Partition(val topicPartition: TopicPartition,
   def fetchOffsetSnapshot(currentLeaderEpoch: Optional[Integer],
                           fetchOnlyFromLeader: Boolean): LogOffsetSnapshot = inReadLock(leaderIsrUpdateLock) {
     // decide whether to only fetch from leader
-    val localReplica = localReplicaWithEpochOrException(currentLeaderEpoch, fetchOnlyFromLeader)
-    localReplica.offsetSnapshot
+    val localLog = localLogWithEpochOrException(currentLeaderEpoch, fetchOnlyFromLeader)
+    localLog.offsetSnapshot
   }
 
   def fetchOffsetSnapshotOrError(currentLeaderEpoch: Optional[Integer],
                                  fetchOnlyFromLeader: Boolean): Either[LogOffsetSnapshot, Errors] = {
     inReadLock(leaderIsrUpdateLock) {
-      getLocalReplica(localBrokerId, currentLeaderEpoch, fetchOnlyFromLeader)
+      getLocalLog(currentLeaderEpoch, fetchOnlyFromLeader)
         .left.map(_.offsetSnapshot)
     }
   }
@@ -1024,21 +1057,13 @@ class Partition(val topicPartition: TopicPartition,
                                      maxNumOffsets: Int,
                                      isFromConsumer: Boolean,
                                      fetchOnlyFromLeader: Boolean): Seq[Long] = inReadLock(leaderIsrUpdateLock) {
-    val localReplica = localReplicaWithEpochOrException(Optional.empty(), fetchOnlyFromLeader)
-    val allOffsets = logManager.getLog(topicPartition) match {
-      case Some(log) =>
-        log.legacyFetchOffsetsBefore(timestamp, maxNumOffsets)
-      case None =>
-        if (timestamp == ListOffsetRequest.LATEST_TIMESTAMP || timestamp == ListOffsetRequest.EARLIEST_TIMESTAMP)
-          Seq(0L)
-        else
-          Nil
-    }
+    val localLog = localLogWithEpochOrException(Optional.empty(), fetchOnlyFromLeader)
+    val allOffsets = localLog.legacyFetchOffsetsBefore(timestamp, maxNumOffsets)
 
     if (!isFromConsumer) {
       allOffsets
     } else {
-      val hw = localReplica.highWatermark
+      val hw = localLog.highWatermark
       if (allOffsets.exists(_ > hw))
         hw +: allOffsets.dropWhile(_ > hw)
       else
@@ -1048,7 +1073,7 @@ class Partition(val topicPartition: TopicPartition,
 
   def logStartOffset: Long = {
     inReadLock(leaderIsrUpdateLock) {
-      leaderReplicaIfLocal.map(_.log.get.logStartOffset).getOrElse(-1)
+      leaderLogIfLocal.map(_.logStartOffset).getOrElse(-1)
     }
   }
 
@@ -1059,20 +1084,20 @@ class Partition(val topicPartition: TopicPartition,
    * Return low watermark of the partition.
    */
   def deleteRecordsOnLeader(offset: Long): LogDeleteRecordsResult = inReadLock(leaderIsrUpdateLock) {
-    leaderReplicaIfLocal match {
-      case Some(leaderReplica) =>
-        if (!leaderReplica.log.get.config.delete)
+    leaderLogIfLocal match {
+      case Some(leaderLog) =>
+        if (!leaderLog.config.delete)
           throw new PolicyViolationException(s"Records of partition $topicPartition can not be deleted due to the configured policy")
 
         val convertedOffset = if (offset == DeleteRecordsRequest.HIGH_WATERMARK)
-          leaderReplica.highWatermark
+          leaderLog.highWatermark
         else
           offset
 
         if (convertedOffset < 0)
           throw new OffsetOutOfRangeException(s"The offset $convertedOffset for partition $topicPartition is not valid")
 
-        leaderReplica.maybeIncrementLogStartOffset(convertedOffset)
+        leaderLog.maybeIncrementLogStartOffset(convertedOffset)
         LogDeleteRecordsResult(
           requestedOffset = convertedOffset,
           lowWatermark = lowWatermarkIfLeader)
@@ -1126,10 +1151,10 @@ class Partition(val topicPartition: TopicPartition,
                                leaderEpoch: Int,
                                fetchOnlyFromLeader: Boolean): EpochEndOffset = {
     inReadLock(leaderIsrUpdateLock) {
-      val localReplicaOrError = getLocalReplica(localBrokerId, currentLeaderEpoch, fetchOnlyFromLeader)
-      localReplicaOrError match {
-        case Left(replica) =>
-          replica.endOffsetForEpoch(leaderEpoch) match {
+      val localLogOrError = getLocalLog(currentLeaderEpoch, fetchOnlyFromLeader)
+      localLogOrError match {
+        case Left(localLog) =>
+          localLog.endOffsetForEpoch(leaderEpoch) match {
             case Some(epochAndOffset) => new EpochEndOffset(NONE, epochAndOffset.leaderEpoch, epochAndOffset.offset)
             case None => new EpochEndOffset(NONE, UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET)
           }
@@ -1139,19 +1164,19 @@ class Partition(val topicPartition: TopicPartition,
     }
   }
 
-  private def expandIsr(newIsr: Set[Replica]): Unit = {
-    val newLeaderAndIsr = new LeaderAndIsr(localBrokerId, leaderEpoch, newIsr.map(_.brokerId).toList, zkVersion)
+  private def expandIsr(newIsr: Set[Int]): Unit = {
+    val newLeaderAndIsr = new LeaderAndIsr(localBrokerId, leaderEpoch, newIsr.toList, zkVersion)
     val zkVersionOpt = stateStore.expandIsr(controllerEpoch, newLeaderAndIsr)
     maybeUpdateIsrAndVersion(newIsr, zkVersionOpt)
   }
 
-  private def shrinkIsr(newIsr: Set[Replica]): Unit = {
-    val newLeaderAndIsr = new LeaderAndIsr(localBrokerId, leaderEpoch, newIsr.map(_.brokerId).toList, zkVersion)
+  private def shrinkIsr(newIsr: Set[Int]): Unit = {
+    val newLeaderAndIsr = new LeaderAndIsr(localBrokerId, leaderEpoch, newIsr.toList, zkVersion)
     val zkVersionOpt = stateStore.shrinkIsr(controllerEpoch, newLeaderAndIsr)
     maybeUpdateIsrAndVersion(newIsr, zkVersionOpt)
   }
 
-  private def maybeUpdateIsrAndVersion(isr: Set[Replica], zkVersionOpt: Option[Int]): Unit = {
+  private def maybeUpdateIsrAndVersion(isr: Set[Int], zkVersionOpt: Option[Int]): Unit = {
     zkVersionOpt match {
       case Some(newVersion) =>
         inSyncReplicas = isr
@@ -1171,13 +1196,13 @@ class Partition(val topicPartition: TopicPartition,
   override def hashCode: Int =
     31 + topic.hashCode + 17 * partitionId
 
-  override def toString(): String = {
+  override def toString: String = {
     val partitionString = new StringBuilder
     partitionString.append("Topic: " + topic)
     partitionString.append("; Partition: " + partitionId)
     partitionString.append("; Leader: " + leaderReplicaIdOpt)
-    partitionString.append("; AllReplicas: " + allReplicasMap.keys.mkString(","))
-    partitionString.append("; InSyncReplicas: " + inSyncReplicas.map(_.brokerId).mkString(","))
+    partitionString.append("; AllReplicas: " + remoteReplicasMap.keys.mkString(","))
+    partitionString.append("; InSyncReplicas: " + inSyncReplicas.mkString(","))
     partitionString.toString
   }
 }
diff --git a/core/src/main/scala/kafka/cluster/Replica.scala b/core/src/main/scala/kafka/cluster/Replica.scala
index 831233ea08132..1c61fad443676 100644
--- a/core/src/main/scala/kafka/cluster/Replica.scala
+++ b/core/src/main/scala/kafka/cluster/Replica.scala
@@ -17,20 +17,12 @@
 
 package kafka.cluster
 
-import kafka.log.{Log, LogOffsetSnapshot}
+import kafka.log.{Log}
 import kafka.utils.Logging
-import kafka.server.{LogOffsetMetadata, OffsetAndEpoch}
-import org.apache.kafka.common.{KafkaException, TopicPartition}
-import org.apache.kafka.common.errors.OffsetOutOfRangeException
-import org.apache.kafka.common.utils.Time
-
-class Replica(val brokerId: Int,
-              val topicPartition: TopicPartition,
-              time: Time = Time.SYSTEM,
-              initialHighWatermarkValue: Long = 0L,
-              @volatile var log: Option[Log] = None) extends Logging {
-  // the high watermark offset value, in non-leader replicas only its message offsets are kept
-  @volatile private[this] var _highWatermarkMetadata = new LogOffsetMetadata(initialHighWatermarkValue)
+import kafka.server.{LogOffsetMetadata}
+import org.apache.kafka.common.{TopicPartition}
+
+class Replica(val brokerId: Int, val topicPartition: TopicPartition) extends Logging {
   // the log end offset value, kept in all replicas;
   // for local replica it is the log's end offset, for remote replicas its value is only updated by follower fetch
   @volatile private[this] var _logEndOffsetMetadata = LogOffsetMetadata.UnknownOffsetMetadata
@@ -50,12 +42,13 @@ class Replica(val brokerId: Int,
   // the LEO of leader at time t. This is used to determine the lag of this follower and ISR of this partition.
   @volatile private[this] var _lastCaughtUpTimeMs = 0L
 
-  def isLocal: Boolean = log.isDefined
+  def logStartOffset: Long = _logStartOffset
 
-  def lastCaughtUpTimeMs: Long = _lastCaughtUpTimeMs
+  def logEndOffsetMetadata: LogOffsetMetadata = _logEndOffsetMetadata
+
+  def logEndOffset: Long = logEndOffsetMetadata.messageOffset
 
-  info(s"Replica loaded for partition $topicPartition with initial high watermark $initialHighWatermarkValue")
-  log.foreach(_.onHighWatermarkIncremented(initialHighWatermarkValue))
+  def lastCaughtUpTimeMs: Long = _lastCaughtUpTimeMs
 
   /*
    * If the FetchRequest reads up to the log end offset of the leader when the current fetch request is received,
@@ -78,168 +71,39 @@ class Replica(val brokerId: Int,
     else if (followerFetchOffsetMetadata.messageOffset >= lastFetchLeaderLogEndOffset)
       _lastCaughtUpTimeMs = math.max(_lastCaughtUpTimeMs, lastFetchTimeMs)
 
-    logStartOffset = followerStartOffset
-    logEndOffsetMetadata = followerFetchOffsetMetadata
+    _logStartOffset = followerStartOffset
+    _logEndOffsetMetadata = followerFetchOffsetMetadata
     lastFetchLeaderLogEndOffset = leaderEndOffset
     lastFetchTimeMs = followerFetchTimeMs
+    trace(s"Updated state of replica to $this")
   }
 
-  def resetLastCaughtUpTime(curLeaderLogEndOffset: Long, curTimeMs: Long, lastCaughtUpTimeMs: Long) {
+  def resetLastCaughtUpTime(curLeaderLogEndOffset: Long, curTimeMs: Long, lastCaughtUpTimeMs: Long): Unit = {
     lastFetchLeaderLogEndOffset = curLeaderLogEndOffset
     lastFetchTimeMs = curTimeMs
     _lastCaughtUpTimeMs = lastCaughtUpTimeMs
+    trace(s"Reset state of replica to $this")
   }
 
-  private def logEndOffsetMetadata_=(newLogEndOffset: LogOffsetMetadata) {
-    if (isLocal) {
-      throw new KafkaException(s"Should not set log end offset on partition $topicPartition's local replica $brokerId")
-    } else {
-      _logEndOffsetMetadata = newLogEndOffset
-      trace(s"Setting log end offset for replica $brokerId for partition $topicPartition to [${_logEndOffsetMetadata}]")
-    }
-  }
-
-  def latestEpoch: Option[Int] = {
-    if (isLocal) {
-      log.get.latestEpoch
-    } else {
-      throw new KafkaException(s"Cannot get latest epoch of non-local replica of $topicPartition")
-    }
-  }
-
-  def endOffsetForEpoch(leaderEpoch: Int): Option[OffsetAndEpoch] = {
-    if (isLocal) {
-      log.get.endOffsetForEpoch(leaderEpoch)
-    } else {
-      throw new KafkaException(s"Cannot lookup end offset for epoch of non-local replica of $topicPartition")
-    }
-  }
-
-  def logEndOffsetMetadata: LogOffsetMetadata =
-    if (isLocal)
-      log.get.logEndOffsetMetadata
-    else
-      _logEndOffsetMetadata
-
-  def logEndOffset: Long =
-    logEndOffsetMetadata.messageOffset
-
-  /**
-   * Increment the log start offset if the new offset is greater than the previous log start offset. The replica
-   * must be local and the new log start offset must be lower than the current high watermark.
-   */
-  def maybeIncrementLogStartOffset(newLogStartOffset: Long) {
-    if (isLocal) {
-      if (newLogStartOffset > highWatermark)
-        throw new OffsetOutOfRangeException(s"Cannot increment the log start offset to $newLogStartOffset of partition $topicPartition " +
-          s"since it is larger than the high watermark $highWatermark")
-      log.get.maybeIncrementLogStartOffset(newLogStartOffset)
-    } else {
-      throw new KafkaException(s"Should not try to delete records on partition $topicPartition's non-local replica $brokerId")
-    }
-  }
-
-  private def logStartOffset_=(newLogStartOffset: Long) {
-    if (isLocal) {
-      throw new KafkaException(s"Should not set log start offset on partition $topicPartition's local replica $brokerId " +
-                               s"without attempting to delete records of the log")
-    } else {
-      _logStartOffset = newLogStartOffset
-      trace(s"Setting log start offset for remote replica $brokerId for partition $topicPartition to [$newLogStartOffset]")
-    }
-  }
-
-  def logStartOffset: Long =
-    if (isLocal)
-      log.get.logStartOffset
-    else
-      _logStartOffset
-
-  def highWatermarkMetadata_=(newHighWatermarkMetadata: LogOffsetMetadata) {
-    if (isLocal) {
-      if (newHighWatermarkMetadata.messageOffset < 0)
-        throw new IllegalArgumentException("High watermark offset should be non-negative")
-
-      _highWatermarkMetadata = newHighWatermarkMetadata
-      log.foreach(_.onHighWatermarkIncremented(newHighWatermarkMetadata.messageOffset))
-      trace(s"Setting high watermark for replica $brokerId partition $topicPartition to [$newHighWatermarkMetadata]")
-    } else {
-      throw new KafkaException(s"Should not set high watermark on partition $topicPartition's non-local replica $brokerId")
-    }
-  }
-
-  def highWatermark_=(newHighWatermark: Long): Unit = {
-    highWatermarkMetadata = LogOffsetMetadata(newHighWatermark)
-  }
-
-  def highWatermarkMetadata: LogOffsetMetadata = _highWatermarkMetadata
-
-  def highWatermark: Long = _highWatermarkMetadata.messageOffset
-
-  /**
-   * The last stable offset (LSO) is defined as the first offset such that all lower offsets have been "decided."
-   * Non-transactional messages are considered decided immediately, but transactional messages are only decided when
-   * the corresponding COMMIT or ABORT marker is written. This implies that the last stable offset will be equal
-   * to the high watermark if there are no transactional messages in the log. Note also that the LSO cannot advance
-   * beyond the high watermark.
-   */
-  def lastStableOffsetMetadata: LogOffsetMetadata = {
-    log.map { log =>
-      log.firstUnstableOffset match {
-        case Some(offsetMetadata) if offsetMetadata.messageOffset < highWatermark => offsetMetadata
-        case _ => highWatermarkMetadata
-      }
-    }.getOrElse(throw new KafkaException(s"Cannot fetch last stable offset on partition $topicPartition's " +
-      s"non-local replica $brokerId"))
-  }
-
-  def lastStableOffset: Long = lastStableOffsetMetadata.messageOffset
-
-  /*
-   * Convert hw to local offset metadata by reading the log at the hw offset.
-   * If the hw offset is out of range, return the first offset of the first log segment as the offset metadata.
-   */
-  def maybeFetchHighWatermarkOffsetMetadata(): Unit = {
-    if (!isLocal)
-      throw new KafkaException(s"Should not construct complete high watermark on partition $topicPartition's non-local replica $brokerId")
-
-    if (highWatermarkMetadata.messageOffsetOnly) {
-      highWatermarkMetadata = log.get.convertToOffsetMetadata(highWatermark).getOrElse {
-        log.get.convertToOffsetMetadata(logStartOffset).getOrElse {
-          val firstSegmentOffset = log.get.logSegments.head.baseOffset
-          new LogOffsetMetadata(firstSegmentOffset, firstSegmentOffset, 0)
-        }
-      }
-    }
-  }
-
-  def offsetSnapshot: LogOffsetSnapshot = {
-    LogOffsetSnapshot(
-      logStartOffset = logStartOffset,
-      logEndOffset = logEndOffsetMetadata,
-      highWatermark =  highWatermarkMetadata,
-      lastStableOffset = lastStableOffsetMetadata)
-  }
-
-  override def equals(that: Any): Boolean = that match {
-    case other: Replica => brokerId == other.brokerId && topicPartition == other.topicPartition
-    case _ => false
-  }
-
-  override def hashCode: Int = 31 + topicPartition.hashCode + 17 * brokerId
-
   override def toString: String = {
     val replicaString = new StringBuilder
     replicaString.append("Replica(replicaId=" + brokerId)
     replicaString.append(s", topic=${topicPartition.topic}")
     replicaString.append(s", partition=${topicPartition.partition}")
-    replicaString.append(s", isLocal=$isLocal")
     replicaString.append(s", lastCaughtUpTimeMs=$lastCaughtUpTimeMs")
-    if (isLocal) {
-      replicaString.append(s", highWatermark=$highWatermarkMetadata")
-      replicaString.append(s", lastStableOffset=$lastStableOffsetMetadata")
-    }
+    replicaString.append(s", logStartOffset=$logStartOffset")
+    replicaString.append(s", logEndOffset=$logEndOffset")
+    replicaString.append(s", logEndOffsetMetadata=$logEndOffsetMetadata")
+    replicaString.append(s", lastFetchLeaderLogEndOffset=$lastFetchLeaderLogEndOffset")
+    replicaString.append(s", lastFetchTimeMs=$lastFetchTimeMs")
     replicaString.append(")")
     replicaString.toString
   }
+
+  override def equals(that: Any): Boolean = that match {
+    case other: Replica => brokerId == other.brokerId && topicPartition == other.topicPartition
+    case _ => false
+  }
+
+  override def hashCode: Int = 31 + topicPartition.hashCode + 17 * brokerId
 }
diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index 38c36179d95a0..52aface6be7c1 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -266,7 +266,7 @@ class Log(@volatile var dir: File,
    * equals the log end offset (which may never happen for a partition under consistent load). This is needed to
    * prevent the log start offset (which is exposed in fetch responses) from getting ahead of the high watermark.
    */
-  @volatile private var replicaHighWatermark: Option[Long] = None
+  @volatile private[this] var _highWatermarkMetadata: LogOffsetMetadata = LogOffsetMetadata(0)
 
   /* the actual segments of the log */
   private val segments: ConcurrentNavigableMap[java.lang.Long, LogSegment] = new ConcurrentSkipListMap[java.lang.Long, LogSegment]
@@ -285,7 +285,7 @@ class Log(@volatile var dir: File,
     val nextOffset = loadSegments()
 
     /* Calculate the offset of the next message */
-    nextOffsetMetadata = new LogOffsetMetadata(nextOffset, activeSegment.baseOffset, activeSegment.size)
+    nextOffsetMetadata = LogOffsetMetadata(nextOffset, activeSegment.baseOffset, activeSegment.size)
 
     leaderEpochCache.foreach(_.truncateFromEnd(nextOffsetMetadata.messageOffset))
 
@@ -304,6 +304,67 @@ class Log(@volatile var dir: File,
       s"log end offset $logEndOffset in ${time.milliseconds() - startMs} ms")
   }
 
+  def highWatermark: Long = highWatermarkMetadata.messageOffset
+
+  def highWatermark_=(newHighWatermark: Long): Unit = {
+    highWatermarkMetadata = LogOffsetMetadata(newHighWatermark)
+  }
+
+  def highWatermarkMetadata: LogOffsetMetadata = _highWatermarkMetadata
+
+  def highWatermarkMetadata_=(newHighWatermark: LogOffsetMetadata) {
+    if (newHighWatermark.messageOffset < 0)
+      throw new IllegalArgumentException("High watermark offset should be non-negative")
+
+    lock synchronized {
+      _highWatermarkMetadata = newHighWatermark
+      producerStateManager.onHighWatermarkUpdated(newHighWatermark.messageOffset)
+      updateFirstUnstableOffset()
+    }
+    trace(s"Setting high watermark [$newHighWatermark]")
+  }
+
+  /*
+   * Convert hw to local offset metadata by reading the log at the hw offset.
+   * If the hw offset is out of range, return the first offset of the first log segment as the offset metadata.
+   */
+  def maybeFetchHighWatermarkOffsetMetadata(): Unit = {
+    if (highWatermarkMetadata.messageOffsetOnly) {
+      highWatermarkMetadata = convertToOffsetMetadata(highWatermark).getOrElse {
+        convertToOffsetMetadata(logStartOffset).getOrElse {
+          val firstSegmentOffset = logSegments.head.baseOffset
+          LogOffsetMetadata(firstSegmentOffset, firstSegmentOffset, 0)
+        }
+      }
+    }
+  }
+
+  /**
+    * The last stable offset (LSO) is defined as the first offset such that all lower offsets have been "decided."
+    * Non-transactional messages are considered decided immediately, but transactional messages are only decided when
+    * the corresponding COMMIT or ABORT marker is written. This implies that the last stable offset will be equal
+    * to the high watermark if there are no transactional messages in the log. Note also that the LSO cannot advance
+    * beyond the high watermark.
+    */
+  def lastStableOffsetMetadata: LogOffsetMetadata = {
+    firstUnstableOffset match {
+      case Some(offsetMetadata) if offsetMetadata.messageOffset < highWatermark => offsetMetadata
+      case _ => highWatermarkMetadata
+    }
+  }
+
+  def lastStableOffset: Long = lastStableOffsetMetadata.messageOffset
+
+  def lastStableOffsetLag: Long = highWatermark - lastStableOffset
+
+  def offsetSnapshot: LogOffsetSnapshot = {
+    LogOffsetSnapshot(
+      logStartOffset = logStartOffset,
+      logEndOffset = logEndOffsetMetadata,
+      highWatermark =  highWatermarkMetadata,
+      lastStableOffset = lastStableOffsetMetadata)
+  }
+
   private val tags = {
     val maybeFutureTag = if (isFuture) Map("is-future" -> "true") else Map.empty[String, String]
     Map("topic" -> topicPartition.topic, "partition" -> topicPartition.partition.toString) ++ maybeFutureTag
@@ -577,7 +638,7 @@ class Log(@volatile var dir: File,
   }
 
   private def updateLogEndOffset(messageOffset: Long) {
-    nextOffsetMetadata = new LogOffsetMetadata(messageOffset, activeSegment.baseOffset, activeSegment.size)
+    nextOffsetMetadata = LogOffsetMetadata(messageOffset, activeSegment.baseOffset, activeSegment.size)
   }
 
   /**
@@ -1026,14 +1087,6 @@ class Log(@volatile var dir: File,
     }
   }
 
-  def onHighWatermarkIncremented(highWatermark: Long): Unit = {
-    lock synchronized {
-      replicaHighWatermark = Some(highWatermark)
-      producerStateManager.onHighWatermarkUpdated(highWatermark)
-      updateFirstUnstableOffset()
-    }
-  }
-
   private def updateFirstUnstableOffset(): Unit = lock synchronized {
     checkIfMemoryMappedBufferClosed()
     val updatedFirstStableOffset = producerStateManager.firstUnstableOffset match {
@@ -1055,6 +1108,10 @@ class Log(@volatile var dir: File,
    * Increment the log start offset if the provided offset is larger.
    */
   def maybeIncrementLogStartOffset(newLogStartOffset: Long) {
+    if (newLogStartOffset > highWatermark)
+      throw new OffsetOutOfRangeException(s"Cannot increment the log start offset to $newLogStartOffset of partition $topicPartition " +
+        s"since it is larger than the high watermark ${highWatermark}")
+
     // We don't have to write the log start offset to log-start-offset-checkpoint immediately.
     // The deleteRecordsOffset may be lost only if all in-sync replicas of this broker are shutdown
     // in an unclean manner within log.flush.start.offset.checkpoint.interval.ms. The chance of this happening is low.
@@ -1502,10 +1559,9 @@ class Log(@volatile var dir: File,
    * @return the segments ready to be deleted
    */
   private def deletableSegments(predicate: (LogSegment, Option[LogSegment]) => Boolean): Iterable[LogSegment] = {
-    if (segments.isEmpty || replicaHighWatermark.isEmpty) {
+    if (segments.isEmpty) {
       Seq.empty
     } else {
-      val highWatermark = replicaHighWatermark.get
       val deletable = ArrayBuffer.empty[LogSegment]
       var segmentEntry = segments.firstEntry
       while (segmentEntry != null) {
@@ -1914,7 +1970,18 @@ class Log(@volatile var dir: File,
     }
   }
 
-  override def toString = "Log(" + dir + ")"
+  override def toString: String = {
+    val logString = new StringBuilder
+    logString.append(s"Log(dir=$dir")
+    logString.append(s", topic=${topicPartition.topic}")
+    logString.append(s", partition=${topicPartition.partition}")
+    logString.append(s", highWatermark=$highWatermarkMetadata")
+    logString.append(s", lastStableOffset=$lastStableOffsetMetadata")
+    logString.append(s", logStartOffset=$logStartOffset")
+    logString.append(s", logEndOffset=$logEndOffset")
+    logString.append(")")
+    logString.toString
+  }
 
   /**
    * This method performs an asynchronous log segment delete by doing the following:
diff --git a/core/src/main/scala/kafka/log/LogManager.scala b/core/src/main/scala/kafka/log/LogManager.scala
index 91cf79e07b2a6..6455746b589ae 100755
--- a/core/src/main/scala/kafka/log/LogManager.scala
+++ b/core/src/main/scala/kafka/log/LogManager.scala
@@ -803,6 +803,7 @@ class LogManager(logDirs: Seq[File],
         throw new KafkaStorageException(s"The future replica for $topicPartition is offline")
 
       destLog.renameDir(Log.logDirName(topicPartition))
+      destLog.highWatermarkMetadata = sourceLog.highWatermarkMetadata
       // Now that future replica has been successfully renamed to be the current replica
       // Update the cached map and log cleaner as appropriate.
       futureLogs.remove(topicPartition)
diff --git a/core/src/main/scala/kafka/log/LogSegment.scala b/core/src/main/scala/kafka/log/LogSegment.scala
index ecd85f99734fe..08182e8e0fbc4 100755
--- a/core/src/main/scala/kafka/log/LogSegment.scala
+++ b/core/src/main/scala/kafka/log/LogSegment.scala
@@ -301,7 +301,7 @@ class LogSegment private[log] (val log: FileRecords,
       return null
 
     val startPosition = startOffsetAndSize.position
-    val offsetMetadata = new LogOffsetMetadata(startOffset, this.baseOffset, startPosition)
+    val offsetMetadata = LogOffsetMetadata(startOffset, this.baseOffset, startPosition)
 
     val adjustedMaxSize =
       if (minOneMessage) math.max(maxSize, startOffsetAndSize.size)
diff --git a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
index 203cc626a62f7..0aec85ad6e98a 100755
--- a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
+++ b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
@@ -631,7 +631,7 @@ abstract class AbstractFetcherThread(name: String,
     } finally partitionMapLock.unlock()
   }
 
-  def partitionCount() = {
+  def partitionCount(): Int = {
     partitionMapLock.lockInterruptibly()
     try partitionStates.size
     finally partitionMapLock.unlock()
diff --git a/core/src/main/scala/kafka/server/DelayedDeleteRecords.scala b/core/src/main/scala/kafka/server/DelayedDeleteRecords.scala
index 236c8d103fd17..852f72d8c9ef8 100644
--- a/core/src/main/scala/kafka/server/DelayedDeleteRecords.scala
+++ b/core/src/main/scala/kafka/server/DelayedDeleteRecords.scala
@@ -74,7 +74,7 @@ class DelayedDeleteRecords(delayMs: Long,
       if (status.acksPending) {
         val (lowWatermarkReached, error, lw) = replicaManager.getPartition(topicPartition) match {
           case HostedPartition.Online(partition) =>
-            partition.leaderReplicaIfLocal match {
+            partition.leaderLogIfLocal match {
               case Some(_) =>
                 val leaderLW = partition.lowWatermarkIfLeader
                 (leaderLW >= status.requiredOffset, Errors.NONE, leaderLW)
diff --git a/core/src/main/scala/kafka/server/LogOffsetMetadata.scala b/core/src/main/scala/kafka/server/LogOffsetMetadata.scala
index 67afac69e378d..6423cfcf9eabc 100644
--- a/core/src/main/scala/kafka/server/LogOffsetMetadata.scala
+++ b/core/src/main/scala/kafka/server/LogOffsetMetadata.scala
@@ -21,7 +21,7 @@ import kafka.log.Log
 import org.apache.kafka.common.KafkaException
 
 object LogOffsetMetadata {
-  val UnknownOffsetMetadata = new LogOffsetMetadata(-1, 0, 0)
+  val UnknownOffsetMetadata = LogOffsetMetadata(-1, 0, 0)
   val UnknownFilePosition = -1
 
   class OffsetOrdering extends Ordering[LogOffsetMetadata] {
diff --git a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
index 8b45501bd6f64..18f89f7423965 100644
--- a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
+++ b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
@@ -56,15 +56,15 @@ class ReplicaAlterLogDirsThread(name: String,
   private var inProgressPartition: Option[TopicPartition] = None
 
   override protected def latestEpoch(topicPartition: TopicPartition): Option[Int] = {
-    replicaMgr.futureLocalReplicaOrException(topicPartition).latestEpoch
+    replicaMgr.futureLocalLogOrException(topicPartition).latestEpoch
   }
 
   override protected def logEndOffset(topicPartition: TopicPartition): Long = {
-    replicaMgr.futureLocalReplicaOrException(topicPartition).logEndOffset
+    replicaMgr.futureLocalLogOrException(topicPartition).logEndOffset
   }
 
   override protected def endOffsetForEpoch(topicPartition: TopicPartition, epoch: Int): Option[OffsetAndEpoch] = {
-    replicaMgr.futureLocalReplicaOrException(topicPartition).endOffsetForEpoch(epoch)
+    replicaMgr.futureLocalLogOrException(topicPartition).endOffsetForEpoch(epoch)
   }
 
   def fetchFromLeader(fetchRequest: FetchRequest.Builder): Seq[(TopicPartition, FetchData)] = {
@@ -102,17 +102,16 @@ class ReplicaAlterLogDirsThread(name: String,
                                     fetchOffset: Long,
                                     partitionData: PartitionData[Records]): Option[LogAppendInfo] = {
     val partition = replicaMgr.nonOfflinePartition(topicPartition).get
-    val futureReplica = partition.futureLocalReplicaOrException
+    val futureLog = partition.futureLocalLogOrException
     val records = toMemoryRecords(partitionData.records)
 
-    if (fetchOffset != futureReplica.logEndOffset)
+    if (fetchOffset != futureLog.logEndOffset)
       throw new IllegalStateException("Offset mismatch for the future replica %s: fetched offset = %d, log end offset = %d.".format(
-        topicPartition, fetchOffset, futureReplica.logEndOffset))
+        topicPartition, fetchOffset, futureLog.logEndOffset))
 
     val logAppendInfo = partition.appendRecordsToFollowerOrFutureReplica(records, isFuture = true)
-    val futureReplicaHighWatermark = futureReplica.logEndOffset.min(partitionData.highWatermark)
-    futureReplica.highWatermark = futureReplicaHighWatermark
-    futureReplica.maybeIncrementLogStartOffset(partitionData.logStartOffset)
+    futureLog.highWatermark = futureLog.logEndOffset.min(partitionData.highWatermark)
+    futureLog.maybeIncrementLogStartOffset(partitionData.logStartOffset)
 
     if (partition.maybeReplaceCurrentWithFutureReplica())
       removePartitions(Set(topicPartition))
@@ -228,7 +227,7 @@ class ReplicaAlterLogDirsThread(name: String,
     val partitionsWithError = mutable.Set[TopicPartition]()
 
     try {
-      val logStartOffset = replicaMgr.futureLocalReplicaOrException(tp).logStartOffset
+      val logStartOffset = replicaMgr.futureLocalLogOrException(tp).logStartOffset
       requestMap.put(tp, new FetchRequest.PartitionData(fetchState.fetchOffset, logStartOffset,
         fetchSize, Optional.of(fetchState.currentLeaderEpoch)))
     } catch {
diff --git a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
index 947e16aad58e5..ccf9bc134b311 100644
--- a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
+++ b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
@@ -99,15 +99,15 @@ class ReplicaFetcherThread(name: String,
   private val fetchSessionHandler = new FetchSessionHandler(logContext, sourceBroker.id)
 
   override protected def latestEpoch(topicPartition: TopicPartition): Option[Int] = {
-    replicaMgr.localReplicaOrException(topicPartition).latestEpoch
+    replicaMgr.localLogOrException(topicPartition).latestEpoch
   }
 
   override protected def logEndOffset(topicPartition: TopicPartition): Long = {
-    replicaMgr.localReplicaOrException(topicPartition).logEndOffset
+    replicaMgr.localLogOrException(topicPartition).logEndOffset
   }
 
   override protected def endOffsetForEpoch(topicPartition: TopicPartition, epoch: Int): Option[OffsetAndEpoch] = {
-    replicaMgr.localReplicaOrException(topicPartition).endOffsetForEpoch(epoch)
+    replicaMgr.localLogOrException(topicPartition).endOffsetForEpoch(epoch)
   }
 
   override def initiateShutdown(): Boolean = {
@@ -144,32 +144,32 @@ class ReplicaFetcherThread(name: String,
                                     fetchOffset: Long,
                                     partitionData: FetchData): Option[LogAppendInfo] = {
     val partition = replicaMgr.nonOfflinePartition(topicPartition).get
-    val replica = partition.localReplicaOrException
+    val log = partition.localLogOrException
     val records = toMemoryRecords(partitionData.records)
 
     maybeWarnIfOversizedRecords(records, topicPartition)
 
-    if (fetchOffset != replica.logEndOffset)
+    if (fetchOffset != log.logEndOffset)
       throw new IllegalStateException("Offset mismatch for partition %s: fetched offset = %d, log end offset = %d.".format(
-        topicPartition, fetchOffset, replica.logEndOffset))
+        topicPartition, fetchOffset, log.logEndOffset))
 
     if (isTraceEnabled)
       trace("Follower has replica log end offset %d for partition %s. Received %d messages and leader hw %d"
-        .format(replica.logEndOffset, topicPartition, records.sizeInBytes, partitionData.highWatermark))
+        .format(log.logEndOffset, topicPartition, records.sizeInBytes, partitionData.highWatermark))
 
     // Append the leader's messages to the log
     val logAppendInfo = partition.appendRecordsToFollowerOrFutureReplica(records, isFuture = false)
 
     if (isTraceEnabled)
       trace("Follower has replica log end offset %d after appending %d bytes of messages for partition %s"
-        .format(replica.logEndOffset, records.sizeInBytes, topicPartition))
-    val followerHighWatermark = replica.logEndOffset.min(partitionData.highWatermark)
+        .format(log.logEndOffset, records.sizeInBytes, topicPartition))
+    val followerHighWatermark = log.logEndOffset.min(partitionData.highWatermark)
     val leaderLogStartOffset = partitionData.logStartOffset
     // for the follower replica, we do not need to keep
     // its segment base offset the physical position,
     // these values will be computed upon making the leader
-    replica.highWatermark = followerHighWatermark
-    replica.maybeIncrementLogStartOffset(leaderLogStartOffset)
+    log.highWatermark = followerHighWatermark
+    log.maybeIncrementLogStartOffset(leaderLogStartOffset)
     if (isTraceEnabled)
       trace(s"Follower set replica high watermark for partition $topicPartition to $followerHighWatermark")
 
@@ -245,7 +245,7 @@ class ReplicaFetcherThread(name: String,
       // We will not include a replica in the fetch request if it should be throttled.
       if (fetchState.isReadyForFetch && !shouldFollowerThrottle(quota, topicPartition)) {
         try {
-          val logStartOffset = replicaMgr.localReplicaOrException(topicPartition).logStartOffset
+          val logStartOffset = replicaMgr.localLogOrException(topicPartition).logStartOffset
           builder.add(topicPartition, new FetchRequest.PartitionData(
             fetchState.fetchOffset, logStartOffset, fetchSize, Optional.of(fetchState.currentLeaderEpoch)))
         } catch {
@@ -278,13 +278,13 @@ class ReplicaFetcherThread(name: String,
    */
   override def truncate(tp: TopicPartition, offsetTruncationState: OffsetTruncationState): Unit = {
     val partition = replicaMgr.nonOfflinePartition(tp).get
-    val replica = partition.localReplicaOrException
+    val log = partition.localLogOrException
 
     partition.truncateTo(offsetTruncationState.offset, isFuture = false)
 
-    if (offsetTruncationState.offset < replica.highWatermark)
+    if (offsetTruncationState.offset < log.highWatermark)
       warn(s"Truncating $tp to offset ${offsetTruncationState.offset} below high watermark " +
-        s"${replica.highWatermark}")
+        s"${log.highWatermark}")
 
     // mark the future replica for truncation only when we do last truncation
     if (offsetTruncationState.truncationCompleted)
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index 7ded9854dbbb6..b1079748fb045 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -24,7 +24,7 @@ import java.util.concurrent.locks.Lock
 
 import com.yammer.metrics.core.{Gauge, Meter}
 import kafka.api._
-import kafka.cluster.{BrokerEndPoint, Partition, Replica}
+import kafka.cluster.{BrokerEndPoint, Partition}
 import kafka.controller.{KafkaController, StateChangeLogger}
 import kafka.log._
 import kafka.metrics.KafkaMetricsGroup
@@ -445,24 +445,20 @@ class ReplicaManager(val config: KafkaConfig,
     }
   }
 
-  def localReplicaOrException(topicPartition: TopicPartition): Replica = {
-    getPartitionOrException(topicPartition, expectLeader = false).localReplicaOrException
+  def localLogOrException(topicPartition: TopicPartition): Log = {
+    getPartitionOrException(topicPartition, expectLeader = false).localLogOrException
   }
 
-  def futureLocalReplicaOrException(topicPartition: TopicPartition): Replica = {
-    getPartitionOrException(topicPartition, expectLeader = false).futureLocalReplicaOrException
+  def futureLocalLogOrException(topicPartition: TopicPartition): Log = {
+    getPartitionOrException(topicPartition, expectLeader = false).futureLocalLogOrException
   }
 
-  def futureLocalReplica(topicPartition: TopicPartition): Option[Replica] = {
-    nonOfflinePartition(topicPartition).flatMap(_.futureLocalReplica)
-  }
-
-  def localReplica(topicPartition: TopicPartition): Option[Replica] = {
-    nonOfflinePartition(topicPartition).flatMap(_.localReplica)
+  def localLog(topicPartition: TopicPartition): Option[Log] = {
+    nonOfflinePartition(topicPartition).flatMap(_.log)
   }
 
   def getLogDir(topicPartition: TopicPartition): Option[String] = {
-    localReplica(topicPartition).flatMap(_.log).map(_.dir.getParent)
+    localLog(topicPartition).map(_.dir.getParent)
   }
 
   /**
@@ -599,9 +595,9 @@ class ReplicaManager(val config: KafkaConfig,
           // 2) Respond with ReplicaNotAvailableException for this partition in the AlterReplicaLogDirsResponse
           logManager.maybeUpdatePreferredLogDir(topicPartition, destinationDir)
 
-          // throw ReplicaNotAvailableException if replica does not exit for the given partition
+          // throw ReplicaNotAvailableException if replica does not exist for the given partition
           val partition = getPartitionOrException(topicPartition, expectLeader = false)
-          partition.localReplicaOrException
+          partition.localLogOrException
 
           // If the destinationLDir is different from the current log directory of the replica:
           // - If there is no offline log directory, create the future log in the destinationDir (if it does not exist) and
@@ -610,11 +606,11 @@ class ReplicaManager(val config: KafkaConfig,
           //   so that we can avoid creating future log for the same partition in multiple log directories.
           val highWatermarkCheckpoints = new LazyOffsetCheckpoints(this.highWatermarkCheckpoints)
           if (partition.maybeCreateFutureReplica(destinationDir, highWatermarkCheckpoints)) {
-            val futureReplica = futureLocalReplicaOrException(topicPartition)
+            val futureLog = futureLocalLogOrException(topicPartition)
             logManager.abortAndPauseCleaning(topicPartition)
 
             val initialFetchState = InitialFetchState(BrokerEndPoint(config.brokerId, "localhost", -1),
-              partition.getLeaderEpoch, futureReplica.highWatermark)
+              partition.getLeaderEpoch, futureLog.highWatermark)
             replicaAlterLogDirsManager.addFetcherForPartitions(Map(topicPartition -> initialFetchState))
           }
 
@@ -674,12 +670,12 @@ class ReplicaManager(val config: KafkaConfig,
   }
 
   def getLogEndOffsetLag(topicPartition: TopicPartition, logEndOffset: Long, isFuture: Boolean): Long = {
-    localReplica(topicPartition) match {
-      case Some(replica) =>
+    localLog(topicPartition) match {
+      case Some(log) =>
         if (isFuture)
-          replica.logEndOffset - logEndOffset
+          log.logEndOffset - logEndOffset
         else
-          math.max(replica.highWatermark - logEndOffset, 0)
+          math.max(log.highWatermark - logEndOffset, 0)
       case None =>
         // return -1L to indicate that the LEO lag is not available if the replica is not created or is offline
         DescribeLogDirsResponse.INVALID_OFFSET_LAG
@@ -1009,13 +1005,11 @@ class ReplicaManager(val config: KafkaConfig,
    *  the quota is exceeded and the replica is not in sync.
    */
   def shouldLeaderThrottle(quota: ReplicaQuota, topicPartition: TopicPartition, replicaId: Int): Boolean = {
-    val isReplicaInSync = nonOfflinePartition(topicPartition).exists { partition =>
-      partition.getReplica(replicaId).exists(partition.inSyncReplicas.contains)
-    }
+    val isReplicaInSync = nonOfflinePartition(topicPartition).exists(_.inSyncReplicas.contains(replicaId))
     !isReplicaInSync && quota.isThrottled(topicPartition) && quota.isQuotaExceeded
   }
 
-  def getLogConfig(topicPartition: TopicPartition): Option[LogConfig] = localReplica(topicPartition).flatMap(_.log.map(_.config))
+  def getLogConfig(topicPartition: TopicPartition): Option[LogConfig] = localLog(topicPartition).map(_.config)
 
   def getMagic(topicPartition: TopicPartition): Option[Byte] = getLogConfig(topicPartition).map(_.messageFormatVersion.recordVersion.value)
 
@@ -1132,7 +1126,7 @@ class ReplicaManager(val config: KafkaConfig,
            * In this case ReplicaManager.allPartitions will map this topic-partition to an empty Partition object.
            * we need to map this topic-partition to OfflinePartition instead.
            */
-          if (localReplica(topicPartition).isEmpty)
+          if (localLog(topicPartition).isEmpty)
             markPartitionOffline(topicPartition)
         }
 
@@ -1144,18 +1138,19 @@ class ReplicaManager(val config: KafkaConfig,
         for (partition <- newPartitions) {
           val topicPartition = partition.topicPartition
           if (logManager.getLog(topicPartition, isFuture = true).isDefined) {
-            partition.localReplica.foreach { replica =>
+            partition.log.foreach { log =>
               val leader = BrokerEndPoint(config.brokerId, "localhost", -1)
 
               // Add future replica to partition's map
-              partition.getOrCreateReplica(Request.FutureLocalReplicaId, isNew = false, highWatermarkCheckpoints)
+              partition.createLogIfNotExists(Request.FutureLocalReplicaId, isNew = false, isFutureReplica = true,
+                highWatermarkCheckpoints)
 
               // pause cleaning for partitions that are being moved and start ReplicaAlterDirThread to move
               // replica from source dir to destination dir
               logManager.abortAndPauseCleaning(topicPartition)
 
               futureReplicasAndInitialOffset.put(topicPartition, InitialFetchState(leader,
-                partition.getLeaderEpoch, replica.highWatermark))
+                partition.getLeaderEpoch, log.highWatermark))
             }
           }
         }
@@ -1304,7 +1299,8 @@ class ReplicaManager(val config: KafkaConfig,
                 s"but cannot become follower since the new leader $newLeaderBrokerId is unavailable.")
               // Create the local replica even if the leader is unavailable. This is required to ensure that we include
               // the partition's high watermark in the checkpoint file (see KAFKA-1647)
-              partition.getOrCreateReplica(localBrokerId, isNew = partitionStateInfo.isNew, highWatermarkCheckpoints)
+              partition.createLogIfNotExists(localBrokerId, isNew = partitionStateInfo.isNew, isFutureReplica = false,
+                highWatermarkCheckpoints)
           }
         } catch {
           case e: KafkaStorageException =>
@@ -1350,7 +1346,7 @@ class ReplicaManager(val config: KafkaConfig,
         val partitionsToMakeFollowerWithLeaderAndOffset = partitionsToMakeFollower.map { partition =>
           val leader = metadataCache.getAliveBrokers.find(_.id == partition.leaderReplicaIdOpt.get).get
             .brokerEndPoint(config.interBrokerListenerName)
-          val fetchOffset = partition.localReplicaOrException.highWatermark
+          val fetchOffset = partition.localLogOrException.highWatermark
           partition.topicPartition -> InitialFetchState(leader, partition.getLeaderEpoch, fetchOffset)
        }.toMap
 
@@ -1415,7 +1411,7 @@ class ReplicaManager(val config: KafkaConfig,
             } else {
               warn(s"Leader $localBrokerId failed to record follower $followerId's position " +
                 s"${readResult.info.fetchOffsetMetadata.messageOffset} since the replica is not recognized to be " +
-                s"one of the assigned replicas ${partition.assignedReplicas.map(_.brokerId).mkString(",")} " +
+                s"one of the assigned replicas ${partition.allReplicaIds.mkString(",")} " +
                 s"for partition $topicPartition. Empty records will be returned for this partition.")
               readResult.withEmptyFetchInfo
             }
@@ -1429,22 +1425,22 @@ class ReplicaManager(val config: KafkaConfig,
   }
 
   private def leaderPartitionsIterator: Iterator[Partition] =
-    nonOfflinePartitionsIterator.filter(_.leaderReplicaIfLocal.isDefined)
+    nonOfflinePartitionsIterator.filter(_.leaderLogIfLocal.isDefined)
 
   def getLogEndOffset(topicPartition: TopicPartition): Option[Long] =
-    nonOfflinePartition(topicPartition).flatMap(_.leaderReplicaIfLocal.map(_.logEndOffset))
+    nonOfflinePartition(topicPartition).flatMap(_.leaderLogIfLocal.map(_.logEndOffset))
 
   // Flushes the highwatermark value for all partitions to the highwatermark file
   def checkpointHighWatermarks() {
-    val replicas = nonOfflinePartitionsIterator.flatMap { partition =>
-      val replicasList: mutable.Set[Replica] = mutable.Set()
-      partition.localReplica.foreach(replicasList.add)
-      partition.futureLocalReplica.foreach(replicasList.add)
-      replicasList
-    }.filter(_.log.isDefined).toBuffer
-    val replicasByDir = replicas.groupBy(_.log.get.dir.getParent)
-    for ((dir, reps) <- replicasByDir) {
-      val hwms = reps.map(r => r.topicPartition -> r.highWatermark).toMap
+    val localLogs = nonOfflinePartitionsIterator.flatMap { partition =>
+      val logsList: mutable.Set[Log] = mutable.Set()
+      partition.log.foreach(logsList.add)
+      partition.futureLog.foreach(logsList.add)
+      logsList
+    }.toBuffer
+    val logsByDir = localLogs.groupBy(_.dir.getParent)
+    for ((dir, logs) <- logsByDir) {
+      val hwms = logs.map(log => log.topicPartition -> log.highWatermark).toMap
       try {
         highWatermarkCheckpoints.get(dir).foreach(_.write(hwms))
       } catch {
@@ -1468,15 +1464,11 @@ class ReplicaManager(val config: KafkaConfig,
     info(s"Stopping serving replicas in dir $dir")
     replicaStateChangeLock synchronized {
       val newOfflinePartitions = nonOfflinePartitionsIterator.filter { partition =>
-        partition.localReplica.exists { replica =>
-          replica.log.isDefined && replica.log.get.dir.getParent == dir
-        }
+        partition.log.exists { _.dir.getParent == dir }
       }.map(_.topicPartition).toSet
 
       val partitionsWithOfflineFutureReplica = nonOfflinePartitionsIterator.filter { partition =>
-        partition.futureLocalReplica.exists { replica =>
-          replica.log.isDefined && replica.log.get.dir.getParent == dir
-        }
+        partition.futureLog.exists { _.dir.getParent == dir }
       }.toSet
 
       replicaFetcherManager.removeFetcherForPartitions(newOfflinePartitions)
diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index d622a7963de10..306bcb5fb91d6 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -849,7 +849,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     assertEquals(3L, lowWatermark)
 
     for (i <- 0 until brokerCount)
-      assertEquals(3, servers(i).replicaManager.localReplica(topicPartition).get.logStartOffset)
+      assertEquals(3, servers(i).replicaManager.localLog(topicPartition).get.logStartOffset)
   }
 
   @Test
@@ -858,16 +858,16 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     val followerIndex = if (leaders(0) != servers(0).config.brokerId) 0 else 1
 
     def waitForFollowerLog(expectedStartOffset: Long, expectedEndOffset: Long): Unit = {
-      TestUtils.waitUntilTrue(() => servers(followerIndex).replicaManager.localReplica(topicPartition) != None,
+      TestUtils.waitUntilTrue(() => servers(followerIndex).replicaManager.localLog(topicPartition) != None,
                               "Expected follower to create replica for partition")
 
       // wait until the follower discovers that log start offset moved beyond its HW
       TestUtils.waitUntilTrue(() => {
-        servers(followerIndex).replicaManager.localReplica(topicPartition).get.logStartOffset == expectedStartOffset
+        servers(followerIndex).replicaManager.localLog(topicPartition).get.logStartOffset == expectedStartOffset
       }, s"Expected follower to discover new log start offset $expectedStartOffset")
 
       TestUtils.waitUntilTrue(() => {
-        servers(followerIndex).replicaManager.localReplica(topicPartition).get.logEndOffset == expectedEndOffset
+        servers(followerIndex).replicaManager.localLog(topicPartition).get.logEndOffset == expectedEndOffset
       }, s"Expected follower to catch up to log end offset $expectedEndOffset")
     }
 
@@ -888,7 +888,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
 
     // after the new replica caught up, all replicas should have same log start offset
     for (i <- 0 until brokerCount)
-      assertEquals(3, servers(i).replicaManager.localReplica(topicPartition).get.logStartOffset)
+      assertEquals(3, servers(i).replicaManager.localLog(topicPartition).get.logStartOffset)
 
     // kill the same follower again, produce more records, and delete records beyond follower's LOE
     killBroker(followerIndex)
@@ -912,8 +912,8 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     result.all().get()
     // make sure we are in the expected state after delete records
     for (i <- 0 until brokerCount) {
-      assertEquals(3, servers(i).replicaManager.localReplica(topicPartition).get.logStartOffset)
-      assertEquals(expectedLEO, servers(i).replicaManager.localReplica(topicPartition).get.logEndOffset)
+      assertEquals(3, servers(i).replicaManager.localLog(topicPartition).get.logStartOffset)
+      assertEquals(expectedLEO, servers(i).replicaManager.localLog(topicPartition).get.logEndOffset)
     }
 
     // we will create another dir just for one server
@@ -927,8 +927,8 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     }, "timed out waiting for replica movement")
 
     // once replica moved, its LSO and LEO should match other replicas
-    assertEquals(3, servers(0).replicaManager.localReplica(topicPartition).get.logStartOffset)
-    assertEquals(expectedLEO, servers(0).replicaManager.localReplica(topicPartition).get.logEndOffset)
+    assertEquals(3, servers.head.replicaManager.localLog(topicPartition).get.logStartOffset)
+    assertEquals(expectedLEO, servers.head.replicaManager.localLog(topicPartition).get.logEndOffset)
   }
 
   @Test
diff --git a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
index 16a117be4e811..1bd1c81aa47d1 100644
--- a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
+++ b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
@@ -132,7 +132,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
 
     // wait until all the followers have synced the last HW with leader
     TestUtils.waitUntilTrue(() => servers.forall(server =>
-      server.replicaManager.localReplica(tp).get.highWatermark == numRecords
+      server.replicaManager.localLog(tp).get.highWatermark == numRecords
     ), "Failed to update high watermark for followers after timeout")
 
     val scheduler = new BounceBrokerScheduler(numIters)
diff --git a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
index d604cd0b7950b..2b36da88b8cd9 100644
--- a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
@@ -99,14 +99,14 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     val newLeaderServer = servers.find(_.config.brokerId == 101).get
 
     TestUtils.waitUntilTrue (
-      () => newLeaderServer.replicaManager.nonOfflinePartition(topicPartition).flatMap(_.leaderReplicaIfLocal).isDefined,
+      () => newLeaderServer.replicaManager.nonOfflinePartition(topicPartition).flatMap(_.leaderLogIfLocal).isDefined,
       "broker 101 should be the new leader", pause = 1L
     )
 
-    assertEquals(100, newLeaderServer.replicaManager.localReplicaOrException(topicPartition)
+    assertEquals(100, newLeaderServer.replicaManager.localLogOrException(topicPartition)
       .highWatermark)
     val newFollowerServer = servers.find(_.config.brokerId == 102).get
-    TestUtils.waitUntilTrue(() => newFollowerServer.replicaManager.localReplicaOrException(topicPartition)
+    TestUtils.waitUntilTrue(() => newFollowerServer.replicaManager.localLogOrException(topicPartition)
       .highWatermark == 100,
       "partition follower's highWatermark should be 100")
   }
diff --git a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
index 650dce441f2d2..bb1270d24af60 100644
--- a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
+++ b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
@@ -41,9 +41,9 @@ import org.junit.{After, Before, Test}
 import org.junit.Assert._
 import org.mockito.Mockito.{doNothing, mock, when}
 import org.scalatest.Assertions.assertThrows
+import org.mockito.ArgumentMatchers
 import org.mockito.invocation.InvocationOnMock
 import org.mockito.stubbing.Answer
-import org.mockito.ArgumentMatchers
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable.ListBuffer
@@ -125,7 +125,7 @@ class PartitionTest {
     assertEquals(4, log.logEndOffset)
 
     val partition = setupPartitionWithMocks(leaderEpoch = leaderEpoch, isLeader = true, log = log)
-    assertEquals(Some(4), partition.leaderReplicaIfLocal.map(_.logEndOffset))
+    assertEquals(Some(4), partition.leaderLogIfLocal.map(_.logEndOffset))
 
     val epochEndOffset = partition.lastOffsetForLeaderEpoch(currentLeaderEpoch = Optional.of[Integer](leaderEpoch),
       leaderEpoch = leaderEpoch, fetchOnlyFromLeader = true)
@@ -153,7 +153,7 @@ class PartitionTest {
     assertEquals(4, log.logEndOffset)
 
     val partition = setupPartitionWithMocks(leaderEpoch = leaderEpoch, isLeader = true, log = log)
-    assertEquals(Some(4), partition.leaderReplicaIfLocal.map(_.logEndOffset))
+    assertEquals(Some(4), partition.leaderLogIfLocal.map(_.logEndOffset))
     assertEquals(None, log.latestEpoch)
 
     val epochEndOffset = partition.lastOffsetForLeaderEpoch(currentLeaderEpoch = Optional.of[Integer](leaderEpoch),
@@ -168,7 +168,7 @@ class PartitionTest {
     val latch = new CountDownLatch(1)
 
     logManager.maybeUpdatePreferredLogDir(topicPartition, logDir1.getAbsolutePath)
-    partition.getOrCreateReplica(brokerId, isNew = true, offsetCheckpoints)
+    partition.createLogIfNotExists(brokerId, isNew = true, isFutureReplica = false, offsetCheckpoints)
     logManager.maybeUpdatePreferredLogDir(topicPartition, logDir2.getAbsolutePath)
     partition.maybeCreateFutureReplica(logDir2.getAbsolutePath, offsetCheckpoints)
 
@@ -192,7 +192,7 @@ class PartitionTest {
     latch.countDown()
     thread1.join()
     thread2.join()
-    assertEquals(None, partition.futureLocalReplica)
+    assertEquals(None, partition.futureLog)
   }
 
   @Test
@@ -200,13 +200,12 @@ class PartitionTest {
   // active segment
   def testMaybeReplaceCurrentWithFutureReplicaDifferentBaseOffsets(): Unit = {
     logManager.maybeUpdatePreferredLogDir(topicPartition, logDir1.getAbsolutePath)
-    val currentReplica = partition.getOrCreateReplica(brokerId, isNew = true, offsetCheckpoints)
+    partition.createLogIfNotExists(brokerId, isNew = true, isFutureReplica = false, offsetCheckpoints)
     logManager.maybeUpdatePreferredLogDir(topicPartition, logDir2.getAbsolutePath)
     partition.maybeCreateFutureReplica(logDir2.getAbsolutePath, offsetCheckpoints)
-    val futureReplica = partition.futureLocalReplicaOrException
 
     // Write records with duplicate keys to current replica and roll at offset 6
-    val currentLog = currentReplica.log.get
+    val currentLog = partition.log.get
     currentLog.appendAsLeader(MemoryRecords.withRecords(0L, CompressionType.NONE, 0,
       new SimpleRecord("k1".getBytes, "v1".getBytes),
       new SimpleRecord("k1".getBytes, "v2".getBytes),
@@ -231,7 +230,8 @@ class PartitionTest {
     builder.appendWithOffset(6L, new SimpleRecord("k3".getBytes, "v7".getBytes))
     builder.appendWithOffset(7L, new SimpleRecord("k4".getBytes, "v8".getBytes))
 
-    futureReplica.log.get.appendAsFollower(builder.build())
+    val futureLog = partition.futureLocalLogOrException
+    futureLog.appendAsFollower(builder.build())
 
     assertTrue(partition.maybeReplaceCurrentWithFutureReplica())
   }
@@ -483,15 +483,14 @@ class PartitionTest {
     assertTrue("Expected first makeLeader() to return 'leader changed'",
       partition.makeLeader(controllerId, leaderState, 0, offsetCheckpoints))
     assertEquals("Current leader epoch", leaderEpoch, partition.getLeaderEpoch)
-    assertEquals("ISR", Set[Integer](leader, follower2), partition.inSyncReplicas.map(_.brokerId))
+    assertEquals("ISR", Set[Integer](leader, follower2), partition.inSyncReplicas)
 
     // after makeLeader(() call, partition should know about all the replicas
-    val leaderReplica = partition.getReplica(leader).get
-
     // append records with initial leader epoch
     partition.appendRecordsToLeader(batch1, isFromClient = true)
     partition.appendRecordsToLeader(batch2, isFromClient = true)
-    assertEquals("Expected leader's HW not move", leaderReplica.logStartOffset, leaderReplica.highWatermark)
+    assertEquals("Expected leader's HW not move", partition.localLogOrException.logStartOffset,
+      partition.localLogOrException.highWatermark)
 
     // let the follower in ISR move leader's HW to move further but below LEO
     def updateFollowerFetchState(followerId: Int, fetchOffsetMetadata: LogOffsetMetadata): Unit = {
@@ -500,7 +499,7 @@ class PartitionTest {
         followerFetchOffsetMetadata = fetchOffsetMetadata,
         followerStartOffset = 0L,
         followerFetchTimeMs = time.milliseconds(),
-        leaderEndOffset = leaderReplica.logEndOffset)
+        leaderEndOffset = partition.localLogOrException.logEndOffset)
     }
 
     def fetchOffsetsForTimestamp(timestamp: Long, isolation: Option[IsolationLevel]): Either[ApiException, Option[TimestampAndOffset]] = {
@@ -527,7 +526,7 @@ class PartitionTest {
     updateFollowerFetchState(follower2, LogOffsetMetadata(2))
 
     // At this point, the leader has gotten 5 writes, but followers have only fetched two
-    assertEquals(2, partition.localReplica.get.highWatermark)
+    assertEquals(2, partition.localLogOrException.highWatermark)
 
     // Get the LEO
     fetchOffsetsForTimestamp(ListOffsetRequest.LATEST_TIMESTAMP, None) match {
@@ -617,7 +616,7 @@ class PartitionTest {
   private def setupPartitionWithMocks(leaderEpoch: Int,
                                       isLeader: Boolean,
                                       log: Log = logManager.getOrCreateLog(topicPartition, logConfig)): Partition = {
-    val replica = partition.getOrCreateReplica(brokerId, isNew = false, offsetCheckpoints)
+    partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
 
     val controllerId = 0
     val controllerEpoch = 0
@@ -629,13 +628,12 @@ class PartitionTest {
         partition.makeLeader(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
           leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
       assertEquals(leaderEpoch, partition.getLeaderEpoch)
-      assertEquals(Some(replica), partition.leaderReplicaIfLocal)
     } else {
       assertTrue("Expected become follower transition to succeed",
         partition.makeFollower(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId + 1,
           leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
       assertEquals(leaderEpoch, partition.getLeaderEpoch)
-      assertEquals(None, partition.leaderReplicaIfLocal)
+      assertEquals(None, partition.leaderLogIfLocal)
     }
 
     partition
@@ -643,20 +641,22 @@ class PartitionTest {
 
   @Test
   def testAppendRecordsAsFollowerBelowLogStartOffset(): Unit = {
-    val replica = partition.getOrCreateReplica(brokerId, isNew = false, offsetCheckpoints)
+    partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    val log = partition.localLogOrException
+
     val initialLogStartOffset = 5L
     partition.truncateFullyAndStartAt(initialLogStartOffset, isFuture = false)
     assertEquals(s"Log end offset after truncate fully and start at $initialLogStartOffset:",
-                 initialLogStartOffset, replica.logEndOffset)
+                 initialLogStartOffset, log.logEndOffset)
     assertEquals(s"Log start offset after truncate fully and start at $initialLogStartOffset:",
-                 initialLogStartOffset, replica.logStartOffset)
+                 initialLogStartOffset, log.logStartOffset)
 
     // verify that we cannot append records that do not contain log start offset even if the log is empty
     assertThrows[UnexpectedAppendOffsetException] {
       // append one record with offset = 3
       partition.appendRecordsToFollowerOrFutureReplica(createRecords(List(new SimpleRecord("k1".getBytes, "v1".getBytes)), baseOffset = 3L), isFuture = false)
     }
-    assertEquals(s"Log end offset should not change after failure to append", initialLogStartOffset, replica.logEndOffset)
+    assertEquals(s"Log end offset should not change after failure to append", initialLogStartOffset, log.logEndOffset)
 
     // verify that we can append records that contain log start offset, even when first
     // offset < log start offset if the log is empty
@@ -666,13 +666,13 @@ class PartitionTest {
                                      new SimpleRecord("k3".getBytes, "v3".getBytes)),
                                 baseOffset = newLogStartOffset)
     partition.appendRecordsToFollowerOrFutureReplica(records, isFuture = false)
-    assertEquals(s"Log end offset after append of 3 records with base offset $newLogStartOffset:", 7L, replica.logEndOffset)
-    assertEquals(s"Log start offset after append of 3 records with base offset $newLogStartOffset:", newLogStartOffset, replica.logStartOffset)
+    assertEquals(s"Log end offset after append of 3 records with base offset $newLogStartOffset:", 7L, log.logEndOffset)
+    assertEquals(s"Log start offset after append of 3 records with base offset $newLogStartOffset:", newLogStartOffset, log.logStartOffset)
 
     // and we can append more records after that
     partition.appendRecordsToFollowerOrFutureReplica(createRecords(List(new SimpleRecord("k1".getBytes, "v1".getBytes)), baseOffset = 7L), isFuture = false)
-    assertEquals(s"Log end offset after append of 1 record at offset 7:", 8L, replica.logEndOffset)
-    assertEquals(s"Log start offset not expected to change:", newLogStartOffset, replica.logStartOffset)
+    assertEquals(s"Log end offset after append of 1 record at offset 7:", 8L, log.logEndOffset)
+    assertEquals(s"Log start offset not expected to change:", newLogStartOffset, log.logStartOffset)
 
     // but we cannot append to offset < log start if the log is not empty
     assertThrows[UnexpectedAppendOffsetException] {
@@ -681,12 +681,12 @@ class PartitionTest {
                                    baseOffset = 3L)
       partition.appendRecordsToFollowerOrFutureReplica(records2, isFuture = false)
     }
-    assertEquals(s"Log end offset should not change after failure to append", 8L, replica.logEndOffset)
+    assertEquals(s"Log end offset should not change after failure to append", 8L, log.logEndOffset)
 
     // we still can append to next offset
     partition.appendRecordsToFollowerOrFutureReplica(createRecords(List(new SimpleRecord("k1".getBytes, "v1".getBytes)), baseOffset = 8L), isFuture = false)
-    assertEquals(s"Log end offset after append of 1 record at offset 8:", 9L, replica.logEndOffset)
-    assertEquals(s"Log start offset not expected to change:", newLogStartOffset, replica.logStartOffset)
+    assertEquals(s"Log end offset after append of 1 record at offset 8:", 9L, log.logEndOffset)
+    assertEquals(s"Log start offset not expected to change:", newLogStartOffset, log.logStartOffset)
   }
 
   @Test
@@ -699,13 +699,12 @@ class PartitionTest {
 
     doNothing().when(delayedOperations).checkAndCompleteFetch()
 
-    val replica = partition.getOrCreateReplica(brokerId, isNew = false, offsetCheckpoints)
+    partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
 
     assertTrue("Expected become leader transition to succeed",
       partition.makeLeader(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
         leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
     assertEquals(leaderEpoch, partition.getLeaderEpoch)
-    assertEquals(Some(replica), partition.leaderReplicaIfLocal)
 
     val records = createTransactionalRecords(List(
       new SimpleRecord("k1".getBytes, "v1".getBytes),
@@ -736,7 +735,7 @@ class PartitionTest {
     assertEquals(0L, fetchLatestOffset(isolationLevel = Some(IsolationLevel.READ_UNCOMMITTED)).offset)
     assertEquals(0L, fetchLatestOffset(isolationLevel = Some(IsolationLevel.READ_COMMITTED)).offset)
 
-    replica.highWatermark = 1L
+    partition.log.get.highWatermark = 1L
 
     assertEquals(3L, fetchLatestOffset(isolationLevel = None).offset)
     assertEquals(1L, fetchLatestOffset(isolationLevel = Some(IsolationLevel.READ_UNCOMMITTED)).offset)
@@ -749,14 +748,19 @@ class PartitionTest {
 
   @Test
   def testGetReplica(): Unit = {
-    assertEquals(None, partition.localReplica)
+    assertEquals(None, partition.log)
     assertThrows[ReplicaNotAvailableException] {
-      partition.localReplicaOrException
+      partition.localLogOrException
     }
 
-    val replica = partition.getOrCreateReplica(brokerId, isNew = false, offsetCheckpoints)
-    assertEquals(Some(replica), partition.localReplica)
-    assertEquals(replica, partition.localReplicaOrException)
+    assertThrows[IllegalArgumentException] {
+      val replica = partition.getOrCreateReplica(brokerId)
+    }
+
+    val remoteReplicaId = brokerId + 1;
+    val replica = partition.getOrCreateReplica(remoteReplicaId)
+    assertEquals(replica.brokerId, remoteReplicaId)
+    assertEquals(replica.topicPartition, partition.topicPartition)
   }
 
   @Test
@@ -807,15 +811,14 @@ class PartitionTest {
     assertTrue("Expected first makeLeader() to return 'leader changed'",
                partition.makeLeader(controllerId, leaderState, 0, offsetCheckpoints))
     assertEquals("Current leader epoch", leaderEpoch, partition.getLeaderEpoch)
-    assertEquals("ISR", Set[Integer](leader, follower2), partition.inSyncReplicas.map(_.brokerId))
+    assertEquals("ISR", Set[Integer](leader, follower2), partition.inSyncReplicas)
 
     // after makeLeader(() call, partition should know about all the replicas
-    val leaderReplica = partition.getReplica(leader).get
-
     // append records with initial leader epoch
     val lastOffsetOfFirstBatch = partition.appendRecordsToLeader(batch1, isFromClient = true).lastOffset
     partition.appendRecordsToLeader(batch2, isFromClient = true)
-    assertEquals("Expected leader's HW not move", leaderReplica.logStartOffset, leaderReplica.highWatermark)
+    assertEquals("Expected leader's HW not move", partition.localLogOrException.logStartOffset,
+      partition.log.get.highWatermark)
 
     // let the follower in ISR move leader's HW to move further but below LEO
     def updateFollowerFetchState(followerId: Int, fetchOffsetMetadata: LogOffsetMetadata): Unit = {
@@ -824,12 +827,12 @@ class PartitionTest {
         followerFetchOffsetMetadata = fetchOffsetMetadata,
         followerStartOffset = 0L,
         followerFetchTimeMs = time.milliseconds(),
-        leaderEndOffset = leaderReplica.logEndOffset)
+        leaderEndOffset = partition.localLogOrException.logEndOffset)
     }
 
     updateFollowerFetchState(follower2, LogOffsetMetadata(0))
     updateFollowerFetchState(follower2, LogOffsetMetadata(lastOffsetOfFirstBatch))
-    assertEquals("Expected leader's HW", lastOffsetOfFirstBatch, leaderReplica.highWatermark)
+    assertEquals("Expected leader's HW", lastOffsetOfFirstBatch, partition.log.get.highWatermark)
 
     // current leader becomes follower and then leader again (without any new records appended)
     val followerState = new LeaderAndIsrRequest.PartitionState(controllerEpoch, follower2, leaderEpoch + 1, isr, 1,
@@ -840,7 +843,7 @@ class PartitionTest {
       replicas, false)
     assertTrue("Expected makeLeader() to return 'leader changed' after makeFollower()",
                partition.makeLeader(controllerEpoch, newLeaderState, 2, offsetCheckpoints))
-    val currentLeaderEpochStartOffset = leaderReplica.logEndOffset
+    val currentLeaderEpochStartOffset = partition.localLogOrException.logEndOffset
 
     // append records with the latest leader epoch
     partition.appendRecordsToLeader(batch3, isFromClient = true)
@@ -848,14 +851,14 @@ class PartitionTest {
     // fetch from follower not in ISR from log start offset should not add this follower to ISR
     updateFollowerFetchState(follower1, LogOffsetMetadata(0))
     updateFollowerFetchState(follower1, LogOffsetMetadata(lastOffsetOfFirstBatch))
-    assertEquals("ISR", Set[Integer](leader, follower2), partition.inSyncReplicas.map(_.brokerId))
+    assertEquals("ISR", Set[Integer](leader, follower2), partition.inSyncReplicas)
 
     // fetch from the follower not in ISR from start offset of the current leader epoch should
     // add this follower to ISR
     when(stateStore.expandIsr(controllerEpoch, new LeaderAndIsr(leader, leaderEpoch + 2,
       List(leader, follower2, follower1), 1))).thenReturn(Some(2))
     updateFollowerFetchState(follower1, LogOffsetMetadata(currentLeaderEpochStartOffset))
-    assertEquals("ISR", Set[Integer](leader, follower1, follower2), partition.inSyncReplicas.map(_.brokerId))
+    assertEquals("ISR", Set[Integer](leader, follower1, follower2), partition.inSyncReplicas)
   }
 
   /**
@@ -876,11 +879,10 @@ class PartitionTest {
 
     val topicPartitions = (0 until 5).map { i => new TopicPartition("test-topic", i) }
     val logs = topicPartitions.map { tp => logManager.getOrCreateLog(tp, logConfig) }
-    val replicas = logs.map { log => new Replica(brokerId, log.topicPartition, time, log = Some(log)) }
     val partitions = ListBuffer.empty[Partition]
 
-    replicas.foreach { replica =>
-      val tp = replica.topicPartition
+    logs.foreach { log =>
+      val tp = log.topicPartition
       val delayedOperations: DelayedOperations = mock(classOf[DelayedOperations])
       val partition = new Partition(tp,
         replicaLagTimeMaxMs = Defaults.ReplicaLagTimeMaxMs,
@@ -902,7 +904,7 @@ class PartitionTest {
           }
         })
 
-      partition.addReplicaIfNotExists(replica)
+      partition.setLog(log, isFutureLog = false)
       val leaderState = new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
         leaderEpoch, isr, 1, replicaIds, true)
       partition.makeLeader(controllerId, leaderState, 0, offsetCheckpoints)
@@ -1007,7 +1009,7 @@ class PartitionTest {
 
     doNothing().when(delayedOperations).checkAndCompleteFetch()
 
-    partition.getOrCreateReplica(brokerId, isNew = false, offsetCheckpoints)
+    partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
 
     val initializeTimeMs = time.milliseconds()
     assertTrue("Expected become leader transition to succeed",
@@ -1058,11 +1060,11 @@ class PartitionTest {
 
     doNothing().when(delayedOperations).checkAndCompleteFetch()
 
-    partition.getOrCreateReplica(brokerId, isNew = false, offsetCheckpoints)
+    partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
     assertTrue("Expected become leader transition to succeed",
       partition.makeLeader(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
         leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
-    assertEquals(Set(brokerId), partition.inSyncReplicas.map(_.brokerId))
+    assertEquals(Set(brokerId), partition.inSyncReplicas)
 
     val remoteReplica = partition.getReplica(remoteBrokerId).get
     assertEquals(LogOffsetMetadata.UnknownOffsetMetadata.messageOffset, remoteReplica.logEndOffset)
@@ -1074,7 +1076,7 @@ class PartitionTest {
       followerFetchTimeMs = time.milliseconds(),
       leaderEndOffset = 6L)
 
-    assertEquals(Set(brokerId), partition.inSyncReplicas.map(_.brokerId))
+    assertEquals(Set(brokerId), partition.inSyncReplicas)
     assertEquals(3L, remoteReplica.logEndOffset)
     assertEquals(0L, remoteReplica.logStartOffset)
 
@@ -1092,7 +1094,7 @@ class PartitionTest {
       followerFetchTimeMs = time.milliseconds(),
       leaderEndOffset = 6L)
 
-    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas.map(_.brokerId))
+    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas)
     assertEquals(10L, remoteReplica.logEndOffset)
     assertEquals(0L, remoteReplica.logStartOffset)
   }
@@ -1111,11 +1113,11 @@ class PartitionTest {
 
     doNothing().when(delayedOperations).checkAndCompleteFetch()
 
-    partition.getOrCreateReplica(brokerId, isNew = false, offsetCheckpoints)
+    partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
     assertTrue("Expected become leader transition to succeed",
       partition.makeLeader(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
         leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
-    assertEquals(Set(brokerId), partition.inSyncReplicas.map(_.brokerId))
+    assertEquals(Set(brokerId), partition.inSyncReplicas)
 
     val remoteReplica = partition.getReplica(remoteBrokerId).get
     assertEquals(LogOffsetMetadata.UnknownOffsetMetadata.messageOffset, remoteReplica.logEndOffset)
@@ -1136,7 +1138,7 @@ class PartitionTest {
       leaderEndOffset = 10L)
 
     // Follower state is updated, but the ISR has not expanded
-    assertEquals(Set(brokerId), partition.inSyncReplicas.map(_.brokerId))
+    assertEquals(Set(brokerId), partition.inSyncReplicas)
     assertEquals(10L, remoteReplica.logEndOffset)
     assertEquals(0L, remoteReplica.logStartOffset)
   }
@@ -1156,12 +1158,12 @@ class PartitionTest {
     doNothing().when(delayedOperations).checkAndCompleteFetch()
 
     val initializeTimeMs = time.milliseconds()
-    partition.getOrCreateReplica(brokerId, isNew = false, offsetCheckpoints)
+    partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
     assertTrue("Expected become leader transition to succeed",
       partition.makeLeader(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
         leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
-    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas.map(_.brokerId))
-    assertEquals(0L, partition.localReplicaOrException.highWatermark)
+    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas)
+    assertEquals(0L, partition.localLogOrException.highWatermark)
 
     val remoteReplica = partition.getReplica(remoteBrokerId).get
     assertEquals(initializeTimeMs, remoteReplica.lastCaughtUpTimeMs)
@@ -1170,7 +1172,7 @@ class PartitionTest {
 
     // On initialization, the replica is considered caught up and should not be removed
     partition.maybeShrinkIsr(10000)
-    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas.map(_.brokerId))
+    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas)
 
     // If enough time passes without a fetch update, the ISR should shrink
     time.sleep(10001)
@@ -1182,8 +1184,8 @@ class PartitionTest {
     when(stateStore.shrinkIsr(controllerEpoch, updatedLeaderAndIsr)).thenReturn(Some(2))
 
     partition.maybeShrinkIsr(10000)
-    assertEquals(Set(brokerId), partition.inSyncReplicas.map(_.brokerId))
-    assertEquals(10L, partition.localReplicaOrException.highWatermark)
+    assertEquals(Set(brokerId), partition.inSyncReplicas)
+    assertEquals(10L, partition.localLogOrException.highWatermark)
   }
 
   @Test
@@ -1201,12 +1203,12 @@ class PartitionTest {
     doNothing().when(delayedOperations).checkAndCompleteFetch()
 
     val initializeTimeMs = time.milliseconds()
-    partition.getOrCreateReplica(brokerId, isNew = false, offsetCheckpoints)
+    partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
     assertTrue("Expected become leader transition to succeed",
       partition.makeLeader(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
         leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
-    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas.map(_.brokerId))
-    assertEquals(0L, partition.localReplicaOrException.highWatermark)
+    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas)
+    assertEquals(0L, partition.localLogOrException.highWatermark)
 
     val remoteReplica = partition.getReplica(remoteBrokerId).get
     assertEquals(initializeTimeMs, remoteReplica.lastCaughtUpTimeMs)
@@ -1222,7 +1224,7 @@ class PartitionTest {
       followerFetchTimeMs = firstFetchTimeMs,
       leaderEndOffset = 10L)
     assertEquals(initializeTimeMs, remoteReplica.lastCaughtUpTimeMs)
-    assertEquals(5L, partition.localReplicaOrException.highWatermark)
+    assertEquals(5L, partition.localLogOrException.highWatermark)
     assertEquals(5L, remoteReplica.logEndOffset)
     assertEquals(0L, remoteReplica.logStartOffset)
 
@@ -1236,14 +1238,14 @@ class PartitionTest {
       followerFetchTimeMs = time.milliseconds(),
       leaderEndOffset = 15L)
     assertEquals(firstFetchTimeMs, remoteReplica.lastCaughtUpTimeMs)
-    assertEquals(10L, partition.localReplicaOrException.highWatermark)
+    assertEquals(10L, partition.localLogOrException.highWatermark)
     assertEquals(10L, remoteReplica.logEndOffset)
     assertEquals(0L, remoteReplica.logStartOffset)
 
     // The ISR should not be shrunk because the follower has caught up with the leader at the
     // time of the first fetch.
     partition.maybeShrinkIsr(10000)
-    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas.map(_.brokerId))
+    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas)
   }
 
   @Test
@@ -1261,12 +1263,12 @@ class PartitionTest {
     doNothing().when(delayedOperations).checkAndCompleteFetch()
 
     val initializeTimeMs = time.milliseconds()
-    partition.getOrCreateReplica(brokerId, isNew = false, offsetCheckpoints)
+    partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
     assertTrue("Expected become leader transition to succeed",
       partition.makeLeader(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
         leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
-    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas.map(_.brokerId))
-    assertEquals(0L, partition.localReplicaOrException.highWatermark)
+    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas)
+    assertEquals(0L, partition.localLogOrException.highWatermark)
 
     val remoteReplica = partition.getReplica(remoteBrokerId).get
     assertEquals(initializeTimeMs, remoteReplica.lastCaughtUpTimeMs)
@@ -1280,7 +1282,7 @@ class PartitionTest {
       followerFetchTimeMs = time.milliseconds(),
       leaderEndOffset = 10L)
     assertEquals(initializeTimeMs, remoteReplica.lastCaughtUpTimeMs)
-    assertEquals(10L, partition.localReplicaOrException.highWatermark)
+    assertEquals(10L, partition.localLogOrException.highWatermark)
     assertEquals(10L, remoteReplica.logEndOffset)
     assertEquals(0L, remoteReplica.logStartOffset)
 
@@ -1289,7 +1291,7 @@ class PartitionTest {
 
     // The ISR should not be shrunk because the follower is caught up to the leader's log end
     partition.maybeShrinkIsr(10000)
-    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas.map(_.brokerId))
+    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas)
   }
 
   @Test
@@ -1307,12 +1309,12 @@ class PartitionTest {
     doNothing().when(delayedOperations).checkAndCompleteFetch()
 
     val initializeTimeMs = time.milliseconds()
-    partition.getOrCreateReplica(brokerId, isNew = false, offsetCheckpoints)
+    partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
     assertTrue("Expected become leader transition to succeed",
       partition.makeLeader(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
         leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
-    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas.map(_.brokerId))
-    assertEquals(0L, partition.localReplicaOrException.highWatermark)
+    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas)
+    assertEquals(0L, partition.localLogOrException.highWatermark)
 
     val remoteReplica = partition.getReplica(remoteBrokerId).get
     assertEquals(initializeTimeMs, remoteReplica.lastCaughtUpTimeMs)
@@ -1330,8 +1332,8 @@ class PartitionTest {
     when(stateStore.shrinkIsr(controllerEpoch, updatedLeaderAndIsr)).thenReturn(None)
 
     partition.maybeShrinkIsr(10000)
-    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas.map(_.brokerId))
-    assertEquals(0L, partition.localReplicaOrException.highWatermark)
+    assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicas)
+    assertEquals(0L, partition.localLogOrException.highWatermark)
   }
 
   @Test
@@ -1348,7 +1350,7 @@ class PartitionTest {
     val leaderState = new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
       6, replicas, 1, replicas, false)
     partition.makeLeader(controllerId, leaderState, 0, offsetCheckpoints)
-    assertEquals(4, partition.localReplicaOrException.highWatermark)
+    assertEquals(4, partition.localLogOrException.highWatermark)
   }
 
   @Test
diff --git a/core/src/test/scala/unit/kafka/cluster/ReplicaTest.scala b/core/src/test/scala/unit/kafka/cluster/ReplicaTest.scala
index 64e298fa1eaf6..31192f06f41ed 100644
--- a/core/src/test/scala/unit/kafka/cluster/ReplicaTest.scala
+++ b/core/src/test/scala/unit/kafka/cluster/ReplicaTest.scala
@@ -21,7 +21,6 @@ import java.util.Properties
 import kafka.log.{Log, LogConfig, LogManager}
 import kafka.server.{BrokerTopicStats, LogDirFailureChannel}
 import kafka.utils.{MockTime, TestUtils}
-import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.OffsetOutOfRangeException
 import org.apache.kafka.common.utils.Utils
 import org.junit.{After, Before, Test}
@@ -34,7 +33,6 @@ class ReplicaTest {
   val time = new MockTime()
   val brokerTopicStats = new BrokerTopicStats
   var log: Log = _
-  var replica: Replica = _
 
   @Before
   def setup(): Unit = {
@@ -53,11 +51,6 @@ class ReplicaTest {
       maxProducerIdExpirationMs = 60 * 60 * 1000,
       producerIdExpirationCheckIntervalMs = LogManager.ProducerIdExpirationCheckIntervalMs,
       logDirFailureChannel = new LogDirFailureChannel(10))
-
-    replica = new Replica(brokerId = 0,
-      topicPartition = new TopicPartition("foo", 0),
-      time = time,
-      log = Some(log))
   }
 
   @After
@@ -70,13 +63,9 @@ class ReplicaTest {
   @Test
   def testSegmentDeletionWithHighWatermarkInitialization(): Unit = {
     val initialHighWatermark = 25L
-    replica = new Replica(brokerId = 0,
-      topicPartition = new TopicPartition("foo", 0),
-      time = time,
-      initialHighWatermarkValue = initialHighWatermark,
-      log = Some(log))
+    log.highWatermark = initialHighWatermark
 
-    assertEquals(initialHighWatermark, replica.highWatermark)
+    assertEquals(initialHighWatermark, log.highWatermark)
 
     val expiredTimestamp = time.milliseconds() - 1000
     for (i <- 0 until 100) {
@@ -87,7 +76,7 @@ class ReplicaTest {
     val initialNumSegments = log.numberOfSegments
     log.deleteOldSegments()
     assertTrue(log.numberOfSegments < initialNumSegments)
-    assertTrue(replica.logStartOffset <= initialHighWatermark)
+    assertTrue(log.logStartOffset <= initialHighWatermark)
   }
 
   @Test
@@ -100,25 +89,25 @@ class ReplicaTest {
 
     // ensure we have at least a few segments so the test case is not trivial
     assertTrue(log.numberOfSegments > 5)
-    assertEquals(0L, replica.highWatermark)
-    assertEquals(0L, replica.logStartOffset)
-    assertEquals(100L, replica.logEndOffset)
+    assertEquals(0L, log.highWatermark)
+    assertEquals(0L, log.logStartOffset)
+    assertEquals(100L, log.logEndOffset)
 
     for (hw <- 0 to 100) {
-      replica.highWatermark = hw
-      assertEquals(hw, replica.highWatermark)
+      log.highWatermark = hw
+      assertEquals(hw, log.highWatermark)
       log.deleteOldSegments()
-      assertTrue(replica.logStartOffset <= hw)
+      assertTrue(log.logStartOffset <= hw)
 
       // verify that all segments up to the high watermark have been deleted
 
       log.logSegments.headOption.foreach { segment =>
         assertTrue(segment.baseOffset <= hw)
-        assertTrue(segment.baseOffset >= replica.logStartOffset)
+        assertTrue(segment.baseOffset >= log.logStartOffset)
       }
       log.logSegments.tail.foreach { segment =>
         assertTrue(segment.baseOffset > hw)
-        assertTrue(segment.baseOffset >= replica.logStartOffset)
+        assertTrue(segment.baseOffset >= log.logStartOffset)
       }
     }
 
@@ -134,8 +123,7 @@ class ReplicaTest {
       log.appendAsLeader(records, leaderEpoch = 0)
     }
 
-    replica.highWatermark = 25L
-    replica.maybeIncrementLogStartOffset(26L)
+    log.highWatermark = 25L
+    log.maybeIncrementLogStartOffset(26L)
   }
-
 }
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
index 7c83cded3bb7f..15b47ca76339c 100644
--- a/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
@@ -20,7 +20,7 @@ package kafka.log
 import java.io.File
 import java.util.Properties
 
-import kafka.server.{BrokerTopicStats, LogDirFailureChannel}
+import kafka.server.{BrokerTopicStats, LogDirFailureChannel, LogOffsetMetadata}
 import kafka.utils._
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.record._
@@ -218,7 +218,7 @@ class LogCleanerManagerTest extends Logging {
     log.appendAsLeader(records, leaderEpoch = 0)
     log.roll()
     log.appendAsLeader(records, leaderEpoch = 0)
-    log.onHighWatermarkIncremented(2L)
+    log.highWatermarkMetadata = LogOffsetMetadata(2L)
 
     // simulate cleanup thread working on the log partition
     val deletableLog = cleanerManager.pauseCleaningForNonCompactedPartitions()
@@ -403,7 +403,7 @@ class LogCleanerManagerTest extends Logging {
     log.appendAsLeader(MemoryRecords.withTransactionalRecords(CompressionType.NONE, producerId, producerEpoch, sequence + 2,
       new SimpleRecord(time.milliseconds(), "3".getBytes, "c".getBytes)), leaderEpoch = 0)
     log.roll()
-    log.onHighWatermarkIncremented(3L)
+    log.highWatermarkMetadata = LogOffsetMetadata(3L)
 
     time.sleep(compactionLag + 1)
     // although the compaction lag has been exceeded, the undecided data should not be cleaned
@@ -415,7 +415,7 @@ class LogCleanerManagerTest extends Logging {
     log.appendAsLeader(MemoryRecords.withEndTransactionMarker(time.milliseconds(), producerId, producerEpoch,
       new EndTransactionMarker(ControlRecordType.ABORT, 15)), leaderEpoch = 0, isFromClient = false)
     log.roll()
-    log.onHighWatermarkIncremented(4L)
+    log.highWatermarkMetadata = LogOffsetMetadata(4L)
 
     // the first segment should now become cleanable immediately
     cleanableOffsets = LogCleanerManager.cleanableOffsets(log, topicPartition,
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerParameterizedIntegrationTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerParameterizedIntegrationTest.scala
index 61e3ea57604d0..d0ef078b8fc95 100755
--- a/core/src/test/scala/unit/kafka/log/LogCleanerParameterizedIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerParameterizedIntegrationTest.scala
@@ -22,7 +22,7 @@ import java.util.Properties
 
 import kafka.api.KAFKA_0_11_0_IV0
 import kafka.api.{KAFKA_0_10_0_IV1, KAFKA_0_9_0}
-import kafka.server.KafkaConfig
+import kafka.server.{KafkaConfig, LogOffsetMetadata}
 import kafka.server.checkpoints.OffsetCheckpointFile
 import kafka.utils._
 import org.apache.kafka.common.TopicPartition
@@ -102,7 +102,7 @@ class LogCleanerParameterizedIntegrationTest(compressionCodec: String) extends A
       val messages = writeDups(numKeys = numKeys, numDups = 3, log = log, codec = codec)
       val startSize = log.size
 
-      log.onHighWatermarkIncremented(log.logEndOffset)
+      log.highWatermarkMetadata = LogOffsetMetadata(log.logEndOffset)
 
       val firstDirty = log.activeSegment.baseOffset
       cleaner.startup()
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala
index 825fa90f959e2..42ad8a08d1481 100755
--- a/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala
@@ -24,7 +24,7 @@ import java.util.Properties
 import java.util.concurrent.{CountDownLatch, TimeUnit}
 
 import kafka.common._
-import kafka.server.{BrokerTopicStats, LogDirFailureChannel}
+import kafka.server.{BrokerTopicStats, LogDirFailureChannel, LogOffsetMetadata}
 import kafka.utils._
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.CorruptRecordException
@@ -126,8 +126,9 @@ class LogCleanerTest {
     val t = new Thread() {
       override def run(): Unit = {
         deleteStartLatch.await(5000, TimeUnit.MILLISECONDS)
+        log.highWatermark = log.activeSegment.baseOffset
         log.maybeIncrementLogStartOffset(log.activeSegment.baseOffset)
-        log.onHighWatermarkIncremented(log.activeSegment.baseOffset)
+        log.highWatermarkMetadata = LogOffsetMetadata(log.activeSegment.baseOffset)
         log.deleteOldSegments()
         deleteCompleteLatch.countDown()
       }
diff --git a/core/src/test/scala/unit/kafka/log/LogManagerTest.scala b/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
index ed1824df04d2a..94ae3eddb2727 100755
--- a/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
@@ -20,7 +20,7 @@ package kafka.log
 import java.io._
 import java.util.{Collections, Properties}
 
-import kafka.server.FetchDataInfo
+import kafka.server.{FetchDataInfo, LogOffsetMetadata}
 import kafka.server.checkpoints.OffsetCheckpointFile
 import kafka.utils._
 import org.apache.kafka.common.{KafkaException, TopicPartition}
@@ -119,7 +119,8 @@ class LogManagerTest {
       offset = info.lastOffset
     }
     assertTrue("There should be more than one segment now.", log.numberOfSegments > 1)
-    log.onHighWatermarkIncremented(log.logEndOffset)
+    log.highWatermarkMetadata = LogOffsetMetadata(log.logEndOffset)
+    log.highWatermark = log.logEndOffset
 
     log.logSegments.foreach(_.log.file.setLastModified(time.milliseconds))
 
@@ -173,7 +174,8 @@ class LogManagerTest {
       offset = info.firstOffset.get
     }
 
-    log.onHighWatermarkIncremented(log.logEndOffset)
+    log.highWatermarkMetadata = LogOffsetMetadata(log.logEndOffset)
+    log.highWatermark = log.logEndOffset
     assertEquals("Check we have the expected number of segments.", numMessages * setSize / config.segmentSize, log.numberOfSegments)
 
     // this cleanup shouldn't find any expired segments but should delete some to reduce size
diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala
index dc9a423e34556..8fb65ff213202 100755
--- a/core/src/test/scala/unit/kafka/log/LogTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogTest.scala
@@ -317,6 +317,7 @@ class LogTest {
 
     // Increment the log start offset
     val startOffset = 4
+    log.highWatermark = log.logEndOffset
     log.maybeIncrementLogStartOffset(startOffset)
     assertTrue(log.logEndOffset > log.logStartOffset)
 
@@ -830,6 +831,7 @@ class LogTest {
       producerEpoch = epoch, sequence = 0), leaderEpoch = 0)
     assertEquals(2, log.activeProducersWithLastSequence.size)
 
+    log.highWatermark = log.logEndOffset
     log.maybeIncrementLogStartOffset(1L)
 
     assertEquals(1, log.activeProducersWithLastSequence.size)
@@ -862,8 +864,8 @@ class LogTest {
     assertEquals(2, log.logSegments.size)
     assertEquals(2, log.activeProducersWithLastSequence.size)
 
+    log.highWatermark = log.logEndOffset
     log.maybeIncrementLogStartOffset(1L)
-    log.onHighWatermarkIncremented(log.logEndOffset)
     log.deleteOldSegments()
 
     assertEquals(1, log.logSegments.size)
@@ -921,7 +923,7 @@ class LogTest {
     assertEquals(3, log.logSegments.size)
     assertEquals(Set(pid1, pid2), log.activeProducersWithLastSequence.keySet)
 
-    log.onHighWatermarkIncremented(log.logEndOffset)
+    log.highWatermark = log.logEndOffset
     log.deleteOldSegments()
 
     assertEquals(2, log.logSegments.size)
@@ -1010,7 +1012,7 @@ class LogTest {
     log.appendAsLeader(records, leaderEpoch = 0)
     val commitAppendInfo = log.appendAsLeader(endTxnRecords(ControlRecordType.ABORT, pid, epoch),
       isFromClient = false, leaderEpoch = 0)
-    log.onHighWatermarkIncremented(commitAppendInfo.lastOffset + 1)
+    log.highWatermark = commitAppendInfo.lastOffset + 1
 
     // now there should be no first unstable offset
     assertEquals(None, log.firstUnstableOffset)
@@ -1018,7 +1020,7 @@ class LogTest {
     log.close()
 
     val reopenedLog = createLog(logDir, logConfig)
-    reopenedLog.onHighWatermarkIncremented(commitAppendInfo.lastOffset + 1)
+    reopenedLog.highWatermark = commitAppendInfo.lastOffset + 1
     assertEquals(None, reopenedLog.firstUnstableOffset)
   }
 
@@ -1557,7 +1559,7 @@ class LogTest {
       assertEquals(currOffset, messagesToAppend)
 
       // time goes by; the log file is deleted
-      log.onHighWatermarkIncremented(currOffset)
+      log.highWatermark = currOffset
       log.deleteOldSegments()
 
       assertEquals("Deleting segments shouldn't have changed the logEndOffset", currOffset, log.logEndOffset)
@@ -2033,7 +2035,7 @@ class LogTest {
     val segments = log.logSegments.toArray
     val oldFiles = segments.map(_.log.file) ++ segments.map(_.lazyOffsetIndex.file)
 
-    log.onHighWatermarkIncremented(log.logEndOffset)
+    log.highWatermark = log.logEndOffset
     log.deleteOldSegments()
 
     assertEquals("Only one segment should remain.", 1, log.numberOfSegments)
@@ -2063,7 +2065,7 @@ class LogTest {
       log.appendAsLeader(createRecords, leaderEpoch = 0)
 
     // expire all segments
-    log.onHighWatermarkIncremented(log.logEndOffset)
+    log.highWatermark = log.logEndOffset
     log.deleteOldSegments()
     log.close()
     log = createLog(logDir, logConfig)
@@ -2818,7 +2820,7 @@ class LogTest {
 
     // only segments with offset before the current high watermark are eligible for deletion
     for (hw <- 25 to 30) {
-      log.onHighWatermarkIncremented(hw)
+      log.highWatermark = hw
       log.deleteOldSegments()
       assertTrue(log.logStartOffset <= hw)
       log.logSegments.foreach { segment =>
@@ -2831,7 +2833,7 @@ class LogTest {
     }
 
     // expire all segments
-    log.onHighWatermarkIncremented(log.logEndOffset)
+    log.highWatermark = log.logEndOffset
     log.deleteOldSegments()
     assertEquals("The deleted segments should be gone.", 1, log.numberOfSegments)
     assertEquals("Epoch entries should have gone.", 1, epochCache(log).epochEntries.size)
@@ -2875,7 +2877,7 @@ class LogTest {
       log.appendAsLeader(createRecords, leaderEpoch = 0)
     assertEquals("should have 3 segments", 3, log.numberOfSegments)
     assertEquals(log.logStartOffset, 0)
-    log.onHighWatermarkIncremented(log.logEndOffset)
+    log.highWatermark = log.logEndOffset
 
     log.maybeIncrementLogStartOffset(1)
     log.deleteOldSegments()
@@ -2907,7 +2909,7 @@ class LogTest {
     for (_ <- 0 until 15)
       log.appendAsLeader(createRecords, leaderEpoch = 0)
 
-    log.onHighWatermarkIncremented(log.logEndOffset)
+    log.highWatermark = log.logEndOffset
     log.deleteOldSegments()
     assertEquals("should have 2 segments", 2,log.numberOfSegments)
   }
@@ -2922,7 +2924,7 @@ class LogTest {
     for (_ <- 0 until 15)
       log.appendAsLeader(createRecords, leaderEpoch = 0)
 
-    log.onHighWatermarkIncremented(log.logEndOffset)
+    log.highWatermark = log.logEndOffset
     log.deleteOldSegments()
     assertEquals("should have 3 segments", 3,log.numberOfSegments)
   }
@@ -2937,7 +2939,7 @@ class LogTest {
     for (_ <- 0 until 15)
       log.appendAsLeader(createRecords, leaderEpoch = 0)
 
-    log.onHighWatermarkIncremented(log.logEndOffset)
+    log.highWatermark = log.logEndOffset
     log.deleteOldSegments()
     assertEquals("There should be 1 segment remaining", 1, log.numberOfSegments)
   }
@@ -2952,7 +2954,7 @@ class LogTest {
     for (_ <- 0 until 15)
       log.appendAsLeader(createRecords, leaderEpoch = 0)
 
-    log.onHighWatermarkIncremented(log.logEndOffset)
+    log.highWatermark = log.logEndOffset
     log.deleteOldSegments()
     assertEquals("There should be 3 segments remaining", 3, log.numberOfSegments)
   }
@@ -2971,7 +2973,7 @@ class LogTest {
     log.logSegments.head.lastModified = mockTime.milliseconds - 20000
 
     val segments = log.numberOfSegments
-    log.onHighWatermarkIncremented(log.logEndOffset)
+    log.highWatermark = log.logEndOffset
     log.deleteOldSegments()
     assertEquals("There should be 3 segments remaining", segments, log.numberOfSegments)
   }
@@ -2986,7 +2988,7 @@ class LogTest {
     for (_ <- 0 until 15)
       log.appendAsLeader(createRecords, leaderEpoch = 0)
 
-    log.onHighWatermarkIncremented(log.logEndOffset)
+    log.highWatermark = log.logEndOffset
     log.deleteOldSegments()
     assertEquals("There should be 1 segment remaining", 1, log.numberOfSegments)
   }
@@ -3004,12 +3006,13 @@ class LogTest {
 
     // Three segments should be created
     assertEquals(3, log.logSegments.count(_ => true))
+    log.highWatermark = log.logEndOffset
     log.maybeIncrementLogStartOffset(recordsPerSegment)
 
     // The first segment, which is entirely before the log start offset, should be deleted
     // Of the remaining the segments, the first can overlap the log start offset and the rest must have a base offset
     // greater than the start offset
-    log.onHighWatermarkIncremented(log.logEndOffset)
+    log.highWatermark = log.logEndOffset
     log.deleteOldSegments()
     assertEquals("There should be 2 segments remaining", 2, log.numberOfSegments)
     assertTrue(log.logSegments.head.baseOffset <= log.logStartOffset)
@@ -3081,7 +3084,7 @@ class LogTest {
     cache.assign(2, 10)
 
     //When first segment is removed
-    log.onHighWatermarkIncremented(log.logEndOffset)
+    log.highWatermark = log.logEndOffset
     log.deleteOldSegments()
 
     //The oldest epoch entry should have been removed
@@ -3106,7 +3109,7 @@ class LogTest {
     cache.assign(2, 10)
 
     //When first segment removed (up to offset 5)
-    log.onHighWatermarkIncremented(log.logEndOffset)
+    log.highWatermark = log.logEndOffset
     log.deleteOldSegments()
 
     //The first entry should have gone from (0,0) => (0,5)
@@ -3261,7 +3264,7 @@ class LogTest {
 
     // first unstable offset is not updated until the high watermark is advanced
     assertEquals(Some(firstAppendInfo.firstOffset.get), log.firstUnstableOffset.map(_.messageOffset))
-    log.onHighWatermarkIncremented(commitAppendInfo.lastOffset + 1)
+    log.highWatermark = commitAppendInfo.lastOffset + 1
 
     // now there should be no first unstable offset
     assertEquals(None, log.firstUnstableOffset)
@@ -3568,6 +3571,7 @@ class LogTest {
 
     assertEquals(Some(0L), log.firstUnstableOffset.map(_.messageOffset))
 
+    log.highWatermark = log.logEndOffset
     log.maybeIncrementLogStartOffset(5L)
 
     // the first unstable offset should be lower bounded by the log start offset
@@ -3592,8 +3596,9 @@ class LogTest {
 
     assertEquals(Some(0L), log.firstUnstableOffset.map(_.messageOffset))
 
+    log.highWatermark = log.logEndOffset
     log.maybeIncrementLogStartOffset(8L)
-    log.onHighWatermarkIncremented(log.logEndOffset)
+    log.highWatermark = log.logEndOffset
     log.deleteOldSegments()
     assertEquals(1, log.logSegments.size)
 
@@ -3630,7 +3635,7 @@ class LogTest {
     assertEquals(Some(0L), log.firstUnstableOffset.map(_.messageOffset))
 
     // Even if the high watermark is updated, the first unstable offset does not move
-    log.onHighWatermarkIncremented(12L)
+    log.highWatermark = 12L
     assertEquals(Some(0L), log.firstUnstableOffset.map(_.messageOffset))
 
     log.close()
@@ -3638,7 +3643,7 @@ class LogTest {
     val reopenedLog = createLog(logDir, logConfig)
     assertEquals(12L, reopenedLog.logEndOffset)
     assertEquals(2, reopenedLog.activeSegment.txnIndex.allAbortedTxns.size)
-    reopenedLog.onHighWatermarkIncremented(12L)
+    reopenedLog.highWatermark = 12L
     assertEquals(None, reopenedLog.firstUnstableOffset.map(_.messageOffset))
   }
 
@@ -3680,7 +3685,7 @@ class LogTest {
     // now first producer's transaction is aborted
     val abortAppendInfo = log.appendAsLeader(endTxnRecords(ControlRecordType.ABORT, pid1, epoch),
       isFromClient = false, leaderEpoch = 0)
-    log.onHighWatermarkIncremented(abortAppendInfo.lastOffset + 1)
+    log.highWatermark = abortAppendInfo.lastOffset + 1
 
     // LSO should now point to one less than the first offset of the second transaction
     assertEquals(Some(secondAppendInfo.firstOffset.get), log.firstUnstableOffset.map(_.messageOffset))
@@ -3688,7 +3693,7 @@ class LogTest {
     // commit the second transaction
     val commitAppendInfo = log.appendAsLeader(endTxnRecords(ControlRecordType.COMMIT, pid2, epoch),
       isFromClient = false, leaderEpoch = 0)
-    log.onHighWatermarkIncremented(commitAppendInfo.lastOffset + 1)
+    log.highWatermark = commitAppendInfo.lastOffset + 1
 
     // now there should be no first unstable offset
     assertEquals(None, log.firstUnstableOffset)
@@ -3725,7 +3730,7 @@ class LogTest {
     // now abort the transaction
     val appendInfo = log.appendAsLeader(endTxnRecords(ControlRecordType.ABORT, pid, epoch),
       isFromClient = false, leaderEpoch = 0)
-    log.onHighWatermarkIncremented(appendInfo.lastOffset + 1)
+    log.highWatermark = appendInfo.lastOffset + 1
     assertEquals(None, log.firstUnstableOffset.map(_.messageOffset))
 
     // now check that a fetch includes the aborted transaction
diff --git a/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala b/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala
index 6f4979bac519f..3db3c22111194 100644
--- a/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala
@@ -211,7 +211,7 @@ class ProducerStateManagerTest {
     val producerAppendInfo = new ProducerAppendInfo(partition, producerId, ProducerStateEntry.empty(producerId), ValidationType.Full)
     producerAppendInfo.append(producerEpoch, seq, seq, time.milliseconds(), offset, offset, isTransactional = true)
 
-    val logOffsetMetadata = new LogOffsetMetadata(messageOffset = offset, segmentBaseOffset = 990000L,
+    val logOffsetMetadata = LogOffsetMetadata(messageOffset = offset, segmentBaseOffset = 990000L,
       relativePositionInSegment = 234224)
     producerAppendInfo.maybeCacheTxnFirstOffsetMetadata(logOffsetMetadata)
     stateManager.update(producerAppendInfo)
@@ -283,7 +283,7 @@ class ProducerStateManagerTest {
 
     // use some other offset to simulate a follower append where the log offset metadata won't typically
     // match any of the transaction first offsets
-    val logOffsetMetadata = new LogOffsetMetadata(messageOffset = offset - 23429, segmentBaseOffset = 990000L,
+    val logOffsetMetadata = LogOffsetMetadata(messageOffset = offset - 23429, segmentBaseOffset = 990000L,
       relativePositionInSegment = 234224)
     producerAppendInfo.maybeCacheTxnFirstOffsetMetadata(logOffsetMetadata)
     stateManager.update(producerAppendInfo)
diff --git a/core/src/test/scala/unit/kafka/server/HighwatermarkPersistenceTest.scala b/core/src/test/scala/unit/kafka/server/HighwatermarkPersistenceTest.scala
index 61c521b153d11..92820d62ced77 100755
--- a/core/src/test/scala/unit/kafka/server/HighwatermarkPersistenceTest.scala
+++ b/core/src/test/scala/unit/kafka/server/HighwatermarkPersistenceTest.scala
@@ -75,18 +75,17 @@ class HighwatermarkPersistenceTest {
       val partition0 = replicaManager.createPartition(tp0)
       // create leader and follower replicas
       val log0 = logManagers.head.getOrCreateLog(new TopicPartition(topic, 0), LogConfig())
-      val leaderReplicaPartition0 = new Replica(configs.head.brokerId, tp0, time, 0, Some(log0))
-      partition0.addReplicaIfNotExists(leaderReplicaPartition0)
-      val followerReplicaPartition0 = new Replica(configs.last.brokerId, tp0, time)
+      partition0.setLog(log0, isFutureLog = false)
+      val followerReplicaPartition0 = new Replica(configs.last.brokerId, tp0)
       partition0.addReplicaIfNotExists(followerReplicaPartition0)
       replicaManager.checkpointHighWatermarks()
       fooPartition0Hw = hwmFor(replicaManager, topic, 0)
-      assertEquals(leaderReplicaPartition0.highWatermark, fooPartition0Hw)
+      assertEquals(log0.highWatermark, fooPartition0Hw)
       // set the high watermark for local replica
-      partition0.localReplica.get.highWatermark = 5L
+      partition0.localLogOrException.highWatermark = 5L
       replicaManager.checkpointHighWatermarks()
       fooPartition0Hw = hwmFor(replicaManager, topic, 0)
-      assertEquals(leaderReplicaPartition0.highWatermark, fooPartition0Hw)
+      assertEquals(log0.highWatermark, fooPartition0Hw)
       EasyMock.verify(zkClient)
     } finally {
       // shutdown the replica manager upon test completion
@@ -121,16 +120,15 @@ class HighwatermarkPersistenceTest {
       // create leader log
       val topic1Log0 = logManagers.head.getOrCreateLog(t1p0, LogConfig())
       // create a local replica for topic1
-      val leaderReplicaTopic1Partition0 = new Replica(configs.head.brokerId, t1p0, time, 0, Some(topic1Log0))
-      topic1Partition0.addReplicaIfNotExists(leaderReplicaTopic1Partition0)
+      topic1Partition0.setLog(topic1Log0, isFutureLog = false)
       replicaManager.checkpointHighWatermarks()
       topic1Partition0Hw = hwmFor(replicaManager, topic1, 0)
-      assertEquals(leaderReplicaTopic1Partition0.highWatermark, topic1Partition0Hw)
+      assertEquals(topic1Log0.highWatermark, topic1Partition0Hw)
       // set the high watermark for local replica
-      topic1Partition0.localReplica.get.highWatermark = 5L
+      topic1Partition0.localLogOrException.highWatermark = 5L
       replicaManager.checkpointHighWatermarks()
       topic1Partition0Hw = hwmFor(replicaManager, topic1, 0)
-      assertEquals(5L, leaderReplicaTopic1Partition0.highWatermark)
+      assertEquals(5L, topic1Log0.highWatermark)
       assertEquals(5L, topic1Partition0Hw)
       // add another partition and set highwatermark
       val t2p0 = new TopicPartition(topic2, 0)
@@ -138,17 +136,16 @@ class HighwatermarkPersistenceTest {
       // create leader log
       val topic2Log0 = logManagers.head.getOrCreateLog(t2p0, LogConfig())
       // create a local replica for topic2
-      val leaderReplicaTopic2Partition0 =  new Replica(configs.head.brokerId, t2p0, time, 0, Some(topic2Log0))
-      topic2Partition0.addReplicaIfNotExists(leaderReplicaTopic2Partition0)
+      topic2Partition0.setLog(topic2Log0, isFutureLog = false)
       replicaManager.checkpointHighWatermarks()
       var topic2Partition0Hw = hwmFor(replicaManager, topic2, 0)
-      assertEquals(leaderReplicaTopic2Partition0.highWatermark, topic2Partition0Hw)
+      assertEquals(topic2Log0.highWatermark, topic2Partition0Hw)
       // set the highwatermark for local replica
-      topic2Partition0.localReplica.get.highWatermark = 15L
-      assertEquals(15L, leaderReplicaTopic2Partition0.highWatermark)
+      topic2Partition0.localLogOrException.highWatermark = 15L
+      assertEquals(15L, topic2Log0.highWatermark)
       // change the highwatermark for topic1
-      topic1Partition0.localReplica.get.highWatermark = 10L
-      assertEquals(10L, leaderReplicaTopic1Partition0.highWatermark)
+      topic1Partition0.localLogOrException.highWatermark = 10L
+      assertEquals(10L, topic1Log0.highWatermark)
       replicaManager.checkpointHighWatermarks()
       // verify checkpointed hw for topic 2
       topic2Partition0Hw = hwmFor(replicaManager, topic2, 0)
@@ -169,5 +166,4 @@ class HighwatermarkPersistenceTest {
     replicaManager.highWatermarkCheckpoints(new File(replicaManager.config.logDirs.head).getAbsolutePath).read.getOrElse(
       new TopicPartition(topic, partition), 0L)
   }
-
 }
diff --git a/core/src/test/scala/unit/kafka/server/ISRExpirationTest.scala b/core/src/test/scala/unit/kafka/server/ISRExpirationTest.scala
index 0ee0fa164d9d2..efdadbb311d24 100644
--- a/core/src/test/scala/unit/kafka/server/ISRExpirationTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ISRExpirationTest.scala
@@ -76,25 +76,24 @@ class IsrExpirationTest {
 
     // create one partition and all replicas
     val partition0 = getPartitionWithAllReplicasInIsr(topic, 0, time, configs.head, log)
-    assertEquals("All replicas should be in ISR", configs.map(_.brokerId).toSet, partition0.inSyncReplicas.map(_.brokerId))
-    val leaderReplica = partition0.getReplica(configs.head.brokerId).get
+    assertEquals("All replicas should be in ISR", configs.map(_.brokerId).toSet, partition0.inSyncReplicas)
 
     // let the follower catch up to the Leader logEndOffset - 1
-    for (replica <- partition0.assignedReplicas - leaderReplica)
+    for (replica <- partition0.remoteReplicas)
       replica.updateFetchState(
-        followerFetchOffsetMetadata = new LogOffsetMetadata(leaderLogEndOffset - 1),
+        followerFetchOffsetMetadata = LogOffsetMetadata(leaderLogEndOffset - 1),
         followerStartOffset = 0L,
         followerFetchTimeMs= time.milliseconds,
         leaderEndOffset = leaderLogEndOffset)
-    var partition0OSR = partition0.getOutOfSyncReplicas(leaderReplica, configs.head.replicaLagTimeMaxMs)
-    assertEquals("No replica should be out of sync", Set.empty[Int], partition0OSR.map(_.brokerId))
+    var partition0OSR = partition0.getOutOfSyncReplicas(configs.head.replicaLagTimeMaxMs)
+    assertEquals("No replica should be out of sync", Set.empty[Int], partition0OSR)
 
     // let some time pass
     time.sleep(150)
 
     // now follower hasn't pulled any data for > replicaMaxLagTimeMs ms. So it is stuck
-    partition0OSR = partition0.getOutOfSyncReplicas(leaderReplica, configs.head.replicaLagTimeMaxMs)
-    assertEquals("Replica 1 should be out of sync", Set(configs.last.brokerId), partition0OSR.map(_.brokerId))
+    partition0OSR = partition0.getOutOfSyncReplicas(configs.head.replicaLagTimeMaxMs)
+    assertEquals("Replica 1 should be out of sync", Set(configs.last.brokerId), partition0OSR)
     EasyMock.verify(log)
   }
 
@@ -107,14 +106,13 @@ class IsrExpirationTest {
 
     // create one partition and all replicas
     val partition0 = getPartitionWithAllReplicasInIsr(topic, 0, time, configs.head, log)
-    assertEquals("All replicas should be in ISR", configs.map(_.brokerId).toSet, partition0.inSyncReplicas.map(_.brokerId))
-    val leaderReplica = partition0.getReplica(configs.head.brokerId).get
+    assertEquals("All replicas should be in ISR", configs.map(_.brokerId).toSet, partition0.inSyncReplicas)
 
     // Let enough time pass for the replica to be considered stuck
     time.sleep(150)
 
-    val partition0OSR = partition0.getOutOfSyncReplicas(leaderReplica, configs.head.replicaLagTimeMaxMs)
-    assertEquals("Replica 1 should be out of sync", Set(configs.last.brokerId), partition0OSR.map(_.brokerId))
+    val partition0OSR = partition0.getOutOfSyncReplicas(configs.head.replicaLagTimeMaxMs)
+    assertEquals("Replica 1 should be out of sync", Set(configs.last.brokerId), partition0OSR)
     EasyMock.verify(log)
   }
 
@@ -128,50 +126,48 @@ class IsrExpirationTest {
     val log = logMock
     // add one partition
     val partition0 = getPartitionWithAllReplicasInIsr(topic, 0, time, configs.head, log)
-    assertEquals("All replicas should be in ISR", configs.map(_.brokerId).toSet, partition0.inSyncReplicas.map(_.brokerId))
-    val leaderReplica = partition0.getReplica(configs.head.brokerId).get
-
+    assertEquals("All replicas should be in ISR", configs.map(_.brokerId).toSet, partition0.inSyncReplicas)
     // Make the remote replica not read to the end of log. It should be not be out of sync for at least 100 ms
-    for (replica <- partition0.assignedReplicas - leaderReplica)
+    for (replica <- partition0.remoteReplicas)
       replica.updateFetchState(
-        followerFetchOffsetMetadata = new LogOffsetMetadata(leaderLogEndOffset - 2),
+        followerFetchOffsetMetadata = LogOffsetMetadata(leaderLogEndOffset - 2),
         followerStartOffset = 0L,
         followerFetchTimeMs= time.milliseconds,
         leaderEndOffset = leaderLogEndOffset)
 
     // Simulate 2 fetch requests spanning more than 100 ms which do not read to the end of the log.
     // The replicas will no longer be in ISR. We do 2 fetches because we want to simulate the case where the replica is lagging but is not stuck
-    var partition0OSR = partition0.getOutOfSyncReplicas(leaderReplica, configs.head.replicaLagTimeMaxMs)
-    assertEquals("No replica should be out of sync", Set.empty[Int], partition0OSR.map(_.brokerId))
+    var partition0OSR = partition0.getOutOfSyncReplicas(configs.head.replicaLagTimeMaxMs)
+    assertEquals("No replica should be out of sync", Set.empty[Int], partition0OSR)
 
     time.sleep(75)
 
-    (partition0.assignedReplicas - leaderReplica).foreach { r =>
+    partition0.remoteReplicas.foreach { r =>
       r.updateFetchState(
-        followerFetchOffsetMetadata = new LogOffsetMetadata(leaderLogEndOffset - 1),
+        followerFetchOffsetMetadata = LogOffsetMetadata(leaderLogEndOffset - 1),
         followerStartOffset = 0L,
         followerFetchTimeMs= time.milliseconds,
         leaderEndOffset = leaderLogEndOffset)
     }
-    partition0OSR = partition0.getOutOfSyncReplicas(leaderReplica, configs.head.replicaLagTimeMaxMs)
-    assertEquals("No replica should be out of sync", Set.empty[Int], partition0OSR.map(_.brokerId))
+    partition0OSR = partition0.getOutOfSyncReplicas(configs.head.replicaLagTimeMaxMs)
+    assertEquals("No replica should be out of sync", Set.empty[Int], partition0OSR)
 
     time.sleep(75)
 
     // The replicas will no longer be in ISR
-    partition0OSR = partition0.getOutOfSyncReplicas(leaderReplica, configs.head.replicaLagTimeMaxMs)
-    assertEquals("Replica 1 should be out of sync", Set(configs.last.brokerId), partition0OSR.map(_.brokerId))
+    partition0OSR = partition0.getOutOfSyncReplicas(configs.head.replicaLagTimeMaxMs)
+    assertEquals("Replica 1 should be out of sync", Set(configs.last.brokerId), partition0OSR)
 
     // Now actually make a fetch to the end of the log. The replicas should be back in ISR
-    (partition0.assignedReplicas - leaderReplica).foreach { r =>
+    partition0.remoteReplicas.foreach { r =>
       r.updateFetchState(
-        followerFetchOffsetMetadata = new LogOffsetMetadata(leaderLogEndOffset),
+        followerFetchOffsetMetadata = LogOffsetMetadata(leaderLogEndOffset),
         followerStartOffset = 0L,
         followerFetchTimeMs= time.milliseconds,
         leaderEndOffset = leaderLogEndOffset)
     }
-    partition0OSR = partition0.getOutOfSyncReplicas(leaderReplica, configs.head.replicaLagTimeMaxMs)
-    assertEquals("No replica should be out of sync", Set.empty[Int], partition0OSR.map(_.brokerId))
+    partition0OSR = partition0.getOutOfSyncReplicas(configs.head.replicaLagTimeMaxMs)
+    assertEquals("No replica should be out of sync", Set.empty[Int], partition0OSR)
 
     EasyMock.verify(log)
   }
@@ -185,26 +181,25 @@ class IsrExpirationTest {
 
     // create one partition and all replicas
     val partition0 = getPartitionWithAllReplicasInIsr(topic, 0, time, configs.head, log)
-    assertEquals("All replicas should be in ISR", configs.map(_.brokerId).toSet, partition0.inSyncReplicas.map(_.brokerId))
-    val leaderReplica = partition0.getReplica(configs.head.brokerId).get
+    assertEquals("All replicas should be in ISR", configs.map(_.brokerId).toSet, partition0.inSyncReplicas)
 
     // let the follower catch up to the Leader logEndOffset
-    for (replica <- partition0.assignedReplicas - leaderReplica)
+    for (replica <- partition0.remoteReplicas)
       replica.updateFetchState(
-        followerFetchOffsetMetadata = new LogOffsetMetadata(leaderLogEndOffset),
+        followerFetchOffsetMetadata = LogOffsetMetadata(leaderLogEndOffset),
         followerStartOffset = 0L,
         followerFetchTimeMs= time.milliseconds,
         leaderEndOffset = leaderLogEndOffset)
 
-    var partition0OSR = partition0.getOutOfSyncReplicas(leaderReplica, configs.head.replicaLagTimeMaxMs)
-    assertEquals("No replica should be out of sync", Set.empty[Int], partition0OSR.map(_.brokerId))
+    var partition0OSR = partition0.getOutOfSyncReplicas(configs.head.replicaLagTimeMaxMs)
+    assertEquals("No replica should be out of sync", Set.empty[Int], partition0OSR)
 
     // let some time pass
     time.sleep(150)
 
     // even though follower hasn't pulled any data for > replicaMaxLagTimeMs ms, the follower has already caught up. So it is not out-of-sync.
-    partition0OSR = partition0.getOutOfSyncReplicas(leaderReplica, configs.head.replicaLagTimeMaxMs)
-    assertEquals("No replica should be out of sync", Set.empty[Int], partition0OSR.map(_.brokerId))
+    partition0OSR = partition0.getOutOfSyncReplicas(configs.head.replicaLagTimeMaxMs)
+    assertEquals("No replica should be out of sync", Set.empty[Int], partition0OSR)
     EasyMock.verify(log)
   }
 
@@ -213,16 +208,16 @@ class IsrExpirationTest {
     val leaderId = config.brokerId
     val tp = new TopicPartition(topic, partitionId)
     val partition = replicaManager.createPartition(tp)
-    val leaderReplica = new Replica(leaderId, tp, time, 0, Some(localLog))
+    partition.setLog(localLog, isFutureLog = false)
 
-    val allReplicas = getFollowerReplicas(partition, leaderId, time) :+ leaderReplica
+    val allReplicas = getFollowerReplicas(partition, leaderId, time)
     allReplicas.foreach(r => partition.addReplicaIfNotExists(r))
     // set in sync replicas for this partition to all the assigned replicas
-    partition.inSyncReplicas = allReplicas.toSet
+    partition.inSyncReplicas = allReplicas.map(_.brokerId).toSet + leaderId
     // set lastCaughtUpTime to current time
-    for (replica <- partition.assignedReplicas - leaderReplica)
+    for (replica <- partition.remoteReplicas)
       replica.updateFetchState(
-        followerFetchOffsetMetadata = new LogOffsetMetadata(0L),
+        followerFetchOffsetMetadata = LogOffsetMetadata(0L),
         followerStartOffset = 0L,
         followerFetchTimeMs= time.milliseconds,
         leaderEndOffset = 0L)
@@ -235,15 +230,15 @@ class IsrExpirationTest {
   private def logMock: Log = {
     val log: Log = EasyMock.createMock(classOf[Log])
     EasyMock.expect(log.dir).andReturn(TestUtils.tempDir()).anyTimes()
-    EasyMock.expect(log.onHighWatermarkIncremented(0L))
     EasyMock.expect(log.logEndOffsetMetadata).andReturn(LogOffsetMetadata(leaderLogEndOffset)).anyTimes()
+    EasyMock.expect(log.logEndOffset).andReturn(leaderLogEndOffset).anyTimes()
     EasyMock.replay(log)
     log
   }
 
   private def getFollowerReplicas(partition: Partition, leaderId: Int, time: Time): Seq[Replica] = {
     configs.filter(_.brokerId != leaderId).map { config =>
-      new Replica(config.brokerId, partition.topicPartition, time)
+      new Replica(config.brokerId, partition.topicPartition)
     }
   }
 }
diff --git a/core/src/test/scala/unit/kafka/server/LogDirFailureTest.scala b/core/src/test/scala/unit/kafka/server/LogDirFailureTest.scala
index f83562c2c1e76..03ce3a5a34dd4 100644
--- a/core/src/test/scala/unit/kafka/server/LogDirFailureTest.scala
+++ b/core/src/test/scala/unit/kafka/server/LogDirFailureTest.scala
@@ -113,7 +113,7 @@ class LogDirFailureTest extends IntegrationTestHarness {
     // so that ReplicaFetcherThread on the follower will get response from leader immediately
     val anotherPartitionWithTheSameLeader = (1 until partitionNum).find { i =>
       leaderServer.replicaManager.nonOfflinePartition(new TopicPartition(topic, i))
-        .flatMap(_.leaderReplicaIfLocal).isDefined
+        .flatMap(_.leaderLogIfLocal).isDefined
     }.get
     val record = new ProducerRecord[Array[Byte], Array[Byte]](topic, anotherPartitionWithTheSameLeader, topic.getBytes, "message".getBytes)
     // When producer.send(...).get returns, it is guaranteed that ReplicaFetcherThread on the follower
@@ -145,8 +145,8 @@ class LogDirFailureTest extends IntegrationTestHarness {
     TestUtils.consumeRecords(consumer, 1)
 
     // Make log directory of the partition on the leader broker inaccessible by replacing it with a file
-    val replica = leaderServer.replicaManager.localReplicaOrException(partition)
-    val logDir = replica.log.get.dir.getParentFile
+    val localLog = leaderServer.replicaManager.localLogOrException(partition)
+    val logDir = localLog.dir.getParentFile
     CoreUtils.swallow(Utils.delete(logDir), this)
     logDir.createNewFile()
     assertTrue(logDir.isFile)
@@ -164,7 +164,7 @@ class LogDirFailureTest extends IntegrationTestHarness {
 
     // Wait for ReplicaHighWatermarkCheckpoint to happen so that the log directory of the topic will be offline
     TestUtils.waitUntilTrue(() => !leaderServer.logManager.isLogDirOnline(logDir.getAbsolutePath), "Expected log directory offline", 3000L)
-    assertTrue(leaderServer.replicaManager.localReplica(partition).isEmpty)
+    assertTrue(leaderServer.replicaManager.localLog(partition).isEmpty)
 
     // The second send() should fail due to either KafkaStorageException or NotLeaderForPartitionException
     try {
diff --git a/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala b/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala
index 7b52e7b345300..46fcee5272605 100755
--- a/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala
+++ b/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala
@@ -78,7 +78,8 @@ class LogOffsetTest extends BaseRequestTest {
       log.appendAsLeader(TestUtils.singletonRecords(value = Integer.toString(42).getBytes()), leaderEpoch = 0)
     log.flush()
 
-    log.onHighWatermarkIncremented(log.logEndOffset)
+    log.highWatermarkMetadata = LogOffsetMetadata(log.logEndOffset)
+    log.highWatermark = log.logEndOffset
     log.maybeIncrementLogStartOffset(3)
     log.deleteOldSegments()
 
diff --git a/core/src/test/scala/unit/kafka/server/LogRecoveryTest.scala b/core/src/test/scala/unit/kafka/server/LogRecoveryTest.scala
index e868f6c4a8b19..ee7472fa59d7c 100755
--- a/core/src/test/scala/unit/kafka/server/LogRecoveryTest.scala
+++ b/core/src/test/scala/unit/kafka/server/LogRecoveryTest.scala
@@ -104,7 +104,7 @@ class LogRecoveryTest extends ZooKeeperTestHarness {
 
     // give some time for the follower 1 to record leader HW
     TestUtils.waitUntilTrue(() =>
-      server2.replicaManager.localReplica(topicPartition).get.highWatermark == numMessages,
+      server2.replicaManager.localLogOrException(topicPartition).highWatermark == numMessages,
       "Failed to update high watermark for follower after timeout")
 
     servers.foreach(_.replicaManager.checkpointHighWatermarks())
@@ -166,7 +166,7 @@ class LogRecoveryTest extends ZooKeeperTestHarness {
 
     // give some time for follower 1 to record leader HW of 60
     TestUtils.waitUntilTrue(() =>
-      server2.replicaManager.localReplica(topicPartition).get.highWatermark == hw,
+      server2.replicaManager.localLogOrException(topicPartition).highWatermark == hw,
       "Failed to update high watermark for follower after timeout")
     // shutdown the servers to allow the hw to be checkpointed
     servers.foreach(_.shutdown())
@@ -180,7 +180,7 @@ class LogRecoveryTest extends ZooKeeperTestHarness {
     val hw = 20L
     // give some time for follower 1 to record leader HW of 600
     TestUtils.waitUntilTrue(() =>
-      server2.replicaManager.localReplica(topicPartition).get.highWatermark == hw,
+      server2.replicaManager.localLogOrException(topicPartition).highWatermark == hw,
       "Failed to update high watermark for follower after timeout")
     // shutdown the servers to allow the hw to be checkpointed
     servers.foreach(_.shutdown())
@@ -199,7 +199,7 @@ class LogRecoveryTest extends ZooKeeperTestHarness {
 
     // allow some time for the follower to get the leader HW
     TestUtils.waitUntilTrue(() =>
-      server2.replicaManager.localReplica(topicPartition).get.highWatermark == hw,
+      server2.replicaManager.localLogOrException(topicPartition).highWatermark == hw,
       "Failed to update high watermark for follower after timeout")
     // kill the server hosting the preferred replica
     server1.shutdown()
@@ -226,11 +226,11 @@ class LogRecoveryTest extends ZooKeeperTestHarness {
     hw += 2
 
     // allow some time for the follower to create replica
-    TestUtils.waitUntilTrue(() => server1.replicaManager.localReplica(topicPartition).nonEmpty,
+    TestUtils.waitUntilTrue(() => server1.replicaManager.localLog(topicPartition).nonEmpty,
       "Failed to create replica in follower after timeout")
     // allow some time for the follower to get the leader HW
     TestUtils.waitUntilTrue(() =>
-      server1.replicaManager.localReplica(topicPartition).get.highWatermark == hw,
+      server1.replicaManager.localLogOrException(topicPartition).highWatermark == hw,
       "Failed to update high watermark for follower after timeout")
     // shutdown the servers to allow the hw to be checkpointed
     servers.foreach(_.shutdown())
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala
index d149b8aea599f..6607272b9c0ea 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala
@@ -18,8 +18,8 @@ package kafka.server
 
 import java.util.Optional
 
-import kafka.cluster.{BrokerEndPoint, Partition, Replica}
-import kafka.log.LogManager
+import kafka.cluster.{BrokerEndPoint, Partition}
+import kafka.log.{Log, LogManager}
 import kafka.server.AbstractFetcherThread.ResultWithPartitions
 import kafka.utils.{DelayedItem, TestUtils}
 import org.apache.kafka.common.TopicPartition
@@ -28,7 +28,7 @@ import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.requests.{EpochEndOffset, OffsetsForLeaderEpochRequest}
 import org.apache.kafka.common.requests.EpochEndOffset.{UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET}
 import org.easymock.EasyMock._
-import org.easymock.{Capture, CaptureType, EasyMock, IAnswer}
+import org.easymock.{Capture, CaptureType, EasyMock, IAnswer, IExpectationSetters}
 import org.junit.Assert._
 import org.junit.Test
 
@@ -153,11 +153,11 @@ class ReplicaAlterLogDirsThreadTest {
     val config = KafkaConfig.fromProps(TestUtils.createBrokerConfig(1, "localhost:1234"))
     val quotaManager: ReplicationQuotaManager = createNiceMock(classOf[ReplicationQuotaManager])
     val logManager: LogManager = createMock(classOf[LogManager])
-    val replicaT1p0: Replica = createNiceMock(classOf[Replica])
-    val replicaT1p1: Replica = createNiceMock(classOf[Replica])
+    val logT1p0: Log = createNiceMock(classOf[Log])
+    val logT1p1: Log = createNiceMock(classOf[Log])
     // one future replica mock because our mocking methods return same values for both future replicas
-    val futureReplicaT1p0: Replica = createNiceMock(classOf[Replica])
-    val futureReplicaT1p1: Replica = createNiceMock(classOf[Replica])
+    val futureLogT1p0: Log = createNiceMock(classOf[Log])
+    val futureLogT1p1: Log = createNiceMock(classOf[Log])
     val partitionT1p0: Partition = createMock(classOf[Partition])
     val partitionT1p1: Partition = createMock(classOf[Partition])
     val replicaManager: ReplicaManager = createMock(classOf[ReplicaManager])
@@ -173,33 +173,32 @@ class ReplicaAlterLogDirsThreadTest {
       .andStubReturn(partitionT1p0)
     expect(replicaManager.getPartitionOrException(t1p1, expectLeader = false))
       .andStubReturn(partitionT1p1)
-    expect(replicaManager.futureLocalReplicaOrException(t1p0)).andStubReturn(futureReplicaT1p0)
-    expect(replicaManager.futureLocalReplicaOrException(t1p1)).andStubReturn(futureReplicaT1p1)
+    expect(replicaManager.futureLocalLogOrException(t1p0)).andStubReturn(futureLogT1p0)
+    expect(replicaManager.futureLocalLogOrException(t1p1)).andStubReturn(futureLogT1p1)
     expect(partitionT1p0.truncateTo(capture(truncateCaptureT1p0), anyBoolean())).anyTimes()
     expect(partitionT1p1.truncateTo(capture(truncateCaptureT1p1), anyBoolean())).anyTimes()
 
-    expect(futureReplicaT1p0.logEndOffset).andReturn(futureReplicaLEO).anyTimes()
-    expect(futureReplicaT1p1.logEndOffset).andReturn(futureReplicaLEO).anyTimes()
+    expect(futureLogT1p0.logEndOffset).andReturn(futureReplicaLEO).anyTimes()
+    expect(futureLogT1p1.logEndOffset).andReturn(futureReplicaLEO).anyTimes()
 
-    expect(futureReplicaT1p0.latestEpoch).andReturn(Some(leaderEpoch)).anyTimes()
-    expect(futureReplicaT1p0.endOffsetForEpoch(leaderEpoch)).andReturn(
+    expect(futureLogT1p0.latestEpoch).andReturn(Some(leaderEpoch)).anyTimes()
+    expect(futureLogT1p0.endOffsetForEpoch(leaderEpoch)).andReturn(
       Some(OffsetAndEpoch(futureReplicaLEO, leaderEpoch))).anyTimes()
     expect(partitionT1p0.lastOffsetForLeaderEpoch(Optional.of(1), leaderEpoch, fetchOnlyFromLeader = false))
       .andReturn(new EpochEndOffset(leaderEpoch, replicaT1p0LEO))
       .anyTimes()
 
-    expect(futureReplicaT1p1.latestEpoch).andReturn(Some(leaderEpoch)).anyTimes()
-    expect(futureReplicaT1p1.endOffsetForEpoch(leaderEpoch)).andReturn(
+    expect(futureLogT1p1.latestEpoch).andReturn(Some(leaderEpoch)).anyTimes()
+    expect(futureLogT1p1.endOffsetForEpoch(leaderEpoch)).andReturn(
       Some(OffsetAndEpoch(futureReplicaLEO, leaderEpoch))).anyTimes()
     expect(partitionT1p1.lastOffsetForLeaderEpoch(Optional.of(1), leaderEpoch, fetchOnlyFromLeader = false))
       .andReturn(new EpochEndOffset(leaderEpoch, replicaT1p1LEO))
       .anyTimes()
 
     expect(replicaManager.logManager).andReturn(logManager).anyTimes()
-    stubWithFetchMessages(replicaT1p0, replicaT1p1, futureReplicaT1p0, partitionT1p0, replicaManager, responseCallback)
+    stubWithFetchMessages(logT1p0, logT1p1, futureLogT1p0, partitionT1p0, replicaManager, responseCallback)
 
-    replay(replicaManager, logManager, quotaManager, replicaT1p0, replicaT1p1,
-      futureReplicaT1p0, partitionT1p0, partitionT1p1)
+    replay(replicaManager, logManager, quotaManager, partitionT1p0, partitionT1p1, logT1p0, logT1p1, futureLogT1p0, futureLogT1p1)
 
     //Create the thread
     val endPoint = new BrokerEndPoint(0, "localhost", 1000)
@@ -231,9 +230,9 @@ class ReplicaAlterLogDirsThreadTest {
     val config = KafkaConfig.fromProps(TestUtils.createBrokerConfig(1, "localhost:1234"))
     val quotaManager: ReplicationQuotaManager = createNiceMock(classOf[ReplicationQuotaManager])
     val logManager: LogManager = createMock(classOf[LogManager])
-    val replica: Replica = createNiceMock(classOf[Replica])
+    val log: Log = createNiceMock(classOf[Log])
     // one future replica mock because our mocking methods return same values for both future replicas
-    val futureReplica: Replica = createNiceMock(classOf[Replica])
+    val futureLog: Log = createNiceMock(classOf[Log])
     val partition: Partition = createMock(classOf[Partition])
     val replicaManager: ReplicaManager = createMock(classOf[ReplicaManager])
     val responseCallback: Capture[Seq[(TopicPartition, FetchPartitionData)] => Unit]  = EasyMock.newCapture()
@@ -247,31 +246,31 @@ class ReplicaAlterLogDirsThreadTest {
     //Stubs
     expect(replicaManager.getPartitionOrException(t1p0, expectLeader = false))
       .andStubReturn(partition)
-    expect(replicaManager.futureLocalReplicaOrException(t1p0)).andStubReturn(futureReplica)
+    expect(replicaManager.futureLocalLogOrException(t1p0)).andStubReturn(futureLog)
 
     expect(partition.truncateTo(capture(truncateToCapture), EasyMock.eq(true))).anyTimes()
-    expect(futureReplica.logEndOffset).andReturn(futureReplicaLEO).anyTimes()
-    expect(futureReplica.latestEpoch).andReturn(Some(leaderEpoch)).once()
-    expect(futureReplica.latestEpoch).andReturn(Some(leaderEpoch - 2)).once()
+    expect(futureLog.logEndOffset).andReturn(futureReplicaLEO).anyTimes()
+    expect(futureLog.latestEpoch).andReturn(Some(leaderEpoch)).once()
+    expect(futureLog.latestEpoch).andReturn(Some(leaderEpoch - 2)).once()
 
     // leader replica truncated and fetched new offsets with new leader epoch
     expect(partition.lastOffsetForLeaderEpoch(Optional.of(1), leaderEpoch, fetchOnlyFromLeader = false))
       .andReturn(new EpochEndOffset(leaderEpoch - 1, replicaLEO))
       .anyTimes()
     // but future replica does not know about this leader epoch, so returns a smaller leader epoch
-    expect(futureReplica.endOffsetForEpoch(leaderEpoch - 1)).andReturn(
+    expect(futureLog.endOffsetForEpoch(leaderEpoch - 1)).andReturn(
       Some(OffsetAndEpoch(futureReplicaLEO, leaderEpoch - 2))).anyTimes()
     // finally, the leader replica knows about the leader epoch and returns end offset
     expect(partition.lastOffsetForLeaderEpoch(Optional.of(1), leaderEpoch - 2, fetchOnlyFromLeader = false))
       .andReturn(new EpochEndOffset(leaderEpoch - 2, replicaEpochEndOffset))
       .anyTimes()
-    expect(futureReplica.endOffsetForEpoch(leaderEpoch - 2)).andReturn(
+    expect(futureLog.endOffsetForEpoch(leaderEpoch - 2)).andReturn(
       Some(OffsetAndEpoch(futureReplicaEpochEndOffset, leaderEpoch - 2))).anyTimes()
 
     expect(replicaManager.logManager).andReturn(logManager).anyTimes()
-    stubWithFetchMessages(replica, replica, futureReplica, partition, replicaManager, responseCallback)
+    stubWithFetchMessages(log, null, futureLog, partition, replicaManager, responseCallback)
 
-    replay(replicaManager, logManager, quotaManager, replica, futureReplica, partition)
+    replay(replicaManager, logManager, quotaManager, partition, log, futureLog)
 
     //Create the thread
     val endPoint = new BrokerEndPoint(0, "localhost", 1000)
@@ -305,8 +304,8 @@ class ReplicaAlterLogDirsThreadTest {
     val config = KafkaConfig.fromProps(TestUtils.createBrokerConfig(1, "localhost:1234"))
     val quotaManager: ReplicationQuotaManager = createNiceMock(classOf[ReplicationQuotaManager])
     val logManager: LogManager = createMock(classOf[LogManager])
-    val replica: Replica = createNiceMock(classOf[Replica])
-    val futureReplica: Replica = createNiceMock(classOf[Replica])
+    val log: Log = createNiceMock(classOf[Log])
+    val futureLog: Log = createNiceMock(classOf[Log])
     val partition: Partition = createMock(classOf[Partition])
     val replicaManager: ReplicaManager = createMock(classOf[ReplicaManager])
     val responseCallback: Capture[Seq[(TopicPartition, FetchPartitionData)] => Unit]  = EasyMock.newCapture()
@@ -318,16 +317,15 @@ class ReplicaAlterLogDirsThreadTest {
     expect(replicaManager.getPartitionOrException(t1p0, expectLeader = false))
       .andStubReturn(partition)
     expect(partition.truncateTo(capture(truncated), isFuture = EasyMock.eq(true))).anyTimes()
-    expect(replicaManager.futureLocalReplicaOrException(t1p0)).andStubReturn(futureReplica)
+    expect(replicaManager.futureLocalLogOrException(t1p0)).andStubReturn(futureLog)
 
-    expect(futureReplica.logEndOffset).andReturn(futureReplicaLEO).anyTimes()
     expect(replicaManager.logManager).andReturn(logManager).anyTimes()
 
     // pretend this is a completely new future replica, with no leader epochs recorded
-    expect(futureReplica.latestEpoch).andReturn(None).anyTimes()
+    expect(futureLog.latestEpoch).andReturn(None).anyTimes()
 
-    stubWithFetchMessages(replica, replica, futureReplica, partition, replicaManager, responseCallback)
-    replay(replicaManager, logManager, quotaManager, replica, futureReplica, partition)
+    stubWithFetchMessages(log, null, futureLog, partition, replicaManager, responseCallback)
+    replay(replicaManager, logManager, quotaManager, partition, log, futureLog)
 
     //Create the thread
     val endPoint = new BrokerEndPoint(0, "localhost", 1000)
@@ -359,8 +357,8 @@ class ReplicaAlterLogDirsThreadTest {
     val config = KafkaConfig.fromProps(TestUtils.createBrokerConfig(1, "localhost:1234"))
     val quotaManager: ReplicationQuotaManager = createNiceMock(classOf[ReplicationQuotaManager])
     val logManager: LogManager = createMock(classOf[LogManager])
-    val replica: Replica = createNiceMock(classOf[Replica])
-    val futureReplica: Replica = createNiceMock(classOf[Replica])
+    val log: Log = createNiceMock(classOf[Log])
+    val futureLog: Log = createNiceMock(classOf[Log])
     val partition: Partition = createMock(classOf[Partition])
     val replicaManager: ReplicaManager = createMock(classOf[ReplicaManager])
     val responseCallback: Capture[Seq[(TopicPartition, FetchPartitionData)] => Unit]  = EasyMock.newCapture()
@@ -374,14 +372,13 @@ class ReplicaAlterLogDirsThreadTest {
       .andStubReturn(partition)
     expect(partition.truncateTo(capture(truncated), isFuture = EasyMock.eq(true))).once()
 
-    expect(replicaManager.futureLocalReplicaOrException(t1p0)).andStubReturn(futureReplica)
-    expect(futureReplica.latestEpoch).andStubReturn(Some(futureReplicaLeaderEpoch))
-    expect(futureReplica.endOffsetForEpoch(futureReplicaLeaderEpoch)).andReturn(
+    expect(replicaManager.futureLocalLogOrException(t1p0)).andStubReturn(futureLog)
+    expect(futureLog.logEndOffset).andReturn(futureReplicaLEO).anyTimes()
+    expect(futureLog.latestEpoch).andStubReturn(Some(futureReplicaLeaderEpoch))
+    expect(futureLog.endOffsetForEpoch(futureReplicaLeaderEpoch)).andReturn(
       Some(OffsetAndEpoch(futureReplicaLEO, futureReplicaLeaderEpoch)))
-    expect(futureReplica.logEndOffset).andReturn(futureReplicaLEO).anyTimes()
-    expect(replicaManager.localReplica(t1p0)).andReturn(Some(replica)).anyTimes()
-    expect(replicaManager.futureLocalReplica(t1p0)).andReturn(Some(futureReplica)).anyTimes()
-    expect(replicaManager.futureLocalReplicaOrException(t1p0)).andReturn(futureReplica).anyTimes()
+    expect(replicaManager.localLog(t1p0)).andReturn(Some(log)).anyTimes()
+    expect(replicaManager.futureLocalLogOrException(t1p0)).andReturn(futureLog).anyTimes()
 
     // this will cause fetchEpochsFromLeader return an error with undefined offset
     expect(partition.lastOffsetForLeaderEpoch(Optional.of(1), futureReplicaLeaderEpoch, fetchOnlyFromLeader = false))
@@ -406,7 +403,7 @@ class ReplicaAlterLogDirsThreadTest {
         }
       }).anyTimes()
 
-    replay(replicaManager, logManager, quotaManager, replica, futureReplica, partition)
+    replay(replicaManager, logManager, quotaManager, partition, log, futureLog)
 
     //Create the thread
     val endPoint = new BrokerEndPoint(0, "localhost", 1000)
@@ -442,8 +439,8 @@ class ReplicaAlterLogDirsThreadTest {
     val config = KafkaConfig.fromProps(TestUtils.createBrokerConfig(1, "localhost:1234"))
     val quotaManager: ReplicationQuotaManager = createNiceMock(classOf[ReplicationQuotaManager])
     val logManager: LogManager = createMock(classOf[LogManager])
-    val replica: Replica = createNiceMock(classOf[Replica])
-    val futureReplica: Replica = createNiceMock(classOf[Replica])
+    val log: Log = createNiceMock(classOf[Log])
+    val futureLog: Log = createNiceMock(classOf[Log])
     val partition: Partition = createMock(classOf[Partition])
     val replicaManager: ReplicaManager = createMock(classOf[ReplicaManager])
     val responseCallback: Capture[Seq[(TopicPartition, FetchPartitionData)] => Unit]  = EasyMock.newCapture()
@@ -458,15 +455,15 @@ class ReplicaAlterLogDirsThreadTest {
         .andReturn(new EpochEndOffset(leaderEpoch, replicaLEO))
     expect(partition.truncateTo(futureReplicaLEO, isFuture = true)).once()
 
-    expect(replicaManager.futureLocalReplicaOrException(t1p0)).andStubReturn(futureReplica)
-    expect(futureReplica.latestEpoch).andStubReturn(Some(leaderEpoch))
-    expect(futureReplica.logEndOffset).andReturn(futureReplicaLEO).anyTimes()
-    expect(futureReplica.endOffsetForEpoch(leaderEpoch)).andReturn(
+    expect(replicaManager.futureLocalLogOrException(t1p0)).andStubReturn(futureLog)
+    expect(futureLog.latestEpoch).andStubReturn(Some(leaderEpoch))
+    expect(futureLog.logEndOffset).andStubReturn(futureReplicaLEO)
+    expect(futureLog.endOffsetForEpoch(leaderEpoch)).andReturn(
       Some(OffsetAndEpoch(futureReplicaLEO, leaderEpoch)))
     expect(replicaManager.logManager).andReturn(logManager).anyTimes()
-    stubWithFetchMessages(replica, replica, futureReplica, partition, replicaManager, responseCallback)
+    stubWithFetchMessages(log, null, futureLog, partition, replicaManager, responseCallback)
 
-    replay(replicaManager, logManager, quotaManager, replica, futureReplica, partition)
+    replay(replicaManager, logManager, quotaManager, partition, log, futureLog)
 
     //Create the fetcher thread
     val endPoint = new BrokerEndPoint(0, "localhost", 1000)
@@ -496,17 +493,16 @@ class ReplicaAlterLogDirsThreadTest {
     val config = KafkaConfig.fromProps(TestUtils.createBrokerConfig(1, "localhost:1234"))
     val quotaManager: ReplicationQuotaManager = createNiceMock(classOf[ReplicationQuotaManager])
     val logManager: LogManager = createMock(classOf[LogManager])
-    val replica: Replica = createNiceMock(classOf[Replica])
-    val futureReplica: Replica = createNiceMock(classOf[Replica])
+    val log: Log = createNiceMock(classOf[Log])
+    val futureLog: Log = createNiceMock(classOf[Log])
     val partition: Partition = createMock(classOf[Partition])
     val replicaManager: ReplicaManager = createMock(classOf[ReplicaManager])
 
     //Stubs
-    expect(futureReplica.logStartOffset).andReturn(123).anyTimes()
     expect(replicaManager.logManager).andReturn(logManager).anyTimes()
-    stub(replica, replica, futureReplica, partition, replicaManager)
+    stub(log, null, futureLog, partition, replicaManager)
 
-    replay(replicaManager, logManager, quotaManager, replica, futureReplica, partition)
+    replay(replicaManager, logManager, quotaManager, partition, log)
 
     //Create the fetcher thread
     val endPoint = new BrokerEndPoint(0, "localhost", 1000)
@@ -546,17 +542,18 @@ class ReplicaAlterLogDirsThreadTest {
     val config = KafkaConfig.fromProps(TestUtils.createBrokerConfig(1, "localhost:1234"))
     val quotaManager: ReplicationQuotaManager = createNiceMock(classOf[ReplicationQuotaManager])
     val logManager: LogManager = createMock(classOf[LogManager])
-    val replica: Replica = createNiceMock(classOf[Replica])
-    val futureReplica: Replica = createNiceMock(classOf[Replica])
+    val log: Log = createNiceMock(classOf[Log])
+    val futureLog: Log = createNiceMock(classOf[Log])
     val partition: Partition = createMock(classOf[Partition])
     val replicaManager: ReplicaManager = createMock(classOf[ReplicaManager])
 
     //Stubs
-    expect(futureReplica.logStartOffset).andReturn(123).anyTimes()
+    val startOffset = 123
+    expect(futureLog.logStartOffset).andReturn(startOffset).anyTimes()
     expect(replicaManager.logManager).andReturn(logManager).anyTimes()
-    stub(replica, replica, futureReplica, partition, replicaManager)
+    stub(log, null, futureLog, partition, replicaManager)
 
-    replay(replicaManager, logManager, quotaManager, replica, futureReplica, partition)
+    replay(replicaManager, logManager, quotaManager, partition, log, futureLog)
 
     //Create the fetcher thread
     val endPoint = new BrokerEndPoint(0, "localhost", 1000)
@@ -609,23 +606,21 @@ class ReplicaAlterLogDirsThreadTest {
     assertFalse(partitionsWithError3.nonEmpty)
   }
 
-  def stub(replicaT1p0: Replica, replicaT1p1: Replica, futureReplica: Replica, partition: Partition, replicaManager: ReplicaManager) = {
-    expect(replicaManager.localReplica(t1p0)).andReturn(Some(replicaT1p0)).anyTimes()
-    expect(replicaManager.futureLocalReplica(t1p0)).andReturn(Some(futureReplica)).anyTimes()
-    expect(replicaManager.localReplicaOrException(t1p0)).andReturn(replicaT1p0).anyTimes()
-    expect(replicaManager.futureLocalReplicaOrException(t1p0)).andReturn(futureReplica).anyTimes()
+  def stub(logT1p0: Log, logT1p1: Log, futureLog: Log, partition: Partition,
+           replicaManager: ReplicaManager): IExpectationSetters[Option[Partition]] = {
+    expect(replicaManager.localLog(t1p0)).andReturn(Some(logT1p0)).anyTimes()
+    expect(replicaManager.localLogOrException(t1p0)).andReturn(logT1p0).anyTimes()
+    expect(replicaManager.futureLocalLogOrException(t1p0)).andReturn(futureLog).anyTimes()
     expect(replicaManager.nonOfflinePartition(t1p0)).andReturn(Some(partition)).anyTimes()
-    expect(replicaManager.localReplica(t1p1)).andReturn(Some(replicaT1p1)).anyTimes()
-    expect(replicaManager.futureLocalReplica(t1p1)).andReturn(Some(futureReplica)).anyTimes()
-    expect(replicaManager.localReplicaOrException(t1p1)).andReturn(replicaT1p1).anyTimes()
-    expect(replicaManager.futureLocalReplicaOrException(t1p1)).andReturn(futureReplica).anyTimes()
+    expect(replicaManager.localLog(t1p1)).andReturn(Some(logT1p1)).anyTimes()
+    expect(replicaManager.localLogOrException(t1p1)).andReturn(logT1p1).anyTimes()
+    expect(replicaManager.futureLocalLogOrException(t1p1)).andReturn(futureLog).anyTimes()
     expect(replicaManager.nonOfflinePartition(t1p1)).andReturn(Some(partition)).anyTimes()
   }
 
-  def stubWithFetchMessages(replicaT1p0: Replica, replicaT1p1: Replica, futureReplica: Replica,
-                            partition: Partition, replicaManager: ReplicaManager,
-                            responseCallback: Capture[Seq[(TopicPartition, FetchPartitionData)] => Unit]) = {
-    stub(replicaT1p0, replicaT1p1, futureReplica, partition, replicaManager)
+  def stubWithFetchMessages(logT1p0: Log, logT1p1: Log, futureLog: Log, partition: Partition, replicaManager: ReplicaManager,
+          responseCallback: Capture[Seq[(TopicPartition, FetchPartitionData)] => Unit]): IExpectationSetters[Unit] = {
+    stub(logT1p0, logT1p1, futureLog, partition, replicaManager)
     expect(replicaManager.fetchMessages(
       EasyMock.anyLong(),
       EasyMock.anyInt(),
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala
index 5400f8698243d..bb4cbdad47d85 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala
@@ -18,8 +18,8 @@ package kafka.server
 
 import java.util.Optional
 
-import kafka.cluster.{BrokerEndPoint, Partition, Replica}
-import kafka.log.LogManager
+import kafka.cluster.{BrokerEndPoint, Partition}
+import kafka.log.{Log, LogManager}
 import kafka.server.QuotaFactory.UnboundedQuota
 import kafka.server.epoch.util.ReplicaFetcherMockBlockingSend
 import kafka.utils.TestUtils
@@ -79,28 +79,29 @@ class ReplicaFetcherThreadTest {
     val quota: ReplicationQuotaManager = createNiceMock(classOf[ReplicationQuotaManager])
     val logManager: LogManager = createMock(classOf[LogManager])
     val replicaAlterLogDirsManager: ReplicaAlterLogDirsManager = createMock(classOf[ReplicaAlterLogDirsManager])
-    val replica: Replica = createNiceMock(classOf[Replica])
+    val log: Log = createNiceMock(classOf[Log])
     val partition: Partition = createMock(classOf[Partition])
     val replicaManager: ReplicaManager = createMock(classOf[ReplicaManager])
 
     val leaderEpoch = 5
 
     //Stubs
-    expect(replica.logEndOffset).andReturn(0).anyTimes()
-    expect(replica.highWatermark).andReturn(0L).anyTimes()
-    expect(replica.latestEpoch).andReturn(Some(leaderEpoch)).once()
-    expect(replica.latestEpoch).andReturn(Some(leaderEpoch)).once()
-    expect(replica.latestEpoch).andReturn(None).once()  // t2p1 doesnt support epochs
-    expect(replica.endOffsetForEpoch(leaderEpoch)).andReturn(
+    expect(partition.localLogOrException).andReturn(log).anyTimes()
+    expect(log.logEndOffset).andReturn(0).anyTimes()
+    expect(log.highWatermark).andReturn(0).anyTimes()
+    expect(log.latestEpoch).andReturn(Some(leaderEpoch)).once()
+    expect(log.latestEpoch).andReturn(Some(leaderEpoch)).once()
+    expect(log.latestEpoch).andReturn(None).once()  // t2p1 doesnt support epochs
+    expect(log.endOffsetForEpoch(leaderEpoch)).andReturn(
       Some(OffsetAndEpoch(0, leaderEpoch))).anyTimes()
     expect(replicaManager.logManager).andReturn(logManager).anyTimes()
     expect(replicaManager.replicaAlterLogDirsManager).andReturn(replicaAlterLogDirsManager).anyTimes()
-    stub(replica, partition, replicaManager)
+    stub(partition, replicaManager, log)
 
     //Expectations
     expect(partition.truncateTo(anyLong(), anyBoolean())).times(3)
 
-    replay(replicaManager, logManager, quota, replica, partition)
+    replay(replicaManager, logManager, quota, partition, log)
 
     //Define the offsets for the OffsetsForLeaderEpochResponse
     val offsets = Map(t1p0 -> new EpochEndOffset(leaderEpoch, 1),
@@ -210,26 +211,26 @@ class ReplicaFetcherThreadTest {
     //Setup all dependencies
     val logManager: LogManager = createMock(classOf[LogManager])
     val replicaAlterLogDirsManager: ReplicaAlterLogDirsManager = createMock(classOf[ReplicaAlterLogDirsManager])
-    val replica: Replica = createNiceMock(classOf[Replica])
+    val log: Log = createNiceMock(classOf[Log])
     val partition: Partition = createMock(classOf[Partition])
     val replicaManager: ReplicaManager = createMock(classOf[ReplicaManager])
 
     val leaderEpoch = 5
 
     //Stubs
-    expect(replica.logEndOffset).andReturn(0).anyTimes()
-    expect(replica.highWatermark).andReturn(0L).anyTimes()
-    expect(replica.latestEpoch).andReturn(Some(leaderEpoch)).anyTimes()
-    expect(replica.endOffsetForEpoch(leaderEpoch)).andReturn(
+    expect(partition.localLogOrException).andReturn(log).anyTimes()
+    expect(log.highWatermark).andReturn(0).anyTimes()
+    expect(log.latestEpoch).andReturn(Some(leaderEpoch)).anyTimes()
+    expect(log.endOffsetForEpoch(leaderEpoch)).andReturn(
       Some(OffsetAndEpoch(0, leaderEpoch))).anyTimes()
     expect(replicaManager.logManager).andReturn(logManager).anyTimes()
     expect(replicaManager.replicaAlterLogDirsManager).andReturn(replicaAlterLogDirsManager).anyTimes()
-    stub(replica, partition, replicaManager)
+    stub(partition, replicaManager, log)
 
     //Expectations
     expect(partition.truncateTo(anyLong(), anyBoolean())).times(2)
 
-    replay(replicaManager, logManager, replica, partition)
+    replay(replicaManager, logManager, partition, log)
 
     //Define the offsets for the OffsetsForLeaderEpochResponse
     val offsets = Map(t1p0 -> new EpochEndOffset(leaderEpoch, 1), t1p1 -> new EpochEndOffset(leaderEpoch, 1)).asJava
@@ -270,7 +271,7 @@ class ReplicaFetcherThreadTest {
     val quota: ReplicationQuotaManager = createNiceMock(classOf[ReplicationQuotaManager])
     val logManager: LogManager = createMock(classOf[LogManager])
     val replicaAlterLogDirsManager: ReplicaAlterLogDirsManager = createMock(classOf[ReplicaAlterLogDirsManager])
-    val replica: Replica = createNiceMock(classOf[Replica])
+    val log: Log = createNiceMock(classOf[Log])
     val partition: Partition = createMock(classOf[Partition])
     val replicaManager: ReplicaManager = createMock(classOf[ReplicaManager])
 
@@ -279,23 +280,26 @@ class ReplicaFetcherThreadTest {
 
     //Stubs
     expect(partition.truncateTo(capture(truncateToCapture), anyBoolean())).anyTimes()
-    expect(replica.logEndOffset).andReturn(initialLEO).anyTimes()
-    expect(replica.highWatermark).andReturn(initialLEO - 1).anyTimes()
-    expect(replica.latestEpoch).andReturn(Some(leaderEpoch)).anyTimes()
-    expect(replica.endOffsetForEpoch(leaderEpoch)).andReturn(
+    expect(partition.localLogOrException).andReturn(log).anyTimes()
+    expect(log.highWatermark).andReturn(initialLEO - 1).anyTimes()
+    expect(log.latestEpoch).andReturn(Some(leaderEpoch)).anyTimes()
+    expect(log.endOffsetForEpoch(leaderEpoch)).andReturn(
       Some(OffsetAndEpoch(initialLEO, leaderEpoch))).anyTimes()
+    expect(log.logEndOffset).andReturn(initialLEO).anyTimes()
+    expect(replicaManager.localLogOrException(anyObject(classOf[TopicPartition]))).andReturn(log).anyTimes()
     expect(replicaManager.logManager).andReturn(logManager).anyTimes()
     expect(replicaManager.replicaAlterLogDirsManager).andReturn(replicaAlterLogDirsManager).anyTimes()
-    stub(replica, partition, replicaManager)
+    stub(partition, replicaManager, log)
 
-    replay(replicaManager, logManager, quota, replica, partition)
+    replay(replicaManager, logManager, quota, partition, log)
 
     //Define the offsets for the OffsetsForLeaderEpochResponse, these are used for truncation
     val offsetsReply = Map(t1p0 -> new EpochEndOffset(leaderEpoch, 156), t2p1 -> new EpochEndOffset(leaderEpoch, 172)).asJava
 
     //Create the thread
     val mockNetwork = new ReplicaFetcherMockBlockingSend(offsetsReply, brokerEndPoint, new SystemTime())
-    val thread = new ReplicaFetcherThread("bob", 0, brokerEndPoint, configs(0), failedPartitions, replicaManager, new Metrics(), new SystemTime(), quota, Some(mockNetwork))
+    val thread = new ReplicaFetcherThread("bob", 0, brokerEndPoint, configs.head, failedPartitions, replicaManager,
+      new Metrics(), new SystemTime(), quota, Some(mockNetwork))
     thread.addPartitions(Map(t1p0 -> offsetAndEpoch(0L), t2p1 -> offsetAndEpoch(0L)))
 
     //Run it
@@ -318,7 +322,7 @@ class ReplicaFetcherThreadTest {
     val quota: ReplicationQuotaManager = createNiceMock(classOf[ReplicationQuotaManager])
     val logManager: LogManager = createMock(classOf[LogManager])
     val replicaAlterLogDirsManager: ReplicaAlterLogDirsManager = createMock(classOf[ReplicaAlterLogDirsManager])
-    val replica: Replica = createNiceMock(classOf[Replica])
+    val log: Log = createNiceMock(classOf[Log])
     val partition: Partition = createMock(classOf[Partition])
     val replicaManager: ReplicaManager = createMock(classOf[ReplicaManager])
 
@@ -328,15 +332,17 @@ class ReplicaFetcherThreadTest {
 
     //Stubs
     expect(partition.truncateTo(capture(truncateToCapture), anyBoolean())).anyTimes()
-    expect(replica.logEndOffset).andReturn(initialLEO).anyTimes()
-    expect(replica.highWatermark).andReturn(initialLEO - 3).anyTimes()
-    expect(replica.latestEpoch).andReturn(Some(leaderEpochAtFollower)).anyTimes()
-    expect(replica.endOffsetForEpoch(leaderEpochAtLeader)).andReturn(None).anyTimes()
+    expect(partition.localLogOrException).andReturn(log).anyTimes()
+    expect(log.highWatermark).andReturn(initialLEO - 3).anyTimes()
+    expect(log.latestEpoch).andReturn(Some(leaderEpochAtFollower)).anyTimes()
+    expect(log.endOffsetForEpoch(leaderEpochAtLeader)).andReturn(None).anyTimes()
+    expect(log.logEndOffset).andReturn(initialLEO).anyTimes()
+    expect(replicaManager.localLogOrException(anyObject(classOf[TopicPartition]))).andReturn(log).anyTimes()
     expect(replicaManager.logManager).andReturn(logManager).anyTimes()
     expect(replicaManager.replicaAlterLogDirsManager).andReturn(replicaAlterLogDirsManager).anyTimes()
-    stub(replica, partition, replicaManager)
+    stub(partition, replicaManager, log)
 
-    replay(replicaManager, logManager, quota, replica, partition)
+    replay(replicaManager, logManager, quota, partition, log)
 
     //Define the offsets for the OffsetsForLeaderEpochResponse, these are used for truncation
     val offsetsReply = Map(t1p0 -> new EpochEndOffset(leaderEpochAtLeader, 156),
@@ -344,7 +350,8 @@ class ReplicaFetcherThreadTest {
 
     //Create the thread
     val mockNetwork = new ReplicaFetcherMockBlockingSend(offsetsReply, brokerEndPoint, new SystemTime())
-    val thread = new ReplicaFetcherThread("bob", 0, brokerEndPoint, configs(0), failedPartitions, replicaManager, new Metrics(), new SystemTime(), quota, Some(mockNetwork))
+    val thread = new ReplicaFetcherThread("bob", 0, brokerEndPoint, configs.head, failedPartitions,
+      replicaManager, new Metrics(), new SystemTime(), quota, Some(mockNetwork))
     thread.addPartitions(Map(t1p0 -> offsetAndEpoch(0L), t2p1 -> offsetAndEpoch(0L)))
 
     //Run it
@@ -370,7 +377,7 @@ class ReplicaFetcherThreadTest {
     val quota: ReplicationQuotaManager = createNiceMock(classOf[ReplicationQuotaManager])
     val logManager: LogManager = createMock(classOf[LogManager])
     val replicaAlterLogDirsManager: ReplicaAlterLogDirsManager = createMock(classOf[ReplicaAlterLogDirsManager])
-    val replica: Replica = createNiceMock(classOf[Replica])
+    val log: Log = createNiceMock(classOf[Log])
     val partition: Partition = createMock(classOf[Partition])
     val replicaManager: ReplicaManager = createMock(classOf[ReplicaManager])
 
@@ -378,18 +385,20 @@ class ReplicaFetcherThreadTest {
 
     // Stubs
     expect(partition.truncateTo(capture(truncateToCapture), anyBoolean())).anyTimes()
-    expect(replica.logEndOffset).andReturn(initialLEO).anyTimes()
-    expect(replica.highWatermark).andReturn(initialLEO - 2).anyTimes()
-    expect(replica.latestEpoch).andReturn(Some(5)).anyTimes()
-    expect(replica.endOffsetForEpoch(4)).andReturn(
+    expect(partition.localLogOrException).andReturn(log).anyTimes()
+    expect(log.highWatermark).andReturn(initialLEO - 2).anyTimes()
+    expect(log.latestEpoch).andReturn(Some(5)).anyTimes()
+    expect(log.endOffsetForEpoch(4)).andReturn(
       Some(OffsetAndEpoch(120, 3))).anyTimes()
-    expect(replica.endOffsetForEpoch(3)).andReturn(
+    expect(log.endOffsetForEpoch(3)).andReturn(
       Some(OffsetAndEpoch(120, 3))).anyTimes()
+    expect(log.logEndOffset).andReturn(initialLEO).anyTimes()
+    expect(replicaManager.localLogOrException(anyObject(classOf[TopicPartition]))).andReturn(log).anyTimes()
     expect(replicaManager.logManager).andReturn(logManager).anyTimes()
     expect(replicaManager.replicaAlterLogDirsManager).andReturn(replicaAlterLogDirsManager).anyTimes()
-    stub(replica, partition, replicaManager)
+    stub(partition, replicaManager, log)
 
-    replay(replicaManager, logManager, quota, replica, partition)
+    replay(replicaManager, logManager, quota, partition, log)
 
     // Define the offsets for the OffsetsForLeaderEpochResponse
     val offsets = Map(t1p0 -> new EpochEndOffset(4, 155), t1p1 -> new EpochEndOffset(4, 143)).asJava
@@ -441,7 +450,7 @@ class ReplicaFetcherThreadTest {
     val quota: ReplicationQuotaManager = createNiceMock(classOf[ReplicationQuotaManager])
     val logManager: LogManager = createMock(classOf[LogManager])
     val replicaAlterLogDirsManager: ReplicaAlterLogDirsManager = createMock(classOf[ReplicaAlterLogDirsManager])
-    val replica: Replica = createNiceMock(classOf[Replica])
+    val log: Log = createNiceMock(classOf[Log])
     val partition: Partition = createMock(classOf[Partition])
     val replicaManager: ReplicaManager = createMock(classOf[ReplicaManager])
 
@@ -449,18 +458,20 @@ class ReplicaFetcherThreadTest {
 
     // Stubs
     expect(partition.truncateTo(capture(truncateToCapture), anyBoolean())).anyTimes()
-    expect(replica.logEndOffset).andReturn(initialLEO).anyTimes()
-    expect(replica.highWatermark).andReturn(initialLEO - 2).anyTimes()
-    expect(replica.latestEpoch).andReturn(Some(5)).anyTimes()
-    expect(replica.endOffsetForEpoch(4)).andReturn(
+    expect(partition.localLogOrException).andReturn(log).anyTimes()
+    expect(log.highWatermark).andReturn(initialLEO - 2).anyTimes()
+    expect(log.latestEpoch).andReturn(Some(5)).anyTimes()
+    expect(log.endOffsetForEpoch(4)).andReturn(
       Some(OffsetAndEpoch(120, 3))).anyTimes()
-    expect(replica.endOffsetForEpoch(3)).andReturn(
+    expect(log.endOffsetForEpoch(3)).andReturn(
       Some(OffsetAndEpoch(120, 3))).anyTimes()
+    expect(log.logEndOffset).andReturn(initialLEO).anyTimes()
+    expect(replicaManager.localLogOrException(anyObject(classOf[TopicPartition]))).andReturn(log).anyTimes()
     expect(replicaManager.logManager).andReturn(logManager).anyTimes()
     expect(replicaManager.replicaAlterLogDirsManager).andReturn(replicaAlterLogDirsManager).anyTimes()
-    stub(replica, partition, replicaManager)
+    stub(partition, replicaManager, log)
 
-    replay(replicaManager, logManager, quota, replica, partition)
+    replay(replicaManager, logManager, quota, partition, log)
 
     // Define the offsets for the OffsetsForLeaderEpochResponse with undefined epoch to simulate
     // older protocol version
@@ -502,7 +513,7 @@ class ReplicaFetcherThreadTest {
     val quota: ReplicationQuotaManager = createNiceMock(classOf[ReplicationQuotaManager])
     val logManager: LogManager = createMock(classOf[LogManager])
     val replicaAlterLogDirsManager: ReplicaAlterLogDirsManager = createMock(classOf[ReplicaAlterLogDirsManager])
-    val replica: Replica = createNiceMock(classOf[Replica])
+    val log: Log = createNiceMock(classOf[Log])
     val partition: Partition = createMock(classOf[Partition])
     val replicaManager: ReplicaManager = createMock(classOf[ReplicaManager])
 
@@ -511,20 +522,20 @@ class ReplicaFetcherThreadTest {
 
     //Stubs
     expect(partition.truncateTo(capture(truncated), anyBoolean())).anyTimes()
-    expect(replica.logEndOffset).andReturn(initialLeo).anyTimes()
-    expect(replica.highWatermark).andReturn(initialFetchOffset).anyTimes()
-    expect(replica.latestEpoch).andReturn(Some(5))
+    expect(partition.localLogOrException).andReturn(log).anyTimes()
+    expect(log.highWatermark).andReturn(initialFetchOffset).anyTimes()
+    expect(log.latestEpoch).andReturn(Some(5))
     expect(replicaManager.logManager).andReturn(logManager).anyTimes()
     expect(replicaManager.replicaAlterLogDirsManager).andReturn(replicaAlterLogDirsManager).anyTimes()
-    stub(replica, partition, replicaManager)
-    replay(replicaManager, logManager, quota, replica, partition)
+    stub(partition, replicaManager, log)
+    replay(replicaManager, logManager, quota, partition, log)
 
     //Define the offsets for the OffsetsForLeaderEpochResponse, these are used for truncation
     val offsetsReply = Map(t1p0 -> new EpochEndOffset(EpochEndOffset.UNDEFINED_EPOCH, EpochEndOffset.UNDEFINED_EPOCH_OFFSET)).asJava
 
     //Create the thread
     val mockNetwork = new ReplicaFetcherMockBlockingSend(offsetsReply, brokerEndPoint, new SystemTime())
-    val thread = new ReplicaFetcherThread("bob", 0, brokerEndPoint, configs(0), failedPartitions, replicaManager, new Metrics(), new SystemTime(), quota, Some(mockNetwork))
+    val thread = new ReplicaFetcherThread("bob", 0, brokerEndPoint, configs.head, failedPartitions, replicaManager, new Metrics(), new SystemTime(), quota, Some(mockNetwork))
     thread.addPartitions(Map(t1p0 -> offsetAndEpoch(initialFetchOffset)))
 
     //Run it
@@ -545,7 +556,7 @@ class ReplicaFetcherThreadTest {
     val quota: ReplicationQuotaManager = createNiceMock(classOf[ReplicationQuotaManager])
     val logManager: LogManager = createMock(classOf[LogManager])
     val replicaAlterLogDirsManager: ReplicaAlterLogDirsManager = createMock(classOf[ReplicaAlterLogDirsManager])
-    val replica: Replica = createNiceMock(classOf[Replica])
+    val log: Log = createNiceMock(classOf[Log])
     val partition: Partition = createMock(classOf[Partition])
     val replicaManager: ReplicaManager = createMock(classOf[ReplicaManager])
 
@@ -554,17 +565,19 @@ class ReplicaFetcherThreadTest {
     val initialLeo = 300
 
     //Stubs
-    expect(replica.highWatermark).andReturn(highWaterMark).anyTimes()
+    expect(log.highWatermark).andReturn(highWaterMark).anyTimes()
     expect(partition.truncateTo(capture(truncated), anyBoolean())).anyTimes()
-    expect(replica.logEndOffset).andReturn(initialLeo).anyTimes()
-    expect(replica.latestEpoch).andReturn(Some(leaderEpoch)).anyTimes()
+    expect(partition.localLogOrException).andReturn(log).anyTimes()
+    expect(log.latestEpoch).andReturn(Some(leaderEpoch)).anyTimes()
     // this is for the last reply with EpochEndOffset(5, 156)
-    expect(replica.endOffsetForEpoch(leaderEpoch)).andReturn(
+    expect(log.endOffsetForEpoch(leaderEpoch)).andReturn(
       Some(OffsetAndEpoch(initialLeo, leaderEpoch))).anyTimes()
+    expect(log.logEndOffset).andReturn(initialLeo).anyTimes()
+    expect(replicaManager.localLogOrException(anyObject(classOf[TopicPartition]))).andReturn(log).anyTimes()
     expect(replicaManager.logManager).andReturn(logManager).anyTimes()
     expect(replicaManager.replicaAlterLogDirsManager).andReturn(replicaAlterLogDirsManager).anyTimes()
-    stub(replica, partition, replicaManager)
-    replay(replicaManager, logManager, quota, replica, partition)
+    stub(partition, replicaManager, log)
+    replay(replicaManager, logManager, quota, partition, log)
 
     //Define the offsets for the OffsetsForLeaderEpochResponse, these are used for truncation
     val offsetsReply = mutable.Map(
@@ -574,7 +587,7 @@ class ReplicaFetcherThreadTest {
 
     //Create the thread
     val mockNetwork = new ReplicaFetcherMockBlockingSend(offsetsReply, brokerEndPoint, new SystemTime())
-    val thread = new ReplicaFetcherThread("bob", 0, brokerEndPoint, configs(0), failedPartitions, replicaManager, new Metrics(), new SystemTime(), quota, Some(mockNetwork))
+    val thread = new ReplicaFetcherThread("bob", 0, brokerEndPoint, configs.head, failedPartitions, replicaManager, new Metrics(), new SystemTime(), quota, Some(mockNetwork))
     thread.addPartitions(Map(t1p0 -> offsetAndEpoch(0L), t1p1 -> offsetAndEpoch(0L)))
 
     //Run thread 3 times
@@ -602,7 +615,7 @@ class ReplicaFetcherThreadTest {
     val quota: ReplicationQuotaManager = createNiceMock(classOf[ReplicationQuotaManager])
     val logManager: LogManager = createNiceMock(classOf[LogManager])
     val replicaAlterLogDirsManager: ReplicaAlterLogDirsManager = createMock(classOf[ReplicaAlterLogDirsManager])
-    val replica: Replica = createNiceMock(classOf[Replica])
+    val log: Log = createNiceMock(classOf[Log])
     val partition: Partition = createMock(classOf[Partition])
     val replicaManager: ReplicaManager = createNiceMock(classOf[ReplicaManager])
 
@@ -610,16 +623,16 @@ class ReplicaFetcherThreadTest {
 
     //Stub return values
     expect(partition.truncateTo(0L, false)).times(2)
-    expect(replica.logEndOffset).andReturn(0).anyTimes()
-    expect(replica.highWatermark).andReturn(0).anyTimes()
-    expect(replica.latestEpoch).andReturn(Some(leaderEpoch)).anyTimes()
-    expect(replica.endOffsetForEpoch(leaderEpoch)).andReturn(
+    expect(partition.localLogOrException).andReturn(log).anyTimes()
+    expect(log.highWatermark).andReturn(0).anyTimes()
+    expect(log.latestEpoch).andReturn(Some(leaderEpoch)).anyTimes()
+    expect(log.endOffsetForEpoch(leaderEpoch)).andReturn(
       Some(OffsetAndEpoch(0, leaderEpoch))).anyTimes()
     expect(replicaManager.logManager).andReturn(logManager).anyTimes()
     expect(replicaManager.replicaAlterLogDirsManager).andReturn(replicaAlterLogDirsManager).anyTimes()
-    stub(replica, partition, replicaManager)
+    stub(partition, replicaManager, log)
 
-    replay(replicaManager, logManager, quota, replica, partition)
+    replay(replicaManager, logManager, quota, partition, log)
 
     //Define the offsets for the OffsetsForLeaderEpochResponse
     val offsetsReply = Map(
@@ -655,21 +668,23 @@ class ReplicaFetcherThreadTest {
     val quota: ReplicationQuotaManager = createNiceMock(classOf[ReplicationQuotaManager])
     val logManager: LogManager = createNiceMock(classOf[LogManager])
     val replicaAlterLogDirsManager: ReplicaAlterLogDirsManager = createMock(classOf[ReplicaAlterLogDirsManager])
-    val replica: Replica = createNiceMock(classOf[Replica])
+    val log: Log = createNiceMock(classOf[Log])
     val partition: Partition = createMock(classOf[Partition])
     val replicaManager: ReplicaManager = createNiceMock(classOf[ReplicaManager])
 
     //Stub return values
     expect(partition.truncateTo(capture(truncateToCapture), anyBoolean())).once
-    expect(replica.logEndOffset).andReturn(initialLEO).anyTimes()
-    expect(replica.highWatermark).andReturn(initialLEO - 2).anyTimes()
-    expect(replica.latestEpoch).andReturn(Some(5)).anyTimes()
-    expect(replica.endOffsetForEpoch(5)).andReturn(Some(OffsetAndEpoch(initialLEO, 5))).anyTimes()
+    expect(partition.localLogOrException).andReturn(log).anyTimes()
+    expect(log.highWatermark).andReturn(initialLEO - 2).anyTimes()
+    expect(log.latestEpoch).andReturn(Some(5)).anyTimes()
+    expect(log.endOffsetForEpoch(5)).andReturn(Some(OffsetAndEpoch(initialLEO, 5))).anyTimes()
+    expect(log.logEndOffset).andReturn(initialLEO).anyTimes()
+    expect(replicaManager.localLogOrException(anyObject(classOf[TopicPartition]))).andReturn(log).anyTimes()
     expect(replicaManager.logManager).andReturn(logManager).anyTimes()
     expect(replicaManager.replicaAlterLogDirsManager).andReturn(replicaAlterLogDirsManager).anyTimes()
-    stub(replica, partition, replicaManager)
+    stub(partition, replicaManager, log)
 
-    replay(replicaManager, logManager, quota, replica, partition)
+    replay(replicaManager, logManager, quota, partition, log)
 
     //Define the offsets for the OffsetsForLeaderEpochResponse
     val offsetsReply = Map(
@@ -678,7 +693,8 @@ class ReplicaFetcherThreadTest {
 
     //Create the fetcher thread
     val mockNetwork = new ReplicaFetcherMockBlockingSend(offsetsReply, brokerEndPoint, new SystemTime())
-    val thread = new ReplicaFetcherThread("bob", 0, brokerEndPoint, config, failedPartitions, replicaManager, new Metrics(), new SystemTime(), quota, Some(mockNetwork))
+    val thread = new ReplicaFetcherThread("bob", 0, brokerEndPoint, config, failedPartitions, replicaManager, new Metrics(),
+      new SystemTime(), quota, Some(mockNetwork))
 
     //When
     thread.addPartitions(Map(t1p0 -> offsetAndEpoch(0L), t1p1 -> offsetAndEpoch(0L)))
@@ -728,13 +744,12 @@ class ReplicaFetcherThreadTest {
     verify(mockBlockingSend)
   }
 
-  def stub(replica: Replica, partition: Partition, replicaManager: ReplicaManager): Unit = {
-    expect(replicaManager.localReplicaOrException(t1p0)).andReturn(replica).anyTimes()
+  def stub(partition: Partition, replicaManager: ReplicaManager, log: Log): Unit = {
+    expect(replicaManager.localLogOrException(t1p0)).andReturn(log).anyTimes()
     expect(replicaManager.nonOfflinePartition(t1p0)).andReturn(Some(partition)).anyTimes()
-    expect(replicaManager.localReplicaOrException(t1p1)).andReturn(replica).anyTimes()
+    expect(replicaManager.localLogOrException(t1p1)).andReturn(log).anyTimes()
     expect(replicaManager.nonOfflinePartition(t1p1)).andReturn(Some(partition)).anyTimes()
-    expect(replicaManager.localReplicaOrException(t2p1)).andReturn(replica).anyTimes()
+    expect(replicaManager.localLogOrException(t2p1)).andReturn(log).anyTimes()
     expect(replicaManager.nonOfflinePartition(t2p1)).andReturn(Some(partition)).anyTimes()
-    expect(partition.localReplicaOrException).andReturn(replica).anyTimes()
   }
 }
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala
index d298003674c36..e1a3e2c6212df 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala
@@ -149,7 +149,7 @@ class ReplicaManagerQuotasTest {
   def testCompleteInDelayedFetchWithReplicaThrottling(): Unit = {
     // Set up DelayedFetch where there is data to return to a follower replica, either in-sync or out of sync
     def setupDelayedFetch(isReplicaInSync: Boolean): DelayedFetch = {
-      val endOffsetMetadata = new LogOffsetMetadata(messageOffset = 100L, segmentBaseOffset = 0L, relativePositionInSegment = 500)
+      val endOffsetMetadata = LogOffsetMetadata(messageOffset = 100L, segmentBaseOffset = 0L, relativePositionInSegment = 500)
       val partition: Partition = EasyMock.createMock(classOf[Partition])
 
       val offsetSnapshot = LogOffsetSnapshot(
@@ -170,7 +170,7 @@ class ReplicaManagerQuotasTest {
       EasyMock.replay(replicaManager, partition)
 
       val tp = new TopicPartition("t1", 0)
-      val fetchPartitionStatus = FetchPartitionStatus(new LogOffsetMetadata(messageOffset = 50L, segmentBaseOffset = 0L,
+      val fetchPartitionStatus = FetchPartitionStatus(LogOffsetMetadata(messageOffset = 50L, segmentBaseOffset = 0L,
          relativePositionInSegment = 250), new PartitionData(50, 0, 1, Optional.empty()))
       val fetchMetadata = FetchMetadata(fetchMinBytes = 1,
         fetchMaxBytes = 1000,
@@ -198,7 +198,9 @@ class ReplicaManagerQuotasTest {
     val log: Log = createNiceMock(classOf[Log])
     expect(log.logStartOffset).andReturn(0L).anyTimes()
     expect(log.logEndOffset).andReturn(20L).anyTimes()
-    expect(log.logEndOffsetMetadata).andReturn(new LogOffsetMetadata(20L)).anyTimes()
+    expect(log.highWatermark).andReturn(5).anyTimes()
+    expect(log.lastStableOffset).andReturn(5).anyTimes()
+    expect(log.logEndOffsetMetadata).andReturn(LogOffsetMetadata(20L)).anyTimes()
 
     //if we ask for len 1 return a message
     expect(log.read(anyObject(),
@@ -207,7 +209,7 @@ class ReplicaManagerQuotasTest {
       minOneMessage = anyBoolean(),
       includeAbortedTxns = EasyMock.eq(false))).andReturn(
       FetchDataInfo(
-        new LogOffsetMetadata(0L, 0L, 0),
+        LogOffsetMetadata(0L, 0L, 0),
         MemoryRecords.withRecords(CompressionType.NONE, record)
       )).anyTimes()
 
@@ -218,7 +220,7 @@ class ReplicaManagerQuotasTest {
       minOneMessage = anyBoolean(),
       includeAbortedTxns = EasyMock.eq(false))).andReturn(
       FetchDataInfo(
-        new LogOffsetMetadata(0L, 0L, 0),
+        LogOffsetMetadata(0L, 0L, 0),
         MemoryRecords.EMPTY
       )).anyTimes()
     replay(log)
@@ -231,25 +233,25 @@ class ReplicaManagerQuotasTest {
     expect(logManager.liveLogDirs).andReturn(Array.empty[File]).anyTimes()
     replay(logManager)
 
+    val leaderBrokerId = configs.head.brokerId
     replicaManager = new ReplicaManager(configs.head, metrics, time, zkClient, scheduler, logManager,
       new AtomicBoolean(false), QuotaFactory.instantiate(configs.head, metrics, time, ""),
-      new BrokerTopicStats, new MetadataCache(configs.head.brokerId), new LogDirFailureChannel(configs.head.logDirs.size))
+      new BrokerTopicStats, new MetadataCache(leaderBrokerId), new LogDirFailureChannel(configs.head.logDirs.size))
 
     //create the two replicas
     for ((p, _) <- fetchInfo) {
       val partition = replicaManager.createPartition(p)
-      val leaderReplica = new Replica(configs.head.brokerId, p, time, 0, Some(log))
-      leaderReplica.highWatermark = 5
-      partition.leaderReplicaIdOpt = Some(leaderReplica.brokerId)
-      val followerReplica = new Replica(configs.last.brokerId, p, time, 0, Some(log))
-      val allReplicas = Set(leaderReplica, followerReplica)
-      allReplicas.foreach(partition.addReplicaIfNotExists)
+      log.highWatermark = 5
+      partition.leaderReplicaIdOpt = Some(leaderBrokerId)
+      partition.setLog(log, isFutureLog = false)
+
+      val followerReplica = new Replica(configs.last.brokerId, p)
+      val allReplicas : Set[Int] = Set(leaderBrokerId, followerReplica.brokerId)
+      partition.addReplicaIfNotExists(followerReplica)
       if (bothReplicasInSync) {
         partition.inSyncReplicas = allReplicas
-        followerReplica.highWatermark = 5
       } else {
-        partition.inSyncReplicas = Set(leaderReplica)
-        followerReplica.highWatermark = 0
+        partition.inSyncReplicas = Set(leaderBrokerId)
       }
     }
   }
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index 64a29ed387708..57786507fe3fc 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -87,7 +87,8 @@ class ReplicaManagerTest {
       new MetadataCache(config.brokerId), new LogDirFailureChannel(config.logDirs.size))
     try {
       val partition = rm.createPartition(new TopicPartition(topic, 1))
-      partition.getOrCreateReplica(1, isNew = false, new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints))
+      partition.createLogIfNotExists(1, isNew = false, isFutureReplica = false,
+        new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints))
       rm.checkpointHighWatermarks()
     } finally {
       // shutdown the replica manager upon test completion
@@ -106,7 +107,8 @@ class ReplicaManagerTest {
       new MetadataCache(config.brokerId), new LogDirFailureChannel(config.logDirs.size))
     try {
       val partition = rm.createPartition(new TopicPartition(topic, 1))
-      partition.getOrCreateReplica(1, isNew = false, new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints))
+      partition.createLogIfNotExists(1, isNew = false, isFutureReplica = false,
+        new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints))
       rm.checkpointHighWatermarks()
     } finally {
       // shutdown the replica manager upon test completion
@@ -160,7 +162,8 @@ class ReplicaManagerTest {
       val brokerList = Seq[Integer](0, 1).asJava
 
       val partition = rm.createPartition(new TopicPartition(topic, 0))
-      partition.getOrCreateReplica(0, isNew = false, new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints))
+      partition.createLogIfNotExists(0, isNew = false, isFutureReplica = false,
+        new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints))
       // Make this replica the leader.
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
         collection.immutable.Map(new TopicPartition(topic, 0) ->
@@ -168,7 +171,7 @@ class ReplicaManagerTest {
         Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
       rm.becomeLeaderOrFollower(0, leaderAndIsrRequest1, (_, _) => ())
       rm.getPartitionOrException(new TopicPartition(topic, 0), expectLeader = true)
-          .localReplicaOrException
+          .localLogOrException
 
       val records = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("first message".getBytes()))
       val appendResult = appendRecords(rm, new TopicPartition(topic, 0), records).onFire { response =>
@@ -204,7 +207,8 @@ class ReplicaManagerTest {
       val brokerList = Seq[Integer](0, 1).asJava
 
       val partition = replicaManager.createPartition(new TopicPartition(topic, 0))
-      partition.getOrCreateReplica(0, isNew = false, new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
+      partition.createLogIfNotExists(0, isNew = false, isFutureReplica = false,
+        new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
 
       // Make this replica the leader.
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
@@ -213,7 +217,7 @@ class ReplicaManagerTest {
         Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
       replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest1, (_, _) => ())
       replicaManager.getPartitionOrException(new TopicPartition(topic, 0), expectLeader = true)
-        .localReplicaOrException
+        .localLogOrException
 
       val producerId = 234L
       val epoch = 5.toShort
@@ -255,7 +259,8 @@ class ReplicaManagerTest {
       val brokerList = Seq[Integer](0, 1).asJava
 
       val partition = replicaManager.createPartition(new TopicPartition(topic, 0))
-      partition.getOrCreateReplica(0, isNew = false, new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
+      partition.createLogIfNotExists(0, isNew = false, isFutureReplica = false,
+        new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
 
       // Make this replica the leader.
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
@@ -264,7 +269,7 @@ class ReplicaManagerTest {
         Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
       replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest1, (_, _) => ())
       replicaManager.getPartitionOrException(new TopicPartition(topic, 0), expectLeader = true)
-        .localReplicaOrException
+        .localLogOrException
 
       val producerId = 234L
       val epoch = 5.toShort
@@ -351,7 +356,8 @@ class ReplicaManagerTest {
     try {
       val brokerList = Seq[Integer](0, 1).asJava
       val partition = replicaManager.createPartition(new TopicPartition(topic, 0))
-      partition.getOrCreateReplica(0, isNew = false, new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
+      partition.createLogIfNotExists(0, isNew = false, isFutureReplica = false,
+        new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
 
       // Make this replica the leader.
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
@@ -360,7 +366,7 @@ class ReplicaManagerTest {
         Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
       replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest1, (_, _) => ())
       replicaManager.getPartitionOrException(new TopicPartition(topic, 0), expectLeader = true)
-        .localReplicaOrException
+        .localLogOrException
 
       val producerId = 234L
       val epoch = 5.toShort
@@ -417,7 +423,8 @@ class ReplicaManagerTest {
       val brokerList = Seq[Integer](0, 1, 2).asJava
 
       val partition = rm.createPartition(new TopicPartition(topic, 0))
-      partition.getOrCreateReplica(0, isNew = false, new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints))
+      partition.createLogIfNotExists(0, isNew = false, isFutureReplica = false,
+        new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints))
 
       // Make this replica the leader.
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
@@ -426,7 +433,7 @@ class ReplicaManagerTest {
         Set(new Node(0, "host1", 0), new Node(1, "host2", 1), new Node(2, "host2", 2)).asJava).build()
       rm.becomeLeaderOrFollower(0, leaderAndIsrRequest1, (_, _) => ())
       rm.getPartitionOrException(new TopicPartition(topic, 0), expectLeader = true)
-        .localReplicaOrException
+        .localLogOrException
 
       // Append a couple of messages.
       for(i <- 1 to 2) {
@@ -479,7 +486,6 @@ class ReplicaManagerTest {
 
       assertTrue(partition.getReplica(1).isDefined)
       val followerReplica = partition.getReplica(1).get
-      assertEquals(None, followerReplica.log)
       assertEquals(-1L, followerReplica.logStartOffset)
       assertEquals(-1L, followerReplica.logEndOffset)
 
@@ -553,8 +559,8 @@ class ReplicaManagerTest {
       val tp0 = new TopicPartition(topic, 0)
       val tp1 = new TopicPartition(topic, 1)
       val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
-      replicaManager.createPartition(tp0).getOrCreateReplica(0, isNew = false, offsetCheckpoints)
-      replicaManager.createPartition(tp1).getOrCreateReplica(0, isNew = false, offsetCheckpoints)
+      replicaManager.createPartition(tp0).createLogIfNotExists(0, isNew = false, isFutureReplica = false, offsetCheckpoints)
+      replicaManager.createPartition(tp1).createLogIfNotExists(0, isNew = false, isFutureReplica = false, offsetCheckpoints)
       val partition0Replicas = Seq[Integer](0, 1).asJava
       val partition1Replicas = Seq[Integer](0, 2).asJava
       val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
@@ -609,12 +615,12 @@ class ReplicaManagerTest {
         responseCallback = fetchCallback,
         isolationLevel = IsolationLevel.READ_UNCOMMITTED
       )
-      val tp0Replica = replicaManager.localReplica(tp0)
-      assertTrue(tp0Replica.isDefined)
-      assertEquals("hw should be incremented", 1, tp0Replica.get.highWatermark)
+      val tp0Log = replicaManager.localLog(tp0)
+      assertTrue(tp0Log.isDefined)
+      assertEquals("hw should be incremented", 1, tp0Log.get.highWatermark)
 
-      replicaManager.localReplica(tp1)
-      val tp1Replica = replicaManager.localReplica(tp1)
+      replicaManager.localLog(tp1)
+      val tp1Replica = replicaManager.localLog(tp1)
       assertTrue(tp1Replica.isDefined)
       assertEquals("hw should not be incremented", 0, tp1Replica.get.highWatermark)
 
@@ -646,7 +652,7 @@ class ReplicaManagerTest {
     // Initialize partition state to follower, with leader = 1, leaderEpoch = 1
     val partition = replicaManager.createPartition(new TopicPartition(topic, topicPartition))
     val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
-    partition.getOrCreateReplica(followerBrokerId, isNew = false, offsetCheckpoints)
+    partition.createLogIfNotExists(followerBrokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
     partition.makeFollower(controllerId,
       leaderAndIsrPartitionState(leaderEpoch, leaderBrokerId, aliveBrokerIds),
       correlationId, offsetCheckpoints)
@@ -713,6 +719,8 @@ class ReplicaManagerTest {
       override def latestEpoch: Option[Int] = Some(leaderEpochFromLeader)
 
       override def logEndOffsetMetadata = LogOffsetMetadata(localLogOffset)
+
+      override def logEndOffset: Long = localLogOffset
     }
 
     // Expect to call LogManager.truncateTo exactly once
diff --git a/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala b/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala
index e7e6e8fab6609..1f3179c4eb7e6 100644
--- a/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala
+++ b/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala
@@ -82,7 +82,9 @@ class SimpleFetchTest {
     EasyMock.expect(log.logStartOffset).andReturn(0).anyTimes()
     EasyMock.expect(log.logEndOffset).andReturn(leaderLEO).anyTimes()
     EasyMock.expect(log.dir).andReturn(TestUtils.tempDir()).anyTimes()
-    EasyMock.expect(log.logEndOffsetMetadata).andReturn(new LogOffsetMetadata(leaderLEO)).anyTimes()
+    EasyMock.expect(log.logEndOffsetMetadata).andReturn(LogOffsetMetadata(leaderLEO)).anyTimes()
+    EasyMock.expect(log.highWatermark).andReturn(partitionHW).anyTimes()
+    EasyMock.expect(log.lastStableOffset).andReturn(partitionHW).anyTimes()
     EasyMock.expect(log.read(
       startOffset = 0,
       maxLength = fetchSize,
@@ -90,7 +92,7 @@ class SimpleFetchTest {
       minOneMessage = true,
       includeAbortedTxns = false))
       .andReturn(FetchDataInfo(
-        new LogOffsetMetadata(0L, 0L, 0),
+        LogOffsetMetadata(0L, 0L, 0),
         MemoryRecords.withRecords(CompressionType.NONE, recordToHW)
       )).anyTimes()
     EasyMock.expect(log.read(
@@ -100,7 +102,7 @@ class SimpleFetchTest {
       minOneMessage = true,
       includeAbortedTxns = false))
       .andReturn(FetchDataInfo(
-        new LogOffsetMetadata(0L, 0L, 0),
+        LogOffsetMetadata(0L, 0L, 0),
         MemoryRecords.withRecords(CompressionType.NONE, recordToLEO)
       )).anyTimes()
     EasyMock.replay(log)
@@ -120,22 +122,22 @@ class SimpleFetchTest {
     val partition = replicaManager.createPartition(new TopicPartition(topic, partitionId))
 
     // create the leader replica with the local log
-    val leaderReplica = new Replica(configs.head.brokerId, partition.topicPartition, time, 0, Some(log))
-    leaderReplica.highWatermark = partitionHW
-    partition.leaderReplicaIdOpt = Some(leaderReplica.brokerId)
+    log.highWatermark = partitionHW
+    partition.leaderReplicaIdOpt = Some(configs.head.brokerId)
+    partition.setLog(log, false)
 
     // create the follower replica with defined log end offset
-    val followerReplica= new Replica(configs(1).brokerId, partition.topicPartition, time)
-    val leo = new LogOffsetMetadata(followerLEO, 0L, followerLEO.toInt)
+    val followerReplica= new Replica(configs(1).brokerId, partition.topicPartition)
+    val leo = LogOffsetMetadata(followerLEO, 0L, followerLEO.toInt)
     followerReplica.updateFetchState(
       followerFetchOffsetMetadata = leo,
       followerStartOffset = 0L,
       followerFetchTimeMs= time.milliseconds,
       leaderEndOffset = leo.messageOffset)
+    partition.addReplicaIfNotExists(followerReplica)
 
     // add both of them to ISR
-    val allReplicas = List(leaderReplica, followerReplica)
-    allReplicas.foreach(partition.addReplicaIfNotExists)
+    val allReplicas = List(configs.head.brokerId, followerReplica.brokerId)
     partition.inSyncReplicas = allReplicas.toSet
   }
 
diff --git a/core/src/test/scala/unit/kafka/server/epoch/EpochDrivenReplicationProtocolAcceptanceTest.scala b/core/src/test/scala/unit/kafka/server/epoch/EpochDrivenReplicationProtocolAcceptanceTest.scala
index 834954144cd3b..3e481099aad12 100644
--- a/core/src/test/scala/unit/kafka/server/epoch/EpochDrivenReplicationProtocolAcceptanceTest.scala
+++ b/core/src/test/scala/unit/kafka/server/epoch/EpochDrivenReplicationProtocolAcceptanceTest.scala
@@ -444,7 +444,7 @@ class EpochDrivenReplicationProtocolAcceptanceTest extends ZooKeeperTestHarness
 
   private def awaitISR(tp: TopicPartition): Unit = {
     TestUtils.waitUntilTrue(() => {
-      leader.replicaManager.nonOfflinePartition(tp).get.inSyncReplicas.map(_.brokerId).size == 2
+      leader.replicaManager.nonOfflinePartition(tp).get.inSyncReplicas.size == 2
     }, "Timed out waiting for replicas to join ISR")
   }
 
diff --git a/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala b/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala
index cf78bac8d617c..ef3ac857328ac 100644
--- a/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala
@@ -145,7 +145,7 @@ class LeaderEpochIntegrationTest extends ZooKeeperTestHarness with Logging {
 
     brokers += createServer(fromProps(createBrokerConfig(101, zkConnect)))
 
-    def leo() = brokers(1).replicaManager.localReplica(tp).get.logEndOffset
+    def leo() = brokers(1).replicaManager.localLog(tp).get.logEndOffset
 
     TestUtils.createTopic(zkClient, tp.topic, Map(tp.partition -> Seq(101)), brokers)
     producer = createProducer(getBrokerListStrFromServers(brokers), acks = -1)
diff --git a/core/src/test/scala/unit/kafka/server/epoch/OffsetsForLeaderEpochTest.scala b/core/src/test/scala/unit/kafka/server/epoch/OffsetsForLeaderEpochTest.scala
index eba4167edf1f1..08be8a28f7967 100644
--- a/core/src/test/scala/unit/kafka/server/epoch/OffsetsForLeaderEpochTest.scala
+++ b/core/src/test/scala/unit/kafka/server/epoch/OffsetsForLeaderEpochTest.scala
@@ -20,7 +20,6 @@ import java.io.File
 import java.util.Optional
 import java.util.concurrent.atomic.AtomicBoolean
 
-import kafka.cluster.Replica
 import kafka.log.{Log, LogManager}
 import kafka.server._
 import kafka.utils.{MockTime, TestUtils}
@@ -58,8 +57,7 @@ class OffsetsForLeaderEpochTest {
       QuotaFactory.instantiate(config, metrics, time, ""), new BrokerTopicStats,
       new MetadataCache(config.brokerId), new LogDirFailureChannel(config.logDirs.size))
     val partition = replicaManager.createPartition(tp)
-    val leaderReplica = new Replica(config.brokerId, partition.topicPartition, time, 0, Some(mockLog))
-    partition.addReplicaIfNotExists(leaderReplica)
+    partition.setLog(mockLog, isFutureLog = false)
     partition.leaderReplicaIdOpt = Some(config.brokerId)
 
     //When
diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index 5c9284ff6b8b7..98fbc4e4dc25f 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -848,7 +848,7 @@ object TestUtils extends Logging {
   }
 
   def isLeaderLocalOnBroker(topic: String, partitionId: Int, server: KafkaServer): Boolean = {
-    server.replicaManager.nonOfflinePartition(new TopicPartition(topic, partitionId)).exists(_.leaderReplicaIfLocal.isDefined)
+    server.replicaManager.nonOfflinePartition(new TopicPartition(topic, partitionId)).exists(_.leaderLogIfLocal.isDefined)
   }
 
   def findLeaderEpoch(brokerId: Int,
@@ -929,7 +929,7 @@ object TestUtils extends Logging {
     def newLeaderExists: Option[Int] = {
       servers.find { server =>
         server.config.brokerId != oldLeader &&
-          server.replicaManager.nonOfflinePartition(tp).exists(_.leaderReplicaIfLocal.isDefined)
+          server.replicaManager.nonOfflinePartition(tp).exists(_.leaderLogIfLocal.isDefined)
       }.map(_.config.brokerId)
     }
 
@@ -944,7 +944,7 @@ object TestUtils extends Logging {
                              timeout: Long = JTestUtils.DEFAULT_MAX_WAIT_MS): Int = {
     def leaderIfExists: Option[Int] = {
       servers.find { server =>
-        server.replicaManager.nonOfflinePartition(tp).exists(_.leaderReplicaIfLocal.isDefined)
+        server.replicaManager.nonOfflinePartition(tp).exists(_.leaderLogIfLocal.isDefined)
       }.map(_.config.brokerId)
     }
 

From 1b9e1073885951697f34950a1ea706c93826e871 Mon Sep 17 00:00:00 2001
From: Boyang Chen 
Date: Mon, 17 Jun 2019 10:58:43 -0700
Subject: [PATCH 0372/1071] KAFKA-7853: Refactor coordinator config (#6854)

An attempt to refactor current coordinator logic.

Reviewers: Stanislav Kozlovski , Konstantine Karantasis , Guozhang Wang 
---
 .../kafka/clients/CommonClientConfigs.java    |  38 ++++++-
 .../kafka/clients/GroupRebalanceConfig.java   | 100 +++++++++++++++++
 .../clients/consumer/ConsumerConfig.java      |  37 ++----
 .../kafka/clients/consumer/KafkaConsumer.java |  29 ++---
 .../internals/AbstractCoordinator.java        | 106 +++++++-----------
 .../internals/ConsumerCoordinator.java        |  51 ++++-----
 .../clients/consumer/internals/Heartbeat.java |  37 +++---
 .../clients/CommonClientConfigsTest.java      |   2 +-
 .../clients/consumer/KafkaConsumerTest.java   |  42 ++++---
 .../internals/AbstractCoordinatorTest.java    |  34 ++++--
 .../internals/ConsumerCoordinatorTest.java    |  96 ++++++++++------
 .../consumer/internals/HeartbeatTest.java     |  21 +++-
 .../distributed/DistributedConfig.java        |  13 +--
 .../distributed/WorkerCoordinator.java        |  22 +---
 .../distributed/WorkerGroupMember.java        |   9 +-
 .../WorkerCoordinatorIncrementalTest.java     |  37 +++---
 .../distributed/WorkerCoordinatorTest.java    |  38 ++++---
 17 files changed, 409 insertions(+), 303 deletions(-)
 create mode 100644 clients/src/main/java/org/apache/kafka/clients/GroupRebalanceConfig.java

diff --git a/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java b/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java
index 49465dc2e3dd7..d8428a75cf627 100644
--- a/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java
+++ b/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java
@@ -26,7 +26,7 @@
 import java.util.Map;
 
 /**
- * Some configurations shared by both producer and consumer
+ * Configurations shared by Kafka client applications: producer, consumer, connect, etc.
  */
 public class CommonClientConfigs {
     private static final Logger log = LoggerFactory.getLogger(CommonClientConfigs.class);
@@ -101,6 +101,42 @@ public class CommonClientConfigs {
                                                          + "elapses the client will resend the request if necessary or fail the request if "
                                                          + "retries are exhausted.";
 
+    public static final String GROUP_ID_CONFIG = "group.id";
+    public static final String GROUP_ID_DOC = "A unique string that identifies the consumer group this consumer belongs to. This property is required if the consumer uses either the group management functionality by using subscribe(topic) or the Kafka-based offset management strategy.";
+
+    public static final String GROUP_INSTANCE_ID_CONFIG = "group.instance.id";
+    public static final String GROUP_INSTANCE_ID_DOC = "A unique identifier of the consumer instance provided by end user. " +
+            "Only non-empty strings are permitted. If set, the consumer is treated as a static member, " +
+            "which means that only one instance with this ID is allowed in the consumer group at any time. " +
+            "This can be used in combination with a larger session timeout to avoid group rebalances caused by transient unavailability " +
+            "(e.g. process restarts). If not set, the consumer will join the group as a dynamic member, which is the traditional behavior.";
+
+    public static final String MAX_POLL_INTERVAL_MS_CONFIG = "max.poll.interval.ms";
+    public static final String MAX_POLL_INTERVAL_MS_DOC = "The maximum delay between invocations of poll() when using " +
+            "consumer group management. This places an upper bound on the amount of time that the consumer can be idle " +
+            "before fetching more records. If poll() is not called before expiration of this timeout, then the consumer " +
+            "is considered failed and the group will rebalance in order to reassign the partitions to another member. ";
+
+    public static final String REBALANCE_TIMEOUT_MS_CONFIG = "rebalance.timeout.ms";
+    public static final String REBALANCE_TIMEOUT_MS_DOC = "The maximum allowed time for each worker to join the group " +
+            "once a rebalance has begun. This is basically a limit on the amount of time needed for all tasks to " +
+            "flush any pending data and commit offsets. If the timeout is exceeded, then the worker will be removed " +
+            "from the group, which will cause offset commit failures.";
+
+    public static final String SESSION_TIMEOUT_MS_CONFIG = "session.timeout.ms";
+    public static final String SESSION_TIMEOUT_MS_DOC = "The timeout used to detect client failures when using " +
+            "Kafka's group management facility. The client sends periodic heartbeats to indicate its liveness " +
+            "to the broker. If no heartbeats are received by the broker before the expiration of this session timeout, " +
+            "then the broker will remove this client from the group and initiate a rebalance. Note that the value " +
+            "must be in the allowable range as configured in the broker configuration by group.min.session.timeout.ms " +
+            "and group.max.session.timeout.ms.";
+
+    public static final String HEARTBEAT_INTERVAL_MS_CONFIG = "heartbeat.interval.ms";
+    public static final String HEARTBEAT_INTERVAL_MS_DOC = "The expected time between heartbeats to the consumer " +
+            "coordinator when using Kafka's group management facilities. Heartbeats are used to ensure that the " +
+            "consumer's session stays active and to facilitate rebalancing when new consumers join or leave the group. " +
+            "The value must be set lower than session.timeout.ms, but typically should be set no higher " +
+            "than 1/3 of that value. It can be adjusted even lower to control the expected time for normal rebalances.";
 
     /**
      * Postprocess the configuration so that exponential backoff is disabled when reconnect backoff
diff --git a/clients/src/main/java/org/apache/kafka/clients/GroupRebalanceConfig.java b/clients/src/main/java/org/apache/kafka/clients/GroupRebalanceConfig.java
new file mode 100644
index 0000000000000..006800a3e5c4e
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/GroupRebalanceConfig.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients;
+
+import org.apache.kafka.common.config.AbstractConfig;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+
+import java.util.Locale;
+import java.util.Optional;
+
+/**
+ * Class to extract group rebalance related configs.
+ */
+public class GroupRebalanceConfig {
+
+    public enum ProtocolType {
+        CONSUMER,
+        CONNECT;
+
+        @Override
+        public String toString() {
+            return super.toString().toLowerCase(Locale.ROOT);
+        }
+    }
+
+    public final int sessionTimeoutMs;
+    public final int rebalanceTimeoutMs;
+    public final int heartbeatIntervalMs;
+    public final String groupId;
+    public final Optional groupInstanceId;
+    public final long retryBackoffMs;
+    public final boolean leaveGroupOnClose;
+
+    public GroupRebalanceConfig(AbstractConfig config, ProtocolType protocolType) {
+        this.sessionTimeoutMs = config.getInt(CommonClientConfigs.SESSION_TIMEOUT_MS_CONFIG);
+
+        // Consumer and Connect use different config names for defining rebalance timeout
+        if (protocolType == ProtocolType.CONSUMER) {
+            this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG);
+        } else {
+            this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.REBALANCE_TIMEOUT_MS_CONFIG);
+        }
+
+        this.heartbeatIntervalMs = config.getInt(CommonClientConfigs.HEARTBEAT_INTERVAL_MS_CONFIG);
+        this.groupId = config.getString(CommonClientConfigs.GROUP_ID_CONFIG);
+
+        // Static membership is only introduced in consumer API.
+        if (protocolType == ProtocolType.CONSUMER) {
+            String groupInstanceId = config.getString(CommonClientConfigs.GROUP_INSTANCE_ID_CONFIG);
+            if (groupInstanceId != null) {
+                JoinGroupRequest.validateGroupInstanceId(groupInstanceId);
+                this.groupInstanceId = Optional.of(groupInstanceId);
+            } else {
+                this.groupInstanceId = Optional.empty();
+            }
+        } else {
+            this.groupInstanceId = Optional.empty();
+        }
+
+        this.retryBackoffMs = config.getLong(CommonClientConfigs.RETRY_BACKOFF_MS_CONFIG);
+
+        // Internal leave group config is only defined in Consumer.
+        if (protocolType == ProtocolType.CONSUMER) {
+            this.leaveGroupOnClose = config.getBoolean("internal.leave.group.on.close");
+        } else {
+            this.leaveGroupOnClose = true;
+        }
+    }
+
+    // For testing purpose.
+    public GroupRebalanceConfig(final int sessionTimeoutMs,
+                                final int rebalanceTimeoutMs,
+                                final int heartbeatIntervalMs,
+                                String groupId,
+                                Optional groupInstanceId,
+                                long retryBackoffMs,
+                                boolean leaveGroupOnClose) {
+        this.sessionTimeoutMs = sessionTimeoutMs;
+        this.rebalanceTimeoutMs = rebalanceTimeoutMs;
+        this.heartbeatIntervalMs = heartbeatIntervalMs;
+        this.groupId = groupId;
+        this.groupInstanceId = groupInstanceId;
+        this.retryBackoffMs = retryBackoffMs;
+        this.leaveGroupOnClose = leaveGroupOnClose;
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
index 010fff81fa591..7eb34d4aeee78 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
@@ -50,50 +50,33 @@ public class ConsumerConfig extends AbstractConfig {
     /**
      * group.id
      */
-    public static final String GROUP_ID_CONFIG = "group.id";
-    private static final String GROUP_ID_DOC = "A unique string that identifies the consumer group this consumer belongs to. This property is required if the consumer uses either the group management functionality by using subscribe(topic) or the Kafka-based offset management strategy.";
+    public static final String GROUP_ID_CONFIG = CommonClientConfigs.GROUP_ID_CONFIG;
+    private static final String GROUP_ID_DOC = CommonClientConfigs.GROUP_ID_DOC;
 
     /**
      * group.instance.id
      */
-    public static final String GROUP_INSTANCE_ID_CONFIG = "group.instance.id";
-    private static final String GROUP_INSTANCE_ID_DOC = "A unique identifier of the consumer instance provided by end user. " +
-            "Only non-empty strings are permitted. If set, the consumer is treated as a static member, " +
-            "which means that only one instance with this ID is allowed in the consumer group at any time. " +
-            "This can be used in combination with a larger session timeout to avoid group rebalances caused by transient unavailability " +
-            "(e.g. process restarts). If not set, the consumer will join the group as a dynamic member, which is the traditional behavior.";
+    public static final String GROUP_INSTANCE_ID_CONFIG = CommonClientConfigs.GROUP_INSTANCE_ID_CONFIG;
+    private static final String GROUP_INSTANCE_ID_DOC = CommonClientConfigs.GROUP_INSTANCE_ID_DOC;
 
     /** max.poll.records */
     public static final String MAX_POLL_RECORDS_CONFIG = "max.poll.records";
     private static final String MAX_POLL_RECORDS_DOC = "The maximum number of records returned in a single call to poll().";
 
     /** max.poll.interval.ms */
-    public static final String MAX_POLL_INTERVAL_MS_CONFIG = "max.poll.interval.ms";
-    private static final String MAX_POLL_INTERVAL_MS_DOC = "The maximum delay between invocations of poll() when using " +
-            "consumer group management. This places an upper bound on the amount of time that the consumer can be idle " +
-            "before fetching more records. If poll() is not called before expiration of this timeout, then the consumer " +
-            "is considered failed and the group will rebalance in order to reassign the partitions to another member. ";
-
+    public static final String MAX_POLL_INTERVAL_MS_CONFIG = CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG;
+    private static final String MAX_POLL_INTERVAL_MS_DOC = CommonClientConfigs.MAX_POLL_INTERVAL_MS_DOC;
     /**
      * session.timeout.ms
      */
-    public static final String SESSION_TIMEOUT_MS_CONFIG = "session.timeout.ms";
-    private static final String SESSION_TIMEOUT_MS_DOC = "The timeout used to detect consumer failures when using " +
-            "Kafka's group management facility. The consumer sends periodic heartbeats to indicate its liveness " +
-            "to the broker. If no heartbeats are received by the broker before the expiration of this session timeout, " +
-            "then the broker will remove this consumer from the group and initiate a rebalance. Note that the value " +
-            "must be in the allowable range as configured in the broker configuration by group.min.session.timeout.ms " +
-            "and group.max.session.timeout.ms.";
+    public static final String SESSION_TIMEOUT_MS_CONFIG = CommonClientConfigs.SESSION_TIMEOUT_MS_CONFIG;
+    private static final String SESSION_TIMEOUT_MS_DOC = CommonClientConfigs.SESSION_TIMEOUT_MS_DOC;
 
     /**
      * heartbeat.interval.ms
      */
-    public static final String HEARTBEAT_INTERVAL_MS_CONFIG = "heartbeat.interval.ms";
-    private static final String HEARTBEAT_INTERVAL_MS_DOC = "The expected time between heartbeats to the consumer " +
-            "coordinator when using Kafka's group management facilities. Heartbeats are used to ensure that the " +
-            "consumer's session stays active and to facilitate rebalancing when new consumers join or leave the group. " +
-            "The value must be set lower than session.timeout.ms, but typically should be set no higher " +
-            "than 1/3 of that value. It can be adjusted even lower to control the expected time for normal rebalances.";
+    public static final String HEARTBEAT_INTERVAL_MS_CONFIG = CommonClientConfigs.HEARTBEAT_INTERVAL_MS_CONFIG;
+    private static final String HEARTBEAT_INTERVAL_MS_DOC = CommonClientConfigs.HEARTBEAT_INTERVAL_MS_DOC;
 
     /**
      * bootstrap.servers
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index 5be065d5f2621..79afafa388654 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -19,6 +19,7 @@
 import org.apache.kafka.clients.ApiVersions;
 import org.apache.kafka.clients.ClientDnsLookup;
 import org.apache.kafka.clients.ClientUtils;
+import org.apache.kafka.clients.GroupRebalanceConfig;
 import org.apache.kafka.clients.Metadata;
 import org.apache.kafka.clients.NetworkClient;
 import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator;
@@ -27,7 +28,6 @@
 import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient;
 import org.apache.kafka.clients.consumer.internals.Fetcher;
 import org.apache.kafka.clients.consumer.internals.FetcherMetricsRegistry;
-import org.apache.kafka.clients.consumer.internals.Heartbeat;
 import org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener;
 import org.apache.kafka.clients.consumer.internals.PartitionAssignor;
 import org.apache.kafka.clients.consumer.internals.SubscriptionState;
@@ -50,7 +50,6 @@
 import org.apache.kafka.common.network.ChannelBuilder;
 import org.apache.kafka.common.network.Selector;
 import org.apache.kafka.common.requests.IsolationLevel;
-import org.apache.kafka.common.requests.JoinGroupRequest;
 import org.apache.kafka.common.requests.MetadataRequest;
 import org.apache.kafka.common.serialization.Deserializer;
 import org.apache.kafka.common.utils.AppInfoParser;
@@ -568,7 +567,6 @@ public class KafkaConsumer implements Consumer {
     private final Logger log;
     private final String clientId;
     private String groupId;
-    private Optional groupInstanceId;
     private final ConsumerCoordinator coordinator;
     private final Deserializer keyDeserializer;
     private final Deserializer valueDeserializer;
@@ -674,15 +672,14 @@ private KafkaConsumer(ConsumerConfig config, Deserializer keyDeserializer, De
             this.clientId = clientId;
             this.groupId = config.getString(ConsumerConfig.GROUP_ID_CONFIG);
 
+            GroupRebalanceConfig groupRebalanceConfig = new GroupRebalanceConfig(config,
+                                                                                 GroupRebalanceConfig.ProtocolType.CONSUMER);
             LogContext logContext;
             // If group.instance.id is set, we will append it to the log context.
-            String groupInstanceId = config.getString(ConsumerConfig.GROUP_INSTANCE_ID_CONFIG);
-            if (groupInstanceId != null) {
-                JoinGroupRequest.validateGroupInstanceId(groupInstanceId);
-                this.groupInstanceId = Optional.of(groupInstanceId);
-                logContext = new LogContext("[Consumer instanceId=" + groupInstanceId + ", clientId=" + clientId + ", groupId=" + groupId + "] ");
+            if (groupRebalanceConfig.groupInstanceId.isPresent()) {
+                logContext = new LogContext("[Consumer instanceId=" + groupRebalanceConfig.groupInstanceId.get() +
+                        ", clientId=" + clientId + ", groupId=" + groupId + "] ");
             } else {
-                this.groupInstanceId = Optional.empty();
                 logContext = new LogContext("[Consumer clientId=" + clientId + ", groupId=" + groupId + "] ");
             }
 
@@ -773,28 +770,20 @@ else if (enableAutoCommit)
                     ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG,
                     PartitionAssignor.class);
 
-            int maxPollIntervalMs = config.getInt(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG);
-            int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG);
             // no coordinator will be constructed for the default (null) group id
             this.coordinator = groupId == null ? null :
-                new ConsumerCoordinator(logContext,
+                new ConsumerCoordinator(groupRebalanceConfig,
+                        logContext,
                         this.client,
-                        groupId,
-                        this.groupInstanceId,
-                        maxPollIntervalMs,
-                        sessionTimeoutMs,
-                        new Heartbeat(time, sessionTimeoutMs, heartbeatIntervalMs, maxPollIntervalMs, retryBackoffMs),
                         assignors,
                         this.metadata,
                         this.subscriptions,
                         metrics,
                         metricGrpPrefix,
                         this.time,
-                        retryBackoffMs,
                         enableAutoCommit,
                         config.getInt(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG),
-                        this.interceptors,
-                        config.getBoolean(ConsumerConfig.LEAVE_GROUP_ON_CLOSE_CONFIG));
+                        this.interceptors);
             this.fetcher = new Fetcher<>(
                     logContext,
                     this.client,
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index 30277b3819b5c..efe6cb75ce5aa 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -17,6 +17,7 @@
 package org.apache.kafka.clients.consumer.internals;
 
 import org.apache.kafka.clients.ClientResponse;
+import org.apache.kafka.clients.GroupRebalanceConfig;
 import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.Node;
 import org.apache.kafka.common.errors.AuthenticationException;
@@ -71,7 +72,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -114,15 +114,11 @@ private enum MemberState {
     }
 
     private final Logger log;
-    private final int sessionTimeoutMs;
     private final GroupCoordinatorMetrics sensors;
     private final Heartbeat heartbeat;
-    protected final int rebalanceTimeoutMs;
-    protected final String groupId;
-    protected final Optional groupInstanceId;
+    private final GroupRebalanceConfig rebalanceConfig;
     protected final ConsumerNetworkClient client;
     protected final Time time;
-    protected final long retryBackoffMs;
 
     private HeartbeatThread heartbeatThread = null;
     private boolean rejoinNeeded = true;
@@ -133,52 +129,24 @@ private enum MemberState {
     private Generation generation = Generation.NO_GENERATION;
 
     private RequestFuture findCoordinatorFuture = null;
-    private final boolean leaveGroupOnClose;
 
     /**
      * Initialize the coordination manager.
      */
-    public AbstractCoordinator(LogContext logContext,
+    public AbstractCoordinator(GroupRebalanceConfig rebalanceConfig,
+                               LogContext logContext,
                                ConsumerNetworkClient client,
-                               String groupId,
-                               Optional groupInstanceId,
-                               int rebalanceTimeoutMs,
-                               int sessionTimeoutMs,
-                               Heartbeat heartbeat,
                                Metrics metrics,
                                String metricGrpPrefix,
-                               Time time,
-                               long retryBackoffMs,
-                               boolean leaveGroupOnClose) {
+                               Time time) {
+        Objects.requireNonNull(rebalanceConfig.groupId,
+                               "Expected a non-null group id for coordinator construction");
+        this.rebalanceConfig = rebalanceConfig;
         this.log = logContext.logger(AbstractCoordinator.class);
         this.client = client;
         this.time = time;
-        this.groupId = Objects.requireNonNull(groupId,
-                "Expected a non-null group id for coordinator construction");
-        this.groupInstanceId = groupInstanceId;
-        this.rebalanceTimeoutMs = rebalanceTimeoutMs;
-        this.sessionTimeoutMs = sessionTimeoutMs;
-        this.heartbeat = heartbeat;
+        this.heartbeat = new Heartbeat(rebalanceConfig, time);
         this.sensors = new GroupCoordinatorMetrics(metrics, metricGrpPrefix);
-        this.retryBackoffMs = retryBackoffMs;
-        this.leaveGroupOnClose = leaveGroupOnClose;
-    }
-
-    public AbstractCoordinator(LogContext logContext,
-                               ConsumerNetworkClient client,
-                               String groupId,
-                               Optional groupInstanceId,
-                               int rebalanceTimeoutMs,
-                               int sessionTimeoutMs,
-                               int heartbeatIntervalMs,
-                               Metrics metrics,
-                               String metricGrpPrefix,
-                               Time time,
-                               long retryBackoffMs,
-                               boolean leaveGroupOnClose) {
-        this(logContext, client, groupId, groupInstanceId, rebalanceTimeoutMs, sessionTimeoutMs,
-                new Heartbeat(time, sessionTimeoutMs, heartbeatIntervalMs, rebalanceTimeoutMs, retryBackoffMs),
-                metrics, metricGrpPrefix, time, retryBackoffMs, leaveGroupOnClose);
     }
 
     /**
@@ -263,7 +231,7 @@ protected synchronized boolean ensureCoordinatorReady(final Timer timer) {
                 // we found the coordinator, but the connection has failed, so mark
                 // it dead and backoff before retrying discovery
                 markCoordinatorUnknown();
-                timer.sleep(retryBackoffMs);
+                timer.sleep(rebalanceConfig.retryBackoffMs);
             }
         } while (coordinatorUnknown() && timer.notExpired());
 
@@ -438,7 +406,7 @@ boolean joinGroupIfNeeded(final Timer timer) {
                 else if (!future.isRetriable())
                     throw exception;
 
-                timer.sleep(retryBackoffMs);
+                timer.sleep(rebalanceConfig.retryBackoffMs);
             }
         }
         return true;
@@ -505,13 +473,13 @@ RequestFuture sendJoinGroupRequest() {
         log.info("(Re-)joining group");
         JoinGroupRequest.Builder requestBuilder = new JoinGroupRequest.Builder(
                 new JoinGroupRequestData()
-                        .setGroupId(groupId)
-                        .setSessionTimeoutMs(this.sessionTimeoutMs)
+                        .setGroupId(rebalanceConfig.groupId)
+                        .setSessionTimeoutMs(this.rebalanceConfig.sessionTimeoutMs)
                         .setMemberId(this.generation.memberId)
-                        .setGroupInstanceId(this.groupInstanceId.orElse(null))
+                        .setGroupInstanceId(this.rebalanceConfig.groupInstanceId.orElse(null))
                         .setProtocolType(protocolType())
                         .setProtocols(metadata())
-                        .setRebalanceTimeoutMs(this.rebalanceTimeoutMs)
+                        .setRebalanceTimeoutMs(this.rebalanceConfig.rebalanceTimeoutMs)
         );
 
         log.debug("Sending JoinGroup ({}) to coordinator {}", requestBuilder, this.coordinator);
@@ -519,7 +487,7 @@ RequestFuture sendJoinGroupRequest() {
         // Note that we override the request timeout using the rebalance timeout since that is the
         // maximum time that it may block on the coordinator. We add an extra 5 seconds for small delays.
 
-        int joinGroupTimeoutMs = Math.max(rebalanceTimeoutMs, rebalanceTimeoutMs + 5000);
+        int joinGroupTimeoutMs = Math.max(rebalanceConfig.rebalanceTimeoutMs, rebalanceConfig.rebalanceTimeoutMs + 5000);
         return client.send(coordinator, requestBuilder, joinGroupTimeoutMs)
                 .compose(new JoinGroupResponseHandler());
     }
@@ -573,9 +541,9 @@ public void handle(JoinGroupResponse joinResponse, RequestFuture fut
                 // log the error and re-throw the exception
                 log.error("Attempt to join group failed due to fatal error: {}", error.message());
                 if (error == Errors.GROUP_MAX_SIZE_REACHED) {
-                    future.raise(new GroupMaxSizeReachedException(groupId));
+                    future.raise(new GroupMaxSizeReachedException(rebalanceConfig.groupId));
                 } else if (error == Errors.GROUP_AUTHORIZATION_FAILED) {
-                    future.raise(new GroupAuthorizationException(groupId));
+                    future.raise(new GroupAuthorizationException(rebalanceConfig.groupId));
                 } else {
                     future.raise(error);
                 }
@@ -606,9 +574,9 @@ private RequestFuture onJoinFollower() {
         SyncGroupRequest.Builder requestBuilder =
                 new SyncGroupRequest.Builder(
                         new SyncGroupRequestData()
-                                .setGroupId(groupId)
+                                .setGroupId(rebalanceConfig.groupId)
                                 .setMemberId(generation.memberId)
-                                .setGroupInstanceId(this.groupInstanceId.orElse(null))
+                                .setGroupInstanceId(this.rebalanceConfig.groupInstanceId.orElse(null))
                                 .setGenerationId(generation.generationId)
                                 .setAssignments(Collections.emptyList())
                 );
@@ -633,9 +601,9 @@ private RequestFuture onJoinLeader(JoinGroupResponse joinResponse) {
             SyncGroupRequest.Builder requestBuilder =
                     new SyncGroupRequest.Builder(
                             new SyncGroupRequestData()
-                                    .setGroupId(groupId)
+                                    .setGroupId(rebalanceConfig.groupId)
                                     .setMemberId(generation.memberId)
-                                    .setGroupInstanceId(this.groupInstanceId.orElse(null))
+                                    .setGroupInstanceId(this.rebalanceConfig.groupInstanceId.orElse(null))
                                     .setGenerationId(generation.generationId)
                                     .setAssignments(groupAssignmentList)
                     );
@@ -665,7 +633,7 @@ public void handle(SyncGroupResponse syncResponse,
                 requestRejoin();
 
                 if (error == Errors.GROUP_AUTHORIZATION_FAILED) {
-                    future.raise(new GroupAuthorizationException(groupId));
+                    future.raise(new GroupAuthorizationException(rebalanceConfig.groupId));
                 } else if (error == Errors.REBALANCE_IN_PROGRESS) {
                     log.debug("SyncGroup failed because the group began another rebalance");
                     future.raise(error);
@@ -701,7 +669,7 @@ private RequestFuture sendFindCoordinatorRequest(Node node) {
                 new FindCoordinatorRequest.Builder(
                         new FindCoordinatorRequestData()
                             .setKeyType(CoordinatorType.GROUP.id())
-                            .setKey(this.groupId));
+                            .setKey(this.rebalanceConfig.groupId));
         return client.send(node, requestBuilder)
                 .compose(new FindCoordinatorResponseHandler());
     }
@@ -731,7 +699,7 @@ public void onSuccess(ClientResponse resp, RequestFuture future) {
                 }
                 future.complete(null);
             } else if (error == Errors.GROUP_AUTHORIZATION_FAILED) {
-                future.raise(new GroupAuthorizationException(groupId));
+                future.raise(new GroupAuthorizationException(rebalanceConfig.groupId));
             } else {
                 log.debug("Group coordinator lookup failed: {}", findCoordinatorResponse.data().errorMessage());
                 future.raise(error);
@@ -854,7 +822,7 @@ protected void close(Timer timer) {
             // Synchronize after closing the heartbeat thread since heartbeat thread
             // needs this lock to complete and terminate after close flag is set.
             synchronized (this) {
-                if (leaveGroupOnClose) {
+                if (rebalanceConfig.leaveGroupOnClose) {
                     maybeLeaveGroup();
                 }
 
@@ -883,7 +851,7 @@ public synchronized void maybeLeaveGroup() {
             // attempt any resending if the request fails or times out.
             log.info("Member {} sending LeaveGroup request to coordinator {}", generation.memberId, coordinator);
             LeaveGroupRequest.Builder request = new LeaveGroupRequest.Builder(new LeaveGroupRequestData()
-                    .setGroupId(groupId).setMemberId(generation.memberId));
+                    .setGroupId(rebalanceConfig.groupId).setMemberId(generation.memberId));
             client.send(coordinator, request)
                     .compose(new LeaveGroupResponseHandler());
             client.pollNoWakeup();
@@ -893,7 +861,7 @@ public synchronized void maybeLeaveGroup() {
     }
 
     protected boolean isDynamicMember() {
-        return !groupInstanceId.isPresent();
+        return !rebalanceConfig.groupInstanceId.isPresent();
     }
 
     private class LeaveGroupResponseHandler extends CoordinatorResponseHandler {
@@ -915,9 +883,9 @@ synchronized RequestFuture sendHeartbeatRequest() {
         log.debug("Sending Heartbeat request to coordinator {}", coordinator);
         HeartbeatRequest.Builder requestBuilder =
                 new HeartbeatRequest.Builder(new HeartbeatRequestData()
-                        .setGroupId(groupId)
+                        .setGroupId(rebalanceConfig.groupId)
                         .setMemberId(this.generation.memberId)
-                        .setGroupInstanceId(this.groupInstanceId.orElse(null))
+                        .setGroupInstanceId(this.rebalanceConfig.groupInstanceId.orElse(null))
                         .setGenerationId(this.generation.generationId));
         return client.send(coordinator, requestBuilder)
                 .compose(new HeartbeatResponseHandler());
@@ -953,7 +921,7 @@ public void handle(HeartbeatResponse heartbeatResponse, RequestFuture futu
                 resetGeneration();
                 future.raise(error);
             } else if (error == Errors.GROUP_AUTHORIZATION_FAILED) {
-                future.raise(new GroupAuthorizationException(groupId));
+                future.raise(new GroupAuthorizationException(rebalanceConfig.groupId));
             } else {
                 future.raise(new KafkaException("Unexpected error in heartbeat response: " + error.message()));
             }
@@ -1051,7 +1019,7 @@ private class HeartbeatThread extends KafkaThread implements AutoCloseable {
         private AtomicReference failed = new AtomicReference<>(null);
 
         private HeartbeatThread() {
-            super(HEARTBEAT_THREAD_PREFIX + (groupId.isEmpty() ? "" : " | " + groupId), true);
+            super(HEARTBEAT_THREAD_PREFIX + (rebalanceConfig.groupId.isEmpty() ? "" : " | " + rebalanceConfig.groupId), true);
         }
 
         public void enable() {
@@ -1113,7 +1081,7 @@ public void run() {
                             if (findCoordinatorFuture != null || lookupCoordinator().failed())
                                 // the immediate future check ensures that we backoff properly in the case that no
                                 // brokers are available to connect to.
-                                AbstractCoordinator.this.wait(retryBackoffMs);
+                                AbstractCoordinator.this.wait(rebalanceConfig.retryBackoffMs);
                         } else if (heartbeat.sessionTimeoutExpired(now)) {
                             // the session timeout has expired without seeing a successful heartbeat, so we should
                             // probably make sure the coordinator is still healthy.
@@ -1131,7 +1099,7 @@ public void run() {
                         } else if (!heartbeat.shouldHeartbeat(now)) {
                             // poll again after waiting for the retry backoff in case the heartbeat failed or the
                             // coordinator disconnected
-                            AbstractCoordinator.this.wait(retryBackoffMs);
+                            AbstractCoordinator.this.wait(rebalanceConfig.retryBackoffMs);
                         } else {
                             heartbeat.sentHeartbeat(now);
 
@@ -1153,7 +1121,7 @@ public void onFailure(RuntimeException e) {
                                             // however, then the session timeout may expire before we can rejoin.
                                             heartbeat.receiveHeartbeat();
                                         } else if (e instanceof FencedInstanceIdException) {
-                                            log.error("Caught fenced group.instance.id {} error in heartbeat thread", groupInstanceId);
+                                            log.error("Caught fenced group.instance.id {} error in heartbeat thread", rebalanceConfig.groupInstanceId);
                                             heartbeatThread.failed.set(e);
                                             heartbeatThread.disable();
                                         } else {
@@ -1243,4 +1211,8 @@ private static class UnjoinedGroupException extends RetriableException {
 
     }
 
+    // For testing only
+    public Heartbeat heartbeat() {
+        return heartbeat;
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index fbf5db42af3e7..a590a1e794ee6 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.clients.consumer.internals;
 
+import org.apache.kafka.clients.GroupRebalanceConfig;
 import org.apache.kafka.clients.consumer.CommitFailedException;
 import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
 import org.apache.kafka.clients.consumer.OffsetAndMetadata;
@@ -63,7 +64,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -74,6 +74,7 @@
  * This class manages the coordination process with the consumer coordinator.
  */
 public final class ConsumerCoordinator extends AbstractCoordinator {
+    private final GroupRebalanceConfig rebalanceConfig;
     private final Logger log;
     private final List assignors;
     private final ConsumerMetadata metadata;
@@ -120,36 +121,25 @@ private boolean sameRequest(final Set currentRequest, final Gene
     /**
      * Initialize the coordination manager.
      */
-    public ConsumerCoordinator(LogContext logContext,
+    public ConsumerCoordinator(GroupRebalanceConfig rebalanceConfig,
+                               LogContext logContext,
                                ConsumerNetworkClient client,
-                               String groupId,
-                               Optional groupInstanceId,
-                               int rebalanceTimeoutMs,
-                               int sessionTimeoutMs,
-                               Heartbeat heartbeat,
                                List assignors,
                                ConsumerMetadata metadata,
                                SubscriptionState subscriptions,
                                Metrics metrics,
                                String metricGrpPrefix,
                                Time time,
-                               long retryBackoffMs,
                                boolean autoCommitEnabled,
                                int autoCommitIntervalMs,
-                               ConsumerInterceptors interceptors,
-                               boolean leaveGroupOnClose) {
-        super(logContext,
+                               ConsumerInterceptors interceptors) {
+        super(rebalanceConfig,
+              logContext,
               client,
-              groupId,
-              groupInstanceId,
-              rebalanceTimeoutMs,
-              sessionTimeoutMs,
-              heartbeat,
               metrics,
               metricGrpPrefix,
-              time,
-              retryBackoffMs,
-              leaveGroupOnClose);
+              time);
+        this.rebalanceConfig = rebalanceConfig;
         this.log = logContext.logger(ConsumerCoordinator.class);
         this.metadata = metadata;
         this.metadataSnapshot = new MetadataSnapshot(subscriptions, metadata.fetch(), metadata.updateVersion());
@@ -459,7 +449,7 @@ protected Map performAssignment(String leaderId,
     @Override
     protected void onJoinPrepare(int generation, String memberId) {
         // commit offsets prior to rebalance if auto-commit enabled
-        maybeAutoCommitOffsetsSync(time.timer(rebalanceTimeoutMs));
+        maybeAutoCommitOffsetsSync(time.timer(rebalanceConfig.rebalanceTimeoutMs));
 
         // execute the user's callback before rebalance
         ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
@@ -558,7 +548,7 @@ public Map fetchCommittedOffsets(final Set offsets,
             if (future.failed() && !future.isRetriable())
                 throw future.exception();
 
-            timer.sleep(retryBackoffMs);
+            timer.sleep(rebalanceConfig.retryBackoffMs);
         } while (timer.notExpired());
 
         return false;
@@ -723,7 +714,7 @@ private void doAutoCommitOffsetsAsync() {
                 if (exception instanceof RetriableException) {
                     log.debug("Asynchronous auto-commit of offsets {} failed due to retriable error: {}", offsets,
                         exception);
-                    nextAutoCommitTimer.updateAndReset(retryBackoffMs);
+                    nextAutoCommitTimer.updateAndReset(rebalanceConfig.retryBackoffMs);
                 } else {
                     log.warn("Asynchronous auto-commit of offsets {} failed: {}", offsets, exception.getMessage());
                 }
@@ -813,10 +804,10 @@ private RequestFuture sendOffsetCommitRequest(final Map(requestTopicDataMap.values()))
         );
 
@@ -857,7 +848,7 @@ public void handle(OffsetCommitResponse commitResponse, RequestFuture futu
                         }
 
                         if (error == Errors.GROUP_AUTHORIZATION_FAILED) {
-                            future.raise(new GroupAuthorizationException(groupId));
+                            future.raise(new GroupAuthorizationException(rebalanceConfig.groupId));
                             return;
                         } else if (error == Errors.TOPIC_AUTHORIZATION_FAILED) {
                             unauthorizedTopics.add(tp.topic());
@@ -929,7 +920,7 @@ private RequestFuture> sendOffsetFetchReq
 
         log.debug("Fetching committed offsets for partitions: {}", partitions);
         // construct the request
-        OffsetFetchRequest.Builder requestBuilder = new OffsetFetchRequest.Builder(this.groupId,
+        OffsetFetchRequest.Builder requestBuilder = new OffsetFetchRequest.Builder(this.rebalanceConfig.groupId,
                 new ArrayList<>(partitions));
 
         // send the request with a callback
@@ -952,7 +943,7 @@ public void handle(OffsetFetchResponse response, RequestFuture= sessionTimeoutMs)
+    public Heartbeat(GroupRebalanceConfig config,
+                     Time time) {
+        if (config.heartbeatIntervalMs >= config.sessionTimeoutMs)
             throw new IllegalArgumentException("Heartbeat must be set lower than the session timeout");
-
+        this.rebalanceConfig = config;
         this.time = time;
-        this.sessionTimeoutMs = sessionTimeoutMs;
-        this.heartbeatIntervalMs = heartbeatIntervalMs;
-        this.maxPollIntervalMs = maxPollIntervalMs;
-        this.retryBackoffMs = retryBackoffMs;
-        this.heartbeatTimer = time.timer(heartbeatIntervalMs);
-        this.sessionTimer = time.timer(sessionTimeoutMs);
+        this.heartbeatTimer = time.timer(config.heartbeatIntervalMs);
+        this.sessionTimer = time.timer(config.sessionTimeoutMs);
+        this.maxPollIntervalMs = config.rebalanceTimeoutMs;
         this.pollTimer = time.timer(maxPollIntervalMs);
     }
 
@@ -66,17 +59,17 @@ public void poll(long now) {
     public void sentHeartbeat(long now) {
         this.lastHeartbeatSend = now;
         update(now);
-        heartbeatTimer.reset(heartbeatIntervalMs);
+        heartbeatTimer.reset(rebalanceConfig.heartbeatIntervalMs);
     }
 
     public void failHeartbeat() {
         update(time.milliseconds());
-        heartbeatTimer.reset(retryBackoffMs);
+        heartbeatTimer.reset(rebalanceConfig.retryBackoffMs);
     }
 
     public void receiveHeartbeat() {
         update(time.milliseconds());
-        sessionTimer.reset(sessionTimeoutMs);
+        sessionTimer.reset(rebalanceConfig.sessionTimeoutMs);
     }
 
     public boolean shouldHeartbeat(long now) {
@@ -100,14 +93,14 @@ public boolean sessionTimeoutExpired(long now) {
 
     public void resetTimeouts() {
         update(time.milliseconds());
-        sessionTimer.reset(sessionTimeoutMs);
+        sessionTimer.reset(rebalanceConfig.sessionTimeoutMs);
         pollTimer.reset(maxPollIntervalMs);
-        heartbeatTimer.reset(heartbeatIntervalMs);
+        heartbeatTimer.reset(rebalanceConfig.heartbeatIntervalMs);
     }
 
     public void resetSessionTimeout() {
         update(time.milliseconds());
-        sessionTimer.reset(sessionTimeoutMs);
+        sessionTimer.reset(rebalanceConfig.sessionTimeoutMs);
     }
 
     public boolean pollTimeoutExpired(long now) {
diff --git a/clients/src/test/java/org/apache/kafka/clients/CommonClientConfigsTest.java b/clients/src/test/java/org/apache/kafka/clients/CommonClientConfigsTest.java
index 63a9312846001..d65e6d1cbdc50 100644
--- a/clients/src/test/java/org/apache/kafka/clients/CommonClientConfigsTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/CommonClientConfigsTest.java
@@ -58,7 +58,7 @@ public TestConfig(Map props) {
     }
 
     @Test
-    public void testExponentialBackoffDefaults() throws Exception {
+    public void testExponentialBackoffDefaults() {
         TestConfig defaultConf = new TestConfig(Collections.emptyMap());
         assertEquals(Long.valueOf(50L),
                 defaultConf.getLong(CommonClientConfigs.RECONNECT_BACKOFF_MS_CONFIG));
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
index 405ec6892a2b9..c1adf1932ec6b 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
@@ -18,6 +18,7 @@
 
 import org.apache.kafka.clients.ApiVersions;
 import org.apache.kafka.clients.ClientRequest;
+import org.apache.kafka.clients.GroupRebalanceConfig;
 import org.apache.kafka.clients.KafkaClient;
 import org.apache.kafka.clients.MockClient;
 import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator;
@@ -27,7 +28,6 @@
 import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient;
 import org.apache.kafka.clients.consumer.internals.ConsumerProtocol;
 import org.apache.kafka.clients.consumer.internals.Fetcher;
-import org.apache.kafka.clients.consumer.internals.Heartbeat;
 import org.apache.kafka.clients.consumer.internals.PartitionAssignor;
 import org.apache.kafka.clients.consumer.internals.SubscriptionState;
 import org.apache.kafka.common.Cluster;
@@ -1895,27 +1895,25 @@ private KafkaConsumer newConsumer(Time time,
         ConsumerNetworkClient consumerClient = new ConsumerNetworkClient(loggerFactory, client, metadata, time,
                 retryBackoffMs, requestTimeoutMs, heartbeatIntervalMs);
 
-        Heartbeat heartbeat = new Heartbeat(time, sessionTimeoutMs, heartbeatIntervalMs, rebalanceTimeoutMs, retryBackoffMs);
-        ConsumerCoordinator consumerCoordinator = new ConsumerCoordinator(
-                loggerFactory,
-                consumerClient,
-                groupId,
-                groupInstanceId,
-                rebalanceTimeoutMs,
-                sessionTimeoutMs,
-                heartbeat,
-                assignors,
-                metadata,
-                subscription,
-                metrics,
-                metricGroupPrefix,
-                time,
-                retryBackoffMs,
-                autoCommitEnabled,
-                autoCommitIntervalMs,
-                interceptors,
-                true);
-
+        GroupRebalanceConfig rebalanceConfig = new GroupRebalanceConfig(sessionTimeoutMs,
+                                                                        rebalanceTimeoutMs,
+                                                                        heartbeatIntervalMs,
+                                                                        groupId,
+                                                                        groupInstanceId,
+                                                                        retryBackoffMs,
+                                                                        true);
+        ConsumerCoordinator consumerCoordinator = new ConsumerCoordinator(rebalanceConfig,
+                                                                          loggerFactory,
+                                                                          consumerClient,
+                                                                          assignors,
+                                                                          metadata,
+                                                                          subscription,
+                                                                          metrics,
+                                                                          metricGroupPrefix,
+                                                                          time,
+                                                                          autoCommitEnabled,
+                                                                          autoCommitIntervalMs,
+                                                                          interceptors);
         Fetcher fetcher = new Fetcher<>(
                 loggerFactory,
                 consumerClient,
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
index 0fc5f62b69adb..659ef5f781ae8 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.clients.consumer.internals;
 
+import org.apache.kafka.clients.GroupRebalanceConfig;
 import org.apache.kafka.clients.MockClient;
 import org.apache.kafka.clients.consumer.OffsetResetStrategy;
 import org.apache.kafka.common.Node;
@@ -104,14 +105,30 @@ false, false, new SubscriptionState(logContext, OffsetResetStrategy.EARLIEST),
                 logContext, new ClusterResourceListeners());
 
         this.mockClient = new MockClient(mockTime, metadata);
-        this.consumerClient = new ConsumerNetworkClient(logContext, mockClient, metadata, mockTime,
-                retryBackoffMs, REQUEST_TIMEOUT_MS, HEARTBEAT_INTERVAL_MS);
+        this.consumerClient = new ConsumerNetworkClient(logContext,
+                                                        mockClient,
+                                                        metadata,
+                                                        mockTime,
+                                                        retryBackoffMs,
+                                                        REQUEST_TIMEOUT_MS,
+                                                        HEARTBEAT_INTERVAL_MS);
         Metrics metrics = new Metrics();
 
         mockClient.updateMetadata(TestUtils.metadataUpdateWith(1, emptyMap()));
         this.node = metadata.fetch().nodes().get(0);
         this.coordinatorNode = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port());
-        this.coordinator = new DummyCoordinator(consumerClient, metrics, mockTime, rebalanceTimeoutMs, retryBackoffMs, groupInstanceId);
+
+        GroupRebalanceConfig rebalanceConfig = new GroupRebalanceConfig(SESSION_TIMEOUT_MS,
+                                                                        rebalanceTimeoutMs,
+                                                                        HEARTBEAT_INTERVAL_MS,
+                                                                        GROUP_ID,
+                                                                        groupInstanceId,
+                                                                        retryBackoffMs,
+                                                                        !groupInstanceId.isPresent());
+        this.coordinator = new DummyCoordinator(rebalanceConfig,
+                                                consumerClient,
+                                                metrics,
+                                                mockTime);
     }
 
     @Test
@@ -850,14 +867,11 @@ public static class DummyCoordinator extends AbstractCoordinator {
         private int onJoinCompleteInvokes = 0;
         private boolean wakeupOnJoinComplete = false;
 
-        public DummyCoordinator(ConsumerNetworkClient client,
+        public DummyCoordinator(GroupRebalanceConfig rebalanceConfig,
+                                ConsumerNetworkClient client,
                                 Metrics metrics,
-                                Time time,
-                                int rebalanceTimeoutMs,
-                                int retryBackoffMs,
-                                Optional groupInstanceId) {
-            super(new LogContext(), client, GROUP_ID, groupInstanceId, rebalanceTimeoutMs,
-                    SESSION_TIMEOUT_MS, HEARTBEAT_INTERVAL_MS, metrics, METRIC_GROUP_PREFIX, time, retryBackoffMs, !groupInstanceId.isPresent());
+                                Time time) {
+            super(rebalanceConfig, new LogContext(), client, metrics, METRIC_GROUP_PREFIX, time);
         }
 
         @Override
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index 4b377ea2017f2..c54540eccf54a 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -17,6 +17,7 @@
 package org.apache.kafka.clients.consumer.internals;
 
 import org.apache.kafka.clients.ClientResponse;
+import org.apache.kafka.clients.GroupRebalanceConfig;
 import org.apache.kafka.clients.MockClient;
 import org.apache.kafka.clients.consumer.CommitFailedException;
 import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
@@ -118,8 +119,7 @@ public class ConsumerCoordinatorTest {
     private final int autoCommitIntervalMs = 2000;
     private final int requestTimeoutMs = 30000;
     private final MockTime time = new MockTime();
-    private final Heartbeat heartbeat = new Heartbeat(time, sessionTimeoutMs, heartbeatIntervalMs,
-            rebalanceTimeoutMs, retryBackoffMs);
+    private GroupRebalanceConfig rebalanceConfig;
 
     private MockPartitionAssignor partitionAssignor = new MockPartitionAssignor();
     private List assignors = Collections.singletonList(partitionAssignor);
@@ -153,8 +153,21 @@ public void setup() {
         this.rebalanceListener = new MockRebalanceListener();
         this.mockOffsetCommitCallback = new MockCommitCallback();
         this.partitionAssignor.clear();
+        this.rebalanceConfig = buildRebalanceConfig(Optional.empty());
+        this.coordinator = buildCoordinator(rebalanceConfig,
+                                            metrics,
+                                            assignors,
+                                            false);
+    }
 
-        this.coordinator = buildCoordinator(metrics, assignors, false, Optional.empty());
+    private GroupRebalanceConfig buildRebalanceConfig(Optional groupInstanceId) {
+        return new GroupRebalanceConfig(sessionTimeoutMs,
+                                        rebalanceTimeoutMs,
+                                        heartbeatIntervalMs,
+                                        groupId,
+                                        groupInstanceId,
+                                        retryBackoffMs,
+                                        !groupInstanceId.isPresent());
     }
 
     @After
@@ -741,10 +754,10 @@ public void testUpdateLastHeartbeatPollWhenCoordinatorUnknown() throws Exception
 
         assertTrue(coordinator.coordinatorUnknown());
         assertFalse(coordinator.poll(time.timer(0)));
-        assertEquals(time.milliseconds(), heartbeat.lastPollTime());
+        assertEquals(time.milliseconds(), coordinator.heartbeat().lastPollTime());
 
         time.sleep(rebalanceTimeoutMs - 1);
-        assertFalse(heartbeat.pollTimeoutExpired(time.milliseconds()));
+        assertFalse(coordinator.heartbeat().pollTimeoutExpired(time.milliseconds()));
     }
 
     @Test
@@ -1044,9 +1057,6 @@ public boolean matches(AbstractRequest body) {
 
     @Test
     public void testWakeupFromAssignmentCallback() {
-        ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors,
-                false, Optional.empty());
-
         final String topic = "topic1";
         TopicPartition partition = new TopicPartition(topic, 0);
         final String consumerId = "follower";
@@ -1162,7 +1172,10 @@ private void testInternalTopicInclusion(boolean includeInternalTopics) {
         metadata = new ConsumerMetadata(0, Long.MAX_VALUE, includeInternalTopics,
                 false, subscriptions, new LogContext(), new ClusterResourceListeners());
         client = new MockClient(time, metadata);
-        coordinator = buildCoordinator(new Metrics(), assignors, false, Optional.empty());
+        coordinator = buildCoordinator(rebalanceConfig,
+                                       new Metrics(),
+                                       assignors,
+                                       false);
 
         subscriptions.subscribe(Pattern.compile(".*"), rebalanceListener);
 
@@ -1290,8 +1303,10 @@ private void testInFlightRequestsFailedAfterCoordinatorMarkedDead(Errors error)
     public void testAutoCommitDynamicAssignment() {
         final String consumerId = "consumer";
 
-        ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors,
-                true, groupInstanceId);
+        ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig,
+                                                           new Metrics(),
+                                                           assignors,
+                                                           true);
 
         subscriptions.subscribe(singleton(topic1), rebalanceListener);
         joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p));
@@ -1306,8 +1321,10 @@ public void testAutoCommitDynamicAssignment() {
     @Test
     public void testAutoCommitRetryBackoff() {
         final String consumerId = "consumer";
-        ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors,
-                true, groupInstanceId);
+        ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig,
+                                                           new Metrics(),
+                                                           assignors,
+                                                           true);
         subscriptions.subscribe(singleton(topic1), rebalanceListener);
         joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p));
 
@@ -1340,7 +1357,10 @@ public void testAutoCommitRetryBackoff() {
     @Test
     public void testAutoCommitAwaitsInterval() {
         final String consumerId = "consumer";
-        ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, true, groupInstanceId);
+        ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig,
+                                                           new Metrics(),
+                                                           assignors,
+                                                           true);
         subscriptions.subscribe(singleton(topic1), rebalanceListener);
         joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p));
 
@@ -1378,8 +1398,10 @@ public void testAutoCommitAwaitsInterval() {
     public void testAutoCommitDynamicAssignmentRebalance() {
         final String consumerId = "consumer";
 
-        ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors,
-                true, groupInstanceId);
+        ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig,
+                                                           new Metrics(),
+                                                           assignors,
+                                                           true);
 
         subscriptions.subscribe(singleton(topic1), rebalanceListener);
 
@@ -1404,8 +1426,10 @@ public void testAutoCommitDynamicAssignmentRebalance() {
 
     @Test
     public void testAutoCommitManualAssignment() {
-        ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors,
-                true, groupInstanceId);
+        ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig,
+                                                           new Metrics(),
+                                                           assignors,
+                                                           true);
 
         subscriptions.assignFromUser(singleton(t1p));
         subscriptions.seek(t1p, 100);
@@ -1421,8 +1445,10 @@ public void testAutoCommitManualAssignment() {
 
     @Test
     public void testAutoCommitManualAssignmentCoordinatorUnknown() {
-        ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors,
-                true, groupInstanceId);
+        ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig,
+                                                           new Metrics(),
+                                                           assignors,
+                                                           true);
 
         subscriptions.assignFromUser(singleton(t1p));
         subscriptions.seek(t1p, 100);
@@ -2066,8 +2092,10 @@ public void testHeartbeatThreadClose() throws Exception {
 
     @Test
     public void testAutoCommitAfterCoordinatorBackToService() {
-        ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors,
-                true, groupInstanceId);
+        ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig,
+                                                           new Metrics(),
+                                                           assignors,
+                                                           true);
 
         subscriptions.assignFromUser(Collections.singleton(t1p));
         subscriptions.seek(t1p, 100L);
@@ -2125,8 +2153,11 @@ private ConsumerCoordinator prepareCoordinatorForCloseTest(final boolean useGrou
                                                                final boolean autoCommit,
                                                                final Optional groupInstanceId) {
         final String consumerId = "consumer";
-        ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors,
-                autoCommit, groupInstanceId);
+        rebalanceConfig = buildRebalanceConfig(groupInstanceId);
+        ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig,
+                                                           new Metrics(),
+                                                           assignors,
+                                                           autoCommit);
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
         if (useGroupManagement) {
@@ -2216,30 +2247,23 @@ public boolean matches(AbstractRequest body) {
         assertEquals("leaveGroupRequested should be " + shouldLeaveGroup, shouldLeaveGroup, leaveGroupRequested.get());
     }
 
-    private ConsumerCoordinator buildCoordinator(final Metrics metrics,
+    private ConsumerCoordinator buildCoordinator(final GroupRebalanceConfig rebalanceConfig,
+                                                 final Metrics metrics,
                                                  final List assignors,
-                                                 final boolean autoCommitEnabled,
-                                                 final Optional groupInstanceId) {
+                                                 final boolean autoCommitEnabled) {
         return new ConsumerCoordinator(
+                rebalanceConfig,
                 new LogContext(),
                 consumerClient,
-                groupId,
-                groupInstanceId,
-                rebalanceTimeoutMs,
-                sessionTimeoutMs,
-                heartbeat,
                 assignors,
                 metadata,
                 subscriptions,
                 metrics,
                 "consumer" + groupId,
                 time,
-                retryBackoffMs,
                 autoCommitEnabled,
                 autoCommitIntervalMs,
-                null,
-                !groupInstanceId.isPresent()
-        );
+                null);
     }
 
     private FindCoordinatorResponse groupCoordinatorResponse(Node node, Errors error) {
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatTest.java
index c382de610dcb9..b014bec637c1c 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatTest.java
@@ -16,23 +16,38 @@
  */
 package org.apache.kafka.clients.consumer.internals;
 
+import org.apache.kafka.clients.GroupRebalanceConfig;
 import org.apache.kafka.common.utils.MockTime;
 
+import org.junit.Before;
 import org.junit.Test;
 
+import java.util.Optional;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 public class HeartbeatTest {
-
     private int sessionTimeoutMs = 300;
     private int heartbeatIntervalMs = 100;
     private int maxPollIntervalMs = 900;
     private long retryBackoffMs = 10L;
     private MockTime time = new MockTime();
-    private Heartbeat heartbeat = new Heartbeat(time, sessionTimeoutMs, heartbeatIntervalMs,
-            maxPollIntervalMs, retryBackoffMs);
+
+    private Heartbeat heartbeat;
+
+    @Before
+    public void setUp() {
+        GroupRebalanceConfig rebalanceConfig = new GroupRebalanceConfig(sessionTimeoutMs,
+                                                                        maxPollIntervalMs,
+                                                                        heartbeatIntervalMs,
+                                                                        "group_id",
+                                                                        Optional.empty(),
+                                                                        retryBackoffMs,
+                                                                        true);
+        heartbeat = new Heartbeat(rebalanceConfig, time);
+    }
 
     @Test
     public void testShouldHeartbeat() {
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java
index c7ff6d82f8615..bc3bc19cce7f3 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java
@@ -39,13 +39,13 @@ public class DistributedConfig extends WorkerConfig {
     /**
      * group.id
      */
-    public static final String GROUP_ID_CONFIG = "group.id";
+    public static final String GROUP_ID_CONFIG = CommonClientConfigs.GROUP_ID_CONFIG;
     private static final String GROUP_ID_DOC = "A unique string that identifies the Connect cluster group this worker belongs to.";
 
     /**
      * session.timeout.ms
      */
-    public static final String SESSION_TIMEOUT_MS_CONFIG = "session.timeout.ms";
+    public static final String SESSION_TIMEOUT_MS_CONFIG = CommonClientConfigs.SESSION_TIMEOUT_MS_CONFIG;
     private static final String SESSION_TIMEOUT_MS_DOC = "The timeout used to detect worker failures. " +
             "The worker sends periodic heartbeats to indicate its liveness to the broker. If no heartbeats are " +
             "received by the broker before the expiration of this session timeout, then the broker will remove the " +
@@ -56,7 +56,7 @@ public class DistributedConfig extends WorkerConfig {
     /**
      * heartbeat.interval.ms
      */
-    public static final String HEARTBEAT_INTERVAL_MS_CONFIG = "heartbeat.interval.ms";
+    public static final String HEARTBEAT_INTERVAL_MS_CONFIG = CommonClientConfigs.HEARTBEAT_INTERVAL_MS_CONFIG;
     private static final String HEARTBEAT_INTERVAL_MS_DOC = "The expected time between heartbeats to the group " +
             "coordinator when using Kafka's group management facilities. Heartbeats are used to ensure that the " +
             "worker's session stays active and to facilitate rebalancing when new members join or leave the group. " +
@@ -66,11 +66,8 @@ public class DistributedConfig extends WorkerConfig {
     /**
      * rebalance.timeout.ms
      */
-    public static final String REBALANCE_TIMEOUT_MS_CONFIG = "rebalance.timeout.ms";
-    private static final String REBALANCE_TIMEOUT_MS_DOC = "The maximum allowed time for each worker to join the group " +
-            "once a rebalance has begun. This is basically a limit on the amount of time needed for all tasks to " +
-            "flush any pending data and commit offsets. If the timeout is exceeded, then the worker will be removed " +
-            "from the group, which will cause offset commit failures.";
+    public static final String REBALANCE_TIMEOUT_MS_CONFIG = CommonClientConfigs.REBALANCE_TIMEOUT_MS_CONFIG;
+    private static final String REBALANCE_TIMEOUT_MS_DOC = CommonClientConfigs.REBALANCE_TIMEOUT_MS_DOC;
 
     /**
      * worker.sync.timeout.ms
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java
index 230a2725ae46e..0b855a01e2dc2 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java
@@ -18,6 +18,7 @@
 
 import org.apache.kafka.clients.consumer.internals.AbstractCoordinator;
 import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient;
+import org.apache.kafka.clients.GroupRebalanceConfig;
 import org.apache.kafka.common.metrics.Measurable;
 import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
@@ -39,7 +40,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Optional;
 
 import static org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection;
 import static org.apache.kafka.common.message.JoinGroupResponseData.JoinGroupResponseMember;
@@ -70,33 +70,23 @@ public class WorkerCoordinator extends AbstractCoordinator implements Closeable
     /**
      * Initialize the coordination manager.
      */
-    public WorkerCoordinator(LogContext logContext,
+    public WorkerCoordinator(GroupRebalanceConfig config,
+                             LogContext logContext,
                              ConsumerNetworkClient client,
-                             String groupId,
-                             int rebalanceTimeoutMs,
-                             int sessionTimeoutMs,
-                             int heartbeatIntervalMs,
                              Metrics metrics,
                              String metricGrpPrefix,
                              Time time,
-                             long retryBackoffMs,
                              String restUrl,
                              ConfigBackingStore configStorage,
                              WorkerRebalanceListener listener,
                              ConnectProtocolCompatibility protocolCompatibility,
                              int maxDelay) {
-        super(logContext,
+        super(config,
+              logContext,
               client,
-              groupId,
-              Optional.empty(),
-              rebalanceTimeoutMs,
-              sessionTimeoutMs,
-              heartbeatIntervalMs,
               metrics,
               metricGrpPrefix,
-              time,
-              retryBackoffMs,
-              true);
+              time);
         this.log = logContext.logger(WorkerCoordinator.class);
         this.restUrl = restUrl;
         this.configStorage = configStorage;
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerGroupMember.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerGroupMember.java
index 99ea3a4e6e164..94cf97df1fa1a 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerGroupMember.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerGroupMember.java
@@ -23,6 +23,7 @@
 import org.apache.kafka.clients.Metadata;
 import org.apache.kafka.clients.NetworkClient;
 import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient;
+import org.apache.kafka.clients.GroupRebalanceConfig;
 import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.internals.ClusterResourceListeners;
 import org.apache.kafka.common.metrics.JmxReporter;
@@ -79,8 +80,6 @@ public WorkerGroupMember(DistributedConfig config,
             this.clientId = clientId;
             this.log = logContext.logger(WorkerGroupMember.class);
 
-            String groupId = config.getString(DistributedConfig.GROUP_ID_CONFIG);
-
             Map metricsTags = new LinkedHashMap<>();
             metricsTags.put("client-id", clientId);
             MetricConfig metricConfig = new MetricConfig().samples(config.getInt(CommonClientConfigs.METRICS_NUM_SAMPLES_CONFIG))
@@ -124,16 +123,12 @@ public WorkerGroupMember(DistributedConfig config,
                     config.getInt(CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG),
                     Integer.MAX_VALUE);
             this.coordinator = new WorkerCoordinator(
+                    new GroupRebalanceConfig(config, GroupRebalanceConfig.ProtocolType.CONNECT),
                     logContext,
                     this.client,
-                    groupId,
-                    config.getInt(DistributedConfig.REBALANCE_TIMEOUT_MS_CONFIG),
-                    config.getInt(DistributedConfig.SESSION_TIMEOUT_MS_CONFIG),
-                    config.getInt(DistributedConfig.HEARTBEAT_INTERVAL_MS_CONFIG),
                     metrics,
                     metricGrpPrefix,
                     this.time,
-                    retryBackoffMs,
                     restUrl,
                     configStorage,
                     listener,
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java
index f06976ae3744d..b1d3ec6147d98 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.connect.runtime.distributed;
 
+import org.apache.kafka.clients.GroupRebalanceConfig;
 import org.apache.kafka.clients.Metadata;
 import org.apache.kafka.clients.MockClient;
 import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient;
@@ -45,6 +46,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 import static org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocol;
 import static org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection;
@@ -93,6 +95,7 @@ public class WorkerCoordinatorIncrementalTest {
     private MockRebalanceListener rebalanceListener;
     @Mock
     private KafkaConfigBackingStore configStorage;
+    private GroupRebalanceConfig rebalanceConfig;
     private WorkerCoordinator coordinator;
     private int rebalanceDelay = DistributedConfig.SCHEDULED_REBALANCE_MAX_DELAY_MS_DEFAULT;
 
@@ -150,22 +153,24 @@ public void setup() {
 
         this.configStorageCalls = 0;
 
-        this.coordinator = new WorkerCoordinator(
-                loggerFactory,
-                consumerClient,
-                groupId,
-                rebalanceTimeoutMs,
-                sessionTimeoutMs,
-                heartbeatIntervalMs,
-                metrics,
-                "worker" + groupId,
-                time,
-                retryBackoffMs,
-                expectedUrl(leaderId),
-                configStorage,
-                rebalanceListener,
-                compatibility,
-                rebalanceDelay);
+        this.rebalanceConfig = new GroupRebalanceConfig(sessionTimeoutMs,
+                                                        rebalanceTimeoutMs,
+                                                        heartbeatIntervalMs,
+                                                        groupId,
+                                                        Optional.empty(),
+                                                        retryBackoffMs,
+                                                        true);
+        this.coordinator = new WorkerCoordinator(rebalanceConfig,
+                                                 loggerFactory,
+                                                 consumerClient,
+                                                 metrics,
+                                                 "worker" + groupId,
+                                                 time,
+                                                 expectedUrl(leaderId),
+                                                 configStorage,
+                                                 rebalanceListener,
+                                                 compatibility,
+                                                 rebalanceDelay);
 
         configState1 = clusterConfigState(offset, 2, 4);
     }
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorTest.java
index 182d6bd976478..eac13d158eb5c 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.connect.runtime.distributed;
 
+import org.apache.kafka.clients.GroupRebalanceConfig;
 import org.apache.kafka.clients.Metadata;
 import org.apache.kafka.clients.MockClient;
 import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient;
@@ -56,6 +57,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -90,6 +92,7 @@ public class WorkerCoordinatorTest {
     private ConsumerNetworkClient consumerClient;
     private MockRebalanceListener rebalanceListener;
     @Mock private KafkaConfigBackingStore configStorage;
+    private GroupRebalanceConfig rebalanceConfig;
     private WorkerCoordinator coordinator;
 
     private ClusterConfigState configState1;
@@ -125,23 +128,24 @@ public void setup() {
         this.metrics = new Metrics(time);
         this.rebalanceListener = new MockRebalanceListener();
         this.configStorage = PowerMock.createMock(KafkaConfigBackingStore.class);
-
-        this.coordinator = new WorkerCoordinator(
-                logContext,
-                consumerClient,
-                groupId,
-                rebalanceTimeoutMs,
-                sessionTimeoutMs,
-                heartbeatIntervalMs,
-                metrics,
-                "consumer" + groupId,
-                time,
-                retryBackoffMs,
-                LEADER_URL,
-                configStorage,
-                rebalanceListener,
-                compatibility,
-                0);
+        this.rebalanceConfig = new GroupRebalanceConfig(sessionTimeoutMs,
+                                                        rebalanceTimeoutMs,
+                                                        heartbeatIntervalMs,
+                                                        groupId,
+                                                        Optional.empty(),
+                                                        retryBackoffMs,
+                                                        true);
+        this.coordinator = new WorkerCoordinator(rebalanceConfig,
+                                                 logContext,
+                                                 consumerClient,
+                                                 metrics,
+                                                 "consumer" + groupId,
+                                                 time,
+                                                 LEADER_URL,
+                                                 configStorage,
+                                                 rebalanceListener,
+                                                 compatibility,
+                                                 0);
 
         configState1 = new ClusterConfigState(
                 1L,

From 47f908fa73fb7bbaec553635e75bffddd7a473f9 Mon Sep 17 00:00:00 2001
From: Boyang Chen 
Date: Mon, 17 Jun 2019 11:25:22 -0700
Subject: [PATCH 0373/1071] KAFKA-8539; Add group.instance.id to Subscription
 (#6936)

This PR is part of KIP-345's effort to utilize this new field for more stable topic partition assignment. We add the group instance id to the `Subscription` object to allow partition assignors to make stickier assignments. More details [here](https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances#KIP-345:Introducestaticmembershipprotocoltoreduceconsumerrebalances-ClientBehaviorChanges).

Reviewers: Jason Gustafson 
---
 .../internals/ConsumerCoordinator.java        |  7 ++--
 .../consumer/internals/ConsumerProtocol.java  | 30 ++++++++++-----
 .../consumer/internals/PartitionAssignor.java | 24 +++++++++---
 .../clients/consumer/KafkaConsumerTest.java   |  2 +-
 .../internals/ConsumerCoordinatorTest.java    |  2 +-
 .../internals/ConsumerProtocolTest.java       | 37 +++++++++++++++----
 6 files changed, 75 insertions(+), 27 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index a590a1e794ee6..b2b6f96984334 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -388,9 +388,10 @@ protected Map performAssignment(String leaderId,
 
         Set allSubscribedTopics = new HashSet<>();
         Map subscriptions = new HashMap<>();
-        for (JoinGroupResponseData.JoinGroupResponseMember memberSubScription : allSubscriptions) {
-            Subscription subscription = ConsumerProtocol.deserializeSubscription(ByteBuffer.wrap(memberSubScription.metadata()));
-            subscriptions.put(memberSubScription.memberId(), subscription);
+        for (JoinGroupResponseData.JoinGroupResponseMember memberSubscription : allSubscriptions) {
+            Subscription subscription = ConsumerProtocol.deserializeSubscription(ByteBuffer.wrap(memberSubscription.metadata()),
+                                                                                 Optional.ofNullable(memberSubscription.groupInstanceId()));
+            subscriptions.put(memberSubscription.memberId(), subscription);
             allSubscribedTopics.addAll(subscription.topics());
         }
 
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java
index b4ad4514eb60b..d3737f79e35eb 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java
@@ -27,8 +27,10 @@
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
 
@@ -177,17 +179,22 @@ public static ByteBuffer serializeSubscription(PartitionAssignor.Subscription su
         }
     }
 
-    public static PartitionAssignor.Subscription deserializeSubscriptionV0(ByteBuffer buffer) {
+    public static PartitionAssignor.Subscription deserializeSubscriptionV0(ByteBuffer buffer,
+                                                                           Optional groupInstanceId) {
         Struct struct = SUBSCRIPTION_V0.read(buffer);
         ByteBuffer userData = struct.getBytes(USER_DATA_KEY_NAME);
         List topics = new ArrayList<>();
         for (Object topicObj : struct.getArray(TOPICS_KEY_NAME))
             topics.add((String) topicObj);
-
-        return new PartitionAssignor.Subscription(CONSUMER_PROTOCOL_V0, topics, userData);
+        return new PartitionAssignor.Subscription(CONSUMER_PROTOCOL_V0,
+                                                  topics,
+                                                  userData,
+                                                  Collections.emptyList(),
+                                                  groupInstanceId);
     }
 
-    public static PartitionAssignor.Subscription deserializeSubscriptionV1(ByteBuffer buffer) {
+    public static PartitionAssignor.Subscription deserializeSubscriptionV1(ByteBuffer buffer,
+                                                                           Optional groupInstanceId) {
         Struct struct = SUBSCRIPTION_V1.read(buffer);
         ByteBuffer userData = struct.getBytes(USER_DATA_KEY_NAME);
         List topics = new ArrayList<>();
@@ -203,10 +210,15 @@ public static PartitionAssignor.Subscription deserializeSubscriptionV1(ByteBuffe
             }
         }
 
-        return new PartitionAssignor.Subscription(CONSUMER_PROTOCOL_V1, topics, userData, ownedPartitions);
+        return new PartitionAssignor.Subscription(CONSUMER_PROTOCOL_V1,
+                                                  topics,
+                                                  userData,
+                                                  ownedPartitions,
+                                                  groupInstanceId);
     }
 
-    public static PartitionAssignor.Subscription deserializeSubscription(ByteBuffer buffer) {
+    public static PartitionAssignor.Subscription deserializeSubscription(ByteBuffer buffer,
+                                                                         Optional groupInstanceId) {
         Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer);
         Short version = header.getShort(VERSION_KEY_NAME);
 
@@ -215,14 +227,14 @@ public static PartitionAssignor.Subscription deserializeSubscription(ByteBuffer
 
         switch (version) {
             case CONSUMER_PROTOCOL_V0:
-                return deserializeSubscriptionV0(buffer);
+                return deserializeSubscriptionV0(buffer, groupInstanceId);
 
             case CONSUMER_PROTOCOL_V1:
-                return deserializeSubscriptionV1(buffer);
+                return deserializeSubscriptionV1(buffer, groupInstanceId);
 
             // assume all higher versions can be parsed as V1
             default:
-                return deserializeSubscriptionV1(buffer);
+                return deserializeSubscriptionV1(buffer, groupInstanceId);
         }
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
index 5c76fd66dfef0..921a55bd715a2 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
@@ -24,6 +24,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 
 import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.CONSUMER_PROTOCOL_V0;
@@ -130,12 +131,18 @@ class Subscription {
         private final List topics;
         private final ByteBuffer userData;
         private final List ownedPartitions;
+        private final Optional groupInstanceId;
 
-        Subscription(Short version, List topics, ByteBuffer userData, List ownedPartitions) {
+        Subscription(Short version,
+                     List topics,
+                     ByteBuffer userData,
+                     List ownedPartitions,
+                     Optional groupInstanceId) {
             this.version = version;
             this.topics = topics;
             this.userData = userData;
             this.ownedPartitions = ownedPartitions;
+            this.groupInstanceId = groupInstanceId;
 
             if (version < CONSUMER_PROTOCOL_V0)
                 throw new SchemaException("Unsupported subscription version: " + version);
@@ -145,11 +152,14 @@ class Subscription {
         }
 
         Subscription(Short version, List topics, ByteBuffer userData) {
-            this(version, topics, userData, Collections.emptyList());
+            this(version, topics, userData, Collections.emptyList(), Optional.empty());
         }
 
-        public Subscription(List topics, ByteBuffer userData, List ownedPartitions) {
-            this(CONSUMER_PROTOCOL_V1, topics, userData, ownedPartitions);
+        public Subscription(List topics,
+                            ByteBuffer userData,
+                            List ownedPartitions,
+                            Optional groupInstanceId) {
+            this(CONSUMER_PROTOCOL_V1, topics, userData, ownedPartitions, groupInstanceId);
         }
 
         public Subscription(List topics, ByteBuffer userData) {
@@ -176,13 +186,17 @@ public ByteBuffer userData() {
             return userData;
         }
 
+        public Optional groupInstanceId() {
+            return groupInstanceId;
+        }
+
         @Override
         public String toString() {
             return "Subscription(" +
                     "version=" + version +
                     ", topics=" + topics +
                     ", ownedPartitions=" + ownedPartitions +
-                    ')';
+                    ", group.instance.id=" + groupInstanceId + ")";
         }
     }
 
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
index c1adf1932ec6b..cff71c34e4553 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
@@ -1692,7 +1692,7 @@ public boolean matches(AbstractRequest body) {
                 assertTrue(protocolIterator.hasNext());
 
                 ByteBuffer protocolMetadata = ByteBuffer.wrap(protocolIterator.next().metadata());
-                PartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(protocolMetadata);
+                PartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(protocolMetadata, Optional.empty());
                 return subscribedTopics.equals(new HashSet<>(subscription.topics()));
             }
         }, joinGroupFollowerResponse(assignor, 1, "memberId", "leaderId", Errors.NONE), coordinator);
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index c54540eccf54a..8aeec3c729953 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -600,7 +600,7 @@ public boolean matches(AbstractRequest body) {
                 JoinGroupRequestData.JoinGroupRequestProtocol protocolMetadata = protocolIterator.next();
 
                 ByteBuffer metadata = ByteBuffer.wrap(protocolMetadata.metadata());
-                PartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(metadata);
+                PartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(metadata, Optional.empty());
                 metadata.rewind();
                 return subscription.topics().containsAll(updatedSubscriptionSet);
             }
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java
index 8a8ba0a82e199..07eafa2b07994 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java
@@ -33,6 +33,7 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 
 import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.CONSUMER_PROTOCOL_HEADER_SCHEMA;
@@ -44,6 +45,7 @@
 import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.VERSION_KEY_NAME;
 import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
@@ -51,46 +53,64 @@ public class ConsumerProtocolTest {
 
     private final TopicPartition tp1 = new TopicPartition("foo", 1);
     private final TopicPartition tp2 = new TopicPartition("bar", 2);
+    private final Optional groupInstanceId = Optional.of("instance.id");
 
     @Test
     public void serializeDeserializeMetadata() {
         Subscription subscription = new Subscription(Arrays.asList("foo", "bar"));
         ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription);
-        Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer);
+        Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer, Optional.empty());
         assertEquals(subscription.topics(), parsedSubscription.topics());
         assertEquals(0, parsedSubscription.userData().limit());
+        assertFalse(parsedSubscription.groupInstanceId().isPresent());
+    }
+
+    @Test
+    public void serializeDeserializeMetadataAndGroupInstanceId() {
+        Subscription subscription = new Subscription(Arrays.asList("foo", "bar"));
+        ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription);
+
+        Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer, groupInstanceId);
+        assertEquals(subscription.topics(), parsedSubscription.topics());
+        assertEquals(groupInstanceId, parsedSubscription.groupInstanceId());
     }
 
     @Test
     public void serializeDeserializeNullSubscriptionUserData() {
         Subscription subscription = new Subscription(Arrays.asList("foo", "bar"), null);
         ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription);
-        Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer);
+        Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer, Optional.empty());
         assertEquals(subscription.topics(), parsedSubscription.topics());
         assertNull(parsedSubscription.userData());
+        assertFalse(parsedSubscription.groupInstanceId().isPresent());
     }
 
     @Test
     public void deserializeOldSubscriptionVersion() {
         Subscription subscription = new Subscription((short) 0, Arrays.asList("foo", "bar"), null);
         ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription);
-        Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer);
+        Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer, groupInstanceId);
         assertEquals(parsedSubscription.topics(), parsedSubscription.topics());
         assertNull(parsedSubscription.userData());
         assertTrue(parsedSubscription.ownedPartitions().isEmpty());
+        assertEquals(groupInstanceId, parsedSubscription.groupInstanceId());
     }
 
     @Test
     public void deserializeNewSubscriptionWithOldVersion() {
-        Subscription subscription = new Subscription((short) 1, Arrays.asList("foo", "bar"), null, Collections.singletonList(tp2));
+        Subscription subscription = new Subscription((short) 1,
+                                                     Arrays.asList("foo", "bar"),
+                                                     null, Collections.singletonList(tp2),
+                                                     Optional.empty());
         ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription);
         // ignore the version assuming it is the old byte code, as it will blindly deserialize as V0
         Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer);
         header.getShort(VERSION_KEY_NAME);
-        Subscription parsedSubscription = ConsumerProtocol.deserializeSubscriptionV0(buffer);
+        Subscription parsedSubscription = ConsumerProtocol.deserializeSubscriptionV0(buffer, Optional.empty());
         assertEquals(subscription.topics(), parsedSubscription.topics());
         assertNull(parsedSubscription.userData());
         assertTrue(parsedSubscription.ownedPartitions().isEmpty());
+        assertFalse(parsedSubscription.groupInstanceId().isPresent());
     }
 
     @Test
@@ -121,9 +141,10 @@ public void deserializeFutureSubscriptionVersion() {
 
         buffer.flip();
 
-        Subscription subscription = ConsumerProtocol.deserializeSubscription(buffer);
-        assertEquals(Collections.singletonList("topic"), subscription.topics());
-        assertEquals(Collections.singletonList(tp2), subscription.ownedPartitions());
+        Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer, groupInstanceId);
+        assertEquals(Collections.singletonList("topic"), parsedSubscription.topics());
+        assertEquals(Collections.singletonList(tp2), parsedSubscription.ownedPartitions());
+        assertEquals(groupInstanceId, parsedSubscription.groupInstanceId());
     }
 
     @Test

From 1ae92914e28919a97520e91bfd0e588d55eb1774 Mon Sep 17 00:00:00 2001
From: Boyang Chen 
Date: Mon, 17 Jun 2019 13:46:48 -0700
Subject: [PATCH 0374/1071] HOTFIX: Fix optional import in ConsumerCoordinator
 (#6953)

This was caused by back-to-back merging of #6854 (which removed the Optional import) and #6936 (which needed the import).

Reviewers: Jason Gustafson 
---
 .../kafka/clients/consumer/internals/ConsumerCoordinator.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index b2b6f96984334..de65a325f75e4 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -64,6 +64,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.atomic.AtomicBoolean;

From 52cd59bdb4de89c15f41984859469100f919dd6f Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Mon, 17 Jun 2019 14:30:05 -0700
Subject: [PATCH 0375/1071] MINOR: Simplify controller election utilities
 (#6944)

This patch simplifies the controller election API. We were passing `LeaderIsrAndControllerEpoch` into the election utilities even though we just needed `LeaderAndIsr`. We also remove some unneeded collection copies `doElectLeaderForPartitions`.

Reviewers: Ismael Juma 
---
 .../scala/kafka/controller/Election.scala     | 58 ++++++++--------
 .../controller/PartitionStateMachine.scala    | 68 +++++++++----------
 .../MockPartitionStateMachine.scala           | 40 +++++------
 3 files changed, 82 insertions(+), 84 deletions(-)

diff --git a/core/src/main/scala/kafka/controller/Election.scala b/core/src/main/scala/kafka/controller/Election.scala
index 3896e00bb9ba4..e30f69ad04fa5 100644
--- a/core/src/main/scala/kafka/controller/Election.scala
+++ b/core/src/main/scala/kafka/controller/Election.scala
@@ -24,21 +24,21 @@ case class ElectionResult(topicPartition: TopicPartition, leaderAndIsr: Option[L
 object Election {
 
   private def leaderForOffline(partition: TopicPartition,
-                               leaderIsrAndControllerEpochOpt: Option[LeaderIsrAndControllerEpoch],
+                               leaderAndIsrOpt: Option[LeaderAndIsr],
                                uncleanLeaderElectionEnabled: Boolean,
                                controllerContext: ControllerContext): ElectionResult = {
 
     val assignment = controllerContext.partitionReplicaAssignment(partition)
     val liveReplicas = assignment.filter(replica => controllerContext.isReplicaOnline(replica, partition))
-    leaderIsrAndControllerEpochOpt match {
-      case Some(leaderIsrAndControllerEpoch) =>
-        val isr = leaderIsrAndControllerEpoch.leaderAndIsr.isr
+    leaderAndIsrOpt match {
+      case Some(leaderAndIsr) =>
+        val isr = leaderAndIsr.isr
         val leaderOpt = PartitionLeaderElectionAlgorithms.offlinePartitionLeaderElection(
           assignment, isr, liveReplicas.toSet, uncleanLeaderElectionEnabled, controllerContext)
         val newLeaderAndIsrOpt = leaderOpt.map { leader =>
           val newIsr = if (isr.contains(leader)) isr.filter(replica => controllerContext.isReplicaOnline(replica, partition))
           else List(leader)
-          leaderIsrAndControllerEpoch.leaderAndIsr.newLeaderAndIsr(leader, newIsr)
+          leaderAndIsr.newLeaderAndIsr(leader, newIsr)
         }
         ElectionResult(partition, newLeaderAndIsrOpt, liveReplicas)
 
@@ -59,22 +59,22 @@ object Election {
    */
   def leaderForOffline(
     controllerContext: ControllerContext,
-    partitionsWithUncleanLeaderElectionState: Seq[(TopicPartition, Option[LeaderIsrAndControllerEpoch], Boolean)]
+    partitionsWithUncleanLeaderElectionState: Seq[(TopicPartition, Option[LeaderAndIsr], Boolean)]
   ): Seq[ElectionResult] = {
     partitionsWithUncleanLeaderElectionState.map {
-      case (partition, leaderIsrAndControllerEpochOpt, uncleanLeaderElectionEnabled) =>
-        leaderForOffline(partition, leaderIsrAndControllerEpochOpt, uncleanLeaderElectionEnabled, controllerContext)
+      case (partition, leaderAndIsrOpt, uncleanLeaderElectionEnabled) =>
+        leaderForOffline(partition, leaderAndIsrOpt, uncleanLeaderElectionEnabled, controllerContext)
     }
   }
 
   private def leaderForReassign(partition: TopicPartition,
-                                leaderIsrAndControllerEpoch: LeaderIsrAndControllerEpoch,
+                                leaderAndIsr: LeaderAndIsr,
                                 controllerContext: ControllerContext): ElectionResult = {
     val reassignment = controllerContext.partitionsBeingReassigned(partition).newReplicas
     val liveReplicas = reassignment.filter(replica => controllerContext.isReplicaOnline(replica, partition))
-    val isr = leaderIsrAndControllerEpoch.leaderAndIsr.isr
+    val isr = leaderAndIsr.isr
     val leaderOpt = PartitionLeaderElectionAlgorithms.reassignPartitionLeaderElection(reassignment, isr, liveReplicas.toSet)
-    val newLeaderAndIsrOpt = leaderOpt.map(leader => leaderIsrAndControllerEpoch.leaderAndIsr.newLeader(leader))
+    val newLeaderAndIsrOpt = leaderOpt.map(leader => leaderAndIsr.newLeader(leader))
     ElectionResult(partition, newLeaderAndIsrOpt, reassignment)
   }
 
@@ -82,26 +82,26 @@ object Election {
    * Elect leaders for partitions that are undergoing reassignment.
    *
    * @param controllerContext Context with the current state of the cluster
-   * @param leaderIsrAndControllerEpochs A sequence of tuples representing the partitions that need election
+   * @param leaderAndIsrs A sequence of tuples representing the partitions that need election
    *                                     and their respective leader/ISR states
    *
    * @return The election results
    */
   def leaderForReassign(controllerContext: ControllerContext,
-                        leaderIsrAndControllerEpochs: Seq[(TopicPartition, LeaderIsrAndControllerEpoch)]): Seq[ElectionResult] = {
-    leaderIsrAndControllerEpochs.map { case (partition, leaderIsrAndControllerEpoch) =>
-      leaderForReassign(partition, leaderIsrAndControllerEpoch, controllerContext)
+                        leaderAndIsrs: Seq[(TopicPartition, LeaderAndIsr)]): Seq[ElectionResult] = {
+    leaderAndIsrs.map { case (partition, leaderAndIsr) =>
+      leaderForReassign(partition, leaderAndIsr, controllerContext)
     }
   }
 
   private def leaderForPreferredReplica(partition: TopicPartition,
-                                        leaderIsrAndControllerEpoch: LeaderIsrAndControllerEpoch,
+                                        leaderAndIsr: LeaderAndIsr,
                                         controllerContext: ControllerContext): ElectionResult = {
     val assignment = controllerContext.partitionReplicaAssignment(partition)
     val liveReplicas = assignment.filter(replica => controllerContext.isReplicaOnline(replica, partition))
-    val isr = leaderIsrAndControllerEpoch.leaderAndIsr.isr
+    val isr = leaderAndIsr.isr
     val leaderOpt = PartitionLeaderElectionAlgorithms.preferredReplicaPartitionLeaderElection(assignment, isr, liveReplicas.toSet)
-    val newLeaderAndIsrOpt = leaderOpt.map(leader => leaderIsrAndControllerEpoch.leaderAndIsr.newLeader(leader))
+    val newLeaderAndIsrOpt = leaderOpt.map(leader => leaderAndIsr.newLeader(leader))
     ElectionResult(partition, newLeaderAndIsrOpt, assignment)
   }
 
@@ -109,30 +109,30 @@ object Election {
    * Elect preferred leaders.
    *
    * @param controllerContext Context with the current state of the cluster
-   * @param leaderIsrAndControllerEpochs A sequence of tuples representing the partitions that need election
+   * @param leaderAndIsrs A sequence of tuples representing the partitions that need election
    *                                     and their respective leader/ISR states
    *
    * @return The election results
    */
   def leaderForPreferredReplica(controllerContext: ControllerContext,
-                                leaderIsrAndControllerEpochs: Seq[(TopicPartition, LeaderIsrAndControllerEpoch)]): Seq[ElectionResult] = {
-    leaderIsrAndControllerEpochs.map { case (partition, leaderIsrAndControllerEpoch) =>
-      leaderForPreferredReplica(partition, leaderIsrAndControllerEpoch, controllerContext)
+                                leaderAndIsrs: Seq[(TopicPartition, LeaderAndIsr)]): Seq[ElectionResult] = {
+    leaderAndIsrs.map { case (partition, leaderAndIsr) =>
+      leaderForPreferredReplica(partition, leaderAndIsr, controllerContext)
     }
   }
 
   private def leaderForControlledShutdown(partition: TopicPartition,
-                                          leaderIsrAndControllerEpoch: LeaderIsrAndControllerEpoch,
+                                          leaderAndIsr: LeaderAndIsr,
                                           shuttingDownBrokerIds: Set[Int],
                                           controllerContext: ControllerContext): ElectionResult = {
     val assignment = controllerContext.partitionReplicaAssignment(partition)
     val liveOrShuttingDownReplicas = assignment.filter(replica =>
       controllerContext.isReplicaOnline(replica, partition, includeShuttingDownBrokers = true))
-    val isr = leaderIsrAndControllerEpoch.leaderAndIsr.isr
+    val isr = leaderAndIsr.isr
     val leaderOpt = PartitionLeaderElectionAlgorithms.controlledShutdownPartitionLeaderElection(assignment, isr,
       liveOrShuttingDownReplicas.toSet, shuttingDownBrokerIds)
     val newIsr = isr.filter(replica => !shuttingDownBrokerIds.contains(replica))
-    val newLeaderAndIsrOpt = leaderOpt.map(leader => leaderIsrAndControllerEpoch.leaderAndIsr.newLeaderAndIsr(leader, newIsr))
+    val newLeaderAndIsrOpt = leaderOpt.map(leader => leaderAndIsr.newLeaderAndIsr(leader, newIsr))
     ElectionResult(partition, newLeaderAndIsrOpt, liveOrShuttingDownReplicas)
   }
 
@@ -140,16 +140,16 @@ object Election {
    * Elect leaders for partitions whose current leaders are shutting down.
    *
    * @param controllerContext Context with the current state of the cluster
-   * @param leaderIsrAndControllerEpochs A sequence of tuples representing the partitions that need election
+   * @param leaderAndIsrs A sequence of tuples representing the partitions that need election
    *                                     and their respective leader/ISR states
    *
    * @return The election results
    */
   def leaderForControlledShutdown(controllerContext: ControllerContext,
-                                  leaderIsrAndControllerEpochs: Seq[(TopicPartition, LeaderIsrAndControllerEpoch)]): Seq[ElectionResult] = {
+                                  leaderAndIsrs: Seq[(TopicPartition, LeaderAndIsr)]): Seq[ElectionResult] = {
     val shuttingDownBrokerIds = controllerContext.shuttingDownBrokerIds.toSet
-    leaderIsrAndControllerEpochs.map { case (partition, leaderIsrAndControllerEpoch) =>
-      leaderForControlledShutdown(partition, leaderIsrAndControllerEpoch, shuttingDownBrokerIds, controllerContext)
+    leaderAndIsrs.map { case (partition, leaderAndIsr) =>
+      leaderForControlledShutdown(partition, leaderAndIsr, shuttingDownBrokerIds, controllerContext)
     }
   }
 }
diff --git a/core/src/main/scala/kafka/controller/PartitionStateMachine.scala b/core/src/main/scala/kafka/controller/PartitionStateMachine.scala
index ab4e8d4b4365d..fca78ef4afb8c 100755
--- a/core/src/main/scala/kafka/controller/PartitionStateMachine.scala
+++ b/core/src/main/scala/kafka/controller/PartitionStateMachine.scala
@@ -371,17 +371,28 @@ class ZkPartitionStateMachine(config: KafkaConfig,
         return (partitions.map(_ -> Left(e))(breakOut), Seq.empty)
     }
     val failedElections = mutable.Map.empty[TopicPartition, Either[Exception, LeaderAndIsr]]
-    val leaderIsrAndControllerEpochPerPartition = mutable.Buffer.empty[(TopicPartition, LeaderIsrAndControllerEpoch)]
+    val validLeaderAndIsrs = mutable.Buffer.empty[(TopicPartition, LeaderAndIsr)]
+
     getDataResponses.foreach { getDataResponse =>
       val partition = getDataResponse.ctx.get.asInstanceOf[TopicPartition]
       val currState = partitionState(partition)
       if (getDataResponse.resultCode == Code.OK) {
-        val leaderIsrAndControllerEpochOpt = TopicPartitionStateZNode.decode(getDataResponse.data, getDataResponse.stat)
-        if (leaderIsrAndControllerEpochOpt.isEmpty) {
-          val exception = new StateChangeFailedException(s"LeaderAndIsr information doesn't exist for partition $partition in $currState state")
-          failedElections.put(partition, Left(exception))
+        TopicPartitionStateZNode.decode(getDataResponse.data, getDataResponse.stat) match {
+          case Some(leaderIsrAndControllerEpoch) =>
+            if (leaderIsrAndControllerEpoch.controllerEpoch > controllerContext.epoch) {
+              val failMsg = s"Aborted leader election for partition $partition since the LeaderAndIsr path was " +
+                s"already written by another controller. This probably means that the current controller $controllerId went through " +
+                s"a soft failure and another controller was elected with epoch ${leaderIsrAndControllerEpoch.controllerEpoch}."
+              failedElections.put(partition, Left(new StateChangeFailedException(failMsg)))
+            } else {
+              validLeaderAndIsrs += partition -> leaderIsrAndControllerEpoch.leaderAndIsr
+            }
+
+          case None =>
+            val exception = new StateChangeFailedException(s"LeaderAndIsr information doesn't exist for partition $partition in $currState state")
+            failedElections.put(partition, Left(exception))
         }
-        leaderIsrAndControllerEpochPerPartition += partition -> leaderIsrAndControllerEpochOpt.get
+
       } else if (getDataResponse.resultCode == Code.NONODE) {
         val exception = new StateChangeFailedException(s"LeaderAndIsr information doesn't exist for partition $partition in $currState state")
         failedElections.put(partition, Left(exception))
@@ -390,33 +401,23 @@ class ZkPartitionStateMachine(config: KafkaConfig,
       }
     }
 
-    val (invalidPartitionsForElection, validPartitionsForElection) = leaderIsrAndControllerEpochPerPartition.partition { case (_, leaderIsrAndControllerEpoch) =>
-      leaderIsrAndControllerEpoch.controllerEpoch > controllerContext.epoch
-    }
-    invalidPartitionsForElection.foreach { case (partition, leaderIsrAndControllerEpoch) =>
-      val failMsg = s"aborted leader election for partition $partition since the LeaderAndIsr path was " +
-        s"already written by another controller. This probably means that the current controller $controllerId went through " +
-        s"a soft failure and another controller was elected with epoch ${leaderIsrAndControllerEpoch.controllerEpoch}."
-      failedElections.put(partition, Left(new StateChangeFailedException(failMsg)))
-    }
-
-    if (validPartitionsForElection.isEmpty) {
+    if (validLeaderAndIsrs.isEmpty) {
       return (failedElections.toMap, Seq.empty)
     }
 
     val (partitionsWithoutLeaders, partitionsWithLeaders) = partitionLeaderElectionStrategy match {
       case OfflinePartitionLeaderElectionStrategy(allowUnclean) =>
         val partitionsWithUncleanLeaderElectionState = collectUncleanLeaderElectionState(
-          validPartitionsForElection,
+          validLeaderAndIsrs,
           allowUnclean
         )
         leaderForOffline(controllerContext, partitionsWithUncleanLeaderElectionState).partition(_.leaderAndIsr.isEmpty)
       case ReassignPartitionLeaderElectionStrategy =>
-        leaderForReassign(controllerContext, validPartitionsForElection).partition(_.leaderAndIsr.isEmpty)
+        leaderForReassign(controllerContext, validLeaderAndIsrs).partition(_.leaderAndIsr.isEmpty)
       case PreferredReplicaPartitionLeaderElectionStrategy =>
-        leaderForPreferredReplica(controllerContext, validPartitionsForElection).partition(_.leaderAndIsr.isEmpty)
+        leaderForPreferredReplica(controllerContext, validLeaderAndIsrs).partition(_.leaderAndIsr.isEmpty)
       case ControlledShutdownPartitionLeaderElectionStrategy =>
-        leaderForControlledShutdown(controllerContext, validPartitionsForElection).partition(_.leaderAndIsr.isEmpty)
+        leaderForControlledShutdown(controllerContext, validLeaderAndIsrs).partition(_.leaderAndIsr.isEmpty)
     }
     partitionsWithoutLeaders.foreach { electionResult =>
       val partition = electionResult.topicPartition
@@ -454,21 +455,18 @@ class ZkPartitionStateMachine(config: KafkaConfig,
    *         3. allow unclean
    */
   private def collectUncleanLeaderElectionState(
-    leaderIsrAndControllerEpochs: Seq[(TopicPartition, LeaderIsrAndControllerEpoch)],
+    leaderAndIsrs: Seq[(TopicPartition, LeaderAndIsr)],
     allowUnclean: Boolean
-  ): Seq[(TopicPartition, Option[LeaderIsrAndControllerEpoch], Boolean)] = {
-    val (partitionsWithNoLiveInSyncReplicas, partitionsWithLiveInSyncReplicas) = leaderIsrAndControllerEpochs.partition {
-      case (partition, leaderIsrAndControllerEpoch) =>
-        val liveInSyncReplicas = leaderIsrAndControllerEpoch
-          .leaderAndIsr
-          .isr
-          .filter(replica => controllerContext.isReplicaOnline(replica, partition))
+  ): Seq[(TopicPartition, Option[LeaderAndIsr], Boolean)] = {
+    val (partitionsWithNoLiveInSyncReplicas, partitionsWithLiveInSyncReplicas) = leaderAndIsrs.partition {
+      case (partition, leaderAndIsr) =>
+        val liveInSyncReplicas = leaderAndIsr.isr.filter(controllerContext.isReplicaOnline(_, partition))
         liveInSyncReplicas.isEmpty
     }
 
     val electionForPartitionWithoutLiveReplicas = if (allowUnclean) {
-      partitionsWithNoLiveInSyncReplicas.map { case (partition, leaderIsrAndControllerEpoch) =>
-        (partition, Option(leaderIsrAndControllerEpoch), true)
+      partitionsWithNoLiveInSyncReplicas.map { case (partition, leaderAndIsr) =>
+        (partition, Option(leaderAndIsr), true)
       }
     } else {
       val (logConfigs, failed) = zkClient.getLogConfigs(
@@ -476,14 +474,14 @@ class ZkPartitionStateMachine(config: KafkaConfig,
         config.originals()
       )
 
-      partitionsWithNoLiveInSyncReplicas.map { case (partition, leaderIsrAndControllerEpoch) =>
+      partitionsWithNoLiveInSyncReplicas.map { case (partition, leaderAndIsr) =>
         if (failed.contains(partition.topic)) {
           logFailedStateChange(partition, partitionState(partition), OnlinePartition, failed(partition.topic))
           (partition, None, false)
         } else {
           (
             partition,
-            Option(leaderIsrAndControllerEpoch),
+            Option(leaderAndIsr),
             logConfigs(partition.topic).uncleanLeaderElectionEnable.booleanValue()
           )
         }
@@ -491,8 +489,8 @@ class ZkPartitionStateMachine(config: KafkaConfig,
     }
 
     electionForPartitionWithoutLiveReplicas ++
-    partitionsWithLiveInSyncReplicas.map { case (partition, leaderIsrAndControllerEpoch) =>
-      (partition, Option(leaderIsrAndControllerEpoch), false)
+    partitionsWithLiveInSyncReplicas.map { case (partition, leaderAndIsr) =>
+      (partition, Option(leaderAndIsr), false)
     }
   }
 
diff --git a/core/src/test/scala/unit/kafka/controller/MockPartitionStateMachine.scala b/core/src/test/scala/unit/kafka/controller/MockPartitionStateMachine.scala
index 0e2c8bef1c51e..94fd1773ebc2d 100644
--- a/core/src/test/scala/unit/kafka/controller/MockPartitionStateMachine.scala
+++ b/core/src/test/scala/unit/kafka/controller/MockPartitionStateMachine.scala
@@ -20,7 +20,8 @@ import kafka.api.LeaderAndIsr
 import kafka.common.StateChangeFailedException
 import kafka.controller.Election._
 import org.apache.kafka.common.TopicPartition
-import scala.collection.breakOut
+
+import scala.collection.{breakOut, mutable}
 
 class MockPartitionStateMachine(controllerContext: ControllerContext,
                                 uncleanLeaderElectionEnabled: Boolean)
@@ -68,34 +69,33 @@ class MockPartitionStateMachine(controllerContext: ControllerContext,
     partitions: Seq[TopicPartition],
     leaderElectionStrategy: PartitionLeaderElectionStrategy
   ): Map[TopicPartition, Either[Throwable, LeaderAndIsr]] = {
-    val leaderIsrAndControllerEpochPerPartition = partitions.map { partition =>
-      partition -> controllerContext.partitionLeadershipInfo(partition)
-    }
-
-    val (invalidPartitionsForElection, validPartitionsForElection) = leaderIsrAndControllerEpochPerPartition.partition { case (_, leaderIsrAndControllerEpoch) =>
-      leaderIsrAndControllerEpoch.controllerEpoch > controllerContext.epoch
-    }
-
-    val failedElections = invalidPartitionsForElection.map { case (partition, leaderIsrAndControllerEpoch) =>
-      val failMsg = s"aborted leader election for partition $partition since the LeaderAndIsr path was " +
-        s"already written by another controller. This probably means that the current controller went through " +
-        s"a soft failure and another controller was elected with epoch ${leaderIsrAndControllerEpoch.controllerEpoch}."
-
-      partition -> Left(new StateChangeFailedException(failMsg))
+    val failedElections = mutable.Map.empty[TopicPartition, Either[Throwable, LeaderAndIsr]]
+    val validLeaderAndIsrs = mutable.Buffer.empty[(TopicPartition, LeaderAndIsr)]
+
+    for (partition <- partitions) {
+      val leaderIsrAndControllerEpoch = controllerContext.partitionLeadershipInfo(partition)
+      if (leaderIsrAndControllerEpoch.controllerEpoch > controllerContext.epoch) {
+        val failMsg = s"Aborted leader election for partition $partition since the LeaderAndIsr path was " +
+          s"already written by another controller. This probably means that the current controller went through " +
+          s"a soft failure and another controller was elected with epoch ${leaderIsrAndControllerEpoch.controllerEpoch}."
+        failedElections.put(partition, Left(new StateChangeFailedException(failMsg)))
+      } else {
+        validLeaderAndIsrs.append((partition, leaderIsrAndControllerEpoch.leaderAndIsr))
+      }
     }
 
     val electionResults = leaderElectionStrategy match {
       case OfflinePartitionLeaderElectionStrategy(isUnclean) =>
-        val partitionsWithUncleanLeaderElectionState = validPartitionsForElection.map { case (partition, leaderIsrAndControllerEpoch) =>
-          (partition, Some(leaderIsrAndControllerEpoch), isUnclean || uncleanLeaderElectionEnabled)
+        val partitionsWithUncleanLeaderElectionState = validLeaderAndIsrs.map { case (partition, leaderAndIsr) =>
+          (partition, Some(leaderAndIsr), isUnclean || uncleanLeaderElectionEnabled)
         }
         leaderForOffline(controllerContext, partitionsWithUncleanLeaderElectionState)
       case ReassignPartitionLeaderElectionStrategy =>
-        leaderForReassign(controllerContext, validPartitionsForElection)
+        leaderForReassign(controllerContext, validLeaderAndIsrs)
       case PreferredReplicaPartitionLeaderElectionStrategy =>
-        leaderForPreferredReplica(controllerContext, validPartitionsForElection)
+        leaderForPreferredReplica(controllerContext, validLeaderAndIsrs)
       case ControlledShutdownPartitionLeaderElectionStrategy =>
-        leaderForControlledShutdown(controllerContext, validPartitionsForElection)
+        leaderForControlledShutdown(controllerContext, validLeaderAndIsrs)
     }
 
     val results: Map[TopicPartition, Either[Exception, LeaderAndIsr]] = electionResults.map { electionResult =>

From 6d6366cd5563ba2a7bd894611ef037cdea99273a Mon Sep 17 00:00:00 2001
From: Florian Hussonnois 
Date: Tue, 18 Jun 2019 00:02:52 +0200
Subject: [PATCH 0376/1071] KAFKA-6958: Overload KTable methods to allow to
 name operation name using the new Named class (#6412)

Sub-task required to allow to define custom processor names with KStreams DSL(KIP-307). This is the 4th PR for KIP-307.

Reviewers: John Roesler , Bill Bejeck 
---
 .../apache/kafka/streams/kstream/KTable.java  | 1177 +++++++++++++++--
 .../streams/kstream/internals/KTableImpl.java |  193 ++-
 .../kafka/streams/StreamsBuilderTest.java     |   29 +
 .../kstream/internals/KTableImplTest.java     |    2 +-
 4 files changed, 1272 insertions(+), 129 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java
index 36cd17b2de74d..39e5e225b2c79 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java
@@ -90,6 +90,68 @@ public interface KTable {
      */
     KTable filter(final Predicate predicate);
 
+    /**
+     * Create a new {@code KTable} that consists of all records of this {@code KTable} which satisfy the given
+     * predicate, with default serializers, deserializers, and state store.
+     * All records that do not satisfy the predicate are dropped.
+     * For each {@code KTable} update, the filter is evaluated based on the current update
+     * record and then an update record is produced for the result {@code KTable}.
+     * This is a stateless record-by-record operation.
+     * 
    
+     * Note that {@code filter} for a changelog stream works differently than {@link KStream#filter(Predicate)
+     * record stream filters}, because {@link KeyValue records} with {@code null} values (so-called tombstone records)
+     * have delete semantics.
+     * Thus, for tombstones the provided filter predicate is not evaluated but the tombstone record is forwarded
+     * directly if required (i.e., if there is anything to be deleted).
+     * Furthermore, for each record that gets dropped (i.e., does not satisfy the given predicate) a tombstone record
+     * is forwarded.
+     *
+     * @param predicate a filter {@link Predicate} that is applied to each record
+     * @param named     a {@link Named} config used to name the processor in the topology
+     * @return a {@code KTable} that contains only those records that satisfy the given predicate
+     * @see #filterNot(Predicate)
+     */
+    KTable filter(final Predicate predicate, final Named named);
+
+    /**
+     * Create a new {@code KTable} that consists of all records of this {@code KTable} which satisfy the given
+     * predicate, with the {@link Serde key serde}, {@link Serde value serde}, and the underlying
+     * {@link KeyValueStore materialized state storage} configured in the {@link Materialized} instance.
+     * All records that do not satisfy the predicate are dropped.
+     * For each {@code KTable} update, the filter is evaluated based on the current update
+     * record and then an update record is produced for the result {@code KTable}.
+     * This is a stateless record-by-record operation.
+     * 
    
+     * Note that {@code filter} for a changelog stream works differently than {@link KStream#filter(Predicate)
+     * record stream filters}, because {@link KeyValue records} with {@code null} values (so-called tombstone records)
+     * have delete semantics.
+     * Thus, for tombstones the provided filter predicate is not evaluated but the tombstone record is forwarded
+     * directly if required (i.e., if there is anything to be deleted).
+     * Furthermore, for each record that gets dropped (i.e., does not satisfy the given predicate) a tombstone record
+     * is forwarded.
+     * 
    
+     * To query the local {@link KeyValueStore} it must be obtained via
+     * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}:
+     * 
    {@code
+     * KafkaStreams streams = ... // filtering words
+     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
+     * K key = "some-word";
+     * V valueForKey = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
+     * }
    
+     * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
+     * query the value of the key on a parallel running instance of your Kafka Streams application.
+     * The store name to query with is specified by {@link Materialized#as(String)} or {@link Materialized#as(KeyValueBytesStoreSupplier)}.
+     * 
    
+     *
+     * @param predicate     a filter {@link Predicate} that is applied to each record
+     * @param materialized  a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
+     *                      should be materialized. Cannot be {@code null}
+     * @return a {@code KTable} that contains only those records that satisfy the given predicate
+     * @see #filterNot(Predicate, Materialized)
+     */
+    KTable filter(final Predicate predicate,
+                        final Materialized> materialized);
+
     /**
      * Create a new {@code KTable} that consists of all records of this {@code KTable} which satisfy the given
      * predicate, with the {@link Serde key serde}, {@link Serde value serde}, and the underlying
@@ -121,12 +183,14 @@ public interface KTable {
      * 
    
      *
      * @param predicate     a filter {@link Predicate} that is applied to each record
+     * @param named         a {@link Named} config used to name the processor in the topology
      * @param materialized  a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
      *                      should be materialized. Cannot be {@code null}
      * @return a {@code KTable} that contains only those records that satisfy the given predicate
      * @see #filterNot(Predicate, Materialized)
      */
     KTable filter(final Predicate predicate,
+                        final Named named,
                         final Materialized> materialized);
 
     /**
@@ -151,6 +215,29 @@ KTable filter(final Predicate predicate,
      */
     KTable filterNot(final Predicate predicate);
 
+    /**
+     * Create a new {@code KTable} that consists all records of this {@code KTable} which do not satisfy the
+     * given predicate, with default serializers, deserializers, and state store.
+     * All records that do satisfy the predicate are dropped.
+     * For each {@code KTable} update, the filter is evaluated based on the current update
+     * record and then an update record is produced for the result {@code KTable}.
+     * This is a stateless record-by-record operation.
+     * 
    
+     * Note that {@code filterNot} for a changelog stream works differently than {@link KStream#filterNot(Predicate)
+     * record stream filters}, because {@link KeyValue records} with {@code null} values (so-called tombstone records)
+     * have delete semantics.
+     * Thus, for tombstones the provided filter predicate is not evaluated but the tombstone record is forwarded
+     * directly if required (i.e., if there is anything to be deleted).
+     * Furthermore, for each record that gets dropped (i.e., does satisfy the given predicate) a tombstone record is
+     * forwarded.
+     *
+     * @param predicate a filter {@link Predicate} that is applied to each record
+     * @param named     a {@link Named} config used to name the processor in the topology
+     * @return a {@code KTable} that contains only those records that do not satisfy the given predicate
+     * @see #filter(Predicate)
+     */
+    KTable filterNot(final Predicate predicate, final Named named);
+
     /**
      * Create a new {@code KTable} that consists all records of this {@code KTable} which do not satisfy the
      * given predicate, with the {@link Serde key serde}, {@link Serde value serde}, and the underlying
@@ -189,6 +276,45 @@ KTable filter(final Predicate predicate,
     KTable filterNot(final Predicate predicate,
                            final Materialized> materialized);
 
+    /**
+     * Create a new {@code KTable} that consists all records of this {@code KTable} which do not satisfy the
+     * given predicate, with the {@link Serde key serde}, {@link Serde value serde}, and the underlying
+     * {@link KeyValueStore materialized state storage} configured in the {@link Materialized} instance.
+     * All records that do satisfy the predicate are dropped.
+     * For each {@code KTable} update, the filter is evaluated based on the current update
+     * record and then an update record is produced for the result {@code KTable}.
+     * This is a stateless record-by-record operation.
+     * 
    
+     * Note that {@code filterNot} for a changelog stream works differently than {@link KStream#filterNot(Predicate)
+     * record stream filters}, because {@link KeyValue records} with {@code null} values (so-called tombstone records)
+     * have delete semantics.
+     * Thus, for tombstones the provided filter predicate is not evaluated but the tombstone record is forwarded
+     * directly if required (i.e., if there is anything to be deleted).
+     * Furthermore, for each record that gets dropped (i.e., does satisfy the given predicate) a tombstone record is
+     * forwarded.
+     * 
    
+     * To query the local {@link KeyValueStore} it must be obtained via
+     * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}:
+     * 
    {@code
+     * KafkaStreams streams = ... // filtering words
+     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
+     * K key = "some-word";
+     * V valueForKey = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
+     * }
    
+     * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
+     * query the value of the key on a parallel running instance of your Kafka Streams application.
+     * The store name to query with is specified by {@link Materialized#as(String)} or {@link Materialized#as(KeyValueBytesStoreSupplier)}.
+     * 
    
+     * @param predicate a filter {@link Predicate} that is applied to each record
+     * @param named     a {@link Named} config used to name the processor in the topology
+     * @param materialized  a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
+     *                      should be materialized. Cannot be {@code null}
+     * @return a {@code KTable} that contains only those records that do not satisfy the given predicate
+     * @see #filter(Predicate, Materialized)
+     */
+    KTable filterNot(final Predicate predicate,
+                           final Named named,
+                           final Materialized> materialized);
 
     /**
      * Create a new {@code KTable} by transforming the value of each record in this {@code KTable} into a new value
@@ -201,11 +327,7 @@ KTable filterNot(final Predicate predicate,
      * The example below counts the number of token of the value string.
      * 
    {@code
      * KTable inputTable = builder.table("topic");
-     * KTable outputTable = inputTable.mapValue(new ValueMapper {
-     *     Integer apply(String value) {
-     *         return value.split(" ").length;
-     *     }
-     * });
+     * KTable outputTable = inputTable.mapValues(value -> value.split(" ").length);
      * }
    
      * 
    
      * This operation preserves data co-location with respect to the key.
@@ -224,6 +346,38 @@ KTable filterNot(final Predicate predicate,
      */
      KTable mapValues(final ValueMapper mapper);
 
+    /**
+     * Create a new {@code KTable} by transforming the value of each record in this {@code KTable} into a new value
+     * (with possibly a new type) in the new {@code KTable}, with default serializers, deserializers, and state store.
+     * For each {@code KTable} update the provided {@link ValueMapper} is applied to the value of the updated record and
+     * computes a new value for it, resulting in an updated record for the result {@code KTable}.
+     * Thus, an input record {@code } can be transformed into an output record {@code }.
+     * This is a stateless record-by-record operation.
+     * 
    
+     * The example below counts the number of token of the value string.
+     * 
    {@code
+     * KTable inputTable = builder.table("topic");
+     * KTable outputTable = inputTable.mapValues(value -> value.split(" ").length, Named.as("countTokenValue"));
+     * }
    
+     * 
    
+     * This operation preserves data co-location with respect to the key.
+     * Thus, no internal data redistribution is required if a key based operator (like a join) is applied to
+     * the result {@code KTable}.
+     * 
    
+     * Note that {@code mapValues} for a changelog stream works differently than {@link KStream#mapValues(ValueMapper)
+     * record stream filters}, because {@link KeyValue records} with {@code null} values (so-called tombstone records)
+     * have delete semantics.
+     * Thus, for tombstones the provided value-mapper is not evaluated but the tombstone record is forwarded directly to
+     * delete the corresponding record in the result {@code KTable}.
+     *
+     * @param mapper a {@link ValueMapper} that computes a new output value
+     * @param named  a {@link Named} config used to name the processor in the topology
+     * @param    the value type of the result {@code KTable}
+     * @return a {@code KTable} that contains records with unmodified keys and new values (possibly of different type)
+     */
+     KTable mapValues(final ValueMapper mapper,
+                                 final Named named);
+
     /**
      * Create a new {@code KTable} by transforming the value of each record in this {@code KTable} into a new value
      * (with possibly a new type) in the new {@code KTable}, with default serializers, deserializers, and state store.
@@ -235,11 +389,8 @@ KTable filterNot(final Predicate predicate,
      * The example below counts the number of token of value and key strings.
      * 
    {@code
      * KTable inputTable = builder.table("topic");
-     * KTable outputTable = inputTable.mapValue(new ValueMapperWithKey {
-     *     Integer apply(String readOnlyKey, String value) {
-     *          return readOnlyKey.split(" ").length + value.split(" ").length;
-     *     }
-     * });
+     * KTable outputTable =
+     *  inputTable.mapValues((readOnlyKey, value) -> readOnlyKey.split(" ").length + value.split(" ").length);
      * }
    
      * 
    
      * Note that the key is read-only and should not be modified, as this can lead to corrupt partitioning.
@@ -259,6 +410,86 @@ KTable filterNot(final Predicate predicate,
      */
      KTable mapValues(final ValueMapperWithKey mapper);
 
+    /**
+     * Create a new {@code KTable} by transforming the value of each record in this {@code KTable} into a new value
+     * (with possibly a new type) in the new {@code KTable}, with default serializers, deserializers, and state store.
+     * For each {@code KTable} update the provided {@link ValueMapperWithKey} is applied to the value of the update
+     * record and computes a new value for it, resulting in an updated record for the result {@code KTable}.
+     * Thus, an input record {@code } can be transformed into an output record {@code }.
+     * This is a stateless record-by-record operation.
+     * 
    
+     * The example below counts the number of token of value and key strings.
+     * 
    {@code
+     * KTable inputTable = builder.table("topic");
+     * KTable outputTable =
+     *  inputTable.mapValues((readOnlyKey, value) -> readOnlyKey.split(" ").length + value.split(" ").length, Named.as("countTokenValueAndKey"));
+     * }
    
+     * 
    
+     * Note that the key is read-only and should not be modified, as this can lead to corrupt partitioning.
+     * This operation preserves data co-location with respect to the key.
+     * Thus, no internal data redistribution is required if a key based operator (like a join) is applied to
+     * the result {@code KTable}.
+     * 
    
+     * Note that {@code mapValues} for a changelog stream works differently than {@link KStream#mapValues(ValueMapperWithKey)
+     * record stream filters}, because {@link KeyValue records} with {@code null} values (so-called tombstone records)
+     * have delete semantics.
+     * Thus, for tombstones the provided value-mapper is not evaluated but the tombstone record is forwarded directly to
+     * delete the corresponding record in the result {@code KTable}.
+     *
+     * @param mapper a {@link ValueMapperWithKey} that computes a new output value
+     * @param named  a {@link Named} config used to name the processor in the topology
+     * @param    the value type of the result {@code KTable}
+     * @return a {@code KTable} that contains records with unmodified keys and new values (possibly of different type)
+     */
+     KTable mapValues(final ValueMapperWithKey mapper,
+                                 final Named named);
+
+    /**
+     * Create a new {@code KTable} by transforming the value of each record in this {@code KTable} into a new value
+     * (with possibly a new type) in the new {@code KTable}, with the {@link Serde key serde}, {@link Serde value serde},
+     * and the underlying {@link KeyValueStore materialized state storage} configured in the {@link Materialized}
+     * instance.
+     * For each {@code KTable} update the provided {@link ValueMapper} is applied to the value of the updated record and
+     * computes a new value for it, resulting in an updated record for the result {@code KTable}.
+     * Thus, an input record {@code } can be transformed into an output record {@code }.
+     * This is a stateless record-by-record operation.
+     * 
    
+     * The example below counts the number of token of the value string.
+     * 
    {@code
+     * KTable inputTable = builder.table("topic");
+     * KTable outputTable = inputTable.mapValue(new ValueMapper {
+     *     Integer apply(String value) {
+     *         return value.split(" ").length;
+     *     }
+     * });
+     * }
    
+     * 
    
+     * To query the local {@link KeyValueStore} representing outputTable above it must be obtained via
+     * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}:
+     * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
+     * query the value of the key on a parallel running instance of your Kafka Streams application.
+     * The store name to query with is specified by {@link Materialized#as(String)} or {@link Materialized#as(KeyValueBytesStoreSupplier)}.
+     * 
    
+     * This operation preserves data co-location with respect to the key.
+     * Thus, no internal data redistribution is required if a key based operator (like a join) is applied to
+     * the result {@code KTable}.
+     * 
    
+     * Note that {@code mapValues} for a changelog stream works differently than {@link KStream#mapValues(ValueMapper)
+     * record stream filters}, because {@link KeyValue records} with {@code null} values (so-called tombstone records)
+     * have delete semantics.
+     * Thus, for tombstones the provided value-mapper is not evaluated but the tombstone record is forwarded directly to
+     * delete the corresponding record in the result {@code KTable}.
+     *
+     * @param mapper a {@link ValueMapper} that computes a new output value
+     * @param materialized  a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
+     *                      should be materialized. Cannot be {@code null}
+     * @param    the value type of the result {@code KTable}
+     *
+     * @return a {@code KTable} that contains records with unmodified keys and new values (possibly of different type)
+     */
+     KTable mapValues(final ValueMapper mapper,
+                                 final Materialized> materialized);
+
     /**
      * Create a new {@code KTable} by transforming the value of each record in this {@code KTable} into a new value
      * (with possibly a new type) in the new {@code KTable}, with the {@link Serde key serde}, {@link Serde value serde},
@@ -296,6 +527,7 @@ KTable filterNot(final Predicate predicate,
      * delete the corresponding record in the result {@code KTable}.
      *
      * @param mapper a {@link ValueMapper} that computes a new output value
+     * @param named  a {@link Named} config used to name the processor in the topology
      * @param materialized  a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
      *                      should be materialized. Cannot be {@code null}
      * @param    the value type of the result {@code KTable}
@@ -303,6 +535,7 @@ KTable filterNot(final Predicate predicate,
      * @return a {@code KTable} that contains records with unmodified keys and new values (possibly of different type)
      */
      KTable mapValues(final ValueMapper mapper,
+                                 final Named named,
                                  final Materialized> materialized);
 
     /**
@@ -353,22 +586,83 @@  KTable mapValues(final ValueMapperWithKey> materialized);
 
     /**
-     * Convert this changelog stream to a {@link KStream}.
-     * 
    
-     * Note that this is a logical operation and only changes the "interpretation" of the stream, i.e., each record of
-     * this changelog stream is no longer treated as an updated record (cf. {@link KStream} vs {@code KTable}).
-     *
-     * @return a {@link KStream} that contains the same records as this {@code KTable}
-     */
-    KStream toStream();
-
-    /**
-     * Convert this changelog stream to a {@link KStream} using the given {@link KeyValueMapper} to select the new key.
+     * Create a new {@code KTable} by transforming the value of each record in this {@code KTable} into a new value
+     * (with possibly a new type) in the new {@code KTable}, with the {@link Serde key serde}, {@link Serde value serde},
+     * and the underlying {@link KeyValueStore materialized state storage} configured in the {@link Materialized}
+     * instance.
+     * For each {@code KTable} update the provided {@link ValueMapperWithKey} is applied to the value of the update
+     * record and computes a new value for it, resulting in an updated record for the result {@code KTable}.
+     * Thus, an input record {@code } can be transformed into an output record {@code }.
+     * This is a stateless record-by-record operation.
      * 
    
-     * For example, you can compute the new key as the length of the value string.
+     * The example below counts the number of token of value and key strings.
      * 
    {@code
-     * KTable table = builder.table("topic");
-     * KTable keyedStream = table.toStream(new KeyValueMapper {
+     * KTable inputTable = builder.table("topic");
+     * KTable outputTable = inputTable.mapValue(new ValueMapperWithKey {
+     *     Integer apply(String readOnlyKey, String value) {
+     *          return readOnlyKey.split(" ").length + value.split(" ").length;
+     *     }
+     * });
+     * }
    
+     * 
    
+     * To query the local {@link KeyValueStore} representing outputTable above it must be obtained via
+     * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}:
+     * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
+     * query the value of the key on a parallel running instance of your Kafka Streams application.
+     * The store name to query with is specified by {@link Materialized#as(String)} or {@link Materialized#as(KeyValueBytesStoreSupplier)}.
+     * 
    
+     * Note that the key is read-only and should not be modified, as this can lead to corrupt partitioning.
+     * This operation preserves data co-location with respect to the key.
+     * Thus, no internal data redistribution is required if a key based operator (like a join) is applied to
+     * the result {@code KTable}.
+     * 
    
+     * Note that {@code mapValues} for a changelog stream works differently than {@link KStream#mapValues(ValueMapper)
+     * record stream filters}, because {@link KeyValue records} with {@code null} values (so-called tombstone records)
+     * have delete semantics.
+     * Thus, for tombstones the provided value-mapper is not evaluated but the tombstone record is forwarded directly to
+     * delete the corresponding record in the result {@code KTable}.
+     *
+     * @param mapper a {@link ValueMapperWithKey} that computes a new output value
+     * @param named  a {@link Named} config used to name the processor in the topology
+     * @param materialized  a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
+     *                      should be materialized. Cannot be {@code null}
+     * @param    the value type of the result {@code KTable}
+     *
+     * @return a {@code KTable} that contains records with unmodified keys and new values (possibly of different type)
+     */
+     KTable mapValues(final ValueMapperWithKey mapper,
+                                 final Named named,
+                                 final Materialized> materialized);
+
+    /**
+     * Convert this changelog stream to a {@link KStream}.
+     * 
    
+     * Note that this is a logical operation and only changes the "interpretation" of the stream, i.e., each record of
+     * this changelog stream is no longer treated as an updated record (cf. {@link KStream} vs {@code KTable}).
+     *
+     * @return a {@link KStream} that contains the same records as this {@code KTable}
+     */
+    KStream toStream();
+
+    /**
+     * Convert this changelog stream to a {@link KStream}.
+     * 
    
+     * Note that this is a logical operation and only changes the "interpretation" of the stream, i.e., each record of
+     * this changelog stream is no longer treated as an updated record (cf. {@link KStream} vs {@code KTable}).
+     *
+     * @param named  a {@link Named} config used to name the processor in the topology
+     *
+     * @return a {@link KStream} that contains the same records as this {@code KTable}
+     */
+    KStream toStream(final Named named);
+
+    /**
+     * Convert this changelog stream to a {@link KStream} using the given {@link KeyValueMapper} to select the new key.
+     * 
    
+     * For example, you can compute the new key as the length of the value string.
+     * 
    {@code
+     * KTable table = builder.table("topic");
+     * KTable keyedStream = table.toStream(new KeyValueMapper {
      *     Integer apply(String key, String value) {
      *         return value.length();
      *     }
@@ -389,6 +683,35 @@  KTable mapValues(final ValueMapperWithKey KStream toStream(final KeyValueMapper mapper);
 
+    /**
+     * Convert this changelog stream to a {@link KStream} using the given {@link KeyValueMapper} to select the new key.
+     * 
    
+     * For example, you can compute the new key as the length of the value string.
+     * 
    {@code
+     * KTable table = builder.table("topic");
+     * KTable keyedStream = table.toStream(new KeyValueMapper {
+     *     Integer apply(String key, String value) {
+     *         return value.length();
+     *     }
+     * });
+     * }
    
+     * Setting a new key might result in an internal data redistribution if a key based operator (like an aggregation or
+     * join) is applied to the result {@link KStream}.
+     * 
    
+     * This operation is equivalent to calling
+     * {@code table.}{@link #toStream() toStream}{@code ().}{@link KStream#selectKey(KeyValueMapper) selectKey(KeyValueMapper)}.
+     * 
    
+     * Note that {@link #toStream()} is a logical operation and only changes the "interpretation" of the stream, i.e.,
+     * each record of this changelog stream is no longer treated as an updated record (cf. {@link KStream} vs {@code KTable}).
+     *
+     * @param mapper a {@link KeyValueMapper} that computes a new key for each record
+     * @param named  a {@link Named} config used to name the processor in the topology
+     * @param  the new key type of the result stream
+     * @return a {@link KStream} that contains the same records as this {@code KTable}
+     */
+     KStream toStream(final KeyValueMapper mapper,
+                                 final Named named);
+
     /**
      * Suppress some updates from this changelog stream, determined by the supplied {@link Suppressed} configuration.
      *
@@ -472,6 +795,160 @@  KTable mapValues(final ValueMapperWithKey KTable transformValues(final ValueTransformerWithKeySupplier transformerSupplier,
                                        final String... stateStoreNames);
 
+    /**
+     * Create a new {@code KTable} by transforming the value of each record in this {@code KTable} into a new value
+     * (with possibly a new type), with default serializers, deserializers, and state store.
+     * A {@link ValueTransformerWithKey} (provided by the given {@link ValueTransformerWithKeySupplier}) is applied to each input
+     * record value and computes a new value for it.
+     * Thus, an input record {@code } can be transformed into an output record {@code }.
+     * This is similar to {@link #mapValues(ValueMapperWithKey)}, but more flexible, allowing access to additional state-stores,
+     * and access to the {@link ProcessorContext}.
+     * Furthermore, via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long)} the processing progress can be observed and additional
+     * periodic actions can be performed.
+     * 
    
+     * If the downstream topology uses aggregation functions, (e.g. {@link KGroupedTable#reduce}, {@link KGroupedTable#aggregate}, etc),
+     * care must be taken when dealing with state, (either held in state-stores or transformer instances), to ensure correct aggregate results.
+     * In contrast, if the resulting KTable is materialized, (cf. {@link #transformValues(ValueTransformerWithKeySupplier, Materialized, String...)}),
+     * such concerns are handled for you.
+     * 
    
+     * In order to assign a state, the state must be created and registered beforehand:
+     * 
    {@code
+     * // create store
+     * StoreBuilder> keyValueStoreBuilder =
+     *         Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore("myValueTransformState"),
+     *                 Serdes.String(),
+     *                 Serdes.String());
+     * // register store
+     * builder.addStateStore(keyValueStoreBuilder);
+     *
+     * KTable outputTable = inputTable.transformValues(new ValueTransformerWithKeySupplier() { ... }, "myValueTransformState");
+     * }
    
+     * 
    
+     * Within the {@link ValueTransformerWithKey}, the state is obtained via the
+     * {@link ProcessorContext}.
+     * To trigger periodic actions via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) punctuate()},
+     * a schedule must be registered.
+     * 
    {@code
+     * new ValueTransformerWithKeySupplier() {
+     *     ValueTransformerWithKey get() {
+     *         return new ValueTransformerWithKey() {
+     *             private KeyValueStore state;
+     *
+     *             void init(ProcessorContext context) {
+     *                 this.state = (KeyValueStore)context.getStateStore("myValueTransformState");
+     *                 context.schedule(Duration.ofSeconds(1), PunctuationType.WALL_CLOCK_TIME, new Punctuator(..)); // punctuate each 1000ms, can access this.state
+     *             }
+     *
+     *             NewValueType transform(K readOnlyKey, V value) {
+     *                 // can access this.state and use read-only key
+     *                 return new NewValueType(readOnlyKey); // or null
+     *             }
+     *
+     *             void close() {
+     *                 // can access this.state
+     *             }
+     *         }
+     *     }
+     * }
+     * }
    
+     * 
    
+     * Note that the key is read-only and should not be modified, as this can lead to corrupt partitioning.
+     * Setting a new value preserves data co-location with respect to the key.
+     *
+     * @param transformerSupplier a instance of {@link ValueTransformerWithKeySupplier} that generates a
+     *                            {@link ValueTransformerWithKey}.
+     *                            At least one transformer instance will be created per streaming task.
+     *                            Transformers do not need to be thread-safe.
+     * @param named               a {@link Named} config used to name the processor in the topology
+     * @param stateStoreNames     the names of the state stores used by the processor
+     * @param                 the value type of the result table
+     * @return a {@code KTable} that contains records with unmodified key and new values (possibly of different type)
+     * @see #mapValues(ValueMapper)
+     * @see #mapValues(ValueMapperWithKey)
+     */
+     KTable transformValues(final ValueTransformerWithKeySupplier transformerSupplier,
+                                       final Named named,
+                                       final String... stateStoreNames);
+
+    /**
+     * Create a new {@code KTable} by transforming the value of each record in this {@code KTable} into a new value
+     * (with possibly a new type), with the {@link Serde key serde}, {@link Serde value serde}, and the underlying
+     * {@link KeyValueStore materialized state storage} configured in the {@link Materialized} instance.
+     * A {@link ValueTransformerWithKey} (provided by the given {@link ValueTransformerWithKeySupplier}) is applied to each input
+     * record value and computes a new value for it.
+     * This is similar to {@link #mapValues(ValueMapperWithKey)}, but more flexible, allowing stateful, rather than stateless,
+     * record-by-record operation, access to additional state-stores, and access to the {@link ProcessorContext}.
+     * Furthermore, via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long)} the processing progress can be observed and additional
+     * periodic actions can be performed.
+     * The resulting {@code KTable} is materialized into another state store (additional to the provided state store names)
+     * as specified by the user via {@link Materialized} parameter, and is queryable through its given name.
+     * 
    
+     * In order to assign a state, the state must be created and registered beforehand:
+     * 
    {@code
+     * // create store
+     * StoreBuilder> keyValueStoreBuilder =
+     *         Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore("myValueTransformState"),
+     *                 Serdes.String(),
+     *                 Serdes.String());
+     * // register store
+     * builder.addStateStore(keyValueStoreBuilder);
+     *
+     * KTable outputTable = inputTable.transformValues(
+     *     new ValueTransformerWithKeySupplier() { ... },
+     *     Materialized.>as("outputTable")
+     *                                 .withKeySerde(Serdes.String())
+     *                                 .withValueSerde(Serdes.String()),
+     *     "myValueTransformState");
+     * }
    
+     * 
    
+     * Within the {@link ValueTransformerWithKey}, the state is obtained via the
+     * {@link ProcessorContext}.
+     * To trigger periodic actions via {@link org.apache.kafka.streams.processor.Punctuator#punctuate(long) punctuate()},
+     * a schedule must be registered.
+     * 
    {@code
+     * new ValueTransformerWithKeySupplier() {
+     *     ValueTransformerWithKey get() {
+     *         return new ValueTransformerWithKey() {
+     *             private KeyValueStore state;
+     *
+     *             void init(ProcessorContext context) {
+     *                 this.state = (KeyValueStore)context.getStateStore("myValueTransformState");
+     *                 context.schedule(Duration.ofSeconds(1), PunctuationType.WALL_CLOCK_TIME, new Punctuator(..)); // punctuate each 1000ms, can access this.state
+     *             }
+     *
+     *             NewValueType transform(K readOnlyKey, V value) {
+     *                 // can access this.state and use read-only key
+     *                 return new NewValueType(readOnlyKey); // or null
+     *             }
+     *
+     *             void close() {
+     *                 // can access this.state
+     *             }
+     *         }
+     *     }
+     * }
+     * }
    
+     * 
    
+     * Note that the key is read-only and should not be modified, as this can lead to corrupt partitioning.
+     * Setting a new value preserves data co-location with respect to the key.
+     *
+     * @param transformerSupplier a instance of {@link ValueTransformerWithKeySupplier} that generates a
+     *                            {@link ValueTransformerWithKey}.
+     *                            At least one transformer instance will be created per streaming task.
+     *                            Transformers do not need to be thread-safe.
+     * @param materialized        an instance of {@link Materialized} used to describe how the state store of the
+     *                            resulting table should be materialized.
+     *                            Cannot be {@code null}
+     * @param stateStoreNames     the names of the state stores used by the processor
+     * @param                 the value type of the result table
+     * @return a {@code KTable} that contains records with unmodified key and new values (possibly of different type)
+     * @see #mapValues(ValueMapper)
+     * @see #mapValues(ValueMapperWithKey)
+     */
+     KTable transformValues(final ValueTransformerWithKeySupplier transformerSupplier,
+                                       final Materialized> materialized,
+                                       final String... stateStoreNames);
+
     /**
      * Create a new {@code KTable} by transforming the value of each record in this {@code KTable} into a new value
      * (with possibly a new type), with the {@link Serde key serde}, {@link Serde value serde}, and the underlying
@@ -541,6 +1018,7 @@  KTable transformValues(final ValueTransformerWithKeySupplier                the value type of the result table
      * @return a {@code KTable} that contains records with unmodified key and new values (possibly of different type)
@@ -549,6 +1027,7 @@  KTable transformValues(final ValueTransformerWithKeySupplier KTable transformValues(final ValueTransformerWithKeySupplier transformerSupplier,
                                        final Materialized> materialized,
+                                       final Named named,
                                        final String... stateStoreNames);
 
     /**
@@ -640,20 +1119,416 @@  KGroupedTable groupBy(final KeyValueMapper          the key type of the result {@link KGroupedTable}
-     * @param           the value type of the result {@link KGroupedTable}
-     * @return a {@link KGroupedTable} that contains the re-grouped records of the original {@code KTable}
+     * @param selector      a {@link KeyValueMapper} that computes a new grouping key and value to be aggregated
+     * @param grouped       the {@link Grouped} instance used to specify {@link org.apache.kafka.common.serialization.Serdes}
+     *                      and the name for a repartition topic if repartitioning is required.
+     * @param           the key type of the result {@link KGroupedTable}
+     * @param           the value type of the result {@link KGroupedTable}
+     * @return a {@link KGroupedTable} that contains the re-grouped records of the original {@code KTable}
+     */
+     KGroupedTable groupBy(final KeyValueMapper> selector,
+                                           final Grouped grouped);
+
+    /**
+     * Join records of this {@code KTable} with another {@code KTable}'s records using non-windowed inner equi join,
+     * with default serializers, deserializers, and state store.
+     * The join is a primary key join with join attribute {@code thisKTable.key == otherKTable.key}.
+     * The result is an ever updating {@code KTable} that represents the current (i.e., processing time) result
+     * of the join.
+     * 
    
+     * The join is computed by (1) updating the internal state of one {@code KTable} and (2) performing a lookup for a
+     * matching record in the current (i.e., processing time) internal state of the other {@code KTable}.
+     * This happens in a symmetric way, i.e., for each update of either {@code this} or the {@code other} input
+     * {@code KTable} the result gets updated.
+     * 
    
+     * For each {@code KTable} record that finds a corresponding record in the other {@code KTable} the provided
+     * {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record.
+     * The key of the result record is the same as for both joining input records.
+     * 
    
+     * Note that {@link KeyValue records} with {@code null} values (so-called tombstone records) have delete semantics.
+     * Thus, for input tombstones the provided value-joiner is not called but a tombstone record is forwarded
+     * directly to delete a record in the result {@code KTable} if required (i.e., if there is anything to be deleted).
+     * 
    
+     * Input records with {@code null} key will be dropped and no join computation is performed.
+     * 
    
+     * Example:
+     *

    + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
    thisKTablethisStateotherKTableotherStateresult updated record
    <K1:A><K1:A>
    <K1:A><K1:b><K1:b><K1:ValueJoiner(A,b)>
    <K1:C><K1:C><K1:b><K1:ValueJoiner(C,b)>
    <K1:C><K1:null><K1:null>
    + * Both input streams (or to be more precise, their underlying source topics) need to have the same number of + * partitions. + * + * @param other the other {@code KTable} to be joined with this {@code KTable} + * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records + * @param the value type of the other {@code KTable} + * @param the value type of the result {@code KTable} + * @return a {@code KTable} that contains join-records for each key and values computed by the given + * {@link ValueJoiner}, one for each matched record-pair with the same key + * @see #leftJoin(KTable, ValueJoiner) + * @see #outerJoin(KTable, ValueJoiner) + */ + KTable join(final KTable other, + final ValueJoiner joiner); + + /** + * Join records of this {@code KTable} with another {@code KTable}'s records using non-windowed inner equi join, + * with default serializers, deserializers, and state store. + * The join is a primary key join with join attribute {@code thisKTable.key == otherKTable.key}. + * The result is an ever updating {@code KTable} that represents the current (i.e., processing time) result + * of the join. + *

    + * The join is computed by (1) updating the internal state of one {@code KTable} and (2) performing a lookup for a + * matching record in the current (i.e., processing time) internal state of the other {@code KTable}. + * This happens in a symmetric way, i.e., for each update of either {@code this} or the {@code other} input + * {@code KTable} the result gets updated. + *

    + * For each {@code KTable} record that finds a corresponding record in the other {@code KTable} the provided + * {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record. + * The key of the result record is the same as for both joining input records. + *

    + * Note that {@link KeyValue records} with {@code null} values (so-called tombstone records) have delete semantics. + * Thus, for input tombstones the provided value-joiner is not called but a tombstone record is forwarded + * directly to delete a record in the result {@code KTable} if required (i.e., if there is anything to be deleted). + *

    + * Input records with {@code null} key will be dropped and no join computation is performed. + *

    + * Example: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
    thisKTablethisStateotherKTableotherStateresult updated record
    <K1:A><K1:A>
    <K1:A><K1:b><K1:b><K1:ValueJoiner(A,b)>
    <K1:C><K1:C><K1:b><K1:ValueJoiner(C,b)>
    <K1:C><K1:null><K1:null>
    + * Both input streams (or to be more precise, their underlying source topics) need to have the same number of + * partitions. + * + * @param other the other {@code KTable} to be joined with this {@code KTable} + * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records + * @param named a {@link Named} config used to name the processor in the topology + * @param the value type of the other {@code KTable} + * @param the value type of the result {@code KTable} + * @return a {@code KTable} that contains join-records for each key and values computed by the given + * {@link ValueJoiner}, one for each matched record-pair with the same key + * @see #leftJoin(KTable, ValueJoiner) + * @see #outerJoin(KTable, ValueJoiner) + */ + KTable join(final KTable other, + final ValueJoiner joiner, + final Named named); + + /** + * Join records of this {@code KTable} with another {@code KTable}'s records using non-windowed inner equi join, + * with the {@link Materialized} instance for configuration of the {@link Serde key serde}, + * {@link Serde the result table's value serde}, and {@link KeyValueStore state store}. + * The join is a primary key join with join attribute {@code thisKTable.key == otherKTable.key}. + * The result is an ever updating {@code KTable} that represents the current (i.e., processing time) result + * of the join. + *

    + * The join is computed by (1) updating the internal state of one {@code KTable} and (2) performing a lookup for a + * matching record in the current (i.e., processing time) internal state of the other {@code KTable}. + * This happens in a symmetric way, i.e., for each update of either {@code this} or the {@code other} input + * {@code KTable} the result gets updated. + *

    + * For each {@code KTable} record that finds a corresponding record in the other {@code KTable} the provided + * {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record. + * The key of the result record is the same as for both joining input records. + *

    + * Note that {@link KeyValue records} with {@code null} values (so-called tombstone records) have delete semantics. + * Thus, for input tombstones the provided value-joiner is not called but a tombstone record is forwarded + * directly to delete a record in the result {@code KTable} if required (i.e., if there is anything to be deleted). + *

    + * Input records with {@code null} key will be dropped and no join computation is performed. + *

    + * Example: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
    thisKTablethisStateotherKTableotherStateresult updated record
    <K1:A><K1:A>
    <K1:A><K1:b><K1:b><K1:ValueJoiner(A,b)>
    <K1:C><K1:C><K1:b><K1:ValueJoiner(C,b)>
    <K1:C><K1:null><K1:null>
    + * Both input streams (or to be more precise, their underlying source topics) need to have the same number of + * partitions. + * + * @param other the other {@code KTable} to be joined with this {@code KTable} + * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records + * @param materialized an instance of {@link Materialized} used to describe how the state store should be materialized. + * Cannot be {@code null} + * @param the value type of the other {@code KTable} + * @param the value type of the result {@code KTable} + * @return a {@code KTable} that contains join-records for each key and values computed by the given + * {@link ValueJoiner}, one for each matched record-pair with the same key + * @see #leftJoin(KTable, ValueJoiner, Materialized) + * @see #outerJoin(KTable, ValueJoiner, Materialized) + */ + KTable join(final KTable other, + final ValueJoiner joiner, + final Materialized> materialized); + + /** + * Join records of this {@code KTable} with another {@code KTable}'s records using non-windowed inner equi join, + * with the {@link Materialized} instance for configuration of the {@link Serde key serde}, + * {@link Serde the result table's value serde}, and {@link KeyValueStore state store}. + * The join is a primary key join with join attribute {@code thisKTable.key == otherKTable.key}. + * The result is an ever updating {@code KTable} that represents the current (i.e., processing time) result + * of the join. + *

    + * The join is computed by (1) updating the internal state of one {@code KTable} and (2) performing a lookup for a + * matching record in the current (i.e., processing time) internal state of the other {@code KTable}. + * This happens in a symmetric way, i.e., for each update of either {@code this} or the {@code other} input + * {@code KTable} the result gets updated. + *

    + * For each {@code KTable} record that finds a corresponding record in the other {@code KTable} the provided + * {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record. + * The key of the result record is the same as for both joining input records. + *

    + * Note that {@link KeyValue records} with {@code null} values (so-called tombstone records) have delete semantics. + * Thus, for input tombstones the provided value-joiner is not called but a tombstone record is forwarded + * directly to delete a record in the result {@code KTable} if required (i.e., if there is anything to be deleted). + *

    + * Input records with {@code null} key will be dropped and no join computation is performed. + *

    + * Example: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
    thisKTablethisStateotherKTableotherStateresult updated record
    <K1:A><K1:A>
    <K1:A><K1:b><K1:b><K1:ValueJoiner(A,b)>
    <K1:C><K1:C><K1:b><K1:ValueJoiner(C,b)>
    <K1:C><K1:null><K1:null>
    + * Both input streams (or to be more precise, their underlying source topics) need to have the same number of + * partitions. + * + * @param other the other {@code KTable} to be joined with this {@code KTable} + * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records + * @param named a {@link Named} config used to name the processor in the topology + * @param materialized an instance of {@link Materialized} used to describe how the state store should be materialized. + * Cannot be {@code null} + * @param the value type of the other {@code KTable} + * @param the value type of the result {@code KTable} + * @return a {@code KTable} that contains join-records for each key and values computed by the given + * {@link ValueJoiner}, one for each matched record-pair with the same key + * @see #leftJoin(KTable, ValueJoiner, Materialized) + * @see #outerJoin(KTable, ValueJoiner, Materialized) + */ + KTable join(final KTable other, + final ValueJoiner joiner, + final Named named, + final Materialized> materialized); + + /** + * Join records of this {@code KTable} (left input) with another {@code KTable}'s (right input) records using + * non-windowed left equi join, with default serializers, deserializers, and state store. + * The join is a primary key join with join attribute {@code thisKTable.key == otherKTable.key}. + * In contrast to {@link #join(KTable, ValueJoiner) inner-join}, all records from left {@code KTable} will produce + * an output record (cf. below). + * The result is an ever updating {@code KTable} that represents the current (i.e., processing time) result + * of the join. + *

    + * The join is computed by (1) updating the internal state of one {@code KTable} and (2) performing a lookup for a + * matching record in the current (i.e., processing time) internal state of the other {@code KTable}. + * This happens in a symmetric way, i.e., for each update of either {@code this} or the {@code other} input + * {@code KTable} the result gets updated. + *

    + * For each {@code KTable} record that finds a corresponding record in the other {@code KTable}'s state the + * provided {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record. + * Additionally, for each record of left {@code KTable} that does not find a corresponding record in the + * right {@code KTable}'s state the provided {@link ValueJoiner} will be called with {@code rightValue = + * null} to compute a value (with arbitrary type) for the result record. + * The key of the result record is the same as for both joining input records. + *

    + * Note that {@link KeyValue records} with {@code null} values (so-called tombstone records) have delete semantics. + * For example, for left input tombstones the provided value-joiner is not called but a tombstone record is + * forwarded directly to delete a record in the result {@code KTable} if required (i.e., if there is anything to be + * deleted). + *

    + * Input records with {@code null} key will be dropped and no join computation is performed. + *

    + * Example: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
    thisKTablethisStateotherKTableotherStateresult updated record
    <K1:A><K1:A><K1:ValueJoiner(A,null)>
    <K1:A><K1:b><K1:b><K1:ValueJoiner(A,b)>
    <K1:null><K1:b><K1:null>
    <K1:null>
    + * Both input streams (or to be more precise, their underlying source topics) need to have the same number of + * partitions. + * + * @param other the other {@code KTable} to be joined with this {@code KTable} + * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records + * @param the value type of the other {@code KTable} + * @param the value type of the result {@code KTable} + * @return a {@code KTable} that contains join-records for each key and values computed by the given + * {@link ValueJoiner}, one for each matched record-pair with the same key plus one for each non-matching record of + * left {@code KTable} + * @see #join(KTable, ValueJoiner) + * @see #outerJoin(KTable, ValueJoiner) */ - KGroupedTable groupBy(final KeyValueMapper> selector, - final Grouped grouped); + KTable leftJoin(final KTable other, + final ValueJoiner joiner); /** - * Join records of this {@code KTable} with another {@code KTable}'s records using non-windowed inner equi join, - * with default serializers, deserializers, and state store. + * Join records of this {@code KTable} (left input) with another {@code KTable}'s (right input) records using + * non-windowed left equi join, with default serializers, deserializers, and state store. * The join is a primary key join with join attribute {@code thisKTable.key == otherKTable.key}. + * In contrast to {@link #join(KTable, ValueJoiner) inner-join}, all records from left {@code KTable} will produce + * an output record (cf. below). * The result is an ever updating {@code KTable} that represents the current (i.e., processing time) result * of the join. *

    @@ -662,13 +1537,17 @@ KGroupedTable groupBy(final KeyValueMapper - * For each {@code KTable} record that finds a corresponding record in the other {@code KTable} the provided - * {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record. + * For each {@code KTable} record that finds a corresponding record in the other {@code KTable}'s state the + * provided {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record. + * Additionally, for each record of left {@code KTable} that does not find a corresponding record in the + * right {@code KTable}'s state the provided {@link ValueJoiner} will be called with {@code rightValue = + * null} to compute a value (with arbitrary type) for the result record. * The key of the result record is the same as for both joining input records. *

    * Note that {@link KeyValue records} with {@code null} values (so-called tombstone records) have delete semantics. - * Thus, for input tombstones the provided value-joiner is not called but a tombstone record is forwarded - * directly to delete a record in the result {@code KTable} if required (i.e., if there is anything to be deleted). + * For example, for left input tombstones the provided value-joiner is not called but a tombstone record is + * forwarded directly to delete a record in the result {@code KTable} if required (i.e., if there is anything to be + * deleted). *

    * Input records with {@code null} key will be dropped and no join computation is performed. *

    @@ -686,7 +1565,7 @@ KGroupedTable groupBy(final KeyValueMapper<K1:A> * * - * + * <K1:ValueJoiner(A,null)> * * * @@ -696,18 +1575,18 @@ KGroupedTable groupBy(final KeyValueMapper<K1:ValueJoiner(A,b)> * * - * <K1:C> - * <K1:C> + * <K1:null> + * * * <K1:b> - * <K1:ValueJoiner(C,b)> + * <K1:null> * * * - * <K1:C> - * <K1:null> * * <K1:null> + * + * * * * Both input streams (or to be more precise, their underlying source topics) need to have the same number of @@ -715,21 +1594,26 @@ KGroupedTable groupBy(final KeyValueMapper the value type of the other {@code KTable} * @param the value type of the result {@code KTable} * @return a {@code KTable} that contains join-records for each key and values computed by the given - * {@link ValueJoiner}, one for each matched record-pair with the same key - * @see #leftJoin(KTable, ValueJoiner) + * {@link ValueJoiner}, one for each matched record-pair with the same key plus one for each non-matching record of + * left {@code KTable} + * @see #join(KTable, ValueJoiner) * @see #outerJoin(KTable, ValueJoiner) */ - KTable join(final KTable other, - final ValueJoiner joiner); + KTable leftJoin(final KTable other, + final ValueJoiner joiner, + final Named named); /** - * Join records of this {@code KTable} with another {@code KTable}'s records using non-windowed inner equi join, - * with the {@link Materialized} instance for configuration of the {@link Serde key serde}, + * Join records of this {@code KTable} (left input) with another {@code KTable}'s (right input) records using + * non-windowed left equi join, with the {@link Materialized} instance for configuration of the {@link Serde key serde}, * {@link Serde the result table's value serde}, and {@link KeyValueStore state store}. * The join is a primary key join with join attribute {@code thisKTable.key == otherKTable.key}. + * In contrast to {@link #join(KTable, ValueJoiner) inner-join}, all records from left {@code KTable} will produce + * an output record (cf. below). * The result is an ever updating {@code KTable} that represents the current (i.e., processing time) result * of the join. *

    @@ -738,13 +1622,17 @@ KTable join(final KTable other, * This happens in a symmetric way, i.e., for each update of either {@code this} or the {@code other} input * {@code KTable} the result gets updated. *

    - * For each {@code KTable} record that finds a corresponding record in the other {@code KTable} the provided - * {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record. + * For each {@code KTable} record that finds a corresponding record in the other {@code KTable}'s state the + * provided {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record. + * Additionally, for each record of left {@code KTable} that does not find a corresponding record in the + * right {@code KTable}'s state the provided {@link ValueJoiner} will be called with {@code rightValue = + * null} to compute a value (with arbitrary type) for the result record. * The key of the result record is the same as for both joining input records. *

    * Note that {@link KeyValue records} with {@code null} values (so-called tombstone records) have delete semantics. - * Thus, for input tombstones the provided value-joiner is not called but a tombstone record is forwarded - * directly to delete a record in the result {@code KTable} if required (i.e., if there is anything to be deleted). + * For example, for left input tombstones the provided value-joiner is not called but a tombstone record is + * forwarded directly to delete a record in the result {@code KTable} if required (i.e., if there is anything to be + * deleted). *

    * Input records with {@code null} key will be dropped and no join computation is performed. *

    @@ -762,7 +1650,7 @@ KTable join(final KTable other, * <K1:A> * * - * + * <K1:ValueJoiner(A,null)> * * * @@ -772,18 +1660,18 @@ KTable join(final KTable other, * <K1:ValueJoiner(A,b)> * * - * <K1:C> - * <K1:C> + * <K1:null> + * * * <K1:b> - * <K1:ValueJoiner(C,b)> + * <K1:null> * * * - * <K1:C> - * <K1:null> * * <K1:null> + * + * * * * Both input streams (or to be more precise, their underlying source topics) need to have the same number of @@ -796,18 +1684,19 @@ KTable join(final KTable other, * @param the value type of the other {@code KTable} * @param the value type of the result {@code KTable} * @return a {@code KTable} that contains join-records for each key and values computed by the given - * {@link ValueJoiner}, one for each matched record-pair with the same key - * @see #leftJoin(KTable, ValueJoiner, Materialized) + * {@link ValueJoiner}, one for each matched record-pair with the same key plus one for each non-matching record of + * left {@code KTable} + * @see #join(KTable, ValueJoiner, Materialized) * @see #outerJoin(KTable, ValueJoiner, Materialized) */ - KTable join(final KTable other, - final ValueJoiner joiner, - final Materialized> materialized); - + KTable leftJoin(final KTable other, + final ValueJoiner joiner, + final Materialized> materialized); /** * Join records of this {@code KTable} (left input) with another {@code KTable}'s (right input) records using - * non-windowed left equi join, with default serializers, deserializers, and state store. + * non-windowed left equi join, with the {@link Materialized} instance for configuration of the {@link Serde key serde}, + * {@link Serde the result table's value serde}, and {@link KeyValueStore state store}. * The join is a primary key join with join attribute {@code thisKTable.key == otherKTable.key}. * In contrast to {@link #join(KTable, ValueJoiner) inner-join}, all records from left {@code KTable} will produce * an output record (cf. below). @@ -874,26 +1763,30 @@ KTable join(final KTable other, * Both input streams (or to be more precise, their underlying source topics) need to have the same number of * partitions. * - * @param other the other {@code KTable} to be joined with this {@code KTable} - * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records - * @param the value type of the other {@code KTable} - * @param the value type of the result {@code KTable} + * @param other the other {@code KTable} to be joined with this {@code KTable} + * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records + * @param named a {@link Named} config used to name the processor in the topology + * @param materialized an instance of {@link Materialized} used to describe how the state store should be materialized. + * Cannot be {@code null} + * @param the value type of the other {@code KTable} + * @param the value type of the result {@code KTable} * @return a {@code KTable} that contains join-records for each key and values computed by the given * {@link ValueJoiner}, one for each matched record-pair with the same key plus one for each non-matching record of * left {@code KTable} - * @see #join(KTable, ValueJoiner) - * @see #outerJoin(KTable, ValueJoiner) + * @see #join(KTable, ValueJoiner, Materialized) + * @see #outerJoin(KTable, ValueJoiner, Materialized) */ KTable leftJoin(final KTable other, - final ValueJoiner joiner); + final ValueJoiner joiner, + final Named named, + final Materialized> materialized); /** * Join records of this {@code KTable} (left input) with another {@code KTable}'s (right input) records using - * non-windowed left equi join, with the {@link Materialized} instance for configuration of the {@link Serde key serde}, - * {@link Serde the result table's value serde}, and {@link KeyValueStore state store}. + * non-windowed outer equi join, with default serializers, deserializers, and state store. * The join is a primary key join with join attribute {@code thisKTable.key == otherKTable.key}. - * In contrast to {@link #join(KTable, ValueJoiner) inner-join}, all records from left {@code KTable} will produce - * an output record (cf. below). + * In contrast to {@link #join(KTable, ValueJoiner) inner-join} or {@link #leftJoin(KTable, ValueJoiner) left-join}, + * all records from both input {@code KTable}s will produce an output record (cf. below). * The result is an ever updating {@code KTable} that represents the current (i.e., processing time) result * of the join. *

    @@ -904,15 +1797,14 @@ KTable leftJoin(final KTable other, *

    * For each {@code KTable} record that finds a corresponding record in the other {@code KTable}'s state the * provided {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record. - * Additionally, for each record of left {@code KTable} that does not find a corresponding record in the - * right {@code KTable}'s state the provided {@link ValueJoiner} will be called with {@code rightValue = - * null} to compute a value (with arbitrary type) for the result record. + * Additionally, for each record that does not find a corresponding record in the corresponding other + * {@code KTable}'s state the provided {@link ValueJoiner} will be called with {@code null} value for the + * corresponding other value to compute a value (with arbitrary type) for the result record. * The key of the result record is the same as for both joining input records. *

    * Note that {@link KeyValue records} with {@code null} values (so-called tombstone records) have delete semantics. - * For example, for left input tombstones the provided value-joiner is not called but a tombstone record is - * forwarded directly to delete a record in the result {@code KTable} if required (i.e., if there is anything to be - * deleted). + * Thus, for input tombstones the provided value-joiner is not called but a tombstone record is forwarded directly + * to delete a record in the result {@code KTable} if required (i.e., if there is anything to be deleted). *

    * Input records with {@code null} key will be dropped and no join computation is performed. *

    @@ -944,34 +1836,31 @@ KTable leftJoin(final KTable other, * * * <K1:b> - * <K1:null> + * <K1:ValueJoiner(null,b)> * * * * * <K1:null> * - * + * <K1:null> * * * Both input streams (or to be more precise, their underlying source topics) need to have the same number of * partitions. * - * @param other the other {@code KTable} to be joined with this {@code KTable} - * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records - * @param materialized an instance of {@link Materialized} used to describe how the state store should be materialized. - * Cannot be {@code null} - * @param the value type of the other {@code KTable} - * @param the value type of the result {@code KTable} + * @param other the other {@code KTable} to be joined with this {@code KTable} + * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records + * @param the value type of the other {@code KTable} + * @param the value type of the result {@code KTable} * @return a {@code KTable} that contains join-records for each key and values computed by the given * {@link ValueJoiner}, one for each matched record-pair with the same key plus one for each non-matching record of - * left {@code KTable} - * @see #join(KTable, ValueJoiner, Materialized) - * @see #outerJoin(KTable, ValueJoiner, Materialized) + * both {@code KTable}s + * @see #join(KTable, ValueJoiner) + * @see #leftJoin(KTable, ValueJoiner) */ - KTable leftJoin(final KTable other, - final ValueJoiner joiner, - final Materialized> materialized); + KTable outerJoin(final KTable other, + final ValueJoiner joiner); /** @@ -1044,6 +1933,7 @@ KTable leftJoin(final KTable other, * * @param other the other {@code KTable} to be joined with this {@code KTable} * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records + * @param named a {@link Named} config used to name the processor in the topology * @param the value type of the other {@code KTable} * @param the value type of the result {@code KTable} * @return a {@code KTable} that contains join-records for each key and values computed by the given @@ -1053,7 +1943,94 @@ KTable leftJoin(final KTable other, * @see #leftJoin(KTable, ValueJoiner) */ KTable outerJoin(final KTable other, - final ValueJoiner joiner); + final ValueJoiner joiner, + final Named named); + + /** + * Join records of this {@code KTable} (left input) with another {@code KTable}'s (right input) records using + * non-windowed outer equi join, with the {@link Materialized} instance for configuration of the {@link Serde key serde}, + * {@link Serde the result table's value serde}, and {@link KeyValueStore state store}. + * The join is a primary key join with join attribute {@code thisKTable.key == otherKTable.key}. + * In contrast to {@link #join(KTable, ValueJoiner) inner-join} or {@link #leftJoin(KTable, ValueJoiner) left-join}, + * all records from both input {@code KTable}s will produce an output record (cf. below). + * The result is an ever updating {@code KTable} that represents the current (i.e., processing time) result + * of the join. + *

    + * The join is computed by (1) updating the internal state of one {@code KTable} and (2) performing a lookup for a + * matching record in the current (i.e., processing time) internal state of the other {@code KTable}. + * This happens in a symmetric way, i.e., for each update of either {@code this} or the {@code other} input + * {@code KTable} the result gets updated. + *

    + * For each {@code KTable} record that finds a corresponding record in the other {@code KTable}'s state the + * provided {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record. + * Additionally, for each record that does not find a corresponding record in the corresponding other + * {@code KTable}'s state the provided {@link ValueJoiner} will be called with {@code null} value for the + * corresponding other value to compute a value (with arbitrary type) for the result record. + * The key of the result record is the same as for both joining input records. + *

    + * Note that {@link KeyValue records} with {@code null} values (so-called tombstone records) have delete semantics. + * Thus, for input tombstones the provided value-joiner is not called but a tombstone record is forwarded directly + * to delete a record in the result {@code KTable} if required (i.e., if there is anything to be deleted). + *

    + * Input records with {@code null} key will be dropped and no join computation is performed. + *

    + * Example: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
    thisKTablethisStateotherKTableotherStateresult updated record
    <K1:A><K1:A><K1:ValueJoiner(A,null)>
    <K1:A><K1:b><K1:b><K1:ValueJoiner(A,b)>
    <K1:null><K1:b><K1:ValueJoiner(null,b)>
    <K1:null><K1:null>
    + * Both input streams (or to be more precise, their underlying source topics) need to have the same number of + * partitions. + * + * @param other the other {@code KTable} to be joined with this {@code KTable} + * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records + * @param materialized an instance of {@link Materialized} used to describe how the state store should be materialized. + * Cannot be {@code null} + * @param the value type of the other {@code KTable} + * @param the value type of the result {@code KTable} + * @return a {@code KTable} that contains join-records for each key and values computed by the given + * {@link ValueJoiner}, one for each matched record-pair with the same key plus one for each non-matching record of + * both {@code KTable}s + * @see #join(KTable, ValueJoiner) + * @see #leftJoin(KTable, ValueJoiner) + */ + KTable outerJoin(final KTable other, + final ValueJoiner joiner, + final Materialized> materialized); + /** * Join records of this {@code KTable} (left input) with another {@code KTable}'s (right input) records using @@ -1126,6 +2103,7 @@ KTable outerJoin(final KTable other, * * @param other the other {@code KTable} to be joined with this {@code KTable} * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records + * @param named a {@link Named} config used to name the processor in the topology * @param materialized an instance of {@link Materialized} used to describe how the state store should be materialized. * Cannot be {@code null} * @param the value type of the other {@code KTable} @@ -1138,6 +2116,7 @@ KTable outerJoin(final KTable other, */ KTable outerJoin(final KTable other, final ValueJoiner joiner, + final Named named, final Materialized> materialized); /** diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java index 666a109bbc012..4bc102a746d92 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java @@ -25,6 +25,7 @@ import org.apache.kafka.streams.kstream.KTable; import org.apache.kafka.streams.kstream.KeyValueMapper; import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Named; import org.apache.kafka.streams.kstream.Predicate; import org.apache.kafka.streams.kstream.Suppressed; import org.apache.kafka.streams.kstream.ValueJoiner; @@ -113,6 +114,7 @@ public String queryableStoreName() { } private KTable doFilter(final Predicate predicate, + final Named named, final MaterializedInternal> materializedInternal, final boolean filterNot) { final Serde keySerde; @@ -140,8 +142,7 @@ private KTable doFilter(final Predicate predicate, queryableStoreName = null; storeBuilder = null; } - - final String name = builder.newProcessorName(FILTER_NAME); + final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, FILTER_NAME); final KTableProcessorSupplier processorSupplier = new KTableFilter<>(this, predicate, filterNot, queryableStoreName); @@ -171,36 +172,64 @@ private KTable doFilter(final Predicate predicate, @Override public KTable filter(final Predicate predicate) { Objects.requireNonNull(predicate, "predicate can't be null"); - return doFilter(predicate, null, false); + return doFilter(predicate, NamedInternal.empty(), null, false); + } + + @Override + public KTable filter(final Predicate predicate, final Named named) { + Objects.requireNonNull(predicate, "predicate can't be null"); + return doFilter(predicate, named, null, false); } @Override public KTable filter(final Predicate predicate, + final Named named, final Materialized> materialized) { Objects.requireNonNull(predicate, "predicate can't be null"); Objects.requireNonNull(materialized, "materialized can't be null"); final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized); - return doFilter(predicate, materializedInternal, false); + return doFilter(predicate, named, materializedInternal, false); + } + + @Override + public KTable filter(final Predicate predicate, + final Materialized> materialized) { + return filter(predicate, NamedInternal.empty(), materialized); } @Override public KTable filterNot(final Predicate predicate) { Objects.requireNonNull(predicate, "predicate can't be null"); - return doFilter(predicate, null, true); + return doFilter(predicate, NamedInternal.empty(), null, true); + } + + @Override + public KTable filterNot(final Predicate predicate, + final Named named) { + Objects.requireNonNull(predicate, "predicate can't be null"); + return doFilter(predicate, named, null, true); + } + + @Override + public KTable filterNot(final Predicate predicate, + final Materialized> materialized) { + return filterNot(predicate, NamedInternal.empty(), materialized); } @Override public KTable filterNot(final Predicate predicate, + final Named named, final Materialized> materialized) { Objects.requireNonNull(predicate, "predicate can't be null"); Objects.requireNonNull(materialized, "materialized can't be null"); final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized); - - return doFilter(predicate, materializedInternal, true); + final NamedInternal renamed = new NamedInternal(named); + return doFilter(predicate, renamed, materializedInternal, true); } private KTable doMapValues(final ValueMapperWithKey mapper, + final Named named, final MaterializedInternal> materializedInternal) { final Serde keySerde; final Serde valueSerde; @@ -225,7 +254,7 @@ private KTable doMapValues(final ValueMapperWithKey processorSupplier = new KTableMapValues<>(this, mapper, queryableStoreName); @@ -260,54 +289,101 @@ private KTable doMapValues(final ValueMapperWithKey KTable mapValues(final ValueMapper mapper) { Objects.requireNonNull(mapper, "mapper can't be null"); - return doMapValues(withKey(mapper), null); + return doMapValues(withKey(mapper), NamedInternal.empty(), null); + } + + @Override + public KTable mapValues(final ValueMapper mapper, + final Named named) { + Objects.requireNonNull(mapper, "mapper can't be null"); + return doMapValues(withKey(mapper), named, null); } @Override public KTable mapValues(final ValueMapperWithKey mapper) { Objects.requireNonNull(mapper, "mapper can't be null"); - return doMapValues(mapper, null); + return doMapValues(mapper, NamedInternal.empty(), null); + } + + @Override + public KTable mapValues(final ValueMapperWithKey mapper, + final Named named) { + Objects.requireNonNull(mapper, "mapper can't be null"); + return doMapValues(mapper, named, null); + } + + @Override + public KTable mapValues(final ValueMapper mapper, + final Materialized> materialized) { + return mapValues(mapper, NamedInternal.empty(), materialized); } @Override public KTable mapValues(final ValueMapper mapper, + final Named named, final Materialized> materialized) { Objects.requireNonNull(mapper, "mapper can't be null"); Objects.requireNonNull(materialized, "materialized can't be null"); final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized); - return doMapValues(withKey(mapper), materializedInternal); + return doMapValues(withKey(mapper), named, materializedInternal); } @Override public KTable mapValues(final ValueMapperWithKey mapper, final Materialized> materialized) { + return mapValues(mapper, NamedInternal.empty(), materialized); + } + + @Override + public KTable mapValues(final ValueMapperWithKey mapper, + final Named named, + final Materialized> materialized) { Objects.requireNonNull(mapper, "mapper can't be null"); Objects.requireNonNull(materialized, "materialized can't be null"); final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized); - return doMapValues(mapper, materializedInternal); + + return doMapValues(mapper, named, materializedInternal); + } + + @Override + public KTable transformValues(final ValueTransformerWithKeySupplier transformerSupplier, + final String... stateStoreNames) { + return doTransformValues(transformerSupplier, null, NamedInternal.empty(), stateStoreNames); } @Override public KTable transformValues(final ValueTransformerWithKeySupplier transformerSupplier, + final Named named, final String... stateStoreNames) { - return doTransformValues(transformerSupplier, null, stateStoreNames); + Objects.requireNonNull(named, "processorName can't be null"); + return doTransformValues(transformerSupplier, null, new NamedInternal(named), stateStoreNames); } @Override public KTable transformValues(final ValueTransformerWithKeySupplier transformerSupplier, final Materialized> materialized, final String... stateStoreNames) { + return transformValues(transformerSupplier, materialized, NamedInternal.empty(), stateStoreNames); + } + + @Override + public KTable transformValues(final ValueTransformerWithKeySupplier transformerSupplier, + final Materialized> materialized, + final Named named, + final String... stateStoreNames) { Objects.requireNonNull(materialized, "materialized can't be null"); + Objects.requireNonNull(named, "named can't be null"); final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized); - return doTransformValues(transformerSupplier, materializedInternal, stateStoreNames); + return doTransformValues(transformerSupplier, materializedInternal, new NamedInternal(named), stateStoreNames); } private KTable doTransformValues(final ValueTransformerWithKeySupplier transformerSupplier, final MaterializedInternal> materializedInternal, + final NamedInternal namedInternal, final String... stateStoreNames) { Objects.requireNonNull(stateStoreNames, "stateStoreNames"); final Serde keySerde; @@ -331,7 +407,7 @@ private KTable doTransformValues(final ValueTransformerWithKeySuppli storeBuilder = null; } - final String name = builder.newProcessorName(TRANSFORMVALUES_NAME); + final String name = namedInternal.orElseGenerateWithPrefix(builder, TRANSFORMVALUES_NAME); final KTableProcessorSupplier processorSupplier = new KTableTransformValues<>( this, @@ -364,8 +440,14 @@ private KTable doTransformValues(final ValueTransformerWithKeySuppli @Override public KStream toStream() { - final String name = builder.newProcessorName(TOSTREAM_NAME); + return toStream(NamedInternal.empty()); + } + + @Override + public KStream toStream(final Named named) { + Objects.requireNonNull(named, "named can't be null"); + final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, TOSTREAM_NAME); final ProcessorSupplier> kStreamMapValues = new KStreamMapValues<>((key, change) -> change.newValue); final ProcessorParameters processorParameters = unsafeCastProcessorParametersToCompletelyDifferentType( new ProcessorParameters<>(kStreamMapValues, name) @@ -387,6 +469,12 @@ public KStream toStream(final KeyValueMapper KStream toStream(final KeyValueMapper mapper, + final Named named) { + return toStream(named).selectKey(mapper); + } + @Override public KTable suppress(final Suppressed suppressed) { final String name; @@ -407,8 +495,7 @@ public KTable suppress(final Suppressed suppressed) { storeName, this ); - - + final ProcessorGraphNode> node = new StatefulProcessorNode<>( name, new ProcessorParameters<>(suppressionSupplier, name), @@ -452,64 +539,112 @@ private SuppressedInternal buildSuppress(final Suppressed suppress @Override public KTable join(final KTable other, final ValueJoiner joiner) { - return doJoin(other, joiner, null, false, false); + return doJoin(other, joiner, NamedInternal.empty(), null, false, false); + } + + @Override + public KTable join(final KTable other, + final ValueJoiner joiner, + final Named named) { + return doJoin(other, joiner, named, null, false, false); + } + + @Override + public KTable join(final KTable other, + final ValueJoiner joiner, + final Materialized> materialized) { + return join(other, joiner, NamedInternal.empty(), materialized); } @Override public KTable join(final KTable other, final ValueJoiner joiner, + final Named named, final Materialized> materialized) { Objects.requireNonNull(materialized, "materialized can't be null"); final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized, builder, MERGE_NAME); - return doJoin(other, joiner, materializedInternal, false, false); + return doJoin(other, joiner, named, materializedInternal, false, false); } @Override public KTable outerJoin(final KTable other, final ValueJoiner joiner) { - return doJoin(other, joiner, null, true, true); + return outerJoin(other, joiner, NamedInternal.empty()); + } + + @Override + public KTable outerJoin(final KTable other, + final ValueJoiner joiner, + final Named named) { + return doJoin(other, joiner, named, null, true, true); + } + + @Override + public KTable outerJoin(final KTable other, + final ValueJoiner joiner, + final Materialized> materialized) { + return outerJoin(other, joiner, NamedInternal.empty(), materialized); } @Override public KTable outerJoin(final KTable other, final ValueJoiner joiner, + final Named named, final Materialized> materialized) { Objects.requireNonNull(materialized, "materialized can't be null"); final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized, builder, MERGE_NAME); - return doJoin(other, joiner, materializedInternal, true, true); + return doJoin(other, joiner, named, materializedInternal, true, true); } @Override public KTable leftJoin(final KTable other, final ValueJoiner joiner) { - return doJoin(other, joiner, null, true, false); + return leftJoin(other, joiner, NamedInternal.empty()); + } + + @Override + public KTable leftJoin(final KTable other, + final ValueJoiner joiner, + final Named named) { + return doJoin(other, joiner, named, null, true, false); } @Override public KTable leftJoin(final KTable other, final ValueJoiner joiner, final Materialized> materialized) { + return leftJoin(other, joiner, NamedInternal.empty(), materialized); + } + + @Override + public KTable leftJoin(final KTable other, + final ValueJoiner joiner, + final Named named, + final Materialized> materialized) { Objects.requireNonNull(materialized, "materialized can't be null"); final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized, builder, MERGE_NAME); - return doJoin(other, joiner, materializedInternal, true, false); + return doJoin(other, joiner, named, materializedInternal, true, false); } @SuppressWarnings("unchecked") private KTable doJoin(final KTable other, final ValueJoiner joiner, + final Named joinName, final MaterializedInternal> materializedInternal, final boolean leftOuter, final boolean rightOuter) { Objects.requireNonNull(other, "other can't be null"); Objects.requireNonNull(joiner, "joiner can't be null"); + Objects.requireNonNull(joinName, "joinName can't be null"); - final String joinMergeName = builder.newProcessorName(MERGE_NAME); + final NamedInternal renamed = new NamedInternal(joinName); + final String joinMergeName = renamed.orElseGenerateWithPrefix(builder, MERGE_NAME); final Set allSourceNodes = ensureJoinableWith((AbstractStream) other); if (leftOuter) { @@ -533,8 +668,8 @@ private KTable doJoin(final KTable other, joinOther = new KTableKTableOuterJoin<>((KTableImpl) other, this, reverseJoiner(joiner)); } - final String joinThisName = builder.newProcessorName(JOINTHIS_NAME); - final String joinOtherName = builder.newProcessorName(JOINOTHER_NAME); + final String joinThisName = renamed.suffixWithOrElseGet("-join-this", builder, JOINTHIS_NAME); + final String joinOtherName = renamed.suffixWithOrElseGet("-join-other", builder, JOINOTHER_NAME); final ProcessorParameters> joinThisProcessorParameters = new ProcessorParameters<>(joinThis, joinThisName); final ProcessorParameters> joinOtherProcessorParameters = new ProcessorParameters<>(joinOther, joinOtherName); @@ -605,7 +740,8 @@ public KGroupedTable groupBy(final KeyValueMapper grouped) { Objects.requireNonNull(selector, "selector can't be null"); Objects.requireNonNull(grouped, "grouped can't be null"); - final String selectName = builder.newProcessorName(SELECT_NAME); + final GroupedInternal groupedInternal = new GroupedInternal<>(grouped); + final String selectName = new NamedInternal(groupedInternal.name()).orElseGenerateWithPrefix(builder, SELECT_NAME); final KTableProcessorSupplier> selectSupplier = new KTableRepartitionMap<>(this, selector); final ProcessorParameters> processorParameters = new ProcessorParameters<>(selectSupplier, selectName); @@ -616,7 +752,6 @@ public KGroupedTable groupBy(final KeyValueMapper groupedInternal = new GroupedInternal<>(grouped); return new KGroupedTableImpl<>( builder, selectName, diff --git a/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java index 49477b0fb1136..68dd3ac4ac34a 100644 --- a/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java @@ -705,6 +705,35 @@ public void shouldUseSpecifiedNameForFlatTransformValueWithKeyOperation() { assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME); } + @Test + @SuppressWarnings("unchecked") + public void shouldUseSpecifiedNameForToStream() { + builder.table(STREAM_TOPIC) + .toStream(Named.as("to-stream")); + + builder.build(); + final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build(); + assertSpecifiedNameForOperation(topology, + "KSTREAM-SOURCE-0000000001", + "KTABLE-SOURCE-0000000002", + "to-stream"); + } + + @Test + @SuppressWarnings("unchecked") + public void shouldUseSpecifiedNameForToStreamWithMapper() { + builder.table(STREAM_TOPIC) + .toStream(KeyValue::pair, Named.as("to-stream")); + + builder.build(); + final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build(); + assertSpecifiedNameForOperation(topology, + "KSTREAM-SOURCE-0000000001", + "KTABLE-SOURCE-0000000002", + "to-stream", + "KSTREAM-KEY-SELECT-0000000004"); + } + private static void assertSpecifiedNameForOperation(final ProcessorTopology topology, final String... expected) { final List processors = topology.processors(); assertEquals("Invalid number of expected processors", expected.length, processors.size()); diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java index e9f688bbf4da3..f245fec380a89 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java @@ -351,7 +351,7 @@ public void shouldCreateSourceAndSinkNodesForRepartitioningTopic() throws Except @Test(expected = NullPointerException.class) public void shouldNotAllowNullSelectorOnToStream() { - table.toStream(null); + table.toStream((KeyValueMapper) null); } @Test(expected = NullPointerException.class) From c7db82b59a145735fce227436c6191d1825fe384 Mon Sep 17 00:00:00 2001 From: Boyang Chen Date: Mon, 17 Jun 2019 17:11:11 -0700 Subject: [PATCH 0377/1071] MINOR: rename subscription construction function (#6954) Per discussion on #6936, some nit fixes to the Subscription initialization path. Reviewers: Guozhang Wang --- .../consumer/internals/ConsumerCoordinator.java | 4 ++-- .../consumer/internals/ConsumerProtocol.java | 15 ++++++--------- .../kafka/clients/consumer/KafkaConsumerTest.java | 2 +- .../internals/ConsumerCoordinatorTest.java | 2 +- .../consumer/internals/ConsumerProtocolTest.java | 12 ++++++------ 5 files changed, 16 insertions(+), 19 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index de65a325f75e4..477f24bdadd3d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -390,8 +390,8 @@ protected Map performAssignment(String leaderId, Set allSubscribedTopics = new HashSet<>(); Map subscriptions = new HashMap<>(); for (JoinGroupResponseData.JoinGroupResponseMember memberSubscription : allSubscriptions) { - Subscription subscription = ConsumerProtocol.deserializeSubscription(ByteBuffer.wrap(memberSubscription.metadata()), - Optional.ofNullable(memberSubscription.groupInstanceId())); + Subscription subscription = ConsumerProtocol.buildSubscription(ByteBuffer.wrap(memberSubscription.metadata()), + Optional.ofNullable(memberSubscription.groupInstanceId())); subscriptions.put(memberSubscription.memberId(), subscription); allSubscribedTopics.addAll(subscription.topics()); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java index d3737f79e35eb..9e7961ee55338 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java @@ -179,8 +179,7 @@ public static ByteBuffer serializeSubscription(PartitionAssignor.Subscription su } } - public static PartitionAssignor.Subscription deserializeSubscriptionV0(ByteBuffer buffer, - Optional groupInstanceId) { + public static PartitionAssignor.Subscription buildSubscriptionV0(ByteBuffer buffer, Optional groupInstanceId) { Struct struct = SUBSCRIPTION_V0.read(buffer); ByteBuffer userData = struct.getBytes(USER_DATA_KEY_NAME); List topics = new ArrayList<>(); @@ -193,8 +192,7 @@ public static PartitionAssignor.Subscription deserializeSubscriptionV0(ByteBuffe groupInstanceId); } - public static PartitionAssignor.Subscription deserializeSubscriptionV1(ByteBuffer buffer, - Optional groupInstanceId) { + public static PartitionAssignor.Subscription buildSubscriptionV1(ByteBuffer buffer, Optional groupInstanceId) { Struct struct = SUBSCRIPTION_V1.read(buffer); ByteBuffer userData = struct.getBytes(USER_DATA_KEY_NAME); List topics = new ArrayList<>(); @@ -217,8 +215,7 @@ public static PartitionAssignor.Subscription deserializeSubscriptionV1(ByteBuffe groupInstanceId); } - public static PartitionAssignor.Subscription deserializeSubscription(ByteBuffer buffer, - Optional groupInstanceId) { + public static PartitionAssignor.Subscription buildSubscription(ByteBuffer buffer, Optional groupInstanceId) { Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer); Short version = header.getShort(VERSION_KEY_NAME); @@ -227,14 +224,14 @@ public static PartitionAssignor.Subscription deserializeSubscription(ByteBuffer switch (version) { case CONSUMER_PROTOCOL_V0: - return deserializeSubscriptionV0(buffer, groupInstanceId); + return buildSubscriptionV0(buffer, groupInstanceId); case CONSUMER_PROTOCOL_V1: - return deserializeSubscriptionV1(buffer, groupInstanceId); + return buildSubscriptionV1(buffer, groupInstanceId); // assume all higher versions can be parsed as V1 default: - return deserializeSubscriptionV1(buffer, groupInstanceId); + return buildSubscriptionV1(buffer, groupInstanceId); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java index cff71c34e4553..0f9b956663f16 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java @@ -1692,7 +1692,7 @@ public boolean matches(AbstractRequest body) { assertTrue(protocolIterator.hasNext()); ByteBuffer protocolMetadata = ByteBuffer.wrap(protocolIterator.next().metadata()); - PartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(protocolMetadata, Optional.empty()); + PartitionAssignor.Subscription subscription = ConsumerProtocol.buildSubscription(protocolMetadata, Optional.empty()); return subscribedTopics.equals(new HashSet<>(subscription.topics())); } }, joinGroupFollowerResponse(assignor, 1, "memberId", "leaderId", Errors.NONE), coordinator); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java index 8aeec3c729953..f4b87d2f16e2c 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java @@ -600,7 +600,7 @@ public boolean matches(AbstractRequest body) { JoinGroupRequestData.JoinGroupRequestProtocol protocolMetadata = protocolIterator.next(); ByteBuffer metadata = ByteBuffer.wrap(protocolMetadata.metadata()); - PartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(metadata, Optional.empty()); + PartitionAssignor.Subscription subscription = ConsumerProtocol.buildSubscription(metadata, Optional.empty()); metadata.rewind(); return subscription.topics().containsAll(updatedSubscriptionSet); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java index 07eafa2b07994..d89c5a7dc4344 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java @@ -59,7 +59,7 @@ public class ConsumerProtocolTest { public void serializeDeserializeMetadata() { Subscription subscription = new Subscription(Arrays.asList("foo", "bar")); ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription); - Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer, Optional.empty()); + Subscription parsedSubscription = ConsumerProtocol.buildSubscription(buffer, Optional.empty()); assertEquals(subscription.topics(), parsedSubscription.topics()); assertEquals(0, parsedSubscription.userData().limit()); assertFalse(parsedSubscription.groupInstanceId().isPresent()); @@ -70,7 +70,7 @@ public void serializeDeserializeMetadataAndGroupInstanceId() { Subscription subscription = new Subscription(Arrays.asList("foo", "bar")); ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription); - Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer, groupInstanceId); + Subscription parsedSubscription = ConsumerProtocol.buildSubscription(buffer, groupInstanceId); assertEquals(subscription.topics(), parsedSubscription.topics()); assertEquals(groupInstanceId, parsedSubscription.groupInstanceId()); } @@ -79,7 +79,7 @@ public void serializeDeserializeMetadataAndGroupInstanceId() { public void serializeDeserializeNullSubscriptionUserData() { Subscription subscription = new Subscription(Arrays.asList("foo", "bar"), null); ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription); - Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer, Optional.empty()); + Subscription parsedSubscription = ConsumerProtocol.buildSubscription(buffer, Optional.empty()); assertEquals(subscription.topics(), parsedSubscription.topics()); assertNull(parsedSubscription.userData()); assertFalse(parsedSubscription.groupInstanceId().isPresent()); @@ -89,7 +89,7 @@ public void serializeDeserializeNullSubscriptionUserData() { public void deserializeOldSubscriptionVersion() { Subscription subscription = new Subscription((short) 0, Arrays.asList("foo", "bar"), null); ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription); - Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer, groupInstanceId); + Subscription parsedSubscription = ConsumerProtocol.buildSubscription(buffer, groupInstanceId); assertEquals(parsedSubscription.topics(), parsedSubscription.topics()); assertNull(parsedSubscription.userData()); assertTrue(parsedSubscription.ownedPartitions().isEmpty()); @@ -106,7 +106,7 @@ public void deserializeNewSubscriptionWithOldVersion() { // ignore the version assuming it is the old byte code, as it will blindly deserialize as V0 Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer); header.getShort(VERSION_KEY_NAME); - Subscription parsedSubscription = ConsumerProtocol.deserializeSubscriptionV0(buffer, Optional.empty()); + Subscription parsedSubscription = ConsumerProtocol.buildSubscriptionV0(buffer, Optional.empty()); assertEquals(subscription.topics(), parsedSubscription.topics()); assertNull(parsedSubscription.userData()); assertTrue(parsedSubscription.ownedPartitions().isEmpty()); @@ -141,7 +141,7 @@ public void deserializeFutureSubscriptionVersion() { buffer.flip(); - Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer, groupInstanceId); + Subscription parsedSubscription = ConsumerProtocol.buildSubscription(buffer, groupInstanceId); assertEquals(Collections.singletonList("topic"), parsedSubscription.topics()); assertEquals(Collections.singletonList(tp2), parsedSubscription.ownedPartitions()); assertEquals(groupInstanceId, parsedSubscription.groupInstanceId()); From c6ddd31887c3008da97d8a96dc53a408f420fcad Mon Sep 17 00:00:00 2001 From: Konstantine Karantasis Date: Mon, 17 Jun 2019 21:53:20 -0700 Subject: [PATCH 0378/1071] MINOR: Update docs for KIP-415 (#6958) Update docs with KIP-415 details for incremental cooperative rebalancing Author: Konstantine Karantasis Reviewer: Randall Hauch --- docs/connect.html | 35 ++++++++++++++++++++++++++++++++++- docs/upgrade.html | 35 +++++++++++++++++++---------------- 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/docs/connect.html b/docs/connect.html index baefc1b417e4f..a92bb04849fa1 100644 --- a/docs/connect.html +++ b/docs/connect.html @@ -508,7 +508,39 @@

    Kafka Connect

    - When a connector is first submitted to the cluster, the workers rebalance the full set of connectors in the cluster and their tasks so that each worker has approximately the same amount of work. This same rebalancing procedure is also used when connectors increase or decrease the number of tasks they require, or when a connector's configuration is changed. You can use the REST API to view the current status of a connector and its tasks, including the id of the worker to which each was assigned. For example, querying the status of a file source (using GET /connectors/file-source/status) might produce output like the following: + When a connector is first submitted to the cluster, a rebalance is triggered between the Connect workers in order to distribute the load that consists of the tasks of the new connector. + This same rebalancing procedure is also used when connectors increase or decrease the number of tasks they require, when a connector's configuration is changed, or when a + worker is added or removed from the group as part of an intentional upgrade of the Connect cluster or due to a failure. +

    + +

    + In versions prior to 2.3.0, the Connect workers would rebalance the full set of connectors and their tasks in the cluster as a simple way to make sure that each worker has approximately the same amount of work. + This behavior can be still enabled by setting connect.protocol=eager. +

    + +

    + Starting with 2.3.0, Kafka Connect is using by default a protocol that performs + incremental cooperative rebalancing + that incrementally balances the connectors and tasks across the Connect workers, affecting only tasks that are new, to be removed, or need to move from one worker to another. + Other tasks are not stopped and restarted during the rebalance, as they would have been with the old protocol. +

    + +

    + If a Connect worker leaves the group, intentionally or due to a failure, Connect waits for scheduled.rebalance.max.delay.ms before triggering a rebalance. + This delay defaults to five minutes (300000ms) to tolerate failures or upgrades of workers without immediately redistributing the load of a departing worker. + If this worker returns within the configured delay, it gets its previously assigned tasks in full. + However, this means that the tasks will remain unassigned until the time specified by scheduled.rebalance.max.delay.ms elapses. + If a worker does not return within that time limit, Connect will reassign those tasks among the remaining workers in the Connect cluster. +

    + +

    + The new Connect protocol is enabled when all the workers that form the Connect cluster are configured with connect.protocol=compatible, which is also the default value when this property is missing. + Therefore, upgrading to the new Connect protocol happens automatically when all the workers upgrade to 2.3.0. + A rolling upgrade of the Connect cluster will activate incremental cooperative rebalancing when the last worker joins on version 2.3.0. +

    + +

    + You can use the REST API to view the current status of a connector and its tasks, including the ID of the worker to which each was assigned. For example, the GET /connectors/file-source/status request shows the status of a connector named file-source: 

    @@ -537,6 +569,7 @@ 
    Kafka Connect
     
  • RUNNING: The connector/task is running.
    • 
     
  • PAUSED: The connector/task has been administratively paused.
    • 
     
  • FAILED: The connector/task has failed (usually by raising an exception, which is reported in the status output).
    • 
+    
  • DESTROYED: The connector/task has been administratively removed and will stop appearing in the Connect cluster.
    • 
     
 
     
    
diff --git a/docs/upgrade.html b/docs/upgrade.html
index 4f483e6b4f084..4668e6380edc6 100644
--- a/docs/upgrade.html
+++ b/docs/upgrade.html
@@ -19,14 +19,32 @@
 
 
diff --git a/docs/configuration.html b/docs/configuration.html
index 112c8445e8687..dc17333e9dbb3 100644
--- a/docs/configuration.html
+++ b/docs/configuration.html
@@ -285,7 +285,7 @@ 
    3.6 Kafka Streams Configs<
   Below is the configuration of the Kafka Streams client library.
   
 
-  
    3.7 AdminClient Configs
    
+  
    3.7 Admin Configs
    
   Below is the configuration of the Kafka Admin client library.
   
 
diff --git a/docs/security.html b/docs/security.html
index e5e5532545049..f281bfff4166f 100644
--- a/docs/security.html
+++ b/docs/security.html
@@ -920,7 +920,7 @@ 
    7.3 Authentication using SASLTypical steps for delegation token usage are:
    
         
      
         
    1. User authenticates with the Kafka cluster via SASL or SSL, and obtains a delegation token. This can be done
-            using AdminClient APIs or using kafka-delegation-tokens.sh script.
      2. 
+            using Admin APIs or using kafka-delegation-tokens.sh script.
         
    2. User securely passes the delegation token to Kafka clients for authenticating with the Kafka cluster.
      3. 
         
    3. Token owner/renewer can renew/expire the delegation tokens.
      4. 
         
    
@@ -944,7 +944,7 @@ 
    7.3 Authentication using SASL
 
         
  • Creating Delegation Tokens
    
-        
    Tokens can be created by using AdminClient APIs or using kafka-delegation-tokens.sh script.
+        
    Tokens can be created by using Admin APIs or using kafka-delegation-tokens.sh script.
             Delegation token requests (create/renew/expire/describe) should be issued only on SASL or SSL authenticated channels.
             Tokens can not be requests if the initial authentication is done through delegation token.
             kafka-delegation-tokens.sh script examples are given below.
    
@@ -1281,8 +1281,8 @@ 
    Examples
                 Note that for consumer option we must also specify the consumer group.
                 In order to remove a principal from producer or consumer role we just need to pass --remove option. 
    • 
 
-        
  • AdminClient API based acl management
    
-            Users having Alter permission on ClusterResource can use AdminClient API for ACL management. kafka-acls.sh script supports AdminClient API to manage ACLs without interacting with zookeeper/authorizer directly.
+        
  • Admin API based acl management
    
+            Users having Alter permission on ClusterResource can use Admin API for ACL management. kafka-acls.sh script supports AdminClient API to manage ACLs without interacting with zookeeper/authorizer directly.
             All the above examples can be executed by using --bootstrap-server option. For example:
 
             
    diff --git a/docs/toc.html b/docs/toc.html
index a8189216e9175..1223d20d3c25e 100644
--- a/docs/toc.html
+++ b/docs/toc.html
@@ -35,7 +35,7 @@
                 
  • 2.2 Consumer API
                 
  • 2.3 Streams API
                 
  • 2.4 Connect API
-                
  • 2.5 AdminClient API
+                
  • 2.5 Admin API
             
         
    • 
         
  • 3. Configuration
diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaClientSupplier.java b/streams/src/main/java/org/apache/kafka/streams/KafkaClientSupplier.java
index 4ed277018f3ed..cc9d27e02d0fe 100644
--- a/streams/src/main/java/org/apache/kafka/streams/KafkaClientSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/KafkaClientSupplier.java
@@ -16,7 +16,7 @@
  */
 package org.apache.kafka.streams;
 
-import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.producer.Producer;
 import org.apache.kafka.streams.kstream.GlobalKTable;
@@ -31,12 +31,12 @@
  */
 public interface KafkaClientSupplier {
     /**
-     * Create an {@link AdminClient} which is used for internal topic management.
+     * Create an {@link Admin} which is used for internal topic management.
      *
      * @param config Supplied by the {@link java.util.Properties} given to the {@link KafkaStreams}
-     * @return an instance of {@link AdminClient}
+     * @return an instance of {@link Admin}
      */
-    AdminClient getAdminClient(final Map config);
+    Admin getAdminClient(final Map config);
 
     /**
      * Create a {@link Producer} which is used to write records to sink topics.
diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
index 02586f99d15f3..bbf9f9c509745 100644
--- a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
+++ b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
@@ -16,7 +16,7 @@
  */
 package org.apache.kafka.streams;
 
-import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.clients.consumer.KafkaConsumer;
 import org.apache.kafka.clients.producer.KafkaProducer;
 import org.apache.kafka.clients.producer.ProducerConfig;
@@ -141,7 +141,7 @@ public class KafkaStreams implements AutoCloseable {
     private final StreamsMetadataState streamsMetadataState;
     private final ScheduledExecutorService stateDirCleaner;
     private final QueryableStoreProvider queryableStoreProvider;
-    private final AdminClient adminClient;
+    private final Admin adminClient;
 
     private GlobalStreamThread globalStreamThread;
     private KafkaStreams.StateListener stateListener;
diff --git a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
index 6d93b9977df87..f08eecaac4743 100644
--- a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
+++ b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
@@ -17,7 +17,7 @@
 package org.apache.kafka.streams;
 
 import org.apache.kafka.clients.CommonClientConfigs;
-import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.clients.admin.AdminClientConfig;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.clients.consumer.KafkaConsumer;
@@ -58,7 +58,7 @@
 
 /**
  * Configuration for a {@link KafkaStreams} instance.
- * Can also be used to configure the Kafka Streams internal {@link KafkaConsumer}, {@link KafkaProducer} and {@link AdminClient}.
+ * Can also be used to configure the Kafka Streams internal {@link KafkaConsumer}, {@link KafkaProducer} and {@link Admin}.
  * To avoid consumer/producer/admin property conflicts, you should prefix those properties using
  * {@link #consumerPrefix(String)}, {@link #producerPrefix(String)} and {@link #adminClientPrefix(String)}, respectively.
  * 
    
@@ -198,7 +198,7 @@ public class StreamsConfig extends AbstractConfig {
     public static final String PRODUCER_PREFIX = "producer.";
 
     /**
-     * Prefix used to isolate {@link org.apache.kafka.clients.admin.AdminClient admin} configs from other client configs.
+     * Prefix used to isolate {@link Admin admin} configs from other client configs.
      * It is recommended to use {@link #adminClientPrefix(String)} to add this prefix to {@link ProducerConfig producer
      * properties}.
      */
@@ -1116,7 +1116,7 @@ public Map getProducerConfigs(final String clientId) {
     }
 
     /**
-     * Get the configs for the {@link org.apache.kafka.clients.admin.AdminClient admin client}.
+     * Get the configs for the {@link Admin admin client}.
      * @param clientId clientId
      * @return Map of the admin client configuration.
      */
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/DefaultKafkaClientSupplier.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/DefaultKafkaClientSupplier.java
index 69331b462ec42..f56f834493e8a 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/DefaultKafkaClientSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/DefaultKafkaClientSupplier.java
@@ -18,7 +18,7 @@
 
 import java.util.Map;
 
-import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.KafkaConsumer;
 import org.apache.kafka.clients.producer.KafkaProducer;
@@ -29,9 +29,9 @@
 
 public class DefaultKafkaClientSupplier implements KafkaClientSupplier {
     @Override
-    public AdminClient getAdminClient(final Map config) {
+    public Admin getAdminClient(final Map config) {
         // create a new client upon each call; but expect this call to be only triggered once so this should be fine
-        return AdminClient.create(config);
+        return Admin.create(config);
     }
 
     @Override
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java
index 3cb06f69e0e05..320ce114c61e7 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java
@@ -16,7 +16,7 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
-import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.clients.admin.AdminClientConfig;
 import org.apache.kafka.clients.admin.CreateTopicsResult;
 import org.apache.kafka.clients.admin.DescribeTopicsResult;
@@ -53,12 +53,12 @@ private InternalAdminClientConfig(final Map props) {
     private final Map defaultTopicConfigs = new HashMap<>();
 
     private final short replicationFactor;
-    private final AdminClient adminClient;
+    private final Admin adminClient;
 
     private final int retries;
     private final long retryBackOffMs;
 
-    public InternalTopicManager(final AdminClient adminClient,
+    public InternalTopicManager(final Admin adminClient,
                                 final StreamsConfig streamsConfig) {
         this.adminClient = adminClient;
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index ea1e29d0bcba6..1633b30223b62 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -16,7 +16,7 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
-import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
@@ -566,7 +566,7 @@ StandbyTask createTask(final Consumer consumer,
     public static StreamThread create(final InternalTopologyBuilder builder,
                                       final StreamsConfig config,
                                       final KafkaClientSupplier clientSupplier,
-                                      final AdminClient adminClient,
+                                      final Admin adminClient,
                                       final UUID processId,
                                       final String clientId,
                                       final Metrics metrics,
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
index c136fdb806dd4..6b3ab001a73d9 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
@@ -16,7 +16,7 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
-import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.clients.admin.DeleteRecordsResult;
 import org.apache.kafka.clients.admin.RecordsToDelete;
 import org.apache.kafka.clients.consumer.Consumer;
@@ -57,7 +57,7 @@ public class TaskManager {
     private final StreamThread.AbstractTaskCreator standbyTaskCreator;
     private final StreamsMetadataState streamsMetadataState;
 
-    final AdminClient adminClient;
+    final Admin adminClient;
     private DeleteRecordsResult deleteRecordsResult;
 
     // following information is updated during rebalance phase by the partition assignor
@@ -74,7 +74,7 @@ public class TaskManager {
                 final StreamsMetadataState streamsMetadataState,
                 final StreamThread.AbstractTaskCreator taskCreator,
                 final StreamThread.AbstractTaskCreator standbyTaskCreator,
-                final AdminClient adminClient,
+                final Admin adminClient,
                 final AssignedStreamsTasks active,
                 final AssignedStandbyTasks standby) {
         this.changelogReader = changelogReader;
@@ -285,7 +285,7 @@ void shutdown(final boolean clean) {
         }
     }
 
-    AdminClient getAdminClient() {
+    Admin getAdminClient() {
         return adminClient;
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/AbstractResetIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/AbstractResetIntegrationTest.java
index d8c0570465309..675286bd259c8 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/AbstractResetIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/AbstractResetIntegrationTest.java
@@ -18,7 +18,7 @@
 
 import kafka.tools.StreamsResetter;
 import org.apache.kafka.clients.CommonClientConfigs;
-import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.clients.admin.ConsumerGroupDescription;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.clients.producer.ProducerConfig;
@@ -75,7 +75,7 @@ public abstract class AbstractResetIntegrationTest {
 
     private static MockTime mockTime;
     private static KafkaStreams streams;
-    private static AdminClient adminClient = null;
+    private static Admin adminClient = null;
 
     abstract Map getClientSslConfig();
 
@@ -95,7 +95,7 @@ public static void afterClassCleanup() {
 
     private void prepareEnvironment() {
         if (adminClient == null) {
-            adminClient = AdminClient.create(commonClientConfig);
+            adminClient = Admin.create(commonClientConfig);
         }
 
         boolean timeSet = false;
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/InternalTopicIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/InternalTopicIntegrationTest.java
index 0ee027862345b..cdb369c8bfe94 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/InternalTopicIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/InternalTopicIntegrationTest.java
@@ -18,7 +18,7 @@
 
 import kafka.log.LogConfig;
 import kafka.utils.MockTime;
-import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.clients.admin.AdminClientConfig;
 import org.apache.kafka.clients.admin.Config;
 import org.apache.kafka.clients.admin.ConfigEntry;
@@ -113,7 +113,7 @@ private void produceData(final List inputValues) throws Exception {
     }
 
     private Properties getTopicProperties(final String changelog) {
-        try (final AdminClient adminClient = createAdminClient()) {
+        try (final Admin adminClient = createAdminClient()) {
             final ConfigResource configResource = new ConfigResource(ConfigResource.Type.TOPIC, changelog);
             try {
                 final Config config = adminClient.describeConfigs(Collections.singletonList(configResource)).values().get(configResource).get();
@@ -130,10 +130,10 @@ private Properties getTopicProperties(final String changelog) {
         }
     }
 
-    private AdminClient createAdminClient() {
+    private Admin createAdminClient() {
         final Properties adminClientConfig = new Properties();
         adminClientConfig.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
-        return AdminClient.create(adminClientConfig);
+        return Admin.create(adminClientConfig);
     }
 
     @Test
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/PurgeRepartitionTopicIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/PurgeRepartitionTopicIntegrationTest.java
index 0cfa97a1f9d07..efae747952de4 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/PurgeRepartitionTopicIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/PurgeRepartitionTopicIntegrationTest.java
@@ -16,7 +16,7 @@
  */
 package org.apache.kafka.streams.integration;
 
-import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.clients.admin.Config;
 import org.apache.kafka.clients.producer.ProducerConfig;
 import org.apache.kafka.common.TopicPartition;
@@ -60,7 +60,7 @@ public class PurgeRepartitionTopicIntegrationTest {
     private static final String APPLICATION_ID = "restore-test";
     private static final String REPARTITION_TOPIC = APPLICATION_ID + "-KSTREAM-AGGREGATE-STATE-STORE-0000000002-repartition";
 
-    private static AdminClient adminClient;
+    private static Admin adminClient;
     private static KafkaStreams kafkaStreams;
     private static final Integer PURGE_INTERVAL_MS = 10;
     private static final Integer PURGE_SEGMENT_BYTES = 2000;
@@ -148,7 +148,7 @@ public void setup() {
         // create admin client for verification
         final Properties adminConfig = new Properties();
         adminConfig.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
-        adminClient = AdminClient.create(adminConfig);
+        adminClient = Admin.create(adminConfig);
 
         final Properties streamsConfiguration = new Properties();
         streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, APPLICATION_ID);
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/utils/KafkaEmbedded.java b/streams/src/test/java/org/apache/kafka/streams/integration/utils/KafkaEmbedded.java
index eeee3bf7c8db5..43438a5ca40e9 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/utils/KafkaEmbedded.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/utils/KafkaEmbedded.java
@@ -22,7 +22,7 @@
 import kafka.utils.MockTime;
 import kafka.utils.TestUtils;
 import org.apache.kafka.clients.CommonClientConfigs;
-import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.clients.admin.AdminClientConfig;
 import org.apache.kafka.clients.admin.NewTopic;
 import org.apache.kafka.common.config.SslConfigs;
@@ -180,7 +180,7 @@ public void createTopic(final String topic,
         final NewTopic newTopic = new NewTopic(topic, partitions, (short) replication);
         newTopic.configs(topicConfig);
 
-        try (final AdminClient adminClient = createAdminClient()) {
+        try (final Admin adminClient = createAdminClient()) {
             adminClient.createTopics(Collections.singletonList(newTopic)).all().get();
         } catch (final InterruptedException | ExecutionException e) {
             throw new RuntimeException(e);
@@ -188,7 +188,7 @@ public void createTopic(final String topic,
     }
 
     @SuppressWarnings("WeakerAccess")
-    public AdminClient createAdminClient() {
+    public Admin createAdminClient() {
         final Properties adminClientConfig = new Properties();
         adminClientConfig.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList());
         final Object listeners = effectiveConfig.get(KafkaConfig$.MODULE$.ListenersProp());
@@ -197,13 +197,13 @@ public AdminClient createAdminClient() {
             adminClientConfig.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, ((Password) effectiveConfig.get(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG)).value());
             adminClientConfig.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL");
         }
-        return AdminClient.create(adminClientConfig);
+        return Admin.create(adminClientConfig);
     }
 
     @SuppressWarnings("WeakerAccess")
     public void deleteTopic(final String topic) {
         log.debug("Deleting topic { name: {} }", topic);
-        try (final AdminClient adminClient = createAdminClient()) {
+        try (final Admin adminClient = createAdminClient()) {
             adminClient.deleteTopics(Collections.singletonList(topic)).all().get();
         } catch (final InterruptedException | ExecutionException e) {
             if (!(e.getCause() instanceof UnknownTopicOrPartitionException)) {
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java
index 7d7d4e63fdfab..77e0254b9f058 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java
@@ -17,7 +17,7 @@
 
 package org.apache.kafka.streams.processor.internals;
 
-import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.clients.admin.DeleteRecordsResult;
 import org.apache.kafka.clients.admin.DeletedRecords;
 import org.apache.kafka.clients.admin.RecordsToDelete;
@@ -87,7 +87,7 @@ public class TaskManagerTest {
     @Mock(type = MockType.NICE)
     private StreamThread.AbstractTaskCreator standbyTaskCreator;
     @Mock(type = MockType.NICE)
-    private AdminClient adminClient;
+    private Admin adminClient;
     @Mock(type = MockType.NICE)
     private StreamTask streamTask;
     @Mock(type = MockType.NICE)
diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/EosTestDriver.java b/streams/src/test/java/org/apache/kafka/streams/tests/EosTestDriver.java
index 3187359a1cd2d..5daf066f386f8 100644
--- a/streams/src/test/java/org/apache/kafka/streams/tests/EosTestDriver.java
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/EosTestDriver.java
@@ -16,9 +16,8 @@
  */
 package org.apache.kafka.streams.tests;
 
-import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.clients.admin.ConsumerGroupDescription;
-import org.apache.kafka.clients.admin.KafkaAdminClient;
 import org.apache.kafka.clients.admin.ListConsumerGroupOffsetsResult;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
@@ -146,7 +145,7 @@ public static void verify(final String kafka, final boolean withRepartitioning)
         props.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, IsolationLevel.READ_COMMITTED.toString().toLowerCase(Locale.ROOT));
 
         final Map committedOffsets;
-        try (final AdminClient adminClient = KafkaAdminClient.create(props)) {
+        try (final Admin adminClient = Admin.create(props)) {
             ensureStreamsApplicationDown(adminClient);
 
             committedOffsets = getCommittedOffsets(adminClient, withRepartitioning);
@@ -218,7 +217,7 @@ public static void verify(final String kafka, final boolean withRepartitioning)
         System.out.flush();
     }
 
-    private static void ensureStreamsApplicationDown(final AdminClient adminClient) {
+    private static void ensureStreamsApplicationDown(final Admin adminClient) {
 
         final long maxWaitTime = System.currentTimeMillis() + MAX_IDLE_TIME_MS;
         ConsumerGroupDescription description;
@@ -236,7 +235,7 @@ private static void ensureStreamsApplicationDown(final AdminClient adminClient)
     }
 
 
-    private static Map getCommittedOffsets(final AdminClient adminClient,
+    private static Map getCommittedOffsets(final Admin adminClient,
                                                                  final boolean withRepartitioning) {
         final Map topicPartitionOffsetAndMetadataMap;
 
@@ -617,7 +616,7 @@ private static List getAllPartitions(final KafkaConsumer c
     }
 
 
-    private static ConsumerGroupDescription getConsumerGroupDescription(final AdminClient adminClient) {
+    private static ConsumerGroupDescription getConsumerGroupDescription(final Admin adminClient) {
         final ConsumerGroupDescription description;
         try {
             description = adminClient.describeConsumerGroups(Collections.singleton(EosTestClient.APP_ID))
diff --git a/streams/src/test/java/org/apache/kafka/test/MockClientSupplier.java b/streams/src/test/java/org/apache/kafka/test/MockClientSupplier.java
index d3430f2c727c9..4330d6c85cf4f 100644
--- a/streams/src/test/java/org/apache/kafka/test/MockClientSupplier.java
+++ b/streams/src/test/java/org/apache/kafka/test/MockClientSupplier.java
@@ -16,7 +16,7 @@
  */
 package org.apache.kafka.test;
 
-import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.clients.admin.MockAdminClient;
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.MockConsumer;
@@ -57,7 +57,7 @@ public void setClusterForAdminClient(final Cluster cluster) {
     }
 
     @Override
-    public AdminClient getAdminClient(final Map config) {
+    public Admin getAdminClient(final Map config) {
         return new MockAdminClient(cluster.nodes(), cluster.nodeById(0));
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/test/MockInternalTopicManager.java b/streams/src/test/java/org/apache/kafka/test/MockInternalTopicManager.java
index 8ab50b8fa5995..1eebf2479d701 100644
--- a/streams/src/test/java/org/apache/kafka/test/MockInternalTopicManager.java
+++ b/streams/src/test/java/org/apache/kafka/test/MockInternalTopicManager.java
@@ -16,7 +16,7 @@
  */
 package org.apache.kafka.test;
 
-import org.apache.kafka.clients.admin.KafkaAdminClient;
+import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.clients.consumer.MockConsumer;
 import org.apache.kafka.common.PartitionInfo;
 import org.apache.kafka.streams.StreamsConfig;
@@ -30,7 +30,6 @@
 import java.util.Set;
 
 
-
 public class MockInternalTopicManager extends InternalTopicManager {
 
     final public Map readyTopics = new HashMap<>();
@@ -38,7 +37,7 @@ public class MockInternalTopicManager extends InternalTopicManager {
 
     public MockInternalTopicManager(final StreamsConfig streamsConfig,
                                     final MockConsumer restoreConsumer) {
-        super(KafkaAdminClient.create(streamsConfig.originals()), streamsConfig);
+        super(Admin.create(streamsConfig.originals()), streamsConfig);
 
         this.restoreConsumer = restoreConsumer;
     }
diff --git a/tools/src/main/java/org/apache/kafka/tools/ClientCompatibilityTest.java b/tools/src/main/java/org/apache/kafka/tools/ClientCompatibilityTest.java
index 887bdc4d77ff7..89e6b44c8d2be 100644
--- a/tools/src/main/java/org/apache/kafka/tools/ClientCompatibilityTest.java
+++ b/tools/src/main/java/org/apache/kafka/tools/ClientCompatibilityTest.java
@@ -20,7 +20,7 @@
 import net.sourceforge.argparse4j.inf.ArgumentParser;
 import net.sourceforge.argparse4j.inf.ArgumentParserException;
 import net.sourceforge.argparse4j.inf.Namespace;
-import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.clients.admin.AdminClientConfig;
 import org.apache.kafka.clients.admin.NewTopic;
 import org.apache.kafka.clients.admin.TopicListing;
@@ -247,7 +247,7 @@ public void testProduce() throws Exception {
     void testAdminClient() throws Throwable {
         Properties adminProps = new Properties();
         adminProps.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, testConfig.bootstrapServer);
-        try (final AdminClient client = AdminClient.create(adminProps)) {
+        try (final Admin client = Admin.create(adminProps)) {
             while (true) {
                 Collection nodes = client.describeCluster().nodes().get();
                 if (nodes.size() == testConfig.numClusterNodes) {
@@ -297,7 +297,7 @@ void testAdminClient() throws Throwable {
         }
     }
 
-    private void createTopicsResultTest(AdminClient client, Collection topics)
+    private void createTopicsResultTest(Admin client, Collection topics)
             throws InterruptedException, ExecutionException {
         while (true) {
             try {
diff --git a/tools/src/main/java/org/apache/kafka/trogdor/common/WorkerUtils.java b/tools/src/main/java/org/apache/kafka/trogdor/common/WorkerUtils.java
index cb765cc496698..faf2d964bfa5f 100644
--- a/tools/src/main/java/org/apache/kafka/trogdor/common/WorkerUtils.java
+++ b/tools/src/main/java/org/apache/kafka/trogdor/common/WorkerUtils.java
@@ -17,7 +17,7 @@
 
 package org.apache.kafka.trogdor.common;
 
-import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.clients.admin.AdminClientConfig;
 import org.apache.kafka.clients.admin.DescribeTopicsOptions;
 import org.apache.kafka.clients.admin.DescribeTopicsResult;
@@ -131,7 +131,7 @@ public static void createTopics(
         // this method wraps the call to createTopics() that takes admin client, so that we can
         // unit test the functionality with MockAdminClient. The exception is caught and
         // re-thrown so that admin client is closed when the method returns.
-        try (AdminClient adminClient
+        try (Admin adminClient
                  = createAdminClient(bootstrapServers, commonClientConf, adminClientConf)) {
             createTopics(log, adminClient, topics, failOnExisting);
         } catch (Exception e) {
@@ -148,7 +148,7 @@ public static void createTopics(
      * @throws Throwable if creation of one or more topics fails (except for the cases above).
      */
     static void createTopics(
-        Logger log, AdminClient adminClient,
+        Logger log, Admin adminClient,
         Map topics, boolean failOnExisting) throws Throwable {
         if (topics.isEmpty()) {
             log.warn("Request to create topics has an empty topic list.");
@@ -174,7 +174,7 @@ static void createTopics(
      * @return                Collection of topics names that already exist.
      * @throws Throwable if creation of one or more topics fails (except for topic exists case).
      */
-    private static Collection createTopics(Logger log, AdminClient adminClient,
+    private static Collection createTopics(Logger log, Admin adminClient,
                                                    Collection topics) throws Throwable {
         long startMs = Time.SYSTEM.milliseconds();
         int tries = 0;
@@ -249,7 +249,7 @@ private static Collection createTopics(Logger log, AdminClient adminClie
      * described in 'topicsInfo'
      */
     static void verifyTopics(
-        Logger log, AdminClient adminClient,
+        Logger log, Admin adminClient,
         Collection topicsToVerify, Map topicsInfo, int retryCount, long retryBackoffMs) throws Throwable {
 
         Map topicDescriptionMap = topicDescriptions(topicsToVerify, adminClient,
@@ -270,7 +270,7 @@ static void verifyTopics(
     }
 
     private static Map topicDescriptions(Collection topicsToVerify,
-                                                                   AdminClient adminClient,
+                                                                   Admin adminClient,
                                                                    int retryCount, long retryBackoffMs)
             throws ExecutionException, InterruptedException {
         UnknownTopicOrPartitionException lastException = null;
@@ -300,7 +300,7 @@ private static Map topicDescriptions(Collection getMatchingTopicPartitions(
-        AdminClient adminClient, String topicRegex, int startPartition, int endPartition)
+        Admin adminClient, String topicRegex, int startPartition, int endPartition)
         throws Throwable {
         final Pattern topicNamePattern = Pattern.compile(topicRegex);
 
@@ -332,7 +332,7 @@ static Collection getMatchingTopicPartitions(
         return out;
     }
 
-    private static AdminClient createAdminClient(
+    private static Admin createAdminClient(
         String bootstrapServers,
         Map commonClientConf, Map adminClientConf) {
         Properties props = new Properties();
@@ -341,6 +341,6 @@ private static AdminClient createAdminClient(
         // first add common client config, and then admin client config to properties, possibly
         // over-writing default or common properties.
         addConfigsToProperties(props, commonClientConf, adminClientConf);
-        return AdminClient.create(props);
+        return Admin.create(props);
     }
 }
diff --git a/tools/src/main/java/org/apache/kafka/trogdor/workload/ConnectionStressWorker.java b/tools/src/main/java/org/apache/kafka/trogdor/workload/ConnectionStressWorker.java
index 5c5db48c6d4dc..cef8d2d0d569d 100644
--- a/tools/src/main/java/org/apache/kafka/trogdor/workload/ConnectionStressWorker.java
+++ b/tools/src/main/java/org/apache/kafka/trogdor/workload/ConnectionStressWorker.java
@@ -26,7 +26,7 @@
 import org.apache.kafka.clients.ManualMetadataUpdater;
 import org.apache.kafka.clients.NetworkClient;
 import org.apache.kafka.clients.NetworkClientUtils;
-import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.clients.admin.AdminClientConfig;
 import org.apache.kafka.clients.producer.ProducerConfig;
 import org.apache.kafka.common.Cluster;
@@ -207,7 +207,7 @@ static class FetchMetadataStressor  implements Stressor {
 
         @Override
         public boolean tryConnect() {
-            try (AdminClient client = AdminClient.create(this.props)) {
+            try (Admin client = Admin.create(this.props)) {
                 client.describeCluster().nodes().get();
             } catch (RuntimeException e) {
                 return false;

From f98e176746d663fadedbcd3c18312a7f476a20c8 Mon Sep 17 00:00:00 2001
From: Boyang Chen 
Date: Mon, 22 Jul 2019 16:13:02 -0700
Subject: [PATCH 0474/1071] KAFKA-8678; Fix leave group protocol bug in
 throttling and error response (#7101)

This is a bug fix PR to resolve errors introduced in https://github.com/apache/kafka/pull/6188. The PR fixes 2 things:

1. throttle time should be set on version >= 1 instead of version >= 2
2. `getErrorResponse` should set throwable exception within LeaveGroupResponseData

The patch also adds more unit tests to guarantee correctness for leave group protocol.

Reviewers: Guozhang Wang , Jason Gustafson 
---
 .../common/requests/LeaveGroupRequest.java    | 11 ++-
 .../common/requests/LeaveGroupResponse.java   | 12 +++
 .../requests/LeaveGroupRequestTest.java       | 60 +++++++++++++
 .../requests/LeaveGroupResponseTest.java      | 85 +++++++++++++++++++
 4 files changed, 164 insertions(+), 4 deletions(-)
 create mode 100644 clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupRequestTest.java
 create mode 100644 clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java

diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java
index 20fdf4ffac78e..e6e239c757f06 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java
@@ -19,6 +19,7 @@
 import org.apache.kafka.common.message.LeaveGroupRequestData;
 import org.apache.kafka.common.message.LeaveGroupResponseData;
 import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.protocol.types.Struct;
 
 import java.nio.ByteBuffer;
@@ -65,11 +66,13 @@ public LeaveGroupRequestData data() {
 
     @Override
     public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
-        LeaveGroupResponseData response = new LeaveGroupResponseData();
-        if (version() >= 2) {
-            response.setThrottleTimeMs(throttleTimeMs);
+        LeaveGroupResponseData responseData = new LeaveGroupResponseData()
+                                                  .setErrorCode(Errors.forException(e).code());
+
+        if (version() >= 1) {
+            responseData.setThrottleTimeMs(throttleTimeMs);
         }
-        return new LeaveGroupResponse(response);
+        return new LeaveGroupResponse(responseData);
     }
 
     public static LeaveGroupRequest parse(ByteBuffer buffer, short version) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java
index 3f67bb0b1ad30..6390c0f6d909a 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java
@@ -24,6 +24,7 @@
 import java.nio.ByteBuffer;
 import java.util.Collections;
 import java.util.Map;
+import java.util.Objects;
 
 public class LeaveGroupResponse extends AbstractResponse {
 
@@ -72,4 +73,15 @@ public static LeaveGroupResponse parse(ByteBuffer buffer, short versionId) {
     public boolean shouldClientThrottle(short version) {
         return version >= 2;
     }
+
+    @Override
+    public boolean equals(Object other) {
+        return other instanceof LeaveGroupResponse &&
+                   ((LeaveGroupResponse) other).data.equals(this.data);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hashCode(data);
+    }
 }
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupRequestTest.java
new file mode 100644
index 0000000000000..9fa9d3b6d0ba8
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupRequestTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.requests;
+
+import org.apache.kafka.common.message.LeaveGroupRequestData;
+import org.apache.kafka.common.message.LeaveGroupResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class LeaveGroupRequestTest {
+
+    @Test
+    public void testLeaveConstructor() {
+        final String groupId = "group_id";
+        final String memberId = "member_id";
+        final int throttleTimeMs = 10;
+
+        final LeaveGroupRequestData expectedData = new LeaveGroupRequestData()
+                                                       .setGroupId(groupId)
+                                                       .setMemberId(memberId);
+
+        final LeaveGroupRequest.Builder builder =
+            new LeaveGroupRequest.Builder(new LeaveGroupRequestData()
+                                              .setGroupId(groupId)
+                                              .setMemberId(memberId));
+
+        for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) {
+            LeaveGroupRequest request = builder.build(version);
+            assertEquals(expectedData, request.data());
+
+            int expectedThrottleTime = version >= 1 ? throttleTimeMs
+                                           : AbstractResponse.DEFAULT_THROTTLE_TIME;
+            LeaveGroupResponse expectedResponse = new LeaveGroupResponse(
+                new LeaveGroupResponseData()
+                    .setErrorCode(Errors.NOT_CONTROLLER.code())
+                    .setThrottleTimeMs(expectedThrottleTime)
+            );
+
+            assertEquals(expectedResponse, request.getErrorResponse(throttleTimeMs,
+                                                                    Errors.NOT_CONTROLLER.exception()));
+        }
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java
new file mode 100644
index 0000000000000..a1368b54dcbfa
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.requests;
+
+import org.apache.kafka.common.message.LeaveGroupResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.Map;
+
+import static org.apache.kafka.common.requests.AbstractResponse.DEFAULT_THROTTLE_TIME;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class LeaveGroupResponseTest {
+    private final int throttleTimeMs = 10;
+
+    @Test
+    public void testConstructor() {
+        Map expectedErrorCounts = Collections.singletonMap(Errors.NOT_COORDINATOR, 1);
+
+        LeaveGroupResponseData responseData = new LeaveGroupResponseData()
+                                                  .setErrorCode(Errors.NOT_COORDINATOR.code())
+                                                  .setThrottleTimeMs(throttleTimeMs);
+        for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) {
+            LeaveGroupResponse leaveGroupResponse = new LeaveGroupResponse(responseData.toStruct(version), version);
+
+            assertEquals(expectedErrorCounts, leaveGroupResponse.errorCounts());
+
+            if (version >= 1) {
+                assertEquals(throttleTimeMs, leaveGroupResponse.throttleTimeMs());
+            } else {
+                assertEquals(DEFAULT_THROTTLE_TIME, leaveGroupResponse.throttleTimeMs());
+            }
+
+            assertEquals(Errors.NOT_COORDINATOR, leaveGroupResponse.error());
+        }
+    }
+
+    @Test
+    public void testShouldThrottle() {
+        // A dummy setup is ok.
+        LeaveGroupResponse response = new LeaveGroupResponse(new LeaveGroupResponseData());
+        for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) {
+            if (version >= 2) {
+                assertTrue(response.shouldClientThrottle(version));
+            } else {
+                assertFalse(response.shouldClientThrottle(version));
+            }
+        }
+    }
+
+    @Test
+    public void testEquality() {
+        LeaveGroupResponseData responseData = new LeaveGroupResponseData()
+            .setErrorCode(Errors.NONE.code())
+            .setThrottleTimeMs(throttleTimeMs);
+        for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) {
+            LeaveGroupResponse primaryResponse = new LeaveGroupResponse(responseData.toStruct(version), version);
+
+            LeaveGroupResponse secondaryResponse = new LeaveGroupResponse(responseData.toStruct(version), version);
+
+            assertEquals(primaryResponse, primaryResponse);
+            assertEquals(primaryResponse, secondaryResponse);
+            assertEquals(primaryResponse.hashCode(), secondaryResponse.hashCode());
+        }
+    }
+}

From 6b6a6b930fda853fd91dfbe85b4462e17654f804 Mon Sep 17 00:00:00 2001
From: Igor Soarez 
Date: Tue, 23 Jul 2019 16:57:38 +0100
Subject: [PATCH 0475/1071] KAFKA-8526; Fallback to other log dirs after
 getOrCreateLog failure (#6969)

 LogManager#getOrCreateLog() selects a log dir for the new replica from
 _liveLogDirs, if disk failure is discovered at this point, before
 LogDirFailureHandler finds out, try using other log dirs before failing
 the operation.

Reviewers: Anna Povzner , Jason Gustafson 
---
 .../src/main/scala/kafka/log/LogManager.scala | 109 ++++++++++--------
 .../scala/unit/kafka/log/LogManagerTest.scala |  50 +++++++-
 2 files changed, 112 insertions(+), 47 deletions(-)

diff --git a/core/src/main/scala/kafka/log/LogManager.scala b/core/src/main/scala/kafka/log/LogManager.scala
index 320f346873397..6c724c3eecd56 100755
--- a/core/src/main/scala/kafka/log/LogManager.scala
+++ b/core/src/main/scala/kafka/log/LogManager.scala
@@ -34,6 +34,7 @@ import org.apache.kafka.common.errors.{KafkaStorageException, LogDirNotFoundExce
 import scala.collection.JavaConverters._
 import scala.collection._
 import scala.collection.mutable.ArrayBuffer
+import scala.util.{Failure, Success, Try}
 
 /**
  * The entry point to the kafka log management subsystem. The log manager is responsible for log creation, retrieval, and cleaning.
@@ -676,7 +677,7 @@ class LogManager(logDirs: Seq[File],
         if (!isNew && offlineLogDirs.nonEmpty)
           throw new KafkaStorageException(s"Can not create log for $topicPartition because log directories ${offlineLogDirs.mkString(",")} are offline")
 
-        val logDir = {
+        val logDirs: List[File] = {
           val preferredLogDir = preferredLogDirs.get(topicPartition)
 
           if (isFuture) {
@@ -687,55 +688,70 @@ class LogManager(logDirs: Seq[File],
           }
 
           if (preferredLogDir != null)
-            preferredLogDir
+            List(new File(preferredLogDir))
           else
-            nextLogDir().getAbsolutePath
+            nextLogDirs()
         }
-        if (!isLogDirOnline(logDir))
-          throw new KafkaStorageException(s"Can not create log for $topicPartition because log directory $logDir is offline")
-
-        try {
-          val dir = {
-            if (isFuture)
-              new File(logDir, Log.logFutureDirName(topicPartition))
-            else
-              new File(logDir, Log.logDirName(topicPartition))
-          }
-          Files.createDirectories(dir.toPath)
-
-          val log = Log(
-            dir = dir,
-            config = config,
-            logStartOffset = 0L,
-            recoveryPoint = 0L,
-            maxProducerIdExpirationMs = maxPidExpirationMs,
-            producerIdExpirationCheckIntervalMs = LogManager.ProducerIdExpirationCheckIntervalMs,
-            scheduler = scheduler,
-            time = time,
-            brokerTopicStats = brokerTopicStats,
-            logDirFailureChannel = logDirFailureChannel)
 
+        val logDirName = {
           if (isFuture)
-            futureLogs.put(topicPartition, log)
+            Log.logFutureDirName(topicPartition)
           else
-            currentLogs.put(topicPartition, log)
+            Log.logDirName(topicPartition)
+        }
 
-          info(s"Created log for partition $topicPartition in $logDir with properties " +
-            s"{${config.originals.asScala.mkString(", ")}}.")
-          // Remove the preferred log dir since it has already been satisfied
-          preferredLogDirs.remove(topicPartition)
+        val logDir = logDirs
+          .toStream // to prevent actually mapping the whole list, lazy map
+          .map(createLogDirectory(_, logDirName))
+          .find(_.isSuccess)
+          .getOrElse(Failure(new KafkaStorageException("No log directories available. Tried " + logDirs.map(_.getAbsolutePath).mkString(", "))))
+          .get // If Failure, will throw
+
+        val log = Log(
+          dir = logDir,
+          config = config,
+          logStartOffset = 0L,
+          recoveryPoint = 0L,
+          maxProducerIdExpirationMs = maxPidExpirationMs,
+          producerIdExpirationCheckIntervalMs = LogManager.ProducerIdExpirationCheckIntervalMs,
+          scheduler = scheduler,
+          time = time,
+          brokerTopicStats = brokerTopicStats,
+          logDirFailureChannel = logDirFailureChannel)
 
-          log
-        } catch {
-          case e: IOException =>
-            val msg = s"Error while creating log for $topicPartition in dir $logDir"
-            logDirFailureChannel.maybeAddOfflineLogDir(logDir, msg, e)
-            throw new KafkaStorageException(msg, e)
-        }
+        if (isFuture)
+          futureLogs.put(topicPartition, log)
+        else
+          currentLogs.put(topicPartition, log)
+
+        info(s"Created log for partition $topicPartition in $logDir with properties " + s"{${config.originals.asScala.mkString(", ")}}.")
+        // Remove the preferred log dir since it has already been satisfied
+        preferredLogDirs.remove(topicPartition)
+
+        log
       }
     }
   }
 
+  private[log] def createLogDirectory(logDir: File, logDirName: String): Try[File] = {
+    val logDirPath = logDir.getAbsolutePath
+    if (isLogDirOnline(logDirPath)) {
+      val dir = new File(logDirPath, logDirName)
+      try {
+        Files.createDirectories(dir.toPath)
+        Success(dir)
+      } catch {
+        case e: IOException =>
+          val msg = s"Error while creating log for $logDirName in dir $logDirPath"
+          logDirFailureChannel.maybeAddOfflineLogDir(logDirPath, msg, e)
+          warn(msg, e)
+          Failure(new KafkaStorageException(msg, e))
+      }
+    } else {
+      Failure(new KafkaStorageException(s"Can not create log $logDirName because log directory $logDirPath is offline"))
+    }
+  }
+
   /**
    *  Delete logs marked for deletion. Delete all logs for which `currentDefaultConfig.fileDeleteDelayMs`
    *  has elapsed after the delete was scheduled. Logs for which this interval has not yet elapsed will be
@@ -869,13 +885,13 @@ class LogManager(logDirs: Seq[File],
   }
 
   /**
-   * Choose the next directory in which to create a log. Currently this is done
-   * by calculating the number of partitions in each directory and then choosing the
-   * data directory with the fewest partitions.
+   * Provides the full ordered list of suggested directories for the next partition.
+   * Currently this is done by calculating the number of partitions in each directory and then sorting the
+   * data directories by fewest partitions.
    */
-  private def nextLogDir(): File = {
+  private def nextLogDirs(): List[File] = {
     if(_liveLogDirs.size == 1) {
-      _liveLogDirs.peek()
+      List(_liveLogDirs.peek())
     } else {
       // count the number of logs in each parent directory (including 0 for empty directories
       val logCounts = allLogs.groupBy(_.dir.getParent).mapValues(_.size)
@@ -883,8 +899,9 @@ class LogManager(logDirs: Seq[File],
       val dirCounts = (zeros ++ logCounts).toBuffer
 
       // choose the directory with the least logs in it
-      val leastLoaded = dirCounts.sortBy(_._2).head
-      new File(leastLoaded._1)
+      dirCounts.sortBy(_._2).map {
+        case (path: String, _: Int) => new File(path)
+      }.toList
     }
   }
 
diff --git a/core/src/test/scala/unit/kafka/log/LogManagerTest.scala b/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
index 1e6d2dc2420e5..bfbd423371867 100755
--- a/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
@@ -23,11 +23,18 @@ import java.util.{Collections, Properties}
 import kafka.server.FetchDataInfo
 import kafka.server.checkpoints.OffsetCheckpointFile
 import kafka.utils._
-import org.apache.kafka.common.{KafkaException, TopicPartition}
 import org.apache.kafka.common.errors.OffsetOutOfRangeException
 import org.apache.kafka.common.utils.Utils
+import org.apache.kafka.common.{KafkaException, TopicPartition}
 import org.junit.Assert._
 import org.junit.{After, Before, Test}
+import org.mockito.ArgumentMatchers.any
+import org.mockito.Mockito.{doAnswer, spy}
+import org.mockito.invocation.InvocationOnMock
+import org.mockito.stubbing.Answer
+
+import scala.collection.mutable
+import scala.util.{Failure, Try}
 
 class LogManagerTest {
 
@@ -95,6 +102,47 @@ class LogManagerTest {
     log.appendAsLeader(TestUtils.singletonRecords("test".getBytes()), leaderEpoch = 0)
   }
 
+  @Test
+  def testCreateLogWithLogDirFallback() {
+    // Configure a number of directories one level deeper in logDir,
+    // so they all get cleaned up in tearDown().
+    val dirs = (0 to 4)
+      .map(_.toString)
+      .map(logDir.toPath.resolve(_).toFile)
+
+    // Create a new LogManager with the configured directories and an overridden createLogDirectory.
+    logManager.shutdown()
+    logManager = spy(createLogManager(dirs))
+    val brokenDirs = mutable.Set[File]()
+    doAnswer(new Answer[Try[File]] {
+      override def answer(invocation: InvocationOnMock): Try[File] = {
+        // The first half of directories tried will fail, the rest goes through.
+        val logDir = invocation.getArgument[File](0)
+        if (brokenDirs.contains(logDir) || brokenDirs.size < dirs.length / 2) {
+          brokenDirs.add(logDir)
+          Failure(new Throwable("broken dir"))
+        } else {
+          invocation.callRealMethod().asInstanceOf[Try[File]]
+        }
+      }
+    }).when(logManager).createLogDirectory(any(), any())
+    logManager.startup()
+
+    // Request creating a new log.
+    // LogManager should try using all configured log directories until one succeeds.
+    logManager.getOrCreateLog(new TopicPartition(name, 0), logConfig, isNew = true)
+
+    // Verify that half the directories were considered broken,
+    assertEquals(dirs.length / 2, brokenDirs.size)
+
+    // and that exactly one log file was created,
+    val containsLogFile: File => Boolean = dir => new File(dir, name + "-0").exists()
+    assertEquals("More than one log file created", 1, dirs.count(containsLogFile))
+
+    // and that it wasn't created in one of the broken directories.
+    assertFalse(brokenDirs.exists(containsLogFile))
+  }
+
   /**
    * Test that get on a non-existent returns None and no log is created.
    */

From a8aedc85ebfadcf1472acafe2e0311a73d3040be Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Tue, 23 Jul 2019 18:54:20 -0500
Subject: [PATCH 0476/1071] KAFKA-8696: clean up Sum/Count/Total metrics
 (#7057)

* Clean up one redundant and one misplaced metric
* Clarify the relationship among these metrics to avoid future confusion

Reviewers: Matthias J. Sax , Bill Bejeck , Guozhang Wang 
---
 .../internals/AbstractCoordinator.java        |  4 +-
 .../clients/consumer/internals/Fetcher.java   |  4 +-
 .../kafka/common/metrics/MeasurableStat.java  |  2 +-
 .../kafka/common/metrics/stats/Count.java     | 30 ++-------
 .../metrics/stats}/CumulativeCount.java       | 22 +++----
 .../common/metrics/stats/CumulativeSum.java   | 50 +++++++++++++++
 .../kafka/common/metrics/stats/Meter.java     | 24 ++++---
 .../kafka/common/metrics/stats/Rate.java      | 27 ++------
 .../kafka/common/metrics/stats/Sum.java       | 30 ++-------
 .../kafka/common/metrics/stats/Total.java     | 36 +++--------
 .../common/metrics/stats/WindowedCount.java   | 35 ++++++++++
 .../common/metrics/stats/WindowedSum.java     | 48 ++++++++++++++
 .../apache/kafka/common/network/Selector.java | 18 +++---
 .../kafka/common/metrics/JmxReporterTest.java | 14 ++--
 .../kafka/common/metrics/KafkaMbeanTest.java  |  8 +--
 .../kafka/common/metrics/MetricsTest.java     | 64 +++++++++----------
 .../kafka/common/metrics/SensorTest.java      |  4 +-
 .../kafka/common/metrics/stats/MeterTest.java |  2 +-
 .../org/apache/kafka/test/MetricsBench.java   |  4 +-
 .../apache/kafka/connect/runtime/Worker.java  | 14 ++--
 .../kafka/connect/runtime/WorkerSinkTask.java | 10 +--
 .../connect/runtime/WorkerSourceTask.java     |  6 +-
 .../distributed/DistributedHerder.java        |  4 +-
 .../runtime/errors/ErrorHandlingMetrics.java  | 16 ++---
 .../scala/kafka/network/SocketServer.scala    |  5 +-
 .../kafka/server/ClientQuotaManager.scala     |  4 +-
 .../kstream/internals/metrics/Sensors.java    |  8 +--
 .../processor/internals/StreamTask.java       |  8 +--
 .../internals/metrics/StreamsMetricsImpl.java |  5 +-
 .../internals/RecordCollectorTest.java        |  4 +-
 .../processor/internals/StandbyTaskTest.java  |  4 +-
 .../kafka/streams/TopologyTestDriver.java     |  8 +--
 32 files changed, 292 insertions(+), 230 deletions(-)
 rename {streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics => clients/src/main/java/org/apache/kafka/common/metrics/stats}/CumulativeCount.java (66%)
 create mode 100644 clients/src/main/java/org/apache/kafka/common/metrics/stats/CumulativeSum.java
 create mode 100644 clients/src/main/java/org/apache/kafka/common/metrics/stats/WindowedCount.java
 create mode 100644 clients/src/main/java/org/apache/kafka/common/metrics/stats/WindowedSum.java

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index d5faa6e79dabc..b92a4a6109107 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -42,9 +42,9 @@
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
-import org.apache.kafka.common.metrics.stats.Count;
 import org.apache.kafka.common.metrics.stats.Max;
 import org.apache.kafka.common.metrics.stats.Meter;
+import org.apache.kafka.common.metrics.stats.WindowedCount;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.requests.FindCoordinatorRequest;
 import org.apache.kafka.common.requests.FindCoordinatorRequest.CoordinatorType;
@@ -961,7 +961,7 @@ public void onSuccess(ClientResponse clientResponse, RequestFuture future) {
     }
 
     protected Meter createMeter(Metrics metrics, String groupName, String baseName, String descriptiveName) {
-        return new Meter(new Count(),
+        return new Meter(new WindowedCount(),
                 metrics.metricName(baseName + "-rate", groupName,
                         String.format("The number of %s per second", descriptiveName)),
                 metrics.metricName(baseName + "-total", groupName,
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
index 08ab9fbbc5084..d4d028d38dc73 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
@@ -47,10 +47,10 @@
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
-import org.apache.kafka.common.metrics.stats.Count;
 import org.apache.kafka.common.metrics.stats.Max;
 import org.apache.kafka.common.metrics.stats.Meter;
 import org.apache.kafka.common.metrics.stats.Min;
+import org.apache.kafka.common.metrics.stats.WindowedCount;
 import org.apache.kafka.common.metrics.stats.Value;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
@@ -1657,7 +1657,7 @@ private FetchManagerMetrics(Metrics metrics, FetcherMetricsRegistry metricsRegis
             this.fetchLatency = metrics.sensor("fetch-latency");
             this.fetchLatency.add(metrics.metricInstance(metricsRegistry.fetchLatencyAvg), new Avg());
             this.fetchLatency.add(metrics.metricInstance(metricsRegistry.fetchLatencyMax), new Max());
-            this.fetchLatency.add(new Meter(new Count(), metrics.metricInstance(metricsRegistry.fetchRequestRate),
+            this.fetchLatency.add(new Meter(new WindowedCount(), metrics.metricInstance(metricsRegistry.fetchRequestRate),
                     metrics.metricInstance(metricsRegistry.fetchRequestTotal)));
 
             this.recordsFetchLag = metrics.sensor("records-lag");
diff --git a/clients/src/main/java/org/apache/kafka/common/metrics/MeasurableStat.java b/clients/src/main/java/org/apache/kafka/common/metrics/MeasurableStat.java
index aedac9a764c82..035449e7363a2 100644
--- a/clients/src/main/java/org/apache/kafka/common/metrics/MeasurableStat.java
+++ b/clients/src/main/java/org/apache/kafka/common/metrics/MeasurableStat.java
@@ -19,7 +19,7 @@
 /**
  * A MeasurableStat is a {@link Stat} that is also {@link Measurable} (i.e. can produce a single floating point value).
  * This is the interface used for most of the simple statistics such as {@link org.apache.kafka.common.metrics.stats.Avg},
- * {@link org.apache.kafka.common.metrics.stats.Max}, {@link org.apache.kafka.common.metrics.stats.Count}, etc.
+ * {@link org.apache.kafka.common.metrics.stats.Max}, {@link org.apache.kafka.common.metrics.stats.CumulativeCount}, etc.
  */
 public interface MeasurableStat extends Stat, Measurable {
 
diff --git a/clients/src/main/java/org/apache/kafka/common/metrics/stats/Count.java b/clients/src/main/java/org/apache/kafka/common/metrics/stats/Count.java
index 3da91c4e908c6..2bef7cf93d8bb 100644
--- a/clients/src/main/java/org/apache/kafka/common/metrics/stats/Count.java
+++ b/clients/src/main/java/org/apache/kafka/common/metrics/stats/Count.java
@@ -16,30 +16,14 @@
  */
 package org.apache.kafka.common.metrics.stats;
 
-import java.util.List;
-
-import org.apache.kafka.common.metrics.MetricConfig;
-
 /**
  * A {@link SampledStat} that maintains a simple count of what it has seen.
+ * This is a special kind of {@link WindowedSum} that always records a value of {@code 1} instead of the provided value.
+ *
+ * See also {@link CumulativeCount} for a non-sampled version of this metric.
+ *
+ * @deprecated since 2.4 . Use {@link WindowedCount} instead
  */
-public class Count extends SampledStat {
-
-    public Count() {
-        super(0);
-    }
-
-    @Override
-    protected void update(Sample sample, MetricConfig config, double value, long now) {
-        sample.value += 1.0;
-    }
-
-    @Override
-    public double combine(List samples, MetricConfig config, long now) {
-        double total = 0.0;
-        for (Sample sample : samples)
-            total += sample.value;
-        return total;
-    }
-
+@Deprecated
+public class Count extends WindowedCount {
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/CumulativeCount.java b/clients/src/main/java/org/apache/kafka/common/metrics/stats/CumulativeCount.java
similarity index 66%
rename from streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/CumulativeCount.java
rename to clients/src/main/java/org/apache/kafka/common/metrics/stats/CumulativeCount.java
index 2c12c2b6e9d41..85591b5cf726f 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/CumulativeCount.java
+++ b/clients/src/main/java/org/apache/kafka/common/metrics/stats/CumulativeCount.java
@@ -14,25 +14,21 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.kafka.streams.processor.internals.metrics;
+package org.apache.kafka.common.metrics.stats;
 
-import org.apache.kafka.common.metrics.MeasurableStat;
 import org.apache.kafka.common.metrics.MetricConfig;
 
 /**
- * A non-SampledStat version of Count for measuring -total metrics in streams
+ * A non-sampled version of {@link WindowedCount} maintained over all time.
+ *
+ * This is a special kind of {@link CumulativeSum} that always records {@code 1} instead of the provided value.
+ * In other words, it counts the number of
+ * {@link CumulativeCount#record(MetricConfig, double, long)} invocations,
+ * instead of summing the recorded values.
  */
-public class CumulativeCount implements MeasurableStat {
-
-    private double count = 0.0;
-
+public class CumulativeCount extends CumulativeSum {
     @Override
     public void record(final MetricConfig config, final double value, final long timeMs) {
-        count += 1;
-    }
-
-    @Override
-    public double measure(final MetricConfig config, final long now) {
-        return count;
+        super.record(config, 1, timeMs);
     }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/metrics/stats/CumulativeSum.java b/clients/src/main/java/org/apache/kafka/common/metrics/stats/CumulativeSum.java
new file mode 100644
index 0000000000000..13f12a1bb09d4
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/metrics/stats/CumulativeSum.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.metrics.stats;
+
+import org.apache.kafka.common.metrics.MeasurableStat;
+import org.apache.kafka.common.metrics.MetricConfig;
+
+/**
+ * An non-sampled cumulative total maintained over all time.
+ * This is a non-sampled version of {@link WindowedSum}.
+ *
+ * See also {@link CumulativeCount} if you just want to increment the value by 1 on each recording.
+ */
+public class CumulativeSum implements MeasurableStat {
+
+    private double total;
+
+    public CumulativeSum() {
+        total = 0.0;
+    }
+
+    public CumulativeSum(double value) {
+        total = value;
+    }
+
+    @Override
+    public void record(MetricConfig config, double value, long now) {
+        total += value;
+    }
+
+    @Override
+    public double measure(MetricConfig config, long now) {
+        return total;
+    }
+
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/metrics/stats/Meter.java b/clients/src/main/java/org/apache/kafka/common/metrics/stats/Meter.java
index 91d4461d2b52f..a6bdc9f3c108f 100644
--- a/clients/src/main/java/org/apache/kafka/common/metrics/stats/Meter.java
+++ b/clients/src/main/java/org/apache/kafka/common/metrics/stats/Meter.java
@@ -23,48 +23,46 @@
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.metrics.CompoundStat;
 import org.apache.kafka.common.metrics.MetricConfig;
-import org.apache.kafka.common.metrics.stats.Rate.SampledTotal;
 
 
 /**
  * A compound stat that includes a rate metric and a cumulative total metric.
  */
 public class Meter implements CompoundStat {
-
     private final MetricName rateMetricName;
     private final MetricName totalMetricName;
     private final Rate rate;
-    private final Total total;
+    private final CumulativeSum total;
 
     /**
-     * Construct a Meter with seconds as time unit and {@link SampledTotal} stats for Rate
+     * Construct a Meter with seconds as time unit
      */
     public Meter(MetricName rateMetricName, MetricName totalMetricName) {
-        this(TimeUnit.SECONDS, new SampledTotal(), rateMetricName, totalMetricName);
+        this(TimeUnit.SECONDS, new WindowedSum(), rateMetricName, totalMetricName);
     }
 
     /**
-     * Construct a Meter with provided time unit and {@link SampledTotal} stats for Rate
+     * Construct a Meter with provided time unit
      */
     public Meter(TimeUnit unit, MetricName rateMetricName, MetricName totalMetricName) {
-        this(unit, new SampledTotal(), rateMetricName, totalMetricName);
+        this(unit, new WindowedSum(), rateMetricName, totalMetricName);
     }
 
     /**
-     * Construct a Meter with seconds as time unit and provided {@link SampledStat} stats for Rate
+     * Construct a Meter with seconds as time unit
      */
     public Meter(SampledStat rateStat, MetricName rateMetricName, MetricName totalMetricName) {
         this(TimeUnit.SECONDS, rateStat, rateMetricName, totalMetricName);
     }
 
     /**
-     * Construct a Meter with provided time unit and provided {@link SampledStat} stats for Rate
+     * Construct a Meter with provided time unit
      */
     public Meter(TimeUnit unit, SampledStat rateStat, MetricName rateMetricName, MetricName totalMetricName) {
-        if (!(rateStat instanceof SampledTotal) && !(rateStat instanceof Count)) {
-            throw new IllegalArgumentException("Meter is supported only for SampledTotal and Count");
+        if (!(rateStat instanceof WindowedSum)) {
+            throw new IllegalArgumentException("Meter is supported only for WindowedCount or WindowedSum.");
         }
-        this.total = new Total();
+        this.total = new CumulativeSum();
         this.rate = new Rate(unit, rateStat);
         this.rateMetricName = rateMetricName;
         this.totalMetricName = totalMetricName;
@@ -81,7 +79,7 @@ public List stats() {
     public void record(MetricConfig config, double value, long timeMs) {
         rate.record(config, value, timeMs);
         // Total metrics with Count stat should record 1.0 (as recorded in the count)
-        double totalValue = (rate.stat instanceof Count) ? 1.0 : value;
+        double totalValue = (rate.stat instanceof WindowedCount) ? 1.0 : value;
         total.record(config, totalValue, timeMs);
     }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/metrics/stats/Rate.java b/clients/src/main/java/org/apache/kafka/common/metrics/stats/Rate.java
index a56734cd86dfb..604f860ae4f2d 100644
--- a/clients/src/main/java/org/apache/kafka/common/metrics/stats/Rate.java
+++ b/clients/src/main/java/org/apache/kafka/common/metrics/stats/Rate.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.common.metrics.stats;
 
-import java.util.List;
 import java.util.Locale;
 import java.util.concurrent.TimeUnit;
 
@@ -40,7 +39,7 @@ public Rate() {
     }
 
     public Rate(TimeUnit unit) {
-        this(unit, new SampledTotal());
+        this(unit, new WindowedSum());
     }
 
     public Rate(SampledStat stat) {
@@ -115,24 +114,10 @@ private double convert(long timeMs) {
         }
     }
 
-    public static class SampledTotal extends SampledStat {
-
-        public SampledTotal() {
-            super(0.0d);
-        }
-
-        @Override
-        protected void update(Sample sample, MetricConfig config, double value, long timeMs) {
-            sample.value += value;
-        }
-
-        @Override
-        public double combine(List samples, MetricConfig config, long now) {
-            double total = 0.0;
-            for (Sample sample : samples)
-                total += sample.value;
-            return total;
-        }
-
+    /**
+     * @deprecated since 2.4 Use {@link WindowedSum} instead.
+     */
+    @Deprecated
+    public static class SampledTotal extends WindowedSum {
     }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/metrics/stats/Sum.java b/clients/src/main/java/org/apache/kafka/common/metrics/stats/Sum.java
index b40e9cdb649a0..17188b835c434 100644
--- a/clients/src/main/java/org/apache/kafka/common/metrics/stats/Sum.java
+++ b/clients/src/main/java/org/apache/kafka/common/metrics/stats/Sum.java
@@ -16,30 +16,14 @@
  */
 package org.apache.kafka.common.metrics.stats;
 
-import java.util.List;
-
-import org.apache.kafka.common.metrics.MetricConfig;
-
 /**
  * A {@link SampledStat} that maintains the sum of what it has seen.
+ * This is a sampled version of {@link CumulativeSum}.
+ *
+ * See also {@link WindowedCount} if you want to increment the value by 1 on each recording.
+ *
+ * @deprecated since 2.4 . Use {@link WindowedSum} instead
  */
-public class Sum extends SampledStat {
-
-    public Sum() {
-        super(0);
-    }
-
-    @Override
-    protected void update(Sample sample, MetricConfig config, double value, long now) {
-        sample.value += value;
-    }
-
-    @Override
-    public double combine(List samples, MetricConfig config, long now) {
-        double total = 0.0;
-        for (Sample sample : samples)
-            total += sample.value;
-        return total;
-    }
-
+@Deprecated
+public class Sum extends WindowedSum {
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/metrics/stats/Total.java b/clients/src/main/java/org/apache/kafka/common/metrics/stats/Total.java
index b8a83f5004bd6..23f7d040f8f90 100644
--- a/clients/src/main/java/org/apache/kafka/common/metrics/stats/Total.java
+++ b/clients/src/main/java/org/apache/kafka/common/metrics/stats/Total.java
@@ -16,32 +16,14 @@
  */
 package org.apache.kafka.common.metrics.stats;
 
-import org.apache.kafka.common.metrics.MeasurableStat;
-import org.apache.kafka.common.metrics.MetricConfig;
-
 /**
- * An un-windowed cumulative total maintained over all time.
+ * An non-sampled cumulative total maintained over all time.
+ * This is a non-sampled version of {@link WindowedSum}.
+ *
+ * See also {@link CumulativeCount} if you just want to increment the value by 1 on each recording.
+ *
+ * @deprecated since 2.4 . Use {@link CumulativeSum} instead.
  */
-public class Total implements MeasurableStat {
-
-    private double total;
-
-    public Total() {
-        this.total = 0.0;
-    }
-
-    public Total(double value) {
-        this.total = value;
-    }
-
-    @Override
-    public void record(MetricConfig config, double value, long now) {
-        this.total += value;
-    }
-
-    @Override
-    public double measure(MetricConfig config, long now) {
-        return this.total;
-    }
-
-}
+@Deprecated
+public class Total extends CumulativeSum {
+}
\ No newline at end of file
diff --git a/clients/src/main/java/org/apache/kafka/common/metrics/stats/WindowedCount.java b/clients/src/main/java/org/apache/kafka/common/metrics/stats/WindowedCount.java
new file mode 100644
index 0000000000000..825f404ef02d1
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/metrics/stats/WindowedCount.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.metrics.stats;
+
+import org.apache.kafka.common.metrics.MetricConfig;
+
+/**
+ * A {@link SampledStat} that maintains a simple count of what it has seen.
+ * This is a special kind of {@link WindowedSum} that always records a value of {@code 1} instead of the provided value.
+ * In other words, it counts the number of
+ * {@link WindowedCount#record(MetricConfig, double, long)} invocations,
+ * instead of summing the recorded values.
+ *
+ * See also {@link CumulativeCount} for a non-sampled version of this metric.
+ */
+public class WindowedCount extends WindowedSum {
+    @Override
+    protected void update(Sample sample, MetricConfig config, double value, long now) {
+        super.update(sample, config, 1.0, now);
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/metrics/stats/WindowedSum.java b/clients/src/main/java/org/apache/kafka/common/metrics/stats/WindowedSum.java
new file mode 100644
index 0000000000000..14aa5620fcc44
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/metrics/stats/WindowedSum.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.metrics.stats;
+
+import org.apache.kafka.common.metrics.MetricConfig;
+
+import java.util.List;
+
+/**
+ * A {@link SampledStat} that maintains the sum of what it has seen.
+ * This is a sampled version of {@link CumulativeSum}.
+ *
+ * See also {@link WindowedCount} if you want to increment the value by 1 on each recording.
+ */
+public class WindowedSum extends SampledStat {
+
+    public WindowedSum() {
+        super(0);
+    }
+
+    @Override
+    protected void update(Sample sample, MetricConfig config, double value, long now) {
+        sample.value += value;
+    }
+
+    @Override
+    public double combine(List samples, MetricConfig config, long now) {
+        double total = 0.0;
+        for (Sample sample : samples)
+            total += sample.value;
+        return total;
+    }
+
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selector.java b/clients/src/main/java/org/apache/kafka/common/network/Selector.java
index 20e24b7be4a2c..2652906099d78 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/Selector.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/Selector.java
@@ -23,11 +23,11 @@
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
-import org.apache.kafka.common.metrics.stats.Count;
+import org.apache.kafka.common.metrics.stats.CumulativeSum;
 import org.apache.kafka.common.metrics.stats.Max;
 import org.apache.kafka.common.metrics.stats.Meter;
+import org.apache.kafka.common.metrics.stats.WindowedCount;
 import org.apache.kafka.common.metrics.stats.SampledStat;
-import org.apache.kafka.common.metrics.stats.Total;
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.common.utils.Utils;
@@ -1084,7 +1084,7 @@ public SelectorMetrics(Metrics metrics, String metricGrpPrefix, Map metricValueFunc) {
         s.add(metrics.metricName("test.min", "grp1"), new Min());
         s.add(new Meter(TimeUnit.SECONDS, metrics.metricName("test.rate", "grp1"),
                 metrics.metricName("test.total", "grp1")));
-        s.add(new Meter(TimeUnit.SECONDS, new Count(), metrics.metricName("test.occurences", "grp1"),
+        s.add(new Meter(TimeUnit.SECONDS, new WindowedCount(), metrics.metricName("test.occurences", "grp1"),
                 metrics.metricName("test.occurences.total", "grp1")));
-        s.add(metrics.metricName("test.count", "grp1"), new Count());
+        s.add(metrics.metricName("test.count", "grp1"), new WindowedCount());
         s.add(new Percentiles(100, -100, 100, BucketSizing.CONSTANT,
                              new Percentile(metrics.metricName("test.median", "grp1"), 50.0),
                              new Percentile(metrics.metricName("test.perc99_9", "grp1"), 99.9)));
 
         Sensor s2 = metrics.sensor("test.sensor2");
-        s2.add(metrics.metricName("s2.total", "grp1"), new Total());
+        s2.add(metrics.metricName("s2.total", "grp1"), new CumulativeSum());
         s2.record(5.0);
 
         int sum = 0;
@@ -162,15 +162,15 @@ private void verifyStats(Function metricValueFunc) {
     @Test
     public void testHierarchicalSensors() {
         Sensor parent1 = metrics.sensor("test.parent1");
-        parent1.add(metrics.metricName("test.parent1.count", "grp1"), new Count());
+        parent1.add(metrics.metricName("test.parent1.count", "grp1"), new WindowedCount());
         Sensor parent2 = metrics.sensor("test.parent2");
-        parent2.add(metrics.metricName("test.parent2.count", "grp1"), new Count());
+        parent2.add(metrics.metricName("test.parent2.count", "grp1"), new WindowedCount());
         Sensor child1 = metrics.sensor("test.child1", parent1, parent2);
-        child1.add(metrics.metricName("test.child1.count", "grp1"), new Count());
+        child1.add(metrics.metricName("test.child1.count", "grp1"), new WindowedCount());
         Sensor child2 = metrics.sensor("test.child2", parent1);
-        child2.add(metrics.metricName("test.child2.count", "grp1"), new Count());
+        child2.add(metrics.metricName("test.child2.count", "grp1"), new WindowedCount());
         Sensor grandchild = metrics.sensor("test.grandchild", child1);
-        grandchild.add(metrics.metricName("test.grandchild.count", "grp1"), new Count());
+        grandchild.add(metrics.metricName("test.grandchild.count", "grp1"), new WindowedCount());
 
         /* increment each sensor one time */
         parent1.record();
@@ -222,15 +222,15 @@ public void testRemoveChildSensor() {
     public void testRemoveSensor() {
         int size = metrics.metrics().size();
         Sensor parent1 = metrics.sensor("test.parent1");
-        parent1.add(metrics.metricName("test.parent1.count", "grp1"), new Count());
+        parent1.add(metrics.metricName("test.parent1.count", "grp1"), new WindowedCount());
         Sensor parent2 = metrics.sensor("test.parent2");
-        parent2.add(metrics.metricName("test.parent2.count", "grp1"), new Count());
+        parent2.add(metrics.metricName("test.parent2.count", "grp1"), new WindowedCount());
         Sensor child1 = metrics.sensor("test.child1", parent1, parent2);
-        child1.add(metrics.metricName("test.child1.count", "grp1"), new Count());
+        child1.add(metrics.metricName("test.child1.count", "grp1"), new WindowedCount());
         Sensor child2 = metrics.sensor("test.child2", parent2);
-        child2.add(metrics.metricName("test.child2.count", "grp1"), new Count());
+        child2.add(metrics.metricName("test.child2.count", "grp1"), new WindowedCount());
         Sensor grandChild1 = metrics.sensor("test.gchild2", child2);
-        grandChild1.add(metrics.metricName("test.gchild2.count", "grp1"), new Count());
+        grandChild1.add(metrics.metricName("test.gchild2.count", "grp1"), new WindowedCount());
 
         Sensor sensor = metrics.getSensor("test.parent1");
         assertNotNull(sensor);
@@ -268,10 +268,10 @@ public void testRemoveSensor() {
     @Test
     public void testRemoveInactiveMetrics() {
         Sensor s1 = metrics.sensor("test.s1", null, 1);
-        s1.add(metrics.metricName("test.s1.count", "grp1"), new Count());
+        s1.add(metrics.metricName("test.s1.count", "grp1"), new WindowedCount());
 
         Sensor s2 = metrics.sensor("test.s2", null, 3);
-        s2.add(metrics.metricName("test.s2.count", "grp1"), new Count());
+        s2.add(metrics.metricName("test.s2.count", "grp1"), new WindowedCount());
 
         Metrics.ExpireSensorTask purger = metrics.new ExpireSensorTask();
         purger.run();
@@ -309,7 +309,7 @@ public void testRemoveInactiveMetrics() {
 
         // After purging, it should be possible to recreate a metric
         s1 = metrics.sensor("test.s1", null, 1);
-        s1.add(metrics.metricName("test.s1.count", "grp1"), new Count());
+        s1.add(metrics.metricName("test.s1.count", "grp1"), new WindowedCount());
         assertNotNull("Sensor test.s1 must be present", metrics.getSensor("test.s1"));
         assertNotNull("MetricName test.s1.count must be present",
                 metrics.metrics().get(metrics.metricName("test.s1.count", "grp1")));
@@ -318,8 +318,8 @@ public void testRemoveInactiveMetrics() {
     @Test
     public void testRemoveMetric() {
         int size = metrics.metrics().size();
-        metrics.addMetric(metrics.metricName("test1", "grp1"), new Count());
-        metrics.addMetric(metrics.metricName("test2", "grp1"), new Count());
+        metrics.addMetric(metrics.metricName("test1", "grp1"), new WindowedCount());
+        metrics.addMetric(metrics.metricName("test2", "grp1"), new WindowedCount());
 
         assertNotNull(metrics.removeMetric(metrics.metricName("test1", "grp1")));
         assertNull(metrics.metrics().get(metrics.metricName("test1", "grp1")));
@@ -333,7 +333,7 @@ public void testRemoveMetric() {
 
     @Test
     public void testEventWindowing() {
-        Count count = new Count();
+        WindowedCount count = new WindowedCount();
         MetricConfig config = new MetricConfig().eventWindow(1).samples(2);
         count.record(config, 1.0, time.milliseconds());
         count.record(config, 1.0, time.milliseconds());
@@ -344,7 +344,7 @@ public void testEventWindowing() {
 
     @Test
     public void testTimeWindowing() {
-        Count count = new Count();
+        WindowedCount count = new WindowedCount();
         MetricConfig config = new MetricConfig().timeWindow(1, TimeUnit.MILLISECONDS).samples(2);
         count.record(config, 1.0, time.milliseconds());
         time.sleep(1);
@@ -397,8 +397,8 @@ public void testSampledStatReturnsNaNWhenNoValuesExist() {
      */
     @Test
     public void testSampledStatReturnsInitialValueWhenNoValuesExist() {
-        Count count = new Count();
-        Rate.SampledTotal sampledTotal = new Rate.SampledTotal();
+        WindowedCount count = new WindowedCount();
+        WindowedSum sampledTotal = new WindowedSum();
         long windowMs = 100;
         int samples = 2;
         MetricConfig config = new MetricConfig().timeWindow(windowMs, TimeUnit.MILLISECONDS).samples(samples);
@@ -415,14 +415,14 @@ public void testSampledStatReturnsInitialValueWhenNoValuesExist() {
     @Test(expected = IllegalArgumentException.class)
     public void testDuplicateMetricName() {
         metrics.sensor("test").add(metrics.metricName("test", "grp1"), new Avg());
-        metrics.sensor("test2").add(metrics.metricName("test", "grp1"), new Total());
+        metrics.sensor("test2").add(metrics.metricName("test", "grp1"), new CumulativeSum());
     }
 
     @Test
     public void testQuotas() {
         Sensor sensor = metrics.sensor("test");
-        sensor.add(metrics.metricName("test1.total", "grp1"), new Total(), new MetricConfig().quota(Quota.upperBound(5.0)));
-        sensor.add(metrics.metricName("test2.total", "grp1"), new Total(), new MetricConfig().quota(Quota.lowerBound(0.0)));
+        sensor.add(metrics.metricName("test1.total", "grp1"), new CumulativeSum(), new MetricConfig().quota(Quota.upperBound(5.0)));
+        sensor.add(metrics.metricName("test2.total", "grp1"), new CumulativeSum(), new MetricConfig().quota(Quota.lowerBound(0.0)));
         sensor.record(5.0);
         try {
             sensor.record(1.0);
@@ -503,7 +503,7 @@ public void testRateWindowing() throws Exception {
         MetricName countRateMetricName = metrics.metricName("test.count.rate", "grp1");
         MetricName countTotalMetricName = metrics.metricName("test.count.total", "grp1");
         s.add(new Meter(TimeUnit.SECONDS, rateMetricName, totalMetricName));
-        s.add(new Meter(TimeUnit.SECONDS, new Count(), countRateMetricName, countTotalMetricName));
+        s.add(new Meter(TimeUnit.SECONDS, new WindowedCount(), countRateMetricName, countTotalMetricName));
         KafkaMetric totalMetric = metrics.metrics().get(totalMetricName);
         KafkaMetric countTotalMetric = metrics.metrics().get(countTotalMetricName);
 
@@ -825,10 +825,10 @@ private Sensor createSensor(StatType statType, int index) {
                     sensor.add(metrics.metricName("test.metric.avg", "avg", tags), new Avg());
                     break;
                 case TOTAL:
-                    sensor.add(metrics.metricName("test.metric.total", "total", tags), new Total());
+                    sensor.add(metrics.metricName("test.metric.total", "total", tags), new CumulativeSum());
                     break;
                 case COUNT:
-                    sensor.add(metrics.metricName("test.metric.count", "count", tags), new Count());
+                    sensor.add(metrics.metricName("test.metric.count", "count", tags), new WindowedCount());
                     break;
                 case MAX:
                     sensor.add(metrics.metricName("test.metric.max", "max", tags), new Max());
@@ -843,7 +843,7 @@ private Sensor createSensor(StatType statType, int index) {
                     sensor.add(metrics.metricName("test.metric.simpleRate", "simpleRate", tags), new SimpleRate());
                     break;
                 case SUM:
-                    sensor.add(metrics.metricName("test.metric.sum", "sum", tags), new Sum());
+                    sensor.add(metrics.metricName("test.metric.sum", "sum", tags), new WindowedSum());
                     break;
                 case VALUE:
                     sensor.add(metrics.metricName("test.metric.value", "value", tags), new Value());
diff --git a/clients/src/test/java/org/apache/kafka/common/metrics/SensorTest.java b/clients/src/test/java/org/apache/kafka/common/metrics/SensorTest.java
index 8e3dfeb3b9fad..fc7cfe24def9a 100644
--- a/clients/src/test/java/org/apache/kafka/common/metrics/SensorTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/metrics/SensorTest.java
@@ -20,7 +20,7 @@
 import org.apache.kafka.common.metrics.stats.Avg;
 import org.apache.kafka.common.metrics.stats.Meter;
 import org.apache.kafka.common.metrics.stats.Rate;
-import org.apache.kafka.common.metrics.stats.Sum;
+import org.apache.kafka.common.metrics.stats.WindowedSum;
 import org.apache.kafka.common.utils.MockTime;
 import org.apache.kafka.common.utils.SystemTime;
 import org.apache.kafka.common.utils.Time;
@@ -128,7 +128,7 @@ public void testIdempotentAdd() {
         }
 
         // note that adding a different metric with the same name is also a no-op
-        assertTrue(sensor.add(metrics.metricName("test-metric", "test-group"), new Sum()));
+        assertTrue(sensor.add(metrics.metricName("test-metric", "test-group"), new WindowedSum()));
 
         // so after all this, we still just have the original metric registered
         assertEquals(1, sensor.metrics().size());
diff --git a/clients/src/test/java/org/apache/kafka/common/metrics/stats/MeterTest.java b/clients/src/test/java/org/apache/kafka/common/metrics/stats/MeterTest.java
index 8204771d794db..27198ea1e79d0 100644
--- a/clients/src/test/java/org/apache/kafka/common/metrics/stats/MeterTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/metrics/stats/MeterTest.java
@@ -44,7 +44,7 @@ public void testMeter() {
         assertEquals(rateMetricName, rate.name());
         assertEquals(totalMetricName, total.name());
         Rate rateStat = (Rate) rate.stat();
-        Total totalStat = (Total) total.stat();
+        CumulativeSum totalStat = (CumulativeSum) total.stat();
 
         MetricConfig config = new MetricConfig();
         double nextValue = 0.0;
diff --git a/clients/src/test/java/org/apache/kafka/test/MetricsBench.java b/clients/src/test/java/org/apache/kafka/test/MetricsBench.java
index 379db2f526d1d..93cbf6d2a8910 100644
--- a/clients/src/test/java/org/apache/kafka/test/MetricsBench.java
+++ b/clients/src/test/java/org/apache/kafka/test/MetricsBench.java
@@ -21,11 +21,11 @@
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
-import org.apache.kafka.common.metrics.stats.Count;
 import org.apache.kafka.common.metrics.stats.Max;
 import org.apache.kafka.common.metrics.stats.Percentile;
 import org.apache.kafka.common.metrics.stats.Percentiles;
 import org.apache.kafka.common.metrics.stats.Percentiles.BucketSizing;
+import org.apache.kafka.common.metrics.stats.WindowedCount;
 
 public class MetricsBench {
 
@@ -37,7 +37,7 @@ public static void main(String[] args) {
             Sensor child = metrics.sensor("child", parent);
             for (Sensor sensor : Arrays.asList(parent, child)) {
                 sensor.add(metrics.metricName(sensor.name() + ".avg", "grp1"), new Avg());
-                sensor.add(metrics.metricName(sensor.name() + ".count", "grp1"), new Count());
+                sensor.add(metrics.metricName(sensor.name() + ".count", "grp1"), new WindowedCount());
                 sensor.add(metrics.metricName(sensor.name() + ".max", "grp1"), new Max());
                 sensor.add(new Percentiles(1024,
                         0.0,
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
index f848a18f06f9e..fd90dd149ba51 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
@@ -24,8 +24,8 @@
 import org.apache.kafka.common.config.ConfigValue;
 import org.apache.kafka.common.config.provider.ConfigProvider;
 import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.metrics.stats.CumulativeSum;
 import org.apache.kafka.common.metrics.stats.Frequencies;
-import org.apache.kafka.common.metrics.stats.Total;
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.connect.connector.Connector;
@@ -863,13 +863,13 @@ public Double metricValue(long now) {
             connectorStartupResults.add(connectorStartupResultFrequencies);
 
             connectorStartupAttempts = metricGroup.sensor("connector-startup-attempts");
-            connectorStartupAttempts.add(metricGroup.metricName(registry.connectorStartupAttemptsTotal), new Total());
+            connectorStartupAttempts.add(metricGroup.metricName(registry.connectorStartupAttemptsTotal), new CumulativeSum());
 
             connectorStartupSuccesses = metricGroup.sensor("connector-startup-successes");
-            connectorStartupSuccesses.add(metricGroup.metricName(registry.connectorStartupSuccessTotal), new Total());
+            connectorStartupSuccesses.add(metricGroup.metricName(registry.connectorStartupSuccessTotal), new CumulativeSum());
 
             connectorStartupFailures = metricGroup.sensor("connector-startup-failures");
-            connectorStartupFailures.add(metricGroup.metricName(registry.connectorStartupFailureTotal), new Total());
+            connectorStartupFailures.add(metricGroup.metricName(registry.connectorStartupFailureTotal), new CumulativeSum());
 
             MetricName taskFailurePct = metricGroup.metricName(registry.taskStartupFailurePercentage);
             MetricName taskSuccessPct = metricGroup.metricName(registry.taskStartupSuccessPercentage);
@@ -878,13 +878,13 @@ public Double metricValue(long now) {
             taskStartupResults.add(taskStartupResultFrequencies);
 
             taskStartupAttempts = metricGroup.sensor("task-startup-attempts");
-            taskStartupAttempts.add(metricGroup.metricName(registry.taskStartupAttemptsTotal), new Total());
+            taskStartupAttempts.add(metricGroup.metricName(registry.taskStartupAttemptsTotal), new CumulativeSum());
 
             taskStartupSuccesses = metricGroup.sensor("task-startup-successes");
-            taskStartupSuccesses.add(metricGroup.metricName(registry.taskStartupSuccessTotal), new Total());
+            taskStartupSuccesses.add(metricGroup.metricName(registry.taskStartupSuccessTotal), new CumulativeSum());
 
             taskStartupFailures = metricGroup.sensor("task-startup-failures");
-            taskStartupFailures.add(metricGroup.metricName(registry.taskStartupFailureTotal), new Total());
+            taskStartupFailures.add(metricGroup.metricName(registry.taskStartupFailureTotal), new CumulativeSum());
         }
 
         void close() {
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
index f21c500699e07..395c93e0f772a 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
@@ -27,9 +27,9 @@
 import org.apache.kafka.common.errors.WakeupException;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
+import org.apache.kafka.common.metrics.stats.CumulativeSum;
 import org.apache.kafka.common.metrics.stats.Max;
 import org.apache.kafka.common.metrics.stats.Rate;
-import org.apache.kafka.common.metrics.stats.Total;
 import org.apache.kafka.common.metrics.stats.Value;
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.connect.data.SchemaAndValue;
@@ -705,11 +705,11 @@ public SinkTaskMetricsGroup(ConnectorTaskId id, ConnectMetrics connectMetrics) {
 
             sinkRecordRead = metricGroup.sensor("sink-record-read");
             sinkRecordRead.add(metricGroup.metricName(registry.sinkRecordReadRate), new Rate());
-            sinkRecordRead.add(metricGroup.metricName(registry.sinkRecordReadTotal), new Total());
+            sinkRecordRead.add(metricGroup.metricName(registry.sinkRecordReadTotal), new CumulativeSum());
 
             sinkRecordSend = metricGroup.sensor("sink-record-send");
             sinkRecordSend.add(metricGroup.metricName(registry.sinkRecordSendRate), new Rate());
-            sinkRecordSend.add(metricGroup.metricName(registry.sinkRecordSendTotal), new Total());
+            sinkRecordSend.add(metricGroup.metricName(registry.sinkRecordSendTotal), new CumulativeSum());
 
             sinkRecordActiveCount = metricGroup.sensor("sink-record-active-count");
             sinkRecordActiveCount.add(metricGroup.metricName(registry.sinkRecordActiveCount), new Value());
@@ -724,11 +724,11 @@ public SinkTaskMetricsGroup(ConnectorTaskId id, ConnectMetrics connectMetrics) {
 
             offsetCompletion = metricGroup.sensor("offset-commit-completion");
             offsetCompletion.add(metricGroup.metricName(registry.sinkRecordOffsetCommitCompletionRate), new Rate());
-            offsetCompletion.add(metricGroup.metricName(registry.sinkRecordOffsetCommitCompletionTotal), new Total());
+            offsetCompletion.add(metricGroup.metricName(registry.sinkRecordOffsetCommitCompletionTotal), new CumulativeSum());
 
             offsetCompletionSkip = metricGroup.sensor("offset-commit-completion-skip");
             offsetCompletionSkip.add(metricGroup.metricName(registry.sinkRecordOffsetCommitSkipRate), new Rate());
-            offsetCompletionSkip.add(metricGroup.metricName(registry.sinkRecordOffsetCommitSkipTotal), new Total());
+            offsetCompletionSkip.add(metricGroup.metricName(registry.sinkRecordOffsetCommitSkipTotal), new CumulativeSum());
 
             putBatchTime = metricGroup.sensor("put-batch-time");
             putBatchTime.add(metricGroup.metricName(registry.sinkRecordPutBatchTimeMax), new Max());
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
index 6e94c6f2e4b0d..6e1152be91168 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
@@ -24,9 +24,9 @@
 import org.apache.kafka.common.header.internals.RecordHeaders;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
+import org.apache.kafka.common.metrics.stats.CumulativeSum;
 import org.apache.kafka.common.metrics.stats.Max;
 import org.apache.kafka.common.metrics.stats.Rate;
-import org.apache.kafka.common.metrics.stats.Total;
 import org.apache.kafka.common.metrics.stats.Value;
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.connect.errors.ConnectException;
@@ -591,11 +591,11 @@ public SourceTaskMetricsGroup(ConnectorTaskId id, ConnectMetrics connectMetrics)
 
             sourceRecordPoll = metricGroup.sensor("source-record-poll");
             sourceRecordPoll.add(metricGroup.metricName(registry.sourceRecordPollRate), new Rate());
-            sourceRecordPoll.add(metricGroup.metricName(registry.sourceRecordPollTotal), new Total());
+            sourceRecordPoll.add(metricGroup.metricName(registry.sourceRecordPollTotal), new CumulativeSum());
 
             sourceRecordWrite = metricGroup.sensor("source-record-write");
             sourceRecordWrite.add(metricGroup.metricName(registry.sourceRecordWriteRate), new Rate());
-            sourceRecordWrite.add(metricGroup.metricName(registry.sourceRecordWriteTotal), new Total());
+            sourceRecordWrite.add(metricGroup.metricName(registry.sourceRecordWriteTotal), new CumulativeSum());
 
             pollTime = metricGroup.sensor("poll-batch-time");
             pollTime.add(metricGroup.metricName(registry.sourceRecordPollBatchTimeMax), new Max());
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
index b1a41c0b4ba64..86caeeb4af6f1 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
@@ -22,8 +22,8 @@
 import org.apache.kafka.common.errors.WakeupException;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
+import org.apache.kafka.common.metrics.stats.CumulativeSum;
 import org.apache.kafka.common.metrics.stats.Max;
-import org.apache.kafka.common.metrics.stats.Total;
 import org.apache.kafka.common.utils.Exit;
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.common.utils.Time;
@@ -1483,7 +1483,7 @@ public Double metricValue(long now) {
             });
 
             rebalanceCompletedCounts = metricGroup.sensor("completed-rebalance-count");
-            rebalanceCompletedCounts.add(metricGroup.metricName(registry.rebalanceCompletedTotal), new Total());
+            rebalanceCompletedCounts.add(metricGroup.metricName(registry.rebalanceCompletedTotal), new CumulativeSum());
 
             rebalanceTime = metricGroup.sensor("rebalance-time");
             rebalanceTime.add(metricGroup.metricName(registry.rebalanceTimeMax), new Max());
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/ErrorHandlingMetrics.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/ErrorHandlingMetrics.java
index c589012d02cc2..0deecd129a06e 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/ErrorHandlingMetrics.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/ErrorHandlingMetrics.java
@@ -17,7 +17,7 @@
 package org.apache.kafka.connect.runtime.errors;
 
 import org.apache.kafka.common.metrics.Sensor;
-import org.apache.kafka.common.metrics.stats.Total;
+import org.apache.kafka.common.metrics.stats.CumulativeSum;
 import org.apache.kafka.common.utils.SystemTime;
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.connect.runtime.ConnectMetrics;
@@ -62,25 +62,25 @@ public ErrorHandlingMetrics(ConnectorTaskId id, ConnectMetrics connectMetrics) {
         metricGroup.close();
 
         recordProcessingFailures = metricGroup.sensor("total-record-failures");
-        recordProcessingFailures.add(metricGroup.metricName(registry.recordProcessingFailures), new Total());
+        recordProcessingFailures.add(metricGroup.metricName(registry.recordProcessingFailures), new CumulativeSum());
 
         recordProcessingErrors = metricGroup.sensor("total-record-errors");
-        recordProcessingErrors.add(metricGroup.metricName(registry.recordProcessingErrors), new Total());
+        recordProcessingErrors.add(metricGroup.metricName(registry.recordProcessingErrors), new CumulativeSum());
 
         recordsSkipped = metricGroup.sensor("total-records-skipped");
-        recordsSkipped.add(metricGroup.metricName(registry.recordsSkipped), new Total());
+        recordsSkipped.add(metricGroup.metricName(registry.recordsSkipped), new CumulativeSum());
 
         retries = metricGroup.sensor("total-retries");
-        retries.add(metricGroup.metricName(registry.retries), new Total());
+        retries.add(metricGroup.metricName(registry.retries), new CumulativeSum());
 
         errorsLogged = metricGroup.sensor("total-errors-logged");
-        errorsLogged.add(metricGroup.metricName(registry.errorsLogged), new Total());
+        errorsLogged.add(metricGroup.metricName(registry.errorsLogged), new CumulativeSum());
 
         dlqProduceRequests = metricGroup.sensor("deadletterqueue-produce-requests");
-        dlqProduceRequests.add(metricGroup.metricName(registry.dlqProduceRequests), new Total());
+        dlqProduceRequests.add(metricGroup.metricName(registry.dlqProduceRequests), new CumulativeSum());
 
         dlqProduceFailures = metricGroup.sensor("deadletterqueue-produce-failures");
-        dlqProduceFailures.add(metricGroup.metricName(registry.dlqProduceFailures), new Total());
+        dlqProduceFailures.add(metricGroup.metricName(registry.dlqProduceFailures), new CumulativeSum());
 
         metricGroup.addValueMetric(registry.lastErrorTimestamp, now -> lastErrorTime);
     }
diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala
index 619d2605ac9b4..69f02ae5799be 100644
--- a/core/src/main/scala/kafka/network/SocketServer.scala
+++ b/core/src/main/scala/kafka/network/SocketServer.scala
@@ -39,8 +39,7 @@ import org.apache.kafka.common.config.ConfigException
 import org.apache.kafka.common.{KafkaException, Reconfigurable}
 import org.apache.kafka.common.memory.{MemoryPool, SimpleMemoryPool}
 import org.apache.kafka.common.metrics._
-import org.apache.kafka.common.metrics.stats.Meter
-import org.apache.kafka.common.metrics.stats.Total
+import org.apache.kafka.common.metrics.stats.{CumulativeSum, Meter}
 import org.apache.kafka.common.network.KafkaChannel.ChannelMuteEvent
 import org.apache.kafka.common.network.{ChannelBuilder, ChannelBuilders, KafkaChannel, ListenerName, ListenerReconfigurable, Selectable, Send, Selector => KSelector}
 import org.apache.kafka.common.protocol.ApiKeys
@@ -712,7 +711,7 @@ private[kafka] class Processor(val id: Int,
     Map(NetworkProcessorMetricTag -> id.toString)
   )
 
-  val expiredConnectionsKilledCount = new Total()
+  val expiredConnectionsKilledCount = new CumulativeSum()
   private val expiredConnectionsKilledCountMetricName = metrics.metricName("expired-connections-killed-count", "socket-server-metrics", metricTags)
   metrics.addMetric(expiredConnectionsKilledCountMetricName, expiredConnectionsKilledCount)
 
diff --git a/core/src/main/scala/kafka/server/ClientQuotaManager.scala b/core/src/main/scala/kafka/server/ClientQuotaManager.scala
index 959144cf49901..5451b5caa35df 100644
--- a/core/src/main/scala/kafka/server/ClientQuotaManager.scala
+++ b/core/src/main/scala/kafka/server/ClientQuotaManager.scala
@@ -27,7 +27,7 @@ import kafka.utils.{Logging, ShutdownableThread}
 import org.apache.kafka.common.{Cluster, MetricName}
 import org.apache.kafka.common.metrics._
 import org.apache.kafka.common.metrics.Metrics
-import org.apache.kafka.common.metrics.stats.{Avg, Rate, Total}
+import org.apache.kafka.common.metrics.stats.{Avg, CumulativeSum, Rate}
 import org.apache.kafka.common.security.auth.KafkaPrincipal
 import org.apache.kafka.common.utils.{Sanitizer, Time}
 import org.apache.kafka.server.quota.{ClientQuotaCallback, ClientQuotaEntity, ClientQuotaType}
@@ -179,7 +179,7 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig,
   private val delayQueueSensor = metrics.sensor(quotaType + "-delayQueue")
   delayQueueSensor.add(metrics.metricName("queue-size",
     quotaType.toString,
-    "Tracks the size of the delay queue"), new Total())
+    "Tracks the size of the delay queue"), new CumulativeSum())
   start() // Use start method to keep spotbugs happy
   private def start() {
     throttledChannelReaper.start()
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/metrics/Sensors.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/metrics/Sensors.java
index 4ecaeb4189755..038b8ac77e9a3 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/metrics/Sensors.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/metrics/Sensors.java
@@ -19,10 +19,10 @@
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
+import org.apache.kafka.common.metrics.stats.CumulativeSum;
 import org.apache.kafka.common.metrics.stats.Max;
 import org.apache.kafka.common.metrics.stats.Rate;
-import org.apache.kafka.common.metrics.stats.Sum;
-import org.apache.kafka.common.metrics.stats.Total;
+import org.apache.kafka.common.metrics.stats.WindowedSum;
 import org.apache.kafka.streams.processor.internals.InternalProcessorContext;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 
@@ -106,7 +106,7 @@ public static Sensor suppressionEmitSensor(final InternalProcessorContext contex
                 "The average number of occurrence of suppression-emit operation per second.",
                 tags
             ),
-            new Rate(TimeUnit.SECONDS, new Sum())
+            new Rate(TimeUnit.SECONDS, new WindowedSum())
         );
         sensor.add(
             new MetricName(
@@ -115,7 +115,7 @@ public static Sensor suppressionEmitSensor(final InternalProcessorContext contex
                 "The total number of occurrence of suppression-emit operations.",
                 tags
             ),
-            new Total()
+            new CumulativeSum()
         );
         return sensor;
     }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
index 210412be92fc6..836330f242617 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
@@ -28,9 +28,10 @@
 import org.apache.kafka.common.errors.TimeoutException;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
-import org.apache.kafka.common.metrics.stats.Count;
+import org.apache.kafka.common.metrics.stats.CumulativeCount;
 import org.apache.kafka.common.metrics.stats.Max;
 import org.apache.kafka.common.metrics.stats.Rate;
+import org.apache.kafka.common.metrics.stats.WindowedCount;
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.errors.DeserializationExceptionHandler;
@@ -43,7 +44,6 @@
 import org.apache.kafka.streams.processor.Punctuator;
 import org.apache.kafka.streams.processor.TaskId;
 import org.apache.kafka.streams.processor.TimestampExtractor;
-import org.apache.kafka.streams.processor.internals.metrics.CumulativeCount;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.apache.kafka.streams.state.internals.ThreadCache;
@@ -112,7 +112,7 @@ protected static final class TaskMetrics {
             );
             taskCommitTimeSensor.add(
                 new MetricName("commit-rate", group, "The average number of occurrence of commit operation per second.", tagMap),
-                new Rate(TimeUnit.SECONDS, new Count())
+                new Rate(TimeUnit.SECONDS, new WindowedCount())
             );
             taskCommitTimeSensor.add(
                 new MetricName("commit-total", group, "The total number of occurrence of commit operations.", tagMap),
@@ -123,7 +123,7 @@ protected static final class TaskMetrics {
             taskEnforcedProcessSensor = metrics.taskLevelSensor(taskName, "enforced-processing", Sensor.RecordingLevel.DEBUG, parent);
             taskEnforcedProcessSensor.add(
                     new MetricName("enforced-processing-rate", group, "The average number of occurrence of enforced-processing operation per second.", tagMap),
-                    new Rate(TimeUnit.SECONDS, new Count())
+                    new Rate(TimeUnit.SECONDS, new WindowedCount())
             );
             taskEnforcedProcessSensor.add(
                     new MetricName("enforced-processing-total", group, "The total number of occurrence of enforced-processing operations.", tagMap),
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
index 46b5669d6011c..b6bfcc5c8513a 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
@@ -22,9 +22,10 @@
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.Sensor.RecordingLevel;
 import org.apache.kafka.common.metrics.stats.Avg;
-import org.apache.kafka.common.metrics.stats.Count;
+import org.apache.kafka.common.metrics.stats.CumulativeCount;
 import org.apache.kafka.common.metrics.stats.Max;
 import org.apache.kafka.common.metrics.stats.Rate;
+import org.apache.kafka.common.metrics.stats.WindowedCount;
 import org.apache.kafka.streams.StreamsMetrics;
 
 import java.util.Arrays;
@@ -445,7 +446,7 @@ public static void addInvocationRateAndCount(final Sensor sensor,
                 descriptionOfRate,
                 tags
             ),
-            new Rate(TimeUnit.SECONDS, new Count())
+            new Rate(TimeUnit.SECONDS, new WindowedCount())
         );
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/RecordCollectorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RecordCollectorTest.java
index a7da2cbec76cb..47dd61b2a9057 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/RecordCollectorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RecordCollectorTest.java
@@ -33,7 +33,7 @@
 import org.apache.kafka.common.header.internals.RecordHeaders;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
-import org.apache.kafka.common.metrics.stats.Sum;
+import org.apache.kafka.common.metrics.stats.WindowedSum;
 import org.apache.kafka.common.serialization.ByteArraySerializer;
 import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.common.utils.LogContext;
@@ -215,7 +215,7 @@ public void shouldRecordSkippedMetricAndLogWarningIfSendFailsWithContinueExcepti
         final Sensor sensor = metrics.sensor("skipped-records");
         final LogCaptureAppender logCaptureAppender = LogCaptureAppender.createAndRegister();
         final MetricName metricName = new MetricName("name", "group", "description", Collections.emptyMap());
-        sensor.add(metricName, new Sum());
+        sensor.add(metricName, new WindowedSum());
         final RecordCollector collector = new RecordCollectorImpl(
             "test",
             logContext,
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
index 0400128282e3d..2faa078d6751e 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
@@ -27,7 +27,7 @@
 import org.apache.kafka.common.metrics.KafkaMetric;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
-import org.apache.kafka.common.metrics.stats.Total;
+import org.apache.kafka.common.metrics.stats.CumulativeSum;
 import org.apache.kafka.common.record.TimestampType;
 import org.apache.kafka.common.serialization.IntegerDeserializer;
 import org.apache.kafka.common.serialization.IntegerSerializer;
@@ -692,7 +692,7 @@ void closeStateManager(final boolean clean) throws ProcessorStateException {
     private MetricName setupCloseTaskMetric() {
         final MetricName metricName = new MetricName("name", "group", "description", Collections.emptyMap());
         final Sensor sensor = streamsMetrics.threadLevelSensor("task-closed", Sensor.RecordingLevel.INFO);
-        sensor.add(metricName, new Total());
+        sensor.add(metricName, new CumulativeSum());
         return metricName;
     }
 
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
index 48c038c2d5f65..2b1428f9f8968 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
@@ -32,9 +32,9 @@
 import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
-import org.apache.kafka.common.metrics.stats.Count;
+import org.apache.kafka.common.metrics.stats.CumulativeSum;
 import org.apache.kafka.common.metrics.stats.Rate;
-import org.apache.kafka.common.metrics.stats.Total;
+import org.apache.kafka.common.metrics.stats.WindowedCount;
 import org.apache.kafka.common.record.TimestampType;
 import org.apache.kafka.common.serialization.ByteArraySerializer;
 import org.apache.kafka.common.serialization.Deserializer;
@@ -289,12 +289,12 @@ public List partitionsFor(final String topic) {
                                                 threadLevelGroup,
                                                 "The average per-second number of skipped records",
                                                 streamsMetrics.tagMap()),
-                                 new Rate(TimeUnit.SECONDS, new Count()));
+                                 new Rate(TimeUnit.SECONDS, new WindowedCount()));
         skippedRecordsSensor.add(new MetricName("skipped-records-total",
                                                 threadLevelGroup,
                                                 "The total number of skipped records",
                                                 streamsMetrics.tagMap()),
-                                 new Total());
+                                 new CumulativeSum());
         final ThreadCache cache = new ThreadCache(
             new LogContext("topology-test-driver "),
             Math.max(0, streamsConfig.getLong(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG)),

From e9c61f690e548fb8c587d777e5c8f71149a38fea Mon Sep 17 00:00:00 2001
From: Dhruvil Shah 
Date: Wed, 24 Jul 2019 22:17:05 -0700
Subject: [PATCH 0477/1071] MINOR: Ensure in-memory metadata is removed before
 physical deletion of segment (#7106)

Minor refactoring of the places where we delete log segments, to ensure we always remove the in-memory metadata of the segment before performing physical deletion.

Reviewers: Ismael Juma , NIkhil Bhatia , Jun Rao 
---
 core/src/main/scala/kafka/log/Log.scala       | 72 +++++++++++--------
 .../test/scala/unit/kafka/log/LogTest.scala   | 21 ++++++
 2 files changed, 64 insertions(+), 29 deletions(-)

diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index b13823932d30b..5e12e315bbfab 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -746,8 +746,10 @@ class Log(@volatile var dir: File,
     // if we have the clean shutdown marker, skip recovery
     if (!hasCleanShutdownFile) {
       // okay we need to actually recover this log
-      val unflushed = logSegments(this.recoveryPoint, Long.MaxValue).iterator
-      while (unflushed.hasNext) {
+      val unflushed = logSegments(this.recoveryPoint, Long.MaxValue).toIterator
+      var truncated = false
+
+      while (unflushed.hasNext && !truncated) {
         val segment = unflushed.next
         info(s"Recovering unflushed segment ${segment.baseOffset}")
         val truncatedBytes =
@@ -763,7 +765,8 @@ class Log(@volatile var dir: File,
         if (truncatedBytes > 0) {
           // we had an invalid message, delete all remaining log
           warn(s"Corruption found in segment ${segment.baseOffset}, truncating to offset ${segment.readNextOffset}")
-          unflushed.foreach(deleteSegment)
+          removeAndDeleteSegments(unflushed.toList, asyncDelete = true)
+          truncated = true
         }
       }
     }
@@ -773,7 +776,7 @@ class Log(@volatile var dir: File,
       if (logEndOffset < logStartOffset) {
         warn(s"Deleting all segments because logEndOffset ($logEndOffset) is smaller than logStartOffset ($logStartOffset). " +
           "This could happen if segment files were deleted from the file system.")
-        logSegments.foreach(deleteSegment)
+        removeAndDeleteSegments(logSegments, asyncDelete = true)
       }
     }
 
@@ -1653,7 +1656,7 @@ class Log(@volatile var dir: File,
         lock synchronized {
           checkIfMemoryMappedBufferClosed()
           // remove the segments for lookups
-          deletable.foreach(deleteSegment)
+          removeAndDeleteSegments(deletable, asyncDelete = true)
           maybeIncrementLogStartOffset(segments.firstEntry.getValue.baseOffset)
         }
       }
@@ -1829,7 +1832,7 @@ class Log(@volatile var dir: File,
                  s"=max(provided offset = $expectedNextOffset, LEO = $logEndOffset) while it already " +
                  s"exists and is active with size 0. Size of time index: ${activeSegment.timeIndex.entries}," +
                  s" size of offset index: ${activeSegment.offsetIndex.entries}.")
-            deleteSegment(activeSegment)
+            removeAndDeleteSegments(Seq(activeSegment), asyncDelete = true)
           } else {
             throw new KafkaException(s"Trying to roll a new log segment for topic partition $topicPartition with start offset $newOffset" +
                                      s" =max(provided offset = $expectedNextOffset, LEO = $logEndOffset) while it already exists. Existing " +
@@ -1959,8 +1962,7 @@ class Log(@volatile var dir: File,
       lock synchronized {
         checkIfMemoryMappedBufferClosed()
         removeLogMetrics()
-        logSegments.foreach(_.deleteIfExists())
-        segments.clear()
+        removeAndDeleteSegments(logSegments, asyncDelete = false)
         leaderEpochCache.foreach(_.clear())
         Utils.delete(dir)
         // File handlers will be closed if this log is deleted
@@ -2011,7 +2013,7 @@ class Log(@volatile var dir: File,
             truncateFullyAndStartAt(targetOffset)
           } else {
             val deletable = logSegments.filter(segment => segment.baseOffset > targetOffset)
-            deletable.foreach(deleteSegment)
+            removeAndDeleteSegments(deletable, asyncDelete = true)
             activeSegment.truncateTo(targetOffset)
             updateLogEndOffset(targetOffset)
             this.recoveryPoint = math.min(targetOffset, this.recoveryPoint)
@@ -2035,8 +2037,7 @@ class Log(@volatile var dir: File,
       debug(s"Truncate and start at offset $newOffset")
       lock synchronized {
         checkIfMemoryMappedBufferClosed()
-        val segmentsToDelete = logSegments.toList
-        segmentsToDelete.foreach(deleteSegment)
+        removeAndDeleteSegments(logSegments, asyncDelete = true)
         addSegment(LogSegment.open(dir,
           baseOffset = newOffset,
           config = config,
@@ -2099,30 +2100,36 @@ class Log(@volatile var dir: File,
   }
 
   /**
-   * This method performs an asynchronous log segment delete by doing the following:
+   * This method deletes the given log segments by doing the following for each of them:
    * 
      
    *   
    1. It removes the segment from the segment map so that it will no longer be used for reads.
    *   
    2. It renames the index and log files by appending .deleted to the respective file name
-   *   
    3. It schedules an asynchronous delete operation to occur in the future
+   *   
    4. It can either schedule an asynchronous delete operation to occur in the future or perform the deletion synchronously
    * 
    
-   * This allows reads to happen concurrently without synchronization and without the possibility of physically
-   * deleting a file while it is being read from.
+   * Asynchronous deletion allows reads to happen concurrently without synchronization and without the possibility of
+   * physically deleting a file while it is being read.
    *
    * This method does not need to convert IOException to KafkaStorageException because it is either called before all logs are loaded
    * or the immediate caller will catch and handle IOException
    *
-   * @param segment The log segment to schedule for deletion
+   * @param segments The log segments to schedule for deletion
+   * @param asyncDelete Whether the segment files should be deleted asynchronously
    */
-  private def deleteSegment(segment: LogSegment) {
-    info(s"Scheduling log segment [baseOffset ${segment.baseOffset}, size ${segment.size}] for deletion.")
+  private def removeAndDeleteSegments(segments: Iterable[LogSegment], asyncDelete: Boolean): Unit = {
     lock synchronized {
-      segments.remove(segment.baseOffset)
-      asyncDeleteSegment(segment)
+      // As most callers hold an iterator into the `segments` collection and `removeAndDeleteSegment` mutates it by
+      // removing the deleted segment, we should force materialization of the iterator here, so that results of the
+      // iteration remain valid and deterministic.
+      val toDelete = segments.toList
+      toDelete.foreach { segment =>
+        this.segments.remove(segment.baseOffset)
+      }
+      deleteSegmentFiles(toDelete, asyncDelete)
     }
   }
 
   /**
-   * Perform an asynchronous delete on the given file.
+   * Perform physical deletion for the given file. Allows the file to be deleted asynchronously or synchronously.
    *
    * This method assumes that the file exists and the method is not thread-safe.
    *
@@ -2131,15 +2138,22 @@ class Log(@volatile var dir: File,
    *
    * @throws IOException if the file can't be renamed and still exists
    */
-  private def asyncDeleteSegment(segment: LogSegment) {
-    segment.changeFileSuffixes("", Log.DeletedFileSuffix)
-    def deleteSeg() {
-      info(s"Deleting segment ${segment.baseOffset}")
+  private def deleteSegmentFiles(segments: Iterable[LogSegment], asyncDelete: Boolean) {
+    segments.foreach(_.changeFileSuffixes("", Log.DeletedFileSuffix))
+
+    def deleteSegments() {
+      info(s"Deleting segments $segments")
       maybeHandleIOException(s"Error while deleting segments for $topicPartition in dir ${dir.getParent}") {
-        segment.deleteIfExists()
+        segments.foreach(_.deleteIfExists())
       }
     }
-    scheduler.schedule("delete-file", deleteSeg _, delay = config.fileDeleteDelayMs)
+
+    if (asyncDelete) {
+      info(s"Scheduling segments for deletion $segments")
+      scheduler.schedule("delete-file", () => deleteSegments, delay = config.fileDeleteDelayMs)
+    } else {
+      deleteSegments()
+    }
   }
 
   /**
@@ -2194,8 +2208,8 @@ class Log(@volatile var dir: File,
         // remove the index entry
         if (seg.baseOffset != sortedNewSegments.head.baseOffset)
           segments.remove(seg.baseOffset)
-        // delete segment
-        asyncDeleteSegment(seg)
+        // delete segment files
+        deleteSegmentFiles(List(seg), asyncDelete = true)
       }
       // okay we are safe now, remove the swap suffix
       sortedNewSegments.foreach(_.changeFileSuffixes(Log.SwapFileSuffix, ""))
diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala
index 6bb9301a898d1..46494e788d138 100755
--- a/core/src/test/scala/unit/kafka/log/LogTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogTest.scala
@@ -378,6 +378,27 @@ class LogTest {
     assertEquals(startOffset, log.logEndOffset)
   }
 
+  @Test
+  def testLogDelete(): Unit = {
+    val logConfig = LogTest.createLogConfig()
+    val log = createLog(logDir, logConfig)
+
+    for (i <- 0 to 100) {
+      val record = new SimpleRecord(mockTime.milliseconds, i.toString.getBytes)
+      log.appendAsLeader(TestUtils.records(List(record)), leaderEpoch = 0)
+      log.roll()
+    }
+
+    assertTrue(log.logSegments.size > 0)
+    assertFalse(logDir.listFiles.isEmpty)
+
+    // delete the log
+    log.delete()
+
+    assertEquals(0, log.logSegments.size)
+    assertFalse(logDir.exists)
+  }
+
   private def testProducerSnapshotsRecoveryAfterUncleanShutdown(messageFormatVersion: String): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 64 * 10, messageFormatVersion = messageFormatVersion)
     var log = createLog(logDir, logConfig)

From 69d86a197f86ad4c6f1636b5ab4678907e30a4c0 Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Thu, 25 Jul 2019 13:02:09 -0700
Subject: [PATCH 0478/1071] KAFKA-8179: add public ConsumerPartitionAssignor
 interface (#7108)

Main changes of this PR:

* Deprecate old consumer.internal.PartitionAssignor and add public consumer.ConsumerPartitionAssignor with all OOTB assignors migrated to new interface
* Refactor assignor's assignment/subscription related classes for easier to evolve API
* Removed version number from classes as it is only needed for serialization/deserialization
* Other previously-discussed cleanup included in this PR:

* Remove Assignment.error added in pt 1
* Remove ConsumerCoordinator#adjustAssignment added in pt 2

Reviewers: Boyang Chen , Jason Gustafson , Guozhang Wang 
---
 .../kafka/clients/admin/KafkaAdminClient.java |   4 +-
 .../clients/consumer/ConsumerConfig.java      |   2 +-
 .../consumer/ConsumerGroupMetadata.java       |  50 +++++
 .../consumer/ConsumerPartitionAssignor.java   | 206 ++++++++++++++++++
 .../kafka/clients/consumer/KafkaConsumer.java |   7 +-
 .../clients/consumer/StickyAssignor.java      |  11 +-
 .../internals/AbstractPartitionAssignor.java  |  18 +-
 .../internals/ConsumerCoordinator.java        |  86 ++------
 .../consumer/internals/ConsumerProtocol.java  |  95 +++-----
 .../consumer/internals/PartitionAssignor.java | 142 +-----------
 .../consumer/internals/SubscriptionState.java |   8 +
 .../clients/admin/KafkaAdminClientTest.java   |   6 +-
 .../clients/consumer/KafkaConsumerTest.java   |  69 +++---
 .../clients/consumer/RangeAssignorTest.java   |   2 +-
 .../consumer/RoundRobinAssignorTest.java      |   2 +-
 .../clients/consumer/StickyAssignorTest.java  |   2 +-
 .../internals/ConsumerCoordinatorTest.java    |  31 +--
 .../internals/ConsumerProtocolTest.java       |  45 +---
 .../group/GroupMetadataManagerTest.scala      |   2 +-
 .../internals/StreamsPartitionAssignor.java   |  20 +-
 .../StreamsPartitionAssignorTest.java         | 126 ++++++-----
 .../streams/tests/StreamsUpgradeTest.java     |  21 +-
 22 files changed, 495 insertions(+), 460 deletions(-)
 create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerGroupMetadata.java
 create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java

diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
index 349fc2a4d3d4d..f2ee21e9da617 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
@@ -29,9 +29,9 @@
 import org.apache.kafka.clients.admin.DeleteAclsResult.FilterResults;
 import org.apache.kafka.clients.admin.DescribeReplicaLogDirsResult.ReplicaLogDirInfo;
 import org.apache.kafka.clients.admin.internals.AdminMetadataManager;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor;
 import org.apache.kafka.clients.consumer.OffsetAndMetadata;
 import org.apache.kafka.clients.consumer.internals.ConsumerProtocol;
-import org.apache.kafka.clients.consumer.internals.PartitionAssignor;
 import org.apache.kafka.common.Cluster;
 import org.apache.kafka.common.ConsumerGroupState;
 import org.apache.kafka.common.ElectionType;
@@ -2724,7 +2724,7 @@ void handleResponse(AbstractResponse abstractResponse) {
                     for (DescribedGroupMember groupMember : members) {
                         Set partitions = Collections.emptySet();
                         if (groupMember.memberAssignment().length > 0) {
-                            final PartitionAssignor.Assignment assignment = ConsumerProtocol.
+                            final ConsumerPartitionAssignor.Assignment assignment = ConsumerProtocol.
                                 deserializeAssignment(ByteBuffer.wrap(groupMember.memberAssignment()));
                             partitions = new HashSet<>(assignment.partitions());
                         }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
index 7eb34d4aeee78..8a18bd5a77af3 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
@@ -102,7 +102,7 @@ public class ConsumerConfig extends AbstractConfig {
      * partition.assignment.strategy
      */
     public static final String PARTITION_ASSIGNMENT_STRATEGY_CONFIG = "partition.assignment.strategy";
-    private static final String PARTITION_ASSIGNMENT_STRATEGY_DOC = "The class name of the partition assignment strategy that the client will use to distribute partition ownership amongst consumer instances when group management is used";
+    private static final String PARTITION_ASSIGNMENT_STRATEGY_DOC = "The class name or class type of the assignor implementing the partition assignment strategy that the client will use to distribute partition ownership amongst consumer instances when group management is used. A custom assignor that implements ConsumerPartitionAssignor can be plugged in";
 
     /**
      * auto.offset.reset
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerGroupMetadata.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerGroupMetadata.java
new file mode 100644
index 0000000000000..e17894b213d0e
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerGroupMetadata.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.consumer;
+
+import java.util.Optional;
+
+public class ConsumerGroupMetadata {
+    private String groupId;
+    private int generationId;
+    private String memberId;
+    Optional groupInstanceId;
+
+    public ConsumerGroupMetadata(String groupId, int generationId, String memberId, Optional groupInstanceId) {
+        this.groupId = groupId;
+        this.generationId = generationId;
+        this.memberId = memberId;
+        this.groupInstanceId = groupInstanceId;
+    }
+
+    public String groupId() {
+        return groupId;
+    }
+
+    public int generationId() {
+        return generationId;
+    }
+
+    public String memberId() {
+        return memberId;
+    }
+
+    public Optional groupInstanceId() {
+        return groupInstanceId;
+    }
+
+}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java
new file mode 100644
index 0000000000000..72d5d6e806bd7
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.consumer;
+
+import java.nio.ByteBuffer;
+import java.util.Optional;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.kafka.common.Cluster;
+import org.apache.kafka.common.TopicPartition;
+
+/**
+ * This interface is used to define custom partition assignment for use in
+ * {@link org.apache.kafka.clients.consumer.KafkaConsumer}. Members of the consumer group subscribe
+ * to the topics they are interested in and forward their subscriptions to a Kafka broker serving
+ * as the group coordinator. The coordinator selects one member to perform the group assignment and
+ * propagates the subscriptions of all members to it. Then {@link #assign(Cluster, GroupSubscription)} is called
+ * to perform the assignment and the results are forwarded back to each respective members
+ *
+ * In some cases, it is useful to forward additional metadata to the assignor in order to make
+ * assignment decisions. For this, you can override {@link #subscriptionUserData(Set)} and provide custom
+ * userData in the returned Subscription. For example, to have a rack-aware assignor, an implementation
+ * can use this user data to forward the rackId belonging to each member.
+ */
+public interface ConsumerPartitionAssignor {
+
+    /**
+     * Return serialized data that will be included in the {@link Subscription} sent to the leader
+     * and can be leveraged in {@link #assign(Cluster, GroupSubscription)} ((e.g. local host/rack information)
+     *
+     * @return optional join subscription user data
+     */
+    default ByteBuffer subscriptionUserData(Set topics) {
+        return null;
+    }
+
+    /**
+     * Perform the group assignment given the member subscriptions and current cluster metadata.
+     * @param metadata Current topic/broker metadata known by consumer
+     * @param subscriptions Subscriptions from all members including metadata provided through {@link #subscriptionUserData(Set)}
+     * @return A map from the members to their respective assignment. This should have one entry
+     *         for each member in the input subscription map.
+     */
+    GroupAssignment assign(Cluster metadata, GroupSubscription subscriptions);
+
+    /**
+     * Callback which is invoked when a group member receives its assignment from the leader.
+     * @param assignment The local member's assignment as provided by the leader in {@link #assign(Cluster, GroupSubscription)}
+     * @param metadata Additional metadata on the consumer (optional)
+     */
+    default void onAssignment(Assignment assignment, ConsumerGroupMetadata metadata) {
+    }
+
+    /**
+     * Indicate which rebalance protocol this assignor works with;
+     * By default it should always work with {@link RebalanceProtocol#EAGER}.
+     */
+    default List supportedProtocols() {
+        return Collections.singletonList(RebalanceProtocol.EAGER);
+    }
+
+    /**
+     * Return the version of the assignor which indicates how the user metadata encodings
+     * and the assignment algorithm gets evolved.
+     */
+    default short version() {
+        return (short) 0;
+    }
+
+    /**
+     * Unique name for this assignor (e.g. "range" or "roundrobin" or "sticky"). Note, this is not required
+     * to be the same as the class name specified in {@link ConsumerConfig#PARTITION_ASSIGNMENT_STRATEGY_CONFIG}
+     * @return non-null unique name
+     */
+    String name();
+
+    final class Subscription {
+        private final List topics;
+        private final ByteBuffer userData;
+        private final List ownedPartitions;
+        private Optional groupInstanceId;
+
+        public Subscription(List topics, ByteBuffer userData, List ownedPartitions) {
+            this.topics = topics;
+            this.userData = userData;
+            this.ownedPartitions = ownedPartitions;
+            this.groupInstanceId = Optional.empty();
+        }
+
+        public Subscription(List topics, ByteBuffer userData) {
+            this(topics, userData, Collections.emptyList());
+        }
+
+        public Subscription(List topics) {
+            this(topics, null, Collections.emptyList());
+        }
+
+        public List topics() {
+            return topics;
+        }
+
+        public ByteBuffer userData() {
+            return userData;
+        }
+
+        public List ownedPartitions() {
+            return ownedPartitions;
+        }
+
+        public void setGroupInstanceId(Optional groupInstanceId) {
+            this.groupInstanceId = groupInstanceId;
+        }
+
+        public Optional groupInstanceId() {
+            return groupInstanceId;
+        }
+    }
+
+    final class Assignment {
+        private List partitions;
+        private ByteBuffer userData;
+
+        public Assignment(List partitions, ByteBuffer userData) {
+            this.partitions = partitions;
+            this.userData = userData;
+        }
+
+        public Assignment(List partitions) {
+            this(partitions, null);
+        }
+
+        public List partitions() {
+            return partitions;
+        }
+
+        public ByteBuffer userData() {
+            return userData;
+        }
+    }
+
+    final class GroupSubscription {
+        private final Map subscriptions;
+
+        public GroupSubscription(Map subscriptions) {
+            this.subscriptions = subscriptions;
+        }
+
+        public Map groupSubscription() {
+            return subscriptions;
+        }
+    }
+
+    final class GroupAssignment {
+        private final Map assignments;
+
+        public GroupAssignment(Map assignments) {
+            this.assignments = assignments;
+        }
+
+        public Map groupAssignment() {
+            return assignments;
+        }
+    }
+
+    enum RebalanceProtocol {
+        EAGER((byte) 0), COOPERATIVE((byte) 1);
+
+        private final byte id;
+
+        RebalanceProtocol(byte id) {
+            this.id = id;
+        }
+
+        public byte id() {
+            return id;
+        }
+
+        public static RebalanceProtocol forId(byte id) {
+            switch (id) {
+                case 0:
+                    return EAGER;
+                case 1:
+                    return COOPERATIVE;
+                default:
+                    throw new IllegalArgumentException("Unknown rebalance protocol id: " + id);
+            }
+        }
+    }
+
+}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index fa5cc99a18879..30944b330da60 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -29,7 +29,6 @@
 import org.apache.kafka.clients.consumer.internals.Fetcher;
 import org.apache.kafka.clients.consumer.internals.FetcherMetricsRegistry;
 import org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener;
-import org.apache.kafka.clients.consumer.internals.PartitionAssignor;
 import org.apache.kafka.clients.consumer.internals.SubscriptionState;
 import org.apache.kafka.common.Cluster;
 import org.apache.kafka.common.KafkaException;
@@ -581,7 +580,7 @@ public class KafkaConsumer implements Consumer {
     private final long requestTimeoutMs;
     private final int defaultApiTimeoutMs;
     private volatile boolean closed = false;
-    private List assignors;
+    private List assignors;
 
     // currentThread holds the threadId of the current thread accessing KafkaConsumer
     // and is used to prevent multi-threaded access
@@ -768,7 +767,7 @@ else if (enableAutoCommit)
                     heartbeatIntervalMs); //Will avoid blocking an extended period of time to prevent heartbeat thread starvation
             this.assignors = config.getConfiguredInstances(
                     ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG,
-                    PartitionAssignor.class);
+                    ConsumerPartitionAssignor.class);
 
             // no coordinator will be constructed for the default (null) group id
             this.coordinator = groupId == null ? null :
@@ -833,7 +832,7 @@ else if (enableAutoCommit)
                   long retryBackoffMs,
                   long requestTimeoutMs,
                   int defaultApiTimeoutMs,
-                  List assignors,
+                  List assignors,
                   String groupId) {
         this.log = logContext.logger(getClass());
         this.clientId = clientId;
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/StickyAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/StickyAssignor.java
index 3c7d010f5f7ae..3311cd8909403 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/StickyAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/StickyAssignor.java
@@ -365,18 +365,17 @@ private void prepopulateCurrentAssignments(Map subscriptio
     }
 
     @Override
-    public void onAssignment(Assignment assignment, int generation) {
+    public void onAssignment(Assignment assignment, ConsumerGroupMetadata metadata) {
         memberAssignment = assignment.partitions();
-        this.generation = generation;
+        this.generation = metadata.generationId();
     }
 
     @Override
-    public Subscription subscription(Set topics) {
+    public ByteBuffer subscriptionUserData(Set topics) {
         if (memberAssignment == null)
-            return new Subscription(new ArrayList<>(topics));
+            return null;
 
-        return new Subscription(new ArrayList<>(topics),
-                serializeTopicPartitionAssignment(new ConsumerUserData(memberAssignment, Optional.of(generation))));
+        return serializeTopicPartitionAssignment(new ConsumerUserData(memberAssignment, Optional.of(generation)));
     }
 
     @Override
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractPartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractPartitionAssignor.java
index 2487daa4eaa55..3b966b0736bbe 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractPartitionAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractPartitionAssignor.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.clients.consumer.internals;
 
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor;
 import org.apache.kafka.common.Cluster;
 import org.apache.kafka.common.TopicPartition;
 import org.slf4j.Logger;
@@ -33,7 +34,7 @@
  * Abstract assignor implementation which does some common grunt work (in particular collecting
  * partition counts which are always needed in assignors).
  */
-public abstract class AbstractPartitionAssignor implements PartitionAssignor {
+public abstract class AbstractPartitionAssignor implements ConsumerPartitionAssignor {
     private static final Logger log = LoggerFactory.getLogger(AbstractPartitionAssignor.class);
 
     /**
@@ -47,12 +48,8 @@ public abstract Map> assign(Map pa
                                                              Map subscriptions);
 
     @Override
-    public Subscription subscription(Set topics) {
-        return new Subscription(new ArrayList<>(topics));
-    }
-
-    @Override
-    public Map assign(Cluster metadata, Map subscriptions) {
+    public GroupAssignment assign(Cluster metadata, GroupSubscription groupSubscriptions) {
+        Map subscriptions = groupSubscriptions.groupSubscription();
         Set allSubscribedTopics = new HashSet<>();
         for (Map.Entry subscriptionEntry : subscriptions.entrySet())
             allSubscribedTopics.addAll(subscriptionEntry.getValue().topics());
@@ -72,12 +69,7 @@ public Map assign(Cluster metadata, Map assignments = new HashMap<>();
         for (Map.Entry> assignmentEntry : rawAssignments.entrySet())
             assignments.put(assignmentEntry.getKey(), new Assignment(assignmentEntry.getValue()));
-        return assignments;
-    }
-
-    @Override
-    public void onAssignment(Assignment assignment) {
-        // this assignor maintains no internal state, so nothing to do
+        return new GroupAssignment(assignments);
     }
 
     protected static  void put(Map> map, K key, V value) {
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 4866986d37e85..a28119dd4b0f4 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -18,13 +18,16 @@
 
 import org.apache.kafka.clients.GroupRebalanceConfig;
 import org.apache.kafka.clients.consumer.CommitFailedException;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor;
+import org.apache.kafka.clients.consumer.ConsumerGroupMetadata;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.GroupSubscription;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription;
 import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
 import org.apache.kafka.clients.consumer.OffsetAndMetadata;
 import org.apache.kafka.clients.consumer.OffsetCommitCallback;
 import org.apache.kafka.clients.consumer.RetriableCommitFailedException;
-import org.apache.kafka.clients.consumer.internals.PartitionAssignor.Assignment;
-import org.apache.kafka.clients.consumer.internals.PartitionAssignor.RebalanceProtocol;
-import org.apache.kafka.clients.consumer.internals.PartitionAssignor.Subscription;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.RebalanceProtocol;
 import org.apache.kafka.common.Cluster;
 import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.Node;
@@ -78,7 +81,7 @@
 public final class ConsumerCoordinator extends AbstractCoordinator {
     private final GroupRebalanceConfig rebalanceConfig;
     private final Logger log;
-    private final List assignors;
+    private final List assignors;
     private final ConsumerMetadata metadata;
     private final ConsumerCoordinatorMetrics sensors;
     private final SubscriptionState subscriptions;
@@ -128,7 +131,7 @@ private boolean sameRequest(final Set currentRequest, final Gene
     public ConsumerCoordinator(GroupRebalanceConfig rebalanceConfig,
                                LogContext logContext,
                                ConsumerNetworkClient client,
-                               List assignors,
+                               List assignors,
                                ConsumerMetadata metadata,
                                SubscriptionState subscriptions,
                                Metrics metrics,
@@ -170,13 +173,13 @@ public ConsumerCoordinator(GroupRebalanceConfig rebalanceConfig,
         if (!assignors.isEmpty()) {
             List supportedProtocols = new ArrayList<>(assignors.get(0).supportedProtocols());
 
-            for (PartitionAssignor assignor : assignors) {
+            for (ConsumerPartitionAssignor assignor : assignors) {
                 supportedProtocols.retainAll(assignor.supportedProtocols());
             }
 
             if (supportedProtocols.isEmpty()) {
                 throw new IllegalArgumentException("Specified assignors " +
-                    assignors.stream().map(PartitionAssignor::name).collect(Collectors.toSet()) +
+                    assignors.stream().map(ConsumerPartitionAssignor::name).collect(Collectors.toSet()) +
                     " do not have commonly supported rebalance protocol");
             }
 
@@ -201,8 +204,10 @@ protected JoinGroupRequestData.JoinGroupRequestProtocolCollection metadata() {
         this.joinedSubscription = subscriptions.subscription();
         JoinGroupRequestData.JoinGroupRequestProtocolCollection protocolSet = new JoinGroupRequestData.JoinGroupRequestProtocolCollection();
 
-        for (PartitionAssignor assignor : assignors) {
-            Subscription subscription = assignor.subscription(joinedSubscription);
+        for (ConsumerPartitionAssignor assignor : assignors) {
+            Subscription subscription = new Subscription(new ArrayList<>(joinedSubscription),
+                                                         assignor.subscriptionUserData(joinedSubscription),
+                                                         subscriptions.assignedPartitionsList());
             ByteBuffer metadata = ConsumerProtocol.serializeSubscription(subscription);
 
             protocolSet.add(new JoinGroupRequestData.JoinGroupRequestProtocol()
@@ -220,8 +225,8 @@ public void updatePatternSubscription(Cluster cluster) {
             metadata.requestUpdateForNewTopics();
     }
 
-    private PartitionAssignor lookupAssignor(String name) {
-        for (PartitionAssignor assignor : this.assignors) {
+    private ConsumerPartitionAssignor lookupAssignor(String name) {
+        for (ConsumerPartitionAssignor assignor : this.assignors) {
             if (assignor.name().equals(name))
                 return assignor;
         }
@@ -261,7 +266,7 @@ protected void onJoinComplete(int generation,
         if (!isLeader)
             assignmentSnapshot = null;
 
-        PartitionAssignor assignor = lookupAssignor(assignmentStrategy);
+        ConsumerPartitionAssignor assignor = lookupAssignor(assignmentStrategy);
         if (assignor == null)
             throw new IllegalStateException("Coordinator selected invalid assignment protocol: " + assignmentStrategy);
 
@@ -285,7 +290,8 @@ protected void onJoinComplete(int generation,
         maybeUpdateJoinedSubscription(assignedPartitions);
 
         // give the assignor a chance to update internal state based on the received assignment
-        assignor.onAssignment(assignment, generation);
+        ConsumerGroupMetadata metadata = new ConsumerGroupMetadata(rebalanceConfig.groupId, generation, memberId, rebalanceConfig.groupInstanceId);
+        assignor.onAssignment(assignment, metadata);
 
         // reschedule the auto commit starting from now
         if (autoCommitEnabled)
@@ -314,10 +320,6 @@ protected void onJoinComplete(int generation,
             case COOPERATIVE:
                 assignAndRevoke(listener, assignedPartitions, ownedPartitions);
 
-                if (assignment.error() == ConsumerProtocol.AssignmentError.NEED_REJOIN) {
-                    requestRejoin();
-                }
-
                 break;
         }
 
@@ -470,20 +472,17 @@ private void updateGroupSubscription(Set topics) {
     protected Map performAssignment(String leaderId,
                                                         String assignmentStrategy,
                                                         List allSubscriptions) {
-        PartitionAssignor assignor = lookupAssignor(assignmentStrategy);
+        ConsumerPartitionAssignor assignor = lookupAssignor(assignmentStrategy);
         if (assignor == null)
             throw new IllegalStateException("Coordinator selected invalid assignment protocol: " + assignmentStrategy);
 
         Set allSubscribedTopics = new HashSet<>();
         Map subscriptions = new HashMap<>();
-        // collect all the owned partitions
-        Map ownedPartitions = new HashMap<>();
         for (JoinGroupResponseData.JoinGroupResponseMember memberSubscription : allSubscriptions) {
             Subscription subscription = ConsumerProtocol.deserializeSubscription(ByteBuffer.wrap(memberSubscription.metadata()));
             subscription.setGroupInstanceId(Optional.ofNullable(memberSubscription.groupInstanceId()));
             subscriptions.put(memberSubscription.memberId(), subscription);
             allSubscribedTopics.addAll(subscription.topics());
-            ownedPartitions.putAll(subscription.ownedPartitions().stream().collect(Collectors.toMap(item -> item, item -> memberSubscription.memberId())));
         }
 
         // the leader will begin watching for changes to any of the topics the group is interested in,
@@ -494,16 +493,7 @@ protected Map performAssignment(String leaderId,
 
         log.debug("Performing assignment using strategy {} with subscriptions {}", assignor.name(), subscriptions);
 
-        Map assignments = assignor.assign(metadata.fetch(), subscriptions);
-
-        switch (protocol) {
-            case EAGER:
-                break;
-
-            case COOPERATIVE:
-                adjustAssignment(ownedPartitions, assignments);
-                break;
-        }
+        Map assignments = assignor.assign(metadata.fetch(), new GroupSubscription(subscriptions)).groupAssignment();
 
         // user-customized assignor may have created some topics that are not in the subscription list
         // and assign their partitions to the members; in this case we would like to update the leader's
@@ -547,40 +537,6 @@ protected Map performAssignment(String leaderId,
         return groupAssignment;
     }
 
-    private void adjustAssignment(final Map ownedPartitions,
-                                  final Map assignments) {
-        boolean revocationsNeeded = false;
-        Set assignedPartitions = new HashSet<>();
-        for (final Map.Entry entry : assignments.entrySet()) {
-            final Assignment assignment = entry.getValue();
-            assignedPartitions.addAll(assignment.partitions());
-
-            // update the assignment if the partition is owned by another different owner
-            List updatedPartitions = assignment.partitions().stream()
-                .filter(tp -> ownedPartitions.containsKey(tp) && !entry.getKey().equals(ownedPartitions.get(tp)))
-                .collect(Collectors.toList());
-            if (!updatedPartitions.equals(assignment.partitions())) {
-                assignment.updatePartitions(updatedPartitions);
-                revocationsNeeded = true;
-            }
-        }
-
-        // for all owned but not assigned partitions, blindly add them to assignment
-        for (final Map.Entry entry : ownedPartitions.entrySet()) {
-            final TopicPartition tp = entry.getKey();
-            if (!assignedPartitions.contains(tp)) {
-                assignments.get(entry.getValue()).partitions().add(tp);
-            }
-        }
-
-        // if revocations are triggered, tell everyone to re-join immediately.
-        if (revocationsNeeded) {
-            for (final Assignment assignment : assignments.values()) {
-                assignment.setError(ConsumerProtocol.AssignmentError.NEED_REJOIN);
-            }
-        }
-    }
-
     @Override
     protected void onJoinPrepare(int generation, String memberId) {
         // commit offsets prior to rebalance if auto-commit enabled
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java
index d05d5b06906a2..e852e62cd46b6 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java
@@ -16,6 +16,9 @@
  */
 package org.apache.kafka.clients.consumer.internals;
 
+import java.util.Collections;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.protocol.types.ArrayOf;
 import org.apache.kafka.common.protocol.types.Field;
@@ -49,7 +52,6 @@
  *     Topic            => String
  *     Partitions       => [int32]
  *   UserData           => Bytes
- *   ErrorCode          => [int16]
  * 
    • 
  *
  * Version 0 format:
@@ -85,11 +87,11 @@ public class ConsumerProtocol {
     public static final String TOPIC_PARTITIONS_KEY_NAME = "topic_partitions";
     public static final String USER_DATA_KEY_NAME = "user_data";
 
-    public static final Field.Int16 ERROR_CODE = new Field.Int16("error_code", "Assignment error code");
-
     public static final short CONSUMER_PROTOCOL_V0 = 0;
     public static final short CONSUMER_PROTOCOL_V1 = 1;
 
+    public static final short CONSUMER_PROTOCL_LATEST_VERSION = CONSUMER_PROTOCOL_V1;
+
     public static final Schema CONSUMER_PROTOCOL_HEADER_SCHEMA = new Schema(
             new Field(VERSION_KEY_NAME, Type.INT16));
     private static final Struct CONSUMER_PROTOCOL_HEADER_V0 = new Struct(CONSUMER_PROTOCOL_HEADER_SCHEMA)
@@ -116,36 +118,9 @@ public class ConsumerProtocol {
 
     public static final Schema ASSIGNMENT_V1 = new Schema(
         new Field(TOPIC_PARTITIONS_KEY_NAME, new ArrayOf(TOPIC_ASSIGNMENT_V0)),
-        new Field(USER_DATA_KEY_NAME, Type.NULLABLE_BYTES),
-        ERROR_CODE);
-
-    public enum AssignmentError {
-        NONE(0),
-        NEED_REJOIN(1);
+        new Field(USER_DATA_KEY_NAME, Type.NULLABLE_BYTES));
 
-        private final short code;
-
-        AssignmentError(final int code) {
-            this.code = (short) code;
-        }
-
-        public short code() {
-            return code;
-        }
-
-        public static AssignmentError fromCode(final short code) {
-            switch (code) {
-                case 0:
-                    return NONE;
-                case 1:
-                    return NEED_REJOIN;
-                default:
-                    throw new IllegalArgumentException("Unknown error code: " + code);
-            }
-        }
-    }
-
-    public static ByteBuffer serializeSubscriptionV0(PartitionAssignor.Subscription subscription) {
+    public static ByteBuffer serializeSubscriptionV0(Subscription subscription) {
         Struct struct = new Struct(SUBSCRIPTION_V0);
         struct.set(USER_DATA_KEY_NAME, subscription.userData());
         struct.set(TOPICS_KEY_NAME, subscription.topics().toArray());
@@ -157,7 +132,7 @@ public static ByteBuffer serializeSubscriptionV0(PartitionAssignor.Subscription
         return buffer;
     }
 
-    public static ByteBuffer serializeSubscriptionV1(PartitionAssignor.Subscription subscription) {
+    public static ByteBuffer serializeSubscriptionV1(Subscription subscription) {
         Struct struct = new Struct(SUBSCRIPTION_V1);
         struct.set(USER_DATA_KEY_NAME, subscription.userData());
         struct.set(TOPICS_KEY_NAME, subscription.topics().toArray());
@@ -178,8 +153,12 @@ public static ByteBuffer serializeSubscriptionV1(PartitionAssignor.Subscription
         return buffer;
     }
 
-    public static ByteBuffer serializeSubscription(PartitionAssignor.Subscription subscription) {
-        switch (subscription.version()) {
+    public static ByteBuffer serializeSubscription(Subscription subscription) {
+        return serializeSubscription(subscription, CONSUMER_PROTOCL_LATEST_VERSION);
+    }
+
+    public static ByteBuffer serializeSubscription(Subscription subscription, short version) {
+        switch (version) {
             case CONSUMER_PROTOCOL_V0:
                 return serializeSubscriptionV0(subscription);
 
@@ -192,17 +171,17 @@ public static ByteBuffer serializeSubscription(PartitionAssignor.Subscription su
         }
     }
 
-    public static PartitionAssignor.Subscription deserializeSubscriptionV0(ByteBuffer buffer) {
+    public static Subscription deserializeSubscriptionV0(ByteBuffer buffer) {
         Struct struct = SUBSCRIPTION_V0.read(buffer);
         ByteBuffer userData = struct.getBytes(USER_DATA_KEY_NAME);
         List topics = new ArrayList<>();
         for (Object topicObj : struct.getArray(TOPICS_KEY_NAME))
             topics.add((String) topicObj);
 
-        return new PartitionAssignor.Subscription(CONSUMER_PROTOCOL_V0, topics, userData);
+        return new Subscription(topics, userData, Collections.emptyList());
     }
 
-    public static PartitionAssignor.Subscription deserializeSubscriptionV1(ByteBuffer buffer) {
+    public static Subscription deserializeSubscriptionV1(ByteBuffer buffer) {
         Struct struct = SUBSCRIPTION_V1.read(buffer);
         ByteBuffer userData = struct.getBytes(USER_DATA_KEY_NAME);
         List topics = new ArrayList<>();
@@ -218,10 +197,10 @@ public static PartitionAssignor.Subscription deserializeSubscriptionV1(ByteBuffe
             }
         }
 
-        return new PartitionAssignor.Subscription(CONSUMER_PROTOCOL_V1, topics, userData, ownedPartitions);
+        return new Subscription(topics, userData, ownedPartitions);
     }
 
-    public static PartitionAssignor.Subscription deserializeSubscription(ByteBuffer buffer) {
+    public static Subscription deserializeSubscription(ByteBuffer buffer) {
         Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer);
         Short version = header.getShort(VERSION_KEY_NAME);
 
@@ -241,7 +220,7 @@ public static PartitionAssignor.Subscription deserializeSubscription(ByteBuffer
         }
     }
 
-    public static ByteBuffer serializeAssignmentV0(PartitionAssignor.Assignment assignment) {
+    public static ByteBuffer serializeAssignmentV0(Assignment assignment) {
         Struct struct = new Struct(ASSIGNMENT_V0);
         struct.set(USER_DATA_KEY_NAME, assignment.userData());
         List topicAssignments = new ArrayList<>();
@@ -261,7 +240,7 @@ public static ByteBuffer serializeAssignmentV0(PartitionAssignor.Assignment assi
         return buffer;
     }
 
-    public static ByteBuffer serializeAssignmentV1(PartitionAssignor.Assignment assignment) {
+    public static ByteBuffer serializeAssignmentV1(Assignment assignment) {
         Struct struct = new Struct(ASSIGNMENT_V1);
         struct.set(USER_DATA_KEY_NAME, assignment.userData());
         List topicAssignments = new ArrayList<>();
@@ -273,7 +252,6 @@ public static ByteBuffer serializeAssignmentV1(PartitionAssignor.Assignment assi
             topicAssignments.add(topicAssignment);
         }
         struct.set(TOPIC_PARTITIONS_KEY_NAME, topicAssignments.toArray());
-        struct.set(ERROR_CODE, assignment.error().code);
 
         ByteBuffer buffer = ByteBuffer.allocate(CONSUMER_PROTOCOL_HEADER_V1.sizeOf() + ASSIGNMENT_V1.sizeOf(struct));
         CONSUMER_PROTOCOL_HEADER_V1.writeTo(buffer);
@@ -282,8 +260,12 @@ public static ByteBuffer serializeAssignmentV1(PartitionAssignor.Assignment assi
         return buffer;
     }
 
-    public static ByteBuffer serializeAssignment(PartitionAssignor.Assignment assignment) {
-        switch (assignment.version()) {
+    public static ByteBuffer serializeAssignment(Assignment assignment) {
+        return serializeAssignment(assignment, CONSUMER_PROTOCL_LATEST_VERSION);
+    }
+
+    public static ByteBuffer serializeAssignment(Assignment assignment, short version) {
+        switch (version) {
             case CONSUMER_PROTOCOL_V0:
                 return serializeAssignmentV0(assignment);
 
@@ -296,7 +278,7 @@ public static ByteBuffer serializeAssignment(PartitionAssignor.Assignment assign
         }
     }
 
-    public static PartitionAssignor.Assignment deserializeAssignmentV0(ByteBuffer buffer) {
+    public static Assignment deserializeAssignmentV0(ByteBuffer buffer) {
         Struct struct = ASSIGNMENT_V0.read(buffer);
         ByteBuffer userData = struct.getBytes(USER_DATA_KEY_NAME);
         List partitions = new ArrayList<>();
@@ -307,27 +289,14 @@ public static PartitionAssignor.Assignment deserializeAssignmentV0(ByteBuffer bu
                 partitions.add(new TopicPartition(topic, (Integer) partitionObj));
             }
         }
-        return new PartitionAssignor.Assignment(CONSUMER_PROTOCOL_V0, partitions, userData);
+        return new Assignment(partitions, userData);
     }
 
-    public static PartitionAssignor.Assignment deserializeAssignmentV1(ByteBuffer buffer) {
-        Struct struct = ASSIGNMENT_V1.read(buffer);
-        ByteBuffer userData = struct.getBytes(USER_DATA_KEY_NAME);
-        List partitions = new ArrayList<>();
-        for (Object structObj : struct.getArray(TOPIC_PARTITIONS_KEY_NAME)) {
-            Struct assignment = (Struct) structObj;
-            String topic = assignment.getString(TOPIC_KEY_NAME);
-            for (Object partitionObj : assignment.getArray(PARTITIONS_KEY_NAME)) {
-                partitions.add(new TopicPartition(topic, (Integer) partitionObj));
-            }
-        }
-
-        AssignmentError error = AssignmentError.fromCode(struct.get(ERROR_CODE));
-
-        return new PartitionAssignor.Assignment(CONSUMER_PROTOCOL_V1, partitions, userData, error);
+    public static Assignment deserializeAssignmentV1(ByteBuffer buffer) {
+        return deserializeAssignmentV0(buffer);
     }
 
-    public static PartitionAssignor.Assignment deserializeAssignment(ByteBuffer buffer) {
+    public static Assignment deserializeAssignment(ByteBuffer buffer) {
         Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer);
         Short version = header.getShort(VERSION_KEY_NAME);
 
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
index c26f68462ea4b..b3f2ada5c4193 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
@@ -18,18 +18,12 @@
 
 import org.apache.kafka.common.Cluster;
 import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.protocol.types.SchemaException;
 
 import java.nio.ByteBuffer;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
 
-import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.CONSUMER_PROTOCOL_V0;
-import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.CONSUMER_PROTOCOL_V1;
-
 /**
  * This interface is used to define custom partition assignment for use in
  * {@link org.apache.kafka.clients.consumer.KafkaConsumer}. Members of the consumer group subscribe
@@ -43,6 +37,7 @@
  * userData in the returned Subscription. For example, to have a rack-aware assignor, an implementation
  * can use this user data to forward the rackId belonging to each member.
  */
+@Deprecated
 public interface PartitionAssignor {
 
     /**
@@ -79,21 +74,6 @@ default void onAssignment(Assignment assignment, int generation) {
         onAssignment(assignment);
     }
 
-    /**
-     * Indicate which rebalance protocol this assignor works with;
-     * By default it should always work with {@link RebalanceProtocol#EAGER}.
-     */
-    default List supportedProtocols() {
-        return Collections.singletonList(RebalanceProtocol.EAGER);
-    }
-
-    /**
-     * Return the version of the assignor which indicates how the user metadata encodings
-     * and the assignment algorithm gets evolved.
-     */
-    default short version() {
-        return (short) 0;
-    }
 
     /**
      * Unique name for this assignor (e.g. "range" or "roundrobin" or "sticky")
@@ -101,156 +81,52 @@ default short version() {
      */
     String name();
 
-    enum RebalanceProtocol {
-        EAGER((byte) 0), COOPERATIVE((byte) 1);
-
-        private final byte id;
-
-        RebalanceProtocol(byte id) {
-            this.id = id;
-        }
-
-        public byte id() {
-            return id;
-        }
-
-        public static RebalanceProtocol forId(byte id) {
-            switch (id) {
-                case 0:
-                    return EAGER;
-                case 1:
-                    return COOPERATIVE;
-                default:
-                    throw new IllegalArgumentException("Unknown rebalance protocol id: " + id);
-            }
-        }
-    }
-
     class Subscription {
-        private final Short version;
         private final List topics;
         private final ByteBuffer userData;
-        private final List ownedPartitions;
-        private Optional groupInstanceId;
 
-        Subscription(Short version,
-                     List topics,
-                     ByteBuffer userData,
-                     List ownedPartitions) {
-            this.version = version;
+        public Subscription(List topics, ByteBuffer userData) {
             this.topics = topics;
             this.userData = userData;
-            this.ownedPartitions = ownedPartitions;
-            this.groupInstanceId = Optional.empty();
-
-            if (version < CONSUMER_PROTOCOL_V0)
-                throw new SchemaException("Unsupported subscription version: " + version);
-
-            if (version < CONSUMER_PROTOCOL_V1 && !ownedPartitions.isEmpty())
-                throw new IllegalArgumentException("Subscription version smaller than 1 should not have owned partitions");
-        }
-
-        Subscription(Short version, List topics, ByteBuffer userData) {
-            this(version, topics, userData, Collections.emptyList());
-        }
-
-        public Subscription(List topics, ByteBuffer userData, List ownedPartitions) {
-            this(CONSUMER_PROTOCOL_V1, topics, userData, ownedPartitions);
-        }
-
-        public Subscription(List topics, ByteBuffer userData) {
-            this(CONSUMER_PROTOCOL_V1, topics, userData);
         }
 
         public Subscription(List topics) {
             this(topics, ByteBuffer.wrap(new byte[0]));
         }
 
-        Short version() {
-            return version;
-        }
-
         public List topics() {
             return topics;
         }
 
-        public List ownedPartitions() {
-            return ownedPartitions;
-        }
-
         public ByteBuffer userData() {
             return userData;
         }
 
-        public void setGroupInstanceId(Optional groupInstanceId) {
-            this.groupInstanceId = groupInstanceId;
-        }
-
-        public Optional groupInstanceId() {
-            return groupInstanceId;
-        }
-
         @Override
         public String toString() {
             return "Subscription(" +
-                    "version=" + version +
-                    ", topics=" + topics +
-                    ", ownedPartitions=" + ownedPartitions +
-                    ", group.instance.id=" + groupInstanceId + ")";
+                "topics=" + topics +
+                ')';
         }
     }
 
     class Assignment {
-        private final Short version;
-        private List partitions;
+        private final List partitions;
         private final ByteBuffer userData;
-        private ConsumerProtocol.AssignmentError error;
 
-        Assignment(Short version, List partitions, ByteBuffer userData, ConsumerProtocol.AssignmentError error) {
-            this.version = version;
+        public Assignment(List partitions, ByteBuffer userData) {
             this.partitions = partitions;
             this.userData = userData;
-            this.error = error;
-
-            if (version < CONSUMER_PROTOCOL_V0)
-                throw new SchemaException("Unsupported subscription version: " + version);
-
-            if (version < CONSUMER_PROTOCOL_V1 && error != ConsumerProtocol.AssignmentError.NONE)
-                throw new IllegalArgumentException("Assignment version smaller than 1 should not have error code.");
-        }
-
-        Assignment(Short version, List partitions, ByteBuffer userData) {
-            this(version, partitions, userData, ConsumerProtocol.AssignmentError.NONE);
-        }
-
-        public Assignment(List partitions, ByteBuffer userData) {
-            this(CONSUMER_PROTOCOL_V1, partitions, userData);
         }
 
         public Assignment(List partitions) {
             this(partitions, ByteBuffer.wrap(new byte[0]));
         }
 
-        Short version() {
-            return version;
-        }
-
         public List partitions() {
             return partitions;
         }
 
-        public ConsumerProtocol.AssignmentError error() {
-            return error;
-        }
-
-        public void updatePartitions(List partitions) {
-            this.partitions = partitions;
-        }
-
-        public void setError(ConsumerProtocol.AssignmentError error) {
-            this.error = error;
-        }
-
         public ByteBuffer userData() {
             return userData;
         }
@@ -258,10 +134,8 @@ public ByteBuffer userData() {
         @Override
         public String toString() {
             return "Assignment(" +
-                    "version=" + version +
-                    ", partitions=" + partitions +
-                    ", error=" + error +
-                    ')';
+                "partitions=" + partitions +
+                ')';
         }
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
index 3f1cf98be9ee5..af834ce71b1db 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.clients.consumer.internals;
 
+import java.util.ArrayList;
 import org.apache.kafka.clients.Metadata;
 import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
 import org.apache.kafka.clients.consumer.NoOffsetForPartitionException;
@@ -387,6 +388,13 @@ public synchronized Set assignedPartitions() {
         return new HashSet<>(this.assignment.partitionSet());
     }
 
+    /**
+     * @return a modifiable copy of the currently assigned partitions as a list
+     */
+    public synchronized List assignedPartitionsList() {
+        return new ArrayList<>(this.assignment.partitionSet());
+    }
+
     /**
      * Provides the number of assigned partitions in a thread safe manner.
      * @return the number of assigned partitions.
diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
index 711c8f92de849..769f58c3caf40 100644
--- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
@@ -21,9 +21,9 @@
 import org.apache.kafka.clients.MockClient;
 import org.apache.kafka.clients.NodeApiVersions;
 import org.apache.kafka.clients.admin.DeleteAclsResult.FilterResults;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor;
 import org.apache.kafka.clients.consumer.OffsetAndMetadata;
 import org.apache.kafka.clients.consumer.internals.ConsumerProtocol;
-import org.apache.kafka.clients.consumer.internals.PartitionAssignor;
 import org.apache.kafka.common.Cluster;
 import org.apache.kafka.common.ElectionType;
 import org.apache.kafka.common.KafkaException;
@@ -1222,7 +1222,7 @@ public void testDescribeConsumerGroups() throws Exception {
             topicPartitions.add(1, myTopicPartition1);
             topicPartitions.add(2, myTopicPartition2);
 
-            final ByteBuffer memberAssignment = ConsumerProtocol.serializeAssignment(new PartitionAssignor.Assignment(topicPartitions));
+            final ByteBuffer memberAssignment = ConsumerProtocol.serializeAssignment(new ConsumerPartitionAssignor.Assignment(topicPartitions));
             byte[] memberAssignmentBytes = new byte[memberAssignment.remaining()];
             memberAssignment.get(memberAssignmentBytes);
 
@@ -1282,7 +1282,7 @@ public void testDescribeMultipleConsumerGroups() throws Exception {
             topicPartitions.add(1, myTopicPartition1);
             topicPartitions.add(2, myTopicPartition2);
 
-            final ByteBuffer memberAssignment = ConsumerProtocol.serializeAssignment(new PartitionAssignor.Assignment(topicPartitions));
+            final ByteBuffer memberAssignment = ConsumerProtocol.serializeAssignment(new ConsumerPartitionAssignor.Assignment(topicPartitions));
             byte[] memberAssignmentBytes = new byte[memberAssignment.remaining()];
             memberAssignment.get(memberAssignmentBytes);
 
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
index c1adf1932ec6b..1227c27ad6035 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
@@ -28,7 +28,6 @@
 import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient;
 import org.apache.kafka.clients.consumer.internals.ConsumerProtocol;
 import org.apache.kafka.clients.consumer.internals.Fetcher;
-import org.apache.kafka.clients.consumer.internals.PartitionAssignor;
 import org.apache.kafka.clients.consumer.internals.SubscriptionState;
 import org.apache.kafka.common.Cluster;
 import org.apache.kafka.common.KafkaException;
@@ -396,7 +395,7 @@ public void verifyHeartbeatSent() throws Exception {
         initMetadata(client, Collections.singletonMap(topic, 1));
         Node node = metadata.fetch().nodes().get(0);
 
-        PartitionAssignor assignor = new RoundRobinAssignor();
+        ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
 
         KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
 
@@ -430,7 +429,7 @@ public void verifyHeartbeatSentWhenFetchedDataReady() throws Exception {
 
         initMetadata(client, Collections.singletonMap(topic, 1));
         Node node = metadata.fetch().nodes().get(0);
-        PartitionAssignor assignor = new RoundRobinAssignor();
+        ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
 
         KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
         consumer.subscribe(singleton(topic), getConsumerRebalanceListener(consumer));
@@ -465,7 +464,7 @@ public void verifyPollTimesOutDuringMetadataUpdate() {
         initMetadata(client, Collections.singletonMap(topic, 1));
         Node node = metadata.fetch().nodes().get(0);
 
-        final PartitionAssignor assignor = new RoundRobinAssignor();
+        final ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
 
         final KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
         consumer.subscribe(singleton(topic), getConsumerRebalanceListener(consumer));
@@ -489,7 +488,7 @@ public void verifyDeprecatedPollDoesNotTimeOutDuringMetadataUpdate() {
         initMetadata(client, Collections.singletonMap(topic, 1));
         Node node = metadata.fetch().nodes().get(0);
 
-        final PartitionAssignor assignor = new RoundRobinAssignor();
+        final ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
 
         final KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
         consumer.subscribe(singleton(topic), getConsumerRebalanceListener(consumer));
@@ -512,7 +511,7 @@ public void verifyNoCoordinatorLookupForManualAssignmentWithSeek() {
         MockClient client = new MockClient(time, metadata);
 
         initMetadata(client, Collections.singletonMap(topic, 1));
-        PartitionAssignor assignor = new RoundRobinAssignor();
+        ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
 
         KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
         consumer.assign(singleton(tp0));
@@ -587,7 +586,7 @@ public void testMissingOffsetNoResetPolicy() {
         initMetadata(client, Collections.singletonMap(topic, 1));
         Node node = metadata.fetch().nodes().get(0);
 
-        PartitionAssignor assignor = new RoundRobinAssignor();
+        ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
 
         KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor,
                 true, groupId, groupInstanceId);
@@ -611,7 +610,7 @@ public void testResetToCommittedOffset() {
         initMetadata(client, Collections.singletonMap(topic, 1));
         Node node = metadata.fetch().nodes().get(0);
 
-        PartitionAssignor assignor = new RoundRobinAssignor();
+        ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
 
         KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor,
                 true, groupId, groupInstanceId);
@@ -636,7 +635,7 @@ public void testResetUsingAutoResetPolicy() {
         initMetadata(client, Collections.singletonMap(topic, 1));
         Node node = metadata.fetch().nodes().get(0);
 
-        PartitionAssignor assignor = new RoundRobinAssignor();
+        ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
 
         KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor,
                 true, groupId, groupInstanceId);
@@ -663,7 +662,7 @@ public void testOffsetIsValidAfterSeek() {
         initMetadata(client, Collections.singletonMap(topic, 1));
         Node node = metadata.fetch().nodes().get(0);
 
-        PartitionAssignor assignor = new RoundRobinAssignor();
+        ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
 
         KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor,
                 true, groupId, Optional.empty());
@@ -686,7 +685,7 @@ public void testCommitsFetchedDuringAssign() {
         initMetadata(client, Collections.singletonMap(topic, 2));
         Node node = metadata.fetch().nodes().get(0);
 
-        PartitionAssignor assignor = new RoundRobinAssignor();
+        ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
 
         KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
         consumer.assign(singletonList(tp0));
@@ -724,7 +723,7 @@ public void testAutoCommitSentBeforePositionUpdate() {
         initMetadata(client, Collections.singletonMap(topic, 1));
         Node node = metadata.fetch().nodes().get(0);
 
-        PartitionAssignor assignor = new RoundRobinAssignor();
+        ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
 
         KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
         consumer.subscribe(singleton(topic), getConsumerRebalanceListener(consumer));
@@ -764,7 +763,7 @@ public void testRegexSubscription() {
         initMetadata(client, partitionCounts);
         Node node = metadata.fetch().nodes().get(0);
 
-        PartitionAssignor assignor = new RoundRobinAssignor();
+        ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
 
         KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
         prepareRebalance(client, node, singleton(topic), assignor, singletonList(tp0), null);
@@ -782,7 +781,7 @@ public void testRegexSubscription() {
 
     @Test
     public void testChangingRegexSubscription() {
-        PartitionAssignor assignor = new RoundRobinAssignor();
+        ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
 
         String otherTopic = "other";
         TopicPartition otherTopicPartition = new TopicPartition(otherTopic, 0);
@@ -828,7 +827,7 @@ public void testWakeupWithFetchDataAvailable() throws Exception {
         initMetadata(client, Collections.singletonMap(topic, 1));
         Node node = metadata.fetch().nodes().get(0);
 
-        PartitionAssignor assignor = new RoundRobinAssignor();
+        ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
 
         KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
         consumer.subscribe(singleton(topic), getConsumerRebalanceListener(consumer));
@@ -878,7 +877,7 @@ public void testPollThrowsInterruptExceptionIfInterrupted() {
         initMetadata(client, Collections.singletonMap(topic, 1));
         Node node = metadata.fetch().nodes().get(0);
 
-        final PartitionAssignor assignor = new RoundRobinAssignor();
+        final ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
 
         KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, false, groupInstanceId);
         consumer.subscribe(singleton(topic), getConsumerRebalanceListener(consumer));
@@ -908,7 +907,7 @@ public void fetchResponseWithUnexpectedPartitionIsIgnored() {
         initMetadata(client, Collections.singletonMap(topic, 1));
         Node node = metadata.fetch().nodes().get(0);
 
-        PartitionAssignor assignor = new RangeAssignor();
+        ConsumerPartitionAssignor assignor = new RangeAssignor();
 
         KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
         consumer.subscribe(singletonList(topic), getConsumerRebalanceListener(consumer));
@@ -948,7 +947,7 @@ public void testSubscriptionChangesWithAutoCommitEnabled() {
         initMetadata(client, tpCounts);
         Node node = metadata.fetch().nodes().get(0);
 
-        PartitionAssignor assignor = new RangeAssignor();
+        ConsumerPartitionAssignor assignor = new RangeAssignor();
 
         KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
 
@@ -1062,7 +1061,7 @@ public void testSubscriptionChangesWithAutoCommitDisabled() {
         initMetadata(client, tpCounts);
         Node node = metadata.fetch().nodes().get(0);
 
-        PartitionAssignor assignor = new RangeAssignor();
+        ConsumerPartitionAssignor assignor = new RangeAssignor();
 
         KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, false, groupInstanceId);
 
@@ -1124,7 +1123,7 @@ public void testManualAssignmentChangeWithAutoCommitEnabled() {
         initMetadata(client, tpCounts);
         Node node = metadata.fetch().nodes().get(0);
 
-        PartitionAssignor assignor = new RangeAssignor();
+        ConsumerPartitionAssignor assignor = new RangeAssignor();
 
         KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
 
@@ -1180,7 +1179,7 @@ public void testManualAssignmentChangeWithAutoCommitDisabled() {
         initMetadata(client, tpCounts);
         Node node = metadata.fetch().nodes().get(0);
 
-        PartitionAssignor assignor = new RangeAssignor();
+        ConsumerPartitionAssignor assignor = new RangeAssignor();
 
         KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, false, groupInstanceId);
 
@@ -1234,7 +1233,7 @@ public void testOffsetOfPausedPartitions() {
         initMetadata(client, Collections.singletonMap(topic, 2));
         Node node = metadata.fetch().nodes().get(0);
 
-        PartitionAssignor assignor = new RangeAssignor();
+        ConsumerPartitionAssignor assignor = new RangeAssignor();
 
         KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
 
@@ -1429,7 +1428,7 @@ public void shouldAttemptToRejoinGroupAfterSyncGroupFailed() throws Exception {
         initMetadata(client, Collections.singletonMap(topic, 1));
         Node node = metadata.fetch().nodes().get(0);
 
-        PartitionAssignor assignor = new RoundRobinAssignor();
+        ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
 
         KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, false, groupInstanceId);
         consumer.subscribe(singleton(topic), getConsumerRebalanceListener(consumer));
@@ -1457,7 +1456,7 @@ public boolean matches(AbstractRequest body) {
                 coordinator);
 
         // join group
-        final ByteBuffer byteBuffer = ConsumerProtocol.serializeSubscription(new PartitionAssignor.Subscription(singletonList(topic)));
+        final ByteBuffer byteBuffer = ConsumerProtocol.serializeSubscription(new ConsumerPartitionAssignor.Subscription(singletonList(topic)));
 
         // This member becomes the leader
         String memberId = "memberId";
@@ -1512,7 +1511,7 @@ private void consumerCloseTest(final long closeTimeoutMs,
         initMetadata(client, Collections.singletonMap(topic, 1));
         Node node = metadata.fetch().nodes().get(0);
 
-        PartitionAssignor assignor = new RoundRobinAssignor();
+        ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
 
         final KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, false, Optional.empty());
         consumer.subscribe(singleton(topic), getConsumerRebalanceListener(consumer));
@@ -1649,7 +1648,7 @@ private KafkaConsumer consumerWithPendingAuthenticationError() {
         initMetadata(client, singletonMap(topic, 1));
         Node node = metadata.fetch().nodes().get(0);
 
-        PartitionAssignor assignor = new RangeAssignor();
+        ConsumerPartitionAssignor assignor = new RangeAssignor();
 
         client.createPendingAuthenticationError(node, 0);
         return newConsumer(time, client, subscription, metadata, assignor, false, groupInstanceId);
@@ -1675,7 +1674,7 @@ private ConsumerMetadata createMetadata(SubscriptionState subscription) {
                                     subscription, new LogContext(), new ClusterResourceListeners());
     }
 
-    private Node prepareRebalance(MockClient client, Node node, final Set subscribedTopics, PartitionAssignor assignor, List partitions, Node coordinator) {
+    private Node prepareRebalance(MockClient client, Node node, final Set subscribedTopics, ConsumerPartitionAssignor assignor, List partitions, Node coordinator) {
         if (coordinator == null) {
             // lookup coordinator
             client.prepareResponseFrom(FindCoordinatorResponse.prepareResponse(Errors.NONE, node), node);
@@ -1692,7 +1691,7 @@ public boolean matches(AbstractRequest body) {
                 assertTrue(protocolIterator.hasNext());
 
                 ByteBuffer protocolMetadata = ByteBuffer.wrap(protocolIterator.next().metadata());
-                PartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(protocolMetadata);
+                ConsumerPartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(protocolMetadata);
                 return subscribedTopics.equals(new HashSet<>(subscription.topics()));
             }
         }, joinGroupFollowerResponse(assignor, 1, "memberId", "leaderId", Errors.NONE), coordinator);
@@ -1703,7 +1702,7 @@ public boolean matches(AbstractRequest body) {
         return coordinator;
     }
 
-    private Node prepareRebalance(MockClient client, Node node, PartitionAssignor assignor, List partitions, Node coordinator) {
+    private Node prepareRebalance(MockClient client, Node node, ConsumerPartitionAssignor assignor, List partitions, Node coordinator) {
         if (coordinator == null) {
             // lookup coordinator
             client.prepareResponseFrom(FindCoordinatorResponse.prepareResponse(Errors.NONE, node), node);
@@ -1764,7 +1763,7 @@ private OffsetCommitResponse offsetCommitResponse(Map re
         return new OffsetCommitResponse(responseData);
     }
 
-    private JoinGroupResponse joinGroupFollowerResponse(PartitionAssignor assignor, int generationId, String memberId, String leaderId, Errors error) {
+    private JoinGroupResponse joinGroupFollowerResponse(ConsumerPartitionAssignor assignor, int generationId, String memberId, String leaderId, Errors error) {
         return new JoinGroupResponse(
                 new JoinGroupResponseData()
                         .setErrorCode(error.code())
@@ -1777,7 +1776,7 @@ private JoinGroupResponse joinGroupFollowerResponse(PartitionAssignor assignor,
     }
 
     private SyncGroupResponse syncGroupResponse(List partitions, Errors error) {
-        ByteBuffer buf = ConsumerProtocol.serializeAssignment(new PartitionAssignor.Assignment(partitions));
+        ByteBuffer buf = ConsumerProtocol.serializeAssignment(new ConsumerPartitionAssignor.Assignment(partitions));
         return new SyncGroupResponse(
                 new SyncGroupResponseData()
                         .setErrorCode(error.code())
@@ -1848,7 +1847,7 @@ private KafkaConsumer newConsumer(Time time,
                                                       KafkaClient client,
                                                       SubscriptionState subscription,
                                                       ConsumerMetadata metadata,
-                                                      PartitionAssignor assignor,
+                                                      ConsumerPartitionAssignor assignor,
                                                       boolean autoCommitEnabled,
                                                       Optional groupInstanceId) {
         return newConsumer(time, client, subscription, metadata, assignor, autoCommitEnabled, groupId, groupInstanceId);
@@ -1865,7 +1864,7 @@ private KafkaConsumer newConsumer(Time time,
                                                       KafkaClient client,
                                                       SubscriptionState subscription,
                                                       ConsumerMetadata metadata,
-                                                      PartitionAssignor assignor,
+                                                      ConsumerPartitionAssignor assignor,
                                                       boolean autoCommitEnabled,
                                                       String groupId,
                                                       Optional groupInstanceId) {
@@ -1885,7 +1884,7 @@ private KafkaConsumer newConsumer(Time time,
         Deserializer keyDeserializer = new StringDeserializer();
         Deserializer valueDeserializer = new StringDeserializer();
 
-        List assignors = singletonList(assignor);
+        List assignors = singletonList(assignor);
         ConsumerInterceptors interceptors = new ConsumerInterceptors<>(Collections.emptyList());
 
         Metrics metrics = new Metrics();
@@ -1985,7 +1984,7 @@ public void testSubscriptionOnInvalidTopic() {
         initMetadata(client, Collections.singletonMap(topic, 1));
         Cluster cluster = metadata.fetch();
 
-        PartitionAssignor assignor = new RoundRobinAssignor();
+        ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
 
         String invalidTopicName = "topic abc";  // Invalid topic name due to space
 
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/RangeAssignorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/RangeAssignorTest.java
index f08ca144bf65f..118e60a3d1287 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/RangeAssignorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/RangeAssignorTest.java
@@ -16,7 +16,7 @@
  */
 package org.apache.kafka.clients.consumer;
 
-import org.apache.kafka.clients.consumer.internals.PartitionAssignor.Subscription;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription;
 import org.apache.kafka.common.TopicPartition;
 import org.junit.Test;
 
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/RoundRobinAssignorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/RoundRobinAssignorTest.java
index fa6840649a167..02fb9ffba403e 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/RoundRobinAssignorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/RoundRobinAssignorTest.java
@@ -17,7 +17,7 @@
 package org.apache.kafka.clients.consumer;
 
 import org.apache.kafka.clients.consumer.internals.AbstractPartitionAssignor.MemberInfo;
-import org.apache.kafka.clients.consumer.internals.PartitionAssignor.Subscription;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription;
 import org.apache.kafka.common.TopicPartition;
 import org.junit.Test;
 
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/StickyAssignorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/StickyAssignorTest.java
index 89f0d37ff0ccd..6dd306200006b 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/StickyAssignorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/StickyAssignorTest.java
@@ -30,7 +30,7 @@
 import java.util.Set;
 
 import org.apache.kafka.clients.consumer.StickyAssignor.ConsumerUserData;
-import org.apache.kafka.clients.consumer.internals.PartitionAssignor.Subscription;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.protocol.types.Struct;
 import org.apache.kafka.common.utils.CollectionUtils;
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index a0878fb8899fa..a81e73e022dfd 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -20,6 +20,7 @@
 import org.apache.kafka.clients.GroupRebalanceConfig;
 import org.apache.kafka.clients.MockClient;
 import org.apache.kafka.clients.consumer.CommitFailedException;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor;
 import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
 import org.apache.kafka.clients.consumer.OffsetAndMetadata;
 import org.apache.kafka.clients.consumer.OffsetCommitCallback;
@@ -125,9 +126,9 @@ public class ConsumerCoordinatorTest {
     private final MockTime time = new MockTime();
     private GroupRebalanceConfig rebalanceConfig;
 
-    private final PartitionAssignor.RebalanceProtocol protocol;
+    private final ConsumerPartitionAssignor.RebalanceProtocol protocol;
     private final MockPartitionAssignor partitionAssignor;
-    private final List assignors;
+    private final List assignors;
     private MockClient client;
     private MetadataResponse metadataResponse = TestUtils.metadataUpdateWith(1, new HashMap() {
         {
@@ -144,7 +145,7 @@ public class ConsumerCoordinatorTest {
     private MockCommitCallback mockOffsetCommitCallback;
     private ConsumerCoordinator coordinator;
 
-    public ConsumerCoordinatorTest(final PartitionAssignor.RebalanceProtocol protocol) {
+    public ConsumerCoordinatorTest(final ConsumerPartitionAssignor.RebalanceProtocol protocol) {
         this.protocol = protocol;
         this.partitionAssignor = new MockPartitionAssignor(Collections.singletonList(protocol));
         this.assignors = Collections.singletonList(partitionAssignor);
@@ -153,7 +154,7 @@ public ConsumerCoordinatorTest(final PartitionAssignor.RebalanceProtocol protoco
     @Parameterized.Parameters(name = "rebalance protocol = {0}")
     public static Collection data() {
         final List values = new ArrayList<>();
-        for (final PartitionAssignor.RebalanceProtocol protocol: PartitionAssignor.RebalanceProtocol.values()) {
+        for (final ConsumerPartitionAssignor.RebalanceProtocol protocol: ConsumerPartitionAssignor.RebalanceProtocol.values()) {
             values.add(new Object[]{protocol});
         }
         return values;
@@ -198,20 +199,20 @@ public void teardown() {
 
     @Test
     public void testSelectRebalanceProtcol() {
-        List assignors = new ArrayList<>();
-        assignors.add(new MockPartitionAssignor(Collections.singletonList(PartitionAssignor.RebalanceProtocol.EAGER)));
-        assignors.add(new MockPartitionAssignor(Collections.singletonList(PartitionAssignor.RebalanceProtocol.COOPERATIVE)));
+        List assignors = new ArrayList<>();
+        assignors.add(new MockPartitionAssignor(Collections.singletonList(ConsumerPartitionAssignor.RebalanceProtocol.EAGER)));
+        assignors.add(new MockPartitionAssignor(Collections.singletonList(ConsumerPartitionAssignor.RebalanceProtocol.COOPERATIVE)));
 
         // no commonly supported protocols
         assertThrows(IllegalArgumentException.class, () -> buildCoordinator(rebalanceConfig, new Metrics(), assignors, false));
 
         assignors.clear();
-        assignors.add(new MockPartitionAssignor(Arrays.asList(PartitionAssignor.RebalanceProtocol.EAGER, PartitionAssignor.RebalanceProtocol.COOPERATIVE)));
-        assignors.add(new MockPartitionAssignor(Arrays.asList(PartitionAssignor.RebalanceProtocol.EAGER, PartitionAssignor.RebalanceProtocol.COOPERATIVE)));
+        assignors.add(new MockPartitionAssignor(Arrays.asList(ConsumerPartitionAssignor.RebalanceProtocol.EAGER, ConsumerPartitionAssignor.RebalanceProtocol.COOPERATIVE)));
+        assignors.add(new MockPartitionAssignor(Arrays.asList(ConsumerPartitionAssignor.RebalanceProtocol.EAGER, ConsumerPartitionAssignor.RebalanceProtocol.COOPERATIVE)));
 
         // select higher indexed (more advanced) protocols
         try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors, false)) {
-            assertEquals(PartitionAssignor.RebalanceProtocol.COOPERATIVE, coordinator.getProtocol());
+            assertEquals(ConsumerPartitionAssignor.RebalanceProtocol.COOPERATIVE, coordinator.getProtocol());
         }
     }
 
@@ -553,7 +554,7 @@ public boolean matches(AbstractRequest body) {
         final int addCount = 1;
 
         // with eager protocol we will call revoke on the old assignment as well
-        if (protocol == PartitionAssignor.RebalanceProtocol.EAGER) {
+        if (protocol == ConsumerPartitionAssignor.RebalanceProtocol.EAGER) {
             revokeCount += 1;
         }
 
@@ -670,7 +671,7 @@ public boolean matches(AbstractRequest body) {
                 JoinGroupRequestData.JoinGroupRequestProtocol protocolMetadata = protocolIterator.next();
 
                 ByteBuffer metadata = ByteBuffer.wrap(protocolMetadata.metadata());
-                PartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(metadata);
+                ConsumerPartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(metadata);
                 metadata.rewind();
                 return subscription.topics().containsAll(updatedSubscription);
             }
@@ -2326,7 +2327,7 @@ public boolean matches(AbstractRequest body) {
 
     private ConsumerCoordinator buildCoordinator(final GroupRebalanceConfig rebalanceConfig,
                                                  final Metrics metrics,
-                                                 final List assignors,
+                                                 final List assignors,
                                                  final boolean autoCommitEnabled) {
         return new ConsumerCoordinator(
                 rebalanceConfig,
@@ -2385,7 +2386,7 @@ private JoinGroupResponse joinGroupLeaderResponse(int generationId,
                                                       Errors error) {
         List metadata = new ArrayList<>();
         for (Map.Entry> subscriptionEntry : subscriptions.entrySet()) {
-            PartitionAssignor.Subscription subscription = new PartitionAssignor.Subscription(subscriptionEntry.getValue());
+            ConsumerPartitionAssignor.Subscription subscription = new ConsumerPartitionAssignor.Subscription(subscriptionEntry.getValue());
             ByteBuffer buf = ConsumerProtocol.serializeSubscription(subscription);
             metadata.add(new JoinGroupResponseData.JoinGroupResponseMember()
                     .setMemberId(subscriptionEntry.getKey())
@@ -2416,7 +2417,7 @@ private JoinGroupResponse joinGroupFollowerResponse(int generationId, String mem
     }
 
     private SyncGroupResponse syncGroupResponse(List partitions, Errors error) {
-        ByteBuffer buf = ConsumerProtocol.serializeAssignment(new PartitionAssignor.Assignment(partitions));
+        ByteBuffer buf = ConsumerProtocol.serializeAssignment(new ConsumerPartitionAssignor.Assignment(partitions));
         return new SyncGroupResponse(
                 new SyncGroupResponseData()
                         .setErrorCode(error.code())
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java
index 9e601b034e529..3cf7be51665c4 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java
@@ -16,8 +16,8 @@
  */
 package org.apache.kafka.clients.consumer.internals;
 
-import org.apache.kafka.clients.consumer.internals.PartitionAssignor.Assignment;
-import org.apache.kafka.clients.consumer.internals.PartitionAssignor.Subscription;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.protocol.types.ArrayOf;
 import org.apache.kafka.common.protocol.types.Field;
@@ -39,7 +39,6 @@
 import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.TOPIC_PARTITIONS_KEY_NAME;
 import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.USER_DATA_KEY_NAME;
 import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.VERSION_KEY_NAME;
-import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
 import static org.apache.kafka.test.TestUtils.toSet;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -54,7 +53,7 @@ public class ConsumerProtocolTest {
 
     @Test
     public void serializeDeserializeMetadata() {
-        Subscription subscription = new Subscription(Arrays.asList("foo", "bar"));
+        Subscription subscription = new Subscription(Arrays.asList("foo", "bar"), ByteBuffer.wrap(new byte[0]));
         ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription);
         Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer);
         assertEquals(subscription.topics(), parsedSubscription.topics());
@@ -64,7 +63,7 @@ public void serializeDeserializeMetadata() {
 
     @Test
     public void serializeDeserializeMetadataAndGroupInstanceId() {
-        Subscription subscription = new Subscription(Arrays.asList("foo", "bar"));
+        Subscription subscription = new Subscription(Arrays.asList("foo", "bar"), ByteBuffer.wrap(new byte[0]));
         ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription);
 
         Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer);
@@ -85,8 +84,8 @@ public void serializeDeserializeNullSubscriptionUserData() {
 
     @Test
     public void deserializeOldSubscriptionVersion() {
-        Subscription subscription = new Subscription((short) 0, Arrays.asList("foo", "bar"), null);
-        ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription);
+        Subscription subscription = new Subscription(Arrays.asList("foo", "bar"), null);
+        ByteBuffer buffer = ConsumerProtocol.serializeSubscriptionV0(subscription);
         Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer);
         assertEquals(parsedSubscription.topics(), parsedSubscription.topics());
         assertNull(parsedSubscription.userData());
@@ -95,7 +94,7 @@ public void deserializeOldSubscriptionVersion() {
 
     @Test
     public void deserializeNewSubscriptionWithOldVersion() {
-        Subscription subscription = new Subscription((short) 1, Arrays.asList("foo", "bar"), null, Collections.singletonList(tp2));
+        Subscription subscription = new Subscription(Arrays.asList("foo", "bar"), null, Collections.singletonList(tp2));
         ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription);
         // ignore the version assuming it is the old byte code, as it will blindly deserialize as V0
         Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer);
@@ -145,7 +144,7 @@ public void deserializeFutureSubscriptionVersion() {
     @Test
     public void serializeDeserializeAssignment() {
         List partitions = Arrays.asList(tp1, tp2);
-        ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new Assignment(partitions));
+        ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new Assignment(partitions, ByteBuffer.wrap(new byte[0])));
         Assignment parsedAssignment = ConsumerProtocol.deserializeAssignment(buffer);
         assertEquals(toSet(partitions), toSet(parsedAssignment.partitions()));
         assertEquals(0, parsedAssignment.userData().limit());
@@ -160,29 +159,6 @@ public void deserializeNullAssignmentUserData() {
         assertNull(parsedAssignment.userData());
     }
 
-    @Test
-    public void deserializeOldAssignmentVersion() {
-        List partitions = Arrays.asList(tp1, tp2);
-        ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new Assignment((short) 0, partitions, null));
-        Assignment parsedAssignment = ConsumerProtocol.deserializeAssignment(buffer);
-        assertEquals(toSet(partitions), toSet(parsedAssignment.partitions()));
-        assertNull(parsedAssignment.userData());
-        assertEquals(ConsumerProtocol.AssignmentError.NONE, parsedAssignment.error());
-    }
-
-    @Test
-    public void deserializeNewAssignmentWithOldVersion() {
-        List partitions = Collections.singletonList(tp1);
-        ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new Assignment((short) 1, partitions, null, ConsumerProtocol.AssignmentError.NEED_REJOIN));
-        // ignore the version assuming it is the old byte code, as it will blindly deserialize as 0
-        Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer);
-        header.getShort(VERSION_KEY_NAME);
-        Assignment parsedAssignment = ConsumerProtocol.deserializeAssignmentV0(buffer);
-        assertEquals(toSet(partitions), toSet(parsedAssignment.partitions()));
-        assertNull(parsedAssignment.userData());
-        assertEquals(ConsumerProtocol.AssignmentError.NONE, parsedAssignment.error());
-    }
-
     @Test
     public void deserializeFutureAssignmentVersion() {
         // verify that a new version which adds a field is still parseable
@@ -191,7 +167,6 @@ public void deserializeFutureAssignmentVersion() {
         Schema assignmentSchemaV100 = new Schema(
                 new Field(TOPIC_PARTITIONS_KEY_NAME, new ArrayOf(TOPIC_ASSIGNMENT_V0)),
                 new Field(USER_DATA_KEY_NAME, Type.BYTES),
-                ERROR_CODE,
                 new Field("foo", Type.STRING));
 
         Struct assignmentV100 = new Struct(assignmentSchemaV100);
@@ -200,7 +175,6 @@ public void deserializeFutureAssignmentVersion() {
                         .set(ConsumerProtocol.TOPIC_KEY_NAME, tp1.topic())
                         .set(ConsumerProtocol.PARTITIONS_KEY_NAME, new Object[]{tp1.partition()})});
         assignmentV100.set(USER_DATA_KEY_NAME, ByteBuffer.wrap(new byte[0]));
-        assignmentV100.set(ERROR_CODE.name, ConsumerProtocol.AssignmentError.NEED_REJOIN.code());
         assignmentV100.set("foo", "bar");
 
         Struct headerV100 = new Struct(CONSUMER_PROTOCOL_HEADER_SCHEMA);
@@ -212,8 +186,7 @@ public void deserializeFutureAssignmentVersion() {
 
         buffer.flip();
 
-        PartitionAssignor.Assignment assignment = ConsumerProtocol.deserializeAssignment(buffer);
+        Assignment assignment = ConsumerProtocol.deserializeAssignment(buffer);
         assertEquals(toSet(Collections.singletonList(tp1)), toSet(assignment.partitions()));
-        assertEquals(ConsumerProtocol.AssignmentError.NEED_REJOIN, assignment.error());
     }
 }
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
index 6761b0cad94c8..4e06716677909 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
@@ -31,8 +31,8 @@ import kafka.server.{FetchDataInfo, KafkaConfig, LogOffsetMetadata, ReplicaManag
 import kafka.server.HostedPartition
 import kafka.utils.{KafkaScheduler, MockTime, TestUtils}
 import kafka.zk.KafkaZkClient
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription
 import org.apache.kafka.clients.consumer.internals.ConsumerProtocol
-import org.apache.kafka.clients.consumer.internals.PartitionAssignor.Subscription
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.internals.Topic
 import org.apache.kafka.common.protocol.Errors
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
index 714bb0ea21041..fa5f5115d65c1 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
@@ -16,8 +16,10 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
+import java.nio.ByteBuffer;
 import org.apache.kafka.clients.CommonClientConfigs;
-import org.apache.kafka.clients.consumer.internals.PartitionAssignor;
+import org.apache.kafka.clients.consumer.ConsumerGroupMetadata;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor;
 import org.apache.kafka.common.Cluster;
 import org.apache.kafka.common.Configurable;
 import org.apache.kafka.common.KafkaException;
@@ -56,7 +58,7 @@
 import static org.apache.kafka.common.utils.Utils.getHost;
 import static org.apache.kafka.common.utils.Utils.getPort;
 
-public class StreamsPartitionAssignor implements PartitionAssignor, Configurable {
+public class StreamsPartitionAssignor implements ConsumerPartitionAssignor, Configurable {
 
     final static int UNKNOWN = -1;
     private final static int VERSION_ONE = 1;
@@ -309,7 +311,7 @@ public String name() {
     }
 
     @Override
-    public Subscription subscription(final Set topics) {
+    public ByteBuffer subscriptionUserData(final Set topics) {
         // Adds the following information to subscription
         // 1. Client UUID (a unique id assigned to an instance of KafkaStreams)
         // 2. Task ids of previously running tasks
@@ -327,7 +329,7 @@ public Subscription subscription(final Set topics) {
 
         taskManager.updateSubscriptionsFromMetadata(topics);
 
-        return new Subscription(new ArrayList<>(topics), data.encode());
+        return data.encode();
     }
 
     private Map errorAssignment(final Map clientsMetadata,
@@ -371,8 +373,8 @@ private Map errorAssignment(final Map
      * 3. within each client, tasks are assigned to consumer clients in round-robin manner.
      */
     @Override
-    public Map assign(final Cluster metadata,
-                                          final Map subscriptions) {
+    public GroupAssignment assign(final Cluster metadata, final GroupSubscription groupSubscriptions) {
+        final Map subscriptions = groupSubscriptions.groupSubscription();
         // construct the client metadata from the decoded subscription info
         final Map clientMetadataMap = new HashMap<>();
         final Set futureConsumers = new HashSet<>();
@@ -446,7 +448,7 @@ public Map assign(final Cluster metadata,
                     !metadata.topics().contains(topic)) {
                     log.error("Missing source topic {} durign assignment. Returning error {}.",
                               topic, Error.INCOMPLETE_SOURCE_TOPIC_METADATA.name());
-                    return errorAssignment(clientMetadataMap, topic, Error.INCOMPLETE_SOURCE_TOPIC_METADATA.code);
+                    return new GroupAssignment(errorAssignment(clientMetadataMap, topic, Error.INCOMPLETE_SOURCE_TOPIC_METADATA.code));
                 }
             }
             for (final InternalTopicConfig topic: topicsInfo.repartitionSourceTopics.values()) {
@@ -644,7 +646,7 @@ public Map assign(final Cluster metadata,
             assignment = computeNewAssignment(clientMetadataMap, partitionsForTask, partitionsByHostState, minReceivedMetadataVersion);
         }
 
-        return assignment;
+        return new GroupAssignment(assignment);
     }
 
     private Map computeNewAssignment(final Map clientsMetadata,
@@ -777,7 +779,7 @@ List> interleaveTasksByGroupId(final Collection taskIds, fi
      * @throws TaskAssignmentException if there is no task id for one of the partitions specified
      */
     @Override
-    public void onAssignment(final Assignment assignment) {
+    public void onAssignment(final Assignment assignment, final ConsumerGroupMetadata metadata) {
         final List partitions = new ArrayList<>(assignment.partitions());
         partitions.sort(PARTITION_COMPARATOR);
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
index 5fa6653ba3acc..616deaf39040c 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
@@ -16,7 +16,8 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
-import org.apache.kafka.clients.consumer.internals.PartitionAssignor;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.GroupSubscription;
 import org.apache.kafka.common.Cluster;
 import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.Node;
@@ -146,7 +147,7 @@ private void mockTaskManager(final Set prevTasks,
         EasyMock.replay(taskManager);
     }
 
-    private Map subscriptions;
+    private Map subscriptions;
 
     @Before
     public void setUp() {
@@ -200,7 +201,9 @@ public void testSubscription() {
         mockTaskManager(prevTasks, cachedTasks, processId, builder);
 
         configurePartitionAssignor(Collections.emptyMap());
-        final PartitionAssignor.Subscription subscription = partitionAssignor.subscription(Utils.mkSet("topic1", "topic2"));
+
+        final Set topics = Utils.mkSet("topic1", "topic2");
+        final ConsumerPartitionAssignor.Subscription subscription = new ConsumerPartitionAssignor.Subscription(new ArrayList<>(topics), partitionAssignor.subscriptionUserData(topics));
 
         Collections.sort(subscription.topics());
         assertEquals(asList("topic1", "topic2"), subscription.topics());
@@ -236,16 +239,16 @@ public void testAssignBasic() {
         partitionAssignor.setInternalTopicManager(new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer));
 
         subscriptions.put("consumer10",
-                new PartitionAssignor.Subscription(topics,
+                new ConsumerPartitionAssignor.Subscription(topics,
                         new SubscriptionInfo(uuid1, prevTasks10, standbyTasks10, userEndPoint).encode()));
         subscriptions.put("consumer11",
-                new PartitionAssignor.Subscription(topics,
+                new ConsumerPartitionAssignor.Subscription(topics,
                         new SubscriptionInfo(uuid1, prevTasks11, standbyTasks11, userEndPoint).encode()));
         subscriptions.put("consumer20",
-                new PartitionAssignor.Subscription(topics,
+                new ConsumerPartitionAssignor.Subscription(topics,
                         new SubscriptionInfo(uuid2, prevTasks20, standbyTasks20, userEndPoint).encode()));
 
-        final Map assignments = partitionAssignor.assign(metadata, subscriptions);
+        final Map assignments = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
 
         // check assigned partitions
         assertEquals(Utils.mkSet(Utils.mkSet(t1p0, t2p0), Utils.mkSet(t1p1, t2p1)),
@@ -320,13 +323,13 @@ public void shouldAssignEvenlyAcrossConsumersOneClientMultipleThreads() {
         partitionAssignor.setInternalTopicManager(new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer));
 
         subscriptions.put("consumer10",
-                          new PartitionAssignor.Subscription(topics,
+                          new ConsumerPartitionAssignor.Subscription(topics,
                                   new SubscriptionInfo(uuid1, new HashSet<>(), new HashSet<>(), userEndPoint).encode()));
         subscriptions.put("consumer11",
-                          new PartitionAssignor.Subscription(topics,
+                          new ConsumerPartitionAssignor.Subscription(topics,
                                   new SubscriptionInfo(uuid1, new HashSet<>(), new HashSet<>(), userEndPoint).encode()));
 
-        final Map assignments = partitionAssignor.assign(localMetadata, subscriptions);
+        final Map assignments = partitionAssignor.assign(localMetadata, new GroupSubscription(subscriptions)).groupAssignment();
 
         // check assigned partitions
         assertEquals(Utils.mkSet(Utils.mkSet(t2p2, t1p0, t1p2, t2p0), Utils.mkSet(t1p1, t2p1, t1p3, t2p3)),
@@ -365,10 +368,10 @@ public void testAssignWithPartialTopology() {
 
         // will throw exception if it fails
         subscriptions.put("consumer10",
-                new PartitionAssignor.Subscription(topics,
+                new ConsumerPartitionAssignor.Subscription(topics,
                         new SubscriptionInfo(uuid1, emptyTasks, emptyTasks, userEndPoint).encode()
         ));
-        final Map assignments = partitionAssignor.assign(metadata, subscriptions);
+        final Map assignments = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
 
         // check assignment info
         final AssignmentInfo info10 = checkAssignment(Utils.mkSet("topic1"), assignments.get("consumer10"));
@@ -399,12 +402,12 @@ public void testAssignEmptyMetadata() {
         configurePartitionAssignor(Collections.emptyMap());
 
         subscriptions.put("consumer10",
-                new PartitionAssignor.Subscription(topics,
+                new ConsumerPartitionAssignor.Subscription(topics,
                         new SubscriptionInfo(uuid1, prevTasks10, standbyTasks10, userEndPoint).encode()
                 ));
 
         // initially metadata is empty
-        Map assignments = partitionAssignor.assign(emptyMetadata, subscriptions);
+        Map assignments = partitionAssignor.assign(emptyMetadata, new GroupSubscription(subscriptions)).groupAssignment();
 
         // check assigned partitions
         assertEquals(Collections.emptySet(),
@@ -417,7 +420,7 @@ public void testAssignEmptyMetadata() {
         assertEquals(0, allActiveTasks.size());
 
         // then metadata gets populated
-        assignments = partitionAssignor.assign(metadata, subscriptions);
+        assignments = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
         // check assigned partitions
         assertEquals(Utils.mkSet(Utils.mkSet(t1p0, t2p0, t1p0, t2p0, t1p1, t2p1, t1p2, t2p2)),
             Utils.mkSet(new HashSet<>(assignments.get("consumer10").partitions())));
@@ -455,16 +458,16 @@ public void testAssignWithNewTasks() {
         partitionAssignor.setInternalTopicManager(new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer));
 
         subscriptions.put("consumer10",
-                new PartitionAssignor.Subscription(topics,
+                new ConsumerPartitionAssignor.Subscription(topics,
                         new SubscriptionInfo(uuid1, prevTasks10, emptyTasks, userEndPoint).encode()));
         subscriptions.put("consumer11",
-                new PartitionAssignor.Subscription(topics,
+                new ConsumerPartitionAssignor.Subscription(topics,
                         new SubscriptionInfo(uuid1, prevTasks11, emptyTasks, userEndPoint).encode()));
         subscriptions.put("consumer20",
-                new PartitionAssignor.Subscription(topics,
+                new ConsumerPartitionAssignor.Subscription(topics,
                         new SubscriptionInfo(uuid2, prevTasks20, emptyTasks, userEndPoint).encode()));
 
-        final Map assignments = partitionAssignor.assign(metadata, subscriptions);
+        final Map assignments = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
 
         // check assigned partitions: since there is no previous task for topic 3 it will be assigned randomly so we cannot check exact match
         // also note that previously assigned partitions / tasks may not stay on the previous host since we may assign the new task first and
@@ -521,16 +524,16 @@ public void testAssignWithStates() {
         partitionAssignor.setInternalTopicManager(new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer));
 
         subscriptions.put("consumer10",
-                new PartitionAssignor.Subscription(topics,
+                new ConsumerPartitionAssignor.Subscription(topics,
                         new SubscriptionInfo(uuid1, emptyTasks, emptyTasks, userEndPoint).encode()));
         subscriptions.put("consumer11",
-                new PartitionAssignor.Subscription(topics,
+                new ConsumerPartitionAssignor.Subscription(topics,
                         new SubscriptionInfo(uuid1, emptyTasks, emptyTasks, userEndPoint).encode()));
         subscriptions.put("consumer20",
-                new PartitionAssignor.Subscription(topics,
+                new ConsumerPartitionAssignor.Subscription(topics,
                         new SubscriptionInfo(uuid2, emptyTasks, emptyTasks, userEndPoint).encode()));
 
-        final Map assignments = partitionAssignor.assign(metadata, subscriptions);
+        final Map assignments = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
 
         // check assigned partition size: since there is no previous task and there are two sub-topologies the assignment is random so we cannot check exact match
         assertEquals(2, assignments.get("consumer10").partitions().size());
@@ -609,16 +612,16 @@ public void testAssignWithStandbyReplicas() {
         partitionAssignor.setInternalTopicManager(new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer));
 
         subscriptions.put("consumer10",
-                new PartitionAssignor.Subscription(topics,
+                new ConsumerPartitionAssignor.Subscription(topics,
                         new SubscriptionInfo(uuid1, prevTasks00, standbyTasks01, userEndPoint).encode()));
         subscriptions.put("consumer11",
-                new PartitionAssignor.Subscription(topics,
+                new ConsumerPartitionAssignor.Subscription(topics,
                         new SubscriptionInfo(uuid1, prevTasks01, standbyTasks02, userEndPoint).encode()));
         subscriptions.put("consumer20",
-                new PartitionAssignor.Subscription(topics,
+                new ConsumerPartitionAssignor.Subscription(topics,
                         new SubscriptionInfo(uuid2, prevTasks02, standbyTasks00, "any:9097").encode()));
 
-        final Map assignments = partitionAssignor.assign(metadata, subscriptions);
+        final Map assignments = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
 
         // the first consumer
         final AssignmentInfo info10 = checkAssignment(allTopics, assignments.get("consumer10"));
@@ -666,7 +669,7 @@ public void testOnAssignment() {
         standbyTasks.put(task2, Utils.mkSet(t3p2));
 
         final AssignmentInfo info = new AssignmentInfo(activeTaskList, standbyTasks, hostState);
-        final PartitionAssignor.Assignment assignment = new PartitionAssignor.Assignment(asList(t3p0, t3p3), info.encode());
+        final ConsumerPartitionAssignor.Assignment assignment = new ConsumerPartitionAssignor.Assignment(asList(t3p0, t3p3), info.encode());
 
         final Capture capturedCluster = EasyMock.newCapture();
         taskManager.setPartitionsByHostState(hostState);
@@ -677,7 +680,7 @@ public void testOnAssignment() {
         EasyMock.expectLastCall();
         EasyMock.replay(taskManager);
 
-        partitionAssignor.onAssignment(assignment);
+        partitionAssignor.onAssignment(assignment, null);
 
         EasyMock.verify(taskManager);
 
@@ -704,10 +707,10 @@ public void testAssignWithInternalTopics() {
         partitionAssignor.setInternalTopicManager(internalTopicManager);
 
         subscriptions.put("consumer10",
-                new PartitionAssignor.Subscription(topics,
+                new ConsumerPartitionAssignor.Subscription(topics,
                         new SubscriptionInfo(uuid1, emptyTasks, emptyTasks, userEndPoint).encode())
         );
-        partitionAssignor.assign(metadata, subscriptions);
+        partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
 
         // check prepared internal topics
         assertEquals(1, internalTopicManager.readyTopics.size());
@@ -738,10 +741,10 @@ public void testAssignWithInternalTopicThatsSourceIsAnotherInternalTopic() {
         partitionAssignor.setInternalTopicManager(internalTopicManager);
 
         subscriptions.put("consumer10",
-                new PartitionAssignor.Subscription(topics,
+                new ConsumerPartitionAssignor.Subscription(topics,
                         new SubscriptionInfo(uuid1, emptyTasks, emptyTasks, userEndPoint).encode())
         );
-        partitionAssignor.assign(metadata, subscriptions);
+        partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
 
         // check prepared internal topics
         assertEquals(2, internalTopicManager.readyTopics.size());
@@ -790,11 +793,11 @@ public void shouldGenerateTasksForAllCreatedPartitions() {
         partitionAssignor.setInternalTopicManager(mockInternalTopicManager);
 
         subscriptions.put(client,
-                new PartitionAssignor.Subscription(
+                new ConsumerPartitionAssignor.Subscription(
                         asList("topic1", "topic3"),
                         new SubscriptionInfo(uuid, emptyTasks, emptyTasks, userEndPoint).encode())
         );
-        final Map assignment = partitionAssignor.assign(metadata, subscriptions);
+        final Map assignment = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
 
         final Map expectedCreatedInternalTopics = new HashMap<>();
         expectedCreatedInternalTopics.put(applicationId + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 4);
@@ -841,7 +844,8 @@ public void shouldAddUserDefinedEndPointToSubscription() {
             uuid1,
             builder);
         configurePartitionAssignor(Collections.singletonMap(StreamsConfig.APPLICATION_SERVER_CONFIG, userEndPoint));
-        final PartitionAssignor.Subscription subscription = partitionAssignor.subscription(Utils.mkSet("input"));
+        final Set topics = Utils.mkSet("input");
+        final ConsumerPartitionAssignor.Subscription subscription = new ConsumerPartitionAssignor.Subscription(new ArrayList<>(topics), partitionAssignor.subscriptionUserData(topics));
         final SubscriptionInfo subscriptionInfo = SubscriptionInfo.decode(subscription.userData());
         assertEquals("localhost:8080", subscriptionInfo.userEndPoint());
     }
@@ -863,11 +867,11 @@ public void shouldMapUserEndPointToTopicPartitions() {
         partitionAssignor.setInternalTopicManager(new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer));
 
         subscriptions.put("consumer1",
-                new PartitionAssignor.Subscription(topics,
+                new ConsumerPartitionAssignor.Subscription(topics,
                         new SubscriptionInfo(uuid1, emptyTasks, emptyTasks, userEndPoint).encode())
         );
-        final Map assignments = partitionAssignor.assign(metadata, subscriptions);
-        final PartitionAssignor.Assignment consumerAssignment = assignments.get("consumer1");
+        final Map assignments = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
+        final ConsumerPartitionAssignor.Assignment consumerAssignment = assignments.get("consumer1");
         final AssignmentInfo assignmentInfo = AssignmentInfo.decode(consumerAssignment.userData());
         final Set topicPartitions = assignmentInfo.partitionsByHost().get(new HostInfo("localhost", 8080));
         assertEquals(
@@ -961,11 +965,11 @@ public void shouldNotLoopInfinitelyOnMissingMetadataAndShouldNotCreateRelatedTas
         partitionAssignor.setInternalTopicManager(mockInternalTopicManager);
 
         subscriptions.put(client,
-                new PartitionAssignor.Subscription(
+                new ConsumerPartitionAssignor.Subscription(
                         Collections.singletonList("unknownTopic"),
                         new SubscriptionInfo(uuid, emptyTasks, emptyTasks, userEndPoint).encode())
         );
-        final Map assignment = partitionAssignor.assign(metadata, subscriptions);
+        final Map assignment = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
 
         assertThat(mockInternalTopicManager.readyTopics.isEmpty(), equalTo(true));
 
@@ -985,7 +989,7 @@ public void shouldUpdateClusterMetadataAndHostInfoOnAssignment() {
         EasyMock.expectLastCall();
         EasyMock.replay(taskManager);
 
-        partitionAssignor.onAssignment(createAssignment(hostState));
+        partitionAssignor.onAssignment(createAssignment(hostState), null);
 
         EasyMock.verify(taskManager);
     }
@@ -1015,18 +1019,18 @@ public void shouldNotAddStandbyTaskPartitionsToPartitionsForHost() {
             mockClientSupplier.restoreConsumer));
 
         subscriptions.put("consumer1",
-                new PartitionAssignor.Subscription(
+                new ConsumerPartitionAssignor.Subscription(
                         Collections.singletonList("topic1"),
                         new SubscriptionInfo(uuid, emptyTasks, emptyTasks, userEndPoint).encode())
         );
         subscriptions.put("consumer2",
-                new PartitionAssignor.Subscription(
+                new ConsumerPartitionAssignor.Subscription(
                         Collections.singletonList("topic1"),
                         new SubscriptionInfo(UUID.randomUUID(), emptyTasks, emptyTasks, "other:9090").encode())
         );
         final Set allPartitions = Utils.mkSet(t1p0, t1p1, t1p2);
-        final Map assign = partitionAssignor.assign(metadata, subscriptions);
-        final PartitionAssignor.Assignment consumer1Assignment = assign.get("consumer1");
+        final Map assign = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
+        final ConsumerPartitionAssignor.Assignment consumer1Assignment = assign.get("consumer1");
         final AssignmentInfo assignmentInfo = AssignmentInfo.decode(consumer1Assignment.userData());
         final Set consumer1partitions = assignmentInfo.partitionsByHost().get(new HostInfo("localhost", 8080));
         final Set consumer2Partitions = assignmentInfo.partitionsByHost().get(new HostInfo("other", 9090));
@@ -1109,12 +1113,12 @@ public void shouldReturnLowestAssignmentVersionForDifferentSubscriptionVersionsV
     private void shouldReturnLowestAssignmentVersionForDifferentSubscriptionVersions(final int smallestVersion,
                                                                                      final int otherVersion) {
         subscriptions.put("consumer1",
-                new PartitionAssignor.Subscription(
+                new ConsumerPartitionAssignor.Subscription(
                         Collections.singletonList("topic1"),
                         new SubscriptionInfo(smallestVersion, UUID.randomUUID(), emptyTasks, emptyTasks, null).encode())
         );
         subscriptions.put("consumer2",
-                new PartitionAssignor.Subscription(
+                new ConsumerPartitionAssignor.Subscription(
                         Collections.singletonList("topic1"),
                         new SubscriptionInfo(otherVersion, UUID.randomUUID(), emptyTasks, emptyTasks, null).encode()
                 )
@@ -1126,7 +1130,7 @@ private void shouldReturnLowestAssignmentVersionForDifferentSubscriptionVersions
             UUID.randomUUID(),
             builder);
         partitionAssignor.configure(configProps());
-        final Map assignment = partitionAssignor.assign(metadata, subscriptions);
+        final Map assignment = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
 
         assertThat(assignment.size(), equalTo(2));
         assertThat(AssignmentInfo.decode(assignment.get("consumer1").userData()).version(), equalTo(smallestVersion));
@@ -1142,7 +1146,8 @@ public void shouldDownGradeSubscriptionToVersion1() {
             builder);
         configurePartitionAssignor(Collections.singletonMap(StreamsConfig.UPGRADE_FROM_CONFIG, StreamsConfig.UPGRADE_FROM_0100));
 
-        final PartitionAssignor.Subscription subscription = partitionAssignor.subscription(Utils.mkSet("topic1"));
+        final Set topics = Utils.mkSet("topic1");
+        final ConsumerPartitionAssignor.Subscription subscription = new ConsumerPartitionAssignor.Subscription(new ArrayList<>(topics), partitionAssignor.subscriptionUserData(topics));
 
         assertThat(SubscriptionInfo.decode(subscription.userData()).version(), equalTo(1));
     }
@@ -1180,7 +1185,8 @@ private void shouldDownGradeSubscriptionToVersion2(final Object upgradeFromValue
             builder);
         configurePartitionAssignor(Collections.singletonMap(StreamsConfig.UPGRADE_FROM_CONFIG, upgradeFromValue));
 
-        final PartitionAssignor.Subscription subscription = partitionAssignor.subscription(Utils.mkSet("topic1"));
+        final Set topics = Utils.mkSet("topic1");
+        final ConsumerPartitionAssignor.Subscription subscription = new ConsumerPartitionAssignor.Subscription(new ArrayList<>(topics), partitionAssignor.subscriptionUserData(topics));
 
         assertThat(SubscriptionInfo.decode(subscription.userData()).version(), equalTo(2));
     }
@@ -1200,12 +1206,12 @@ public void shouldReturnUnchangedAssignmentForOldInstancesAndEmptyAssignmentForF
         };
 
         subscriptions.put("consumer1",
-                new PartitionAssignor.Subscription(
+                new ConsumerPartitionAssignor.Subscription(
                         Collections.singletonList("topic1"),
                         new SubscriptionInfo(UUID.randomUUID(), activeTasks, standbyTasks, null).encode())
         );
         subscriptions.put("future-consumer",
-                new PartitionAssignor.Subscription(
+                new ConsumerPartitionAssignor.Subscription(
                         Collections.singletonList("topic1"),
                         encodeFutureSubscription())
         );
@@ -1216,7 +1222,7 @@ public void shouldReturnUnchangedAssignmentForOldInstancesAndEmptyAssignmentForF
             UUID.randomUUID(),
             builder);
         partitionAssignor.configure(configProps());
-        final Map assignment = partitionAssignor.assign(metadata, subscriptions);
+        final Map assignment = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
 
         assertThat(assignment.size(), equalTo(2));
         assertThat(
@@ -1252,12 +1258,12 @@ private ByteBuffer encodeFutureSubscription() {
 
     private void shouldThrowIfPreVersionProbingSubscriptionAndFutureSubscriptionIsMixed(final int oldVersion) {
         subscriptions.put("consumer1",
-                new PartitionAssignor.Subscription(
+                new ConsumerPartitionAssignor.Subscription(
                         Collections.singletonList("topic1"),
                         new SubscriptionInfo(oldVersion, UUID.randomUUID(), emptyTasks, emptyTasks, null).encode())
         );
         subscriptions.put("future-consumer",
-                new PartitionAssignor.Subscription(
+                new ConsumerPartitionAssignor.Subscription(
                         Collections.singletonList("topic1"),
                         encodeFutureSubscription())
         );
@@ -1270,24 +1276,24 @@ private void shouldThrowIfPreVersionProbingSubscriptionAndFutureSubscriptionIsMi
         partitionAssignor.configure(configProps());
 
         try {
-            partitionAssignor.assign(metadata, subscriptions);
+            partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
             fail("Should have thrown IllegalStateException");
         } catch (final IllegalStateException expected) {
             // pass
         }
     }
 
-    private PartitionAssignor.Assignment createAssignment(final Map> firstHostState) {
+    private ConsumerPartitionAssignor.Assignment createAssignment(final Map> firstHostState) {
         final AssignmentInfo info = new AssignmentInfo(Collections.emptyList(),
                                                        Collections.emptyMap(),
                                                        firstHostState);
 
-        return new PartitionAssignor.Assignment(
+        return new ConsumerPartitionAssignor.Assignment(
                 Collections.emptyList(), info.encode());
     }
 
     private AssignmentInfo checkAssignment(final Set expectedTopics,
-                                           final PartitionAssignor.Assignment assignment) {
+                                           final ConsumerPartitionAssignor.Assignment assignment) {
 
         // This assumed 1) DefaultPartitionGrouper is used, and 2) there is an only one topic group.
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
index 27bee810d85fd..0b2d0b319aa5a 100644
--- a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
@@ -18,8 +18,9 @@
 
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.ConsumerGroupMetadata;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor;
 import org.apache.kafka.clients.consumer.KafkaConsumer;
-import org.apache.kafka.clients.consumer.internals.PartitionAssignor;
 import org.apache.kafka.common.Cluster;
 import org.apache.kafka.common.PartitionInfo;
 import org.apache.kafka.common.TopicPartition;
@@ -113,7 +114,7 @@ public FutureStreamsPartitionAssignor() {
         }
 
         @Override
-        public Subscription subscription(final Set topics) {
+        public ByteBuffer subscriptionUserData(final Set topics) {
             // Adds the following information to subscription
             // 1. Client UUID (a unique id assigned to an instance of KafkaStreams)
             // 2. Task ids of previously running tasks
@@ -133,13 +134,13 @@ public Subscription subscription(final Set topics) {
 
             taskManager.updateSubscriptionsFromMetadata(topics);
 
-            return new Subscription(new ArrayList<>(topics), data.encode());
+            return data.encode();
         }
 
         @Override
-        public void onAssignment(final PartitionAssignor.Assignment assignment) {
+        public void onAssignment(final ConsumerPartitionAssignor.Assignment assignment, final ConsumerGroupMetadata metadata) {
             try {
-                super.onAssignment(assignment);
+                super.onAssignment(assignment, metadata);
                 return;
             } catch (final TaskAssignmentException cannotProcessFutureVersion) {
                 // continue
@@ -183,15 +184,15 @@ public void onAssignment(final PartitionAssignor.Assignment assignment) {
         }
 
         @Override
-        public Map assign(final Cluster metadata,
-                                              final Map subscriptions) {
+        public GroupAssignment assign(final Cluster metadata, final GroupSubscription groupSubscription) {
+            final Map subscriptions = groupSubscription.groupSubscription();
             Map assignment = null;
 
             final Map downgradedSubscriptions = new HashMap<>();
             for (final Subscription subscription : subscriptions.values()) {
                 final SubscriptionInfo info = SubscriptionInfo.decode(subscription.userData());
                 if (info.version() < SubscriptionInfo.LATEST_SUPPORTED_VERSION + 1) {
-                    assignment = super.assign(metadata, subscriptions);
+                    assignment = super.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
                     break;
                 }
             }
@@ -219,7 +220,7 @@ public Map assign(final Cluster metadata,
                                 info.userEndPoint())
                                 .encode()));
                 }
-                assignment = super.assign(metadata, downgradedSubscriptions);
+                assignment = super.assign(metadata, new GroupSubscription(downgradedSubscriptions)).groupAssignment();
                 bumpUsedVersion = true;
                 bumpSupportedVersion = true;
             }
@@ -238,7 +239,7 @@ public Map assign(final Cluster metadata,
                             .encode()));
             }
 
-            return newAssignment;
+            return new GroupAssignment(newAssignment);
         }
     }
 

From 79341a12f235510d9f78af24d979102c3d8882e1 Mon Sep 17 00:00:00 2001
From: Boyang Chen 
Date: Fri, 26 Jul 2019 15:31:31 -0700
Subject: [PATCH 0479/1071] KAFKA-8715; Fix buggy reliance on state timestamp
 in static member.id generation (#7116)

The bug is that we accidentally used the current state timestamp for the group instead of the real current time. When a group is first loaded, this timestamp is not initialized, so this resulted in a `NoSuchElementException`. Additionally this violated the intended uniqueness of the memberId, which could have broken the group instance fencing. Fix is made and unit test to make sure the timestamp is properly encoded within the returned member.id.

Reviewers: Ismael Juma , Guozhang Wang , Jason Gustafson 
---
 .../kafka/coordinator/group/GroupMetadata.scala |  2 +-
 .../group/GroupCoordinatorTest.scala            | 17 +++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala
index 58a68a2c18b16..4d283bc8d62e8 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala
@@ -365,7 +365,7 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
       case None =>
         clientId + GroupMetadata.MemberIdDelimiter + UUID.randomUUID().toString
       case Some(instanceId) =>
-        instanceId + GroupMetadata.MemberIdDelimiter + currentStateTimestamp.get
+        instanceId + GroupMetadata.MemberIdDelimiter + UUID.randomUUID().toString
     }
   }
 
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
index 0a7bcc74e2804..d72bfc8f5cebc 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
@@ -873,6 +873,23 @@ class GroupCoordinatorTest {
     assertEquals(Errors.FENCED_INSTANCE_ID, invalidHeartbeatResult)
   }
 
+  @Test
+  def shouldGetDifferentStaticMemberIdAfterEachRejoin(): Unit = {
+    val initialResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
+
+    val timeAdvance = 1
+    var lastMemberId = initialResult.leaderId
+    for (_ <- 1 to 5) {
+      EasyMock.reset(replicaManager)
+
+      val joinGroupResult = staticJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID,
+        leaderInstanceId, protocolType, protocols, clockAdvance = timeAdvance)
+      assertTrue(joinGroupResult.memberId.startsWith(leaderInstanceId.get))
+      assertNotEquals(lastMemberId, joinGroupResult.memberId)
+      lastMemberId = joinGroupResult.memberId
+    }
+  }
+
   @Test
   def testOffsetCommitDeadGroup() {
     val memberId = "memberId"

From 74c90f46c34727be9484e9826ff543b451ada775 Mon Sep 17 00:00:00 2001
From: Boyang Chen 
Date: Fri, 26 Jul 2019 23:13:37 -0700
Subject: [PATCH 0480/1071] KAFKA-8221; Add batch leave group request (#6714)

This patch is part of KIP-345. We are aiming to support batch leave group request issued from admin client. This diff is the first effort to bump leave group request version.

Reviewers: Guozhang Wang , Jason Gustafson 
---
 checkstyle/suppressions.xml                   |   2 +-
 .../internals/AbstractCoordinator.java        |  26 +-
 .../common/requests/LeaveGroupRequest.java    |  48 ++-
 .../common/requests/LeaveGroupResponse.java   |  86 ++++-
 .../common/message/LeaveGroupRequest.json     |  15 +-
 .../common/message/LeaveGroupResponse.json    |  15 +-
 .../internals/AbstractCoordinatorTest.java    | 280 ++++++++++-------
 .../internals/ConsumerCoordinatorTest.java    | 294 +++++++-----------
 .../kafka/common/message/MessageTest.java     |  44 ++-
 .../requests/LeaveGroupRequestTest.java       |  88 +++++-
 .../requests/LeaveGroupResponseTest.java      |  86 ++++-
 .../common/requests/RequestResponseTest.java  |  37 ++-
 .../coordinator/group/GroupCoordinator.scala  | 105 +++++--
 .../main/scala/kafka/server/KafkaApis.scala   |  53 ++--
 .../kafka/api/AuthorizerIntegrationTest.scala |   9 +-
 .../GroupCoordinatorConcurrencyTest.scala     |  24 +-
 .../group/GroupCoordinatorTest.scala          | 238 +++++++++++---
 .../unit/kafka/server/KafkaApisTest.scala     |  58 ++++
 .../unit/kafka/server/RequestQuotaTest.scala  |   9 +-
 .../processor/internals/AssignedTasks.java    |   1 -
 20 files changed, 1065 insertions(+), 453 deletions(-)

diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index 03321f12a5f21..908e1148d6323 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -62,7 +62,7 @@
 
     
     
+              files="(Sender|Fetcher|KafkaConsumer|Metrics|RequestResponse|TransactionManager|KafkaAdminClient|Message)Test.java"/>
 
     
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index b92a4a6109107..fc385f67e1022 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -35,7 +35,8 @@
 import org.apache.kafka.common.message.HeartbeatRequestData;
 import org.apache.kafka.common.message.JoinGroupRequestData;
 import org.apache.kafka.common.message.JoinGroupResponseData;
-import org.apache.kafka.common.message.LeaveGroupRequestData;
+import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity;
+import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse;
 import org.apache.kafka.common.message.SyncGroupRequestData;
 import org.apache.kafka.common.metrics.Measurable;
 import org.apache.kafka.common.metrics.MetricConfig;
@@ -843,7 +844,8 @@ protected void close(Timer timer) {
      * Leave the current group and reset local generation/memberId.
      * @param leaveReason reason to attempt leaving the group
      */
-    public synchronized void maybeLeaveGroup(String leaveReason) {
+    public synchronized RequestFuture maybeLeaveGroup(String leaveReason) {
+        RequestFuture future = null;
         // Starting from 2.3, only dynamic members will send LeaveGroupRequest to the broker,
         // consumer with valid group.instance.id is viewed as static member that never sends LeaveGroup,
         // and the membership expiration is only controlled by session timeout.
@@ -853,14 +855,18 @@ public synchronized void maybeLeaveGroup(String leaveReason) {
             // attempt any resending if the request fails or times out.
             log.info("Member {} sending LeaveGroup request to coordinator {} due to {}",
                      generation.memberId, coordinator, leaveReason);
-            LeaveGroupRequest.Builder request = new LeaveGroupRequest.Builder(new LeaveGroupRequestData()
-                    .setGroupId(rebalanceConfig.groupId).setMemberId(generation.memberId));
-            client.send(coordinator, request)
+            LeaveGroupRequest.Builder request = new LeaveGroupRequest.Builder(
+                rebalanceConfig.groupId,
+                Collections.singletonList(new MemberIdentity()
+                                              .setMemberId(generation.memberId))
+            );
+            future = client.send(coordinator, request)
                     .compose(new LeaveGroupResponseHandler());
             client.pollNoWakeup();
         }
 
         resetGeneration();
+        return future;
     }
 
     protected boolean isDynamicMember() {
@@ -870,12 +876,18 @@ protected boolean isDynamicMember() {
     private class LeaveGroupResponseHandler extends CoordinatorResponseHandler {
         @Override
         public void handle(LeaveGroupResponse leaveResponse, RequestFuture future) {
-            Errors error = leaveResponse.error();
+            final List members = leaveResponse.memberResponses();
+            if (members.size() > 1) {
+                future.raise(new IllegalStateException("The expected leave group response " +
+                                                           "should only contain no more than one member info, however get " + members));
+            }
+
+            final Errors error = leaveResponse.error();
             if (error == Errors.NONE) {
                 log.debug("LeaveGroup request returned successfully");
                 future.complete(null);
             } else {
-                log.debug("LeaveGroup request failed with error: {}", error.message());
+                log.error("LeaveGroup request failed with error: {}", error.message());
                 future.raise(error);
             }
         }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java
index e6e239c757f06..ac77379ae2bf8 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java
@@ -16,33 +16,64 @@
  */
 package org.apache.kafka.common.requests;
 
+import org.apache.kafka.common.errors.UnsupportedVersionException;
 import org.apache.kafka.common.message.LeaveGroupRequestData;
+import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity;
 import org.apache.kafka.common.message.LeaveGroupResponseData;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.protocol.MessageUtil;
 import org.apache.kafka.common.protocol.types.Struct;
 
 import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.List;
 
 public class LeaveGroupRequest extends AbstractRequest {
 
     public static class Builder extends AbstractRequest.Builder {
+        private final String groupId;
+        private final List members;
 
-        private final LeaveGroupRequestData data;
-
-        public Builder(LeaveGroupRequestData data) {
+        public Builder(String groupId, List members) {
             super(ApiKeys.LEAVE_GROUP);
-            this.data = data;
+            this.groupId = groupId;
+            this.members = members;
+            if (members.isEmpty()) {
+                throw new IllegalArgumentException("leaving members should not be empty");
+            }
         }
 
+        /**
+         * Based on the request version to choose fields.
+         */
         @Override
         public LeaveGroupRequest build(short version) {
+            final LeaveGroupRequestData data;
+            // Starting from version 3, all the leave group request will be in batch.
+            if (version >= 3) {
+                data = new LeaveGroupRequestData()
+                           .setGroupId(groupId)
+                           .setMembers(members);
+            } else {
+                if (members.size() != 1) {
+                    throw new UnsupportedVersionException("Version " + version + " leave group request only " +
+                                                              "supports single member instance than " + members.size() + " members");
+                }
+
+                data = new LeaveGroupRequestData()
+                           .setGroupId(groupId)
+                           .setMemberId(members.get(0).memberId());
+            }
             return new LeaveGroupRequest(data, version);
         }
 
         @Override
         public String toString() {
-            return data.toString();
+            return "(type=LeaveGroupRequest" +
+                       ", groupId=" + groupId +
+                       ", members=" + MessageUtil.deepToString(members.iterator()) +
+                       ")";
         }
     }
     private final LeaveGroupRequestData data;
@@ -64,6 +95,13 @@ public LeaveGroupRequestData data() {
         return data;
     }
 
+    public List members() {
+        // Before version 3, leave group request is still in single mode
+        return version <= 2 ? Collections.singletonList(
+            new MemberIdentity()
+                .setMemberId(data.memberId())) : data.members();
+    }
+
     @Override
     public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
         LeaveGroupResponseData responseData = new LeaveGroupResponseData()
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java
index 6390c0f6d909a..97a583f3d226f 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java
@@ -17,47 +17,117 @@
 package org.apache.kafka.common.requests;
 
 import org.apache.kafka.common.message.LeaveGroupResponseData;
+import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.protocol.types.Struct;
 
 import java.nio.ByteBuffer;
-import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
+/**
+ * Possible error codes.
+ *
+ * Top level errors:
+ * - {@link Errors#COORDINATOR_LOAD_IN_PROGRESS}
+ * - {@link Errors#COORDINATOR_NOT_AVAILABLE}
+ * - {@link Errors#NOT_COORDINATOR}
+ * - {@link Errors#GROUP_AUTHORIZATION_FAILED}
+ *
+ * Member level errors:
+ * - {@link Errors#FENCED_INSTANCE_ID}
+ * - {@link Errors#UNKNOWN_MEMBER_ID}
+ *
+ * If the top level error code is set, normally this indicates that broker early stops the request
+ * handling due to some severe global error, so it is expected to see the member level errors to be empty.
+ * For older version response, we may populate member level error towards top level because older client
+ * couldn't parse member level.
+ */
 public class LeaveGroupResponse extends AbstractResponse {
 
-    private final LeaveGroupResponseData data;
+    public final LeaveGroupResponseData data;
 
     public LeaveGroupResponse(LeaveGroupResponseData data) {
         this.data = data;
     }
 
+    public LeaveGroupResponse(List memberResponses,
+                              Errors topLevelError,
+                              final int throttleTimeMs,
+                              final short version) {
+        if (version <= 2) {
+            // Populate member level error.
+            final short errorCode = getError(topLevelError, memberResponses).code();
+
+            this.data = new LeaveGroupResponseData()
+                            .setErrorCode(errorCode);
+        } else {
+            this.data = new LeaveGroupResponseData()
+                            .setErrorCode(topLevelError.code())
+                            .setMembers(memberResponses);
+        }
+
+        if (version >= 1) {
+            this.data.setThrottleTimeMs(throttleTimeMs);
+        }
+    }
+
     public LeaveGroupResponse(Struct struct) {
         short latestVersion = (short) (LeaveGroupResponseData.SCHEMAS.length - 1);
         this.data = new LeaveGroupResponseData(struct, latestVersion);
     }
+
     public LeaveGroupResponse(Struct struct, short version) {
         this.data = new LeaveGroupResponseData(struct, version);
     }
 
-    public LeaveGroupResponseData data() {
-        return data;
-    }
-
     @Override
     public int throttleTimeMs() {
         return data.throttleTimeMs();
     }
 
+    public List memberResponses() {
+        return data.members();
+    }
+
     public Errors error() {
-        return Errors.forCode(data.errorCode());
+        return getError(Errors.forCode(data.errorCode()), data.members());
+    }
+
+    private static Errors getError(Errors topLevelError, List memberResponses) {
+        if (topLevelError != Errors.NONE) {
+            return topLevelError;
+        } else {
+            for (MemberResponse memberResponse : memberResponses) {
+                Errors memberError = Errors.forCode(memberResponse.errorCode());
+                if (memberError != Errors.NONE) {
+                    return memberError;
+                }
+            }
+            return Errors.NONE;
+        }
     }
 
     @Override
     public Map errorCounts() {
-        return Collections.singletonMap(Errors.forCode(data.errorCode()), 1);
+        Map combinedErrorCounts = new HashMap<>();
+        // Top level error.
+        Errors topLevelError = Errors.forCode(data.errorCode());
+        if (topLevelError != Errors.NONE) {
+            updateErrorCounts(combinedErrorCounts, topLevelError);
+        }
+
+        // Member level error.
+        for (MemberResponse memberResponse : data.members()) {
+            Errors memberError = Errors.forCode(memberResponse.errorCode());
+            if (memberError != Errors.NONE) {
+                updateErrorCounts(combinedErrorCounts, memberError);
+            }
+        }
+        return combinedErrorCounts;
     }
 
     @Override
diff --git a/clients/src/main/resources/common/message/LeaveGroupRequest.json b/clients/src/main/resources/common/message/LeaveGroupRequest.json
index 7c536da19d241..9f98f06501bca 100644
--- a/clients/src/main/resources/common/message/LeaveGroupRequest.json
+++ b/clients/src/main/resources/common/message/LeaveGroupRequest.json
@@ -18,11 +18,20 @@
   "type": "request",
   "name": "LeaveGroupRequest",
   // Version 1 and 2 are the same as version 0.
-  "validVersions": "0-2",
+  // Version 3 defines batch processing scheme with group.instance.id + member.id for identity
+  "validVersions": "0-3",
   "fields": [
     { "name": "GroupId", "type": "string", "versions": "0+", "entityType": "groupId",
       "about": "The ID of the group to leave." },
-    { "name": "MemberId", "type": "string", "versions": "0+",
-      "about": "The member ID to remove from the group." }
+    { "name": "MemberId", "type": "string", "versions": "0-2",
+      "about": "The member ID to remove from the group." },
+    { "name": "Members", "type": "[]MemberIdentity", "versions": "3+",
+      "about": "List of leaving member identities.", "fields": [
+      { "name": "MemberId", "type": "string", "versions": "3+",
+        "about": "The member ID to remove from the group." },
+      { "name": "GroupInstanceId", "type": "string",
+        "versions": "3+", "nullableVersions": "3+", "default": "null",
+        "about": "The group instance ID to remove from the group." }
+    ]}
   ]
 }
diff --git a/clients/src/main/resources/common/message/LeaveGroupResponse.json b/clients/src/main/resources/common/message/LeaveGroupResponse.json
index 0d887cdcb0289..2bbf63df236de 100644
--- a/clients/src/main/resources/common/message/LeaveGroupResponse.json
+++ b/clients/src/main/resources/common/message/LeaveGroupResponse.json
@@ -19,11 +19,22 @@
   "name": "LeaveGroupResponse",
   // Version 1 adds the throttle time.
   // Starting in version 2, on quota violation, brokers send out responses before throttling.
-  "validVersions": "0-2",
+  // Starting in version 3, we will make leave group request into batch mode.
+  "validVersions": "0-3",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "1+", "ignorable": true,
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
     { "name": "ErrorCode", "type": "int16", "versions": "0+",
-      "about": "The error code, or 0 if there was no error." }
+      "about": "The error code, or 0 if there was no error." },
+
+    { "name": "Members", "type": "[]MemberResponse", "versions": "3+",
+      "about": "List of leaving member responses.", "fields": [
+      { "name": "MemberId", "type": "string", "versions": "3+",
+        "about": "The member ID to remove from the group." },
+      { "name": "GroupInstanceId", "type": "string", "versions": "3+", "nullableVersions": "3+",
+        "about": "The group instance ID to remove from the group." },
+      { "name": "ErrorCode", "type": "int16", "versions": "3+",
+        "about": "The error code, or 0 if there was no error." }
+    ]}
   ]
 }
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
index 659ef5f781ae8..e0264b3f35917 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
@@ -18,25 +18,32 @@
 
 import org.apache.kafka.clients.GroupRebalanceConfig;
 import org.apache.kafka.clients.MockClient;
+import org.apache.kafka.clients.NodeApiVersions;
 import org.apache.kafka.clients.consumer.OffsetResetStrategy;
 import org.apache.kafka.common.Node;
 import org.apache.kafka.common.errors.AuthenticationException;
 import org.apache.kafka.common.errors.FencedInstanceIdException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
 import org.apache.kafka.common.errors.WakeupException;
 import org.apache.kafka.common.internals.ClusterResourceListeners;
 import org.apache.kafka.common.message.HeartbeatResponseData;
 import org.apache.kafka.common.message.JoinGroupRequestData;
 import org.apache.kafka.common.message.JoinGroupResponseData;
+import org.apache.kafka.common.message.LeaveGroupResponseData;
+import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse;
 import org.apache.kafka.common.message.SyncGroupResponseData;
 import org.apache.kafka.common.metrics.Metrics;
+import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.requests.AbstractRequest;
+import org.apache.kafka.common.requests.ApiVersionsResponse;
 import org.apache.kafka.common.requests.FindCoordinatorResponse;
 import org.apache.kafka.common.requests.HeartbeatRequest;
 import org.apache.kafka.common.requests.HeartbeatResponse;
 import org.apache.kafka.common.requests.JoinGroupRequest;
 import org.apache.kafka.common.requests.JoinGroupResponse;
 import org.apache.kafka.common.requests.LeaveGroupRequest;
+import org.apache.kafka.common.requests.LeaveGroupResponse;
 import org.apache.kafka.common.requests.SyncGroupRequest;
 import org.apache.kafka.common.requests.SyncGroupResponse;
 import org.apache.kafka.common.utils.LogContext;
@@ -48,6 +55,7 @@
 import org.junit.Test;
 
 import java.nio.ByteBuffer;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -63,6 +71,7 @@
 import static java.util.Collections.emptyMap;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNotSame;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertThrows;
@@ -75,7 +84,6 @@ public class AbstractCoordinatorTest {
     private static final int SESSION_TIMEOUT_MS = 10000;
     private static final int HEARTBEAT_INTERVAL_MS = 3000;
     private static final int RETRY_BACKOFF_MS = 100;
-    private static final int LONG_RETRY_BACKOFF_MS = 10000;
     private static final int REQUEST_TIMEOUT_MS = 40000;
     private static final String GROUP_ID = "dummy-group";
     private static final String METRIC_GROUP_PREFIX = "consumer";
@@ -87,6 +95,10 @@ public class AbstractCoordinatorTest {
     private ConsumerNetworkClient consumerClient;
     private DummyCoordinator coordinator;
 
+    private final String memberId = "memberId";
+    private final String leaderId = "leaderId";
+    private final int defaultGeneration = -1;
+
     private void setupCoordinator() {
         setupCoordinator(RETRY_BACKOFF_MS, REBALANCE_TIMEOUT_MS,
             Optional.empty());
@@ -164,7 +176,7 @@ public void testTimeoutAndRetryJoinGroupIfNeeded() throws Exception {
             assertFalse(firstAttempt.get());
             assertTrue(consumerClient.hasPendingRequests(coordinatorNode));
 
-            mockClient.respond(joinGroupFollowerResponse(1, "memberId", "leaderId", Errors.NONE));
+            mockClient.respond(joinGroupFollowerResponse(1, memberId, leaderId, Errors.NONE));
             mockClient.prepareResponse(syncGroupResponse(Errors.NONE));
 
             Timer secondAttemptTimer = mockTime.timer(REQUEST_TIMEOUT_MS);
@@ -183,10 +195,7 @@ public void testGroupMaxSizeExceptionIsFatal() {
         mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady(mockTime.timer(0));
 
-        final String memberId = "memberId";
-        final int generation = -1;
-
-        mockClient.prepareResponse(joinGroupFollowerResponse(generation, memberId, JoinGroupResponse.UNKNOWN_MEMBER_ID, Errors.GROUP_MAX_SIZE_REACHED));
+        mockClient.prepareResponse(joinGroupFollowerResponse(defaultGeneration, memberId, JoinGroupResponse.UNKNOWN_MEMBER_ID, Errors.GROUP_MAX_SIZE_REACHED));
 
         RequestFuture future = coordinator.sendJoinGroupRequest();
         assertTrue(consumerClient.poll(future, mockTime.timer(REQUEST_TIMEOUT_MS)));
@@ -232,23 +241,14 @@ public void testJoinGroupRequestWithMemberIdRequired() {
         mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady(mockTime.timer(0));
 
-        final String memberId = "memberId";
-        final int generation = -1;
-
-        mockClient.prepareResponse(joinGroupFollowerResponse(generation, memberId, JoinGroupResponse.UNKNOWN_MEMBER_ID, Errors.MEMBER_ID_REQUIRED));
+        mockClient.prepareResponse(joinGroupFollowerResponse(defaultGeneration, memberId, JoinGroupResponse.UNKNOWN_MEMBER_ID, Errors.MEMBER_ID_REQUIRED));
 
-        mockClient.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                if (!(body instanceof JoinGroupRequest)) {
-                    return false;
-                }
-                JoinGroupRequest joinGroupRequest = (JoinGroupRequest) body;
-                if (!joinGroupRequest.data().memberId().equals(memberId)) {
-                    return false;
-                }
-                return true;
+        mockClient.prepareResponse(body -> {
+            if (!(body instanceof JoinGroupRequest)) {
+                return false;
             }
+            JoinGroupRequest joinGroupRequest = (JoinGroupRequest) body;
+            return joinGroupRequest.data().memberId().equals(memberId);
         }, joinGroupResponse(Errors.UNKNOWN_MEMBER_ID));
 
         RequestFuture future = coordinator.sendJoinGroupRequest();
@@ -256,7 +256,7 @@ public boolean matches(AbstractRequest body) {
         assertEquals(Errors.MEMBER_ID_REQUIRED.message(), future.exception().getMessage());
         assertTrue(coordinator.rejoinNeededOrPending());
         assertTrue(coordinator.hasValidMemberId());
-        assertTrue(coordinator.hasMatchingGenerationId(generation));
+        assertTrue(coordinator.hasMatchingGenerationId(defaultGeneration));
         future = coordinator.sendJoinGroupRequest();
         assertTrue(consumerClient.poll(future, mockTime.timer(REBALANCE_TIMEOUT_MS)));
     }
@@ -267,10 +267,7 @@ public void testJoinGroupRequestWithFencedInstanceIdException() {
         mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady(mockTime.timer(0));
 
-        final String memberId = "memberId";
-        final int generation = -1;
-
-        mockClient.prepareResponse(joinGroupFollowerResponse(generation, memberId, JoinGroupResponse.UNKNOWN_MEMBER_ID, Errors.FENCED_INSTANCE_ID));
+        mockClient.prepareResponse(joinGroupFollowerResponse(defaultGeneration, memberId, JoinGroupResponse.UNKNOWN_MEMBER_ID, Errors.FENCED_INSTANCE_ID));
 
         RequestFuture future = coordinator.sendJoinGroupRequest();
         assertTrue(consumerClient.poll(future, mockTime.timer(REQUEST_TIMEOUT_MS)));
@@ -284,7 +281,6 @@ public void testSyncGroupRequestWithFencedInstanceIdException() {
         setupCoordinator();
         mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
 
-        final String memberId = "memberId";
         final int generation = -1;
 
         mockClient.prepareResponse(joinGroupFollowerResponse(generation, memberId, JoinGroupResponse.UNKNOWN_MEMBER_ID, Errors.NONE));
@@ -298,7 +294,6 @@ public void testHeartbeatRequestWithFencedInstanceIdException() throws Interrupt
         setupCoordinator();
         mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
 
-        final String memberId = "memberId";
         final int generation = -1;
 
         mockClient.prepareResponse(joinGroupFollowerResponse(generation, memberId, JoinGroupResponse.UNKNOWN_MEMBER_ID, Errors.NONE));
@@ -325,17 +320,14 @@ public void testJoinGroupRequestWithGroupInstanceIdNotFound() {
         mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady(mockTime.timer(0));
 
-        final String memberId = "memberId";
-        final int generation = -1;
-
-        mockClient.prepareResponse(joinGroupFollowerResponse(generation, memberId, JoinGroupResponse.UNKNOWN_MEMBER_ID, Errors.UNKNOWN_MEMBER_ID));
+        mockClient.prepareResponse(joinGroupFollowerResponse(defaultGeneration, memberId, JoinGroupResponse.UNKNOWN_MEMBER_ID, Errors.UNKNOWN_MEMBER_ID));
 
         RequestFuture future = coordinator.sendJoinGroupRequest();
 
         assertTrue(consumerClient.poll(future, mockTime.timer(REQUEST_TIMEOUT_MS)));
         assertEquals(Errors.UNKNOWN_MEMBER_ID.message(), future.exception().getMessage());
         assertTrue(coordinator.rejoinNeededOrPending());
-        assertTrue(coordinator.hasMatchingGenerationId(generation));
+        assertTrue(coordinator.hasMatchingGenerationId(defaultGeneration));
     }
 
     @Test
@@ -348,19 +340,16 @@ private void checkLeaveGroupRequestSent(Optional groupInstanceId) {
         setupCoordinator(RETRY_BACKOFF_MS, Integer.MAX_VALUE, groupInstanceId);
 
         mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        mockClient.prepareResponse(joinGroupFollowerResponse(1, "memberId", "leaderId", Errors.NONE));
+        mockClient.prepareResponse(joinGroupFollowerResponse(1, memberId, leaderId, Errors.NONE));
         mockClient.prepareResponse(syncGroupResponse(Errors.NONE));
 
         final RuntimeException e = new RuntimeException();
 
         // raise the error when the coordinator tries to send leave group request.
-        mockClient.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                if (body instanceof LeaveGroupRequest)
-                    throw e;
-                return false;
-            }
+        mockClient.prepareResponse(body -> {
+            if (body instanceof LeaveGroupRequest)
+                throw e;
+            return false;
         }, heartbeatResponse(Errors.UNKNOWN_SERVER_ERROR));
 
         try {
@@ -378,24 +367,106 @@ public boolean matches(AbstractRequest body) {
         }
     }
 
+    @Test
+    public void testHandleNormalLeaveGroupResponse() {
+        MemberResponse memberResponse = new MemberResponse()
+                                            .setMemberId(memberId)
+                                            .setErrorCode(Errors.NONE.code());
+        LeaveGroupResponse response =
+            leaveGroupResponse(Collections.singletonList(memberResponse));
+        RequestFuture leaveGroupFuture = setupLeaveGroup(response);
+        assertNotNull(leaveGroupFuture);
+        assertTrue(leaveGroupFuture.succeeded());
+    }
+
+    @Test
+    public void testHandleMultipleMembersLeaveGroupResponse() {
+        MemberResponse memberResponse = new MemberResponse()
+                                            .setMemberId(memberId)
+                                            .setErrorCode(Errors.NONE.code());
+        LeaveGroupResponse response =
+            leaveGroupResponse(Arrays.asList(memberResponse, memberResponse));
+        RequestFuture leaveGroupFuture = setupLeaveGroup(response);
+        assertNotNull(leaveGroupFuture);
+        assertTrue(leaveGroupFuture.exception() instanceof IllegalStateException);
+    }
+
+    @Test
+    public void testHandleLeaveGroupResponseWithEmptyMemberResponse() {
+        LeaveGroupResponse response =
+            leaveGroupResponse(Collections.emptyList());
+        RequestFuture leaveGroupFuture = setupLeaveGroup(response);
+        assertNotNull(leaveGroupFuture);
+        assertTrue(leaveGroupFuture.succeeded());
+    }
+
+    @Test
+    public void testHandleLeaveGroupResponseWithException() {
+        MemberResponse memberResponse = new MemberResponse()
+                                            .setMemberId(memberId)
+                                            .setErrorCode(Errors.UNKNOWN_MEMBER_ID.code());
+        LeaveGroupResponse response =
+            leaveGroupResponse(Collections.singletonList(memberResponse));
+        RequestFuture leaveGroupFuture = setupLeaveGroup(response);
+        assertNotNull(leaveGroupFuture);
+        assertTrue(leaveGroupFuture.exception() instanceof UnknownMemberIdException);
+    }
+
+    @Test
+    public void testHandleSingleLeaveGroupRequest() {
+        setupCoordinator(RETRY_BACKOFF_MS, Integer.MAX_VALUE, Optional.empty());
+        mockClient.setNodeApiVersions(NodeApiVersions.create(Collections.singletonList(
+            new ApiVersionsResponse.ApiVersion(ApiKeys.LEAVE_GROUP, (short) 2, (short) 2))));
+
+        LeaveGroupResponse expectedResponse = leaveGroupResponse(Collections.singletonList(
+            new MemberResponse()
+                .setErrorCode(Errors.NONE.code())
+                .setMemberId(memberId)));
+        mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
+        mockClient.prepareResponse(joinGroupFollowerResponse(1, memberId, leaderId, Errors.NONE));
+        mockClient.prepareResponse(syncGroupResponse(Errors.NONE));
+        mockClient.prepareResponse(body -> {
+            if (body instanceof LeaveGroupRequest) {
+                LeaveGroupRequest request = (LeaveGroupRequest) body;
+                return request.data().memberId().equals(memberId)
+                    && request.data().members().isEmpty();
+            } else {
+                return false;
+            }
+        }, expectedResponse);
+
+        coordinator.ensureActiveGroup();
+        RequestFuture leaveGroupFuture = coordinator.maybeLeaveGroup("test single leave group");
+        assertTrue(leaveGroupFuture.succeeded());
+    }
+
+    private RequestFuture setupLeaveGroup(LeaveGroupResponse leaveGroupResponse) {
+        setupCoordinator(RETRY_BACKOFF_MS, Integer.MAX_VALUE, Optional.empty());
+
+        mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
+        mockClient.prepareResponse(joinGroupFollowerResponse(1, memberId, leaderId, Errors.NONE));
+        mockClient.prepareResponse(syncGroupResponse(Errors.NONE));
+        mockClient.prepareResponse(leaveGroupResponse);
+
+        coordinator.ensureActiveGroup();
+        return coordinator.maybeLeaveGroup("test maybe leave group");
+    }
+
     @Test
     public void testUncaughtExceptionInHeartbeatThread() throws Exception {
         setupCoordinator();
 
         mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        mockClient.prepareResponse(joinGroupFollowerResponse(1, "memberId", "leaderId", Errors.NONE));
+        mockClient.prepareResponse(joinGroupFollowerResponse(1, memberId, leaderId, Errors.NONE));
         mockClient.prepareResponse(syncGroupResponse(Errors.NONE));
 
         final RuntimeException e = new RuntimeException();
 
         // raise the error when the background thread tries to send a heartbeat
-        mockClient.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                if (body instanceof HeartbeatRequest)
-                    throw e;
-                return false;
-            }
+        mockClient.prepareResponse(body -> {
+            if (body instanceof HeartbeatRequest)
+                throw e;
+            return false;
         }, heartbeatResponse(Errors.UNKNOWN_SERVER_ERROR));
 
         try {
@@ -414,21 +485,19 @@ public boolean matches(AbstractRequest body) {
 
     @Test
     public void testPollHeartbeatAwakesHeartbeatThread() throws Exception {
-        setupCoordinator(LONG_RETRY_BACKOFF_MS);
+        final int longRetryBackoffMs = 10000;
+        setupCoordinator(longRetryBackoffMs);
 
         mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        mockClient.prepareResponse(joinGroupFollowerResponse(1, "memberId", "leaderId", Errors.NONE));
+        mockClient.prepareResponse(joinGroupFollowerResponse(1, memberId, leaderId, Errors.NONE));
         mockClient.prepareResponse(syncGroupResponse(Errors.NONE));
 
         coordinator.ensureActiveGroup();
 
         final CountDownLatch heartbeatDone = new CountDownLatch(1);
-        mockClient.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                heartbeatDone.countDown();
-                return body instanceof HeartbeatRequest;
-            }
+        mockClient.prepareResponse(body -> {
+            heartbeatDone.countDown();
+            return body instanceof HeartbeatRequest;
         }, heartbeatResponse(Errors.NONE));
 
         mockTime.sleep(HEARTBEAT_INTERVAL_MS);
@@ -473,7 +542,7 @@ public boolean matches(AbstractRequest body) {
                     throw new WakeupException();
                 return isJoinGroupRequest;
             }
-        }, joinGroupFollowerResponse(1, "memberId", "leaderId", Errors.NONE));
+        }, joinGroupFollowerResponse(1, memberId, leaderId, Errors.NONE));
         mockClient.prepareResponse(syncGroupResponse(Errors.NONE));
         AtomicBoolean heartbeatReceived = prepareFirstHeartbeat();
 
@@ -511,7 +580,7 @@ public boolean matches(AbstractRequest body) {
                     throw new WakeupException();
                 return isJoinGroupRequest;
             }
-        }, joinGroupFollowerResponse(1, "memberId", "leaderId", Errors.NONE));
+        }, joinGroupFollowerResponse(1, memberId, leaderId, Errors.NONE));
         mockClient.prepareResponse(syncGroupResponse(Errors.NONE));
         AtomicBoolean heartbeatReceived = prepareFirstHeartbeat();
 
@@ -540,16 +609,13 @@ public void testWakeupAfterJoinGroupReceived() throws Exception {
         setupCoordinator();
 
         mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        mockClient.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                boolean isJoinGroupRequest = body instanceof JoinGroupRequest;
-                if (isJoinGroupRequest)
-                    // wakeup after the request returns
-                    consumerClient.wakeup();
-                return isJoinGroupRequest;
-            }
-        }, joinGroupFollowerResponse(1, "memberId", "leaderId", Errors.NONE));
+        mockClient.prepareResponse(body -> {
+            boolean isJoinGroupRequest = body instanceof JoinGroupRequest;
+            if (isJoinGroupRequest)
+                // wakeup after the request returns
+                consumerClient.wakeup();
+            return isJoinGroupRequest;
+        }, joinGroupFollowerResponse(1, memberId, leaderId, Errors.NONE));
         mockClient.prepareResponse(syncGroupResponse(Errors.NONE));
         AtomicBoolean heartbeatReceived = prepareFirstHeartbeat();
 
@@ -576,16 +642,13 @@ public void testWakeupAfterJoinGroupReceivedExternalCompletion() throws Exceptio
         setupCoordinator();
 
         mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        mockClient.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                boolean isJoinGroupRequest = body instanceof JoinGroupRequest;
-                if (isJoinGroupRequest)
-                    // wakeup after the request returns
-                    consumerClient.wakeup();
-                return isJoinGroupRequest;
-            }
-        }, joinGroupFollowerResponse(1, "memberId", "leaderId", Errors.NONE));
+        mockClient.prepareResponse(body -> {
+            boolean isJoinGroupRequest = body instanceof JoinGroupRequest;
+            if (isJoinGroupRequest)
+                // wakeup after the request returns
+                consumerClient.wakeup();
+            return isJoinGroupRequest;
+        }, joinGroupFollowerResponse(1, memberId, leaderId, Errors.NONE));
         mockClient.prepareResponse(syncGroupResponse(Errors.NONE));
         AtomicBoolean heartbeatReceived = prepareFirstHeartbeat();
 
@@ -614,7 +677,7 @@ public void testWakeupAfterSyncGroupSent() throws Exception {
         setupCoordinator();
 
         mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        mockClient.prepareResponse(joinGroupFollowerResponse(1, "memberId", "leaderId", Errors.NONE));
+        mockClient.prepareResponse(joinGroupFollowerResponse(1, memberId, leaderId, Errors.NONE));
         mockClient.prepareResponse(new MockClient.RequestMatcher() {
             private int invocations = 0;
             @Override
@@ -652,7 +715,7 @@ public void testWakeupAfterSyncGroupSentExternalCompletion() throws Exception {
         setupCoordinator();
 
         mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        mockClient.prepareResponse(joinGroupFollowerResponse(1, "memberId", "leaderId", Errors.NONE));
+        mockClient.prepareResponse(joinGroupFollowerResponse(1, memberId, leaderId, Errors.NONE));
         mockClient.prepareResponse(new MockClient.RequestMatcher() {
             private int invocations = 0;
             @Override
@@ -692,16 +755,13 @@ public void testWakeupAfterSyncGroupReceived() throws Exception {
         setupCoordinator();
 
         mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        mockClient.prepareResponse(joinGroupFollowerResponse(1, "memberId", "leaderId", Errors.NONE));
-        mockClient.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                boolean isSyncGroupRequest = body instanceof SyncGroupRequest;
-                if (isSyncGroupRequest)
-                    // wakeup after the request returns
-                    consumerClient.wakeup();
-                return isSyncGroupRequest;
-            }
+        mockClient.prepareResponse(joinGroupFollowerResponse(1, memberId, leaderId, Errors.NONE));
+        mockClient.prepareResponse(body -> {
+            boolean isSyncGroupRequest = body instanceof SyncGroupRequest;
+            if (isSyncGroupRequest)
+                // wakeup after the request returns
+                consumerClient.wakeup();
+            return isSyncGroupRequest;
         }, syncGroupResponse(Errors.NONE));
         AtomicBoolean heartbeatReceived = prepareFirstHeartbeat();
 
@@ -728,16 +788,13 @@ public void testWakeupAfterSyncGroupReceivedExternalCompletion() throws Exceptio
         setupCoordinator();
 
         mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        mockClient.prepareResponse(joinGroupFollowerResponse(1, "memberId", "leaderId", Errors.NONE));
-        mockClient.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                boolean isSyncGroupRequest = body instanceof SyncGroupRequest;
-                if (isSyncGroupRequest)
-                    // wakeup after the request returns
-                    consumerClient.wakeup();
-                return isSyncGroupRequest;
-            }
+        mockClient.prepareResponse(joinGroupFollowerResponse(1, memberId, leaderId, Errors.NONE));
+        mockClient.prepareResponse(body -> {
+            boolean isSyncGroupRequest = body instanceof SyncGroupRequest;
+            if (isSyncGroupRequest)
+                // wakeup after the request returns
+                consumerClient.wakeup();
+            return isSyncGroupRequest;
         }, syncGroupResponse(Errors.NONE));
         AtomicBoolean heartbeatReceived = prepareFirstHeartbeat();
 
@@ -765,7 +822,7 @@ public void testWakeupInOnJoinComplete() throws Exception {
 
         coordinator.wakeupOnJoinComplete = true;
         mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        mockClient.prepareResponse(joinGroupFollowerResponse(1, "memberId", "leaderId", Errors.NONE));
+        mockClient.prepareResponse(joinGroupFollowerResponse(1, memberId, leaderId, Errors.NONE));
         mockClient.prepareResponse(syncGroupResponse(Errors.NONE));
         AtomicBoolean heartbeatReceived = prepareFirstHeartbeat();
 
@@ -806,14 +863,11 @@ public void testAuthenticationErrorInEnsureCoordinatorReady() {
 
     private AtomicBoolean prepareFirstHeartbeat() {
         final AtomicBoolean heartbeatReceived = new AtomicBoolean(false);
-        mockClient.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                boolean isHeartbeatRequest = body instanceof HeartbeatRequest;
-                if (isHeartbeatRequest)
-                    heartbeatReceived.set(true);
-                return isHeartbeatRequest;
-            }
+        mockClient.prepareResponse(body -> {
+            boolean isHeartbeatRequest = body instanceof HeartbeatRequest;
+            if (isHeartbeatRequest)
+                heartbeatReceived.set(true);
+            return isHeartbeatRequest;
         }, heartbeatResponse(Errors.UNKNOWN_SERVER_ERROR));
         return heartbeatReceived;
     }
@@ -861,6 +915,12 @@ private SyncGroupResponse syncGroupResponse(Errors error) {
         );
     }
 
+    private LeaveGroupResponse leaveGroupResponse(List members) {
+        return new LeaveGroupResponse(new LeaveGroupResponseData()
+                .setErrorCode(Errors.NONE.code())
+                .setMembers(members));
+    }
+
     public static class DummyCoordinator extends AbstractCoordinator {
 
         private int onJoinPrepareInvokes = 0;
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index a81e73e022dfd..11273cb6f1a17 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -44,6 +44,7 @@
 import org.apache.kafka.common.message.HeartbeatResponseData;
 import org.apache.kafka.common.message.JoinGroupRequestData;
 import org.apache.kafka.common.message.JoinGroupResponseData;
+import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity;
 import org.apache.kafka.common.message.LeaveGroupResponseData;
 import org.apache.kafka.common.message.OffsetCommitRequestData;
 import org.apache.kafka.common.message.OffsetCommitResponseData;
@@ -51,7 +52,6 @@
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.record.RecordBatch;
-import org.apache.kafka.common.requests.AbstractRequest;
 import org.apache.kafka.common.requests.FindCoordinatorResponse;
 import org.apache.kafka.common.requests.HeartbeatResponse;
 import org.apache.kafka.common.requests.JoinGroupRequest;
@@ -473,14 +473,11 @@ public void testNormalJoinGroupLeader() {
         partitionAssignor.prepare(singletonMap(consumerId, assigned));
 
         client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE));
-        client.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                SyncGroupRequest sync = (SyncGroupRequest) body;
-                return sync.data.memberId().equals(consumerId) &&
-                        sync.data.generationId() == 1 &&
-                        sync.groupAssignments().containsKey(consumerId);
-            }
+        client.prepareResponse(body -> {
+            SyncGroupRequest sync = (SyncGroupRequest) body;
+            return sync.data.memberId().equals(consumerId) &&
+                    sync.data.generationId() == 1 &&
+                    sync.groupAssignments().containsKey(consumerId);
         }, syncGroupResponse(assigned, Errors.NONE));
         coordinator.poll(time.timer(Long.MAX_VALUE));
 
@@ -514,28 +511,22 @@ public void testOutdatedCoordinatorAssignment() {
         client.prepareResponse(
                 joinGroupLeaderResponse(
                     1, consumerId, singletonMap(consumerId, oldSubscription), Errors.NONE));
-        client.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                SyncGroupRequest sync = (SyncGroupRequest) body;
-                return sync.data.memberId().equals(consumerId) &&
-                        sync.data.generationId() == 1 &&
-                        sync.groupAssignments().containsKey(consumerId);
-            }
+        client.prepareResponse(body -> {
+            SyncGroupRequest sync = (SyncGroupRequest) body;
+            return sync.data.memberId().equals(consumerId) &&
+                    sync.data.generationId() == 1 &&
+                    sync.groupAssignments().containsKey(consumerId);
         }, syncGroupResponse(oldAssignment, Errors.NONE));
 
         // Second correct assignment for subscription
         client.prepareResponse(
                 joinGroupLeaderResponse(
                     1, consumerId, singletonMap(consumerId, newSubscription), Errors.NONE));
-        client.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                SyncGroupRequest sync = (SyncGroupRequest) body;
-                return sync.data.memberId().equals(consumerId) &&
-                        sync.data.generationId() == 1 &&
-                        sync.groupAssignments().containsKey(consumerId);
-            }
+        client.prepareResponse(body -> {
+            SyncGroupRequest sync = (SyncGroupRequest) body;
+            return sync.data.memberId().equals(consumerId) &&
+                    sync.data.generationId() == 1 &&
+                    sync.groupAssignments().containsKey(consumerId);
         }, syncGroupResponse(newAssignment, Errors.NONE));
 
         // Poll once so that the join group future gets created and complete
@@ -587,14 +578,11 @@ public void testPatternJoinGroupLeader() {
         partitionAssignor.prepare(singletonMap(consumerId, assigned));
 
         client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE));
-        client.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                SyncGroupRequest sync = (SyncGroupRequest) body;
-                return sync.data.memberId().equals(consumerId) &&
-                        sync.data.generationId() == 1 &&
-                        sync.groupAssignments().containsKey(consumerId);
-            }
+        client.prepareResponse(body -> {
+            SyncGroupRequest sync = (SyncGroupRequest) body;
+            return sync.data.memberId().equals(consumerId) &&
+                    sync.data.generationId() == 1 &&
+                    sync.groupAssignments().containsKey(consumerId);
         }, syncGroupResponse(assigned, Errors.NONE));
         // expect client to force updating the metadata, if yes gives it both topics
         client.prepareMetadataUpdate(metadataResponse);
@@ -634,15 +622,12 @@ public void testMetadataRefreshDuringRebalance() {
         final List updatedSubscription = Arrays.asList(topic1, topic2);
 
         client.prepareResponse(joinGroupLeaderResponse(1, consumerId, initialSubscription, Errors.NONE));
-        client.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                final Map updatedPartitions = new HashMap<>();
-                for (String topic : updatedSubscription)
-                    updatedPartitions.put(topic, 1);
-                client.updateMetadata(TestUtils.metadataUpdateWith(1, updatedPartitions));
-                return true;
-            }
+        client.prepareResponse(body -> {
+            final Map updatedPartitions = new HashMap<>();
+            for (String topic : updatedSubscription)
+                updatedPartitions.put(topic, 1);
+            client.updateMetadata(TestUtils.metadataUpdateWith(1, updatedPartitions));
+            return true;
         }, syncGroupResponse(oldAssigned, Errors.NONE));
         coordinator.poll(time.timer(Long.MAX_VALUE));
 
@@ -661,20 +646,17 @@ public boolean matches(AbstractRequest body) {
         partitionAssignor.prepare(singletonMap(consumerId, newAssigned));
 
         // we expect to see a second rebalance with the new-found topics
-        client.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                JoinGroupRequest join = (JoinGroupRequest) body;
-                Iterator protocolIterator =
-                        join.data().protocols().iterator();
-                assertTrue(protocolIterator.hasNext());
-                JoinGroupRequestData.JoinGroupRequestProtocol protocolMetadata = protocolIterator.next();
-
-                ByteBuffer metadata = ByteBuffer.wrap(protocolMetadata.metadata());
-                ConsumerPartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(metadata);
-                metadata.rewind();
-                return subscription.topics().containsAll(updatedSubscription);
-            }
+        client.prepareResponse(body -> {
+            JoinGroupRequest join = (JoinGroupRequest) body;
+            Iterator protocolIterator =
+                    join.data().protocols().iterator();
+            assertTrue(protocolIterator.hasNext());
+            JoinGroupRequestData.JoinGroupRequestProtocol protocolMetadata = protocolIterator.next();
+
+            ByteBuffer metadata = ByteBuffer.wrap(protocolMetadata.metadata());
+            ConsumerPartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(metadata);
+            metadata.rewind();
+            return subscription.topics().containsAll(updatedSubscription);
         }, joinGroupLeaderResponse(2, consumerId, updatedSubscriptions, Errors.NONE));
         client.prepareResponse(syncGroupResponse(newAssigned, Errors.NONE));
 
@@ -707,14 +689,11 @@ public void testForceMetadataRefreshForPatternSubscriptionDuringRebalance() {
         client.prepareMetadataUpdate(metadataResponse);
 
         client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE));
-        client.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                SyncGroupRequest sync = (SyncGroupRequest) body;
-                return sync.data.memberId().equals(consumerId) &&
-                    sync.data.generationId() == 1 &&
-                    sync.groupAssignments().isEmpty();
-            }
+        client.prepareResponse(body -> {
+            SyncGroupRequest sync = (SyncGroupRequest) body;
+            return sync.data.memberId().equals(consumerId) &&
+                sync.data.generationId() == 1 &&
+                sync.groupAssignments().isEmpty();
         }, syncGroupResponse(singletonList(t1p), Errors.NONE));
 
         partitionAssignor.prepare(singletonMap(consumerId, singletonList(t1p)));
@@ -786,14 +765,11 @@ public void testNormalJoinGroupFollower() {
 
         // normal join group
         client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE));
-        client.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                SyncGroupRequest sync = (SyncGroupRequest) body;
-                return sync.data.memberId().equals(consumerId) &&
-                        sync.data.generationId() == 1 &&
-                        sync.groupAssignments().isEmpty();
-            }
+        client.prepareResponse(body -> {
+            SyncGroupRequest sync = (SyncGroupRequest) body;
+            return sync.data.memberId().equals(consumerId) &&
+                    sync.data.generationId() == 1 &&
+                    sync.groupAssignments().isEmpty();
         }, syncGroupResponse(assigned, Errors.NONE));
 
         coordinator.joinGroupIfNeeded(time.timer(Long.MAX_VALUE));
@@ -854,14 +830,11 @@ public void testPatternJoinGroupFollower() {
 
         // normal join group
         client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE));
-        client.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                SyncGroupRequest sync = (SyncGroupRequest) body;
-                return sync.data.memberId().equals(consumerId) &&
-                        sync.data.generationId() == 1 &&
-                        sync.groupAssignments().isEmpty();
-            }
+        client.prepareResponse(body -> {
+            SyncGroupRequest sync = (SyncGroupRequest) body;
+            return sync.data.memberId().equals(consumerId) &&
+                    sync.data.generationId() == 1 &&
+                    sync.groupAssignments().isEmpty();
         }, syncGroupResponse(assigned, Errors.NONE));
         // expect client to force updating the metadata, if yes gives it both topics
         client.prepareMetadataUpdate(metadataResponse);
@@ -885,15 +858,12 @@ public void testLeaveGroupOnClose() {
         joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p));
 
         final AtomicBoolean received = new AtomicBoolean(false);
-        client.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                received.set(true);
-                LeaveGroupRequest leaveRequest = (LeaveGroupRequest) body;
-                return leaveRequest.data().memberId().equals(consumerId) &&
-                        leaveRequest.data().groupId().equals(groupId);
-            }
-        }, new LeaveGroupResponse(new LeaveGroupResponseData().setErrorCode(Errors.NONE.code())));
+        client.prepareResponse(body -> {
+            received.set(true);
+            LeaveGroupRequest leaveRequest = (LeaveGroupRequest) body;
+            return validateLeaveGroup(groupId, consumerId, leaveRequest);
+        }, new LeaveGroupResponse(
+            new LeaveGroupResponseData().setErrorCode(Errors.NONE.code())));
         coordinator.close(time.timer(0));
         assertTrue(received.get());
     }
@@ -906,14 +876,10 @@ public void testMaybeLeaveGroup() {
         joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p));
 
         final AtomicBoolean received = new AtomicBoolean(false);
-        client.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                received.set(true);
-                LeaveGroupRequest leaveRequest = (LeaveGroupRequest) body;
-                return leaveRequest.data().memberId().equals(consumerId) &&
-                        leaveRequest.data().groupId().equals(groupId);
-            }
+        client.prepareResponse(body -> {
+            received.set(true);
+            LeaveGroupRequest leaveRequest = (LeaveGroupRequest) body;
+            return validateLeaveGroup(groupId, consumerId, leaveRequest);
         }, new LeaveGroupResponse(new LeaveGroupResponseData().setErrorCode(Errors.NONE.code())));
         coordinator.maybeLeaveGroup("test maybe leave group");
         assertTrue(received.get());
@@ -922,6 +888,15 @@ public boolean matches(AbstractRequest body) {
         assertNull(generation);
     }
 
+    private boolean validateLeaveGroup(String groupId,
+                                       String consumerId,
+                                       LeaveGroupRequest leaveRequest) {
+        List members = leaveRequest.data().members();
+        return leaveRequest.data().groupId().equals(groupId) &&
+                   members.size() == 1 &&
+                   members.get(0).memberId().equals(consumerId);
+    }
+
     /**
      * This test checks if a consumer that has a valid member ID but an invalid generation
      * ({@link org.apache.kafka.clients.consumer.internals.AbstractCoordinator.Generation#NO_GENERATION})
@@ -944,13 +919,10 @@ public void testPendingMemberShouldLeaveGroup() {
         coordinator.joinGroupIfNeeded(time.timer(0));
 
         final AtomicBoolean received = new AtomicBoolean(false);
-        client.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                received.set(true);
-                LeaveGroupRequest leaveRequest = (LeaveGroupRequest) body;
-                return leaveRequest.data().memberId().equals(consumerId);
-            }
+        client.prepareResponse(body -> {
+            received.set(true);
+            LeaveGroupRequest leaveRequest = (LeaveGroupRequest) body;
+            return validateLeaveGroup(groupId, consumerId, leaveRequest);
         }, new LeaveGroupResponse(new LeaveGroupResponseData().setErrorCode(Errors.NONE.code())));
 
         coordinator.maybeLeaveGroup("pending member leaves");
@@ -986,12 +958,9 @@ public void testUnknownMemberIdOnSyncGroup() {
         client.prepareResponse(syncGroupResponse(Collections.emptyList(), Errors.UNKNOWN_MEMBER_ID));
 
         // now we should see a new join with the empty UNKNOWN_MEMBER_ID
-        client.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                JoinGroupRequest joinRequest = (JoinGroupRequest) body;
-                return joinRequest.data().memberId().equals(JoinGroupRequest.UNKNOWN_MEMBER_ID);
-            }
+        client.prepareResponse(body -> {
+            JoinGroupRequest joinRequest = (JoinGroupRequest) body;
+            return joinRequest.data().memberId().equals(JoinGroupRequest.UNKNOWN_MEMBER_ID);
         }, joinGroupFollowerResponse(2, consumerId, "leader", Errors.NONE));
         client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE));
 
@@ -1038,12 +1007,9 @@ public void testIllegalGenerationOnSyncGroup() {
         client.prepareResponse(syncGroupResponse(Collections.emptyList(), Errors.ILLEGAL_GENERATION));
 
         // then let the full join/sync finish successfully
-        client.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                JoinGroupRequest joinRequest = (JoinGroupRequest) body;
-                return joinRequest.data().memberId().equals(JoinGroupRequest.UNKNOWN_MEMBER_ID);
-            }
+        client.prepareResponse(body -> {
+            JoinGroupRequest joinRequest = (JoinGroupRequest) body;
+            return joinRequest.data().memberId().equals(JoinGroupRequest.UNKNOWN_MEMBER_ID);
         }, joinGroupFollowerResponse(2, consumerId, "leader", Errors.NONE));
         client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE));
 
@@ -1106,22 +1072,19 @@ public void testUpdateMetadataDuringRebalance() {
         partitionAssignor.prepare(singletonMap(consumerId, Arrays.asList(tp1)));
 
         client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE));
-        client.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                SyncGroupRequest sync = (SyncGroupRequest) body;
-                if (sync.data.memberId().equals(consumerId) &&
-                        sync.data.generationId() == 1 &&
-                        sync.groupAssignments().containsKey(consumerId)) {
-                    // trigger the metadata update including both topics after the sync group request has been sent
-                    Map topicPartitionCounts = new HashMap<>();
-                    topicPartitionCounts.put(topic1, 1);
-                    topicPartitionCounts.put(topic2, 1);
-                    client.updateMetadata(TestUtils.metadataUpdateWith(1, topicPartitionCounts));
-                    return true;
-                }
-                return false;
+        client.prepareResponse(body -> {
+            SyncGroupRequest sync = (SyncGroupRequest) body;
+            if (sync.data.memberId().equals(consumerId) &&
+                    sync.data.generationId() == 1 &&
+                    sync.groupAssignments().containsKey(consumerId)) {
+                // trigger the metadata update including both topics after the sync group request has been sent
+                Map topicPartitionCounts = new HashMap<>();
+                topicPartitionCounts.put(topic1, 1);
+                topicPartitionCounts.put(topic2, 1);
+                client.updateMetadata(TestUtils.metadataUpdateWith(1, topicPartitionCounts));
+                return true;
             }
+            return false;
         }, syncGroupResponse(Collections.singletonList(tp1), Errors.NONE));
         coordinator.poll(time.timer(Long.MAX_VALUE));
 
@@ -1581,13 +1544,10 @@ public void testCommitAfterLeaveGroup() {
         subscriptions.assignFromUser(singleton(t1p));
 
         // the client should not reuse generation/memberId from auto-subscribed generation
-        client.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                OffsetCommitRequest commitRequest = (OffsetCommitRequest) body;
-                return commitRequest.data().memberId().equals(OffsetCommitRequest.DEFAULT_MEMBER_ID) &&
-                        commitRequest.data().generationId() == OffsetCommitRequest.DEFAULT_GENERATION_ID;
-            }
+        client.prepareResponse(body -> {
+            OffsetCommitRequest commitRequest = (OffsetCommitRequest) body;
+            return commitRequest.data().memberId().equals(OffsetCommitRequest.DEFAULT_MEMBER_ID) &&
+                    commitRequest.data().generationId() == OffsetCommitRequest.DEFAULT_GENERATION_ID;
         }, offsetCommitResponse(singletonMap(t1p, Errors.NONE)));
 
         AtomicBoolean success = new AtomicBoolean(false);
@@ -2302,21 +2262,15 @@ public void run() {
     private void gracefulCloseTest(ConsumerCoordinator coordinator, boolean shouldLeaveGroup) throws Exception {
         final AtomicBoolean commitRequested = new AtomicBoolean();
         final AtomicBoolean leaveGroupRequested = new AtomicBoolean();
-        client.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                commitRequested.set(true);
-                OffsetCommitRequest commitRequest = (OffsetCommitRequest) body;
-                return commitRequest.data().groupId().equals(groupId);
-            }
+        client.prepareResponse(body -> {
+            commitRequested.set(true);
+            OffsetCommitRequest commitRequest = (OffsetCommitRequest) body;
+            return commitRequest.data().groupId().equals(groupId);
         }, new OffsetCommitResponse(new OffsetCommitResponseData()));
-        client.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                leaveGroupRequested.set(true);
-                LeaveGroupRequest leaveRequest = (LeaveGroupRequest) body;
-                return leaveRequest.data().groupId().equals(groupId);
-            }
+        client.prepareResponse(body -> {
+            leaveGroupRequested.set(true);
+            LeaveGroupRequest leaveRequest = (LeaveGroupRequest) body;
+            return leaveRequest.data().groupId().equals(groupId);
         }, new LeaveGroupResponse(new LeaveGroupResponseData()
                 .setErrorCode(Errors.NONE.code())));
 
@@ -2494,37 +2448,31 @@ private void respondToOffsetCommitRequest(final Map expect
     }
 
     private MockClient.RequestMatcher offsetCommitRequestMatcher(final Map expectedOffsets) {
-        return new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                OffsetCommitRequest req = (OffsetCommitRequest) body;
-                Map offsets = req.offsets();
-                if (offsets.size() != expectedOffsets.size())
-                    return false;
+        return body -> {
+            OffsetCommitRequest req = (OffsetCommitRequest) body;
+            Map offsets = req.offsets();
+            if (offsets.size() != expectedOffsets.size())
+                return false;
 
-                for (Map.Entry expectedOffset : expectedOffsets.entrySet()) {
-                    if (!offsets.containsKey(expectedOffset.getKey())) {
+            for (Map.Entry expectedOffset : expectedOffsets.entrySet()) {
+                if (!offsets.containsKey(expectedOffset.getKey())) {
+                    return false;
+                } else {
+                    Long actualOffset = offsets.get(expectedOffset.getKey());
+                    if (!actualOffset.equals(expectedOffset.getValue())) {
                         return false;
-                    } else {
-                        Long actualOffset = offsets.get(expectedOffset.getKey());
-                        if (!actualOffset.equals(expectedOffset.getValue())) {
-                            return false;
-                        }
                     }
                 }
-                return true;
             }
+            return true;
         };
     }
 
     private OffsetCommitCallback callback(final Map expectedOffsets,
                                           final AtomicBoolean success) {
-        return new OffsetCommitCallback() {
-            @Override
-            public void onComplete(Map offsets, Exception exception) {
-                if (expectedOffsets.equals(offsets) && exception == null)
-                    success.set(true);
-            }
+        return (offsets, exception) -> {
+            if (expectedOffsets.equals(offsets) && exception == null)
+                success.set(true);
         };
     }
 
diff --git a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
index 5213ec2a8f736..58c650a011c80 100644
--- a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
@@ -21,8 +21,10 @@
 import org.apache.kafka.common.message.AddPartitionsToTxnRequestData.AddPartitionsToTxnTopic;
 import org.apache.kafka.common.message.AddPartitionsToTxnRequestData.AddPartitionsToTxnTopicCollection;
 import org.apache.kafka.common.message.JoinGroupResponseData.JoinGroupResponseMember;
+import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.ByteBufferAccessor;
+import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.protocol.Message;
 import org.apache.kafka.common.protocol.types.ArrayOf;
 import org.apache.kafka.common.protocol.types.BoundField;
@@ -48,6 +50,10 @@
 import static org.junit.Assert.fail;
 
 public final class MessageTest {
+
+    private final String memberId = "memberId";
+    private final String instanceId = "instanceId";
+
     @Rule
     final public Timeout globalTimeout = Timeout.millis(120000);
 
@@ -115,7 +121,7 @@ public void testMetadataVersions() throws Exception {
     public void testHeartbeatVersions() throws Exception {
         Supplier newRequest = () -> new HeartbeatRequestData()
                 .setGroupId("groupId")
-                .setMemberId("memberId")
+                .setMemberId(memberId)
                 .setGenerationId(15);
         testAllMessageRoundTrips(newRequest.get());
         testAllMessageRoundTrips(newRequest.get().setGroupInstanceId(null));
@@ -126,7 +132,7 @@ public void testHeartbeatVersions() throws Exception {
     public void testJoinGroupRequestVersions() throws Exception {
         Supplier newRequest = () -> new JoinGroupRequestData()
                 .setGroupId("groupId")
-                .setMemberId("memberId")
+                .setMemberId(memberId)
                 .setProtocolType("consumer")
                 .setProtocols(new JoinGroupRequestData.JoinGroupRequestProtocolCollection())
                 .setSessionTimeoutMs(10000);
@@ -138,7 +144,6 @@ public void testJoinGroupRequestVersions() throws Exception {
 
     @Test
     public void testJoinGroupResponseVersions() throws Exception {
-        String memberId = "memberId";
         Supplier newResponse = () -> new JoinGroupResponseData()
                 .setMemberId(memberId)
                 .setLeader(memberId)
@@ -153,16 +158,31 @@ public void testJoinGroupResponseVersions() throws Exception {
         testAllMessageRoundTripsFromVersion((short) 5, newResponse.get().members().get(0).setGroupInstanceId("instanceId"));
     }
 
+    @Test
+    public void testLeaveGroupResponseVersions() throws Exception {
+        Supplier newResponse = () -> new LeaveGroupResponseData()
+                                                                 .setErrorCode(Errors.NOT_COORDINATOR.code());
+
+        testAllMessageRoundTrips(newResponse.get());
+        testAllMessageRoundTripsFromVersion((short) 1, newResponse.get().setThrottleTimeMs(1000));
+
+        testAllMessageRoundTripsFromVersion((short) 3, newResponse.get().setMembers(
+            Collections.singletonList(new MemberResponse()
+            .setMemberId(memberId)
+            .setGroupInstanceId(instanceId))
+        ));
+    }
+
     @Test
     public void testSyncGroupDefaultGroupInstanceId() throws Exception {
         Supplier request = () -> new SyncGroupRequestData()
                 .setGroupId("groupId")
-                .setMemberId("memberId")
+                .setMemberId(memberId)
                 .setGenerationId(15)
                 .setAssignments(new ArrayList<>());
         testAllMessageRoundTrips(request.get());
         testAllMessageRoundTrips(request.get().setGroupInstanceId(null));
-        testAllMessageRoundTripsFromVersion((short) 3, request.get().setGroupInstanceId("instanceId"));
+        testAllMessageRoundTripsFromVersion((short) 3, request.get().setGroupInstanceId(instanceId));
     }
 
     @Test
@@ -173,12 +193,12 @@ public void testOffsetCommitDefaultGroupInstanceId() throws Exception {
 
         Supplier request = () -> new OffsetCommitRequestData()
                 .setGroupId("groupId")
-                .setMemberId("memberId")
+                .setMemberId(memberId)
                 .setTopics(new ArrayList<>())
                 .setGenerationId(15);
         testAllMessageRoundTripsFromVersion((short) 1, request.get());
         testAllMessageRoundTripsFromVersion((short) 1, request.get().setGroupInstanceId(null));
-        testAllMessageRoundTripsFromVersion((short) 7, request.get().setGroupInstanceId("instanceId"));
+        testAllMessageRoundTripsFromVersion((short) 7, request.get().setGroupInstanceId(instanceId));
     }
 
     @Test
@@ -317,7 +337,7 @@ public void testRequestSchemas() throws Exception {
      * Test that the JSON response files match the schemas accessible through the ApiKey class.
      */
     @Test
-    public void testResponseSchemas() throws Exception {
+    public void testResponseSchemas() {
         for (ApiKeys apiKey : ApiKeys.values()) {
             Schema[] manualSchemas = apiKey.responseSchemas;
             Schema[] generatedSchemas = ApiMessageType.fromApiKey(apiKey.id).responseSchemas();
@@ -439,17 +459,17 @@ public void testNonIgnorableFieldWithDefaultNull() throws Exception {
         verifySizeRaisesUve((short) 0, "groupInstanceId", new HeartbeatRequestData()
                 .setGroupId("groupId")
                 .setGenerationId(15)
-                .setMemberId("memberId")
-                .setGroupInstanceId("instanceId"));
+                .setMemberId(memberId)
+                .setGroupInstanceId(instanceId));
         verifySizeSucceeds((short) 0, new HeartbeatRequestData()
                 .setGroupId("groupId")
                 .setGenerationId(15)
-                .setMemberId("memberId")
+                .setMemberId(memberId)
                 .setGroupInstanceId(null));
         verifySizeSucceeds((short) 0, new HeartbeatRequestData()
                 .setGroupId("groupId")
                 .setGenerationId(15)
-                .setMemberId("memberId"));
+                .setMemberId(memberId));
     }
 
     private void verifySizeRaisesUve(short version, String problemFieldName,
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupRequestTest.java
index 9fa9d3b6d0ba8..2ff928bea5c0c 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupRequestTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupRequestTest.java
@@ -16,34 +16,97 @@
  */
 package org.apache.kafka.common.requests;
 
+import org.apache.kafka.common.errors.UnsupportedVersionException;
 import org.apache.kafka.common.message.LeaveGroupRequestData;
+import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity;
 import org.apache.kafka.common.message.LeaveGroupResponseData;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
+import org.junit.Before;
 import org.junit.Test;
 
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 public class LeaveGroupRequestTest {
 
+    private final String groupId = "group_id";
+    private final String memberIdOne = "member_1";
+    private final String instanceIdOne = "instance_1";
+    private final String memberIdTwo = "member_2";
+    private final String instanceIdTwo = "instance_2";
+
+    private final int throttleTimeMs = 10;
+
+    private LeaveGroupRequest.Builder builder;
+    private List members;
+
+    @Before
+    public void setUp() {
+        members = Arrays.asList(new MemberIdentity()
+                                         .setMemberId(memberIdOne)
+                                         .setGroupInstanceId(instanceIdOne),
+                                new MemberIdentity()
+                                         .setMemberId(memberIdTwo)
+                                         .setGroupInstanceId(instanceIdTwo));
+        builder = new LeaveGroupRequest.Builder(
+            groupId,
+            members
+        );
+    }
+
     @Test
-    public void testLeaveConstructor() {
-        final String groupId = "group_id";
-        final String memberId = "member_id";
-        final int throttleTimeMs = 10;
+    public void testMultiLeaveConstructor() {
+        final LeaveGroupRequestData expectedData = new LeaveGroupRequestData()
+                                                       .setGroupId(groupId)
+                                                       .setMembers(members);
+
+        for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) {
+            try {
+                LeaveGroupRequest request = builder.build(version);
+                if (version <= 2) {
+                    fail("Older version " + version +
+                             " request data should not be created due to non-single members");
+                }
+                assertEquals(expectedData, request.data());
+                assertEquals(members, request.members());
 
+                LeaveGroupResponse expectedResponse = new LeaveGroupResponse(
+                    Collections.emptyList(),
+                    Errors.COORDINATOR_LOAD_IN_PROGRESS,
+                    throttleTimeMs,
+                    version
+                );
+
+                assertEquals(expectedResponse, request.getErrorResponse(throttleTimeMs,
+                                                                        Errors.COORDINATOR_LOAD_IN_PROGRESS.exception()));
+            } catch (UnsupportedVersionException e) {
+                assertTrue(e.getMessage().contains("leave group request only supports single member instance"));
+            }
+        }
+
+    }
+
+    @Test
+    public void testSingleLeaveConstructor() {
         final LeaveGroupRequestData expectedData = new LeaveGroupRequestData()
                                                        .setGroupId(groupId)
-                                                       .setMemberId(memberId);
+                                                       .setMemberId(memberIdOne);
+        List singleMember = Collections.singletonList(
+            new MemberIdentity()
+                .setMemberId(memberIdOne));
 
-        final LeaveGroupRequest.Builder builder =
-            new LeaveGroupRequest.Builder(new LeaveGroupRequestData()
-                                              .setGroupId(groupId)
-                                              .setMemberId(memberId));
+        builder = new LeaveGroupRequest.Builder(groupId, singleMember);
 
-        for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) {
+        for (short version = 0; version <= 2; version++) {
             LeaveGroupRequest request = builder.build(version);
             assertEquals(expectedData, request.data());
+            assertEquals(singleMember, request.members());
 
             int expectedThrottleTime = version >= 1 ? throttleTimeMs
                                            : AbstractResponse.DEFAULT_THROTTLE_TIME;
@@ -57,4 +120,9 @@ public void testLeaveConstructor() {
                                                                     Errors.NOT_CONTROLLER.exception()));
         }
     }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testBuildEmptyMembers() {
+        new LeaveGroupRequest.Builder(groupId, Collections.emptyList());
+    }
 }
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java
index a1368b54dcbfa..187640bfdeb71 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java
@@ -17,11 +17,16 @@
 package org.apache.kafka.common.requests;
 
 import org.apache.kafka.common.message.LeaveGroupResponseData;
+import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
+import org.junit.Before;
 import org.junit.Test;
 
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static org.apache.kafka.common.requests.AbstractResponse.DEFAULT_THROTTLE_TIME;
@@ -30,10 +35,31 @@
 import static org.junit.Assert.assertTrue;
 
 public class LeaveGroupResponseTest {
+
+    private final String memberIdOne = "member_1";
+    private final String instanceIdOne = "instance_1";
+    private final String memberIdTwo = "member_2";
+    private final String instanceIdTwo = "instance_2";
+
     private final int throttleTimeMs = 10;
 
+    private List memberResponses;
+
+    @Before
+    public void setUp() {
+        memberResponses = Arrays.asList(new MemberResponse()
+                                            .setMemberId(memberIdOne)
+                                            .setGroupInstanceId(instanceIdOne)
+                                            .setErrorCode(Errors.UNKNOWN_MEMBER_ID.code()),
+                                        new MemberResponse()
+                                            .setMemberId(memberIdTwo)
+                                            .setGroupInstanceId(instanceIdTwo)
+                                            .setErrorCode(Errors.FENCED_INSTANCE_ID.code())
+        );
+    }
+
     @Test
-    public void testConstructor() {
+    public void testConstructorWithStruct() {
         Map expectedErrorCounts = Collections.singletonMap(Errors.NOT_COORDINATOR, 1);
 
         LeaveGroupResponseData responseData = new LeaveGroupResponseData()
@@ -54,9 +80,40 @@ public void testConstructor() {
         }
     }
 
+
+    @Test
+    public void testConstructorWithMemberResponses() {
+        Map expectedErrorCounts = new HashMap<>();
+        expectedErrorCounts.put(Errors.UNKNOWN_MEMBER_ID, 1);
+        expectedErrorCounts.put(Errors.FENCED_INSTANCE_ID, 1);
+
+        for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) {
+            LeaveGroupResponse leaveGroupResponse = new LeaveGroupResponse(memberResponses,
+                                                                           Errors.NONE,
+                                                                           throttleTimeMs,
+                                                                           version);
+
+            if (version >= 3) {
+                assertEquals(expectedErrorCounts, leaveGroupResponse.errorCounts());
+                assertEquals(memberResponses, leaveGroupResponse.memberResponses());
+            } else {
+                assertEquals(Collections.singletonMap(Errors.UNKNOWN_MEMBER_ID, 1),
+                             leaveGroupResponse.errorCounts());
+                assertEquals(Collections.emptyList(), leaveGroupResponse.memberResponses());
+            }
+
+            if (version >= 1) {
+                assertEquals(throttleTimeMs, leaveGroupResponse.throttleTimeMs());
+            } else {
+                assertEquals(DEFAULT_THROTTLE_TIME, leaveGroupResponse.throttleTimeMs());
+            }
+
+            assertEquals(Errors.UNKNOWN_MEMBER_ID, leaveGroupResponse.error());
+        }
+    }
+
     @Test
     public void testShouldThrottle() {
-        // A dummy setup is ok.
         LeaveGroupResponse response = new LeaveGroupResponse(new LeaveGroupResponseData());
         for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) {
             if (version >= 2) {
@@ -68,7 +125,7 @@ public void testShouldThrottle() {
     }
 
     @Test
-    public void testEquality() {
+    public void testEqualityWithStruct() {
         LeaveGroupResponseData responseData = new LeaveGroupResponseData()
             .setErrorCode(Errors.NONE.code())
             .setThrottleTimeMs(throttleTimeMs);
@@ -80,6 +137,29 @@ public void testEquality() {
             assertEquals(primaryResponse, primaryResponse);
             assertEquals(primaryResponse, secondaryResponse);
             assertEquals(primaryResponse.hashCode(), secondaryResponse.hashCode());
+
+        }
+    }
+
+    @Test
+    public void testEqualityWithMemberResponses() {
+        for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) {
+            List localResponses = version > 2 ? memberResponses : memberResponses.subList(0, 1);
+            LeaveGroupResponse primaryResponse = new LeaveGroupResponse(localResponses,
+                                                                        Errors.NONE,
+                                                                        throttleTimeMs,
+                                                                        version);
+
+            // The order of members should not alter result data.
+            Collections.reverse(localResponses);
+            LeaveGroupResponse reversedResponse = new LeaveGroupResponse(localResponses,
+                                                                         Errors.NONE,
+                                                                         throttleTimeMs,
+                                                                         version);
+
+            assertEquals(primaryResponse, primaryResponse);
+            assertEquals(primaryResponse, reversedResponse);
+            assertEquals(primaryResponse.hashCode(), reversedResponse.hashCode());
         }
     }
 }
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index 10da9f55f8605..98552108a1ebc 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -55,6 +55,7 @@
 import org.apache.kafka.common.message.DeleteTopicsResponseData;
 import org.apache.kafka.common.message.DescribeGroupsRequestData;
 import org.apache.kafka.common.message.DescribeGroupsResponseData;
+import org.apache.kafka.common.message.DescribeGroupsResponseData.DescribedGroup;
 import org.apache.kafka.common.message.ElectLeadersResponseData.PartitionResult;
 import org.apache.kafka.common.message.ElectLeadersResponseData.ReplicaElectionResult;
 import org.apache.kafka.common.message.FindCoordinatorRequestData;
@@ -69,7 +70,7 @@
 import org.apache.kafka.common.message.InitProducerIdResponseData;
 import org.apache.kafka.common.message.JoinGroupRequestData;
 import org.apache.kafka.common.message.JoinGroupResponseData;
-import org.apache.kafka.common.message.LeaveGroupRequestData;
+import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity;
 import org.apache.kafka.common.message.LeaveGroupResponseData;
 import org.apache.kafka.common.message.ListGroupsRequestData;
 import org.apache.kafka.common.message.ListGroupsResponseData;
@@ -125,7 +126,6 @@
 import java.util.Set;
 
 import static java.util.Arrays.asList;
-import static java.util.Collections.singletonList;
 import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID;
 import static org.apache.kafka.test.TestUtils.toBuffer;
 import static org.junit.Assert.assertEquals;
@@ -184,14 +184,14 @@ public void testSerialization() throws Exception {
         checkErrorResponse(createListOffsetRequest(2), new UnknownServerException(), true);
         checkResponse(createListOffsetResponse(2), 2, true);
         checkRequest(MetadataRequest.Builder.allTopics().build((short) 2), true);
-        checkRequest(createMetadataRequest(1, singletonList("topic1")), true);
-        checkErrorResponse(createMetadataRequest(1, singletonList("topic1")), new UnknownServerException(), true);
+        checkRequest(createMetadataRequest(1, Collections.singletonList("topic1")), true);
+        checkErrorResponse(createMetadataRequest(1, Collections.singletonList("topic1")), new UnknownServerException(), true);
         checkResponse(createMetadataResponse(), 2, true);
-        checkErrorResponse(createMetadataRequest(2, singletonList("topic1")), new UnknownServerException(), true);
+        checkErrorResponse(createMetadataRequest(2, Collections.singletonList("topic1")), new UnknownServerException(), true);
         checkResponse(createMetadataResponse(), 3, true);
-        checkErrorResponse(createMetadataRequest(3, singletonList("topic1")), new UnknownServerException(), true);
+        checkErrorResponse(createMetadataRequest(3, Collections.singletonList("topic1")), new UnknownServerException(), true);
         checkResponse(createMetadataResponse(), 4, true);
-        checkErrorResponse(createMetadataRequest(4, singletonList("topic1")), new UnknownServerException(), true);
+        checkErrorResponse(createMetadataRequest(4, Collections.singletonList("topic1")), new UnknownServerException(), true);
         checkRequest(OffsetFetchRequest.forAllPartitions("group1"), true);
         checkErrorResponse(OffsetFetchRequest.forAllPartitions("group1"), new NotCoordinatorException("Not Coordinator"), true);
         checkRequest(createOffsetFetchRequest(0), true);
@@ -262,7 +262,7 @@ public void testSerialization() throws Exception {
         checkOlderFetchVersions();
         checkResponse(createMetadataResponse(), 0, true);
         checkResponse(createMetadataResponse(), 1, true);
-        checkErrorResponse(createMetadataRequest(1, singletonList("topic1")), new UnknownServerException(), true);
+        checkErrorResponse(createMetadataRequest(1, Collections.singletonList("topic1")), new UnknownServerException(), true);
         checkRequest(createOffsetCommitRequest(0), true);
         checkErrorResponse(createOffsetCommitRequest(0), new UnknownServerException(), true);
         checkRequest(createOffsetCommitRequest(1), true);
@@ -723,8 +723,7 @@ public void testJoinGroupRequestVersion0RebalanceTimeout() {
     public void testOffsetFetchRequestBuilderToString() {
         String allTopicPartitionsString = OffsetFetchRequest.Builder.allTopicPartitions("someGroup").toString();
         assertTrue(allTopicPartitionsString.contains(""));
-        String string = new OffsetFetchRequest.Builder("group1",
-                singletonList(new TopicPartition("test11", 1))).toString();
+        String string = new OffsetFetchRequest.Builder("group1", Collections.singletonList(new TopicPartition("test11", 1))).toString();
         assertTrue(string.contains("test11"));
         assertTrue(string.contains("group1"));
     }
@@ -896,14 +895,22 @@ private DescribeGroupsResponse createDescribeGroupResponse() {
         DescribeGroupsResponseData describeGroupsResponseData = new DescribeGroupsResponseData();
         DescribeGroupsResponseData.DescribedGroupMember member = DescribeGroupsResponse.groupMember("memberId", null,
                 clientId, clientHost, new byte[0], new byte[0]);
-        DescribeGroupsResponseData.DescribedGroup metadata = DescribeGroupsResponse.groupMetadata("test-group", Errors.NONE,
-                "STABLE", "consumer", "roundrobin", asList(member), Collections.emptySet());
+        DescribedGroup metadata = DescribeGroupsResponse.groupMetadata("test-group",
+                                                                       Errors.NONE,
+                                                                       "STABLE",
+                                                                       "consumer",
+                                                                       "roundrobin",
+                                                                       Collections.singletonList(member),
+                                                                       Collections.emptySet());
         describeGroupsResponseData.groups().add(metadata);
         return new DescribeGroupsResponse(describeGroupsResponseData);
     }
 
     private LeaveGroupRequest createLeaveGroupRequest() {
-        return new LeaveGroupRequest.Builder(new LeaveGroupRequestData().setGroupId("group1").setMemberId("consumer1")).build();
+        return new LeaveGroupRequest.Builder(
+            "group1", Collections.singletonList(new MemberIdentity()
+                                                    .setMemberId("consumer1"))
+            ).build();
     }
 
     private LeaveGroupResponse createLeaveGroupResponse() {
@@ -1040,7 +1047,7 @@ private OffsetCommitResponse createOffsetCommitResponse() {
     }
 
     private OffsetFetchRequest createOffsetFetchRequest(int version) {
-        return new OffsetFetchRequest.Builder("group1", singletonList(new TopicPartition("test11", 1)))
+        return new OffsetFetchRequest.Builder("group1", Collections.singletonList(new TopicPartition("test11", 1)))
                 .build((short) version);
     }
 
@@ -1188,7 +1195,7 @@ private SaslHandshakeRequest createSaslHandshakeRequest() {
     private SaslHandshakeResponse createSaslHandshakeResponse() {
         return new SaslHandshakeResponse(
                 new SaslHandshakeResponseData()
-                .setErrorCode(Errors.NONE.code()).setMechanisms(singletonList("GSSAPI")));
+                .setErrorCode(Errors.NONE.code()).setMechanisms(Collections.singletonList("GSSAPI")));
     }
 
     private SaslAuthenticateRequest createSaslAuthenticateRequest() {
diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
index 7f6641ca5617b..6a57d59aa496a 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
@@ -28,6 +28,7 @@ import kafka.zk.KafkaZkClient
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.internals.Topic
 import org.apache.kafka.common.message.JoinGroupResponseData.JoinGroupResponseMember
+import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
 import org.apache.kafka.common.record.RecordBatch.{NO_PRODUCER_EPOCH, NO_PRODUCER_ID}
 import org.apache.kafka.common.requests._
@@ -252,7 +253,7 @@ class GroupCoordinator(val brokerId: Int,
       } else if (group.isPendingMember(memberId)) {
         // A rejoining pending member will be accepted. Note that pending member will never be a static member.
         if (groupInstanceId.isDefined) {
-          throw new IllegalStateException(s"the static member $groupInstanceId was unexpectedly to be assigned " +
+          throw new IllegalStateException(s"the static member $groupInstanceId was not expected to be assigned " +
             s"into pending member bucket with member id $memberId")
         } else {
           addMemberAndRebalance(rebalanceTimeoutMs, sessionTimeoutMs, memberId, groupInstanceId,
@@ -419,36 +420,58 @@ class GroupCoordinator(val brokerId: Int,
     }
   }
 
-  def handleLeaveGroup(groupId: String, memberId: String, responseCallback: Errors => Unit): Unit = {
-    validateGroupStatus(groupId, ApiKeys.LEAVE_GROUP).foreach { error =>
-      responseCallback(error)
-      return
-    }
-
-    groupManager.getGroup(groupId) match {
+  def handleLeaveGroup(groupId: String,
+                       leavingMembers: List[MemberIdentity],
+                       responseCallback: LeaveGroupResult => Unit) {
+    validateGroupStatus(groupId, ApiKeys.LEAVE_GROUP) match {
+      case Some(error) =>
+        responseCallback(leaveError(error, List.empty))
       case None =>
-        responseCallback(Errors.UNKNOWN_MEMBER_ID)
-
-      case Some(group) =>
-        group.inLock {
-          if (group.is(Dead)) {
-            responseCallback(Errors.COORDINATOR_NOT_AVAILABLE)
-          } else if (group.isPendingMember(memberId)) {
-            // if a pending member is leaving, it needs to be removed from the pending list, heartbeat cancelled
-            // and if necessary, prompt a JoinGroup completion.
-            info(s"Pending member $memberId is leaving group ${group.groupId}.")
-            removePendingMemberAndUpdateGroup(group, memberId)
-            heartbeatPurgatory.checkAndComplete(MemberKey(group.groupId, memberId))
-            responseCallback(Errors.NONE)
-          } else if (!group.has(memberId)) {
-            responseCallback(Errors.UNKNOWN_MEMBER_ID)
-          } else {
-            val member = group.get(memberId)
-            removeHeartbeatForLeavingMember(group, member)
-            info(s"Member ${member.memberId} in group ${group.groupId} has left, removing it from the group")
-            removeMemberAndUpdateGroup(group, member, s"removing member $memberId on LeaveGroup")
-            responseCallback(Errors.NONE)
-          }
+        groupManager.getGroup(groupId) match {
+          case None =>
+            responseCallback(leaveError(Errors.NONE, leavingMembers.map {leavingMember =>
+              memberLeaveError(leavingMember, Errors.UNKNOWN_MEMBER_ID)
+            }))
+          case Some(group) =>
+            group.inLock {
+              if (group.is(Dead)) {
+                responseCallback(leaveError(Errors.COORDINATOR_NOT_AVAILABLE, List.empty))
+              } else {
+                val memberErrors = leavingMembers.map { leavingMember =>
+                  val memberId = leavingMember.memberId
+                  val groupInstanceId = Option(leavingMember.groupInstanceId)
+                  if (memberId != JoinGroupRequest.UNKNOWN_MEMBER_ID
+                    && group.isStaticMemberFenced(memberId, groupInstanceId)) {
+                    memberLeaveError(leavingMember, Errors.FENCED_INSTANCE_ID)
+                  } else if (group.isPendingMember(memberId)) {
+                    if (groupInstanceId.isDefined) {
+                      throw new IllegalStateException(s"the static member $groupInstanceId was not expected to be leaving " +
+                        s"from pending member bucket with member id $memberId")
+                    } else {
+                      // if a pending member is leaving, it needs to be removed from the pending list, heartbeat cancelled
+                      // and if necessary, prompt a JoinGroup completion.
+                      info(s"Pending member $memberId is leaving group ${group.groupId}.")
+                      removePendingMemberAndUpdateGroup(group, memberId)
+                      heartbeatPurgatory.checkAndComplete(MemberKey(group.groupId, memberId))
+                      memberLeaveError(leavingMember, Errors.NONE)
+                    }
+                  } else if (!group.has(memberId) && !group.hasStaticMember(groupInstanceId)) {
+                    memberLeaveError(leavingMember, Errors.UNKNOWN_MEMBER_ID)
+                  } else {
+                    val member = if (group.hasStaticMember(groupInstanceId))
+                      group.get(group.getStaticMemberId(groupInstanceId))
+                    else
+                      group.get(memberId)
+                    removeHeartbeatForLeavingMember(group, member)
+                    info(s"Member[group.instance.id ${member.groupInstanceId}, member.id ${member.memberId}] " +
+                      s"in group ${group.groupId} has left, removing it from the group")
+                    removeMemberAndUpdateGroup(group, member, s"removing member $memberId on LeaveGroup")
+                    memberLeaveError(leavingMember, Errors.NONE)
+                  }
+                }
+                responseCallback(leaveError(Errors.NONE, memberErrors))
+              }
+            }
         }
     }
   }
@@ -1106,6 +1129,21 @@ object GroupCoordinator {
       leaderId = GroupCoordinator.NoLeader,
       error = error)
   }
+
+  private def memberLeaveError(memberIdentity: MemberIdentity,
+                               error: Errors): LeaveMemberResponse = {
+    LeaveMemberResponse(
+      memberId = memberIdentity.memberId,
+      groupInstanceId = Option(memberIdentity.groupInstanceId),
+      error = error)
+  }
+
+  private def leaveError(topLevelError: Errors,
+                         memberResponses: List[LeaveMemberResponse]): LeaveGroupResult = {
+    LeaveGroupResult(
+      topLevelError = topLevelError,
+      memberResponses = memberResponses)
+  }
 }
 
 case class GroupConfig(groupMinSessionTimeoutMs: Int,
@@ -1122,3 +1160,10 @@ case class JoinGroupResult(members: List[JoinGroupResponseMember],
 
 case class SyncGroupResult(memberAssignment: Array[Byte],
                            error: Errors)
+
+case class LeaveMemberResponse(memberId: String,
+                               groupInstanceId: Option[String],
+                               error: Errors)
+
+case class LeaveGroupResult(topLevelError: Errors,
+                            memberResponses : List[LeaveMemberResponse])
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index cec7470c162e6..ecd6c9453201c 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -31,7 +31,7 @@ import kafka.api.{ApiVersion, KAFKA_0_11_0_IV0, KAFKA_2_3_IV0}
 import kafka.cluster.Partition
 import kafka.common.OffsetAndMetadata
 import kafka.controller.KafkaController
-import kafka.coordinator.group.{GroupCoordinator, JoinGroupResult, SyncGroupResult}
+import kafka.coordinator.group.{GroupCoordinator, JoinGroupResult, LeaveGroupResult, SyncGroupResult}
 import kafka.coordinator.transaction.{InitProducerIdResult, TransactionCoordinator}
 import kafka.message.ZStdCompressionCodec
 import kafka.network.RequestChannel
@@ -62,6 +62,7 @@ import org.apache.kafka.common.message.HeartbeatResponseData
 import org.apache.kafka.common.message.InitProducerIdResponseData
 import org.apache.kafka.common.message.JoinGroupResponseData
 import org.apache.kafka.common.message.LeaveGroupResponseData
+import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse
 import org.apache.kafka.common.message.ListGroupsResponseData
 import org.apache.kafka.common.message.OffsetCommitRequestData
 import org.apache.kafka.common.message.OffsetCommitResponseData
@@ -1514,9 +1515,9 @@ class KafkaApis(val requestChannel: RequestChannel,
     def sendResponseCallback(error: Errors) {
       def createResponse(requestThrottleMs: Int): AbstractResponse = {
         val response = new HeartbeatResponse(
-            new HeartbeatResponseData()
-              .setThrottleTimeMs(requestThrottleMs)
-              .setErrorCode(error.code))
+          new HeartbeatResponseData()
+            .setThrottleTimeMs(requestThrottleMs)
+            .setErrorCode(error.code))
         trace("Sending heartbeat response %s for correlation id %d to client %s."
           .format(response, request.header.correlationId, request.header.clientId))
         response
@@ -1549,29 +1550,37 @@ class KafkaApis(val requestChannel: RequestChannel,
   def handleLeaveGroupRequest(request: RequestChannel.Request) {
     val leaveGroupRequest = request.body[LeaveGroupRequest]
 
-    // the callback for sending a leave-group response
-    def sendResponseCallback(error: Errors) {
-      def createResponse(requestThrottleMs: Int): AbstractResponse = {
-        val response = new LeaveGroupResponse(new LeaveGroupResponseData()
-          .setThrottleTimeMs(requestThrottleMs)
-          .setErrorCode(error.code()))
-        trace("Sending leave group response %s for correlation id %d to client %s."
-          .format(response, request.header.correlationId, request.header.clientId))
-        response
-      }
-      sendResponseMaybeThrottle(request, createResponse)
-    }
+    val members = leaveGroupRequest.members().asScala.toList
 
-    if (!authorize(request.session, Read, Resource(Group, leaveGroupRequest.data().groupId(), LITERAL))) {
-      sendResponseMaybeThrottle(request, requestThrottleMs =>
+    if (!authorize(request.session, Read, Resource(Group, leaveGroupRequest.data.groupId, LITERAL))) {
+      sendResponseMaybeThrottle(request, requestThrottleMs => {
         new LeaveGroupResponse(new LeaveGroupResponseData()
           .setThrottleTimeMs(requestThrottleMs)
-          .setErrorCode(Errors.GROUP_AUTHORIZATION_FAILED.code())))
+          .setErrorCode(Errors.GROUP_AUTHORIZATION_FAILED.code)
+        )
+      })
     } else {
-      // let the coordinator to handle leave-group
+      def sendResponseCallback(leaveGroupResult : LeaveGroupResult) {
+        val memberResponses = leaveGroupResult.memberResponses.map(
+          leaveGroupResult =>
+            new MemberResponse()
+              .setErrorCode(leaveGroupResult.error.code)
+              .setMemberId(leaveGroupResult.memberId)
+              .setGroupInstanceId(leaveGroupResult.groupInstanceId.orNull)
+        )
+        def createResponse(requestThrottleMs: Int): AbstractResponse = {
+          new LeaveGroupResponse(
+            memberResponses.asJava,
+            leaveGroupResult.topLevelError,
+            requestThrottleMs,
+            leaveGroupRequest.version)
+        }
+        sendResponseMaybeThrottle(request, createResponse)
+      }
+
       groupCoordinator.handleLeaveGroup(
-        leaveGroupRequest.data().groupId(),
-        leaveGroupRequest.data().memberId(),
+        leaveGroupRequest.data.groupId,
+        members,
         sendResponseCallback)
     }
   }
diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
index 41bfb0d6645ef..cb5f1aadb041c 100644
--- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
@@ -46,7 +46,7 @@ import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData
 import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData.{AlterConfigsResource, AlterableConfig, AlterableConfigCollection}
 import org.apache.kafka.common.message.JoinGroupRequestData
 import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection
-import org.apache.kafka.common.message.LeaveGroupRequestData
+import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
 import org.apache.kafka.common.message.OffsetCommitRequestData
 import org.apache.kafka.common.message.SyncGroupRequestData
 import org.apache.kafka.common.network.ListenerName
@@ -398,9 +398,10 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
       .setMemberId(JoinGroupRequest.UNKNOWN_MEMBER_ID)).build()
 
   private def leaveGroupRequest = new LeaveGroupRequest.Builder(
-    new LeaveGroupRequestData()
-      .setGroupId(group)
-      .setMemberId(JoinGroupRequest.UNKNOWN_MEMBER_ID)).build()
+    group, Collections.singletonList(
+      new MemberIdentity()
+        .setMemberId(JoinGroupRequest.UNKNOWN_MEMBER_ID)
+    )).build()
 
   private def deleteGroupsRequest = new DeleteGroupsRequest.Builder(
     new DeleteGroupsRequestData()
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorConcurrencyTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorConcurrencyTest.scala
index 1cee665f1c893..b85035f64f2c6 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorConcurrencyTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorConcurrencyTest.scala
@@ -26,6 +26,7 @@ import kafka.coordinator.group.GroupCoordinatorConcurrencyTest._
 import kafka.server.{DelayedOperationPurgatory, KafkaConfig}
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.internals.Topic
+import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.requests.JoinGroupRequest
 import org.apache.kafka.common.utils.Time
@@ -153,8 +154,8 @@ class GroupCoordinatorConcurrencyTest extends AbstractCoordinatorConcurrencyTest
   }
 
 
-  class JoinGroupOperation extends GroupOperation[JoinGroupResult, JoinGroupCallback] {
-    override def responseCallback(responsePromise: Promise[JoinGroupResult]): JoinGroupCallback = {
+  class JoinGroupOperation extends GroupOperation[JoinGroupCallbackParams, JoinGroupCallback] {
+    override def responseCallback(responsePromise: Promise[JoinGroupCallbackParams]): JoinGroupCallback = {
       val callback: JoinGroupCallback = responsePromise.success(_)
       callback
     }
@@ -263,21 +264,28 @@ class GroupCoordinatorConcurrencyTest extends AbstractCoordinatorConcurrencyTest
 
   class LeaveGroupOperation extends GroupOperation[LeaveGroupCallbackParams, LeaveGroupCallback] {
     override def responseCallback(responsePromise: Promise[LeaveGroupCallbackParams]): LeaveGroupCallback = {
-      val callback: LeaveGroupCallback = error => responsePromise.success(error)
+      val callback: LeaveGroupCallback = result => responsePromise.success(result)
       callback
     }
     override def runWithCallback(member: GroupMember, responseCallback: LeaveGroupCallback): Unit = {
-      groupCoordinator.handleLeaveGroup(member.group.groupId, member.memberId, responseCallback)
+      val memberIdentity = new MemberIdentity()
+          .setMemberId(member.memberId)
+      groupCoordinator.handleLeaveGroup(member.group.groupId, List(memberIdentity), responseCallback)
     }
     override def awaitAndVerify(member: GroupMember): Unit = {
-       val error = await(member, DefaultSessionTimeout)
-       assertEquals(Errors.NONE, error)
+      val leaveGroupResult = await(member, DefaultSessionTimeout)
+
+      val memberResponses = leaveGroupResult.memberResponses
+      GroupCoordinatorTest.verifyLeaveGroupResult(leaveGroupResult, Errors.NONE, List(Errors.NONE))
+      assertEquals(member.memberId, memberResponses.head.memberId)
+      assertEquals(None, memberResponses.head.groupInstanceId)
     }
   }
 }
 
 object GroupCoordinatorConcurrencyTest {
 
+  type JoinGroupCallbackParams = JoinGroupResult
   type JoinGroupCallback = JoinGroupResult => Unit
   type SyncGroupCallbackParams = (Array[Byte], Errors)
   type SyncGroupCallback = SyncGroupResult => Unit
@@ -285,8 +293,8 @@ object GroupCoordinatorConcurrencyTest {
   type HeartbeatCallback = Errors => Unit
   type CommitOffsetCallbackParams = Map[TopicPartition, Errors]
   type CommitOffsetCallback = Map[TopicPartition, Errors] => Unit
-  type LeaveGroupCallbackParams = Errors
-  type LeaveGroupCallback = Errors => Unit
+  type LeaveGroupCallbackParams = LeaveGroupResult
+  type LeaveGroupCallback = LeaveGroupResult => Unit
   type CompleteTxnCallbackParams = Errors
   type CompleteTxnCallback = Errors => Unit
 
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
index d72bfc8f5cebc..cdf1518229987 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
@@ -20,7 +20,7 @@ package kafka.coordinator.group
 import java.util.Optional
 
 import kafka.common.OffsetAndMetadata
-import kafka.server.{DelayedOperationPurgatory, KafkaConfig, HostedPartition, ReplicaManager}
+import kafka.server.{DelayedOperationPurgatory, HostedPartition, KafkaConfig, ReplicaManager}
 import kafka.utils._
 import kafka.utils.timer.MockTimer
 import org.apache.kafka.common.TopicPartition
@@ -35,6 +35,7 @@ import java.util.concurrent.locks.ReentrantLock
 import kafka.cluster.Partition
 import kafka.zk.KafkaZkClient
 import org.apache.kafka.common.internals.Topic
+import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
 import org.junit.Assert._
 import org.junit.{After, Assert, Before, Test}
 import org.scalatest.Assertions.intercept
@@ -45,6 +46,8 @@ import scala.concurrent.duration.Duration
 import scala.concurrent.{Await, Future, Promise, TimeoutException}
 
 class GroupCoordinatorTest {
+  import GroupCoordinatorTest._
+
   type JoinGroupCallback = JoinGroupResult => Unit
   type SyncGroupCallbackParams = (Array[Byte], Errors)
   type SyncGroupCallback = SyncGroupResult => Unit
@@ -52,8 +55,7 @@ class GroupCoordinatorTest {
   type HeartbeatCallback = Errors => Unit
   type CommitOffsetCallbackParams = Map[TopicPartition, Errors]
   type CommitOffsetCallback = Map[TopicPartition, Errors] => Unit
-  type LeaveGroupCallbackParams = Errors
-  type LeaveGroupCallback = Errors => Unit
+  type LeaveGroupCallback = LeaveGroupResult => Unit
 
   val ClientId = "consumer-test"
   val ClientHost = "localhost"
@@ -950,6 +952,24 @@ class GroupCoordinatorTest {
     assertTrue(message.contains(followerInstanceId.get))
   }
 
+  @Test
+  def staticMemberLeaveWithIllegalStateAsPendingMember() {
+    val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
+    val group = groupCoordinator.groupManager.getGroup(groupId).get
+    group.addPendingMember(rebalanceResult.followerId)
+    group.remove(rebalanceResult.followerId)
+    EasyMock.reset(replicaManager)
+
+    // Illegal state exception shall trigger since follower id resides in pending member bucket.
+    val expectedException = intercept[IllegalStateException] {
+      singleLeaveGroup(groupId, rebalanceResult.followerId, followerInstanceId)
+    }
+
+    val message = expectedException.getMessage
+    assertTrue(message.contains(rebalanceResult.followerId))
+    assertTrue(message.contains(followerInstanceId.get))
+  }
+
   @Test
   def staticMemberReJoinWithIllegalStateAsUnknownMember() {
     staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
@@ -1036,8 +1056,8 @@ class GroupCoordinatorTest {
 
     // Send a special leave group request from static follower, moving group towards PreparingRebalance
     EasyMock.reset(replicaManager)
-    val followerLeaveGroupResult = leaveGroup(groupId, rebalanceResult.followerId)
-    assertEquals(Errors.NONE, followerLeaveGroupResult)
+    val followerLeaveGroupResults = singleLeaveGroup(groupId, rebalanceResult.followerId)
+    verifyLeaveGroupResult(followerLeaveGroupResults)
     assertGroupState(groupState = PreparingRebalance)
 
     timer.advanceClock(DefaultRebalanceTimeout + 1)
@@ -1791,8 +1811,8 @@ class GroupCoordinatorTest {
     val pending = setupGroupWithPendingMember()
 
     EasyMock.reset(replicaManager)
-    val leaveGroupResult = leaveGroup(groupId, pending.memberId)
-    assertEquals(Errors.NONE, leaveGroupResult)
+    val leaveGroupResults = singleLeaveGroup(groupId, pending.memberId)
+    verifyLeaveGroupResult(leaveGroupResults)
 
     assertGroupState(groupState = CompletingRebalance)
     assertEquals(2, group().allMembers.size)
@@ -2004,8 +2024,8 @@ class GroupCoordinatorTest {
 
     // and leaves.
     EasyMock.reset(replicaManager)
-    val leaveGroupResult = leaveGroup(groupId, assignedMemberId)
-    assertEquals(Errors.NONE, leaveGroupResult)
+    val leaveGroupResults = singleLeaveGroup(groupId, assignedMemberId)
+    verifyLeaveGroupResult(leaveGroupResults)
 
     // The simple offset commit should now fail
     EasyMock.reset(replicaManager)
@@ -2436,25 +2456,14 @@ class GroupCoordinatorTest {
   def testLeaveGroupWrongCoordinator() {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
 
-    val leaveGroupResult = leaveGroup(otherGroupId, memberId)
-    assertEquals(Errors.NOT_COORDINATOR, leaveGroupResult)
+    val leaveGroupResults = singleLeaveGroup(otherGroupId, memberId)
+    verifyLeaveGroupResult(leaveGroupResults, Errors.NOT_COORDINATOR)
   }
 
   @Test
   def testLeaveGroupUnknownGroup() {
-    val leaveGroupResult = leaveGroup(groupId, memberId)
-    assertEquals(Errors.UNKNOWN_MEMBER_ID, leaveGroupResult)
-  }
-
-  @Test
-  def testLeaveDeadGroup() {
-    val memberId = "memberId"
-
-    val deadGroupId = "deadGroupId"
-
-    groupCoordinator.groupManager.addGroup(new GroupMetadata(deadGroupId, Dead, new MockTime()))
-    val leaveGroupResult = leaveGroup(deadGroupId, memberId)
-    assertEquals(Errors.COORDINATOR_NOT_AVAILABLE, leaveGroupResult)
+    val leaveGroupResults = singleLeaveGroup(groupId, memberId)
+    verifyLeaveGroupResult(leaveGroupResults, Errors.NONE, List(Errors.UNKNOWN_MEMBER_ID))
   }
 
   @Test
@@ -2467,8 +2476,27 @@ class GroupCoordinatorTest {
     assertEquals(Errors.NONE, joinGroupError)
 
     EasyMock.reset(replicaManager)
-    val leaveGroupResult = leaveGroup(groupId, otherMemberId)
-    assertEquals(Errors.UNKNOWN_MEMBER_ID, leaveGroupResult)
+    val leaveGroupResults = singleLeaveGroup(groupId, otherMemberId)
+    verifyLeaveGroupResult(leaveGroupResults, Errors.NONE, List(Errors.UNKNOWN_MEMBER_ID))
+  }
+
+  @Test
+  def testSingleLeaveDeadGroup() {
+    val deadGroupId = "deadGroupId"
+
+    groupCoordinator.groupManager.addGroup(new GroupMetadata(deadGroupId, Dead, new MockTime()))
+    val leaveGroupResults = singleLeaveGroup(deadGroupId, memberId)
+    verifyLeaveGroupResult(leaveGroupResults, Errors.COORDINATOR_NOT_AVAILABLE)
+  }
+
+  @Test
+  def testBatchLeaveDeadGroup() {
+    val deadGroupId = "deadGroupId"
+
+    groupCoordinator.groupManager.addGroup(new GroupMetadata(deadGroupId, Dead, new MockTime()))
+    val leaveGroupResults = batchLeaveGroup(deadGroupId,
+      List(new MemberIdentity().setMemberId(memberId), new MemberIdentity().setMemberId(memberId)))
+    verifyLeaveGroupResult(leaveGroupResults, Errors.COORDINATOR_NOT_AVAILABLE)
   }
 
   @Test
@@ -2481,8 +2509,116 @@ class GroupCoordinatorTest {
     assertEquals(Errors.NONE, joinGroupError)
 
     EasyMock.reset(replicaManager)
-    val leaveGroupResult = leaveGroup(groupId, assignedMemberId)
-    assertEquals(Errors.NONE, leaveGroupResult)
+    val leaveGroupResults = singleLeaveGroup(groupId, assignedMemberId)
+    verifyLeaveGroupResult(leaveGroupResults)
+  }
+
+  @Test
+  def testLeaveGroupWithFencedInstanceId() {
+    val joinGroupResult = staticJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, leaderInstanceId, protocolType, protocolSuperset)
+    assertEquals(Errors.NONE, joinGroupResult.error)
+
+    EasyMock.reset(replicaManager)
+    val leaveGroupResults = singleLeaveGroup(groupId, "some_member", leaderInstanceId)
+    verifyLeaveGroupResult(leaveGroupResults, Errors.NONE, List(Errors.FENCED_INSTANCE_ID))
+  }
+
+  @Test
+  def testLeaveGroupStaticMemberWithUnknownMemberId() {
+    val joinGroupResult = staticJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, leaderInstanceId, protocolType, protocolSuperset)
+    assertEquals(Errors.NONE, joinGroupResult.error)
+
+    EasyMock.reset(replicaManager)
+    // Having unknown member id will not affect the request processing.
+    val leaveGroupResults = singleLeaveGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, leaderInstanceId)
+    verifyLeaveGroupResult(leaveGroupResults, Errors.NONE, List(Errors.NONE))
+  }
+
+  @Test
+  def testStaticMembersValidBatchLeaveGroup() {
+    staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
+
+    EasyMock.reset(replicaManager)
+    val leaveGroupResults = batchLeaveGroup(groupId, List(new MemberIdentity()
+      .setGroupInstanceId(leaderInstanceId.get), new MemberIdentity().setGroupInstanceId(followerInstanceId.get)))
+
+    verifyLeaveGroupResult(leaveGroupResults, Errors.NONE, List(Errors.NONE, Errors.NONE))
+  }
+
+  @Test
+  def testStaticMembersWrongCoordinatorBatchLeaveGroup() {
+    staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
+
+    EasyMock.reset(replicaManager)
+    val leaveGroupResults = batchLeaveGroup("invalid-group", List(new MemberIdentity()
+      .setGroupInstanceId(leaderInstanceId.get), new MemberIdentity().setGroupInstanceId(followerInstanceId.get)))
+
+    verifyLeaveGroupResult(leaveGroupResults, Errors.NOT_COORDINATOR)
+  }
+
+  @Test
+  def testStaticMembersUnknownGroupBatchLeaveGroup() {
+    val leaveGroupResults = batchLeaveGroup(groupId, List(new MemberIdentity()
+      .setGroupInstanceId(leaderInstanceId.get), new MemberIdentity().setGroupInstanceId(followerInstanceId.get)))
+
+    verifyLeaveGroupResult(leaveGroupResults, Errors.NONE, List(Errors.UNKNOWN_MEMBER_ID, Errors.UNKNOWN_MEMBER_ID))
+  }
+
+  @Test
+  def testStaticMembersFencedInstanceBatchLeaveGroup() {
+    staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
+
+    EasyMock.reset(replicaManager)
+    val leaveGroupResults = batchLeaveGroup(groupId, List(new MemberIdentity()
+      .setGroupInstanceId(leaderInstanceId.get), new MemberIdentity()
+      .setGroupInstanceId(followerInstanceId.get)
+      .setMemberId("invalid-member")))
+
+    verifyLeaveGroupResult(leaveGroupResults, Errors.NONE, List(Errors.NONE, Errors.FENCED_INSTANCE_ID))
+  }
+
+  @Test
+  def testStaticMembersUnknownInstanceBatchLeaveGroup() {
+    staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
+
+    EasyMock.reset(replicaManager)
+    val leaveGroupResults = batchLeaveGroup(groupId, List(new MemberIdentity()
+      .setGroupInstanceId("unknown-instance"), new MemberIdentity()
+      .setGroupInstanceId(followerInstanceId.get)))
+
+    verifyLeaveGroupResult(leaveGroupResults, Errors.NONE, List(Errors.UNKNOWN_MEMBER_ID, Errors.NONE))
+  }
+
+  @Test
+  def testPendingMemberBatchLeaveGroup() {
+    val pendingMember = setupGroupWithPendingMember()
+
+    EasyMock.reset(replicaManager)
+    val leaveGroupResults = batchLeaveGroup(groupId, List(new MemberIdentity()
+      .setGroupInstanceId("unknown-instance"), new MemberIdentity()
+      .setMemberId(pendingMember.memberId)))
+
+    verifyLeaveGroupResult(leaveGroupResults, Errors.NONE, List(Errors.UNKNOWN_MEMBER_ID, Errors.NONE))
+  }
+
+  @Test
+  def testPendingMemberWithUnexpectedInstanceIdBatchLeaveGroup() {
+    val pendingMember = setupGroupWithPendingMember()
+
+    EasyMock.reset(replicaManager)
+
+    // Bypass the FENCED_INSTANCE_ID check by defining pending member as a static member.
+    val instanceId = "instanceId"
+    val pendingMemberId = pendingMember.memberId
+    getGroup(groupId).addStaticMember(Option(instanceId), pendingMemberId)
+    val expectedException = intercept[IllegalStateException] {
+      batchLeaveGroup(groupId, List(new MemberIdentity().setGroupInstanceId("unknown-instance"),
+        new MemberIdentity().setGroupInstanceId(instanceId).setMemberId(pendingMemberId)))
+    }
+
+    val message = expectedException.getMessage
+    assertTrue(message.contains(instanceId))
+    assertTrue(message.contains(pendingMemberId))
   }
 
   @Test
@@ -2622,8 +2758,8 @@ class GroupCoordinatorTest {
     val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols)
 
     EasyMock.reset(replicaManager)
-    val leaveGroupResult = leaveGroup(groupId, joinGroupResult.memberId)
-    assertEquals(Errors.NONE, leaveGroupResult)
+    val leaveGroupResults = singleLeaveGroup(groupId, joinGroupResult.memberId)
+    verifyLeaveGroupResult(leaveGroupResults)
 
     val groupTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupPartitionId)
     val partition: Partition = EasyMock.niceMock(classOf[Partition])
@@ -2663,8 +2799,8 @@ class GroupCoordinatorTest {
     assertEquals(assignedMemberId, describeGroupResult._2.members.head.memberId)
 
     EasyMock.reset(replicaManager)
-    val leaveGroupResult = leaveGroup(groupId, assignedMemberId)
-    assertEquals(Errors.NONE, leaveGroupResult)
+    val leaveGroupResults = singleLeaveGroup(groupId, assignedMemberId)
+    verifyLeaveGroupResult(leaveGroupResults)
 
     val groupTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupPartitionId)
     val partition: Partition = EasyMock.niceMock(classOf[Partition])
@@ -2785,6 +2921,13 @@ class GroupCoordinatorTest {
     (responseFuture, responseCallback)
   }
 
+  private def setupLeaveGroupCallback: (Future[LeaveGroupResult], LeaveGroupCallback) = {
+    val responsePromise = Promise[LeaveGroupResult]
+    val responseFuture = responsePromise.future
+    val responseCallback: LeaveGroupCallback = result => responsePromise.success(result)
+    (responseFuture, responseCallback)
+  }
+
   private def sendJoinGroup(groupId: String,
                             memberId: String,
                             protocolType: String,
@@ -2980,15 +3123,26 @@ class GroupCoordinatorTest {
     result
   }
 
-  private def leaveGroup(groupId: String, consumerId: String): LeaveGroupCallbackParams = {
-    val (responseFuture, responseCallback) = setupHeartbeatCallback
+  private def singleLeaveGroup(groupId: String,
+                               consumerId: String,
+                               groupInstanceId: Option[String] = None): LeaveGroupResult = {
+    val singleMemberIdentity = List(
+      new MemberIdentity()
+        .setMemberId(consumerId)
+        .setGroupInstanceId(groupInstanceId.orNull))
+    batchLeaveGroup(groupId, singleMemberIdentity)
+  }
+
+  private def batchLeaveGroup(groupId: String,
+                              memberIdentities: List[MemberIdentity]): LeaveGroupResult = {
+    val (responseFuture, responseCallback) = setupLeaveGroupCallback
 
     EasyMock.expect(replicaManager.getPartition(new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupPartitionId)))
       .andReturn(HostedPartition.None)
     EasyMock.expect(replicaManager.getMagic(EasyMock.anyObject())).andReturn(Some(RecordBatch.MAGIC_VALUE_V1)).anyTimes()
     EasyMock.replay(replicaManager)
 
-    groupCoordinator.handleLeaveGroup(groupId, consumerId, responseCallback)
+    groupCoordinator.handleLeaveGroup(groupId, memberIdentities, responseCallback)
     Await.result(responseFuture, Duration(40, TimeUnit.MILLISECONDS))
   }
 
@@ -3003,3 +3157,17 @@ class GroupCoordinatorTest {
     OffsetAndMetadata(offset, "", timer.time.milliseconds())
   }
 }
+
+object GroupCoordinatorTest {
+  def verifyLeaveGroupResult(leaveGroupResult: LeaveGroupResult,
+                             expectedTopLevelError: Errors = Errors.NONE,
+                             expectedMemberLevelErrors: List[Errors] = List.empty) {
+    assertEquals(expectedTopLevelError, leaveGroupResult.topLevelError)
+    if (expectedMemberLevelErrors.nonEmpty) {
+      assertEquals(expectedMemberLevelErrors.size, leaveGroupResult.memberResponses.size)
+      for (i <- expectedMemberLevelErrors.indices) {
+            assertEquals(expectedMemberLevelErrors(i), leaveGroupResult.memberResponses(i).error)
+          }
+    }
+  }
+}
diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
index c26c3fd21d354..44f0301cd7e0b 100644
--- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
@@ -50,6 +50,7 @@ import org.easymock.{Capture, EasyMock, IAnswer}
 import EasyMock._
 import org.apache.kafka.common.message.{HeartbeatRequestData, JoinGroupRequestData, OffsetCommitRequestData, OffsetCommitResponseData, SyncGroupRequestData}
 import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocol
+import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
 import org.apache.kafka.common.replica.ClientMetadata
 import org.junit.Assert.{assertEquals, assertNull, assertTrue}
 import org.junit.{After, Test}
@@ -643,6 +644,63 @@ class KafkaApisTest {
     EasyMock.replay(groupCoordinator)
   }
 
+  @Test
+  def testMultipleLeaveGroup() {
+    val groupId = "groupId"
+
+    val leaveMemberList = List(
+      new MemberIdentity()
+        .setMemberId("member-1")
+        .setGroupInstanceId("instance-1"),
+      new MemberIdentity()
+        .setMemberId("member-2")
+        .setGroupInstanceId("instance-2")
+    )
+
+    EasyMock.expect(groupCoordinator.handleLeaveGroup(
+      EasyMock.eq(groupId),
+      EasyMock.eq(leaveMemberList),
+      anyObject()
+    ))
+
+    val (_, leaveRequest) = buildRequest(
+      new LeaveGroupRequest.Builder(
+        groupId,
+        leaveMemberList.asJava)
+    )
+
+    createKafkaApis().handleLeaveGroupRequest(leaveRequest)
+
+    EasyMock.replay(groupCoordinator)
+  }
+
+  @Test
+  def testSingleLeaveGroup() {
+    val groupId = "groupId"
+    val memberId = "member"
+
+    val singleLeaveMember = List(
+      new MemberIdentity()
+        .setMemberId(memberId)
+    )
+
+    EasyMock.expect(groupCoordinator.handleLeaveGroup(
+      EasyMock.eq(groupId),
+      EasyMock.eq(singleLeaveMember),
+      anyObject()
+    ))
+
+    val (_, leaveRequest) = buildRequest(
+      new LeaveGroupRequest.Builder(
+        groupId,
+        singleLeaveMember.asJava)
+    )
+
+    createKafkaApis().handleLeaveGroupRequest(leaveRequest)
+
+    EasyMock.replay(groupCoordinator)
+  }
+
   /**
    * Return pair of listener names in the metadataCache: PLAINTEXT and LISTENER2 respectively.
    */
diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
index 7fc3de9d4319e..242ab219686f0 100644
--- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
+++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
@@ -39,7 +39,7 @@ import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData
 import org.apache.kafka.common.message.InitProducerIdRequestData
 import org.apache.kafka.common.message.JoinGroupRequestData
 import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection
-import org.apache.kafka.common.message.LeaveGroupRequestData
+import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
 import org.apache.kafka.common.message.ListGroupsRequestData
 import org.apache.kafka.common.message.OffsetCommitRequestData
 import org.apache.kafka.common.message.SaslAuthenticateRequestData
@@ -325,9 +325,10 @@ class RequestQuotaTest extends BaseRequestTest {
 
         case ApiKeys.LEAVE_GROUP =>
           new LeaveGroupRequest.Builder(
-            new LeaveGroupRequestData()
-              .setGroupId("test-leave-group")
-              .setMemberId(JoinGroupRequest.UNKNOWN_MEMBER_ID)
+            "test-leave-group",
+            Collections.singletonList(
+              new MemberIdentity()
+                .setMemberId(JoinGroupRequest.UNKNOWN_MEMBER_ID))
           )
 
         case ApiKeys.SYNC_GROUP =>
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
index a9baa3f15cb9d..e1bfe37247267 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
@@ -278,7 +278,6 @@ Set previousTaskIds() {
     int commit() {
         int committed = 0;
         RuntimeException firstException = null;
-
         for (final Iterator it = running().iterator(); it.hasNext(); ) {
             final T task = it.next();
             try {

From 9ad7bce492869e7d971bb7c83becf795622fa4dd Mon Sep 17 00:00:00 2001
From: Arlo Louis O'Keeffe 
Date: Sun, 28 Jul 2019 08:35:25 +0200
Subject: [PATCH 0481/1071] MINOR: Close ZKDatabase in EmbeddedZookeeper
 (#6237)

And remove redundant call. Closing ZKDatabase is necessary to allow the data
directory to be deleted when running on Windows.

Reviewers: Ismael Juma 
---
 core/src/test/scala/unit/kafka/zk/EmbeddedZookeeper.scala | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/test/scala/unit/kafka/zk/EmbeddedZookeeper.scala b/core/src/test/scala/unit/kafka/zk/EmbeddedZookeeper.scala
index 288b62e2d5aa4..28b592eaf7af8 100755
--- a/core/src/test/scala/unit/kafka/zk/EmbeddedZookeeper.scala
+++ b/core/src/test/scala/unit/kafka/zk/EmbeddedZookeeper.scala
@@ -49,7 +49,7 @@ class EmbeddedZookeeper() extends Logging {
   val port = zookeeper.getClientPort
 
   def shutdown(): Unit = {
-    CoreUtils.swallow(zookeeper.shutdown(), this)
+    // Also shuts down ZooKeeperServer
     CoreUtils.swallow(factory.shutdown(), this)
 
     def isDown(): Boolean = {
@@ -60,6 +60,7 @@ class EmbeddedZookeeper() extends Logging {
     }
 
     Iterator.continually(isDown()).exists(identity)
+    CoreUtils.swallow(zookeeper.getZKDatabase().close(), this)
 
     Utils.delete(logDir)
     Utils.delete(snapshotDir)

From acf507ce9cd0a6037da3e4c561bd4c50ba526f8a Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Sun, 28 Jul 2019 09:38:26 -0700
Subject: [PATCH 0482/1071] MINOR: Remove unused TopicAndPartition and remove
 unused symbols (#7119)

With the removal of ZkUtils and AdminUtils, TopicAndPartition is finally
unused.

Reviewers: Manikumar Reddy 
---
 .../main/scala/kafka/admin/AclCommand.scala   |  4 +--
 .../kafka/common/TopicAndPartition.scala      | 30 -------------------
 .../kafka/controller/KafkaController.scala    |  2 +-
 .../group/GroupMetadataManager.scala          |  2 +-
 .../scala/kafka/log/LogCleanerManager.scala   |  2 +-
 .../security/auth/SimpleAclAuthorizer.scala   |  5 ++--
 .../main/scala/kafka/server/KafkaApis.scala   |  2 +-
 .../main/scala/kafka/server/KafkaConfig.scala |  4 +--
 .../main/scala/kafka/zk/KafkaZkClient.scala   |  2 +-
 9 files changed, 11 insertions(+), 42 deletions(-)
 delete mode 100644 core/src/main/scala/kafka/common/TopicAndPartition.scala

diff --git a/core/src/main/scala/kafka/admin/AclCommand.scala b/core/src/main/scala/kafka/admin/AclCommand.scala
index 7ab379d2b0dc1..6f5b06cb51890 100644
--- a/core/src/main/scala/kafka/admin/AclCommand.scala
+++ b/core/src/main/scala/kafka/admin/AclCommand.scala
@@ -280,9 +280,9 @@ object AclCommand extends Logging {
     private def getAcls(authorizer: Authorizer, filter: ResourcePatternFilter,
                         listPrincipal: Option[KafkaPrincipal] = None): Map[Resource, Set[Acl]] =
       if (listPrincipal.isEmpty)
-        authorizer.getAcls().filter { case (resource, acl) => filter.matches(resource.toPattern) }
+        authorizer.getAcls().filter { case (resource, _) => filter.matches(resource.toPattern) }
       else
-        authorizer.getAcls(listPrincipal.get).filter { case (resource, acl) => filter.matches(resource.toPattern) }
+        authorizer.getAcls(listPrincipal.get).filter { case (resource, _) => filter.matches(resource.toPattern) }
 
   }
 
diff --git a/core/src/main/scala/kafka/common/TopicAndPartition.scala b/core/src/main/scala/kafka/common/TopicAndPartition.scala
deleted file mode 100644
index 6c276952f5aa7..0000000000000
--- a/core/src/main/scala/kafka/common/TopicAndPartition.scala
+++ /dev/null
@@ -1,30 +0,0 @@
-package kafka.common
-
-import org.apache.kafka.common.TopicPartition
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Convenience case class since (topic, partition) pairs are ubiquitous.
- */
-case class TopicAndPartition(topic: String, partition: Int) {
-
-  def this(topicPartition: TopicPartition) = this(topicPartition.topic, topicPartition.partition)
-
-  override def toString: String = s"$topic-$partition"
-}
diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala
index 75aeba2a10646..4ca4b49109ba0 100644
--- a/core/src/main/scala/kafka/controller/KafkaController.scala
+++ b/core/src/main/scala/kafka/controller/KafkaController.scala
@@ -976,7 +976,7 @@ class KafkaController(val config: KafkaConfig,
               finalLeaderIsrAndControllerEpoch = Some(LeaderIsrAndControllerEpoch(leaderAndIsr, epoch))
               info(s"Updated leader epoch for partition $partition to ${leaderAndIsr.leaderEpoch}")
               true
-            case (partition, Left(e)) =>
+            case (_, Left(e)) =>
               throw e
           }.getOrElse(false)
         case None =>
diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
index 6ef8ff39870ed..c6669755c5de4 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
@@ -312,7 +312,7 @@ class GroupMetadataManager(brokerId: Int,
     // construct the message set to append
     if (filteredOffsetMetadata.isEmpty) {
       // compute the final error codes for the commit response
-      val commitStatus = offsetMetadata.map { case (k, v) => k -> Errors.OFFSET_METADATA_TOO_LARGE }
+      val commitStatus = offsetMetadata.map { case (k, _) => k -> Errors.OFFSET_METADATA_TOO_LARGE }
       responseCallback(commitStatus)
       None
     } else {
diff --git a/core/src/main/scala/kafka/log/LogCleanerManager.scala b/core/src/main/scala/kafka/log/LogCleanerManager.scala
index 2397e02c8f489..d46dd944bb2b9 100755
--- a/core/src/main/scala/kafka/log/LogCleanerManager.scala
+++ b/core/src/main/scala/kafka/log/LogCleanerManager.scala
@@ -333,7 +333,7 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
       case None => false
       case Some(state) =>
         state match {
-          case LogCleaningPaused(s) =>
+          case _: LogCleaningPaused =>
             true
           case _ =>
             false
diff --git a/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala b/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala
index 2eba792e81638..b4cbcfa604cb6 100644
--- a/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala
+++ b/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala
@@ -271,15 +271,14 @@ class SimpleAclAuthorizer extends Authorizer with Logging {
         for (rType <- resourceTypes) {
           val resourceType = Try(ResourceType.fromString(rType))
           resourceType match {
-            case Success(resourceTypeObj) => {
+            case Success(resourceTypeObj) =>
               val resourceNames = zkClient.getResourceNames(store.patternType, resourceTypeObj)
               for (resourceName <- resourceNames) {
                 val resource = new Resource(resourceTypeObj, resourceName, store.patternType)
                 val versionedAcls = getAclsFromZk(resource)
                 updateCache(resource, versionedAcls)
               }
-            }
-            case Failure(f) => warn(s"Ignoring unknown ResourceType: $rType")
+            case Failure(_) => warn(s"Ignoring unknown ResourceType: $rType")
           }
         }
       })
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index ecd6c9453201c..2a0fbd35d049b 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -1336,7 +1336,7 @@ class KafkaApis(val requestChannel: RequestChannel,
   private def authorizedOperations(session: RequestChannel.Session, resource: Resource): Int = {
     val authorizedOps = authorizer match {
       case None => resource.resourceType.supportedOperations
-      case Some(auth) => resource.resourceType.supportedOperations
+      case Some(_) => resource.resourceType.supportedOperations
         .filter(operation => authorize(session, operation, resource))
     }
 
diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index 4765031e1146f..5223ebd058e08 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -1311,8 +1311,8 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO
 
   def interBrokerListenerName = getInterBrokerListenerNameAndSecurityProtocol._1
   def interBrokerSecurityProtocol = getInterBrokerListenerNameAndSecurityProtocol._2
-  def controlPlaneListenerName = getControlPlaneListenerNameAndSecurityProtocol.map { case (listenerName, securityProtocol) => listenerName }
-  def controlPlaneSecurityProtocol = getControlPlaneListenerNameAndSecurityProtocol.map { case (listenerName, securityProtocol) => securityProtocol }
+  def controlPlaneListenerName = getControlPlaneListenerNameAndSecurityProtocol.map { case (listenerName, _) => listenerName }
+  def controlPlaneSecurityProtocol = getControlPlaneListenerNameAndSecurityProtocol.map { case (_, securityProtocol) => securityProtocol }
   def saslMechanismInterBrokerProtocol = getString(KafkaConfig.SaslMechanismInterBrokerProtocolProp)
   val saslInterBrokerHandshakeRequestEnable = interBrokerProtocolVersion >= KAFKA_0_10_0_IV1
 
diff --git a/core/src/main/scala/kafka/zk/KafkaZkClient.scala b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
index ce824927dd615..ad3069ce1e7b8 100644
--- a/core/src/main/scala/kafka/zk/KafkaZkClient.scala
+++ b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
@@ -1721,7 +1721,7 @@ class KafkaZkClient private[zk] (zooKeeperClient: ZooKeeperClient, isSecure: Boo
           SetDataOp(path, data, 0)))
       )
       val stat = response.resultCode match {
-        case code@ Code.OK =>
+        case Code.OK =>
           val setDataResult = response.zkOpResults(1).rawOpResult.asInstanceOf[SetDataResult]
           setDataResult.getStat
         case Code.NODEEXISTS =>

From 39bcc8447c906506d63b8df156cf90174bbb8b78 Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Sun, 28 Jul 2019 18:18:59 -0700
Subject: [PATCH 0483/1071] MINOR: Fix Streams metadata upgrade system test
 (#7118)

Reviewers: A. Sophie Blee-Goldmann , Guozhang Wang 
---
 bin/kafka-run-class.sh | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/bin/kafka-run-class.sh b/bin/kafka-run-class.sh
index 44e20bafcbd95..7098925b940c4 100755
--- a/bin/kafka-run-class.sh
+++ b/bin/kafka-run-class.sh
@@ -57,10 +57,12 @@ fi
 
 # run ./gradlew copyDependantLibs to get all dependant jars in a local dir
 shopt -s nullglob
-for dir in "$base_dir"/core/build/dependant-libs-${SCALA_VERSION}*;
-do
-  CLASSPATH="$CLASSPATH:$dir/*"
-done
+if [ -z "$UPGRADE_KAFKA_STREAMS_TEST_VERSION" ]; then
+  for dir in "$base_dir"/core/build/dependant-libs-${SCALA_VERSION}*;
+  do
+    CLASSPATH="$CLASSPATH:$dir/*"
+  done
+fi
 
 for file in "$base_dir"/examples/build/libs/kafka-examples*.jar;
 do
@@ -110,6 +112,14 @@ else
       CLASSPATH="$file":"$CLASSPATH"
     fi
   done
+  if [ "$SHORT_VERSION_NO_DOTS" = "0100" ]; then
+    CLASSPATH="/opt/kafka-$UPGRADE_KAFKA_STREAMS_TEST_VERSION/libs/zkclient-0.8.jar":"$CLASSPATH"
+    CLASSPATH="/opt/kafka-$UPGRADE_KAFKA_STREAMS_TEST_VERSION/libs/zookeeper-3.4.6.jar":"$CLASSPATH"
+  fi
+  if [ "$SHORT_VERSION_NO_DOTS" = "0101" ]; then
+    CLASSPATH="/opt/kafka-$UPGRADE_KAFKA_STREAMS_TEST_VERSION/libs/zkclient-0.9.jar":"$CLASSPATH"
+    CLASSPATH="/opt/kafka-$UPGRADE_KAFKA_STREAMS_TEST_VERSION/libs/zookeeper-3.4.8.jar":"$CLASSPATH"
+  fi
 fi
 
 for file in "$rocksdb_lib_dir"/rocksdb*.jar;

From 81900d0ba0a5839a1e5dc876897bab1c24b3bd94 Mon Sep 17 00:00:00 2001
From: Stanislav Kozlovski 
Date: Mon, 29 Jul 2019 22:35:55 +0100
Subject: [PATCH 0484/1071] KAFKA-8345: KIP-455 Protocol changes (part 1)
 (#7114)

Add a new exception, NoReassignmentInProgressException.  Modify LeaderAndIsrRequest to include the AddingRepicas and RemovingReplicas fields.  Add the ListPartitionReassignments and AlterPartitionReassignments RPCs.

Reviewers: Colin P. McCabe , Viktor Somogyi 
---
 .../NoReassignmentInProgressException.java    |  31 +++++
 .../apache/kafka/common/protocol/ApiKeys.java |  10 +-
 .../apache/kafka/common/protocol/Errors.java  |   5 +-
 .../common/requests/AbstractRequest.java      |   4 +
 .../common/requests/AbstractResponse.java     |   4 +
 .../AlterPartitionReassignmentsRequest.java   | 114 ++++++++++++++++++
 .../AlterPartitionReassignmentsResponse.java  |  84 +++++++++++++
 .../common/requests/LeaderAndIsrRequest.java  |  86 ++++++++++++-
 .../common/requests/LeaderAndIsrResponse.java |   4 +-
 .../ListPartitionReassignmentsRequest.java    | 104 ++++++++++++++++
 .../ListPartitionReassignmentsResponse.java   |  75 ++++++++++++
 .../AlterPartitionReassignmentsRequest.json   |  37 ++++++
 .../AlterPartitionReassignmentsResponse.json  |  43 +++++++
 .../common/message/LeaderAndIsrRequest.json   |   8 +-
 .../common/message/LeaderAndIsrResponse.json  |   4 +-
 .../ListPartitionReassignmentsRequest.json    |  32 +++++
 .../ListPartitionReassignmentsResponse.json   |  45 +++++++
 .../kafka/common/message/MessageTest.java     |  43 ++++++-
 .../common/requests/RequestResponseTest.java  |  67 ++++++++++
 .../main/scala/kafka/server/KafkaApis.scala   |  30 +++++
 .../kafka/api/AuthorizerIntegrationTest.scala |  38 +++++-
 .../unit/kafka/server/RequestQuotaTest.scala  |  14 +++
 22 files changed, 867 insertions(+), 15 deletions(-)
 create mode 100644 clients/src/main/java/org/apache/kafka/common/errors/NoReassignmentInProgressException.java
 create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsRequest.java
 create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java
 create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsRequest.java
 create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java
 create mode 100644 clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json
 create mode 100644 clients/src/main/resources/common/message/AlterPartitionReassignmentsResponse.json
 create mode 100644 clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json
 create mode 100644 clients/src/main/resources/common/message/ListPartitionReassignmentsResponse.json

diff --git a/clients/src/main/java/org/apache/kafka/common/errors/NoReassignmentInProgressException.java b/clients/src/main/java/org/apache/kafka/common/errors/NoReassignmentInProgressException.java
new file mode 100644
index 0000000000000..9fd8a73c80916
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/errors/NoReassignmentInProgressException.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.common.errors;
+
+/**
+ * Thrown if a reassignment cannot be cancelled because none is in progress.
+ */
+public class NoReassignmentInProgressException extends ApiException {
+    public NoReassignmentInProgressException(String message) {
+        super(message);
+    }
+
+    public NoReassignmentInProgressException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
index 171a3cd412b34..18d8fd62e9766 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
@@ -44,6 +44,10 @@
 import org.apache.kafka.common.message.LeaveGroupResponseData;
 import org.apache.kafka.common.message.ListGroupsRequestData;
 import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData;
+import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData;
+import org.apache.kafka.common.message.ListPartitionReassignmentsRequestData;
+import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData;
 import org.apache.kafka.common.message.MetadataRequestData;
 import org.apache.kafka.common.message.MetadataResponseData;
 import org.apache.kafka.common.message.OffsetCommitRequestData;
@@ -193,7 +197,11 @@ public Struct parseResponse(short version, ByteBuffer buffer) {
     ELECT_LEADERS(43, "ElectLeaders", ElectLeadersRequestData.SCHEMAS,
             ElectLeadersResponseData.SCHEMAS),
     INCREMENTAL_ALTER_CONFIGS(44, "IncrementalAlterConfigs", IncrementalAlterConfigsRequestData.SCHEMAS,
-                              IncrementalAlterConfigsResponseData.SCHEMAS);
+                              IncrementalAlterConfigsResponseData.SCHEMAS),
+    ALTER_PARTITION_REASSIGNMENTS(45, "AlterPartitionReassignments", AlterPartitionReassignmentsRequestData.SCHEMAS,
+                                  AlterPartitionReassignmentsResponseData.SCHEMAS),
+    LIST_PARTITION_REASSIGNMENTS(46, "ListPartitionReassignments", ListPartitionReassignmentsRequestData.SCHEMAS,
+                                 ListPartitionReassignmentsResponseData.SCHEMAS);
 
     private static final ApiKeys[] ID_TO_TYPE;
     private static final int MIN_API_KEY = 0;
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java
index 89bc0515d7dfa..9f11fc89adf05 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java
@@ -65,6 +65,7 @@
 import org.apache.kafka.common.errors.ElectionNotNeededException;
 import org.apache.kafka.common.errors.EligibleLeadersNotAvailableException;
 import org.apache.kafka.common.errors.NetworkException;
+import org.apache.kafka.common.errors.NoReassignmentInProgressException;
 import org.apache.kafka.common.errors.NotControllerException;
 import org.apache.kafka.common.errors.NotCoordinatorException;
 import org.apache.kafka.common.errors.NotEnoughReplicasAfterAppendException;
@@ -309,7 +310,9 @@ public enum Errors {
             FencedInstanceIdException::new),
     ELIGIBLE_LEADERS_NOT_AVAILABLE(83, "Eligible topic partition leaders are not available",
             EligibleLeadersNotAvailableException::new),
-    ELECTION_NOT_NEEDED(84, "Leader election not needed for topic partition", ElectionNotNeededException::new);
+    ELECTION_NOT_NEEDED(84, "Leader election not needed for topic partition", ElectionNotNeededException::new),
+    NO_REASSIGNMENT_IN_PROGRESS(85, "No partition reassignment is in progress.",
+            NoReassignmentInProgressException::new);
 
     private static final Logger log = LoggerFactory.getLogger(Errors.class);
 
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
index c8ff90d0fdea4..58bf1283a77a1 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
@@ -233,6 +233,10 @@ public static AbstractRequest parseRequest(ApiKeys apiKey, short apiVersion, Str
                 return new ElectLeadersRequest(struct, apiVersion);
             case INCREMENTAL_ALTER_CONFIGS:
                 return new IncrementalAlterConfigsRequest(struct, apiVersion);
+            case ALTER_PARTITION_REASSIGNMENTS:
+                return new AlterPartitionReassignmentsRequest(struct, apiVersion);
+            case LIST_PARTITION_REASSIGNMENTS:
+                return new ListPartitionReassignmentsRequest(struct, apiVersion);
             default:
                 throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseRequest`, the " +
                         "code should be updated to do so.", apiKey));
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
index 9eddf66b6fb7b..2e433e8b52515 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
@@ -160,6 +160,10 @@ public static AbstractResponse parseResponse(ApiKeys apiKey, Struct struct, shor
                 return new ElectLeadersResponse(struct, version);
             case INCREMENTAL_ALTER_CONFIGS:
                 return new IncrementalAlterConfigsResponse(struct, version);
+            case ALTER_PARTITION_REASSIGNMENTS:
+                return new AlterPartitionReassignmentsResponse(struct, version);
+            case LIST_PARTITION_REASSIGNMENTS:
+                return new ListPartitionReassignmentsResponse(struct, version);
             default:
                 throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseResponse`, the " +
                         "code should be updated to do so.", apiKey));
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsRequest.java
new file mode 100644
index 0000000000000..7b2f848f614f9
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsRequest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.common.requests;
+
+import org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData;
+import org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData.ReassignableTopic;
+import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData;
+import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignablePartitionResponse;
+import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignableTopicResponse;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.types.Struct;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class AlterPartitionReassignmentsRequest extends AbstractRequest {
+
+    public static class Builder extends AbstractRequest.Builder {
+        private final AlterPartitionReassignmentsRequestData data;
+
+        public Builder(AlterPartitionReassignmentsRequestData data) {
+            super(ApiKeys.ALTER_PARTITION_REASSIGNMENTS);
+            this.data = data;
+        }
+
+        @Override
+        public AlterPartitionReassignmentsRequest build(short version) {
+            return new AlterPartitionReassignmentsRequest(data, version);
+        }
+
+        @Override
+        public String toString() {
+            return data.toString();
+        }
+    }
+
+    private final AlterPartitionReassignmentsRequestData data;
+    private final short version;
+
+    private AlterPartitionReassignmentsRequest(AlterPartitionReassignmentsRequestData data, short version) {
+        super(ApiKeys.ALTER_PARTITION_REASSIGNMENTS, version);
+        this.data = data;
+        this.version = version;
+    }
+
+    AlterPartitionReassignmentsRequest(Struct struct, short version) {
+        super(ApiKeys.ALTER_PARTITION_REASSIGNMENTS, version);
+        this.data = new AlterPartitionReassignmentsRequestData(struct, version);
+        this.version = version;
+    }
+
+    public static AlterPartitionReassignmentsRequest parse(ByteBuffer buffer, short version) {
+        return new AlterPartitionReassignmentsRequest(
+                ApiKeys.ALTER_PARTITION_REASSIGNMENTS.parseRequest(version, buffer),
+                version
+        );
+    }
+
+    public AlterPartitionReassignmentsRequestData data() {
+        return data;
+    }
+
+    /**
+     * Visible for testing.
+     */
+    @Override
+    public Struct toStruct() {
+        return data.toStruct(version);
+    }
+
+    @Override
+    public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
+        ApiError apiError = ApiError.fromThrowable(e);
+        List topicResponses = new ArrayList<>();
+
+        for (ReassignableTopic topic : data.topics()) {
+            List partitionResponses = topic.partitions().stream().map(partition ->
+                    new ReassignablePartitionResponse()
+                            .setPartitionIndex(partition.partitionIndex())
+                            .setErrorCode(apiError.error().code())
+                            .setErrorMessage(apiError.message())
+            ).collect(Collectors.toList());
+            topicResponses.add(
+                    new ReassignableTopicResponse()
+                            .setName(topic.name())
+                            .setPartitions(partitionResponses)
+            );
+        }
+
+        AlterPartitionReassignmentsResponseData responseData = new AlterPartitionReassignmentsResponseData()
+                .setResponses(topicResponses)
+                .setErrorCode(apiError.error().code())
+                .setErrorMessage(apiError.message())
+                .setThrottleTimeMs(throttleTimeMs);
+        return new AlterPartitionReassignmentsResponse(responseData);
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java
new file mode 100644
index 0000000000000..db1cfabebfa8b
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.common.requests;
+
+import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData;
+import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignableTopicResponse;
+import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignablePartitionResponse;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.protocol.types.Struct;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+
+public class AlterPartitionReassignmentsResponse extends AbstractResponse {
+
+    private final AlterPartitionReassignmentsResponseData data;
+
+    public AlterPartitionReassignmentsResponse(Struct struct) {
+        this(struct, ApiKeys.ALTER_PARTITION_REASSIGNMENTS.latestVersion());
+    }
+
+    AlterPartitionReassignmentsResponse(AlterPartitionReassignmentsResponseData data) {
+        this.data = data;
+    }
+
+    AlterPartitionReassignmentsResponse(Struct struct, short version) {
+        this.data = new AlterPartitionReassignmentsResponseData(struct, version);
+    }
+
+    public static AlterPartitionReassignmentsResponse parse(ByteBuffer buffer, short version) {
+        return new AlterPartitionReassignmentsResponse(ApiKeys.ALTER_PARTITION_REASSIGNMENTS.responseSchema(version).read(buffer), version);
+    }
+
+    public AlterPartitionReassignmentsResponseData data() {
+        return data;
+    }
+
+    @Override
+    public boolean shouldClientThrottle(short version) {
+        return true;
+    }
+
+    @Override
+    public int throttleTimeMs() {
+        return data.throttleTimeMs();
+    }
+
+    @Override
+    public Map errorCounts() {
+        Map counts = new HashMap<>();
+        Errors topLevelErr = Errors.forCode(data.errorCode());
+        counts.put(topLevelErr, counts.getOrDefault(topLevelErr, 0) + 1);
+
+        for (ReassignableTopicResponse topicResponse : data.responses()) {
+            for (ReassignablePartitionResponse partitionResponse : topicResponse.partitions()) {
+                Errors error = Errors.forCode(partitionResponse.errorCode());
+                counts.put(error, counts.getOrDefault(error, 0) + 1);
+            }
+        }
+        return counts;
+    }
+
+    @Override
+    protected Struct toStruct(short version) {
+        return data.toStruct(version);
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
index 230c80734408a..02c14e3274a99 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
@@ -33,6 +33,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.Collections;
 
 import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
 import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
@@ -49,6 +50,10 @@ public class LeaderAndIsrRequest extends AbstractControlRequest {
     private static final Field.Array ISR = new Field.Array("isr", INT32, "The in sync replica ids.");
     private static final Field.Int32 ZK_VERSION = new Field.Int32("zk_version", "The ZK version.");
     private static final Field.Array REPLICAS = new Field.Array("replicas", INT32, "The replica ids.");
+    private static final Field.Array ADDING_REPLICAS = new Field.Array("adding_replicas", INT32,
+            "The replica ids we are in the process of adding to the replica set during a reassignment.");
+    private static final Field.Array REMOVING_REPLICAS = new Field.Array("removing_replicas", INT32,
+            "The replica ids we are in the process of removing from the replica set during a reassignment.");
     private static final Field.Bool IS_NEW = new Field.Bool("is_new", "Whether the replica should have existed on the broker or not");
 
     // live_leaders fields
@@ -89,11 +94,28 @@ public class LeaderAndIsrRequest extends AbstractControlRequest {
             REPLICAS,
             IS_NEW);
 
+    private static final Field PARTITION_STATES_V3 = PARTITION_STATES.withFields(
+            PARTITION_ID,
+            CONTROLLER_EPOCH,
+            LEADER,
+            LEADER_EPOCH,
+            ISR,
+            ZK_VERSION,
+            REPLICAS,
+            ADDING_REPLICAS,
+            REMOVING_REPLICAS,
+            IS_NEW);
+
     // TOPIC_STATES_V2 normalizes TOPIC_STATES_V1 to make it more memory efficient
     private static final Field TOPIC_STATES_V2 = TOPIC_STATES.withFields(
             TOPIC_NAME,
             PARTITION_STATES_V2);
 
+    // TOPIC_STATES_V3 adds two new fields - adding_replicas and removing_replicas
+    private static final Field TOPIC_STATES_V3 = TOPIC_STATES.withFields(
+            TOPIC_NAME,
+            PARTITION_STATES_V3);
+
     private static final Field LIVE_LEADERS_V0 = LIVE_LEADERS.withFields(
             END_POINT_ID,
             HOST,
@@ -122,8 +144,17 @@ public class LeaderAndIsrRequest extends AbstractControlRequest {
             TOPIC_STATES_V2,
             LIVE_LEADERS_V0);
 
+    // LEADER_AND_ISR_REQUEST_V3 added two new fields - adding_replicas and removing_replicas.
+    // These fields respectively specify the replica IDs we want to add or remove as part of a reassignment
+    private static final Schema LEADER_AND_ISR_REQUEST_V3 = new Schema(
+            CONTROLLER_ID,
+            CONTROLLER_EPOCH,
+            BROKER_EPOCH,
+            TOPIC_STATES_V3,
+            LIVE_LEADERS_V0);
+
     public static Schema[] schemaVersions() {
-        return new Schema[]{LEADER_AND_ISR_REQUEST_V0, LEADER_AND_ISR_REQUEST_V1, LEADER_AND_ISR_REQUEST_V2};
+        return new Schema[]{LEADER_AND_ISR_REQUEST_V0, LEADER_AND_ISR_REQUEST_V1, LEADER_AND_ISR_REQUEST_V2, LEADER_AND_ISR_REQUEST_V3};
     }
 
     public static class Builder extends AbstractControlRequest.Builder {
@@ -223,7 +254,7 @@ protected Struct toStruct() {
                 for (Map.Entry partitionEntry : partitionMap.entrySet()) {
                     Struct partitionStateData = topicStateData.instance(PARTITION_STATES);
                     partitionStateData.set(PARTITION_ID, partitionEntry.getKey());
-                    partitionEntry.getValue().setStruct(partitionStateData);
+                    partitionEntry.getValue().setStruct(partitionStateData, version);
                     partitionStatesData.add(partitionStateData);
                 }
                 topicStateData.set(PARTITION_STATES, partitionStatesData.toArray());
@@ -237,7 +268,7 @@ protected Struct toStruct() {
                 TopicPartition topicPartition = entry.getKey();
                 partitionStateData.set(TOPIC_NAME, topicPartition.topic());
                 partitionStateData.set(PARTITION_ID, topicPartition.partition());
-                entry.getValue().setStruct(partitionStateData);
+                entry.getValue().setStruct(partitionStateData, version);
                 partitionStatesData.add(partitionStateData);
             }
             struct.set(PARTITION_STATES, partitionStatesData.toArray());
@@ -269,6 +300,7 @@ public LeaderAndIsrResponse getErrorResponse(int throttleTimeMs, Throwable e) {
             case 0:
             case 1:
             case 2:
+            case 3:
                 return new LeaderAndIsrResponse(error, responses);
             default:
                 throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
@@ -298,6 +330,8 @@ public static LeaderAndIsrRequest parse(ByteBuffer buffer, short version) {
 
     public static final class PartitionState {
         public final BasePartitionState basePartitionState;
+        public final List addingReplicas;
+        public final List removingReplicas;
         public final boolean isNew;
 
         public PartitionState(int controllerEpoch,
@@ -307,7 +341,29 @@ public PartitionState(int controllerEpoch,
                               int zkVersion,
                               List replicas,
                               boolean isNew) {
+            this(controllerEpoch,
+                 leader,
+                 leaderEpoch,
+                 isr,
+                 zkVersion,
+                 replicas,
+                 Collections.emptyList(),
+                 Collections.emptyList(),
+                 isNew);
+        }
+
+        public PartitionState(int controllerEpoch,
+                              int leader,
+                              int leaderEpoch,
+                              List isr,
+                              int zkVersion,
+                              List replicas,
+                              List addingReplicas,
+                              List removingReplicas,
+                              boolean isNew) {
             this.basePartitionState = new BasePartitionState(controllerEpoch, leader, leaderEpoch, isr, zkVersion, replicas);
+            this.addingReplicas = addingReplicas;
+            this.removingReplicas = removingReplicas;
             this.isNew = isNew;
         }
 
@@ -329,6 +385,21 @@ private PartitionState(Struct struct) {
                 replicas.add((Integer) r);
 
             this.basePartitionState = new BasePartitionState(controllerEpoch, leader, leaderEpoch, isr, zkVersion, replicas);
+
+            List addingReplicas = new ArrayList<>();
+            if (struct.hasField(ADDING_REPLICAS)) {
+                for (Object r : struct.get(ADDING_REPLICAS))
+                    addingReplicas.add((Integer) r);
+            }
+            this.addingReplicas = addingReplicas;
+
+            List removingReplicas = new ArrayList<>();
+            if (struct.hasField(REMOVING_REPLICAS)) {
+                for (Object r : struct.get(REMOVING_REPLICAS))
+                    removingReplicas.add((Integer) r);
+            }
+            this.removingReplicas = removingReplicas;
+
             this.isNew = struct.getOrElse(IS_NEW, false);
         }
 
@@ -340,18 +411,23 @@ public String toString() {
                 ", isr=" + Utils.join(basePartitionState.isr, ",") +
                 ", zkVersion=" + basePartitionState.zkVersion +
                 ", replicas=" + Utils.join(basePartitionState.replicas, ",") +
+                ", addingReplicas=" + Utils.join(addingReplicas, ",") +
+                ", removingReplicas=" + Utils.join(removingReplicas, ",") +
                 ", isNew=" + isNew + ")";
         }
 
-        private void setStruct(Struct struct) {
+        private void setStruct(Struct struct, short version) {
             struct.set(CONTROLLER_EPOCH, basePartitionState.controllerEpoch);
             struct.set(LEADER, basePartitionState.leader);
             struct.set(LEADER_EPOCH, basePartitionState.leaderEpoch);
             struct.set(ISR, basePartitionState.isr.toArray());
             struct.set(ZK_VERSION, basePartitionState.zkVersion);
             struct.set(REPLICAS, basePartitionState.replicas.toArray());
+            if (version >= 3) {
+                struct.set(ADDING_REPLICAS, addingReplicas.toArray());
+                struct.set(REMOVING_REPLICAS, removingReplicas.toArray());
+            }
             struct.setIfExists(IS_NEW, isNew);
         }
     }
-
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java
index 3ab9bf79de4a2..3b80222eff0e7 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java
@@ -50,8 +50,10 @@ public class LeaderAndIsrResponse extends AbstractResponse {
 
     private static final Schema LEADER_AND_ISR_RESPONSE_V2 = LEADER_AND_ISR_RESPONSE_V1;
 
+    private static final Schema LEADER_AND_ISR_RESPONSE_V3 = LEADER_AND_ISR_RESPONSE_V2;
+
     public static Schema[] schemaVersions() {
-        return new Schema[]{LEADER_AND_ISR_RESPONSE_V0, LEADER_AND_ISR_RESPONSE_V1, LEADER_AND_ISR_RESPONSE_V2};
+        return new Schema[]{LEADER_AND_ISR_RESPONSE_V0, LEADER_AND_ISR_RESPONSE_V1, LEADER_AND_ISR_RESPONSE_V2, LEADER_AND_ISR_RESPONSE_V3};
     }
 
     /**
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsRequest.java
new file mode 100644
index 0000000000000..471147bdb2a25
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsRequest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.requests;
+
+import org.apache.kafka.common.message.ListPartitionReassignmentsRequestData;
+import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData;
+import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingPartitionReassignment;
+import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingTopicReassignment;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.types.Struct;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.stream.Collectors;
+
+
+public class ListPartitionReassignmentsRequest extends AbstractRequest {
+
+    public static class Builder extends AbstractRequest.Builder {
+        private final ListPartitionReassignmentsRequestData data;
+
+        public Builder(ListPartitionReassignmentsRequestData data) {
+            super(ApiKeys.LIST_PARTITION_REASSIGNMENTS);
+            this.data = data;
+        }
+
+        @Override
+        public ListPartitionReassignmentsRequest build(short version) {
+            return new ListPartitionReassignmentsRequest(data, version);
+        }
+
+        @Override
+        public String toString() {
+            return data.toString();
+        }
+    }
+
+    private ListPartitionReassignmentsRequestData data;
+    private final short version;
+
+    private ListPartitionReassignmentsRequest(ListPartitionReassignmentsRequestData data, short version) {
+        super(ApiKeys.LIST_PARTITION_REASSIGNMENTS, version);
+        this.data = data;
+        this.version = version;
+    }
+
+    ListPartitionReassignmentsRequest(Struct struct, short version) {
+        super(ApiKeys.LIST_PARTITION_REASSIGNMENTS, version);
+        this.data = new ListPartitionReassignmentsRequestData(struct, version);
+        this.version = version;
+    }
+
+    public static ListPartitionReassignmentsRequest parse(ByteBuffer buffer, short version) {
+        return new ListPartitionReassignmentsRequest(
+                ApiKeys.LIST_PARTITION_REASSIGNMENTS.parseRequest(version, buffer), version
+        );
+    }
+
+    public ListPartitionReassignmentsRequestData data() {
+        return data;
+    }
+
+    /**
+     * Visible for testing.
+     */
+    @Override
+    public Struct toStruct() {
+        return data.toStruct(version);
+    }
+
+    @Override
+    public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
+        ApiError apiError = ApiError.fromThrowable(e);
+
+        List ongoingTopicReassignments = data.topics().stream().map(topic ->
+                new OngoingTopicReassignment()
+                        .setName(topic.name())
+                        .setPartitions(topic.partitionIndexes().stream().map(partitionIndex ->
+                                new OngoingPartitionReassignment().setPartitionIndex(partitionIndex)).collect(Collectors.toList())
+                        )
+        ).collect(Collectors.toList());
+
+        ListPartitionReassignmentsResponseData responseData = new ListPartitionReassignmentsResponseData()
+                .setTopics(ongoingTopicReassignments)
+                .setErrorCode(apiError.error().code())
+                .setErrorMessage(apiError.message())
+                .setThrottleTimeMs(throttleTimeMs);
+        return new ListPartitionReassignmentsResponse(responseData);
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java
new file mode 100644
index 0000000000000..9513e88303044
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.requests;
+
+import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.protocol.types.Struct;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+
+public class ListPartitionReassignmentsResponse extends AbstractResponse {
+
+    private final ListPartitionReassignmentsResponseData data;
+
+    public ListPartitionReassignmentsResponse(Struct struct) {
+        this(struct, ApiKeys.LIST_PARTITION_REASSIGNMENTS.latestVersion());
+    }
+
+    ListPartitionReassignmentsResponse(ListPartitionReassignmentsResponseData responseData) {
+        this.data = responseData;
+    }
+
+    ListPartitionReassignmentsResponse(Struct struct, short version) {
+        this.data = new ListPartitionReassignmentsResponseData(struct, version);
+    }
+
+    public static ListPartitionReassignmentsResponse parse(ByteBuffer buffer, short version) {
+        return new ListPartitionReassignmentsResponse(ApiKeys.LIST_PARTITION_REASSIGNMENTS.responseSchema(version).read(buffer), version);
+    }
+
+    public ListPartitionReassignmentsResponseData data() {
+        return data;
+    }
+
+    @Override
+    public boolean shouldClientThrottle(short version) {
+        return true;
+    }
+
+    @Override
+    public int throttleTimeMs() {
+        return data.throttleTimeMs();
+    }
+
+    @Override
+    public Map errorCounts() {
+        Map counts = new HashMap<>();
+        Errors topLevelErr = Errors.forCode(data.errorCode());
+        counts.put(topLevelErr, 1);
+
+        return counts;
+    }
+
+    @Override
+    protected Struct toStruct(short version) {
+        return data.toStruct(version);
+    }
+}
diff --git a/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json b/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json
new file mode 100644
index 0000000000000..f962e1e65db37
--- /dev/null
+++ b/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+{
+  "apiKey": 45,
+  "type": "request",
+  "name": "AlterPartitionReassignmentsRequest",
+  "validVersions": "0",
+  "fields": [
+    { "name": "TimeoutMs", "type": "int32", "versions": "0+", "default": "60000",
+      "about": "The time in ms to wait for the request to complete." },
+    { "name": "Topics", "type": "[]ReassignableTopic", "versions": "0+",
+      "about": "The topics to reassign.", "fields": [
+      { "name": "Name", "type": "string", "versions": "0+", "entityType": "topicName",
+        "about": "The topic name." },
+      { "name": "Partitions", "type": "[]ReassignablePartition", "versions": "0+",
+        "about": "The partitions to reassign.", "fields": [
+        { "name": "PartitionIndex", "type": "int32", "versions": "0+",
+          "about": "The partition index." },
+        { "name": "Replicas", "type": "[]int32", "versions": "0+", "nullableVersions": "0+", "default": "null",
+          "about": "The replicas to place the partitions on, or null to cancel a pending reassignment for this partition." }
+      ]}
+    ]}
+  ]
+}
diff --git a/clients/src/main/resources/common/message/AlterPartitionReassignmentsResponse.json b/clients/src/main/resources/common/message/AlterPartitionReassignmentsResponse.json
new file mode 100644
index 0000000000000..d04959678f64f
--- /dev/null
+++ b/clients/src/main/resources/common/message/AlterPartitionReassignmentsResponse.json
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+{
+  "apiKey": 45,
+  "type": "response",
+  "name": "AlterPartitionReassignmentsResponse",
+  "validVersions": "0",
+  "fields": [
+    { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
+      "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
+    { "name": "ErrorCode", "type": "int16", "versions": "0+",
+      "about": "The top-level error code, or 0 if there was no error." },
+    { "name": "ErrorMessage", "type": "string", "versions": "0+", "nullableVersions": "0+",
+      "about": "The top-level error message, or null if there was no error." },
+    { "name": "Responses", "type": "[]ReassignableTopicResponse", "versions": "0+",
+      "about": "The responses to topics to reassign.", "fields": [
+      { "name": "Name", "type": "string", "versions": "0+", "entityType": "topicName",
+        "about": "The topic name" },
+      { "name": "Partitions", "type": "[]ReassignablePartitionResponse", "versions": "0+",
+        "about": "The responses to partitions to reassign", "fields": [
+        { "name": "PartitionIndex", "type": "int32", "versions": "0+",
+          "about": "The partition index." },
+        { "name": "ErrorCode", "type": "int16", "versions": "0+",
+          "about": "The error code for this partition, or 0 if there was no error." },
+        { "name": "ErrorMessage", "type": "string", "versions": "0+", "nullableVersions": "0+",
+          "about": "The error message for this partition, or null if there was no error." }
+      ]}
+    ]}
+  ]
+}
diff --git a/clients/src/main/resources/common/message/LeaderAndIsrRequest.json b/clients/src/main/resources/common/message/LeaderAndIsrRequest.json
index a449b869ad8e9..c43d2f4b1633b 100644
--- a/clients/src/main/resources/common/message/LeaderAndIsrRequest.json
+++ b/clients/src/main/resources/common/message/LeaderAndIsrRequest.json
@@ -20,7 +20,9 @@
   // Version 1 adds IsNew.
   //
   // Version 2 adds broker epoch and reorganizes the partitions by topic.
-  "validVersions": "0-2",
+  //
+  // Version 3 adds AddingReplicas and RemovingReplicas
+  "validVersions": "0-3",
   "fields": [
     { "name": "ControllerId", "type": "int32", "versions": "0+", "entityType": "brokerId",
       "about": "The current controller ID." },
@@ -68,6 +70,10 @@
         "about": "The ZooKeeper version." },
       { "name": "Replicas", "type": "[]int32", "versions": "0+",
         "about": "The replica IDs." },
+      { "name": "AddingReplicas", "type": "[]int32", "versions": "3+", "ignorable": true,
+        "about": "The replica IDs that we are adding this partition to, or null if no replicas are being added." },
+      { "name": "RemovingReplicas", "type": "[]int32", "versions": "3+", "ignorable": true,
+        "about": "The replica IDs that we are removing this partition from, or null if no replicas are being removed." },
       { "name": "IsNew", "type": "bool", "versions": "1+", "default": "false", "ignorable": true,
         "about": "Whether the replica should have existed on the broker or not." }
     ]}
diff --git a/clients/src/main/resources/common/message/LeaderAndIsrResponse.json b/clients/src/main/resources/common/message/LeaderAndIsrResponse.json
index 8f4bf635c793b..06bb088e179ba 100644
--- a/clients/src/main/resources/common/message/LeaderAndIsrResponse.json
+++ b/clients/src/main/resources/common/message/LeaderAndIsrResponse.json
@@ -20,7 +20,9 @@
   // Version 1 adds KAFKA_STORAGE_ERROR as a valid error code.
   //
   // Version 2 is the same as version 1.
-  "validVersions": "0-2",
+  //
+  // Version 3 is the same as version 2.
+  "validVersions": "0-3",
   "fields": [
     { "name": "ErrorCode", "type": "int16", "versions": "0+",
       "about": "The error code, or 0 if there was no error." },
diff --git a/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json b/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json
new file mode 100644
index 0000000000000..d0ebf8bd7d622
--- /dev/null
+++ b/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+{
+  "apiKey": 46,
+  "type": "request",
+  "name": "ListPartitionReassignmentsRequest",
+  "validVersions": "0",
+  "fields": [
+    { "name": "TimeoutMs", "type": "int32", "versions": "0+", "default": "60000",
+      "about": "The time in ms to wait for the request to complete." },
+    { "name": "Topics", "type": "[]ListPartitionReassignmentsTopics", "versions": "0+", "nullableVersions": "0+",
+      "about": "The topics to list partition reassignments for, or null to list everything.", "fields": [
+      { "name": "Name", "type": "string", "versions": "0+", "entityType": "topicName",
+        "about": "The topic name" },
+      { "name": "PartitionIndexes", "type": "[]int32", "versions": "0+",
+        "about": "The partitions to list partition reassignments for." }
+    ]}
+  ]
+}
diff --git a/clients/src/main/resources/common/message/ListPartitionReassignmentsResponse.json b/clients/src/main/resources/common/message/ListPartitionReassignmentsResponse.json
new file mode 100644
index 0000000000000..b79e0523b32ee
--- /dev/null
+++ b/clients/src/main/resources/common/message/ListPartitionReassignmentsResponse.json
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+{
+  "apiKey": 46,
+  "type": "response",
+  "name": "ListPartitionReassignmentsResponse",
+  "validVersions": "0",
+  "fields": [
+    { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
+      "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
+    { "name": "ErrorCode", "type": "int16", "versions": "0+",
+      "about": "The top-level error code, or 0 if there was no error" },
+    { "name": "ErrorMessage", "type": "string", "versions": "0+", "nullableVersions": "0+",
+      "about": "The top-level error message, or null if there was no error." },
+    { "name": "Topics", "type": "[]OngoingTopicReassignment", "versions": "0+",
+      "about": "The ongoing reassignments for each topic.", "fields": [
+      { "name": "Name", "type": "string", "versions": "0+", "entityType": "topicName",
+        "about": "The topic name." },
+      { "name": "Partitions", "type": "[]OngoingPartitionReassignment", "versions": "0+",
+        "about": "The ongoing reassignments for each partition.", "fields": [
+        { "name": "PartitionIndex", "type": "int32", "versions": "0+",
+          "about": "The index of the partition." },
+        { "name": "Replicas", "type": "[]int32", "versions": "0+",
+          "about": "The current replica set." },
+        { "name": "AddingReplicas", "type": "[]int32", "versions": "0+",
+          "about": "The set of replicas we are currently adding." },
+        { "name": "RemovingReplicas", "type": "[]int32", "versions": "0+",
+          "about": "The set of replicas we are currently removing." }
+      ]}
+    ]}
+  ]
+}
diff --git a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
index 58c650a011c80..bdfce3f32f1b1 100644
--- a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
@@ -233,12 +233,53 @@ public void testOffsetForLeaderEpochVersions() throws Exception {
 
     }
 
+    @Test
+    public void testLeaderAndIsrVersions() throws Exception {
+        // Version 3 adds two new fields - AddingReplicas and RemovingReplicas
+        LeaderAndIsrRequestData.LeaderAndIsrRequestTopicState partitionStateNoAddingRemovingReplicas =
+                new LeaderAndIsrRequestData.LeaderAndIsrRequestTopicState()
+                        .setName("topic")
+                        .setPartitionStatesV0(
+                                Collections.singletonList(
+                                        new LeaderAndIsrRequestData.LeaderAndIsrRequestPartition()
+                                                .setPartitionIndex(0)
+                                                .setReplicas(Collections.singletonList(0))
+                                )
+                        );
+        LeaderAndIsrRequestData.LeaderAndIsrRequestTopicState partitionStateWithAddingRemovingReplicas =
+                new LeaderAndIsrRequestData.LeaderAndIsrRequestTopicState()
+                        .setName("topic")
+                        .setPartitionStatesV0(
+                                Collections.singletonList(
+                                        new LeaderAndIsrRequestData.LeaderAndIsrRequestPartition()
+                                                .setPartitionIndex(0)
+                                                .setReplicas(Collections.singletonList(0))
+                                                .setAddingReplicas(Collections.singletonList(1))
+                                                .setRemovingReplicas(Collections.singletonList(1))
+                                )
+                        );
+        testAllMessageRoundTripsBetweenVersions(
+                (short) 2,
+                (short) 3,
+                new LeaderAndIsrRequestData().setTopicStates(Collections.singletonList(partitionStateWithAddingRemovingReplicas)),
+                new LeaderAndIsrRequestData().setTopicStates(Collections.singletonList(partitionStateNoAddingRemovingReplicas)));
+        testAllMessageRoundTripsFromVersion((short) 3, new LeaderAndIsrRequestData().setTopicStates(Collections.singletonList(partitionStateWithAddingRemovingReplicas)));
+    }
+
     private void testAllMessageRoundTrips(Message message) throws Exception {
         testAllMessageRoundTripsFromVersion(message.lowestSupportedVersion(), message);
     }
 
     private void testAllMessageRoundTripsBeforeVersion(short beforeVersion, Message message, Message expected) throws Exception {
-        for (short version = 0; version < beforeVersion; version++) {
+        testAllMessageRoundTripsBetweenVersions((short) 0, beforeVersion, message, expected);
+    }
+
+    /**
+     * @param startVersion - the version we want to start at, inclusive
+     * @param endVersion - the version we want to end at, exclusive
+     */
+    private void testAllMessageRoundTripsBetweenVersions(short startVersion, short endVersion, Message message, Message expected) throws Exception {
+        for (short version = startVersion; version < endVersion; version++) {
             testMessageRoundTrip(version, message, expected);
         }
     }
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index 98552108a1ebc..70adcca9f8d71 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -66,6 +66,10 @@
 import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData;
 import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData.AlterConfigsResourceResponse;
 import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData;
+import org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData;
+import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData;
+import org.apache.kafka.common.message.ListPartitionReassignmentsRequestData;
+import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData;
 import org.apache.kafka.common.message.InitProducerIdRequestData;
 import org.apache.kafka.common.message.InitProducerIdResponseData;
 import org.apache.kafka.common.message.JoinGroupRequestData;
@@ -362,6 +366,12 @@ public void testSerialization() throws Exception {
         checkRequest(createIncrementalAlterConfigsRequest(), true);
         checkErrorResponse(createIncrementalAlterConfigsRequest(), new UnknownServerException(), true);
         checkResponse(createIncrementalAlterConfigsResponse(), 0, true);
+        checkRequest(createAlterPartitionReassignmentsRequest(), true);
+        checkErrorResponse(createAlterPartitionReassignmentsRequest(), new UnknownServerException(), true);
+        checkResponse(createAlterPartitionReassignmentsResponse(), 0, true);
+        checkRequest(createListPartitionReassignmentsRequest(), true);
+        checkErrorResponse(createListPartitionReassignmentsRequest(), new UnknownServerException(), true);
+        checkResponse(createListPartitionReassignmentsResponse(), 0, true);
     }
 
     @Test
@@ -1630,4 +1640,61 @@ private IncrementalAlterConfigsResponse createIncrementalAlterConfigsResponse()
                 .setErrorMessage("Duplicate Keys"));
         return new IncrementalAlterConfigsResponse(data);
     }
+
+    private AlterPartitionReassignmentsRequest createAlterPartitionReassignmentsRequest() {
+        AlterPartitionReassignmentsRequestData data = new AlterPartitionReassignmentsRequestData();
+        data.topics().add(
+                new AlterPartitionReassignmentsRequestData.ReassignableTopic().setName("topic").setPartitions(
+                        Collections.singletonList(
+                                new AlterPartitionReassignmentsRequestData.ReassignablePartition().setPartitionIndex(0).setReplicas(null)
+                        )
+                )
+        );
+        return new AlterPartitionReassignmentsRequest.Builder(data).build((short) 0);
+    }
+
+    private AlterPartitionReassignmentsResponse createAlterPartitionReassignmentsResponse() {
+        AlterPartitionReassignmentsResponseData data = new AlterPartitionReassignmentsResponseData();
+        data.responses().add(
+                new AlterPartitionReassignmentsResponseData.ReassignableTopicResponse()
+                        .setName("topic")
+                        .setPartitions(Collections.singletonList(
+                                new AlterPartitionReassignmentsResponseData.ReassignablePartitionResponse()
+                                        .setPartitionIndex(0)
+                                        .setErrorCode(Errors.NO_REASSIGNMENT_IN_PROGRESS.code())
+                                        .setErrorMessage("No reassignment is in progress for topic topic partition 0")
+                                )
+                        )
+        );
+        return new AlterPartitionReassignmentsResponse(data);
+    }
+
+    private ListPartitionReassignmentsRequest createListPartitionReassignmentsRequest() {
+        ListPartitionReassignmentsRequestData data = new ListPartitionReassignmentsRequestData();
+        data.setTopics(
+            Collections.singletonList(
+                new ListPartitionReassignmentsRequestData.ListPartitionReassignmentsTopics()
+                    .setName("topic")
+                    .setPartitionIndexes(Collections.singletonList(1))
+            )
+        );
+        return new ListPartitionReassignmentsRequest.Builder(data).build((short) 0);
+    }
+
+    private ListPartitionReassignmentsResponse createListPartitionReassignmentsResponse() {
+        ListPartitionReassignmentsResponseData data = new ListPartitionReassignmentsResponseData();
+        data.topics().add(
+                new ListPartitionReassignmentsResponseData.OngoingTopicReassignment()
+                        .setName("topic")
+                        .setPartitions(Collections.singletonList(
+                                new ListPartitionReassignmentsResponseData.OngoingPartitionReassignment()
+                                        .setPartitionIndex(0)
+                                        .setReplicas(Arrays.asList(1, 2))
+                                        .setAddingReplicas(Collections.singletonList(2))
+                                        .setRemovingReplicas(Collections.singletonList(1))
+                                )
+                        )
+        );
+        return new ListPartitionReassignmentsResponse(data);
+    }
 }
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 2a0fbd35d049b..3ec6b2314d193 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -52,6 +52,8 @@ import org.apache.kafka.common.message.CreateTopicsResponseData
 import org.apache.kafka.common.message.CreateTopicsResponseData.{CreatableTopicResult, CreatableTopicResultCollection}
 import org.apache.kafka.common.message.DeleteGroupsResponseData
 import org.apache.kafka.common.message.DeleteGroupsResponseData.{DeletableGroupResult, DeletableGroupResultCollection}
+import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData
+import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData
 import org.apache.kafka.common.message.DeleteTopicsResponseData
 import org.apache.kafka.common.message.DeleteTopicsResponseData.{DeletableTopicResult, DeletableTopicResultCollection}
 import org.apache.kafka.common.message.DescribeGroupsResponseData
@@ -178,6 +180,8 @@ class KafkaApis(val requestChannel: RequestChannel,
         case ApiKeys.DELETE_GROUPS => handleDeleteGroupsRequest(request)
         case ApiKeys.ELECT_LEADERS => handleElectReplicaLeader(request)
         case ApiKeys.INCREMENTAL_ALTER_CONFIGS => handleIncrementalAlterConfigsRequest(request)
+        case ApiKeys.ALTER_PARTITION_REASSIGNMENTS => handleAlterPartitionReassignmentsRequest(request)
+        case ApiKeys.LIST_PARTITION_REASSIGNMENTS => handleListPartitionReassignmentsRequest(request)
       }
     } catch {
       case e: FatalExitError => throw e
@@ -2299,6 +2303,32 @@ class KafkaApis(val requestChannel: RequestChannel,
       new AlterConfigsResponse(requestThrottleMs, (authorizedResult ++ unauthorizedResult).asJava))
   }
 
+  def handleAlterPartitionReassignmentsRequest(request: RequestChannel.Request): Unit = {
+    authorizeClusterAlter(request)
+    val alterPartitionReassignmentsRequest = request.body[AlterPartitionReassignmentsRequest]
+
+    sendResponseMaybeThrottle(request, requestThrottleMs =>
+      new AlterPartitionReassignmentsResponse(
+        new AlterPartitionReassignmentsResponseData().setThrottleTimeMs(requestThrottleMs)
+          .setErrorCode(Errors.UNSUPPORTED_VERSION.code()).setErrorMessage(Errors.UNSUPPORTED_VERSION.message())
+          .toStruct(0)
+      )
+    )
+  }
+
+  def handleListPartitionReassignmentsRequest(request: RequestChannel.Request): Unit = {
+    authorizeClusterDescribe(request)
+    val listPartitionReassignmentsRequest = request.body[ListPartitionReassignmentsRequest]
+
+    sendResponseMaybeThrottle(request, requestThrottleMs =>
+      new ListPartitionReassignmentsResponse(
+        new ListPartitionReassignmentsResponseData().setThrottleTimeMs(requestThrottleMs)
+          .setErrorCode(Errors.UNSUPPORTED_VERSION.code()).setErrorMessage(Errors.UNSUPPORTED_VERSION.message())
+          .toStruct(0)
+      )
+    )
+  }
+
   private def configsAuthorizationApiError(session: RequestChannel.Session, resource: ConfigResource): ApiError = {
     val error = resource.`type` match {
       case ConfigResource.Type.BROKER => Errors.CLUSTER_AUTHORIZATION_FAILED
diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
index cb5f1aadb041c..ef2e6fd5b13c4 100644
--- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
@@ -44,6 +44,8 @@ import org.apache.kafka.common.message.FindCoordinatorRequestData
 import org.apache.kafka.common.message.HeartbeatRequestData
 import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData
 import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData.{AlterConfigsResource, AlterableConfig, AlterableConfigCollection}
+import org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData
+import org.apache.kafka.common.message.ListPartitionReassignmentsRequestData
 import org.apache.kafka.common.message.JoinGroupRequestData
 import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection
 import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
@@ -164,7 +166,9 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
       ApiKeys.DESCRIBE_LOG_DIRS -> classOf[DescribeLogDirsResponse],
       ApiKeys.CREATE_PARTITIONS -> classOf[CreatePartitionsResponse],
       ApiKeys.ELECT_LEADERS -> classOf[ElectLeadersResponse],
-      ApiKeys.INCREMENTAL_ALTER_CONFIGS -> classOf[IncrementalAlterConfigsResponse]
+      ApiKeys.INCREMENTAL_ALTER_CONFIGS -> classOf[IncrementalAlterConfigsResponse],
+      ApiKeys.ALTER_PARTITION_REASSIGNMENTS -> classOf[AlterPartitionReassignmentsResponse],
+      ApiKeys.LIST_PARTITION_REASSIGNMENTS -> classOf[ListPartitionReassignmentsResponse]
     )
 
   val requestKeyToError = Map[ApiKeys, Nothing => Errors](
@@ -212,7 +216,9 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
     ApiKeys.CREATE_PARTITIONS -> ((resp: CreatePartitionsResponse) => resp.errors.asScala.find(_._1 == topic).get._2.error),
     ApiKeys.ELECT_LEADERS -> ((resp: ElectLeadersResponse) => Errors.forCode(resp.data().errorCode())),
     ApiKeys.INCREMENTAL_ALTER_CONFIGS -> ((resp: IncrementalAlterConfigsResponse) =>
-      IncrementalAlterConfigsResponse.fromResponseData(resp.data()).get(new ConfigResource(ConfigResource.Type.TOPIC, tp.topic)).error)
+      IncrementalAlterConfigsResponse.fromResponseData(resp.data()).get(new ConfigResource(ConfigResource.Type.TOPIC, tp.topic)).error),
+    ApiKeys.ALTER_PARTITION_REASSIGNMENTS -> ((resp: AlterPartitionReassignmentsResponse) => Errors.forCode(resp.data().errorCode())),
+    ApiKeys.LIST_PARTITION_REASSIGNMENTS -> ((resp: ListPartitionReassignmentsResponse) => Errors.forCode(resp.data().errorCode()))
   )
 
   val requestKeysToAcls = Map[ApiKeys, Map[Resource, Set[Acl]]](
@@ -252,7 +258,9 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
     ApiKeys.DESCRIBE_LOG_DIRS -> clusterDescribeAcl,
     ApiKeys.CREATE_PARTITIONS -> topicAlterAcl,
     ApiKeys.ELECT_LEADERS -> clusterAlterAcl,
-    ApiKeys.INCREMENTAL_ALTER_CONFIGS -> topicAlterConfigsAcl
+    ApiKeys.INCREMENTAL_ALTER_CONFIGS -> topicAlterConfigsAcl,
+    ApiKeys.ALTER_PARTITION_REASSIGNMENTS -> clusterAlterAcl,
+    ApiKeys.LIST_PARTITION_REASSIGNMENTS -> clusterDescribeAcl
   )
 
   @Before
@@ -485,6 +493,26 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
     10000
   ).build()
 
+  private def alterPartitionReassignmentsRequest = new AlterPartitionReassignmentsRequest.Builder(
+    new AlterPartitionReassignmentsRequestData().setTopics(
+      List(new AlterPartitionReassignmentsRequestData.ReassignableTopic()
+        .setName(topic)
+        .setPartitions(
+          List(new AlterPartitionReassignmentsRequestData.ReassignablePartition().setPartitionIndex(tp.partition())).asJava
+        )).asJava
+    )
+  ).build()
+
+  private def listPartitionReassignmentsRequest = new ListPartitionReassignmentsRequest.Builder(
+    new ListPartitionReassignmentsRequestData().setTopics(
+      List(new ListPartitionReassignmentsRequestData.ListPartitionReassignmentsTopics()
+        .setName(topic)
+        .setPartitionIndexes(
+          List(new Integer(tp.partition)).asJava
+        )).asJava
+    )
+  ).build()
+
   @Test
   def testAuthorizationWithTopicExisting() {
     val requestKeyToRequest = mutable.LinkedHashMap[ApiKeys, AbstractRequest](
@@ -520,7 +548,9 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
       // Check StopReplica last since some APIs depend on replica availability
       ApiKeys.STOP_REPLICA -> stopReplicaRequest,
       ApiKeys.ELECT_LEADERS -> electLeadersRequest,
-      ApiKeys.INCREMENTAL_ALTER_CONFIGS -> incrementalAlterConfigsRequest
+      ApiKeys.INCREMENTAL_ALTER_CONFIGS -> incrementalAlterConfigsRequest,
+      ApiKeys.ALTER_PARTITION_REASSIGNMENTS -> alterPartitionReassignmentsRequest,
+      ApiKeys.LIST_PARTITION_REASSIGNMENTS -> listPartitionReassignmentsRequest
     )
 
     for ((key, request) <- requestKeyToRequest) {
diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
index 242ab219686f0..b09fc02846f97 100644
--- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
+++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
@@ -41,6 +41,8 @@ import org.apache.kafka.common.message.JoinGroupRequestData
 import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection
 import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
 import org.apache.kafka.common.message.ListGroupsRequestData
+import org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData
+import org.apache.kafka.common.message.ListPartitionReassignmentsRequestData
 import org.apache.kafka.common.message.OffsetCommitRequestData
 import org.apache.kafka.common.message.SaslAuthenticateRequestData
 import org.apache.kafka.common.message.SaslHandshakeRequestData
@@ -465,6 +467,16 @@ class RequestQuotaTest extends BaseRequestTest {
           new IncrementalAlterConfigsRequest.Builder(
             new IncrementalAlterConfigsRequestData())
 
+        case ApiKeys.ALTER_PARTITION_REASSIGNMENTS =>
+          new AlterPartitionReassignmentsRequest.Builder(
+            new AlterPartitionReassignmentsRequestData()
+          )
+
+        case ApiKeys.LIST_PARTITION_REASSIGNMENTS =>
+          new ListPartitionReassignmentsRequest.Builder(
+            new ListPartitionReassignmentsRequestData()
+          )
+
         case _ =>
           throw new IllegalArgumentException("Unsupported API key " + apiKey)
     }
@@ -568,6 +580,8 @@ class RequestQuotaTest extends BaseRequestTest {
       case ApiKeys.ELECT_LEADERS => new ElectLeadersResponse(response).throttleTimeMs
       case ApiKeys.INCREMENTAL_ALTER_CONFIGS =>
         new IncrementalAlterConfigsResponse(response, ApiKeys.INCREMENTAL_ALTER_CONFIGS.latestVersion()).throttleTimeMs
+      case ApiKeys.ALTER_PARTITION_REASSIGNMENTS => new AlterPartitionReassignmentsResponse(response).throttleTimeMs
+      case ApiKeys.LIST_PARTITION_REASSIGNMENTS => new ListPartitionReassignmentsResponse(response).throttleTimeMs
       case requestId => throw new IllegalArgumentException(s"No throttle time for $requestId")
     }
   }

From 204710832eb4787b185264a0c8e5fe89a3db9d44 Mon Sep 17 00:00:00 2001
From: Boyang Chen 
Date: Tue, 30 Jul 2019 08:29:45 -0700
Subject: [PATCH 0485/1071] KAFKA-8640; Use generated classes in OffsetFetch
 request and response (#7062)

Reviewers: Jason Gustafson 
---
 .../apache/kafka/common/protocol/ApiKeys.java |   6 +-
 .../kafka/common/protocol/CommonFields.java   |   3 -
 .../common/requests/AbstractResponse.java     |   2 +-
 .../common/requests/OffsetCommitResponse.java |   1 -
 .../common/requests/OffsetFetchRequest.java   | 194 +++++----------
 .../common/requests/OffsetFetchResponse.java  | 228 +++++++-----------
 .../common/message/OffsetFetchRequest.json    |   5 +-
 .../common/message/OffsetFetchResponse.json   |   2 +-
 .../clients/admin/KafkaAdminClientTest.java   |   4 +-
 .../kafka/common/message/MessageTest.java     |  66 +++++
 .../requests/OffsetFetchRequestTest.java      | 115 +++++++++
 .../requests/OffsetFetchResponseTest.java     | 223 +++++++++++++++++
 .../common/requests/RequestResponseTest.java  |  12 +-
 .../kafka/api/AuthorizerIntegrationTest.scala |   2 +-
 .../unit/kafka/server/RequestQuotaTest.scala  |   2 +-
 15 files changed, 569 insertions(+), 296 deletions(-)
 create mode 100644 clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchRequestTest.java
 create mode 100644 clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java

diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
index 18d8fd62e9766..3f0f5e854013d 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
@@ -52,6 +52,8 @@
 import org.apache.kafka.common.message.MetadataResponseData;
 import org.apache.kafka.common.message.OffsetCommitRequestData;
 import org.apache.kafka.common.message.OffsetCommitResponseData;
+import org.apache.kafka.common.message.OffsetFetchRequestData;
+import org.apache.kafka.common.message.OffsetFetchResponseData;
 import org.apache.kafka.common.message.SaslAuthenticateRequestData;
 import org.apache.kafka.common.message.SaslAuthenticateResponseData;
 import org.apache.kafka.common.message.SaslHandshakeRequestData;
@@ -99,8 +101,6 @@
 import org.apache.kafka.common.requests.LeaderAndIsrResponse;
 import org.apache.kafka.common.requests.ListOffsetRequest;
 import org.apache.kafka.common.requests.ListOffsetResponse;
-import org.apache.kafka.common.requests.OffsetFetchRequest;
-import org.apache.kafka.common.requests.OffsetFetchResponse;
 import org.apache.kafka.common.requests.OffsetsForLeaderEpochRequest;
 import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse;
 import org.apache.kafka.common.requests.ProduceRequest;
@@ -138,7 +138,7 @@ public enum ApiKeys {
     CONTROLLED_SHUTDOWN(7, "ControlledShutdown", true, ControlledShutdownRequestData.SCHEMAS,
             ControlledShutdownResponseData.SCHEMAS),
     OFFSET_COMMIT(8, "OffsetCommit", OffsetCommitRequestData.SCHEMAS, OffsetCommitResponseData.SCHEMAS),
-    OFFSET_FETCH(9, "OffsetFetch", OffsetFetchRequest.schemaVersions(), OffsetFetchResponse.schemaVersions()),
+    OFFSET_FETCH(9, "OffsetFetch", OffsetFetchRequestData.SCHEMAS, OffsetFetchResponseData.SCHEMAS),
     FIND_COORDINATOR(10, "FindCoordinator", FindCoordinatorRequestData.SCHEMAS,
         FindCoordinatorResponseData.SCHEMAS),
     JOIN_GROUP(11, "JoinGroup", JoinGroupRequestData.SCHEMAS, JoinGroupResponseData.SCHEMAS),
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/CommonFields.java b/clients/src/main/java/org/apache/kafka/common/protocol/CommonFields.java
index 5fdc37f044eb2..5da38aa13ed53 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/CommonFields.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/CommonFields.java
@@ -36,9 +36,6 @@ public class CommonFields {
 
     // Group APIs
     public static final Field.Str GROUP_ID = new Field.Str("group_id", "The unique group identifier");
-    public static final Field.Int32 GENERATION_ID = new Field.Int32("generation_id", "The generation of the group.");
-    public static final Field.Str MEMBER_ID = new Field.Str("member_id", "The member id assigned by the group " +
-            "coordinator or null if joining for the first time.");
 
     // Transactional APIs
     public static final Field.Str TRANSACTIONAL_ID = new Field.Str("transactional_id", "The transactional id corresponding to the transaction.");
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
index 2e433e8b52515..eb52fb805ae52 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
@@ -81,7 +81,7 @@ public static AbstractResponse parseResponse(ApiKeys apiKey, Struct struct, shor
             case OFFSET_COMMIT:
                 return new OffsetCommitResponse(struct, version);
             case OFFSET_FETCH:
-                return new OffsetFetchResponse(struct);
+                return new OffsetFetchResponse(struct, version);
             case FIND_COORDINATOR:
                 return new FindCoordinatorResponse(struct, version);
             case JOIN_GROUP:
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
index 7f2603ec969fa..fee2e472cc1af 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
@@ -102,7 +102,6 @@ public Map errorCounts() {
                 errorMap.put(new TopicPartition(topic.name(), partition.partitionIndex()),
                         Errors.forCode(partition.errorCode()));
             }
-
         }
         return errorCounts(errorMap);
     }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java
index d2f5c888cec19..f616ac1df810a 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java
@@ -18,13 +18,11 @@
 
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.OffsetFetchRequestData;
+import org.apache.kafka.common.message.OffsetFetchRequestData.OffsetFetchRequestTopic;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
-import org.apache.kafka.common.utils.CollectionUtils;
-import org.apache.kafka.common.utils.Utils;
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
@@ -33,84 +31,45 @@
 import java.util.Map;
 import java.util.Optional;
 
-import static org.apache.kafka.common.protocol.CommonFields.GROUP_ID;
-import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
-import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
-
 public class OffsetFetchRequest extends AbstractRequest {
-    // top level fields
-    private static final Field.ComplexArray TOPICS = new Field.ComplexArray("topics",
-            "Topics to fetch offsets. If the topic array is null fetch offsets for all topics.");
-
-    // topic level fields
-    private static final Field.ComplexArray PARTITIONS = new Field.ComplexArray("partitions",
-            "Partitions to fetch offsets.");
-
-    /*
-     * Wire formats of version 0 and 1 are the same, but with different functionality.
-     * Wire format of version 2 is similar to version 1, with the exception of
-     * - accepting 'null' as list of topics
-     * - returning a top level error code
-     * Version 0 will read the offsets from ZK.
-     * Version 1 will read the offsets from Kafka.
-     * Version 2 will read the offsets from Kafka, and returns all associated topic partition offsets if
-     * a 'null' is passed instead of a list of specific topic partitions. It also returns a top level error code
-     * for group or coordinator level errors.
-     */
-    private static final Field PARTITIONS_V0 = PARTITIONS.withFields(
-            PARTITION_ID);
-
-    private static final Field TOPICS_V0 = TOPICS.withFields("Topics to fetch offsets.",
-            TOPIC_NAME,
-            PARTITIONS_V0);
-
-    private static final Schema OFFSET_FETCH_REQUEST_V0 = new Schema(
-            GROUP_ID,
-            TOPICS_V0);
-
-    // V1 begins support for fetching offsets from the internal __consumer_offsets topic
-    private static final Schema OFFSET_FETCH_REQUEST_V1 = OFFSET_FETCH_REQUEST_V0;
-
-    // V2 adds top-level error code to the response as well as allowing a null offset array to indicate fetch
-    // of all committed offsets for a group
-    private static final Field TOPICS_V2 = TOPICS.nullableWithFields(
-            TOPIC_NAME,
-            PARTITIONS_V0);
-    private static final Schema OFFSET_FETCH_REQUEST_V2 = new Schema(
-            GROUP_ID,
-            TOPICS_V2);
-
-    // V3 request is the same as v2. Throttle time has been added to v3 response
-    private static final Schema OFFSET_FETCH_REQUEST_V3 = OFFSET_FETCH_REQUEST_V2;
-
-    // V4 bump used to indicate that on quota violation brokers send out responses before throttling.
-    private static final Schema OFFSET_FETCH_REQUEST_V4 = OFFSET_FETCH_REQUEST_V3;
-
-    // V5 adds the leader epoch of the committed offset in the response
-    private static final Schema OFFSET_FETCH_REQUEST_V5 = OFFSET_FETCH_REQUEST_V4;
-
-    public static Schema[] schemaVersions() {
-        return new Schema[] {OFFSET_FETCH_REQUEST_V0, OFFSET_FETCH_REQUEST_V1, OFFSET_FETCH_REQUEST_V2,
-            OFFSET_FETCH_REQUEST_V3, OFFSET_FETCH_REQUEST_V4, OFFSET_FETCH_REQUEST_V5};
-    }
+
+    private static final List ALL_TOPIC_PARTITIONS = null;
+    public final OffsetFetchRequestData data;
 
     public static class Builder extends AbstractRequest.Builder {
-        private static final List ALL_TOPIC_PARTITIONS = null;
-        private final String groupId;
-        private final List partitions;
+
+        public final OffsetFetchRequestData data;
 
         public Builder(String groupId, List partitions) {
             super(ApiKeys.OFFSET_FETCH);
-            this.groupId = groupId;
-            this.partitions = partitions;
+
+            final List topics;
+            if (partitions != null) {
+                Map offsetFetchRequestTopicMap = new HashMap<>();
+                for (TopicPartition topicPartition : partitions) {
+                    String topicName = topicPartition.topic();
+                    OffsetFetchRequestTopic topic = offsetFetchRequestTopicMap.getOrDefault(
+                        topicName, new OffsetFetchRequestTopic().setName(topicName));
+                    topic.partitionIndexes().add(topicPartition.partition());
+                    offsetFetchRequestTopicMap.put(topicName, topic);
+                }
+                topics = new ArrayList<>(offsetFetchRequestTopicMap.values());
+            } else {
+                // If passed in partition list is null, it is requesting offsets for all topic partitions.
+                topics = ALL_TOPIC_PARTITIONS;
+            }
+
+            this.data = new OffsetFetchRequestData()
+                            .setGroupId(groupId)
+                            .setTopics(topics);
         }
 
         public static Builder allTopicPartitions(String groupId) {
-            return new Builder(groupId, ALL_TOPIC_PARTITIONS);
+            return new Builder(groupId, null);
         }
 
         public boolean isAllTopicPartitions() {
-            return this.partitions == ALL_TOPIC_PARTITIONS;
+            return this.data.topics() == ALL_TOPIC_PARTITIONS;
         }
 
         @Override
@@ -118,54 +77,40 @@ public OffsetFetchRequest build(short version) {
             if (isAllTopicPartitions() && version < 2)
                 throw new UnsupportedVersionException("The broker only supports OffsetFetchRequest " +
                         "v" + version + ", but we need v2 or newer to request all topic partitions.");
-            return new OffsetFetchRequest(groupId, partitions, version);
+            return new OffsetFetchRequest(data, version);
         }
 
         @Override
         public String toString() {
-            StringBuilder bld = new StringBuilder();
-            String partitionsString = partitions == null ? "" : Utils.join(partitions, ",");
-            bld.append("(type=OffsetFetchRequest, ").
-                    append("groupId=").append(groupId).
-                    append(", partitions=").append(partitionsString).
-                    append(")");
-            return bld.toString();
+            return data.toString();
         }
     }
 
-    private final String groupId;
-    private final List partitions;
+    public List partitions() {
+        if (isAllPartitions()) {
+            return null;
+        }
+        List partitions = new ArrayList<>();
+        for (OffsetFetchRequestTopic topic : data.topics()) {
+            for (Integer partitionIndex : topic.partitionIndexes()) {
+                partitions.add(new TopicPartition(topic.name(), partitionIndex));
+            }
+        }
+        return partitions;
+    }
 
-    public static OffsetFetchRequest forAllPartitions(String groupId) {
-        return new OffsetFetchRequest.Builder(groupId, null).build((short) 2);
+    public String groupId() {
+        return data.groupId();
     }
 
-    private OffsetFetchRequest(String groupId, List partitions, short version) {
+    private OffsetFetchRequest(OffsetFetchRequestData data, short version) {
         super(ApiKeys.OFFSET_FETCH, version);
-        this.groupId = groupId;
-        this.partitions = partitions;
+        this.data = data;
     }
 
     public OffsetFetchRequest(Struct struct, short version) {
         super(ApiKeys.OFFSET_FETCH, version);
-
-        Object[] topicArray = struct.get(TOPICS);
-        if (topicArray != null) {
-            partitions = new ArrayList<>();
-            for (Object topicResponseObj : topicArray) {
-                Struct topicResponse = (Struct) topicResponseObj;
-                String topic = topicResponse.get(TOPIC_NAME);
-                for (Object partitionResponseObj : topicResponse.get(PARTITIONS)) {
-                    Struct partitionResponse = (Struct) partitionResponseObj;
-                    int partition = partitionResponse.get(PARTITION_ID);
-                    partitions.add(new TopicPartition(topic, partition));
-                }
-            }
-        } else
-            partitions = null;
-
-
-        groupId = struct.get(GROUP_ID);
+        this.data = new OffsetFetchRequestData(struct, version);
     }
 
     public OffsetFetchResponse getErrorResponse(Errors error) {
@@ -183,8 +128,12 @@ public OffsetFetchResponse getErrorResponse(int throttleTimeMs, Errors error) {
                     OffsetFetchResponse.NO_METADATA,
                     error);
 
-            for (TopicPartition partition : this.partitions)
-                responsePartitions.put(partition, partitionError);
+            for (OffsetFetchRequestTopic topic : this.data.topics()) {
+                for (int partitionIndex : topic.partitionIndexes()) {
+                    responsePartitions.put(
+                        new TopicPartition(topic.name(), partitionIndex), partitionError);
+                }
+            }
         }
 
         switch (versionId) {
@@ -207,47 +156,16 @@ public OffsetFetchResponse getErrorResponse(int throttleTimeMs, Throwable e) {
         return getErrorResponse(throttleTimeMs, Errors.forException(e));
     }
 
-    public String groupId() {
-        return groupId;
-    }
-
-    public List partitions() {
-        return partitions;
-    }
-
     public static OffsetFetchRequest parse(ByteBuffer buffer, short version) {
         return new OffsetFetchRequest(ApiKeys.OFFSET_FETCH.parseRequest(version, buffer), version);
     }
 
     public boolean isAllPartitions() {
-        return partitions == null;
+        return data.topics() == ALL_TOPIC_PARTITIONS;
     }
 
     @Override
     protected Struct toStruct() {
-        Struct struct = new Struct(ApiKeys.OFFSET_FETCH.requestSchema(version()));
-        struct.set(GROUP_ID, groupId);
-        if (partitions != null) {
-            Map> topicsData = CollectionUtils.groupPartitionsByTopic(partitions);
-
-            List topicArray = new ArrayList<>();
-            for (Map.Entry> entries : topicsData.entrySet()) {
-                Struct topicData = struct.instance(TOPICS);
-                topicData.set(TOPIC_NAME, entries.getKey());
-                List partitionArray = new ArrayList<>();
-                for (Integer partitionId : entries.getValue()) {
-                    Struct partitionData = topicData.instance(PARTITIONS);
-                    partitionData.set(PARTITION_ID, partitionId);
-                    partitionArray.add(partitionData);
-                }
-                topicData.set(PARTITIONS, partitionArray.toArray());
-                topicArray.add(topicData);
-            }
-            struct.set(TOPICS, topicArray.toArray());
-        } else
-            struct.set(TOPICS, null);
-
-        return struct;
+        return data.toStruct(version());
     }
-
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java
index ddf5c3ba1281a..722ef4e468a7a 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java
@@ -17,29 +17,23 @@
 package org.apache.kafka.common.requests;
 
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.message.OffsetFetchResponseData;
+import org.apache.kafka.common.message.OffsetFetchResponseData.OffsetFetchResponsePartition;
+import org.apache.kafka.common.message.OffsetFetchResponseData.OffsetFetchResponseTopic;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
-import org.apache.kafka.common.utils.CollectionUtils;
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 
-import static org.apache.kafka.common.protocol.CommonFields.COMMITTED_LEADER_EPOCH;
-import static org.apache.kafka.common.protocol.CommonFields.COMMITTED_METADATA;
-import static org.apache.kafka.common.protocol.CommonFields.COMMITTED_OFFSET;
-import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
-import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
-import static org.apache.kafka.common.protocol.CommonFields.THROTTLE_TIME_MS;
-import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
+import static org.apache.kafka.common.record.RecordBatch.NO_PARTITION_LEADER_EPOCH;
 
 /**
  * Possible error codes:
@@ -55,77 +49,21 @@
  *   - {@link Errors#GROUP_AUTHORIZATION_FAILED}
  */
 public class OffsetFetchResponse extends AbstractResponse {
-    private static final Field.ComplexArray TOPICS = new Field.ComplexArray("responses",
-            "Responses by topic for fetched offsets");
-
-    // topic level fields
-    private static final Field.ComplexArray PARTITIONS = new Field.ComplexArray("partition_responses",
-            "Responses by partition for fetched offsets");
-
-    private static final Field PARTITIONS_V0 = PARTITIONS.withFields(
-            PARTITION_ID,
-            COMMITTED_OFFSET,
-            COMMITTED_METADATA,
-            ERROR_CODE);
-
-    private static final Field TOPICS_V0 = TOPICS.withFields(
-            TOPIC_NAME,
-            PARTITIONS_V0);
-
-    private static final Schema OFFSET_FETCH_RESPONSE_V0 = new Schema(
-            TOPICS_V0);
-
-    // V1 begins support for fetching offsets from the internal __consumer_offsets topic
-    private static final Schema OFFSET_FETCH_RESPONSE_V1 = OFFSET_FETCH_RESPONSE_V0;
-
-    // V2 adds top-level error code
-    private static final Schema OFFSET_FETCH_RESPONSE_V2 = new Schema(
-            TOPICS_V0,
-            ERROR_CODE);
-
-    // V3 request includes throttle time
-    private static final Schema OFFSET_FETCH_RESPONSE_V3 = new Schema(
-            THROTTLE_TIME_MS,
-            TOPICS_V0,
-            ERROR_CODE);
-
-    // V4 bump used to indicate that on quota violation brokers send out responses before throttling.
-    private static final Schema OFFSET_FETCH_RESPONSE_V4 = OFFSET_FETCH_RESPONSE_V3;
-
-    // V5 adds the leader epoch to the committed offset
-    private static final Field PARTITIONS_V5 = PARTITIONS.withFields(
-            PARTITION_ID,
-            COMMITTED_OFFSET,
-            COMMITTED_LEADER_EPOCH,
-            COMMITTED_METADATA,
-            ERROR_CODE);
-
-    private static final Field TOPICS_V5 = TOPICS.withFields(
-            TOPIC_NAME,
-            PARTITIONS_V5);
-
-    private static final Schema OFFSET_FETCH_RESPONSE_V5 = new Schema(
-            THROTTLE_TIME_MS,
-            TOPICS_V5,
-            ERROR_CODE);
-
-    public static Schema[] schemaVersions() {
-        return new Schema[] {OFFSET_FETCH_RESPONSE_V0, OFFSET_FETCH_RESPONSE_V1, OFFSET_FETCH_RESPONSE_V2,
-            OFFSET_FETCH_RESPONSE_V3, OFFSET_FETCH_RESPONSE_V4, OFFSET_FETCH_RESPONSE_V5};
-    }
-
     public static final long INVALID_OFFSET = -1L;
     public static final String NO_METADATA = "";
     public static final PartitionData UNKNOWN_PARTITION = new PartitionData(INVALID_OFFSET,
-            Optional.empty(), NO_METADATA, Errors.UNKNOWN_TOPIC_OR_PARTITION);
+                                                                            Optional.empty(),
+                                                                            NO_METADATA,
+                                                                            Errors.UNKNOWN_TOPIC_OR_PARTITION);
     public static final PartitionData UNAUTHORIZED_PARTITION = new PartitionData(INVALID_OFFSET,
-            Optional.empty(), NO_METADATA, Errors.TOPIC_AUTHORIZATION_FAILED);
+                                                                                 Optional.empty(),
+                                                                                 NO_METADATA,
+                                                                                 Errors.TOPIC_AUTHORIZATION_FAILED);
+    private static final List PARTITION_ERRORS = Arrays.asList(
+        Errors.UNKNOWN_TOPIC_OR_PARTITION, Errors.TOPIC_AUTHORIZATION_FAILED);
 
-    private static final List PARTITION_ERRORS = Collections.singletonList(Errors.UNKNOWN_TOPIC_OR_PARTITION);
-
-    private final Map responseData;
+    public final OffsetFetchResponseData data;
     private final Errors error;
-    private final int throttleTimeMs;
 
     public static final class PartitionData {
         public final long offset;
@@ -146,6 +84,32 @@ public PartitionData(long offset,
         public boolean hasError() {
             return this.error != Errors.NONE;
         }
+
+        @Override
+        public boolean equals(Object other) {
+            if (!(other instanceof PartitionData))
+                return false;
+            PartitionData otherPartition = (PartitionData) other;
+            return this.offset == otherPartition.offset
+                       && this.leaderEpoch.equals(otherPartition.leaderEpoch)
+                       && this.metadata.equals(otherPartition.metadata)
+                       && this.error.equals(otherPartition.error);
+        }
+
+        @Override
+        public String toString() {
+            return "PartitionData("
+                       + "offset=" + offset
+                       + ", leaderEpoch=" + leaderEpoch.orElse(NO_PARTITION_LEADER_EPOCH)
+                       + ", metadata=" + metadata
+                       + ", error='" + error.toString()
+                       + ")";
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(offset, leaderEpoch, metadata, error);
+        }
     }
 
     /**
@@ -164,31 +128,41 @@ public OffsetFetchResponse(Errors error, Map resp
      * @param responseData Fetched offset information grouped by topic-partition
      */
     public OffsetFetchResponse(int throttleTimeMs, Errors error, Map responseData) {
-        this.throttleTimeMs = throttleTimeMs;
-        this.responseData = responseData;
+        Map offsetFetchResponseTopicMap = new HashMap<>();
+        for (Map.Entry entry : responseData.entrySet()) {
+            String topicName = entry.getKey().topic();
+            OffsetFetchResponseTopic topic = offsetFetchResponseTopicMap.getOrDefault(
+                topicName, new OffsetFetchResponseTopic().setName(topicName));
+            PartitionData partitionData = entry.getValue();
+            topic.partitions().add(new OffsetFetchResponsePartition()
+                                       .setPartitionIndex(entry.getKey().partition())
+                                       .setErrorCode(partitionData.error.code())
+                                       .setCommittedOffset(partitionData.offset)
+                                       .setCommittedLeaderEpoch(
+                                           partitionData.leaderEpoch.orElse(NO_PARTITION_LEADER_EPOCH))
+                                       .setMetadata(partitionData.metadata)
+            );
+            offsetFetchResponseTopicMap.put(topicName, topic);
+        }
+
+        this.data = new OffsetFetchResponseData()
+            .setTopics(new ArrayList<>(offsetFetchResponseTopicMap.values()))
+            .setErrorCode(error.code())
+            .setThrottleTimeMs(throttleTimeMs);
         this.error = error;
     }
 
-    public OffsetFetchResponse(Struct struct) {
-        this.throttleTimeMs = struct.getOrElse(THROTTLE_TIME_MS, DEFAULT_THROTTLE_TIME);
-        Errors topLevelError = Errors.NONE;
-        this.responseData = new HashMap<>();
-        for (Object topicResponseObj : struct.get(TOPICS)) {
-            Struct topicResponse = (Struct) topicResponseObj;
-            String topic = topicResponse.get(TOPIC_NAME);
-            for (Object partitionResponseObj : topicResponse.get(PARTITIONS)) {
-                Struct partitionResponse = (Struct) partitionResponseObj;
-                int partition = partitionResponse.get(PARTITION_ID);
-                long offset = partitionResponse.get(COMMITTED_OFFSET);
-                String metadata = partitionResponse.get(COMMITTED_METADATA);
-                Optional leaderEpochOpt = RequestUtils.getLeaderEpoch(partitionResponse, COMMITTED_LEADER_EPOCH);
-
-                Errors error = Errors.forCode(partitionResponse.get(ERROR_CODE));
-                if (error != Errors.NONE && !PARTITION_ERRORS.contains(error))
-                    topLevelError = error;
+    public OffsetFetchResponse(Struct struct, short version) {
+        this.data = new OffsetFetchResponseData(struct, version);
 
-                PartitionData partitionData = new PartitionData(offset, leaderEpochOpt, metadata, error);
-                this.responseData.put(new TopicPartition(topic, partition), partitionData);
+        Errors topLevelError = Errors.NONE;
+        for (OffsetFetchResponseTopic topic : data.topics()) {
+            for (OffsetFetchResponsePartition partition : topic.partitions()) {
+                Errors partitionError = Errors.forCode(partition.errorCode());
+                if (partitionError != Errors.NONE && !PARTITION_ERRORS.contains(partitionError)) {
+                    topLevelError = partitionError;
+                    break;
+                }
             }
         }
 
@@ -196,28 +170,20 @@ public OffsetFetchResponse(Struct struct) {
         // for older versions there is no top-level error in the response and all errors are partition errors,
         // so if there is a group or coordinator error at the partition level use that as the top-level error.
         // this way clients can depend on the top-level error regardless of the offset fetch version.
-        this.error = struct.hasField(ERROR_CODE) ? Errors.forCode(struct.get(ERROR_CODE)) : topLevelError;
-    }
-
-    public void maybeThrowFirstPartitionError() {
-        Collection partitionsData = this.responseData.values();
-        for (PartitionData data : partitionsData) {
-            if (data.hasError())
-                throw data.error.exception();
-        }
+        this.error = version >= 2 ? Errors.forCode(data.errorCode()) : topLevelError;
     }
 
     @Override
     public int throttleTimeMs() {
-        return throttleTimeMs;
+        return data.throttleTimeMs();
     }
 
     public boolean hasError() {
-        return this.error != Errors.NONE;
+        return error != Errors.NONE;
     }
 
     public Errors error() {
-        return this.error;
+        return error;
     }
 
     @Override
@@ -226,43 +192,27 @@ public Map errorCounts() {
     }
 
     public Map responseData() {
+        Map responseData = new HashMap<>();
+        for (OffsetFetchResponseTopic topic : data.topics()) {
+            for (OffsetFetchResponsePartition partition : topic.partitions()) {
+                responseData.put(new TopicPartition(topic.name(), partition.partitionIndex()),
+                                 new PartitionData(partition.committedOffset(),
+                                                   RequestUtils.getLeaderEpoch(partition.committedLeaderEpoch()),
+                                                   partition.metadata(),
+                                                   Errors.forCode(partition.errorCode()))
+                );
+            }
+        }
         return responseData;
     }
 
     public static OffsetFetchResponse parse(ByteBuffer buffer, short version) {
-        return new OffsetFetchResponse(ApiKeys.OFFSET_FETCH.parseResponse(version, buffer));
+        return new OffsetFetchResponse(ApiKeys.OFFSET_FETCH.parseResponse(version, buffer), version);
     }
 
     @Override
     protected Struct toStruct(short version) {
-        Struct struct = new Struct(ApiKeys.OFFSET_FETCH.responseSchema(version));
-        struct.setIfExists(THROTTLE_TIME_MS, throttleTimeMs);
-
-        Map> topicsData = CollectionUtils.groupPartitionDataByTopic(responseData);
-        List topicArray = new ArrayList<>();
-        for (Map.Entry> entries : topicsData.entrySet()) {
-            Struct topicData = struct.instance(TOPICS);
-            topicData.set(TOPIC_NAME, entries.getKey());
-            List partitionArray = new ArrayList<>();
-            for (Map.Entry partitionEntry : entries.getValue().entrySet()) {
-                PartitionData fetchPartitionData = partitionEntry.getValue();
-                Struct partitionData = topicData.instance(PARTITIONS);
-                partitionData.set(PARTITION_ID, partitionEntry.getKey());
-                partitionData.set(COMMITTED_OFFSET, fetchPartitionData.offset);
-                RequestUtils.setLeaderEpochIfExists(partitionData, COMMITTED_LEADER_EPOCH, fetchPartitionData.leaderEpoch);
-                partitionData.set(COMMITTED_METADATA, fetchPartitionData.metadata);
-                partitionData.set(ERROR_CODE, fetchPartitionData.error.code());
-                partitionArray.add(partitionData);
-            }
-            topicData.set(PARTITIONS, partitionArray.toArray());
-            topicArray.add(topicData);
-        }
-        struct.set(TOPICS, topicArray.toArray());
-
-        if (version > 1)
-            struct.set(ERROR_CODE, this.error.code());
-
-        return struct;
+        return data.toStruct(version);
     }
 
     @Override
diff --git a/clients/src/main/resources/common/message/OffsetFetchRequest.json b/clients/src/main/resources/common/message/OffsetFetchRequest.json
index 4ff781bd58bdd..0449fda7c0382 100644
--- a/clients/src/main/resources/common/message/OffsetFetchRequest.json
+++ b/clients/src/main/resources/common/message/OffsetFetchRequest.json
@@ -17,10 +17,13 @@
   "apiKey": 9,
   "type": "request",
   "name": "OffsetFetchRequest",
+  // In version 0, the request read offsets from ZK.
+  //
   // Starting in version 1, the broker supports fetching offsets from the internal __consumer_offsets topic.
   //
   // Starting in version 2, the request can contain a null topics array to indicate that offsets
-  // for all topics should be fetched.
+  // for all topics should be fetched. It also returns a top level error code
+  // for group or coordinator level errors.
   //
   // Version 3, 4, and 5 are the same as version 2.
   "validVersions": "0-5",
diff --git a/clients/src/main/resources/common/message/OffsetFetchResponse.json b/clients/src/main/resources/common/message/OffsetFetchResponse.json
index eb0bbbc293c34..69756cc523fca 100644
--- a/clients/src/main/resources/common/message/OffsetFetchResponse.json
+++ b/clients/src/main/resources/common/message/OffsetFetchResponse.json
@@ -40,7 +40,7 @@
           "about": "The partition index." },
         { "name": "CommittedOffset", "type": "int64", "versions": "0+",
           "about": "The committed message offset." },
-        { "name": "CommittedLeaderEpoch", "type": "int32", "versions": "5+",
+        { "name": "CommittedLeaderEpoch", "type": "int32", "versions": "5+", "default": "-1",
           "about": "The leader epoch." },
         { "name": "Metadata", "type": "string", "versions": "0+", "nullableVersions": "0+",
           "about": "The partition metadata." },
diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
index 769f58c3caf40..7ec7c24a3a0c7 100644
--- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
@@ -1377,12 +1377,12 @@ public void testDescribeConsumerGroupOffsets() throws Exception {
         try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster)) {
             env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
 
-            //Retriable FindCoordinatorResponse errors should be retried
+            // Retriable FindCoordinatorResponse errors should be retried
             env.kafkaClient().prepareResponse(prepareFindCoordinatorResponse(Errors.COORDINATOR_NOT_AVAILABLE,  Node.noNode()));
 
             env.kafkaClient().prepareResponse(prepareFindCoordinatorResponse(Errors.NONE, env.cluster().controller()));
 
-            //Retriable  errors should be retried
+            // Retriable errors should be retried
             env.kafkaClient().prepareResponse(new OffsetFetchResponse(Errors.COORDINATOR_NOT_AVAILABLE, Collections.emptyMap()));
             env.kafkaClient().prepareResponse(new OffsetFetchResponse(Errors.COORDINATOR_LOAD_IN_PROGRESS, Collections.emptyMap()));
 
diff --git a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
index bdfce3f32f1b1..473912ceda345 100644
--- a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
@@ -22,6 +22,9 @@
 import org.apache.kafka.common.message.AddPartitionsToTxnRequestData.AddPartitionsToTxnTopicCollection;
 import org.apache.kafka.common.message.JoinGroupResponseData.JoinGroupResponseMember;
 import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse;
+import org.apache.kafka.common.message.OffsetFetchRequestData.OffsetFetchRequestTopic;
+import org.apache.kafka.common.message.OffsetFetchResponseData.OffsetFetchResponsePartition;
+import org.apache.kafka.common.message.OffsetFetchResponseData.OffsetFetchResponseTopic;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.ByteBufferAccessor;
 import org.apache.kafka.common.protocol.Errors;
@@ -29,6 +32,7 @@
 import org.apache.kafka.common.protocol.types.ArrayOf;
 import org.apache.kafka.common.protocol.types.BoundField;
 import org.apache.kafka.common.protocol.types.Schema;
+import org.apache.kafka.common.protocol.types.SchemaException;
 import org.apache.kafka.common.protocol.types.Struct;
 import org.apache.kafka.common.protocol.types.Type;
 import org.apache.kafka.common.utils.Utils;
@@ -46,6 +50,7 @@
 
 import static java.util.Collections.singletonList;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -266,6 +271,67 @@ public void testLeaderAndIsrVersions() throws Exception {
         testAllMessageRoundTripsFromVersion((short) 3, new LeaderAndIsrRequestData().setTopicStates(Collections.singletonList(partitionStateWithAddingRemovingReplicas)));
     }
 
+    @Test
+    public void testOffsetFetchVersions() throws Exception {
+        String groupId = "groupId";
+        String topicName = "topic";
+
+        testAllMessageRoundTrips(new OffsetFetchRequestData()
+                                     .setTopics(new ArrayList<>())
+                                     .setGroupId(groupId));
+
+        testAllMessageRoundTrips(new OffsetFetchRequestData()
+                                     .setGroupId(groupId)
+                                     .setTopics(Collections.singletonList(
+                                         new OffsetFetchRequestTopic()
+                                             .setName(topicName)
+                                             .setPartitionIndexes(Collections.singletonList(5))))
+        );
+
+        OffsetFetchRequestData allPartitionData = new OffsetFetchRequestData()
+                                                      .setGroupId(groupId)
+                                                      .setTopics(null);
+        for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) {
+            if (version < 2) {
+                final short finalVersion = version;
+                assertThrows(SchemaException.class, () -> testAllMessageRoundTripsFromVersion(finalVersion, allPartitionData));
+            } else {
+                testAllMessageRoundTripsFromVersion(version, allPartitionData);
+            }
+        }
+
+        Supplier response =
+            () -> new OffsetFetchResponseData()
+                      .setTopics(Collections.singletonList(
+                          new OffsetFetchResponseTopic()
+                              .setName(topicName)
+                              .setPartitions(Collections.singletonList(
+                                  new OffsetFetchResponsePartition()
+                                      .setPartitionIndex(5)
+                                      .setMetadata(null)
+                                      .setCommittedOffset(100)
+                                      .setCommittedLeaderEpoch(3)
+                                      .setErrorCode(Errors.UNKNOWN_TOPIC_OR_PARTITION.code())))))
+                      .setErrorCode(Errors.NOT_COORDINATOR.code())
+                      .setThrottleTimeMs(10);
+        for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) {
+            OffsetFetchResponseData responseData = response.get();
+            if (version <= 1) {
+                responseData.setErrorCode(Errors.NONE.code());
+            }
+
+            if (version <= 2) {
+                responseData.setThrottleTimeMs(0);
+            }
+
+            if (version <= 4) {
+                responseData.topics().get(0).partitions().get(0).setCommittedLeaderEpoch(-1);
+            }
+
+            testAllMessageRoundTripsFromVersion(version, responseData);
+        }
+    }
+
     private void testAllMessageRoundTrips(Message message) throws Exception {
         testAllMessageRoundTripsFromVersion(message.lowestSupportedVersion(), message);
     }
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchRequestTest.java
new file mode 100644
index 0000000000000..3bafb76561ae8
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchRequestTest.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.requests;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.OffsetFetchResponse.PartitionData;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.apache.kafka.common.requests.AbstractResponse.DEFAULT_THROTTLE_TIME;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+public class OffsetFetchRequestTest {
+
+    private final String topicOne = "topic1";
+    private final int partitionOne = 1;
+    private final String topicTwo = "topic2";
+    private final int partitionTwo = 2;
+    private final String groupId = "groupId";
+
+    private OffsetFetchRequest.Builder builder;
+    private List partitions;
+
+    @Before
+    public void setUp() {
+        partitions = Arrays.asList(new TopicPartition(topicOne, partitionOne),
+                                   new TopicPartition(topicTwo, partitionTwo));
+        builder = new OffsetFetchRequest.Builder(
+            groupId,
+            partitions
+        );
+    }
+
+    @Test
+    public void testConstructor() {
+        assertFalse(builder.isAllTopicPartitions());
+        int throttleTimeMs = 10;
+
+        Map expectedData = new HashMap<>();
+        for (TopicPartition partition : partitions) {
+            expectedData.put(partition, new PartitionData(
+                OffsetFetchResponse.INVALID_OFFSET,
+                Optional.empty(),
+                OffsetFetchResponse.NO_METADATA,
+                Errors.NONE
+            ));
+        }
+
+        for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) {
+            OffsetFetchRequest request = builder.build(version);
+            assertFalse(request.isAllPartitions());
+            assertEquals(groupId, request.groupId());
+            assertEquals(partitions, request.partitions());
+
+            OffsetFetchResponse response = request.getErrorResponse(throttleTimeMs, Errors.NONE);
+            assertEquals(Errors.NONE, response.error());
+            assertFalse(response.hasError());
+            assertEquals(Collections.singletonMap(Errors.NONE, 1), response.errorCounts());
+
+            if (version <= 1) {
+                assertEquals(expectedData, response.responseData());
+            }
+
+            if (version >= 3) {
+                assertEquals(throttleTimeMs, response.throttleTimeMs());
+            } else {
+                assertEquals(DEFAULT_THROTTLE_TIME, response.throttleTimeMs());
+            }
+        }
+    }
+
+    @Test
+    public void testConstructorFailForUnsupportedAllPartition() {
+        builder = OffsetFetchRequest.Builder.allTopicPartitions(groupId);
+        for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) {
+            short finalVersion = version;
+            if (version <= 1) {
+                assertThrows(UnsupportedVersionException.class, () -> builder.build(finalVersion));
+            } else {
+                OffsetFetchRequest request = builder.build(finalVersion);
+                assertEquals(groupId, request.groupId());
+                assertNull(request.partitions());
+                assertTrue(request.isAllPartitions());
+            }
+        }
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java
new file mode 100644
index 0000000000000..d3ff161ac0b07
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.requests;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.message.OffsetFetchResponseData;
+import org.apache.kafka.common.message.OffsetFetchResponseData.OffsetFetchResponsePartition;
+import org.apache.kafka.common.message.OffsetFetchResponseData.OffsetFetchResponseTopic;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.protocol.types.Struct;
+import org.apache.kafka.common.record.RecordBatch;
+import org.apache.kafka.common.requests.OffsetFetchResponse.PartitionData;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
+import static org.apache.kafka.common.requests.AbstractResponse.DEFAULT_THROTTLE_TIME;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class OffsetFetchResponseTest {
+
+    private final int throttleTimeMs = 10;
+    private final int offset = 100;
+    private final String metadata = "metadata";
+
+    private final String topicOne = "topic1";
+    private final int partitionOne = 1;
+    private final Optional leaderEpochOne = Optional.of(1);
+    private final String topicTwo = "topic2";
+    private final int partitionTwo = 2;
+    private final Optional leaderEpochTwo = Optional.of(2);
+
+    private Map partitionDataMap;
+
+    @Before
+    public void setUp() {
+        partitionDataMap = new HashMap<>();
+        partitionDataMap.put(new TopicPartition(topicOne, partitionOne), new PartitionData(
+            offset,
+            leaderEpochOne,
+            metadata,
+            Errors.TOPIC_AUTHORIZATION_FAILED
+        ));
+        partitionDataMap.put(new TopicPartition(topicTwo, partitionTwo), new PartitionData(
+            offset,
+            leaderEpochTwo,
+            metadata,
+            Errors.UNKNOWN_TOPIC_OR_PARTITION
+        ));
+    }
+
+    @Test
+    public void testConstructor() {
+        OffsetFetchResponse response = new OffsetFetchResponse(throttleTimeMs, Errors.NOT_COORDINATOR, partitionDataMap);
+        assertEquals(Errors.NOT_COORDINATOR, response.error());
+        assertEquals(Collections.singletonMap(Errors.NOT_COORDINATOR, 1), response.errorCounts());
+
+        assertEquals(throttleTimeMs, response.throttleTimeMs());
+
+        Map responseData = response.responseData();
+        assertEquals(partitionDataMap, responseData);
+        responseData.forEach(
+            (tp, data) -> assertTrue(data.hasError())
+        );
+    }
+
+    /**
+     * Test behavior changes over the versions. Refer to resources.common.messages.OffsetFetchResponse.json
+     */
+    @Test
+    public void testStructBuild() {
+        partitionDataMap.put(new TopicPartition(topicTwo, partitionTwo), new PartitionData(
+            offset,
+            leaderEpochTwo,
+            metadata,
+            Errors.GROUP_AUTHORIZATION_FAILED
+        ));
+
+        OffsetFetchResponse latestResponse = new OffsetFetchResponse(throttleTimeMs, Errors.NONE, partitionDataMap);
+
+        for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) {
+            Struct struct = latestResponse.data.toStruct(version);
+
+            OffsetFetchResponse oldResponse =  new OffsetFetchResponse(struct, version);
+
+            if (version <= 1) {
+                assertFalse(struct.hasField(ERROR_CODE));
+
+                // Partition level error populated in older versions.
+                assertEquals(Errors.GROUP_AUTHORIZATION_FAILED, oldResponse.error());
+                assertEquals(Collections.singletonMap(Errors.GROUP_AUTHORIZATION_FAILED, 1), oldResponse.errorCounts());
+
+            } else {
+                assertTrue(struct.hasField(ERROR_CODE));
+
+                assertEquals(Errors.NONE, oldResponse.error());
+                assertEquals(Collections.singletonMap(Errors.NONE, 1), oldResponse.errorCounts());
+            }
+
+            if (version <= 2) {
+                assertEquals(DEFAULT_THROTTLE_TIME, oldResponse.throttleTimeMs());
+            } else {
+                assertEquals(throttleTimeMs, oldResponse.throttleTimeMs());
+            }
+
+            Map expectedDataMap = new HashMap<>();
+            for (Map.Entry entry : partitionDataMap.entrySet()) {
+                PartitionData partitionData = entry.getValue();
+                expectedDataMap.put(entry.getKey(), new PartitionData(
+                    partitionData.offset,
+                    version <= 4 ? Optional.empty() : partitionData.leaderEpoch,
+                    partitionData.metadata,
+                    partitionData.error
+                ));
+            }
+
+            Map responseData = oldResponse.responseData();
+            assertEquals(expectedDataMap, responseData);
+
+            responseData.forEach(
+                (tp, data) -> assertTrue(data.hasError())
+            );
+        }
+    }
+
+    @Test
+    public void testShouldThrottle() {
+        OffsetFetchResponse response = new OffsetFetchResponse(throttleTimeMs, Errors.NONE, partitionDataMap);
+        for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) {
+            if (version >= 4) {
+                assertTrue(response.shouldClientThrottle(version));
+            } else {
+                assertFalse(response.shouldClientThrottle(version));
+            }
+        }
+    }
+
+    @Test
+    public void testNullableMetadata() {
+        partitionDataMap.clear();
+        partitionDataMap.put(new TopicPartition(topicOne, partitionOne),
+                             new PartitionData(
+                                 offset,
+                                 leaderEpochOne,
+                                 null,
+                                 Errors.UNKNOWN_TOPIC_OR_PARTITION)
+        );
+
+        OffsetFetchResponse response = new OffsetFetchResponse(throttleTimeMs, Errors.GROUP_AUTHORIZATION_FAILED, partitionDataMap);
+        OffsetFetchResponseData expectedData =
+            new OffsetFetchResponseData()
+                .setErrorCode(Errors.GROUP_AUTHORIZATION_FAILED.code())
+                .setThrottleTimeMs(throttleTimeMs)
+                .setTopics(Collections.singletonList(
+                    new OffsetFetchResponseTopic()
+                        .setName(topicOne)
+                        .setPartitions(Collections.singletonList(
+                            new OffsetFetchResponsePartition()
+                                .setPartitionIndex(partitionOne)
+                                .setCommittedOffset(offset)
+                                .setCommittedLeaderEpoch(leaderEpochOne.orElse(-1))
+                                .setErrorCode(Errors.UNKNOWN_TOPIC_OR_PARTITION.code())
+                                .setMetadata(null))
+                        ))
+                );
+        assertEquals(expectedData, response.data);
+    }
+
+    @Test
+    public void testUseDefaultLeaderEpoch() {
+        final Optional emptyLeaderEpoch = Optional.empty();
+        partitionDataMap.clear();
+
+        partitionDataMap.put(new TopicPartition(topicOne, partitionOne),
+                             new PartitionData(
+                                 offset,
+                                 emptyLeaderEpoch,
+                                 metadata,
+                                 Errors.UNKNOWN_TOPIC_OR_PARTITION)
+        );
+
+        OffsetFetchResponse response = new OffsetFetchResponse(throttleTimeMs, Errors.NOT_COORDINATOR, partitionDataMap);
+        OffsetFetchResponseData expectedData =
+            new OffsetFetchResponseData()
+                .setErrorCode(Errors.NOT_COORDINATOR.code())
+                .setThrottleTimeMs(throttleTimeMs)
+                .setTopics(Collections.singletonList(
+                new OffsetFetchResponseTopic()
+                    .setName(topicOne)
+                    .setPartitions(Collections.singletonList(
+                        new OffsetFetchResponsePartition()
+                            .setPartitionIndex(partitionOne)
+                            .setCommittedOffset(offset)
+                            .setCommittedLeaderEpoch(RecordBatch.NO_PARTITION_LEADER_EPOCH)
+                            .setErrorCode(Errors.UNKNOWN_TOPIC_OR_PARTITION.code())
+                            .setMetadata(metadata))
+                    ))
+                );
+        assertEquals(expectedData, response.data);
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index 70adcca9f8d71..4218eff46e26a 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -196,12 +196,12 @@ public void testSerialization() throws Exception {
         checkErrorResponse(createMetadataRequest(3, Collections.singletonList("topic1")), new UnknownServerException(), true);
         checkResponse(createMetadataResponse(), 4, true);
         checkErrorResponse(createMetadataRequest(4, Collections.singletonList("topic1")), new UnknownServerException(), true);
-        checkRequest(OffsetFetchRequest.forAllPartitions("group1"), true);
-        checkErrorResponse(OffsetFetchRequest.forAllPartitions("group1"), new NotCoordinatorException("Not Coordinator"), true);
+        checkRequest(OffsetFetchRequest.Builder.allTopicPartitions("group1").build(), true);
+        checkErrorResponse(OffsetFetchRequest.Builder.allTopicPartitions("group1").build(), new NotCoordinatorException("Not Coordinator"), true);
         checkRequest(createOffsetFetchRequest(0), true);
         checkRequest(createOffsetFetchRequest(1), true);
         checkRequest(createOffsetFetchRequest(2), true);
-        checkRequest(OffsetFetchRequest.forAllPartitions("group1"), true);
+        checkRequest(OffsetFetchRequest.Builder.allTopicPartitions("group1").build(), true);
         checkErrorResponse(createOffsetFetchRequest(0), new UnknownServerException(), true);
         checkErrorResponse(createOffsetFetchRequest(1), new UnknownServerException(), true);
         checkErrorResponse(createOffsetFetchRequest(2), new UnknownServerException(), true);
@@ -732,8 +732,10 @@ public void testJoinGroupRequestVersion0RebalanceTimeout() {
     @Test
     public void testOffsetFetchRequestBuilderToString() {
         String allTopicPartitionsString = OffsetFetchRequest.Builder.allTopicPartitions("someGroup").toString();
-        assertTrue(allTopicPartitionsString.contains(""));
-        String string = new OffsetFetchRequest.Builder("group1", Collections.singletonList(new TopicPartition("test11", 1))).toString();
+
+        assertTrue(allTopicPartitionsString.contains("groupId='someGroup', topics=null"));
+        String string = new OffsetFetchRequest.Builder("group1",
+            Collections.singletonList(new TopicPartition("test11", 1))).toString();
         assertTrue(string.contains("test11"));
         assertTrue(string.contains("group1"));
     }
diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
index ef2e6fd5b13c4..8f3f24fc1ef04 100644
--- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
@@ -1132,7 +1132,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
     // note there's only one broker, so no need to lookup the group coordinator
 
     // without describe permission on the topic, we shouldn't be able to fetch offsets
-    val offsetFetchRequest = requests.OffsetFetchRequest.forAllPartitions(group)
+    val offsetFetchRequest = requests.OffsetFetchRequest.Builder.allTopicPartitions(group).build()
     var offsetFetchResponse = sendOffsetFetchRequest(offsetFetchRequest, anySocketServer)
     assertEquals(Errors.NONE, offsetFetchResponse.error)
     assertTrue(offsetFetchResponse.responseData.isEmpty)
diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
index b09fc02846f97..a8d29fd02f242 100644
--- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
+++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
@@ -542,7 +542,7 @@ class RequestQuotaTest extends BaseRequestTest {
         new MetadataResponse(response, ApiKeys.DESCRIBE_GROUPS.latestVersion).throttleTimeMs
       case ApiKeys.OFFSET_COMMIT =>
         new OffsetCommitResponse(response, ApiKeys.OFFSET_COMMIT.latestVersion).throttleTimeMs
-      case ApiKeys.OFFSET_FETCH => new OffsetFetchResponse(response).throttleTimeMs
+      case ApiKeys.OFFSET_FETCH => new OffsetFetchResponse(response, ApiKeys.OFFSET_FETCH.latestVersion).throttleTimeMs
       case ApiKeys.FIND_COORDINATOR =>
         new FindCoordinatorResponse(response, ApiKeys.FIND_COORDINATOR.latestVersion).throttleTimeMs
       case ApiKeys.JOIN_GROUP => new JoinGroupResponse(response).throttleTimeMs

From c6286b2b3e0d573773844f9b0a2931e9c2058f58 Mon Sep 17 00:00:00 2001
From: huxi 
Date: Tue, 30 Jul 2019 23:44:11 +0800
Subject: [PATCH 0486/1071] KAFKA-8442; Include ISR in Metadata response even
 if there is no leader (#6836)

Currently the Metadata response returns an empty ISR if there is no active leader. The behavior is inconsistent since other fields such as the replica list and offline replicas are included. This patch changes the behavior to return the current known ISR. This fixes a problem with the topic describe command which fails to report ISR when a leader is offline.

Reviewers: Jason Gustafson 
---
 core/src/main/scala/kafka/admin/TopicCommand.scala         | 5 ++++-
 core/src/main/scala/kafka/server/MetadataCache.scala       | 7 +++----
 .../unit/kafka/admin/TopicCommandWithAdminClientTest.scala | 2 +-
 .../test/scala/unit/kafka/server/MetadataCacheTest.scala   | 2 +-
 4 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/core/src/main/scala/kafka/admin/TopicCommand.scala b/core/src/main/scala/kafka/admin/TopicCommand.scala
index f2e197ae122ab..a25e546b713b7 100755
--- a/core/src/main/scala/kafka/admin/TopicCommand.scala
+++ b/core/src/main/scala/kafka/admin/TopicCommand.scala
@@ -120,7 +120,10 @@ object TopicCommand extends Logging {
       opts.reportUnavailablePartitions && hasUnavailablePartitions(partitionDescription)
     }
     private def hasUnderMinIsrPartitions(partitionDescription: PartitionDescription) = {
-      partitionDescription.isr.size < partitionDescription.minIsrCount
+      if (partitionDescription.leader.isDefined)
+        partitionDescription.isr.size < partitionDescription.minIsrCount
+      else
+        partitionDescription.minIsrCount > 0
     }
     private def hasAtMinIsrPartitions(partitionDescription: PartitionDescription) = {
       partitionDescription.isr.size == partitionDescription.minIsrCount
diff --git a/core/src/main/scala/kafka/server/MetadataCache.scala b/core/src/main/scala/kafka/server/MetadataCache.scala
index 8b8e159be9889..7d10bb004251e 100755
--- a/core/src/main/scala/kafka/server/MetadataCache.scala
+++ b/core/src/main/scala/kafka/server/MetadataCache.scala
@@ -82,6 +82,8 @@ class MetadataCache(brokerId: Int) extends Logging {
         val replicaInfo = getEndpoints(snapshot, replicas, listenerName, errorUnavailableEndpoints)
         val offlineReplicaInfo = getEndpoints(snapshot, partitionState.offlineReplicas.asScala, listenerName, errorUnavailableEndpoints)
 
+        val isr = partitionState.basePartitionState.isr.asScala
+        val isrInfo = getEndpoints(snapshot, isr, listenerName, errorUnavailableEndpoints)
         maybeLeader match {
           case None =>
             val error = if (!snapshot.aliveBrokers.contains(brokerId)) { // we are already holding the read lock
@@ -92,13 +94,10 @@ class MetadataCache(brokerId: Int) extends Logging {
               if (errorUnavailableListeners) Errors.LISTENER_NOT_FOUND else Errors.LEADER_NOT_AVAILABLE
             }
             new MetadataResponse.PartitionMetadata(error, partitionId.toInt, Node.noNode(),
-              Optional.empty(), replicaInfo.asJava, java.util.Collections.emptyList(),
+              Optional.empty(), replicaInfo.asJava, isrInfo.asJava,
               offlineReplicaInfo.asJava)
 
           case Some(leader) =>
-            val isr = partitionState.basePartitionState.isr.asScala
-            val isrInfo = getEndpoints(snapshot, isr, listenerName, errorUnavailableEndpoints)
-
             if (replicaInfo.size < replicas.size) {
               debug(s"Error while fetching metadata for $topicPartition: replica information not available for " +
                 s"following brokers ${replicas.filterNot(replicaInfo.map(_.id).contains).mkString(",")}")
diff --git a/core/src/test/scala/unit/kafka/admin/TopicCommandWithAdminClientTest.scala b/core/src/test/scala/unit/kafka/admin/TopicCommandWithAdminClientTest.scala
index 8813b59811749..a04ef0dac7fa1 100644
--- a/core/src/test/scala/unit/kafka/admin/TopicCommandWithAdminClientTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/TopicCommandWithAdminClientTest.scala
@@ -602,7 +602,7 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
           topicService.describeTopic(new TopicCommandOptions(Array("--topic", testTopicName, "--unavailable-partitions"))))
       val rows = output.split("\n")
       assertTrue(rows(0).startsWith(s"\tTopic: $testTopicName"))
-      assertTrue(rows(0).endsWith("Leader: none\tReplicas: 0\tIsr: "))
+      assertTrue(rows(0).contains("Leader: none\tReplicas: 0\tIsr:"))
     } finally {
       restartDeadBrokers()
     }
diff --git a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala
index 5ddabc0b6f7a0..9f73fbea98883 100644
--- a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala
+++ b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala
@@ -182,7 +182,7 @@ class MetadataCacheTest {
     val partitionMetadata = partitionMetadatas.get(0)
     assertEquals(0, partitionMetadata.partition)
     assertEquals(expectedError, partitionMetadata.error)
-    assertTrue(partitionMetadata.isr.isEmpty)
+    assertFalse(partitionMetadata.isr.isEmpty)
     assertEquals(1, partitionMetadata.replicas.size)
     assertEquals(0, partitionMetadata.replicas.get(0).id)
   }

From a48b5d900c6b5c9c52a97124a1b51aff3636c32c Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Tue, 30 Jul 2019 08:50:13 -0700
Subject: [PATCH 0487/1071] KAFKA-8717; Reuse cached offset metadata when
 reading from log (#7081)

Although we currently cache offset metadata for the high watermark and last stable offset, we don't use it when reading from the log. Instead we always look it up from the index. This patch pushes fetch isolation into `Log.read` so that we are able to reuse the cached offset metadata.

Reviewers: Jun Rao 
---
 .../main/scala/kafka/cluster/Partition.scala  |  18 +-
 .../group/GroupMetadataManager.scala          |   8 +-
 .../transaction/TransactionStateManager.scala |  14 +-
 core/src/main/scala/kafka/log/Log.scala       |  99 ++++++---
 .../src/main/scala/kafka/log/LogSegment.scala |  32 +--
 .../scala/other/kafka/StressTestLog.scala     |   9 +-
 .../group/GroupMetadataManagerTest.scala      |  15 +-
 ...ransactionCoordinatorConcurrencyTest.scala |   7 +-
 .../TransactionStateManagerTest.scala         |   7 +-
 .../kafka/log/BrokerCompressionTest.scala     |   8 +-
 .../scala/unit/kafka/log/LogManagerTest.scala |   4 +-
 .../scala/unit/kafka/log/LogSegmentTest.scala |  46 ++--
 .../test/scala/unit/kafka/log/LogTest.scala   | 208 +++++++++++++++---
 .../server/ReplicaManagerQuotasTest.scala     |  10 +-
 .../unit/kafka/server/SimpleFetchTest.scala   |  10 +-
 ...venReplicationProtocolAcceptanceTest.scala |   4 +-
 .../epoch/LeaderEpochIntegrationTest.scala    |   4 +-
 17 files changed, 314 insertions(+), 189 deletions(-)

diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala
index f18ddd2ca03e9..87cbe54f05563 100755
--- a/core/src/main/scala/kafka/cluster/Partition.scala
+++ b/core/src/main/scala/kafka/cluster/Partition.scala
@@ -948,25 +948,13 @@ class Partition(val topicPartition: TopicPartition,
     // decide whether to only fetch from leader
     val localLog = localLogWithEpochOrException(currentLeaderEpoch, fetchOnlyFromLeader)
 
-    /* Read the LogOffsetMetadata prior to performing the read from the log.
-     * We use the LogOffsetMetadata to determine if a particular replica is in-sync or not.
-     * Using the log end offset after performing the read can lead to a race condition
-     * where data gets appended to the log immediately after the replica has consumed from it
-     * This can cause a replica to always be out of sync.
-     */
+    // Note we use the log end offset prior to the read. This ensures that any appends following
+    // the fetch do not prevent a follower from coming into sync.
     val initialHighWatermark = localLog.highWatermark
     val initialLogStartOffset = localLog.logStartOffset
     val initialLogEndOffset = localLog.logEndOffset
     val initialLastStableOffset = localLog.lastStableOffset
-
-    val maxOffsetOpt = fetchIsolation match {
-      case FetchLogEnd => None
-      case FetchHighWatermark => Some(initialHighWatermark)
-      case FetchTxnCommitted => Some(initialLastStableOffset)
-    }
-
-    val fetchedData = localLog.read(fetchOffset, maxBytes, maxOffsetOpt, minOneMessage,
-          includeAbortedTxns = fetchIsolation == FetchTxnCommitted)
+    val fetchedData = localLog.read(fetchOffset, maxBytes, fetchIsolation, minOneMessage)
 
     LogReadInfo(
       fetchedData = fetchedData,
diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
index c6669755c5de4..03c3e37752040 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
@@ -29,7 +29,7 @@ import com.yammer.metrics.core.Gauge
 import kafka.api.{ApiVersion, KAFKA_0_10_1_IV0, KAFKA_2_1_IV0, KAFKA_2_1_IV1, KAFKA_2_3_IV0}
 import kafka.common.{MessageFormatter, OffsetAndMetadata}
 import kafka.metrics.KafkaMetricsGroup
-import kafka.server.ReplicaManager
+import kafka.server.{FetchLogEnd, ReplicaManager}
 import kafka.utils.CoreUtils.inLock
 import kafka.utils._
 import kafka.zk.KafkaZkClient
@@ -529,8 +529,10 @@ class GroupMetadataManager(brokerId: Int,
         val removedGroups = mutable.Set[String]()
 
         while (currOffset < logEndOffset && !shuttingDown.get()) {
-          val fetchDataInfo = log.read(currOffset, config.loadBufferSize, maxOffset = None,
-            minOneMessage = true, includeAbortedTxns = false)
+          val fetchDataInfo = log.read(currOffset,
+            maxLength = config.loadBufferSize,
+            isolation = FetchLogEnd,
+            minOneMessage = true)
           val memRecords = fetchDataInfo.records match {
             case records: MemoryRecords => records
             case fileRecords: FileRecords =>
diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala
index 92cba507fb50a..45ee4e998aec2 100644
--- a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala
+++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala
@@ -24,8 +24,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock
 
 import kafka.log.LogConfig
 import kafka.message.UncompressedCodec
-import kafka.server.Defaults
-import kafka.server.ReplicaManager
+import kafka.server.{Defaults, FetchLogEnd, ReplicaManager}
 import kafka.utils.CoreUtils.{inReadLock, inWriteLock}
 import kafka.utils.{Logging, Pool, Scheduler}
 import kafka.zk.KafkaZkClient
@@ -297,12 +296,13 @@ class TransactionStateManager(brokerId: Int,
         var currOffset = log.logStartOffset
 
         try {
-          while (currOffset < logEndOffset
-            && !shuttingDown.get()
-            && inReadLock(stateLock) {loadingPartitions.exists { idAndEpoch: TransactionPartitionAndLeaderEpoch =>
+          while (currOffset < logEndOffset && !shuttingDown.get() && inReadLock(stateLock) {
+            loadingPartitions.exists { idAndEpoch: TransactionPartitionAndLeaderEpoch =>
               idAndEpoch.txnPartitionId == topicPartition.partition && idAndEpoch.coordinatorEpoch == coordinatorEpoch}}) {
-            val fetchDataInfo = log.read(currOffset, config.transactionLogLoadBufferSize, maxOffset = None,
-              minOneMessage = true, includeAbortedTxns = false)
+            val fetchDataInfo = log.read(currOffset,
+              maxLength = config.transactionLogLoadBufferSize,
+              isolation = FetchLogEnd,
+              minOneMessage = true)
             val memRecords = fetchDataInfo.records match {
               case records: MemoryRecords => records
               case fileRecords: FileRecords =>
diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index 5e12e315bbfab..f8df2115e9aa0 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -34,7 +34,7 @@ import kafka.message.{BrokerCompressionCodec, CompressionCodec, NoCompressionCod
 import kafka.metrics.KafkaMetricsGroup
 import kafka.server.checkpoints.LeaderEpochCheckpointFile
 import kafka.server.epoch.LeaderEpochFileCache
-import kafka.server.{BrokerTopicStats, FetchDataInfo, LogDirFailureChannel, LogOffsetMetadata, OffsetAndEpoch}
+import kafka.server.{BrokerTopicStats, FetchDataInfo, FetchHighWatermark, FetchIsolation, FetchLogEnd, FetchTxnCommitted, LogDirFailureChannel, LogOffsetMetadata, OffsetAndEpoch}
 import kafka.utils._
 import org.apache.kafka.common.errors._
 import org.apache.kafka.common.record.FileRecords.TimestampAndOffset
@@ -844,6 +844,8 @@ class Log(@volatile var dir: File,
       // and we can skip the loading. This is an optimization for users which are not yet using
       // idempotent/transactional features yet.
       if (lastOffset > producerStateManager.mapEndOffset && !isEmptyBeforeTruncation) {
+        val segmentOfLastOffset = floorLogSegment(lastOffset)
+
         logSegments(producerStateManager.mapEndOffset, lastOffset).foreach { segment =>
           val startOffset = Utils.max(segment.baseOffset, producerStateManager.mapEndOffset, logStartOffset)
           producerStateManager.updateMapEndOffset(startOffset)
@@ -851,7 +853,18 @@ class Log(@volatile var dir: File,
           if (offsetsToSnapshot.contains(Some(segment.baseOffset)))
             producerStateManager.takeSnapshot()
 
-          val fetchDataInfo = segment.read(startOffset, Some(lastOffset), Int.MaxValue)
+          val maxPosition = if (segmentOfLastOffset.contains(segment)) {
+            Option(segment.translateOffset(lastOffset))
+              .map(_.position)
+              .getOrElse(segment.size)
+          } else {
+            segment.size
+          }
+
+          val fetchDataInfo = segment.read(startOffset,
+            maxSize = Int.MaxValue,
+            maxPosition = maxPosition,
+            minOneMessage = false)
           if (fetchDataInfo != null)
             loadProducersFromLog(producerStateManager, fetchDataInfo.records)
         }
@@ -1389,43 +1402,60 @@ class Log(@volatile var dir: File,
     }
   }
 
+  private def emptyFetchDataInfo(fetchOffsetMetadata: LogOffsetMetadata,
+                                 includeAbortedTxns: Boolean): FetchDataInfo = {
+    val abortedTransactions =
+      if (includeAbortedTxns) Some(List.empty[AbortedTransaction])
+      else None
+    FetchDataInfo(fetchOffsetMetadata,
+      MemoryRecords.EMPTY,
+      firstEntryIncomplete = false,
+      abortedTransactions = abortedTransactions)
+  }
+
   /**
    * Read messages from the log.
    *
    * @param startOffset The offset to begin reading at
    * @param maxLength The maximum number of bytes to read
-   * @param maxOffset The offset to read up to, exclusive. (i.e. this offset NOT included in the resulting message set)
+   * @param isolation The fetch isolation, which controls the maximum offset we are allowed to read
    * @param minOneMessage If this is true, the first message will be returned even if it exceeds `maxLength` (if one exists)
-   * @param includeAbortedTxns Whether or not to lookup aborted transactions for fetched data
    * @throws OffsetOutOfRangeException If startOffset is beyond the log end offset or before the log start offset
    * @return The fetch data information including fetch starting offset metadata and messages read.
    */
   def read(startOffset: Long,
            maxLength: Int,
-           maxOffset: Option[Long],
-           minOneMessage: Boolean,
-           includeAbortedTxns: Boolean): FetchDataInfo = {
+           isolation: FetchIsolation,
+           minOneMessage: Boolean): FetchDataInfo = {
     maybeHandleIOException(s"Exception while reading from $topicPartition in dir ${dir.getParent}") {
       trace(s"Reading $maxLength bytes from offset $startOffset of length $size bytes")
 
-      // Because we don't use lock for reading, the synchronization is a little bit tricky.
+      val includeAbortedTxns = isolation == FetchTxnCommitted
+
+      // Because we don't use the lock for reading, the synchronization is a little bit tricky.
       // We create the local variables to avoid race conditions with updates to the log.
-      val currentNextOffsetMetadata = nextOffsetMetadata
-      val next = currentNextOffsetMetadata.messageOffset
-      if (startOffset == next) {
-        val abortedTransactions =
-          if (includeAbortedTxns) Some(List.empty[AbortedTransaction])
-          else None
-        return FetchDataInfo(currentNextOffsetMetadata, MemoryRecords.EMPTY, firstEntryIncomplete = false,
-          abortedTransactions = abortedTransactions)
-      }
+      val endOffsetMetadata = nextOffsetMetadata
+      val endOffset = nextOffsetMetadata.messageOffset
+      if (startOffset == endOffset)
+        return emptyFetchDataInfo(endOffsetMetadata, includeAbortedTxns)
 
       var segmentEntry = segments.floorEntry(startOffset)
 
       // return error on attempt to read beyond the log end offset or read below log start offset
-      if (startOffset > next || segmentEntry == null || startOffset < logStartOffset)
+      if (startOffset > endOffset || segmentEntry == null || startOffset < logStartOffset)
         throw new OffsetOutOfRangeException(s"Received request for offset $startOffset for partition $topicPartition, " +
-          s"but we only have log segments in the range $logStartOffset to $next.")
+          s"but we only have log segments in the range $logStartOffset to $endOffset.")
+
+      val maxOffsetMetadata = isolation match {
+        case FetchLogEnd => nextOffsetMetadata
+        case FetchHighWatermark => fetchHighWatermarkMetadata
+        case FetchTxnCommitted => fetchLastStableOffsetMetadata
+      }
+
+      if (startOffset > maxOffsetMetadata.messageOffset) {
+        val startOffsetMetadata = convertToOffsetMetadataOrThrow(startOffset)
+        return emptyFetchDataInfo(startOffsetMetadata, includeAbortedTxns)
+      }
 
       // Do the read on the segment with a base offset less than the target offset
       // but if that segment doesn't contain any messages with an offset greater than that
@@ -1433,24 +1463,16 @@ class Log(@volatile var dir: File,
       while (segmentEntry != null) {
         val segment = segmentEntry.getValue
 
-        // If the fetch occurs on the active segment, there might be a race condition where two fetch requests occur after
-        // the message is appended but before the nextOffsetMetadata is updated. In that case the second fetch may
-        // cause OffsetOutOfRangeException. To solve that, we cap the reading up to exposed position instead of the log
-        // end of the active segment.
         val maxPosition = {
-          if (segmentEntry == segments.lastEntry) {
-            val exposedPos = nextOffsetMetadata.relativePositionInSegment.toLong
-            // Check the segment again in case a new segment has just rolled out.
-            if (segmentEntry != segments.lastEntry)
-            // New log segment has rolled out, we can read up to the file end.
-              segment.size
-            else
-              exposedPos
+          // Use the max offset position if it is on this segment; otherwise, the segment size is the limit.
+          if (maxOffsetMetadata.segmentBaseOffset == segment.baseOffset) {
+            maxOffsetMetadata.relativePositionInSegment
           } else {
             segment.size
           }
         }
-        val fetchInfo = segment.read(startOffset, maxOffset, maxLength, maxPosition, minOneMessage)
+
+        val fetchInfo = segment.read(startOffset, maxLength, maxPosition, minOneMessage)
         if (fetchInfo == null) {
           segmentEntry = segments.higherEntry(segmentEntry.getKey)
         } else {
@@ -1623,9 +1645,8 @@ class Log(@volatile var dir: File,
   private def convertToOffsetMetadataOrThrow(offset: Long): LogOffsetMetadata = {
     val fetchDataInfo = read(offset,
       maxLength = 1,
-      maxOffset = None,
-      minOneMessage = false,
-      includeAbortedTxns = false)
+      isolation = FetchLogEnd,
+      minOneMessage = false)
     fetchDataInfo.fetchOffsetMetadata
   }
 
@@ -2086,6 +2107,14 @@ class Log(@volatile var dir: File,
     }
   }
 
+  /**
+   * Get the largest log segment with a base offset less than or equal to the given offset, if one exists.
+   * @return the optional log segment
+   */
+  private def floorLogSegment(offset: Long): Option[LogSegment] = {
+    Option(segments.floorEntry(offset)).map(_.getValue)
+  }
+
   override def toString: String = {
     val logString = new StringBuilder
     logString.append(s"Log(dir=$dir")
diff --git a/core/src/main/scala/kafka/log/LogSegment.scala b/core/src/main/scala/kafka/log/LogSegment.scala
index 37b2f9da48096..4c16291c170ae 100755
--- a/core/src/main/scala/kafka/log/LogSegment.scala
+++ b/core/src/main/scala/kafka/log/LogSegment.scala
@@ -43,8 +43,8 @@ import scala.math._
  * A segment with a base offset of [base_offset] would be stored in two files, a [base_offset].index and a [base_offset].log file.
  *
  * @param log The file records containing log entries
- * @param offsetIndex The offset index
- * @param timeIndex The timestamp index
+ * @param lazyOffsetIndex The offset index
+ * @param lazyTimeIndex The timestamp index
  * @param txnIndex The transaction index
  * @param baseOffset A lower bound on the offsets in this segment
  * @param indexIntervalBytes The approximate number of bytes between entries in the index
@@ -279,7 +279,6 @@ class LogSegment private[log] (val log: FileRecords,
    * no more than maxSize bytes and will end before maxOffset if a maxOffset is specified.
    *
    * @param startOffset A lower bound on the first offset to include in the message set we read
-   * @param maxOffset An optional maximum offset for the message set we read
    * @param maxSize The maximum number of bytes to include in the message set we read
    * @param maxPosition The maximum position in the log segment that should be exposed for read
    * @param minOneMessage If this is true, the first message will be returned even if it exceeds `maxSize` (if one exists)
@@ -288,12 +287,13 @@ class LogSegment private[log] (val log: FileRecords,
    *         or null if the startOffset is larger than the largest offset in this log
    */
   @threadsafe
-  def read(startOffset: Long, maxOffset: Option[Long], maxSize: Int, maxPosition: Long = size,
+  def read(startOffset: Long,
+           maxSize: Int,
+           maxPosition: Long = size,
            minOneMessage: Boolean = false): FetchDataInfo = {
     if (maxSize < 0)
       throw new IllegalArgumentException(s"Invalid max size $maxSize for log read from segment $log")
 
-    val logSize = log.sizeInBytes // this may change, need to save a consistent copy
     val startOffsetAndSize = translateOffset(startOffset)
 
     // if the start position is already off the end of the log, return null
@@ -312,25 +312,7 @@ class LogSegment private[log] (val log: FileRecords,
       return FetchDataInfo(offsetMetadata, MemoryRecords.EMPTY)
 
     // calculate the length of the message set to read based on whether or not they gave us a maxOffset
-    val fetchSize: Int = maxOffset match {
-      case None =>
-        // no max offset, just read until the max position
-        min((maxPosition - startPosition).toInt, adjustedMaxSize)
-      case Some(offset) =>
-        // there is a max offset, translate it to a file position and use that to calculate the max read size;
-        // when the leader of a partition changes, it's possible for the new leader's high watermark to be less than the
-        // true high watermark in the previous leader for a short window. In this window, if a consumer fetches on an
-        // offset between new leader's high watermark and the log end offset, we want to return an empty response.
-        if (offset < startOffset)
-          return FetchDataInfo(offsetMetadata, MemoryRecords.EMPTY, firstEntryIncomplete = false)
-        val mapping = translateOffset(offset, startPosition)
-        val endPosition =
-          if (mapping == null)
-            logSize // the max offset is off the end of the log, use the end of the file
-          else
-            mapping.position
-        min(min(maxPosition, endPosition) - startPosition, adjustedMaxSize).toInt
-    }
+    val fetchSize: Int = min((maxPosition - startPosition).toInt, adjustedMaxSize)
 
     FetchDataInfo(offsetMetadata, log.slice(startPosition, fetchSize),
       firstEntryIncomplete = adjustedMaxSize < startOffsetAndSize.size)
@@ -467,7 +449,7 @@ class LogSegment private[log] (val log: FileRecords,
    */
   @threadsafe
   def readNextOffset: Long = {
-    val fetchData = read(offsetIndex.lastOffset, None, log.sizeInBytes)
+    val fetchData = read(offsetIndex.lastOffset, log.sizeInBytes)
     if (fetchData == null)
       baseOffset
     else
diff --git a/core/src/test/scala/other/kafka/StressTestLog.scala b/core/src/test/scala/other/kafka/StressTestLog.scala
index 7821a0e007d3c..6ff3a9874550c 100755
--- a/core/src/test/scala/other/kafka/StressTestLog.scala
+++ b/core/src/test/scala/other/kafka/StressTestLog.scala
@@ -21,7 +21,7 @@ import java.util.Properties
 import java.util.concurrent.atomic._
 
 import kafka.log._
-import kafka.server.{BrokerTopicStats, LogDirFailureChannel}
+import kafka.server.{BrokerTopicStats, FetchLogEnd, LogDirFailureChannel}
 import kafka.utils._
 import org.apache.kafka.clients.consumer.OffsetOutOfRangeException
 import org.apache.kafka.common.record.FileRecords
@@ -132,10 +132,9 @@ object StressTestLog {
     override def work() {
       try {
         log.read(currentOffset,
-          maxLength = 1024,
-          maxOffset = Some(currentOffset + 1),
-          minOneMessage = true,
-          includeAbortedTxns = false).records match {
+          maxLength = 1,
+          isolation = FetchLogEnd,
+          minOneMessage = true).records match {
           case read: FileRecords if read.sizeInBytes > 0 => {
             val first = read.batches.iterator.next()
             require(first.lastOffset == currentOffset, "We should either read nothing or the message we asked for.")
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
index 4e06716677909..faca4479b17a3 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
@@ -23,12 +23,12 @@ import java.nio.ByteBuffer
 import java.util.Collections
 import java.util.Optional
 import java.util.concurrent.locks.ReentrantLock
+
 import kafka.api._
 import kafka.cluster.Partition
 import kafka.common.OffsetAndMetadata
 import kafka.log.{Log, LogAppendInfo}
-import kafka.server.{FetchDataInfo, KafkaConfig, LogOffsetMetadata, ReplicaManager}
-import kafka.server.HostedPartition
+import kafka.server.{FetchDataInfo, FetchLogEnd, HostedPartition, KafkaConfig, LogOffsetMetadata, ReplicaManager}
 import kafka.utils.{KafkaScheduler, MockTime, TestUtils}
 import kafka.zk.KafkaZkClient
 import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription
@@ -43,6 +43,7 @@ import org.easymock.{Capture, EasyMock, IAnswer}
 import org.junit.Assert.{assertEquals, assertFalse, assertNull, assertTrue}
 import org.junit.{Before, Test}
 import org.scalatest.Assertions.fail
+
 import scala.collection.JavaConverters._
 import scala.collection._
 
@@ -1875,9 +1876,8 @@ class GroupMetadataManagerTest {
     EasyMock.expect(logMock.logStartOffset).andReturn(startOffset).anyTimes()
     EasyMock.expect(logMock.read(EasyMock.eq(startOffset),
       maxLength = EasyMock.anyInt(),
-      maxOffset = EasyMock.eq(None),
-      minOneMessage = EasyMock.eq(true),
-      includeAbortedTxns = EasyMock.eq(false)))
+      isolation = EasyMock.eq(FetchLogEnd),
+      minOneMessage = EasyMock.eq(true)))
       .andReturn(FetchDataInfo(LogOffsetMetadata(startOffset), mockRecords))
     EasyMock.expect(replicaManager.getLog(groupMetadataTopicPartition)).andStubReturn(Some(logMock))
     EasyMock.expect(replicaManager.getLogEndOffset(groupMetadataTopicPartition)).andStubReturn(Some(18))
@@ -1981,9 +1981,8 @@ class GroupMetadataManagerTest {
     EasyMock.expect(logMock.logStartOffset).andStubReturn(startOffset)
     EasyMock.expect(logMock.read(EasyMock.eq(startOffset),
       maxLength = EasyMock.anyInt(),
-      maxOffset = EasyMock.eq(None),
-      minOneMessage = EasyMock.eq(true),
-      includeAbortedTxns = EasyMock.eq(false)))
+      isolation = EasyMock.eq(FetchLogEnd),
+      minOneMessage = EasyMock.eq(true)))
       .andReturn(FetchDataInfo(LogOffsetMetadata(startOffset), fileRecordsMock))
 
     EasyMock.expect(fileRecordsMock.sizeInBytes()).andStubReturn(records.sizeInBytes)
diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala
index 9719a486475e9..bc6ed9330e3da 100644
--- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala
@@ -22,7 +22,7 @@ import kafka.coordinator.AbstractCoordinatorConcurrencyTest
 import kafka.coordinator.AbstractCoordinatorConcurrencyTest._
 import kafka.coordinator.transaction.TransactionCoordinatorConcurrencyTest._
 import kafka.log.Log
-import kafka.server.{DelayedOperationPurgatory, FetchDataInfo, KafkaConfig, LogOffsetMetadata, MetadataCache}
+import kafka.server.{DelayedOperationPurgatory, FetchDataInfo, FetchLogEnd, KafkaConfig, LogOffsetMetadata, MetadataCache}
 import kafka.utils.timer.MockTimer
 import kafka.utils.{Pool, TestUtils}
 import org.apache.kafka.clients.{ClientResponse, NetworkClient}
@@ -257,9 +257,8 @@ class TransactionCoordinatorConcurrencyTest extends AbstractCoordinatorConcurren
     EasyMock.expect(logMock.logStartOffset).andStubReturn(startOffset)
     EasyMock.expect(logMock.read(EasyMock.eq(startOffset),
       maxLength = EasyMock.anyInt(),
-      maxOffset = EasyMock.eq(None),
-      minOneMessage = EasyMock.eq(true),
-      includeAbortedTxns = EasyMock.eq(false)))
+      isolation = EasyMock.eq(FetchLogEnd),
+      minOneMessage = EasyMock.eq(true)))
       .andReturn(FetchDataInfo(LogOffsetMetadata(startOffset), fileRecordsMock))
 
     EasyMock.expect(fileRecordsMock.sizeInBytes()).andStubReturn(records.sizeInBytes)
diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala
index 08d0b3260f84d..14745e7659065 100644
--- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala
@@ -20,7 +20,7 @@ import java.nio.ByteBuffer
 import java.util.concurrent.locks.ReentrantLock
 
 import kafka.log.Log
-import kafka.server.{FetchDataInfo, LogOffsetMetadata, ReplicaManager}
+import kafka.server.{FetchDataInfo, FetchLogEnd, LogOffsetMetadata, ReplicaManager}
 import kafka.utils.{MockScheduler, Pool}
 import org.scalatest.Assertions.fail
 import kafka.zk.KafkaZkClient
@@ -584,9 +584,8 @@ class TransactionStateManagerTest {
     EasyMock.expect(logMock.logStartOffset).andStubReturn(startOffset)
     EasyMock.expect(logMock.read(EasyMock.eq(startOffset),
       maxLength = EasyMock.anyInt(),
-      maxOffset = EasyMock.eq(None),
-      minOneMessage = EasyMock.eq(true),
-      includeAbortedTxns = EasyMock.eq(false)))
+      isolation = EasyMock.eq(FetchLogEnd),
+      minOneMessage = EasyMock.eq(true)))
       .andReturn(FetchDataInfo(LogOffsetMetadata(startOffset), fileRecordsMock))
 
     EasyMock.expect(fileRecordsMock.sizeInBytes()).andStubReturn(records.sizeInBytes)
diff --git a/core/src/test/scala/unit/kafka/log/BrokerCompressionTest.scala b/core/src/test/scala/unit/kafka/log/BrokerCompressionTest.scala
index 557cef3955514..d7a725b19f3ad 100755
--- a/core/src/test/scala/unit/kafka/log/BrokerCompressionTest.scala
+++ b/core/src/test/scala/unit/kafka/log/BrokerCompressionTest.scala
@@ -28,7 +28,7 @@ import org.apache.kafka.common.record.{CompressionType, MemoryRecords, RecordBat
 import org.apache.kafka.common.utils.Utils
 import java.util.{Collection, Properties}
 
-import kafka.server.{BrokerTopicStats, LogDirFailureChannel}
+import kafka.server.{BrokerTopicStats, FetchLogEnd, LogDirFailureChannel}
 
 import scala.collection.JavaConverters._
 
@@ -64,8 +64,10 @@ class BrokerCompressionTest(messageCompression: String, brokerCompression: Strin
           new SimpleRecord("hello".getBytes), new SimpleRecord("there".getBytes)), leaderEpoch = 0)
 
     def readBatch(offset: Int): RecordBatch = {
-      val fetchInfo = log.read(offset, 4096, maxOffset = None,
-        includeAbortedTxns = false, minOneMessage = true)
+      val fetchInfo = log.read(offset,
+        maxLength = 4096,
+        isolation = FetchLogEnd,
+        minOneMessage = true)
       fetchInfo.records.batches.iterator.next()
     }
 
diff --git a/core/src/test/scala/unit/kafka/log/LogManagerTest.scala b/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
index bfbd423371867..684f0b21db6a5 100755
--- a/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
@@ -20,7 +20,7 @@ package kafka.log
 import java.io._
 import java.util.{Collections, Properties}
 
-import kafka.server.FetchDataInfo
+import kafka.server.{FetchDataInfo, FetchLogEnd}
 import kafka.server.checkpoints.OffsetCheckpointFile
 import kafka.utils._
 import org.apache.kafka.common.errors.OffsetOutOfRangeException
@@ -465,7 +465,7 @@ class LogManagerTest {
   }
 
   private def readLog(log: Log, offset: Long, maxLength: Int = 1024): FetchDataInfo = {
-    log.read(offset, maxLength, maxOffset = None, minOneMessage = true, includeAbortedTxns = false)
+    log.read(offset, maxLength, isolation = FetchLogEnd, minOneMessage = true)
   }
 
 }
diff --git a/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala b/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala
index 0ebc0eff0b046..c3a6899ed3093 100644
--- a/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala
@@ -67,7 +67,7 @@ class LogSegmentTest {
   @Test
   def testReadOnEmptySegment() {
     val seg = createSegment(40)
-    val read = seg.read(startOffset = 40, maxSize = 300, maxOffset = None)
+    val read = seg.read(startOffset = 40, maxSize = 300)
     assertNull("Read beyond the last offset in the segment should be null", read)
   }
 
@@ -80,28 +80,10 @@ class LogSegmentTest {
     val seg = createSegment(40)
     val ms = records(50, "hello", "there", "little", "bee")
     seg.append(53, RecordBatch.NO_TIMESTAMP, -1L, ms)
-    val read = seg.read(startOffset = 41, maxSize = 300, maxOffset = None).records
+    val read = seg.read(startOffset = 41, maxSize = 300).records
     checkEquals(ms.records.iterator, read.records.iterator)
   }
 
-  /**
-   * If we set the startOffset and maxOffset for the read to be the same value
-   * we should get only the first message in the log
-   */
-  @Test
-  def testMaxOffset() {
-    val baseOffset = 50
-    val seg = createSegment(baseOffset)
-    val ms = records(baseOffset, "hello", "there", "beautiful")
-    seg.append(52, RecordBatch.NO_TIMESTAMP, -1L, ms)
-    def validate(offset: Long) =
-      assertEquals(ms.records.asScala.filter(_.offset == offset).toList,
-                   seg.read(startOffset = offset, maxSize = 1024, maxOffset = Some(offset+1)).records.records.asScala.toList)
-    validate(50)
-    validate(51)
-    validate(52)
-  }
-
   /**
    * If we read from an offset beyond the last offset in the segment we should get null
    */
@@ -110,7 +92,7 @@ class LogSegmentTest {
     val seg = createSegment(40)
     val ms = records(50, "hello", "there")
     seg.append(51, RecordBatch.NO_TIMESTAMP, -1L, ms)
-    val read = seg.read(startOffset = 52, maxSize = 200, maxOffset = None)
+    val read = seg.read(startOffset = 52, maxSize = 200)
     assertNull("Read beyond the last offset in the segment should give null", read)
   }
 
@@ -125,7 +107,7 @@ class LogSegmentTest {
     seg.append(51, RecordBatch.NO_TIMESTAMP, -1L, ms)
     val ms2 = records(60, "alpha", "beta")
     seg.append(61, RecordBatch.NO_TIMESTAMP, -1L, ms2)
-    val read = seg.read(startOffset = 55, maxSize = 200, maxOffset = None)
+    val read = seg.read(startOffset = 55, maxSize = 200)
     checkEquals(ms2.records.iterator, read.records.records.iterator)
   }
 
@@ -143,11 +125,11 @@ class LogSegmentTest {
       val ms2 = records(offset + 1, "hello")
       seg.append(offset + 1, RecordBatch.NO_TIMESTAMP, -1L, ms2)
       // check that we can read back both messages
-      val read = seg.read(offset, None, 10000)
+      val read = seg.read(offset, 10000)
       assertEquals(List(ms1.records.iterator.next(), ms2.records.iterator.next()), read.records.records.asScala.toList)
       // now truncate off the last message
       seg.truncateTo(offset + 1)
-      val read2 = seg.read(offset, None, 10000)
+      val read2 = seg.read(offset, 10000)
       assertEquals(1, read2.records.records.asScala.size)
       checkEquals(ms1.records.iterator, read2.records.records.iterator)
       offset += 1
@@ -230,7 +212,7 @@ class LogSegmentTest {
     assertEquals(0, seg.timeWaitedForRoll(time.milliseconds(), RecordBatch.NO_TIMESTAMP))
     assertFalse(seg.timeIndex.isFull)
     assertFalse(seg.offsetIndex.isFull)
-    assertNull("Segment should be empty.", seg.read(0, None, 1024))
+    assertNull("Segment should be empty.", seg.read(0, 1024))
 
     seg.append(41, RecordBatch.NO_TIMESTAMP, -1L, records(40, "hello", "there"))
   }
@@ -299,8 +281,10 @@ class LogSegmentTest {
     val indexFile = seg.lazyOffsetIndex.file
     TestUtils.writeNonsenseToFile(indexFile, 5, indexFile.length.toInt)
     seg.recover(new ProducerStateManager(topicPartition, logDir))
-    for(i <- 0 until 100)
-      assertEquals(i, seg.read(i, Some(i + 1), 1024).records.records.iterator.next().offset)
+    for(i <- 0 until 100) {
+      val records = seg.read(i, 1, minOneMessage = true).records.records
+      assertEquals(i, records.iterator.next().offset)
+    }
   }
 
   @Test
@@ -439,7 +423,7 @@ class LogSegmentTest {
     seg.append(51, RecordBatch.NO_TIMESTAMP, -1L, ms)
     val ms2 = records(60, "alpha", "beta")
     seg.append(61, RecordBatch.NO_TIMESTAMP, -1L, ms2)
-    val read = seg.read(startOffset = 55, maxSize = 200, maxOffset = None)
+    val read = seg.read(startOffset = 55, maxSize = 200)
     checkEquals(ms2.records.iterator, read.records.records.iterator)
   }
 
@@ -460,7 +444,7 @@ class LogSegmentTest {
     seg.append(51, RecordBatch.NO_TIMESTAMP, -1L, ms)
     val ms2 = records(60, "alpha", "beta")
     seg.append(61, RecordBatch.NO_TIMESTAMP, -1L, ms2)
-    val read = seg.read(startOffset = 55, maxSize = 200, maxOffset = None)
+    val read = seg.read(startOffset = 55, maxSize = 200)
     checkEquals(ms2.records.iterator, read.records.records.iterator)
     val oldSize = seg.log.sizeInBytes()
     val oldPosition = seg.log.channel.position
@@ -474,7 +458,7 @@ class LogSegmentTest {
       initFileSize = 512 * 1024 * 1024, preallocate = true)
     segments += segReopen
 
-    val readAgain = segReopen.read(startOffset = 55, maxSize = 200, maxOffset = None)
+    val readAgain = segReopen.read(startOffset = 55, maxSize = 200)
     checkEquals(ms2.records.iterator, readAgain.records.records.iterator)
     val size = segReopen.log.sizeInBytes()
     val position = segReopen.log.channel.position
@@ -503,7 +487,7 @@ class LogSegmentTest {
     seg.truncateTo(offset + 1)
 
     //Then we should still truncate the record that was present (i.e. offset + 3 is gone)
-    val log = seg.read(offset, None, 10000)
+    val log = seg.read(offset, 10000)
     assertEquals(offset, log.records.batches.iterator.next().baseOffset())
     assertEquals(1, log.records.batches.asScala.size)
   }
diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala
index 46494e788d138..fa7be7eb88c43 100755
--- a/core/src/test/scala/unit/kafka/log/LogTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogTest.scala
@@ -28,7 +28,7 @@ import kafka.common.{OffsetsOutOfOrderException, UnexpectedAppendOffsetException
 import kafka.log.Log.DeleteDirSuffix
 import kafka.server.checkpoints.LeaderEpochCheckpointFile
 import kafka.server.epoch.{EpochEntry, LeaderEpochFileCache}
-import kafka.server.{BrokerTopicStats, FetchDataInfo, KafkaConfig, LogDirFailureChannel, LogOffsetMetadata}
+import kafka.server.{BrokerTopicStats, FetchDataInfo, FetchHighWatermark, FetchIsolation, FetchLogEnd, FetchTxnCommitted, KafkaConfig, LogDirFailureChannel, LogOffsetMetadata}
 import kafka.utils._
 import org.apache.kafka.common.{KafkaException, TopicPartition}
 import org.apache.kafka.common.errors._
@@ -77,7 +77,7 @@ class LogTest {
 
   @Test
   def testHighWatermarkMaintenance(): Unit = {
-    val logConfig = LogTest.createLogConfig(segmentMs = 1 * 60 * 60L)
+    val logConfig = LogTest.createLogConfig(segmentBytes = 1024 * 1024)
     val log = createLog(logDir, logConfig)
 
     val records = TestUtils.records(List(
@@ -116,6 +116,142 @@ class LogTest {
     assertHighWatermark(3L)
   }
 
+  private def assertNonEmptyFetch(log: Log, offset: Long, isolation: FetchIsolation): Unit = {
+    val readInfo = log.read(startOffset = offset,
+      maxLength = Int.MaxValue,
+      isolation = isolation,
+      minOneMessage = true)
+
+    assertFalse(readInfo.firstEntryIncomplete)
+    assertTrue(readInfo.records.sizeInBytes > 0)
+
+    val upperBoundOffset = isolation match {
+      case FetchLogEnd => log.logEndOffset
+      case FetchHighWatermark => log.highWatermark
+      case FetchTxnCommitted => log.lastStableOffset
+    }
+
+    for (record <- readInfo.records.records.asScala)
+      assertTrue(record.offset < upperBoundOffset)
+
+    assertEquals(offset, readInfo.fetchOffsetMetadata.messageOffset)
+    assertValidLogOffsetMetadata(log, readInfo.fetchOffsetMetadata)
+  }
+
+  private def assertEmptyFetch(log: Log, offset: Long, isolation: FetchIsolation): Unit = {
+    val readInfo = log.read(startOffset = offset,
+      maxLength = Int.MaxValue,
+      isolation = isolation,
+      minOneMessage = true)
+    assertFalse(readInfo.firstEntryIncomplete)
+    assertEquals(0, readInfo.records.sizeInBytes)
+    assertEquals(offset, readInfo.fetchOffsetMetadata.messageOffset)
+    assertValidLogOffsetMetadata(log, readInfo.fetchOffsetMetadata)
+  }
+
+  @Test
+  def testFetchUpToLogEndOffset(): Unit = {
+    val logConfig = LogTest.createLogConfig(segmentBytes = 1024 * 1024)
+    val log = createLog(logDir, logConfig)
+
+    log.appendAsLeader(TestUtils.records(List(
+      new SimpleRecord("0".getBytes),
+      new SimpleRecord("1".getBytes),
+      new SimpleRecord("2".getBytes)
+    )), leaderEpoch = 0)
+    log.appendAsLeader(TestUtils.records(List(
+      new SimpleRecord("3".getBytes),
+      new SimpleRecord("4".getBytes)
+    )), leaderEpoch = 0)
+
+    (log.logStartOffset until log.logEndOffset).foreach { offset =>
+      assertNonEmptyFetch(log, offset, FetchLogEnd)
+    }
+  }
+
+  @Test
+  def testFetchUpToHighWatermark(): Unit = {
+    val logConfig = LogTest.createLogConfig(segmentBytes = 1024 * 1024)
+    val log = createLog(logDir, logConfig)
+
+    log.appendAsLeader(TestUtils.records(List(
+      new SimpleRecord("0".getBytes),
+      new SimpleRecord("1".getBytes),
+      new SimpleRecord("2".getBytes)
+    )), leaderEpoch = 0)
+    log.appendAsLeader(TestUtils.records(List(
+      new SimpleRecord("3".getBytes),
+      new SimpleRecord("4".getBytes)
+    )), leaderEpoch = 0)
+
+    def assertHighWatermarkBoundedFetches(): Unit = {
+      (log.logStartOffset until log.highWatermark).foreach { offset =>
+        assertNonEmptyFetch(log, offset, FetchHighWatermark)
+      }
+
+      (log.highWatermark to log.logEndOffset).foreach { offset =>
+        assertEmptyFetch(log, offset, FetchHighWatermark)
+      }
+    }
+
+    assertHighWatermarkBoundedFetches()
+
+    log.updateHighWatermark(3L)
+    assertHighWatermarkBoundedFetches()
+
+    log.updateHighWatermark(5L)
+    assertHighWatermarkBoundedFetches()
+  }
+
+  @Test
+  def testFetchUpToLastStableOffset(): Unit = {
+    val logConfig = LogTest.createLogConfig(segmentBytes = 1024 * 1024)
+    val log = createLog(logDir, logConfig)
+    val epoch = 0.toShort
+
+    val producerId1 = 1L
+    val producerId2 = 2L
+
+    val appendProducer1 = appendTransactionalAsLeader(log, producerId1, epoch)
+    val appendProducer2 = appendTransactionalAsLeader(log, producerId2, epoch)
+
+    appendProducer1(5)
+    appendNonTransactionalAsLeader(log, 3)
+    appendProducer2(2)
+    appendProducer1(4)
+    appendNonTransactionalAsLeader(log, 2)
+    appendProducer1(10)
+
+    def assertLsoBoundedFetches(): Unit = {
+      (log.logStartOffset until log.lastStableOffset).foreach { offset =>
+        assertNonEmptyFetch(log, offset, FetchTxnCommitted)
+      }
+
+      (log.lastStableOffset to log.logEndOffset).foreach { offset =>
+        assertEmptyFetch(log, offset, FetchTxnCommitted)
+      }
+    }
+
+    assertLsoBoundedFetches()
+
+    log.updateHighWatermark(log.logEndOffset)
+    assertLsoBoundedFetches()
+
+    appendEndTxnMarkerAsLeader(log, producerId1, epoch, ControlRecordType.COMMIT)
+    assertEquals(0L, log.lastStableOffset)
+
+    log.updateHighWatermark(log.logEndOffset)
+    assertEquals(8L, log.lastStableOffset)
+    assertLsoBoundedFetches()
+
+    appendEndTxnMarkerAsLeader(log, producerId2, epoch, ControlRecordType.ABORT)
+    assertEquals(8L, log.lastStableOffset)
+
+    log.updateHighWatermark(log.logEndOffset)
+    assertEquals(log.logEndOffset, log.lastStableOffset)
+    assertLsoBoundedFetches()
+  }
+
   @Test
   def testLogDeleteDirName(): Unit = {
     val name1 = Log.logDeleteDirName(new TopicPartition("foo", 3))
@@ -228,8 +364,8 @@ class LogTest {
     log.appendAsFollower(records2)
 
     assertEquals("Expect two records in the log", 2, log.logEndOffset)
-    assertEquals(0, readLog(log, 0, 100, Some(1)).records.batches.iterator.next().lastOffset)
-    assertEquals(1, readLog(log, 1, 100, Some(2)).records.batches.iterator.next().lastOffset)
+    assertEquals(0, readLog(log, 0, 1).records.batches.iterator.next().lastOffset)
+    assertEquals(1, readLog(log, 1, 1).records.batches.iterator.next().lastOffset)
 
     // roll so that active segment is empty
     log.roll()
@@ -243,7 +379,7 @@ class LogTest {
       baseOffset = 2L, partitionLeaderEpoch = 0)
     log.appendAsFollower(records3)
     assertTrue(log.activeSegment.offsetIndex.maxEntries > 1)
-    assertEquals(2, readLog(log, 2, 100, Some(3)).records.batches.iterator.next().lastOffset)
+    assertEquals(2, readLog(log, 2, 1).records.batches.iterator.next().lastOffset)
     assertEquals("Expect two segments.", 2, log.numberOfSegments)
   }
 
@@ -449,10 +585,9 @@ class LogTest {
           val wrapper = new LogSegment(segment.log, segment.lazyOffsetIndex, segment.lazyTimeIndex, segment.txnIndex, segment.baseOffset,
             segment.indexIntervalBytes, segment.rollJitterMs, mockTime) {
 
-            override def read(startOffset: Long, maxOffset: Option[Long], maxSize: Int, maxPosition: Long,
-                              minOneMessage: Boolean): FetchDataInfo = {
+            override def read(startOffset: Long, maxSize: Int, maxPosition: Long, minOneMessage: Boolean): FetchDataInfo = {
               segmentsWithReads += this
-              super.read(startOffset, maxOffset, maxSize, maxPosition, minOneMessage)
+              super.read(startOffset, maxSize, maxPosition, minOneMessage)
             }
 
             override def recover(producerStateManager: ProducerStateManager,
@@ -1392,7 +1527,7 @@ class LogTest {
       log.appendAsLeader(TestUtils.singletonRecords(value = value), leaderEpoch = 0)
 
     for(i <- values.indices) {
-      val read = readLog(log, i, 100, Some(i+1)).records.batches.iterator.next()
+      val read = readLog(log, i, 1).records.batches.iterator.next()
       assertEquals("Offset read should match order appended.", i, read.lastOffset)
       val actual = read.iterator.next()
       assertNull("Key should be null", actual.key)
@@ -1469,16 +1604,13 @@ class LogTest {
       val idx = messageIds.indexWhere(_ >= i)
       val reads = Seq(
         readLog(log, i, 1),
-        readLog(log, i, 100),
-        readLog(log, i, 100, Some(10000))
+        readLog(log, i, 100000),
+        readLog(log, i, 100)
       ).map(_.records.records.iterator.next())
       reads.foreach { read =>
         assertEquals("Offset read should match message id.", messageIds(idx), read.offset)
         assertEquals("Message should match appended.", records(idx), new SimpleRecord(read))
       }
-
-      val fetchedData = readLog(log, i, 1, Some(1))
-      assertEquals(Seq.empty, fetchedData.records.batches.asScala.toIndexedSeq)
     }
   }
 
@@ -1538,9 +1670,6 @@ class LogTest {
     } catch {
       case _: OffsetOutOfRangeException => // This is good.
     }
-
-    assertEquals("Reading from below the specified maxOffset should produce 0 byte read.", 0,
-      readLog(log, 1025, 1000, Some(1024)).records.sizeInBytes)
   }
 
   /**
@@ -1572,8 +1701,7 @@ class LogTest {
       assertEquals(s"Timestamps not equal at offset $offset", expected.timestamp, actual.timestamp)
       offset = head.lastOffset + 1
     }
-    val lastRead = readLog(log, startOffset = numMessages, maxLength = 1024*1024,
-      maxOffset = Some(numMessages + 1)).records
+    val lastRead = readLog(log, startOffset = numMessages, maxLength = 1024*1024).records
     assertEquals("Should be no more messages", 0, lastRead.records.asScala.size)
 
     // check that rolling the log forced a flushed, the flush is async so retry in case of failure
@@ -2886,7 +3014,7 @@ class LogTest {
       log.deleteOldSegments()
       assertTrue(log.logStartOffset <= hw)
       log.logSegments.foreach { segment =>
-        val segmentFetchInfo = segment.read(startOffset = segment.baseOffset, maxOffset = None, maxSize = Int.MaxValue)
+        val segmentFetchInfo = segment.read(startOffset = segment.baseOffset, maxSize = Int.MaxValue)
         val segmentLastOffsetOpt = segmentFetchInfo.records.records.asScala.lastOption.map(_.offset)
         segmentLastOffsetOpt.foreach { lastOffset =>
           assertTrue(lastOffset >= hw)
@@ -3099,7 +3227,7 @@ class LogTest {
 
     //Then leader epoch should be set on messages
     for (i <- records.indices) {
-      val read = readLog(log, i, 100, Some(i+1)).records.batches.iterator.next()
+      val read = readLog(log, i, 1).records.batches.iterator.next()
       assertEquals("Should have set leader epoch", 72, read.partitionLeaderEpoch)
     }
   }
@@ -3613,12 +3741,25 @@ class LogTest {
   }
 
   private def assertValidLogOffsetMetadata(log: Log, offsetMetadata: LogOffsetMetadata): Unit = {
-    val readInfo = log.read(startOffset = offsetMetadata.messageOffset,
-      maxLength = 2048,
-      maxOffset = None,
-      minOneMessage = false,
-      includeAbortedTxns = false)
-    assertEquals(offsetMetadata, readInfo.fetchOffsetMetadata)
+    assertFalse(offsetMetadata.messageOffsetOnly)
+
+    val segmentBaseOffset = offsetMetadata.segmentBaseOffset
+    val segmentOpt = log.logSegments(segmentBaseOffset, segmentBaseOffset + 1).headOption
+    assertTrue(segmentOpt.isDefined)
+
+    val segment = segmentOpt.get
+    assertEquals(segmentBaseOffset, segment.baseOffset)
+    assertTrue(offsetMetadata.relativePositionInSegment <= segment.size)
+
+    val readInfo = segment.read(offsetMetadata.messageOffset,
+      maxSize = 2048,
+      maxPosition = segment.size,
+      minOneMessage = false)
+
+    if (offsetMetadata.relativePositionInSegment < segment.size)
+      assertEquals(offsetMetadata, readInfo.fetchOffsetMetadata)
+    else
+      assertNull(readInfo)
   }
 
   @Test(expected = classOf[TransactionCoordinatorFencedException])
@@ -3863,7 +4004,10 @@ class LogTest {
     assertEquals(None, log.firstUnstableOffset)
 
     // now check that a fetch includes the aborted transaction
-    val fetchDataInfo = log.read(0L, 2048, maxOffset = None, minOneMessage = true, includeAbortedTxns = true)
+    val fetchDataInfo = log.read(0L,
+      maxLength = 2048,
+      isolation = FetchTxnCommitted,
+      minOneMessage = true)
     assertEquals(1, fetchDataInfo.abortedTransactions.size)
 
     assertTrue(fetchDataInfo.abortedTransactions.isDefined)
@@ -4003,10 +4147,12 @@ class LogTest {
       expectDeletedFiles)
   }
 
-  private def readLog(log: Log, startOffset: Long, maxLength: Int,
-                      maxOffset: Option[Long] = None,
+  private def readLog(log: Log,
+                      startOffset: Long,
+                      maxLength: Int,
+                      isolation: FetchIsolation = FetchLogEnd,
                       minOneMessage: Boolean = true): FetchDataInfo = {
-    log.read(startOffset, maxLength, maxOffset, minOneMessage, includeAbortedTxns = false)
+    log.read(startOffset, maxLength, isolation, minOneMessage)
   }
 
 }
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala
index 7637bfcba537d..e0dbf8cee4edd 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala
@@ -212,9 +212,8 @@ class ReplicaManagerQuotasTest {
     //if we ask for len 1 return a message
     expect(log.read(anyObject(),
       maxLength = geq(1),
-      maxOffset = anyObject(),
-      minOneMessage = anyBoolean(),
-      includeAbortedTxns = EasyMock.eq(false))).andReturn(
+      isolation = anyObject(),
+      minOneMessage = anyBoolean())).andReturn(
       FetchDataInfo(
         LogOffsetMetadata(0L, 0L, 0),
         MemoryRecords.withRecords(CompressionType.NONE, record)
@@ -223,9 +222,8 @@ class ReplicaManagerQuotasTest {
     //if we ask for len = 0, return 0 messages
     expect(log.read(anyObject(),
       maxLength = EasyMock.eq(0),
-      maxOffset = anyObject(),
-      minOneMessage = anyBoolean(),
-      includeAbortedTxns = EasyMock.eq(false))).andReturn(
+      isolation = anyObject(),
+      minOneMessage = anyBoolean())).andReturn(
       FetchDataInfo(
         LogOffsetMetadata(0L, 0L, 0),
         MemoryRecords.EMPTY
diff --git a/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala b/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala
index 1aec9d8b52728..5e9c482e60853 100644
--- a/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala
+++ b/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala
@@ -90,9 +90,8 @@ class SimpleFetchTest {
     EasyMock.expect(log.read(
       startOffset = 0,
       maxLength = fetchSize,
-      maxOffset = Some(partitionHW),
-      minOneMessage = true,
-      includeAbortedTxns = false))
+      isolation = FetchHighWatermark,
+      minOneMessage = true))
       .andReturn(FetchDataInfo(
         LogOffsetMetadata(0L, 0L, 0),
         MemoryRecords.withRecords(CompressionType.NONE, recordToHW)
@@ -100,9 +99,8 @@ class SimpleFetchTest {
     EasyMock.expect(log.read(
       startOffset = 0,
       maxLength = fetchSize,
-      maxOffset = None,
-      minOneMessage = true,
-      includeAbortedTxns = false))
+      isolation = FetchLogEnd,
+      minOneMessage = true))
       .andReturn(FetchDataInfo(
         LogOffsetMetadata(0L, 0L, 0),
         MemoryRecords.withRecords(CompressionType.NONE, recordToLEO)
diff --git a/core/src/test/scala/unit/kafka/server/epoch/EpochDrivenReplicationProtocolAcceptanceTest.scala b/core/src/test/scala/unit/kafka/server/epoch/EpochDrivenReplicationProtocolAcceptanceTest.scala
index 01ba69f13752c..84de813fde7dd 100644
--- a/core/src/test/scala/unit/kafka/server/epoch/EpochDrivenReplicationProtocolAcceptanceTest.scala
+++ b/core/src/test/scala/unit/kafka/server/epoch/EpochDrivenReplicationProtocolAcceptanceTest.scala
@@ -367,7 +367,7 @@ class EpochDrivenReplicationProtocolAcceptanceTest extends ZooKeeperTestHarness
     printSegments()
 
     def crcSeq(broker: KafkaServer, partition: Int = 0): Seq[Long] = {
-      val batches = getLog(broker, partition).activeSegment.read(0, None, Integer.MAX_VALUE)
+      val batches = getLog(broker, partition).activeSegment.read(0, Integer.MAX_VALUE)
         .records.batches().asScala.toSeq
       batches.map(_.checksum)
     }
@@ -438,7 +438,7 @@ class EpochDrivenReplicationProtocolAcceptanceTest extends ZooKeeperTestHarness
   private def epochCache(broker: KafkaServer): LeaderEpochFileCache = getLog(broker, 0).leaderEpochCache.get
 
   private def latestRecord(leader: KafkaServer, offset: Int = -1, partition: Int = 0): RecordBatch = {
-    getLog(leader, partition).activeSegment.read(0, None, Integer.MAX_VALUE)
+    getLog(leader, partition).activeSegment.read(0, Integer.MAX_VALUE)
       .records.batches().asScala.toSeq.last
   }
 
diff --git a/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala b/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala
index e6f91560863f4..bdd853c205031 100644
--- a/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala
@@ -245,10 +245,10 @@ class LeaderEpochIntegrationTest extends ZooKeeperTestHarness with Logging {
       val leo = broker.getLogManager().getLog(tp).get.logEndOffset
       result = result && leo > 0 && brokers.forall { broker =>
         broker.getLogManager().getLog(tp).get.logSegments.iterator.forall { segment =>
-          if (segment.read(minOffset, None, Integer.MAX_VALUE) == null) {
+          if (segment.read(minOffset, Integer.MAX_VALUE) == null) {
             false
           } else {
-            segment.read(minOffset, None, Integer.MAX_VALUE)
+            segment.read(minOffset, Integer.MAX_VALUE)
               .records.batches().iterator().asScala.forall(
               expectedLeaderEpoch == _.partitionLeaderEpoch()
             )

From 73ed9eac2c875a12d933af48b7a7817ea57ad447 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Tue, 30 Jul 2019 13:35:02 -0700
Subject: [PATCH 0488/1071] MINOR: Refactor abstractConfig#configuredInstance
 (#7129)

Reviewers: Jason Gustafson 
---
 .../kafka/common/config/AbstractConfig.java   | 50 ++++++++++---------
 1 file changed, 26 insertions(+), 24 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java b/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java
index a48856ffb53c6..832837da625b9 100644
--- a/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java
+++ b/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java
@@ -355,6 +355,29 @@ public void logUnused() {
             log.warn("The configuration '{}' was supplied but isn't a known config.", key);
     }
 
+    private  T getConfiguredInstance(Object klass, Class t, Map configPairs) {
+        if (klass == null)
+            return null;
+
+        Object o;
+        if (klass instanceof String) {
+            try {
+                o = Utils.newInstance((String) klass, t);
+            } catch (ClassNotFoundException e) {
+                throw new KafkaException("Class " + klass + " cannot be found", e);
+            }
+        } else if (klass instanceof Class) {
+            o = Utils.newInstance((Class) klass);
+        } else
+            throw new KafkaException("Unexpected element of type " + klass.getClass().getName() + ", expected String or Class");
+        if (!t.isInstance(o))
+            throw new KafkaException(klass + " is not an instance of " + t.getName());
+        if (o instanceof Configurable)
+            ((Configurable) o).configure(configPairs);
+
+        return t.cast(o);
+    }
+
     /**
      * Get a configured instance of the give class specified by the given configuration key. If the object implements
      * Configurable configure it using the configuration.
@@ -365,14 +388,8 @@ public void logUnused() {
      */
     public  T getConfiguredInstance(String key, Class t) {
         Class c = getClass(key);
-        if (c == null)
-            return null;
-        Object o = Utils.newInstance(c);
-        if (!t.isInstance(o))
-            throw new KafkaException(c.getName() + " is not an instance of " + t.getName());
-        if (o instanceof Configurable)
-            ((Configurable) o).configure(originals());
-        return t.cast(o);
+
+        return getConfiguredInstance(c, t, originals());
     }
 
     /**
@@ -400,7 +417,6 @@ public  List getConfiguredInstances(String key, Class t, Map List getConfiguredInstances(List classNames, Class t, M
         Map configPairs = originals();
         configPairs.putAll(configOverrides);
         for (Object klass : classNames) {
-            Object o;
-            if (klass instanceof String) {
-                try {
-                    o = Utils.newInstance((String) klass, t);
-                } catch (ClassNotFoundException e) {
-                    throw new KafkaException(klass + " ClassNotFoundException exception occurred", e);
-                }
-            } else if (klass instanceof Class) {
-                o = Utils.newInstance((Class) klass);
-            } else
-                throw new KafkaException("List contains element of type " + klass.getClass().getName() + ", expected String or Class");
-            if (!t.isInstance(o))
-                throw new KafkaException(klass + " is not an instance of " + t.getName());
-            if (o instanceof Configurable)
-                ((Configurable) o).configure(configPairs);
+            Object o = getConfiguredInstance(klass, t, configPairs);
             objects.add(t.cast(o));
         }
         return objects;

From de8ce78a90632b0d79e0a6ad6abb638982376b52 Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Wed, 31 Jul 2019 16:19:36 +0100
Subject: [PATCH 0489/1071] MINOR: Tolerate limited data loss for upgrade tests
 with old message format (#7102)

To avoid transient system test failures, tolerate a small amount of data loss due to truncation in upgrade system tests using older message format prior to KIP-101, where data loss was possible.

Reviewers: Ismael Juma 
---
 tests/kafkatest/tests/core/upgrade_test.py    |  6 ++++-
 .../tests/produce_consume_validate.py         |  6 ++++-
 tests/kafkatest/utils/util.py                 | 22 +++++++++++++++----
 3 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/tests/kafkatest/tests/core/upgrade_test.py b/tests/kafkatest/tests/core/upgrade_test.py
index c0fe61cfb1954..c390ff9792ae1 100644
--- a/tests/kafkatest/tests/core/upgrade_test.py
+++ b/tests/kafkatest/tests/core/upgrade_test.py
@@ -23,7 +23,7 @@
 from kafkatest.services.zookeeper import ZookeeperService
 from kafkatest.tests.produce_consume_validate import ProduceConsumeValidateTest
 from kafkatest.utils import is_int
-from kafkatest.version import LATEST_0_8_2, LATEST_0_9, LATEST_0_10, LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, LATEST_2_0, LATEST_2_1, LATEST_2_2, LATEST_2_3, V_0_9_0_0, DEV_BRANCH, KafkaVersion
+from kafkatest.version import LATEST_0_8_2, LATEST_0_9, LATEST_0_10, LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, LATEST_2_0, LATEST_2_1, LATEST_2_2, LATEST_2_3, V_0_9_0_0, V_0_11_0_0, DEV_BRANCH, KafkaVersion
 
 class TestUpgrade(ProduceConsumeValidateTest):
 
@@ -129,6 +129,10 @@ def test_upgrade(self, from_kafka_version, to_message_format_version, compressio
         if from_kafka_version <= LATEST_0_10_0:
             assert self.kafka.cluster_id() is None
 
+        # With older message formats before KIP-101, message loss may occur due to truncation
+        # after leader change. Tolerate limited data loss for this case to avoid transient test failures.
+        self.may_truncate_acked_records = False if from_kafka_version >= V_0_11_0_0 else True
+
         new_consumer = from_kafka_version >= V_0_9_0_0
         # TODO - reduce the timeout
         self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka,
diff --git a/tests/kafkatest/tests/produce_consume_validate.py b/tests/kafkatest/tests/produce_consume_validate.py
index 0d2b20cf1b0db..d915524e69576 100644
--- a/tests/kafkatest/tests/produce_consume_validate.py
+++ b/tests/kafkatest/tests/produce_consume_validate.py
@@ -47,6 +47,9 @@ def __init__(self, test_context):
         self.consumer_init_timeout_sec = 0
         self.enable_idempotence = False
 
+        # Allow tests to tolerate some data loss by overriding this for tests using older message formats
+        self.may_truncate_acked_records = False
+
     def start_producer_and_consumer(self):
         # Start background producer and consumer
         self.consumer.start()
@@ -125,7 +128,8 @@ def check_lost_data(missing_records):
             return self.kafka.search_data_files(self.topic, missing_records)
 
         succeeded, error_msg = validate_delivery(self.producer.acked, messages_consumed,
-                                                 self.enable_idempotence, check_lost_data)
+                                                 self.enable_idempotence, check_lost_data,
+                                                 self.may_truncate_acked_records)
 
         # Collect all logs if validation fails
         if not succeeded:
diff --git a/tests/kafkatest/utils/util.py b/tests/kafkatest/utils/util.py
index b9ccaf84f2fe6..60f60075c1263 100644
--- a/tests/kafkatest/utils/util.py
+++ b/tests/kafkatest/utils/util.py
@@ -14,6 +14,7 @@
 
 from kafkatest import __version__ as __kafkatest_version__
 
+import math
 import re
 import time
 
@@ -137,7 +138,7 @@ def annotate_data_lost(data_lost, msg, number_validated):
             "This suggests they were lost on their way to the consumer." % number_validated
     return msg
 
-def validate_delivery(acked, consumed, idempotence_enabled=False, check_lost_data=None):
+def validate_delivery(acked, consumed, idempotence_enabled=False, check_lost_data=None, may_truncate_acked_records=False):
     """Check that each acked message was consumed."""
     success = True
     msg = ""
@@ -149,13 +150,26 @@ def validate_delivery(acked, consumed, idempotence_enabled=False, check_lost_dat
     # Were all acked messages consumed?
     if len(missing) > 0:
         msg = annotate_missing_msgs(missing, acked, consumed, msg)
-        success = False
         
         # Did we miss anything due to data loss?
         if check_lost_data:
-            to_validate = list(missing)[0:1000 if len(missing) > 1000 else len(missing)]
+            max_truncate_count = 100 if may_truncate_acked_records else 0
+            max_validate_count = max(1000, max_truncate_count)
+
+            to_validate = list(missing)[0:min(len(missing), max_validate_count)]
             data_lost = check_lost_data(to_validate)
-            msg = annotate_data_lost(data_lost, msg, len(to_validate))
+
+            # With older versions of message format before KIP-101, data loss could occur due to truncation.
+            # These records won't be in the data logs. Tolerate limited data loss for this case.
+            if len(missing) < max_truncate_count and len(data_lost) == len(missing):
+               msg += "The %s missing messages were not present in Kafka's data files. This suggests data loss " \
+                   "due to truncation, which is possible with older message formats and hence are ignored " \
+                   "by this test. The messages lost: %s\n" % (len(data_lost), str(data_lost))
+            else:
+                msg = annotate_data_lost(data_lost, msg, len(to_validate))
+                success = False
+        else:
+            success = False
 
     # Are there duplicates?
     if len(set(consumed)) != len(consumed):

From 211745d0b3346b69451932358f45987dc790858f Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Wed, 31 Jul 2019 08:27:38 -0700
Subject: [PATCH 0490/1071] MINOR: Small cleanups in TopicCommand describe
 handling (#7136)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This patch contains a few small cleanups to make topic describe logic a little clearer. It also fixes a minor inconsistency in the output between the --zookeeper and --bootstrap-server behavior when the leader is unknown. Previously we printed -1 when --zookeeper was used, and now we print "none." The patch consolidates the output logic for describing topics and partitions in order to avoid inconsistencies like this in the future.

Reviewers: José Armando García Sancio , Guozhang Wang 
---
 .../main/scala/kafka/admin/TopicCommand.scala | 216 ++++++++++--------
 .../main/scala/kafka/api/LeaderAndIsr.scala   |   4 +
 .../unit/kafka/admin/TopicCommandTest.scala   |   4 +-
 3 files changed, 126 insertions(+), 98 deletions(-)

diff --git a/core/src/main/scala/kafka/admin/TopicCommand.scala b/core/src/main/scala/kafka/admin/TopicCommand.scala
index a25e546b713b7..9233b228bfe53 100755
--- a/core/src/main/scala/kafka/admin/TopicCommand.scala
+++ b/core/src/main/scala/kafka/admin/TopicCommand.scala
@@ -28,8 +28,8 @@ import kafka.utils.Implicits._
 import kafka.utils._
 import kafka.zk.{AdminZkClient, KafkaZkClient}
 import org.apache.kafka.clients.CommonClientConfigs
-import org.apache.kafka.clients.admin.{Admin, ListTopicsOptions, NewPartitions, NewTopic, AdminClient => JAdminClient}
-import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.clients.admin.{Admin, ConfigEntry, ListTopicsOptions, NewPartitions, NewTopic, AdminClient => JAdminClient, Config => JConfig}
+import org.apache.kafka.common.{Node, TopicPartition, TopicPartitionInfo}
 import org.apache.kafka.common.config.ConfigResource.Type
 import org.apache.kafka.common.config.{ConfigResource, TopicConfig}
 import org.apache.kafka.common.errors.{InvalidTopicException, TopicExistsException}
@@ -78,8 +78,6 @@ object TopicCommand extends Logging {
     }
   }
 
-
-
   class CommandTopicPartition(opts: TopicCommandOptions) {
     val name: String = opts.topic.get
     val partitions: Option[Integer] = opts.partitions
@@ -94,54 +92,97 @@ object TopicCommand extends Logging {
     def ifTopicDoesntExist(): Boolean = opts.ifNotExists
   }
 
-  case class PartitionDescription(
-                                   topic: String,
-                                   partition: Int,
-                                   leader: Option[Int],
-                                   assignedReplicas: Seq[Int],
-                                   isr: Seq[Int],
-                                   minIsrCount: Int,
-                                   markedForDeletion: Boolean,
-                                   describeConfigs: Boolean)
+  case class TopicDescription(topic: String,
+                              numPartitions: Int,
+                              replicationFactor: Int,
+                              config: JConfig,
+                              markedForDeletion: Boolean) {
+
+    def printDescription(): Unit = {
+      val configsAsString = config.entries.asScala.filter(!_.isDefault).map { ce => s"${ce.name}=${ce.value}" }.mkString(",")
+      print(s"Topic: $topic")
+      print(s"\tPartitionCount: $numPartitions")
+      print(s"\tReplicationFactor: $replicationFactor")
+      print(s"\tConfigs: $configsAsString")
+      print(if (markedForDeletion) "\tMarkedForDeletion: true" else "")
+      println()
+    }
+  }
 
-  class DescribeOptions(opts: TopicCommandOptions, liveBrokers: Set[Int]) {
-    val describeConfigs: Boolean = !opts.reportUnavailablePartitions && !opts.reportUnderReplicatedPartitions && !opts.reportUnderMinIsrPartitions && !opts.reportAtMinIsrPartitions
-    val describePartitions: Boolean = !opts.reportOverriddenConfigs
-    private def hasUnderReplicatedPartitions(partitionDescription: PartitionDescription) = {
-      partitionDescription.isr.size < partitionDescription.assignedReplicas.size
+  case class PartitionDescription(topic: String,
+                                  info: TopicPartitionInfo,
+                                  config: Option[JConfig],
+                                  markedForDeletion: Boolean) {
+
+    private def minIsrCount: Option[Int] = {
+      config.map(_.get(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG).value.toInt)
     }
-    private def shouldPrintUnderReplicatedPartitions(partitionDescription: PartitionDescription) = {
-      opts.reportUnderReplicatedPartitions && hasUnderReplicatedPartitions(partitionDescription)
+
+    def hasUnderReplicatedPartitions: Boolean = {
+      info.isr.size < info.replicas.size
     }
-    private def hasUnavailablePartitions(partitionDescription: PartitionDescription) = {
-      partitionDescription.leader.isEmpty || !liveBrokers.contains(partitionDescription.leader.get)
+
+    private def hasLeader: Boolean = {
+      info.leader != null
     }
-    private def shouldPrintUnavailablePartitions(partitionDescription: PartitionDescription) = {
-      opts.reportUnavailablePartitions && hasUnavailablePartitions(partitionDescription)
+
+    def hasUnderMinIsrPartitions: Boolean = {
+      !hasLeader || minIsrCount.exists(info.isr.size < _)
     }
-    private def hasUnderMinIsrPartitions(partitionDescription: PartitionDescription) = {
-      if (partitionDescription.leader.isDefined)
-        partitionDescription.isr.size < partitionDescription.minIsrCount
-      else
-        partitionDescription.minIsrCount > 0
+
+    def isAtMinIsrPartitions: Boolean =  {
+      minIsrCount.contains(info.isr.size)
+    }
+
+    def hasUnavailablePartitions(liveBrokers: Set[Int]): Boolean = {
+      !hasLeader || !liveBrokers.contains(info.leader.id)
+    }
+
+    def printDescription(): Unit = {
+      print("\tTopic: " + topic)
+      print("\tPartition: " + info.partition)
+      print("\tLeader: " + (if (hasLeader) info.leader.id else "none"))
+      print("\tReplicas: " + info.replicas.asScala.map(_.id).mkString(","))
+      print("\tIsr: " + info.isr.asScala.map(_.id).mkString(","))
+      print(if (markedForDeletion) "\tMarkedForDeletion: true" else "")
+      println()
+    }
+
+  }
+
+  class DescribeOptions(opts: TopicCommandOptions, liveBrokers: Set[Int]) {
+    val describeConfigs: Boolean =
+      !opts.reportUnavailablePartitions &&
+      !opts.reportUnderReplicatedPartitions &&
+      !opts.reportUnderMinIsrPartitions &&
+      !opts.reportAtMinIsrPartitions
+    val describePartitions: Boolean = !opts.reportOverriddenConfigs
+
+    private def shouldPrintUnderReplicatedPartitions(partitionDescription: PartitionDescription): Boolean = {
+      opts.reportUnderReplicatedPartitions && partitionDescription.hasUnderReplicatedPartitions
     }
-    private def hasAtMinIsrPartitions(partitionDescription: PartitionDescription) = {
-      partitionDescription.isr.size == partitionDescription.minIsrCount
+    private def shouldPrintUnavailablePartitions(partitionDescription: PartitionDescription): Boolean = {
+      opts.reportUnavailablePartitions && partitionDescription.hasUnavailablePartitions(liveBrokers)
     }
-    private def shouldPrintUnderMinIsrPartitions(partitionDescription: PartitionDescription) = {
-      opts.reportUnderMinIsrPartitions && hasUnderMinIsrPartitions(partitionDescription)
+    private def shouldPrintUnderMinIsrPartitions(partitionDescription: PartitionDescription): Boolean = {
+      opts.reportUnderMinIsrPartitions && partitionDescription.hasUnderMinIsrPartitions
     }
-    private def shouldPrintAtMinIsrPartitions(partitionDescription: PartitionDescription) = {
-      opts.reportAtMinIsrPartitions && hasAtMinIsrPartitions(partitionDescription)
+    private def shouldPrintAtMinIsrPartitions(partitionDescription: PartitionDescription): Boolean = {
+      opts.reportAtMinIsrPartitions && partitionDescription.isAtMinIsrPartitions
     }
 
-    def shouldPrintTopicPartition(partitionDesc: PartitionDescription): Boolean = {
+    private def shouldPrintTopicPartition(partitionDesc: PartitionDescription): Boolean = {
       describeConfigs ||
         shouldPrintUnderReplicatedPartitions(partitionDesc) ||
         shouldPrintUnavailablePartitions(partitionDesc) ||
         shouldPrintUnderMinIsrPartitions(partitionDesc) ||
         shouldPrintAtMinIsrPartitions(partitionDesc)
     }
+
+    def maybePrintPartitionDescription(desc: PartitionDescription): Unit = {
+      if (shouldPrintTopicPartition(desc))
+        desc.printDescription()
+    }
   }
 
   trait TopicService extends AutoCloseable {
@@ -234,35 +275,24 @@ object TopicCommand extends Logging {
       val describeOptions = new DescribeOptions(opts, liveBrokers.toSet)
 
       for (td <- topicDescriptions) {
-        val sortedPartitions = td.partitions().asScala.sortBy(_.partition())
+        val topicName = td.name
+        val config = allConfigs.get(new ConfigResource(Type.TOPIC, topicName)).get()
+        val sortedPartitions = td.partitions.asScala.sortBy(_.partition)
+
         if (describeOptions.describeConfigs) {
-          val config = allConfigs.get(new ConfigResource(Type.TOPIC, td.name())).get()
           val hasNonDefault = config.entries().asScala.exists(!_.isDefault)
           if (!opts.reportOverriddenConfigs || hasNonDefault) {
             val numPartitions = td.partitions().size
-            val replicationFactor = td.partitions().iterator().next().replicas().size
-            val configsAsString = config.entries().asScala.filter(!_.isDefault).map { ce => s"${ce.name()}=${ce.value()}" }.mkString(",")
-            println(s"Topic:${td.name()}\tPartitionCount:$numPartitions\tReplicationFactor:$replicationFactor\tConfigs:$configsAsString")
+            val replicationFactor = td.partitions.iterator.next().replicas.size
+            val topicDesc = TopicDescription(topicName, numPartitions, replicationFactor, config, markedForDeletion = false)
+            topicDesc.printDescription()
           }
         }
+
         if (describeOptions.describePartitions) {
-          val computedMinIsrCount = if (opts.reportUnderMinIsrPartitions || opts.reportAtMinIsrPartitions)
-            allConfigs.get(new ConfigResource(ConfigResource.Type.TOPIC, td.name())).get().get(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG).value().toInt else 0
           for (partition <- sortedPartitions) {
-            val partitionDesc = PartitionDescription(
-              topic = td.name(),
-              partition.partition(),
-              leader = Option(partition.leader()).map(_.id()),
-              assignedReplicas = partition.replicas().asScala.map(_.id()),
-              isr = partition.isr().asScala.map(_.id()),
-              minIsrCount = computedMinIsrCount,
-              markedForDeletion = false,
-              describeOptions.describeConfigs
-            )
-
-            if (describeOptions.shouldPrintTopicPartition(partitionDesc)) {
-              printPartition(partitionDesc)
-            }
+            val partitionDesc = PartitionDescription(topicName, partition, Some(config), markedForDeletion = false)
+            describeOptions.maybePrintPartitionDescription(partitionDesc)
           }
         }
       }
@@ -358,8 +388,9 @@ object TopicCommand extends Logging {
     override def describeTopic(opts: TopicCommandOptions): Unit = {
       val topics = getTopics(opts.topic, opts.excludeInternalTopics)
       ensureTopicExists(topics, opts.topic, !opts.ifExists)
-      val liveBrokers = zkClient.getAllBrokersInCluster.map(_.id).toSet
-      val describeOptions = new DescribeOptions(opts, liveBrokers)
+      val liveBrokers = zkClient.getAllBrokersInCluster.map(broker => broker.id -> broker).toMap
+      val liveBrokerIds = liveBrokers.keySet
+      val describeOptions = new DescribeOptions(opts, liveBrokerIds)
       val adminZkClient = new AdminZkClient(zkClient)
 
       for (topic <- topics) {
@@ -371,28 +402,32 @@ object TopicCommand extends Logging {
               if (!opts.reportOverriddenConfigs || configs.nonEmpty) {
                 val numPartitions = topicPartitionAssignment.size
                 val replicationFactor = topicPartitionAssignment.head._2.size
-                val configsAsString = configs.map { case (k, v) => s"$k=$v" }.mkString(",")
-                val markedForDeletionString = if (markedForDeletion) "\tMarkedForDeletion:true" else ""
-                println(s"Topic:$topic\tPartitionCount:$numPartitions\tReplicationFactor:$replicationFactor\tConfigs:$configsAsString$markedForDeletionString")
+                val config = new JConfig(configs.map{ case (k, v) => new ConfigEntry(k, v) }.asJavaCollection)
+                val topicDesc = TopicDescription(topic, numPartitions, replicationFactor, config, markedForDeletion)
+                topicDesc.printDescription()
               }
             }
             if (describeOptions.describePartitions) {
               for ((partitionId, assignedReplicas) <- topicPartitionAssignment.toSeq.sortBy(_._1)) {
-                val leaderIsrEpoch = zkClient.getTopicPartitionState(new TopicPartition(topic, partitionId))
-                val partitionDesc = PartitionDescription(
-                  topic,
-                  partitionId,
-                  leader = if (leaderIsrEpoch.isEmpty) None else Option(leaderIsrEpoch.get.leaderAndIsr.leader),
-                  assignedReplicas,
-                  isr = if (leaderIsrEpoch.isEmpty) Seq.empty[Int] else leaderIsrEpoch.get.leaderAndIsr.isr,
-                  minIsrCount = 0,
-                  markedForDeletion,
-                  describeOptions.describeConfigs
-                )
-
-                if (describeOptions.shouldPrintTopicPartition(partitionDesc)) {
-                  printPartition(partitionDesc)
+                val tp = new TopicPartition(topic, partitionId)
+                val (leaderOpt, isr) =  zkClient.getTopicPartitionState(tp).map(_.leaderAndIsr) match {
+                  case Some(leaderAndIsr) => (leaderAndIsr.leaderOpt, leaderAndIsr.isr)
+                  case None => (None, Seq.empty[Int])
+                }
+
+                def asNode(brokerId: Int): Node = {
+                  liveBrokers.get(brokerId) match {
+                    case Some(broker) => broker.node(broker.endPoints.head.listenerName)
+                    case None => new Node(brokerId, "", -1)
+                  }
                 }
+
+                val info = new TopicPartitionInfo(partitionId, leaderOpt.map(asNode).orNull,
+                  assignedReplicas.map(asNode).toList.asJava,
+                  isr.map(asNode).toList.asJava)
+
+                val partitionDesc = PartitionDescription(topic, info, config = None, markedForDeletion)
+                describeOptions.maybePrintPartitionDescription(partitionDesc)
               }
             }
           case None =>
@@ -449,18 +484,6 @@ object TopicCommand extends Logging {
     }
   }
 
-  private def printPartition(tp: PartitionDescription): Unit = {
-    val markedForDeletionString =
-    if (tp.markedForDeletion && !tp.describeConfigs) "\tMarkedForDeletion: true" else ""
-    print("\tTopic: " + tp.topic)
-    print("\tPartition: " + tp.partition)
-    print("\tLeader: " + (if(tp.leader.isDefined) tp.leader.get else "none"))
-    print("\tReplicas: " + tp.assignedReplicas.mkString(","))
-    print("\tIsr: " + tp.isr.mkString(","))
-    print(markedForDeletionString)
-    println()
-  }
-
   private def doGetTopics(allTopics: Seq[String], topicWhitelist: Option[String], excludeInternalTopics: Boolean): Seq[String] = {
     if (topicWhitelist.isDefined) {
       val topicsFilter = Whitelist(topicWhitelist.get)
@@ -565,26 +588,27 @@ object TopicCommand extends Logging {
                                         "broker_id_for_part2_replica1 : broker_id_for_part2_replica2 , ...")
                            .ofType(classOf[String])
     private val reportUnderReplicatedPartitionsOpt = parser.accepts("under-replicated-partitions",
-                                                            "if set when describing topics, only show under replicated partitions")
+      "if set when describing topics, only show under replicated partitions")
     private val reportUnavailablePartitionsOpt = parser.accepts("unavailable-partitions",
-                                                            "if set when describing topics, only show partitions whose leader is not available")
+      "if set when describing topics, only show partitions whose leader is not available")
     private val reportUnderMinIsrPartitionsOpt = parser.accepts("under-min-isr-partitions",
-                                                            "if set when describing topics, only show partitions whose isr count is less than the configured minimum. Not supported with the --zookeeper option.")
+      "if set when describing topics, only show partitions whose isr count is less than the configured minimum. Not supported with the --zookeeper option.")
     private val reportAtMinIsrPartitionsOpt = parser.accepts("at-min-isr-partitions",
-                                                            "if set when describing topics, only show partitions whose isr count is equal to the configured minimum. Not supported with the --zookeeper option.")
+      "if set when describing topics, only show partitions whose isr count is equal to the configured minimum. Not supported with the --zookeeper option.")
     private val topicsWithOverridesOpt = parser.accepts("topics-with-overrides",
-                                                "if set when describing topics, only show topics that have overridden configs")
+      "if set when describing topics, only show topics that have overridden configs")
     private val ifExistsOpt = parser.accepts("if-exists",
-                                     "if set when altering or deleting or describing topics, the action will only execute if the topic exists. Not supported with the --bootstrap-server option.")
+      "if set when altering or deleting or describing topics, the action will only execute if the topic exists. Not supported with the --bootstrap-server option.")
     private val ifNotExistsOpt = parser.accepts("if-not-exists",
-                                        "if set when creating topics, the action will only execute if the topic does not already exist. Not supported with the --bootstrap-server option.")
+      "if set when creating topics, the action will only execute if the topic does not already exist. Not supported with the --bootstrap-server option.")
 
     private val disableRackAware = parser.accepts("disable-rack-aware", "Disable rack aware replica assignment")
 
     // This is not currently used, but we keep it for compatibility
     parser.accepts("force", "Suppress console prompts")
 
-    private val excludeInternalTopicOpt = parser.accepts("exclude-internal", "exclude internal topics when running list or describe command. The internal topics will be listed by default")
+    private val excludeInternalTopicOpt = parser.accepts("exclude-internal",
+      "exclude internal topics when running list or describe command. The internal topics will be listed by default")
 
     options = parser.parse(args : _*)
 
diff --git a/core/src/main/scala/kafka/api/LeaderAndIsr.scala b/core/src/main/scala/kafka/api/LeaderAndIsr.scala
index cb59575ea7a4b..ebfa710d3de00 100644
--- a/core/src/main/scala/kafka/api/LeaderAndIsr.scala
+++ b/core/src/main/scala/kafka/api/LeaderAndIsr.scala
@@ -40,6 +40,10 @@ case class LeaderAndIsr(leader: Int,
 
   def newEpochAndZkVersion = newLeaderAndIsr(leader, isr)
 
+  def leaderOpt: Option[Int] = {
+    if (leader == LeaderAndIsr.NoLeader) None else Some(leader)
+  }
+
   override def toString: String = {
     s"LeaderAndIsr(leader=$leader, leaderEpoch=$leaderEpoch, isr=$isr, zkVersion=$zkVersion)"
   }
diff --git a/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala b/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala
index 35502f54d0706..668da77a81376 100644
--- a/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala
@@ -290,14 +290,14 @@ class TopicCommandTest extends ZooKeeperTestHarness with Logging with RackAwareT
 
     val output = TestUtils.grabConsoleOutput(
       topicService.describeTopic(new TopicCommandOptions(Array("--topic", testTopicName))))
-    assertTrue("The output should contain the modified config", output.contains("Configs:cleanup.policy=compact"))
+    assertTrue("The output should contain the modified config", output.contains("Configs: cleanup.policy=compact"))
 
     topicService.alterTopic(new TopicCommandOptions(
       Array("--topic", testTopicName, "--config", "cleanup.policy=delete")))
 
     val output2 = TestUtils.grabConsoleOutput(
       topicService.describeTopic(new TopicCommandOptions(Array("--topic", testTopicName))))
-    assertTrue("The output should contain the modified config", output2.contains("Configs:cleanup.policy=delete"))
+    assertTrue("The output should contain the modified config", output2.contains("Configs: cleanup.policy=delete"))
   }
 
   @Test

From cdea3480c693d270064ebcb3871d08dee053ce7f Mon Sep 17 00:00:00 2001
From: Gemma Singleton <49679170+gemma-singleton@users.noreply.github.com>
Date: Wed, 31 Jul 2019 18:06:55 +0200
Subject: [PATCH 0491/1071] Update KafkaConfig.scala (#7113)

Better clarify the auto leader rebalance config documentation to reflect additional leader.imbalance.per.broker.percentage config description.

Reviewers: Stanislav Kozlovski , Jun Rao 
---
 core/src/main/scala/kafka/server/KafkaConfig.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index 5223ebd058e08..a2db65379796a 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -688,7 +688,7 @@ object KafkaConfig {
   val FetchPurgatoryPurgeIntervalRequestsDoc = "The purge interval (in number of requests) of the fetch request purgatory"
   val ProducerPurgatoryPurgeIntervalRequestsDoc = "The purge interval (in number of requests) of the producer request purgatory"
   val DeleteRecordsPurgatoryPurgeIntervalRequestsDoc = "The purge interval (in number of requests) of the delete records request purgatory"
-  val AutoLeaderRebalanceEnableDoc = "Enables auto leader balancing. A background thread checks and triggers leader balance if required at regular intervals"
+  val AutoLeaderRebalanceEnableDoc = "Enables auto leader balancing. A background thread checks the distribution of partition leaders at regular intervals, configurable by `leader.imbalance.check.interval.seconds`. If the leader imbalance exceeds `leader.imbalance.per.broker.percentage`, leader rebalance to the preferred leader for partitions is triggered."
   val LeaderImbalancePerBrokerPercentageDoc = "The ratio of leader imbalance allowed per broker. The controller would trigger a leader balance if it goes above this value per broker. The value is specified in percentage."
   val LeaderImbalanceCheckIntervalSecondsDoc = "The frequency with which the partition rebalance check is triggered by the controller"
   val UncleanLeaderElectionEnableDoc = "Indicates whether to enable replicas not in the ISR set to be elected as leader as a last resort, even though doing so may result in data loss"

From 6fbac3cfa88bd3eba36b2d3e254d7ce5e11a550b Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Wed, 31 Jul 2019 13:53:38 -0700
Subject: [PATCH 0492/1071]  KAFKA-8179: PartitionAssignorAdapter (#7110)

Follow up to new PartitionAssignor interface merged in 7108 is merged

Adds a PartitionAssignorAdapter class to maintain backwards compatibility

Reviewers: Boyang Chen , Jason Gustafson , Guozhang Wang 
---
 .../kafka/clients/admin/KafkaAdminClient.java |   4 +-
 .../clients/consumer/ConsumerConfig.java      |   2 +-
 .../consumer/ConsumerPartitionAssignor.java   |  10 +-
 .../kafka/clients/consumer/KafkaConsumer.java |   7 +-
 .../internals/AbstractPartitionAssignor.java  |   4 +-
 .../internals/ConsumerCoordinator.java        |   3 +-
 .../consumer/internals/PartitionAssignor.java |   3 +
 .../internals/PartitionAssignorAdapter.java   | 136 ++++++++++++++
 .../PartitionAssignorAdapterTest.java         | 173 ++++++++++++++++++
 docs/upgrade.html                             |   2 +
 .../internals/StreamsPartitionAssignor.java   |   4 +-
 11 files changed, 333 insertions(+), 15 deletions(-)
 create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignorAdapter.java
 create mode 100644 clients/src/test/java/org/apache/kafka/clients/consumer/internals/PartitionAssignorAdapterTest.java

diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
index f2ee21e9da617..227a03b966c7a 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
@@ -29,7 +29,7 @@
 import org.apache.kafka.clients.admin.DeleteAclsResult.FilterResults;
 import org.apache.kafka.clients.admin.DescribeReplicaLogDirsResult.ReplicaLogDirInfo;
 import org.apache.kafka.clients.admin.internals.AdminMetadataManager;
-import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment;
 import org.apache.kafka.clients.consumer.OffsetAndMetadata;
 import org.apache.kafka.clients.consumer.internals.ConsumerProtocol;
 import org.apache.kafka.common.Cluster;
@@ -2724,7 +2724,7 @@ void handleResponse(AbstractResponse abstractResponse) {
                     for (DescribedGroupMember groupMember : members) {
                         Set partitions = Collections.emptySet();
                         if (groupMember.memberAssignment().length > 0) {
-                            final ConsumerPartitionAssignor.Assignment assignment = ConsumerProtocol.
+                            final Assignment assignment = ConsumerProtocol.
                                 deserializeAssignment(ByteBuffer.wrap(groupMember.memberAssignment()));
                             partitions = new HashSet<>(assignment.partitions());
                         }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
index 8a18bd5a77af3..2e4507af1d4d4 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
@@ -102,7 +102,7 @@ public class ConsumerConfig extends AbstractConfig {
      * partition.assignment.strategy
      */
     public static final String PARTITION_ASSIGNMENT_STRATEGY_CONFIG = "partition.assignment.strategy";
-    private static final String PARTITION_ASSIGNMENT_STRATEGY_DOC = "The class name or class type of the assignor implementing the partition assignment strategy that the client will use to distribute partition ownership amongst consumer instances when group management is used. A custom assignor that implements ConsumerPartitionAssignor can be plugged in";
+    private static final String PARTITION_ASSIGNMENT_STRATEGY_DOC = "A list of class names or class types, ordered by preference, of supported assignors responsible for the partition assignment strategy that the client will use to distribute partition ownership amongst consumer instances when group management is used. Implementing the org.apache.kafka.clients.consumer.ConsumerPartitionAssignor interface allows you to plug in a custom assignment strategy.";
 
     /**
      * auto.offset.reset
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java
index 72d5d6e806bd7..07e153e33f35e 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java
@@ -44,7 +44,9 @@ public interface ConsumerPartitionAssignor {
      * Return serialized data that will be included in the {@link Subscription} sent to the leader
      * and can be leveraged in {@link #assign(Cluster, GroupSubscription)} ((e.g. local host/rack information)
      *
-     * @return optional join subscription user data
+     * @param topics Topics subscribed to through {@link org.apache.kafka.clients.consumer.KafkaConsumer#subscribe(java.util.Collection)}
+     *               and variants
+     * @return nullable subscription user data
      */
     default ByteBuffer subscriptionUserData(Set topics) {
         return null;
@@ -53,11 +55,11 @@ default ByteBuffer subscriptionUserData(Set topics) {
     /**
      * Perform the group assignment given the member subscriptions and current cluster metadata.
      * @param metadata Current topic/broker metadata known by consumer
-     * @param subscriptions Subscriptions from all members including metadata provided through {@link #subscriptionUserData(Set)}
-     * @return A map from the members to their respective assignment. This should have one entry
+     * @param groupSubscription Subscriptions from all members including metadata provided through {@link #subscriptionUserData(Set)}
+     * @return A map from the members to their respective assignments. This should have one entry
      *         for each member in the input subscription map.
      */
-    GroupAssignment assign(Cluster metadata, GroupSubscription subscriptions);
+    GroupAssignment assign(Cluster metadata, GroupSubscription groupSubscription);
 
     /**
      * Callback which is invoked when a group member receives its assignment from the leader.
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index 30944b330da60..b33b1f20796e8 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -16,6 +16,8 @@
  */
 package org.apache.kafka.clients.consumer;
 
+import static org.apache.kafka.clients.consumer.internals.PartitionAssignorAdapter.getAssignorInstances;
+
 import org.apache.kafka.clients.ApiVersions;
 import org.apache.kafka.clients.ClientDnsLookup;
 import org.apache.kafka.clients.ClientUtils;
@@ -765,9 +767,8 @@ else if (enableAutoCommit)
                     retryBackoffMs,
                     config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG),
                     heartbeatIntervalMs); //Will avoid blocking an extended period of time to prevent heartbeat thread starvation
-            this.assignors = config.getConfiguredInstances(
-                    ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG,
-                    ConsumerPartitionAssignor.class);
+
+            this.assignors = getAssignorInstances(config.getList(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG), config.originals());
 
             // no coordinator will be constructed for the default (null) group id
             this.coordinator = groupId == null ? null :
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractPartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractPartitionAssignor.java
index 3b966b0736bbe..ed0282b406299 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractPartitionAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractPartitionAssignor.java
@@ -48,8 +48,8 @@ public abstract Map> assign(Map pa
                                                              Map subscriptions);
 
     @Override
-    public GroupAssignment assign(Cluster metadata, GroupSubscription groupSubscriptions) {
-        Map subscriptions = groupSubscriptions.groupSubscription();
+    public GroupAssignment assign(Cluster metadata, GroupSubscription groupSubscription) {
+        Map subscriptions = groupSubscription.groupSubscription();
         Set allSubscribedTopics = new HashSet<>();
         for (Map.Entry subscriptionEntry : subscriptions.entrySet())
             allSubscribedTopics.addAll(subscriptionEntry.getValue().topics());
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index a28119dd4b0f4..aae16b92dc3c2 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -204,8 +204,9 @@ protected JoinGroupRequestData.JoinGroupRequestProtocolCollection metadata() {
         this.joinedSubscription = subscriptions.subscription();
         JoinGroupRequestData.JoinGroupRequestProtocolCollection protocolSet = new JoinGroupRequestData.JoinGroupRequestProtocolCollection();
 
+        List topics = new ArrayList<>(joinedSubscription);
         for (ConsumerPartitionAssignor assignor : assignors) {
-            Subscription subscription = new Subscription(new ArrayList<>(joinedSubscription),
+            Subscription subscription = new Subscription(topics,
                                                          assignor.subscriptionUserData(joinedSubscription),
                                                          subscriptions.assignedPartitionsList());
             ByteBuffer metadata = ConsumerProtocol.serializeSubscription(subscription);
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
index b3f2ada5c4193..1ecb15c8d4f25 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
@@ -36,6 +36,9 @@
  * assignment decisions. For this, you can override {@link #subscription(Set)} and provide custom
  * userData in the returned Subscription. For example, to have a rack-aware assignor, an implementation
  * can use this user data to forward the rackId belonging to each member.
+ *
+ * This interface has been deprecated in 2.4, custom assignors should now implement
+ * {@link org.apache.kafka.clients.consumer.ConsumerPartitionAssignor}
  */
 @Deprecated
 public interface PartitionAssignor {
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignorAdapter.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignorAdapter.java
new file mode 100644
index 0000000000000..d1d5e20f32209
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignorAdapter.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.consumer.internals;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.kafka.clients.consumer.ConsumerGroupMetadata;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor;
+import org.apache.kafka.common.Cluster;
+import org.apache.kafka.common.Configurable;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This adapter class is used to ensure backwards compatibility for those who have implemented the {@link PartitionAssignor}
+ * interface, which has been deprecated in favor of the new {@link org.apache.kafka.clients.consumer.ConsumerPartitionAssignor}
+ */
+@SuppressWarnings("deprecation")
+public class PartitionAssignorAdapter implements ConsumerPartitionAssignor {
+
+    private static final Logger LOG = LoggerFactory.getLogger(PartitionAssignorAdapter.class);
+    private final PartitionAssignor oldAssignor;
+
+    PartitionAssignorAdapter(PartitionAssignor oldAssignor) {
+        this.oldAssignor = oldAssignor;
+    }
+
+    @Override
+    public ByteBuffer subscriptionUserData(Set topics) {
+        return oldAssignor.subscription(topics).userData();
+    }
+
+    @Override
+    public GroupAssignment assign(Cluster metadata, GroupSubscription groupSubscription) {
+        return toNewGroupAssignment(oldAssignor.assign(metadata, toOldGroupSubscription(groupSubscription)));
+    }
+
+    @Override
+    public void onAssignment(Assignment assignment, ConsumerGroupMetadata metadata) {
+        oldAssignor.onAssignment(toOldAssignment(assignment), metadata.generationId());
+    }
+
+    @Override
+    public String name() {
+        return oldAssignor.name();
+    }
+
+    private static PartitionAssignor.Assignment toOldAssignment(Assignment newAssignment) {
+        return new PartitionAssignor.Assignment(newAssignment.partitions(), newAssignment.userData());
+    }
+
+    private static Map toOldGroupSubscription(GroupSubscription newSubscriptions) {
+        Map oldSubscriptions = new HashMap<>();
+        for (Map.Entry entry : newSubscriptions.groupSubscription().entrySet()) {
+            String member = entry.getKey();
+            Subscription newSubscription = entry.getValue();
+            oldSubscriptions.put(member, new PartitionAssignor.Subscription(
+                newSubscription.topics(), newSubscription.userData()));
+        }
+        return oldSubscriptions;
+    }
+
+    private static GroupAssignment toNewGroupAssignment(Map oldAssignments) {
+        Map newAssignments = new HashMap<>();
+        for (Map.Entry entry : oldAssignments.entrySet()) {
+            String member = entry.getKey();
+            PartitionAssignor.Assignment oldAssignment = entry.getValue();
+            newAssignments.put(member, new Assignment(oldAssignment.partitions(), oldAssignment.userData()));
+        }
+        return new GroupAssignment(newAssignments);
+    }
+
+    /**
+     * Get a list of configured instances of {@link org.apache.kafka.clients.consumer.ConsumerPartitionAssignor}
+     * based on the class names/types specified by {@link org.apache.kafka.clients.consumer.ConsumerConfig#PARTITION_ASSIGNMENT_STRATEGY_CONFIG}
+     * where any instances of the old {@link PartitionAssignor} interface are wrapped in an adapter to the new
+     * {@link org.apache.kafka.clients.consumer.ConsumerPartitionAssignor} interface
+     */
+    public static List getAssignorInstances(List assignorClasses, Map configs) {
+        List assignors = new ArrayList<>();
+
+        if (assignorClasses == null)
+            return assignors;
+
+        for (Object klass : assignorClasses) {
+            // first try to get the class if passed in as a string
+            if (klass instanceof String) {
+                try {
+                    klass = Class.forName((String) klass, true, Utils.getContextOrKafkaClassLoader());
+                } catch (ClassNotFoundException classNotFound) {
+                    throw new KafkaException(klass + " ClassNotFoundException exception occurred", classNotFound);
+                }
+            }
+
+            if (klass instanceof Class) {
+                Object assignor = Utils.newInstance((Class) klass);
+                if (assignor instanceof Configurable)
+                    ((Configurable) assignor).configure(configs);
+
+                if (assignor instanceof ConsumerPartitionAssignor) {
+                    assignors.add((ConsumerPartitionAssignor) assignor);
+                } else if (assignor instanceof PartitionAssignor) {
+                    assignors.add(new PartitionAssignorAdapter((PartitionAssignor) assignor));
+                    LOG.warn("The PartitionAssignor interface has been deprecated, "
+                        + "please implement the ConsumerPartitionAssignor interface instead.");
+                } else {
+                    throw new KafkaException(klass + " is not an instance of " + PartitionAssignor.class.getName()
+                        + " or an instance of " + ConsumerPartitionAssignor.class.getName());
+                }
+            } else {
+                throw new KafkaException("List contains element of type " + klass.getClass().getName() + ", expected String or Class");
+            }
+        }
+        return assignors;
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PartitionAssignorAdapterTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PartitionAssignorAdapterTest.java
new file mode 100644
index 0000000000000..52950cba4cc6c
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PartitionAssignorAdapterTest.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.consumer.internals;
+
+import static org.apache.kafka.clients.consumer.internals.PartitionAssignorAdapter.getAssignorInstances;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.Set;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.ConsumerGroupMetadata;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.GroupSubscription;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.consumer.StickyAssignor;
+import org.apache.kafka.common.Cluster;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.junit.Test;
+
+public class PartitionAssignorAdapterTest {
+
+    private List classNames;
+    private List classTypes;
+
+    @Test
+    public void shouldInstantiateNewAssignors() {
+        classNames = Arrays.asList(StickyAssignor.class.getName());
+        List assignors = getAssignorInstances(classNames, Collections.emptyMap());
+        assertTrue(StickyAssignor.class.isInstance(assignors.get(0)));
+    }
+
+    @Test
+    public void shouldAdaptOldAssignors() {
+        classNames = Arrays.asList(OldPartitionAssignor.class.getName());
+        List assignors = getAssignorInstances(classNames, Collections.emptyMap());
+        assertTrue(PartitionAssignorAdapter.class.isInstance(assignors.get(0)));
+    }
+
+    @Test
+    public void shouldThrowKafkaExceptionOnNonAssignor() {
+        classNames = Arrays.asList(String.class.getName());
+        assertThrows(KafkaException.class, () -> getAssignorInstances(classNames, Collections.emptyMap()));
+    }
+
+    @Test
+    public void shouldThrowKafkaExceptionOnAssignorNotFound() {
+        classNames = Arrays.asList("Non-existent assignor");
+        assertThrows(KafkaException.class, () -> getAssignorInstances(classNames, Collections.emptyMap()));
+    }
+
+    @Test
+    public void shouldInstantiateFromListOfOldAndNewClassTypes() {
+        Properties props = new Properties();
+        props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999");
+
+        classTypes = Arrays.asList(StickyAssignor.class, OldPartitionAssignor.class);
+
+        props.put(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, classTypes);
+        KafkaConsumer consumer = new KafkaConsumer<>(
+            props, new StringDeserializer(), new StringDeserializer());
+
+        consumer.close();
+    }
+
+    @Test
+    public void shouldThrowKafkaExceptionOnListWithNonAssignorClassType() {
+        Properties props = new Properties();
+        props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999");
+
+        classTypes = Arrays.asList(StickyAssignor.class, OldPartitionAssignor.class, String.class);
+
+        props.put(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, classTypes);
+        assertThrows(KafkaException.class, () -> new KafkaConsumer<>(
+            props, new StringDeserializer(), new StringDeserializer()));
+    }
+
+    @Test
+    @SuppressWarnings("deprecation")
+    public void testOnAssignment() {
+        OldPartitionAssignor oldAssignor = new OldPartitionAssignor();
+        ConsumerPartitionAssignor adaptedAssignor = new PartitionAssignorAdapter(oldAssignor);
+
+        TopicPartition tp1 = new TopicPartition("tp1", 1);
+        TopicPartition tp2 = new TopicPartition("tp2", 2);
+        List partitions = Arrays.asList(tp1, tp2);
+
+        adaptedAssignor.onAssignment(new Assignment(partitions), new ConsumerGroupMetadata("", 1, "", Optional.empty()));
+
+        assertEquals(oldAssignor.partitions, partitions);
+    }
+
+    @Test
+    public void testAssign() {
+        ConsumerPartitionAssignor adaptedAssignor = new PartitionAssignorAdapter(new OldPartitionAssignor());
+
+        Map subscriptions = new HashMap<>();
+        subscriptions.put("C1", new Subscription(Arrays.asList("topic1")));
+        subscriptions.put("C2", new Subscription(Arrays.asList("topic1", "topic2")));
+        subscriptions.put("C3", new Subscription(Arrays.asList("topic2", "topic3")));
+        GroupSubscription groupSubscription = new GroupSubscription(subscriptions);
+
+        Map assignments = adaptedAssignor.assign(null, groupSubscription).groupAssignment();
+
+        assertEquals(assignments.get("C1").partitions(), Arrays.asList(new TopicPartition("topic1", 1)));
+        assertEquals(assignments.get("C2").partitions(), Arrays.asList(new TopicPartition("topic1", 1), new TopicPartition("topic2", 1)));
+        assertEquals(assignments.get("C3").partitions(), Arrays.asList(new TopicPartition("topic2", 1), new TopicPartition("topic3", 1)));
+    }
+
+    /*
+     * Dummy assignor just gives each consumer partition 1 of each topic it's subscribed to
+     */
+    @SuppressWarnings("deprecation")
+    public static class OldPartitionAssignor implements PartitionAssignor {
+
+        List partitions = null;
+
+        @Override
+        public Subscription subscription(Set topics) {
+            return new Subscription(new ArrayList<>(topics), null);
+        }
+
+        @Override
+        public Map assign(Cluster metadata, Map subscriptions) {
+            Map assignments = new HashMap<>();
+            for (Map.Entry entry : subscriptions.entrySet()) {
+                List partitions = new ArrayList<>();
+                for (String topic : entry.getValue().topics()) {
+                    partitions.add(new TopicPartition(topic, 1));
+                }
+                assignments.put(entry.getKey(), new Assignment(partitions, null));
+            }
+            return assignments;
+        }
+
+        @Override
+        public void onAssignment(Assignment assignment) {
+            partitions = assignment.partitions();
+        }
+
+        @Override
+        public String name() {
+            return "old-assignor";
+        }
+    }
+
+}
diff --git a/docs/upgrade.html b/docs/upgrade.html
index 9d0e73803a644..0ae59046d0b27 100644
--- a/docs/upgrade.html
+++ b/docs/upgrade.html
@@ -29,6 +29,8 @@ 
    Notable changes in 2
         avoid potential misuse. The constructor TopicAuthorizationException(String) which was previously used for a single
         unauthorized topic was changed similarly.
     
+    
  • The internal PartitionAssignor interface has been deprecated and replaced with a new ConsumerPartitionAssignor in the public API. Users
+        implementing a custom PartitionAssignor should migrate to the new interface as soon as possible.
    • 
 
 
 
    Upgrading from 0.8.x, 0.9.x, 0.10.0.x, 0.10.1.x, 0.10.2.x, 0.11.0.x, 1.0.x, 1.1.x, 2.0.x or 2.1.x or 2.2.x to 2.3.0
    
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
index fa5f5115d65c1..36196a68ddba1 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
@@ -373,8 +373,8 @@ private Map errorAssignment(final Map
      * 3. within each client, tasks are assigned to consumer clients in round-robin manner.
      */
     @Override
-    public GroupAssignment assign(final Cluster metadata, final GroupSubscription groupSubscriptions) {
-        final Map subscriptions = groupSubscriptions.groupSubscription();
+    public GroupAssignment assign(final Cluster metadata, final GroupSubscription groupSubscription) {
+        final Map subscriptions = groupSubscription.groupSubscription();
         // construct the client metadata from the decoded subscription info
         final Map clientMetadataMap = new HashMap<>();
         final Set futureConsumers = new HashSet<>();

From 2c2b30d96b7d30037d3ee69ebbf985cd88557af9 Mon Sep 17 00:00:00 2001
From: jolshan 
Date: Wed, 31 Jul 2019 14:00:49 -0700
Subject: [PATCH 0493/1071] MINOR: Add RandomComponentPayloadGenerator and
 update Trogdor documentation (#7103)

Add a new RandomComponentPayloadGenerator that gives a payload based on random selection of another PayloadGenerator.  Additionally, add an example that uses a non-default PayloadGenerator configuration to TROGDOR.md.

Reviewers: Colin P. McCabe 
---
 TROGDOR.md                                    |  30 +++++
 .../trogdor/workload/PayloadGenerator.java    |   3 +-
 .../trogdor/workload/RandomComponent.java     |  49 ++++++++
 .../RandomComponentPayloadGenerator.java      | 114 ++++++++++++++++++
 .../workload/PayloadGeneratorTest.java        |  97 ++++++++++++++-
 5 files changed, 290 insertions(+), 3 deletions(-)
 create mode 100644 tools/src/main/java/org/apache/kafka/trogdor/workload/RandomComponent.java
 create mode 100644 tools/src/main/java/org/apache/kafka/trogdor/workload/RandomComponentPayloadGenerator.java

diff --git a/TROGDOR.md b/TROGDOR.md
index ad8d8af797e02..3891857562f93 100644
--- a/TROGDOR.md
+++ b/TROGDOR.md
@@ -87,6 +87,36 @@ The task specification is usually written as JSON.  For example, this task speci
         "durationMs": 30000,
         "partitions": [["node1", "node2"], ["node3"]]
     }
+    
+This task runs a simple ProduceBench test on a cluster with one producer node, 5 topics, and 10,000 messages per second. 
+The keys are generated sequentially and the configured partitioner (DefaultPartitioner) is used. 
+
+    {
+        "class": "org.apache.kafka.trogdor.workload.ProduceBenchSpec",
+        "durationMs": 10000000,
+        "producerNode": "node0",
+        "bootstrapServers": "localhost:9092",
+        "targetMessagesPerSec": 10000,
+        "maxMessages": 50000,
+        "activeTopics": {
+            "foo[1-3]": {
+                "numPartitions": 10,
+                "replicationFactor": 1
+            }
+        },
+        "inactiveTopics": {
+             "foo[4-5]": {
+                 "numPartitions": 10,
+                 "replicationFactor": 1
+             }
+        },
+        "keyGenerator": {
+             "type": "sequential",
+             "size": 8,
+             "offset": 1
+        },
+        "useConfiguredPartitioner": true
+     }
 
 Tasks are submitted to the coordinator.  Once the coordinator determines that it is time for the task to start, it creates workers on agent processes.  The workers run until the task is done.
 
diff --git a/tools/src/main/java/org/apache/kafka/trogdor/workload/PayloadGenerator.java b/tools/src/main/java/org/apache/kafka/trogdor/workload/PayloadGenerator.java
index 3c574bac810df..b06ba016158fc 100644
--- a/tools/src/main/java/org/apache/kafka/trogdor/workload/PayloadGenerator.java
+++ b/tools/src/main/java/org/apache/kafka/trogdor/workload/PayloadGenerator.java
@@ -34,7 +34,8 @@
     @JsonSubTypes.Type(value = ConstantPayloadGenerator.class, name = "constant"),
     @JsonSubTypes.Type(value = SequentialPayloadGenerator.class, name = "sequential"),
     @JsonSubTypes.Type(value = UniformRandomPayloadGenerator.class, name = "uniformRandom"),
-    @JsonSubTypes.Type(value = NullPayloadGenerator.class, name = "null")
+    @JsonSubTypes.Type(value = NullPayloadGenerator.class, name = "null"),
+    @JsonSubTypes.Type(value = RandomComponentPayloadGenerator.class, name = "randomComponent")
     })
 public interface PayloadGenerator {
     /**
diff --git a/tools/src/main/java/org/apache/kafka/trogdor/workload/RandomComponent.java b/tools/src/main/java/org/apache/kafka/trogdor/workload/RandomComponent.java
new file mode 100644
index 0000000000000..b5973a8c74443
--- /dev/null
+++ b/tools/src/main/java/org/apache/kafka/trogdor/workload/RandomComponent.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.trogdor.workload;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Contains a percent value represented as an integer between 1 and 100 and a PayloadGenerator to specify
+ * how often that PayloadGenerator should be used. 
+ */
+public class RandomComponent {
+    private final int percent;
+    private final PayloadGenerator component;
+    
+
+    @JsonCreator
+    public RandomComponent(@JsonProperty("percent") int percent,
+                           @JsonProperty("component") PayloadGenerator component) {
+        this.percent = percent;
+        this.component = component;
+    }
+
+    @JsonProperty
+    public int percent() {
+        return percent;
+    }
+
+    @JsonProperty
+    public PayloadGenerator component() {
+        return component;
+    }
+}
+
diff --git a/tools/src/main/java/org/apache/kafka/trogdor/workload/RandomComponentPayloadGenerator.java b/tools/src/main/java/org/apache/kafka/trogdor/workload/RandomComponentPayloadGenerator.java
new file mode 100644
index 0000000000000..be50a4427419e
--- /dev/null
+++ b/tools/src/main/java/org/apache/kafka/trogdor/workload/RandomComponentPayloadGenerator.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.trogdor.workload;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+
+/**
+ * A PayloadGenerator which generates pseudo-random payloads based on other PayloadGenerators.
+ *
+ * Given a seed and non-null list of RandomComponents, RandomComponentPayloadGenerator
+ * will use any given generator in its list of components a percentage of the time based on the 
+ * percent field in the RandomComponent. These percent fields must be integers greater than 0 
+ * and together add up to 100. The payloads generated can be reproduced from run to run.
+ * 
+ * An example of how to include this generator in a Trogdor taskSpec is shown below.
+ * #{@code
+ *    "keyGenerator": {
+ *        "type": "randomComponent",
+ *        "seed": 456,
+ *        "components": [
+ *          {
+ *            "percent": 50,
+ *            "component": {
+ *              "type": "null"
+ *            }
+ *          },
+ *          {
+ *            "percent": 50,
+ *            "component": {
+ *              "type": "uniformRandom",
+ *              "size": 4,
+ *              "seed": 123,
+ *              "padding": 0
+ *            }
+ *          }
+ *        ]
+ *    }
+ * }
+ */
+public class RandomComponentPayloadGenerator implements PayloadGenerator {
+    private final long seed;
+    private final List components;
+    private final Random random = new Random();
+
+    @JsonCreator
+    public RandomComponentPayloadGenerator(@JsonProperty("seed") long seed,
+                                           @JsonProperty("components") List components) {
+        this.seed = seed;
+        if (components == null || components.isEmpty()) {
+            throw new IllegalArgumentException("Components must be a specified, non-empty list of RandomComponents.");
+        }
+        int sum = 0;
+        for (RandomComponent component : components) {
+            if (component.percent() < 1) {
+                throw new IllegalArgumentException("Percent value must be greater than zero.");
+            }
+            sum += component.percent();
+        }
+        if (sum != 100) {
+            throw new IllegalArgumentException("Components must be a list of RandomComponents such that the percent fields sum to 100");
+        }
+        this.components = new ArrayList<>(components);
+    }
+
+    @JsonProperty
+    public long seed() {
+        return seed;
+    }
+
+    @JsonProperty
+    public List components() {
+        return components;
+    }
+
+    @Override
+    public byte[] generate(long position) {
+        int randPercent;
+        synchronized (random) {
+            random.setSeed(seed + position);
+            randPercent = random.nextInt(100);
+        }
+        int curPercent = 0;
+        RandomComponent com = components.get(0);
+        for (RandomComponent component : components) {
+            curPercent += component.percent();
+            if (curPercent > randPercent) {
+                com = component;
+                break;
+            }
+        }
+        return com.component().generate(position);
+    }
+}
diff --git a/tools/src/test/java/org/apache/kafka/trogdor/workload/PayloadGeneratorTest.java b/tools/src/test/java/org/apache/kafka/trogdor/workload/PayloadGeneratorTest.java
index 0909dc0d05c36..9ee654b2c08cd 100644
--- a/tools/src/test/java/org/apache/kafka/trogdor/workload/PayloadGeneratorTest.java
+++ b/tools/src/test/java/org/apache/kafka/trogdor/workload/PayloadGeneratorTest.java
@@ -24,10 +24,13 @@
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.List;
 
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
-
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThrows;
 
 public class PayloadGeneratorTest {
     @Rule
@@ -104,7 +107,11 @@ private static void testReproducible(PayloadGenerator generator) {
         byte[] val = generator.generate(123);
         generator.generate(456);
         byte[] val2 = generator.generate(123);
-        assertArrayEquals(val, val2);
+        if (val == null) {
+            assertNull(val2);
+        } else {
+            assertArrayEquals(val, val2);
+        }
     }
 
     @Test
@@ -123,6 +130,92 @@ public void testUniformRandomPayloadGeneratorPaddingBytes() {
         assertArrayEquals(val1End, val2End);
         assertArrayEquals(val1End, val3End);
     }
+    
+    @Test
+    public void testRandomComponentPayloadGenerator() {
+        NullPayloadGenerator nullGenerator = new NullPayloadGenerator();
+        RandomComponent nullConfig = new RandomComponent(50, nullGenerator);
+        
+        UniformRandomPayloadGenerator uniformGenerator =
+            new UniformRandomPayloadGenerator(5, 123, 0);
+        RandomComponent uniformConfig = new RandomComponent(50, uniformGenerator);
+        
+        SequentialPayloadGenerator sequentialGenerator =
+            new SequentialPayloadGenerator(4, 10);
+        RandomComponent sequentialConfig = new RandomComponent(75, sequentialGenerator);
+        
+        ConstantPayloadGenerator constantGenerator =
+            new ConstantPayloadGenerator(4, new byte[0]);
+        RandomComponent constantConfig = new RandomComponent(25, constantGenerator);
+        
+        List components1 = new ArrayList<>(Arrays.asList(nullConfig, uniformConfig));
+        List components2 = new ArrayList<>(Arrays.asList(sequentialConfig, constantConfig));
+        byte[] expected = new byte[4];
+
+        PayloadIterator iter = new PayloadIterator(
+            new RandomComponentPayloadGenerator(4, components1));
+        int notNull = 0;
+        int isNull = 0;
+        while (notNull < 1000 || isNull < 1000) {
+            byte[] cur = iter.next();
+            if (cur == null) {
+                isNull++;
+            } else {
+                notNull++;
+            }
+        }
+        
+        iter = new PayloadIterator(
+            new RandomComponentPayloadGenerator(123, components2));
+        int isZeroBytes = 0;
+        int isNotZeroBytes = 0;
+        while (isZeroBytes < 500 || isNotZeroBytes < 1500) {
+            byte[] cur = iter.next();
+            if (Arrays.equals(expected, cur)) {
+                isZeroBytes++;
+            } else {
+                isNotZeroBytes++;
+            }
+        }
+        
+        RandomComponent uniformConfig2 = new RandomComponent(25, uniformGenerator);
+        RandomComponent sequentialConfig2 = new RandomComponent(25, sequentialGenerator);
+        RandomComponent nullConfig2 = new RandomComponent(25, nullGenerator);
+        
+        List components3 = new ArrayList<>(Arrays.asList(sequentialConfig2, uniformConfig2, nullConfig));
+        List components4 = new ArrayList<>(Arrays.asList(uniformConfig2, sequentialConfig2, constantConfig, nullConfig2));
+        
+        testReproducible(new RandomComponentPayloadGenerator(4, components1));
+        testReproducible(new RandomComponentPayloadGenerator(123, components2));
+        testReproducible(new RandomComponentPayloadGenerator(50, components3));
+        testReproducible(new RandomComponentPayloadGenerator(0, components4));
+    } 
+    
+    @Test
+    public void testRandomComponentPayloadGeneratorErrors() {
+        NullPayloadGenerator nullGenerator = new NullPayloadGenerator();
+        RandomComponent nullConfig = new RandomComponent(25, nullGenerator);
+        UniformRandomPayloadGenerator uniformGenerator =
+            new UniformRandomPayloadGenerator(5, 123, 0);
+        RandomComponent uniformConfig = new RandomComponent(25, uniformGenerator);
+        ConstantPayloadGenerator constantGenerator =
+            new ConstantPayloadGenerator(4, new byte[0]);
+        RandomComponent constantConfig = new RandomComponent(-25, constantGenerator);
+        
+        List components1 = new ArrayList<>(Arrays.asList(nullConfig, uniformConfig));
+        List components2 = new ArrayList<>(Arrays.asList(
+             nullConfig, constantConfig, uniformConfig, nullConfig, uniformConfig, uniformConfig));
+     
+        assertThrows(IllegalArgumentException.class, () -> {
+            new PayloadIterator(new RandomComponentPayloadGenerator(1, new ArrayList<>()));
+        });
+        assertThrows(IllegalArgumentException.class, () -> {
+            new PayloadIterator(new RandomComponentPayloadGenerator(13, components2));
+        });
+        assertThrows(IllegalArgumentException.class, () -> {
+            new PayloadIterator(new RandomComponentPayloadGenerator(123, components1));
+        });
+    }  
 
     @Test
     public void testPayloadIterator() {

From a028f597d7ea042c414e9669d2261c1423a96f95 Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Wed, 31 Jul 2019 14:19:46 -0700
Subject: [PATCH 0494/1071] KAFKA-8731: InMemorySessionStore throws
 NullPointerException on startup (#7132)

Reviewers:  Matthias J. Sax , Bill Bejeck 
---
 .../streams/state/internals/InMemorySessionStore.java     | 8 ++++++++
 .../streams/state/internals/SessionBytesStoreTest.java    | 5 +++++
 2 files changed, 13 insertions(+)

diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java
index f3b85657278dc..ebe9878fe2721 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java
@@ -120,7 +120,15 @@ public void put(final Windowed sessionKey, final byte[] aggregate) {
     @Override
     public void remove(final Windowed sessionKey) {
         final ConcurrentNavigableMap> keyMap = endTimeMap.get(sessionKey.window().end());
+        if (keyMap == null) {
+            return;
+        }
+
         final ConcurrentNavigableMap startTimeMap = keyMap.get(sessionKey.key());
+        if (startTimeMap == null) {
+            return;
+        }
+
         startTimeMap.remove(sessionKey.window().start());
 
         if (startTimeMap.isEmpty()) {
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/SessionBytesStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/SessionBytesStoreTest.java
index a6cef81b3777e..9b29e8b9ff607 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/SessionBytesStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/SessionBytesStoreTest.java
@@ -472,6 +472,11 @@ public void shouldLogAndMeasureExpiredRecords() {
         assertThat(messages, hasItem("Skipping record for expired segment."));
     }
 
+    @Test
+    public void shouldNotThrowExceptionRemovingNonexistentKey() {
+        sessionStore.remove(new Windowed<>("a", new SessionWindow(0, 1)));
+    }
+
     @Test(expected = NullPointerException.class)
     public void shouldThrowNullPointerExceptionOnFindSessionsNullKey() {
         sessionStore.findSessions(null, 1L, 2L);

From 06e246d4a11c63d1b46516442de81f370f89a0ee Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Thu, 1 Aug 2019 11:31:00 -0700
Subject: [PATCH 0495/1071] KAFKA-8456: Stabilize flaky
 StoreUpgradeIntegrationTest (#6941)

Reviewers: Boyang Chen , Guozhang Wang , Bill Bejeck , A. Sophie Blee-Goldman 
---
 .../StoreUpgradeIntegrationTest.java          | 273 ++++++++++--------
 1 file changed, 150 insertions(+), 123 deletions(-)

diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/StoreUpgradeIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/StoreUpgradeIntegrationTest.java
index ba5b08f632e83..9a82ad9f89e81 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/StoreUpgradeIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/StoreUpgradeIntegrationTest.java
@@ -329,56 +329,65 @@ private  void processKeyValueAndVerifyPlainCount(final K key,
                 IntegerSerializer.class),
             CLUSTER.time);
 
-        TestUtils.waitForCondition(() -> {
-            try {
-                final ReadOnlyKeyValueStore store =
-                    kafkaStreams.store(STORE_NAME, QueryableStoreTypes.keyValueStore());
-                try (final KeyValueIterator all = store.all()) {
-                    final List> storeContent = new LinkedList<>();
-                    while (all.hasNext()) {
-                        storeContent.add(all.next());
+        TestUtils.waitForCondition(
+            () -> {
+                try {
+                    final ReadOnlyKeyValueStore store =
+                        kafkaStreams.store(STORE_NAME, QueryableStoreTypes.keyValueStore());
+                    try (final KeyValueIterator all = store.all()) {
+                        final List> storeContent = new LinkedList<>();
+                        while (all.hasNext()) {
+                            storeContent.add(all.next());
+                        }
+                        return storeContent.equals(expectedStoreContent);
                     }
-                    return storeContent.equals(expectedStoreContent);
+                } catch (final Exception swallow) {
+                    swallow.printStackTrace();
+                    System.err.println(swallow.getMessage());
+                    return false;
                 }
-            } catch (final Exception swallow) {
-                swallow.printStackTrace();
-                System.err.println(swallow.getMessage());
-                return false;
-            }
-        }, "Could not get expected result in time.");
+            },
+            60_000L,
+            "Could not get expected result in time.");
     }
 
     private  void verifyCountWithTimestamp(final K key,
                                               final long value,
                                               final long timestamp) throws Exception {
-        TestUtils.waitForCondition(() -> {
-            try {
-                final ReadOnlyKeyValueStore> store =
-                    kafkaStreams.store(STORE_NAME, QueryableStoreTypes.timestampedKeyValueStore());
-                final ValueAndTimestamp count = store.get(key);
-                return count.value() == value && count.timestamp() == timestamp;
-            } catch (final Exception swallow) {
-                swallow.printStackTrace();
-                System.err.println(swallow.getMessage());
-                return false;
-            }
-        }, "Could not get expected result in time.");
+        TestUtils.waitForCondition(
+            () -> {
+                try {
+                    final ReadOnlyKeyValueStore> store =
+                        kafkaStreams.store(STORE_NAME, QueryableStoreTypes.timestampedKeyValueStore());
+                    final ValueAndTimestamp count = store.get(key);
+                    return count.value() == value && count.timestamp() == timestamp;
+                } catch (final Exception swallow) {
+                    swallow.printStackTrace();
+                    System.err.println(swallow.getMessage());
+                    return false;
+                }
+            },
+            60_000L,
+            "Could not get expected result in time.");
     }
 
     private  void verifyCountWithSurrogateTimestamp(final K key,
                                                        final long value) throws Exception {
-        TestUtils.waitForCondition(() -> {
-            try {
-                final ReadOnlyKeyValueStore> store =
-                    kafkaStreams.store(STORE_NAME, QueryableStoreTypes.timestampedKeyValueStore());
-                final ValueAndTimestamp count = store.get(key);
-                return count.value() == value && count.timestamp() == -1L;
-            } catch (final Exception swallow) {
-                swallow.printStackTrace();
-                System.err.println(swallow.getMessage());
-                return false;
-            }
-        }, "Could not get expected result in time.");
+        TestUtils.waitForCondition(
+            () -> {
+                try {
+                    final ReadOnlyKeyValueStore> store =
+                        kafkaStreams.store(STORE_NAME, QueryableStoreTypes.timestampedKeyValueStore());
+                    final ValueAndTimestamp count = store.get(key);
+                    return count.value() == value && count.timestamp() == -1L;
+                } catch (final Exception swallow) {
+                    swallow.printStackTrace();
+                    System.err.println(swallow.getMessage());
+                    return false;
+                }
+            },
+            60_000L,
+            "Could not get expected result in time.");
     }
 
     private  void processKeyValueAndVerifyCount(final K key,
@@ -394,23 +403,26 @@ private  void processKeyValueAndVerifyCount(final K key,
                 IntegerSerializer.class),
             timestamp);
 
-        TestUtils.waitForCondition(() -> {
-            try {
-                final ReadOnlyKeyValueStore> store =
-                    kafkaStreams.store(STORE_NAME, QueryableStoreTypes.timestampedKeyValueStore());
-                try (final KeyValueIterator> all = store.all()) {
-                    final List>> storeContent = new LinkedList<>();
-                    while (all.hasNext()) {
-                        storeContent.add(all.next());
+        TestUtils.waitForCondition(
+            () -> {
+                try {
+                    final ReadOnlyKeyValueStore> store =
+                        kafkaStreams.store(STORE_NAME, QueryableStoreTypes.timestampedKeyValueStore());
+                    try (final KeyValueIterator> all = store.all()) {
+                        final List>> storeContent = new LinkedList<>();
+                        while (all.hasNext()) {
+                            storeContent.add(all.next());
+                        }
+                        return storeContent.equals(expectedStoreContent);
                     }
-                    return storeContent.equals(expectedStoreContent);
+                } catch (final Exception swallow) {
+                    swallow.printStackTrace();
+                    System.err.println(swallow.getMessage());
+                    return false;
                 }
-            } catch (final Exception swallow) {
-                swallow.printStackTrace();
-                System.err.println(swallow.getMessage());
-                return false;
-            }
-        }, "Could not get expected result in time.");
+            },
+            60_000L,
+            "Could not get expected result in time.");
     }
 
     private  void processKeyValueAndVerifyCountWithTimestamp(final K key,
@@ -426,23 +438,26 @@ private  void processKeyValueAndVerifyCountWithTimestamp(final K key,
                 IntegerSerializer.class),
             timestamp);
 
-        TestUtils.waitForCondition(() -> {
-            try {
-                final ReadOnlyKeyValueStore> store =
-                    kafkaStreams.store(STORE_NAME, QueryableStoreTypes.timestampedKeyValueStore());
-                try (final KeyValueIterator> all = store.all()) {
-                    final List>> storeContent = new LinkedList<>();
-                    while (all.hasNext()) {
-                        storeContent.add(all.next());
+        TestUtils.waitForCondition(
+            () -> {
+                try {
+                    final ReadOnlyKeyValueStore> store =
+                        kafkaStreams.store(STORE_NAME, QueryableStoreTypes.timestampedKeyValueStore());
+                    try (final KeyValueIterator> all = store.all()) {
+                        final List>> storeContent = new LinkedList<>();
+                        while (all.hasNext()) {
+                            storeContent.add(all.next());
+                        }
+                        return storeContent.equals(expectedStoreContent);
                     }
-                    return storeContent.equals(expectedStoreContent);
+                } catch (final Exception swallow) {
+                    swallow.printStackTrace();
+                    System.err.println(swallow.getMessage());
+                    return false;
                 }
-            } catch (final Exception swallow) {
-                swallow.printStackTrace();
-                System.err.println(swallow.getMessage());
-                return false;
-            }
-        }, "Could not get expected result in time.");
+            },
+            60_000L,
+            "Could not get expected result in time.");
     }
 
     @Test
@@ -789,56 +804,65 @@ private  void processWindowedKeyValueAndVerifyPlainCount(final K key,
                 IntegerSerializer.class),
             CLUSTER.time);
 
-        TestUtils.waitForCondition(() -> {
-            try {
-                final ReadOnlyWindowStore store =
-                    kafkaStreams.store(STORE_NAME, QueryableStoreTypes.windowStore());
-                try (final KeyValueIterator, V> all = store.all()) {
-                    final List, V>> storeContent = new LinkedList<>();
-                    while (all.hasNext()) {
-                        storeContent.add(all.next());
+        TestUtils.waitForCondition(
+            () -> {
+                try {
+                    final ReadOnlyWindowStore store =
+                        kafkaStreams.store(STORE_NAME, QueryableStoreTypes.windowStore());
+                    try (final KeyValueIterator, V> all = store.all()) {
+                        final List, V>> storeContent = new LinkedList<>();
+                        while (all.hasNext()) {
+                            storeContent.add(all.next());
+                        }
+                        return storeContent.equals(expectedStoreContent);
                     }
-                    return storeContent.equals(expectedStoreContent);
+                } catch (final Exception swallow) {
+                    swallow.printStackTrace();
+                    System.err.println(swallow.getMessage());
+                    return false;
                 }
-            } catch (final Exception swallow) {
-                swallow.printStackTrace();
-                System.err.println(swallow.getMessage());
-                return false;
-            }
-        }, "Could not get expected result in time.");
+            },
+            60_000L,
+            "Could not get expected result in time.");
     }
 
     private  void verifyWindowedCountWithSurrogateTimestamp(final Windowed key,
                                                                final long value) throws Exception {
-        TestUtils.waitForCondition(() -> {
-            try {
-                final ReadOnlyWindowStore> store =
-                    kafkaStreams.store(STORE_NAME, QueryableStoreTypes.timestampedWindowStore());
-                final ValueAndTimestamp count = store.fetch(key.key(), key.window().start());
-                return count.value() == value && count.timestamp() == -1L;
-            } catch (final Exception swallow) {
-                swallow.printStackTrace();
-                System.err.println(swallow.getMessage());
-                return false;
-            }
-        }, "Could not get expected result in time.");
+        TestUtils.waitForCondition(
+            () -> {
+                try {
+                    final ReadOnlyWindowStore> store =
+                        kafkaStreams.store(STORE_NAME, QueryableStoreTypes.timestampedWindowStore());
+                    final ValueAndTimestamp count = store.fetch(key.key(), key.window().start());
+                    return count.value() == value && count.timestamp() == -1L;
+                } catch (final Exception swallow) {
+                    swallow.printStackTrace();
+                    System.err.println(swallow.getMessage());
+                    return false;
+                }
+            },
+            60_000L,
+            "Could not get expected result in time.");
     }
 
     private  void verifyWindowedCountWithTimestamp(final Windowed key,
                                                       final long value,
                                                       final long timestamp) throws Exception {
-        TestUtils.waitForCondition(() -> {
-            try {
-                final ReadOnlyWindowStore> store =
-                    kafkaStreams.store(STORE_NAME, QueryableStoreTypes.timestampedWindowStore());
-                final ValueAndTimestamp count = store.fetch(key.key(), key.window().start());
-                return count.value() == value && count.timestamp() == timestamp;
-            } catch (final Exception swallow) {
-                swallow.printStackTrace();
-                System.err.println(swallow.getMessage());
-                return false;
-            }
-        }, "Could not get expected result in time.");
+        TestUtils.waitForCondition(
+            () -> {
+                try {
+                    final ReadOnlyWindowStore> store =
+                        kafkaStreams.store(STORE_NAME, QueryableStoreTypes.timestampedWindowStore());
+                    final ValueAndTimestamp count = store.fetch(key.key(), key.window().start());
+                    return count.value() == value && count.timestamp() == timestamp;
+                } catch (final Exception swallow) {
+                    swallow.printStackTrace();
+                    System.err.println(swallow.getMessage());
+                    return false;
+                }
+            },
+            60_000L,
+            "Could not get expected result in time.");
     }
 
     private  void processKeyValueAndVerifyWindowedCountWithTimestamp(final K key,
@@ -854,23 +878,26 @@ private  void processKeyValueAndVerifyWindowedCountWithTimestamp(final K k
                 IntegerSerializer.class),
             timestamp);
 
-        TestUtils.waitForCondition(() -> {
-            try {
-                final ReadOnlyWindowStore> store =
-                    kafkaStreams.store(STORE_NAME, QueryableStoreTypes.timestampedWindowStore());
-                try (final KeyValueIterator, ValueAndTimestamp> all = store.all()) {
-                    final List, ValueAndTimestamp>> storeContent = new LinkedList<>();
-                    while (all.hasNext()) {
-                        storeContent.add(all.next());
+        TestUtils.waitForCondition(
+            () -> {
+                try {
+                    final ReadOnlyWindowStore> store =
+                        kafkaStreams.store(STORE_NAME, QueryableStoreTypes.timestampedWindowStore());
+                    try (final KeyValueIterator, ValueAndTimestamp> all = store.all()) {
+                        final List, ValueAndTimestamp>> storeContent = new LinkedList<>();
+                        while (all.hasNext()) {
+                            storeContent.add(all.next());
+                        }
+                        return storeContent.equals(expectedStoreContent);
                     }
-                    return storeContent.equals(expectedStoreContent);
+                } catch (final Exception swallow) {
+                    swallow.printStackTrace();
+                    System.err.println(swallow.getMessage());
+                    return false;
                 }
-            } catch (final Exception swallow) {
-                swallow.printStackTrace();
-                System.err.println(swallow.getMessage());
-                return false;
-            }
-        }, "Could not get expected result in time.");
+            },
+            60_000L,
+            "Could not get expected result in time.");
     }
 
     private static class KeyValueProcessor implements Processor {

From d997dc65a434558961a842fd2006a6ed36cd1ac7 Mon Sep 17 00:00:00 2001
From: cadonna 
Date: Thu, 1 Aug 2019 20:47:21 +0200
Subject: [PATCH 0496/1071] MINOR: Fix wrong debug log message (#7137)

Reviewers: Bill Bejeck , Matthias J. Sax 
---
 .../streams/processor/internals/ProcessorStateManager.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorStateManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorStateManager.java
index 7e16abcf4fb6d..2e80c94e5f570 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorStateManager.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorStateManager.java
@@ -119,7 +119,7 @@ public ProcessorStateManager(final TaskId taskId,
             checkpointFile = null;
         }
 
-        log.debug("Created state store manager for task {} with the acquired state dir lock", taskId);
+        log.debug("Created state store manager for task {}", taskId);
     }
 
 

From 717c55be971df862c55f55d245b9997f1d6f998c Mon Sep 17 00:00:00 2001
From: Justine Olshan 
Date: Thu, 1 Aug 2019 14:36:12 -0700
Subject: [PATCH 0497/1071] KAFKA-8601: Implement KIP-480: Sticky Partitioning
 for keyless records (#6997)

Implement KIP-480, which specifies that the default partitioner should use a "sticky" partitioning strategy for records that have a null key.

Reviewers: Colin P. McCabe , Lucas Bradstreet , Stanislav Kozlovski , Jun Rao , Kamal Chandraprakash  
---
 checkstyle/suppressions.xml                   |   2 +-
 .../kafka/clients/producer/KafkaProducer.java |  27 ++-
 .../kafka/clients/producer/Partitioner.java   |  10 ++
 .../internals/DefaultPartitioner.java         |  50 ++----
 .../producer/internals/RecordAccumulator.java |  18 +-
 .../internals/StickyPartitionCache.java       |  76 +++++++++
 .../internals/TransactionManager.java         |   4 +-
 .../internals/DefaultPartitionerTest.java     |  80 ++-------
 .../internals/RecordAccumulatorTest.java      | 151 +++++++++++++----
 .../producer/internals/SenderTest.java        | 160 +++++++++---------
 .../internals/StickyPartitionCacheTest.java   | 110 ++++++++++++
 .../internals/TransactionManagerTest.java     | 135 +++++++++++----
 12 files changed, 569 insertions(+), 254 deletions(-)
 create mode 100644 clients/src/main/java/org/apache/kafka/clients/producer/internals/StickyPartitionCache.java
 create mode 100644 clients/src/test/java/org/apache/kafka/clients/producer/internals/StickyPartitionCacheTest.java

diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index 908e1148d6323..0bb2193c049ef 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -55,7 +55,7 @@
               files="AbstractRequest.java|KerberosLogin.java|WorkerSinkTaskTest.java|TransactionManagerTest.java|SenderTest.java"/>
 
     
+              files="(BufferPool|Fetcher|MetricName|Node|ConfigDef|RecordBatch|SslFactory|SslTransportLayer|MetadataResponse|KerberosLogin|Selector|Sender|Serdes|TokenInformation|Agent|Values|PluginUtils|MiniTrogdorCluster|TasksRequest|KafkaProducer).java"/>
 
     
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
index 5bda54664d08f..c586fb41be414 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
@@ -907,15 +907,36 @@ private Future doSend(ProducerRecord record, Callback call
                     compressionType, serializedKey, serializedValue, headers);
             ensureValidRecordSize(serializedSize);
             long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp();
-            log.trace("Sending record {} with callback {} to topic {} partition {}", record, callback, record.topic(), partition);
+            if (log.isTraceEnabled()) {
+                log.trace("Attempting to append record {} with callback {} to topic {} partition {}", record, callback, record.topic(), partition);
+            }
             // producer callback will make sure to call both 'callback' and interceptor callback
             Callback interceptCallback = new InterceptorCallback<>(callback, this.interceptors, tp);
 
+            if (transactionManager != null && transactionManager.isTransactional()) {
+                transactionManager.failIfNotReadyForSend();
+            }
+            RecordAccumulator.RecordAppendResult result = accumulator.append(tp, timestamp, serializedKey,
+                    serializedValue, headers, interceptCallback, remainingWaitMs, true);
+            
+            if (result.abortForNewBatch) {
+                int prevPartition = partition;
+                partitioner.onNewBatch(record.topic(), cluster, prevPartition);
+                partition = partition(record, serializedKey, serializedValue, cluster);
+                tp = new TopicPartition(record.topic(), partition);
+                if (log.isTraceEnabled()) {
+                    log.trace("Retrying because of a new batch, sending the record to topic {} partition {}. The old partition was {}", record.topic(), partition, prevPartition);
+                }
+                // producer callback will make sure to call both 'callback' and interceptor callback
+                interceptCallback = new InterceptorCallback<>(callback, this.interceptors, tp);
+
+                result = accumulator.append(tp, timestamp, serializedKey,
+                    serializedValue, headers, interceptCallback, remainingWaitMs, false);
+            }
+            
             if (transactionManager != null && transactionManager.isTransactional())
                 transactionManager.maybeAddPartitionToTransaction(tp);
 
-            RecordAccumulator.RecordAppendResult result = accumulator.append(tp, timestamp, serializedKey,
-                    serializedValue, headers, interceptCallback, remainingWaitMs);
             if (result.batchIsFull || result.newBatchCreated) {
                 log.trace("Waking up the sender since topic {} partition {} is either full or getting a new batch", record.topic(), partition);
                 this.sender.wakeup();
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/Partitioner.java b/clients/src/main/java/org/apache/kafka/clients/producer/Partitioner.java
index a3a2fbea86408..1baeb0f648ba7 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/Partitioner.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/Partitioner.java
@@ -44,4 +44,14 @@ public interface Partitioner extends Configurable, Closeable {
      */
     public void close();
 
+
+    /**
+     * Notifies the partitioner a new batch is about to be created. When using the sticky partitioner,
+     * this method can change the chosen sticky partition for the new batch. 
+     * @param topic The topic name
+     * @param cluster The current cluster metadata
+     * @param prevPartition The partition previously selected for the record that triggered a new batch
+     */
+    default public void onNewBatch(String topic, Cluster cluster, int prevPartition) {
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/DefaultPartitioner.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/DefaultPartitioner.java
index 72f8f4bd2a6e9..85c5e4eb9f216 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/DefaultPartitioner.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/DefaultPartitioner.java
@@ -18,10 +18,6 @@
 
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.kafka.clients.producer.Partitioner;
 import org.apache.kafka.common.Cluster;
@@ -33,11 +29,13 @@
  * 
      
  * 
    • If a partition is specified in the record, use it
  * 
    • If no partition is specified but a key is present choose a partition based on a hash of the key
- * 
    • If no partition or key is present choose a partition in a round-robin fashion
+ * 
    • If no partition or key is present choose the sticky partition that changes when the batch is full.
+ * 
+ * See KIP-480 for details about sticky partitioning.
  */
 public class DefaultPartitioner implements Partitioner {
 
-    private final ConcurrentMap topicCounterMap = new ConcurrentHashMap<>();
+    private final StickyPartitionCache stickyPartitionCache = new StickyPartitionCache();
 
     public void configure(Map configs) {}
 
@@ -52,36 +50,22 @@ public void configure(Map configs) {}
      * @param cluster The current cluster metadata
      */
     public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {
+        if (keyBytes == null) {
+            return stickyPartitionCache.partition(topic, cluster);
+        } 
         List partitions = cluster.partitionsForTopic(topic);
         int numPartitions = partitions.size();
-        if (keyBytes == null) {
-            int nextValue = nextValue(topic);
-            List availablePartitions = cluster.availablePartitionsForTopic(topic);
-            if (!availablePartitions.isEmpty()) {
-                int part = Utils.toPositive(nextValue) % availablePartitions.size();
-                return availablePartitions.get(part).partition();
-            } else {
-                // no partitions are available, give a non-available partition
-                return Utils.toPositive(nextValue) % numPartitions;
-            }
-        } else {
-            // hash the keyBytes to choose a partition
-            return Utils.toPositive(Utils.murmur2(keyBytes)) % numPartitions;
-        }
-    }
-
-    private int nextValue(String topic) {
-        AtomicInteger counter = topicCounterMap.get(topic);
-        if (null == counter) {
-            counter = new AtomicInteger(ThreadLocalRandom.current().nextInt());
-            AtomicInteger currentCounter = topicCounterMap.putIfAbsent(topic, counter);
-            if (currentCounter != null) {
-                counter = currentCounter;
-            }
-        }
-        return counter.getAndIncrement();
+        // hash the keyBytes to choose a partition
+        return Utils.toPositive(Utils.murmur2(keyBytes)) % numPartitions;
     }
 
     public void close() {}
-
+    
+    /**
+     * If a batch completed for the current sticky partition, change the sticky partition. 
+     * Alternately, if no sticky partition has been determined, set one.
+     */
+    public void onNewBatch(String topic, Cluster cluster, int prevPartition) {
+        stickyPartitionCache.nextPartition(topic, cluster, prevPartition);
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
index 91fb8c92c2e0a..4f5aa3e4726e4 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
@@ -179,6 +179,8 @@ public double measure(MetricConfig config, long now) {
      * @param headers the Headers for the record
      * @param callback The user-supplied callback to execute when the request is complete
      * @param maxTimeToBlock The maximum time in milliseconds to block for buffer memory to be available
+     * @param abortOnNewBatch A boolean that indicates returning before a new batch is created and 
+     *                        running the the partitioner's onNewBatch method before trying to append again
      */
     public RecordAppendResult append(TopicPartition tp,
                                      long timestamp,
@@ -186,7 +188,8 @@ public RecordAppendResult append(TopicPartition tp,
                                      byte[] value,
                                      Header[] headers,
                                      Callback callback,
-                                     long maxTimeToBlock) throws InterruptedException {
+                                     long maxTimeToBlock,
+                                     boolean abortOnNewBatch) throws InterruptedException {
         // We keep track of the number of appending thread to make sure we do not miss batches in
         // abortIncompleteBatches().
         appendsInProgress.incrementAndGet();
@@ -204,6 +207,11 @@ public RecordAppendResult append(TopicPartition tp,
             }
 
             // we don't have an in-progress record batch try to allocate a new batch
+            if (abortOnNewBatch) {
+                // Return a result that will cause another call to append.
+                return new RecordAppendResult(null, false, false, true);
+            }
+            
             byte maxUsableMagic = apiVersions.maxUsableProduceMagic();
             int size = Math.max(this.batchSize, AbstractRecords.estimateSizeInBytesUpperBound(maxUsableMagic, compression, key, value, headers));
             log.trace("Allocating a new {} byte message buffer for topic {} partition {}", size, tp.topic(), tp.partition());
@@ -228,7 +236,7 @@ public RecordAppendResult append(TopicPartition tp,
 
                 // Don't deallocate this buffer in the finally block as it's being used in the record batch
                 buffer = null;
-                return new RecordAppendResult(future, dq.size() > 1 || batch.isFull(), true);
+                return new RecordAppendResult(future, dq.size() > 1 || batch.isFull(), true, false);
             }
         } finally {
             if (buffer != null)
@@ -261,7 +269,7 @@ private RecordAppendResult tryAppend(long timestamp, byte[] key, byte[] value, H
             if (future == null)
                 last.closeForRecordAppends();
             else
-                return new RecordAppendResult(future, deque.size() > 1 || last.isFull(), false);
+                return new RecordAppendResult(future, deque.size() > 1 || last.isFull(), false, false);
         }
         return null;
     }
@@ -787,11 +795,13 @@ public final static class RecordAppendResult {
         public final FutureRecordMetadata future;
         public final boolean batchIsFull;
         public final boolean newBatchCreated;
+        public final boolean abortForNewBatch;
 
-        public RecordAppendResult(FutureRecordMetadata future, boolean batchIsFull, boolean newBatchCreated) {
+        public RecordAppendResult(FutureRecordMetadata future, boolean batchIsFull, boolean newBatchCreated, boolean abortForNewBatch) {
             this.future = future;
             this.batchIsFull = batchIsFull;
             this.newBatchCreated = newBatchCreated;
+            this.abortForNewBatch = abortForNewBatch;
         }
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/StickyPartitionCache.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/StickyPartitionCache.java
new file mode 100644
index 0000000000000..117eb5f080e9b
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/StickyPartitionCache.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.producer.internals;
+
+import org.apache.kafka.common.Cluster;
+import org.apache.kafka.common.PartitionInfo;
+
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.kafka.common.utils.Utils;
+
+/**
+ * An internal class that implements a cache used for sticky partitioning behavior. The cache tracks the current sticky
+ * partition for any given topic. This class should not be used externally. 
+ */
+public class StickyPartitionCache {
+    private final ConcurrentMap indexCache;
+    public StickyPartitionCache() {
+        this.indexCache = new ConcurrentHashMap<>();
+    }
+
+    public int partition(String topic, Cluster cluster) {
+        Integer part = indexCache.get(topic);
+        if (part == null) {
+            return nextPartition(topic, cluster, -1);
+        }
+        return part;
+    }
+
+    public int nextPartition(String topic, Cluster cluster, int prevPartition) {
+        List partitions = cluster.partitionsForTopic(topic);
+        Integer oldPart = indexCache.get(topic);
+        Integer newPart = oldPart;
+        // Check that the current sticky partition for the topic is either not set or that the partition that 
+        // triggered the new batch matches the sticky partition that needs to be changed.
+        if (oldPart == null || oldPart == prevPartition) {
+            List availablePartitions = cluster.availablePartitionsForTopic(topic);
+            Integer random = Utils.toPositive(ThreadLocalRandom.current().nextInt());
+            if (availablePartitions.size() < 1) {
+                newPart = random % partitions.size();
+            } else if (availablePartitions.size() == 1) {
+                newPart = availablePartitions.get(0).partition();
+            } else {
+                while (newPart == null || newPart.equals(oldPart)) {
+                    random = Utils.toPositive(ThreadLocalRandom.current().nextInt());
+                    newPart = availablePartitions.get(random % availablePartitions.size()).partition();
+                }
+            }
+            // Only change the sticky partition if it is null or prevPartition matches the current sticky partition.
+            if (oldPart == null) {
+                indexCache.putIfAbsent(topic, newPart);
+            } else {
+                indexCache.replace(topic, prevPartition, newPart);
+            }
+            return indexCache.get(topic);
+        }
+        return indexCache.get(topic);
+    }
+
+}
\ No newline at end of file
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java
index 03b023183d1cc..cd091542dc8db 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java
@@ -338,8 +338,6 @@ public synchronized TransactionalRequestResult sendOffsetsToTransaction(Map partitions = asList(new PartitionInfo(topic, 1, null, nodes, nodes),
-                                                    new PartitionInfo(topic, 2, node1, nodes, nodes),
-                                                    new PartitionInfo(topic, 0, node0, nodes, nodes));
-    private Cluster cluster = new Cluster("clusterId", asList(node0, node1, node2), partitions,
-            Collections.emptySet(), Collections.emptySet());
+    private final static List PARTITIONS = asList(new PartitionInfo(TOPIC, 1, null, NODES, NODES),
+                                                    new PartitionInfo(TOPIC, 2, NODES[1], NODES, NODES),
+                                                    new PartitionInfo(TOPIC, 0, NODES[0], NODES, NODES));
 
     @Test
     public void testKeyPartitionIsStable() {
-        int partition = partitioner.partition("test",  null, keyBytes, null, null, cluster);
-        assertEquals("Same key should yield same partition", partition, partitioner.partition("test", null, keyBytes, null, null, cluster));
-    }
-
-    @Test
-    public void testRoundRobinWithUnavailablePartitions() {
-        // When there are some unavailable partitions, we want to make sure that (1) we always pick an available partition,
-        // and (2) the available partitions are selected in a round robin way.
-        int countForPart0 = 0;
-        int countForPart2 = 0;
-        for (int i = 1; i <= 100; i++) {
-            int part = partitioner.partition("test", null, null, null, null, cluster);
-            assertTrue("We should never choose a leader-less node in round robin", part == 0 || part == 2);
-            if (part == 0)
-                countForPart0++;
-            else
-                countForPart2++;
-        }
-        assertEquals("The distribution between two available partitions should be even", countForPart0, countForPart2);
-    }
-
-    @Test
-    public void testRoundRobin() throws InterruptedException {
-        final String topicA = "topicA";
-        final String topicB = "topicB";
-
-        List allPartitions = asList(new PartitionInfo(topicA, 0, node0, nodes, nodes),
-                new PartitionInfo(topicA, 1, node1, nodes, nodes),
-                new PartitionInfo(topicA, 2, node2, nodes, nodes),
-                new PartitionInfo(topicB, 0, node0, nodes, nodes)
-                );
-        Cluster testCluster = new Cluster("clusterId", asList(node0, node1, node2), allPartitions,
-                Collections.emptySet(), Collections.emptySet());
-
-        final Map partitionCount = new HashMap<>();
-
-        for (int i = 0; i < 30; ++i) {
-            int partition = partitioner.partition(topicA, null, null, null, null, testCluster);
-            Integer count = partitionCount.get(partition);
-            if (null == count) count = 0;
-            partitionCount.put(partition, count + 1);
-
-            if (i % 5 == 0) {
-                partitioner.partition(topicB, null, null, null, null, testCluster);
-            }
-        }
-
-        assertEquals(10, (int) partitionCount.get(0));
-        assertEquals(10, (int) partitionCount.get(1));
-        assertEquals(10, (int) partitionCount.get(2));
+        final Partitioner partitioner = new DefaultPartitioner();
+        final Cluster cluster = new Cluster("clusterId", asList(NODES), PARTITIONS,
+            Collections.emptySet(), Collections.emptySet());
+        int partition = partitioner.partition("test",  null, KEY_BYTES, null, null, cluster);
+        assertEquals("Same key should yield same partition", partition, partitioner.partition("test", null, KEY_BYTES, null, null, cluster));
     }
 }
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java
index a58f3b5977bef..cdb164af5b2dd 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java
@@ -19,6 +19,7 @@
 import org.apache.kafka.clients.ApiVersions;
 import org.apache.kafka.clients.NodeApiVersions;
 import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.Partitioner;
 import org.apache.kafka.clients.producer.RecordMetadata;
 import org.apache.kafka.common.Cluster;
 import org.apache.kafka.common.KafkaException;
@@ -107,7 +108,7 @@ public void testFull() throws Exception {
         int appends = expectedNumAppends(batchSize);
         for (int i = 0; i < appends; i++) {
             // append to the first batch
-            accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
+            accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
             Deque partitionBatches = accum.batches().get(tp1);
             assertEquals(1, partitionBatches.size());
 
@@ -118,7 +119,7 @@ public void testFull() throws Exception {
 
         // this append doesn't fit in the first batch, so a new batch is created and the first batch is closed
 
-        accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
+        accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
         Deque partitionBatches = accum.batches().get(tp1);
         assertEquals(2, partitionBatches.size());
         Iterator partitionBatchesIterator = partitionBatches.iterator();
@@ -153,7 +154,7 @@ private void testAppendLarge(CompressionType compressionType) throws Exception {
         byte[] value = new byte[2 * batchSize];
         RecordAccumulator accum = createTestRecordAccumulator(
                 batchSize + DefaultRecordBatch.RECORD_BATCH_OVERHEAD, 10 * 1024, compressionType, 0);
-        accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
+        accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
         assertEquals("Our partition's leader should be ready", Collections.singleton(node1), accum.ready(cluster, time.milliseconds()).readyNodes);
 
         Deque batches = accum.batches().get(tp1);
@@ -192,7 +193,7 @@ private void testAppendLargeOldMessageFormat(CompressionType compressionType) th
 
         RecordAccumulator accum = createTestRecordAccumulator(
                 batchSize + DefaultRecordBatch.RECORD_BATCH_OVERHEAD, 10 * 1024, compressionType, 0);
-        accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
+        accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
         assertEquals("Our partition's leader should be ready", Collections.singleton(node1), accum.ready(cluster, time.milliseconds()).readyNodes);
 
         Deque batches = accum.batches().get(tp1);
@@ -216,7 +217,7 @@ public void testLinger() throws Exception {
         int lingerMs = 10;
         RecordAccumulator accum = createTestRecordAccumulator(
                 1024 + DefaultRecordBatch.RECORD_BATCH_OVERHEAD, 10 * 1024, CompressionType.NONE, lingerMs);
-        accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
+        accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
         assertEquals("No partitions should be ready", 0, accum.ready(cluster, time.milliseconds()).readyNodes.size());
         time.sleep(10);
         assertEquals("Our partition's leader should be ready", Collections.singleton(node1), accum.ready(cluster, time.milliseconds()).readyNodes);
@@ -239,7 +240,7 @@ public void testPartialDrain() throws Exception {
         List partitions = asList(tp1, tp2);
         for (TopicPartition tp : partitions) {
             for (int i = 0; i < appends; i++)
-                accum.append(tp, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
+                accum.append(tp, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
         }
         assertEquals("Partition's leader should be ready", Collections.singleton(node1), accum.ready(cluster, time.milliseconds()).readyNodes);
 
@@ -261,7 +262,7 @@ public void testStressfulSituation() throws Exception {
                 public void run() {
                     for (int i = 0; i < msgs; i++) {
                         try {
-                            accum.append(new TopicPartition(topic, i % numParts), 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
+                            accum.append(new TopicPartition(topic, i % numParts), 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
                         } catch (Exception e) {
                             e.printStackTrace();
                         }
@@ -305,7 +306,7 @@ public void testNextReadyCheckDelay() throws Exception {
 
         // Partition on node1 only
         for (int i = 0; i < appends; i++)
-            accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
+            accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
         RecordAccumulator.ReadyCheckResult result = accum.ready(cluster, time.milliseconds());
         assertEquals("No nodes should be ready.", 0, result.readyNodes.size());
         assertEquals("Next check time should be the linger time", lingerMs, result.nextReadyCheckDelayMs);
@@ -314,14 +315,14 @@ public void testNextReadyCheckDelay() throws Exception {
 
         // Add partition on node2 only
         for (int i = 0; i < appends; i++)
-            accum.append(tp3, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
+            accum.append(tp3, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
         result = accum.ready(cluster, time.milliseconds());
         assertEquals("No nodes should be ready.", 0, result.readyNodes.size());
         assertEquals("Next check time should be defined by node1, half remaining linger time", lingerMs / 2, result.nextReadyCheckDelayMs);
 
         // Add data for another partition on node1, enough to make data sendable immediately
         for (int i = 0; i < appends + 1; i++)
-            accum.append(tp2, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
+            accum.append(tp2, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
         result = accum.ready(cluster, time.milliseconds());
         assertEquals("Node1 should be ready", Collections.singleton(node1), result.readyNodes);
         // Note this can actually be < linger time because it may use delays from partitions that aren't sendable
@@ -343,7 +344,7 @@ CompressionType.NONE, lingerMs, retryBackoffMs, deliveryTimeoutMs, metrics, metr
             new BufferPool(totalSize, batchSize, metrics, time, metricGrpName));
 
         long now = time.milliseconds();
-        accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
+        accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
         RecordAccumulator.ReadyCheckResult result = accum.ready(cluster, now + lingerMs + 1);
         assertEquals("Node1 should be ready", Collections.singleton(node1), result.readyNodes);
         Map> batches = accum.drain(cluster, result.readyNodes, Integer.MAX_VALUE, now + lingerMs + 1);
@@ -355,7 +356,7 @@ CompressionType.NONE, lingerMs, retryBackoffMs, deliveryTimeoutMs, metrics, metr
         accum.reenqueue(batches.get(0).get(0), now);
 
         // Put message for partition 1 into accumulator
-        accum.append(tp2, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
+        accum.append(tp2, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
         result = accum.ready(cluster, now + lingerMs + 1);
         assertEquals("Node1 should be ready", Collections.singleton(node1), result.readyNodes);
 
@@ -381,7 +382,7 @@ public void testFlush() throws Exception {
                 4 * 1024 + DefaultRecordBatch.RECORD_BATCH_OVERHEAD, 64 * 1024, CompressionType.NONE, lingerMs);
 
         for (int i = 0; i < 100; i++) {
-            accum.append(new TopicPartition(topic, i % 3), 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
+            accum.append(new TopicPartition(topic, i % 3), 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
             assertTrue(accum.hasIncomplete());
         }
         RecordAccumulator.ReadyCheckResult result = accum.ready(cluster, time.milliseconds());
@@ -419,7 +420,7 @@ public void run() {
     public void testAwaitFlushComplete() throws Exception {
         RecordAccumulator accum = createTestRecordAccumulator(
             4 * 1024 + DefaultRecordBatch.RECORD_BATCH_OVERHEAD, 64 * 1024, CompressionType.NONE, Integer.MAX_VALUE);
-        accum.append(new TopicPartition(topic, 0), 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
+        accum.append(new TopicPartition(topic, 0), 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
 
         accum.beginFlush();
         assertTrue(accum.flushInProgress());
@@ -448,7 +449,7 @@ public void onCompletion(RecordMetadata metadata, Exception exception) {
             }
         }
         for (int i = 0; i < numRecords; i++)
-            accum.append(new TopicPartition(topic, i % 3), 0L, key, value, null, new TestCallback(), maxBlockTimeMs);
+            accum.append(new TopicPartition(topic, i % 3), 0L, key, value, null, new TestCallback(), maxBlockTimeMs, false);
         RecordAccumulator.ReadyCheckResult result = accum.ready(cluster, time.milliseconds());
         assertFalse(result.readyNodes.isEmpty());
         Map> drained = accum.drain(cluster, result.readyNodes, Integer.MAX_VALUE, time.milliseconds());
@@ -489,7 +490,7 @@ public void onCompletion(RecordMetadata metadata, Exception exception) {
             }
         }
         for (int i = 0; i < numRecords; i++)
-            accum.append(new TopicPartition(topic, i % 3), 0L, key, value, null, new TestCallback(), maxBlockTimeMs);
+            accum.append(new TopicPartition(topic, i % 3), 0L, key, value, null, new TestCallback(), maxBlockTimeMs, false);
         RecordAccumulator.ReadyCheckResult result = accum.ready(cluster, time.milliseconds());
         assertFalse(result.readyNodes.isEmpty());
         Map> drained = accum.drain(cluster, result.readyNodes, Integer.MAX_VALUE,
@@ -528,7 +529,7 @@ private void doExpireBatchSingle(int deliveryTimeoutMs) throws InterruptedExcept
         for (Boolean mute: muteStates) {
             if (time.milliseconds() < System.currentTimeMillis())
                 time.setCurrentTimeMs(System.currentTimeMillis());
-            accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
+            accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
             assertEquals("No partition should be ready.", 0, accum.ready(cluster, time.milliseconds()).readyNodes.size());
 
             time.sleep(lingerMs);
@@ -577,11 +578,11 @@ public void testExpiredBatches() throws InterruptedException {
 
         // Test batches not in retry
         for (int i = 0; i < appends; i++) {
-            accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
+            accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
             assertEquals("No partitions should be ready.", 0, accum.ready(cluster, time.milliseconds()).readyNodes.size());
         }
         // Make the batches ready due to batch full
-        accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, 0);
+        accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, 0, false);
         Set readyNodes = accum.ready(cluster, time.milliseconds()).readyNodes;
         assertEquals("Our partition's leader should be ready", Collections.singleton(node1), readyNodes);
         // Advance the clock to expire the batch.
@@ -611,7 +612,7 @@ public void testExpiredBatches() throws InterruptedException {
 
         // Test batches in retry.
         // Create a retried batch
-        accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, 0);
+        accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, 0, false);
         time.sleep(lingerMs);
         readyNodes = accum.ready(cluster, time.milliseconds()).readyNodes;
         assertEquals("Our partition's leader should be ready", Collections.singleton(node1), readyNodes);
@@ -635,7 +636,7 @@ public void testExpiredBatches() throws InterruptedException {
         assertEquals("All batches should have been expired.", 0, expiredBatches.size());
 
         // Test that when being throttled muted batches are expired before the throttle time is over.
-        accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, 0);
+        accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, 0, false);
         time.sleep(lingerMs);
         readyNodes = accum.ready(cluster, time.milliseconds()).readyNodes;
         assertEquals("Our partition's leader should be ready", Collections.singleton(node1), readyNodes);
@@ -668,7 +669,7 @@ public void testMutedPartitions() throws InterruptedException {
                 batchSize + DefaultRecordBatch.RECORD_BATCH_OVERHEAD, 10 * batchSize, CompressionType.NONE, 10);
         int appends = expectedNumAppends(batchSize);
         for (int i = 0; i < appends; i++) {
-            accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
+            accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
             assertEquals("No partitions should be ready.", 0, accum.ready(cluster, now).readyNodes.size());
         }
         time.sleep(2000);
@@ -710,7 +711,7 @@ public void testIdempotenceWithOldMagic() throws InterruptedException {
         RecordAccumulator accum = new RecordAccumulator(logContext, batchSize + DefaultRecordBatch.RECORD_BATCH_OVERHEAD,
             CompressionType.NONE, lingerMs, retryBackoffMs, deliveryTimeoutMs, metrics, metricGrpName, time, apiVersions, new TransactionManager(),
             new BufferPool(totalSize, batchSize, metrics, time, metricGrpName));
-        accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, 0);
+        accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, 0, false);
     }
 
     @Test
@@ -810,7 +811,7 @@ public void testSplitFrequency() throws InterruptedException {
                 int dice = random.nextInt(100);
                 byte[] value = (dice < goodCompRatioPercentage) ?
                         bytesWithGoodCompression(random) : bytesWithPoorCompression(random, 100);
-                accum.append(tp1, 0L, null, value, Record.EMPTY_HEADERS, null, 0);
+                accum.append(tp1, 0L, null, value, Record.EMPTY_HEADERS, null, 0, false);
                 BatchDrainedResult result = completeOrSplitBatches(accum, batchSize);
                 numSplit += result.numSplit;
                 numBatches += result.numBatches;
@@ -833,7 +834,7 @@ public void testSoonToExpireBatchesArePickedUpForExpiry() throws InterruptedExce
         RecordAccumulator accum = createTestRecordAccumulator(
             batchSize + DefaultRecordBatch.RECORD_BATCH_OVERHEAD, 10 * batchSize, CompressionType.NONE, lingerMs);
 
-        accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
+        accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
         Set readyNodes = accum.ready(cluster, time.milliseconds()).readyNodes;
         Map> drained = accum.drain(cluster, readyNodes, Integer.MAX_VALUE, time.milliseconds());
         assertTrue(drained.isEmpty());
@@ -848,7 +849,7 @@ public void testSoonToExpireBatchesArePickedUpForExpiry() throws InterruptedExce
         //assertTrue(accum.soonToExpireInFlightBatches().isEmpty());
 
         // Queue another batch and advance clock such that batch expiry time is earlier than request timeout.
-        accum.append(tp2, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
+        accum.append(tp2, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
         time.sleep(lingerMs * 4);
 
         // Now drain and check that accumulator picked up the drained batch because its expiry is soon.
@@ -872,8 +873,8 @@ public void testExpiredBatchesRetry() throws InterruptedException {
             batchSize + DefaultRecordBatch.RECORD_BATCH_OVERHEAD, 10 * batchSize, CompressionType.NONE, lingerMs);
 
         // Test batches in retry.
-        for (Boolean mute: muteStates) {
-            accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, 0);
+        for (Boolean mute : muteStates) {
+            accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, 0, false);
             time.sleep(lingerMs);
             readyNodes = accum.ready(cluster, time.milliseconds()).readyNodes;
             assertEquals("Our partition's leader should be ready", Collections.singleton(node1), readyNodes);
@@ -891,8 +892,82 @@ public void testExpiredBatchesRetry() throws InterruptedException {
             time.sleep(deliveryTimeoutMs - rtt);
             accum.drain(cluster, Collections.singleton(node1), Integer.MAX_VALUE, time.milliseconds());
             expiredBatches = accum.expiredBatches(time.milliseconds());
-            assertEquals("RecordAccumulator has expired batches if the partition is not muted", mute  ? 1 : 0, expiredBatches.size());
+            assertEquals("RecordAccumulator has expired batches if the partition is not muted", mute ? 1 : 0, expiredBatches.size());
+        }
+    }
+        
+    @Test
+    public void testStickyBatches() throws Exception {
+        long now = time.milliseconds();
+
+        // Test case assumes that the records do not fill the batch completely
+        int batchSize = 1025;
+
+        Partitioner partitioner = new DefaultPartitioner();
+        RecordAccumulator accum = createTestRecordAccumulator(3200,
+                batchSize + DefaultRecordBatch.RECORD_BATCH_OVERHEAD, 10L * batchSize, CompressionType.NONE, 10);
+        int expectedAppends = expectedNumAppendsNoKey(batchSize);
+        
+        // Create first batch
+        int partition = partitioner.partition(topic, null, null, "value", value, cluster);
+        TopicPartition tp = new TopicPartition(topic, partition);
+        accum.append(tp, 0L, null, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
+        int appends = 1;
+        
+        boolean switchPartition = false;
+        while (!switchPartition) {
+            // Append to the first batch
+            partition = partitioner.partition(topic, null, null, "value", value, cluster);
+            tp = new TopicPartition(topic, partition);
+            RecordAccumulator.RecordAppendResult result = accum.append(tp, 0L, null, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, true);
+            Deque partitionBatches1 = accum.batches().get(tp1);
+            Deque partitionBatches2 = accum.batches().get(tp2);
+            Deque partitionBatches3 = accum.batches().get(tp3);
+            int numBatches = (partitionBatches1 == null ? 0 : partitionBatches1.size()) + (partitionBatches2 == null ? 0 : partitionBatches2.size()) + (partitionBatches3 == null ? 0 : partitionBatches3.size());
+            // Only one batch is created because the partition is sticky.
+            assertEquals(1, numBatches);
+
+            switchPartition = result.abortForNewBatch;
+            // We only appended if we do not retry.
+            if (!switchPartition) {
+                appends++;
+                assertEquals("No partitions should be ready.", 0, accum.ready(cluster, now).readyNodes.size());
+            }
+        }
+        
+        // Batch should be full.
+        assertEquals(1, accum.ready(cluster, time.milliseconds()).readyNodes.size());
+        assertEquals(appends, expectedAppends);
+        switchPartition = false;
+        
+        // KafkaProducer would call this method in this case, make second batch
+        partitioner.onNewBatch(topic, cluster, partition);
+        partition = partitioner.partition(topic, null, null, "value", value, cluster);
+        tp = new TopicPartition(topic, partition);
+        accum.append(tp, 0L, null, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
+        appends++;
+        
+        // These appends all go into the second batch
+        while (!switchPartition) {
+            partition = partitioner.partition(topic, null, null, "value", value, cluster);
+            tp = new TopicPartition(topic, partition);
+            RecordAccumulator.RecordAppendResult result = accum.append(tp, 0L, null, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, true);
+            Deque partitionBatches1 = accum.batches().get(tp1);
+            Deque partitionBatches2 = accum.batches().get(tp2);
+            Deque partitionBatches3 = accum.batches().get(tp3);
+            int numBatches = (partitionBatches1 == null ? 0 : partitionBatches1.size()) + (partitionBatches2 == null ? 0 : partitionBatches2.size()) + (partitionBatches3 == null ? 0 : partitionBatches3.size());
+            // Only two batches because the new partition is also sticky.
+            assertEquals(2, numBatches);
+            
+            switchPartition = result.abortForNewBatch;
+            // We only appended if we do not retry.
+            if (!switchPartition) {
+                appends++;
+            }
         }
+        
+        // There should be two full batches now.
+        assertEquals(appends, 2 * expectedAppends);
     }
 
     private int prepareSplitBatches(RecordAccumulator accum, long seed, int recordSize, int numRecords)
@@ -904,7 +979,7 @@ private int prepareSplitBatches(RecordAccumulator accum, long seed, int recordSi
         CompressionRatioEstimator.setEstimation(tp1.topic(), CompressionType.GZIP, 0.1f);
         // Append 20 records of 100 bytes size with poor compression ratio should make the batch too big.
         for (int i = 0; i < numRecords; i++) {
-            accum.append(tp1, 0L, null, bytesWithPoorCompression(random, recordSize), Record.EMPTY_HEADERS, null, 0);
+            accum.append(tp1, 0L, null, bytesWithPoorCompression(random, recordSize), Record.EMPTY_HEADERS, null, 0, false);
         }
 
         RecordAccumulator.ReadyCheckResult result = accum.ready(cluster, time.milliseconds());
@@ -989,6 +1064,22 @@ private int expectedNumAppends(int batchSize) {
             size += recordSize;
         }
     }
+    
+     /**
+     * Return the offset delta when there is no key.
+     */
+    private int expectedNumAppendsNoKey(int batchSize) {
+        int size = 0;
+        int offsetDelta = 0;
+        while (true) {
+            int recordSize = DefaultRecord.sizeInBytes(offsetDelta, 0, 0, value.length,
+                Record.EMPTY_HEADERS);
+            if (size + recordSize > batchSize)
+                return offsetDelta;
+            offsetDelta += 1;
+            size += recordSize;
+        }
+    }
 
 
     private RecordAccumulator createTestRecordAccumulator(int batchSize, long totalSize, CompressionType type, int lingerMs) {
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java
index 194176d8e2235..56308c1e38a09 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java
@@ -153,7 +153,7 @@ public void tearDown() {
     public void testSimple() throws Exception {
         long offset = 0;
         Future future = accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(),
-                null, null, MAX_BLOCK_TIMEOUT).future;
+                null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce(); // connect
         sender.runOnce(); // send produce request
         assertEquals("We should have a single produce request in flight.", 1, client.inFlightRequestCount());
@@ -180,7 +180,7 @@ public void testMessageFormatDownConversion() throws Exception {
         apiVersions.update("0", NodeApiVersions.create());
 
         Future future = accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(),
-                null, null, MAX_BLOCK_TIMEOUT).future;
+                null, null, MAX_BLOCK_TIMEOUT, false).future;
 
         // now the partition leader supports only v2
         apiVersions.update("0", NodeApiVersions.create(Collections.singleton(
@@ -220,14 +220,14 @@ public void testDownConversionForMismatchedMagicValues() throws Exception {
         apiVersions.update("0", NodeApiVersions.create());
 
         Future future1 = accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(),
-                null, null, MAX_BLOCK_TIMEOUT).future;
+                null, null, MAX_BLOCK_TIMEOUT, false).future;
 
         // now the partition leader supports only v2
         apiVersions.update("0", NodeApiVersions.create(Collections.singleton(
                 new ApiVersionsResponse.ApiVersion(ApiKeys.PRODUCE, (short) 0, (short) 2))));
 
         Future future2 = accumulator.append(tp1, 0L, "key".getBytes(), "value".getBytes(),
-                null, null, MAX_BLOCK_TIMEOUT).future;
+                null, null, MAX_BLOCK_TIMEOUT, false).future;
 
         // start off support produce request v3
         apiVersions.update("0", NodeApiVersions.create());
@@ -321,7 +321,7 @@ public void testSenderMetricsTemplates() throws Exception {
                 1, metricsRegistry, time, REQUEST_TIMEOUT, RETRY_BACKOFF_MS, null, apiVersions);
 
         // Append a message so that topic metrics are created
-        accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT);
+        accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false);
         sender.runOnce(); // connect
         sender.runOnce(); // send produce request
         client.respond(produceResponse(tp0, 0, Errors.NONE, 0));
@@ -348,7 +348,7 @@ public void testRetries() throws Exception {
             Sender sender = new Sender(logContext, client, metadata, this.accumulator, false, MAX_REQUEST_SIZE, ACKS_ALL,
                     maxRetries, senderMetrics, time, REQUEST_TIMEOUT, RETRY_BACKOFF_MS, null, apiVersions);
             // do a successful retry
-            Future future = accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+            Future future = accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
             sender.runOnce(); // connect
             sender.runOnce(); // send produce request
             String id = client.requests().peek().destination();
@@ -377,7 +377,7 @@ public void testRetries() throws Exception {
             assertEquals(0, sender.inFlightBatches(tp0).size());
 
             // do an unsuccessful retry
-            future = accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+            future = accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
             sender.runOnce(); // send produce request
             assertEquals(1, sender.inFlightBatches(tp0).size());
             for (int i = 0; i < maxRetries + 1; i++) {
@@ -411,7 +411,7 @@ public void testSendInOrder() throws Exception {
 
             // Send the first message.
             TopicPartition tp2 = new TopicPartition("test", 1);
-            accumulator.append(tp2, 0L, "key1".getBytes(), "value1".getBytes(), null, null, MAX_BLOCK_TIMEOUT);
+            accumulator.append(tp2, 0L, "key1".getBytes(), "value1".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false);
             sender.runOnce(); // connect
             sender.runOnce(); // send produce request
             String id = client.requests().peek().destination();
@@ -424,7 +424,7 @@ public void testSendInOrder() throws Exception {
 
             time.sleep(900);
             // Now send another message to tp2
-            accumulator.append(tp2, 0L, "key2".getBytes(), "value2".getBytes(), null, null, MAX_BLOCK_TIMEOUT);
+            accumulator.append(tp2, 0L, "key2".getBytes(), "value2".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false);
 
             // Update metadata before sender receives response from broker 0. Now partition 2 moves to broker 0
             MetadataResponse metadataUpdate2 = TestUtils.metadataUpdateWith(1, Collections.singletonMap("test", 2));
@@ -454,7 +454,7 @@ public void onCompletion(RecordMetadata metadata, Exception exception) {
                 if (exception instanceof TimeoutException) {
                     expiryCallbackCount.incrementAndGet();
                     try {
-                        accumulator.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
+                        accumulator.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
                     } catch (InterruptedException e) {
                         throw new RuntimeException("Unexpected interruption", e);
                     }
@@ -464,7 +464,7 @@ public void onCompletion(RecordMetadata metadata, Exception exception) {
         };
 
         for (int i = 0; i < messagesPerBatch; i++)
-            accumulator.append(tp1, 0L, key, value, null, callback, maxBlockTimeMs);
+            accumulator.append(tp1, 0L, key, value, null, callback, maxBlockTimeMs, false);
 
         // Advance the clock to expire the first batch.
         time.sleep(10000);
@@ -497,7 +497,7 @@ public void testMetadataTopicExpiry() throws Exception {
         long offset = 0;
         client.updateMetadata(TestUtils.metadataUpdateWith(1, Collections.singletonMap("test", 2)));
 
-        Future future = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future future = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         assertTrue("Topic not added to metadata", metadata.containsTopic(tp0.topic()));
         client.updateMetadata(TestUtils.metadataUpdateWith(1, Collections.singletonMap("test", 2)));
@@ -514,7 +514,7 @@ public void testMetadataTopicExpiry() throws Exception {
         time.sleep(ProducerMetadata.TOPIC_EXPIRY_MS);
         client.updateMetadata(TestUtils.metadataUpdateWith(1, Collections.singletonMap("test", 2)));
         assertFalse("Unused topic has not been expired", metadata.containsTopic(tp0.topic()));
-        future = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        future = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         assertTrue("Topic not added to metadata", metadata.containsTopic(tp0.topic()));
         client.updateMetadata(TestUtils.metadataUpdateWith(1, Collections.singletonMap("test", 2)));
@@ -556,7 +556,7 @@ public void testClusterAuthorizationExceptionInInitProducerIdRequest() throws Ex
     @Test
     public void testCanRetryWithoutIdempotence() throws Exception {
         // do a successful retry
-        Future future = accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future future = accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce(); // connect
         sender.runOnce(); // send produce request
         String id = client.requests().peek().destination();
@@ -595,7 +595,7 @@ public void testIdempotenceWithMultipleInflights() throws Exception {
         assertEquals(0, transactionManager.sequenceNumber(tp0).longValue());
 
         // Send first ProduceRequest
-        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         String nodeId = client.requests().peek().destination();
         Node node = new Node(Integer.valueOf(nodeId), "localhost", 0);
@@ -604,7 +604,7 @@ public void testIdempotenceWithMultipleInflights() throws Exception {
         assertEquals(OptionalInt.empty(), transactionManager.lastAckedSequence(tp0));
 
         // Send second ProduceRequest
-        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         assertEquals(2, client.inFlightRequestCount());
         assertEquals(2, transactionManager.sequenceNumber(tp0).longValue());
@@ -645,7 +645,7 @@ public void testIdempotenceWithMultipleInflightsRetriedInOrder() throws Exceptio
         assertEquals(0, transactionManager.sequenceNumber(tp0).longValue());
 
         // Send first ProduceRequest
-        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         String nodeId = client.requests().peek().destination();
         Node node = new Node(Integer.valueOf(nodeId), "localhost", 0);
@@ -654,11 +654,11 @@ public void testIdempotenceWithMultipleInflightsRetriedInOrder() throws Exceptio
         assertEquals(OptionalInt.empty(), transactionManager.lastAckedSequence(tp0));
 
         // Send second ProduceRequest
-        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
 
          // Send third ProduceRequest
-        Future request3 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request3 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
 
         assertEquals(3, client.inFlightRequestCount());
@@ -673,7 +673,7 @@ public void testIdempotenceWithMultipleInflightsRetriedInOrder() throws Exceptio
         sender.runOnce(); // receive response 0
 
         // Queue the fourth request, it shouldn't be sent until the first 3 complete.
-        Future request4 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request4 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
 
         assertEquals(2, client.inFlightRequestCount());
         assertEquals(OptionalInt.empty(), transactionManager.lastAckedSequence(tp0));
@@ -745,7 +745,7 @@ public void testIdempotenceWithMultipleInflightsWhereFirstFailsFatallyAndSequenc
         assertEquals(0, transactionManager.sequenceNumber(tp0).longValue());
 
         // Send first ProduceRequest
-        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         String nodeId = client.requests().peek().destination();
         Node node = new Node(Integer.valueOf(nodeId), "localhost", 0);
@@ -754,7 +754,7 @@ public void testIdempotenceWithMultipleInflightsWhereFirstFailsFatallyAndSequenc
         assertEquals(OptionalInt.empty(), transactionManager.lastAckedSequence(tp0));
 
         // Send second ProduceRequest
-        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         assertEquals(2, client.inFlightRequestCount());
         assertEquals(2, transactionManager.sequenceNumber(tp0).longValue());
@@ -804,8 +804,8 @@ public void testMustNotRetryOutOfOrderSequenceForNextBatch() throws Exception {
         assertEquals(0, transactionManager.sequenceNumber(tp0).longValue());
 
         // Send first ProduceRequest with multiple messages.
-        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
-        accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT);
+        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
+        accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false);
         sender.runOnce();
         String nodeId = client.requests().peek().destination();
         Node node = new Node(Integer.valueOf(nodeId), "localhost", 0);
@@ -819,7 +819,7 @@ public void testMustNotRetryOutOfOrderSequenceForNextBatch() throws Exception {
         sender.runOnce();
 
         // Send second ProduceRequest
-        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         assertEquals(1, client.inFlightRequestCount());
         assertEquals(3, transactionManager.sequenceNumber(tp0).longValue());
@@ -846,7 +846,7 @@ public void testCorrectHandlingOfOutOfOrderResponses() throws Exception {
         assertEquals(0, transactionManager.sequenceNumber(tp0).longValue());
 
         // Send first ProduceRequest
-        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         String nodeId = client.requests().peek().destination();
         Node node = new Node(Integer.valueOf(nodeId), "localhost", 0);
@@ -855,7 +855,7 @@ public void testCorrectHandlingOfOutOfOrderResponses() throws Exception {
         assertEquals(OptionalInt.empty(), transactionManager.lastAckedSequence(tp0));
 
         // Send second ProduceRequest
-        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         assertEquals(2, client.inFlightRequestCount());
         assertEquals(2, transactionManager.sequenceNumber(tp0).longValue());
@@ -928,14 +928,14 @@ public void testCorrectHandlingOfOutOfOrderResponsesWhenSecondSucceeds() throws
         assertEquals(0, transactionManager.sequenceNumber(tp0).longValue());
 
         // Send first ProduceRequest
-        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         String nodeId = client.requests().peek().destination();
         Node node = new Node(Integer.valueOf(nodeId), "localhost", 0);
         assertEquals(1, client.inFlightRequestCount());
 
         // Send second ProduceRequest
-        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         assertEquals(2, client.inFlightRequestCount());
         assertFalse(request1.isDone());
@@ -996,7 +996,7 @@ public void testExpiryOfUnsentBatchesShouldNotCauseUnresolvedSequences() throws
         assertEquals(0, transactionManager.sequenceNumber(tp0).longValue());
 
         // Send first ProduceRequest
-        Future request1 = accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request1 = accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         Node node = metadata.fetch().nodes().get(0);
         time.sleep(10000L);
         client.disconnect(node.idString());
@@ -1018,13 +1018,13 @@ public void testExpiryOfFirstBatchShouldNotCauseUnresolvedSequencesIfFutureBatch
         assertEquals(0, transactionManager.sequenceNumber(tp0).longValue());
 
         // Send first ProduceRequest
-        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();  // send request
         // We separate the two appends by 1 second so that the two batches
         // don't expire at the same time.
         time.sleep(1000L);
 
-        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();  // send request
         assertEquals(2, client.inFlightRequestCount());
         assertEquals(2, sender.inFlightBatches(tp0).size());
@@ -1046,7 +1046,7 @@ public void testExpiryOfFirstBatchShouldNotCauseUnresolvedSequencesIfFutureBatch
         assertEquals(0, sender.inFlightBatches(tp0).size());
 
         // let's enqueue another batch, which should not be dequeued until the unresolved state is clear.
-        Future request3 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request3 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         time.sleep(20);
         assertFalse(request2.isDone());
 
@@ -1085,11 +1085,11 @@ public void testExpiryOfFirstBatchShouldCauseResetIfFutureBatchesFail() throws E
         assertEquals(0, transactionManager.sequenceNumber(tp0).longValue());
 
         // Send first ProduceRequest
-        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();  // send request
 
         time.sleep(1000L);
-        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();  // send request
 
         assertEquals(2, client.inFlightRequestCount());
@@ -1106,7 +1106,7 @@ public void testExpiryOfFirstBatchShouldCauseResetIfFutureBatchesFail() throws E
         assertFutureFailure(request1, TimeoutException.class);
         assertTrue(transactionManager.hasUnresolvedSequence(tp0));
         // let's enqueue another batch, which should not be dequeued until the unresolved state is clear.
-        Future request3 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request3 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
 
         time.sleep(20);
         assertFalse(request2.isDone());
@@ -1138,7 +1138,7 @@ public void testExpiryOfAllSentBatchesShouldCauseUnresolvedSequences() throws Ex
         assertEquals(0, transactionManager.sequenceNumber(tp0).longValue());
 
         // Send first ProduceRequest
-        Future request1 = accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request1 = accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();  // send request
         sendIdempotentProducerResponse(0, tp0, Errors.NOT_LEADER_FOR_PARTITION, -1);
 
@@ -1178,9 +1178,9 @@ public void testResetOfProducerStateShouldAllowQueuedBatchesToDrain() throws Exc
                 senderMetrics, time, REQUEST_TIMEOUT, RETRY_BACKOFF_MS, transactionManager, apiVersions);
 
         Future failedResponse = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         Future successfulResponse = accumulator.append(tp1, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();  // connect and send.
 
         assertEquals(1, client.inFlightRequestCount());
@@ -1220,9 +1220,9 @@ public void testCloseWithProducerIdReset() throws Exception {
             senderMetrics, time, REQUEST_TIMEOUT, RETRY_BACKOFF_MS, transactionManager, apiVersions);
 
         Future failedResponse = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-            "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+            "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         Future successfulResponse = accumulator.append(tp1, time.milliseconds(), "key".getBytes(),
-            "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+            "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();  // connect and send.
 
         assertEquals(1, client.inFlightRequestCount());
@@ -1259,9 +1259,9 @@ public void testForceCloseWithProducerIdReset() throws Exception {
             senderMetrics, time, REQUEST_TIMEOUT, RETRY_BACKOFF_MS, transactionManager, apiVersions);
 
         Future failedResponse = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-            "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+            "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         Future successfulResponse = accumulator.append(tp1, time.milliseconds(), "key".getBytes(),
-            "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+            "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();  // connect and send.
 
         assertEquals(1, client.inFlightRequestCount());
@@ -1295,9 +1295,9 @@ public void testBatchesDrainedWithOldProducerIdShouldFailWithOutOfOrderSequenceO
                 senderMetrics, time, REQUEST_TIMEOUT, RETRY_BACKOFF_MS, transactionManager, apiVersions);
 
         Future failedResponse = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         Future successfulResponse = accumulator.append(tp1, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();  // connect.
         sender.runOnce();  // send.
 
@@ -1341,7 +1341,7 @@ public void testCorrectHandlingOfDuplicateSequenceError() throws Exception {
         assertEquals(0, transactionManager.sequenceNumber(tp0).longValue());
 
         // Send first ProduceRequest
-        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         String nodeId = client.requests().peek().destination();
         Node node = new Node(Integer.valueOf(nodeId), "localhost", 0);
@@ -1350,7 +1350,7 @@ public void testCorrectHandlingOfDuplicateSequenceError() throws Exception {
         assertEquals(OptionalInt.empty(), transactionManager.lastAckedSequence(tp0));
 
         // Send second ProduceRequest
-        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         assertEquals(2, client.inFlightRequestCount());
         assertEquals(2, transactionManager.sequenceNumber(tp0).longValue());
@@ -1394,7 +1394,7 @@ public void testUnknownProducerHandlingWhenRetentionLimitReached() throws Except
         assertEquals(0, transactionManager.sequenceNumber(tp0).longValue());
 
         // Send first ProduceRequest
-        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
 
         assertEquals(1, client.inFlightRequestCount());
@@ -1411,8 +1411,8 @@ public void testUnknownProducerHandlingWhenRetentionLimitReached() throws Except
         assertEquals(OptionalLong.of(1000L), transactionManager.lastAckedOffset(tp0));
 
         // Send second ProduceRequest, a single batch with 2 records.
-        accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT);
-        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false);
+        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         assertEquals(3, transactionManager.sequenceNumber(tp0).longValue());
         assertEquals(OptionalInt.of(0), transactionManager.lastAckedSequence(tp0));
@@ -1452,7 +1452,7 @@ public void testUnknownProducerErrorShouldBeRetriedWhenLogStartOffsetIsUnknown()
         assertEquals(0, transactionManager.sequenceNumber(tp0).longValue());
 
         // Send first ProduceRequest
-        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
 
         assertEquals(1, client.inFlightRequestCount());
@@ -1469,7 +1469,7 @@ public void testUnknownProducerErrorShouldBeRetriedWhenLogStartOffsetIsUnknown()
         assertEquals(OptionalLong.of(1000L), transactionManager.lastAckedOffset(tp0));
 
         // Send second ProduceRequest
-        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         assertEquals(2, transactionManager.sequenceNumber(tp0).longValue());
         assertEquals(OptionalInt.of(0), transactionManager.lastAckedSequence(tp0));
@@ -1510,7 +1510,7 @@ public void testUnknownProducerErrorShouldBeRetriedForFutureBatchesWhenFirstFail
         assertEquals(0, transactionManager.sequenceNumber(tp0).longValue());
 
         // Send first ProduceRequest
-        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
 
         assertEquals(1, client.inFlightRequestCount());
@@ -1527,14 +1527,14 @@ public void testUnknownProducerErrorShouldBeRetriedForFutureBatchesWhenFirstFail
         assertEquals(OptionalLong.of(1000L), transactionManager.lastAckedOffset(tp0));
 
         // Send second ProduceRequest
-        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         assertEquals(2, transactionManager.sequenceNumber(tp0).longValue());
         assertEquals(OptionalInt.of(0), transactionManager.lastAckedSequence(tp0));
 
         // Send the third ProduceRequest, in parallel with the second. It should be retried even though the
         // lastAckedOffset > logStartOffset when its UnknownProducerResponse comes back.
-        Future request3 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request3 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         assertEquals(3, transactionManager.sequenceNumber(tp0).longValue());
         assertEquals(OptionalInt.of(0), transactionManager.lastAckedSequence(tp0));
@@ -1599,7 +1599,7 @@ public void testShouldRaiseOutOfOrderSequenceExceptionToUserIfLogWasNotTruncated
         assertEquals(0, transactionManager.sequenceNumber(tp0).longValue());
 
         // Send first ProduceRequest
-        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
 
         assertEquals(1, client.inFlightRequestCount());
@@ -1616,7 +1616,7 @@ public void testShouldRaiseOutOfOrderSequenceExceptionToUserIfLogWasNotTruncated
         assertEquals(OptionalLong.of(1000L), transactionManager.lastAckedOffset(tp0));
 
         // Send second ProduceRequest,
-        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         assertEquals(2, transactionManager.sequenceNumber(tp0).longValue());
         assertEquals(OptionalInt.of(0), transactionManager.lastAckedSequence(tp0));
@@ -1660,7 +1660,7 @@ public void testClusterAuthorizationExceptionInProduceRequest() throws Exception
 
         // cluster authorization is a fatal error for the producer
         Future future = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(),
-                null, null, MAX_BLOCK_TIMEOUT).future;
+                null, null, MAX_BLOCK_TIMEOUT, false).future;
         client.prepareResponse(new MockClient.RequestMatcher() {
             @Override
             public boolean matches(AbstractRequest body) {
@@ -1687,11 +1687,11 @@ public void testCancelInFlightRequestAfterFatalError() throws Exception {
 
         // cluster authorization is a fatal error for the producer
         Future future1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(),
-                null, null, MAX_BLOCK_TIMEOUT).future;
+                null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
 
         Future future2 = accumulator.append(tp1, time.milliseconds(), "key".getBytes(), "value".getBytes(),
-                null, null, MAX_BLOCK_TIMEOUT).future;
+                null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
 
         client.respond(new MockClient.RequestMatcher() {
@@ -1728,7 +1728,7 @@ public void testUnsupportedForMessageFormatInProduceRequest() throws Exception {
         assertTrue(transactionManager.hasProducerId());
 
         Future future = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(),
-                null, null, MAX_BLOCK_TIMEOUT).future;
+                null, null, MAX_BLOCK_TIMEOUT, false).future;
         client.prepareResponse(new MockClient.RequestMatcher() {
             @Override
             public boolean matches(AbstractRequest body) {
@@ -1753,7 +1753,7 @@ public void testUnsupportedVersionInProduceRequest() throws Exception {
         assertTrue(transactionManager.hasProducerId());
 
         Future future = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(),
-                null, null, MAX_BLOCK_TIMEOUT).future;
+                null, null, MAX_BLOCK_TIMEOUT, false).future;
         client.prepareUnsupportedVersionResponse(new MockClient.RequestMatcher() {
             @Override
             public boolean matches(AbstractRequest body) {
@@ -1783,7 +1783,7 @@ public void testSequenceNumberIncrement() throws InterruptedException {
         Sender sender = new Sender(logContext, client, metadata, this.accumulator, true, MAX_REQUEST_SIZE, ACKS_ALL, maxRetries,
                 senderMetrics, time, REQUEST_TIMEOUT, RETRY_BACKOFF_MS, transactionManager, apiVersions);
 
-        Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         client.prepareResponse(new MockClient.RequestMatcher() {
             @Override
             public boolean matches(AbstractRequest body) {
@@ -1825,7 +1825,7 @@ public void testAbortRetryWhenProducerIdChanges() throws InterruptedException {
         Sender sender = new Sender(logContext, client, metadata, this.accumulator, true, MAX_REQUEST_SIZE, ACKS_ALL, maxRetries,
                 senderMetrics, time, REQUEST_TIMEOUT, RETRY_BACKOFF_MS, transactionManager, apiVersions);
 
-        Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();  // connect.
         sender.runOnce();  // send.
         String id = client.requests().peek().destination();
@@ -1864,7 +1864,7 @@ public void testResetWhenOutOfOrderSequenceReceived() throws InterruptedExceptio
         Sender sender = new Sender(logContext, client, metadata, this.accumulator, true, MAX_REQUEST_SIZE, ACKS_ALL, maxRetries,
                 senderMetrics, time, REQUEST_TIMEOUT, RETRY_BACKOFF_MS, transactionManager, apiVersions);
 
-        Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();  // connect.
         sender.runOnce();  // send.
 
@@ -1898,6 +1898,7 @@ public void testTransactionalSplitBatchAndSend() throws Exception {
         doInitTransactions(txnManager, producerIdAndEpoch);
 
         txnManager.beginTransaction();
+        txnManager.failIfNotReadyForSend();
         txnManager.maybeAddPartitionToTransaction(tp);
         client.prepareResponse(new AddPartitionsToTxnResponse(0, Collections.singletonMap(tp, Errors.NONE)));
         sender.runOnce();
@@ -1927,9 +1928,9 @@ private void testSplitBatchAndSend(TransactionManager txnManager,
             client.prepareMetadataUpdate(metadataUpdate1);
             // Send the first message.
             Future f1 =
-                    accumulator.append(tp, 0L, "key1".getBytes(), new byte[batchSize / 2], null, null, MAX_BLOCK_TIMEOUT).future;
+                    accumulator.append(tp, 0L, "key1".getBytes(), new byte[batchSize / 2], null, null, MAX_BLOCK_TIMEOUT, false).future;
             Future f2 =
-                    accumulator.append(tp, 0L, "key2".getBytes(), new byte[batchSize / 2], null, null, MAX_BLOCK_TIMEOUT).future;
+                    accumulator.append(tp, 0L, "key2".getBytes(), new byte[batchSize / 2], null, null, MAX_BLOCK_TIMEOUT, false).future;
             sender.runOnce(); // connect
             sender.runOnce(); // send produce request
 
@@ -1999,7 +2000,7 @@ public void testNoDoubleDeallocation() throws Exception {
 
         // Send first ProduceRequest
         Future request1 =
-            accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+            accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();  // send request
         assertEquals(1, client.inFlightRequestCount());
         assertEquals(1, sender.inFlightBatches(tp0).size());
@@ -2024,7 +2025,7 @@ public void testInflightBatchesExpireOnDeliveryTimeout() throws InterruptedExcep
         setupWithTransactionState(null, true, null);
 
         // Send first ProduceRequest
-        Future request = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();  // send request
         assertEquals(1, client.inFlightRequestCount());
         assertEquals("Expect one in-flight batch in accumulator", 1, sender.inFlightBatches(tp0).size());
@@ -2050,7 +2051,7 @@ public void testWhenFirstBatchExpireNoSendSecondBatchIfGuaranteeOrder() throws I
         setupWithTransactionState(null, true, null);
 
         // Send first ProduceRequest
-        accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT);
+        accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false);
         sender.runOnce();  // send request
         assertEquals(1, client.inFlightRequestCount());
         assertEquals(1, sender.inFlightBatches(tp0).size());
@@ -2058,7 +2059,7 @@ public void testWhenFirstBatchExpireNoSendSecondBatchIfGuaranteeOrder() throws I
         time.sleep(deliveryTimeoutMs / 2);
 
         // Send second ProduceRequest
-        accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT);
+        accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false);
         sender.runOnce();  // must not send request because the partition is muted
         assertEquals(1, client.inFlightRequestCount());
         assertEquals(1, sender.inFlightBatches(tp0).size());
@@ -2083,7 +2084,7 @@ public void testExpiredBatchDoesNotRetry() throws Exception {
         // Send first ProduceRequest
         Future request1 =
             accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null,
-                MAX_BLOCK_TIMEOUT).future;
+                MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();  // send request
         assertEquals(1, client.inFlightRequestCount());
         time.sleep(deliverTimeoutMs);
@@ -2112,10 +2113,10 @@ public void testExpiredBatchDoesNotSplitOnMessageTooLargeError() throws Exceptio
         // create a producer batch with more than one record so it is eligible to split
         Future request1 =
             accumulator.append(tp0, time.milliseconds(), "key1".getBytes(), "value1".getBytes(), null, null,
-                MAX_BLOCK_TIMEOUT).future;
+                MAX_BLOCK_TIMEOUT, false).future;
         Future request2 =
             accumulator.append(tp0, time.milliseconds(), "key2".getBytes(), "value2".getBytes(), null, null,
-                MAX_BLOCK_TIMEOUT).future;
+                MAX_BLOCK_TIMEOUT, false).future;
 
         sender.runOnce();  // send request
         assertEquals(1, client.inFlightRequestCount());
@@ -2143,7 +2144,7 @@ public void testResetNextBatchExpiry() throws Exception {
         setupWithTransactionState(null);
 
         accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(), null, null,
-                MAX_BLOCK_TIMEOUT);
+                MAX_BLOCK_TIMEOUT, false);
 
         sender.runOnce();
         sender.runOnce();
@@ -2167,8 +2168,8 @@ public void testExpiredBatchesInMultiplePartitions() throws Exception {
         setupWithTransactionState(null, true, null);
 
         // Send multiple ProduceRequest across multiple partitions.
-        Future request1 = accumulator.append(tp0, time.milliseconds(), "k1".getBytes(), "v1".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
-        Future request2 = accumulator.append(tp1, time.milliseconds(), "k2".getBytes(), "v2".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
+        Future request1 = accumulator.append(tp0, time.milliseconds(), "k1".getBytes(), "v1".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
+        Future request2 = accumulator.append(tp1, time.milliseconds(), "k2".getBytes(), "v2".getBytes(), null, null, MAX_BLOCK_TIMEOUT, false).future;
 
         // Send request.
         sender.runOnce();
@@ -2217,6 +2218,7 @@ public void testTransactionalRequestsSentOnShutdown() {
             doInitTransactions(txnManager, producerIdAndEpoch);
 
             txnManager.beginTransaction();
+            txnManager.failIfNotReadyForSend();
             txnManager.maybeAddPartitionToTransaction(tp);
             client.prepareResponse(new AddPartitionsToTxnResponse(0, Collections.singletonMap(tp, Errors.NONE)));
             sender.runOnce();
@@ -2249,6 +2251,7 @@ public void testIncompleteTransactionAbortOnShutdown() {
             doInitTransactions(txnManager, producerIdAndEpoch);
 
             txnManager.beginTransaction();
+            txnManager.failIfNotReadyForSend();
             txnManager.maybeAddPartitionToTransaction(tp);
             client.prepareResponse(new AddPartitionsToTxnResponse(0, Collections.singletonMap(tp, Errors.NONE)));
             sender.runOnce();
@@ -2280,6 +2283,7 @@ public void testForceShutdownWithIncompleteTransaction() {
             doInitTransactions(txnManager, producerIdAndEpoch);
 
             txnManager.beginTransaction();
+            txnManager.failIfNotReadyForSend();
             txnManager.maybeAddPartitionToTransaction(tp);
             client.prepareResponse(new AddPartitionsToTxnResponse(0, Collections.singletonMap(tp, Errors.NONE)));
             sender.runOnce();
@@ -2442,7 +2446,7 @@ private void setupWithTransactionState(TransactionManager transactionManager, bo
 
     private void assertSendFailure(Class expectedError) throws Exception {
         Future future = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(),
-                null, null, MAX_BLOCK_TIMEOUT).future;
+                null, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         assertTrue(future.isDone());
         try {
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/StickyPartitionCacheTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/StickyPartitionCacheTest.java
new file mode 100644
index 0000000000000..6878ba1d6871a
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/StickyPartitionCacheTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.producer.internals;
+
+import org.apache.kafka.common.Cluster;
+import org.apache.kafka.common.Node;
+import org.apache.kafka.common.PartitionInfo;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.List;
+
+import static java.util.Arrays.asList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+public class StickyPartitionCacheTest {
+    private final static Node[] NODES = new Node[] {
+        new Node(0, "localhost", 99),
+        new Node(1, "localhost", 100),
+        new Node(12, "localhost", 101)
+    };
+    final static String TOPIC_A = "topicA";
+    final static String TOPIC_B = "topicB";
+    final static String TOPIC_C = "topicC";
+
+    @Test
+    public void testStickyPartitionCache() {
+        List allPartitions = asList(new PartitionInfo(TOPIC_A, 0, NODES[0], NODES, NODES),
+            new PartitionInfo(TOPIC_A, 1, NODES[1], NODES, NODES),
+            new PartitionInfo(TOPIC_A, 2, NODES[2], NODES, NODES),
+            new PartitionInfo(TOPIC_B, 0, NODES[0], NODES, NODES)
+        );
+        Cluster testCluster = new Cluster("clusterId", asList(NODES), allPartitions,
+            Collections.emptySet(), Collections.emptySet());
+        StickyPartitionCache stickyPartitionCache = new StickyPartitionCache();
+
+        int partA = stickyPartitionCache.partition(TOPIC_A, testCluster);
+        assertEquals(partA, stickyPartitionCache.partition(TOPIC_A, testCluster));
+
+        int partB = stickyPartitionCache.partition(TOPIC_B, testCluster);
+        assertEquals(partB, stickyPartitionCache.partition(TOPIC_B, testCluster));
+
+        int changedPartA = stickyPartitionCache.nextPartition(TOPIC_A, testCluster, partA);
+        assertEquals(changedPartA, stickyPartitionCache.partition(TOPIC_A, testCluster));
+        assertNotEquals(partA, changedPartA);
+        int changedPartA2 = stickyPartitionCache.partition(TOPIC_A, testCluster);
+        assertEquals(changedPartA2, changedPartA);
+
+        // We do not want to change partitions because the previous partition does not match the current sticky one.
+        int changedPartA3 = stickyPartitionCache.nextPartition(TOPIC_A, testCluster, partA);
+        assertEquals(changedPartA3, changedPartA2);
+
+        // Check that the we can still use the partitioner when there is only one partition
+        int changedPartB = stickyPartitionCache.nextPartition(TOPIC_B, testCluster, partB);
+        assertEquals(changedPartB, stickyPartitionCache.partition(TOPIC_B, testCluster));
+    }
+    
+    @Test
+    public void unavailablePartitionsTest() {
+        // Partition 1 in topic A and partition 0 in topic B are unavailable partitions.
+        List allPartitions = asList(new PartitionInfo(TOPIC_A, 0, NODES[0], NODES, NODES),
+            new PartitionInfo(TOPIC_A, 1, null, NODES, NODES),
+            new PartitionInfo(TOPIC_A, 2, NODES[2], NODES, NODES),
+            new PartitionInfo(TOPIC_B, 0, null, NODES, NODES),
+            new PartitionInfo(TOPIC_B, 1, NODES[0], NODES, NODES),
+            new PartitionInfo(TOPIC_C, 0, null, NODES, NODES)
+        );
+        
+        Cluster testCluster = new Cluster("clusterId", asList(NODES[0], NODES[1]), allPartitions,
+            Collections.emptySet(), Collections.emptySet());
+        StickyPartitionCache stickyPartitionCache = new StickyPartitionCache();
+        
+        // Assure we never choose partition 1 because it is unavailable.
+        int partA = stickyPartitionCache.partition(TOPIC_A, testCluster);
+        assertNotEquals(1, partA);
+        for (int aPartitions = 0; aPartitions < 100; aPartitions++) {
+            partA = stickyPartitionCache.nextPartition(TOPIC_A, testCluster, partA);
+            assertNotEquals(1, stickyPartitionCache.partition(TOPIC_A, testCluster));
+        }
+        
+        // Assure we always choose partition 1 for topic B.
+        int partB = stickyPartitionCache.partition(TOPIC_B, testCluster);
+        assertEquals(1, partB);
+        for (int bPartitions = 0; bPartitions < 100; bPartitions++) {
+            partB = stickyPartitionCache.nextPartition(TOPIC_B, testCluster, partB);
+            assertEquals(1, stickyPartitionCache.partition(TOPIC_B, testCluster));
+        }
+        
+        // Assure that we still choose the partition when there are no partitions available.
+        int partC = stickyPartitionCache.partition(TOPIC_C, testCluster);
+        assertEquals(0, partC);
+        partC = stickyPartitionCache.nextPartition(TOPIC_C, testCluster, partC);
+        assertEquals(0, partC);
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java
index 03b13d3fdf05b..cca5771002cd8 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java
@@ -151,9 +151,10 @@ public void testSenderShutdownWithPendingTransactions() throws Exception {
         doInitTransactions(pid, epoch);
         transactionManager.beginTransaction();
 
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
         FutureRecordMetadata sendFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         prepareAddPartitionsToTxn(tp0, Errors.NONE);
         prepareProduceResponse(Errors.NONE, pid, epoch);
@@ -177,6 +178,7 @@ public void testEndTxnNotSentIfIncompleteBatches() {
         doInitTransactions(pid, epoch);
         transactionManager.beginTransaction();
 
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
         prepareAddPartitionsToTxn(tp0, Errors.NONE);
         sender.runOnce();
@@ -245,6 +247,7 @@ public void testHasOngoingTransactionSuccessfulAbort() {
         transactionManager.beginTransaction();
         assertTrue(transactionManager.hasOngoingTransaction());
 
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(partition);
         assertTrue(transactionManager.hasOngoingTransaction());
 
@@ -272,6 +275,7 @@ public void testHasOngoingTransactionSuccessfulCommit() {
         transactionManager.beginTransaction();
         assertTrue(transactionManager.hasOngoingTransaction());
 
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(partition);
         assertTrue(transactionManager.hasOngoingTransaction());
 
@@ -299,6 +303,7 @@ public void testHasOngoingTransactionAbortableError() {
         transactionManager.beginTransaction();
         assertTrue(transactionManager.hasOngoingTransaction());
 
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(partition);
         assertTrue(transactionManager.hasOngoingTransaction());
 
@@ -329,6 +334,7 @@ public void testHasOngoingTransactionFatalError() {
         transactionManager.beginTransaction();
         assertTrue(transactionManager.hasOngoingTransaction());
 
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(partition);
         assertTrue(transactionManager.hasOngoingTransaction());
 
@@ -347,6 +353,7 @@ public void testMaybeAddPartitionToTransaction() {
         doInitTransactions(pid, epoch);
         transactionManager.beginTransaction();
 
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(partition);
         assertTrue(transactionManager.hasPartitionsToAdd());
         assertFalse(transactionManager.isPartitionAdded(partition));
@@ -360,6 +367,7 @@ public void testMaybeAddPartitionToTransaction() {
         assertFalse(transactionManager.isPartitionPendingAdd(partition));
 
         // adding the partition again should not have any effect
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(partition);
         assertFalse(transactionManager.hasPartitionsToAdd());
         assertTrue(transactionManager.isPartitionAdded(partition));
@@ -374,6 +382,7 @@ public void testAddPartitionToTransactionOverridesRetryBackoffForConcurrentTrans
         doInitTransactions(pid, epoch);
         transactionManager.beginTransaction();
 
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(partition);
         assertTrue(transactionManager.hasPartitionsToAdd());
         assertFalse(transactionManager.isPartitionAdded(partition));
@@ -395,6 +404,7 @@ public void testAddPartitionToTransactionRetainsRetryBackoffForRegularRetriableE
         doInitTransactions(pid, epoch);
         transactionManager.beginTransaction();
 
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(partition);
         assertTrue(transactionManager.hasPartitionsToAdd());
         assertFalse(transactionManager.isPartitionAdded(partition));
@@ -416,6 +426,7 @@ public void testAddPartitionToTransactionRetainsRetryBackoffWhenPartitionsAlread
         doInitTransactions(pid, epoch);
         transactionManager.beginTransaction();
 
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(partition);
         assertTrue(transactionManager.hasPartitionsToAdd());
         assertFalse(transactionManager.isPartitionAdded(partition));
@@ -426,6 +437,7 @@ public void testAddPartitionToTransactionRetainsRetryBackoffWhenPartitionsAlread
         assertTrue(transactionManager.isPartitionAdded(partition));
 
         TopicPartition otherPartition = new TopicPartition("foo", 1);
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(otherPartition);
 
         prepareAddPartitionsToTxn(otherPartition, Errors.CONCURRENT_TRANSACTIONS);
@@ -436,6 +448,7 @@ public void testAddPartitionToTransactionRetainsRetryBackoffWhenPartitionsAlread
 
     @Test(expected = IllegalStateException.class)
     public void testMaybeAddPartitionToTransactionBeforeInitTransactions() {
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(new TopicPartition("foo", 0));
     }
 
@@ -444,6 +457,7 @@ public void testMaybeAddPartitionToTransactionBeforeBeginTransaction() {
         long pid = 13131L;
         short epoch = 1;
         doInitTransactions(pid, epoch);
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(new TopicPartition("foo", 0));
     }
 
@@ -454,6 +468,7 @@ public void testMaybeAddPartitionToTransactionAfterAbortableError() {
         doInitTransactions(pid, epoch);
         transactionManager.beginTransaction();
         transactionManager.transitionToAbortableError(new KafkaException());
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(new TopicPartition("foo", 0));
     }
 
@@ -463,6 +478,7 @@ public void testMaybeAddPartitionToTransactionAfterFatalError() {
         short epoch = 1;
         doInitTransactions(pid, epoch);
         transactionManager.transitionToFatalError(new KafkaException());
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(new TopicPartition("foo", 0));
     }
 
@@ -474,6 +490,7 @@ public void testIsSendToPartitionAllowedWithPendingPartitionAfterAbortableError(
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
         transactionManager.transitionToAbortableError(new KafkaException());
 
@@ -489,6 +506,7 @@ public void testIsSendToPartitionAllowedWithInFlightPartitionAddAfterAbortableEr
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         // Send the AddPartitionsToTxn request and leave it in-flight
@@ -507,6 +525,7 @@ public void testIsSendToPartitionAllowedWithPendingPartitionAfterFatalError() {
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
         transactionManager.transitionToFatalError(new KafkaException());
 
@@ -522,6 +541,7 @@ public void testIsSendToPartitionAllowedWithInFlightPartitionAddAfterFatalError(
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         // Send the AddPartitionsToTxn request and leave it in-flight
@@ -541,6 +561,7 @@ public void testIsSendToPartitionAllowedWithAddedPartitionAfterAbortableError()
 
         transactionManager.beginTransaction();
 
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
         prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid);
         sender.runOnce();
@@ -559,6 +580,7 @@ public void testIsSendToPartitionAllowedWithAddedPartitionAfterFatalError() {
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
         prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid);
         sender.runOnce();
@@ -774,10 +796,11 @@ public void testBasicTransaction() throws InterruptedException {
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         assertFalse(responseFuture.isDone());
         prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid);
@@ -1190,7 +1213,9 @@ public void testTopicAuthorizationFailureInAddPartitions() {
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp1);
         Map errors = new HashMap<>();
         errors.put(tp0, Errors.TOPIC_AUTHORIZATION_FAILED);
@@ -1222,10 +1247,11 @@ public void testRecoveryFromAbortableErrorTransactionNotStarted() throws Excepti
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(unauthorizedPartition);
 
         Future responseFuture = accumulator.append(unauthorizedPartition, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         prepareAddPartitionsToTxn(singletonMap(unauthorizedPartition, Errors.TOPIC_AUTHORIZATION_FAILED));
         sender.runOnce();
@@ -1245,10 +1271,11 @@ public void testRecoveryFromAbortableErrorTransactionNotStarted() throws Excepti
         // ensure we can now start a new transaction
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         prepareAddPartitionsToTxn(singletonMap(tp0, Errors.NONE));
         sender.runOnce();
@@ -1277,17 +1304,19 @@ public void testRecoveryFromAbortableErrorTransactionStarted() throws Exception
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
         prepareAddPartitionsToTxn(tp0, Errors.NONE);
 
         Future authorizedTopicProduceFuture = accumulator.append(unauthorizedPartition, time.milliseconds(),
-                "key".getBytes(), "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "key".getBytes(), "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
         sender.runOnce();
         assertTrue(transactionManager.isPartitionAdded(tp0));
 
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(unauthorizedPartition);
         Future unauthorizedTopicProduceFuture = accumulator.append(unauthorizedPartition, time.milliseconds(),
-                "key".getBytes(), "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "key".getBytes(), "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
         prepareAddPartitionsToTxn(singletonMap(unauthorizedPartition, Errors.TOPIC_AUTHORIZATION_FAILED));
         sender.runOnce();
         assertTrue(transactionManager.hasAbortableError());
@@ -1309,10 +1338,11 @@ public void testRecoveryFromAbortableErrorTransactionStarted() throws Exception
         // ensure we can now start a new transaction
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         FutureRecordMetadata nextTransactionFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         prepareAddPartitionsToTxn(singletonMap(tp0, Errors.NONE));
         sender.runOnce();
@@ -1341,11 +1371,12 @@ public void testRecoveryFromAbortableErrorProduceRequestInRetry() throws Excepti
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
         prepareAddPartitionsToTxn(tp0, Errors.NONE);
 
         Future authorizedTopicProduceFuture = accumulator.append(tp0, time.milliseconds(),
-                "key".getBytes(), "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "key".getBytes(), "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
         sender.runOnce();
         assertTrue(transactionManager.isPartitionAdded(tp0));
 
@@ -1355,9 +1386,10 @@ public void testRecoveryFromAbortableErrorProduceRequestInRetry() throws Excepti
         assertFalse(authorizedTopicProduceFuture.isDone());
         assertTrue(accumulator.hasIncomplete());
 
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(unauthorizedPartition);
         Future unauthorizedTopicProduceFuture = accumulator.append(unauthorizedPartition, time.milliseconds(),
-                "key".getBytes(), "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "key".getBytes(), "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
         prepareAddPartitionsToTxn(singletonMap(unauthorizedPartition, Errors.TOPIC_AUTHORIZATION_FAILED));
         sender.runOnce();
         assertTrue(transactionManager.hasAbortableError());
@@ -1383,10 +1415,11 @@ public void testRecoveryFromAbortableErrorProduceRequestInRetry() throws Excepti
         // ensure we can now start a new transaction
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         FutureRecordMetadata nextTransactionFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         prepareAddPartitionsToTxn(singletonMap(tp0, Errors.NONE));
         sender.runOnce();
@@ -1415,6 +1448,7 @@ public void testTransactionalIdAuthorizationFailureInAddPartitions() {
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp);
 
         prepareAddPartitionsToTxn(tp, Errors.TRANSACTIONAL_ID_AUTHORIZATION_FAILED);
@@ -1434,10 +1468,11 @@ public void testFlushPendingPartitionsOnCommit() throws InterruptedException {
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         assertFalse(responseFuture.isDone());
 
@@ -1478,11 +1513,12 @@ public void testMultipleAddPartitionsPerForOneProduce() throws InterruptedExcept
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
-        // User does one producer.sed
+        // User does one producer.send
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         assertFalse(responseFuture.isDone());
         prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid);
@@ -1495,9 +1531,10 @@ public void testMultipleAddPartitionsPerForOneProduce() throws InterruptedExcept
         assertTrue(transactionManager.transactionContainsPartition(tp0));
 
         // In the mean time, the user does a second produce to a different partition
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp1);
         Future secondResponseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         prepareAddPartitionsToTxnResponse(Errors.NONE, tp1, epoch, pid);
         prepareProduceResponse(Errors.NONE, pid, epoch);
@@ -1529,10 +1566,11 @@ public void testProducerFencedException() throws InterruptedException {
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         assertFalse(responseFuture.isDone());
         prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid);
@@ -1567,10 +1605,11 @@ public void testDisallowCommitOnProduceFailure() throws InterruptedException {
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         TransactionalRequestResult commitResult = transactionManager.beginCommit();
         assertFalse(responseFuture.isDone());
@@ -1616,10 +1655,11 @@ public void testAllowAbortOnProduceFailure() throws InterruptedException {
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         assertFalse(responseFuture.isDone());
         prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid);
@@ -1644,10 +1684,11 @@ public void testAbortableErrorWhileAbortInProgress() throws InterruptedException
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         assertFalse(responseFuture.isDone());
         prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid);
@@ -1681,10 +1722,11 @@ public void testCommitTransactionWithUnsentProduceRequest() throws Exception {
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         prepareAddPartitionsToTxn(tp0, Errors.NONE);
         sender.runOnce();
@@ -1730,10 +1772,11 @@ public void testCommitTransactionWithInFlightProduceRequest() throws Exception {
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         prepareAddPartitionsToTxn(tp0, Errors.NONE);
         sender.runOnce();
@@ -1785,10 +1828,11 @@ public void testFindCoordinatorAllowedInAbortableErrorState() throws Interrupted
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         assertFalse(responseFuture.isDone());
         sender.runOnce();  // Send AddPartitionsRequest
@@ -1813,10 +1857,11 @@ public void testCancelUnsentAddPartitionsAndProduceOnAbort() throws InterruptedE
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         assertFalse(responseFuture.isDone());
 
@@ -1844,11 +1889,12 @@ public void testAbortResendsAddPartitionErrorIfRetried() throws InterruptedExcep
         doInitTransactions(producerId, producerEpoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
         prepareAddPartitionsToTxnResponse(Errors.UNKNOWN_TOPIC_OR_PARTITION, tp0, producerEpoch, producerId);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         sender.runOnce();  // Send AddPartitions and let it fail
         assertFalse(responseFuture.isDone());
@@ -1882,12 +1928,13 @@ public void testAbortResendsProduceRequestIfRetried() throws Exception {
         doInitTransactions(producerId, producerEpoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
         prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, producerEpoch, producerId);
         prepareProduceResponse(Errors.REQUEST_TIMED_OUT, producerId, producerEpoch);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         sender.runOnce();  // Send AddPartitions
         sender.runOnce();  // Send ProduceRequest and let it fail
@@ -1919,10 +1966,11 @@ public void testHandlingOfUnknownTopicPartitionErrorOnAddPartitions() throws Int
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         assertFalse(responseFuture.isDone());
         prepareAddPartitionsToTxnResponse(Errors.UNKNOWN_TOPIC_OR_PARTITION, tp0, epoch, pid);
@@ -2008,6 +2056,7 @@ public void shouldNotSendAbortTxnRequestWhenOnlyAddPartitionsRequestFailed() {
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         prepareAddPartitionsToTxnResponse(Errors.TOPIC_AUTHORIZATION_FAILED, tp0, epoch, pid);
@@ -2079,12 +2128,14 @@ public void testNoDrainWhenPartitionsPending() throws InterruptedException {
         final short epoch = 1;
         doInitTransactions(pid, epoch);
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
         accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT);
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false);
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp1);
         accumulator.append(tp1, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT);
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false);
 
         assertFalse(transactionManager.isSendToPartitionAllowed(tp0));
         assertFalse(transactionManager.isSendToPartitionAllowed(tp1));
@@ -2116,12 +2167,14 @@ public void testAllowDrainInAbortableErrorState() throws InterruptedException {
         final short epoch = 1;
         doInitTransactions(pid, epoch);
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp1);
         prepareAddPartitionsToTxn(tp1, Errors.NONE);
         sender.runOnce();  // Send AddPartitions, tp1 should be in the transaction now.
 
         assertTrue(transactionManager.transactionContainsPartition(tp1));
 
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
         prepareAddPartitionsToTxn(tp0, Errors.TOPIC_AUTHORIZATION_FAILED);
         sender.runOnce();  // Send AddPartitions, should be in abortable state.
@@ -2135,7 +2188,7 @@ public void testAllowDrainInAbortableErrorState() throws InterruptedException {
         Cluster cluster = new Cluster(null, Collections.singletonList(node1), Collections.singletonList(part1),
                 Collections.emptySet(), Collections.emptySet());
         accumulator.append(tp1, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT);
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false);
         Map> drainedBatches = accumulator.drain(cluster, Collections.singleton(node1),
                 Integer.MAX_VALUE,
                 time.milliseconds());
@@ -2155,7 +2208,7 @@ public void testRaiseErrorWhenNoPartitionsPendingOnDrain() throws InterruptedExc
         transactionManager.beginTransaction();
         // Don't execute transactionManager.maybeAddPartitionToTransaction(tp0). This should result in an error on drain.
         accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT);
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false);
         Node node1 = new Node(0, "localhost", 1111);
         PartitionInfo part1 = new PartitionInfo(topic, 0, node1, null, null);
 
@@ -2178,10 +2231,11 @@ public void resendFailedProduceRequestAfterAbortableError() throws Exception {
         doInitTransactions(pid, epoch);
         transactionManager.beginTransaction();
 
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid);
         prepareProduceResponse(Errors.NOT_LEADER_FOR_PARTITION, pid, epoch);
@@ -2206,10 +2260,11 @@ public void testTransitionToAbortableErrorOnBatchExpiry() throws InterruptedExce
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         assertFalse(responseFuture.isDone());
 
@@ -2252,13 +2307,15 @@ public void testTransitionToAbortableErrorOnMultipleBatchExpiry() throws Interru
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp1);
 
         Future firstBatchResponse = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
         Future secondBatchResponse = accumulator.append(tp1, time.milliseconds(), "key".getBytes(),
-               "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+               "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         assertFalse(firstBatchResponse.isDone());
         assertFalse(secondBatchResponse.isDone());
@@ -2317,10 +2374,11 @@ public void testDropCommitOnBatchExpiry() throws InterruptedException {
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         assertFalse(responseFuture.isDone());
 
@@ -2383,10 +2441,11 @@ public void testTransitionToFatalErrorWhenRetriedBatchIsExpired() throws Interru
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
 
         assertFalse(responseFuture.isDone());
 
@@ -2581,10 +2640,11 @@ private void verifyCommitOrAbortTranscationRetriable(TransactionResult firstTran
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT);
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false);
 
         prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid);
 
@@ -2622,10 +2682,11 @@ private void verifyAddPartitionsFailsWithPartitionLevelError(final Errors error)
         doInitTransactions(pid, epoch);
 
         transactionManager.beginTransaction();
+        transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
         assertFalse(responseFuture.isDone());
         prepareAddPartitionsToTxn(tp0, error);
         sender.runOnce();  // attempt send addPartitions.

From 1546fc30af520f8d133aa3454cdc8a036bae0f3e Mon Sep 17 00:00:00 2001
From: Sean Glover 
Date: Thu, 1 Aug 2019 20:10:19 -0400
Subject: [PATCH 0498/1071] KAFKA-7548; KafkaConsumer should not discard
 fetched data for paused partitions (#6988)

This is an updated implementation of #5844 by @MayureshGharat (with Mayuresh's permission). As described in the original ticket:

> Today when we call KafkaConsumer.poll(), it will fetch data from Kafka asynchronously and is put in to a local buffer (completedFetches).
>
> If now we pause some TopicPartitions and call KafkaConsumer.poll(), we might throw away any buffered data that we might have in the local buffer for these TopicPartitions. Generally, if an application is calling pause on some TopicPartitions, it is likely to resume those TopicPartitions in near future, which would require KafkaConsumer to re-issue a fetch for the same data that it had buffered earlier for these TopicPartitions. This is a wasted effort from the application's point of view.

This patch fixes the problem by retaining the paused data in the completed fetches queue, essentially moving it to the back on each call to `fetchedRecords`.

Reviewers: Jason Gustafson 
---
 .../kafka/clients/consumer/KafkaConsumer.java |   2 +-
 .../clients/consumer/internals/Fetcher.java   | 120 +++++++++----
 .../consumer/internals/FetcherTest.java       | 165 ++++++++++++++++++
 3 files changed, 253 insertions(+), 34 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index b33b1f20796e8..0d0efe1b663fc 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -1270,7 +1270,7 @@ private Map>> pollForFetches(Timer tim
         client.poll(pollTimer, () -> {
             // since a fetch might be completed by the background thread, we need this poll condition
             // to ensure that we do not block unnecessarily in poll()
-            return !fetcher.hasCompletedFetches();
+            return !fetcher.hasAvailableFetches();
         });
         timer.update(pollTimer.currentTimeMs());
 
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
index d4d028d38dc73..c1c42bd06ca4f 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
@@ -50,8 +50,8 @@
 import org.apache.kafka.common.metrics.stats.Max;
 import org.apache.kafka.common.metrics.stats.Meter;
 import org.apache.kafka.common.metrics.stats.Min;
-import org.apache.kafka.common.metrics.stats.WindowedCount;
 import org.apache.kafka.common.metrics.stats.Value;
+import org.apache.kafka.common.metrics.stats.WindowedCount;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.record.BufferSupplier;
@@ -81,6 +81,7 @@
 
 import java.io.Closeable;
 import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -93,6 +94,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.PriorityQueue;
+import java.util.Queue;
 import java.util.Set;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -140,7 +142,7 @@ public class Fetcher implements Closeable {
     private final ConsumerMetadata metadata;
     private final FetchManagerMetrics sensors;
     private final SubscriptionState subscriptions;
-    private final ConcurrentLinkedQueue completedFetches;
+    private final ConcurrentLinkedQueue completedFetches;
     private final BufferSupplier decompressionBufferSupplier = BufferSupplier.create();
     private final Deserializer keyDeserializer;
     private final Deserializer valueDeserializer;
@@ -216,13 +218,22 @@ private static class ListOffsetData {
     }
 
     /**
-     * Return whether we have any completed fetches pending return to the user. This method is thread-safe.
+     * Return whether we have any completed fetches pending return to the user. This method is thread-safe. Has
+     * visibility for testing.
      * @return true if there are completed fetches, false otherwise
      */
-    public boolean hasCompletedFetches() {
+    protected boolean hasCompletedFetches() {
         return !completedFetches.isEmpty();
     }
 
+    /**
+     * Return whether we have any completed fetches that are fetchable. This method is thread-safe.
+     * @return true if there are completed fetches that can be returned, false otherwise
+     */
+    public boolean hasAvailableFetches() {
+        return completedFetches.stream().anyMatch(fetch -> subscriptions.isFetchable(fetch.partition));
+    }
+
     /**
      * Set-up a fetch request for any node that we have assigned partitions for which doesn't already have
      * an in-flight fetch or pending fetch data.
@@ -291,8 +302,11 @@ public void onSuccess(ClientResponse resp) {
 
                                             log.debug("Fetch {} at offset {} for partition {} returned fetch data {}",
                                                     isolationLevel, fetchOffset, partition, fetchData);
-                                            completedFetches.add(new CompletedFetch(partition, fetchOffset, fetchData, metricAggregator,
-                                                    resp.requestHeader().apiVersion()));
+
+                                            CompletedFetch completedFetch = new CompletedFetch(partition, fetchOffset,
+                                                    fetchData, metricAggregator, resp.requestHeader().apiVersion());
+
+                                            completedFetches.add(parseCompletedFetch(completedFetch));
                                         }
                                     }
 
@@ -577,33 +591,45 @@ private Map beginningOrEndOffset(Collection>> fetchedRecords() {
         Map>> fetched = new HashMap<>();
+        Queue pausedCompletedFetches = new ArrayDeque<>();
         int recordsRemaining = maxPollRecords;
 
         try {
             while (recordsRemaining > 0) {
                 if (nextInLineRecords == null || nextInLineRecords.isFetched) {
-                    CompletedFetch completedFetch = completedFetches.peek();
-                    if (completedFetch == null) break;
-
-                    try {
-                        nextInLineRecords = parseCompletedFetch(completedFetch);
-                    } catch (Exception e) {
-                        // Remove a completedFetch upon a parse with exception if (1) it contains no records, and
-                        // (2) there are no fetched records with actual content preceding this exception.
-                        // The first condition ensures that the completedFetches is not stuck with the same completedFetch
-                        // in cases such as the TopicAuthorizationException, and the second condition ensures that no
-                        // potential data loss due to an exception in a following record.
-                        FetchResponse.PartitionData partition = completedFetch.partitionData;
-                        if (fetched.isEmpty() && (partition.records == null || partition.records.sizeInBytes() == 0)) {
-                            completedFetches.poll();
+                    PartitionRecords records = completedFetches.peek();
+                    if (records == null) break;
+
+                    if (records.notInitialized()) {
+                        try {
+                            nextInLineRecords = initializePartitionRecords(records);
+                        } catch (Exception e) {
+                            // Remove a completedFetch upon a parse with exception if (1) it contains no records, and
+                            // (2) there are no fetched records with actual content preceding this exception.
+                            // The first condition ensures that the completedFetches is not stuck with the same completedFetch
+                            // in cases such as the TopicAuthorizationException, and the second condition ensures that no
+                            // potential data loss due to an exception in a following record.
+                            FetchResponse.PartitionData partition = records.completedFetch.partitionData;
+                            if (fetched.isEmpty() && (partition.records == null || partition.records.sizeInBytes() == 0)) {
+                                completedFetches.poll();
+                            }
+                            throw e;
                         }
-                        throw e;
+                    } else {
+                        nextInLineRecords = records;
                     }
                     completedFetches.poll();
+                } else if (subscriptions.isPaused(nextInLineRecords.partition)) {
+                    // when the partition is paused we add the records back to the completedFetches queue instead of draining
+                    // them so that they can be returned on a subsequent poll if the partition is resumed at that time
+                    log.debug("Skipping fetching records for assigned partition {} because it is paused", nextInLineRecords.partition);
+                    pausedCompletedFetches.add(nextInLineRecords);
+                    nextInLineRecords = null;
                 } else {
                     List> records = fetchRecords(nextInLineRecords, recordsRemaining);
-                    TopicPartition partition = nextInLineRecords.partition;
+
                     if (!records.isEmpty()) {
+                        TopicPartition partition = nextInLineRecords.partition;
                         List> currentRecords = fetched.get(partition);
                         if (currentRecords == null) {
                             fetched.put(partition, records);
@@ -623,7 +649,12 @@ public Map>> fetchedRecords() {
         } catch (KafkaException e) {
             if (fetched.isEmpty())
                 throw e;
+        } finally {
+            // add any polled completed fetches for paused partitions back to the completed fetches queue to be
+            // re-evaluated in the next poll
+            completedFetches.addAll(pausedCompletedFetches);
         }
+
         return fetched;
     }
 
@@ -670,7 +701,9 @@ private List> fetchRecords(PartitionRecords partitionRecord
             }
         }
 
+        log.trace("Draining fetched records for partition {}", partitionRecords.partition);
         partitionRecords.drain();
+
         return emptyList();
     }
 
@@ -1025,7 +1058,7 @@ private List fetchablePartitions() {
         if (nextInLineRecords != null && !nextInLineRecords.isFetched) {
             exclude.add(nextInLineRecords.partition);
         }
-        for (CompletedFetch completedFetch : completedFetches) {
+        for (PartitionRecords completedFetch : completedFetches) {
             exclude.add(completedFetch.partition);
         }
         return subscriptions.fetchablePartitions(tp -> !exclude.contains(tp));
@@ -1125,20 +1158,31 @@ private  Map> regroupPartitionMapByNode(Map partition = completedFetch.partitionData;
+
+        Iterator batches = partition.records.batches().iterator();
+        return new PartitionRecords(tp, completedFetch, batches);
+    }
+
+    /**
+     * Initialize a PartitionRecords object.
+     */
+    private PartitionRecords initializePartitionRecords(PartitionRecords partitionRecordsToInitialize) {
+        CompletedFetch completedFetch = partitionRecordsToInitialize.completedFetch;
+        TopicPartition tp = completedFetch.partition;
+        FetchResponse.PartitionData partition = completedFetch.partitionData;
         long fetchOffset = completedFetch.fetchedOffset;
         PartitionRecords partitionRecords = null;
         Errors error = partition.error;
 
         try {
-            if (!subscriptions.isFetchable(tp)) {
-                // this can happen when a rebalance happened or a partition consumption paused
-                // while fetch is still in-flight
-                log.debug("Ignoring fetched records for partition {} since it is no longer fetchable", tp);
+            if (!subscriptions.hasValidPosition(tp)) {
+                // this can happen when a rebalance happened while fetch is still in-flight
+                log.debug("Ignoring fetched records for partition {} since it no longer has valid position", tp);
             } else if (error == Errors.NONE) {
                 // we are interested in this fetch only if the beginning offset matches the
                 // current consumed position
@@ -1152,7 +1196,7 @@ private PartitionRecords parseCompletedFetch(CompletedFetch completedFetch) {
                 log.trace("Preparing to read {} bytes of data for partition {} with offset {}",
                         partition.records.sizeInBytes(), tp, position);
                 Iterator batches = partition.records.batches().iterator();
-                partitionRecords = new PartitionRecords(tp, completedFetch, batches);
+                partitionRecords = partitionRecordsToInitialize;
 
                 if (!batches.hasNext() && partition.records.sizeInBytes() > 0) {
                     if (completedFetch.responseVersion < 3) {
@@ -1196,6 +1240,8 @@ private PartitionRecords parseCompletedFetch(CompletedFetch completedFetch) {
                     });
                 }
 
+
+                partitionRecordsToInitialize.initialized = true;
             } else if (error == Errors.NOT_LEADER_FOR_PARTITION ||
                        error == Errors.REPLICA_NOT_AVAILABLE ||
                        error == Errors.KAFKA_STORAGE_ERROR ||
@@ -1286,13 +1332,16 @@ private Optional maybeLeaderEpoch(int leaderEpoch) {
      * @param assignedPartitions  newly assigned {@link TopicPartition}
      */
     public void clearBufferedDataForUnassignedPartitions(Collection assignedPartitions) {
-        Iterator itr = completedFetches.iterator();
-        while (itr.hasNext()) {
-            TopicPartition tp = itr.next().partition;
+        Iterator completedFetchesItr = completedFetches.iterator();
+        while (completedFetchesItr.hasNext()) {
+            PartitionRecords records = completedFetchesItr.next();
+            TopicPartition tp = records.partition;
             if (!assignedPartitions.contains(tp)) {
-                itr.remove();
+                records.drain();
+                completedFetchesItr.remove();
             }
         }
+
         if (nextInLineRecords != null && !assignedPartitions.contains(nextInLineRecords.partition)) {
             nextInLineRecords.drain();
             nextInLineRecords = null;
@@ -1345,6 +1394,7 @@ private class PartitionRecords {
         private boolean isFetched = false;
         private Exception cachedRecordException = null;
         private boolean corruptLastRecord = false;
+        private boolean initialized = false;
 
         private PartitionRecords(TopicPartition partition,
                                  CompletedFetch completedFetch,
@@ -1546,6 +1596,10 @@ private boolean containsAbortMarker(RecordBatch batch) {
             Record firstRecord = batchIterator.next();
             return ControlRecordType.ABORT == ControlRecordType.parse(firstRecord.key());
         }
+
+        private boolean notInitialized() {
+            return !this.initialized;
+        }
     }
 
     private static class CompletedFetch {
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
index 44c00c48a3ee6..5f66689520f36 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
@@ -897,6 +897,171 @@ public void testFetchOnPausedPartition() {
         assertTrue(client.requests().isEmpty());
     }
 
+    @Test
+    public void testFetchOnCompletedFetchesForPausedAndResumedPartitions() {
+        buildFetcher();
+
+        assignFromUser(singleton(tp0));
+        subscriptions.seek(tp0, 0);
+
+        assertEquals(1, fetcher.sendFetches());
+
+        subscriptions.pause(tp0);
+
+        client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0));
+        consumerClient.poll(time.timer(0));
+
+        Map>> fetchedRecords = fetchedRecords();
+        assertEquals("Should not return any records when partition is paused", 0, fetchedRecords.size());
+        assertTrue("Should still contain completed fetches", fetcher.hasCompletedFetches());
+        assertFalse("Should not have any available (non-paused) completed fetches", fetcher.hasAvailableFetches());
+        assertNull(fetchedRecords.get(tp0));
+        assertEquals(0, fetcher.sendFetches());
+
+        subscriptions.resume(tp0);
+
+        assertTrue("Should have available (non-paused) completed fetches", fetcher.hasAvailableFetches());
+
+        consumerClient.poll(time.timer(0));
+        fetchedRecords = fetchedRecords();
+        assertEquals("Should return records when partition is resumed", 1, fetchedRecords.size());
+        assertNotNull(fetchedRecords.get(tp0));
+        assertEquals(3, fetchedRecords.get(tp0).size());
+
+        consumerClient.poll(time.timer(0));
+        fetchedRecords = fetchedRecords();
+        assertEquals("Should not return records after previously paused partitions are fetched", 0, fetchedRecords.size());
+        assertFalse("Should no longer contain completed fetches", fetcher.hasCompletedFetches());
+    }
+
+    @Test
+    public void testFetchOnCompletedFetchesForSomePausedPartitions() {
+        buildFetcher();
+
+        Map>> fetchedRecords;
+
+        assignFromUser(Utils.mkSet(tp0, tp1));
+
+        // seek to tp0 and tp1 in two polls to generate 2 complete requests and responses
+
+        // #1 seek, request, poll, response
+        subscriptions.seek(tp0, 1);
+        assertEquals(1, fetcher.sendFetches());
+        client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0));
+        consumerClient.poll(time.timer(0));
+
+        // #2 seek, request, poll, response
+        subscriptions.seek(tp1, 1);
+        assertEquals(1, fetcher.sendFetches());
+        client.prepareResponse(fullFetchResponse(tp1, this.nextRecords, Errors.NONE, 100L, 0));
+
+        subscriptions.pause(tp0);
+        consumerClient.poll(time.timer(0));
+
+        fetchedRecords = fetchedRecords();
+        assertEquals("Should return completed fetch for unpaused partitions", 1, fetchedRecords.size());
+        assertTrue("Should still contain completed fetches", fetcher.hasCompletedFetches());
+        assertNotNull(fetchedRecords.get(tp1));
+        assertNull(fetchedRecords.get(tp0));
+
+        fetchedRecords = fetchedRecords();
+        assertEquals("Should return no records for remaining paused partition", 0, fetchedRecords.size());
+        assertTrue("Should still contain completed fetches", fetcher.hasCompletedFetches());
+    }
+
+    @Test
+    public void testFetchOnCompletedFetchesForAllPausedPartitions() {
+        buildFetcher();
+
+        Map>> fetchedRecords;
+
+        assignFromUser(Utils.mkSet(tp0, tp1));
+
+        // seek to tp0 and tp1 in two polls to generate 2 complete requests and responses
+
+        // #1 seek, request, poll, response
+        subscriptions.seek(tp0, 1);
+        assertEquals(1, fetcher.sendFetches());
+        client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0));
+        consumerClient.poll(time.timer(0));
+
+        // #2 seek, request, poll, response
+        subscriptions.seek(tp1, 1);
+        assertEquals(1, fetcher.sendFetches());
+        client.prepareResponse(fullFetchResponse(tp1, this.nextRecords, Errors.NONE, 100L, 0));
+
+        subscriptions.pause(tp0);
+        subscriptions.pause(tp1);
+
+        consumerClient.poll(time.timer(0));
+
+        fetchedRecords = fetchedRecords();
+        assertEquals("Should return no records for all paused partitions", 0, fetchedRecords.size());
+        assertTrue("Should still contain completed fetches", fetcher.hasCompletedFetches());
+        assertFalse("Should not have any available (non-paused) completed fetches", fetcher.hasAvailableFetches());
+    }
+
+    @Test
+    public void testPartialFetchWithPausedPartitions() {
+        // this test sends creates a completed fetch with 3 records and a max poll of 2 records to assert
+        // that a fetch that must be returned over at least 2 polls can be cached successfully when its partition is
+        // paused, then returned successfully after its been resumed again later
+        buildFetcher(2);
+
+        Map>> fetchedRecords;
+
+        assignFromUser(Utils.mkSet(tp0, tp1));
+
+        subscriptions.seek(tp0, 1);
+        assertEquals(1, fetcher.sendFetches());
+        client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0));
+        consumerClient.poll(time.timer(0));
+
+        fetchedRecords = fetchedRecords();
+
+        assertEquals("Should return 2 records from fetch with 3 records", 2, fetchedRecords.get(tp0).size());
+        assertFalse("Should have no completed fetches", fetcher.hasCompletedFetches());
+
+        subscriptions.pause(tp0);
+        consumerClient.poll(time.timer(0));
+
+        fetchedRecords = fetchedRecords();
+
+        assertEquals("Should return no records for paused partitions", 0, fetchedRecords.size());
+        assertTrue("Should have 1 entry in completed fetches", fetcher.hasCompletedFetches());
+        assertFalse("Should not have any available (non-paused) completed fetches", fetcher.hasAvailableFetches());
+
+        subscriptions.resume(tp0);
+
+        consumerClient.poll(time.timer(0));
+
+        fetchedRecords = fetchedRecords();
+
+        assertEquals("Should return last remaining record", 1, fetchedRecords.get(tp0).size());
+        assertFalse("Should have no completed fetches", fetcher.hasCompletedFetches());
+    }
+
+    @Test
+    public void testFetchDiscardedAfterPausedPartitionResumedAndSeekedToNewOffset() {
+        buildFetcher();
+        assignFromUser(singleton(tp0));
+        subscriptions.seek(tp0, 0);
+
+        assertEquals(1, fetcher.sendFetches());
+        subscriptions.pause(tp0);
+        client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0));
+
+        subscriptions.seek(tp0, 3);
+        subscriptions.resume(tp0);
+        consumerClient.poll(time.timer(0));
+
+        assertTrue("Should have 1 entry in completed fetches", fetcher.hasCompletedFetches());
+        Map>> fetchedRecords = fetchedRecords();
+        assertEquals("Should not return any records because we seeked to a new offset", 0, fetchedRecords.size());
+        assertNull(fetchedRecords.get(tp0));
+        assertFalse("Should have no completed fetches", fetcher.hasCompletedFetches());
+    }
+
     @Test
     public void testFetchNotLeaderForPartition() {
         buildFetcher();

From 22d4ccd8acbece69e1e1897714b18ff95ff37694 Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Fri, 2 Aug 2019 06:09:51 -0700
Subject: [PATCH 0499/1071] MINOR: Update docs to reflect the ZK 3.5.5 upgrade
 (#7149)

Reviewers: Manikumar Reddy 
---
 docs/ops.html      | 2 +-
 docs/security.html | 2 +-
 docs/upgrade.html  | 1 +
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/ops.html b/docs/ops.html
index b4370e1ab2a5e..f94afc39e03cd 100644
--- a/docs/ops.html
+++ b/docs/ops.html
@@ -2014,7 +2014,7 @@ 
      Others
      
   
      6.7 ZooKeeper
      
 
   
      Stable version
      
-  The current stable branch is 3.4 and the latest release of that branch is 3.4.9.
+  The current stable branch is 3.5. Kafka is regularly updated to include the latest release in the 3.5 series.
 
   
      Operationalizing ZooKeeper
      
   Operationally, we do the following for a healthy ZooKeeper installation:
diff --git a/docs/security.html b/docs/security.html
index f281bfff4166f..ea103b70e42ef 100644
--- a/docs/security.html
+++ b/docs/security.html
@@ -1847,7 +1847,7 @@ 
      7.6.2 Migrating cluste
     
      7.6.3 Migrating the ZooKeeper ensemble
      
     It is also necessary to enable authentication on the ZooKeeper ensemble. To do it, we need to perform a rolling restart of the server and set a few properties. Please refer to the ZooKeeper documentation for more detail:
     
        
-        
      1. Apache ZooKeeper documentation
        2. 
+        
      2. Apache ZooKeeper documentation
        3. 
         
      3. Apache ZooKeeper wiki
        4. 
     
      
 
diff --git a/docs/upgrade.html b/docs/upgrade.html
index 0ae59046d0b27..e55e3b8dc4310 100644
--- a/docs/upgrade.html
+++ b/docs/upgrade.html
@@ -21,6 +21,7 @@
 
 
      Notable changes in 2.4.0
      
 
        
+    
      • ZooKeeper has been upgraded from 3.4.14 to 3.5.5. TLS and dynamic reconfiguration are supported by the new version.
        • 
     
      • The bin/kafka-preferred-replica-election.sh command line tool has been deprecated. It has been replaced by bin/kafka-leader-election.sh.
        • 
     
      • The methods electPreferredLeaders in the Java AdminClient class have been deprecated in favor of the methods electLeaders.
        • 
     
      • Scala code leveraging the NewTopic(String, int, short) constructor with literal values will need to explicitly call toShort on the second literal.
        • 

From a7d0fdd534ef55533a868ea7388bbc081ee42718 Mon Sep 17 00:00:00 2001
From: cadonna 
Date: Fri, 2 Aug 2019 18:51:03 +0200
Subject: [PATCH 0500/1071] KAFKA-8578: Add basic functionality to expose
 RocksDB metrics (#6979)

* Adds RocksDBMetrics class that provides methods to get sensors from the Kafka metrics registry and to setup the sensors to record RocksDB metrics

* Extends StreamsMetricsImpl with functionality to add the required metrics to the sensors.

Reviewers: Boyang Chen , Bill Bejeck , Matthias J. Sax , John Roesler , Guozhang Wang 
---
 .../internals/metrics/StreamsMetricsImpl.java |  83 +++-
 .../internals/metrics/RocksDBMetrics.java     | 382 ++++++++++++++++++
 .../metrics/StreamsMetricsImplTest.java       | 113 +++++-
 .../internals/metrics/RocksDBMetricsTest.java | 283 +++++++++++++
 4 files changed, 852 insertions(+), 9 deletions(-)
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetrics.java
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsTest.java

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
index b6bfcc5c8513a..ae3d9534f8dd0 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
@@ -23,9 +23,12 @@
 import org.apache.kafka.common.metrics.Sensor.RecordingLevel;
 import org.apache.kafka.common.metrics.stats.Avg;
 import org.apache.kafka.common.metrics.stats.CumulativeCount;
+import org.apache.kafka.common.metrics.stats.CumulativeSum;
 import org.apache.kafka.common.metrics.stats.Max;
 import org.apache.kafka.common.metrics.stats.Rate;
+import org.apache.kafka.common.metrics.stats.Value;
 import org.apache.kafka.common.metrics.stats.WindowedCount;
+import org.apache.kafka.common.metrics.stats.WindowedSum;
 import org.apache.kafka.streams.StreamsMetrics;
 
 import java.util.Arrays;
@@ -54,17 +57,21 @@ public class StreamsMetricsImpl implements StreamsMetrics {
 
     public static final String THREAD_ID_TAG = "client-id";
     public static final String TASK_ID_TAG = "task-id";
+    public static final String STORE_ID_TAG = "state-id";
 
     public static final String ALL_TASKS = "all";
 
     public static final String LATENCY_SUFFIX = "-latency";
     public static final String AVG_SUFFIX = "-avg";
     public static final String MAX_SUFFIX = "-max";
+    public static final String MIN_SUFFIX = "-min";
     public static final String RATE_SUFFIX = "-rate";
     public static final String TOTAL_SUFFIX = "-total";
+    public static final String RATIO_SUFFIX = "-ratio";
 
     public static final String THREAD_LEVEL_GROUP = "stream-metrics";
     public static final String TASK_LEVEL_GROUP = "stream-task-metrics";
+    public static final String STATE_LEVEL_GROUP = "stream-state-metrics";
 
     public static final String PROCESSOR_NODE_METRICS_GROUP = "stream-processor-node-metrics";
     public static final String PROCESSOR_NODE_ID_TAG = "processor-node-id";
@@ -123,6 +130,18 @@ public final void removeAllThreadLevelSensors() {
         }
     }
 
+    public Map taskLevelTagMap(final String taskName) {
+        final Map tagMap = threadLevelTagMap();
+        tagMap.put(TASK_ID_TAG, taskName);
+        return tagMap;
+    }
+
+    public Map storeLevelTagMap(final String taskName, final String storeType, final String storeName) {
+        final Map tagMap = taskLevelTagMap(taskName);
+        tagMap.put(storeType + "-" + STORE_ID_TAG, storeName);
+        return tagMap;
+    }
+
     public final Sensor taskLevelSensor(final String taskName,
                                         final String sensorName,
                                         final RecordingLevel recordingLevel,
@@ -237,9 +256,7 @@ public final Sensor storeLevelSensor(final String taskName,
             if (!storeLevelSensors.containsKey(key)) {
                 storeLevelSensors.put(key, new LinkedList<>());
             }
-
             final String fullSensorName = key + SENSOR_NAME_DELIMITER + sensorName;
-
             final Sensor sensor = metrics.sensor(fullSensorName, recordingLevel, parents);
 
             storeLevelSensors.get(key).push(fullSensorName);
@@ -454,12 +471,62 @@ public static void addInvocationRateAndCount(final Sensor sensor,
                                                  final String group,
                                                  final Map tags,
                                                  final String operation) {
-        addInvocationRateAndCount(sensor,
-                                  group,
-                                  tags,
-                                  operation,
-                                  "The total number of " + operation,
-                                  "The average per-second number of " + operation);
+        addInvocationRateAndCount(
+            sensor,
+            group,
+            tags,
+            operation,
+            "The total number of " + operation,
+            "The average per-second number of " + operation
+        );
+    }
+
+    public static void addRateOfSumAndSumMetricsToSensor(final Sensor sensor,
+                                                         final String group,
+                                                         final Map tags,
+                                                         final String operation,
+                                                         final String descriptionOfRate,
+                                                         final String descriptionOfTotal) {
+        addRateOfSumMetricToSensor(sensor, group, tags, operation, descriptionOfRate);
+        addSumMetricToSensor(sensor, group, tags, operation, descriptionOfTotal);
+    }
+
+    public static void addRateOfSumMetricToSensor(final Sensor sensor,
+                                                  final String group,
+                                                  final Map tags,
+                                                  final String operation,
+                                                  final String description) {
+        sensor.add(new MetricName(operation + RATE_SUFFIX, group, description, tags),
+                   new Rate(TimeUnit.SECONDS, new WindowedSum()));
+    }
+
+    public static void addSumMetricToSensor(final Sensor sensor,
+                                            final String group,
+                                            final Map tags,
+                                            final String operation,
+                                            final String description) {
+        sensor.add(new MetricName(operation + TOTAL_SUFFIX, group, description, tags), new CumulativeSum());
+    }
+
+    public static void addValueMetricToSensor(final Sensor sensor,
+                                              final String group,
+                                              final Map tags,
+                                              final String name,
+                                              final String description) {
+        sensor.add(new MetricName(name, group, description, tags), new Value());
+    }
+
+    public static void addAvgAndSumMetricsToSensor(final Sensor sensor,
+                                                   final String group,
+                                                   final Map tags,
+                                                   final String metricNamePrefix,
+                                                   final String descriptionOfAvg,
+                                                   final String descriptionOfTotal) {
+        sensor.add(new MetricName(metricNamePrefix + AVG_SUFFIX, group, descriptionOfAvg, tags), new Avg());
+        sensor.add(
+            new MetricName(metricNamePrefix + TOTAL_SUFFIX, group, descriptionOfTotal, tags),
+            new CumulativeSum()
+        );
     }
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetrics.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetrics.java
new file mode 100644
index 0000000000000..95cb52206ea85
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetrics.java
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals.metrics;
+
+import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.metrics.Sensor.RecordingLevel;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.AVG_SUFFIX;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.MAX_SUFFIX;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.MIN_SUFFIX;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.RATIO_SUFFIX;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.STATE_LEVEL_GROUP;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addRateOfSumMetricToSensor;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addRateOfSumAndSumMetricsToSensor;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addAvgAndSumMetricsToSensor;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addSumMetricToSensor;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addValueMetricToSensor;
+
+public class RocksDBMetrics {
+    private RocksDBMetrics() {}
+
+    private static final String BYTES_WRITTEN_TO_DB = "bytes-written";
+    private static final String BYTES_READ_FROM_DB = "bytes-read";
+    private static final String MEMTABLE_BYTES_FLUSHED = "memtable-bytes-flushed";
+    private static final String MEMTABLE_HIT_RATIO = "memtable-hit" + RATIO_SUFFIX;
+    private static final String MEMTABLE_FLUSH_TIME = "memtable-flush-time";
+    private static final String MEMTABLE_FLUSH_TIME_AVG = MEMTABLE_FLUSH_TIME + AVG_SUFFIX;
+    private static final String MEMTABLE_FLUSH_TIME_MIN = MEMTABLE_FLUSH_TIME + MIN_SUFFIX;
+    private static final String MEMTABLE_FLUSH_TIME_MAX = MEMTABLE_FLUSH_TIME + MAX_SUFFIX;
+    private static final String WRITE_STALL_DURATION = "write-stall-duration";
+    private static final String BLOCK_CACHE_DATA_HIT_RATIO = "block-cache-data-hit" + RATIO_SUFFIX;
+    private static final String BLOCK_CACHE_INDEX_HIT_RATIO = "block-cache-index-hit" + RATIO_SUFFIX;
+    private static final String BLOCK_CACHE_FILTER_HIT_RATIO = "block-cache-filter-hit" + RATIO_SUFFIX;
+    private static final String BYTES_READ_DURING_COMPACTION = "bytes-read-compaction";
+    private static final String BYTES_WRITTEN_DURING_COMPACTION = "bytes-written-compaction";
+    private static final String COMPACTION_TIME = "compaction-time";
+    private static final String COMPACTION_TIME_AVG = COMPACTION_TIME + AVG_SUFFIX;
+    private static final String COMPACTION_TIME_MIN = COMPACTION_TIME + MIN_SUFFIX;
+    private static final String COMPACTION_TIME_MAX = COMPACTION_TIME + MAX_SUFFIX;
+    private static final String NUMBER_OF_OPEN_FILES = "number-open-files";
+    private static final String NUMBER_OF_FILE_ERRORS = "number-file-errors";
+
+    private static final String BYTES_WRITTEN_TO_DB_RATE_DESCRIPTION =
+        "Average number of bytes written per second to the RocksDB state store";
+    private static final String BYTES_WRITTEN_TO_DB_TOTAL_DESCRIPTION =
+        "Total number of bytes written to the RocksDB state store";
+    private static final String BYTES_READ_FROM_DB_RATE_DESCRIPTION =
+        "Average number of bytes read per second from the RocksDB state store";
+    private static final String BYTES_READ_FROM_DB_TOTAL_DESCRIPTION =
+        "Total number of bytes read from the RocksDB state store";
+    private static final String MEMTABLE_BYTES_FLUSHED_RATE_DESCRIPTION =
+        "Average number of bytes flushed per second from the memtable to disk";
+    private static final String MEMTABLE_BYTES_FLUSHED_TOTAL_DESCRIPTION =
+        "Total number of bytes flushed from the memtable to disk";
+    private static final String MEMTABLE_HIT_RATIO_DESCRIPTION =
+        "Ratio of memtable hits relative to all lookups to the memtable";
+    private static final String MEMTABLE_FLUSH_TIME_AVG_DESCRIPTION =
+        "Average time spent on flushing the memtable to disk in ms";
+    private static final String MEMTABLE_FLUSH_TIME_MIN_DESCRIPTION =
+        "Minimum time spent on flushing the memtable to disk in ms";
+    private static final String MEMTABLE_FLUSH_TIME_MAX_DESCRIPTION =
+        "Maximum time spent on flushing the memtable to disk in ms";
+    private static final String WRITE_STALL_DURATION_AVG_DESCRIPTION = "Average duration of write stalls in ms";
+    private static final String WRITE_STALL_DURATION_TOTAL_DESCRIPTION = "Total duration of write stalls in ms";
+    private static final String BLOCK_CACHE_DATA_HIT_RATIO_DESCRIPTION =
+        "Ratio of block cache hits for data relative to all lookups for data to the block cache";
+    private static final String BLOCK_CACHE_INDEX_HIT_RATIO_DESCRIPTION =
+        "Ratio of block cache hits for indexes relative to all lookups for indexes to the block cache";
+    private static final String BLOCK_CACHE_FILTER_HIT_RATIO_DESCRIPTION =
+        "Ratio of block cache hits for filters relative to all lookups for filters to the block cache";
+    private static final String BYTES_READ_DURING_COMPACTION_DESCRIPTION =
+        "Average number of bytes read per second during compaction";
+    private static final String BYTES_WRITTEN_DURING_COMPACTION_DESCRIPTION =
+        "Average number of bytes written per second during compaction";
+    private static final String COMPACTION_TIME_AVG_DESCRIPTION = "Average time spent on compaction in ms";
+    private static final String COMPACTION_TIME_MIN_DESCRIPTION = "Minimum time spent on compaction in ms";
+    private static final String COMPACTION_TIME_MAX_DESCRIPTION = "Maximum time spent on compaction in ms";
+    private static final String NUMBER_OF_OPEN_FILES_DESCRIPTION = "Number of currently open files";
+    private static final String NUMBER_OF_FILE_ERRORS_DESCRIPTION = "Total number of file errors occurred";
+
+    public static class RocksDBMetricContext {
+        private final String taskName;
+        private final String storeType;
+        private final String storeName;
+
+        public RocksDBMetricContext(final String taskName, final String storeType, final String storeName) {
+            this.taskName = taskName;
+            this.storeType = "rocksdb-" + storeType;
+            this.storeName = storeName;
+        }
+
+        public String taskName() {
+            return taskName;
+        }
+        public String storeType() {
+            return storeType;
+        }
+        public String storeName() {
+            return storeName;
+        }
+    }
+
+    public static Sensor bytesWrittenToDatabaseSensor(final StreamsMetricsImpl streamsMetrics,
+                                                      final RocksDBMetricContext metricContext) {
+        final Sensor sensor = createSensor(streamsMetrics, metricContext, BYTES_WRITTEN_TO_DB);
+        addRateOfSumAndSumMetricsToSensor(
+            sensor,
+            STATE_LEVEL_GROUP,
+            streamsMetrics
+                .storeLevelTagMap(metricContext.taskName(), metricContext.storeType(), metricContext.storeName()),
+            BYTES_WRITTEN_TO_DB,
+            BYTES_WRITTEN_TO_DB_RATE_DESCRIPTION,
+            BYTES_WRITTEN_TO_DB_TOTAL_DESCRIPTION
+        );
+        return sensor;
+    }
+
+    public static Sensor bytesReadFromDatabaseSensor(final StreamsMetricsImpl streamsMetrics,
+                                                     final RocksDBMetricContext metricContext) {
+        final Sensor sensor = createSensor(streamsMetrics, metricContext, BYTES_READ_FROM_DB);
+        addRateOfSumAndSumMetricsToSensor(
+            sensor,
+            STATE_LEVEL_GROUP,
+            streamsMetrics
+                .storeLevelTagMap(metricContext.taskName(), metricContext.storeType(), metricContext.storeName()),
+            BYTES_READ_FROM_DB,
+            BYTES_READ_FROM_DB_RATE_DESCRIPTION,
+            BYTES_READ_FROM_DB_TOTAL_DESCRIPTION
+        );
+        return sensor;
+    }
+
+    public static Sensor memtableBytesFlushedSensor(final StreamsMetricsImpl streamsMetrics,
+                                                    final RocksDBMetricContext metricContext) {
+        final Sensor sensor = createSensor(streamsMetrics, metricContext, MEMTABLE_BYTES_FLUSHED);
+        addRateOfSumAndSumMetricsToSensor(
+            sensor,
+            STATE_LEVEL_GROUP,
+            streamsMetrics
+                .storeLevelTagMap(metricContext.taskName(), metricContext.storeType(), metricContext.storeName()),
+            MEMTABLE_BYTES_FLUSHED,
+            MEMTABLE_BYTES_FLUSHED_RATE_DESCRIPTION,
+            MEMTABLE_BYTES_FLUSHED_TOTAL_DESCRIPTION
+        );
+        return sensor;
+    }
+
+    public static Sensor memtableHitRatioSensor(final StreamsMetricsImpl streamsMetrics,
+                                                final RocksDBMetricContext metricContext) {
+        final Sensor sensor = createSensor(streamsMetrics, metricContext, MEMTABLE_HIT_RATIO);
+        addValueMetricToSensor(
+            sensor,
+            STATE_LEVEL_GROUP,
+            streamsMetrics
+                .storeLevelTagMap(metricContext.taskName(), metricContext.storeType(), metricContext.storeName()),
+            MEMTABLE_HIT_RATIO,
+            MEMTABLE_HIT_RATIO_DESCRIPTION
+        );
+        return sensor;
+    }
+
+    public static Sensor memtableAvgFlushTimeSensor(final StreamsMetricsImpl streamsMetrics,
+                                                    final RocksDBMetricContext metricContext) {
+        final Sensor sensor = createSensor(streamsMetrics, metricContext, MEMTABLE_FLUSH_TIME_AVG);
+        addValueMetricToSensor(
+            sensor,
+            STATE_LEVEL_GROUP,
+            streamsMetrics
+                .storeLevelTagMap(metricContext.taskName(), metricContext.storeType(), metricContext.storeName()),
+            MEMTABLE_FLUSH_TIME_AVG,
+            MEMTABLE_FLUSH_TIME_AVG_DESCRIPTION
+        );
+        return sensor;
+    }
+
+    public static Sensor memtableMinFlushTimeSensor(final StreamsMetricsImpl streamsMetrics,
+                                                    final RocksDBMetricContext metricContext) {
+        final Sensor sensor = createSensor(streamsMetrics, metricContext, MEMTABLE_FLUSH_TIME_MIN);
+        addValueMetricToSensor(
+            sensor,
+            STATE_LEVEL_GROUP,
+            streamsMetrics
+                .storeLevelTagMap(metricContext.taskName(), metricContext.storeType(), metricContext.storeName()),
+            MEMTABLE_FLUSH_TIME_MIN,
+            MEMTABLE_FLUSH_TIME_MIN_DESCRIPTION
+        );
+        return sensor;
+    }
+
+    public static Sensor memtableMaxFlushTimeSensor(final StreamsMetricsImpl streamsMetrics,
+                                                    final RocksDBMetricContext metricContext) {
+        final Sensor sensor = createSensor(streamsMetrics, metricContext, MEMTABLE_FLUSH_TIME_MAX);
+        addValueMetricToSensor(
+            sensor,
+            STATE_LEVEL_GROUP,
+            streamsMetrics
+                .storeLevelTagMap(metricContext.taskName(), metricContext.storeType(), metricContext.storeName()),
+            MEMTABLE_FLUSH_TIME_MAX,
+            MEMTABLE_FLUSH_TIME_MAX_DESCRIPTION
+        );
+        return sensor;
+    }
+
+    public static Sensor writeStallDurationSensor(final StreamsMetricsImpl streamsMetrics,
+                                                  final RocksDBMetricContext metricContext) {
+        final Sensor sensor = createSensor(streamsMetrics, metricContext, WRITE_STALL_DURATION);
+        addAvgAndSumMetricsToSensor(
+            sensor,
+            STATE_LEVEL_GROUP,
+            streamsMetrics
+                .storeLevelTagMap(metricContext.taskName(), metricContext.storeType(), metricContext.storeName()),
+            WRITE_STALL_DURATION,
+            WRITE_STALL_DURATION_AVG_DESCRIPTION,
+            WRITE_STALL_DURATION_TOTAL_DESCRIPTION
+        );
+        return sensor;
+    }
+
+    public static Sensor blockCacheDataHitRatioSensor(final StreamsMetricsImpl streamsMetrics,
+                                                      final RocksDBMetricContext metricContext) {
+        final Sensor sensor = createSensor(streamsMetrics, metricContext, BLOCK_CACHE_DATA_HIT_RATIO);
+        addValueMetricToSensor(
+            sensor,
+            STATE_LEVEL_GROUP,
+            streamsMetrics
+                .storeLevelTagMap(metricContext.taskName(), metricContext.storeType(), metricContext.storeName()),
+            BLOCK_CACHE_DATA_HIT_RATIO,
+            BLOCK_CACHE_DATA_HIT_RATIO_DESCRIPTION
+        );
+        return sensor;
+    }
+
+    public static Sensor blockCacheIndexHitRatioSensor(final StreamsMetricsImpl streamsMetrics,
+                                                       final RocksDBMetricContext metricContext) {
+        final Sensor sensor = createSensor(streamsMetrics, metricContext, BLOCK_CACHE_INDEX_HIT_RATIO);
+        addValueMetricToSensor(
+            sensor,
+            STATE_LEVEL_GROUP,
+            streamsMetrics.storeLevelTagMap(metricContext.taskName(), metricContext.storeType(), metricContext.storeName()),
+            BLOCK_CACHE_INDEX_HIT_RATIO,
+            BLOCK_CACHE_INDEX_HIT_RATIO_DESCRIPTION
+        );
+        return sensor;
+    }
+
+    public static Sensor blockCacheFilterHitRatioSensor(final StreamsMetricsImpl streamsMetrics,
+                                                        final RocksDBMetricContext metricContext) {
+        final Sensor sensor = createSensor(streamsMetrics, metricContext, BLOCK_CACHE_FILTER_HIT_RATIO);
+        addValueMetricToSensor(
+            sensor,
+            STATE_LEVEL_GROUP,
+            streamsMetrics
+                .storeLevelTagMap(metricContext.taskName(), metricContext.storeType(), metricContext.storeName()),
+            BLOCK_CACHE_FILTER_HIT_RATIO,
+            BLOCK_CACHE_FILTER_HIT_RATIO_DESCRIPTION
+        );
+        return sensor;
+    }
+
+    public static Sensor bytesReadDuringCompactionSensor(final StreamsMetricsImpl streamsMetrics,
+                                                         final RocksDBMetricContext metricContext) {
+        final Sensor sensor = createSensor(streamsMetrics, metricContext, BYTES_READ_DURING_COMPACTION);
+        addRateOfSumMetricToSensor(
+            sensor,
+            STATE_LEVEL_GROUP,
+            streamsMetrics
+                .storeLevelTagMap(metricContext.taskName(), metricContext.storeType(), metricContext.storeName()),
+            BYTES_READ_DURING_COMPACTION,
+            BYTES_READ_DURING_COMPACTION_DESCRIPTION
+        );
+        return sensor;
+    }
+
+    public static Sensor bytesWrittenDuringCompactionSensor(final StreamsMetricsImpl streamsMetrics,
+                                                            final RocksDBMetricContext metricContext) {
+        final Sensor sensor = createSensor(streamsMetrics, metricContext, BYTES_WRITTEN_DURING_COMPACTION);
+        addRateOfSumMetricToSensor(
+            sensor,
+            STATE_LEVEL_GROUP,
+            streamsMetrics
+                .storeLevelTagMap(metricContext.taskName(), metricContext.storeType(), metricContext.storeName()),
+            BYTES_WRITTEN_DURING_COMPACTION,
+            BYTES_WRITTEN_DURING_COMPACTION_DESCRIPTION
+        );
+        return sensor;
+    }
+
+    public static Sensor compactionTimeAvgSensor(final StreamsMetricsImpl streamsMetrics,
+                                                 final RocksDBMetricContext metricContext) {
+        final Sensor sensor = createSensor(streamsMetrics, metricContext, COMPACTION_TIME_AVG);
+        addValueMetricToSensor(
+            sensor,
+            STATE_LEVEL_GROUP,
+            streamsMetrics
+                .storeLevelTagMap(metricContext.taskName(), metricContext.storeType(), metricContext.storeName()),
+            COMPACTION_TIME_AVG,
+            COMPACTION_TIME_AVG_DESCRIPTION
+        );
+        return sensor;
+    }
+
+    public static Sensor compactionTimeMinSensor(final StreamsMetricsImpl streamsMetrics,
+                                                 final RocksDBMetricContext metricContext) {
+        final Sensor sensor = createSensor(streamsMetrics, metricContext, COMPACTION_TIME_MIN);
+        addValueMetricToSensor(
+            sensor,
+            STATE_LEVEL_GROUP,
+            streamsMetrics
+                .storeLevelTagMap(metricContext.taskName(), metricContext.storeType(), metricContext.storeName()),
+            COMPACTION_TIME_MIN,
+            COMPACTION_TIME_MIN_DESCRIPTION
+        );
+        return sensor;
+    }
+
+    public static Sensor compactionTimeMaxSensor(final StreamsMetricsImpl streamsMetrics,
+                                                 final RocksDBMetricContext metricContext) {
+        final Sensor sensor = createSensor(streamsMetrics, metricContext, COMPACTION_TIME_MAX);
+        addValueMetricToSensor(
+            sensor,
+            STATE_LEVEL_GROUP,
+            streamsMetrics
+                .storeLevelTagMap(metricContext.taskName(), metricContext.storeType(), metricContext.storeName()),
+            COMPACTION_TIME_MAX,
+            COMPACTION_TIME_MAX_DESCRIPTION
+        );
+        return sensor;
+    }
+
+    public static Sensor numberOfOpenFilesSensor(final StreamsMetricsImpl streamsMetrics,
+                                                 final RocksDBMetricContext metricContext) {
+        final Sensor sensor = createSensor(streamsMetrics, metricContext, NUMBER_OF_OPEN_FILES);
+        addValueMetricToSensor(
+            sensor,
+            STATE_LEVEL_GROUP,
+            streamsMetrics
+                .storeLevelTagMap(metricContext.taskName(), metricContext.storeType(), metricContext.storeName()),
+            NUMBER_OF_OPEN_FILES,
+            NUMBER_OF_OPEN_FILES_DESCRIPTION
+        );
+        return sensor;
+    }
+
+    public static Sensor numberOfFileErrorsSensor(final StreamsMetricsImpl streamsMetrics,
+                                                  final RocksDBMetricContext metricContext) {
+        final Sensor sensor = createSensor(streamsMetrics, metricContext, NUMBER_OF_FILE_ERRORS);
+        addSumMetricToSensor(
+            sensor,
+            STATE_LEVEL_GROUP,
+            streamsMetrics
+                .storeLevelTagMap(metricContext.taskName(), metricContext.storeType(), metricContext.storeName()),
+            NUMBER_OF_FILE_ERRORS,
+            NUMBER_OF_FILE_ERRORS_DESCRIPTION
+        );
+        return sensor;
+    }
+
+    private static Sensor createSensor(final StreamsMetricsImpl streamsMetrics,
+                                       final RocksDBMetricContext metricContext,
+                                       final String sensorName) {
+        return streamsMetrics.storeLevelSensor(
+            metricContext.taskName(),
+            metricContext.storeName(),
+            sensorName,
+            RecordingLevel.DEBUG);
+    }
+}
\ No newline at end of file
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java
index f381a58c1233e..678d9f30635ab 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.streams.processor.internals.metrics;
 
-
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.metrics.KafkaMetric;
 import org.apache.kafka.common.metrics.MetricConfig;
@@ -28,6 +27,7 @@
 import org.easymock.EasyMockSupport;
 import org.junit.Test;
 
+import java.time.Duration;
 import java.util.Collections;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
@@ -37,6 +37,8 @@
 import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.PROCESSOR_NODE_METRICS_GROUP;
 import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addAvgMaxLatency;
 import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addInvocationRateAndCount;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -49,6 +51,15 @@ public class StreamsMetricsImplTest extends EasyMockSupport {
     private final static String SENSOR_NAME_DELIMITER = ".s.";
     private final static String INTERNAL_PREFIX = "internal";
 
+    private final Metrics metrics = new Metrics();
+    private final Sensor sensor = metrics.sensor("dummy");
+    private final String metricNamePrefix = "metric";
+    private final String group = "group";
+    private final Map tags = mkMap(mkEntry("tag", "value"));
+    private final String description1 = "description number one";
+    private final String description2 = "description number two";
+    private final MockTime time = new MockTime(0);
+
     @Test
     public void shouldGetThreadLevelSensor() {
         final Metrics metrics = mock(Metrics.class);
@@ -230,6 +241,106 @@ public void testTotalMetricDoesntDecrease() {
             assertEquals(i, Math.round(totalMetric.measurable().measure(config, time.milliseconds())));
             sensor.record(latency, time.milliseconds());
         }
+    }
+
+    @Test
+    public void shouldGetStoreLevelTagMap() {
+        final String threadName = "test-thread";
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, threadName);
+        final String taskName = "test-task";
+        final String storeType = "remote-window";
+        final String storeName = "window-keeper";
+
+        final Map tagMap = streamsMetrics.storeLevelTagMap(taskName, storeType, storeName);
 
+        assertThat(tagMap.size(), equalTo(3));
+        assertThat(tagMap.get(StreamsMetricsImpl.THREAD_ID_TAG), equalTo(threadName));
+        assertThat(tagMap.get(StreamsMetricsImpl.TASK_ID_TAG), equalTo(taskName));
+        assertThat(tagMap.get(storeType + "-" + StreamsMetricsImpl.STORE_ID_TAG), equalTo(storeName));
+    }
+
+    @Test
+    public void shouldAddAmountRateAndSum() {
+        StreamsMetricsImpl
+            .addRateOfSumAndSumMetricsToSensor(sensor, group, tags, metricNamePrefix, description1, description2);
+
+        final double valueToRecord1 = 18.0;
+        final double valueToRecord2 = 72.0;
+        final long defaultWindowSizeInSeconds = Duration.ofMillis(new MetricConfig().timeWindowMs()).getSeconds();
+        final double expectedRateMetricValue = (valueToRecord1 + valueToRecord2) / defaultWindowSizeInSeconds;
+        verifyMetric(metricNamePrefix + "-rate", description1, valueToRecord1, valueToRecord2, expectedRateMetricValue);
+        final double expectedSumMetricValue = 2 * valueToRecord1 + 2 * valueToRecord2; // values are recorded once for each metric verification
+        verifyMetric(metricNamePrefix + "-total", description2, valueToRecord1, valueToRecord2, expectedSumMetricValue);
+        assertThat(metrics.metrics().size(), equalTo(2 + 1)); // one metric is added automatically in the constructor of Metrics
+    }
+
+    @Test
+    public void shouldAddSum() {
+        StreamsMetricsImpl.addSumMetricToSensor(sensor, group, tags, metricNamePrefix, description1);
+
+        final double valueToRecord1 = 18.0;
+        final double valueToRecord2 = 42.0;
+        final double expectedSumMetricValue = valueToRecord1 + valueToRecord2;
+        verifyMetric(metricNamePrefix + "-total", description1, valueToRecord1, valueToRecord2, expectedSumMetricValue);
+        assertThat(metrics.metrics().size(), equalTo(1 + 1)); // one metric is added automatically in the constructor of Metrics
+    }
+
+    @Test
+    public void shouldAddAmountRate() {
+        StreamsMetricsImpl.addRateOfSumMetricToSensor(sensor, group, tags, metricNamePrefix, description1);
+
+        final double valueToRecord1 = 18.0;
+        final double valueToRecord2 = 72.0;
+        final long defaultWindowSizeInSeconds = Duration.ofMillis(new MetricConfig().timeWindowMs()).getSeconds();
+        final double expectedRateMetricValue = (valueToRecord1 + valueToRecord2) / defaultWindowSizeInSeconds;
+        verifyMetric(metricNamePrefix + "-rate", description1, valueToRecord1, valueToRecord2, expectedRateMetricValue);
+        assertThat(metrics.metrics().size(), equalTo(1 + 1)); // one metric is added automatically in the constructor of Metrics
+    }
+
+    @Test
+    public void shouldAddValue() {
+        StreamsMetricsImpl.addValueMetricToSensor(sensor, group, tags, metricNamePrefix, description1);
+
+        final KafkaMetric ratioMetric = metrics.metric(new MetricName(metricNamePrefix, group, description1, tags));
+        assertThat(ratioMetric, is(notNullValue()));
+        final MetricConfig metricConfig = new MetricConfig();
+        final double value1 = 42.0;
+        sensor.record(value1);
+        assertThat(ratioMetric.measurable().measure(metricConfig, time.milliseconds()), equalTo(42.0));
+        final double value2 = 18.0;
+        sensor.record(value2);
+        assertThat(ratioMetric.measurable().measure(metricConfig, time.milliseconds()), equalTo(18.0));
+        assertThat(metrics.metrics().size(), equalTo(1 + 1)); // one metric is added automatically in the constructor of Metrics
+    }
+
+    @Test
+    public void shouldAddAvgAndTotalMetricsToSensor() {
+        StreamsMetricsImpl
+            .addAvgAndSumMetricsToSensor(sensor, group, tags, metricNamePrefix, description1, description2);
+
+        final double valueToRecord1 = 18.0;
+        final double valueToRecord2 = 42.0;
+        final double expectedAvgMetricValue = (valueToRecord1 + valueToRecord2) / 2;
+        verifyMetric(metricNamePrefix + "-avg", description1, valueToRecord1, valueToRecord2, expectedAvgMetricValue);
+        final double expectedSumMetricValue = 2 * valueToRecord1 + 2 * valueToRecord2; // values are recorded once for each metric verification
+        verifyMetric(metricNamePrefix + "-total", description2, valueToRecord1, valueToRecord2, expectedSumMetricValue);
+        assertThat(metrics.metrics().size(), equalTo(2 + 1)); // one metric is added automatically in the constructor of Metrics
+    }
+
+    private void verifyMetric(final String name,
+                              final String description,
+                              final double valueToRecord1,
+                              final double valueToRecord2,
+                              final double expectedMetricValue) {
+        final KafkaMetric metric = metrics
+            .metric(new MetricName(name, group, description, tags));
+        assertThat(metric, is(notNullValue()));
+        assertThat(metric.metricName().description(), equalTo(description));
+        sensor.record(valueToRecord1, time.milliseconds());
+        sensor.record(valueToRecord2, time.milliseconds());
+        assertThat(
+            metric.measurable().measure(new MetricConfig(), time.milliseconds()),
+            equalTo(expectedMetricValue)
+        );
     }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsTest.java
new file mode 100644
index 0000000000000..6c8b9d873ff85
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsTest.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals.metrics;
+
+import org.apache.kafka.common.metrics.Metrics;
+import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.metrics.Sensor.RecordingLevel;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.state.internals.metrics.RocksDBMetrics.RocksDBMetricContext;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.util.Collections;
+import java.util.Map;
+
+import static org.easymock.EasyMock.expect;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.powermock.api.easymock.PowerMock.createStrictMock;
+import static org.powermock.api.easymock.PowerMock.mockStatic;
+import static org.powermock.api.easymock.PowerMock.replay;
+import static org.powermock.api.easymock.PowerMock.replayAll;
+import static org.powermock.api.easymock.PowerMock.verify;
+import static org.powermock.api.easymock.PowerMock.verifyAll;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(StreamsMetricsImpl.class)
+public class RocksDBMetricsTest {
+
+    private static final String STATE_LEVEL_GROUP = "stream-state-metrics";
+    private static final String STORE_TYPE_PREFIX = "rocksdb-";
+    private final String taskName = "task";
+    private final String storeType = "storeType";
+    private final String storeName = "store";
+    private final Metrics metrics = new Metrics();
+    private final Sensor sensor = metrics.sensor("dummy");
+    private final StreamsMetricsImpl streamsMetrics = createStrictMock(StreamsMetricsImpl.class);
+    private final Map tags = Collections.singletonMap("hello", "world");
+
+    private interface SensorCreator {
+        Sensor sensor(final StreamsMetricsImpl streamsMetrics, final RocksDBMetricContext metricContext);
+    }
+
+    @Test
+    public void shouldGetBytesWrittenSensor() {
+        final String metricNamePrefix = "bytes-written";
+        final String descriptionOfTotal = "Total number of bytes written to the RocksDB state store";
+        final String descriptionOfRate = "Average number of bytes written per second to the RocksDB state store";
+        verifyRateAndTotalSensor(
+            metricNamePrefix,
+            descriptionOfTotal,
+            descriptionOfRate,
+            RocksDBMetrics::bytesWrittenToDatabaseSensor
+        );
+    }
+
+    @Test
+    public void shouldGetBytesReadSensor() {
+        final String metricNamePrefix = "bytes-read";
+        final String descriptionOfTotal = "Total number of bytes read from the RocksDB state store";
+        final String descriptionOfRate = "Average number of bytes read per second from the RocksDB state store";
+        verifyRateAndTotalSensor(
+            metricNamePrefix,
+            descriptionOfTotal,
+            descriptionOfRate,
+            RocksDBMetrics::bytesReadFromDatabaseSensor
+        );
+    }
+
+    @Test
+    public void shouldGetMemtableHitRatioSensor() {
+        final String metricNamePrefix = "memtable-hit-ratio";
+        final String description = "Ratio of memtable hits relative to all lookups to the memtable";
+        verifyValueSensor(metricNamePrefix, description, RocksDBMetrics::memtableHitRatioSensor);
+    }
+
+    @Test
+    public void shouldGetMemtableBytesFlushedSensor() {
+        final String metricNamePrefix = "memtable-bytes-flushed";
+        final String descriptionOfTotal = "Total number of bytes flushed from the memtable to disk";
+        final String descriptionOfRate = "Average number of bytes flushed per second from the memtable to disk";
+        verifyRateAndTotalSensor(
+            metricNamePrefix,
+            descriptionOfTotal,
+            descriptionOfRate,
+            RocksDBMetrics::memtableBytesFlushedSensor
+        );
+    }
+
+    @Test
+    public void shouldGetMemtableAvgFlushTimeSensor() {
+        final String metricNamePrefix = "memtable-flush-time-avg";
+        final String description = "Average time spent on flushing the memtable to disk in ms";
+        verifyValueSensor(metricNamePrefix, description, RocksDBMetrics::memtableAvgFlushTimeSensor);
+    }
+
+    @Test
+    public void shouldGetMemtableMinFlushTimeSensor() {
+        final String metricNamePrefix = "memtable-flush-time-min";
+        final String description = "Minimum time spent on flushing the memtable to disk in ms";
+        verifyValueSensor(metricNamePrefix, description, RocksDBMetrics::memtableMinFlushTimeSensor);
+    }
+
+    @Test
+    public void shouldGetMemtableMaxFlushTimeSensor() {
+        final String metricNamePrefix = "memtable-flush-time-max";
+        final String description = "Maximum time spent on flushing the memtable to disk in ms";
+        verifyValueSensor(metricNamePrefix, description, RocksDBMetrics::memtableMaxFlushTimeSensor);
+    }
+
+    @Test
+    public void shouldGetWriteStallDurationSensor() {
+        final String metricNamePrefix = "write-stall-duration";
+        final String descriptionOfAvg = "Average duration of write stalls in ms";
+        final String descriptionOfTotal = "Total duration of write stalls in ms";
+        setupStreamsMetricsMock(metricNamePrefix);
+        StreamsMetricsImpl.addAvgAndSumMetricsToSensor(
+            sensor,
+            STATE_LEVEL_GROUP,
+            tags,
+            metricNamePrefix,
+            descriptionOfAvg,
+            descriptionOfTotal
+        );
+
+        replayCallAndVerify(RocksDBMetrics::writeStallDurationSensor);
+    }
+
+    @Test
+    public void shouldGetBlockCacheDataHitRatioSensor() {
+        final String metricNamePrefix = "block-cache-data-hit-ratio";
+        final String description =
+            "Ratio of block cache hits for data relative to all lookups for data to the block cache";
+        verifyValueSensor(metricNamePrefix, description, RocksDBMetrics::blockCacheDataHitRatioSensor);
+    }
+
+    @Test
+    public void shouldGetBlockCacheIndexHitRatioSensor() {
+        final String metricNamePrefix = "block-cache-index-hit-ratio";
+        final String description =
+            "Ratio of block cache hits for indexes relative to all lookups for indexes to the block cache";
+        verifyValueSensor(metricNamePrefix, description, RocksDBMetrics::blockCacheIndexHitRatioSensor);
+    }
+
+    @Test
+    public void shouldGetBlockCacheFilterHitRatioSensor() {
+        final String metricNamePrefix = "block-cache-filter-hit-ratio";
+        final String description =
+            "Ratio of block cache hits for filters relative to all lookups for filters to the block cache";
+        verifyValueSensor(metricNamePrefix, description, RocksDBMetrics::blockCacheFilterHitRatioSensor);
+    }
+
+    @Test
+    public void shouldGetBytesReadDuringCompactionSensor() {
+        final String metricNamePrefix = "bytes-read-compaction";
+        final String description = "Average number of bytes read per second during compaction";
+        verifyRateSensor(metricNamePrefix, description, RocksDBMetrics::bytesReadDuringCompactionSensor);
+    }
+
+    @Test
+    public void shouldGetBytesWrittenDuringCompactionSensor() {
+        final String metricNamePrefix = "bytes-written-compaction";
+        final String description = "Average number of bytes written per second during compaction";
+        verifyRateSensor(metricNamePrefix, description, RocksDBMetrics::bytesWrittenDuringCompactionSensor);
+    }
+
+    @Test
+    public void shouldGetCompactionTimeAvgSensor() {
+        final String metricNamePrefix = "compaction-time-avg";
+        final String description = "Average time spent on compaction in ms";
+        verifyValueSensor(metricNamePrefix, description, RocksDBMetrics::compactionTimeAvgSensor);
+    }
+
+    @Test
+    public void shouldGetCompactionTimeMinSensor() {
+        final String metricNamePrefix = "compaction-time-min";
+        final String description = "Minimum time spent on compaction in ms";
+        verifyValueSensor(metricNamePrefix, description, RocksDBMetrics::compactionTimeMinSensor);
+    }
+
+    @Test
+    public void shouldGetCompactionTimeMaxSensor() {
+        final String metricNamePrefix = "compaction-time-max";
+        final String description = "Maximum time spent on compaction in ms";
+        verifyValueSensor(metricNamePrefix, description, RocksDBMetrics::compactionTimeMaxSensor);
+    }
+
+    @Test
+    public void shouldGetNumberOfOpenFilesSensor() {
+        final String metricNamePrefix = "number-open-files";
+        final String description = "Number of currently open files";
+        verifyValueSensor(metricNamePrefix, description, RocksDBMetrics::numberOfOpenFilesSensor);
+    }
+
+    @Test
+    public void shouldGetNumberOfFilesErrors() {
+        final String metricNamePrefix = "number-file-errors";
+        final String description = "Total number of file errors occurred";
+        setupStreamsMetricsMock(metricNamePrefix);
+        StreamsMetricsImpl.addSumMetricToSensor(sensor, STATE_LEVEL_GROUP, tags, metricNamePrefix, description);
+
+        replayCallAndVerify(RocksDBMetrics::numberOfFileErrorsSensor);
+    }
+
+    private void verifyRateAndTotalSensor(final String metricNamePrefix,
+                                          final String descriptionOfTotal,
+                                          final String descriptionOfRate,
+                                          final SensorCreator sensorCreator) {
+        setupStreamsMetricsMock(metricNamePrefix);
+        StreamsMetricsImpl.addRateOfSumAndSumMetricsToSensor(
+            sensor,
+            STATE_LEVEL_GROUP,
+            tags,
+            metricNamePrefix,
+            descriptionOfRate,
+            descriptionOfTotal
+        );
+
+        replayCallAndVerify(sensorCreator);
+    }
+
+    private void verifyRateSensor(final String metricNamePrefix,
+                                  final String description,
+                                  final SensorCreator sensorCreator) {
+        setupStreamsMetricsMock(metricNamePrefix);
+        StreamsMetricsImpl.addRateOfSumMetricToSensor(sensor, STATE_LEVEL_GROUP, tags, metricNamePrefix, description);
+
+        replayCallAndVerify(sensorCreator);
+    }
+
+    private void verifyValueSensor(final String metricNamePrefix,
+                                   final String description,
+                                   final SensorCreator sensorCreator) {
+        setupStreamsMetricsMock(metricNamePrefix);
+        StreamsMetricsImpl.addValueMetricToSensor(sensor, STATE_LEVEL_GROUP, tags, metricNamePrefix, description);
+
+        replayCallAndVerify(sensorCreator);
+    }
+
+    private void setupStreamsMetricsMock(final String metricNamePrefix) {
+        mockStatic(StreamsMetricsImpl.class);
+        expect(streamsMetrics.storeLevelSensor(
+            taskName,
+            storeName,
+            metricNamePrefix,
+            RecordingLevel.DEBUG
+        )).andReturn(sensor);
+        expect(streamsMetrics.storeLevelTagMap(
+            taskName,
+            STORE_TYPE_PREFIX + storeType,
+            storeName
+        )).andReturn(tags);
+    }
+
+    private void replayCallAndVerify(final SensorCreator sensorCreator) {
+        replayAll();
+        replay(StreamsMetricsImpl.class);
+
+        final Sensor sensor =
+            sensorCreator.sensor(streamsMetrics, new RocksDBMetricContext(taskName, storeType, storeName));
+
+        verifyAll();
+        verify(StreamsMetricsImpl.class);
+
+        assertThat(sensor, is(this.sensor));
+    }
+}

From a99e0111114d1cb8c762494ac195cf84e6425bb3 Mon Sep 17 00:00:00 2001
From: Stanislav Kozlovski 
Date: Fri, 2 Aug 2019 11:51:35 -0700
Subject: [PATCH 0501/1071] KAFKA-7800; Dynamic log levels admin API (KIP-412)


What
----


References
----------
[**KIP-412**](https://cwiki.apache.org/confluence/display/KAFKA/KIP-412%3A+Extend+Admin+API+to+support+dynamic+application+log+levels)
[**KAFKA-7800**](https://issues.apache.org/jira/browse/KAFKA-7800)
[**Discussion Thread**](http://mail-archives.apache.org/mod_mbox/kafka-dev/201901.mbox/%3CCANZZNGyeVw8q%3Dx9uOQS-18wL3FEmnOwpBnpJ9x3iMLdXY3gEug%40mail.gmail.com%3E)
[**Vote Thread**](http://mail-archives.apache.org/mod_mbox/kafka-dev/201902.mbox/%3CCANZZNGzpTJg5YX1Gpe5S%3DHSr%3DXGvmxvYLTdA3jWq_qwH-UvorQ%40mail.gmail.com%3E)



Test&Review
------------
Test cases covered:
* DescribeConfigs
* Alter the log level with and without validateOnly, validate the results with DescribeConfigs

Open questions / Follow ups
--------------------------
If you're a reviewer, I'd appreciate your thoughts on these questions I have open:
1. Should we add synchronization to the Log4jController methods? - Seems like we don't get much value from it
2. Should we instantiate a new Log4jController instead of it having static methods? - All operations are stateless, so I thought static methods would do well
3. A logger which does not have a set value returns "null" (as seen in the unit tests). Should we just return the Root logger's level?

Author: Stanislav Kozlovski 

Reviewers: Gwen Shapira

Closes #6903 from stanislavkozlovski/KAFKA-7800-dynamic-log-levels-admin-ap
---
 .../kafka/clients/admin/ConfigEntry.java      |   1 +
 .../kafka/clients/admin/KafkaAdminClient.java |  17 +-
 .../kafka/common/config/ConfigResource.java   |   2 +-
 .../kafka/common/config/LogLevelConfig.java   |  71 ++++++
 .../requests/DescribeConfigsResponse.java     |   3 +-
 .../scala/kafka/admin/ConfigCommand.scala     | 138 +++++++----
 .../scala/kafka/server/AdminManager.scala     |  69 +++++-
 .../main/scala/kafka/server/KafkaApis.scala   |   8 +-
 .../scala/kafka/utils/Log4jController.scala   |  92 ++++---
 .../api/AdminClientIntegrationTest.scala      | 229 +++++++++++++++++-
 .../kafka/api/AuthorizerIntegrationTest.scala |  34 ++-
 .../unit/kafka/admin/ConfigCommandTest.scala  | 164 ++++++++++++-
 12 files changed, 719 insertions(+), 109 deletions(-)
 create mode 100644 clients/src/main/java/org/apache/kafka/common/config/LogLevelConfig.java

diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ConfigEntry.java b/clients/src/main/java/org/apache/kafka/clients/admin/ConfigEntry.java
index 7775b6a744a0d..42cc6274980e7 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/ConfigEntry.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/ConfigEntry.java
@@ -189,6 +189,7 @@ public String toString() {
      */
     public enum ConfigSource {
         DYNAMIC_TOPIC_CONFIG,           // dynamic topic config that is configured for a specific topic
+        DYNAMIC_BROKER_LOGGER_CONFIG,   // dynamic broker logger config that is configured for a specific broker
         DYNAMIC_BROKER_CONFIG,          // dynamic broker config that is configured for a specific broker
         DYNAMIC_DEFAULT_BROKER_CONFIG,  // dynamic broker config that is configured as default for all brokers in the cluster
         STATIC_BROKER_CONFIG,           // static broker config provided as broker properties at start up (e.g. server.properties file)
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
index 227a03b966c7a..8092eec82fd72 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
@@ -1762,7 +1762,7 @@ public DescribeConfigsResult describeConfigs(Collection configRe
         final Collection unifiedRequestResources = new ArrayList<>(configResources.size());
 
         for (ConfigResource resource : configResources) {
-            if (resource.type() == ConfigResource.Type.BROKER && !resource.isDefault()) {
+            if (dependsOnSpecificNode(resource)) {
                 brokerFutures.put(resource, new KafkaFutureImpl<>());
                 brokerResources.add(resource);
             } else {
@@ -1887,6 +1887,9 @@ private ConfigEntry.ConfigSource configSource(DescribeConfigsResponse.ConfigSour
             case STATIC_BROKER_CONFIG:
                 configSource = ConfigEntry.ConfigSource.STATIC_BROKER_CONFIG;
                 break;
+            case DYNAMIC_BROKER_LOGGER_CONFIG:
+                configSource = ConfigEntry.ConfigSource.DYNAMIC_BROKER_LOGGER_CONFIG;
+                break;
             case DEFAULT_CONFIG:
                 configSource = ConfigEntry.ConfigSource.DEFAULT_CONFIG;
                 break;
@@ -1906,7 +1909,7 @@ public AlterConfigsResult alterConfigs(Map configs, fina
         final Collection unifiedRequestResources = new ArrayList<>();
 
         for (ConfigResource resource : configs.keySet()) {
-            if (resource.type() == ConfigResource.Type.BROKER && !resource.isDefault()) {
+            if (dependsOnSpecificNode(resource)) {
                 NodeProvider nodeProvider = new ConstantNodeIdProvider(Integer.parseInt(resource.name()));
                 allFutures.putAll(alterConfigs(configs, options, Collections.singleton(resource), nodeProvider));
             } else
@@ -1971,7 +1974,7 @@ public AlterConfigsResult incrementalAlterConfigs(Map unifiedRequestResources = new ArrayList<>();
 
         for (ConfigResource resource : configs.keySet()) {
-            if (resource.type() == ConfigResource.Type.BROKER && !resource.isDefault()) {
+            if (dependsOnSpecificNode(resource)) {
                 NodeProvider nodeProvider = new ConstantNodeIdProvider(Integer.parseInt(resource.name()));
                 allFutures.putAll(incrementalAlterConfigs(configs, options, Collections.singleton(resource), nodeProvider));
             } else
@@ -3070,4 +3073,12 @@ void handleFailure(Throwable throwable) {
 
         return new ElectLeadersResult(electionFuture);
     }
+
+    /**
+     * Returns a boolean indicating whether the resource needs to go to a specific node
+     */
+    private boolean dependsOnSpecificNode(ConfigResource resource) {
+        return (resource.type() == ConfigResource.Type.BROKER && !resource.isDefault())
+                || resource.type() == ConfigResource.Type.BROKER_LOGGER;
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/config/ConfigResource.java b/clients/src/main/java/org/apache/kafka/common/config/ConfigResource.java
index 5343a6bcdfd3d..8870238f638a1 100644
--- a/clients/src/main/java/org/apache/kafka/common/config/ConfigResource.java
+++ b/clients/src/main/java/org/apache/kafka/common/config/ConfigResource.java
@@ -33,7 +33,7 @@ public final class ConfigResource {
      * Type of resource.
      */
     public enum Type {
-        BROKER((byte) 4), TOPIC((byte) 2), UNKNOWN((byte) 0);
+        BROKER_LOGGER((byte) 8), BROKER((byte) 4), TOPIC((byte) 2), UNKNOWN((byte) 0);
 
         private static final Map TYPES = Collections.unmodifiableMap(
             Arrays.stream(values()).collect(Collectors.toMap(Type::id, Function.identity()))
diff --git a/clients/src/main/java/org/apache/kafka/common/config/LogLevelConfig.java b/clients/src/main/java/org/apache/kafka/common/config/LogLevelConfig.java
new file mode 100644
index 0000000000000..fe7e2eb6669e7
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/config/LogLevelConfig.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.common.config;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * This class holds definitions for log level configurations related to Kafka's application logging. See KIP-412 for additional information
+ */
+public class LogLevelConfig {
+    /*
+     * NOTE: DO NOT CHANGE EITHER CONFIG NAMES AS THESE ARE PART OF THE PUBLIC API AND CHANGE WILL BREAK USER CODE.
+     */
+
+    /**
+     * The FATAL level designates a very severe error
+     * that will lead the Kafka broker to abort.
+     */
+    public static final String FATAL_LOG_LEVEL = "FATAL";
+
+    /**
+     * The ERROR level designates error events that
+     * might still allow the broker to continue running.
+     */
+    public static final String ERROR_LOG_LEVEL = "ERROR";
+
+    /**
+     * The WARN level designates potentially harmful situations.
+     */
+    public static final String WARN_LOG_LEVEL = "WARN";
+
+    /**
+     * The INFO level designates informational messages
+     * that highlight normal Kafka events at a coarse-grained level
+     */
+    public static final String INFO_LOG_LEVEL = "INFO";
+
+    /**
+     * The DEBUG level designates fine-grained
+     * informational events that are most useful to debug Kafka
+     */
+    public static final String DEBUG_LOG_LEVEL = "DEBUG";
+
+    /**
+     * The TRACE level designates finer-grained
+     * informational events than the DEBUG level.
+     */
+    public static final String TRACE_LOG_LEVEL = "TRACE";
+
+    public static final Set VALID_LOG_LEVELS = new HashSet<>(Arrays.asList(
+            FATAL_LOG_LEVEL, ERROR_LOG_LEVEL, WARN_LOG_LEVEL,
+            INFO_LOG_LEVEL, DEBUG_LOG_LEVEL, TRACE_LOG_LEVEL
+    ));
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java
index 51c35d56f22dc..6d424f2f3e378 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java
@@ -179,7 +179,8 @@ public enum ConfigSource {
         DYNAMIC_BROKER_CONFIG((byte) 2),
         DYNAMIC_DEFAULT_BROKER_CONFIG((byte) 3),
         STATIC_BROKER_CONFIG((byte) 4),
-        DEFAULT_CONFIG((byte) 5);
+        DEFAULT_CONFIG((byte) 5),
+        DYNAMIC_BROKER_LOGGER_CONFIG((byte) 6);
 
         final byte id;
         private static final ConfigSource[] VALUES = values();
diff --git a/core/src/main/scala/kafka/admin/ConfigCommand.scala b/core/src/main/scala/kafka/admin/ConfigCommand.scala
index 7edc4a41045a7..781cc1a7fcb5b 100644
--- a/core/src/main/scala/kafka/admin/ConfigCommand.scala
+++ b/core/src/main/scala/kafka/admin/ConfigCommand.scala
@@ -28,8 +28,8 @@ import kafka.utils.{CommandDefaultOptions, CommandLineUtils, Exit, PasswordEncod
 import kafka.utils.Implicits._
 import kafka.zk.{AdminZkClient, KafkaZkClient}
 import org.apache.kafka.clients.CommonClientConfigs
-import org.apache.kafka.clients.admin.{Admin, AlterConfigsOptions, ConfigEntry, DescribeConfigsOptions, AdminClient => JAdminClient, Config => JConfig}
-import org.apache.kafka.common.config.ConfigResource
+import org.apache.kafka.clients.admin.{Admin, AlterConfigOp, AlterConfigsOptions, ConfigEntry, DescribeConfigsOptions, AdminClient => JAdminClient, Config => JConfig}
+import org.apache.kafka.common.config.{ConfigResource, LogLevelConfig}
 import org.apache.kafka.common.config.types.Password
 import org.apache.kafka.common.errors.InvalidConfigurationException
 import org.apache.kafka.common.security.JaasUtils
@@ -41,7 +41,6 @@ import scala.collection._
 
 
 /**
- * This script can be used to change configs for topics/clients/brokers dynamically
  * This script can be used to change configs for topics/clients/users/brokers dynamically
  * An entity described or altered by the command may be one of:
  * 
          
@@ -50,12 +49,15 @@ import scala.collection._
  *     
        •  user: --entity-type users --entity-name 
  *     
        •  : --entity-type users --entity-name  --entity-type clients --entity-name 
  *     
        •  broker: --entity-type brokers --entity-name 
+ *     
        •  broker-logger: --entity-type broker-loggers --entity-name 
  * 
        
  * --entity-default may be used instead of --entity-name when describing or altering default configuration for users and clients.
  *
  */
 object ConfigCommand extends Config {
 
+  val BrokerLoggerConfigType = "broker-loggers"
+  val BrokerSupportedConfigTypes = Seq(ConfigType.Broker, BrokerLoggerConfigType)
   val DefaultScramIterations = 4096
   // Dynamic broker configs can only be updated using the new AdminClient once brokers have started
   // so that configs may be fully validated. Prior to starting brokers, updates may be performed using
@@ -274,49 +276,61 @@ object ConfigCommand extends Config {
     val adminClient = JAdminClient.create(props)
     val entityName = if (opts.options.has(opts.entityName))
       opts.options.valueOf(opts.entityName)
-    else if (opts.options.has(opts.entityDefault))
+    else // default entity
       ""
-    else
-      throw new IllegalArgumentException("At least one of --entity-name or --entity-default must be specified with --bootstrap-server")
 
     val entityTypes = opts.options.valuesOf(opts.entityType).asScala
     if (entityTypes.size != 1)
-      throw new IllegalArgumentException("Exactly one --entity-type must be specified with --bootstrap-server")
-    if (entityTypes.head != ConfigType.Broker)
-      throw new IllegalArgumentException(s"--zookeeper option must be specified for entity-type $entityTypes")
+      throw new IllegalArgumentException(s"Exactly one --entity-type (out of ${BrokerSupportedConfigTypes.mkString(",")}) must be specified with --bootstrap-server")
 
     try {
       if (opts.options.has(opts.alterOpt))
-        alterBrokerConfig(adminClient, opts, entityName)
+        alterBrokerConfig(adminClient, opts, entityTypes.head, entityName)
       else if (opts.options.has(opts.describeOpt))
-        describeBrokerConfig(adminClient, opts, entityName)
+        describeBrokerConfig(adminClient, opts, entityTypes.head, entityName)
     } finally {
       adminClient.close()
     }
   }
 
-  private[admin] def alterBrokerConfig(adminClient: Admin, opts: ConfigCommandOptions, entityName: String) {
+  private[admin] def alterBrokerConfig(adminClient: Admin, opts: ConfigCommandOptions,
+                                       entityType: String, entityName: String) {
     val configsToBeAdded = parseConfigsToBeAdded(opts).asScala.map { case (k, v) => (k, new ConfigEntry(k, v)) }
     val configsToBeDeleted = parseConfigsToBeDeleted(opts)
 
-    // compile the final set of configs
-    val configResource = new ConfigResource(ConfigResource.Type.BROKER, entityName)
-    val oldConfig = brokerConfig(adminClient, entityName, includeSynonyms = false)
+    if (entityType == ConfigType.Broker) {
+      val configResource = new ConfigResource(ConfigResource.Type.BROKER, entityName)
+      val oldConfig = brokerConfig(adminClient, entityName, includeSynonyms = false)
         .map { entry => (entry.name, entry) }.toMap
 
-    // fail the command if any of the configs to be deleted does not exist
-    val invalidConfigs = configsToBeDeleted.filterNot(oldConfig.contains)
-    if (invalidConfigs.nonEmpty)
-      throw new InvalidConfigurationException(s"Invalid config(s): ${invalidConfigs.mkString(",")}")
-
-    val newEntries = oldConfig ++ configsToBeAdded -- configsToBeDeleted
-    val sensitiveEntries = newEntries.filter(_._2.value == null)
-    if (sensitiveEntries.nonEmpty)
-      throw new InvalidConfigurationException(s"All sensitive broker config entries must be specified for --alter, missing entries: ${sensitiveEntries.keySet}")
-    val newConfig = new JConfig(newEntries.asJava.values)
-
-    val alterOptions = new AlterConfigsOptions().timeoutMs(30000).validateOnly(false)
-    adminClient.alterConfigs(Map(configResource -> newConfig).asJava, alterOptions).all().get(60, TimeUnit.SECONDS)
+      // fail the command if any of the configs to be deleted does not exist
+      val invalidConfigs = configsToBeDeleted.filterNot(oldConfig.contains)
+      if (invalidConfigs.nonEmpty)
+        throw new InvalidConfigurationException(s"Invalid config(s): ${invalidConfigs.mkString(",")}")
+
+      val newEntries = oldConfig ++ configsToBeAdded -- configsToBeDeleted
+      val sensitiveEntries = newEntries.filter(_._2.value == null)
+      if (sensitiveEntries.nonEmpty)
+        throw new InvalidConfigurationException(s"All sensitive broker config entries must be specified for --alter, missing entries: ${sensitiveEntries.keySet}")
+      val newConfig = new JConfig(newEntries.asJava.values)
+
+      val alterOptions = new AlterConfigsOptions().timeoutMs(30000).validateOnly(false)
+      adminClient.alterConfigs(Map(configResource -> newConfig).asJava, alterOptions).all().get(60, TimeUnit.SECONDS)
+    } else if (entityType == BrokerLoggerConfigType) {
+      val configResource = new ConfigResource(ConfigResource.Type.BROKER_LOGGER, entityName)
+      val validLoggers = brokerLoggerConfigs(adminClient, entityName).map(_.name)
+      // fail the command if any of the configured broker loggers do not exist
+      val invalidBrokerLoggers = configsToBeDeleted.filterNot(validLoggers.contains) ++ configsToBeAdded.keys.filterNot(validLoggers.contains)
+      if (invalidBrokerLoggers.nonEmpty)
+        throw new InvalidConfigurationException(s"Invalid broker logger(s): ${invalidBrokerLoggers.mkString(",")}")
+
+      val alterOptions = new AlterConfigsOptions().timeoutMs(30000).validateOnly(false)
+      val alterLogLevelEntries = (configsToBeAdded.values.map(new AlterConfigOp(_, AlterConfigOp.OpType.SET))
+        ++ configsToBeDeleted.map { k => new AlterConfigOp(new ConfigEntry(k, ""), AlterConfigOp.OpType.DELETE) }
+      ).asJavaCollection
+
+      adminClient.incrementalAlterConfigs(Map(configResource -> alterLogLevelEntries).asJava, alterOptions).all().get(60, TimeUnit.SECONDS)
+    }
 
     if (entityName.nonEmpty)
       println(s"Completed updating config for broker: $entityName.")
@@ -324,8 +338,13 @@ object ConfigCommand extends Config {
       println(s"Completed updating default config for brokers in the cluster,")
   }
 
-  private def describeBrokerConfig(adminClient: Admin, opts: ConfigCommandOptions, entityName: String) {
-    val configs = brokerConfig(adminClient, entityName, includeSynonyms = true)
+  private def describeBrokerConfig(adminClient: Admin, opts: ConfigCommandOptions,
+                                   entityType: String, entityName: String) {
+    val configs = if (entityType == ConfigType.Broker)
+      brokerConfig(adminClient, entityName, includeSynonyms = true)
+    else // broker logger
+      brokerLoggerConfigs(adminClient, entityName)
+
     if (entityName.nonEmpty)
       println(s"Configs for broker $entityName are:")
     else
@@ -349,6 +368,15 @@ object ConfigCommand extends Config {
       .toSeq
   }
 
+  /**
+    * Returns all the valid broker logger configurations
+    */
+  private def brokerLoggerConfigs(adminClient: Admin, entityName: String): Seq[ConfigEntry] = {
+    val configResource = new ConfigResource(ConfigResource.Type.BROKER_LOGGER, entityName)
+    val configs = adminClient.describeConfigs(Collections.singleton(configResource)).all.get(30, TimeUnit.SECONDS)
+    configs.get(configResource).entries.asScala.toSeq
+  }
+
   case class Entity(entityType: String, sanitizedName: Option[String]) {
     val entityPath = sanitizedName match {
       case Some(n) => entityType + "/" + n
@@ -445,7 +473,7 @@ object ConfigCommand extends Config {
     if (opts.options.has(opts.alterOpt) && names.size != types.size)
       throw new IllegalArgumentException("--entity-name or --entity-default must be specified with each --entity-type for --alter")
 
-    val reverse = types.size == 2 && types(0) == ConfigType.Client
+    val reverse = types.size == 2 && types.head == ConfigType.Client
     val entityTypes = if (reverse) types.reverse else types
     val sortedNames = (if (reverse && names.length == 2) names.reverse else names).iterator
 
@@ -483,7 +511,7 @@ object ConfigCommand extends Config {
       .ofType(classOf[String])
     val alterOpt = parser.accepts("alter", "Alter the configuration for the entity.")
     val describeOpt = parser.accepts("describe", "List configs for the given entity.")
-    val entityType = parser.accepts("entity-type", "Type of entity (topics/clients/users/brokers)")
+    val entityType = parser.accepts("entity-type", "Type of entity (topics/clients/users/brokers/broker-loggers)")
             .withRequiredArg
             .ofType(classOf[String])
     val entityName = parser.accepts("entity-name", "Name of entity (topic name/client id/user principal name/broker id)")
@@ -514,36 +542,58 @@ object ConfigCommand extends Config {
       val actions = Seq(alterOpt, describeOpt).count(options.has _)
       if(actions != 1)
         CommandLineUtils.printUsageAndDie(parser, "Command must include exactly one action: --describe, --alter")
-
       // check required args
       CommandLineUtils.checkInvalidArgs(parser, options, alterOpt, Set(describeOpt))
       CommandLineUtils.checkInvalidArgs(parser, options, describeOpt, Set(alterOpt, addConfig, deleteConfig))
+
       val entityTypeVals = options.valuesOf(entityType).asScala
+      val (allowedEntityTypes, connectOptString) = if (options.has(bootstrapServerOpt))
+        (BrokerSupportedConfigTypes, "--bootstrap-server")
+      else
+        (ConfigType.all, "--zookeeper")
+
+      entityTypeVals.foreach(entityTypeVal =>
+        if (!allowedEntityTypes.contains(entityTypeVal))
+          throw new IllegalArgumentException(s"Invalid entity-type $entityTypeVal, --entity-type must be one of ${allowedEntityTypes.mkString(",")} with the $connectOptString argument")
+      )
+      if (entityTypeVals.isEmpty)
+        throw new IllegalArgumentException("At least one --entity-type must be specified")
+      else if (entityTypeVals.size > 1 && !entityTypeVals.toSet.equals(Set(ConfigType.User, ConfigType.Client)))
+        throw new IllegalArgumentException(s"Only '${ConfigType.User}' and '${ConfigType.Client}' entity types may be specified together")
 
-      if (options.has(bootstrapServerOpt) == options.has(zkConnectOpt))
+      if (!options.has(bootstrapServerOpt) && !options.has(zkConnectOpt))
+        throw new IllegalArgumentException("One of the required --bootstrap-server or --zookeeper arguments must be specified")
+      else if (options.has(bootstrapServerOpt) && options.has(zkConnectOpt))
         throw new IllegalArgumentException("Only one of --bootstrap-server or --zookeeper must be specified")
+      else if (options.has(bootstrapServerOpt) && !options.has(entityName) && !options.has(entityDefault))
+        throw new IllegalArgumentException(s"At least one of --entity-name or --entity-default must be specified with --bootstrap-server")
+
+      if (options.has(entityName) && (entityTypeVals.contains(ConfigType.Broker) || entityTypeVals.contains(BrokerLoggerConfigType))) {
+        val brokerId = options.valueOf(entityName)
+        try brokerId.toInt catch {
+          case _: NumberFormatException =>
+            throw new IllegalArgumentException(s"The entity name for ${entityTypeVals.head} must be a valid integer broker id , but it is: $brokerId")
+        }
+      }
+
       if (entityTypeVals.contains(ConfigType.Client) || entityTypeVals.contains(ConfigType.Topic) || entityTypeVals.contains(ConfigType.User))
         CommandLineUtils.checkRequiredArgs(parser, options, zkConnectOpt, entityType)
-      if(options.has(alterOpt)) {
+
+      if (options.has(describeOpt) && entityTypeVals.contains(BrokerLoggerConfigType) && !options.has(entityName))
+        throw new IllegalArgumentException(s"--entity-name must be specified with --describe of ${entityTypeVals.mkString(",")}")
+
+      if (options.has(alterOpt)) {
         if (entityTypeVals.contains(ConfigType.User) || entityTypeVals.contains(ConfigType.Client) || entityTypeVals.contains(ConfigType.Broker)) {
           if (!options.has(entityName) && !options.has(entityDefault))
             throw new IllegalArgumentException("--entity-name or --entity-default must be specified with --alter of users, clients or brokers")
         } else if (!options.has(entityName))
-            throw new IllegalArgumentException(s"--entity-name must be specified with --alter of ${entityTypeVals}")
+          throw new IllegalArgumentException(s"--entity-name must be specified with --alter of ${entityTypeVals.mkString(",")}")
 
         val isAddConfigPresent: Boolean = options.has(addConfig)
         val isDeleteConfigPresent: Boolean = options.has(deleteConfig)
         if(! isAddConfigPresent && ! isDeleteConfigPresent)
           throw new IllegalArgumentException("At least one of --add-config or --delete-config must be specified with --alter")
       }
-      entityTypeVals.foreach(entityTypeVal =>
-        if (!ConfigType.all.contains(entityTypeVal))
-          throw new IllegalArgumentException(s"Invalid entity-type ${entityTypeVal}, --entity-type must be one of ${ConfigType.all}")
-      )
-      if (entityTypeVals.isEmpty)
-        throw new IllegalArgumentException("At least one --entity-type must be specified")
-      else if (entityTypeVals.size > 1 && !entityTypeVals.toSet.equals(Set(ConfigType.User, ConfigType.Client)))
-        throw new IllegalArgumentException(s"Only '${ConfigType.User}' and '${ConfigType.Client}' entity types may be specified together")
     }
   }
 }
diff --git a/core/src/main/scala/kafka/server/AdminManager.scala b/core/src/main/scala/kafka/server/AdminManager.scala
index 1ed55fbfff114..1daaeec6bb6a8 100644
--- a/core/src/main/scala/kafka/server/AdminManager.scala
+++ b/core/src/main/scala/kafka/server/AdminManager.scala
@@ -21,13 +21,14 @@ import java.util.{Collections, Properties}
 import kafka.admin.{AdminOperationException, AdminUtils}
 import kafka.common.TopicAlreadyMarkedForDeletionException
 import kafka.log.LogConfig
+import kafka.utils.Log4jController
 import kafka.metrics.KafkaMetricsGroup
 import kafka.utils._
 import kafka.zk.{AdminZkClient, KafkaZkClient}
 import org.apache.kafka.clients.admin.AlterConfigOp
 import org.apache.kafka.clients.admin.AlterConfigOp.OpType
 import org.apache.kafka.common.config.ConfigDef.ConfigKey
-import org.apache.kafka.common.config.{AbstractConfig, ConfigDef, ConfigException, ConfigResource}
+import org.apache.kafka.common.config.{AbstractConfig, ConfigDef, ConfigException, ConfigResource, LogLevelConfig}
 import org.apache.kafka.common.errors.{ApiException, InvalidConfigurationException, InvalidPartitionsException, InvalidReplicaAssignmentException, InvalidRequestException, ReassignmentInProgressException, UnknownTopicOrPartitionException}
 import org.apache.kafka.common.internals.Topic
 import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic
@@ -347,8 +348,16 @@ class AdminManager(val config: KafkaConfig,
               createResponseConfig(allConfigs(config),
                 createBrokerConfigEntry(perBrokerConfig = true, includeSynonyms))
             else
-              throw new InvalidRequestException(s"Unexpected broker id, expected ${config.brokerId} or empty string, but received $resource.name")
+              throw new InvalidRequestException(s"Unexpected broker id, expected ${config.brokerId} or empty string, but received ${resource.name}")
 
+          case ConfigResource.Type.BROKER_LOGGER =>
+            if (resource.name == null || resource.name.isEmpty)
+              throw new InvalidRequestException("Broker id must not be empty")
+            else if (resourceNameToBrokerId(resource.name) != config.brokerId)
+              throw new InvalidRequestException(s"Unexpected broker id, expected ${config.brokerId} but received ${resource.name}")
+            else
+              createResponseConfig(Log4jController.loggers,
+                (name, value) => new DescribeConfigsResponse.ConfigEntry(name, value.toString, ConfigSource.DYNAMIC_BROKER_LOGGER_CONFIG, false, false, List.empty.asJava))
           case resourceType => throw new InvalidRequestException(s"Unsupported resource type: $resourceType")
         }
         resource -> resourceConfig
@@ -428,13 +437,24 @@ class AdminManager(val config: KafkaConfig,
     resource -> ApiError.NONE
   }
 
+  private def alterLogLevelConfigs(alterConfigOps: List[AlterConfigOp]): Unit = {
+    alterConfigOps.foreach { alterConfigOp =>
+      val loggerName = alterConfigOp.configEntry().name()
+      val logLevel = alterConfigOp.configEntry().value()
+      alterConfigOp.opType() match {
+        case OpType.SET => Log4jController.logLevel(loggerName, logLevel)
+        case OpType.DELETE => Log4jController.unsetLogLevel(loggerName)
+      }
+    }
+  }
+
   private def getBrokerId(resource: ConfigResource) = {
     if (resource.name == null || resource.name.isEmpty)
       None
     else {
       val id = resourceNameToBrokerId(resource.name)
       if (id != this.config.brokerId)
-        throw new InvalidRequestException(s"Unexpected broker id, expected ${this.config.brokerId}, but received $resource.name")
+        throw new InvalidRequestException(s"Unexpected broker id, expected ${this.config.brokerId}, but received ${resource.name}")
       Some(id)
     }
   }
@@ -451,7 +471,7 @@ class AdminManager(val config: KafkaConfig,
   def incrementalAlterConfigs(configs: Map[ConfigResource, List[AlterConfigOp]], validateOnly: Boolean): Map[ConfigResource, ApiError] = {
     configs.map { case (resource, alterConfigOps) =>
       try {
-        //throw InvalidRequestException if any duplicate keys
+        // throw InvalidRequestException if any duplicate keys
         val duplicateKeys = alterConfigOps.groupBy(config => config.configEntry().name())
           .mapValues(_.size).filter(_._2 > 1).keys.toSet
         if (duplicateKeys.nonEmpty)
@@ -475,6 +495,14 @@ class AdminManager(val config: KafkaConfig,
             val configProps = this.config.dynamicConfig.fromPersistentProps(persistentProps, perBrokerConfig)
             prepareIncrementalConfigs(alterConfigOps, configProps, KafkaConfig.configKeys)
             alterBrokerConfigs(resource, validateOnly, configProps, configEntriesMap)
+
+          case ConfigResource.Type.BROKER_LOGGER =>
+            getBrokerId(resource)
+            validateLogLevelConfigs(alterConfigOps)
+
+            if (!validateOnly)
+              alterLogLevelConfigs(alterConfigOps)
+            resource -> ApiError.NONE
           case resourceType =>
             throw new InvalidRequestException(s"AlterConfigs is only supported for topics and brokers, but resource type is $resourceType")
         }
@@ -495,6 +523,35 @@ class AdminManager(val config: KafkaConfig,
     }.toMap
   }
 
+  private def validateLogLevelConfigs(alterConfigOps: List[AlterConfigOp]): Unit = {
+    def validateLoggerNameExists(loggerName: String): Unit = {
+      if (!Log4jController.loggerExists(loggerName))
+        throw new ConfigException(s"Logger $loggerName does not exist!")
+    }
+
+    alterConfigOps.foreach { alterConfigOp =>
+      val loggerName = alterConfigOp.configEntry().name()
+      alterConfigOp.opType() match {
+        case OpType.SET =>
+          validateLoggerNameExists(loggerName)
+          val logLevel = alterConfigOp.configEntry().value()
+          if (!LogLevelConfig.VALID_LOG_LEVELS.contains(logLevel)) {
+            val validLevelsStr = LogLevelConfig.VALID_LOG_LEVELS.asScala.mkString(", ")
+            throw new ConfigException(
+              s"Cannot set the log level of $loggerName to $logLevel as it is not a supported log level. " +
+              s"Valid log levels are $validLevelsStr"
+            )
+          }
+        case OpType.DELETE =>
+          validateLoggerNameExists(loggerName)
+          if (loggerName == Log4jController.ROOT_LOGGER)
+            throw new InvalidRequestException(s"Removing the log level of the ${Log4jController.ROOT_LOGGER} logger is not allowed")
+        case OpType.APPEND => throw new InvalidRequestException(s"${OpType.APPEND} operation is not allowed for the ${ConfigResource.Type.BROKER_LOGGER} resource")
+        case OpType.SUBTRACT => throw new InvalidRequestException(s"${OpType.SUBTRACT} operation is not allowed for the ${ConfigResource.Type.BROKER_LOGGER} resource")
+      }
+    }
+  }
+
   private def prepareIncrementalConfigs(alterConfigOps: List[AlterConfigOp], configProps: Properties, configKeys: Map[String, ConfigKey]): Unit = {
 
     def listType(configName: String, configKeys: Map[String, ConfigKey]): Boolean = {
@@ -512,14 +569,14 @@ class AdminManager(val config: KafkaConfig,
           if (!listType(alterConfigOp.configEntry().name(), configKeys))
             throw new InvalidRequestException(s"Config value append is not allowed for config key: ${alterConfigOp.configEntry().name()}")
           val oldValueList = configProps.getProperty(alterConfigOp.configEntry().name()).split(",").toList
-          val newValueList =  oldValueList ::: alterConfigOp.configEntry().value().split(",").toList
+          val newValueList = oldValueList ::: alterConfigOp.configEntry().value().split(",").toList
           configProps.setProperty(alterConfigOp.configEntry().name(), newValueList.mkString(","))
         }
         case OpType.SUBTRACT => {
           if (!listType(alterConfigOp.configEntry().name(), configKeys))
             throw new InvalidRequestException(s"Config value subtract is not allowed for config key: ${alterConfigOp.configEntry().name()}")
           val oldValueList = configProps.getProperty(alterConfigOp.configEntry().name()).split(",").toList
-          val newValueList =  oldValueList.diff(alterConfigOp.configEntry().value().split(",").toList)
+          val newValueList = oldValueList.diff(alterConfigOp.configEntry().value().split(",").toList)
           configProps.setProperty(alterConfigOp.configEntry().name(), newValueList.mkString(","))
         }
       }
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 3ec6b2314d193..a88cd92f8267d 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -2288,6 +2288,8 @@ class KafkaApis(val requestChannel: RequestChannel,
     val alterConfigsRequest = request.body[AlterConfigsRequest]
     val (authorizedResources, unauthorizedResources) = alterConfigsRequest.configs.asScala.partition { case (resource, _) =>
       resource.`type` match {
+        case ConfigResource.Type.BROKER_LOGGER =>
+          throw new InvalidRequestException(s"AlterConfigs is deprecated and does not support the resource type ${ConfigResource.Type.BROKER_LOGGER}")
         case ConfigResource.Type.BROKER =>
           authorize(request.session, AlterConfigs, Resource.ClusterResource)
         case ConfigResource.Type.TOPIC =>
@@ -2331,7 +2333,7 @@ class KafkaApis(val requestChannel: RequestChannel,
 
   private def configsAuthorizationApiError(session: RequestChannel.Session, resource: ConfigResource): ApiError = {
     val error = resource.`type` match {
-      case ConfigResource.Type.BROKER => Errors.CLUSTER_AUTHORIZATION_FAILED
+      case ConfigResource.Type.BROKER | ConfigResource.Type.BROKER_LOGGER => Errors.CLUSTER_AUTHORIZATION_FAILED
       case ConfigResource.Type.TOPIC => Errors.TOPIC_AUTHORIZATION_FAILED
       case rt => throw new InvalidRequestException(s"Unexpected resource type $rt for resource ${resource.name}")
     }
@@ -2349,7 +2351,7 @@ class KafkaApis(val requestChannel: RequestChannel,
 
     val (authorizedResources, unauthorizedResources) = configs.partition { case (resource, _) =>
       resource.`type` match {
-        case ConfigResource.Type.BROKER =>
+        case ConfigResource.Type.BROKER | ConfigResource.Type.BROKER_LOGGER =>
           authorize(request.session, AlterConfigs, Resource.ClusterResource)
         case ConfigResource.Type.TOPIC =>
           authorize(request.session, AlterConfigs, Resource(Topic, resource.name, LITERAL))
@@ -2370,7 +2372,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     val describeConfigsRequest = request.body[DescribeConfigsRequest]
     val (authorizedResources, unauthorizedResources) = describeConfigsRequest.resources.asScala.partition { resource =>
       resource.`type` match {
-        case ConfigResource.Type.BROKER => authorize(request.session, DescribeConfigs, Resource.ClusterResource)
+        case ConfigResource.Type.BROKER | ConfigResource.Type.BROKER_LOGGER => authorize(request.session, DescribeConfigs, Resource.ClusterResource)
         case ConfigResource.Type.TOPIC =>
           authorize(request.session, DescribeConfigs, Resource(Topic, resource.name, LITERAL))
         case rt => throw new InvalidRequestException(s"Unexpected resource type $rt for resource ${resource.name}")
diff --git a/core/src/main/scala/kafka/utils/Log4jController.scala b/core/src/main/scala/kafka/utils/Log4jController.scala
index 95d07334a61d5..ba0649c0ca2db 100755
--- a/core/src/main/scala/kafka/utils/Log4jController.scala
+++ b/core/src/main/scala/kafka/utils/Log4jController.scala
@@ -22,69 +22,95 @@ import java.util.Locale
 
 import org.apache.log4j.{Level, LogManager, Logger}
 
+import scala.collection.mutable
+import scala.collection.JavaConverters._
 
-/**
- * An MBean that allows the user to dynamically alter log4j levels at runtime.
- * The companion object contains the singleton instance of this class and
- * registers the MBean. The [[kafka.utils.Logging]] trait forces initialization
- * of the companion object.
- */
-private class Log4jController extends Log4jControllerMBean {
 
-  def getLoggers = {
-    val lst = new util.ArrayList[String]()
-    lst.add("root=" + existingLogger("root").getLevel.toString)
+object Log4jController {
+  val ROOT_LOGGER = "root"
+
+  /**
+    * Returns a map of the log4j loggers and their assigned log level.
+    * If a logger does not have a log level assigned, we return the root logger's log level
+    */
+  def loggers: mutable.Map[String, String] = {
+    val logs = new mutable.HashMap[String, String]()
+    val rootLoggerLvl = existingLogger(ROOT_LOGGER).getLevel.toString
+    logs.put(ROOT_LOGGER, rootLoggerLvl)
+
     val loggers = LogManager.getCurrentLoggers
     while (loggers.hasMoreElements) {
       val logger = loggers.nextElement().asInstanceOf[Logger]
       if (logger != null) {
-        val level =  if (logger != null) logger.getLevel else null
-        lst.add("%s=%s".format(logger.getName, if (level != null) level.toString else "null"))
+        val level = if (logger.getLevel != null) logger.getLevel.toString else rootLoggerLvl
+        logs.put(logger.getName, level)
       }
     }
-    lst
+    logs
   }
 
+  /**
+    * Sets the log level of a particular logger
+    */
+  def logLevel(loggerName: String, logLevel: String): Boolean = {
+    val log = existingLogger(loggerName)
+    if (!loggerName.trim.isEmpty && !logLevel.trim.isEmpty && log != null) {
+      log.setLevel(Level.toLevel(logLevel.toUpperCase(Locale.ROOT)))
+      true
+    }
+    else false
+  }
 
-  private def newLogger(loggerName: String) =
-    if (loggerName == "root")
-      LogManager.getRootLogger
-    else LogManager.getLogger(loggerName)
+  def unsetLogLevel(loggerName: String): Boolean = {
+    val log = existingLogger(loggerName)
+    if (!loggerName.trim.isEmpty && log != null) {
+      log.setLevel(null)
+      true
+    }
+    else false
+  }
 
+  def loggerExists(loggerName: String): Boolean = existingLogger(loggerName) != null
 
   private def existingLogger(loggerName: String) =
-    if (loggerName == "root")
+    if (loggerName == ROOT_LOGGER)
       LogManager.getRootLogger
     else LogManager.exists(loggerName)
+}
 
+/**
+ * An MBean that allows the user to dynamically alter log4j levels at runtime.
+ * The companion object contains the singleton instance of this class and
+ * registers the MBean. The [[kafka.utils.Logging]] trait forces initialization
+ * of the companion object.
+ */
+class Log4jController extends Log4jControllerMBean {
 
-  def getLogLevel(loggerName: String) = {
-    val log = existingLogger(loggerName)
+  def getLoggers: util.List[String] = {
+    Log4jController.loggers.map {
+      case (logger, level) => s"$logger=$level"
+    }.toList.asJava
+  }
+
+
+  def getLogLevel(loggerName: String): String = {
+    val log = Log4jController.existingLogger(loggerName)
     if (log != null) {
       val level = log.getLevel
       if (level != null)
         log.getLevel.toString
-      else "Null log level."
+      else
+        Log4jController.existingLogger(Log4jController.ROOT_LOGGER).getLevel.toString
     }
     else "No such logger."
   }
 
-
-  def setLogLevel(loggerName: String, level: String) = {
-    val log = newLogger(loggerName)
-    if (!loggerName.trim.isEmpty && !level.trim.isEmpty && log != null) {
-      log.setLevel(Level.toLevel(level.toUpperCase(Locale.ROOT)))
-      true
-    }
-    else false
-  }
-
+  def setLogLevel(loggerName: String, level: String): Boolean = Log4jController.logLevel(loggerName, level)
 }
 
 
-private trait Log4jControllerMBean {
+trait Log4jControllerMBean {
   def getLoggers: java.util.List[String]
   def getLogLevel(logger: String): String
   def setLogLevel(logger: String, level: String): Boolean
 }
-
diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index 7f04de1819e08..ff8e379ef5bcb 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -24,12 +24,13 @@ import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger}
 import java.util.concurrent.{CountDownLatch, ExecutionException, TimeUnit}
 import java.util.{Collections, Properties}
 import java.{time, util}
+
 import kafka.log.LogConfig
 import kafka.security.auth.{Cluster, Group, Topic}
 import kafka.server.{Defaults, KafkaConfig, KafkaServer}
 import kafka.utils.Implicits._
 import kafka.utils.TestUtils._
-import kafka.utils.{Logging, TestUtils}
+import kafka.utils.{Log4jController, Logging, TestUtils}
 import kafka.zk.KafkaZkClient
 import org.apache.kafka.clients.admin._
 import org.apache.kafka.clients.consumer.{ConsumerConfig, KafkaConsumer}
@@ -40,7 +41,7 @@ import org.apache.kafka.common.ElectionType
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.TopicPartitionReplica
 import org.apache.kafka.common.acl._
-import org.apache.kafka.common.config.ConfigResource
+import org.apache.kafka.common.config.{ConfigResource, LogLevelConfig}
 import org.apache.kafka.common.errors._
 import org.apache.kafka.common.requests.{DeleteRecordsRequest, MetadataResponse}
 import org.apache.kafka.common.resource.{PatternType, ResourcePattern, ResourceType}
@@ -49,6 +50,7 @@ import org.junit.Assert._
 import org.junit.rules.Timeout
 import org.junit.{After, Before, Rule, Test}
 import org.scalatest.Assertions.intercept
+
 import scala.collection.JavaConverters._
 import scala.collection.Seq
 import scala.compat.java8.OptionConverters._
@@ -68,6 +70,8 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
   def globalTimeout = Timeout.millis(120000)
 
   var client: Admin = null
+  var brokerLoggerConfigResource: ConfigResource = null
+  var changedBrokerLoggers = scala.collection.mutable.Set[String]()
 
   val topic = "topic"
   val partition = 0
@@ -77,10 +81,12 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
   override def setUp(): Unit = {
     super.setUp
     TestUtils.waitUntilBrokerMetadataIsPropagated(servers)
+    brokerLoggerConfigResource = new ConfigResource(ConfigResource.Type.BROKER_LOGGER, servers.head.config.brokerId.toString)
   }
 
   @After
   override def tearDown(): Unit = {
+    teardownBrokerLoggers()
     if (client != null)
       Utils.closeQuietly(client, "AdminClient")
     super.tearDown()
@@ -1819,6 +1825,225 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
       classOf[InvalidTopicException])
     client.close()
   }
+
+  @Test
+  def testDescribeConfigsForLog4jLogLevels(): Unit = {
+    client = AdminClient.create(createConfig())
+
+    val loggerConfig = describeBrokerLoggers()
+    val rootLogLevel = loggerConfig.get(Log4jController.ROOT_LOGGER).value()
+    val logCleanerLogLevelConfig = loggerConfig.get("kafka.cluster.Replica")
+    assertEquals(rootLogLevel, logCleanerLogLevelConfig.value()) // we expect an undefined log level to be the same as the root logger
+    assertEquals("kafka.cluster.Replica", logCleanerLogLevelConfig.name())
+    assertEquals(ConfigEntry.ConfigSource.DYNAMIC_BROKER_LOGGER_CONFIG, logCleanerLogLevelConfig.source())
+    assertEquals(false, logCleanerLogLevelConfig.isReadOnly)
+    assertEquals(false, logCleanerLogLevelConfig.isSensitive)
+    assertTrue(logCleanerLogLevelConfig.synonyms().isEmpty)
+  }
+
+  @Test
+  def testIncrementalAlterConfigsForLog4jLogLevels(): Unit = {
+    client = AdminClient.create(createConfig())
+
+    val initialLoggerConfig = describeBrokerLoggers()
+    val initialRootLogLevel = initialLoggerConfig.get(Log4jController.ROOT_LOGGER).value()
+    assertEquals(initialRootLogLevel, initialLoggerConfig.get("kafka.controller.KafkaController").value())
+    assertEquals(initialRootLogLevel, initialLoggerConfig.get("kafka.log.LogCleaner").value())
+    assertEquals(initialRootLogLevel, initialLoggerConfig.get("kafka.server.ReplicaManager").value())
+
+    val newRootLogLevel = LogLevelConfig.DEBUG_LOG_LEVEL
+    val alterRootLoggerEntry = Seq(
+      new AlterConfigOp(new ConfigEntry(Log4jController.ROOT_LOGGER, newRootLogLevel), AlterConfigOp.OpType.SET)
+    ).asJavaCollection
+    // Test validateOnly does not change anything
+    alterBrokerLoggers(alterRootLoggerEntry, validateOnly = true)
+    val validatedLoggerConfig = describeBrokerLoggers()
+    assertEquals(initialRootLogLevel, validatedLoggerConfig.get(Log4jController.ROOT_LOGGER).value())
+    assertEquals(initialRootLogLevel, validatedLoggerConfig.get("kafka.controller.KafkaController").value())
+    assertEquals(initialRootLogLevel, validatedLoggerConfig.get("kafka.log.LogCleaner").value())
+    assertEquals(initialRootLogLevel, validatedLoggerConfig.get("kafka.server.ReplicaManager").value())
+    assertEquals(initialRootLogLevel, validatedLoggerConfig.get("kafka.zookeeper.ZooKeeperClient").value())
+
+    // test that we can change them and unset loggers still use the root's log level
+    alterBrokerLoggers(alterRootLoggerEntry)
+    val changedRootLoggerConfig = describeBrokerLoggers()
+    assertEquals(newRootLogLevel, changedRootLoggerConfig.get(Log4jController.ROOT_LOGGER).value())
+    assertEquals(newRootLogLevel, changedRootLoggerConfig.get("kafka.controller.KafkaController").value())
+    assertEquals(newRootLogLevel, changedRootLoggerConfig.get("kafka.log.LogCleaner").value())
+    assertEquals(newRootLogLevel, changedRootLoggerConfig.get("kafka.server.ReplicaManager").value())
+    assertEquals(newRootLogLevel, changedRootLoggerConfig.get("kafka.zookeeper.ZooKeeperClient").value())
+
+    // alter the ZK client's logger so we can later test resetting it
+    val alterZKLoggerEntry = Seq(
+      new AlterConfigOp(new ConfigEntry("kafka.zookeeper.ZooKeeperClient", LogLevelConfig.ERROR_LOG_LEVEL), AlterConfigOp.OpType.SET)
+    ).asJavaCollection
+    alterBrokerLoggers(alterZKLoggerEntry)
+    val changedZKLoggerConfig = describeBrokerLoggers()
+    assertEquals(LogLevelConfig.ERROR_LOG_LEVEL, changedZKLoggerConfig.get("kafka.zookeeper.ZooKeeperClient").value())
+
+    // properly test various set operations and one delete
+    val alterLogLevelsEntries = Seq(
+      new AlterConfigOp(new ConfigEntry("kafka.controller.KafkaController", LogLevelConfig.INFO_LOG_LEVEL), AlterConfigOp.OpType.SET),
+      new AlterConfigOp(new ConfigEntry("kafka.log.LogCleaner", LogLevelConfig.ERROR_LOG_LEVEL), AlterConfigOp.OpType.SET),
+      new AlterConfigOp(new ConfigEntry("kafka.server.ReplicaManager", LogLevelConfig.TRACE_LOG_LEVEL), AlterConfigOp.OpType.SET),
+      new AlterConfigOp(new ConfigEntry("kafka.zookeeper.ZooKeeperClient", ""), AlterConfigOp.OpType.DELETE) // should reset to the root logger level
+    ).asJavaCollection
+    alterBrokerLoggers(alterLogLevelsEntries)
+    val alteredLoggerConfig = describeBrokerLoggers()
+    assertEquals(newRootLogLevel, alteredLoggerConfig.get(Log4jController.ROOT_LOGGER).value())
+    assertEquals(LogLevelConfig.INFO_LOG_LEVEL, alteredLoggerConfig.get("kafka.controller.KafkaController").value())
+    assertEquals(LogLevelConfig.ERROR_LOG_LEVEL, alteredLoggerConfig.get("kafka.log.LogCleaner").value())
+    assertEquals(LogLevelConfig.TRACE_LOG_LEVEL, alteredLoggerConfig.get("kafka.server.ReplicaManager").value())
+    assertEquals(newRootLogLevel, alteredLoggerConfig.get("kafka.zookeeper.ZooKeeperClient").value())
+  }
+
+  /**
+    * 1. Assume ROOT logger == TRACE
+    * 2. Change kafka.controller.KafkaController logger to INFO
+    * 3. Unset kafka.controller.KafkaController via AlterConfigOp.OpType.DELETE (resets it to the root logger - TRACE)
+    * 4. Change ROOT logger to ERROR
+    * 5. Ensure the kafka.controller.KafkaController logger's level is ERROR (the curent root logger level)
+    */
+  @Test
+  def testIncrementalAlterConfigsForLog4jLogLevelsCanResetLoggerToCurrentRoot(): Unit = {
+    client = AdminClient.create(createConfig())
+    // step 1 - configure root logger
+    val initialRootLogLevel = LogLevelConfig.TRACE_LOG_LEVEL
+    val alterRootLoggerEntry = Seq(
+      new AlterConfigOp(new ConfigEntry(Log4jController.ROOT_LOGGER, initialRootLogLevel), AlterConfigOp.OpType.SET)
+    ).asJavaCollection
+    alterBrokerLoggers(alterRootLoggerEntry)
+    val initialLoggerConfig = describeBrokerLoggers()
+    assertEquals(initialRootLogLevel, initialLoggerConfig.get(Log4jController.ROOT_LOGGER).value())
+    assertEquals(initialRootLogLevel, initialLoggerConfig.get("kafka.controller.KafkaController").value())
+
+    // step 2 - change KafkaController logger to INFO
+    val alterControllerLoggerEntry = Seq(
+      new AlterConfigOp(new ConfigEntry("kafka.controller.KafkaController", LogLevelConfig.INFO_LOG_LEVEL), AlterConfigOp.OpType.SET)
+    ).asJavaCollection
+    alterBrokerLoggers(alterControllerLoggerEntry)
+    val changedControllerLoggerConfig = describeBrokerLoggers()
+    assertEquals(initialRootLogLevel, changedControllerLoggerConfig.get(Log4jController.ROOT_LOGGER).value())
+    assertEquals(LogLevelConfig.INFO_LOG_LEVEL, changedControllerLoggerConfig.get("kafka.controller.KafkaController").value())
+
+    // step 3 - unset KafkaController logger
+    val deleteControllerLoggerEntry = Seq(
+      new AlterConfigOp(new ConfigEntry("kafka.controller.KafkaController", ""), AlterConfigOp.OpType.DELETE)
+    ).asJavaCollection
+    alterBrokerLoggers(deleteControllerLoggerEntry)
+    val deletedControllerLoggerConfig = describeBrokerLoggers()
+    assertEquals(initialRootLogLevel, deletedControllerLoggerConfig.get(Log4jController.ROOT_LOGGER).value())
+    assertEquals(initialRootLogLevel, deletedControllerLoggerConfig.get("kafka.controller.KafkaController").value())
+
+    val newRootLogLevel = LogLevelConfig.ERROR_LOG_LEVEL
+    val newAlterRootLoggerEntry = Seq(
+      new AlterConfigOp(new ConfigEntry(Log4jController.ROOT_LOGGER, newRootLogLevel), AlterConfigOp.OpType.SET)
+    ).asJavaCollection
+    alterBrokerLoggers(newAlterRootLoggerEntry)
+    val newRootLoggerConfig = describeBrokerLoggers()
+    assertEquals(newRootLogLevel, newRootLoggerConfig.get(Log4jController.ROOT_LOGGER).value())
+    assertEquals(newRootLogLevel, newRootLoggerConfig.get("kafka.controller.KafkaController").value())
+  }
+
+  @Test
+  def testIncrementalAlterConfigsForLog4jLogLevelsCannotResetRootLogger(): Unit = {
+    client = AdminClient.create(createConfig())
+    val deleteRootLoggerEntry = Seq(
+      new AlterConfigOp(new ConfigEntry(Log4jController.ROOT_LOGGER, ""), AlterConfigOp.OpType.DELETE)
+    ).asJavaCollection
+
+    assertTrue(intercept[ExecutionException](alterBrokerLoggers(deleteRootLoggerEntry)).getCause.isInstanceOf[InvalidRequestException])
+  }
+
+  @Test
+  def testIncrementalAlterConfigsForLog4jLogLevelsDoesNotWorkWithInvalidConfigs(): Unit = {
+    client = AdminClient.create(createConfig())
+    val validLoggerName = "kafka.server.KafkaRequestHandler"
+    val expectedValidLoggerLogLevel = describeBrokerLoggers().get(validLoggerName)
+    def assertLogLevelDidNotChange(): Unit = {
+      assertEquals(
+        expectedValidLoggerLogLevel,
+        describeBrokerLoggers().get(validLoggerName)
+      )
+    }
+
+    val appendLogLevelEntries = Seq(
+      new AlterConfigOp(new ConfigEntry("kafka.server.KafkaRequestHandler", LogLevelConfig.INFO_LOG_LEVEL), AlterConfigOp.OpType.SET), // valid
+      new AlterConfigOp(new ConfigEntry("kafka.network.SocketServer", LogLevelConfig.ERROR_LOG_LEVEL), AlterConfigOp.OpType.APPEND) // append is not supported
+    ).asJavaCollection
+    assertTrue(intercept[ExecutionException](alterBrokerLoggers(appendLogLevelEntries)).getCause.isInstanceOf[InvalidRequestException])
+    assertLogLevelDidNotChange()
+
+    val subtractLogLevelEntries = Seq(
+      new AlterConfigOp(new ConfigEntry("kafka.server.KafkaRequestHandler", LogLevelConfig.INFO_LOG_LEVEL), AlterConfigOp.OpType.SET), // valid
+      new AlterConfigOp(new ConfigEntry("kafka.network.SocketServer", LogLevelConfig.ERROR_LOG_LEVEL), AlterConfigOp.OpType.SUBTRACT) // subtract is not supported
+    ).asJavaCollection
+    assertTrue(intercept[ExecutionException](alterBrokerLoggers(subtractLogLevelEntries)).getCause.isInstanceOf[InvalidRequestException])
+    assertLogLevelDidNotChange()
+
+    val invalidLogLevelLogLevelEntries = Seq(
+      new AlterConfigOp(new ConfigEntry("kafka.server.KafkaRequestHandler", LogLevelConfig.INFO_LOG_LEVEL), AlterConfigOp.OpType.SET), // valid
+      new AlterConfigOp(new ConfigEntry("kafka.network.SocketServer", "OFF"), AlterConfigOp.OpType.SET) // OFF is not a valid log level
+    ).asJavaCollection
+    assertTrue(intercept[ExecutionException](alterBrokerLoggers(invalidLogLevelLogLevelEntries)).getCause.isInstanceOf[InvalidRequestException])
+    assertLogLevelDidNotChange()
+
+    val invalidLoggerNameLogLevelEntries = Seq(
+      new AlterConfigOp(new ConfigEntry("kafka.server.KafkaRequestHandler", LogLevelConfig.INFO_LOG_LEVEL), AlterConfigOp.OpType.SET), // valid
+      new AlterConfigOp(new ConfigEntry("Some Other LogCleaner", LogLevelConfig.ERROR_LOG_LEVEL), AlterConfigOp.OpType.SET) // invalid logger name is not supported
+    ).asJavaCollection
+    assertTrue(intercept[ExecutionException](alterBrokerLoggers(invalidLoggerNameLogLevelEntries)).getCause.isInstanceOf[InvalidRequestException])
+    assertLogLevelDidNotChange()
+  }
+
+  /**
+    * The AlterConfigs API is deprecated and should not support altering log levels
+    */
+  @Test
+  def testAlterConfigsForLog4jLogLevelsDoesNotWork(): Unit = {
+    client = AdminClient.create(createConfig())
+
+    val alterLogLevelsEntries = Seq(
+      new ConfigEntry("kafka.controller.KafkaController", LogLevelConfig.INFO_LOG_LEVEL)
+    ).asJavaCollection
+    val alterResult = client.alterConfigs(Map(brokerLoggerConfigResource -> new Config(alterLogLevelsEntries)).asJava)
+    assertTrue(intercept[ExecutionException](alterResult.values.get(brokerLoggerConfigResource).get).getCause.isInstanceOf[InvalidRequestException])
+  }
+
+  def alterBrokerLoggers(entries: util.Collection[AlterConfigOp], validateOnly: Boolean = false): Unit = {
+    if (!validateOnly) {
+      for (entry <- entries.asScala)
+        changedBrokerLoggers.add(entry.configEntry().name())
+    }
+
+    client.incrementalAlterConfigs(Map(brokerLoggerConfigResource -> entries).asJava, new AlterConfigsOptions().validateOnly(validateOnly))
+      .values.get(brokerLoggerConfigResource).get()
+  }
+
+  def describeBrokerLoggers(): Config =
+    client.describeConfigs(Collections.singletonList(brokerLoggerConfigResource)).values.get(brokerLoggerConfigResource).get()
+
+  /**
+    * Due to the fact that log4j is not re-initialized across tests, changing a logger's log level persists across test classes.
+    * We need to clean up the changes done while testing.
+    */
+  def teardownBrokerLoggers(): Unit = {
+    if (changedBrokerLoggers.nonEmpty) {
+      val validLoggers = describeBrokerLoggers().entries().asScala.filterNot(_.name().equals(Log4jController.ROOT_LOGGER)).map(_.name).toSet
+      val unsetBrokerLoggersEntries = changedBrokerLoggers
+        .intersect(validLoggers)
+        .map { logger => new AlterConfigOp(new ConfigEntry(logger, ""), AlterConfigOp.OpType.DELETE) }
+        .asJavaCollection
+
+      // ensure that we first reset the root logger to an arbitrary log level. Note that we cannot reset it to its original value
+      alterBrokerLoggers(List(
+        new AlterConfigOp(new ConfigEntry(Log4jController.ROOT_LOGGER, LogLevelConfig.FATAL_LOG_LEVEL), AlterConfigOp.OpType.SET)
+      ).asJavaCollection)
+      alterBrokerLoggers(unsetBrokerLoggersEntries)
+
+      changedBrokerLoggers.clear()
+    }
+  }
 }
 
 object AdminClientIntegrationTest {
diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
index 8f3f24fc1ef04..387a7f9d113ea 100644
--- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
@@ -31,7 +31,7 @@ import org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener
 import org.apache.kafka.clients.producer._
 import org.apache.kafka.common.ElectionType
 import org.apache.kafka.common.acl.{AccessControlEntry, AccessControlEntryFilter, AclBinding, AclBindingFilter, AclOperation, AclPermissionType}
-import org.apache.kafka.common.config.ConfigResource
+import org.apache.kafka.common.config.{ConfigResource, LogLevelConfig}
 import org.apache.kafka.common.errors._
 import org.apache.kafka.common.internals.Topic.GROUP_METADATA_TOPIC_NAME
 import org.apache.kafka.common.message.ControlledShutdownRequestData
@@ -101,6 +101,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   val clusterCreateAcl = Map(Resource.ClusterResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Create)))
   val clusterAlterAcl = Map(Resource.ClusterResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Alter)))
   val clusterDescribeAcl = Map(Resource.ClusterResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)))
+  val clusterAlterConfigsAcl = Map(Resource.ClusterResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, AlterConfigs)))
   val clusterIdempotentWriteAcl = Map(Resource.ClusterResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, IdempotentWrite)))
   val topicCreateAcl = Map(createTopicResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Create)))
   val topicReadAcl = Map(topicResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)))
@@ -215,8 +216,13 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
       if (resp.logDirInfos.size() > 0) resp.logDirInfos.asScala.head._2.error else Errors.CLUSTER_AUTHORIZATION_FAILED),
     ApiKeys.CREATE_PARTITIONS -> ((resp: CreatePartitionsResponse) => resp.errors.asScala.find(_._1 == topic).get._2.error),
     ApiKeys.ELECT_LEADERS -> ((resp: ElectLeadersResponse) => Errors.forCode(resp.data().errorCode())),
-    ApiKeys.INCREMENTAL_ALTER_CONFIGS -> ((resp: IncrementalAlterConfigsResponse) =>
-      IncrementalAlterConfigsResponse.fromResponseData(resp.data()).get(new ConfigResource(ConfigResource.Type.TOPIC, tp.topic)).error),
+    ApiKeys.INCREMENTAL_ALTER_CONFIGS -> ((resp: IncrementalAlterConfigsResponse) => {
+      val topicResourceError = IncrementalAlterConfigsResponse.fromResponseData(resp.data()).get(new ConfigResource(ConfigResource.Type.TOPIC, tp.topic))
+      if (topicResourceError == null)
+        IncrementalAlterConfigsResponse.fromResponseData(resp.data()).get(new ConfigResource(ConfigResource.Type.BROKER_LOGGER, brokerId.toString)).error
+      else
+        topicResourceError.error()
+    }),
     ApiKeys.ALTER_PARTITION_REASSIGNMENTS -> ((resp: AlterPartitionReassignmentsResponse) => Errors.forCode(resp.data().errorCode())),
     ApiKeys.LIST_PARTITION_REASSIGNMENTS -> ((resp: ListPartitionReassignmentsResponse) => Errors.forCode(resp.data().errorCode()))
   )
@@ -650,6 +656,28 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
     sendRequestAndVerifyResponseError(key, request, resources, isAuthorized = true)
   }
 
+  @Test
+  def testIncrementalAlterConfigsRequestRequiresClusterPermissionForBrokerLogger(): Unit = {
+    val data = new IncrementalAlterConfigsRequestData
+    val alterableConfig = new AlterableConfig().setName("kafka.controller.KafkaController").
+      setValue(LogLevelConfig.DEBUG_LOG_LEVEL).setConfigOperation(AlterConfigOp.OpType.DELETE.id())
+    val alterableConfigSet = new AlterableConfigCollection
+    alterableConfigSet.add(alterableConfig)
+    data.resources().add(new AlterConfigsResource().
+      setResourceName(brokerId.toString).setResourceType(ConfigResource.Type.BROKER_LOGGER.id()).
+      setConfigs(alterableConfigSet))
+    val key = ApiKeys.INCREMENTAL_ALTER_CONFIGS
+    val request = new IncrementalAlterConfigsRequest.Builder(data).build()
+
+    removeAllAcls()
+    val resources = Set(topicResource.resourceType, Resource.ClusterResource.resourceType)
+    sendRequestAndVerifyResponseError(key, request, resources, isAuthorized = false)
+
+    val clusterAcls = clusterAlterConfigsAcl(Resource.ClusterResource)
+    addAndVerifyAcls(clusterAcls, Resource.ClusterResource)
+    sendRequestAndVerifyResponseError(key, request, resources, isAuthorized = true)
+  }
+
   @Test
   def testOffsetsForLeaderEpochClusterPermission(): Unit = {
     val key = ApiKeys.OFFSET_FOR_LEADER_EPOCH
diff --git a/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala b/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala
index bd26a61d22d60..e3396bbcae247 100644
--- a/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala
@@ -22,7 +22,7 @@ import java.util.Properties
 import kafka.admin.ConfigCommand.ConfigCommandOptions
 import kafka.api.ApiVersion
 import kafka.cluster.{Broker, EndPoint}
-import kafka.server.{ConfigEntityName, KafkaConfig}
+import kafka.server.{ConfigEntityName, ConfigType, KafkaConfig}
 import kafka.utils.{Exit, Logging}
 import kafka.zk.{AdminZkClient, BrokerInfo, KafkaZkClient, ZooKeeperTestHarness}
 import org.apache.kafka.clients.admin._
@@ -101,33 +101,44 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging {
     testArgumentParse("brokers")
   }
 
-  def testArgumentParse(entityType: String) = {
+  @Test
+  def shouldParseArgumentsForBrokerLoggersEntityType() {
+    testArgumentParse("broker-loggers",
+      zkConfig = false)
+  }
+
+  def testArgumentParse(entityType: String, zkConfig: Boolean=true): Unit = {
+    val connectOpts = if (zkConfig)
+      ("--zookeeper", zkConnect)
+    else
+      ("--bootstrap-server", "localhost:9092")
+
     // Should parse correctly
-    var createOpts = new ConfigCommandOptions(Array("--zookeeper", zkConnect,
-      "--entity-name", "x",
+    var createOpts = new ConfigCommandOptions(Array(connectOpts._1, connectOpts._2,
+      "--entity-name", "1",
       "--entity-type", entityType,
       "--describe"))
     createOpts.checkArgs()
 
     // For --alter and added config
-    createOpts = new ConfigCommandOptions(Array("--zookeeper", zkConnect,
-      "--entity-name", "x",
+    createOpts = new ConfigCommandOptions(Array(connectOpts._1, connectOpts._2,
+      "--entity-name", "1",
       "--entity-type", entityType,
       "--alter",
       "--add-config", "a=b,c=d"))
     createOpts.checkArgs()
 
     // For alter and deleted config
-    createOpts = new ConfigCommandOptions(Array("--zookeeper", zkConnect,
-      "--entity-name", "x",
+    createOpts = new ConfigCommandOptions(Array(connectOpts._1, connectOpts._2,
+      "--entity-name", "1",
       "--entity-type", entityType,
       "--alter",
       "--delete-config", "a,b,c"))
     createOpts.checkArgs()
 
     // For alter and both added, deleted config
-    createOpts = new ConfigCommandOptions(Array("--zookeeper", zkConnect,
-      "--entity-name", "x",
+    createOpts = new ConfigCommandOptions(Array(connectOpts._1, connectOpts._2,
+      "--entity-name", "1",
       "--entity-type", entityType,
       "--alter",
       "--add-config", "a=b,c=d",
@@ -143,8 +154,8 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging {
     assertEquals(1, deletedProps.size)
     assertEquals("a", deletedProps.head)
 
-    createOpts = new ConfigCommandOptions(Array("--zookeeper", zkConnect,
-      "--entity-name", "x",
+    createOpts = new ConfigCommandOptions(Array(connectOpts._1, connectOpts._2,
+      "--entity-name", "1",
       "--entity-type", entityType,
       "--alter",
       "--add-config", "a=b,c=,d=e,f="))
@@ -165,6 +176,13 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging {
     ConfigCommand.alterConfig(null, createOpts, new DummyAdminZkClient(zkClient))
   }
 
+  @Test(expected = classOf[IllegalArgumentException])
+  def shouldFailIfBrokerEntityTypeIsNotAnInteger(): Unit = {
+    val createOpts = new ConfigCommandOptions(Array("--zookeeper", zkConnect,
+      "--entity-name", "A", "--entity-type", "brokers", "--alter", "--add-config", "a=b,c=d"))
+    ConfigCommand.alterConfig(null, createOpts, new DummyAdminZkClient(zkClient))
+  }
+
   @Test
   def shouldAddClientConfig(): Unit = {
     val createOpts = new ConfigCommandOptions(Array("--zookeeper", zkConnect,
@@ -228,6 +246,71 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging {
     verifyAlterBrokerConfig(node, "1", List("--entity-name", "1"))
   }
 
+  @Test
+  def shouldAddBrokerLoggerConfig(): Unit = {
+    val node = new Node(1, "localhost", 9092)
+    verifyAlterBrokerLoggerConfig(node, "1", "1", List(
+      new ConfigEntry("kafka.log.LogCleaner", "INFO"),
+      new ConfigEntry("kafka.server.ReplicaManager", "INFO"),
+      new ConfigEntry("kafka.server.KafkaApi", "INFO")
+    ))
+  }
+
+  @Test
+  def testNoSpecifiedEntityOptionWithDescribeBrokersInZKIsAllowed(): Unit = {
+    val optsList = List("--zookeeper", "localhost:9092",
+      "--entity-type", ConfigType.Broker,
+      "--describe"
+    )
+
+    new ConfigCommandOptions(optsList.toArray).checkArgs()
+  }
+
+  @Test(expected = classOf[IllegalArgumentException])
+  def testNoSpecifiedEntityOptionWithDescribeBrokersInBootstrapServerIsNotAllowed(): Unit = {
+    val optsList = List("--bootstrap-server", "localhost:9092",
+      "--entity-type", ConfigType.Broker,
+      "--describe"
+    )
+
+    new ConfigCommandOptions(optsList.toArray).checkArgs()
+  }
+
+  @Test(expected = classOf[IllegalArgumentException])
+  def testEntityDefaultOptionWithDescribeBrokerLoggerIsNotAllowed(): Unit = {
+    val node = new Node(1, "localhost", 9092)
+    val optsList = List("--bootstrap-server", "localhost:9092",
+      "--entity-type", ConfigCommand.BrokerLoggerConfigType,
+      "--entity-default",
+      "--describe"
+    )
+
+    new ConfigCommandOptions(optsList.toArray).checkArgs()
+  }
+
+  @Test(expected = classOf[IllegalArgumentException])
+  def testEntityDefaultOptionWithAlterBrokerLoggerIsNotAllowed(): Unit = {
+    val node = new Node(1, "localhost", 9092)
+    val optsList = List("--bootstrap-server", "localhost:9092",
+      "--entity-type", ConfigCommand.BrokerLoggerConfigType,
+      "--entity-default",
+      "--alter",
+      "--add-config", "kafka.log.LogCleaner=DEBUG"
+    )
+
+    new ConfigCommandOptions(optsList.toArray).checkArgs()
+  }
+
+  @Test(expected = classOf[InvalidConfigurationException])
+  def shouldRaiseInvalidConfigurationExceptionWhenAddingInvalidBrokerLoggerConfig(): Unit = {
+    val node = new Node(1, "localhost", 9092)
+    // verifyAlterBrokerLoggerConfig tries to alter kafka.log.LogCleaner, kafka.server.ReplicaManager and kafka.server.KafkaApi
+    // yet, we make it so DescribeConfigs returns only one logger, implying that kafka.server.ReplicaManager and kafka.log.LogCleaner are invalid
+    verifyAlterBrokerLoggerConfig(node, "1", "1", List(
+      new ConfigEntry("kafka.server.KafkaApi", "INFO")
+    ))
+  }
+
   @Test
   def shouldAddDefaultBrokerDynamicConfig(): Unit = {
     val node = new Node(1, "localhost", 9092)
@@ -274,11 +357,66 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging {
       }
     }
     EasyMock.replay(alterResult, describeResult)
-    ConfigCommand.alterBrokerConfig(mockAdminClient, alterOpts, resourceName)
+    ConfigCommand.alterBrokerConfig(mockAdminClient, alterOpts, ConfigType.Broker, resourceName)
     assertEquals(Map("message.max.bytes" -> "10", "num.io.threads" -> "5"), brokerConfigs.toMap)
     EasyMock.reset(alterResult, describeResult)
   }
 
+  def verifyAlterBrokerLoggerConfig(node: Node, resourceName: String, entityName: String,
+                                    describeConfigEntries: List[ConfigEntry]): Unit = {
+    val optsList = List("--bootstrap-server", "localhost:9092",
+      "--entity-type", ConfigCommand.BrokerLoggerConfigType,
+      "--alter",
+      "--entity-name", entityName,
+      "--add-config", "kafka.log.LogCleaner=DEBUG",
+      "--delete-config", "kafka.server.ReplicaManager,kafka.server.KafkaApi")
+    val alterOpts = new ConfigCommandOptions(optsList.toArray)
+    var alteredConfigs = false
+
+    val resource = new ConfigResource(ConfigResource.Type.BROKER_LOGGER, resourceName)
+    val future = new KafkaFutureImpl[util.Map[ConfigResource, Config]]
+    future.complete(util.Collections.singletonMap(resource, new Config(describeConfigEntries.asJava)))
+    val describeResult: DescribeConfigsResult = EasyMock.createNiceMock(classOf[DescribeConfigsResult])
+    EasyMock.expect(describeResult.all()).andReturn(future).once()
+
+    val alterFuture = new KafkaFutureImpl[Void]
+    alterFuture.complete(null)
+    val alterResult: AlterConfigsResult = EasyMock.createNiceMock(classOf[AlterConfigsResult])
+    EasyMock.expect(alterResult.all()).andReturn(alterFuture)
+
+    val mockAdminClient = new MockAdminClient(util.Collections.singletonList(node), node) {
+      override def describeConfigs(resources: util.Collection[ConfigResource]): DescribeConfigsResult = {
+        assertEquals(1, resources.size)
+        val resource = resources.iterator.next
+        assertEquals(ConfigResource.Type.BROKER_LOGGER, resource.`type`)
+        assertEquals(resourceName, resource.name)
+        describeResult
+      }
+
+      override def incrementalAlterConfigs(configs: util.Map[ConfigResource, util.Collection[AlterConfigOp]], options: AlterConfigsOptions): AlterConfigsResult = {
+        assertEquals(1, configs.size)
+        val entry = configs.entrySet.iterator.next
+        val resource = entry.getKey
+        val alterConfigOps = entry.getValue
+        assertEquals(ConfigResource.Type.BROKER_LOGGER, resource.`type`)
+        assertEquals(3, alterConfigOps.size)
+
+        val expectedConfigOps = List(
+          new AlterConfigOp(new ConfigEntry("kafka.log.LogCleaner", "DEBUG"), AlterConfigOp.OpType.SET),
+          new AlterConfigOp(new ConfigEntry("kafka.server.ReplicaManager", ""), AlterConfigOp.OpType.DELETE),
+          new AlterConfigOp(new ConfigEntry("kafka.server.KafkaApi", ""), AlterConfigOp.OpType.DELETE)
+        )
+        assertEquals(expectedConfigOps, alterConfigOps.asScala.toList)
+        alteredConfigs = true
+        alterResult
+      }
+    }
+    EasyMock.replay(alterResult, describeResult)
+    ConfigCommand.alterBrokerConfig(mockAdminClient, alterOpts, ConfigCommand.BrokerLoggerConfigType, resourceName)
+    assertTrue(alteredConfigs)
+    EasyMock.reset(alterResult, describeResult)
+  }
+
   @Test
   def shouldSupportCommaSeparatedValues(): Unit = {
     val createOpts = new ConfigCommandOptions(Array("--zookeeper", zkConnect,

From 96c575a2c7d5740efe9b4fdbc5a6c22c558a3f8f Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Fri, 2 Aug 2019 21:37:42 -0700
Subject: [PATCH 0502/1071] MINOR: Fix binary compatibility break in
 KafkaClientSupplier.getAdminClient (#7157)

Changing the return type in an interface is a binary incompatible change.

Reviewers: Jason Gustafson 
---
 .../org/apache/kafka/streams/KafkaClientSupplier.java     | 8 ++++----
 .../processor/internals/DefaultKafkaClientSupplier.java   | 6 +++---
 .../java/org/apache/kafka/test/MockClientSupplier.java    | 4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaClientSupplier.java b/streams/src/main/java/org/apache/kafka/streams/KafkaClientSupplier.java
index cc9d27e02d0fe..4ed277018f3ed 100644
--- a/streams/src/main/java/org/apache/kafka/streams/KafkaClientSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/KafkaClientSupplier.java
@@ -16,7 +16,7 @@
  */
 package org.apache.kafka.streams;
 
-import org.apache.kafka.clients.admin.Admin;
+import org.apache.kafka.clients.admin.AdminClient;
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.producer.Producer;
 import org.apache.kafka.streams.kstream.GlobalKTable;
@@ -31,12 +31,12 @@
  */
 public interface KafkaClientSupplier {
     /**
-     * Create an {@link Admin} which is used for internal topic management.
+     * Create an {@link AdminClient} which is used for internal topic management.
      *
      * @param config Supplied by the {@link java.util.Properties} given to the {@link KafkaStreams}
-     * @return an instance of {@link Admin}
+     * @return an instance of {@link AdminClient}
      */
-    Admin getAdminClient(final Map config);
+    AdminClient getAdminClient(final Map config);
 
     /**
      * Create a {@link Producer} which is used to write records to sink topics.
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/DefaultKafkaClientSupplier.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/DefaultKafkaClientSupplier.java
index f56f834493e8a..69331b462ec42 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/DefaultKafkaClientSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/DefaultKafkaClientSupplier.java
@@ -18,7 +18,7 @@
 
 import java.util.Map;
 
-import org.apache.kafka.clients.admin.Admin;
+import org.apache.kafka.clients.admin.AdminClient;
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.KafkaConsumer;
 import org.apache.kafka.clients.producer.KafkaProducer;
@@ -29,9 +29,9 @@
 
 public class DefaultKafkaClientSupplier implements KafkaClientSupplier {
     @Override
-    public Admin getAdminClient(final Map config) {
+    public AdminClient getAdminClient(final Map config) {
         // create a new client upon each call; but expect this call to be only triggered once so this should be fine
-        return Admin.create(config);
+        return AdminClient.create(config);
     }
 
     @Override
diff --git a/streams/src/test/java/org/apache/kafka/test/MockClientSupplier.java b/streams/src/test/java/org/apache/kafka/test/MockClientSupplier.java
index 4330d6c85cf4f..d3430f2c727c9 100644
--- a/streams/src/test/java/org/apache/kafka/test/MockClientSupplier.java
+++ b/streams/src/test/java/org/apache/kafka/test/MockClientSupplier.java
@@ -16,7 +16,7 @@
  */
 package org.apache.kafka.test;
 
-import org.apache.kafka.clients.admin.Admin;
+import org.apache.kafka.clients.admin.AdminClient;
 import org.apache.kafka.clients.admin.MockAdminClient;
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.MockConsumer;
@@ -57,7 +57,7 @@ public void setClusterForAdminClient(final Cluster cluster) {
     }
 
     @Override
-    public Admin getAdminClient(final Map config) {
+    public AdminClient getAdminClient(final Map config) {
         return new MockAdminClient(cluster.nodes(), cluster.nodeById(0));
     }
 

From c76f5651fa1b406d8f7a88b20e435ad4f14c4797 Mon Sep 17 00:00:00 2001
From: Victoria Bialas 
Date: Sat, 3 Aug 2019 13:51:43 -0700
Subject: [PATCH 0503/1071] MINOR: Fix typo in docs (#7158)

Reviewer: Matthias J. Sax 
---
 docs/uses.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/uses.html b/docs/uses.html
index 945b8960c1406..09bc45fc3215a 100644
--- a/docs/uses.html
+++ b/docs/uses.html
@@ -60,7 +60,7 @@ 
        Stream Processin
 Many users of Kafka process data in processing pipelines consisting of multiple stages, where raw input data is consumed from Kafka topics and then
 aggregated, enriched, or otherwise transformed into new topics for further consumption or follow-up processing.
 For example, a processing pipeline for recommending news articles might crawl article content from RSS feeds and publish it to an "articles" topic;
-further processing might normalize or deduplicate this content and published the cleansed article content to a new topic;
+further processing might normalize or deduplicate this content and publish the cleansed article content to a new topic;
 a final processing stage might attempt to recommend this content to users.
 Such processing pipelines create graphs of real-time data flows based on the individual topics.
 Starting in 0.10.0.0, a light-weight but powerful stream processing library called Kafka Streams

From 320f7b0d7afd75c6d76ee33770461e93bef43689 Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Sat, 3 Aug 2019 14:03:15 -0700
Subject: [PATCH 0504/1071] MINOR: Avoid dividing by zero (#7143)

Reviews: A. Sophie Blee-Goldman , Bill Bejeck , Bruno Cadonna , Boyang Chen , Guozhang Wang 
---
 .../kafka/streams/processor/internals/StreamThread.java  | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index 1633b30223b62..d3efa9e0e05de 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -1042,7 +1042,7 @@ private boolean maybePunctuate() {
      *                               or if the task producer got fenced (EOS)
      */
     boolean maybeCommit() {
-        int committed = 0;
+        final int committed;
 
         if (now - lastCommitMs > commitTimeMs) {
             if (log.isTraceEnabled()) {
@@ -1050,7 +1050,7 @@ boolean maybeCommit() {
                     taskManager.activeTaskIds(), taskManager.standbyTaskIds(), now - lastCommitMs, commitTimeMs);
             }
 
-            committed += taskManager.commitAll();
+            committed = taskManager.commitAll();
             if (committed > 0) {
                 final long intervalCommitLatency = advanceNowAndComputeLatency();
                 commitSensor.record(intervalCommitLatency / (double) committed, now);
@@ -1067,11 +1067,10 @@ boolean maybeCommit() {
             lastCommitMs = now;
             processStandbyRecords = true;
         } else {
-            final int commitPerRequested = taskManager.maybeCommitActiveTasksPerUserRequested();
-            if (commitPerRequested > 0) {
+            committed = taskManager.maybeCommitActiveTasksPerUserRequested();
+            if (committed > 0) {
                 final long requestCommitLatency = advanceNowAndComputeLatency();
                 commitSensor.record(requestCommitLatency / (double) committed, now);
-                committed += commitPerRequested;
             }
         }
 

From 0b1dc1ca7be3566a4367d7d3aae2c691cf94f48e Mon Sep 17 00:00:00 2001
From: anatasiavela 
Date: Sat, 3 Aug 2019 21:00:46 -0700
Subject: [PATCH 0505/1071] KAFKA-6263; Expose metrics for group and
 transaction metadata loading duration

[JIRA](https://issues.apache.org/jira/browse/KAFKA-6263)

- Add metrics to provide visibility for how long group metadata and transaction metadata take to load in order to understand some inactivity seen in the consumer groups
- Tests include mocking load times by creating a delay after each are loaded and ensuring the measured JMX metric is as it should be

Author: anatasiavela 

Reviewers: Gwen Shapira, Jason Gustafson

Closes #7045 from anatasiavela/KAFKA-6263
---
 .../coordinator/group/GroupCoordinator.scala  | 16 ++++---
 .../group/GroupMetadataManager.scala          | 20 +++++++-
 .../transaction/TransactionCoordinator.scala  |  2 +-
 .../transaction/TransactionStateManager.scala | 21 +++++++--
 .../main/scala/kafka/server/KafkaServer.scala |  2 +-
 .../GroupCoordinatorConcurrencyTest.scala     |  3 +-
 .../group/GroupCoordinatorTest.scala          |  3 +-
 .../group/GroupMetadataManagerTest.scala      | 47 ++++++++++++++++++-
 ...ransactionCoordinatorConcurrencyTest.scala |  3 +-
 .../TransactionStateManagerTest.scala         | 39 ++++++++++++++-
 docs/ops.html                                 | 20 ++++++++
 11 files changed, 157 insertions(+), 19 deletions(-)

diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
index 6a57d59aa496a..78749460cb261 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
@@ -28,6 +28,7 @@ import kafka.zk.KafkaZkClient
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.internals.Topic
 import org.apache.kafka.common.message.JoinGroupResponseData.JoinGroupResponseMember
+import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
 import org.apache.kafka.common.record.RecordBatch.{NO_PRODUCER_EPOCH, NO_PRODUCER_ID}
@@ -55,7 +56,8 @@ class GroupCoordinator(val brokerId: Int,
                        val groupManager: GroupMetadataManager,
                        val heartbeatPurgatory: DelayedOperationPurgatory[DelayedHeartbeat],
                        val joinPurgatory: DelayedOperationPurgatory[DelayedJoin],
-                       time: Time) extends Logging {
+                       time: Time,
+                       metrics: Metrics) extends Logging {
   import GroupCoordinator._
 
   type JoinCallback = JoinGroupResult => Unit
@@ -1084,10 +1086,11 @@ object GroupCoordinator {
   def apply(config: KafkaConfig,
             zkClient: KafkaZkClient,
             replicaManager: ReplicaManager,
-            time: Time): GroupCoordinator = {
+            time: Time,
+            metrics: Metrics): GroupCoordinator = {
     val heartbeatPurgatory = DelayedOperationPurgatory[DelayedHeartbeat]("Heartbeat", config.brokerId)
     val joinPurgatory = DelayedOperationPurgatory[DelayedJoin]("Rebalance", config.brokerId)
-    apply(config, zkClient, replicaManager, heartbeatPurgatory, joinPurgatory, time)
+    apply(config, zkClient, replicaManager, heartbeatPurgatory, joinPurgatory, time, metrics)
   }
 
   private[group] def offsetConfig(config: KafkaConfig) = OffsetConfig(
@@ -1108,7 +1111,8 @@ object GroupCoordinator {
             replicaManager: ReplicaManager,
             heartbeatPurgatory: DelayedOperationPurgatory[DelayedHeartbeat],
             joinPurgatory: DelayedOperationPurgatory[DelayedJoin],
-            time: Time): GroupCoordinator = {
+            time: Time,
+            metrics: Metrics): GroupCoordinator = {
     val offsetConfig = this.offsetConfig(config)
     val groupConfig = GroupConfig(groupMinSessionTimeoutMs = config.groupMinSessionTimeoutMs,
       groupMaxSessionTimeoutMs = config.groupMaxSessionTimeoutMs,
@@ -1116,8 +1120,8 @@ object GroupCoordinator {
       groupInitialRebalanceDelayMs = config.groupInitialRebalanceDelay)
 
     val groupMetadataManager = new GroupMetadataManager(config.brokerId, config.interBrokerProtocolVersion,
-      offsetConfig, replicaManager, zkClient, time)
-    new GroupCoordinator(config.brokerId, groupConfig, offsetConfig, groupMetadataManager, heartbeatPurgatory, joinPurgatory, time)
+      offsetConfig, replicaManager, zkClient, time, metrics)
+    new GroupCoordinator(config.brokerId, groupConfig, offsetConfig, groupMetadataManager, heartbeatPurgatory, joinPurgatory, time, metrics)
   }
 
   def joinError(memberId: String, error: Errors): JoinGroupResult = {
diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
index 03c3e37752040..7d8499f1dc99a 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
@@ -36,6 +36,8 @@ import kafka.zk.KafkaZkClient
 import org.apache.kafka.clients.consumer.ConsumerRecord
 import org.apache.kafka.common.{KafkaException, TopicPartition}
 import org.apache.kafka.common.internals.Topic
+import org.apache.kafka.common.metrics.stats.{Avg, Max}
+import org.apache.kafka.common.metrics.{MetricConfig, Metrics}
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.protocol.types.Type._
 import org.apache.kafka.common.protocol.types._
@@ -53,7 +55,8 @@ class GroupMetadataManager(brokerId: Int,
                            config: OffsetConfig,
                            replicaManager: ReplicaManager,
                            zkClient: KafkaZkClient,
-                           time: Time) extends Logging with KafkaMetricsGroup {
+                           time: Time,
+                           metrics: Metrics) extends Logging with KafkaMetricsGroup {
 
   private val compressionType: CompressionType = CompressionType.forId(config.offsetsTopicCompressionCodec.codec)
 
@@ -82,6 +85,16 @@ class GroupMetadataManager(brokerId: Int,
    * We use this structure to quickly find the groups which need to be updated by the commit/abort marker. */
   private val openGroupsForProducer = mutable.HashMap[Long, mutable.Set[String]]()
 
+  /* setup metrics*/
+  val partitionLoadSensor = metrics.sensor("PartitionLoadTime")
+
+  partitionLoadSensor.add(metrics.metricName("partition-load-time-max",
+    "group-coordinator-metrics",
+    "The max time it took to load the partitions in the last 30sec"), new Max())
+  partitionLoadSensor.add(metrics.metricName("partition-load-time-avg",
+    "group-coordinator-metrics",
+    "The avg time it took to load the partitions in the last 30sec"), new Avg())
+
   this.logIdent = s"[GroupMetadataManager brokerId=$brokerId] "
 
   private def recreateGauge[T](name: String, gauge: Gauge[T]): Gauge[T] = {
@@ -498,7 +511,10 @@ class GroupMetadataManager(brokerId: Int,
     try {
       val startMs = time.milliseconds()
       doLoadGroupsAndOffsets(topicPartition, onGroupLoaded)
-      info(s"Finished loading offsets and group metadata from $topicPartition in ${time.milliseconds() - startMs} milliseconds.")
+      val endMs = time.milliseconds()
+      val timeLapse = endMs - startMs
+      partitionLoadSensor.record(timeLapse, endMs, false)
+      info(s"Finished loading offsets and group metadata from $topicPartition in $timeLapse milliseconds.")
     } catch {
       case t: Throwable => error(s"Error loading offsets from $topicPartition", t)
     } finally {
diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala
index 9d4eed69fd371..6d99889d0181e 100644
--- a/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala
+++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala
@@ -55,7 +55,7 @@ object TransactionCoordinator {
     // we do not need to turn on reaper thread since no tasks will be expired and there are no completed tasks to be purged
     val txnMarkerPurgatory = DelayedOperationPurgatory[DelayedTxnMarker]("txn-marker-purgatory", config.brokerId,
       reaperEnabled = false, timerEnabled = false)
-    val txnStateManager = new TransactionStateManager(config.brokerId, zkClient, scheduler, replicaManager, txnConfig, time)
+    val txnStateManager = new TransactionStateManager(config.brokerId, zkClient, scheduler, replicaManager, txnConfig, time, metrics)
 
     val logContext = new LogContext(s"[TransactionCoordinator id=${config.brokerId}] ")
     val txnMarkerChannelManager = TransactionMarkerChannelManager(config, metrics, metadataCache, txnStateManager,
diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala
index 45ee4e998aec2..38caed5752672 100644
--- a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala
+++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala
@@ -30,6 +30,8 @@ import kafka.utils.{Logging, Pool, Scheduler}
 import kafka.zk.KafkaZkClient
 import org.apache.kafka.common.{KafkaException, TopicPartition}
 import org.apache.kafka.common.internals.Topic
+import org.apache.kafka.common.metrics.stats.{Avg, Max}
+import org.apache.kafka.common.metrics.{MetricConfig, Metrics}
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.record.{FileRecords, MemoryRecords, SimpleRecord}
 import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse
@@ -70,7 +72,8 @@ class TransactionStateManager(brokerId: Int,
                               scheduler: Scheduler,
                               replicaManager: ReplicaManager,
                               config: TransactionConfig,
-                              time: Time) extends Logging {
+                              time: Time,
+                              metrics: Metrics) extends Logging {
 
   this.logIdent = "[Transaction State Manager " + brokerId + "]: "
 
@@ -94,6 +97,16 @@ class TransactionStateManager(brokerId: Int,
   /** number of partitions for the transaction log topic */
   private val transactionTopicPartitionCount = getTransactionTopicPartitionCount
 
+  /** setup metrics*/
+  private val partitionLoadSensor = metrics.sensor("PartitionLoadTime")
+
+  partitionLoadSensor.add(metrics.metricName("partition-load-time-max",
+    "transaction-coordinator-metrics",
+    "The max time it took to load the partitions in the last 30sec"), new Max())
+  partitionLoadSensor.add(metrics.metricName("partition-load-time-avg",
+    "transaction-coordinator-metrics",
+    "The avg time it took to load the partitions in the last 30sec"), new Avg())
+
   // visible for testing only
   private[transaction] def addLoadingPartition(partitionId: Int, coordinatorEpoch: Int): Unit = {
     val partitionAndLeaderEpoch = TransactionPartitionAndLeaderEpoch(partitionId, coordinatorEpoch)
@@ -339,8 +352,10 @@ class TransactionStateManager(brokerId: Int,
                 currOffset = batch.nextOffset
               }
             }
-
-            info(s"Finished loading ${loadedTransactions.size} transaction metadata from $topicPartition in ${time.milliseconds() - startMs} milliseconds")
+            val endMs = time.milliseconds()
+            val timeLapse = endMs - startMs
+            partitionLoadSensor.record(timeLapse, endMs, false)
+            info(s"Finished loading ${loadedTransactions.size} transaction metadata from $topicPartition in $timeLapse milliseconds")
           }
         } catch {
           case t: Throwable => error(s"Error loading transactions from transaction log $topicPartition", t)
diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala
index 07ffe9dd266e9..6c433b7474e97 100755
--- a/core/src/main/scala/kafka/server/KafkaServer.scala
+++ b/core/src/main/scala/kafka/server/KafkaServer.scala
@@ -276,7 +276,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
 
         /* start group coordinator */
         // Hardcode Time.SYSTEM for now as some Streams tests fail otherwise, it would be good to fix the underlying issue
-        groupCoordinator = GroupCoordinator(config, zkClient, replicaManager, Time.SYSTEM)
+        groupCoordinator = GroupCoordinator(config, zkClient, replicaManager, Time.SYSTEM, metrics)
         groupCoordinator.startup()
 
         /* start transaction coordinator, with a separate background thread scheduler for transaction expiration and log loading */
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorConcurrencyTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorConcurrencyTest.scala
index b85035f64f2c6..ec31fc96e8554 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorConcurrencyTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorConcurrencyTest.scala
@@ -26,6 +26,7 @@ import kafka.coordinator.group.GroupCoordinatorConcurrencyTest._
 import kafka.server.{DelayedOperationPurgatory, KafkaConfig}
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.internals.Topic
+import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.requests.JoinGroupRequest
@@ -84,7 +85,7 @@ class GroupCoordinatorConcurrencyTest extends AbstractCoordinatorConcurrencyTest
     val heartbeatPurgatory = new DelayedOperationPurgatory[DelayedHeartbeat]("Heartbeat", timer, config.brokerId, reaperEnabled = false)
     val joinPurgatory = new DelayedOperationPurgatory[DelayedJoin]("Rebalance", timer, config.brokerId, reaperEnabled = false)
 
-    groupCoordinator = GroupCoordinator(config, zkClient, replicaManager, heartbeatPurgatory, joinPurgatory, timer.time)
+    groupCoordinator = GroupCoordinator(config, zkClient, replicaManager, heartbeatPurgatory, joinPurgatory, timer.time, new Metrics())
     groupCoordinator.startup(false)
   }
 
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
index cdf1518229987..5a35e33a0eb48 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
@@ -35,6 +35,7 @@ import java.util.concurrent.locks.ReentrantLock
 import kafka.cluster.Partition
 import kafka.zk.KafkaZkClient
 import org.apache.kafka.common.internals.Topic
+import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
 import org.junit.Assert._
 import org.junit.{After, Assert, Before, Test}
@@ -111,7 +112,7 @@ class GroupCoordinatorTest {
     val heartbeatPurgatory = new DelayedOperationPurgatory[DelayedHeartbeat]("Heartbeat", timer, config.brokerId, reaperEnabled = false)
     val joinPurgatory = new DelayedOperationPurgatory[DelayedJoin]("Rebalance", timer, config.brokerId, reaperEnabled = false)
 
-    groupCoordinator = GroupCoordinator(config, zkClient, replicaManager, heartbeatPurgatory, joinPurgatory, timer.time)
+    groupCoordinator = GroupCoordinator(config, zkClient, replicaManager, heartbeatPurgatory, joinPurgatory, timer.time, new Metrics())
     groupCoordinator.startup(enableMetadataExpiration = false)
 
     // add the partition into the owned partition list
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
index faca4479b17a3..dbcf5eda56e91 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
@@ -19,11 +19,12 @@ package kafka.coordinator.group
 
 import com.yammer.metrics.Metrics
 import com.yammer.metrics.core.Gauge
+import java.lang.management.ManagementFactory
 import java.nio.ByteBuffer
 import java.util.Collections
 import java.util.Optional
 import java.util.concurrent.locks.ReentrantLock
-
+import javax.management.ObjectName
 import kafka.api._
 import kafka.cluster.Partition
 import kafka.common.OffsetAndMetadata
@@ -35,6 +36,7 @@ import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription
 import org.apache.kafka.clients.consumer.internals.ConsumerProtocol
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.internals.Topic
+import org.apache.kafka.common.metrics.{JmxReporter, Metrics => kMetrics}
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.record._
 import org.apache.kafka.common.requests.OffsetFetchResponse
@@ -56,6 +58,7 @@ class GroupMetadataManagerTest {
   var zkClient: KafkaZkClient = null
   var partition: Partition = null
   var defaultOffsetRetentionMs = Long.MaxValue
+  var metrics: kMetrics = null
 
   val groupId = "foo"
   val groupInstanceId = Some("bar")
@@ -87,9 +90,10 @@ class GroupMetadataManagerTest {
     EasyMock.expect(zkClient.getTopicPartitionCount(Topic.GROUP_METADATA_TOPIC_NAME)).andReturn(Some(2))
     EasyMock.replay(zkClient)
 
+    metrics = new kMetrics()
     time = new MockTime
     replicaManager = EasyMock.createNiceMock(classOf[ReplicaManager])
-    groupMetadataManager = new GroupMetadataManager(0, ApiVersion.latestVersion, offsetConfig, replicaManager, zkClient, time)
+    groupMetadataManager = new GroupMetadataManager(0, ApiVersion.latestVersion, offsetConfig, replicaManager, zkClient, time, metrics)
     partition = EasyMock.niceMock(classOf[Partition])
   }
 
@@ -2051,4 +2055,43 @@ class GroupMetadataManagerTest {
     group.transitionTo(CompletingRebalance)
     expectMetrics(groupMetadataManager, 1, 0, 1)
   }
+
+  @Test
+  def testPartitionLoadMetric(): Unit = {
+    val server = ManagementFactory.getPlatformMBeanServer
+    val mBeanName = "kafka.server:type=group-coordinator-metrics"
+    val reporter = new JmxReporter("kafka.server")
+    metrics.addReporter(reporter)
+
+    def partitionLoadTime(attribute: String): Double = {
+      server.getAttribute(new ObjectName(mBeanName), attribute).asInstanceOf[Double]
+    }
+
+    assertTrue(server.isRegistered(new ObjectName(mBeanName)))
+    assertEquals(Double.NaN, partitionLoadTime( "partition-load-time-max"), 0)
+    assertEquals(Double.NaN, partitionLoadTime("partition-load-time-avg"), 0)
+    assertTrue(reporter.containsMbean(mBeanName))
+
+    val groupMetadataTopicPartition = groupTopicPartition
+    val startOffset = 15L
+    val memberId = "98098230493"
+    val committedOffsets = Map(
+      new TopicPartition("foo", 0) -> 23L,
+      new TopicPartition("foo", 1) -> 455L,
+      new TopicPartition("bar", 0) -> 8992L
+    )
+
+    val offsetCommitRecords = createCommittedOffsetRecords(committedOffsets)
+    val groupMetadataRecord = buildStableGroupRecordWithMember(generation = 15,
+      protocolType = "consumer", protocol = "range", memberId)
+    val records = MemoryRecords.withRecords(startOffset, CompressionType.NONE,
+      (offsetCommitRecords ++ Seq(groupMetadataRecord)).toArray: _*)
+
+    expectGroupMetadataLoad(groupMetadataTopicPartition, startOffset, records)
+    EasyMock.replay(replicaManager)
+    groupMetadataManager.loadGroupsAndOffsets(groupMetadataTopicPartition, _ => ())
+
+    assertTrue(partitionLoadTime("partition-load-time-max") >= 0.0)
+    assertTrue(partitionLoadTime( "partition-load-time-avg") >= 0.0)
+  }
 }
diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala
index bc6ed9330e3da..238ef4b5a3b08 100644
--- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala
@@ -28,6 +28,7 @@ import kafka.utils.{Pool, TestUtils}
 import org.apache.kafka.clients.{ClientResponse, NetworkClient}
 import org.apache.kafka.common.{Node, TopicPartition}
 import org.apache.kafka.common.internals.Topic.TRANSACTION_STATE_TOPIC_NAME
+import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
 import org.apache.kafka.common.record.{CompressionType, FileRecords, MemoryRecords, SimpleRecord}
 import org.apache.kafka.common.requests._
@@ -68,7 +69,7 @@ class TransactionCoordinatorConcurrencyTest extends AbstractCoordinatorConcurren
       .anyTimes()
     EasyMock.replay(zkClient)
 
-    txnStateManager = new TransactionStateManager(0, zkClient, scheduler, replicaManager, txnConfig, time)
+    txnStateManager = new TransactionStateManager(0, zkClient, scheduler, replicaManager, txnConfig, time, new Metrics())
     for (i <- 0 until numPartitions)
       txnStateManager.addLoadedTransactionsToCache(i, coordinatorEpoch, new Pool[String, TransactionMetadata]())
 
diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala
index 14745e7659065..4e778ddea63ac 100644
--- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala
@@ -16,8 +16,10 @@
  */
 package kafka.coordinator.transaction
 
+import java.lang.management.ManagementFactory
 import java.nio.ByteBuffer
 import java.util.concurrent.locks.ReentrantLock
+import javax.management.ObjectName
 
 import kafka.log.Log
 import kafka.server.{FetchDataInfo, FetchLogEnd, LogOffsetMetadata, ReplicaManager}
@@ -26,6 +28,7 @@ import org.scalatest.Assertions.fail
 import kafka.zk.KafkaZkClient
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.internals.Topic.TRANSACTION_STATE_TOPIC_NAME
+import org.apache.kafka.common.metrics.{JmxReporter, Metrics}
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.record._
 import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse
@@ -59,9 +62,10 @@ class TransactionStateManagerTest {
     .anyTimes()
 
   EasyMock.replay(zkClient)
+  val metrics = new Metrics()
 
   val txnConfig = TransactionConfig()
-  val transactionManager: TransactionStateManager = new TransactionStateManager(0, zkClient, scheduler, replicaManager, txnConfig, time)
+  val transactionManager: TransactionStateManager = new TransactionStateManager(0, zkClient, scheduler, replicaManager, txnConfig, time, metrics)
 
   val transactionalId1: String = "one"
   val transactionalId2: String = "two"
@@ -627,4 +631,37 @@ class TransactionStateManagerTest {
 
     EasyMock.replay(replicaManager)
   }
+
+  @Test
+  def testPartitionLoadMetric(): Unit = {
+    val server = ManagementFactory.getPlatformMBeanServer
+    val mBeanName = "kafka.server:type=transaction-coordinator-metrics"
+    val reporter = new JmxReporter("kafka.server")
+    metrics.addReporter(reporter)
+
+    def partitionLoadTime(attribute: String): Double = {
+      server.getAttribute(new ObjectName(mBeanName), attribute).asInstanceOf[Double]
+    }
+
+    assertTrue(server.isRegistered(new ObjectName(mBeanName)))
+    assertEquals(Double.NaN, partitionLoadTime( "partition-load-time-max"), 0)
+    assertEquals(Double.NaN, partitionLoadTime("partition-load-time-avg"), 0)
+    assertTrue(reporter.containsMbean(mBeanName))
+
+    txnMetadata1.state = Ongoing
+    txnMetadata1.addPartitions(Set[TopicPartition](new TopicPartition("topic1", 1),
+      new TopicPartition("topic1", 1)))
+
+    txnRecords += new SimpleRecord(txnMessageKeyBytes1, TransactionLog.valueToBytes(txnMetadata1.prepareNoTransit()))
+
+    val startOffset = 15L
+    val records = MemoryRecords.withRecords(startOffset, CompressionType.NONE, txnRecords.toArray: _*)
+
+    prepareTxnLog(topicPartition, startOffset, records)
+    transactionManager.loadTransactionsForTxnTopicPartition(partitionId, 0, (_, _, _, _, _) => ())
+    scheduler.tick()
+
+    assertTrue(partitionLoadTime("partition-load-time-max") >= 0)
+    assertTrue(partitionLoadTime( "partition-load-time-avg") >= 0)
+  }
 }
diff --git a/docs/ops.html b/docs/ops.html
index f94afc39e03cd..984b80d5990b2 100644
--- a/docs/ops.html
+++ b/docs/ops.html
@@ -1029,6 +1029,26 @@ 
        Security Considerations for Remote Mon
         Connection status of broker's ZooKeeper session which may be one of
             Disconnected|SyncConnected|AuthFailed|ConnectedReadOnly|SaslAuthenticated|Expired.
       
+      
+        Max time to load group metadata
+        kafka.server:type=group-coordinator-metrics,name=partition-load-time-max
+        maximum time, in milliseconds, it took to load offsets and group metadata from the consumer offset partitions loaded in the last 30 seconds
+      
+      
+        Avg time to load group metadata
+        kafka.server:type=group-coordinator-metrics,name=partition-load-time-avg
+        average time, in milliseconds, it took to load offsets and group metadata from the consumer offset partitions loaded in the last 30 seconds
+      
+      
+        Max time to load transaction metadata
+        kafka.server:type=transaction-coordinator-metrics,name=partition-load-time-max
+        maximum time, in milliseconds, it took to load transaction metadata from the consumer offset partitions loaded in the last 30 seconds
+      
+      
+        Avg time to load transaction metadata
+        kafka.server:type=transaction-coordinator-metrics,name=partition-load-time-avg
+        average time, in milliseconds, it took to load transaction metadata from the consumer offset partitions loaded in the last 30 seconds
+      
   
 
   
        Common monitoring metrics for producer/consumer/connect/streams
        

From f70ece26d1cce23556f8f69ca2ccb2bb9e4f4de1 Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Sat, 3 Aug 2019 23:59:06 -0700
Subject: [PATCH 0506/1071] MINOR: Fix potential bug in
 LogConfig.getConfigValue and improve test coverage (#7159)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

LogConfig.getConfigValue would throw a NoSuchElementException if any log
config was defined without a server default mapping.

Added a unit test for `getConfigValue` and a sanity test for
`toHtml`/`toRst`/`toEnrichedRst`, which were previously not exercised during
the test suite.

Reviewers: Jason Gustafson , José Armando García Sancio 
---
 .../clients/producer/ProducerConfig.java      |  2 +-
 core/src/main/scala/kafka/log/LogConfig.scala | 18 ++++++--
 .../scala/unit/kafka/log/LogConfigTest.scala  | 46 ++++++++++++++++++-
 3 files changed, 60 insertions(+), 6 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
index 4a4aebaf3248d..0d87284c1cf50 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
@@ -405,7 +405,7 @@ public static Set configNames() {
     }
 
     public static ConfigDef configDef() {
-        return  new ConfigDef(CONFIG);
+        return new ConfigDef(CONFIG);
     }
 
     public static void main(String[] args) {
diff --git a/core/src/main/scala/kafka/log/LogConfig.scala b/core/src/main/scala/kafka/log/LogConfig.scala
index bcc3f1208f397..c61e023a3682f 100755
--- a/core/src/main/scala/kafka/log/LogConfig.scala
+++ b/core/src/main/scala/kafka/log/LogConfig.scala
@@ -181,9 +181,17 @@ object LogConfig {
     "[PartitionId]:[BrokerId],[PartitionId]:[BrokerId]:... or alternatively the wildcard '*' can be used to throttle " +
     "all replicas for this topic."
 
-  private class LogConfigDef extends ConfigDef {
+  private[log] val ServerDefaultHeaderName = "Server Default Property"
+
+  // Package private for testing
+  private[log] class LogConfigDef(base: ConfigDef) extends ConfigDef(base) {
+    def this() = this(new ConfigDef)
 
     private final val serverDefaultConfigNames = mutable.Map[String, String]()
+    base match {
+      case b: LogConfigDef => serverDefaultConfigNames ++= b.serverDefaultConfigNames
+      case _ =>
+    }
 
     def define(name: String, defType: ConfigDef.Type, defaultValue: Any, validator: Validator,
                importance: ConfigDef.Importance, doc: String, serverDefaultConfigName: String): LogConfigDef = {
@@ -206,11 +214,12 @@ object LogConfig {
       this
     }
 
-    override def headers = List("Name", "Description", "Type", "Default", "Valid Values", "Server Default Property", "Importance").asJava
+    override def headers = List("Name", "Description", "Type", "Default", "Valid Values", ServerDefaultHeaderName,
+      "Importance").asJava
 
     override def getConfigValue(key: ConfigKey, headerName: String): String = {
       headerName match {
-        case "Server Default Property" => serverDefaultConfigNames.get(key.name).get
+        case ServerDefaultHeaderName => serverDefaultConfigNames.getOrElse(key.name, null)
         case _ => super.getConfigValue(key, headerName)
       }
     }
@@ -218,6 +227,9 @@ object LogConfig {
     def serverConfigName(configName: String): Option[String] = serverDefaultConfigNames.get(configName)
   }
 
+  // Package private for testing, return a copy since it's a mutable global variable
+  private[log] def configDefCopy: LogConfigDef = new LogConfigDef(configDef)
+
   private val configDef: LogConfigDef = {
     import org.apache.kafka.common.config.ConfigDef.Importance._
     import org.apache.kafka.common.config.ConfigDef.Range._
diff --git a/core/src/test/scala/unit/kafka/log/LogConfigTest.scala b/core/src/test/scala/unit/kafka/log/LogConfigTest.scala
index 2cd79041fb479..6f3332f98350c 100644
--- a/core/src/test/scala/unit/kafka/log/LogConfigTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogConfigTest.scala
@@ -19,9 +19,11 @@ package kafka.log
 
 import java.util.Properties
 
-import kafka.server.{ThrottledReplicaListValidator, KafkaConfig, KafkaServer}
+import kafka.server.{KafkaConfig, KafkaServer, ThrottledReplicaListValidator}
 import kafka.utils.TestUtils
-import org.apache.kafka.common.config.ConfigException
+import org.apache.kafka.common.config.ConfigDef.Importance.MEDIUM
+import org.apache.kafka.common.config.ConfigDef.Type.INT
+import org.apache.kafka.common.config.{ConfigException, TopicConfig}
 import org.junit.{Assert, Test}
 import org.junit.Assert._
 import org.scalatest.Assertions._
@@ -113,6 +115,46 @@ class LogConfigTest {
     assertFalse(isValid("100:0,10 :   "))
   }
 
+  /* Sanity check that toHtml produces one of the expected configs */
+  @Test
+  def testToHtml(): Unit = {
+    val html = LogConfig.configDefCopy.toHtmlTable
+    val expectedConfig = "file.delete.delay.ms"
+    assertTrue(s"Could not find `$expectedConfig` in:\n $html", html.contains(expectedConfig))
+  }
+
+  /* Sanity check that toEnrichedRst produces one of the expected configs */
+  @Test
+  def testToEnrichedRst(): Unit = {
+    val rst = LogConfig.configDefCopy.toEnrichedRst
+    val expectedConfig = "``file.delete.delay.ms``"
+    assertTrue(s"Could not find `$expectedConfig` in:\n $rst", rst.contains(expectedConfig))
+  }
+
+  /* Sanity check that toEnrichedRst produces one of the expected configs */
+  @Test
+  def testToRst(): Unit = {
+    val rst = LogConfig.configDefCopy.toRst
+    val expectedConfig = "``file.delete.delay.ms``"
+    assertTrue(s"Could not find `$expectedConfig` in:\n $rst", rst.contains(expectedConfig))
+  }
+
+  @Test
+  def testGetConfigValue(): Unit = {
+    // Add a config that doesn't set the `serverDefaultConfigName`
+    val configDef = LogConfig.configDefCopy
+    val configNameWithNoServerMapping = "log.foo"
+    configDef.define(configNameWithNoServerMapping, INT, 1, MEDIUM, s"$configNameWithNoServerMapping doc")
+
+    val deleteDelayKey = configDef.configKeys.get(TopicConfig.FILE_DELETE_DELAY_MS_CONFIG)
+    val deleteDelayServerDefault = configDef.getConfigValue(deleteDelayKey, LogConfig.ServerDefaultHeaderName)
+    assertEquals(KafkaConfig.LogDeleteDelayMsProp, deleteDelayServerDefault)
+
+    val keyWithNoServerMapping = configDef.configKeys.get(configNameWithNoServerMapping)
+    val nullServerDefault = configDef.getConfigValue(keyWithNoServerMapping, LogConfig.ServerDefaultHeaderName)
+    assertNull(nullServerDefault)
+  }
+
   private def isValid(configValue: String): Boolean = {
     try {
       ThrottledReplicaListValidator.ensureValidString("", configValue)

From 7663a6c44daae5d72f38cbba79d728416e11167d Mon Sep 17 00:00:00 2001
From: cadonna 
Date: Tue, 6 Aug 2019 17:51:08 +0200
Subject: [PATCH 0507/1071] Minor: Refactor methods to add metrics to sensor in
 `StreamsMetricsImpl` (#7161)

Renames method names in StreamsMetricsImpl to make them consistent.

Reviewers: A. Sophie Blee-Goldman , Guozhang Wang 
---
 .../kstream/internals/metrics/Sensors.java    |  2 +-
 .../processor/internals/ProcessorNode.java    | 13 ++---
 .../internals/metrics/StreamsMetricsImpl.java | 50 +++++++++----------
 .../internals/metrics/ThreadMetrics.java      | 30 +++++------
 .../AbstractRocksDBSegmentedBytesStore.java   |  4 +-
 .../state/internals/InMemorySessionStore.java |  8 +--
 .../state/internals/InMemoryWindowStore.java  |  4 +-
 .../state/internals/metrics/Sensors.java      | 12 ++---
 .../metrics/StreamsMetricsImplTest.java       | 20 ++++----
 .../internals/metrics/ThreadMetricsTest.java  | 26 +++++-----
 10 files changed, 85 insertions(+), 84 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/metrics/Sensors.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/metrics/Sensors.java
index 038b8ac77e9a3..363ec6e71e1db 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/metrics/Sensors.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/metrics/Sensors.java
@@ -44,7 +44,7 @@ public static Sensor lateRecordDropSensor(final InternalProcessorContext context
             LATE_RECORD_DROP,
             Sensor.RecordingLevel.INFO
         );
-        StreamsMetricsImpl.addInvocationRateAndCount(
+        StreamsMetricsImpl.addInvocationRateAndCountToSensor(
             sensor,
             PROCESSOR_NODE_METRICS_GROUP,
             metrics.tagMap("task-id", context.taskId().toString(), PROCESSOR_NODE_ID_TAG, context.currentNode().name()),
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorNode.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorNode.java
index 01e3e56bbb5b6..bc66edee307c7 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorNode.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorNode.java
@@ -33,8 +33,7 @@
 
 import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.PROCESSOR_NODE_ID_TAG;
 import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.PROCESSOR_NODE_METRICS_GROUP;
-import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addAvgMaxLatency;
-import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addInvocationRateAndCount;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addAvgAndMaxLatencyToSensor;
 
 public class ProcessorNode {
 
@@ -232,12 +231,14 @@ private static Sensor createTaskAndNodeLatencyAndThroughputSensors(final String
                                                                            final Map taskTags,
                                                                            final Map nodeTags) {
             final Sensor parent = metrics.taskLevelSensor(taskName, operation, Sensor.RecordingLevel.DEBUG);
-            addAvgMaxLatency(parent, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation);
-            addInvocationRateAndCount(parent, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation);
+            addAvgAndMaxLatencyToSensor(parent, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation);
+            StreamsMetricsImpl
+                .addInvocationRateAndCountToSensor(parent, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation);
 
             final Sensor sensor = metrics.nodeLevelSensor(taskName, processorNodeName, operation, Sensor.RecordingLevel.DEBUG, parent);
-            addAvgMaxLatency(sensor, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation);
-            addInvocationRateAndCount(sensor, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation);
+            addAvgAndMaxLatencyToSensor(sensor, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation);
+            StreamsMetricsImpl
+                .addInvocationRateAndCountToSensor(sensor, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation);
 
             return sensor;
         }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
index ae3d9534f8dd0..5ac2f33dabc1d 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
@@ -344,13 +344,13 @@ public Sensor addLatencyAndThroughputSensor(final String scopeName,
 
         // first add the global operation metrics if not yet, with the global tags only
         final Sensor parent = metrics.sensor(externalParentSensorName(operationName), recordingLevel);
-        addAvgMaxLatency(parent, group, allTagMap, operationName);
-        addInvocationRateAndCount(parent, group, allTagMap, operationName);
+        addAvgAndMaxLatencyToSensor(parent, group, allTagMap, operationName);
+        addInvocationRateAndCountToSensor(parent, group, allTagMap, operationName);
 
         // add the operation metrics with additional tags
         final Sensor sensor = metrics.sensor(externalChildSensorName(operationName, entityName), recordingLevel, parent);
-        addAvgMaxLatency(sensor, group, tagMap, operationName);
-        addInvocationRateAndCount(sensor, group, tagMap, operationName);
+        addAvgAndMaxLatencyToSensor(sensor, group, tagMap, operationName);
+        addInvocationRateAndCountToSensor(sensor, group, tagMap, operationName);
 
         parentSensors.put(sensor, parent);
 
@@ -374,11 +374,11 @@ public Sensor addThroughputSensor(final String scopeName,
 
         // first add the global operation metrics if not yet, with the global tags only
         final Sensor parent = metrics.sensor(externalParentSensorName(operationName), recordingLevel);
-        addInvocationRateAndCount(parent, group, allTagMap, operationName);
+        addInvocationRateAndCountToSensor(parent, group, allTagMap, operationName);
 
         // add the operation metrics with additional tags
         final Sensor sensor = metrics.sensor(externalChildSensorName(operationName, entityName), recordingLevel, parent);
-        addInvocationRateAndCount(sensor, group, tagMap, operationName);
+        addInvocationRateAndCountToSensor(sensor, group, tagMap, operationName);
 
         parentSensors.put(sensor, parent);
 
@@ -397,10 +397,10 @@ private String externalParentSensorName(final String operationName) {
     }
 
 
-    public static void addAvgAndMax(final Sensor sensor,
-                                    final String group,
-                                    final Map tags,
-                                    final String operation) {
+    public static void addAvgAndMaxToSensor(final Sensor sensor,
+                                            final String group,
+                                            final Map tags,
+                                            final String operation) {
         sensor.add(
             new MetricName(
                 operation + AVG_SUFFIX,
@@ -419,10 +419,10 @@ public static void addAvgAndMax(final Sensor sensor,
         );
     }
 
-    public static void addAvgMaxLatency(final Sensor sensor,
-                                        final String group,
-                                        final Map tags,
-                                        final String operation) {
+    public static void addAvgAndMaxLatencyToSensor(final Sensor sensor,
+                                                   final String group,
+                                                   final Map tags,
+                                                   final String operation) {
         sensor.add(
             new MetricName(
                 operation + "-latency-avg",
@@ -441,12 +441,12 @@ public static void addAvgMaxLatency(final Sensor sensor,
         );
     }
 
-    public static void addInvocationRateAndCount(final Sensor sensor,
-                                                 final String group,
-                                                 final Map tags,
-                                                 final String operation,
-                                                 final String descriptionOfInvocation,
-                                                 final String descriptionOfRate) {
+    public static void addInvocationRateAndCountToSensor(final Sensor sensor,
+                                                         final String group,
+                                                         final Map tags,
+                                                         final String operation,
+                                                         final String descriptionOfInvocation,
+                                                         final String descriptionOfRate) {
         sensor.add(
             new MetricName(
                 operation + TOTAL_SUFFIX,
@@ -467,11 +467,11 @@ public static void addInvocationRateAndCount(final Sensor sensor,
         );
     }
 
-    public static void addInvocationRateAndCount(final Sensor sensor,
-                                                 final String group,
-                                                 final Map tags,
-                                                 final String operation) {
-        addInvocationRateAndCount(
+    public static void addInvocationRateAndCountToSensor(final Sensor sensor,
+                                                         final String group,
+                                                         final Map tags,
+                                                         final String operation) {
+        addInvocationRateAndCountToSensor(
             sensor,
             group,
             tags,
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetrics.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetrics.java
index e177667b35f57..f8b7836fee18e 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetrics.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetrics.java
@@ -26,8 +26,8 @@
 import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.TASK_ID_TAG;
 import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.TASK_LEVEL_GROUP;
 import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.THREAD_LEVEL_GROUP;
-import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addAvgAndMax;
-import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addInvocationRateAndCount;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addAvgAndMaxToSensor;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addInvocationRateAndCountToSensor;
 
 public class ThreadMetrics {
     private ThreadMetrics() {}
@@ -74,7 +74,7 @@ private ThreadMetrics() {}
 
     public static Sensor createTaskSensor(final StreamsMetricsImpl streamsMetrics) {
         final Sensor createTaskSensor = streamsMetrics.threadLevelSensor(CREATE_TASK, RecordingLevel.INFO);
-        addInvocationRateAndCount(createTaskSensor,
+        addInvocationRateAndCountToSensor(createTaskSensor,
                                   THREAD_LEVEL_GROUP,
                                   streamsMetrics.threadLevelTagMap(),
                                   CREATE_TASK,
@@ -85,7 +85,7 @@ public static Sensor createTaskSensor(final StreamsMetricsImpl streamsMetrics) {
 
     public static Sensor closeTaskSensor(final StreamsMetricsImpl streamsMetrics) {
         final Sensor closeTaskSensor = streamsMetrics.threadLevelSensor(CLOSE_TASK, RecordingLevel.INFO);
-        addInvocationRateAndCount(closeTaskSensor,
+        addInvocationRateAndCountToSensor(closeTaskSensor,
                                   THREAD_LEVEL_GROUP,
                                   streamsMetrics.threadLevelTagMap(),
                                   CLOSE_TASK,
@@ -97,8 +97,8 @@ public static Sensor closeTaskSensor(final StreamsMetricsImpl streamsMetrics) {
     public static Sensor commitSensor(final StreamsMetricsImpl streamsMetrics) {
         final Sensor commitSensor = streamsMetrics.threadLevelSensor(COMMIT, Sensor.RecordingLevel.INFO);
         final Map tagMap = streamsMetrics.threadLevelTagMap();
-        addAvgAndMax(commitSensor, THREAD_LEVEL_GROUP, tagMap, COMMIT_LATENCY);
-        addInvocationRateAndCount(commitSensor,
+        addAvgAndMaxToSensor(commitSensor, THREAD_LEVEL_GROUP, tagMap, COMMIT_LATENCY);
+        addInvocationRateAndCountToSensor(commitSensor,
                                   THREAD_LEVEL_GROUP,
                                   tagMap,
                                   COMMIT,
@@ -110,8 +110,8 @@ public static Sensor commitSensor(final StreamsMetricsImpl streamsMetrics) {
     public static Sensor pollSensor(final StreamsMetricsImpl streamsMetrics) {
         final Sensor pollSensor = streamsMetrics.threadLevelSensor(POLL, Sensor.RecordingLevel.INFO);
         final Map tagMap = streamsMetrics.threadLevelTagMap();
-        addAvgAndMax(pollSensor, THREAD_LEVEL_GROUP, tagMap, POLL_LATENCY);
-        addInvocationRateAndCount(pollSensor,
+        addAvgAndMaxToSensor(pollSensor, THREAD_LEVEL_GROUP, tagMap, POLL_LATENCY);
+        addInvocationRateAndCountToSensor(pollSensor,
                                   THREAD_LEVEL_GROUP,
                                   tagMap,
                                   POLL,
@@ -123,8 +123,8 @@ public static Sensor pollSensor(final StreamsMetricsImpl streamsMetrics) {
     public static Sensor processSensor(final StreamsMetricsImpl streamsMetrics) {
         final Sensor processSensor = streamsMetrics.threadLevelSensor(PROCESS, Sensor.RecordingLevel.INFO);
         final Map tagMap = streamsMetrics.threadLevelTagMap();
-        addAvgAndMax(processSensor, THREAD_LEVEL_GROUP, tagMap, PROCESS_LATENCY);
-        addInvocationRateAndCount(processSensor,
+        addAvgAndMaxToSensor(processSensor, THREAD_LEVEL_GROUP, tagMap, PROCESS_LATENCY);
+        addInvocationRateAndCountToSensor(processSensor,
                                   THREAD_LEVEL_GROUP,
                                   tagMap,
                                   PROCESS,
@@ -137,8 +137,8 @@ public static Sensor processSensor(final StreamsMetricsImpl streamsMetrics) {
     public static Sensor punctuateSensor(final StreamsMetricsImpl streamsMetrics) {
         final Sensor punctuateSensor = streamsMetrics.threadLevelSensor(PUNCTUATE, Sensor.RecordingLevel.INFO);
         final Map tagMap = streamsMetrics.threadLevelTagMap();
-        addAvgAndMax(punctuateSensor, THREAD_LEVEL_GROUP, tagMap, PUNCTUATE_LATENCY);
-        addInvocationRateAndCount(punctuateSensor,
+        addAvgAndMaxToSensor(punctuateSensor, THREAD_LEVEL_GROUP, tagMap, PUNCTUATE_LATENCY);
+        addInvocationRateAndCountToSensor(punctuateSensor,
                                   THREAD_LEVEL_GROUP,
                                   tagMap,
                                   PUNCTUATE,
@@ -150,7 +150,7 @@ public static Sensor punctuateSensor(final StreamsMetricsImpl streamsMetrics) {
 
     public static Sensor skipRecordSensor(final StreamsMetricsImpl streamsMetrics) {
         final Sensor skippedRecordsSensor = streamsMetrics.threadLevelSensor(SKIP_RECORD, Sensor.RecordingLevel.INFO);
-        addInvocationRateAndCount(skippedRecordsSensor,
+        addInvocationRateAndCountToSensor(skippedRecordsSensor,
                                   THREAD_LEVEL_GROUP,
                                   streamsMetrics.threadLevelTagMap(),
                                   SKIP_RECORD,
@@ -163,11 +163,11 @@ public static Sensor skipRecordSensor(final StreamsMetricsImpl streamsMetrics) {
     public static Sensor commitOverTasksSensor(final StreamsMetricsImpl streamsMetrics) {
         final Sensor commitOverTasksSensor = streamsMetrics.threadLevelSensor(COMMIT, Sensor.RecordingLevel.DEBUG);
         final Map tagMap = streamsMetrics.threadLevelTagMap(TASK_ID_TAG, ALL_TASKS);
-        addAvgAndMax(commitOverTasksSensor,
+        addAvgAndMaxToSensor(commitOverTasksSensor,
                      TASK_LEVEL_GROUP,
                      tagMap,
                      COMMIT_LATENCY);
-        addInvocationRateAndCount(commitOverTasksSensor,
+        addInvocationRateAndCountToSensor(commitOverTasksSensor,
                                   TASK_LEVEL_GROUP,
                                   tagMap,
                                   COMMIT,
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/AbstractRocksDBSegmentedBytesStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/AbstractRocksDBSegmentedBytesStore.java
index ef18d3ca28b62..97dc8d5bc5f6b 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/AbstractRocksDBSegmentedBytesStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/AbstractRocksDBSegmentedBytesStore.java
@@ -41,7 +41,7 @@
 import java.util.Set;
 
 import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.EXPIRED_WINDOW_RECORD_DROP;
-import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addInvocationRateAndCount;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addInvocationRateAndCountToSensor;
 
 public class AbstractRocksDBSegmentedBytesStore implements SegmentedBytesStore {
     private static final Logger LOG = LoggerFactory.getLogger(AbstractRocksDBSegmentedBytesStore.class);
@@ -182,7 +182,7 @@ public void init(final ProcessorContext context,
             EXPIRED_WINDOW_RECORD_DROP,
             Sensor.RecordingLevel.INFO
         );
-        addInvocationRateAndCount(
+        addInvocationRateAndCountToSensor(
             expiredRecordSensor,
             "stream-" + metricScope + "-metrics",
             metrics.tagMap("task-id", taskName, metricScope + "-id", name()),
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java
index ebe9878fe2721..6c64b049ffb3c 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java
@@ -16,9 +16,6 @@
  */
 package org.apache.kafka.streams.state.internals;
 
-import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.EXPIRED_WINDOW_RECORD_DROP;
-import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addInvocationRateAndCount;
-
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -43,6 +40,9 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.EXPIRED_WINDOW_RECORD_DROP;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addInvocationRateAndCountToSensor;
+
 public class InMemorySessionStore implements SessionStore {
 
     private static final Logger LOG = LoggerFactory.getLogger(InMemorySessionStore.class);
@@ -82,7 +82,7 @@ public void init(final ProcessorContext context, final StateStore root) {
             EXPIRED_WINDOW_RECORD_DROP,
             Sensor.RecordingLevel.INFO
         );
-        addInvocationRateAndCount(
+        addInvocationRateAndCountToSensor(
             expiredRecordSensor,
             "stream-" + metricScope + "-metrics",
             metrics.tagMap("task-id", taskName, metricScope + "-id", name()),
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryWindowStore.java
index 8063410212eb6..1a3e26b7fd2a7 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryWindowStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryWindowStore.java
@@ -43,7 +43,7 @@
 import java.util.NoSuchElementException;
 
 import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.EXPIRED_WINDOW_RECORD_DROP;
-import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addInvocationRateAndCount;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addInvocationRateAndCountToSensor;
 import static org.apache.kafka.streams.state.internals.WindowKeySchema.extractStoreKeyBytes;
 import static org.apache.kafka.streams.state.internals.WindowKeySchema.extractStoreTimestamp;
 
@@ -98,7 +98,7 @@ public void init(final ProcessorContext context, final StateStore root) {
             EXPIRED_WINDOW_RECORD_DROP,
             Sensor.RecordingLevel.INFO
         );
-        addInvocationRateAndCount(
+        addInvocationRateAndCountToSensor(
             expiredRecordSensor,
             "stream-" + metricScope + "-metrics",
             metrics.tagMap("task-id", taskName, metricScope + "-id", name()),
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/Sensors.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/Sensors.java
index 13a39c652f5f9..8ed4d47f24d53 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/Sensors.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/Sensors.java
@@ -27,8 +27,8 @@
 
 import java.util.Map;
 
-import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addAvgMaxLatency;
-import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addInvocationRateAndCount;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addAvgAndMaxLatencyToSensor;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addInvocationRateAndCountToSensor;
 
 public final class Sensors {
     private Sensors() {}
@@ -42,11 +42,11 @@ public static Sensor createTaskAndStoreLatencyAndThroughputSensors(final Sensor.
                                                                        final Map taskTags,
                                                                        final Map storeTags) {
         final Sensor taskSensor = metrics.taskLevelSensor(taskName, operation, level);
-        addAvgMaxLatency(taskSensor, metricsGroup, taskTags, operation);
-        addInvocationRateAndCount(taskSensor, metricsGroup, taskTags, operation);
+        addAvgAndMaxLatencyToSensor(taskSensor, metricsGroup, taskTags, operation);
+        addInvocationRateAndCountToSensor(taskSensor, metricsGroup, taskTags, operation);
         final Sensor sensor = metrics.storeLevelSensor(taskName, storeName, operation, level, taskSensor);
-        addAvgMaxLatency(sensor, metricsGroup, storeTags, operation);
-        addInvocationRateAndCount(sensor, metricsGroup, storeTags, operation);
+        addAvgAndMaxLatencyToSensor(sensor, metricsGroup, storeTags, operation);
+        addInvocationRateAndCountToSensor(sensor, metricsGroup, storeTags, operation);
         return sensor;
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java
index 678d9f30635ab..4fd6f88a95df0 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java
@@ -35,8 +35,8 @@
 import static org.apache.kafka.common.utils.Utils.mkEntry;
 import static org.apache.kafka.common.utils.Utils.mkMap;
 import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.PROCESSOR_NODE_METRICS_GROUP;
-import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addAvgMaxLatency;
-import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addInvocationRateAndCount;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addAvgAndMaxLatencyToSensor;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addInvocationRateAndCountToSensor;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -131,14 +131,14 @@ public void testMutiLevelSensorRemoval() {
         final Map nodeTags = mkMap(mkEntry("nkey", "value"));
 
         final Sensor parent1 = metrics.taskLevelSensor(taskName, operation, Sensor.RecordingLevel.DEBUG);
-        addAvgMaxLatency(parent1, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation);
-        addInvocationRateAndCount(parent1, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation, "", "");
+        addAvgAndMaxLatencyToSensor(parent1, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation);
+        addInvocationRateAndCountToSensor(parent1, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation, "", "");
 
         final int numberOfTaskMetrics = registry.metrics().size();
 
         final Sensor sensor1 = metrics.nodeLevelSensor(taskName, processorNodeName, operation, Sensor.RecordingLevel.DEBUG, parent1);
-        addAvgMaxLatency(sensor1, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation);
-        addInvocationRateAndCount(sensor1, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation, "", "");
+        addAvgAndMaxLatencyToSensor(sensor1, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation);
+        addInvocationRateAndCountToSensor(sensor1, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation, "", "");
 
         assertThat(registry.metrics().size(), greaterThan(numberOfTaskMetrics));
 
@@ -147,14 +147,14 @@ public void testMutiLevelSensorRemoval() {
         assertThat(registry.metrics().size(), equalTo(numberOfTaskMetrics));
 
         final Sensor parent2 = metrics.taskLevelSensor(taskName, operation, Sensor.RecordingLevel.DEBUG);
-        addAvgMaxLatency(parent2, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation);
-        addInvocationRateAndCount(parent2, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation, "", "");
+        addAvgAndMaxLatencyToSensor(parent2, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation);
+        addInvocationRateAndCountToSensor(parent2, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation, "", "");
 
         assertThat(registry.metrics().size(), equalTo(numberOfTaskMetrics));
 
         final Sensor sensor2 = metrics.nodeLevelSensor(taskName, processorNodeName, operation, Sensor.RecordingLevel.DEBUG, parent2);
-        addAvgMaxLatency(sensor2, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation);
-        addInvocationRateAndCount(sensor2, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation, "", "");
+        addAvgAndMaxLatencyToSensor(sensor2, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation);
+        addInvocationRateAndCountToSensor(sensor2, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation, "", "");
 
         assertThat(registry.metrics().size(), greaterThan(numberOfTaskMetrics));
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetricsTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetricsTest.java
index 89395d9c51a8c..739f028092155 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetricsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetricsTest.java
@@ -59,7 +59,7 @@ public void shouldGetCreateTaskSensor() {
         mockStatic(StreamsMetricsImpl.class);
         expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.INFO)).andReturn(dummySensor);
         expect(streamsMetrics.threadLevelTagMap()).andReturn(dummyTagMap);
-        StreamsMetricsImpl.addInvocationRateAndCount(
+        StreamsMetricsImpl.addInvocationRateAndCountToSensor(
             dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
 
         replayAll();
@@ -81,7 +81,7 @@ public void shouldGetCloseTaskSensor() {
         mockStatic(StreamsMetricsImpl.class);
         expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.INFO)).andReturn(dummySensor);
         expect(streamsMetrics.threadLevelTagMap()).andReturn(dummyTagMap);
-        StreamsMetricsImpl.addInvocationRateAndCount(
+        StreamsMetricsImpl.addInvocationRateAndCountToSensor(
             dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
 
         replayAll();
@@ -104,9 +104,9 @@ public void shouldGetCommitSensor() {
         mockStatic(StreamsMetricsImpl.class);
         expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.INFO)).andReturn(dummySensor);
         expect(streamsMetrics.threadLevelTagMap()).andReturn(dummyTagMap);
-        StreamsMetricsImpl.addInvocationRateAndCount(
+        StreamsMetricsImpl.addInvocationRateAndCountToSensor(
             dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
-        StreamsMetricsImpl.addAvgAndMax(
+        StreamsMetricsImpl.addAvgAndMaxToSensor(
             dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operationLatency);
 
         replayAll();
@@ -129,9 +129,9 @@ public void shouldGetPollSensor() {
         mockStatic(StreamsMetricsImpl.class);
         expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.INFO)).andReturn(dummySensor);
         expect(streamsMetrics.threadLevelTagMap()).andReturn(dummyTagMap);
-        StreamsMetricsImpl.addInvocationRateAndCount(
+        StreamsMetricsImpl.addInvocationRateAndCountToSensor(
             dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
-        StreamsMetricsImpl.addAvgAndMax(
+        StreamsMetricsImpl.addAvgAndMaxToSensor(
             dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operationLatency);
 
         replayAll();
@@ -154,9 +154,9 @@ public void shouldGetProcessSensor() {
         mockStatic(StreamsMetricsImpl.class);
         expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.INFO)).andReturn(dummySensor);
         expect(streamsMetrics.threadLevelTagMap()).andReturn(dummyTagMap);
-        StreamsMetricsImpl.addInvocationRateAndCount(
+        StreamsMetricsImpl.addInvocationRateAndCountToSensor(
             dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
-        StreamsMetricsImpl.addAvgAndMax(
+        StreamsMetricsImpl.addAvgAndMaxToSensor(
             dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operationLatency);
 
         replayAll();
@@ -179,9 +179,9 @@ public void shouldGetPunctuateSensor() {
         mockStatic(StreamsMetricsImpl.class);
         expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.INFO)).andReturn(dummySensor);
         expect(streamsMetrics.threadLevelTagMap()).andReturn(dummyTagMap);
-        StreamsMetricsImpl.addInvocationRateAndCount(
+        StreamsMetricsImpl.addInvocationRateAndCountToSensor(
             dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
-        StreamsMetricsImpl.addAvgAndMax(
+        StreamsMetricsImpl.addAvgAndMaxToSensor(
             dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operationLatency);
 
         replayAll();
@@ -203,7 +203,7 @@ public void shouldGetSkipRecordSensor() {
         mockStatic(StreamsMetricsImpl.class);
         expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.INFO)).andReturn(dummySensor);
         expect(streamsMetrics.threadLevelTagMap()).andReturn(dummyTagMap);
-        StreamsMetricsImpl.addInvocationRateAndCount(
+        StreamsMetricsImpl.addInvocationRateAndCountToSensor(
             dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
 
         replayAll();
@@ -226,9 +226,9 @@ public void shouldGetCommitOverTasksSensor() {
         mockStatic(StreamsMetricsImpl.class);
         expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.DEBUG)).andReturn(dummySensor);
         expect(streamsMetrics.threadLevelTagMap(TASK_ID_TAG, ALL_TASKS)).andReturn(dummyTagMap);
-        StreamsMetricsImpl.addInvocationRateAndCount(
+        StreamsMetricsImpl.addInvocationRateAndCountToSensor(
             dummySensor, TASK_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
-        StreamsMetricsImpl.addAvgAndMax(
+        StreamsMetricsImpl.addAvgAndMaxToSensor(
             dummySensor, TASK_LEVEL_GROUP, dummyTagMap, operationLatency);
 
         replayAll();

From 7ebcd50fbf70dcf401094f6f28dc615820835fb1 Mon Sep 17 00:00:00 2001
From: mjarvie <30092781+mjarvie@users.noreply.github.com>
Date: Tue, 6 Aug 2019 10:05:44 -0700
Subject: [PATCH 0508/1071] KAFKA-8736: Streams performance improvement, use
 isEmpty() rather than size() == 0 (#7164)

Reviewers: A. Sophie Blee-Goldman , Matthias J. Sax , Guozhang Wang 
---
 .../org/apache/kafka/streams/state/internals/NamedCache.java  | 4 ++++
 .../org/apache/kafka/streams/state/internals/ThreadCache.java | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
index 0201f20f9fbf4..ec53dfa5d46cd 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
@@ -266,6 +266,10 @@ public long size() {
         return cache.size();
     }
 
+    public boolean isEmpty() {
+        return cache.isEmpty();
+    }
+
     synchronized Iterator> subMapIterator(final Bytes from, final Bytes to) {
         return cache.subMap(from, true, to, true).entrySet().iterator();
     }
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java
index 0db6c78a5772e..116b1ef8b7d1a 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java
@@ -193,7 +193,7 @@ public MemoryLRUCacheBytesIterator all(final String namespace) {
         }
         return new MemoryLRUCacheBytesIterator(cache.allIterator());
     }
-    
+
     public long size() {
         long size = 0;
         for (final NamedCache cache : caches.values()) {
@@ -235,7 +235,7 @@ private void maybeEvict(final String namespace) {
             // a put on another cache. So even though the sizeInBytes() is
             // still > maxCacheSizeBytes there is nothing to evict from this
             // namespaced cache.
-            if (cache.size() == 0) {
+            if (cache.isEmpty()) {
                 return;
             }
             cache.evict();

From e4e2dd31ca66330aefdbb226eb25968bb14fee64 Mon Sep 17 00:00:00 2001
From: "Colin P. Mccabe" 
Date: Tue, 6 Aug 2019 12:25:23 -0700
Subject: [PATCH 0509/1071] MINOR: some small style fixes to
 RoundRobinPartitioner

Author: Colin P. Mccabe 

Reviewers: Gwen Shapira

Closes #7058 from cmccabe/rr-style-fixes
---
 .../producer/RoundRobinPartitioner.java       |  2 +-
 .../producer/RoundRobinPartitionerTest.java   | 60 ++++++++++---------
 2 files changed, 33 insertions(+), 29 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/RoundRobinPartitioner.java b/clients/src/main/java/org/apache/kafka/clients/producer/RoundRobinPartitioner.java
index 8c21164f3e979..80c47252b13f2 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/RoundRobinPartitioner.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/RoundRobinPartitioner.java
@@ -35,7 +35,6 @@
  *
  */
 public class RoundRobinPartitioner implements Partitioner {
-
     private final ConcurrentMap topicCounterMap = new ConcurrentHashMap<>();
 
     public void configure(Map configs) {}
@@ -50,6 +49,7 @@ public void configure(Map configs) {}
      * @param valueBytes serialized value to partition on or null
      * @param cluster The current cluster metadata
      */
+    @Override
     public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {
         List partitions = cluster.partitionsForTopic(topic);
         int numPartitions = partitions.size();
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/RoundRobinPartitionerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/RoundRobinPartitionerTest.java
index d1c326bbc88ce..c6bb616047061 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/RoundRobinPartitionerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/RoundRobinPartitionerTest.java
@@ -30,29 +30,30 @@
 import static org.junit.Assert.assertTrue;
 
 public class RoundRobinPartitionerTest {
-    private byte[] keyBytes = "key".getBytes();
-    private Partitioner partitioner = new RoundRobinPartitioner();
-    private Node node0 = new Node(0, "localhost", 99);
-    private Node node1 = new Node(1, "localhost", 100);
-    private Node node2 = new Node(2, "localhost", 101);
-    private Node[] nodes = new Node[] {node0, node1, node2};
-    private String topic = "test";
-    // Intentionally make the partition list not in partition order to test the edge
-    // cases.
-    private List partitions = asList(new PartitionInfo(topic, 1, null, nodes, nodes),
-            new PartitionInfo(topic, 2, node1, nodes, nodes), new PartitionInfo(topic, 0, node0, nodes, nodes));
-    private Cluster cluster = new Cluster("clusterId", asList(node0, node1, node2), partitions,
-            Collections.emptySet(), Collections.emptySet());
+    private final static Node[] NODES = new Node[] {
+        new Node(0, "localhost", 99),
+        new Node(1, "localhost", 100),
+        new Node(2, "localhost", 101)
+    };
 
     @Test
     public void testRoundRobinWithUnavailablePartitions() {
+        // Intentionally make the partition list not in partition order to test the edge
+        // cases.
+        List partitions = asList(
+                new PartitionInfo("test", 1, null, NODES, NODES),
+                new PartitionInfo("test", 2, NODES[1], NODES, NODES),
+                new PartitionInfo("test", 0, NODES[0], NODES, NODES));
         // When there are some unavailable partitions, we want to make sure that (1) we
         // always pick an available partition,
         // and (2) the available partitions are selected in a round robin way.
         int countForPart0 = 0;
         int countForPart2 = 0;
+        Partitioner partitioner = new RoundRobinPartitioner();
+        Cluster cluster = new Cluster("clusterId", asList(NODES[0], NODES[1], NODES[2]), partitions,
+            Collections.emptySet(), Collections.emptySet());
         for (int i = 1; i <= 100; i++) {
-            int part = partitioner.partition(topic, null, null, null, null, cluster);
+            int part = partitioner.partition("test", null, null, null, null, cluster);
             assertTrue("We should never choose a leader-less node in round robin", part == 0 || part == 2);
             if (part == 0)
                 countForPart0++;
@@ -67,14 +68,16 @@ public void testRoundRobinWithKeyBytes() throws InterruptedException {
         final String topicA = "topicA";
         final String topicB = "topicB";
 
-        List allPartitions = asList(new PartitionInfo(topicA, 0, node0, nodes, nodes),
-                new PartitionInfo(topicA, 1, node1, nodes, nodes), new PartitionInfo(topicA, 2, node2, nodes, nodes),
-                new PartitionInfo(topicB, 0, node0, nodes, nodes));
-        Cluster testCluster = new Cluster("clusterId", asList(node0, node1, node2), allPartitions,
+        List allPartitions = asList(new PartitionInfo(topicA, 0, NODES[0], NODES, NODES),
+                new PartitionInfo(topicA, 1, NODES[1], NODES, NODES), new PartitionInfo(topicA, 2, NODES[2], NODES, NODES),
+                new PartitionInfo(topicB, 0, NODES[0], NODES, NODES));
+        Cluster testCluster = new Cluster("clusterId", asList(NODES[0], NODES[1], NODES[2]), allPartitions,
                 Collections.emptySet(), Collections.emptySet());
 
         final Map partitionCount = new HashMap<>();
 
+        final byte[] keyBytes = "key".getBytes();
+        Partitioner partitioner = new RoundRobinPartitioner();
         for (int i = 0; i < 30; ++i) {
             int partition = partitioner.partition(topicA, null, keyBytes, null, null, testCluster);
             Integer count = partitionCount.get(partition);
@@ -87,9 +90,9 @@ public void testRoundRobinWithKeyBytes() throws InterruptedException {
             }
         }
 
-        assertEquals(10, (int) partitionCount.get(0));
-        assertEquals(10, (int) partitionCount.get(1));
-        assertEquals(10, (int) partitionCount.get(2));
+        assertEquals(10, partitionCount.get(0).intValue());
+        assertEquals(10, partitionCount.get(1).intValue());
+        assertEquals(10, partitionCount.get(2).intValue());
     }
     
     @Test
@@ -97,14 +100,15 @@ public void testRoundRobinWithNullKeyBytes() throws InterruptedException {
         final String topicA = "topicA";
         final String topicB = "topicB";
 
-        List allPartitions = asList(new PartitionInfo(topicA, 0, node0, nodes, nodes),
-                new PartitionInfo(topicA, 1, node1, nodes, nodes), new PartitionInfo(topicA, 2, node2, nodes, nodes),
-                new PartitionInfo(topicB, 0, node0, nodes, nodes));
-        Cluster testCluster = new Cluster("clusterId", asList(node0, node1, node2), allPartitions,
+        List allPartitions = asList(new PartitionInfo(topicA, 0, NODES[0], NODES, NODES),
+                new PartitionInfo(topicA, 1, NODES[1], NODES, NODES), new PartitionInfo(topicA, 2, NODES[2], NODES, NODES),
+                new PartitionInfo(topicB, 0, NODES[0], NODES, NODES));
+        Cluster testCluster = new Cluster("clusterId", asList(NODES[0], NODES[1], NODES[2]), allPartitions,
                 Collections.emptySet(), Collections.emptySet());
 
         final Map partitionCount = new HashMap<>();
 
+        Partitioner partitioner = new RoundRobinPartitioner();
         for (int i = 0; i < 30; ++i) {
             int partition = partitioner.partition(topicA, null, null, null, null, testCluster);
             Integer count = partitionCount.get(partition);
@@ -117,8 +121,8 @@ public void testRoundRobinWithNullKeyBytes() throws InterruptedException {
             }
         }
 
-        assertEquals(10, (int) partitionCount.get(0));
-        assertEquals(10, (int) partitionCount.get(1));
-        assertEquals(10, (int) partitionCount.get(2));
+        assertEquals(10, partitionCount.get(0).intValue());
+        assertEquals(10, partitionCount.get(1).intValue());
+        assertEquals(10, partitionCount.get(2).intValue());
     }    
 }

From 97b731b0866c1e78b9136e8bc088f8a1a19b966a Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Tue, 6 Aug 2019 23:01:10 -0700
Subject: [PATCH 0510/1071] MINOR: Upgrade jackson-databind to 2.9.9.3 (#7125)

2.9.9.1 and 2.9.9.2 include security fixes while 2.9.9.3 fixes a regression
introduced in 2.9.9.2.

Reviewers: Manikumar Reddy 
---
 gradle/dependencies.gradle | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle
index 5dfbf043bc613..d4368338995f6 100644
--- a/gradle/dependencies.gradle
+++ b/gradle/dependencies.gradle
@@ -72,6 +72,7 @@ versions += [
   httpclient: "4.5.8",
   easymock: "4.0.2",
   jackson: "2.9.9",
+  jacksonDatabind: "2.9.9.3",
   jacoco: "0.8.3",
   jetty: "9.4.18.v20190429",
   jersey: "2.28",
@@ -132,7 +133,7 @@ libs += [
   bcpkix: "org.bouncycastle:bcpkix-jdk15on:$versions.bcpkix",
   commonsCli: "commons-cli:commons-cli:$versions.commonsCli",
   easymock: "org.easymock:easymock:$versions.easymock",
-  jacksonDatabind: "com.fasterxml.jackson.core:jackson-databind:$versions.jackson",
+  jacksonDatabind: "com.fasterxml.jackson.core:jackson-databind:$versions.jacksonDatabind",
   jacksonDataformatCsv: "com.fasterxml.jackson.dataformat:jackson-dataformat-csv:$versions.jackson",
   jacksonModuleScala: "com.fasterxml.jackson.module:jackson-module-scala_$versions.baseScala:$versions.jackson",
   jacksonJDK8Datatypes: "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:$versions.jackson",

From 926fb35d9dcefd45c1e1d276ee7252b15875f23e Mon Sep 17 00:00:00 2001
From: Mickael Maison 
Date: Wed, 7 Aug 2019 13:32:26 +0530
Subject: [PATCH 0511/1071] KAFKA-8599: Use automatic RPC generation in
 ExpireDelegationToken

Author: Mickael Maison 

Reviewers: Manikumar Reddy , Viktor Somogyi 

Closes #7098 from mimaison/KAFKA-8599
---
 .../kafka/clients/admin/KafkaAdminClient.java |  8 +-
 .../apache/kafka/common/protocol/ApiKeys.java |  6 +-
 .../kafka/common/protocol/types/Struct.java   |  1 +
 .../common/requests/AbstractResponse.java     |  2 +-
 .../ExpireDelegationTokenRequest.java         | 77 ++++++-------------
 .../ExpireDelegationTokenResponse.java        | 70 ++++-------------
 .../common/requests/RequestResponseTest.java  | 13 +++-
 .../main/scala/kafka/server/KafkaApis.scala   |  7 +-
 .../unit/kafka/server/RequestQuotaTest.scala  |  8 +-
 9 files changed, 73 insertions(+), 119 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
index 8092eec82fd72..5256e368891ca 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
@@ -73,6 +73,7 @@
 import org.apache.kafka.common.message.DescribeGroupsRequestData;
 import org.apache.kafka.common.message.DescribeGroupsResponseData.DescribedGroup;
 import org.apache.kafka.common.message.DescribeGroupsResponseData.DescribedGroupMember;
+import org.apache.kafka.common.message.ExpireDelegationTokenRequestData;
 import org.apache.kafka.common.message.FindCoordinatorRequestData;
 import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData;
 import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData.AlterableConfigCollection;
@@ -2473,8 +2474,11 @@ public ExpireDelegationTokenResult expireDelegationToken(final byte[] hmac, fina
             new LeastLoadedNodeProvider()) {
 
             @Override
-            AbstractRequest.Builder createRequest(int timeoutMs) {
-                return new ExpireDelegationTokenRequest.Builder(hmac, options.expiryTimePeriodMs());
+            AbstractRequest.Builder createRequest(int timeoutMs) {
+                return new ExpireDelegationTokenRequest.Builder(
+                        new ExpireDelegationTokenRequestData()
+                            .setHmac(hmac)
+                            .setExpiryTimePeriodMs(options.expiryTimePeriodMs()));
             }
 
             @Override
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
index 3f0f5e854013d..e05f692305f75 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
@@ -30,6 +30,8 @@
 import org.apache.kafka.common.message.DescribeGroupsResponseData;
 import org.apache.kafka.common.message.ElectLeadersRequestData;
 import org.apache.kafka.common.message.ElectLeadersResponseData;
+import org.apache.kafka.common.message.ExpireDelegationTokenRequestData;
+import org.apache.kafka.common.message.ExpireDelegationTokenResponseData;
 import org.apache.kafka.common.message.FindCoordinatorRequestData;
 import org.apache.kafka.common.message.FindCoordinatorResponseData;
 import org.apache.kafka.common.message.HeartbeatRequestData;
@@ -93,8 +95,6 @@
 import org.apache.kafka.common.requests.DescribeLogDirsResponse;
 import org.apache.kafka.common.requests.EndTxnRequest;
 import org.apache.kafka.common.requests.EndTxnResponse;
-import org.apache.kafka.common.requests.ExpireDelegationTokenRequest;
-import org.apache.kafka.common.requests.ExpireDelegationTokenResponse;
 import org.apache.kafka.common.requests.FetchRequest;
 import org.apache.kafka.common.requests.FetchResponse;
 import org.apache.kafka.common.requests.LeaderAndIsrRequest;
@@ -191,7 +191,7 @@ public Struct parseResponse(short version, ByteBuffer buffer) {
             CreatePartitionsResponse.schemaVersions()),
     CREATE_DELEGATION_TOKEN(38, "CreateDelegationToken", CreateDelegationTokenRequestData.SCHEMAS, CreateDelegationTokenResponseData.SCHEMAS),
     RENEW_DELEGATION_TOKEN(39, "RenewDelegationToken", RenewDelegationTokenRequest.schemaVersions(), RenewDelegationTokenResponse.schemaVersions()),
-    EXPIRE_DELEGATION_TOKEN(40, "ExpireDelegationToken", ExpireDelegationTokenRequest.schemaVersions(), ExpireDelegationTokenResponse.schemaVersions()),
+    EXPIRE_DELEGATION_TOKEN(40, "ExpireDelegationToken", ExpireDelegationTokenRequestData.SCHEMAS, ExpireDelegationTokenResponseData.SCHEMAS),
     DESCRIBE_DELEGATION_TOKEN(41, "DescribeDelegationToken", DescribeDelegationTokenRequest.schemaVersions(), DescribeDelegationTokenResponse.schemaVersions()),
     DELETE_GROUPS(42, "DeleteGroups", DeleteGroupsRequestData.SCHEMAS, DeleteGroupsResponseData.SCHEMAS),
     ELECT_LEADERS(43, "ElectLeaders", ElectLeadersRequestData.SCHEMAS,
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
index 3114aeaeac5a3..e47a2cdce2a6a 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
@@ -290,6 +290,7 @@ public byte[] getByteArray(String name) {
         ByteBuffer buf = (ByteBuffer) result;
         byte[] arr = new byte[buf.remaining()];
         buf.get(arr);
+        buf.flip();
         return arr;
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
index eb52fb805ae52..da2b837054660 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
@@ -151,7 +151,7 @@ public static AbstractResponse parseResponse(ApiKeys apiKey, Struct struct, shor
             case RENEW_DELEGATION_TOKEN:
                 return new RenewDelegationTokenResponse(struct);
             case EXPIRE_DELEGATION_TOKEN:
-                return new ExpireDelegationTokenResponse(struct);
+                return new ExpireDelegationTokenResponse(struct, version);
             case DESCRIBE_DELEGATION_TOKEN:
                 return new DescribeDelegationTokenResponse(struct);
             case DELETE_GROUPS:
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java
index 5b996763cca9e..ca6d2d67d1f79 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java
@@ -16,102 +16,69 @@
  */
 package org.apache.kafka.common.requests;
 
+import java.nio.ByteBuffer;
+
+import org.apache.kafka.common.message.ExpireDelegationTokenRequestData;
+import org.apache.kafka.common.message.ExpireDelegationTokenResponseData;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
 
-import java.nio.ByteBuffer;
-
-import static org.apache.kafka.common.protocol.types.Type.BYTES;
-import static org.apache.kafka.common.protocol.types.Type.INT64;
-
 public class ExpireDelegationTokenRequest extends AbstractRequest {
 
-    private static final String HMAC_KEY_NAME = "hmac";
-    private static final String EXPIRY_TIME_PERIOD_KEY_NAME = "expiry_time_period";
-    private final ByteBuffer hmac;
-    private final long expiryTimePeriod;
-
-    private static final Schema TOKEN_EXPIRE_REQUEST_V0 = new Schema(
-        new Field(HMAC_KEY_NAME, BYTES, "HMAC of the delegation token to be expired."),
-        new Field(EXPIRY_TIME_PERIOD_KEY_NAME, INT64, "expiry time period in milli seconds."));
-
-    /**
-     * The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
-     */
-    private static final Schema TOKEN_EXPIRE_REQUEST_V1 = TOKEN_EXPIRE_REQUEST_V0;
+    private final ExpireDelegationTokenRequestData data;
 
-    private ExpireDelegationTokenRequest(short version, ByteBuffer hmac, long renewTimePeriod) {
+    private ExpireDelegationTokenRequest(ExpireDelegationTokenRequestData data, short version) {
         super(ApiKeys.EXPIRE_DELEGATION_TOKEN, version);
-
-        this.hmac = hmac;
-        this.expiryTimePeriod = renewTimePeriod;
+        this.data = data;
     }
 
-    public ExpireDelegationTokenRequest(Struct struct, short versionId) {
-        super(ApiKeys.EXPIRE_DELEGATION_TOKEN, versionId);
-
-        hmac = struct.getBytes(HMAC_KEY_NAME);
-        expiryTimePeriod = struct.getLong(EXPIRY_TIME_PERIOD_KEY_NAME);
+    public ExpireDelegationTokenRequest(Struct struct, short version) {
+        super(ApiKeys.EXPIRE_DELEGATION_TOKEN, version);
+        this.data = new ExpireDelegationTokenRequestData(struct, version);
     }
 
     public static ExpireDelegationTokenRequest parse(ByteBuffer buffer, short version) {
         return new ExpireDelegationTokenRequest(ApiKeys.EXPIRE_DELEGATION_TOKEN.parseRequest(version, buffer), version);
     }
 
-    public static Schema[] schemaVersions() {
-        return new Schema[] {TOKEN_EXPIRE_REQUEST_V0, TOKEN_EXPIRE_REQUEST_V1};
-    }
-
     @Override
     protected Struct toStruct() {
-        short version = version();
-        Struct struct = new Struct(ApiKeys.EXPIRE_DELEGATION_TOKEN.requestSchema(version));
-
-        struct.set(HMAC_KEY_NAME, hmac);
-        struct.set(EXPIRY_TIME_PERIOD_KEY_NAME, expiryTimePeriod);
-
-        return struct;
+        return data.toStruct(version());
     }
 
     @Override
     public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
-        return new ExpireDelegationTokenResponse(throttleTimeMs, Errors.forException(e));
+        return new ExpireDelegationTokenResponse(
+                new ExpireDelegationTokenResponseData()
+                    .setErrorCode(Errors.forException(e).code())
+                    .setThrottleTimeMs(throttleTimeMs));
     }
 
     public ByteBuffer hmac() {
-        return hmac;
+        return ByteBuffer.wrap(data.hmac());
     }
 
     public long expiryTimePeriod() {
-        return expiryTimePeriod;
+        return data.expiryTimePeriodMs();
     }
 
     public static class Builder extends AbstractRequest.Builder {
-        private final ByteBuffer hmac;
-        private final long expiryTimePeriod;
+        private final ExpireDelegationTokenRequestData data;
 
-        public Builder(byte[] hmac, long expiryTimePeriod) {
+        public Builder(ExpireDelegationTokenRequestData data) {
             super(ApiKeys.EXPIRE_DELEGATION_TOKEN);
-            this.hmac = ByteBuffer.wrap(hmac);
-            this.expiryTimePeriod = expiryTimePeriod;
+            this.data = data;
         }
 
         @Override
         public ExpireDelegationTokenRequest build(short version) {
-            return new ExpireDelegationTokenRequest(version, hmac, expiryTimePeriod);
+            return new ExpireDelegationTokenRequest(data, version);
         }
 
         @Override
         public String toString() {
-            StringBuilder bld = new StringBuilder();
-            bld.append("(type: ExpireDelegationTokenRequest").
-                append(", hmac=").append(hmac).
-                append(", expiryTimePeriod=").append(expiryTimePeriod).
-                append(")");
-            return bld.toString();
+            return data.toString();
         }
     }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java
index 9491a3546a1b9..16a6e8c5b40f5 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java
@@ -16,92 +16,56 @@
  */
 package org.apache.kafka.common.requests;
 
-import org.apache.kafka.common.protocol.ApiKeys;
-import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
-import org.apache.kafka.common.protocol.types.Struct;
-
 import java.nio.ByteBuffer;
+import java.util.Collections;
 import java.util.Map;
 
-import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
-import static org.apache.kafka.common.protocol.CommonFields.THROTTLE_TIME_MS;
-import static org.apache.kafka.common.protocol.types.Type.INT64;
+import org.apache.kafka.common.message.ExpireDelegationTokenResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.protocol.types.Struct;
 
 public class ExpireDelegationTokenResponse extends AbstractResponse {
 
-    private static final String EXPIRY_TIMESTAMP_KEY_NAME = "expiry_timestamp";
-
-    private final Errors error;
-    private final long expiryTimestamp;
-    private final int throttleTimeMs;
+    private final ExpireDelegationTokenResponseData data;
 
-    private  static final Schema TOKEN_EXPIRE_RESPONSE_V0 = new Schema(
-        ERROR_CODE,
-        new Field(EXPIRY_TIMESTAMP_KEY_NAME, INT64, "timestamp (in msec) at which this token expires.."),
-        THROTTLE_TIME_MS);
-
-    /**
-     * The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
-     */
-    private static final Schema TOKEN_EXPIRE_RESPONSE_V1 = TOKEN_EXPIRE_RESPONSE_V0;
-
-    public ExpireDelegationTokenResponse(int throttleTimeMs, Errors error, long expiryTimestamp) {
-        this.throttleTimeMs = throttleTimeMs;
-        this.error = error;
-        this.expiryTimestamp = expiryTimestamp;
-    }
-
-    public ExpireDelegationTokenResponse(int throttleTimeMs, Errors error) {
-        this(throttleTimeMs, error, -1);
+    public ExpireDelegationTokenResponse(ExpireDelegationTokenResponseData data) {
+        this.data = data;
     }
 
-    public ExpireDelegationTokenResponse(Struct struct) {
-        error = Errors.forCode(struct.get(ERROR_CODE));
-        this.expiryTimestamp = struct.getLong(EXPIRY_TIMESTAMP_KEY_NAME);
-        this.throttleTimeMs = struct.getOrElse(THROTTLE_TIME_MS, DEFAULT_THROTTLE_TIME);
+    public ExpireDelegationTokenResponse(Struct struct, short version) {
+        this.data = new ExpireDelegationTokenResponseData(struct, version);
     }
 
     public static ExpireDelegationTokenResponse parse(ByteBuffer buffer, short version) {
-        return new ExpireDelegationTokenResponse(ApiKeys.EXPIRE_DELEGATION_TOKEN.responseSchema(version).read(buffer));
-    }
-
-    public static Schema[] schemaVersions() {
-        return new Schema[] {TOKEN_EXPIRE_RESPONSE_V0, TOKEN_EXPIRE_RESPONSE_V1};
+        return new ExpireDelegationTokenResponse(ApiKeys.EXPIRE_DELEGATION_TOKEN.responseSchema(version).read(buffer), version);
     }
 
     public Errors error() {
-        return error;
+        return Errors.forCode(data.errorCode());
     }
 
     public long expiryTimestamp() {
-        return expiryTimestamp;
+        return data.expiryTimestampMs();
     }
 
     @Override
     public Map errorCounts() {
-        return errorCounts(error);
+        return Collections.singletonMap(error(), 1);
     }
 
     @Override
     protected Struct toStruct(short version) {
-        Struct struct = new Struct(ApiKeys.EXPIRE_DELEGATION_TOKEN.responseSchema(version));
-
-        struct.set(ERROR_CODE, error.code());
-        struct.set(EXPIRY_TIMESTAMP_KEY_NAME, expiryTimestamp);
-        struct.setIfExists(THROTTLE_TIME_MS, throttleTimeMs);
-
-        return struct;
+        return data.toStruct(version);
     }
 
     @Override
     public int throttleTimeMs() {
-        return throttleTimeMs;
+        return data.throttleTimeMs();
     }
 
     public boolean hasError() {
-        return this.error != Errors.NONE;
+        return error() != Errors.NONE;
     }
 
     @Override
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index 4218eff46e26a..0b8d98d73a10f 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -58,6 +58,8 @@
 import org.apache.kafka.common.message.DescribeGroupsResponseData.DescribedGroup;
 import org.apache.kafka.common.message.ElectLeadersResponseData.PartitionResult;
 import org.apache.kafka.common.message.ElectLeadersResponseData.ReplicaElectionResult;
+import org.apache.kafka.common.message.ExpireDelegationTokenRequestData;
+import org.apache.kafka.common.message.ExpireDelegationTokenResponseData;
 import org.apache.kafka.common.message.FindCoordinatorRequestData;
 import org.apache.kafka.common.message.HeartbeatRequestData;
 import org.apache.kafka.common.message.HeartbeatResponseData;
@@ -1551,11 +1553,18 @@ private RenewDelegationTokenResponse createRenewTokenResponse() {
     }
 
     private ExpireDelegationTokenRequest createExpireTokenRequest() {
-        return new ExpireDelegationTokenRequest.Builder("test".getBytes(), System.currentTimeMillis()).build();
+        ExpireDelegationTokenRequestData data = new ExpireDelegationTokenRequestData()
+                .setHmac("test".getBytes())
+                .setExpiryTimePeriodMs(System.currentTimeMillis());
+        return new ExpireDelegationTokenRequest.Builder(data).build();
     }
 
     private ExpireDelegationTokenResponse createExpireTokenResponse() {
-        return new ExpireDelegationTokenResponse(20, Errors.NONE, System.currentTimeMillis());
+        ExpireDelegationTokenResponseData data = new ExpireDelegationTokenResponseData()
+                .setThrottleTimeMs(20)
+                .setErrorCode(Errors.NONE.code())
+                .setExpiryTimestampMs(System.currentTimeMillis());
+        return new ExpireDelegationTokenResponse(data);
     }
 
     private DescribeDelegationTokenRequest createDescribeTokenRequest() {
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index a88cd92f8267d..2b4998240683b 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -59,6 +59,7 @@ import org.apache.kafka.common.message.DeleteTopicsResponseData.{DeletableTopicR
 import org.apache.kafka.common.message.DescribeGroupsResponseData
 import org.apache.kafka.common.message.ElectLeadersResponseData.PartitionResult
 import org.apache.kafka.common.message.ElectLeadersResponseData.ReplicaElectionResult
+import org.apache.kafka.common.message.ExpireDelegationTokenResponseData
 import org.apache.kafka.common.message.FindCoordinatorResponseData
 import org.apache.kafka.common.message.HeartbeatResponseData
 import org.apache.kafka.common.message.InitProducerIdResponseData
@@ -2484,7 +2485,11 @@ class KafkaApis(val requestChannel: RequestChannel,
       trace("Sending expire token response for correlation id %d to client %s."
         .format(request.header.correlationId, request.header.clientId))
       sendResponseMaybeThrottle(request, requestThrottleMs =>
-        new ExpireDelegationTokenResponse(requestThrottleMs, error, expiryTimestamp))
+        new ExpireDelegationTokenResponse(
+            new ExpireDelegationTokenResponseData()
+              .setThrottleTimeMs(requestThrottleMs)
+              .setErrorCode(error.code)
+              .setExpiryTimestampMs(expiryTimestamp)))
     }
 
     if (!allowTokenRequests(request))
diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
index a8d29fd02f242..db7ab699712a7 100644
--- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
+++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
@@ -33,6 +33,7 @@ import org.apache.kafka.common.message.CreateTopicsRequestData.{CreatableTopic,
 import org.apache.kafka.common.message.DeleteGroupsRequestData
 import org.apache.kafka.common.message.DeleteTopicsRequestData
 import org.apache.kafka.common.message.DescribeGroupsRequestData
+import org.apache.kafka.common.message.ExpireDelegationTokenRequestData
 import org.apache.kafka.common.message.FindCoordinatorRequestData
 import org.apache.kafka.common.message.HeartbeatRequestData
 import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData
@@ -444,7 +445,10 @@ class RequestQuotaTest extends BaseRequestTest {
           )
 
         case ApiKeys.EXPIRE_DELEGATION_TOKEN =>
-          new ExpireDelegationTokenRequest.Builder("".getBytes, 1000)
+          new ExpireDelegationTokenRequest.Builder(
+              new ExpireDelegationTokenRequestData()
+                .setHmac("".getBytes)
+                .setExpiryTimePeriodMs(1000L))
 
         case ApiKeys.DESCRIBE_DELEGATION_TOKEN =>
           new DescribeDelegationTokenRequest.Builder(Collections.singletonList(SecurityUtils.parseKafkaPrincipal("User:test")))
@@ -573,7 +577,7 @@ class RequestQuotaTest extends BaseRequestTest {
       case ApiKeys.CREATE_PARTITIONS => new CreatePartitionsResponse(response).throttleTimeMs
       case ApiKeys.CREATE_DELEGATION_TOKEN => new CreateDelegationTokenResponse(response, ApiKeys.CREATE_DELEGATION_TOKEN.latestVersion).throttleTimeMs
       case ApiKeys.DESCRIBE_DELEGATION_TOKEN=> new DescribeDelegationTokenResponse(response).throttleTimeMs
-      case ApiKeys.EXPIRE_DELEGATION_TOKEN => new ExpireDelegationTokenResponse(response).throttleTimeMs
+      case ApiKeys.EXPIRE_DELEGATION_TOKEN => new ExpireDelegationTokenResponse(response, ApiKeys.EXPIRE_DELEGATION_TOKEN.latestVersion).throttleTimeMs
       case ApiKeys.RENEW_DELEGATION_TOKEN => new RenewDelegationTokenResponse(response).throttleTimeMs
       case ApiKeys.DELETE_GROUPS => new DeleteGroupsResponse(response).throttleTimeMs
       case ApiKeys.OFFSET_FOR_LEADER_EPOCH => new OffsetsForLeaderEpochResponse(response).throttleTimeMs

From 66d81a0e50680916f541703049dd4e911542b5ed Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Thu, 8 Aug 2019 06:01:22 -0700
Subject: [PATCH 0512/1071] MINOR: Update dependencies for Kafka 2.4 (#7126)

Scala 2.12.9 brings another 5% ~ 10% improvement in compiler performance,
improved compatibility with JDK 11/12/13, and experimental infrastructure for
build pipelining.

zstd update includes performance improvements, among which the
primary improvement is that decompression is ~7% faster.

Level | v1.4.0 | v1.4.1 | Delta
-- | -- | -- | --
1 | 1390 MB/s | 1453 MB/s | +4.5%
3 | 1208 MB/s | 1301 MB/s | +7.6%
5 | 1129 MB/s | 1233 MB/s | +9.2%
7 | 1224 MB/s | 1347 MB/s | +10.0%
16 | 1278 MB/s | 1430 MB/s | +11.8%

Jetty 9.4.19 includes a number of bug fixes:
https://github.com/eclipse/jetty.project/releases/tag/jetty-9.4.19.v20190610

Mockito 3.0.0 switched the Java requirement from 7 to 8.

Several updates to owaspDepCheckPlugin (4.0.2 -> 5.2.1).

The rest are patch updates.

Reviewers: Rajini Sivaram 
---
 bin/kafka-run-class.sh          |  2 +-
 bin/windows/kafka-run-class.bat |  2 +-
 gradle.properties               |  2 +-
 gradle/dependencies.gradle      | 20 ++++++++++----------
 4 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/bin/kafka-run-class.sh b/bin/kafka-run-class.sh
index 7098925b940c4..dc0235e1cbece 100755
--- a/bin/kafka-run-class.sh
+++ b/bin/kafka-run-class.sh
@@ -48,7 +48,7 @@ should_include_file() {
 base_dir=$(dirname $0)/..
 
 if [ -z "$SCALA_VERSION" ]; then
-  SCALA_VERSION=2.12.8
+  SCALA_VERSION=2.12.9
 fi
 
 if [ -z "$SCALA_BINARY_VERSION" ]; then
diff --git a/bin/windows/kafka-run-class.bat b/bin/windows/kafka-run-class.bat
index 3bec536e67a73..cc9f3d6a26021 100755
--- a/bin/windows/kafka-run-class.bat
+++ b/bin/windows/kafka-run-class.bat
@@ -27,7 +27,7 @@ set BASE_DIR=%CD%
 popd
 
 IF ["%SCALA_VERSION%"] EQU [""] (
-  set SCALA_VERSION=2.12.8
+  set SCALA_VERSION=2.12.9
 )
 
 IF ["%SCALA_BINARY_VERSION%"] EQU [""] (
diff --git a/gradle.properties b/gradle.properties
index c4b58b94339cb..9c4949629c2fe 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -21,6 +21,6 @@ group=org.apache.kafka
 #  - tests/kafkatest/version.py (variable DEV_VERSION)
 #  - kafka-merge-pr.py
 version=2.4.0-SNAPSHOT
-scalaVersion=2.12.8
+scalaVersion=2.12.9
 task=build
 org.gradle.jvmargs=-Xmx1024m -Xss2m
diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle
index d4368338995f6..f3d0e31d37dbb 100644
--- a/gradle/dependencies.gradle
+++ b/gradle/dependencies.gradle
@@ -30,7 +30,7 @@ ext {
 
 // Add Scala version
 def defaultScala211Version = '2.11.12'
-def defaultScala212Version = '2.12.8'
+def defaultScala212Version = '2.12.9'
 def defaultScala213Version = '2.13.0'
 if (hasProperty('scalaVersion')) {
   if (scalaVersion == '2.11') {
@@ -63,18 +63,18 @@ versions += [
   apacheda: "1.0.2",
   apacheds: "2.0.0-M24",
   argparse4j: "0.7.0",
-  bcpkix: "1.61",
+  bcpkix: "1.62",
   checkstyle: "8.20",
   commonsCli: "1.4",
   gradle: "5.4.1",
   gradleVersionsPlugin: "0.21.0",
   grgit: "3.1.1",
-  httpclient: "4.5.8",
+  httpclient: "4.5.9",
   easymock: "4.0.2",
   jackson: "2.9.9",
   jacksonDatabind: "2.9.9.3",
   jacoco: "0.8.3",
-  jetty: "9.4.18.v20190429",
+  jetty: "9.4.19.v20190610",
   jersey: "2.28",
   jmh: "1.21",
   hamcrest: "2.1",
@@ -97,25 +97,25 @@ versions += [
   lz4: "1.6.0",
   mavenArtifact: "3.6.1",
   metrics: "2.2.0",
-  mockito: "2.27.0",
-  owaspDepCheckPlugin: "4.0.2",
+  mockito: "3.0.0",
+  owaspDepCheckPlugin: "5.2.1",
   powermock: "2.0.2",
   reflections: "0.9.11",
   rocksDB: "5.18.3",
-  scalaCollectionCompat: "2.1.0",
+  scalaCollectionCompat: "2.1.2",
   scalafmt: "1.5.1",
   scalaJava8Compat : "0.9.0",
   scalatest: "3.0.8",
   scoverage: "1.4.0",
   scoveragePlugin: "2.5.0",
   shadowPlugin: "4.0.4",
-  slf4j: "1.7.26",
+  slf4j: "1.7.27",
   snappy: "1.1.7.3",
   spotbugs: "3.1.12",
   spotbugsPlugin: "1.6.9",
-  spotlessPlugin: "3.23.0",
+  spotlessPlugin: "3.23.1",
   zookeeper: "3.5.5",
-  zstd: "1.4.0-1"
+  zstd: "1.4.2-1"
 ]
 
 libs += [

From e867a58425876767b952e06892c72b5e13066acc Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Thu, 8 Aug 2019 14:31:22 -0700
Subject: [PATCH 0513/1071] KAFKA-8179: Part 3, Add PartitionsLost API for
 resetGenerations and metadata/subscription change (#6884)

1. Add onPartitionsLost into the RebalanceListener, which will be triggered when the consumer found that the generation is reset due to fatal errors in response handling.

2. Semantical behavior change: with COOPERATIVE protocol, if the revoked / lost partitions are empty, do not trigger the corresponding callback at all. For added partitions though, even if it is empty we would still trigger the callback as a way to notify the rebalance event; with EAGER protocol, revoked / assigned callbacks are always triggered.

The ordering of the callback would be the following:

a. Callback onPartitionsRevoked / onPartitionsLost triggered.
b. Update the assignment (both revoked and added).
c. Callback onPartitionsAssigned triggered.

In this way we are assured that users can still access the partitions being revoked, whereas they can also access the partitions being added.

3. Semantical behavior change (KAFKA-4600): if the rebalance listener throws an exception, pass it along all the way to the consumer.poll caller, but still completes the rest of the actions. Also, the newly assigned partitions list does not gets affected with exception thrown since it is just for notifying the users.

4. Semantical behavior change: the ConsumerCoordinator would not try to modify assignor's returned assignments, instead it will validate that assignments and set the error code accordingly: if there are overlaps between added / revoked partitions, it is a fatal error and would be communicated to all members to stop; if revoked is not empty, it is an error indicate re-join; otherwise, it is normal.

5. Minor: with the error code removed from the Assignment, ConsumerCoordinator will request re-join if the revoked partitions list is not empty.

6. Updated ConsumerCoordinatorTest accordingly. Also found a minor bug in MetadataUpdate that removed topic would still be retained with null value of num.partitions.

6. Updated a few other flaky tests that are exposed due to this change.

Reviewers: John Roesler , A. Sophie Blee-Goldman , Jason Gustafson 
---
 .../consumer/ConsumerPartitionAssignor.java   |  17 +
 .../consumer/ConsumerRebalanceListener.java   |  98 +++++-
 .../kafka/clients/consumer/KafkaConsumer.java |  19 +-
 .../internals/AbstractCoordinator.java        |  62 ++--
 .../internals/ConsumerCoordinator.java        | 332 +++++++++++++-----
 .../consumer/internals/PartitionAssignor.java |   1 -
 .../consumer/internals/SubscriptionState.java |  55 +--
 .../clients/consumer/KafkaConsumerTest.java   |  64 +++-
 .../consumer/RoundRobinAssignorTest.java      |  20 +-
 .../internals/ConsumerCoordinatorTest.java    | 184 ++++++----
 .../internals/SubscriptionStateTest.java      |  40 ++-
 .../main/scala/kafka/tools/MirrorMaker.scala  |   2 +
 .../kafka/api/AbstractConsumerTest.scala      |  14 +-
 .../kafka/api/ConsumerBounceTest.scala        |  13 +-
 .../kafka/api/PlaintextConsumerTest.scala     |   8 +-
 .../apache/kafka/streams/KafkaStreams.java    |   5 +-
 .../processor/internals/StreamThread.java     |   5 +-
 .../StreamTableJoinIntegrationTest.java       |   5 +-
 .../utils/IntegrationTestUtils.java           |  17 +-
 19 files changed, 686 insertions(+), 275 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java
index 07e153e33f35e..f9a42171a0447 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java
@@ -180,6 +180,23 @@ public Map groupAssignment() {
         }
     }
 
+    /**
+     * The rebalance protocol defines partition assignment and revocation semantics. The purpose is to establish a
+     * consistent set of rules that all consumers in a group follow in order to transfer ownership of a partition.
+     * {@link ConsumerPartitionAssignor} implementors can claim supporting one or more rebalance protocols via the
+     * {@link ConsumerPartitionAssignor#supportedProtocols()}, and it is their responsibility to respect the rules
+     * of those protocols in their {@link ConsumerPartitionAssignor#assign(Cluster, GroupSubscription)} implementations.
+     * Failures to follow the rules of the supported protocols would lead to runtime error or undefined behavior.
+     *
+     * The {@link RebalanceProtocol#EAGER} rebalance protocol requires a consumer to always revoke all its owned
+     * partitions before participating in a rebalance event. It therefore allows a complete reshuffling of the assignment.
+     *
+     * {@link RebalanceProtocol#COOPERATIVE} rebalance protocol allows a consumer to retain its currently owned
+     * partitions before participating in a rebalance event. The assignor should not reassign any owned partitions
+     * immediately, but instead may indicate consumers the need for partition revocation so that the revoked
+     * partitions can be reassigned to other consumers in the next rebalance event. This is designed for sticky assignment
+     * logic which attempts to minimize partition reassignment with cooperative adjustments.
+     */
     enum RebalanceProtocol {
         EAGER((byte) 0), COOPERATIVE((byte) 1);
 
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
index 74e8b060c73fa..6046ef954682d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.clients.consumer;
 
+import java.time.Duration;
 import java.util.Collection;
 
 import org.apache.kafka.common.TopicPartition;
@@ -29,7 +30,7 @@
  * 
        
  * When Kafka is managing the group membership, a partition re-assignment will be triggered any time the members of the group change or the subscription
  * of the members changes. This can occur when processes die, new process instances are added or old instances come back to life after failure.
- * Rebalances can also be triggered by changes affecting the subscribed topics (e.g. when the number of partitions is
+ * Partition re-assignments can also be triggered by changes affecting the subscribed topics (e.g. when the number of partitions is
  * administratively adjusted).
  * 
        
  * There are many uses for this functionality. One common use is saving offsets in a custom store. By saving offsets in
@@ -47,11 +48,43 @@
  * This callback will only execute in the user thread as part of the {@link Consumer#poll(java.time.Duration) poll(long)} call
  * whenever partition assignment changes.
  * 
        
- * It is guaranteed that all consumer processes will invoke {@link #onPartitionsRevoked(Collection) onPartitionsRevoked} prior to
- * any process invoking {@link #onPartitionsAssigned(Collection) onPartitionsAssigned}. So if offsets or other state is saved in the
- * {@link #onPartitionsRevoked(Collection) onPartitionsRevoked} call it is guaranteed to be saved by the time the process taking over that
- * partition has their {@link #onPartitionsAssigned(Collection) onPartitionsAssigned} callback called to load the state.
+ * Under normal conditions, if a partition is reassigned from one consumer to another, then the old consumer will
+ * always invoke {@link #onPartitionsRevoked(Collection) onPartitionsRevoked} for that partition prior to the new consumer
+ * invoking {@link #onPartitionsAssigned(Collection) onPartitionsAssigned} for the same partition. So if offsets or other state is saved in the
+ * {@link #onPartitionsRevoked(Collection) onPartitionsRevoked} call by one consumer member, it will be always accessible by the time the
+ * other consumer member taking over that partition and triggering its {@link #onPartitionsAssigned(Collection) onPartitionsAssigned} callback to load the state.
  * 
        
+ * You can think of revocation as a graceful way to give up ownership of a partition. In some cases, the consumer may not have an opportunity to do so.
+ * For example, if the session times out, then the partitions may be reassigned before we have a chance to revoke them gracefully.
+ * For this case, we have a third callback {@link #onPartitionsLost(Collection)}. The difference between this function and
+ * {@link #onPartitionsRevoked(Collection)} is that upon invocation of {@link #onPartitionsLost(Collection)}, the partitions
+ * may already be owned by some other members in the group and therefore users would not be able to commit its consumed offsets for example.
+ * Users could implement these two functions differently (by default,
+ * {@link #onPartitionsLost(Collection)} will be calling {@link #onPartitionsRevoked(Collection)} directly); for example, in the
+ * {@link #onPartitionsLost(Collection)} we should not need to store the offsets since we know these partitions are no longer owned by the consumer
+ * at that time.
+ * 
        
+ * During a rebalance event, the {@link #onPartitionsAssigned(Collection) onPartitionsAssigned} function will always be triggered exactly once when
+ * the rebalance completes. That is, even if there is no newly assigned partitions for a consumer member, its {@link #onPartitionsAssigned(Collection) onPartitionsAssigned}
+ * will still be triggered with an empty collection of partitions. As a result this function can be used also to notify when a rebalance event has happened.
+ * On the other hand, {@link #onPartitionsRevoked(Collection)} and {@link #onPartitionsLost(Collection)}
+ * will only be triggered when there are non-empty partitions revoked or lost from this consumer member during a rebalance event.
+ * 
        
+ * It is possible
+ * for a {@link org.apache.kafka.common.errors.WakeupException} or {@link org.apache.kafka.common.errors.InterruptException}
+ * to be raised from one of these nested invocations. In this case, the exception will be propagated to the current
+ * invocation of {@link KafkaConsumer#poll(java.time.Duration)} in which this callback is being executed. This means it is not
+ * necessary to catch these exceptions and re-attempt to wakeup or interrupt the consumer thread.
+ * Also if the callback function implementation itself throws an exception, this exception will be propagated to the current
+ * invocation of {@link KafkaConsumer#poll(java.time.Duration)} as well.
+ * 
        
+ * Note that callbacks only serve as notification of an assignment change.
+ * They cannot be used to express acceptance of the change.
+ * Hence throwing an exception from a callback does not affect the assignment in any way,
+ * as it will be propagated all the way up to the {@link KafkaConsumer#poll(java.time.Duration)} call.
+ * If user captures the exception in the caller, the callback is still assumed successful and no further retries will be attempted.
+ * 
        
+ *
  * Here is pseudo-code for a callback implementation for saving offsets:
  * 
          * {@code
@@ -68,6 +101,10 @@
  *              saveOffsetInExternalStore(consumer.position(partition));
  *       }
  *
+ *       public void onPartitionsLost(Collection partitions) {
+ *           // do not need to save the offsets since these partitions are probably owned by other consumers already
+ *       }
+ *
  *       public void onPartitionsAssigned(Collection partitions) {
  *           // read the offsets from an external store using some custom code not described here
  *           for(TopicPartition partition: partitions)
@@ -82,7 +119,9 @@ public interface ConsumerRebalanceListener {
     /**
      * A callback method the user can implement to provide handling of offset commits to a customized store on the start
      * of a rebalance operation. This method will be called before a rebalance operation starts and after the consumer
-     * stops fetching data. It is recommended that offsets should be committed in this callback to either Kafka or a
+     * stops fetching data. It can also be called when consumer is being closed ({@link KafkaConsumer#close(Duration)})
+     * or is unsubscribing ({@link KafkaConsumer#unsubscribe()}).
+     * It is recommended that offsets should be committed in this callback to either Kafka or a
      * custom offset store to prevent duplicate data.
      * 
        
      * For examples on usage of this API, see Usage Examples section of {@link KafkaConsumer KafkaConsumer}
@@ -91,11 +130,12 @@ public interface ConsumerRebalanceListener {
      * 
        
      * It is common for the revocation callback to use the consumer instance in order to commit offsets. It is possible
      * for a {@link org.apache.kafka.common.errors.WakeupException} or {@link org.apache.kafka.common.errors.InterruptException}
-     * to be raised from one these nested invocations. In this case, the exception will be propagated to the current
+     * to be raised from one of these nested invocations. In this case, the exception will be propagated to the current
      * invocation of {@link KafkaConsumer#poll(java.time.Duration)} in which this callback is being executed. This means it is not
      * necessary to catch these exceptions and re-attempt to wakeup or interrupt the consumer thread.
      *
-     * @param partitions The list of partitions that were assigned to the consumer on the last rebalance
+     * @param partitions The list of partitions that were assigned to the consumer and now need to be revoked (may not
+     *                   include all currently assigned partitions, i.e. there may still be some partitions left)
      * @throws org.apache.kafka.common.errors.WakeupException If raised from a nested call to {@link KafkaConsumer}
      * @throws org.apache.kafka.common.errors.InterruptException If raised from a nested call to {@link KafkaConsumer}
      */
@@ -106,20 +146,52 @@ public interface ConsumerRebalanceListener {
      * partition re-assignment. This method will be called after the partition re-assignment completes and before the
      * consumer starts fetching data, and only as the result of a {@link Consumer#poll(java.time.Duration) poll(long)} call.
      * 
        
-     * It is guaranteed that all the processes in a consumer group will execute their
+     * It is guaranteed that under normal conditions all the processes in a consumer group will execute their
      * {@link #onPartitionsRevoked(Collection)} callback before any instance executes its
-     * {@link #onPartitionsAssigned(Collection)} callback.
+     * {@link #onPartitionsAssigned(Collection)} callback. During exceptional scenarios, partitions may be migrated
+     * without the old owner being notified (i.e. their {@link #onPartitionsRevoked(Collection)} callback not triggered),
+     * and later when the old owner consumer realized this event, the {@link #onPartitionsLost(Collection)} (Collection)} callback
+     * will be triggered by the consumer then.
      * 
        
      * It is common for the assignment callback to use the consumer instance in order to query offsets. It is possible
      * for a {@link org.apache.kafka.common.errors.WakeupException} or {@link org.apache.kafka.common.errors.InterruptException}
-     * to be raised from one these nested invocations. In this case, the exception will be propagated to the current
+     * to be raised from one of these nested invocations. In this case, the exception will be propagated to the current
      * invocation of {@link KafkaConsumer#poll(java.time.Duration)} in which this callback is being executed. This means it is not
      * necessary to catch these exceptions and re-attempt to wakeup or interrupt the consumer thread.
      *
-     * @param partitions The list of partitions that are now assigned to the consumer (may include partitions previously
-     *            assigned to the consumer)
+     * @param partitions The list of partitions that are now assigned to the consumer (previously owned partitions will
+     *                   NOT be included, i.e. this list will only include newly added partitions)
      * @throws org.apache.kafka.common.errors.WakeupException If raised from a nested call to {@link KafkaConsumer}
      * @throws org.apache.kafka.common.errors.InterruptException If raised from a nested call to {@link KafkaConsumer}
      */
     void onPartitionsAssigned(Collection partitions);
+
+    /**
+     * A callback method you can implement to provide handling of cleaning up resources for partitions that have already
+     * been reassigned to other consumers. This method will not be called during normal execution as the owned partitions would
+     * first be revoked by calling the {@link ConsumerRebalanceListener#onPartitionsRevoked}, before being reassigned
+     * to other consumers during a rebalance event. However, during exceptional scenarios when the consumer realized that it
+     * does not own this partition any longer, i.e. not revoked via a normal rebalance event, then this method would be invoked.
+     * 
        
+     * For example, this function is called if a consumer's session timeout has expired, or if a fatal error has been
+     * received indicating the consumer is no longer part of the group.
+     * 
        
+     * By default it will just trigger {@link ConsumerRebalanceListener#onPartitionsRevoked}; for users who want to distinguish
+     * the handling logic of revoked partitions v.s. lost partitions, they can override the default implementation.
+     * 
        
+     * It is possible
+     * for a {@link org.apache.kafka.common.errors.WakeupException} or {@link org.apache.kafka.common.errors.InterruptException}
+     * to be raised from one of these nested invocations. In this case, the exception will be propagated to the current
+     * invocation of {@link KafkaConsumer#poll(java.time.Duration)} in which this callback is being executed. This means it is not
+     * necessary to catch these exceptions and re-attempt to wakeup or interrupt the consumer thread.
+     *
+     * @param partitions The list of partitions that were assigned to the consumer and now have been reassigned
+     *                   to other consumers (may not include all currently assigned partitions, i.e. there may still
+     *                   be some partitions left)
+     * @throws org.apache.kafka.common.errors.WakeupException If raised from a nested call to {@link KafkaConsumer}
+     * @throws org.apache.kafka.common.errors.InterruptException If raised from a nested call to {@link KafkaConsumer}
+     */
+    default void onPartitionsLost(Collection partitions) {
+        onPartitionsRevoked(partitions);
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index 0d0efe1b663fc..be3e176c2eeb9 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -1046,14 +1046,18 @@ public void subscribe(Pattern pattern) {
     /**
      * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)} or {@link #subscribe(Pattern)}.
      * This also clears any partitions directly assigned through {@link #assign(Collection)}.
+     *
+     * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. rebalance callback errors)
      */
     public void unsubscribe() {
         acquireAndEnsureOpen();
         try {
             fetcher.clearBufferedDataForUnassignedPartitions(Collections.emptySet());
             this.subscriptions.unsubscribe();
-            if (this.coordinator != null)
+            if (this.coordinator != null) {
+                this.coordinator.onLeavePrepare();
                 this.coordinator.maybeLeaveGroup("the consumer unsubscribed from all topics");
+            }
             log.info("Unsubscribed all topics or patterns and assigned partitions");
         } finally {
             release();
@@ -1176,7 +1180,8 @@ public ConsumerRecords poll(final long timeoutMs) {
      * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed
      *             topics or to the configured groupId. See the exception for more details
      * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or
-     *             session timeout, errors deserializing key/value pairs, or any new error cases in future versions)
+     *             session timeout, errors deserializing key/value pairs, your rebalance callback thrown exceptions,
+     *             or any new error cases in future versions)
      * @throws java.lang.IllegalArgumentException if the timeout value is negative
      * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any
      *             partitions to consume from
@@ -1190,6 +1195,9 @@ public ConsumerRecords poll(final Duration timeout) {
         return poll(time.timer(timeout), true);
     }
 
+    /**
+     * @throws KafkaException if the rebalance callback throws exception
+     */
     private ConsumerRecords poll(final Timer timer, final boolean includeMetadataInTimeout) {
         acquireAndEnsureOpen();
         try {
@@ -1244,6 +1252,9 @@ boolean updateAssignmentMetadataIfNeeded(final Timer timer) {
         return updateFetchPositions(timer);
     }
 
+    /**
+     * @throws KafkaException if the rebalance callback throws exception
+     */
     private Map>> pollForFetches(Timer timer) {
         long pollTimeout = coordinator == null ? timer.remainingMs() :
                 Math.min(coordinator.timeToNextPoll(timer.currentTimeMs()), timer.remainingMs());
@@ -2162,10 +2173,12 @@ public void close(Duration timeout) {
         acquire();
         try {
             if (!closed) {
-                closed = true;
+                // need to close before setting the flag since the close function
+                // itself may trigger rebalance callback that needs the consumer to be open still
                 close(timeout.toMillis(), false);
             }
         } finally {
+            closed = true;
             release();
         }
     }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index fc385f67e1022..4c71a89f3bdcd 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -46,6 +46,7 @@
 import org.apache.kafka.common.metrics.stats.Max;
 import org.apache.kafka.common.metrics.stats.Meter;
 import org.apache.kafka.common.metrics.stats.WindowedCount;
+import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.requests.FindCoordinatorRequest;
 import org.apache.kafka.common.requests.FindCoordinatorRequest.CoordinatorType;
@@ -201,6 +202,13 @@ protected abstract void onJoinComplete(int generation,
                                            String protocol,
                                            ByteBuffer memberAssignment);
 
+    /**
+     * Invoked prior to each leave group event. This is typically used to cleanup assigned partitions;
+     * note it is triggered by the consumer's API caller thread (i.e. background heartbeat thread would
+     * not trigger it even if it tries to force leaving group upon heartbeat session expiration)
+     */
+    protected void onLeavePrepare() {}
+
     /**
      * Visible for testing.
      *
@@ -314,6 +322,7 @@ public void ensureActiveGroup() {
      * Ensure the group is active (i.e., joined and synced)
      *
      * @param timer Timer bounding how long this method can block
+     * @throws KafkaException if the callback throws exception
      * @return true iff the group is active
      */
     boolean ensureActiveGroup(final Timer timer) {
@@ -362,6 +371,7 @@ private void closeHeartbeatThread() {
      * Visible for testing.
      *
      * @param timer Timer bounding how long this method can block
+     * @throws KafkaException if the callback throws exception
      * @return true iff the operation succeeded
      */
     boolean joinGroupIfNeeded(final Timer timer) {
@@ -376,8 +386,10 @@ boolean joinGroupIfNeeded(final Timer timer) {
             // refresh which changes the matched subscription set) can occur while another rebalance is
             // still in progress.
             if (needsJoinPrepare) {
-                onJoinPrepare(generation.generationId, generation.memberId);
+                // need to set the flag before calling onJoinPrepare since the user callback may throw
+                // exception, in which case upon retry we should not retry onJoinPrepare either.
                 needsJoinPrepare = false;
+                onJoinPrepare(generation.generationId, generation.memberId);
             }
 
             final RequestFuture future = initiateJoinGroup();
@@ -522,7 +534,7 @@ public void handle(JoinGroupResponse joinResponse, RequestFuture fut
                 future.raise(error);
             } else if (error == Errors.UNKNOWN_MEMBER_ID) {
                 // reset the member id and retry immediately
-                resetGeneration();
+                resetGenerationOnResponseError(ApiKeys.JOIN_GROUP, error);
                 log.debug("Attempt to join group failed due to unknown member id.");
                 future.raise(error);
             } else if (error == Errors.COORDINATOR_NOT_AVAILABLE
@@ -645,7 +657,7 @@ public void handle(SyncGroupResponse syncResponse,
                 } else if (error == Errors.UNKNOWN_MEMBER_ID
                         || error == Errors.ILLEGAL_GENERATION) {
                     log.debug("SyncGroup failed: {}", error.message());
-                    resetGeneration();
+                    resetGenerationOnResponseError(ApiKeys.SYNC_GROUP, error);
                     future.raise(error);
                 } else if (error == Errors.COORDINATOR_NOT_AVAILABLE
                         || error == Errors.NOT_COORDINATOR) {
@@ -795,14 +807,20 @@ final synchronized boolean hasValidMemberId() {
         return generation != null && generation.hasMemberId();
     }
 
-
-    /**
-     * Reset the generation and memberId because we have fallen out of the group.
-     */
-    protected synchronized void resetGeneration() {
+    private synchronized void resetGeneration() {
         this.generation = Generation.NO_GENERATION;
-        this.rejoinNeeded = true;
         this.state = MemberState.UNJOINED;
+        this.rejoinNeeded = true;
+    }
+
+    synchronized void resetGenerationOnResponseError(ApiKeys api, Errors error) {
+        log.debug("Resetting generation after encountering " + error + " from " + api + " response");
+        resetGeneration();
+    }
+
+    synchronized void resetGenerationOnLeaveGroup() {
+        log.debug("Resetting generation due to consumer pro-actively leaving the group");
+        resetGeneration();
     }
 
     protected synchronized void requestRejoin() {
@@ -817,6 +835,9 @@ public final void close() {
         close(time.timer(0));
     }
 
+    /**
+     * @throws KafkaException if the rebalance callback throws exception
+     */
     protected void close(Timer timer) {
         try {
             closeHeartbeatThread();
@@ -825,6 +846,7 @@ protected void close(Timer timer) {
             // needs this lock to complete and terminate after close flag is set.
             synchronized (this) {
                 if (rebalanceConfig.leaveGroupOnClose) {
+                    onLeavePrepare();
                     maybeLeaveGroup("the consumer is being closed");
                 }
 
@@ -841,31 +863,31 @@ protected void close(Timer timer) {
     }
 
     /**
-     * Leave the current group and reset local generation/memberId.
-     * @param leaveReason reason to attempt leaving the group
+     * @throws KafkaException if the rebalance callback throws exception
      */
     public synchronized RequestFuture maybeLeaveGroup(String leaveReason) {
         RequestFuture future = null;
+
         // Starting from 2.3, only dynamic members will send LeaveGroupRequest to the broker,
         // consumer with valid group.instance.id is viewed as static member that never sends LeaveGroup,
         // and the membership expiration is only controlled by session timeout.
         if (isDynamicMember() && !coordinatorUnknown() &&
-                state != MemberState.UNJOINED && generation.hasMemberId()) {
+            state != MemberState.UNJOINED && generation.hasMemberId()) {
             // this is a minimal effort attempt to leave the group. we do not
             // attempt any resending if the request fails or times out.
             log.info("Member {} sending LeaveGroup request to coordinator {} due to {}",
-                     generation.memberId, coordinator, leaveReason);
+                generation.memberId, coordinator, leaveReason);
             LeaveGroupRequest.Builder request = new LeaveGroupRequest.Builder(
                 rebalanceConfig.groupId,
-                Collections.singletonList(new MemberIdentity()
-                                              .setMemberId(generation.memberId))
+                Collections.singletonList(new MemberIdentity().setMemberId(generation.memberId))
             );
-            future = client.send(coordinator, request)
-                    .compose(new LeaveGroupResponseHandler());
+
+            future = client.send(coordinator, request).compose(new LeaveGroupResponseHandler());
             client.pollNoWakeup();
         }
 
-        resetGeneration();
+        resetGenerationOnLeaveGroup();
+
         return future;
     }
 
@@ -926,14 +948,14 @@ public void handle(HeartbeatResponse heartbeatResponse, RequestFuture futu
                 future.raise(error);
             } else if (error == Errors.ILLEGAL_GENERATION) {
                 log.info("Attempt to heartbeat failed since generation {} is not current", generation.generationId);
-                resetGeneration();
+                resetGenerationOnResponseError(ApiKeys.HEARTBEAT, error);
                 future.raise(error);
             } else if (error == Errors.FENCED_INSTANCE_ID) {
                 log.error("Received fatal exception: group.instance.id gets fenced");
                 future.raise(error);
             } else if (error == Errors.UNKNOWN_MEMBER_ID) {
                 log.info("Attempt to heartbeat failed for since member id {} is not valid.", generation.memberId);
-                resetGeneration();
+                resetGenerationOnResponseError(ApiKeys.HEARTBEAT, error);
                 future.raise(error);
             } else if (error == Errors.GROUP_AUTHORIZATION_FAILED) {
                 future.raise(GroupAuthorizationException.forGroupId(rebalanceConfig.groupId));
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index aae16b92dc3c2..a3aaface7822d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -18,6 +18,7 @@
 
 import org.apache.kafka.clients.GroupRebalanceConfig;
 import org.apache.kafka.clients.consumer.CommitFailedException;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor;
 import org.apache.kafka.clients.consumer.ConsumerGroupMetadata;
 import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.GroupSubscription;
@@ -48,6 +49,7 @@
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
 import org.apache.kafka.common.metrics.stats.Max;
+import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.record.RecordBatch;
 import org.apache.kafka.common.requests.OffsetCommitRequest;
@@ -73,6 +75,7 @@
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
 /**
@@ -235,27 +238,80 @@ private ConsumerPartitionAssignor lookupAssignor(String name) {
     }
 
     private void maybeUpdateJoinedSubscription(Set assignedPartitions) {
-        // Check if the assignment contains some topics that were not in the original
-        // subscription, if yes we will obey what leader has decided and add these topics
-        // into the subscriptions as long as they still match the subscribed pattern
-
-        Set addedTopics = new HashSet<>();
-        //this is a copy because its handed to listener below
-        for (TopicPartition tp : assignedPartitions) {
-            if (!joinedSubscription.contains(tp.topic()))
-                addedTopics.add(tp.topic());
+        if (subscriptions.hasPatternSubscription()) {
+            // Check if the assignment contains some topics that were not in the original
+            // subscription, if yes we will obey what leader has decided and add these topics
+            // into the subscriptions as long as they still match the subscribed pattern
+
+            Set addedTopics = new HashSet<>();
+            // this is a copy because its handed to listener below
+            for (TopicPartition tp : assignedPartitions) {
+                if (!joinedSubscription.contains(tp.topic()))
+                    addedTopics.add(tp.topic());
+            }
+
+            if (!addedTopics.isEmpty()) {
+                Set newSubscription = new HashSet<>(subscriptions.subscription());
+                Set newJoinedSubscription = new HashSet<>(joinedSubscription);
+                newSubscription.addAll(addedTopics);
+                newJoinedSubscription.addAll(addedTopics);
+
+                if (this.subscriptions.subscribeFromPattern(newSubscription))
+                    metadata.requestUpdateForNewTopics();
+                this.joinedSubscription = newJoinedSubscription;
+            }
+        }
+    }
+
+    private Exception invokePartitionsAssigned(final Set assignedPartitions) {
+        log.info("Adding newly assigned partitions: {}", Utils.join(assignedPartitions, ", "));
+
+        ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
+        try {
+            listener.onPartitionsAssigned(assignedPartitions);
+        } catch (WakeupException | InterruptException e) {
+            throw e;
+        } catch (Exception e) {
+            log.error("User provided listener {} failed on invocation of onPartitionsAssigned for partitions {}",
+                listener.getClass().getName(), assignedPartitions, e);
+            return e;
+        }
+
+        return null;
+    }
+
+    private Exception invokePartitionsRevoked(final Set revokedPartitions) {
+        log.info("Revoke previously assigned partitions {}", Utils.join(revokedPartitions, ", "));
+
+        ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
+        try {
+            listener.onPartitionsRevoked(revokedPartitions);
+        } catch (WakeupException | InterruptException e) {
+            throw e;
+        } catch (Exception e) {
+            log.error("User provided listener {} failed on invocation of onPartitionsRevoked for partitions {}",
+                listener.getClass().getName(), revokedPartitions, e);
+            return e;
         }
 
-        if (!addedTopics.isEmpty()) {
-            Set newSubscription = new HashSet<>(subscriptions.subscription());
-            Set newJoinedSubscription = new HashSet<>(joinedSubscription);
-            newSubscription.addAll(addedTopics);
-            newJoinedSubscription.addAll(addedTopics);
+        return null;
+    }
+
+    private Exception invokePartitionsLost(final Set lostPartitions) {
+        log.info("Lost previously assigned partitions {}", Utils.join(lostPartitions, ", "));
 
-            if (this.subscriptions.subscribeFromPattern(newSubscription))
-                metadata.requestUpdateForNewTopics();
-            this.joinedSubscription = newJoinedSubscription;
+        ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
+        try {
+            listener.onPartitionsLost(lostPartitions);
+        } catch (WakeupException | InterruptException e) {
+            throw e;
+        } catch (Exception e) {
+            log.error("User provided listener {} failed on invocation of onPartitionsLost for partitions {}",
+                listener.getClass().getName(), lostPartitions, e);
+            return e;
         }
+
+        return null;
     }
 
     @Override
@@ -263,6 +319,8 @@ protected void onJoinComplete(int generation,
                                   String memberId,
                                   String assignmentStrategy,
                                   ByteBuffer assignmentBuffer) {
+        log.debug("Executing onJoinComplete with generation {} and memberId {}", generation, memberId);
+
         // only the leader is responsible for monitoring for metadata changes (i.e. partition changes)
         if (!isLeader)
             assignmentSnapshot = null;
@@ -274,7 +332,10 @@ protected void onJoinComplete(int generation,
         Set ownedPartitions = new HashSet<>(subscriptions.assignedPartitions());
 
         Assignment assignment = ConsumerProtocol.deserializeAssignment(assignmentBuffer);
-        if (!subscriptions.assignFromSubscribed(assignment.partitions())) {
+
+        Set assignedPartitions = new HashSet<>(assignment.partitions());
+
+        if (!subscriptions.checkAssignmentMatchedSubscription(assignedPartitions)) {
             log.warn("We received an assignment {} that doesn't match our current subscription {}; it is likely " +
                 "that the subscription has changed since we joined the group. Will try re-join the group with current subscription",
                 assignment.partitions(), subscriptions.prettyString());
@@ -284,8 +345,6 @@ protected void onJoinComplete(int generation,
             return;
         }
 
-        Set assignedPartitions = subscriptions.assignedPartitions();
-
         // The leader may have assigned partitions which match our subscription pattern, but which
         // were not explicitly requested, so we update the joined subscription here.
         maybeUpdateJoinedSubscription(assignedPartitions);
@@ -299,63 +358,52 @@ protected void onJoinComplete(int generation,
             this.nextAutoCommitTimer.updateAndReset(autoCommitIntervalMs);
 
         // execute the user's callback after rebalance
-        ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
+        final AtomicReference firstException = new AtomicReference<>(null);
+        Set addedPartitions = new HashSet<>(assignedPartitions);
+        addedPartitions.removeAll(ownedPartitions);
 
         switch (protocol) {
             case EAGER:
-                if (!ownedPartitions.isEmpty()) {
-                    log.info("Coordinator has owned partitions {} that are not revoked with {} protocol, " +
-                        "it is likely client is woken up before a previous pending rebalance completes its callback", ownedPartitions, protocol);
-                }
+                // assign partitions that are not yet owned
+                subscriptions.assignFromSubscribed(assignedPartitions);
+
+                firstException.compareAndSet(null, invokePartitionsAssigned(addedPartitions));
 
-                log.info("Setting newly assigned partitions: {}", Utils.join(assignedPartitions, ", "));
-                try {
-                    listener.onPartitionsAssigned(assignedPartitions);
-                } catch (WakeupException | InterruptException e) {
-                    throw e;
-                } catch (Exception e) {
-                    log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e);
-                }
                 break;
 
             case COOPERATIVE:
-                assignAndRevoke(listener, assignedPartitions, ownedPartitions);
-
-                break;
-        }
+                Set revokedPartitions = new HashSet<>(ownedPartitions);
+                revokedPartitions.removeAll(assignedPartitions);
+
+                log.info("Updating with newly assigned partitions: {}, compare with already owned partitions: {}, " +
+                        "newly added partitions: {}, revoking partitions: {}",
+                    Utils.join(assignedPartitions, ", "),
+                    Utils.join(ownedPartitions, ", "),
+                    Utils.join(addedPartitions, ", "),
+                    Utils.join(revokedPartitions, ", "));
+
+                // revoke partitions that was previously owned but no longer assigned;
+                // note that we should only change the assignment AFTER we've triggered
+                // the revoke callback
+                if (!revokedPartitions.isEmpty()) {
+                    firstException.compareAndSet(null, invokePartitionsRevoked(revokedPartitions));
+                }
 
-    }
+                subscriptions.assignFromSubscribed(assignedPartitions);
 
-    private void assignAndRevoke(final ConsumerRebalanceListener listener,
-                                 final Set assignedPartitions,
-                                 final Set ownedPartitions) {
-        Set addedPartitions = new HashSet<>(assignedPartitions);
-        Set revokedPartitions = new HashSet<>(ownedPartitions);
-        addedPartitions.removeAll(ownedPartitions);
-        revokedPartitions.removeAll(assignedPartitions);
+                // add partitions that were not previously owned but are now assigned
+                firstException.compareAndSet(null, invokePartitionsAssigned(addedPartitions));
 
-        log.info("Updating with newly assigned partitions: {}, compare with already owned partitions: {}, " +
-                "newly added partitions: {}, revoking partitions: {}",
-            Utils.join(assignedPartitions, ", "),
-            Utils.join(ownedPartitions, ", "),
-            Utils.join(addedPartitions, ", "),
-            Utils.join(revokedPartitions, ", "));
+                // if revoked any partitions, need to re-join the group afterwards
+                if (!revokedPartitions.isEmpty()) {
+                    requestRejoin();
+                }
 
-        try {
-            listener.onPartitionsAssigned(addedPartitions);
-        } catch (WakeupException | InterruptException e) {
-            throw e;
-        } catch (Exception e) {
-            log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e);
+                break;
         }
 
-        try {
-            listener.onPartitionsRevoked(revokedPartitions);
-        } catch (WakeupException | InterruptException e) {
-            throw e;
-        } catch (Exception e) {
-            log.error("User provided listener {} failed on partition revocation", listener.getClass().getName(), e);
-        }
+        if (firstException.get() != null)
+            throw new KafkaException("User rebalance callback throws an error", firstException.get());
     }
 
     void maybeUpdateSubscriptionMetadata() {
@@ -380,6 +428,7 @@ void maybeUpdateSubscriptionMetadata() {
      * Returns early if the timeout expires
      *
      * @param timer Timer bounding how long this method can block
+     * @throws KafkaException if the rebalance callback throws an exception
      * @return true iff the operation succeeded
      */
     public boolean poll(Timer timer) {
@@ -389,8 +438,8 @@ public boolean poll(Timer timer) {
 
         if (subscriptions.partitionsAutoAssigned()) {
             if (protocol == null) {
-                throw new IllegalStateException("User confingure ConsumerConfig#PARTITION_ASSIGNMENT_STRATEGY_CONFIG to empty " +
-                    "while trying to subscribe for group protocol to auto assign partitions");
+                throw new IllegalStateException("User configured " + ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG +
+                    " to empty while trying to subscribe for group protocol to auto assign partitions");
             }
             // Always update the heartbeat last poll time so that the heartbeat thread does not leave the
             // group proactively due to application inactivity even if (say) the coordinator cannot be found.
@@ -479,11 +528,16 @@ protected Map performAssignment(String leaderId,
 
         Set allSubscribedTopics = new HashSet<>();
         Map subscriptions = new HashMap<>();
+
+        // collect all the owned partitions
+        Map> ownedPartitions = new HashMap<>();
+
         for (JoinGroupResponseData.JoinGroupResponseMember memberSubscription : allSubscriptions) {
             Subscription subscription = ConsumerProtocol.deserializeSubscription(ByteBuffer.wrap(memberSubscription.metadata()));
             subscription.setGroupInstanceId(Optional.ofNullable(memberSubscription.groupInstanceId()));
             subscriptions.put(memberSubscription.memberId(), subscription);
             allSubscribedTopics.addAll(subscription.topics());
+            ownedPartitions.put(memberSubscription.memberId(), subscription.ownedPartitions());
         }
 
         // the leader will begin watching for changes to any of the topics the group is interested in,
@@ -496,6 +550,10 @@ protected Map performAssignment(String leaderId,
 
         Map assignments = assignor.assign(metadata.fetch(), new GroupSubscription(subscriptions)).groupAssignment();
 
+        if (protocol == RebalanceProtocol.COOPERATIVE) {
+            validateCooperativeAssignment(ownedPartitions, assignments);
+        }
+
         // user-customized assignor may have created some topics that are not in the subscription list
         // and assign their partitions to the members; in this case we would like to update the leader's
         // own metadata with the newly added topics so that it will not trigger a subsequent rebalance
@@ -538,51 +596,137 @@ protected Map performAssignment(String leaderId,
         return groupAssignment;
     }
 
+    /**
+     * Used by COOPERATIVE rebalance protocol only.
+     *
+     * Validate the assignments returned by the assignor such that no owned partitions are going to
+     * be reassigned to a different consumer directly: if the assignor wants to reassign an owned partition,
+     * it must first remove it from the new assignment of the current owner so that it is not assigned to any
+     * member, and then in the next rebalance it can finally reassign those partitions not owned by anyone to consumers.
+     */
+    private void validateCooperativeAssignment(final Map> ownedPartitions,
+                                               final Map assignments) {
+        Set totalRevokedPartitions = new HashSet<>();
+        Set totalAddedPartitions = new HashSet<>();
+        for (final Map.Entry entry : assignments.entrySet()) {
+            final Assignment assignment = entry.getValue();
+            final Set addedPartitions = new HashSet<>(assignment.partitions());
+            addedPartitions.removeAll(ownedPartitions.get(entry.getKey()));
+            final Set revokedPartitions = new HashSet<>(ownedPartitions.get(entry.getKey()));
+            revokedPartitions.removeAll(assignment.partitions());
+
+            totalAddedPartitions.addAll(addedPartitions);
+            totalRevokedPartitions.addAll(revokedPartitions);
+        }
+
+        // if there are overlap between revoked partitions and added partitions, it means some partitions
+        // immediately gets re-assigned to another member while it is still claimed by some member
+        totalAddedPartitions.retainAll(totalRevokedPartitions);
+        if (!totalAddedPartitions.isEmpty()) {
+            log.error("With the COOPERATIVE protocol, owned partitions cannot be " +
+                "reassigned to other members; however the assignor has reassigned partitions {} which are still owned " +
+                "by some members; return the error code to all members to let them stop", totalAddedPartitions);
+
+            throw new IllegalStateException("Assignor supporting the COOPERATIVE protocol violates its requirements");
+        }
+    }
+
     @Override
     protected void onJoinPrepare(int generation, String memberId) {
+        log.debug("Executing onJoinPrepare with generation {} and memberId {}", generation, memberId);
         // commit offsets prior to rebalance if auto-commit enabled
         maybeAutoCommitOffsetsSync(time.timer(rebalanceConfig.rebalanceTimeoutMs));
 
-        // execute the user's callback before rebalance
-        ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
+        // the generation / member-id can possibly be reset by the heartbeat thread
+        // upon getting errors or heartbeat timeouts; in this case whatever is previously
+        // owned partitions would be lost, we should trigger the callback and cleanup the assignment;
+        // otherwise we can proceed normally and revoke the partitions depending on the protocol,
+        // and in that case we should only change the assignment AFTER the revoke callback is triggered
+        // so that users can still access the previously owned partitions to commit offsets etc.
+        Exception exception = null;
+        final Set revokedPartitions;
+        if (generation == Generation.NO_GENERATION.generationId &&
+            memberId.equals(Generation.NO_GENERATION.memberId)) {
+            revokedPartitions = new HashSet<>(subscriptions.assignedPartitions());
+
+            if (!revokedPartitions.isEmpty()) {
+                log.info("Giving away all assigned partitions as lost since generation has been reset," +
+                    "indicating that consumer is no longer part of the group");
+                exception = invokePartitionsLost(revokedPartitions);
 
-        switch (protocol) {
-            case EAGER:
-                Set revokedPartitions = new HashSet<>(subscriptions.assignedPartitions());
-                log.info("Revoking previously assigned partitions {}", revokedPartitions);
-                try {
-                    listener.onPartitionsRevoked(revokedPartitions);
-                } catch (WakeupException | InterruptException e) {
-                    throw e;
-                } catch (Exception e) {
-                    log.error("User provided listener {} failed on partition revocation", listener.getClass().getName(), e);
-                }
-
-                // also clear the assigned partitions since all have been revoked
                 subscriptions.assignFromSubscribed(Collections.emptySet());
+            }
+        } else {
+            switch (protocol) {
+                case EAGER:
+                    // revoke all partitions
+                    revokedPartitions = new HashSet<>(subscriptions.assignedPartitions());
+                    exception = invokePartitionsRevoked(revokedPartitions);
 
-                break;
+                    subscriptions.assignFromSubscribed(Collections.emptySet());
 
-            case COOPERATIVE:
-                break;
+                    break;
+
+                case COOPERATIVE:
+                    // only revoke those partitions that are not in the subscription any more.
+                    Set ownedPartitions = new HashSet<>(subscriptions.assignedPartitions());
+                    revokedPartitions = ownedPartitions.stream()
+                        .filter(tp -> !subscriptions.subscription().contains(tp.topic()))
+                        .collect(Collectors.toSet());
+
+                    if (!revokedPartitions.isEmpty()) {
+                        exception = invokePartitionsRevoked(revokedPartitions);
+
+                        ownedPartitions.removeAll(revokedPartitions);
+                        subscriptions.assignFromSubscribed(ownedPartitions);
+                    }
+
+                    break;
+            }
         }
 
+
         isLeader = false;
         subscriptions.resetGroupSubscription();
+
+        if (exception != null) {
+            throw new KafkaException("User rebalance callback throws an error", exception);
+        }
+    }
+
+    @Override
+    public void onLeavePrepare() {
+        // we should reset assignment and trigger the callback before leaving group
+        Set droppedPartitions = new HashSet<>(subscriptions.assignedPartitions());
+
+        if (subscriptions.partitionsAutoAssigned() && !droppedPartitions.isEmpty()) {
+            final Exception e = invokePartitionsRevoked(droppedPartitions);
+
+            subscriptions.assignFromSubscribed(Collections.emptySet());
+
+            if (e != null) {
+                throw new KafkaException("User rebalance callback throws an error", e);
+            }
+        }
     }
 
+    /**
+     * @throws KafkaException if the callback throws exception
+     */
     @Override
     public boolean rejoinNeededOrPending() {
         if (!subscriptions.partitionsAutoAssigned())
             return false;
 
-        // we need to rejoin if we performed the assignment and metadata has changed
+        // we need to rejoin if we performed the assignment and metadata has changed;
+        // also for those owned-but-no-longer-existed partitions we should drop them as lost
         if (assignmentSnapshot != null && !assignmentSnapshot.matches(metadataSnapshot))
             return true;
 
         // we need to join if our subscription has changed since the last join
-        if (joinedSubscription != null && !joinedSubscription.equals(subscriptions.subscription()))
+        if (joinedSubscription != null && !joinedSubscription.equals(subscriptions.subscription())) {
             return true;
+        }
 
         return super.rejoinNeededOrPending();
     }
@@ -661,6 +805,9 @@ public Map fetchCommittedOffsets(final Set futu
                         } else if (error == Errors.UNKNOWN_MEMBER_ID
                                 || error == Errors.ILLEGAL_GENERATION) {
                             // need to reset generation and re-join group
-                            resetGeneration();
+                            resetGenerationOnResponseError(ApiKeys.OFFSET_COMMIT, error);
                             future.raise(new CommitFailedException());
                             return;
                         } else {
@@ -1121,8 +1268,11 @@ private static class MetadataSnapshot {
 
         private MetadataSnapshot(SubscriptionState subscription, Cluster cluster, int version) {
             Map partitionsPerTopic = new HashMap<>();
-            for (String topic : subscription.groupSubscription())
-                partitionsPerTopic.put(topic, cluster.partitionCountForTopic(topic));
+            for (String topic : subscription.groupSubscription()) {
+                Integer numPartitions = cluster.partitionCountForTopic(topic);
+                if (numPartitions != null)
+                    partitionsPerTopic.put(topic, numPartitions);
+            }
             this.partitionsPerTopic = partitionsPerTopic;
             this.version = version;
         }
@@ -1130,6 +1280,10 @@ private MetadataSnapshot(SubscriptionState subscription, Cluster cluster, int ve
         boolean matches(MetadataSnapshot other) {
             return version == other.version || partitionsPerTopic.equals(other.partitionsPerTopic);
         }
+
+        Map partitionsPerTopic() {
+            return partitionsPerTopic;
+        }
     }
 
     private static class OffsetCommitCompletion {
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
index 1ecb15c8d4f25..a66921d872070 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
@@ -77,7 +77,6 @@ default void onAssignment(Assignment assignment, int generation) {
         onAssignment(assignment);
     }
 
-
     /**
      * Unique name for this assignor (e.g. "range" or "roundrobin" or "sticky")
      * @return non-null unique name
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
index af834ce71b1db..7c965ae58a55d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
@@ -238,42 +238,42 @@ public synchronized boolean assignFromUser(Set partitions) {
     }
 
     /**
-     * Change the assignment to the specified partitions returned from the coordinator, note this is
-     * different from {@link #assignFromUser(Set)} which directly set the assignment from user inputs.
-     *
      * @return true if assignments matches subscription, otherwise false
      */
-    public synchronized boolean assignFromSubscribed(Collection assignments) {
-        if (!this.partitionsAutoAssigned())
-            throw new IllegalArgumentException("Attempt to dynamically assign partitions while manual assignment in use");
-
-        boolean assignmentMatchedSubscription = true;
+    public synchronized boolean checkAssignmentMatchedSubscription(Collection assignments) {
         for (TopicPartition topicPartition : assignments) {
             if (this.subscribedPattern != null) {
-                assignmentMatchedSubscription = this.subscribedPattern.matcher(topicPartition.topic()).matches();
-                if (!assignmentMatchedSubscription) {
+                if (!this.subscribedPattern.matcher(topicPartition.topic()).matches()) {
                     log.info("Assigned partition {} for non-subscribed topic regex pattern; subscription pattern is {}",
-                            topicPartition,
-                            this.subscribedPattern);
-                    break;
+                        topicPartition,
+                        this.subscribedPattern);
+
+                    return false;
                 }
             } else {
-                assignmentMatchedSubscription = this.subscription.contains(topicPartition.topic());
-                if (!assignmentMatchedSubscription) {
+                if (!this.subscription.contains(topicPartition.topic())) {
                     log.info("Assigned partition {} for non-subscribed topic; subscription is {}", topicPartition, this.subscription);
-                    break;
+
+                    return false;
                 }
             }
         }
 
-        if (assignmentMatchedSubscription) {
-            Map assignedPartitionStates = partitionToStateMap(
-                    assignments);
-            assignmentId++;
-            this.assignment.set(assignedPartitionStates);
-        }
+        return true;
+    }
 
-        return assignmentMatchedSubscription;
+    /**
+     * Change the assignment to the specified partitions returned from the coordinator, note this is
+     * different from {@link #assignFromUser(Set)} which directly set the assignment from user inputs.
+     */
+    public synchronized void assignFromSubscribed(Collection assignments) {
+        if (!this.partitionsAutoAssigned())
+            throw new IllegalArgumentException("Attempt to dynamically assign partitions while manual assignment in use");
+
+
+        Map assignedPartitionStates = partitionToStateMap(assignments);
+        assignmentId++;
+        this.assignment.set(assignedPartitionStates);
     }
 
     private void registerRebalanceListener(ConsumerRebalanceListener listener) {
@@ -652,8 +652,13 @@ public synchronized void resume(TopicPartition tp) {
     }
 
     synchronized void requestFailed(Set partitions, long nextRetryTimeMs) {
-        for (TopicPartition partition : partitions)
-            assignedState(partition).requestFailed(nextRetryTimeMs);
+        for (TopicPartition partition : partitions) {
+            // by the time the request failed, the assignment may no longer
+            // contain this partition any more, in which case we would just ignore.
+            final TopicPartitionState state = assignedStateOrNull(partition);
+            if (state != null)
+                state.requestFailed(nextRetryTimeMs);
+        }
     }
 
     synchronized void movePartitionToEnd(TopicPartition tp) {
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
index 1227c27ad6035..7892d6d1c92db 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
@@ -1332,7 +1332,7 @@ public void testCloseInterrupt() throws Exception {
     }
 
     @Test
-    public void closeShouldBeIdempotent() {
+    public void testCloseShouldBeIdempotent() {
         KafkaConsumer consumer = newConsumer((String) null);
         consumer.close();
         consumer.close();
@@ -1419,7 +1419,7 @@ public void testMetricConfigRecordingLevel() {
     }
 
     @Test
-    public void shouldAttemptToRejoinGroupAfterSyncGroupFailed() throws Exception {
+    public void testShouldAttemptToRejoinGroupAfterSyncGroupFailed() throws Exception {
         Time time = new MockTime();
         SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.EARLIEST);
         ConsumerMetadata metadata = createMetadata(subscription);
@@ -1639,6 +1639,52 @@ public void testCommittedAuthenticationFaiure() {
         consumer.committed(tp0);
     }
 
+    @Test
+    public void testRebalanceException() {
+        Time time = new MockTime();
+        SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.EARLIEST);
+        ConsumerMetadata metadata = createMetadata(subscription);
+        MockClient client = new MockClient(time, metadata);
+
+        initMetadata(client, Collections.singletonMap(topic, 1));
+        Node node = metadata.fetch().nodes().get(0);
+
+        ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
+
+        KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
+
+        consumer.subscribe(singleton(topic), getExceptionConsumerRebalanceListener());
+        Node coordinator = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port());
+
+        client.prepareResponseFrom(FindCoordinatorResponse.prepareResponse(Errors.NONE, node), node);
+        client.prepareResponseFrom(joinGroupFollowerResponse(assignor, 1, "memberId", "leaderId", Errors.NONE), coordinator);
+        client.prepareResponseFrom(syncGroupResponse(singletonList(tp0), Errors.NONE), coordinator);
+
+        // assign throws
+        try {
+            consumer.updateAssignmentMetadataIfNeeded(time.timer(Long.MAX_VALUE));
+            fail("Should throw exception");
+        } catch (Throwable e) {
+            assertEquals("boom!", e.getCause().getMessage());
+        }
+
+        // the assignment is still updated regardless of the exception
+        assertEquals(singleton(tp0), subscription.assignedPartitions());
+
+        // close's revoke throws
+        try {
+            consumer.close(Duration.ofMillis(0));
+            fail("Should throw exception");
+        } catch (Throwable e) {
+            assertEquals("boom!", e.getCause().getCause().getMessage());
+        }
+
+        consumer.close(Duration.ofMillis(0));
+
+        // the assignment is still updated regardless of the exception
+        assertTrue(subscription.assignedPartitions().isEmpty());
+    }
+
     private KafkaConsumer consumerWithPendingAuthenticationError() {
         Time time = new MockTime();
         SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.EARLIEST);
@@ -1669,6 +1715,20 @@ public void onPartitionsAssigned(Collection partitions) {
         };
     }
 
+    private ConsumerRebalanceListener getExceptionConsumerRebalanceListener() {
+        return new ConsumerRebalanceListener() {
+            @Override
+            public void onPartitionsRevoked(Collection partitions) {
+                throw new RuntimeException("boom!");
+            }
+
+            @Override
+            public void onPartitionsAssigned(Collection partitions) {
+                throw new RuntimeException("boom!");
+            }
+        };
+    }
+
     private ConsumerMetadata createMetadata(SubscriptionState subscription) {
         return new ConsumerMetadata(0, Long.MAX_VALUE, false, false,
                                     subscription, new LogContext(), new ClusterResourceListeners());
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/RoundRobinAssignorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/RoundRobinAssignorTest.java
index 02fb9ffba403e..e7622807834ad 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/RoundRobinAssignorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/RoundRobinAssignorTest.java
@@ -188,14 +188,10 @@ public void testTwoStaticConsumersTwoTopicsSixPartitions() {
         partitionsPerTopic.put(topic2, 3);
 
         Map consumers = new HashMap<>();
-        Subscription consumer1Subscription = new Subscription(topics(topic1, topic2),
-                                                              null,
-                                                              Collections.emptyList());
+        Subscription consumer1Subscription = new Subscription(topics(topic1, topic2), null);
         consumer1Subscription.setGroupInstanceId(Optional.of(instance1));
         consumers.put(consumer1, consumer1Subscription);
-        Subscription consumer2Subscription = new Subscription(topics(topic1, topic2),
-                                                              null,
-                                                              Collections.emptyList());
+        Subscription consumer2Subscription = new Subscription(topics(topic1, topic2), null);
         consumer2Subscription.setGroupInstanceId(Optional.of(instance2));
         consumers.put(consumer2, consumer2Subscription);
         Map> assignment = assignor.assign(partitionsPerTopic, consumers);
@@ -219,9 +215,7 @@ public void testOneStaticConsumerAndOneDynamicConsumerTwoTopicsSixPartitions() {
 
         Map consumers = new HashMap<>();
 
-        Subscription consumer1Subscription = new Subscription(topics(topic1, topic2),
-                                                              null,
-                                                              Collections.emptyList());
+        Subscription consumer1Subscription = new Subscription(topics(topic1, topic2), null);
         consumer1Subscription.setGroupInstanceId(Optional.of(instance1));
         consumers.put(consumer1, consumer1Subscription);
         consumers.put(consumer2, new Subscription(topics(topic1, topic2)));
@@ -257,9 +251,7 @@ public void testStaticMemberAssignmentPersistent() {
         partitionsPerTopic.put(topic2, 3);
         Map consumers = new HashMap<>();
         for (MemberInfo m : staticMemberInfos) {
-            Subscription subscription = new Subscription(topics(topic1, topic2),
-                                                         null,
-                                                         Collections.emptyList());
+            Subscription subscription = new Subscription(topics(topic1, topic2), null);
             subscription.setGroupInstanceId(m.groupInstanceId);
             consumers.put(m.memberId, subscription);
         }
@@ -333,9 +325,7 @@ private Map> checkStaticAssignment(String topic1,
         partitionsPerTopic.put(topic2, 3);
         Map consumers = new HashMap<>();
         for (MemberInfo m : staticMemberInfos) {
-            Subscription subscription = new Subscription(topics(topic1, topic2),
-                                                         null,
-                                                         Collections.emptyList());
+            Subscription subscription = new Subscription(topics(topic1, topic2), null);
             subscription.setGroupInstanceId(m.groupInstanceId);
             consumers.put(m.memberId, subscription);
         }
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index 11273cb6f1a17..91d3529ea10e4 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -52,6 +52,7 @@
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.record.RecordBatch;
+import org.apache.kafka.common.requests.AbstractRequest;
 import org.apache.kafka.common.requests.FindCoordinatorResponse;
 import org.apache.kafka.common.requests.HeartbeatResponse;
 import org.apache.kafka.common.requests.JoinGroupRequest;
@@ -100,6 +101,8 @@
 import static java.util.Collections.singleton;
 import static java.util.Collections.singletonList;
 import static java.util.Collections.singletonMap;
+import static org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.RebalanceProtocol.COOPERATIVE;
+import static org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.RebalanceProtocol.EAGER;
 import static org.apache.kafka.test.TestUtils.toSet;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -201,18 +204,18 @@ public void teardown() {
     public void testSelectRebalanceProtcol() {
         List assignors = new ArrayList<>();
         assignors.add(new MockPartitionAssignor(Collections.singletonList(ConsumerPartitionAssignor.RebalanceProtocol.EAGER)));
-        assignors.add(new MockPartitionAssignor(Collections.singletonList(ConsumerPartitionAssignor.RebalanceProtocol.COOPERATIVE)));
+        assignors.add(new MockPartitionAssignor(Collections.singletonList(COOPERATIVE)));
 
         // no commonly supported protocols
         assertThrows(IllegalArgumentException.class, () -> buildCoordinator(rebalanceConfig, new Metrics(), assignors, false));
 
         assignors.clear();
-        assignors.add(new MockPartitionAssignor(Arrays.asList(ConsumerPartitionAssignor.RebalanceProtocol.EAGER, ConsumerPartitionAssignor.RebalanceProtocol.COOPERATIVE)));
-        assignors.add(new MockPartitionAssignor(Arrays.asList(ConsumerPartitionAssignor.RebalanceProtocol.EAGER, ConsumerPartitionAssignor.RebalanceProtocol.COOPERATIVE)));
+        assignors.add(new MockPartitionAssignor(Arrays.asList(ConsumerPartitionAssignor.RebalanceProtocol.EAGER, COOPERATIVE)));
+        assignors.add(new MockPartitionAssignor(Arrays.asList(ConsumerPartitionAssignor.RebalanceProtocol.EAGER, COOPERATIVE)));
 
         // select higher indexed (more advanced) protocols
         try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors, false)) {
-            assertEquals(ConsumerPartitionAssignor.RebalanceProtocol.COOPERATIVE, coordinator.getProtocol());
+            assertEquals(COOPERATIVE, coordinator.getProtocol());
         }
     }
 
@@ -247,7 +250,7 @@ public void testGroupReadUnauthorized() {
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
 
-        client.prepareResponse(joinGroupLeaderResponse(0, "memberId", Collections.>emptyMap(),
+        client.prepareResponse(joinGroupLeaderResponse(0, "memberId", Collections.emptyMap(),
                 Errors.GROUP_AUTHORIZATION_FAILED));
         coordinator.poll(time.timer(Long.MAX_VALUE));
     }
@@ -274,7 +277,7 @@ public void testCoordinatorNotAvailable() {
     }
 
     @Test
-    public void testManyInFlightAsyncCommitsWithCoordinatorDisconnect() throws Exception {
+    public void testManyInFlightAsyncCommitsWithCoordinatorDisconnect() {
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
 
@@ -302,7 +305,7 @@ public void onComplete(Map offsets, Exception
     }
 
     @Test
-    public void testCoordinatorUnknownInUnsentCallbacksAfterCoordinatorDead() throws Exception {
+    public void testCoordinatorUnknownInUnsentCallbacksAfterCoordinatorDead() {
         // When the coordinator is marked dead, all unsent or in-flight requests are cancelled
         // with a disconnect error. This test case ensures that the corresponding callbacks see
         // the coordinator as unknown which prevents additional retries to the same coordinator.
@@ -389,10 +392,15 @@ public void testIllegalGeneration() {
         assertTrue(future.failed());
         assertEquals(Errors.ILLEGAL_GENERATION.exception(), future.exception());
         assertTrue(coordinator.rejoinNeededOrPending());
+
+        coordinator.poll(time.timer(0));
+
+        assertEquals(1, rebalanceListener.lostCount);
+        assertEquals(Collections.singleton(t1p), rebalanceListener.lost);
     }
 
     @Test
-    public void testUnknownConsumerId() {
+    public void testUnknownMemberId() {
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
 
@@ -413,6 +421,11 @@ public void testUnknownConsumerId() {
         assertTrue(future.failed());
         assertEquals(Errors.UNKNOWN_MEMBER_ID.exception(), future.exception());
         assertTrue(coordinator.rejoinNeededOrPending());
+
+        coordinator.poll(time.timer(0));
+
+        assertEquals(1, rebalanceListener.lostCount);
+        assertEquals(Collections.singleton(t1p), rebalanceListener.lost);
     }
 
     @Test
@@ -448,7 +461,7 @@ public void testJoinGroupInvalidGroupId() {
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
 
-        client.prepareResponse(joinGroupLeaderResponse(0, consumerId, Collections.>emptyMap(),
+        client.prepareResponse(joinGroupLeaderResponse(0, consumerId, Collections.emptyMap(),
                 Errors.INVALID_GROUP_ID));
         coordinator.poll(time.timer(Long.MAX_VALUE));
     }
@@ -484,8 +497,8 @@ public void testNormalJoinGroupLeader() {
         assertFalse(coordinator.rejoinNeededOrPending());
         assertEquals(toSet(assigned), subscriptions.assignedPartitions());
         assertEquals(subscription, subscriptions.groupSubscription());
-        assertEquals(1, rebalanceListener.revokedCount);
-        assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked);
+        assertEquals(0, rebalanceListener.revokedCount);
+        assertNull(rebalanceListener.revoked);
         assertEquals(1, rebalanceListener.assignedCount);
         assertEquals(getAdded(owned, assigned), rebalanceListener.assigned);
     }
@@ -538,23 +551,13 @@ public void testOutdatedCoordinatorAssignment() {
 
         coordinator.poll(time.timer(Long.MAX_VALUE));
 
-        final Collection revoked = getRevoked(owned, newAssignment);
         final Collection assigned = getAdded(owned, newAssignment);
 
-        int revokeCount = 1;
-        final int addCount = 1;
-
-        // with eager protocol we will call revoke on the old assignment as well
-        if (protocol == ConsumerPartitionAssignor.RebalanceProtocol.EAGER) {
-            revokeCount += 1;
-        }
-
         assertFalse(coordinator.rejoinNeededOrPending());
         assertEquals(toSet(newAssignment), subscriptions.assignedPartitions());
         assertEquals(toSet(newSubscription), subscriptions.groupSubscription());
-        assertEquals(revokeCount, rebalanceListener.revokedCount);
-        assertEquals(revoked, rebalanceListener.revoked);
-        assertEquals(addCount, rebalanceListener.assignedCount);
+        assertEquals(protocol == EAGER ? 1 : 0, rebalanceListener.revokedCount);
+        assertEquals(1, rebalanceListener.assignedCount);
         assertEquals(assigned, rebalanceListener.assigned);
     }
 
@@ -593,8 +596,9 @@ public void testPatternJoinGroupLeader() {
         assertEquals(2, subscriptions.numAssignedPartitions());
         assertEquals(2, subscriptions.groupSubscription().size());
         assertEquals(2, subscriptions.subscription().size());
-        assertEquals(1, rebalanceListener.revokedCount);
-        assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked);
+        // callback not triggered at all since there's nothing to be revoked
+        assertEquals(0, rebalanceListener.revokedCount);
+        assertNull(rebalanceListener.revoked);
         assertEquals(1, rebalanceListener.assignedCount);
         assertEquals(getAdded(owned, assigned), rebalanceListener.assigned);
     }
@@ -604,8 +608,6 @@ public void testMetadataRefreshDuringRebalance() {
         final String consumerId = "leader";
         final List owned = Collections.emptyList();
         final List oldAssigned = Arrays.asList(t1p);
-
-
         subscriptions.subscribe(Pattern.compile(".*"), rebalanceListener);
         client.updateMetadata(TestUtils.metadataUpdateWith(1, singletonMap(topic1, 1)));
         coordinator.maybeUpdateSubscriptionMetadata();
@@ -635,14 +637,15 @@ public void testMetadataRefreshDuringRebalance() {
         assertFalse(coordinator.rejoinNeededOrPending());
         assertEquals(singleton(topic1), subscriptions.subscription());
         assertEquals(toSet(oldAssigned), subscriptions.assignedPartitions());
-        assertEquals(1, rebalanceListener.revokedCount);
-        assertEquals(getRevoked(owned, oldAssigned), rebalanceListener.revoked);
+        // nothing to be revoked and hence no callback triggered
+        assertEquals(0, rebalanceListener.revokedCount);
+        assertNull(rebalanceListener.revoked);
         assertEquals(1, rebalanceListener.assignedCount);
         assertEquals(getAdded(owned, oldAssigned), rebalanceListener.assigned);
 
         List newAssigned = Arrays.asList(t1p, t2p);
 
-        Map> updatedSubscriptions = singletonMap(consumerId, Arrays.asList(topic1, topic2));
+        final Map> updatedSubscriptions = singletonMap(consumerId, Arrays.asList(topic1, topic2));
         partitionAssignor.prepare(singletonMap(consumerId, newAssigned));
 
         // we expect to see a second rebalance with the new-found topics
@@ -658,17 +661,63 @@ public void testMetadataRefreshDuringRebalance() {
             metadata.rewind();
             return subscription.topics().containsAll(updatedSubscription);
         }, joinGroupLeaderResponse(2, consumerId, updatedSubscriptions, Errors.NONE));
-        client.prepareResponse(syncGroupResponse(newAssigned, Errors.NONE));
+        client.prepareResponse(new MockClient.RequestMatcher() {
+            // update the metadata again back to topic1
+            @Override
+            public boolean matches(AbstractRequest body) {
+                client.updateMetadata(TestUtils.metadataUpdateWith(1, singletonMap(topic1, 1)));
+                return true;
+            }
+        }, syncGroupResponse(newAssigned, Errors.NONE));
 
         coordinator.poll(time.timer(Long.MAX_VALUE));
 
+        Collection revoked = getRevoked(oldAssigned, newAssigned);
+        int revokedCount = revoked.isEmpty() ? 0 : 1;
+
         assertFalse(coordinator.rejoinNeededOrPending());
         assertEquals(toSet(updatedSubscription), subscriptions.subscription());
         assertEquals(toSet(newAssigned), subscriptions.assignedPartitions());
-        assertEquals(2, rebalanceListener.revokedCount);
-        assertEquals(getRevoked(oldAssigned, newAssigned), rebalanceListener.revoked);
+        assertEquals(revokedCount, rebalanceListener.revokedCount);
+        assertEquals(revoked.isEmpty() ? null : revoked, rebalanceListener.revoked);
         assertEquals(2, rebalanceListener.assignedCount);
         assertEquals(getAdded(oldAssigned, newAssigned), rebalanceListener.assigned);
+
+        // we expect to see a third rebalance with the new-found topics
+        partitionAssignor.prepare(singletonMap(consumerId, oldAssigned));
+
+        client.prepareResponse(new MockClient.RequestMatcher() {
+            @Override
+            public boolean matches(AbstractRequest body) {
+                JoinGroupRequest join = (JoinGroupRequest) body;
+                Iterator protocolIterator =
+                    join.data().protocols().iterator();
+                assertTrue(protocolIterator.hasNext());
+                JoinGroupRequestData.JoinGroupRequestProtocol protocolMetadata = protocolIterator.next();
+
+                ByteBuffer metadata = ByteBuffer.wrap(protocolMetadata.metadata());
+                ConsumerPartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(metadata);
+                metadata.rewind();
+                return subscription.topics().contains(topic1);
+            }
+        }, joinGroupLeaderResponse(3, consumerId, initialSubscription, Errors.NONE));
+        client.prepareResponse(syncGroupResponse(oldAssigned, Errors.NONE));
+
+        coordinator.poll(time.timer(Long.MAX_VALUE));
+
+        revoked = getRevoked(newAssigned, oldAssigned);
+        assertFalse(revoked.isEmpty());
+        revokedCount += 1;
+        Collection added = getAdded(newAssigned, oldAssigned);
+
+        assertFalse(coordinator.rejoinNeededOrPending());
+        assertEquals(singleton(topic1), subscriptions.subscription());
+        assertEquals(toSet(oldAssigned), subscriptions.assignedPartitions());
+        assertEquals(revokedCount, rebalanceListener.revokedCount);
+        assertEquals(revoked.isEmpty() ? null : revoked, rebalanceListener.revoked);
+        assertEquals(3, rebalanceListener.assignedCount);
+        assertEquals(added, rebalanceListener.assigned);
+        assertEquals(0, rebalanceListener.lostCount);
     }
 
     @Test
@@ -745,8 +794,8 @@ public void testWakeupDuringJoin() {
 
         assertFalse(coordinator.rejoinNeededOrPending());
         assertEquals(toSet(assigned), subscriptions.assignedPartitions());
-        assertEquals(1, rebalanceListener.revokedCount);
-        assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked);
+        assertEquals(0, rebalanceListener.revokedCount);
+        assertNull(rebalanceListener.revoked);
         assertEquals(1, rebalanceListener.assignedCount);
         assertEquals(getAdded(owned, assigned), rebalanceListener.assigned);
     }
@@ -777,8 +826,8 @@ public void testNormalJoinGroupFollower() {
         assertFalse(coordinator.rejoinNeededOrPending());
         assertEquals(toSet(assigned), subscriptions.assignedPartitions());
         assertEquals(subscription, subscriptions.groupSubscription());
-        assertEquals(1, rebalanceListener.revokedCount);
-        assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked);
+        assertEquals(0, rebalanceListener.revokedCount);
+        assertNull(rebalanceListener.revoked);
         assertEquals(1, rebalanceListener.assignedCount);
         assertEquals(getAdded(owned, assigned), rebalanceListener.assigned);
     }
@@ -844,8 +893,8 @@ public void testPatternJoinGroupFollower() {
         assertFalse(coordinator.rejoinNeededOrPending());
         assertEquals(assigned.size(), subscriptions.numAssignedPartitions());
         assertEquals(subscription, subscriptions.subscription());
-        assertEquals(1, rebalanceListener.revokedCount);
-        assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked);
+        assertEquals(0, rebalanceListener.revokedCount);
+        assertNull(rebalanceListener.revoked);
         assertEquals(1, rebalanceListener.assignedCount);
         assertEquals(getAdded(owned, assigned), rebalanceListener.assigned);
     }
@@ -955,7 +1004,7 @@ public void testUnknownMemberIdOnSyncGroup() {
 
         // join initially, but let coordinator returns unknown member id
         client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE));
-        client.prepareResponse(syncGroupResponse(Collections.emptyList(), Errors.UNKNOWN_MEMBER_ID));
+        client.prepareResponse(syncGroupResponse(Collections.emptyList(), Errors.UNKNOWN_MEMBER_ID));
 
         // now we should see a new join with the empty UNKNOWN_MEMBER_ID
         client.prepareResponse(body -> {
@@ -981,7 +1030,7 @@ public void testRebalanceInProgressOnSyncGroup() {
 
         // join initially, but let coordinator rebalance on sync
         client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE));
-        client.prepareResponse(syncGroupResponse(Collections.emptyList(), Errors.REBALANCE_IN_PROGRESS));
+        client.prepareResponse(syncGroupResponse(Collections.emptyList(), Errors.REBALANCE_IN_PROGRESS));
 
         // then let the full join/sync finish successfully
         client.prepareResponse(joinGroupFollowerResponse(2, consumerId, "leader", Errors.NONE));
@@ -1004,7 +1053,7 @@ public void testIllegalGenerationOnSyncGroup() {
 
         // join initially, but let coordinator rebalance on sync
         client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE));
-        client.prepareResponse(syncGroupResponse(Collections.emptyList(), Errors.ILLEGAL_GENERATION));
+        client.prepareResponse(syncGroupResponse(Collections.emptyList(), Errors.ILLEGAL_GENERATION));
 
         // then let the full join/sync finish successfully
         client.prepareResponse(body -> {
@@ -1140,7 +1189,7 @@ public void onPartitionsAssigned(Collection partitions) {
         coordinator.poll(time.timer(Long.MAX_VALUE));
 
         assertFalse(coordinator.rejoinNeededOrPending());
-        assertEquals(1, rebalanceListener.revokedCount);
+        assertEquals(0, rebalanceListener.revokedCount);
         assertEquals(2, rebalanceListener.assignedCount);
     }
 
@@ -1180,6 +1229,7 @@ private void unavailableTopicTest(boolean patternSubscribe, Set unavaila
         client.prepareResponse(syncGroupResponse(Collections.emptyList(), Errors.NONE));
         coordinator.poll(time.timer(Long.MAX_VALUE));
         assertFalse(coordinator.rejoinNeededOrPending());
+        // callback not triggered since there's nothing to be assigned
         assertEquals(Collections.emptySet(), rebalanceListener.assigned);
         assertTrue("Metadata refresh not requested for unavailable partitions", metadata.updateRequested());
 
@@ -1242,8 +1292,8 @@ public void testRejoinGroup() {
         // join the group once
         joinAsFollowerAndReceiveAssignment("consumer", coordinator, assigned);
 
-        assertEquals(1, rebalanceListener.revokedCount);
-        assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked);
+        assertEquals(0, rebalanceListener.revokedCount);
+        assertNull(rebalanceListener.revoked);
         assertEquals(1, rebalanceListener.assignedCount);
         assertEquals(getAdded(owned, assigned), rebalanceListener.assigned);
 
@@ -1255,10 +1305,12 @@ public void testRejoinGroup() {
         client.prepareResponse(syncGroupResponse(assigned, Errors.NONE));
         coordinator.joinGroupIfNeeded(time.timer(Long.MAX_VALUE));
 
-        assertEquals(2, rebalanceListener.revokedCount);
-        assertEquals(getRevoked(assigned, assigned), rebalanceListener.revoked);
+        Collection revoked = getRevoked(assigned, assigned);
+        Collection added = getAdded(assigned, assigned);
+        assertEquals(revoked.isEmpty() ? 0 : 1, rebalanceListener.revokedCount);
+        assertEquals(revoked.isEmpty() ? null : revoked, rebalanceListener.revoked);
         assertEquals(2, rebalanceListener.assignedCount);
-        assertEquals(getAdded(assigned, assigned), rebalanceListener.assigned);
+        assertEquals(added, rebalanceListener.assigned);
     }
 
     @Test
@@ -1279,8 +1331,9 @@ public void testDisconnectInJoin() {
 
         assertFalse(coordinator.rejoinNeededOrPending());
         assertEquals(toSet(assigned), subscriptions.assignedPartitions());
-        assertEquals(1, rebalanceListener.revokedCount);
-        assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked);
+        // nothing to be revoked hence callback not triggered
+        assertEquals(0, rebalanceListener.revokedCount);
+        assertNull(rebalanceListener.revoked);
         assertEquals(1, rebalanceListener.assignedCount);
         assertEquals(getAdded(owned, assigned), rebalanceListener.assigned);
     }
@@ -1348,8 +1401,7 @@ private void testInFlightRequestsFailedAfterCoordinatorMarkedDead(Errors error)
     public void testAutoCommitDynamicAssignment() {
         final String consumerId = "consumer";
 
-        try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors,
-            true)
+        try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors, true)
         ) {
             subscriptions.subscribe(singleton(topic1), rebalanceListener);
             joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p));
@@ -1365,8 +1417,7 @@ public void testAutoCommitDynamicAssignment() {
     public void testAutoCommitRetryBackoff() {
         final String consumerId = "consumer";
 
-        try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors,
-            true)) {
+        try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors, true)) {
             subscriptions.subscribe(singleton(topic1), rebalanceListener);
             joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p));
 
@@ -1439,8 +1490,7 @@ public void testAutoCommitAwaitsInterval() {
     public void testAutoCommitDynamicAssignmentRebalance() {
         final String consumerId = "consumer";
 
-        try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors,
-            true)) {
+        try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors, true)) {
             subscriptions.subscribe(singleton(topic1), rebalanceListener);
             client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
             coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
@@ -1657,7 +1707,7 @@ public void testAsyncCommitCallbacksInvokedPriorToSyncCommitCompletion() throws
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
 
-        final List committedOffsets = Collections.synchronizedList(new ArrayList());
+        final List committedOffsets = Collections.synchronizedList(new ArrayList<>());
         final OffsetAndMetadata firstOffset = new OffsetAndMetadata(0L);
         final OffsetAndMetadata secondOffset = new OffsetAndMetadata(1L);
 
@@ -1929,7 +1979,7 @@ public void testRefreshOffsetWithNoFetchableOffsets() {
         assertEquals(Collections.singleton(t1p), subscriptions.missingFetchPositions());
         assertEquals(Collections.emptySet(), subscriptions.partitionsNeedingReset(time.milliseconds()));
         assertFalse(subscriptions.hasAllFetchPositions());
-        assertEquals(null, subscriptions.position(t1p));
+        assertNull(subscriptions.position(t1p));
     }
 
     @Test
@@ -1977,7 +2027,7 @@ public void testAuthenticationFailureInEnsureActiveGroup() {
     public void testThreadSafeAssignedPartitionsMetric() throws Exception {
         // Get the assigned-partitions metric
         final Metric metric = metrics.metric(new MetricName("assigned-partitions", "consumer" + groupId + "-coordinator-metrics",
-                "", Collections.emptyMap()));
+                "", Collections.emptyMap()));
 
         // Start polling the metric in the background
         final AtomicBoolean doStop = new AtomicBoolean();
@@ -2203,8 +2253,9 @@ private ConsumerCoordinator prepareCoordinatorForCloseTest(final boolean useGrou
             client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE));
             client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE));
             coordinator.joinGroupIfNeeded(time.timer(Long.MAX_VALUE));
-        } else
+        } else {
             subscriptions.assignFromUser(singleton(t1p));
+        }
 
         subscriptions.seek(t1p, 100);
         coordinator.poll(time.timer(Long.MAX_VALUE));
@@ -2277,6 +2328,11 @@ private void gracefulCloseTest(ConsumerCoordinator coordinator, boolean shouldLe
         coordinator.close();
         assertTrue("Commit not requested", commitRequested.get());
         assertEquals("leaveGroupRequested should be " + shouldLeaveGroup, shouldLeaveGroup, leaveGroupRequested.get());
+
+        if (shouldLeaveGroup) {
+            assertEquals(1, rebalanceListener.revokedCount);
+            assertEquals(singleton(t1p), rebalanceListener.revoked);
+        }
     }
 
     private ConsumerCoordinator buildCoordinator(final GroupRebalanceConfig rebalanceConfig,
@@ -2488,12 +2544,13 @@ public void onComplete(Map offsets, Exception
     }
 
     private static class MockRebalanceListener implements ConsumerRebalanceListener {
+        public Collection lost;
         public Collection revoked;
         public Collection assigned;
+        public int lostCount = 0;
         public int revokedCount = 0;
         public int assignedCount = 0;
 
-
         @Override
         public void onPartitionsAssigned(Collection partitions) {
             this.assigned = partitions;
@@ -2506,5 +2563,10 @@ public void onPartitionsRevoked(Collection partitions) {
             revokedCount++;
         }
 
+        @Override
+        public void onPartitionsLost(Collection partitions) {
+            this.lost = partitions;
+            lostCount++;
+        }
     }
 }
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java
index 484b9de0a9bce..c3ce02a79a155 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java
@@ -62,7 +62,7 @@ public void partitionAssignment() {
         state.seek(tp0, 1);
         assertTrue(state.isFetchable(tp0));
         assertEquals(1L, state.position(tp0).offset);
-        state.assignFromUser(Collections.emptySet());
+        state.assignFromUser(Collections.emptySet());
         assertTrue(state.assignedPartitions().isEmpty());
         assertEquals(0, state.numAssignedPartitions());
         assertFalse(state.isAssigned(tp0));
@@ -88,7 +88,8 @@ public void partitionAssignmentChangeOnTopicSubscription() {
         assertTrue(state.assignedPartitions().isEmpty());
         assertEquals(0, state.numAssignedPartitions());
 
-        assertTrue(state.assignFromSubscribed(singleton(t1p0)));
+        assertTrue(state.checkAssignmentMatchedSubscription(singleton(t1p0)));
+        state.assignFromSubscribed(singleton(t1p0));
         // assigned partitions should immediately change
         assertEquals(singleton(t1p0), state.assignedPartitions());
         assertEquals(1, state.numAssignedPartitions());
@@ -116,13 +117,17 @@ public void partitionAssignmentChangeOnPatternSubscription() {
         assertTrue(state.assignedPartitions().isEmpty());
         assertEquals(0, state.numAssignedPartitions());
 
-        assertTrue(state.assignFromSubscribed(singleton(tp1)));
+        assertTrue(state.checkAssignmentMatchedSubscription(singleton(tp1)));
+        state.assignFromSubscribed(singleton(tp1));
+
         // assigned partitions should immediately change
         assertEquals(singleton(tp1), state.assignedPartitions());
         assertEquals(1, state.numAssignedPartitions());
         assertEquals(singleton(topic), state.subscription());
 
-        assertTrue(state.assignFromSubscribed(Collections.singletonList(t1p0)));
+        assertTrue(state.checkAssignmentMatchedSubscription(singleton(t1p0)));
+        state.assignFromSubscribed(singleton(t1p0));
+
         // assigned partitions should immediately change
         assertEquals(singleton(t1p0), state.assignedPartitions());
         assertEquals(1, state.numAssignedPartitions());
@@ -138,7 +143,9 @@ public void partitionAssignmentChangeOnPatternSubscription() {
         assertEquals(singleton(t1p0), state.assignedPartitions());
         assertEquals(1, state.numAssignedPartitions());
 
-        assertTrue(state.assignFromSubscribed(Collections.singletonList(tp0)));
+        assertTrue(state.checkAssignmentMatchedSubscription(singleton(tp0)));
+        state.assignFromSubscribed(singleton(tp0));
+
         // assigned partitions should immediately change
         assertEquals(singleton(tp0), state.assignedPartitions());
         assertEquals(1, state.numAssignedPartitions());
@@ -164,7 +171,8 @@ public void verifyAssignmentId() {
 
         Set autoAssignment = Utils.mkSet(t1p0);
         state.subscribe(singleton(topic1), rebalanceListener);
-        assertTrue(state.assignFromSubscribed(autoAssignment));
+        assertTrue(state.checkAssignmentMatchedSubscription(autoAssignment));
+        state.assignFromSubscribed(autoAssignment);
         assertEquals(3, state.assignmentId());
         assertEquals(autoAssignment, state.assignedPartitions());
     }
@@ -192,10 +200,14 @@ public void topicSubscription() {
         assertTrue(state.assignedPartitions().isEmpty());
         assertEquals(0, state.numAssignedPartitions());
         assertTrue(state.partitionsAutoAssigned());
-        assertTrue(state.assignFromSubscribed(singleton(tp0)));
+        assertTrue(state.checkAssignmentMatchedSubscription(singleton(tp0)));
+        state.assignFromSubscribed(singleton(tp0));
+
         state.seek(tp0, 1);
         assertEquals(1L, state.position(tp0).offset);
-        assertTrue(state.assignFromSubscribed(singleton(tp1)));
+        assertTrue(state.checkAssignmentMatchedSubscription(singleton(tp1)));
+        state.assignFromSubscribed(singleton(tp1));
+
         assertTrue(state.isAssigned(tp1));
         assertFalse(state.isAssigned(tp0));
         assertFalse(state.isFetchable(tp1));
@@ -217,21 +229,23 @@ public void partitionPause() {
     @Test(expected = IllegalStateException.class)
     public void invalidPositionUpdate() {
         state.subscribe(singleton(topic), rebalanceListener);
-        assertTrue(state.assignFromSubscribed(singleton(tp0)));
+        assertTrue(state.checkAssignmentMatchedSubscription(singleton(tp0)));
+        state.assignFromSubscribed(singleton(tp0));
+
         state.position(tp0, new SubscriptionState.FetchPosition(0, Optional.empty(), leaderAndEpoch));
     }
 
     @Test
     public void cantAssignPartitionForUnsubscribedTopics() {
         state.subscribe(singleton(topic), rebalanceListener);
-        assertFalse(state.assignFromSubscribed(Collections.singletonList(t1p0)));
+        assertFalse(state.checkAssignmentMatchedSubscription(Collections.singletonList(t1p0)));
     }
 
     @Test
     public void cantAssignPartitionForUnmatchedPattern() {
         state.subscribe(Pattern.compile(".*t"), rebalanceListener);
         state.subscribeFromPattern(new HashSet<>(Collections.singletonList(topic)));
-        assertFalse(state.assignFromSubscribed(Collections.singletonList(t1p0)));
+        assertFalse(state.checkAssignmentMatchedSubscription(Collections.singletonList(t1p0)));
     }
 
     @Test(expected = IllegalStateException.class)
@@ -291,7 +305,9 @@ public void unsubscribeUserSubscribe() {
     public void unsubscription() {
         state.subscribe(Pattern.compile(".*"), rebalanceListener);
         state.subscribeFromPattern(new HashSet<>(Arrays.asList(topic, topic1)));
-        assertTrue(state.assignFromSubscribed(singleton(tp1)));
+        assertTrue(state.checkAssignmentMatchedSubscription(singleton(tp1)));
+        state.assignFromSubscribed(singleton(tp1));
+
         assertEquals(singleton(tp1), state.assignedPartitions());
         assertEquals(1, state.numAssignedPartitions());
 
diff --git a/core/src/main/scala/kafka/tools/MirrorMaker.scala b/core/src/main/scala/kafka/tools/MirrorMaker.scala
index b6cd60b201776..8a603c36e1b52 100755
--- a/core/src/main/scala/kafka/tools/MirrorMaker.scala
+++ b/core/src/main/scala/kafka/tools/MirrorMaker.scala
@@ -354,6 +354,8 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
                                           customRebalanceListener: Option[ConsumerRebalanceListener])
     extends ConsumerRebalanceListener {
 
+    override def onPartitionsLost(partitions: util.Collection[TopicPartition]) {}
+
     override def onPartitionsRevoked(partitions: util.Collection[TopicPartition]) {
       producer.flush()
       commitOffsets(consumerWrapper)
diff --git a/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala b/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala
index 210ceb2d4d006..45743b2c97713 100644
--- a/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala
@@ -61,6 +61,8 @@ abstract class AbstractConsumerTest extends BaseRequestTest {
   this.consumerConfig.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
   this.consumerConfig.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false")
   this.consumerConfig.setProperty(ConsumerConfig.METADATA_MAX_AGE_CONFIG, "100")
+  this.consumerConfig.setProperty(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, "6000")
+
 
   override protected def brokerPropertyOverrides(properties: Properties): Unit = {
     properties.setProperty(KafkaConfig.ControlledShutdownEnableProp, "false") // speed up shutdown
@@ -332,28 +334,28 @@ abstract class AbstractConsumerTest extends BaseRequestTest {
     @volatile var thrownException: Option[Throwable] = None
     @volatile var receivedMessages = 0
 
-    @volatile private var partitionAssignment: Set[TopicPartition] = partitionsToAssign
+    @volatile private var partitionAssignment: mutable.Set[TopicPartition] = new mutable.HashSet[TopicPartition]()
     @volatile private var subscriptionChanged = false
     private var topicsSubscription = topicsToSubscribe
 
     val rebalanceListener: ConsumerRebalanceListener = new ConsumerRebalanceListener {
       override def onPartitionsAssigned(partitions: util.Collection[TopicPartition]) = {
-        partitionAssignment = collection.immutable.Set(consumer.assignment().asScala.toArray: _*)
+        partitionAssignment ++= partitions.toArray(new Array[TopicPartition](0))
       }
 
       override def onPartitionsRevoked(partitions: util.Collection[TopicPartition]) = {
-        partitionAssignment = Set.empty[TopicPartition]
+        partitionAssignment --= partitions.toArray(new Array[TopicPartition](0))
       }
     }
 
-    if (partitionAssignment.isEmpty) {
+    if (partitionsToAssign.isEmpty) {
       consumer.subscribe(topicsToSubscribe.asJava, rebalanceListener)
     } else {
-      consumer.assign(partitionAssignment.asJava)
+      consumer.assign(partitionsToAssign.asJava)
     }
 
     def consumerAssignment(): Set[TopicPartition] = {
-      partitionAssignment
+      partitionAssignment.toSet
     }
 
     /**
diff --git a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
index 778e3b885ccac..634bcda16a7bb 100644
--- a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
+++ b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
@@ -246,7 +246,9 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
     killBroker(findCoordinator(manualGroup))
 
     val future1 = submitCloseAndValidate(consumer1, Long.MaxValue, None, gracefulCloseTimeMs)
+
     val future2 = submitCloseAndValidate(consumer2, Long.MaxValue, None, gracefulCloseTimeMs)
+
     future1.get
     future2.get
 
@@ -286,7 +288,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
 
     servers.foreach(server => killBroker(server.config.brokerId))
     val closeTimeout = 2000
-    val future1 = submitCloseAndValidate(consumer1, closeTimeout, Some(closeTimeout), Some(closeTimeout))
+    val future1 = submitCloseAndValidate(consumer1, closeTimeout, None, Some(closeTimeout))
     val future2 = submitCloseAndValidate(consumer2, Long.MaxValue, Some(requestTimeout), Some(requestTimeout))
     future1.get
     future2.get
@@ -384,13 +386,8 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
 
     def subscribeAndPoll(consumer: KafkaConsumer[Array[Byte], Array[Byte]], revokeSemaphore: Option[Semaphore] = None): Future[Any] = {
       executor.submit(CoreUtils.runnable {
-          consumer.subscribe(Collections.singletonList(topic), new ConsumerRebalanceListener {
-            def onPartitionsAssigned(partitions: Collection[TopicPartition]) {
-            }
-            def onPartitionsRevoked(partitions: Collection[TopicPartition]) {
-              revokeSemaphore.foreach(s => s.release())
-            }
-          })
+        consumer.subscribe(Collections.singletonList(topic))
+        revokeSemaphore.foreach(s => s.release())
         // requires to used deprecated `poll(long)` to trigger metadata update
           consumer.poll(0L)
         }, 0)
diff --git a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
index 33c14ebe12ff0..30b1d13aad81e 100644
--- a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
+++ b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
@@ -185,14 +185,14 @@ class PlaintextConsumerTest extends BaseConsumerTest {
     // rebalance to get the initial assignment
     awaitRebalance(consumer, listener)
     assertEquals(1, listener.callsToAssigned)
-    assertEquals(1, listener.callsToRevoked)
+    assertEquals(0, listener.callsToRevoked)
 
     Thread.sleep(3500)
 
     // we should fall out of the group and need to rebalance
     awaitRebalance(consumer, listener)
     assertEquals(2, listener.callsToAssigned)
-    assertEquals(2, listener.callsToRevoked)
+    assertEquals(1, listener.callsToRevoked)
   }
 
   @Test
@@ -207,8 +207,9 @@ class PlaintextConsumerTest extends BaseConsumerTest {
     var committedPosition: Long = -1
 
     val listener = new TestConsumerReassignmentListener {
+      override def onPartitionsLost(partitions: util.Collection[TopicPartition]): Unit = {}
       override def onPartitionsRevoked(partitions: util.Collection[TopicPartition]): Unit = {
-        if (callsToRevoked > 0) {
+        if (!partitions.isEmpty && partitions.contains(tp)) {
           // on the second rebalance (after we have joined the group initially), sleep longer
           // than session timeout and then try a commit. We should still be in the group,
           // so the commit should succeed
@@ -1641,6 +1642,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
 
     // stop polling and close one of the consumers, should trigger partition re-assignment among alive consumers
     timeoutPoller.shutdown()
+    consumerPollers -= timeoutPoller
     if (closeConsumer)
       timeoutConsumer.close()
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
index bbf9f9c509745..412d5e4166947 100644
--- a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
+++ b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
@@ -190,8 +190,11 @@ public class KafkaStreams implements AutoCloseable {
      * - Of special importance: If the global stream thread dies, or all stream threads die (or both) then
      *   the instance will be in the ERROR state. The user will need to close it.
      */
+    // TODO: the current transitions from other states directly to RUNNING is due to
+    //       the fact that onPartitionsRevoked may not be triggered. we need to refactor the
+    //       state diagram more thoroughly after we refactor StreamsPartitionAssignor to support COOPERATIVE
     public enum State {
-        CREATED(1, 3), REBALANCING(2, 3, 5), RUNNING(1, 3, 5), PENDING_SHUTDOWN(4), NOT_RUNNING, ERROR(3);
+        CREATED(1, 2, 3), REBALANCING(2, 3, 5), RUNNING(1, 2, 3, 5), PENDING_SHUTDOWN(4), NOT_RUNNING, ERROR(3);
 
         private final Set validTransitions = new HashSet<>();
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index d3efa9e0e05de..cd5fc83856f3a 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -127,7 +127,10 @@ public class StreamThread extends Thread {
      * 
      
      */
     public enum State implements ThreadStateTransitionValidator {
-        CREATED(1, 5), STARTING(2, 5), PARTITIONS_REVOKED(3, 5), PARTITIONS_ASSIGNED(2, 4, 5), RUNNING(2, 5), PENDING_SHUTDOWN(6), DEAD;
+        // TODO: the current transitions from other states directly to PARTITIONS_REVOKED is due to
+        //       the fact that onPartitionsRevoked may not be triggered. we need to refactor the
+        //       state diagram more thoroughly after we refactor StreamsPartitionAssignor to support COOPERATIVE
+        CREATED(1, 5), STARTING(2, 3, 5), PARTITIONS_REVOKED(3, 5), PARTITIONS_ASSIGNED(2, 3, 4, 5), RUNNING(2, 3, 5), PENDING_SHUTDOWN(6), DEAD;
 
         private final Set validTransitions = new HashSet<>();
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/StreamTableJoinIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/StreamTableJoinIntegrationTest.java
index 772c91d7b36af..4ddab57099d33 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/StreamTableJoinIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/StreamTableJoinIntegrationTest.java
@@ -79,11 +79,10 @@ public void testShouldAutoShutdownOnIncompleteMetadata() throws InterruptedExcep
         streams.setStreamThreadStateListener(listener);
         streams.start();
 
-        TestUtils.waitForCondition(listener::revokedToPendingShutdownSeen, "Did not seen thread state transited to PENDING_SHUTDOWN");
+        TestUtils.waitForCondition(listener::transitToPendingShutdownSeen, "Did not seen thread state transited to PENDING_SHUTDOWN");
 
         streams.close();
-        assertTrue(listener.createdToRevokedSeen());
-        assertTrue(listener.revokedToPendingShutdownSeen());
+        assertTrue(listener.transitToPendingShutdownSeen());
     }
 
     @Test
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java b/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java
index 46515aa84690c..a0a61b978e749 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java
@@ -79,25 +79,18 @@ public class IntegrationTestUtils {
      * Records state transition for StreamThread
      */
     public static class StateListenerStub implements StreamThread.StateListener {
-        boolean startingToRevokedSeen = false;
-        boolean revokedToPendingShutdownSeen = false;
+        boolean toPendingShutdownSeen = false;
         @Override
         public void onChange(final Thread thread,
                              final ThreadStateTransitionValidator newState,
                              final ThreadStateTransitionValidator oldState) {
-            if (oldState == StreamThread.State.STARTING && newState == StreamThread.State.PARTITIONS_REVOKED) {
-                startingToRevokedSeen = true;
-            } else if (oldState == StreamThread.State.PARTITIONS_REVOKED && newState == StreamThread.State.PENDING_SHUTDOWN) {
-                revokedToPendingShutdownSeen = true;
+            if (newState == StreamThread.State.PENDING_SHUTDOWN) {
+                toPendingShutdownSeen = true;
             }
         }
 
-        public boolean revokedToPendingShutdownSeen() {
-            return revokedToPendingShutdownSeen;
-        }
-
-        public boolean createdToRevokedSeen() {
-            return startingToRevokedSeen;
+        public boolean transitToPendingShutdownSeen() {
+            return toPendingShutdownSeen;
         }
     }
 

From 5e543ae10832808a9d7efef363603f07695d4368 Mon Sep 17 00:00:00 2001
From: Mickael Maison 
Date: Fri, 9 Aug 2019 12:49:38 +0530
Subject: [PATCH 0514/1071] KAFKA-8598: Use automatic RPC generation in
 RenewDelegationToken

Author: Mickael Maison 

Reviewers: Manikumar Reddy , Viktor Somogyi 

Closes #7038 from mimaison/KAFKA-8598
---
 .../kafka/clients/admin/KafkaAdminClient.java |  8 +-
 .../apache/kafka/common/protocol/ApiKeys.java |  6 +-
 .../common/requests/AbstractResponse.java     |  2 +-
 .../requests/RenewDelegationTokenRequest.java | 83 +++++--------------
 .../RenewDelegationTokenResponse.java         | 70 ++++------------
 .../common/requests/RequestResponseTest.java  | 14 +++-
 .../main/scala/kafka/server/KafkaApis.scala   | 11 ++-
 .../unit/kafka/server/RequestQuotaTest.scala  |  8 +-
 8 files changed, 75 insertions(+), 127 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
index 5256e368891ca..7bcad4100436f 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
@@ -82,6 +82,7 @@
 import org.apache.kafka.common.message.ListGroupsRequestData;
 import org.apache.kafka.common.message.ListGroupsResponseData;
 import org.apache.kafka.common.message.MetadataRequestData;
+import org.apache.kafka.common.message.RenewDelegationTokenRequestData;
 import org.apache.kafka.common.metrics.JmxReporter;
 import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
@@ -2443,8 +2444,11 @@ public RenewDelegationTokenResult renewDelegationToken(final byte[] hmac, final
             new LeastLoadedNodeProvider()) {
 
             @Override
-            AbstractRequest.Builder createRequest(int timeoutMs) {
-                return new RenewDelegationTokenRequest.Builder(hmac, options.renewTimePeriodMs());
+            AbstractRequest.Builder createRequest(int timeoutMs) {
+                return new RenewDelegationTokenRequest.Builder(
+                        new RenewDelegationTokenRequestData()
+                        .setHmac(hmac)
+                        .setRenewPeriodMs(options.renewTimePeriodMs()));
             }
 
             @Override
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
index e05f692305f75..912e26bc99b3b 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
@@ -56,6 +56,8 @@
 import org.apache.kafka.common.message.OffsetCommitResponseData;
 import org.apache.kafka.common.message.OffsetFetchRequestData;
 import org.apache.kafka.common.message.OffsetFetchResponseData;
+import org.apache.kafka.common.message.RenewDelegationTokenRequestData;
+import org.apache.kafka.common.message.RenewDelegationTokenResponseData;
 import org.apache.kafka.common.message.SaslAuthenticateRequestData;
 import org.apache.kafka.common.message.SaslAuthenticateResponseData;
 import org.apache.kafka.common.message.SaslHandshakeRequestData;
@@ -105,8 +107,6 @@
 import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse;
 import org.apache.kafka.common.requests.ProduceRequest;
 import org.apache.kafka.common.requests.ProduceResponse;
-import org.apache.kafka.common.requests.RenewDelegationTokenRequest;
-import org.apache.kafka.common.requests.RenewDelegationTokenResponse;
 import org.apache.kafka.common.requests.StopReplicaRequest;
 import org.apache.kafka.common.requests.StopReplicaResponse;
 import org.apache.kafka.common.requests.TxnOffsetCommitRequest;
@@ -190,7 +190,7 @@ public Struct parseResponse(short version, ByteBuffer buffer) {
     CREATE_PARTITIONS(37, "CreatePartitions", CreatePartitionsRequest.schemaVersions(),
             CreatePartitionsResponse.schemaVersions()),
     CREATE_DELEGATION_TOKEN(38, "CreateDelegationToken", CreateDelegationTokenRequestData.SCHEMAS, CreateDelegationTokenResponseData.SCHEMAS),
-    RENEW_DELEGATION_TOKEN(39, "RenewDelegationToken", RenewDelegationTokenRequest.schemaVersions(), RenewDelegationTokenResponse.schemaVersions()),
+    RENEW_DELEGATION_TOKEN(39, "RenewDelegationToken", RenewDelegationTokenRequestData.SCHEMAS, RenewDelegationTokenResponseData.SCHEMAS),
     EXPIRE_DELEGATION_TOKEN(40, "ExpireDelegationToken", ExpireDelegationTokenRequestData.SCHEMAS, ExpireDelegationTokenResponseData.SCHEMAS),
     DESCRIBE_DELEGATION_TOKEN(41, "DescribeDelegationToken", DescribeDelegationTokenRequest.schemaVersions(), DescribeDelegationTokenResponse.schemaVersions()),
     DELETE_GROUPS(42, "DeleteGroups", DeleteGroupsRequestData.SCHEMAS, DeleteGroupsResponseData.SCHEMAS),
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
index da2b837054660..13d18a2d72401 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
@@ -149,7 +149,7 @@ public static AbstractResponse parseResponse(ApiKeys apiKey, Struct struct, shor
             case CREATE_DELEGATION_TOKEN:
                 return new CreateDelegationTokenResponse(struct, version);
             case RENEW_DELEGATION_TOKEN:
-                return new RenewDelegationTokenResponse(struct);
+                return new RenewDelegationTokenResponse(struct, version);
             case EXPIRE_DELEGATION_TOKEN:
                 return new ExpireDelegationTokenResponse(struct, version);
             case DESCRIBE_DELEGATION_TOKEN:
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenRequest.java
index d73561abbb88e..2c83f06adda62 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenRequest.java
@@ -16,102 +16,65 @@
  */
 package org.apache.kafka.common.requests;
 
+import java.nio.ByteBuffer;
+
+import org.apache.kafka.common.message.RenewDelegationTokenRequestData;
+import org.apache.kafka.common.message.RenewDelegationTokenResponseData;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
 
-import java.nio.ByteBuffer;
-
-import static org.apache.kafka.common.protocol.types.Type.BYTES;
-import static org.apache.kafka.common.protocol.types.Type.INT64;
-
 public class RenewDelegationTokenRequest extends AbstractRequest {
 
-    private static final String HMAC_KEY_NAME = "hmac";
-    private static final String RENEW_TIME_PERIOD_KEY_NAME = "renew_time_period";
-    private final ByteBuffer hmac;
-    private final long renewTimePeriod;
+    private final RenewDelegationTokenRequestData data;
 
-    public static final Schema TOKEN_RENEW_REQUEST_V0 = new Schema(
-        new Field(HMAC_KEY_NAME, BYTES, "HMAC of the delegation token to be renewed."),
-        new Field(RENEW_TIME_PERIOD_KEY_NAME, INT64, "Renew time period in milli seconds."));
-
-    /**
-     * The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
-     */
-    public static final Schema TOKEN_RENEW_REQUEST_V1 = TOKEN_RENEW_REQUEST_V0;
-
-    private RenewDelegationTokenRequest(short version, ByteBuffer hmac, long renewTimePeriod) {
+    public RenewDelegationTokenRequest(RenewDelegationTokenRequestData data, short version) {
         super(ApiKeys.RENEW_DELEGATION_TOKEN, version);
-
-        this.hmac = hmac;
-        this.renewTimePeriod = renewTimePeriod;
+        this.data = data;
     }
 
-    public RenewDelegationTokenRequest(Struct struct, short versionId) {
-        super(ApiKeys.RENEW_DELEGATION_TOKEN, versionId);
-
-        hmac = struct.getBytes(HMAC_KEY_NAME);
-        renewTimePeriod = struct.getLong(RENEW_TIME_PERIOD_KEY_NAME);
+    public RenewDelegationTokenRequest(Struct struct, short version) {
+        super(ApiKeys.RENEW_DELEGATION_TOKEN, version);
+        this.data = new RenewDelegationTokenRequestData(struct, version);
     }
 
     public static RenewDelegationTokenRequest parse(ByteBuffer buffer, short version) {
         return new RenewDelegationTokenRequest(ApiKeys.RENEW_DELEGATION_TOKEN.parseRequest(version, buffer), version);
     }
 
-    public static Schema[] schemaVersions() {
-        return new Schema[] {TOKEN_RENEW_REQUEST_V0, TOKEN_RENEW_REQUEST_V1};
-    }
-
     @Override
     protected Struct toStruct() {
-        short version = version();
-        Struct struct = new Struct(ApiKeys.RENEW_DELEGATION_TOKEN.requestSchema(version));
-
-        struct.set(HMAC_KEY_NAME, hmac);
-        struct.set(RENEW_TIME_PERIOD_KEY_NAME, renewTimePeriod);
+        return data.toStruct(version());
+    }
 
-        return struct;
+    public RenewDelegationTokenRequestData data() {
+        return data;
     }
 
     @Override
     public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
-        return new RenewDelegationTokenResponse(throttleTimeMs, Errors.forException(e));
-    }
-
-    public ByteBuffer hmac() {
-        return hmac;
-    }
-
-    public long renewTimePeriod() {
-        return renewTimePeriod;
+        return new RenewDelegationTokenResponse(
+                new RenewDelegationTokenResponseData()
+                    .setThrottleTimeMs(throttleTimeMs)
+                    .setErrorCode(Errors.forException(e).code()));
     }
 
     public static class Builder extends AbstractRequest.Builder {
-        private final ByteBuffer hmac;
-        private final long renewTimePeriod;
+        private final RenewDelegationTokenRequestData data;
 
-        public Builder(byte[] hmac, long renewTimePeriod) {
+        public Builder(RenewDelegationTokenRequestData data) {
             super(ApiKeys.RENEW_DELEGATION_TOKEN);
-            this.hmac = ByteBuffer.wrap(hmac);
-            this.renewTimePeriod = renewTimePeriod;
+            this.data = data;
         }
 
         @Override
         public RenewDelegationTokenRequest build(short version) {
-            return new RenewDelegationTokenRequest(version, hmac, renewTimePeriod);
+            return new RenewDelegationTokenRequest(data, version);
         }
 
         @Override
         public String toString() {
-            StringBuilder bld = new StringBuilder();
-            bld.append("(type: RenewDelegationTokenRequest").
-                append(", hmac=").append(hmac).
-                append(", renewTimePeriod=").append(renewTimePeriod).
-                append(")");
-            return bld.toString();
+            return data.toString();
         }
     }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java
index dc961e176df3a..35cd6152632b6 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java
@@ -16,92 +16,56 @@
  */
 package org.apache.kafka.common.requests;
 
-import org.apache.kafka.common.protocol.ApiKeys;
-import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
-import org.apache.kafka.common.protocol.types.Struct;
-
 import java.nio.ByteBuffer;
+import java.util.Collections;
 import java.util.Map;
 
-import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
-import static org.apache.kafka.common.protocol.CommonFields.THROTTLE_TIME_MS;
-import static org.apache.kafka.common.protocol.types.Type.INT64;
+import org.apache.kafka.common.message.RenewDelegationTokenResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.protocol.types.Struct;
 
 public class RenewDelegationTokenResponse extends AbstractResponse {
 
-    private static final String EXPIRY_TIMESTAMP_KEY_NAME = "expiry_timestamp";
-
-    private final Errors error;
-    private final long expiryTimestamp;
-    private final int throttleTimeMs;
+    private final RenewDelegationTokenResponseData data;
 
-    private static final Schema TOKEN_RENEW_RESPONSE_V0 = new Schema(
-            ERROR_CODE,
-            new Field(EXPIRY_TIMESTAMP_KEY_NAME, INT64, "timestamp (in msec) at which this token expires.."),
-            THROTTLE_TIME_MS);
-
-    /**
-     * The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
-     */
-    private static final Schema TOKEN_RENEW_RESPONSE_V1 = TOKEN_RENEW_RESPONSE_V0;
-
-    public RenewDelegationTokenResponse(int throttleTimeMs, Errors error, long expiryTimestamp) {
-        this.throttleTimeMs = throttleTimeMs;
-        this.error = error;
-        this.expiryTimestamp = expiryTimestamp;
-    }
-
-    public RenewDelegationTokenResponse(int throttleTimeMs, Errors error) {
-        this(throttleTimeMs, error, -1);
+    public RenewDelegationTokenResponse(RenewDelegationTokenResponseData data) {
+        this.data = data;
     }
 
-    public RenewDelegationTokenResponse(Struct struct) {
-        error = Errors.forCode(struct.get(ERROR_CODE));
-        expiryTimestamp = struct.getLong(EXPIRY_TIMESTAMP_KEY_NAME);
-        this.throttleTimeMs = struct.getOrElse(THROTTLE_TIME_MS, DEFAULT_THROTTLE_TIME);
+    public RenewDelegationTokenResponse(Struct struct, short version) {
+        data = new RenewDelegationTokenResponseData(struct, version);
     }
 
     public static RenewDelegationTokenResponse parse(ByteBuffer buffer, short version) {
-        return new RenewDelegationTokenResponse(ApiKeys.RENEW_DELEGATION_TOKEN.responseSchema(version).read(buffer));
-    }
-
-    public static Schema[] schemaVersions() {
-        return new Schema[] {TOKEN_RENEW_RESPONSE_V0, TOKEN_RENEW_RESPONSE_V1};
+        return new RenewDelegationTokenResponse(ApiKeys.RENEW_DELEGATION_TOKEN.responseSchema(version).read(buffer), version);
     }
 
     @Override
     public Map errorCounts() {
-        return errorCounts(error);
+        return Collections.singletonMap(error(), 1);
     }
 
     @Override
     protected Struct toStruct(short version) {
-        Struct struct = new Struct(ApiKeys.RENEW_DELEGATION_TOKEN.responseSchema(version));
-
-        struct.set(ERROR_CODE, error.code());
-        struct.set(EXPIRY_TIMESTAMP_KEY_NAME, expiryTimestamp);
-        struct.setIfExists(THROTTLE_TIME_MS, throttleTimeMs);
-
-        return struct;
+        return data.toStruct(version);
     }
 
     @Override
     public int throttleTimeMs() {
-        return throttleTimeMs;
+        return data.throttleTimeMs();
     }
 
     public Errors error() {
-        return error;
+        return Errors.forCode(data.errorCode());
     }
 
     public long expiryTimestamp() {
-        return expiryTimestamp;
+        return data.expiryTimestampMs();
     }
 
     public boolean hasError() {
-        return this.error != Errors.NONE;
+        return error() != Errors.NONE;
     }
 
     @Override
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index 0b8d98d73a10f..a979d4cdee545 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -82,6 +82,8 @@
 import org.apache.kafka.common.message.ListGroupsResponseData;
 import org.apache.kafka.common.message.OffsetCommitRequestData;
 import org.apache.kafka.common.message.OffsetCommitResponseData;
+import org.apache.kafka.common.message.RenewDelegationTokenRequestData;
+import org.apache.kafka.common.message.RenewDelegationTokenResponseData;
 import org.apache.kafka.common.message.SaslAuthenticateRequestData;
 import org.apache.kafka.common.message.SaslAuthenticateResponseData;
 import org.apache.kafka.common.message.SaslHandshakeRequestData;
@@ -1020,7 +1022,6 @@ private MetadataResponse createMetadataResponse() {
         return MetadataResponse.prepareResponse(asList(node), null, MetadataResponse.NO_CONTROLLER_ID, allTopicMetadata);
     }
 
-    @SuppressWarnings("deprecation")
     private OffsetCommitRequest createOffsetCommitRequest(int version) {
         return new OffsetCommitRequest.Builder(new OffsetCommitRequestData()
                 .setGroupId("group1")
@@ -1545,11 +1546,18 @@ private CreateDelegationTokenResponse createCreateTokenResponse() {
     }
 
     private RenewDelegationTokenRequest createRenewTokenRequest() {
-        return new RenewDelegationTokenRequest.Builder("test".getBytes(), System.currentTimeMillis()).build();
+        RenewDelegationTokenRequestData data = new RenewDelegationTokenRequestData()
+                .setHmac("test".getBytes())
+                .setRenewPeriodMs(System.currentTimeMillis());
+        return new RenewDelegationTokenRequest.Builder(data).build();
     }
 
     private RenewDelegationTokenResponse createRenewTokenResponse() {
-        return new RenewDelegationTokenResponse(20, Errors.NONE, System.currentTimeMillis());
+        RenewDelegationTokenResponseData data = new RenewDelegationTokenResponseData()
+                .setThrottleTimeMs(20)
+                .setErrorCode(Errors.NONE.code())
+                .setExpiryTimestampMs(System.currentTimeMillis());
+        return new RenewDelegationTokenResponse(data);
     }
 
     private ExpireDelegationTokenRequest createExpireTokenRequest() {
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 2b4998240683b..b71e4b0876c1b 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -69,6 +69,7 @@ import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse
 import org.apache.kafka.common.message.ListGroupsResponseData
 import org.apache.kafka.common.message.OffsetCommitRequestData
 import org.apache.kafka.common.message.OffsetCommitResponseData
+import org.apache.kafka.common.message.RenewDelegationTokenResponseData
 import org.apache.kafka.common.message.SaslAuthenticateResponseData
 import org.apache.kafka.common.message.SaslHandshakeResponseData
 import org.apache.kafka.common.message.SyncGroupResponseData
@@ -2462,7 +2463,11 @@ class KafkaApis(val requestChannel: RequestChannel,
       trace("Sending renew token response %s for correlation id %d to client %s."
         .format(request.header.correlationId, request.header.clientId))
       sendResponseMaybeThrottle(request, requestThrottleMs =>
-        new RenewDelegationTokenResponse(requestThrottleMs, error, expiryTimestamp))
+        new RenewDelegationTokenResponse(
+             new RenewDelegationTokenResponseData()
+               .setThrottleTimeMs(requestThrottleMs)
+               .setErrorCode(error.code)
+               .setExpiryTimestampMs(expiryTimestamp)))
     }
 
     if (!allowTokenRequests(request))
@@ -2470,8 +2475,8 @@ class KafkaApis(val requestChannel: RequestChannel,
     else {
       tokenManager.renewToken(
         request.session.principal,
-        renewTokenRequest.hmac,
-        renewTokenRequest.renewTimePeriod(),
+        ByteBuffer.wrap(renewTokenRequest.data.hmac),
+        renewTokenRequest.data.renewPeriodMs,
         sendResponseCallback
       )
     }
diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
index db7ab699712a7..d5c5d7faebb2d 100644
--- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
+++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
@@ -45,6 +45,7 @@ import org.apache.kafka.common.message.ListGroupsRequestData
 import org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData
 import org.apache.kafka.common.message.ListPartitionReassignmentsRequestData
 import org.apache.kafka.common.message.OffsetCommitRequestData
+import org.apache.kafka.common.message.RenewDelegationTokenRequestData
 import org.apache.kafka.common.message.SaslAuthenticateRequestData
 import org.apache.kafka.common.message.SaslHandshakeRequestData
 import org.apache.kafka.common.message.SyncGroupRequestData
@@ -454,7 +455,10 @@ class RequestQuotaTest extends BaseRequestTest {
           new DescribeDelegationTokenRequest.Builder(Collections.singletonList(SecurityUtils.parseKafkaPrincipal("User:test")))
 
         case ApiKeys.RENEW_DELEGATION_TOKEN =>
-          new RenewDelegationTokenRequest.Builder("".getBytes, 1000)
+          new RenewDelegationTokenRequest.Builder(
+              new RenewDelegationTokenRequestData()
+                .setHmac("".getBytes)
+                .setRenewPeriodMs(1000L))
 
         case ApiKeys.DELETE_GROUPS =>
           new DeleteGroupsRequest.Builder(new DeleteGroupsRequestData()
@@ -577,8 +581,8 @@ class RequestQuotaTest extends BaseRequestTest {
       case ApiKeys.CREATE_PARTITIONS => new CreatePartitionsResponse(response).throttleTimeMs
       case ApiKeys.CREATE_DELEGATION_TOKEN => new CreateDelegationTokenResponse(response, ApiKeys.CREATE_DELEGATION_TOKEN.latestVersion).throttleTimeMs
       case ApiKeys.DESCRIBE_DELEGATION_TOKEN=> new DescribeDelegationTokenResponse(response).throttleTimeMs
+      case ApiKeys.RENEW_DELEGATION_TOKEN => new RenewDelegationTokenResponse(response, ApiKeys.RENEW_DELEGATION_TOKEN.latestVersion).throttleTimeMs
       case ApiKeys.EXPIRE_DELEGATION_TOKEN => new ExpireDelegationTokenResponse(response, ApiKeys.EXPIRE_DELEGATION_TOKEN.latestVersion).throttleTimeMs
-      case ApiKeys.RENEW_DELEGATION_TOKEN => new RenewDelegationTokenResponse(response).throttleTimeMs
       case ApiKeys.DELETE_GROUPS => new DeleteGroupsResponse(response).throttleTimeMs
       case ApiKeys.OFFSET_FOR_LEADER_EPOCH => new OffsetsForLeaderEpochResponse(response).throttleTimeMs
       case ApiKeys.ELECT_LEADERS => new ElectLeadersResponse(response).throttleTimeMs

From 600cc48fa56ca675421eaf383929e89a04b529f1 Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Fri, 9 Aug 2019 06:16:32 -0700
Subject: [PATCH 0515/1071] KAFKA-8748: Fix flaky testDescribeLogDirsRequest
 (#7182)

The introduction of KIP-480: Sticky Producer Partitioner had the
side effect that generateAndProduceMessages can often write
messages to a lower number of partitions to improve batching.

testDescribeLogDirsRequest (and potentially other tests) relies
on the messages being written somewhat uniformly to the topic
partitions. We fix the issue by including a monotonically
increasing key in the produced messages.

I also included a couple of minor clean-ups I noticed while
debugging the issue.

The test failed very frequently when executed locally before the
change and it passed 100 times consecutively after the change.

Reviewers: Manikumar Reddy 
---
 .../unit/kafka/server/DescribeLogDirsRequestTest.scala     | 3 ++-
 core/src/test/scala/unit/kafka/utils/JaasTestUtils.scala   | 5 +----
 core/src/test/scala/unit/kafka/utils/TestUtils.scala       | 7 +++++--
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/core/src/test/scala/unit/kafka/server/DescribeLogDirsRequestTest.scala b/core/src/test/scala/unit/kafka/server/DescribeLogDirsRequestTest.scala
index a1e4c733efb87..5e2707a646625 100644
--- a/core/src/test/scala/unit/kafka/server/DescribeLogDirsRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/DescribeLogDirsRequestTest.scala
@@ -57,7 +57,8 @@ class DescribeLogDirsRequestTest extends BaseRequestTest {
     val log1 = servers.head.logManager.getLog(tp1).get
     assertEquals(log0.size, replicaInfo0.size)
     assertEquals(log1.size, replicaInfo1.size)
-    assertTrue(servers.head.logManager.getLog(tp0).get.logEndOffset > 0)
+    val logEndOffset = servers.head.logManager.getLog(tp0).get.logEndOffset
+    assertTrue(s"LogEndOffset '$logEndOffset' should be > 0", logEndOffset > 0)
     assertEquals(servers.head.replicaManager.getLogEndOffsetLag(tp0, log0.logEndOffset, false), replicaInfo0.offsetLag)
     assertEquals(servers.head.replicaManager.getLogEndOffsetLag(tp1, log1.logEndOffset, false), replicaInfo1.offsetLag)
   }
diff --git a/core/src/test/scala/unit/kafka/utils/JaasTestUtils.scala b/core/src/test/scala/unit/kafka/utils/JaasTestUtils.scala
index fb7b07e7baff9..65ff4bae89ec4 100644
--- a/core/src/test/scala/unit/kafka/utils/JaasTestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/JaasTestUtils.scala
@@ -155,10 +155,7 @@ object JaasTestUtils {
   val serviceName = "kafka"
 
   def saslConfigs(saslProperties: Option[Properties]): Properties = {
-    val result = saslProperties match {
-      case Some(properties) => properties
-      case None => new Properties
-    }
+    val result = saslProperties.getOrElse(new Properties)
     // IBM Kerberos module doesn't support the serviceName JAAS property, hence it needs to be
     // passed as a Kafka property
     if (Java.isIbmJdk && !result.contains(KafkaConfig.SaslKerberosServiceNameProp))
diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index ffaf47b84c1bb..960ab4c03a038 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -53,7 +53,7 @@ import org.apache.kafka.common.internals.Topic
 import org.apache.kafka.common.network.{ListenerName, Mode}
 import org.apache.kafka.common.record._
 import org.apache.kafka.common.security.auth.SecurityProtocol
-import org.apache.kafka.common.serialization.{ByteArrayDeserializer, ByteArraySerializer, Deserializer, Serializer}
+import org.apache.kafka.common.serialization.{ByteArrayDeserializer, ByteArraySerializer, Deserializer, IntegerSerializer, Serializer}
 import org.apache.kafka.common.utils.Time
 import org.apache.kafka.common.utils.Utils._
 import org.apache.kafka.test.{TestSslUtils, TestUtils => JTestUtils}
@@ -1065,7 +1065,10 @@ object TestUtils extends Logging {
                                  numMessages: Int,
                                  acks: Int = -1): Seq[String] = {
     val values = (0 until numMessages).map(x =>  s"test-$x")
-    val records = values.map(v => new ProducerRecord[Array[Byte], Array[Byte]](topic, v.getBytes))
+    val intSerializer = new IntegerSerializer()
+    val records = values.zipWithIndex.map { case (v, i) =>
+      new ProducerRecord(topic, intSerializer.serialize(topic, i), v.getBytes)
+    }
     produceMessages(servers, records, acks)
     values
   }

From 3f0064974f5040da5bedd07f3eb382ac63424537 Mon Sep 17 00:00:00 2001
From: Stanislav Kozlovski 
Date: Fri, 9 Aug 2019 16:05:25 +0100
Subject: [PATCH 0516/1071] MINOR: Ignore dynamic log4j log level tests (#7183)

We recently introduced a bunch of flaky tests in the AdminClientIntegrationTest.
These tests are failing very frequently. We should ignore the tests in order to
make the build stable until we have a fix.

Reviewers: Ismael Juma 
---
 .../integration/kafka/api/AdminClientIntegrationTest.scala | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index ff8e379ef5bcb..b7a2c6ab9e920 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -48,7 +48,7 @@ import org.apache.kafka.common.resource.{PatternType, ResourcePattern, ResourceT
 import org.apache.kafka.common.utils.{Time, Utils}
 import org.junit.Assert._
 import org.junit.rules.Timeout
-import org.junit.{After, Before, Rule, Test}
+import org.junit.{After, Before, Ignore, Rule, Test}
 import org.scalatest.Assertions.intercept
 
 import scala.collection.JavaConverters._
@@ -1842,6 +1842,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
   }
 
   @Test
+  @Ignore // To be re-enabled once KAFKA-8779 is resolved
   def testIncrementalAlterConfigsForLog4jLogLevels(): Unit = {
     client = AdminClient.create(createConfig())
 
@@ -1905,6 +1906,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     * 5. Ensure the kafka.controller.KafkaController logger's level is ERROR (the curent root logger level)
     */
   @Test
+  @Ignore // To be re-enabled once KAFKA-8779 is resolved
   def testIncrementalAlterConfigsForLog4jLogLevelsCanResetLoggerToCurrentRoot(): Unit = {
     client = AdminClient.create(createConfig())
     // step 1 - configure root logger
@@ -1946,6 +1948,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
   }
 
   @Test
+  @Ignore // To be re-enabled once KAFKA-8779 is resolved
   def testIncrementalAlterConfigsForLog4jLogLevelsCannotResetRootLogger(): Unit = {
     client = AdminClient.create(createConfig())
     val deleteRootLoggerEntry = Seq(
@@ -1956,6 +1959,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
   }
 
   @Test
+  @Ignore // To be re-enabled once KAFKA-8779 is resolved
   def testIncrementalAlterConfigsForLog4jLogLevelsDoesNotWorkWithInvalidConfigs(): Unit = {
     client = AdminClient.create(createConfig())
     val validLoggerName = "kafka.server.KafkaRequestHandler"
@@ -2000,6 +2004,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     * The AlterConfigs API is deprecated and should not support altering log levels
     */
   @Test
+  @Ignore // To be re-enabled once KAFKA-8779 is resolved
   def testAlterConfigsForLog4jLogLevelsDoesNotWork(): Unit = {
     client = AdminClient.create(createConfig())
 

From e9a35fe02effe10dcbec2daed4ef66c40d35716c Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Fri, 9 Aug 2019 14:33:20 -0700
Subject: [PATCH 0517/1071] MINOR: Bump system test version from 2.2.0 to 2.2.1
 (#6873)

Reviewers: Boyang Chen , Bill Bejeck 
---
 gradle/dependencies.gradle | 2 +-
 tests/docker/Dockerfile    | 4 ++--
 tests/kafkatest/version.py | 3 ++-
 vagrant/base.sh            | 6 ++----
 4 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle
index f3d0e31d37dbb..e75526aaa9dc6 100644
--- a/gradle/dependencies.gradle
+++ b/gradle/dependencies.gradle
@@ -93,7 +93,7 @@ versions += [
   kafka_11: "1.1.1",
   kafka_20: "2.0.1",
   kafka_21: "2.1.1",
-  kafka_22: "2.2.0",
+  kafka_22: "2.2.1",
   lz4: "1.6.0",
   mavenArtifact: "3.6.1",
   metrics: "2.2.0",
diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile
index 72991ea497da2..27a1362e02a38 100644
--- a/tests/docker/Dockerfile
+++ b/tests/docker/Dockerfile
@@ -55,7 +55,7 @@ RUN mkdir -p "/opt/kafka-1.0.2" && chmod a+rw /opt/kafka-1.0.2 && curl -s "$KAFK
 RUN mkdir -p "/opt/kafka-1.1.1" && chmod a+rw /opt/kafka-1.1.1 && curl -s "$KAFKA_MIRROR/kafka_2.11-1.1.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-1.1.1"
 RUN mkdir -p "/opt/kafka-2.0.1" && chmod a+rw /opt/kafka-2.0.1 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.0.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.0.1"
 RUN mkdir -p "/opt/kafka-2.1.1" && chmod a+rw /opt/kafka-2.1.1 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.1.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.1.1"
-RUN mkdir -p "/opt/kafka-2.2.0" && chmod a+rw /opt/kafka-2.2.0 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.2.0.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.2.0"
+RUN mkdir -p "/opt/kafka-2.2.1" && chmod a+rw /opt/kafka-2.2.1 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.2.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.2.1"
 RUN mkdir -p "/opt/kafka-2.3.0" && chmod a+rw /opt/kafka-2.3.0 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.3.0.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.3.0"
 
 # Streams test dependencies
@@ -67,7 +67,7 @@ RUN curl -s "$KAFKA_MIRROR/kafka-streams-1.0.2-test.jar" -o /opt/kafka-1.0.2/lib
 RUN curl -s "$KAFKA_MIRROR/kafka-streams-1.1.1-test.jar" -o /opt/kafka-1.1.1/libs/kafka-streams-1.1.1-test.jar
 RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.0.1-test.jar" -o /opt/kafka-2.0.1/libs/kafka-streams-2.0.1-test.jar
 RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.1.1-test.jar" -o /opt/kafka-2.1.1/libs/kafka-streams-2.1.1-test.jar
-RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.2.0-test.jar" -o /opt/kafka-2.2.0/libs/kafka-streams-2.2.0-test.jar
+RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.2.1-test.jar" -o /opt/kafka-2.2.1/libs/kafka-streams-2.2.1-test.jar
 
 # The version of Kibosh to use for testing.
 # If you update this, also update vagrant/base.sy
diff --git a/tests/kafkatest/version.py b/tests/kafkatest/version.py
index 6623548f29be4..573e2ed4de853 100644
--- a/tests/kafkatest/version.py
+++ b/tests/kafkatest/version.py
@@ -124,7 +124,8 @@ def get_version(node=None):
 
 # 2.2.x versions
 V_2_2_0 = KafkaVersion("2.2.0")
-LATEST_2_2 = V_2_2_0
+V_2_2_1 = KafkaVersion("2.2.1")
+LATEST_2_2 = V_2_2_1
 
 # 2.3.x versions
 V_2_3_0 = KafkaVersion("2.3.0")
diff --git a/vagrant/base.sh b/vagrant/base.sh
index 758aaf987c4e9..2445e12cfd352 100755
--- a/vagrant/base.sh
+++ b/vagrant/base.sh
@@ -129,12 +129,10 @@ get_kafka 1.1.1 2.11
 chmod a+rw /opt/kafka-1.1.1
 get_kafka 2.0.1 2.12
 chmod a+rw /opt/kafka-2.0.1
-get_kafka 2.1.0 2.12
-chmod a+rw /opt/kafka-2.1.0
 get_kafka 2.1.1 2.12
 chmod a+rw /opt/kafka-2.1.1
-get_kafka 2.2.0 2.12
-chmod a+rw /opt/kafka-2.2.0
+get_kafka 2.2.1 2.12
+chmod a+rw /opt/kafka-2.2.1
 get_kafka 2.3.0 2.12
 chmod a+rw /opt/kafka-2.3.0
 

From 7511ccf68dc41a78bdef8f34e57d94b600666232 Mon Sep 17 00:00:00 2001
From: Chris Egerton 
Date: Sun, 11 Aug 2019 07:56:05 -0700
Subject: [PATCH 0518/1071] KAFKA-8550: Fix plugin loading of aliased
 converters in Connect (#6959)

Connector validation fails if an alias is used for the converter since the validation for that is done via `ConfigDef.validateAll(...)`, which in turn invokes `Class.forName(...)` on the alias. Even though the class is successfully loaded by the DelegatingClassLoader, some Java implementations will refuse to return a class from `Class.forName(...)` whose name differs from the argument provided.

This commit alters `ConfigDef.parseType(...)` to first invoke `ClassLoader.loadClass(...)` on the class using our class loader in order to get a handle on the actual class object to be loaded, then invoke `Class.forName(...)` with the fully-qualified class name of the to-be-loaded class and return the result. The invocation of `Class.forName(...)` is necessary in order to allow static initialization to take place; simply calling `ClassLoader.loadClass(...)` is insufficient.

Also corrected a unit test that relied upon the old behavior.

Author: Chris Egerton 
Reviewers: Robert Yokota , Randall Hauch 
---
 .../apache/kafka/common/config/ConfigDef.java | 13 ++++++++---
 .../common/config/AbstractConfigTest.java     |  2 +-
 .../kafka/common/config/ConfigDefTest.java    | 23 +++++++++++++++++++
 3 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java b/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java
index 362b1a6c1cb81..fa99931783e00 100644
--- a/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java
+++ b/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java
@@ -707,9 +707,16 @@ else if (value instanceof String)
                 case CLASS:
                     if (value instanceof Class)
                         return value;
-                    else if (value instanceof String)
-                        return Class.forName(trimmed, true, Utils.getContextOrKafkaClassLoader());
-                    else
+                    else if (value instanceof String) {
+                        ClassLoader contextOrKafkaClassLoader = Utils.getContextOrKafkaClassLoader();
+                        // Use loadClass here instead of Class.forName because the name we use here may be an alias
+                        // and not match the name of the class that gets loaded. If that happens, Class.forName can
+                        // throw an exception.
+                        Class klass = contextOrKafkaClassLoader.loadClass(trimmed);
+                        // Invoke forName here with the true name of the requested class to cause class
+                        // initialization to take place.
+                        return Class.forName(klass.getName(), true, contextOrKafkaClassLoader);
+                    } else
                         throw new ConfigException(name, value, "Expected a Class instance or class name.");
                 default:
                     throw new IllegalStateException("Unknown type.");
diff --git a/clients/src/test/java/org/apache/kafka/common/config/AbstractConfigTest.java b/clients/src/test/java/org/apache/kafka/common/config/AbstractConfigTest.java
index f6864754ca79b..b5fff6f78d74d 100644
--- a/clients/src/test/java/org/apache/kafka/common/config/AbstractConfigTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/config/AbstractConfigTest.java
@@ -265,7 +265,7 @@ public RestrictedClassLoader() {
             @Override
             protected Class findClass(String name) throws ClassNotFoundException {
                 if (name.equals(ClassTestConfig.DEFAULT_CLASS.getName()) || name.equals(ClassTestConfig.RESTRICTED_CLASS.getName()))
-                    return null;
+                    throw new ClassNotFoundException();
                 else
                     return ClassTestConfig.class.getClassLoader().loadClass(name);
             }
diff --git a/clients/src/test/java/org/apache/kafka/common/config/ConfigDefTest.java b/clients/src/test/java/org/apache/kafka/common/config/ConfigDefTest.java
index 974d39f934d24..701e9f44919bd 100644
--- a/clients/src/test/java/org/apache/kafka/common/config/ConfigDefTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/config/ConfigDefTest.java
@@ -639,6 +639,29 @@ public void testConvertValueToStringNestedClass() throws ClassNotFoundException
         assertEquals(NestedClass.class, Class.forName(actual));
     }
 
+    @Test
+    public void testClassWithAlias() {
+        final String alias = "PluginAlias";
+        ClassLoader originalClassLoader = Thread.currentThread().getContextClassLoader();
+        try {
+            // Could try to use the Plugins class from Connect here, but this should simulate enough
+            // of the aliasing logic to suffice for this test.
+            Thread.currentThread().setContextClassLoader(new ClassLoader(originalClassLoader) {
+                @Override
+                public Class loadClass(String name, boolean resolve) throws ClassNotFoundException {
+                    if (alias.equals(name)) {
+                        return NestedClass.class;
+                    } else {
+                        return super.loadClass(name, resolve);
+                    }
+                }
+            });
+            ConfigDef.parseType("Test config", alias, Type.CLASS);
+        } finally {
+            Thread.currentThread().setContextClassLoader(originalClassLoader);
+        }
+    }
+
     private class NestedClass {
     }
 }

From 35ed7b14cc8f943b4fbc9945a93eecbff7b72ab2 Mon Sep 17 00:00:00 2001
From: Lucas Bradstreet 
Date: Sun, 11 Aug 2019 11:44:30 -0700
Subject: [PATCH 0519/1071] MINOR: Eliminate unnecessary map operations in
 RecordAccumulator.isMuted (#7193)

Avoids calling both `containsKey` and `get` from isMuted, when only a single get is necessary.
Also avoid calling `remove` unless necessary. This could be a reduction of map operations
from 3 to 1.

isMuted showed up as a hotspot in profiling when using the producer with high numbers of partitions.

Reviewers: Ismael Juma 
---
 .../producer/internals/RecordAccumulator.java      | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
index 4f5aa3e4726e4..4d272f34ca4af 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
@@ -275,10 +275,18 @@ private RecordAppendResult tryAppend(long timestamp, byte[] key, byte[] value, H
     }
 
     private boolean isMuted(TopicPartition tp, long now) {
-        boolean result = muted.containsKey(tp) && muted.get(tp) > now;
-        if (!result)
+        // Take care to avoid unnecessary map look-ups because this method is a hotspot if producing to a
+        // large number of partitions
+        Long throttleUntilTime = muted.get(tp);
+        if (throttleUntilTime == null)
+            return false;
+
+        if (now >= throttleUntilTime) {
             muted.remove(tp);
-        return result;
+            return false;
+        }
+        
+        return true;
     }
 
     public void resetNextBatchExpiryTime() {

From 558bb1d0692b3ba6cf9c46d7170be6d31656024b Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Sun, 11 Aug 2019 15:31:42 -0700
Subject: [PATCH 0520/1071] KAFKA-8782: Close metrics in QuotaManagerTests
 (#7191)

Since `Metrics` was constructed with `enableExpiration=false`, this was
not a source of flakiness given the current implementation. This could
change in the future, so good to follow the class contract.

Included a few clean-ups with regards to redundant casts and type parameters
as well as usage of try with resources for inline usage of `Metrics`.

Reviewers: Manikumar Reddy 
---
 .../apache/kafka/common/metrics/Metrics.java  |  6 ++--
 .../kafka/common/metrics/MetricsTest.java     |  7 ++--
 .../kafka/common/metrics/SensorTest.java      | 36 +++++++++----------
 .../common/metrics/stats/FrequenciesTest.java |  3 +-
 .../kafka/server/ClientQuotaManagerTest.scala | 28 ++++++---------
 .../server/ReplicationQuotaManagerTest.scala  | 17 ++++-----
 6 files changed, 43 insertions(+), 54 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/metrics/Metrics.java b/clients/src/main/java/org/apache/kafka/common/metrics/Metrics.java
index 91c245dd5a186..e512e8cca3a02 100644
--- a/clients/src/main/java/org/apache/kafka/common/metrics/Metrics.java
+++ b/clients/src/main/java/org/apache/kafka/common/metrics/Metrics.java
@@ -90,7 +90,7 @@ public Metrics() {
      * Expiration of Sensors is disabled.
      */
     public Metrics(Time time) {
-        this(new MetricConfig(), new ArrayList(0), time);
+        this(new MetricConfig(), new ArrayList<>(0), time);
     }
 
     /**
@@ -98,7 +98,7 @@ public Metrics(Time time) {
      * Expiration of Sensors is disabled.
      */
     public Metrics(MetricConfig defaultConfig, Time time) {
-        this(defaultConfig, new ArrayList(0), time);
+        this(defaultConfig, new ArrayList<>(0), time);
     }
 
 
@@ -108,7 +108,7 @@ public Metrics(MetricConfig defaultConfig, Time time) {
      * @param defaultConfig The default config to use for all metrics that don't override their config
      */
     public Metrics(MetricConfig defaultConfig) {
-        this(defaultConfig, new ArrayList(0), Time.SYSTEM);
+        this(defaultConfig, new ArrayList<>(0), Time.SYSTEM);
     }
 
     /**
diff --git a/clients/src/test/java/org/apache/kafka/common/metrics/MetricsTest.java b/clients/src/test/java/org/apache/kafka/common/metrics/MetricsTest.java
index 0b1ea85323543..035704578de6a 100644
--- a/clients/src/test/java/org/apache/kafka/common/metrics/MetricsTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/metrics/MetricsTest.java
@@ -63,7 +63,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-@SuppressWarnings("deprecation")
 public class MetricsTest {
     private static final Logger log = LoggerFactory.getLogger(MetricsTest.class);
 
@@ -75,7 +74,7 @@ public class MetricsTest {
 
     @Before
     public void setup() {
-        this.metrics = new Metrics(config, Arrays.asList((MetricsReporter) new JmxReporter()), time, true);
+        this.metrics = new Metrics(config, Arrays.asList(new JmxReporter()), time, true);
     }
 
     @After
@@ -631,7 +630,7 @@ public void testMetricInstances() {
         Map childTagsWithValues = new HashMap<>();
         childTagsWithValues.put("child-tag", "child-tag-value");
 
-        try (Metrics inherited = new Metrics(new MetricConfig().tags(parentTagsWithValues), Arrays.asList((MetricsReporter) new JmxReporter()), time, true)) {
+        try (Metrics inherited = new Metrics(new MetricConfig().tags(parentTagsWithValues), Arrays.asList(new JmxReporter()), time, true)) {
             MetricName inheritedMetric = inherited.metricInstance(SampleMetrics.METRIC_WITH_INHERITED_TAGS, childTagsWithValues);
 
             Map filledOutTags = inheritedMetric.tags();
@@ -731,7 +730,7 @@ synchronized void processMetrics() {
 
         final LockingReporter reporter = new LockingReporter();
         this.metrics.close();
-        this.metrics = new Metrics(config, Arrays.asList((MetricsReporter) reporter), new MockTime(10), true);
+        this.metrics = new Metrics(config, Arrays.asList(reporter), new MockTime(10), true);
         final Deque sensors = new ConcurrentLinkedDeque<>();
         SensorCreator sensorCreator = new SensorCreator(metrics);
 
diff --git a/clients/src/test/java/org/apache/kafka/common/metrics/SensorTest.java b/clients/src/test/java/org/apache/kafka/common/metrics/SensorTest.java
index fc7cfe24def9a..d1f3a91ce35ae 100644
--- a/clients/src/test/java/org/apache/kafka/common/metrics/SensorTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/metrics/SensorTest.java
@@ -86,25 +86,23 @@ public void testShouldRecord() {
     public void testExpiredSensor() {
         MetricConfig config = new MetricConfig();
         Time mockTime = new MockTime();
-        Metrics metrics =  new Metrics(config, Arrays.asList((MetricsReporter) new JmxReporter()), mockTime, true);
-
-        long inactiveSensorExpirationTimeSeconds = 60L;
-        Sensor sensor = new Sensor(metrics, "sensor", null, config, mockTime,
-            inactiveSensorExpirationTimeSeconds, Sensor.RecordingLevel.INFO);
-
-        assertTrue(sensor.add(metrics.metricName("test1", "grp1"), new Avg()));
-
-        Map emptyTags = Collections.emptyMap();
-        MetricName rateMetricName = new MetricName("rate", "test", "", emptyTags);
-        MetricName totalMetricName = new MetricName("total", "test", "", emptyTags);
-        Meter meter = new Meter(rateMetricName, totalMetricName);
-        assertTrue(sensor.add(meter));
-
-        mockTime.sleep(TimeUnit.SECONDS.toMillis(inactiveSensorExpirationTimeSeconds + 1));
-        assertFalse(sensor.add(metrics.metricName("test3", "grp1"), new Avg()));
-        assertFalse(sensor.add(meter));
-
-        metrics.close();
+        try (Metrics metrics = new Metrics(config, Arrays.asList(new JmxReporter()), mockTime, true)) {
+            long inactiveSensorExpirationTimeSeconds = 60L;
+            Sensor sensor = new Sensor(metrics, "sensor", null, config, mockTime,
+                    inactiveSensorExpirationTimeSeconds, Sensor.RecordingLevel.INFO);
+
+            assertTrue(sensor.add(metrics.metricName("test1", "grp1"), new Avg()));
+
+            Map emptyTags = Collections.emptyMap();
+            MetricName rateMetricName = new MetricName("rate", "test", "", emptyTags);
+            MetricName totalMetricName = new MetricName("total", "test", "", emptyTags);
+            Meter meter = new Meter(rateMetricName, totalMetricName);
+            assertTrue(sensor.add(meter));
+
+            mockTime.sleep(TimeUnit.SECONDS.toMillis(inactiveSensorExpirationTimeSeconds + 1));
+            assertFalse(sensor.add(metrics.metricName("test3", "grp1"), new Avg()));
+            assertFalse(sensor.add(meter));
+        }
     }
 
     @Test
diff --git a/clients/src/test/java/org/apache/kafka/common/metrics/stats/FrequenciesTest.java b/clients/src/test/java/org/apache/kafka/common/metrics/stats/FrequenciesTest.java
index e3e623fae078e..bf7647a9afb40 100644
--- a/clients/src/test/java/org/apache/kafka/common/metrics/stats/FrequenciesTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/metrics/stats/FrequenciesTest.java
@@ -22,7 +22,6 @@
 import org.apache.kafka.common.metrics.JmxReporter;
 import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
-import org.apache.kafka.common.metrics.MetricsReporter;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.utils.MockTime;
 import org.apache.kafka.common.utils.Time;
@@ -46,7 +45,7 @@ public class FrequenciesTest {
     public void setup() {
         config = new MetricConfig().eventWindow(50).samples(2);
         time = new MockTime();
-        metrics = new Metrics(config, Arrays.asList((MetricsReporter) new JmxReporter()), time, true);
+        metrics = new Metrics(config, Arrays.asList(new JmxReporter()), time, true);
     }
 
     @After
diff --git a/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala
index e10d4b2d7c847..d52a8a6b18faa 100644
--- a/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala
@@ -33,14 +33,20 @@ import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
 import org.apache.kafka.common.utils.{MockTime, Sanitizer}
 import org.easymock.EasyMock
 import org.junit.Assert.{assertEquals, assertTrue}
-import org.junit.{Before, Test}
+import org.junit.{After, Test}
 
 class ClientQuotaManagerTest {
   private val time = new MockTime
-
+  private val metrics = new Metrics(new MetricConfig(), Collections.emptyList(), time)
   private val config = ClientQuotaManagerConfig(quotaBytesPerSecondDefault = 500)
 
   var numCallbacks: Int = 0
+
+  @After
+  def tearDown(): Unit = {
+    metrics.close()
+  }
+
   def callback (response: RequestChannel.Response) {
     // Count how many times this callback is called for notifyThrottlingDone().
     response match {
@@ -49,11 +55,6 @@ class ClientQuotaManagerTest {
     }
   }
 
-  @Before
-  def beforeMethod() {
-    numCallbacks = 0
-  }
-
   private def buildRequest[T <: AbstractRequest](builder: AbstractRequest.Builder[T],
                                                  listenerName: ListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)): (T, RequestChannel.Request) = {
 
@@ -81,7 +82,7 @@ class ClientQuotaManagerTest {
   }
 
   private def testQuotaParsing(config: ClientQuotaManagerConfig, client1: UserClient, client2: UserClient, randomClient: UserClient, defaultConfigClient: UserClient) {
-    val clientMetrics = new ClientQuotaManager(config, newMetrics, Produce, time, "")
+    val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, "")
 
     try {
       // Case 1: Update the quota. Assert that the new quota value is returned
@@ -193,7 +194,7 @@ class ClientQuotaManagerTest {
   @Test
   def testQuotaConfigPrecedence() {
     val quotaManager = new ClientQuotaManager(ClientQuotaManagerConfig(quotaBytesPerSecondDefault=Long.MaxValue),
-        newMetrics, Produce, time, "")
+      metrics, Produce, time, "")
 
     def checkQuota(user: String, clientId: String, expectedBound: Int, value: Int, expectThrottle: Boolean) {
       assertEquals(expectedBound, quotaManager.quota(user, clientId).bound, 0.0)
@@ -265,7 +266,6 @@ class ClientQuotaManagerTest {
 
   @Test
   def testQuotaViolation() {
-    val metrics = newMetrics
     val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, "")
     val queueSizeMetric = metrics.metrics().get(metrics.metricName("queue-size", "Produce", ""))
     try {
@@ -313,7 +313,6 @@ class ClientQuotaManagerTest {
 
   @Test
   def testRequestPercentageQuotaViolation() {
-    val metrics = newMetrics
     val quotaManager = new ClientRequestQuotaManager(config, metrics, time, "", None)
     quotaManager.updateQuota(Some("ANONYMOUS"), Some("test-client"), Some("test-client"), Some(Quota.upperBound(1)))
     val queueSizeMetric = metrics.metrics().get(metrics.metricName("queue-size", "Request", ""))
@@ -376,7 +375,6 @@ class ClientQuotaManagerTest {
 
   @Test
   def testExpireThrottleTimeSensor() {
-    val metrics = newMetrics
     val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, "")
     try {
       maybeRecord(clientMetrics, "ANONYMOUS", "client1", 100)
@@ -396,7 +394,6 @@ class ClientQuotaManagerTest {
 
   @Test
   def testExpireQuotaSensors() {
-    val metrics = newMetrics
     val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, "")
     try {
       maybeRecord(clientMetrics, "ANONYMOUS", "client1", 100)
@@ -420,7 +417,6 @@ class ClientQuotaManagerTest {
 
   @Test
   def testClientIdNotSanitized() {
-    val metrics = newMetrics
     val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, "")
     val clientId = "client@#$%"
     try {
@@ -437,10 +433,6 @@ class ClientQuotaManagerTest {
     }
   }
 
-  def newMetrics: Metrics = {
-    new Metrics(new MetricConfig(), Collections.emptyList(), time)
-  }
-
   private case class UserClient(val user: String, val clientId: String, val configUser: Option[String] = None, val configClientId: Option[String] = None) {
     // The class under test expects only sanitized client configs. We pass both the default value (which should not be
     // sanitized to ensure it remains unique) and non-default values, so we need to take care in generating the sanitized
diff --git a/core/src/test/scala/unit/kafka/server/ReplicationQuotaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicationQuotaManagerTest.scala
index 284d9ab048963..da9c5340cbd33 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicationQuotaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicationQuotaManagerTest.scala
@@ -23,16 +23,22 @@ import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.metrics.{MetricConfig, Metrics, Quota}
 import org.apache.kafka.common.utils.MockTime
 import org.junit.Assert.{assertEquals, assertFalse, assertTrue}
-import org.junit.Test
+import org.junit.{After, Test}
 
 import scala.collection.JavaConverters._
 
 class ReplicationQuotaManagerTest {
   private val time = new MockTime
+  private val metrics = new Metrics(new MetricConfig(), Collections.emptyList(), time)
+
+  @After
+  def tearDown: Unit = {
+    metrics.close()
+  }
 
   @Test
   def shouldThrottleOnlyDefinedReplicas(): Unit = {
-    val quota = new ReplicationQuotaManager(ReplicationQuotaManagerConfig(), newMetrics, QuotaType.Fetch, time)
+    val quota = new ReplicationQuotaManager(ReplicationQuotaManagerConfig(), metrics, QuotaType.Fetch, time)
     quota.markThrottled("topic1", Seq(1, 2, 3))
 
     assertTrue(quota.isThrottled(tp1(1)))
@@ -43,7 +49,6 @@ class ReplicationQuotaManagerTest {
 
   @Test
   def shouldExceedQuotaThenReturnBackBelowBoundAsTimePasses(): Unit = {
-    val metrics = newMetrics()
     val quota = new ReplicationQuotaManager(ReplicationQuotaManagerConfig(numQuotaSamples = 10, quotaWindowSizeSeconds = 1), metrics, LeaderReplication, time)
 
     //Given
@@ -105,7 +110,7 @@ class ReplicationQuotaManagerTest {
 
   @Test
   def shouldSupportWildcardThrottledReplicas(): Unit = {
-    val quota = new ReplicationQuotaManager(ReplicationQuotaManagerConfig(), newMetrics, LeaderReplication, time)
+    val quota = new ReplicationQuotaManager(ReplicationQuotaManagerConfig(), metrics, LeaderReplication, time)
 
     //When
     quota.markThrottled("MyTopic")
@@ -116,8 +121,4 @@ class ReplicationQuotaManagerTest {
   }
 
   private def tp1(id: Int): TopicPartition = new TopicPartition("topic1", id)
-
-  private def newMetrics(): Metrics = {
-    new Metrics(new MetricConfig(), Collections.emptyList(), time)
-  }
 }

From 4148d79880948a928a211c88d7280c748780c8aa Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Sun, 11 Aug 2019 20:42:24 -0700
Subject: [PATCH 0521/1071] MINOR: Remove Utils.notNull, use
 Objects.requireNonNull instead (#7194)

---
 .../producer/internals/RecordAccumulator.java      |  5 +++--
 .../kafka/clients/producer/internals/Sender.java   |  8 ++++----
 .../java/org/apache/kafka/common/MetricName.java   | 11 +++++------
 .../apache/kafka/common/MetricNameTemplate.java    | 10 ++++------
 .../apache/kafka/common/TopicPartitionReplica.java |  5 ++---
 .../org/apache/kafka/common/metrics/Metrics.java   | 12 ++++++------
 .../org/apache/kafka/common/metrics/Sensor.java    | 10 +++++-----
 .../java/org/apache/kafka/common/utils/Utils.java  | 14 --------------
 .../processor/internals/InternalTopicManager.java  |  3 ++-
 9 files changed, 31 insertions(+), 47 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
index 4d272f34ca4af..fb335e4229612 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
@@ -25,6 +25,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -54,7 +55,6 @@
 import org.apache.kafka.common.utils.CopyOnWriteMap;
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.common.utils.Time;
-import org.apache.kafka.common.utils.Utils;
 import org.slf4j.Logger;
 
 /**
@@ -229,7 +229,8 @@ public RecordAppendResult append(TopicPartition tp,
 
                 MemoryRecordsBuilder recordsBuilder = recordsBuilder(buffer, maxUsableMagic);
                 ProducerBatch batch = new ProducerBatch(tp, recordsBuilder, time.milliseconds());
-                FutureRecordMetadata future = Utils.notNull(batch.tryAppend(timestamp, key, value, headers, callback, time.milliseconds()));
+                FutureRecordMetadata future = Objects.requireNonNull(batch.tryAppend(timestamp, key, value, headers,
+                        callback, time.milliseconds()));
 
                 dq.addLast(batch);
                 incomplete.add(batch);
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java
index dfa6424afe863..4cd743d3d77ce 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java
@@ -54,7 +54,6 @@
 import org.apache.kafka.common.requests.RequestHeader;
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.common.utils.Time;
-import org.apache.kafka.common.utils.Utils;
 import org.slf4j.Logger;
 
 import java.io.IOException;
@@ -64,6 +63,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import static org.apache.kafka.common.record.RecordBatch.NO_TIMESTAMP;
 
@@ -923,17 +923,17 @@ public void updateProduceRequestMetrics(Map> batche
 
                     // per-topic record send rate
                     String topicRecordsCountName = "topic." + topic + ".records-per-batch";
-                    Sensor topicRecordCount = Utils.notNull(this.metrics.getSensor(topicRecordsCountName));
+                    Sensor topicRecordCount = Objects.requireNonNull(this.metrics.getSensor(topicRecordsCountName));
                     topicRecordCount.record(batch.recordCount);
 
                     // per-topic bytes send rate
                     String topicByteRateName = "topic." + topic + ".bytes";
-                    Sensor topicByteRate = Utils.notNull(this.metrics.getSensor(topicByteRateName));
+                    Sensor topicByteRate = Objects.requireNonNull(this.metrics.getSensor(topicByteRateName));
                     topicByteRate.record(batch.estimatedSizeInBytes());
 
                     // per-topic compression rate
                     String topicCompressionRateName = "topic." + topic + ".compression-rate";
-                    Sensor topicCompressionRate = Utils.notNull(this.metrics.getSensor(topicCompressionRateName));
+                    Sensor topicCompressionRate = Objects.requireNonNull(this.metrics.getSensor(topicCompressionRateName));
                     topicCompressionRate.record(batch.compressionRatio());
 
                     // global metrics
diff --git a/clients/src/main/java/org/apache/kafka/common/MetricName.java b/clients/src/main/java/org/apache/kafka/common/MetricName.java
index 0af77a038b4b4..9d695b5cf5efd 100644
--- a/clients/src/main/java/org/apache/kafka/common/MetricName.java
+++ b/clients/src/main/java/org/apache/kafka/common/MetricName.java
@@ -17,8 +17,7 @@
 package org.apache.kafka.common;
 
 import java.util.Map;
-
-import org.apache.kafka.common.utils.Utils;
+import java.util.Objects;
 
 /**
  * The MetricName class encapsulates a metric's name, logical group and its related attributes. It should be constructed using metrics.MetricName(...).
@@ -78,10 +77,10 @@ public final class MetricName {
      * @param tags        additional key/value attributes of the metric
      */
     public MetricName(String name, String group, String description, Map tags) {
-        this.name = Utils.notNull(name);
-        this.group = Utils.notNull(group);
-        this.description = Utils.notNull(description);
-        this.tags = Utils.notNull(tags);
+        this.name = Objects.requireNonNull(name);
+        this.group = Objects.requireNonNull(group);
+        this.description = Objects.requireNonNull(description);
+        this.tags = Objects.requireNonNull(tags);
     }
 
     public String name() {
diff --git a/clients/src/main/java/org/apache/kafka/common/MetricNameTemplate.java b/clients/src/main/java/org/apache/kafka/common/MetricNameTemplate.java
index abe121f7ac972..f9c4ef5f730e6 100644
--- a/clients/src/main/java/org/apache/kafka/common/MetricNameTemplate.java
+++ b/clients/src/main/java/org/apache/kafka/common/MetricNameTemplate.java
@@ -21,8 +21,6 @@
 import java.util.Objects;
 import java.util.Set;
 
-import org.apache.kafka.common.utils.Utils;
-
 /**
  * A template for a MetricName. It contains a name, group, and description, as
  * well as all the tags that will be used to create the mBean name. Tag values
@@ -46,10 +44,10 @@ public class MetricNameTemplate {
      * @param tagsNames the set of metric tag names, which can/should be a set that maintains order; may not be null
      */
     public MetricNameTemplate(String name, String group, String description, Set tagsNames) {
-        this.name = Utils.notNull(name);
-        this.group = Utils.notNull(group);
-        this.description = Utils.notNull(description);
-        this.tags = new LinkedHashSet<>(Utils.notNull(tagsNames));
+        this.name = Objects.requireNonNull(name);
+        this.group = Objects.requireNonNull(group);
+        this.description = Objects.requireNonNull(description);
+        this.tags = new LinkedHashSet<>(Objects.requireNonNull(tagsNames));
     }
 
     /**
diff --git a/clients/src/main/java/org/apache/kafka/common/TopicPartitionReplica.java b/clients/src/main/java/org/apache/kafka/common/TopicPartitionReplica.java
index 244260295a108..0a7c419c933bd 100644
--- a/clients/src/main/java/org/apache/kafka/common/TopicPartitionReplica.java
+++ b/clients/src/main/java/org/apache/kafka/common/TopicPartitionReplica.java
@@ -16,9 +16,8 @@
  */
 package org.apache.kafka.common;
 
-import org.apache.kafka.common.utils.Utils;
-
 import java.io.Serializable;
+import java.util.Objects;
 
 
 /**
@@ -32,7 +31,7 @@ public final class TopicPartitionReplica implements Serializable {
     private final String topic;
 
     public TopicPartitionReplica(String topic, int partition, int brokerId) {
-        this.topic = Utils.notNull(topic);
+        this.topic = Objects.requireNonNull(topic);
         this.partition = partition;
         this.brokerId = brokerId;
     }
diff --git a/clients/src/main/java/org/apache/kafka/common/metrics/Metrics.java b/clients/src/main/java/org/apache/kafka/common/metrics/Metrics.java
index e512e8cca3a02..d9b4b034f0f7d 100644
--- a/clients/src/main/java/org/apache/kafka/common/metrics/Metrics.java
+++ b/clients/src/main/java/org/apache/kafka/common/metrics/Metrics.java
@@ -20,7 +20,6 @@
 import org.apache.kafka.common.MetricNameTemplate;
 import org.apache.kafka.common.utils.KafkaThread;
 import org.apache.kafka.common.utils.Time;
-import org.apache.kafka.common.utils.Utils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -33,6 +32,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Objects;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
@@ -134,7 +134,7 @@ public Metrics(MetricConfig defaultConfig, List reporters, Time
         this.sensors = new ConcurrentHashMap<>();
         this.metrics = new ConcurrentHashMap<>();
         this.childrenSensors = new ConcurrentHashMap<>();
-        this.reporters = Utils.notNull(reporters);
+        this.reporters = Objects.requireNonNull(reporters);
         this.time = time;
         for (MetricsReporter reporter : reporters)
             reporter.init(new ArrayList());
@@ -313,7 +313,7 @@ public MetricConfig config() {
      * @return Return the sensor or null if no such sensor exists
      */
     public Sensor getSensor(String name) {
-        return this.sensors.get(Utils.notNull(name));
+        return this.sensors.get(Objects.requireNonNull(name));
     }
 
     /**
@@ -500,8 +500,8 @@ public void addMetric(MetricName metricName, MetricConfig config, Measurable mea
      */
     public void addMetric(MetricName metricName, MetricConfig config, MetricValueProvider metricValueProvider) {
         KafkaMetric m = new KafkaMetric(new Object(),
-                                        Utils.notNull(metricName),
-                                        Utils.notNull(metricValueProvider),
+                                        Objects.requireNonNull(metricName),
+                                        Objects.requireNonNull(metricValueProvider),
                                         config == null ? this.config : config,
                                         time);
         registerMetric(m);
@@ -545,7 +545,7 @@ public synchronized KafkaMetric removeMetric(MetricName metricName) {
      * Add a MetricReporter
      */
     public synchronized void addReporter(MetricsReporter reporter) {
-        Utils.notNull(reporter).init(new ArrayList<>(metrics.values()));
+        Objects.requireNonNull(reporter).init(new ArrayList<>(metrics.values()));
         this.reporters.add(reporter);
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/metrics/Sensor.java b/clients/src/main/java/org/apache/kafka/common/metrics/Sensor.java
index 1af9419bc757a..7d52d01f20640 100644
--- a/clients/src/main/java/org/apache/kafka/common/metrics/Sensor.java
+++ b/clients/src/main/java/org/apache/kafka/common/metrics/Sensor.java
@@ -19,7 +19,6 @@
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.metrics.CompoundStat.NamedMeasurable;
 import org.apache.kafka.common.utils.Time;
-import org.apache.kafka.common.utils.Utils;
 
 import java.util.ArrayList;
 import java.util.HashSet;
@@ -28,6 +27,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
@@ -107,7 +107,7 @@ public boolean shouldRecord(final int configId) {
            long inactiveSensorExpirationTimeSeconds, RecordingLevel recordingLevel) {
         super();
         this.registry = registry;
-        this.name = Utils.notNull(name);
+        this.name = Objects.requireNonNull(name);
         this.parents = parents == null ? new Sensor[0] : parents;
         this.metrics = new LinkedHashMap<>();
         this.stats = new ArrayList<>();
@@ -238,7 +238,7 @@ public synchronized boolean add(CompoundStat stat, MetricConfig config) {
         if (hasExpired())
             return false;
 
-        this.stats.add(Utils.notNull(stat));
+        this.stats.add(Objects.requireNonNull(stat));
         Object lock = metricLock();
         for (NamedMeasurable m : stat.stats()) {
             final KafkaMetric metric = new KafkaMetric(lock, m.name(), m.stat(), config == null ? this.config : config, time);
@@ -276,8 +276,8 @@ public synchronized boolean add(final MetricName metricName, final MeasurableSta
         } else {
             final KafkaMetric metric = new KafkaMetric(
                 metricLock(),
-                Utils.notNull(metricName),
-                Utils.notNull(stat),
+                Objects.requireNonNull(metricName),
+                Objects.requireNonNull(stat),
                 config == null ? this.config : config,
                 time
             );
diff --git a/clients/src/main/java/org/apache/kafka/common/utils/Utils.java b/clients/src/main/java/org/apache/kafka/common/utils/Utils.java
index dbaf165ea5df3..c7b70af60a164 100755
--- a/clients/src/main/java/org/apache/kafka/common/utils/Utils.java
+++ b/clients/src/main/java/org/apache/kafka/common/utils/Utils.java
@@ -287,20 +287,6 @@ public static byte[] copyArray(byte[] src) {
         return Arrays.copyOf(src, src.length);
     }
 
-    /**
-     * Check that the parameter t is not null
-     *
-     * @param t The object to check
-     * @return t if it isn't null
-     * @throws NullPointerException if t is null.
-     */
-    public static  T notNull(T t) {
-        if (t == null)
-            throw new NullPointerException();
-        else
-            return t;
-    }
-
     /**
      * Sleep for a bit
      * @param ms The duration of the sleep
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java
index 320ce114c61e7..ffd744bc7df02 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java
@@ -35,6 +35,7 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 
@@ -107,7 +108,7 @@ public void makeReady(final Map topics) {
                 final Set newTopics = new HashSet<>();
 
                 for (final String topicName : topicsNotReady) {
-                    final InternalTopicConfig internalTopicConfig = Utils.notNull(topics.get(topicName));
+                    final InternalTopicConfig internalTopicConfig = Objects.requireNonNull(topics.get(topicName));
                     final Map topicConfig = internalTopicConfig.getProperties(defaultTopicConfigs, windowChangeLogAdditionalRetention);
 
                     log.debug("Going to create topic {} with {} partitions and config {}.",

From 0c2d1c390d9e1653d25654bef9418af1fc68fdb7 Mon Sep 17 00:00:00 2001
From: Vikas Singh <50422828+soondenana@users.noreply.github.com>
Date: Mon, 12 Aug 2019 12:53:28 -0700
Subject: [PATCH 0522/1071] MINOR: Format AdminUtils::assignReplicasToBrokers
 java documentation (#7173)

The assignReplicasToBrokers method has helpful but a large unformatted
javadoc comment that results in a big blob in generated html. This
change formats the comment so that generated javadoc is nice.

Reviewers: Stanislav Kozlovski , Jason Gustafson 
---
 .../main/scala/kafka/admin/AdminUtils.scala   | 70 +++++++++++--------
 1 file changed, 41 insertions(+), 29 deletions(-)

diff --git a/core/src/main/scala/kafka/admin/AdminUtils.scala b/core/src/main/scala/kafka/admin/AdminUtils.scala
index c3748cff58b1f..ce4052dbe990f 100644
--- a/core/src/main/scala/kafka/admin/AdminUtils.scala
+++ b/core/src/main/scala/kafka/admin/AdminUtils.scala
@@ -31,58 +31,70 @@ object AdminUtils extends Logging {
   /**
    * There are 3 goals of replica assignment:
    *
-   * 1. Spread the replicas evenly among brokers.
-   * 2. For partitions assigned to a particular broker, their other replicas are spread over the other brokers.
-   * 3. If all brokers have rack information, assign the replicas for each partition to different racks if possible
+   * 
        
+   * 
      1.  Spread the replicas evenly among brokers.
        2. 
+   * 
      2.  For partitions assigned to a particular broker, their other replicas are spread over the other brokers.
        3. 
+   * 
      3.  If all brokers have rack information, assign the replicas for each partition to different racks if possible
        4. 
+   * 
      
    *
    * To achieve this goal for replica assignment without considering racks, we:
-   * 1. Assign the first replica of each partition by round-robin, starting from a random position in the broker list.
-   * 2. Assign the remaining replicas of each partition with an increasing shift.
+   * 
        
+   * 
      1.  Assign the first replica of each partition by round-robin, starting from a random position in the broker list.
        2. 
+   * 
      2.  Assign the remaining replicas of each partition with an increasing shift.
        3. 
+   * 
      
    *
    * Here is an example of assigning
-   * broker-0  broker-1  broker-2  broker-3  broker-4
-   * p0        p1        p2        p3        p4       (1st replica)
-   * p5        p6        p7        p8        p9       (1st replica)
-   * p4        p0        p1        p2        p3       (2nd replica)
-   * p8        p9        p5        p6        p7       (2nd replica)
-   * p3        p4        p0        p1        p2       (3nd replica)
-   * p7        p8        p9        p5        p6       (3nd replica)
+   * 
+   * 
+   * 
+   * 
+   * 
+   * 
+   * 
+   * 
+   * 
      broker-0broker-1broker-2broker-3broker-4 
      p0      p1      p2      p3      p4      (1st replica)
      p5      p6      p7      p8      p9      (1st replica)
      p4      p0      p1      p2      p3      (2nd replica)
      p8      p9      p5      p6      p7      (2nd replica)
      p3      p4      p0      p1      p2      (3nd replica)
      p7      p8      p9      p5      p6      (3nd replica)
      
    *
+   * 
      
    * To create rack aware assignment, this API will first create a rack alternated broker list. For example,
-   * from this brokerID -> rack mapping:
-   *
+   * from this brokerID -> rack mapping:
      
    * 0 -> "rack1", 1 -> "rack3", 2 -> "rack3", 3 -> "rack2", 4 -> "rack2", 5 -> "rack1"
-   *
+   * 

      
+   * 
      
    * The rack alternated list will be:
-   *
+   * 
      
    * 0, 3, 1, 5, 4, 2
-   *
+   * 

      
+   * 
      
    * Then an easy round-robin assignment can be applied. Assume 6 partitions with replication factor of 3, the assignment
    * will be:
-   *
-   * 0 -> 0,3,1
-   * 1 -> 3,1,5
-   * 2 -> 1,5,4
-   * 3 -> 5,4,2
-   * 4 -> 4,2,0
-   * 5 -> 2,0,3
-   *
+   * 
      
+   * 0 -> 0,3,1 
      
+   * 1 -> 3,1,5 
      
+   * 2 -> 1,5,4 
      
+   * 3 -> 5,4,2 
      
+   * 4 -> 4,2,0 
      
+   * 5 -> 2,0,3 
      
+   * 
      
+   * 
      
    * Once it has completed the first round-robin, if there are more partitions to assign, the algorithm will start
    * shifting the followers. This is to ensure we will not always get the same set of sequences.
    * In this case, if there is another partition to assign (partition #6), the assignment will be:
-   *
+   * 
      
    * 6 -> 0,4,2 (instead of repeating 0,3,1 as partition 0)
-   *
+   * 

      
+   * 
      
    * The rack aware assignment always chooses the 1st replica of the partition using round robin on the rack alternated
    * broker list. For rest of the replicas, it will be biased towards brokers on racks that do not have
    * any replica assignment, until every rack has a replica. Then the assignment will go back to round-robin on
    * the broker list.
-   *
+   * 
      
+   * 
      
+   * 
      
    * As the result, if the number of replicas is equal to or greater than the number of racks, it will ensure that
    * each rack will get at least one replica. Otherwise, each rack will get at most one replica. In a perfect
    * situation where the number of replicas is the same as the number of racks and each rack has the same number of
    * brokers, it guarantees that the replica distribution is even across brokers and racks.
-   *
+   * 
      
    * @return a Map from partition id to replica ids
    * @throws AdminOperationException If rack information is supplied but it is incomplete, or if it is not possible to
    *                                 assign each replica to a unique rack.

From 88087e91dd4eed1ee3c3e12961db84b7b56c34a0 Mon Sep 17 00:00:00 2001
From: Justine Olshan 
Date: Mon, 12 Aug 2019 14:17:28 -0700
Subject: [PATCH 0523/1071] MINOR: Clean up the sticky partitioner code a bit
 (#7151)

Reviewers: Colin P. McCabe , Lucas Bradstreet 
---
 .../kafka/clients/producer/KafkaProducer.java |  2 +-
 .../internals/StickyPartitionCache.java       |  4 +-
 .../internals/TransactionManagerTest.java     | 72 +++++++++----------
 docs/upgrade.html                             |  4 ++
 4 files changed, 43 insertions(+), 39 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
index c586fb41be414..3b3589fa5b95c 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
@@ -925,7 +925,7 @@ private Future doSend(ProducerRecord record, Callback call
                 partition = partition(record, serializedKey, serializedValue, cluster);
                 tp = new TopicPartition(record.topic(), partition);
                 if (log.isTraceEnabled()) {
-                    log.trace("Retrying because of a new batch, sending the record to topic {} partition {}. The old partition was {}", record.topic(), partition, prevPartition);
+                    log.trace("Retrying append due to new batch creation for topic {} partition {}. The old partition was {}", record.topic(), partition, prevPartition);
                 }
                 // producer callback will make sure to call both 'callback' and interceptor callback
                 interceptCallback = new InterceptorCallback<>(callback, this.interceptors, tp);
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/StickyPartitionCache.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/StickyPartitionCache.java
index 117eb5f080e9b..bfe1ac6c6fc3d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/StickyPartitionCache.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/StickyPartitionCache.java
@@ -51,14 +51,14 @@ public int nextPartition(String topic, Cluster cluster, int prevPartition) {
         // triggered the new batch matches the sticky partition that needs to be changed.
         if (oldPart == null || oldPart == prevPartition) {
             List availablePartitions = cluster.availablePartitionsForTopic(topic);
-            Integer random = Utils.toPositive(ThreadLocalRandom.current().nextInt());
             if (availablePartitions.size() < 1) {
+                Integer random = Utils.toPositive(ThreadLocalRandom.current().nextInt());
                 newPart = random % partitions.size();
             } else if (availablePartitions.size() == 1) {
                 newPart = availablePartitions.get(0).partition();
             } else {
                 while (newPart == null || newPart.equals(oldPart)) {
-                    random = Utils.toPositive(ThreadLocalRandom.current().nextInt());
+                    Integer random = Utils.toPositive(ThreadLocalRandom.current().nextInt());
                     newPart = availablePartitions.get(random % availablePartitions.size()).partition();
                 }
             }
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java
index cca5771002cd8..24869aa7f73ac 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java
@@ -154,7 +154,7 @@ public void testSenderShutdownWithPendingTransactions() throws Exception {
         transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
         FutureRecordMetadata sendFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         prepareAddPartitionsToTxn(tp0, Errors.NONE);
         prepareProduceResponse(Errors.NONE, pid, epoch);
@@ -800,7 +800,7 @@ public void testBasicTransaction() throws InterruptedException {
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         assertFalse(responseFuture.isDone());
         prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid);
@@ -1251,7 +1251,7 @@ public void testRecoveryFromAbortableErrorTransactionNotStarted() throws Excepti
         transactionManager.maybeAddPartitionToTransaction(unauthorizedPartition);
 
         Future responseFuture = accumulator.append(unauthorizedPartition, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         prepareAddPartitionsToTxn(singletonMap(unauthorizedPartition, Errors.TOPIC_AUTHORIZATION_FAILED));
         sender.runOnce();
@@ -1275,7 +1275,7 @@ public void testRecoveryFromAbortableErrorTransactionNotStarted() throws Excepti
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         prepareAddPartitionsToTxn(singletonMap(tp0, Errors.NONE));
         sender.runOnce();
@@ -1309,14 +1309,14 @@ public void testRecoveryFromAbortableErrorTransactionStarted() throws Exception
         prepareAddPartitionsToTxn(tp0, Errors.NONE);
 
         Future authorizedTopicProduceFuture = accumulator.append(unauthorizedPartition, time.milliseconds(),
-                "key".getBytes(), "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "key".getBytes(), "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         assertTrue(transactionManager.isPartitionAdded(tp0));
 
         transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(unauthorizedPartition);
         Future unauthorizedTopicProduceFuture = accumulator.append(unauthorizedPartition, time.milliseconds(),
-                "key".getBytes(), "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "key".getBytes(), "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
         prepareAddPartitionsToTxn(singletonMap(unauthorizedPartition, Errors.TOPIC_AUTHORIZATION_FAILED));
         sender.runOnce();
         assertTrue(transactionManager.hasAbortableError());
@@ -1342,7 +1342,7 @@ public void testRecoveryFromAbortableErrorTransactionStarted() throws Exception
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         FutureRecordMetadata nextTransactionFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         prepareAddPartitionsToTxn(singletonMap(tp0, Errors.NONE));
         sender.runOnce();
@@ -1376,7 +1376,7 @@ public void testRecoveryFromAbortableErrorProduceRequestInRetry() throws Excepti
         prepareAddPartitionsToTxn(tp0, Errors.NONE);
 
         Future authorizedTopicProduceFuture = accumulator.append(tp0, time.milliseconds(),
-                "key".getBytes(), "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "key".getBytes(), "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
         sender.runOnce();
         assertTrue(transactionManager.isPartitionAdded(tp0));
 
@@ -1389,7 +1389,7 @@ public void testRecoveryFromAbortableErrorProduceRequestInRetry() throws Excepti
         transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(unauthorizedPartition);
         Future unauthorizedTopicProduceFuture = accumulator.append(unauthorizedPartition, time.milliseconds(),
-                "key".getBytes(), "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "key".getBytes(), "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
         prepareAddPartitionsToTxn(singletonMap(unauthorizedPartition, Errors.TOPIC_AUTHORIZATION_FAILED));
         sender.runOnce();
         assertTrue(transactionManager.hasAbortableError());
@@ -1419,7 +1419,7 @@ public void testRecoveryFromAbortableErrorProduceRequestInRetry() throws Excepti
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         FutureRecordMetadata nextTransactionFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         prepareAddPartitionsToTxn(singletonMap(tp0, Errors.NONE));
         sender.runOnce();
@@ -1472,7 +1472,7 @@ public void testFlushPendingPartitionsOnCommit() throws InterruptedException {
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         assertFalse(responseFuture.isDone());
 
@@ -1518,7 +1518,7 @@ public void testMultipleAddPartitionsPerForOneProduce() throws InterruptedExcept
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         assertFalse(responseFuture.isDone());
         prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid);
@@ -1534,7 +1534,7 @@ public void testMultipleAddPartitionsPerForOneProduce() throws InterruptedExcept
         transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp1);
         Future secondResponseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         prepareAddPartitionsToTxnResponse(Errors.NONE, tp1, epoch, pid);
         prepareProduceResponse(Errors.NONE, pid, epoch);
@@ -1570,7 +1570,7 @@ public void testProducerFencedException() throws InterruptedException {
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         assertFalse(responseFuture.isDone());
         prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid);
@@ -1609,7 +1609,7 @@ public void testDisallowCommitOnProduceFailure() throws InterruptedException {
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         TransactionalRequestResult commitResult = transactionManager.beginCommit();
         assertFalse(responseFuture.isDone());
@@ -1659,7 +1659,7 @@ public void testAllowAbortOnProduceFailure() throws InterruptedException {
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         assertFalse(responseFuture.isDone());
         prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid);
@@ -1688,7 +1688,7 @@ public void testAbortableErrorWhileAbortInProgress() throws InterruptedException
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         assertFalse(responseFuture.isDone());
         prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid);
@@ -1726,7 +1726,7 @@ public void testCommitTransactionWithUnsentProduceRequest() throws Exception {
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         prepareAddPartitionsToTxn(tp0, Errors.NONE);
         sender.runOnce();
@@ -1776,7 +1776,7 @@ public void testCommitTransactionWithInFlightProduceRequest() throws Exception {
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         prepareAddPartitionsToTxn(tp0, Errors.NONE);
         sender.runOnce();
@@ -1832,7 +1832,7 @@ public void testFindCoordinatorAllowedInAbortableErrorState() throws Interrupted
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         assertFalse(responseFuture.isDone());
         sender.runOnce();  // Send AddPartitionsRequest
@@ -1861,7 +1861,7 @@ public void testCancelUnsentAddPartitionsAndProduceOnAbort() throws InterruptedE
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         assertFalse(responseFuture.isDone());
 
@@ -1894,7 +1894,7 @@ public void testAbortResendsAddPartitionErrorIfRetried() throws InterruptedExcep
         prepareAddPartitionsToTxnResponse(Errors.UNKNOWN_TOPIC_OR_PARTITION, tp0, producerEpoch, producerId);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         sender.runOnce();  // Send AddPartitions and let it fail
         assertFalse(responseFuture.isDone());
@@ -1934,7 +1934,7 @@ public void testAbortResendsProduceRequestIfRetried() throws Exception {
         prepareProduceResponse(Errors.REQUEST_TIMED_OUT, producerId, producerEpoch);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         sender.runOnce();  // Send AddPartitions
         sender.runOnce();  // Send ProduceRequest and let it fail
@@ -1970,7 +1970,7 @@ public void testHandlingOfUnknownTopicPartitionErrorOnAddPartitions() throws Int
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         assertFalse(responseFuture.isDone());
         prepareAddPartitionsToTxnResponse(Errors.UNKNOWN_TOPIC_OR_PARTITION, tp0, epoch, pid);
@@ -2131,11 +2131,11 @@ public void testNoDrainWhenPartitionsPending() throws InterruptedException {
         transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp0);
         accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false);
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false);
         transactionManager.failIfNotReadyForSend();
         transactionManager.maybeAddPartitionToTransaction(tp1);
         accumulator.append(tp1, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false);
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false);
 
         assertFalse(transactionManager.isSendToPartitionAllowed(tp0));
         assertFalse(transactionManager.isSendToPartitionAllowed(tp1));
@@ -2188,7 +2188,7 @@ public void testAllowDrainInAbortableErrorState() throws InterruptedException {
         Cluster cluster = new Cluster(null, Collections.singletonList(node1), Collections.singletonList(part1),
                 Collections.emptySet(), Collections.emptySet());
         accumulator.append(tp1, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false);
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false);
         Map> drainedBatches = accumulator.drain(cluster, Collections.singleton(node1),
                 Integer.MAX_VALUE,
                 time.milliseconds());
@@ -2208,7 +2208,7 @@ public void testRaiseErrorWhenNoPartitionsPendingOnDrain() throws InterruptedExc
         transactionManager.beginTransaction();
         // Don't execute transactionManager.maybeAddPartitionToTransaction(tp0). This should result in an error on drain.
         accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false);
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false);
         Node node1 = new Node(0, "localhost", 1111);
         PartitionInfo part1 = new PartitionInfo(topic, 0, node1, null, null);
 
@@ -2235,7 +2235,7 @@ public void resendFailedProduceRequestAfterAbortableError() throws Exception {
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid);
         prepareProduceResponse(Errors.NOT_LEADER_FOR_PARTITION, pid, epoch);
@@ -2264,7 +2264,7 @@ public void testTransitionToAbortableErrorOnBatchExpiry() throws InterruptedExce
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         assertFalse(responseFuture.isDone());
 
@@ -2313,9 +2313,9 @@ public void testTransitionToAbortableErrorOnMultipleBatchExpiry() throws Interru
         transactionManager.maybeAddPartitionToTransaction(tp1);
 
         Future firstBatchResponse = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
         Future secondBatchResponse = accumulator.append(tp1, time.milliseconds(), "key".getBytes(),
-               "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+               "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         assertFalse(firstBatchResponse.isDone());
         assertFalse(secondBatchResponse.isDone());
@@ -2378,7 +2378,7 @@ public void testDropCommitOnBatchExpiry() throws InterruptedException {
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         assertFalse(responseFuture.isDone());
 
@@ -2445,7 +2445,7 @@ public void testTransitionToFatalErrorWhenRetriedBatchIsExpired() throws Interru
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
 
         assertFalse(responseFuture.isDone());
 
@@ -2644,7 +2644,7 @@ private void verifyCommitOrAbortTranscationRetriable(TransactionResult firstTran
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false);
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false);
 
         prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid);
 
@@ -2686,7 +2686,7 @@ private void verifyAddPartitionsFailsWithPartitionLevelError(final Errors error)
         transactionManager.maybeAddPartitionToTransaction(tp0);
 
         Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
-                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT,  false).future;
+                "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT, false).future;
         assertFalse(responseFuture.isDone());
         prepareAddPartitionsToTxn(tp0, error);
         sender.runOnce();  // attempt send addPartitions.
diff --git a/docs/upgrade.html b/docs/upgrade.html
index e55e3b8dc4310..5fe06a4483ce9 100644
--- a/docs/upgrade.html
+++ b/docs/upgrade.html
@@ -32,6 +32,10 @@ 
      Notable changes in 2
     
      • 
     
    • The internal PartitionAssignor interface has been deprecated and replaced with a new ConsumerPartitionAssignor in the public API. Users
         implementing a custom PartitionAssignor should migrate to the new interface as soon as possible.
      • 
+    
    • The DefaultPartitioner now uses a sticky partitioning strategy. This means that records for specific topic with null keys and no assigned partition 
+        will be sent to the same partition until the batch is ready to be sent. When a new batch is created, a new partition is chosen. This decreases latency to produce, but
+        it may result in uneven distribution of records across partitions in edge cases. Generally users will not be impacted, but this difference may be noticeable in tests and
+        other situations producing records for a very short amount of time.
      • 
 
    
 
 
    Upgrading from 0.8.x, 0.9.x, 0.10.0.x, 0.10.1.x, 0.10.2.x, 0.11.0.x, 1.0.x, 1.1.x, 2.0.x or 2.1.x or 2.2.x to 2.3.0
    

From 794637232cea44ee458839a293091473e81fb01e Mon Sep 17 00:00:00 2001
From: Arjun Satish 
Date: Tue, 13 Aug 2019 07:40:12 -0700
Subject: [PATCH 0524/1071] KAFKA-8774: Regex can be found anywhere in config
 value (#7197)

Corrected the AbstractHerder to correctly identify task configs that contain variables for externalized secrets. The original method incorrectly used `matcher.matches()` instead of `matcher.find()`. The former method expects the entire string to match the regex, whereas the second one can find a pattern anywhere within the input string (which fits this use case more correctly).

Added unit tests to cover various cases of a config with externalized secrets, and updated system tests to cover case where config value contains additional characters besides secret that requires regex pattern to be found anywhere in the string (as opposed to complete match).

Author: Arjun Satish 
Reviewer: Randall Hauch 
---
 .../kafka/connect/runtime/AbstractHerder.java |  5 ++--
 .../connect/runtime/AbstractHerderTest.java   | 28 +++++++++++++++++++
 .../tests/connect/connect_rest_test.py        |  2 +-
 3 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
index f66029c5f7402..83d81bb347da0 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
@@ -598,12 +598,13 @@ public static List> reverseTransform(String connName,
         return result;
     }
 
-    private static Set keysWithVariableValues(Map rawConfig, Pattern pattern) {
+    // Visible for testing
+    static Set keysWithVariableValues(Map rawConfig, Pattern pattern) {
         Set keys = new HashSet<>();
         for (Map.Entry config : rawConfig.entrySet()) {
             if (config.getValue() != null) {
                 Matcher matcher = pattern.matcher(config.getValue());
-                if (matcher.matches()) {
+                if (matcher.find()) {
                     keys.add(config.getKey());
                 }
             }
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java
index 2c341bccfa77d..801891675a313 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java
@@ -20,6 +20,7 @@
 import org.apache.kafka.clients.producer.ProducerConfig;
 import org.apache.kafka.common.config.ConfigDef;
 import org.apache.kafka.common.config.ConfigException;
+import org.apache.kafka.common.config.ConfigTransformer;
 import org.apache.kafka.common.config.SaslConfigs;
 import org.apache.kafka.common.security.oauthbearer.internals.unsecured.OAuthBearerUnsecuredLoginCallbackHandler;
 import org.apache.kafka.connect.connector.ConnectRecord;
@@ -62,6 +63,7 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import static org.apache.kafka.connect.runtime.AbstractHerder.keysWithVariableValues;
 import static org.powermock.api.easymock.PowerMock.verifyAll;
 import static org.powermock.api.easymock.PowerMock.replayAll;
 import static org.easymock.EasyMock.strictMock;
@@ -474,6 +476,32 @@ public void testReverseTransformConfigs() {
         assertFalse(reverseTransformed.get(0).containsKey(TEST_KEY3));
     }
 
+    @Test
+    public void testConfigProviderRegex() {
+        testConfigProviderRegex("\"${::}\"");
+        testConfigProviderRegex("${::}");
+        testConfigProviderRegex("\"${:/a:somevar}\"");
+        testConfigProviderRegex("\"${file::somevar}\"");
+        testConfigProviderRegex("${file:/a/b/c:}");
+        testConfigProviderRegex("${file:/tmp/somefile.txt:somevar}");
+        testConfigProviderRegex("\"${file:/tmp/somefile.txt:somevar}\"");
+        testConfigProviderRegex("plain.PlainLoginModule required username=\"${file:/tmp/somefile.txt:somevar}\"");
+        testConfigProviderRegex("plain.PlainLoginModule required username=${file:/tmp/somefile.txt:somevar}");
+        testConfigProviderRegex("plain.PlainLoginModule required username=${file:/tmp/somefile.txt:somevar} not null");
+        testConfigProviderRegex("plain.PlainLoginModule required username=${file:/tmp/somefile.txt:somevar} password=${file:/tmp/somefile.txt:othervar}");
+        testConfigProviderRegex("plain.PlainLoginModule required username", false);
+    }
+
+    private void testConfigProviderRegex(String rawConnConfig) {
+        testConfigProviderRegex(rawConnConfig, true);
+    }
+    
+    private void testConfigProviderRegex(String rawConnConfig, boolean expected) {
+        Set keys = keysWithVariableValues(Collections.singletonMap("key", rawConnConfig), ConfigTransformer.DEFAULT_PATTERN);
+        boolean actual = keys != null && !keys.isEmpty() && keys.contains("key");
+        assertEquals(String.format("%s should have matched regex", rawConnConfig), expected, actual);
+    }
+
     private AbstractHerder createConfigValidationHerder(Class connectorClass,
                                                         ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy) {
 
diff --git a/tests/kafkatest/tests/connect/connect_rest_test.py b/tests/kafkatest/tests/connect/connect_rest_test.py
index 13dcf046d4075..67c04170b358a 100644
--- a/tests/kafkatest/tests/connect/connect_rest_test.py
+++ b/tests/kafkatest/tests/connect/connect_rest_test.py
@@ -44,7 +44,7 @@ class ConnectRestApiTest(KafkaTest):
     INPUT_FILE2 = "/mnt/connect.input2"
     OUTPUT_FILE = "/mnt/connect.output"
 
-    TOPIC = "${file:%s:topic.external}" % ConnectServiceBase.EXTERNAL_CONFIGS_FILE
+    TOPIC = "topic-${file:%s:topic.external}" % ConnectServiceBase.EXTERNAL_CONFIGS_FILE
     TOPIC_TEST = "test"
 
     DEFAULT_BATCH_SIZE = "2000"

From e2c8612d01f5e6e97476b1656fe728b2f316efef Mon Sep 17 00:00:00 2001
From: Paul 
Date: Tue, 13 Aug 2019 10:16:55 -0500
Subject: [PATCH 0525/1071] KAFKA-7941: Catch TimeoutException in KafkaBasedLog
 worker thread (#6283)

When calling readLogToEnd(), the KafkaBasedLog worker thread should catch TimeoutException and log a warning, which can occur if brokers are unavailable, otherwise the worker thread terminates.

Includes an enhancement to MockConsumer that allows simulating exceptions not just when polling but also when querying for offsets, which is necessary for testing the fix.

Author: Paul Whalen 
Reviewers: Randall Hauch , Arjun Satish , Ryanne Dolan 
---
 .../kafka/clients/consumer/MockConsumer.java  | 35 +++++++--
 .../kafka/connect/util/KafkaBasedLog.java     |  5 ++
 .../kafka/connect/util/KafkaBasedLogTest.java | 76 ++++++++++++++++++-
 .../internals/GlobalStateManagerImplTest.java |  2 +-
 .../internals/GlobalStreamThreadTest.java     |  2 +-
 .../internals/StoreChangelogReaderTest.java   |  2 +-
 .../processor/internals/StreamThreadTest.java |  2 +-
 7 files changed, 112 insertions(+), 12 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java
index 660a11238b3d2..947746f3ce1af 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java
@@ -61,7 +61,8 @@ public class MockConsumer implements Consumer {
     private final Set paused;
 
     private Map>> records;
-    private KafkaException exception;
+    private KafkaException pollException;
+    private KafkaException offsetsException;
     private AtomicBoolean wakeup;
     private boolean closed;
 
@@ -74,7 +75,7 @@ public MockConsumer(OffsetResetStrategy offsetResetStrategy) {
         this.beginningOffsets = new HashMap<>();
         this.endOffsets = new HashMap<>();
         this.pollTasks = new LinkedList<>();
-        this.exception = null;
+        this.pollException = null;
         this.wakeup = new AtomicBoolean(false);
         this.committed = new HashMap<>();
     }
@@ -173,9 +174,9 @@ public synchronized ConsumerRecords poll(final Duration timeout) {
             throw new WakeupException();
         }
 
-        if (exception != null) {
-            RuntimeException exception = this.exception;
-            this.exception = null;
+        if (pollException != null) {
+            RuntimeException exception = this.pollException;
+            this.pollException = null;
             throw exception;
         }
 
@@ -220,8 +221,20 @@ public synchronized void addRecord(ConsumerRecord record) {
         recs.add(record);
     }
 
+    /**
+     * @deprecated Use {@link #setPollException(KafkaException)} instead
+     */
+    @Deprecated
     public synchronized void setException(KafkaException exception) {
-        this.exception = exception;
+        setPollException(exception);
+    }
+
+    public synchronized void setPollException(KafkaException exception) {
+        this.pollException = exception;
+    }
+
+    public synchronized void setOffsetsException(KafkaException exception) {
+        this.offsetsException = exception;
     }
 
     @Override
@@ -393,6 +406,11 @@ public synchronized Map offsetsForTimes(Map<
 
     @Override
     public synchronized Map beginningOffsets(Collection partitions) {
+        if (offsetsException != null) {
+            RuntimeException exception = this.offsetsException;
+            this.offsetsException = null;
+            throw exception;
+        }
         Map result = new HashMap<>();
         for (TopicPartition tp : partitions) {
             Long beginningOffset = beginningOffsets.get(tp);
@@ -405,6 +423,11 @@ public synchronized Map beginningOffsets(Collection endOffsets(Collection partitions) {
+        if (offsetsException != null) {
+            RuntimeException exception = this.offsetsException;
+            this.offsetsException = null;
+            throw exception;
+        }
         Map result = new HashMap<>();
         for (TopicPartition tp : partitions) {
             Long endOffset = getEndOffset(endOffsets.get(tp));
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java b/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java
index 9d77d21767a89..e78276a264dc8 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java
@@ -28,6 +28,7 @@
 import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.PartitionInfo;
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.TimeoutException;
 import org.apache.kafka.common.errors.WakeupException;
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.common.utils.Utils;
@@ -312,6 +313,10 @@ public void run() {
                         try {
                             readToLogEnd();
                             log.trace("Finished read to end log for topic {}", topic);
+                        } catch (TimeoutException e) {
+                            log.warn("Timeout while reading log to end for topic '{}'. Retrying automatically. " +
+                                "This may occur when brokers are unavailable or unreachable. Reason: {}", topic, e.getMessage());
+                            continue;
                         } catch (WakeupException e) {
                             // Either received another get() call and need to retry reading to end of log or stop() was
                             // called. Both are handled by restarting this loop.
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/KafkaBasedLogTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/KafkaBasedLogTest.java
index 1af6e343521a6..080f9434d13f6 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/util/KafkaBasedLogTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/KafkaBasedLogTest.java
@@ -30,6 +30,7 @@
 import org.apache.kafka.common.PartitionInfo;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.errors.LeaderNotAvailableException;
+import org.apache.kafka.common.errors.TimeoutException;
 import org.apache.kafka.common.errors.WakeupException;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.record.TimestampType;
@@ -370,7 +371,7 @@ public void run() {
     }
 
     @Test
-    public void testConsumerError() throws Exception {
+    public void testPollConsumerError() throws Exception {
         expectStart();
         expectStop();
 
@@ -388,7 +389,7 @@ public void run() {
                 consumer.schedulePollTask(new Runnable() {
                     @Override
                     public void run() {
-                        consumer.setException(Errors.COORDINATOR_NOT_AVAILABLE.exception());
+                        consumer.setPollException(Errors.COORDINATOR_NOT_AVAILABLE.exception());
                     }
                 });
 
@@ -423,6 +424,77 @@ public void run() {
         PowerMock.verifyAll();
     }
 
+    @Test
+    public void testGetOffsetsConsumerErrorOnReadToEnd() throws Exception {
+        expectStart();
+
+        // Producer flushes when read to log end is called
+        producer.flush();
+        PowerMock.expectLastCall();
+
+        expectStop();
+
+        PowerMock.replayAll();
+        final CountDownLatch finishedLatch = new CountDownLatch(1);
+        Map endOffsets = new HashMap<>();
+        endOffsets.put(TP0, 0L);
+        endOffsets.put(TP1, 0L);
+        consumer.updateEndOffsets(endOffsets);
+        store.start();
+        final AtomicBoolean getInvoked = new AtomicBoolean(false);
+        final FutureCallback readEndFutureCallback = new FutureCallback<>(new Callback() {
+            @Override
+            public void onCompletion(Throwable error, Void result) {
+                getInvoked.set(true);
+            }
+        });
+        consumer.schedulePollTask(new Runnable() {
+            @Override
+            public void run() {
+                // Once we're synchronized in a poll, start the read to end and schedule the exact set of poll events
+                // that should follow. This readToEnd call will immediately wakeup this consumer.poll() call without
+                // returning any data.
+                Map newEndOffsets = new HashMap<>();
+                newEndOffsets.put(TP0, 1L);
+                newEndOffsets.put(TP1, 1L);
+                consumer.updateEndOffsets(newEndOffsets);
+                // Set exception to occur when getting offsets to read log to end.  It'll be caught in the work thread,
+                // which will retry and eventually get the correct offsets and read log to end.
+                consumer.setOffsetsException(new TimeoutException("Failed to get offsets by times"));
+                store.readToEnd(readEndFutureCallback);
+
+                // Should keep polling until it reaches current log end offset for all partitions
+                consumer.scheduleNopPollTask();
+                consumer.scheduleNopPollTask();
+                consumer.schedulePollTask(new Runnable() {
+                    @Override
+                    public void run() {
+                        consumer.addRecord(new ConsumerRecord<>(TOPIC, 0, 0, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, TP0_KEY, TP0_VALUE));
+                        consumer.addRecord(new ConsumerRecord<>(TOPIC, 1, 0, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, TP0_KEY, TP0_VALUE_NEW));
+                    }
+                });
+
+                consumer.schedulePollTask(new Runnable() {
+                    @Override
+                    public void run() {
+                        finishedLatch.countDown();
+                    }
+                });
+            }
+        });
+        readEndFutureCallback.get(10000, TimeUnit.MILLISECONDS);
+        assertTrue(getInvoked.get());
+        assertTrue(finishedLatch.await(10000, TimeUnit.MILLISECONDS));
+        assertEquals(CONSUMER_ASSIGNMENT, consumer.assignment());
+        assertEquals(1L, consumer.position(TP0));
+
+        store.stop();
+
+        assertFalse(Whitebox.getInternalState(store, "thread").isAlive());
+        assertTrue(consumer.closed());
+        PowerMock.verifyAll();
+    }
+
     @Test
     public void testProducerError() throws Exception {
         expectStart();
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStateManagerImplTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStateManagerImplTest.java
index 8e3eb39390b63..425c074615753 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStateManagerImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStateManagerImplTest.java
@@ -302,7 +302,7 @@ public void shouldRestoreRecordsUpToHighwatermark() {
     @Test
     public void shouldRecoverFromInvalidOffsetExceptionAndRestoreRecords() {
         initializeConsumer(2, 0, t1);
-        consumer.setException(new InvalidOffsetException("Try Again!") {
+        consumer.setPollException(new InvalidOffsetException("Try Again!") {
             public Set partitions() {
                 return Collections.singleton(t1);
             }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStreamThreadTest.java
index a5db9246a0d07..508b000789ee6 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStreamThreadTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStreamThreadTest.java
@@ -236,7 +236,7 @@ public void shouldDieOnInvalidOffsetException() throws Exception {
             10 * 1000,
             "Input record never consumed");
 
-        mockConsumer.setException(new InvalidOffsetException("Try Again!") {
+        mockConsumer.setPollException(new InvalidOffsetException("Try Again!") {
             @Override
             public Set partitions() {
                 return Collections.singleton(topicPartition);
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java
index b13ed535d625f..bf59c21f12c15 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java
@@ -158,7 +158,7 @@ public void shouldRestoreAllMessagesFromBeginningWhenCheckpointNull() {
     public void shouldRecoverFromInvalidOffsetExceptionAndFinishRestore() {
         final int messages = 10;
         setupConsumer(messages, topicPartition);
-        consumer.setException(new InvalidOffsetException("Try Again!") {
+        consumer.setPollException(new InvalidOffsetException("Try Again!") {
             @Override
             public Set partitions() {
                 return Collections.singleton(topicPartition);
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
index aff5d6c74861c..ceadaecc812c8 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
@@ -1409,7 +1409,7 @@ public void shouldRecoverFromInvalidOffsetExceptionOnRestoreAndFinishRestore() t
                 () -> mockRestoreConsumer.position(changelogPartition) == 1L,
                 "Never restore first record");
 
-            mockRestoreConsumer.setException(new InvalidOffsetException("Try Again!") {
+            mockRestoreConsumer.setPollException(new InvalidOffsetException("Try Again!") {
                 @Override
                 public Set partitions() {
                     return changelogPartitionSet;

From e5657bc910bbf82638a8b6c3e96c30101cf27560 Mon Sep 17 00:00:00 2001
From: Randall Hauch 
Date: Tue, 13 Aug 2019 11:14:41 -0700
Subject: [PATCH 0526/1071] KAFKA-8391; Improved the Connect integration tests
 to make them less flaky
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Added the ability for the connector handles and task handles, which are used by the monitorable source and sink connectors used to verify the functionality of the Connect framework, to record the number of times the connector and tasks have each been started, and to allow a test to obtain a `RestartLatch` that can be used to block until the connectors and/or tasks have been restarted a specified number of types.

Typically, a test will get the `ConnectorHandle` for a connector, and call the `ConnectorHandle.expectedRestarts(int)` method with the expected number of times that the connector and/or tasks will be restarted, and will hold onto the resulting `RestartLatch`. The test will then change the connector (or otherwise cause the connector to restart) one or more times as desired, and then call `RestartLatch.await(long, TimeUnit)` to block the test up to a specified duration for the connector and all tasks to be started the specified number of times.

This commit also increases several of the maximum wait times used in other integration tests. It doesn’t hurt to potentially wait longer, since most test runs will not need to wait the maximum amount of time anyway. However, in the rare cases that do need that extra time, waiting a bit more is fine if we can reduce the flakiness and minimize test failures that happened to time out too early.

Unit tests were added for the new `RestartLatch` and `StopAndStartCounter` utility classes. This PR only affects the tests and does not affect any runtime code or API.

**This should be merged on `trunk` and backported to the `2.3.x` branch.**

Author: Randall Hauch 

Reviewers: Konstantine Karantasis, Arjun Satish

Closes #7019 from rhauch/kafka-8391
---
 .../ConnectWorkerIntegrationTest.java         |   7 +-
 .../connect/integration/ConnectorHandle.java  | 135 ++++++++++++-
 .../ErrorHandlingIntegrationTest.java         |   5 +-
 .../ExampleConnectIntegrationTest.java        |   5 +-
 .../integration/MonitorableSinkConnector.java |   6 +
 .../MonitorableSourceConnector.java           |   4 +
 ...alanceSourceConnectorsIntegrationTest.java |  23 ++-
 .../integration/StartAndStopCounter.java      | 179 ++++++++++++++++++
 .../integration/StartAndStopCounterTest.java  | 116 ++++++++++++
 .../integration/StartAndStopLatch.java        | 118 ++++++++++++
 .../integration/StartAndStopLatchTest.java    | 137 ++++++++++++++
 .../kafka/connect/integration/TaskHandle.java |  85 +++++++--
 12 files changed, 790 insertions(+), 30 deletions(-)
 create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/integration/StartAndStopCounter.java
 create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/integration/StartAndStopCounterTest.java
 create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/integration/StartAndStopLatch.java
 create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/integration/StartAndStopLatchTest.java

diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java
index 09363cdd3afaf..ca63a07e0a7a6 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java
@@ -35,6 +35,7 @@
 import java.util.Optional;
 import java.util.Properties;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 
 import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLASS_CONFIG;
 import static org.apache.kafka.connect.runtime.ConnectorConfig.KEY_CONVERTER_CLASS_CONFIG;
@@ -53,9 +54,9 @@ public class ConnectWorkerIntegrationTest {
     private static final Logger log = LoggerFactory.getLogger(ConnectWorkerIntegrationTest.class);
 
     private static final int NUM_TOPIC_PARTITIONS = 3;
-    private static final int CONNECTOR_SETUP_DURATION_MS = 15_000;
-    private static final int WORKER_SETUP_DURATION_MS = 20_000;
-    private static final int OFFSET_COMMIT_INTERVAL_MS = 30_000;
+    private static final long CONNECTOR_SETUP_DURATION_MS = TimeUnit.SECONDS.toMillis(30);
+    private static final long WORKER_SETUP_DURATION_MS = TimeUnit.SECONDS.toMillis(60);
+    private static final long OFFSET_COMMIT_INTERVAL_MS = TimeUnit.SECONDS.toMillis(30);
     private static final int NUM_WORKERS = 3;
     private static final String CONNECTOR_NAME = "simple-source";
 
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorHandle.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorHandle.java
index 0df0f8cad21bc..a4a461288301b 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorHandle.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorHandle.java
@@ -21,10 +21,12 @@
 import org.slf4j.LoggerFactory;
 
 import java.util.Collection;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
 /**
@@ -36,6 +38,7 @@ public class ConnectorHandle {
 
     private final String connectorName;
     private final Map taskHandles = new ConcurrentHashMap<>();
+    private final StartAndStopCounter startAndStopCounter = new StartAndStopCounter();
 
     private CountDownLatch recordsRemainingLatch;
     private CountDownLatch recordsToCommitLatch;
@@ -152,7 +155,7 @@ public void commit(int batchSize) {
      * @param timeout max duration to wait for records
      * @throws InterruptedException if another threads interrupts this one while waiting for records
      */
-    public void awaitRecords(int timeout) throws InterruptedException {
+    public void awaitRecords(long timeout) throws InterruptedException {
         if (recordsRemainingLatch == null || expectedRecords < 0) {
             throw new IllegalStateException("expectedRecords() was not set for this connector?");
         }
@@ -174,7 +177,7 @@ public void awaitRecords(int timeout) throws InterruptedException {
      * @param  timeout duration to wait for commits
      * @throws InterruptedException if another threads interrupts this one while waiting for commits
      */
-    public void awaitCommits(int timeout) throws InterruptedException {
+    public void awaitCommits(long timeout) throws InterruptedException {
         if (recordsToCommitLatch == null || expectedCommits < 0) {
             throw new IllegalStateException("expectedCommits() was not set for this connector?");
         }
@@ -189,6 +192,134 @@ public void awaitCommits(int timeout) throws InterruptedException {
         }
     }
 
+    /**
+     * Record that this connector has been started. This should be called by the connector under
+     * test.
+     *
+     * @see #expectedStarts(int)
+     */
+    public void recordConnectorStart() {
+        startAndStopCounter.recordStart();
+    }
+
+    /**
+     * Record that this connector has been stopped. This should be called by the connector under
+     * test.
+     *
+     * @see #expectedStarts(int)
+     */
+    public void recordConnectorStop() {
+        startAndStopCounter.recordStop();
+    }
+
+    /**
+     * Obtain a {@link StartAndStopLatch} that can be used to wait until the connector using this handle
+     * and all tasks using {@link TaskHandle} have completed the expected number of
+     * starts, starting the counts at the time this method is called.
+     *
+     * 
    A test can call this method, specifying the number of times the connector and tasks
+     * will each be stopped and started from that point (typically {@code expectedStarts(1)}).
+     * The test should then change the connector or otherwise cause the connector to restart one or
+     * more times, and then can call {@link StartAndStopLatch#await(long, TimeUnit)} to wait up to a
+     * specified duration for the connector and all tasks to be started at least the specified
+     * number of times.
+     *
+     * 
    This method does not track the number of times the connector and tasks are stopped, and
+     * only tracks the number of times the connector and tasks are started.
+     *
+     * @param expectedStarts the minimum number of starts that are expected once this method is
+     *                       called
+     * @return the latch that can be used to wait for the starts to complete; never null
+     */
+    public StartAndStopLatch expectedStarts(int expectedStarts) {
+        return expectedStarts(expectedStarts, true);
+    }
+
+    /**
+     * Obtain a {@link StartAndStopLatch} that can be used to wait until the connector using this handle
+     * and optionally all tasks using {@link TaskHandle} have completed the expected number of
+     * starts, starting the counts at the time this method is called.
+     *
+     * 
    A test can call this method, specifying the number of times the connector and tasks
+     * will each be stopped and started from that point (typically {@code expectedStarts(1)}).
+     * The test should then change the connector or otherwise cause the connector to restart one or
+     * more times, and then can call {@link StartAndStopLatch#await(long, TimeUnit)} to wait up to a
+     * specified duration for the connector and all tasks to be started at least the specified
+     * number of times.
+     *
+     * 
    This method does not track the number of times the connector and tasks are stopped, and
+     * only tracks the number of times the connector and tasks are started.
+     *
+     * @param expectedStarts the minimum number of starts that are expected once this method is
+     *                       called
+     * @param includeTasks  true if the latch should also wait for the tasks to be stopped the
+     *                      specified minimum number of times
+     * @return the latch that can be used to wait for the starts to complete; never null
+     */
+    public StartAndStopLatch expectedStarts(int expectedStarts, boolean includeTasks) {
+        List taskLatches = null;
+        if (includeTasks) {
+            taskLatches = taskHandles.values().stream()
+                                     .map(task -> task.expectedStarts(expectedStarts))
+                                     .collect(Collectors.toList());
+        }
+        return startAndStopCounter.expectedStarts(expectedStarts, taskLatches);
+    }
+
+    /**
+     * Obtain a {@link StartAndStopLatch} that can be used to wait until the connector using this handle
+     * and optionally all tasks using {@link TaskHandle} have completed the minimum number of
+     * stops, starting the counts at the time this method is called.
+     *
+     * 
    A test can call this method, specifying the number of times the connector and tasks
+     * will each be stopped from that point (typically {@code expectedStops(1)}).
+     * The test should then change the connector or otherwise cause the connector to stop (or
+     * restart) one or more times, and then can call
+     * {@link StartAndStopLatch#await(long, TimeUnit)} to wait up to a specified duration for the
+     * connector and all tasks to be started at least the specified number of times.
+     *
+     * 
    This method does not track the number of times the connector and tasks are stopped, and
+     * only tracks the number of times the connector and tasks are started.
+     *
+     * @param expectedStops the minimum number of starts that are expected once this method is
+     *                      called
+     * @return the latch that can be used to wait for the starts to complete; never null
+     */
+    public StartAndStopLatch expectedStops(int expectedStops) {
+        return expectedStops(expectedStops, true);
+    }
+
+    /**
+     * Obtain a {@link StartAndStopLatch} that can be used to wait until the connector using this handle
+     * and optionally all tasks using {@link TaskHandle} have completed the minimum number of
+     * stops, starting the counts at the time this method is called.
+     *
+     * 
    A test can call this method, specifying the number of times the connector and tasks
+     * will each be stopped from that point (typically {@code expectedStops(1)}).
+     * The test should then change the connector or otherwise cause the connector to stop (or
+     * restart) one or more times, and then can call
+     * {@link StartAndStopLatch#await(long, TimeUnit)} to wait up to a specified duration for the
+     * connector and all tasks to be started at least the specified number of times.
+     *
+     * 
    This method does not track the number of times the connector and tasks are stopped, and
+     * only tracks the number of times the connector and tasks are started.
+     *
+     * @param expectedStops the minimum number of starts that are expected once this method is
+     *                      called
+     * @param includeTasks  true if the latch should also wait for the tasks to be stopped the
+     *                      specified minimum number of times
+     * @return the latch that can be used to wait for the starts to complete; never null
+     */
+    public StartAndStopLatch expectedStops(int expectedStops, boolean includeTasks) {
+        List taskLatches = null;
+        if (includeTasks) {
+            taskLatches = taskHandles.values().stream()
+                                     .map(task -> task.expectedStops(expectedStops))
+                                     .collect(Collectors.toList());
+        }
+        return startAndStopCounter.expectedStops(expectedStops, taskLatches);
+    }
+
     @Override
     public String toString() {
         return "ConnectorHandle{" +
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ErrorHandlingIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ErrorHandlingIntegrationTest.java
index 67bcc748c6509..33e6cf58234b2 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ErrorHandlingIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ErrorHandlingIntegrationTest.java
@@ -37,6 +37,7 @@
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLASS_CONFIG;
 import static org.apache.kafka.connect.runtime.ConnectorConfig.ERRORS_LOG_ENABLE_CONFIG;
@@ -75,8 +76,8 @@ public class ErrorHandlingIntegrationTest {
     private static final int EXPECTED_CORRECT_RECORDS = 19;
     private static final int EXPECTED_INCORRECT_RECORDS = 1;
     private static final int NUM_TASKS = 1;
-    private static final int CONNECTOR_SETUP_DURATION_MS = 5000;
-    private static final int CONSUME_MAX_DURATION_MS = 5000;
+    private static final long CONNECTOR_SETUP_DURATION_MS = TimeUnit.SECONDS.toMillis(60);
+    private static final long CONSUME_MAX_DURATION_MS = TimeUnit.SECONDS.toMillis(30);
 
     private EmbeddedConnectCluster connect;
     private ConnectorHandle connectorHandle;
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExampleConnectIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExampleConnectIntegrationTest.java
index 35a76a591965b..4da89d7406648 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExampleConnectIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExampleConnectIntegrationTest.java
@@ -31,6 +31,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
+import java.util.concurrent.TimeUnit;
 
 import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLASS_CONFIG;
 import static org.apache.kafka.connect.runtime.ConnectorConfig.KEY_CONVERTER_CLASS_CONFIG;
@@ -55,8 +56,8 @@ public class ExampleConnectIntegrationTest {
 
     private static final int NUM_RECORDS_PRODUCED = 2000;
     private static final int NUM_TOPIC_PARTITIONS = 3;
-    private static final int RECORD_TRANSFER_DURATION_MS = 30000;
-    private static final int CONNECTOR_SETUP_DURATION_MS = 15000;
+    private static final long RECORD_TRANSFER_DURATION_MS = TimeUnit.SECONDS.toMillis(30);
+    private static final long CONNECTOR_SETUP_DURATION_MS = TimeUnit.SECONDS.toMillis(60);
     private static final int NUM_TASKS = 3;
     private static final int NUM_WORKERS = 3;
     private static final String CONNECTOR_NAME = "simple-conn";
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSinkConnector.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSinkConnector.java
index 06145de977a3a..ed6f6b96f98c2 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSinkConnector.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSinkConnector.java
@@ -47,12 +47,15 @@ public class MonitorableSinkConnector extends TestSinkConnector {
 
     private String connectorName;
     private Map commonConfigs;
+    private ConnectorHandle connectorHandle;
 
     @Override
     public void start(Map props) {
+        connectorHandle = RuntimeHandles.get().connectorHandle(props.get("name"));
         connectorName = props.get("name");
         commonConfigs = props;
         log.info("Starting connector {}", props.get("name"));
+        connectorHandle.recordConnectorStart();
     }
 
     @Override
@@ -74,6 +77,7 @@ public List> taskConfigs(int maxTasks) {
 
     @Override
     public void stop() {
+        connectorHandle.recordConnectorStop();
     }
 
     @Override
@@ -107,6 +111,7 @@ public void start(Map props) {
             connectorName = props.get("connector.name");
             taskHandle = RuntimeHandles.get().connectorHandle(connectorName).taskHandle(taskId);
             log.debug("Starting task {}", taskId);
+            taskHandle.recordTaskStart();
         }
 
         @Override
@@ -148,6 +153,7 @@ public Map preCommit(Map props) {
         connectorName = connectorHandle.name();
         commonConfigs = props;
         log.info("Started {} connector {}", this.getClass().getSimpleName(), connectorName);
+        connectorHandle.recordConnectorStart();
     }
 
     @Override
@@ -77,6 +78,7 @@ public List> taskConfigs(int maxTasks) {
     @Override
     public void stop() {
         log.info("Stopped {} connector {}", this.getClass().getSimpleName(), connectorName);
+        connectorHandle.recordConnectorStop();
     }
 
     @Override
@@ -116,6 +118,7 @@ public void start(Map props) {
             startingSeqno = Optional.ofNullable((Long) offset.get("saved")).orElse(0L);
             log.info("Started {} task {} with properties {}", this.getClass().getSimpleName(), taskId, props);
             throttler = new ThroughputThrottler(throughput, System.currentTimeMillis());
+            taskHandle.recordTaskStart();
         }
 
         @Override
@@ -156,6 +159,7 @@ public void commitRecord(SourceRecord record) {
         public void stop() {
             log.info("Stopped {} task {}", this.getClass().getSimpleName(), taskId);
             stopped = true;
+            taskHandle.recordTaskStop();
         }
     }
 }
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java
index d3cc8dbfe58dc..ad93534bc45f8 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java
@@ -36,6 +36,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Properties;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
@@ -61,8 +62,8 @@ public class RebalanceSourceConnectorsIntegrationTest {
     private static final Logger log = LoggerFactory.getLogger(RebalanceSourceConnectorsIntegrationTest.class);
 
     private static final int NUM_TOPIC_PARTITIONS = 3;
-    private static final int CONNECTOR_SETUP_DURATION_MS = 30_000;
-    private static final int WORKER_SETUP_DURATION_MS = 30_000;
+    private static final long CONNECTOR_SETUP_DURATION_MS = TimeUnit.SECONDS.toMillis(30);
+    private static final long WORKER_SETUP_DURATION_MS = TimeUnit.SECONDS.toMillis(60);
     private static final int NUM_TASKS = 4;
     private static final String CONNECTOR_NAME = "seq-source1";
     private static final String TOPIC_NAME = "sequential-topic";
@@ -156,26 +157,29 @@ public void testReconfigConnector() throws Exception {
                 CONNECTOR_SETUP_DURATION_MS, "Connector tasks did not start in time.");
 
         int numRecordsProduced = 100;
-        int recordTransferDurationMs = 5000;
+        long recordTransferDurationMs = TimeUnit.SECONDS.toMillis(30);
 
         // consume all records from the source topic or fail, to ensure that they were correctly produced
         int recordNum = connect.kafka().consume(numRecordsProduced, recordTransferDurationMs, TOPIC_NAME).count();
         assertTrue("Not enough records produced by source connector. Expected at least: " + numRecordsProduced + " + but got " + recordNum,
                 recordNum >= numRecordsProduced);
 
+        // expect that we're going to restart the connector and its tasks
+        StartAndStopLatch restartLatch = connectorHandle.expectedStarts(1);
+
         // Reconfigure the source connector by changing the Kafka topic used as output
         props.put(TOPIC_CONFIG, anotherTopic);
         connect.configureConnector(CONNECTOR_NAME, props);
 
+        // Wait for the connector *and tasks* to be restarted
+        assertTrue("Failed to alter connector configuration and see connector and tasks restart "
+                   + "within " + CONNECTOR_SETUP_DURATION_MS + "ms",
+                restartLatch.await(CONNECTOR_SETUP_DURATION_MS, TimeUnit.MILLISECONDS));
+
+        // And wait for the Connect to show the connectors and tasks are running
         waitForCondition(() -> this.assertConnectorAndTasksRunning(CONNECTOR_NAME, NUM_TASKS).orElse(false),
                 CONNECTOR_SETUP_DURATION_MS, "Connector tasks did not start in time.");
 
-        // expect all records to be produced by the connector
-        connectorHandle.expectedRecords(numRecordsProduced);
-
-        // expect all records to be produced by the connector
-        connectorHandle.expectedCommits(numRecordsProduced);
-
         // consume all records from the source topic or fail, to ensure that they were correctly produced
         recordNum = connect.kafka().consume(numRecordsProduced, recordTransferDurationMs, anotherTopic).count();
         assertTrue("Not enough records produced by source connector. Expected at least: " + numRecordsProduced + " + but got " + recordNum,
@@ -320,6 +324,7 @@ private Optional assertConnectorAndTasksRunning(String connectorName, i
                     && info.tasks().size() == numTasks
                     && info.connector().state().equals(AbstractStatus.State.RUNNING.toString())
                     && info.tasks().stream().allMatch(s -> s.state().equals(AbstractStatus.State.RUNNING.toString()));
+            log.debug("Found connector and tasks running: {}", result);
             return Optional.of(result);
         } catch (Exception e) {
             log.error("Could not check connector state info.", e);
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/StartAndStopCounter.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/StartAndStopCounter.java
new file mode 100644
index 0000000000000..9ddfa1f685c97
--- /dev/null
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/StartAndStopCounter.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.connect.integration;
+
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.kafka.common.utils.Time;
+
+public class StartAndStopCounter {
+
+    private final AtomicInteger startCounter = new AtomicInteger(0);
+    private final AtomicInteger stopCounter = new AtomicInteger(0);
+    private final List restartLatches = new CopyOnWriteArrayList<>();
+    private final Time clock;
+
+    public StartAndStopCounter() {
+        this(Time.SYSTEM);
+    }
+
+    public StartAndStopCounter(Time clock) {
+        this.clock = clock != null ? clock : Time.SYSTEM;
+    }
+
+    /**
+     * Record a start.
+     */
+    public void recordStart() {
+        startCounter.incrementAndGet();
+        restartLatches.forEach(StartAndStopLatch::recordStart);
+    }
+
+    /**
+     * Record a stop.
+     */
+    public void recordStop() {
+        stopCounter.incrementAndGet();
+        restartLatches.forEach(StartAndStopLatch::recordStop);
+    }
+
+    /**
+     * Get the number of starts.
+     *
+     * @return the number of starts
+     */
+    public int starts() {
+        return startCounter.get();
+    }
+
+    /**
+     * Get the number of stops.
+     *
+     * @return the number of stops
+     */
+    public int stops() {
+        return stopCounter.get();
+    }
+
+    /**
+     * Obtain a {@link StartAndStopLatch} that can be used to wait until the expected number of restarts
+     * has been completed.
+     *
+     * @param expectedStarts   the expected number of starts; may be 0
+     * @param expectedStops    the expected number of stops; may be 0
+     * @return the latch; never null
+     */
+    public StartAndStopLatch expectedRestarts(int expectedStarts, int expectedStops) {
+        return expectedRestarts(expectedStarts, expectedStops, null);
+    }
+
+    /**
+     * Obtain a {@link StartAndStopLatch} that can be used to wait until the expected number of restarts
+     * has been completed.
+     *
+     * @param expectedStarts   the expected number of starts; may be 0
+     * @param expectedStops    the expected number of stops; may be 0
+     * @param dependents       any dependent latches that must also complete in order for the
+     *                         resulting latch to complete
+     * @return the latch; never null
+     */
+    public StartAndStopLatch expectedRestarts(int expectedStarts, int expectedStops, List dependents) {
+        StartAndStopLatch latch = new StartAndStopLatch(expectedStarts, expectedStops, this::remove, dependents, clock);
+        restartLatches.add(latch);
+        return latch;
+    }
+
+    /**
+     * Obtain a {@link StartAndStopLatch} that can be used to wait until the expected number of restarts
+     * has been completed.
+     *
+     * @param expectedRestarts the expected number of restarts
+     * @return the latch; never null
+     */
+    public StartAndStopLatch expectedRestarts(int expectedRestarts) {
+        return expectedRestarts(expectedRestarts, expectedRestarts);
+    }
+
+    /**
+     * Obtain a {@link StartAndStopLatch} that can be used to wait until the expected number of restarts
+     * has been completed.
+     *
+     * @param expectedRestarts the expected number of restarts
+     * @param dependents       any dependent latches that must also complete in order for the
+     *                         resulting latch to complete
+     * @return the latch; never null
+     */
+    public StartAndStopLatch expectedRestarts(int expectedRestarts, List dependents) {
+        return expectedRestarts(expectedRestarts, expectedRestarts, dependents);
+    }
+
+    /**
+     * Obtain a {@link StartAndStopLatch} that can be used to wait until the expected number of starts
+     * has been completed.
+     *
+     * @param expectedStarts the expected number of starts
+     * @return the latch; never null
+     */
+    public StartAndStopLatch expectedStarts(int expectedStarts) {
+        return expectedRestarts(expectedStarts, 0);
+    }
+
+    /**
+     * Obtain a {@link StartAndStopLatch} that can be used to wait until the expected number of starts
+     * has been completed.
+     *
+     * @param expectedStarts the expected number of starts
+     * @param dependents     any dependent latches that must also complete in order for the
+     *                       resulting latch to complete
+     * @return the latch; never null
+     */
+    public StartAndStopLatch expectedStarts(int expectedStarts, List dependents) {
+        return expectedRestarts(expectedStarts, 0, dependents);
+    }
+
+
+    /**
+     * Obtain a {@link StartAndStopLatch} that can be used to wait until the expected number of
+     * stops has been completed.
+     *
+     * @param expectedStops the expected number of stops
+     * @return the latch; never null
+     */
+    public StartAndStopLatch expectedStops(int expectedStops) {
+        return expectedRestarts(0, expectedStops);
+    }
+
+    /**
+     * Obtain a {@link StartAndStopLatch} that can be used to wait until the expected number of
+     * stops has been completed.
+     *
+     * @param expectedStops the expected number of stops
+     * @param dependents    any dependent latches that must also complete in order for the
+     *                      resulting latch to complete
+     * @return the latch; never null
+     */
+    public StartAndStopLatch expectedStops(int expectedStops, List dependents) {
+        return expectedRestarts(0, expectedStops, dependents);
+    }
+
+    protected void remove(StartAndStopLatch restartLatch) {
+        restartLatches.remove(restartLatch);
+    }
+}
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/StartAndStopCounterTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/StartAndStopCounterTest.java
new file mode 100644
index 0000000000000..7820a6d94d8e3
--- /dev/null
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/StartAndStopCounterTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.connect.integration;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.Time;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class StartAndStopCounterTest {
+
+    private StartAndStopCounter counter;
+    private Time clock;
+    private ExecutorService waiters;
+    private StartAndStopLatch latch;
+
+    @Before
+    public void setup() {
+        clock = new MockTime();
+        counter = new StartAndStopCounter(clock);
+    }
+
+    @After
+    public void teardown() {
+        if (waiters != null) {
+            try {
+                waiters.shutdownNow();
+            } finally {
+                waiters = null;
+            }
+        }
+    }
+
+    @Test
+    public void shouldRecordStarts() {
+        assertEquals(0, counter.starts());
+        counter.recordStart();
+        assertEquals(1, counter.starts());
+        counter.recordStart();
+        assertEquals(2, counter.starts());
+        assertEquals(2, counter.starts());
+    }
+
+    @Test
+    public void shouldRecordStops() {
+        assertEquals(0, counter.stops());
+        counter.recordStop();
+        assertEquals(1, counter.stops());
+        counter.recordStop();
+        assertEquals(2, counter.stops());
+        assertEquals(2, counter.stops());
+    }
+
+    @Test
+    public void shouldExpectRestarts() throws Exception {
+        waiters = Executors.newSingleThreadExecutor();
+
+        latch = counter.expectedRestarts(1);
+        Future future = asyncAwait(100, TimeUnit.MILLISECONDS);
+
+        clock.sleep(1000);
+        counter.recordStop();
+        counter.recordStart();
+        assertTrue(future.get(200, TimeUnit.MILLISECONDS));
+        assertTrue(future.isDone());
+    }
+    @Test
+    public void shouldFailToWaitForRestartThatNeverHappens() throws Exception {
+        waiters = Executors.newSingleThreadExecutor();
+
+        latch = counter.expectedRestarts(1);
+        Future future = asyncAwait(100, TimeUnit.MILLISECONDS);
+
+        clock.sleep(1000);
+        // Record a stop but NOT a start
+        counter.recordStop();
+        assertFalse(future.get(200, TimeUnit.MILLISECONDS));
+        assertTrue(future.isDone());
+    }
+
+    private Future asyncAwait(long duration, TimeUnit unit) {
+        return waiters.submit(() -> {
+            try {
+                return latch.await(duration, unit);
+            } catch (InterruptedException e) {
+                Thread.interrupted();
+                return false;
+            }
+        });
+    }
+}
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/StartAndStopLatch.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/StartAndStopLatch.java
new file mode 100644
index 0000000000000..b77007c5f2e13
--- /dev/null
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/StartAndStopLatch.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.connect.integration;
+
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+import org.apache.kafka.common.utils.Time;
+
+/**
+ * A latch that can be used to count down the number of times a connector and/or tasks have
+ * been started and stopped.
+ */
+public class StartAndStopLatch {
+    private final CountDownLatch startLatch;
+    private final CountDownLatch stopLatch;
+    private final List dependents;
+    private final Consumer uponCompletion;
+    private final Time clock;
+
+    StartAndStopLatch(int expectedStarts, int expectedStops, Consumer uponCompletion,
+                 List dependents, Time clock) {
+        this.startLatch = new CountDownLatch(expectedStarts < 0 ? 0 : expectedStarts);
+        this.stopLatch = new CountDownLatch(expectedStops < 0 ? 0 : expectedStops);
+        this.dependents = dependents;
+        this.uponCompletion = uponCompletion;
+        this.clock = clock;
+    }
+
+    protected void recordStart() {
+        startLatch.countDown();
+    }
+
+    protected void recordStop() {
+        stopLatch.countDown();
+    }
+
+    /**
+     * Causes the current thread to wait until the latch has counted down the starts and
+     * stops to zero, unless the thread is {@linkplain Thread#interrupt interrupted},
+     * or the specified waiting time elapses.
+     *
+     * 
    If the current counts are zero then this method returns immediately
+     * with the value {@code true}.
+     *
+     * 
    If the current count is greater than zero then the current
+     * thread becomes disabled for thread scheduling purposes and lies
+     * dormant until one of three things happen:
+     * 
      
+     * 
    • The counts reach zero due to invocations of the {@link #recordStart()} and
+     * {@link #recordStop()} methods; or
+     * 
    • Some other thread {@linkplain Thread#interrupt interrupts}
+     * the current thread; or
+     * 
    • The specified waiting time elapses.
+     * 
    
+     *
+     * 
    If the count reaches zero then the method returns with the
+     * value {@code true}.
+     *
+     * 
    If the current thread:
+     * 
      
+     * 
    • has its interrupted status set on entry to this method; or
+     * 
    • is {@linkplain Thread#interrupt interrupted} while waiting,
+     * 
    
+     * then {@link InterruptedException} is thrown and the current thread's
+     * interrupted status is cleared.
+     *
+     * 
    If the specified waiting time elapses then the value {@code false}
+     * is returned.  If the time is less than or equal to zero, the method
+     * will not wait at all.
+     *
+     * @param timeout the maximum time to wait
+     * @param unit    the time unit of the {@code timeout} argument
+     * @return {@code true} if the counts reached zero and {@code false}
+     *         if the waiting time elapsed before the counts reached zero
+     * @throws InterruptedException if the current thread is interrupted
+     *         while waiting
+     */
+    public boolean await(long timeout, TimeUnit unit) throws InterruptedException {
+        final long start = clock.milliseconds();
+        final long end = start + unit.toMillis(timeout);
+        if (!startLatch.await(end - start, TimeUnit.MILLISECONDS)) {
+            return false;
+        }
+        if (!stopLatch.await(end - clock.milliseconds(), TimeUnit.MILLISECONDS)) {
+            return false;
+        }
+
+        if (dependents != null) {
+            for (StartAndStopLatch dependent : dependents) {
+                if (!dependent.await(end - clock.milliseconds(), TimeUnit.MILLISECONDS)) {
+                    return false;
+                }
+            }
+        }
+        if (uponCompletion != null) {
+            uponCompletion.accept(this);
+        }
+        return true;
+    }
+}
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/StartAndStopLatchTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/StartAndStopLatchTest.java
new file mode 100644
index 0000000000000..d2732ea4bb28d
--- /dev/null
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/StartAndStopLatchTest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.connect.integration;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.Time;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class StartAndStopLatchTest {
+
+    private Time clock;
+    private StartAndStopLatch latch;
+    private List dependents;
+    private AtomicBoolean completed = new AtomicBoolean();
+    private ExecutorService waiters;
+    private Future future;
+
+    @Before
+    public void setup() {
+        clock = new MockTime();
+        waiters = Executors.newSingleThreadExecutor();
+    }
+
+    @After
+    public void teardown() {
+        if (waiters != null) {
+            waiters.shutdownNow();
+        }
+    }
+
+    @Test
+    public void shouldReturnFalseWhenAwaitingForStartToNeverComplete() throws Throwable {
+        latch = new StartAndStopLatch(1, 1, this::complete, dependents, clock);
+        future = asyncAwait(100);
+        clock.sleep(10);
+        assertFalse(future.get(200, TimeUnit.MILLISECONDS));
+        assertTrue(future.isDone());
+    }
+
+    @Test
+    public void shouldReturnFalseWhenAwaitingForStopToNeverComplete() throws Throwable {
+        latch = new StartAndStopLatch(1, 1, this::complete, dependents, clock);
+        future = asyncAwait(100);
+        latch.recordStart();
+        clock.sleep(10);
+        assertFalse(future.get(200, TimeUnit.MILLISECONDS));
+        assertTrue(future.isDone());
+    }
+
+    @Test
+    public void shouldReturnTrueWhenAwaitingForStartAndStopToComplete() throws Throwable {
+        latch = new StartAndStopLatch(1, 1, this::complete, dependents, clock);
+        future = asyncAwait(100);
+        latch.recordStart();
+        latch.recordStop();
+        clock.sleep(10);
+        assertTrue(future.get(200, TimeUnit.MILLISECONDS));
+        assertTrue(future.isDone());
+    }
+
+    @Test
+    public void shouldReturnFalseWhenAwaitingForDependentLatchToComplete() throws Throwable {
+        StartAndStopLatch depLatch = new StartAndStopLatch(1, 1, this::complete, null, clock);
+        dependents = Collections.singletonList(depLatch);
+        latch = new StartAndStopLatch(1, 1, this::complete, dependents, clock);
+
+        future = asyncAwait(100);
+        latch.recordStart();
+        latch.recordStop();
+        clock.sleep(10);
+        assertFalse(future.get(200, TimeUnit.MILLISECONDS));
+        assertTrue(future.isDone());
+    }
+
+    @Test
+    public void shouldReturnTrueWhenAwaitingForStartAndStopAndDependentLatch() throws Throwable {
+        StartAndStopLatch depLatch = new StartAndStopLatch(1, 1, this::complete, null, clock);
+        dependents = Collections.singletonList(depLatch);
+        latch = new StartAndStopLatch(1, 1, this::complete, dependents, clock);
+
+        future = asyncAwait(100);
+        latch.recordStart();
+        latch.recordStop();
+        depLatch.recordStart();
+        depLatch.recordStop();
+        clock.sleep(10);
+        assertTrue(future.get(200, TimeUnit.MILLISECONDS));
+        assertTrue(future.isDone());
+    }
+
+    private Future asyncAwait(long duration) {
+        return asyncAwait(duration, TimeUnit.MILLISECONDS);
+    }
+
+    private Future asyncAwait(long duration, TimeUnit unit) {
+        return waiters.submit(() -> {
+            try {
+                return latch.await(duration, unit);
+            } catch (InterruptedException e) {
+                Thread.interrupted();
+                return false;
+            }
+        });
+    }
+
+    private void complete(StartAndStopLatch latch) {
+        completed.set(true);
+    }
+}
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/TaskHandle.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/TaskHandle.java
index 6081ea3cdcd7c..1159cb8fcc714 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/TaskHandle.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/TaskHandle.java
@@ -36,6 +36,7 @@ public class TaskHandle {
     private final String taskId;
     private final ConnectorHandle connectorHandle;
     private final AtomicInteger partitionsAssigned = new AtomicInteger(0);
+    private final StartAndStopCounter startAndStopCounter = new StartAndStopCounter();
 
     private CountDownLatch recordsRemainingLatch;
     private CountDownLatch recordsToCommitLatch;
@@ -129,21 +130,33 @@ public int partitionsAssigned() {
     }
 
     /**
-     * Wait for this task to meet the expected number of records as defined by {@code
-     * expectedRecords}.
+     * Wait up to the specified number of milliseconds for this task to meet the expected number of
+     * records as defined by {@code expectedRecords}.
      *
-     * @param  timeout duration to wait for records
+     * @param timeoutMillis number of milliseconds to wait for records
      * @throws InterruptedException if another threads interrupts this one while waiting for records
      */
-    public void awaitRecords(int timeout) throws InterruptedException {
+    public void awaitRecords(long timeoutMillis) throws InterruptedException {
+        awaitRecords(timeoutMillis, TimeUnit.MILLISECONDS);
+    }
+
+    /**
+     * Wait up to the specified timeout for this task to meet the expected number of records as
+     * defined by {@code expectedRecords}.
+     *
+     * @param timeout duration to wait for records
+     * @param unit    the unit of duration; may not be null
+     * @throws InterruptedException if another threads interrupts this one while waiting for records
+     */
+    public void awaitRecords(long timeout, TimeUnit unit) throws InterruptedException {
         if (recordsRemainingLatch == null) {
             throw new IllegalStateException("Illegal state encountered. expectedRecords() was not set for this task?");
         }
-        if (!recordsRemainingLatch.await(timeout, TimeUnit.MILLISECONDS)) {
+        if (!recordsRemainingLatch.await(timeout, unit)) {
             String msg = String.format(
                     "Insufficient records seen by task %s in %d millis. Records expected=%d, actual=%d",
                     taskId,
-                    timeout,
+                    unit.toMillis(timeout),
                     expectedRecords,
                     expectedRecords - recordsRemainingLatch.getCount());
             throw new DataException(msg);
@@ -153,21 +166,33 @@ public void awaitRecords(int timeout) throws InterruptedException {
     }
 
     /**
-     * Wait for this task to meet the expected number of commits as defined by {@code
-     * expectedCommits}.
+     * Wait up to the specified timeout in milliseconds for this task to meet the expected number
+     * of commits as defined by {@code expectedCommits}.
      *
-     * @param  timeout duration to wait for commits
+     * @param timeoutMillis number of milliseconds to wait for commits
      * @throws InterruptedException if another threads interrupts this one while waiting for commits
      */
-    public void awaitCommits(int timeout) throws InterruptedException {
+    public void awaitCommits(long timeoutMillis) throws InterruptedException {
+        awaitCommits(timeoutMillis, TimeUnit.MILLISECONDS);
+    }
+
+    /**
+     * Wait up to the specified timeout for this task to meet the expected number of commits as
+     * defined by {@code expectedCommits}.
+     *
+     * @param timeout duration to wait for commits
+     * @param unit    the unit of duration; may not be null
+     * @throws InterruptedException if another threads interrupts this one while waiting for commits
+     */
+    public void awaitCommits(long timeout, TimeUnit unit) throws InterruptedException {
         if (recordsToCommitLatch == null) {
             throw new IllegalStateException("Illegal state encountered. expectedRecords() was not set for this task?");
         }
-        if (!recordsToCommitLatch.await(timeout, TimeUnit.MILLISECONDS)) {
+        if (!recordsToCommitLatch.await(timeout, unit)) {
             String msg = String.format(
                     "Insufficient records seen by task %s in %d millis. Records expected=%d, actual=%d",
                     taskId,
-                    timeout,
+                    unit.toMillis(timeout),
                     expectedCommits,
                     expectedCommits - recordsToCommitLatch.getCount());
             throw new DataException(msg);
@@ -176,6 +201,42 @@ public void awaitCommits(int timeout) throws InterruptedException {
                   taskId, expectedCommits - recordsToCommitLatch.getCount(), expectedCommits);
     }
 
+    /**
+     * Record that this task has been stopped. This should be called by the task.
+     */
+    public void recordTaskStart() {
+        startAndStopCounter.recordStart();
+    }
+
+    /**
+     * Record that this task has been stopped. This should be called by the task.
+     */
+    public void recordTaskStop() {
+        startAndStopCounter.recordStop();
+    }
+
+    /**
+     * Obtain a {@link StartAndStopLatch} that can be used to wait until this task has completed the
+     * expected number of starts.
+     *
+     * @param expectedStarts    the expected number of starts
+     * @return the latch; never null
+     */
+    public StartAndStopLatch expectedStarts(int expectedStarts) {
+        return startAndStopCounter.expectedStarts(expectedStarts);
+    }
+
+    /**
+     * Obtain a {@link StartAndStopLatch} that can be used to wait until this task has completed the
+     * expected number of starts.
+     *
+     * @param expectedStops    the expected number of stops
+     * @return the latch; never null
+     */
+    public StartAndStopLatch expectedStops(int expectedStops) {
+        return startAndStopCounter.expectedStops(expectedStops);
+    }
+
     @Override
     public String toString() {
         return "Handle{" +

From ff9e95cb09907739c17b4f4681b11c525515b995 Mon Sep 17 00:00:00 2001
From: David Arthur 
Date: Tue, 13 Aug 2019 15:33:05 -0400
Subject: [PATCH 0527/1071] MINOR: Add fetch from follower system test (#7166)

This adds a basic system test that enables rack-aware brokers with the rack-aware replica selector for fetch from followers (KIP-392). The test asserts that the follower was read from at least once and that all the messages that were produced were successfully consumed.

Reviewers: Jason Gustafson 
---
 .../tests/core/fetch_from_follower_test.py    | 144 ++++++++++++++++++
 1 file changed, 144 insertions(+)
 create mode 100644 tests/kafkatest/tests/core/fetch_from_follower_test.py

diff --git a/tests/kafkatest/tests/core/fetch_from_follower_test.py b/tests/kafkatest/tests/core/fetch_from_follower_test.py
new file mode 100644
index 0000000000000..fde1bafe07c41
--- /dev/null
+++ b/tests/kafkatest/tests/core/fetch_from_follower_test.py
@@ -0,0 +1,144 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+from collections import defaultdict
+
+from ducktape.mark.resource import cluster
+
+from kafkatest.directory_layout.kafka_path import KafkaPathResolverMixin
+from kafkatest.services.console_consumer import ConsoleConsumer
+from kafkatest.services.kafka import KafkaService
+from kafkatest.services.monitor.jmx import JmxMixin
+from kafkatest.services.verifiable_producer import VerifiableProducer
+from kafkatest.services.zookeeper import ZookeeperService
+from kafkatest.tests.produce_consume_validate import ProduceConsumeValidateTest
+from kafkatest.utils import is_int
+
+
+class JmxTool(JmxMixin, KafkaPathResolverMixin):
+    """
+    Simple helper class for using the JmxTool directly instead of as a mix-in
+    """
+    def __init__(self, text_context, *args, **kwargs):
+        JmxMixin.__init__(self, num_nodes=1, *args, **kwargs)
+        self.context = text_context
+
+    @property
+    def logger(self):
+        return self.context.logger
+
+
+class FetchFromFollowerTest(ProduceConsumeValidateTest):
+
+    RACK_AWARE_REPLICA_SELECTOR = "org.apache.kafka.common.replica.RackAwareReplicaSelector"
+    METADATA_MAX_AGE_MS = 3000
+
+    def __init__(self, test_context):
+        super(FetchFromFollowerTest, self).__init__(test_context=test_context)
+        self.jmx_tool = JmxTool(test_context, jmx_poll_ms=100)
+        self.topic = "test_topic"
+        self.zk = ZookeeperService(test_context, num_nodes=1)
+        self.kafka = KafkaService(test_context,
+                                  num_nodes=3,
+                                  zk=self.zk,
+                                  topics={
+                                      self.topic: {
+                                          "partitions": 1,
+                                          "replication-factor": 3,
+                                          "configs": {"min.insync.replicas": 1}},
+                                  },
+                                  server_prop_overides=[
+                                      ["replica.selector.class", self.RACK_AWARE_REPLICA_SELECTOR]
+                                  ],
+                                  per_node_server_prop_overrides={
+                                      1: [("broker.rack", "rack-a")],
+                                      2: [("broker.rack", "rack-b")],
+                                      3: [("broker.rack", "rack-c")]
+                                  })
+
+        self.producer_throughput = 1000
+        self.num_producers = 1
+        self.num_consumers = 1
+
+    def min_cluster_size(self):
+        return super(FetchFromFollowerTest, self).min_cluster_size() + self.num_producers * 2 + self.num_consumers * 2
+
+    def setUp(self):
+        self.zk.start()
+        self.kafka.start()
+
+    @cluster(num_nodes=9)
+    def test_consumer_preferred_read_replica(self):
+        """
+        This test starts up brokers with "broker.rack" and "replica.selector.class" configurations set. The replica
+        selector is set to the rack-aware implementation. One of the brokers has a different rack than the other two.
+        We then use a console consumer with the "client.rack" set to the same value as the differing broker. After
+        producing some records, we verify that the client has been informed of the preferred replica and that all the
+        records are properly consumed.
+        """
+
+        # Find the leader, configure consumer to be on a different rack
+        leader_node = self.kafka.leader(self.topic, 0)
+        leader_idx = self.kafka.idx(leader_node)
+        non_leader_idx = 2 if leader_idx != 2 else 1
+        non_leader_rack = "rack-b" if leader_idx != 2 else "rack-a"
+
+        self.logger.debug("Leader %d %s" % (leader_idx, leader_node))
+        self.logger.debug("Non-Leader %d %s" % (non_leader_idx, non_leader_rack))
+
+        self.producer = VerifiableProducer(self.test_context, self.num_producers, self.kafka, self.topic,
+                                           throughput=self.producer_throughput)
+        self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic,
+                                        client_id="console-consumer", group_id="test-consumer-group-1",
+                                        consumer_timeout_ms=60000, message_validator=is_int,
+                                        consumer_properties={"client.rack": non_leader_rack, "metadata.max.age.ms": self.METADATA_MAX_AGE_MS})
+
+        # Start up and let some data get produced
+        self.start_producer_and_consumer()
+        time.sleep(self.METADATA_MAX_AGE_MS * 2. / 1000)
+
+        consumer_node = self.consumer.nodes[0]
+        consumer_idx = self.consumer.idx(consumer_node)
+        read_replica_attribute = "preferred-read-replica"
+        read_replica_mbean = "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=%s,topic=%s,partition=%d" % \
+                  ("console-consumer", self.topic, 0)
+        self.jmx_tool.jmx_object_names = [read_replica_mbean]
+        self.jmx_tool.jmx_attributes = [read_replica_attribute]
+        self.jmx_tool.start_jmx_tool(consumer_idx, consumer_node)
+
+        # Wait for at least one interval of "metadata.max.age.ms"
+        time.sleep(self.METADATA_MAX_AGE_MS * 2. / 1000)
+
+        # Read the JMX output
+        self.jmx_tool.read_jmx_output(consumer_idx, consumer_node)
+
+        all_captured_preferred_read_replicas = defaultdict(int)
+        self.logger.debug(self.jmx_tool.jmx_stats)
+
+        for ts, data in self.jmx_tool.jmx_stats[0].items():
+            for k, v in data.items():
+                if k.endswith(read_replica_attribute):
+                    all_captured_preferred_read_replicas[int(v)] += 1
+
+        self.logger.debug("Saw the following preferred read replicas %s",
+                          dict(all_captured_preferred_read_replicas.items()))
+
+        assert all_captured_preferred_read_replicas[non_leader_idx] > 0, \
+            "Expected to see broker %d (%s) as a preferred replica" % (non_leader_idx, non_leader_rack)
+
+        # Validate consumed messages
+        self.stop_producer_and_consumer()
+        self.validate()

From f8078f380381604f1c7b07204b98e9c08d5e0af1 Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Tue, 13 Aug 2019 14:54:58 -0700
Subject: [PATCH 0528/1071] KAFKA-8736: Track size in InMemoryKeyValueStore
 (#7177)

InMemoryKeyValueStore uses ConcurrentSkipListMap#size which takes linear time as it iterates over the entire map. We should just track size ourselves for approximateNumEntries

Reviewers: Guozhang Wang , Matthias J. Sax 
---
 .../internals/InMemoryKeyValueStore.java      | 23 ++++++++-----------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStore.java
index aa7b2cc17baa8..2d68214371ab1 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStore.java
@@ -35,6 +35,7 @@ public class InMemoryKeyValueStore implements KeyValueStore {
     private final String name;
     private final ConcurrentNavigableMap map = new ConcurrentSkipListMap<>();
     private volatile boolean open = false;
+    private long size = 0L; // SkipListMap#size is O(N) so we just do our best to track it
 
     private static final Logger LOG = LoggerFactory.getLogger(InMemoryKeyValueStore.class);
 
@@ -50,17 +51,10 @@ public String name() {
     @Override
     public void init(final ProcessorContext context,
                      final StateStore root) {
-
+        size = 0;
         if (root != null) {
             // register the store
-            context.register(root, (key, value) -> {
-                // this is a delete
-                if (value == null) {
-                    delete(Bytes.wrap(key));
-                } else {
-                    put(Bytes.wrap(key), value);
-                }
-            });
+            context.register(root, (key, value) -> put(Bytes.wrap(key), value));
         }
 
         open = true;
@@ -84,9 +78,9 @@ public byte[] get(final Bytes key) {
     @Override
     public void put(final Bytes key, final byte[] value) {
         if (value == null) {
-            map.remove(key);
+            size -= map.remove(key) == null ? 0 : 1;
         } else {
-            map.put(key, value);
+            size += map.put(key, value) == null ? 1 : 0;
         }
     }
 
@@ -108,7 +102,9 @@ public void putAll(final List> entries) {
 
     @Override
     public byte[] delete(final Bytes key) {
-        return map.remove(key);
+        final byte[] oldValue = map.remove(key);
+        size -= oldValue == null ? 0 : 1;
+        return oldValue;
     }
 
     @Override
@@ -135,7 +131,7 @@ public KeyValueIterator all() {
 
     @Override
     public long approximateNumEntries() {
-        return map.size();
+        return size;
     }
 
     @Override
@@ -146,6 +142,7 @@ public void flush() {
     @Override
     public void close() {
         map.clear();
+        size = 0;
         open = false;
     }
 

From e07b46dd0424717829f5e12fb0fa91065b943fcd Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Tue, 13 Aug 2019 16:30:48 -0700
Subject: [PATCH 0529/1071] MINOR: remove unnecessary #remove overrides (#7178)

Iterator#remove has a default implementation that throws UnsupportedOperatorException so there's no need to override it with the same thing.

Should be cherry-picked back to whenever we switched to Java 8

Reviewers: Bill Bejeck , Matthias J. Sax , Guozhang Wang 
---
 .../kafka/streams/state/KeyValueIterator.java |  3 ++-
 ...bstractMergedSortedCacheStoreIterator.java |  5 -----
 .../internals/CompositeKeyValueIterator.java  |  4 ----
 .../DelegatingPeekingKeyValueIterator.java    |  5 -----
 .../internals/FilteredCacheIterator.java      |  9 ---------
 .../internals/InMemoryKeyValueStore.java      |  5 -----
 .../state/internals/KeyValueIterators.java    |  3 ---
 .../internals/MemoryNavigableLRUCache.java    |  5 -----
 .../state/internals/MeteredKeyValueStore.java |  5 -----
 .../internals/MeteredWindowStoreIterator.java |  5 -----
 .../MeteredWindowedKeyValueIterator.java      |  5 -----
 .../internals/RocksDBTimestampedStore.java    |  5 -----
 .../state/internals/RocksDbIterator.java      |  5 -----
 .../state/internals/SegmentIterator.java      |  6 ++----
 .../streams/state/internals/ThreadCache.java  |  5 -----
 .../internals/WindowStoreIteratorWrapper.java | 10 ----------
 .../WrappedSessionStoreIterator.java          |  5 -----
 .../kafka/streams/state/NoOpWindowStore.java  |  4 ----
 .../internals/ReadOnlyWindowStoreStub.java    | 20 -------------------
 .../test/GenericInMemoryKeyValueStore.java    |  5 -----
 ...nericInMemoryTimestampedKeyValueStore.java |  5 -----
 .../kafka/test/KeyValueIteratorStub.java      |  4 ----
 .../kafka/test/ReadOnlySessionStoreStub.java  |  4 ----
 23 files changed, 4 insertions(+), 128 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/state/KeyValueIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/KeyValueIterator.java
index 70a142b505c5d..b1f5e2c5485db 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/KeyValueIterator.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/KeyValueIterator.java
@@ -23,9 +23,10 @@
 
 /**
  * Iterator interface of {@link KeyValue}.
- *
+ * 
    
  * Users must call its {@code close} method explicitly upon completeness to release resources,
  * or use try-with-resources statement (available since JDK7) for this {@link Closeable} class.
+ * Note that {@code remove()} is not supported.
  *
  * @param  Type of keys
  * @param  Type of values
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/AbstractMergedSortedCacheStoreIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/AbstractMergedSortedCacheStoreIterator.java
index dfcd763971ff9..16bdbeb8449f4 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/AbstractMergedSortedCacheStoreIterator.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/AbstractMergedSortedCacheStoreIterator.java
@@ -147,11 +147,6 @@ public K peekNextKey() {
         }
     }
 
-    @Override
-    public void remove() {
-        throw new UnsupportedOperationException("remove() is not supported in " + getClass().getName());
-    }
-
     @Override
     public void close() {
         cacheIterator.close();
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeKeyValueIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeKeyValueIterator.java
index faccc161a6e6c..4ac6fee2e6cec 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeKeyValueIterator.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeKeyValueIterator.java
@@ -67,8 +67,4 @@ public KeyValue next() {
         return current.next();
     }
 
-    @Override
-    public void remove() {
-        throw new UnsupportedOperationException("Remove not supported");
-    }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/DelegatingPeekingKeyValueIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/DelegatingPeekingKeyValueIterator.java
index 20a434a9a2c4e..245c9e813f1c1 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/DelegatingPeekingKeyValueIterator.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/DelegatingPeekingKeyValueIterator.java
@@ -78,11 +78,6 @@ public synchronized KeyValue next() {
         return result;
     }
 
-    @Override
-    public void remove() {
-        throw new UnsupportedOperationException("remove() is not supported in " + getClass().getName());
-    }
-
     @Override
     public KeyValue peekNext() {
         if (!hasNext()) {
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/FilteredCacheIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/FilteredCacheIterator.java
index a26f8cf05d024..9b1bfd8b21fd0 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/FilteredCacheIterator.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/FilteredCacheIterator.java
@@ -61,10 +61,6 @@ private KeyValue cachedPair(final KeyValue next() {
 
     }
 
-    @Override
-    public void remove() {
-        throw new UnsupportedOperationException();
-    }
-
     @Override
     public KeyValue peekNext() {
         if (!hasNext()) {
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStore.java
index 2d68214371ab1..9ec5a98f7b76a 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStore.java
@@ -164,11 +164,6 @@ public KeyValue next() {
             return new KeyValue<>(entry.getKey(), entry.getValue());
         }
 
-        @Override
-        public void remove() {
-            iter.remove();
-        }
-
         @Override
         public void close() {
             // do nothing
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/KeyValueIterators.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/KeyValueIterators.java
index bef6f490406a9..29c3009cdf4de 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/KeyValueIterators.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/KeyValueIterators.java
@@ -46,9 +46,6 @@ public KeyValue next() {
             throw new NoSuchElementException();
         }
 
-        @Override
-        public void remove() {
-        }
     }
 
     private static class EmptyWindowStoreIterator extends EmptyKeyValueIterator
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MemoryNavigableLRUCache.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MemoryNavigableLRUCache.java
index 4bf42def92eb7..6e0deaa667eb9 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MemoryNavigableLRUCache.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MemoryNavigableLRUCache.java
@@ -82,11 +82,6 @@ public KeyValue next() {
             return new KeyValue<>(lastKey, entries.get(lastKey));
         }
 
-        @Override
-        public void remove() {
-            // do nothing
-        }
-
         @Override
         public void close() {
             // do nothing
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStore.java
index 51da3ed1bf602..4f2df4c7ff0e4 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStore.java
@@ -305,11 +305,6 @@ public KeyValue next() {
                 outerValue(keyValue.value));
         }
 
-        @Override
-        public void remove() {
-            iter.remove();
-        }
-
         @Override
         public void close() {
             try {
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreIterator.java
index 2cf70a46b35ff..ea665a663eb30 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreIterator.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreIterator.java
@@ -56,11 +56,6 @@ public KeyValue next() {
         return KeyValue.pair(next.key, serdes.valueFrom(next.value));
     }
 
-    @Override
-    public void remove() {
-        iter.remove();
-    }
-
     @Override
     public void close() {
         try {
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowedKeyValueIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowedKeyValueIterator.java
index a12c6eba9bbd3..9e84a18a2f771 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowedKeyValueIterator.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowedKeyValueIterator.java
@@ -63,11 +63,6 @@ private Windowed windowedKey(final Windowed bytesKey) {
         return new Windowed<>(key, bytesKey.window());
     }
 
-    @Override
-    public void remove() {
-        iter.remove();
-    }
-
     @Override
     public void close() {
         try {
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStore.java
index 74f091936d28a..58cd5c4b50293 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStore.java
@@ -341,11 +341,6 @@ public KeyValue makeNext() {
             return next;
         }
 
-        @Override
-        public void remove() {
-            throw new UnsupportedOperationException("RocksDB iterator does not support remove()");
-        }
-
         @Override
         public synchronized void close() {
             openIterators.remove(this);
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDbIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDbIterator.java
index b26e7af251cb3..9fa747a45b9c0 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDbIterator.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDbIterator.java
@@ -67,11 +67,6 @@ private KeyValue getKeyValue() {
         return new KeyValue<>(new Bytes(iter.key()), iter.value());
     }
 
-    @Override
-    public void remove() {
-        throw new UnsupportedOperationException("RocksDB iterator does not support remove()");
-    }
-
     @Override
     public synchronized void close() {
         openIterators.remove(this);
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/SegmentIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/SegmentIterator.java
index 0d90bd8caaeea..e4f828bca512f 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/SegmentIterator.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/SegmentIterator.java
@@ -47,6 +47,7 @@ class SegmentIterator implements KeyValueIterator next() {
         if (!hasNext()) {
             throw new NoSuchElementException();
         }
         return currentIterator.next();
     }
-
-    public void remove() {
-        throw new UnsupportedOperationException("remove() is not supported in " + getClass().getName());
-    }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java
index 116b1ef8b7d1a..1fe99c51b2ff9 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java
@@ -317,11 +317,6 @@ private void internalNext() {
             nextEntry = new KeyValue<>(cacheKey, entry);
         }
 
-        @Override
-        public void remove() {
-            throw new UnsupportedOperationException("remove not supported by MemoryLRUCacheBytesIterator");
-        }
-
         @Override
         public void close() {
             // do nothing
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowStoreIteratorWrapper.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowStoreIteratorWrapper.java
index 4095445a925c2..14acb13be85ed 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowStoreIteratorWrapper.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowStoreIteratorWrapper.java
@@ -66,11 +66,6 @@ public KeyValue next() {
             return KeyValue.pair(timestamp, next.value);
         }
 
-        @Override
-        public void remove() {
-            throw new UnsupportedOperationException("remove() is not supported in " + getClass().getName());
-        }
-
         @Override
         public void close() {
             bytesIterator.close();
@@ -104,11 +99,6 @@ public KeyValue, byte[]> next() {
             return KeyValue.pair(WindowKeySchema.fromStoreBytesKey(next.key.get(), windowSize), next.value);
         }
 
-        @Override
-        public void remove() {
-            throw new UnsupportedOperationException("remove() is not supported in " + getClass().getName());
-        }
-
         @Override
         public void close() {
             bytesIterator.close();
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/WrappedSessionStoreIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/WrappedSessionStoreIterator.java
index 281297c62b037..ce26029af4fbd 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/WrappedSessionStoreIterator.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/WrappedSessionStoreIterator.java
@@ -49,9 +49,4 @@ public KeyValue, byte[]> next() {
         final KeyValue next = bytesIterator.next();
         return KeyValue.pair(SessionKeySchema.from(next.key), next.value);
     }
-
-    @Override
-    public void remove() {
-        throw new UnsupportedOperationException("remove() is not supported in " + getClass().getName());
-    }
 }
\ No newline at end of file
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/NoOpWindowStore.java b/streams/src/test/java/org/apache/kafka/streams/state/NoOpWindowStore.java
index 34d9050c74870..0e8a7b9b30475 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/NoOpWindowStore.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/NoOpWindowStore.java
@@ -45,10 +45,6 @@ public boolean hasNext() {
         public KeyValue next() {
             throw new NoSuchElementException();
         }
-
-        @Override
-        public void remove() {
-        }
     }
 
     private static final WindowStoreIterator EMPTY_WINDOW_STORE_ITERATOR = new EmptyWindowStoreIterator();
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/ReadOnlyWindowStoreStub.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/ReadOnlyWindowStoreStub.java
index aad7403871890..468d551d793c0 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/ReadOnlyWindowStoreStub.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/ReadOnlyWindowStoreStub.java
@@ -121,11 +121,6 @@ public KeyValue, V> next() {
                 return iterator.next();
             }
 
-
-            @Override
-            public void remove() {
-                throw new UnsupportedOperationException("remove() not supported in " + getClass().getName());
-            }
         };
     }
 
@@ -168,11 +163,6 @@ public KeyValue, V> next() {
                 return iterator.next();
             }
 
-
-            @Override
-            public void remove() {
-                throw new UnsupportedOperationException("remove() not supported in " + getClass().getName());
-            }
         };
     }
 
@@ -219,11 +209,6 @@ public KeyValue, V> next() {
                 return iterator.next();
             }
 
-
-            @Override
-            public void remove() {
-                throw new UnsupportedOperationException("remove() not supported in " + getClass().getName());
-            }
         };
     }
 
@@ -298,10 +283,5 @@ public boolean hasNext() {
         public KeyValue next() {
             return underlying.next();
         }
-
-        @Override
-        public void remove() {
-            throw new UnsupportedOperationException("remove() not supported in " + getClass().getName());
-        }
     }
 }
diff --git a/streams/src/test/java/org/apache/kafka/test/GenericInMemoryKeyValueStore.java b/streams/src/test/java/org/apache/kafka/test/GenericInMemoryKeyValueStore.java
index a5e345efc5b45..649dc5b377293 100644
--- a/streams/src/test/java/org/apache/kafka/test/GenericInMemoryKeyValueStore.java
+++ b/streams/src/test/java/org/apache/kafka/test/GenericInMemoryKeyValueStore.java
@@ -171,11 +171,6 @@ public KeyValue next() {
             return new KeyValue<>(entry.getKey(), entry.getValue());
         }
 
-        @Override
-        public void remove() {
-            iter.remove();
-        }
-
         @Override
         public void close() {
             // do nothing
diff --git a/streams/src/test/java/org/apache/kafka/test/GenericInMemoryTimestampedKeyValueStore.java b/streams/src/test/java/org/apache/kafka/test/GenericInMemoryTimestampedKeyValueStore.java
index 67a67c925bd3f..b1b75a16324bd 100644
--- a/streams/src/test/java/org/apache/kafka/test/GenericInMemoryTimestampedKeyValueStore.java
+++ b/streams/src/test/java/org/apache/kafka/test/GenericInMemoryTimestampedKeyValueStore.java
@@ -172,11 +172,6 @@ public KeyValue> next() {
             return new KeyValue<>(entry.getKey(), entry.getValue());
         }
 
-        @Override
-        public void remove() {
-            iter.remove();
-        }
-
         @Override
         public void close() {
             // do nothing
diff --git a/streams/src/test/java/org/apache/kafka/test/KeyValueIteratorStub.java b/streams/src/test/java/org/apache/kafka/test/KeyValueIteratorStub.java
index 2ee5a6fcf9e22..aa3a4e99b58bd 100644
--- a/streams/src/test/java/org/apache/kafka/test/KeyValueIteratorStub.java
+++ b/streams/src/test/java/org/apache/kafka/test/KeyValueIteratorStub.java
@@ -49,8 +49,4 @@ public KeyValue next() {
         return iterator.next();
     }
 
-    @Override
-    public void remove() {
-
-    }
 }
diff --git a/streams/src/test/java/org/apache/kafka/test/ReadOnlySessionStoreStub.java b/streams/src/test/java/org/apache/kafka/test/ReadOnlySessionStoreStub.java
index 39ff614751e44..4f6d5debb0ada 100644
--- a/streams/src/test/java/org/apache/kafka/test/ReadOnlySessionStoreStub.java
+++ b/streams/src/test/java/org/apache/kafka/test/ReadOnlySessionStoreStub.java
@@ -83,10 +83,6 @@ public KeyValue, V> next() {
                     return it.next();
                 }
 
-                @Override
-                public void remove() {
-                    throw new UnsupportedOperationException();
-                }
             }
         );
     }

From 245e9caa4418f9c2241ce1e518d5159ec6e53ad1 Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Tue, 13 Aug 2019 18:37:45 -0700
Subject: [PATCH 0530/1071] KAFKA-8791: RocksDBTimestampedStore should open in
 regular mode by default (#7201)

Reviewers: A. Sophie Blee-Goldman , Bill Bejeck , Richard Yu , Guozhang Wang 
---
 .../internals/RocksDBTimestampedStore.java    |  32 +-
 .../RocksDBTimestampedStoreTest.java          | 341 ++++++++++++------
 2 files changed, 240 insertions(+), 133 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStore.java
index 58cd5c4b50293..820f7f512fcc0 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStore.java
@@ -70,20 +70,7 @@ void openRocksDB(final DBOptions dbOptions,
 
         try {
             db = RocksDB.open(dbOptions, dbDir.getAbsolutePath(), columnFamilyDescriptors, columnFamilies);
-
-            final ColumnFamilyHandle noTimestampColumnFamily = columnFamilies.get(0);
-
-            final RocksIterator noTimestampsIter = db.newIterator(noTimestampColumnFamily);
-            noTimestampsIter.seekToFirst();
-            if (noTimestampsIter.isValid()) {
-                log.info("Opening store {} in upgrade mode", name);
-                dbAccessor = new DualColumnFamilyAccessor(noTimestampColumnFamily, columnFamilies.get(1));
-            } else {
-                log.info("Opening store {} in regular mode", name);
-                dbAccessor = new SingleColumnFamilyAccessor(columnFamilies.get(1));
-                noTimestampColumnFamily.close();
-            }
-            noTimestampsIter.close();
+            setDbAccessor(columnFamilies.get(0), columnFamilies.get(1));
         } catch (final RocksDBException e) {
             if ("Column family not found: : keyValueWithTimestamp".equals(e.getMessage())) {
                 try {
@@ -92,14 +79,27 @@ void openRocksDB(final DBOptions dbOptions,
                 } catch (final RocksDBException fatal) {
                     throw new ProcessorStateException("Error opening store " + name + " at location " + dbDir.toString(), fatal);
                 }
-                log.info("Opening store {} in upgrade mode", name);
-                dbAccessor = new DualColumnFamilyAccessor(columnFamilies.get(0), columnFamilies.get(1));
+                setDbAccessor(columnFamilies.get(0), columnFamilies.get(1));
             } else {
                 throw new ProcessorStateException("Error opening store " + name + " at location " + dbDir.toString(), e);
             }
         }
     }
 
+    private void setDbAccessor(final ColumnFamilyHandle noTimestampColumnFamily,
+                               final ColumnFamilyHandle withTimestampColumnFamily) {
+        final RocksIterator noTimestampsIter = db.newIterator(noTimestampColumnFamily);
+        noTimestampsIter.seekToFirst();
+        if (noTimestampsIter.isValid()) {
+            log.info("Opening store {} in upgrade mode", name);
+            dbAccessor = new DualColumnFamilyAccessor(noTimestampColumnFamily, withTimestampColumnFamily);
+        } else {
+            log.info("Opening store {} in regular mode", name);
+            dbAccessor = new SingleColumnFamilyAccessor(withTimestampColumnFamily);
+            noTimestampColumnFamily.close();
+        }
+        noTimestampsIter.close();
+    }
 
 
     private class DualColumnFamilyAccessor implements RocksDBAccessor {
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStoreTest.java
index f49527b0d7a5a..e7ad30d2d0bc0 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStoreTest.java
@@ -46,6 +46,79 @@ RocksDBStore getRocksDBStore() {
         return new RocksDBTimestampedStore(DB_NAME);
     }
 
+    @Test
+    public void shouldOpenNewStoreInRegularMode() {
+        LogCaptureAppender.setClassLoggerToDebug(RocksDBTimestampedStore.class);
+
+        final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
+        rocksDBStore.init(context, rocksDBStore);
+        assertThat(appender.getMessages(), hasItem("Opening store " + DB_NAME + " in regular mode"));
+        LogCaptureAppender.unregister(appender);
+
+        try (final KeyValueIterator iterator = rocksDBStore.all()) {
+            assertThat(iterator.hasNext(), is(false));
+        }
+    }
+
+    @Test
+    public void shouldOpenExistingStoreInRegularMode() throws Exception {
+        LogCaptureAppender.setClassLoggerToDebug(RocksDBTimestampedStore.class);
+
+        // prepare store
+        rocksDBStore.init(context, rocksDBStore);
+        rocksDBStore.put(new Bytes("key".getBytes()), "timestamped".getBytes());
+        rocksDBStore.close();
+
+        // re-open store
+        final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
+        rocksDBStore = getRocksDBStore();
+        rocksDBStore.init(context, rocksDBStore);
+        assertThat(appender.getMessages(), hasItem("Opening store " + DB_NAME + " in regular mode"));
+        LogCaptureAppender.unregister(appender);
+
+        rocksDBStore.close();
+
+        // verify store
+        final DBOptions dbOptions = new DBOptions();
+        final ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions();
+
+        final List columnFamilyDescriptors = asList(
+            new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnFamilyOptions),
+            new ColumnFamilyDescriptor("keyValueWithTimestamp".getBytes(StandardCharsets.UTF_8), columnFamilyOptions));
+        final List columnFamilies = new ArrayList<>(columnFamilyDescriptors.size());
+
+        RocksDB db = null;
+        ColumnFamilyHandle noTimestampColumnFamily = null, withTimestampColumnFamily = null;
+        try {
+            db = RocksDB.open(
+                dbOptions,
+                new File(new File(context.stateDir(), "rocksdb"), DB_NAME).getAbsolutePath(),
+                columnFamilyDescriptors,
+                columnFamilies);
+
+            noTimestampColumnFamily = columnFamilies.get(0);
+            withTimestampColumnFamily = columnFamilies.get(1);
+
+            assertThat(db.get(noTimestampColumnFamily, "key".getBytes()), new IsNull<>());
+            assertThat(db.getLongProperty(noTimestampColumnFamily, "rocksdb.estimate-num-keys"), is(0L));
+            assertThat(db.get(withTimestampColumnFamily, "key".getBytes()).length, is(11));
+            assertThat(db.getLongProperty(withTimestampColumnFamily, "rocksdb.estimate-num-keys"), is(1L));
+        } finally {
+            // Order of closing must follow: ColumnFamilyHandle > RocksDB > DBOptions > ColumnFamilyOptions
+            if (noTimestampColumnFamily != null) {
+                noTimestampColumnFamily.close();
+            }
+            if (withTimestampColumnFamily != null) {
+                withTimestampColumnFamily.close();
+            }
+            if (db != null) {
+                db.close();
+            }
+            dbOptions.close();
+            columnFamilyOptions.close();
+        }
+    }
+
     @Test
     public void shouldMigrateDataFromDefaultToTimestampColumnFamily() throws Exception {
         prepareOldStore();
@@ -139,70 +212,70 @@ public void shouldMigrateDataFromDefaultToTimestampColumnFamily() throws Excepti
 
     private void iteratorsShouldNotMigrateData() {
         // iterating should not migrate any data, but return all key over both CF (plus surrogate timestamps for old CF)
-        final KeyValueIterator itAll = rocksDBStore.all();
-        {
-            final KeyValue keyValue = itAll.next();
-            assertArrayEquals("key1".getBytes(), keyValue.key.get());
-            // unknown timestamp == -1 plus value == 1
-            assertArrayEquals(new byte[]{-1, -1, -1, -1, -1, -1, -1, -1, '1'}, keyValue.value);
-        }
-        {
-            final KeyValue keyValue = itAll.next();
-            assertArrayEquals("key11".getBytes(), keyValue.key.get());
-            assertArrayEquals(new byte[]{'t', 'i', 'm', 'e', 's', 't', 'a', 'm', 'p', '+', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1'}, keyValue.value);
-        }
-        {
-            final KeyValue keyValue = itAll.next();
-            assertArrayEquals("key2".getBytes(), keyValue.key.get());
-            assertArrayEquals(new byte[]{'t', 'i', 'm', 'e', 's', 't', 'a', 'm', 'p', '+', '2', '2'}, keyValue.value);
-        }
-        {
-            final KeyValue keyValue = itAll.next();
-            assertArrayEquals("key4".getBytes(), keyValue.key.get());
-            // unknown timestamp == -1 plus value == 4444
-            assertArrayEquals(new byte[]{-1, -1, -1, -1, -1, -1, -1, -1, '4', '4', '4', '4'}, keyValue.value);
-        }
-        {
-            final KeyValue keyValue = itAll.next();
-            assertArrayEquals("key5".getBytes(), keyValue.key.get());
-            // unknown timestamp == -1 plus value == 55555
-            assertArrayEquals(new byte[]{-1, -1, -1, -1, -1, -1, -1, -1, '5', '5', '5', '5', '5'}, keyValue.value);
+        try (final KeyValueIterator itAll = rocksDBStore.all()) {
+            {
+                final KeyValue keyValue = itAll.next();
+                assertArrayEquals("key1".getBytes(), keyValue.key.get());
+                // unknown timestamp == -1 plus value == 1
+                assertArrayEquals(new byte[]{-1, -1, -1, -1, -1, -1, -1, -1, '1'}, keyValue.value);
+            }
+            {
+                final KeyValue keyValue = itAll.next();
+                assertArrayEquals("key11".getBytes(), keyValue.key.get());
+                assertArrayEquals(new byte[]{'t', 'i', 'm', 'e', 's', 't', 'a', 'm', 'p', '+', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1'}, keyValue.value);
+            }
+            {
+                final KeyValue keyValue = itAll.next();
+                assertArrayEquals("key2".getBytes(), keyValue.key.get());
+                assertArrayEquals(new byte[]{'t', 'i', 'm', 'e', 's', 't', 'a', 'm', 'p', '+', '2', '2'}, keyValue.value);
+            }
+            {
+                final KeyValue keyValue = itAll.next();
+                assertArrayEquals("key4".getBytes(), keyValue.key.get());
+                // unknown timestamp == -1 plus value == 4444
+                assertArrayEquals(new byte[]{-1, -1, -1, -1, -1, -1, -1, -1, '4', '4', '4', '4'}, keyValue.value);
+            }
+            {
+                final KeyValue keyValue = itAll.next();
+                assertArrayEquals("key5".getBytes(), keyValue.key.get());
+                // unknown timestamp == -1 plus value == 55555
+                assertArrayEquals(new byte[]{-1, -1, -1, -1, -1, -1, -1, -1, '5', '5', '5', '5', '5'}, keyValue.value);
+            }
+            {
+                final KeyValue keyValue = itAll.next();
+                assertArrayEquals("key7".getBytes(), keyValue.key.get());
+                // unknown timestamp == -1 plus value == 7777777
+                assertArrayEquals(new byte[]{-1, -1, -1, -1, -1, -1, -1, -1, '7', '7', '7', '7', '7', '7', '7'}, keyValue.value);
+            }
+            {
+                final KeyValue keyValue = itAll.next();
+                assertArrayEquals("key8".getBytes(), keyValue.key.get());
+                assertArrayEquals(new byte[]{'t', 'i', 'm', 'e', 's', 't', 'a', 'm', 'p', '+', '8', '8', '8', '8', '8', '8', '8', '8'}, keyValue.value);
+            }
+            assertFalse(itAll.hasNext());
         }
-        {
-            final KeyValue keyValue = itAll.next();
-            assertArrayEquals("key7".getBytes(), keyValue.key.get());
-            // unknown timestamp == -1 plus value == 7777777
-            assertArrayEquals(new byte[]{-1, -1, -1, -1, -1, -1, -1, -1, '7', '7', '7', '7', '7', '7', '7'}, keyValue.value);
-        }
-        {
-            final KeyValue keyValue = itAll.next();
-            assertArrayEquals("key8".getBytes(), keyValue.key.get());
-            assertArrayEquals(new byte[]{'t', 'i', 'm', 'e', 's', 't', 'a', 'm', 'p', '+', '8', '8', '8', '8', '8', '8', '8', '8'}, keyValue.value);
-        }
-        assertFalse(itAll.hasNext());
-        itAll.close();
-
-        final KeyValueIterator it =
-            rocksDBStore.range(new Bytes("key2".getBytes()), new Bytes("key5".getBytes()));
-        {
-            final KeyValue keyValue = it.next();
-            assertArrayEquals("key2".getBytes(), keyValue.key.get());
-            assertArrayEquals(new byte[]{'t', 'i', 'm', 'e', 's', 't', 'a', 'm', 'p', '+', '2', '2'}, keyValue.value);
-        }
-        {
-            final KeyValue keyValue = it.next();
-            assertArrayEquals("key4".getBytes(), keyValue.key.get());
-            // unknown timestamp == -1 plus value == 4444
-            assertArrayEquals(new byte[]{-1, -1, -1, -1, -1, -1, -1, -1, '4', '4', '4', '4'}, keyValue.value);
-        }
-        {
-            final KeyValue keyValue = it.next();
-            assertArrayEquals("key5".getBytes(), keyValue.key.get());
-            // unknown timestamp == -1 plus value == 55555
-            assertArrayEquals(new byte[]{-1, -1, -1, -1, -1, -1, -1, -1, '5', '5', '5', '5', '5'}, keyValue.value);
+
+        try (final KeyValueIterator it =
+                rocksDBStore.range(new Bytes("key2".getBytes()), new Bytes("key5".getBytes()))) {
+            {
+                final KeyValue keyValue = it.next();
+                assertArrayEquals("key2".getBytes(), keyValue.key.get());
+                assertArrayEquals(new byte[]{'t', 'i', 'm', 'e', 's', 't', 'a', 'm', 'p', '+', '2', '2'}, keyValue.value);
+            }
+            {
+                final KeyValue keyValue = it.next();
+                assertArrayEquals("key4".getBytes(), keyValue.key.get());
+                // unknown timestamp == -1 plus value == 4444
+                assertArrayEquals(new byte[]{-1, -1, -1, -1, -1, -1, -1, -1, '4', '4', '4', '4'}, keyValue.value);
+            }
+            {
+                final KeyValue keyValue = it.next();
+                assertArrayEquals("key5".getBytes(), keyValue.key.get());
+                // unknown timestamp == -1 plus value == 55555
+                assertArrayEquals(new byte[]{-1, -1, -1, -1, -1, -1, -1, -1, '5', '5', '5', '5', '5'}, keyValue.value);
+            }
+            assertFalse(it.hasNext());
         }
-        assertFalse(it.hasNext());
-        it.close();
     }
 
     private void verifyOldAndNewColumnFamily() throws Exception {
@@ -214,40 +287,60 @@ private void verifyOldAndNewColumnFamily() throws Exception {
             new ColumnFamilyDescriptor("keyValueWithTimestamp".getBytes(StandardCharsets.UTF_8), columnFamilyOptions));
         final List columnFamilies = new ArrayList<>(columnFamilyDescriptors.size());
 
-        RocksDB db = RocksDB.open(
-            dbOptions,
-            new File(new File(context.stateDir(), "rocksdb"), DB_NAME).getAbsolutePath(),
-            columnFamilyDescriptors,
-            columnFamilies);
-
-        ColumnFamilyHandle noTimestampColumnFamily = columnFamilies.get(0);
-        final ColumnFamilyHandle withTimestampColumnFamily = columnFamilies.get(1);
-
-        assertThat(db.get(noTimestampColumnFamily, "unknown".getBytes()), new IsNull<>());
-        assertThat(db.get(noTimestampColumnFamily, "key1".getBytes()), new IsNull<>());
-        assertThat(db.get(noTimestampColumnFamily, "key2".getBytes()), new IsNull<>());
-        assertThat(db.get(noTimestampColumnFamily, "key3".getBytes()), new IsNull<>());
-        assertThat(db.get(noTimestampColumnFamily, "key4".getBytes()), new IsNull<>());
-        assertThat(db.get(noTimestampColumnFamily, "key5".getBytes()), new IsNull<>());
-        assertThat(db.get(noTimestampColumnFamily, "key6".getBytes()), new IsNull<>());
-        assertThat(db.get(noTimestampColumnFamily, "key7".getBytes()).length, is(7));
-        assertThat(db.get(noTimestampColumnFamily, "key8".getBytes()), new IsNull<>());
-        assertThat(db.get(noTimestampColumnFamily, "key11".getBytes()), new IsNull<>());
-        assertThat(db.get(noTimestampColumnFamily, "key12".getBytes()), new IsNull<>());
-
-        assertThat(db.get(withTimestampColumnFamily, "unknown".getBytes()), new IsNull<>());
-        assertThat(db.get(withTimestampColumnFamily, "key1".getBytes()).length, is(8 + 1));
-        assertThat(db.get(withTimestampColumnFamily, "key2".getBytes()).length, is(12));
-        assertThat(db.get(withTimestampColumnFamily, "key3".getBytes()), new IsNull<>());
-        assertThat(db.get(withTimestampColumnFamily, "key4".getBytes()).length, is(8 + 4));
-        assertThat(db.get(withTimestampColumnFamily, "key5".getBytes()).length, is(8 + 5));
-        assertThat(db.get(withTimestampColumnFamily, "key6".getBytes()), new IsNull<>());
-        assertThat(db.get(withTimestampColumnFamily, "key7".getBytes()), new IsNull<>());
-        assertThat(db.get(withTimestampColumnFamily, "key8".getBytes()).length, is(18));
-        assertThat(db.get(withTimestampColumnFamily, "key11".getBytes()).length, is(21));
-        assertThat(db.get(withTimestampColumnFamily, "key12".getBytes()), new IsNull<>());
-
-        db.close();
+        RocksDB db = null;
+        ColumnFamilyHandle noTimestampColumnFamily = null, withTimestampColumnFamily = null;
+        boolean errorOccurred = false;
+        try {
+            db = RocksDB.open(
+                dbOptions,
+                new File(new File(context.stateDir(), "rocksdb"), DB_NAME).getAbsolutePath(),
+                columnFamilyDescriptors,
+                columnFamilies);
+
+            noTimestampColumnFamily = columnFamilies.get(0);
+            withTimestampColumnFamily = columnFamilies.get(1);
+
+            assertThat(db.get(noTimestampColumnFamily, "unknown".getBytes()), new IsNull<>());
+            assertThat(db.get(noTimestampColumnFamily, "key1".getBytes()), new IsNull<>());
+            assertThat(db.get(noTimestampColumnFamily, "key2".getBytes()), new IsNull<>());
+            assertThat(db.get(noTimestampColumnFamily, "key3".getBytes()), new IsNull<>());
+            assertThat(db.get(noTimestampColumnFamily, "key4".getBytes()), new IsNull<>());
+            assertThat(db.get(noTimestampColumnFamily, "key5".getBytes()), new IsNull<>());
+            assertThat(db.get(noTimestampColumnFamily, "key6".getBytes()), new IsNull<>());
+            assertThat(db.get(noTimestampColumnFamily, "key7".getBytes()).length, is(7));
+            assertThat(db.get(noTimestampColumnFamily, "key8".getBytes()), new IsNull<>());
+            assertThat(db.get(noTimestampColumnFamily, "key11".getBytes()), new IsNull<>());
+            assertThat(db.get(noTimestampColumnFamily, "key12".getBytes()), new IsNull<>());
+
+            assertThat(db.get(withTimestampColumnFamily, "unknown".getBytes()), new IsNull<>());
+            assertThat(db.get(withTimestampColumnFamily, "key1".getBytes()).length, is(8 + 1));
+            assertThat(db.get(withTimestampColumnFamily, "key2".getBytes()).length, is(12));
+            assertThat(db.get(withTimestampColumnFamily, "key3".getBytes()), new IsNull<>());
+            assertThat(db.get(withTimestampColumnFamily, "key4".getBytes()).length, is(8 + 4));
+            assertThat(db.get(withTimestampColumnFamily, "key5".getBytes()).length, is(8 + 5));
+            assertThat(db.get(withTimestampColumnFamily, "key6".getBytes()), new IsNull<>());
+            assertThat(db.get(withTimestampColumnFamily, "key7".getBytes()), new IsNull<>());
+            assertThat(db.get(withTimestampColumnFamily, "key8".getBytes()).length, is(18));
+            assertThat(db.get(withTimestampColumnFamily, "key11".getBytes()).length, is(21));
+            assertThat(db.get(withTimestampColumnFamily, "key12".getBytes()), new IsNull<>());
+        } catch (final RuntimeException fatal) {
+            errorOccurred = true;
+        } finally {
+            // Order of closing must follow: ColumnFamilyHandle > RocksDB > DBOptions > ColumnFamilyOptions
+            if (noTimestampColumnFamily != null) {
+                noTimestampColumnFamily.close();
+            }
+            if (withTimestampColumnFamily != null) {
+                withTimestampColumnFamily.close();
+            }
+            if (db != null) {
+                db.close();
+            }
+            if (errorOccurred) {
+                dbOptions.close();
+                columnFamilyOptions.close();
+            }
+        }
 
         // check that still in upgrade mode
         LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
@@ -258,15 +351,28 @@ private void verifyOldAndNewColumnFamily() throws Exception {
 
         // clear old CF
         columnFamilies.clear();
-        db = RocksDB.open(
-            dbOptions,
-            new File(new File(context.stateDir(), "rocksdb"), DB_NAME).getAbsolutePath(),
-            columnFamilyDescriptors,
-            columnFamilies);
-
-        noTimestampColumnFamily = columnFamilies.get(0);
-        db.delete(noTimestampColumnFamily, "key7".getBytes());
-        db.close();
+        db = null;
+        noTimestampColumnFamily = null;
+        try {
+            db = RocksDB.open(
+                dbOptions,
+                new File(new File(context.stateDir(), "rocksdb"), DB_NAME).getAbsolutePath(),
+                columnFamilyDescriptors,
+                columnFamilies);
+
+            noTimestampColumnFamily = columnFamilies.get(0);
+            db.delete(noTimestampColumnFamily, "key7".getBytes());
+        } finally {
+            // Order of closing must follow: ColumnFamilyHandle > RocksDB > DBOptions > ColumnFamilyOptions
+            if (noTimestampColumnFamily != null) {
+                noTimestampColumnFamily.close();
+            }
+            if (db != null) {
+                db.close();
+            }
+            dbOptions.close();
+            columnFamilyOptions.close();
+        }
 
         // check that still in regular mode
         appender = LogCaptureAppender.createAndRegister();
@@ -277,17 +383,18 @@ private void verifyOldAndNewColumnFamily() throws Exception {
 
     private void prepareOldStore() {
         final RocksDBStore keyValueStore = new RocksDBStore(DB_NAME);
-        keyValueStore.init(context, keyValueStore);
-
-        keyValueStore.put(new Bytes("key1".getBytes()), "1".getBytes());
-        keyValueStore.put(new Bytes("key2".getBytes()), "22".getBytes());
-        keyValueStore.put(new Bytes("key3".getBytes()), "333".getBytes());
-        keyValueStore.put(new Bytes("key4".getBytes()), "4444".getBytes());
-        keyValueStore.put(new Bytes("key5".getBytes()), "55555".getBytes());
-        keyValueStore.put(new Bytes("key6".getBytes()), "666666".getBytes());
-        keyValueStore.put(new Bytes("key7".getBytes()), "7777777".getBytes());
-
-        keyValueStore.close();
+        try {
+            keyValueStore.init(context, keyValueStore);
+
+            keyValueStore.put(new Bytes("key1".getBytes()), "1".getBytes());
+            keyValueStore.put(new Bytes("key2".getBytes()), "22".getBytes());
+            keyValueStore.put(new Bytes("key3".getBytes()), "333".getBytes());
+            keyValueStore.put(new Bytes("key4".getBytes()), "4444".getBytes());
+            keyValueStore.put(new Bytes("key5".getBytes()), "55555".getBytes());
+            keyValueStore.put(new Bytes("key6".getBytes()), "666666".getBytes());
+            keyValueStore.put(new Bytes("key7".getBytes()), "7777777".getBytes());
+        } finally {
+            keyValueStore.close();
+        }
     }
-
 }

From 2ca30191d2bb430a4b27b125177406cf263d444f Mon Sep 17 00:00:00 2001
From: Lucas Bradstreet 
Date: Tue, 13 Aug 2019 21:35:03 -0700
Subject: [PATCH 0531/1071] MINOR: Avoid unnecessary leaderFor calls when
 ProducerBatch queue empty (#7196)

The RecordAccumulator ready calls `leaderFor` unnecessarily when the ProducerBatch
queue is empty. When producing to many partitions, the queue is often empty and the
`leaderFor` call can be expensive in comparison. Remove the unnecessary call.

Reviewers: Ismael Juma 
---
 .../producer/internals/RecordAccumulator.java | 21 ++++++++++---------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
index fb335e4229612..745382d88a720 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
@@ -459,18 +459,19 @@ public ReadyCheckResult ready(Cluster cluster, long nowMs) {
 
         boolean exhausted = this.free.queued() > 0;
         for (Map.Entry> entry : this.batches.entrySet()) {
-            TopicPartition part = entry.getKey();
             Deque deque = entry.getValue();
-
-            Node leader = cluster.leaderFor(part);
             synchronized (deque) {
-                if (leader == null && !deque.isEmpty()) {
-                    // This is a partition for which leader is not known, but messages are available to send.
-                    // Note that entries are currently not removed from batches when deque is empty.
-                    unknownLeaderTopics.add(part.topic());
-                } else if (!readyNodes.contains(leader) && !isMuted(part, nowMs)) {
-                    ProducerBatch batch = deque.peekFirst();
-                    if (batch != null) {
+                // When producing to a large number of partitions, this path is hot and deques are often empty.
+                // We check whether a batch exists first to avoid the more expensive checks whenever possible.
+                ProducerBatch batch = deque.peekFirst();
+                if (batch != null) {
+                    TopicPartition part = entry.getKey();
+                    Node leader = cluster.leaderFor(part);
+                    if (leader == null) {
+                        // This is a partition for which leader is not known, but messages are available to send.
+                        // Note that entries are currently not removed from batches when deque is empty.
+                        unknownLeaderTopics.add(part.topic());
+                    } else if (!readyNodes.contains(leader) && !isMuted(part, nowMs)) {
                         long waitedTimeMs = batch.waitedTimeMs(nowMs);
                         boolean backingOff = batch.attempts() > 0 && waitedTimeMs < retryBackoffMs;
                         long timeToWaitMs = backingOff ? retryBackoffMs : lingerMs;

From 39158bd944f7c64e32634cec8171515b93466ffa Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Tue, 13 Aug 2019 22:39:42 -0700
Subject: [PATCH 0532/1071] KAFKA-8765: Remove interface annotations in Streams
 API (#7174)

Reviewers: Bruno Cadonna , Guozhang Wang 
---
 .../src/main/java/org/apache/kafka/streams/KafkaStreams.java   | 2 --
 .../src/main/java/org/apache/kafka/streams/StreamsMetrics.java | 2 --
 .../java/org/apache/kafka/streams/kstream/GlobalKTable.java    | 2 --
 .../java/org/apache/kafka/streams/kstream/KGroupedStream.java  | 2 --
 .../java/org/apache/kafka/streams/kstream/KGroupedTable.java   | 2 --
 .../main/java/org/apache/kafka/streams/kstream/KStream.java    | 2 --
 .../src/main/java/org/apache/kafka/streams/kstream/KTable.java | 2 --
 .../org/apache/kafka/streams/kstream/internals/TimeWindow.java | 2 --
 .../kafka/streams/kstream/internals/UnlimitedWindow.java       | 2 --
 .../streams/processor/ExtractRecordMetadataTimestamp.java      | 2 --
 .../apache/kafka/streams/processor/FailOnInvalidTimestamp.java | 2 --
 .../kafka/streams/processor/LogAndSkipOnInvalidTimestamp.java  | 2 --
 .../java/org/apache/kafka/streams/processor/Processor.java     | 3 ---
 .../org/apache/kafka/streams/processor/ProcessorContext.java   | 2 --
 .../apache/kafka/streams/processor/StateRestoreCallback.java   | 3 ---
 .../org/apache/kafka/streams/processor/TimestampExtractor.java | 2 --
 .../org/apache/kafka/streams/processor/TopicNameExtractor.java | 3 ---
 .../streams/processor/UsePreviousTimeOnInvalidTimestamp.java   | 2 --
 .../kafka/streams/processor/WallclockTimestampExtractor.java   | 2 --
 .../src/main/java/org/apache/kafka/streams/state/Stores.java   | 2 --
 .../java/org/apache/kafka/streams/state/StreamsMetadata.java   | 3 +--
 .../main/java/org/apache/kafka/streams/TopologyTestDriver.java | 2 --
 .../apache/kafka/streams/processor/MockProcessorContext.java   | 2 --
 .../org/apache/kafka/streams/test/ConsumerRecordFactory.java   | 2 --
 24 files changed, 1 insertion(+), 51 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
index 412d5e4166947..a0cc7a5204423 100644
--- a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
+++ b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
@@ -23,7 +23,6 @@
 import org.apache.kafka.common.Metric;
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.common.config.ConfigException;
 import org.apache.kafka.common.metrics.JmxReporter;
 import org.apache.kafka.common.metrics.MetricConfig;
@@ -122,7 +121,6 @@
  * @see org.apache.kafka.streams.StreamsBuilder
  * @see org.apache.kafka.streams.Topology
  */
-@InterfaceStability.Evolving
 public class KafkaStreams implements AutoCloseable {
 
     private static final String JMX_PREFIX = "kafka.streams";
diff --git a/streams/src/main/java/org/apache/kafka/streams/StreamsMetrics.java b/streams/src/main/java/org/apache/kafka/streams/StreamsMetrics.java
index d4f685b5debd4..db16a784fcfb7 100644
--- a/streams/src/main/java/org/apache/kafka/streams/StreamsMetrics.java
+++ b/streams/src/main/java/org/apache/kafka/streams/StreamsMetrics.java
@@ -18,7 +18,6 @@
 
 import org.apache.kafka.common.Metric;
 import org.apache.kafka.common.MetricName;
-import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.common.metrics.Sensor;
 
 import java.util.Map;
@@ -26,7 +25,6 @@
 /**
  * The Kafka Streams metrics interface for adding metric sensors and collecting metric values.
  */
-@InterfaceStability.Evolving
 public interface StreamsMetrics {
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/GlobalKTable.java b/streams/src/main/java/org/apache/kafka/streams/kstream/GlobalKTable.java
index e58f67fc5b3de..ba64abee6c825 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/GlobalKTable.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/GlobalKTable.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.streams.kstream;
 
-import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.streams.KafkaStreams;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsBuilder;
@@ -65,7 +64,6 @@
  * @see KStream#join(GlobalKTable, KeyValueMapper, ValueJoiner)
  * @see KStream#leftJoin(GlobalKTable, KeyValueMapper, ValueJoiner)
  */
-@InterfaceStability.Evolving
 public interface GlobalKTable {
     /**
      * Get the name of the local state store that can be used to query this {@code GlobalKTable}.
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java
index 121d0a4cb9f98..7675bcb051d62 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.streams.kstream;
 
-import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.streams.KafkaStreams;
 import org.apache.kafka.streams.KeyValue;
@@ -40,7 +39,6 @@
  * @param  Type of values
  * @see KStream
  */
-@InterfaceStability.Evolving
 public interface KGroupedStream {
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java
index 30f348c2eaf34..74fb88055bec8 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.streams.kstream;
 
-import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.streams.KafkaStreams;
 import org.apache.kafka.streams.StreamsConfig;
@@ -38,7 +37,6 @@
  * @param  Type of values
  * @see KTable
  */
-@InterfaceStability.Evolving
 public interface KGroupedTable {
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java
index 8313addb6ff68..b1d8422fd281a 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java
@@ -17,7 +17,6 @@
 package org.apache.kafka.streams.kstream;
 
 import org.apache.kafka.clients.producer.internals.DefaultPartitioner;
-import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsBuilder;
@@ -52,7 +51,6 @@
  * @see KGroupedStream
  * @see StreamsBuilder#stream(String)
  */
-@InterfaceStability.Evolving
 public interface KStream {
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java
index 39e5e225b2c79..a2b9bafd9a78a 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.streams.kstream;
 
-import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.streams.KafkaStreams;
@@ -65,7 +64,6 @@
  * @see GlobalKTable
  * @see StreamsBuilder#table(String)
  */
-@InterfaceStability.Evolving
 public interface KTable {
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TimeWindow.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TimeWindow.java
index ac712826a35c9..4037d8f761aa7 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TimeWindow.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TimeWindow.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
-import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.streams.kstream.Window;
 
 /**
@@ -32,7 +31,6 @@
  * @see org.apache.kafka.streams.kstream.TimeWindows
  * @see org.apache.kafka.streams.processor.TimestampExtractor
  */
-@InterfaceStability.Unstable
 public class TimeWindow extends Window {
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/UnlimitedWindow.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/UnlimitedWindow.java
index 12125ec080764..9b29b9e90a838 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/UnlimitedWindow.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/UnlimitedWindow.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
-import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.streams.kstream.Window;
 
 /**
@@ -32,7 +31,6 @@
  * @see org.apache.kafka.streams.kstream.UnlimitedWindows
  * @see org.apache.kafka.streams.processor.TimestampExtractor
  */
-@InterfaceStability.Unstable
 public class UnlimitedWindow extends Window {
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/ExtractRecordMetadataTimestamp.java b/streams/src/main/java/org/apache/kafka/streams/processor/ExtractRecordMetadataTimestamp.java
index 3c7428a8c235f..fc07b40a346d4 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/ExtractRecordMetadataTimestamp.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/ExtractRecordMetadataTimestamp.java
@@ -17,7 +17,6 @@
 package org.apache.kafka.streams.processor;
 
 import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.common.annotation.InterfaceStability;
 
 /**
  * Retrieves embedded metadata timestamps from Kafka messages.
@@ -43,7 +42,6 @@
  * @see UsePreviousTimeOnInvalidTimestamp
  * @see WallclockTimestampExtractor
  */
-@InterfaceStability.Evolving
 abstract class ExtractRecordMetadataTimestamp implements TimestampExtractor {
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/FailOnInvalidTimestamp.java b/streams/src/main/java/org/apache/kafka/streams/processor/FailOnInvalidTimestamp.java
index 40d3e0ea5955c..92987b2c91dbd 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/FailOnInvalidTimestamp.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/FailOnInvalidTimestamp.java
@@ -17,7 +17,6 @@
 package org.apache.kafka.streams.processor;
 
 import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.streams.errors.StreamsException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -45,7 +44,6 @@
  * @see UsePreviousTimeOnInvalidTimestamp
  * @see WallclockTimestampExtractor
  */
-@InterfaceStability.Evolving
 public class FailOnInvalidTimestamp extends ExtractRecordMetadataTimestamp {
     private static final Logger log = LoggerFactory.getLogger(FailOnInvalidTimestamp.class);
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/LogAndSkipOnInvalidTimestamp.java b/streams/src/main/java/org/apache/kafka/streams/processor/LogAndSkipOnInvalidTimestamp.java
index b759e5bd4249e..9fbc2f5a52555 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/LogAndSkipOnInvalidTimestamp.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/LogAndSkipOnInvalidTimestamp.java
@@ -17,7 +17,6 @@
 package org.apache.kafka.streams.processor;
 
 import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.common.annotation.InterfaceStability;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -46,7 +45,6 @@
  * @see UsePreviousTimeOnInvalidTimestamp
  * @see WallclockTimestampExtractor
  */
-@InterfaceStability.Evolving
 public class LogAndSkipOnInvalidTimestamp extends ExtractRecordMetadataTimestamp {
     private static final Logger log = LoggerFactory.getLogger(LogAndSkipOnInvalidTimestamp.class);
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/Processor.java b/streams/src/main/java/org/apache/kafka/streams/processor/Processor.java
index c10af9e8c99bb..4046f2f116f8e 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/Processor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/Processor.java
@@ -16,8 +16,6 @@
  */
 package org.apache.kafka.streams.processor;
 
-import org.apache.kafka.common.annotation.InterfaceStability;
-
 import java.time.Duration;
 
 /**
@@ -26,7 +24,6 @@
  * @param  the type of keys
  * @param  the type of values
  */
-@InterfaceStability.Evolving
 public interface Processor {
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/ProcessorContext.java b/streams/src/main/java/org/apache/kafka/streams/processor/ProcessorContext.java
index b654410c73e63..1971c67b29832 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/ProcessorContext.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/ProcessorContext.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.streams.processor;
 
-import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.common.header.Headers;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.streams.StreamsMetrics;
@@ -29,7 +28,6 @@
 /**
  * Processor context interface.
  */
-@InterfaceStability.Evolving
 public interface ProcessorContext {
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/StateRestoreCallback.java b/streams/src/main/java/org/apache/kafka/streams/processor/StateRestoreCallback.java
index 91732c3dfab92..2e896c8a015a0 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/StateRestoreCallback.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/StateRestoreCallback.java
@@ -16,13 +16,10 @@
  */
 package org.apache.kafka.streams.processor;
 
-import org.apache.kafka.common.annotation.InterfaceStability;
-
 /**
  * Restoration logic for log-backed state stores upon restart,
  * it takes one record at a time from the logs to apply to the restoring state.
  */
-@InterfaceStability.Evolving
 public interface StateRestoreCallback {
 
     void restore(byte[] key, byte[] value);
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/TimestampExtractor.java b/streams/src/main/java/org/apache/kafka/streams/processor/TimestampExtractor.java
index 1e6d6cd65c173..30d82085cc834 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/TimestampExtractor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/TimestampExtractor.java
@@ -17,14 +17,12 @@
 package org.apache.kafka.streams.processor;
 
 import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.streams.kstream.KTable;
 
 /**
  * An interface that allows the Kafka Streams framework to extract a timestamp from an instance of {@link ConsumerRecord}.
  * The extracted timestamp is defined as milliseconds.
  */
-@InterfaceStability.Evolving
 public interface TimestampExtractor {
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/TopicNameExtractor.java b/streams/src/main/java/org/apache/kafka/streams/processor/TopicNameExtractor.java
index 5d79751384e2e..0faa6109ceadd 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/TopicNameExtractor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/TopicNameExtractor.java
@@ -16,12 +16,9 @@
  */
 package org.apache.kafka.streams.processor;
 
-import org.apache.kafka.common.annotation.InterfaceStability;
-
 /**
  * An interface that allows to dynamically determine the name of the Kafka topic to send at the sink node of the topology.
  */
-@InterfaceStability.Evolving
 public interface TopicNameExtractor {
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/UsePreviousTimeOnInvalidTimestamp.java b/streams/src/main/java/org/apache/kafka/streams/processor/UsePreviousTimeOnInvalidTimestamp.java
index 89e2fd3729bdf..6807c6d257176 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/UsePreviousTimeOnInvalidTimestamp.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/UsePreviousTimeOnInvalidTimestamp.java
@@ -17,7 +17,6 @@
 package org.apache.kafka.streams.processor;
 
 import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.streams.errors.StreamsException;
 
 /**
@@ -43,7 +42,6 @@
  * @see LogAndSkipOnInvalidTimestamp
  * @see WallclockTimestampExtractor
  */
-@InterfaceStability.Evolving
 public class UsePreviousTimeOnInvalidTimestamp extends ExtractRecordMetadataTimestamp {
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/WallclockTimestampExtractor.java b/streams/src/main/java/org/apache/kafka/streams/processor/WallclockTimestampExtractor.java
index baa1cb6dbea2e..aa1df0f5400c5 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/WallclockTimestampExtractor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/WallclockTimestampExtractor.java
@@ -17,7 +17,6 @@
 package org.apache.kafka.streams.processor;
 
 import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.common.annotation.InterfaceStability;
 
 /**
  * Retrieves current wall clock timestamps as {@link System#currentTimeMillis()}.
@@ -31,7 +30,6 @@
  * @see LogAndSkipOnInvalidTimestamp
  * @see UsePreviousTimeOnInvalidTimestamp
  */
-@InterfaceStability.Evolving
 public class WallclockTimestampExtractor implements TimestampExtractor {
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/Stores.java b/streams/src/main/java/org/apache/kafka/streams/state/Stores.java
index 2f81fd3470545..f92a0de9d4f85 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/Stores.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/Stores.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.streams.state;
 
-import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.common.utils.Time;
@@ -76,7 +75,6 @@
  * topology.addStateStore(storeBuilder, "processorName");
  * }
  */
-@InterfaceStability.Evolving
 public final class Stores {
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/StreamsMetadata.java b/streams/src/main/java/org/apache/kafka/streams/state/StreamsMetadata.java
index a14b211bd7344..16af13e54bc2a 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/StreamsMetadata.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/StreamsMetadata.java
@@ -17,7 +17,6 @@
 package org.apache.kafka.streams.state;
 
 import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.streams.KafkaStreams;
 
 import java.util.Collections;
@@ -30,7 +29,6 @@
  * the instance and the Set of {@link TopicPartition}s available on the instance.
  * NOTE: This is a point in time view. It may change when rebalances happen.
  */
-@InterfaceStability.Evolving
 public class StreamsMetadata {
     /**
      * Sentinel to indicate that the StreamsMetadata is currently unavailable. This can occur during rebalance
@@ -69,6 +67,7 @@ public String host() {
         return hostInfo.host();
     }
 
+    @SuppressWarnings("unused")
     public int port() {
         return hostInfo.port();
     }
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
index 2b1428f9f8968..d0ca6eae253d3 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
@@ -27,7 +27,6 @@
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.PartitionInfo;
 import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.common.header.internals.RecordHeaders;
 import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
@@ -186,7 +185,6 @@
  * @see ConsumerRecordFactory
  * @see OutputVerifier
  */
-@InterfaceStability.Evolving
 public class TopologyTestDriver implements Closeable {
 
     private static final Logger log = LoggerFactory.getLogger(TopologyTestDriver.class);
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java
index ed0274649bf23..c478b6ba2bf08 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.streams.processor;
 
-import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.common.header.Headers;
 import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
@@ -56,7 +55,6 @@
  * If you require more automated tests, we recommend wrapping your {@link Processor} in a minimal source-processor-sink
  * {@link Topology} and using the {@link TopologyTestDriver}.
  */
-@InterfaceStability.Evolving
 public class MockProcessorContext implements ProcessorContext, RecordCollector.Supplier {
     // Immutable fields ================================================
     private final StreamsMetricsImpl metrics;
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/test/ConsumerRecordFactory.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/test/ConsumerRecordFactory.java
index 87ec7c1fcc396..e0fcefb983de7 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/test/ConsumerRecordFactory.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/test/ConsumerRecordFactory.java
@@ -17,7 +17,6 @@
 package org.apache.kafka.streams.test;
 
 import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.common.header.Headers;
 import org.apache.kafka.common.header.internals.RecordHeaders;
 import org.apache.kafka.common.record.TimestampType;
@@ -38,7 +37,6 @@
  *
  * @see TopologyTestDriver
  */
-@InterfaceStability.Evolving
 public class ConsumerRecordFactory {
     private final String topicName;
     private final Serializer keySerializer;

From f125c2dfe22a9a51f2881f63673c9b85a863ba14 Mon Sep 17 00:00:00 2001
From: Gwen Shapira 
Date: Tue, 13 Aug 2019 23:07:49 -0700
Subject: [PATCH 0533/1071] KAFKA-8792; Default ZK configuration to disable
 AdminServer

Kafka ships with default ZK configuration. With the upgrade to ZK 3.5, our defaults include running ZK's AdminServer on port 8080. This is an unfortunate default as it tends to cause conflicts.

I suggest we default to disable ZK's AdminServer in the default ZK configs that we ship. Users who want to use AdminServer can enable it and set the port to something that works for them. Realistically, in most production environments, a different ZK server will be used anyway. So this is mostly to save new users who are trying Kafka on their own machine from running into accidental and frustrating port conflicts.

Author: Gwen Shapira 

Reviewers: Ismael Juma

Closes #7203 from gwenshap/zk_disable_adminserver
---
 config/zookeeper.properties | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/config/zookeeper.properties b/config/zookeeper.properties
index 74cbf90428f81..90f4332ec31cf 100644
--- a/config/zookeeper.properties
+++ b/config/zookeeper.properties
@@ -18,3 +18,7 @@ dataDir=/tmp/zookeeper
 clientPort=2181
 # disable the per-ip limit on the number of connections since this is a non-production config
 maxClientCnxns=0
+# Disable the adminserver by default to avoid port conflicts.
+# Set the port to something non-conflicting if choosing to enable this
+admin.enableServer=false
+# admin.serverPort=8080

From e6fe4c13deb9ca814513a8c12d79dd76af658762 Mon Sep 17 00:00:00 2001
From: Bruno Cadonna 
Date: Wed, 14 Aug 2019 16:58:28 +0200
Subject: [PATCH 0534/1071] MINOR: Correct typo in test name
 `TimetampedSegmentsTest` (#7210)

Reviewers: Sophie Blee-Goldman , Stanislav Kozlovski , Bill Bejeck 
---
 ...TimetampedSegmentsTest.java => TimestampedSegmentsTest.java} | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename streams/src/test/java/org/apache/kafka/streams/state/internals/{TimetampedSegmentsTest.java => TimestampedSegmentsTest.java} (99%)

diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/TimetampedSegmentsTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/TimestampedSegmentsTest.java
similarity index 99%
rename from streams/src/test/java/org/apache/kafka/streams/state/internals/TimetampedSegmentsTest.java
rename to streams/src/test/java/org/apache/kafka/streams/state/internals/TimestampedSegmentsTest.java
index c51988721f55e..ccc88d0c31f53 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/TimetampedSegmentsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/TimestampedSegmentsTest.java
@@ -41,7 +41,7 @@
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
-public class TimetampedSegmentsTest {
+public class TimestampedSegmentsTest {
 
     private static final int NUM_SEGMENTS = 5;
     private static final long SEGMENT_INTERVAL = 100L;

From 5129ab53ee9a2e46a22a13b1d5300ee139c4999f Mon Sep 17 00:00:00 2001
From: Stanislav Kozlovski 
Date: Wed, 14 Aug 2019 17:25:17 +0100
Subject: [PATCH 0535/1071] KAFKA-8345: KIP-455: Admin API changes (Part 2)
 (#7120)

Add the AlterPartitionReassignments and ListPartitionReassignments APIs.  Also remove an unused methodlength suppression for KafkaAdminClient.

Reviewers: Colin P. McCabe , Viktor Somogyi 
---
 checkstyle/suppressions.xml                   |   2 +-
 .../org/apache/kafka/clients/admin/Admin.java | 115 ++++++++
 .../AlterPartitionReassignmentsOptions.java   |  31 +++
 .../AlterPartitionReassignmentsResult.java    |  59 +++++
 .../kafka/clients/admin/KafkaAdminClient.java | 250 ++++++++++++++++++
 .../ListPartitionReassignmentsOptions.java    |  29 ++
 .../ListPartitionReassignmentsResult.java     |  43 +++
 .../admin/NewPartitionReassignment.java       |  44 +++
 .../clients/admin/PartitionReassignment.java  |  60 +++++
 .../AlterPartitionReassignmentsResponse.java  |   2 +-
 .../ListPartitionReassignmentsRequest.java    |  21 +-
 .../ListPartitionReassignmentsResponse.java   |   2 +-
 .../ListPartitionReassignmentsRequest.json    |   2 +-
 .../clients/admin/KafkaAdminClientTest.java   | 215 +++++++++++++++
 .../kafka/clients/admin/MockAdminClient.java  |  12 +
 .../common/requests/RequestResponseTest.java  |   6 +-
 16 files changed, 878 insertions(+), 15 deletions(-)
 create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/AlterPartitionReassignmentsOptions.java
 create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/AlterPartitionReassignmentsResult.java
 create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/ListPartitionReassignmentsOptions.java
 create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/ListPartitionReassignmentsResult.java
 create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/NewPartitionReassignment.java
 create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/PartitionReassignment.java

diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index 0bb2193c049ef..475b9b0cd6087 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -52,7 +52,7 @@
               files="(ConsumerCoordinator|Fetcher|Sender|KafkaProducer|BufferPool|ConfigDef|RecordAccumulator|KerberosLogin|AbstractRequest|AbstractResponse|Selector|SslFactory|SslTransportLayer|SaslClientAuthenticator|SaslClientCallbackHandler|SaslServerAuthenticator|SchemaGenerator|AbstractCoordinator).java"/>
 
     
+              files="AbstractRequest.java|KerberosLogin.java|WorkerSinkTaskTest.java|TransactionManagerTest.java|SenderTest.java|KafkaAdminClient.java"/>
 
     
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java b/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java
index c1be7a7dac6d6..8fdd21b1ad54c 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java
@@ -21,6 +21,7 @@
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
@@ -930,6 +931,120 @@ ElectLeadersResult electLeaders(
         Set partitions,
         ElectLeadersOptions options);
 
+
+    /**
+     * Change the reassignments for one or more partitions.
+     * Providing an empty Optional (e.g via {@link Optional#empty()}) will revert the reassignment for the associated partition.
+     *
+     * This is a convenience method for {@link #alterPartitionReassignments(Map, AlterPartitionReassignmentsOptions)}
+     * with default options.  See the overload for more details.
+     */
+    default AlterPartitionReassignmentsResult alterPartitionReassignments(
+        Map> reassignments) {
+        return alterPartitionReassignments(reassignments, new AlterPartitionReassignmentsOptions());
+    }
+
+    /**
+     * Change the reassignments for one or more partitions.
+     * Providing an empty Optional (e.g via {@link Optional#empty()}) will revert the reassignment for the associated partition.
+     *
+     * 
    The following exceptions can be anticipated when calling {@code get()} on the futures obtained from
+     * the returned {@code AlterPartitionReassignmentsResult}:
    
+     * 
      
+     *   
    • {@link org.apache.kafka.common.errors.ClusterAuthorizationException}
+     *   If the authenticated user didn't have alter access to the cluster.
      • 
+     *   
    • {@link org.apache.kafka.common.errors.UnknownTopicOrPartitionException}
+     *   If the topic or partition does not exist within the cluster.
      • 
+     *   
    • {@link org.apache.kafka.common.errors.TimeoutException}
+     *   if the request timed out before the controller could record the new assignments.
      • 
+     *   
    • {@link org.apache.kafka.common.errors.InvalidReplicaAssignmentException}
+     *   If the specified assignment was not valid.
      • 
+     *   
    • {@link org.apache.kafka.common.errors.NoReassignmentInProgressException}
+     *   If there was an attempt to cancel a reassignment for a partition which was not being reassigned.
      • 
+     * 
    
+     *
+     * @param reassignments   The reassignments to add, modify, or remove.
+     * @param options         The options to use.
+     * @return                The result.
+     */
+    AlterPartitionReassignmentsResult alterPartitionReassignments(
+        Map> reassignments,
+        AlterPartitionReassignmentsOptions options);
+
+
+    /**
+     * List all of the current partition reassignments
+     *
+     * This is a convenience method for {@link #listPartitionReassignments(ListPartitionReassignmentsOptions)}
+     * with default options. See the overload for more details.
+     */
+    default ListPartitionReassignmentsResult listPartitionReassignments() {
+        return listPartitionReassignments(new ListPartitionReassignmentsOptions());
+    }
+
+    /**
+     * List the current reassignments for the given partitions
+     *
+     * This is a convenience method for {@link #listPartitionReassignments(Set, ListPartitionReassignmentsOptions)}
+     * with default options. See the overload for more details.
+     */
+    default ListPartitionReassignmentsResult listPartitionReassignments(Set partitions) {
+        return listPartitionReassignments(partitions, new ListPartitionReassignmentsOptions());
+    }
+
+    /**
+     * List the current reassignments for the given partitions
+     *
+     * 
    The following exceptions can be anticipated when calling {@code get()} on the futures obtained from
+     * the returned {@code ListPartitionReassignmentsResult}:
    
+     * 
      
+     *   
    • {@link org.apache.kafka.common.errors.ClusterAuthorizationException}
+     *   If the authenticated user doesn't have alter access to the cluster.
      • 
+     *   
    • {@link org.apache.kafka.common.errors.UnknownTopicOrPartitionException}
+     *   If a given topic or partition does not exist.
      • 
+     *   
    • {@link org.apache.kafka.common.errors.TimeoutException}
+     *   If the request timed out before the controller could list the current reassignments.
      • 
+     * 
    
+     *
+     * @param partitions      The topic partitions to list reassignments for.
+     * @param options         The options to use.
+     * @return                The result.
+     */
+    default ListPartitionReassignmentsResult listPartitionReassignments(
+        Set partitions,
+        ListPartitionReassignmentsOptions options) {
+        return listPartitionReassignments(Optional.of(partitions), options);
+    }
+
+    /**
+     * List all of the current partition reassignments
+     *
+     * 
    The following exceptions can be anticipated when calling {@code get()} on the futures obtained from
+     * the returned {@code ListPartitionReassignmentsResult}:
    
+     * 
      
+     *   
    • {@link org.apache.kafka.common.errors.ClusterAuthorizationException}
+     *   If the authenticated user doesn't have alter access to the cluster.
      • 
+     *   
    • {@link org.apache.kafka.common.errors.UnknownTopicOrPartitionException}
+     *   If a given topic or partition does not exist.
      • 
+     *   
    • {@link org.apache.kafka.common.errors.TimeoutException}
+     *   If the request timed out before the controller could list the current reassignments.
      • 
+     * 
    
+     *
+     * @param options         The options to use.
+     * @return                The result.
+     */
+    default ListPartitionReassignmentsResult listPartitionReassignments(ListPartitionReassignmentsOptions options) {
+        return listPartitionReassignments(Optional.empty(), options);
+    }
+
+    /**
+     * @param partitions the partitions we want to get reassignment for, or an empty optional if we want to get the reassignments for all partitions in the cluster
+     * @param options         The options to use.
+     * @return                The result.
+     */
+    ListPartitionReassignmentsResult listPartitionReassignments(Optional> partitions,
+                                                                ListPartitionReassignmentsOptions options);
+
     /**
      * Get the metrics kept by the adminClient
      */
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/AlterPartitionReassignmentsOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/AlterPartitionReassignmentsOptions.java
new file mode 100644
index 0000000000000..bee9c70ab0b19
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/AlterPartitionReassignmentsOptions.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.clients.admin;
+
+import org.apache.kafka.common.annotation.InterfaceStability;
+
+import java.util.Map;
+
+/**
+ * Options for {@link AdminClient#alterPartitionReassignments(Map, AlterPartitionReassignmentsOptions)}
+ *
+ * The API of this class is evolving. See {@link AdminClient} for details.
+ */
+@InterfaceStability.Evolving
+public class AlterPartitionReassignmentsOptions extends AbstractOptions {
+}
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/AlterPartitionReassignmentsResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/AlterPartitionReassignmentsResult.java
new file mode 100644
index 0000000000000..2009ab5b6b7f2
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/AlterPartitionReassignmentsResult.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.clients.admin;
+
+import org.apache.kafka.common.KafkaFuture;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.annotation.InterfaceStability;
+
+import java.util.Map;
+
+/**
+ * The result of {@link AdminClient#alterPartitionReassignments(Map, AlterPartitionReassignmentsOptions)}.
+ *
+ * The API of this class is evolving. See {@link AdminClient} for details.
+ */
+@InterfaceStability.Evolving
+public class AlterPartitionReassignmentsResult {
+    private final Map> futures;
+
+    AlterPartitionReassignmentsResult(Map> futures) {
+        this.futures = futures;
+    }
+
+    /**
+     * Return a map from partitions to futures which can be used to check the status of the reassignment.
+     *
+     * Possible error codes:
+     *
+     * INVALID_REPLICA_ASSIGNMENT (39) -  if the specified replica assignment was not valid -- for example, if it included negative numbers, repeated numbers, or specified a broker ID that the controller was not aware of.
+     * NO_REASSIGNMENT_IN_PROGRESS (85) - if the request wants to cancel reassignments but none exist
+     * UNKNOWN (-1)
+     *
+     */
+    public Map> values() {
+        return futures;
+    }
+
+    /**
+     * Return a future which succeeds only if all the reassignments were successfully initiated.
+     */
+    public KafkaFuture all() {
+        return KafkaFuture.allOf(futures.values().toArray(new KafkaFuture[0]));
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
index 7bcad4100436f..936dc65a5bd7a 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
@@ -61,6 +61,8 @@
 import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
 import org.apache.kafka.common.errors.UnsupportedVersionException;
 import org.apache.kafka.common.internals.KafkaFutureImpl;
+import org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData;
+import org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData.ReassignableTopic;
 import org.apache.kafka.common.message.CreateDelegationTokenRequestData;
 import org.apache.kafka.common.message.CreateDelegationTokenRequestData.CreatableRenewers;
 import org.apache.kafka.common.message.CreateDelegationTokenResponseData;
@@ -81,6 +83,7 @@
 import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData.AlterConfigsResource;
 import org.apache.kafka.common.message.ListGroupsRequestData;
 import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.ListPartitionReassignmentsRequestData;
 import org.apache.kafka.common.message.MetadataRequestData;
 import org.apache.kafka.common.message.RenewDelegationTokenRequestData;
 import org.apache.kafka.common.metrics.JmxReporter;
@@ -95,6 +98,8 @@
 import org.apache.kafka.common.requests.AbstractResponse;
 import org.apache.kafka.common.requests.AlterConfigsRequest;
 import org.apache.kafka.common.requests.AlterConfigsResponse;
+import org.apache.kafka.common.requests.AlterPartitionReassignmentsRequest;
+import org.apache.kafka.common.requests.AlterPartitionReassignmentsResponse;
 import org.apache.kafka.common.requests.AlterReplicaLogDirsRequest;
 import org.apache.kafka.common.requests.AlterReplicaLogDirsResponse;
 import org.apache.kafka.common.requests.ApiError;
@@ -140,6 +145,8 @@
 import org.apache.kafka.common.requests.IncrementalAlterConfigsResponse;
 import org.apache.kafka.common.requests.ListGroupsRequest;
 import org.apache.kafka.common.requests.ListGroupsResponse;
+import org.apache.kafka.common.requests.ListPartitionReassignmentsRequest;
+import org.apache.kafka.common.requests.ListPartitionReassignmentsResponse;
 import org.apache.kafka.common.requests.MetadataRequest;
 import org.apache.kafka.common.requests.MetadataResponse;
 import org.apache.kafka.common.requests.OffsetFetchRequest;
@@ -170,7 +177,9 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
+import java.util.TreeMap;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -179,6 +188,12 @@
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
+import static org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData.ReassignablePartition;
+import static org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignableTopicResponse;
+import static org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignablePartitionResponse;
+import static org.apache.kafka.common.message.ListPartitionReassignmentsRequestData.ListPartitionReassignmentsTopics;
+import static org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingTopicReassignment;
+import static org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingPartitionReassignment;
 import static org.apache.kafka.common.requests.MetadataRequest.convertToMetadataRequestTopic;
 import static org.apache.kafka.common.utils.Utils.closeQuietly;
 
@@ -3082,6 +3097,241 @@ void handleFailure(Throwable throwable) {
         return new ElectLeadersResult(electionFuture);
     }
 
+    @Override
+    public AlterPartitionReassignmentsResult alterPartitionReassignments(
+            Map> reassignments,
+            AlterPartitionReassignmentsOptions options) {
+        final Map> futures = new HashMap<>();
+        final Map>> topicsToReassignments = new TreeMap<>();
+        for (Map.Entry> entry : reassignments.entrySet()) {
+            String topic = entry.getKey().topic();
+            int partition = entry.getKey().partition();
+            TopicPartition topicPartition = new TopicPartition(topic, partition);
+            Optional reassignment = entry.getValue();
+            KafkaFutureImpl future = new KafkaFutureImpl<>();
+            futures.put(topicPartition, future);
+
+            if (topicNameIsUnrepresentable(topic)) {
+                future.completeExceptionally(new InvalidTopicException("The given topic name '" +
+                        topic + "' cannot be represented in a request."));
+            } else if (topicPartition.partition() < 0) {
+                future.completeExceptionally(new InvalidTopicException("The given partition index " +
+                        topicPartition.partition() + " is not valid."));
+            } else {
+                Map> partitionReassignments =
+                        topicsToReassignments.get(topicPartition.topic());
+                if (partitionReassignments == null) {
+                    partitionReassignments = new TreeMap<>();
+                    topicsToReassignments.put(topic, partitionReassignments);
+                }
+
+                partitionReassignments.put(partition, reassignment);
+            }
+        }
+
+        final long now = time.milliseconds();
+        Call call = new Call("alterPartitionReassignments", calcDeadlineMs(now, options.timeoutMs()),
+                new ControllerNodeProvider()) {
+
+            @Override
+            public AbstractRequest.Builder createRequest(int timeoutMs) {
+                AlterPartitionReassignmentsRequestData data =
+                        new AlterPartitionReassignmentsRequestData();
+                for (Map.Entry>> entry :
+                        topicsToReassignments.entrySet()) {
+                    String topicName = entry.getKey();
+                    Map> partitionsToReassignments = entry.getValue();
+
+                    List reassignablePartitions = new ArrayList<>();
+                    for (Map.Entry> partitionEntry :
+                            partitionsToReassignments.entrySet()) {
+                        int partitionIndex = partitionEntry.getKey();
+                        Optional reassignment = partitionEntry.getValue();
+
+                        ReassignablePartition reassignablePartition = new ReassignablePartition()
+                                .setPartitionIndex(partitionIndex)
+                                .setReplicas(reassignment.map(NewPartitionReassignment::targetBrokers).orElse(null));
+                        reassignablePartitions.add(reassignablePartition);
+                    }
+
+                    ReassignableTopic reassignableTopic = new ReassignableTopic()
+                            .setName(topicName)
+                            .setPartitions(reassignablePartitions);
+                    data.topics().add(reassignableTopic);
+                }
+                data.setTimeoutMs(timeoutMs);
+                return new AlterPartitionReassignmentsRequest.Builder(data);
+            }
+
+            @Override
+            public void handleResponse(AbstractResponse abstractResponse) {
+                AlterPartitionReassignmentsResponse response = (AlterPartitionReassignmentsResponse) abstractResponse;
+                Map errors = new HashMap<>();
+                int receivedResponsesCount = 0;
+
+                Errors topLevelError = Errors.forCode(response.data().errorCode());
+                switch (topLevelError) {
+                    case NONE:
+                        receivedResponsesCount += validateTopicResponses(response.data().responses(), errors);
+                        break;
+                    case NOT_CONTROLLER:
+                        handleNotControllerError(topLevelError);
+                        break;
+                    default:
+                        for (ReassignableTopicResponse topicResponse : response.data().responses()) {
+                            String topicName = topicResponse.name();
+                            for (ReassignablePartitionResponse partition : topicResponse.partitions()) {
+                                errors.put(
+                                        new TopicPartition(topicName, partition.partitionIndex()),
+                                        new ApiError(topLevelError, topLevelError.message()).exception()
+                                );
+                                receivedResponsesCount += 1;
+                            }
+                        }
+                        break;
+                }
+
+                assertResponseCountMatch(errors, receivedResponsesCount);
+                for (Map.Entry entry : errors.entrySet()) {
+                    ApiException exception = entry.getValue();
+                    if (exception == null)
+                        futures.get(entry.getKey()).complete(null);
+                    else
+                        futures.get(entry.getKey()).completeExceptionally(exception);
+                }
+            }
+
+            private void assertResponseCountMatch(Map errors, int receivedResponsesCount) {
+                int expectedResponsesCount = topicsToReassignments.values().stream().mapToInt(Map::size).sum();
+                if (errors.values().stream().noneMatch(Objects::nonNull) && receivedResponsesCount != expectedResponsesCount) {
+                    String quantifier = receivedResponsesCount > expectedResponsesCount ? "many" : "less";
+                    throw new UnknownServerException("The server returned too " + quantifier + " results." +
+                        "Expected " + expectedResponsesCount + " but received " + receivedResponsesCount);
+                }
+            }
+
+            private int validateTopicResponses(List topicResponses,
+                                               Map errors) {
+                int receivedResponsesCount = 0;
+
+                for (ReassignableTopicResponse topicResponse : topicResponses) {
+                    String topicName = topicResponse.name();
+                    for (ReassignablePartitionResponse partResponse : topicResponse.partitions()) {
+                        Errors partitionError = Errors.forCode(partResponse.errorCode());
+
+                        TopicPartition tp = new TopicPartition(topicName, partResponse.partitionIndex());
+                        if (partitionError == Errors.NONE) {
+                            errors.put(tp, null);
+                        } else {
+                            errors.put(tp, new ApiError(partitionError, partResponse.errorMessage()).exception());
+                        }
+                        receivedResponsesCount += 1;
+                    }
+                }
+
+                return receivedResponsesCount;
+            }
+
+            @Override
+            void handleFailure(Throwable throwable) {
+                for (KafkaFutureImpl future : futures.values()) {
+                    future.completeExceptionally(throwable);
+                }
+            }
+        };
+        if (!topicsToReassignments.isEmpty()) {
+            runnable.call(call, now);
+        }
+        return new AlterPartitionReassignmentsResult(new HashMap<>(futures));
+    }
+
+    @Override
+    public ListPartitionReassignmentsResult listPartitionReassignments(Optional> partitions,
+                                                                       ListPartitionReassignmentsOptions options) {
+        final KafkaFutureImpl> partitionReassignmentsFuture = new KafkaFutureImpl<>();
+        if (partitions.isPresent()) {
+            for (TopicPartition tp : partitions.get()) {
+                String topic = tp.topic();
+                int partition = tp.partition();
+                if (topicNameIsUnrepresentable(topic)) {
+                    partitionReassignmentsFuture.completeExceptionally(new InvalidTopicException("The given topic name '"
+                            + topic + "' cannot be represented in a request."));
+                } else if (partition < 0) {
+                    partitionReassignmentsFuture.completeExceptionally(new InvalidTopicException("The given partition index " +
+                            partition + " is not valid."));
+                }
+                if (partitionReassignmentsFuture.isCompletedExceptionally())
+                    return new ListPartitionReassignmentsResult(partitionReassignmentsFuture);
+            }
+        }
+        final long now = time.milliseconds();
+        runnable.call(new Call("listPartitionReassignments", calcDeadlineMs(now, options.timeoutMs()),
+            new ControllerNodeProvider()) {
+
+            @Override
+            AbstractRequest.Builder createRequest(int timeoutMs) {
+                ListPartitionReassignmentsRequestData listData = new ListPartitionReassignmentsRequestData();
+                listData.setTimeoutMs(timeoutMs);
+
+                if (partitions.isPresent()) {
+                    Map reassignmentTopicByTopicName = new HashMap<>();
+
+                    for (TopicPartition tp : partitions.get()) {
+                        if (!reassignmentTopicByTopicName.containsKey(tp.topic()))
+                            reassignmentTopicByTopicName.put(tp.topic(), new ListPartitionReassignmentsTopics().setName(tp.topic()));
+
+                        reassignmentTopicByTopicName.get(tp.topic()).partitionIndexes().add(tp.partition());
+                    }
+
+                    listData.setTopics(new ArrayList<>(reassignmentTopicByTopicName.values()));
+                }
+                return new ListPartitionReassignmentsRequest.Builder(listData);
+            }
+
+            @Override
+            void handleResponse(AbstractResponse abstractResponse) {
+                ListPartitionReassignmentsResponse response = (ListPartitionReassignmentsResponse) abstractResponse;
+                Errors error = Errors.forCode(response.data().errorCode());
+                switch (error) {
+                    case NONE:
+                        break;
+                    case NOT_CONTROLLER:
+                        handleNotControllerError(error);
+                        break;
+                    default:
+                        partitionReassignmentsFuture.completeExceptionally(new ApiError(error, response.data().errorMessage()).exception());
+                        break;
+                }
+                Map reassignmentMap = new HashMap<>();
+
+                for (OngoingTopicReassignment topicReassignment : response.data().topics()) {
+                    String topicName = topicReassignment.name();
+                    for (OngoingPartitionReassignment partitionReassignment : topicReassignment.partitions()) {
+                        reassignmentMap.put(
+                            new TopicPartition(topicName, partitionReassignment.partitionIndex()),
+                            new PartitionReassignment(partitionReassignment.replicas(), partitionReassignment.addingReplicas(), partitionReassignment.removingReplicas())
+                        );
+                    }
+                }
+
+                partitionReassignmentsFuture.complete(reassignmentMap);
+            }
+
+            @Override
+            void handleFailure(Throwable throwable) {
+                partitionReassignmentsFuture.completeExceptionally(throwable);
+            }
+        }, now);
+
+        return new ListPartitionReassignmentsResult(partitionReassignmentsFuture);
+    }
+
+    private void handleNotControllerError(Errors error) throws ApiException {
+        metadataManager.clearController();
+        metadataManager.requestUpdate();
+        throw error.exception();
+    }
+
     /**
      * Returns a boolean indicating whether the resource needs to go to a specific node
      */
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ListPartitionReassignmentsOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/ListPartitionReassignmentsOptions.java
new file mode 100644
index 0000000000000..7dcc7a6c3e6d5
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/ListPartitionReassignmentsOptions.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.clients.admin;
+
+import org.apache.kafka.common.annotation.InterfaceStability;
+
+/**
+ * Options for {@link AdminClient#listPartitionReassignments(ListPartitionReassignmentsOptions)}
+ *
+ * The API of this class is evolving. See {@link AdminClient} for details.
+ */
+@InterfaceStability.Evolving
+public class ListPartitionReassignmentsOptions extends AbstractOptions {
+}
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ListPartitionReassignmentsResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/ListPartitionReassignmentsResult.java
new file mode 100644
index 0000000000000..3d7b14c48f04c
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/ListPartitionReassignmentsResult.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.clients.admin;
+
+import org.apache.kafka.common.KafkaFuture;
+import org.apache.kafka.common.TopicPartition;
+
+import java.util.Map;
+
+/**
+ * The result of {@link AdminClient#listPartitionReassignments(ListPartitionReassignmentsOptions)}.
+ *
+ * The API of this class is evolving. See {@link AdminClient} for details.
+ */
+public class ListPartitionReassignmentsResult {
+    private final KafkaFuture> future;
+
+    public ListPartitionReassignmentsResult(KafkaFuture> reassignments) {
+        this.future = reassignments;
+    }
+
+    /**
+     * Return a future which yields a map containing each partition's reassignments
+     */
+    public KafkaFuture> reassignments() {
+        return future;
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/NewPartitionReassignment.java b/clients/src/main/java/org/apache/kafka/clients/admin/NewPartitionReassignment.java
new file mode 100644
index 0000000000000..88564708a6f46
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/NewPartitionReassignment.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.clients.admin;
+
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * A new partition reassignment, which can be applied via {@link AdminClient#alterPartitionReassignments(Map, AlterPartitionReassignmentsOptions)}.
+ */
+public class NewPartitionReassignment {
+    private final List targetBrokers;
+
+    public static Optional of(Integer... brokers) {
+        return Optional.of(new NewPartitionReassignment(Arrays.asList(brokers)));
+    }
+
+    public NewPartitionReassignment(List targetBrokers) {
+        this.targetBrokers = Collections.unmodifiableList(new ArrayList<>(targetBrokers));
+    }
+
+    public List targetBrokers() {
+        return targetBrokers;
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/PartitionReassignment.java b/clients/src/main/java/org/apache/kafka/clients/admin/PartitionReassignment.java
new file mode 100644
index 0000000000000..cc3306ea7d6b3
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/PartitionReassignment.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.clients.admin;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * A partition reassignment, which has been listed via {@link AdminClient#listPartitionReassignments()}.
+ */
+public class PartitionReassignment {
+
+    private final List replicas;
+    private final List addingReplicas;
+    private final List removingReplicas;
+
+    public PartitionReassignment(List replicas, List addingReplicas, List removingReplicas) {
+        this.replicas = Collections.unmodifiableList(replicas);
+        this.addingReplicas = Collections.unmodifiableList(addingReplicas);
+        this.removingReplicas = Collections.unmodifiableList(removingReplicas);
+    }
+
+    /**
+     * The brokers which this partition currently resides on.
+     */
+    public List replicas() {
+        return replicas;
+    }
+
+    /**
+     * The brokers that we are adding this partition to as part of a reassignment.
+     * A subset of replicas.
+     */
+    public List addingReplicas() {
+        return addingReplicas;
+    }
+
+    /**
+     * The brokers that we are removing this partition from as part of a reassignment.
+     * A subset of replicas.
+     */
+    public List removingReplicas() {
+        return removingReplicas;
+    }
+}
\ No newline at end of file
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java
index db1cfabebfa8b..ef235cda34f78 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java
@@ -36,7 +36,7 @@ public AlterPartitionReassignmentsResponse(Struct struct) {
         this(struct, ApiKeys.ALTER_PARTITION_REASSIGNMENTS.latestVersion());
     }
 
-    AlterPartitionReassignmentsResponse(AlterPartitionReassignmentsResponseData data) {
+    public AlterPartitionReassignmentsResponse(AlterPartitionReassignmentsResponseData data) {
         this.data = data;
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsRequest.java
index 471147bdb2a25..0aa5e557556ea 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsRequest.java
@@ -24,9 +24,11 @@
 import org.apache.kafka.common.protocol.types.Struct;
 
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
 
+import static org.apache.kafka.common.message.ListPartitionReassignmentsRequestData.ListPartitionReassignmentsTopics;
 
 public class ListPartitionReassignmentsRequest extends AbstractRequest {
 
@@ -86,14 +88,17 @@ public Struct toStruct() {
     public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
         ApiError apiError = ApiError.fromThrowable(e);
 
-        List ongoingTopicReassignments = data.topics().stream().map(topic ->
-                new OngoingTopicReassignment()
-                        .setName(topic.name())
-                        .setPartitions(topic.partitionIndexes().stream().map(partitionIndex ->
-                                new OngoingPartitionReassignment().setPartitionIndex(partitionIndex)).collect(Collectors.toList())
-                        )
-        ).collect(Collectors.toList());
-
+        List ongoingTopicReassignments = new ArrayList<>();
+        if (data.topics() != null) {
+            for (ListPartitionReassignmentsTopics topic : data.topics()) {
+                ongoingTopicReassignments.add(
+                        new OngoingTopicReassignment()
+                                .setName(topic.name())
+                                .setPartitions(topic.partitionIndexes().stream().map(partitionIndex ->
+                                        new OngoingPartitionReassignment().setPartitionIndex(partitionIndex)).collect(Collectors.toList()))
+                );
+            }
+        }
         ListPartitionReassignmentsResponseData responseData = new ListPartitionReassignmentsResponseData()
                 .setTopics(ongoingTopicReassignments)
                 .setErrorCode(apiError.error().code())
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java
index 9513e88303044..0d53e4dd7caee 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java
@@ -33,7 +33,7 @@ public ListPartitionReassignmentsResponse(Struct struct) {
         this(struct, ApiKeys.LIST_PARTITION_REASSIGNMENTS.latestVersion());
     }
 
-    ListPartitionReassignmentsResponse(ListPartitionReassignmentsResponseData responseData) {
+    public ListPartitionReassignmentsResponse(ListPartitionReassignmentsResponseData responseData) {
         this.data = responseData;
     }
 
diff --git a/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json b/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json
index d0ebf8bd7d622..ac871b2bc028f 100644
--- a/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json
+++ b/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json
@@ -21,7 +21,7 @@
   "fields": [
     { "name": "TimeoutMs", "type": "int32", "versions": "0+", "default": "60000",
       "about": "The time in ms to wait for the request to complete." },
-    { "name": "Topics", "type": "[]ListPartitionReassignmentsTopics", "versions": "0+", "nullableVersions": "0+",
+    { "name": "Topics", "type": "[]ListPartitionReassignmentsTopics", "versions": "0+", "nullableVersions": "0+", "default": "null",
       "about": "The topics to list partition reassignments for, or null to list everything.", "fields": [
       { "name": "Name", "type": "string", "versions": "0+", "entityType": "topicName",
         "about": "The topic name" },
diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
index 7ec7c24a3a0c7..ff9e27206b83b 100644
--- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
@@ -52,6 +52,7 @@
 import org.apache.kafka.common.errors.TopicDeletionDisabledException;
 import org.apache.kafka.common.errors.UnknownServerException;
 import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
+import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData;
 import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult;
 import org.apache.kafka.common.message.CreateTopicsResponseData;
 import org.apache.kafka.common.message.DeleteGroupsResponseData;
@@ -66,7 +67,9 @@
 import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData.AlterConfigsResourceResponse;
 import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData;
 import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData;
 import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.AlterPartitionReassignmentsResponse;
 import org.apache.kafka.common.requests.ApiError;
 import org.apache.kafka.common.requests.CreateAclsResponse.AclCreationResponse;
 import org.apache.kafka.common.requests.CreateAclsResponse;
@@ -87,6 +90,7 @@
 import org.apache.kafka.common.requests.FindCoordinatorResponse;
 import org.apache.kafka.common.requests.IncrementalAlterConfigsResponse;
 import org.apache.kafka.common.requests.ListGroupsResponse;
+import org.apache.kafka.common.requests.ListPartitionReassignmentsResponse;
 import org.apache.kafka.common.requests.MetadataRequest;
 import org.apache.kafka.common.requests.MetadataResponse;
 import org.apache.kafka.common.requests.OffsetFetchResponse;
@@ -119,11 +123,16 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
 import static java.util.Arrays.asList;
 import static java.util.Collections.singletonList;
+import static org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignablePartitionResponse;
+import static org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignableTopicResponse;
+import static org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingTopicReassignment;
+import static org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingPartitionReassignment;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
@@ -1561,6 +1570,212 @@ public void testIncrementalAlterConfigs()  throws Exception {
         }
     }
 
+    @Test
+    public void testAlterPartitionReassignments() throws Exception {
+        try (AdminClientUnitTestEnv env = mockClientEnv()) {
+            env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
+
+            TopicPartition tp1 = new TopicPartition("A", 0);
+            TopicPartition tp2 = new TopicPartition("B", 0);
+            Map> reassignments = new HashMap<>();
+            reassignments.put(tp1, Optional.empty());
+            reassignments.put(tp2, NewPartitionReassignment.of(1, 2, 3));
+
+            // 1. server returns less responses than number of partitions we sent
+            AlterPartitionReassignmentsResponseData responseData1 = new AlterPartitionReassignmentsResponseData();
+            ReassignablePartitionResponse normalPartitionResponse = new ReassignablePartitionResponse().setPartitionIndex(0);
+            responseData1.setResponses(Collections.singletonList(
+                    new ReassignableTopicResponse()
+                            .setName("A")
+                            .setPartitions(Collections.singletonList(normalPartitionResponse))));
+            env.kafkaClient().prepareResponse(new AlterPartitionReassignmentsResponse(responseData1));
+            AlterPartitionReassignmentsResult result1 = env.adminClient().alterPartitionReassignments(reassignments);
+            Future future1 = result1.all();
+            Future future2 = result1.values().get(tp1);
+            TestUtils.assertFutureError(future1, UnknownServerException.class);
+            TestUtils.assertFutureError(future2, UnknownServerException.class);
+
+            // 2. NOT_CONTROLLER error handling
+            AlterPartitionReassignmentsResponseData controllerErrResponseData =
+                    new AlterPartitionReassignmentsResponseData()
+                            .setErrorCode(Errors.NOT_CONTROLLER.code())
+                            .setErrorMessage(Errors.NOT_CONTROLLER.message())
+                            .setResponses(Arrays.asList(
+                                new ReassignableTopicResponse()
+                                        .setName("A")
+                                        .setPartitions(Collections.singletonList(normalPartitionResponse)),
+                                new ReassignableTopicResponse()
+                                        .setName("B")
+                                        .setPartitions(Collections.singletonList(normalPartitionResponse)))
+                            );
+            MetadataResponse controllerNodeResponse = MetadataResponse.prepareResponse(env.cluster().nodes(),
+                    env.cluster().clusterResource().clusterId(), 1, Collections.emptyList());
+            AlterPartitionReassignmentsResponseData normalResponse =
+                    new AlterPartitionReassignmentsResponseData()
+                            .setResponses(Arrays.asList(
+                                    new ReassignableTopicResponse()
+                                            .setName("A")
+                                            .setPartitions(Collections.singletonList(normalPartitionResponse)),
+                                    new ReassignableTopicResponse()
+                                            .setName("B")
+                                            .setPartitions(Collections.singletonList(normalPartitionResponse)))
+                            );
+            env.kafkaClient().prepareResponse(new AlterPartitionReassignmentsResponse(controllerErrResponseData));
+            env.kafkaClient().prepareResponse(controllerNodeResponse);
+            env.kafkaClient().prepareResponse(new AlterPartitionReassignmentsResponse(normalResponse));
+            AlterPartitionReassignmentsResult controllerErrResult = env.adminClient().alterPartitionReassignments(reassignments);
+            controllerErrResult.all().get();
+            controllerErrResult.values().get(tp1).get();
+            controllerErrResult.values().get(tp2).get();
+
+            // 3. partition-level error
+            AlterPartitionReassignmentsResponseData partitionLevelErrData =
+                    new AlterPartitionReassignmentsResponseData()
+                            .setResponses(Arrays.asList(
+                                    new ReassignableTopicResponse()
+                                            .setName("A")
+                                            .setPartitions(Collections.singletonList(new ReassignablePartitionResponse()
+                                                .setPartitionIndex(0).setErrorMessage(Errors.INVALID_REPLICA_ASSIGNMENT.message())
+                                                .setErrorCode(Errors.INVALID_REPLICA_ASSIGNMENT.code())
+                                            )),
+                                    new ReassignableTopicResponse()
+                                            .setName("B")
+                                            .setPartitions(Collections.singletonList(normalPartitionResponse)))
+                            );
+            env.kafkaClient().prepareResponse(new AlterPartitionReassignmentsResponse(partitionLevelErrData));
+            AlterPartitionReassignmentsResult partitionLevelErrResult = env.adminClient().alterPartitionReassignments(reassignments);
+            TestUtils.assertFutureError(partitionLevelErrResult.values().get(tp1), Errors.INVALID_REPLICA_ASSIGNMENT.exception().getClass());
+            partitionLevelErrResult.values().get(tp2).get();
+
+            // 4. top-level error
+            AlterPartitionReassignmentsResponseData topLevelErrResponseData =
+                    new AlterPartitionReassignmentsResponseData()
+                            .setErrorCode(Errors.CLUSTER_AUTHORIZATION_FAILED.code())
+                            .setErrorMessage(Errors.CLUSTER_AUTHORIZATION_FAILED.message())
+                            .setResponses(Arrays.asList(
+                                    new ReassignableTopicResponse()
+                                            .setName("A")
+                                            .setPartitions(Collections.singletonList(normalPartitionResponse)),
+                                    new ReassignableTopicResponse()
+                                            .setName("B")
+                                            .setPartitions(Collections.singletonList(normalPartitionResponse)))
+                            );
+            env.kafkaClient().prepareResponse(new AlterPartitionReassignmentsResponse(topLevelErrResponseData));
+            AlterPartitionReassignmentsResult topLevelErrResult = env.adminClient().alterPartitionReassignments(reassignments);
+            TestUtils.assertFutureError(topLevelErrResult.all(), Errors.CLUSTER_AUTHORIZATION_FAILED.exception().getClass());
+            TestUtils.assertFutureError(topLevelErrResult.values().get(tp1), Errors.CLUSTER_AUTHORIZATION_FAILED.exception().getClass());
+            TestUtils.assertFutureError(topLevelErrResult.values().get(tp2), Errors.CLUSTER_AUTHORIZATION_FAILED.exception().getClass());
+
+            // 5. unrepresentable topic name error
+            TopicPartition invalidTopicTP = new TopicPartition("", 0);
+            TopicPartition invalidPartitionTP = new TopicPartition("ABC", -1);
+            Map> invalidTopicReassignments = new HashMap<>();
+            invalidTopicReassignments.put(invalidPartitionTP, NewPartitionReassignment.of(1, 2, 3));
+            invalidTopicReassignments.put(invalidTopicTP, NewPartitionReassignment.of(1, 2, 3));
+            invalidTopicReassignments.put(tp1, NewPartitionReassignment.of(1, 2, 3));
+
+            AlterPartitionReassignmentsResponseData singlePartResponseData =
+                    new AlterPartitionReassignmentsResponseData()
+                            .setResponses(Collections.singletonList(
+                                    new ReassignableTopicResponse()
+                                            .setName("A")
+                                            .setPartitions(Collections.singletonList(normalPartitionResponse)))
+                            );
+            env.kafkaClient().prepareResponse(new AlterPartitionReassignmentsResponse(singlePartResponseData));
+            AlterPartitionReassignmentsResult unrepresentableTopicResult = env.adminClient().alterPartitionReassignments(invalidTopicReassignments);
+            TestUtils.assertFutureError(unrepresentableTopicResult.values().get(invalidTopicTP), InvalidTopicException.class);
+            TestUtils.assertFutureError(unrepresentableTopicResult.values().get(invalidPartitionTP), InvalidTopicException.class);
+            unrepresentableTopicResult.values().get(tp1).get();
+
+            // Test success scenario
+            AlterPartitionReassignmentsResponseData noErrResponseData =
+                    new AlterPartitionReassignmentsResponseData()
+                            .setErrorCode(Errors.NONE.code())
+                            .setErrorMessage(Errors.NONE.message())
+                            .setResponses(Arrays.asList(
+                                    new ReassignableTopicResponse()
+                                            .setName("A")
+                                            .setPartitions(Collections.singletonList(normalPartitionResponse)),
+                                    new ReassignableTopicResponse()
+                                            .setName("B")
+                                            .setPartitions(Collections.singletonList(normalPartitionResponse)))
+                            );
+            env.kafkaClient().prepareResponse(new AlterPartitionReassignmentsResponse(noErrResponseData));
+            AlterPartitionReassignmentsResult noErrResult = env.adminClient().alterPartitionReassignments(reassignments);
+            noErrResult.all().get();
+            noErrResult.values().get(tp1).get();
+            noErrResult.values().get(tp2).get();
+        }
+    }
+
+    @Test
+    public void testListPartitionReassignments() throws Exception {
+        try (AdminClientUnitTestEnv env = mockClientEnv()) {
+            env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
+
+            TopicPartition tp1 = new TopicPartition("A", 0);
+            OngoingPartitionReassignment tp1PartitionReassignment = new OngoingPartitionReassignment()
+                    .setPartitionIndex(0)
+                    .setRemovingReplicas(Arrays.asList(1, 2, 3))
+                    .setAddingReplicas(Arrays.asList(4, 5, 6))
+                    .setReplicas(Arrays.asList(1, 2, 3, 4, 5, 6));
+            OngoingTopicReassignment tp1Reassignment = new OngoingTopicReassignment().setName("A")
+                    .setPartitions(Collections.singletonList(tp1PartitionReassignment));
+
+            TopicPartition tp2 = new TopicPartition("B", 0);
+            OngoingPartitionReassignment tp2PartitionReassignment = new OngoingPartitionReassignment()
+                    .setPartitionIndex(0)
+                    .setRemovingReplicas(Arrays.asList(1, 2, 3))
+                    .setAddingReplicas(Arrays.asList(4, 5, 6))
+                    .setReplicas(Arrays.asList(1, 2, 3, 4, 5, 6));
+            OngoingTopicReassignment tp2Reassignment = new OngoingTopicReassignment().setName("B")
+                    .setPartitions(Collections.singletonList(tp2PartitionReassignment));
+
+            // 1. NOT_CONTROLLER error handling
+            ListPartitionReassignmentsResponseData notControllerData = new ListPartitionReassignmentsResponseData()
+                    .setErrorCode(Errors.NOT_CONTROLLER.code())
+                    .setErrorMessage(Errors.NOT_CONTROLLER.message());
+            MetadataResponse controllerNodeResponse = MetadataResponse.prepareResponse(env.cluster().nodes(),
+                    env.cluster().clusterResource().clusterId(), 1, Collections.emptyList());
+            ListPartitionReassignmentsResponseData reassignmentsData = new ListPartitionReassignmentsResponseData()
+                    .setTopics(Arrays.asList(tp1Reassignment, tp2Reassignment));
+            env.kafkaClient().prepareResponse(new ListPartitionReassignmentsResponse(notControllerData));
+            env.kafkaClient().prepareResponse(controllerNodeResponse);
+            env.kafkaClient().prepareResponse(new ListPartitionReassignmentsResponse(reassignmentsData));
+
+            ListPartitionReassignmentsResult noControllerResult = env.adminClient().listPartitionReassignments();
+            noControllerResult.reassignments().get(); // no error
+
+            // 2. UNKNOWN_TOPIC_OR_EXCEPTION_ERROR
+            ListPartitionReassignmentsResponseData unknownTpData = new ListPartitionReassignmentsResponseData()
+                    .setErrorCode(Errors.UNKNOWN_TOPIC_OR_PARTITION.code())
+                    .setErrorMessage(Errors.UNKNOWN_TOPIC_OR_PARTITION.message());
+            env.kafkaClient().prepareResponse(new ListPartitionReassignmentsResponse(unknownTpData));
+
+            ListPartitionReassignmentsResult unknownTpResult = env.adminClient().listPartitionReassignments(new HashSet<>(Arrays.asList(tp1, tp2)));
+            TestUtils.assertFutureError(unknownTpResult.reassignments(), UnknownTopicOrPartitionException.class);
+
+            // 3. Success
+            ListPartitionReassignmentsResponseData responseData = new ListPartitionReassignmentsResponseData()
+                    .setTopics(Arrays.asList(tp1Reassignment, tp2Reassignment));
+            env.kafkaClient().prepareResponse(new ListPartitionReassignmentsResponse(responseData));
+            ListPartitionReassignmentsResult responseResult = env.adminClient().listPartitionReassignments();
+
+            Map reassignments = responseResult.reassignments().get();
+
+            PartitionReassignment tp1Result = reassignments.get(tp1);
+            assertEquals(tp1PartitionReassignment.addingReplicas(), tp1Result.addingReplicas());
+            assertEquals(tp1PartitionReassignment.removingReplicas(), tp1Result.removingReplicas());
+            assertEquals(tp1PartitionReassignment.replicas(), tp1Result.replicas());
+            assertEquals(tp1PartitionReassignment.replicas(), tp1Result.replicas());
+            PartitionReassignment tp2Result = reassignments.get(tp2);
+            assertEquals(tp2PartitionReassignment.addingReplicas(), tp2Result.addingReplicas());
+            assertEquals(tp2PartitionReassignment.removingReplicas(), tp2Result.removingReplicas());
+            assertEquals(tp2PartitionReassignment.replicas(), tp2Result.replicas());
+            assertEquals(tp2PartitionReassignment.replicas(), tp2Result.replicas());
+        }
+    }
+
     private static MemberDescription convertToMemberDescriptions(DescribedGroupMember member,
                                                                  MemberAssignment assignment) {
         return new MemberDescription(member.memberId(),
diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java b/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java
index 7ca9ce4abbe1d..baaf6613841e3 100644
--- a/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java
+++ b/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java
@@ -40,6 +40,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 
 public class MockAdminClient extends AdminClient {
@@ -420,6 +421,17 @@ public DescribeReplicaLogDirsResult describeReplicaLogDirs(Collection> reassignments,
+                                                                         AlterPartitionReassignmentsOptions options) {
+        throw new UnsupportedOperationException("Not implemented yet");
+    }
+
+    @Override
+    public ListPartitionReassignmentsResult listPartitionReassignments(Optional> partitions, ListPartitionReassignmentsOptions options) {
+        throw new UnsupportedOperationException("Not implemented yet");
+    }
+
     @Override
     public void close(Duration timeout) {}
 
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index a979d4cdee545..8645d795b8463 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -1702,8 +1702,8 @@ private ListPartitionReassignmentsRequest createListPartitionReassignmentsReques
 
     private ListPartitionReassignmentsResponse createListPartitionReassignmentsResponse() {
         ListPartitionReassignmentsResponseData data = new ListPartitionReassignmentsResponseData();
-        data.topics().add(
-                new ListPartitionReassignmentsResponseData.OngoingTopicReassignment()
+        data.setTopics(Collections.singletonList(
+            new ListPartitionReassignmentsResponseData.OngoingTopicReassignment()
                         .setName("topic")
                         .setPartitions(Collections.singletonList(
                                 new ListPartitionReassignmentsResponseData.OngoingPartitionReassignment()
@@ -1713,7 +1713,7 @@ private ListPartitionReassignmentsResponse createListPartitionReassignmentsRespo
                                         .setRemovingReplicas(Collections.singletonList(1))
                                 )
                         )
-        );
+        ));
         return new ListPartitionReassignmentsResponse(data);
     }
 }

From 4d169f5a859ac1d4a34134ef5b6def574769aa43 Mon Sep 17 00:00:00 2001
From: David Jacot 
Date: Wed, 14 Aug 2019 09:56:03 -0700
Subject: [PATCH 0536/1071] KAFKA-7335; Store clusterId locally to ensure
 broker joins the right cluster (#7189)

This patch stores `clusterId` in the `meta.properties` file. During startup, the broker checks that it joins the correct cluster and fails fast otherwise.

The `meta.properties' is versioned. I have decided to not bump the version because 1) the clusterId is null anyway if not present in the file; and 2) bumping it means that rolling back to a previous version won't work.

I have refactored the way the metadata is read and written as it was strongly coupled with the brokerId bits. Now, the metadata is read independently during the startup and used to 1) check the clusterId and 2) get or generate the brokerId (as before).

Reviewers: Stanislav Kozlovski , Jason Gustafson 
---
 .../InconsistentBrokerMetadataException.scala |  27 ++++
 .../InconsistentClusterIdException.scala      |  27 ++++
 .../server/BrokerMetadataCheckpoint.scala     |  14 ++-
 .../main/scala/kafka/server/KafkaServer.scala | 115 +++++++++++-------
 .../server/ServerGenerateClusterIdTest.scala  |  97 ++++++++++++++-
 5 files changed, 232 insertions(+), 48 deletions(-)
 create mode 100644 core/src/main/scala/kafka/common/InconsistentBrokerMetadataException.scala
 create mode 100644 core/src/main/scala/kafka/common/InconsistentClusterIdException.scala

diff --git a/core/src/main/scala/kafka/common/InconsistentBrokerMetadataException.scala b/core/src/main/scala/kafka/common/InconsistentBrokerMetadataException.scala
new file mode 100644
index 0000000000000..2b11512e44ce2
--- /dev/null
+++ b/core/src/main/scala/kafka/common/InconsistentBrokerMetadataException.scala
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.common
+
+/**
+ * Indicates the BrokerMetadata stored in logDirs is not consistent across logDirs.
+ */
+class InconsistentBrokerMetadataException(message: String, cause: Throwable) extends RuntimeException(message, cause) {
+  def this(message: String) = this(message, null)
+  def this(cause: Throwable) = this(null, cause)
+  def this() = this(null, null)
+}
diff --git a/core/src/main/scala/kafka/common/InconsistentClusterIdException.scala b/core/src/main/scala/kafka/common/InconsistentClusterIdException.scala
new file mode 100644
index 0000000000000..6868dd8780de9
--- /dev/null
+++ b/core/src/main/scala/kafka/common/InconsistentClusterIdException.scala
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.common
+
+/**
+ * Indicates the clusterId stored in logDirs is not consistent with the clusterIs stored in ZK.
+ */
+class InconsistentClusterIdException(message: String, cause: Throwable) extends RuntimeException(message, cause) {
+  def this(message: String) = this(message, null)
+  def this(cause: Throwable) = this(null, cause)
+  def this() = this(null, null)
+}
diff --git a/core/src/main/scala/kafka/server/BrokerMetadataCheckpoint.scala b/core/src/main/scala/kafka/server/BrokerMetadataCheckpoint.scala
index 2b915a0f44592..ffbcb5df05968 100755
--- a/core/src/main/scala/kafka/server/BrokerMetadataCheckpoint.scala
+++ b/core/src/main/scala/kafka/server/BrokerMetadataCheckpoint.scala
@@ -24,7 +24,13 @@ import java.util.Properties
 import kafka.utils._
 import org.apache.kafka.common.utils.Utils
 
-case class BrokerMetadata(brokerId: Int)
+case class BrokerMetadata(brokerId: Int,
+                          clusterId: Option[String]) {
+
+  override def toString: String  = {
+    s"BrokerMetadata(brokerId=$brokerId, clusterId=${clusterId.map(_.toString).getOrElse("None")})"
+  }
+}
 
 /**
   * This class saves broker's metadata to a file
@@ -38,6 +44,9 @@ class BrokerMetadataCheckpoint(val file: File) extends Logging {
         val brokerMetaProps = new Properties()
         brokerMetaProps.setProperty("version", 0.toString)
         brokerMetaProps.setProperty("broker.id", brokerMetadata.brokerId.toString)
+        brokerMetadata.clusterId.foreach { clusterId =>
+          brokerMetaProps.setProperty("cluster.id", clusterId)
+        }
         val temp = new File(file.getAbsolutePath + ".tmp")
         val fileOutputStream = new FileOutputStream(temp)
         try {
@@ -66,7 +75,8 @@ class BrokerMetadataCheckpoint(val file: File) extends Logging {
         version match {
           case 0 =>
             val brokerId = brokerMetaProps.getIntInRange("broker.id", (0, Int.MaxValue))
-            return Some(BrokerMetadata(brokerId))
+            val clusterId = Option(brokerMetaProps.getString("cluster.id", null))
+            return Some(BrokerMetadata(brokerId, clusterId))
           case _ =>
             throw new IOException("Unrecognized version of the server meta.properties file: " + version)
         }
diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala
index 6c433b7474e97..21c53b3b91235 100755
--- a/core/src/main/scala/kafka/server/KafkaServer.scala
+++ b/core/src/main/scala/kafka/server/KafkaServer.scala
@@ -26,7 +26,7 @@ import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger}
 import com.yammer.metrics.core.Gauge
 import kafka.api.{KAFKA_0_9_0, KAFKA_2_2_IV0}
 import kafka.cluster.Broker
-import kafka.common.{GenerateBrokerIdException, InconsistentBrokerIdException}
+import kafka.common.{GenerateBrokerIdException, InconsistentBrokerIdException, InconsistentClusterIdException, InconsistentBrokerMetadataException}
 import kafka.controller.KafkaController
 import kafka.coordinator.group.GroupCoordinator
 import kafka.coordinator.transaction.TransactionCoordinator
@@ -210,9 +210,17 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
         _clusterId = getOrGenerateClusterId(zkClient)
         info(s"Cluster ID = $clusterId")
 
+        /* load metadata */
+        val (preloadedBrokerMetadataCheckpoint, initialOfflineDirs) = getBrokerMetadataAndOfflineDirs
+
+        /* check cluster id */
+        if (preloadedBrokerMetadataCheckpoint.clusterId.isDefined && preloadedBrokerMetadataCheckpoint.clusterId.get != clusterId)
+          throw new InconsistentClusterIdException(
+            s"The Cluster ID ${clusterId} doesn't match stored clusterId ${preloadedBrokerMetadataCheckpoint.clusterId} in meta.properties. " +
+            s"The broker is trying to join the wrong cluster. Configured zookeeper.connect may be wrong.")
+
         /* generate brokerId */
-        val (brokerId, initialOfflineDirs) = getBrokerIdAndOfflineDirs
-        config.brokerId = brokerId
+        config.brokerId = getOrGenerateBrokerId(preloadedBrokerMetadataCheckpoint)
         logContext = new LogContext(s"[KafkaServer id=${config.brokerId}] ")
         this.logIdent = logContext.logPrefix
 
@@ -261,8 +269,8 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
         val brokerInfo = createBrokerInfo
         val brokerEpoch = zkClient.registerBroker(brokerInfo)
 
-        // Now that the broker id is successfully registered, checkpoint it
-        checkpointBrokerId(config.brokerId)
+        // Now that the broker is successfully registered, checkpoint its metadata
+        checkpointBrokerMetadata(BrokerMetadata(config.brokerId, Some(clusterId)))
 
         /* start token manager */
         tokenManager = new DelegationTokenManager(config, tokenCache, time , zkClient)
@@ -674,28 +682,24 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
   def boundPort(listenerName: ListenerName): Int = socketServer.boundPort(listenerName)
 
   /**
-    * Generates new brokerId if enabled or reads from meta.properties based on following conditions
-    * 
      
-    * 
    1.  config has no broker.id provided and broker id generation is enabled, generates a broker.id based on Zookeeper's sequence
-    * 
    2.  stored broker.id in meta.properties doesn't match in all the log.dirs throws InconsistentBrokerIdException
-    * 
    3.  config has broker.id and meta.properties contains broker.id if they don't match throws InconsistentBrokerIdException
-    * 
    4.  config has broker.id and there is no meta.properties file, creates new meta.properties and stores broker.id
-    * 
        
-    *
-    * The log directories whose meta.properties can not be accessed due to IOException will be returned to the caller
-    *
-    * @return A 2-tuple containing the brokerId and a sequence of offline log directories.
-    */
-  private def getBrokerIdAndOfflineDirs: (Int, Seq[String]) = {
-    var brokerId = config.brokerId
-    val brokerIdSet = mutable.HashSet[Int]()
+   * Reads the BrokerMetadata. If the BrokerMetadata doesn't match in all the log.dirs, InconsistentBrokerMetadataException is
+   * thrown.
+   *
+   * The log directories whose meta.properties can not be accessed due to IOException will be returned to the caller
+   *
+   * @return A 2-tuple containing the brokerMetadata and a sequence of offline log directories.
+   */
+  private def getBrokerMetadataAndOfflineDirs: (BrokerMetadata, Seq[String]) = {
+    val brokerMetadataMap = mutable.HashMap[String, BrokerMetadata]()
+    val brokerMetadataSet = mutable.HashSet[BrokerMetadata]()
     val offlineDirs = mutable.ArrayBuffer.empty[String]
 
     for (logDir <- config.logDirs) {
       try {
         val brokerMetadataOpt = brokerMetadataCheckpoints(logDir).read()
         brokerMetadataOpt.foreach { brokerMetadata =>
-          brokerIdSet.add(brokerMetadata.brokerId)
+          brokerMetadataMap += (logDir -> brokerMetadata)
+          brokerMetadataSet += brokerMetadata
         }
       } catch {
         case e: IOException =>
@@ -704,39 +708,60 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
       }
     }
 
-    if (brokerIdSet.size > 1)
-      throw new InconsistentBrokerIdException(
-        s"Failed to match broker.id across log.dirs. This could happen if multiple brokers shared a log directory (log.dirs) " +
-        s"or partial data was manually copied from another broker. Found $brokerIdSet")
-    else if (brokerId >= 0 && brokerIdSet.size == 1 && brokerIdSet.last != brokerId)
-      throw new InconsistentBrokerIdException(
-        s"Configured broker.id $brokerId doesn't match stored broker.id ${brokerIdSet.last} in meta.properties. " +
-        s"If you moved your data, make sure your configured broker.id matches. " +
-        s"If you intend to create a new broker, you should remove all data in your data directories (log.dirs).")
-    else if (brokerIdSet.isEmpty && brokerId < 0 && config.brokerIdGenerationEnable) // generate a new brokerId from Zookeeper
-      brokerId = generateBrokerId
-    else if (brokerIdSet.size == 1) // pick broker.id from meta.properties
-      brokerId = brokerIdSet.last
+    if (brokerMetadataSet.size > 1) {
+      val builder = StringBuilder.newBuilder
 
+      for ((logDir, brokerMetadata) <- brokerMetadataMap)
+        builder ++= s"- $logDir -> $brokerMetadata\n"
 
-    (brokerId, offlineDirs)
+      throw new InconsistentBrokerMetadataException(
+        s"BrokerMetadata is not consistent across log.dirs. This could happen if multiple brokers shared a log directory (log.dirs) " +
+        s"or partial data was manually copied from another broker. Found:\n${builder.toString()}"
+      )
+    } else if (brokerMetadataSet.size == 1)
+      (brokerMetadataSet.last, offlineDirs)
+    else
+      (BrokerMetadata(-1, None), offlineDirs)
   }
 
-  private def checkpointBrokerId(brokerId: Int) {
-    var logDirsWithoutMetaProps: List[String] = List()
-
+  /**
+   * Checkpoint the BrokerMetadata to all the online log.dirs
+   *
+   * @param brokerMetadata
+   */
+  private def checkpointBrokerMetadata(brokerMetadata: BrokerMetadata) = {
     for (logDir <- config.logDirs if logManager.isLogDirOnline(new File(logDir).getAbsolutePath)) {
-      val brokerMetadataOpt = brokerMetadataCheckpoints(logDir).read()
-      if (brokerMetadataOpt.isEmpty)
-        logDirsWithoutMetaProps ++= List(logDir)
-    }
-
-    for (logDir <- logDirsWithoutMetaProps) {
       val checkpoint = brokerMetadataCheckpoints(logDir)
-      checkpoint.write(BrokerMetadata(brokerId))
+      checkpoint.write(brokerMetadata)
     }
   }
 
+  /**
+   * Generates new brokerId if enabled or reads from meta.properties based on following conditions
+   * 
          
+   * 
        1.  config has no broker.id provided and broker id generation is enabled, generates a broker.id based on Zookeeper's sequence
+   * 
        2.  config has broker.id and meta.properties contains broker.id if they don't match throws InconsistentBrokerIdException
+   * 
        3.  config has broker.id and there is no meta.properties file, creates new meta.properties and stores broker.id
+   * 
            
+   *
+   * @return The brokerId.
+   */
+  private def getOrGenerateBrokerId(brokerMetadata: BrokerMetadata): Int = {
+    val brokerId = config.brokerId
+
+    if (brokerId >= 0 && brokerMetadata.brokerId >= 0 && brokerMetadata.brokerId != brokerId)
+      throw new InconsistentBrokerIdException(
+        s"Configured broker.id $brokerId doesn't match stored broker.id ${brokerMetadata.brokerId} in meta.properties. " +
+        s"If you moved your data, make sure your configured broker.id matches. " +
+        s"If you intend to create a new broker, you should remove all data in your data directories (log.dirs).")
+    else if (brokerMetadata.brokerId < 0 && brokerId < 0 && config.brokerIdGenerationEnable) // generate a new brokerId from Zookeeper
+      generateBrokerId
+    else if (brokerMetadata.brokerId >= 0) // pick broker.id from meta.properties
+      brokerMetadata.brokerId
+    else
+      brokerId
+  }
+
   /**
     * Return a sequence id generated by updating the broker sequence id path in ZK.
     * Users can provide brokerId in the config. To avoid conflicts between ZK generated
diff --git a/core/src/test/scala/unit/kafka/server/ServerGenerateClusterIdTest.scala b/core/src/test/scala/unit/kafka/server/ServerGenerateClusterIdTest.scala
index e00e6c13cd0c6..4581e23c0349b 100755
--- a/core/src/test/scala/unit/kafka/server/ServerGenerateClusterIdTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ServerGenerateClusterIdTest.scala
@@ -16,20 +16,28 @@
   */
 package kafka.server
 
+import java.io.File
+
+import kafka.common.{InconsistentBrokerMetadataException, InconsistentClusterIdException}
+
 import scala.concurrent._
 import ExecutionContext.Implicits._
 import scala.concurrent.duration._
 import kafka.utils.TestUtils
 import kafka.zk.ZooKeeperTestHarness
 import org.junit.Assert._
-import org.junit.{Before, After, Test}
+import org.junit.{After, Before, Test}
+import org.scalatest.Assertions.assertThrows
 import org.apache.kafka.test.TestUtils.isValidClusterId
 
+import scala.collection.Seq
+
 class ServerGenerateClusterIdTest extends ZooKeeperTestHarness {
   var config1: KafkaConfig = null
   var config2: KafkaConfig = null
   var config3: KafkaConfig = null
   var servers: Seq[KafkaServer] = Seq()
+  val brokerMetaPropsFile = "meta.properties"
 
   @Before
   override def setUp() {
@@ -137,4 +145,91 @@ class ServerGenerateClusterIdTest extends ZooKeeperTestHarness {
     TestUtils.verifyNonDaemonThreadsStatus(this.getClass.getName)
   }
 
+  @Test
+  def testConsistentClusterIdFromZookeeperAndFromMetaProps() = {
+    // Check at the first boot
+    val server = TestUtils.createServer(config1)
+    val clusterId = server.clusterId
+
+    assertTrue(verifyBrokerMetadata(server.config.logDirs, clusterId))
+
+    server.shutdown()
+
+    // Check again after reboot
+    server.startup()
+
+    assertEquals(clusterId, server.clusterId)
+    assertTrue(verifyBrokerMetadata(server.config.logDirs, server.clusterId))
+
+    server.shutdown()
+
+    TestUtils.verifyNonDaemonThreadsStatus(this.getClass.getName)
+  }
+
+  @Test
+  def testInconsistentClusterIdFromZookeeperAndFromMetaProps() = {
+    forgeBrokerMetadata(config1.logDirs, config1.brokerId, "aclusterid")
+
+    val server = new KafkaServer(config1, threadNamePrefix = Option(this.getClass.getName))
+
+    // Startup fails
+    assertThrows[InconsistentClusterIdException] {
+      server.startup()
+    }
+
+    server.shutdown()
+
+    TestUtils.verifyNonDaemonThreadsStatus(this.getClass.getName)
+  }
+
+  @Test
+  def testInconsistentBrokerMetadataBetweenMultipleLogDirs() {
+    // Add multiple logDirs with different BrokerMetadata
+    val logDir1 = TestUtils.tempDir().getAbsolutePath
+    val logDir2 = TestUtils.tempDir().getAbsolutePath
+    val logDirs = logDir1 + "," + logDir2
+
+    forgeBrokerMetadata(logDir1, 1, "ebwOKU-zSieInaFQh_qP4g")
+    forgeBrokerMetadata(logDir2, 1, "blaOKU-zSieInaFQh_qP4g")
+
+    val props = TestUtils.createBrokerConfig(1, zkConnect)
+    props.setProperty("log.dir", logDirs)
+    val config = KafkaConfig.fromProps(props)
+
+    val server = new KafkaServer(config, threadNamePrefix = Option(this.getClass.getName))
+
+    // Startup fails
+    assertThrows[InconsistentBrokerMetadataException] {
+      server.startup()
+    }
+
+    server.shutdown()
+
+    TestUtils.verifyNonDaemonThreadsStatus(this.getClass.getName)
+  }
+
+  def forgeBrokerMetadata(logDirs: Seq[String], brokerId: Int, clusterId: String) {
+    for (logDir <- logDirs) {
+      forgeBrokerMetadata(logDir, brokerId, clusterId)
+    }
+  }
+
+  def forgeBrokerMetadata(logDir: String, brokerId: Int, clusterId: String) {
+    val checkpoint = new BrokerMetadataCheckpoint(
+      new File(logDir + File.separator + brokerMetaPropsFile))
+    checkpoint.write(BrokerMetadata(brokerId, Option(clusterId)))
+  }
+
+  def verifyBrokerMetadata(logDirs: Seq[String], clusterId: String): Boolean = {
+    for (logDir <- logDirs) {
+      val brokerMetadataOpt = new BrokerMetadataCheckpoint(
+        new File(logDir + File.separator + brokerMetaPropsFile)).read()
+      brokerMetadataOpt match {
+        case Some(brokerMetadata) =>
+          if (brokerMetadata.clusterId.isDefined && brokerMetadata.clusterId.get != clusterId) return false
+        case _ => return false
+      }
+    }
+    true
+  }
 }

From ebf78c04c9aa841f7bd702afdbc782cc07b35ad8 Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Wed, 14 Aug 2019 19:44:38 -0700
Subject: [PATCH 0537/1071] KAFKA-8788: Optimize client metadata handling with
 a large number of partitions (#7192)

Credit to @lbradstreet for profiling the producer with a large number of partitions.

Cache `topicMetadata`, `brokers` and `controller` in the `MetadataResponse`
the first time it's needed avoid unnecessary recomputation. We were previously
computing`brokersMap` 4 times per partition in one code path that was invoked from
multiple places. This is a regression introduced via a42f16f980 and first released
in 2.3.0.

The `Cluster` constructor became significantly more allocation heavy due to
2c44e77e2f20, first released in 2.2.0. Replaced `merge` calls with more verbose,
but more efficient code. Added a test to verify that the returned collections are
unmodifiable.

Add `topicAuthorizedOperations` and `clusterAuthorizedOperations` to
`MetadataResponse` and remove `data()` method.

Reviewers: Manikumar Reddy , Lucas Bradstreet , Colin P. McCabe , Stanislav Kozlovski , Justine Olshan 
---
 .../kafka/clients/admin/KafkaAdminClient.java |   4 +-
 .../java/org/apache/kafka/common/Cluster.java |  55 +++++-
 .../common/requests/MetadataResponse.java     | 164 +++++++++++-------
 .../consumer/internals/FetcherTest.java       |   2 +-
 .../internals/StickyPartitionCacheTest.java   |   9 +-
 .../org/apache/kafka/common/ClusterTest.java  |  50 +++++-
 6 files changed, 203 insertions(+), 81 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
index 936dc65a5bd7a..3ed981909576b 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
@@ -1541,7 +1541,7 @@ void handleResponse(AbstractResponse abstractResponse) {
                     }
                     partitions.sort(Comparator.comparingInt(TopicPartitionInfo::partition));
                     TopicDescription topicDescription = new TopicDescription(topicName, isInternal, partitions,
-                        validAclOperations(response.data().topics().find(topicName).topicAuthorizedOperations()));
+                        validAclOperations(response.topicAuthorizedOperations(topicName).get()));
                     future.complete(topicDescription);
                 }
             }
@@ -1600,7 +1600,7 @@ void handleResponse(AbstractResponse abstractResponse) {
                 controllerFuture.complete(controller(response));
                 clusterIdFuture.complete(response.clusterId());
                 authorizedOperationsFuture.complete(
-                        validAclOperations(response.data().clusterAuthorizedOperations()));
+                        validAclOperations(response.clusterAuthorizedOperations()));
             }
 
             private Node controller(MetadataResponse response) {
diff --git a/clients/src/main/java/org/apache/kafka/common/Cluster.java b/clients/src/main/java/org/apache/kafka/common/Cluster.java
index e69be4282eb74..c765cdc0b0c46 100644
--- a/clients/src/main/java/org/apache/kafka/common/Cluster.java
+++ b/clients/src/main/java/org/apache/kafka/common/Cluster.java
@@ -16,8 +16,6 @@
  */
 package org.apache.kafka.common;
 
-import org.apache.kafka.common.utils.Utils;
-
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -108,23 +106,64 @@ private Cluster(String clusterId,
 
         // Index the nodes for quick lookup
         Map tmpNodesById = new HashMap<>();
-        for (Node node : nodes)
+        Map> tmpPartitionsByNode = new HashMap<>(nodes.size());
+        for (Node node : nodes) {
             tmpNodesById.put(node.id(), node);
+            // Populate the map here to make it easy to add the partitions per node efficiently when iterating over
+            // the partitions
+            tmpPartitionsByNode.put(node.id(), new ArrayList<>());
+        }
         this.nodesById = Collections.unmodifiableMap(tmpNodesById);
 
         // index the partition infos by topic, topic+partition, and node
+        // note that this code is performance sensitive if there are a large number of partitions so we are careful
+        // to avoid unnecessary work
         Map tmpPartitionsByTopicPartition = new HashMap<>(partitions.size());
         Map> tmpPartitionsByTopic = new HashMap<>();
-        Map> tmpAvailablePartitionsByTopic = new HashMap<>();
-        Map> tmpPartitionsByNode = new HashMap<>();
         for (PartitionInfo p : partitions) {
             tmpPartitionsByTopicPartition.put(new TopicPartition(p.topic(), p.partition()), p);
-            tmpPartitionsByTopic.merge(p.topic(), Collections.singletonList(p), Utils::concatListsUnmodifiable);
+            List partitionsForTopic = tmpPartitionsByTopic.get(p.topic());
+            if (partitionsForTopic == null) {
+                partitionsForTopic = new ArrayList<>();
+                tmpPartitionsByTopic.put(p.topic(), partitionsForTopic);
+            }
+            partitionsForTopic.add(p);
             if (p.leader() != null) {
-                tmpAvailablePartitionsByTopic.merge(p.topic(), Collections.singletonList(p), Utils::concatListsUnmodifiable);
-                tmpPartitionsByNode.merge(p.leader().id(), Collections.singletonList(p), Utils::concatListsUnmodifiable);
+                // The broker guarantees that if a partition has a non-null leader, it is one of the brokers returned
+                // in the metadata response
+                List partitionsForNode = Objects.requireNonNull(tmpPartitionsByNode.get(p.leader().id()));
+                partitionsForNode.add(p);
             }
         }
+
+        // Update the values of `tmpPartitionsByNode` to contain unmodifiable lists
+        for (Map.Entry> entry : tmpPartitionsByNode.entrySet()) {
+            tmpPartitionsByNode.put(entry.getKey(), Collections.unmodifiableList(entry.getValue()));
+        }
+
+        // Populate `tmpAvailablePartitionsByTopic` and update the values of `tmpPartitionsByTopic` to contain
+        // unmodifiable lists
+        Map> tmpAvailablePartitionsByTopic = new HashMap<>(tmpPartitionsByTopic.size());
+        for (Map.Entry> entry : tmpPartitionsByTopic.entrySet()) {
+            String topic = entry.getKey();
+            List partitionsForTopic = Collections.unmodifiableList(entry.getValue());
+            tmpPartitionsByTopic.put(topic, partitionsForTopic);
+            // Optimise for the common case where all partitions are available
+            boolean foundUnavailablePartition = partitionsForTopic.stream().anyMatch(p -> p.leader() == null);
+            List availablePartitionsForTopic;
+            if (foundUnavailablePartition) {
+                availablePartitionsForTopic = new ArrayList<>(partitionsForTopic.size());
+                for (PartitionInfo p : partitionsForTopic) {
+                    if (p.leader() != null)
+                        availablePartitionsForTopic.add(p);
+                }
+                availablePartitionsForTopic = Collections.unmodifiableList(availablePartitionsForTopic);
+            } else {
+                availablePartitionsForTopic = partitionsForTopic;
+            }
+            tmpAvailablePartitionsByTopic.put(topic, availablePartitionsForTopic);
+        }
+
         this.partitionsByTopicPartition = Collections.unmodifiableMap(tmpPartitionsByTopicPartition);
         this.partitionsByTopic = Collections.unmodifiableMap(tmpPartitionsByTopic);
         this.availablePartitionsByTopic = Collections.unmodifiableMap(tmpAvailablePartitionsByTopic);
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java
index 39b6180eed67a..ef5381b444376 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java
@@ -32,6 +32,7 @@
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -57,17 +58,13 @@ public class MetadataResponse extends AbstractResponse {
 
     public static final int AUTHORIZED_OPERATIONS_OMITTED = Integer.MIN_VALUE;
 
-    private MetadataResponseData data;
+    private final MetadataResponseData data;
+    private volatile Holder holder;
 
     public MetadataResponse(MetadataResponseData data) {
         this.data = data;
     }
 
-    private Map brokersMap() {
-        return data.brokers().stream().collect(
-            Collectors.toMap(MetadataResponseBroker::nodeId, b -> new Node(b.nodeId(), b.host(), b.port(), b.rack())));
-    }
-
     public MetadataResponse(Struct struct, short version) {
         this(new MetadataResponseData(struct, version));
     }
@@ -77,28 +74,6 @@ protected Struct toStruct(short version) {
         return data.toStruct(version);
     }
 
-    public MetadataResponseData data() {
-        return data;
-    }
-
-    private List convertToNodes(Map brokers, List brokerIds) {
-        List nodes = new ArrayList<>(brokerIds.size());
-        for (Integer brokerId : brokerIds)
-            if (brokers.containsKey(brokerId))
-                nodes.add(brokers.get(brokerId));
-            else
-                nodes.add(new Node(brokerId, "", -1));
-        return nodes;
-    }
-
-    private Node getControllerNode(int controllerId, Collection brokers) {
-        for (Node broker : brokers) {
-            if (broker.id() == controllerId)
-                return broker;
-        }
-        return null;
-    }
-
     @Override
     public int throttleTimeMs() {
         return data.throttleTimeMs();
@@ -145,7 +120,6 @@ public Cluster cluster() {
         Set internalTopics = new HashSet<>();
         List partitions = new ArrayList<>();
         for (TopicMetadata metadata : topicMetadata()) {
-
             if (metadata.error == Errors.NONE) {
                 if (metadata.isInternal)
                     internalTopics.add(metadata.topic);
@@ -154,13 +128,30 @@ public Cluster cluster() {
                 }
             }
         }
-        return new Cluster(data.clusterId(), brokersMap().values(), partitions, topicsByError(Errors.TOPIC_AUTHORIZATION_FAILED),
+        return new Cluster(data.clusterId(), brokers(), partitions, topicsByError(Errors.TOPIC_AUTHORIZATION_FAILED),
                 topicsByError(Errors.INVALID_TOPIC_EXCEPTION), internalTopics, controller());
     }
 
     /**
-     * Transform a topic and PartitionMetadata into PartitionInfo
-     * @return
+     * Returns a 32-bit bitfield to represent authorized operations for this topic.
+     */
+    public Optional topicAuthorizedOperations(String topicName) {
+        MetadataResponseTopic topic = data.topics().find(topicName);
+        if (topic == null)
+            return Optional.empty();
+        else
+            return Optional.of(topic.topicAuthorizedOperations());
+    }
+
+    /**
+     * Returns a 32-bit bitfield to represent authorized operations for this cluster.
+     */
+    public int clusterAuthorizedOperations() {
+        return data.clusterAuthorizedOperations();
+    }
+
+    /**
+     * Transform a topic and PartitionMetadata into PartitionInfo.
      */
     public static PartitionInfo partitionMetaToInfo(String topic, PartitionMetadata partitionMetadata) {
         return new PartitionInfo(
@@ -172,12 +163,22 @@ public static PartitionInfo partitionMetaToInfo(String topic, PartitionMetadata
                 partitionMetadata.offlineReplicas().toArray(new Node[0]));
     }
 
+    private Holder holder() {
+        if (holder == null) {
+            synchronized (data) {
+                if (holder == null)
+                    holder = new Holder(data);
+            }
+        }
+        return holder;
+    }
+
     /**
      * Get all brokers returned in metadata response
      * @return the brokers
      */
     public Collection brokers() {
-        return new ArrayList<>(brokersMap().values());
+        return holder().brokers;
     }
 
     /**
@@ -185,30 +186,7 @@ public Collection brokers() {
      * @return the topicMetadata
      */
     public Collection topicMetadata() {
-        List topicMetadataList = new ArrayList<>();
-        for (MetadataResponseTopic topicMetadata : data.topics()) {
-            Errors topicError = Errors.forCode(topicMetadata.errorCode());
-            String topic = topicMetadata.name();
-            boolean isInternal = topicMetadata.isInternal();
-            List partitionMetadataList = new ArrayList<>();
-
-            for (MetadataResponsePartition partitionMetadata : topicMetadata.partitions()) {
-                Errors partitionError = Errors.forCode(partitionMetadata.errorCode());
-                int partitionIndex = partitionMetadata.partitionIndex();
-                int leader = partitionMetadata.leaderId();
-                Optional leaderEpoch = RequestUtils.getLeaderEpoch(partitionMetadata.leaderEpoch());
-                Node leaderNode = leader == -1 ? null : brokersMap().get(leader);
-                List replicaNodes = convertToNodes(brokersMap(), partitionMetadata.replicaNodes());
-                List isrNodes = convertToNodes(brokersMap(), partitionMetadata.isrNodes());
-                List offlineNodes = convertToNodes(brokersMap(), partitionMetadata.offlineReplicas());
-                partitionMetadataList.add(new PartitionMetadata(partitionError, partitionIndex, leaderNode, leaderEpoch,
-                    replicaNodes, isrNodes, offlineNodes));
-            }
-
-            topicMetadataList.add(new TopicMetadata(topicError, topic, isInternal, partitionMetadataList,
-                topicMetadata.topicAuthorizedOperations()));
-        }
-        return  topicMetadataList;
+        return holder().topicMetadata;
     }
 
     /**
@@ -216,7 +194,7 @@ public Collection topicMetadata() {
      * @return the controller node or null if it doesn't exist
      */
     public Node controller() {
-        return getControllerNode(data.controllerId(), brokers());
+        return holder().controller;
     }
 
     /**
@@ -381,18 +359,76 @@ public String toString() {
         }
     }
 
-    public static MetadataResponse prepareResponse(int throttleTimeMs, List brokers, String clusterId,
+    private static class Holder {
+        private final Collection brokers;
+        private final Node controller;
+        private final Collection topicMetadata;
+
+        Holder(MetadataResponseData data) {
+            this.brokers = Collections.unmodifiableCollection(createBrokers(data));
+            Map brokerMap = brokers.stream().collect(Collectors.toMap(Node::id, b -> b));
+            this.topicMetadata = createTopicMetadata(data, brokerMap);
+            this.controller = brokerMap.get(data.controllerId());
+        }
+
+        private Collection createBrokers(MetadataResponseData data) {
+            return data.brokers().valuesList().stream().map(b ->
+                    new Node(b.nodeId(), b.host(), b.port(), b.rack())).collect(Collectors.toList());
+        }
+
+        private Collection createTopicMetadata(MetadataResponseData data, Map brokerMap) {
+            List topicMetadataList = new ArrayList<>();
+            for (MetadataResponseTopic topicMetadata : data.topics()) {
+                Errors topicError = Errors.forCode(topicMetadata.errorCode());
+                String topic = topicMetadata.name();
+                boolean isInternal = topicMetadata.isInternal();
+                List partitionMetadataList = new ArrayList<>();
+
+                for (MetadataResponsePartition partitionMetadata : topicMetadata.partitions()) {
+                    Errors partitionError = Errors.forCode(partitionMetadata.errorCode());
+                    int partitionIndex = partitionMetadata.partitionIndex();
+                    int leader = partitionMetadata.leaderId();
+                    Optional leaderEpoch = RequestUtils.getLeaderEpoch(partitionMetadata.leaderEpoch());
+                    Node leaderNode = leader == -1 ? null : brokerMap.get(leader);
+                    List replicaNodes = convertToNodes(brokerMap, partitionMetadata.replicaNodes());
+                    List isrNodes = convertToNodes(brokerMap, partitionMetadata.isrNodes());
+                    List offlineNodes = convertToNodes(brokerMap, partitionMetadata.offlineReplicas());
+                    partitionMetadataList.add(new PartitionMetadata(partitionError, partitionIndex, leaderNode, leaderEpoch,
+                            replicaNodes, isrNodes, offlineNodes));
+                }
+
+                topicMetadataList.add(new TopicMetadata(topicError, topic, isInternal, partitionMetadataList,
+                        topicMetadata.topicAuthorizedOperations()));
+            }
+            return topicMetadataList;
+        }
+
+        private List convertToNodes(Map brokers, List brokerIds) {
+            List nodes = new ArrayList<>(brokerIds.size());
+            for (Integer brokerId : brokerIds) {
+                Node node = brokers.get(brokerId);
+                if (node == null)
+                    nodes.add(new Node(brokerId, "", -1));
+                else
+                    nodes.add(node);
+            }
+            return nodes;
+        }
+
+    }
+
+    public static MetadataResponse prepareResponse(int throttleTimeMs, Collection brokers, String clusterId,
                                                    int controllerId, List topicMetadataList,
                                                    int clusterAuthorizedOperations) {
         MetadataResponseData responseData = new MetadataResponseData();
         responseData.setThrottleTimeMs(throttleTimeMs);
-        brokers.forEach(broker -> {
+        brokers.forEach(broker ->
             responseData.brokers().add(new MetadataResponseBroker()
                 .setNodeId(broker.id())
                 .setHost(broker.host())
                 .setPort(broker.port())
-                .setRack(broker.rack()));
-        });
+                .setRack(broker.rack()))
+        );
 
         responseData.setClusterId(clusterId);
         responseData.setControllerId(controllerId);
@@ -421,13 +457,13 @@ public static MetadataResponse prepareResponse(int throttleTimeMs, List br
         return new MetadataResponse(responseData);
     }
 
-    public static MetadataResponse prepareResponse(int throttleTimeMs, List brokers, String clusterId,
+    public static MetadataResponse prepareResponse(int throttleTimeMs, Collection brokers, String clusterId,
                                                    int controllerId, List topicMetadataList) {
         return prepareResponse(throttleTimeMs, brokers, clusterId, controllerId, topicMetadataList,
                 MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED);
     }
 
-    public static MetadataResponse prepareResponse(List brokers, String clusterId, int controllerId,
+    public static MetadataResponse prepareResponse(Collection brokers, String clusterId, int controllerId,
                                                    List topicMetadata) {
         return prepareResponse(AbstractResponse.DEFAULT_THROTTLE_TIME, brokers, clusterId, controllerId, topicMetadata);
     }
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
index 5f66689520f36..9e2621af53445 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
@@ -1939,7 +1939,7 @@ public void testGetTopicMetadataOfflinePartitions() {
         }
         Node controller = originalResponse.controller();
         MetadataResponse altered = MetadataResponse.prepareResponse(
-            (List) originalResponse.brokers(),
+            originalResponse.brokers(),
             originalResponse.clusterId(),
             controller != null ? controller.id() : MetadataResponse.NO_CONTROLLER_ID,
             altTopics);
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/StickyPartitionCacheTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/StickyPartitionCacheTest.java
index 6878ba1d6871a..9f4a481c9a071 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/StickyPartitionCacheTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/StickyPartitionCacheTest.java
@@ -32,7 +32,8 @@ public class StickyPartitionCacheTest {
     private final static Node[] NODES = new Node[] {
         new Node(0, "localhost", 99),
         new Node(1, "localhost", 100),
-        new Node(12, "localhost", 101)
+        new Node(2, "localhost", 101),
+        new Node(11, "localhost", 102)
     };
     final static String TOPIC_A = "topicA";
     final static String TOPIC_B = "topicB";
@@ -46,7 +47,7 @@ public void testStickyPartitionCache() {
             new PartitionInfo(TOPIC_B, 0, NODES[0], NODES, NODES)
         );
         Cluster testCluster = new Cluster("clusterId", asList(NODES), allPartitions,
-            Collections.emptySet(), Collections.emptySet());
+            Collections.emptySet(), Collections.emptySet());
         StickyPartitionCache stickyPartitionCache = new StickyPartitionCache();
 
         int partA = stickyPartitionCache.partition(TOPIC_A, testCluster);
@@ -81,8 +82,8 @@ public void unavailablePartitionsTest() {
             new PartitionInfo(TOPIC_C, 0, null, NODES, NODES)
         );
         
-        Cluster testCluster = new Cluster("clusterId", asList(NODES[0], NODES[1]), allPartitions,
-            Collections.emptySet(), Collections.emptySet());
+        Cluster testCluster = new Cluster("clusterId", asList(NODES[0], NODES[1], NODES[2]), allPartitions,
+            Collections.emptySet(), Collections.emptySet());
         StickyPartitionCache stickyPartitionCache = new StickyPartitionCache();
         
         // Assure we never choose partition 1 because it is unavailable.
diff --git a/clients/src/test/java/org/apache/kafka/common/ClusterTest.java b/clients/src/test/java/org/apache/kafka/common/ClusterTest.java
index 0a7049bcc8899..2c80d08a3747f 100644
--- a/clients/src/test/java/org/apache/kafka/common/ClusterTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/ClusterTest.java
@@ -22,19 +22,35 @@
 import java.net.InetSocketAddress;
 import java.util.Arrays;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
+import static java.util.Arrays.asList;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
 
 public class ClusterTest {
 
+    private final static Node[] NODES = new Node[] {
+        new Node(0, "localhost", 99),
+        new Node(1, "localhost", 100),
+        new Node(2, "localhost", 101),
+        new Node(11, "localhost", 102)
+    };
+
+    private final static String TOPIC_A = "topicA";
+    private final static String TOPIC_B = "topicB";
+    private final static String TOPIC_C = "topicC";
+    private final static String TOPIC_D = "topicD";
+    private final static String TOPIC_E = "topicE";
+
     @Test
     public void testBootstrap() {
         String ipAddress = "140.211.11.105";
         String hostName = "www.example.com";
         Cluster cluster = Cluster.bootstrap(Arrays.asList(
-                new InetSocketAddress(ipAddress, 9002),
-                new InetSocketAddress(hostName, 9002)
+            new InetSocketAddress(ipAddress, 9002),
+            new InetSocketAddress(hostName, 9002)
         ));
         Set expectedHosts = Utils.mkSet(ipAddress, hostName);
         Set actualHosts = new HashSet<>();
@@ -43,4 +59,34 @@ public void testBootstrap() {
         assertEquals(expectedHosts, actualHosts);
     }
 
+    @Test
+    public void testReturnUnmodifiableCollections() {
+        List allPartitions = asList(new PartitionInfo(TOPIC_A, 0, NODES[0], NODES, NODES),
+            new PartitionInfo(TOPIC_A, 1, null, NODES, NODES),
+            new PartitionInfo(TOPIC_A, 2, NODES[2], NODES, NODES),
+            new PartitionInfo(TOPIC_B, 0, null, NODES, NODES),
+            new PartitionInfo(TOPIC_B, 1, NODES[0], NODES, NODES),
+            new PartitionInfo(TOPIC_C, 0, null, NODES, NODES),
+            new PartitionInfo(TOPIC_D, 0, NODES[1], NODES, NODES),
+            new PartitionInfo(TOPIC_E, 0, NODES[0], NODES, NODES)
+        );
+        Set unauthorizedTopics = Utils.mkSet(TOPIC_C);
+        Set invalidTopics = Utils.mkSet(TOPIC_D);
+        Set internalTopics = Utils.mkSet(TOPIC_E);
+        Cluster cluster = new Cluster("clusterId", asList(NODES), allPartitions, unauthorizedTopics,
+            invalidTopics, internalTopics, NODES[1]);
+
+        assertThrows(UnsupportedOperationException.class, () -> cluster.invalidTopics().add("foo"));
+        assertThrows(UnsupportedOperationException.class, () -> cluster.internalTopics().add("foo"));
+        assertThrows(UnsupportedOperationException.class, () -> cluster.unauthorizedTopics().add("foo"));
+        assertThrows(UnsupportedOperationException.class, () -> cluster.topics().add("foo"));
+        assertThrows(UnsupportedOperationException.class, () -> cluster.nodes().add(NODES[3]));
+        assertThrows(UnsupportedOperationException.class, () -> cluster.partitionsForTopic(TOPIC_A).add(
+            new PartitionInfo(TOPIC_A, 3, NODES[0], NODES, NODES)));
+        assertThrows(UnsupportedOperationException.class, () -> cluster.availablePartitionsForTopic(TOPIC_B).add(
+            new PartitionInfo(TOPIC_B, 2, NODES[0], NODES, NODES)));
+        assertThrows(UnsupportedOperationException.class, () -> cluster.partitionsForNode(NODES[1].id()).add(
+            new PartitionInfo(TOPIC_B, 2, NODES[1], NODES, NODES)));
+    }
+
 }

From b13391e0fe0b249d46e47d886fbdafaaed7c1eef Mon Sep 17 00:00:00 2001
From: David Jacot 
Date: Thu, 15 Aug 2019 09:53:56 -0700
Subject: [PATCH 0538/1071] MINOR: Update the javadoc of
 SocketServer#startup(). (#7215)

Update the javadoc of SockerServer#startup(). SocketServer#startProcessors() does not exist any more and it has been replaced by SocketServer#startDataPlaneProcessors() and SocketServer#startControlPlaneProcessor().

Reviewers: Jason Gustafson 
---
 core/src/main/scala/kafka/network/SocketServer.scala | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala
index 69f02ae5799be..0ebded825980c 100644
--- a/core/src/main/scala/kafka/network/SocketServer.scala
+++ b/core/src/main/scala/kafka/network/SocketServer.scala
@@ -103,7 +103,8 @@ class SocketServer(val config: KafkaConfig,
   /**
    * Start the socket server. Acceptors for all the listeners are started. Processors
    * are started if `startupProcessors` is true. If not, processors are only started when
-   * [[kafka.network.SocketServer#startProcessors()]] is invoked. Delayed starting of processors
+   * [[kafka.network.SocketServer#startDataPlaneProcessors()]] or
+   * [[kafka.network.SocketServer#startControlPlaneProcessor()]] is invoked. Delayed starting of processors
    * is used to delay processing client connections until server is fully initialized, e.g.
    * to ensure that all credentials have been loaded before authentications are performed.
    * Acceptors are always started during `startup` so that the bound port is known when this

From 14215d1b84e937c4656e0984c1ce76d9aac65bdd Mon Sep 17 00:00:00 2001
From: Bob Barrett 
Date: Thu, 15 Aug 2019 13:47:56 -0700
Subject: [PATCH 0539/1071] MINOR: Use max retries for consumer group tests to
 avoid flakiness (#7186)

This patch updates ConsumerGroupCommandTest.scala to use the maximum possible number of AdminClient retries. The test runs will still be bounded by the request timeout. This address flakiness in tests such as testResetOffsetsNotExistingGroup and testResetOffsetsExistingTopic, which was caused by group coordinators being intermittently unavailable.

Reviewers: Ismael Juma , Jason Gustafson 
---
 .../kafka/admin/ConsumerGroupCommand.scala    |  8 ++--
 .../admin/ConsumerGroupCommandTest.scala      |  5 +-
 .../admin/DeleteConsumerGroupsTest.scala      | 24 +++++-----
 .../admin/DescribeConsumerGroupTest.scala     | 46 +++++++++----------
 .../kafka/admin/ListConsumerGroupTest.scala   |  2 +-
 .../scala/unit/kafka/utils/TestUtils.scala    | 23 +++-------
 6 files changed, 51 insertions(+), 57 deletions(-)

diff --git a/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala b/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala
index 3f2ed322a552d..8c4c5e4dbd5b6 100755
--- a/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala
+++ b/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala
@@ -158,9 +158,10 @@ object ConsumerGroupCommand extends Logging {
     }
   }
 
-  class ConsumerGroupService(val opts: ConsumerGroupCommandOptions) {
+  class ConsumerGroupService(val opts: ConsumerGroupCommandOptions,
+                             private[admin] val configOverrides: Map[String, String] = Map.empty) {
 
-    private val adminClient = createAdminClient()
+    private val adminClient = createAdminClient(configOverrides)
 
     // `consumers` are only needed for `describe`, so we instantiate them lazily
     private lazy val consumers: mutable.Map[String, KafkaConsumer[String, String]] = mutable.Map.empty
@@ -528,9 +529,10 @@ object ConsumerGroupCommand extends Logging {
       )
     }
 
-    private def createAdminClient(): Admin = {
+    private def createAdminClient(configOverrides: Map[String, String]): Admin = {
       val props = if (opts.options.has(opts.commandConfigOpt)) Utils.loadProps(opts.options.valueOf(opts.commandConfigOpt)) else new Properties()
       props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, opts.options.valueOf(opts.bootstrapServerOpt))
+      configOverrides.foreach { case (k, v) => props.put(k, v)}
       admin.AdminClient.create(props)
     }
 
diff --git a/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala b/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
index d5eea981635c6..c3989407d3c02 100644
--- a/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
@@ -25,14 +25,15 @@ import kafka.admin.ConsumerGroupCommand.{ConsumerGroupCommandOptions, ConsumerGr
 import kafka.integration.KafkaServerTestHarness
 import kafka.server.KafkaConfig
 import kafka.utils.TestUtils
+import org.apache.kafka.clients.admin.AdminClientConfig
 import org.apache.kafka.clients.consumer.{KafkaConsumer, RangeAssignor}
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.WakeupException
 import org.apache.kafka.common.serialization.StringDeserializer
 import org.junit.{After, Before}
 
-import scala.collection.mutable.ArrayBuffer
 import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
 
 class ConsumerGroupCommandTest extends KafkaServerTestHarness {
   import ConsumerGroupCommandTest._
@@ -84,7 +85,7 @@ class ConsumerGroupCommandTest extends KafkaServerTestHarness {
 
   def getConsumerGroupService(args: Array[String]): ConsumerGroupService = {
     val opts = new ConsumerGroupCommandOptions(args)
-    val service = new ConsumerGroupService(opts)
+    val service = new ConsumerGroupService(opts, Map(AdminClientConfig.RETRIES_CONFIG -> Int.MaxValue.toString))
     consumerGroupService = service :: consumerGroupService
     service
   }
diff --git a/core/src/test/scala/unit/kafka/admin/DeleteConsumerGroupsTest.scala b/core/src/test/scala/unit/kafka/admin/DeleteConsumerGroupsTest.scala
index 769f33adabff6..63fb84a8bf998 100644
--- a/core/src/test/scala/unit/kafka/admin/DeleteConsumerGroupsTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/DeleteConsumerGroupsTest.scala
@@ -71,7 +71,7 @@ class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
 
     TestUtils.waitUntilTrue(() => {
       service.collectGroupMembers(group, false)._2.get.size == 1
-    }, "The group did not initialize as expected.", maxRetries = 3)
+    }, "The group did not initialize as expected.")
 
     val output = TestUtils.grabConsoleOutput(service.deleteGroups())
     assertTrue(s"The expected error (${Errors.NON_EMPTY_GROUP}) was not detected while deleting consumer group. Output was: (${output})",
@@ -89,7 +89,7 @@ class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
 
     TestUtils.waitUntilTrue(() => {
       service.collectGroupMembers(group, false)._2.get.size == 1
-    }, "The group did not initialize as expected.", maxRetries = 3)
+    }, "The group did not initialize as expected.")
 
     val result = service.deleteGroups()
     assertNotNull(s"Group was deleted successfully, but it shouldn't have been. Result was:(${result})", result(group))
@@ -108,13 +108,13 @@ class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
 
     TestUtils.waitUntilTrue(() => {
       service.listGroups().contains(group)
-    }, "The group did not initialize as expected.", maxRetries = 3)
+    }, "The group did not initialize as expected.")
 
     executor.shutdown()
 
     TestUtils.waitUntilTrue(() => {
       service.collectGroupState(group).state == "Empty"
-    }, "The group did not become empty as expected.", maxRetries = 3)
+    }, "The group did not become empty as expected.")
 
     val output = TestUtils.grabConsoleOutput(service.deleteGroups())
     assertTrue(s"The consumer group could not be deleted as expected",
@@ -138,14 +138,14 @@ class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
 
     TestUtils.waitUntilTrue(() => {
       service.listGroups().forall(groupId => groups.keySet.contains(groupId))
-    }, "The group did not initialize as expected.", maxRetries = 3)
+    }, "The group did not initialize as expected.")
 
     // Shutdown consumers to empty out groups
     groups.values.foreach(executor => executor.shutdown())
 
     TestUtils.waitUntilTrue(() => {
       groups.keySet.forall(groupId => service.collectGroupState(groupId).state == "Empty")
-    }, "The group did not become empty as expected.", maxRetries = 3)
+    }, "The group did not become empty as expected.")
 
     val output = TestUtils.grabConsoleOutput(service.deleteGroups()).trim
     val expectedGroupsForDeletion = groups.keySet
@@ -169,13 +169,13 @@ class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
 
     TestUtils.waitUntilTrue(() => {
       service.listGroups().contains(group)
-    }, "The group did not initialize as expected.", maxRetries = 3)
+    }, "The group did not initialize as expected.")
 
     executor.shutdown()
 
     TestUtils.waitUntilTrue(() => {
       service.collectGroupState(group).state == "Empty"
-    }, "The group did not become empty as expected.", maxRetries = 3)
+    }, "The group did not become empty as expected.")
 
     val result = service.deleteGroups()
     assertTrue(s"The consumer group could not be deleted as expected",
@@ -194,13 +194,13 @@ class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
 
     TestUtils.waitUntilTrue(() => {
       service.listGroups().contains(group)
-    }, "The group did not initialize as expected.", maxRetries = 3)
+    }, "The group did not initialize as expected.")
 
     executor.shutdown()
 
     TestUtils.waitUntilTrue(() => {
       service.collectGroupState(group).state == "Empty"
-    }, "The group did not become empty as expected.", maxRetries = 3)
+    }, "The group did not become empty as expected.")
 
     val service2 = getConsumerGroupService(cgcArgs ++ Array("--group", missingGroup))
     val output = TestUtils.grabConsoleOutput(service2.deleteGroups())
@@ -221,13 +221,13 @@ class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
 
     TestUtils.waitUntilTrue(() => {
       service.listGroups().contains(group)
-    }, "The group did not initialize as expected.", maxRetries = 3)
+    }, "The group did not initialize as expected.")
 
     executor.shutdown()
 
     TestUtils.waitUntilTrue(() => {
       service.collectGroupState(group).state == "Empty"
-    }, "The group did not become empty as expected.", maxRetries = 3)
+    }, "The group did not become empty as expected.")
 
     val service2 = getConsumerGroupService(cgcArgs ++ Array("--group", missingGroup))
     val result = service2.deleteGroups()
diff --git a/core/src/test/scala/unit/kafka/admin/DescribeConsumerGroupTest.scala b/core/src/test/scala/unit/kafka/admin/DescribeConsumerGroupTest.scala
index dcad72a87a83c..68a4f3146bf3a 100644
--- a/core/src/test/scala/unit/kafka/admin/DescribeConsumerGroupTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/DescribeConsumerGroupTest.scala
@@ -127,7 +127,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
       TestUtils.waitUntilTrue(() => {
         val (output, error) = TestUtils.grabConsoleOutputAndError(service.describeGroups())
         output.trim.split("\n").length == 2 && error.isEmpty
-      }, s"Expected a data row and no error in describe results with describe type ${describeType.mkString(" ")}.", maxRetries = 3)
+      }, s"Expected a data row and no error in describe results with describe type ${describeType.mkString(" ")}.")
     }
   }
 
@@ -152,7 +152,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
         val (output, error) = TestUtils.grabConsoleOutputAndError(service.describeGroups())
         val numLines = output.trim.split("\n").filterNot(line => line.isEmpty).length
         (numLines == expectedNumLines) && error.isEmpty
-      }, s"Expected a data row and no error in describe results with describe type ${describeType.mkString(" ")}.", maxRetries = 3)
+      }, s"Expected a data row and no error in describe results with describe type ${describeType.mkString(" ")}.")
     }
   }
 
@@ -176,7 +176,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
         val (output, error) = TestUtils.grabConsoleOutputAndError(service.describeGroups())
         val numLines = output.trim.split("\n").filterNot(line => line.isEmpty).length
         (numLines == expectedNumLines) && error.isEmpty
-      }, s"Expected a data row and no error in describe results with describe type ${describeType.mkString(" ")}.", maxRetries = 3)
+      }, s"Expected a data row and no error in describe results with describe type ${describeType.mkString(" ")}.")
     }
   }
 
@@ -198,7 +198,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
         assignments.get.filter(_.group == group).head.consumerId.exists(_.trim != ConsumerGroupCommand.MISSING_COLUMN_VALUE) &&
         assignments.get.filter(_.group == group).head.clientId.exists(_.trim != ConsumerGroupCommand.MISSING_COLUMN_VALUE) &&
         assignments.get.filter(_.group == group).head.host.exists(_.trim != ConsumerGroupCommand.MISSING_COLUMN_VALUE)
-    }, s"Expected a 'Stable' group status, rows and valid values for consumer id / client id / host columns in describe results for group $group.", maxRetries = 3)
+    }, s"Expected a 'Stable' group status, rows and valid values for consumer id / client id / host columns in describe results for group $group.")
   }
 
   @Test
@@ -222,7 +222,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
           case None =>
             false
         })
-    }, s"Expected a 'Stable' group status, rows and valid member information for group $group.", maxRetries = 3)
+    }, s"Expected a 'Stable' group status, rows and valid member information for group $group.")
 
     val (_, assignments) = service.collectGroupMembers(group, true)
     assignments match {
@@ -251,7 +251,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
         state.assignmentStrategy == "range" &&
         state.coordinator != null &&
         servers.map(_.config.brokerId).toList.contains(state.coordinator.id)
-    }, s"Expected a 'Stable' group status, with one member and round robin assignment strategy for group $group.", maxRetries = 3)
+    }, s"Expected a 'Stable' group status, with one member and round robin assignment strategy for group $group.")
   }
 
   @Test
@@ -270,7 +270,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
         state.assignmentStrategy == "roundrobin" &&
         state.coordinator != null &&
         servers.map(_.config.brokerId).toList.contains(state.coordinator.id)
-    }, s"Expected a 'Stable' group status, with one member and round robin assignment strategy for group $group.", maxRetries = 3)
+    }, s"Expected a 'Stable' group status, with one member and round robin assignment strategy for group $group.")
   }
 
   @Test
@@ -287,13 +287,13 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
       TestUtils.waitUntilTrue(() => {
         val (output, error) = TestUtils.grabConsoleOutputAndError(service.describeGroups())
         output.trim.split("\n").length == 2 && error.isEmpty
-      }, s"Expected describe group results with one data row for describe type '${describeType.mkString(" ")}'", maxRetries = 3)
+      }, s"Expected describe group results with one data row for describe type '${describeType.mkString(" ")}'")
 
       // stop the consumer so the group has no active member anymore
       executor.shutdown()
       TestUtils.waitUntilTrue(() => {
         TestUtils.grabConsoleError(service.describeGroups()).contains(s"Consumer group '$group' has no active members.")
-      }, s"Expected no active member in describe group results with describe type ${describeType.mkString(" ")}", maxRetries = 3)
+      }, s"Expected no active member in describe group results with describe type ${describeType.mkString(" ")}")
     }
   }
 
@@ -310,7 +310,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
     TestUtils.waitUntilTrue(() => {
       val (state, assignments) = service.collectGroupOffsets(group)
       state.contains("Stable") && assignments.exists(_.exists(_.group == group))
-    }, "Expected the group to initially become stable, and to find group in assignments after initial offset commit.", maxRetries = 3)
+    }, "Expected the group to initially become stable, and to find group in assignments after initial offset commit.")
 
     // stop the consumer so the group has no active member anymore
     executor.shutdown()
@@ -343,7 +343,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
     TestUtils.waitUntilTrue(() => {
       val (state, assignments) = service.collectGroupMembers(group, false)
       state.contains("Stable") && assignments.exists(_.exists(_.group == group))
-    }, "Expected the group to initially become stable, and to find group in assignments after initial offset commit.", maxRetries = 3)
+    }, "Expected the group to initially become stable, and to find group in assignments after initial offset commit.")
 
     // stop the consumer so the group has no active member anymore
     executor.shutdown()
@@ -351,7 +351,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
     TestUtils.waitUntilTrue(() => {
       val (state, assignments) = service.collectGroupMembers(group, false)
       state.contains("Empty") && assignments.isDefined && assignments.get.isEmpty
-    }, s"Expected no member in describe group members results for group '$group'", maxRetries = 3)
+    }, s"Expected no member in describe group members results for group '$group'")
   }
 
   @Test
@@ -370,7 +370,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
         state.numMembers == 1 &&
         state.coordinator != null &&
         servers.map(_.config.brokerId).toList.contains(state.coordinator.id)
-    }, s"Expected the group '$group' to initially become stable, and have a single member.", maxRetries = 3)
+    }, s"Expected the group '$group' to initially become stable, and have a single member.")
 
     // stop the consumer so the group has no active member anymore
     executor.shutdown()
@@ -378,7 +378,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
     TestUtils.waitUntilTrue(() => {
       val state = service.collectGroupState(group)
       state.state == "Empty" && state.numMembers == 0 && state.assignmentStrategy == ""
-    }, s"Expected the group '$group' to become empty after the only member leaving.", maxRetries = 3)
+    }, s"Expected the group '$group' to become empty after the only member leaving.")
   }
 
   @Test
@@ -396,7 +396,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
         val (output, error) = TestUtils.grabConsoleOutputAndError(service.describeGroups())
         val expectedNumRows = if (describeTypeMembers.contains(describeType)) 3 else 2
         error.isEmpty && output.trim.split("\n").size == expectedNumRows
-      }, s"Expected a single data row in describe group result with describe type '${describeType.mkString(" ")}'", maxRetries = 3)
+      }, s"Expected a single data row in describe group result with describe type '${describeType.mkString(" ")}'")
     }
   }
 
@@ -416,7 +416,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
         assignments.isDefined &&
         assignments.get.count(_.group == group) == 1 &&
         assignments.get.count { x => x.group == group && x.partition.isDefined } == 1
-    }, "Expected rows for consumers with no assigned partitions in describe group results", maxRetries = 3)
+    }, "Expected rows for consumers with no assigned partitions in describe group results")
   }
 
   @Test
@@ -437,7 +437,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
         assignments.get.count { x => x.group == group && x.numPartitions == 1 } == 1 &&
         assignments.get.count { x => x.group == group && x.numPartitions == 0 } == 1 &&
         assignments.get.count(_.assignment.nonEmpty) == 0
-    }, "Expected rows for consumers with no assigned partitions in describe group results", maxRetries = 3)
+    }, "Expected rows for consumers with no assigned partitions in describe group results")
 
     val (state, assignments) = service.collectGroupMembers(group, true)
     assertTrue("Expected additional columns in verbose version of describe members",
@@ -457,7 +457,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
     TestUtils.waitUntilTrue(() => {
       val state = service.collectGroupState(group)
       state.state == "Stable" && state.numMembers == 2
-    }, "Expected two consumers in describe group results", maxRetries = 3)
+    }, "Expected two consumers in describe group results")
   }
 
   @Test
@@ -477,7 +477,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
         val (output, error) = TestUtils.grabConsoleOutputAndError(service.describeGroups())
         val expectedNumRows = if (describeTypeState.contains(describeType)) 2 else 3
         error.isEmpty && output.trim.split("\n").size == expectedNumRows
-      }, s"Expected a single data row in describe group result with describe type '${describeType.mkString(" ")}'", maxRetries = 3)
+      }, s"Expected a single data row in describe group result with describe type '${describeType.mkString(" ")}'")
     }
   }
 
@@ -500,7 +500,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
         assignments.get.count(_.group == group) == 2 &&
         assignments.get.count{ x => x.group == group && x.partition.isDefined} == 2 &&
         assignments.get.count{ x => x.group == group && x.partition.isEmpty} == 0
-    }, "Expected two rows (one row per consumer) in describe group results.", maxRetries = 3)
+    }, "Expected two rows (one row per consumer) in describe group results.")
   }
 
   @Test
@@ -522,7 +522,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
         assignments.get.count(_.group == group) == 2 &&
         assignments.get.count{ x => x.group == group && x.numPartitions == 1 } == 2 &&
         assignments.get.count{ x => x.group == group && x.numPartitions == 0 } == 0
-    }, "Expected two rows (one row per consumer) in describe group members results.", maxRetries = 3)
+    }, "Expected two rows (one row per consumer) in describe group members results.")
 
     val (state, assignments) = service.collectGroupMembers(group, true)
     assertTrue("Expected additional columns in verbose version of describe members",
@@ -544,7 +544,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
     TestUtils.waitUntilTrue(() => {
       val state = service.collectGroupState(group)
       state.state == "Stable" && state.group == group && state.numMembers == 2
-    }, "Expected a stable group with two members in describe group state result.", maxRetries = 3)
+    }, "Expected a stable group with two members in describe group state result.")
   }
 
   @Test
@@ -562,7 +562,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
     TestUtils.waitUntilTrue(() => {
       val (state, assignments) = service.collectGroupOffsets(group)
       state.contains("Empty") && assignments.isDefined && assignments.get.count(_.group == group) == 2
-    }, "Expected a stable group with two members in describe group state result.", maxRetries = 3)
+    }, "Expected a stable group with two members in describe group state result.")
   }
 
   @Test
diff --git a/core/src/test/scala/unit/kafka/admin/ListConsumerGroupTest.scala b/core/src/test/scala/unit/kafka/admin/ListConsumerGroupTest.scala
index 1a35c4ccd502e..32f6614f9285a 100644
--- a/core/src/test/scala/unit/kafka/admin/ListConsumerGroupTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ListConsumerGroupTest.scala
@@ -36,7 +36,7 @@ class ListConsumerGroupTest extends ConsumerGroupCommandTest {
     TestUtils.waitUntilTrue(() => {
       foundGroups = service.listGroups().toSet
       expectedGroups == foundGroups
-    }, s"Expected --list to show groups $expectedGroups, but found $foundGroups.", maxRetries = 3)
+    }, s"Expected --list to show groups $expectedGroups, but found $foundGroups.")
   }
 
   @Test(expected = classOf[OptionException])
diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index 960ab4c03a038..30ca2d8bdb444 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -808,27 +808,18 @@ object TestUtils extends Logging {
     * @param msg error message
     * @param waitTimeMs maximum time to wait and retest the condition before failing the test
     * @param pause delay between condition checks
-    * @param maxRetries maximum number of retries to check the given condition if a retriable exception is thrown
     */
   def waitUntilTrue(condition: () => Boolean, msg: => String,
-                    waitTimeMs: Long = JTestUtils.DEFAULT_MAX_WAIT_MS, pause: Long = 100L, maxRetries: Int = 0): Unit = {
+                    waitTimeMs: Long = JTestUtils.DEFAULT_MAX_WAIT_MS, pause: Long = 100L): Unit = {
     val startTime = System.currentTimeMillis()
-    var retry = 0
     while (true) {
-      try {
-        if (condition())
-          return
-        if (System.currentTimeMillis() > startTime + waitTimeMs)
-          fail(msg)
-        Thread.sleep(waitTimeMs.min(pause))
-      }
-      catch {
-        case e: RetriableException if retry < maxRetries =>
-          debug("Retrying after error", e)
-          retry += 1
-        case e : Throwable => throw e
-      }
+      if (condition())
+        return
+      if (System.currentTimeMillis() > startTime + waitTimeMs)
+        fail(msg)
+      Thread.sleep(waitTimeMs.min(pause))
     }
+
     // should never hit here
     throw new RuntimeException("unexpected error")
   }

From 83a575d4b92a7a49917244dbf7cdee0d2c5aa253 Mon Sep 17 00:00:00 2001
From: Mickael Maison 
Date: Thu, 15 Aug 2019 23:49:29 +0100
Subject: [PATCH 0540/1071] MINOR: Fix bugs in handling zero-length
 ImplicitLinkedHashCollections (#7163)

Reviewers: Colin P. McCabe 
---
 .../common/utils/ImplicitLinkedHashCollection.java   |  2 +-
 .../utils/ImplicitLinkedHashMultiCollection.java     |  4 ++--
 .../utils/ImplicitLinkedHashCollectionTest.java      | 12 +++++++++++-
 .../utils/ImplicitLinkedHashMultiCollectionTest.java | 10 ++++++++++
 4 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/utils/ImplicitLinkedHashCollection.java b/clients/src/main/java/org/apache/kafka/common/utils/ImplicitLinkedHashCollection.java
index e060629b817c7..fba7d7ac584ec 100644
--- a/clients/src/main/java/org/apache/kafka/common/utils/ImplicitLinkedHashCollection.java
+++ b/clients/src/main/java/org/apache/kafka/common/utils/ImplicitLinkedHashCollection.java
@@ -304,7 +304,7 @@ final int slot(Element[] curElements, Object e) {
      * @return                  The match index, or INVALID_INDEX if no match was found.
      */
     final private int findIndexOfEqualElement(Object key) {
-        if (key == null) {
+        if (key == null || size == 0) {
             return INVALID_INDEX;
         }
         int slot = slot(elements, key);
diff --git a/clients/src/main/java/org/apache/kafka/common/utils/ImplicitLinkedHashMultiCollection.java b/clients/src/main/java/org/apache/kafka/common/utils/ImplicitLinkedHashMultiCollection.java
index 85714d63bfd15..b5ae8f9793a46 100644
--- a/clients/src/main/java/org/apache/kafka/common/utils/ImplicitLinkedHashMultiCollection.java
+++ b/clients/src/main/java/org/apache/kafka/common/utils/ImplicitLinkedHashMultiCollection.java
@@ -91,7 +91,7 @@ int addInternal(Element newElement, Element[] addElements) {
      */
     @Override
     int findElementToRemove(Object key) {
-        if (key == null) {
+        if (key == null || size() == 0) {
             return INVALID_INDEX;
         }
         int slot = slot(elements, key);
@@ -120,7 +120,7 @@ int findElementToRemove(Object key) {
      * @return          All of the matching elements.
      */
     final public List findAll(E key) {
-        if (key == null) {
+        if (key == null || size() == 0) {
             return Collections.emptyList();
         }
         ArrayList results = new ArrayList<>();
diff --git a/clients/src/test/java/org/apache/kafka/common/utils/ImplicitLinkedHashCollectionTest.java b/clients/src/test/java/org/apache/kafka/common/utils/ImplicitLinkedHashCollectionTest.java
index 8c102dd6a2d4a..389c24e456b96 100644
--- a/clients/src/test/java/org/apache/kafka/common/utils/ImplicitLinkedHashCollectionTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/utils/ImplicitLinkedHashCollectionTest.java
@@ -31,6 +31,7 @@
 import java.util.Set;
 
 import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertEquals;
@@ -271,7 +272,7 @@ public void testListViewModification() {
     @Test
     public void testEmptyListIterator() {
         ImplicitLinkedHashCollection coll = new ImplicitLinkedHashCollection<>();
-        ListIterator iter = coll.valuesList().listIterator();
+        ListIterator iter = coll.valuesList().listIterator();
         assertFalse(iter.hasNext());
         assertFalse(iter.hasPrevious());
         assertEquals(0, iter.nextIndex());
@@ -513,6 +514,14 @@ public void testEquals() {
         assertNotEquals(coll2, coll3);
     }
 
+    @Test
+    public void testFindContainsRemoveOnEmptyCollection() {
+        ImplicitLinkedHashCollection coll = new ImplicitLinkedHashCollection<>();
+        assertNull(coll.find(new TestElement(2)));
+        assertFalse(coll.contains(new TestElement(2)));
+        assertFalse(coll.remove(new TestElement(2)));
+    }
+
     private void addRandomElement(Random random, LinkedHashSet existing,
                                   ImplicitLinkedHashCollection set) {
         int next;
@@ -523,6 +532,7 @@ private void addRandomElement(Random random, LinkedHashSet existing,
         set.add(new TestElement(next));
     }
 
+    @SuppressWarnings("unlikely-arg-type")
     private void removeRandomElement(Random random, Collection existing,
                                      ImplicitLinkedHashCollection coll) {
         int removeIdx = random.nextInt(existing.size());
diff --git a/clients/src/test/java/org/apache/kafka/common/utils/ImplicitLinkedHashMultiCollectionTest.java b/clients/src/test/java/org/apache/kafka/common/utils/ImplicitLinkedHashMultiCollectionTest.java
index 8d2b850d98c37..ad87b55c493c4 100644
--- a/clients/src/test/java/org/apache/kafka/common/utils/ImplicitLinkedHashMultiCollectionTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/utils/ImplicitLinkedHashMultiCollectionTest.java
@@ -28,6 +28,7 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -44,6 +45,15 @@ public void testNullForbidden() {
         assertFalse(multiSet.add(null));
     }
 
+    @Test
+    public void testFindFindAllContainsRemoveOnEmptyCollection() {
+        ImplicitLinkedHashMultiCollection coll = new ImplicitLinkedHashMultiCollection<>();
+        assertNull(coll.find(new TestElement(2)));
+        assertFalse(coll.contains(new TestElement(2)));
+        assertFalse(coll.remove(new TestElement(2)));
+        assertTrue(coll.findAll(new TestElement(2)).isEmpty());
+    }
+
     @Test
     public void testInsertDelete() {
         ImplicitLinkedHashMultiCollection multiSet = new ImplicitLinkedHashMultiCollection<>(100);

From bd7d92e0c3dae4841f23f796f50384e6653111a2 Mon Sep 17 00:00:00 2001
From: Jeff Huang <47870461+jeffhuang26@users.noreply.github.com>
Date: Fri, 16 Aug 2019 00:43:24 -0700
Subject: [PATCH 0541/1071] KAFKA-8592; Fix for resolving variables for dynamic
 config as per KIP-421. (#7031)

Reviewers: Rajini Sivaram , Stanislav Kozlovski 
---
 .../kafka/common/config/AbstractConfig.java   |   2 +-
 .../kafka/server/DynamicBrokerConfig.scala    |  18 ++-
 .../scala/kafka/server/DynamicConfig.scala    |   3 +-
 .../DynamicBrokerReconfigurationTest.scala    | 142 ++++++++++++++----
 4 files changed, 132 insertions(+), 33 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java b/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java
index 832837da625b9..3992a4171e2b7 100644
--- a/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java
+++ b/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java
@@ -53,7 +53,7 @@ public class AbstractConfig {
 
     private final ConfigDef definition;
 
-    private static final String CONFIG_PROVIDERS_CONFIG = "config.providers";
+    public static final String CONFIG_PROVIDERS_CONFIG = "config.providers";
 
     private static final String CONFIG_PROVIDERS_PARAM = ".param.";
 
diff --git a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
index c384afb032ee7..37183be2d510a 100755
--- a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
+++ b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
@@ -28,7 +28,7 @@ import kafka.server.DynamicBrokerConfig._
 import kafka.utils.{CoreUtils, Logging, PasswordEncoder}
 import kafka.zk.{AdminZkClient, KafkaZkClient}
 import org.apache.kafka.common.Reconfigurable
-import org.apache.kafka.common.config.{ConfigDef, ConfigException, SslConfigs}
+import org.apache.kafka.common.config.{ConfigDef, ConfigException, SslConfigs, AbstractConfig}
 import org.apache.kafka.common.metrics.MetricsReporter
 import org.apache.kafka.common.config.types.Password
 import org.apache.kafka.common.network.{ListenerName, ListenerReconfigurable}
@@ -177,6 +177,15 @@ object DynamicBrokerConfig {
       (name -> mode)
     }.toMap.asJava
   }
+
+  private[server] def resolveVariableConfigs(propsOriginal: Properties): Properties = {
+    val props = new Properties
+    val config = new AbstractConfig(new ConfigDef(), propsOriginal, false)
+    config.originals.asScala.filter(!_._1.startsWith(AbstractConfig.CONFIG_PROVIDERS_CONFIG)).foreach {case (key: String, value: Object) => {
+      props.put(key, value)
+    }}
+    props
+  }
 }
 
 class DynamicBrokerConfig(private val kafkaConfig: KafkaConfig) extends Logging {
@@ -405,14 +414,15 @@ class DynamicBrokerConfig(private val kafkaConfig: KafkaConfig) extends Logging
    * Note: The caller must acquire the read or write lock before invoking this method.
    */
   private def validatedKafkaProps(propsOverride: Properties, perBrokerConfig: Boolean): Map[String, String] = {
-    validateConfigs(propsOverride, perBrokerConfig)
+    val propsResolved = DynamicBrokerConfig.resolveVariableConfigs(propsOverride)
+    validateConfigs(propsResolved, perBrokerConfig)
     val newProps = mutable.Map[String, String]()
     newProps ++= staticBrokerConfigs
     if (perBrokerConfig) {
       overrideProps(newProps, dynamicDefaultConfigs)
-      overrideProps(newProps, propsOverride.asScala)
+      overrideProps(newProps, propsResolved.asScala)
     } else {
-      overrideProps(newProps, propsOverride.asScala)
+      overrideProps(newProps, propsResolved.asScala)
       overrideProps(newProps, dynamicBrokerConfigs)
     }
     newProps
diff --git a/core/src/main/scala/kafka/server/DynamicConfig.scala b/core/src/main/scala/kafka/server/DynamicConfig.scala
index 1a401d2c326d9..e5974d3642dc8 100644
--- a/core/src/main/scala/kafka/server/DynamicConfig.scala
+++ b/core/src/main/scala/kafka/server/DynamicConfig.scala
@@ -115,7 +115,8 @@ object DynamicConfig {
       val unknownKeys = propKeys.filter(!names.contains(_))
       require(unknownKeys.isEmpty, s"Unknown Dynamic Configuration: $unknownKeys.")
     }
+    val propResolved = DynamicBrokerConfig.resolveVariableConfigs(props)
     //ValidateValues
-    configDef.parse(props)
+    configDef.parse(propResolved)
   }
 }
diff --git a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
index 8b720fabbfb76..3951c53bfae35 100644
--- a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
+++ b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
@@ -18,7 +18,7 @@
 
 package kafka.server
 
-import java.io.{Closeable, File, FileWriter}
+import java.io.{Closeable, File, FileWriter, IOException, Reader, StringReader}
 import java.nio.file.{Files, Paths, StandardCopyOption}
 import java.lang.management.ManagementFactory
 import java.security.KeyStore
@@ -49,6 +49,7 @@ import org.apache.kafka.common.{ClusterResource, ClusterResourceListener, Reconf
 import org.apache.kafka.common.config.{ConfigException, ConfigResource}
 import org.apache.kafka.common.config.SslConfigs._
 import org.apache.kafka.common.config.types.Password
+import org.apache.kafka.common.config.provider.FileConfigProvider
 import org.apache.kafka.common.errors.{AuthenticationException, InvalidRequestException}
 import org.apache.kafka.common.internals.Topic
 import org.apache.kafka.common.metrics.{KafkaMetric, MetricsReporter}
@@ -216,6 +217,76 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
     verifySslConfig("", invalidSslProperties, configDesc)
   }
 
+  @Test
+  def testUpdatesUsingConfigProvider(): Unit = {
+    val PollingIntervalVal = "${file:polling.interval:interval}"
+    val PollingIntervalUpdateVal = "${file:polling.interval:updinterval}"
+    val SslTruststoreTypeVal = "${file:ssl.truststore.type:storetype}"
+    val SslKeystorePasswordVal = "${file:ssl.keystore.password:password}"
+
+    val configPrefix = listenerPrefix(SecureExternal)
+    var brokerConfigs = describeConfig(adminClients.head, servers).entries.asScala
+    // the following are values before updated
+    assertTrue("Initial value of polling interval", brokerConfigs.find(_.name == TestMetricsReporter.PollingIntervalProp) == None)
+    assertTrue("Initial value of ssl truststore type", brokerConfigs.find(_.name == configPrefix+KafkaConfig.SslTruststoreTypeProp) == None)
+    assertTrue("Initial value of ssl keystore password", brokerConfigs.find(_.name == configPrefix+KafkaConfig.SslKeystorePasswordProp).get.value == null)
+
+    // setup ssl properties
+    val secProps = securityProps(sslProperties1, KEYSTORE_PROPS, configPrefix)
+
+    // configure config providers and properties need be updated
+    val updatedProps = new Properties
+    updatedProps.setProperty("config.providers", "file")
+    updatedProps.setProperty("config.providers.file.class", "kafka.server.MockFileConfigProvider")
+    updatedProps.put(KafkaConfig.MetricReporterClassesProp, classOf[TestMetricsReporter].getName)
+
+    // 1. update Integer property using config provider
+    updatedProps.put(TestMetricsReporter.PollingIntervalProp, PollingIntervalVal)
+
+    // 2. update String property using config provider
+    updatedProps.put(configPrefix+KafkaConfig.SslTruststoreTypeProp, SslTruststoreTypeVal)
+
+    // merge two properties
+    updatedProps ++= secProps
+
+    // 3. update password property using config provider
+    updatedProps.put(configPrefix+KafkaConfig.SslKeystorePasswordProp, SslKeystorePasswordVal)
+
+    alterConfigsUsingConfigCommand(updatedProps)
+    waitForConfig(TestMetricsReporter.PollingIntervalProp, "1000")
+    waitForConfig(configPrefix+KafkaConfig.SslTruststoreTypeProp, "JKS")
+    waitForConfig(configPrefix+KafkaConfig.SslKeystorePasswordProp, "ServerPassword")
+
+    // wait for MetricsReporter
+    val reporters = TestMetricsReporter.waitForReporters(servers.size)
+    reporters.foreach { reporter =>
+      reporter.verifyState(reconfigureCount = 0, deleteCount = 0, pollingInterval = 1000)
+      assertFalse("No metrics found", reporter.kafkaMetrics.isEmpty)
+    }
+
+    // fetch from ZK, values should be unresolved
+    val props = fetchBrokerConfigsFromZooKeeper(servers.head)
+    assertTrue("polling interval is not updated in ZK", props.getProperty(TestMetricsReporter.PollingIntervalProp) == PollingIntervalVal)
+    assertTrue("store type is not updated in ZK", props.getProperty(configPrefix+KafkaConfig.SslTruststoreTypeProp) == SslTruststoreTypeVal)
+    assertTrue("keystore password is not updated in ZK", props.getProperty(configPrefix+KafkaConfig.SslKeystorePasswordProp) == SslKeystorePasswordVal)
+
+    // verify the update
+    // 1. verify update not occurring if the value of property is same.
+    alterConfigsUsingConfigCommand(updatedProps)
+    waitForConfig(TestMetricsReporter.PollingIntervalProp, "1000")
+    reporters.foreach { reporter =>
+      reporter.verifyState(reconfigureCount = 0, deleteCount = 0, pollingInterval = 1000)
+    }
+
+    // 2. verify update occurring if the value of property changed.
+    updatedProps.put(TestMetricsReporter.PollingIntervalProp, PollingIntervalUpdateVal)
+    alterConfigsUsingConfigCommand(updatedProps)
+    waitForConfig(TestMetricsReporter.PollingIntervalProp, "2000")
+    reporters.foreach { reporter =>
+      reporter.verifyState(reconfigureCount = 1, deleteCount = 0, pollingInterval = 2000)
+    }
+  }
+
   @Test
   def testKeyStoreAlter(): Unit = {
     val topic2 = "testtopic2"
@@ -733,12 +804,12 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
     servers.foreach { server =>
       assertEquals(classOf[TestMetricsReporter].getName, server.config.originals.get(KafkaConfig.MetricReporterClassesProp))
     }
-    newReporters.foreach(_.verifyState(reconfigureCount = 1, deleteCount = 0, pollingInterval = 2000))
+    newReporters.foreach(_.verifyState(reconfigureCount = 0, deleteCount = 0, pollingInterval = 2000))
 
     // Verify that validation failure of custom config fails reconfiguration and leaves config unchanged
     newProps.put(TestMetricsReporter.PollingIntervalProp, "invalid")
     reconfigureServers(newProps, perBrokerConfig = false, (TestMetricsReporter.PollingIntervalProp, "2000"), expectFailure = true)
-    newReporters.foreach(_.verifyState(reconfigureCount = 1, deleteCount = 0, pollingInterval = 2000))
+    newReporters.foreach(_.verifyState(reconfigureCount = 0, deleteCount = 0, pollingInterval = 2000))
 
     // Delete reporters
     configureMetricsReporters(Seq.empty[Class[_]], newProps)
@@ -751,7 +822,13 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
     alterConfigsOnServer(servers.head, newProps)
     TestUtils.waitUntilTrue(() => !TestMetricsReporter.testReporters.isEmpty, "Metrics reporter not created")
     val perBrokerReporter = TestMetricsReporter.waitForReporters(1).head
-    perBrokerReporter.verifyState(reconfigureCount = 1, deleteCount = 0, pollingInterval = 4000)
+    perBrokerReporter.verifyState(reconfigureCount = 0, deleteCount = 0, pollingInterval = 4000)
+
+    // update TestMetricsReporter.PollingIntervalProp to 3000
+    newProps.put(TestMetricsReporter.PollingIntervalProp, "3000")
+    alterConfigsOnServer(servers.head, newProps)
+    perBrokerReporter.verifyState(reconfigureCount = 1, deleteCount = 0, pollingInterval = 3000)
+
     servers.tail.foreach { server => assertEquals("", server.config.originals.get(KafkaConfig.MetricReporterClassesProp)) }
 
     // Verify that produce/consume worked throughout this test without any retries in producer
@@ -1167,25 +1244,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
   private def alterSslKeystoreUsingConfigCommand(props: Properties, listener: String): Unit = {
     val configPrefix = listenerPrefix(listener)
     val newProps = securityProps(props, KEYSTORE_PROPS, configPrefix)
-
-    val propsFile = TestUtils.tempFile()
-    val propsWriter = new FileWriter(propsFile)
-    try {
-      clientProps(SecurityProtocol.SSL).asScala.foreach {
-        case (k, v) => propsWriter.write(s"$k=$v\n")
-      }
-    } finally {
-      propsWriter.close()
-    }
-
-    servers.foreach { server =>
-      val args = Array("--bootstrap-server", TestUtils.bootstrapServers(servers, new ListenerName(SecureInternal)),
-        "--command-config", propsFile.getAbsolutePath,
-        "--alter", "--add-config", newProps.asScala.map { case (k, v) => s"$k=$v" }.mkString(","),
-        "--entity-type", "brokers",
-        "--entity-name", server.config.brokerId.toString)
-      ConfigCommand.main(args)
-    }
+    alterConfigsUsingConfigCommand(newProps)
     waitForConfig(s"$configPrefix$SSL_KEYSTORE_LOCATION_CONFIG", props.getProperty(SSL_KEYSTORE_LOCATION_CONFIG))
   }
 
@@ -1304,7 +1363,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
   }
 
   private def configureMetricsReporters(reporters: Seq[Class[_]], props: Properties,
-                                       perBrokerConfig: Boolean = false): Unit = {
+                                        perBrokerConfig: Boolean = false): Unit = {
     val reporterStr = reporters.map(_.getName).mkString(",")
     props.put(KafkaConfig.MetricReporterClassesProp, reporterStr)
     reconfigureServers(props, perBrokerConfig, (KafkaConfig.MetricReporterClassesProp, reporterStr))
@@ -1417,6 +1476,27 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
     }
   }
 
+  private def alterConfigsUsingConfigCommand(props: Properties): Unit = {
+    val propsFile = TestUtils.tempFile()
+    val propsWriter = new FileWriter(propsFile)
+    try {
+      clientProps(SecurityProtocol.SSL).asScala.foreach {
+        case (k, v) => propsWriter.write(s"$k=$v\n")
+      }
+    } finally {
+      propsWriter.close()
+    }
+
+    servers.foreach { server =>
+      val args = Array("--bootstrap-server", TestUtils.bootstrapServers(servers, new ListenerName(SecureInternal)),
+        "--command-config", propsFile.getAbsolutePath,
+        "--alter", "--add-config", props.asScala.map { case (k, v) => s"$k=$v" }.mkString(","),
+        "--entity-type", "brokers",
+        "--entity-name", server.config.brokerId.toString)
+      ConfigCommand.main(args)
+    }
+  }
+
   private abstract class ClientBuilder[T]() {
     protected var _bootstrapServers: Option[String] = None
     protected var _listenerName = SecureExternal
@@ -1629,8 +1709,8 @@ class TestMetricsReporter extends MetricsReporter with Reconfigurable with Close
   }
 
   override def validateReconfiguration(configs: util.Map[String, _]): Unit = {
-    val pollingInterval = configs.get(PollingIntervalProp).toString
-    if (configs.get(PollingIntervalProp).toString.toInt <= 0)
+    val pollingInterval = configs.get(PollingIntervalProp).toString.toInt
+    if (pollingInterval <= 0)
       throw new ConfigException(s"Invalid polling interval $pollingInterval")
   }
 
@@ -1646,7 +1726,7 @@ class TestMetricsReporter extends MetricsReporter with Reconfigurable with Close
   def verifyState(reconfigureCount: Int, deleteCount: Int, pollingInterval: Int): Unit = {
     assertEquals(1, initializeCount)
     assertEquals(1, configureCount)
-    assertEquals(reconfigureCount, reconfigureCount)
+    assertEquals(reconfigureCount, this.reconfigureCount)
     assertEquals(deleteCount, closeCount)
     assertEquals(1, clusterUpdateCount)
     assertEquals(pollingInterval, this.pollingInterval)
@@ -1659,3 +1739,11 @@ class TestMetricsReporter extends MetricsReporter with Reconfigurable with Close
     assertTrue("Invalid metric value", total > 0.0)
   }
 }
+
+
+class MockFileConfigProvider extends FileConfigProvider {
+  @throws(classOf[IOException])
+  override def reader(path: String): Reader = {
+    new StringReader("key=testKey\npassword=ServerPassword\ninterval=1000\nupdinterval=2000\nstoretype=JKS");
+  }
+}

From b8605c9bc1331b876f9d134ebc8fcf9bfc684c8e Mon Sep 17 00:00:00 2001
From: Viktor Somogyi-Vass 
Date: Fri, 16 Aug 2019 23:05:54 +0530
Subject: [PATCH 0542/1071] KAFKA-8600: Use RPC generation for
 DescribeDelegationTokens protocol

Refactors the DescribeDelegationToken to use the generated RPC classes.

Author: Viktor Somogyi-Vass 
Author: Viktor Somogyi 

Reviewers: Mickael Maison , Manikumar Reddy 

Closes #7154 from viktorsomogyi/refactor-describe-dt
---
 .../apache/kafka/common/protocol/ApiKeys.java |   6 +-
 .../common/requests/AbstractResponse.java     |   2 +-
 .../DescribeDelegationTokenRequest.java       | 103 +++--------
 .../DescribeDelegationTokenResponse.java      | 175 +++++-------------
 .../main/scala/kafka/server/KafkaApis.scala   |   5 +-
 .../unit/kafka/server/RequestQuotaTest.scala  |  32 +---
 6 files changed, 82 insertions(+), 241 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
index 912e26bc99b3b..3b45e5882f4e0 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
@@ -26,6 +26,8 @@
 import org.apache.kafka.common.message.DeleteGroupsResponseData;
 import org.apache.kafka.common.message.DeleteTopicsRequestData;
 import org.apache.kafka.common.message.DeleteTopicsResponseData;
+import org.apache.kafka.common.message.DescribeDelegationTokenRequestData;
+import org.apache.kafka.common.message.DescribeDelegationTokenResponseData;
 import org.apache.kafka.common.message.DescribeGroupsRequestData;
 import org.apache.kafka.common.message.DescribeGroupsResponseData;
 import org.apache.kafka.common.message.ElectLeadersRequestData;
@@ -91,8 +93,6 @@
 import org.apache.kafka.common.requests.DescribeAclsResponse;
 import org.apache.kafka.common.requests.DescribeConfigsRequest;
 import org.apache.kafka.common.requests.DescribeConfigsResponse;
-import org.apache.kafka.common.requests.DescribeDelegationTokenRequest;
-import org.apache.kafka.common.requests.DescribeDelegationTokenResponse;
 import org.apache.kafka.common.requests.DescribeLogDirsRequest;
 import org.apache.kafka.common.requests.DescribeLogDirsResponse;
 import org.apache.kafka.common.requests.EndTxnRequest;
@@ -192,7 +192,7 @@ public Struct parseResponse(short version, ByteBuffer buffer) {
     CREATE_DELEGATION_TOKEN(38, "CreateDelegationToken", CreateDelegationTokenRequestData.SCHEMAS, CreateDelegationTokenResponseData.SCHEMAS),
     RENEW_DELEGATION_TOKEN(39, "RenewDelegationToken", RenewDelegationTokenRequestData.SCHEMAS, RenewDelegationTokenResponseData.SCHEMAS),
     EXPIRE_DELEGATION_TOKEN(40, "ExpireDelegationToken", ExpireDelegationTokenRequestData.SCHEMAS, ExpireDelegationTokenResponseData.SCHEMAS),
-    DESCRIBE_DELEGATION_TOKEN(41, "DescribeDelegationToken", DescribeDelegationTokenRequest.schemaVersions(), DescribeDelegationTokenResponse.schemaVersions()),
+    DESCRIBE_DELEGATION_TOKEN(41, "DescribeDelegationToken", DescribeDelegationTokenRequestData.SCHEMAS, DescribeDelegationTokenResponseData.SCHEMAS),
     DELETE_GROUPS(42, "DeleteGroups", DeleteGroupsRequestData.SCHEMAS, DeleteGroupsResponseData.SCHEMAS),
     ELECT_LEADERS(43, "ElectLeaders", ElectLeadersRequestData.SCHEMAS,
             ElectLeadersResponseData.SCHEMAS),
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
index 13d18a2d72401..b84a2c03d1eaa 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
@@ -153,7 +153,7 @@ public static AbstractResponse parseResponse(ApiKeys apiKey, Struct struct, shor
             case EXPIRE_DELEGATION_TOKEN:
                 return new ExpireDelegationTokenResponse(struct, version);
             case DESCRIBE_DELEGATION_TOKEN:
-                return new DescribeDelegationTokenResponse(struct);
+                return new DescribeDelegationTokenResponse(struct, version);
             case DELETE_GROUPS:
                 return new DeleteGroupsResponse(struct, version);
             case ELECT_LEADERS:
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java
index 21e14608c3691..7db3209d621fb 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java
@@ -16,122 +16,69 @@
  */
 package org.apache.kafka.common.requests;
 
+import org.apache.kafka.common.message.DescribeDelegationTokenRequestData;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.ArrayOf;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
 import org.apache.kafka.common.security.auth.KafkaPrincipal;
 
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
 import java.util.List;
-
-import static org.apache.kafka.common.protocol.CommonFields.PRINCIPAL_NAME;
-import static org.apache.kafka.common.protocol.CommonFields.PRINCIPAL_TYPE;
+import java.util.stream.Collectors;
 
 public class DescribeDelegationTokenRequest extends AbstractRequest {
-    private static final String OWNER_KEY_NAME = "owners";
-
-    private final List owners;
-
-    public static final Schema TOKEN_DESCRIBE_REQUEST_V0 = new Schema(
-        new Field(OWNER_KEY_NAME, ArrayOf.nullable(new Schema(PRINCIPAL_TYPE, PRINCIPAL_NAME)), "An array of token owners."));
 
-    /**
-     * The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
-     */
-    public static final Schema TOKEN_DESCRIBE_REQUEST_V1 = TOKEN_DESCRIBE_REQUEST_V0;
+    private final DescribeDelegationTokenRequestData data;
 
     public static class Builder extends AbstractRequest.Builder {
-        // describe token for the given list of owners, or null if we want to describe all tokens.
-        private final List owners;
+        private final DescribeDelegationTokenRequestData data;
 
         public Builder(List owners) {
             super(ApiKeys.DESCRIBE_DELEGATION_TOKEN);
-            this.owners = owners;
+            this.data = new DescribeDelegationTokenRequestData()
+                .setOwners(owners == null ? null : owners
+                    .stream()
+                    .map(owner -> new DescribeDelegationTokenRequestData.DescribeDelegationTokenOwner()
+                            .setPrincipalName(owner.getName())
+                            .setPrincipalType(owner.getPrincipalType()))
+                    .collect(Collectors.toList()));
         }
 
         @Override
         public DescribeDelegationTokenRequest build(short version) {
-            return new DescribeDelegationTokenRequest(version, owners);
+            return new DescribeDelegationTokenRequest(data, version);
         }
 
         @Override
         public String toString() {
-            StringBuilder bld = new StringBuilder();
-            bld.append("(type: DescribeDelegationTokenRequest").
-                append(", owners=").append(owners).
-                append(")");
-            return bld.toString();
+            return data.toString();
         }
     }
 
-    private DescribeDelegationTokenRequest(short version, List owners) {
+    public DescribeDelegationTokenRequest(Struct struct, short version) {
         super(ApiKeys.DESCRIBE_DELEGATION_TOKEN, version);
-        this.owners = owners;
+        this.data = new DescribeDelegationTokenRequestData(struct, version);
     }
 
-    public DescribeDelegationTokenRequest(Struct struct, short versionId) {
-        super(ApiKeys.DESCRIBE_DELEGATION_TOKEN, versionId);
-
-        Object[] ownerArray = struct.getArray(OWNER_KEY_NAME);
-
-        if (ownerArray != null) {
-            owners = new ArrayList<>();
-            for (Object ownerObj : ownerArray) {
-                Struct ownerObjStruct = (Struct) ownerObj;
-                String principalType = ownerObjStruct.get(PRINCIPAL_TYPE);
-                String principalName = ownerObjStruct.get(PRINCIPAL_NAME);
-                owners.add(new KafkaPrincipal(principalType, principalName));
-            }
-        } else
-            owners = null;
-    }
-
-    public static Schema[] schemaVersions() {
-        return new Schema[]{TOKEN_DESCRIBE_REQUEST_V0, TOKEN_DESCRIBE_REQUEST_V1};
+    public DescribeDelegationTokenRequest(DescribeDelegationTokenRequestData data, short version) {
+        super(ApiKeys.DESCRIBE_DELEGATION_TOKEN, version);
+        this.data = data;
     }
 
     @Override
     protected Struct toStruct() {
-        short version = version();
-        Struct struct = new Struct(ApiKeys.DESCRIBE_DELEGATION_TOKEN.requestSchema(version));
-
-        if (owners == null) {
-            struct.set(OWNER_KEY_NAME, null);
-        } else {
-            Object[] ownersArray = new Object[owners.size()];
-
-            int i = 0;
-            for (KafkaPrincipal principal: owners) {
-                Struct ownerStruct = struct.instance(OWNER_KEY_NAME);
-                ownerStruct.set(PRINCIPAL_TYPE, principal.getPrincipalType());
-                ownerStruct.set(PRINCIPAL_NAME, principal.getName());
-                ownersArray[i++] = ownerStruct;
-            }
-
-            struct.set(OWNER_KEY_NAME, ownersArray);
-        }
-
-        return struct;
-    }
-
-    @Override
-    public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
-        return new DescribeDelegationTokenResponse(throttleTimeMs, Errors.forException(e));
+        return data.toStruct(version());
     }
 
-    public List owners() {
-        return owners;
+    public DescribeDelegationTokenRequestData data() {
+        return data;
     }
 
     public boolean ownersListEmpty() {
-        return owners != null && owners.isEmpty();
+        return data.owners() != null && data.owners().isEmpty();
     }
 
-    public static DescribeDelegationTokenRequest parse(ByteBuffer buffer, short version) {
-        return new DescribeDelegationTokenRequest(ApiKeys.DESCRIBE_DELEGATION_TOKEN.parseRequest(version, buffer), version);
+    @Override
+    public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
+        return new DescribeDelegationTokenResponse(throttleTimeMs, Errors.forException(e));
     }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java
index 64ed27bfd0003..91e7db0fccb1e 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java
@@ -16,11 +16,11 @@
  */
 package org.apache.kafka.common.requests;
 
+import org.apache.kafka.common.message.DescribeDelegationTokenResponseData;
+import org.apache.kafka.common.message.DescribeDelegationTokenResponseData.DescribedDelegationToken;
+import org.apache.kafka.common.message.DescribeDelegationTokenResponseData.DescribedDelegationTokenRenewer;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.ArrayOf;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
 import org.apache.kafka.common.security.auth.KafkaPrincipal;
 import org.apache.kafka.common.security.token.delegation.DelegationToken;
@@ -30,169 +30,82 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-
-import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
-import static org.apache.kafka.common.protocol.CommonFields.PRINCIPAL_NAME;
-import static org.apache.kafka.common.protocol.CommonFields.PRINCIPAL_TYPE;
-import static org.apache.kafka.common.protocol.CommonFields.THROTTLE_TIME_MS;
-import static org.apache.kafka.common.protocol.types.Type.BYTES;
-import static org.apache.kafka.common.protocol.types.Type.INT64;
-import static org.apache.kafka.common.protocol.types.Type.STRING;
+import java.util.stream.Collectors;
 
 public class DescribeDelegationTokenResponse extends AbstractResponse {
 
-    private static final String TOKEN_DETAILS_KEY_NAME = "token_details";
-    private static final String ISSUE_TIMESTAMP_KEY_NAME = "issue_timestamp";
-    private static final String EXPIRY_TIMESTAMP_NAME = "expiry_timestamp";
-    private static final String MAX_TIMESTAMP_NAME = "max_timestamp";
-    private static final String TOKEN_ID_KEY_NAME = "token_id";
-    private static final String HMAC_KEY_NAME = "hmac";
-    private static final String OWNER_KEY_NAME = "owner";
-    private static final String RENEWERS_KEY_NAME = "renewers";
-
-    private final Errors error;
-    private final List tokens;
-    private final int throttleTimeMs;
-
-    private static final Schema TOKEN_DETAILS_V0 = new Schema(
-        new Field(OWNER_KEY_NAME, new Schema(PRINCIPAL_TYPE, PRINCIPAL_NAME), "token owner."),
-        new Field(ISSUE_TIMESTAMP_KEY_NAME, INT64, "timestamp (in msec) when this token was generated."),
-        new Field(EXPIRY_TIMESTAMP_NAME, INT64, "timestamp (in msec) at which this token expires."),
-        new Field(MAX_TIMESTAMP_NAME, INT64, "max life time of this token."),
-        new Field(TOKEN_ID_KEY_NAME, STRING, "UUID to ensure uniqueness."),
-        new Field(HMAC_KEY_NAME, BYTES, "HMAC of the delegation token to be expired."),
-        new Field(RENEWERS_KEY_NAME, new ArrayOf(new Schema(PRINCIPAL_TYPE, PRINCIPAL_NAME)),
-            "An array of token renewers. Renewer is an Kafka PrincipalType and name string," +
-                " who is allowed to renew this token before the max lifetime expires."));
-
-    private static final Schema TOKEN_DESCRIBE_RESPONSE_V0 = new Schema(
-        ERROR_CODE,
-        new Field(TOKEN_DETAILS_KEY_NAME, new ArrayOf(TOKEN_DETAILS_V0)),
-        THROTTLE_TIME_MS);
-
-    /**
-     * The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
-     */
-    private static final Schema TOKEN_DESCRIBE_RESPONSE_V1 = TOKEN_DESCRIBE_RESPONSE_V0;
+    private final DescribeDelegationTokenResponseData data;
 
     public DescribeDelegationTokenResponse(int throttleTimeMs, Errors error, List tokens) {
-        this.throttleTimeMs = throttleTimeMs;
-        this.error = error;
-        this.tokens = tokens;
+        List describedDelegationTokenList = tokens
+            .stream()
+            .map(dt -> new DescribedDelegationToken()
+                .setTokenId(dt.tokenInfo().tokenId())
+                .setPrincipalType(dt.tokenInfo().owner().getPrincipalType())
+                .setPrincipalName(dt.tokenInfo().owner().getName())
+                .setIssueTimestamp(dt.tokenInfo().issueTimestamp())
+                .setMaxTimestamp(dt.tokenInfo().maxTimestamp())
+                .setExpiryTimestamp(dt.tokenInfo().expiryTimestamp())
+                .setHmac(dt.hmac())
+                .setRenewers(dt.tokenInfo().renewers()
+                    .stream()
+                    .map(r -> new DescribedDelegationTokenRenewer().setPrincipalName(r.getName()).setPrincipalType(r.getPrincipalType()))
+                    .collect(Collectors.toList())))
+            .collect(Collectors.toList());
+
+        this.data = new DescribeDelegationTokenResponseData()
+            .setThrottleTimeMs(throttleTimeMs)
+            .setErrorCode(error.code())
+            .setTokens(describedDelegationTokenList);
     }
 
     public DescribeDelegationTokenResponse(int throttleTimeMs, Errors error) {
         this(throttleTimeMs, error, new ArrayList<>());
     }
 
-    public DescribeDelegationTokenResponse(Struct struct) {
-        Object[] requestStructs = struct.getArray(TOKEN_DETAILS_KEY_NAME);
-        List tokens = new ArrayList<>();
-
-        for (Object requestStructObj : requestStructs) {
-            Struct singleRequestStruct = (Struct) requestStructObj;
-
-            Struct ownerStruct = (Struct) singleRequestStruct.get(OWNER_KEY_NAME);
-            KafkaPrincipal owner = new KafkaPrincipal(ownerStruct.get(PRINCIPAL_TYPE), ownerStruct.get(PRINCIPAL_NAME));
-            long issueTimestamp = singleRequestStruct.getLong(ISSUE_TIMESTAMP_KEY_NAME);
-            long expiryTimestamp = singleRequestStruct.getLong(EXPIRY_TIMESTAMP_NAME);
-            long maxTimestamp = singleRequestStruct.getLong(MAX_TIMESTAMP_NAME);
-            String tokenId = singleRequestStruct.getString(TOKEN_ID_KEY_NAME);
-            ByteBuffer hmac = singleRequestStruct.getBytes(HMAC_KEY_NAME);
-
-            Object[] renewerArray = singleRequestStruct.getArray(RENEWERS_KEY_NAME);
-            List  renewers = new ArrayList<>();
-            if (renewerArray != null) {
-                for (Object renewerObj : renewerArray) {
-                    Struct renewerObjStruct = (Struct) renewerObj;
-                    String principalType = renewerObjStruct.get(PRINCIPAL_TYPE);
-                    String principalName = renewerObjStruct.get(PRINCIPAL_NAME);
-                    renewers.add(new KafkaPrincipal(principalType, principalName));
-                }
-            }
-
-            TokenInformation tokenInfo = new TokenInformation(tokenId, owner, renewers, issueTimestamp, maxTimestamp, expiryTimestamp);
-
-            byte[] hmacBytes = new byte[hmac.remaining()];
-            hmac.get(hmacBytes);
-
-            DelegationToken tokenDetails = new DelegationToken(tokenInfo, hmacBytes);
-            tokens.add(tokenDetails);
-        }
-
-        this.tokens = tokens;
-        error = Errors.forCode(struct.get(ERROR_CODE));
-        this.throttleTimeMs = struct.getOrElse(THROTTLE_TIME_MS, DEFAULT_THROTTLE_TIME);
+    public DescribeDelegationTokenResponse(Struct struct, short version) {
+        this.data = new DescribeDelegationTokenResponseData(struct, version);
     }
 
     public static DescribeDelegationTokenResponse parse(ByteBuffer buffer, short version) {
-        return new DescribeDelegationTokenResponse(ApiKeys.DESCRIBE_DELEGATION_TOKEN.responseSchema(version).read(buffer));
+        return new DescribeDelegationTokenResponse(ApiKeys.DESCRIBE_DELEGATION_TOKEN.responseSchema(version).read(buffer), version);
     }
 
     @Override
     public Map errorCounts() {
-        return errorCounts(error);
+        return errorCounts(error());
     }
 
     @Override
     protected Struct toStruct(short version) {
-        Struct struct = new Struct(ApiKeys.DESCRIBE_DELEGATION_TOKEN.responseSchema(version));
-        List tokenDetailsStructs = new ArrayList<>(tokens.size());
-
-        struct.set(ERROR_CODE, error.code());
-
-        for (DelegationToken token : tokens) {
-            TokenInformation tokenInfo = token.tokenInfo();
-            Struct singleRequestStruct = struct.instance(TOKEN_DETAILS_KEY_NAME);
-            Struct ownerStruct = singleRequestStruct.instance(OWNER_KEY_NAME);
-            ownerStruct.set(PRINCIPAL_TYPE, tokenInfo.owner().getPrincipalType());
-            ownerStruct.set(PRINCIPAL_NAME, tokenInfo.owner().getName());
-            singleRequestStruct.set(OWNER_KEY_NAME, ownerStruct);
-            singleRequestStruct.set(ISSUE_TIMESTAMP_KEY_NAME, tokenInfo.issueTimestamp());
-            singleRequestStruct.set(EXPIRY_TIMESTAMP_NAME, tokenInfo.expiryTimestamp());
-            singleRequestStruct.set(MAX_TIMESTAMP_NAME, tokenInfo.maxTimestamp());
-            singleRequestStruct.set(TOKEN_ID_KEY_NAME, tokenInfo.tokenId());
-            singleRequestStruct.set(HMAC_KEY_NAME, ByteBuffer.wrap(token.hmac()));
-
-            Object[] renewersArray = new Object[tokenInfo.renewers().size()];
-
-            int i = 0;
-            for (KafkaPrincipal principal: tokenInfo.renewers()) {
-                Struct renewerStruct = singleRequestStruct.instance(RENEWERS_KEY_NAME);
-                renewerStruct.set(PRINCIPAL_TYPE, principal.getPrincipalType());
-                renewerStruct.set(PRINCIPAL_NAME, principal.getName());
-                renewersArray[i++] = renewerStruct;
-            }
-
-            singleRequestStruct.set(RENEWERS_KEY_NAME, renewersArray);
-
-            tokenDetailsStructs.add(singleRequestStruct);
-        }
-        struct.set(TOKEN_DETAILS_KEY_NAME, tokenDetailsStructs.toArray());
-        struct.setIfExists(THROTTLE_TIME_MS, throttleTimeMs);
-
-        return struct;
-    }
-
-    public static Schema[] schemaVersions() {
-        return new Schema[]{TOKEN_DESCRIBE_RESPONSE_V0, TOKEN_DESCRIBE_RESPONSE_V1};
+        return data.toStruct(version);
     }
 
     @Override
     public int throttleTimeMs() {
-        return throttleTimeMs;
+        return data.throttleTimeMs();
     }
 
     public Errors error() {
-        return error;
+        return Errors.forCode(data.errorCode());
     }
 
     public List tokens() {
-        return tokens;
+        return data.tokens()
+            .stream()
+            .map(ddt -> new DelegationToken(new TokenInformation(
+                ddt.tokenId(),
+                new KafkaPrincipal(ddt.principalType(), ddt.principalName()),
+                ddt.renewers()
+                    .stream()
+                    .map(ddtr -> new KafkaPrincipal(ddtr.principalType(), ddtr.principalName()))
+                    .collect(Collectors.toList()), ddt.issueTimestamp(), ddt.maxTimestamp(), ddt.expiryTimestamp()),
+                ddt.hmac()))
+            .collect(Collectors.toList());
     }
 
     public boolean hasError() {
-        return this.error != Errors.NONE;
+        return error() != Errors.NONE;
     }
 
     @Override
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index b71e4b0876c1b..4df57a95aa404 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -2531,7 +2531,10 @@ class KafkaApis(val requestChannel: RequestChannel,
         sendResponseCallback(Errors.NONE, List())
       }
       else {
-        val owners = if (describeTokenRequest.owners == null) None else Some(describeTokenRequest.owners.asScala.toList)
+        val owners = if (describeTokenRequest.data.owners == null)
+          None
+        else
+          Some(describeTokenRequest.data.owners.asScala.map(p => new KafkaPrincipal(p.principalType(), p.principalName())).toList)
         def authorizeToken(tokenId: String) = authorize(request.session, Describe, Resource(kafka.security.auth.DelegationToken, tokenId, LITERAL))
         def eligible(token: TokenInformation) = DelegationTokenManager.filterToken(requestPrincipal, owners, token, authorizeToken)
         val tokens =  tokenManager.getTokens(eligible)
diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
index d5c5d7faebb2d..d2ad89748f5e6 100644
--- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
+++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
@@ -21,34 +21,13 @@ import kafka.log.LogConfig
 import kafka.network.RequestChannel.Session
 import kafka.security.auth._
 import kafka.utils.TestUtils
-import org.apache.kafka.common.ElectionType
-import org.apache.kafka.common.Node
-import org.apache.kafka.common.TopicPartition
-import org.apache.kafka.common.acl.{AccessControlEntry, AccessControlEntryFilter, AclBinding, AclBindingFilter, AclOperation, AclPermissionType}
+import org.apache.kafka.common.{ElectionType, Node, TopicPartition}
+import org.apache.kafka.common.acl._
 import org.apache.kafka.common.config.ConfigResource
-import org.apache.kafka.common.message.ControlledShutdownRequestData
-import org.apache.kafka.common.message.CreateDelegationTokenRequestData
-import org.apache.kafka.common.message.CreateTopicsRequestData
 import org.apache.kafka.common.message.CreateTopicsRequestData.{CreatableTopic, CreatableTopicCollection}
-import org.apache.kafka.common.message.DeleteGroupsRequestData
-import org.apache.kafka.common.message.DeleteTopicsRequestData
-import org.apache.kafka.common.message.DescribeGroupsRequestData
-import org.apache.kafka.common.message.ExpireDelegationTokenRequestData
-import org.apache.kafka.common.message.FindCoordinatorRequestData
-import org.apache.kafka.common.message.HeartbeatRequestData
-import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData
-import org.apache.kafka.common.message.InitProducerIdRequestData
-import org.apache.kafka.common.message.JoinGroupRequestData
 import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection
 import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
-import org.apache.kafka.common.message.ListGroupsRequestData
-import org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData
-import org.apache.kafka.common.message.ListPartitionReassignmentsRequestData
-import org.apache.kafka.common.message.OffsetCommitRequestData
-import org.apache.kafka.common.message.RenewDelegationTokenRequestData
-import org.apache.kafka.common.message.SaslAuthenticateRequestData
-import org.apache.kafka.common.message.SaslHandshakeRequestData
-import org.apache.kafka.common.message.SyncGroupRequestData
+import org.apache.kafka.common.message._
 import org.apache.kafka.common.metrics.{KafkaMetric, Quota, Sensor}
 import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.protocol.ApiKeys
@@ -58,8 +37,7 @@ import org.apache.kafka.common.requests.CreateAclsRequest.AclCreation
 import org.apache.kafka.common.requests._
 import org.apache.kafka.common.resource.{PatternType, ResourcePattern, ResourcePatternFilter, ResourceType => AdminResourceType}
 import org.apache.kafka.common.security.auth.{AuthenticationContext, KafkaPrincipal, KafkaPrincipalBuilder, SecurityProtocol}
-import org.apache.kafka.common.utils.Sanitizer
-import org.apache.kafka.common.utils.SecurityUtils
+import org.apache.kafka.common.utils.{Sanitizer, SecurityUtils}
 import org.junit.Assert._
 import org.junit.{After, Before, Test}
 
@@ -580,7 +558,7 @@ class RequestQuotaTest extends BaseRequestTest {
       case ApiKeys.DESCRIBE_LOG_DIRS => new DescribeLogDirsResponse(response).throttleTimeMs
       case ApiKeys.CREATE_PARTITIONS => new CreatePartitionsResponse(response).throttleTimeMs
       case ApiKeys.CREATE_DELEGATION_TOKEN => new CreateDelegationTokenResponse(response, ApiKeys.CREATE_DELEGATION_TOKEN.latestVersion).throttleTimeMs
-      case ApiKeys.DESCRIBE_DELEGATION_TOKEN=> new DescribeDelegationTokenResponse(response).throttleTimeMs
+      case ApiKeys.DESCRIBE_DELEGATION_TOKEN=> new DescribeDelegationTokenResponse(response, ApiKeys.DESCRIBE_DELEGATION_TOKEN.latestVersion).throttleTimeMs
       case ApiKeys.RENEW_DELEGATION_TOKEN => new RenewDelegationTokenResponse(response, ApiKeys.RENEW_DELEGATION_TOKEN.latestVersion).throttleTimeMs
       case ApiKeys.EXPIRE_DELEGATION_TOKEN => new ExpireDelegationTokenResponse(response, ApiKeys.EXPIRE_DELEGATION_TOKEN.latestVersion).throttleTimeMs
       case ApiKeys.DELETE_GROUPS => new DeleteGroupsResponse(response).throttleTimeMs

From e2d16b504c717038ba6030b40661df80b92353fe Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Fri, 16 Aug 2019 10:41:30 -0700
Subject: [PATCH 0543/1071] KAFKA-8802: ConcurrentSkipListMap shows performance
 regression in cache and in-memory store (#7212)

Reverts the TreeMap -> ConcurrentSkipListMap change that caused a performance regression in 2.3, and fixes the ConcurrentModificationException by copying (just) the key set to iterate over

Reviewers: Bill Bejeck , Guozhang Wang 
---
 .../internals/InMemoryKeyValueStore.java      | 37 ++++++++++---------
 .../streams/state/internals/NamedCache.java   | 17 ++++++---
 .../streams/state/internals/ThreadCache.java  | 24 ++++++------
 3 files changed, 42 insertions(+), 36 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStore.java
index 9ec5a98f7b76a..27ec409dc4d31 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStore.java
@@ -17,8 +17,10 @@
 package org.apache.kafka.streams.state.internals;
 
 import java.util.List;
-import java.util.concurrent.ConcurrentNavigableMap;
-import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.NavigableMap;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.processor.ProcessorContext;
@@ -27,13 +29,12 @@
 import org.apache.kafka.streams.state.KeyValueStore;
 
 import java.util.Iterator;
-import java.util.Map;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class InMemoryKeyValueStore implements KeyValueStore {
     private final String name;
-    private final ConcurrentNavigableMap map = new ConcurrentSkipListMap<>();
+    private final NavigableMap map = new TreeMap<>();
     private volatile boolean open = false;
     private long size = 0L; // SkipListMap#size is O(N) so we just do our best to track it
 
@@ -71,12 +72,12 @@ public boolean isOpen() {
     }
 
     @Override
-    public byte[] get(final Bytes key) {
+    public synchronized byte[] get(final Bytes key) {
         return map.get(key);
     }
 
     @Override
-    public void put(final Bytes key, final byte[] value) {
+    public synchronized void put(final Bytes key, final byte[] value) {
         if (value == null) {
             size -= map.remove(key) == null ? 0 : 1;
         } else {
@@ -85,7 +86,7 @@ public void put(final Bytes key, final byte[] value) {
     }
 
     @Override
-    public byte[] putIfAbsent(final Bytes key, final byte[] value) {
+    public synchronized byte[] putIfAbsent(final Bytes key, final byte[] value) {
         final byte[] originalValue = get(key);
         if (originalValue == null) {
             put(key, value);
@@ -101,14 +102,14 @@ public void putAll(final List> entries) {
     }
 
     @Override
-    public byte[] delete(final Bytes key) {
+    public synchronized byte[] delete(final Bytes key) {
         final byte[] oldValue = map.remove(key);
         size -= oldValue == null ? 0 : 1;
         return oldValue;
     }
 
     @Override
-    public KeyValueIterator range(final Bytes from, final Bytes to) {
+    public synchronized KeyValueIterator range(final Bytes from, final Bytes to) {
 
         if (from.compareTo(to) > 0) {
             LOG.warn("Returning empty iterator for fetch with invalid key range: from > to. "
@@ -119,14 +120,14 @@ public KeyValueIterator range(final Bytes from, final Bytes to) {
 
         return new DelegatingPeekingKeyValueIterator<>(
             name,
-            new InMemoryKeyValueIterator(map.subMap(from, true, to, true).entrySet().iterator()));
+            new InMemoryKeyValueIterator(map.subMap(from, true, to, true).keySet()));
     }
 
     @Override
-    public KeyValueIterator all() {
+    public synchronized KeyValueIterator all() {
         return new DelegatingPeekingKeyValueIterator<>(
             name,
-            new InMemoryKeyValueIterator(map.entrySet().iterator()));
+            new InMemoryKeyValueIterator(map.keySet()));
     }
 
     @Override
@@ -146,11 +147,11 @@ public void close() {
         open = false;
     }
 
-    private static class InMemoryKeyValueIterator implements KeyValueIterator {
-        private final Iterator> iter;
+    private class InMemoryKeyValueIterator implements KeyValueIterator {
+        private final Iterator iter;
 
-        private InMemoryKeyValueIterator(final Iterator> iter) {
-            this.iter = iter;
+        private InMemoryKeyValueIterator(final Set keySet) {
+            this.iter = new TreeSet<>(keySet).iterator();
         }
 
         @Override
@@ -160,8 +161,8 @@ public boolean hasNext() {
 
         @Override
         public KeyValue next() {
-            final Map.Entry entry = iter.next();
-            return new KeyValue<>(entry.getKey(), entry.getValue());
+            final Bytes key = iter.next();
+            return new KeyValue<>(key, map.get(key));
         }
 
         @Override
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
index ec53dfa5d46cd..a1d0aab277052 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
@@ -17,7 +17,8 @@
 package org.apache.kafka.streams.state.internals;
 
 import java.util.NavigableMap;
-import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.TreeMap;
+import java.util.TreeSet;
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
@@ -39,7 +40,7 @@
 class NamedCache {
     private static final Logger log = LoggerFactory.getLogger(NamedCache.class);
     private final String name;
-    private final NavigableMap cache = new ConcurrentSkipListMap<>();
+    private final NavigableMap cache = new TreeMap<>();
     private final Set dirtyKeys = new LinkedHashSet<>();
     private ThreadCache.DirtyEntryFlushListener listener;
     private LRUNode tail;
@@ -270,12 +271,16 @@ public boolean isEmpty() {
         return cache.isEmpty();
     }
 
-    synchronized Iterator> subMapIterator(final Bytes from, final Bytes to) {
-        return cache.subMap(from, true, to, true).entrySet().iterator();
+    synchronized Iterator keyRange(final Bytes from, final Bytes to) {
+        return keySetIterator(cache.navigableKeySet().subSet(from, true, to, true));
     }
 
-    synchronized Iterator> allIterator() {
-        return cache.entrySet().iterator();
+    private Iterator keySetIterator(final Set keySet) {
+        return new TreeSet<>(keySet).iterator();
+    }
+
+    synchronized Iterator allKeys() {
+        return keySetIterator(cache.navigableKeySet());
     }
 
     synchronized LRUCacheEntry first() {
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java
index 1fe99c51b2ff9..2899573d4b6f2 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java
@@ -20,7 +20,6 @@
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
-import org.apache.kafka.streams.state.internals.NamedCache.LRUNode;
 import org.slf4j.Logger;
 
 import java.util.Collections;
@@ -181,17 +180,17 @@ public LRUCacheEntry delete(final String namespace, final Bytes key) {
     public MemoryLRUCacheBytesIterator range(final String namespace, final Bytes from, final Bytes to) {
         final NamedCache cache = getCache(namespace);
         if (cache == null) {
-            return new MemoryLRUCacheBytesIterator(Collections.emptyIterator());
+            return new MemoryLRUCacheBytesIterator(Collections.emptyIterator(), new NamedCache(namespace, this.metrics));
         }
-        return new MemoryLRUCacheBytesIterator(cache.subMapIterator(from, to));
+        return new MemoryLRUCacheBytesIterator(cache.keyRange(from, to), cache);
     }
 
     public MemoryLRUCacheBytesIterator all(final String namespace) {
         final NamedCache cache = getCache(namespace);
         if (cache == null) {
-            return new MemoryLRUCacheBytesIterator(Collections.emptyIterator());
+            return new MemoryLRUCacheBytesIterator(Collections.emptyIterator(), new NamedCache(namespace, this.metrics));
         }
-        return new MemoryLRUCacheBytesIterator(cache.allIterator());
+        return new MemoryLRUCacheBytesIterator(cache.allKeys(), cache);
     }
 
     public long size() {
@@ -261,11 +260,13 @@ private synchronized NamedCache getOrCreateCache(final String name) {
     }
 
     static class MemoryLRUCacheBytesIterator implements PeekingKeyValueIterator {
-        private final Iterator> underlying;
+        private final Iterator keys;
+        private final NamedCache cache;
         private KeyValue nextEntry;
 
-        MemoryLRUCacheBytesIterator(final Iterator> underlying) {
-            this.underlying = underlying;
+        MemoryLRUCacheBytesIterator(final Iterator keys, final NamedCache cache) {
+            this.keys = keys;
+            this.cache = cache;
         }
 
         public Bytes peekNextKey() {
@@ -289,7 +290,7 @@ public boolean hasNext() {
                 return true;
             }
 
-            while (underlying.hasNext() && nextEntry == null) {
+            while (keys.hasNext() && nextEntry == null) {
                 internalNext();
             }
 
@@ -307,9 +308,8 @@ public KeyValue next() {
         }
 
         private void internalNext() {
-            final Map.Entry mapEntry = underlying.next();
-            final Bytes cacheKey = mapEntry.getKey();
-            final LRUCacheEntry entry = mapEntry.getValue().entry();
+            final Bytes cacheKey = keys.next();
+            final LRUCacheEntry entry = cache.get(cacheKey);
             if (entry == null) {
                 return;
             }

From 33d06082117d971cdcddd4f01392006b543f3c01 Mon Sep 17 00:00:00 2001
From: Justine Olshan 
Date: Fri, 16 Aug 2019 13:28:10 -0700
Subject: [PATCH 0544/1071] MINOR: Remove Deprecated Scala Procedure Syntax
 (#7214)

Reviewers: Colin P. McCabe , Stanislav Kozlovski 
---
 .../main/scala/kafka/admin/AclCommand.scala   |  10 +-
 .../admin/BrokerApiVersionsCommand.scala      |   8 +-
 .../scala/kafka/admin/ConfigCommand.scala     |  14 +-
 .../kafka/admin/ConsumerGroupCommand.scala    |   6 +-
 .../kafka/admin/DelegationTokenCommand.scala  |   2 +-
 ...referredReplicaLeaderElectionCommand.scala |  10 +-
 .../admin/ReassignPartitionsCommand.scala     |  10 +-
 .../main/scala/kafka/admin/TopicCommand.scala |  12 +-
 .../kafka/admin/ZkSecurityMigrator.scala      |   8 +-
 core/src/main/scala/kafka/api/ApiUtils.scala  |   2 +-
 .../main/scala/kafka/cluster/Partition.scala  |   8 +-
 core/src/main/scala/kafka/common/Config.scala |   2 +-
 .../kafka/common/InterBrokerSendThread.scala  |   2 +-
 .../scala/kafka/common/MessageFormatter.scala |   4 +-
 .../scala/kafka/common/MessageReader.scala    |   4 +-
 .../ZkNodeChangeNotificationListener.scala    |   8 +-
 .../controller/ControllerChannelManager.scala |  18 +-
 .../kafka/controller/ControllerContext.scala  |   2 +-
 .../kafka/controller/KafkaController.scala    |  44 ++--
 .../controller/PartitionStateMachine.scala    |   6 +-
 .../controller/ReplicaStateMachine.scala      |   6 +-
 .../controller/TopicDeletionManager.scala     |  16 +-
 .../coordinator/group/GroupCoordinator.scala  |  62 ++---
 .../coordinator/group/GroupMetadata.scala     |  18 +-
 .../group/GroupMetadataManager.scala          |  24 +-
 .../transaction/ProducerIdManager.scala       |   2 +-
 .../transaction/TransactionCoordinator.scala  |   8 +-
 .../transaction/TransactionLog.scala          |   2 +-
 .../TransactionMarkerChannelManager.scala     |   2 +-
 .../transaction/TransactionStateManager.scala |  12 +-
 .../main/scala/kafka/log/AbstractIndex.scala  |  10 +-
 core/src/main/scala/kafka/log/Log.scala       |  22 +-
 .../src/main/scala/kafka/log/LogCleaner.scala |  44 ++--
 .../scala/kafka/log/LogCleanerManager.scala   |  16 +-
 core/src/main/scala/kafka/log/LogConfig.scala |   6 +-
 .../src/main/scala/kafka/log/LogManager.scala |  16 +-
 .../src/main/scala/kafka/log/LogSegment.scala |  16 +-
 .../main/scala/kafka/log/LogValidator.scala   |   4 +-
 .../main/scala/kafka/log/OffsetIndex.scala    |  10 +-
 core/src/main/scala/kafka/log/OffsetMap.scala |  12 +-
 .../kafka/log/ProducerStateManager.scala      |  12 +-
 core/src/main/scala/kafka/log/TimeIndex.scala |  10 +-
 .../metrics/KafkaCSVMetricsReporter.scala     |   6 +-
 .../kafka/metrics/KafkaMetricsReporter.scala  |   7 +-
 .../scala/kafka/network/RequestChannel.scala  |  14 +-
 .../scala/kafka/network/SocketServer.scala    |  30 +--
 .../kafka/security/CredentialProvider.scala   |   2 +-
 .../security/auth/SimpleAclAuthorizer.scala   |  16 +-
 .../kafka/server/AbstractFetcherManager.scala |  10 +-
 .../kafka/server/AbstractFetcherThread.scala  |  26 +-
 .../scala/kafka/server/AdminManager.scala     |   8 +-
 .../scala/kafka/server/BrokerStates.scala     |   4 +-
 .../kafka/server/ClientQuotaManager.scala     |   6 +-
 .../server/ClientRequestQuotaManager.scala    |   2 +-
 .../scala/kafka/server/ConfigHandler.scala    |  12 +-
 .../server/DelayedCreatePartitions.scala      |   2 +-
 .../kafka/server/DelayedDeleteRecords.scala   |   6 +-
 .../kafka/server/DelayedDeleteTopics.scala    |   2 +-
 .../scala/kafka/server/DelayedFetch.scala     |   4 +-
 .../scala/kafka/server/DelayedOperation.scala |  12 +-
 .../scala/kafka/server/DelayedProduce.scala   |   6 +-
 .../kafka/server/DelegationTokenManager.scala |  10 +-
 .../kafka/server/DynamicBrokerConfig.scala    |   2 +-
 .../kafka/server/DynamicConfigManager.scala   |   4 +-
 .../main/scala/kafka/server/KafkaApis.scala   |  90 +++----
 .../main/scala/kafka/server/KafkaConfig.scala |   4 +-
 .../kafka/server/KafkaRequestHandler.scala    |  14 +-
 .../main/scala/kafka/server/KafkaServer.scala |   7 +-
 .../kafka/server/KafkaServerStartable.scala   |   6 +-
 .../scala/kafka/server/MetadataCache.scala    |   2 +-
 .../scala/kafka/server/QuotaFactory.scala     |   2 +-
 .../server/ReplicaAlterLogDirsManager.scala   |   2 +-
 .../server/ReplicaAlterLogDirsThread.scala    |   2 +-
 .../server/ReplicaFetcherBlockingSend.scala   |   4 +-
 .../kafka/server/ReplicaFetcherManager.scala  |   2 +-
 .../scala/kafka/server/ReplicaManager.scala   |  22 +-
 .../server/ReplicationQuotaManager.scala      |  10 +-
 .../server/checkpoints/CheckpointFile.scala   |   2 +-
 .../LeaderEpochCheckpointFile.scala           |   2 +-
 .../checkpoints/OffsetCheckpointFile.scala    |   2 +-
 .../scala/kafka/tools/ConsoleConsumer.scala   |  36 +--
 .../scala/kafka/tools/ConsoleProducer.scala   |   4 +-
 .../kafka/tools/ConsumerPerformance.scala     |   6 +-
 .../scala/kafka/tools/DumpLogSegments.scala   |  16 +-
 .../scala/kafka/tools/EndToEndLatency.scala   |   4 +-
 core/src/main/scala/kafka/tools/JmxTool.scala |   2 +-
 .../main/scala/kafka/tools/MirrorMaker.scala  |  40 +--
 .../kafka/tools/ReplicaVerificationTool.scala |  16 +-
 .../kafka/tools/StateChangeLogMerger.scala    |   2 +-
 .../scala/kafka/utils/CommandLineUtils.scala  |   8 +-
 .../main/scala/kafka/utils/CoreUtils.scala    |   4 +-
 .../src/main/scala/kafka/utils/FileLock.scala |   4 +-
 .../scala/kafka/utils/KafkaScheduler.scala    |   8 +-
 core/src/main/scala/kafka/utils/Pool.scala    |   2 +-
 .../main/scala/kafka/utils/Throttler.scala    |   4 +-
 .../kafka/utils/VerifiableProperties.scala    |   2 +-
 .../main/scala/kafka/utils/VersionInfo.scala  |   2 +-
 .../main/scala/kafka/utils/timer/Timer.scala  |   2 +-
 .../main/scala/kafka/zk/AdminZkClient.scala   |  12 +-
 .../main/scala/kafka/zk/KafkaZkClient.scala   |   4 +-
 .../kafka/api/AbstractConsumerTest.scala      |   8 +-
 .../api/AdminClientIntegrationTest.scala      |   2 +-
 .../kafka/api/AuthorizerIntegrationTest.scala | 118 ++++-----
 .../kafka/api/BaseConsumerTest.scala          |   4 +-
 .../kafka/api/BaseProducerSendTest.scala      |  32 +--
 .../integration/kafka/api/BaseQuotaTest.scala |  20 +-
 .../kafka/api/ClientIdQuotaTest.scala         |   8 +-
 .../kafka/api/ConsumerBounceTest.scala        |  32 +--
 .../kafka/api/CustomQuotaCallbackTest.scala   |   6 +-
 ...gationTokenEndToEndAuthorizationTest.scala |   6 +-
 .../DescribeAuthorizedOperationsTest.scala    |   2 +-
 .../kafka/api/EndToEndAuthorizationTest.scala |  22 +-
 .../kafka/api/EndToEndClusterIdTest.scala     |  14 +-
 .../api/GroupCoordinatorIntegrationTest.scala |   2 +-
 .../kafka/api/IntegrationTestHarness.scala    |   4 +-
 .../kafka/api/LogAppendTimeTest.scala         |   4 +-
 .../integration/kafka/api/MetricsTest.scala   |   2 +-
 .../kafka/api/PlaintextConsumerTest.scala     | 104 ++++----
 .../PlaintextEndToEndAuthorizationTest.scala  |   4 +-
 .../kafka/api/PlaintextProducerSendTest.scala |  12 +-
 .../kafka/api/ProducerCompressionTest.scala   |   6 +-
 .../api/ProducerFailureHandlingTest.scala     |  28 +-
 .../api/RackAwareAutoTopicCreationTest.scala  |   2 +-
 ...aslClientsWithInvalidCredentialsTest.scala |  20 +-
 .../api/SaslEndToEndAuthorizationTest.scala   |   2 +-
 .../api/SaslMultiMechanismConsumerTest.scala  |   2 +-
 .../api/SaslPlainPlaintextConsumerTest.scala  |   2 +-
 ...aslPlainSslEndToEndAuthorizationTest.scala |  14 +-
 ...aslScramSslEndToEndAuthorizationTest.scala |   4 +-
 .../integration/kafka/api/SaslSetup.scala     |   6 +-
 .../SaslSslAdminClientIntegrationTest.scala   |   2 +-
 .../api/SslEndToEndAuthorizationTest.scala    |   2 +-
 .../kafka/api/TransactionsBounceTest.scala    |   4 +-
 .../kafka/api/TransactionsTest.scala          |   2 +-
 .../kafka/api/UserClientIdQuotaTest.scala     |   8 +-
 .../integration/kafka/api/UserQuotaTest.scala |   8 +-
 .../network/DynamicConnectionQuotaTest.scala  |   6 +-
 .../DynamicBrokerReconfigurationTest.scala    |   6 +-
 .../server/GssapiAuthenticationTest.scala     |   2 +-
 ...nersWithSameSecurityProtocolBaseTest.scala |   2 +-
 .../kafka/server/ScramServerStartupTest.scala |   2 +-
 .../kafka/security/minikdc/MiniKdc.scala      |  20 +-
 .../kafka/tools/LogCompactionTester.scala     |   8 +-
 .../kafka/ReplicationQuotasTestRig.scala      |  12 +-
 .../scala/other/kafka/StressTestLog.scala     |  12 +-
 .../other/kafka/TestLinearWriteSpeed.scala    |   8 +-
 .../unit/kafka/admin/AclCommandTest.scala     |  14 +-
 .../unit/kafka/admin/AddPartitionsTest.scala  |   2 +-
 .../unit/kafka/admin/AdminRackAwareTest.scala |  30 +--
 .../unit/kafka/admin/ConfigCommandTest.scala  |  22 +-
 .../admin/ConsumerGroupCommandTest.scala      |  10 +-
 .../admin/DeleteConsumerGroupsTest.scala      |  22 +-
 .../unit/kafka/admin/DeleteTopicTest.scala    |  28 +-
 .../admin/DescribeConsumerGroupTest.scala     |  62 ++---
 .../admin/LeaderElectionCommandTest.scala     |   4 +-
 .../kafka/admin/ListConsumerGroupTest.scala   |   4 +-
 ...rredReplicaLeaderElectionCommandTest.scala |  32 +--
 .../unit/kafka/admin/RackAwareTest.scala      |   2 +-
 .../admin/ReassignPartitionsClusterTest.scala |  36 +--
 .../ReassignPartitionsCommandArgsTest.scala   |   4 +-
 .../admin/ReassignPartitionsCommandTest.scala |  24 +-
 .../admin/ResetConsumerGroupOffsetTest.scala  |  46 ++--
 .../unit/kafka/admin/TopicCommandTest.scala   |  28 +-
 .../TopicCommandWithAdminClientTest.scala     |   8 +-
 .../scala/unit/kafka/api/ApiUtilsTest.scala   |   4 +-
 ...ZkNodeChangeNotificationListenerTest.scala |   4 +-
 .../controller/ControllerFailoverTest.scala   |   4 +-
 .../ControllerIntegrationTest.scala           |   6 +-
 .../AbstractCoordinatorConcurrencyTest.scala  |  14 +-
 .../GroupCoordinatorConcurrencyTest.scala     |  10 +-
 .../group/GroupCoordinatorTest.scala          | 242 +++++++++---------
 .../group/GroupMetadataManagerTest.scala      | 100 ++++----
 .../coordinator/group/GroupMetadataTest.scala |  58 ++---
 .../group/MemberMetadataTest.scala            |  12 +-
 .../transaction/ProducerIdManagerTest.scala   |   4 +-
 ...ransactionCoordinatorConcurrencyTest.scala |   4 +-
 .../transaction/TransactionLogTest.scala      |   4 +-
 .../TransactionStateManagerTest.scala         |  14 +-
 .../integration/KafkaServerTestHarness.scala  |  12 +-
 ...tricsDuringTopicCreationDeletionTest.scala |   8 +-
 .../kafka/integration/MinIsrConfigTest.scala  |   2 +-
 .../UncleanLeaderElectionTest.scala           |   6 +-
 .../kafka/log/BrokerCompressionTest.scala     |   4 +-
 .../kafka/log/LogCleanerIntegrationTest.scala |   2 +-
 ...gCleanerParameterizedIntegrationTest.scala |   8 +-
 .../scala/unit/kafka/log/LogCleanerTest.scala |  12 +-
 .../scala/unit/kafka/log/LogConfigTest.scala  |  12 +-
 .../scala/unit/kafka/log/LogManagerTest.scala |  40 +--
 .../scala/unit/kafka/log/LogSegmentTest.scala |  36 +--
 .../test/scala/unit/kafka/log/LogTest.scala   | 166 ++++++------
 .../unit/kafka/log/LogValidatorTest.scala     |  98 +++----
 .../unit/kafka/log/OffsetIndexTest.scala      |  20 +-
 .../scala/unit/kafka/log/OffsetMapTest.scala  |   8 +-
 .../kafka/log/ProducerStateManagerTest.scala  |   6 +-
 .../scala/unit/kafka/log/TimeIndexTest.scala  |  12 +-
 .../unit/kafka/metrics/KafkaTimerTest.scala   |   4 +-
 .../unit/kafka/metrics/MetricsTest.scala      |   4 +-
 .../unit/kafka/network/SocketServerTest.scala |  50 ++--
 .../auth/SimpleAclAuthorizerTest.scala        |  34 +--
 .../security/auth/ZkAuthorizationTest.scala   |   4 +-
 .../DelegationTokenManagerTest.scala          |   2 +-
 .../kafka/server/ApiVersionsRequestTest.scala |   6 +-
 .../unit/kafka/server/BaseRequestTest.scala   |   4 +-
 .../server/BrokerEpochIntegrationTest.scala   |  10 +-
 .../kafka/server/ClientQuotaManagerTest.scala |  30 +--
 .../server/CreateTopicsRequestTest.scala      |   8 +-
 .../CreateTopicsRequestWithPolicyTest.scala   |   4 +-
 .../kafka/server/DelayedOperationTest.scala   |  26 +-
 .../server/DeleteTopicsRequestTest.scala      |   6 +-
 ...opicsRequestWithDeletionDisabledTest.scala |   2 +-
 .../server/DynamicBrokerConfigTest.scala      |   2 +-
 .../server/DynamicConfigChangeTest.scala      |  24 +-
 .../unit/kafka/server/DynamicConfigTest.scala |   8 +-
 .../kafka/server/EdgeCaseRequestTest.scala    |  14 +-
 .../server/HighwatermarkPersistenceTest.scala |   6 +-
 .../unit/kafka/server/IsrExpirationTest.scala |  12 +-
 .../unit/kafka/server/KafkaApisTest.scala     |  14 +-
 .../unit/kafka/server/KafkaConfigTest.scala   |  66 ++---
 .../KafkaMetricReporterClusterIdTest.scala    |   8 +-
 ...aMetricReporterExceptionHandlingTest.scala |  18 +-
 .../unit/kafka/server/KafkaServerTest.scala   |   2 +-
 .../kafka/server/LeaderElectionTest.scala     |   6 +-
 .../unit/kafka/server/LogDirFailureTest.scala |  14 +-
 .../unit/kafka/server/LogOffsetTest.scala     |  16 +-
 .../unit/kafka/server/MetadataCacheTest.scala |  14 +-
 .../kafka/server/MetadataRequestTest.scala    |  18 +-
 .../kafka/server/ProduceRequestTest.scala     |   6 +-
 .../unit/kafka/server/ReplicaFetchTest.scala  |   6 +-
 .../kafka/server/ReplicaManagerTest.scala     |  18 +-
 .../unit/kafka/server/RequestQuotaTest.scala  |  34 +--
 .../server/SaslApiVersionsRequestTest.scala   |   8 +-
 .../server/ServerGenerateBrokerIdTest.scala   |  16 +-
 .../server/ServerGenerateClusterIdTest.scala  |  16 +-
 .../kafka/server/ServerShutdownTest.scala     |  16 +-
 .../unit/kafka/server/ServerStartupTest.scala |   2 +-
 .../unit/kafka/server/SimpleFetchTest.scala   |   6 +-
 .../ThrottledChannelExpirationTest.scala      |   6 +-
 ...venReplicationProtocolAcceptanceTest.scala |   4 +-
 .../epoch/LeaderEpochFileCacheTest.scala      |  10 +-
 .../epoch/LeaderEpochIntegrationTest.scala    |   4 +-
 .../util/ReplicaFetcherMockBlockingSend.scala |   2 +-
 .../kafka/tools/ConsoleConsumerTest.scala     |  30 +--
 .../kafka/tools/ConsoleProducerTest.scala     |   4 +-
 .../unit/kafka/tools/MirrorMakerTest.scala    |   6 +-
 .../kafka/utils/CommandLineUtilsTest.scala    |  12 +-
 .../unit/kafka/utils/CoreUtilsTest.scala      |  22 +-
 .../unit/kafka/utils/JaasTestUtils.scala      |   2 +-
 .../scala/unit/kafka/utils/JsonTest.scala     |   8 +-
 .../unit/kafka/utils/LogCaptureAppender.scala |   2 +-
 .../unit/kafka/utils/MockScheduler.scala      |  10 +-
 .../scala/unit/kafka/utils/MockTime.scala     |   2 +-
 .../kafka/utils/ReplicationUtilsTest.scala    |   4 +-
 .../unit/kafka/utils/SchedulerTest.scala      |  16 +-
 .../scala/unit/kafka/utils/TestUtils.scala    |  40 +--
 .../unit/kafka/utils/ThrottlerTest.scala      |   2 +-
 .../unit/kafka/utils/TopicFilterTest.scala    |   2 +-
 .../unit/kafka/utils/timer/MockTimer.scala    |   2 +-
 .../kafka/utils/timer/TimerTaskListTest.scala |   2 +-
 .../unit/kafka/utils/timer/TimerTest.scala    |   2 +-
 .../unit/kafka/zk/AdminZkClientTest.scala     |  24 +-
 .../unit/kafka/zk/KafkaZkClientTest.scala     |  36 +--
 .../zk/ReassignPartitionsZNodeTest.scala      |   6 +-
 .../unit/kafka/zk/ZkFourLetterWords.scala     |   2 +-
 .../unit/kafka/zk/ZooKeeperTestHarness.scala  |  10 +-
 .../kafka/zookeeper/ZooKeeperClientTest.scala |  10 +-
 265 files changed, 1902 insertions(+), 1902 deletions(-)

diff --git a/core/src/main/scala/kafka/admin/AclCommand.scala b/core/src/main/scala/kafka/admin/AclCommand.scala
index 6f5b06cb51890..63c8466352d80 100644
--- a/core/src/main/scala/kafka/admin/AclCommand.scala
+++ b/core/src/main/scala/kafka/admin/AclCommand.scala
@@ -41,7 +41,7 @@ object AclCommand extends Logging {
 
   private val Newline = scala.util.Properties.lineSeparator
 
-  def main(args: Array[String]) {
+  def main(args: Array[String]): Unit = {
 
     val opts = new AclCommandOptions(args)
 
@@ -80,7 +80,7 @@ object AclCommand extends Logging {
 
   class AdminClientService(val opts: AclCommandOptions) extends AclCommandService with Logging {
 
-    private def withAdminClient(opts: AclCommandOptions)(f: Admin => Unit) {
+    private def withAdminClient(opts: AclCommandOptions)(f: Admin => Unit): Unit = {
       val props = if (opts.options.has(opts.commandConfigOpt))
         Utils.loadProps(opts.options.valueOf(opts.commandConfigOpt))
       else
@@ -185,7 +185,7 @@ object AclCommand extends Logging {
 
   class AuthorizerService(val opts: AclCommandOptions) extends AclCommandService with Logging {
 
-    private def withAuthorizer()(f: Authorizer => Unit) {
+    private def withAuthorizer()(f: Authorizer => Unit): Unit = {
       val defaultProps = Map(KafkaConfig.ZkEnableSecureAclsProp -> JaasUtils.isZkSecurityEnabled)
       val authorizerProperties =
         if (opts.options.has(opts.authorizerPropertiesOpt)) {
@@ -258,7 +258,7 @@ object AclCommand extends Logging {
       }
     }
 
-    private def removeAcls(authorizer: Authorizer, acls: Set[Acl], filter: ResourcePatternFilter) {
+    private def removeAcls(authorizer: Authorizer, acls: Set[Acl], filter: ResourcePatternFilter): Unit = {
       getAcls(authorizer, filter)
         .keys
         .foreach(resource =>
@@ -573,7 +573,7 @@ object AclCommand extends Logging {
 
     options = parser.parse(args: _*)
 
-    def checkArgs() {
+    def checkArgs(): Unit = {
       if (options.has(bootstrapServerOpt) && options.has(authorizerOpt))
         CommandLineUtils.printUsageAndDie(parser, "Only one of --bootstrap-server or --authorizer must be specified")
 
diff --git a/core/src/main/scala/kafka/admin/BrokerApiVersionsCommand.scala b/core/src/main/scala/kafka/admin/BrokerApiVersionsCommand.scala
index 4562f37d7981d..f08385f090cb4 100644
--- a/core/src/main/scala/kafka/admin/BrokerApiVersionsCommand.scala
+++ b/core/src/main/scala/kafka/admin/BrokerApiVersionsCommand.scala
@@ -92,7 +92,7 @@ object BrokerApiVersionsCommand {
     options = parser.parse(args : _*)
     checkArgs()
 
-    def checkArgs() {
+    def checkArgs(): Unit = {
       CommandLineUtils.printHelpAndExitIfNeeded(this, "This tool helps to retrieve broker version information.")
       // check required args
       CommandLineUtils.checkRequiredArgs(parser, options, bootstrapServerOpt)
@@ -111,7 +111,7 @@ object BrokerApiVersionsCommand {
     val pendingFutures = new ConcurrentLinkedQueue[RequestFuture[ClientResponse]]()
 
     val networkThread = new KafkaThread("admin-client-network-thread", new Runnable {
-      override def run() {
+      override def run(): Unit = {
         try {
           while (running)
             client.poll(time.timer(Long.MaxValue))
@@ -169,7 +169,7 @@ object BrokerApiVersionsCommand {
     /**
      * Wait until there is a non-empty list of brokers in the cluster.
      */
-    def awaitBrokers() {
+    def awaitBrokers(): Unit = {
       var nodes = List[Node]()
       do {
         nodes = findAllBrokers()
@@ -192,7 +192,7 @@ object BrokerApiVersionsCommand {
         broker -> Try[NodeApiVersions](new NodeApiVersions(getApiVersions(broker).asJava))
       }.toMap
 
-    def close() {
+    def close(): Unit = {
       running = false
       try {
         client.close()
diff --git a/core/src/main/scala/kafka/admin/ConfigCommand.scala b/core/src/main/scala/kafka/admin/ConfigCommand.scala
index 781cc1a7fcb5b..db213db7fe4f8 100644
--- a/core/src/main/scala/kafka/admin/ConfigCommand.scala
+++ b/core/src/main/scala/kafka/admin/ConfigCommand.scala
@@ -111,7 +111,7 @@ object ConfigCommand extends Config {
     }
   }
 
-  private[admin] def alterConfig(zkClient: KafkaZkClient, opts: ConfigCommandOptions, adminZkClient: AdminZkClient) {
+  private[admin] def alterConfig(zkClient: KafkaZkClient, opts: ConfigCommandOptions, adminZkClient: AdminZkClient): Unit = {
     val configsToBeAdded = parseConfigsToBeAdded(opts)
     val configsToBeDeleted = parseConfigsToBeDeleted(opts)
     val entity = parseEntity(opts)
@@ -157,7 +157,7 @@ object ConfigCommand extends Config {
     println(s"Completed Updating config for entity: $entity.")
   }
 
-  private def preProcessScramCredentials(configsToBeAdded: Properties) {
+  private def preProcessScramCredentials(configsToBeAdded: Properties): Unit = {
     def scramCredential(mechanism: ScramMechanism, credentialStr: String): String = {
       val pattern = "(?:iterations=([0-9]*),)?password=(.*)".r
       val (iterations, password) = credentialStr match {
@@ -194,7 +194,7 @@ object ConfigCommand extends Config {
    * Password configs are encrypted using the secret `KafkaConfig.PasswordEncoderSecretProp`.
    * The secret is removed from `configsToBeAdded` and will not be persisted in ZooKeeper.
    */
-  private def preProcessBrokerConfigs(configsToBeAdded: Properties, perBrokerConfig: Boolean) {
+  private def preProcessBrokerConfigs(configsToBeAdded: Properties, perBrokerConfig: Boolean): Unit = {
     val passwordEncoderConfigs = new Properties
     passwordEncoderConfigs ++= configsToBeAdded.asScala.filter { case (key, _) => key.startsWith("password.encoder.") }
     if (!passwordEncoderConfigs.isEmpty) {
@@ -221,7 +221,7 @@ object ConfigCommand extends Config {
     }
   }
 
-  private def describeConfig(zkClient: KafkaZkClient, opts: ConfigCommandOptions, adminZkClient: AdminZkClient) {
+  private def describeConfig(zkClient: KafkaZkClient, opts: ConfigCommandOptions, adminZkClient: AdminZkClient): Unit = {
     val configEntity = parseEntity(opts)
     val describeAllUsers = configEntity.root.entityType == ConfigType.User && !configEntity.root.sanitizedName.isDefined && !configEntity.child.isDefined
     val entities = configEntity.getAllEntities(zkClient)
@@ -294,7 +294,7 @@ object ConfigCommand extends Config {
   }
 
   private[admin] def alterBrokerConfig(adminClient: Admin, opts: ConfigCommandOptions,
-                                       entityType: String, entityName: String) {
+                                       entityType: String, entityName: String): Unit = {
     val configsToBeAdded = parseConfigsToBeAdded(opts).asScala.map { case (k, v) => (k, new ConfigEntry(k, v)) }
     val configsToBeDeleted = parseConfigsToBeDeleted(opts)
 
@@ -339,7 +339,7 @@ object ConfigCommand extends Config {
   }
 
   private def describeBrokerConfig(adminClient: Admin, opts: ConfigCommandOptions,
-                                   entityType: String, entityName: String) {
+                                   entityType: String, entityName: String): Unit = {
     val configs = if (entityType == ConfigType.Broker)
       brokerConfig(adminClient, entityName, includeSynonyms = true)
     else // broker logger
@@ -537,7 +537,7 @@ object ConfigCommand extends Config {
 
     val allOpts: Set[OptionSpec[_]] = Set(alterOpt, describeOpt, entityType, entityName, addConfig, deleteConfig, helpOpt)
 
-    def checkArgs() {
+    def checkArgs(): Unit = {
       // should have exactly one action
       val actions = Seq(alterOpt, describeOpt).count(options.has _)
       if(actions != 1)
diff --git a/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala b/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala
index 8c4c5e4dbd5b6..73c54faf0b7e8 100755
--- a/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala
+++ b/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala
@@ -42,7 +42,7 @@ import scala.reflect.ClassTag
 
 object ConsumerGroupCommand extends Logging {
 
-  def main(args: Array[String]) {
+  def main(args: Array[String]): Unit = {
     val opts = new ConsumerGroupCommandOptions(args)
 
     CommandLineUtils.printHelpAndExitIfNeeded(opts, "This tool helps to list all consumer groups, describe a consumer group, delete consumer group info, or reset consumer group offsets.")
@@ -522,7 +522,7 @@ object ConsumerGroupCommand extends Logging {
       successfulLogTimestampOffsets ++ getLogEndOffsets(groupId, unsuccessfulOffsetsForTimes.keySet.toSeq)
     }
 
-    def close() {
+    def close(): Unit = {
       adminClient.close()
       consumers.values.foreach(consumer =>
         Option(consumer).foreach(_.close())
@@ -922,7 +922,7 @@ object ConsumerGroupCommand extends Logging {
     val allResetOffsetScenarioOpts: Set[OptionSpec[_]] = Set(resetToOffsetOpt, resetShiftByOpt,
       resetToDatetimeOpt, resetByDurationOpt, resetToEarliestOpt, resetToLatestOpt, resetToCurrentOpt, resetFromFileOpt)
 
-    def checkArgs() {
+    def checkArgs(): Unit = {
 
       CommandLineUtils.checkRequiredArgs(parser, options, bootstrapServerOpt)
 
diff --git a/core/src/main/scala/kafka/admin/DelegationTokenCommand.scala b/core/src/main/scala/kafka/admin/DelegationTokenCommand.scala
index 5db38f4319187..316698baea397 100644
--- a/core/src/main/scala/kafka/admin/DelegationTokenCommand.scala
+++ b/core/src/main/scala/kafka/admin/DelegationTokenCommand.scala
@@ -196,7 +196,7 @@ object DelegationTokenCommand extends Logging {
 
     options = parser.parse(args : _*)
 
-    def checkArgs() {
+    def checkArgs(): Unit = {
       // check required args
       CommandLineUtils.checkRequiredArgs(parser, options, bootstrapServerOpt, commandConfigOpt)
 
diff --git a/core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala b/core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala
index cb8c89985cfdc..04765a8b20600 100755
--- a/core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala
+++ b/core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala
@@ -105,7 +105,7 @@ object PreferredReplicaLeaderElectionCommand extends Logging {
   }
 
   def writePreferredReplicaElectionData(zkClient: KafkaZkClient,
-                                        partitionsUndergoingPreferredReplicaElection: Set[TopicPartition]) {
+                                        partitionsUndergoingPreferredReplicaElection: Set[TopicPartition]): Unit = {
     try {
       zkClient.createPreferredReplicaElection(partitionsUndergoingPreferredReplicaElection.toSet)
       println("Created preferred replica election path with %s".format(partitionsUndergoingPreferredReplicaElection.mkString(",")))
@@ -158,8 +158,8 @@ object PreferredReplicaLeaderElectionCommand extends Logging {
     /** Elect the preferred leader for the given {@code partitionsForElection}.
       * If the given {@code partitionsForElection} are None then elect the preferred leader for all partitions.
       */
-    def electPreferredLeaders(partitionsForElection: Option[Set[TopicPartition]]) : Unit
-    def close() : Unit
+    def electPreferredLeaders(partitionsForElection: Option[Set[TopicPartition]]): Unit
+    def close(): Unit
   }
 
   class ZkCommand(zkConnect: String, isSecure: Boolean, timeout: Int)
@@ -169,7 +169,7 @@ object PreferredReplicaLeaderElectionCommand extends Logging {
     val time = Time.SYSTEM
     zkClient = KafkaZkClient(zkConnect, isSecure, timeout, timeout, Int.MaxValue, time)
 
-    override def electPreferredLeaders(partitionsFromUser: Option[Set[TopicPartition]]) {
+    override def electPreferredLeaders(partitionsFromUser: Option[Set[TopicPartition]]): Unit = {
       try {
         val topics =
           partitionsFromUser match {
@@ -283,7 +283,7 @@ object PreferredReplicaLeaderElectionCommand extends Logging {
 }
 
 class PreferredReplicaLeaderElectionCommand(zkClient: KafkaZkClient, partitionsFromUser: scala.collection.Set[TopicPartition]) {
-  def moveLeaderToPreferredReplica() = {
+  def moveLeaderToPreferredReplica(): Unit = {
     try {
       val topics = partitionsFromUser.map(_.topic).toSet
       val partitionsFromZk = zkClient.getPartitionsForTopics(topics).flatMap { case (topic, partitions) =>
diff --git a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala
index 37269373508d1..83d2e009c08af 100755
--- a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala
+++ b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala
@@ -84,7 +84,7 @@ object ReassignPartitionsCommand extends Logging {
     }
   }
 
-  def verifyAssignment(zkClient: KafkaZkClient, adminClientOpt: Option[Admin], opts: ReassignPartitionsCommandOptions) {
+  def verifyAssignment(zkClient: KafkaZkClient, adminClientOpt: Option[Admin], opts: ReassignPartitionsCommandOptions): Unit = {
     val jsonFile = opts.options.valueOf(opts.reassignmentJsonFileOpt)
     val jsonString = Utils.readFileAsString(jsonFile)
     verifyAssignment(zkClient, adminClientOpt, jsonString)
@@ -160,7 +160,7 @@ object ReassignPartitionsCommand extends Logging {
     }
   }
 
-  def generateAssignment(zkClient: KafkaZkClient, opts: ReassignPartitionsCommandOptions) {
+  def generateAssignment(zkClient: KafkaZkClient, opts: ReassignPartitionsCommandOptions): Unit = {
     val topicsToMoveJsonFile = opts.options.valueOf(opts.topicsToMoveJsonFileOpt)
     val brokerListToReassign = opts.options.valueOf(opts.brokerListOpt).split(',').map(_.toInt)
     val duplicateReassignments = CoreUtils.duplicates(brokerListToReassign)
@@ -196,7 +196,7 @@ object ReassignPartitionsCommand extends Logging {
     (partitionsToBeReassigned, currentAssignment)
   }
 
-  def executeAssignment(zkClient: KafkaZkClient, adminClientOpt: Option[Admin], opts: ReassignPartitionsCommandOptions) {
+  def executeAssignment(zkClient: KafkaZkClient, adminClientOpt: Option[Admin], opts: ReassignPartitionsCommandOptions): Unit = {
     val reassignmentJsonFile =  opts.options.valueOf(opts.reassignmentJsonFileOpt)
     val reassignmentJsonString = Utils.readFileAsString(reassignmentJsonFile)
     val interBrokerThrottle = opts.options.valueOf(opts.interBrokerThrottleOpt)
@@ -205,7 +205,7 @@ object ReassignPartitionsCommand extends Logging {
     executeAssignment(zkClient, adminClientOpt, reassignmentJsonString, Throttle(interBrokerThrottle, replicaAlterLogDirsThrottle), timeoutMs)
   }
 
-  def executeAssignment(zkClient: KafkaZkClient, adminClientOpt: Option[Admin], reassignmentJsonString: String, throttle: Throttle, timeoutMs: Long = 10000L) {
+  def executeAssignment(zkClient: KafkaZkClient, adminClientOpt: Option[Admin], reassignmentJsonString: String, throttle: Throttle, timeoutMs: Long = 10000L): Unit = {
     val (partitionAssignment, replicaAssignment) = parseAndValidate(zkClient, reassignmentJsonString)
     val adminZkClient = new AdminZkClient(zkClient)
     val reassignPartitionsCommand = new ReassignPartitionsCommand(zkClient, adminClientOpt, partitionAssignment.toMap, replicaAssignment, adminZkClient)
@@ -531,7 +531,7 @@ class ReassignPartitionsCommand(zkClient: KafkaZkClient,
     * Limit the throttle on currently moving replicas. Note that this command can use used to alter the throttle, but
     * it may not alter all limits originally set, if some of the brokers have completed their rebalance.
     */
-  def maybeLimit(throttle: Throttle) {
+  def maybeLimit(throttle: Throttle): Unit = {
     if (throttle.interBrokerLimit >= 0 || throttle.replicaAlterLogDirsLimit >= 0) {
       val existingBrokers = existingAssignment().values.flatten.toSeq
       val proposedBrokers = proposedPartitionAssignment.values.flatten.toSeq ++ proposedReplicaAssignment.keys.toSeq.map(_.brokerId())
diff --git a/core/src/main/scala/kafka/admin/TopicCommand.scala b/core/src/main/scala/kafka/admin/TopicCommand.scala
index 9233b228bfe53..986e555cc83e5 100755
--- a/core/src/main/scala/kafka/admin/TopicCommand.scala
+++ b/core/src/main/scala/kafka/admin/TopicCommand.scala
@@ -193,11 +193,11 @@ object TopicCommand extends Logging {
           "collide. To avoid issues it is best to use either, but not both.")
       createTopic(topic)
     }
-    def createTopic(topic: CommandTopicPartition)
-    def listTopics(opts: TopicCommandOptions)
-    def alterTopic(opts: TopicCommandOptions)
-    def describeTopic(opts: TopicCommandOptions)
-    def deleteTopic(opts: TopicCommandOptions)
+    def createTopic(topic: CommandTopicPartition): Unit
+    def listTopics(opts: TopicCommandOptions): Unit
+    def alterTopic(opts: TopicCommandOptions): Unit
+    def describeTopic(opts: TopicCommandOptions): Unit
+    def deleteTopic(opts: TopicCommandOptions): Unit
     def getTopics(topicWhitelist: Option[String], excludeInternalTopics: Boolean = false): Seq[String]
   }
 
@@ -649,7 +649,7 @@ object TopicCommand extends Logging {
     def topicConfig: Option[util.List[String]] = valuesAsOption(configOpt)
     def configsToDelete: Option[util.List[String]] = valuesAsOption(deleteConfigOpt)
 
-    def checkArgs() {
+    def checkArgs(): Unit = {
       if (args.length == 0)
         CommandLineUtils.printUsageAndDie(parser, "Create, delete, describe, or change a topic.")
 
diff --git a/core/src/main/scala/kafka/admin/ZkSecurityMigrator.scala b/core/src/main/scala/kafka/admin/ZkSecurityMigrator.scala
index 65ce0148e2611..0790fde01c89e 100644
--- a/core/src/main/scala/kafka/admin/ZkSecurityMigrator.scala
+++ b/core/src/main/scala/kafka/admin/ZkSecurityMigrator.scala
@@ -61,7 +61,7 @@ object ZkSecurityMigrator extends Logging {
                       + "znodes as part of the process of setting up ZooKeeper "
                       + "authentication.")
 
-  def run(args: Array[String]) {
+  def run(args: Array[String]): Unit = {
     val jaasFile = System.getProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM)
     val opts = new ZkSecurityMigratorOptions(args)
 
@@ -99,7 +99,7 @@ object ZkSecurityMigrator extends Logging {
     migrator.run()
   }
 
-  def main(args: Array[String]) {
+  def main(args: Array[String]): Unit = {
     try {
       run(args)
     } catch {
@@ -159,7 +159,7 @@ class ZkSecurityMigrator(zkClient: KafkaZkClient) extends Logging {
     def processResult(rc: Int,
                       path: String,
                       ctx: Object,
-                      children: java.util.List[String]) {
+                      children: java.util.List[String]): Unit = {
       val zkHandle = zkSecurityMigratorUtils.currentZooKeeper
       val promise = ctx.asInstanceOf[Promise[String]]
       Code.get(rc) match {
@@ -193,7 +193,7 @@ class ZkSecurityMigrator(zkClient: KafkaZkClient) extends Logging {
     def processResult(rc: Int,
                       path: String,
                       ctx: Object,
-                      stat: Stat) {
+                      stat: Stat): Unit = {
       val zkHandle = zkSecurityMigratorUtils.currentZooKeeper
       val promise = ctx.asInstanceOf[Promise[String]]
 
diff --git a/core/src/main/scala/kafka/api/ApiUtils.scala b/core/src/main/scala/kafka/api/ApiUtils.scala
index 4a0c8b0c636b7..1644358f7bb3a 100644
--- a/core/src/main/scala/kafka/api/ApiUtils.scala
+++ b/core/src/main/scala/kafka/api/ApiUtils.scala
@@ -45,7 +45,7 @@ object ApiUtils {
    * @param buffer The buffer to write to
    * @param string The string to write
    */
-  def writeShortString(buffer: ByteBuffer, string: String) {
+  def writeShortString(buffer: ByteBuffer, string: String): Unit = {
     if(string == null) {
       buffer.putShort(-1)
     } else {
diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala
index 87cbe54f05563..03395a4e4e568 100755
--- a/core/src/main/scala/kafka/cluster/Partition.scala
+++ b/core/src/main/scala/kafka/cluster/Partition.scala
@@ -413,7 +413,7 @@ class Partition(val topicPartition: TopicPartition,
     }
   }
 
-  def removeFutureLocalReplica(deleteFromLogDir: Boolean = true) {
+  def removeFutureLocalReplica(deleteFromLogDir: Boolean = true): Unit = {
     inWriteLock(leaderIsrUpdateLock) {
       futureLog = None
       if (deleteFromLogDir)
@@ -450,7 +450,7 @@ class Partition(val topicPartition: TopicPartition,
     } else false
   }
 
-  def delete() {
+  def delete(): Unit = {
     // need to hold the lock to prevent appendMessagesToLeader() from hitting I/O exceptions due to log being deleted
     inWriteLock(leaderIsrUpdateLock) {
       remoteReplicasMap.clear()
@@ -1076,7 +1076,7 @@ class Partition(val topicPartition: TopicPartition,
     * @param offset offset to be used for truncation
     * @param isFuture True iff the truncation should be performed on the future log of this partition
     */
-  def truncateTo(offset: Long, isFuture: Boolean) {
+  def truncateTo(offset: Long, isFuture: Boolean): Unit = {
     // The read lock is needed to prevent the follower replica from being truncated while ReplicaAlterDirThread
     // is executing maybeDeleteAndSwapFutureReplica() to replace follower replica with the future replica.
     inReadLock(leaderIsrUpdateLock) {
@@ -1090,7 +1090,7 @@ class Partition(val topicPartition: TopicPartition,
     * @param newOffset The new offset to start the log with
     * @param isFuture True iff the truncation should be performed on the future log of this partition
     */
-  def truncateFullyAndStartAt(newOffset: Long, isFuture: Boolean) {
+  def truncateFullyAndStartAt(newOffset: Long, isFuture: Boolean): Unit = {
     // The read lock is needed to prevent the follower replica from being truncated while ReplicaAlterDirThread
     // is executing maybeDeleteAndSwapFutureReplica() to replace follower replica with the future replica.
     inReadLock(leaderIsrUpdateLock) {
diff --git a/core/src/main/scala/kafka/common/Config.scala b/core/src/main/scala/kafka/common/Config.scala
index 4110ba7b07315..f56cca8bd0528 100644
--- a/core/src/main/scala/kafka/common/Config.scala
+++ b/core/src/main/scala/kafka/common/Config.scala
@@ -23,7 +23,7 @@ import org.apache.kafka.common.errors.InvalidConfigurationException
 
 trait Config extends Logging {
 
-  def validateChars(prop: String, value: String) {
+  def validateChars(prop: String, value: String): Unit = {
     val legalChars = "[a-zA-Z0-9\\._\\-]"
     val rgx = new Regex(legalChars + "*")
 
diff --git a/core/src/main/scala/kafka/common/InterBrokerSendThread.scala b/core/src/main/scala/kafka/common/InterBrokerSendThread.scala
index aedaac79d2774..1551704203a91 100644
--- a/core/src/main/scala/kafka/common/InterBrokerSendThread.scala
+++ b/core/src/main/scala/kafka/common/InterBrokerSendThread.scala
@@ -51,7 +51,7 @@ abstract class InterBrokerSendThread(name: String,
     awaitShutdown()
   }
 
-  override def doWork() {
+  override def doWork(): Unit = {
     var now = time.milliseconds()
 
     generateRequests().foreach { request =>
diff --git a/core/src/main/scala/kafka/common/MessageFormatter.scala b/core/src/main/scala/kafka/common/MessageFormatter.scala
index ef3c7238b7b11..9f6c3fa94adc5 100644
--- a/core/src/main/scala/kafka/common/MessageFormatter.scala
+++ b/core/src/main/scala/kafka/common/MessageFormatter.scala
@@ -30,10 +30,10 @@ import org.apache.kafka.clients.consumer.ConsumerRecord
   */
 trait MessageFormatter {
 
-  def init(props: Properties) {}
+  def init(props: Properties): Unit = {}
 
   def writeTo(consumerRecord: ConsumerRecord[Array[Byte], Array[Byte]], output: PrintStream): Unit
 
-  def close() {}
+  def close(): Unit = {}
 
 }
diff --git a/core/src/main/scala/kafka/common/MessageReader.scala b/core/src/main/scala/kafka/common/MessageReader.scala
index 56b55ce891f9e..de456e16ae532 100644
--- a/core/src/main/scala/kafka/common/MessageReader.scala
+++ b/core/src/main/scala/kafka/common/MessageReader.scala
@@ -30,10 +30,10 @@ import org.apache.kafka.clients.producer.ProducerRecord
   */
 trait MessageReader {
 
-  def init(inputStream: InputStream, props: Properties) {}
+  def init(inputStream: InputStream, props: Properties): Unit = {}
 
   def readMessage(): ProducerRecord[Array[Byte], Array[Byte]]
 
-  def close() {}
+  def close(): Unit = {}
 
 }
diff --git a/core/src/main/scala/kafka/common/ZkNodeChangeNotificationListener.scala b/core/src/main/scala/kafka/common/ZkNodeChangeNotificationListener.scala
index 9a160e6ccfcde..42341cf72e47b 100644
--- a/core/src/main/scala/kafka/common/ZkNodeChangeNotificationListener.scala
+++ b/core/src/main/scala/kafka/common/ZkNodeChangeNotificationListener.scala
@@ -32,7 +32,7 @@ import scala.util.{Failure, Try}
  * Handle the notificationMessage.
  */
 trait NotificationHandler {
-  def processNotification(notificationMessage: Array[Byte])
+  def processNotification(notificationMessage: Array[Byte]): Unit
 }
 
 /**
@@ -60,7 +60,7 @@ class ZkNodeChangeNotificationListener(private val zkClient: KafkaZkClient,
   private val thread = new ChangeEventProcessThread(s"$seqNodeRoot-event-process-thread")
   private val isClosed = new AtomicBoolean(false)
 
-  def init() {
+  def init(): Unit = {
     zkClient.registerStateChangeHandler(ZkStateChangeHandler)
     zkClient.registerZNodeChildChangeHandler(ChangeNotificationHandler)
     addChangeNotification()
@@ -78,7 +78,7 @@ class ZkNodeChangeNotificationListener(private val zkClient: KafkaZkClient,
   /**
    * Process notifications
    */
-  private def processNotifications() {
+  private def processNotifications(): Unit = {
     try {
       val notifications = zkClient.getChildren(seqNodeRoot).sorted
       if (notifications.nonEmpty) {
@@ -126,7 +126,7 @@ class ZkNodeChangeNotificationListener(private val zkClient: KafkaZkClient,
    * @param now
    * @param notifications
    */
-  private def purgeObsoleteNotifications(now: Long, notifications: Seq[String]) {
+  private def purgeObsoleteNotifications(now: Long, notifications: Seq[String]): Unit = {
     for (notification <- notifications.sorted) {
       val notificationNode = seqNodeRoot + "/" + notification
       val (data, stat) = zkClient.getDataAndStat(notificationNode)
diff --git a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
index 3719a318ff572..7bd494ef028aa 100755
--- a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
+++ b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
@@ -81,7 +81,7 @@ class ControllerChannelManager(controllerContext: ControllerContext,
   }
 
   def sendRequest(brokerId: Int, request: AbstractControlRequest.Builder[_ <: AbstractControlRequest],
-                  callback: AbstractResponse => Unit = null) {
+                  callback: AbstractResponse => Unit = null): Unit = {
     brokerLock synchronized {
       val stateInfoOpt = brokerStateInfo.get(brokerId)
       stateInfoOpt match {
@@ -93,7 +93,7 @@ class ControllerChannelManager(controllerContext: ControllerContext,
     }
   }
 
-  def addBroker(broker: Broker) {
+  def addBroker(broker: Broker): Unit = {
     // be careful here. Maybe the startup() API has already started the request send thread
     brokerLock synchronized {
       if (!brokerStateInfo.contains(broker.id)) {
@@ -103,13 +103,13 @@ class ControllerChannelManager(controllerContext: ControllerContext,
     }
   }
 
-  def removeBroker(brokerId: Int) {
+  def removeBroker(brokerId: Int): Unit = {
     brokerLock synchronized {
       removeExistingBroker(brokerStateInfo(brokerId))
     }
   }
 
-  private def addNewBroker(broker: Broker) {
+  private def addNewBroker(broker: Broker): Unit = {
     val messageQueue = new LinkedBlockingQueue[QueueItem]
     debug(s"Controller ${config.brokerId} trying to connect to broker ${broker.id}")
     val controllerToBrokerListenerName = config.controlPlaneListenerName.getOrElse(config.interBrokerListenerName)
@@ -188,7 +188,7 @@ class ControllerChannelManager(controllerContext: ControllerContext,
 
   private def brokerMetricTags(brokerId: Int) = Map("broker-id" -> brokerId.toString)
 
-  private def removeExistingBroker(brokerState: ControllerBrokerStateInfo) {
+  private def removeExistingBroker(brokerState: ControllerBrokerStateInfo): Unit = {
     try {
       // Shutdown the RequestSendThread before closing the NetworkClient to avoid the concurrent use of the
       // non-threadsafe classes as described in KAFKA-4959.
@@ -206,7 +206,7 @@ class ControllerChannelManager(controllerContext: ControllerContext,
     }
   }
 
-  protected def startRequestSendThread(brokerId: Int) {
+  protected def startRequestSendThread(brokerId: Int): Unit = {
     val requestThread = brokerStateInfo(brokerId).requestSendThread
     if (requestThread.getState == Thread.State.NEW)
       requestThread.start()
@@ -352,7 +352,7 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
                   request: AbstractControlRequest.Builder[_ <: AbstractControlRequest],
                   callback: AbstractResponse => Unit = null): Unit
 
-  def newBatch() {
+  def newBatch(): Unit = {
     // raise error if the previous batch is not empty
     if (leaderAndIsrRequestMap.nonEmpty)
       throw new IllegalStateException("Controller to broker state change requests batch is not empty while creating " +
@@ -366,7 +366,7 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
         s"$updateMetadataRequestPartitionInfoMap might be lost ")
   }
 
-  def clear() {
+  def clear(): Unit = {
     leaderAndIsrRequestMap.clear()
     stopReplicaRequestMap.clear()
     updateMetadataRequestBrokerSet.clear()
@@ -407,7 +407,7 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
   def addUpdateMetadataRequestForBrokers(brokerIds: Seq[Int],
                                          partitions: collection.Set[TopicPartition]): Unit = {
 
-    def updateMetadataRequestPartitionInfo(partition: TopicPartition, beingDeleted: Boolean) {
+    def updateMetadataRequestPartitionInfo(partition: TopicPartition, beingDeleted: Boolean): Unit = {
       val leaderIsrAndControllerEpochOpt = controllerContext.partitionLeadershipInfo.get(partition)
       leaderIsrAndControllerEpochOpt match {
         case Some(l @ LeaderIsrAndControllerEpoch(leaderAndIsr, controllerEpoch)) =>
diff --git a/core/src/main/scala/kafka/controller/ControllerContext.scala b/core/src/main/scala/kafka/controller/ControllerContext.scala
index 38443a7fc3aaa..85a4abab0012a 100644
--- a/core/src/main/scala/kafka/controller/ControllerContext.scala
+++ b/core/src/main/scala/kafka/controller/ControllerContext.scala
@@ -99,7 +99,7 @@ class ControllerContext {
     }.toSet
   }
 
-  def setLiveBrokerAndEpochs(brokerAndEpochs: Map[Broker, Long]) {
+  def setLiveBrokerAndEpochs(brokerAndEpochs: Map[Broker, Long]): Unit = {
     liveBrokers = brokerAndEpochs.keySet
     liveBrokerEpochs =
       brokerAndEpochs map { case (broker, brokerEpoch) => (broker.id, brokerEpoch)}
diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala
index 4ca4b49109ba0..a502af056be8c 100644
--- a/core/src/main/scala/kafka/controller/KafkaController.scala
+++ b/core/src/main/scala/kafka/controller/KafkaController.scala
@@ -239,7 +239,7 @@ class KafkaController(val config: KafkaConfig,
    * If it encounters any unexpected exception/error while becoming controller, it resigns as the current controller.
    * This ensures another controller election will be triggered and there will always be an actively serving controller
    */
-  private def onControllerFailover() {
+  private def onControllerFailover(): Unit = {
     info("Registering handlers")
 
     // before reading source of truth from zookeeper, register the listeners to get broker/topic callbacks
@@ -300,7 +300,7 @@ class KafkaController(val config: KafkaConfig,
    * This callback is invoked by the zookeeper leader elector when the current broker resigns as the controller. This is
    * required to clean up internal controller data structures
    */
-  private def onControllerResignation() {
+  private def onControllerResignation(): Unit = {
     debug("Resigning")
     // de-register listeners
     zkClient.unregisterZNodeChildChangeHandler(isrChangeNotificationHandler.path)
@@ -343,7 +343,7 @@ class KafkaController(val config: KafkaConfig,
    * to all these brokers to query the state of their replicas. Replicas with an offline log directory respond with
    * KAFKA_STORAGE_ERROR, which will be handled by the LeaderAndIsrResponseReceived event.
    */
-  private def onBrokerLogDirFailure(brokerIds: Seq[Int]) {
+  private def onBrokerLogDirFailure(brokerIds: Seq[Int]): Unit = {
     // send LeaderAndIsrRequest for all replicas on those brokers to see if they are still online.
     info(s"Handling log directory failure for brokers ${brokerIds.mkString(",")}")
     val replicasOnBrokers = controllerContext.replicasOnBrokers(brokerIds.toSet)
@@ -364,7 +364,7 @@ class KafkaController(val config: KafkaConfig,
    * 2. Even if we do refresh the cache, there is no guarantee that by the time the leader and ISR request reaches
    *    every broker that it is still valid.  Brokers check the leader epoch to determine validity of the request.
    */
-  private def onBrokerStartup(newBrokers: Seq[Int]) {
+  private def onBrokerStartup(newBrokers: Seq[Int]): Unit = {
     info(s"New broker startup callback for ${newBrokers.mkString(",")}")
     newBrokers.foreach(controllerContext.replicasOnOfflineDirs.remove)
     val newBrokersSet = newBrokers.toSet
@@ -420,7 +420,7 @@ class KafkaController(val config: KafkaConfig,
    * This callback is invoked by the replica state machine's broker change listener with the list of failed brokers
    * as input. It will call onReplicaBecomeOffline(...) with the list of replicas on those failed brokers as input.
    */
-  private def onBrokerFailure(deadBrokers: Seq[Int]) {
+  private def onBrokerFailure(deadBrokers: Seq[Int]): Unit = {
     info(s"Broker failure callback for ${deadBrokers.mkString(",")}")
     deadBrokers.foreach(controllerContext.replicasOnOfflineDirs.remove)
     val deadBrokersThatWereShuttingDown =
@@ -433,7 +433,7 @@ class KafkaController(val config: KafkaConfig,
     unregisterBrokerModificationsHandler(deadBrokers)
   }
 
-  private def onBrokerUpdate(updatedBrokerId: Int) {
+  private def onBrokerUpdate(updatedBrokerId: Int): Unit = {
     info(s"Broker info update callback for $updatedBrokerId")
     sendUpdateMetadataRequest(controllerContext.liveOrShuttingDownBrokerIds.toSeq, Set.empty)
   }
@@ -485,7 +485,7 @@ class KafkaController(val config: KafkaConfig,
    * 1. Move the newly created partitions to the NewPartition state
    * 2. Move the newly created partitions from NewPartition->OnlinePartition state
    */
-  private def onNewPartitionCreation(newPartitions: Set[TopicPartition]) {
+  private def onNewPartitionCreation(newPartitions: Set[TopicPartition]): Unit = {
     info(s"New partition creation callback for ${newPartitions.mkString(",")}")
     partitionStateMachine.handleStateChanges(newPartitions.toSeq, NewPartition)
     replicaStateMachine.handleStateChanges(controllerContext.replicasForPartition(newPartitions).toSeq, NewReplica)
@@ -538,7 +538,7 @@ class KafkaController(val config: KafkaConfig,
    * Note that we have to update AR in ZK with RAR last since it's the only place where we store OAR persistently.
    * This way, if the controller crashes before that step, we can still recover.
    */
-  private def onPartitionReassignment(topicPartition: TopicPartition, reassignedPartitionContext: ReassignedPartitionsContext) {
+  private def onPartitionReassignment(topicPartition: TopicPartition, reassignedPartitionContext: ReassignedPartitionsContext): Unit = {
     val reassignedReplicas = reassignedPartitionContext.newReplicas
     if (!areReplicasInIsr(topicPartition, reassignedReplicas)) {
       info(s"New replicas ${reassignedReplicas.mkString(",")} for partition $topicPartition being reassigned not yet " +
@@ -590,7 +590,7 @@ class KafkaController(val config: KafkaConfig,
    *
    * @throws IllegalStateException if a partition is not in `partitionsBeingReassigned`
    */
-  private def maybeTriggerPartitionReassignment(topicPartitions: Set[TopicPartition]) {
+  private def maybeTriggerPartitionReassignment(topicPartitions: Set[TopicPartition]): Unit = {
     val partitionsToBeRemovedFromReassignment = scala.collection.mutable.Set.empty[TopicPartition]
     topicPartitions.foreach { tp =>
       if (topicDeletionManager.isTopicQueuedUpForDeletion(tp.topic)) {
@@ -687,7 +687,7 @@ class KafkaController(val config: KafkaConfig,
     }
   }
 
-  private def initializeControllerContext() {
+  private def initializeControllerContext(): Unit = {
     // update controller cache with delete topic information
     val curBrokerAndEpochs = zkClient.getAllBrokerAndEpochsInCluster
     controllerContext.setLiveBrokerAndEpochs(curBrokerAndEpochs)
@@ -731,7 +731,7 @@ class KafkaController(val config: KafkaConfig,
     pendingPreferredReplicaElections
   }
 
-  private def initializePartitionReassignment() {
+  private def initializePartitionReassignment(): Unit = {
     // read the partitions being reassigned from zookeeper path /admin/reassign_partitions
     val partitionsBeingReassigned = zkClient.getPartitionReassignment
     info(s"Partitions being reassigned: $partitionsBeingReassigned")
@@ -755,7 +755,7 @@ class KafkaController(val config: KafkaConfig,
     (topicsToBeDeleted, topicsIneligibleForDeletion)
   }
 
-  private def updateLeaderAndIsrCache(partitions: Seq[TopicPartition] = controllerContext.allPartitions.toSeq) {
+  private def updateLeaderAndIsrCache(partitions: Seq[TopicPartition] = controllerContext.allPartitions.toSeq): Unit = {
     val leaderIsrAndControllerEpochs = zkClient.getTopicPartitionStates(partitions)
     leaderIsrAndControllerEpochs.foreach { case (partition, leaderIsrAndControllerEpoch) =>
       controllerContext.partitionLeadershipInfo.put(partition, leaderIsrAndControllerEpoch)
@@ -769,7 +769,7 @@ class KafkaController(val config: KafkaConfig,
   }
 
   private def moveReassignedPartitionLeaderIfRequired(topicPartition: TopicPartition,
-                                                      reassignedPartitionContext: ReassignedPartitionsContext) {
+                                                      reassignedPartitionContext: ReassignedPartitionsContext): Unit = {
     val reassignedReplicas = reassignedPartitionContext.newReplicas
     val currentLeader = controllerContext.partitionLeadershipInfo(topicPartition).leaderAndIsr.leader
     // change the assigned replica list to just the reassigned replicas in the cache so it gets sent out on the LeaderAndIsr
@@ -798,7 +798,7 @@ class KafkaController(val config: KafkaConfig,
 
   private def stopOldReplicasOfReassignedPartition(topicPartition: TopicPartition,
                                                    reassignedPartitionContext: ReassignedPartitionsContext,
-                                                   oldReplicas: Set[Int]) {
+                                                   oldReplicas: Set[Int]): Unit = {
     // first move the replica to offline state (the controller removes it from the ISR)
     val replicasToBeDeleted = oldReplicas.map(PartitionAndReplica(topicPartition, _))
     replicaStateMachine.handleStateChanges(replicasToBeDeleted.toSeq, OfflineReplica)
@@ -810,7 +810,7 @@ class KafkaController(val config: KafkaConfig,
   }
 
   private def updateAssignedReplicasForPartition(partition: TopicPartition,
-                                                 replicas: Seq[Int]) {
+                                                 replicas: Seq[Int]): Unit = {
     controllerContext.updatePartitionReplicaAssignment(partition, replicas)
     val setDataResponse = zkClient.setTopicAssignmentRaw(partition.topic, controllerContext.partitionReplicaAssignmentForTopic(partition.topic), controllerContext.epochZkVersion)
     setDataResponse.resultCode match {
@@ -825,7 +825,7 @@ class KafkaController(val config: KafkaConfig,
 
   private def startNewReplicasForReassignedPartition(topicPartition: TopicPartition,
                                                      reassignedPartitionContext: ReassignedPartitionsContext,
-                                                     newReplicas: Set[Int]) {
+                                                     newReplicas: Set[Int]): Unit = {
     // send the start replica request to the brokers in the reassigned replicas list that are not in the assigned
     // replicas list
     newReplicas.foreach { replica =>
@@ -833,7 +833,7 @@ class KafkaController(val config: KafkaConfig,
     }
   }
 
-  private def updateLeaderEpochAndSendRequest(partition: TopicPartition, replicasToReceiveRequest: Seq[Int], newAssignedReplicas: Seq[Int]) {
+  private def updateLeaderEpochAndSendRequest(partition: TopicPartition, replicasToReceiveRequest: Seq[Int], newAssignedReplicas: Seq[Int]): Unit = {
     val stateChangeLog = stateChangeLogger.withControllerEpoch(controllerContext.epoch)
     updateLeaderEpoch(partition) match {
       case Some(updatedLeaderIsrAndControllerEpoch) =>
@@ -869,7 +869,7 @@ class KafkaController(val config: KafkaConfig,
     }
   }
 
-  private def unregisterPartitionReassignmentIsrChangeHandlers() {
+  private def unregisterPartitionReassignmentIsrChangeHandlers(): Unit = {
     controllerContext.partitionsBeingReassigned.values.foreach(_.unregisterReassignIsrChangeHandler(zkClient))
   }
 
@@ -881,7 +881,7 @@ class KafkaController(val config: KafkaConfig,
    * `ControllerContext.partitionsBeingReassigned` must be populated with all partitions being reassigned before this
    * method is invoked to avoid premature deletion of the `reassign_partitions` znode.
    */
-  private def removePartitionsFromReassignedPartitions(partitionsToBeRemoved: Set[TopicPartition]) {
+  private def removePartitionsFromReassignedPartitions(partitionsToBeRemoved: Set[TopicPartition]): Unit = {
     partitionsToBeRemoved.map(controllerContext.partitionsBeingReassigned).foreach { reassignContext =>
       reassignContext.unregisterReassignIsrChangeHandler(zkClient)
     }
@@ -908,7 +908,7 @@ class KafkaController(val config: KafkaConfig,
   }
 
   private def removePartitionsFromPreferredReplicaElection(partitionsToBeRemoved: Set[TopicPartition],
-                                                           isTriggeredByAutoRebalance : Boolean) {
+                                                           isTriggeredByAutoRebalance : Boolean): Unit = {
     for (partition <- partitionsToBeRemoved) {
       // check the status
       val currentLeader = controllerContext.partitionLeadershipInfo(partition).leaderAndIsr.leader
@@ -933,7 +933,7 @@ class KafkaController(val config: KafkaConfig,
    *
    * @param brokers The brokers that the update metadata request should be sent to
    */
-  private[controller] def sendUpdateMetadataRequest(brokers: Seq[Int], partitions: Set[TopicPartition]) {
+  private[controller] def sendUpdateMetadataRequest(brokers: Seq[Int], partitions: Set[TopicPartition]): Unit = {
     try {
       brokerRequestBatch.newBatch()
       brokerRequestBatch.addUpdateMetadataRequestForBrokers(brokers, partitions)
@@ -1471,7 +1471,7 @@ class KafkaController(val config: KafkaConfig,
   }
 
   private def processIsrChangeNotification(): Unit = {
-    def processUpdateNotifications(partitions: Seq[TopicPartition]) {
+    def processUpdateNotifications(partitions: Seq[TopicPartition]): Unit = {
       val liveBrokers: Seq[Int] = controllerContext.liveOrShuttingDownBrokerIds.toSeq
       debug(s"Sending MetadataRequest to Brokers: $liveBrokers for TopicPartitions: $partitions")
       sendUpdateMetadataRequest(liveBrokers, partitions.toSet)
diff --git a/core/src/main/scala/kafka/controller/PartitionStateMachine.scala b/core/src/main/scala/kafka/controller/PartitionStateMachine.scala
index 8669f736880aa..94e8b7e0aa35e 100755
--- a/core/src/main/scala/kafka/controller/PartitionStateMachine.scala
+++ b/core/src/main/scala/kafka/controller/PartitionStateMachine.scala
@@ -34,7 +34,7 @@ abstract class PartitionStateMachine(controllerContext: ControllerContext) exten
   /**
    * Invoked on successful controller election.
    */
-  def startup() {
+  def startup(): Unit = {
     info("Initializing partition state")
     initializePartitionState()
     info("Triggering online partition state changes")
@@ -45,7 +45,7 @@ abstract class PartitionStateMachine(controllerContext: ControllerContext) exten
   /**
    * Invoked on controller shutdown.
    */
-  def shutdown() {
+  def shutdown(): Unit = {
     info("Stopped partition state machine")
   }
 
@@ -79,7 +79,7 @@ abstract class PartitionStateMachine(controllerContext: ControllerContext) exten
    * Invoked on startup of the partition's state machine to set the initial state for all existing partitions in
    * zookeeper
    */
-  private def initializePartitionState() {
+  private def initializePartitionState(): Unit = {
     for (topicPartition <- controllerContext.allPartitions) {
       // check if leader and isr path exists for partition. If not, then it is in NEW state
       controllerContext.partitionLeadershipInfo.get(topicPartition) match {
diff --git a/core/src/main/scala/kafka/controller/ReplicaStateMachine.scala b/core/src/main/scala/kafka/controller/ReplicaStateMachine.scala
index e74f4dc28959e..610315498194d 100644
--- a/core/src/main/scala/kafka/controller/ReplicaStateMachine.scala
+++ b/core/src/main/scala/kafka/controller/ReplicaStateMachine.scala
@@ -32,7 +32,7 @@ abstract class ReplicaStateMachine(controllerContext: ControllerContext) extends
   /**
    * Invoked on successful controller election.
    */
-  def startup() {
+  def startup(): Unit = {
     info("Initializing replica state")
     initializeReplicaState()
     info("Triggering online replica state changes")
@@ -46,7 +46,7 @@ abstract class ReplicaStateMachine(controllerContext: ControllerContext) extends
   /**
    * Invoked on controller shutdown.
    */
-  def shutdown() {
+  def shutdown(): Unit = {
     info("Stopped replica state machine")
   }
 
@@ -54,7 +54,7 @@ abstract class ReplicaStateMachine(controllerContext: ControllerContext) extends
    * Invoked on startup of the replica's state machine to set the initial state for replicas of all existing partitions
    * in zookeeper
    */
-  private def initializeReplicaState() {
+  private def initializeReplicaState(): Unit = {
     controllerContext.allPartitions.foreach { partition =>
       val replicas = controllerContext.partitionReplicaAssignment(partition)
       replicas.foreach { replicaId =>
diff --git a/core/src/main/scala/kafka/controller/TopicDeletionManager.scala b/core/src/main/scala/kafka/controller/TopicDeletionManager.scala
index b15b795ff54de..d032b3b499dbf 100755
--- a/core/src/main/scala/kafka/controller/TopicDeletionManager.scala
+++ b/core/src/main/scala/kafka/controller/TopicDeletionManager.scala
@@ -117,7 +117,7 @@ class TopicDeletionManager(config: KafkaConfig,
    * i.e. all replicas of all partitions of that topic are deleted successfully.
    * @param topics Topics that should be deleted
    */
-  def enqueueTopicsForDeletion(topics: Set[String]) {
+  def enqueueTopicsForDeletion(topics: Set[String]): Unit = {
     if (isDeleteTopicEnabled) {
       controllerContext.queueTopicDeletion(topics)
       resumeDeletions()
@@ -130,7 +130,7 @@ class TopicDeletionManager(config: KafkaConfig,
    * 2. Partition reassignment completes. Any partitions belonging to topics queued up for deletion finished reassignment
    * @param topics Topics for which deletion can be resumed
    */
-  def resumeDeletionForTopics(topics: Set[String] = Set.empty) {
+  def resumeDeletionForTopics(topics: Set[String] = Set.empty): Unit = {
     if (isDeleteTopicEnabled) {
       val topicsToResumeDeletion = topics & controllerContext.topicsToBeDeleted
       if (topicsToResumeDeletion.nonEmpty) {
@@ -147,7 +147,7 @@ class TopicDeletionManager(config: KafkaConfig,
    * ineligible for deletion until further notice.
    * @param replicas Replicas for which deletion has failed
    */
-  def failReplicaDeletion(replicas: Set[PartitionAndReplica]) {
+  def failReplicaDeletion(replicas: Set[PartitionAndReplica]): Unit = {
     if (isDeleteTopicEnabled) {
       val replicasThatFailedToDelete = replicas.filter(r => isTopicQueuedUpForDeletion(r.topic))
       if (replicasThatFailedToDelete.nonEmpty) {
@@ -202,7 +202,7 @@ class TopicDeletionManager(config: KafkaConfig,
    * the topic if all replicas of a topic have been successfully deleted
    * @param replicas Replicas that were successfully deleted by the broker
    */
-  def completeReplicaDeletion(replicas: Set[PartitionAndReplica]) {
+  def completeReplicaDeletion(replicas: Set[PartitionAndReplica]): Unit = {
     val successfullyDeletedReplicas = replicas.filter(r => isTopicQueuedUpForDeletion(r.topic))
     debug(s"Deletion successfully completed for replicas ${successfullyDeletedReplicas.mkString(",")}")
     replicaStateMachine.handleStateChanges(successfullyDeletedReplicas.toSeq, ReplicaDeletionSuccessful)
@@ -235,7 +235,7 @@ class TopicDeletionManager(config: KafkaConfig,
     replicaStateMachine.handleStateChanges(failedReplicas.toSeq, OfflineReplica)
   }
 
-  private def completeDeleteTopic(topic: String) {
+  private def completeDeleteTopic(topic: String): Unit = {
     // deregister partition change listener on the deleted topic. This is to prevent the partition change listener
     // firing before the new topic listener when a deleted topic gets auto created
     client.mutePartitionModifications(topic)
@@ -255,7 +255,7 @@ class TopicDeletionManager(config: KafkaConfig,
    * {@link LeaderAndIsr#LeaderDuringDelete}. This lets each broker know that this topic is being deleted and can be
    * removed from their caches.
    */
-  private def onTopicDeletion(topics: Set[String]) {
+  private def onTopicDeletion(topics: Set[String]): Unit = {
     info(s"Topic deletion callback for ${topics.mkString(",")}")
     // send update metadata so that brokers stop serving data for topics to be deleted
     val partitions = topics.flatMap(controllerContext.partitionsForTopic)
@@ -291,7 +291,7 @@ class TopicDeletionManager(config: KafkaConfig,
    * 2. Move all alive replicas to ReplicaDeletionStarted state so they can be deleted successfully
    * @param replicasForTopicsToBeDeleted
    */
-  private def startReplicaDeletion(replicasForTopicsToBeDeleted: Set[PartitionAndReplica]) {
+  private def startReplicaDeletion(replicasForTopicsToBeDeleted: Set[PartitionAndReplica]): Unit = {
     replicasForTopicsToBeDeleted.groupBy(_.topic).keys.foreach { topic =>
       val aliveReplicasForTopic = controllerContext.allLiveReplicas().filter(p => p.topic == topic)
       val deadReplicasForTopic = replicasForTopicsToBeDeleted -- aliveReplicasForTopic
@@ -321,7 +321,7 @@ class TopicDeletionManager(config: KafkaConfig,
    * 3. Move all replicas to ReplicaDeletionStarted state. This will send StopReplicaRequest with deletePartition=true. And
    *    will delete all persistent data from all replicas of the respective partitions
    */
-  private def onPartitionDeletion(partitionsToBeDeleted: Set[TopicPartition]) {
+  private def onPartitionDeletion(partitionsToBeDeleted: Set[TopicPartition]): Unit = {
     info(s"Partition deletion callback for ${partitionsToBeDeleted.mkString(",")}")
     val replicasPerPartition = controllerContext.replicasForPartition(partitionsToBeDeleted)
     startReplicaDeletion(replicasPerPartition)
diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
index 78749460cb261..9e52e516b05b8 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
@@ -83,7 +83,7 @@ class GroupCoordinator(val brokerId: Int,
   /**
    * Startup logic executed at the same time when the server starts up.
    */
-  def startup(enableMetadataExpiration: Boolean = true) {
+  def startup(enableMetadataExpiration: Boolean = true): Unit = {
     info("Starting up.")
     groupManager.startup(enableMetadataExpiration)
     isActive.set(true)
@@ -94,7 +94,7 @@ class GroupCoordinator(val brokerId: Int,
    * Shutdown logic executed at the same time when server shuts down.
    * Ordering of actions should be reversed from the startup process.
    */
-  def shutdown() {
+  def shutdown(): Unit = {
     info("Shutting down.")
     isActive.set(false)
     groupManager.shutdown()
@@ -242,7 +242,7 @@ class GroupCoordinator(val brokerId: Int,
                           sessionTimeoutMs: Int,
                           protocolType: String,
                           protocols: List[(String, Array[Byte])],
-                          responseCallback: JoinCallback) {
+                          responseCallback: JoinCallback): Unit = {
     group.inLock {
       if (group.is(Dead)) {
         // if the group is marked as dead, it means some other thread has just removed the group
@@ -358,7 +358,7 @@ class GroupCoordinator(val brokerId: Int,
                           memberId: String,
                           groupInstanceId: Option[String],
                           groupAssignment: Map[String, Array[Byte]],
-                          responseCallback: SyncCallback) {
+                          responseCallback: SyncCallback): Unit = {
     group.inLock {
       if (group.is(Dead)) {
         // if the group is marked as dead, it means some other thread has just removed the group
@@ -424,7 +424,7 @@ class GroupCoordinator(val brokerId: Int,
 
   def handleLeaveGroup(groupId: String,
                        leavingMembers: List[MemberIdentity],
-                       responseCallback: LeaveGroupResult => Unit) {
+                       responseCallback: LeaveGroupResult => Unit): Unit = {
     validateGroupStatus(groupId, ApiKeys.LEAVE_GROUP) match {
       case Some(error) =>
         responseCallback(leaveError(error, List.empty))
@@ -523,7 +523,7 @@ class GroupCoordinator(val brokerId: Int,
                       memberId: String,
                       groupInstanceId: Option[String],
                       generationId: Int,
-                      responseCallback: Errors => Unit) {
+                      responseCallback: Errors => Unit): Unit = {
     validateGroupStatus(groupId, ApiKeys.HEARTBEAT).foreach { error =>
       if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS)
         // the group is still loading, so respond just blindly
@@ -596,7 +596,7 @@ class GroupCoordinator(val brokerId: Int,
                           groupInstanceId: Option[String],
                           generationId: Int,
                           offsetMetadata: immutable.Map[TopicPartition, OffsetAndMetadata],
-                          responseCallback: immutable.Map[TopicPartition, Errors] => Unit) {
+                          responseCallback: immutable.Map[TopicPartition, Errors] => Unit): Unit = {
     validateGroupStatus(groupId, ApiKeys.OFFSET_COMMIT) match {
       case Some(error) => responseCallback(offsetMetadata.map { case (k, _) => k -> error })
       case None =>
@@ -621,7 +621,7 @@ class GroupCoordinator(val brokerId: Int,
 
   def scheduleHandleTxnCompletion(producerId: Long,
                                   offsetsPartitions: Iterable[TopicPartition],
-                                  transactionResult: TransactionResult) {
+                                  transactionResult: TransactionResult): Unit = {
     require(offsetsPartitions.forall(_.topic == Topic.GROUP_METADATA_TOPIC_NAME))
     val isCommit = transactionResult == TransactionResult.COMMIT
     groupManager.scheduleHandleTxnCompletion(producerId, offsetsPartitions.map(_.partition).toSet, isCommit)
@@ -634,7 +634,7 @@ class GroupCoordinator(val brokerId: Int,
                               producerId: Long,
                               producerEpoch: Short,
                               offsetMetadata: immutable.Map[TopicPartition, OffsetAndMetadata],
-                              responseCallback: immutable.Map[TopicPartition, Errors] => Unit) {
+                              responseCallback: immutable.Map[TopicPartition, Errors] => Unit): Unit = {
     group.inLock {
       if (group.is(Dead)) {
         // if the group is marked as dead, it means some other thread has just removed the group
@@ -710,7 +710,7 @@ class GroupCoordinator(val brokerId: Int,
     }
   }
 
-  def handleDeletedPartitions(topicPartitions: Seq[TopicPartition]) {
+  def handleDeletedPartitions(topicPartitions: Seq[TopicPartition]): Unit = {
     val offsetsRemoved = groupManager.cleanupGroupMetadata(groupManager.currentGroups, group => {
       group.removeOffsets(topicPartitions)
     })
@@ -745,7 +745,7 @@ class GroupCoordinator(val brokerId: Int,
       None
   }
 
-  private def onGroupUnloaded(group: GroupMetadata) {
+  private def onGroupUnloaded(group: GroupMetadata): Unit = {
     group.inLock {
       info(s"Unloading group metadata for ${group.groupId} with generation ${group.generationId}")
       val previousState = group.currentState
@@ -769,7 +769,7 @@ class GroupCoordinator(val brokerId: Int,
     }
   }
 
-  private def onGroupLoaded(group: GroupMetadata) {
+  private def onGroupLoaded(group: GroupMetadata): Unit = {
     group.inLock {
       info(s"Loading group metadata for ${group.groupId} with generation ${group.generationId}")
       assert(group.is(Stable) || group.is(Empty))
@@ -781,27 +781,27 @@ class GroupCoordinator(val brokerId: Int,
     }
   }
 
-  def handleGroupImmigration(offsetTopicPartitionId: Int) {
+  def handleGroupImmigration(offsetTopicPartitionId: Int): Unit = {
     groupManager.scheduleLoadGroupAndOffsets(offsetTopicPartitionId, onGroupLoaded)
   }
 
-  def handleGroupEmigration(offsetTopicPartitionId: Int) {
+  def handleGroupEmigration(offsetTopicPartitionId: Int): Unit = {
     groupManager.removeGroupsForPartition(offsetTopicPartitionId, onGroupUnloaded)
   }
 
-  private def setAndPropagateAssignment(group: GroupMetadata, assignment: Map[String, Array[Byte]]) {
+  private def setAndPropagateAssignment(group: GroupMetadata, assignment: Map[String, Array[Byte]]): Unit = {
     assert(group.is(CompletingRebalance))
     group.allMemberMetadata.foreach(member => member.assignment = assignment(member.memberId))
     propagateAssignment(group, Errors.NONE)
   }
 
-  private def resetAndPropagateAssignmentError(group: GroupMetadata, error: Errors) {
+  private def resetAndPropagateAssignmentError(group: GroupMetadata, error: Errors): Unit = {
     assert(group.is(CompletingRebalance))
     group.allMemberMetadata.foreach(_.assignment = Array.empty)
     propagateAssignment(group, error)
   }
 
-  private def propagateAssignment(group: GroupMetadata, error: Errors) {
+  private def propagateAssignment(group: GroupMetadata, error: Errors): Unit = {
     for (member <- group.allMemberMetadata) {
       if (group.maybeInvokeSyncCallback(member, SyncGroupResult(member.assignment, error))) {
         // reset the session timeout for members after propagating the member's assignment.
@@ -816,11 +816,11 @@ class GroupCoordinator(val brokerId: Int,
   /**
    * Complete existing DelayedHeartbeats for the given member and schedule the next one
    */
-  private def completeAndScheduleNextHeartbeatExpiration(group: GroupMetadata, member: MemberMetadata) {
+  private def completeAndScheduleNextHeartbeatExpiration(group: GroupMetadata, member: MemberMetadata): Unit = {
     completeAndScheduleNextExpiration(group, member, member.sessionTimeoutMs)
   }
 
-  private def completeAndScheduleNextExpiration(group: GroupMetadata, member: MemberMetadata, timeoutMs: Long) {
+  private def completeAndScheduleNextExpiration(group: GroupMetadata, member: MemberMetadata, timeoutMs: Long): Unit = {
     // complete current heartbeat expectation
     member.latestHeartbeat = time.milliseconds()
     val memberKey = MemberKey(member.groupId, member.memberId)
@@ -835,14 +835,14 @@ class GroupCoordinator(val brokerId: Int,
   /**
     * Add pending member expiration to heartbeat purgatory
     */
-  private def addPendingMemberExpiration(group: GroupMetadata, pendingMemberId: String, timeoutMs: Long) {
+  private def addPendingMemberExpiration(group: GroupMetadata, pendingMemberId: String, timeoutMs: Long): Unit = {
     val pendingMemberKey = MemberKey(group.groupId, pendingMemberId)
     val deadline = time.milliseconds() + timeoutMs
     val delayedHeartbeat = new DelayedHeartbeat(this, group, pendingMemberId, isPending = true, deadline, timeoutMs)
     heartbeatPurgatory.tryCompleteElseWatch(delayedHeartbeat, Seq(pendingMemberKey))
   }
 
-  private def removeHeartbeatForLeavingMember(group: GroupMetadata, member: MemberMetadata) {
+  private def removeHeartbeatForLeavingMember(group: GroupMetadata, member: MemberMetadata): Unit = {
     member.isLeaving = true
     val memberKey = MemberKey(member.groupId, member.memberId)
     heartbeatPurgatory.checkAndComplete(memberKey)
@@ -857,7 +857,7 @@ class GroupCoordinator(val brokerId: Int,
                                     protocolType: String,
                                     protocols: List[(String, Array[Byte])],
                                     group: GroupMetadata,
-                                    callback: JoinCallback) {
+                                    callback: JoinCallback): Unit = {
     val member = new MemberMetadata(memberId, group.groupId, groupInstanceId,
       clientId, clientHost, rebalanceTimeoutMs,
       sessionTimeoutMs, protocolType, protocols)
@@ -888,19 +888,19 @@ class GroupCoordinator(val brokerId: Int,
   private def updateMemberAndRebalance(group: GroupMetadata,
                                        member: MemberMetadata,
                                        protocols: List[(String, Array[Byte])],
-                                       callback: JoinCallback) {
+                                       callback: JoinCallback): Unit = {
     group.updateMember(member, protocols, callback)
     maybePrepareRebalance(group, s"Updating metadata for member ${member.memberId}")
   }
 
-  private def maybePrepareRebalance(group: GroupMetadata, reason: String) {
+  private def maybePrepareRebalance(group: GroupMetadata, reason: String): Unit = {
     group.inLock {
       if (group.canRebalance)
         prepareRebalance(group, reason)
     }
   }
 
-  private def prepareRebalance(group: GroupMetadata, reason: String) {
+  private def prepareRebalance(group: GroupMetadata, reason: String): Unit = {
     // if any members are awaiting sync, cancel their request and have them rejoin
     if (group.is(CompletingRebalance))
       resetAndPropagateAssignmentError(group, Errors.REBALANCE_IN_PROGRESS)
@@ -924,7 +924,7 @@ class GroupCoordinator(val brokerId: Int,
     joinPurgatory.tryCompleteElseWatch(delayedRebalance, Seq(groupKey))
   }
 
-  private def removeMemberAndUpdateGroup(group: GroupMetadata, member: MemberMetadata, reason: String) {
+  private def removeMemberAndUpdateGroup(group: GroupMetadata, member: MemberMetadata, reason: String): Unit = {
     // New members may timeout with a pending JoinGroup while the group is still rebalancing, so we have
     // to invoke the callback before removing the member. We return UNKNOWN_MEMBER_ID so that the consumer
     // will retry the JoinGroup request if is still active.
@@ -940,7 +940,7 @@ class GroupCoordinator(val brokerId: Int,
     }
   }
 
-  private def removePendingMemberAndUpdateGroup(group: GroupMetadata, memberId: String) {
+  private def removePendingMemberAndUpdateGroup(group: GroupMetadata, memberId: String): Unit = {
     group.removePendingMember(memberId)
 
     if (group.is(PreparingRebalance)) {
@@ -956,11 +956,11 @@ class GroupCoordinator(val brokerId: Int,
     }
   }
 
-  def onExpireJoin() {
+  def onExpireJoin(): Unit = {
     // TODO: add metrics for restabilize timeouts
   }
 
-  def onCompleteJoin(group: GroupMetadata) {
+  def onCompleteJoin(group: GroupMetadata): Unit = {
     group.inLock {
       // remove dynamic members who haven't joined the group yet
       group.notYetRejoinedMembers.filterNot(_.isStaticMember) foreach { failedMember =>
@@ -1038,7 +1038,7 @@ class GroupCoordinator(val brokerId: Int,
     }
   }
 
-  def onExpireHeartbeat(group: GroupMetadata, memberId: String, isPending: Boolean, heartbeatDeadline: Long) {
+  def onExpireHeartbeat(group: GroupMetadata, memberId: String, isPending: Boolean, heartbeatDeadline: Long): Unit = {
     group.inLock {
       if (isPending) {
         info(s"Pending member $memberId in group ${group.groupId} has been removed after session timeout expiration.")
@@ -1055,7 +1055,7 @@ class GroupCoordinator(val brokerId: Int,
     }
   }
 
-  def onCompleteHeartbeat() {
+  def onCompleteHeartbeat(): Unit = {
     // TODO: add metrics for complete heartbeats
   }
 
diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala
index 4d283bc8d62e8..ca42b0f098c0b 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala
@@ -219,7 +219,7 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
   def protocolOrNull: String = protocol.orNull
   def currentStateTimestampOrDefault: Long = currentStateTimestamp.getOrElse(-1)
 
-  def add(member: MemberMetadata, callback: JoinCallback = null) {
+  def add(member: MemberMetadata, callback: JoinCallback = null): Unit = {
     if (members.isEmpty)
       this.protocolType = Some(member.protocolType)
 
@@ -236,7 +236,7 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
       numMembersAwaitingJoin += 1
   }
 
-  def remove(memberId: String) {
+  def remove(memberId: String): Unit = {
     members.remove(memberId).foreach { member =>
       member.supportedProtocols.foreach{ case (protocol, _) => supportedProtocols(protocol) -= 1 }
       if (member.isAwaitingJoin)
@@ -387,7 +387,7 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
 
   def canRebalance = GroupMetadata.validPreviousStates(PreparingRebalance).contains(state)
 
-  def transitionTo(groupState: GroupState) {
+  def transitionTo(groupState: GroupState): Unit = {
     assertValidTransition(groupState)
     state = groupState
     currentStateTimestamp = Some(time.milliseconds())
@@ -504,12 +504,12 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
   }
 
   def initializeOffsets(offsets: collection.Map[TopicPartition, CommitRecordMetadataAndOffset],
-                        pendingTxnOffsets: Map[Long, mutable.Map[TopicPartition, CommitRecordMetadataAndOffset]]) {
+                        pendingTxnOffsets: Map[Long, mutable.Map[TopicPartition, CommitRecordMetadataAndOffset]]): Unit = {
     this.offsets ++= offsets
     this.pendingTransactionalOffsetCommits ++= pendingTxnOffsets
   }
 
-  def onOffsetCommitAppend(topicPartition: TopicPartition, offsetWithCommitRecordMetadata: CommitRecordMetadataAndOffset) {
+  def onOffsetCommitAppend(topicPartition: TopicPartition, offsetWithCommitRecordMetadata: CommitRecordMetadataAndOffset): Unit = {
     if (pendingOffsetCommits.contains(topicPartition)) {
       if (offsetWithCommitRecordMetadata.appendedBatchOffset.isEmpty)
         throw new IllegalStateException("Cannot complete offset commit write without providing the metadata of the record " +
@@ -534,12 +534,12 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
     }
   }
 
-  def prepareOffsetCommit(offsets: Map[TopicPartition, OffsetAndMetadata]) {
+  def prepareOffsetCommit(offsets: Map[TopicPartition, OffsetAndMetadata]): Unit = {
     receivedConsumerOffsetCommits = true
     pendingOffsetCommits ++= offsets
   }
 
-  def prepareTxnOffsetCommit(producerId: Long, offsets: Map[TopicPartition, OffsetAndMetadata]) {
+  def prepareTxnOffsetCommit(producerId: Long, offsets: Map[TopicPartition, OffsetAndMetadata]): Unit = {
     trace(s"TxnOffsetCommit for producer $producerId and group $groupId with offsets $offsets is pending")
     receivedTransactionalOffsetCommits = true
     val producerOffsets = pendingTransactionalOffsetCommits.getOrElseUpdate(producerId,
@@ -571,7 +571,7 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
   }
 
   def onTxnOffsetCommitAppend(producerId: Long, topicPartition: TopicPartition,
-                              commitRecordMetadataAndOffset: CommitRecordMetadataAndOffset) {
+                              commitRecordMetadataAndOffset: CommitRecordMetadataAndOffset): Unit = {
     pendingTransactionalOffsetCommits.get(producerId) match {
       case Some(pendingOffset) =>
         if (pendingOffset.contains(topicPartition)
@@ -688,7 +688,7 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
 
   def hasOffsets = offsets.nonEmpty || pendingOffsetCommits.nonEmpty || pendingTransactionalOffsetCommits.nonEmpty
 
-  private def assertValidTransition(targetState: GroupState) {
+  private def assertValidTransition(targetState: GroupState): Unit = {
     if (!GroupMetadata.validPreviousStates(targetState).contains(state))
       throw new IllegalStateException("Group %s should be in the %s states before moving to %s state. Instead it is in %s state"
         .format(groupId, GroupMetadata.validPreviousStates(targetState).mkString(","), targetState, state))
diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
index 7d8499f1dc99a..330ccd0668463 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
@@ -149,7 +149,7 @@ class GroupMetadataManager(brokerId: Int,
       })
     })
 
-  def startup(enableMetadataExpiration: Boolean) {
+  def startup(enableMetadataExpiration: Boolean): Unit = {
     scheduler.startup()
     if (enableMetadataExpiration) {
       scheduler.schedule(name = "delete-expired-group-metadata",
@@ -230,7 +230,7 @@ class GroupMetadataManager(brokerId: Int,
         val generationId = group.generationId
 
         // set the callback function to insert the created group into cache after log append completed
-        def putCacheCallback(responseStatus: Map[TopicPartition, PartitionResponse]) {
+        def putCacheCallback(responseStatus: Map[TopicPartition, PartitionResponse]): Unit = {
           // the append response should only contain the topics partition
           if (responseStatus.size != 1 || !responseStatus.contains(groupMetadataPartition))
             throw new IllegalStateException("Append status %s should only have one partition %s"
@@ -353,7 +353,7 @@ class GroupMetadataManager(brokerId: Int,
           val entries = Map(offsetTopicPartition -> builder.build())
 
           // set the callback function to insert offsets into cache after log append completed
-          def putCacheCallback(responseStatus: Map[TopicPartition, PartitionResponse]) {
+          def putCacheCallback(responseStatus: Map[TopicPartition, PartitionResponse]): Unit = {
             // the append response should only contain the topics partition
             if (responseStatus.size != 1 || !responseStatus.contains(offsetTopicPartition))
               throw new IllegalStateException("Append status %s should only have one partition %s"
@@ -497,7 +497,7 @@ class GroupMetadataManager(brokerId: Int,
   /**
    * Asynchronously read the partition from the offsets topic and populate the cache
    */
-  def scheduleLoadGroupAndOffsets(offsetsPartition: Int, onGroupLoaded: GroupMetadata => Unit) {
+  def scheduleLoadGroupAndOffsets(offsetsPartition: Int, onGroupLoaded: GroupMetadata => Unit): Unit = {
     val topicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, offsetsPartition)
     if (addLoadingPartition(offsetsPartition)) {
       info(s"Scheduling loading of offsets and group metadata from $topicPartition")
@@ -507,7 +507,7 @@ class GroupMetadataManager(brokerId: Int,
     }
   }
 
-  private[group] def loadGroupsAndOffsets(topicPartition: TopicPartition, onGroupLoaded: GroupMetadata => Unit) {
+  private[group] def loadGroupsAndOffsets(topicPartition: TopicPartition, onGroupLoaded: GroupMetadata => Unit): Unit = {
     try {
       val startMs = time.milliseconds()
       doLoadGroupsAndOffsets(topicPartition, onGroupLoaded)
@@ -525,7 +525,7 @@ class GroupMetadataManager(brokerId: Int,
     }
   }
 
-  private def doLoadGroupsAndOffsets(topicPartition: TopicPartition, onGroupLoaded: GroupMetadata => Unit) {
+  private def doLoadGroupsAndOffsets(topicPartition: TopicPartition, onGroupLoaded: GroupMetadata => Unit): Unit = {
     def logEndOffset: Long = replicaManager.getLogEndOffset(topicPartition).getOrElse(-1L)
 
     replicaManager.getLog(topicPartition) match {
@@ -706,12 +706,12 @@ class GroupMetadataManager(brokerId: Int,
    * @param offsetsPartition Groups belonging to this partition of the offsets topic will be deleted from the cache.
    */
   def removeGroupsForPartition(offsetsPartition: Int,
-                               onGroupUnloaded: GroupMetadata => Unit) {
+                               onGroupUnloaded: GroupMetadata => Unit): Unit = {
     val topicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, offsetsPartition)
     info(s"Scheduling unloading of offsets and group metadata from $topicPartition")
     scheduler.schedule(topicPartition.toString, () => removeGroupsAndOffsets)
 
-    def removeGroupsAndOffsets() {
+    def removeGroupsAndOffsets(): Unit = {
       var numOffsetsRemoved = 0
       var numGroupsRemoved = 0
 
@@ -881,7 +881,7 @@ class GroupMetadataManager(brokerId: Int,
   }
 
 
-  def shutdown() {
+  def shutdown(): Unit = {
     shuttingDown.set(true)
     if (scheduler.isStarted)
       scheduler.shutdown()
@@ -911,7 +911,7 @@ class GroupMetadataManager(brokerId: Int,
    *
    * NOTE: this is for test only
    */
-  private[group] def addPartitionOwnership(partition: Int) {
+  private[group] def addPartitionOwnership(partition: Int): Unit = {
     inLock(partitionLock) {
       ownedPartitions.add(partition)
     }
@@ -1402,7 +1402,7 @@ object GroupMetadataManager {
   // Formatter for use with tools such as console consumer: Consumer should also set exclude.internal.topics to false.
   // (specify --formatter "kafka.coordinator.group.GroupMetadataManager\$OffsetsMessageFormatter" when consuming __consumer_offsets)
   class OffsetsMessageFormatter extends MessageFormatter {
-    def writeTo(consumerRecord: ConsumerRecord[Array[Byte], Array[Byte]], output: PrintStream) {
+    def writeTo(consumerRecord: ConsumerRecord[Array[Byte], Array[Byte]], output: PrintStream): Unit = {
       Option(consumerRecord.key).map(key => GroupMetadataManager.readMessageKey(ByteBuffer.wrap(key))).foreach {
         // Only print if the message is an offset record.
         // We ignore the timestamp of the message because GroupMetadataMessage has its own timestamp.
@@ -1423,7 +1423,7 @@ object GroupMetadataManager {
 
   // Formatter for use with tools to read group metadata history
   class GroupMetadataMessageFormatter extends MessageFormatter {
-    def writeTo(consumerRecord: ConsumerRecord[Array[Byte], Array[Byte]], output: PrintStream) {
+    def writeTo(consumerRecord: ConsumerRecord[Array[Byte], Array[Byte]], output: PrintStream): Unit = {
       Option(consumerRecord.key).map(key => GroupMetadataManager.readMessageKey(ByteBuffer.wrap(key))).foreach {
         // Only print if the message is a group metadata record.
         // We ignore the timestamp of the message because GroupMetadataMessage has its own timestamp.
diff --git a/core/src/main/scala/kafka/coordinator/transaction/ProducerIdManager.scala b/core/src/main/scala/kafka/coordinator/transaction/ProducerIdManager.scala
index 5c22c8e2cb23a..b12dbf8124f91 100644
--- a/core/src/main/scala/kafka/coordinator/transaction/ProducerIdManager.scala
+++ b/core/src/main/scala/kafka/coordinator/transaction/ProducerIdManager.scala
@@ -150,7 +150,7 @@ class ProducerIdManager(val brokerId: Int, val zkClient: KafkaZkClient) extends
     }
   }
 
-  def shutdown() {
+  def shutdown(): Unit = {
     info(s"Shutdown complete: last producerId assigned $nextProducerId")
   }
 }
diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala
index 6d99889d0181e..3f6ec34f2bcbd 100644
--- a/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala
+++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala
@@ -274,11 +274,11 @@ class TransactionCoordinator(brokerId: Int,
     }
   }
 
-  def handleTxnImmigration(txnTopicPartitionId: Int, coordinatorEpoch: Int) {
+  def handleTxnImmigration(txnTopicPartitionId: Int, coordinatorEpoch: Int): Unit = {
     txnManager.loadTransactionsForTxnTopicPartition(txnTopicPartitionId, coordinatorEpoch, txnMarkerChannelManager.addTxnMarkersToSend)
   }
 
-  def handleTxnEmigration(txnTopicPartitionId: Int, coordinatorEpoch: Int) {
+  def handleTxnEmigration(txnTopicPartitionId: Int, coordinatorEpoch: Int): Unit = {
       txnManager.removeTransactionsForTxnTopicPartition(txnTopicPartitionId, coordinatorEpoch)
       txnMarkerChannelManager.removeMarkersForTxnTopicPartition(txnTopicPartitionId)
   }
@@ -493,7 +493,7 @@ class TransactionCoordinator(brokerId: Int,
   /**
    * Startup logic executed at the same time when the server starts up.
    */
-  def startup(enableTransactionalIdExpiration: Boolean = true) {
+  def startup(enableTransactionalIdExpiration: Boolean = true): Unit = {
     info("Starting up.")
     scheduler.startup()
     scheduler.schedule("transaction-abort",
@@ -513,7 +513,7 @@ class TransactionCoordinator(brokerId: Int,
    * Shutdown logic executed at the same time when server shuts down.
    * Ordering of actions should be reversed from the startup process.
    */
-  def shutdown() {
+  def shutdown(): Unit = {
     info("Shutting down.")
     isActive.set(false)
     scheduler.shutdown()
diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionLog.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionLog.scala
index f74e8ad2dfa6a..ef5b59f7cd735 100644
--- a/core/src/main/scala/kafka/coordinator/transaction/TransactionLog.scala
+++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionLog.scala
@@ -252,7 +252,7 @@ object TransactionLog {
 
   // Formatter for use with tools to read transaction log messages
   class TransactionLogMessageFormatter extends MessageFormatter {
-    def writeTo(consumerRecord: ConsumerRecord[Array[Byte], Array[Byte]], output: PrintStream) {
+    def writeTo(consumerRecord: ConsumerRecord[Array[Byte], Array[Byte]], output: PrintStream): Unit = {
       Option(consumerRecord.key).map(key => readTxnRecordKey(ByteBuffer.wrap(key))).foreach { txnKey =>
         val transactionalId = txnKey.transactionalId
         val value = consumerRecord.value
diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerChannelManager.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerChannelManager.scala
index 436ea2ea115a2..34cc6391193d3 100644
--- a/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerChannelManager.scala
+++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerChannelManager.scala
@@ -173,7 +173,7 @@ class TransactionMarkerChannelManager(config: KafkaConfig,
   // visible for testing
   private[transaction] def queueForUnknownBroker = markersQueueForUnknownBroker
 
-  private[transaction] def addMarkersForBroker(broker: Node, txnTopicPartition: Int, txnIdAndMarker: TxnIdAndMarkerEntry) {
+  private[transaction] def addMarkersForBroker(broker: Node, txnTopicPartition: Int, txnIdAndMarker: TxnIdAndMarkerEntry): Unit = {
     val brokerId = broker.id
 
     // we do not synchronize on the update of the broker node with the enqueuing,
diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala
index 38caed5752672..e7ac4a8520abe 100644
--- a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala
+++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala
@@ -145,7 +145,7 @@ class TransactionStateManager(brokerId: Int,
     }
   }
 
-  def enableTransactionalIdExpiration() {
+  def enableTransactionalIdExpiration(): Unit = {
     scheduler.schedule("transactionalId-expiration", () => {
       val now = time.milliseconds()
       inReadLock(stateLock) {
@@ -389,7 +389,7 @@ class TransactionStateManager(brokerId: Int,
    * When this broker becomes a leader for a transaction log partition, load this partition and
    * populate the transaction metadata cache with the transactional ids.
    */
-  def loadTransactionsForTxnTopicPartition(partitionId: Int, coordinatorEpoch: Int, sendTxnMarkers: SendTxnMarkersCallback) {
+  def loadTransactionsForTxnTopicPartition(partitionId: Int, coordinatorEpoch: Int, sendTxnMarkers: SendTxnMarkersCallback): Unit = {
     validateTransactionTopicPartitionCountIsStable()
 
     val topicPartition = new TopicPartition(Topic.TRANSACTION_STATE_TOPIC_NAME, partitionId)
@@ -400,7 +400,7 @@ class TransactionStateManager(brokerId: Int,
       loadingPartitions.add(partitionAndLeaderEpoch)
     }
 
-    def loadTransactions() {
+    def loadTransactions(): Unit = {
       info(s"Loading transaction metadata from $topicPartition")
       val loadedTransactions = loadTransactionMetadata(topicPartition, coordinatorEpoch)
 
@@ -445,7 +445,7 @@ class TransactionStateManager(brokerId: Int,
    * When this broker becomes a follower for a transaction log partition, clear out the cache for corresponding transactional ids
    * that belong to that partition.
    */
-  def removeTransactionsForTxnTopicPartition(partitionId: Int, coordinatorEpoch: Int) {
+  def removeTransactionsForTxnTopicPartition(partitionId: Int, coordinatorEpoch: Int): Unit = {
     validateTransactionTopicPartitionCountIsStable()
 
     val topicPartition = new TopicPartition(Topic.TRANSACTION_STATE_TOPIC_NAME, partitionId)
@@ -456,7 +456,7 @@ class TransactionStateManager(brokerId: Int,
       leavingPartitions.add(partitionAndLeaderEpoch)
     }
 
-    def removeTransactions() {
+    def removeTransactions(): Unit = {
       inWriteLock(stateLock) {
         if (leavingPartitions.contains(partitionAndLeaderEpoch)) {
           transactionMetadataCache.remove(partitionId) match {
@@ -648,7 +648,7 @@ class TransactionStateManager(brokerId: Int,
     }
   }
 
-  def shutdown() {
+  def shutdown(): Unit = {
     shuttingDown.set(true)
     loadingPartitions.clear()
     transactionMetadataCache.clear()
diff --git a/core/src/main/scala/kafka/log/AbstractIndex.scala b/core/src/main/scala/kafka/log/AbstractIndex.scala
index 05237c44193fc..242d0745302bc 100644
--- a/core/src/main/scala/kafka/log/AbstractIndex.scala
+++ b/core/src/main/scala/kafka/log/AbstractIndex.scala
@@ -205,7 +205,7 @@ abstract class AbstractIndex[K, V](@volatile var file: File, val baseOffset: Lon
    *
    * @throws IOException if rename fails
    */
-  def renameTo(f: File) {
+  def renameTo(f: File): Unit = {
     try Utils.atomicMoveWithFallback(file.toPath, f.toPath)
     finally file = f
   }
@@ -213,7 +213,7 @@ abstract class AbstractIndex[K, V](@volatile var file: File, val baseOffset: Lon
   /**
    * Flush the data in the index to disk
    */
-  def flush() {
+  def flush(): Unit = {
     inLock(lock) {
       mmap.force()
     }
@@ -235,7 +235,7 @@ abstract class AbstractIndex[K, V](@volatile var file: File, val baseOffset: Lon
    * Trim this segment to fit just the valid entries, deleting all trailing unwritten bytes from
    * the file.
    */
-  def trimToValidSize() {
+  def trimToValidSize(): Unit = {
     inLock(lock) {
       resize(entrySize * _entries)
     }
@@ -247,7 +247,7 @@ abstract class AbstractIndex[K, V](@volatile var file: File, val baseOffset: Lon
   def sizeInBytes = entrySize * _entries
 
   /** Close the index */
-  def close() {
+  def close(): Unit = {
     trimToValidSize()
     closeHandler()
   }
@@ -318,7 +318,7 @@ abstract class AbstractIndex[K, V](@volatile var file: File, val baseOffset: Lon
   /**
    * Forcefully free the buffer's mmap.
    */
-  protected[log] def forceUnmap() {
+  protected[log] def forceUnmap(): Unit = {
     try MappedByteBuffers.unmap(file.getAbsolutePath, mmap)
     finally mmap = null // Accessing unmapped mmap crashes JVM by SEGV so we null it out to be safe
   }
diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index f8df2115e9aa0..09cffa6d7d42b 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -726,7 +726,7 @@ class Log(@volatile var dir: File,
     }
   }
 
-  private def updateLogEndOffset(messageOffset: Long) {
+  private def updateLogEndOffset(messageOffset: Long): Unit = {
     nextOffsetMetadata = LogOffsetMetadata(messageOffset, activeSegment.baseOffset, activeSegment.size)
 
     // Update the high watermark in case it has gotten ahead of the log end offset
@@ -924,7 +924,7 @@ class Log(@volatile var dir: File,
    * Close this log.
    * The memory mapped buffer for index files of this log will be left open until the log is deleted.
    */
-  def close() {
+  def close(): Unit = {
     debug("Closing log")
     lock synchronized {
       checkIfMemoryMappedBufferClosed()
@@ -944,7 +944,7 @@ class Log(@volatile var dir: File,
    *
    * @throws KafkaStorageException if rename fails
    */
-  def renameDir(name: String) {
+  def renameDir(name: String): Unit = {
     lock synchronized {
       maybeHandleIOException(s"Error while renaming dir for $topicPartition in log dir ${dir.getParent}") {
         val renamedDir = new File(dir.getParent, name)
@@ -964,7 +964,7 @@ class Log(@volatile var dir: File,
   /**
    * Close file handlers used by log but don't write to disk. This is called if the log directory is offline
    */
-  def closeHandlers() {
+  def closeHandlers(): Unit = {
     debug("Closing handlers")
     lock synchronized {
       logSegments.foreach(_.closeHandlers())
@@ -1222,7 +1222,7 @@ class Log(@volatile var dir: File,
   /**
    * Increment the log start offset if the provided offset is larger.
    */
-  def maybeIncrementLogStartOffset(newLogStartOffset: Long) {
+  def maybeIncrementLogStartOffset(newLogStartOffset: Long): Unit = {
     if (newLogStartOffset > highWatermark)
       throw new OffsetOutOfRangeException(s"Cannot increment the log start offset to $newLogStartOffset of partition $topicPartition " +
         s"since it is larger than the high watermark $highWatermark")
@@ -1920,7 +1920,7 @@ class Log(@volatile var dir: File,
    *
    * @param offset The offset to flush up to (non-inclusive); the new recovery point
    */
-  def flush(offset: Long) : Unit = {
+  def flush(offset: Long): Unit = {
     maybeHandleIOException(s"Error while flushing log for $topicPartition in dir ${dir.getParent} with offset $offset") {
       if (offset <= this.recoveryPoint)
         return
@@ -1978,7 +1978,7 @@ class Log(@volatile var dir: File,
   /**
    * Completely delete this log directory and all contents from the file system with no delay
    */
-  private[log] def delete() {
+  private[log] def delete(): Unit = {
     maybeHandleIOException(s"Error while deleting log for $topicPartition in dir ${dir.getParent}") {
       lock synchronized {
         checkIfMemoryMappedBufferClosed()
@@ -2053,7 +2053,7 @@ class Log(@volatile var dir: File,
    *
    *  @param newOffset The new offset to start the log with
    */
-  private[log] def truncateFullyAndStartAt(newOffset: Long) {
+  private[log] def truncateFullyAndStartAt(newOffset: Long): Unit = {
     maybeHandleIOException(s"Error while truncating the entire log for $topicPartition in dir ${dir.getParent}") {
       debug(s"Truncate and start at offset $newOffset")
       lock synchronized {
@@ -2167,10 +2167,10 @@ class Log(@volatile var dir: File,
    *
    * @throws IOException if the file can't be renamed and still exists
    */
-  private def deleteSegmentFiles(segments: Iterable[LogSegment], asyncDelete: Boolean) {
+  private def deleteSegmentFiles(segments: Iterable[LogSegment], asyncDelete: Boolean): Unit = {
     segments.foreach(_.changeFileSuffixes("", Log.DeletedFileSuffix))
 
-    def deleteSegments() {
+    def deleteSegments(): Unit = {
       info(s"Deleting segments $segments")
       maybeHandleIOException(s"Error while deleting segments for $topicPartition in dir ${dir.getParent}") {
         segments.foreach(_.deleteIfExists())
@@ -2217,7 +2217,7 @@ class Log(@volatile var dir: File,
    * @param oldSegments The old log segments to delete from the log
    * @param isRecoveredSwapFile true if the new segment was created from a swap file during recovery after a crash
    */
-  private[log] def replaceSegments(newSegments: Seq[LogSegment], oldSegments: Seq[LogSegment], isRecoveredSwapFile: Boolean = false) {
+  private[log] def replaceSegments(newSegments: Seq[LogSegment], oldSegments: Seq[LogSegment], isRecoveredSwapFile: Boolean = false): Unit = {
     lock synchronized {
       val sortedNewSegments = newSegments.sortBy(_.baseOffset)
       // Some old segments may have been removed from index and scheduled for async deletion after the caller reads segments
diff --git a/core/src/main/scala/kafka/log/LogCleaner.scala b/core/src/main/scala/kafka/log/LogCleaner.scala
index e09f2dcf9e5f5..955733b402f35 100644
--- a/core/src/main/scala/kafka/log/LogCleaner.scala
+++ b/core/src/main/scala/kafka/log/LogCleaner.scala
@@ -142,7 +142,7 @@ class LogCleaner(initialConfig: CleanerConfig,
   /**
    * Start the background cleaning
    */
-  def startup() {
+  def startup(): Unit = {
     info("Starting the log cleaner")
     (0 until config.numThreads).foreach { i =>
       val cleaner = new CleanerThread(i)
@@ -154,7 +154,7 @@ class LogCleaner(initialConfig: CleanerConfig,
   /**
    * Stop the background cleaning
    */
-  def shutdown() {
+  def shutdown(): Unit = {
     info("Shutting down the log cleaner.")
     cleaners.foreach(_.shutdown())
     cleaners.clear()
@@ -191,14 +191,14 @@ class LogCleaner(initialConfig: CleanerConfig,
    *  Abort the cleaning of a particular partition, if it's in progress. This call blocks until the cleaning of
    *  the partition is aborted.
    */
-  def abortCleaning(topicPartition: TopicPartition) {
+  def abortCleaning(topicPartition: TopicPartition): Unit = {
     cleanerManager.abortCleaning(topicPartition)
   }
 
   /**
    * Update checkpoint file, removing topics and partitions that no longer exist
    */
-  def updateCheckpoints(dataDir: File) {
+  def updateCheckpoints(dataDir: File): Unit = {
     cleanerManager.updateCheckpoints(dataDir, update=None)
   }
 
@@ -206,14 +206,14 @@ class LogCleaner(initialConfig: CleanerConfig,
     cleanerManager.alterCheckpointDir(topicPartition, sourceLogDir, destLogDir)
   }
 
-  def handleLogDirFailure(dir: String) {
+  def handleLogDirFailure(dir: String): Unit = {
     cleanerManager.handleLogDirFailure(dir)
   }
 
   /**
    * Truncate cleaner offset checkpoint for the given partition if its checkpointed offset is larger than the given offset
    */
-  def maybeTruncateCheckpoint(dataDir: File, topicPartition: TopicPartition, offset: Long) {
+  def maybeTruncateCheckpoint(dataDir: File, topicPartition: TopicPartition, offset: Long): Unit = {
     cleanerManager.maybeTruncateCheckpoint(dataDir, topicPartition, offset)
   }
 
@@ -221,14 +221,14 @@ class LogCleaner(initialConfig: CleanerConfig,
    *  Abort the cleaning of a particular partition if it's in progress, and pause any future cleaning of this partition.
    *  This call blocks until the cleaning of the partition is aborted and paused.
    */
-  def abortAndPauseCleaning(topicPartition: TopicPartition) {
+  def abortAndPauseCleaning(topicPartition: TopicPartition): Unit = {
     cleanerManager.abortAndPauseCleaning(topicPartition)
   }
 
   /**
     *  Resume the cleaning of paused partitions.
     */
-  def resumeCleaning(topicPartitions: Iterable[TopicPartition]) {
+  def resumeCleaning(topicPartitions: Iterable[TopicPartition]): Unit = {
     cleanerManager.resumeCleaning(topicPartitions)
   }
 
@@ -293,7 +293,7 @@ class LogCleaner(initialConfig: CleanerConfig,
     @volatile var lastStats: CleanerStats = new CleanerStats()
     @volatile var lastPreCleanStats: PreCleanStats = new PreCleanStats()
 
-    private def checkDone(topicPartition: TopicPartition) {
+    private def checkDone(topicPartition: TopicPartition): Unit = {
       if (!isRunning)
         throw new ThreadShutdownException
       cleanerManager.checkCleaningAborted(topicPartition)
@@ -303,7 +303,7 @@ class LogCleaner(initialConfig: CleanerConfig,
      * The main loop for the cleaner thread
      * Clean a log if there is a dirty log available, otherwise sleep for a bit
      */
-    override def doWork() {
+    override def doWork(): Unit = {
       val cleaned = cleanFilthiestLog()
       if (!cleaned)
         pause(config.backOffMs, TimeUnit.MILLISECONDS)
@@ -374,7 +374,7 @@ class LogCleaner(initialConfig: CleanerConfig,
     /**
      * Log out statistics on a single run of the cleaner.
      */
-    def recordStats(id: Int, name: String, from: Long, to: Long, stats: CleanerStats) {
+    def recordStats(id: Int, name: String, from: Long, to: Long, stats: CleanerStats): Unit = {
       this.lastStats = stats
       def mb(bytes: Double) = bytes / (1024*1024)
       val message =
@@ -630,7 +630,7 @@ private[log] class Cleaner(val id: Int,
                              maxLogMessageSize: Int,
                              transactionMetadata: CleanedTransactionMetadata,
                              lastRecordsOfActiveProducers: Map[Long, LastRecord],
-                             stats: CleanerStats) {
+                             stats: CleanerStats): Unit = {
     val logCleanerFilter: RecordFilter = new RecordFilter {
       var discardBatchRecords: Boolean = _
 
@@ -783,7 +783,7 @@ private[log] class Cleaner(val id: Int,
   /**
    * Double the I/O buffer capacity
    */
-  def growBuffers(maxLogMessageSize: Int) {
+  def growBuffers(maxLogMessageSize: Int): Unit = {
     val maxBufferSize = math.max(maxLogMessageSize, maxIoBufferSize)
     if(readBuffer.capacity >= maxBufferSize || writeBuffer.capacity >= maxBufferSize)
       throw new IllegalStateException("This log contains a message larger than maximum allowable size of %s.".format(maxBufferSize))
@@ -796,7 +796,7 @@ private[log] class Cleaner(val id: Int,
   /**
    * Restore the I/O buffer capacity to its original size
    */
-  def restoreBuffers() {
+  def restoreBuffers(): Unit = {
     if(this.readBuffer.capacity > this.ioBufferSize)
       this.readBuffer = ByteBuffer.allocate(this.ioBufferSize)
     if(this.writeBuffer.capacity > this.ioBufferSize)
@@ -873,7 +873,7 @@ private[log] class Cleaner(val id: Int,
                                   start: Long,
                                   end: Long,
                                   map: OffsetMap,
-                                  stats: CleanerStats) {
+                                  stats: CleanerStats): Unit = {
     map.clear()
     val dirty = log.logSegments(start, end).toBuffer
     info("Building offset map for log %s for %d segments in offset range [%d, %d).".format(log.name, dirty.size, start, end))
@@ -1002,33 +1002,33 @@ private class CleanerStats(time: Time = Time.SYSTEM) {
   var messagesWritten = 0L
   var bufferUtilization = 0.0d
 
-  def readMessages(messagesRead: Int, bytesRead: Int) {
+  def readMessages(messagesRead: Int, bytesRead: Int): Unit = {
     this.messagesRead += messagesRead
     this.bytesRead += bytesRead
   }
 
-  def invalidMessage() {
+  def invalidMessage(): Unit = {
     invalidMessagesRead += 1
   }
 
-  def recopyMessages(messagesWritten: Int, bytesWritten: Int) {
+  def recopyMessages(messagesWritten: Int, bytesWritten: Int): Unit = {
     this.messagesWritten += messagesWritten
     this.bytesWritten += bytesWritten
   }
 
-  def indexMessagesRead(size: Int) {
+  def indexMessagesRead(size: Int): Unit = {
     mapMessagesRead += size
   }
 
-  def indexBytesRead(size: Int) {
+  def indexBytesRead(size: Int): Unit = {
     mapBytesRead += size
   }
 
-  def indexDone() {
+  def indexDone(): Unit = {
     mapCompleteTime = time.milliseconds
   }
 
-  def allDone() {
+  def allDone(): Unit = {
     endTime = time.milliseconds
   }
 
diff --git a/core/src/main/scala/kafka/log/LogCleanerManager.scala b/core/src/main/scala/kafka/log/LogCleanerManager.scala
index d46dd944bb2b9..9e215d2a88798 100755
--- a/core/src/main/scala/kafka/log/LogCleanerManager.scala
+++ b/core/src/main/scala/kafka/log/LogCleanerManager.scala
@@ -247,7 +247,7 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
    *  the partition is aborted.
    *  This is implemented by first abortAndPausing and then resuming the cleaning of the partition.
    */
-  def abortCleaning(topicPartition: TopicPartition) {
+  def abortCleaning(topicPartition: TopicPartition): Unit = {
     inLock(lock) {
       abortAndPauseCleaning(topicPartition)
       resumeCleaning(Seq(topicPartition))
@@ -267,7 +267,7 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
    *  6. If the partition is already paused, a new call to this function
    *     will increase the paused count by one.
    */
-  def abortAndPauseCleaning(topicPartition: TopicPartition) {
+  def abortAndPauseCleaning(topicPartition: TopicPartition): Unit = {
     inLock(lock) {
       inProgress.get(topicPartition) match {
         case None =>
@@ -290,7 +290,7 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
     *  Resume the cleaning of paused partitions.
     *  Each call of this function will undo one pause.
     */
-  def resumeCleaning(topicPartitions: Iterable[TopicPartition]){
+  def resumeCleaning(topicPartitions: Iterable[TopicPartition]): Unit = {
     inLock(lock) {
       topicPartitions.foreach {
         topicPartition =>
@@ -344,14 +344,14 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
   /**
    *  Check if the cleaning for a partition is aborted. If so, throw an exception.
    */
-  def checkCleaningAborted(topicPartition: TopicPartition) {
+  def checkCleaningAborted(topicPartition: TopicPartition): Unit = {
     inLock(lock) {
       if (isCleaningInState(topicPartition, LogCleaningAborted))
         throw new LogCleaningAbortedException()
     }
   }
 
-  def updateCheckpoints(dataDir: File, update: Option[(TopicPartition,Long)]) {
+  def updateCheckpoints(dataDir: File, update: Option[(TopicPartition,Long)]): Unit = {
     inLock(lock) {
       val checkpoint = checkpoints(dataDir)
       if (checkpoint != null) {
@@ -390,14 +390,14 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
     }
   }
 
-  def handleLogDirFailure(dir: String) {
+  def handleLogDirFailure(dir: String): Unit = {
     info(s"Stopping cleaning logs in dir $dir")
     inLock(lock) {
       checkpoints = checkpoints.filter { case (k, _) => k.getAbsolutePath != dir }
     }
   }
 
-  def maybeTruncateCheckpoint(dataDir: File, topicPartition: TopicPartition, offset: Long) {
+  def maybeTruncateCheckpoint(dataDir: File, topicPartition: TopicPartition, offset: Long): Unit = {
     inLock(lock) {
       if (logs.get(topicPartition).config.compact) {
         val checkpoint = checkpoints(dataDir)
@@ -413,7 +413,7 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
   /**
    * Save out the endOffset and remove the given log from the in-progress set, if not aborted.
    */
-  def doneCleaning(topicPartition: TopicPartition, dataDir: File, endOffset: Long) {
+  def doneCleaning(topicPartition: TopicPartition, dataDir: File, endOffset: Long): Unit = {
     inLock(lock) {
       inProgress.get(topicPartition) match {
         case Some(LogCleaningInProgress) =>
diff --git a/core/src/main/scala/kafka/log/LogConfig.scala b/core/src/main/scala/kafka/log/LogConfig.scala
index c61e023a3682f..6d93d7a276e76 100755
--- a/core/src/main/scala/kafka/log/LogConfig.scala
+++ b/core/src/main/scala/kafka/log/LogConfig.scala
@@ -112,7 +112,7 @@ case class LogConfig(props: java.util.Map[_, _], overriddenConfigs: Set[String]
 
 object LogConfig {
 
-  def main(args: Array[String]) {
+  def main(args: Array[String]): Unit = {
     println(configDef.toHtmlTable)
   }
 
@@ -313,7 +313,7 @@ object LogConfig {
   /**
    * Check that property names are valid
    */
-  def validateNames(props: Properties) {
+  def validateNames(props: Properties): Unit = {
     val names = configNames
     for(name <- props.asScala.keys)
       if (!names.contains(name))
@@ -334,7 +334,7 @@ object LogConfig {
   /**
    * Check that the given properties contain only valid log config names and that all values can be parsed and are valid
    */
-  def validate(props: Properties) {
+  def validate(props: Properties): Unit = {
     validateNames(props)
     val valueMaps = configDef.parse(props)
     validateValues(valueMaps)
diff --git a/core/src/main/scala/kafka/log/LogManager.scala b/core/src/main/scala/kafka/log/LogManager.scala
index 6c724c3eecd56..f01a74b8c14d6 100755
--- a/core/src/main/scala/kafka/log/LogManager.scala
+++ b/core/src/main/scala/kafka/log/LogManager.scala
@@ -188,7 +188,7 @@ class LogManager(logDirs: Seq[File],
   }
 
   // dir should be an absolute path
-  def handleLogDirFailure(dir: String) {
+  def handleLogDirFailure(dir: String): Unit = {
     info(s"Stopping serving logs in dir $dir")
     logCreationOrDeletionLock synchronized {
       _liveLogDirs.remove(new File(dir))
@@ -388,7 +388,7 @@ class LogManager(logDirs: Seq[File],
   /**
    *  Start the background threads to flush logs and do log cleanup
    */
-  def startup() {
+  def startup(): Unit = {
     /* Schedule the cleanup task to delete old logs */
     if (scheduler != null) {
       info("Starting log cleanup with a period of %d ms.".format(retentionCheckMs))
@@ -425,7 +425,7 @@ class LogManager(logDirs: Seq[File],
   /**
    * Close all the logs
    */
-  def shutdown() {
+  def shutdown(): Unit = {
     info("Shutting down.")
 
     removeMetric("OfflineLogDirectoryCount")
@@ -497,7 +497,7 @@ class LogManager(logDirs: Seq[File],
    * @param partitionOffsets Partition logs that need to be truncated
    * @param isFuture True iff the truncation should be performed on the future log of the specified partitions
    */
-  def truncateTo(partitionOffsets: Map[TopicPartition, Long], isFuture: Boolean) {
+  def truncateTo(partitionOffsets: Map[TopicPartition, Long], isFuture: Boolean): Unit = {
     val affectedLogs = ArrayBuffer.empty[Log]
     for ((topicPartition, truncateOffset) <- partitionOffsets) {
       val log = {
@@ -538,7 +538,7 @@ class LogManager(logDirs: Seq[File],
    * @param newOffset The new offset to start the log with
    * @param isFuture True iff the truncation should be performed on the future log of the specified partition
    */
-  def truncateFullyAndStartAt(topicPartition: TopicPartition, newOffset: Long, isFuture: Boolean) {
+  def truncateFullyAndStartAt(topicPartition: TopicPartition, newOffset: Long, isFuture: Boolean): Unit = {
     val log = {
       if (isFuture)
         futureLogs.get(topicPartition)
@@ -569,7 +569,7 @@ class LogManager(logDirs: Seq[File],
    * Write out the current recovery point for all logs to a text file in the log directory
    * to avoid recovering the whole log on startup.
    */
-  def checkpointLogRecoveryOffsets() {
+  def checkpointLogRecoveryOffsets(): Unit = {
     logsByDir.foreach { case (dir, partitionToLogMap) =>
       liveLogDirs.find(_.getAbsolutePath.equals(dir)).foreach { f =>
         checkpointRecoveryOffsetsAndCleanSnapshot(f, partitionToLogMap.values.toSeq)
@@ -581,7 +581,7 @@ class LogManager(logDirs: Seq[File],
    * Write out the current log start offset for all logs to a text file in the log directory
    * to avoid exposing data that have been deleted by DeleteRecordsRequest
    */
-  def checkpointLogStartOffsets() {
+  def checkpointLogStartOffsets(): Unit = {
     liveLogDirs.foreach(checkpointLogStartOffsetsInDir)
   }
 
@@ -909,7 +909,7 @@ class LogManager(logDirs: Seq[File],
    * Delete any eligible logs. Return the number of segments deleted.
    * Only consider logs that are not compacted.
    */
-  def cleanupLogs() {
+  def cleanupLogs(): Unit = {
     debug("Beginning log cleanup...")
     var total = 0
     val startMs = time.milliseconds
diff --git a/core/src/main/scala/kafka/log/LogSegment.scala b/core/src/main/scala/kafka/log/LogSegment.scala
index 4c16291c170ae..f5d740be4ffee 100755
--- a/core/src/main/scala/kafka/log/LogSegment.scala
+++ b/core/src/main/scala/kafka/log/LogSegment.scala
@@ -234,7 +234,7 @@ class LogSegment private[log] (val log: FileRecords,
   }
 
   @nonthreadsafe
-  def updateTxnIndex(completedTxn: CompletedTxn, lastStableOffset: Long) {
+  def updateTxnIndex(completedTxn: CompletedTxn, lastStableOffset: Long): Unit = {
     if (completedTxn.isAborted) {
       trace(s"Writing aborted transaction $completedTxn to transaction index, last stable offset is $lastStableOffset")
       txnIndex.append(new AbortedTxn(completedTxn, lastStableOffset))
@@ -383,7 +383,7 @@ class LogSegment private[log] (val log: FileRecords,
     truncated
   }
 
-  private def loadLargestTimestamp() {
+  private def loadLargestTimestamp(): Unit = {
     // Get the last time index entry. If the time index is empty, it will return (-1, baseOffset)
     val lastTimeIndexEntry = timeIndex.lastEntry
     maxTimestampSoFar = lastTimeIndexEntry.timestamp
@@ -462,7 +462,7 @@ class LogSegment private[log] (val log: FileRecords,
    * Flush this log segment to disk
    */
   @threadsafe
-  def flush() {
+  def flush(): Unit = {
     LogFlushStats.logFlushTimer.time {
       log.flush()
       offsetIndex.flush()
@@ -486,7 +486,7 @@ class LogSegment private[log] (val log: FileRecords,
    * Change the suffix for the index and log file for this log segment
    * IOException from this method should be handled by the caller
    */
-  def changeFileSuffixes(oldSuffix: String, newSuffix: String) {
+  def changeFileSuffixes(oldSuffix: String, newSuffix: String): Unit = {
     log.renameTo(new File(CoreUtils.replaceSuffix(log.file.getPath, oldSuffix, newSuffix)))
     offsetIndex.renameTo(new File(CoreUtils.replaceSuffix(lazyOffsetIndex.file.getPath, oldSuffix, newSuffix)))
     timeIndex.renameTo(new File(CoreUtils.replaceSuffix(lazyTimeIndex.file.getPath, oldSuffix, newSuffix)))
@@ -498,7 +498,7 @@ class LogSegment private[log] (val log: FileRecords,
    *
    * The time index entry appended will be used to decide when to delete the segment.
    */
-  def onBecomeInactiveSegment() {
+  def onBecomeInactiveSegment(): Unit = {
     timeIndex.maybeAppend(maxTimestampSoFar, offsetOfMaxTimestampSoFar, skipFullCheck = true)
     offsetIndex.trimToValidSize()
     timeIndex.trimToValidSize()
@@ -579,7 +579,7 @@ class LogSegment private[log] (val log: FileRecords,
   /**
    * Close this log segment
    */
-  def close() {
+  def close(): Unit = {
     CoreUtils.swallow(timeIndex.maybeAppend(maxTimestampSoFar, offsetOfMaxTimestampSoFar, skipFullCheck = true), this)
     CoreUtils.swallow(offsetIndex.close(), this)
     CoreUtils.swallow(timeIndex.close(), this)
@@ -590,7 +590,7 @@ class LogSegment private[log] (val log: FileRecords,
   /**
     * Close file handlers used by the log segment but don't write to disk. This is used when the disk may have failed
     */
-  def closeHandlers() {
+  def closeHandlers(): Unit = {
     CoreUtils.swallow(offsetIndex.closeHandler(), this)
     CoreUtils.swallow(timeIndex.closeHandler(), this)
     CoreUtils.swallow(log.closeHandlers(), this)
@@ -600,7 +600,7 @@ class LogSegment private[log] (val log: FileRecords,
   /**
    * Delete this log segment from the filesystem.
    */
-  def deleteIfExists() {
+  def deleteIfExists(): Unit = {
     def delete(delete: () => Boolean, fileType: String, file: File, logIfMissing: Boolean): Unit = {
       try {
         if (delete())
diff --git a/core/src/main/scala/kafka/log/LogValidator.scala b/core/src/main/scala/kafka/log/LogValidator.scala
index 775ea6e8b6a39..b0412bf89147b 100644
--- a/core/src/main/scala/kafka/log/LogValidator.scala
+++ b/core/src/main/scala/kafka/log/LogValidator.scala
@@ -422,7 +422,7 @@ private[kafka] object LogValidator extends Logging {
       recordConversionStats = recordConversionStats)
   }
 
-  private def validateKey(record: Record, compactedTopic: Boolean) {
+  private def validateKey(record: Record, compactedTopic: Boolean): Unit = {
     if (compactedTopic && !record.hasKey)
       throw new InvalidRecordException("Compacted topic cannot accept message without key.")
   }
@@ -435,7 +435,7 @@ private[kafka] object LogValidator extends Logging {
                                 record: Record,
                                 now: Long,
                                 timestampType: TimestampType,
-                                timestampDiffMaxMs: Long) {
+                                timestampDiffMaxMs: Long): Unit = {
     if (timestampType == TimestampType.CREATE_TIME
       && record.timestamp != RecordBatch.NO_TIMESTAMP
       && math.abs(record.timestamp - now) > timestampDiffMaxMs)
diff --git a/core/src/main/scala/kafka/log/OffsetIndex.scala b/core/src/main/scala/kafka/log/OffsetIndex.scala
index d043d0a50271a..204124a5182ce 100755
--- a/core/src/main/scala/kafka/log/OffsetIndex.scala
+++ b/core/src/main/scala/kafka/log/OffsetIndex.scala
@@ -138,7 +138,7 @@ class OffsetIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writabl
    * Append an entry for the given offset/location pair to the index. This entry must have a larger offset than all subsequent entries.
    * @throws IndexOffsetOverflowException if the offset causes index offset to overflow
    */
-  def append(offset: Long, position: Int) {
+  def append(offset: Long, position: Int): Unit = {
     inLock(lock) {
       require(!isFull, "Attempt to append to a full index (size = " + _entries + ").")
       if (_entries == 0 || offset > _lastOffset) {
@@ -157,7 +157,7 @@ class OffsetIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writabl
 
   override def truncate() = truncateToEntries(0)
 
-  override def truncateTo(offset: Long) {
+  override def truncateTo(offset: Long): Unit = {
     inLock(lock) {
       val idx = mmap.duplicate
       val slot = largestLowerBoundSlotFor(idx, offset, IndexSearchType.KEY)
@@ -181,7 +181,7 @@ class OffsetIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writabl
   /**
    * Truncates index to a known number of entries.
    */
-  private def truncateToEntries(entries: Int) {
+  private def truncateToEntries(entries: Int): Unit = {
     inLock(lock) {
       _entries = entries
       mmap.position(_entries * entrySize)
@@ -191,7 +191,7 @@ class OffsetIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writabl
     }
   }
 
-  override def sanityCheck() {
+  override def sanityCheck(): Unit = {
     if (_entries != 0 && _lastOffset < baseOffset)
       throw new CorruptIndexException(s"Corrupt index found, index file (${file.getAbsolutePath}) has non-zero size " +
         s"but the last offset is ${_lastOffset} which is less than the base offset $baseOffset.")
@@ -226,7 +226,7 @@ class LazyOffsetIndex(@volatile private var _file: File, baseOffset: Long, maxIn
       _file
   }
 
-  def file_=(f: File) {
+  def file_=(f: File): Unit = {
     if (offsetIndex.isDefined)
       offsetIndex.get.file = f
     else
diff --git a/core/src/main/scala/kafka/log/OffsetMap.scala b/core/src/main/scala/kafka/log/OffsetMap.scala
index 219bed3c6d22a..22b5305203e90 100755
--- a/core/src/main/scala/kafka/log/OffsetMap.scala
+++ b/core/src/main/scala/kafka/log/OffsetMap.scala
@@ -25,10 +25,10 @@ import org.apache.kafka.common.utils.Utils
 
 trait OffsetMap {
   def slots: Int
-  def put(key: ByteBuffer, offset: Long)
+  def put(key: ByteBuffer, offset: Long): Unit
   def get(key: ByteBuffer): Long
-  def updateLatestOffset(offset: Long)
-  def clear()
+  def updateLatestOffset(offset: Long): Unit
+  def clear(): Unit
   def size: Int
   def utilization: Double = size.toDouble / slots
   def latestOffset: Long
@@ -81,7 +81,7 @@ class SkimpyOffsetMap(val memory: Int, val hashAlgorithm: String = "MD5") extend
    * @param key The key
    * @param offset The offset
    */
-  override def put(key: ByteBuffer, offset: Long) {
+  override def put(key: ByteBuffer, offset: Long): Unit = {
     require(entries < slots, "Attempt to add a new entry to a full offset map.")
     lookups += 1
     hashInto(key, hash1)
@@ -144,7 +144,7 @@ class SkimpyOffsetMap(val memory: Int, val hashAlgorithm: String = "MD5") extend
   /**
    * Change the salt used for key hashing making all existing keys unfindable.
    */
-  override def clear() {
+  override def clear(): Unit = {
     this.entries = 0
     this.lookups = 0L
     this.probes = 0L
@@ -191,7 +191,7 @@ class SkimpyOffsetMap(val memory: Int, val hashAlgorithm: String = "MD5") extend
    * @param key The key to hash
    * @param buffer The buffer to store the hash into
    */
-  private def hashInto(key: ByteBuffer, buffer: Array[Byte]) {
+  private def hashInto(key: ByteBuffer, buffer: Array[Byte]): Unit = {
     key.mark()
     digest.update(key)
     key.reset()
diff --git a/core/src/main/scala/kafka/log/ProducerStateManager.scala b/core/src/main/scala/kafka/log/ProducerStateManager.scala
index 3024352152daf..995bf851eb035 100644
--- a/core/src/main/scala/kafka/log/ProducerStateManager.scala
+++ b/core/src/main/scala/kafka/log/ProducerStateManager.scala
@@ -418,7 +418,7 @@ object ProducerStateManager {
     }
   }
 
-  private def writeSnapshot(file: File, entries: mutable.Map[Long, ProducerStateEntry]) {
+  private def writeSnapshot(file: File, entries: mutable.Map[Long, ProducerStateEntry]): Unit = {
     val struct = new Struct(PidSnapshotMapSchema)
     struct.set(VersionField, ProducerSnapshotVersion)
     struct.set(CrcField, 0L) // we'll fill this after writing the entries
@@ -464,7 +464,7 @@ object ProducerStateManager {
   // visible for testing
   private[log] def deleteSnapshotsBefore(dir: File, offset: Long): Unit = deleteSnapshotFiles(dir, _ < offset)
 
-  private def deleteSnapshotFiles(dir: File, predicate: Long => Boolean = _ => true) {
+  private def deleteSnapshotFiles(dir: File, predicate: Long => Boolean = _ => true): Unit = {
     listSnapshotFiles(dir).filter(file => predicate(offsetFromFile(file))).foreach { file =>
       Files.deleteIfExists(file.toPath)
     }
@@ -551,7 +551,7 @@ class ProducerStateManager(val topicPartition: TopicPartition,
 
   def isEmpty: Boolean = producers.isEmpty && unreplicatedTxns.isEmpty
 
-  private def loadFromSnapshot(logStartOffset: Long, currentTime: Long) {
+  private def loadFromSnapshot(logStartOffset: Long, currentTime: Long): Unit = {
     while (true) {
       latestSnapshotFile match {
         case Some(file) =>
@@ -592,7 +592,7 @@ class ProducerStateManager(val topicPartition: TopicPartition,
   /**
    * Expire any producer ids which have been idle longer than the configured maximum expiration timeout.
    */
-  def removeExpiredProducers(currentTimeMs: Long) {
+  def removeExpiredProducers(currentTimeMs: Long): Unit = {
     producers.retain { case (_, lastEntry) =>
       !isProducerExpired(currentTimeMs, lastEntry)
     }
@@ -706,7 +706,7 @@ class ProducerStateManager(val topicPartition: TopicPartition,
    * should no longer be retained: these producers will be removed if and when we need to load state from
    * the snapshot.
    */
-  def truncateHead(logStartOffset: Long) {
+  def truncateHead(logStartOffset: Long): Unit = {
     val evictedProducerEntries = producers.filter { case (_, producerState) =>
       !isProducerRetained(producerState, logStartOffset)
     }
@@ -745,7 +745,7 @@ class ProducerStateManager(val topicPartition: TopicPartition,
   /**
    * Truncate the producer id mapping and remove all snapshots. This resets the state of the mapping.
    */
-  def truncate() {
+  def truncate(): Unit = {
     producers.clear()
     ongoingTxns.clear()
     unreplicatedTxns.clear()
diff --git a/core/src/main/scala/kafka/log/TimeIndex.scala b/core/src/main/scala/kafka/log/TimeIndex.scala
index f9bc929178ad4..a5f4e79d982cb 100644
--- a/core/src/main/scala/kafka/log/TimeIndex.scala
+++ b/core/src/main/scala/kafka/log/TimeIndex.scala
@@ -110,7 +110,7 @@ class TimeIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writable:
    * @param skipFullCheck To skip checking whether the segment is full or not. We only skip the check when the segment
    *                      gets rolled or the segment is closed.
    */
-  def maybeAppend(timestamp: Long, offset: Long, skipFullCheck: Boolean = false) {
+  def maybeAppend(timestamp: Long, offset: Long, skipFullCheck: Boolean = false): Unit = {
     inLock(lock) {
       if (!skipFullCheck)
         require(!isFull, "Attempt to append to a full time index (size = " + _entries + ").")
@@ -165,7 +165,7 @@ class TimeIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writable:
    * Remove all entries from the index which have an offset greater than or equal to the given offset.
    * Truncating to an offset larger than the largest in the index has no effect.
    */
-  override def truncateTo(offset: Long) {
+  override def truncateTo(offset: Long): Unit = {
     inLock(lock) {
       val idx = mmap.duplicate
       val slot = largestLowerBoundSlotFor(idx, offset, IndexSearchType.VALUE)
@@ -199,7 +199,7 @@ class TimeIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writable:
   /**
    * Truncates index to a known number of entries.
    */
-  private def truncateToEntries(entries: Int) {
+  private def truncateToEntries(entries: Int): Unit = {
     inLock(lock) {
       _entries = entries
       mmap.position(_entries * entrySize)
@@ -208,7 +208,7 @@ class TimeIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writable:
     }
   }
 
-  override def sanityCheck() {
+  override def sanityCheck(): Unit = {
     val lastTimestamp = lastEntry.timestamp
     val lastOffset = lastEntry.offset
     if (_entries != 0 && lastTimestamp < timestamp(mmap, 0))
@@ -248,7 +248,7 @@ class LazyTimeIndex(@volatile private var _file: File, baseOffset: Long, maxInde
       _file
   }
 
-  def file_=(f: File) {
+  def file_=(f: File): Unit = {
     if (timeIndex.isDefined)
       timeIndex.get.file = f
     else
diff --git a/core/src/main/scala/kafka/metrics/KafkaCSVMetricsReporter.scala b/core/src/main/scala/kafka/metrics/KafkaCSVMetricsReporter.scala
index 81c20a7e117bc..82317277814c5 100755
--- a/core/src/main/scala/kafka/metrics/KafkaCSVMetricsReporter.scala
+++ b/core/src/main/scala/kafka/metrics/KafkaCSVMetricsReporter.scala
@@ -45,7 +45,7 @@ private class KafkaCSVMetricsReporter extends KafkaMetricsReporter
   override def getMBeanName = "kafka:type=kafka.metrics.KafkaCSVMetricsReporter"
 
 
-  override def init(props: VerifiableProperties) {
+  override def init(props: VerifiableProperties): Unit = {
     synchronized {
       if (!initialized) {
         val metricsConfig = new KafkaMetricsConfig(props)
@@ -62,7 +62,7 @@ private class KafkaCSVMetricsReporter extends KafkaMetricsReporter
   }
 
 
-  override def startReporter(pollingPeriodSecs: Long) {
+  override def startReporter(pollingPeriodSecs: Long): Unit = {
     synchronized {
       if (initialized && !running) {
         underlying.start(pollingPeriodSecs, TimeUnit.SECONDS)
@@ -73,7 +73,7 @@ private class KafkaCSVMetricsReporter extends KafkaMetricsReporter
   }
 
 
-  override def stopReporter() {
+  override def stopReporter(): Unit = {
     synchronized {
       if (initialized && running) {
         underlying.shutdown()
diff --git a/core/src/main/scala/kafka/metrics/KafkaMetricsReporter.scala b/core/src/main/scala/kafka/metrics/KafkaMetricsReporter.scala
index a373af0418fa8..814cdb2b91b7b 100755
--- a/core/src/main/scala/kafka/metrics/KafkaMetricsReporter.scala
+++ b/core/src/main/scala/kafka/metrics/KafkaMetricsReporter.scala
@@ -35,9 +35,8 @@ import scala.collection.mutable.ArrayBuffer
  * registered MBean is compliant with the standard MBean convention.
  */
 trait KafkaMetricsReporterMBean {
-  def startReporter(pollingPeriodInSeconds: Long)
-  def stopReporter()
-
+  def startReporter(pollingPeriodInSeconds: Long): Unit
+  def stopReporter(): Unit
   /**
    *
    * @return The name with which the MBean will be registered.
@@ -49,7 +48,7 @@ trait KafkaMetricsReporterMBean {
   * Implement {@link org.apache.kafka.common.ClusterResourceListener} to receive cluster metadata once it's available. Please see the class documentation for ClusterResourceListener for more information.
   */
 trait KafkaMetricsReporter {
-  def init(props: VerifiableProperties)
+  def init(props: VerifiableProperties): Unit
 }
 
 object KafkaMetricsReporter {
diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala
index 35d64f0e2901b..4a8c290d945e8 100644
--- a/core/src/main/scala/kafka/network/RequestChannel.scala
+++ b/core/src/main/scala/kafka/network/RequestChannel.scala
@@ -115,7 +115,7 @@ object RequestChannel extends Logging {
       math.max(apiLocalCompleteTimeNanos - requestDequeueTimeNanos, 0L)
     }
 
-    def updateRequestMetrics(networkThreadTimeNanos: Long, response: Response) {
+    def updateRequestMetrics(networkThreadTimeNanos: Long, response: Response): Unit = {
       val endTimeNanos = Time.SYSTEM.nanoseconds
       // In some corner cases, apiLocalCompleteTimeNanos may not be set when the request completes if the remote
       // processing time is really small. This value is set in KafkaApis from a request handling thread.
@@ -308,12 +308,12 @@ class RequestChannel(val queueSize: Int, val metricNamePrefix : String) extends
   }
 
   /** Send a request to be handled, potentially blocking until there is room in the queue for the request */
-  def sendRequest(request: RequestChannel.Request) {
+  def sendRequest(request: RequestChannel.Request): Unit = {
     requestQueue.put(request)
   }
 
   /** Send a response back to the socket server to be sent over the network */
-  def sendResponse(response: RequestChannel.Response) {
+  def sendResponse(response: RequestChannel.Response): Unit = {
     if (isTraceEnabled) {
       val requestHeader = response.request.header
       val message = response match {
@@ -347,17 +347,17 @@ class RequestChannel(val queueSize: Int, val metricNamePrefix : String) extends
   def receiveRequest(): RequestChannel.BaseRequest =
     requestQueue.take()
 
-  def updateErrorMetrics(apiKey: ApiKeys, errors: collection.Map[Errors, Integer]) {
+  def updateErrorMetrics(apiKey: ApiKeys, errors: collection.Map[Errors, Integer]): Unit = {
     errors.foreach { case (error, count) =>
       metrics(apiKey.name).markErrorMeter(error, count)
     }
   }
 
-  def clear() {
+  def clear(): Unit = {
     requestQueue.clear()
   }
 
-  def shutdown() {
+  def shutdown(): Unit = {
     clear()
     metrics.close()
   }
@@ -453,7 +453,7 @@ class RequestMetrics(name: String) extends KafkaMetricsGroup {
     }
   }
 
-  def markErrorMeter(error: Errors, count: Int) {
+  def markErrorMeter(error: Errors, count: Int): Unit = {
     errorMeters(error).getOrCreateMeter().mark(count.toLong)
   }
 
diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala
index 0ebded825980c..0b88894888c2d 100644
--- a/core/src/main/scala/kafka/network/SocketServer.scala
+++ b/core/src/main/scala/kafka/network/SocketServer.scala
@@ -113,7 +113,7 @@ class SocketServer(val config: KafkaConfig,
    *
    * @param startupProcessors Flag indicating whether `Processor`s must be started.
    */
-  def startup(startupProcessors: Boolean = true) {
+  def startup(startupProcessors: Boolean = true): Unit = {
     this.synchronized {
       connectionQuotas = new ConnectionQuotas(config, time)
       createControlPlaneAcceptorAndProcessor(config.controlPlaneListener)
@@ -523,7 +523,7 @@ private[kafka] class Acceptor(val endPoint: EndPoint,
   /**
    * Accept loop that checks for new connection attempts
    */
-  def run() {
+  def run(): Unit = {
     serverChannel.register(nioSelector, SelectionKey.OP_ACCEPT)
     startupComplete()
     try {
@@ -750,7 +750,7 @@ private[kafka] class Processor(val id: Int,
   // closed, connection ids are not reused while requests from the closed connection are being processed.
   private var nextConnectionIndex = 0
 
-  override def run() {
+  override def run(): Unit = {
     startupComplete()
     try {
       while (isRunning) {
@@ -781,14 +781,14 @@ private[kafka] class Processor(val id: Int,
     }
   }
 
-  private def processException(errorMessage: String, throwable: Throwable) {
+  private def processException(errorMessage: String, throwable: Throwable): Unit = {
     throwable match {
       case e: ControlThrowable => throw e
       case e => error(errorMessage, e)
     }
   }
 
-  private def processChannelException(channelId: String, errorMessage: String, throwable: Throwable) {
+  private def processChannelException(channelId: String, errorMessage: String, throwable: Throwable): Unit = {
     if (openOrClosingChannel(channelId).isDefined) {
       error(s"Closing socket for $channelId because of error", throwable)
       close(channelId)
@@ -796,7 +796,7 @@ private[kafka] class Processor(val id: Int,
     processException(errorMessage, throwable)
   }
 
-  private def processNewResponses() {
+  private def processNewResponses(): Unit = {
     var currentResponse: RequestChannel.Response = null
     while ({currentResponse = dequeueResponse(); currentResponse != null}) {
       val channelId = currentResponse.request.context.connectionId
@@ -837,7 +837,7 @@ private[kafka] class Processor(val id: Int,
   }
 
   // `protected` for test usage
-  protected[network] def sendResponse(response: RequestChannel.Response, responseSend: Send) {
+  protected[network] def sendResponse(response: RequestChannel.Response, responseSend: Send): Unit = {
     val connectionId = response.request.context.connectionId
     trace(s"Socket server received response to send to $connectionId, registering for write and sending data: $response")
     // `channel` can be None if the connection was closed remotely or if selector closed it for being idle for too long
@@ -858,7 +858,7 @@ private[kafka] class Processor(val id: Int,
     override def get(): java.lang.Long = time.nanoseconds()
   }
 
-  private def poll() {
+  private def poll(): Unit = {
     val pollTimeout = if (newConnections.isEmpty) 300 else 0
     try selector.poll(pollTimeout)
     catch {
@@ -869,7 +869,7 @@ private[kafka] class Processor(val id: Int,
     }
   }
 
-  private def processCompletedReceives() {
+  private def processCompletedReceives(): Unit = {
     selector.completedReceives.asScala.foreach { receive =>
       try {
         openOrClosingChannel(receive.source) match {
@@ -907,7 +907,7 @@ private[kafka] class Processor(val id: Int,
     }
   }
 
-  private def processCompletedSends() {
+  private def processCompletedSends(): Unit = {
     selector.completedSends.asScala.foreach { send =>
       try {
         val response = inflightResponses.remove(send.destination).getOrElse {
@@ -936,7 +936,7 @@ private[kafka] class Processor(val id: Int,
     request.updateRequestMetrics(networkThreadTimeNanos, response)
   }
 
-  private def processDisconnected() {
+  private def processDisconnected(): Unit = {
     selector.disconnected.keySet.asScala.foreach { connectionId =>
       try {
         val remoteHost = ConnectionId.fromString(connectionId).getOrElse {
@@ -1005,7 +1005,7 @@ private[kafka] class Processor(val id: Int,
    * in each iteration is limited to ensure that traffic and connection close notifications of
    * existing channels are handled promptly.
    */
-  private def configureNewConnections() {
+  private def configureNewConnections(): Unit = {
     var connectionsProcessed = 0
     while (connectionsProcessed < connectionQueueSize && !newConnections.isEmpty) {
       val channel = newConnections.poll()
@@ -1027,7 +1027,7 @@ private[kafka] class Processor(val id: Int,
   /**
    * Close the selector and all open connections
    */
-  private def closeAll() {
+  private def closeAll(): Unit = {
     selector.channels.asScala.foreach { channel =>
       close(channel.id)
     }
@@ -1111,7 +1111,7 @@ class ConnectionQuotas(config: KafkaConfig, time: Time) extends Logging {
   private val maxConnectionsPerListener = mutable.Map[ListenerName, ListenerConnectionQuota]()
   @volatile private var totalCount = 0
 
-  def inc(listenerName: ListenerName, address: InetAddress, acceptorBlockedPercentMeter: com.yammer.metrics.core.Meter) {
+  def inc(listenerName: ListenerName, address: InetAddress, acceptorBlockedPercentMeter: com.yammer.metrics.core.Meter): Unit = {
     counts.synchronized {
       waitForConnectionSlot(listenerName, acceptorBlockedPercentMeter)
 
@@ -1164,7 +1164,7 @@ class ConnectionQuotas(config: KafkaConfig, time: Time) extends Logging {
     }
   }
 
-  def dec(listenerName: ListenerName, address: InetAddress) {
+  def dec(listenerName: ListenerName, address: InetAddress): Unit = {
     counts.synchronized {
       val count = counts.getOrElse(address,
         throw new IllegalArgumentException(s"Attempted to decrease connection count for address with no connections, address: $address"))
diff --git a/core/src/main/scala/kafka/security/CredentialProvider.scala b/core/src/main/scala/kafka/security/CredentialProvider.scala
index f761b8b281a8d..9aa8bc915d4a7 100644
--- a/core/src/main/scala/kafka/security/CredentialProvider.scala
+++ b/core/src/main/scala/kafka/security/CredentialProvider.scala
@@ -31,7 +31,7 @@ class CredentialProvider(scramMechanisms: Collection[String], val tokenCache: De
   val credentialCache = new CredentialCache
   ScramCredentialUtils.createCache(credentialCache, scramMechanisms)
 
-  def updateCredentials(username: String, config: Properties) {
+  def updateCredentials(username: String, config: Properties): Unit = {
     for (mechanism <- ScramMechanism.values()) {
       val cache = credentialCache.cache(mechanism.mechanismName, classOf[ScramCredential])
       if (cache != null) {
diff --git a/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala b/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala
index b4cbcfa604cb6..126b2b13d1de2 100644
--- a/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala
+++ b/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala
@@ -76,7 +76,7 @@ class SimpleAclAuthorizer extends Authorizer with Logging {
   /**
    * Guaranteed to be called before any authorize call is made.
    */
-  override def configure(javaConfigs: util.Map[String, _]) {
+  override def configure(javaConfigs: util.Map[String, _]): Unit = {
     val configs = javaConfigs.asScala
     val props = new java.util.Properties()
     configs.foreach { case (key, value) => props.put(key, value.toString) }
@@ -181,7 +181,7 @@ class SimpleAclAuthorizer extends Authorizer with Logging {
     }
   }
 
-  override def addAcls(acls: Set[Acl], resource: Resource) {
+  override def addAcls(acls: Set[Acl], resource: Resource): Unit = {
     if (acls != null && acls.nonEmpty) {
       if (!extendedAclSupport && resource.patternType == PatternType.PREFIXED) {
         throw new UnsupportedVersionException(s"Adding ACLs on prefixed resource patterns requires " +
@@ -259,12 +259,12 @@ class SimpleAclAuthorizer extends Authorizer with Logging {
     }
   }
 
-  def close() {
+  def close(): Unit = {
     aclChangeListeners.foreach(listener => listener.close())
     if (zkClient != null) zkClient.close()
   }
 
-  private def loadCache() {
+  private def loadCache(): Unit = {
     inWriteLock(lock) {
       ZkAclStore.stores.foreach(store => {
         val resourceTypes = zkClient.getResourceTypes(store.patternType)
@@ -290,7 +290,7 @@ class SimpleAclAuthorizer extends Authorizer with Logging {
       .map(store => store.createListener(AclChangedNotificationHandler, zkClient))
   }
 
-  private def logAuditMessage(principal: KafkaPrincipal, authorized: Boolean, operation: Operation, resource: Resource, host: String) {
+  private def logAuditMessage(principal: KafkaPrincipal, authorized: Boolean, operation: Operation, resource: Resource, host: String): Unit = {
     def logMessage: String = {
       val authResult = if (authorized) "Allowed" else "Denied"
       s"Principal = $principal is $authResult Operation = $operation from host = $host on resource = $resource"
@@ -370,7 +370,7 @@ class SimpleAclAuthorizer extends Authorizer with Logging {
     zkClient.getVersionedAclsForResource(resource)
   }
 
-  private def updateCache(resource: Resource, versionedAcls: VersionedAcls) {
+  private def updateCache(resource: Resource, versionedAcls: VersionedAcls): Unit = {
     if (versionedAcls.acls.nonEmpty) {
       aclCache = aclCache + (resource -> versionedAcls)
     } else {
@@ -378,7 +378,7 @@ class SimpleAclAuthorizer extends Authorizer with Logging {
     }
   }
 
-  private def updateAclChangedFlag(resource: Resource) {
+  private def updateAclChangedFlag(resource: Resource): Unit = {
       zkClient.createAclChangeNotification(resource)
   }
 
@@ -387,7 +387,7 @@ class SimpleAclAuthorizer extends Authorizer with Logging {
   }
 
   object AclChangedNotificationHandler extends AclChangeNotificationHandler {
-    override def processNotification(resource: Resource) {
+    override def processNotification(resource: Resource): Unit = {
       inWriteLock(lock) {
         val versionedAcls = getAclsFromZk(resource)
         updateCache(resource, versionedAcls)
diff --git a/core/src/main/scala/kafka/server/AbstractFetcherManager.scala b/core/src/main/scala/kafka/server/AbstractFetcherManager.scala
index 53152ebd7541f..d76fbfc0acb8b 100755
--- a/core/src/main/scala/kafka/server/AbstractFetcherManager.scala
+++ b/core/src/main/scala/kafka/server/AbstractFetcherManager.scala
@@ -117,7 +117,7 @@ abstract class AbstractFetcherManager[T <: AbstractFetcherThread](val name: Stri
   }
 
   // This method is only needed by ReplicaAlterDirManager
-  def markPartitionsForTruncation(brokerId: Int, topicPartition: TopicPartition, truncationOffset: Long) {
+  def markPartitionsForTruncation(brokerId: Int, topicPartition: TopicPartition, truncationOffset: Long): Unit = {
     lock synchronized {
       val fetcherId = getFetcherId(topicPartition)
       val brokerIdAndFetcherId = BrokerIdAndFetcherId(brokerId, fetcherId)
@@ -130,7 +130,7 @@ abstract class AbstractFetcherManager[T <: AbstractFetcherThread](val name: Stri
   // to be defined in subclass to create a specific fetcher
   def createFetcherThread(fetcherId: Int, sourceBroker: BrokerEndPoint): T
 
-  def addFetcherForPartitions(partitionAndOffsets: Map[TopicPartition, InitialFetchState]) {
+  def addFetcherForPartitions(partitionAndOffsets: Map[TopicPartition, InitialFetchState]): Unit = {
     lock synchronized {
       val partitionsPerFetcher = partitionAndOffsets.groupBy { case (topicPartition, brokerAndInitialFetchOffset) =>
         BrokerAndFetcherId(brokerAndInitialFetchOffset.leader, getFetcherId(topicPartition))
@@ -168,7 +168,7 @@ abstract class AbstractFetcherManager[T <: AbstractFetcherThread](val name: Stri
     }
   }
 
-  def removeFetcherForPartitions(partitions: Set[TopicPartition]) {
+  def removeFetcherForPartitions(partitions: Set[TopicPartition]): Unit = {
     lock synchronized {
       for (fetcher <- fetcherThreadMap.values)
         fetcher.removePartitions(partitions)
@@ -178,7 +178,7 @@ abstract class AbstractFetcherManager[T <: AbstractFetcherThread](val name: Stri
       info(s"Removed fetcher for partitions $partitions")
   }
 
-  def shutdownIdleFetcherThreads() {
+  def shutdownIdleFetcherThreads(): Unit = {
     lock synchronized {
       val keysToBeRemoved = new mutable.HashSet[BrokerIdAndFetcherId]
       for ((key, fetcher) <- fetcherThreadMap) {
@@ -191,7 +191,7 @@ abstract class AbstractFetcherManager[T <: AbstractFetcherThread](val name: Stri
     }
   }
 
-  def closeAllFetchers() {
+  def closeAllFetchers(): Unit = {
     lock synchronized {
       for ( (_, fetcher) <- fetcherThreadMap) {
         fetcher.initiateShutdown()
diff --git a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
index 2dd59445b4cf0..34da9e23780e4 100755
--- a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
+++ b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
@@ -97,7 +97,7 @@ abstract class AbstractFetcherThread(name: String,
 
   protected def isOffsetForLeaderEpochSupported: Boolean
 
-  override def shutdown() {
+  override def shutdown(): Unit = {
     initiateShutdown()
     inLock(partitionMapLock) {
       partitionMapCond.signalAll()
@@ -109,7 +109,7 @@ abstract class AbstractFetcherThread(name: String,
     fetcherLagStats.unregister()
   }
 
-  override def doWork() {
+  override def doWork(): Unit = {
     maybeTruncate()
     maybeFetch()
   }
@@ -135,7 +135,7 @@ abstract class AbstractFetcherThread(name: String,
   }
 
   // deal with partitions with errors, potentially due to leadership changes
-  private def handlePartitionsWithErrors(partitions: Iterable[TopicPartition], methodName: String) {
+  private def handlePartitionsWithErrors(partitions: Iterable[TopicPartition], methodName: String): Unit = {
     if (partitions.nonEmpty) {
       debug(s"Handling errors in $methodName for partitions $partitions")
       delayPartitions(partitions, fetchBackOffMs)
@@ -383,7 +383,7 @@ abstract class AbstractFetcherThread(name: String,
     }
   }
 
-  def markPartitionsForTruncation(topicPartition: TopicPartition, truncationOffset: Long) {
+  def markPartitionsForTruncation(topicPartition: TopicPartition, truncationOffset: Long): Unit = {
     partitionMapLock.lockInterruptibly()
     try {
       Option(partitionStates.stateValue(topicPartition)).foreach { state =>
@@ -405,7 +405,7 @@ abstract class AbstractFetcherThread(name: String,
   }
 
 
-  def addPartitions(initialFetchStates: Map[TopicPartition, OffsetAndEpoch]) {
+  def addPartitions(initialFetchStates: Map[TopicPartition, OffsetAndEpoch]): Unit = {
     partitionMapLock.lockInterruptibly()
     try {
       initialFetchStates.foreach { case (tp, initialFetchState) =>
@@ -433,7 +433,7 @@ abstract class AbstractFetcherThread(name: String,
     *
     * @param fetchOffsets the partitions to update fetch offset and maybe mark truncation complete
     */
-  private def updateFetchOffsetAndMaybeMarkTruncationComplete(fetchOffsets: Map[TopicPartition, OffsetTruncationState]) {
+  private def updateFetchOffsetAndMaybeMarkTruncationComplete(fetchOffsets: Map[TopicPartition, OffsetTruncationState]): Unit = {
     val newStates: Map[TopicPartition, PartitionFetchState] = partitionStates.partitionStates.asScala
       .map { state =>
         val currentFetchState = state.value
@@ -606,7 +606,7 @@ abstract class AbstractFetcherThread(name: String,
     }
   }
 
-  def delayPartitions(partitions: Iterable[TopicPartition], delay: Long) {
+  def delayPartitions(partitions: Iterable[TopicPartition], delay: Long): Unit = {
     partitionMapLock.lockInterruptibly()
     try {
       for (partition <- partitions) {
@@ -621,7 +621,7 @@ abstract class AbstractFetcherThread(name: String,
     } finally partitionMapLock.unlock()
   }
 
-  def removePartitions(topicPartitions: Set[TopicPartition]) {
+  def removePartitions(topicPartitions: Set[TopicPartition]): Unit = {
     partitionMapLock.lockInterruptibly()
     try {
       topicPartitions.foreach { topicPartition =>
@@ -690,13 +690,13 @@ class FetcherLagMetrics(metricId: ClientIdTopicPartition) extends KafkaMetricsGr
     tags
   )
 
-  def lag_=(newLag: Long) {
+  def lag_=(newLag: Long): Unit = {
     lagVal.set(newLag)
   }
 
   def lag = lagVal.get
 
-  def unregister() {
+  def unregister(): Unit = {
     removeMetric(FetcherMetrics.ConsumerLag, tags)
   }
 }
@@ -717,12 +717,12 @@ class FetcherLagStats(metricId: ClientIdAndBroker) {
       false
   }
 
-  def unregister(topicPartition: TopicPartition) {
+  def unregister(topicPartition: TopicPartition): Unit = {
     val lagMetrics = stats.remove(ClientIdTopicPartition(metricId.clientId, topicPartition))
     if (lagMetrics != null) lagMetrics.unregister()
   }
 
-  def unregister() {
+  def unregister(): Unit = {
     stats.keys.toBuffer.foreach { key: ClientIdTopicPartition =>
       unregister(key.topicPartition)
     }
@@ -738,7 +738,7 @@ class FetcherStats(metricId: ClientIdAndBroker) extends KafkaMetricsGroup {
 
   val byteRate = newMeter(FetcherMetrics.BytesPerSec, "bytes", TimeUnit.SECONDS, tags)
 
-  def unregister() {
+  def unregister(): Unit = {
     removeMetric(FetcherMetrics.RequestsPerSec, tags)
     removeMetric(FetcherMetrics.BytesPerSec, tags)
   }
diff --git a/core/src/main/scala/kafka/server/AdminManager.scala b/core/src/main/scala/kafka/server/AdminManager.scala
index 1daaeec6bb6a8..fc3c93432f366 100644
--- a/core/src/main/scala/kafka/server/AdminManager.scala
+++ b/core/src/main/scala/kafka/server/AdminManager.scala
@@ -69,7 +69,7 @@ class AdminManager(val config: KafkaConfig,
   /**
     * Try to complete delayed topic operations with the request key
     */
-  def tryCompleteDelayedTopicOperations(topic: String) {
+  def tryCompleteDelayedTopicOperations(topic: String): Unit = {
     val key = TopicKey(topic)
     val completed = topicPurgatory.checkAndComplete(key)
     debug(s"Request key ${key.keyLabel} unblocked $completed topic requests.")
@@ -82,7 +82,7 @@ class AdminManager(val config: KafkaConfig,
   def createTopics(timeout: Int,
                    validateOnly: Boolean,
                    toCreate: Map[String, CreatableTopic],
-                   responseCallback: Map[String, ApiError] => Unit) {
+                   responseCallback: Map[String, ApiError] => Unit): Unit = {
 
     // 1. map over topics creating assignment and calling zookeeper
     val brokers = metadataCache.getAliveBrokers.map { b => kafka.admin.BrokerMetadata(b.id, b.rack) }
@@ -190,7 +190,7 @@ class AdminManager(val config: KafkaConfig,
     */
   def deleteTopics(timeout: Int,
                    topics: Set[String],
-                   responseCallback: Map[String, Errors] => Unit) {
+                   responseCallback: Map[String, Errors] => Unit): Unit = {
 
     // 1. map over topics calling the asynchronous delete
     val metadata = topics.map { topic =>
@@ -583,7 +583,7 @@ class AdminManager(val config: KafkaConfig,
     }
   }
 
-  def shutdown() {
+  def shutdown(): Unit = {
     topicPurgatory.shutdown()
     CoreUtils.swallow(createTopicPolicy.foreach(_.close()), this)
     CoreUtils.swallow(alterConfigPolicy.foreach(_.close()), this)
diff --git a/core/src/main/scala/kafka/server/BrokerStates.scala b/core/src/main/scala/kafka/server/BrokerStates.scala
index 2b66beb6328a5..f53ed833c22e9 100644
--- a/core/src/main/scala/kafka/server/BrokerStates.scala
+++ b/core/src/main/scala/kafka/server/BrokerStates.scala
@@ -67,12 +67,12 @@ case object BrokerShuttingDown extends BrokerStates { val state: Byte = 7 }
 case class BrokerState() {
   @volatile var currentState: Byte = NotRunning.state
 
-  def newState(newState: BrokerStates) {
+  def newState(newState: BrokerStates): Unit = {
     this.newState(newState.state)
   }
 
   // Allowing undefined custom state
-  def newState(newState: Byte) {
+  def newState(newState: Byte): Unit = {
     currentState = newState
   }
 }
diff --git a/core/src/main/scala/kafka/server/ClientQuotaManager.scala b/core/src/main/scala/kafka/server/ClientQuotaManager.scala
index 5451b5caa35df..730cc618122f0 100644
--- a/core/src/main/scala/kafka/server/ClientQuotaManager.scala
+++ b/core/src/main/scala/kafka/server/ClientQuotaManager.scala
@@ -181,7 +181,7 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig,
     quotaType.toString,
     "Tracks the size of the delay queue"), new CumulativeSum())
   start() // Use start method to keep spotbugs happy
-  private def start() {
+  private def start(): Unit = {
     throttledChannelReaper.start()
   }
 
@@ -285,7 +285,7 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig,
    * quota violation. The aggregate value will subsequently be used for throttling when the
    * next request is processed.
    */
-  def recordNoThrottle(clientSensors: ClientSensors, value: Double) {
+  def recordNoThrottle(clientSensors: ClientSensors, value: Double): Unit = {
     clientSensors.quotaSensor.record(value, time.milliseconds(), false)
   }
 
@@ -410,7 +410,7 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig,
    * @param sanitizedClientId sanitized client ID to override if quota applies to  or 
    * @param quota             custom quota to apply or None if quota override is being removed
    */
-  def updateQuota(sanitizedUser: Option[String], clientId: Option[String], sanitizedClientId: Option[String], quota: Option[Quota]) {
+  def updateQuota(sanitizedUser: Option[String], clientId: Option[String], sanitizedClientId: Option[String], quota: Option[Quota]): Unit = {
     /*
      * Acquire the write lock to apply changes in the quota objects.
      * This method changes the quota in the overriddenQuota map and applies the update on the actual KafkaMetric object (if it exists).
diff --git a/core/src/main/scala/kafka/server/ClientRequestQuotaManager.scala b/core/src/main/scala/kafka/server/ClientRequestQuotaManager.scala
index 7bab61605bddb..eb921dca7d32f 100644
--- a/core/src/main/scala/kafka/server/ClientRequestQuotaManager.scala
+++ b/core/src/main/scala/kafka/server/ClientRequestQuotaManager.scala
@@ -36,7 +36,7 @@ class ClientRequestQuotaManager(private val config: ClientQuotaManagerConfig,
   val maxThrottleTimeMs = TimeUnit.SECONDS.toMillis(this.config.quotaWindowSizeSeconds)
   def exemptSensor = getOrCreateSensor(exemptSensorName, exemptMetricName)
 
-  def recordExempt(value: Double) {
+  def recordExempt(value: Double): Unit = {
     exemptSensor.record(value)
   }
 
diff --git a/core/src/main/scala/kafka/server/ConfigHandler.scala b/core/src/main/scala/kafka/server/ConfigHandler.scala
index 0c3362dfbab8e..a80d0f51334a7 100644
--- a/core/src/main/scala/kafka/server/ConfigHandler.scala
+++ b/core/src/main/scala/kafka/server/ConfigHandler.scala
@@ -41,7 +41,7 @@ import scala.util.Try
   * The ConfigHandler is used to process config change notifications received by the DynamicConfigManager
   */
 trait ConfigHandler {
-  def processConfigChanges(entityName: String, value: Properties)
+  def processConfigChanges(entityName: String, value: Properties): Unit
 }
 
 /**
@@ -50,7 +50,7 @@ trait ConfigHandler {
   */
 class TopicConfigHandler(private val logManager: LogManager, kafkaConfig: KafkaConfig, val quotas: QuotaManagers, kafkaController: KafkaController) extends ConfigHandler with Logging  {
 
-  def processConfigChanges(topic: String, topicConfig: Properties) {
+  def processConfigChanges(topic: String, topicConfig: Properties): Unit = {
     // Validate the configurations.
     val configNamesToExclude = excludedConfigs(topic, topicConfig)
 
@@ -117,7 +117,7 @@ class TopicConfigHandler(private val logManager: LogManager, kafkaConfig: KafkaC
  */
 class QuotaConfigHandler(private val quotaManagers: QuotaManagers) {
 
-  def updateQuotaConfig(sanitizedUser: Option[String], sanitizedClientId: Option[String], config: Properties) {
+  def updateQuotaConfig(sanitizedUser: Option[String], sanitizedClientId: Option[String], config: Properties): Unit = {
     val clientId = sanitizedClientId.map(Sanitizer.desanitize)
     val producerQuota =
       if (config.containsKey(DynamicConfig.Client.ProducerByteRateOverrideProp))
@@ -146,7 +146,7 @@ class QuotaConfigHandler(private val quotaManagers: QuotaManagers) {
  */
 class ClientIdConfigHandler(private val quotaManagers: QuotaManagers) extends QuotaConfigHandler(quotaManagers) with ConfigHandler {
 
-  def processConfigChanges(sanitizedClientId: String, clientConfig: Properties) {
+  def processConfigChanges(sanitizedClientId: String, clientConfig: Properties): Unit = {
     updateQuotaConfig(None, Some(sanitizedClientId), clientConfig)
   }
 }
@@ -158,7 +158,7 @@ class ClientIdConfigHandler(private val quotaManagers: QuotaManagers) extends Qu
  */
 class UserConfigHandler(private val quotaManagers: QuotaManagers, val credentialProvider: CredentialProvider) extends QuotaConfigHandler(quotaManagers) with ConfigHandler {
 
-  def processConfigChanges(quotaEntityPath: String, config: Properties) {
+  def processConfigChanges(quotaEntityPath: String, config: Properties): Unit = {
     // Entity path is  or /clients/
     val entities = quotaEntityPath.split("/")
     if (entities.length != 1 && entities.length != 3)
@@ -179,7 +179,7 @@ class UserConfigHandler(private val quotaManagers: QuotaManagers, val credential
 class BrokerConfigHandler(private val brokerConfig: KafkaConfig,
                           private val quotaManagers: QuotaManagers) extends ConfigHandler with Logging {
 
-  def processConfigChanges(brokerId: String, properties: Properties) {
+  def processConfigChanges(brokerId: String, properties: Properties): Unit = {
     def getOrDefault(prop: String): Long = {
       if (properties.containsKey(prop))
         properties.getProperty(prop).toLong
diff --git a/core/src/main/scala/kafka/server/DelayedCreatePartitions.scala b/core/src/main/scala/kafka/server/DelayedCreatePartitions.scala
index 0a9948311af0c..cfa017852d40f 100644
--- a/core/src/main/scala/kafka/server/DelayedCreatePartitions.scala
+++ b/core/src/main/scala/kafka/server/DelayedCreatePartitions.scala
@@ -61,7 +61,7 @@ class DelayedCreatePartitions(delayMs: Long,
   /**
     * Check for partitions that are still missing a leader, update their error code and call the responseCallback
     */
-  override def onComplete() {
+  override def onComplete(): Unit = {
     trace(s"Completing operation for $createMetadata")
     val results = createMetadata.map { metadata =>
       // ignore topics that already have errors
diff --git a/core/src/main/scala/kafka/server/DelayedDeleteRecords.scala b/core/src/main/scala/kafka/server/DelayedDeleteRecords.scala
index 4af82522eac2f..e83ce0a8af44d 100644
--- a/core/src/main/scala/kafka/server/DelayedDeleteRecords.scala
+++ b/core/src/main/scala/kafka/server/DelayedDeleteRecords.scala
@@ -103,7 +103,7 @@ class DelayedDeleteRecords(delayMs: Long,
       false
   }
 
-  override def onExpiration() {
+  override def onExpiration(): Unit = {
     deleteRecordsStatus.foreach { case (topicPartition, status) =>
       if (status.acksPending) {
         DelayedDeleteRecordsMetrics.recordExpiration(topicPartition)
@@ -114,7 +114,7 @@ class DelayedDeleteRecords(delayMs: Long,
   /**
    * Upon completion, return the current response status along with the error code per partition
    */
-  override def onComplete() {
+  override def onComplete(): Unit = {
     val responseStatus = deleteRecordsStatus.map { case (k, status) => k -> status.responseStatus }
     responseCallback(responseStatus)
   }
@@ -124,7 +124,7 @@ object DelayedDeleteRecordsMetrics extends KafkaMetricsGroup {
 
   private val aggregateExpirationMeter = newMeter("ExpiresPerSec", "requests", TimeUnit.SECONDS)
 
-  def recordExpiration(partition: TopicPartition) {
+  def recordExpiration(partition: TopicPartition): Unit = {
     aggregateExpirationMeter.mark()
   }
 }
diff --git a/core/src/main/scala/kafka/server/DelayedDeleteTopics.scala b/core/src/main/scala/kafka/server/DelayedDeleteTopics.scala
index 95d6f50515c62..bb15a27948c73 100644
--- a/core/src/main/scala/kafka/server/DelayedDeleteTopics.scala
+++ b/core/src/main/scala/kafka/server/DelayedDeleteTopics.scala
@@ -57,7 +57,7 @@ class DelayedDeleteTopics(delayMs: Long,
   /**
     * Check for partitions that still exist, update their error code and call the responseCallback
     */
-  override def onComplete() {
+  override def onComplete(): Unit = {
     trace(s"Completing operation for $deleteMetadata")
     val results = deleteMetadata.map { metadata =>
       // ignore topics that already have errors
diff --git a/core/src/main/scala/kafka/server/DelayedFetch.scala b/core/src/main/scala/kafka/server/DelayedFetch.scala
index c3dbde9a0d88f..849fb1d93d496 100644
--- a/core/src/main/scala/kafka/server/DelayedFetch.scala
+++ b/core/src/main/scala/kafka/server/DelayedFetch.scala
@@ -149,7 +149,7 @@ class DelayedFetch(delayMs: Long,
       false
   }
 
-  override def onExpiration() {
+  override def onExpiration(): Unit = {
     if (fetchMetadata.isFromFollower)
       DelayedFetchMetrics.followerExpiredRequestMeter.mark()
     else
@@ -159,7 +159,7 @@ class DelayedFetch(delayMs: Long,
   /**
    * Upon completion, read whatever data is available and pass to the complete callback
    */
-  override def onComplete() {
+  override def onComplete(): Unit = {
     val logReadResults = replicaManager.readFromLocalLog(
       replicaId = fetchMetadata.replicaId,
       fetchOnlyFromLeader = fetchMetadata.fetchOnlyLeader,
diff --git a/core/src/main/scala/kafka/server/DelayedOperation.scala b/core/src/main/scala/kafka/server/DelayedOperation.scala
index 33187bb86dc6f..5e965c4fb6362 100644
--- a/core/src/main/scala/kafka/server/DelayedOperation.scala
+++ b/core/src/main/scala/kafka/server/DelayedOperation.scala
@@ -329,7 +329,7 @@ final class DelayedOperationPurgatory[T <: DelayedOperation](purgatoryName: Stri
    * Return the watch list of the given key, note that we need to
    * grab the removeWatchersLock to avoid the operation being added to a removed watcher list
    */
-  private def watchForOperation(key: Any, operation: T) {
+  private def watchForOperation(key: Any, operation: T): Unit = {
     val wl = watcherList(key)
     inLock(wl.watchersLock) {
       val watcher = wl.watchersByKey.getAndMaybePut(key)
@@ -340,7 +340,7 @@ final class DelayedOperationPurgatory[T <: DelayedOperation](purgatoryName: Stri
   /*
    * Remove the key from watcher lists if its list is empty
    */
-  private def removeKeyIfEmpty(key: Any, watchers: Watchers) {
+  private def removeKeyIfEmpty(key: Any, watchers: Watchers): Unit = {
     val wl = watcherList(key)
     inLock(wl.watchersLock) {
       // if the current key is no longer correlated to the watchers to remove, skip
@@ -356,7 +356,7 @@ final class DelayedOperationPurgatory[T <: DelayedOperation](purgatoryName: Stri
   /**
    * Shutdown the expire reaper thread
    */
-  def shutdown() {
+  def shutdown(): Unit = {
     if (reaperEnabled)
       expirationReaper.shutdown()
     timeoutTimer.shutdown()
@@ -374,7 +374,7 @@ final class DelayedOperationPurgatory[T <: DelayedOperation](purgatoryName: Stri
     def isEmpty: Boolean = operations.isEmpty
 
     // add the element to watch
-    def watch(t: T) {
+    def watch(t: T): Unit = {
       operations.add(t)
     }
 
@@ -432,7 +432,7 @@ final class DelayedOperationPurgatory[T <: DelayedOperation](purgatoryName: Stri
     }
   }
 
-  def advanceClock(timeoutMs: Long) {
+  def advanceClock(timeoutMs: Long): Unit = {
     timeoutTimer.advanceClock(timeoutMs)
 
     // Trigger a purge if the number of completed but still being watched operations is larger than
@@ -458,7 +458,7 @@ final class DelayedOperationPurgatory[T <: DelayedOperation](purgatoryName: Stri
     "ExpirationReaper-%d-%s".format(brokerId, purgatoryName),
     false) {
 
-    override def doWork() {
+    override def doWork(): Unit = {
       advanceClock(200L)
     }
   }
diff --git a/core/src/main/scala/kafka/server/DelayedProduce.scala b/core/src/main/scala/kafka/server/DelayedProduce.scala
index 14b6aca23128e..6ec20c1d518ff 100644
--- a/core/src/main/scala/kafka/server/DelayedProduce.scala
+++ b/core/src/main/scala/kafka/server/DelayedProduce.scala
@@ -112,7 +112,7 @@ class DelayedProduce(delayMs: Long,
       false
   }
 
-  override def onExpiration() {
+  override def onExpiration(): Unit = {
     produceMetadata.produceStatus.foreach { case (topicPartition, status) =>
       if (status.acksPending) {
         debug(s"Expiring produce request for partition $topicPartition with status $status")
@@ -124,7 +124,7 @@ class DelayedProduce(delayMs: Long,
   /**
    * Upon completion, return the current response status along with the error code per partition
    */
-  override def onComplete() {
+  override def onComplete(): Unit = {
     val responseStatus = produceMetadata.produceStatus.map { case (k, status) => k -> status.responseStatus }
     responseCallback(responseStatus)
   }
@@ -141,7 +141,7 @@ object DelayedProduceMetrics extends KafkaMetricsGroup {
              tags = Map("topic" -> key.topic, "partition" -> key.partition.toString))
   private val partitionExpirationMeters = new Pool[TopicPartition, Meter](valueFactory = Some(partitionExpirationMeterFactory))
 
-  def recordExpiration(partition: TopicPartition) {
+  def recordExpiration(partition: TopicPartition): Unit = {
     aggregateExpirationMeter.mark()
     partitionExpirationMeters.getAndMaybePut(partition).mark()
   }
diff --git a/core/src/main/scala/kafka/server/DelegationTokenManager.scala b/core/src/main/scala/kafka/server/DelegationTokenManager.scala
index 90025f429898f..93053e23570cd 100644
--- a/core/src/main/scala/kafka/server/DelegationTokenManager.scala
+++ b/core/src/main/scala/kafka/server/DelegationTokenManager.scala
@@ -198,7 +198,7 @@ class DelegationTokenManager(val config: KafkaConfig,
     }
   }
 
-  private def loadCache() {
+  private def loadCache(): Unit = {
     lock.synchronized {
       val tokens = zkClient.getChildren(DelegationTokensZNode.path)
       info(s"Loading the token cache. Total token count: ${tokens.size}")
@@ -261,7 +261,7 @@ class DelegationTokenManager(val config: KafkaConfig,
   def createToken(owner: KafkaPrincipal,
                   renewers: List[KafkaPrincipal],
                   maxLifeTimeMs: Long,
-                  responseCallback: CreateResponseCallback) {
+                  responseCallback: CreateResponseCallback): Unit = {
 
     if (!config.tokenAuthEnabled) {
       responseCallback(CreateTokenResult(-1, -1, -1, "", Array[Byte](), Errors.DELEGATION_TOKEN_AUTH_DISABLED))
@@ -295,7 +295,7 @@ class DelegationTokenManager(val config: KafkaConfig,
   def renewToken(principal: KafkaPrincipal,
                  hmac: ByteBuffer,
                  renewLifeTimeMs: Long,
-                 renewCallback: RenewResponseCallback) {
+                 renewCallback: RenewResponseCallback): Unit = {
 
     if (!config.tokenAuthEnabled) {
       renewCallback(Errors.DELEGATION_TOKEN_AUTH_DISABLED, -1)
@@ -395,7 +395,7 @@ class DelegationTokenManager(val config: KafkaConfig,
   def expireToken(principal: KafkaPrincipal,
                   hmac: ByteBuffer,
                   expireLifeTimeMs: Long,
-                  expireResponseCallback: ExpireResponseCallback) {
+                  expireResponseCallback: ExpireResponseCallback): Unit = {
 
     if (!config.tokenAuthEnabled) {
       expireResponseCallback(Errors.DELEGATION_TOKEN_AUTH_DISABLED, -1)
@@ -477,7 +477,7 @@ class DelegationTokenManager(val config: KafkaConfig,
   }
 
   object TokenChangedNotificationHandler extends NotificationHandler {
-    override def processNotification(tokenIdBytes: Array[Byte]) {
+    override def processNotification(tokenIdBytes: Array[Byte]): Unit = {
       lock.synchronized {
         val tokenId = new String(tokenIdBytes, StandardCharsets.UTF_8)
         info(s"Processing Token Notification for tokenId: $tokenId")
diff --git a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
index 37183be2d510a..3294c284170f4 100755
--- a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
+++ b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
@@ -120,7 +120,7 @@ object DynamicBrokerConfig {
     }
   }
 
-  def validateConfigs(props: Properties, perBrokerConfig: Boolean): Unit =  {
+  def validateConfigs(props: Properties, perBrokerConfig: Boolean): Unit = {
     def checkInvalidProps(invalidPropNames: Set[String], errorMessage: String): Unit = {
       if (invalidPropNames.nonEmpty)
         throw new ConfigException(s"$errorMessage: $invalidPropNames")
diff --git a/core/src/main/scala/kafka/server/DynamicConfigManager.scala b/core/src/main/scala/kafka/server/DynamicConfigManager.scala
index be2edcfb55fba..4a80a6f15ea85 100644
--- a/core/src/main/scala/kafka/server/DynamicConfigManager.scala
+++ b/core/src/main/scala/kafka/server/DynamicConfigManager.scala
@@ -108,7 +108,7 @@ class DynamicConfigManager(private val zkClient: KafkaZkClient,
       }
     }
 
-    private def processEntityConfigChangeVersion1(jsonBytes: Array[Byte], js: JsonObject) {
+    private def processEntityConfigChangeVersion1(jsonBytes: Array[Byte], js: JsonObject): Unit = {
       val validConfigTypes = Set(ConfigType.Topic, ConfigType.Client)
       val entityType = js.get("entity_type").flatMap(_.to[Option[String]]).filter(validConfigTypes).getOrElse {
         throw new IllegalArgumentException("Version 1 config change notification must have 'entity_type' set to " +
@@ -126,7 +126,7 @@ class DynamicConfigManager(private val zkClient: KafkaZkClient,
 
     }
 
-    private def processEntityConfigChangeVersion2(jsonBytes: Array[Byte], js: JsonObject) {
+    private def processEntityConfigChangeVersion2(jsonBytes: Array[Byte], js: JsonObject): Unit = {
 
       val entityPath = js.get("entity_path").flatMap(_.to[Option[String]]).getOrElse {
         throw new IllegalArgumentException(s"Version 2 config change notification must specify 'entity_path'. " +
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 4df57a95aa404..77b137783910a 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -125,14 +125,14 @@ class KafkaApis(val requestChannel: RequestChannel,
   this.logIdent = "[KafkaApi-%d] ".format(brokerId)
   val adminZkClient = new AdminZkClient(zkClient)
 
-  def close() {
+  def close(): Unit = {
     info("Shutdown complete.")
   }
 
   /**
    * Top-level method that handles all requests and multiplexes to the right api
    */
-  def handle(request: RequestChannel.Request) {
+  def handle(request: RequestChannel.Request): Unit = {
     try {
       trace(s"Handling request:${request.requestDesc(true)} from connection ${request.context.connectionId};" +
         s"securityProtocol:${request.context.securityProtocol},principal:${request.context.principal}")
@@ -193,14 +193,14 @@ class KafkaApis(val requestChannel: RequestChannel,
     }
   }
 
-  def handleLeaderAndIsrRequest(request: RequestChannel.Request) {
+  def handleLeaderAndIsrRequest(request: RequestChannel.Request): Unit = {
     // ensureTopicExists is only for client facing requests
     // We can't have the ensureTopicExists check here since the controller sends it as an advisory to all brokers so they
     // stop serving data to clients for the topic being deleted
     val correlationId = request.header.correlationId
     val leaderAndIsrRequest = request.body[LeaderAndIsrRequest]
 
-    def onLeadershipChange(updatedLeaders: Iterable[Partition], updatedFollowers: Iterable[Partition]) {
+    def onLeadershipChange(updatedLeaders: Iterable[Partition], updatedFollowers: Iterable[Partition]): Unit = {
       // for each new leader or follower, call coordinator to handle consumer group migration.
       // this callback is invoked under the replica state change lock to ensure proper order of
       // leadership changes
@@ -232,7 +232,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     }
   }
 
-  def handleStopReplicaRequest(request: RequestChannel.Request) {
+  def handleStopReplicaRequest(request: RequestChannel.Request): Unit = {
     // ensureTopicExists is only for client facing requests
     // We can't have the ensureTopicExists check here since the controller sends it as an advisory to all brokers so they
     // stop serving data to clients for the topic being deleted
@@ -263,7 +263,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     CoreUtils.swallow(replicaManager.replicaFetcherManager.shutdownIdleFetcherThreads(), this)
   }
 
-  def handleUpdateMetadataRequest(request: RequestChannel.Request) {
+  def handleUpdateMetadataRequest(request: RequestChannel.Request): Unit = {
     val correlationId = request.header.correlationId
     val updateMetadataRequest = request.body[UpdateMetadataRequest]
 
@@ -300,7 +300,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     }
   }
 
-  def handleControlledShutdownRequest(request: RequestChannel.Request) {
+  def handleControlledShutdownRequest(request: RequestChannel.Request): Unit = {
     // ensureTopicExists is only for client facing requests
     // We can't have the ensureTopicExists check here since the controller sends it as an advisory to all brokers so they
     // stop serving data to clients for the topic being deleted
@@ -323,14 +323,14 @@ class KafkaApis(val requestChannel: RequestChannel,
   /**
    * Handle an offset commit request
    */
-  def handleOffsetCommitRequest(request: RequestChannel.Request) {
+  def handleOffsetCommitRequest(request: RequestChannel.Request): Unit = {
     val header = request.header
     val offsetCommitRequest = request.body[OffsetCommitRequest]
 
     val unauthorizedTopicErrors = mutable.Map[TopicPartition, Errors]()
     val nonExistingTopicErrors = mutable.Map[TopicPartition, Errors]()
     // the callback for sending an offset commit response
-    def sendResponseCallback(commitStatus: Map[TopicPartition, Errors]) {
+    def sendResponseCallback(commitStatus: Map[TopicPartition, Errors]): Unit = {
       val combinedCommitStatus = commitStatus ++ unauthorizedTopicErrors ++ nonExistingTopicErrors
       if (isDebugEnabled)
         combinedCommitStatus.foreach { case (topicPartition, error) =>
@@ -455,7 +455,7 @@ class KafkaApis(val requestChannel: RequestChannel,
   /**
    * Handle a produce request
    */
-  def handleProduceRequest(request: RequestChannel.Request) {
+  def handleProduceRequest(request: RequestChannel.Request): Unit = {
     val produceRequest = request.body[ProduceRequest]
     val numBytesAppended = request.header.toStruct.sizeOf + request.sizeOfBodyInBytes
 
@@ -494,7 +494,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     }
 
     // the callback for sending a produce response
-    def sendResponseCallback(responseStatus: Map[TopicPartition, PartitionResponse]) {
+    def sendResponseCallback(responseStatus: Map[TopicPartition, PartitionResponse]): Unit = {
       val mergedResponseStatus = responseStatus ++ unauthorizedTopicResponses ++ nonExistingTopicResponses ++ invalidRequestResponses
       var errorInResponse = false
 
@@ -581,7 +581,7 @@ class KafkaApis(val requestChannel: RequestChannel,
   /**
    * Handle a fetch request
    */
-  def handleFetchRequest(request: RequestChannel.Request) {
+  def handleFetchRequest(request: RequestChannel.Request): Unit = {
     val versionId = request.header.apiVersion
     val clientId = request.header.clientId
     val fetchRequest = request.body[FetchRequest]
@@ -842,7 +842,7 @@ class KafkaApis(val requestChannel: RequestChannel,
   def replicationQuota(fetchRequest: FetchRequest): ReplicaQuota =
     if (fetchRequest.isFromFollower) quotas.leader else UnboundedQuota
 
-  def handleListOffsetRequest(request: RequestChannel.Request) {
+  def handleListOffsetRequest(request: RequestChannel.Request): Unit = {
     val version = request.header.apiVersion()
 
     val mergedResponseMap = if (version == 0)
@@ -1065,7 +1065,7 @@ class KafkaApis(val requestChannel: RequestChannel,
   /**
    * Handle a topic metadata request
    */
-  def handleTopicMetadataRequest(request: RequestChannel.Request) {
+  def handleTopicMetadataRequest(request: RequestChannel.Request): Unit = {
     val metadataRequest = request.body[MetadataRequest]
     val requestVersion = request.header.apiVersion
 
@@ -1152,7 +1152,7 @@ class KafkaApis(val requestChannel: RequestChannel,
   /**
    * Handle an offset fetch request
    */
-  def handleOffsetFetchRequest(request: RequestChannel.Request) {
+  def handleOffsetFetchRequest(request: RequestChannel.Request): Unit = {
     val header = request.header
     val offsetFetchRequest = request.body[OffsetFetchRequest]
 
@@ -1226,7 +1226,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     sendResponseMaybeThrottle(request, createResponse)
   }
 
-  def handleFindCoordinatorRequest(request: RequestChannel.Request) {
+  def handleFindCoordinatorRequest(request: RequestChannel.Request): Unit = {
     val findCoordinatorRequest = request.body[FindCoordinatorRequest]
 
     if (findCoordinatorRequest.data.keyType == CoordinatorType.GROUP.id &&
@@ -1286,7 +1286,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     }
   }
 
-  def handleDescribeGroupRequest(request: RequestChannel.Request) {
+  def handleDescribeGroupRequest(request: RequestChannel.Request): Unit = {
 
     def sendResponseCallback(describeGroupsResponseData: DescribeGroupsResponseData): Unit = {
       def createResponse(requestThrottleMs: Int): AbstractResponse = {
@@ -1349,7 +1349,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     Utils.to32BitField(authorizedOps.map(operation => operation.toJava.code().asInstanceOf[JByte]).asJava)
   }
 
-  def handleListGroupsRequest(request: RequestChannel.Request) {
+  def handleListGroupsRequest(request: RequestChannel.Request): Unit = {
     val (error, groups) = groupCoordinator.handleListGroups()
     if (authorize(request.session, Describe, Resource.ClusterResource))
       // With describe cluster access all groups are returned. We keep this alternative for backward compatibility.
@@ -1376,11 +1376,11 @@ class KafkaApis(val requestChannel: RequestChannel,
     }
   }
 
-  def handleJoinGroupRequest(request: RequestChannel.Request) {
+  def handleJoinGroupRequest(request: RequestChannel.Request): Unit = {
     val joinGroupRequest = request.body[JoinGroupRequest]
 
     // the callback for sending a join-group response
-    def sendResponseCallback(joinResult: JoinGroupResult) {
+    def sendResponseCallback(joinResult: JoinGroupResult): Unit = {
       def createResponse(requestThrottleMs: Int): AbstractResponse = {
         val responseBody = new JoinGroupResponse(
           new JoinGroupResponseData()
@@ -1450,10 +1450,10 @@ class KafkaApis(val requestChannel: RequestChannel,
     }
   }
 
-  def handleSyncGroupRequest(request: RequestChannel.Request) {
+  def handleSyncGroupRequest(request: RequestChannel.Request): Unit = {
     val syncGroupRequest = request.body[SyncGroupRequest]
 
-    def sendResponseCallback(syncGroupResult: SyncGroupResult) {
+    def sendResponseCallback(syncGroupResult: SyncGroupResult): Unit = {
       sendResponseMaybeThrottle(request, requestThrottleMs =>
         new SyncGroupResponse(
           new SyncGroupResponseData()
@@ -1514,11 +1514,11 @@ class KafkaApis(val requestChannel: RequestChannel,
     })
   }
 
-  def handleHeartbeatRequest(request: RequestChannel.Request) {
+  def handleHeartbeatRequest(request: RequestChannel.Request): Unit = {
     val heartbeatRequest = request.body[HeartbeatRequest]
 
     // the callback for sending a heartbeat response
-    def sendResponseCallback(error: Errors) {
+    def sendResponseCallback(error: Errors): Unit = {
       def createResponse(requestThrottleMs: Int): AbstractResponse = {
         val response = new HeartbeatResponse(
           new HeartbeatResponseData()
@@ -1553,7 +1553,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     }
   }
 
-  def handleLeaveGroupRequest(request: RequestChannel.Request) {
+  def handleLeaveGroupRequest(request: RequestChannel.Request): Unit = {
     val leaveGroupRequest = request.body[LeaveGroupRequest]
 
     val members = leaveGroupRequest.members().asScala.toList
@@ -1566,7 +1566,7 @@ class KafkaApis(val requestChannel: RequestChannel,
         )
       })
     } else {
-      def sendResponseCallback(leaveGroupResult : LeaveGroupResult) {
+      def sendResponseCallback(leaveGroupResult : LeaveGroupResult): Unit = {
         val memberResponses = leaveGroupResult.memberResponses.map(
           leaveGroupResult =>
             new MemberResponse()
@@ -1591,19 +1591,19 @@ class KafkaApis(val requestChannel: RequestChannel,
     }
   }
 
-  def handleSaslHandshakeRequest(request: RequestChannel.Request) {
+  def handleSaslHandshakeRequest(request: RequestChannel.Request): Unit = {
     val responseData = new SaslHandshakeResponseData().setErrorCode(Errors.ILLEGAL_SASL_STATE.code)
     sendResponseMaybeThrottle(request, _ => new SaslHandshakeResponse(responseData))
   }
 
-  def handleSaslAuthenticateRequest(request: RequestChannel.Request) {
+  def handleSaslAuthenticateRequest(request: RequestChannel.Request): Unit = {
     val responseData = new SaslAuthenticateResponseData()
       .setErrorCode(Errors.ILLEGAL_SASL_STATE.code)
       .setErrorMessage("SaslAuthenticate request received after successful authentication")
     sendResponseMaybeThrottle(request, _ => new SaslAuthenticateResponse(responseData))
   }
 
-  def handleApiVersionsRequest(request: RequestChannel.Request) {
+  def handleApiVersionsRequest(request: RequestChannel.Request): Unit = {
     // Note that broker returns its full list of supported ApiKeys and versions regardless of current
     // authentication state (e.g., before SASL authentication on an SASL listener, do note that no
     // Kafka protocol requests may take place on a SSL listener before the SSL handshake is finished).
@@ -1621,7 +1621,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     sendResponseMaybeThrottle(request, createResponseCallback)
   }
 
-  def handleCreateTopicsRequest(request: RequestChannel.Request) {
+  def handleCreateTopicsRequest(request: RequestChannel.Request): Unit = {
     def sendResponseCallback(results: CreatableTopicResultCollection): Unit = {
       def createResponse(requestThrottleMs: Int): AbstractResponse = {
         val responseData = new CreateTopicsResponseData().
@@ -1718,7 +1718,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     }
   }
 
-  def handleDeleteTopicsRequest(request: RequestChannel.Request) {
+  def handleDeleteTopicsRequest(request: RequestChannel.Request): Unit = {
     def sendResponseCallback(results: DeletableTopicResultCollection): Unit = {
       def createResponse(requestThrottleMs: Int): AbstractResponse = {
         val responseData = new DeleteTopicsResponseData()
@@ -1784,7 +1784,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     }
   }
 
-  def handleDeleteRecordsRequest(request: RequestChannel.Request) {
+  def handleDeleteRecordsRequest(request: RequestChannel.Request): Unit = {
     val deleteRecordsRequest = request.body[DeleteRecordsRequest]
 
     val unauthorizedTopicResponses = mutable.Map[TopicPartition, DeleteRecordsResponse.PartitionResponse]()
@@ -1803,7 +1803,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     }
 
     // the callback for sending a DeleteRecordsResponse
-    def sendResponseCallback(authorizedTopicResponses: Map[TopicPartition, DeleteRecordsResponse.PartitionResponse]) {
+    def sendResponseCallback(authorizedTopicResponses: Map[TopicPartition, DeleteRecordsResponse.PartitionResponse]): Unit = {
       val mergedResponseStatus = authorizedTopicResponses ++ unauthorizedTopicResponses ++ nonExistingTopicResponses
       mergedResponseStatus.foreach { case (topicPartition, status) =>
         if (status.error != Errors.NONE) {
@@ -1866,7 +1866,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     val transactionalId = endTxnRequest.transactionalId
 
     if (authorize(request.session, Write, Resource(TransactionalId, transactionalId, LITERAL))) {
-      def sendResponseCallback(error: Errors) {
+      def sendResponseCallback(error: Errors): Unit = {
         def createResponse(requestThrottleMs: Int): AbstractResponse = {
           val responseBody = new EndTxnResponse(requestThrottleMs, error)
           trace(s"Completed ${endTxnRequest.transactionalId}'s EndTxnRequest with command: ${endTxnRequest.command}, errors: $error from client ${request.header.clientId}.")
@@ -2105,7 +2105,7 @@ class KafkaApis(val requestChannel: RequestChannel,
       }
 
       // the callback for sending an offset commit response
-      def sendResponseCallback(authorizedTopicErrors: Map[TopicPartition, Errors]) {
+      def sendResponseCallback(authorizedTopicErrors: Map[TopicPartition, Errors]): Unit = {
         val combinedCommitStatus = authorizedTopicErrors ++ unauthorizedTopicErrors ++ nonExistingTopicErrors
         if (isDebugEnabled)
           combinedCommitStatus.foreach { case (topicPartition, error) =>
@@ -2421,11 +2421,11 @@ class KafkaApis(val requestChannel: RequestChannel,
     sendResponseMaybeThrottle(request, throttleTimeMs => new DescribeLogDirsResponse(throttleTimeMs, logDirInfos.asJava))
   }
 
-  def handleCreateTokenRequest(request: RequestChannel.Request) {
+  def handleCreateTokenRequest(request: RequestChannel.Request): Unit = {
     val createTokenRequest = request.body[CreateDelegationTokenRequest]
 
     // the callback for sending a create token response
-    def sendResponseCallback(createResult: CreateTokenResult) {
+    def sendResponseCallback(createResult: CreateTokenResult): Unit = {
       trace("Sending create token response for correlation id %d to client %s."
         .format(request.header.correlationId, request.header.clientId))
       sendResponseMaybeThrottle(request, requestThrottleMs =>
@@ -2455,11 +2455,11 @@ class KafkaApis(val requestChannel: RequestChannel,
     }
   }
 
-  def handleRenewTokenRequest(request: RequestChannel.Request) {
+  def handleRenewTokenRequest(request: RequestChannel.Request): Unit = {
     val renewTokenRequest = request.body[RenewDelegationTokenRequest]
 
     // the callback for sending a renew token response
-    def sendResponseCallback(error: Errors, expiryTimestamp: Long) {
+    def sendResponseCallback(error: Errors, expiryTimestamp: Long): Unit = {
       trace("Sending renew token response %s for correlation id %d to client %s."
         .format(request.header.correlationId, request.header.clientId))
       sendResponseMaybeThrottle(request, requestThrottleMs =>
@@ -2482,11 +2482,11 @@ class KafkaApis(val requestChannel: RequestChannel,
     }
   }
 
-  def handleExpireTokenRequest(request: RequestChannel.Request) {
+  def handleExpireTokenRequest(request: RequestChannel.Request): Unit = {
     val expireTokenRequest = request.body[ExpireDelegationTokenRequest]
 
     // the callback for sending a expire token response
-    def sendResponseCallback(error: Errors, expiryTimestamp: Long) {
+    def sendResponseCallback(error: Errors, expiryTimestamp: Long): Unit = {
       trace("Sending expire token response for correlation id %d to client %s."
         .format(request.header.correlationId, request.header.clientId))
       sendResponseMaybeThrottle(request, requestThrottleMs =>
@@ -2509,11 +2509,11 @@ class KafkaApis(val requestChannel: RequestChannel,
     }
   }
 
-  def handleDescribeTokensRequest(request: RequestChannel.Request) {
+  def handleDescribeTokensRequest(request: RequestChannel.Request): Unit = {
     val describeTokenRequest = request.body[DescribeDelegationTokenRequest]
 
     // the callback for sending a describe token response
-    def sendResponseCallback(error: Errors, tokenDetails: List[DelegationToken]) {
+    def sendResponseCallback(error: Errors, tokenDetails: List[DelegationToken]): Unit = {
       sendResponseMaybeThrottle(request, requestThrottleMs =>
         new DescribeDelegationTokenResponse(requestThrottleMs, error, tokenDetails.asJava))
       trace("Sending describe token response for correlation id %d to client %s."
@@ -2664,7 +2664,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     request.temporaryMemoryBytes = conversionStats.temporaryMemoryBytes
   }
 
-  private def handleError(request: RequestChannel.Request, e: Throwable) {
+  private def handleError(request: RequestChannel.Request, e: Throwable): Unit = {
     val mayThrottle = e.isInstanceOf[ClusterAuthorizationException] || !request.header.apiKey.clusterAction
     error("Error when handling request: " +
       s"clientId=${request.header.clientId}, " +
@@ -2687,7 +2687,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     sendResponse(request, Some(createResponse(throttleTimeMs)), onComplete)
   }
 
-  private def sendErrorResponseMaybeThrottle(request: RequestChannel.Request, error: Throwable) {
+  private def sendErrorResponseMaybeThrottle(request: RequestChannel.Request, error: Throwable): Unit = {
     val throttleTimeMs = quotas.request.maybeRecordAndGetThrottleTimeMs(request)
     quotas.request.throttle(request, throttleTimeMs, sendResponse)
     sendErrorOrCloseConnection(request, error, throttleTimeMs)
diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index a2db65379796a..74c397b9190d5 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -254,7 +254,7 @@ object KafkaConfig {
 
   private val LogConfigPrefix = "log."
 
-  def main(args: Array[String]) {
+  def main(args: Array[String]): Unit = {
     System.out.println(configDef.toHtmlTable(DynamicBrokerConfig.dynamicConfigUpdateModes))
   }
 
@@ -1463,7 +1463,7 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO
 
   validateValues()
 
-  private def validateValues() {
+  private def validateValues(): Unit = {
     if(brokerIdGenerationEnable) {
       require(brokerId >= -1 && brokerId <= maxReservedBrokerId, "broker.id must be equal or greater than -1 and not greater than reserved.broker.max.id")
     } else {
diff --git a/core/src/main/scala/kafka/server/KafkaRequestHandler.scala b/core/src/main/scala/kafka/server/KafkaRequestHandler.scala
index 0397795e6db9e..f6999d30d8ddf 100755
--- a/core/src/main/scala/kafka/server/KafkaRequestHandler.scala
+++ b/core/src/main/scala/kafka/server/KafkaRequestHandler.scala
@@ -43,7 +43,7 @@ class KafkaRequestHandler(id: Int,
   private val shutdownComplete = new CountDownLatch(1)
   @volatile private var stopped = false
 
-  def run() {
+  def run(): Unit = {
     while (!stopped) {
       // We use a single meter for aggregate idle percentage for the thread pool.
       // Since meter is calculated as total_recorded_value / time_window and
@@ -188,7 +188,7 @@ class BrokerTopicMetrics(name: Option[String]) extends KafkaMetricsGroup {
     }
   }
 
-  def close() {
+  def close(): Unit = {
     removeMetricHelper(BrokerTopicStats.MessagesInPerSec, tags)
     removeMetricHelper(BrokerTopicStats.BytesInPerSec, tags)
     removeMetricHelper(BrokerTopicStats.BytesOutPerSec, tags)
@@ -231,20 +231,20 @@ class BrokerTopicStats {
   def topicStats(topic: String): BrokerTopicMetrics =
     stats.getAndMaybePut(topic)
 
-  def updateReplicationBytesIn(value: Long) {
+  def updateReplicationBytesIn(value: Long): Unit = {
     allTopicsStats.replicationBytesInRate.foreach { metric =>
       metric.mark(value)
     }
   }
 
-  private def updateReplicationBytesOut(value: Long) {
+  private def updateReplicationBytesOut(value: Long): Unit = {
     allTopicsStats.replicationBytesOutRate.foreach { metric =>
       metric.mark(value)
     }
   }
 
   // This method only removes metrics only used for leader
-  def removeOldLeaderMetrics(topic: String) {
+  def removeOldLeaderMetrics(topic: String): Unit = {
     val topicMetrics = topicStats(topic)
     if (topicMetrics != null) {
       topicMetrics.removeMetricHelper(BrokerTopicStats.MessagesInPerSec, topicMetrics.tags)
@@ -264,13 +264,13 @@ class BrokerTopicStats {
       topicMetrics.removeMetricHelper(BrokerTopicStats.ReplicationBytesInPerSec, topicMetrics.tags)
   }
 
-  def removeMetrics(topic: String) {
+  def removeMetrics(topic: String): Unit = {
     val metrics = stats.remove(topic)
     if (metrics != null)
       metrics.close()
   }
 
-  def updateBytesOut(topic: String, isFollower: Boolean, value: Long) {
+  def updateBytesOut(topic: String, isFollower: Boolean, value: Long): Unit = {
     if (isFollower) {
       updateReplicationBytesOut(value)
     } else {
diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala
index 21c53b3b91235..68eef3ab215b0 100755
--- a/core/src/main/scala/kafka/server/KafkaServer.scala
+++ b/core/src/main/scala/kafka/server/KafkaServer.scala
@@ -189,7 +189,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
    * Start up API for bringing up a single instance of the Kafka server.
    * Instantiates the LogManager, the SocketServer and the request handlers - KafkaRequestHandlers
    */
-  def startup() {
+  def startup(): Unit = {
     try {
       info("starting")
 
@@ -429,7 +429,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
   /**
    * Performs controlled shutdown
    */
-  private def controlledShutdown() {
+  private def controlledShutdown(): Unit = {
 
     def node(broker: Broker): Node = broker.node(config.interBrokerListenerName)
 
@@ -584,7 +584,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
    * Shutdown API for shutting down a single instance of the Kafka server.
    * Shuts down the LogManager, the SocketServer and the log cleaner scheduler thread
    */
-  def shutdown() {
+  def shutdown(): Unit = {
     try {
       info("shutting down")
 
@@ -724,6 +724,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
       (BrokerMetadata(-1, None), offlineDirs)
   }
 
+
   /**
    * Checkpoint the BrokerMetadata to all the online log.dirs
    *
diff --git a/core/src/main/scala/kafka/server/KafkaServerStartable.scala b/core/src/main/scala/kafka/server/KafkaServerStartable.scala
index f5fc8ce91f704..515162cbb4928 100644
--- a/core/src/main/scala/kafka/server/KafkaServerStartable.scala
+++ b/core/src/main/scala/kafka/server/KafkaServerStartable.scala
@@ -36,7 +36,7 @@ class KafkaServerStartable(val staticServerConfig: KafkaConfig, reporters: Seq[K
 
   def this(serverConfig: KafkaConfig) = this(serverConfig, Seq.empty)
 
-  def startup() {
+  def startup(): Unit = {
     try server.startup()
     catch {
       case _: Throwable =>
@@ -46,7 +46,7 @@ class KafkaServerStartable(val staticServerConfig: KafkaConfig, reporters: Seq[K
     }
   }
 
-  def shutdown() {
+  def shutdown(): Unit = {
     try server.shutdown()
     catch {
       case _: Throwable =>
@@ -60,7 +60,7 @@ class KafkaServerStartable(val staticServerConfig: KafkaConfig, reporters: Seq[K
    * Allow setting broker state from the startable.
    * This is needed when a custom kafka server startable want to emit new states that it introduces.
    */
-  def setServerState(newState: Byte) {
+  def setServerState(newState: Byte): Unit = {
     server.brokerState.newState(newState)
   }
 
diff --git a/core/src/main/scala/kafka/server/MetadataCache.scala b/core/src/main/scala/kafka/server/MetadataCache.scala
index 7d10bb004251e..e0203478e0776 100755
--- a/core/src/main/scala/kafka/server/MetadataCache.scala
+++ b/core/src/main/scala/kafka/server/MetadataCache.scala
@@ -169,7 +169,7 @@ class MetadataCache(brokerId: Int) extends Logging {
   private def addOrUpdatePartitionInfo(partitionStates: mutable.AnyRefMap[String, mutable.LongMap[UpdateMetadataRequest.PartitionState]],
                                        topic: String,
                                        partitionId: Int,
-                                       stateInfo: UpdateMetadataRequest.PartitionState) {
+                                       stateInfo: UpdateMetadataRequest.PartitionState): Unit = {
     val infos = partitionStates.getOrElseUpdate(topic, mutable.LongMap())
     infos(partitionId) = stateInfo
   }
diff --git a/core/src/main/scala/kafka/server/QuotaFactory.scala b/core/src/main/scala/kafka/server/QuotaFactory.scala
index ed04dcf096fb8..c941163d017c4 100644
--- a/core/src/main/scala/kafka/server/QuotaFactory.scala
+++ b/core/src/main/scala/kafka/server/QuotaFactory.scala
@@ -48,7 +48,7 @@ object QuotaFactory extends Logging {
                            follower: ReplicationQuotaManager,
                            alterLogDirs: ReplicationQuotaManager,
                            clientQuotaCallback: Option[ClientQuotaCallback]) {
-    def shutdown() {
+    def shutdown(): Unit = {
       fetch.shutdown
       produce.shutdown
       request.shutdown
diff --git a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsManager.scala b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsManager.scala
index d5ab4d6e549a2..5b6c5b18da20f 100644
--- a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsManager.scala
@@ -34,7 +34,7 @@ class ReplicaAlterLogDirsManager(brokerConfig: KafkaConfig,
       quotaManager, brokerTopicStats)
   }
 
-  def shutdown() {
+  def shutdown(): Unit = {
     info("shutting down")
     closeAllFetchers()
     info("shutdown completed")
diff --git a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
index 24dbb3dbd36cc..fdb2bfd6c0365 100644
--- a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
+++ b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
@@ -71,7 +71,7 @@ class ReplicaAlterLogDirsThread(name: String,
     var partitionData: Seq[(TopicPartition, FetchResponse.PartitionData[Records])] = null
     val request = fetchRequest.build()
 
-    def processResponseCallback(responsePartitionData: Seq[(TopicPartition, FetchPartitionData)]) {
+    def processResponseCallback(responsePartitionData: Seq[(TopicPartition, FetchPartitionData)]): Unit = {
       partitionData = responsePartitionData.map { case (tp, data) =>
         val abortedTransactions = data.abortedTransactions.map(_.asJava).orNull
         val lastStableOffset = data.lastStableOffset.getOrElse(FetchResponse.INVALID_LAST_STABLE_OFFSET)
diff --git a/core/src/main/scala/kafka/server/ReplicaFetcherBlockingSend.scala b/core/src/main/scala/kafka/server/ReplicaFetcherBlockingSend.scala
index 8e631fec00f7a..ee2dd608c0a4d 100644
--- a/core/src/main/scala/kafka/server/ReplicaFetcherBlockingSend.scala
+++ b/core/src/main/scala/kafka/server/ReplicaFetcherBlockingSend.scala
@@ -35,9 +35,9 @@ trait BlockingSend {
 
   def sendRequest(requestBuilder: AbstractRequest.Builder[_ <: AbstractRequest]): ClientResponse
 
-  def initiateClose()
+  def initiateClose(): Unit
 
-  def close()
+  def close(): Unit
 }
 
 class ReplicaFetcherBlockingSend(sourceBroker: BrokerEndPoint,
diff --git a/core/src/main/scala/kafka/server/ReplicaFetcherManager.scala b/core/src/main/scala/kafka/server/ReplicaFetcherManager.scala
index 3426290b7a78c..d547e1b5d769b 100644
--- a/core/src/main/scala/kafka/server/ReplicaFetcherManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaFetcherManager.scala
@@ -39,7 +39,7 @@ class ReplicaFetcherManager(brokerConfig: KafkaConfig,
       metrics, time, quotaManager)
   }
 
-  def shutdown() {
+  def shutdown(): Unit = {
     info("shutting down")
     closeAllFetchers()
     info("shutdown completed")
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index 1e61d76e869a4..12cb507d35005 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -216,7 +216,7 @@ class ReplicaManager(val config: KafkaConfig,
   private var logDirFailureHandler: LogDirFailureHandler = null
 
   private class LogDirFailureHandler(name: String, haltBrokerOnDirFailure: Boolean) extends ShutdownableThread(name) {
-    override def doWork() {
+    override def doWork(): Unit = {
       val newOfflineLogDir = logDirFailureChannel.takeNextOfflineLogDir()
       if (haltBrokerOnDirFailure) {
         fatal(s"Halting broker because dir $newOfflineLogDir is offline")
@@ -276,7 +276,7 @@ class ReplicaManager(val config: KafkaConfig,
       scheduler.schedule("highwatermark-checkpoint", checkpointHighWatermarks _, period = config.replicaHighWatermarkCheckpointIntervalMs, unit = TimeUnit.MILLISECONDS)
   }
 
-  def recordIsrChange(topicPartition: TopicPartition) {
+  def recordIsrChange(topicPartition: TopicPartition): Unit = {
     isrChangeSet synchronized {
       isrChangeSet += topicPartition
       lastIsrChangeMs.set(System.currentTimeMillis())
@@ -289,7 +289,7 @@ class ReplicaManager(val config: KafkaConfig,
    * This allows an occasional ISR change to be propagated within a few seconds, and avoids overwhelming controller and
    * other brokers when large amount of ISR change occurs.
    */
-  def maybePropagateIsrChanges() {
+  def maybePropagateIsrChanges(): Unit = {
     val now = System.currentTimeMillis()
     isrChangeSet synchronized {
       if (isrChangeSet.nonEmpty &&
@@ -319,7 +319,7 @@ class ReplicaManager(val config: KafkaConfig,
     debug("Request key %s unblocked %d ElectLeader.".format(key.keyLabel, completed))
   }
 
-  def startup() {
+  def startup(): Unit = {
     // start ISR expiration thread
     // A follower can lag behind leader for up to config.replicaLagTimeMaxMs x 1.5 before it is removed from ISR
     scheduler.schedule("isr-expiration", maybeShrinkIsr _, period = config.replicaLagTimeMaxMs / 2, unit = TimeUnit.MILLISECONDS)
@@ -485,7 +485,7 @@ class ReplicaManager(val config: KafkaConfig,
                     entriesPerPartition: Map[TopicPartition, MemoryRecords],
                     responseCallback: Map[TopicPartition, PartitionResponse] => Unit,
                     delayedProduceLock: Option[Lock] = None,
-                    recordConversionStatsCallback: Map[TopicPartition, RecordConversionStats] => Unit = _ => ()) {
+                    recordConversionStatsCallback: Map[TopicPartition, RecordConversionStats] => Unit = _ => ()): Unit = {
     if (isValidRequiredAcks(requiredAcks)) {
       val sTime = time.milliseconds
       val localProduceResults = appendToLocalLog(internalTopicsAllowed = internalTopicsAllowed,
@@ -696,7 +696,7 @@ class ReplicaManager(val config: KafkaConfig,
 
   def deleteRecords(timeout: Long,
                     offsetPerPartition: Map[TopicPartition, Long],
-                    responseCallback: Map[TopicPartition, DeleteRecordsResponse.PartitionResponse] => Unit) {
+                    responseCallback: Map[TopicPartition, DeleteRecordsResponse.PartitionResponse] => Unit): Unit = {
     val timeBeforeLocalDeleteRecords = time.milliseconds
     val localDeleteRecordsResults = deleteRecordsOnLocalLog(offsetPerPartition)
     debug("Delete records on local log in %d ms".format(time.milliseconds - timeBeforeLocalDeleteRecords))
@@ -832,7 +832,7 @@ class ReplicaManager(val config: KafkaConfig,
                     quota: ReplicaQuota,
                     responseCallback: Seq[(TopicPartition, FetchPartitionData)] => Unit,
                     isolationLevel: IsolationLevel,
-                    clientMetadata: Option[ClientMetadata]) {
+                    clientMetadata: Option[ClientMetadata]): Unit = {
     val isFromFollower = Request.isValidBrokerId(replicaId)
 
     val fetchIsolation = if (isFromFollower || replicaId == Request.FutureLocalReplicaId)
@@ -1565,7 +1565,7 @@ class ReplicaManager(val config: KafkaConfig,
     nonOfflinePartition(topicPartition).flatMap(_.leaderLogIfLocal.map(_.logEndOffset))
 
   // Flushes the highwatermark value for all partitions to the highwatermark file
-  def checkpointHighWatermarks() {
+  def checkpointHighWatermarks(): Unit = {
     val localLogs = nonOfflinePartitionsIterator.flatMap { partition =>
       val logsList: mutable.Set[Log] = mutable.Set()
       partition.log.foreach(logsList.add)
@@ -1592,7 +1592,7 @@ class ReplicaManager(val config: KafkaConfig,
 
   // logDir should be an absolute path
   // sendZkNotification is needed for unit test
-  def handleLogDirFailure(dir: String, sendZkNotification: Boolean = true) {
+  def handleLogDirFailure(dir: String, sendZkNotification: Boolean = true): Unit = {
     if (!logManager.isLogDirOnline(dir))
       return
     info(s"Stopping serving replicas in dir $dir")
@@ -1627,7 +1627,7 @@ class ReplicaManager(val config: KafkaConfig,
     info(s"Stopped serving replicas in dir $dir")
   }
 
-  def removeMetrics() {
+  def removeMetrics(): Unit = {
     removeMetric("LeaderCount")
     removeMetric("PartitionCount")
     removeMetric("OfflineReplicaCount")
@@ -1637,7 +1637,7 @@ class ReplicaManager(val config: KafkaConfig,
   }
 
   // High watermark do not need to be checkpointed only when under unit tests
-  def shutdown(checkpointHW: Boolean = true) {
+  def shutdown(checkpointHW: Boolean = true): Unit = {
     info("Shutting down")
     removeMetrics()
     if (logDirFailureHandler != null)
diff --git a/core/src/main/scala/kafka/server/ReplicationQuotaManager.scala b/core/src/main/scala/kafka/server/ReplicationQuotaManager.scala
index 59d8172a1f714..c4024645564cf 100644
--- a/core/src/main/scala/kafka/server/ReplicationQuotaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicationQuotaManager.scala
@@ -86,7 +86,7 @@ class ReplicationQuotaManager(val config: ReplicationQuotaManagerConfig,
     *
     * @param quota
     */
-  def updateQuota(quota: Quota) {
+  def updateQuota(quota: Quota): Unit = {
     inWriteLock(lock) {
       this.quota = quota
       //The metric could be expired by another thread, so use a local variable and null check.
@@ -132,7 +132,7 @@ class ReplicationQuotaManager(val config: ReplicationQuotaManagerConfig,
     *
     * @param value
     */
-  def record(value: Long) {
+  def record(value: Long): Unit = {
     try {
       sensor().record(value)
     } catch {
@@ -149,7 +149,7 @@ class ReplicationQuotaManager(val config: ReplicationQuotaManagerConfig,
     * @param partitions the set of throttled partitions
     * @return
     */
-  def markThrottled(topic: String, partitions: Seq[Int]) {
+  def markThrottled(topic: String, partitions: Seq[Int]): Unit = {
     throttledPartitions.put(topic, partitions)
   }
 
@@ -159,7 +159,7 @@ class ReplicationQuotaManager(val config: ReplicationQuotaManagerConfig,
     * @param topic
     * @return
     */
-  def markThrottled(topic: String) {
+  def markThrottled(topic: String): Unit = {
     markThrottled(topic, AllReplicas)
   }
 
@@ -169,7 +169,7 @@ class ReplicationQuotaManager(val config: ReplicationQuotaManagerConfig,
     * @param topic
     * @return
     */
-  def removeThrottle(topic: String) {
+  def removeThrottle(topic: String): Unit = {
     throttledPartitions.remove(topic)
   }
 
diff --git a/core/src/main/scala/kafka/server/checkpoints/CheckpointFile.scala b/core/src/main/scala/kafka/server/checkpoints/CheckpointFile.scala
index c14c1ef5bc8d9..8a05775e0869c 100644
--- a/core/src/main/scala/kafka/server/checkpoints/CheckpointFile.scala
+++ b/core/src/main/scala/kafka/server/checkpoints/CheckpointFile.scala
@@ -45,7 +45,7 @@ class CheckpointFile[T](val file: File,
   try Files.createFile(file.toPath) // create the file if it doesn't exist
   catch { case _: FileAlreadyExistsException => }
 
-  def write(entries: Iterable[T]) {
+  def write(entries: Iterable[T]): Unit = {
     lock synchronized {
       try {
         // write to temp file and then swap with the existing file
diff --git a/core/src/main/scala/kafka/server/checkpoints/LeaderEpochCheckpointFile.scala b/core/src/main/scala/kafka/server/checkpoints/LeaderEpochCheckpointFile.scala
index 79a11c7dcc893..6b88bebf6b4a4 100644
--- a/core/src/main/scala/kafka/server/checkpoints/LeaderEpochCheckpointFile.scala
+++ b/core/src/main/scala/kafka/server/checkpoints/LeaderEpochCheckpointFile.scala
@@ -25,7 +25,7 @@ import kafka.server.epoch.EpochEntry
 import scala.collection._
 
 trait LeaderEpochCheckpoint {
-  def write(epochs: Seq[EpochEntry])
+  def write(epochs: Seq[EpochEntry]): Unit
   def read(): Seq[EpochEntry]
 }
 
diff --git a/core/src/main/scala/kafka/server/checkpoints/OffsetCheckpointFile.scala b/core/src/main/scala/kafka/server/checkpoints/OffsetCheckpointFile.scala
index b33036825722d..fbe2f6e2bb036 100644
--- a/core/src/main/scala/kafka/server/checkpoints/OffsetCheckpointFile.scala
+++ b/core/src/main/scala/kafka/server/checkpoints/OffsetCheckpointFile.scala
@@ -45,7 +45,7 @@ object OffsetCheckpointFile {
 }
 
 trait OffsetCheckpoint {
-  def write(epochs: Seq[EpochEntry])
+  def write(epochs: Seq[EpochEntry]): Unit
   def read(): Seq[EpochEntry]
 }
 
diff --git a/core/src/main/scala/kafka/tools/ConsoleConsumer.scala b/core/src/main/scala/kafka/tools/ConsoleConsumer.scala
index d246e7b9007b3..22917c9594146 100755
--- a/core/src/main/scala/kafka/tools/ConsoleConsumer.scala
+++ b/core/src/main/scala/kafka/tools/ConsoleConsumer.scala
@@ -48,7 +48,7 @@ object ConsoleConsumer extends Logging {
 
   private val shutdownLatch = new CountDownLatch(1)
 
-  def main(args: Array[String]) {
+  def main(args: Array[String]): Unit = {
     val conf = new ConsumerConfig(args)
     try {
       run(conf)
@@ -62,7 +62,7 @@ object ConsoleConsumer extends Logging {
     }
   }
 
-  def run(conf: ConsumerConfig) {
+  def run(conf: ConsumerConfig): Unit = {
     val timeoutMs = if (conf.timeoutMs >= 0) conf.timeoutMs else Long.MaxValue
     val consumer = new KafkaConsumer(consumerProps(conf), new ByteArrayDeserializer, new ByteArrayDeserializer)
 
@@ -84,9 +84,9 @@ object ConsoleConsumer extends Logging {
     }
   }
 
-  def addShutdownHook(consumer: ConsumerWrapper, conf: ConsumerConfig) {
+  def addShutdownHook(consumer: ConsumerWrapper, conf: ConsumerConfig): Unit = {
     Runtime.getRuntime.addShutdownHook(new Thread() {
-      override def run() {
+      override def run(): Unit = {
         consumer.wakeup()
 
         shutdownLatch.await()
@@ -99,7 +99,7 @@ object ConsoleConsumer extends Logging {
   }
 
   def process(maxMessages: Integer, formatter: MessageFormatter, consumer: ConsumerWrapper, output: PrintStream,
-              skipMessageOnError: Boolean) {
+              skipMessageOnError: Boolean): Unit = {
     while (messageCount < maxMessages || maxMessages == -1) {
       val msg: ConsumerRecord[Array[Byte], Array[Byte]] = try {
         consumer.receive()
@@ -133,7 +133,7 @@ object ConsoleConsumer extends Logging {
     }
   }
 
-  def reportRecordCount() {
+  def reportRecordCount(): Unit = {
     System.err.println(s"Processed a total of $messageCount messages")
   }
 
@@ -168,7 +168,7 @@ object ConsoleConsumer extends Logging {
     * In case both --from-beginning and an explicit value are specified an error is thrown if these
     * are conflicting.
     */
-  def setAutoOffsetResetValue(config: ConsumerConfig, props: Properties) {
+  def setAutoOffsetResetValue(config: ConsumerConfig, props: Properties): Unit = {
     val (earliestConfigValue, latestConfigValue) = ("earliest", "latest")
 
     if (props.containsKey(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG)) {
@@ -394,7 +394,7 @@ object ConsoleConsumer extends Logging {
     consumerInit()
     var recordIter = Collections.emptyList[ConsumerRecord[Array[Byte], Array[Byte]]]().iterator()
 
-    def consumerInit() {
+    def consumerInit(): Unit = {
       (topic, partitionId, offset, whitelist) match {
         case (Some(topic), Some(partitionId), Some(offset), None) =>
           seek(topic, partitionId, offset)
@@ -413,7 +413,7 @@ object ConsoleConsumer extends Logging {
       }
     }
 
-    def seek(topic: String, partitionId: Int, offset: Long) {
+    def seek(topic: String, partitionId: Int, offset: Long): Unit = {
       val topicPartition = new TopicPartition(topic, partitionId)
       consumer.assign(Collections.singletonList(topicPartition))
       offset match {
@@ -423,7 +423,7 @@ object ConsoleConsumer extends Logging {
       }
     }
 
-    def resetUnconsumedOffsets() {
+    def resetUnconsumedOffsets(): Unit = {
       val smallestUnconsumedOffsets = collection.mutable.Map[TopicPartition, Long]()
       while (recordIter.hasNext) {
         val record = recordIter.next()
@@ -448,7 +448,7 @@ object ConsoleConsumer extends Logging {
       this.consumer.wakeup()
     }
 
-    def cleanup() {
+    def cleanup(): Unit = {
       resetUnconsumedOffsets()
       this.consumer.close()
     }
@@ -466,7 +466,7 @@ class DefaultMessageFormatter extends MessageFormatter {
   var keyDeserializer: Option[Deserializer[_]] = None
   var valueDeserializer: Option[Deserializer[_]] = None
 
-  override def init(props: Properties) {
+  override def init(props: Properties): Unit = {
     if (props.containsKey("print.timestamp"))
       printTimestamp = props.getProperty("print.timestamp").trim.equalsIgnoreCase("true")
     if (props.containsKey("print.key"))
@@ -500,7 +500,7 @@ class DefaultMessageFormatter extends MessageFormatter {
     newProps
   }
 
-  def writeTo(consumerRecord: ConsumerRecord[Array[Byte], Array[Byte]], output: PrintStream) {
+  def writeTo(consumerRecord: ConsumerRecord[Array[Byte], Array[Byte]], output: PrintStream): Unit = {
 
     def writeSeparator(columnSeparator: Boolean): Unit = {
       if (columnSeparator)
@@ -509,7 +509,7 @@ class DefaultMessageFormatter extends MessageFormatter {
         output.write(lineSeparator)
     }
 
-    def write(deserializer: Option[Deserializer[_]], sourceBytes: Array[Byte], topic: String) {
+    def write(deserializer: Option[Deserializer[_]], sourceBytes: Array[Byte], topic: String): Unit = {
       val nonNullBytes = Option(sourceBytes).getOrElse("null".getBytes(StandardCharsets.UTF_8))
       val convertedBytes = deserializer.map(_.deserialize(topic, nonNullBytes).toString.
         getBytes(StandardCharsets.UTF_8)).getOrElse(nonNullBytes)
@@ -553,15 +553,15 @@ class LoggingMessageFormatter extends MessageFormatter with LazyLogging {
 }
 
 class NoOpMessageFormatter extends MessageFormatter {
-  override def init(props: Properties) {}
+  override def init(props: Properties): Unit = {}
 
-  def writeTo(consumerRecord: ConsumerRecord[Array[Byte], Array[Byte]], output: PrintStream){}
+  def writeTo(consumerRecord: ConsumerRecord[Array[Byte], Array[Byte]], output: PrintStream): Unit = {}
 }
 
 class ChecksumMessageFormatter extends MessageFormatter {
   private var topicStr: String = _
 
-  override def init(props: Properties) {
+  override def init(props: Properties): Unit = {
     topicStr = props.getProperty("topic")
     if (topicStr != null)
       topicStr = topicStr + ":"
@@ -569,7 +569,7 @@ class ChecksumMessageFormatter extends MessageFormatter {
       topicStr = ""
   }
 
-  def writeTo(consumerRecord: ConsumerRecord[Array[Byte], Array[Byte]], output: PrintStream) {
+  def writeTo(consumerRecord: ConsumerRecord[Array[Byte], Array[Byte]], output: PrintStream): Unit = {
     output.println(topicStr + "checksum:" + consumerRecord.checksum)
   }
 }
diff --git a/core/src/main/scala/kafka/tools/ConsoleProducer.scala b/core/src/main/scala/kafka/tools/ConsoleProducer.scala
index e6f526b3ea66c..16f6af89b65ad 100644
--- a/core/src/main/scala/kafka/tools/ConsoleProducer.scala
+++ b/core/src/main/scala/kafka/tools/ConsoleProducer.scala
@@ -45,7 +45,7 @@ object ConsoleProducer {
         val producer = new KafkaProducer[Array[Byte], Array[Byte]](producerProps(config))
 
         Runtime.getRuntime.addShutdownHook(new Thread() {
-          override def run() {
+          override def run(): Unit = {
             producer.close()
           }
         })
@@ -252,7 +252,7 @@ object ConsoleProducer {
     var ignoreError = false
     var lineNumber = 0
 
-    override def init(inputStream: InputStream, props: Properties) {
+    override def init(inputStream: InputStream, props: Properties): Unit = {
       topic = props.getProperty("topic")
       if (props.containsKey("parse.key"))
         parseKey = props.getProperty("parse.key").trim.equalsIgnoreCase("true")
diff --git a/core/src/main/scala/kafka/tools/ConsumerPerformance.scala b/core/src/main/scala/kafka/tools/ConsumerPerformance.scala
index 226341e4122bc..278d69486b012 100644
--- a/core/src/main/scala/kafka/tools/ConsumerPerformance.scala
+++ b/core/src/main/scala/kafka/tools/ConsumerPerformance.scala
@@ -100,7 +100,7 @@ object ConsumerPerformance extends LazyLogging {
               totalMessagesRead: AtomicLong,
               totalBytesRead: AtomicLong,
               joinTime: AtomicLong,
-              testStartTime: Long) {
+              testStartTime: Long): Unit = {
     var bytesRead = 0L
     var messagesRead = 0L
     var lastBytesRead = 0L
@@ -109,11 +109,11 @@ object ConsumerPerformance extends LazyLogging {
     var joinTimeMsInSingleRound = 0L
 
     consumer.subscribe(topics.asJava, new ConsumerRebalanceListener {
-      def onPartitionsAssigned(partitions: util.Collection[TopicPartition]) {
+      def onPartitionsAssigned(partitions: util.Collection[TopicPartition]): Unit = {
         joinTime.addAndGet(System.currentTimeMillis - joinStart)
         joinTimeMsInSingleRound += System.currentTimeMillis - joinStart
       }
-      def onPartitionsRevoked(partitions: util.Collection[TopicPartition]) {
+      def onPartitionsRevoked(partitions: util.Collection[TopicPartition]): Unit = {
         joinStart = System.currentTimeMillis
       }})
 
diff --git a/core/src/main/scala/kafka/tools/DumpLogSegments.scala b/core/src/main/scala/kafka/tools/DumpLogSegments.scala
index 5998902245ce3..5840c28bd6428 100755
--- a/core/src/main/scala/kafka/tools/DumpLogSegments.scala
+++ b/core/src/main/scala/kafka/tools/DumpLogSegments.scala
@@ -39,7 +39,7 @@ object DumpLogSegments {
   // visible for testing
   private[tools] val RecordIndent = "|"
 
-  def main(args: Array[String]) {
+  def main(args: Array[String]): Unit = {
     val opts = new DumpLogSegmentsOptions(args)
     CommandLineUtils.printHelpAndExitIfNeeded(opts, "This tool helps to parse a log file and dump its contents to the console, useful for debugging a seemingly corrupt log segment.")
     opts.checkArgs()
@@ -119,7 +119,7 @@ object DumpLogSegments {
                                indexSanityOnly: Boolean,
                                verifyOnly: Boolean,
                                misMatchesForIndexFilesMap: mutable.Map[String, List[(Long, Long)]],
-                               maxMessageSize: Int) {
+                               maxMessageSize: Int): Unit = {
     val startOffset = file.getName.split("\\.")(0).toLong
     val logFile = new File(file.getAbsoluteFile.getParent, file.getName.split("\\.")(0) + Log.LogFileSuffix)
     val fileRecords = FileRecords.open(logFile, false)
@@ -156,7 +156,7 @@ object DumpLogSegments {
                                    indexSanityOnly: Boolean,
                                    verifyOnly: Boolean,
                                    timeIndexDumpErrors: TimeIndexDumpErrors,
-                                   maxMessageSize: Int) {
+                                   maxMessageSize: Int): Unit = {
     val startOffset = file.getName.split("\\.")(0).toLong
     val logFile = new File(file.getAbsoluteFile.getParent, file.getName.split("\\.")(0) + Log.LogFileSuffix)
     val fileRecords = FileRecords.open(logFile, false)
@@ -326,7 +326,7 @@ object DumpLogSegments {
                       nonConsecutivePairsForLogFilesMap: mutable.Map[String, List[(Long, Long)]],
                       isDeepIteration: Boolean,
                       maxMessageSize: Int,
-                      parser: MessageParser[_, _]) {
+                      parser: MessageParser[_, _]): Unit = {
     val startOffset = file.getName.split("\\.")(0).toLong
     println("Starting offset: " + startOffset)
     val fileRecords = FileRecords.open(file, false)
@@ -401,28 +401,28 @@ object DumpLogSegments {
     val outOfOrderTimestamp = mutable.Map[String, ArrayBuffer[(Long, Long)]]()
     val shallowOffsetNotFound = mutable.Map[String, ArrayBuffer[(Long, Long)]]()
 
-    def recordMismatchTimeIndex(file: File, indexTimestamp: Long, logTimestamp: Long) {
+    def recordMismatchTimeIndex(file: File, indexTimestamp: Long, logTimestamp: Long): Unit = {
       val misMatchesSeq = misMatchesForTimeIndexFilesMap.getOrElse(file.getAbsolutePath, new ArrayBuffer[(Long, Long)]())
       if (misMatchesSeq.isEmpty)
         misMatchesForTimeIndexFilesMap.put(file.getAbsolutePath, misMatchesSeq)
       misMatchesSeq += ((indexTimestamp, logTimestamp))
     }
 
-    def recordOutOfOrderIndexTimestamp(file: File, indexTimestamp: Long, prevIndexTimestamp: Long) {
+    def recordOutOfOrderIndexTimestamp(file: File, indexTimestamp: Long, prevIndexTimestamp: Long): Unit = {
       val outOfOrderSeq = outOfOrderTimestamp.getOrElse(file.getAbsolutePath, new ArrayBuffer[(Long, Long)]())
       if (outOfOrderSeq.isEmpty)
         outOfOrderTimestamp.put(file.getAbsolutePath, outOfOrderSeq)
       outOfOrderSeq += ((indexTimestamp, prevIndexTimestamp))
     }
 
-    def recordShallowOffsetNotFound(file: File, indexOffset: Long, logOffset: Long) {
+    def recordShallowOffsetNotFound(file: File, indexOffset: Long, logOffset: Long): Unit = {
       val shallowOffsetNotFoundSeq = shallowOffsetNotFound.getOrElse(file.getAbsolutePath, new ArrayBuffer[(Long, Long)]())
       if (shallowOffsetNotFoundSeq.isEmpty)
         shallowOffsetNotFound.put(file.getAbsolutePath, shallowOffsetNotFoundSeq)
       shallowOffsetNotFoundSeq += ((indexOffset, logOffset))
     }
 
-    def printErrors() {
+    def printErrors(): Unit = {
       misMatchesForTimeIndexFilesMap.foreach {
         case (fileName, listOfMismatches) => {
           System.err.println("Found timestamp mismatch in :" + fileName)
diff --git a/core/src/main/scala/kafka/tools/EndToEndLatency.scala b/core/src/main/scala/kafka/tools/EndToEndLatency.scala
index 810758446af53..e97be2f30c62b 100755
--- a/core/src/main/scala/kafka/tools/EndToEndLatency.scala
+++ b/core/src/main/scala/kafka/tools/EndToEndLatency.scala
@@ -49,7 +49,7 @@ object EndToEndLatency {
   private val defaultReplicationFactor: Short = 1
   private val defaultNumPartitions: Int = 1
 
-  def main(args: Array[String]) {
+  def main(args: Array[String]): Unit = {
     if (args.length != 5 && args.length != 6) {
       System.err.println("USAGE: java " + getClass.getName + " broker_list topic num_messages producer_acks message_size_bytes [optional] properties_file")
       Exit.exit(1)
@@ -88,7 +88,7 @@ object EndToEndLatency {
     producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer")
     val producer = new KafkaProducer[Array[Byte], Array[Byte]](producerProps)
 
-    def finalise() {
+    def finalise(): Unit = {
       consumer.commitSync()
       producer.close()
       consumer.close()
diff --git a/core/src/main/scala/kafka/tools/JmxTool.scala b/core/src/main/scala/kafka/tools/JmxTool.scala
index d035692d04183..7b677019a04ef 100644
--- a/core/src/main/scala/kafka/tools/JmxTool.scala
+++ b/core/src/main/scala/kafka/tools/JmxTool.scala
@@ -40,7 +40,7 @@ import kafka.utils.{CommandLineUtils, Exit, Logging}
   */
 object JmxTool extends Logging {
 
-  def main(args: Array[String]) {
+  def main(args: Array[String]): Unit = {
     // Parse command line
     val parser = new OptionParser(false)
     val objectNameOpt =
diff --git a/core/src/main/scala/kafka/tools/MirrorMaker.scala b/core/src/main/scala/kafka/tools/MirrorMaker.scala
index 8a603c36e1b52..a37fa6e58e819 100755
--- a/core/src/main/scala/kafka/tools/MirrorMaker.scala
+++ b/core/src/main/scala/kafka/tools/MirrorMaker.scala
@@ -81,7 +81,7 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
       def value = numDroppedMessages.get()
     })
 
-  def main(args: Array[String]) {
+  def main(args: Array[String]): Unit = {
 
     info("Starting mirror maker")
     try {
@@ -162,7 +162,7 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
     }
   }
 
-  def cleanShutdown() {
+  def cleanShutdown(): Unit = {
     if (isShuttingDown.compareAndSet(false, true)) {
       info("Start clean shutdown.")
       // Shutdown consumer threads.
@@ -177,7 +177,7 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
     }
   }
 
-  private def maybeSetDefaultProperty(properties: Properties, propertyName: String, defaultValue: String) {
+  private def maybeSetDefaultProperty(properties: Properties, propertyName: String, defaultValue: String): Unit = {
     val propertyValue = properties.getProperty(propertyName)
     properties.setProperty(propertyName, Option(propertyValue).getOrElse(defaultValue))
     if (properties.getProperty(propertyName) != defaultValue)
@@ -204,7 +204,7 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
         record.value,
         record.headers)
 
-    override def run() {
+    override def run(): Unit = {
       info(s"Starting mirror maker thread $threadName")
       try {
         consumerWrapper.init()
@@ -260,7 +260,7 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
       }
     }
 
-    def maybeFlushAndCommitOffsets() {
+    def maybeFlushAndCommitOffsets(): Unit = {
       if (System.currentTimeMillis() - lastOffsetCommitMs > offsetCommitIntervalMs) {
         debug("Committing MirrorMaker state.")
         producer.flush()
@@ -269,7 +269,7 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
       }
     }
 
-    def shutdown() {
+    def shutdown(): Unit = {
       try {
         info(s"$threadName shutting down")
         shuttingDown = true
@@ -281,7 +281,7 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
       }
     }
 
-    def awaitShutdown() {
+    def awaitShutdown(): Unit = {
       try {
         shutdownLatch.await()
         info("Mirror maker thread shutdown complete")
@@ -303,7 +303,7 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
     // Visible for testing
     private[tools] val offsets = new HashMap[TopicPartition, Long]()
 
-    def init() {
+    def init(): Unit = {
       debug("Initiating consumer")
       val consumerRebalanceListener = new InternalRebalanceListener(this, customRebalanceListener)
       whitelistOpt.foreach { whitelist =>
@@ -336,15 +336,15 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
       record
     }
 
-    def wakeup() {
+    def wakeup(): Unit = {
       consumer.wakeup()
     }
 
-    def close() {
+    def close(): Unit = {
       consumer.close()
     }
 
-    def commit() {
+    def commit(): Unit = {
       consumer.commitSync(offsets.map { case (tp, offset) => (tp, new OffsetAndMetadata(offset)) }.asJava)
       offsets.clear()
     }
@@ -354,15 +354,15 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
                                           customRebalanceListener: Option[ConsumerRebalanceListener])
     extends ConsumerRebalanceListener {
 
-    override def onPartitionsLost(partitions: util.Collection[TopicPartition]) {}
+    override def onPartitionsLost(partitions: util.Collection[TopicPartition]): Unit = {}
 
-    override def onPartitionsRevoked(partitions: util.Collection[TopicPartition]) {
+    override def onPartitionsRevoked(partitions: util.Collection[TopicPartition]): Unit = {
       producer.flush()
       commitOffsets(consumerWrapper)
       customRebalanceListener.foreach(_.onPartitionsRevoked(partitions))
     }
 
-    override def onPartitionsAssigned(partitions: util.Collection[TopicPartition]) {
+    override def onPartitionsAssigned(partitions: util.Collection[TopicPartition]): Unit = {
       customRebalanceListener.foreach(_.onPartitionsAssigned(partitions))
     }
   }
@@ -371,7 +371,7 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
 
     val producer = new KafkaProducer[Array[Byte], Array[Byte]](producerProps)
 
-    def send(record: ProducerRecord[Array[Byte], Array[Byte]]) {
+    def send(record: ProducerRecord[Array[Byte], Array[Byte]]): Unit = {
       if (sync) {
         this.producer.send(record).get()
       } else {
@@ -380,15 +380,15 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
       }
     }
 
-    def flush() {
+    def flush(): Unit = {
       this.producer.flush()
     }
 
-    def close() {
+    def close(): Unit = {
       this.producer.close()
     }
 
-    def close(timeout: Long) {
+    def close(timeout: Long): Unit = {
       this.producer.close(Duration.ofMillis(timeout))
     }
   }
@@ -396,7 +396,7 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
   private class MirrorMakerProducerCallback (topic: String, key: Array[Byte], value: Array[Byte])
     extends ErrorLoggingCallback(topic, key, value, false) {
 
-    override def onCompletion(metadata: RecordMetadata, exception: Exception) {
+    override def onCompletion(metadata: RecordMetadata, exception: Exception): Unit = {
       if (exception != null) {
         // Use default call back to log error. This means the max retries of producer has reached and message
         // still could not be sent.
@@ -520,7 +520,7 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
       val numStreams = options.valueOf(numStreamsOpt).intValue()
 
       Runtime.getRuntime.addShutdownHook(new Thread("MirrorMakerShutdownHook") {
-        override def run() {
+        override def run(): Unit = {
           cleanShutdown()
         }
       })
diff --git a/core/src/main/scala/kafka/tools/ReplicaVerificationTool.scala b/core/src/main/scala/kafka/tools/ReplicaVerificationTool.scala
index 2c275261027e2..8455b0d97e45f 100644
--- a/core/src/main/scala/kafka/tools/ReplicaVerificationTool.scala
+++ b/core/src/main/scala/kafka/tools/ReplicaVerificationTool.scala
@@ -201,7 +201,7 @@ object ReplicaVerificationTool extends Logging {
     }
 
     Runtime.getRuntime.addShutdownHook(new Thread() {
-      override def run() {
+      override def run(): Unit = {
         info("Stopping all fetchers")
         fetcherThreads.foreach(_.shutdown())
       }
@@ -272,31 +272,31 @@ private class ReplicaBuffer(expectedReplicasPerTopicPartition: Map[TopicPartitio
   private var maxLagTopicAndPartition: TopicPartition = null
   initialize()
 
-  def createNewFetcherBarrier() {
+  def createNewFetcherBarrier(): Unit = {
     fetcherBarrier.set(new CountDownLatch(expectedNumFetchers))
   }
 
   def getFetcherBarrier() = fetcherBarrier.get
 
-  def createNewVerificationBarrier() {
+  def createNewVerificationBarrier(): Unit = {
     verificationBarrier.set(new CountDownLatch(1))
   }
 
   def getVerificationBarrier() = verificationBarrier.get
 
-  private def initialize() {
+  private def initialize(): Unit = {
     for (topicPartition <- expectedReplicasPerTopicPartition.keySet)
       recordsCache.put(topicPartition, new Pool[Int, FetchResponse.PartitionData[MemoryRecords]])
     setInitialOffsets()
   }
 
 
-  private def setInitialOffsets() {
+  private def setInitialOffsets(): Unit = {
     for ((tp, offset) <- initialOffsets)
       fetchOffsetMap.put(tp, offset)
   }
 
-  def addFetchedData(topicAndPartition: TopicPartition, replicaId: Int, partitionData: FetchResponse.PartitionData[MemoryRecords]) {
+  def addFetchedData(topicAndPartition: TopicPartition, replicaId: Int, partitionData: FetchResponse.PartitionData[MemoryRecords]): Unit = {
     recordsCache.get(topicAndPartition).put(replicaId, partitionData)
   }
 
@@ -304,7 +304,7 @@ private class ReplicaBuffer(expectedReplicasPerTopicPartition: Map[TopicPartitio
     fetchOffsetMap.get(topicAndPartition)
   }
 
-  def verifyCheckSum(println: String => Unit) {
+  def verifyCheckSum(println: String => Unit): Unit = {
     debug("Begin verification")
     maxLag = -1L
     for ((topicPartition, fetchResponsePerReplica) <- recordsCache) {
@@ -390,7 +390,7 @@ private class ReplicaFetcher(name: String, sourceBroker: Node, topicPartitions:
   private val fetchEndpoint = new ReplicaFetcherBlockingSend(sourceBroker, new ConsumerConfig(consumerConfig), new Metrics(), Time.SYSTEM, fetcherId,
     s"broker-${Request.DebuggingConsumerId}-fetcher-$fetcherId")
 
-  override def doWork() {
+  override def doWork(): Unit = {
 
     val fetcherBarrier = replicaBuffer.getFetcherBarrier()
     val verificationBarrier = replicaBuffer.getVerificationBarrier()
diff --git a/core/src/main/scala/kafka/tools/StateChangeLogMerger.scala b/core/src/main/scala/kafka/tools/StateChangeLogMerger.scala
index 8edb8d29d0c80..49eaccbe537d5 100755
--- a/core/src/main/scala/kafka/tools/StateChangeLogMerger.scala
+++ b/core/src/main/scala/kafka/tools/StateChangeLogMerger.scala
@@ -57,7 +57,7 @@ object StateChangeLogMerger extends Logging {
   var startDate: Date = null
   var endDate: Date = null
 
-  def main(args: Array[String]) {
+  def main(args: Array[String]): Unit = {
 
     // Parse input arguments.
     val parser = new OptionParser(false)
diff --git a/core/src/main/scala/kafka/utils/CommandLineUtils.scala b/core/src/main/scala/kafka/utils/CommandLineUtils.scala
index b19da3ec15271..3576f4990f2f6 100644
--- a/core/src/main/scala/kafka/utils/CommandLineUtils.scala
+++ b/core/src/main/scala/kafka/utils/CommandLineUtils.scala
@@ -62,7 +62,7 @@ object CommandLineUtils extends Logging {
   /**
    * Check that all the listed options are present
    */
-  def checkRequiredArgs(parser: OptionParser, options: OptionSet, required: OptionSpec[_]*) {
+  def checkRequiredArgs(parser: OptionParser, options: OptionSet, required: OptionSpec[_]*): Unit = {
     for (arg <- required) {
       if (!options.has(arg))
         printUsageAndDie(parser, "Missing required argument \"" + arg + "\"")
@@ -72,7 +72,7 @@ object CommandLineUtils extends Logging {
   /**
    * Check that none of the listed options are present
    */
-  def checkInvalidArgs(parser: OptionParser, options: OptionSet, usedOption: OptionSpec[_], invalidOptions: Set[OptionSpec[_]]) {
+  def checkInvalidArgs(parser: OptionParser, options: OptionSet, usedOption: OptionSpec[_], invalidOptions: Set[OptionSpec[_]]): Unit = {
     if (options.has(usedOption)) {
       for (arg <- invalidOptions) {
         if (options.has(arg))
@@ -84,7 +84,7 @@ object CommandLineUtils extends Logging {
   /**
     * Check that none of the listed options are present with the combination of used options
     */
-  def checkInvalidArgsSet(parser: OptionParser, options: OptionSet, usedOptions: Set[OptionSpec[_]], invalidOptions: Set[OptionSpec[_]]) {
+  def checkInvalidArgsSet(parser: OptionParser, options: OptionSet, usedOptions: Set[OptionSpec[_]], invalidOptions: Set[OptionSpec[_]]): Unit = {
     if (usedOptions.count(options.has) == usedOptions.size) {
       for (arg <- invalidOptions) {
         if (options.has(arg))
@@ -132,7 +132,7 @@ object CommandLineUtils extends Logging {
     * 3) otherwise, use the default value of {@code spec}.
     * A {@code null} value means to remove {@code key} from the {@code props}.
     */
-  def maybeMergeOptions[V](props: Properties, key: String, options: OptionSet, spec: OptionSpec[V]) {
+  def maybeMergeOptions[V](props: Properties, key: String, options: OptionSet, spec: OptionSpec[V]): Unit = {
     if (options.has(spec) || !props.containsKey(key)) {
       val value = options.valueOf(spec)
       if (value == null)
diff --git a/core/src/main/scala/kafka/utils/CoreUtils.scala b/core/src/main/scala/kafka/utils/CoreUtils.scala
index 8f1fd781857af..2abac1bcbcba7 100755
--- a/core/src/main/scala/kafka/utils/CoreUtils.scala
+++ b/core/src/main/scala/kafka/utils/CoreUtils.scala
@@ -83,7 +83,7 @@ object CoreUtils {
     * @param logging The logging instance to use for logging the thrown exception.
     * @param logLevel The log level to use for logging.
     */
-  def swallow(action: => Unit, logging: Logging, logLevel: Level = Level.WARN) {
+  def swallow(action: => Unit, logging: Logging, logLevel: Level = Level.WARN): Unit = {
     try {
       action
     } catch {
@@ -159,7 +159,7 @@ object CoreUtils {
    * Unregister the mbean with the given name, if there is one registered
    * @param name The mbean name to unregister
    */
-  def unregisterMBean(name: String) {
+  def unregisterMBean(name: String): Unit = {
     val mbs = ManagementFactory.getPlatformMBeanServer()
     mbs synchronized {
       val objName = new ObjectName(name)
diff --git a/core/src/main/scala/kafka/utils/FileLock.scala b/core/src/main/scala/kafka/utils/FileLock.scala
index fd31b12a2b0d1..c635f76dff3a2 100644
--- a/core/src/main/scala/kafka/utils/FileLock.scala
+++ b/core/src/main/scala/kafka/utils/FileLock.scala
@@ -34,7 +34,7 @@ class FileLock(val file: File) extends Logging {
   /**
    * Lock the file or throw an exception if the lock is already held
    */
-  def lock() {
+  def lock(): Unit = {
     this synchronized {
       trace(s"Acquiring lock on ${file.getAbsolutePath}")
       flock = channel.lock()
@@ -62,7 +62,7 @@ class FileLock(val file: File) extends Logging {
   /**
    * Unlock the lock if it is held
    */
-  def unlock() {
+  def unlock(): Unit = {
     this synchronized {
       trace(s"Releasing lock on ${file.getAbsolutePath}")
       if(flock != null)
diff --git a/core/src/main/scala/kafka/utils/KafkaScheduler.scala b/core/src/main/scala/kafka/utils/KafkaScheduler.scala
index cee44787992ea..f7f5995b4fc0c 100755
--- a/core/src/main/scala/kafka/utils/KafkaScheduler.scala
+++ b/core/src/main/scala/kafka/utils/KafkaScheduler.scala
@@ -32,13 +32,13 @@ trait Scheduler {
   /**
    * Initialize this scheduler so it is ready to accept scheduling of tasks
    */
-  def startup()
+  def startup(): Unit
   
   /**
    * Shutdown this scheduler. When this method is complete no more executions of background tasks will occur. 
    * This includes tasks scheduled with a delayed execution.
    */
-  def shutdown()
+  def shutdown(): Unit
   
   /**
    * Check if the scheduler has been started
@@ -72,7 +72,7 @@ class KafkaScheduler(val threads: Int,
   private var executor: ScheduledThreadPoolExecutor = null
   private val schedulerThreadId = new AtomicInteger(0)
 
-  override def startup() {
+  override def startup(): Unit = {
     debug("Initializing task scheduler.")
     this synchronized {
       if(isStarted)
@@ -88,7 +88,7 @@ class KafkaScheduler(val threads: Int,
     }
   }
   
-  override def shutdown() {
+  override def shutdown(): Unit = {
     debug("Shutting down task scheduler.")
     // We use the local variable to avoid NullPointerException if another thread shuts down scheduler at same time.
     val cachedExecutor = this.executor
diff --git a/core/src/main/scala/kafka/utils/Pool.scala b/core/src/main/scala/kafka/utils/Pool.scala
index 5dd61946f0d29..c963524a9c8c3 100644
--- a/core/src/main/scala/kafka/utils/Pool.scala
+++ b/core/src/main/scala/kafka/utils/Pool.scala
@@ -73,7 +73,7 @@ class Pool[K,V](valueFactory: Option[K => V] = None) extends Iterable[(K, V)] {
 
   def values: Iterable[V] = pool.values.asScala
 
-  def clear() { pool.clear() }
+  def clear(): Unit = { pool.clear() }
   
   override def size: Int = pool.size
   
diff --git a/core/src/main/scala/kafka/utils/Throttler.scala b/core/src/main/scala/kafka/utils/Throttler.scala
index 9fe3cdcf13f44..cce6270cf02e8 100644
--- a/core/src/main/scala/kafka/utils/Throttler.scala
+++ b/core/src/main/scala/kafka/utils/Throttler.scala
@@ -49,7 +49,7 @@ class Throttler(desiredRatePerSec: Double,
   private var periodStartNs: Long = time.nanoseconds
   private var observedSoFar: Double = 0.0
   
-  def maybeThrottle(observed: Double) {
+  def maybeThrottle(observed: Double): Unit = {
     val msPerSec = TimeUnit.SECONDS.toMillis(1)
     val nsPerSec = TimeUnit.SECONDS.toNanos(1)
 
@@ -83,7 +83,7 @@ class Throttler(desiredRatePerSec: Double,
 
 object Throttler {
   
-  def main(args: Array[String]) {
+  def main(args: Array[String]): Unit = {
     val rand = new Random()
     val throttler = new Throttler(100000, 100, true, time = Time.SYSTEM)
     val interval = 30000
diff --git a/core/src/main/scala/kafka/utils/VerifiableProperties.scala b/core/src/main/scala/kafka/utils/VerifiableProperties.scala
index 5d70db5e11d06..8a0f33837c1d8 100755
--- a/core/src/main/scala/kafka/utils/VerifiableProperties.scala
+++ b/core/src/main/scala/kafka/utils/VerifiableProperties.scala
@@ -214,7 +214,7 @@ class VerifiableProperties(val props: Properties) extends Logging {
     }
   }
 
-  def verify() {
+  def verify(): Unit = {
     info("Verifying properties")
     val propNames = Collections.list(props.propertyNames).asScala.map(_.toString).sorted
     for(key <- propNames) {
diff --git a/core/src/main/scala/kafka/utils/VersionInfo.scala b/core/src/main/scala/kafka/utils/VersionInfo.scala
index 9910dfc22e726..9d3130e6685d3 100644
--- a/core/src/main/scala/kafka/utils/VersionInfo.scala
+++ b/core/src/main/scala/kafka/utils/VersionInfo.scala
@@ -21,7 +21,7 @@ import org.apache.kafka.common.utils.AppInfoParser
 
 object VersionInfo {
 
-  def main(args: Array[String]) {
+  def main(args: Array[String]): Unit = {
     System.out.println(getVersionString)
     System.exit(0)
   }
diff --git a/core/src/main/scala/kafka/utils/timer/Timer.scala b/core/src/main/scala/kafka/utils/timer/Timer.scala
index ae8caff863658..79994a5af9b70 100644
--- a/core/src/main/scala/kafka/utils/timer/Timer.scala
+++ b/core/src/main/scala/kafka/utils/timer/Timer.scala
@@ -122,7 +122,7 @@ class SystemTimer(executorName: String,
 
   def size: Int = taskCounter.get
 
-  override def shutdown() {
+  override def shutdown(): Unit = {
     taskExecutor.shutdown()
   }
 
diff --git a/core/src/main/scala/kafka/zk/AdminZkClient.scala b/core/src/main/scala/kafka/zk/AdminZkClient.scala
index b10a089f058a6..f0e57991c78a9 100644
--- a/core/src/main/scala/kafka/zk/AdminZkClient.scala
+++ b/core/src/main/scala/kafka/zk/AdminZkClient.scala
@@ -50,7 +50,7 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
                   partitions: Int,
                   replicationFactor: Int,
                   topicConfig: Properties = new Properties,
-                  rackAwareMode: RackAwareMode = RackAwareMode.Enforced) {
+                  rackAwareMode: RackAwareMode = RackAwareMode.Enforced): Unit = {
     val brokerMetadatas = getBrokerMetadatas(rackAwareMode)
     val replicaAssignment = AdminUtils.assignReplicasToBrokers(brokerMetadatas, partitions, replicationFactor)
     createTopicWithAssignment(topic, topicConfig, replicaAssignment)
@@ -134,7 +134,7 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
     LogConfig.validate(config)
   }
 
-  private def writeTopicPartitionAssignment(topic: String, replicaAssignment: Map[Int, Seq[Int]], isUpdate: Boolean) {
+  private def writeTopicPartitionAssignment(topic: String, replicaAssignment: Map[Int, Seq[Int]], isUpdate: Boolean): Unit = {
     try {
       val assignment = replicaAssignment.map { case (partitionId, replicas) => (new TopicPartition(topic,partitionId), replicas) }.toMap
 
@@ -154,7 +154,7 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
    * Creates a delete path for a given topic
    * @param topic
    */
-  def deleteTopic(topic: String) {
+  def deleteTopic(topic: String): Unit = {
     if (zkClient.topicExists(topic)) {
       try {
         zkClient.createDeleteTopicPath(topic)
@@ -289,7 +289,7 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
    *                 existing configs need to be deleted, it should be done prior to invoking this API
    *
    */
-  def changeClientIdConfig(sanitizedClientId: String, configs: Properties) {
+  def changeClientIdConfig(sanitizedClientId: String, configs: Properties): Unit = {
     DynamicConfig.Client.validate(configs)
     changeEntityConfig(ConfigType.Client, sanitizedClientId, configs)
   }
@@ -304,7 +304,7 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
    *                 existing configs need to be deleted, it should be done prior to invoking this API
    *
    */
-  def changeUserOrUserClientIdConfig(sanitizedEntityName: String, configs: Properties) {
+  def changeUserOrUserClientIdConfig(sanitizedEntityName: String, configs: Properties): Unit = {
     if (sanitizedEntityName == ConfigEntityName.Default || sanitizedEntityName.contains("/clients"))
       DynamicConfig.Client.validate(configs)
     else
@@ -373,7 +373,7 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
     DynamicConfig.Broker.validate(configs)
   }
 
-  private def changeEntityConfig(rootEntityType: String, fullSanitizedEntityName: String, configs: Properties) {
+  private def changeEntityConfig(rootEntityType: String, fullSanitizedEntityName: String, configs: Properties): Unit = {
     val sanitizedEntityPath = rootEntityType + '/' + fullSanitizedEntityName
     zkClient.setOrCreateEntityConfigs(rootEntityType, fullSanitizedEntityName, configs)
 
diff --git a/core/src/main/scala/kafka/zk/KafkaZkClient.scala b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
index ad3069ce1e7b8..7adb27e4f74a6 100644
--- a/core/src/main/scala/kafka/zk/KafkaZkClient.scala
+++ b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
@@ -985,7 +985,7 @@ class KafkaZkClient private[zk] (zooKeeperClient: ZooKeeperClient, isSecure: Boo
    * @param partitions
    * @throws KeeperException if there is an error while creating the znode
    */
-  def createPreferredReplicaElection(partitions: Set[TopicPartition]): Unit  = {
+  def createPreferredReplicaElection(partitions: Set[TopicPartition]): Unit = {
     createRecursive(PreferredReplicaElectionZNode.path, PreferredReplicaElectionZNode.encode(partitions))
   }
 
@@ -1155,7 +1155,7 @@ class KafkaZkClient private[zk] (zooKeeperClient: ZooKeeperClient, isSecure: Boo
     createResponse.maybeThrow
   }
 
-  def propagateLogDirEvent(brokerId: Int) {
+  def propagateLogDirEvent(brokerId: Int): Unit = {
     val logDirEventNotificationPath: String = createSequentialPersistentPath(
       LogDirEventNotificationZNode.path + "/" + LogDirEventNotificationSequenceZNode.SequenceNumberPrefix,
       LogDirEventNotificationSequenceZNode.encode(brokerId))
diff --git a/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala b/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala
index 45743b2c97713..4853424ed35c1 100644
--- a/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala
@@ -74,7 +74,7 @@ abstract class AbstractConsumerTest extends BaseRequestTest {
   }
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
 
     // create the test topic with all the brokers as replicas
@@ -85,12 +85,12 @@ abstract class AbstractConsumerTest extends BaseRequestTest {
     var callsToAssigned = 0
     var callsToRevoked = 0
 
-    def onPartitionsAssigned(partitions: java.util.Collection[TopicPartition]) {
+    def onPartitionsAssigned(partitions: java.util.Collection[TopicPartition]): Unit = {
       info("onPartitionsAssigned called.")
       callsToAssigned += 1
     }
 
-    def onPartitionsRevoked(partitions: java.util.Collection[TopicPartition]) {
+    def onPartitionsRevoked(partitions: java.util.Collection[TopicPartition]): Unit = {
       info("onPartitionsRevoked called.")
       callsToRevoked += 1
     }
@@ -121,7 +121,7 @@ abstract class AbstractConsumerTest extends BaseRequestTest {
                                         startingTimestamp: Long = 0L,
                                         timestampType: TimestampType = TimestampType.CREATE_TIME,
                                         tp: TopicPartition = tp,
-                                        maxPollRecords: Int = Int.MaxValue) {
+                                        maxPollRecords: Int = Int.MaxValue): Unit = {
     val records = consumeRecords(consumer, numRecords, maxPollRecords = maxPollRecords)
     val now = System.currentTimeMillis()
     for (i <- 0 until numRecords) {
diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index b7a2c6ab9e920..328949e26087f 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -1175,7 +1175,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
       try {
         // Start a consumer in a thread that will subscribe to a new group.
         val consumerThread = new Thread {
-          override def run {
+          override def run : Unit = {
             consumer.subscribe(Collections.singleton(testTopicName))
 
             try {
diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
index 387a7f9d113ea..8e746c7888d83 100644
--- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
@@ -270,7 +270,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   )
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     doSetup(createOffsetsTopic = false)
 
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, ClusterAction)), Resource.ClusterResource)
@@ -520,7 +520,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   ).build()
 
   @Test
-  def testAuthorizationWithTopicExisting() {
+  def testAuthorizationWithTopicExisting(): Unit = {
     val requestKeyToRequest = mutable.LinkedHashMap[ApiKeys, AbstractRequest](
       ApiKeys.METADATA -> createMetadataRequest(allowAutoTopicCreation = true),
       ApiKeys.PRODUCE -> createProduceRequest,
@@ -583,7 +583,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
    * even if the topic doesn't exist, request APIs should not leak the topic name
    */
   @Test
-  def testAuthorizationWithTopicNotExisting() {
+  def testAuthorizationWithTopicNotExisting(): Unit = {
     adminZkClient.deleteTopic(topic)
     TestUtils.verifyTopicDeletion(zkClient, topic, 1, servers)
     adminZkClient.deleteTopic(deleteTopic)
@@ -627,7 +627,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testCreateTopicAuthorizationWithClusterCreate() {
+  def testCreateTopicAuthorizationWithClusterCreate(): Unit = {
     removeAllAcls()
     val resources = Set[ResourceType](Topic)
 
@@ -639,7 +639,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testFetchFollowerRequest() {
+  def testFetchFollowerRequest(): Unit = {
     val key = ApiKeys.FETCH
     val request = createFetchFollowerRequest
 
@@ -696,7 +696,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testProduceWithNoTopicAccess() {
+  def testProduceWithNoTopicAccess(): Unit = {
     try {
       val producer = createProducer()
       sendRecords(producer, numRecords, tp)
@@ -707,7 +707,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testProduceWithTopicDescribe() {
+  def testProduceWithTopicDescribe(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), topicResource)
     try {
       val producer = createProducer()
@@ -720,7 +720,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testProduceWithTopicRead() {
+  def testProduceWithTopicRead(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
     try {
       val producer = createProducer()
@@ -733,23 +733,23 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testProduceWithTopicWrite() {
+  def testProduceWithTopicWrite(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
     val producer = createProducer()
     sendRecords(producer, numRecords, tp)
   }
 
   @Test
-  def testCreatePermissionOnTopicToWriteToNonExistentTopic() {
+  def testCreatePermissionOnTopicToWriteToNonExistentTopic(): Unit = {
     testCreatePermissionNeededToWriteToNonExistentTopic(Topic)
   }
 
   @Test
-  def testCreatePermissionOnClusterToWriteToNonExistentTopic() {
+  def testCreatePermissionOnClusterToWriteToNonExistentTopic(): Unit = {
     testCreatePermissionNeededToWriteToNonExistentTopic(Cluster)
   }
 
-  private def testCreatePermissionNeededToWriteToNonExistentTopic(resType: ResourceType) {
+  private def testCreatePermissionNeededToWriteToNonExistentTopic(resType: ResourceType): Unit = {
     val topicPartition = new TopicPartition(createTopic, 0)
     val newTopicResource = Resource(Topic, createTopic, LITERAL)
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), newTopicResource)
@@ -817,7 +817,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test(expected = classOf[KafkaException])
-  def testConsumeWithoutTopicDescribeAccess() {
+  def testConsumeWithoutTopicDescribeAccess(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
@@ -833,7 +833,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testConsumeWithTopicDescribe() {
+  def testConsumeWithTopicDescribe(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
@@ -852,7 +852,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testConsumeWithTopicWrite() {
+  def testConsumeWithTopicWrite(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
@@ -872,7 +872,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testConsumeWithTopicAndGroupRead() {
+  def testConsumeWithTopicAndGroupRead(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
@@ -887,7 +887,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testPatternSubscriptionWithNoTopicAccess() {
+  def testPatternSubscriptionWithNoTopicAccess(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
@@ -902,7 +902,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testPatternSubscriptionWithTopicDescribeOnlyAndGroupRead() {
+  def testPatternSubscriptionWithTopicDescribeOnlyAndGroupRead(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
@@ -921,7 +921,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testPatternSubscriptionWithTopicAndGroupRead() {
+  def testPatternSubscriptionWithTopicAndGroupRead(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
@@ -950,7 +950,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testPatternSubscriptionMatchingInternalTopic() {
+  def testPatternSubscriptionMatchingInternalTopic(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
@@ -975,7 +975,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testPatternSubscriptionMatchingInternalTopicWithDescribeOnlyPermission() {
+  def testPatternSubscriptionMatchingInternalTopicWithDescribeOnlyPermission(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
@@ -1000,7 +1000,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testPatternSubscriptionNotMatchingInternalTopic() {
+  def testPatternSubscriptionNotMatchingInternalTopic(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
@@ -1018,20 +1018,20 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 }
 
   @Test
-  def testCreatePermissionOnTopicToReadFromNonExistentTopic() {
+  def testCreatePermissionOnTopicToReadFromNonExistentTopic(): Unit = {
     testCreatePermissionNeededToReadFromNonExistentTopic("newTopic",
       Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Create)),
       Topic)
   }
 
   @Test
-  def testCreatePermissionOnClusterToReadFromNonExistentTopic() {
+  def testCreatePermissionOnClusterToReadFromNonExistentTopic(): Unit = {
     testCreatePermissionNeededToReadFromNonExistentTopic("newTopic",
       Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Create)),
       Cluster)
   }
 
-  private def testCreatePermissionNeededToReadFromNonExistentTopic(newTopic: String, acls: Set[Acl], resType: ResourceType) {
+  private def testCreatePermissionNeededToReadFromNonExistentTopic(newTopic: String, acls: Set[Acl], resType: ResourceType): Unit = {
     val topicPartition = new TopicPartition(newTopic, 0)
     val newTopicResource = Resource(Topic, newTopic, LITERAL)
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), newTopicResource)
@@ -1053,7 +1053,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testCreatePermissionMetadataRequestAutoCreate() {
+  def testCreatePermissionMetadataRequestAutoCreate(): Unit = {
     val readAcls = topicReadAcl.get(topicResource).get
     addAndVerifyAcls(readAcls, topicResource)
     assertTrue(zkClient.topicExists(topicResource.name))
@@ -1078,20 +1078,20 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test(expected = classOf[AuthorizationException])
-  def testCommitWithNoAccess() {
+  def testCommitWithNoAccess(): Unit = {
     val consumer = createConsumer()
     consumer.commitSync(Map(tp -> new OffsetAndMetadata(5)).asJava)
   }
 
   @Test(expected = classOf[KafkaException])
-  def testCommitWithNoTopicAccess() {
+  def testCommitWithNoTopicAccess(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
     val consumer = createConsumer()
     consumer.commitSync(Map(tp -> new OffsetAndMetadata(5)).asJava)
   }
 
   @Test(expected = classOf[TopicAuthorizationException])
-  def testCommitWithTopicWrite() {
+  def testCommitWithTopicWrite(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
     val consumer = createConsumer()
@@ -1099,7 +1099,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test(expected = classOf[TopicAuthorizationException])
-  def testCommitWithTopicDescribe() {
+  def testCommitWithTopicDescribe(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), topicResource)
     val consumer = createConsumer()
@@ -1107,14 +1107,14 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test(expected = classOf[GroupAuthorizationException])
-  def testCommitWithNoGroupAccess() {
+  def testCommitWithNoGroupAccess(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
     val consumer = createConsumer()
     consumer.commitSync(Map(tp -> new OffsetAndMetadata(5)).asJava)
   }
 
   @Test
-  def testCommitWithTopicAndGroupRead() {
+  def testCommitWithTopicAndGroupRead(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
     val consumer = createConsumer()
@@ -1122,14 +1122,14 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test(expected = classOf[AuthorizationException])
-  def testOffsetFetchWithNoAccess() {
+  def testOffsetFetchWithNoAccess(): Unit = {
     val consumer = createConsumer()
     consumer.assign(List(tp).asJava)
     consumer.position(tp)
   }
 
   @Test(expected = classOf[GroupAuthorizationException])
-  def testOffsetFetchWithNoGroupAccess() {
+  def testOffsetFetchWithNoGroupAccess(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
     val consumer = createConsumer()
     consumer.assign(List(tp).asJava)
@@ -1137,7 +1137,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test(expected = classOf[KafkaException])
-  def testOffsetFetchWithNoTopicAccess() {
+  def testOffsetFetchWithNoTopicAccess(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
     val consumer = createConsumer()
     consumer.assign(List(tp).asJava)
@@ -1145,7 +1145,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testFetchAllOffsetsTopicAuthorization() {
+  def testFetchAllOffsetsTopicAuthorization(): Unit = {
     val offset = 15L
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
@@ -1174,7 +1174,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testOffsetFetchTopicDescribe() {
+  def testOffsetFetchTopicDescribe(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), groupResource)
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), topicResource)
     val consumer = createConsumer()
@@ -1183,7 +1183,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testOffsetFetchWithTopicAndGroupRead() {
+  def testOffsetFetchWithTopicAndGroupRead(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
     val consumer = createConsumer()
@@ -1192,47 +1192,47 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test(expected = classOf[TopicAuthorizationException])
-  def testMetadataWithNoTopicAccess() {
+  def testMetadataWithNoTopicAccess(): Unit = {
     val consumer = createConsumer()
     consumer.partitionsFor(topic)
   }
 
   @Test
-  def testMetadataWithTopicDescribe() {
+  def testMetadataWithTopicDescribe(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), topicResource)
     val consumer = createConsumer()
     consumer.partitionsFor(topic)
   }
 
   @Test(expected = classOf[TopicAuthorizationException])
-  def testListOffsetsWithNoTopicAccess() {
+  def testListOffsetsWithNoTopicAccess(): Unit = {
     val consumer = createConsumer()
     consumer.endOffsets(Set(tp).asJava)
   }
 
   @Test
-  def testListOffsetsWithTopicDescribe() {
+  def testListOffsetsWithTopicDescribe(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), topicResource)
     val consumer = createConsumer()
     consumer.endOffsets(Set(tp).asJava)
   }
 
   @Test
-  def testDescribeGroupApiWithNoGroupAcl() {
+  def testDescribeGroupApiWithNoGroupAcl(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), topicResource)
     val result = createAdminClient().describeConsumerGroups(Seq(group).asJava)
     TestUtils.assertFutureExceptionTypeEquals(result.describedGroups().get(group), classOf[GroupAuthorizationException])
   }
 
   @Test
-  def testDescribeGroupApiWithGroupDescribe() {
+  def testDescribeGroupApiWithGroupDescribe(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), groupResource)
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), topicResource)
     createAdminClient().describeConsumerGroups(Seq(group).asJava).describedGroups().get(group).get()
   }
 
   @Test
-  def testDescribeGroupCliWithGroupDescribe() {
+  def testDescribeGroupCliWithGroupDescribe(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), groupResource)
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), topicResource)
 
@@ -1244,7 +1244,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testListGroupApiWithAndWithoutListGroupAcls() {
+  def testListGroupApiWithAndWithoutListGroupAcls(): Unit = {
     // write some record to the topic
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
     val producer = createProducer()
@@ -1288,7 +1288,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testDeleteGroupApiWithDeleteGroupAcl() {
+  def testDeleteGroupApiWithDeleteGroupAcl(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Delete)), groupResource)
@@ -1299,7 +1299,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testDeleteGroupApiWithNoDeleteGroupAcl() {
+  def testDeleteGroupApiWithNoDeleteGroupAcl(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
     val consumer = createConsumer()
@@ -1310,13 +1310,13 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testDeleteGroupApiWithNoDeleteGroupAcl2() {
+  def testDeleteGroupApiWithNoDeleteGroupAcl2(): Unit = {
     val result = createAdminClient().deleteConsumerGroups(Seq(group).asJava)
     TestUtils.assertFutureExceptionTypeEquals(result.deletedGroups().get(group), classOf[GroupAuthorizationException])
   }
 
   @Test
-  def testUnauthorizedDeleteTopicsWithoutDescribe() {
+  def testUnauthorizedDeleteTopicsWithoutDescribe(): Unit = {
     val response = connectAndSend(deleteTopicsRequest, ApiKeys.DELETE_TOPICS)
     val version = ApiKeys.DELETE_TOPICS.latestVersion
     val deleteResponse = DeleteTopicsResponse.parse(response, version)
@@ -1324,7 +1324,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testUnauthorizedDeleteTopicsWithDescribe() {
+  def testUnauthorizedDeleteTopicsWithDescribe(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), deleteTopicResource)
     val response = connectAndSend(deleteTopicsRequest, ApiKeys.DELETE_TOPICS)
     val version = ApiKeys.DELETE_TOPICS.latestVersion
@@ -1334,7 +1334,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testDeleteTopicsWithWildCardAuth() {
+  def testDeleteTopicsWithWildCardAuth(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Delete)), Resource(Topic, "*", LITERAL))
     val response = connectAndSend(deleteTopicsRequest, ApiKeys.DELETE_TOPICS)
     val version = ApiKeys.DELETE_TOPICS.latestVersion
@@ -1344,7 +1344,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testUnauthorizedDeleteRecordsWithoutDescribe() {
+  def testUnauthorizedDeleteRecordsWithoutDescribe(): Unit = {
     val response = connectAndSend(deleteRecordsRequest, ApiKeys.DELETE_RECORDS)
     val version = ApiKeys.DELETE_RECORDS.latestVersion
     val deleteRecordsResponse = DeleteRecordsResponse.parse(response, version)
@@ -1352,7 +1352,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testUnauthorizedDeleteRecordsWithDescribe() {
+  def testUnauthorizedDeleteRecordsWithDescribe(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), deleteTopicResource)
     val response = connectAndSend(deleteRecordsRequest, ApiKeys.DELETE_RECORDS)
     val version = ApiKeys.DELETE_RECORDS.latestVersion
@@ -1361,7 +1361,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testDeleteRecordsWithWildCardAuth() {
+  def testDeleteRecordsWithWildCardAuth(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Delete)), Resource(Topic, "*", LITERAL))
     val response = connectAndSend(deleteRecordsRequest, ApiKeys.DELETE_RECORDS)
     val version = ApiKeys.DELETE_RECORDS.latestVersion
@@ -1371,7 +1371,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testUnauthorizedCreatePartitions() {
+  def testUnauthorizedCreatePartitions(): Unit = {
     val response = connectAndSend(createPartitionsRequest, ApiKeys.CREATE_PARTITIONS)
     val version = ApiKeys.CREATE_PARTITIONS.latestVersion
     val createPartitionsResponse = CreatePartitionsResponse.parse(response, version)
@@ -1379,7 +1379,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   @Test
-  def testCreatePartitionsWithWildCardAuth() {
+  def testCreatePartitionsWithWildCardAuth(): Unit = {
     addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Alter)), Resource(Topic, "*", LITERAL))
     val response = connectAndSend(createPartitionsRequest, ApiKeys.CREATE_PARTITIONS)
     val version = ApiKeys.CREATE_PARTITIONS.latestVersion
@@ -1658,7 +1658,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   private def sendRecords(producer: KafkaProducer[Array[Byte], Array[Byte]],
                           numRecords: Int,
-                          tp: TopicPartition) {
+                          tp: TopicPartition): Unit = {
     val futures = (0 until numRecords).map { i =>
       producer.send(new ProducerRecord(tp.topic(), tp.partition(), i.toString.getBytes, i.toString.getBytes))
     }
@@ -1678,7 +1678,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
                              numRecords: Int = 1,
                              startingOffset: Int = 0,
                              topic: String = topic,
-                             part: Int = part) {
+                             part: Int = part): Unit = {
     val records = TestUtils.consumeRecords(consumer, numRecords)
 
     for (i <- 0 until numRecords) {
diff --git a/core/src/test/scala/integration/kafka/api/BaseConsumerTest.scala b/core/src/test/scala/integration/kafka/api/BaseConsumerTest.scala
index e715525d65592..2b0700b38f869 100644
--- a/core/src/test/scala/integration/kafka/api/BaseConsumerTest.scala
+++ b/core/src/test/scala/integration/kafka/api/BaseConsumerTest.scala
@@ -31,7 +31,7 @@ import scala.collection.Seq
 abstract class BaseConsumerTest extends AbstractConsumerTest {
 
   @Test
-  def testSimpleConsumption() {
+  def testSimpleConsumption(): Unit = {
     val numRecords = 10000
     val producer = createProducer()
     sendRecords(producer, numRecords, tp)
@@ -49,7 +49,7 @@ abstract class BaseConsumerTest extends AbstractConsumerTest {
   }
 
   @Test
-  def testCoordinatorFailover() {
+  def testCoordinatorFailover(): Unit = {
     val listener = new TestConsumerReassignmentListener()
     this.consumerConfig.setProperty(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "5001")
     this.consumerConfig.setProperty(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, "2000")
diff --git a/core/src/test/scala/integration/kafka/api/BaseProducerSendTest.scala b/core/src/test/scala/integration/kafka/api/BaseProducerSendTest.scala
index 2f57f8df16e4b..357dc44aa31fe 100644
--- a/core/src/test/scala/integration/kafka/api/BaseProducerSendTest.scala
+++ b/core/src/test/scala/integration/kafka/api/BaseProducerSendTest.scala
@@ -57,13 +57,13 @@ abstract class BaseProducerSendTest extends KafkaServerTestHarness {
   private val numRecords = 100
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
     consumer = TestUtils.createConsumer(TestUtils.getBrokerListStrFromServers(servers), securityProtocol = SecurityProtocol.PLAINTEXT)
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     consumer.close()
     // Ensure that all producers are closed since unclosed producers impact other tests when Kafka server ports are reused
     producers.foreach(_.close())
@@ -100,14 +100,14 @@ abstract class BaseProducerSendTest extends KafkaServerTestHarness {
    * 2. Last message of the non-blocking send should return the correct offset metadata
    */
   @Test
-  def testSendOffset() {
+  def testSendOffset(): Unit = {
     val producer = createProducer(brokerList)
     val partition = 0
 
     object callback extends Callback {
       var offset = 0L
 
-      def onCompletion(metadata: RecordMetadata, exception: Exception) {
+      def onCompletion(metadata: RecordMetadata, exception: Exception): Unit = {
         if (exception == null) {
           assertEquals(offset, metadata.offset())
           assertEquals(topic, metadata.topic())
@@ -172,7 +172,7 @@ abstract class BaseProducerSendTest extends KafkaServerTestHarness {
   }
 
   @Test
-  def testSendCompressedMessageWithCreateTime() {
+  def testSendCompressedMessageWithCreateTime(): Unit = {
     val producer = createProducer(brokerList = brokerList,
       compressionType = "gzip",
       lingerMs = Int.MaxValue,
@@ -181,14 +181,14 @@ abstract class BaseProducerSendTest extends KafkaServerTestHarness {
   }
 
   @Test
-  def testSendNonCompressedMessageWithCreateTime() {
+  def testSendNonCompressedMessageWithCreateTime(): Unit = {
     val producer = createProducer(brokerList = brokerList, lingerMs = Int.MaxValue, deliveryTimeoutMs = Int.MaxValue)
     sendAndVerifyTimestamp(producer, TimestampType.CREATE_TIME)
   }
 
   protected def sendAndVerify(producer: KafkaProducer[Array[Byte], Array[Byte]],
                               numRecords: Int = numRecords,
-                              timeoutMs: Long = 20000L) {
+                              timeoutMs: Long = 20000L): Unit = {
     val partition = 0
     try {
       createTopic(topic, 1, 2)
@@ -212,7 +212,7 @@ abstract class BaseProducerSendTest extends KafkaServerTestHarness {
     }
   }
 
-  protected def sendAndVerifyTimestamp(producer: KafkaProducer[Array[Byte], Array[Byte]], timestampType: TimestampType) {
+  protected def sendAndVerifyTimestamp(producer: KafkaProducer[Array[Byte], Array[Byte]], timestampType: TimestampType): Unit = {
     val partition = 0
 
     val baseTimestamp = 123456L
@@ -222,7 +222,7 @@ abstract class BaseProducerSendTest extends KafkaServerTestHarness {
       var offset = 0L
       var timestampDiff = 1L
 
-      def onCompletion(metadata: RecordMetadata, exception: Exception) {
+      def onCompletion(metadata: RecordMetadata, exception: Exception): Unit = {
         if (exception == null) {
           assertEquals(offset, metadata.offset)
           assertEquals(topic, metadata.topic)
@@ -273,7 +273,7 @@ abstract class BaseProducerSendTest extends KafkaServerTestHarness {
    * After close() returns, all messages should be sent with correct returned offset metadata
    */
   @Test
-  def testClose() {
+  def testClose(): Unit = {
     val producer = createProducer(brokerList)
 
     try {
@@ -306,7 +306,7 @@ abstract class BaseProducerSendTest extends KafkaServerTestHarness {
    * The specified partition-id should be respected
    */
   @Test
-  def testSendToPartition() {
+  def testSendToPartition(): Unit = {
     val producer = createProducer(brokerList)
 
     try {
@@ -351,7 +351,7 @@ abstract class BaseProducerSendTest extends KafkaServerTestHarness {
     * succeed as long as the partition is included in the metadata.
     */
   @Test
-  def testSendBeforeAndAfterPartitionExpansion() {
+  def testSendBeforeAndAfterPartitionExpansion(): Unit = {
     val producer = createProducer(brokerList, maxBlockMs = 5 * 1000L)
 
     // create topic
@@ -417,7 +417,7 @@ abstract class BaseProducerSendTest extends KafkaServerTestHarness {
    * Test that flush immediately sends all accumulated requests.
    */
   @Test
-  def testFlush() {
+  def testFlush(): Unit = {
     val producer = createProducer(brokerList, lingerMs = Int.MaxValue, deliveryTimeoutMs = Int.MaxValue)
     try {
       createTopic(topic, 2, 2)
@@ -438,7 +438,7 @@ abstract class BaseProducerSendTest extends KafkaServerTestHarness {
    * Test close with zero timeout from caller thread
    */
   @Test
-  def testCloseWithZeroTimeoutFromCallerThread() {
+  def testCloseWithZeroTimeoutFromCallerThread(): Unit = {
     createTopic(topic, 2, 2)
     val partition = 0
     consumer.assign(List(new TopicPartition(topic, partition)).asJava)
@@ -467,7 +467,7 @@ abstract class BaseProducerSendTest extends KafkaServerTestHarness {
    * Test close with zero and non-zero timeout from sender thread
    */
   @Test
-  def testCloseWithZeroTimeoutFromSenderThread() {
+  def testCloseWithZeroTimeoutFromSenderThread(): Unit = {
     createTopic(topic, 1, 2)
     val partition = 0
     consumer.assign(List(new TopicPartition(topic, partition)).asJava)
@@ -475,7 +475,7 @@ abstract class BaseProducerSendTest extends KafkaServerTestHarness {
 
     // Test closing from sender thread.
     class CloseCallback(producer: KafkaProducer[Array[Byte], Array[Byte]], sendRecords: Boolean) extends Callback {
-      override def onCompletion(metadata: RecordMetadata, exception: Exception) {
+      override def onCompletion(metadata: RecordMetadata, exception: Exception): Unit = {
         // Trigger another batch in accumulator before close the producer. These messages should
         // not be sent.
         if (sendRecords)
diff --git a/core/src/test/scala/integration/kafka/api/BaseQuotaTest.scala b/core/src/test/scala/integration/kafka/api/BaseQuotaTest.scala
index d9bc646410fee..651eef6188706 100644
--- a/core/src/test/scala/integration/kafka/api/BaseQuotaTest.scala
+++ b/core/src/test/scala/integration/kafka/api/BaseQuotaTest.scala
@@ -66,7 +66,7 @@ abstract class BaseQuotaTest extends IntegrationTestHarness {
   var quotaTestClients: QuotaTestClients = _
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
 
     val numPartitions = 1
@@ -77,7 +77,7 @@ abstract class BaseQuotaTest extends IntegrationTestHarness {
   }
 
   @Test
-  def testThrottledProducerConsumer() {
+  def testThrottledProducerConsumer(): Unit = {
     val numRecords = 1000
     val produced = quotaTestClients.produceUntilThrottled(numRecords)
     quotaTestClients.verifyProduceThrottle(expectThrottle = true)
@@ -88,7 +88,7 @@ abstract class BaseQuotaTest extends IntegrationTestHarness {
   }
 
   @Test
-  def testProducerConsumerOverrideUnthrottled() {
+  def testProducerConsumerOverrideUnthrottled(): Unit = {
     // Give effectively unlimited quota for producer and consumer
     val props = new Properties()
     props.put(DynamicConfig.Client.ProducerByteRateOverrideProp, Long.MaxValue.toString)
@@ -107,7 +107,7 @@ abstract class BaseQuotaTest extends IntegrationTestHarness {
   }
 
   @Test
-  def testQuotaOverrideDelete() {
+  def testQuotaOverrideDelete(): Unit = {
     // Override producer and consumer quotas to unlimited
     quotaTestClients.overrideQuotas(Long.MaxValue, Long.MaxValue, Int.MaxValue)
     quotaTestClients.waitForQuotaUpdate(Long.MaxValue, Long.MaxValue, Int.MaxValue)
@@ -132,7 +132,7 @@ abstract class BaseQuotaTest extends IntegrationTestHarness {
   }
 
   @Test
-  def testThrottledRequest() {
+  def testThrottledRequest(): Unit = {
     quotaTestClients.overrideQuotas(Long.MaxValue, Long.MaxValue, 0.1)
     quotaTestClients.waitForQuotaUpdate(Long.MaxValue, Long.MaxValue, 0.1)
 
@@ -168,8 +168,8 @@ abstract class QuotaTestClients(topic: String,
                                 val consumer: KafkaConsumer[Array[Byte], Array[Byte]]) {
 
   def userPrincipal: KafkaPrincipal
-  def overrideQuotas(producerQuota: Long, consumerQuota: Long, requestQuota: Double)
-  def removeQuotaOverrides()
+  def overrideQuotas(producerQuota: Long, consumerQuota: Long, requestQuota: Double): Unit
+  def removeQuotaOverrides(): Unit
 
   def quotaMetricTags(clientId: String): Map[String, String]
 
@@ -248,7 +248,7 @@ abstract class QuotaTestClients(topic: String,
     leaderNode.metrics.metrics.get(metricName)
   }
 
-  def verifyProducerClientThrottleTimeMetric(expectThrottle: Boolean) {
+  def verifyProducerClientThrottleTimeMetric(expectThrottle: Boolean): Unit = {
     val tags = new HashMap[String, String]
     tags.put("client-id", producerClientId)
     val avgMetric = producer.metrics.get(new MetricName("produce-throttle-time-avg", "producer-metrics", "", tags))
@@ -261,7 +261,7 @@ abstract class QuotaTestClients(topic: String,
       assertEquals("Should not have been throttled", 0.0, metricValue(maxMetric), 0.0)
   }
 
-  def verifyConsumerClientThrottleTimeMetric(expectThrottle: Boolean, maxThrottleTime: Option[Double] = None) {
+  def verifyConsumerClientThrottleTimeMetric(expectThrottle: Boolean, maxThrottleTime: Option[Double] = None): Unit = {
     val tags = new HashMap[String, String]
     tags.put("client-id", consumerClientId)
     val avgMetric = consumer.metrics.get(new MetricName("fetch-throttle-time-avg", "consumer-fetch-manager-metrics", "", tags))
@@ -284,7 +284,7 @@ abstract class QuotaTestClients(topic: String,
     props
   }
 
-  def waitForQuotaUpdate(producerQuota: Long, consumerQuota: Long, requestQuota: Double, server: KafkaServer = leaderNode) {
+  def waitForQuotaUpdate(producerQuota: Long, consumerQuota: Long, requestQuota: Double, server: KafkaServer = leaderNode): Unit = {
     TestUtils.retry(10000) {
       val quotaManagers = server.dataPlaneRequestProcessor.quotas
       val overrideProducerQuota = quota(quotaManagers.produce, userPrincipal, producerClientId)
diff --git a/core/src/test/scala/integration/kafka/api/ClientIdQuotaTest.scala b/core/src/test/scala/integration/kafka/api/ClientIdQuotaTest.scala
index ef8fb413e5262..386c39d969655 100644
--- a/core/src/test/scala/integration/kafka/api/ClientIdQuotaTest.scala
+++ b/core/src/test/scala/integration/kafka/api/ClientIdQuotaTest.scala
@@ -27,7 +27,7 @@ class ClientIdQuotaTest extends BaseQuotaTest {
   override def consumerClientId = "QuotasTestConsumer-!@#$%^&*()"
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     this.serverConfig.setProperty(KafkaConfig.ProducerQuotaBytesPerSecondDefaultProp, defaultProducerQuota.toString)
     this.serverConfig.setProperty(KafkaConfig.ConsumerQuotaBytesPerSecondDefaultProp, defaultConsumerQuota.toString)
     super.setUp()
@@ -43,7 +43,7 @@ class ClientIdQuotaTest extends BaseQuotaTest {
         Map("user" -> "", "client-id" -> clientId)
       }
 
-      override def overrideQuotas(producerQuota: Long, consumerQuota: Long, requestQuota: Double) {
+      override def overrideQuotas(producerQuota: Long, consumerQuota: Long, requestQuota: Double): Unit = {
         val producerProps = new Properties()
         producerProps.put(DynamicConfig.Client.ProducerByteRateOverrideProp, producerQuota.toString)
         producerProps.put(DynamicConfig.Client.RequestPercentageOverrideProp, requestQuota.toString)
@@ -55,13 +55,13 @@ class ClientIdQuotaTest extends BaseQuotaTest {
         updateQuotaOverride(consumerClientId, consumerProps)
       }
 
-      override def removeQuotaOverrides() {
+      override def removeQuotaOverrides(): Unit = {
         val emptyProps = new Properties
         updateQuotaOverride(producerClientId, emptyProps)
         updateQuotaOverride(consumerClientId, emptyProps)
       }
 
-      private def updateQuotaOverride(clientId: String, properties: Properties) {
+      private def updateQuotaOverride(clientId: String, properties: Properties): Unit = {
         adminZkClient.changeClientIdConfig(Sanitizer.sanitize(clientId), properties)
       }
     }
diff --git a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
index 634bcda16a7bb..209eac0f506be 100644
--- a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
+++ b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
@@ -65,7 +65,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     try {
       consumerPollers.foreach(_.shutdown())
       executor.shutdownNow()
@@ -84,7 +84,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
    * 1. Produce a bunch of messages
    * 2. Then consume the messages while killing and restarting brokers at random
    */
-  def consumeWithBrokerFailures(numIters: Int) {
+  def consumeWithBrokerFailures(numIters: Int): Unit = {
     val numRecords = 1000
     val producer = createProducer()
     sendRecords(producer, numRecords)
@@ -122,7 +122,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
   @Test
   def testSeekAndCommitWithBrokerFailures() = seekAndCommitWithBrokerFailures(5)
 
-  def seekAndCommitWithBrokerFailures(numIters: Int) {
+  def seekAndCommitWithBrokerFailures(numIters: Int): Unit = {
     val numRecords = 1000
     val producer = createProducer()
     sendRecords(producer, numRecords)
@@ -159,7 +159,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
   }
 
   @Test
-  def testSubscribeWhenTopicUnavailable() {
+  def testSubscribeWhenTopicUnavailable(): Unit = {
     val numRecords = 1000
     val newtopic = "newtopic"
 
@@ -172,7 +172,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
 
     val producer = createProducer()
 
-    def sendRecords(numRecords: Int, topic: String) {
+    def sendRecords(numRecords: Int, topic: String): Unit = {
       var remainingRecords = numRecords
       val endTimeMs = System.currentTimeMillis + 20000
       while (remainingRecords > 0 && System.currentTimeMillis < endTimeMs) {
@@ -210,7 +210,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
   }
 
   @Test
-  def testClose() {
+  def testClose(): Unit = {
     val numRecords = 10
     val producer = createProducer()
     sendRecords(producer, numRecords)
@@ -225,7 +225,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
    * and leave group. New consumer instance should be able join group and start consuming from
    * last committed offset.
    */
-  private def checkCloseGoodPath(numRecords: Int, groupId: String) {
+  private def checkCloseGoodPath(numRecords: Int, groupId: String): Unit = {
     val consumer = createConsumerAndReceive(groupId, false, numRecords)
     val future = submitCloseAndValidate(consumer, Long.MaxValue, None, gracefulCloseTimeMs)
     future.get
@@ -238,7 +238,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
    * Close of consumers using manual assignment should complete with successful commits since a
    * broker is available.
    */
-  private def checkCloseWithCoordinatorFailure(numRecords: Int, dynamicGroup: String, manualGroup: String) {
+  private def checkCloseWithCoordinatorFailure(numRecords: Int, dynamicGroup: String, manualGroup: String): Unit = {
     val consumer1 = createConsumerAndReceive(dynamicGroup, false, numRecords)
     val consumer2 = createConsumerAndReceive(manualGroup, true, numRecords)
 
@@ -277,7 +277,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
    * there is no coordinator, but close should timeout and return. If close is invoked with a very
    * large timeout, close should timeout after request timeout.
    */
-  private def checkCloseWithClusterFailure(numRecords: Int, group1: String, group2: String) {
+  private def checkCloseWithClusterFailure(numRecords: Int, group1: String, group2: String): Unit = {
     val consumer1 = createConsumerAndReceive(group1, false, numRecords)
 
     val requestTimeout = 6000
@@ -373,7 +373,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
    * close should terminate immediately without sending leave group.
    */
   @Test
-  def testCloseDuringRebalance() {
+  def testCloseDuringRebalance(): Unit = {
     val topic = "closetest"
     createTopic(topic, 10, brokerCount)
     this.consumerConfig.setProperty(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, "60000")
@@ -382,7 +382,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
     checkCloseDuringRebalance("group1", topic, executor, true)
   }
 
-  private def checkCloseDuringRebalance(groupId: String, topic: String, executor: ExecutorService, brokersAvailableDuringClose: Boolean) {
+  private def checkCloseDuringRebalance(groupId: String, topic: String, executor: ExecutorService, brokersAvailableDuringClose: Boolean): Unit = {
 
     def subscribeAndPoll(consumer: KafkaConsumer[Array[Byte], Array[Byte]], revokeSemaphore: Option[Semaphore] = None): Future[Any] = {
       executor.submit(CoreUtils.runnable {
@@ -393,7 +393,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
         }, 0)
     }
 
-    def waitForRebalance(timeoutMs: Long, future: Future[Any], otherConsumers: KafkaConsumer[Array[Byte], Array[Byte]]*) {
+    def waitForRebalance(timeoutMs: Long, future: Future[Any], otherConsumers: KafkaConsumer[Array[Byte], Array[Byte]]*): Unit = {
       val startMs = System.currentTimeMillis
       while (System.currentTimeMillis < startMs + timeoutMs && !future.isDone)
           otherConsumers.foreach(consumer => consumer.poll(time.Duration.ofMillis(100L)))
@@ -471,16 +471,16 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
     }, 0)
   }
 
-  private def checkClosedState(groupId: String, committedRecords: Int) {
+  private def checkClosedState(groupId: String, committedRecords: Int): Unit = {
     // Check that close was graceful with offsets committed and leave group sent.
     // New instance of consumer should be assigned partitions immediately and should see committed offsets.
     val assignSemaphore = new Semaphore(0)
     val consumer = createConsumerWithGroupId(groupId)
     consumer.subscribe(Collections.singletonList(topic),  new ConsumerRebalanceListener {
-      def onPartitionsAssigned(partitions: Collection[TopicPartition]) {
+      def onPartitionsAssigned(partitions: Collection[TopicPartition]): Unit = {
         assignSemaphore.release()
       }
-      def onPartitionsRevoked(partitions: Collection[TopicPartition]) {
+      def onPartitionsRevoked(partitions: Collection[TopicPartition]): Unit = {
       }})
     consumer.poll(time.Duration.ofSeconds(3L))
     assertTrue("Assignment did not complete on time", assignSemaphore.tryAcquire(1, TimeUnit.SECONDS))
@@ -515,7 +515,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
   private def sendRecords(producer: KafkaProducer[Array[Byte], Array[Byte]],
                           numRecords: Int,
                           topic: String = this.topic,
-                          numPartitions: Option[Int] = None) {
+                          numPartitions: Option[Int] = None): Unit = {
     var partitionIndex = 0
     def getPartition: Int = {
       numPartitions match {
diff --git a/core/src/test/scala/integration/kafka/api/CustomQuotaCallbackTest.scala b/core/src/test/scala/integration/kafka/api/CustomQuotaCallbackTest.scala
index b661eae58c83f..15cb6b98166a3 100644
--- a/core/src/test/scala/integration/kafka/api/CustomQuotaCallbackTest.scala
+++ b/core/src/test/scala/integration/kafka/api/CustomQuotaCallbackTest.scala
@@ -62,7 +62,7 @@ class CustomQuotaCallbackTest extends IntegrationTestHarness with SaslSetup {
   val defaultConsumeQuota = 1000 * 1000 * 1000
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     startSasl(jaasSections(kafkaServerSaslMechanisms, Some("SCRAM-SHA-256"), KafkaSasl, JaasTestUtils.KafkaServerContextName))
     this.serverConfig.setProperty(KafkaConfig.ProducerQuotaBytesPerSecondDefaultProp, Long.MaxValue.toString)
     this.serverConfig.setProperty(KafkaConfig.ConsumerQuotaBytesPerSecondDefaultProp, Long.MaxValue.toString)
@@ -85,14 +85,14 @@ class CustomQuotaCallbackTest extends IntegrationTestHarness with SaslSetup {
     super.tearDown()
   }
 
-  override def configureSecurityBeforeServersStart() {
+  override def configureSecurityBeforeServersStart(): Unit = {
     super.configureSecurityBeforeServersStart()
     zkClient.makeSurePersistentPathExists(ConfigEntityChangeNotificationZNode.path)
     createScramCredentials(zkConnect, JaasTestUtils.KafkaScramAdmin, JaasTestUtils.KafkaScramAdminPassword)
   }
 
   @Test
-  def testCustomQuotaCallback() {
+  def testCustomQuotaCallback(): Unit = {
     // Large quota override, should not throttle
     var brokerId = 0
     var user = createGroupWithOneUser("group0_user1", brokerId)
diff --git a/core/src/test/scala/integration/kafka/api/DelegationTokenEndToEndAuthorizationTest.scala b/core/src/test/scala/integration/kafka/api/DelegationTokenEndToEndAuthorizationTest.scala
index 65df797df622e..fa41baa8ddd46 100644
--- a/core/src/test/scala/integration/kafka/api/DelegationTokenEndToEndAuthorizationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/DelegationTokenEndToEndAuthorizationTest.scala
@@ -47,14 +47,14 @@ class DelegationTokenEndToEndAuthorizationTest extends EndToEndAuthorizationTest
 
   this.serverConfig.setProperty(KafkaConfig.DelegationTokenMasterKeyProp, "testKey")
 
-  override def configureSecurityBeforeServersStart() {
+  override def configureSecurityBeforeServersStart(): Unit = {
     super.configureSecurityBeforeServersStart()
     zkClient.makeSurePersistentPathExists(ConfigEntityChangeNotificationZNode.path)
     // Create broker admin credentials before starting brokers
     createScramCredentials(zkConnect, kafkaPrincipal, kafkaPassword)
   }
 
-  override def configureSecurityAfterServersStart() {
+  override def configureSecurityAfterServersStart(): Unit = {
     super.configureSecurityAfterServersStart()
 
     // create scram credential for user "scram-user"
@@ -79,7 +79,7 @@ class DelegationTokenEndToEndAuthorizationTest extends EndToEndAuthorizationTest
   }
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     startSasl(jaasSections(kafkaServerSaslMechanisms, Option(kafkaClientSaslMechanism), Both))
     super.setUp()
   }
diff --git a/core/src/test/scala/integration/kafka/api/DescribeAuthorizedOperationsTest.scala b/core/src/test/scala/integration/kafka/api/DescribeAuthorizedOperationsTest.scala
index d484f465cb9eb..a28e4dddeb62b 100644
--- a/core/src/test/scala/integration/kafka/api/DescribeAuthorizedOperationsTest.scala
+++ b/core/src/test/scala/integration/kafka/api/DescribeAuthorizedOperationsTest.scala
@@ -45,7 +45,7 @@ class DescribeAuthorizedOperationsTest extends IntegrationTestHarness with SaslS
 
   override protected lazy val trustStoreFile = Some(File.createTempFile("truststore", ".jks"))
 
-  override def configureSecurityBeforeServersStart() {
+  override def configureSecurityBeforeServersStart(): Unit = {
     val authorizer = CoreUtils.createObject[Authorizer](classOf[SimpleAclAuthorizer].getName)
     val topicResource = Resource(Topic, Resource.WildCardResource, PatternType.LITERAL)
 
diff --git a/core/src/test/scala/integration/kafka/api/EndToEndAuthorizationTest.scala b/core/src/test/scala/integration/kafka/api/EndToEndAuthorizationTest.scala
index 57b8c394b2f7d..ede80ac31d661 100644
--- a/core/src/test/scala/integration/kafka/api/EndToEndAuthorizationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/EndToEndAuthorizationTest.scala
@@ -60,7 +60,7 @@ import scala.collection.JavaConverters._
 abstract class EndToEndAuthorizationTest extends IntegrationTestHarness with SaslSetup {
   override val brokerCount = 3
 
-  override def configureSecurityBeforeServersStart() {
+  override def configureSecurityBeforeServersStart(): Unit = {
     AclCommand.main(clusterActionArgs)
     AclCommand.main(topicBrokerReadAclArgs)
   }
@@ -179,7 +179,7 @@ abstract class EndToEndAuthorizationTest extends IntegrationTestHarness with Sas
     * Starts MiniKDC and only then sets up the parent trait.
     */
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
     servers.foreach { s =>
       TestUtils.waitAndVerifyAcls(ClusterActionAcl, s.dataPlaneRequestProcessor.authorizer.get, Resource.ClusterResource)
@@ -193,7 +193,7 @@ abstract class EndToEndAuthorizationTest extends IntegrationTestHarness with Sas
     * Closes MiniKDC last when tearing down.
     */
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     super.tearDown()
     closeSasl()
   }
@@ -210,7 +210,7 @@ abstract class EndToEndAuthorizationTest extends IntegrationTestHarness with Sas
     confirmReauthenticationMetrics
   }
 
-  protected def confirmReauthenticationMetrics() : Unit = {
+  protected def confirmReauthenticationMetrics(): Unit = {
     val expiredConnectionsKilledCountTotal = getGauge("ExpiredConnectionsKilledCount").value()
     servers.foreach { s =>
         val numExpiredKilled = TestUtils.totalMetricValue(s, "expired-connections-killed-count")
@@ -272,7 +272,7 @@ abstract class EndToEndAuthorizationTest extends IntegrationTestHarness with Sas
     confirmReauthenticationMetrics
   }
 
-  private def setWildcardResourceAcls() {
+  private def setWildcardResourceAcls(): Unit = {
     AclCommand.main(produceConsumeWildcardAclArgs)
     servers.foreach { s =>
       TestUtils.waitAndVerifyAcls(TopicReadAcl ++ TopicWriteAcl ++ TopicDescribeAcl ++ TopicCreateAcl ++ TopicBrokerReadAcl, s.dataPlaneRequestProcessor.authorizer.get, wildcardTopicResource)
@@ -280,7 +280,7 @@ abstract class EndToEndAuthorizationTest extends IntegrationTestHarness with Sas
     }
   }
 
-  private def setPrefixedResourceAcls() {
+  private def setPrefixedResourceAcls(): Unit = {
     AclCommand.main(produceConsumePrefixedAclsArgs)
     servers.foreach { s =>
       TestUtils.waitAndVerifyAcls(TopicReadAcl ++ TopicWriteAcl ++ TopicDescribeAcl ++ TopicCreateAcl, s.dataPlaneRequestProcessor.authorizer.get, prefixedTopicResource)
@@ -288,7 +288,7 @@ abstract class EndToEndAuthorizationTest extends IntegrationTestHarness with Sas
     }
   }
 
-  private def setReadAndWriteAcls(tp: TopicPartition) {
+  private def setReadAndWriteAcls(tp: TopicPartition): Unit = {
     AclCommand.main(produceAclArgs(tp.topic))
     AclCommand.main(consumeAclArgs(tp.topic))
     servers.foreach { s =>
@@ -298,13 +298,13 @@ abstract class EndToEndAuthorizationTest extends IntegrationTestHarness with Sas
     }
   }
 
-  protected def setAclsAndProduce(tp: TopicPartition) {
+  protected def setAclsAndProduce(tp: TopicPartition): Unit = {
     setReadAndWriteAcls(tp)
     val producer = createProducer()
     sendRecords(producer, numRecords, tp)
   }
 
-  private def setConsumerGroupAcls() {
+  private def setConsumerGroupAcls(): Unit = {
     AclCommand.main(groupAclArgs)
     servers.foreach { s =>
       TestUtils.waitAndVerifyAcls(GroupReadAcl, s.dataPlaneRequestProcessor.authorizer.get, groupResource)
@@ -500,7 +500,7 @@ abstract class EndToEndAuthorizationTest extends IntegrationTestHarness with Sas
   }
 
   protected final def sendRecords(producer: KafkaProducer[Array[Byte], Array[Byte]],
-                                  numRecords: Int, tp: TopicPartition) {
+                                  numRecords: Int, tp: TopicPartition): Unit = {
     val futures = (0 until numRecords).map { i =>
       val record = new ProducerRecord(tp.topic(), tp.partition(), s"$i".getBytes, s"$i".getBytes)
       debug(s"Sending this record: $record")
@@ -518,7 +518,7 @@ abstract class EndToEndAuthorizationTest extends IntegrationTestHarness with Sas
                                      startingOffset: Int = 0,
                                      topic: String = topic,
                                      part: Int = part,
-                                     timeout: Long = 10000) {
+                                     timeout: Long = 10000): Unit = {
     val records = TestUtils.consumeRecords(consumer, numRecords, timeout)
 
     for (i <- 0 until numRecords) {
diff --git a/core/src/test/scala/integration/kafka/api/EndToEndClusterIdTest.scala b/core/src/test/scala/integration/kafka/api/EndToEndClusterIdTest.scala
index 04dcf28a68094..072c44d1a4d90 100644
--- a/core/src/test/scala/integration/kafka/api/EndToEndClusterIdTest.scala
+++ b/core/src/test/scala/integration/kafka/api/EndToEndClusterIdTest.scala
@@ -55,7 +55,7 @@ object EndToEndClusterIdTest {
 
   class MockConsumerMetricsReporter extends MockMetricsReporter with ClusterResourceListener {
 
-    override def onUpdate(clusterMetadata: ClusterResource) {
+    override def onUpdate(clusterMetadata: ClusterResource): Unit = {
       MockConsumerMetricsReporter.CLUSTER_META.set(clusterMetadata)
     }
   }
@@ -66,7 +66,7 @@ object EndToEndClusterIdTest {
 
   class MockProducerMetricsReporter extends MockMetricsReporter with ClusterResourceListener {
 
-    override def onUpdate(clusterMetadata: ClusterResource) {
+    override def onUpdate(clusterMetadata: ClusterResource): Unit = {
       MockProducerMetricsReporter.CLUSTER_META.set(clusterMetadata)
     }
   }
@@ -77,7 +77,7 @@ object EndToEndClusterIdTest {
 
   class MockBrokerMetricsReporter extends MockMetricsReporter with ClusterResourceListener {
 
-    override def onUpdate(clusterMetadata: ClusterResource) {
+    override def onUpdate(clusterMetadata: ClusterResource): Unit = {
       MockBrokerMetricsReporter.CLUSTER_META.set(clusterMetadata)
     }
   }
@@ -107,7 +107,7 @@ class EndToEndClusterIdTest extends KafkaServerTestHarness {
   }
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
     MockDeserializer.resetStaticVariables
     // create the consumer offset topic
@@ -115,7 +115,7 @@ class EndToEndClusterIdTest extends KafkaServerTestHarness {
   }
 
   @Test
-  def testEndToEnd() {
+  def testEndToEnd(): Unit = {
     val appendStr = "mock"
     MockConsumerInterceptor.resetCounters()
     MockProducerInterceptor.resetCounters()
@@ -183,7 +183,7 @@ class EndToEndClusterIdTest extends KafkaServerTestHarness {
     MockProducerInterceptor.resetCounters()
   }
 
-  private def sendRecords(producer: KafkaProducer[Array[Byte], Array[Byte]], numRecords: Int, tp: TopicPartition) {
+  private def sendRecords(producer: KafkaProducer[Array[Byte], Array[Byte]], numRecords: Int, tp: TopicPartition): Unit = {
     val futures = (0 until numRecords).map { i =>
       val record = new ProducerRecord(tp.topic(), tp.partition(), s"$i".getBytes, s"$i".getBytes)
       debug(s"Sending this record: $record")
@@ -200,7 +200,7 @@ class EndToEndClusterIdTest extends KafkaServerTestHarness {
                              numRecords: Int,
                              startingOffset: Int = 0,
                              topic: String = topic,
-                             part: Int = part) {
+                             part: Int = part): Unit = {
     val records = TestUtils.consumeRecords(consumer, numRecords)
 
     for (i <- 0 until numRecords) {
diff --git a/core/src/test/scala/integration/kafka/api/GroupCoordinatorIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/GroupCoordinatorIntegrationTest.scala
index 61845cf194505..e29867b886900 100644
--- a/core/src/test/scala/integration/kafka/api/GroupCoordinatorIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/GroupCoordinatorIntegrationTest.scala
@@ -38,7 +38,7 @@ class GroupCoordinatorIntegrationTest extends KafkaServerTestHarness {
   }
 
   @Test
-  def testGroupCoordinatorPropagatesOffsetsTopicCompressionCodec() {
+  def testGroupCoordinatorPropagatesOffsetsTopicCompressionCodec(): Unit = {
     val consumer = TestUtils.createConsumer(TestUtils.getBrokerListStrFromServers(servers))
     val offsetMap = Map(
       new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, 0) -> new OffsetAndMetadata(10, "")
diff --git a/core/src/test/scala/integration/kafka/api/IntegrationTestHarness.scala b/core/src/test/scala/integration/kafka/api/IntegrationTestHarness.scala
index 4efd26211d59f..557b278dfa70d 100644
--- a/core/src/test/scala/integration/kafka/api/IntegrationTestHarness.scala
+++ b/core/src/test/scala/integration/kafka/api/IntegrationTestHarness.scala
@@ -80,7 +80,7 @@ abstract class IntegrationTestHarness extends KafkaServerTestHarness {
   }
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     doSetup(createOffsetsTopic = true)
   }
 
@@ -148,7 +148,7 @@ abstract class IntegrationTestHarness extends KafkaServerTestHarness {
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     producers.foreach(_.close(Duration.ZERO))
     consumers.foreach(_.wakeup())
     consumers.foreach(_.close(Duration.ZERO))
diff --git a/core/src/test/scala/integration/kafka/api/LogAppendTimeTest.scala b/core/src/test/scala/integration/kafka/api/LogAppendTimeTest.scala
index 8d2e66eaa24f3..74a774c682f67 100644
--- a/core/src/test/scala/integration/kafka/api/LogAppendTimeTest.scala
+++ b/core/src/test/scala/integration/kafka/api/LogAppendTimeTest.scala
@@ -42,13 +42,13 @@ class LogAppendTimeTest extends IntegrationTestHarness {
   private val topic = "topic"
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
     createTopic(topic)
   }
 
   @Test
-  def testProduceConsume() {
+  def testProduceConsume(): Unit = {
     val producer = createProducer()
     val now = System.currentTimeMillis()
     val createTime = now - TimeUnit.DAYS.toMillis(1)
diff --git a/core/src/test/scala/integration/kafka/api/MetricsTest.scala b/core/src/test/scala/integration/kafka/api/MetricsTest.scala
index f0ff496e8b9cb..83c75dfb77f2b 100644
--- a/core/src/test/scala/integration/kafka/api/MetricsTest.scala
+++ b/core/src/test/scala/integration/kafka/api/MetricsTest.scala
@@ -134,7 +134,7 @@ class MetricsTest extends IntegrationTestHarness with SaslSetup {
   }
 
   private def verifyKafkaRateMetricsHaveCumulativeCount(producer: KafkaProducer[Array[Byte], Array[Byte]],
-                                                        consumer: KafkaConsumer[Array[Byte], Array[Byte]]): Unit =  {
+                                                        consumer: KafkaConsumer[Array[Byte], Array[Byte]]): Unit = {
 
     def exists(name: String, rateMetricName: MetricName, allMetricNames: Set[MetricName]): Boolean = {
       allMetricNames.contains(new MetricName(name, rateMetricName.group, "", rateMetricName.tags))
diff --git a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
index 30b1d13aad81e..acb0d6b17ba76 100644
--- a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
+++ b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
@@ -44,7 +44,7 @@ import scala.collection.mutable
 class PlaintextConsumerTest extends BaseConsumerTest {
 
   @Test
-  def testHeaders() {
+  def testHeaders(): Unit = {
     val numRecords = 1
     val record = new ProducerRecord(tp.topic, tp.partition, null, "key".getBytes, "value".getBytes)
 
@@ -158,7 +158,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testMaxPollRecords() {
+  def testMaxPollRecords(): Unit = {
     val maxPollRecords = 2
     val numRecords = 10000
 
@@ -172,7 +172,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testMaxPollIntervalMs() {
+  def testMaxPollIntervalMs(): Unit = {
     this.consumerConfig.setProperty(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, 3000.toString)
     this.consumerConfig.setProperty(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, 500.toString)
     this.consumerConfig.setProperty(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 2000.toString)
@@ -196,7 +196,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testMaxPollIntervalMsDelayInRevocation() {
+  def testMaxPollIntervalMsDelayInRevocation(): Unit = {
     this.consumerConfig.setProperty(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, 5000.toString)
     this.consumerConfig.setProperty(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, 500.toString)
     this.consumerConfig.setProperty(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 1000.toString)
@@ -236,7 +236,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testMaxPollIntervalMsDelayInAssignment() {
+  def testMaxPollIntervalMsDelayInAssignment(): Unit = {
     this.consumerConfig.setProperty(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, 5000.toString)
     this.consumerConfig.setProperty(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, 500.toString)
     this.consumerConfig.setProperty(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 1000.toString)
@@ -260,7 +260,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testAutoCommitOnClose() {
+  def testAutoCommitOnClose(): Unit = {
     this.consumerConfig.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true")
     val consumer = createConsumer()
 
@@ -283,7 +283,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testAutoCommitOnCloseAfterWakeup() {
+  def testAutoCommitOnCloseAfterWakeup(): Unit = {
     this.consumerConfig.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true")
     val consumer = createConsumer()
 
@@ -310,7 +310,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testAutoOffsetReset() {
+  def testAutoOffsetReset(): Unit = {
     val producer = createProducer()
     sendRecords(producer, numRecords = 1, tp)
 
@@ -320,7 +320,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testGroupConsumption() {
+  def testGroupConsumption(): Unit = {
     val producer = createProducer()
     sendRecords(producer, numRecords = 10, tp)
 
@@ -339,7 +339,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
    * of that topic are assigned to it.
    */
   @Test
-  def testPatternSubscription() {
+  def testPatternSubscription(): Unit = {
     val numRecords = 10000
     val producer = createProducer()
     sendRecords(producer, numRecords, tp)
@@ -396,7 +396,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
    * that it is the subscription call that triggers a metadata refresh, and not the timeout.
    */
   @Test
-  def testSubsequentPatternSubscription() {
+  def testSubsequentPatternSubscription(): Unit = {
     this.consumerConfig.setProperty(ConsumerConfig.METADATA_MAX_AGE_CONFIG, "30000")
     val consumer = createConsumer()
 
@@ -447,7 +447,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
    * assignments are cleared right away.
    */
   @Test
-  def testPatternUnsubscription() {
+  def testPatternUnsubscription(): Unit = {
     val numRecords = 10000
     val producer = createProducer()
     sendRecords(producer, numRecords, tp)
@@ -473,7 +473,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testCommitMetadata() {
+  def testCommitMetadata(): Unit = {
     val consumer = createConsumer()
     consumer.assign(List(tp).asJava)
 
@@ -494,7 +494,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testAsyncCommit() {
+  def testAsyncCommit(): Unit = {
     val consumer = createConsumer()
     consumer.assign(List(tp).asJava)
 
@@ -513,7 +513,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testExpandingTopicSubscriptions() {
+  def testExpandingTopicSubscriptions(): Unit = {
     val otherTopic = "other"
     val initialAssignment = Set(new TopicPartition(topic, 0), new TopicPartition(topic, 1))
     val consumer = createConsumer()
@@ -527,7 +527,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testShrinkingTopicSubscriptions() {
+  def testShrinkingTopicSubscriptions(): Unit = {
     val otherTopic = "other"
     createTopic(otherTopic, 2, brokerCount)
     val initialAssignment = Set(new TopicPartition(topic, 0), new TopicPartition(topic, 1), new TopicPartition(otherTopic, 0), new TopicPartition(otherTopic, 1))
@@ -541,7 +541,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testPartitionsFor() {
+  def testPartitionsFor(): Unit = {
     val numParts = 2
     createTopic("part-test", numParts, 1)
     val consumer = createConsumer()
@@ -551,20 +551,20 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testPartitionsForAutoCreate() {
+  def testPartitionsForAutoCreate(): Unit = {
     val consumer = createConsumer()
     val partitions = consumer.partitionsFor("non-exist-topic")
     assertFalse(partitions.isEmpty)
   }
 
   @Test(expected = classOf[InvalidTopicException])
-  def testPartitionsForInvalidTopic() {
+  def testPartitionsForInvalidTopic(): Unit = {
     val consumer = createConsumer()
     consumer.partitionsFor(";3# ads,{234")
   }
 
   @Test
-  def testSeek() {
+  def testSeek(): Unit = {
     val consumer = createConsumer()
     val totalRecords = 50L
     val mid = totalRecords / 2
@@ -606,7 +606,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
       startingTimestamp = mid.toLong, tp = tp2)
   }
 
-  private def sendCompressedMessages(numRecords: Int, tp: TopicPartition) {
+  private def sendCompressedMessages(numRecords: Int, tp: TopicPartition): Unit = {
     val producerProps = new Properties()
     producerProps.setProperty(ProducerConfig.COMPRESSION_TYPE_CONFIG, CompressionType.GZIP.name)
     producerProps.setProperty(ProducerConfig.LINGER_MS_CONFIG, Int.MaxValue.toString)
@@ -618,7 +618,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testPositionAndCommit() {
+  def testPositionAndCommit(): Unit = {
     val producer = createProducer()
     sendRecords(producer, numRecords = 5, tp)
 
@@ -650,7 +650,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testPartitionPauseAndResume() {
+  def testPartitionPauseAndResume(): Unit = {
     val partitions = List(tp).asJava
     val producer = createProducer()
     sendRecords(producer, numRecords = 5, tp)
@@ -666,7 +666,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testFetchInvalidOffset() {
+  def testFetchInvalidOffset(): Unit = {
     this.consumerConfig.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none")
     val consumer = createConsumer()
 
@@ -694,7 +694,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testFetchRecordLargerThanFetchMaxBytes() {
+  def testFetchRecordLargerThanFetchMaxBytes(): Unit = {
     val maxFetchBytes = 10 * 1024
     this.consumerConfig.setProperty(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, maxFetchBytes.toString)
     checkLargeRecord(maxFetchBytes + 1)
@@ -762,7 +762,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testFetchRecordLargerThanMaxPartitionFetchBytes() {
+  def testFetchRecordLargerThanMaxPartitionFetchBytes(): Unit = {
     val maxPartitionFetchBytes = 10 * 1024
     this.consumerConfig.setProperty(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, maxPartitionFetchBytes.toString)
     checkLargeRecord(maxPartitionFetchBytes + 1)
@@ -813,7 +813,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testRoundRobinAssignment() {
+  def testRoundRobinAssignment(): Unit = {
     // 1 consumer using round-robin assignment
     this.consumerConfig.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "roundrobin-group")
     this.consumerConfig.setProperty(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, classOf[RoundRobinAssignor].getName)
@@ -849,7 +849,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testMultiConsumerRoundRobinAssignment() {
+  def testMultiConsumerRoundRobinAssignment(): Unit = {
     this.consumerConfig.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "roundrobin-group")
     this.consumerConfig.setProperty(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, classOf[RoundRobinAssignor].getName)
 
@@ -886,7 +886,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
    *    will move to consumer #10, leading to a total of (#par mod 9) partition movement
    */
   @Test
-  def testMultiConsumerStickyAssignment() {
+  def testMultiConsumerStickyAssignment(): Unit = {
 
     def reverse(m: Map[Long, Set[TopicPartition]]) =
       m.values.toSet.flatten.map(v => (v, m.keys.filter(m(_).contains(v)).head)).toMap
@@ -932,7 +932,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
    * As a result, it is testing the default assignment strategy set by BaseConsumerTest
    */
   @Test
-  def testMultiConsumerDefaultAssignment() {
+  def testMultiConsumerDefaultAssignment(): Unit = {
     // use consumers and topics defined in this class + one more topic
     val producer = createProducer()
     sendRecords(producer, numRecords = 100, tp)
@@ -977,7 +977,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testInterceptors() {
+  def testInterceptors(): Unit = {
     val appendStr = "mock"
     MockConsumerInterceptor.resetCounters()
     MockProducerInterceptor.resetCounters()
@@ -1041,7 +1041,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testAutoCommitIntercept() {
+  def testAutoCommitIntercept(): Unit = {
     val topic2 = "topic2"
     createTopic(topic2, 2, brokerCount)
 
@@ -1091,7 +1091,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testInterceptorsWithWrongKeyValue() {
+  def testInterceptorsWithWrongKeyValue(): Unit = {
     val appendStr = "mock"
     // create producer with interceptor that has different key and value types from the producer
     val producerProps = new Properties()
@@ -1117,7 +1117,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testConsumeMessagesWithCreateTime() {
+  def testConsumeMessagesWithCreateTime(): Unit = {
     val numRecords = 50
     // Test non-compressed messages
     val producer = createProducer()
@@ -1135,7 +1135,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testConsumeMessagesWithLogAppendTime() {
+  def testConsumeMessagesWithLogAppendTime(): Unit = {
     val topicName = "testConsumeMessagesWithLogAppendTime"
     val topicProps = new Properties()
     topicProps.setProperty(LogConfig.MessageTimestampTypeProp, "LogAppendTime")
@@ -1163,7 +1163,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testListTopics() {
+  def testListTopics(): Unit = {
     val numParts = 2
     val topic1 = "part-test-topic-1"
     val topic2 = "part-test-topic-2"
@@ -1183,7 +1183,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testOffsetsForTimes() {
+  def testOffsetsForTimes(): Unit = {
     val numParts = 2
     val topic1 = "part-test-topic-1"
     val topic2 = "part-test-topic-2"
@@ -1246,7 +1246,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testEarliestOrLatestOffsets() {
+  def testEarliestOrLatestOffsets(): Unit = {
     val topic0 = "topicWithNewMessageFormat"
     val topic1 = "topicWithOldMessageFormat"
     val producer = createProducer()
@@ -1274,7 +1274,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testUnsubscribeTopic() {
+  def testUnsubscribeTopic(): Unit = {
     this.consumerConfig.setProperty(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "100") // timeout quickly to avoid slow test
     this.consumerConfig.setProperty(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, "30")
     val consumer = createConsumer()
@@ -1290,7 +1290,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testPauseStateNotPreservedByRebalance() {
+  def testPauseStateNotPreservedByRebalance(): Unit = {
     this.consumerConfig.setProperty(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "100") // timeout quickly to avoid slow test
     this.consumerConfig.setProperty(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, "30")
     val consumer = createConsumer()
@@ -1310,7 +1310,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testCommitSpecifiedOffsets() {
+  def testCommitSpecifiedOffsets(): Unit = {
     val producer = createProducer()
     sendRecords(producer, numRecords = 5, tp)
     sendRecords(producer, numRecords = 7, tp2)
@@ -1337,7 +1337,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testAutoCommitOnRebalance() {
+  def testAutoCommitOnRebalance(): Unit = {
     val topic2 = "topic2"
     createTopic(topic2, 2, brokerCount)
 
@@ -1376,7 +1376,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testPerPartitionLeadMetricsCleanUpWithSubscribe() {
+  def testPerPartitionLeadMetricsCleanUpWithSubscribe(): Unit = {
     val numMessages = 1000
     val topic2 = "topic2"
     createTopic(topic2, 2, brokerCount)
@@ -1415,7 +1415,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testPerPartitionLagMetricsCleanUpWithSubscribe() {
+  def testPerPartitionLagMetricsCleanUpWithSubscribe(): Unit = {
     val numMessages = 1000
     val topic2 = "topic2"
     createTopic(topic2, 2, brokerCount)
@@ -1455,7 +1455,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testPerPartitionLeadMetricsCleanUpWithAssign() {
+  def testPerPartitionLeadMetricsCleanUpWithAssign(): Unit = {
     val numMessages = 1000
     // Test assign
     // send some messages.
@@ -1484,7 +1484,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testPerPartitionLagMetricsCleanUpWithAssign() {
+  def testPerPartitionLagMetricsCleanUpWithAssign(): Unit = {
     val numMessages = 1000
     // Test assign
     // send some messages.
@@ -1515,7 +1515,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testPerPartitionLagMetricsWhenReadCommitted() {
+  def testPerPartitionLagMetricsWhenReadCommitted(): Unit = {
     val numMessages = 1000
     // send some messages.
     val producer = createProducer()
@@ -1538,7 +1538,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testPerPartitionLeadWithMaxPollRecords() {
+  def testPerPartitionLeadWithMaxPollRecords(): Unit = {
     val numMessages = 1000
     val maxPollRecords = 10
     val producer = createProducer()
@@ -1560,7 +1560,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testPerPartitionLagWithMaxPollRecords() {
+  def testPerPartitionLagWithMaxPollRecords(): Unit = {
     val numMessages = 1000
     val maxPollRecords = 10
     val producer = createProducer()
@@ -1583,7 +1583,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
   }
 
   @Test
-  def testQuotaMetricsNotCreatedIfNoQuotasConfigured() {
+  def testQuotaMetricsNotCreatedIfNoQuotasConfigured(): Unit = {
     val numRecords = 1000
     val producer = createProducer()
     sendRecords(producer, numRecords, tp)
@@ -1593,7 +1593,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
     consumer.seek(tp, 0)
     consumeAndVerifyRecords(consumer = consumer, numRecords = numRecords, startingOffset = 0)
 
-    def assertNoMetric(broker: KafkaServer, name: String, quotaType: QuotaType, clientId: String) {
+    def assertNoMetric(broker: KafkaServer, name: String, quotaType: QuotaType, clientId: String): Unit = {
         val metricName = broker.metrics.metricName("throttle-time",
                                   quotaType.toString,
                                   "",
@@ -1611,7 +1611,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
     servers.foreach(assertNoMetric(_, "request-time", QuotaType.Request, consumerClientId))
     servers.foreach(assertNoMetric(_, "throttle-time", QuotaType.Request, consumerClientId))
 
-    def assertNoExemptRequestMetric(broker: KafkaServer) {
+    def assertNoExemptRequestMetric(broker: KafkaServer): Unit = {
         val metricName = broker.metrics.metricName("exempt-request-time", QuotaType.Request.toString, "")
         assertNull("Metric should not hanve been created " + metricName, broker.metrics.metric(metricName))
     }
diff --git a/core/src/test/scala/integration/kafka/api/PlaintextEndToEndAuthorizationTest.scala b/core/src/test/scala/integration/kafka/api/PlaintextEndToEndAuthorizationTest.scala
index 5189e826f2879..b8549c577c69a 100644
--- a/core/src/test/scala/integration/kafka/api/PlaintextEndToEndAuthorizationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/PlaintextEndToEndAuthorizationTest.scala
@@ -71,13 +71,13 @@ class PlaintextEndToEndAuthorizationTest extends EndToEndAuthorizationTest {
   override val kafkaPrincipal = "server"
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     startSasl(jaasSections(List.empty, None, ZkSasl))
     super.setUp()
   }
 
   @Test
-  def testListenerName() {
+  def testListenerName(): Unit = {
     // To check the client listener name, establish a session on the server by sending any request eg sendRecords
     val producer = createProducer()
     intercept[TopicAuthorizationException](sendRecords(producer, numRecords = 1, tp))
diff --git a/core/src/test/scala/integration/kafka/api/PlaintextProducerSendTest.scala b/core/src/test/scala/integration/kafka/api/PlaintextProducerSendTest.scala
index 1bb8989315799..1dda60c589a60 100644
--- a/core/src/test/scala/integration/kafka/api/PlaintextProducerSendTest.scala
+++ b/core/src/test/scala/integration/kafka/api/PlaintextProducerSendTest.scala
@@ -31,7 +31,7 @@ import org.junit.Test
 class PlaintextProducerSendTest extends BaseProducerSendTest {
 
   @Test(expected = classOf[SerializationException])
-  def testWrongSerializer() {
+  def testWrongSerializer(): Unit = {
     val producerProps = new Properties()
     producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList)
     producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
@@ -42,7 +42,7 @@ class PlaintextProducerSendTest extends BaseProducerSendTest {
   }
 
   @Test
-  def testBatchSizeZero() {
+  def testBatchSizeZero(): Unit = {
     val producer = createProducer(brokerList = brokerList,
       lingerMs = Int.MaxValue,
       deliveryTimeoutMs = Int.MaxValue,
@@ -51,7 +51,7 @@ class PlaintextProducerSendTest extends BaseProducerSendTest {
   }
 
   @Test
-  def testSendCompressedMessageWithLogAppendTime() {
+  def testSendCompressedMessageWithLogAppendTime(): Unit = {
     val producer = createProducer(brokerList = brokerList,
       compressionType = "gzip",
       lingerMs = Int.MaxValue,
@@ -60,7 +60,7 @@ class PlaintextProducerSendTest extends BaseProducerSendTest {
   }
 
   @Test
-  def testSendNonCompressedMessageWithLogAppendTime() {
+  def testSendNonCompressedMessageWithLogAppendTime(): Unit = {
     val producer = createProducer(brokerList = brokerList, lingerMs = Int.MaxValue, deliveryTimeoutMs = Int.MaxValue)
     sendAndVerifyTimestamp(producer, TimestampType.LOG_APPEND_TIME)
   }
@@ -71,7 +71,7 @@ class PlaintextProducerSendTest extends BaseProducerSendTest {
    * The topic should be created upon sending the first message
    */
   @Test
-  def testAutoCreateTopic() {
+  def testAutoCreateTopic(): Unit = {
     val producer = createProducer(brokerList)
     try {
       // Send a message to auto-create the topic
@@ -87,7 +87,7 @@ class PlaintextProducerSendTest extends BaseProducerSendTest {
   }
 
   @Test
-  def testSendWithInvalidCreateTime() {
+  def testSendWithInvalidCreateTime(): Unit = {
     val topicProps = new Properties()
     topicProps.setProperty(LogConfig.MessageTimestampDifferenceMaxMsProp, "1000")
     createTopic(topic, 1, 2, topicProps)
diff --git a/core/src/test/scala/integration/kafka/api/ProducerCompressionTest.scala b/core/src/test/scala/integration/kafka/api/ProducerCompressionTest.scala
index 24193521f91ee..d65c43e71b556 100755
--- a/core/src/test/scala/integration/kafka/api/ProducerCompressionTest.scala
+++ b/core/src/test/scala/integration/kafka/api/ProducerCompressionTest.scala
@@ -42,14 +42,14 @@ class ProducerCompressionTest(compression: String) extends ZooKeeperTestHarness
   private var server: KafkaServer = null
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
     val props = TestUtils.createBrokerConfig(brokerId, zkConnect)
     server = TestUtils.createServer(KafkaConfig.fromProps(props))
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     TestUtils.shutdownServers(Seq(server))
     super.tearDown()
   }
@@ -60,7 +60,7 @@ class ProducerCompressionTest(compression: String) extends ZooKeeperTestHarness
    * Compressed messages should be able to sent and consumed correctly
    */
   @Test
-  def testCompression() {
+  def testCompression(): Unit = {
 
     val producerProps = new Properties()
     val bootstrapServers = TestUtils.getBrokerListStrFromServers(Seq(server))
diff --git a/core/src/test/scala/integration/kafka/api/ProducerFailureHandlingTest.scala b/core/src/test/scala/integration/kafka/api/ProducerFailureHandlingTest.scala
index 8d69b5f83ba13..beb52323e4689 100644
--- a/core/src/test/scala/integration/kafka/api/ProducerFailureHandlingTest.scala
+++ b/core/src/test/scala/integration/kafka/api/ProducerFailureHandlingTest.scala
@@ -61,7 +61,7 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness {
   private val topic2 = "topic-2"
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
 
     producer1 = TestUtils.createProducer(brokerList, acks = 0, retries = 0, requestTimeoutMs = 30000, maxBlockMs = 10000L,
@@ -73,7 +73,7 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness {
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     if (producer1 != null) producer1.close()
     if (producer2 != null) producer2.close()
     if (producer3 != null) producer3.close()
@@ -86,7 +86,7 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness {
    * With ack == 0 the future metadata will have no exceptions with offset -1
    */
   @Test
-  def testTooLargeRecordWithAckZero() {
+  def testTooLargeRecordWithAckZero(): Unit = {
     // create topic
     createTopic(topic1, replicationFactor = numServers)
 
@@ -103,7 +103,7 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness {
    * With ack == 1 the future metadata will throw ExecutionException caused by RecordTooLargeException
    */
   @Test
-  def testTooLargeRecordWithAckOne() {
+  def testTooLargeRecordWithAckOne(): Unit = {
     // create topic
     createTopic(topic1, replicationFactor = numServers)
 
@@ -114,7 +114,7 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness {
     }
   }
 
-  private def checkTooLargeRecordForReplicationWithAckAll(maxFetchSize: Int) {
+  private def checkTooLargeRecordForReplicationWithAckAll(maxFetchSize: Int): Unit = {
     val maxMessageSize = maxFetchSize + 100
     val topicConfig = new Properties
     topicConfig.setProperty(LogConfig.MinInSyncReplicasProp, numServers.toString)
@@ -134,13 +134,13 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness {
 
   /** This should succeed as the replica fetcher thread can handle oversized messages since KIP-74 */
   @Test
-  def testPartitionTooLargeForReplicationWithAckAll() {
+  def testPartitionTooLargeForReplicationWithAckAll(): Unit = {
     checkTooLargeRecordForReplicationWithAckAll(replicaFetchMaxPartitionBytes)
   }
 
   /** This should succeed as the replica fetcher thread can handle oversized messages since KIP-74 */
   @Test
-  def testResponseTooLargeForReplicationWithAckAll() {
+  def testResponseTooLargeForReplicationWithAckAll(): Unit = {
     checkTooLargeRecordForReplicationWithAckAll(replicaFetchMaxResponseBytes)
   }
 
@@ -148,7 +148,7 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness {
    * With non-exist-topic the future metadata should return ExecutionException caused by TimeoutException
    */
   @Test
-  def testNonExistentTopic() {
+  def testNonExistentTopic(): Unit = {
     // send a record with non-exist topic
     val record = new ProducerRecord(topic2, null, "key".getBytes, "value".getBytes)
     intercept[ExecutionException] {
@@ -167,7 +167,7 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness {
    *    TimeoutException
    */
   @Test
-  def testWrongBrokerList() {
+  def testWrongBrokerList(): Unit = {
     // create topic
     createTopic(topic1, replicationFactor = numServers)
 
@@ -186,7 +186,7 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness {
     * when partition is higher than the upper bound of partitions.
     */
   @Test
-  def testInvalidPartition() {
+  def testInvalidPartition(): Unit = {
     // create topic with a single partition
     createTopic(topic1, numPartitions = 1, replicationFactor = numServers)
 
@@ -204,7 +204,7 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness {
    * The send call after producer closed should throw IllegalStateException
    */
   @Test
-  def testSendAfterClosed() {
+  def testSendAfterClosed(): Unit = {
     // create topic
     createTopic(topic1, replicationFactor = numServers)
 
@@ -230,7 +230,7 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness {
   }
 
   @Test
-  def testCannotSendToInternalTopic() {
+  def testCannotSendToInternalTopic(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
     val thrown = intercept[ExecutionException] {
       producer2.send(new ProducerRecord(Topic.GROUP_METADATA_TOPIC_NAME, "test".getBytes, "test".getBytes)).get
@@ -239,7 +239,7 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness {
   }
 
   @Test
-  def testNotEnoughReplicas() {
+  def testNotEnoughReplicas(): Unit = {
     val topicName = "minisrtest"
     val topicProps = new Properties()
     topicProps.put("min.insync.replicas",(numServers+1).toString)
@@ -259,7 +259,7 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness {
   }
 
   @Test
-  def testNotEnoughReplicasAfterBrokerShutdown() {
+  def testNotEnoughReplicasAfterBrokerShutdown(): Unit = {
     val topicName = "minisrtest2"
     val topicProps = new Properties()
     topicProps.put("min.insync.replicas", numServers.toString)
diff --git a/core/src/test/scala/integration/kafka/api/RackAwareAutoTopicCreationTest.scala b/core/src/test/scala/integration/kafka/api/RackAwareAutoTopicCreationTest.scala
index 5fc626bb0bbe5..4ba3fd5eac945 100644
--- a/core/src/test/scala/integration/kafka/api/RackAwareAutoTopicCreationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/RackAwareAutoTopicCreationTest.scala
@@ -43,7 +43,7 @@ class RackAwareAutoTopicCreationTest extends KafkaServerTestHarness with RackAwa
   private val topic = "topic"
 
   @Test
-  def testAutoCreateTopic() {
+  def testAutoCreateTopic(): Unit = {
     val producer = TestUtils.createProducer(brokerList)
     try {
       // Send a message to auto-create the topic
diff --git a/core/src/test/scala/integration/kafka/api/SaslClientsWithInvalidCredentialsTest.scala b/core/src/test/scala/integration/kafka/api/SaslClientsWithInvalidCredentialsTest.scala
index faf2fbcc718a3..0155a739335ac 100644
--- a/core/src/test/scala/integration/kafka/api/SaslClientsWithInvalidCredentialsTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SaslClientsWithInvalidCredentialsTest.scala
@@ -50,7 +50,7 @@ class SaslClientsWithInvalidCredentialsTest extends IntegrationTestHarness with
   val numPartitions = 1
   val tp = new TopicPartition(topic, 0)
 
-  override def configureSecurityBeforeServersStart() {
+  override def configureSecurityBeforeServersStart(): Unit = {
     super.configureSecurityBeforeServersStart()
     zkClient.makeSurePersistentPathExists(ConfigEntityChangeNotificationZNode.path)
     // Create broker credentials before starting brokers
@@ -72,7 +72,7 @@ class SaslClientsWithInvalidCredentialsTest extends IntegrationTestHarness with
   }
 
   @Test
-  def testProducerWithAuthenticationFailure() {
+  def testProducerWithAuthenticationFailure(): Unit = {
     val producer = createProducer()
     verifyAuthenticationException(sendOneRecord(producer, maxWaitMs = 10000))
     verifyAuthenticationException(producer.partitionsFor(topic))
@@ -82,7 +82,7 @@ class SaslClientsWithInvalidCredentialsTest extends IntegrationTestHarness with
   }
 
   @Test
-  def testTransactionalProducerWithAuthenticationFailure() {
+  def testTransactionalProducerWithAuthenticationFailure(): Unit = {
     val txProducer = createTransactionalProducer()
     verifyAuthenticationException(txProducer.initTransactions())
 
@@ -96,21 +96,21 @@ class SaslClientsWithInvalidCredentialsTest extends IntegrationTestHarness with
   }
 
   @Test
-  def testConsumerWithAuthenticationFailure() {
+  def testConsumerWithAuthenticationFailure(): Unit = {
     val consumer = createConsumer()
     consumer.subscribe(List(topic).asJava)
     verifyConsumerWithAuthenticationFailure(consumer)
   }
 
   @Test
-  def testManualAssignmentConsumerWithAuthenticationFailure() {
+  def testManualAssignmentConsumerWithAuthenticationFailure(): Unit = {
     val consumer = createConsumer()
     consumer.assign(List(tp).asJava)
     verifyConsumerWithAuthenticationFailure(consumer)
   }
 
   @Test
-  def testManualAssignmentConsumerWithAutoCommitDisabledWithAuthenticationFailure() {
+  def testManualAssignmentConsumerWithAutoCommitDisabledWithAuthenticationFailure(): Unit = {
     this.consumerConfig.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false.toString)
     val consumer = createConsumer()
     consumer.assign(List(tp).asJava)
@@ -118,7 +118,7 @@ class SaslClientsWithInvalidCredentialsTest extends IntegrationTestHarness with
     verifyConsumerWithAuthenticationFailure(consumer)
   }
 
-  private def verifyConsumerWithAuthenticationFailure(consumer: KafkaConsumer[Array[Byte], Array[Byte]]) {
+  private def verifyConsumerWithAuthenticationFailure(consumer: KafkaConsumer[Array[Byte], Array[Byte]]): Unit = {
     verifyAuthenticationException(consumer.poll(Duration.ofMillis(1000)))
     verifyAuthenticationException(consumer.partitionsFor(topic))
 
@@ -129,7 +129,7 @@ class SaslClientsWithInvalidCredentialsTest extends IntegrationTestHarness with
   }
 
   @Test
-  def testKafkaAdminClientWithAuthenticationFailure() {
+  def testKafkaAdminClientWithAuthenticationFailure(): Unit = {
     val props = TestUtils.adminClientSecurityConfigs(securityProtocol, trustStoreFile, clientSaslProperties)
     props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList)
     val adminClient = AdminClient.create(props)
@@ -157,7 +157,7 @@ class SaslClientsWithInvalidCredentialsTest extends IntegrationTestHarness with
   }
 
   @Test
-  def testConsumerGroupServiceWithAuthenticationFailure() {
+  def testConsumerGroupServiceWithAuthenticationFailure(): Unit = {
     val consumerGroupService: ConsumerGroupService = prepareConsumerGroupService
 
     val consumer = createConsumer()
@@ -168,7 +168,7 @@ class SaslClientsWithInvalidCredentialsTest extends IntegrationTestHarness with
   }
 
   @Test
-  def testConsumerGroupServiceWithAuthenticationSuccess() {
+  def testConsumerGroupServiceWithAuthenticationSuccess(): Unit = {
     createClientCredential()
     val consumerGroupService: ConsumerGroupService = prepareConsumerGroupService
 
diff --git a/core/src/test/scala/integration/kafka/api/SaslEndToEndAuthorizationTest.scala b/core/src/test/scala/integration/kafka/api/SaslEndToEndAuthorizationTest.scala
index f51ec6e3636cc..a1b867ef753dd 100644
--- a/core/src/test/scala/integration/kafka/api/SaslEndToEndAuthorizationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SaslEndToEndAuthorizationTest.scala
@@ -35,7 +35,7 @@ abstract class SaslEndToEndAuthorizationTest extends EndToEndAuthorizationTest {
   protected def kafkaServerSaslMechanisms: List[String]
   
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     // create static config including client login context with credentials for JaasTestUtils 'client2'
     startSasl(jaasSections(kafkaServerSaslMechanisms, Option(kafkaClientSaslMechanism), Both))
     // set dynamic properties with credentials for JaasTestUtils 'client1' so that dynamic JAAS configuration is also
diff --git a/core/src/test/scala/integration/kafka/api/SaslMultiMechanismConsumerTest.scala b/core/src/test/scala/integration/kafka/api/SaslMultiMechanismConsumerTest.scala
index 94b5e6f673255..b24937a039432 100644
--- a/core/src/test/scala/integration/kafka/api/SaslMultiMechanismConsumerTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SaslMultiMechanismConsumerTest.scala
@@ -44,7 +44,7 @@ class SaslMultiMechanismConsumerTest extends BaseConsumerTest with SaslSetup {
   }
 
   @Test
-  def testMultipleBrokerMechanisms() {
+  def testMultipleBrokerMechanisms(): Unit = {
     val plainSaslProducer = createProducer()
     val plainSaslConsumer = createConsumer()
 
diff --git a/core/src/test/scala/integration/kafka/api/SaslPlainPlaintextConsumerTest.scala b/core/src/test/scala/integration/kafka/api/SaslPlainPlaintextConsumerTest.scala
index c32f7c2958e05..f31575323a06c 100644
--- a/core/src/test/scala/integration/kafka/api/SaslPlainPlaintextConsumerTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SaslPlainPlaintextConsumerTest.scala
@@ -59,7 +59,7 @@ class SaslPlainPlaintextConsumerTest extends BaseConsumerTest with SaslSetup {
    * when zookeeper.set.acl=false, even if ZooKeeper is SASL-enabled.
    */
   @Test
-  def testZkAclsDisabled() {
+  def testZkAclsDisabled(): Unit = {
     TestUtils.verifyUnsecureZkAcls(zkClient)
   }
 }
diff --git a/core/src/test/scala/integration/kafka/api/SaslPlainSslEndToEndAuthorizationTest.scala b/core/src/test/scala/integration/kafka/api/SaslPlainSslEndToEndAuthorizationTest.scala
index 828b056b72146..0c1559106c955 100644
--- a/core/src/test/scala/integration/kafka/api/SaslPlainSslEndToEndAuthorizationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SaslPlainSslEndToEndAuthorizationTest.scala
@@ -59,8 +59,8 @@ object SaslPlainSslEndToEndAuthorizationTest {
   }
 
   class TestServerCallbackHandler extends AuthenticateCallbackHandler {
-    def configure(configs: java.util.Map[String, _], saslMechanism: String, jaasConfigEntries: java.util.List[AppConfigurationEntry]) {}
-    def handle(callbacks: Array[Callback]) {
+    def configure(configs: java.util.Map[String, _], saslMechanism: String, jaasConfigEntries: java.util.List[AppConfigurationEntry]): Unit = {}
+    def handle(callbacks: Array[Callback]): Unit = {
       var username: String = null
       for (callback <- callbacks) {
         if (callback.isInstanceOf[NameCallback])
@@ -72,12 +72,12 @@ object SaslPlainSslEndToEndAuthorizationTest {
           throw new UnsupportedCallbackException(callback)
       }
     }
-    def close() {}
+    def close(): Unit = {}
   }
 
   class TestClientCallbackHandler extends AuthenticateCallbackHandler {
-    def configure(configs: java.util.Map[String, _], saslMechanism: String, jaasConfigEntries: java.util.List[AppConfigurationEntry]) {}
-    def handle(callbacks: Array[Callback]) {
+    def configure(configs: java.util.Map[String, _], saslMechanism: String, jaasConfigEntries: java.util.List[AppConfigurationEntry]): Unit = {}
+    def handle(callbacks: Array[Callback]): Unit = {
       val subject = Subject.getSubject(AccessController.getContext())
       val username = subject.getPublicCredentials(classOf[String]).iterator().next()
       for (callback <- callbacks) {
@@ -90,7 +90,7 @@ object SaslPlainSslEndToEndAuthorizationTest {
           throw new UnsupportedCallbackException(callback)
       }
     }
-    def close() {}
+    def close(): Unit = {}
   }
 }
 
@@ -136,7 +136,7 @@ class SaslPlainSslEndToEndAuthorizationTest extends SaslEndToEndAuthorizationTes
    * have expected ACLs.
    */
   @Test
-  def testAcls() {
+  def testAcls(): Unit = {
     TestUtils.verifySecureZkAcls(zkClient, 1)
   }
 }
diff --git a/core/src/test/scala/integration/kafka/api/SaslScramSslEndToEndAuthorizationTest.scala b/core/src/test/scala/integration/kafka/api/SaslScramSslEndToEndAuthorizationTest.scala
index 8652a6fedbcd1..bc425911baa83 100644
--- a/core/src/test/scala/integration/kafka/api/SaslScramSslEndToEndAuthorizationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SaslScramSslEndToEndAuthorizationTest.scala
@@ -30,7 +30,7 @@ class SaslScramSslEndToEndAuthorizationTest extends SaslEndToEndAuthorizationTes
   override val kafkaPrincipal = JaasTestUtils.KafkaScramAdmin
   private val kafkaPassword = JaasTestUtils.KafkaScramAdminPassword
 
-  override def configureSecurityBeforeServersStart() {
+  override def configureSecurityBeforeServersStart(): Unit = {
     super.configureSecurityBeforeServersStart()
     zkClient.makeSurePersistentPathExists(ConfigEntityChangeNotificationZNode.path)
     // Create broker credentials before starting brokers
@@ -38,7 +38,7 @@ class SaslScramSslEndToEndAuthorizationTest extends SaslEndToEndAuthorizationTes
   }
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
     // Create client credentials after starting brokers so that dynamic credential creation is also tested
     createScramCredentials(zkConnect, JaasTestUtils.KafkaScramUser, JaasTestUtils.KafkaScramPassword)
diff --git a/core/src/test/scala/integration/kafka/api/SaslSetup.scala b/core/src/test/scala/integration/kafka/api/SaslSetup.scala
index c0cd9a0c4d76e..7b06eb40bdf9d 100644
--- a/core/src/test/scala/integration/kafka/api/SaslSetup.scala
+++ b/core/src/test/scala/integration/kafka/api/SaslSetup.scala
@@ -53,7 +53,7 @@ trait SaslSetup {
   private var serverKeytabFile: Option[File] = None
   private var clientKeytabFile: Option[File] = None
 
-  def startSasl(jaasSections: Seq[JaasSection]) {
+  def startSasl(jaasSections: Seq[JaasSection]): Unit = {
     // Important if tests leak consumers, producers or brokers
     LoginManager.closeAll()
     val hasKerberos = jaasSections.exists(_.modules.exists {
@@ -108,14 +108,14 @@ trait SaslSetup {
     }
   }
 
-  private def writeJaasConfigurationToFile(jaasSections: Seq[JaasSection]) {
+  private def writeJaasConfigurationToFile(jaasSections: Seq[JaasSection]): Unit = {
     val file = JaasTestUtils.writeJaasContextsToFile(jaasSections)
     System.setProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM, file.getAbsolutePath)
     // This will cause a reload of the Configuration singleton when `getConfiguration` is called
     Configuration.setConfiguration(null)
   }
 
-  def closeSasl() {
+  def closeSasl(): Unit = {
     if (kdc != null)
       kdc.stop()
     // Important if tests leak consumers, producers or brokers
diff --git a/core/src/test/scala/integration/kafka/api/SaslSslAdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/SaslSslAdminClientIntegrationTest.scala
index f4c9adfbc34e3..a44bb33c7efe5 100644
--- a/core/src/test/scala/integration/kafka/api/SaslSslAdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SaslSslAdminClientIntegrationTest.scala
@@ -37,7 +37,7 @@ class SaslSslAdminClientIntegrationTest extends AdminClientIntegrationTest with
   override protected def securityProtocol = SecurityProtocol.SASL_SSL
   override protected lazy val trustStoreFile = Some(File.createTempFile("truststore", ".jks"))
 
-  override def configureSecurityBeforeServersStart() {
+  override def configureSecurityBeforeServersStart(): Unit = {
     val authorizer = CoreUtils.createObject[Authorizer](classOf[SimpleAclAuthorizer].getName)
     try {
       authorizer.configure(this.configs.head.originals())
diff --git a/core/src/test/scala/integration/kafka/api/SslEndToEndAuthorizationTest.scala b/core/src/test/scala/integration/kafka/api/SslEndToEndAuthorizationTest.scala
index 8354ee06a4d5a..fe087ec864037 100644
--- a/core/src/test/scala/integration/kafka/api/SslEndToEndAuthorizationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SslEndToEndAuthorizationTest.scala
@@ -67,7 +67,7 @@ class SslEndToEndAuthorizationTest extends EndToEndAuthorizationTest {
   override val clientPrincipal = s"O=A client,CN=$clientCn"
   override val kafkaPrincipal = "server"
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     startSasl(jaasSections(List.empty, None, ZkSasl))
     super.setUp()
   }
diff --git a/core/src/test/scala/integration/kafka/api/TransactionsBounceTest.scala b/core/src/test/scala/integration/kafka/api/TransactionsBounceTest.scala
index eabafccffb2b8..b24698bed6713 100644
--- a/core/src/test/scala/integration/kafka/api/TransactionsBounceTest.scala
+++ b/core/src/test/scala/integration/kafka/api/TransactionsBounceTest.scala
@@ -71,7 +71,7 @@ class TransactionsBounceTest extends KafkaServerTestHarness {
   }
 
   @Test
-  def testBrokerFailure() {
+  def testBrokerFailure(): Unit = {
     // basic idea is to seed a topic with 10000 records, and copy it transactionally while bouncing brokers
     // constantly through the period.
     val consumerGroup = "myGroup"
@@ -181,7 +181,7 @@ class TransactionsBounceTest extends KafkaServerTestHarness {
       (0 until numPartitions).foreach(partition => TestUtils.waitUntilLeaderIsElectedOrChanged(zkClient, outputTopic, partition))
     }
 
-    override def shutdown(){
+    override def shutdown(): Unit = {
       super.shutdown()
    }
   }
diff --git a/core/src/test/scala/integration/kafka/api/TransactionsTest.scala b/core/src/test/scala/integration/kafka/api/TransactionsTest.scala
index 5b1888addd615..254ccadc89096 100644
--- a/core/src/test/scala/integration/kafka/api/TransactionsTest.scala
+++ b/core/src/test/scala/integration/kafka/api/TransactionsTest.scala
@@ -393,7 +393,7 @@ class TransactionsTest extends KafkaServerTestHarness {
   }
 
   @Test
-  def testFencingOnSend() {
+  def testFencingOnSend(): Unit = {
     val producer1 = transactionalProducers(0)
     val producer2 = transactionalProducers(1)
     val consumer = transactionalConsumers(0)
diff --git a/core/src/test/scala/integration/kafka/api/UserClientIdQuotaTest.scala b/core/src/test/scala/integration/kafka/api/UserClientIdQuotaTest.scala
index 828c98b414bdf..9a3c4df0c6ad8 100644
--- a/core/src/test/scala/integration/kafka/api/UserClientIdQuotaTest.scala
+++ b/core/src/test/scala/integration/kafka/api/UserClientIdQuotaTest.scala
@@ -31,7 +31,7 @@ class UserClientIdQuotaTest extends BaseQuotaTest {
   override def consumerClientId = "QuotasTestConsumer-!@#$%^&*()"
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     this.serverConfig.setProperty(KafkaConfig.SslClientAuthProp, "required")
     this.serverConfig.setProperty(KafkaConfig.ProducerQuotaBytesPerSecondDefaultProp, Long.MaxValue.toString)
     this.serverConfig.setProperty(KafkaConfig.ConsumerQuotaBytesPerSecondDefaultProp, Long.MaxValue.toString)
@@ -51,7 +51,7 @@ class UserClientIdQuotaTest extends BaseQuotaTest {
         Map("user" -> Sanitizer.sanitize(userPrincipal.getName), "client-id" -> clientId)
       }
 
-      override def overrideQuotas(producerQuota: Long, consumerQuota: Long, requestQuota: Double) {
+      override def overrideQuotas(producerQuota: Long, consumerQuota: Long, requestQuota: Double): Unit = {
         val producerProps = new Properties()
         producerProps.setProperty(DynamicConfig.Client.ProducerByteRateOverrideProp, producerQuota.toString)
         producerProps.setProperty(DynamicConfig.Client.RequestPercentageOverrideProp, requestQuota.toString)
@@ -63,7 +63,7 @@ class UserClientIdQuotaTest extends BaseQuotaTest {
         updateQuotaOverride(userPrincipal.getName, consumerClientId, consumerProps)
       }
 
-      override def removeQuotaOverrides() {
+      override def removeQuotaOverrides(): Unit = {
         val emptyProps = new Properties
         adminZkClient.changeUserOrUserClientIdConfig(Sanitizer.sanitize(userPrincipal.getName) +
           "/clients/" + Sanitizer.sanitize(producerClientId), emptyProps)
@@ -71,7 +71,7 @@ class UserClientIdQuotaTest extends BaseQuotaTest {
           "/clients/" + Sanitizer.sanitize(consumerClientId), emptyProps)
       }
 
-      private def updateQuotaOverride(userPrincipal: String, clientId: String, properties: Properties) {
+      private def updateQuotaOverride(userPrincipal: String, clientId: String, properties: Properties): Unit = {
         adminZkClient.changeUserOrUserClientIdConfig(Sanitizer.sanitize(userPrincipal) + "/clients/" + Sanitizer.sanitize(clientId), properties)
       }
     }
diff --git a/core/src/test/scala/integration/kafka/api/UserQuotaTest.scala b/core/src/test/scala/integration/kafka/api/UserQuotaTest.scala
index d60b8751c7048..37870d9121b0c 100644
--- a/core/src/test/scala/integration/kafka/api/UserQuotaTest.scala
+++ b/core/src/test/scala/integration/kafka/api/UserQuotaTest.scala
@@ -33,7 +33,7 @@ class UserQuotaTest extends BaseQuotaTest with SaslSetup {
   override protected val clientSaslProperties = Some(kafkaClientSaslProperties(kafkaClientSaslMechanism))
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     startSasl(jaasSections(kafkaServerSaslMechanisms, Some("GSSAPI"), KafkaSasl, JaasTestUtils.KafkaServerContextName))
     this.serverConfig.setProperty(KafkaConfig.ProducerQuotaBytesPerSecondDefaultProp, Long.MaxValue.toString)
     this.serverConfig.setProperty(KafkaConfig.ConsumerQuotaBytesPerSecondDefaultProp, Long.MaxValue.toString)
@@ -59,18 +59,18 @@ class UserQuotaTest extends BaseQuotaTest with SaslSetup {
         Map("user" -> userPrincipal.getName, "client-id" -> "")
       }
 
-      override def overrideQuotas(producerQuota: Long, consumerQuota: Long, requestQuota: Double) {
+      override def overrideQuotas(producerQuota: Long, consumerQuota: Long, requestQuota: Double): Unit = {
         val props = quotaProperties(producerQuota, consumerQuota, requestQuota)
         updateQuotaOverride(props)
       }
 
-      override def removeQuotaOverrides() {
+      override def removeQuotaOverrides(): Unit = {
         val emptyProps = new Properties
         updateQuotaOverride(emptyProps)
         updateQuotaOverride(emptyProps)
       }
 
-      private def updateQuotaOverride(properties: Properties) {
+      private def updateQuotaOverride(properties: Properties): Unit = {
         adminZkClient.changeUserOrUserClientIdConfig(Sanitizer.sanitize(userPrincipal.getName), properties)
       }
     }
diff --git a/core/src/test/scala/integration/kafka/network/DynamicConnectionQuotaTest.scala b/core/src/test/scala/integration/kafka/network/DynamicConnectionQuotaTest.scala
index 21578dffaa834..871953addd3d8 100644
--- a/core/src/test/scala/integration/kafka/network/DynamicConnectionQuotaTest.scala
+++ b/core/src/test/scala/integration/kafka/network/DynamicConnectionQuotaTest.scala
@@ -70,10 +70,10 @@ class DynamicConnectionQuotaTest extends BaseRequestTest {
   }
 
   @Test
-  def testDynamicConnectionQuota() {
+  def testDynamicConnectionQuota(): Unit = {
     val maxConnectionsPerIP = 5
 
-    def connectAndVerify() {
+    def connectAndVerify(): Unit = {
       val socket = connect()
       try {
         sendAndReceive(produceRequest, ApiKeys.PRODUCE, socket)
@@ -100,7 +100,7 @@ class DynamicConnectionQuotaTest extends BaseRequestTest {
   def testDynamicListenerConnectionQuota(): Unit = {
     val initialConnectionCount = connectionCount
 
-    def connectAndVerify() {
+    def connectAndVerify(): Unit = {
       val socket = connect("PLAINTEXT")
       socket.setSoTimeout(1000)
       try {
diff --git a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
index 3951c53bfae35..3a9920e6eb294 100644
--- a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
+++ b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
@@ -145,7 +145,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     clientThreads.foreach(_.interrupt())
     clientThreads.foreach(_.initiateShutdown())
     clientThreads.foreach(_.join(5 * 1000))
@@ -1422,7 +1422,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
     val executor = Executors.newSingleThreadExecutor
     executors += executor
     val future = executor.submit(new Runnable() {
-      def run() {
+      def run(): Unit = {
         producer.send(new ProducerRecord(topic, "key", "value")).get
       }
     })
@@ -1434,7 +1434,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
     val executor = Executors.newSingleThreadExecutor
     executors += executor
     val future = executor.submit(new Runnable() {
-      def run() {
+      def run(): Unit = {
         consumer.commitSync()
       }
     })
diff --git a/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala b/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala
index 8842171185a18..f5fed6c608c44 100644
--- a/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala
+++ b/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala
@@ -56,7 +56,7 @@ class GssapiAuthenticationTest extends IntegrationTestHarness with SaslSetup {
   private val failedAuthenticationDelayMs = 2000
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     startSasl(jaasSections(kafkaServerSaslMechanisms, Option(kafkaClientSaslMechanism), Both))
     serverConfig.put(KafkaConfig.SslClientAuthProp, "required")
     serverConfig.put(KafkaConfig.FailedAuthenticationDelayMsProp, failedAuthenticationDelayMs.toString)
diff --git a/core/src/test/scala/integration/kafka/server/MultipleListenersWithSameSecurityProtocolBaseTest.scala b/core/src/test/scala/integration/kafka/server/MultipleListenersWithSameSecurityProtocolBaseTest.scala
index 86d05a79f7f80..6e811886ac40a 100644
--- a/core/src/test/scala/integration/kafka/server/MultipleListenersWithSameSecurityProtocolBaseTest.scala
+++ b/core/src/test/scala/integration/kafka/server/MultipleListenersWithSameSecurityProtocolBaseTest.scala
@@ -148,7 +148,7 @@ abstract class MultipleListenersWithSameSecurityProtocolBaseTest extends ZooKeep
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     producers.values.foreach(_.close())
     consumers.values.foreach(_.close())
     TestUtils.shutdownServers(servers)
diff --git a/core/src/test/scala/integration/kafka/server/ScramServerStartupTest.scala b/core/src/test/scala/integration/kafka/server/ScramServerStartupTest.scala
index 4c2c64030b4c4..70c30ee7dcbf4 100644
--- a/core/src/test/scala/integration/kafka/server/ScramServerStartupTest.scala
+++ b/core/src/test/scala/integration/kafka/server/ScramServerStartupTest.scala
@@ -46,7 +46,7 @@ class ScramServerStartupTest extends IntegrationTestHarness with SaslSetup {
   override protected val serverSaslProperties = Some(kafkaServerSaslProperties(kafkaServerSaslMechanisms, kafkaClientSaslMechanism))
   override protected val clientSaslProperties = Some(kafkaClientSaslProperties(kafkaClientSaslMechanism))
 
-  override def configureSecurityBeforeServersStart() {
+  override def configureSecurityBeforeServersStart(): Unit = {
     super.configureSecurityBeforeServersStart()
     zkClient.makeSurePersistentPathExists(ConfigEntityChangeNotificationZNode.path)
     // Create credentials before starting brokers
diff --git a/core/src/test/scala/kafka/security/minikdc/MiniKdc.scala b/core/src/test/scala/kafka/security/minikdc/MiniKdc.scala
index 9b3058122aaa2..256091ced732a 100644
--- a/core/src/test/scala/kafka/security/minikdc/MiniKdc.scala
+++ b/core/src/test/scala/kafka/security/minikdc/MiniKdc.scala
@@ -113,7 +113,7 @@ class MiniKdc(config: Properties, workDir: File) extends Logging {
 
   def host: String = config.getProperty(MiniKdc.KdcBindAddress)
 
-  def start() {
+  def start(): Unit = {
     if (kdc != null)
       throw new RuntimeException("KDC already started")
     if (closed)
@@ -123,7 +123,7 @@ class MiniKdc(config: Properties, workDir: File) extends Logging {
     initJvmKerberosConfig()
   }
 
-  private def initDirectoryService() {
+  private def initDirectoryService(): Unit = {
     ds = new DefaultDirectoryService
     ds.setInstanceLayout(new InstanceLayout(workDir))
     ds.setCacheService(new CacheService)
@@ -187,9 +187,9 @@ class MiniKdc(config: Properties, workDir: File) extends Logging {
     ds.getAdminSession.add(entry)
   }
 
-  private def initKdcServer() {
+  private def initKdcServer(): Unit = {
 
-    def addInitialEntriesToDirectoryService(bindAddress: String) {
+    def addInitialEntriesToDirectoryService(bindAddress: String): Unit = {
       val map = Map (
         "0" -> orgName.toLowerCase(Locale.ENGLISH),
         "1" -> orgDomain.toLowerCase(Locale.ENGLISH),
@@ -245,7 +245,7 @@ class MiniKdc(config: Properties, workDir: File) extends Logging {
     refreshJvmKerberosConfig()
   }
 
-  private def writeKrb5Conf() {
+  private def writeKrb5Conf(): Unit = {
     val stringBuilder = new StringBuilder
     val reader = new BufferedReader(
       new InputStreamReader(MiniKdc.getResourceAsStream("minikdc-krb5.conf"), StandardCharsets.UTF_8))
@@ -268,7 +268,7 @@ class MiniKdc(config: Properties, workDir: File) extends Logging {
     klass.getMethod("refresh").invoke(klass)
   }
 
-  def stop() {
+  def stop(): Unit = {
     if (!closed) {
       closed = true
       if (kdc != null) {
@@ -291,7 +291,7 @@ class MiniKdc(config: Properties, workDir: File) extends Logging {
     * @param principal principal name, do not include the domain.
     * @param password  password.
     */
-  private def createPrincipal(principal: String, password: String) {
+  private def createPrincipal(principal: String, password: String): Unit = {
     val ldifContent = s"""
       |dn: uid=$principal,ou=users,dc=${orgName.toLowerCase(Locale.ENGLISH)},dc=${orgDomain.toLowerCase(Locale.ENGLISH)}
       |objectClass: top
@@ -316,7 +316,7 @@ class MiniKdc(config: Properties, workDir: File) extends Logging {
     * @param keytabFile keytab file to add the created principals
     * @param principals principals to add to the KDC, do not include the domain.
     */
-  def createPrincipal(keytabFile: File, principals: String*) {
+  def createPrincipal(keytabFile: File, principals: String*): Unit = {
     val generatedPassword = UUID.randomUUID.toString
     val keytab = new Keytab
     val entries = principals.flatMap { principal =>
@@ -347,7 +347,7 @@ object MiniKdc {
   val JavaSecurityKrb5Conf = "java.security.krb5.conf"
   val SunSecurityKrb5Debug = "sun.security.krb5.debug"
 
-  def main(args: Array[String]) {
+  def main(args: Array[String]): Unit = {
     args match {
       case Array(workDirPath, configPath, keytabPath, principals@ _*) if principals.nonEmpty =>
         val workDir = new File(workDirPath)
@@ -370,7 +370,7 @@ object MiniKdc {
     }
   }
 
-  private def start(workDir: File, config: Properties, keytabFile: File, principals: Seq[String]) {
+  private def start(workDir: File, config: Properties, keytabFile: File, principals: Seq[String]): Unit = {
     val miniKdc = new MiniKdc(config, workDir)
     miniKdc.start()
     miniKdc.createPrincipal(keytabFile, principals: _*)
diff --git a/core/src/test/scala/kafka/tools/LogCompactionTester.scala b/core/src/test/scala/kafka/tools/LogCompactionTester.scala
index 4360b2b266819..9edac52bffcfc 100755
--- a/core/src/test/scala/kafka/tools/LogCompactionTester.scala
+++ b/core/src/test/scala/kafka/tools/LogCompactionTester.scala
@@ -58,7 +58,7 @@ object LogCompactionTester {
   //maximum line size while reading produced/consumed record text file
   private val ReadAheadLimit = 4906
 
-  def main(args: Array[String]) {
+  def main(args: Array[String]): Unit = {
     val parser = new OptionParser(false)
     val numMessagesOpt = parser.accepts("messages", "The number of messages to send or consume.")
       .withRequiredArg
@@ -157,7 +157,7 @@ object LogCompactionTester {
 
   def lineCount(filPath: Path): Int = Files.readAllLines(filPath).size
 
-  def validateOutput(producedDataFile: File, consumedDataFile: File) {
+  def validateOutput(producedDataFile: File, consumedDataFile: File): Unit = {
     val producedReader = externalSort(producedDataFile)
     val consumedReader = externalSort(consumedDataFile)
     val produced = valuesIterator(producedReader)
@@ -192,7 +192,7 @@ object LogCompactionTester {
     Utils.delete(consumedDedupedFile)
   }
 
-  def require(requirement: Boolean, message: => Any) {
+  def require(requirement: Boolean, message: => Any): Unit = {
     if (!requirement) {
       System.err.println(s"Data validation failed : $message")
       Exit.exit(1)
@@ -242,7 +242,7 @@ object LogCompactionTester {
     val builder = new ProcessBuilder("sort", "--key=1,2", "--stable", "--buffer-size=20%", "--temporary-directory=" + Files.createTempDirectory("log_compaction_test"), file.getAbsolutePath)
     val process = builder.start
     new Thread() {
-      override def run() {
+      override def run(): Unit = {
         val exitCode = process.waitFor()
         if (exitCode != 0) {
           System.err.println("Process exited abnormally.")
diff --git a/core/src/test/scala/other/kafka/ReplicationQuotasTestRig.scala b/core/src/test/scala/other/kafka/ReplicationQuotasTestRig.scala
index 4ec8665fb09cf..a9e241ead5b37 100644
--- a/core/src/test/scala/other/kafka/ReplicationQuotasTestRig.scala
+++ b/core/src/test/scala/other/kafka/ReplicationQuotasTestRig.scala
@@ -74,7 +74,7 @@ object ReplicationQuotasTestRig {
       Exit.exit(0)
   }
 
-  def run(config: ExperimentDef, journal: Journal, displayChartsOnScreen: Boolean) {
+  def run(config: ExperimentDef, journal: Journal, displayChartsOnScreen: Boolean): Unit = {
     val experiment = new Experiment()
     try {
       experiment.setUp
@@ -101,18 +101,18 @@ object ReplicationQuotasTestRig {
     val leaderRates = mutable.Map[Int, Array[Double]]()
     val followerRates = mutable.Map[Int, Array[Double]]()
 
-    def startBrokers(brokerIds: Seq[Int]) {
+    def startBrokers(brokerIds: Seq[Int]): Unit = {
       println("Starting Brokers")
       servers = brokerIds.map(i => createBrokerConfig(i, zkConnect))
         .map(c => createServer(KafkaConfig.fromProps(c)))
     }
 
-    override def tearDown() {
+    override def tearDown(): Unit = {
       TestUtils.shutdownServers(servers)
       super.tearDown()
     }
 
-    def run(config: ExperimentDef, journal: Journal, displayChartsOnScreen: Boolean) {
+    def run(config: ExperimentDef, journal: Journal, displayChartsOnScreen: Boolean): Unit = {
       experimentName = config.name
       val brokers = (100 to 100 + config.brokers)
       var count = 0
@@ -187,7 +187,7 @@ object ReplicationQuotasTestRig {
       println(s"Worst case duration is ${config.targetBytesPerBrokerMB * 1000 * 1000/ config.throttle}")
     }
 
-    def waitForReassignmentToComplete() {
+    def waitForReassignmentToComplete(): Unit = {
       waitUntilTrue(() => {
         printRateMetrics()
         !zkClient.reassignPartitionsInProgress()
@@ -248,7 +248,7 @@ object ReplicationQuotasTestRig {
       rates.put(brokerId, leaderRatesBroker)
     }
 
-    def printRateMetrics() {
+    def printRateMetrics(): Unit = {
       for (broker <- servers) {
         val leaderRate: Double = measuredRate(broker, QuotaType.LeaderReplication)
         if (broker.config.brokerId == 100)
diff --git a/core/src/test/scala/other/kafka/StressTestLog.scala b/core/src/test/scala/other/kafka/StressTestLog.scala
index 6ff3a9874550c..8a5eefb859aad 100755
--- a/core/src/test/scala/other/kafka/StressTestLog.scala
+++ b/core/src/test/scala/other/kafka/StressTestLog.scala
@@ -34,7 +34,7 @@ import org.apache.kafka.common.utils.Utils
 object StressTestLog {
   val running = new AtomicBoolean(true)
 
-  def main(args: Array[String]) {
+  def main(args: Array[String]): Unit = {
     val dir = TestUtils.randomPartitionLogDir(TestUtils.tempDir())
     val time = new MockTime
     val logProperties = new Properties()
@@ -77,7 +77,7 @@ object StressTestLog {
   abstract class WorkerThread extends Thread {
     val threadInfo = "Thread: " + Thread.currentThread.getName + " Class: " + getClass.getName
 
-    override def run() {
+    override def run(): Unit = {
       try {
         while(running.get)
           work()
@@ -90,7 +90,7 @@ object StressTestLog {
       }
     }
 
-    def work()
+    def work(): Unit
     def isMakingProgress(): Boolean
   }
 
@@ -108,7 +108,7 @@ object StressTestLog {
       false
     }
 
-    def checkProgress() {
+    def checkProgress(): Unit = {
       // Check if we are making progress every 500ms
       val curTime = System.currentTimeMillis
       if ((curTime - lastProgressCheckTime) > 500) {
@@ -119,7 +119,7 @@ object StressTestLog {
   }
 
   class WriterThread(val log: Log) extends WorkerThread with LogProgress {
-    override def work() {
+    override def work(): Unit = {
       val logAppendInfo = log.appendAsLeader(TestUtils.singletonRecords(currentOffset.toString.getBytes), 0)
       require(logAppendInfo.firstOffset.forall(_ == currentOffset) && logAppendInfo.lastOffset == currentOffset)
       currentOffset += 1
@@ -129,7 +129,7 @@ object StressTestLog {
   }
 
   class ReaderThread(val log: Log) extends WorkerThread with LogProgress {
-    override def work() {
+    override def work(): Unit = {
       try {
         log.read(currentOffset,
           maxLength = 1,
diff --git a/core/src/test/scala/other/kafka/TestLinearWriteSpeed.scala b/core/src/test/scala/other/kafka/TestLinearWriteSpeed.scala
index 3c8dfa2b13e78..bf21deeae357e 100755
--- a/core/src/test/scala/other/kafka/TestLinearWriteSpeed.scala
+++ b/core/src/test/scala/other/kafka/TestLinearWriteSpeed.scala
@@ -174,7 +174,7 @@ object TestLinearWriteSpeed {
 
   trait Writable {
     def write(): Int
-    def close()
+    def close(): Unit
   }
 
   class MmapWritable(val file: File, size: Long, val content: ByteBuffer) extends Writable {
@@ -187,7 +187,7 @@ object TestLinearWriteSpeed {
       content.rewind()
       content.limit()
     }
-    def close() {
+    def close(): Unit = {
       raf.close()
       Utils.delete(file)
     }
@@ -202,7 +202,7 @@ object TestLinearWriteSpeed {
       content.rewind()
       content.limit()
     }
-    def close() {
+    def close(): Unit = {
       channel.close()
       Utils.delete(file)
     }
@@ -216,7 +216,7 @@ object TestLinearWriteSpeed {
       log.appendAsLeader(messages, leaderEpoch = 0)
       messages.sizeInBytes
     }
-    def close() {
+    def close(): Unit = {
       log.close()
       Utils.delete(log.dir)
     }
diff --git a/core/src/test/scala/unit/kafka/admin/AclCommandTest.scala b/core/src/test/scala/unit/kafka/admin/AclCommandTest.scala
index dc260e8a60002..024de8dad4c8b 100644
--- a/core/src/test/scala/unit/kafka/admin/AclCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/AclCommandTest.scala
@@ -106,7 +106,7 @@ class AclCommandTest extends ZooKeeperTestHarness with Logging {
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     TestUtils.shutdownServers(servers)
     super.tearDown()
   }
@@ -128,7 +128,7 @@ class AclCommandTest extends ZooKeeperTestHarness with Logging {
     adminArgs = Array("--bootstrap-server", TestUtils.bootstrapServers(servers, listenerName))
   }
 
-  private def testAclCli(cmdArgs: Array[String]) {
+  private def testAclCli(cmdArgs: Array[String]): Unit = {
     for ((resources, resourceCmd) <- ResourceToCommand) {
       for (permissionType <- PermissionType.values) {
         val operationToCmd = ResourceToOperations(resources)
@@ -156,7 +156,7 @@ class AclCommandTest extends ZooKeeperTestHarness with Logging {
     testProducerConsumerCli(adminArgs)
   }
 
-  private def testProducerConsumerCli(cmdArgs: Array[String]) {
+  private def testProducerConsumerCli(cmdArgs: Array[String]): Unit = {
     for ((cmd, resourcesToAcls) <- CmdToResourcesToAcl) {
       val resourceCommand: Array[String] = resourcesToAcls.keys.map(ResourceToCommand).foldLeft(Array[String]())(_ ++ _)
       AclCommand.main(cmdArgs ++ getCmd(Allow) ++ resourceCommand ++ cmd :+ "--add")
@@ -203,14 +203,14 @@ class AclCommandTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test(expected = classOf[IllegalArgumentException])
-  def testInvalidAuthorizerProperty() {
+  def testInvalidAuthorizerProperty(): Unit = {
     val args = Array("--authorizer-properties", "zookeeper.connect " + zkConnect)
     val aclCommandService = new AclCommand.AuthorizerService(new AclCommandOptions(args))
     aclCommandService.listAcls()
   }
 
   @Test
-  def testPatternTypes() {
+  def testPatternTypes(): Unit = {
     Exit.setExitProcedure { (status, _) =>
       if (status == 1)
         throw new RuntimeException("Exiting command")
@@ -238,7 +238,7 @@ class AclCommandTest extends ZooKeeperTestHarness with Logging {
     }
   }
 
-  private def testRemove(cmdArgs: Array[String], resources: Set[Resource], resourceCmd: Array[String]) {
+  private def testRemove(cmdArgs: Array[String], resources: Set[Resource], resourceCmd: Array[String]): Unit = {
     for (resource <- resources) {
       AclCommand.main(cmdArgs ++ resourceCmd :+ "--remove" :+ "--force")
       withAuthorizer() { authorizer =>
@@ -258,7 +258,7 @@ class AclCommandTest extends ZooKeeperTestHarness with Logging {
     Users.foldLeft(cmd) ((cmd, user) => cmd ++ Array(principalCmd, user.toString))
   }
 
-  private def withAuthorizer()(f: Authorizer => Unit) {
+  private def withAuthorizer()(f: Authorizer => Unit): Unit = {
     val kafkaConfig = KafkaConfig.fromProps(brokerProps, doLog = false)
     val authZ = new SimpleAclAuthorizer
     try {
diff --git a/core/src/test/scala/unit/kafka/admin/AddPartitionsTest.scala b/core/src/test/scala/unit/kafka/admin/AddPartitionsTest.scala
index a5f1bab09c2c5..69f0e41e5ae7f 100755
--- a/core/src/test/scala/unit/kafka/admin/AddPartitionsTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/AddPartitionsTest.scala
@@ -49,7 +49,7 @@ class AddPartitionsTest extends BaseRequestTest {
   val topic5Assignment = Map(1->Seq(0,1))
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
 
     createTopic(topic1, partitionReplicaAssignment = topic1Assignment)
diff --git a/core/src/test/scala/unit/kafka/admin/AdminRackAwareTest.scala b/core/src/test/scala/unit/kafka/admin/AdminRackAwareTest.scala
index b065ed2e4631c..48bddd453f1f4 100644
--- a/core/src/test/scala/unit/kafka/admin/AdminRackAwareTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/AdminRackAwareTest.scala
@@ -27,7 +27,7 @@ import scala.collection.Map
 class AdminRackAwareTest extends RackAwareTest with Logging {
 
   @Test
-  def testGetRackAlternatedBrokerListAndAssignReplicasToBrokers() {
+  def testGetRackAlternatedBrokerListAndAssignReplicasToBrokers(): Unit = {
     val rackMap = Map(0 -> "rack1", 1 -> "rack3", 2 -> "rack3", 3 -> "rack2", 4 -> "rack2", 5 -> "rack1")
     val newList = AdminUtils.getRackAlternatedBrokerList(rackMap)
     assertEquals(List(0, 3, 1, 5, 4, 2), newList)
@@ -45,7 +45,7 @@ class AdminRackAwareTest extends RackAwareTest with Logging {
   }
 
   @Test
-  def testAssignmentWithRackAware() {
+  def testAssignmentWithRackAware(): Unit = {
     val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack2", 2 -> "rack2", 3 -> "rack3", 4 -> "rack3", 5 -> "rack1")
     val numPartitions = 6
     val replicationFactor = 3
@@ -56,7 +56,7 @@ class AdminRackAwareTest extends RackAwareTest with Logging {
   }
 
   @Test
-  def testAssignmentWithRackAwareWithRandomStartIndex() {
+  def testAssignmentWithRackAwareWithRandomStartIndex(): Unit = {
     val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack2", 2 -> "rack2", 3 -> "rack3", 4 -> "rack3", 5 -> "rack1")
     val numPartitions = 6
     val replicationFactor = 3
@@ -67,7 +67,7 @@ class AdminRackAwareTest extends RackAwareTest with Logging {
   }
 
   @Test
-  def testAssignmentWithRackAwareWithUnevenReplicas() {
+  def testAssignmentWithRackAwareWithUnevenReplicas(): Unit = {
     val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack2", 2 -> "rack2", 3 -> "rack3", 4 -> "rack3", 5 -> "rack1")
     val numPartitions = 13
     val replicationFactor = 3
@@ -78,7 +78,7 @@ class AdminRackAwareTest extends RackAwareTest with Logging {
   }
 
   @Test
-  def testAssignmentWithRackAwareWithUnevenRacks() {
+  def testAssignmentWithRackAwareWithUnevenRacks(): Unit = {
     val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack1", 2 -> "rack2", 3 -> "rack3", 4 -> "rack3", 5 -> "rack1")
     val numPartitions = 12
     val replicationFactor = 3
@@ -89,7 +89,7 @@ class AdminRackAwareTest extends RackAwareTest with Logging {
   }
 
   @Test
-  def testAssignmentWith2ReplicasRackAware() {
+  def testAssignmentWith2ReplicasRackAware(): Unit = {
     val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack2", 2 -> "rack2", 3 -> "rack3", 4 -> "rack3", 5 -> "rack1")
     val numPartitions = 12
     val replicationFactor = 2
@@ -100,7 +100,7 @@ class AdminRackAwareTest extends RackAwareTest with Logging {
   }
 
   @Test
-  def testRackAwareExpansion() {
+  def testRackAwareExpansion(): Unit = {
     val brokerRackMapping = Map(6 -> "rack1", 7 -> "rack2", 8 -> "rack2", 9 -> "rack3", 10 -> "rack3", 11 -> "rack1")
     val numPartitions = 12
     val replicationFactor = 2
@@ -111,7 +111,7 @@ class AdminRackAwareTest extends RackAwareTest with Logging {
   }
 
   @Test
-  def testAssignmentWith2ReplicasRackAwareWith6Partitions() {
+  def testAssignmentWith2ReplicasRackAwareWith6Partitions(): Unit = {
     val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack2", 2 -> "rack2", 3 -> "rack3", 4 -> "rack3", 5 -> "rack1")
     val numPartitions = 6
     val replicationFactor = 2
@@ -122,7 +122,7 @@ class AdminRackAwareTest extends RackAwareTest with Logging {
   }
 
   @Test
-  def testAssignmentWith2ReplicasRackAwareWith6PartitionsAnd3Brokers() {
+  def testAssignmentWith2ReplicasRackAwareWith6PartitionsAnd3Brokers(): Unit = {
     val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack2", 4 -> "rack3")
     val numPartitions = 3
     val replicationFactor = 2
@@ -131,7 +131,7 @@ class AdminRackAwareTest extends RackAwareTest with Logging {
   }
 
   @Test
-  def testLargeNumberPartitionsAssignment() {
+  def testLargeNumberPartitionsAssignment(): Unit = {
     val numPartitions = 96
     val replicationFactor = 3
     val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack2", 2 -> "rack2", 3 -> "rack3", 4 -> "rack3", 5 -> "rack1",
@@ -143,7 +143,7 @@ class AdminRackAwareTest extends RackAwareTest with Logging {
   }
 
   @Test
-  def testMoreReplicasThanRacks() {
+  def testMoreReplicasThanRacks(): Unit = {
     val numPartitions = 6
     val replicationFactor = 5
     val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack2", 2 -> "rack2", 3 -> "rack3", 4 -> "rack3", 5 -> "rack2")
@@ -155,7 +155,7 @@ class AdminRackAwareTest extends RackAwareTest with Logging {
   }
 
   @Test
-  def testLessReplicasThanRacks() {
+  def testLessReplicasThanRacks(): Unit = {
     val numPartitions = 6
     val replicationFactor = 2
     val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack2", 2 -> "rack2", 3 -> "rack3", 4 -> "rack3", 5 -> "rack2")
@@ -168,7 +168,7 @@ class AdminRackAwareTest extends RackAwareTest with Logging {
   }
 
   @Test
-  def testSingleRack() {
+  def testSingleRack(): Unit = {
     val numPartitions = 6
     val replicationFactor = 3
     val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack1", 2 -> "rack1", 3 -> "rack1", 4 -> "rack1", 5 -> "rack1")
@@ -182,7 +182,7 @@ class AdminRackAwareTest extends RackAwareTest with Logging {
   }
 
   @Test
-  def testSkipBrokerWithReplicaAlreadyAssigned() {
+  def testSkipBrokerWithReplicaAlreadyAssigned(): Unit = {
     val rackInfo = Map(0 -> "a", 1 -> "b", 2 -> "c", 3 -> "a", 4 -> "a")
     val brokerList = 0 to 4
     val numPartitions = 6
@@ -196,7 +196,7 @@ class AdminRackAwareTest extends RackAwareTest with Logging {
   }
 
   @Test
-  def testReplicaAssignment() {
+  def testReplicaAssignment(): Unit = {
     val brokerMetadatas = (0 to 4).map(new BrokerMetadata(_, None))
 
     // test 0 replication factor
diff --git a/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala b/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala
index e3396bbcae247..3da7a35352c58 100644
--- a/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala
@@ -87,22 +87,22 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def shouldParseArgumentsForClientsEntityType() {
+  def shouldParseArgumentsForClientsEntityType(): Unit = {
     testArgumentParse("clients")
   }
 
   @Test
-  def shouldParseArgumentsForTopicsEntityType() {
+  def shouldParseArgumentsForTopicsEntityType(): Unit = {
     testArgumentParse("topics")
   }
 
   @Test
-  def shouldParseArgumentsForBrokersEntityType() {
+  def shouldParseArgumentsForBrokersEntityType(): Unit = {
     testArgumentParse("brokers")
   }
 
   @Test
-  def shouldParseArgumentsForBrokerLoggersEntityType() {
+  def shouldParseArgumentsForBrokerLoggersEntityType(): Unit = {
     testArgumentParse("broker-loggers",
       zkConfig = false)
   }
@@ -646,7 +646,7 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def testQuotaConfigEntity() {
+  def testQuotaConfigEntity(): Unit = {
 
     def createOpts(entityType: String, entityName: Option[String], otherArgs: Array[String]) : ConfigCommandOptions = {
       val optArray = Array("--zookeeper", zkConnect,
@@ -658,7 +658,7 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging {
       new ConfigCommandOptions(optArray ++ nameArray ++ otherArgs)
     }
 
-    def checkEntity(entityType: String, entityName: Option[String], expectedEntityName: String, otherArgs: Array[String]) {
+    def checkEntity(entityType: String, entityName: Option[String], expectedEntityName: String, otherArgs: Array[String]): Unit = {
       val opts = createOpts(entityType, entityName, otherArgs)
       opts.checkArgs()
       val entity = ConfigCommand.parseEntity(opts)
@@ -666,7 +666,7 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging {
       assertEquals(expectedEntityName, entity.fullSanitizedName)
     }
 
-    def checkInvalidEntity(entityType: String, entityName: Option[String], otherArgs: Array[String]) {
+    def checkInvalidEntity(entityType: String, entityName: Option[String], otherArgs: Array[String]): Unit = {
       val opts = createOpts(entityType, entityName, otherArgs)
       try {
         opts.checkArgs()
@@ -718,8 +718,8 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def testUserClientQuotaOpts() {
-    def checkEntity(expectedEntityType: String, expectedEntityName: String, args: String*) {
+  def testUserClientQuotaOpts(): Unit = {
+    def checkEntity(expectedEntityType: String, expectedEntityName: String, args: String*): Unit = {
       val opts = new ConfigCommandOptions(Array("--zookeeper", zkConnect) ++ args)
       opts.checkArgs()
       val entity = ConfigCommand.parseEntity(opts)
@@ -760,10 +760,10 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def testQuotaDescribeEntities() {
+  def testQuotaDescribeEntities(): Unit = {
     val zkClient: KafkaZkClient = EasyMock.createNiceMock(classOf[KafkaZkClient])
 
-    def checkEntities(opts: Array[String], expectedFetches: Map[String, Seq[String]], expectedEntityNames: Seq[String]) {
+    def checkEntities(opts: Array[String], expectedFetches: Map[String, Seq[String]], expectedEntityNames: Seq[String]): Unit = {
       val entity = ConfigCommand.parseEntity(new ConfigCommandOptions(opts :+ "--describe"))
       expectedFetches.foreach {
         case (name, values) => EasyMock.expect(zkClient.getAllEntitiesWithConfig(name)).andReturn(values)
diff --git a/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala b/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
index c3989407d3c02..fdadec41e3727 100644
--- a/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
@@ -52,7 +52,7 @@ class ConsumerGroupCommandTest extends KafkaServerTestHarness {
   }
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
     createTopic(topic, 1, 1)
   }
@@ -131,7 +131,7 @@ object ConsumerGroupCommandTest {
 
     def subscribe(): Unit
 
-    def run() {
+    def run(): Unit = {
       try {
         subscribe()
         while (true)
@@ -143,7 +143,7 @@ object ConsumerGroupCommandTest {
       }
     }
 
-    def shutdown() {
+    def shutdown(): Unit = {
       consumer.wakeup()
     }
   }
@@ -173,12 +173,12 @@ object ConsumerGroupCommandTest {
     private val executor: ExecutorService = Executors.newFixedThreadPool(numThreads)
     private val consumers = new ArrayBuffer[AbstractConsumerRunnable]()
 
-    def submit(consumerThread: AbstractConsumerRunnable) {
+    def submit(consumerThread: AbstractConsumerRunnable): Unit = {
       consumers += consumerThread
       executor.submit(consumerThread)
     }
 
-    def shutdown() {
+    def shutdown(): Unit = {
       consumers.foreach(_.shutdown())
       executor.shutdown()
       executor.awaitTermination(5000, TimeUnit.MILLISECONDS)
diff --git a/core/src/test/scala/unit/kafka/admin/DeleteConsumerGroupsTest.scala b/core/src/test/scala/unit/kafka/admin/DeleteConsumerGroupsTest.scala
index 63fb84a8bf998..35828f0a21d9f 100644
--- a/core/src/test/scala/unit/kafka/admin/DeleteConsumerGroupsTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/DeleteConsumerGroupsTest.scala
@@ -26,14 +26,14 @@ import org.junit.Test
 class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
 
   @Test(expected = classOf[OptionException])
-  def testDeleteWithTopicOption() {
+  def testDeleteWithTopicOption(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
     val cgcArgs = Array("--bootstrap-server", brokerList, "--delete", "--group", group, "--topic")
     getConsumerGroupService(cgcArgs)
   }
 
   @Test
-  def testDeleteCmdNonExistingGroup() {
+  def testDeleteCmdNonExistingGroup(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
     val missingGroup = "missing.group"
 
@@ -46,7 +46,7 @@ class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDeleteNonExistingGroup() {
+  def testDeleteNonExistingGroup(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
     val missingGroup = "missing.group"
 
@@ -61,7 +61,7 @@ class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDeleteCmdNonEmptyGroup() {
+  def testDeleteCmdNonEmptyGroup(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
 
     // run one consumer in the group
@@ -79,7 +79,7 @@ class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDeleteNonEmptyGroup() {
+  def testDeleteNonEmptyGroup(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
 
     // run one consumer in the group
@@ -98,7 +98,7 @@ class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDeleteCmdEmptyGroup() {
+  def testDeleteCmdEmptyGroup(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
 
     // run one consumer in the group
@@ -122,7 +122,7 @@ class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDeleteCmdAllGroups() {
+  def testDeleteCmdAllGroups(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
 
     // Create 3 groups with 1 consumer per each
@@ -159,7 +159,7 @@ class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDeleteEmptyGroup() {
+  def testDeleteEmptyGroup(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
 
     // run one consumer in the group
@@ -183,7 +183,7 @@ class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDeleteCmdWithMixOfSuccessAndError() {
+  def testDeleteCmdWithMixOfSuccessAndError(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
     val missingGroup = "missing.group"
 
@@ -210,7 +210,7 @@ class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDeleteWithMixOfSuccessAndError() {
+  def testDeleteWithMixOfSuccessAndError(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
     val missingGroup = "missing.group"
 
@@ -240,7 +240,7 @@ class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
 
 
   @Test(expected = classOf[OptionException])
-  def testDeleteWithUnrecognizedNewConsumerOption() {
+  def testDeleteWithUnrecognizedNewConsumerOption(): Unit = {
     val cgcArgs = Array("--new-consumer", "--bootstrap-server", brokerList, "--delete", "--group", group)
     getConsumerGroupService(cgcArgs)
   }
diff --git a/core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala b/core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala
index dfdef94cba4e0..fa28be915bd1f 100644
--- a/core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala
@@ -41,13 +41,13 @@ class DeleteTopicTest extends ZooKeeperTestHarness {
   val expectedReplicaAssignment = Map(0 -> List(0, 1, 2))
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     TestUtils.shutdownServers(servers)
     super.tearDown()
   }
 
   @Test
-  def testDeleteTopicWithAllAliveReplicas() {
+  def testDeleteTopicWithAllAliveReplicas(): Unit = {
     val topic = "test"
     servers = createTestTopicAndCluster(topic)
     // start topic deletion
@@ -56,7 +56,7 @@ class DeleteTopicTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testResumeDeleteTopicWithRecoveredFollower() {
+  def testResumeDeleteTopicWithRecoveredFollower(): Unit = {
     val topicPartition = new TopicPartition("test", 0)
     val topic = topicPartition.topic
     servers = createTestTopicAndCluster(topic)
@@ -80,7 +80,7 @@ class DeleteTopicTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testResumeDeleteTopicOnControllerFailover() {
+  def testResumeDeleteTopicOnControllerFailover(): Unit = {
     val topicPartition = new TopicPartition("test", 0)
     val topic = topicPartition.topic
     servers = createTestTopicAndCluster(topic)
@@ -106,7 +106,7 @@ class DeleteTopicTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testPartitionReassignmentDuringDeleteTopic() {
+  def testPartitionReassignmentDuringDeleteTopic(): Unit = {
     val expectedReplicaAssignment = Map(0 -> List(0, 1, 2))
     val topic = "test"
     val topicPartition = new TopicPartition(topic, 0)
@@ -177,7 +177,7 @@ class DeleteTopicTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testIncreasePartitionCountDuringDeleteTopic() {
+  def testIncreasePartitionCountDuringDeleteTopic(): Unit = {
     val expectedReplicaAssignment = Map(0 -> List(0, 1, 2))
     val topic = "test"
     val topicPartition = new TopicPartition(topic, 0)
@@ -234,7 +234,7 @@ class DeleteTopicTest extends ZooKeeperTestHarness {
 
 
   @Test
-  def testDeleteTopicDuringAddPartition() {
+  def testDeleteTopicDuringAddPartition(): Unit = {
     val topic = "test"
     servers = createTestTopicAndCluster(topic)
     val leaderIdOpt = zkClient.getLeaderForPartition(new TopicPartition(topic, 0))
@@ -262,7 +262,7 @@ class DeleteTopicTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testAddPartitionDuringDeleteTopic() {
+  def testAddPartitionDuringDeleteTopic(): Unit = {
     zkClient.createTopLevelPaths()
     val topic = "test"
     servers = createTestTopicAndCluster(topic)
@@ -280,7 +280,7 @@ class DeleteTopicTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testRecreateTopicAfterDeletion() {
+  def testRecreateTopicAfterDeletion(): Unit = {
     val expectedReplicaAssignment = Map(0 -> List(0, 1, 2))
     val topic = "test"
     val topicPartition = new TopicPartition(topic, 0)
@@ -296,7 +296,7 @@ class DeleteTopicTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testDeleteNonExistingTopic() {
+  def testDeleteNonExistingTopic(): Unit = {
     val topicPartition = new TopicPartition("test", 0)
     val topic = topicPartition.topic
     servers = createTestTopicAndCluster(topic)
@@ -319,7 +319,7 @@ class DeleteTopicTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testDeleteTopicWithCleaner() {
+  def testDeleteTopicWithCleaner(): Unit = {
     val topicName = "test"
     val topicPartition = new TopicPartition(topicName, 0)
     val topic = topicPartition.topic
@@ -349,7 +349,7 @@ class DeleteTopicTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testDeleteTopicAlreadyMarkedAsDeleted() {
+  def testDeleteTopicAlreadyMarkedAsDeleted(): Unit = {
     val topicPartition = new TopicPartition("test", 0)
     val topic = topicPartition.topic
     servers = createTestTopicAndCluster(topic)
@@ -397,7 +397,7 @@ class DeleteTopicTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testDisableDeleteTopic() {
+  def testDisableDeleteTopic(): Unit = {
     val topicPartition = new TopicPartition("test", 0)
     val topic = topicPartition.topic
     servers = createTestTopicAndCluster(topic, deleteTopicEnabled = false)
@@ -415,7 +415,7 @@ class DeleteTopicTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testDeletingPartiallyDeletedTopic() {
+  def testDeletingPartiallyDeletedTopic(): Unit = {
     /**
       * A previous controller could have deleted some partitions of a topic from ZK, but not all partitions, and then crashed.
       * In that case, the new controller should be able to handle the partially deleted topic, and finish the deletion.
diff --git a/core/src/test/scala/unit/kafka/admin/DescribeConsumerGroupTest.scala b/core/src/test/scala/unit/kafka/admin/DescribeConsumerGroupTest.scala
index 68a4f3146bf3a..e4d167cd3b6b9 100644
--- a/core/src/test/scala/unit/kafka/admin/DescribeConsumerGroupTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/DescribeConsumerGroupTest.scala
@@ -37,7 +37,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   private val describeTypes = describeTypeOffsets ++ describeTypeMembers ++ describeTypeState
 
   @Test
-  def testDescribeNonExistingGroup() {
+  def testDescribeNonExistingGroup(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
     val missingGroup = "missing.group"
 
@@ -53,14 +53,14 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test(expected = classOf[OptionException])
-  def testDescribeWithMultipleSubActions() {
+  def testDescribeWithMultipleSubActions(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
     val cgcArgs = Array("--bootstrap-server", brokerList, "--describe", "--group", group, "--members", "--state")
     getConsumerGroupService(cgcArgs)
   }
 
   @Test
-  def testDescribeOffsetsOfNonExistingGroup() {
+  def testDescribeOffsetsOfNonExistingGroup(): Unit = {
     val group = "missing.group"
     TestUtils.createOffsetsTopic(zkClient, servers)
 
@@ -76,7 +76,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeMembersOfNonExistingGroup() {
+  def testDescribeMembersOfNonExistingGroup(): Unit = {
     val group = "missing.group"
     TestUtils.createOffsetsTopic(zkClient, servers)
 
@@ -96,7 +96,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeStateOfNonExistingGroup() {
+  def testDescribeStateOfNonExistingGroup(): Unit = {
     val group = "missing.group"
     TestUtils.createOffsetsTopic(zkClient, servers)
 
@@ -114,7 +114,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeExistingGroup() {
+  def testDescribeExistingGroup(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
 
     for (describeType <- describeTypes) {
@@ -132,7 +132,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeExistingGroups() {
+  def testDescribeExistingGroups(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
 
     // Create N single-threaded consumer groups from a single-partition topic
@@ -157,7 +157,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeAllExistingGroups() {
+  def testDescribeAllExistingGroups(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
 
     // Create N single-threaded consumer groups from a single-partition topic
@@ -181,7 +181,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeOffsetsOfExistingGroup() {
+  def testDescribeOffsetsOfExistingGroup(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
 
     // run one consumer in the group consuming from a single-partition topic
@@ -202,7 +202,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeMembersOfExistingGroup() {
+  def testDescribeMembersOfExistingGroup(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
 
     // run one consumer in the group consuming from a single-partition topic
@@ -236,7 +236,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeStateOfExistingGroup() {
+  def testDescribeStateOfExistingGroup(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
 
     // run one consumer in the group consuming from a single-partition topic
@@ -255,7 +255,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeStateOfExistingGroupWithRoundRobinAssignor() {
+  def testDescribeStateOfExistingGroupWithRoundRobinAssignor(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
 
     // run one consumer in the group consuming from a single-partition topic
@@ -274,7 +274,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeExistingGroupWithNoMembers() {
+  def testDescribeExistingGroupWithNoMembers(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
 
     for (describeType <- describeTypes) {
@@ -298,7 +298,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeOffsetsOfExistingGroupWithNoMembers() {
+  def testDescribeOffsetsOfExistingGroupWithNoMembers(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
 
     // run one consumer in the group consuming from a single-partition topic
@@ -331,7 +331,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeMembersOfExistingGroupWithNoMembers() {
+  def testDescribeMembersOfExistingGroupWithNoMembers(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
 
     // run one consumer in the group consuming from a single-partition topic
@@ -355,7 +355,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeStateOfExistingGroupWithNoMembers() {
+  def testDescribeStateOfExistingGroupWithNoMembers(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
 
     // run one consumer in the group consuming from a single-partition topic
@@ -382,7 +382,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeWithConsumersWithoutAssignedPartitions() {
+  def testDescribeWithConsumersWithoutAssignedPartitions(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
 
     for (describeType <- describeTypes) {
@@ -401,7 +401,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeOffsetsWithConsumersWithoutAssignedPartitions() {
+  def testDescribeOffsetsWithConsumersWithoutAssignedPartitions(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
 
     // run two consumers in the group consuming from a single-partition topic
@@ -420,7 +420,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeMembersWithConsumersWithoutAssignedPartitions() {
+  def testDescribeMembersWithConsumersWithoutAssignedPartitions(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
 
     // run two consumers in the group consuming from a single-partition topic
@@ -445,7 +445,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeStateWithConsumersWithoutAssignedPartitions() {
+  def testDescribeStateWithConsumersWithoutAssignedPartitions(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
 
     // run two consumers in the group consuming from a single-partition topic
@@ -461,7 +461,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeWithMultiPartitionTopicAndMultipleConsumers() {
+  def testDescribeWithMultiPartitionTopicAndMultipleConsumers(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
     val topic2 = "foo2"
     createTopic(topic2, 2, 1)
@@ -482,7 +482,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeOffsetsWithMultiPartitionTopicAndMultipleConsumers() {
+  def testDescribeOffsetsWithMultiPartitionTopicAndMultipleConsumers(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
     val topic2 = "foo2"
     createTopic(topic2, 2, 1)
@@ -504,7 +504,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeMembersWithMultiPartitionTopicAndMultipleConsumers() {
+  def testDescribeMembersWithMultiPartitionTopicAndMultipleConsumers(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
     val topic2 = "foo2"
     createTopic(topic2, 2, 1)
@@ -530,7 +530,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeStateWithMultiPartitionTopicAndMultipleConsumers() {
+  def testDescribeStateWithMultiPartitionTopicAndMultipleConsumers(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
     val topic2 = "foo2"
     createTopic(topic2, 2, 1)
@@ -548,7 +548,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeSimpleConsumerGroup() {
+  def testDescribeSimpleConsumerGroup(): Unit = {
     // Ensure that the offsets of consumers which don't use group management are still displayed
 
     TestUtils.createOffsetsTopic(zkClient, servers)
@@ -566,7 +566,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeGroupWithShortInitializationTimeout() {
+  def testDescribeGroupWithShortInitializationTimeout(): Unit = {
     // Let creation of the offsets topic happen during group initialization to ensure that initialization doesn't
     // complete before the timeout expires
 
@@ -587,7 +587,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeGroupOffsetsWithShortInitializationTimeout() {
+  def testDescribeGroupOffsetsWithShortInitializationTimeout(): Unit = {
     // Let creation of the offsets topic happen during group initialization to ensure that initialization doesn't
     // complete before the timeout expires
 
@@ -607,7 +607,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeGroupMembersWithShortInitializationTimeout() {
+  def testDescribeGroupMembersWithShortInitializationTimeout(): Unit = {
     // Let creation of the offsets topic happen during group initialization to ensure that initialization doesn't
     // complete before the timeout expires
 
@@ -633,7 +633,7 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testDescribeGroupStateWithShortInitializationTimeout() {
+  def testDescribeGroupStateWithShortInitializationTimeout(): Unit = {
     // Let creation of the offsets topic happen during group initialization to ensure that initialization doesn't
     // complete before the timeout expires
 
@@ -653,14 +653,14 @@ class DescribeConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test(expected = classOf[joptsimple.OptionException])
-  def testDescribeWithUnrecognizedNewConsumerOption() {
+  def testDescribeWithUnrecognizedNewConsumerOption(): Unit = {
     val cgcArgs = Array("--new-consumer", "--bootstrap-server", brokerList, "--describe", "--group", group)
     getConsumerGroupService(cgcArgs)
     fail("Expected an error due to presence of unrecognized --new-consumer option")
   }
 
   @Test
-  def testDescribeNonOffsetCommitGroup() {
+  def testDescribeNonOffsetCommitGroup(): Unit = {
     TestUtils.createOffsetsTopic(zkClient, servers)
 
     val customProps = new Properties
diff --git a/core/src/test/scala/unit/kafka/admin/LeaderElectionCommandTest.scala b/core/src/test/scala/unit/kafka/admin/LeaderElectionCommandTest.scala
index 13ad590102453..8418b4b2caaa3 100644
--- a/core/src/test/scala/unit/kafka/admin/LeaderElectionCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/LeaderElectionCommandTest.scala
@@ -49,7 +49,7 @@ final class LeaderElectionCommandTest extends ZooKeeperTestHarness {
   val broker3 = 2
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
 
     val brokerConfigs = TestUtils.createBrokerConfigs(3, zkConnect, false)
@@ -63,7 +63,7 @@ final class LeaderElectionCommandTest extends ZooKeeperTestHarness {
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     TestUtils.shutdownServers(servers)
 
     super.tearDown()
diff --git a/core/src/test/scala/unit/kafka/admin/ListConsumerGroupTest.scala b/core/src/test/scala/unit/kafka/admin/ListConsumerGroupTest.scala
index 32f6614f9285a..7429a43a1f102 100644
--- a/core/src/test/scala/unit/kafka/admin/ListConsumerGroupTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ListConsumerGroupTest.scala
@@ -23,7 +23,7 @@ import kafka.utils.TestUtils
 class ListConsumerGroupTest extends ConsumerGroupCommandTest {
 
   @Test
-  def testListConsumerGroups() {
+  def testListConsumerGroups(): Unit = {
     val simpleGroup = "simple-group"
     addSimpleGroupExecutor(group = simpleGroup)
     addConsumerGroupExecutor(numConsumers = 1)
@@ -40,7 +40,7 @@ class ListConsumerGroupTest extends ConsumerGroupCommandTest {
   }
 
   @Test(expected = classOf[OptionException])
-  def testListWithUnrecognizedNewConsumerOption() {
+  def testListWithUnrecognizedNewConsumerOption(): Unit = {
     val cgcArgs = Array("--new-consumer", "--bootstrap-server", brokerList, "--list")
     getConsumerGroupService(cgcArgs)
   }
diff --git a/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala b/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala
index 1fc57a15ce855..09fe4ccaa5186 100644
--- a/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala
@@ -43,13 +43,13 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
   var servers: Seq[KafkaServer] = Seq()
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     TestUtils.shutdownServers(servers)
     super.tearDown()
   }
 
   private def createTestTopicAndCluster(topicPartition: Map[TopicPartition, List[Int]],
-                                        authorizer: Option[String] = None) {
+                                        authorizer: Option[String] = None): Unit = {
 
     val brokerConfigs = TestUtils.createBrokerConfigs(3, zkConnect, false)
     brokerConfigs.foreach(p => p.setProperty("auto.leader.rebalance.enable", "false"))
@@ -62,7 +62,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
   }
 
   private def createTestTopicAndCluster(partitionsAndAssignments: Map[TopicPartition, List[Int]],
-                                        brokerConfigs: Seq[Properties]) {
+                                        brokerConfigs: Seq[Properties]): Unit = {
     // create brokers
     servers = brokerConfigs.map(b => TestUtils.createServer(KafkaConfig.fromProps(b)))
     // create the topic
@@ -83,7 +83,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
   }
 
   /** Bounce the given targetServer and wait for all servers to get metadata for the given partition */
-  private def bounceServer(targetServer: Int, partition: TopicPartition) {
+  private def bounceServer(targetServer: Int, partition: TopicPartition): Unit = {
     debug(s"Shutting down server $targetServer so a non-preferred replica becomes leader")
     servers(targetServer).shutdown()
     debug(s"Starting server $targetServer now that a non-preferred replica is leader")
@@ -120,7 +120,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
 
   /** Test the case multiple values are given for --bootstrap-broker */
   @Test
-  def testMultipleBrokersGiven() {
+  def testMultipleBrokersGiven(): Unit = {
     createTestTopicAndCluster(testPartitionAndAssignment)
     bounceServer(testPartitionPreferredLeader, testPartition)
     // Check the leader for the partition is not the preferred one
@@ -133,7 +133,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
 
   /** Test the case when an invalid broker is given for --bootstrap-broker */
   @Test
-  def testInvalidBrokerGiven() {
+  def testInvalidBrokerGiven(): Unit = {
     try {
       PreferredReplicaLeaderElectionCommand.run(Array(
         "--bootstrap-server", "example.com:1234"),
@@ -147,7 +147,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
 
   /** Test the case where no partitions are given (=> elect all partitions) */
   @Test
-  def testNoPartitionsGiven() {
+  def testNoPartitionsGiven(): Unit = {
     createTestTopicAndCluster(testPartitionAndAssignment)
     bounceServer(testPartitionPreferredLeader, testPartition)
     // Check the leader for the partition is not the preferred one
@@ -169,7 +169,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
 
   /** Test the case where a list of partitions is given */
   @Test
-  def testSingletonPartitionGiven() {
+  def testSingletonPartitionGiven(): Unit = {
     createTestTopicAndCluster(testPartitionAndAssignment)
     bounceServer(testPartitionPreferredLeader, testPartition)
     // Check the leader for the partition is not the preferred one
@@ -188,7 +188,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
 
   /** Test the case where a topic does not exist */
   @Test
-  def testTopicDoesNotExist() {
+  def testTopicDoesNotExist(): Unit = {
     val nonExistentPartition = new TopicPartition("does.not.exist", 0)
     val nonExistentPartitionAssignment = List(1, 2, 0)
     val nonExistentPartitionAndAssignment = Map(nonExistentPartition -> nonExistentPartitionAssignment)
@@ -213,7 +213,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
 
   /** Test the case where several partitions are given */
   @Test
-  def testMultiplePartitionsSameAssignment() {
+  def testMultiplePartitionsSameAssignment(): Unit = {
     val testPartitionA = new TopicPartition("testA", 0)
     val testPartitionB = new TopicPartition("testB", 0)
     val testPartitionAssignment = List(1, 2, 0)
@@ -240,7 +240,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
 
   /** What happens when the preferred replica is already the leader? */
   @Test
-  def testNoopElection() {
+  def testNoopElection(): Unit = {
     createTestTopicAndCluster(testPartitionAndAssignment)
     // Don't bounce the server. Doublecheck the leader for the partition is the preferred one
     assertEquals(testPartitionPreferredLeader, awaitLeader(testPartition))
@@ -259,7 +259,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
 
   /** What happens if the preferred replica is offline? */
   @Test
-  def testWithOfflinePreferredReplica() {
+  def testWithOfflinePreferredReplica(): Unit = {
     createTestTopicAndCluster(testPartitionAndAssignment)
     bounceServer(testPartitionPreferredLeader, testPartition)
     // Check the leader for the partition is not the preferred one
@@ -289,7 +289,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
 
   /** What happens if the controller gets killed just before an election? */
   @Test
-  def testTimeout() {
+  def testTimeout(): Unit = {
     createTestTopicAndCluster(testPartitionAndAssignment)
     bounceServer(testPartitionPreferredLeader, testPartition)
     // Check the leader for the partition is not the preferred one
@@ -317,7 +317,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
 
   /** Test the case where client is not authorized */
   @Test
-  def testAuthzFailure() {
+  def testAuthzFailure(): Unit = {
     createTestTopicAndCluster(testPartitionAndAssignment, Some(classOf[PreferredReplicaLeaderElectionCommandTestAuthorizer].getName))
     bounceServer(testPartitionPreferredLeader, testPartition)
     // Check the leader for the partition is not the preferred one
@@ -343,7 +343,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
   }
 
   @Test
-  def testPreferredReplicaJsonData() {
+  def testPreferredReplicaJsonData(): Unit = {
     // write preferred replica json data to zk path
     val partitionsForPreferredReplicaElection = Set(new TopicPartition("test", 1), new TopicPartition("test2", 1))
     PreferredReplicaLeaderElectionCommand.writePreferredReplicaElectionData(zkClient, partitionsForPreferredReplicaElection)
@@ -354,7 +354,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
   }
 
   @Test
-  def testBasicPreferredReplicaElection() {
+  def testBasicPreferredReplicaElection(): Unit = {
     val expectedReplicaAssignment = Map(0  -> List(0, 1, 2))
     val topic = "test"
     val partition = 0
diff --git a/core/src/test/scala/unit/kafka/admin/RackAwareTest.scala b/core/src/test/scala/unit/kafka/admin/RackAwareTest.scala
index facc7458333dc..4f4c92e0406bf 100644
--- a/core/src/test/scala/unit/kafka/admin/RackAwareTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/RackAwareTest.scala
@@ -28,7 +28,7 @@ trait RackAwareTest {
                                replicationFactor: Int,
                                verifyRackAware: Boolean = true,
                                verifyLeaderDistribution: Boolean = true,
-                               verifyReplicasDistribution: Boolean = true) {
+                               verifyReplicasDistribution: Boolean = true): Unit = {
     // always verify that no broker will be assigned for more than one replica
     for ((_, brokerList) <- assignment) {
       assertEquals("More than one replica is assigned to same broker for the same partition", brokerList.toSet.size, brokerList.size)
diff --git a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
index 8bb64bca89e24..957ab9b6ae67d 100644
--- a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
@@ -45,11 +45,11 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
   def zkUpdateDelay(): Unit = Thread.sleep(delayMs)
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
   }
 
-  def startBrokers(brokerIds: Seq[Int]) {
+  def startBrokers(brokerIds: Seq[Int]): Unit = {
     servers = brokerIds.map { i =>
       val props = createBrokerConfig(i, zkConnect, enableControlledShutdown = false, logDirCount = 3)
       // shorter backoff to reduce test durations when no active partitions are eligible for fetching due to throttling
@@ -72,7 +72,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     if (adminClient != null) {
       adminClient.close()
       adminClient = null
@@ -133,7 +133,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def shouldMoveSinglePartitionWithinBroker() {
+  def shouldMoveSinglePartitionWithinBroker(): Unit = {
     // Given a single replica on server 100
     startBrokers(Seq(100, 101))
     adminClient = createAdminClient(servers)
@@ -150,7 +150,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def shouldExpandCluster() {
+  def shouldExpandCluster(): Unit = {
     val brokers = Array(100, 101, 102)
     startBrokers(brokers)
     adminClient = createAdminClient(servers)
@@ -192,7 +192,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def shouldShrinkCluster() {
+  def shouldShrinkCluster(): Unit = {
     //Given partitions on 3 of 3 brokers
     val brokers = Array(100, 101, 102)
     startBrokers(brokers)
@@ -214,7 +214,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def shouldMoveSubsetOfPartitions() {
+  def shouldMoveSubsetOfPartitions(): Unit = {
     //Given partitions on 3 of 3 brokers
     val brokers = Array(100, 101, 102)
     startBrokers(brokers)
@@ -265,7 +265,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def shouldExecuteThrottledReassignment() {
+  def shouldExecuteThrottledReassignment(): Unit = {
 
     //Given partitions on 3 of 3 brokers
     val brokers = Array(100, 101, 102)
@@ -309,7 +309,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
 
 
   @Test
-  def shouldOnlyThrottleMovingReplicas() {
+  def shouldOnlyThrottleMovingReplicas(): Unit = {
     //Given 6 brokers, two topics
     val brokers = Array(100, 101, 102, 103, 104, 105)
     startBrokers(brokers)
@@ -354,7 +354,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def shouldChangeThrottleOnRerunAndRemoveOnVerify() {
+  def shouldChangeThrottleOnRerunAndRemoveOnVerify(): Unit = {
     //Given partitions on 3 of 3 brokers
     val brokers = Array(100, 101, 102)
     startBrokers(brokers)
@@ -405,7 +405,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test(expected = classOf[AdminCommandFailedException])
-  def shouldFailIfProposedDoesNotMatchExisting() {
+  def shouldFailIfProposedDoesNotMatchExisting(): Unit = {
     //Given a single replica on server 100
     startBrokers(Seq(100, 101))
     createTopic(zkClient, topicName, Map(0 -> Seq(100)), servers = servers)
@@ -416,7 +416,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test(expected = classOf[AdminCommandFailedException])
-  def shouldFailIfProposedHasEmptyReplicaList() {
+  def shouldFailIfProposedHasEmptyReplicaList(): Unit = {
     //Given a single replica on server 100
     startBrokers(Seq(100, 101))
     createTopic(zkClient, topicName, Map(0 -> Seq(100)), servers = servers)
@@ -427,7 +427,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test(expected = classOf[AdminCommandFailedException])
-  def shouldFailIfProposedHasInvalidBrokerID() {
+  def shouldFailIfProposedHasInvalidBrokerID(): Unit = {
     //Given a single replica on server 100
     startBrokers(Seq(100, 101))
     createTopic(zkClient, topicName, Map(0 -> Seq(100)), servers = servers)
@@ -438,7 +438,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test(expected = classOf[AdminCommandFailedException])
-  def shouldFailIfProposedHasInvalidLogDir() {
+  def shouldFailIfProposedHasInvalidLogDir(): Unit = {
     // Given a single replica on server 100
     startBrokers(Seq(100, 101))
     adminClient = createAdminClient(servers)
@@ -450,7 +450,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test(expected = classOf[AdminCommandFailedException])
-  def shouldFailIfProposedHasInconsistentReplicasAndLogDirs() {
+  def shouldFailIfProposedHasInconsistentReplicasAndLogDirs(): Unit = {
     // Given a single replica on server 100
     startBrokers(Seq(100, 101))
     adminClient = createAdminClient(servers)
@@ -463,7 +463,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def shouldPerformThrottledReassignmentOverVariousTopics() {
+  def shouldPerformThrottledReassignmentOverVariousTopics(): Unit = {
     val throttle = Throttle(1000L)
 
     startBrokers(Seq(0, 1, 2, 3))
@@ -508,7 +508,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
    * often enough to detect a regression.
    */
   @Test
-  def shouldPerformMultipleReassignmentOperationsOverVariousTopics() {
+  def shouldPerformMultipleReassignmentOperationsOverVariousTopics(): Unit = {
     startBrokers(Seq(0, 1, 2, 3))
 
     createTopic(zkClient, "orders", Map(0 -> List(0, 1, 2), 1 -> List(0, 1, 2)), servers)
@@ -627,7 +627,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     assertEquals(Seq.empty, zkClient.getReplicasForPartition(new TopicPartition("customers", 0)))
   }
 
-  def waitForReassignmentToComplete(pause: Long = 100L) {
+  def waitForReassignmentToComplete(pause: Long = 100L): Unit = {
     waitUntilTrue(() => !zkClient.reassignPartitionsInProgress,
       s"Znode ${ReassignPartitionsZNode.path} wasn't deleted", pause = pause)
   }
diff --git a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsCommandArgsTest.scala b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsCommandArgsTest.scala
index b42fce69e9508..7037f2980ce8e 100644
--- a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsCommandArgsTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsCommandArgsTest.scala
@@ -23,12 +23,12 @@ import org.junit.{After, Before, Test}
 class ReassignPartitionsCommandArgsTest {
 
   @Before
-  def setUp() {
+  def setUp(): Unit = {
     Exit.setExitProcedure((_, message) => throw new IllegalArgumentException(message.orNull))
   }
 
   @After
-  def tearDown() {
+  def tearDown(): Unit = {
     Exit.resetExitProcedure()
   }
 
diff --git a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsCommandTest.scala b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsCommandTest.scala
index 79620e743505b..ac1bc8d224c1f 100644
--- a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsCommandTest.scala
@@ -44,13 +44,13 @@ class ReassignPartitionsCommandTest extends ZooKeeperTestHarness with Logging {
   var calls = 0
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     TestUtils.shutdownServers(servers)
     super.tearDown()
   }
 
   @Test
-  def shouldFindMovingReplicas() {
+  def shouldFindMovingReplicas(): Unit = {
     val control = new TopicPartition("topic1", 1) -> Seq(100, 102)
     val assigner = new ReassignPartitionsCommand(null, null, null, null, null)
 
@@ -73,7 +73,7 @@ class ReassignPartitionsCommandTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def shouldFindMovingReplicasWhenProposedIsSubsetOfExisting() {
+  def shouldFindMovingReplicasWhenProposedIsSubsetOfExisting(): Unit = {
     val assigner = new ReassignPartitionsCommand(null, null, null, null, null)
 
     //Given we have more existing partitions than we are proposing
@@ -108,7 +108,7 @@ class ReassignPartitionsCommandTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def shouldFindMovingReplicasMultiplePartitions() {
+  def shouldFindMovingReplicasMultiplePartitions(): Unit = {
     val control = new TopicPartition("topic1", 2) -> Seq(100, 102)
     val assigner = new ReassignPartitionsCommand(null, null, null, null, null)
 
@@ -133,7 +133,7 @@ class ReassignPartitionsCommandTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def shouldFindMovingReplicasMultipleTopics() {
+  def shouldFindMovingReplicasMultipleTopics(): Unit = {
     val control = new TopicPartition("topic1", 1) -> Seq(100, 102)
     val assigner = new ReassignPartitionsCommand(null, null, null, null, null)
 
@@ -165,7 +165,7 @@ class ReassignPartitionsCommandTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def shouldFindMovingReplicasMultipleTopicsAndPartitions() {
+  def shouldFindMovingReplicasMultipleTopicsAndPartitions(): Unit = {
     val assigner = new ReassignPartitionsCommand(null, null, null, null, null)
 
     //Given
@@ -208,7 +208,7 @@ class ReassignPartitionsCommandTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def shouldFindTwoMovingReplicasInSamePartition() {
+  def shouldFindTwoMovingReplicasInSamePartition(): Unit = {
     val control = new TopicPartition("topic1", 1) -> Seq(100, 102)
     val assigner = new ReassignPartitionsCommand(null, null, null, null, null)
 
@@ -426,7 +426,7 @@ class ReassignPartitionsCommandTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def testPartitionReassignmentWithLeaderInNewReplicas() {
+  def testPartitionReassignmentWithLeaderInNewReplicas(): Unit = {
     val expectedReplicaAssignment = Map(0  -> List(0, 1, 2))
     val topic = "test"
     // create brokers
@@ -455,7 +455,7 @@ class ReassignPartitionsCommandTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def testPartitionReassignmentWithLeaderNotInNewReplicas() {
+  def testPartitionReassignmentWithLeaderNotInNewReplicas(): Unit = {
     val expectedReplicaAssignment = Map(0  -> List(0, 1, 2))
     val topic = "test"
     // create brokers
@@ -483,7 +483,7 @@ class ReassignPartitionsCommandTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def testPartitionReassignmentNonOverlappingReplicas() {
+  def testPartitionReassignmentNonOverlappingReplicas(): Unit = {
     val expectedReplicaAssignment = Map(0  -> List(0, 1))
     val topic = "test"
     // create brokers
@@ -511,7 +511,7 @@ class ReassignPartitionsCommandTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def testReassigningNonExistingPartition() {
+  def testReassigningNonExistingPartition(): Unit = {
     val topic = "test"
     // create brokers
     servers = TestUtils.createBrokerConfigs(4, zkConnect, false).map(b => TestUtils.createServer(KafkaConfig.fromProps(b)))
@@ -526,7 +526,7 @@ class ReassignPartitionsCommandTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def testResumePartitionReassignmentThatWasCompleted() {
+  def testResumePartitionReassignmentThatWasCompleted(): Unit = {
     val expectedReplicaAssignment = Map(0  -> List(0, 1))
     val topic = "test"
     // create the topic
diff --git a/core/src/test/scala/unit/kafka/admin/ResetConsumerGroupOffsetTest.scala b/core/src/test/scala/unit/kafka/admin/ResetConsumerGroupOffsetTest.scala
index a078419450b4c..838444ce748f1 100644
--- a/core/src/test/scala/unit/kafka/admin/ResetConsumerGroupOffsetTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ResetConsumerGroupOffsetTest.scala
@@ -31,7 +31,7 @@ import org.junit.Test
 class TimeConversionTests {
 
   @Test
-  def testDateTimeFormats() {
+  def testDateTimeFormats(): Unit = {
     //check valid formats
     invokeGetDateTimeMethod(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS"))
     invokeGetDateTimeMethod(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ"))
@@ -55,7 +55,7 @@ class TimeConversionTests {
     }
   }
 
-  private def invokeGetDateTimeMethod(format: SimpleDateFormat) {
+  private def invokeGetDateTimeMethod(format: SimpleDateFormat): Unit = {
     val checkpoint = new Date()
     val timestampString = format.format(checkpoint)
     ConsumerGroupCommand.convertTimestamp(timestampString)
@@ -104,7 +104,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testResetOffsetsNotExistingGroup() {
+  def testResetOffsetsNotExistingGroup(): Unit = {
     val group = "missing.group"
     val args = buildArgsForGroup(group, "--all-topics", "--to-current", "--execute")
     val consumerGroupCommand = getConsumerGroupService(args)
@@ -178,7 +178,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testResetOffsetsToLocalDateTime() {
+  def testResetOffsetsToLocalDateTime(): Unit = {
     val format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS")
     val calendar = Calendar.getInstance()
     calendar.add(Calendar.DATE, -1)
@@ -194,7 +194,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testResetOffsetsToZonedDateTime() {
+  def testResetOffsetsToZonedDateTime(): Unit = {
     val format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")
 
     produceMessages(topic, 50)
@@ -210,28 +210,28 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testResetOffsetsByDuration() {
+  def testResetOffsetsByDuration(): Unit = {
     val args = buildArgsForGroup(group, "--all-topics", "--by-duration", "PT1M", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     resetAndAssertOffsets(args, expectedOffset = 0)
   }
 
   @Test
-  def testResetOffsetsByDurationToEarliest() {
+  def testResetOffsetsByDurationToEarliest(): Unit = {
     val args = buildArgsForGroup(group, "--all-topics", "--by-duration", "PT0.1S", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     resetAndAssertOffsets(args, expectedOffset = 100)
   }
 
   @Test
-  def testResetOffsetsToEarliest() {
+  def testResetOffsetsToEarliest(): Unit = {
     val args = buildArgsForGroup(group, "--all-topics", "--to-earliest", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     resetAndAssertOffsets(args, expectedOffset = 0)
   }
 
   @Test
-  def testResetOffsetsToLatest() {
+  def testResetOffsetsToLatest(): Unit = {
     val args = buildArgsForGroup(group, "--all-topics", "--to-latest", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     produceMessages(topic, 100)
@@ -239,7 +239,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testResetOffsetsToCurrentOffset() {
+  def testResetOffsetsToCurrentOffset(): Unit = {
     val args = buildArgsForGroup(group, "--all-topics", "--to-current", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     produceMessages(topic, 100)
@@ -247,14 +247,14 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testResetOffsetsToSpecificOffset() {
+  def testResetOffsetsToSpecificOffset(): Unit = {
     val args = buildArgsForGroup(group, "--all-topics", "--to-offset", "1", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     resetAndAssertOffsets(args, expectedOffset = 1)
   }
 
   @Test
-  def testResetOffsetsShiftPlus() {
+  def testResetOffsetsShiftPlus(): Unit = {
     val args = buildArgsForGroup(group, "--all-topics", "--shift-by", "50", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     produceMessages(topic, 100)
@@ -262,7 +262,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testResetOffsetsShiftMinus() {
+  def testResetOffsetsShiftMinus(): Unit = {
     val args = buildArgsForGroup(group, "--all-topics", "--shift-by", "-50", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     produceMessages(topic, 100)
@@ -270,7 +270,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testResetOffsetsShiftByLowerThanEarliest() {
+  def testResetOffsetsShiftByLowerThanEarliest(): Unit = {
     val args = buildArgsForGroup(group, "--all-topics", "--shift-by", "-150", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     produceMessages(topic, 100)
@@ -278,7 +278,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testResetOffsetsShiftByHigherThanLatest() {
+  def testResetOffsetsShiftByHigherThanLatest(): Unit = {
     val args = buildArgsForGroup(group, "--all-topics", "--shift-by", "150", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     produceMessages(topic, 100)
@@ -286,14 +286,14 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testResetOffsetsToEarliestOnOneTopic() {
+  def testResetOffsetsToEarliestOnOneTopic(): Unit = {
     val args = buildArgsForGroup(group, "--topic", topic, "--to-earliest", "--execute")
     produceConsumeAndShutdown(topic, group, totalMessages = 100)
     resetAndAssertOffsets(args, expectedOffset = 0)
   }
 
   @Test
-  def testResetOffsetsToEarliestOnOneTopicAndPartition() {
+  def testResetOffsetsToEarliestOnOneTopicAndPartition(): Unit = {
     val topic = "bar"
     createTopic(topic, 2, 1)
 
@@ -312,7 +312,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testResetOffsetsToEarliestOnTopics() {
+  def testResetOffsetsToEarliestOnTopics(): Unit = {
     val topic1 = "topic1"
     val topic2 = "topic2"
     createTopic(topic1, 1, 1)
@@ -337,7 +337,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
   }
 
   @Test
-  def testResetOffsetsToEarliestOnTopicsAndPartitions() {
+  def testResetOffsetsToEarliestOnTopicsAndPartitions(): Unit = {
     val topic1 = "topic1"
     val topic2 = "topic2"
 
@@ -367,7 +367,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
 
   @Test
   // This one deals with old CSV export/import format for a single --group arg: "topic,partition,offset" to support old behavior
-  def testResetOffsetsExportImportPlanSingleGroupArg() {
+  def testResetOffsetsExportImportPlanSingleGroupArg(): Unit = {
     val topic = "bar"
     val tp0 = new TopicPartition(topic, 0)
     val tp1 = new TopicPartition(topic, 1)
@@ -398,7 +398,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
   @Test
   // This one deals with universal CSV export/import file format "group,topic,partition,offset",
   // supporting multiple --group args or --all-groups arg
-  def testResetOffsetsExportImportPlan() {
+  def testResetOffsetsExportImportPlan(): Unit = {
     val group1 = group + "1"
     val group2 = group + "2"
     val topic1 = "bar1"
@@ -443,7 +443,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
   }
 
   @Test(expected = classOf[OptionException])
-  def testResetWithUnrecognizedNewConsumerOption() {
+  def testResetWithUnrecognizedNewConsumerOption(): Unit = {
     val cgcArgs = Array("--new-consumer", "--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
       "--to-offset", "2", "--export")
     getConsumerGroupService(cgcArgs)
@@ -455,7 +455,7 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
     TestUtils.produceMessages(servers, records, acks = 1)
   }
 
-  private def produceConsumeAndShutdown(topic: String, group: String, totalMessages: Int, numConsumers: Int = 1) {
+  private def produceConsumeAndShutdown(topic: String, group: String, totalMessages: Int, numConsumers: Int = 1): Unit = {
     produceMessages(topic, totalMessages)
     val executor = addConsumerGroupExecutor(numConsumers = numConsumers, topic = topic, group = group)
     awaitConsumerProgress(topic, group, totalMessages)
diff --git a/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala b/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala
index 668da77a81376..652bf95593b47 100644
--- a/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala
@@ -76,7 +76,7 @@ class TopicCommandTest extends ZooKeeperTestHarness with Logging with RackAwareT
   }
 
   @Test
-  def testCreateIfNotExists() {
+  def testCreateIfNotExists(): Unit = {
     // create brokers
     val brokers = List(0, 1, 2)
     TestUtils.createBrokersInZk(zkClient, brokers)
@@ -115,7 +115,7 @@ class TopicCommandTest extends ZooKeeperTestHarness with Logging with RackAwareT
   }
 
   @Test
-  def testCreateWithInvalidReplicationFactor() {
+  def testCreateWithInvalidReplicationFactor(): Unit = {
     val brokers = List(0)
     TestUtils.createBrokersInZk(zkClient, brokers)
 
@@ -260,7 +260,7 @@ class TopicCommandTest extends ZooKeeperTestHarness with Logging with RackAwareT
   }
 
   @Test
-  def testAlterIfExists() {
+  def testAlterIfExists(): Unit = {
     // create brokers
     val brokers = List(0, 1, 2)
     TestUtils.createBrokersInZk(zkClient, brokers)
@@ -277,7 +277,7 @@ class TopicCommandTest extends ZooKeeperTestHarness with Logging with RackAwareT
   }
 
   @Test
-  def testAlterConfigs() {
+  def testAlterConfigs(): Unit = {
     // create brokers
     val brokers = List(0, 1, 2)
     TestUtils.createBrokersInZk(zkClient, brokers)
@@ -301,7 +301,7 @@ class TopicCommandTest extends ZooKeeperTestHarness with Logging with RackAwareT
   }
 
   @Test
-  def testConfigPreservationAcrossPartitionAlteration() {
+  def testConfigPreservationAcrossPartitionAlteration(): Unit = {
     val numPartitionsOriginal = 1
     val cleanupKey = "cleanup.policy"
     val cleanupVal = "compact"
@@ -331,7 +331,7 @@ class TopicCommandTest extends ZooKeeperTestHarness with Logging with RackAwareT
   }
 
   @Test
-  def testTopicDeletion() {
+  def testTopicDeletion(): Unit = {
 
     val numPartitionsOriginal = 1
 
@@ -369,7 +369,7 @@ class TopicCommandTest extends ZooKeeperTestHarness with Logging with RackAwareT
   }
 
   @Test
-  def testDeleteIfExists() {
+  def testDeleteIfExists(): Unit = {
     // create brokers
     val brokers = List(0, 1, 2)
     TestUtils.createBrokersInZk(zkClient, brokers)
@@ -405,7 +405,7 @@ class TopicCommandTest extends ZooKeeperTestHarness with Logging with RackAwareT
   }
 
   @Test
-  def testDescribeIfTopicNotExists() {
+  def testDescribeIfTopicNotExists(): Unit = {
     // create brokers
     val brokers = List(0, 1, 2)
     TestUtils.createBrokersInZk(zkClient, brokers)
@@ -428,7 +428,7 @@ class TopicCommandTest extends ZooKeeperTestHarness with Logging with RackAwareT
   }
 
   @Test
-  def testCreateAlterTopicWithRackAware() {
+  def testCreateAlterTopicWithRackAware(): Unit = {
     val rackInfo = Map(0 -> "rack1", 1 -> "rack2", 2 -> "rack2", 3 -> "rack1", 4 -> "rack3", 5 -> "rack3")
     TestUtils.createBrokersInZk(toBrokerMetadata(rackInfo), zkClient)
 
@@ -485,7 +485,7 @@ class TopicCommandTest extends ZooKeeperTestHarness with Logging with RackAwareT
   }
 
   @Test
-  def testDescribeAndListTopicsMarkedForDeletion() {
+  def testDescribeAndListTopicsMarkedForDeletion(): Unit = {
     val brokers = List(0)
     val markedForDeletionDescribe = "MarkedForDeletion"
     val markedForDeletionList = "marked for deletion"
@@ -499,20 +499,20 @@ class TopicCommandTest extends ZooKeeperTestHarness with Logging with RackAwareT
     topicService.deleteTopic(new TopicCommandOptions(Array("--topic", testTopicName)))
 
     // Test describe topics
-    def describeTopicsWithConfig() {
+    def describeTopicsWithConfig(): Unit = {
       topicService.describeTopic(new TopicCommandOptions(Array("--describe")))
     }
     val outputWithConfig = TestUtils.grabConsoleOutput(describeTopicsWithConfig())
     assertTrue(outputWithConfig.contains(testTopicName) && outputWithConfig.contains(markedForDeletionDescribe))
 
-    def describeTopicsNoConfig() {
+    def describeTopicsNoConfig(): Unit = {
       topicService.describeTopic(new TopicCommandOptions(Array("--describe", "--unavailable-partitions")))
     }
     val outputNoConfig = TestUtils.grabConsoleOutput(describeTopicsNoConfig())
     assertTrue(outputNoConfig.contains(testTopicName) && outputNoConfig.contains(markedForDeletionDescribe))
 
     // Test list topics
-    def listTopics() {
+    def listTopics(): Unit = {
       topicService.listTopics(new TopicCommandOptions(Array("--list")))
     }
     val output = TestUtils.grabConsoleOutput(listTopics())
@@ -520,7 +520,7 @@ class TopicCommandTest extends ZooKeeperTestHarness with Logging with RackAwareT
   }
 
   @Test
-  def testDescribeAndListTopicsWithoutInternalTopics() {
+  def testDescribeAndListTopicsWithoutInternalTopics(): Unit = {
     val brokers = List(0)
     TestUtils.createBrokersInZk(zkClient, brokers)
 
diff --git a/core/src/test/scala/unit/kafka/admin/TopicCommandWithAdminClientTest.scala b/core/src/test/scala/unit/kafka/admin/TopicCommandWithAdminClientTest.scala
index a04ef0dac7fa1..13d68e5026308 100644
--- a/core/src/test/scala/unit/kafka/admin/TopicCommandWithAdminClientTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/TopicCommandWithAdminClientTest.scala
@@ -65,7 +65,7 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
   private val _testName = new TestName
   @Rule def testName = _testName
 
-  def assertExitCode(expected: Int, method: () => Unit) {
+  def assertExitCode(expected: Int, method: () => Unit): Unit = {
     def mockExitProcedure(exitCode: Int, exitMessage: Option[String]): Nothing = {
       assertEquals(expected, exitCode)
       throw new RuntimeException
@@ -80,7 +80,7 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
     }
   }
 
-  def assertCheckArgsExitCode(expected: Int, options: TopicCommandOptions) {
+  def assertCheckArgsExitCode(expected: Int, options: TopicCommandOptions): Unit = {
     assertExitCode(expected, options.checkArgs _)
   }
 
@@ -101,7 +101,7 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
   }
 
   @Before
-  def setup() {
+  def setup(): Unit = {
     // create adminClient
     val props = new Properties()
     props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokerList)
@@ -254,7 +254,7 @@ class TopicCommandWithAdminClientTest extends KafkaServerTestHarness with Loggin
   }
 
   @Test
-  def testCreateWithInvalidReplicationFactor() {
+  def testCreateWithInvalidReplicationFactor(): Unit = {
     intercept[IllegalArgumentException] {
       topicService.createTopic(new TopicCommandOptions(
         Array("--partitions", "2", "--replication-factor", (Short.MaxValue+1).toString, "--topic", testTopicName)))
diff --git a/core/src/test/scala/unit/kafka/api/ApiUtilsTest.scala b/core/src/test/scala/unit/kafka/api/ApiUtilsTest.scala
index 46adedffab79e..08994c87ad01a 100644
--- a/core/src/test/scala/unit/kafka/api/ApiUtilsTest.scala
+++ b/core/src/test/scala/unit/kafka/api/ApiUtilsTest.scala
@@ -33,7 +33,7 @@ object ApiUtilsTest {
 class ApiUtilsTest {
 
   @Test
-  def testShortStringNonASCII() {
+  def testShortStringNonASCII(): Unit = {
     // Random-length strings
     for(_ <- 0 to 100) {
       // Since we're using UTF-8 encoding, each encoded byte will be one to four bytes long 
@@ -46,7 +46,7 @@ class ApiUtilsTest {
   }
 
   @Test
-  def testShortStringASCII() {
+  def testShortStringASCII(): Unit = {
     // Random-length strings
     for(_ <- 0 to 100) {
       val s: String = TestUtils.randomString(math.abs(ApiUtilsTest.rnd.nextInt()) % Short.MaxValue)  
diff --git a/core/src/test/scala/unit/kafka/common/ZkNodeChangeNotificationListenerTest.scala b/core/src/test/scala/unit/kafka/common/ZkNodeChangeNotificationListenerTest.scala
index 12f55ed1b3dd8..05b56aae64232 100644
--- a/core/src/test/scala/unit/kafka/common/ZkNodeChangeNotificationListenerTest.scala
+++ b/core/src/test/scala/unit/kafka/common/ZkNodeChangeNotificationListenerTest.scala
@@ -47,7 +47,7 @@ class ZkNodeChangeNotificationListenerTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testProcessNotification() {
+  def testProcessNotification(): Unit = {
     val notificationMessage1 = Resource(Group, "messageA", LITERAL)
     val notificationMessage2 = Resource(Group, "messageB", LITERAL)
 
@@ -78,7 +78,7 @@ class ZkNodeChangeNotificationListenerTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testSwallowsProcessorException() : Unit = {
+  def testSwallowsProcessorException(): Unit = {
     notificationHandler.setThrowSize(2)
     notificationListener = new ZkNodeChangeNotificationListener(zkClient, LiteralAclChangeStore.aclChangePath,
       ZkAclChangeStore.SequenceNumberPrefix, notificationHandler, changeExpirationMs)
diff --git a/core/src/test/scala/unit/kafka/controller/ControllerFailoverTest.scala b/core/src/test/scala/unit/kafka/controller/ControllerFailoverTest.scala
index e19c5d49eb0ba..abbf6721b176e 100644
--- a/core/src/test/scala/unit/kafka/controller/ControllerFailoverTest.scala
+++ b/core/src/test/scala/unit/kafka/controller/ControllerFailoverTest.scala
@@ -45,7 +45,7 @@ class ControllerFailoverTest extends KafkaServerTestHarness with Logging {
     .map(KafkaConfig.fromProps(_, overridingProps))
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     super.tearDown()
     this.metrics.close()
   }
@@ -55,7 +55,7 @@ class ControllerFailoverTest extends KafkaServerTestHarness with Logging {
    * for the background of this test case
    */
   @Test
-  def testHandleIllegalStateException() {
+  def testHandleIllegalStateException(): Unit = {
     val initialController = servers.find(_.kafkaController.isActive).map(_.kafkaController).getOrElse {
       fail("Could not find controller")
     }
diff --git a/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala b/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala
index 473fb03da7c42..70d26b8a49803 100644
--- a/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala
@@ -46,13 +46,13 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
   val firstControllerEpochZkVersion = KafkaController.InitialControllerEpochZkVersion + 1
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp
     servers = Seq.empty[KafkaServer]
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     TestUtils.shutdownServers(servers)
     super.tearDown
   }
@@ -440,7 +440,7 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testControlledShutdown() {
+  def testControlledShutdown(): Unit = {
     val expectedReplicaAssignment = Map(0  -> List(0, 1, 2))
     val topic = "test"
     val partition = 0
diff --git a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala
index e695078f8056e..709d5a04b47a6 100644
--- a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala
@@ -52,7 +52,7 @@ abstract class AbstractCoordinatorConcurrencyTest[M <: CoordinatorMember] {
   val random = new Random
 
   @Before
-  def setUp() {
+  def setUp(): Unit = {
 
     replicaManager = EasyMock.partialMockBuilder(classOf[TestReplicaManager]).createMock()
     replicaManager.createDelayedProducePurgatory(timer)
@@ -61,7 +61,7 @@ abstract class AbstractCoordinatorConcurrencyTest[M <: CoordinatorMember] {
   }
 
   @After
-  def tearDown() {
+  def tearDown(): Unit = {
     EasyMock.reset(replicaManager)
     if (executor != null)
       executor.shutdownNow()
@@ -70,7 +70,7 @@ abstract class AbstractCoordinatorConcurrencyTest[M <: CoordinatorMember] {
   /**
     * Verify that concurrent operations run in the normal sequence produce the expected results.
     */
-  def verifyConcurrentOperations(createMembers: String => Set[M], operations: Seq[Operation]) {
+  def verifyConcurrentOperations(createMembers: String => Set[M], operations: Seq[Operation]): Unit = {
     OrderedOperationSequence(createMembers("verifyConcurrentOperations"), operations).run()
   }
 
@@ -78,7 +78,7 @@ abstract class AbstractCoordinatorConcurrencyTest[M <: CoordinatorMember] {
     * Verify that arbitrary operations run in some random sequence don't leave the coordinator
     * in a bad state. Operations in the normal sequence should continue to work as expected.
     */
-  def verifyConcurrentRandomSequences(createMembers: String => Set[M], operations: Seq[Operation]) {
+  def verifyConcurrentRandomSequences(createMembers: String => Set[M], operations: Seq[Operation]): Unit = {
     EasyMock.reset(replicaManager)
     for (i <- 0 to 10) {
       // Run some random operations
@@ -89,7 +89,7 @@ abstract class AbstractCoordinatorConcurrencyTest[M <: CoordinatorMember] {
     }
   }
 
-  def verifyConcurrentActions(actions: Set[Action]) {
+  def verifyConcurrentActions(actions: Set[Action]): Unit = {
     val futures = actions.map(executor.submit)
     futures.map(_.get)
     enableCompletion()
@@ -177,7 +177,7 @@ object AbstractCoordinatorConcurrencyTest {
                                entriesPerPartition: Map[TopicPartition, MemoryRecords],
                                responseCallback: Map[TopicPartition, PartitionResponse] => Unit,
                                delayedProduceLock: Option[Lock] = None,
-                               processingStatsCallback: Map[TopicPartition, RecordConversionStats] => Unit = _ => ()) {
+                               processingStatsCallback: Map[TopicPartition, RecordConversionStats] => Unit = _ => ()): Unit = {
 
       if (entriesPerPartition.isEmpty)
         return
@@ -194,7 +194,7 @@ object AbstractCoordinatorConcurrencyTest {
           else
             false
         }
-        override def onComplete() {
+        override def onComplete(): Unit = {
           responseCallback(entriesPerPartition.map {
             case (tp, _) =>
               (tp, new PartitionResponse(Errors.NONE, 0L, RecordBatch.NO_TIMESTAMP, 0L))
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorConcurrencyTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorConcurrencyTest.scala
index ec31fc96e8554..207aab134d655 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorConcurrencyTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorConcurrencyTest.scala
@@ -68,7 +68,7 @@ class GroupCoordinatorConcurrencyTest extends AbstractCoordinatorConcurrencyTest
   var groupCoordinator: GroupCoordinator = _
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
 
     EasyMock.expect(zkClient.getTopicPartitionCount(Topic.GROUP_METADATA_TOPIC_NAME))
@@ -90,7 +90,7 @@ class GroupCoordinatorConcurrencyTest extends AbstractCoordinatorConcurrencyTest
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     try {
       if (groupCoordinator != null)
         groupCoordinator.shutdown()
@@ -106,17 +106,17 @@ class GroupCoordinatorConcurrencyTest extends AbstractCoordinatorConcurrencyTest
   }
 
   @Test
-  def testConcurrentGoodPathSequence() {
+  def testConcurrentGoodPathSequence(): Unit = {
     verifyConcurrentOperations(createGroupMembers, allOperations)
   }
 
   @Test
-  def testConcurrentTxnGoodPathSequence() {
+  def testConcurrentTxnGoodPathSequence(): Unit = {
     verifyConcurrentOperations(createGroupMembers, allOperationsWithTxn)
   }
 
   @Test
-  def testConcurrentRandomSequence() {
+  def testConcurrentRandomSequence(): Unit = {
     verifyConcurrentRandomSequences(createGroupMembers, allOperationsWithTxn)
   }
 
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
index 5a35e33a0eb48..1b7b68f528d36 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
@@ -88,7 +88,7 @@ class GroupCoordinatorTest {
   private val otherGroupId = "otherGroup"
 
   @Before
-  def setUp() {
+  def setUp(): Unit = {
     val props = TestUtils.createBrokerConfig(nodeId = 0, zkConnect = "")
     props.setProperty(KafkaConfig.GroupMinSessionTimeoutMsProp, GroupMinSessionTimeout.toString)
     props.setProperty(KafkaConfig.GroupMaxSessionTimeoutMsProp, GroupMaxSessionTimeout.toString)
@@ -121,7 +121,7 @@ class GroupCoordinatorTest {
   }
 
   @After
-  def tearDown() {
+  def tearDown(): Unit = {
     EasyMock.reset(replicaManager)
     if (groupCoordinator != null)
       groupCoordinator.shutdown()
@@ -189,7 +189,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testOffsetsRetentionMsIntegerOverflow() {
+  def testOffsetsRetentionMsIntegerOverflow(): Unit = {
     val props = TestUtils.createBrokerConfig(nodeId = 0, zkConnect = "")
     props.setProperty(KafkaConfig.OffsetsRetentionMinutesProp, Integer.MAX_VALUE.toString)
     val config = KafkaConfig.fromProps(props)
@@ -198,7 +198,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testJoinGroupWrongCoordinator() {
+  def testJoinGroupWrongCoordinator(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
 
     var joinGroupResult = dynamicJoinGroup(otherGroupId, memberId, protocolType, protocols)
@@ -210,7 +210,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testJoinGroupShouldReceiveErrorIfGroupOverMaxSize() {
+  def testJoinGroupShouldReceiveErrorIfGroupOverMaxSize(): Unit = {
     val futures = ArrayBuffer[Future[JoinGroupResult]]()
     val rebalanceTimeout = GroupInitialRebalanceDelay * 2
 
@@ -232,7 +232,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testJoinGroupSessionTimeoutTooSmall() {
+  def testJoinGroupSessionTimeoutTooSmall(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
 
     val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols, sessionTimeout = GroupMinSessionTimeout - 1)
@@ -241,7 +241,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testJoinGroupSessionTimeoutTooLarge() {
+  def testJoinGroupSessionTimeoutTooLarge(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
 
     val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols, sessionTimeout = GroupMaxSessionTimeout + 1)
@@ -250,7 +250,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testJoinGroupUnknownConsumerNewGroup() {
+  def testJoinGroupUnknownConsumerNewGroup(): Unit = {
     var joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols)
     assertEquals(Errors.UNKNOWN_MEMBER_ID, joinGroupResult.error)
 
@@ -260,7 +260,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testInvalidGroupId() {
+  def testInvalidGroupId(): Unit = {
     val groupId = ""
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
 
@@ -269,7 +269,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testValidJoinGroup() {
+  def testValidJoinGroup(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
 
     val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols)
@@ -278,7 +278,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testJoinGroupInconsistentProtocolType() {
+  def testJoinGroupInconsistentProtocolType(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
     val otherMemberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
 
@@ -291,7 +291,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testJoinGroupWithEmptyProtocolType() {
+  def testJoinGroupWithEmptyProtocolType(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
 
     var joinGroupResult = dynamicJoinGroup(groupId, memberId, "", protocols)
@@ -303,7 +303,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testJoinGroupWithEmptyGroupProtocol() {
+  def testJoinGroupWithEmptyGroupProtocol(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
 
     val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, List())
@@ -352,7 +352,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testJoinGroupInconsistentGroupProtocol() {
+  def testJoinGroupInconsistentGroupProtocol(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
 
     val otherMemberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
@@ -369,7 +369,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testJoinGroupUnknownConsumerExistingGroup() {
+  def testJoinGroupUnknownConsumerExistingGroup(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
     val otherMemberId = "memberId"
 
@@ -382,7 +382,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testJoinGroupUnknownConsumerNewDeadGroup() {
+  def testJoinGroupUnknownConsumerNewDeadGroup(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
 
     val deadGroupId = "deadGroupId"
@@ -393,7 +393,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testSyncDeadGroup() {
+  def testSyncDeadGroup(): Unit = {
     val memberId = "memberId"
 
     val deadGroupId = "deadGroupId"
@@ -404,7 +404,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testJoinGroupSecondJoinInconsistentProtocol() {
+  def testJoinGroupSecondJoinInconsistentProtocol(): Unit = {
     var responseFuture = sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols, requireKnownMemberId = true)
     var joinGroupResult = Await.result(responseFuture, Duration(DefaultRebalanceTimeout + 1, TimeUnit.MILLISECONDS))
     assertEquals(Errors.MEMBER_ID_REQUIRED, joinGroupResult.error)
@@ -425,13 +425,13 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberJoinAsFirstMember() {
+  def staticMemberJoinAsFirstMember(): Unit = {
     val joinGroupResult = staticJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, groupInstanceId, protocolType, protocols)
     assertEquals(Errors.NONE, joinGroupResult.error)
   }
 
   @Test
-  def staticMemberReJoinWithExplicitUnknownMemberId() {
+  def staticMemberReJoinWithExplicitUnknownMemberId(): Unit = {
     var joinGroupResult = staticJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, groupInstanceId, protocolType, protocols)
     assertEquals(Errors.NONE, joinGroupResult.error)
 
@@ -442,7 +442,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberFenceDuplicateRejoinedFollower() {
+  def staticMemberFenceDuplicateRejoinedFollower(): Unit = {
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
 
     EasyMock.reset(replicaManager)
@@ -476,7 +476,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberFenceDuplicateSyncingFollowerAfterMemberIdChanged() {
+  def staticMemberFenceDuplicateSyncingFollowerAfterMemberIdChanged(): Unit = {
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
 
     EasyMock.reset(replicaManager)
@@ -539,7 +539,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberFenceDuplicateRejoiningFollowerAfterMemberIdChanged() {
+  def staticMemberFenceDuplicateRejoiningFollowerAfterMemberIdChanged(): Unit = {
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
 
     EasyMock.reset(replicaManager)
@@ -587,7 +587,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberRejoinWithKnownMemberId() {
+  def staticMemberRejoinWithKnownMemberId(): Unit = {
     var joinGroupResult = staticJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, groupInstanceId, protocolType, protocols)
     assertEquals(Errors.NONE, joinGroupResult.error)
 
@@ -609,7 +609,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberRejoinWithLeaderIdAndUnknownMemberId() {
+  def staticMemberRejoinWithLeaderIdAndUnknownMemberId(): Unit = {
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
 
     // A static leader rejoin with unknown id will not trigger rebalance, and no assignment will be returned.
@@ -640,7 +640,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberRejoinWithLeaderIdAndKnownMemberId() {
+  def staticMemberRejoinWithLeaderIdAndKnownMemberId(): Unit = {
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId, sessionTimeout = DefaultRebalanceTimeout / 2)
 
     // A static leader with known id rejoin will trigger rebalance.
@@ -659,7 +659,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberRejoinWithLeaderIdAndUnexpectedDeadGroup() {
+  def staticMemberRejoinWithLeaderIdAndUnexpectedDeadGroup(): Unit = {
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
 
     getGroup(groupId).transitionTo(Dead)
@@ -670,7 +670,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberRejoinWithLeaderIdAndUnexpectedEmptyGroup() {
+  def staticMemberRejoinWithLeaderIdAndUnexpectedEmptyGroup(): Unit = {
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
 
     getGroup(groupId).transitionTo(PreparingRebalance)
@@ -682,7 +682,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberRejoinWithFollowerIdAndChangeOfProtocol() {
+  def staticMemberRejoinWithFollowerIdAndChangeOfProtocol(): Unit = {
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId, sessionTimeout = DefaultSessionTimeout * 2)
 
     // A static follower rejoin with changed protocol will trigger rebalance.
@@ -705,7 +705,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberRejoinWithUnknownMemberIdAndChangeOfProtocol() {
+  def staticMemberRejoinWithUnknownMemberIdAndChangeOfProtocol(): Unit = {
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
 
     // A static follower rejoin with protocol changing to leader protocol subset won't trigger rebalance.
@@ -739,7 +739,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberRejoinWithKnownLeaderIdToTriggerRebalanceAndFollowerWithChangeofProtocol() {
+  def staticMemberRejoinWithKnownLeaderIdToTriggerRebalanceAndFollowerWithChangeofProtocol(): Unit = {
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
 
     // A static leader rejoin with known member id will trigger rebalance.
@@ -788,7 +788,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberRejoinAsFollowerWithUnknownMemberId() {
+  def staticMemberRejoinAsFollowerWithUnknownMemberId(): Unit = {
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
 
     EasyMock.reset(replicaManager)
@@ -813,7 +813,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberRejoinAsFollowerWithKnownMemberIdAndNoProtocolChange() {
+  def staticMemberRejoinAsFollowerWithKnownMemberIdAndNoProtocolChange(): Unit = {
     val rebalanceResult  = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
 
     // A static follower rejoin with no protocol change will not trigger rebalance.
@@ -833,7 +833,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberRejoinAsFollowerWithMismatchedMemberId() {
+  def staticMemberRejoinAsFollowerWithMismatchedMemberId(): Unit = {
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
 
     EasyMock.reset(replicaManager)
@@ -842,7 +842,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberRejoinAsLeaderWithMismatchedMemberId() {
+  def staticMemberRejoinAsLeaderWithMismatchedMemberId(): Unit = {
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
 
     EasyMock.reset(replicaManager)
@@ -851,7 +851,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberSyncAsLeaderWithInvalidMemberId() {
+  def staticMemberSyncAsLeaderWithInvalidMemberId(): Unit = {
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
 
     EasyMock.reset(replicaManager)
@@ -860,7 +860,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberHeartbeatLeaderWithInvalidMemberId() {
+  def staticMemberHeartbeatLeaderWithInvalidMemberId(): Unit = {
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
 
     EasyMock.reset(replicaManager)
@@ -894,7 +894,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testOffsetCommitDeadGroup() {
+  def testOffsetCommitDeadGroup(): Unit = {
     val memberId = "memberId"
 
     val deadGroupId = "deadGroupId"
@@ -907,7 +907,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberCommitOffsetWithInvalidMemberId() {
+  def staticMemberCommitOffsetWithInvalidMemberId(): Unit = {
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
 
     EasyMock.reset(replicaManager)
@@ -926,7 +926,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberJoinWithUnknownInstanceIdAndKnownMemberId() {
+  def staticMemberJoinWithUnknownInstanceIdAndKnownMemberId(): Unit = {
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
 
     EasyMock.reset(replicaManager)
@@ -936,7 +936,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberJoinWithIllegalStateAsPendingMember() {
+  def staticMemberJoinWithIllegalStateAsPendingMember(): Unit = {
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
     val group = groupCoordinator.groupManager.getGroup(groupId).get
     group.addPendingMember(rebalanceResult.followerId)
@@ -954,7 +954,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberLeaveWithIllegalStateAsPendingMember() {
+  def staticMemberLeaveWithIllegalStateAsPendingMember(): Unit = {
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
     val group = groupCoordinator.groupManager.getGroup(groupId).get
     group.addPendingMember(rebalanceResult.followerId)
@@ -972,7 +972,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberReJoinWithIllegalStateAsUnknownMember() {
+  def staticMemberReJoinWithIllegalStateAsUnknownMember(): Unit = {
     staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
     val group = groupCoordinator.groupManager.getGroup(groupId).get
     group.transitionTo(PreparingRebalance)
@@ -991,7 +991,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def staticMemberReJoinWithIllegalArgumentAsMissingOldMember() {
+  def staticMemberReJoinWithIllegalArgumentAsMissingOldMember(): Unit = {
     staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
     val group = groupCoordinator.groupManager.getGroup(groupId).get
     val invalidMemberId = "invalid_member_id"
@@ -1008,7 +1008,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testLeaderFailToRejoinBeforeFinalRebalanceTimeoutWithLongSessionTimeout() {
+  def testLeaderFailToRejoinBeforeFinalRebalanceTimeoutWithLongSessionTimeout(): Unit = {
     groupStuckInRebalanceTimeoutDueToNonjoinedStaticMember()
 
     timer.advanceClock(DefaultRebalanceTimeout + 1)
@@ -1020,7 +1020,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testLeaderRejoinBeforeFinalRebalanceTimeoutWithLongSessionTimeout() {
+  def testLeaderRejoinBeforeFinalRebalanceTimeoutWithLongSessionTimeout(): Unit = {
     groupStuckInRebalanceTimeoutDueToNonjoinedStaticMember()
 
     EasyMock.reset(replicaManager)
@@ -1038,7 +1038,7 @@ class GroupCoordinatorTest {
     assertEquals(3, getGroup(groupId).generationId)
   }
 
-  def groupStuckInRebalanceTimeoutDueToNonjoinedStaticMember() {
+  def groupStuckInRebalanceTimeoutDueToNonjoinedStaticMember(): Unit = {
     val longSessionTimeout = DefaultSessionTimeout * 2
     val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId, sessionTimeout = longSessionTimeout)
 
@@ -1070,7 +1070,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testStaticMemberFollowerFailToRejoinBeforeRebalanceTimeout() {
+  def testStaticMemberFollowerFailToRejoinBeforeRebalanceTimeout(): Unit = {
     // Increase session timeout so that the follower won't be evicted when rebalance timeout is reached.
     val initialRebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId, sessionTimeout = DefaultRebalanceTimeout * 2)
 
@@ -1105,7 +1105,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testStaticMemberLeaderFailToRejoinBeforeRebalanceTimeout() {
+  def testStaticMemberLeaderFailToRejoinBeforeRebalanceTimeout(): Unit = {
     // Increase session timeout so that the leader won't be evicted when rebalance timeout is reached.
     val initialRebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId, sessionTimeout = DefaultRebalanceTimeout * 2)
 
@@ -1203,7 +1203,7 @@ class GroupCoordinatorTest {
                                    groupId: String,
                                    expectedGroupState: GroupState,
                                    expectedLeaderId: String = JoinGroupRequest.UNKNOWN_MEMBER_ID,
-                                   expectedMemberId: String = JoinGroupRequest.UNKNOWN_MEMBER_ID) {
+                                   expectedMemberId: String = JoinGroupRequest.UNKNOWN_MEMBER_ID): Unit = {
     assertEquals(expectedError, joinGroupResult.error)
     assertEquals(expectedGeneration, joinGroupResult.generationId)
     assertEquals(expectedGroupInstanceIds.size, joinGroupResult.members.size)
@@ -1220,19 +1220,19 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testHeartbeatWrongCoordinator() {
+  def testHeartbeatWrongCoordinator(): Unit = {
     val heartbeatResult = heartbeat(otherGroupId, memberId, -1)
     assertEquals(Errors.NOT_COORDINATOR, heartbeatResult)
   }
 
   @Test
-  def testHeartbeatUnknownGroup() {
+  def testHeartbeatUnknownGroup(): Unit = {
     val heartbeatResult = heartbeat(groupId, memberId, -1)
     assertEquals(Errors.UNKNOWN_MEMBER_ID, heartbeatResult)
   }
 
   @Test
-  def testheartbeatDeadGroup() {
+  def testheartbeatDeadGroup(): Unit = {
     val memberId = "memberId"
 
     val deadGroupId = "deadGroupId"
@@ -1243,7 +1243,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testheartbeatEmptyGroup() {
+  def testheartbeatEmptyGroup(): Unit = {
     val memberId = "memberId"
 
     val group = new GroupMetadata(groupId, Empty, new MockTime())
@@ -1258,7 +1258,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testHeartbeatUnknownConsumerExistingGroup() {
+  def testHeartbeatUnknownConsumerExistingGroup(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
     val otherMemberId = "memberId"
 
@@ -1278,7 +1278,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testHeartbeatRebalanceInProgress() {
+  def testHeartbeatRebalanceInProgress(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
 
     val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols)
@@ -1292,7 +1292,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testHeartbeatIllegalGeneration() {
+  def testHeartbeatIllegalGeneration(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
 
     val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols)
@@ -1311,7 +1311,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testValidHeartbeat() {
+  def testValidHeartbeat(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
 
     val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols)
@@ -1331,7 +1331,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testSessionTimeout() {
+  def testSessionTimeout(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
 
     val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols)
@@ -1358,7 +1358,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testHeartbeatMaintainsSession() {
+  def testHeartbeatMaintainsSession(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
     val sessionTimeout = 1000
 
@@ -1387,7 +1387,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testCommitMaintainsSession() {
+  def testCommitMaintainsSession(): Unit = {
     val sessionTimeout = 1000
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
     val tp = new TopicPartition("topic", 0)
@@ -1418,7 +1418,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testSessionTimeoutDuringRebalance() {
+  def testSessionTimeoutDuringRebalance(): Unit = {
     // create a group with a single member
     val firstJoinResult = dynamicJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols,
       rebalanceTimeout = 2000, sessionTimeout = 1000)
@@ -1454,7 +1454,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testRebalanceCompletesBeforeMemberJoins() {
+  def testRebalanceCompletesBeforeMemberJoins(): Unit = {
     // create a group with a single member
     val firstJoinResult = staticJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, leaderInstanceId, protocolType, protocols,
       rebalanceTimeout = 1200, sessionTimeout = 1000)
@@ -1527,7 +1527,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testSyncGroupEmptyAssignment() {
+  def testSyncGroupEmptyAssignment(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
 
     val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols)
@@ -1548,7 +1548,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testSyncGroupNotCoordinator() {
+  def testSyncGroupNotCoordinator(): Unit = {
     val generation = 1
 
     val syncGroupResult = syncGroupFollower(otherGroupId, generation, memberId)
@@ -1556,13 +1556,13 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testSyncGroupFromUnknownGroup() {
+  def testSyncGroupFromUnknownGroup(): Unit = {
     val syncGroupResult = syncGroupFollower(groupId, 1, memberId)
     assertEquals(Errors.UNKNOWN_MEMBER_ID, syncGroupResult._2)
   }
 
   @Test
-  def testSyncGroupFromUnknownMember() {
+  def testSyncGroupFromUnknownMember(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
 
     val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols)
@@ -1582,7 +1582,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testSyncGroupFromIllegalGeneration() {
+  def testSyncGroupFromIllegalGeneration(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
 
     val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols)
@@ -1597,7 +1597,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testJoinGroupFromUnchangedFollowerDoesNotRebalance() {
+  def testJoinGroupFromUnchangedFollowerDoesNotRebalance(): Unit = {
     // to get a group of two members:
     // 1. join and sync with a single member (because we can't immediately join with two members)
     // 2. join and sync with the first member and a new member
@@ -1638,7 +1638,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testJoinGroupFromUnchangedLeaderShouldRebalance() {
+  def testJoinGroupFromUnchangedLeaderShouldRebalance(): Unit = {
     val firstJoinResult = dynamicJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols)
     val firstMemberId = firstJoinResult.memberId
     val firstGenerationId = firstJoinResult.generationId
@@ -1666,7 +1666,7 @@ class GroupCoordinatorTest {
     * which should remain stable throughout this test.
     */
   @Test
-  def testSecondMemberPartiallyJoinAndTimeout() {
+  def testSecondMemberPartiallyJoinAndTimeout(): Unit = {
     val firstJoinResult = dynamicJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols)
     val firstMemberId = firstJoinResult.memberId
     val firstGenerationId = firstJoinResult.generationId
@@ -1773,7 +1773,7 @@ class GroupCoordinatorTest {
     * operation
     */
   @Test
-  def testJoinGroupCompletionWhenPendingMemberJoins() {
+  def testJoinGroupCompletionWhenPendingMemberJoins(): Unit = {
     val pendingMember = setupGroupWithPendingMember()
 
     // compete join group for the pending member
@@ -1791,7 +1791,7 @@ class GroupCoordinatorTest {
     * cause the group to return to a CompletingRebalance state.
     */
   @Test
-  def testJoinGroupCompletionWhenPendingMemberTimesOut() {
+  def testJoinGroupCompletionWhenPendingMemberTimesOut(): Unit = {
     setupGroupWithPendingMember()
 
     // Advancing Clock by > 100 (session timeout for third and fourth member)
@@ -1847,7 +1847,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testLeaderFailureInSyncGroup() {
+  def testLeaderFailureInSyncGroup(): Unit = {
     // to get a group of two members:
     // 1. join and sync with a single member (because we can't immediately join with two members)
     // 2. join and sync with the first member and a new member
@@ -1891,7 +1891,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testSyncGroupFollowerAfterLeader() {
+  def testSyncGroupFollowerAfterLeader(): Unit = {
     // to get a group of two members:
     // 1. join and sync with a single member (because we can't immediately join with two members)
     // 2. join and sync with the first member and a new member
@@ -1940,7 +1940,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testSyncGroupLeaderAfterFollower() {
+  def testSyncGroupLeaderAfterFollower(): Unit = {
     // to get a group of two members:
     // 1. join and sync with a single member (because we can't immediately join with two members)
     // 2. join and sync with the first member and a new member
@@ -1992,7 +1992,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testCommitOffsetFromUnknownGroup() {
+  def testCommitOffsetFromUnknownGroup(): Unit = {
     val generationId = 1
     val tp = new TopicPartition("topic", 0)
     val offset = offsetAndMetadata(0)
@@ -2002,7 +2002,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testCommitOffsetWithDefaultGeneration() {
+  def testCommitOffsetWithDefaultGeneration(): Unit = {
     val tp = new TopicPartition("topic", 0)
     val offset = offsetAndMetadata(0)
 
@@ -2064,7 +2064,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testCommitAndFetchOffsetsWithEmptyGroup() {
+  def testCommitAndFetchOffsetsWithEmptyGroup(): Unit = {
     // For backwards compatibility, the coordinator supports committing/fetching offsets with an empty groupId.
     // To allow inspection and removal of the empty group, we must also support DescribeGroups and DeleteGroups
 
@@ -2102,7 +2102,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testBasicFetchTxnOffsets() {
+  def testBasicFetchTxnOffsets(): Unit = {
     val tp = new TopicPartition("topic", 0)
     val offset = offsetAndMetadata(0)
     val producerId = 1000L
@@ -2129,7 +2129,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testFetchTxnOffsetsWithAbort() {
+  def testFetchTxnOffsetsWithAbort(): Unit = {
     val tp = new TopicPartition("topic", 0)
     val offset = offsetAndMetadata(0)
     val producerId = 1000L
@@ -2153,7 +2153,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testFetchTxnOffsetsIgnoreSpuriousCommit() {
+  def testFetchTxnOffsetsIgnoreSpuriousCommit(): Unit = {
     val tp = new TopicPartition("topic", 0)
     val offset = offsetAndMetadata(0)
     val producerId = 1000L
@@ -2182,7 +2182,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testFetchTxnOffsetsOneProducerMultipleGroups() {
+  def testFetchTxnOffsetsOneProducerMultipleGroups(): Unit = {
     // One producer, two groups located on separate offsets topic partitions.
     // Both group have pending offset commits.
     // Marker for only one partition is received. That commit should be materialized while the other should not.
@@ -2261,7 +2261,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testFetchTxnOffsetsMultipleProducersOneGroup() {
+  def testFetchTxnOffsetsMultipleProducersOneGroup(): Unit = {
     // One group, two producers
     // Different producers will commit offsets for different partitions.
     // Each partition's offsets should be materialized when the corresponding producer's marker is received.
@@ -2335,7 +2335,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testFetchAllOffsets() {
+  def testFetchAllOffsets(): Unit = {
     val tp1 = new TopicPartition("topic", 0)
     val tp2 = new TopicPartition("topic", 1)
     val tp3 = new TopicPartition("other-topic", 0)
@@ -2361,7 +2361,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testCommitOffsetInCompletingRebalance() {
+  def testCommitOffsetInCompletingRebalance(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
     val tp = new TopicPartition("topic", 0)
     val offset = offsetAndMetadata(0)
@@ -2378,7 +2378,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testCommitOffsetInCompletingRebalanceFromUnknownMemberId() {
+  def testCommitOffsetInCompletingRebalanceFromUnknownMemberId(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
     val tp = new TopicPartition("topic", 0)
     val offset = offsetAndMetadata(0)
@@ -2394,7 +2394,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testCommitOffsetInCompletingRebalanceFromIllegalGeneration() {
+  def testCommitOffsetInCompletingRebalanceFromIllegalGeneration(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
     val tp = new TopicPartition("topic", 0)
     val offset = offsetAndMetadata(0)
@@ -2411,7 +2411,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testHeartbeatDuringRebalanceCausesRebalanceInProgress() {
+  def testHeartbeatDuringRebalanceCausesRebalanceInProgress(): Unit = {
     // First start up a group (with a slightly larger timeout to give us time to heartbeat when the rebalance starts)
     val joinGroupResult = dynamicJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols)
     val assignedConsumerId = joinGroupResult.memberId
@@ -2430,7 +2430,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testGenerationIdIncrementsOnRebalance() {
+  def testGenerationIdIncrementsOnRebalance(): Unit = {
     val joinGroupResult = dynamicJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols)
     val initialGenerationId = joinGroupResult.generationId
     val joinGroupError = joinGroupResult.error
@@ -2454,7 +2454,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testLeaveGroupWrongCoordinator() {
+  def testLeaveGroupWrongCoordinator(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
 
     val leaveGroupResults = singleLeaveGroup(otherGroupId, memberId)
@@ -2462,13 +2462,13 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testLeaveGroupUnknownGroup() {
+  def testLeaveGroupUnknownGroup(): Unit = {
     val leaveGroupResults = singleLeaveGroup(groupId, memberId)
     verifyLeaveGroupResult(leaveGroupResults, Errors.NONE, List(Errors.UNKNOWN_MEMBER_ID))
   }
 
   @Test
-  def testLeaveGroupUnknownConsumerExistingGroup() {
+  def testLeaveGroupUnknownConsumerExistingGroup(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
     val otherMemberId = "consumerId"
 
@@ -2482,7 +2482,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testSingleLeaveDeadGroup() {
+  def testSingleLeaveDeadGroup(): Unit = {
     val deadGroupId = "deadGroupId"
 
     groupCoordinator.groupManager.addGroup(new GroupMetadata(deadGroupId, Dead, new MockTime()))
@@ -2491,7 +2491,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testBatchLeaveDeadGroup() {
+  def testBatchLeaveDeadGroup(): Unit = {
     val deadGroupId = "deadGroupId"
 
     groupCoordinator.groupManager.addGroup(new GroupMetadata(deadGroupId, Dead, new MockTime()))
@@ -2501,7 +2501,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testValidLeaveGroup() {
+  def testValidLeaveGroup(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
 
     val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols)
@@ -2515,7 +2515,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testLeaveGroupWithFencedInstanceId() {
+  def testLeaveGroupWithFencedInstanceId(): Unit = {
     val joinGroupResult = staticJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, leaderInstanceId, protocolType, protocolSuperset)
     assertEquals(Errors.NONE, joinGroupResult.error)
 
@@ -2525,7 +2525,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testLeaveGroupStaticMemberWithUnknownMemberId() {
+  def testLeaveGroupStaticMemberWithUnknownMemberId(): Unit = {
     val joinGroupResult = staticJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, leaderInstanceId, protocolType, protocolSuperset)
     assertEquals(Errors.NONE, joinGroupResult.error)
 
@@ -2536,7 +2536,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testStaticMembersValidBatchLeaveGroup() {
+  def testStaticMembersValidBatchLeaveGroup(): Unit = {
     staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
 
     EasyMock.reset(replicaManager)
@@ -2547,7 +2547,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testStaticMembersWrongCoordinatorBatchLeaveGroup() {
+  def testStaticMembersWrongCoordinatorBatchLeaveGroup(): Unit = {
     staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
 
     EasyMock.reset(replicaManager)
@@ -2558,7 +2558,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testStaticMembersUnknownGroupBatchLeaveGroup() {
+  def testStaticMembersUnknownGroupBatchLeaveGroup(): Unit = {
     val leaveGroupResults = batchLeaveGroup(groupId, List(new MemberIdentity()
       .setGroupInstanceId(leaderInstanceId.get), new MemberIdentity().setGroupInstanceId(followerInstanceId.get)))
 
@@ -2566,7 +2566,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testStaticMembersFencedInstanceBatchLeaveGroup() {
+  def testStaticMembersFencedInstanceBatchLeaveGroup(): Unit = {
     staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
 
     EasyMock.reset(replicaManager)
@@ -2579,7 +2579,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testStaticMembersUnknownInstanceBatchLeaveGroup() {
+  def testStaticMembersUnknownInstanceBatchLeaveGroup(): Unit = {
     staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
 
     EasyMock.reset(replicaManager)
@@ -2591,7 +2591,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testPendingMemberBatchLeaveGroup() {
+  def testPendingMemberBatchLeaveGroup(): Unit = {
     val pendingMember = setupGroupWithPendingMember()
 
     EasyMock.reset(replicaManager)
@@ -2603,7 +2603,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testPendingMemberWithUnexpectedInstanceIdBatchLeaveGroup() {
+  def testPendingMemberWithUnexpectedInstanceIdBatchLeaveGroup(): Unit = {
     val pendingMember = setupGroupWithPendingMember()
 
     EasyMock.reset(replicaManager)
@@ -2623,7 +2623,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testListGroupsIncludesStableGroups() {
+  def testListGroupsIncludesStableGroups(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
     val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols)
     val assignedMemberId = joinGroupResult.memberId
@@ -2642,7 +2642,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testListGroupsIncludesRebalancingGroups() {
+  def testListGroupsIncludesRebalancingGroups(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
     val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols)
     assertEquals(Errors.NONE, joinGroupResult.error)
@@ -2654,14 +2654,14 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testDescribeGroupWrongCoordinator() {
+  def testDescribeGroupWrongCoordinator(): Unit = {
     EasyMock.reset(replicaManager)
     val (error, _) = groupCoordinator.handleDescribeGroup(otherGroupId)
     assertEquals(Errors.NOT_COORDINATOR, error)
   }
 
   @Test
-  def testDescribeGroupInactiveGroup() {
+  def testDescribeGroupInactiveGroup(): Unit = {
     EasyMock.reset(replicaManager)
     val (error, summary) = groupCoordinator.handleDescribeGroup(groupId)
     assertEquals(Errors.NONE, error)
@@ -2669,7 +2669,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testDescribeGroupStableForDynamicMember() {
+  def testDescribeGroupStableForDynamicMember(): Unit = {
     val joinGroupResult = dynamicJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols)
     val assignedMemberId = joinGroupResult.memberId
     val generationId = joinGroupResult.generationId
@@ -2691,7 +2691,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testDescribeGroupStableForStaticMember() {
+  def testDescribeGroupStableForStaticMember(): Unit = {
     val joinGroupResult = staticJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, leaderInstanceId, protocolType, protocols)
     val assignedMemberId = joinGroupResult.memberId
     val generationId = joinGroupResult.generationId
@@ -2714,7 +2714,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testDescribeGroupRebalancing() {
+  def testDescribeGroupRebalancing(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
     val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols)
     val joinGroupError = joinGroupResult.error
@@ -2732,7 +2732,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testDeleteNonEmptyGroup() {
+  def testDeleteNonEmptyGroup(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
     dynamicJoinGroup(groupId, memberId, protocolType, protocols)
 
@@ -2741,20 +2741,20 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testDeleteGroupWithInvalidGroupId() {
+  def testDeleteGroupWithInvalidGroupId(): Unit = {
     val invalidGroupId = null
     val result = groupCoordinator.handleDeleteGroups(Set(invalidGroupId))
     assert(result.size == 1 && result.contains(invalidGroupId) && result.get(invalidGroupId).contains(Errors.INVALID_GROUP_ID))
   }
 
   @Test
-  def testDeleteGroupWithWrongCoordinator() {
+  def testDeleteGroupWithWrongCoordinator(): Unit = {
     val result = groupCoordinator.handleDeleteGroups(Set(otherGroupId))
     assert(result.size == 1 && result.contains(otherGroupId) && result.get(otherGroupId).contains(Errors.NOT_COORDINATOR))
   }
 
   @Test
-  def testDeleteEmptyGroup() {
+  def testDeleteEmptyGroup(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
     val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols)
 
@@ -2776,7 +2776,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def testDeleteEmptyGroupWithStoredOffsets() {
+  def testDeleteEmptyGroupWithStoredOffsets(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
 
     val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols)
@@ -2819,7 +2819,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def shouldDelayInitialRebalanceByGroupInitialRebalanceDelayOnEmptyGroup() {
+  def shouldDelayInitialRebalanceByGroupInitialRebalanceDelayOnEmptyGroup(): Unit = {
     val firstJoinFuture = sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols)
     timer.advanceClock(GroupInitialRebalanceDelay / 2)
     verifyDelayedTaskNotCompleted(firstJoinFuture)
@@ -2838,7 +2838,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def shouldResetRebalanceDelayWhenNewMemberJoinsGroupInInitialRebalance() {
+  def shouldResetRebalanceDelayWhenNewMemberJoinsGroupInInitialRebalance(): Unit = {
     val rebalanceTimeout = GroupInitialRebalanceDelay * 3
     val firstMemberJoinFuture = sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols, rebalanceTimeout = rebalanceTimeout)
     EasyMock.reset(replicaManager)
@@ -2862,7 +2862,7 @@ class GroupCoordinatorTest {
   }
 
   @Test
-  def shouldDelayRebalanceUptoRebalanceTimeout() {
+  def shouldDelayRebalanceUptoRebalanceTimeout(): Unit = {
     val rebalanceTimeout = GroupInitialRebalanceDelay * 2
     val firstMemberJoinFuture = sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols, rebalanceTimeout = rebalanceTimeout)
     EasyMock.reset(replicaManager)
@@ -3162,7 +3162,7 @@ class GroupCoordinatorTest {
 object GroupCoordinatorTest {
   def verifyLeaveGroupResult(leaveGroupResult: LeaveGroupResult,
                              expectedTopLevelError: Errors = Errors.NONE,
-                             expectedMemberLevelErrors: List[Errors] = List.empty) {
+                             expectedMemberLevelErrors: List[Errors] = List.empty): Unit = {
     assertEquals(expectedTopLevelError, leaveGroupResult.topLevelError)
     if (expectedMemberLevelErrors.nonEmpty) {
       assertEquals(expectedMemberLevelErrors.size, leaveGroupResult.memberResponses.size)
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
index dbcf5eda56e91..f961d8e18d53c 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
@@ -69,7 +69,7 @@ class GroupMetadataManagerTest {
   val sessionTimeout = 10000
 
   @Before
-  def setUp() {
+  def setUp(): Unit = {
     val config = KafkaConfig.fromProps(TestUtils.createBrokerConfig(nodeId = 0, zkConnect = ""))
 
     val offsetConfig = OffsetConfig(maxMetadataSize = config.offsetMetadataMaxSize,
@@ -98,7 +98,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testLoadOffsetsWithoutGroup() {
+  def testLoadOffsetsWithoutGroup(): Unit = {
     val groupMetadataTopicPartition = groupTopicPartition
     val startOffset = 15L
 
@@ -126,7 +126,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testLoadEmptyGroupWithOffsets() {
+  def testLoadEmptyGroupWithOffsets(): Unit = {
     val groupMetadataTopicPartition = groupTopicPartition
     val generation = 15
     val protocolType = "consumer"
@@ -161,7 +161,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testLoadTransactionalOffsetsWithoutGroup() {
+  def testLoadTransactionalOffsetsWithoutGroup(): Unit = {
     val groupMetadataTopicPartition = groupTopicPartition
     val producerId = 1000L
     val producerEpoch: Short = 2
@@ -195,7 +195,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testDoNotLoadAbortedTransactionalOffsetCommits() {
+  def testDoNotLoadAbortedTransactionalOffsetCommits(): Unit = {
     val groupMetadataTopicPartition = groupTopicPartition
     val producerId = 1000L
     val producerEpoch: Short = 2
@@ -259,7 +259,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testLoadWithCommittedAndAbortedTransactionalOffsetCommits() {
+  def testLoadWithCommittedAndAbortedTransactionalOffsetCommits(): Unit = {
     // A test which loads a log with a mix of committed and aborted transactional offset committed messages.
     val groupMetadataTopicPartition = groupTopicPartition
     val producerId = 1000L
@@ -305,7 +305,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testLoadWithCommittedAndAbortedAndPendingTransactionalOffsetCommits() {
+  def testLoadWithCommittedAndAbortedAndPendingTransactionalOffsetCommits(): Unit = {
     val groupMetadataTopicPartition = groupTopicPartition
     val producerId = 1000L
     val producerEpoch: Short = 2
@@ -509,7 +509,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testGroupNotExists() {
+  def testGroupNotExists(): Unit = {
     // group is not owned
     assertFalse(groupMetadataManager.groupNotExists(groupId))
 
@@ -557,7 +557,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testLoadOffsetsWithTombstones() {
+  def testLoadOffsetsWithTombstones(): Unit = {
     val groupMetadataTopicPartition = groupTopicPartition
     val startOffset = 15L
 
@@ -592,7 +592,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testLoadOffsetsAndGroup() {
+  def testLoadOffsetsAndGroup(): Unit = {
     val groupMetadataTopicPartition = groupTopicPartition
     val generation = 935
     val protocolType = "consumer"
@@ -632,7 +632,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testLoadGroupWithTombstone() {
+  def testLoadGroupWithTombstone(): Unit = {
     val groupMetadataTopicPartition = groupTopicPartition
     val startOffset = 15L
     val memberId = "98098230493"
@@ -652,7 +652,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testLoadGroupWithLargeGroupMetadataRecord() {
+  def testLoadGroupWithLargeGroupMetadataRecord(): Unit = {
     val groupMetadataTopicPartition = groupTopicPartition
     val startOffset = 15L
     val committedOffsets = Map(
@@ -772,14 +772,14 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testAddGroup() {
+  def testAddGroup(): Unit = {
     val group = new GroupMetadata("foo", Empty, time)
     assertEquals(group, groupMetadataManager.addGroup(group))
     assertEquals(group, groupMetadataManager.addGroup(new GroupMetadata("foo", Empty, time)))
   }
 
   @Test
-  def testloadGroupWithStaticMember() {
+  def testloadGroupWithStaticMember(): Unit = {
     val generation = 27
     val protocolType = "consumer"
     val staticMemberId = "staticMemberId"
@@ -805,7 +805,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testReadFromOldGroupMetadata() {
+  def testReadFromOldGroupMetadata(): Unit = {
     val generation = 1
     val protocol = "range"
     val memberId = "memberId"
@@ -826,7 +826,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testStoreEmptyGroup() {
+  def testStoreEmptyGroup(): Unit = {
     val generation = 27
     val protocolType = "consumer"
 
@@ -837,7 +837,7 @@ class GroupMetadataManagerTest {
     EasyMock.replay(replicaManager)
 
     var maybeError: Option[Errors] = None
-    def callback(error: Errors) {
+    def callback(error: Errors): Unit = {
       maybeError = Some(error)
     }
 
@@ -856,7 +856,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testStoreEmptySimpleGroup() {
+  def testStoreEmptySimpleGroup(): Unit = {
     val group = new GroupMetadata(groupId, Empty, time)
     groupMetadataManager.addGroup(group)
 
@@ -864,7 +864,7 @@ class GroupMetadataManagerTest {
     EasyMock.replay(replicaManager)
 
     var maybeError: Option[Errors] = None
-    def callback(error: Errors) {
+    def callback(error: Errors): Unit = {
       maybeError = Some(error)
     }
 
@@ -885,7 +885,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testStoreGroupErrorMapping() {
+  def testStoreGroupErrorMapping(): Unit = {
     assertStoreGroupErrorMapping(Errors.NONE, Errors.NONE)
     assertStoreGroupErrorMapping(Errors.UNKNOWN_TOPIC_OR_PARTITION, Errors.COORDINATOR_NOT_AVAILABLE)
     assertStoreGroupErrorMapping(Errors.NOT_ENOUGH_REPLICAS, Errors.COORDINATOR_NOT_AVAILABLE)
@@ -897,7 +897,7 @@ class GroupMetadataManagerTest {
     assertStoreGroupErrorMapping(Errors.CORRUPT_MESSAGE, Errors.CORRUPT_MESSAGE)
   }
 
-  private def assertStoreGroupErrorMapping(appendError: Errors, expectedError: Errors) {
+  private def assertStoreGroupErrorMapping(appendError: Errors, expectedError: Errors): Unit = {
     EasyMock.reset(replicaManager)
 
     val group = new GroupMetadata(groupId, Empty, time)
@@ -907,7 +907,7 @@ class GroupMetadataManagerTest {
     EasyMock.replay(replicaManager)
 
     var maybeError: Option[Errors] = None
-    def callback(error: Errors) {
+    def callback(error: Errors): Unit = {
       maybeError = Some(error)
     }
 
@@ -918,7 +918,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testStoreNonEmptyGroup() {
+  def testStoreNonEmptyGroup(): Unit = {
     val memberId = "memberId"
     val clientId = "clientId"
     val clientHost = "localhost"
@@ -936,7 +936,7 @@ class GroupMetadataManagerTest {
     EasyMock.replay(replicaManager)
 
     var maybeError: Option[Errors] = None
-    def callback(error: Errors) {
+    def callback(error: Errors): Unit = {
       maybeError = Some(error)
     }
 
@@ -947,7 +947,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testStoreNonEmptyGroupWhenCoordinatorHasMoved() {
+  def testStoreNonEmptyGroupWhenCoordinatorHasMoved(): Unit = {
     EasyMock.expect(replicaManager.getMagic(EasyMock.anyObject())).andReturn(None)
     val memberId = "memberId"
     val clientId = "clientId"
@@ -964,7 +964,7 @@ class GroupMetadataManagerTest {
     EasyMock.replay(replicaManager)
 
     var maybeError: Option[Errors] = None
-    def callback(error: Errors) {
+    def callback(error: Errors): Unit = {
       maybeError = Some(error)
     }
 
@@ -975,7 +975,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testCommitOffset() {
+  def testCommitOffset(): Unit = {
     val memberId = ""
     val topicPartition = new TopicPartition("foo", 0)
     val offset = 37
@@ -991,7 +991,7 @@ class GroupMetadataManagerTest {
     EasyMock.replay(replicaManager)
 
     var commitErrors: Option[immutable.Map[TopicPartition, Errors]] = None
-    def callback(errors: immutable.Map[TopicPartition, Errors]) {
+    def callback(errors: immutable.Map[TopicPartition, Errors]): Unit = {
       commitErrors = Some(errors)
     }
 
@@ -1015,7 +1015,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testTransactionalCommitOffsetCommitted() {
+  def testTransactionalCommitOffsetCommitted(): Unit = {
     val memberId = ""
     val topicPartition = new TopicPartition("foo", 0)
     val offset = 37
@@ -1034,7 +1034,7 @@ class GroupMetadataManagerTest {
     EasyMock.replay(replicaManager)
 
     var commitErrors: Option[immutable.Map[TopicPartition, Errors]] = None
-    def callback(errors: immutable.Map[TopicPartition, Errors]) {
+    def callback(errors: immutable.Map[TopicPartition, Errors]): Unit = {
       commitErrors = Some(errors)
     }
 
@@ -1056,7 +1056,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testTransactionalCommitOffsetAppendFailure() {
+  def testTransactionalCommitOffsetAppendFailure(): Unit = {
     val memberId = ""
     val topicPartition = new TopicPartition("foo", 0)
     val offset = 37
@@ -1074,7 +1074,7 @@ class GroupMetadataManagerTest {
     EasyMock.replay(replicaManager)
 
     var commitErrors: Option[immutable.Map[TopicPartition, Errors]] = None
-    def callback(errors: immutable.Map[TopicPartition, Errors]) {
+    def callback(errors: immutable.Map[TopicPartition, Errors]): Unit = {
       commitErrors = Some(errors)
     }
 
@@ -1095,7 +1095,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testTransactionalCommitOffsetAborted() {
+  def testTransactionalCommitOffsetAborted(): Unit = {
     val memberId = ""
     val topicPartition = new TopicPartition("foo", 0)
     val offset = 37
@@ -1113,7 +1113,7 @@ class GroupMetadataManagerTest {
     EasyMock.replay(replicaManager)
 
     var commitErrors: Option[immutable.Map[TopicPartition, Errors]] = None
-    def callback(errors: immutable.Map[TopicPartition, Errors]) {
+    def callback(errors: immutable.Map[TopicPartition, Errors]): Unit = {
       commitErrors = Some(errors)
     }
 
@@ -1134,7 +1134,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testCommitOffsetWhenCoordinatorHasMoved() {
+  def testCommitOffsetWhenCoordinatorHasMoved(): Unit = {
     EasyMock.expect(replicaManager.getMagic(EasyMock.anyObject())).andReturn(None)
     val memberId = ""
     val topicPartition = new TopicPartition("foo", 0)
@@ -1150,7 +1150,7 @@ class GroupMetadataManagerTest {
     EasyMock.replay(replicaManager)
 
     var commitErrors: Option[immutable.Map[TopicPartition, Errors]] = None
-    def callback(errors: immutable.Map[TopicPartition, Errors]) {
+    def callback(errors: immutable.Map[TopicPartition, Errors]): Unit = {
       commitErrors = Some(errors)
     }
 
@@ -1164,7 +1164,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testCommitOffsetFailure() {
+  def testCommitOffsetFailure(): Unit = {
     assertCommitOffsetErrorMapping(Errors.UNKNOWN_TOPIC_OR_PARTITION, Errors.COORDINATOR_NOT_AVAILABLE)
     assertCommitOffsetErrorMapping(Errors.NOT_ENOUGH_REPLICAS, Errors.COORDINATOR_NOT_AVAILABLE)
     assertCommitOffsetErrorMapping(Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND, Errors.COORDINATOR_NOT_AVAILABLE)
@@ -1193,7 +1193,7 @@ class GroupMetadataManagerTest {
     EasyMock.replay(replicaManager)
 
     var commitErrors: Option[immutable.Map[TopicPartition, Errors]] = None
-    def callback(errors: immutable.Map[TopicPartition, Errors]) {
+    def callback(errors: immutable.Map[TopicPartition, Errors]): Unit = {
       commitErrors = Some(errors)
     }
 
@@ -1214,7 +1214,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testExpireOffset() {
+  def testExpireOffset(): Unit = {
     val memberId = ""
     val topicPartition1 = new TopicPartition("foo", 0)
     val topicPartition2 = new TopicPartition("foo", 1)
@@ -1236,7 +1236,7 @@ class GroupMetadataManagerTest {
     EasyMock.replay(replicaManager)
 
     var commitErrors: Option[immutable.Map[TopicPartition, Errors]] = None
-    def callback(errors: immutable.Map[TopicPartition, Errors]) {
+    def callback(errors: immutable.Map[TopicPartition, Errors]): Unit = {
       commitErrors = Some(errors)
     }
 
@@ -1269,7 +1269,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testGroupMetadataRemoval() {
+  def testGroupMetadataRemoval(): Unit = {
     val topicPartition1 = new TopicPartition("foo", 0)
     val topicPartition2 = new TopicPartition("foo", 1)
 
@@ -1317,7 +1317,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testGroupMetadataRemovalWithLogAppendTime() {
+  def testGroupMetadataRemovalWithLogAppendTime(): Unit = {
     val topicPartition1 = new TopicPartition("foo", 0)
     val topicPartition2 = new TopicPartition("foo", 1)
 
@@ -1366,7 +1366,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testExpireGroupWithOffsetsOnly() {
+  def testExpireGroupWithOffsetsOnly(): Unit = {
     // verify that the group is removed properly, but no tombstone is written if
     // this is a group which is only using kafka for offset storage
 
@@ -1391,7 +1391,7 @@ class GroupMetadataManagerTest {
     EasyMock.replay(replicaManager)
 
     var commitErrors: Option[immutable.Map[TopicPartition, Errors]] = None
-    def callback(errors: immutable.Map[TopicPartition, Errors]) {
+    def callback(errors: immutable.Map[TopicPartition, Errors]): Unit = {
       commitErrors = Some(errors)
     }
 
@@ -1438,7 +1438,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testOffsetExpirationSemantics() {
+  def testOffsetExpirationSemantics(): Unit = {
     val memberId = "memberId"
     val clientId = "clientId"
     val clientHost = "localhost"
@@ -1476,7 +1476,7 @@ class GroupMetadataManagerTest {
     EasyMock.replay(replicaManager)
 
     var commitErrors: Option[immutable.Map[TopicPartition, Errors]] = None
-    def callback(errors: immutable.Map[TopicPartition, Errors]) {
+    def callback(errors: immutable.Map[TopicPartition, Errors]): Unit = {
       commitErrors = Some(errors)
     }
 
@@ -1600,7 +1600,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testOffsetExpirationOfSimpleConsumer() {
+  def testOffsetExpirationOfSimpleConsumer(): Unit = {
     val memberId = "memberId"
     val topic = "foo"
     val topicPartition1 = new TopicPartition(topic, 0)
@@ -1624,7 +1624,7 @@ class GroupMetadataManagerTest {
     EasyMock.replay(replicaManager)
 
     var commitErrors: Option[immutable.Map[TopicPartition, Errors]] = None
-    def callback(errors: immutable.Map[TopicPartition, Errors]) {
+    def callback(errors: immutable.Map[TopicPartition, Errors]): Unit = {
       commitErrors = Some(errors)
     }
 
@@ -1847,7 +1847,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testLoadOffsetsWithEmptyControlBatch() {
+  def testLoadOffsetsWithEmptyControlBatch(): Unit = {
     val groupMetadataTopicPartition = groupTopicPartition
     val startOffset = 15L
     val generation = 15
@@ -2044,7 +2044,7 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testMetrics() {
+  def testMetrics(): Unit = {
     groupMetadataManager.cleanupGroupMetadata()
     expectMetrics(groupMetadataManager, 0, 0, 0)
     val group = new GroupMetadata("foo2", Stable, time)
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala
index 177bef726eb9f..6436e959bc98a 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala
@@ -41,32 +41,32 @@ class GroupMetadataTest {
   private var member: MemberMetadata = null
 
   @Before
-  def setUp() {
+  def setUp(): Unit = {
     group = new GroupMetadata("groupId", Empty, Time.SYSTEM)
     member = new MemberMetadata(memberId, groupId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs,
       protocolType, List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte])))
   }
 
   @Test
-  def testCanRebalanceWhenStable() {
+  def testCanRebalanceWhenStable(): Unit = {
     assertTrue(group.canRebalance)
   }
 
   @Test
-  def testCanRebalanceWhenCompletingRebalance() {
+  def testCanRebalanceWhenCompletingRebalance(): Unit = {
     group.transitionTo(PreparingRebalance)
     group.transitionTo(CompletingRebalance)
     assertTrue(group.canRebalance)
   }
 
   @Test
-  def testCannotRebalanceWhenPreparingRebalance() {
+  def testCannotRebalanceWhenPreparingRebalance(): Unit = {
     group.transitionTo(PreparingRebalance)
     assertFalse(group.canRebalance)
   }
 
   @Test
-  def testCannotRebalanceWhenDead() {
+  def testCannotRebalanceWhenDead(): Unit = {
     group.transitionTo(PreparingRebalance)
     group.transitionTo(Empty)
     group.transitionTo(Dead)
@@ -74,19 +74,19 @@ class GroupMetadataTest {
   }
 
   @Test
-  def testStableToPreparingRebalanceTransition() {
+  def testStableToPreparingRebalanceTransition(): Unit = {
     group.transitionTo(PreparingRebalance)
     assertState(group, PreparingRebalance)
   }
 
   @Test
-  def testStableToDeadTransition() {
+  def testStableToDeadTransition(): Unit = {
     group.transitionTo(Dead)
     assertState(group, Dead)
   }
 
   @Test
-  def testAwaitingRebalanceToPreparingRebalanceTransition() {
+  def testAwaitingRebalanceToPreparingRebalanceTransition(): Unit = {
     group.transitionTo(PreparingRebalance)
     group.transitionTo(CompletingRebalance)
     group.transitionTo(PreparingRebalance)
@@ -94,21 +94,21 @@ class GroupMetadataTest {
   }
 
   @Test
-  def testPreparingRebalanceToDeadTransition() {
+  def testPreparingRebalanceToDeadTransition(): Unit = {
     group.transitionTo(PreparingRebalance)
     group.transitionTo(Dead)
     assertState(group, Dead)
   }
 
   @Test
-  def testPreparingRebalanceToEmptyTransition() {
+  def testPreparingRebalanceToEmptyTransition(): Unit = {
     group.transitionTo(PreparingRebalance)
     group.transitionTo(Empty)
     assertState(group, Empty)
   }
 
   @Test
-  def testEmptyToDeadTransition() {
+  def testEmptyToDeadTransition(): Unit = {
     group.transitionTo(PreparingRebalance)
     group.transitionTo(Empty)
     group.transitionTo(Dead)
@@ -116,7 +116,7 @@ class GroupMetadataTest {
   }
 
   @Test
-  def testAwaitingRebalanceToStableTransition() {
+  def testAwaitingRebalanceToStableTransition(): Unit = {
     group.transitionTo(PreparingRebalance)
     group.transitionTo(CompletingRebalance)
     group.transitionTo(Stable)
@@ -124,12 +124,12 @@ class GroupMetadataTest {
   }
 
   @Test(expected = classOf[IllegalStateException])
-  def testEmptyToStableIllegalTransition() {
+  def testEmptyToStableIllegalTransition(): Unit = {
     group.transitionTo(Stable)
   }
 
   @Test
-  def testStableToStableIllegalTransition() {
+  def testStableToStableIllegalTransition(): Unit = {
     group.transitionTo(PreparingRebalance)
     group.transitionTo(CompletingRebalance)
     group.transitionTo(Stable)
@@ -142,30 +142,30 @@ class GroupMetadataTest {
   }
 
   @Test(expected = classOf[IllegalStateException])
-  def testEmptyToAwaitingRebalanceIllegalTransition() {
+  def testEmptyToAwaitingRebalanceIllegalTransition(): Unit = {
     group.transitionTo(CompletingRebalance)
   }
 
   @Test(expected = classOf[IllegalStateException])
-  def testPreparingRebalanceToPreparingRebalanceIllegalTransition() {
+  def testPreparingRebalanceToPreparingRebalanceIllegalTransition(): Unit = {
     group.transitionTo(PreparingRebalance)
     group.transitionTo(PreparingRebalance)
   }
 
   @Test(expected = classOf[IllegalStateException])
-  def testPreparingRebalanceToStableIllegalTransition() {
+  def testPreparingRebalanceToStableIllegalTransition(): Unit = {
     group.transitionTo(PreparingRebalance)
     group.transitionTo(Stable)
   }
 
   @Test(expected = classOf[IllegalStateException])
-  def testAwaitingRebalanceToAwaitingRebalanceIllegalTransition() {
+  def testAwaitingRebalanceToAwaitingRebalanceIllegalTransition(): Unit = {
     group.transitionTo(PreparingRebalance)
     group.transitionTo(CompletingRebalance)
     group.transitionTo(CompletingRebalance)
   }
 
-  def testDeadToDeadIllegalTransition() {
+  def testDeadToDeadIllegalTransition(): Unit = {
     group.transitionTo(PreparingRebalance)
     group.transitionTo(Dead)
     group.transitionTo(Dead)
@@ -173,28 +173,28 @@ class GroupMetadataTest {
   }
 
   @Test(expected = classOf[IllegalStateException])
-  def testDeadToStableIllegalTransition() {
+  def testDeadToStableIllegalTransition(): Unit = {
     group.transitionTo(PreparingRebalance)
     group.transitionTo(Dead)
     group.transitionTo(Stable)
   }
 
   @Test(expected = classOf[IllegalStateException])
-  def testDeadToPreparingRebalanceIllegalTransition() {
+  def testDeadToPreparingRebalanceIllegalTransition(): Unit = {
     group.transitionTo(PreparingRebalance)
     group.transitionTo(Dead)
     group.transitionTo(PreparingRebalance)
   }
 
   @Test(expected = classOf[IllegalStateException])
-  def testDeadToAwaitingRebalanceIllegalTransition() {
+  def testDeadToAwaitingRebalanceIllegalTransition(): Unit = {
     group.transitionTo(PreparingRebalance)
     group.transitionTo(Dead)
     group.transitionTo(CompletingRebalance)
   }
 
   @Test
-  def testSelectProtocol() {
+  def testSelectProtocol(): Unit = {
     val memberId = "memberId"
     val member = new MemberMetadata(memberId, groupId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs,
       protocolType, List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte])))
@@ -220,13 +220,13 @@ class GroupMetadataTest {
   }
 
   @Test(expected = classOf[IllegalStateException])
-  def testSelectProtocolRaisesIfNoMembers() {
+  def testSelectProtocolRaisesIfNoMembers(): Unit = {
     group.selectProtocol
     fail()
   }
 
   @Test
-  def testSelectProtocolChoosesCompatibleProtocol() {
+  def testSelectProtocolChoosesCompatibleProtocol(): Unit = {
     val memberId = "memberId"
     val member = new MemberMetadata(memberId, groupId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs,
       protocolType, List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte])))
@@ -241,7 +241,7 @@ class GroupMetadataTest {
   }
 
   @Test
-  def testSupportsProtocols() {
+  def testSupportsProtocols(): Unit = {
     // by default, the group supports everything
     assertTrue(group.supportsProtocols(protocolType, Set("roundrobin", "range")))
 
@@ -263,7 +263,7 @@ class GroupMetadataTest {
   }
 
   @Test
-  def testInitNextGeneration() {
+  def testInitNextGeneration(): Unit = {
     member.supportedProtocols = List(("roundrobin", Array.empty[Byte]))
 
     group.transitionTo(PreparingRebalance)
@@ -279,7 +279,7 @@ class GroupMetadataTest {
   }
 
   @Test
-  def testInitNextGenerationEmptyGroup() {
+  def testInitNextGenerationEmptyGroup(): Unit = {
     assertEquals(Empty, group.currentState)
     assertEquals(0, group.generationId)
     assertNull(group.protocolOrNull)
@@ -546,7 +546,7 @@ class GroupMetadataTest {
     assertFalse(member.isAwaitingSync)
   }
 
-  private def assertState(group: GroupMetadata, targetState: GroupState) {
+  private def assertState(group: GroupMetadata, targetState: GroupState): Unit = {
     val states: Set[GroupState] = Set(Stable, PreparingRebalance, CompletingRebalance, Dead)
     val otherStates = states - targetState
     otherStates.foreach { otherState =>
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/MemberMetadataTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/MemberMetadataTest.scala
index 6c818d08a793c..d348cc8c931c3 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/MemberMetadataTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/MemberMetadataTest.scala
@@ -33,7 +33,7 @@ class MemberMetadataTest {
 
 
   @Test
-  def testMatchesSupportedProtocols() {
+  def testMatchesSupportedProtocols(): Unit = {
     val protocols = List(("range", Array.empty[Byte]))
 
     val member = new MemberMetadata(memberId, groupId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs,
@@ -45,7 +45,7 @@ class MemberMetadataTest {
   }
 
   @Test
-  def testVoteForPreferredProtocol() {
+  def testVoteForPreferredProtocol(): Unit = {
     val protocols = List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte]))
 
     val member = new MemberMetadata(memberId, groupId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs,
@@ -55,7 +55,7 @@ class MemberMetadataTest {
   }
 
   @Test
-  def testMetadata() {
+  def testMetadata(): Unit = {
     val protocols = List(("range", Array[Byte](0)), ("roundrobin", Array[Byte](1)))
 
     val member = new MemberMetadata(memberId, groupId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs,
@@ -65,7 +65,7 @@ class MemberMetadataTest {
   }
 
   @Test(expected = classOf[IllegalArgumentException])
-  def testMetadataRaisesOnUnsupportedProtocol() {
+  def testMetadataRaisesOnUnsupportedProtocol(): Unit = {
     val protocols = List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte]))
 
     val member = new MemberMetadata(memberId, groupId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs,
@@ -75,7 +75,7 @@ class MemberMetadataTest {
   }
 
   @Test(expected = classOf[IllegalArgumentException])
-  def testVoteRaisesOnNoSupportedProtocols() {
+  def testVoteRaisesOnNoSupportedProtocols(): Unit = {
     val protocols = List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte]))
 
     val member = new MemberMetadata(memberId, groupId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs,
@@ -85,7 +85,7 @@ class MemberMetadataTest {
   }
 
   @Test
-  def testHasValidGroupInstanceId() {
+  def testHasValidGroupInstanceId(): Unit = {
     val protocols = List(("range", Array[Byte](0)), ("roundrobin", Array[Byte](1)))
 
     val member = new MemberMetadata(memberId, groupId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs,
diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/ProducerIdManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/ProducerIdManagerTest.scala
index b2cc4a51a592f..148d22b57f881 100644
--- a/core/src/test/scala/unit/kafka/coordinator/transaction/ProducerIdManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/transaction/ProducerIdManagerTest.scala
@@ -32,7 +32,7 @@ class ProducerIdManagerTest {
   }
 
   @Test
-  def testGetProducerId() {
+  def testGetProducerId(): Unit = {
     var zkVersion: Option[Int] = None
     var data: Array[Byte] = null
     EasyMock.expect(zkClient.getDataAndVersion(EasyMock.anyString)).andAnswer(new IAnswer[(Option[Array[Byte]], Int)] {
@@ -75,7 +75,7 @@ class ProducerIdManagerTest {
   }
 
   @Test(expected = classOf[KafkaException])
-  def testExceedProducerIdLimit() {
+  def testExceedProducerIdLimit(): Unit = {
     EasyMock.expect(zkClient.getDataAndVersion(EasyMock.anyString)).andAnswer(new IAnswer[(Option[Array[Byte]], Int)] {
       override def answer(): (Option[Array[Byte]], Int) = {
         val json = ProducerIdManager.generateProducerIdBlockJson(
diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala
index 238ef4b5a3b08..3031671f4b43d 100644
--- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala
@@ -61,7 +61,7 @@ class TransactionCoordinatorConcurrencyTest extends AbstractCoordinatorConcurren
     (0 until numPartitions).map { i => (i, mutable.ArrayBuffer[SimpleRecord]()) }.toMap
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
 
     EasyMock.expect(zkClient.getTopicPartitionCount(TRANSACTION_STATE_TOPIC_NAME))
@@ -115,7 +115,7 @@ class TransactionCoordinatorConcurrencyTest extends AbstractCoordinatorConcurren
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     try {
       EasyMock.reset(zkClient, replicaManager)
       transactionCoordinator.shutdown()
diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionLogTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionLogTest.scala
index 21e92701b720f..5df399e67683b 100644
--- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionLogTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionLogTest.scala
@@ -38,7 +38,7 @@ class TransactionLogTest {
     new TopicPartition("topic2", 2))
 
   @Test
-  def shouldThrowExceptionWriteInvalidTxn() {
+  def shouldThrowExceptionWriteInvalidTxn(): Unit = {
     val transactionalId = "transactionalId"
     val producerId = 23423L
 
@@ -51,7 +51,7 @@ class TransactionLogTest {
   }
 
   @Test
-  def shouldReadWriteMessages() {
+  def shouldReadWriteMessages(): Unit = {
     val pidMappings = Map[String, Long]("zero" -> 0L,
       "one" -> 1L,
       "two" -> 2L,
diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala
index 4e778ddea63ac..d152888d4c132 100644
--- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala
@@ -78,20 +78,20 @@ class TransactionStateManagerTest {
   var expectedError: Errors = Errors.NONE
 
   @Before
-  def setUp() {
+  def setUp(): Unit = {
     // make sure the transactional id hashes to the assigning partition id
     assertEquals(partitionId, transactionManager.partitionFor(transactionalId1))
     assertEquals(partitionId, transactionManager.partitionFor(transactionalId2))
   }
 
   @After
-  def tearDown() {
+  def tearDown(): Unit = {
     EasyMock.reset(zkClient, replicaManager)
     transactionManager.shutdown()
   }
 
   @Test
-  def testValidateTransactionTimeout() {
+  def testValidateTransactionTimeout(): Unit = {
     assertTrue(transactionManager.validateTransactionTimeoutMs(1))
     assertFalse(transactionManager.validateTransactionTimeoutMs(-1))
     assertFalse(transactionManager.validateTransactionTimeoutMs(0))
@@ -100,7 +100,7 @@ class TransactionStateManagerTest {
   }
 
   @Test
-  def testAddGetPids() {
+  def testAddGetPids(): Unit = {
     transactionManager.addLoadedTransactionsToCache(partitionId, coordinatorEpoch, new Pool[String, TransactionMetadata]())
 
     assertEquals(Right(None), transactionManager.getTransactionState(transactionalId1))
@@ -113,7 +113,7 @@ class TransactionStateManagerTest {
   }
 
   @Test
-  def testLoadAndRemoveTransactionsForPartition() {
+  def testLoadAndRemoveTransactionsForPartition(): Unit = {
     // generate transaction log messages for two pids traces:
 
     // pid1's transaction started with two partitions
@@ -310,7 +310,7 @@ class TransactionStateManagerTest {
   }
 
   @Test
-  def testAppendFailToUnknownError() {
+  def testAppendFailToUnknownError(): Unit = {
     transactionManager.addLoadedTransactionsToCache(partitionId, coordinatorEpoch, new Pool[String, TransactionMetadata]())
     transactionManager.putTransactionStateIfNotExists(transactionalId1, txnMetadata1)
 
@@ -330,7 +330,7 @@ class TransactionStateManagerTest {
   }
 
   @Test
-  def testPendingStateNotResetOnRetryAppend() {
+  def testPendingStateNotResetOnRetryAppend(): Unit = {
     transactionManager.addLoadedTransactionsToCache(partitionId, coordinatorEpoch, new Pool[String, TransactionMetadata]())
     transactionManager.putTransactionStateIfNotExists(transactionalId1, txnMetadata1)
 
diff --git a/core/src/test/scala/unit/kafka/integration/KafkaServerTestHarness.scala b/core/src/test/scala/unit/kafka/integration/KafkaServerTestHarness.scala
index dd6b8d09334ef..2b252503ed9e3 100755
--- a/core/src/test/scala/unit/kafka/integration/KafkaServerTestHarness.scala
+++ b/core/src/test/scala/unit/kafka/integration/KafkaServerTestHarness.scala
@@ -61,13 +61,13 @@ abstract class KafkaServerTestHarness extends ZooKeeperTestHarness {
    *
    * The default implementation of this method is a no-op.
    */
-  def configureSecurityBeforeServersStart() {}
+  def configureSecurityBeforeServersStart(): Unit = {}
 
   /**
    * Override this in case Tokens or security credentials needs to be created after `servers` are started.
    * The default implementation of this method is a no-op.
    */
-  def configureSecurityAfterServersStart() {}
+  def configureSecurityAfterServersStart(): Unit = {}
 
   def configs: Seq[KafkaConfig] = {
     if (instanceConfigs == null)
@@ -87,7 +87,7 @@ abstract class KafkaServerTestHarness extends ZooKeeperTestHarness {
   protected def brokerTime(brokerId: Int): Time = Time.SYSTEM
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
 
     if (configs.isEmpty)
@@ -109,7 +109,7 @@ abstract class KafkaServerTestHarness extends ZooKeeperTestHarness {
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     if (servers != null) {
       TestUtils.shutdownServers(servers)
     }
@@ -143,7 +143,7 @@ abstract class KafkaServerTestHarness extends ZooKeeperTestHarness {
     index
   }
 
-  def killBroker(index: Int) {
+  def killBroker(index: Int): Unit = {
     if(alive(index)) {
       servers(index).shutdown()
       servers(index).awaitShutdown()
@@ -154,7 +154,7 @@ abstract class KafkaServerTestHarness extends ZooKeeperTestHarness {
   /**
    * Restart any dead brokers
    */
-  def restartDeadBrokers() {
+  def restartDeadBrokers(): Unit = {
     for(i <- servers.indices if !alive(i)) {
       servers(i).startup()
       alive(i) = true
diff --git a/core/src/test/scala/unit/kafka/integration/MetricsDuringTopicCreationDeletionTest.scala b/core/src/test/scala/unit/kafka/integration/MetricsDuringTopicCreationDeletionTest.scala
index 20ac71c10063d..0acc184cd1a30 100644
--- a/core/src/test/scala/unit/kafka/integration/MetricsDuringTopicCreationDeletionTest.scala
+++ b/core/src/test/scala/unit/kafka/integration/MetricsDuringTopicCreationDeletionTest.scala
@@ -52,7 +52,7 @@ class MetricsDuringTopicCreationDeletionTest extends KafkaServerTestHarness with
     .map(KafkaConfig.fromProps(_, overridingProps))
 
   @Before
-  override def setUp {
+  override def setUp: Unit = {
     // Do some Metrics Registry cleanup by removing the metrics that this test checks.
     // This is a test workaround to the issue that prior harness runs may have left a populated registry.
     // see https://issues.apache.org/jira/browse/KAFKA-4605
@@ -68,7 +68,7 @@ class MetricsDuringTopicCreationDeletionTest extends KafkaServerTestHarness with
    * checking all metrics we care in a single test is faster though it would be more elegant to have 3 @Test methods
    */
   @Test
-  def testMetricsDuringTopicCreateDelete() {
+  def testMetricsDuringTopicCreateDelete(): Unit = {
 
     // For UnderReplicatedPartitions, because of https://issues.apache.org/jira/browse/KAFKA-4605
     // we can't access the metrics value of each server. So instead we directly invoke the method
@@ -88,7 +88,7 @@ class MetricsDuringTopicCreationDeletionTest extends KafkaServerTestHarness with
     // Thread checking the metric continuously
     running = true
     val thread = new Thread(new Runnable {
-      def run() {
+      def run(): Unit = {
         while (running) {
           for ( s <- servers if running) {
             underReplicatedPartitionCount = s.replicaManager.underReplicatedPartitionCount
@@ -131,7 +131,7 @@ class MetricsDuringTopicCreationDeletionTest extends KafkaServerTestHarness with
            ._2.asInstanceOf[Gauge[Int]]
   }
 
-  private def createDeleteTopics() {
+  private def createDeleteTopics(): Unit = {
     for (l <- 1 to createDeleteIterations if running) {
       // Create topics
       for (t <- topics if running) {
diff --git a/core/src/test/scala/unit/kafka/integration/MinIsrConfigTest.scala b/core/src/test/scala/unit/kafka/integration/MinIsrConfigTest.scala
index 1f01dc3be2b99..8ea5af789e8f6 100644
--- a/core/src/test/scala/unit/kafka/integration/MinIsrConfigTest.scala
+++ b/core/src/test/scala/unit/kafka/integration/MinIsrConfigTest.scala
@@ -30,7 +30,7 @@ class MinIsrConfigTest extends KafkaServerTestHarness {
   def generateConfigs = TestUtils.createBrokerConfigs(1, zkConnect).map(KafkaConfig.fromProps(_, overridingProps))
 
   @Test
-  def testDefaultKafkaConfig() {
+  def testDefaultKafkaConfig(): Unit = {
     assert(servers.head.getLogManager().initialDefaultConfig.minInSyncReplicas == 5)
   }
 
diff --git a/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala b/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala
index 64cafb49eb1c1..4996eec523223 100755
--- a/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala
+++ b/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala
@@ -62,7 +62,7 @@ class UncleanLeaderElectionTest extends ZooKeeperTestHarness {
   val networkProcessorLogger = Logger.getLogger(classOf[kafka.network.Processor])
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
 
     configProps1 = createBrokerConfig(brokerId1, zkConnect)
@@ -80,7 +80,7 @@ class UncleanLeaderElectionTest extends ZooKeeperTestHarness {
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     servers.foreach(server => shutdownServer(server))
     servers.foreach(server => CoreUtils.delete(server.config.logDirs))
 
@@ -91,7 +91,7 @@ class UncleanLeaderElectionTest extends ZooKeeperTestHarness {
     super.tearDown()
   }
 
-  private def startBrokers(cluster: Seq[Properties]) {
+  private def startBrokers(cluster: Seq[Properties]): Unit = {
     for (props <- cluster) {
       val config = KafkaConfig.fromProps(props)
       val server = createServer(config)
diff --git a/core/src/test/scala/unit/kafka/log/BrokerCompressionTest.scala b/core/src/test/scala/unit/kafka/log/BrokerCompressionTest.scala
index d7a725b19f3ad..8e4093ea83b39 100755
--- a/core/src/test/scala/unit/kafka/log/BrokerCompressionTest.scala
+++ b/core/src/test/scala/unit/kafka/log/BrokerCompressionTest.scala
@@ -41,7 +41,7 @@ class BrokerCompressionTest(messageCompression: String, brokerCompression: Strin
   val logConfig = LogConfig()
 
   @After
-  def tearDown() {
+  def tearDown(): Unit = {
     Utils.delete(tmpDir)
   }
 
@@ -49,7 +49,7 @@ class BrokerCompressionTest(messageCompression: String, brokerCompression: Strin
    * Test broker-side compression configuration
    */
   @Test
-  def testBrokerSideCompression() {
+  def testBrokerSideCompression(): Unit = {
     val messageCompressionCode = CompressionCodec.getCompressionCodec(messageCompression)
     val logProps = new Properties()
     logProps.put(LogConfig.CompressionTypeProp, brokerCompression)
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerIntegrationTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerIntegrationTest.scala
index 2d342facf5225..df6df0bd4ed11 100644
--- a/core/src/test/scala/unit/kafka/log/LogCleanerIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerIntegrationTest.scala
@@ -42,7 +42,7 @@ class LogCleanerIntegrationTest extends AbstractLogCleanerIntegrationTest {
   val topicPartitions = Array(new TopicPartition("log", 0), new TopicPartition("log", 1), new TopicPartition("log", 2))
 
   @Test(timeout = DEFAULT_MAX_WAIT_MS)
-  def testMarksPartitionsAsOfflineAndPopulatesUncleanableMetrics() {
+  def testMarksPartitionsAsOfflineAndPopulatesUncleanableMetrics(): Unit = {
     val largeMessageKey = 20
     val (_, largeMessageSet) = createLargeSingleMessageSet(largeMessageKey, RecordBatch.CURRENT_MAGIC_VALUE)
     val maxMessageSize = largeMessageSet.sizeInBytes
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerParameterizedIntegrationTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerParameterizedIntegrationTest.scala
index 4074191d36a50..815405319df8c 100755
--- a/core/src/test/scala/unit/kafka/log/LogCleanerParameterizedIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerParameterizedIntegrationTest.scala
@@ -48,7 +48,7 @@ class LogCleanerParameterizedIntegrationTest(compressionCodec: String) extends A
 
 
   @Test
-  def cleanerTest() {
+  def cleanerTest(): Unit = {
     val largeMessageKey = 20
     val (largeMessageValue, largeMessageSet) = createLargeSingleMessageSet(largeMessageKey, RecordBatch.CURRENT_MAGIC_VALUE)
     val maxMessageSize = largeMessageSet.sizeInBytes
@@ -226,7 +226,7 @@ class LogCleanerParameterizedIntegrationTest(compressionCodec: String) extends A
   }
 
   @Test
-  def cleanerConfigUpdateTest() {
+  def cleanerConfigUpdateTest(): Unit = {
     val largeMessageKey = 20
     val (largeMessageValue, largeMessageSet) = createLargeSingleMessageSet(largeMessageKey, RecordBatch.CURRENT_MAGIC_VALUE)
     val maxMessageSize = largeMessageSet.sizeInBytes
@@ -275,7 +275,7 @@ class LogCleanerParameterizedIntegrationTest(compressionCodec: String) extends A
     assertTrue(s"log should have been compacted: startSize=$startSize compactedSize=$compactedSize", startSize > compactedSize)
   }
 
-  private def checkLastCleaned(topic: String, partitionId: Int, firstDirty: Long) {
+  private def checkLastCleaned(topic: String, partitionId: Int, firstDirty: Long): Unit = {
     // wait until cleaning up to base_offset, note that cleaning happens only when "log dirty ratio" is higher than
     // LogConfig.MinCleanableDirtyRatioProp
     val topicPartition = new TopicPartition(topic, partitionId)
@@ -285,7 +285,7 @@ class LogCleanerParameterizedIntegrationTest(compressionCodec: String) extends A
       lastCleaned >= firstDirty)
   }
 
-  private def checkLogAfterAppendingDups(log: Log, startSize: Long, appends: Seq[(Int, String, Long)]) {
+  private def checkLogAfterAppendingDups(log: Log, startSize: Long, appends: Seq[(Int, String, Long)]): Unit = {
     val read = readFromLog(log)
     assertEquals("Contents of the map shouldn't change", toMap(appends), toMap(read))
     assertTrue(startSize > log.size)
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala
index 40ae5a8164524..caa985d18bde5 100755
--- a/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala
@@ -682,7 +682,7 @@ class LogCleanerTest {
    * Test log cleaning with logs containing messages larger than default message size
    */
   @Test
-  def testLargeMessage() {
+  def testLargeMessage(): Unit = {
     val largeMessageSize = 1024 * 1024
     // Create cleaner with very small default max message size
     val cleaner = makeCleaner(Int.MaxValue, maxMessageSize=1024)
@@ -713,7 +713,7 @@ class LogCleanerTest {
    * Test log cleaning with logs containing messages larger than topic's max message size
    */
   @Test
-  def testMessageLargerThanMaxMessageSize() {
+  def testMessageLargerThanMaxMessageSize(): Unit = {
     val (log, offsetMap) = createLogWithMessagesLargerThanMaxSize(largeMessageSize = 1024 * 1024)
 
     val cleaner = makeCleaner(Int.MaxValue, maxMessageSize=1024)
@@ -727,7 +727,7 @@ class LogCleanerTest {
    * where header is corrupt
    */
   @Test
-  def testMessageLargerThanMaxMessageSizeWithCorruptHeader() {
+  def testMessageLargerThanMaxMessageSizeWithCorruptHeader(): Unit = {
     val (log, offsetMap) = createLogWithMessagesLargerThanMaxSize(largeMessageSize = 1024 * 1024)
     val file = new RandomAccessFile(log.logSegments.head.log.file, "rw")
     file.seek(Records.MAGIC_OFFSET)
@@ -745,7 +745,7 @@ class LogCleanerTest {
    * where message size is corrupt and larger than bytes available in log segment.
    */
   @Test
-  def testCorruptMessageSizeLargerThanBytesAvailable() {
+  def testCorruptMessageSizeLargerThanBytesAvailable(): Unit = {
     val (log, offsetMap) = createLogWithMessagesLargerThanMaxSize(largeMessageSize = 1024 * 1024)
     val file = new RandomAccessFile(log.logSegments.head.log.file, "rw")
     file.setLength(1024)
@@ -1247,7 +1247,7 @@ class LogCleanerTest {
     val end = 500
     writeToLog(log, (start until end) zip (start until end))
 
-    def checkRange(map: FakeOffsetMap, start: Int, end: Int) {
+    def checkRange(map: FakeOffsetMap, start: Int, end: Int): Unit = {
       val stats = new CleanerStats()
       cleaner.buildOffsetMap(log, start, end, map, stats)
       val endOffset = map.latestOffset + 1
@@ -1467,7 +1467,7 @@ class LogCleanerTest {
    * This test verifies that messages corrupted by KAFKA-4298 are fixed by the cleaner
    */
   @Test
-  def testCleanCorruptMessageSet() {
+  def testCleanCorruptMessageSet(): Unit = {
     val codec = CompressionType.GZIP
 
     val logProps = new Properties()
diff --git a/core/src/test/scala/unit/kafka/log/LogConfigTest.scala b/core/src/test/scala/unit/kafka/log/LogConfigTest.scala
index 6f3332f98350c..e39f697efcc13 100644
--- a/core/src/test/scala/unit/kafka/log/LogConfigTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogConfigTest.scala
@@ -38,7 +38,7 @@ class LogConfigTest {
    * keys from LogConfig to KafkaConfig are not missing values.
    */
   @Test
-  def ensureNoStaticInitializationOrderDependency() {
+  def ensureNoStaticInitializationOrderDependency(): Unit = {
     // Access any KafkaConfig val to load KafkaConfig object before LogConfig.
     assertTrue(KafkaConfig.LogRetentionTimeMillisProp != null)
     assertTrue(LogConfig.configNames.forall { config =>
@@ -48,7 +48,7 @@ class LogConfigTest {
   }
 
   @Test
-  def testKafkaConfigToProps() {
+  def testKafkaConfigToProps(): Unit = {
     val millisInHour = 60L * 60L * 1000L
     val kafkaProps = TestUtils.createBrokerConfig(nodeId = 0, zkConnect = "")
     kafkaProps.put(KafkaConfig.LogRollTimeHoursProp, "2")
@@ -63,14 +63,14 @@ class LogConfigTest {
   }
 
   @Test
-  def testFromPropsEmpty() {
+  def testFromPropsEmpty(): Unit = {
     val p = new Properties()
     val config = LogConfig(p)
     Assert.assertEquals(LogConfig(), config)
   }
 
   @Test
-  def testFromPropsInvalid() {
+  def testFromPropsInvalid(): Unit = {
     LogConfig.configNames.foreach(name => name match {
       case LogConfig.UncleanLeaderElectionEnableProp => assertPropertyInvalid(name, "not a boolean")
       case LogConfig.RetentionBytesProp => assertPropertyInvalid(name, "not_a_number")
@@ -94,7 +94,7 @@ class LogConfigTest {
   }
 
   @Test
-  def shouldValidateThrottledReplicasConfig() {
+  def shouldValidateThrottledReplicasConfig(): Unit = {
     assertTrue(isValid("*"))
     assertTrue(isValid("* "))
     assertTrue(isValid(""))
@@ -164,7 +164,7 @@ class LogConfigTest {
     }
   }
 
-  private def assertPropertyInvalid(name: String, values: AnyRef*) {
+  private def assertPropertyInvalid(name: String, values: AnyRef*): Unit = {
     values.foreach((value) => {
       val props = new Properties
       props.setProperty(name, value.toString)
diff --git a/core/src/test/scala/unit/kafka/log/LogManagerTest.scala b/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
index 684f0b21db6a5..91f2a405251a1 100755
--- a/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
@@ -53,14 +53,14 @@ class LogManagerTest {
   val veryLargeLogFlushInterval = 10000000L
 
   @Before
-  def setUp() {
+  def setUp(): Unit = {
     logDir = TestUtils.tempDir()
     logManager = createLogManager()
     logManager.startup()
   }
 
   @After
-  def tearDown() {
+  def tearDown(): Unit = {
     if (logManager != null)
       logManager.shutdown()
     Utils.delete(logDir)
@@ -72,7 +72,7 @@ class LogManagerTest {
    * Test that getOrCreateLog on a non-existent log creates a new log and that we can append to the new log.
    */
   @Test
-  def testCreateLog() {
+  def testCreateLog(): Unit = {
     val log = logManager.getOrCreateLog(new TopicPartition(name, 0), logConfig)
     assertEquals(1, logManager.liveLogDirs.size)
 
@@ -86,7 +86,7 @@ class LogManagerTest {
    * The LogManager is configured with one invalid log directory which should be marked as offline.
    */
   @Test
-  def testCreateLogWithInvalidLogDir() {
+  def testCreateLogWithInvalidLogDir(): Unit = {
     // Configure the log dir with the Nul character as the path, which causes dir.getCanonicalPath() to throw an
     // IOException. This simulates the scenario where the disk is not properly mounted (which is hard to achieve in
     // a unit test)
@@ -103,7 +103,7 @@ class LogManagerTest {
   }
 
   @Test
-  def testCreateLogWithLogDirFallback() {
+  def testCreateLogWithLogDirFallback(): Unit = {
     // Configure a number of directories one level deeper in logDir,
     // so they all get cleaned up in tearDown().
     val dirs = (0 to 4)
@@ -147,7 +147,7 @@ class LogManagerTest {
    * Test that get on a non-existent returns None and no log is created.
    */
   @Test
-  def testGetNonExistentLog() {
+  def testGetNonExistentLog(): Unit = {
     val log = logManager.getLog(new TopicPartition(name, 0))
     assertEquals("No log should be found.", None, log)
     val logFile = new File(logDir, name + "-0")
@@ -158,7 +158,7 @@ class LogManagerTest {
    * Test time-based log cleanup. First append messages, then set the time into the future and run cleanup.
    */
   @Test
-  def testCleanupExpiredSegments() {
+  def testCleanupExpiredSegments(): Unit = {
     val log = logManager.getOrCreateLog(new TopicPartition(name, 0), logConfig)
     var offset = 0L
     for(_ <- 0 until 200) {
@@ -198,7 +198,7 @@ class LogManagerTest {
    * Test size-based cleanup. Append messages, then run cleanup and check that segments are deleted.
    */
   @Test
-  def testCleanupSegmentsToMaintainSize() {
+  def testCleanupSegmentsToMaintainSize(): Unit = {
     val setSize = TestUtils.singletonRecords("test".getBytes()).sizeInBytes
     logManager.shutdown()
     val logProps = new Properties()
@@ -248,7 +248,7 @@ class LogManagerTest {
     * LogCleaner.CleanerThread handles all logs where compaction is enabled.
     */
   @Test
-  def testDoesntCleanLogsWithCompactDeletePolicy() {
+  def testDoesntCleanLogsWithCompactDeletePolicy(): Unit = {
     testDoesntCleanLogs(LogConfig.Compact + "," + LogConfig.Delete)
   }
 
@@ -257,11 +257,11 @@ class LogManagerTest {
     * LogCleaner.CleanerThread handles all logs where compaction is enabled.
     */
   @Test
-  def testDoesntCleanLogsWithCompactPolicy() {
+  def testDoesntCleanLogsWithCompactPolicy(): Unit = {
     testDoesntCleanLogs(LogConfig.Compact)
   }
 
-  private def testDoesntCleanLogs(policy: String) {
+  private def testDoesntCleanLogs(policy: String): Unit = {
     val logProps = new Properties()
     logProps.put(LogConfig.CleanupPolicyProp, policy)
     val log = logManager.getOrCreateLog(new TopicPartition(name, 0), LogConfig.fromProps(logConfig.originals, logProps))
@@ -285,7 +285,7 @@ class LogManagerTest {
    * Test that flush is invoked by the background scheduler thread.
    */
   @Test
-  def testTimeBasedFlush() {
+  def testTimeBasedFlush(): Unit = {
     logManager.shutdown()
     val logProps = new Properties()
     logProps.put(LogConfig.FlushMsProp, 1000: java.lang.Integer)
@@ -307,7 +307,7 @@ class LogManagerTest {
    * Test that new logs that are created are assigned to the least loaded log directory
    */
   @Test
-  def testLeastLoadedAssignment() {
+  def testLeastLoadedAssignment(): Unit = {
     // create a log manager with multiple data directories
     val dirs = Seq(TestUtils.tempDir(),
                      TestUtils.tempDir(),
@@ -328,7 +328,7 @@ class LogManagerTest {
    * Test that it is not possible to open two log managers using the same data directory
    */
   @Test
-  def testTwoLogManagersUsingSameDirFails() {
+  def testTwoLogManagersUsingSameDirFails(): Unit = {
     try {
       createLogManager()
       fail("Should not be able to create a second log manager instance with the same data directory")
@@ -341,7 +341,7 @@ class LogManagerTest {
    * Test that recovery points are correctly written out to disk
    */
   @Test
-  def testCheckpointRecoveryPoints() {
+  def testCheckpointRecoveryPoints(): Unit = {
     verifyCheckpointRecovery(Seq(new TopicPartition("test-a", 1), new TopicPartition("test-b", 1)), logManager, logDir)
   }
 
@@ -349,7 +349,7 @@ class LogManagerTest {
    * Test that recovery points directory checking works with trailing slash
    */
   @Test
-  def testRecoveryDirectoryMappingWithTrailingSlash() {
+  def testRecoveryDirectoryMappingWithTrailingSlash(): Unit = {
     logManager.shutdown()
     logManager = TestUtils.createLogManager(logDirs = Seq(new File(TestUtils.tempDir().getAbsolutePath + File.separator)))
     logManager.startup()
@@ -360,14 +360,14 @@ class LogManagerTest {
    * Test that recovery points directory checking works with relative directory
    */
   @Test
-  def testRecoveryDirectoryMappingWithRelativeDirectory() {
+  def testRecoveryDirectoryMappingWithRelativeDirectory(): Unit = {
     logManager.shutdown()
     logManager = createLogManager(Seq(new File("data", logDir.getName).getAbsoluteFile))
     logManager.startup()
     verifyCheckpointRecovery(Seq(new TopicPartition("test-a", 1)), logManager, logManager.liveLogDirs.head)
   }
 
-  private def verifyCheckpointRecovery(topicPartitions: Seq[TopicPartition], logManager: LogManager, logDir: File) {
+  private def verifyCheckpointRecovery(topicPartitions: Seq[TopicPartition], logManager: LogManager, logDir: File): Unit = {
     val logs = topicPartitions.map(logManager.getOrCreateLog(_, logConfig))
     logs.foreach { log =>
       for (_ <- 0 until 50)
@@ -393,7 +393,7 @@ class LogManagerTest {
   }
 
   @Test
-  def testFileReferencesAfterAsyncDelete() {
+  def testFileReferencesAfterAsyncDelete(): Unit = {
     val log = logManager.getOrCreateLog(new TopicPartition(name, 0), logConfig)
     val activeSegment = log.activeSegment
     val logName = activeSegment.log.file.getName
@@ -436,7 +436,7 @@ class LogManagerTest {
   }
 
   @Test
-  def testCheckpointForOnlyAffectedLogs() {
+  def testCheckpointForOnlyAffectedLogs(): Unit = {
     val tps = Seq(
       new TopicPartition("test-a", 0),
       new TopicPartition("test-a", 1),
diff --git a/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala b/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala
index c3a6899ed3093..5e0a1a83c3af0 100644
--- a/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala
@@ -56,7 +56,7 @@ class LogSegmentTest {
   }
 
   @After
-  def teardown() {
+  def teardown(): Unit = {
     segments.foreach(_.close())
     Utils.delete(logDir)
   }
@@ -65,7 +65,7 @@ class LogSegmentTest {
    * A read on an empty log segment should return null
    */
   @Test
-  def testReadOnEmptySegment() {
+  def testReadOnEmptySegment(): Unit = {
     val seg = createSegment(40)
     val read = seg.read(startOffset = 40, maxSize = 300)
     assertNull("Read beyond the last offset in the segment should be null", read)
@@ -76,7 +76,7 @@ class LogSegmentTest {
    * beginning with the first message in the segment
    */
   @Test
-  def testReadBeforeFirstOffset() {
+  def testReadBeforeFirstOffset(): Unit = {
     val seg = createSegment(40)
     val ms = records(50, "hello", "there", "little", "bee")
     seg.append(53, RecordBatch.NO_TIMESTAMP, -1L, ms)
@@ -88,7 +88,7 @@ class LogSegmentTest {
    * If we read from an offset beyond the last offset in the segment we should get null
    */
   @Test
-  def testReadAfterLast() {
+  def testReadAfterLast(): Unit = {
     val seg = createSegment(40)
     val ms = records(50, "hello", "there")
     seg.append(51, RecordBatch.NO_TIMESTAMP, -1L, ms)
@@ -101,7 +101,7 @@ class LogSegmentTest {
    * with the least offset greater than the given startOffset.
    */
   @Test
-  def testReadFromGap() {
+  def testReadFromGap(): Unit = {
     val seg = createSegment(40)
     val ms = records(50, "hello", "there")
     seg.append(51, RecordBatch.NO_TIMESTAMP, -1L, ms)
@@ -116,7 +116,7 @@ class LogSegmentTest {
    * the first but not the second message.
    */
   @Test
-  def testTruncate() {
+  def testTruncate(): Unit = {
     val seg = createSegment(40)
     var offset = 40
     for (_ <- 0 until 30) {
@@ -137,7 +137,7 @@ class LogSegmentTest {
   }
 
   @Test
-  def testTruncateEmptySegment() {
+  def testTruncateEmptySegment(): Unit = {
     // This tests the scenario in which the follower truncates to an empty segment. In this
     // case we must ensure that the index is resized so that the log segment is not mistakenly
     // rolled due to a full index
@@ -175,7 +175,7 @@ class LogSegmentTest {
   }
 
   @Test
-  def testReloadLargestTimestampAndNextOffsetAfterTruncation() {
+  def testReloadLargestTimestampAndNextOffsetAfterTruncation(): Unit = {
     val numMessages = 30
     val seg = createSegment(40, 2 * records(0, "hello").sizeInBytes - 1)
     var offset = 40
@@ -198,7 +198,7 @@ class LogSegmentTest {
    * Test truncating the whole segment, and check that we can reappend with the original offset.
    */
   @Test
-  def testTruncateFull() {
+  def testTruncateFull(): Unit = {
     // test the case where we fully truncate the log
     val time = new MockTime
     val seg = createSegment(40, time = time)
@@ -221,7 +221,7 @@ class LogSegmentTest {
    * Append messages with timestamp and search message by timestamp.
    */
   @Test
-  def testFindOffsetByTimestamp() {
+  def testFindOffsetByTimestamp(): Unit = {
     val messageSize = records(0, s"msg00").sizeInBytes
     val seg = createSegment(40, messageSize * 2 - 1)
     // Produce some messages
@@ -247,7 +247,7 @@ class LogSegmentTest {
    * Test that offsets are assigned sequentially and that the nextOffset variable is incremented
    */
   @Test
-  def testNextOffsetCalculation() {
+  def testNextOffsetCalculation(): Unit = {
     val seg = createSegment(40)
     assertEquals(40, seg.readNextOffset)
     seg.append(52, RecordBatch.NO_TIMESTAMP, -1L, records(50, "hello", "there", "you"))
@@ -258,7 +258,7 @@ class LogSegmentTest {
    * Test that we can change the file suffixes for the log and index files
    */
   @Test
-  def testChangeFileSuffixes() {
+  def testChangeFileSuffixes(): Unit = {
     val seg = createSegment(40)
     val logFile = seg.log.file
     val indexFile = seg.lazyOffsetIndex.file
@@ -274,7 +274,7 @@ class LogSegmentTest {
    * and recover the segment, the entries should all be readable.
    */
   @Test
-  def testRecoveryFixesCorruptIndex() {
+  def testRecoveryFixesCorruptIndex(): Unit = {
     val seg = createSegment(0)
     for(i <- 0 until 100)
       seg.append(i, RecordBatch.NO_TIMESTAMP, -1L, records(i, i.toString))
@@ -365,7 +365,7 @@ class LogSegmentTest {
    * and recover the segment, the entries should all be readable.
    */
   @Test
-  def testRecoveryFixesCorruptTimeIndex() {
+  def testRecoveryFixesCorruptTimeIndex(): Unit = {
     val seg = createSegment(0)
     for(i <- 0 until 100)
       seg.append(i, i * 10, i, records(i, i.toString))
@@ -383,7 +383,7 @@ class LogSegmentTest {
    * Randomly corrupt a log a number of times and attempt recovery.
    */
   @Test
-  def testRecoveryWithCorruptMessage() {
+  def testRecoveryWithCorruptMessage(): Unit = {
     val messagesAppended = 20
     for (_ <- 0 until 10) {
       val seg = createSegment(0)
@@ -417,7 +417,7 @@ class LogSegmentTest {
 
   /* create a segment with   pre allocate, put message to it and verify */
   @Test
-  def testCreateWithInitFileSizeAppendMessage() {
+  def testCreateWithInitFileSizeAppendMessage(): Unit = {
     val seg = createSegment(40, false, 512*1024*1024, true)
     val ms = records(50, "hello", "there")
     seg.append(51, RecordBatch.NO_TIMESTAMP, -1L, ms)
@@ -429,7 +429,7 @@ class LogSegmentTest {
 
   /* create a segment with   pre allocate and clearly shut down*/
   @Test
-  def testCreateWithInitFileSizeClearShutdown() {
+  def testCreateWithInitFileSizeClearShutdown(): Unit = {
     val tempDir = TestUtils.tempDir()
     val logConfig = LogConfig(Map(
       LogConfig.IndexIntervalBytesProp -> 10,
@@ -469,7 +469,7 @@ class LogSegmentTest {
   }
 
   @Test
-  def shouldTruncateEvenIfOffsetPointsToAGapInTheLog() {
+  def shouldTruncateEvenIfOffsetPointsToAGapInTheLog(): Unit = {
     val seg = createSegment(40)
     val offset = 40
 
diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala
index fa7be7eb88c43..10ac4ad7b560f 100755
--- a/core/src/test/scala/unit/kafka/log/LogTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogTest.scala
@@ -57,18 +57,18 @@ class LogTest {
   val mockTime = new MockTime()
 
   @Before
-  def setUp() {
+  def setUp(): Unit = {
     val props = TestUtils.createBrokerConfig(0, "127.0.0.1:1", port = -1)
     config = KafkaConfig.fromProps(props)
   }
 
   @After
-  def tearDown() {
+  def tearDown(): Unit = {
     brokerTopicStats.close()
     Utils.delete(tmpDir)
   }
 
-  def createEmptyLogs(dir: File, offsets: Int*) {
+  def createEmptyLogs(dir: File, offsets: Int*): Unit = {
     for(offset <- offsets) {
       Log.logFile(dir, offset).createNewFile()
       Log.offsetIndexFile(dir, offset).createNewFile()
@@ -269,7 +269,7 @@ class LogTest {
   }
 
   @Test
-  def testOffsetFromFile() {
+  def testOffsetFromFile(): Unit = {
     val offset = 23423423L
 
     val logFile = Log.logFile(tmpDir, offset)
@@ -290,7 +290,7 @@ class LogTest {
    * using the mock clock to force the log to roll and checks the number of segments.
    */
   @Test
-  def testTimeBasedLogRoll() {
+  def testTimeBasedLogRoll(): Unit = {
     def createRecords = TestUtils.singletonRecords("test".getBytes)
     val logConfig = LogTest.createLogConfig(segmentMs = 1 * 60 * 60L)
 
@@ -338,7 +338,7 @@ class LogTest {
   }
 
   @Test
-  def testRollSegmentThatAlreadyExists() {
+  def testRollSegmentThatAlreadyExists(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentMs = 1 * 60 * 60L)
 
     // create a log
@@ -641,7 +641,7 @@ class LogTest {
   }
 
   @Test
-  def testProducerIdMapOffsetUpdatedForNonIdempotentData() {
+  def testProducerIdMapOffsetUpdatedForNonIdempotentData(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 2048 * 5)
     val log = createLog(logDir, logConfig)
     val records = TestUtils.records(List(new SimpleRecord(mockTime.milliseconds, "key".getBytes, "value".getBytes)))
@@ -849,7 +849,7 @@ class LogTest {
   }
 
   @Test
-  def testRebuildProducerIdMapWithCompactedData() {
+  def testRebuildProducerIdMapWithCompactedData(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 2048 * 5)
     val log = createLog(logDir, logConfig)
     val pid = 1L
@@ -892,7 +892,7 @@ class LogTest {
   }
 
   @Test
-  def testRebuildProducerStateWithEmptyCompactedBatch() {
+  def testRebuildProducerStateWithEmptyCompactedBatch(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 2048 * 5)
     val log = createLog(logDir, logConfig)
     val pid = 1L
@@ -933,7 +933,7 @@ class LogTest {
   }
 
   @Test
-  def testUpdateProducerIdMapWithCompactedData() {
+  def testUpdateProducerIdMapWithCompactedData(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 2048 * 5)
     val log = createLog(logDir, logConfig)
     val pid = 1L
@@ -966,7 +966,7 @@ class LogTest {
   }
 
   @Test
-  def testProducerIdMapTruncateTo() {
+  def testProducerIdMapTruncateTo(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 2048 * 5)
     val log = createLog(logDir, logConfig)
     log.appendAsLeader(TestUtils.records(List(new SimpleRecord("a".getBytes))), leaderEpoch = 0)
@@ -990,7 +990,7 @@ class LogTest {
   }
 
   @Test
-  def testProducerIdMapTruncateToWithNoSnapshots() {
+  def testProducerIdMapTruncateToWithNoSnapshots(): Unit = {
     // This ensures that the upgrade optimization path cannot be hit after initial loading
     val logConfig = LogTest.createLogConfig(segmentBytes = 2048 * 5)
     val log = createLog(logDir, logConfig)
@@ -1080,7 +1080,7 @@ class LogTest {
   }
 
   @Test
-  def testProducerIdMapTruncateFullyAndStartAt() {
+  def testProducerIdMapTruncateFullyAndStartAt(): Unit = {
     val records = TestUtils.singletonRecords("foo".getBytes)
     val logConfig = LogTest.createLogConfig(segmentBytes = records.sizeInBytes, retentionBytes = records.sizeInBytes * 2)
     val log = createLog(logDir, logConfig)
@@ -1102,7 +1102,7 @@ class LogTest {
   }
 
   @Test
-  def testProducerIdExpirationOnSegmentDeletion() {
+  def testProducerIdExpirationOnSegmentDeletion(): Unit = {
     val pid1 = 1L
     val records = TestUtils.records(Seq(new SimpleRecord("foo".getBytes)), producerId = pid1, producerEpoch = 0, sequence = 0)
     val logConfig = LogTest.createLogConfig(segmentBytes = records.sizeInBytes, retentionBytes = records.sizeInBytes * 2)
@@ -1128,7 +1128,7 @@ class LogTest {
   }
 
   @Test
-  def testTakeSnapshotOnRollAndDeleteSnapshotOnRecoveryPointCheckpoint() {
+  def testTakeSnapshotOnRollAndDeleteSnapshotOnRecoveryPointCheckpoint(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 2048 * 5)
     val log = createLog(logDir, logConfig)
     log.appendAsLeader(TestUtils.singletonRecords("a".getBytes), leaderEpoch = 0)
@@ -1232,7 +1232,7 @@ class LogTest {
   }
 
   @Test
-  def testPeriodicProducerIdExpiration() {
+  def testPeriodicProducerIdExpiration(): Unit = {
     val maxProducerIdExpirationMs = 200
     val producerIdExpirationCheckIntervalMs = 100
 
@@ -1327,7 +1327,7 @@ class LogTest {
   }
 
   @Test
-  def testMultipleProducerIdsPerMemoryRecord() : Unit = {
+  def testMultipleProducerIdsPerMemoryRecord(): Unit = {
     // create a log
     val log = createLog(logDir, LogConfig())
 
@@ -1373,7 +1373,7 @@ class LogTest {
   }
 
   @Test
-  def testDuplicateAppendToFollower() : Unit = {
+  def testDuplicateAppendToFollower(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 1024 * 1024 * 5)
     val log = createLog(logDir, logConfig)
     val epoch: Short = 0
@@ -1394,7 +1394,7 @@ class LogTest {
   }
 
   @Test
-  def testMultipleProducersWithDuplicatesInSingleAppend() : Unit = {
+  def testMultipleProducersWithDuplicatesInSingleAppend(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 1024 * 1024 * 5)
     val log = createLog(logDir, logConfig)
 
@@ -1465,7 +1465,7 @@ class LogTest {
    * using the mock clock to force the log to roll and checks the number of segments.
    */
   @Test
-  def testTimeBasedLogRollJitter() {
+  def testTimeBasedLogRollJitter(): Unit = {
     var set = TestUtils.singletonRecords(value = "test".getBytes, timestamp = mockTime.milliseconds)
     val maxJitter = 20 * 60L
     // create a log
@@ -1488,7 +1488,7 @@ class LogTest {
    * Test that appending more than the maximum segment size rolls the log
    */
   @Test
-  def testSizeBasedLogRoll() {
+  def testSizeBasedLogRoll(): Unit = {
     def createRecords = TestUtils.singletonRecords(value = "test".getBytes, timestamp = mockTime.milliseconds)
     val setSize = createRecords.sizeInBytes
     val msgPerSeg = 10
@@ -1508,7 +1508,7 @@ class LogTest {
    * Test that we can open and append to an empty log
    */
   @Test
-  def testLoadEmptyLog() {
+  def testLoadEmptyLog(): Unit = {
     createEmptyLogs(logDir, 0)
     val log = createLog(logDir, LogConfig())
     log.appendAsLeader(TestUtils.singletonRecords(value = "test".getBytes, timestamp = mockTime.milliseconds), leaderEpoch = 0)
@@ -1518,7 +1518,7 @@ class LogTest {
    * This test case appends a bunch of messages and checks that we can read them all back using sequential offsets.
    */
   @Test
-  def testAppendAndReadWithSequentialOffsets() {
+  def testAppendAndReadWithSequentialOffsets(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 71)
     val log = createLog(logDir, logConfig)
     val values = (0 until 100 by 2).map(id => id.toString.getBytes).toArray
@@ -1542,7 +1542,7 @@ class LogTest {
    * from any offset less than the logEndOffset including offsets not appended.
    */
   @Test
-  def testAppendAndReadWithNonSequentialOffsets() {
+  def testAppendAndReadWithNonSequentialOffsets(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 72)
     val log = createLog(logDir, logConfig)
     val messageIds = ((0 until 50) ++ (50 until 200 by 7)).toArray
@@ -1566,7 +1566,7 @@ class LogTest {
    * first segment has the greatest lower bound on the offset.
    */
   @Test
-  def testReadAtLogGap() {
+  def testReadAtLogGap(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 300)
     val log = createLog(logDir, logConfig)
 
@@ -1582,7 +1582,7 @@ class LogTest {
   }
 
   @Test(expected = classOf[KafkaStorageException])
-  def testLogRollAfterLogHandlerClosed() {
+  def testLogRollAfterLogHandlerClosed(): Unit = {
     val logConfig = LogTest.createLogConfig()
     val log = createLog(logDir,  logConfig)
     log.closeHandlers()
@@ -1590,7 +1590,7 @@ class LogTest {
   }
 
   @Test
-  def testReadWithMinMessage() {
+  def testReadWithMinMessage(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 72)
     val log = createLog(logDir,  logConfig)
     val messageIds = ((0 until 50) ++ (50 until 200 by 7)).toArray
@@ -1615,7 +1615,7 @@ class LogTest {
   }
 
   @Test
-  def testReadWithTooSmallMaxLength() {
+  def testReadWithTooSmallMaxLength(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 72)
     val log = createLog(logDir,  logConfig)
     val messageIds = ((0 until 50) ++ (50 until 200 by 7)).toArray
@@ -1647,7 +1647,7 @@ class LogTest {
    * - reading beyond the log end offset should throw an OffsetOutOfRangeException
    */
   @Test
-  def testReadOutOfRange() {
+  def testReadOutOfRange(): Unit = {
     createEmptyLogs(logDir, 1024)
     // set up replica log starting with offset 1024 and with one message (at offset 1024)
     val logConfig = LogTest.createLogConfig(segmentBytes = 1024)
@@ -1677,7 +1677,7 @@ class LogTest {
    * and then reads them all back and checks that the message read and offset matches what was appended.
    */
   @Test
-  def testLogRolls() {
+  def testLogRolls(): Unit = {
     /* create a multipart log with 100 messages */
     val logConfig = LogTest.createLogConfig(segmentBytes = 100)
     val log = createLog(logDir, logConfig)
@@ -1714,7 +1714,7 @@ class LogTest {
    * Test reads at offsets that fall within compressed message set boundaries.
    */
   @Test
-  def testCompressedMessages() {
+  def testCompressedMessages(): Unit = {
     /* this log should roll after every messageset */
     val logConfig = LogTest.createLogConfig(segmentBytes = 110)
     val log = createLog(logDir, logConfig)
@@ -1736,7 +1736,7 @@ class LogTest {
    * Test garbage collecting old segments
    */
   @Test
-  def testThatGarbageCollectingSegmentsDoesntChangeOffset() {
+  def testThatGarbageCollectingSegmentsDoesntChangeOffset(): Unit = {
     for(messagesToAppend <- List(0, 1, 25)) {
       logDir.mkdirs()
       // first test a log segment starting at 0
@@ -1770,7 +1770,7 @@ class LogTest {
    * appending a message set larger than the config.segmentSize setting and checking that an exception is thrown.
    */
   @Test
-  def testMessageSetSizeCheck() {
+  def testMessageSetSizeCheck(): Unit = {
     val messageSet = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("You".getBytes), new SimpleRecord("bethe".getBytes))
     // append messages to log
     val configSegmentSize = messageSet.sizeInBytes - 1
@@ -1786,7 +1786,7 @@ class LogTest {
   }
 
   @Test
-  def testCompactedTopicConstraints() {
+  def testCompactedTopicConstraints(): Unit = {
     val keyedMessage = new SimpleRecord("and here it is".getBytes, "this message has a key".getBytes)
     val anotherKeyedMessage = new SimpleRecord("another key".getBytes, "this message also has a key".getBytes)
     val unkeyedMessage = new SimpleRecord("this message does not have a key".getBytes)
@@ -1832,7 +1832,7 @@ class LogTest {
    * setting and checking that an exception is thrown.
    */
   @Test
-  def testMessageSizeCheck() {
+  def testMessageSizeCheck(): Unit = {
     val first = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("You".getBytes), new SimpleRecord("bethe".getBytes))
     val second = MemoryRecords.withRecords(CompressionType.NONE,
       new SimpleRecord("change (I need more bytes)... blah blah blah.".getBytes),
@@ -1857,7 +1857,7 @@ class LogTest {
    * Append a bunch of messages to a log and then re-open it both with and without recovery and check that the log re-initializes correctly.
    */
   @Test
-  def testLogRecoversToCorrectOffset() {
+  def testLogRecoversToCorrectOffset(): Unit = {
     val numMessages = 100
     val messageSize = 100
     val segmentSize = 7 * messageSize
@@ -1880,7 +1880,7 @@ class LogTest {
     }
     log.close()
 
-    def verifyRecoveredLog(log: Log, expectedRecoveryPoint: Long) {
+    def verifyRecoveredLog(log: Log, expectedRecoveryPoint: Long): Unit = {
       assertEquals(s"Unexpected recovery point", expectedRecoveryPoint, log.recoveryPoint)
       assertEquals(s"Should have $numMessages messages when log is reopened w/o recovery", numMessages, log.logEndOffset)
       assertEquals("Should have same last index offset as before.", lastIndexOffset, log.activeSegment.offsetIndex.lastOffset)
@@ -1904,7 +1904,7 @@ class LogTest {
    * Test building the time index on the follower by setting assignOffsets to false.
    */
   @Test
-  def testBuildTimeIndexWhenNotAssigningOffsets() {
+  def testBuildTimeIndexWhenNotAssigningOffsets(): Unit = {
     val numMessages = 100
     val logConfig = LogTest.createLogConfig(segmentBytes = 10000, indexIntervalBytes = 1)
     val log = createLog(logDir, logConfig)
@@ -1923,7 +1923,7 @@ class LogTest {
    * Test that if we manually delete an index segment it is rebuilt when the log is re-opened
    */
   @Test
-  def testIndexRebuild() {
+  def testIndexRebuild(): Unit = {
     // publish the messages and close the log
     val numMessages = 200
     val logConfig = LogTest.createLogConfig(segmentBytes = 200, indexIntervalBytes = 1)
@@ -1996,7 +1996,7 @@ class LogTest {
    * Test that if messages format version of the messages in a segment is before 0.10.0, the time index should be empty.
    */
   @Test
-  def testRebuildTimeIndexForOldMessages() {
+  def testRebuildTimeIndexForOldMessages(): Unit = {
     val numMessages = 200
     val segmentSize = 200
     val logConfig = LogTest.createLogConfig(segmentBytes = segmentSize, indexIntervalBytes = 1, messageFormatVersion = "0.9.0")
@@ -2022,7 +2022,7 @@ class LogTest {
    * Test that if we have corrupted an index segment it is rebuilt when the log is re-opened
    */
   @Test
-  def testCorruptIndexRebuild() {
+  def testCorruptIndexRebuild(): Unit = {
     // publish the messages and close the log
     val numMessages = 200
     val logConfig = LogTest.createLogConfig(segmentBytes = 200, indexIntervalBytes = 1)
@@ -2064,7 +2064,7 @@ class LogTest {
    * Test the Log truncate operations
    */
   @Test
-  def testTruncateTo() {
+  def testTruncateTo(): Unit = {
     def createRecords = TestUtils.singletonRecords(value = "test".getBytes, timestamp = mockTime.milliseconds)
     val setSize = createRecords.sizeInBytes
     val msgPerSeg = 10
@@ -2119,7 +2119,7 @@ class LogTest {
    * Verify that when we truncate a log the index of the last segment is resized to the max index size to allow more appends
    */
   @Test
-  def testIndexResizingAtTruncation() {
+  def testIndexResizingAtTruncation(): Unit = {
     val setSize = TestUtils.singletonRecords(value = "test".getBytes, timestamp = mockTime.milliseconds).sizeInBytes
     val msgPerSeg = 10
     val segmentSize = msgPerSeg * setSize  // each segment will be 10 messages
@@ -2155,7 +2155,7 @@ class LogTest {
    * When we open a log any index segments without an associated log segment should be deleted.
    */
   @Test
-  def testBogusIndexSegmentsAreRemoved() {
+  def testBogusIndexSegmentsAreRemoved(): Unit = {
     val bogusIndex1 = Log.offsetIndexFile(logDir, 0)
     val bogusTimeIndex1 = Log.timeIndexFile(logDir, 0)
     val bogusIndex2 = Log.offsetIndexFile(logDir, 5)
@@ -2190,7 +2190,7 @@ class LogTest {
    * Verify that truncation works correctly after re-opening the log
    */
   @Test
-  def testReopenThenTruncate() {
+  def testReopenThenTruncate(): Unit = {
     def createRecords = TestUtils.singletonRecords(value = "test".getBytes, timestamp = mockTime.milliseconds)
     // create a log
     val logConfig = LogTest.createLogConfig(segmentBytes = createRecords.sizeInBytes * 5, segmentIndexBytes = 1000, indexIntervalBytes = 10000)
@@ -2210,7 +2210,7 @@ class LogTest {
    * Test that deleted files are deleted after the appropriate time.
    */
   @Test
-  def testAsyncDelete() {
+  def testAsyncDelete(): Unit = {
     def createRecords = TestUtils.singletonRecords(value = "test".getBytes, timestamp = mockTime.milliseconds - 1000L)
     val asyncDeleteMs = 1000
     val logConfig = LogTest.createLogConfig(segmentBytes = createRecords.sizeInBytes * 5, segmentIndexBytes = 1000, indexIntervalBytes = 10000,
@@ -2245,7 +2245,7 @@ class LogTest {
    * Any files ending in .deleted should be removed when the log is re-opened.
    */
   @Test
-  def testOpenDeletesObsoleteFiles() {
+  def testOpenDeletesObsoleteFiles(): Unit = {
     def createRecords = TestUtils.singletonRecords(value = "test".getBytes, timestamp = mockTime.milliseconds - 1000)
     val logConfig = LogTest.createLogConfig(segmentBytes = createRecords.sizeInBytes * 5, segmentIndexBytes = 1000, retentionMs = 999)
     var log = createLog(logDir, logConfig)
@@ -2263,7 +2263,7 @@ class LogTest {
   }
 
   @Test
-  def testAppendMessageWithNullPayload() {
+  def testAppendMessageWithNullPayload(): Unit = {
     val log = createLog(logDir, LogConfig())
     log.appendAsLeader(TestUtils.singletonRecords(value = null), leaderEpoch = 0)
     val head = readLog(log, 0, 4096).records.records.iterator.next()
@@ -2272,7 +2272,7 @@ class LogTest {
   }
 
   @Test
-  def testAppendWithOutOfOrderOffsetsThrowsException() {
+  def testAppendWithOutOfOrderOffsetsThrowsException(): Unit = {
     val log = createLog(logDir, LogConfig())
 
     val appendOffsets = Seq(0L, 1L, 3L, 2L, 4L)
@@ -2293,7 +2293,7 @@ class LogTest {
   }
 
   @Test
-  def testAppendBelowExpectedOffsetThrowsException() {
+  def testAppendBelowExpectedOffsetThrowsException(): Unit = {
     val log = createLog(logDir, LogConfig())
     val records = (0 until 2).map(id => new SimpleRecord(id.toString.getBytes)).toArray
     records.foreach(record => log.appendAsLeader(MemoryRecords.withRecords(CompressionType.NONE, record), leaderEpoch = 0))
@@ -2311,7 +2311,7 @@ class LogTest {
   }
 
   @Test
-  def testAppendEmptyLogBelowLogStartOffsetThrowsException() {
+  def testAppendEmptyLogBelowLogStartOffsetThrowsException(): Unit = {
     createEmptyLogs(logDir, 7)
     val log = createLog(logDir, LogConfig(), brokerTopicStats = brokerTopicStats)
     assertEquals(7L, log.logStartOffset)
@@ -2347,7 +2347,7 @@ class LogTest {
   }
 
   @Test
-  def testCorruptLog() {
+  def testCorruptLog(): Unit = {
     // append some messages to create some segments
     val logConfig = LogTest.createLogConfig(segmentBytes = 1000, indexIntervalBytes = 1, maxMessageBytes = 64 * 1024)
     def createRecords = TestUtils.singletonRecords(value = "test".getBytes, timestamp = mockTime.milliseconds)
@@ -2815,7 +2815,7 @@ class LogTest {
   }
 
   @Test
-  def testCleanShutdownFile() {
+  def testCleanShutdownFile(): Unit = {
     // append some messages to create some segments
     val logConfig = LogTest.createLogConfig(segmentBytes = 1000, indexIntervalBytes = 1, maxMessageBytes = 64 * 1024)
     def createRecords = TestUtils.singletonRecords(value = "test".getBytes, timestamp = mockTime.milliseconds)
@@ -2838,7 +2838,7 @@ class LogTest {
   }
 
   @Test
-  def testParseTopicPartitionName() {
+  def testParseTopicPartitionName(): Unit = {
     val topic = "test_topic"
     val partition = "143"
     val dir = new File(logDir, topicPartitionName(topic, partition))
@@ -2852,7 +2852,7 @@ class LogTest {
    * are parsed correctly by `Log.parseTopicPartitionName` (see KAFKA-5232 for details).
    */
   @Test
-  def testParseTopicPartitionNameWithPeriodForDeletedTopic() {
+  def testParseTopicPartitionNameWithPeriodForDeletedTopic(): Unit = {
     val topic = "foo.bar-testtopic"
     val partition = "42"
     val dir = new File(logDir, Log.logDeleteDirName(new TopicPartition(topic, partition.toInt)))
@@ -2862,7 +2862,7 @@ class LogTest {
   }
 
   @Test
-  def testParseTopicPartitionNameForEmptyName() {
+  def testParseTopicPartitionNameForEmptyName(): Unit = {
     try {
       val dir = new File("")
       Log.parseTopicPartitionName(dir)
@@ -2873,7 +2873,7 @@ class LogTest {
   }
 
   @Test
-  def testParseTopicPartitionNameForNull() {
+  def testParseTopicPartitionNameForNull(): Unit = {
     try {
       val dir: File = null
       Log.parseTopicPartitionName(dir)
@@ -2884,7 +2884,7 @@ class LogTest {
   }
 
   @Test
-  def testParseTopicPartitionNameForMissingSeparator() {
+  def testParseTopicPartitionNameForMissingSeparator(): Unit = {
     val topic = "test_topic"
     val partition = "1999"
     val dir = new File(logDir, topic + partition)
@@ -2905,7 +2905,7 @@ class LogTest {
   }
 
   @Test
-  def testParseTopicPartitionNameForMissingTopic() {
+  def testParseTopicPartitionNameForMissingTopic(): Unit = {
     val topic = ""
     val partition = "1999"
     val dir = new File(logDir, topicPartitionName(topic, partition))
@@ -2927,7 +2927,7 @@ class LogTest {
   }
 
   @Test
-  def testParseTopicPartitionNameForMissingPartition() {
+  def testParseTopicPartitionNameForMissingPartition(): Unit = {
     val topic = "test_topic"
     val partition = ""
     val dir = new File(logDir + topicPartitionName(topic, partition))
@@ -2948,7 +2948,7 @@ class LogTest {
   }
 
   @Test
-  def testParseTopicPartitionNameForInvalidPartition() {
+  def testParseTopicPartitionNameForInvalidPartition(): Unit = {
     val topic = "test_topic"
     val partition = "1999a"
     val dir = new File(logDir, topicPartitionName(topic, partition))
@@ -2969,7 +2969,7 @@ class LogTest {
   }
 
   @Test
-  def testParseTopicPartitionNameForExistingInvalidDir() {
+  def testParseTopicPartitionNameForExistingInvalidDir(): Unit = {
     val dir1 = new File(logDir + "/non_kafka_dir")
     try {
       Log.parseTopicPartitionName(dir1)
@@ -2990,7 +2990,7 @@ class LogTest {
     topic + "-" + partition
 
   @Test
-  def testDeleteOldSegments() {
+  def testDeleteOldSegments(): Unit = {
     def createRecords = TestUtils.singletonRecords(value = "test".getBytes, timestamp = mockTime.milliseconds - 1000)
     val logConfig = LogTest.createLogConfig(segmentBytes = createRecords.sizeInBytes * 5, segmentIndexBytes = 1000, retentionMs = 999)
     val log = createLog(logDir, logConfig)
@@ -3040,7 +3040,7 @@ class LogTest {
   }
 
   @Test
-  def testLogDeletionAfterClose() {
+  def testLogDeletionAfterClose(): Unit = {
     def createRecords = TestUtils.singletonRecords(value = "test".getBytes, timestamp = mockTime.milliseconds - 1000)
     val logConfig = LogTest.createLogConfig(segmentBytes = createRecords.sizeInBytes * 5, segmentIndexBytes = 1000, retentionMs = 999)
     val log = createLog(logDir, logConfig)
@@ -3058,7 +3058,7 @@ class LogTest {
   }
 
   @Test
-  def testLogDeletionAfterDeleteRecords() {
+  def testLogDeletionAfterDeleteRecords(): Unit = {
     def createRecords = TestUtils.singletonRecords("test".getBytes)
     val logConfig = LogTest.createLogConfig(segmentBytes = createRecords.sizeInBytes * 5)
     val log = createLog(logDir, logConfig)
@@ -3090,7 +3090,7 @@ class LogTest {
   }
 
   @Test
-  def shouldDeleteSizeBasedSegments() {
+  def shouldDeleteSizeBasedSegments(): Unit = {
     def createRecords = TestUtils.singletonRecords("test".getBytes)
     val logConfig = LogTest.createLogConfig(segmentBytes = createRecords.sizeInBytes * 5, retentionBytes = createRecords.sizeInBytes * 10)
     val log = createLog(logDir, logConfig)
@@ -3105,7 +3105,7 @@ class LogTest {
   }
 
   @Test
-  def shouldNotDeleteSizeBasedSegmentsWhenUnderRetentionSize() {
+  def shouldNotDeleteSizeBasedSegmentsWhenUnderRetentionSize(): Unit = {
     def createRecords = TestUtils.singletonRecords("test".getBytes)
     val logConfig = LogTest.createLogConfig(segmentBytes = createRecords.sizeInBytes * 5, retentionBytes = createRecords.sizeInBytes * 15)
     val log = createLog(logDir, logConfig)
@@ -3120,7 +3120,7 @@ class LogTest {
   }
 
   @Test
-  def shouldDeleteTimeBasedSegmentsReadyToBeDeleted() {
+  def shouldDeleteTimeBasedSegmentsReadyToBeDeleted(): Unit = {
     def createRecords = TestUtils.singletonRecords("test".getBytes, timestamp = 10)
     val logConfig = LogTest.createLogConfig(segmentBytes = createRecords.sizeInBytes * 5, retentionMs = 10000)
     val log = createLog(logDir, logConfig)
@@ -3135,7 +3135,7 @@ class LogTest {
   }
 
   @Test
-  def shouldNotDeleteTimeBasedSegmentsWhenNoneReadyToBeDeleted() {
+  def shouldNotDeleteTimeBasedSegmentsWhenNoneReadyToBeDeleted(): Unit = {
     def createRecords = TestUtils.singletonRecords("test".getBytes, timestamp = mockTime.milliseconds)
     val logConfig = LogTest.createLogConfig(segmentBytes = createRecords.sizeInBytes * 5, retentionMs = 10000000)
     val log = createLog(logDir, logConfig)
@@ -3150,7 +3150,7 @@ class LogTest {
   }
 
   @Test
-  def shouldNotDeleteSegmentsWhenPolicyDoesNotIncludeDelete() {
+  def shouldNotDeleteSegmentsWhenPolicyDoesNotIncludeDelete(): Unit = {
     def createRecords = TestUtils.singletonRecords("test".getBytes, key = "test".getBytes(), timestamp = 10L)
     val logConfig = LogTest.createLogConfig(segmentBytes = createRecords.sizeInBytes * 5, retentionMs = 10000, cleanupPolicy = "compact")
     val log = createLog(logDir, logConfig)
@@ -3169,7 +3169,7 @@ class LogTest {
   }
 
   @Test
-  def shouldDeleteSegmentsReadyToBeDeletedWhenCleanupPolicyIsCompactAndDelete() {
+  def shouldDeleteSegmentsReadyToBeDeletedWhenCleanupPolicyIsCompactAndDelete(): Unit = {
     def createRecords = TestUtils.singletonRecords("test".getBytes, key = "test".getBytes, timestamp = 10L)
     val logConfig = LogTest.createLogConfig(segmentBytes = createRecords.sizeInBytes * 5, retentionMs = 10000, cleanupPolicy = "compact,delete")
     val log = createLog(logDir, logConfig)
@@ -3210,7 +3210,7 @@ class LogTest {
   }
 
   @Test
-  def shouldApplyEpochToMessageOnAppendIfLeader() {
+  def shouldApplyEpochToMessageOnAppendIfLeader(): Unit = {
     val records = (0 until 50).toArray.map(id => new SimpleRecord(id.toString.getBytes))
 
     //Given this partition is on leader epoch 72
@@ -3233,7 +3233,7 @@ class LogTest {
   }
 
   @Test
-  def followerShouldSaveEpochInformationFromReplicatedMessagesToTheEpochCache() {
+  def followerShouldSaveEpochInformationFromReplicatedMessagesToTheEpochCache(): Unit = {
     val messageIds = (0 until 50).toArray
     val records = messageIds.map(id => new SimpleRecord(id.toString.getBytes))
 
@@ -3257,7 +3257,7 @@ class LogTest {
   }
 
   @Test
-  def shouldTruncateLeaderEpochsWhenDeletingSegments() {
+  def shouldTruncateLeaderEpochsWhenDeletingSegments(): Unit = {
     def createRecords = TestUtils.singletonRecords("test".getBytes)
     val logConfig = LogTest.createLogConfig(segmentBytes = createRecords.sizeInBytes * 5, retentionBytes = createRecords.sizeInBytes * 10)
     val log = createLog(logDir, logConfig)
@@ -3282,7 +3282,7 @@ class LogTest {
   }
 
   @Test
-  def shouldUpdateOffsetForLeaderEpochsWhenDeletingSegments() {
+  def shouldUpdateOffsetForLeaderEpochsWhenDeletingSegments(): Unit = {
     def createRecords = TestUtils.singletonRecords("test".getBytes)
     val logConfig = LogTest.createLogConfig(segmentBytes = createRecords.sizeInBytes * 5, retentionBytes = createRecords.sizeInBytes * 10)
     val log = createLog(logDir, logConfig)
@@ -3307,7 +3307,7 @@ class LogTest {
   }
 
   @Test
-  def shouldTruncateLeaderEpochCheckpointFileWhenTruncatingLog() {
+  def shouldTruncateLeaderEpochCheckpointFileWhenTruncatingLog(): Unit = {
     def createRecords(startOffset: Long, epoch: Int): MemoryRecords = {
       TestUtils.records(Seq(new SimpleRecord("value".getBytes)),
         baseOffset = startOffset, partitionLeaderEpoch = epoch)
@@ -3359,7 +3359,7 @@ class LogTest {
    * Append a bunch of messages to a log and then re-open it with recovery and check that the leader epochs are recovered properly.
    */
   @Test
-  def testLogRecoversForLeaderEpoch() {
+  def testLogRecoversForLeaderEpoch(): Unit = {
     val log = createLog(logDir, LogConfig())
     val leaderEpochCache = epochCache(log)
     val firstBatch = singletonRecordsWithLeaderEpoch(value = "random".getBytes, leaderEpoch = 1, offset = 0)
@@ -3410,7 +3410,7 @@ class LogTest {
   }
 
   @Test
-  def testFirstUnstableOffsetNoTransactionalData() {
+  def testFirstUnstableOffsetNoTransactionalData(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 1024 * 1024 * 5)
     val log = createLog(logDir, logConfig)
 
@@ -3424,7 +3424,7 @@ class LogTest {
   }
 
   @Test
-  def testFirstUnstableOffsetWithTransactionalData() {
+  def testFirstUnstableOffsetWithTransactionalData(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 1024 * 1024 * 5)
     val log = createLog(logDir, logConfig)
 
@@ -3898,7 +3898,7 @@ class LogTest {
   }
 
   @Test
-  def testOffsetSnapshot() {
+  def testOffsetSnapshot(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 1024 * 1024 * 5)
     val log = createLog(logDir, logConfig)
 
@@ -3920,7 +3920,7 @@ class LogTest {
   }
 
   @Test
-  def testLastStableOffsetWithMixedProducerData() {
+  def testLastStableOffsetWithMixedProducerData(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 1024 * 1024 * 5)
     val log = createLog(logDir, logConfig)
 
@@ -3972,7 +3972,7 @@ class LogTest {
   }
 
   @Test
-  def testAbortedTransactionSpanningMultipleSegments() {
+  def testAbortedTransactionSpanningMultipleSegments(): Unit = {
     val pid = 137L
     val epoch = 5.toShort
     var seq = 0
@@ -4015,7 +4015,7 @@ class LogTest {
   }
 
   @Test
-  def testLoadPartitionDirWithNoSegmentsShouldNotThrow() {
+  def testLoadPartitionDirWithNoSegmentsShouldNotThrow(): Unit = {
     val dirName = Log.logDeleteDirName(new TopicPartition("foo", 3))
     val logDir = new File(tmpDir, dirName)
     logDir.mkdirs()
diff --git a/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala b/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
index e2a8e17cf07ec..abbb4a4308fc6 100644
--- a/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
@@ -62,13 +62,13 @@ class LogValidatorTest {
     checkMismatchMagic(RecordBatch.MAGIC_VALUE_V1, RecordBatch.MAGIC_VALUE_V0, CompressionType.GZIP)
   }
 
-  private def checkOnlyOneBatch(magic: Byte, sourceCompressionType: CompressionType, targetCompressionType: CompressionType) {
+  private def checkOnlyOneBatch(magic: Byte, sourceCompressionType: CompressionType, targetCompressionType: CompressionType): Unit = {
     assertThrows[InvalidRecordException] {
       validateMessages(createTwoBatchedRecords(magic, 0L, sourceCompressionType), magic, sourceCompressionType, targetCompressionType)
     }
   }
 
-  private def checkAllowMultiBatch(magic: Byte, sourceCompressionType: CompressionType, targetCompressionType: CompressionType) {
+  private def checkAllowMultiBatch(magic: Byte, sourceCompressionType: CompressionType, targetCompressionType: CompressionType): Unit = {
     validateMessages(createTwoBatchedRecords(magic, 0L, sourceCompressionType), magic, sourceCompressionType, targetCompressionType)
   }
 
@@ -96,11 +96,11 @@ class LogValidatorTest {
   }
 
   @Test
-  def testLogAppendTimeNonCompressedV1() {
+  def testLogAppendTimeNonCompressedV1(): Unit = {
     checkLogAppendTimeNonCompressed(RecordBatch.MAGIC_VALUE_V1)
   }
 
-  private def checkLogAppendTimeNonCompressed(magic: Byte) {
+  private def checkLogAppendTimeNonCompressed(magic: Byte): Unit = {
     val now = System.currentTimeMillis()
     // The timestamps should be overwritten
     val records = createRecords(magicValue = magic, timestamp = 1234L, codec = CompressionType.NONE)
@@ -128,16 +128,16 @@ class LogValidatorTest {
       compressed = false)
   }
 
-  def testLogAppendTimeNonCompressedV2() {
+  def testLogAppendTimeNonCompressedV2(): Unit = {
     checkLogAppendTimeNonCompressed(RecordBatch.MAGIC_VALUE_V2)
   }
 
   @Test
-  def testLogAppendTimeWithRecompressionV1() {
+  def testLogAppendTimeWithRecompressionV1(): Unit = {
     checkLogAppendTimeWithRecompression(RecordBatch.MAGIC_VALUE_V1)
   }
 
-  private def checkLogAppendTimeWithRecompression(targetMagic: Byte) {
+  private def checkLogAppendTimeWithRecompression(targetMagic: Byte): Unit = {
     val now = System.currentTimeMillis()
     // The timestamps should be overwritten
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V0, codec = CompressionType.GZIP)
@@ -170,16 +170,16 @@ class LogValidatorTest {
   }
 
   @Test
-  def testLogAppendTimeWithRecompressionV2() {
+  def testLogAppendTimeWithRecompressionV2(): Unit = {
     checkLogAppendTimeWithRecompression(RecordBatch.MAGIC_VALUE_V2)
   }
 
   @Test
-  def testLogAppendTimeWithoutRecompressionV1() {
+  def testLogAppendTimeWithoutRecompressionV1(): Unit = {
     checkLogAppendTimeWithoutRecompression(RecordBatch.MAGIC_VALUE_V1)
   }
 
-  private def checkLogAppendTimeWithoutRecompression(magic: Byte) {
+  private def checkLogAppendTimeWithoutRecompression(magic: Byte): Unit = {
     val now = System.currentTimeMillis()
     // The timestamps should be overwritten
     val records = createRecords(magicValue = magic, timestamp = 1234L, codec = CompressionType.GZIP)
@@ -237,7 +237,7 @@ class LogValidatorTest {
     }
   }
 
-  private def validateRecordBatchWithCountOverrides(lastOffsetDelta: Int, count: Int) {
+  private def validateRecordBatchWithCountOverrides(lastOffsetDelta: Int, count: Int): Unit = {
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V2, timestamp = 1234L, codec = CompressionType.NONE)
     records.buffer.putInt(DefaultRecordBatch.RECORDS_COUNT_OFFSET, count)
     records.buffer.putInt(DefaultRecordBatch.LAST_OFFSET_DELTA_OFFSET, lastOffsetDelta)
@@ -258,16 +258,16 @@ class LogValidatorTest {
   }
 
   @Test
-  def testLogAppendTimeWithoutRecompressionV2() {
+  def testLogAppendTimeWithoutRecompressionV2(): Unit = {
     checkLogAppendTimeWithoutRecompression(RecordBatch.MAGIC_VALUE_V2)
   }
 
   @Test
-  def testNonCompressedV1() {
+  def testNonCompressedV1(): Unit = {
     checkNonCompressed(RecordBatch.MAGIC_VALUE_V1)
   }
 
-  private def checkNonCompressed(magic: Byte) {
+  private def checkNonCompressed(magic: Byte): Unit = {
     val now = System.currentTimeMillis()
     val timestampSeq = Seq(now - 1, now + 1, now)
 
@@ -325,7 +325,7 @@ class LogValidatorTest {
   }
 
   @Test
-  def testNonCompressedV2() {
+  def testNonCompressedV2(): Unit = {
     checkNonCompressed(RecordBatch.MAGIC_VALUE_V2)
   }
 
@@ -400,7 +400,7 @@ class LogValidatorTest {
     checkCreateTimeUpConversionFromV0(RecordBatch.MAGIC_VALUE_V1)
   }
 
-  private def checkCreateTimeUpConversionFromV0(toMagic: Byte) {
+  private def checkCreateTimeUpConversionFromV0(toMagic: Byte): Unit = {
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V0, codec = CompressionType.GZIP)
     val validatedResults = LogValidator.validateMessagesAndAssignOffsets(records,
       offsetCounter = new LongRef(0),
@@ -436,12 +436,12 @@ class LogValidatorTest {
   }
 
   @Test
-  def testCreateTimeUpConversionV0ToV2() {
+  def testCreateTimeUpConversionV0ToV2(): Unit = {
     checkCreateTimeUpConversionFromV0(RecordBatch.MAGIC_VALUE_V2)
   }
 
   @Test
-  def testCreateTimeUpConversionV1ToV2() {
+  def testCreateTimeUpConversionV1ToV2(): Unit = {
     val timestamp = System.currentTimeMillis()
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V1, codec = CompressionType.GZIP, timestamp = timestamp)
     val validatedResults = LogValidator.validateMessagesAndAssignOffsets(records,
@@ -478,11 +478,11 @@ class LogValidatorTest {
   }
 
   @Test
-  def testCompressedV1() {
+  def testCompressedV1(): Unit = {
     checkCompressed(RecordBatch.MAGIC_VALUE_V1)
   }
 
-  private def checkCompressed(magic: Byte) {
+  private def checkCompressed(magic: Byte): Unit = {
     val now = System.currentTimeMillis()
     val timestampSeq = Seq(now - 1, now + 1, now)
 
@@ -540,12 +540,12 @@ class LogValidatorTest {
   }
 
   @Test
-  def testCompressedV2() {
+  def testCompressedV2(): Unit = {
     checkCompressed(RecordBatch.MAGIC_VALUE_V2)
   }
 
   @Test(expected = classOf[InvalidTimestampException])
-  def testInvalidCreateTimeNonCompressedV1() {
+  def testInvalidCreateTimeNonCompressedV1(): Unit = {
     val now = System.currentTimeMillis()
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V1, timestamp = now - 1001L,
       codec = CompressionType.NONE)
@@ -566,7 +566,7 @@ class LogValidatorTest {
   }
 
   @Test(expected = classOf[InvalidTimestampException])
-  def testInvalidCreateTimeNonCompressedV2() {
+  def testInvalidCreateTimeNonCompressedV2(): Unit = {
     val now = System.currentTimeMillis()
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V2, timestamp = now - 1001L,
       codec = CompressionType.NONE)
@@ -587,7 +587,7 @@ class LogValidatorTest {
   }
 
   @Test(expected = classOf[InvalidTimestampException])
-  def testInvalidCreateTimeCompressedV1() {
+  def testInvalidCreateTimeCompressedV1(): Unit = {
     val now = System.currentTimeMillis()
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V1, timestamp = now - 1001L,
       codec = CompressionType.GZIP)
@@ -608,7 +608,7 @@ class LogValidatorTest {
   }
 
   @Test(expected = classOf[InvalidTimestampException])
-  def testInvalidCreateTimeCompressedV2() {
+  def testInvalidCreateTimeCompressedV2(): Unit = {
     val now = System.currentTimeMillis()
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V2, timestamp = now - 1001L,
       codec = CompressionType.GZIP)
@@ -629,7 +629,7 @@ class LogValidatorTest {
   }
 
   @Test
-  def testAbsoluteOffsetAssignmentNonCompressed() {
+  def testAbsoluteOffsetAssignmentNonCompressed(): Unit = {
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V0, codec = CompressionType.NONE)
     val offset = 1234567
     checkOffsets(records, 0)
@@ -649,7 +649,7 @@ class LogValidatorTest {
   }
 
   @Test
-  def testAbsoluteOffsetAssignmentCompressed() {
+  def testAbsoluteOffsetAssignmentCompressed(): Unit = {
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V0, codec = CompressionType.GZIP)
     val offset = 1234567
     checkOffsets(records, 0)
@@ -669,7 +669,7 @@ class LogValidatorTest {
   }
 
   @Test
-  def testRelativeOffsetAssignmentNonCompressedV1() {
+  def testRelativeOffsetAssignmentNonCompressedV1(): Unit = {
     val now = System.currentTimeMillis()
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V1, timestamp = now, codec = CompressionType.NONE)
     val offset = 1234567
@@ -691,7 +691,7 @@ class LogValidatorTest {
   }
 
   @Test
-  def testRelativeOffsetAssignmentNonCompressedV2() {
+  def testRelativeOffsetAssignmentNonCompressedV2(): Unit = {
     val now = System.currentTimeMillis()
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V2, timestamp = now, codec = CompressionType.NONE)
     val offset = 1234567
@@ -713,7 +713,7 @@ class LogValidatorTest {
   }
 
   @Test
-  def testRelativeOffsetAssignmentCompressedV1() {
+  def testRelativeOffsetAssignmentCompressedV1(): Unit = {
     val now = System.currentTimeMillis()
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V1, timestamp = now, codec = CompressionType.GZIP)
     val offset = 1234567
@@ -736,7 +736,7 @@ class LogValidatorTest {
   }
 
   @Test
-  def testRelativeOffsetAssignmentCompressedV2() {
+  def testRelativeOffsetAssignmentCompressedV2(): Unit = {
     val now = System.currentTimeMillis()
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V2, timestamp = now, codec = CompressionType.GZIP)
     val offset = 1234567
@@ -759,7 +759,7 @@ class LogValidatorTest {
   }
 
   @Test
-  def testOffsetAssignmentAfterUpConversionV0ToV1NonCompressed() {
+  def testOffsetAssignmentAfterUpConversionV0ToV1NonCompressed(): Unit = {
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V0, codec = CompressionType.NONE)
     checkOffsets(records, 0)
     val offset = 1234567
@@ -782,7 +782,7 @@ class LogValidatorTest {
   }
 
   @Test
-  def testOffsetAssignmentAfterUpConversionV0ToV2NonCompressed() {
+  def testOffsetAssignmentAfterUpConversionV0ToV2NonCompressed(): Unit = {
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V0, codec = CompressionType.NONE)
     checkOffsets(records, 0)
     val offset = 1234567
@@ -805,7 +805,7 @@ class LogValidatorTest {
   }
 
   @Test
-  def testOffsetAssignmentAfterUpConversionV0ToV1Compressed() {
+  def testOffsetAssignmentAfterUpConversionV0ToV1Compressed(): Unit = {
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V0, codec = CompressionType.GZIP)
     val offset = 1234567
     checkOffsets(records, 0)
@@ -828,7 +828,7 @@ class LogValidatorTest {
   }
 
   @Test
-  def testOffsetAssignmentAfterUpConversionV0ToV2Compressed() {
+  def testOffsetAssignmentAfterUpConversionV0ToV2Compressed(): Unit = {
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V0, codec = CompressionType.GZIP)
     val offset = 1234567
     checkOffsets(records, 0)
@@ -851,7 +851,7 @@ class LogValidatorTest {
   }
 
   @Test(expected = classOf[InvalidRecordException])
-  def testControlRecordsNotAllowedFromClients() {
+  def testControlRecordsNotAllowedFromClients(): Unit = {
     val offset = 1234567
     val endTxnMarker = new EndTransactionMarker(ControlRecordType.COMMIT, 0)
     val records = MemoryRecords.withEndTransactionMarker(23423L, 5, endTxnMarker)
@@ -871,7 +871,7 @@ class LogValidatorTest {
   }
 
   @Test
-  def testControlRecordsNotCompressed() {
+  def testControlRecordsNotCompressed(): Unit = {
     val offset = 1234567
     val endTxnMarker = new EndTransactionMarker(ControlRecordType.COMMIT, 0)
     val records = MemoryRecords.withEndTransactionMarker(23423L, 5, endTxnMarker)
@@ -895,7 +895,7 @@ class LogValidatorTest {
   }
 
   @Test
-  def testOffsetAssignmentAfterDownConversionV1ToV0NonCompressed() {
+  def testOffsetAssignmentAfterDownConversionV1ToV0NonCompressed(): Unit = {
     val offset = 1234567
     val now = System.currentTimeMillis()
     val records = createRecords(RecordBatch.MAGIC_VALUE_V1, now, codec = CompressionType.NONE)
@@ -916,7 +916,7 @@ class LogValidatorTest {
   }
 
   @Test
-  def testOffsetAssignmentAfterDownConversionV1ToV0Compressed() {
+  def testOffsetAssignmentAfterDownConversionV1ToV0Compressed(): Unit = {
     val offset = 1234567
     val now = System.currentTimeMillis()
     val records = createRecords(RecordBatch.MAGIC_VALUE_V1, now, CompressionType.GZIP)
@@ -937,7 +937,7 @@ class LogValidatorTest {
   }
 
   @Test
-  def testOffsetAssignmentAfterUpConversionV1ToV2NonCompressed() {
+  def testOffsetAssignmentAfterUpConversionV1ToV2NonCompressed(): Unit = {
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V1, codec = CompressionType.NONE)
     checkOffsets(records, 0)
     val offset = 1234567
@@ -957,7 +957,7 @@ class LogValidatorTest {
   }
 
   @Test
-  def testOffsetAssignmentAfterUpConversionV1ToV2Compressed() {
+  def testOffsetAssignmentAfterUpConversionV1ToV2Compressed(): Unit = {
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V1, codec = CompressionType.GZIP)
     val offset = 1234567
     checkOffsets(records, 0)
@@ -977,7 +977,7 @@ class LogValidatorTest {
   }
 
   @Test
-  def testOffsetAssignmentAfterDownConversionV2ToV1NonCompressed() {
+  def testOffsetAssignmentAfterDownConversionV2ToV1NonCompressed(): Unit = {
     val offset = 1234567
     val now = System.currentTimeMillis()
     val records = createRecords(RecordBatch.MAGIC_VALUE_V2, now, codec = CompressionType.NONE)
@@ -998,7 +998,7 @@ class LogValidatorTest {
   }
 
   @Test
-  def testOffsetAssignmentAfterDownConversionV2ToV1Compressed() {
+  def testOffsetAssignmentAfterDownConversionV2ToV1Compressed(): Unit = {
     val offset = 1234567
     val now = System.currentTimeMillis()
     val records = createRecords(RecordBatch.MAGIC_VALUE_V2, now, CompressionType.GZIP)
@@ -1019,7 +1019,7 @@ class LogValidatorTest {
   }
 
   @Test(expected = classOf[UnsupportedForMessageFormatException])
-  def testDownConversionOfTransactionalRecordsNotPermitted() {
+  def testDownConversionOfTransactionalRecordsNotPermitted(): Unit = {
     val offset = 1234567
     val producerId = 1344L
     val producerEpoch = 16.toShort
@@ -1042,7 +1042,7 @@ class LogValidatorTest {
   }
 
   @Test(expected = classOf[UnsupportedForMessageFormatException])
-  def testDownConversionOfIdempotentRecordsNotPermitted() {
+  def testDownConversionOfIdempotentRecordsNotPermitted(): Unit = {
     val offset = 1234567
     val producerId = 1344L
     val producerEpoch = 16.toShort
@@ -1065,7 +1065,7 @@ class LogValidatorTest {
   }
 
   @Test
-  def testOffsetAssignmentAfterDownConversionV2ToV0NonCompressed() {
+  def testOffsetAssignmentAfterDownConversionV2ToV0NonCompressed(): Unit = {
     val offset = 1234567
     val now = System.currentTimeMillis()
     val records = createRecords(RecordBatch.MAGIC_VALUE_V2, now, codec = CompressionType.NONE)
@@ -1086,7 +1086,7 @@ class LogValidatorTest {
   }
 
   @Test
-  def testOffsetAssignmentAfterDownConversionV2ToV0Compressed() {
+  def testOffsetAssignmentAfterDownConversionV2ToV0Compressed(): Unit = {
     val offset = 1234567
     val now = System.currentTimeMillis()
     val records = createRecords(RecordBatch.MAGIC_VALUE_V2, now, CompressionType.GZIP)
@@ -1194,7 +1194,7 @@ class LogValidatorTest {
   }
 
   /* check that offsets are assigned consecutively from the given base offset */
-  def checkOffsets(records: MemoryRecords, baseOffset: Long) {
+  def checkOffsets(records: MemoryRecords, baseOffset: Long): Unit = {
     assertTrue("Message set should not be empty", records.records.asScala.nonEmpty)
     var offset = baseOffset
     for (entry <- records.records.asScala) {
@@ -1236,7 +1236,7 @@ class LogValidatorTest {
   /**
     * expectedLogAppendTime is only checked if batch.magic is V2 or higher
     */
-  def validateLogAppendTime(expectedLogAppendTime: Long, expectedBaseTimestamp: Long, batch: RecordBatch) {
+  def validateLogAppendTime(expectedLogAppendTime: Long, expectedBaseTimestamp: Long, batch: RecordBatch): Unit = {
     assertTrue(batch.isValid)
     assertTrue(batch.timestampType == TimestampType.LOG_APPEND_TIME)
     assertEquals(s"Unexpected max timestamp of batch $batch", expectedLogAppendTime, batch.maxTimestamp)
diff --git a/core/src/test/scala/unit/kafka/log/OffsetIndexTest.scala b/core/src/test/scala/unit/kafka/log/OffsetIndexTest.scala
index d8d0484103ff4..b8bc00631be09 100644
--- a/core/src/test/scala/unit/kafka/log/OffsetIndexTest.scala
+++ b/core/src/test/scala/unit/kafka/log/OffsetIndexTest.scala
@@ -38,18 +38,18 @@ class OffsetIndexTest {
   val baseOffset = 45L
   
   @Before
-  def setup() {
+  def setup(): Unit = {
     this.idx = new OffsetIndex(nonExistentTempFile(), baseOffset, maxIndexSize = 30 * 8)
   }
   
   @After
-  def teardown() {
+  def teardown(): Unit = {
     if(this.idx != null)
       this.idx.file.delete()
   }
   
   @Test
-  def randomLookupTest() {
+  def randomLookupTest(): Unit = {
     assertEquals("Not present value should return physical offset 0.", OffsetPosition(idx.baseOffset, 0), idx.lookup(92L))
     
     // append some random values
@@ -77,7 +77,7 @@ class OffsetIndexTest {
   }
   
   @Test
-  def lookupExtremeCases() {
+  def lookupExtremeCases(): Unit = {
     assertEquals("Lookup on empty file", OffsetPosition(idx.baseOffset, 0), idx.lookup(idx.baseOffset))
     for(i <- 0 until idx.maxEntries)
       idx.append(idx.baseOffset + i + 1, i)
@@ -100,7 +100,7 @@ class OffsetIndexTest {
   }
   
   @Test
-  def appendTooMany() {
+  def appendTooMany(): Unit = {
     for(i <- 0 until idx.maxEntries) {
       val offset = idx.baseOffset + i + 1
       idx.append(offset, i)
@@ -109,13 +109,13 @@ class OffsetIndexTest {
   }
   
   @Test(expected = classOf[InvalidOffsetException])
-  def appendOutOfOrder() {
+  def appendOutOfOrder(): Unit = {
     idx.append(51, 0)
     idx.append(50, 1)
   }
 
   @Test
-  def testFetchUpperBoundOffset() {
+  def testFetchUpperBoundOffset(): Unit = {
     val first = OffsetPosition(baseOffset + 0, 0)
     val second = OffsetPosition(baseOffset + 1, 10)
     val third = OffsetPosition(baseOffset + 2, 23)
@@ -137,7 +137,7 @@ class OffsetIndexTest {
   }
 
   @Test
-  def testReopen() {
+  def testReopen(): Unit = {
     val first = OffsetPosition(51, 0)
     val sec = OffsetPosition(52, 1)
     idx.append(first.offset, first.position)
@@ -152,7 +152,7 @@ class OffsetIndexTest {
   }
   
   @Test
-  def truncate() {
+  def truncate(): Unit = {
 	val idx = new OffsetIndex(nonExistentTempFile(), baseOffset = 0L, maxIndexSize = 10 * 8)
 	idx.truncate()
     for(i <- 1 until 10)
@@ -201,7 +201,7 @@ class OffsetIndexTest {
     idx.sanityCheck()
   }
   
-  def assertWriteFails[T](message: String, idx: OffsetIndex, offset: Int, klass: Class[T]) {
+  def assertWriteFails[T](message: String, idx: OffsetIndex, offset: Int, klass: Class[T]): Unit = {
     try {
       idx.append(offset, 1)
       fail(message)
diff --git a/core/src/test/scala/unit/kafka/log/OffsetMapTest.scala b/core/src/test/scala/unit/kafka/log/OffsetMapTest.scala
index e01bc7cf679ae..5fa9389442d6a 100644
--- a/core/src/test/scala/unit/kafka/log/OffsetMapTest.scala
+++ b/core/src/test/scala/unit/kafka/log/OffsetMapTest.scala
@@ -26,7 +26,7 @@ import org.junit.Assert._
 class OffsetMapTest {
   
   @Test
-  def testBasicValidation() {
+  def testBasicValidation(): Unit = {
     validateMap(10)
     validateMap(100)
     validateMap(1000)
@@ -34,7 +34,7 @@ class OffsetMapTest {
   }
   
   @Test
-  def testClear() {
+  def testClear(): Unit = {
     val map = new SkimpyOffsetMap(4000)
     for(i <- 0 until 10)
       map.put(key(i), i)
@@ -46,7 +46,7 @@ class OffsetMapTest {
   }
   
   @Test
-  def testGetWhenFull() {
+  def testGetWhenFull(): Unit = {
     val map = new SkimpyOffsetMap(4096)
     var i = 37L  //any value would do
     while (map.size < map.slots) {
@@ -71,7 +71,7 @@ class OffsetMapTest {
 }
 
 object OffsetMapTest {
-  def main(args: Array[String]) {
+  def main(args: Array[String]): Unit = {
     if(args.length != 2) {
       System.err.println("USAGE: java OffsetMapTest size load")
       Exit.exit(1)
diff --git a/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala b/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala
index 41a68aa8e5332..e98e59e2e271e 100644
--- a/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala
@@ -659,7 +659,7 @@ class ProducerStateManagerTest {
   }
 
   @Test(expected = classOf[UnknownProducerIdException])
-  def testPidExpirationTimeout() {
+  def testPidExpirationTimeout(): Unit = {
     val epoch = 5.toShort
     val sequence = 37
     append(stateManager, producerId, epoch, sequence, 1L)
@@ -669,7 +669,7 @@ class ProducerStateManagerTest {
   }
 
   @Test
-  def testFirstUnstableOffset() {
+  def testFirstUnstableOffset(): Unit = {
     val epoch = 5.toShort
     val sequence = 0
 
@@ -703,7 +703,7 @@ class ProducerStateManagerTest {
   }
 
   @Test
-  def testProducersWithOngoingTransactionsDontExpire() {
+  def testProducersWithOngoingTransactionsDontExpire(): Unit = {
     val epoch = 5.toShort
     val sequence = 0
 
diff --git a/core/src/test/scala/unit/kafka/log/TimeIndexTest.scala b/core/src/test/scala/unit/kafka/log/TimeIndexTest.scala
index 9843012a4f264..bfe8f56882f15 100644
--- a/core/src/test/scala/unit/kafka/log/TimeIndexTest.scala
+++ b/core/src/test/scala/unit/kafka/log/TimeIndexTest.scala
@@ -34,18 +34,18 @@ class TimeIndexTest {
   val baseOffset = 45L
 
   @Before
-  def setup() {
+  def setup(): Unit = {
     this.idx = new TimeIndex(nonExistantTempFile(), baseOffset = baseOffset, maxIndexSize = maxEntries * 12)
   }
 
   @After
-  def teardown() {
+  def teardown(): Unit = {
     if(this.idx != null)
       this.idx.file.delete()
   }
 
   @Test
-  def testLookUp() {
+  def testLookUp(): Unit = {
     // Empty time index
     assertEquals(TimestampOffset(-1L, baseOffset), idx.lookup(100L))
 
@@ -75,7 +75,7 @@ class TimeIndexTest {
   }
 
   @Test
-  def testTruncate() {
+  def testTruncate(): Unit = {
     appendEntries(maxEntries - 1)
     idx.truncate()
     assertEquals(0, idx.entries)
@@ -86,7 +86,7 @@ class TimeIndexTest {
   }
 
   @Test
-  def testAppend() {
+  def testAppend(): Unit = {
     appendEntries(maxEntries - 1)
     intercept[IllegalArgumentException] {
       idx.maybeAppend(10000L, 1000L)
@@ -97,7 +97,7 @@ class TimeIndexTest {
     idx.maybeAppend(10000L, 1000L, true)
   }
 
-  private def appendEntries(numEntries: Int) {
+  private def appendEntries(numEntries: Int): Unit = {
     for (i <- 1 to numEntries)
       idx.maybeAppend(i * 10, i * 10 + baseOffset)
   }
diff --git a/core/src/test/scala/unit/kafka/metrics/KafkaTimerTest.scala b/core/src/test/scala/unit/kafka/metrics/KafkaTimerTest.scala
index 3b3e4c39e491f..4ffa68aa07b1a 100644
--- a/core/src/test/scala/unit/kafka/metrics/KafkaTimerTest.scala
+++ b/core/src/test/scala/unit/kafka/metrics/KafkaTimerTest.scala
@@ -25,7 +25,7 @@ import com.yammer.metrics.core.{MetricsRegistry, Clock}
 class KafkaTimerTest {
 
   @Test
-  def testKafkaTimer() {
+  def testKafkaTimer(): Unit = {
     val clock = new ManualClock
     val testRegistry = new MetricsRegistry(clock)
     val metric = testRegistry.newTimer(this.getClass, "TestTimer")
@@ -52,7 +52,7 @@ class KafkaTimerTest {
       TimeUnit.NANOSECONDS.toMillis(ticksInNanos)
     }
 
-    def addMillis(millis: Long) {
+    def addMillis(millis: Long): Unit = {
       ticksInNanos += TimeUnit.MILLISECONDS.toNanos(millis)
     }
   }
diff --git a/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala b/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
index 27e5dcd394a98..112a2451427ad 100644
--- a/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
+++ b/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
@@ -46,7 +46,7 @@ class MetricsTest extends KafkaServerTestHarness with Logging {
   val nMessages = 2
 
   @Test
-  def testMetricsReporterAfterDeletingTopic() {
+  def testMetricsReporterAfterDeletingTopic(): Unit = {
     val topic = "test-topic-metric"
     createTopic(topic, 1, 1)
     adminZkClient.deleteTopic(topic)
@@ -55,7 +55,7 @@ class MetricsTest extends KafkaServerTestHarness with Logging {
   }
 
   @Test
-  def testBrokerTopicMetricsUnregisteredAfterDeletingTopic() {
+  def testBrokerTopicMetricsUnregisteredAfterDeletingTopic(): Unit = {
     val topic = "test-broker-topic-metric"
     createTopic(topic, 2, 1)
     // Produce a few messages to create the metrics
diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
index 82b8bf98af3ba..d235c62a09629 100644
--- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
+++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
@@ -85,14 +85,14 @@ class SocketServerTest {
   }
 
   @After
-  def tearDown() {
+  def tearDown(): Unit = {
     shutdownServerAndMetrics(server)
     sockets.foreach(_.close())
     sockets.clear()
     kafkaLogger.setLevel(logLevelToRestore)
   }
 
-  def sendRequest(socket: Socket, request: Array[Byte], id: Option[Short] = None, flush: Boolean = true) {
+  def sendRequest(socket: Socket, request: Array[Byte], id: Option[Short] = None, flush: Boolean = true): Unit = {
     val outgoing = new DataOutputStream(socket.getOutputStream)
     id match {
       case Some(id) =>
@@ -123,11 +123,11 @@ class SocketServerTest {
   }
 
   /* A simple request handler that just echos back the response */
-  def processRequest(channel: RequestChannel) {
+  def processRequest(channel: RequestChannel): Unit = {
     processRequest(channel, receiveRequest(channel))
   }
 
-  def processRequest(channel: RequestChannel, request: RequestChannel.Request) {
+  def processRequest(channel: RequestChannel, request: RequestChannel.Request): Unit = {
     val byteBuffer = request.body[AbstractRequest].serialize(request.header)
     byteBuffer.rewind()
 
@@ -179,7 +179,7 @@ class SocketServerTest {
   }
 
   @Test
-  def simpleRequest() {
+  def simpleRequest(): Unit = {
     val plainSocket = connect()
     val serializedBytes = producerRequestBytes()
 
@@ -206,7 +206,7 @@ class SocketServerTest {
   }
 
   @Test
-  def tooBigRequestIsRejected() {
+  def tooBigRequestIsRejected(): Unit = {
     val tooManyBytes = new Array[Byte](server.config.socketRequestMaxBytes + 1)
     new Random().nextBytes(tooManyBytes)
     val socket = connect()
@@ -224,7 +224,7 @@ class SocketServerTest {
   }
 
   @Test
-  def testGracefulClose() {
+  def testGracefulClose(): Unit = {
     val plainSocket = connect()
     val serializedBytes = producerRequestBytes()
 
@@ -253,7 +253,7 @@ class SocketServerTest {
   }
 
   @Test
-  def testConnectionId() {
+  def testConnectionId(): Unit = {
     val sockets = (1 to 5).map(_ => connect())
     val serializedBytes = producerRequestBytes()
 
@@ -270,7 +270,7 @@ class SocketServerTest {
   }
 
   @Test
-  def testIdleConnection() {
+  def testIdleConnection(): Unit = {
     val idleTimeMs = 60000
     val time = new MockTime()
     props.put(KafkaConfig.ConnectionsMaxIdleMsProp, idleTimeMs.toString)
@@ -315,7 +315,7 @@ class SocketServerTest {
   }
 
   @Test
-  def testConnectionIdReuse() {
+  def testConnectionIdReuse(): Unit = {
     val idleTimeMs = 60000
     val time = new MockTime()
     props.put(KafkaConfig.ConnectionsMaxIdleMsProp, idleTimeMs.toString)
@@ -449,7 +449,7 @@ class SocketServerTest {
     server.dataPlaneProcessor(0).openOrClosingChannel(request.context.connectionId)
 
   @Test
-  def testSendActionResponseWithThrottledChannelWhereThrottlingInProgress() {
+  def testSendActionResponseWithThrottledChannelWhereThrottlingInProgress(): Unit = {
     val socket = connect()
     val serializedBytes = producerRequestBytes()
     // SendAction with throttling in progress
@@ -463,7 +463,7 @@ class SocketServerTest {
   }
 
   @Test
-  def testSendActionResponseWithThrottledChannelWhereThrottlingAlreadyDone() {
+  def testSendActionResponseWithThrottledChannelWhereThrottlingAlreadyDone(): Unit = {
     val socket = connect()
     val serializedBytes = producerRequestBytes()
     // SendAction with throttling in progress
@@ -478,7 +478,7 @@ class SocketServerTest {
   }
 
   @Test
-  def testNoOpActionResponseWithThrottledChannelWhereThrottlingInProgress() {
+  def testNoOpActionResponseWithThrottledChannelWhereThrottlingInProgress(): Unit = {
     val socket = connect()
     val serializedBytes = producerRequestBytes()
     // SendAction with throttling in progress
@@ -490,7 +490,7 @@ class SocketServerTest {
   }
 
   @Test
-  def testNoOpActionResponseWithThrottledChannelWhereThrottlingAlreadyDone() {
+  def testNoOpActionResponseWithThrottledChannelWhereThrottlingAlreadyDone(): Unit = {
     val socket = connect()
     val serializedBytes = producerRequestBytes()
     // SendAction with throttling in progress
@@ -503,7 +503,7 @@ class SocketServerTest {
   }
 
   @Test
-  def testSocketsCloseOnShutdown() {
+  def testSocketsCloseOnShutdown(): Unit = {
     // open a connection
     val plainSocket = connect()
     plainSocket.setTcpNoDelay(true)
@@ -530,7 +530,7 @@ class SocketServerTest {
   }
 
   @Test
-  def testMaxConnectionsPerIp() {
+  def testMaxConnectionsPerIp(): Unit = {
     // make the maximum allowable number of connections
     val conns = (0 until server.config.maxConnectionsPerIp).map(_ => connect())
     // now try one more (should fail)
@@ -552,7 +552,7 @@ class SocketServerTest {
   }
 
   @Test
-  def testZeroMaxConnectionsPerIp() {
+  def testZeroMaxConnectionsPerIp(): Unit = {
     val newProps = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 0)
     newProps.setProperty(KafkaConfig.MaxConnectionsPerIpProp, "0")
     newProps.setProperty(KafkaConfig.MaxConnectionsPerIpOverridesProp, "%s:%s".format("127.0.0.1", "5"))
@@ -589,7 +589,7 @@ class SocketServerTest {
   }
 
   @Test
-  def testMaxConnectionsPerIpOverrides() {
+  def testMaxConnectionsPerIpOverrides(): Unit = {
     val overrideNum = server.config.maxConnectionsPerIp + 1
     val overrideProps = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 0)
     overrideProps.put(KafkaConfig.MaxConnectionsPerIpOverridesProp, s"localhost:$overrideNum")
@@ -616,7 +616,7 @@ class SocketServerTest {
   }
 
   @Test
-  def testSslSocketServer() {
+  def testSslSocketServer(): Unit = {
     val trustStoreFile = File.createTempFile("truststore", ".jks")
     val overrideProps = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, interBrokerSecurityProtocol = Some(SecurityProtocol.SSL),
       trustStoreFile = Some(trustStoreFile))
@@ -656,7 +656,7 @@ class SocketServerTest {
   }
 
   @Test
-  def testSessionPrincipal() {
+  def testSessionPrincipal(): Unit = {
     val socket = connect()
     val bytes = new Array[Byte](40)
     sendRequest(socket, bytes, Some(0))
@@ -665,7 +665,7 @@ class SocketServerTest {
 
   /* Test that we update request metrics if the client closes the connection while the broker response is in flight. */
   @Test
-  def testClientDisconnectionUpdatesRequestMetrics() {
+  def testClientDisconnectionUpdatesRequestMetrics(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 0)
     val serverMetrics = new Metrics
     var conn: Socket = null
@@ -675,7 +675,7 @@ class SocketServerTest {
         new Processor(id, time, config.socketRequestMaxBytes, dataPlaneRequestChannel, connectionQuotas,
           config.connectionsMaxIdleMs, config.failedAuthenticationDelayMs, listenerName, protocol, config, metrics,
           credentialProvider, MemoryPool.NONE, new LogContext()) {
-          override protected[network] def sendResponse(response: RequestChannel.Response, responseSend: Send) {
+          override protected[network] def sendResponse(response: RequestChannel.Response, responseSend: Send): Unit = {
             conn.close()
             super.sendResponse(response, responseSend)
           }
@@ -711,7 +711,7 @@ class SocketServerTest {
   }
 
   @Test
-  def testClientDisconnectionWithStagedReceivesFullyProcessed() {
+  def testClientDisconnectionWithStagedReceivesFullyProcessed(): Unit = {
     val serverMetrics = new Metrics
     @volatile var selector: TestableSelector = null
     val overrideConnectionId = "127.0.0.1:1-127.0.0.1:2-0"
@@ -764,7 +764,7 @@ class SocketServerTest {
    * `selector.send` (selector closes old connections, for example).
    */
   @Test
-  def testBrokerSendAfterChannelClosedUpdatesRequestMetrics() {
+  def testBrokerSendAfterChannelClosedUpdatesRequestMetrics(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 0)
     props.setProperty(KafkaConfig.ConnectionsMaxIdleMsProp, "110")
     val serverMetrics = new Metrics
@@ -1288,7 +1288,7 @@ class SocketServerTest {
     @volatile var pollTimeoutOverride: Option[Long] = None
     @volatile var pollCallback: () => Unit = () => {}
 
-    def addFailure(operation: SelectorOperation, exception: Option[Throwable] = None) {
+    def addFailure(operation: SelectorOperation, exception: Option[Throwable] = None): Unit = {
       failures += operation ->
         exception.getOrElse(new IllegalStateException(s"Test exception during $operation"))
     }
diff --git a/core/src/test/scala/unit/kafka/security/auth/SimpleAclAuthorizerTest.scala b/core/src/test/scala/unit/kafka/security/auth/SimpleAclAuthorizerTest.scala
index 1468003f3f011..57d701ff49704 100644
--- a/core/src/test/scala/unit/kafka/security/auth/SimpleAclAuthorizerTest.scala
+++ b/core/src/test/scala/unit/kafka/security/auth/SimpleAclAuthorizerTest.scala
@@ -60,7 +60,7 @@ class SimpleAclAuthorizerTest extends ZooKeeperTestHarness {
   }
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
 
     // Increase maxUpdateRetries to avoid transient failures
@@ -88,7 +88,7 @@ class SimpleAclAuthorizerTest extends ZooKeeperTestHarness {
   }
 
   @Test(expected = classOf[IllegalArgumentException])
-  def testAuthorizeThrowsOnNoneLiteralResource() {
+  def testAuthorizeThrowsOnNoneLiteralResource(): Unit = {
     simpleAclAuthorizer.authorize(session, Read, Resource(Topic, "something", PREFIXED))
   }
 
@@ -106,7 +106,7 @@ class SimpleAclAuthorizerTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testTopicAcl() {
+  def testTopicAcl(): Unit = {
     val user1 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username)
     val user2 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "rob")
     val user3 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "batman")
@@ -161,7 +161,7 @@ class SimpleAclAuthorizerTest extends ZooKeeperTestHarness {
     CustomPrincipals should be compared with their principal type and name
    */
   @Test
-  def testAllowAccessWithCustomPrincipal() {
+  def testAllowAccessWithCustomPrincipal(): Unit = {
     val user = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username)
     val customUserPrincipal = new CustomPrincipal(KafkaPrincipal.USER_TYPE, username)
     val host1 = InetAddress.getByName("192.168.1.1")
@@ -181,7 +181,7 @@ class SimpleAclAuthorizerTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testDenyTakesPrecedence() {
+  def testDenyTakesPrecedence(): Unit = {
     val user = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username)
     val host = InetAddress.getByName("192.168.2.1")
     val session = Session(user, host)
@@ -196,7 +196,7 @@ class SimpleAclAuthorizerTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testAllowAllAccess() {
+  def testAllowAllAccess(): Unit = {
     val allowAllAcl = Acl.AllowAllAcl
 
     changeAclAndVerify(Set.empty[Acl], Set[Acl](allowAllAcl), Set.empty[Acl])
@@ -206,7 +206,7 @@ class SimpleAclAuthorizerTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testSuperUserHasAccess() {
+  def testSuperUserHasAccess(): Unit = {
     val denyAllAcl = new Acl(Acl.WildCardPrincipal, Deny, WildCardHost, All)
 
     changeAclAndVerify(Set.empty[Acl], Set[Acl](denyAllAcl), Set.empty[Acl])
@@ -256,12 +256,12 @@ class SimpleAclAuthorizerTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testNoAclFound() {
+  def testNoAclFound(): Unit = {
     assertFalse("when acls = [],  authorizer should fail close.", simpleAclAuthorizer.authorize(session, Read, resource))
   }
 
   @Test
-  def testNoAclFoundOverride() {
+  def testNoAclFoundOverride(): Unit = {
     val props = TestUtils.createBrokerConfig(1, zkConnect)
     props.put(SimpleAclAuthorizer.AllowEveryoneIfNoAclIsFoundProp, "true")
 
@@ -276,7 +276,7 @@ class SimpleAclAuthorizerTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testAclManagementAPIs() {
+  def testAclManagementAPIs(): Unit = {
     val user1 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username)
     val user2 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "bob")
     val host1 = "host1"
@@ -322,7 +322,7 @@ class SimpleAclAuthorizerTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testLoadCache() {
+  def testLoadCache(): Unit = {
     val user1 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username)
     val acl1 = new Acl(user1, Allow, "host-1", Read)
     val acls = Set[Acl](acl1)
@@ -352,7 +352,7 @@ class SimpleAclAuthorizerTest extends ZooKeeperTestHarness {
    * in the authorizer to avoid the timing window.
    */
   @Test
-  def testChangeListenerTiming() {
+  def testChangeListenerTiming(): Unit = {
     val configureSemaphore = new Semaphore(0)
     val listenerSemaphore = new Semaphore(0)
     val executor = Executors.newSingleThreadExecutor
@@ -381,7 +381,7 @@ class SimpleAclAuthorizerTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testLocalConcurrentModificationOfResourceAcls() {
+  def testLocalConcurrentModificationOfResourceAcls(): Unit = {
     val commonResource = Resource(Topic, "test", LITERAL)
 
     val user1 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username)
@@ -397,7 +397,7 @@ class SimpleAclAuthorizerTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testDistributedConcurrentModificationOfResourceAcls() {
+  def testDistributedConcurrentModificationOfResourceAcls(): Unit = {
     val commonResource = Resource(Topic, "test", LITERAL)
 
     val user1 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username)
@@ -427,7 +427,7 @@ class SimpleAclAuthorizerTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testHighConcurrencyModificationOfResourceAcls() {
+  def testHighConcurrencyModificationOfResourceAcls(): Unit = {
     val commonResource = Resource(Topic, "test", LITERAL)
 
     val acls = (0 to 50).map { i =>
@@ -512,7 +512,7 @@ class SimpleAclAuthorizerTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testHighConcurrencyDeletionOfResourceAcls() {
+  def testHighConcurrencyDeletionOfResourceAcls(): Unit = {
     val acl = new Acl(new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username), Allow, WildCardHost, All)
 
     // Alternate authorizer to keep adding and removing ZooKeeper path
@@ -721,7 +721,7 @@ class SimpleAclAuthorizerTest extends ZooKeeperTestHarness {
     assertEquals(expected, actual)
   }
 
-  private def givenAuthorizerWithProtocolVersion(protocolVersion: Option[ApiVersion]) {
+  private def givenAuthorizerWithProtocolVersion(protocolVersion: Option[ApiVersion]): Unit = {
     simpleAclAuthorizer.close()
 
     val props = TestUtils.createBrokerConfig(0, zkConnect)
diff --git a/core/src/test/scala/unit/kafka/security/auth/ZkAuthorizationTest.scala b/core/src/test/scala/unit/kafka/security/auth/ZkAuthorizationTest.scala
index cc845553181d1..429e1899a67cb 100644
--- a/core/src/test/scala/unit/kafka/security/auth/ZkAuthorizationTest.scala
+++ b/core/src/test/scala/unit/kafka/security/auth/ZkAuthorizationTest.scala
@@ -44,7 +44,7 @@ class ZkAuthorizationTest extends ZooKeeperTestHarness with Logging {
   val authProvider = "zookeeper.authProvider.1"
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     System.setProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM, jaasFile.getAbsolutePath)
     Configuration.setConfiguration(null)
     System.setProperty(authProvider, "org.apache.zookeeper.server.auth.SASLAuthenticationProvider")
@@ -52,7 +52,7 @@ class ZkAuthorizationTest extends ZooKeeperTestHarness with Logging {
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     super.tearDown()
     System.clearProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM)
     System.clearProperty(authProvider)
diff --git a/core/src/test/scala/unit/kafka/security/token/delegation/DelegationTokenManagerTest.scala b/core/src/test/scala/unit/kafka/security/token/delegation/DelegationTokenManagerTest.scala
index ed82f5ed4464c..30e9e6b537752 100644
--- a/core/src/test/scala/unit/kafka/security/token/delegation/DelegationTokenManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/security/token/delegation/DelegationTokenManagerTest.scala
@@ -58,7 +58,7 @@ class DelegationTokenManagerTest extends ZooKeeperTestHarness  {
   var expiryTimeStamp: Long = 0
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
     props = TestUtils.createBrokerConfig(0, zkConnect, enableToken = true)
     props.put(KafkaConfig.SaslEnabledMechanismsProp, ScramMechanism.mechanismNames().asScala.mkString(","))
diff --git a/core/src/test/scala/unit/kafka/server/ApiVersionsRequestTest.scala b/core/src/test/scala/unit/kafka/server/ApiVersionsRequestTest.scala
index ab70aec3791d6..11ecfdef5a3a0 100644
--- a/core/src/test/scala/unit/kafka/server/ApiVersionsRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ApiVersionsRequestTest.scala
@@ -26,7 +26,7 @@ import org.junit.Test
 import scala.collection.JavaConverters._
 
 object ApiVersionsRequestTest {
-  def validateApiVersionsResponse(apiVersionsResponse: ApiVersionsResponse) {
+  def validateApiVersionsResponse(apiVersionsResponse: ApiVersionsResponse): Unit = {
     assertEquals("API keys in ApiVersionsResponse must match API keys supported by broker.", ApiKeys.values.length, apiVersionsResponse.apiVersions.size)
     for (expectedApiVersion: ApiVersion <- ApiVersionsResponse.defaultApiVersionsResponse().apiVersions.asScala) {
       val actualApiVersion = apiVersionsResponse.apiVersion(expectedApiVersion.apiKey)
@@ -43,13 +43,13 @@ class ApiVersionsRequestTest extends BaseRequestTest {
   override def brokerCount: Int = 1
 
   @Test
-  def testApiVersionsRequest() {
+  def testApiVersionsRequest(): Unit = {
     val apiVersionsResponse = sendApiVersionsRequest(new ApiVersionsRequest.Builder().build())
     ApiVersionsRequestTest.validateApiVersionsResponse(apiVersionsResponse)
   }
 
   @Test
-  def testApiVersionsRequestWithUnsupportedVersion() {
+  def testApiVersionsRequestWithUnsupportedVersion(): Unit = {
     val apiVersionsRequest = new ApiVersionsRequest(0)
     val apiVersionsResponse = sendApiVersionsRequest(apiVersionsRequest, Some(Short.MaxValue), 0)
     assertEquals(Errors.UNSUPPORTED_VERSION, apiVersionsResponse.error)
diff --git a/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala b/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala
index 1e94e3f42312d..faeb34df132a0 100644
--- a/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala
@@ -39,7 +39,7 @@ abstract class BaseRequestTest extends IntegrationTestHarness {
   override def brokerCount: Int = 3
 
   // If required, override properties by mutating the passed Properties object
-  protected def brokerPropertyOverrides(properties: Properties) {}
+  protected def brokerPropertyOverrides(properties: Properties): Unit = {}
 
   override def modifyConfigs(props: Seq[Properties]): Unit = {
     props.foreach { p =>
@@ -77,7 +77,7 @@ abstract class BaseRequestTest extends IntegrationTestHarness {
     new Socket("localhost", s.boundPort(ListenerName.forSecurityProtocol(protocol)))
   }
 
-  private def sendRequest(socket: Socket, request: Array[Byte]) {
+  private def sendRequest(socket: Socket, request: Array[Byte]): Unit = {
     val outgoing = new DataOutputStream(socket.getOutputStream)
     outgoing.writeInt(request.length)
     outgoing.write(request)
diff --git a/core/src/test/scala/unit/kafka/server/BrokerEpochIntegrationTest.scala b/core/src/test/scala/unit/kafka/server/BrokerEpochIntegrationTest.scala
index 6c74ce36546f2..d80b5268ac5c8 100755
--- a/core/src/test/scala/unit/kafka/server/BrokerEpochIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/server/BrokerEpochIntegrationTest.scala
@@ -43,7 +43,7 @@ class BrokerEpochIntegrationTest extends ZooKeeperTestHarness {
   var servers: Seq[KafkaServer] = Seq.empty[KafkaServer]
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
     val configs = Seq(
       TestUtils.createBrokerConfig(brokerId1, zkConnect),
@@ -57,7 +57,7 @@ class BrokerEpochIntegrationTest extends ZooKeeperTestHarness {
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     TestUtils.shutdownServers(servers)
     super.tearDown()
   }
@@ -92,16 +92,16 @@ class BrokerEpochIntegrationTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testControlRequestWithCorrectBrokerEpoch() {
+  def testControlRequestWithCorrectBrokerEpoch(): Unit = {
     testControlRequestWithBrokerEpoch(false)
   }
 
   @Test
-  def testControlRequestWithStaleBrokerEpoch() {
+  def testControlRequestWithStaleBrokerEpoch(): Unit = {
     testControlRequestWithBrokerEpoch(true)
   }
 
-  private def testControlRequestWithBrokerEpoch(isEpochInRequestStale: Boolean) {
+  private def testControlRequestWithBrokerEpoch(isEpochInRequestStale: Boolean): Unit = {
     val tp = new TopicPartition("new-topic", 0)
 
     // create topic with 1 partition, 2 replicas, one on each broker
diff --git a/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala
index d52a8a6b18faa..828fb4841a271 100644
--- a/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala
@@ -47,7 +47,7 @@ class ClientQuotaManagerTest {
     metrics.close()
   }
 
-  def callback (response: RequestChannel.Response) {
+  def callback (response: RequestChannel.Response): Unit = {
     // Count how many times this callback is called for notifyThrottlingDone().
     response match {
       case _: StartThrottlingResponse =>
@@ -76,12 +76,12 @@ class ClientQuotaManagerTest {
   }
 
   private def throttle(quotaManager: ClientQuotaManager, user: String, clientId: String, throttleTimeMs: Int,
-                       channelThrottlingCallback: (RequestChannel.Response) => Unit) {
+                       channelThrottlingCallback: (RequestChannel.Response) => Unit): Unit = {
     val (_, request) = buildRequest(FetchRequest.Builder.forConsumer(0, 1000, new util.HashMap[TopicPartition, PartitionData]))
     quotaManager.throttle(request, throttleTimeMs, channelThrottlingCallback)
   }
 
-  private def testQuotaParsing(config: ClientQuotaManagerConfig, client1: UserClient, client2: UserClient, randomClient: UserClient, defaultConfigClient: UserClient) {
+  private def testQuotaParsing(config: ClientQuotaManagerConfig, client1: UserClient, client2: UserClient, randomClient: UserClient, defaultConfigClient: UserClient): Unit = {
     val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, "")
 
     try {
@@ -131,7 +131,7 @@ class ClientQuotaManagerTest {
    * Quota overrides persisted in ZooKeeper in /config/clients/, default persisted in /config/clients/
    */
   @Test
-  def testClientIdQuotaParsing() {
+  def testClientIdQuotaParsing(): Unit = {
     val client1 = UserClient("ANONYMOUS", "p1", None, Some("p1"))
     val client2 = UserClient("ANONYMOUS", "p2", None, Some("p2"))
     val randomClient = UserClient("ANONYMOUS", "random-client-id", None, None)
@@ -144,7 +144,7 @@ class ClientQuotaManagerTest {
    * Quota overrides persisted in ZooKeeper in /config/users/, default persisted in /config/users/
    */
   @Test
-  def testUserQuotaParsing() {
+  def testUserQuotaParsing(): Unit = {
     val client1 = UserClient("User1", "p1", Some("User1"), None)
     val client2 = UserClient("User2", "p2", Some("User2"), None)
     val randomClient = UserClient("RandomUser", "random-client-id", None, None)
@@ -158,7 +158,7 @@ class ClientQuotaManagerTest {
    * Quotas persisted in ZooKeeper in /config/users//clients/, default in /config/users//clients/
    */
   @Test
-  def testUserClientIdQuotaParsing() {
+  def testUserClientIdQuotaParsing(): Unit = {
     val client1 = UserClient("User1", "p1", Some("User1"), Some("p1"))
     val client2 = UserClient("User2", "p2", Some("User2"), Some("p2"))
     val randomClient = UserClient("RandomUser", "random-client-id", None, None)
@@ -171,7 +171,7 @@ class ClientQuotaManagerTest {
    * Tests parsing for  quotas when client-id default quota properties are set.
    */
   @Test
-  def testUserQuotaParsingWithDefaultClientIdQuota() {
+  def testUserQuotaParsingWithDefaultClientIdQuota(): Unit = {
     val client1 = UserClient("User1", "p1", Some("User1"), None)
     val client2 = UserClient("User2", "p2", Some("User2"), None)
     val randomClient = UserClient("RandomUser", "random-client-id", None, None)
@@ -183,7 +183,7 @@ class ClientQuotaManagerTest {
    * Tests parsing for  quotas when client-id default quota properties are set.
    */
   @Test
-  def testUserClientQuotaParsingIdWithDefaultClientIdQuota() {
+  def testUserClientQuotaParsingIdWithDefaultClientIdQuota(): Unit = {
     val client1 = UserClient("User1", "p1", Some("User1"), Some("p1"))
     val client2 = UserClient("User2", "p2", Some("User2"), Some("p2"))
     val randomClient = UserClient("RandomUser", "random-client-id", None, None)
@@ -192,11 +192,11 @@ class ClientQuotaManagerTest {
   }
 
   @Test
-  def testQuotaConfigPrecedence() {
+  def testQuotaConfigPrecedence(): Unit = {
     val quotaManager = new ClientQuotaManager(ClientQuotaManagerConfig(quotaBytesPerSecondDefault=Long.MaxValue),
       metrics, Produce, time, "")
 
-    def checkQuota(user: String, clientId: String, expectedBound: Int, value: Int, expectThrottle: Boolean) {
+    def checkQuota(user: String, clientId: String, expectedBound: Int, value: Int, expectThrottle: Boolean): Unit = {
       assertEquals(expectedBound, quotaManager.quota(user, clientId).bound, 0.0)
       val throttleTimeMs = maybeRecord(quotaManager, user, clientId, value * config.numQuotaSamples)
       if (expectThrottle)
@@ -265,7 +265,7 @@ class ClientQuotaManagerTest {
   }
 
   @Test
-  def testQuotaViolation() {
+  def testQuotaViolation(): Unit = {
     val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, "")
     val queueSizeMetric = metrics.metrics().get(metrics.metricName("queue-size", "Produce", ""))
     try {
@@ -312,7 +312,7 @@ class ClientQuotaManagerTest {
   }
 
   @Test
-  def testRequestPercentageQuotaViolation() {
+  def testRequestPercentageQuotaViolation(): Unit = {
     val quotaManager = new ClientRequestQuotaManager(config, metrics, time, "", None)
     quotaManager.updateQuota(Some("ANONYMOUS"), Some("test-client"), Some("test-client"), Some(Quota.upperBound(1)))
     val queueSizeMetric = metrics.metrics().get(metrics.metricName("queue-size", "Request", ""))
@@ -374,7 +374,7 @@ class ClientQuotaManagerTest {
   }
 
   @Test
-  def testExpireThrottleTimeSensor() {
+  def testExpireThrottleTimeSensor(): Unit = {
     val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, "")
     try {
       maybeRecord(clientMetrics, "ANONYMOUS", "client1", 100)
@@ -393,7 +393,7 @@ class ClientQuotaManagerTest {
   }
 
   @Test
-  def testExpireQuotaSensors() {
+  def testExpireQuotaSensors(): Unit = {
     val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, "")
     try {
       maybeRecord(clientMetrics, "ANONYMOUS", "client1", 100)
@@ -416,7 +416,7 @@ class ClientQuotaManagerTest {
   }
 
   @Test
-  def testClientIdNotSanitized() {
+  def testClientIdNotSanitized(): Unit = {
     val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, "")
     val clientId = "client@#$%"
     try {
diff --git a/core/src/test/scala/unit/kafka/server/CreateTopicsRequestTest.scala b/core/src/test/scala/unit/kafka/server/CreateTopicsRequestTest.scala
index 6d1b771a78a8d..259fadcb7abb0 100644
--- a/core/src/test/scala/unit/kafka/server/CreateTopicsRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/CreateTopicsRequestTest.scala
@@ -24,7 +24,7 @@ import org.junit.Test
 
 class CreateTopicsRequestTest extends AbstractCreateTopicsRequestTest {
   @Test
-  def testValidCreateTopicsRequests() {
+  def testValidCreateTopicsRequests(): Unit = {
     // Generated assignments
     validateValidCreateTopicsRequests(topicsReq(Seq(topicReq("topic1"))))
     validateValidCreateTopicsRequests(topicsReq(Seq(topicReq("topic2", replicationFactor = 3))))
@@ -53,7 +53,7 @@ class CreateTopicsRequestTest extends AbstractCreateTopicsRequestTest {
   }
 
   @Test
-  def testErrorCreateTopicsRequests() {
+  def testErrorCreateTopicsRequests(): Unit = {
     val existingTopic = "existing-topic"
     createTopic(existingTopic, 1, 1)
     // Basic
@@ -113,7 +113,7 @@ class CreateTopicsRequestTest extends AbstractCreateTopicsRequestTest {
   }
 
   @Test
-  def testInvalidCreateTopicsRequests() {
+  def testInvalidCreateTopicsRequests(): Unit = {
     // Partitions/ReplicationFactor and ReplicaAssignment
     validateErrorCreateTopicsRequests(topicsReq(Seq(
       topicReq("bad-args-topic", numPartitions = 10, replicationFactor = 3,
@@ -127,7 +127,7 @@ class CreateTopicsRequestTest extends AbstractCreateTopicsRequestTest {
   }
 
   @Test
-  def testNotController() {
+  def testNotController(): Unit = {
     val req = topicsReq(Seq(topicReq("topic1")))
     val response = sendCreateTopicRequest(req, notControllerSocketServer)
     assertEquals(1, response.errorCounts().get(Errors.NOT_CONTROLLER))
diff --git a/core/src/test/scala/unit/kafka/server/CreateTopicsRequestWithPolicyTest.scala b/core/src/test/scala/unit/kafka/server/CreateTopicsRequestWithPolicyTest.scala
index 1cdd8f1dbefaa..aef408041c282 100644
--- a/core/src/test/scala/unit/kafka/server/CreateTopicsRequestWithPolicyTest.scala
+++ b/core/src/test/scala/unit/kafka/server/CreateTopicsRequestWithPolicyTest.scala
@@ -38,7 +38,7 @@ class CreateTopicsRequestWithPolicyTest extends AbstractCreateTopicsRequestTest
   }
 
   @Test
-  def testValidCreateTopicsRequests() {
+  def testValidCreateTopicsRequests(): Unit = {
     validateValidCreateTopicsRequests(topicsReq(Seq(topicReq("topic1",
       numPartitions = 5))))
 
@@ -56,7 +56,7 @@ class CreateTopicsRequestWithPolicyTest extends AbstractCreateTopicsRequestTest
   }
 
   @Test
-  def testErrorCreateTopicsRequests() {
+  def testErrorCreateTopicsRequests(): Unit = {
     val existingTopic = "existing-topic"
     createTopic(existingTopic, 1, 1)
 
diff --git a/core/src/test/scala/unit/kafka/server/DelayedOperationTest.scala b/core/src/test/scala/unit/kafka/server/DelayedOperationTest.scala
index c1e0b9f7f2aa3..0a81eb7bb90a6 100644
--- a/core/src/test/scala/unit/kafka/server/DelayedOperationTest.scala
+++ b/core/src/test/scala/unit/kafka/server/DelayedOperationTest.scala
@@ -34,19 +34,19 @@ class DelayedOperationTest {
   var executorService: ExecutorService = null
 
   @Before
-  def setUp() {
+  def setUp(): Unit = {
     purgatory = DelayedOperationPurgatory[MockDelayedOperation](purgatoryName = "mock")
   }
 
   @After
-  def tearDown() {
+  def tearDown(): Unit = {
     purgatory.shutdown()
     if (executorService != null)
       executorService.shutdown()
   }
 
   @Test
-  def testRequestSatisfaction() {
+  def testRequestSatisfaction(): Unit = {
     val r1 = new MockDelayedOperation(100000L)
     val r2 = new MockDelayedOperation(100000L)
     assertEquals("With no waiting requests, nothing should be satisfied", 0, purgatory.checkAndComplete("test1"))
@@ -63,7 +63,7 @@ class DelayedOperationTest {
   }
 
   @Test
-  def testRequestExpiry() {
+  def testRequestExpiry(): Unit = {
     val expiration = 20L
     val start = Time.SYSTEM.hiResClockMs
     val r1 = new MockDelayedOperation(expiration)
@@ -78,7 +78,7 @@ class DelayedOperationTest {
   }
 
   @Test
-  def testRequestPurge() {
+  def testRequestPurge(): Unit = {
     val r1 = new MockDelayedOperation(100000L)
     val r2 = new MockDelayedOperation(100000L)
     val r3 = new MockDelayedOperation(100000L)
@@ -110,7 +110,7 @@ class DelayedOperationTest {
   }
 
   @Test
-  def shouldCancelForKeyReturningCancelledOperations() {
+  def shouldCancelForKeyReturningCancelledOperations(): Unit = {
     purgatory.tryCompleteElseWatch(new MockDelayedOperation(10000L), Seq("key"))
     purgatory.tryCompleteElseWatch(new MockDelayedOperation(10000L), Seq("key"))
     purgatory.tryCompleteElseWatch(new MockDelayedOperation(10000L), Seq("key2"))
@@ -122,7 +122,7 @@ class DelayedOperationTest {
   }
 
   @Test
-  def shouldReturnNilOperationsOnCancelForKeyWhenKeyDoesntExist() {
+  def shouldReturnNilOperationsOnCancelForKeyWhenKeyDoesntExist(): Unit = {
     val cancelledOperations = purgatory.cancelForKey("key")
     assertEquals(Nil, cancelledOperations)
   }
@@ -216,12 +216,12 @@ class DelayedOperationTest {
   }
 
   @Test
-  def testDelayedOperationLock() {
+  def testDelayedOperationLock(): Unit = {
     verifyDelayedOperationLock(new MockDelayedOperation(100000L), mismatchedLocks = false)
   }
 
   @Test
-  def testDelayedOperationLockOverride() {
+  def testDelayedOperationLockOverride(): Unit = {
     def newMockOperation = {
       val lock = new ReentrantLock
       new MockDelayedOperation(100000L, Some(lock), Some(lock))
@@ -232,7 +232,7 @@ class DelayedOperationTest {
         mismatchedLocks = true)
   }
 
-  def verifyDelayedOperationLock(mockDelayedOperation: => MockDelayedOperation, mismatchedLocks: Boolean) {
+  def verifyDelayedOperationLock(mockDelayedOperation: => MockDelayedOperation, mismatchedLocks: Boolean): Unit = {
     val key = "key"
     executorService = Executors.newSingleThreadExecutor
     def createDelayedOperations(count: Int): Seq[MockDelayedOperation] = {
@@ -327,7 +327,7 @@ class DelayedOperationTest {
                              extends DelayedOperation(delayMs, lockOpt) {
     var completable = false
 
-    def awaitExpiration() {
+    def awaitExpiration(): Unit = {
       synchronized {
         wait()
       }
@@ -340,11 +340,11 @@ class DelayedOperationTest {
         false
     }
 
-    override def onExpiration() {
+    override def onExpiration(): Unit = {
 
     }
 
-    override def onComplete() {
+    override def onComplete(): Unit = {
       responseLockOpt.foreach { lock =>
         if (!lock.tryLock())
           throw new IllegalStateException("Response callback lock could not be acquired in callback")
diff --git a/core/src/test/scala/unit/kafka/server/DeleteTopicsRequestTest.scala b/core/src/test/scala/unit/kafka/server/DeleteTopicsRequestTest.scala
index 2df7528dfbb6a..b9161a8d07b0c 100644
--- a/core/src/test/scala/unit/kafka/server/DeleteTopicsRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/DeleteTopicsRequestTest.scala
@@ -32,7 +32,7 @@ import java.util.Arrays
 class DeleteTopicsRequestTest extends BaseRequestTest {
 
   @Test
-  def testValidDeleteTopicRequests() {
+  def testValidDeleteTopicRequests(): Unit = {
     val timeout = 10000
     // Single topic
     createTopic("topic-1", 1, 1)
@@ -59,7 +59,7 @@ class DeleteTopicsRequestTest extends BaseRequestTest {
   }
 
   @Test
-  def testErrorDeleteTopicRequests() {
+  def testErrorDeleteTopicRequests(): Unit = {
     val timeout = 30000
     val timeoutTopic = "invalid-timeout"
 
@@ -112,7 +112,7 @@ class DeleteTopicsRequestTest extends BaseRequestTest {
   }
 
   @Test
-  def testNotController() {
+  def testNotController(): Unit = {
     val request = new DeleteTopicsRequest.Builder(
         new DeleteTopicsRequestData()
           .setTopicNames(Collections.singletonList("not-controller"))
diff --git a/core/src/test/scala/unit/kafka/server/DeleteTopicsRequestWithDeletionDisabledTest.scala b/core/src/test/scala/unit/kafka/server/DeleteTopicsRequestWithDeletionDisabledTest.scala
index 6a011653f2e03..ca0ccc0212aca 100644
--- a/core/src/test/scala/unit/kafka/server/DeleteTopicsRequestWithDeletionDisabledTest.scala
+++ b/core/src/test/scala/unit/kafka/server/DeleteTopicsRequestWithDeletionDisabledTest.scala
@@ -41,7 +41,7 @@ class DeleteTopicsRequestWithDeletionDisabledTest extends BaseRequestTest {
   }
 
   @Test
-  def testDeleteRecordsRequest() {
+  def testDeleteRecordsRequest(): Unit = {
     val topic = "topic-1"
     val request = new DeleteTopicsRequest.Builder(
         new DeleteTopicsRequestData()
diff --git a/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala b/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala
index 278f0cede71c9..e9472ae7c5255 100755
--- a/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala
+++ b/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala
@@ -194,7 +194,7 @@ class DynamicBrokerConfigTest {
     verifyConfigUpdate(listenerMaxConnectionsProp, "10", perBrokerConfig = false, expectFailure = false)
   }
 
-  private def verifyConfigUpdate(name: String, value: Object, perBrokerConfig: Boolean, expectFailure: Boolean) {
+  private def verifyConfigUpdate(name: String, value: Object, perBrokerConfig: Boolean, expectFailure: Boolean): Unit = {
     val configProps = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
     configProps.put(KafkaConfig.PasswordEncoderSecretProp, "broker.secret")
     val config = KafkaConfig(configProps)
diff --git a/core/src/test/scala/unit/kafka/server/DynamicConfigChangeTest.scala b/core/src/test/scala/unit/kafka/server/DynamicConfigChangeTest.scala
index 86d977cbf0f40..04afa41f2dd8e 100644
--- a/core/src/test/scala/unit/kafka/server/DynamicConfigChangeTest.scala
+++ b/core/src/test/scala/unit/kafka/server/DynamicConfigChangeTest.scala
@@ -38,7 +38,7 @@ class DynamicConfigChangeTest extends KafkaServerTestHarness {
   def generateConfigs = List(KafkaConfig.fromProps(TestUtils.createBrokerConfig(0, zkConnect)))
 
   @Test
-  def testConfigChange() {
+  def testConfigChange(): Unit = {
     assertTrue("Should contain a ConfigHandler for topics",
       this.servers.head.dynamicConfigHandlers.contains(ConfigType.Topic))
     val oldVal: java.lang.Long = 100000L
@@ -60,7 +60,7 @@ class DynamicConfigChangeTest extends KafkaServerTestHarness {
   }
 
   @Test
-  def testDynamicTopicConfigChange() {
+  def testDynamicTopicConfigChange(): Unit = {
     val tp = new TopicPartition("test", 0)
     val oldSegmentSize = 1000
     val logProps = new Properties()
@@ -86,7 +86,7 @@ class DynamicConfigChangeTest extends KafkaServerTestHarness {
     assertTrue("Log segment size change not applied", log.logSegments.forall(_.size > 1000))
   }
 
-  private def testQuotaConfigChange(user: String, clientId: String, rootEntityType: String, configEntityName: String) {
+  private def testQuotaConfigChange(user: String, clientId: String, rootEntityType: String, configEntityName: String): Unit = {
     assertTrue("Should contain a ConfigHandler for " + rootEntityType ,
                this.servers.head.dynamicConfigHandlers.contains(rootEntityType))
     val props = new Properties()
@@ -129,37 +129,37 @@ class DynamicConfigChangeTest extends KafkaServerTestHarness {
   }
 
   @Test
-  def testClientIdQuotaConfigChange() {
+  def testClientIdQuotaConfigChange(): Unit = {
     testQuotaConfigChange("ANONYMOUS", "testClient", ConfigType.Client, "testClient")
   }
 
   @Test
-  def testUserQuotaConfigChange() {
+  def testUserQuotaConfigChange(): Unit = {
     testQuotaConfigChange("ANONYMOUS", "testClient", ConfigType.User, "ANONYMOUS")
   }
 
   @Test
-  def testUserClientIdQuotaChange() {
+  def testUserClientIdQuotaChange(): Unit = {
     testQuotaConfigChange("ANONYMOUS", "testClient", ConfigType.User, "ANONYMOUS/clients/testClient")
   }
 
   @Test
-  def testDefaultClientIdQuotaConfigChange() {
+  def testDefaultClientIdQuotaConfigChange(): Unit = {
     testQuotaConfigChange("ANONYMOUS", "testClient", ConfigType.Client, "")
   }
 
   @Test
-  def testDefaultUserQuotaConfigChange() {
+  def testDefaultUserQuotaConfigChange(): Unit = {
     testQuotaConfigChange("ANONYMOUS", "testClient", ConfigType.User, "")
   }
 
   @Test
-  def testDefaultUserClientIdQuotaConfigChange() {
+  def testDefaultUserClientIdQuotaConfigChange(): Unit = {
     testQuotaConfigChange("ANONYMOUS", "testClient", ConfigType.User, "/clients/")
   }
 
   @Test
-  def testQuotaInitialization() {
+  def testQuotaInitialization(): Unit = {
     val server = servers.head
     val clientIdProps = new Properties()
     server.shutdown()
@@ -190,7 +190,7 @@ class DynamicConfigChangeTest extends KafkaServerTestHarness {
   }
 
   @Test
-  def testConfigChangeOnNonExistingTopic() {
+  def testConfigChangeOnNonExistingTopic(): Unit = {
     val topic = TestUtils.tempTopic
     try {
       val logProps = new Properties()
@@ -302,7 +302,7 @@ class DynamicConfigChangeTest extends KafkaServerTestHarness {
   }
 
   @Test
-  def shouldParseRegardlessOfWhitespaceAroundValues() {
+  def shouldParseRegardlessOfWhitespaceAroundValues(): Unit = {
     val configHandler: TopicConfigHandler = new TopicConfigHandler(null, null, null, null)
     assertEquals(AllReplicas, parse(configHandler, "* "))
     assertEquals(Seq(), parse(configHandler, " "))
diff --git a/core/src/test/scala/unit/kafka/server/DynamicConfigTest.scala b/core/src/test/scala/unit/kafka/server/DynamicConfigTest.scala
index cf0984960f174..14e3ef5754e97 100644
--- a/core/src/test/scala/unit/kafka/server/DynamicConfigTest.scala
+++ b/core/src/test/scala/unit/kafka/server/DynamicConfigTest.scala
@@ -26,23 +26,23 @@ class DynamicConfigTest  extends ZooKeeperTestHarness {
   private final val someValue: String = "some interesting value"
 
   @Test(expected = classOf[IllegalArgumentException])
-  def shouldFailWhenChangingClientIdUnknownConfig() {
+  def shouldFailWhenChangingClientIdUnknownConfig(): Unit = {
     adminZkClient.changeClientIdConfig("ClientId", propsWith(nonExistentConfig, someValue))
   }
 
   @Test(expected = classOf[IllegalArgumentException])
-  def shouldFailWhenChangingUserUnknownConfig() {
+  def shouldFailWhenChangingUserUnknownConfig(): Unit = {
     adminZkClient.changeUserOrUserClientIdConfig("UserId", propsWith(nonExistentConfig, someValue))
   }
 
   @Test(expected = classOf[ConfigException])
-  def shouldFailLeaderConfigsWithInvalidValues() {
+  def shouldFailLeaderConfigsWithInvalidValues(): Unit = {
     adminZkClient.changeBrokerConfig(Seq(0),
       propsWith(DynamicConfig.Broker.LeaderReplicationThrottledRateProp, "-100"))
   }
 
   @Test(expected = classOf[ConfigException])
-  def shouldFailFollowerConfigsWithInvalidValues() {
+  def shouldFailFollowerConfigsWithInvalidValues(): Unit = {
     adminZkClient.changeBrokerConfig(Seq(0),
       propsWith(DynamicConfig.Broker.FollowerReplicationThrottledRateProp, "-100"))
   }
diff --git a/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala b/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala
index a42610831641e..c267f04e8e116 100755
--- a/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala
@@ -50,7 +50,7 @@ class EdgeCaseRequestTest extends KafkaServerTestHarness {
     new Socket("localhost", s.boundPort(ListenerName.forSecurityProtocol(protocol)))
   }
 
-  private def sendRequest(socket: Socket, request: Array[Byte], id: Option[Short] = None) {
+  private def sendRequest(socket: Socket, request: Array[Byte], id: Option[Short] = None): Unit = {
     val outgoing = new DataOutputStream(socket.getOutputStream)
     id match {
       case Some(id) =>
@@ -98,7 +98,7 @@ class EdgeCaseRequestTest extends KafkaServerTestHarness {
     buffer.array()
   }
 
-  private def verifyDisconnect(request: Array[Byte]) {
+  private def verifyDisconnect(request: Array[Byte]): Unit = {
     val plainSocket = connect()
     try {
       sendRequest(plainSocket, requestHeaderBytes(-1, 0))
@@ -109,7 +109,7 @@ class EdgeCaseRequestTest extends KafkaServerTestHarness {
   }
 
   @Test
-  def testProduceRequestWithNullClientId() {
+  def testProduceRequestWithNullClientId(): Unit = {
     val topic = "topic"
     val topicPartition = new TopicPartition(topic, 0)
     val correlationId = -1
@@ -143,22 +143,22 @@ class EdgeCaseRequestTest extends KafkaServerTestHarness {
   }
 
   @Test
-  def testHeaderOnlyRequest() {
+  def testHeaderOnlyRequest(): Unit = {
     verifyDisconnect(requestHeaderBytes(ApiKeys.PRODUCE.id, 1))
   }
 
   @Test
-  def testInvalidApiKeyRequest() {
+  def testInvalidApiKeyRequest(): Unit = {
     verifyDisconnect(requestHeaderBytes(-1, 0))
   }
 
   @Test
-  def testInvalidApiVersionRequest() {
+  def testInvalidApiVersionRequest(): Unit = {
     verifyDisconnect(requestHeaderBytes(ApiKeys.PRODUCE.id, -1))
   }
 
   @Test
-  def testMalformedHeaderRequest() {
+  def testMalformedHeaderRequest(): Unit = {
     val serializedBytes = {
       // Only send apiKey and apiVersion
       val buffer = ByteBuffer.allocate(
diff --git a/core/src/test/scala/unit/kafka/server/HighwatermarkPersistenceTest.scala b/core/src/test/scala/unit/kafka/server/HighwatermarkPersistenceTest.scala
index d99701a14996e..1efecedf9806e 100755
--- a/core/src/test/scala/unit/kafka/server/HighwatermarkPersistenceTest.scala
+++ b/core/src/test/scala/unit/kafka/server/HighwatermarkPersistenceTest.scala
@@ -48,13 +48,13 @@ class HighwatermarkPersistenceTest {
   }
 
   @After
-  def teardown() {
+  def teardown(): Unit = {
     for (manager <- logManagers; dir <- manager.liveLogDirs)
       Utils.delete(dir)
   }
 
   @Test
-  def testHighWatermarkPersistenceSinglePartition() {
+  def testHighWatermarkPersistenceSinglePartition(): Unit = {
     // mock zkclient
     EasyMock.replay(zkClient)
 
@@ -101,7 +101,7 @@ class HighwatermarkPersistenceTest {
   }
 
   @Test
-  def testHighWatermarkPersistenceMultiplePartitions() {
+  def testHighWatermarkPersistenceMultiplePartitions(): Unit = {
     val topic1 = "foo1"
     val topic2 = "foo2"
     // mock zkclient
diff --git a/core/src/test/scala/unit/kafka/server/IsrExpirationTest.scala b/core/src/test/scala/unit/kafka/server/IsrExpirationTest.scala
index c25899655f04a..e43283ecabb86 100644
--- a/core/src/test/scala/unit/kafka/server/IsrExpirationTest.scala
+++ b/core/src/test/scala/unit/kafka/server/IsrExpirationTest.scala
@@ -52,7 +52,7 @@ class IsrExpirationTest {
   var replicaManager: ReplicaManager = null
 
   @Before
-  def setUp() {
+  def setUp(): Unit = {
     val logManager: LogManager = EasyMock.createMock(classOf[LogManager])
     EasyMock.expect(logManager.liveLogDirs).andReturn(Array.empty[File]).anyTimes()
     EasyMock.replay(logManager)
@@ -63,7 +63,7 @@ class IsrExpirationTest {
   }
 
   @After
-  def tearDown() {
+  def tearDown(): Unit = {
     replicaManager.shutdown(false)
     metrics.close()
   }
@@ -72,7 +72,7 @@ class IsrExpirationTest {
    * Test the case where a follower is caught up but stops making requests to the leader. Once beyond the configured time limit, it should fall out of ISR
    */
   @Test
-  def testIsrExpirationForStuckFollowers() {
+  def testIsrExpirationForStuckFollowers(): Unit = {
     val log = logMock
 
     // create one partition and all replicas
@@ -102,7 +102,7 @@ class IsrExpirationTest {
    * Test the case where a follower never makes a fetch request. It should fall out of ISR because it will be declared stuck
    */
   @Test
-  def testIsrExpirationIfNoFetchRequestMade() {
+  def testIsrExpirationIfNoFetchRequestMade(): Unit = {
     val log = logMock
 
     // create one partition and all replicas
@@ -122,7 +122,7 @@ class IsrExpirationTest {
    * However, any time it makes a request to the LogEndOffset it should be back in the ISR
    */
   @Test
-  def testIsrExpirationForSlowFollowers() {
+  def testIsrExpirationForSlowFollowers(): Unit = {
     // create leader replica
     val log = logMock
     // add one partition
@@ -177,7 +177,7 @@ class IsrExpirationTest {
    * Test the case where a follower has already caught up with same log end offset with the leader. This follower should not be considered as out-of-sync
    */
   @Test
-  def testIsrExpirationForCaughtUpFollowers() {
+  def testIsrExpirationForCaughtUpFollowers(): Unit = {
     val log = logMock
 
     // create one partition and all replicas
diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
index 44f0301cd7e0b..37610403a3925 100644
--- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
@@ -83,7 +83,7 @@ class KafkaApisTest {
   private val time = new MockTime
 
   @After
-  def tearDown() {
+  def tearDown(): Unit = {
     quotas.shutdown()
     metrics.close()
   }
@@ -548,7 +548,7 @@ class KafkaApisTest {
   }
 
   @Test
-  def rejectJoinGroupRequestWhenStaticMembershipNotSupported() {
+  def rejectJoinGroupRequestWhenStaticMembershipNotSupported(): Unit = {
     val capturedResponse = expectNoThrottling()
     EasyMock.replay(clientRequestQuotaManager, requestChannel)
 
@@ -568,7 +568,7 @@ class KafkaApisTest {
   }
 
   @Test
-  def rejectSyncGroupRequestWhenStaticMembershipNotSupported() {
+  def rejectSyncGroupRequestWhenStaticMembershipNotSupported(): Unit = {
     val capturedResponse = expectNoThrottling()
     EasyMock.replay(clientRequestQuotaManager, requestChannel)
 
@@ -587,7 +587,7 @@ class KafkaApisTest {
   }
 
   @Test
-  def rejectHeartbeatRequestWhenStaticMembershipNotSupported() {
+  def rejectHeartbeatRequestWhenStaticMembershipNotSupported(): Unit = {
     val capturedResponse = expectNoThrottling()
     EasyMock.replay(clientRequestQuotaManager, requestChannel)
 
@@ -606,7 +606,7 @@ class KafkaApisTest {
   }
 
   @Test
-  def rejectOffsetCommitRequestWhenStaticMembershipNotSupported() {
+  def rejectOffsetCommitRequestWhenStaticMembershipNotSupported(): Unit = {
     val capturedResponse = expectNoThrottling()
     EasyMock.replay(clientRequestQuotaManager, requestChannel)
 
@@ -645,7 +645,7 @@ class KafkaApisTest {
   }
 
   @Test
-  def testMultipleLeaveGroup() {
+  def testMultipleLeaveGroup(): Unit = {
     val groupId = "groupId"
 
     val leaveMemberList = List(
@@ -675,7 +675,7 @@ class KafkaApisTest {
   }
 
   @Test
-  def testSingleLeaveGroup() {
+  def testSingleLeaveGroup(): Unit = {
     val groupId = "groupId"
     val memberId = "member"
 
diff --git a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
index 850f553a6fc8a..a6e86c66722bb 100755
--- a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
@@ -35,7 +35,7 @@ import org.scalatest.Assertions.intercept
 class KafkaConfigTest {
 
   @Test
-  def testLogRetentionTimeHoursProvided() {
+  def testLogRetentionTimeHoursProvided(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
     props.put(KafkaConfig.LogRetentionTimeHoursProp, "1")
 
@@ -44,7 +44,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testLogRetentionTimeMinutesProvided() {
+  def testLogRetentionTimeMinutesProvided(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
     props.put(KafkaConfig.LogRetentionTimeMinutesProp, "30")
 
@@ -53,7 +53,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testLogRetentionTimeMsProvided() {
+  def testLogRetentionTimeMsProvided(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
     props.put(KafkaConfig.LogRetentionTimeMillisProp, "1800000")
 
@@ -62,7 +62,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testLogRetentionTimeNoConfigProvided() {
+  def testLogRetentionTimeNoConfigProvided(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
 
     val cfg = KafkaConfig.fromProps(props)
@@ -70,7 +70,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testLogRetentionTimeBothMinutesAndHoursProvided() {
+  def testLogRetentionTimeBothMinutesAndHoursProvided(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
     props.put(KafkaConfig.LogRetentionTimeMinutesProp, "30")
     props.put(KafkaConfig.LogRetentionTimeHoursProp, "1")
@@ -80,7 +80,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testLogRetentionTimeBothMinutesAndMsProvided() {
+  def testLogRetentionTimeBothMinutesAndMsProvided(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
     props.put(KafkaConfig.LogRetentionTimeMillisProp, "1800000")
     props.put(KafkaConfig.LogRetentionTimeMinutesProp, "10")
@@ -90,7 +90,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testLogRetentionUnlimited() {
+  def testLogRetentionUnlimited(): Unit = {
     val props1 = TestUtils.createBrokerConfig(0,TestUtils.MockZkConnect, port = 8181)
     val props2 = TestUtils.createBrokerConfig(0,TestUtils.MockZkConnect, port = 8181)
     val props3 = TestUtils.createBrokerConfig(0,TestUtils.MockZkConnect, port = 8181)
@@ -144,7 +144,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testAdvertiseDefaults() {
+  def testAdvertiseDefaults(): Unit = {
     val port = "9999"
     val hostName = "fake-host"
 
@@ -160,7 +160,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testAdvertiseConfigured() {
+  def testAdvertiseConfigured(): Unit = {
     val advertisedHostName = "routable-host"
     val advertisedPort = "1234"
 
@@ -177,7 +177,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testAdvertisePortDefault() {
+  def testAdvertisePortDefault(): Unit = {
     val advertisedHostName = "routable-host"
     val port = "9999"
 
@@ -194,7 +194,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testAdvertiseHostNameDefault() {
+  def testAdvertiseHostNameDefault(): Unit = {
     val hostName = "routable-host"
     val advertisedPort = "9999"
 
@@ -211,7 +211,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testDuplicateListeners() {
+  def testDuplicateListeners(): Unit = {
     val props = new Properties()
     props.put(KafkaConfig.BrokerIdProp, "1")
     props.put(KafkaConfig.ZkConnectProp, "localhost:2181")
@@ -255,7 +255,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testBadListenerProtocol() {
+  def testBadListenerProtocol(): Unit = {
     val props = new Properties()
     props.put(KafkaConfig.BrokerIdProp, "1")
     props.put(KafkaConfig.ZkConnectProp, "localhost:2181")
@@ -354,7 +354,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testCaseInsensitiveListenerProtocol() {
+  def testCaseInsensitiveListenerProtocol(): Unit = {
     val props = new Properties()
     props.put(KafkaConfig.BrokerIdProp, "1")
     props.put(KafkaConfig.ZkConnectProp, "localhost:2181")
@@ -369,7 +369,7 @@ class KafkaConfigTest {
     CoreUtils.listenerListToEndPoints(listenerList, securityProtocolMap)
 
   @Test
-  def testListenerDefaults() {
+  def testListenerDefaults(): Unit = {
     val props = new Properties()
     props.put(KafkaConfig.BrokerIdProp, "1")
     props.put(KafkaConfig.ZkConnectProp, "localhost:2181")
@@ -398,7 +398,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testVersionConfiguration() {
+  def testVersionConfiguration(): Unit = {
     val props = new Properties()
     props.put(KafkaConfig.BrokerIdProp, "1")
     props.put(KafkaConfig.ZkConnectProp, "localhost:2181")
@@ -432,7 +432,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testUncleanLeaderElectionDefault() {
+  def testUncleanLeaderElectionDefault(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
     val serverConfig = KafkaConfig.fromProps(props)
 
@@ -440,7 +440,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testUncleanElectionDisabled() {
+  def testUncleanElectionDisabled(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
     props.put(KafkaConfig.UncleanLeaderElectionEnableProp, String.valueOf(false))
     val serverConfig = KafkaConfig.fromProps(props)
@@ -449,7 +449,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testUncleanElectionEnabled() {
+  def testUncleanElectionEnabled(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
     props.put(KafkaConfig.UncleanLeaderElectionEnableProp, String.valueOf(true))
     val serverConfig = KafkaConfig.fromProps(props)
@@ -458,7 +458,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testUncleanElectionInvalid() {
+  def testUncleanElectionInvalid(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
     props.put(KafkaConfig.UncleanLeaderElectionEnableProp, "invalid")
 
@@ -468,7 +468,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testLogRollTimeMsProvided() {
+  def testLogRollTimeMsProvided(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
     props.put(KafkaConfig.LogRollTimeMillisProp, "1800000")
 
@@ -477,7 +477,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testLogRollTimeBothMsAndHoursProvided() {
+  def testLogRollTimeBothMsAndHoursProvided(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
     props.put(KafkaConfig.LogRollTimeMillisProp, "1800000")
     props.put(KafkaConfig.LogRollTimeHoursProp, "1")
@@ -487,7 +487,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testLogRollTimeNoConfigProvided() {
+  def testLogRollTimeNoConfigProvided(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
 
     val cfg = KafkaConfig.fromProps(props)
@@ -495,7 +495,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testDefaultCompressionType() {
+  def testDefaultCompressionType(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
     val serverConfig = KafkaConfig.fromProps(props)
 
@@ -503,7 +503,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testValidCompressionType() {
+  def testValidCompressionType(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
     props.put("compression.type", "gzip")
     val serverConfig = KafkaConfig.fromProps(props)
@@ -512,7 +512,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testInvalidCompressionType() {
+  def testInvalidCompressionType(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
     props.put(KafkaConfig.CompressionTypeProp, "abc")
     intercept[IllegalArgumentException] {
@@ -521,7 +521,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testInvalidInterBrokerSecurityProtocol() {
+  def testInvalidInterBrokerSecurityProtocol(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
     props.put(KafkaConfig.ListenersProp, "SSL://localhost:0")
     props.put(KafkaConfig.InterBrokerSecurityProtocolProp, SecurityProtocol.PLAINTEXT.toString)
@@ -531,7 +531,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testEqualAdvertisedListenersProtocol() {
+  def testEqualAdvertisedListenersProtocol(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
     props.put(KafkaConfig.ListenersProp, "PLAINTEXT://localhost:9092,SSL://localhost:9093")
     props.put(KafkaConfig.AdvertisedListenersProp, "PLAINTEXT://localhost:9092,SSL://localhost:9093")
@@ -539,7 +539,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testInvalidAdvertisedListenersProtocol() {
+  def testInvalidAdvertisedListenersProtocol(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
     props.put(KafkaConfig.ListenersProp, "TRACE://localhost:9091,SSL://localhost:9093")
     props.put(KafkaConfig.AdvertisedListenersProp, "PLAINTEXT://localhost:9092")
@@ -573,7 +573,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testFromPropsInvalid() {
+  def testFromPropsInvalid(): Unit = {
     def getBaseProperties(): Properties = {
       val validRequiredProperties = new Properties()
       validRequiredProperties.put(KafkaConfig.ZkConnectProp, "127.0.0.1:2181")
@@ -813,7 +813,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testNonroutableAdvertisedListeners() {
+  def testNonroutableAdvertisedListeners(): Unit = {
     val props = new Properties()
     props.put(KafkaConfig.ZkConnectProp, "127.0.0.1:2181")
     props.put(KafkaConfig.ListenersProp, "PLAINTEXT://0.0.0.0:9092")
@@ -821,7 +821,7 @@ class KafkaConfigTest {
   }
 
   @Test
-  def testMaxConnectionsPerIpProp() {
+  def testMaxConnectionsPerIpProp(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
     props.put(KafkaConfig.MaxConnectionsPerIpProp, "0")
     assertFalse(isValidKafkaConfig(props))
@@ -831,7 +831,7 @@ class KafkaConfigTest {
     assertFalse(isValidKafkaConfig(props))
   }
 
-  private def assertPropertyInvalid(validRequiredProps: => Properties, name: String, values: Any*) {
+  private def assertPropertyInvalid(validRequiredProps: => Properties, name: String, values: Any*): Unit = {
     values.foreach((value) => {
       val props = validRequiredProps
       props.setProperty(name, value.toString)
diff --git a/core/src/test/scala/unit/kafka/server/KafkaMetricReporterClusterIdTest.scala b/core/src/test/scala/unit/kafka/server/KafkaMetricReporterClusterIdTest.scala
index 4d514a0550df8..fcba387fc748d 100755
--- a/core/src/test/scala/unit/kafka/server/KafkaMetricReporterClusterIdTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaMetricReporterClusterIdTest.scala
@@ -50,7 +50,7 @@ object KafkaMetricReporterClusterIdTest {
 
   class MockBrokerMetricsReporter extends MockMetricsReporter with ClusterResourceListener {
 
-    override def onUpdate(clusterMetadata: ClusterResource) {
+    override def onUpdate(clusterMetadata: ClusterResource): Unit = {
       MockBrokerMetricsReporter.CLUSTER_META.set(clusterMetadata)
     }
 
@@ -80,7 +80,7 @@ class KafkaMetricReporterClusterIdTest extends ZooKeeperTestHarness {
   var config: KafkaConfig = null
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
     val props = TestUtils.createBrokerConfig(1, zkConnect)
     props.setProperty(KafkaConfig.KafkaMetricsReporterClassesProp, "kafka.server.KafkaMetricReporterClusterIdTest$MockKafkaMetricsReporter")
@@ -93,7 +93,7 @@ class KafkaMetricReporterClusterIdTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testClusterIdPresent() {
+  def testClusterIdPresent(): Unit = {
     assertEquals("", KafkaMetricReporterClusterIdTest.setupError.get())
 
     assertNotNull(KafkaMetricReporterClusterIdTest.MockKafkaMetricsReporter.CLUSTER_META)
@@ -107,7 +107,7 @@ class KafkaMetricReporterClusterIdTest extends ZooKeeperTestHarness {
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     server.shutdown()
     CoreUtils.delete(config.logDirs)
     TestUtils.verifyNonDaemonThreadsStatus(this.getClass.getName)
diff --git a/core/src/test/scala/unit/kafka/server/KafkaMetricReporterExceptionHandlingTest.scala b/core/src/test/scala/unit/kafka/server/KafkaMetricReporterExceptionHandlingTest.scala
index 00b3ebc76dae6..d6f37cd4f05bb 100644
--- a/core/src/test/scala/unit/kafka/server/KafkaMetricReporterExceptionHandlingTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaMetricReporterExceptionHandlingTest.scala
@@ -44,7 +44,7 @@ class KafkaMetricReporterExceptionHandlingTest extends BaseRequestTest {
   }
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
 
     // need a quota prop to register a "throttle-time" metrics after server startup
@@ -54,7 +54,7 @@ class KafkaMetricReporterExceptionHandlingTest extends BaseRequestTest {
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     KafkaMetricReporterExceptionHandlingTest.goodReporterRegistered.set(0)
     KafkaMetricReporterExceptionHandlingTest.badReporterRegistered.set(0)
     
@@ -62,7 +62,7 @@ class KafkaMetricReporterExceptionHandlingTest extends BaseRequestTest {
   }
 
   @Test
-  def testBothReportersAreInvoked() {
+  def testBothReportersAreInvoked(): Unit = {
     val port = anySocketServer.boundPort(ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT))
     val socket = new Socket("localhost", port)
     socket.setSoTimeout(10000)
@@ -88,28 +88,28 @@ object KafkaMetricReporterExceptionHandlingTest {
 
   class GoodReporter extends MetricsReporter {
 
-    def configure(configs: java.util.Map[String, _]) {
+    def configure(configs: java.util.Map[String, _]): Unit = {
     }
 
-    def init(metrics: java.util.List[KafkaMetric]) {
+    def init(metrics: java.util.List[KafkaMetric]): Unit = {
     }
 
-    def metricChange(metric: KafkaMetric) {
+    def metricChange(metric: KafkaMetric): Unit = {
       if (metric.metricName.group == "Request") {
         goodReporterRegistered.incrementAndGet
       }
     }
 
-    def metricRemoval(metric: KafkaMetric) {
+    def metricRemoval(metric: KafkaMetric): Unit = {
     }
 
-    def close() {
+    def close(): Unit = {
     }
   }
 
   class BadReporter extends GoodReporter {
 
-    override def metricChange(metric: KafkaMetric) {
+    override def metricChange(metric: KafkaMetric): Unit = {
       if (metric.metricName.group == "Request") {
         badReporterRegistered.incrementAndGet
         throw new RuntimeException(metric.metricName.toString)
diff --git a/core/src/test/scala/unit/kafka/server/KafkaServerTest.scala b/core/src/test/scala/unit/kafka/server/KafkaServerTest.scala
index fc6d0439c4cce..a285fadf55b79 100755
--- a/core/src/test/scala/unit/kafka/server/KafkaServerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaServerTest.scala
@@ -25,7 +25,7 @@ import org.scalatest.Assertions.intercept
 class KafkaServerTest extends ZooKeeperTestHarness {
 
   @Test
-  def testAlreadyRegisteredAdvertisedListeners() {
+  def testAlreadyRegisteredAdvertisedListeners(): Unit = {
     //start a server with a advertised listener
     val server1 = createServer(1, "myhost", TestUtils.RandomPort)
 
diff --git a/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala b/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala
index c7ceb20e62ad1..6aeb355832cf5 100755
--- a/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala
+++ b/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala
@@ -44,7 +44,7 @@ class LeaderElectionTest extends ZooKeeperTestHarness {
   var staleControllerEpochDetected = false
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
 
     val configProps1 = TestUtils.createBrokerConfig(brokerId1, zkConnect, enableControlledShutdown = false)
@@ -60,7 +60,7 @@ class LeaderElectionTest extends ZooKeeperTestHarness {
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     TestUtils.shutdownServers(servers)
     super.tearDown()
   }
@@ -111,7 +111,7 @@ class LeaderElectionTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testLeaderElectionWithStaleControllerEpoch() {
+  def testLeaderElectionWithStaleControllerEpoch(): Unit = {
     // start 2 brokers
     val topic = "new-topic"
     val partitionId = 0
diff --git a/core/src/test/scala/unit/kafka/server/LogDirFailureTest.scala b/core/src/test/scala/unit/kafka/server/LogDirFailureTest.scala
index 90d5f7894779f..1824e1af50011 100644
--- a/core/src/test/scala/unit/kafka/server/LogDirFailureTest.scala
+++ b/core/src/test/scala/unit/kafka/server/LogDirFailureTest.scala
@@ -51,19 +51,19 @@ class LogDirFailureTest extends IntegrationTestHarness {
   this.serverConfig.setProperty(KafkaConfig.NumReplicaFetchersProp, "1")
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
     createTopic(topic, partitionNum, brokerCount)
   }
 
   @Test
-  def testIOExceptionDuringLogRoll() {
+  def testIOExceptionDuringLogRoll(): Unit = {
     testProduceAfterLogDirFailureOnLeader(Roll)
   }
 
   @Test
   // Broker should halt on any log directory failure if inter-broker protocol < 1.0
-  def brokerWithOldInterBrokerProtocolShouldHaltOnLogDirFailure() {
+  def brokerWithOldInterBrokerProtocolShouldHaltOnLogDirFailure(): Unit = {
     @volatile var statusCodeOption: Option[Int] = None
     Exit.setHaltProcedure { (statusCode, _) =>
       statusCodeOption = Some(statusCode)
@@ -92,12 +92,12 @@ class LogDirFailureTest extends IntegrationTestHarness {
   }
 
   @Test
-  def testIOExceptionDuringCheckpoint() {
+  def testIOExceptionDuringCheckpoint(): Unit = {
     testProduceAfterLogDirFailureOnLeader(Checkpoint)
   }
 
   @Test
-  def testReplicaFetcherThreadAfterLogDirFailureOnFollower() {
+  def testReplicaFetcherThreadAfterLogDirFailureOnFollower(): Unit = {
     this.producerConfig.setProperty(ProducerConfig.RETRIES_CONFIG, "0")
     val producer = createProducer()
     val partition = new TopicPartition(topic, 0)
@@ -127,7 +127,7 @@ class LogDirFailureTest extends IntegrationTestHarness {
     }
   }
 
-  def testProduceAfterLogDirFailureOnLeader(failureType: LogDirFailureType) {
+  def testProduceAfterLogDirFailureOnLeader(failureType: LogDirFailureType): Unit = {
     val consumer = createConsumer()
     subscribeAndWaitForAssignment(topic, consumer)
 
@@ -200,7 +200,7 @@ class LogDirFailureTest extends IntegrationTestHarness {
     assertTrue(offlineReplicas.contains(PartitionAndReplica(new TopicPartition(topic, 0), leaderServerId)))
   }
 
-  private def subscribeAndWaitForAssignment(topic: String, consumer: KafkaConsumer[Array[Byte], Array[Byte]]) {
+  private def subscribeAndWaitForAssignment(topic: String, consumer: KafkaConsumer[Array[Byte], Array[Byte]]): Unit = {
     consumer.subscribe(Collections.singletonList(topic))
     TestUtils.pollUntilTrue(consumer, () => !consumer.assignment.isEmpty, "Expected non-empty assignment")
   }
diff --git a/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala b/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala
index 27f5499ececca..d88540d0a415d 100755
--- a/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala
+++ b/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala
@@ -52,7 +52,7 @@ class LogOffsetTest extends BaseRequestTest {
 
   @deprecated("ListOffsetsRequest V0", since = "")
   @Test
-  def testGetOffsetsForUnknownTopic() {
+  def testGetOffsetsForUnknownTopic(): Unit = {
     val topicPartition = new TopicPartition("foo", 0)
     val request = ListOffsetRequest.Builder.forConsumer(false, IsolationLevel.READ_UNCOMMITTED)
       .setTargetTimes(Map(topicPartition ->
@@ -63,7 +63,7 @@ class LogOffsetTest extends BaseRequestTest {
 
   @deprecated("ListOffsetsRequest V0", since = "")
   @Test
-  def testGetOffsetsAfterDeleteRecords() {
+  def testGetOffsetsAfterDeleteRecords(): Unit = {
     val topic = "kafka-"
     val topicPartition = new TopicPartition(topic, 0)
 
@@ -95,7 +95,7 @@ class LogOffsetTest extends BaseRequestTest {
   }
 
   @Test
-  def testGetOffsetsBeforeLatestTime() {
+  def testGetOffsetsBeforeLatestTime(): Unit = {
     val topic = "kafka-"
     val topicPartition = new TopicPartition(topic, 0)
 
@@ -130,7 +130,7 @@ class LogOffsetTest extends BaseRequestTest {
   }
 
   @Test
-  def testEmptyLogsGetOffsets() {
+  def testEmptyLogsGetOffsets(): Unit = {
     val random = new Random
     val topic = "kafka-"
     val topicPartition = new TopicPartition(topic, random.nextInt(10))
@@ -155,7 +155,7 @@ class LogOffsetTest extends BaseRequestTest {
 
   @deprecated("legacyFetchOffsetsBefore", since = "")
   @Test
-  def testGetOffsetsBeforeNow() {
+  def testGetOffsetsBeforeNow(): Unit = {
     val random = new Random
     val topic = "kafka-"
     val topicPartition = new TopicPartition(topic, random.nextInt(3))
@@ -185,7 +185,7 @@ class LogOffsetTest extends BaseRequestTest {
 
   @deprecated("legacyFetchOffsetsBefore", since = "")
   @Test
-  def testGetOffsetsBeforeEarliestTime() {
+  def testGetOffsetsBeforeEarliestTime(): Unit = {
     val random = new Random
     val topic = "kafka-"
     val topicPartition = new TopicPartition(topic, random.nextInt(3))
@@ -214,7 +214,7 @@ class LogOffsetTest extends BaseRequestTest {
   /* We test that `fetchOffsetsBefore` works correctly if `LogSegment.size` changes after each invocation (simulating
    * a race condition) */
   @Test
-  def testFetchOffsetsBeforeWithChangingSegmentSize() {
+  def testFetchOffsetsBeforeWithChangingSegmentSize(): Unit = {
     val log: Log = EasyMock.niceMock(classOf[Log])
     val logSegment: LogSegment = EasyMock.niceMock(classOf[LogSegment])
     EasyMock.expect(logSegment.size).andStubAnswer(new IAnswer[Int] {
@@ -231,7 +231,7 @@ class LogOffsetTest extends BaseRequestTest {
   /* We test that `fetchOffsetsBefore` works correctly if `Log.logSegments` content and size are
    * different (simulating a race condition) */
   @Test
-  def testFetchOffsetsBeforeWithChangingSegments() {
+  def testFetchOffsetsBeforeWithChangingSegments(): Unit = {
     val log: Log = EasyMock.niceMock(classOf[Log])
     val logSegment: LogSegment = EasyMock.niceMock(classOf[LogSegment])
     EasyMock.expect(log.logSegments).andStubAnswer {
diff --git a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala
index 9f73fbea98883..8d7b27a25d377 100644
--- a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala
+++ b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala
@@ -35,7 +35,7 @@ class MetadataCacheTest {
   val brokerEpoch = 0L
 
   @Test
-  def getTopicMetadataNonExistingTopics() {
+  def getTopicMetadataNonExistingTopics(): Unit = {
     val topic = "topic"
     val cache = new MetadataCache(1)
     val topicMetadata = cache.getTopicMetadata(Set(topic), ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT))
@@ -43,7 +43,7 @@ class MetadataCacheTest {
   }
 
   @Test
-  def getTopicMetadata() {
+  def getTopicMetadata(): Unit = {
     val topic0 = "topic-0"
     val topic1 = "topic-1"
 
@@ -188,7 +188,7 @@ class MetadataCacheTest {
   }
 
   @Test
-  def getTopicMetadataReplicaNotAvailable() {
+  def getTopicMetadataReplicaNotAvailable(): Unit = {
     val topic = "topic"
 
     val cache = new MetadataCache(1)
@@ -248,7 +248,7 @@ class MetadataCacheTest {
   }
 
   @Test
-  def getTopicMetadataIsrNotAvailable() {
+  def getTopicMetadataIsrNotAvailable(): Unit = {
     val topic = "topic"
 
     val cache = new MetadataCache(1)
@@ -308,7 +308,7 @@ class MetadataCacheTest {
   }
 
   @Test
-  def getTopicMetadataWithNonSupportedSecurityProtocol() {
+  def getTopicMetadataWithNonSupportedSecurityProtocol(): Unit = {
     val topic = "topic"
     val cache = new MetadataCache(1)
     val securityProtocol = SecurityProtocol.PLAINTEXT
@@ -333,11 +333,11 @@ class MetadataCacheTest {
   }
 
   @Test
-  def getAliveBrokersShouldNotBeMutatedByUpdateCache() {
+  def getAliveBrokersShouldNotBeMutatedByUpdateCache(): Unit = {
     val topic = "topic"
     val cache = new MetadataCache(1)
 
-    def updateCache(brokerIds: Set[Int]) {
+    def updateCache(brokerIds: Set[Int]): Unit = {
       val brokers = brokerIds.map { brokerId =>
         val securityProtocol = SecurityProtocol.PLAINTEXT
         new Broker(brokerId, Seq(
diff --git a/core/src/test/scala/unit/kafka/server/MetadataRequestTest.scala b/core/src/test/scala/unit/kafka/server/MetadataRequestTest.scala
index 4041b3520b44f..c51444cbe9d07 100644
--- a/core/src/test/scala/unit/kafka/server/MetadataRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/MetadataRequestTest.scala
@@ -35,7 +35,7 @@ import scala.collection.Seq
 
 class MetadataRequestTest extends BaseRequestTest {
 
-  override def brokerPropertyOverrides(properties: Properties) {
+  override def brokerPropertyOverrides(properties: Properties): Unit = {
     properties.setProperty(KafkaConfig.DefaultReplicationFactorProp, "2")
     properties.setProperty(KafkaConfig.RackProp, s"rack/${properties.getProperty(KafkaConfig.BrokerIdProp)}")
   }
@@ -46,20 +46,20 @@ class MetadataRequestTest extends BaseRequestTest {
   }
 
   @Test
-  def testClusterIdWithRequestVersion1() {
+  def testClusterIdWithRequestVersion1(): Unit = {
     val v1MetadataResponse = sendMetadataRequest(MetadataRequest.Builder.allTopics.build(1.toShort))
     val v1ClusterId = v1MetadataResponse.clusterId
     assertNull(s"v1 clusterId should be null", v1ClusterId)
   }
 
   @Test
-  def testClusterIdIsValid() {
+  def testClusterIdIsValid(): Unit = {
     val metadataResponse = sendMetadataRequest(MetadataRequest.Builder.allTopics.build(2.toShort))
     isValidClusterId(metadataResponse.clusterId)
   }
 
   @Test
-  def testControllerId() {
+  def testControllerId(): Unit = {
     val controllerServer = servers.find(_.kafkaController.isActive).get
     val controllerId = controllerServer.config.brokerId
     val metadataResponse = sendMetadataRequest(MetadataRequest.Builder.allTopics.build(1.toShort))
@@ -81,7 +81,7 @@ class MetadataRequestTest extends BaseRequestTest {
   }
 
   @Test
-  def testRack() {
+  def testRack(): Unit = {
     val metadataResponse = sendMetadataRequest(MetadataRequest.Builder.allTopics.build(1.toShort))
     // Validate rack matches what's set in generateConfigs() above
     metadataResponse.brokers.asScala.foreach { broker =>
@@ -90,7 +90,7 @@ class MetadataRequestTest extends BaseRequestTest {
   }
 
   @Test
-  def testIsInternal() {
+  def testIsInternal(): Unit = {
     val internalTopic = Topic.GROUP_METADATA_TOPIC_NAME
     val notInternalTopic = "notInternal"
     // create the topics
@@ -111,7 +111,7 @@ class MetadataRequestTest extends BaseRequestTest {
   }
 
   @Test
-  def testNoTopicsRequest() {
+  def testNoTopicsRequest(): Unit = {
     // create some topics
     createTopic("t1", 3, 2)
     createTopic("t2", 3, 2)
@@ -197,7 +197,7 @@ class MetadataRequestTest extends BaseRequestTest {
   }
 
   @Test
-  def testAllTopicsRequest() {
+  def testAllTopicsRequest(): Unit = {
     // create some topics
     createTopic("t1", 3, 2)
     createTopic("t2", 3, 2)
@@ -250,7 +250,7 @@ class MetadataRequestTest extends BaseRequestTest {
   }
 
   @Test
-  def testReplicaDownResponse() {
+  def testReplicaDownResponse(): Unit = {
     val replicaDownTopic = "replicaDown"
     val replicaCount = 3
 
diff --git a/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala b/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala
index 21644fab4d6ae..a02e6315a3f7d 100644
--- a/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala
@@ -39,7 +39,7 @@ import scala.collection.JavaConverters._
 class ProduceRequestTest extends BaseRequestTest {
 
   @Test
-  def testSimpleProduceRequest() {
+  def testSimpleProduceRequest(): Unit = {
     val (partition, leader) = createTopicAndFindPartitionWithLeader("topic")
 
     def sendAndCheck(memoryRecords: MemoryRecords, expectedOffset: Long): ProduceResponse.PartitionResponse = {
@@ -65,7 +65,7 @@ class ProduceRequestTest extends BaseRequestTest {
   }
 
   @Test
-  def testProduceToNonReplica() {
+  def testProduceToNonReplica(): Unit = {
     val topic = "topic"
     val partition = 0
 
@@ -96,7 +96,7 @@ class ProduceRequestTest extends BaseRequestTest {
   }
 
   @Test
-  def testCorruptLz4ProduceRequest() {
+  def testCorruptLz4ProduceRequest(): Unit = {
     val (partition, leader) = createTopicAndFindPartitionWithLeader("topic")
     val timestamp = 1000000
     val memoryRecords = MemoryRecords.withRecords(CompressionType.LZ4,
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaFetchTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaFetchTest.scala
index 3a63b3ab94ee5..f6f7ff129823e 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaFetchTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaFetchTest.scala
@@ -33,20 +33,20 @@ class ReplicaFetchTest extends ZooKeeperTestHarness  {
   val topic2 = "bar"
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
     val props = createBrokerConfigs(2, zkConnect)
     brokers = props.map(KafkaConfig.fromProps).map(TestUtils.createServer(_))
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     TestUtils.shutdownServers(brokers)
     super.tearDown()
   }
 
   @Test
-  def testReplicaFetcherThread() {
+  def testReplicaFetcherThread(): Unit = {
     val partition = 0
     val testMessageList1 = List("test1", "test2", "test3", "test4")
     val testMessageList2 = List("test5", "test6", "test7", "test8")
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index 196f2dd34d9ab..a4d32c3fc2e8a 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -70,19 +70,19 @@ class ReplicaManagerTest {
   val brokerEpoch = 0L
 
   @Before
-  def setUp() {
+  def setUp(): Unit = {
     kafkaZkClient = EasyMock.createMock(classOf[KafkaZkClient])
     EasyMock.expect(kafkaZkClient.getEntityConfigs(EasyMock.anyString(), EasyMock.anyString())).andReturn(new Properties()).anyTimes()
     EasyMock.replay(kafkaZkClient)
   }
 
   @After
-  def tearDown() {
+  def tearDown(): Unit = {
     metrics.close()
   }
 
   @Test
-  def testHighWaterMarkDirectoryMapping() {
+  def testHighWaterMarkDirectoryMapping(): Unit = {
     val props = TestUtils.createBrokerConfig(1, TestUtils.MockZkConnect)
     val config = KafkaConfig.fromProps(props)
     val mockLogMgr = TestUtils.createLogManager(config.logDirs.map(new File(_)))
@@ -101,7 +101,7 @@ class ReplicaManagerTest {
   }
 
   @Test
-  def testHighwaterMarkRelativeDirectoryMapping() {
+  def testHighwaterMarkRelativeDirectoryMapping(): Unit = {
     val props = TestUtils.createBrokerConfig(1, TestUtils.MockZkConnect)
     props.put("log.dir", TestUtils.tempRelativeDir("data").getAbsolutePath)
     val config = KafkaConfig.fromProps(props)
@@ -121,7 +121,7 @@ class ReplicaManagerTest {
   }
 
   @Test
-  def testIllegalRequiredAcks() {
+  def testIllegalRequiredAcks(): Unit = {
     val props = TestUtils.createBrokerConfig(1, TestUtils.MockZkConnect)
     val config = KafkaConfig.fromProps(props)
     val mockLogMgr = TestUtils.createLogManager(config.logDirs.map(new File(_)))
@@ -148,7 +148,7 @@ class ReplicaManagerTest {
   }
 
   @Test
-  def testClearPurgatoryOnBecomingFollower() {
+  def testClearPurgatoryOnBecomingFollower(): Unit = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect)
     props.put("log.dir", TestUtils.tempRelativeDir("data").getAbsolutePath)
     val config = KafkaConfig.fromProps(props)
@@ -414,7 +414,7 @@ class ReplicaManagerTest {
   }
 
   @Test
-  def testFetchBeyondHighWatermarkReturnEmptyResponse() {
+  def testFetchBeyondHighWatermarkReturnEmptyResponse(): Unit = {
     val rm = setupReplicaManagerWithMockedPurgatories(new MockTimer, aliveBrokerIds = Seq(0, 1, 2))
     try {
       val brokerList = Seq[Integer](0, 1, 2).asJava
@@ -552,7 +552,7 @@ class ReplicaManagerTest {
    * partition should not be affected.
    */
   @Test
-  def testFetchMessagesWhenNotFollowerForOnePartition() {
+  def testFetchMessagesWhenNotFollowerForOnePartition(): Unit = {
     val replicaManager = setupReplicaManagerWithMockedPurgatories(new MockTimer, aliveBrokerIds = Seq(0, 1, 2))
 
     try {
@@ -637,7 +637,7 @@ class ReplicaManagerTest {
     * if the epoch has increased by more than one (which suggests it has missed an update)
     */
   @Test
-  def testBecomeFollowerWhenLeaderIsUnchangedButMissedLeaderUpdate() {
+  def testBecomeFollowerWhenLeaderIsUnchangedButMissedLeaderUpdate(): Unit = {
     val topicPartition = 0
     val followerBrokerId = 0
     val leaderBrokerId = 1
diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
index d2ad89748f5e6..882a7c2702b15 100644
--- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
+++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
@@ -74,7 +74,7 @@ class RequestQuotaTest extends BaseRequestTest {
   }
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     RequestQuotaTest.principal = KafkaPrincipal.ANONYMOUS
     super.setUp()
 
@@ -112,13 +112,13 @@ class RequestQuotaTest extends BaseRequestTest {
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     try executor.shutdownNow()
     finally super.tearDown()
   }
 
   @Test
-  def testResponseThrottleTime() {
+  def testResponseThrottleTime(): Unit = {
     for (apiKey <- RequestQuotaTest.ClientActions)
       submitTest(apiKey, () => checkRequestThrottleTime(apiKey))
 
@@ -126,21 +126,21 @@ class RequestQuotaTest extends BaseRequestTest {
   }
 
   @Test
-  def testResponseThrottleTimeWhenBothProduceAndRequestQuotasViolated() {
+  def testResponseThrottleTimeWhenBothProduceAndRequestQuotasViolated(): Unit = {
     val apiKey = ApiKeys.PRODUCE
     submitTest(apiKey, () => checkSmallQuotaProducerRequestThrottleTime(apiKey))
     waitAndCheckResults()
   }
 
   @Test
-  def testResponseThrottleTimeWhenBothFetchAndRequestQuotasViolated() {
+  def testResponseThrottleTimeWhenBothFetchAndRequestQuotasViolated(): Unit = {
     val apiKey = ApiKeys.FETCH
     submitTest(apiKey, () => checkSmallQuotaConsumerRequestThrottleTime(apiKey))
     waitAndCheckResults()
   }
 
   @Test
-  def testUnthrottledClient() {
+  def testUnthrottledClient(): Unit = {
     for (apiKey <- RequestQuotaTest.ClientActions)
       submitTest(apiKey, () => checkUnthrottledClient(apiKey))
 
@@ -148,7 +148,7 @@ class RequestQuotaTest extends BaseRequestTest {
   }
 
   @Test
-  def testExemptRequestTime() {
+  def testExemptRequestTime(): Unit = {
     for (apiKey <- RequestQuotaTest.ClusterActions)
       submitTest(apiKey, () => checkExemptRequestMetric(apiKey))
 
@@ -156,7 +156,7 @@ class RequestQuotaTest extends BaseRequestTest {
   }
 
   @Test
-  def testUnauthorizedThrottle() {
+  def testUnauthorizedThrottle(): Unit = {
     RequestQuotaTest.principal = RequestQuotaTest.UnauthorizedPrincipal
 
     for (apiKey <- ApiKeys.values)
@@ -497,16 +497,16 @@ class RequestQuotaTest extends BaseRequestTest {
     }
   }
 
-  private def submitTest(apiKey: ApiKeys, test: () => Unit) {
+  private def submitTest(apiKey: ApiKeys, test: () => Unit): Unit = {
     val future = executor.submit(new Runnable() {
-      def run() {
+      def run(): Unit = {
         test.apply()
       }
     })
     tasks += Task(apiKey, future)
   }
 
-  private def waitAndCheckResults() {
+  private def waitAndCheckResults(): Unit = {
     for (task <- tasks) {
       try {
         task.future.get(15, TimeUnit.SECONDS)
@@ -572,7 +572,7 @@ class RequestQuotaTest extends BaseRequestTest {
     }
   }
 
-  private def checkRequestThrottleTime(apiKey: ApiKeys) {
+  private def checkRequestThrottleTime(apiKey: ApiKeys): Unit = {
 
     // Request until throttled using client-id with default small quota
     val clientId = apiKey.toString
@@ -587,7 +587,7 @@ class RequestQuotaTest extends BaseRequestTest {
     assertTrue(s"Throttle time metrics not updated: $client" , throttleTimeMetricValue(clientId) > 0)
   }
 
-  private def checkSmallQuotaProducerRequestThrottleTime(apiKey: ApiKeys) {
+  private def checkSmallQuotaProducerRequestThrottleTime(apiKey: ApiKeys): Unit = {
 
     // Request until throttled using client-id with default small producer quota
     val smallQuotaProducerClient = Client(smallQuotaProducerClientId, apiKey)
@@ -600,7 +600,7 @@ class RequestQuotaTest extends BaseRequestTest {
       throttleTimeMetricValueForQuotaType(smallQuotaProducerClientId, QuotaType.Request).isNaN)
   }
 
-  private def checkSmallQuotaConsumerRequestThrottleTime(apiKey: ApiKeys) {
+  private def checkSmallQuotaConsumerRequestThrottleTime(apiKey: ApiKeys): Unit = {
 
     // Request until throttled using client-id with default small consumer quota
     val smallQuotaConsumerClient =   Client(smallQuotaConsumerClientId, apiKey)
@@ -613,7 +613,7 @@ class RequestQuotaTest extends BaseRequestTest {
       throttleTimeMetricValueForQuotaType(smallQuotaConsumerClientId, QuotaType.Request).isNaN)
   }
 
-  private def checkUnthrottledClient(apiKey: ApiKeys) {
+  private def checkUnthrottledClient(apiKey: ApiKeys): Unit = {
 
     // Test that request from client with large quota is not throttled
     val unthrottledClient = Client(unthrottledClientId, apiKey)
@@ -622,7 +622,7 @@ class RequestQuotaTest extends BaseRequestTest {
     assertTrue(s"Client should not have been throttled: $unthrottledClient", throttleTimeMetricValue(unthrottledClientId).isNaN)
   }
 
-  private def checkExemptRequestMetric(apiKey: ApiKeys) {
+  private def checkExemptRequestMetric(apiKey: ApiKeys): Unit = {
     val exemptTarget = exemptRequestMetricValue + 0.02
     val clientId = apiKey.toString
     val client = Client(clientId, apiKey)
@@ -632,7 +632,7 @@ class RequestQuotaTest extends BaseRequestTest {
     assertTrue(s"Client should not have been throttled: $client", throttleTimeMetricValue(clientId).isNaN)
   }
 
-  private def checkUnauthorizedRequestThrottle(apiKey: ApiKeys) {
+  private def checkUnauthorizedRequestThrottle(apiKey: ApiKeys): Unit = {
     val clientId = "unauthorized-" + apiKey.toString
     val client = Client(clientId, apiKey)
     val throttled = client.runUntil(response => throttleTimeMetricValue(clientId) > 0.0)
diff --git a/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala b/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala
index 7dcc96baaf7d2..675cc900805e9 100644
--- a/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala
@@ -51,7 +51,7 @@ class SaslApiVersionsRequestTest extends BaseRequestTest with SaslSetup {
   }
 
   @Test
-  def testApiVersionsRequestBeforeSaslHandshakeRequest() {
+  def testApiVersionsRequestBeforeSaslHandshakeRequest(): Unit = {
     val plaintextSocket = connect(protocol = securityProtocol)
     try {
       val apiVersionsResponse = sendApiVersionsRequest(plaintextSocket, new ApiVersionsRequest.Builder().build(0))
@@ -63,7 +63,7 @@ class SaslApiVersionsRequestTest extends BaseRequestTest with SaslSetup {
   }
 
   @Test
-  def testApiVersionsRequestAfterSaslHandshakeRequest() {
+  def testApiVersionsRequestAfterSaslHandshakeRequest(): Unit = {
     val plaintextSocket = connect(protocol = securityProtocol)
     try {
       sendSaslHandshakeRequestValidateResponse(plaintextSocket)
@@ -75,7 +75,7 @@ class SaslApiVersionsRequestTest extends BaseRequestTest with SaslSetup {
   }
 
   @Test
-  def testApiVersionsRequestWithUnsupportedVersion() {
+  def testApiVersionsRequestWithUnsupportedVersion(): Unit = {
     val plaintextSocket = connect(protocol = securityProtocol)
     try {
       val apiVersionsRequest = new ApiVersionsRequest(0)
@@ -95,7 +95,7 @@ class SaslApiVersionsRequestTest extends BaseRequestTest with SaslSetup {
     ApiVersionsResponse.parse(response, request.version)
   }
 
-  private def sendSaslHandshakeRequestValidateResponse(socket: Socket) {
+  private def sendSaslHandshakeRequestValidateResponse(socket: Socket): Unit = {
     val request = new SaslHandshakeRequest(new SaslHandshakeRequestData().setMechanism("PLAIN"))
     val response = sendAndReceive(request, ApiKeys.SASL_HANDSHAKE, socket)
     val handshakeResponse = SaslHandshakeResponse.parse(response, request.version)
diff --git a/core/src/test/scala/unit/kafka/server/ServerGenerateBrokerIdTest.scala b/core/src/test/scala/unit/kafka/server/ServerGenerateBrokerIdTest.scala
index d5e6cb16ab34c..3b6e013780170 100755
--- a/core/src/test/scala/unit/kafka/server/ServerGenerateBrokerIdTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ServerGenerateBrokerIdTest.scala
@@ -38,7 +38,7 @@ class ServerGenerateBrokerIdTest extends ZooKeeperTestHarness {
   var servers: Seq[KafkaServer] = Seq()
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
     props1 = TestUtils.createBrokerConfig(-1, zkConnect)
     config1 = KafkaConfig.fromProps(props1)
@@ -47,13 +47,13 @@ class ServerGenerateBrokerIdTest extends ZooKeeperTestHarness {
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     TestUtils.shutdownServers(servers)
     super.tearDown()
   }
 
   @Test
-  def testAutoGenerateBrokerId() {
+  def testAutoGenerateBrokerId(): Unit = {
     var server1 = new KafkaServer(config1, threadNamePrefix = Option(this.getClass.getName))
     server1.startup()
     server1.shutdown()
@@ -67,7 +67,7 @@ class ServerGenerateBrokerIdTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testUserConfigAndGeneratedBrokerId() {
+  def testUserConfigAndGeneratedBrokerId(): Unit = {
     // start the server with broker.id as part of config
     val server1 = new KafkaServer(config1, threadNamePrefix = Option(this.getClass.getName))
     val server2 = new KafkaServer(config2, threadNamePrefix = Option(this.getClass.getName))
@@ -88,7 +88,7 @@ class ServerGenerateBrokerIdTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testDisableGeneratedBrokerId() {
+  def testDisableGeneratedBrokerId(): Unit = {
     val props3 = TestUtils.createBrokerConfig(3, zkConnect)
     props3.put(KafkaConfig.BrokerIdGenerationEnableProp, "false")
     // Set reserve broker ids to cause collision and ensure disabling broker id generation ignores the setting
@@ -103,7 +103,7 @@ class ServerGenerateBrokerIdTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testMultipleLogDirsMetaProps() {
+  def testMultipleLogDirsMetaProps(): Unit = {
     // add multiple logDirs and check if the generate brokerId is stored in all of them
     val logDirs = props1.getProperty("log.dir")+ "," + TestUtils.tempDir().getAbsolutePath +
     "," + TestUtils.tempDir().getAbsolutePath
@@ -127,7 +127,7 @@ class ServerGenerateBrokerIdTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testConsistentBrokerIdFromUserConfigAndMetaProps() {
+  def testConsistentBrokerIdFromUserConfigAndMetaProps(): Unit = {
     // check if configured brokerId and stored brokerId are equal or throw InconsistentBrokerException
     var server1 = new KafkaServer(config1, threadNamePrefix = Option(this.getClass.getName)) //auto generate broker Id
     server1.startup()
@@ -144,7 +144,7 @@ class ServerGenerateBrokerIdTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testBrokerMetadataOnIdCollision() {
+  def testBrokerMetadataOnIdCollision(): Unit = {
     // Start a good server
     val propsA = TestUtils.createBrokerConfig(1, zkConnect)
     val configA = KafkaConfig.fromProps(propsA)
diff --git a/core/src/test/scala/unit/kafka/server/ServerGenerateClusterIdTest.scala b/core/src/test/scala/unit/kafka/server/ServerGenerateClusterIdTest.scala
index 4581e23c0349b..40988455daedb 100755
--- a/core/src/test/scala/unit/kafka/server/ServerGenerateClusterIdTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ServerGenerateClusterIdTest.scala
@@ -40,7 +40,7 @@ class ServerGenerateClusterIdTest extends ZooKeeperTestHarness {
   val brokerMetaPropsFile = "meta.properties"
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
     config1 = KafkaConfig.fromProps(TestUtils.createBrokerConfig(1, zkConnect))
     config2 = KafkaConfig.fromProps(TestUtils.createBrokerConfig(2, zkConnect))
@@ -48,14 +48,14 @@ class ServerGenerateClusterIdTest extends ZooKeeperTestHarness {
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     TestUtils.shutdownServers(servers)
     super.tearDown()
   }
 
 
   @Test
-  def testAutoGenerateClusterId() {
+  def testAutoGenerateClusterId(): Unit = {
     // Make sure that the cluster id doesn't exist yet.
     assertFalse(zkClient.getClusterId.isDefined)
 
@@ -89,7 +89,7 @@ class ServerGenerateClusterIdTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testAutoGenerateClusterIdForKafkaClusterSequential() {
+  def testAutoGenerateClusterIdForKafkaClusterSequential(): Unit = {
     val server1 = TestUtils.createServer(config1)
     val clusterIdFromServer1 = server1.clusterId
 
@@ -119,7 +119,7 @@ class ServerGenerateClusterIdTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testAutoGenerateClusterIdForKafkaClusterParallel() {
+  def testAutoGenerateClusterIdForKafkaClusterParallel(): Unit = {
     val firstBoot = Future.traverse(Seq(config1, config2, config3))(config => Future(TestUtils.createServer(config)))
     servers = Await.result(firstBoot, 100 second)
     val Seq(server1, server2, server3) = servers
@@ -183,7 +183,7 @@ class ServerGenerateClusterIdTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testInconsistentBrokerMetadataBetweenMultipleLogDirs() {
+  def testInconsistentBrokerMetadataBetweenMultipleLogDirs(): Unit = {
     // Add multiple logDirs with different BrokerMetadata
     val logDir1 = TestUtils.tempDir().getAbsolutePath
     val logDir2 = TestUtils.tempDir().getAbsolutePath
@@ -208,13 +208,13 @@ class ServerGenerateClusterIdTest extends ZooKeeperTestHarness {
     TestUtils.verifyNonDaemonThreadsStatus(this.getClass.getName)
   }
 
-  def forgeBrokerMetadata(logDirs: Seq[String], brokerId: Int, clusterId: String) {
+  def forgeBrokerMetadata(logDirs: Seq[String], brokerId: Int, clusterId: String): Unit = {
     for (logDir <- logDirs) {
       forgeBrokerMetadata(logDir, brokerId, clusterId)
     }
   }
 
-  def forgeBrokerMetadata(logDir: String, brokerId: Int, clusterId: String) {
+  def forgeBrokerMetadata(logDir: String, brokerId: Int, clusterId: String): Unit = {
     val checkpoint = new BrokerMetadataCheckpoint(
       new File(logDir + File.separator + brokerMetaPropsFile))
     checkpoint.write(BrokerMetadata(brokerId, Option(clusterId)))
diff --git a/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala b/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala
index f82801eacbf4a..a5883ca0a1029 100755
--- a/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala
@@ -51,14 +51,14 @@ class ServerShutdownTest extends ZooKeeperTestHarness {
   val sent2 = List("more", "messages")
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
     val props = TestUtils.createBrokerConfig(0, zkConnect)
     config = KafkaConfig.fromProps(props)
   }
 
   @Test
-  def testCleanShutdown() {
+  def testCleanShutdown(): Unit = {
 
     def createProducer(server: KafkaServer): KafkaProducer[Integer, String] =
       TestUtils.createProducer(
@@ -122,7 +122,7 @@ class ServerShutdownTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testCleanShutdownWithDeleteTopicEnabled() {
+  def testCleanShutdownWithDeleteTopicEnabled(): Unit = {
     val newProps = TestUtils.createBrokerConfig(0, zkConnect)
     newProps.setProperty("delete.topic.enable", "true")
     val newConfig = KafkaConfig.fromProps(newProps)
@@ -135,7 +135,7 @@ class ServerShutdownTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testCleanShutdownAfterFailedStartup() {
+  def testCleanShutdownAfterFailedStartup(): Unit = {
     val newProps = TestUtils.createBrokerConfig(0, zkConnect)
     newProps.setProperty(KafkaConfig.ZkConnectionTimeoutMsProp, "50")
     newProps.setProperty(KafkaConfig.ZkConnectProp, "some.invalid.hostname.foo.bar.local:65535")
@@ -144,7 +144,7 @@ class ServerShutdownTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testCleanShutdownAfterFailedStartupDueToCorruptLogs() {
+  def testCleanShutdownAfterFailedStartupDueToCorruptLogs(): Unit = {
     val server = new KafkaServer(config)
     server.startup()
     createTopic(zkClient, topic, numPartitions = 1, replicationFactor = 1, servers = Seq(server))
@@ -157,7 +157,7 @@ class ServerShutdownTest extends ZooKeeperTestHarness {
     verifyCleanShutdownAfterFailedStartup[KafkaStorageException](config)
   }
 
-  private def verifyCleanShutdownAfterFailedStartup[E <: Exception](config: KafkaConfig)(implicit exceptionClassTag: ClassTag[E]) {
+  private def verifyCleanShutdownAfterFailedStartup[E <: Exception](config: KafkaConfig)(implicit exceptionClassTag: ClassTag[E]): Unit = {
     val server = new KafkaServer(config, threadNamePrefix = Option(this.getClass.getName))
     try {
       server.startup()
@@ -184,14 +184,14 @@ class ServerShutdownTest extends ZooKeeperTestHarness {
     !t.isDaemon && t.isAlive && t.getName.startsWith(this.getClass.getName)
   }
 
-  def verifyNonDaemonThreadsStatus() {
+  def verifyNonDaemonThreadsStatus(): Unit = {
     assertEquals(0, Thread.getAllStackTraces.keySet.toArray
       .map(_.asInstanceOf[Thread])
       .count(isNonDaemonKafkaThread))
   }
 
   @Test
-  def testConsecutiveShutdown(){
+  def testConsecutiveShutdown(): Unit = {
     val server = new KafkaServer(config)
     server.startup()
     server.shutdown()
diff --git a/core/src/test/scala/unit/kafka/server/ServerStartupTest.scala b/core/src/test/scala/unit/kafka/server/ServerStartupTest.scala
index 1bc025786367d..39795809587db 100755
--- a/core/src/test/scala/unit/kafka/server/ServerStartupTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ServerStartupTest.scala
@@ -30,7 +30,7 @@ class ServerStartupTest extends ZooKeeperTestHarness {
   private var server: KafkaServer = null
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     if (server != null)
       TestUtils.shutdownServers(Seq(server))
     super.tearDown()
diff --git a/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala b/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala
index 5e9c482e60853..dd15914e54359 100644
--- a/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala
+++ b/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala
@@ -68,7 +68,7 @@ class SimpleFetchTest {
   var replicaManager: ReplicaManager = _
 
   @Before
-  def setUp() {
+  def setUp(): Unit = {
     // create nice mock since we don't particularly care about zkclient calls
     val kafkaZkClient: KafkaZkClient = EasyMock.createNiceMock(classOf[KafkaZkClient])
     EasyMock.replay(kafkaZkClient)
@@ -143,7 +143,7 @@ class SimpleFetchTest {
   }
 
   @After
-  def tearDown() {
+  def tearDown(): Unit = {
     replicaManager.shutdown(false)
     metrics.close()
   }
@@ -165,7 +165,7 @@ class SimpleFetchTest {
    * This test also verifies counts of fetch requests recorded by the ReplicaManager
    */
   @Test
-  def testReadFromLog() {
+  def testReadFromLog(): Unit = {
     val brokerTopicStats = new BrokerTopicStats
     val initialTopicCount = brokerTopicStats.topicStats(topic).totalFetchRequestRate.count()
     val initialAllTopicsCount = brokerTopicStats.allTopicsStats.totalFetchRequestRate.count()
diff --git a/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala b/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala
index c46404addca25..15bbcc6e2b906 100644
--- a/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala
@@ -68,13 +68,13 @@ class ThrottledChannelExpirationTest {
   }
 
   @Before
-  def beforeMethod() {
+  def beforeMethod(): Unit = {
     numCallbacksForStartThrottling = 0
     numCallbacksForEndThrottling = 0
   }
 
   @Test
-  def testCallbackInvocationAfterExpiration() {
+  def testCallbackInvocationAfterExpiration(): Unit = {
     val clientMetrics = new ClientQuotaManager(ClientQuotaManagerConfig(), metrics, QuotaType.Produce, time, "")
 
     val delayQueue = new DelayQueue[ThrottledChannel]()
@@ -107,7 +107,7 @@ class ThrottledChannelExpirationTest {
   }
 
   @Test
-  def testThrottledChannelDelay() {
+  def testThrottledChannelDelay(): Unit = {
     val t1: ThrottledChannel = new ThrottledChannel(request, time, 10, callback)
     val t2: ThrottledChannel = new ThrottledChannel(request, time, 20, callback)
     val t3: ThrottledChannel = new ThrottledChannel(request, time, 20, callback)
diff --git a/core/src/test/scala/unit/kafka/server/epoch/EpochDrivenReplicationProtocolAcceptanceTest.scala b/core/src/test/scala/unit/kafka/server/epoch/EpochDrivenReplicationProtocolAcceptanceTest.scala
index 84de813fde7dd..bd9d02668d8fb 100644
--- a/core/src/test/scala/unit/kafka/server/epoch/EpochDrivenReplicationProtocolAcceptanceTest.scala
+++ b/core/src/test/scala/unit/kafka/server/epoch/EpochDrivenReplicationProtocolAcceptanceTest.scala
@@ -60,12 +60,12 @@ class EpochDrivenReplicationProtocolAcceptanceTest extends ZooKeeperTestHarness
   var consumer: KafkaConsumer[Array[Byte], Array[Byte]] = null
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     producer.close()
     TestUtils.shutdownServers(brokers)
     super.tearDown()
diff --git a/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochFileCacheTest.scala b/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochFileCacheTest.scala
index 000fea72ce4f1..af98b4787f8ce 100644
--- a/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochFileCacheTest.scala
+++ b/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochFileCacheTest.scala
@@ -120,13 +120,13 @@ class LeaderEpochFileCacheTest {
   }
 
   @Test
-  def shouldReturnUnsupportedIfNoEpochRecorded(){
+  def shouldReturnUnsupportedIfNoEpochRecorded(): Unit = {
     //Then
     assertEquals((UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET), cache.endOffsetFor(0))
   }
 
   @Test
-  def shouldReturnUnsupportedIfNoEpochRecordedAndUndefinedEpochRequested(){
+  def shouldReturnUnsupportedIfNoEpochRecordedAndUndefinedEpochRequested(): Unit = {
     logEndOffset = 73
 
     //When (say a follower on older message format version) sends request for UNDEFINED_EPOCH
@@ -138,7 +138,7 @@ class LeaderEpochFileCacheTest {
   }
 
   @Test
-  def shouldReturnFirstEpochIfRequestedEpochLessThanFirstEpoch(){
+  def shouldReturnFirstEpochIfRequestedEpochLessThanFirstEpoch(): Unit = {
     cache.assign(epoch = 5, startOffset = 11)
     cache.assign(epoch = 6, startOffset = 12)
     cache.assign(epoch = 7, startOffset = 13)
@@ -179,7 +179,7 @@ class LeaderEpochFileCacheTest {
   }
 
   @Test
-  def shouldReturnNextAvailableEpochIfThereIsNoExactEpochForTheOneRequested(){
+  def shouldReturnNextAvailableEpochIfThereIsNoExactEpochForTheOneRequested(): Unit = {
     //When
     cache.assign(epoch = 0, startOffset = 10)
     cache.assign(epoch = 2, startOffset = 13)
@@ -226,7 +226,7 @@ class LeaderEpochFileCacheTest {
   }
 
   @Test
-  def shouldPersistEpochsBetweenInstances(){
+  def shouldPersistEpochsBetweenInstances(): Unit = {
     val checkpointPath = TestUtils.tempFile().getAbsolutePath
     val checkpoint = new LeaderEpochCheckpointFile(new File(checkpointPath))
 
diff --git a/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala b/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala
index bdd853c205031..4bf8fcca8e105 100644
--- a/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala
@@ -52,7 +52,7 @@ class LeaderEpochIntegrationTest extends ZooKeeperTestHarness with Logging {
   var producer: KafkaProducer[Array[Byte], Array[Byte]] = null
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     if (producer != null)
       producer.close()
     TestUtils.shutdownServers(brokers)
@@ -60,7 +60,7 @@ class LeaderEpochIntegrationTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def shouldAddCurrentLeaderEpochToMessagesAsTheyAreWrittenToLeader() {
+  def shouldAddCurrentLeaderEpochToMessagesAsTheyAreWrittenToLeader(): Unit = {
     brokers ++= (0 to 1).map { id => createServer(fromProps(createBrokerConfig(id, zkConnect))) }
 
     // Given two topics with replication of a single partition
diff --git a/core/src/test/scala/unit/kafka/server/epoch/util/ReplicaFetcherMockBlockingSend.scala b/core/src/test/scala/unit/kafka/server/epoch/util/ReplicaFetcherMockBlockingSend.scala
index ee2cfbf29ab66..f8c528b29108d 100644
--- a/core/src/test/scala/unit/kafka/server/epoch/util/ReplicaFetcherMockBlockingSend.scala
+++ b/core/src/test/scala/unit/kafka/server/epoch/util/ReplicaFetcherMockBlockingSend.scala
@@ -56,7 +56,7 @@ class ReplicaFetcherMockBlockingSend(offsets: java.util.Map[TopicPartition, Epoc
   var currentOffsets: java.util.Map[TopicPartition, EpochEndOffset] = offsets
   private val sourceNode = new Node(sourceBroker.id, sourceBroker.host, sourceBroker.port)
 
-  def setEpochRequestCallback(postEpochFunction: () => Unit){
+  def setEpochRequestCallback(postEpochFunction: () => Unit): Unit = {
     callback = Some(postEpochFunction)
   }
 
diff --git a/core/src/test/scala/unit/kafka/tools/ConsoleConsumerTest.scala b/core/src/test/scala/unit/kafka/tools/ConsoleConsumerTest.scala
index cdc146f366632..a33c07680ee42 100644
--- a/core/src/test/scala/unit/kafka/tools/ConsoleConsumerTest.scala
+++ b/core/src/test/scala/unit/kafka/tools/ConsoleConsumerTest.scala
@@ -43,7 +43,7 @@ class ConsoleConsumerTest {
   }
 
   @Test
-  def shouldResetUnConsumedOffsetsBeforeExit() {
+  def shouldResetUnConsumedOffsetsBeforeExit(): Unit = {
     val topic = "test"
     val maxMessages: Int = 123
     val totalMessages: Int = 700
@@ -75,7 +75,7 @@ class ConsoleConsumerTest {
   }
 
   @Test
-  def shouldLimitReadsToMaxMessageLimit() {
+  def shouldLimitReadsToMaxMessageLimit(): Unit = {
     val consumer = mock(classOf[ConsumerWrapper])
     val formatter = mock(classOf[MessageFormatter])
     val record = new ConsumerRecord("foo", 1, 1, Array[Byte](), Array[Byte]())
@@ -92,7 +92,7 @@ class ConsoleConsumerTest {
   }
 
   @Test
-  def shouldStopWhenOutputCheckErrorFails() {
+  def shouldStopWhenOutputCheckErrorFails(): Unit = {
     val consumer = mock(classOf[ConsumerWrapper])
     val formatter = mock(classOf[MessageFormatter])
     val printStream = mock(classOf[PrintStream])
@@ -113,7 +113,7 @@ class ConsoleConsumerTest {
   }
 
   @Test
-  def shouldParseValidConsumerValidConfig() {
+  def shouldParseValidConsumerValidConfig(): Unit = {
     //Given
     val args: Array[String] = Array(
       "--bootstrap-server", "localhost:9092",
@@ -170,7 +170,7 @@ class ConsoleConsumerTest {
   }
 
   @Test
-  def shouldParseValidSimpleConsumerValidConfigWithStringOffset() {
+  def shouldParseValidSimpleConsumerValidConfigWithStringOffset(): Unit = {
     //Given
     val args: Array[String] = Array(
       "--bootstrap-server", "localhost:9092",
@@ -192,7 +192,7 @@ class ConsoleConsumerTest {
   }
 
   @Test
-  def shouldParseValidConsumerConfigWithAutoOffsetResetLatest() {
+  def shouldParseValidConsumerConfigWithAutoOffsetResetLatest(): Unit = {
     //Given
     val args: Array[String] = Array(
       "--bootstrap-server", "localhost:9092",
@@ -211,7 +211,7 @@ class ConsoleConsumerTest {
   }
 
   @Test
-  def shouldParseValidConsumerConfigWithAutoOffsetResetEarliest() {
+  def shouldParseValidConsumerConfigWithAutoOffsetResetEarliest(): Unit = {
     //Given
     val args: Array[String] = Array(
       "--bootstrap-server", "localhost:9092",
@@ -230,7 +230,7 @@ class ConsoleConsumerTest {
   }
 
   @Test
-  def shouldParseValidConsumerConfigWithAutoOffsetResetAndMatchingFromBeginning() {
+  def shouldParseValidConsumerConfigWithAutoOffsetResetAndMatchingFromBeginning(): Unit = {
     //Given
     val args: Array[String] = Array(
       "--bootstrap-server", "localhost:9092",
@@ -250,7 +250,7 @@ class ConsoleConsumerTest {
   }
 
   @Test
-  def shouldParseValidConsumerConfigWithNoOffsetReset() {
+  def shouldParseValidConsumerConfigWithNoOffsetReset(): Unit = {
     //Given
     val args: Array[String] = Array(
       "--bootstrap-server", "localhost:9092",
@@ -268,7 +268,7 @@ class ConsoleConsumerTest {
   }
 
   @Test(expected = classOf[IllegalArgumentException])
-  def shouldExitOnInvalidConfigWithAutoOffsetResetAndConflictingFromBeginning() {
+  def shouldExitOnInvalidConfigWithAutoOffsetResetAndConflictingFromBeginning(): Unit = {
     Exit.setExitProcedure((_, message) => throw new IllegalArgumentException(message.orNull))
 
     //Given
@@ -287,7 +287,7 @@ class ConsoleConsumerTest {
   }
 
   @Test
-  def shouldParseConfigsFromFile() {
+  def shouldParseConfigsFromFile(): Unit = {
     val propsFile = TestUtils.tempFile()
     val propsStream = Files.newOutputStream(propsFile.toPath)
     propsStream.write("request.timeout.ms=1000\n".getBytes())
@@ -306,7 +306,7 @@ class ConsoleConsumerTest {
   }
 
   @Test
-  def groupIdsProvidedInDifferentPlacesMustMatch() {
+  def groupIdsProvidedInDifferentPlacesMustMatch(): Unit = {
     Exit.setExitProcedure((_, message) => throw new IllegalArgumentException(message.orNull))
 
     // different in all three places
@@ -433,7 +433,7 @@ class ConsoleConsumerTest {
   }
 
   @Test
-  def shouldParseGroupIdFromBeginningGivenTogether() {
+  def shouldParseGroupIdFromBeginningGivenTogether(): Unit = {
     // Start from earliest
     var args: Array[String] = Array(
       "--bootstrap-server", "localhost:9092",
@@ -462,7 +462,7 @@ class ConsoleConsumerTest {
   }
 
   @Test(expected = classOf[IllegalArgumentException])
-  def shouldExitOnGroupIdAndPartitionGivenTogether() {
+  def shouldExitOnGroupIdAndPartitionGivenTogether(): Unit = {
     Exit.setExitProcedure((_, message) => throw new IllegalArgumentException(message.orNull))
     //Given
     val args: Array[String] = Array(
@@ -480,7 +480,7 @@ class ConsoleConsumerTest {
   }
 
   @Test(expected = classOf[IllegalArgumentException])
-  def shouldExitOnOffsetWithoutPartition() {
+  def shouldExitOnOffsetWithoutPartition(): Unit = {
     Exit.setExitProcedure((_, message) => throw new IllegalArgumentException(message.orNull))
     //Given
     val args: Array[String] = Array(
diff --git a/core/src/test/scala/unit/kafka/tools/ConsoleProducerTest.scala b/core/src/test/scala/unit/kafka/tools/ConsoleProducerTest.scala
index ed2044eeab6f3..00efc6a0ca8c7 100644
--- a/core/src/test/scala/unit/kafka/tools/ConsoleProducerTest.scala
+++ b/core/src/test/scala/unit/kafka/tools/ConsoleProducerTest.scala
@@ -44,7 +44,7 @@ class ConsoleProducerTest {
   )
 
   @Test
-  def testValidConfigs() {
+  def testValidConfigs(): Unit = {
     val config = new ConsoleProducer.ProducerConfig(validArgs)
     val producerConfig = new ProducerConfig(ConsoleProducer.producerProps(config))
     assertEquals(util.Arrays.asList("localhost:1001", "localhost:1002"),
@@ -52,7 +52,7 @@ class ConsoleProducerTest {
   }
 
   @Test(expected = classOf[IllegalArgumentException])
-  def testInvalidConfigs() {
+  def testInvalidConfigs(): Unit = {
     Exit.setExitProcedure((_, message) => throw new IllegalArgumentException(message.orNull))
     try {
       new ConsoleProducer.ProducerConfig(invalidArgs)
diff --git a/core/src/test/scala/unit/kafka/tools/MirrorMakerTest.scala b/core/src/test/scala/unit/kafka/tools/MirrorMakerTest.scala
index a3abd4c10e060..de3016bedc9fc 100644
--- a/core/src/test/scala/unit/kafka/tools/MirrorMakerTest.scala
+++ b/core/src/test/scala/unit/kafka/tools/MirrorMakerTest.scala
@@ -26,7 +26,7 @@ import org.junit.Test
 class MirrorMakerTest {
 
   @Test
-  def testDefaultMirrorMakerMessageHandler() {
+  def testDefaultMirrorMakerMessageHandler(): Unit = {
     val now = 12345L
     val consumerRecord = BaseConsumerRecord("topic", 0, 1L, now, TimestampType.CREATE_TIME, "key".getBytes, "value".getBytes)
 
@@ -42,7 +42,7 @@ class MirrorMakerTest {
   }
 
   @Test
-  def testDefaultMirrorMakerMessageHandlerWithNoTimestampInSourceMessage() {
+  def testDefaultMirrorMakerMessageHandlerWithNoTimestampInSourceMessage(): Unit = {
     val consumerRecord = BaseConsumerRecord("topic", 0, 1L, RecordBatch.NO_TIMESTAMP, TimestampType.CREATE_TIME,
       "key".getBytes, "value".getBytes)
 
@@ -58,7 +58,7 @@ class MirrorMakerTest {
   }
 
   @Test
-  def testDefaultMirrorMakerMessageHandlerWithHeaders() {
+  def testDefaultMirrorMakerMessageHandlerWithHeaders(): Unit = {
     val now = 12345L
     val consumerRecord = BaseConsumerRecord("topic", 0, 1L, now, TimestampType.CREATE_TIME, "key".getBytes,
       "value".getBytes)
diff --git a/core/src/test/scala/unit/kafka/utils/CommandLineUtilsTest.scala b/core/src/test/scala/unit/kafka/utils/CommandLineUtilsTest.scala
index c573fce1bf7b8..2977a2b210c3c 100644
--- a/core/src/test/scala/unit/kafka/utils/CommandLineUtilsTest.scala
+++ b/core/src/test/scala/unit/kafka/utils/CommandLineUtilsTest.scala
@@ -27,21 +27,21 @@ class CommandLineUtilsTest {
 
 
   @Test(expected = classOf[java.lang.IllegalArgumentException])
-  def testParseEmptyArg() {
+  def testParseEmptyArg(): Unit = {
     val argArray = Array("my.empty.property=")
 
     CommandLineUtils.parseKeyValueArgs(argArray, acceptMissingValue = false)
   }
 
   @Test(expected = classOf[java.lang.IllegalArgumentException])
-  def testParseEmptyArgWithNoDelimiter() {
+  def testParseEmptyArgWithNoDelimiter(): Unit = {
     val argArray = Array("my.empty.property")
 
     CommandLineUtils.parseKeyValueArgs(argArray, acceptMissingValue = false)
   }
 
   @Test
-  def testParseEmptyArgAsValid() {
+  def testParseEmptyArgAsValid(): Unit = {
     val argArray = Array("my.empty.property=", "my.empty.property1")
     val props = CommandLineUtils.parseKeyValueArgs(argArray)
 
@@ -50,7 +50,7 @@ class CommandLineUtilsTest {
   }
 
   @Test
-  def testParseSingleArg() {
+  def testParseSingleArg(): Unit = {
     val argArray = Array("my.property=value")
     val props = CommandLineUtils.parseKeyValueArgs(argArray)
 
@@ -58,7 +58,7 @@ class CommandLineUtilsTest {
   }
 
   @Test
-  def testParseArgs() {
+  def testParseArgs(): Unit = {
     val argArray = Array("first.property=first","second.property=second")
     val props = CommandLineUtils.parseKeyValueArgs(argArray)
 
@@ -67,7 +67,7 @@ class CommandLineUtilsTest {
   }
 
   @Test
-  def testParseArgsWithMultipleDelimiters() {
+  def testParseArgsWithMultipleDelimiters(): Unit = {
     val argArray = Array("first.property==first", "second.property=second=", "third.property=thi=rd")
     val props = CommandLineUtils.parseKeyValueArgs(argArray)
 
diff --git a/core/src/test/scala/unit/kafka/utils/CoreUtilsTest.scala b/core/src/test/scala/unit/kafka/utils/CoreUtilsTest.scala
index c1263e00eb15c..dace3a5bb2395 100755
--- a/core/src/test/scala/unit/kafka/utils/CoreUtilsTest.scala
+++ b/core/src/test/scala/unit/kafka/utils/CoreUtilsTest.scala
@@ -41,7 +41,7 @@ class CoreUtilsTest extends Logging {
   val clusterIdPattern = Pattern.compile("[a-zA-Z0-9_\\-]+")
 
   @Test
-  def testSwallow() {
+  def testSwallow(): Unit = {
     CoreUtils.swallow(throw new KafkaException("test"), this, Level.INFO)
   }
 
@@ -96,7 +96,7 @@ class CoreUtilsTest extends Logging {
   }
 
   @Test
-  def testCircularIterator() {
+  def testCircularIterator(): Unit = {
     val l = List(1, 2)
     val itl = CoreUtils.circularIterator(l)
     assertEquals(1, itl.next())
@@ -115,7 +115,7 @@ class CoreUtilsTest extends Logging {
   }
 
   @Test
-  def testReadBytes() {
+  def testReadBytes(): Unit = {
     for(testCase <- List("", "a", "abcd")) {
       val bytes = testCase.getBytes
       assertTrue(Arrays.equals(bytes, Utils.readBytes(ByteBuffer.wrap(bytes))))
@@ -123,7 +123,7 @@ class CoreUtilsTest extends Logging {
   }
 
   @Test
-  def testAbs() {
+  def testAbs(): Unit = {
     assertEquals(0, Utils.abs(Integer.MIN_VALUE))
     assertEquals(1, Utils.abs(-1))
     assertEquals(0, Utils.abs(0))
@@ -132,7 +132,7 @@ class CoreUtilsTest extends Logging {
   }
 
   @Test
-  def testReplaceSuffix() {
+  def testReplaceSuffix(): Unit = {
     assertEquals("blah.foo.text", CoreUtils.replaceSuffix("blah.foo.txt", ".txt", ".text"))
     assertEquals("blah.foo", CoreUtils.replaceSuffix("blah.foo.txt", ".txt", ""))
     assertEquals("txt.txt", CoreUtils.replaceSuffix("txt.txt.txt", ".txt", ""))
@@ -140,7 +140,7 @@ class CoreUtilsTest extends Logging {
   }
 
   @Test
-  def testReadInt() {
+  def testReadInt(): Unit = {
     val values = Array(0, 1, -1, Byte.MaxValue, Short.MaxValue, 2 * Short.MaxValue, Int.MaxValue/2, Int.MinValue/2, Int.MaxValue, Int.MinValue, Int.MaxValue)
     val buffer = ByteBuffer.allocate(4 * values.size)
     for(i <- 0 until values.length) {
@@ -150,7 +150,7 @@ class CoreUtilsTest extends Logging {
   }
 
   @Test
-  def testCsvList() {
+  def testCsvList(): Unit = {
     val emptyString:String = ""
     val nullString:String = null
     val emptyList = CoreUtils.parseCsvList(emptyString)
@@ -163,7 +163,7 @@ class CoreUtilsTest extends Logging {
   }
 
   @Test
-  def testCsvMap() {
+  def testCsvMap(): Unit = {
     val emptyString: String = ""
     val emptyMap = CoreUtils.parseCsvMap(emptyString)
     val emptyStringMap = Map.empty[String, String]
@@ -198,7 +198,7 @@ class CoreUtilsTest extends Logging {
   }
 
   @Test
-  def testInLock() {
+  def testInLock(): Unit = {
     val lock = new ReentrantLock()
     val result = inLock(lock) {
       assertTrue("Should be in lock", lock.isHeldByCurrentThread)
@@ -209,7 +209,7 @@ class CoreUtilsTest extends Logging {
   }
 
   @Test
-  def testUrlSafeBase64EncodeUUID() {
+  def testUrlSafeBase64EncodeUUID(): Unit = {
 
     // Test a UUID that has no + or / characters in base64 encoding [a149b4a3-06e1-4b49-a8cb-8a9c4a59fa46 ->(base64)-> oUm0owbhS0moy4qcSln6Rg==]
     val clusterId1 = Base64.getUrlEncoder.withoutPadding.encodeToString(CoreUtils.getBytesFromUuid(UUID.fromString(
@@ -227,7 +227,7 @@ class CoreUtilsTest extends Logging {
   }
 
   @Test
-  def testGenerateUuidAsBase64() {
+  def testGenerateUuidAsBase64(): Unit = {
     val clusterId = CoreUtils.generateUuidAsBase64()
     assertEquals(clusterId.length, 22)
     assertTrue(clusterIdPattern.matcher(clusterId).matches())
diff --git a/core/src/test/scala/unit/kafka/utils/JaasTestUtils.scala b/core/src/test/scala/unit/kafka/utils/JaasTestUtils.scala
index 65ff4bae89ec4..cc9792e41619f 100644
--- a/core/src/test/scala/unit/kafka/utils/JaasTestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/JaasTestUtils.scala
@@ -267,7 +267,7 @@ object JaasTestUtils {
   private def jaasSectionsToString(jaasSections: Seq[JaasSection]): String =
     jaasSections.mkString
 
-  private def writeToFile(file: File, jaasSections: Seq[JaasSection]) {
+  private def writeToFile(file: File, jaasSections: Seq[JaasSection]): Unit = {
     val writer = new BufferedWriter(new FileWriter(file))
     try writer.write(jaasSectionsToString(jaasSections))
     finally writer.close()
diff --git a/core/src/test/scala/unit/kafka/utils/JsonTest.scala b/core/src/test/scala/unit/kafka/utils/JsonTest.scala
index 209fdee2edc31..4e41bb36b05c2 100644
--- a/core/src/test/scala/unit/kafka/utils/JsonTest.scala
+++ b/core/src/test/scala/unit/kafka/utils/JsonTest.scala
@@ -37,7 +37,7 @@ object JsonTest {
 class JsonTest {
 
   @Test
-  def testJsonParse() {
+  def testJsonParse(): Unit = {
     val jnf = JsonNodeFactory.instance
 
     assertEquals(Json.parseFull("{}"), Some(JsonValue(new ObjectNode(jnf))))
@@ -66,7 +66,7 @@ class JsonTest {
   }
 
   @Test
-  def testLegacyEncodeAsString() {
+  def testLegacyEncodeAsString(): Unit = {
     assertEquals("null", Json.legacyEncodeAsString(null))
     assertEquals("1", Json.legacyEncodeAsString(1))
     assertEquals("1", Json.legacyEncodeAsString(1L))
@@ -88,7 +88,7 @@ class JsonTest {
   }
 
   @Test
-  def testEncodeAsString() {
+  def testEncodeAsString(): Unit = {
     assertEquals("null", Json.encodeAsString(null))
     assertEquals("1", Json.encodeAsString(1))
     assertEquals("1", Json.encodeAsString(1L))
@@ -111,7 +111,7 @@ class JsonTest {
   }
 
   @Test
-  def testEncodeAsBytes() {
+  def testEncodeAsBytes(): Unit = {
     assertEquals("null", new String(Json.encodeAsBytes(null), StandardCharsets.UTF_8))
     assertEquals("1", new String(Json.encodeAsBytes(1), StandardCharsets.UTF_8))
     assertEquals("1", new String(Json.encodeAsBytes(1L), StandardCharsets.UTF_8))
diff --git a/core/src/test/scala/unit/kafka/utils/LogCaptureAppender.scala b/core/src/test/scala/unit/kafka/utils/LogCaptureAppender.scala
index 80472e9b19f52..2d071452829ff 100644
--- a/core/src/test/scala/unit/kafka/utils/LogCaptureAppender.scala
+++ b/core/src/test/scala/unit/kafka/utils/LogCaptureAppender.scala
@@ -37,7 +37,7 @@ class LogCaptureAppender extends AppenderSkeleton {
     }
   }
 
-  override def close(): Unit =  {
+  override def close(): Unit = {
     events.synchronized {
       events.clear()
     }
diff --git a/core/src/test/scala/unit/kafka/utils/MockScheduler.scala b/core/src/test/scala/unit/kafka/utils/MockScheduler.scala
index 523be954ec254..a5d51d066d98c 100644
--- a/core/src/test/scala/unit/kafka/utils/MockScheduler.scala
+++ b/core/src/test/scala/unit/kafka/utils/MockScheduler.scala
@@ -41,9 +41,9 @@ class MockScheduler(val time: Time) extends Scheduler {
   
   def isStarted = true
 
-  def startup() {}
+  def startup(): Unit = {}
   
-  def shutdown() {
+  def shutdown(): Unit = {
     this synchronized {
       tasks.foreach(_.fun())
       tasks.clear()
@@ -55,7 +55,7 @@ class MockScheduler(val time: Time) extends Scheduler {
    * when this method is called and the execution happens synchronously in the calling thread.
    * If you are using the scheduler associated with a MockTime instance this call be triggered automatically.
    */
-  def tick() {
+  def tick(): Unit = {
     this synchronized {
       val now = time.milliseconds
       while(tasks.nonEmpty && tasks.head.nextExecution <= now) {
@@ -107,10 +107,10 @@ case class MockTask(name: String, fun: () => Unit, var nextExecution: Long, peri
     false
   }
 
-  def get() {
+  def get(): Unit = {
   }
 
-  def get(timeout: Long, unit: TimeUnit){
+  def get(timeout: Long, unit: TimeUnit): Unit = {
   }
 
   def isCancelled: Boolean = {
diff --git a/core/src/test/scala/unit/kafka/utils/MockTime.scala b/core/src/test/scala/unit/kafka/utils/MockTime.scala
index 2d83d6547f574..bf0e7bd3816a8 100644
--- a/core/src/test/scala/unit/kafka/utils/MockTime.scala
+++ b/core/src/test/scala/unit/kafka/utils/MockTime.scala
@@ -32,7 +32,7 @@ class MockTime(currentTimeMs: Long, currentHiResTimeNs: Long) extends JMockTime(
 
   val scheduler = new MockScheduler(this)
 
-  override def sleep(ms: Long) {
+  override def sleep(ms: Long): Unit = {
     super.sleep(ms)
     scheduler.tick()
   }
diff --git a/core/src/test/scala/unit/kafka/utils/ReplicationUtilsTest.scala b/core/src/test/scala/unit/kafka/utils/ReplicationUtilsTest.scala
index 4bf747136841d..e8de8d9ecfb08 100644
--- a/core/src/test/scala/unit/kafka/utils/ReplicationUtilsTest.scala
+++ b/core/src/test/scala/unit/kafka/utils/ReplicationUtilsTest.scala
@@ -37,7 +37,7 @@ class ReplicationUtilsTest extends ZooKeeperTestHarness {
   private val isr = List(1, 2)
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     super.setUp()
     zkClient.makeSurePersistentPathExists(TopicZNode.path(topic))
     val topicPartition = new TopicPartition(topic, partition)
@@ -47,7 +47,7 @@ class ReplicationUtilsTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testUpdateLeaderAndIsr() {
+  def testUpdateLeaderAndIsr(): Unit = {
     val configs = TestUtils.createBrokerConfigs(1, zkConnect).map(KafkaConfig.fromProps)
     val log: Log = EasyMock.createMock(classOf[Log])
     EasyMock.expect(log.logEndOffset).andReturn(20).anyTimes()
diff --git a/core/src/test/scala/unit/kafka/utils/SchedulerTest.scala b/core/src/test/scala/unit/kafka/utils/SchedulerTest.scala
index 449d840e86541..201216c6d7291 100644
--- a/core/src/test/scala/unit/kafka/utils/SchedulerTest.scala
+++ b/core/src/test/scala/unit/kafka/utils/SchedulerTest.scala
@@ -32,17 +32,17 @@ class SchedulerTest {
   val counter2 = new AtomicInteger(0)
   
   @Before
-  def setup() {
+  def setup(): Unit = {
     scheduler.startup()
   }
   
   @After
-  def teardown() {
+  def teardown(): Unit = {
     scheduler.shutdown()
   }
 
   @Test
-  def testMockSchedulerNonPeriodicTask() {
+  def testMockSchedulerNonPeriodicTask(): Unit = {
     mockTime.scheduler.schedule("test1", counter1.getAndIncrement _, delay=1)
     mockTime.scheduler.schedule("test2", counter2.getAndIncrement _, delay=100)
     assertEquals("Counter1 should not be incremented prior to task running.", 0, counter1.get)
@@ -56,7 +56,7 @@ class SchedulerTest {
   }
 
   @Test
-  def testMockSchedulerPeriodicTask() {
+  def testMockSchedulerPeriodicTask(): Unit = {
     mockTime.scheduler.schedule("test1", counter1.getAndIncrement _, delay=1, period=1)
     mockTime.scheduler.schedule("test2", counter2.getAndIncrement _, delay=100, period=100)
     assertEquals("Counter1 should not be incremented prior to task running.", 0, counter1.get)
@@ -70,14 +70,14 @@ class SchedulerTest {
   }
 
   @Test
-  def testReentrantTaskInMockScheduler() {
+  def testReentrantTaskInMockScheduler(): Unit = {
     mockTime.scheduler.schedule("test1", () => mockTime.scheduler.schedule("test2", counter2.getAndIncrement _, delay=0), delay=1)
     mockTime.sleep(1)
     assertEquals(1, counter2.get)
   }
 
   @Test
-  def testNonPeriodicTask() {
+  def testNonPeriodicTask(): Unit = {
     scheduler.schedule("test", counter1.getAndIncrement _, delay = 0)
     retry(30000) {
       assertEquals(counter1.get, 1)
@@ -87,7 +87,7 @@ class SchedulerTest {
   }
 
   @Test
-  def testPeriodicTask() {
+  def testPeriodicTask(): Unit = {
     scheduler.schedule("test", counter1.getAndIncrement _, delay = 0, period = 5)
     retry(30000){
       assertTrue("Should count to 20", counter1.get >= 20)
@@ -95,7 +95,7 @@ class SchedulerTest {
   }
 
   @Test
-  def testRestart() {
+  def testRestart(): Unit = {
     // schedule a task to increment a counter
     mockTime.scheduler.schedule("test1", counter1.getAndIncrement _, delay=1)
     mockTime.sleep(1)
diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index 30ca2d8bdb444..59427c1f28010 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -207,7 +207,7 @@ object TestUtils extends Logging {
   /**
     * Shutdown `servers` and delete their log directories.
     */
-  def shutdownServers(servers: Seq[KafkaServer]) {
+  def shutdownServers(servers: Seq[KafkaServer]): Unit = {
     import ExecutionContext.Implicits._
     val future = Future.traverse(servers) { s =>
       Future {
@@ -444,7 +444,7 @@ object TestUtils extends Logging {
   /**
    * Check that the buffer content from buffer.position() to buffer.limit() is equal
    */
-  def checkEquals(b1: ByteBuffer, b2: ByteBuffer) {
+  def checkEquals(b1: ByteBuffer, b2: ByteBuffer): Unit = {
     assertEquals("Buffers should have equal length", b1.limit() - b1.position(), b2.limit() - b2.position())
     for(i <- 0 until b1.limit() - b1.position())
       assertEquals("byte " + i + " byte not equal.", b1.get(b1.position() + i), b2.get(b1.position() + i))
@@ -454,7 +454,7 @@ object TestUtils extends Logging {
    * Throw an exception if the two iterators are of differing lengths or contain
    * different messages on their Nth element
    */
-  def checkEquals[T](expected: Iterator[T], actual: Iterator[T]) {
+  def checkEquals[T](expected: Iterator[T], actual: Iterator[T]): Unit = {
     var length = 0
     while(expected.hasNext && actual.hasNext) {
       length += 1
@@ -486,7 +486,7 @@ object TestUtils extends Logging {
    *  Throw an exception if an iterable has different length than expected
    *
    */
-  def checkLength[T](s1: Iterator[T], expectedLength:Int) {
+  def checkLength[T](s1: Iterator[T], expectedLength:Int): Unit = {
     var n = 0
     while (s1.hasNext) {
       n+=1
@@ -499,7 +499,7 @@ object TestUtils extends Logging {
    * Throw an exception if the two iterators are of differing lengths or contain
    * different messages on their Nth element
    */
-  def checkEquals[T](s1: java.util.Iterator[T], s2: java.util.Iterator[T]) {
+  def checkEquals[T](s1: java.util.Iterator[T], s2: java.util.Iterator[T]): Unit = {
     while(s1.hasNext && s2.hasNext)
       assertEquals(s1.next, s2.next)
     assertFalse("Iterators have uneven length--first has more", s1.hasNext)
@@ -672,7 +672,7 @@ object TestUtils extends Logging {
   def makeLeaderForPartition(zkClient: KafkaZkClient,
                              topic: String,
                              leaderPerPartitionMap: scala.collection.immutable.Map[Int, Int],
-                             controllerEpoch: Int) {
+                             controllerEpoch: Int): Unit = {
     val newLeaderIsrAndControllerEpochs = leaderPerPartitionMap.map { case (partition, leader) =>
       val topicPartition = new TopicPartition(topic, partition)
       val newLeaderAndIsr = zkClient.getTopicPartitionState(topicPartition)
@@ -743,7 +743,7 @@ object TestUtils extends Logging {
    * Execute the given block. If it throws an assert error, retry. Repeat
    * until no error is thrown or the time limit elapses
    */
-  def retry(maxWaitMs: Long)(block: => Unit) {
+  def retry(maxWaitMs: Long)(block: => Unit): Unit = {
     var wait = 1L
     val startTime = System.currentTimeMillis()
     while(true) {
@@ -953,7 +953,7 @@ object TestUtils extends Logging {
     leaderIfExists.get
   }
 
-  def writeNonsenseToFile(fileName: File, position: Long, size: Int) {
+  def writeNonsenseToFile(fileName: File, position: Long, size: Int): Unit = {
     val file = new RandomAccessFile(fileName, "rw")
     file.seek(position)
     for (_ <- 0 until size)
@@ -961,7 +961,7 @@ object TestUtils extends Logging {
     file.close()
   }
 
-  def appendNonsenseToFile(file: File, size: Int) {
+  def appendNonsenseToFile(file: File, size: Int): Unit = {
     val outputStream = Files.newOutputStream(file.toPath(), StandardOpenOption.APPEND)
     try {
       for (_ <- 0 until size)
@@ -969,7 +969,7 @@ object TestUtils extends Logging {
     } finally outputStream.close()
   }
 
-  def checkForPhantomInSyncReplicas(zkClient: KafkaZkClient, topic: String, partitionToBeReassigned: Int, assignedReplicas: Seq[Int]) {
+  def checkForPhantomInSyncReplicas(zkClient: KafkaZkClient, topic: String, partitionToBeReassigned: Int, assignedReplicas: Seq[Int]): Unit = {
     val inSyncReplicas = zkClient.getInSyncReplicasForPartition(new TopicPartition(topic, partitionToBeReassigned))
     // in sync replicas should not have any replica that is not in the new assigned replicas
     val phantomInSyncReplicas = inSyncReplicas.get.toSet -- assignedReplicas.toSet
@@ -978,7 +978,7 @@ object TestUtils extends Logging {
   }
 
   def ensureNoUnderReplicatedPartitions(zkClient: KafkaZkClient, topic: String, partitionToBeReassigned: Int, assignedReplicas: Seq[Int],
-                                                servers: Seq[KafkaServer]) {
+                                                servers: Seq[KafkaServer]): Unit = {
     val topicPartition = new TopicPartition(topic, partitionToBeReassigned)
     waitUntilTrue(() => {
         val inSyncReplicas = zkClient.getInSyncReplicasForPartition(topicPartition)
@@ -998,7 +998,7 @@ object TestUtils extends Logging {
       "Reassigned partition [%s,%d] is under-replicated as reported by the leader %d".format(topic, partitionToBeReassigned, leader.get))
   }
 
-  def verifyNonDaemonThreadsStatus(threadNamePrefix: String) {
+  def verifyNonDaemonThreadsStatus(threadNamePrefix: String): Unit = {
     val threadCount = Thread.getAllStackTraces.keySet.asScala.count { t =>
       !t.isDaemon && t.isAlive && t.getName.startsWith(threadNamePrefix)
     }
@@ -1065,7 +1065,7 @@ object TestUtils extends Logging {
   }
 
   def produceMessage(servers: Seq[KafkaServer], topic: String, message: String,
-                     deliveryTimeoutMs: Int = 30 * 1000, requestTimeoutMs: Int = 20 * 1000) {
+                     deliveryTimeoutMs: Int = 30 * 1000, requestTimeoutMs: Int = 20 * 1000): Unit = {
     val producer = createProducer(TestUtils.getBrokerListStrFromServers(servers),
       deliveryTimeoutMs = deliveryTimeoutMs, requestTimeoutMs = requestTimeoutMs)
     try {
@@ -1075,7 +1075,7 @@ object TestUtils extends Logging {
     }
   }
 
-  def verifyTopicDeletion(zkClient: KafkaZkClient, topic: String, numPartitions: Int, servers: Seq[KafkaServer]) {
+  def verifyTopicDeletion(zkClient: KafkaZkClient, topic: String, numPartitions: Int, servers: Seq[KafkaServer]): Unit = {
     val topicPartitions = (0 until numPartitions).map(new TopicPartition(topic, _))
     // wait until admin path for delete topic is deleted, signaling completion of topic deletion
     waitUntilTrue(() => !zkClient.isTopicMarkedForDeletion(topic),
@@ -1153,9 +1153,9 @@ object TestUtils extends Logging {
       override def getAcceptedIssuers: Array[X509Certificate] = {
         null
       }
-      override def checkClientTrusted(certs: Array[X509Certificate], authType: String) {
+      override def checkClientTrusted(certs: Array[X509Certificate], authType: String): Unit = {
       }
-      override def checkServerTrusted(certs: Array[X509Certificate], authType: String) {
+      override def checkServerTrusted(certs: Array[X509Certificate], authType: String): Unit = {
       }
     }
     trustManager
@@ -1207,7 +1207,7 @@ object TestUtils extends Logging {
   /**
    * Verifies that all secure paths in ZK are created with the expected ACL.
    */
-  def verifySecureZkAcls(zkClient: KafkaZkClient, usersWithAccess: Int) {
+  def verifySecureZkAcls(zkClient: KafkaZkClient, usersWithAccess: Int): Unit = {
     secureZkPaths(zkClient).foreach(path => {
       if (zkClient.pathExists(path)) {
         val sensitive = ZkData.sensitivePath(path)
@@ -1225,7 +1225,7 @@ object TestUtils extends Logging {
    * Verifies that secure paths in ZK have no access control. This is
    * the case when zookeeper.set.acl=false and no ACLs have been configured.
    */
-  def verifyUnsecureZkAcls(zkClient: KafkaZkClient) {
+  def verifyUnsecureZkAcls(zkClient: KafkaZkClient): Unit = {
     secureZkPaths(zkClient).foreach(path => {
       if (zkClient.pathExists(path)) {
         val acls = zkClient.getAcl(path)
@@ -1240,9 +1240,9 @@ object TestUtils extends Logging {
     * They all run at the same time in the assertConcurrent method; the chances of triggering a multithreading code error,
     * and thereby failing some assertion are greatly increased.
     */
-  def assertConcurrent(message: String, functions: Seq[() => Any], timeoutMs: Int) {
+  def assertConcurrent(message: String, functions: Seq[() => Any], timeoutMs: Int): Unit = {
 
-    def failWithTimeout() {
+    def failWithTimeout(): Unit = {
       fail(s"$message. Timed out, the concurrent functions took more than $timeoutMs milliseconds")
     }
 
diff --git a/core/src/test/scala/unit/kafka/utils/ThrottlerTest.scala b/core/src/test/scala/unit/kafka/utils/ThrottlerTest.scala
index d26e791ddf932..61a1e281a5ca5 100755
--- a/core/src/test/scala/unit/kafka/utils/ThrottlerTest.scala
+++ b/core/src/test/scala/unit/kafka/utils/ThrottlerTest.scala
@@ -25,7 +25,7 @@ import org.junit.Assert.{assertTrue, assertEquals}
 
 class ThrottlerTest {
   @Test
-  def testThrottleDesiredRate() {
+  def testThrottleDesiredRate(): Unit = {
     val throttleCheckIntervalMs = 100
     val desiredCountPerSec = 1000.0
     val desiredCountPerInterval = desiredCountPerSec * throttleCheckIntervalMs / 1000.0
diff --git a/core/src/test/scala/unit/kafka/utils/TopicFilterTest.scala b/core/src/test/scala/unit/kafka/utils/TopicFilterTest.scala
index d95a7fc54c647..4b623e978c707 100644
--- a/core/src/test/scala/unit/kafka/utils/TopicFilterTest.scala
+++ b/core/src/test/scala/unit/kafka/utils/TopicFilterTest.scala
@@ -24,7 +24,7 @@ import org.junit.Test
 class TopicFilterTest {
 
   @Test
-  def testWhitelists() {
+  def testWhitelists(): Unit = {
 
     val topicFilter1 = Whitelist("white1,white2")
     assertTrue(topicFilter1.isTopicAllowed("white2", excludeInternalTopics = true))
diff --git a/core/src/test/scala/unit/kafka/utils/timer/MockTimer.scala b/core/src/test/scala/unit/kafka/utils/timer/MockTimer.scala
index 17ee578f6a56d..8805b11243fe8 100644
--- a/core/src/test/scala/unit/kafka/utils/timer/MockTimer.scala
+++ b/core/src/test/scala/unit/kafka/utils/timer/MockTimer.scala
@@ -25,7 +25,7 @@ class MockTimer extends Timer {
   val time = new MockTime
   private val taskQueue = mutable.PriorityQueue[TimerTaskEntry]()(Ordering[TimerTaskEntry].reverse)
 
-  def add(timerTask: TimerTask) {
+  def add(timerTask: TimerTask): Unit = {
     if (timerTask.delayMs <= 0)
       timerTask.run()
     else {
diff --git a/core/src/test/scala/unit/kafka/utils/timer/TimerTaskListTest.scala b/core/src/test/scala/unit/kafka/utils/timer/TimerTaskListTest.scala
index 64129e9f87b2c..0680ec202b6a7 100644
--- a/core/src/test/scala/unit/kafka/utils/timer/TimerTaskListTest.scala
+++ b/core/src/test/scala/unit/kafka/utils/timer/TimerTaskListTest.scala
@@ -33,7 +33,7 @@ class TimerTaskListTest {
   }
 
   @Test
-  def testAll() {
+  def testAll(): Unit = {
     val sharedCounter = new AtomicInteger(0)
     val list1 = new TimerTaskList(sharedCounter)
     val list2 = new TimerTaskList(sharedCounter)
diff --git a/core/src/test/scala/unit/kafka/utils/timer/TimerTest.scala b/core/src/test/scala/unit/kafka/utils/timer/TimerTest.scala
index e25a5cbdc7b3a..7b859b6db4a59 100644
--- a/core/src/test/scala/unit/kafka/utils/timer/TimerTest.scala
+++ b/core/src/test/scala/unit/kafka/utils/timer/TimerTest.scala
@@ -39,7 +39,7 @@ class TimerTest {
   private[this] var timer: Timer = null
 
   @Before
-  def setup() {
+  def setup(): Unit = {
     timer = new SystemTimer("test", tickMs = 1, wheelSize = 3)
   }
 
diff --git a/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala b/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala
index 19c0e9685138a..7e2416ebda6e8 100644
--- a/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala
+++ b/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala
@@ -45,13 +45,13 @@ class AdminZkClientTest extends ZooKeeperTestHarness with Logging with RackAware
   var servers: Seq[KafkaServer] = Seq()
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     TestUtils.shutdownServers(servers)
     super.tearDown()
   }
 
   @Test
-  def testManualReplicaAssignment() {
+  def testManualReplicaAssignment(): Unit = {
     val brokers = List(0, 1, 2, 3, 4)
     TestUtils.createBrokersInZk(zkClient, brokers)
 
@@ -91,7 +91,7 @@ class AdminZkClientTest extends ZooKeeperTestHarness with Logging with RackAware
   }
 
   @Test
-  def testTopicCreationInZK() {
+  def testTopicCreationInZK(): Unit = {
     val expectedReplicaAssignment = Map(
       0  -> List(0, 1, 2),
       1  -> List(1, 2, 3),
@@ -139,7 +139,7 @@ class AdminZkClientTest extends ZooKeeperTestHarness with Logging with RackAware
   }
 
   @Test
-  def testTopicCreationWithCollision() {
+  def testTopicCreationWithCollision(): Unit = {
     val topic = "test.topic"
     val collidingTopic = "test_topic"
     TestUtils.createBrokersInZk(zkClient, List(0, 1, 2, 3, 4))
@@ -153,7 +153,7 @@ class AdminZkClientTest extends ZooKeeperTestHarness with Logging with RackAware
   }
 
   @Test
-  def testMockedConcurrentTopicCreation() {
+  def testMockedConcurrentTopicCreation(): Unit = {
     val topic = "test.topic"
 
     // simulate the ZK interactions that can happen when a topic is concurrently created by multiple processes
@@ -169,7 +169,7 @@ class AdminZkClientTest extends ZooKeeperTestHarness with Logging with RackAware
   }
 
   @Test
-  def testConcurrentTopicCreation() {
+  def testConcurrentTopicCreation(): Unit = {
     val topic = "test-concurrent-topic-creation"
     TestUtils.createBrokersInZk(zkClient, List(0, 1, 2, 3, 4))
     val props = new Properties
@@ -195,7 +195,7 @@ class AdminZkClientTest extends ZooKeeperTestHarness with Logging with RackAware
    * then changes the config and checks that the new values take effect.
    */
   @Test
-  def testTopicConfigChange() {
+  def testTopicConfigChange(): Unit = {
     val partitions = 3
     val topic = "my-topic"
     val server = TestUtils.createServer(KafkaConfig.fromProps(TestUtils.createBrokerConfig(0, zkConnect)))
@@ -210,7 +210,7 @@ class AdminZkClientTest extends ZooKeeperTestHarness with Logging with RackAware
       props
     }
 
-    def checkConfig(messageSize: Int, retentionMs: Long, throttledLeaders: String, throttledFollowers: String, quotaManagerIsThrottled: Boolean) {
+    def checkConfig(messageSize: Int, retentionMs: Long, throttledLeaders: String, throttledFollowers: String, quotaManagerIsThrottled: Boolean): Unit = {
       def checkList(actual: util.List[String], expected: String): Unit = {
         assertNotNull(actual)
         if (expected == "")
@@ -268,11 +268,11 @@ class AdminZkClientTest extends ZooKeeperTestHarness with Logging with RackAware
   }
 
   @Test
-  def shouldPropagateDynamicBrokerConfigs() {
+  def shouldPropagateDynamicBrokerConfigs(): Unit = {
     val brokerIds = Seq(0, 1, 2)
     servers = createBrokerConfigs(3, zkConnect).map(fromProps).map(createServer(_))
 
-    def checkConfig(limit: Long) {
+    def checkConfig(limit: Long): Unit = {
       retry(10000) {
         for (server <- servers) {
           assertEquals("Leader Quota Manager was not updated", limit, server.quotaManagers.leader.upperBound)
@@ -313,7 +313,7 @@ class AdminZkClientTest extends ZooKeeperTestHarness with Logging with RackAware
    * Basically, it asserts that notifications are bootstrapped from ZK
    */
   @Test
-  def testBootstrapClientIdConfig() {
+  def testBootstrapClientIdConfig(): Unit = {
     val clientId = "my-client"
     val props = new Properties()
     props.setProperty("producer_byte_rate", "1000")
@@ -334,7 +334,7 @@ class AdminZkClientTest extends ZooKeeperTestHarness with Logging with RackAware
   }
 
   @Test
-  def testGetBrokerMetadatas() {
+  def testGetBrokerMetadatas(): Unit = {
     // broker 4 has no rack information
     val brokerList = 0 to 5
     val rackInfo = Map(0 -> "rack1", 1 -> "rack2", 2 -> "rack2", 3 -> "rack1", 5 -> "rack3")
diff --git a/core/src/test/scala/unit/kafka/zk/KafkaZkClientTest.scala b/core/src/test/scala/unit/kafka/zk/KafkaZkClientTest.scala
index ddadac0e5b11d..ba8dd7e54dda4 100644
--- a/core/src/test/scala/unit/kafka/zk/KafkaZkClientTest.scala
+++ b/core/src/test/scala/unit/kafka/zk/KafkaZkClientTest.scala
@@ -86,7 +86,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
   private val topicPartition = new TopicPartition("topic", 0)
 
   @Test
-  def testSetAndGetConsumerOffset() {
+  def testSetAndGetConsumerOffset(): Unit = {
     val offset = 123L
     // None if no committed offsets
     assertTrue(zkClient.getConsumerOffset(group, topicPartition).isEmpty)
@@ -99,13 +99,13 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testGetConsumerOffsetNoData() {
+  def testGetConsumerOffsetNoData(): Unit = {
     zkClient.createRecursive(ConsumerOffset.path(group, topicPartition.topic, topicPartition.partition))
     assertTrue(zkClient.getConsumerOffset(group, topicPartition).isEmpty)
   }
 
   @Test
-  def testDeleteRecursive() {
+  def testDeleteRecursive(): Unit = {
     zkClient.deleteRecursive("/delete/does-not-exist")
 
     zkClient.createRecursive("/delete/some/random/path")
@@ -133,7 +133,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testCreateRecursive() {
+  def testCreateRecursive(): Unit = {
     zkClient.createRecursive("/create-newrootpath")
     assertTrue(zkClient.pathExists("/create-newrootpath"))
 
@@ -145,7 +145,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testTopicAssignmentMethods() {
+  def testTopicAssignmentMethods(): Unit = {
     assertTrue(zkClient.getAllTopicsInCluster.isEmpty)
 
     // test with non-existing topic
@@ -194,7 +194,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testGetDataAndVersion() {
+  def testGetDataAndVersion(): Unit = {
     val path = "/testpath"
 
     // test with non-existing path
@@ -218,7 +218,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testConditionalUpdatePath() {
+  def testConditionalUpdatePath(): Unit = {
     val path = "/testconditionalpath"
 
     // test with non-existing path
@@ -389,7 +389,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testSetGetAndDeletePartitionReassignment() {
+  def testSetGetAndDeletePartitionReassignment(): Unit = {
     zkClient.createRecursive(AdminZNode.path)
 
     assertEquals(Map.empty, zkClient.getPartitionReassignment)
@@ -419,7 +419,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testGetDataAndStat() {
+  def testGetDataAndStat(): Unit = {
     val path = "/testpath"
 
     // test with non-existing path
@@ -443,7 +443,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testGetChildren() {
+  def testGetChildren(): Unit = {
     val path = "/testpath"
 
     // test with non-existing path
@@ -461,7 +461,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testAclManagementMethods() {
+  def testAclManagementMethods(): Unit = {
     ZkAclStore.stores.foreach(store => {
       assertFalse(zkClient.pathExists(store.aclPath))
       assertFalse(zkClient.pathExists(store.changeStore.aclChangePath))
@@ -565,7 +565,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testDeleteTopicPathMethods() {
+  def testDeleteTopicPathMethods(): Unit = {
     assertFalse(zkClient.isTopicMarkedForDeletion(topic1))
     assertTrue(zkClient.getTopicDeletions.isEmpty)
 
@@ -599,7 +599,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testEntityConfigManagementMethods() {
+  def testEntityConfigManagementMethods(): Unit = {
     assertTrue(zkClient.getEntityConfigs(ConfigType.Topic, topic1).isEmpty)
 
     zkClient.setOrCreateEntityConfigs(ConfigType.Topic, topic1, logProps)
@@ -1057,7 +1057,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testClusterIdMethods() {
+  def testClusterIdMethods(): Unit = {
     val clusterId = CoreUtils.generateUuidAsBase64
 
     zkClient.createOrGetClusterId(clusterId)
@@ -1065,20 +1065,20 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
    }
 
   @Test
-  def testBrokerSequenceIdMethods() {
+  def testBrokerSequenceIdMethods(): Unit = {
     val sequenceId = zkClient.generateBrokerSequenceId()
     assertEquals(sequenceId + 1, zkClient.generateBrokerSequenceId)
   }
 
   @Test
-  def testCreateTopLevelPaths() {
+  def testCreateTopLevelPaths(): Unit = {
     zkClient.createTopLevelPaths()
 
     ZkData.PersistentZkPaths.foreach(path => assertTrue(zkClient.pathExists(path)))
   }
 
   @Test
-  def testPreferredReplicaElectionMethods() {
+  def testPreferredReplicaElectionMethods(): Unit = {
 
     assertTrue(zkClient.getPreferredReplicaElection.isEmpty)
 
@@ -1105,7 +1105,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testDelegationTokenMethods() {
+  def testDelegationTokenMethods(): Unit = {
     assertFalse(zkClient.pathExists(DelegationTokensZNode.path))
     assertFalse(zkClient.pathExists(DelegationTokenChangeNotificationZNode.path))
 
diff --git a/core/src/test/scala/unit/kafka/zk/ReassignPartitionsZNodeTest.scala b/core/src/test/scala/unit/kafka/zk/ReassignPartitionsZNodeTest.scala
index 2f3456f4b0d0b..3ae237b2a8ac6 100644
--- a/core/src/test/scala/unit/kafka/zk/ReassignPartitionsZNodeTest.scala
+++ b/core/src/test/scala/unit/kafka/zk/ReassignPartitionsZNodeTest.scala
@@ -34,20 +34,20 @@ class ReassignPartitionsZNodeTest {
   private val reassignmentJson = """{"version":1,"partitions":[{"topic":"foo","partition":0,"replicas":[1,2]}]}"""
 
   @Test
-  def testEncode() {
+  def testEncode(): Unit = {
     val encodedJsonString = new String(ReassignPartitionsZNode.encode(reassignPartitionData), StandardCharsets.UTF_8)
     assertEquals(reassignmentJson, encodedJsonString)
   }
 
   @Test
-  def testDecodeInvalidJson() {
+  def testDecodeInvalidJson(): Unit = {
     val result = ReassignPartitionsZNode.decode("invalid json".getBytes)
     assertTrue(result.isLeft)
     assertTrue(result.left.get.isInstanceOf[JsonProcessingException])
   }
 
   @Test
-  def testDecodeValidJson() {
+  def testDecodeValidJson(): Unit = {
     val result = ReassignPartitionsZNode.decode(reassignmentJson.getBytes)
     assertTrue(result.isRight)
     val assignmentMap = result.right.get
diff --git a/core/src/test/scala/unit/kafka/zk/ZkFourLetterWords.scala b/core/src/test/scala/unit/kafka/zk/ZkFourLetterWords.scala
index bd0b257772afc..2930ac80c9314 100644
--- a/core/src/test/scala/unit/kafka/zk/ZkFourLetterWords.scala
+++ b/core/src/test/scala/unit/kafka/zk/ZkFourLetterWords.scala
@@ -28,7 +28,7 @@ import java.net.{SocketTimeoutException, Socket, InetAddress, InetSocketAddress}
   * clients, while "srvr" and "cons" give extended details on server and connections respectively.
   */
 object ZkFourLetterWords {
-  def sendStat(host: String, port: Int, timeout: Int) {
+  def sendStat(host: String, port: Int, timeout: Int): Unit = {
     val hostAddress =
       if (host != null) new InetSocketAddress(host, port)
       else new InetSocketAddress(InetAddress.getByName(null), port)
diff --git a/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala b/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala
index 60822eb364481..3f995d946154b 100755
--- a/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala
+++ b/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala
@@ -53,7 +53,7 @@ abstract class ZooKeeperTestHarness extends Logging {
   def zkConnect: String = s"127.0.0.1:$zkPort"
   
   @Before
-  def setUp() {
+  def setUp(): Unit = {
     zookeeper = new EmbeddedZookeeper()
     zkClient = KafkaZkClient(zkConnect, zkAclsEnabled.getOrElse(JaasUtils.isZkSecurityEnabled), zkSessionTimeout,
       zkConnectionTimeout, zkMaxInFlightRequests, Time.SYSTEM)
@@ -61,7 +61,7 @@ abstract class ZooKeeperTestHarness extends Logging {
   }
 
   @After
-  def tearDown() {
+  def tearDown(): Unit = {
     if (zkClient != null)
      zkClient.close()
     if (zookeeper != null)
@@ -101,7 +101,7 @@ object ZooKeeperTestHarness {
    * which is true for core tests where this harness is used.
    */
   @BeforeClass
-  def setUpClass() {
+  def setUpClass(): Unit = {
     verifyNoUnexpectedThreads("@BeforeClass")
   }
 
@@ -109,7 +109,7 @@ object ZooKeeperTestHarness {
    * Verify that tests from the current test class using ZooKeeperTestHarness haven't left behind an unexpected thread
    */
   @AfterClass
-  def tearDownClass() {
+  def tearDownClass(): Unit = {
     verifyNoUnexpectedThreads("@AfterClass")
   }
 
@@ -117,7 +117,7 @@ object ZooKeeperTestHarness {
    * Verifies that threads which are known to cause transient failures in subsequent tests
    * have been shutdown.
    */
-  def verifyNoUnexpectedThreads(context: String) {
+  def verifyNoUnexpectedThreads(context: String): Unit = {
     def allThreads = Thread.getAllStackTraces.keySet.asScala.map(thread => thread.getName)
     val (threads, noUnexpected) = TestUtils.computeUntilTrue(allThreads) { threads =>
       threads.forall(t => unexpectedThreadNames.forall(s => !t.contains(s)))
diff --git a/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala b/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala
index 7ed76aa2708c3..0587def8c16d5 100644
--- a/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala
+++ b/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala
@@ -45,7 +45,7 @@ class ZooKeeperClientTest extends ZooKeeperTestHarness {
   private var zooKeeperClient: ZooKeeperClient = _
 
   @Before
-  override def setUp() {
+  override def setUp(): Unit = {
     ZooKeeperTestHarness.verifyNoUnexpectedThreads("@Before")
     cleanMetricsRegistry()
     super.setUp()
@@ -54,7 +54,7 @@ class ZooKeeperClientTest extends ZooKeeperTestHarness {
   }
 
   @After
-  override def tearDown() {
+  override def tearDown(): Unit = {
     if (zooKeeperClient != null)
       zooKeeperClient.close()
     super.tearDown()
@@ -614,8 +614,8 @@ class ZooKeeperClientTest extends ZooKeeperTestHarness {
     metricName.getName == name && metricName.getGroup == "testMetricGroup" && metricName.getType == "testMetricType"
 
   @Test
-  def testZooKeeperStateChangeRateMetrics() {
-    def checkMeterCount(name: String, expected: Long) {
+  def testZooKeeperStateChangeRateMetrics(): Unit = {
+    def checkMeterCount(name: String, expected: Long): Unit = {
       val meter = Metrics.defaultRegistry.allMetrics.asScala.collectFirst {
         case (metricName, meter: Meter) if isExpectedMetricName(metricName, name) => meter
       }.getOrElse(sys.error(s"Unable to find meter with name $name"))
@@ -653,7 +653,7 @@ class ZooKeeperClientTest extends ZooKeeperTestHarness {
     assertEquals(States.CLOSED, zooKeeperClient.connectionState)
   }
 
-  private def cleanMetricsRegistry() {
+  private def cleanMetricsRegistry(): Unit = {
     val metrics = Metrics.defaultRegistry
     metrics.allMetrics.keySet.asScala.foreach(metrics.removeMetric)
   }

From b9ffe596864b576d81c3ca0ae060ae4ac9006d0c Mon Sep 17 00:00:00 2001
From: Justine Olshan 
Date: Fri, 16 Aug 2019 16:04:03 -0700
Subject: [PATCH 0545/1071] KAFKA-8601: Add UniformStickyPartitioner and tests
 (#7199)

Reviewers: Colin P. McCabe 
---
 .../producer/UniformStickyPartitioner.java    |  65 ++++++
 .../UniformStickyPartitionerTest.java         | 207 ++++++++++++++++++
 2 files changed, 272 insertions(+)
 create mode 100644 clients/src/main/java/org/apache/kafka/clients/producer/UniformStickyPartitioner.java
 create mode 100644 clients/src/test/java/org/apache/kafka/clients/producer/UniformStickyPartitionerTest.java

diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/UniformStickyPartitioner.java b/clients/src/main/java/org/apache/kafka/clients/producer/UniformStickyPartitioner.java
new file mode 100644
index 0000000000000..3b06be47b146f
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/UniformStickyPartitioner.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.producer;
+
+import java.util.Map;
+
+import org.apache.kafka.clients.producer.internals.StickyPartitionCache;
+import org.apache.kafka.common.Cluster;
+
+
+/**
+ * The partitioning strategy:
+ * 
 
 
            Upgrading from 0.8.x, 0.9.x, 0.10.0.x, 0.10.1.x, 0.10.2.x, 0.11.0.x, 1.0.x, 1.1.x, 2.0.x or 2.1.x or 2.2.x to 2.3.0
            
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/EosIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/EosIntegrationTest.java
index 4f5455b4b158f..7523a62837fda 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/EosIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/EosIntegrationTest.java
@@ -334,7 +334,7 @@ public void shouldNotViolateEosIfOneTaskFails() throws Exception {
 
             TestUtils.waitForCondition(
                 () -> commitRequested.get() == 2, MAX_WAIT_TIME_MS,
-                "SteamsTasks did not request commit.");
+                "StreamsTasks did not request commit.");
 
             writeInputData(uncommittedDataBeforeFailure);
 

From 826408e12204d4260c018ee1da7a327193682ada Mon Sep 17 00:00:00 2001
From: Stanislav Kozlovski 
Date: Wed, 28 Aug 2019 05:12:39 +0100
Subject: [PATCH 0567/1071] MINOR: Initialize BrokerTopicMetrics with no topic
 tag greedily (#7198)

This patch fixes the quota system test whose JMX tool relies on the existence
of these metrics.

Reviewers: Guozhang Wang , Nikhil Bhatia , Tu V. Tran , Ismael Juma 
---
 .../kafka/server/KafkaRequestHandler.scala    | 147 +++++++++++-------
 core/src/main/scala/kafka/utils/Pool.scala    |   6 +-
 .../unit/kafka/metrics/MetricsTest.scala      |  27 +++-
 3 files changed, 117 insertions(+), 63 deletions(-)

diff --git a/core/src/main/scala/kafka/server/KafkaRequestHandler.scala b/core/src/main/scala/kafka/server/KafkaRequestHandler.scala
index f6999d30d8ddf..27962f65f8915 100755
--- a/core/src/main/scala/kafka/server/KafkaRequestHandler.scala
+++ b/core/src/main/scala/kafka/server/KafkaRequestHandler.scala
@@ -28,6 +28,7 @@ import org.apache.kafka.common.internals.FatalExitError
 import org.apache.kafka.common.utils.{KafkaThread, Time}
 
 import scala.collection.mutable
+import scala.collection.JavaConverters._
 
 /**
  * A thread that answers kafka requests.
@@ -146,64 +147,92 @@ class BrokerTopicMetrics(name: Option[String]) extends KafkaMetricsGroup {
     case Some(topic) => Map("topic" -> topic)
   }
 
+  case class MeterWrapper(metricType: String, eventType: String) {
+    @volatile private var lazyMeter: Meter = _
+    private val meterLock = new Object
+
+    def meter(): Meter = {
+      var meter = lazyMeter
+      if (meter == null) {
+        meterLock synchronized {
+          meter = lazyMeter
+          if (meter == null) {
+            meter = newMeter(metricType, eventType, TimeUnit.SECONDS, tags)
+            lazyMeter = meter
+          }
+        }
+      }
+      meter
+    }
+
+    def close(): Unit = meterLock synchronized {
+      if (lazyMeter != null) {
+        removeMetric(metricType, tags)
+        lazyMeter = null
+      }
+    }
+
+    if (tags.isEmpty) // greedily initialize the general topic metrics
+      meter()
+  }
+
   // an internal map for "lazy initialization" of certain metrics
-  private val metricTypeMap = new Pool[String, Meter]
-
-  def messagesInRate = metricTypeMap.getAndMaybePut(BrokerTopicStats.MessagesInPerSec,
-    newMeter(BrokerTopicStats.MessagesInPerSec, "messages", TimeUnit.SECONDS, tags))
-  def bytesInRate = metricTypeMap.getAndMaybePut(BrokerTopicStats.BytesInPerSec,
-    newMeter(BrokerTopicStats.BytesInPerSec, "bytes", TimeUnit.SECONDS, tags))
-  def bytesOutRate = metricTypeMap.getAndMaybePut(BrokerTopicStats.BytesOutPerSec,
-    newMeter(BrokerTopicStats.BytesOutPerSec, "bytes", TimeUnit.SECONDS, tags))
-  def bytesRejectedRate = metricTypeMap.getAndMaybePut(BrokerTopicStats.BytesRejectedPerSec,
-    newMeter(BrokerTopicStats.BytesRejectedPerSec, "bytes", TimeUnit.SECONDS, tags))
+  private val metricTypeMap = new Pool[String, MeterWrapper]()
+  metricTypeMap.putAll(Map(
+    BrokerTopicStats.MessagesInPerSec -> MeterWrapper(BrokerTopicStats.MessagesInPerSec, "messages"),
+    BrokerTopicStats.BytesInPerSec -> MeterWrapper(BrokerTopicStats.BytesInPerSec, "bytes"),
+    BrokerTopicStats.BytesOutPerSec -> MeterWrapper(BrokerTopicStats.BytesOutPerSec, "bytes"),
+    BrokerTopicStats.BytesRejectedPerSec -> MeterWrapper(BrokerTopicStats.BytesRejectedPerSec, "bytes"),
+    BrokerTopicStats.FailedProduceRequestsPerSec -> MeterWrapper(BrokerTopicStats.FailedProduceRequestsPerSec, "requests"),
+    BrokerTopicStats.FailedFetchRequestsPerSec -> MeterWrapper(BrokerTopicStats.FailedFetchRequestsPerSec, "requests"),
+    BrokerTopicStats.TotalProduceRequestsPerSec -> MeterWrapper(BrokerTopicStats.TotalProduceRequestsPerSec, "requests"),
+    BrokerTopicStats.TotalFetchRequestsPerSec -> MeterWrapper(BrokerTopicStats.TotalFetchRequestsPerSec, "requests"),
+    BrokerTopicStats.FetchMessageConversionsPerSec -> MeterWrapper(BrokerTopicStats.FetchMessageConversionsPerSec, "requests"),
+    BrokerTopicStats.ProduceMessageConversionsPerSec -> MeterWrapper(BrokerTopicStats.ProduceMessageConversionsPerSec, "requests")
+  ).asJava)
+  if (name.isEmpty) {
+    metricTypeMap.put(BrokerTopicStats.ReplicationBytesInPerSec, MeterWrapper(BrokerTopicStats.ReplicationBytesInPerSec, "bytes"))
+    metricTypeMap.put(BrokerTopicStats.ReplicationBytesOutPerSec, MeterWrapper(BrokerTopicStats.ReplicationBytesOutPerSec, "bytes"))
+  }
+
+  // used for testing only
+  def metricMap: Map[String, MeterWrapper] = metricTypeMap.toMap
+
+  def messagesInRate = metricTypeMap.get(BrokerTopicStats.MessagesInPerSec).meter()
+
+  def bytesInRate = metricTypeMap.get(BrokerTopicStats.BytesInPerSec).meter()
+
+  def bytesOutRate = metricTypeMap.get(BrokerTopicStats.BytesOutPerSec).meter()
+
+  def bytesRejectedRate = metricTypeMap.get(BrokerTopicStats.BytesRejectedPerSec).meter()
+
   private[server] def replicationBytesInRate =
-    if (name.isEmpty) Some(metricTypeMap.getAndMaybePut(
-      BrokerTopicStats.ReplicationBytesInPerSec,
-      newMeter(BrokerTopicStats.ReplicationBytesInPerSec, "bytes", TimeUnit.SECONDS, tags)))
+    if (name.isEmpty) Some(metricTypeMap.get(BrokerTopicStats.ReplicationBytesInPerSec).meter())
     else None
+
   private[server] def replicationBytesOutRate =
-    if (name.isEmpty) Some(metricTypeMap.getAndMaybePut(
-      BrokerTopicStats.ReplicationBytesOutPerSec,
-      newMeter(BrokerTopicStats.ReplicationBytesOutPerSec, "bytes", TimeUnit.SECONDS, tags)))
+    if (name.isEmpty) Some(metricTypeMap.get(BrokerTopicStats.ReplicationBytesOutPerSec).meter())
     else None
-  def failedProduceRequestRate = metricTypeMap.getAndMaybePut(BrokerTopicStats.FailedProduceRequestsPerSec,
-    newMeter(BrokerTopicStats.FailedProduceRequestsPerSec, "requests", TimeUnit.SECONDS, tags))
-  def failedFetchRequestRate = metricTypeMap.getAndMaybePut(BrokerTopicStats.FailedFetchRequestsPerSec,
-    newMeter(BrokerTopicStats.FailedFetchRequestsPerSec, "requests", TimeUnit.SECONDS, tags))
-  def totalProduceRequestRate = metricTypeMap.getAndMaybePut(BrokerTopicStats.TotalProduceRequestsPerSec,
-    newMeter(BrokerTopicStats.TotalProduceRequestsPerSec, "requests", TimeUnit.SECONDS, tags))
-  def totalFetchRequestRate = metricTypeMap.getAndMaybePut(BrokerTopicStats.TotalFetchRequestsPerSec,
-    newMeter(BrokerTopicStats.TotalFetchRequestsPerSec, "requests", TimeUnit.SECONDS, tags))
-  def fetchMessageConversionsRate = metricTypeMap.getAndMaybePut(BrokerTopicStats.FetchMessageConversionsPerSec,
-    newMeter(BrokerTopicStats.FetchMessageConversionsPerSec, "requests", TimeUnit.SECONDS, tags))
-  def produceMessageConversionsRate = metricTypeMap.getAndMaybePut(BrokerTopicStats.ProduceMessageConversionsPerSec,
-    newMeter(BrokerTopicStats.ProduceMessageConversionsPerSec, "requests", TimeUnit.SECONDS, tags))
-
-  // this method helps check with metricTypeMap first before deleting a metric
-  def removeMetricHelper(metricType: String, tags: scala.collection.Map[String, String]): Unit = {
-    val metric: Meter = metricTypeMap.remove(metricType)
-    if (metric != null) {
-      removeMetric(metricType, tags)
-    }
-  }
 
-  def close(): Unit = {
-    removeMetricHelper(BrokerTopicStats.MessagesInPerSec, tags)
-    removeMetricHelper(BrokerTopicStats.BytesInPerSec, tags)
-    removeMetricHelper(BrokerTopicStats.BytesOutPerSec, tags)
-    removeMetricHelper(BrokerTopicStats.BytesRejectedPerSec, tags)
-    if (replicationBytesInRate.isDefined)
-      removeMetricHelper(BrokerTopicStats.ReplicationBytesInPerSec, tags)
-    if (replicationBytesOutRate.isDefined)
-      removeMetricHelper(BrokerTopicStats.ReplicationBytesOutPerSec, tags)
-    removeMetricHelper(BrokerTopicStats.FailedProduceRequestsPerSec, tags)
-    removeMetricHelper(BrokerTopicStats.FailedFetchRequestsPerSec, tags)
-    removeMetricHelper(BrokerTopicStats.TotalProduceRequestsPerSec, tags)
-    removeMetricHelper(BrokerTopicStats.TotalFetchRequestsPerSec, tags)
-    removeMetricHelper(BrokerTopicStats.FetchMessageConversionsPerSec, tags)
-    removeMetricHelper(BrokerTopicStats.ProduceMessageConversionsPerSec, tags)
+  def failedProduceRequestRate = metricTypeMap.get(BrokerTopicStats.FailedProduceRequestsPerSec).meter()
+
+  def failedFetchRequestRate = metricTypeMap.get(BrokerTopicStats.FailedFetchRequestsPerSec).meter()
+
+  def totalProduceRequestRate = metricTypeMap.get(BrokerTopicStats.TotalProduceRequestsPerSec).meter()
+
+  def totalFetchRequestRate = metricTypeMap.get(BrokerTopicStats.TotalFetchRequestsPerSec).meter()
+
+  def fetchMessageConversionsRate = metricTypeMap.get(BrokerTopicStats.FetchMessageConversionsPerSec).meter()
+
+  def produceMessageConversionsRate = metricTypeMap.get(BrokerTopicStats.ProduceMessageConversionsPerSec).meter()
+
+  def closeMetric(metricType: String): Unit = {
+    val meter = metricTypeMap.get(metricType)
+    if (meter != null)
+      meter.close()
   }
+
+  def close(): Unit = metricTypeMap.values.foreach(_.close())
 }
 
 object BrokerTopicStats {
@@ -247,13 +276,13 @@ class BrokerTopicStats {
   def removeOldLeaderMetrics(topic: String): Unit = {
     val topicMetrics = topicStats(topic)
     if (topicMetrics != null) {
-      topicMetrics.removeMetricHelper(BrokerTopicStats.MessagesInPerSec, topicMetrics.tags)
-      topicMetrics.removeMetricHelper(BrokerTopicStats.BytesInPerSec, topicMetrics.tags)
-      topicMetrics.removeMetricHelper(BrokerTopicStats.BytesRejectedPerSec, topicMetrics.tags)
-      topicMetrics.removeMetricHelper(BrokerTopicStats.FailedProduceRequestsPerSec, topicMetrics.tags)
-      topicMetrics.removeMetricHelper(BrokerTopicStats.TotalProduceRequestsPerSec, topicMetrics.tags)
-      topicMetrics.removeMetricHelper(BrokerTopicStats.ProduceMessageConversionsPerSec, topicMetrics.tags)
-      topicMetrics.removeMetricHelper(BrokerTopicStats.ReplicationBytesOutPerSec, topicMetrics.tags)
+      topicMetrics.closeMetric(BrokerTopicStats.MessagesInPerSec)
+      topicMetrics.closeMetric(BrokerTopicStats.BytesInPerSec)
+      topicMetrics.closeMetric(BrokerTopicStats.BytesRejectedPerSec)
+      topicMetrics.closeMetric(BrokerTopicStats.FailedProduceRequestsPerSec)
+      topicMetrics.closeMetric(BrokerTopicStats.TotalProduceRequestsPerSec)
+      topicMetrics.closeMetric(BrokerTopicStats.ProduceMessageConversionsPerSec)
+      topicMetrics.closeMetric(BrokerTopicStats.ReplicationBytesOutPerSec)
     }
   }
 
@@ -261,7 +290,7 @@ class BrokerTopicStats {
   def removeOldFollowerMetrics(topic: String): Unit = {
     val topicMetrics = topicStats(topic)
     if (topicMetrics != null)
-      topicMetrics.removeMetricHelper(BrokerTopicStats.ReplicationBytesInPerSec, topicMetrics.tags)
+      topicMetrics.closeMetric(BrokerTopicStats.ReplicationBytesInPerSec)
   }
 
   def removeMetrics(topic: String): Unit = {
diff --git a/core/src/main/scala/kafka/utils/Pool.scala b/core/src/main/scala/kafka/utils/Pool.scala
index c963524a9c8c3..9a81ce3f9c50e 100644
--- a/core/src/main/scala/kafka/utils/Pool.scala
+++ b/core/src/main/scala/kafka/utils/Pool.scala
@@ -27,9 +27,11 @@ import collection.JavaConverters._
 class Pool[K,V](valueFactory: Option[K => V] = None) extends Iterable[(K, V)] {
 
   private val pool: ConcurrentMap[K, V] = new ConcurrentHashMap[K, V]
-  
+
   def put(k: K, v: V): V = pool.put(k, v)
-  
+
+  def putAll(map: java.util.Map[K, V]): Unit = pool.putAll(map)
+
   def putIfNotExists(k: K, v: V): V = pool.putIfAbsent(k, v)
 
   /**
diff --git a/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala b/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
index 71226b4b8cb8d..7a3b3a9ac5246 100644
--- a/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
+++ b/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
@@ -75,6 +75,20 @@ class MetricsTest extends KafkaServerTestHarness with Logging {
     assertEquals(metrics.keySet.asScala.count(_.getMBeanName == "kafka.server:type=KafkaServer,name=ClusterId"), 1)
   }
 
+  @Test
+  def testGeneralBrokerTopicMetricsAreGreedilyRegistered(): Unit = {
+    val topic = "test-broker-topic-metric"
+    createTopic(topic, 2, 1)
+
+    // The broker metrics for all topics should be greedily registered
+    assertTrue("General topic metrics don't exist", topicMetrics(None).nonEmpty)
+    assertEquals(servers.head.brokerTopicStats.allTopicsStats.metricMap.size, topicMetrics(None).size)
+    // topic metrics should be lazily registered
+    assertTrue("Topic metrics aren't lazily registered", topicMetricGroups(topic).isEmpty)
+    TestUtils.generateAndProduceMessages(servers, topic, nMessages)
+    assertTrue("Topic metrics aren't registered", topicMetricGroups(topic).nonEmpty)
+  }
+
   @Test
   def testWindowsStyleTagNames(): Unit = {
     val path = "C:\\windows-path\\kafka-logs"
@@ -170,9 +184,18 @@ class MetricsTest extends KafkaServerTestHarness with Logging {
       .count
   }
 
+  private def topicMetrics(topic: Option[String]): Set[String] = {
+    val metricNames = Metrics.defaultRegistry.allMetrics().keySet.asScala.map(_.getMBeanName)
+    filterByTopicMetricRegex(metricNames, topic)
+  }
+
   private def topicMetricGroups(topic: String): Set[String] = {
-    val topicMetricRegex = new Regex(".*BrokerTopicMetrics.*("+topic+")$")
     val metricGroups = Metrics.defaultRegistry.groupedMetrics(MetricPredicate.ALL).keySet.asScala
-    metricGroups.filter(topicMetricRegex.pattern.matcher(_).matches)
+    filterByTopicMetricRegex(metricGroups, Some(topic))
+  }
+
+  private def filterByTopicMetricRegex(metrics: Set[String], topic: Option[String]): Set[String] = {
+    val pattern = (".*BrokerTopicMetrics.*" + topic.map(t => s"($t)$$").getOrElse("")).r.pattern
+    metrics.filter(pattern.matcher(_).matches())
   }
 }

From d32a2d12757e24311cae8f495a3b6f2150f3e041 Mon Sep 17 00:00:00 2001
From: Anastasia Vela 
Date: Tue, 27 Aug 2019 22:23:19 -0700
Subject: [PATCH 0568/1071] KAFKA-8837: KafkaMetricReporterClusterIdTest may
 not shutdown ZooKeeperTestHarness (#7255)

- Call `assertNoNonDaemonThreads` in test method instead of tear down method
to avoid situation where parent's class tear down is not invoked.
- Pass the thread prefix in tests that call `assertNoNonDaemonThreads` so that it
works correctly.
- Rename `verifyNonDaemonThreadsStatus` to `assertNoNonDaemonThreads` to
make it clear that it may throw.

Reviewers: Anna Povzner , Jason Gustafson , Ismael Juma 
---
 .../kafka/server/KafkaServerStartable.scala   | 12 ++++++---
 .../KafkaMetricReporterClusterIdTest.scala    |  6 +++--
 .../kafka/server/ReplicaManagerTest.scala     |  2 +-
 .../server/ServerGenerateBrokerIdTest.scala   | 24 ++++++++---------
 .../server/ServerGenerateClusterIdTest.scala  | 26 +++++++++----------
 .../scala/unit/kafka/utils/TestUtils.scala    | 14 ++++++++--
 6 files changed, 50 insertions(+), 34 deletions(-)

diff --git a/core/src/main/scala/kafka/server/KafkaServerStartable.scala b/core/src/main/scala/kafka/server/KafkaServerStartable.scala
index 515162cbb4928..d4f4c152db0a8 100644
--- a/core/src/main/scala/kafka/server/KafkaServerStartable.scala
+++ b/core/src/main/scala/kafka/server/KafkaServerStartable.scala
@@ -25,14 +25,18 @@ import kafka.utils.{Exit, Logging, VerifiableProperties}
 import scala.collection.Seq
 
 object KafkaServerStartable {
-  def fromProps(serverProps: Properties) = {
+  def fromProps(serverProps: Properties): KafkaServerStartable = {
+    fromProps(serverProps, None)
+  }
+
+  def fromProps(serverProps: Properties, threadNamePrefix: Option[String]): KafkaServerStartable = {
     val reporters = KafkaMetricsReporter.startReporters(new VerifiableProperties(serverProps))
-    new KafkaServerStartable(KafkaConfig.fromProps(serverProps, false), reporters)
+    new KafkaServerStartable(KafkaConfig.fromProps(serverProps, false), reporters, threadNamePrefix)
   }
 }
 
-class KafkaServerStartable(val staticServerConfig: KafkaConfig, reporters: Seq[KafkaMetricsReporter]) extends Logging {
-  private val server = new KafkaServer(staticServerConfig, kafkaMetricsReporters = reporters)
+class KafkaServerStartable(val staticServerConfig: KafkaConfig, reporters: Seq[KafkaMetricsReporter], threadNamePrefix: Option[String] = None) extends Logging {
+  private val server = new KafkaServer(staticServerConfig, kafkaMetricsReporters = reporters, threadNamePrefix = threadNamePrefix)
 
   def this(serverConfig: KafkaConfig) = this(serverConfig, Seq.empty)
 
diff --git a/core/src/test/scala/unit/kafka/server/KafkaMetricReporterClusterIdTest.scala b/core/src/test/scala/unit/kafka/server/KafkaMetricReporterClusterIdTest.scala
index fcba387fc748d..62d99c08b53e0 100755
--- a/core/src/test/scala/unit/kafka/server/KafkaMetricReporterClusterIdTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaMetricReporterClusterIdTest.scala
@@ -88,7 +88,7 @@ class KafkaMetricReporterClusterIdTest extends ZooKeeperTestHarness {
     props.setProperty(KafkaConfig.BrokerIdGenerationEnableProp, "true")
     props.setProperty(KafkaConfig.BrokerIdProp, "-1")
     config = KafkaConfig.fromProps(props)
-    server = KafkaServerStartable.fromProps(props)
+    server = KafkaServerStartable.fromProps(props, threadNamePrefix = Option(this.getClass.getName))
     server.startup()
   }
 
@@ -104,13 +104,15 @@ class KafkaMetricReporterClusterIdTest extends ZooKeeperTestHarness {
 
     assertEquals(KafkaMetricReporterClusterIdTest.MockKafkaMetricsReporter.CLUSTER_META.get().clusterId(),
       KafkaMetricReporterClusterIdTest.MockBrokerMetricsReporter.CLUSTER_META.get().clusterId())
+
+    server.shutdown()
+    TestUtils.assertNoNonDaemonThreads(this.getClass.getName)
   }
 
   @After
   override def tearDown(): Unit = {
     server.shutdown()
     CoreUtils.delete(config.logDirs)
-    TestUtils.verifyNonDaemonThreadsStatus(this.getClass.getName)
     super.tearDown()
   }
 }
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index a4d32c3fc2e8a..857d06ecc50c5 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -144,7 +144,7 @@ class ReplicaManagerTest {
       rm.shutdown(checkpointHW = false)
     }
 
-    TestUtils.verifyNonDaemonThreadsStatus(this.getClass.getName)
+    TestUtils.assertNoNonDaemonThreads(this.getClass.getName)
   }
 
   @Test
diff --git a/core/src/test/scala/unit/kafka/server/ServerGenerateBrokerIdTest.scala b/core/src/test/scala/unit/kafka/server/ServerGenerateBrokerIdTest.scala
index 3b6e013780170..8d0e7bb2d6b15 100755
--- a/core/src/test/scala/unit/kafka/server/ServerGenerateBrokerIdTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ServerGenerateBrokerIdTest.scala
@@ -59,11 +59,11 @@ class ServerGenerateBrokerIdTest extends ZooKeeperTestHarness {
     server1.shutdown()
     assertTrue(verifyBrokerMetadata(config1.logDirs, 1001))
     // restart the server check to see if it uses the brokerId generated previously
-    server1 = TestUtils.createServer(config1)
+    server1 = TestUtils.createServer(config1, threadNamePrefix = Option(this.getClass.getName))
     servers = Seq(server1)
     assertEquals(server1.config.brokerId, 1001)
     server1.shutdown()
-    TestUtils.verifyNonDaemonThreadsStatus(this.getClass.getName)
+    TestUtils.assertNoNonDaemonThreads(this.getClass.getName)
   }
 
   @Test
@@ -72,7 +72,7 @@ class ServerGenerateBrokerIdTest extends ZooKeeperTestHarness {
     val server1 = new KafkaServer(config1, threadNamePrefix = Option(this.getClass.getName))
     val server2 = new KafkaServer(config2, threadNamePrefix = Option(this.getClass.getName))
     val props3 = TestUtils.createBrokerConfig(-1, zkConnect)
-    val server3 = new KafkaServer(KafkaConfig.fromProps(props3))
+    val server3 = new KafkaServer(KafkaConfig.fromProps(props3), threadNamePrefix = Option(this.getClass.getName))
     server1.startup()
     assertEquals(server1.config.brokerId, 1001)
     server2.startup()
@@ -84,7 +84,7 @@ class ServerGenerateBrokerIdTest extends ZooKeeperTestHarness {
     assertTrue(verifyBrokerMetadata(server1.config.logDirs, 1001))
     assertTrue(verifyBrokerMetadata(server2.config.logDirs, 0))
     assertTrue(verifyBrokerMetadata(server3.config.logDirs, 1002))
-    TestUtils.verifyNonDaemonThreadsStatus(this.getClass.getName)
+    TestUtils.assertNoNonDaemonThreads(this.getClass.getName)
   }
 
   @Test
@@ -94,12 +94,12 @@ class ServerGenerateBrokerIdTest extends ZooKeeperTestHarness {
     // Set reserve broker ids to cause collision and ensure disabling broker id generation ignores the setting
     props3.put(KafkaConfig.MaxReservedBrokerIdProp, "0")
     val config3 = KafkaConfig.fromProps(props3)
-    val server3 = TestUtils.createServer(config3)
+    val server3 = TestUtils.createServer(config3, threadNamePrefix = Option(this.getClass.getName))
     servers = Seq(server3)
     assertEquals(server3.config.brokerId, 3)
     server3.shutdown()
     assertTrue(verifyBrokerMetadata(server3.config.logDirs, 3))
-    TestUtils.verifyNonDaemonThreadsStatus(this.getClass.getName)
+    TestUtils.assertNoNonDaemonThreads(this.getClass.getName)
   }
 
   @Test
@@ -123,7 +123,7 @@ class ServerGenerateBrokerIdTest extends ZooKeeperTestHarness {
     servers = Seq(server1)
     server1.shutdown()
     assertTrue(verifyBrokerMetadata(config1.logDirs, 1001))
-    TestUtils.verifyNonDaemonThreadsStatus(this.getClass.getName)
+    TestUtils.assertNoNonDaemonThreads(this.getClass.getName)
   }
 
   @Test
@@ -140,7 +140,7 @@ class ServerGenerateBrokerIdTest extends ZooKeeperTestHarness {
       case _: kafka.common.InconsistentBrokerIdException => //success
     }
     server1.shutdown()
-    TestUtils.verifyNonDaemonThreadsStatus(this.getClass.getName)
+    TestUtils.assertNoNonDaemonThreads(this.getClass.getName)
   }
 
   @Test
@@ -148,12 +148,12 @@ class ServerGenerateBrokerIdTest extends ZooKeeperTestHarness {
     // Start a good server
     val propsA = TestUtils.createBrokerConfig(1, zkConnect)
     val configA = KafkaConfig.fromProps(propsA)
-    val serverA = TestUtils.createServer(configA)
+    val serverA = TestUtils.createServer(configA, threadNamePrefix = Option(this.getClass.getName))
 
     // Start a server that collides on the broker id
     val propsB = TestUtils.createBrokerConfig(1, zkConnect)
     val configB = KafkaConfig.fromProps(propsB)
-    val serverB = new KafkaServer(configB)
+    val serverB = new KafkaServer(configB, threadNamePrefix = Option(this.getClass.getName))
     intercept[NodeExistsException] {
       serverB.startup()
     }
@@ -168,7 +168,7 @@ class ServerGenerateBrokerIdTest extends ZooKeeperTestHarness {
     // adjust the broker config and start again
     propsB.setProperty(KafkaConfig.BrokerIdProp, "2")
     val newConfigB = KafkaConfig.fromProps(propsB)
-    val newServerB = TestUtils.createServer(newConfigB)
+    val newServerB = TestUtils.createServer(newConfigB, threadNamePrefix = Option(this.getClass.getName))
     servers = Seq(serverA, newServerB)
 
     serverA.shutdown()
@@ -177,7 +177,7 @@ class ServerGenerateBrokerIdTest extends ZooKeeperTestHarness {
     // verify correct broker metadata was written
     assertTrue(verifyBrokerMetadata(serverA.config.logDirs, 1))
     assertTrue(verifyBrokerMetadata(newServerB.config.logDirs, 2))
-    TestUtils.verifyNonDaemonThreadsStatus(this.getClass.getName)
+    TestUtils.assertNoNonDaemonThreads(this.getClass.getName)
   }
 
   def verifyBrokerMetadata(logDirs: Seq[String], brokerId: Int): Boolean = {
diff --git a/core/src/test/scala/unit/kafka/server/ServerGenerateClusterIdTest.scala b/core/src/test/scala/unit/kafka/server/ServerGenerateClusterIdTest.scala
index 40988455daedb..66d95fce63c03 100755
--- a/core/src/test/scala/unit/kafka/server/ServerGenerateClusterIdTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ServerGenerateClusterIdTest.scala
@@ -59,7 +59,7 @@ class ServerGenerateClusterIdTest extends ZooKeeperTestHarness {
     // Make sure that the cluster id doesn't exist yet.
     assertFalse(zkClient.getClusterId.isDefined)
 
-    var server1 = TestUtils.createServer(config1)
+    var server1 = TestUtils.createServer(config1, threadNamePrefix = Option(this.getClass.getName))
     servers = Seq(server1)
 
     // Validate the cluster id
@@ -73,7 +73,7 @@ class ServerGenerateClusterIdTest extends ZooKeeperTestHarness {
     assertEquals(zkClient.getClusterId, Some(clusterIdOnFirstBoot))
 
     // Restart the server check to confirm that it uses the clusterId generated previously
-    server1 = TestUtils.createServer(config1)
+    server1 = TestUtils.createServer(config1, threadNamePrefix = Option(this.getClass.getName))
     servers = Seq(server1)
 
     val clusterIdOnSecondBoot = server1.clusterId
@@ -85,18 +85,18 @@ class ServerGenerateClusterIdTest extends ZooKeeperTestHarness {
     assertTrue(zkClient.getClusterId.isDefined)
     assertEquals(zkClient.getClusterId, Some(clusterIdOnFirstBoot))
 
-    TestUtils.verifyNonDaemonThreadsStatus(this.getClass.getName)
+    TestUtils.assertNoNonDaemonThreads(this.getClass.getName)
   }
 
   @Test
   def testAutoGenerateClusterIdForKafkaClusterSequential(): Unit = {
-    val server1 = TestUtils.createServer(config1)
+    val server1 = TestUtils.createServer(config1, threadNamePrefix = Option(this.getClass.getName))
     val clusterIdFromServer1 = server1.clusterId
 
-    val server2 = TestUtils.createServer(config2)
+    val server2 = TestUtils.createServer(config2, threadNamePrefix = Option(this.getClass.getName))
     val clusterIdFromServer2 = server2.clusterId
 
-    val server3 = TestUtils.createServer(config3)
+    val server3 = TestUtils.createServer(config3, threadNamePrefix = Option(this.getClass.getName))
     val clusterIdFromServer3 = server3.clusterId
     servers = Seq(server1, server2, server3)
 
@@ -115,12 +115,12 @@ class ServerGenerateClusterIdTest extends ZooKeeperTestHarness {
 
     servers.foreach(_.shutdown())
 
-    TestUtils.verifyNonDaemonThreadsStatus(this.getClass.getName)
+    TestUtils.assertNoNonDaemonThreads(this.getClass.getName)
   }
 
   @Test
   def testAutoGenerateClusterIdForKafkaClusterParallel(): Unit = {
-    val firstBoot = Future.traverse(Seq(config1, config2, config3))(config => Future(TestUtils.createServer(config)))
+    val firstBoot = Future.traverse(Seq(config1, config2, config3))(config => Future(TestUtils.createServer(config, threadNamePrefix = Option(this.getClass.getName))))
     servers = Await.result(firstBoot, 100 second)
     val Seq(server1, server2, server3) = servers
 
@@ -142,13 +142,13 @@ class ServerGenerateClusterIdTest extends ZooKeeperTestHarness {
 
     servers.foreach(_.shutdown())
 
-    TestUtils.verifyNonDaemonThreadsStatus(this.getClass.getName)
+    TestUtils.assertNoNonDaemonThreads(this.getClass.getName)
   }
 
   @Test
   def testConsistentClusterIdFromZookeeperAndFromMetaProps() = {
     // Check at the first boot
-    val server = TestUtils.createServer(config1)
+    val server = TestUtils.createServer(config1, threadNamePrefix = Option(this.getClass.getName))
     val clusterId = server.clusterId
 
     assertTrue(verifyBrokerMetadata(server.config.logDirs, clusterId))
@@ -163,7 +163,7 @@ class ServerGenerateClusterIdTest extends ZooKeeperTestHarness {
 
     server.shutdown()
 
-    TestUtils.verifyNonDaemonThreadsStatus(this.getClass.getName)
+    TestUtils.assertNoNonDaemonThreads(this.getClass.getName)
   }
 
   @Test
@@ -179,7 +179,7 @@ class ServerGenerateClusterIdTest extends ZooKeeperTestHarness {
 
     server.shutdown()
 
-    TestUtils.verifyNonDaemonThreadsStatus(this.getClass.getName)
+    TestUtils.assertNoNonDaemonThreads(this.getClass.getName)
   }
 
   @Test
@@ -205,7 +205,7 @@ class ServerGenerateClusterIdTest extends ZooKeeperTestHarness {
 
     server.shutdown()
 
-    TestUtils.verifyNonDaemonThreadsStatus(this.getClass.getName)
+    TestUtils.assertNoNonDaemonThreads(this.getClass.getName)
   }
 
   def forgeBrokerMetadata(logDirs: Seq[String], brokerId: Int, clusterId: String): Unit = {
diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index 9091dcb30f7a6..2a27ed4e52cb4 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -143,7 +143,15 @@ object TestUtils extends Logging {
    * @param config The configuration of the server
    */
   def createServer(config: KafkaConfig, time: Time = Time.SYSTEM): KafkaServer = {
-    val server = new KafkaServer(config, time)
+    createServer(config, time, None)
+  }
+
+  def createServer(config: KafkaConfig, threadNamePrefix: Option[String]): KafkaServer = {
+    createServer(config, Time.SYSTEM, threadNamePrefix)
+  }
+
+  def createServer(config: KafkaConfig, time: Time, threadNamePrefix: Option[String]): KafkaServer = {
+    val server = new KafkaServer(config, time, threadNamePrefix = threadNamePrefix)
     server.startup()
     server
   }
@@ -998,7 +1006,9 @@ object TestUtils extends Logging {
       "Reassigned partition [%s,%d] is under-replicated as reported by the leader %d".format(topic, partitionToBeReassigned, leader.get))
   }
 
-  def verifyNonDaemonThreadsStatus(threadNamePrefix: String): Unit = {
+  // Note: Call this method in the test itself, rather than the @After method.
+  // Because of the assert, if assertNoNonDaemonThreads fails, nothing after would be executed.
+  def assertNoNonDaemonThreads(threadNamePrefix: String): Unit = {
     val threadCount = Thread.getAllStackTraces.keySet.asScala.count { t =>
       !t.isDaemon && t.isAlive && t.getName.startsWith(threadNamePrefix)
     }

From fcfee618ee440cb78e422963ba7db1fa03620b33 Mon Sep 17 00:00:00 2001
From: Bill Bejeck 
Date: Wed, 28 Aug 2019 12:22:36 -0400
Subject: [PATCH 0569/1071] MINOR: Only send delete request if there are
 offsets in map (#7256)

Currently on commit streams will attempt to delete offsets from repartition topics. However, if a topology does not have any repartition topics, then the recordsToDelete map will be empty.

This PR adds a check that the recordsToDelete is not empty before executing the AdminClient#deleteRecords() method.

Reviewers: A. Sophie Blee-Goldman , Guozhang Wang 
---
 .../kafka/streams/processor/internals/TaskManager.java     | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
index 6b3ab001a73d9..444d6975e692a 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
@@ -449,9 +449,10 @@ void maybePurgeCommitedRecords() {
             for (final Map.Entry entry : active.recordsToDelete().entrySet()) {
                 recordsToDelete.put(entry.getKey(), RecordsToDelete.beforeOffset(entry.getValue()));
             }
-            deleteRecordsResult = adminClient.deleteRecords(recordsToDelete);
-
-            log.trace("Sent delete-records request: {}", recordsToDelete);
+            if (!recordsToDelete.isEmpty()) {
+                deleteRecordsResult = adminClient.deleteRecords(recordsToDelete);
+                log.trace("Sent delete-records request: {}", recordsToDelete);
+            }
         }
     }
 

From d2741e5cbf546117d9eb3f6776f9063bfc240d9d Mon Sep 17 00:00:00 2001
From: Bruno Cadonna 
Date: Thu, 29 Aug 2019 20:26:44 +0200
Subject: [PATCH 0570/1071] MINOR: Remove `activeTaskCheckpointableOffsets`
 from `AbstractTask` (#7253)

Reviewers: cpettitt-confluent <53191309+cpettitt-confluent@users.noreply.github.com>, A. Sophie Blee-Goldman , Guozhang Wang 
---
 .../kafka/streams/processor/internals/AbstractTask.java    | 6 ------
 .../kafka/streams/processor/internals/StreamTask.java      | 7 ++-----
 2 files changed, 2 insertions(+), 11 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java
index 33878017b7ae1..5656c914f7bd3 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java
@@ -34,9 +34,7 @@
 
 import java.io.IOException;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashSet;
-import java.util.Map;
 import java.util.Set;
 
 public abstract class AbstractTask implements Task {
@@ -173,10 +171,6 @@ public String toString(final String indent) {
         return sb.toString();
     }
 
-    protected Map activeTaskCheckpointableOffsets() {
-        return Collections.emptyMap();
-    }
-
     protected void updateOffsetLimits() {
         for (final TopicPartition partition : partitions) {
             try {
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
index d50d5c23cc6c9..d6ee2dff735ac 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
@@ -484,14 +484,11 @@ void commit(final boolean startNewTransaction) {
         taskMetrics.taskCommitTimeSensor.record(time.nanoseconds() - startNs);
     }
 
-    @Override
-    protected Map activeTaskCheckpointableOffsets() {
-        final Map checkpointableOffsets =
-            new HashMap<>(recordCollector.offsets());
+    private Map activeTaskCheckpointableOffsets() {
+        final Map checkpointableOffsets = new HashMap<>(recordCollector.offsets());
         for (final Map.Entry entry : consumedOffsets.entrySet()) {
             checkpointableOffsets.putIfAbsent(entry.getKey(), entry.getValue());
         }
-
         return checkpointableOffsets;
     }
 

From 09ad6b84c558b2f2790dac3317709f7593c81bc8 Mon Sep 17 00:00:00 2001
From: Vikas Singh <50422828+soondenana@users.noreply.github.com>
Date: Thu, 29 Aug 2019 14:12:08 -0700
Subject: [PATCH 0571/1071] MINOR. Fix 2.3.0 streams systest dockerfile typo
 (#7272)

As part of commit 4d1ee26a136997d31dbd6ddca07e09b34c41c77d streams
version 2.3.0 test jar was added, but there was a simple typo in the
path that specified the version.

`ducker-ak up` was failing because of that. Fixed that.

Reviewers: Guozhang Wang 
---
 tests/docker/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile
index 9745a6ec659c8..66d5c4a6779b9 100644
--- a/tests/docker/Dockerfile
+++ b/tests/docker/Dockerfile
@@ -68,7 +68,7 @@ RUN curl -s "$KAFKA_MIRROR/kafka-streams-1.1.1-test.jar" -o /opt/kafka-1.1.1/lib
 RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.0.1-test.jar" -o /opt/kafka-2.0.1/libs/kafka-streams-2.0.1-test.jar
 RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.1.1-test.jar" -o /opt/kafka-2.1.1/libs/kafka-streams-2.1.1-test.jar
 RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.2.1-test.jar" -o /opt/kafka-2.2.1/libs/kafka-streams-2.2.1-test.jar
-RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.3.0-test.jar" -o /opt/kafka-2.2.0/libs/kafka-streams-2.3.0-test.jar
+RUN curl -s "$KAFKA_MIRROR/kafka-streams-2.3.0-test.jar" -o /opt/kafka-2.3.0/libs/kafka-streams-2.3.0-test.jar
 
 # The version of Kibosh to use for testing.
 # If you update this, also update vagrant/base.sy

From fb381cb6c7c9e1183dfeb88dd94c25134049ece9 Mon Sep 17 00:00:00 2001
From: Lucas Bradstreet 
Date: Thu, 29 Aug 2019 18:02:27 -0700
Subject: [PATCH 0572/1071] MINOR: Fix integer overflow in LRUCacheBenchmark
 (#7270)

The jmh LRUCacheBenchmark will exhibit an int overflow when run on a fast machine:

```
java.lang.ArrayIndexOutOfBoundsException: Index -3648 out of bounds for length 10000
	at org.apache.kafka.jmh.cache.LRUCacheBenchmark.testCachePerformance(LRUCacheBenchmark.java:70)
	at org.apache.kafka.jmh.cache.generated.LRUCacheBenchmark_testCachePerformance_jmhTest.testCachePerformance_thrpt_jmhStub(LRUCacheBenchmark_testCachePerformance_jmhTest.java:119)
	at org.apache.kafka.jmh.cache.generated.LRUCacheBenchmark_testCachePerformance_jmhTest.testCachePerformance_Throughput(LRUCacheBenchmark_testCachePerformance_jmhTest.java:83)
```

Reviewers: Jason Gustafson 
---
 .../java/org/apache/kafka/jmh/cache/LRUCacheBenchmark.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/cache/LRUCacheBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/cache/LRUCacheBenchmark.java
index 7d6597993bf03..5b2d004dcd562 100644
--- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/cache/LRUCacheBenchmark.java
+++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/cache/LRUCacheBenchmark.java
@@ -52,7 +52,7 @@ public class LRUCacheBenchmark {
 
     private LRUCache lruCache;
 
-    int counter;
+    private long counter = 0;
 
     @Setup(Level.Trial)
     public void setUp() {
@@ -66,7 +66,7 @@ public void setUp() {
     @Benchmark
     public String testCachePerformance() {
         counter++;
-        int index = counter % DISTINCT_KEYS;
+        int index = (int) (counter % DISTINCT_KEYS);
         String hashkey = keys[index];
         lruCache.put(hashkey, values[index]);
         return lruCache.get(hashkey);

From d18d6b033e09515adff19225f8ec6845ca34c23b Mon Sep 17 00:00:00 2001
From: Bruno Cadonna 
Date: Fri, 30 Aug 2019 15:46:07 +0200
Subject: [PATCH 0573/1071] MINOR: Refactor tag key for store level metrics
 (#7257)

The tag key for store level metrics specified in StreamsMetricsImpl
is unified with the tag keys on thread and task level.

Reviewers: Sophie Blee-Goldman , Bill Bejeck 
---
 .../internals/metrics/StreamsMetricsImpl.java | 14 +++++++----
 .../internals/metrics/ThreadMetrics.java      |  4 ++--
 .../apache/kafka/streams/state/Stores.java    |  4 ++--
 .../InMemorySessionBytesStoreSupplier.java    |  2 +-
 .../InMemoryWindowBytesStoreSupplier.java     |  2 +-
 .../state/internals/MeteredKeyValueStore.java | 13 +++++-----
 .../state/internals/MeteredSessionStore.java  | 13 +++++-----
 .../state/internals/MeteredWindowStore.java   | 15 +++++++-----
 .../RocksDbKeyValueBytesStoreSupplier.java    |  2 +-
 .../RocksDbSessionBytesStoreSupplier.java     |  2 +-
 .../RocksDbWindowBytesStoreSupplier.java      |  2 +-
 .../internals/metrics/ThreadMetricsTest.java  |  4 ++--
 .../internals/MeteredKeyValueStoreTest.java   | 10 ++++----
 .../internals/MeteredSessionStoreTest.java    |  8 +++----
 .../MeteredTimestampedKeyValueStoreTest.java  | 10 ++++----
 .../internals/MeteredWindowStoreTest.java     | 24 +++++++++----------
 16 files changed, 69 insertions(+), 60 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
index c9a0629313b77..34338e1502d98 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
@@ -57,9 +57,9 @@ public class StreamsMetricsImpl implements StreamsMetrics {
 
     public static final String THREAD_ID_TAG = "client-id";
     public static final String TASK_ID_TAG = "task-id";
-    public static final String STORE_ID_TAG = "id";
+    public static final String STORE_ID_TAG = "state-id";
 
-    public static final String ALL_TASKS = "all";
+    public static final String ROLLUP_VALUE = "all";
 
     public static final String LATENCY_SUFFIX = "-latency";
     public static final String AVG_SUFFIX = "-avg";
@@ -69,9 +69,13 @@ public class StreamsMetricsImpl implements StreamsMetrics {
     public static final String TOTAL_SUFFIX = "-total";
     public static final String RATIO_SUFFIX = "-ratio";
 
-    public static final String THREAD_LEVEL_GROUP = "stream-metrics";
-    public static final String TASK_LEVEL_GROUP = "stream-task-metrics";
-    public static final String STATE_LEVEL_GROUP = "stream-state-metrics";
+    public static final String GROUP_PREFIX_WO_DELIMITER = "stream";
+    public static final String GROUP_PREFIX = GROUP_PREFIX_WO_DELIMITER + "-";
+    public static final String GROUP_SUFFIX = "-metrics";
+    public static final String STATE_LEVEL_GROUP_SUFFIX = "-state" + GROUP_SUFFIX;
+    public static final String THREAD_LEVEL_GROUP = GROUP_PREFIX_WO_DELIMITER + GROUP_SUFFIX;
+    public static final String TASK_LEVEL_GROUP = GROUP_PREFIX + "task" + GROUP_SUFFIX;
+    public static final String STATE_LEVEL_GROUP = GROUP_PREFIX + "state" + GROUP_SUFFIX;
 
     public static final String PROCESSOR_NODE_METRICS_GROUP = "stream-processor-node-metrics";
     public static final String PROCESSOR_NODE_ID_TAG = "processor-node-id";
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetrics.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetrics.java
index f8b7836fee18e..8cf8a8b3772aa 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetrics.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetrics.java
@@ -21,7 +21,7 @@
 
 import java.util.Map;
 
-import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.ALL_TASKS;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.ROLLUP_VALUE;
 import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.LATENCY_SUFFIX;
 import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.TASK_ID_TAG;
 import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.TASK_LEVEL_GROUP;
@@ -162,7 +162,7 @@ public static Sensor skipRecordSensor(final StreamsMetricsImpl streamsMetrics) {
 
     public static Sensor commitOverTasksSensor(final StreamsMetricsImpl streamsMetrics) {
         final Sensor commitOverTasksSensor = streamsMetrics.threadLevelSensor(COMMIT, Sensor.RecordingLevel.DEBUG);
-        final Map tagMap = streamsMetrics.threadLevelTagMap(TASK_ID_TAG, ALL_TASKS);
+        final Map tagMap = streamsMetrics.threadLevelTagMap(TASK_ID_TAG, ROLLUP_VALUE);
         addAvgAndMaxToSensor(commitOverTasksSensor,
                      TASK_LEVEL_GROUP,
                      tagMap,
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/Stores.java b/streams/src/main/java/org/apache/kafka/streams/state/Stores.java
index f92a0de9d4f85..0de427d20de62 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/Stores.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/Stores.java
@@ -135,7 +135,7 @@ public KeyValueStore get() {
 
             @Override
             public String metricsScope() {
-                return "in-memory-state";
+                return "in-memory";
             }
         };
     }
@@ -169,7 +169,7 @@ public KeyValueStore get() {
 
             @Override
             public String metricsScope() {
-                return "in-memory-lru-state";
+                return "in-memory-lru";
             }
         };
     }
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionBytesStoreSupplier.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionBytesStoreSupplier.java
index 18e8971ea7f0e..fa4eddced568b 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionBytesStoreSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionBytesStoreSupplier.java
@@ -42,7 +42,7 @@ public SessionStore get() {
 
     @Override
     public String metricsScope() {
-        return "in-memory-session-state";
+        return "in-memory-session";
     }
 
     // In-memory store is not *really* segmented, so just say it is 1 (for ordering consistency with caching enabled)
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryWindowBytesStoreSupplier.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryWindowBytesStoreSupplier.java
index ace4de0e74ef8..042a5398b086a 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryWindowBytesStoreSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryWindowBytesStoreSupplier.java
@@ -52,7 +52,7 @@ public WindowStore get() {
 
     @Override
     public String metricsScope() {
-        return "in-memory-window-state";
+        return "in-memory-window";
     }
 
     @Deprecated
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStore.java
index 4f2df4c7ff0e4..07e171f9eb813 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStore.java
@@ -35,6 +35,7 @@
 import java.util.Map;
 
 import static org.apache.kafka.common.metrics.Sensor.RecordingLevel.DEBUG;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.ROLLUP_VALUE;
 import static org.apache.kafka.streams.state.internals.metrics.Sensors.createTaskAndStoreLatencyAndThroughputSensors;
 
 /**
@@ -53,7 +54,7 @@ public class MeteredKeyValueStore
     final Serde valueSerde;
     StateSerdes serdes;
 
-    private final String metricScope;
+    private final String metricsScope;
     protected final Time time;
     private Sensor putTime;
     private Sensor putIfAbsentTime;
@@ -67,12 +68,12 @@ public class MeteredKeyValueStore
     private String taskName;
 
     MeteredKeyValueStore(final KeyValueStore inner,
-                         final String metricScope,
+                         final String metricsScope,
                          final Time time,
                          final Serde keySerde,
                          final Serde valueSerde) {
         super(inner);
-        this.metricScope = metricScope;
+        this.metricsScope = metricsScope;
         this.time = time != null ? time : Time.SYSTEM;
         this.keySerde = keySerde;
         this.valueSerde = valueSerde;
@@ -84,9 +85,9 @@ public void init(final ProcessorContext context,
         metrics = (StreamsMetricsImpl) context.metrics();
 
         taskName = context.taskId().toString();
-        final String metricsGroup = "stream-" + metricScope + "-metrics";
-        final Map taskTags = metrics.tagMap("task-id", taskName, metricScope + "-id", "all");
-        final Map storeTags = metrics.tagMap("task-id", taskName, metricScope + "-id", name());
+        final String metricsGroup = "stream-" + metricsScope + "-state-metrics";
+        final Map taskTags = metrics.storeLevelTagMap(taskName, metricsScope, ROLLUP_VALUE);
+        final Map storeTags = metrics.storeLevelTagMap(taskName, metricsScope, name());
 
         initStoreSerde(context);
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java
index 94b004e330e75..09030c23ce460 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java
@@ -34,13 +34,14 @@
 import java.util.Objects;
 
 import static org.apache.kafka.common.metrics.Sensor.RecordingLevel.DEBUG;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.ROLLUP_VALUE;
 import static org.apache.kafka.streams.state.internals.metrics.Sensors.createTaskAndStoreLatencyAndThroughputSensors;
 
 public class MeteredSessionStore
     extends WrappedStateStore, Windowed, V>
     implements SessionStore {
 
-    private final String metricScope;
+    private final String metricsScope;
     private final Serde keySerde;
     private final Serde valueSerde;
     private final Time time;
@@ -53,12 +54,12 @@ public class MeteredSessionStore
     private String taskName;
 
     MeteredSessionStore(final SessionStore inner,
-                        final String metricScope,
+                        final String metricsScope,
                         final Serde keySerde,
                         final Serde valueSerde,
                         final Time time) {
         super(inner);
-        this.metricScope = metricScope;
+        this.metricsScope = metricsScope;
         this.keySerde = keySerde;
         this.valueSerde = valueSerde;
         this.time = time;
@@ -76,9 +77,9 @@ public void init(final ProcessorContext context,
         metrics = (StreamsMetricsImpl) context.metrics();
 
         taskName = context.taskId().toString();
-        final String metricsGroup = "stream-" + metricScope + "-metrics";
-        final Map taskTags = metrics.tagMap("task-id", taskName, metricScope + "-id", "all");
-        final Map storeTags = metrics.tagMap("task-id", taskName, metricScope + "-id", name());
+        final String metricsGroup = "stream-" + metricsScope + "-state-metrics";
+        final Map taskTags = metrics.storeLevelTagMap(taskName, metricsScope, ROLLUP_VALUE);
+        final Map storeTags = metrics.storeLevelTagMap(taskName, metricsScope, name());
 
         putTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "put", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
         fetchTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "fetch", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java
index 6d2eaab3d196c..ac86679169af1 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java
@@ -34,6 +34,9 @@
 import java.util.Map;
 
 import static org.apache.kafka.common.metrics.Sensor.RecordingLevel.DEBUG;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.GROUP_PREFIX;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.ROLLUP_VALUE;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.STATE_LEVEL_GROUP_SUFFIX;
 import static org.apache.kafka.streams.state.internals.metrics.Sensors.createTaskAndStoreLatencyAndThroughputSensors;
 
 public class MeteredWindowStore
@@ -41,7 +44,7 @@ public class MeteredWindowStore
     implements WindowStore {
 
     private final long windowSizeMs;
-    private final String metricScope;
+    private final String metricsScope;
     private final Time time;
     final Serde keySerde;
     final Serde valueSerde;
@@ -55,13 +58,13 @@ public class MeteredWindowStore
 
     MeteredWindowStore(final WindowStore inner,
                        final long windowSizeMs,
-                       final String metricScope,
+                       final String metricsScope,
                        final Time time,
                        final Serde keySerde,
                        final Serde valueSerde) {
         super(inner);
         this.windowSizeMs = windowSizeMs;
-        this.metricScope = metricScope;
+        this.metricsScope = metricsScope;
         this.time = time;
         this.keySerde = keySerde;
         this.valueSerde = valueSerde;
@@ -75,9 +78,9 @@ public void init(final ProcessorContext context,
         metrics = (StreamsMetricsImpl) context.metrics();
 
         taskName = context.taskId().toString();
-        final String metricsGroup = "stream-" + metricScope + "-metrics";
-        final Map taskTags = metrics.tagMap("task-id", taskName, metricScope + "-id", "all");
-        final Map storeTags = metrics.tagMap("task-id", taskName, metricScope + "-id", name());
+        final String metricsGroup = GROUP_PREFIX + metricsScope + STATE_LEVEL_GROUP_SUFFIX;
+        final Map taskTags = metrics.storeLevelTagMap(taskName, metricsScope, ROLLUP_VALUE);
+        final Map storeTags = metrics.storeLevelTagMap(taskName, metricsScope, name());
 
         putTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "put", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
         fetchTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "fetch", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDbKeyValueBytesStoreSupplier.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDbKeyValueBytesStoreSupplier.java
index a2be94e905ab5..87be76781fe1f 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDbKeyValueBytesStoreSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDbKeyValueBytesStoreSupplier.java
@@ -45,6 +45,6 @@ public KeyValueStore get() {
 
     @Override
     public String metricsScope() {
-        return "rocksdb-state";
+        return "rocksdb";
     }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDbSessionBytesStoreSupplier.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDbSessionBytesStoreSupplier.java
index 8f305dbf85311..684ebf4143116 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDbSessionBytesStoreSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDbSessionBytesStoreSupplier.java
@@ -48,7 +48,7 @@ public SessionStore get() {
 
     @Override
     public String metricsScope() {
-        return "rocksdb-session-state";
+        return "rocksdb-session";
     }
 
     @Override
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDbWindowBytesStoreSupplier.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDbWindowBytesStoreSupplier.java
index 79f1ee39e040a..42894cbb4f371 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDbWindowBytesStoreSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDbWindowBytesStoreSupplier.java
@@ -74,7 +74,7 @@ public WindowStore get() {
 
     @Override
     public String metricsScope() {
-        return "rocksdb-window-state";
+        return "rocksdb-window";
     }
 
     @Deprecated
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetricsTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetricsTest.java
index 739f028092155..b586258982510 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetricsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetricsTest.java
@@ -27,7 +27,7 @@
 import java.util.Collections;
 import java.util.Map;
 
-import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.ALL_TASKS;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.ROLLUP_VALUE;
 import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.TASK_ID_TAG;
 import static org.easymock.EasyMock.expect;
 import static org.hamcrest.CoreMatchers.is;
@@ -225,7 +225,7 @@ public void shouldGetCommitOverTasksSensor() {
         final String rateDescription = "The average per-second number of commit calls over all tasks";
         mockStatic(StreamsMetricsImpl.class);
         expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.DEBUG)).andReturn(dummySensor);
-        expect(streamsMetrics.threadLevelTagMap(TASK_ID_TAG, ALL_TASKS)).andReturn(dummyTagMap);
+        expect(streamsMetrics.threadLevelTagMap(TASK_ID_TAG, ROLLUP_VALUE)).andReturn(dummyTagMap);
         StreamsMetricsImpl.addInvocationRateAndCountToSensor(
             dummySensor, TASK_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
         StreamsMetricsImpl.addAvgAndMaxToSensor(
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStoreTest.java
index 5cbe95cea2d37..e494532cfe8cf 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStoreTest.java
@@ -65,7 +65,7 @@ public class MeteredKeyValueStoreTest {
     private final Map tags = mkMap(
         mkEntry("client-id", "test"),
         mkEntry("task-id", taskId.toString()),
-        mkEntry("scope-id", "metered")
+        mkEntry("scope-state-id", "metered")
     );
     @Mock(type = MockType.NICE)
     private KeyValueStore inner;
@@ -105,9 +105,9 @@ public void testMetrics() {
         init();
         final JmxReporter reporter = new JmxReporter("kafka.streams");
         metrics.addReporter(reporter);
-        assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-metrics,client-id=%s,task-id=%s,%s-id=%s",
+        assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-state-metrics,client-id=%s,task-id=%s,%s-state-id=%s",
                 "scope", "test", taskId.toString(), "scope", "metered")));
-        assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-metrics,client-id=%s,task-id=%s,%s-id=%s",
+        assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-state-metrics,client-id=%s,task-id=%s,%s-state-id=%s",
                 "scope", "test", taskId.toString(), "scope", "all")));
     }
 
@@ -149,7 +149,7 @@ public void shouldPutIfAbsentAndRecordPutIfAbsentMetric() {
     }
 
     private KafkaMetric metric(final String name) {
-        return this.metrics.metric(new MetricName(name, "stream-scope-metrics", "", this.tags));
+        return this.metrics.metric(new MetricName(name, "stream-scope-state-metrics", "", this.tags));
     }
 
     @SuppressWarnings("unchecked")
@@ -204,7 +204,7 @@ public void shouldGetAllFromInnerStoreAndRecordAllMetric() {
         assertFalse(iterator.hasNext());
         iterator.close();
 
-        final KafkaMetric metric = metric(new MetricName("all-rate", "stream-scope-metrics", "", tags));
+        final KafkaMetric metric = metric(new MetricName("all-rate", "stream-scope-state-metrics", "", tags));
         assertTrue((Double) metric.metricValue() > 0);
         verify(inner);
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java
index 30c382b19c1c9..e21fbc2052629 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java
@@ -67,7 +67,7 @@ public class MeteredSessionStoreTest {
     private final Map tags = mkMap(
         mkEntry("client-id", "test"),
         mkEntry("task-id", taskId.toString()),
-        mkEntry("scope-id", "metered")
+        mkEntry("scope-state-id", "metered")
     );
     private final Metrics metrics = new Metrics();
     private MeteredSessionStore metered;
@@ -104,9 +104,9 @@ public void testMetrics() {
         init();
         final JmxReporter reporter = new JmxReporter("kafka.streams");
         metrics.addReporter(reporter);
-        assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-metrics,client-id=%s,task-id=%s,%s-id=%s",
+        assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-state-metrics,client-id=%s,task-id=%s,%s-state-id=%s",
                 "scope", "test", taskId.toString(), "scope", "metered")));
-        assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-metrics,client-id=%s,task-id=%s,%s-id=%s",
+        assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-state-metrics,client-id=%s,task-id=%s,%s-state-id=%s",
                 "scope", "test", taskId.toString(), "scope", "all")));
     }
 
@@ -287,7 +287,7 @@ public void shouldNotSetFlushListenerOnWrappedNoneCachingStore() {
     }
 
     private KafkaMetric metric(final String name) {
-        return this.metrics.metric(new MetricName(name, "stream-scope-metrics", "", this.tags));
+        return this.metrics.metric(new MetricName(name, "stream-scope-state-metrics", "", this.tags));
     }
 
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedKeyValueStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedKeyValueStoreTest.java
index 606428a66ce7e..47a49455cb3c0 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedKeyValueStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedKeyValueStoreTest.java
@@ -68,7 +68,7 @@ public class MeteredTimestampedKeyValueStoreTest {
     private final Map tags = mkMap(
         mkEntry("client-id", "test"),
         mkEntry("task-id", taskId.toString()),
-        mkEntry("scope-id", "metered")
+        mkEntry("scope-state-id", "metered")
     );
     @Mock(type = MockType.NICE)
     private KeyValueStore inner;
@@ -110,9 +110,9 @@ public void testMetrics() {
         init();
         final JmxReporter reporter = new JmxReporter("kafka.streams");
         metrics.addReporter(reporter);
-        assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-metrics,client-id=%s,task-id=%s,%s-id=%s",
+        assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-state-metrics,client-id=%s,task-id=%s,%s-state-id=%s",
                 "scope", "test", taskId.toString(), "scope", "metered")));
-        assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-metrics,client-id=%s,task-id=%s,%s-id=%s",
+        assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-state-metrics,client-id=%s,task-id=%s,%s-state-id=%s",
                 "scope", "test", taskId.toString(), "scope", "all")));
     }
     @Test
@@ -153,7 +153,7 @@ public void shouldPutIfAbsentAndRecordPutIfAbsentMetric() {
     }
 
     private KafkaMetric metric(final String name) {
-        return this.metrics.metric(new MetricName(name, "stream-scope-metrics", "", this.tags));
+        return this.metrics.metric(new MetricName(name, "stream-scope-state-metrics", "", this.tags));
     }
 
     @SuppressWarnings("unchecked")
@@ -209,7 +209,7 @@ public void shouldGetAllFromInnerStoreAndRecordAllMetric() {
         assertFalse(iterator.hasNext());
         iterator.close();
 
-        final KafkaMetric metric = metric(new MetricName("all-rate", "stream-scope-metrics", "", tags));
+        final KafkaMetric metric = metric(new MetricName("all-rate", "stream-scope-state-metrics", "", tags));
         assertTrue((Double) metric.metricValue() > 0);
         verify(inner);
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java
index c0ed7f634017e..bd5bce1632f17 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java
@@ -94,9 +94,9 @@ public void testMetrics() {
         store.init(context, store);
         final JmxReporter reporter = new JmxReporter("kafka.streams");
         metrics.addReporter(reporter);
-        assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-metrics,client-id=%s,task-id=%s,%s-id=%s",
+        assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-state-metrics,client-id=%s,task-id=%s,%s-state-id=%s",
                 "scope", "test", context.taskId().toString(), "scope", "mocked-store")));
-        assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-metrics,client-id=%s,task-id=%s,%s-id=%s",
+        assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-state-metrics,client-id=%s,task-id=%s,%s-state-id=%s",
                 "scope", "test", context.taskId().toString(), "scope", "all")));
     }
 
@@ -107,8 +107,8 @@ public void shouldRecordRestoreLatencyOnInit() {
         replay(innerStoreMock);
         store.init(context, store);
         final Map metrics = context.metrics().metrics();
-        assertEquals(1.0, getMetricByNameFilterByTags(metrics, "restore-total", "stream-scope-metrics", singletonMap("scope-id", "all")).metricValue());
-        assertEquals(1.0, getMetricByNameFilterByTags(metrics, "restore-total", "stream-scope-metrics", singletonMap("scope-id", "mocked-store")).metricValue());
+        assertEquals(1.0, getMetricByNameFilterByTags(metrics, "restore-total", "stream-scope-state-metrics", singletonMap("scope-state-id", "all")).metricValue());
+        assertEquals(1.0, getMetricByNameFilterByTags(metrics, "restore-total", "stream-scope-state-metrics", singletonMap("scope-state-id", "mocked-store")).metricValue());
     }
 
     @Test
@@ -121,8 +121,8 @@ public void shouldRecordPutLatency() {
         store.init(context, store);
         store.put("a", "a");
         final Map metrics = context.metrics().metrics();
-        assertEquals(1.0, getMetricByNameFilterByTags(metrics, "put-total", "stream-scope-metrics", singletonMap("scope-id", "all")).metricValue());
-        assertEquals(1.0, getMetricByNameFilterByTags(metrics, "put-total", "stream-scope-metrics", singletonMap("scope-id", "mocked-store")).metricValue());
+        assertEquals(1.0, getMetricByNameFilterByTags(metrics, "put-total", "stream-scope-state-metrics", singletonMap("scope-state-id", "all")).metricValue());
+        assertEquals(1.0, getMetricByNameFilterByTags(metrics, "put-total", "stream-scope-state-metrics", singletonMap("scope-state-id", "mocked-store")).metricValue());
         verify(innerStoreMock);
     }
 
@@ -134,8 +134,8 @@ public void shouldRecordFetchLatency() {
         store.init(context, store);
         store.fetch("a", ofEpochMilli(1), ofEpochMilli(1)).close(); // recorded on close;
         final Map metrics = context.metrics().metrics();
-        assertEquals(1.0, getMetricByNameFilterByTags(metrics, "fetch-total", "stream-scope-metrics", singletonMap("scope-id", "all")).metricValue());
-        assertEquals(1.0, getMetricByNameFilterByTags(metrics, "fetch-total", "stream-scope-metrics", singletonMap("scope-id", "mocked-store")).metricValue());
+        assertEquals(1.0, getMetricByNameFilterByTags(metrics, "fetch-total", "stream-scope-state-metrics", singletonMap("scope-state-id", "all")).metricValue());
+        assertEquals(1.0, getMetricByNameFilterByTags(metrics, "fetch-total", "stream-scope-state-metrics", singletonMap("scope-state-id", "mocked-store")).metricValue());
         verify(innerStoreMock);
     }
 
@@ -147,8 +147,8 @@ public void shouldRecordFetchRangeLatency() {
         store.init(context, store);
         store.fetch("a", "b", ofEpochMilli(1), ofEpochMilli(1)).close(); // recorded on close;
         final Map metrics = context.metrics().metrics();
-        assertEquals(1.0, getMetricByNameFilterByTags(metrics, "fetch-total", "stream-scope-metrics", singletonMap("scope-id", "all")).metricValue());
-        assertEquals(1.0, getMetricByNameFilterByTags(metrics, "fetch-total", "stream-scope-metrics", singletonMap("scope-id", "mocked-store")).metricValue());
+        assertEquals(1.0, getMetricByNameFilterByTags(metrics, "fetch-total", "stream-scope-state-metrics", singletonMap("scope-state-id", "all")).metricValue());
+        assertEquals(1.0, getMetricByNameFilterByTags(metrics, "fetch-total", "stream-scope-state-metrics", singletonMap("scope-state-id", "mocked-store")).metricValue());
         verify(innerStoreMock);
     }
 
@@ -161,8 +161,8 @@ public void shouldRecordFlushLatency() {
         store.init(context, store);
         store.flush();
         final Map metrics = context.metrics().metrics();
-        assertEquals(1.0, getMetricByNameFilterByTags(metrics, "flush-total", "stream-scope-metrics", singletonMap("scope-id", "all")).metricValue());
-        assertEquals(1.0, getMetricByNameFilterByTags(metrics, "flush-total", "stream-scope-metrics", singletonMap("scope-id", "mocked-store")).metricValue());
+        assertEquals(1.0, getMetricByNameFilterByTags(metrics, "flush-total", "stream-scope-state-metrics", singletonMap("scope-state-id", "all")).metricValue());
+        assertEquals(1.0, getMetricByNameFilterByTags(metrics, "flush-total", "stream-scope-state-metrics", singletonMap("scope-state-id", "mocked-store")).metricValue());
         verify(innerStoreMock);
     }
 

From 40432e31f7c5b0e0992a51621dc8b1c0e3d3103f Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Fri, 30 Aug 2019 14:48:46 -0700
Subject: [PATCH 0574/1071] MONIR: Check for NULL in case of version probing
 (#7275)

In case of version probing we would skip the logic for setting cluster / assigned tasks; since these values are initialized as null they are vulnerable to NPE when code changes.

Reviewers: A. Sophie Blee-Goldman , Bill Bejeck 
---
 .../kafka/streams/processor/internals/TaskManager.java       | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
index 444d6975e692a..a7fa1fa96706c 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
@@ -112,7 +112,7 @@ void createTasks(final Collection assignment) {
     }
 
     private void addStreamTasks(final Collection assignment) {
-        if (assignedActiveTasks.isEmpty()) {
+        if (assignedActiveTasks == null || assignedActiveTasks.isEmpty()) {
             return;
         }
         final Map> newTasks = new HashMap<>();
@@ -151,8 +151,7 @@ private void addStreamTasks(final Collection assignment) {
     }
 
     private void addStandbyTasks() {
-        final Map> assignedStandbyTasks = this.assignedStandbyTasks;
-        if (assignedStandbyTasks.isEmpty()) {
+        if (assignedStandbyTasks == null || assignedStandbyTasks.isEmpty()) {
             return;
         }
         log.debug("Adding assigned standby tasks {}", assignedStandbyTasks);

From a225347ff20a8d12fa6ffb6788feb64af25a0b3c Mon Sep 17 00:00:00 2001
From: Colin Patrick McCabe 
Date: Fri, 30 Aug 2019 16:07:59 -0700
Subject: [PATCH 0575/1071] KAFKA-8840: Fix bug where
 ClientCompatibilityFeaturesTest fails when running multiple iterations
 (#7260)

Fix a bug where ClientCompatibilityFeaturesTest fails when running multiple iterations.

Also, fix a typo in tests/docker/Dockerfile.

Reviewers: Ismael Juma 
---
 .../tests/client/client_compatibility_features_test.py    | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tests/kafkatest/tests/client/client_compatibility_features_test.py b/tests/kafkatest/tests/client/client_compatibility_features_test.py
index d5a8c0d8a4efb..2e593ba88ff04 100644
--- a/tests/kafkatest/tests/client/client_compatibility_features_test.py
+++ b/tests/kafkatest/tests/client/client_compatibility_features_test.py
@@ -88,7 +88,13 @@ def invoke_compatibility_program(self, features):
         for k, v in features.iteritems():
             cmd = cmd + ("--%s %s " % (k, v))
         results_dir = TestContext.results_dir(self.test_context, 0)
-        os.makedirs(results_dir)
+        try:
+            os.makedirs(results_dir)
+        except OSError as e:
+            if e.errno == errno.EEXIST and os.path.isdir(path):
+                pass
+            else:
+                raise
         ssh_log_file = "%s/%s" % (results_dir, "client_compatibility_test_output.txt")
         try:
           self.logger.info("Running %s" % cmd)

From 364794866f82d2d99f7a610d699478d5055688f0 Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Mon, 2 Sep 2019 14:43:17 +0100
Subject: [PATCH 0576/1071] KAFKA-8760; New Java Authorizer API (KIP-504)
 (#7268)

New Java Authorizer API and a new out-of-the-box authorizer (AclAuthorizer) that implements the new interface.

Reviewers: Manikumar Reddy 
---
 build.gradle                                  |   1 +
 checkstyle/import-control.xml                 |   2 +
 .../org/apache/kafka/common/Endpoint.java     | 101 ++
 .../kafka/common/requests/RequestContext.java |  42 +-
 .../kafka/common/utils/SecurityUtils.java     |  44 +
 .../server/authorizer/AclCreateResult.java    |  50 +
 .../server/authorizer/AclDeleteResult.java    |  97 ++
 .../kafka/server/authorizer/Action.java       | 124 +++
 .../AuthorizableRequestContext.java           |  72 ++
 .../authorizer/AuthorizationResult.java       |  26 +
 .../kafka/server/authorizer/Authorizer.java   | 110 +++
 .../authorizer/AuthorizerServerInfo.java      |  51 +
 .../main/scala/kafka/admin/AclCommand.scala   | 220 ++++-
 .../scala/kafka/admin/ConfigCommand.scala     |   2 +-
 .../src/main/scala/kafka/cluster/Broker.scala |  28 +-
 .../main/scala/kafka/cluster/EndPoint.scala   |   9 +-
 .../transaction/TransactionStateManager.scala |   2 +-
 .../scala/kafka/network/SocketServer.scala    |  31 +-
 .../kafka/security/auth/Authorizer.scala      |   1 +
 .../security/auth/SimpleAclAuthorizer.scala   | 382 ++------
 .../security/authorizer/AclAuthorizer.scala   | 476 ++++++++++
 .../AuthorizerUtils.scala}                    |  56 +-
 .../authorizer/AuthorizerWrapper.scala        | 136 +++
 .../kafka/server/DynamicBrokerConfig.scala    |   4 +
 .../main/scala/kafka/server/KafkaApis.scala   | 401 ++++----
 .../main/scala/kafka/server/KafkaConfig.scala |  20 +-
 .../main/scala/kafka/server/KafkaServer.scala |  21 +-
 .../main/scala/kafka/zk/KafkaZkClient.scala   |   2 +-
 core/src/main/scala/kafka/zk/ZkData.scala     |   2 +-
 .../api/AdminClientIntegrationTest.scala      |   4 +-
 .../kafka/api/AuthorizerIntegrationTest.scala | 384 ++++----
 ...gationTokenEndToEndAuthorizationTest.scala |   2 +-
 .../DescribeAuthorizedOperationsTest.scala    |  30 +-
 .../kafka/api/EndToEndAuthorizationTest.scala |  73 +-
 ...aslClientsWithInvalidCredentialsTest.scala |   2 +-
 ...slGssapiSslEndToEndAuthorizationTest.scala |   2 +
 .../SaslSslAdminClientIntegrationTest.scala   |  21 +-
 .../api/SslAdminClientIntegrationTest.scala   |  90 ++
 .../unit/kafka/admin/AclCommandTest.scala     | 107 ++-
 .../admin/LeaderElectionCommandTest.scala     |   2 +-
 ...rredReplicaLeaderElectionCommandTest.scala |  22 +-
 .../unit/kafka/network/SocketServerTest.scala |  51 +-
 .../auth/SimpleAclAuthorizerTest.scala        |  40 +-
 .../authorizer/AclAuthorizerTest.scala        | 877 ++++++++++++++++++
 .../DelegationTokenManagerTest.scala          |  59 +-
 .../server/DynamicBrokerConfigTest.scala      |  44 +-
 .../unit/kafka/server/KafkaApisTest.scala     |   2 +-
 .../unit/kafka/server/RequestQuotaTest.scala  |  14 +-
 .../scala/unit/kafka/utils/TestUtils.scala    |  15 +-
 49 files changed, 3406 insertions(+), 948 deletions(-)
 create mode 100644 clients/src/main/java/org/apache/kafka/common/Endpoint.java
 create mode 100644 clients/src/main/java/org/apache/kafka/server/authorizer/AclCreateResult.java
 create mode 100644 clients/src/main/java/org/apache/kafka/server/authorizer/AclDeleteResult.java
 create mode 100644 clients/src/main/java/org/apache/kafka/server/authorizer/Action.java
 create mode 100644 clients/src/main/java/org/apache/kafka/server/authorizer/AuthorizableRequestContext.java
 create mode 100644 clients/src/main/java/org/apache/kafka/server/authorizer/AuthorizationResult.java
 create mode 100644 clients/src/main/java/org/apache/kafka/server/authorizer/Authorizer.java
 create mode 100644 clients/src/main/java/org/apache/kafka/server/authorizer/AuthorizerServerInfo.java
 create mode 100644 core/src/main/scala/kafka/security/authorizer/AclAuthorizer.scala
 rename core/src/main/scala/kafka/security/{SecurityUtils.scala => authorizer/AuthorizerUtils.scala} (52%)
 create mode 100644 core/src/main/scala/kafka/security/authorizer/AuthorizerWrapper.scala
 create mode 100644 core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala
 create mode 100644 core/src/test/scala/unit/kafka/security/authorizer/AclAuthorizerTest.scala

diff --git a/build.gradle b/build.gradle
index 9a90040fc3414..323a3ca89e4a9 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1050,6 +1050,7 @@ project(':clients') {
     include "**/org/apache/kafka/common/security/scram/*"
     include "**/org/apache/kafka/common/security/token/delegation/*"
     include "**/org/apache/kafka/common/security/oauthbearer/*"
+    include "**/org/apache/kafka/server/authorizer/*"
     include "**/org/apache/kafka/server/policy/*"
     include "**/org/apache/kafka/server/quota/*"
   }
diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml
index 063c1ba089dc7..44b06a7a381ca 100644
--- a/checkstyle/import-control.xml
+++ b/checkstyle/import-control.xml
@@ -147,6 +147,8 @@
       
       
       
+      
+      
       
       
     
diff --git a/clients/src/main/java/org/apache/kafka/common/Endpoint.java b/clients/src/main/java/org/apache/kafka/common/Endpoint.java
new file mode 100644
index 0000000000000..963d3aea987b7
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/Endpoint.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common;
+
+import java.util.Objects;
+import org.apache.kafka.common.annotation.InterfaceStability;
+import org.apache.kafka.common.security.auth.SecurityProtocol;
+
+/**
+ * Represents a broker endpoint.
+ */
+
+@InterfaceStability.Evolving
+public class Endpoint {
+
+    private final String listener;
+    private final SecurityProtocol securityProtocol;
+    private final String host;
+    private final int port;
+
+    public Endpoint(String listener, SecurityProtocol securityProtocol, String host, int port) {
+        this.listener = listener;
+        this.securityProtocol = securityProtocol;
+        this.host = host;
+        this.port = port;
+    }
+
+    /**
+     * Returns the listener name of this endpoint.
+     */
+    public String listener() {
+        return listener;
+    }
+
+    /**
+     * Returns the security protocol of this endpoint.
+     */
+    public SecurityProtocol securityProtocol() {
+        return securityProtocol;
+    }
+
+    /**
+     * Returns advertised host name of this endpoint.
+     */
+    public String host() {
+        return host;
+    }
+
+    /**
+     * Returns the port to which the listener is bound.
+     */
+    public int port() {
+        return port;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof Endpoint)) {
+            return false;
+        }
+
+        Endpoint that = (Endpoint) o;
+        return Objects.equals(this.listener, that.listener) &&
+            Objects.equals(this.securityProtocol, that.securityProtocol) &&
+            Objects.equals(this.host, that.host) &&
+            this.port == that.port;
+
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(listener, securityProtocol, host, port);
+    }
+
+    @Override
+    public String toString() {
+        return "Endpoint(" +
+            "listener='" + listener + '\'' +
+            ", securityProtocol=" + securityProtocol +
+            ", host='" + host + '\'' +
+            ", port=" + port +
+            ')';
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java
index 663d7460db93e..4002d2a5f44d0 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java
@@ -26,10 +26,11 @@
 
 import java.net.InetAddress;
 import java.nio.ByteBuffer;
+import org.apache.kafka.server.authorizer.AuthorizableRequestContext;
 
 import static org.apache.kafka.common.protocol.ApiKeys.API_VERSIONS;
 
-public class RequestContext {
+public class RequestContext implements AuthorizableRequestContext {
     public final RequestHeader header;
     public final String connectionId;
     public final InetAddress clientAddress;
@@ -89,4 +90,43 @@ public short apiVersion() {
         return header.apiVersion();
     }
 
+    @Override
+    public String listener() {
+        return listenerName.value();
+    }
+
+    @Override
+    public SecurityProtocol securityProtocol() {
+        return securityProtocol;
+    }
+
+    @Override
+    public KafkaPrincipal principal() {
+        return principal;
+    }
+
+    @Override
+    public InetAddress clientAddress() {
+        return clientAddress;
+    }
+
+    @Override
+    public int requestType() {
+        return header.apiKey().id;
+    }
+
+    @Override
+    public int requestVersion() {
+        return header.apiVersion();
+    }
+
+    @Override
+    public String clientId() {
+        return header.clientId();
+    }
+
+    @Override
+    public int correlationId() {
+        return header.correlationId();
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/utils/SecurityUtils.java b/clients/src/main/java/org/apache/kafka/common/utils/SecurityUtils.java
index bd3a22c7a5876..d6595ef9e8417 100644
--- a/clients/src/main/java/org/apache/kafka/common/utils/SecurityUtils.java
+++ b/clients/src/main/java/org/apache/kafka/common/utils/SecurityUtils.java
@@ -16,19 +16,39 @@
  */
 package org.apache.kafka.common.utils;
 
+import org.apache.kafka.common.acl.AclOperation;
 import org.apache.kafka.common.config.SecurityConfig;
+import org.apache.kafka.common.resource.ResourceType;
 import org.apache.kafka.common.security.SecurityProviderCreator;
 import org.apache.kafka.common.security.auth.KafkaPrincipal;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.security.Security;
+import java.util.HashMap;
 import java.util.Map;
 
 public class SecurityUtils {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(SecurityConfig.class);
 
+    private static final Map NAME_TO_RESOURCE_TYPES;
+    private static final Map NAME_TO_OPERATIONS;
+
+    static {
+        NAME_TO_RESOURCE_TYPES = new HashMap<>(ResourceType.values().length);
+        NAME_TO_OPERATIONS = new HashMap<>(AclOperation.values().length);
+
+        for (ResourceType resourceType : ResourceType.values()) {
+            String resourceTypeName = toPascalCase(resourceType.name());
+            NAME_TO_RESOURCE_TYPES.put(resourceTypeName, resourceType);
+        }
+        for (AclOperation operation : AclOperation.values()) {
+            String operationName = toPascalCase(operation.name());
+            NAME_TO_OPERATIONS.put(operationName, operation);
+        }
+    }
+
     public static KafkaPrincipal parseKafkaPrincipal(String str) {
         if (str == null || str.isEmpty()) {
             throw new IllegalArgumentException("expected a string in format principalType:principalName but got " + str);
@@ -65,4 +85,28 @@ public static void addConfiguredSecurityProviders(Map configs) {
         }
     }
 
+    public static ResourceType resourceType(String name) {
+        ResourceType resourceType = NAME_TO_RESOURCE_TYPES.get(name);
+        return resourceType == null ? ResourceType.UNKNOWN : resourceType;
+    }
+
+    public static AclOperation operation(String name) {
+        AclOperation operation = NAME_TO_OPERATIONS.get(name);
+        return operation == null ? AclOperation.UNKNOWN : operation;
+    }
+
+    private static String toPascalCase(String name) {
+        StringBuilder builder = new StringBuilder();
+        boolean capitalizeNext = true;
+        for (char c : name.toCharArray()) {
+            if (c == '_')
+                capitalizeNext = true;
+            else if (capitalizeNext) {
+                builder.append(Character.toUpperCase(c));
+                capitalizeNext = false;
+            } else
+                builder.append(Character.toLowerCase(c));
+        }
+        return builder.toString();
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/server/authorizer/AclCreateResult.java b/clients/src/main/java/org/apache/kafka/server/authorizer/AclCreateResult.java
new file mode 100644
index 0000000000000..7255df5f06373
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/server/authorizer/AclCreateResult.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.server.authorizer;
+
+import org.apache.kafka.common.annotation.InterfaceStability;
+import org.apache.kafka.common.errors.ApiException;
+
+@InterfaceStability.Evolving
+public class AclCreateResult {
+    public static final AclCreateResult SUCCESS = new AclCreateResult();
+
+    private final ApiException exception;
+
+    public AclCreateResult() {
+        this(null);
+    }
+
+    public AclCreateResult(ApiException exception) {
+        this.exception = exception;
+    }
+
+    /**
+     * Returns any exception during create. If exception is null, the request has succeeded.
+     */
+    public ApiException exception() {
+        return exception;
+    }
+
+    /**
+     * Returns true if the request failed.
+     */
+    public boolean failed() {
+        return exception != null;
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/server/authorizer/AclDeleteResult.java b/clients/src/main/java/org/apache/kafka/server/authorizer/AclDeleteResult.java
new file mode 100644
index 0000000000000..ed4449a6ff992
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/server/authorizer/AclDeleteResult.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.server.authorizer;
+
+import java.util.Collections;
+import java.util.Collection;
+import org.apache.kafka.common.acl.AclBinding;
+import org.apache.kafka.common.annotation.InterfaceStability;
+import org.apache.kafka.common.errors.ApiException;
+
+@InterfaceStability.Evolving
+public class AclDeleteResult {
+    private final ApiException exception;
+    private final Collection aclBindingDeleteResults;
+
+    public AclDeleteResult(ApiException exception) {
+        this(Collections.emptySet(), exception);
+    }
+
+    public AclDeleteResult(Collection deleteResults) {
+        this(deleteResults, null);
+    }
+
+    private AclDeleteResult(Collection deleteResults, ApiException exception) {
+        this.aclBindingDeleteResults = deleteResults;
+        this.exception = exception;
+    }
+
+    /**
+     * Returns any exception while attempting to match ACL filter to delete ACLs.
+     */
+    public ApiException exception() {
+        return exception;
+    }
+
+    /**
+     * Returns delete result for each matching ACL binding.
+     */
+    public Collection aclBindingDeleteResults() {
+        return aclBindingDeleteResults;
+    }
+
+
+    /**
+     * Delete result for each ACL binding that matched a delete filter.
+     */
+    public static class AclBindingDeleteResult {
+        private final AclBinding aclBinding;
+        private final ApiException exception;
+
+        public AclBindingDeleteResult(AclBinding aclBinding) {
+            this(aclBinding, null);
+        }
+
+        public AclBindingDeleteResult(AclBinding aclBinding, ApiException exception) {
+            this.aclBinding = aclBinding;
+            this.exception = exception;
+        }
+
+        /**
+         * Returns ACL binding that matched the delete filter. {@link #deleted()} indicates if
+         * the binding was deleted.
+         */
+        public AclBinding aclBinding() {
+            return aclBinding;
+        }
+
+        /**
+         * Returns exception that resulted in failure to delete ACL binding.
+         */
+        public ApiException exception() {
+            return exception;
+        }
+
+        /**
+         * Returns true if ACL binding was deleted, false otherwise.
+         */
+        public boolean deleted() {
+            return exception == null;
+        }
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/server/authorizer/Action.java b/clients/src/main/java/org/apache/kafka/server/authorizer/Action.java
new file mode 100644
index 0000000000000..a62b7f0965b2e
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/server/authorizer/Action.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.server.authorizer;
+
+import java.util.Objects;
+import org.apache.kafka.common.acl.AclOperation;
+import org.apache.kafka.common.annotation.InterfaceStability;
+import org.apache.kafka.common.resource.ResourcePattern;
+
+@InterfaceStability.Evolving
+public class Action {
+
+    private final ResourcePattern resourcePattern;
+    private final AclOperation operation;
+    private final int resourceReferenceCount;
+    private final boolean logIfAllowed;
+    private final boolean logIfDenied;
+
+    public Action(AclOperation operation,
+                  ResourcePattern resourcePattern,
+                  int resourceReferenceCount,
+                  boolean logIfAllowed,
+                  boolean logIfDenied) {
+        this.operation = operation;
+        this.resourcePattern = resourcePattern;
+        this.logIfAllowed = logIfAllowed;
+        this.logIfDenied = logIfDenied;
+        this.resourceReferenceCount = resourceReferenceCount;
+    }
+
+    /**
+     * Resource on which action is being performed.
+     */
+    public ResourcePattern resourcePattern() {
+        return resourcePattern;
+    }
+
+    /**
+     * Operation being performed.
+     */
+    public AclOperation operation() {
+        return operation;
+    }
+
+    /**
+     * Indicates if audit logs tracking ALLOWED access should include this action if result is
+     * ALLOWED. The flag is true if access to a resource is granted while processing the request as a
+     * result of this authorization. The flag is false only for requests used to describe access where
+     * no operation on the resource is actually performed based on the authorization result.
+     */
+    public boolean logIfAllowed() {
+        return logIfAllowed;
+    }
+
+    /**
+     * Indicates if audit logs tracking DENIED access should include this action if result is
+     * DENIED. The flag is true if access to a resource was explicitly requested and request
+     * is denied as a result of this authorization request. The flag is false if request was
+     * filtering out authorized resources (e.g. to subscribe to regex pattern). The flag is also
+     * false if this is an optional authorization where an alternative resource authorization is
+     * applied if this fails (e.g. Cluster:Create which is subsequently overridden by Topic:Create).
+     */
+    public boolean logIfDenied() {
+        return logIfDenied;
+    }
+
+    /**
+     * Number of times the resource being authorized is referenced within the request. For example, a single
+     * request may reference `n` topic partitions of the same topic. Brokers will authorize the topic once
+     * with `resourceReferenceCount=n`. Authorizers may include the count in audit logs.
+     */
+    public int resourceReferenceCount() {
+        return resourceReferenceCount;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof Action)) {
+            return false;
+        }
+
+        Action that = (Action) o;
+        return Objects.equals(this.resourcePattern, that.resourcePattern) &&
+            Objects.equals(this.operation, that.operation) &&
+            this.resourceReferenceCount == that.resourceReferenceCount &&
+            this.logIfAllowed == that.logIfAllowed &&
+            this.logIfDenied == that.logIfDenied;
+
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(resourcePattern, operation, resourceReferenceCount, logIfAllowed, logIfDenied);
+    }
+
+    @Override
+    public String toString() {
+        return "Action(" +
+            ", resourcePattern='" + resourcePattern + '\'' +
+            ", operation='" + operation + '\'' +
+            ", resourceReferenceCount='" + resourceReferenceCount + '\'' +
+            ", logIfAllowed='" + logIfAllowed + '\'' +
+            ", logIfDenied='" + logIfDenied + '\'' +
+            ')';
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/server/authorizer/AuthorizableRequestContext.java b/clients/src/main/java/org/apache/kafka/server/authorizer/AuthorizableRequestContext.java
new file mode 100644
index 0000000000000..49c7290106af2
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/server/authorizer/AuthorizableRequestContext.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.server.authorizer;
+
+import java.net.InetAddress;
+import org.apache.kafka.common.annotation.InterfaceStability;
+import org.apache.kafka.common.security.auth.KafkaPrincipal;
+import org.apache.kafka.common.security.auth.SecurityProtocol;
+
+/**
+ * Request context interface that provides data from request header as well as connection
+ * and authentication information to plugins.
+ */
+@InterfaceStability.Evolving
+public interface AuthorizableRequestContext {
+
+    /**
+     * Returns name of listener on which request was received.
+     */
+    String listener();
+
+    /**
+     * Returns the security protocol for the listener on which request was received.
+     */
+    SecurityProtocol securityProtocol();
+
+    /**
+     * Returns authenticated principal for the connection on which request was received.
+     */
+    KafkaPrincipal principal();
+
+    /**
+     * Returns client IP address from which request was sent.
+     */
+    InetAddress clientAddress();
+
+    /**
+     * 16-bit API key of the request from the request header. See
+     * https://kafka.apache.org/protocol#protocol_api_keys for request types.
+     */
+    int requestType();
+
+    /**
+     * Returns the request version from the request header.
+     */
+    int requestVersion();
+
+    /**
+     * Returns the client id from the request header.
+     */
+    String clientId();
+
+    /**
+     * Returns the correlation id from the request header.
+     */
+    int correlationId();
+}
diff --git a/clients/src/main/java/org/apache/kafka/server/authorizer/AuthorizationResult.java b/clients/src/main/java/org/apache/kafka/server/authorizer/AuthorizationResult.java
new file mode 100644
index 0000000000000..d4ad15d6b0322
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/server/authorizer/AuthorizationResult.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.server.authorizer;
+
+import org.apache.kafka.common.annotation.InterfaceStability;
+
+@InterfaceStability.Evolving
+public enum AuthorizationResult {
+    ALLOWED,
+    DENIED
+}
diff --git a/clients/src/main/java/org/apache/kafka/server/authorizer/Authorizer.java b/clients/src/main/java/org/apache/kafka/server/authorizer/Authorizer.java
new file mode 100644
index 0000000000000..654bcd04912d7
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/server/authorizer/Authorizer.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.server.authorizer;
+
+import java.io.Closeable;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import org.apache.kafka.common.Configurable;
+import org.apache.kafka.common.Endpoint;
+import org.apache.kafka.common.acl.AclBinding;
+import org.apache.kafka.common.acl.AclBindingFilter;
+import org.apache.kafka.common.annotation.InterfaceStability;
+
+/**
+ *
+ * Pluggable authorizer interface for Kafka brokers.
+ *
+ * Startup sequence in brokers:
+ * 
              
+ *   
            1. Broker creates authorizer instance if configured in `authorizer.class.name`.
              2. 
+ *   
            2. Broker configures and starts authorizer instance. Authorizer implementation starts loading its metadata.
              3. 
+ *   
            3. Broker starts SocketServer to accept connections and process requests.
              4. 
+ *   
            4. For each listener, SocketServer waits for authorization metadata to be available in the
+ *       authorizer before accepting connections. The future returned by {@link #start(AuthorizerServerInfo)}
+ *       for each listener must return only when authorizer is ready to authorize requests on the listener.
              5. 
+ *   
            5. Broker accepts connections. For each connection, broker performs authentication and then accepts Kafka requests.
+ *       For each request, broker invokes {@link #authorize(AuthorizableRequestContext, List)} to authorize
+ *       actions performed by the request.
              6. 
+ * 
            
+ *
+ * Authorizer implementation class may optionally implement @{@link org.apache.kafka.common.Reconfigurable}
+ * to enable dynamic reconfiguration without restarting the broker.
+ * 
            
+ * Thread safety: All authorizer operations including authorization and ACL updates must be thread-safe.
+ * 
            
+ */
+@InterfaceStability.Evolving
+public interface Authorizer extends Configurable, Closeable {
+
+    /**
+     * Starts loading authorization metadata and returns futures that can be used to wait until
+     * metadata for authorizing requests on each listener is available. Each listener will be
+     * started only after its metadata is available and authorizer is ready to start authorizing
+     * requests on that listener.
+     *
+     * @param serverInfo Metadata for the broker including broker id and listener endpoints
+     * @return CompletableFutures for each endpoint that completes when authorizer is ready to
+     *         start authorizing requests on that listener.
+     */
+    Map> start(AuthorizerServerInfo serverInfo);
+
+    /**
+     * Authorizes the specified action. Additional metadata for the action is specified
+     * in `requestContext`.
+     *
+     * @param requestContext Request context including request type, security protocol and listener name
+     * @param actions Actions being authorized including resource and operation for each action
+     * @return List of authorization results for each action in the same order as the provided actions
+     */
+    List authorize(AuthorizableRequestContext requestContext, List actions);
+
+    /**
+     * Creates new ACL bindings.
+     *
+     * @param requestContext Request context if the ACL is being created by a broker to handle
+     *        a client request to create ACLs. This may be null if ACLs are created directly in ZooKeeper
+     *        using AclCommand.
+     * @param aclBindings ACL bindings to create
+     *
+     * @return Create result for each ACL binding in the same order as in the input list
+     */
+    List createAcls(AuthorizableRequestContext requestContext, List aclBindings);
+
+    /**
+     * Deletes all ACL bindings that match the provided filters.
+     *
+     * @param requestContext Request context if the ACL is being deleted by a broker to handle
+     *        a client request to delete ACLs. This may be null if ACLs are deleted directly in ZooKeeper
+     *        using AclCommand.
+     * @param aclBindingFilters Filters to match ACL bindings that are to be deleted
+     *
+     * @return Delete result for each filter in the same order as in the input list.
+     *         Each result indicates which ACL bindings were actually deleted as well as any
+     *         bindings that matched but could not be deleted.
+     */
+    List deleteAcls(AuthorizableRequestContext requestContext, List aclBindingFilters);
+
+    /**
+     * Returns ACL bindings which match the provided filter.
+     *
+     * @return Iterator for ACL bindings, which may be populated lazily.
+     */
+    Iterable acls(AclBindingFilter filter);
+}
diff --git a/clients/src/main/java/org/apache/kafka/server/authorizer/AuthorizerServerInfo.java b/clients/src/main/java/org/apache/kafka/server/authorizer/AuthorizerServerInfo.java
new file mode 100644
index 0000000000000..51e23fba57fad
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/server/authorizer/AuthorizerServerInfo.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.server.authorizer;
+
+import java.util.Collection;
+import org.apache.kafka.common.ClusterResource;
+import org.apache.kafka.common.Endpoint;
+import org.apache.kafka.common.annotation.InterfaceStability;
+
+/**
+ * Runtime broker configuration metadata provided to authorizers during start up.
+ */
+@InterfaceStability.Evolving
+public interface AuthorizerServerInfo {
+
+    /**
+     * Returns cluster metadata for the broker running this authorizer including cluster id.
+     */
+    ClusterResource clusterResource();
+
+    /**
+     * Returns broker id. This may be a generated broker id if `broker.id` was not configured.
+     */
+    int brokerId();
+
+    /**
+     * Returns endpoints for all listeners including the advertised host and port to which
+     * the listener is bound.
+     */
+    Collection endpoints();
+
+    /**
+     * Returns the inter-broker endpoint. This is one of the endpoints returned by {@link #endpoints()}.
+     */
+    Endpoint interBrokerEndpoint();
+}
diff --git a/core/src/main/scala/kafka/admin/AclCommand.scala b/core/src/main/scala/kafka/admin/AclCommand.scala
index 63c8466352d80..d4d95a9b79407 100644
--- a/core/src/main/scala/kafka/admin/AclCommand.scala
+++ b/core/src/main/scala/kafka/admin/AclCommand.scala
@@ -22,14 +22,18 @@ import java.util.Properties
 import joptsimple._
 import joptsimple.util.EnumConverter
 import kafka.security.auth._
+import kafka.security.authorizer.AuthorizerUtils
 import kafka.server.KafkaConfig
 import kafka.utils._
 import org.apache.kafka.clients.admin.{Admin, AdminClientConfig, AdminClient => JAdminClient}
 import org.apache.kafka.common.acl._
+import org.apache.kafka.common.acl.AclOperation._
+import org.apache.kafka.common.acl.AclPermissionType.{ALLOW, DENY}
 import org.apache.kafka.common.resource.{PatternType, ResourcePattern, ResourcePatternFilter, Resource => JResource, ResourceType => JResourceType}
 import org.apache.kafka.common.security.JaasUtils
 import org.apache.kafka.common.security.auth.KafkaPrincipal
-import org.apache.kafka.common.utils.{SecurityUtils, Utils}
+import org.apache.kafka.common.utils.{Utils, SecurityUtils => JSecurityUtils}
+import org.apache.kafka.server.authorizer.{Authorizer => JAuthorizer}
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable
@@ -53,7 +57,18 @@ object AclCommand extends Logging {
       if (opts.options.has(opts.bootstrapServerOpt)) {
         new AdminClientService(opts)
       } else {
-        new AuthorizerService(opts)
+        val authorizerClass = if (opts.options.has(opts.authorizerOpt)) {
+          val className = opts.options.valueOf(opts.authorizerOpt)
+          Class.forName(className, true, Utils.getContextOrKafkaClassLoader)
+        } else
+          classOf[SimpleAclAuthorizer]
+
+        if (classOf[JAuthorizer].isAssignableFrom(authorizerClass))
+          new JAuthorizerService(authorizerClass.asSubclass(classOf[JAuthorizer]), opts)
+        else if (classOf[Authorizer].isAssignableFrom(authorizerClass))
+          new AuthorizerService(authorizerClass.asSubclass(classOf[Authorizer]), opts)
+        else
+          throw new IllegalArgumentException(s"Authorizer $authorizerClass does not implement ${classOf[Authorizer]} or ${classOf[JAuthorizer]}.")
       }
     }
 
@@ -99,9 +114,8 @@ object AclCommand extends Logging {
       val resourceToAcl = getResourceToAcls(opts)
       withAdminClient(opts) { adminClient =>
         for ((resource, acls) <- resourceToAcl) {
-          val resourcePattern = resource.toPattern
-          println(s"Adding ACLs for resource `$resourcePattern`: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline")
-          val aclBindings = acls.map(acl => new AclBinding(resourcePattern, getAccessControlEntry(acl))).asJavaCollection
+          println(s"Adding ACLs for resource `$resource`: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline")
+          val aclBindings = acls.map(acl => new AclBinding(resource, acl)).asJavaCollection
           adminClient.createAcls(aclBindings).all().get()
         }
 
@@ -149,23 +163,15 @@ object AclCommand extends Logging {
       }
     }
 
-    private def getAccessControlEntry(acl: Acl): AccessControlEntry = {
-      new AccessControlEntry(acl.principal.toString, acl.host, acl.operation.toJava, acl.permissionType.toJava)
-    }
-
-    private def removeAcls(adminClient: Admin, acls: Set[Acl], filter: ResourcePatternFilter): Unit = {
+    private def removeAcls(adminClient: Admin, acls: Set[AccessControlEntry], filter: ResourcePatternFilter): Unit = {
       if (acls.isEmpty)
         adminClient.deleteAcls(List(new AclBindingFilter(filter, AccessControlEntryFilter.ANY)).asJava).all().get()
       else {
-        val aclBindingFilters = acls.map(acl => new AclBindingFilter(filter, getAccessControlEntryFilter(acl))).toList.asJava
+        val aclBindingFilters = acls.map(acl => new AclBindingFilter(filter, acl.toFilter)).toList.asJava
         adminClient.deleteAcls(aclBindingFilters).all().get()
       }
     }
 
-    private def getAccessControlEntryFilter(acl: Acl): AccessControlEntryFilter = {
-      new AccessControlEntryFilter(acl.principal.toString, acl.host, acl.operation.toJava, acl.permissionType.toJava)
-    }
-
     private def getAcls(adminClient: Admin, filters: Set[ResourcePatternFilter]): Map[ResourcePattern, Set[AccessControlEntry]] = {
       val aclBindings =
         if (filters.isEmpty) adminClient.describeAcls(AclBindingFilter.ANY).values().get().asScala.toList
@@ -183,7 +189,8 @@ object AclCommand extends Logging {
     }
   }
 
-  class AuthorizerService(val opts: AclCommandOptions) extends AclCommandService with Logging {
+  @deprecated("Use JAuthorizerService", "Since 2.4")
+  class AuthorizerService(val authorizerClass: Class[_ <: Authorizer], val opts: AclCommandOptions) extends AclCommandService with Logging {
 
     private def withAuthorizer()(f: Authorizer => Unit): Unit = {
       val defaultProps = Map(KafkaConfig.ZkEnableSecureAclsProp -> JaasUtils.isZkSecurityEnabled)
@@ -195,12 +202,7 @@ object AclCommand extends Logging {
           defaultProps
         }
 
-      val authorizerClass = if (opts.options.has(opts.authorizerOpt))
-        opts.options.valueOf(opts.authorizerOpt)
-      else
-        classOf[SimpleAclAuthorizer].getName
-
-      val authZ = CoreUtils.createObject[Authorizer](authorizerClass)
+      val authZ = Utils.newInstance(authorizerClass)
       try {
         authZ.configure(authorizerProperties.asJava)
         f(authZ)
@@ -213,7 +215,7 @@ object AclCommand extends Logging {
       withAuthorizer() { authorizer =>
         for ((resource, acls) <- resourceToAcl) {
           println(s"Adding ACLs for resource `$resource`: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline")
-          authorizer.addAcls(acls, resource)
+          authorizer.addAcls(acls.map(AuthorizerUtils.convertToAcl), AuthorizerUtils.convertToResource(resource))
         }
 
         listAcls()
@@ -258,12 +260,12 @@ object AclCommand extends Logging {
       }
     }
 
-    private def removeAcls(authorizer: Authorizer, acls: Set[Acl], filter: ResourcePatternFilter): Unit = {
+    private def removeAcls(authorizer: Authorizer, acls: Set[AccessControlEntry], filter: ResourcePatternFilter): Unit = {
       getAcls(authorizer, filter)
         .keys
         .foreach(resource =>
           if (acls.isEmpty) authorizer.removeAcls(resource)
-          else authorizer.removeAcls(acls, resource)
+          else authorizer.removeAcls(acls.map(AuthorizerUtils.convertToAcl), resource)
         )
     }
 
@@ -286,14 +288,130 @@ object AclCommand extends Logging {
 
   }
 
-  private def getResourceToAcls(opts: AclCommandOptions): Map[Resource, Set[Acl]] = {
+  class JAuthorizerService(val authorizerClass: Class[_ <: JAuthorizer], val opts: AclCommandOptions) extends AclCommandService with Logging {
+
+    private def withAuthorizer()(f: JAuthorizer => Unit) {
+      val defaultProps = Map(KafkaConfig.ZkEnableSecureAclsProp -> JaasUtils.isZkSecurityEnabled)
+      val authorizerProperties =
+        if (opts.options.has(opts.authorizerPropertiesOpt)) {
+          val authorizerProperties = opts.options.valuesOf(opts.authorizerPropertiesOpt).asScala
+          defaultProps ++ CommandLineUtils.parseKeyValueArgs(authorizerProperties, acceptMissingValue = false).asScala
+        } else {
+          defaultProps
+        }
+
+      val authZ = Utils.newInstance(authorizerClass)
+      try {
+        authZ.configure(authorizerProperties.asJava)
+        f(authZ)
+      }
+      finally CoreUtils.swallow(authZ.close(), this)
+    }
+
+    def addAcls(): Unit = {
+      val resourceToAcl = getResourceToAcls(opts)
+      withAuthorizer() { authorizer =>
+        for ((resource, acls) <- resourceToAcl) {
+          println(s"Adding ACLs for resource `$resource`: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline")
+          val aclBindings = acls.map(acl => new AclBinding(resource, acl))
+          authorizer.createAcls(null, aclBindings.toList.asJava).asScala.foreach { result =>
+            if (result.failed) {
+              println(s"Error while adding ACLs: ${result.exception.getMessage}")
+              println(Utils.stackTrace(result.exception))
+            }
+          }
+        }
+
+        listAcls()
+      }
+    }
+
+    def removeAcls(): Unit = {
+      withAuthorizer() { authorizer =>
+        val filterToAcl = getResourceFilterToAcls(opts)
+
+        for ((filter, acls) <- filterToAcl) {
+          if (acls.isEmpty) {
+            if (confirmAction(opts, s"Are you sure you want to delete all ACLs for resource filter `$filter`? (y/n)"))
+              removeAcls(authorizer, acls, filter)
+          } else {
+            if (confirmAction(opts, s"Are you sure you want to remove ACLs: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline from resource filter `$filter`? (y/n)"))
+              removeAcls(authorizer, acls, filter)
+          }
+        }
+
+        listAcls()
+      }
+    }
+
+    def listAcls(): Unit = {
+      withAuthorizer() { authorizer =>
+        val filters = getResourceFilter(opts, dieIfNoResourceFound = false)
+        val listPrincipals = getPrincipals(opts, opts.listPrincipalsOpt)
+        val resourceToAcls = getAcls(authorizer, filters)
+
+        if (listPrincipals.isEmpty) {
+          for ((resource, acls) <- resourceToAcls)
+            println(s"Current ACLs for resource `$resource`: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline")
+        } else {
+          listPrincipals.foreach(principal => {
+            println(s"ACLs for principal `$principal`")
+            val filteredResourceToAcls =  resourceToAcls.mapValues(acls =>
+              acls.filter(acl => principal.toString.equals(acl.principal))).filter(entry => entry._2.nonEmpty)
+
+            for ((resource, acls) <- filteredResourceToAcls)
+              println(s"Current ACLs for resource `$resource`: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline")
+          })
+        }
+      }
+    }
+
+    private def removeAcls(authorizer: JAuthorizer, acls: Set[AccessControlEntry], filter: ResourcePatternFilter): Unit = {
+      val result = if (acls.isEmpty)
+        authorizer.deleteAcls(null, List(new AclBindingFilter(filter, AccessControlEntryFilter.ANY)).asJava)
+      else {
+        val aclBindingFilters = acls.map(acl => new AclBindingFilter(filter, acl.toFilter)).toList.asJava
+        authorizer.deleteAcls(null, aclBindingFilters)
+      }
+      result.asScala.foreach { result =>
+        if (result.exception != null) {
+          println(s"Error while removing ACLs: ${result.exception.getMessage}")
+          println(Utils.stackTrace(result.exception))
+        }
+        result.aclBindingDeleteResults.asScala.foreach { deleteResult =>
+          if (deleteResult.exception != null) {
+            println(s"Error while removing ACLs: ${deleteResult.exception.getMessage}")
+            println(Utils.stackTrace(deleteResult.exception))
+          }
+        }
+      }
+    }
+
+    private def getAcls(authorizer: JAuthorizer, filters: Set[ResourcePatternFilter]): Map[ResourcePattern, Set[AccessControlEntry]] = {
+      val aclBindings =
+        if (filters.isEmpty) authorizer.acls(AclBindingFilter.ANY).asScala
+        else {
+          val results = for (filter <- filters) yield {
+            authorizer.acls(new AclBindingFilter(filter, AccessControlEntryFilter.ANY)).asScala
+          }
+          results.reduceLeft(_ ++ _)
+        }
+
+      val resourceToAcls = mutable.Map[ResourcePattern, Set[AccessControlEntry]]().withDefaultValue(Set())
+
+      aclBindings.foreach(aclBinding => resourceToAcls(aclBinding.pattern()) = resourceToAcls(aclBinding.pattern()) + aclBinding.entry())
+      resourceToAcls.toMap
+    }
+  }
+
+  private def getResourceToAcls(opts: AclCommandOptions): Map[ResourcePattern, Set[AccessControlEntry]] = {
     val patternType: PatternType = opts.options.valueOf(opts.resourcePatternType)
     if (!patternType.isSpecific)
       CommandLineUtils.printUsageAndDie(opts.parser, s"A '--resource-pattern-type' value of '$patternType' is not valid when adding acls.")
 
     val resourceToAcl = getResourceFilterToAcls(opts).map {
       case (filter, acls) =>
-        Resource(ResourceType.fromJava(filter.resourceType()), filter.name(), filter.patternType()) -> acls
+        new ResourcePattern(filter.resourceType(), filter.name(), filter.patternType()) -> acls
     }
 
     if (resourceToAcl.values.exists(_.isEmpty))
@@ -302,8 +420,8 @@ object AclCommand extends Logging {
     resourceToAcl
   }
 
-  private def getResourceFilterToAcls(opts: AclCommandOptions): Map[ResourcePatternFilter, Set[Acl]] = {
-    var resourceToAcls = Map.empty[ResourcePatternFilter, Set[Acl]]
+  private def getResourceFilterToAcls(opts: AclCommandOptions): Map[ResourcePatternFilter, Set[AccessControlEntry]] = {
+    var resourceToAcls = Map.empty[ResourcePatternFilter, Set[AccessControlEntry]]
 
     //if none of the --producer or --consumer options are specified , just construct ACLs from CLI options.
     if (!opts.options.has(opts.producerOpt) && !opts.options.has(opts.consumerOpt)) {
@@ -315,33 +433,33 @@ object AclCommand extends Logging {
       resourceToAcls ++= getProducerResourceFilterToAcls(opts)
 
     if (opts.options.has(opts.consumerOpt))
-      resourceToAcls ++= getConsumerResourceFilterToAcls(opts).map { case (k, v) => k -> (v ++ resourceToAcls.getOrElse(k, Set.empty[Acl])) }
+      resourceToAcls ++= getConsumerResourceFilterToAcls(opts).map { case (k, v) => k -> (v ++ resourceToAcls.getOrElse(k, Set.empty[AccessControlEntry])) }
 
     validateOperation(opts, resourceToAcls)
 
     resourceToAcls
   }
 
-  private def getProducerResourceFilterToAcls(opts: AclCommandOptions): Map[ResourcePatternFilter, Set[Acl]] = {
+  private def getProducerResourceFilterToAcls(opts: AclCommandOptions): Map[ResourcePatternFilter, Set[AccessControlEntry]] = {
     val filters = getResourceFilter(opts)
 
     val topics: Set[ResourcePatternFilter] = filters.filter(_.resourceType == JResourceType.TOPIC)
     val transactionalIds: Set[ResourcePatternFilter] = filters.filter(_.resourceType == JResourceType.TRANSACTIONAL_ID)
     val enableIdempotence = opts.options.has(opts.idempotentOpt)
 
-    val topicAcls = getAcl(opts, Set(Write, Describe, Create))
-    val transactionalIdAcls = getAcl(opts, Set(Write, Describe))
+    val topicAcls = getAcl(opts, Set(WRITE, DESCRIBE, CREATE))
+    val transactionalIdAcls = getAcl(opts, Set(WRITE, DESCRIBE))
 
     //Write, Describe, Create permission on topics, Write, Describe on transactionalIds
     topics.map(_ -> topicAcls).toMap ++
       transactionalIds.map(_ -> transactionalIdAcls).toMap ++
         (if (enableIdempotence)
-          Map(ClusterResourceFilter -> getAcl(opts, Set(IdempotentWrite)))
+          Map(ClusterResourceFilter -> getAcl(opts, Set(IDEMPOTENT_WRITE)))
         else
           Map.empty)
   }
 
-  private def getConsumerResourceFilterToAcls(opts: AclCommandOptions): Map[ResourcePatternFilter, Set[Acl]] = {
+  private def getConsumerResourceFilterToAcls(opts: AclCommandOptions): Map[ResourcePatternFilter, Set[AccessControlEntry]] = {
     val filters = getResourceFilter(opts)
 
     val topics: Set[ResourcePatternFilter] = filters.filter(_.resourceType == JResourceType.TOPIC)
@@ -349,19 +467,19 @@ object AclCommand extends Logging {
 
     //Read, Describe on topic, Read on consumerGroup
 
-    val acls = getAcl(opts, Set(Read, Describe))
+    val acls = getAcl(opts, Set(READ, DESCRIBE))
 
-    topics.map(_ -> acls).toMap[ResourcePatternFilter, Set[Acl]] ++
-      groups.map(_ -> getAcl(opts, Set(Read))).toMap[ResourcePatternFilter, Set[Acl]]
+    topics.map(_ -> acls).toMap[ResourcePatternFilter, Set[AccessControlEntry]] ++
+      groups.map(_ -> getAcl(opts, Set(READ))).toMap[ResourcePatternFilter, Set[AccessControlEntry]]
   }
 
-  private def getCliResourceFilterToAcls(opts: AclCommandOptions): Map[ResourcePatternFilter, Set[Acl]] = {
+  private def getCliResourceFilterToAcls(opts: AclCommandOptions): Map[ResourcePatternFilter, Set[AccessControlEntry]] = {
     val acls = getAcl(opts)
     val filters = getResourceFilter(opts)
     filters.map(_ -> acls).toMap
   }
 
-  private def getAcl(opts: AclCommandOptions, operations: Set[Operation]): Set[Acl] = {
+  private def getAcl(opts: AclCommandOptions, operations: Set[AclOperation]): Set[AccessControlEntry] = {
     val allowedPrincipals = getPrincipals(opts, opts.allowPrincipalsOpt)
 
     val deniedPrincipals = getPrincipals(opts, opts.denyPrincipalsOpt)
@@ -370,28 +488,28 @@ object AclCommand extends Logging {
 
     val deniedHosts = getHosts(opts, opts.denyHostsOpt, opts.denyPrincipalsOpt)
 
-    val acls = new collection.mutable.HashSet[Acl]
+    val acls = new collection.mutable.HashSet[AccessControlEntry]
     if (allowedHosts.nonEmpty && allowedPrincipals.nonEmpty)
-      acls ++= getAcls(allowedPrincipals, Allow, operations, allowedHosts)
+      acls ++= getAcls(allowedPrincipals, ALLOW, operations, allowedHosts)
 
     if (deniedHosts.nonEmpty && deniedPrincipals.nonEmpty)
-      acls ++= getAcls(deniedPrincipals, Deny, operations, deniedHosts)
+      acls ++= getAcls(deniedPrincipals, DENY, operations, deniedHosts)
 
     acls.toSet
   }
 
-  private def getAcl(opts: AclCommandOptions): Set[Acl] = {
-    val operations = opts.options.valuesOf(opts.operationsOpt).asScala.map(operation => Operation.fromString(operation.trim)).toSet
+  private def getAcl(opts: AclCommandOptions): Set[AccessControlEntry] = {
+    val operations = opts.options.valuesOf(opts.operationsOpt).asScala.map(operation => Operation.fromString(operation.trim)).map(_.toJava).toSet
     getAcl(opts, operations)
   }
 
-  def getAcls(principals: Set[KafkaPrincipal], permissionType: PermissionType, operations: Set[Operation],
-              hosts: Set[String]): Set[Acl] = {
+  def getAcls(principals: Set[KafkaPrincipal], permissionType: AclPermissionType, operations: Set[AclOperation],
+              hosts: Set[String]): Set[AccessControlEntry] = {
     for {
       principal <- principals
       operation <- operations
       host <- hosts
-    } yield new Acl(principal, permissionType, host, operation)
+    } yield new AccessControlEntry(principal.toString, host, operation, permissionType)
   }
 
   private def getHosts(opts: AclCommandOptions, hostOptionSpec: ArgumentAcceptingOptionSpec[String],
@@ -406,7 +524,7 @@ object AclCommand extends Logging {
 
   private def getPrincipals(opts: AclCommandOptions, principalOptionSpec: ArgumentAcceptingOptionSpec[String]): Set[KafkaPrincipal] = {
     if (opts.options.has(principalOptionSpec))
-      opts.options.valuesOf(principalOptionSpec).asScala.map(s => SecurityUtils.parseKafkaPrincipal(s.trim)).toSet
+      opts.options.valuesOf(principalOptionSpec).asScala.map(s => JSecurityUtils.parseKafkaPrincipal(s.trim)).toSet
     else
       Set.empty[KafkaPrincipal]
   }
@@ -444,10 +562,10 @@ object AclCommand extends Logging {
     StdIn.readLine().equalsIgnoreCase("y")
   }
 
-  private def validateOperation(opts: AclCommandOptions, resourceToAcls: Map[ResourcePatternFilter, Set[Acl]]): Unit = {
+  private def validateOperation(opts: AclCommandOptions, resourceToAcls: Map[ResourcePatternFilter, Set[AccessControlEntry]]): Unit = {
     for ((resource, acls) <- resourceToAcls) {
       val validOps = ResourceType.fromJava(resource.resourceType).supportedOperations + All
-      if ((acls.map(_.operation) -- validOps).nonEmpty)
+      if ((acls.map(_.operation) -- validOps.map(_.toJava)).nonEmpty)
         CommandLineUtils.printUsageAndDie(opts.parser, s"ResourceType ${resource.resourceType} only supports operations ${validOps.mkString(",")}")
     }
   }
diff --git a/core/src/main/scala/kafka/admin/ConfigCommand.scala b/core/src/main/scala/kafka/admin/ConfigCommand.scala
index db213db7fe4f8..1b74eb6fc6e35 100644
--- a/core/src/main/scala/kafka/admin/ConfigCommand.scala
+++ b/core/src/main/scala/kafka/admin/ConfigCommand.scala
@@ -29,7 +29,7 @@ import kafka.utils.Implicits._
 import kafka.zk.{AdminZkClient, KafkaZkClient}
 import org.apache.kafka.clients.CommonClientConfigs
 import org.apache.kafka.clients.admin.{Admin, AlterConfigOp, AlterConfigsOptions, ConfigEntry, DescribeConfigsOptions, AdminClient => JAdminClient, Config => JConfig}
-import org.apache.kafka.common.config.{ConfigResource, LogLevelConfig}
+import org.apache.kafka.common.config.ConfigResource
 import org.apache.kafka.common.config.types.Password
 import org.apache.kafka.common.errors.InvalidConfigurationException
 import org.apache.kafka.common.security.JaasUtils
diff --git a/core/src/main/scala/kafka/cluster/Broker.scala b/core/src/main/scala/kafka/cluster/Broker.scala
index d5ee82cbfc95d..2ca8148153b18 100755
--- a/core/src/main/scala/kafka/cluster/Broker.scala
+++ b/core/src/main/scala/kafka/cluster/Broker.scala
@@ -17,12 +17,17 @@
 
 package kafka.cluster
 
+import java.util
+
 import kafka.common.BrokerEndPointNotAvailableException
-import org.apache.kafka.common.Node
+import kafka.server.KafkaConfig
+import org.apache.kafka.common.{ClusterResource, Endpoint, Node}
 import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.security.auth.SecurityProtocol
+import org.apache.kafka.server.authorizer.AuthorizerServerInfo
 
 import scala.collection.Seq
+import scala.collection.JavaConverters._
 
 /**
  * A Kafka broker.
@@ -59,9 +64,26 @@ case class Broker(id: Int, endPoints: Seq[EndPoint], rack: Option[String]) {
     endPointsMap.get(listenerName).map(endpoint => new Node(id, endpoint.host, endpoint.port, rack.orNull))
 
   def brokerEndPoint(listenerName: ListenerName): BrokerEndPoint = {
-    val endpoint = endPointsMap.getOrElse(listenerName,
-      throw new BrokerEndPointNotAvailableException(s"End point with listener name ${listenerName.value} not found for broker $id"))
+    val endpoint = endPoint(listenerName)
     new BrokerEndPoint(id, endpoint.host, endpoint.port)
   }
 
+  def endPoint(listenerName: ListenerName): EndPoint = {
+    endPointsMap.getOrElse(listenerName,
+      throw new BrokerEndPointNotAvailableException(s"End point with listener name ${listenerName.value} not found for broker $id"))
+  }
+
+  def toServerInfo(clusterId: String, config: KafkaConfig): AuthorizerServerInfo = {
+    val clusterResource: ClusterResource = new ClusterResource(clusterId)
+    val interBrokerEndpoint: Endpoint = endPoint(config.interBrokerListenerName)
+    val brokerEndpoints: util.List[Endpoint] = endPoints.toList.map(_.asInstanceOf[Endpoint]).asJava
+    BrokerEndpointInfo(clusterResource, id, brokerEndpoints, interBrokerEndpoint)
+  }
+
+  case class BrokerEndpointInfo(clusterResource: ClusterResource,
+                                brokerId: Int,
+                                endpoints: util.List[Endpoint],
+                                interBrokerEndpoint: Endpoint)
+    extends AuthorizerServerInfo
+
 }
diff --git a/core/src/main/scala/kafka/cluster/EndPoint.scala b/core/src/main/scala/kafka/cluster/EndPoint.scala
index 2bca5c8797673..4fbae70baadd5 100644
--- a/core/src/main/scala/kafka/cluster/EndPoint.scala
+++ b/core/src/main/scala/kafka/cluster/EndPoint.scala
@@ -17,7 +17,7 @@
 
 package kafka.cluster
 
-import org.apache.kafka.common.KafkaException
+import org.apache.kafka.common.{Endpoint => JEndpoint, KafkaException}
 import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.security.auth.SecurityProtocol
 import org.apache.kafka.common.utils.Utils
@@ -62,7 +62,8 @@ object EndPoint {
 /**
  * Part of the broker definition - matching host/port pair to a protocol
  */
-case class EndPoint(host: String, port: Int, listenerName: ListenerName, securityProtocol: SecurityProtocol) {
+case class EndPoint(override val host: String, override val port: Int, listenerName: ListenerName, override val securityProtocol: SecurityProtocol)
+    extends JEndpoint(Option(listenerName).map(_.value).orNull, securityProtocol, host, port) {
   def connectionString: String = {
     val hostport =
       if (host == null)
@@ -71,4 +72,8 @@ case class EndPoint(host: String, port: Int, listenerName: ListenerName, securit
         Utils.formatAddress(host, port)
     listenerName.value + "://" + hostport
   }
+
+  // to keep spotbugs happy
+  override def equals(o: scala.Any): Boolean = super.equals(o)
+  override def hashCode(): Int = super.hashCode()
 }
diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala
index e7ac4a8520abe..01a24d68ddd94 100644
--- a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala
+++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala
@@ -31,7 +31,7 @@ import kafka.zk.KafkaZkClient
 import org.apache.kafka.common.{KafkaException, TopicPartition}
 import org.apache.kafka.common.internals.Topic
 import org.apache.kafka.common.metrics.stats.{Avg, Max}
-import org.apache.kafka.common.metrics.{MetricConfig, Metrics}
+import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.record.{FileRecords, MemoryRecords, SimpleRecord}
 import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse
diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala
index 0b88894888c2d..adaaabf946f25 100644
--- a/core/src/main/scala/kafka/network/SocketServer.scala
+++ b/core/src/main/scala/kafka/network/SocketServer.scala
@@ -36,7 +36,7 @@ import kafka.security.CredentialProvider
 import kafka.server.{BrokerReconfigurable, KafkaConfig}
 import kafka.utils._
 import org.apache.kafka.common.config.ConfigException
-import org.apache.kafka.common.{KafkaException, Reconfigurable}
+import org.apache.kafka.common.{Endpoint, KafkaException, Reconfigurable}
 import org.apache.kafka.common.memory.{MemoryPool, SimpleMemoryPool}
 import org.apache.kafka.common.metrics._
 import org.apache.kafka.common.metrics.stats.{CumulativeSum, Meter}
@@ -119,8 +119,8 @@ class SocketServer(val config: KafkaConfig,
       createControlPlaneAcceptorAndProcessor(config.controlPlaneListener)
       createDataPlaneAcceptorsAndProcessors(config.numNetworkThreads, config.dataPlaneListeners)
       if (startupProcessors) {
-        startControlPlaneProcessor()
-        startDataPlaneProcessors()
+        startControlPlaneProcessor(Map.empty)
+        startDataPlaneProcessors(Map.empty)
       }
     }
 
@@ -195,9 +195,27 @@ class SocketServer(val config: KafkaConfig,
    * Starts processors of all the data-plane acceptors of this server if they have not already been started.
    * This method is used for delayed starting of data-plane processors if [[kafka.network.SocketServer#startup]]
    * was invoked with `startupProcessors=false`.
+   *
+   * Before starting processors for each endpoint, we ensure that authorizer has all the metadata
+   * to authorize requests on that endpoint by waiting on the provided future. We start inter-broker listener
+   * before other listeners. This allows authorization metadata for other listeners to be stored in Kafka topics
+   * in this cluster.
    */
-  def startDataPlaneProcessors(): Unit = synchronized {
-    dataPlaneAcceptors.values.asScala.foreach { _.startProcessors(DataPlaneThreadPrefix) }
+  def startDataPlaneProcessors(authorizerFutures: Map[Endpoint, CompletableFuture[Void]] = Map.empty): Unit = synchronized {
+    val interBrokerListener = dataPlaneAcceptors.asScala.keySet
+      .find(_.listenerName == config.interBrokerListenerName)
+      .getOrElse(throw new IllegalStateException(s"Inter-broker listener ${config.interBrokerListenerName} not found, endpoints=${dataPlaneAcceptors.keySet}"))
+    val orderedAcceptors = List(dataPlaneAcceptors.get(interBrokerListener)) ++
+      dataPlaneAcceptors.asScala.filterKeys(_ != interBrokerListener).values
+    orderedAcceptors.foreach { acceptor =>
+      val endpoint = acceptor.endPoint
+      debug(s"Wait for authorizer to complete start up on listener ${endpoint.listener}")
+      authorizerFutures.get(endpoint).foreach { future =>
+        future.join()
+      }
+      debug(s"Start processors on listener ${endpoint.listener}")
+      acceptor.startProcessors(DataPlaneThreadPrefix)
+    }
     info(s"Started data-plane processors for ${dataPlaneAcceptors.size} acceptors")
   }
 
@@ -206,8 +224,9 @@ class SocketServer(val config: KafkaConfig,
    * This method is used for delayed starting of control-plane processor if [[kafka.network.SocketServer#startup]]
    * was invoked with `startupProcessors=false`.
    */
-  def startControlPlaneProcessor(): Unit = synchronized {
+  def startControlPlaneProcessor(authorizerFutures: Map[Endpoint, CompletableFuture[Void]] = Map.empty): Unit = synchronized {
     controlPlaneAcceptorOpt.foreach { controlPlaneAcceptor =>
+      authorizerFutures.get(controlPlaneAcceptor.endPoint).foreach(_.get)
       controlPlaneAcceptor.startProcessors(ControlPlaneThreadPrefix)
       info(s"Started control-plane processor for the control-plane acceptor")
     }
diff --git a/core/src/main/scala/kafka/security/auth/Authorizer.scala b/core/src/main/scala/kafka/security/auth/Authorizer.scala
index 9be8e6c704930..7509171313a53 100644
--- a/core/src/main/scala/kafka/security/auth/Authorizer.scala
+++ b/core/src/main/scala/kafka/security/auth/Authorizer.scala
@@ -32,6 +32,7 @@ import org.apache.kafka.common.security.auth.KafkaPrincipal
  * If `authorizer.class.name` has no value specified, then no authorization will be performed, and all operations are
  * permitted.
  */
+@deprecated("Use org.apache.kafka.server.authorizer.Authorizer", "Since 2.4")
 trait Authorizer extends Configurable {
 
   /**
diff --git a/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala b/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala
index 126b2b13d1de2..bdbcf9366a63d 100644
--- a/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala
+++ b/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala
@@ -17,36 +17,32 @@
 package kafka.security.auth
 
 import java.util
-import java.util.concurrent.locks.ReentrantReadWriteLock
 
-import com.typesafe.scalalogging.Logger
-import kafka.api.KAFKA_2_0_IV1
 import kafka.network.RequestChannel.Session
-import kafka.security.auth.SimpleAclAuthorizer.{NoAcls, VersionedAcls}
-import kafka.server.KafkaConfig
-import kafka.utils.CoreUtils.{inReadLock, inWriteLock}
+import kafka.security.authorizer.{AclAuthorizer, AuthorizerUtils}
 import kafka.utils._
-import kafka.zk.{AclChangeNotificationHandler, AclChangeSubscription, KafkaZkClient, ZkAclChangeStore, ZkAclStore, ZkVersion}
-import org.apache.kafka.common.errors.UnsupportedVersionException
-import org.apache.kafka.common.resource.PatternType
+import kafka.zk.ZkVersion
+import org.apache.kafka.common.acl.{AccessControlEntryFilter, AclBinding, AclBindingFilter, AclOperation, AclPermissionType}
+import org.apache.kafka.common.resource.{PatternType, ResourcePatternFilter}
 import org.apache.kafka.common.security.auth.KafkaPrincipal
-import org.apache.kafka.common.utils.{SecurityUtils, Time}
+import org.apache.kafka.server.authorizer.{Action, AuthorizableRequestContext, AuthorizationResult}
 
+import scala.collection.mutable
 import scala.collection.JavaConverters._
-import scala.util.{Failure, Random, Success, Try}
 
+@deprecated("Use kafka.security.authorizer.AclAuthorizer", "Since 2.4")
 object SimpleAclAuthorizer {
   //optional override zookeeper cluster configuration where acls will be stored, if not specified acls will be stored in
   //same zookeeper where all other kafka broker info is stored.
-  val ZkUrlProp = "authorizer.zookeeper.url"
-  val ZkConnectionTimeOutProp = "authorizer.zookeeper.connection.timeout.ms"
-  val ZkSessionTimeOutProp = "authorizer.zookeeper.session.timeout.ms"
-  val ZkMaxInFlightRequests = "authorizer.zookeeper.max.in.flight.requests"
+  val ZkUrlProp = AclAuthorizer.ZkUrlProp
+  val ZkConnectionTimeOutProp = AclAuthorizer.ZkConnectionTimeOutProp
+  val ZkSessionTimeOutProp = AclAuthorizer.ZkSessionTimeOutProp
+  val ZkMaxInFlightRequests = AclAuthorizer.ZkMaxInFlightRequests
 
   //List of users that will be treated as super users and will have access to all the resources for all actions from all hosts, defaults to no super users.
-  val SuperUsersProp = "super.users"
+  val SuperUsersProp = AclAuthorizer.SuperUsersProp
   //If set to true when no acls are found for a resource , authorizer allows access to everyone. Defaults to false.
-  val AllowEveryoneIfNoAclIsFoundProp = "allow.everyone.if.no.acl.found"
+  val AllowEveryoneIfNoAclIsFoundProp = AclAuthorizer.AllowEveryoneIfNoAclIsFoundProp
 
   case class VersionedAcls(acls: Set[Acl], zkVersion: Int) {
     def exists: Boolean = zkVersion != ZkVersion.UnknownVersion
@@ -54,361 +50,115 @@ object SimpleAclAuthorizer {
   val NoAcls = VersionedAcls(Set.empty, ZkVersion.UnknownVersion)
 }
 
+@deprecated("Use kafka.security.authorizer.AclAuthorizer", "Since 2.4")
 class SimpleAclAuthorizer extends Authorizer with Logging {
-  private val authorizerLogger = Logger("kafka.authorizer.logger")
-  private var superUsers = Set.empty[KafkaPrincipal]
-  private var shouldAllowEveryoneIfNoAclIsFound = false
-  private var zkClient: KafkaZkClient = _
-  private var aclChangeListeners: Iterable[AclChangeSubscription] = Iterable.empty
-  private var extendedAclSupport: Boolean = _
 
-  @volatile
-  private var aclCache = new scala.collection.immutable.TreeMap[Resource, VersionedAcls]()(ResourceOrdering)
-  private val lock = new ReentrantReadWriteLock()
+  private val aclAuthorizer = new BaseAuthorizer
 
   // The maximum number of times we should try to update the resource acls in zookeeper before failing;
   // This should never occur, but is a safeguard just in case.
   protected[auth] var maxUpdateRetries = 10
 
-  private val retryBackoffMs = 100
-  private val retryBackoffJitterMs = 50
 
   /**
    * Guaranteed to be called before any authorize call is made.
    */
   override def configure(javaConfigs: util.Map[String, _]): Unit = {
-    val configs = javaConfigs.asScala
-    val props = new java.util.Properties()
-    configs.foreach { case (key, value) => props.put(key, value.toString) }
-
-    superUsers = configs.get(SimpleAclAuthorizer.SuperUsersProp).collect {
-      case str: String if str.nonEmpty => str.split(";").map(s => SecurityUtils.parseKafkaPrincipal(s.trim)).toSet
-    }.getOrElse(Set.empty[KafkaPrincipal])
-
-    shouldAllowEveryoneIfNoAclIsFound = configs.get(SimpleAclAuthorizer.AllowEveryoneIfNoAclIsFoundProp).exists(_.toString.toBoolean)
-
-    // Use `KafkaConfig` in order to get the default ZK config values if not present in `javaConfigs`. Note that this
-    // means that `KafkaConfig.zkConnect` must always be set by the user (even if `SimpleAclAuthorizer.ZkUrlProp` is also
-    // set).
-    val kafkaConfig = KafkaConfig.fromProps(props, doLog = false)
-    val zkUrl = configs.get(SimpleAclAuthorizer.ZkUrlProp).map(_.toString).getOrElse(kafkaConfig.zkConnect)
-    val zkConnectionTimeoutMs = configs.get(SimpleAclAuthorizer.ZkConnectionTimeOutProp).map(_.toString.toInt).getOrElse(kafkaConfig.zkConnectionTimeoutMs)
-    val zkSessionTimeOutMs = configs.get(SimpleAclAuthorizer.ZkSessionTimeOutProp).map(_.toString.toInt).getOrElse(kafkaConfig.zkSessionTimeoutMs)
-    val zkMaxInFlightRequests = configs.get(SimpleAclAuthorizer.ZkMaxInFlightRequests).map(_.toString.toInt).getOrElse(kafkaConfig.zkMaxInFlightRequests)
-
-    val time = Time.SYSTEM
-    zkClient = KafkaZkClient(zkUrl, kafkaConfig.zkEnableSecureAcls, zkSessionTimeOutMs, zkConnectionTimeoutMs,
-      zkMaxInFlightRequests, time, "kafka.security", "SimpleAclAuthorizer", name=Some("Simple ACL authorizer"))
-    zkClient.createAclPaths()
-
-    extendedAclSupport = kafkaConfig.interBrokerProtocolVersion >= KAFKA_2_0_IV1
-
-    // Start change listeners first and then populate the cache so that there is no timing window
-    // between loading cache and processing change notifications.
-    startZkChangeListeners()
-    loadCache()
+    aclAuthorizer.configure(javaConfigs)
   }
 
   override def authorize(session: Session, operation: Operation, resource: Resource): Boolean = {
-    if (resource.patternType != PatternType.LITERAL) {
-      throw new IllegalArgumentException("Only literal resources are supported. Got: " + resource.patternType)
-    }
-
-    // ensure we compare identical classes
-    val sessionPrincipal = session.principal
-    val principal = if (classOf[KafkaPrincipal] != sessionPrincipal.getClass)
-      new KafkaPrincipal(sessionPrincipal.getPrincipalType, sessionPrincipal.getName)
-    else
-      sessionPrincipal
-
-    val host = session.clientAddress.getHostAddress
-
-    def isEmptyAclAndAuthorized(acls: Set[Acl]): Boolean = {
-      if (acls.isEmpty) {
-        // No ACLs found for this resource, permission is determined by value of config allow.everyone.if.no.acl.found
-        authorizerLogger.debug(s"No acl found for resource $resource, authorized = $shouldAllowEveryoneIfNoAclIsFound")
-        shouldAllowEveryoneIfNoAclIsFound
-      } else false
-    }
-
-    def denyAclExists(acls: Set[Acl]): Boolean = {
-      // Check if there are any Deny ACLs which would forbid this operation.
-      aclMatch(operation, resource, principal, host, Deny, acls)
-    }
-
-    def allowAclExists(acls: Set[Acl]): Boolean = {
-      // Check if there are any Allow ACLs which would allow this operation.
-      // Allowing read, write, delete, or alter implies allowing describe.
-      // See #{org.apache.kafka.common.acl.AclOperation} for more details about ACL inheritance.
-      val allowOps = operation match {
-        case Describe => Set[Operation](Describe, Read, Write, Delete, Alter)
-        case DescribeConfigs => Set[Operation](DescribeConfigs, AlterConfigs)
-        case _ => Set[Operation](operation)
-      }
-      allowOps.exists(operation => aclMatch(operation, resource, principal, host, Allow, acls))
-    }
-
-    def aclsAllowAccess = {
-      //we allow an operation if no acls are found and user has configured to allow all users
-      //when no acls are found or if no deny acls are found and at least one allow acls matches.
-      val acls = getMatchingAcls(resource.resourceType, resource.name)
-      isEmptyAclAndAuthorized(acls) || (!denyAclExists(acls) && allowAclExists(acls))
-    }
-
-    // Evaluate if operation is allowed
-    val authorized = isSuperUser(operation, resource, principal, host) || aclsAllowAccess
-
-    logAuditMessage(principal, authorized, operation, resource, host)
-    authorized
+    val requestContext = AuthorizerUtils.sessionToRequestContext(session)
+    val action = new Action(operation.toJava, resource.toPattern, 1, true, true)
+    aclAuthorizer.authorize(requestContext, List(action).asJava).asScala.head == AuthorizationResult.ALLOWED
   }
 
   def isSuperUser(operation: Operation, resource: Resource, principal: KafkaPrincipal, host: String): Boolean = {
-    if (superUsers.contains(principal)) {
-      authorizerLogger.debug(s"principal = $principal is a super user, allowing operation without checking acls.")
-      true
-    } else false
-  }
-
-  private def aclMatch(operation: Operation, resource: Resource, principal: KafkaPrincipal, host: String, permissionType: PermissionType, acls: Set[Acl]): Boolean = {
-    acls.find { acl =>
-      acl.permissionType == permissionType &&
-        (acl.principal == principal || acl.principal == Acl.WildCardPrincipal) &&
-        (operation == acl.operation || acl.operation == All) &&
-        (acl.host == host || acl.host == Acl.WildCardHost)
-    }.exists { acl =>
-      authorizerLogger.debug(s"operation = $operation on resource = $resource from host = $host is $permissionType based on acl = $acl")
-      true
-    }
+    aclAuthorizer.isSuperUser(principal)
   }
 
   override def addAcls(acls: Set[Acl], resource: Resource): Unit = {
+    aclAuthorizer.maxUpdateRetries = maxUpdateRetries
     if (acls != null && acls.nonEmpty) {
-      if (!extendedAclSupport && resource.patternType == PatternType.PREFIXED) {
-        throw new UnsupportedVersionException(s"Adding ACLs on prefixed resource patterns requires " +
-          s"${KafkaConfig.InterBrokerProtocolVersionProp} of $KAFKA_2_0_IV1 or greater")
-      }
-
-      inWriteLock(lock) {
-        updateResourceAcls(resource) { currentAcls =>
-          currentAcls ++ acls
-        }
-      }
+      val bindings = acls.map { acl => AuthorizerUtils.convertToAclBinding(resource, acl) }
+      createAcls(bindings)
     }
   }
 
   override def removeAcls(aclsTobeRemoved: Set[Acl], resource: Resource): Boolean = {
-    inWriteLock(lock) {
-      updateResourceAcls(resource) { currentAcls =>
-        currentAcls -- aclsTobeRemoved
-      }
+    val filters = aclsTobeRemoved.map { acl =>
+      new AclBindingFilter(resource.toPattern.toFilter, AuthorizerUtils.convertToAccessControlEntry(acl).toFilter)
     }
+    deleteAcls(filters)
   }
 
   override def removeAcls(resource: Resource): Boolean = {
-    inWriteLock(lock) {
-      val result = zkClient.deleteResource(resource)
-      updateCache(resource, NoAcls)
-      updateAclChangedFlag(resource)
-      result
-    }
+    val filter = new AclBindingFilter(resource.toPattern.toFilter, AccessControlEntryFilter.ANY)
+    deleteAcls(Set(filter))
   }
 
   override def getAcls(resource: Resource): Set[Acl] = {
-    inReadLock(lock) {
-      aclCache.get(resource).map(_.acls).getOrElse(Set.empty[Acl])
-    }
+    val filter = new AclBindingFilter(resource.toPattern.toFilter, AccessControlEntryFilter.ANY)
+    acls(filter).getOrElse(resource, Set.empty)
   }
 
   override def getAcls(principal: KafkaPrincipal): Map[Resource, Set[Acl]] = {
-    inReadLock(lock) {
-      unorderedAcls.flatMap { case (k, versionedAcls) =>
-        val aclsForPrincipal = versionedAcls.acls.filter(_.principal == principal)
-        if (aclsForPrincipal.nonEmpty)
-          Some(k -> aclsForPrincipal)
-        else
-          None
-      }
-    }
+    val filter = new AclBindingFilter(ResourcePatternFilter.ANY,
+      new AccessControlEntryFilter(principal.toString, null, AclOperation.ANY, AclPermissionType.ANY))
+    acls(filter)
   }
 
   def getMatchingAcls(resourceType: ResourceType, resourceName: String): Set[Acl] = {
-    inReadLock(lock) {
-      val wildcard = aclCache.get(Resource(resourceType, Acl.WildCardResource, PatternType.LITERAL))
-        .map(_.acls)
-        .getOrElse(Set.empty[Acl])
-
-      val literal = aclCache.get(Resource(resourceType, resourceName, PatternType.LITERAL))
-        .map(_.acls)
-        .getOrElse(Set.empty[Acl])
-
-      val prefixed = aclCache
-        .from(Resource(resourceType, resourceName, PatternType.PREFIXED))
-        .to(Resource(resourceType, resourceName.take(1), PatternType.PREFIXED))
-        .filterKeys(resource => resourceName.startsWith(resource.name))
-        .values
-        .flatMap { _.acls }
-        .toSet
-
-      prefixed ++ wildcard ++ literal
-    }
+    val filter = new AclBindingFilter(new ResourcePatternFilter(resourceType.toJava, resourceName, PatternType.MATCH),
+      AccessControlEntryFilter.ANY)
+    acls(filter).flatMap(_._2).toSet
   }
 
   override def getAcls(): Map[Resource, Set[Acl]] = {
-    inReadLock(lock) {
-      unorderedAcls.map { case (k, v) => k -> v.acls }
-    }
+    acls(AclBindingFilter.ANY)
   }
 
   def close(): Unit = {
-    aclChangeListeners.foreach(listener => listener.close())
-    if (zkClient != null) zkClient.close()
+    aclAuthorizer.close()
   }
 
-  private def loadCache(): Unit = {
-    inWriteLock(lock) {
-      ZkAclStore.stores.foreach(store => {
-        val resourceTypes = zkClient.getResourceTypes(store.patternType)
-        for (rType <- resourceTypes) {
-          val resourceType = Try(ResourceType.fromString(rType))
-          resourceType match {
-            case Success(resourceTypeObj) =>
-              val resourceNames = zkClient.getResourceNames(store.patternType, resourceTypeObj)
-              for (resourceName <- resourceNames) {
-                val resource = new Resource(resourceTypeObj, resourceName, store.patternType)
-                val versionedAcls = getAclsFromZk(resource)
-                updateCache(resource, versionedAcls)
-              }
-            case Failure(_) => warn(s"Ignoring unknown ResourceType: $rType")
-          }
-        }
-      })
-    }
+  private def createAcls(bindings: Set[AclBinding]): Unit = {
+    aclAuthorizer.maxUpdateRetries = maxUpdateRetries
+    val results = aclAuthorizer.createAcls(null, bindings.toList.asJava).asScala
+    results.find(_.exception != null).foreach { r => throw r.exception }
   }
 
-  private[auth] def startZkChangeListeners(): Unit = {
-    aclChangeListeners = ZkAclChangeStore.stores
-      .map(store => store.createListener(AclChangedNotificationHandler, zkClient))
+  private def deleteAcls(filters: Set[AclBindingFilter]): Boolean = {
+    aclAuthorizer.maxUpdateRetries = maxUpdateRetries
+    val results = aclAuthorizer.deleteAcls(null, filters.toList.asJava).asScala
+    results.find(_.exception != null).foreach { r => throw r.exception }
+    results.flatMap(_.aclBindingDeleteResults.asScala).find(_.exception != null).foreach { r => throw r.exception }
+    results.exists(r => r.aclBindingDeleteResults.asScala.exists(_.deleted))
   }
 
-  private def logAuditMessage(principal: KafkaPrincipal, authorized: Boolean, operation: Operation, resource: Resource, host: String): Unit = {
-    def logMessage: String = {
-      val authResult = if (authorized) "Allowed" else "Denied"
-      s"Principal = $principal is $authResult Operation = $operation from host = $host on resource = $resource"
+  private def acls(filter: AclBindingFilter): Map[Resource, Set[Acl]] = {
+    val result = mutable.Map[Resource, mutable.Set[Acl]]()
+    aclAuthorizer.acls(filter).asScala.foreach { binding =>
+      val resource = AuthorizerUtils.convertToResource(binding.pattern)
+      val acl = AuthorizerUtils.convertToAcl(binding.entry)
+      result.getOrElseUpdate(resource, mutable.Set()).add(acl)
     }
-
-    if (authorized) authorizerLogger.debug(logMessage)
-    else authorizerLogger.info(logMessage)
+    result.mapValues(_.toSet).toMap
   }
 
-  /**
-    * Safely updates the resources ACLs by ensuring reads and writes respect the expected zookeeper version.
-    * Continues to retry until it successfully updates zookeeper.
-    *
-    * Returns a boolean indicating if the content of the ACLs was actually changed.
-    *
-    * @param resource the resource to change ACLs for
-    * @param getNewAcls function to transform existing acls to new ACLs
-    * @return boolean indicating if a change was made
-    */
-  private def updateResourceAcls(resource: Resource)(getNewAcls: Set[Acl] => Set[Acl]): Boolean = {
-    var currentVersionedAcls =
-      if (aclCache.contains(resource))
-        getAclsFromCache(resource)
-      else
-        getAclsFromZk(resource)
-    var newVersionedAcls: VersionedAcls = null
-    var writeComplete = false
-    var retries = 0
-    while (!writeComplete && retries <= maxUpdateRetries) {
-      val newAcls = getNewAcls(currentVersionedAcls.acls)
-      val (updateSucceeded, updateVersion) =
-        if (newAcls.nonEmpty) {
-          if (currentVersionedAcls.exists)
-            zkClient.conditionalSetAclsForResource(resource, newAcls, currentVersionedAcls.zkVersion)
-          else
-            zkClient.createAclsForResourceIfNotExists(resource, newAcls)
-        } else {
-          trace(s"Deleting path for $resource because it had no ACLs remaining")
-          (zkClient.conditionalDelete(resource, currentVersionedAcls.zkVersion), 0)
-        }
-
-      if (!updateSucceeded) {
-        trace(s"Failed to update ACLs for $resource. Used version ${currentVersionedAcls.zkVersion}. Reading data and retrying update.")
-        Thread.sleep(backoffTime)
-        currentVersionedAcls = getAclsFromZk(resource)
-        retries += 1
-      } else {
-        newVersionedAcls = VersionedAcls(newAcls, updateVersion)
-        writeComplete = updateSucceeded
+  class BaseAuthorizer extends AclAuthorizer {
+    override def logAuditMessage(requestContext: AuthorizableRequestContext, action: Action, authorized: Boolean): Unit = {
+      val principal = requestContext.principal
+      val host = requestContext.clientAddress.getHostAddress
+      val operation = Operation.fromJava(action.operation)
+      val resource = AuthorizerUtils.convertToResource(action.resourcePattern)
+      def logMessage: String = {
+        val authResult = if (authorized) "Allowed" else "Denied"
+        s"Principal = $principal is $authResult Operation = $operation from host = $host on resource = $resource"
       }
-    }
-
-    if(!writeComplete)
-      throw new IllegalStateException(s"Failed to update ACLs for $resource after trying a maximum of $maxUpdateRetries times")
-
-    if (newVersionedAcls.acls != currentVersionedAcls.acls) {
-      debug(s"Updated ACLs for $resource to ${newVersionedAcls.acls} with version ${newVersionedAcls.zkVersion}")
-      updateCache(resource, newVersionedAcls)
-      updateAclChangedFlag(resource)
-      true
-    } else {
-      debug(s"Updated ACLs for $resource, no change was made")
-      updateCache(resource, newVersionedAcls) // Even if no change, update the version
-      false
-    }
-  }
-
-  // Returns Map instead of SortedMap since most callers don't care about ordering. In Scala 2.13, mapping from SortedMap
-  // to Map is restricted by default
-  private def unorderedAcls: Map[Resource, VersionedAcls] = aclCache
-
-  private def getAclsFromCache(resource: Resource): VersionedAcls = {
-    aclCache.getOrElse(resource, throw new IllegalArgumentException(s"ACLs do not exist in the cache for resource $resource"))
-  }
 
-  private def getAclsFromZk(resource: Resource): VersionedAcls = {
-    zkClient.getVersionedAclsForResource(resource)
-  }
-
-  private def updateCache(resource: Resource, versionedAcls: VersionedAcls): Unit = {
-    if (versionedAcls.acls.nonEmpty) {
-      aclCache = aclCache + (resource -> versionedAcls)
-    } else {
-      aclCache = aclCache - resource
-    }
-  }
-
-  private def updateAclChangedFlag(resource: Resource): Unit = {
-      zkClient.createAclChangeNotification(resource)
-  }
-
-  private def backoffTime = {
-    retryBackoffMs + Random.nextInt(retryBackoffJitterMs)
-  }
-
-  object AclChangedNotificationHandler extends AclChangeNotificationHandler {
-    override def processNotification(resource: Resource): Unit = {
-      inWriteLock(lock) {
-        val versionedAcls = getAclsFromZk(resource)
-        updateCache(resource, versionedAcls)
-      }
-    }
-  }
-
-  // Orders by resource type, then resource pattern type and finally reverse ordering by name.
-  private object ResourceOrdering extends Ordering[Resource] {
-
-    def compare(a: Resource, b: Resource): Int = {
-      val rt = a.resourceType compare b.resourceType
-      if (rt != 0)
-        rt
-      else {
-        val rnt = a.patternType compareTo b.patternType
-        if (rnt != 0)
-          rnt
-        else
-          (a.name compare b.name) * -1
-      }
+      if (authorized) authorizerLogger.debug(logMessage)
+      else authorizerLogger.info(logMessage)
     }
   }
 }
diff --git a/core/src/main/scala/kafka/security/authorizer/AclAuthorizer.scala b/core/src/main/scala/kafka/security/authorizer/AclAuthorizer.scala
new file mode 100644
index 0000000000000..5d1880b41374b
--- /dev/null
+++ b/core/src/main/scala/kafka/security/authorizer/AclAuthorizer.scala
@@ -0,0 +1,476 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package kafka.security.authorizer
+
+import java.{lang, util}
+import java.util.concurrent.CompletableFuture
+import java.util.concurrent.locks.ReentrantReadWriteLock
+
+import com.typesafe.scalalogging.Logger
+import kafka.api.KAFKA_2_0_IV1
+import kafka.security.authorizer.AclAuthorizer.VersionedAcls
+import kafka.security.auth.{Acl, Operation, PermissionType, Resource, ResourceType}
+import kafka.security.auth.{All, Allow, Alter, AlterConfigs, Delete, Deny, Describe, DescribeConfigs, Read, Write}
+import kafka.server.KafkaConfig
+import kafka.utils.CoreUtils.{inReadLock, inWriteLock}
+import kafka.utils._
+import kafka.zk._
+import org.apache.kafka.common.Endpoint
+import org.apache.kafka.common.acl._
+import org.apache.kafka.common.errors.{ApiException, InvalidRequestException, UnsupportedVersionException}
+import org.apache.kafka.common.protocol.ApiKeys
+import org.apache.kafka.common.resource.PatternType
+import org.apache.kafka.common.security.auth.KafkaPrincipal
+import org.apache.kafka.common.utils.{Time, SecurityUtils => JSecurityUtils}
+import org.apache.kafka.server.authorizer.AclDeleteResult.AclBindingDeleteResult
+import org.apache.kafka.server.authorizer.{AclCreateResult, AclDeleteResult, Action, AuthorizableRequestContext, AuthorizationResult, Authorizer, AuthorizerServerInfo}
+
+import scala.collection.mutable
+import scala.collection.JavaConverters._
+import scala.util.{Failure, Random, Success, Try}
+
+object AclAuthorizer {
+  // Optional override zookeeper cluster configuration where acls will be stored. If not specified,
+  // acls will be stored in the same zookeeper where all other kafka broker metadata is stored.
+  val ZkUrlProp = "authorizer.zookeeper.url"
+  val ZkConnectionTimeOutProp = "authorizer.zookeeper.connection.timeout.ms"
+  val ZkSessionTimeOutProp = "authorizer.zookeeper.session.timeout.ms"
+  val ZkMaxInFlightRequests = "authorizer.zookeeper.max.in.flight.requests"
+
+  // Semi-colon separated list of users that will be treated as super users and will have access to all the resources
+  // for all actions from all hosts, defaults to no super users.
+  val SuperUsersProp = "super.users"
+  // If set to true when no acls are found for a resource, authorizer allows access to everyone. Defaults to false.
+  val AllowEveryoneIfNoAclIsFoundProp = "allow.everyone.if.no.acl.found"
+
+  case class VersionedAcls(acls: Set[Acl], zkVersion: Int) {
+    def exists: Boolean = zkVersion != ZkVersion.UnknownVersion
+  }
+  val NoAcls = VersionedAcls(Set.empty, ZkVersion.UnknownVersion)
+}
+
+class AclAuthorizer extends Authorizer with Logging {
+  private[security] val authorizerLogger = Logger("kafka.authorizer.logger")
+  private var superUsers = Set.empty[KafkaPrincipal]
+  private var shouldAllowEveryoneIfNoAclIsFound = false
+  private var zkClient: KafkaZkClient = _
+  private var aclChangeListeners: Iterable[AclChangeSubscription] = Iterable.empty
+  private var extendedAclSupport: Boolean = _
+
+  @volatile
+  private var aclCache = new scala.collection.immutable.TreeMap[Resource, VersionedAcls]()(ResourceOrdering)
+  private val lock = new ReentrantReadWriteLock()
+
+  // The maximum number of times we should try to update the resource acls in zookeeper before failing;
+  // This should never occur, but is a safeguard just in case.
+  protected[security] var maxUpdateRetries = 10
+
+  private val retryBackoffMs = 100
+  private val retryBackoffJitterMs = 50
+
+  /**
+   * Guaranteed to be called before any authorize call is made.
+   */
+  override def configure(javaConfigs: util.Map[String, _]): Unit = {
+    val configs = javaConfigs.asScala
+    val props = new java.util.Properties()
+    configs.foreach { case (key, value) => props.put(key, value.toString) }
+
+    superUsers = configs.get(AclAuthorizer.SuperUsersProp).collect {
+      case str: String if str.nonEmpty => str.split(";").map(s => JSecurityUtils.parseKafkaPrincipal(s.trim)).toSet
+    }.getOrElse(Set.empty[KafkaPrincipal])
+
+    shouldAllowEveryoneIfNoAclIsFound = configs.get(AclAuthorizer.AllowEveryoneIfNoAclIsFoundProp).exists(_.toString.toBoolean)
+
+    // Use `KafkaConfig` in order to get the default ZK config values if not present in `javaConfigs`. Note that this
+    // means that `KafkaConfig.zkConnect` must always be set by the user (even if `AclAuthorizer.ZkUrlProp` is also
+    // set).
+    val kafkaConfig = KafkaConfig.fromProps(props, doLog = false)
+    val zkUrl = configs.get(AclAuthorizer.ZkUrlProp).map(_.toString).getOrElse(kafkaConfig.zkConnect)
+    val zkConnectionTimeoutMs = configs.get(AclAuthorizer.ZkConnectionTimeOutProp).map(_.toString.toInt).getOrElse(kafkaConfig.zkConnectionTimeoutMs)
+    val zkSessionTimeOutMs = configs.get(AclAuthorizer.ZkSessionTimeOutProp).map(_.toString.toInt).getOrElse(kafkaConfig.zkSessionTimeoutMs)
+    val zkMaxInFlightRequests = configs.get(AclAuthorizer.ZkMaxInFlightRequests).map(_.toString.toInt).getOrElse(kafkaConfig.zkMaxInFlightRequests)
+
+    val time = Time.SYSTEM
+    zkClient = KafkaZkClient(zkUrl, kafkaConfig.zkEnableSecureAcls, zkSessionTimeOutMs, zkConnectionTimeoutMs,
+      zkMaxInFlightRequests, time, "kafka.security", "AclAuthorizer", name=Some("ACL authorizer"))
+    zkClient.createAclPaths()
+
+    extendedAclSupport = kafkaConfig.interBrokerProtocolVersion >= KAFKA_2_0_IV1
+
+    // Start change listeners first and then populate the cache so that there is no timing window
+    // between loading cache and processing change notifications.
+    startZkChangeListeners()
+    loadCache()
+  }
+
+  override def start(serverInfo: AuthorizerServerInfo): util.Map[Endpoint, CompletableFuture[Void]] = {
+    serverInfo.endpoints.asScala.map { endpoint =>
+      endpoint -> CompletableFuture.completedFuture[Void](null) }.toMap.asJava
+  }
+
+  override def authorize(requestContext: AuthorizableRequestContext, actions: util.List[Action]): util.List[AuthorizationResult] = {
+    actions.asScala.map { action => authorizeAction(requestContext, action) }.asJava
+  }
+
+  override def createAcls(requestContext: AuthorizableRequestContext, aclBindings: util.List[AclBinding]): util.List[AclCreateResult] = {
+    val results = new Array[AclCreateResult](aclBindings.size)
+    val aclsToCreate = aclBindings.asScala.zipWithIndex
+      .filter { case (aclBinding, i) =>
+        try {
+          if (!extendedAclSupport && aclBinding.pattern.patternType == PatternType.PREFIXED) {
+            throw new UnsupportedVersionException(s"Adding ACLs on prefixed resource patterns requires " +
+              s"${KafkaConfig.InterBrokerProtocolVersionProp} of $KAFKA_2_0_IV1 or greater")
+          }
+          AuthorizerUtils.validateAclBinding(aclBinding)
+          true
+        } catch {
+          case e: ApiException =>
+            results(i) = new AclCreateResult(e)
+            false
+          case e: Throwable =>
+            results(i) = new AclCreateResult(new InvalidRequestException("Failed to create ACL", e))
+            false
+        }
+      }.groupBy(_._1.pattern)
+
+    if (aclsToCreate.nonEmpty) {
+      inWriteLock(lock) {
+        aclsToCreate.foreach { case (resource, aclsWithIndex) =>
+          updateResourceAcls(AuthorizerUtils.convertToResource(resource)) { currentAcls =>
+            val newAcls = aclsWithIndex.map { case (acl, index) => AuthorizerUtils.convertToAcl(acl.entry) }
+            currentAcls ++ newAcls
+          }
+          aclsWithIndex.foreach { case (_, index) => results(index) = AclCreateResult.SUCCESS }
+        }
+      }
+    }
+    results.toList.asJava
+  }
+
+  override def deleteAcls(requestContext: AuthorizableRequestContext, aclBindingFilters: util.List[AclBindingFilter]): util.List[AclDeleteResult] = {
+    val deletedBindings = new mutable.HashMap[AclBinding, Int]()
+    val filters = aclBindingFilters.asScala.zipWithIndex
+    inWriteLock(lock) {
+      val resourcesToUpdate = aclCache.keys.map { resource =>
+        val matchingFilters = filters.filter { case (filter, _) =>
+          filter.patternFilter.matches(resource.toPattern)
+        }
+        resource -> matchingFilters
+      }.toMap.filter(_._2.nonEmpty)
+
+      resourcesToUpdate.foreach { case (resource, matchingFilters) =>
+        updateResourceAcls(resource) { currentAcls =>
+          val aclsToRemove = currentAcls.filter { acl =>
+            matchingFilters.exists { case (filter, index) =>
+              val matches = filter.entryFilter.matches(AuthorizerUtils.convertToAccessControlEntry(acl))
+              if (matches)
+                deletedBindings.getOrElseUpdate(AuthorizerUtils.convertToAclBinding(resource, acl), index)
+              matches
+            }
+          }
+          currentAcls -- aclsToRemove
+        }
+      }
+    }
+    val deletedResult = deletedBindings.groupBy(_._2)
+      .mapValues(_.map{ case (binding, _) => new AclBindingDeleteResult(binding) })
+    (0 until aclBindingFilters.size).map { i =>
+      new AclDeleteResult(deletedResult.getOrElse(i, Set.empty[AclBindingDeleteResult]).toSet.asJava)
+    }.asJava
+  }
+
+  override def acls(filter: AclBindingFilter): lang.Iterable[AclBinding] = {
+    inReadLock(lock) {
+      unorderedAcls.flatMap { case (resource, versionedAcls) =>
+        versionedAcls.acls.map(acl => AuthorizerUtils.convertToAclBinding(resource, acl))
+            .filter(filter.matches)
+      }.asJava
+    }
+  }
+
+  override def close(): Unit = {
+    aclChangeListeners.foreach(listener => listener.close())
+    if (zkClient != null) zkClient.close()
+  }
+
+  private def authorizeAction(requestContext: AuthorizableRequestContext, action: Action): AuthorizationResult = {
+    val resource = AuthorizerUtils.convertToResource(action.resourcePattern)
+    if (resource.patternType != PatternType.LITERAL) {
+      throw new IllegalArgumentException("Only literal resources are supported. Got: " + resource.patternType)
+    }
+
+    // ensure we compare identical classes
+    val sessionPrincipal = requestContext.principal
+    val principal = if (classOf[KafkaPrincipal] != sessionPrincipal.getClass)
+      new KafkaPrincipal(sessionPrincipal.getPrincipalType, sessionPrincipal.getName)
+    else
+      sessionPrincipal
+
+    val host = requestContext.clientAddress.getHostAddress
+    val operation = Operation.fromJava(action.operation)
+
+    def isEmptyAclAndAuthorized(acls: Set[Acl]): Boolean = {
+      if (acls.isEmpty) {
+        // No ACLs found for this resource, permission is determined by value of config allow.everyone.if.no.acl.found
+        authorizerLogger.debug(s"No acl found for resource $resource, authorized = $shouldAllowEveryoneIfNoAclIsFound")
+        shouldAllowEveryoneIfNoAclIsFound
+      } else false
+    }
+
+    def denyAclExists(acls: Set[Acl]): Boolean = {
+      // Check if there are any Deny ACLs which would forbid this operation.
+      matchingAclExists(operation, resource, principal, host, Deny, acls)
+    }
+
+    def allowAclExists(acls: Set[Acl]): Boolean = {
+      // Check if there are any Allow ACLs which would allow this operation.
+      // Allowing read, write, delete, or alter implies allowing describe.
+      // See #{org.apache.kafka.common.acl.AclOperation} for more details about ACL inheritance.
+      val allowOps = operation match {
+        case Describe => Set[Operation](Describe, Read, Write, Delete, Alter)
+        case DescribeConfigs => Set[Operation](DescribeConfigs, AlterConfigs)
+        case _ => Set[Operation](operation)
+      }
+      allowOps.exists(operation => matchingAclExists(operation, resource, principal, host, Allow, acls))
+    }
+
+    def aclsAllowAccess = {
+      //we allow an operation if no acls are found and user has configured to allow all users
+      //when no acls are found or if no deny acls are found and at least one allow acls matches.
+      val acls = matchingAcls(resource.resourceType, resource.name)
+      isEmptyAclAndAuthorized(acls) || (!denyAclExists(acls) && allowAclExists(acls))
+    }
+
+    // Evaluate if operation is allowed
+    val authorized = isSuperUser(principal) || aclsAllowAccess
+
+    logAuditMessage(requestContext, action, authorized)
+    if (authorized) AuthorizationResult.ALLOWED else AuthorizationResult.DENIED
+  }
+
+  def isSuperUser(principal: KafkaPrincipal): Boolean = {
+    if (superUsers.contains(principal)) {
+      authorizerLogger.debug(s"principal = $principal is a super user, allowing operation without checking acls.")
+      true
+    } else false
+  }
+
+  private def matchingAcls(resourceType: ResourceType, resourceName: String): Set[Acl] = {
+    inReadLock(lock) {
+      val wildcard = aclCache.get(Resource(resourceType, Acl.WildCardResource, PatternType.LITERAL))
+        .map(_.acls)
+        .getOrElse(Set.empty[Acl])
+
+      val literal = aclCache.get(Resource(resourceType, resourceName, PatternType.LITERAL))
+        .map(_.acls)
+        .getOrElse(Set.empty[Acl])
+
+      val prefixed = aclCache
+        .from(Resource(resourceType, resourceName, PatternType.PREFIXED))
+        .to(Resource(resourceType, resourceName.take(1), PatternType.PREFIXED))
+        .filterKeys(resource => resourceName.startsWith(resource.name))
+        .values
+        .flatMap { _.acls }
+        .toSet
+
+      prefixed ++ wildcard ++ literal
+    }
+  }
+
+  private def matchingAclExists(operation: Operation, resource: Resource, principal: KafkaPrincipal, host: String, permissionType: PermissionType, acls: Set[Acl]): Boolean = {
+    acls.find { acl =>
+      acl.permissionType == permissionType &&
+        (acl.principal == principal || acl.principal == Acl.WildCardPrincipal) &&
+        (operation == acl.operation || acl.operation == All) &&
+        (acl.host == host || acl.host == Acl.WildCardHost)
+    }.exists { acl =>
+      authorizerLogger.debug(s"operation = $operation on resource = $resource from host = $host is $permissionType based on acl = $acl")
+      true
+    }
+  }
+
+  private def loadCache(): Unit = {
+    inWriteLock(lock) {
+      ZkAclStore.stores.foreach(store => {
+        val resourceTypes = zkClient.getResourceTypes(store.patternType)
+        for (rType <- resourceTypes) {
+          val resourceType = Try(ResourceType.fromString(rType))
+          resourceType match {
+            case Success(resourceTypeObj) =>
+              val resourceNames = zkClient.getResourceNames(store.patternType, resourceTypeObj)
+              for (resourceName <- resourceNames) {
+                val resource = new Resource(resourceTypeObj, resourceName, store.patternType)
+                val versionedAcls = getAclsFromZk(resource)
+                updateCache(resource, versionedAcls)
+              }
+            case Failure(_) => warn(s"Ignoring unknown ResourceType: $rType")
+          }
+        }
+      })
+    }
+  }
+
+  private[authorizer] def startZkChangeListeners(): Unit = {
+    aclChangeListeners = ZkAclChangeStore.stores
+      .map(store => store.createListener(AclChangedNotificationHandler, zkClient))
+  }
+
+  def logAuditMessage(requestContext: AuthorizableRequestContext, action: Action, authorized: Boolean): Unit = {
+    def logMessage: String = {
+      val principal = requestContext.principal
+      val operation = Operation.fromJava(action.operation)
+      val host = requestContext.clientAddress.getHostAddress
+      val resource = AuthorizerUtils.convertToResource(action.resourcePattern)
+      val authResult = if (authorized) "Allowed" else "Denied"
+      val apiKey = if (ApiKeys.hasId(requestContext.requestType)) ApiKeys.forId(requestContext.requestType).name else requestContext.requestType
+      val refCount = action.resourceReferenceCount
+      s"Principal = $principal is $authResult Operation = $operation from host = $host on resource = $resource for request = $apiKey with resourceRefCount = $refCount"
+    }
+
+    if (authorized) {
+      // logIfAllowed is true if access is granted to the resource as a result of this authorization.
+      // In this case, log at debug level. If false, no access is actually granted, the result is used
+      // only to determine authorized operations. So log only at trace level.
+      if (action.logIfAllowed)
+        authorizerLogger.debug(logMessage)
+      else
+        authorizerLogger.trace(logMessage)
+    } else {
+      // logIfDenied is true if access to the resource was explicitly requested. Since this is an attempt
+      // to access unauthorized resources, log at info level. If false, this is either a request to determine
+      // authorized operations or a filter (e.g for regex subscriptions) to filter out authorized resources.
+      // In this case, log only at trace level.
+      if (action.logIfDenied)
+        authorizerLogger.info(logMessage)
+      else
+        authorizerLogger.trace(logMessage)
+    }
+  }
+
+  /**
+    * Safely updates the resources ACLs by ensuring reads and writes respect the expected zookeeper version.
+    * Continues to retry until it successfully updates zookeeper.
+    *
+    * Returns a boolean indicating if the content of the ACLs was actually changed.
+    *
+    * @param resource the resource to change ACLs for
+    * @param getNewAcls function to transform existing acls to new ACLs
+    * @return boolean indicating if a change was made
+    */
+  private def updateResourceAcls(resource: Resource)(getNewAcls: Set[Acl] => Set[Acl]): Boolean = {
+    var currentVersionedAcls =
+      if (aclCache.contains(resource))
+        getAclsFromCache(resource)
+      else
+        getAclsFromZk(resource)
+    var newVersionedAcls: VersionedAcls = null
+    var writeComplete = false
+    var retries = 0
+    while (!writeComplete && retries <= maxUpdateRetries) {
+      val newAcls = getNewAcls(currentVersionedAcls.acls)
+      val (updateSucceeded, updateVersion) =
+        if (newAcls.nonEmpty) {
+          if (currentVersionedAcls.exists)
+            zkClient.conditionalSetAclsForResource(resource, newAcls, currentVersionedAcls.zkVersion)
+          else
+            zkClient.createAclsForResourceIfNotExists(resource, newAcls)
+        } else {
+          trace(s"Deleting path for $resource because it had no ACLs remaining")
+          (zkClient.conditionalDelete(resource, currentVersionedAcls.zkVersion), 0)
+        }
+
+      if (!updateSucceeded) {
+        trace(s"Failed to update ACLs for $resource. Used version ${currentVersionedAcls.zkVersion}. Reading data and retrying update.")
+        Thread.sleep(backoffTime)
+        currentVersionedAcls = getAclsFromZk(resource)
+        retries += 1
+      } else {
+        newVersionedAcls = VersionedAcls(newAcls, updateVersion)
+        writeComplete = updateSucceeded
+      }
+    }
+
+    if(!writeComplete)
+      throw new IllegalStateException(s"Failed to update ACLs for $resource after trying a maximum of $maxUpdateRetries times")
+
+    if (newVersionedAcls.acls != currentVersionedAcls.acls) {
+      debug(s"Updated ACLs for $resource to ${newVersionedAcls.acls} with version ${newVersionedAcls.zkVersion}")
+      updateCache(resource, newVersionedAcls)
+      updateAclChangedFlag(resource)
+      true
+    } else {
+      debug(s"Updated ACLs for $resource, no change was made")
+      updateCache(resource, newVersionedAcls) // Even if no change, update the version
+      false
+    }
+  }
+
+  // Returns Map instead of SortedMap since most callers don't care about ordering. In Scala 2.13, mapping from SortedMap
+  // to Map is restricted by default
+  private def unorderedAcls: Map[Resource, VersionedAcls] = aclCache
+
+  private def getAclsFromCache(resource: Resource): VersionedAcls = {
+    aclCache.getOrElse(resource, throw new IllegalArgumentException(s"ACLs do not exist in the cache for resource $resource"))
+  }
+
+  private def getAclsFromZk(resource: Resource): VersionedAcls = {
+    zkClient.getVersionedAclsForResource(resource)
+  }
+
+  private def updateCache(resource: Resource, versionedAcls: VersionedAcls): Unit = {
+    if (versionedAcls.acls.nonEmpty) {
+      aclCache = aclCache + (resource -> versionedAcls)
+    } else {
+      aclCache = aclCache - resource
+    }
+  }
+
+  private def updateAclChangedFlag(resource: Resource): Unit = {
+      zkClient.createAclChangeNotification(resource)
+  }
+
+  private def backoffTime = {
+    retryBackoffMs + Random.nextInt(retryBackoffJitterMs)
+  }
+
+  object AclChangedNotificationHandler extends AclChangeNotificationHandler {
+    override def processNotification(resource: Resource): Unit = {
+      inWriteLock(lock) {
+        val versionedAcls = getAclsFromZk(resource)
+        updateCache(resource, versionedAcls)
+      }
+    }
+  }
+
+  // Orders by resource type, then resource pattern type and finally reverse ordering by name.
+  private object ResourceOrdering extends Ordering[Resource] {
+
+    def compare(a: Resource, b: Resource): Int = {
+      val rt = a.resourceType compare b.resourceType
+      if (rt != 0)
+        rt
+      else {
+        val rnt = a.patternType compareTo b.patternType
+        if (rnt != 0)
+          rnt
+        else
+          (a.name compare b.name) * -1
+      }
+    }
+  }
+}
diff --git a/core/src/main/scala/kafka/security/SecurityUtils.scala b/core/src/main/scala/kafka/security/authorizer/AuthorizerUtils.scala
similarity index 52%
rename from core/src/main/scala/kafka/security/SecurityUtils.scala
rename to core/src/main/scala/kafka/security/authorizer/AuthorizerUtils.scala
index 311e195795d7e..e85e1063d0a3b 100644
--- a/core/src/main/scala/kafka/security/SecurityUtils.scala
+++ b/core/src/main/scala/kafka/security/authorizer/AuthorizerUtils.scala
@@ -15,18 +15,26 @@
   * limitations under the License.
   */
 
-package kafka.security
+package kafka.security.authorizer
 
-import kafka.security.auth.{Acl, Operation, PermissionType, Resource, ResourceType}
-import org.apache.kafka.common.acl.{AccessControlEntry, AclBinding, AclBindingFilter}
+import java.net.InetAddress
+
+import kafka.network.RequestChannel.Session
+import kafka.security.auth._
+import org.apache.kafka.common.acl.{AccessControlEntry, AclBinding, AclBindingFilter, AclOperation}
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.requests.ApiError
-import org.apache.kafka.common.resource.ResourcePattern
+import org.apache.kafka.common.resource.{ResourcePattern, ResourceType => JResourceType}
+import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
 import org.apache.kafka.common.utils.SecurityUtils._
+import org.apache.kafka.server.authorizer.AuthorizableRequestContext
+
 import scala.util.{Failure, Success, Try}
 
 
-object SecurityUtils {
+object AuthorizerUtils {
+  val WildcardPrincipal = "User:*"
+  val WildcardHost = "*"
 
   def convertToResourceAndAcl(filter: AclBindingFilter): Either[ApiError, (Resource, Acl)] = {
     (for {
@@ -44,10 +52,44 @@ object SecurityUtils {
 
   def convertToAclBinding(resource: Resource, acl: Acl): AclBinding = {
     val resourcePattern = new ResourcePattern(resource.resourceType.toJava, resource.name, resource.patternType)
-    val entry = new AccessControlEntry(acl.principal.toString, acl.host.toString,
+    new AclBinding(resourcePattern, convertToAccessControlEntry(acl))
+  }
+
+  def convertToAccessControlEntry(acl: Acl): AccessControlEntry = {
+    new AccessControlEntry(acl.principal.toString, acl.host.toString,
       acl.operation.toJava, acl.permissionType.toJava)
-    new AclBinding(resourcePattern, entry)
+  }
+
+  def convertToAcl(ace: AccessControlEntry): Acl = {
+    new Acl(parseKafkaPrincipal(ace.principal), PermissionType.fromJava(ace.permissionType), ace.host,
+      Operation.fromJava(ace.operation))
+  }
+
+  def convertToResource(resourcePattern: ResourcePattern): Resource = {
+    Resource(ResourceType.fromJava(resourcePattern.resourceType), resourcePattern.name, resourcePattern.patternType)
+  }
+
+  def validateAclBinding(aclBinding: AclBinding): Unit = {
+    if (aclBinding.isUnknown)
+      throw new IllegalArgumentException("ACL binding contains unknown elements")
+  }
+
+  def supportedOperations(resourceType: JResourceType): Set[AclOperation] = {
+    ResourceType.fromJava(resourceType).supportedOperations.map(_.toJava)
   }
 
   def isClusterResource(name: String): Boolean = name.equals(Resource.ClusterResourceName)
+
+  def sessionToRequestContext(session: Session): AuthorizableRequestContext = {
+    new AuthorizableRequestContext {
+      override def clientId(): String = ""
+      override def requestType(): Int = -1
+      override def listener(): String = ""
+      override def clientAddress(): InetAddress = session.clientAddress
+      override def principal(): KafkaPrincipal = session.principal
+      override def securityProtocol(): SecurityProtocol = null
+      override def correlationId(): Int = -1
+      override def requestVersion(): Int = -1
+    }
+  }
 }
diff --git a/core/src/main/scala/kafka/security/authorizer/AuthorizerWrapper.scala b/core/src/main/scala/kafka/security/authorizer/AuthorizerWrapper.scala
new file mode 100644
index 0000000000000..86ee341786a57
--- /dev/null
+++ b/core/src/main/scala/kafka/security/authorizer/AuthorizerWrapper.scala
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.security.authorizer
+
+import java.util.concurrent.CompletableFuture
+import java.{lang, util}
+
+import kafka.network.RequestChannel.Session
+import kafka.security.auth.{Acl, Operation, Resource}
+import org.apache.kafka.common.Endpoint
+import org.apache.kafka.common.acl.{AccessControlEntry, AclBinding, AclBindingFilter}
+import org.apache.kafka.common.errors.{ApiException, InvalidRequestException}
+import org.apache.kafka.common.requests.ApiError
+import org.apache.kafka.common.resource.ResourcePattern
+import org.apache.kafka.server.authorizer.AclDeleteResult.AclBindingDeleteResult
+import org.apache.kafka.server.authorizer.{AuthorizableRequestContext, AuthorizerServerInfo, _}
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+import scala.collection.{Seq, immutable, mutable}
+
+class AuthorizerWrapper(private[kafka] val baseAuthorizer: kafka.security.auth.Authorizer) extends Authorizer {
+
+  override def configure(configs: util.Map[String, _]): Unit = {
+    baseAuthorizer.configure(configs)
+  }
+
+  override def start(serverInfo: AuthorizerServerInfo): util.Map[Endpoint, CompletableFuture[Void]] = {
+    serverInfo.endpoints.asScala.map { endpoint =>
+      endpoint -> CompletableFuture.completedFuture[Void](null) }.toMap.asJava
+  }
+
+  override def authorize(requestContext: AuthorizableRequestContext, actions: util.List[Action]): util.List[AuthorizationResult] = {
+    val session = Session(requestContext.principal, requestContext.clientAddress)
+    actions.asScala.map { action =>
+      val operation = Operation.fromJava(action.operation)
+      if (baseAuthorizer.authorize(session, operation, AuthorizerUtils.convertToResource(action.resourcePattern)))
+        AuthorizationResult.ALLOWED
+      else
+        AuthorizationResult.DENIED
+    }.asJava
+  }
+
+  override def createAcls(requestContext: AuthorizableRequestContext,
+                          aclBindings: util.List[AclBinding]): util.List[AclCreateResult] = {
+    aclBindings.asScala
+      .map { aclBinding =>
+        AuthorizerUtils.convertToResourceAndAcl(aclBinding.toFilter) match {
+          case Left(apiError) => new AclCreateResult(apiError.exception)
+          case Right((resource, acl)) =>
+            try {
+              baseAuthorizer.addAcls(Set(acl), resource)
+              new AclCreateResult
+            } catch {
+              case e: ApiException => new AclCreateResult(e)
+              case e: Throwable => new AclCreateResult(new InvalidRequestException("Failed to create ACL", e))
+            }
+        }
+      }.toList.asJava
+  }
+
+  override def deleteAcls(requestContext: AuthorizableRequestContext,
+                          aclBindingFilters: util.List[AclBindingFilter]): util.List[AclDeleteResult] = {
+    val filters = aclBindingFilters.asScala
+    val results = mutable.Map[Int, AclDeleteResult]()
+    val toDelete = mutable.Map[Int, ArrayBuffer[(Resource, Acl)]]()
+
+    if (filters.forall(_.matchesAtMostOne)) {
+      // Delete based on a list of ACL fixtures.
+      for ((filter, i) <- filters.zipWithIndex) {
+        AuthorizerUtils.convertToResourceAndAcl(filter) match {
+          case Left(apiError) => results.put(i, new AclDeleteResult(apiError.exception))
+          case Right(binding) => toDelete.put(i, ArrayBuffer(binding))
+        }
+      }
+    } else {
+      // Delete based on filters that may match more than one ACL.
+      val aclMap = baseAuthorizer.getAcls()
+      val filtersWithIndex = filters.zipWithIndex
+      for ((resource, acls) <- aclMap; acl <- acls) {
+        val binding = new AclBinding(
+          new ResourcePattern(resource.resourceType.toJava, resource.name, resource.patternType),
+          new AccessControlEntry(acl.principal.toString, acl.host.toString, acl.operation.toJava,
+            acl.permissionType.toJava))
+
+        for ((filter, i) <- filtersWithIndex if filter.matches(binding))
+          toDelete.getOrElseUpdate(i, ArrayBuffer.empty) += ((resource, acl))
+      }
+    }
+
+    for ((i, acls) <- toDelete) {
+      val deletionResults = acls.flatMap { case (resource, acl) =>
+        val aclBinding = AuthorizerUtils.convertToAclBinding(resource, acl)
+        try {
+          if (baseAuthorizer.removeAcls(immutable.Set(acl), resource))
+            Some(new AclBindingDeleteResult(aclBinding))
+          else None
+        } catch {
+          case throwable: Throwable =>
+            Some(new AclBindingDeleteResult(aclBinding, ApiError.fromThrowable(throwable).exception))
+        }
+      }.asJava
+
+      results.put(i, new AclDeleteResult(deletionResults))
+    }
+
+    filters.indices.map { i =>
+      results.getOrElse(i, new AclDeleteResult(Seq.empty[AclBindingDeleteResult].asJava))
+    }.asJava
+  }
+
+  override def acls(filter: AclBindingFilter): lang.Iterable[AclBinding] = {
+    baseAuthorizer.getAcls().flatMap { case (resource, acls) =>
+      acls.map(acl => AuthorizerUtils.convertToAclBinding(resource, acl)).filter(filter.matches)
+    }.asJava
+  }
+
+  override def close(): Unit = {
+    baseAuthorizer.close()
+  }
+}
\ No newline at end of file
diff --git a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
index 3294c284170f4..b517875e8e0b6 100755
--- a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
+++ b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
@@ -233,6 +233,10 @@ class DynamicBrokerConfig(private val kafkaConfig: KafkaConfig) extends Logging
    * directly. They are provided both old and new configs.
    */
   def addReconfigurables(kafkaServer: KafkaServer): Unit = {
+    kafkaServer.authorizer match {
+      case Some(authz: Reconfigurable) => addReconfigurable(authz)
+      case _ =>
+    }
     addReconfigurable(new DynamicMetricsReporters(kafkaConfig.brokerId, kafkaServer))
     addReconfigurable(new DynamicClientQuotaCallback(kafkaConfig.brokerId, kafkaServer))
 
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 77b137783910a..300003fc76628 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -21,7 +21,7 @@ import java.lang.{Byte => JByte}
 import java.lang.{Long => JLong}
 import java.nio.ByteBuffer
 import java.util
-import java.util.Optional
+import java.util.{Collections, Optional}
 import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.atomic.AtomicInteger
 
@@ -35,14 +35,14 @@ import kafka.coordinator.group.{GroupCoordinator, JoinGroupResult, LeaveGroupRes
 import kafka.coordinator.transaction.{InitProducerIdResult, TransactionCoordinator}
 import kafka.message.ZStdCompressionCodec
 import kafka.network.RequestChannel
-import kafka.security.SecurityUtils
-import kafka.security.auth.{Resource, _}
+import kafka.security.authorizer.AuthorizerUtils
 import kafka.server.QuotaFactory.{QuotaManagers, UnboundedQuota}
 import kafka.utils.{CoreUtils, Logging}
 import kafka.zk.{AdminZkClient, KafkaZkClient}
 import org.apache.kafka.clients.admin.{AlterConfigOp, ConfigEntry}
 import org.apache.kafka.clients.admin.AlterConfigOp.OpType
-import org.apache.kafka.common.acl.{AccessControlEntry, AclBinding}
+import org.apache.kafka.common.acl.{AclBinding, AclOperation}
+import org.apache.kafka.common.acl.AclOperation._
 import org.apache.kafka.common.config.ConfigResource
 import org.apache.kafka.common.errors._
 import org.apache.kafka.common.internals.FatalExitError
@@ -85,17 +85,18 @@ import org.apache.kafka.common.requests.DescribeLogDirsResponse.LogDirInfo
 import org.apache.kafka.common.requests.FindCoordinatorRequest.CoordinatorType
 import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse
 import org.apache.kafka.common.requests._
-import org.apache.kafka.common.resource.PatternType.LITERAL
-import org.apache.kafka.common.resource.{PatternType, ResourcePattern}
+import org.apache.kafka.common.resource.Resource.CLUSTER_NAME
+import org.apache.kafka.common.resource.ResourceType._
+import org.apache.kafka.common.resource.{PatternType, Resource, ResourcePattern, ResourceType}
 import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
 import org.apache.kafka.common.security.token.delegation.{DelegationToken, TokenInformation}
 import org.apache.kafka.common.utils.{Time, Utils}
 import org.apache.kafka.common.{Node, TopicPartition}
+import org.apache.kafka.server.authorizer._
 
 import scala.compat.java8.OptionConverters._
 import scala.collection.JavaConverters._
 import scala.collection.{Map, Seq, Set, immutable, mutable}
-import scala.collection.mutable.ArrayBuffer
 import scala.util.{Failure, Success, Try}
 
 
@@ -219,7 +220,7 @@ class KafkaApis(val requestChannel: RequestChannel,
       }
     }
 
-    authorizeClusterAction(request)
+    authorizeClusterOperation(request, CLUSTER_ACTION)
     if (isBrokerEpochStale(leaderAndIsrRequest.brokerEpoch())) {
       // When the broker restarts very quickly, it is possible for this broker to receive request intended
       // for its previous generation so the broker should skip the stale request.
@@ -237,7 +238,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     // We can't have the ensureTopicExists check here since the controller sends it as an advisory to all brokers so they
     // stop serving data to clients for the topic being deleted
     val stopReplicaRequest = request.body[StopReplicaRequest]
-    authorizeClusterAction(request)
+    authorizeClusterOperation(request, CLUSTER_ACTION)
     if (isBrokerEpochStale(stopReplicaRequest.brokerEpoch())) {
       // When the broker restarts very quickly, it is possible for this broker to receive request intended
       // for its previous generation so the broker should skip the stale request.
@@ -267,7 +268,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     val correlationId = request.header.correlationId
     val updateMetadataRequest = request.body[UpdateMetadataRequest]
 
-    authorizeClusterAction(request)
+    authorizeClusterOperation(request, CLUSTER_ACTION)
     if (isBrokerEpochStale(updateMetadataRequest.brokerEpoch())) {
       // When the broker restarts very quickly, it is possible for this broker to receive request intended
       // for its previous generation so the broker should skip the stale request.
@@ -305,7 +306,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     // We can't have the ensureTopicExists check here since the controller sends it as an advisory to all brokers so they
     // stop serving data to clients for the topic being deleted
     val controlledShutdownRequest = request.body[ControlledShutdownRequest]
-    authorizeClusterAction(request)
+    authorizeClusterOperation(request, CLUSTER_ACTION)
 
     def controlledShutdownCallback(controlledShutdownResult: Try[Set[TopicPartition]]): Unit = {
       val response = controlledShutdownResult match {
@@ -344,7 +345,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     }
 
     // reject the request if not authorized to the group
-    if (!authorize(request.session, Read, Resource(Group, offsetCommitRequest.data().groupId, LITERAL))) {
+    if (!authorize(request, READ, GROUP, offsetCommitRequest.data().groupId)) {
       val error = Errors.GROUP_AUTHORIZATION_FAILED
       val responseTopicList = OffsetCommitRequest.getErrorResponseTopics(
         offsetCommitRequest.data().topics(),
@@ -370,10 +371,11 @@ class KafkaApis(val requestChannel: RequestChannel,
     } else {
       val authorizedTopicRequestInfoBldr = immutable.Map.newBuilder[TopicPartition, OffsetCommitRequestData.OffsetCommitRequestPartition]
 
+      val authorizedTopics = filterAuthorized(request, READ, TOPIC, offsetCommitRequest.data.topics.asScala.map(_.name))
       for (topicData <- offsetCommitRequest.data().topics().asScala) {
         for (partitionData <- topicData.partitions().asScala) {
           val topicPartition = new TopicPartition(topicData.name(), partitionData.partitionIndex())
-          if (!authorize(request.session, Read, Resource(Topic, topicData.name(), LITERAL)))
+          if (!authorizedTopics.contains(topicData.name()))
             unauthorizedTopicErrors += (topicPartition -> Errors.TOPIC_AUTHORIZATION_FAILED)
           else if (!metadataCache.contains(topicPartition))
             nonExistingTopicErrors += (topicPartition -> Errors.UNKNOWN_TOPIC_OR_PARTITION)
@@ -449,9 +451,6 @@ class KafkaApis(val requestChannel: RequestChannel,
     }
   }
 
-  private def authorize(session: RequestChannel.Session, operation: Operation, resource: Resource): Boolean =
-    authorizer.forall(_.authorize(session, operation, resource))
-
   /**
    * Handle a produce request
    */
@@ -461,14 +460,14 @@ class KafkaApis(val requestChannel: RequestChannel,
 
     if (produceRequest.hasTransactionalRecords) {
       val isAuthorizedTransactional = produceRequest.transactionalId != null &&
-        authorize(request.session, Write, Resource(TransactionalId, produceRequest.transactionalId, LITERAL))
+        authorize(request, WRITE, TRANSACTIONAL_ID, produceRequest.transactionalId)
       if (!isAuthorizedTransactional) {
         sendErrorResponseMaybeThrottle(request, Errors.TRANSACTIONAL_ID_AUTHORIZATION_FAILED.exception)
         return
       }
       // Note that authorization to a transactionalId implies ProducerId authorization
 
-    } else if (produceRequest.hasIdempotentRecords && !authorize(request.session, IdempotentWrite, Resource.ClusterResource)) {
+    } else if (produceRequest.hasIdempotentRecords && !authorize(request, IDEMPOTENT_WRITE, CLUSTER, CLUSTER_NAME)) {
       sendErrorResponseMaybeThrottle(request, Errors.CLUSTER_AUTHORIZATION_FAILED.exception)
       return
     }
@@ -477,9 +476,11 @@ class KafkaApis(val requestChannel: RequestChannel,
     val nonExistingTopicResponses = mutable.Map[TopicPartition, PartitionResponse]()
     val invalidRequestResponses = mutable.Map[TopicPartition, PartitionResponse]()
     val authorizedRequestInfo = mutable.Map[TopicPartition, MemoryRecords]()
+    val authorizedTopics = filterAuthorized(request, WRITE, TOPIC,
+      produceRequest.partitionRecordsOrFail.asScala.toSeq.map(_._1.topic))
 
     for ((topicPartition, memoryRecords) <- produceRequest.partitionRecordsOrFail.asScala) {
-      if (!authorize(request.session, Write, Resource(Topic, topicPartition.topic, LITERAL)))
+      if (!authorizedTopics.contains(topicPartition.topic))
         unauthorizedTopicResponses += topicPartition -> new PartitionResponse(Errors.TOPIC_AUTHORIZATION_FAILED)
       else if (!metadataCache.contains(topicPartition))
         nonExistingTopicResponses += topicPartition -> new PartitionResponse(Errors.UNKNOWN_TOPIC_OR_PARTITION)
@@ -612,7 +613,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     val interesting = mutable.ArrayBuffer[(TopicPartition, FetchRequest.PartitionData)]()
     if (fetchRequest.isFromFollower) {
       // The follower must have ClusterAction on ClusterResource in order to fetch partition data.
-      if (authorize(request.session, ClusterAction, Resource.ClusterResource)) {
+      if (authorize(request, CLUSTER_ACTION, CLUSTER, CLUSTER_NAME)) {
         fetchContext.foreachPartition { (topicPartition, data) =>
           if (!metadataCache.contains(topicPartition))
             erroneous += topicPartition -> errorResponse(Errors.UNKNOWN_TOPIC_OR_PARTITION)
@@ -626,8 +627,11 @@ class KafkaApis(val requestChannel: RequestChannel,
       }
     } else {
       // Regular Kafka consumers need READ permission on each partition they are fetching.
+      val fetchTopics = new mutable.ArrayBuffer[String]
+      fetchContext.foreachPartition { (topicPartition, _) => fetchTopics += topicPartition.topic }
+      val authorizedTopics = filterAuthorized(request, READ, TOPIC, fetchTopics)
       fetchContext.foreachPartition { (topicPartition, data) =>
-        if (!authorize(request.session, Read, Resource(Topic, topicPartition.topic, LITERAL)))
+        if (!authorizedTopics.contains(topicPartition.topic))
           erroneous += topicPartition -> errorResponse(Errors.TOPIC_AUTHORIZATION_FAILED)
         else if (!metadataCache.contains(topicPartition))
           erroneous += topicPartition -> errorResponse(Errors.UNKNOWN_TOPIC_OR_PARTITION)
@@ -858,8 +862,9 @@ class KafkaApis(val requestChannel: RequestChannel,
     val clientId = request.header.clientId
     val offsetRequest = request.body[ListOffsetRequest]
 
+    val authorizedTopics = filterAuthorized(request, DESCRIBE, TOPIC, offsetRequest.partitionTimestamps.asScala.toSeq.map(_._1.topic))
     val (authorizedRequestInfo, unauthorizedRequestInfo) = offsetRequest.partitionTimestamps.asScala.partition {
-      case (topicPartition, _) => authorize(request.session, Describe, Resource(Topic, topicPartition.topic, LITERAL))
+      case (topicPartition, _) => authorizedTopics.contains(topicPartition.topic)
     }
 
     val unauthorizedResponseStatus = unauthorizedRequestInfo.mapValues(_ =>
@@ -897,8 +902,9 @@ class KafkaApis(val requestChannel: RequestChannel,
     val clientId = request.header.clientId
     val offsetRequest = request.body[ListOffsetRequest]
 
+    val authorizedTopics = filterAuthorized(request, DESCRIBE, TOPIC, offsetRequest.partitionTimestamps.asScala.toSeq.map(_._1.topic))
     val (authorizedRequestInfo, unauthorizedRequestInfo) = offsetRequest.partitionTimestamps.asScala.partition {
-      case (topicPartition, _) => authorize(request.session, Describe, Resource(Topic, topicPartition.topic, LITERAL))
+      case (topicPartition, _) => authorizedTopics.contains(topicPartition.topic)
     }
 
     val unauthorizedResponseStatus = unauthorizedRequestInfo.mapValues(_ => {
@@ -1074,18 +1080,16 @@ class KafkaApis(val requestChannel: RequestChannel,
     else
       metadataRequest.topics.asScala.toSet
 
-    var (authorizedTopics, unauthorizedForDescribeTopics) =
-      topics.partition(topic => authorize(request.session, Describe, Resource(Topic, topic, LITERAL)))
-
+    val authorizedForDescribeTopics = filterAuthorized(request, DESCRIBE, TOPIC, topics.toSeq, logIfDenied = !metadataRequest.isAllTopics)
+    var (authorizedTopics, unauthorizedForDescribeTopics) = topics.partition(authorizedForDescribeTopics.contains)
     var unauthorizedForCreateTopics = Set[String]()
 
     if (authorizedTopics.nonEmpty) {
       val nonExistingTopics = metadataCache.getNonExistingTopics(authorizedTopics)
       if (metadataRequest.allowAutoTopicCreation && config.autoCreateTopicsEnable && nonExistingTopics.nonEmpty) {
-        if (!authorize(request.session, Create, Resource.ClusterResource)) {
-          unauthorizedForCreateTopics = nonExistingTopics.filter { topic =>
-            !authorize(request.session, Create, new Resource(Topic, topic, PatternType.LITERAL))
-          }
+        if (!authorize(request, CREATE, CLUSTER, CLUSTER_NAME, logIfDenied = false)) {
+          val authorizedForCreateTopics = filterAuthorized(request, CREATE, TOPIC, nonExistingTopics.toSeq)
+          unauthorizedForCreateTopics = nonExistingTopics -- authorizedForCreateTopics
           authorizedTopics --= unauthorizedForCreateTopics
         }
       }
@@ -1122,12 +1126,12 @@ class KafkaApis(val requestChannel: RequestChannel,
     if (request.header.apiVersion >= 8) {
       // get cluster authorized operations
       if (metadataRequest.data().includeClusterAuthorizedOperations() &&
-        authorize(request.session, Describe, Resource.ClusterResource))
-        clusterAuthorizedOperations = authorizedOperations(request.session, Resource.ClusterResource)
+        authorize(request, DESCRIBE, CLUSTER, CLUSTER_NAME))
+        clusterAuthorizedOperations = authorizedOperations(request, Resource.CLUSTER)
       // get topic authorized operations
       if (metadataRequest.data().includeTopicAuthorizedOperations())
         topicMetadata.foreach(topicData => {
-          topicData.authorizedOperations(authorizedOperations(request.session, Resource(Topic, topicData.topic(), LITERAL)))
+          topicData.authorizedOperations(authorizedOperations(request, new Resource(ResourceType.TOPIC, topicData.topic())))
         })
     }
 
@@ -1156,18 +1160,20 @@ class KafkaApis(val requestChannel: RequestChannel,
     val header = request.header
     val offsetFetchRequest = request.body[OffsetFetchRequest]
 
-    def authorizeTopicDescribe(partition: TopicPartition) =
-      authorize(request.session, Describe, Resource(Topic, partition.topic, LITERAL))
+    def partitionAuthorized[T](elements: List[T], topic: T => String): (Seq[T], Seq[T]) = {
+      val authorizedTopics = filterAuthorized(request, DESCRIBE, TOPIC, elements.map(topic))
+      elements.partition(element => authorizedTopics.contains(topic.apply(element)))
+    }
 
     def createResponse(requestThrottleMs: Int): AbstractResponse = {
       val offsetFetchResponse =
         // reject the request if not authorized to the group
-        if (!authorize(request.session, Describe, Resource(Group, offsetFetchRequest.groupId, LITERAL)))
+        if (!authorize(request, DESCRIBE, GROUP, offsetFetchRequest.groupId))
           offsetFetchRequest.getErrorResponse(requestThrottleMs, Errors.GROUP_AUTHORIZATION_FAILED)
         else {
           if (header.apiVersion == 0) {
-            val (authorizedPartitions, unauthorizedPartitions) = offsetFetchRequest.partitions.asScala
-              .partition(authorizeTopicDescribe)
+            val (authorizedPartitions, unauthorizedPartitions) =
+              partitionAuthorized[TopicPartition](offsetFetchRequest.partitions.asScala.toList, tp => tp.topic)
 
             // version 0 reads offsets from ZK
             val authorizedPartitionData = authorizedPartitions.map { topicPartition =>
@@ -1201,12 +1207,12 @@ class KafkaApis(val requestChannel: RequestChannel,
                 offsetFetchRequest.getErrorResponse(requestThrottleMs, error)
               else {
                 // clients are not allowed to see offsets for topics that are not authorized for Describe
-                val authorizedPartitionData = allPartitionData.filter { case (topicPartition, _) => authorizeTopicDescribe(topicPartition) }
-                new OffsetFetchResponse(requestThrottleMs, Errors.NONE, authorizedPartitionData.asJava)
+                val (authorizedPartitionData, _) = partitionAuthorized[(TopicPartition, OffsetFetchResponse.PartitionData)](allPartitionData.toList, e => e._1.topic)
+                new OffsetFetchResponse(requestThrottleMs, Errors.NONE, authorizedPartitionData.toMap.asJava)
               }
             } else {
-              val (authorizedPartitions, unauthorizedPartitions) = offsetFetchRequest.partitions.asScala
-                .partition(authorizeTopicDescribe)
+              val (authorizedPartitions, unauthorizedPartitions) =
+                partitionAuthorized[TopicPartition](offsetFetchRequest.partitions.asScala.toList, tp => tp.topic)
               val (error, authorizedPartitionData) = groupCoordinator.handleFetchOffsets(offsetFetchRequest.groupId,
                 Some(authorizedPartitions))
               if (error != Errors.NONE)
@@ -1230,10 +1236,10 @@ class KafkaApis(val requestChannel: RequestChannel,
     val findCoordinatorRequest = request.body[FindCoordinatorRequest]
 
     if (findCoordinatorRequest.data.keyType == CoordinatorType.GROUP.id &&
-        !authorize(request.session, Describe, Resource(Group, findCoordinatorRequest.data.key, LITERAL)))
+        !authorize(request, DESCRIBE, GROUP, findCoordinatorRequest.data.key))
       sendErrorResponseMaybeThrottle(request, Errors.GROUP_AUTHORIZATION_FAILED.exception)
     else if (findCoordinatorRequest.data.keyType == CoordinatorType.TRANSACTION.id &&
-        !authorize(request.session, Describe, Resource(TransactionalId, findCoordinatorRequest.data.key, LITERAL)))
+        !authorize(request, DESCRIBE, TRANSACTIONAL_ID, findCoordinatorRequest.data.key))
       sendErrorResponseMaybeThrottle(request, Errors.TRANSACTIONAL_ID_AUTHORIZATION_FAILED.exception)
     else {
       // get metadata (and create the topic if necessary)
@@ -1300,8 +1306,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     val describeGroupsResponseData = new DescribeGroupsResponseData()
 
     describeRequest.data().groups().asScala.foreach { groupId =>
-      val resource = Resource(Group, groupId, LITERAL)
-      if (!authorize(request.session, Describe, resource)) {
+      if (!authorize(request, DESCRIBE, GROUP, groupId)) {
         describeGroupsResponseData.groups().add(DescribeGroupsResponse.forError(groupId, Errors.GROUP_AUTHORIZATION_FAILED))
       } else {
         val (error, summary) = groupCoordinator.handleDescribeGroup(groupId)
@@ -1325,7 +1330,7 @@ class KafkaApis(val requestChannel: RequestChannel,
 
         if (request.header.apiVersion >= 3) {
           if (error == Errors.NONE && describeRequest.data().includeAuthorizedOperations()) {
-            describedGroup.setAuthorizedOperations(authorizedOperations(request.session, resource))
+            describedGroup.setAuthorizedOperations(authorizedOperations(request, new Resource(ResourceType.GROUP, groupId)))
           } else {
             describedGroup.setAuthorizedOperations(0)
           }
@@ -1338,20 +1343,9 @@ class KafkaApis(val requestChannel: RequestChannel,
     sendResponseCallback(describeGroupsResponseData)
   }
 
-
-  private def authorizedOperations(session: RequestChannel.Session, resource: Resource): Int = {
-    val authorizedOps = authorizer match {
-      case None => resource.resourceType.supportedOperations
-      case Some(_) => resource.resourceType.supportedOperations
-        .filter(operation => authorize(session, operation, resource))
-    }
-
-    Utils.to32BitField(authorizedOps.map(operation => operation.toJava.code().asInstanceOf[JByte]).asJava)
-  }
-
   def handleListGroupsRequest(request: RequestChannel.Request): Unit = {
     val (error, groups) = groupCoordinator.handleListGroups()
-    if (authorize(request.session, Describe, Resource.ClusterResource))
+    if (authorize(request, DESCRIBE, CLUSTER, CLUSTER_NAME))
       // With describe cluster access all groups are returned. We keep this alternative for backward compatibility.
       sendResponseMaybeThrottle(request, requestThrottleMs =>
         new ListGroupsResponse(new ListGroupsResponseData()
@@ -1363,7 +1357,7 @@ class KafkaApis(val requestChannel: RequestChannel,
             .setThrottleTimeMs(requestThrottleMs)
         ))
     else {
-      val filteredGroups = groups.filter(group => authorize(request.session, Describe, new Resource(Group, group.groupId, LITERAL)))
+      val filteredGroups = groups.filter(group => authorize(request, DESCRIBE, GROUP, group.groupId))
       sendResponseMaybeThrottle(request, requestThrottleMs =>
         new ListGroupsResponse(new ListGroupsResponseData()
           .setErrorCode(error.code())
@@ -1412,7 +1406,7 @@ class KafkaApis(val requestChannel: RequestChannel,
         JoinGroupResponse.UNKNOWN_MEMBER_ID,
         Errors.UNSUPPORTED_VERSION
       ))
-    } else if (!authorize(request.session, Read, Resource(Group, joinGroupRequest.data().groupId(), LITERAL))) {
+    } else if (!authorize(request, READ, GROUP, joinGroupRequest.data().groupId())) {
       sendResponseMaybeThrottle(request, requestThrottleMs =>
         new JoinGroupResponse(
           new JoinGroupResponseData()
@@ -1441,7 +1435,7 @@ class KafkaApis(val requestChannel: RequestChannel,
         groupInstanceId,
         requireKnownMemberId,
         request.header.clientId,
-        request.session.clientAddress.toString,
+        request.context.clientAddress.toString,
         joinGroupRequest.data.rebalanceTimeoutMs,
         joinGroupRequest.data.sessionTimeoutMs,
         joinGroupRequest.data.protocolType,
@@ -1468,7 +1462,7 @@ class KafkaApis(val requestChannel: RequestChannel,
       // until we are sure that all brokers support it. If static group being loaded by an older coordinator, it will discard
       // the group.instance.id field, so static members could accidentally become "dynamic", which leads to wrong states.
       sendResponseCallback(SyncGroupResult(Array[Byte](), Errors.UNSUPPORTED_VERSION))
-    } else if (!authorize(request.session, Read, Resource(Group, syncGroupRequest.data.groupId, LITERAL))) {
+    } else if (!authorize(request, READ, GROUP, syncGroupRequest.data.groupId)) {
       sendResponseCallback(SyncGroupResult(Array[Byte](), Errors.GROUP_AUTHORIZATION_FAILED))
     } else {
       val assignmentMap = immutable.Map.newBuilder[String, Array[Byte]]
@@ -1492,7 +1486,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     val groups = deleteGroupsRequest.data.groupsNames.asScala.toSet
 
     val (authorizedGroups, unauthorizedGroups) = groups.partition { group =>
-      authorize(request.session, Delete, Resource(Group, group, LITERAL))
+      authorize(request, DELETE, GROUP, group)
     }
 
     val groupDeletionResult = groupCoordinator.handleDeleteGroups(authorizedGroups) ++
@@ -1536,7 +1530,7 @@ class KafkaApis(val requestChannel: RequestChannel,
       // until we are sure that all brokers support it. If static group being loaded by an older coordinator, it will discard
       // the group.instance.id field, so static members could accidentally become "dynamic", which leads to wrong states.
       sendResponseCallback(Errors.UNSUPPORTED_VERSION)
-    } else if (!authorize(request.session, Read, Resource(Group, heartbeatRequest.data.groupId, LITERAL))) {
+    } else if (!authorize(request, READ, GROUP, heartbeatRequest.data.groupId)) {
       sendResponseMaybeThrottle(request, requestThrottleMs =>
         new HeartbeatResponse(
             new HeartbeatResponseData()
@@ -1558,7 +1552,7 @@ class KafkaApis(val requestChannel: RequestChannel,
 
     val members = leaveGroupRequest.members().asScala.toList
 
-    if (!authorize(request.session, Read, Resource(Group, leaveGroupRequest.data.groupId, LITERAL))) {
+    if (!authorize(request, READ, GROUP, leaveGroupRequest.data().groupId())) {
       sendResponseMaybeThrottle(request, requestThrottleMs => {
         new LeaveGroupResponse(new LeaveGroupResponseData()
           .setThrottleTimeMs(requestThrottleMs)
@@ -1647,13 +1641,15 @@ class KafkaApis(val requestChannel: RequestChannel,
       createTopicsRequest.data.topics.asScala.foreach { case topic =>
         results.add(new CreatableTopicResult().setName(topic.name()))
       }
-      val hasClusterAuthorization = authorize(request.session, Create, Resource.ClusterResource)
+      val hasClusterAuthorization = authorize(request, CREATE, CLUSTER, CLUSTER_NAME, logIfDenied = false)
+      val topics = createTopicsRequest.data.topics.asScala.map(_.name)
+      val authorizedTopics = if (hasClusterAuthorization) topics.toSet else filterAuthorized(request, CREATE, TOPIC, topics.toSeq)
+
       results.asScala.foreach(topic => {
         if (results.findAll(topic.name()).size() > 1) {
           topic.setErrorCode(Errors.INVALID_REQUEST.code())
           topic.setErrorMessage("Found multiple entries for this topic.")
-        } else if ((!hasClusterAuthorization) && (!authorize(request.session, Create,
-              new Resource(Topic, topic.name(), PatternType.LITERAL)))) {
+        } else if (!authorizedTopics.contains(topic.name)) {
           topic.setErrorCode(Errors.TOPIC_AUTHORIZATION_FAILED.code())
           topic.setErrorMessage("Authorization failed.")
         }
@@ -1701,8 +1697,9 @@ class KafkaApis(val requestChannel: RequestChannel,
       // Special handling to add duplicate topics to the response
       val dupes = createPartitionsRequest.duplicates.asScala
       val notDuped = createPartitionsRequest.newPartitions.asScala -- dupes
+      val authorizedTopics = filterAuthorized(request, ALTER, TOPIC, notDuped.toSeq.map(_._1))
       val (authorized, unauthorized) = notDuped.partition { case (topic, _) =>
-        authorize(request.session, Alter, Resource(Topic, topic, LITERAL))
+        authorizedTopics.contains(topic)
       }
 
       val (queuedForDeletion, valid) = authorized.partition { case (topic, _) =>
@@ -1754,8 +1751,9 @@ class KafkaApis(val requestChannel: RequestChannel,
         results.add(new DeletableTopicResult()
           .setName(topic))
       }
+      val authorizedTopics = filterAuthorized(request, DELETE, TOPIC, results.asScala.toSeq.map(_.name))
       results.asScala.foreach(topic => {
-         if (!authorize(request.session, Delete, Resource(Topic, topic.name, LITERAL)))
+         if (!authorizedTopics.contains(topic.name))
            topic.setErrorCode(Errors.TOPIC_AUTHORIZATION_FAILED.code)
          else if (!metadataCache.contains(topic.name))
            topic.setErrorCode(Errors.UNKNOWN_TOPIC_OR_PARTITION.code)
@@ -1791,8 +1789,10 @@ class KafkaApis(val requestChannel: RequestChannel,
     val nonExistingTopicResponses = mutable.Map[TopicPartition, DeleteRecordsResponse.PartitionResponse]()
     val authorizedForDeleteTopicOffsets = mutable.Map[TopicPartition, Long]()
 
+    val authorizedTopics = filterAuthorized(request, DELETE, TOPIC,
+      deleteRecordsRequest.partitionOffsets.asScala.toSeq.map(_._1.topic))
     for ((topicPartition, offset) <- deleteRecordsRequest.partitionOffsets.asScala) {
-      if (!authorize(request.session, Delete, Resource(Topic, topicPartition.topic, LITERAL)))
+      if (!authorizedTopics.contains(topicPartition.topic))
         unauthorizedTopicResponses += topicPartition -> new DeleteRecordsResponse.PartitionResponse(
           DeleteRecordsResponse.INVALID_LOW_WATERMARK, Errors.TOPIC_AUTHORIZATION_FAILED)
       else if (!metadataCache.contains(topicPartition))
@@ -1835,11 +1835,11 @@ class KafkaApis(val requestChannel: RequestChannel,
     val transactionalId = initProducerIdRequest.data.transactionalId
 
     if (transactionalId != null) {
-      if (!authorize(request.session, Write, Resource(TransactionalId, transactionalId, LITERAL))) {
+      if (!authorize(request, WRITE, TRANSACTIONAL_ID, transactionalId)) {
         sendErrorResponseMaybeThrottle(request, Errors.TRANSACTIONAL_ID_AUTHORIZATION_FAILED.exception)
         return
       }
-    } else if (!authorize(request.session, IdempotentWrite, Resource.ClusterResource)) {
+    } else if (!authorize(request, IDEMPOTENT_WRITE, CLUSTER, CLUSTER_NAME)) {
       sendErrorResponseMaybeThrottle(request, Errors.CLUSTER_AUTHORIZATION_FAILED.exception)
       return
     }
@@ -1865,7 +1865,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     val endTxnRequest = request.body[EndTxnRequest]
     val transactionalId = endTxnRequest.transactionalId
 
-    if (authorize(request.session, Write, Resource(TransactionalId, transactionalId, LITERAL))) {
+    if (authorize(request, WRITE, TRANSACTIONAL_ID, transactionalId)) {
       def sendResponseCallback(error: Errors): Unit = {
         def createResponse(requestThrottleMs: Int): AbstractResponse = {
           val responseBody = new EndTxnResponse(requestThrottleMs, error)
@@ -1887,7 +1887,7 @@ class KafkaApis(val requestChannel: RequestChannel,
 
   def handleWriteTxnMarkersRequest(request: RequestChannel.Request): Unit = {
     ensureInterBrokerVersion(KAFKA_0_11_0_IV0)
-    authorizeClusterAction(request)
+    authorizeClusterOperation(request, CLUSTER_ACTION)
     val writeTxnMarkersRequest = request.body[WriteTxnMarkersRequest]
     val errors = new ConcurrentHashMap[java.lang.Long, util.Map[TopicPartition, Errors]]()
     val markers = writeTxnMarkersRequest.markers
@@ -2000,7 +2000,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     val addPartitionsToTxnRequest = request.body[AddPartitionsToTxnRequest]
     val transactionalId = addPartitionsToTxnRequest.transactionalId
     val partitionsToAdd = addPartitionsToTxnRequest.partitions.asScala
-    if (!authorize(request.session, Write, Resource(TransactionalId, transactionalId, LITERAL)))
+    if (!authorize(request, WRITE, TRANSACTIONAL_ID, transactionalId))
       sendResponseMaybeThrottle(request, requestThrottleMs =>
         addPartitionsToTxnRequest.getErrorResponse(requestThrottleMs, Errors.TRANSACTIONAL_ID_AUTHORIZATION_FAILED.exception))
     else {
@@ -2008,9 +2008,10 @@ class KafkaApis(val requestChannel: RequestChannel,
       val nonExistingTopicErrors = mutable.Map[TopicPartition, Errors]()
       val authorizedPartitions = mutable.Set[TopicPartition]()
 
+      val authorizedTopics = filterAuthorized(request, WRITE, TOPIC,
+        partitionsToAdd.toSeq.map(_.topic).filterNot(org.apache.kafka.common.internals.Topic.isInternal))
       for (topicPartition <- partitionsToAdd) {
-        if (org.apache.kafka.common.internals.Topic.isInternal(topicPartition.topic) ||
-            !authorize(request.session, Write, Resource(Topic, topicPartition.topic, LITERAL)))
+        if (!authorizedTopics.contains(topicPartition.topic))
           unauthorizedTopicErrors += topicPartition -> Errors.TOPIC_AUTHORIZATION_FAILED
         else if (!metadataCache.contains(topicPartition))
           nonExistingTopicErrors += topicPartition -> Errors.UNKNOWN_TOPIC_OR_PARTITION
@@ -2054,10 +2055,10 @@ class KafkaApis(val requestChannel: RequestChannel,
     val groupId = addOffsetsToTxnRequest.consumerGroupId
     val offsetTopicPartition = new TopicPartition(GROUP_METADATA_TOPIC_NAME, groupCoordinator.partitionFor(groupId))
 
-    if (!authorize(request.session, Write, Resource(TransactionalId, transactionalId, LITERAL)))
+    if (!authorize(request, WRITE, TRANSACTIONAL_ID, transactionalId))
       sendResponseMaybeThrottle(request, requestThrottleMs =>
         new AddOffsetsToTxnResponse(requestThrottleMs, Errors.TRANSACTIONAL_ID_AUTHORIZATION_FAILED))
-    else if (!authorize(request.session, Read, Resource(Group, groupId, LITERAL)))
+    else if (!authorize(request, READ, GROUP, groupId))
       sendResponseMaybeThrottle(request, requestThrottleMs =>
         new AddOffsetsToTxnResponse(requestThrottleMs, Errors.GROUP_AUTHORIZATION_FAILED))
     else {
@@ -2086,17 +2087,18 @@ class KafkaApis(val requestChannel: RequestChannel,
 
     // authorize for the transactionalId and the consumer group. Note that we skip producerId authorization
     // since it is implied by transactionalId authorization
-    if (!authorize(request.session, Write, Resource(TransactionalId, txnOffsetCommitRequest.transactionalId, LITERAL)))
+    if (!authorize(request, WRITE, TRANSACTIONAL_ID, txnOffsetCommitRequest.transactionalId))
       sendErrorResponseMaybeThrottle(request, Errors.TRANSACTIONAL_ID_AUTHORIZATION_FAILED.exception)
-    else if (!authorize(request.session, Read, Resource(Group, txnOffsetCommitRequest.consumerGroupId, LITERAL)))
+    else if (!authorize(request, READ, GROUP, txnOffsetCommitRequest.consumerGroupId))
       sendErrorResponseMaybeThrottle(request, Errors.GROUP_AUTHORIZATION_FAILED.exception)
     else {
       val unauthorizedTopicErrors = mutable.Map[TopicPartition, Errors]()
       val nonExistingTopicErrors = mutable.Map[TopicPartition, Errors]()
       val authorizedTopicCommittedOffsets = mutable.Map[TopicPartition, TxnOffsetCommitRequest.CommittedOffset]()
+      val authorizedTopics = filterAuthorized(request, READ, TOPIC, txnOffsetCommitRequest.offsets.keySet.asScala.toSeq.map(_.topic))
 
       for ((topicPartition, commitedOffset) <- txnOffsetCommitRequest.offsets.asScala) {
-        if (!authorize(request.session, Read, Resource(Topic, topicPartition.topic, LITERAL)))
+        if (!authorizedTopics.contains(topicPartition.topic))
           unauthorizedTopicErrors += topicPartition -> Errors.TOPIC_AUTHORIZATION_FAILED
         else if (!metadataCache.contains(topicPartition))
           nonExistingTopicErrors += topicPartition -> Errors.UNKNOWN_TOPIC_OR_PARTITION
@@ -2146,7 +2148,7 @@ class KafkaApis(val requestChannel: RequestChannel,
   }
 
   def handleDescribeAcls(request: RequestChannel.Request): Unit = {
-    authorizeClusterDescribe(request)
+    authorizeClusterOperation(request, DESCRIBE)
     val describeAclsRequest = request.body[DescribeAclsRequest]
     authorizer match {
       case None =>
@@ -2154,56 +2156,60 @@ class KafkaApis(val requestChannel: RequestChannel,
           new DescribeAclsResponse(requestThrottleMs,
             new ApiError(Errors.SECURITY_DISABLED, "No Authorizer is configured on the broker"), util.Collections.emptySet()))
       case Some(auth) =>
-        val filter = describeAclsRequest.filter()
-        val returnedAcls = auth.getAcls.toSeq.flatMap { case (resource, acls) =>
-          acls.flatMap { acl =>
-            val fixture = new AclBinding(new ResourcePattern(resource.resourceType.toJava, resource.name, resource.patternType),
-                new AccessControlEntry(acl.principal.toString, acl.host.toString, acl.operation.toJava, acl.permissionType.toJava))
-            Some(fixture).filter(filter.matches)
-          }
-        }
+        val filter = describeAclsRequest.filter
+        val returnedAcls = new util.HashSet[AclBinding]()
+        auth.acls(filter).asScala.foreach(returnedAcls.add)
         sendResponseMaybeThrottle(request, requestThrottleMs =>
-          new DescribeAclsResponse(requestThrottleMs, ApiError.NONE, returnedAcls.asJava))
+          new DescribeAclsResponse(requestThrottleMs, ApiError.NONE, returnedAcls))
     }
   }
 
   def handleCreateAcls(request: RequestChannel.Request): Unit = {
-    authorizeClusterAlter(request)
+    authorizeClusterOperation(request, ALTER)
     val createAclsRequest = request.body[CreateAclsRequest]
+
     authorizer match {
       case None =>
         sendResponseMaybeThrottle(request, requestThrottleMs =>
           createAclsRequest.getErrorResponse(requestThrottleMs,
             new SecurityDisabledException("No Authorizer is configured on the broker.")))
       case Some(auth) =>
-        val aclCreationResults = createAclsRequest.aclCreations.asScala.map { aclCreation =>
-          SecurityUtils.convertToResourceAndAcl(aclCreation.acl.toFilter) match {
-            case Left(apiError) => new AclCreationResponse(apiError)
-            case Right((resource, acl)) => try {
-                if (resource.resourceType.equals(Cluster) && !SecurityUtils.isClusterResource(resource.name))
-                  throw new InvalidRequestException("The only valid name for the CLUSTER resource is " +
-                      Resource.ClusterResourceName)
-                if (resource.name.isEmpty)
-                  throw new InvalidRequestException("Invalid empty resource name")
-                auth.addAcls(immutable.Set(acl), resource)
-
-                debug(s"Added acl $acl to $resource")
-
-                new AclCreationResponse(ApiError.NONE)
-              } catch {
-                case throwable: Throwable =>
-                  debug(s"Failed to add acl $acl to $resource", throwable)
-                  new AclCreationResponse(ApiError.fromThrowable(throwable))
-              }
+
+        val errorResults = mutable.Map[AclBinding, AclCreateResult]()
+        val aclBindings = createAclsRequest.aclCreations.asScala.map(_.acl)
+        val validBindings = aclBindings
+          .filter { acl =>
+            val resource = acl.pattern
+            val throwable = if (resource.resourceType == ResourceType.CLUSTER && !AuthorizerUtils.isClusterResource(resource.name))
+                new InvalidRequestException("The only valid name for the CLUSTER resource is " + CLUSTER_NAME)
+            else if (resource.name.isEmpty)
+              new InvalidRequestException("Invalid empty resource name")
+            else
+              null
+            if (throwable != null) {
+              debug(s"Failed to add acl $acl to $resource", throwable)
+              errorResults(acl) = new AclCreateResult(throwable)
+              false
+            } else
+              true
           }
+        val createResults = auth.createAcls(request.context, validBindings.asJava)
+
+        val aclCreationResults = aclBindings.map { acl =>
+          val result = errorResults.getOrElse(acl, createResults.get(validBindings.indexOf(acl)))
+          if (result.failed)
+            new AclCreationResponse(ApiError.fromThrowable(result.exception))
+          else
+            new AclCreationResponse(ApiError.NONE)
         }
+
         sendResponseMaybeThrottle(request, requestThrottleMs =>
           new CreateAclsResponse(requestThrottleMs, aclCreationResults.asJava))
     }
   }
 
   def handleDeleteAcls(request: RequestChannel.Request): Unit = {
-    authorizeClusterAlter(request)
+    authorizeClusterOperation(request, ALTER)
     val deleteAclsRequest = request.body[DeleteAclsRequest]
     authorizer match {
       case None =>
@@ -2211,51 +2217,16 @@ class KafkaApis(val requestChannel: RequestChannel,
           deleteAclsRequest.getErrorResponse(requestThrottleMs,
             new SecurityDisabledException("No Authorizer is configured on the broker.")))
       case Some(auth) =>
-        val filters = deleteAclsRequest.filters.asScala
-        val filterResponseMap = mutable.Map[Int, AclFilterResponse]()
-        val toDelete = mutable.Map[Int, ArrayBuffer[(Resource, Acl)]]()
-
-        if (filters.forall(_.matchesAtMostOne)) {
-          // Delete based on a list of ACL fixtures.
-          for ((filter, i) <- filters.zipWithIndex) {
-            SecurityUtils.convertToResourceAndAcl(filter) match {
-              case Left(apiError) => filterResponseMap.put(i, new AclFilterResponse(apiError, Seq.empty.asJava))
-              case Right(binding) => toDelete.put(i, ArrayBuffer(binding))
-            }
-          }
-        } else {
-          // Delete based on filters that may match more than one ACL.
-          val aclMap = auth.getAcls()
-          val filtersWithIndex = filters.zipWithIndex
-          for ((resource, acls) <- aclMap; acl <- acls) {
-            val binding = new AclBinding(
-              new ResourcePattern(resource.resourceType.toJava, resource.name, resource.patternType),
-              new AccessControlEntry(acl.principal.toString, acl.host.toString, acl.operation.toJava,
-                acl.permissionType.toJava))
-
-            for ((filter, i) <- filtersWithIndex if filter.matches(binding))
-              toDelete.getOrElseUpdate(i, ArrayBuffer.empty) += ((resource, acl))
-          }
-        }
-
-        for ((i, acls) <- toDelete) {
-          val deletionResults = acls.flatMap { case (resource, acl) =>
-            val aclBinding = SecurityUtils.convertToAclBinding(resource, acl)
-            try {
-              if (auth.removeAcls(immutable.Set(acl), resource))
-                Some(new AclDeletionResult(aclBinding))
-              else None
-            } catch {
-              case throwable: Throwable =>
-                Some(new AclDeletionResult(ApiError.fromThrowable(throwable), aclBinding))
-            }
-          }.asJava
 
-          filterResponseMap.put(i, new AclFilterResponse(deletionResults))
+        val results = auth.deleteAcls(request.context, deleteAclsRequest.filters)
+        def toErrorCode(exception: ApiException): ApiError = {
+          if (exception != null) ApiError.fromThrowable(exception) else ApiError.NONE
         }
-
-        val filterResponses = filters.indices.map { i =>
-          filterResponseMap.getOrElse(i, new AclFilterResponse(Seq.empty.asJava))
+        val filterResponses = results.asScala.map { result =>
+          val deletions = result.aclBindingDeleteResults().asScala.toList.map { deletionResult =>
+            new AclDeletionResult(toErrorCode(deletionResult.exception), deletionResult.aclBinding)
+          }.asJava
+          new AclFilterResponse(toErrorCode(result.exception), deletions)
         }.asJava
         sendResponseMaybeThrottle(request, requestThrottleMs => new DeleteAclsResponse(requestThrottleMs, filterResponses))
     }
@@ -2268,11 +2239,12 @@ class KafkaApis(val requestChannel: RequestChannel,
     // The OffsetsForLeaderEpoch API was initially only used for inter-broker communication and required
     // cluster permission. With KIP-320, the consumer now also uses this API to check for log truncation
     // following a leader change, so we also allow topic describe permission.
-    val (authorizedPartitions, unauthorizedPartitions) = if (isAuthorizedClusterAction(request)) {
+    val (authorizedPartitions, unauthorizedPartitions) = if (authorize(request, CLUSTER_ACTION, CLUSTER, CLUSTER_NAME, logIfDenied = false)) {
       (requestInfo, Map.empty[TopicPartition, OffsetsForLeaderEpochRequest.PartitionData])
     } else {
+      val authorizedTopics = filterAuthorized(request, DESCRIBE, TOPIC, requestInfo.keySet.toSeq.map(_.topic))
       requestInfo.partition {
-        case (tp, _) => authorize(request.session, Describe, Resource(Topic, tp.topic, LITERAL))
+        case (tp, _) => authorizedTopics.contains(tp.topic)
       }
     }
 
@@ -2293,22 +2265,22 @@ class KafkaApis(val requestChannel: RequestChannel,
         case ConfigResource.Type.BROKER_LOGGER =>
           throw new InvalidRequestException(s"AlterConfigs is deprecated and does not support the resource type ${ConfigResource.Type.BROKER_LOGGER}")
         case ConfigResource.Type.BROKER =>
-          authorize(request.session, AlterConfigs, Resource.ClusterResource)
+          authorize(request, ALTER_CONFIGS, CLUSTER, CLUSTER_NAME)
         case ConfigResource.Type.TOPIC =>
-          authorize(request.session, AlterConfigs, Resource(Topic, resource.name, LITERAL))
+          authorize(request, ALTER_CONFIGS, TOPIC, resource.name)
         case rt => throw new InvalidRequestException(s"Unexpected resource type $rt")
       }
     }
     val authorizedResult = adminManager.alterConfigs(authorizedResources, alterConfigsRequest.validateOnly)
     val unauthorizedResult = unauthorizedResources.keys.map { resource =>
-      resource -> configsAuthorizationApiError(request.session, resource)
+      resource -> configsAuthorizationApiError(resource)
     }
     sendResponseMaybeThrottle(request, requestThrottleMs =>
       new AlterConfigsResponse(requestThrottleMs, (authorizedResult ++ unauthorizedResult).asJava))
   }
 
   def handleAlterPartitionReassignmentsRequest(request: RequestChannel.Request): Unit = {
-    authorizeClusterAlter(request)
+    authorizeClusterOperation(request, ALTER)
     val alterPartitionReassignmentsRequest = request.body[AlterPartitionReassignmentsRequest]
 
     sendResponseMaybeThrottle(request, requestThrottleMs =>
@@ -2321,7 +2293,7 @@ class KafkaApis(val requestChannel: RequestChannel,
   }
 
   def handleListPartitionReassignmentsRequest(request: RequestChannel.Request): Unit = {
-    authorizeClusterDescribe(request)
+    authorizeClusterOperation(request, DESCRIBE)
     val listPartitionReassignmentsRequest = request.body[ListPartitionReassignmentsRequest]
 
     sendResponseMaybeThrottle(request, requestThrottleMs =>
@@ -2333,7 +2305,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     )
   }
 
-  private def configsAuthorizationApiError(session: RequestChannel.Session, resource: ConfigResource): ApiError = {
+  private def configsAuthorizationApiError(resource: ConfigResource): ApiError = {
     val error = resource.`type` match {
       case ConfigResource.Type.BROKER | ConfigResource.Type.BROKER_LOGGER => Errors.CLUSTER_AUTHORIZATION_FAILED
       case ConfigResource.Type.TOPIC => Errors.TOPIC_AUTHORIZATION_FAILED
@@ -2354,16 +2326,16 @@ class KafkaApis(val requestChannel: RequestChannel,
     val (authorizedResources, unauthorizedResources) = configs.partition { case (resource, _) =>
       resource.`type` match {
         case ConfigResource.Type.BROKER | ConfigResource.Type.BROKER_LOGGER =>
-          authorize(request.session, AlterConfigs, Resource.ClusterResource)
+          authorize(request, ALTER_CONFIGS, CLUSTER, CLUSTER_NAME)
         case ConfigResource.Type.TOPIC =>
-          authorize(request.session, AlterConfigs, Resource(Topic, resource.name, LITERAL))
+          authorize(request, ALTER_CONFIGS, TOPIC, resource.name)
         case rt => throw new InvalidRequestException(s"Unexpected resource type $rt")
       }
     }
 
     val authorizedResult = adminManager.incrementalAlterConfigs(authorizedResources, alterConfigsRequest.data().validateOnly())
     val unauthorizedResult = unauthorizedResources.keys.map { resource =>
-      resource -> configsAuthorizationApiError(request.session, resource)
+      resource -> configsAuthorizationApiError(resource)
     }
     sendResponseMaybeThrottle(request, requestThrottleMs =>
       new IncrementalAlterConfigsResponse(IncrementalAlterConfigsResponse.toResponseData(requestThrottleMs,
@@ -2374,9 +2346,10 @@ class KafkaApis(val requestChannel: RequestChannel,
     val describeConfigsRequest = request.body[DescribeConfigsRequest]
     val (authorizedResources, unauthorizedResources) = describeConfigsRequest.resources.asScala.partition { resource =>
       resource.`type` match {
-        case ConfigResource.Type.BROKER | ConfigResource.Type.BROKER_LOGGER => authorize(request.session, DescribeConfigs, Resource.ClusterResource)
+        case ConfigResource.Type.BROKER | ConfigResource.Type.BROKER_LOGGER =>
+          authorize(request, DESCRIBE_CONFIGS, CLUSTER, CLUSTER_NAME)
         case ConfigResource.Type.TOPIC =>
-          authorize(request.session, DescribeConfigs, Resource(Topic, resource.name, LITERAL))
+          authorize(request, DESCRIBE_CONFIGS, TOPIC, resource.name)
         case rt => throw new InvalidRequestException(s"Unexpected resource type $rt for resource ${resource.name}")
       }
     }
@@ -2384,7 +2357,7 @@ class KafkaApis(val requestChannel: RequestChannel,
       resource -> Option(describeConfigsRequest.configNames(resource)).map(_.asScala.toSet)
     }.toMap, describeConfigsRequest.includeSynonyms)
     val unauthorizedConfigs = unauthorizedResources.map { resource =>
-      val error = configsAuthorizationApiError(request.session, resource)
+      val error = configsAuthorizationApiError(resource)
       resource -> new DescribeConfigsResponse.Config(error, util.Collections.emptyList[DescribeConfigsResponse.ConfigEntry])
     }
 
@@ -2395,7 +2368,7 @@ class KafkaApis(val requestChannel: RequestChannel,
   def handleAlterReplicaLogDirsRequest(request: RequestChannel.Request): Unit = {
     val alterReplicaDirsRequest = request.body[AlterReplicaLogDirsRequest]
     val responseMap = {
-      if (authorize(request.session, Alter, Resource.ClusterResource))
+      if (authorize(request, ALTER, CLUSTER, CLUSTER_NAME))
         replicaManager.alterReplicaLogDirs(alterReplicaDirsRequest.partitionDirs.asScala)
       else
         alterReplicaDirsRequest.partitionDirs.asScala.keys.map((_, Errors.CLUSTER_AUTHORIZATION_FAILED)).toMap
@@ -2406,7 +2379,7 @@ class KafkaApis(val requestChannel: RequestChannel,
   def handleDescribeLogDirsRequest(request: RequestChannel.Request): Unit = {
     val describeLogDirsDirRequest = request.body[DescribeLogDirsRequest]
     val logDirInfos = {
-      if (authorize(request.session, Describe, Resource.ClusterResource)) {
+      if (authorize(request, DESCRIBE, CLUSTER, CLUSTER_NAME)) {
         val partitions =
           if (describeLogDirsDirRequest.isAllTopicPartitions)
             replicaManager.logManager.allLogs.map(_.topicPartition).toSet
@@ -2429,24 +2402,24 @@ class KafkaApis(val requestChannel: RequestChannel,
       trace("Sending create token response for correlation id %d to client %s."
         .format(request.header.correlationId, request.header.clientId))
       sendResponseMaybeThrottle(request, requestThrottleMs =>
-        CreateDelegationTokenResponse.prepareResponse(requestThrottleMs, createResult.error, request.session.principal, createResult.issueTimestamp,
+        CreateDelegationTokenResponse.prepareResponse(requestThrottleMs, createResult.error, request.context.principal, createResult.issueTimestamp,
           createResult.expiryTimestamp, createResult.maxTimestamp, createResult.tokenId, ByteBuffer.wrap(createResult.hmac)))
     }
 
     if (!allowTokenRequests(request))
       sendResponseMaybeThrottle(request, requestThrottleMs =>
-        CreateDelegationTokenResponse.prepareResponse(requestThrottleMs, Errors.DELEGATION_TOKEN_REQUEST_NOT_ALLOWED, request.session.principal))
+        CreateDelegationTokenResponse.prepareResponse(requestThrottleMs, Errors.DELEGATION_TOKEN_REQUEST_NOT_ALLOWED, request.context.principal))
     else {
       val renewerList = createTokenRequest.data.renewers.asScala.toList.map(entry =>
         new KafkaPrincipal(entry.principalType, entry.principalName))
 
       if (renewerList.exists(principal => principal.getPrincipalType != KafkaPrincipal.USER_TYPE)) {
         sendResponseMaybeThrottle(request, requestThrottleMs =>
-          CreateDelegationTokenResponse.prepareResponse(requestThrottleMs, Errors.INVALID_PRINCIPAL_TYPE, request.session.principal))
+          CreateDelegationTokenResponse.prepareResponse(requestThrottleMs, Errors.INVALID_PRINCIPAL_TYPE, request.context.principal))
       }
       else {
         tokenManager.createToken(
-          request.session.principal,
+          request.context.principal,
           renewerList,
           createTokenRequest.data.maxLifetimeMs,
           sendResponseCallback
@@ -2474,7 +2447,7 @@ class KafkaApis(val requestChannel: RequestChannel,
       sendResponseCallback(Errors.DELEGATION_TOKEN_REQUEST_NOT_ALLOWED, DelegationTokenManager.ErrorTimestamp)
     else {
       tokenManager.renewToken(
-        request.session.principal,
+        request.context.principal,
         ByteBuffer.wrap(renewTokenRequest.data.hmac),
         renewTokenRequest.data.renewPeriodMs,
         sendResponseCallback
@@ -2501,7 +2474,7 @@ class KafkaApis(val requestChannel: RequestChannel,
       sendResponseCallback(Errors.DELEGATION_TOKEN_REQUEST_NOT_ALLOWED, DelegationTokenManager.ErrorTimestamp)
     else {
       tokenManager.expireToken(
-        request.session.principal,
+        request.context.principal,
         expireTokenRequest.hmac(),
         expireTokenRequest.expiryTimePeriod(),
         sendResponseCallback
@@ -2525,7 +2498,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     else if (!config.tokenAuthEnabled)
       sendResponseCallback(Errors.DELEGATION_TOKEN_AUTH_DISABLED, List.empty)
     else {
-      val requestPrincipal = request.session.principal
+      val requestPrincipal = request.context.principal
 
       if (describeTokenRequest.ownersListEmpty()) {
         sendResponseCallback(Errors.NONE, List())
@@ -2535,7 +2508,7 @@ class KafkaApis(val requestChannel: RequestChannel,
           None
         else
           Some(describeTokenRequest.data.owners.asScala.map(p => new KafkaPrincipal(p.principalType(), p.principalName())).toList)
-        def authorizeToken(tokenId: String) = authorize(request.session, Describe, Resource(kafka.security.auth.DelegationToken, tokenId, LITERAL))
+        def authorizeToken(tokenId: String) = authorize(request, DESCRIBE, DELEGATION_TOKEN, tokenId)
         def eligible(token: TokenInformation) = DelegationTokenManager.filterToken(requestPrincipal, owners, token, authorizeToken)
         val tokens =  tokenManager.getTokens(eligible)
         sendResponseCallback(Errors.NONE, tokens)
@@ -2545,10 +2518,10 @@ class KafkaApis(val requestChannel: RequestChannel,
 
   def allowTokenRequests(request: RequestChannel.Request): Boolean = {
     val protocol = request.context.securityProtocol
-    if (request.session.principal.tokenAuthenticated ||
+    if (request.context.principal.tokenAuthenticated ||
       protocol == SecurityProtocol.PLAINTEXT ||
       // disallow requests from 1-way SSL
-      (protocol == SecurityProtocol.SSL && request.session.principal == KafkaPrincipal.ANONYMOUS))
+      (protocol == SecurityProtocol.SSL && request.context.principal == KafkaPrincipal.ANONYMOUS))
       false
     else
       true
@@ -2602,7 +2575,7 @@ class KafkaApis(val requestChannel: RequestChannel,
       })
     }
 
-    if (!authorize(request.session, Alter, Resource.ClusterResource)) {
+    if (!authorize(request, ALTER, CLUSTER, CLUSTER_NAME)) {
       val error = new ApiError(Errors.CLUSTER_AUTHORIZATION_FAILED, null)
       val partitionErrors: Map[TopicPartition, ApiError] =
         electionRequest.topicPartitions.iterator.map(partition => partition -> error).toMap
@@ -2625,23 +2598,61 @@ class KafkaApis(val requestChannel: RequestChannel,
     }
   }
 
-  def authorizeClusterAction(request: RequestChannel.Request): Unit = {
-    if (!isAuthorizedClusterAction(request))
-      throw new ClusterAuthorizationException(s"Request $request is not authorized.")
+  private def authorize(request: RequestChannel.Request,
+                        operation: AclOperation,
+                        resourceType: ResourceType,
+                        resourceName: String,
+                        logIfAllowed: Boolean = true,
+                        logIfDenied: Boolean = true,
+                        refCount: Int = 1): Boolean = {
+    authorizer.forall { authZ =>
+      val resource = new ResourcePattern(resourceType, resourceName, PatternType.LITERAL)
+      val actions = Collections.singletonList(new Action(operation, resource, refCount, logIfAllowed, logIfDenied))
+      authZ.authorize(request.context, actions).asScala.head == AuthorizationResult.ALLOWED
+    }
   }
 
-  private def isAuthorizedClusterAction(request: RequestChannel.Request): Boolean = {
-    authorize(request.session, ClusterAction, Resource.ClusterResource)
+  private def filterAuthorized(request: RequestChannel.Request,
+                               operation: AclOperation,
+                               resourceType: ResourceType,
+                               resourceNames: Seq[String],
+                               logIfAllowed: Boolean = true,
+                               logIfDenied: Boolean = true): Set[String] = {
+    authorizer match {
+      case Some(authZ) =>
+        val resources = resourceNames.groupBy(identity).mapValues(_.size).toList
+        val actions = resources.map { case (resourceName, count) =>
+          val resource = new ResourcePattern(resourceType, resourceName, PatternType.LITERAL)
+          new Action(operation, resource, count, logIfAllowed, logIfDenied)
+        }
+        authZ.authorize(request.context, actions.asJava).asScala
+          .zip(resources.map(_._1)) // zip with resource name
+          .filter(_._1 == AuthorizationResult.ALLOWED) // filter authorized resources
+          .map(_._2).toSet
+      case None =>
+        resourceNames.toSet
+    }
   }
 
-  def authorizeClusterAlter(request: RequestChannel.Request): Unit = {
-    if (!authorize(request.session, Alter, Resource.ClusterResource))
+  private def authorizeClusterOperation(request: RequestChannel.Request, operation: AclOperation): Unit = {
+    if (!authorize(request, operation, CLUSTER, CLUSTER_NAME))
       throw new ClusterAuthorizationException(s"Request $request is not authorized.")
   }
 
-  def authorizeClusterDescribe(request: RequestChannel.Request): Unit = {
-    if (!authorize(request.session, Describe, Resource.ClusterResource))
-      throw new ClusterAuthorizationException(s"Request $request is not authorized.")
+  private def authorizedOperations(request: RequestChannel.Request, resource: Resource): Int = {
+    val supportedOps = AuthorizerUtils.supportedOperations(resource.resourceType).toList
+    val authorizedOps = authorizer match {
+      case Some(authZ) =>
+        val resourcePattern = new ResourcePattern(resource.resourceType, resource.name, PatternType.LITERAL)
+        val actions = supportedOps.map { op => new Action(op, resourcePattern, 1, false, false) }
+        authZ.authorize(request.context, actions.asJava).asScala
+          .zip(supportedOps)
+          .filter(_._1 == AuthorizationResult.ALLOWED)
+          .map(_._2).toSet
+      case None =>
+        supportedOps.toSet
+    }
+    Utils.to32BitField(authorizedOps.map(operation => operation.code().asInstanceOf[JByte]).asJava)
   }
 
   private def updateRecordConversionStats(request: RequestChannel.Request,
diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index 2069e70cec6b5..97f894cb5a600 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -25,6 +25,7 @@ import kafka.cluster.EndPoint
 import kafka.coordinator.group.OffsetConfig
 import kafka.coordinator.transaction.{TransactionLog, TransactionStateManager}
 import kafka.message.{BrokerCompressionCodec, CompressionCodec, ZStdCompressionCodec}
+import kafka.security.authorizer.AuthorizerWrapper
 import kafka.utils.CoreUtils
 import kafka.utils.Implicits._
 import org.apache.kafka.clients.CommonClientConfigs
@@ -38,6 +39,7 @@ import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.record.{LegacyRecord, Records, TimestampType}
 import org.apache.kafka.common.security.auth.SecurityProtocol
 import org.apache.kafka.common.utils.Utils
+import org.apache.kafka.server.authorizer.Authorizer
 
 import scala.collection.JavaConverters._
 import scala.collection.{Map, Seq}
@@ -515,7 +517,9 @@ object KafkaConfig {
   val QueuedMaxRequestBytesDoc = "The number of queued bytes allowed before no more requests are read"
   val RequestTimeoutMsDoc = CommonClientConfigs.REQUEST_TIMEOUT_MS_DOC
   /************* Authorizer Configuration ***********/
-  val AuthorizerClassNameDoc = "The authorizer class that should be used for authorization"
+  val AuthorizerClassNameDoc = s"The fully qualified name of a class that implements s${classOf[Authorizer].getName}" +
+  " interface, which is used by the broker for authorization. This config also supports authorizers that implement the deprecated" +
+  " kafka.security.auth.Authorizer trait which was previously used for authorization."
   /** ********* Socket Server Configuration ***********/
   val PortDoc = "DEPRECATED: only used when listeners is not set. " +
   "Use listeners instead. \n" +
@@ -1173,7 +1177,19 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO
   }
 
   /************* Authorizer Configuration ***********/
-  val authorizerClassName: String = getString(KafkaConfig.AuthorizerClassNameProp)
+  val authorizer: Option[Authorizer] = {
+    val className = getString(KafkaConfig.AuthorizerClassNameProp)
+    if (className == null || className.isEmpty)
+      None
+    else {
+      val authZ = Utils.newInstance(className, classOf[Object]) match {
+        case auth: Authorizer => auth
+        case auth: kafka.security.auth.Authorizer => new AuthorizerWrapper(auth)
+        case auth => throw new ConfigException(s"Authorizer does not implement ${classOf[Authorizer].getName} or kafka.security.auth.Authorizer .")
+      }
+      Some(authZ)
+    }
+  }
 
   /** ********* Socket Server Configuration ***********/
   val hostName = getString(KafkaConfig.HostNameProp)
diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala
index 68eef3ab215b0..353b66a32d161 100755
--- a/core/src/main/scala/kafka/server/KafkaServer.scala
+++ b/core/src/main/scala/kafka/server/KafkaServer.scala
@@ -26,7 +26,7 @@ import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger}
 import com.yammer.metrics.core.Gauge
 import kafka.api.{KAFKA_0_9_0, KAFKA_2_2_IV0}
 import kafka.cluster.Broker
-import kafka.common.{GenerateBrokerIdException, InconsistentBrokerIdException, InconsistentClusterIdException, InconsistentBrokerMetadataException}
+import kafka.common.{GenerateBrokerIdException, InconsistentBrokerIdException, InconsistentBrokerMetadataException, InconsistentClusterIdException}
 import kafka.controller.KafkaController
 import kafka.coordinator.group.GroupCoordinator
 import kafka.coordinator.transaction.TransactionCoordinator
@@ -34,7 +34,6 @@ import kafka.log.{LogConfig, LogManager}
 import kafka.metrics.{KafkaMetricsGroup, KafkaMetricsReporter}
 import kafka.network.SocketServer
 import kafka.security.CredentialProvider
-import kafka.security.auth.Authorizer
 import kafka.utils._
 import kafka.zk.{BrokerInfo, KafkaZkClient}
 import org.apache.kafka.clients.{ApiVersions, ClientDnsLookup, ManualMetadataUpdater, NetworkClient, NetworkClientUtils}
@@ -48,7 +47,8 @@ import org.apache.kafka.common.security.scram.internals.ScramMechanism
 import org.apache.kafka.common.security.token.delegation.internals.DelegationTokenCache
 import org.apache.kafka.common.security.{JaasContext, JaasUtils}
 import org.apache.kafka.common.utils.{AppInfoParser, LogContext, Time}
-import org.apache.kafka.common.{ClusterResource, Node}
+import org.apache.kafka.common.{ClusterResource, Endpoint, Node}
+import org.apache.kafka.server.authorizer.Authorizer
 
 import scala.collection.JavaConverters._
 import scala.collection.{Map, Seq, mutable}
@@ -293,10 +293,13 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
         transactionCoordinator.startup()
 
         /* Get the authorizer and initialize it if one is specified.*/
-        authorizer = Option(config.authorizerClassName).filter(_.nonEmpty).map { authorizerClassName =>
-          val authZ = CoreUtils.createObject[Authorizer](authorizerClassName)
-          authZ.configure(config.originals())
-          authZ
+        authorizer = config.authorizer
+        authorizer.foreach(_.configure(config.originals))
+        val authorizerFutures: Map[Endpoint, CompletableFuture[Void]] = authorizer match {
+          case Some(authZ) =>
+            authZ.start(brokerInfo.broker.toServerInfo(clusterId, config)).asScala
+          case None =>
+            brokerInfo.broker.endPoints.map{ ep => (ep.asInstanceOf[Endpoint], CompletableFuture.completedFuture[Void](null)) }.toMap
         }
 
         val fetchManager = new FetchManager(Time.SYSTEM,
@@ -335,8 +338,8 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
         dynamicConfigManager = new DynamicConfigManager(zkClient, dynamicConfigHandlers)
         dynamicConfigManager.startup()
 
-        socketServer.startDataPlaneProcessors()
-        socketServer.startControlPlaneProcessor()
+        socketServer.startControlPlaneProcessor(authorizerFutures)
+        socketServer.startDataPlaneProcessors(authorizerFutures)
         brokerState.newState(RunningAsBroker)
         shutdownLatch = new CountDownLatch(1)
         startupComplete.set(true)
diff --git a/core/src/main/scala/kafka/zk/KafkaZkClient.scala b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
index 7adb27e4f74a6..21ebe607b09b4 100644
--- a/core/src/main/scala/kafka/zk/KafkaZkClient.scala
+++ b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
@@ -24,7 +24,7 @@ import kafka.cluster.Broker
 import kafka.controller.{KafkaController, LeaderIsrAndControllerEpoch}
 import kafka.log.LogConfig
 import kafka.metrics.KafkaMetricsGroup
-import kafka.security.auth.SimpleAclAuthorizer.{NoAcls, VersionedAcls}
+import kafka.security.authorizer.AclAuthorizer.{NoAcls, VersionedAcls}
 import kafka.security.auth.{Acl, Resource, ResourceType}
 import kafka.server.ConfigType
 import kafka.utils.Logging
diff --git a/core/src/main/scala/kafka/zk/ZkData.scala b/core/src/main/scala/kafka/zk/ZkData.scala
index 66e87649be3e4..cefa94589de89 100644
--- a/core/src/main/scala/kafka/zk/ZkData.scala
+++ b/core/src/main/scala/kafka/zk/ZkData.scala
@@ -26,7 +26,7 @@ import kafka.cluster.{Broker, EndPoint}
 import kafka.common.{NotificationHandler, ZkNodeChangeNotificationListener}
 import kafka.controller.{IsrChangeNotificationHandler, LeaderIsrAndControllerEpoch}
 import kafka.security.auth.Resource.Separator
-import kafka.security.auth.SimpleAclAuthorizer.VersionedAcls
+import kafka.security.authorizer.AclAuthorizer.VersionedAcls
 import kafka.security.auth.{Acl, Resource, ResourceType}
 import kafka.server.{ConfigType, DelegationTokenManager}
 import kafka.utils.Json
diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index 328949e26087f..0484f6a5f78da 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -44,7 +44,7 @@ import org.apache.kafka.common.acl._
 import org.apache.kafka.common.config.{ConfigResource, LogLevelConfig}
 import org.apache.kafka.common.errors._
 import org.apache.kafka.common.requests.{DeleteRecordsRequest, MetadataResponse}
-import org.apache.kafka.common.resource.{PatternType, ResourcePattern, ResourceType}
+import org.apache.kafka.common.resource.{PatternType, Resource, ResourcePattern, ResourceType}
 import org.apache.kafka.common.utils.{Time, Utils}
 import org.junit.Assert._
 import org.junit.rules.Timeout
@@ -76,6 +76,8 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
   val topic = "topic"
   val partition = 0
   val topicPartition = new TopicPartition(topic, partition)
+  val clusterResourcePattern = new ResourcePattern(ResourceType.CLUSTER, Resource.CLUSTER_NAME, PatternType.LITERAL)
+
 
   @Before
   override def setUp(): Unit = {
diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
index 8e746c7888d83..d3fccf5fec249 100644
--- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
@@ -22,7 +22,8 @@ import java.time.Duration
 import kafka.admin.ConsumerGroupCommand.{ConsumerGroupCommandOptions, ConsumerGroupService}
 import kafka.log.LogConfig
 import kafka.network.SocketServer
-import kafka.security.auth._
+import kafka.security.auth.{ResourceType => AuthResourceType, SimpleAclAuthorizer, Topic}
+import kafka.security.authorizer.AuthorizerUtils.WildcardHost
 import kafka.server.{BaseRequestTest, KafkaConfig}
 import kafka.utils.TestUtils
 import org.apache.kafka.clients.admin.{Admin, AdminClient, AdminClientConfig, AlterConfigOp}
@@ -30,7 +31,9 @@ import org.apache.kafka.clients.consumer._
 import org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener
 import org.apache.kafka.clients.producer._
 import org.apache.kafka.common.ElectionType
-import org.apache.kafka.common.acl.{AccessControlEntry, AccessControlEntryFilter, AclBinding, AclBindingFilter, AclOperation, AclPermissionType}
+import org.apache.kafka.common.acl.{AccessControlEntry, AccessControlEntryFilter, AclBinding, AclBindingFilter, AclOperation}
+import org.apache.kafka.common.acl.AclOperation._
+import org.apache.kafka.common.acl.AclPermissionType.{ALLOW, DENY}
 import org.apache.kafka.common.config.{ConfigResource, LogLevelConfig}
 import org.apache.kafka.common.errors._
 import org.apache.kafka.common.internals.Topic.GROUP_METADATA_TOPIC_NAME
@@ -57,7 +60,8 @@ import org.apache.kafka.common.record.{CompressionType, MemoryRecords, RecordBat
 import org.apache.kafka.common.requests.CreateAclsRequest.AclCreation
 import org.apache.kafka.common.requests._
 import org.apache.kafka.common.resource.PatternType.LITERAL
-import org.apache.kafka.common.resource.{ResourcePattern, ResourcePatternFilter, ResourceType => AdminResourceType}
+import org.apache.kafka.common.resource.{Resource, ResourcePattern, ResourcePatternFilter, ResourceType}
+import org.apache.kafka.common.resource.ResourceType._
 import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
 import org.apache.kafka.common.{KafkaException, Node, TopicPartition, requests}
 import org.apache.kafka.test.{TestUtils => JTestUtils}
@@ -88,31 +92,33 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   val logDir = "logDir"
   val deleteRecordsPartition = new TopicPartition(deleteTopic, part)
   val group = "my-group"
-  val topicResource = Resource(Topic, topic, LITERAL)
-  val groupResource = Resource(Group, group, LITERAL)
-  val deleteTopicResource = Resource(Topic, deleteTopic, LITERAL)
-  val transactionalIdResource = Resource(TransactionalId, transactionalId, LITERAL)
-  val createTopicResource = Resource(Topic, createTopic, LITERAL)
-
-  val groupReadAcl = Map(groupResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)))
-  val groupDescribeAcl = Map(groupResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)))
-  val groupDeleteAcl = Map(groupResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Delete)))
-  val clusterAcl = Map(Resource.ClusterResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, ClusterAction)))
-  val clusterCreateAcl = Map(Resource.ClusterResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Create)))
-  val clusterAlterAcl = Map(Resource.ClusterResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Alter)))
-  val clusterDescribeAcl = Map(Resource.ClusterResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)))
-  val clusterAlterConfigsAcl = Map(Resource.ClusterResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, AlterConfigs)))
-  val clusterIdempotentWriteAcl = Map(Resource.ClusterResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, IdempotentWrite)))
-  val topicCreateAcl = Map(createTopicResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Create)))
-  val topicReadAcl = Map(topicResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)))
-  val topicWriteAcl = Map(topicResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)))
-  val topicDescribeAcl = Map(topicResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)))
-  val topicAlterAcl = Map(topicResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Alter)))
-  val topicDeleteAcl = Map(deleteTopicResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Delete)))
-  val topicDescribeConfigsAcl = Map(topicResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, DescribeConfigs)))
-  val topicAlterConfigsAcl = Map(topicResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, AlterConfigs)))
-  val transactionIdWriteAcl = Map(transactionalIdResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)))
-  val transactionalIdDescribeAcl = Map(transactionalIdResource -> Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)))
+  val clusterResource = new ResourcePattern(CLUSTER, Resource.CLUSTER_NAME, LITERAL)
+  val topicResource = new ResourcePattern(TOPIC, topic, LITERAL)
+  val groupResource = new ResourcePattern(GROUP, group, LITERAL)
+  val deleteTopicResource = new ResourcePattern(TOPIC, deleteTopic, LITERAL)
+  val transactionalIdResource = new ResourcePattern(TRANSACTIONAL_ID, transactionalId, LITERAL)
+  val createTopicResource = new ResourcePattern(TOPIC, createTopic, LITERAL)
+  val userPrincipalStr = userPrincipal.toString
+
+  val groupReadAcl = Map(groupResource -> Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)))
+  val groupDescribeAcl = Map(groupResource -> Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)))
+  val groupDeleteAcl = Map(groupResource -> Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DELETE, ALLOW)))
+  val clusterAcl = Map(clusterResource -> Set(new AccessControlEntry(userPrincipalStr, WildcardHost, CLUSTER_ACTION, ALLOW)))
+  val clusterCreateAcl = Map(clusterResource -> Set(new AccessControlEntry(userPrincipalStr, WildcardHost, CREATE, ALLOW)))
+  val clusterAlterAcl = Map(clusterResource -> Set(new AccessControlEntry(userPrincipalStr, WildcardHost, ALTER, ALLOW)))
+  val clusterDescribeAcl = Map(clusterResource -> Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)))
+  val clusterAlterConfigsAcl = Map(clusterResource -> Set(new AccessControlEntry(userPrincipalStr, WildcardHost, ALTER_CONFIGS, ALLOW)))
+  val clusterIdempotentWriteAcl = Map(clusterResource -> Set(new AccessControlEntry(userPrincipalStr, WildcardHost, IDEMPOTENT_WRITE, ALLOW)))
+  val topicCreateAcl = Map(createTopicResource -> Set(new AccessControlEntry(userPrincipalStr, WildcardHost, CREATE, ALLOW)))
+  val topicReadAcl = Map(topicResource -> Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)))
+  val topicWriteAcl = Map(topicResource -> Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)))
+  val topicDescribeAcl = Map(topicResource -> Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)))
+  val topicAlterAcl = Map(topicResource -> Set(new AccessControlEntry(userPrincipalStr, WildcardHost, ALTER, ALLOW)))
+  val topicDeleteAcl = Map(deleteTopicResource -> Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DELETE, ALLOW)))
+  val topicDescribeConfigsAcl = Map(topicResource -> Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE_CONFIGS, ALLOW)))
+  val topicAlterConfigsAcl = Map(topicResource -> Set(new AccessControlEntry(userPrincipalStr, WildcardHost, ALTER_CONFIGS, ALLOW)))
+  val transactionIdWriteAcl = Map(transactionalIdResource -> Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)))
+  val transactionalIdDescribeAcl = Map(transactionalIdResource -> Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)))
 
   val numRecords = 1
   val adminClients = Buffer[Admin]()
@@ -227,7 +233,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
     ApiKeys.LIST_PARTITION_REASSIGNMENTS -> ((resp: ListPartitionReassignmentsResponse) => Errors.forCode(resp.data().errorCode()))
   )
 
-  val requestKeysToAcls = Map[ApiKeys, Map[Resource, Set[Acl]]](
+  val requestKeysToAcls = Map[ApiKeys, Map[ResourcePattern, Set[AccessControlEntry]]](
     ApiKeys.METADATA -> topicDescribeAcl,
     ApiKeys.PRODUCE -> (topicWriteAcl ++ transactionIdWriteAcl ++ clusterIdempotentWriteAcl),
     ApiKeys.FETCH -> topicReadAcl,
@@ -273,7 +279,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   override def setUp(): Unit = {
     doSetup(createOffsetsTopic = false)
 
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, ClusterAction)), Resource.ClusterResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, CLUSTER_ACTION, ALLOW)), clusterResource)
 
     TestUtils.createOffsetsTopic(zkClient, servers)
 
@@ -477,13 +483,13 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   private def createAclsRequest = new CreateAclsRequest.Builder(
     Collections.singletonList(new AclCreation(new AclBinding(
-      new ResourcePattern(AdminResourceType.TOPIC, "mytopic", LITERAL),
-      new AccessControlEntry(userPrincipal.toString, "*", AclOperation.WRITE, AclPermissionType.DENY))))).build()
+      new ResourcePattern(ResourceType.TOPIC, "mytopic", LITERAL),
+      new AccessControlEntry(userPrincipalStr, "*", AclOperation.WRITE, DENY))))).build()
 
   private def deleteAclsRequest = new DeleteAclsRequest.Builder(
     Collections.singletonList(new AclBindingFilter(
-      new ResourcePatternFilter(AdminResourceType.TOPIC, null, LITERAL),
-      new AccessControlEntryFilter(userPrincipal.toString, "*", AclOperation.ANY, AclPermissionType.DENY)))).build()
+      new ResourcePatternFilter(ResourceType.TOPIC, null, LITERAL),
+      new AccessControlEntryFilter(userPrincipalStr, "*", AclOperation.ANY, DENY)))).build()
 
   private def alterReplicaLogDirsRequest = new AlterReplicaLogDirsRequest.Builder(Collections.singletonMap(tp, logDir)).build()
 
@@ -629,7 +635,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   @Test
   def testCreateTopicAuthorizationWithClusterCreate(): Unit = {
     removeAllAcls()
-    val resources = Set[ResourceType](Topic)
+    val resources = Set[ResourceType](TOPIC)
 
     sendRequestAndVerifyResponseError(ApiKeys.CREATE_TOPICS, createTopicsRequest, resources, isAuthorized = false)
 
@@ -644,15 +650,15 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
     val request = createFetchFollowerRequest
 
     removeAllAcls()
-    val resources = Set(topicResource.resourceType, Resource.ClusterResource.resourceType)
+    val resources = Set(topicResource.resourceType, clusterResource.resourceType)
     sendRequestAndVerifyResponseError(key, request, resources, isAuthorized = false)
 
     val readAcls = topicReadAcl(topicResource)
     addAndVerifyAcls(readAcls, topicResource)
     sendRequestAndVerifyResponseError(key, request, resources, isAuthorized = false)
 
-    val clusterAcls = clusterAcl(Resource.ClusterResource)
-    addAndVerifyAcls(clusterAcls, Resource.ClusterResource)
+    val clusterAcls = clusterAcl(clusterResource)
+    addAndVerifyAcls(clusterAcls, clusterResource)
     sendRequestAndVerifyResponseError(key, request, resources, isAuthorized = true)
   }
 
@@ -670,11 +676,11 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
     val request = new IncrementalAlterConfigsRequest.Builder(data).build()
 
     removeAllAcls()
-    val resources = Set(topicResource.resourceType, Resource.ClusterResource.resourceType)
+    val resources = Set(topicResource.resourceType, clusterResource.resourceType)
     sendRequestAndVerifyResponseError(key, request, resources, isAuthorized = false)
 
-    val clusterAcls = clusterAlterConfigsAcl(Resource.ClusterResource)
-    addAndVerifyAcls(clusterAcls, Resource.ClusterResource)
+    val clusterAcls = clusterAlterConfigsAcl(clusterResource)
+    addAndVerifyAcls(clusterAcls, clusterResource)
     sendRequestAndVerifyResponseError(key, request, resources, isAuthorized = true)
   }
 
@@ -685,13 +691,13 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
     removeAllAcls()
 
-    val resources = Set(topicResource.resourceType, Resource.ClusterResource.resourceType)
+    val resources = Set(topicResource.resourceType, clusterResource.resourceType)
     sendRequestAndVerifyResponseError(key, request, resources, isAuthorized = false)
 
     // Although the OffsetsForLeaderEpoch API now accepts topic describe, we should continue
     // allowing cluster action for backwards compatibility
-    val clusterAcls = clusterAcl(Resource.ClusterResource)
-    addAndVerifyAcls(clusterAcls, Resource.ClusterResource)
+    val clusterAcls = clusterAcl(clusterResource)
+    addAndVerifyAcls(clusterAcls, clusterResource)
     sendRequestAndVerifyResponseError(key, request, resources, isAuthorized = true)
   }
 
@@ -708,7 +714,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testProduceWithTopicDescribe(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), topicResource)
     try {
       val producer = createProducer()
       sendRecords(producer, numRecords, tp)
@@ -721,7 +727,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testProduceWithTopicRead(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), topicResource)
     try {
       val producer = createProducer()
       sendRecords(producer, numRecords, tp)
@@ -734,25 +740,25 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testProduceWithTopicWrite(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
     val producer = createProducer()
     sendRecords(producer, numRecords, tp)
   }
 
   @Test
   def testCreatePermissionOnTopicToWriteToNonExistentTopic(): Unit = {
-    testCreatePermissionNeededToWriteToNonExistentTopic(Topic)
+    testCreatePermissionNeededToWriteToNonExistentTopic(TOPIC)
   }
 
   @Test
   def testCreatePermissionOnClusterToWriteToNonExistentTopic(): Unit = {
-    testCreatePermissionNeededToWriteToNonExistentTopic(Cluster)
+    testCreatePermissionNeededToWriteToNonExistentTopic(CLUSTER)
   }
 
   private def testCreatePermissionNeededToWriteToNonExistentTopic(resType: ResourceType): Unit = {
     val topicPartition = new TopicPartition(createTopic, 0)
-    val newTopicResource = Resource(Topic, createTopic, LITERAL)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), newTopicResource)
+    val newTopicResource = new ResourcePattern(TOPIC, createTopic, LITERAL)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), newTopicResource)
     val producer = createProducer()
     try {
       sendRecords(producer, numRecords, topicPartition)
@@ -762,15 +768,15 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
         assertEquals(Collections.singleton(createTopic), e.unauthorizedTopics())
     }
 
-    val resource = if (resType == Topic) newTopicResource else Resource.ClusterResource
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Create)), resource)
+    val resource = if (resType == Topic) newTopicResource else clusterResource
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, CREATE, ALLOW)), resource)
 
     sendRecords(producer, numRecords, topicPartition)
   }
 
   @Test(expected = classOf[TopicAuthorizationException])
   def testConsumeUsingAssignWithNoAccess(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
     removeAllAcls()
@@ -782,12 +788,12 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testSimpleConsumeWithOffsetLookupAndNoGroupAccess(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
     removeAllAcls()
 
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), topicResource)
     try {
       // note this still depends on group access because we haven't set offsets explicitly, which means
       // they will first be fetched from the consumer coordinator (which requires group access)
@@ -802,12 +808,12 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testSimpleConsumeWithExplicitSeekAndNoGroupAccess(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
     removeAllAcls()
 
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), topicResource)
 
     // in this case, we do an explicit seek, so there should be no need to query the coordinator at all
     val consumer = createConsumer()
@@ -818,12 +824,12 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test(expected = classOf[KafkaException])
   def testConsumeWithoutTopicDescribeAccess(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
     removeAllAcls()
 
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
     val consumer = createConsumer()
     consumer.assign(List(tp).asJava)
 
@@ -834,13 +840,13 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testConsumeWithTopicDescribe(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
     removeAllAcls()
 
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), topicResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
     try {
       val consumer = createConsumer()
       consumer.assign(List(tp).asJava)
@@ -853,13 +859,13 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testConsumeWithTopicWrite(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
     removeAllAcls()
 
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
     try {
       val consumer = createConsumer()
       consumer.assign(List(tp).asJava)
@@ -873,13 +879,13 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testConsumeWithTopicAndGroupRead(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
     removeAllAcls()
 
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
 
     val consumer = createConsumer()
     consumer.assign(List(tp).asJava)
@@ -888,12 +894,12 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testPatternSubscriptionWithNoTopicAccess(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
     removeAllAcls()
 
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
 
     val consumer = createConsumer()
     consumer.subscribe(Pattern.compile(topicPattern), new NoOpConsumerRebalanceListener)
@@ -903,13 +909,13 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testPatternSubscriptionWithTopicDescribeOnlyAndGroupRead(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
     removeAllAcls()
 
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), topicResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
     val consumer = createConsumer()
     consumer.subscribe(Pattern.compile(topicPattern))
     try {
@@ -922,26 +928,26 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testPatternSubscriptionWithTopicAndGroupRead(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
 
     // create an unmatched topic
     val unmatchedTopic = "unmatched"
     createTopic(unmatchedTopic)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)),  Resource(Topic, unmatchedTopic, LITERAL))
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)),  new ResourcePattern(TOPIC, unmatchedTopic, LITERAL))
     sendRecords(producer, 1, new TopicPartition(unmatchedTopic, part))
     removeAllAcls()
 
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
     val consumer = createConsumer()
     consumer.subscribe(Pattern.compile(topicPattern))
     consumeRecords(consumer)
 
     // set the subscription pattern to an internal topic that the consumer has read permission to. Since
     // internal topics are not included, we should not be assigned any partitions from this topic
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)),  Resource(Topic,
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)),  new ResourcePattern(TOPIC,
       GROUP_METADATA_TOPIC_NAME, LITERAL))
     consumer.subscribe(Pattern.compile(GROUP_METADATA_TOPIC_NAME))
     consumer.poll(0)
@@ -951,13 +957,13 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testPatternSubscriptionMatchingInternalTopic(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
     removeAllAcls()
 
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
 
     consumerConfig.put(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG, "false")
     val consumer = createConsumer()
@@ -967,7 +973,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
     assertEquals(Set(topic).asJava, consumer.subscription)
 
     // now authorize the user for the internal topic and verify that we can subscribe
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), Resource(Topic,
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), new ResourcePattern(TOPIC,
       GROUP_METADATA_TOPIC_NAME, LITERAL))
     consumer.subscribe(Pattern.compile(GROUP_METADATA_TOPIC_NAME))
     consumer.poll(0)
@@ -976,15 +982,15 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testPatternSubscriptionMatchingInternalTopicWithDescribeOnlyPermission(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
     removeAllAcls()
 
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
-    val internalTopicResource = Resource(Topic, GROUP_METADATA_TOPIC_NAME, LITERAL)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), internalTopicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
+    val internalTopicResource = new ResourcePattern(TOPIC, GROUP_METADATA_TOPIC_NAME, LITERAL)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), internalTopicResource)
 
     consumerConfig.put(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG, "false")
     val consumer = createConsumer()
@@ -1001,13 +1007,13 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testPatternSubscriptionNotMatchingInternalTopic(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
     val producer = createProducer()
     sendRecords(producer, 1, tp)
     removeAllAcls()
 
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
 
     consumerConfig.put(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG, "false")
     val consumer = createConsumer()
@@ -1015,26 +1021,26 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
       consumer.subscribe(Pattern.compile(topicPattern))
       consumeRecords(consumer)
     } finally consumer.close()
-}
+  }
 
   @Test
   def testCreatePermissionOnTopicToReadFromNonExistentTopic(): Unit = {
     testCreatePermissionNeededToReadFromNonExistentTopic("newTopic",
-      Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Create)),
-      Topic)
+      Set(new AccessControlEntry(userPrincipalStr, WildcardHost, CREATE, ALLOW)),
+      TOPIC)
   }
 
   @Test
   def testCreatePermissionOnClusterToReadFromNonExistentTopic(): Unit = {
     testCreatePermissionNeededToReadFromNonExistentTopic("newTopic",
-      Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Create)),
-      Cluster)
+      Set(new AccessControlEntry(userPrincipalStr, WildcardHost, CREATE, ALLOW)),
+      CLUSTER)
   }
 
-  private def testCreatePermissionNeededToReadFromNonExistentTopic(newTopic: String, acls: Set[Acl], resType: ResourceType): Unit = {
+  private def testCreatePermissionNeededToReadFromNonExistentTopic(newTopic: String, acls: Set[AccessControlEntry], resType: ResourceType): Unit = {
     val topicPartition = new TopicPartition(newTopic, 0)
-    val newTopicResource = Resource(Topic, newTopic, LITERAL)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), newTopicResource)
+    val newTopicResource = new ResourcePattern(TOPIC, newTopic, LITERAL)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), newTopicResource)
     addAndVerifyAcls(groupReadAcl(groupResource), groupResource)
     val consumer = createConsumer()
     consumer.assign(List(topicPartition).asJava)
@@ -1043,7 +1049,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
     }.unauthorizedTopics
     assertEquals(Collections.singleton(newTopic), unauthorizedTopics)
 
-    val resource = if (resType == Topic) newTopicResource else Resource.ClusterResource
+    val resource = if (resType == TOPIC) newTopicResource else clusterResource
     addAndVerifyAcls(acls, resource)
 
     TestUtils.waitUntilTrue(() => {
@@ -1085,38 +1091,38 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test(expected = classOf[KafkaException])
   def testCommitWithNoTopicAccess(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
     val consumer = createConsumer()
     consumer.commitSync(Map(tp -> new OffsetAndMetadata(5)).asJava)
   }
 
   @Test(expected = classOf[TopicAuthorizationException])
   def testCommitWithTopicWrite(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
     val consumer = createConsumer()
     consumer.commitSync(Map(tp -> new OffsetAndMetadata(5)).asJava)
   }
 
   @Test(expected = classOf[TopicAuthorizationException])
   def testCommitWithTopicDescribe(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), topicResource)
     val consumer = createConsumer()
     consumer.commitSync(Map(tp -> new OffsetAndMetadata(5)).asJava)
   }
 
   @Test(expected = classOf[GroupAuthorizationException])
   def testCommitWithNoGroupAccess(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), topicResource)
     val consumer = createConsumer()
     consumer.commitSync(Map(tp -> new OffsetAndMetadata(5)).asJava)
   }
 
   @Test
   def testCommitWithTopicAndGroupRead(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), topicResource)
     val consumer = createConsumer()
     consumer.commitSync(Map(tp -> new OffsetAndMetadata(5)).asJava)
   }
@@ -1130,7 +1136,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test(expected = classOf[GroupAuthorizationException])
   def testOffsetFetchWithNoGroupAccess(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), topicResource)
     val consumer = createConsumer()
     consumer.assign(List(tp).asJava)
     consumer.position(tp)
@@ -1138,7 +1144,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test(expected = classOf[KafkaException])
   def testOffsetFetchWithNoTopicAccess(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
     val consumer = createConsumer()
     consumer.assign(List(tp).asJava)
     consumer.position(tp)
@@ -1147,14 +1153,14 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   @Test
   def testFetchAllOffsetsTopicAuthorization(): Unit = {
     val offset = 15L
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), topicResource)
     val consumer = createConsumer()
     consumer.assign(List(tp).asJava)
     consumer.commitSync(Map(tp -> new OffsetAndMetadata(offset)).asJava)
 
     removeAllAcls()
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
 
     // send offset fetch requests directly since the consumer does not expose an API to do so
     // note there's only one broker, so no need to lookup the group coordinator
@@ -1166,7 +1172,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
     assertTrue(offsetFetchResponse.responseData.isEmpty)
 
     // now add describe permission on the topic and verify that the offset can be fetched
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), topicResource)
     offsetFetchResponse = sendOffsetFetchRequest(offsetFetchRequest, anySocketServer)
     assertEquals(Errors.NONE, offsetFetchResponse.error)
     assertTrue(offsetFetchResponse.responseData.containsKey(tp))
@@ -1175,8 +1181,8 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testOffsetFetchTopicDescribe(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), groupResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), topicResource)
     val consumer = createConsumer()
     consumer.assign(List(tp).asJava)
     consumer.position(tp)
@@ -1184,8 +1190,8 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testOffsetFetchWithTopicAndGroupRead(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), topicResource)
     val consumer = createConsumer()
     consumer.assign(List(tp).asJava)
     consumer.position(tp)
@@ -1199,7 +1205,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testMetadataWithTopicDescribe(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), topicResource)
     val consumer = createConsumer()
     consumer.partitionsFor(topic)
   }
@@ -1212,29 +1218,29 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testListOffsetsWithTopicDescribe(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), topicResource)
     val consumer = createConsumer()
     consumer.endOffsets(Set(tp).asJava)
   }
 
   @Test
   def testDescribeGroupApiWithNoGroupAcl(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), topicResource)
     val result = createAdminClient().describeConsumerGroups(Seq(group).asJava)
     TestUtils.assertFutureExceptionTypeEquals(result.describedGroups().get(group), classOf[GroupAuthorizationException])
   }
 
   @Test
   def testDescribeGroupApiWithGroupDescribe(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), groupResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), topicResource)
     createAdminClient().describeConsumerGroups(Seq(group).asJava).describedGroups().get(group).get()
   }
 
   @Test
   def testDescribeGroupCliWithGroupDescribe(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), groupResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), topicResource)
 
     val cgcArgs = Array("--bootstrap-server", brokerList, "--describe", "--group", group)
     val opts = new ConsumerGroupCommandOptions(cgcArgs)
@@ -1246,15 +1252,15 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   @Test
   def testListGroupApiWithAndWithoutListGroupAcls(): Unit = {
     // write some record to the topic
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
     val producer = createProducer()
     sendRecords(producer, numRecords = 1, tp)
 
     // use two consumers to write to two different groups
     val group2 = "other group"
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), Resource(Group, group2, LITERAL))
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), new ResourcePattern(GROUP, group2, LITERAL))
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), topicResource)
     val consumer = createConsumer()
     consumer.subscribe(Collections.singleton(topic))
     consumeRecords(consumer)
@@ -1267,13 +1273,13 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
     // first use cluster describe permission
     removeAllAcls()
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), Resource.ClusterResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), clusterResource)
     // it should list both groups (due to cluster describe permission)
     assertEquals(Set(group, group2), adminClient.listConsumerGroups().all().get().asScala.map(_.groupId()).toSet)
 
     // now replace cluster describe with group read permission
     removeAllAcls()
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
     // it should list only one group now
     val groupList = adminClient.listConsumerGroups().all().get().asScala.toList
     assertEquals(1, groupList.length)
@@ -1289,9 +1295,9 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testDeleteGroupApiWithDeleteGroupAcl(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Delete)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DELETE, ALLOW)), groupResource)
     val consumer = createConsumer()
     consumer.assign(List(tp).asJava)
     consumer.commitSync(Map(tp -> new OffsetAndMetadata(5, "")).asJava)
@@ -1300,8 +1306,8 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testDeleteGroupApiWithNoDeleteGroupAcl(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), groupResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Read)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), topicResource)
     val consumer = createConsumer()
     consumer.assign(List(tp).asJava)
     consumer.commitSync(Map(tp -> new OffsetAndMetadata(5, "")).asJava)
@@ -1325,7 +1331,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testUnauthorizedDeleteTopicsWithDescribe(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), deleteTopicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), deleteTopicResource)
     val response = connectAndSend(deleteTopicsRequest, ApiKeys.DELETE_TOPICS)
     val version = ApiKeys.DELETE_TOPICS.latestVersion
     val deleteResponse = DeleteTopicsResponse.parse(response, version)
@@ -1335,7 +1341,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testDeleteTopicsWithWildCardAuth(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Delete)), Resource(Topic, "*", LITERAL))
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DELETE, ALLOW)), new ResourcePattern(TOPIC, "*", LITERAL))
     val response = connectAndSend(deleteTopicsRequest, ApiKeys.DELETE_TOPICS)
     val version = ApiKeys.DELETE_TOPICS.latestVersion
     val deleteResponse = DeleteTopicsResponse.parse(response, version)
@@ -1353,7 +1359,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testUnauthorizedDeleteRecordsWithDescribe(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), deleteTopicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), deleteTopicResource)
     val response = connectAndSend(deleteRecordsRequest, ApiKeys.DELETE_RECORDS)
     val version = ApiKeys.DELETE_RECORDS.latestVersion
     val deleteRecordsResponse = DeleteRecordsResponse.parse(response, version)
@@ -1362,7 +1368,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testDeleteRecordsWithWildCardAuth(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Delete)), Resource(Topic, "*", LITERAL))
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DELETE, ALLOW)), new ResourcePattern(TOPIC, "*", LITERAL))
     val response = connectAndSend(deleteRecordsRequest, ApiKeys.DELETE_RECORDS)
     val version = ApiKeys.DELETE_RECORDS.latestVersion
     val deleteRecordsResponse = DeleteRecordsResponse.parse(response, version)
@@ -1380,7 +1386,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testCreatePartitionsWithWildCardAuth(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Alter)), Resource(Topic, "*", LITERAL))
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, ALTER, ALLOW)), new ResourcePattern(TOPIC, "*", LITERAL))
     val response = connectAndSend(createPartitionsRequest, ApiKeys.CREATE_PARTITIONS)
     val version = ApiKeys.CREATE_PARTITIONS.latestVersion
     val createPartitionsResponse = CreatePartitionsResponse.parse(response, version)
@@ -1389,7 +1395,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test(expected = classOf[TransactionalIdAuthorizationException])
   def testTransactionalProducerInitTransactionsNoWriteTransactionalIdAcl(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), transactionalIdResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), transactionalIdResource)
     val producer = buildTransactionalProducer()
     producer.initTransactions()
   }
@@ -1402,9 +1408,9 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testSendOffsetsWithNoConsumerGroupDescribeAccess(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, ClusterAction)), Resource.ClusterResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), transactionalIdResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, CLUSTER_ACTION, ALLOW)), clusterResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), transactionalIdResource)
     val producer = buildTransactionalProducer()
     producer.initTransactions()
     producer.beginTransaction()
@@ -1418,8 +1424,8 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testSendOffsetsWithNoConsumerGroupWriteAccess(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), transactionalIdResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), transactionalIdResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), groupResource)
     val producer = buildTransactionalProducer()
     producer.initTransactions()
     producer.beginTransaction()
@@ -1433,7 +1439,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testIdempotentProducerNoIdempotentWriteAclInInitProducerId(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
     val producer = buildIdempotentProducer()
     try {
       // the InitProducerId is sent asynchronously, so we expect the error either in the callback
@@ -1457,8 +1463,8 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testIdempotentProducerNoIdempotentWriteAclInProduce(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, IdempotentWrite)), Resource.ClusterResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, IDEMPOTENT_WRITE, ALLOW)), clusterResource)
 
     val producer = buildIdempotentProducer()
 
@@ -1467,7 +1473,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
     // revoke the IdempotentWrite permission
     removeAllAcls()
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
 
     try {
       // the send should now fail with a cluster auth error
@@ -1490,16 +1496,16 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def shouldInitTransactionsWhenAclSet(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), transactionalIdResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), transactionalIdResource)
     val producer = buildTransactionalProducer()
     producer.initTransactions()
   }
 
   @Test
   def testTransactionalProducerTopicAuthorizationExceptionInSendCallback(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), transactionalIdResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), transactionalIdResource)
     // add describe access so that we can fetch metadata
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), topicResource)
     val producer = buildTransactionalProducer()
     producer.initTransactions()
     producer.beginTransaction()
@@ -1519,9 +1525,9 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def testTransactionalProducerTopicAuthorizationExceptionInCommit(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), transactionalIdResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), transactionalIdResource)
     // add describe access so that we can fetch metadata
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), topicResource)
     val producer = buildTransactionalProducer()
     producer.initTransactions()
     producer.beginTransaction()
@@ -1537,11 +1543,11 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def shouldThrowTransactionalIdAuthorizationExceptionWhenNoTransactionAccessDuringSend(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), transactionalIdResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), transactionalIdResource)
     val producer = buildTransactionalProducer()
     producer.initTransactions()
     removeAllAcls()
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
     try {
       producer.beginTransaction()
       producer.send(new ProducerRecord(tp.topic, tp.partition, "1".getBytes, "1".getBytes)).get
@@ -1554,8 +1560,8 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def shouldThrowTransactionalIdAuthorizationExceptionWhenNoTransactionAccessOnEndTransaction(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), transactionalIdResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), transactionalIdResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
     val producer = buildTransactionalProducer()
     producer.initTransactions()
     producer.beginTransaction()
@@ -1571,9 +1577,9 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def shouldSuccessfullyAbortTransactionAfterTopicAuthorizationException(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), transactionalIdResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Describe)), Resource(Topic, deleteTopic, LITERAL))
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), transactionalIdResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DESCRIBE, ALLOW)), new ResourcePattern(TOPIC, deleteTopic, LITERAL))
     val producer = buildTransactionalProducer()
     producer.initTransactions()
     producer.beginTransaction()
@@ -1591,8 +1597,8 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def shouldThrowTransactionalIdAuthorizationExceptionWhenNoTransactionAccessOnSendOffsetsToTxn(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), transactionalIdResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), transactionalIdResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), groupResource)
     val producer = buildTransactionalProducer()
     producer.initTransactions()
     producer.beginTransaction()
@@ -1608,19 +1614,35 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   @Test
   def shouldSendSuccessfullyWhenIdempotentAndHasCorrectACL(): Unit = {
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, IdempotentWrite)), Resource.ClusterResource)
-    addAndVerifyAcls(Set(new Acl(userPrincipal, Allow, Acl.WildCardHost, Write)), topicResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, IDEMPOTENT_WRITE, ALLOW)), clusterResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, WRITE, ALLOW)), topicResource)
     val producer = buildIdempotentProducer()
     producer.send(new ProducerRecord(tp.topic, tp.partition, "1".getBytes, "1".getBytes)).get
   }
 
-  def removeAllAcls() = {
-    servers.head.dataPlaneRequestProcessor.authorizer.get.getAcls().keys.foreach { resource =>
-      servers.head.dataPlaneRequestProcessor.authorizer.get.removeAcls(resource)
-      TestUtils.waitAndVerifyAcls(Set.empty[Acl], servers.head.dataPlaneRequestProcessor.authorizer.get, resource)
+  // Verify that metadata request without topics works without any ACLs and returns cluster id
+  @Test
+  def testClusterId(): Unit = {
+    val request = new requests.MetadataRequest.Builder(List.empty.asJava, false).build()
+    val apiKey = ApiKeys.METADATA
+    val resp = connectAndSend(request, apiKey)
+    val response = requestKeyToResponseDeserializer(apiKey).getMethod("parse", classOf[ByteBuffer], classOf[Short]).invoke(
+      null, resp, request.version: java.lang.Short).asInstanceOf[MetadataResponse]
+    assertEquals(Collections.emptyMap, response.errorCounts)
+    assertFalse("Cluster id not returned", response.clusterId.isEmpty)
+  }
+
+  def removeAllAcls(): Unit = {
+    val authorizer = servers.head.dataPlaneRequestProcessor.authorizer.get
+    val aclFilter = AclBindingFilter.ANY
+    authorizer.deleteAcls(null, List(aclFilter).asJava).asScala.flatMap { deletion =>
+      deletion.aclBindingDeleteResults().asScala.map(_.aclBinding.pattern).toSet
+    }.foreach { resource =>
+      TestUtils.waitAndVerifyAcls(Set.empty[AccessControlEntry], authorizer, resource)
     }
   }
 
+
   def sendRequestAndVerifyResponseError(apiKey: ApiKeys,
                                         request: AbstractRequest,
                                         resources: Set[ResourceType],
@@ -1632,13 +1654,13 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
     val error = requestKeyToError(apiKey).asInstanceOf[AbstractResponse => Errors](response)
 
     val authorizationErrors = resources.flatMap { resourceType =>
-      if (resourceType == Topic) {
+      if (resourceType == TOPIC) {
         if (isAuthorized)
           Set(Errors.UNKNOWN_TOPIC_OR_PARTITION, Topic.error)
         else
           Set(Topic.error)
       } else {
-        Set(resourceType.error)
+        Set(AuthResourceType.fromJava(resourceType).error)
       }
     }
 
@@ -1647,7 +1669,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
         assertFalse(s"$apiKey should be allowed. Found unexpected authorization error $error", authorizationErrors.contains(error))
       else
         assertTrue(s"$apiKey should be forbidden. Found error $error but expected one of $authorizationErrors", authorizationErrors.contains(error))
-    else if (resources == Set(Topic))
+    else if (resources == Set(TOPIC))
       if (isAuthorized)
         assertEquals(s"$apiKey had an unexpected error", Errors.UNKNOWN_TOPIC_OR_PARTITION, error)
       else
@@ -1669,9 +1691,17 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
     }
   }
 
-  private def addAndVerifyAcls(acls: Set[Acl], resource: Resource) = {
-    servers.head.dataPlaneRequestProcessor.authorizer.get.addAcls(acls, resource)
-    TestUtils.waitAndVerifyAcls(servers.head.dataPlaneRequestProcessor.authorizer.get.getAcls(resource) ++ acls, servers.head.dataPlaneRequestProcessor.authorizer.get, resource)
+  private def addAndVerifyAcls(acls: Set[AccessControlEntry], resource: ResourcePattern): Unit = {
+    val aclBindings = acls.map { acl => new AclBinding(resource, acl) }
+    servers.head.dataPlaneRequestProcessor.authorizer.get
+      .createAcls(null, aclBindings.toList.asJava).asScala.foreach {result =>
+        if (result.failed())
+          throw result.exception()
+      }
+    val aclFilter = new AclBindingFilter(resource.toFilter, AccessControlEntryFilter.ANY)
+    TestUtils.waitAndVerifyAcls(
+      servers.head.dataPlaneRequestProcessor.authorizer.get.acls(aclFilter).asScala.map(_.entry).toSet ++ acls,
+      servers.head.dataPlaneRequestProcessor.authorizer.get, resource)
   }
 
   private def consumeRecords(consumer: Consumer[Array[Byte], Array[Byte]],
diff --git a/core/src/test/scala/integration/kafka/api/DelegationTokenEndToEndAuthorizationTest.scala b/core/src/test/scala/integration/kafka/api/DelegationTokenEndToEndAuthorizationTest.scala
index fa41baa8ddd46..2f1d51cc34f8b 100644
--- a/core/src/test/scala/integration/kafka/api/DelegationTokenEndToEndAuthorizationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/DelegationTokenEndToEndAuthorizationTest.scala
@@ -21,7 +21,7 @@ import java.util
 import kafka.server.KafkaConfig
 import kafka.utils.{JaasTestUtils, TestUtils}
 import kafka.zk.ConfigEntityChangeNotificationZNode
-import org.apache.kafka.clients.admin.{Admin, AdminClient, AdminClientConfig}
+import org.apache.kafka.clients.admin.{AdminClient, AdminClientConfig}
 import org.apache.kafka.common.config.SaslConfigs
 import org.apache.kafka.common.security.auth.SecurityProtocol
 import org.apache.kafka.common.security.scram.ScramCredential
diff --git a/core/src/test/scala/integration/kafka/api/DescribeAuthorizedOperationsTest.scala b/core/src/test/scala/integration/kafka/api/DescribeAuthorizedOperationsTest.scala
index a28e4dddeb62b..a4f76ce91f123 100644
--- a/core/src/test/scala/integration/kafka/api/DescribeAuthorizedOperationsTest.scala
+++ b/core/src/test/scala/integration/kafka/api/DescribeAuthorizedOperationsTest.scala
@@ -16,15 +16,19 @@ import java.io.File
 import java.util
 import java.util.Properties
 
-import kafka.security.auth.{Allow, Alter, Authorizer, Cluster, ClusterAction, Describe, Group, Operation, PermissionType, Resource, SimpleAclAuthorizer, Topic, Acl => AuthAcl}
+import kafka.security.auth.{Cluster, Group, Resource, Topic, Acl => AuthAcl}
+import kafka.security.authorizer.AclAuthorizer
 import kafka.server.KafkaConfig
 import kafka.utils.{CoreUtils, JaasTestUtils, TestUtils}
 import org.apache.kafka.clients.admin._
 import org.apache.kafka.common.acl._
+import org.apache.kafka.common.acl.AclOperation.{ALTER, DESCRIBE, CLUSTER_ACTION}
+import org.apache.kafka.common.acl.AclPermissionType.ALLOW
 import org.apache.kafka.common.resource.{PatternType, ResourcePattern, ResourceType}
 import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
 import org.apache.kafka.common.utils.Utils
-import org.junit.Assert.assertEquals
+import org.apache.kafka.server.authorizer.Authorizer
+import org.junit.Assert.{assertEquals, assertNull}
 import org.junit.{After, Before, Test}
 
 import scala.collection.JavaConverters._
@@ -32,7 +36,7 @@ import scala.collection.JavaConverters._
 class DescribeAuthorizedOperationsTest extends IntegrationTestHarness with SaslSetup {
   override val brokerCount = 1
   this.serverConfig.setProperty(KafkaConfig.ZkEnableSecureAclsProp, "true")
-  this.serverConfig.setProperty(KafkaConfig.AuthorizerClassNameProp, classOf[SimpleAclAuthorizer].getName)
+  this.serverConfig.setProperty(KafkaConfig.AuthorizerClassNameProp, classOf[AclAuthorizer].getName)
 
   var client: Admin = _
   val group1 = "group1"
@@ -46,14 +50,18 @@ class DescribeAuthorizedOperationsTest extends IntegrationTestHarness with SaslS
   override protected lazy val trustStoreFile = Some(File.createTempFile("truststore", ".jks"))
 
   override def configureSecurityBeforeServersStart(): Unit = {
-    val authorizer = CoreUtils.createObject[Authorizer](classOf[SimpleAclAuthorizer].getName)
-    val topicResource = Resource(Topic, Resource.WildCardResource, PatternType.LITERAL)
+    val authorizer = CoreUtils.createObject[Authorizer](classOf[AclAuthorizer].getName)
+    val clusterResource = new ResourcePattern(ResourceType.CLUSTER, Resource.ClusterResource.name, PatternType.LITERAL)
+    val topicResource = new ResourcePattern(ResourceType.TOPIC, Resource.WildCardResource, PatternType.LITERAL)
 
     try {
       authorizer.configure(this.configs.head.originals())
-      authorizer.addAcls(Set(clusterAcl(JaasTestUtils.KafkaServerPrincipalUnqualifiedName, Allow, ClusterAction),
-        clusterAcl(JaasTestUtils.KafkaClientPrincipalUnqualifiedName2, Allow, Alter)), Resource.ClusterResource)
-      authorizer.addAcls(Set(clusterAcl(JaasTestUtils.KafkaClientPrincipalUnqualifiedName2, Allow, Describe)), topicResource)
+      val result = authorizer.createAcls(null, List(
+        new AclBinding(clusterResource, accessControlEntry(JaasTestUtils.KafkaServerPrincipalUnqualifiedName.toString, ALLOW, CLUSTER_ACTION)),
+        new AclBinding(clusterResource, accessControlEntry(JaasTestUtils.KafkaClientPrincipalUnqualifiedName2.toString, ALLOW, ALTER)),
+        new AclBinding(topicResource, accessControlEntry(JaasTestUtils.KafkaClientPrincipalUnqualifiedName2.toString, ALLOW, DESCRIBE))).asJava)
+      result.asScala.foreach { result => assertNull(result.exception) }
+
     } finally {
       authorizer.close()
     }
@@ -66,9 +74,9 @@ class DescribeAuthorizedOperationsTest extends IntegrationTestHarness with SaslS
     TestUtils.waitUntilBrokerMetadataIsPropagated(servers)
   }
 
-  private def clusterAcl(userName: String, permissionType: PermissionType, operation: Operation): AuthAcl = {
-    new AuthAcl(new KafkaPrincipal(KafkaPrincipal.USER_TYPE, userName), permissionType,
-      AuthAcl.WildCardHost, operation)
+  private def accessControlEntry(userName: String, permissionType: AclPermissionType, operation: AclOperation): AccessControlEntry = {
+    new AccessControlEntry(new KafkaPrincipal(KafkaPrincipal.USER_TYPE, userName).toString,
+      AuthAcl.WildCardHost, operation, permissionType)
   }
 
   @After
diff --git a/core/src/test/scala/integration/kafka/api/EndToEndAuthorizationTest.scala b/core/src/test/scala/integration/kafka/api/EndToEndAuthorizationTest.scala
index ede80ac31d661..6b4b7c6748cef 100644
--- a/core/src/test/scala/integration/kafka/api/EndToEndAuthorizationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/EndToEndAuthorizationTest.scala
@@ -23,15 +23,19 @@ import java.io.File
 import java.util.concurrent.ExecutionException
 
 import kafka.admin.AclCommand
-import kafka.security.auth._
+import kafka.security.authorizer.AclAuthorizer
+import kafka.security.authorizer.AuthorizerUtils.WildcardHost
 import kafka.server._
 import kafka.utils._
 import org.apache.kafka.clients.consumer.{Consumer, ConsumerConfig, ConsumerRecords}
 import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord}
-import org.apache.kafka.common.security.auth.KafkaPrincipal
+import org.apache.kafka.common.acl._
+import org.apache.kafka.common.acl.AclOperation._
+import org.apache.kafka.common.acl.AclPermissionType._
 import org.apache.kafka.common.{KafkaException, TopicPartition}
 import org.apache.kafka.common.errors.{GroupAuthorizationException, TopicAuthorizationException}
-import org.apache.kafka.common.resource.PatternType
+import org.apache.kafka.common.resource._
+import org.apache.kafka.common.resource.ResourceType._
 import org.apache.kafka.common.resource.PatternType.{LITERAL, PREFIXED}
 import org.junit.Assert._
 import org.junit.{After, Before, Test}
@@ -77,14 +81,17 @@ abstract class EndToEndAuthorizationTest extends IntegrationTestHarness with Sas
   val kafkaPrincipal: String
 
   override protected lazy val trustStoreFile = Some(File.createTempFile("truststore", ".jks"))
-
-  val topicResource = Resource(Topic, topic, LITERAL)
-  val groupResource = Resource(Group, group, LITERAL)
-  val clusterResource = Resource.ClusterResource
-  val prefixedTopicResource = Resource(Topic, topicPrefix, PREFIXED)
-  val prefixedGroupResource = Resource(Group, groupPrefix, PREFIXED)
-  val wildcardTopicResource = Resource(Topic, wildcard, LITERAL)
-  val wildcardGroupResource = Resource(Group, wildcard, LITERAL)
+  protected def authorizerClass: Class[_] = classOf[AclAuthorizer]
+
+  val topicResource = new ResourcePattern(TOPIC, topic, LITERAL)
+  val groupResource =  new ResourcePattern(GROUP, group, LITERAL)
+  val clusterResource = new ResourcePattern(CLUSTER, Resource.CLUSTER_NAME, LITERAL)
+  val prefixedTopicResource =  new ResourcePattern(TOPIC, topicPrefix, PREFIXED)
+  val prefixedGroupResource =  new ResourcePattern(GROUP, groupPrefix, PREFIXED)
+  val wildcardTopicResource =  new ResourcePattern(TOPIC, wildcard, LITERAL)
+  val wildcardGroupResource =  new ResourcePattern(GROUP, wildcard, LITERAL)
+  def kafkaPrincipalStr = s"$kafkaPrincipalType:$kafkaPrincipal"
+  def clientPrincipalStr = s"$kafkaPrincipalType:$clientPrincipal"
 
   // Arguments to AclCommand to set ACLs.
   def clusterActionArgs: Array[String] = Array("--authorizer-properties",
@@ -92,52 +99,52 @@ abstract class EndToEndAuthorizationTest extends IntegrationTestHarness with Sas
                                           s"--add",
                                           s"--cluster",
                                           s"--operation=ClusterAction",
-                                          s"--allow-principal=$kafkaPrincipalType:$kafkaPrincipal")
+                                          s"--allow-principal=$kafkaPrincipalStr")
   def topicBrokerReadAclArgs: Array[String] = Array("--authorizer-properties",
                                           s"zookeeper.connect=$zkConnect",
                                           s"--add",
                                           s"--topic=$wildcard",
                                           s"--operation=Read",
-                                          s"--allow-principal=$kafkaPrincipalType:$kafkaPrincipal")
+                                          s"--allow-principal=$kafkaPrincipalStr")
   def produceAclArgs(topic: String): Array[String] = Array("--authorizer-properties",
                                           s"zookeeper.connect=$zkConnect",
                                           s"--add",
                                           s"--topic=$topic",
                                           s"--producer",
-                                          s"--allow-principal=$kafkaPrincipalType:$clientPrincipal")
+                                          s"--allow-principal=$clientPrincipalStr")
   def describeAclArgs: Array[String] = Array("--authorizer-properties",
                                           s"zookeeper.connect=$zkConnect",
                                           s"--add",
                                           s"--topic=$topic",
                                           s"--operation=Describe",
-                                          s"--allow-principal=$kafkaPrincipalType:$clientPrincipal")
+                                          s"--allow-principal=$clientPrincipalStr")
   def deleteDescribeAclArgs: Array[String] = Array("--authorizer-properties",
                                           s"zookeeper.connect=$zkConnect",
                                           s"--remove",
                                           s"--force",
                                           s"--topic=$topic",
                                           s"--operation=Describe",
-                                          s"--allow-principal=$kafkaPrincipalType:$clientPrincipal")
+                                          s"--allow-principal=$clientPrincipalStr")
   def deleteWriteAclArgs: Array[String] = Array("--authorizer-properties",
                                           s"zookeeper.connect=$zkConnect",
                                           s"--remove",
                                           s"--force",
                                           s"--topic=$topic",
                                           s"--operation=Write",
-                                          s"--allow-principal=$kafkaPrincipalType:$clientPrincipal")
+                                          s"--allow-principal=$clientPrincipalStr")
   def consumeAclArgs(topic: String): Array[String] = Array("--authorizer-properties",
                                           s"zookeeper.connect=$zkConnect",
                                           s"--add",
                                           s"--topic=$topic",
                                           s"--group=$group",
                                           s"--consumer",
-                                          s"--allow-principal=$kafkaPrincipalType:$clientPrincipal")
+                                          s"--allow-principal=$clientPrincipalStr")
   def groupAclArgs: Array[String] = Array("--authorizer-properties",
                                           s"zookeeper.connect=$zkConnect",
                                           s"--add",
                                           s"--group=$group",
                                           s"--operation=Read",
-                                          s"--allow-principal=$kafkaPrincipalType:$clientPrincipal")
+                                          s"--allow-principal=$clientPrincipalStr")
   def produceConsumeWildcardAclArgs: Array[String] = Array("--authorizer-properties",
                                           s"zookeeper.connect=$zkConnect",
                                           s"--add",
@@ -145,7 +152,7 @@ abstract class EndToEndAuthorizationTest extends IntegrationTestHarness with Sas
                                           s"--group=$wildcard",
                                           s"--consumer",
                                           s"--producer",
-                                          s"--allow-principal=$kafkaPrincipalType:$clientPrincipal")
+                                          s"--allow-principal=$clientPrincipalStr")
   def produceConsumePrefixedAclsArgs: Array[String] = Array("--authorizer-properties",
                                           s"zookeeper.connect=$zkConnect",
                                           s"--add",
@@ -154,19 +161,19 @@ abstract class EndToEndAuthorizationTest extends IntegrationTestHarness with Sas
                                           s"--resource-pattern-type=prefixed",
                                           s"--consumer",
                                           s"--producer",
-                                          s"--allow-principal=$kafkaPrincipalType:$clientPrincipal")
-
-  def ClusterActionAcl = Set(new Acl(new KafkaPrincipal(kafkaPrincipalType, kafkaPrincipal), Allow, Acl.WildCardHost, ClusterAction))
-  def TopicBrokerReadAcl = Set(new Acl(new KafkaPrincipal(kafkaPrincipalType, kafkaPrincipal), Allow, Acl.WildCardHost, Read))
-  def GroupReadAcl = Set(new Acl(new KafkaPrincipal(kafkaPrincipalType, clientPrincipal), Allow, Acl.WildCardHost, Read))
-  def TopicReadAcl = Set(new Acl(new KafkaPrincipal(kafkaPrincipalType, clientPrincipal), Allow, Acl.WildCardHost, Read))
-  def TopicWriteAcl = Set(new Acl(new KafkaPrincipal(kafkaPrincipalType, clientPrincipal), Allow, Acl.WildCardHost, Write))
-  def TopicDescribeAcl = Set(new Acl(new KafkaPrincipal(kafkaPrincipalType, clientPrincipal), Allow, Acl.WildCardHost, Describe))
-  def TopicCreateAcl = Set(new Acl(new KafkaPrincipal(kafkaPrincipalType, clientPrincipal), Allow, Acl.WildCardHost, Create))
+                                          s"--allow-principal=$clientPrincipalStr")
+
+  def ClusterActionAcl = Set(new AccessControlEntry(kafkaPrincipalStr, WildcardHost, CLUSTER_ACTION, ALLOW))
+  def TopicBrokerReadAcl = Set(new AccessControlEntry(kafkaPrincipalStr, WildcardHost, READ, ALLOW))
+  def GroupReadAcl = Set(new AccessControlEntry(clientPrincipalStr, WildcardHost, READ, ALLOW))
+  def TopicReadAcl = Set(new AccessControlEntry(clientPrincipalStr, WildcardHost, READ, ALLOW))
+  def TopicWriteAcl = Set(new AccessControlEntry(clientPrincipalStr, WildcardHost, WRITE, ALLOW))
+  def TopicDescribeAcl = Set(new AccessControlEntry(clientPrincipalStr, WildcardHost, DESCRIBE, ALLOW))
+  def TopicCreateAcl = Set(new AccessControlEntry(clientPrincipalStr, WildcardHost, CREATE, ALLOW))
   // The next two configuration parameters enable ZooKeeper secure ACLs
   // and sets the Kafka authorizer, both necessary to enable security.
   this.serverConfig.setProperty(KafkaConfig.ZkEnableSecureAclsProp, "true")
-  this.serverConfig.setProperty(KafkaConfig.AuthorizerClassNameProp, classOf[SimpleAclAuthorizer].getName)
+  this.serverConfig.setProperty(KafkaConfig.AuthorizerClassNameProp, authorizerClass.getName)
   // Some needed configuration for brokers, producers, and consumers
   this.serverConfig.setProperty(KafkaConfig.OffsetsTopicPartitionsProp, "1")
   this.serverConfig.setProperty(KafkaConfig.OffsetsTopicReplicationFactorProp, "3")
@@ -182,8 +189,8 @@ abstract class EndToEndAuthorizationTest extends IntegrationTestHarness with Sas
   override def setUp(): Unit = {
     super.setUp()
     servers.foreach { s =>
-      TestUtils.waitAndVerifyAcls(ClusterActionAcl, s.dataPlaneRequestProcessor.authorizer.get, Resource.ClusterResource)
-      TestUtils.waitAndVerifyAcls(TopicBrokerReadAcl, s.dataPlaneRequestProcessor.authorizer.get, Resource(Topic, "*", LITERAL))
+      TestUtils.waitAndVerifyAcls(ClusterActionAcl, s.dataPlaneRequestProcessor.authorizer.get, clusterResource)
+      TestUtils.waitAndVerifyAcls(TopicBrokerReadAcl, s.dataPlaneRequestProcessor.authorizer.get, new ResourcePattern(TOPIC, "*", LITERAL))
     }
     // create the test topic with all the brokers as replicas
     createTopic(topic, 1, 3)
@@ -293,7 +300,7 @@ abstract class EndToEndAuthorizationTest extends IntegrationTestHarness with Sas
     AclCommand.main(consumeAclArgs(tp.topic))
     servers.foreach { s =>
       TestUtils.waitAndVerifyAcls(TopicReadAcl ++ TopicWriteAcl ++ TopicDescribeAcl ++ TopicCreateAcl, s.dataPlaneRequestProcessor.authorizer.get,
-        new Resource(Topic, tp.topic, PatternType.LITERAL))
+        new ResourcePattern(TOPIC, tp.topic, LITERAL))
       TestUtils.waitAndVerifyAcls(GroupReadAcl, s.dataPlaneRequestProcessor.authorizer.get, groupResource)
     }
   }
diff --git a/core/src/test/scala/integration/kafka/api/SaslClientsWithInvalidCredentialsTest.scala b/core/src/test/scala/integration/kafka/api/SaslClientsWithInvalidCredentialsTest.scala
index 0155a739335ac..83d32fa27337b 100644
--- a/core/src/test/scala/integration/kafka/api/SaslClientsWithInvalidCredentialsTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SaslClientsWithInvalidCredentialsTest.scala
@@ -18,7 +18,7 @@ import java.util.Collections
 import java.util.concurrent.{ExecutionException, TimeUnit}
 
 import scala.collection.JavaConverters._
-import org.apache.kafka.clients.admin.{Admin, AdminClient, AdminClientConfig}
+import org.apache.kafka.clients.admin.{AdminClient, AdminClientConfig}
 import org.apache.kafka.clients.consumer.{ConsumerConfig, KafkaConsumer}
 import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig, ProducerRecord}
 import org.apache.kafka.common.{KafkaException, TopicPartition}
diff --git a/core/src/test/scala/integration/kafka/api/SaslGssapiSslEndToEndAuthorizationTest.scala b/core/src/test/scala/integration/kafka/api/SaslGssapiSslEndToEndAuthorizationTest.scala
index a630293bb6625..1142bf7a6a029 100644
--- a/core/src/test/scala/integration/kafka/api/SaslGssapiSslEndToEndAuthorizationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SaslGssapiSslEndToEndAuthorizationTest.scala
@@ -16,6 +16,7 @@
   */
 package kafka.api
 
+import kafka.security.auth.SimpleAclAuthorizer
 import kafka.server.KafkaConfig
 import kafka.utils.JaasTestUtils
 
@@ -27,6 +28,7 @@ class SaslGssapiSslEndToEndAuthorizationTest extends SaslEndToEndAuthorizationTe
 
   override protected def kafkaClientSaslMechanism = "GSSAPI"
   override protected def kafkaServerSaslMechanisms = List("GSSAPI")
+  override protected def authorizerClass = classOf[SimpleAclAuthorizer]
 
   // Configure brokers to require SSL client authentication in order to verify that SASL_SSL works correctly even if the
   // client doesn't have a keystore. We want to cover the scenario where a broker requires either SSL client
diff --git a/core/src/test/scala/integration/kafka/api/SaslSslAdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/SaslSslAdminClientIntegrationTest.scala
index a44bb33c7efe5..cad33e07a6100 100644
--- a/core/src/test/scala/integration/kafka/api/SaslSslAdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SaslSslAdminClientIntegrationTest.scala
@@ -16,6 +16,7 @@ import java.io.File
 import java.util
 
 import kafka.security.auth.{All, Allow, Alter, AlterConfigs, Authorizer, ClusterAction, Create, Delete, Deny, Describe, Group, Operation, PermissionType, SimpleAclAuthorizer, Topic, Acl => AuthAcl, Resource => AuthResource}
+import kafka.security.authorizer.AuthorizerWrapper
 import kafka.server.KafkaConfig
 import kafka.utils.{CoreUtils, JaasTestUtils, TestUtils}
 import kafka.utils.TestUtils._
@@ -32,6 +33,7 @@ import scala.util.{Failure, Success, Try}
 
 class SaslSslAdminClientIntegrationTest extends AdminClientIntegrationTest with SaslSetup {
   this.serverConfig.setProperty(KafkaConfig.ZkEnableSecureAclsProp, "true")
+  // This tests the old SimpleAclAuthorizer, we have another test for the new AclAuthorizer
   this.serverConfig.setProperty(KafkaConfig.AuthorizerClassNameProp, classOf[SimpleAclAuthorizer].getName)
 
   override protected def securityProtocol = SecurityProtocol.SASL_SSL
@@ -59,26 +61,30 @@ class SaslSslAdminClientIntegrationTest extends AdminClientIntegrationTest with
 
   @Before
   override def setUp(): Unit = {
-    startSasl(jaasSections(Seq("GSSAPI"), Some("GSSAPI"), Both, JaasTestUtils.KafkaServerContextName))
+    setUpSasl()
     super.setUp()
   }
 
+  def setUpSasl(): Unit = {
+    startSasl(jaasSections(Seq("GSSAPI"), Some("GSSAPI"), Both, JaasTestUtils.KafkaServerContextName))
+  }
+
   private def clusterAcl(permissionType: PermissionType, operation: Operation): AuthAcl = {
     new AuthAcl(new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "*"), permissionType,
       AuthAcl.WildCardHost, operation)
   }
 
-  private def addClusterAcl(permissionType: PermissionType, operation: Operation): Unit = {
+  def addClusterAcl(permissionType: PermissionType, operation: Operation): Unit = {
     val acls = Set(clusterAcl(permissionType, operation))
-    val authorizer = servers.head.dataPlaneRequestProcessor.authorizer.get
+    val authorizer = simpleAclAuthorizer
     val prevAcls = authorizer.getAcls(AuthResource.ClusterResource)
     authorizer.addAcls(acls, AuthResource.ClusterResource)
     TestUtils.waitAndVerifyAcls(prevAcls ++ acls, authorizer, AuthResource.ClusterResource)
   }
 
-  private def removeClusterAcl(permissionType: PermissionType, operation: Operation): Unit = {
+  def removeClusterAcl(permissionType: PermissionType, operation: Operation): Unit = {
     val acls = Set(clusterAcl(permissionType, operation))
-    val authorizer = servers.head.dataPlaneRequestProcessor.authorizer.get
+    val authorizer = simpleAclAuthorizer
     val prevAcls = authorizer.getAcls(AuthResource.ClusterResource)
     Assert.assertTrue(authorizer.removeAcls(acls, AuthResource.ClusterResource))
     TestUtils.waitAndVerifyAcls(prevAcls -- acls, authorizer, AuthResource.ClusterResource)
@@ -412,4 +418,9 @@ class SaslSslAdminClientIntegrationTest extends AdminClientIntegrationTest with
   private def getAcls(allTopicAcls: AclBindingFilter) = {
     client.describeAcls(allTopicAcls).values.get().asScala.toSet
   }
+
+  private def simpleAclAuthorizer: Authorizer = {
+    val authorizerWrapper = servers.head.dataPlaneRequestProcessor.authorizer.get.asInstanceOf[AuthorizerWrapper]
+    authorizerWrapper.baseAuthorizer
+  }
 }
diff --git a/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala
new file mode 100644
index 0000000000000..4ad2e5329f00f
--- /dev/null
+++ b/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala
@@ -0,0 +1,90 @@
+/**
+  * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE
+  * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file
+  * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
+  * License. You may obtain a copy of the License at
+  *
+  * http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+  * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+  * specific language governing permissions and limitations under the License.
+  */
+package kafka.api
+
+import java.io.File
+import java.util.Collections
+
+import kafka.security.authorizer.AclAuthorizer
+import kafka.security.authorizer.AuthorizerUtils.{WildcardHost, WildcardPrincipal}
+import kafka.security.auth.{Operation, PermissionType}
+import kafka.server.KafkaConfig
+import kafka.utils.{CoreUtils, TestUtils}
+import org.apache.kafka.common.acl._
+import org.apache.kafka.common.acl.AclOperation._
+import org.apache.kafka.common.acl.AclPermissionType._
+import org.apache.kafka.common.resource.ResourcePattern
+import org.apache.kafka.common.resource.PatternType._
+import org.apache.kafka.common.resource.ResourceType._
+import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
+import org.apache.kafka.server.authorizer._
+import org.junit.Assert
+
+import scala.collection.JavaConverters._
+
+class SslAdminClientIntegrationTest extends SaslSslAdminClientIntegrationTest {
+  this.serverConfig.setProperty(KafkaConfig.ZkEnableSecureAclsProp, "true")
+  this.serverConfig.setProperty(KafkaConfig.AuthorizerClassNameProp, classOf[AclAuthorizer].getName)
+
+  override protected def securityProtocol = SecurityProtocol.SSL
+  override protected lazy val trustStoreFile = Some(File.createTempFile("truststore", ".jks"))
+
+  override def configureSecurityBeforeServersStart(): Unit = {
+    val authorizer = CoreUtils.createObject[Authorizer](classOf[AclAuthorizer].getName)
+    try {
+      authorizer.configure(this.configs.head.originals())
+      val ace = new AccessControlEntry(WildcardPrincipal, WildcardHost, ALL, ALLOW)
+      authorizer.createAcls(null, List(new AclBinding(new ResourcePattern(TOPIC, "*", LITERAL), ace)).asJava)
+      authorizer.createAcls(null, List(new AclBinding(new ResourcePattern(GROUP, "*", LITERAL), ace)).asJava)
+
+      authorizer.createAcls(null, List(clusterAcl(ALLOW, CREATE),
+                             clusterAcl(ALLOW, DELETE),
+                             clusterAcl(ALLOW, CLUSTER_ACTION),
+                             clusterAcl(ALLOW, ALTER_CONFIGS),
+                             clusterAcl(ALLOW, ALTER))
+        .map(ace => new AclBinding(clusterResourcePattern, ace)).asJava)
+    } finally {
+      authorizer.close()
+    }
+  }
+
+  override def setUpSasl(): Unit = {
+    startSasl(jaasSections(List.empty, None, ZkSasl))
+  }
+
+  override def addClusterAcl(permissionType: PermissionType, operation: Operation): Unit = {
+    val ace = clusterAcl(permissionType.toJava, operation.toJava)
+    val aclBinding = new AclBinding(clusterResourcePattern, ace)
+    val authorizer = servers.head.dataPlaneRequestProcessor.authorizer.get
+    val prevAcls = authorizer.acls(new AclBindingFilter(clusterResourcePattern.toFilter, AccessControlEntryFilter.ANY))
+      .asScala.map(_.entry).toSet
+    authorizer.createAcls(null, Collections.singletonList(aclBinding))
+    TestUtils.waitAndVerifyAcls(prevAcls ++ Set(ace), authorizer, clusterResourcePattern)
+  }
+
+  override def removeClusterAcl(permissionType: PermissionType, operation: Operation): Unit = {
+    val ace = clusterAcl(permissionType.toJava, operation.toJava)
+    val authorizer = servers.head.dataPlaneRequestProcessor.authorizer.get
+    val clusterFilter = new AclBindingFilter(clusterResourcePattern.toFilter, AccessControlEntryFilter.ANY)
+    val prevAcls = authorizer.acls(clusterFilter).asScala.map(_.entry).toSet
+    val deleteFilter = new AclBindingFilter(clusterResourcePattern.toFilter, ace.toFilter)
+    Assert.assertTrue(authorizer.deleteAcls(null, Collections.singletonList(deleteFilter))
+      .get(0).aclBindingDeleteResults().asScala.head.deleted)
+    TestUtils.waitAndVerifyAcls(prevAcls -- Set(ace), authorizer, clusterResourcePattern)
+  }
+
+  private def clusterAcl(permissionType: AclPermissionType, operation: AclOperation): AccessControlEntry = {
+    new AccessControlEntry(new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "*").toString,
+      WildcardHost, operation, permissionType)
+  }
+}
diff --git a/core/src/test/scala/unit/kafka/admin/AclCommandTest.scala b/core/src/test/scala/unit/kafka/admin/AclCommandTest.scala
index 024de8dad4c8b..a2b68a6b57f16 100644
--- a/core/src/test/scala/unit/kafka/admin/AclCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/AclCommandTest.scala
@@ -19,16 +19,21 @@ package kafka.admin
 import java.util.Properties
 
 import kafka.admin.AclCommand.AclCommandOptions
-import kafka.security.auth._
+import kafka.security.auth.Authorizer
+import kafka.security.authorizer.{AclAuthorizer, AuthorizerUtils}
 import kafka.server.{KafkaConfig, KafkaServer}
 import kafka.utils.{Exit, Logging, TestUtils}
 import kafka.zk.ZooKeeperTestHarness
-import org.apache.kafka.common.resource.PatternType
+import org.apache.kafka.common.acl.{AccessControlEntry, AclOperation, AclPermissionType}
+import org.apache.kafka.common.acl.AclOperation._
+import org.apache.kafka.common.acl.AclPermissionType._
+import org.apache.kafka.common.resource.{PatternType, Resource, ResourcePattern}
+import org.apache.kafka.common.resource.ResourceType._
 import org.apache.kafka.common.network.ListenerName
-
 import org.apache.kafka.common.resource.PatternType.{LITERAL, PREFIXED}
 import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
 import org.apache.kafka.common.utils.SecurityUtils
+import org.apache.kafka.server.authorizer.{Authorizer => JAuthorizer}
 import org.junit.{After, Before, Test}
 import org.scalatest.Assertions.intercept
 
@@ -43,51 +48,52 @@ class AclCommandTest extends ZooKeeperTestHarness with Logging {
   private val AllowHostCommand = Array("--allow-host", "host1", "--allow-host", "host2")
   private val DenyHostCommand = Array("--deny-host", "host1", "--deny-host", "host2")
 
-  private val TopicResources = Set(Resource(Topic, "test-1", LITERAL), Resource(Topic, "test-2", LITERAL))
-  private val GroupResources = Set(Resource(Group, "testGroup-1", LITERAL), Resource(Group, "testGroup-2", LITERAL))
-  private val TransactionalIdResources = Set(Resource(TransactionalId, "t0", LITERAL), Resource(TransactionalId, "t1", LITERAL))
-  private val TokenResources = Set(Resource(DelegationToken, "token1", LITERAL), Resource(DelegationToken, "token2", LITERAL))
+  private val ClusterResource = new ResourcePattern(CLUSTER, Resource.CLUSTER_NAME, LITERAL)
+  private val TopicResources = Set(new ResourcePattern(TOPIC, "test-1", LITERAL), new ResourcePattern(TOPIC, "test-2", LITERAL))
+  private val GroupResources = Set(new ResourcePattern(GROUP, "testGroup-1", LITERAL), new ResourcePattern(GROUP, "testGroup-2", LITERAL))
+  private val TransactionalIdResources = Set(new ResourcePattern(TRANSACTIONAL_ID, "t0", LITERAL), new ResourcePattern(TRANSACTIONAL_ID, "t1", LITERAL))
+  private val TokenResources = Set(new ResourcePattern(DELEGATION_TOKEN, "token1", LITERAL), new ResourcePattern(DELEGATION_TOKEN, "token2", LITERAL))
 
-  private val ResourceToCommand = Map[Set[Resource], Array[String]](
+  private val ResourceToCommand = Map[Set[ResourcePattern], Array[String]](
     TopicResources -> Array("--topic", "test-1", "--topic", "test-2"),
-    Set(Resource.ClusterResource) -> Array("--cluster"),
+    Set(ClusterResource) -> Array("--cluster"),
     GroupResources -> Array("--group", "testGroup-1", "--group", "testGroup-2"),
     TransactionalIdResources -> Array("--transactional-id", "t0", "--transactional-id", "t1"),
     TokenResources -> Array("--delegation-token", "token1", "--delegation-token", "token2")
   )
 
-  private val ResourceToOperations = Map[Set[Resource], (Set[Operation], Array[String])](
-    TopicResources -> (Set(Read, Write, Create, Describe, Delete, DescribeConfigs, AlterConfigs, Alter),
-      Array("--operation", "Read" , "--operation", "Write", "--operation", "Create", "--operation", "Describe", "--operation", "Delete",
+  private val ResourceToOperations = Map[Set[ResourcePattern], (Set[AclOperation], Array[String])](
+    TopicResources -> (Set(READ, WRITE, CREATE, DESCRIBE, DELETE, DESCRIBE_CONFIGS, ALTER_CONFIGS, ALTER),
+      Array("--operation", "Read" , "--operation", "WRITE", "--operation", "Create", "--operation", "Describe", "--operation", "Delete",
         "--operation", "DescribeConfigs", "--operation", "AlterConfigs", "--operation", "Alter")),
-    Set(Resource.ClusterResource) -> (Set(Create, ClusterAction, DescribeConfigs, AlterConfigs, IdempotentWrite, Alter, Describe),
+    Set(ClusterResource) -> (Set(CREATE, CLUSTER_ACTION, DESCRIBE_CONFIGS, ALTER_CONFIGS, IDEMPOTENT_WRITE, ALTER, DESCRIBE),
       Array("--operation", "Create", "--operation", "ClusterAction", "--operation", "DescribeConfigs",
         "--operation", "AlterConfigs", "--operation", "IdempotentWrite", "--operation", "Alter", "--operation", "Describe")),
-    GroupResources -> (Set(Read, Describe, Delete), Array("--operation", "Read", "--operation", "Describe", "--operation", "Delete")),
-    TransactionalIdResources -> (Set(Describe, Write), Array("--operation", "Describe", "--operation", "Write")),
-    TokenResources -> (Set(Describe), Array("--operation", "Describe"))
+    GroupResources -> (Set(READ, DESCRIBE, DELETE), Array("--operation", "Read", "--operation", "Describe", "--operation", "Delete")),
+    TransactionalIdResources -> (Set(DESCRIBE, WRITE), Array("--operation", "Describe", "--operation", "WRITE")),
+    TokenResources -> (Set(DESCRIBE), Array("--operation", "Describe"))
   )
 
-  private def ProducerResourceToAcls(enableIdempotence: Boolean = false) = Map[Set[Resource], Set[Acl]](
-    TopicResources -> AclCommand.getAcls(Users, Allow, Set(Write, Describe, Create), Hosts),
-    TransactionalIdResources -> AclCommand.getAcls(Users, Allow, Set(Write, Describe), Hosts),
-    Set(Resource.ClusterResource) -> AclCommand.getAcls(Users, Allow,
-      Set(if (enableIdempotence) Some(IdempotentWrite) else None).flatten, Hosts)
+  private def ProducerResourceToAcls(enableIdempotence: Boolean = false) = Map[Set[ResourcePattern], Set[AccessControlEntry]](
+    TopicResources -> AclCommand.getAcls(Users, ALLOW, Set(WRITE, DESCRIBE, CREATE), Hosts),
+    TransactionalIdResources -> AclCommand.getAcls(Users, ALLOW, Set(WRITE, DESCRIBE), Hosts),
+    Set(ClusterResource) -> AclCommand.getAcls(Users, ALLOW,
+      Set(if (enableIdempotence) Some(IDEMPOTENT_WRITE) else None).flatten, Hosts)
   )
 
-  private val ConsumerResourceToAcls = Map[Set[Resource], Set[Acl]](
-    TopicResources -> AclCommand.getAcls(Users, Allow, Set(Read, Describe), Hosts),
-    GroupResources -> AclCommand.getAcls(Users, Allow, Set(Read), Hosts)
+  private val ConsumerResourceToAcls = Map[Set[ResourcePattern], Set[AccessControlEntry]](
+    TopicResources -> AclCommand.getAcls(Users, ALLOW, Set(READ, DESCRIBE), Hosts),
+    GroupResources -> AclCommand.getAcls(Users, ALLOW, Set(READ), Hosts)
   )
 
-  private val CmdToResourcesToAcl = Map[Array[String], Map[Set[Resource], Set[Acl]]](
+  private val CmdToResourcesToAcl = Map[Array[String], Map[Set[ResourcePattern], Set[AccessControlEntry]]](
     Array[String]("--producer") -> ProducerResourceToAcls(),
     Array[String]("--producer", "--idempotent") -> ProducerResourceToAcls(enableIdempotence = true),
     Array[String]("--consumer") -> ConsumerResourceToAcls,
     Array[String]("--producer", "--consumer") -> ConsumerResourceToAcls.map { case (k, v) => k -> (v ++
-      ProducerResourceToAcls().getOrElse(k, Set.empty[Acl])) },
+      ProducerResourceToAcls().getOrElse(k, Set.empty[AccessControlEntry])) },
     Array[String]("--producer", "--idempotent", "--consumer") -> ConsumerResourceToAcls.map { case (k, v) => k -> (v ++
-      ProducerResourceToAcls(enableIdempotence = true).getOrElse(k, Set.empty[Acl])) }
+      ProducerResourceToAcls(enableIdempotence = true).getOrElse(k, Set.empty[AccessControlEntry])) }
   )
 
   private var brokerProps: Properties = _
@@ -99,8 +105,8 @@ class AclCommandTest extends ZooKeeperTestHarness with Logging {
     super.setUp()
 
     brokerProps = TestUtils.createBrokerConfig(0, zkConnect)
-    brokerProps.put(KafkaConfig.AuthorizerClassNameProp, "kafka.security.auth.SimpleAclAuthorizer")
-    brokerProps.put(SimpleAclAuthorizer.SuperUsersProp, "User:ANONYMOUS")
+    brokerProps.put(KafkaConfig.AuthorizerClassNameProp, classOf[AclAuthorizer].getName)
+    brokerProps.put(AclAuthorizer.SuperUsersProp, "User:ANONYMOUS")
 
     zkArgs = Array("--authorizer-properties", "zookeeper.connect=" + zkConnect)
   }
@@ -130,7 +136,7 @@ class AclCommandTest extends ZooKeeperTestHarness with Logging {
 
   private def testAclCli(cmdArgs: Array[String]): Unit = {
     for ((resources, resourceCmd) <- ResourceToCommand) {
-      for (permissionType <- PermissionType.values) {
+      for (permissionType <- Set(ALLOW, DENY)) {
         val operationToCmd = ResourceToOperations(resources)
         val (acls, cmd) = getAclToCommand(permissionType, operationToCmd._1)
           AclCommand.main(cmdArgs ++ cmd ++ resourceCmd ++ operationToCmd._2 :+ "--add")
@@ -159,7 +165,7 @@ class AclCommandTest extends ZooKeeperTestHarness with Logging {
   private def testProducerConsumerCli(cmdArgs: Array[String]): Unit = {
     for ((cmd, resourcesToAcls) <- CmdToResourcesToAcl) {
       val resourceCommand: Array[String] = resourcesToAcls.keys.map(ResourceToCommand).foldLeft(Array[String]())(_ ++ _)
-      AclCommand.main(cmdArgs ++ getCmd(Allow) ++ resourceCommand ++ cmd :+ "--add")
+      AclCommand.main(cmdArgs ++ getCmd(ALLOW) ++ resourceCommand ++ cmd :+ "--add")
       for ((resources, acls) <- resourcesToAcls) {
         for (resource <- resources) {
           withAuthorizer() { authorizer =>
@@ -188,24 +194,32 @@ class AclCommandTest extends ZooKeeperTestHarness with Logging {
     AclCommand.main(cmdArgs ++ cmd :+ "--add")
 
     withAuthorizer() { authorizer =>
-      val writeAcl = Acl(principal, Allow, Acl.WildCardHost, Write)
-      val describeAcl = Acl(principal, Allow, Acl.WildCardHost, Describe)
-      val createAcl = Acl(principal, Allow, Acl.WildCardHost, Create)
-      TestUtils.waitAndVerifyAcls(Set(writeAcl, describeAcl, createAcl), authorizer, Resource(Topic, "Test-", PREFIXED))
+      val writeAcl = new AccessControlEntry(principal.toString, AuthorizerUtils.WildcardHost, WRITE, ALLOW)
+      val describeAcl = new AccessControlEntry(principal.toString, AuthorizerUtils.WildcardHost, DESCRIBE, ALLOW)
+      val createAcl = new AccessControlEntry(principal.toString, AuthorizerUtils.WildcardHost, CREATE, ALLOW)
+      TestUtils.waitAndVerifyAcls(Set(writeAcl, describeAcl, createAcl), authorizer,
+        new ResourcePattern(TOPIC, "Test-", PREFIXED))
     }
 
     AclCommand.main(cmdArgs ++ cmd :+ "--remove" :+ "--force")
 
     withAuthorizer() { authorizer =>
-      TestUtils.waitAndVerifyAcls(Set.empty[Acl], authorizer, Resource(Cluster, "kafka-cluster", LITERAL))
-      TestUtils.waitAndVerifyAcls(Set.empty[Acl], authorizer, Resource(Topic, "Test-", PREFIXED))
+      TestUtils.waitAndVerifyAcls(Set.empty[AccessControlEntry], authorizer, new ResourcePattern(CLUSTER, "kafka-cluster", LITERAL))
+      TestUtils.waitAndVerifyAcls(Set.empty[AccessControlEntry], authorizer, new ResourcePattern(TOPIC, "Test-", PREFIXED))
     }
   }
 
   @Test(expected = classOf[IllegalArgumentException])
   def testInvalidAuthorizerProperty(): Unit = {
     val args = Array("--authorizer-properties", "zookeeper.connect " + zkConnect)
-    val aclCommandService = new AclCommand.AuthorizerService(new AclCommandOptions(args))
+    val aclCommandService = new AclCommand.AuthorizerService(classOf[Authorizer], new AclCommandOptions(args))
+    aclCommandService.listAcls()
+  }
+
+  @Test(expected = classOf[IllegalArgumentException])
+  def testInvalidJAuthorizerProperty() {
+    val args = Array("--authorizer-properties", "zookeeper.connect " + zkConnect)
+    val aclCommandService = new AclCommand.JAuthorizerService(classOf[JAuthorizer], new AclCommandOptions(args))
     aclCommandService.listAcls()
   }
 
@@ -232,35 +246,36 @@ class AclCommandTest extends ZooKeeperTestHarness with Logging {
         verifyPatternType(listCmd, isValid = patternType != PatternType.UNKNOWN)
         val removeCmd = zkArgs ++ Array("--topic", "Test", "--force", "--remove", "--resource-pattern-type", patternType.toString)
         verifyPatternType(removeCmd, isValid = patternType != PatternType.UNKNOWN)
+
       }
     } finally {
       Exit.resetExitProcedure()
     }
   }
 
-  private def testRemove(cmdArgs: Array[String], resources: Set[Resource], resourceCmd: Array[String]): Unit = {
+  private def testRemove(cmdArgs: Array[String], resources: Set[ResourcePattern], resourceCmd: Array[String]): Unit = {
     for (resource <- resources) {
       AclCommand.main(cmdArgs ++ resourceCmd :+ "--remove" :+ "--force")
       withAuthorizer() { authorizer =>
-        TestUtils.waitAndVerifyAcls(Set.empty[Acl], authorizer, resource)
+        TestUtils.waitAndVerifyAcls(Set.empty[AccessControlEntry], authorizer, resource)
       }
     }
   }
 
-  private def getAclToCommand(permissionType: PermissionType, operations: Set[Operation]): (Set[Acl], Array[String]) = {
+  private def getAclToCommand(permissionType: AclPermissionType, operations: Set[AclOperation]): (Set[AccessControlEntry], Array[String]) = {
     (AclCommand.getAcls(Users, permissionType, operations, Hosts), getCmd(permissionType))
   }
 
-  private def getCmd(permissionType: PermissionType): Array[String] = {
-    val principalCmd = if (permissionType == Allow) "--allow-principal" else "--deny-principal"
-    val cmd = if (permissionType == Allow) AllowHostCommand else DenyHostCommand
+  private def getCmd(permissionType: AclPermissionType): Array[String] = {
+    val principalCmd = if (permissionType == ALLOW) "--allow-principal" else "--deny-principal"
+    val cmd = if (permissionType == ALLOW) AllowHostCommand else DenyHostCommand
 
     Users.foldLeft(cmd) ((cmd, user) => cmd ++ Array(principalCmd, user.toString))
   }
 
-  private def withAuthorizer()(f: Authorizer => Unit): Unit = {
+  private def withAuthorizer()(f: JAuthorizer => Unit): Unit = {
     val kafkaConfig = KafkaConfig.fromProps(brokerProps, doLog = false)
-    val authZ = new SimpleAclAuthorizer
+    val authZ = new AclAuthorizer
     try {
       authZ.configure(kafkaConfig.originals)
       f(authZ)
diff --git a/core/src/test/scala/unit/kafka/admin/LeaderElectionCommandTest.scala b/core/src/test/scala/unit/kafka/admin/LeaderElectionCommandTest.scala
index 8418b4b2caaa3..c30011fbb5cd2 100644
--- a/core/src/test/scala/unit/kafka/admin/LeaderElectionCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/LeaderElectionCommandTest.scala
@@ -26,7 +26,7 @@ import kafka.server.KafkaConfig
 import kafka.server.KafkaServer
 import kafka.utils.TestUtils
 import kafka.zk.ZooKeeperTestHarness
-import org.apache.kafka.clients.admin.{Admin, AdminClient, AdminClientConfig}
+import org.apache.kafka.clients.admin.{AdminClient, AdminClientConfig}
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.TimeoutException
 import org.apache.kafka.common.errors.UnknownTopicOrPartitionException
diff --git a/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala b/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala
index 09fe4ccaa5186..295fc3bd87fd2 100644
--- a/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala
@@ -19,26 +19,30 @@ package kafka.admin
 import java.io.File
 import java.nio.charset.StandardCharsets
 import java.nio.file.{Files, Paths}
+import java.util
 import java.util.Properties
 
 import scala.collection.Seq
-
 import kafka.common.AdminCommandFailedException
-import kafka.network.RequestChannel
-import kafka.security.auth._
+import kafka.security.authorizer.AclAuthorizer
 import kafka.server.{KafkaConfig, KafkaServer}
 import kafka.utils.{Logging, TestUtils}
 import kafka.zk.ZooKeeperTestHarness
 import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.acl.AclOperation
 import org.apache.kafka.common.errors.ClusterAuthorizationException
 import org.apache.kafka.common.errors.PreferredLeaderNotAvailableException
 import org.apache.kafka.common.errors.TimeoutException
 import org.apache.kafka.common.errors.UnknownTopicOrPartitionException
 import org.apache.kafka.common.network.ListenerName
+import org.apache.kafka.common.resource.ResourceType
+import org.apache.kafka.server.authorizer.{Action, AuthorizableRequestContext, AuthorizationResult}
 import org.apache.kafka.test
 import org.junit.Assert._
 import org.junit.{After, Test}
 
+import scala.collection.JavaConverters._
+
 class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness with Logging {
   var servers: Seq[KafkaServer] = Seq()
 
@@ -375,7 +379,13 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
   }
 }
 
-class PreferredReplicaLeaderElectionCommandTestAuthorizer extends SimpleAclAuthorizer {
-  override def authorize(session: RequestChannel.Session, operation: Operation, resource: Resource): Boolean =
-    operation != Alter || resource.resourceType != Cluster
+class PreferredReplicaLeaderElectionCommandTestAuthorizer extends AclAuthorizer {
+  override def authorize(requestContext: AuthorizableRequestContext, actions: util.List[Action]): util.List[AuthorizationResult] = {
+    actions.asScala.map { action =>
+      if (action.operation != AclOperation.ALTER || action.resourcePattern.resourceType != ResourceType.CLUSTER)
+        AuthorizationResult.ALLOWED
+      else
+        AuthorizationResult.DENIED
+    }.asJava
+  }
 }
diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
index d235c62a09629..7072c94f7653c 100644
--- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
+++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
@@ -21,6 +21,7 @@ import java.io._
 import java.net._
 import java.nio.ByteBuffer
 import java.nio.channels.SocketChannel
+import java.util.concurrent.{CompletableFuture, Executors}
 import java.util.{HashMap, Properties, Random}
 
 import com.yammer.metrics.core.{Gauge, Meter}
@@ -30,8 +31,8 @@ import javax.net.ssl._
 import kafka.security.CredentialProvider
 import kafka.server.{KafkaConfig, ThrottledChannel}
 import kafka.utils.Implicits._
-import kafka.utils.TestUtils
-import org.apache.kafka.common.TopicPartition
+import kafka.utils.{CoreUtils, TestUtils}
+import org.apache.kafka.common.{Endpoint, TopicPartition}
 import org.apache.kafka.common.memory.MemoryPool
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.network.KafkaChannel.ChannelMuteState
@@ -190,6 +191,52 @@ class SocketServerTest {
     verifyAcceptorBlockedPercent("PLAINTEXT", expectBlocked = false)
   }
 
+  @Test
+  def testStagedListenerStartup(): Unit = {
+    val testProps = new Properties
+    testProps ++= props
+    testProps.put("listeners", "EXTERNAL://localhost:0,INTERNAL://localhost:0,CONTROLLER://localhost:0")
+    testProps.put("listener.security.protocol.map", "EXTERNAL:PLAINTEXT,INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT")
+    testProps.put("control.plane.listener.name", "CONTROLLER")
+    testProps.put("inter.broker.listener.name", "INTERNAL")
+    val config = KafkaConfig.fromProps(testProps)
+    val testableServer = new TestableSocketServer(config)
+    testableServer.startup(startupProcessors = false)
+    val externalReadyFuture = new CompletableFuture[Void]()
+    val executor = Executors.newSingleThreadExecutor()
+
+    def listenerStarted(listenerName: ListenerName) = {
+      try {
+        val socket = connect(testableServer, listenerName, localAddr = InetAddress.getLocalHost)
+        sendAndReceiveRequest(socket, testableServer)
+        true
+      } catch {
+        case _: Throwable => false
+      }
+    }
+
+    try {
+      testableServer.startControlPlaneProcessor()
+      val socket1 = connect(testableServer, config.controlPlaneListenerName.get, localAddr = InetAddress.getLocalHost)
+      sendAndReceiveControllerRequest(socket1, testableServer)
+
+      val externalListener = new ListenerName("EXTERNAL")
+      val externalEndpoint = new Endpoint(externalListener.value, SecurityProtocol.PLAINTEXT, "localhost", 0)
+      val futures =  Map(externalEndpoint -> externalReadyFuture)
+      val startFuture = executor.submit(CoreUtils.runnable(testableServer.startDataPlaneProcessors(futures)))
+      TestUtils.waitUntilTrue(() => listenerStarted(config.interBrokerListenerName), "Inter-broker listener not started")
+      assertFalse("Socket server startup did not wait for future to complete", startFuture.isDone)
+
+      assertFalse(listenerStarted(externalListener))
+
+      externalReadyFuture.complete(null)
+      TestUtils.waitUntilTrue(() => listenerStarted(externalListener), "External listener not started")
+    } finally {
+      executor.shutdownNow()
+      shutdownServerAndMetrics(testableServer)
+    }
+  }
+
   @Test
   def testControlPlaneRequest(): Unit = {
     val testProps = new Properties
diff --git a/core/src/test/scala/unit/kafka/security/auth/SimpleAclAuthorizerTest.scala b/core/src/test/scala/unit/kafka/security/auth/SimpleAclAuthorizerTest.scala
index 57d701ff49704..d5169646e1616 100644
--- a/core/src/test/scala/unit/kafka/security/auth/SimpleAclAuthorizerTest.scala
+++ b/core/src/test/scala/unit/kafka/security/auth/SimpleAclAuthorizerTest.scala
@@ -19,13 +19,12 @@ package kafka.security.auth
 import java.net.InetAddress
 import java.nio.charset.StandardCharsets.UTF_8
 import java.util.UUID
-import java.util.concurrent.{Executors, Semaphore, TimeUnit}
 
 import kafka.api.{ApiVersion, KAFKA_2_0_IV0, KAFKA_2_0_IV1}
 import kafka.network.RequestChannel.Session
 import kafka.security.auth.Acl.{WildCardHost, WildCardResource}
 import kafka.server.KafkaConfig
-import kafka.utils.{CoreUtils, TestUtils}
+import kafka.utils.TestUtils
 import kafka.zk.{ZkAclStore, ZooKeeperTestHarness}
 import kafka.zookeeper.{GetChildrenRequest, GetDataRequest, ZooKeeperClient}
 import org.apache.kafka.common.errors.UnsupportedVersionException
@@ -36,6 +35,7 @@ import org.apache.kafka.common.utils.Time
 import org.junit.Assert._
 import org.junit.{After, Before, Test}
 
+@deprecated("Use AclAuthorizer", "Since 2.4")
 class SimpleAclAuthorizerTest extends ZooKeeperTestHarness {
 
   private val allowReadAcl = Acl(Acl.WildCardPrincipal, Allow, WildCardHost, Read)
@@ -88,7 +88,7 @@ class SimpleAclAuthorizerTest extends ZooKeeperTestHarness {
   }
 
   @Test(expected = classOf[IllegalArgumentException])
-  def testAuthorizeThrowsOnNoneLiteralResource(): Unit = {
+  def testAuthorizeThrowsOnNonLiteralResource(): Unit = {
     simpleAclAuthorizer.authorize(session, Read, Resource(Topic, "something", PREFIXED))
   }
 
@@ -346,40 +346,6 @@ class SimpleAclAuthorizerTest extends ZooKeeperTestHarness {
     }
   }
 
-  /**
-   * Verify that there is no timing window between loading ACL cache and setting
-   * up ZK change listener. Cache must be loaded before creating change listener
-   * in the authorizer to avoid the timing window.
-   */
-  @Test
-  def testChangeListenerTiming(): Unit = {
-    val configureSemaphore = new Semaphore(0)
-    val listenerSemaphore = new Semaphore(0)
-    val executor = Executors.newSingleThreadExecutor
-    val simpleAclAuthorizer3 = new SimpleAclAuthorizer {
-      override private[auth] def startZkChangeListeners(): Unit = {
-        configureSemaphore.release()
-        listenerSemaphore.acquireUninterruptibly()
-        super.startZkChangeListeners()
-      }
-    }
-    try {
-      val future = executor.submit(CoreUtils.runnable(simpleAclAuthorizer3.configure(config.originals)))
-      configureSemaphore.acquire()
-      val user1 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username)
-      val acls = Set(new Acl(user1, Deny, "host-1", Read))
-      simpleAclAuthorizer.addAcls(acls, resource)
-
-      listenerSemaphore.release()
-      future.get(10, TimeUnit.SECONDS)
-
-      assertEquals(acls, simpleAclAuthorizer3.getAcls(resource))
-    } finally {
-      simpleAclAuthorizer3.close()
-      executor.shutdownNow()
-    }
-  }
-
   @Test
   def testLocalConcurrentModificationOfResourceAcls(): Unit = {
     val commonResource = Resource(Topic, "test", LITERAL)
diff --git a/core/src/test/scala/unit/kafka/security/authorizer/AclAuthorizerTest.scala b/core/src/test/scala/unit/kafka/security/authorizer/AclAuthorizerTest.scala
new file mode 100644
index 0000000000000..aac9594f39a05
--- /dev/null
+++ b/core/src/test/scala/unit/kafka/security/authorizer/AclAuthorizerTest.scala
@@ -0,0 +1,877 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package kafka.security.authorizer
+
+import java.net.InetAddress
+import java.nio.charset.StandardCharsets.UTF_8
+import java.util.UUID
+import java.util.concurrent.{Executors, Semaphore, TimeUnit}
+
+import kafka.api.{ApiVersion, KAFKA_2_0_IV0, KAFKA_2_0_IV1}
+import kafka.security.auth.Resource
+import kafka.security.authorizer.AuthorizerUtils.{WildcardHost, WildcardPrincipal}
+import kafka.server.KafkaConfig
+import kafka.utils.{CoreUtils, TestUtils}
+import kafka.zk.{ZkAclStore, ZooKeeperTestHarness}
+import kafka.zookeeper.{GetChildrenRequest, GetDataRequest, ZooKeeperClient}
+import org.apache.kafka.common.acl._
+import org.apache.kafka.common.acl.AclOperation._
+import org.apache.kafka.common.acl.AclPermissionType.{ALLOW, DENY}
+import org.apache.kafka.common.errors.UnsupportedVersionException
+import org.apache.kafka.common.network.ListenerName
+import org.apache.kafka.common.protocol.ApiKeys
+import org.apache.kafka.common.requests.{RequestContext, RequestHeader}
+import org.apache.kafka.common.resource.{PatternType, ResourcePattern, ResourcePatternFilter, ResourceType}
+import org.apache.kafka.common.resource.ResourcePattern.WILDCARD_RESOURCE
+import org.apache.kafka.common.resource.ResourceType._
+import org.apache.kafka.common.resource.PatternType.{LITERAL, MATCH, PREFIXED}
+import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
+import org.apache.kafka.server.authorizer._
+import org.apache.kafka.common.utils.{SecurityUtils => JSecurityUtils, Time}
+import org.junit.Assert._
+import org.junit.{After, Before, Test}
+
+import scala.collection.JavaConverters._
+
+class AclAuthorizerTest extends ZooKeeperTestHarness {
+
+  private val allowReadAcl = new AccessControlEntry(WildcardPrincipal, WildcardHost, READ, ALLOW)
+  private val allowWriteAcl = new AccessControlEntry(WildcardPrincipal, WildcardHost, WRITE, ALLOW)
+  private val denyReadAcl = new AccessControlEntry(WildcardPrincipal, WildcardHost, READ, DENY)
+
+  private val wildCardResource = new ResourcePattern(TOPIC, WILDCARD_RESOURCE, LITERAL)
+  private val prefixedResource = new ResourcePattern(TOPIC, "foo", PREFIXED)
+  private val clusterResource = new ResourcePattern(CLUSTER, Resource.ClusterResourceName, LITERAL)
+  private val wildcardPrincipal = JSecurityUtils.parseKafkaPrincipal(WildcardPrincipal)
+
+  private val aclAuthorizer = new AclAuthorizer
+  private val aclAuthorizer2 = new AclAuthorizer
+  private var resource: ResourcePattern = _
+  private val superUsers = "User:superuser1; User:superuser2"
+  private val username = "alice"
+  private val principal = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username)
+  private val requestContext = newRequestContext(principal, InetAddress.getByName("192.168.0.1"))
+  private var config: KafkaConfig = _
+  private var zooKeeperClient: ZooKeeperClient = _
+
+  class CustomPrincipal(principalType: String, name: String) extends KafkaPrincipal(principalType, name) {
+    override def equals(o: scala.Any): Boolean = false
+  }
+
+  @Before
+  override def setUp(): Unit = {
+    super.setUp()
+
+    // Increase maxUpdateRetries to avoid transient failures
+    aclAuthorizer.maxUpdateRetries = Int.MaxValue
+    aclAuthorizer2.maxUpdateRetries = Int.MaxValue
+
+    val props = TestUtils.createBrokerConfig(0, zkConnect)
+    props.put(AclAuthorizer.SuperUsersProp, superUsers)
+
+    config = KafkaConfig.fromProps(props)
+    aclAuthorizer.configure(config.originals)
+    aclAuthorizer2.configure(config.originals)
+    resource = new ResourcePattern(TOPIC, "foo-" + UUID.randomUUID(), LITERAL)
+
+    zooKeeperClient = new ZooKeeperClient(zkConnect, zkSessionTimeout, zkConnectionTimeout, zkMaxInFlightRequests,
+      Time.SYSTEM, "kafka.test", "AclAuthorizerTest")
+  }
+
+  @After
+  override def tearDown(): Unit = {
+    aclAuthorizer.close()
+    aclAuthorizer2.close()
+    zooKeeperClient.close()
+    super.tearDown()
+  }
+
+  @Test(expected = classOf[IllegalArgumentException])
+  def testAuthorizeThrowsOnNonLiteralResource(): Unit = {
+    authorize(aclAuthorizer, requestContext, READ, new ResourcePattern(TOPIC, "something", PREFIXED))
+  }
+
+  @Test
+  def testAuthorizeWithEmptyResourceName(): Unit = {
+    assertFalse(authorize(aclAuthorizer, requestContext, READ, new ResourcePattern(GROUP, "", LITERAL)))
+    addAcls(aclAuthorizer, Set(allowReadAcl), new ResourcePattern(GROUP, WILDCARD_RESOURCE, LITERAL))
+    assertTrue(authorize(aclAuthorizer, requestContext, READ, new ResourcePattern(GROUP, "", LITERAL)))
+  }
+
+  // Authorizing the empty resource is not supported because we create a znode with the resource name.
+  @Test(expected = classOf[IllegalArgumentException])
+  def testEmptyAclThrowsException(): Unit = {
+    addAcls(aclAuthorizer, Set(allowReadAcl), new ResourcePattern(GROUP, "", LITERAL))
+  }
+
+  @Test
+  def testTopicAcl(): Unit = {
+    val user1 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username)
+    val user2 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "rob")
+    val user3 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "batman")
+    val host1 = InetAddress.getByName("192.168.1.1")
+    val host2 = InetAddress.getByName("192.168.1.2")
+
+    //user1 has READ access from host1 and host2.
+    val acl1 = new AccessControlEntry(user1.toString, host1.getHostAddress, READ, ALLOW)
+    val acl2 = new AccessControlEntry(user1.toString, host2.getHostAddress, READ, ALLOW)
+
+    //user1 does not have  READ access from host1.
+    val acl3 = new AccessControlEntry(user1.toString, host1.getHostAddress, READ, DENY)
+
+    //user1 has WRITE access from host1 only.
+    val acl4 = new AccessControlEntry(user1.toString, host1.getHostAddress, WRITE, ALLOW)
+
+    //user1 has DESCRIBE access from all hosts.
+    val acl5 = new AccessControlEntry(user1.toString, WildcardHost, DESCRIBE, ALLOW)
+
+    //user2 has READ access from all hosts.
+    val acl6 = new AccessControlEntry(user2.toString, WildcardHost, READ, ALLOW)
+
+    //user3 has WRITE access from all hosts.
+    val acl7 = new AccessControlEntry(user3.toString, WildcardHost, WRITE, ALLOW)
+
+    val acls = Set(acl1, acl2, acl3, acl4, acl5, acl6, acl7)
+
+    changeAclAndVerify(Set.empty, acls, Set.empty)
+
+    val host1Context = newRequestContext(user1, host1)
+    val host2Context = newRequestContext(user1, host2)
+
+    assertTrue("User1 should have READ access from host2", authorize(aclAuthorizer, host2Context, READ, resource))
+    assertFalse("User1 should not have READ access from host1 due to denyAcl", authorize(aclAuthorizer, host1Context, READ, resource))
+    assertTrue("User1 should have WRITE access from host1", authorize(aclAuthorizer, host1Context, WRITE, resource))
+    assertFalse("User1 should not have WRITE access from host2 as no allow acl is defined", authorize(aclAuthorizer, host2Context, WRITE, resource))
+    assertTrue("User1 should not have DESCRIBE access from host1", authorize(aclAuthorizer, host1Context, DESCRIBE, resource))
+    assertTrue("User1 should have DESCRIBE access from host2", authorize(aclAuthorizer, host2Context, DESCRIBE, resource))
+    assertFalse("User1 should not have edit access from host1", authorize(aclAuthorizer, host1Context, ALTER, resource))
+    assertFalse("User1 should not have edit access from host2", authorize(aclAuthorizer, host2Context, ALTER, resource))
+
+    //test if user has READ and write access they also get describe access
+    val user2Context = newRequestContext(user2, host1)
+    val user3Context = newRequestContext(user3, host1)
+    assertTrue("User2 should have DESCRIBE access from host1", authorize(aclAuthorizer, user2Context, DESCRIBE, resource))
+    assertTrue("User3 should have DESCRIBE access from host2", authorize(aclAuthorizer, user3Context, DESCRIBE, resource))
+    assertTrue("User2 should have READ access from host1", authorize(aclAuthorizer, user2Context, READ, resource))
+    assertTrue("User3 should have WRITE access from host2", authorize(aclAuthorizer, user3Context, WRITE, resource))
+  }
+
+  /**
+    CustomPrincipals should be compared with their principal type and name
+   */
+  @Test
+  def testAllowAccessWithCustomPrincipal(): Unit = {
+    val user = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username)
+    val customUserPrincipal = new CustomPrincipal(KafkaPrincipal.USER_TYPE, username)
+    val host1 = InetAddress.getByName("192.168.1.1")
+    val host2 = InetAddress.getByName("192.168.1.2")
+
+    // user has READ access from host2 but not from host1
+    val acl1 = new AccessControlEntry(user.toString, host1.getHostAddress, READ, DENY)
+    val acl2 = new AccessControlEntry(user.toString, host2.getHostAddress, READ, ALLOW)
+    val acls = Set(acl1, acl2)
+    changeAclAndVerify(Set.empty, acls, Set.empty)
+
+    val host1Context = newRequestContext(customUserPrincipal, host1)
+    val host2Context = newRequestContext(customUserPrincipal, host2)
+
+    assertTrue("User1 should have READ access from host2", authorize(aclAuthorizer, host2Context, READ, resource))
+    assertFalse("User1 should not have READ access from host1 due to denyAcl", authorize(aclAuthorizer, host1Context, READ, resource))
+  }
+
+  @Test
+  def testDenyTakesPrecedence(): Unit = {
+    val user = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username)
+    val host = InetAddress.getByName("192.168.2.1")
+    val session = newRequestContext(user, host)
+
+    val allowAll = new AccessControlEntry(WildcardPrincipal, WildcardHost, AclOperation.ALL, ALLOW)
+    val denyAcl = new AccessControlEntry(user.toString, host.getHostAddress, AclOperation.ALL, DENY)
+    val acls = Set(allowAll, denyAcl)
+
+    changeAclAndVerify(Set.empty, acls, Set.empty)
+
+    assertFalse("deny should take precedence over allow.", authorize(aclAuthorizer, session, READ, resource))
+  }
+
+  @Test
+  def testAllowAllAccess(): Unit = {
+    val allowAllAcl = new AccessControlEntry(WildcardPrincipal, WildcardHost, AclOperation.ALL, ALLOW)
+
+    changeAclAndVerify(Set.empty, Set(allowAllAcl), Set.empty)
+
+    val context = newRequestContext(new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "random"), InetAddress.getByName("192.0.4.4"))
+    assertTrue("allow all acl should allow access to all.", authorize(aclAuthorizer, context, READ, resource))
+  }
+
+  @Test
+  def testSuperUserHasAccess(): Unit = {
+    val denyAllAcl = new AccessControlEntry(WildcardPrincipal, WildcardHost, AclOperation.ALL, DENY)
+
+    changeAclAndVerify(Set.empty, Set(denyAllAcl), Set.empty)
+
+    val session1 = newRequestContext(new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "superuser1"), InetAddress.getByName("192.0.4.4"))
+    val session2 = newRequestContext(new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "superuser2"), InetAddress.getByName("192.0.4.4"))
+
+    assertTrue("superuser always has access, no matter what acls.", authorize(aclAuthorizer, session1, READ, resource))
+    assertTrue("superuser always has access, no matter what acls.", authorize(aclAuthorizer, session2, READ, resource))
+  }
+
+  /**
+    CustomPrincipals should be compared with their principal type and name
+   */
+  @Test
+  def testSuperUserWithCustomPrincipalHasAccess(): Unit = {
+    val denyAllAcl = new AccessControlEntry(WildcardPrincipal, WildcardHost, AclOperation.ALL, DENY)
+    changeAclAndVerify(Set.empty, Set(denyAllAcl), Set.empty)
+
+    val session = newRequestContext(new CustomPrincipal(KafkaPrincipal.USER_TYPE, "superuser1"), InetAddress.getByName("192.0.4.4"))
+
+    assertTrue("superuser with custom principal always has access, no matter what acls.", authorize(aclAuthorizer, session, READ, resource))
+  }
+
+  @Test
+  def testWildCardAcls(): Unit = {
+    assertFalse("when acls = [], authorizer should fail close.", authorize(aclAuthorizer, requestContext, READ, resource))
+
+    val user1 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username)
+    val host1 = InetAddress.getByName("192.168.3.1")
+    val readAcl = new AccessControlEntry(user1.toString, host1.getHostAddress, READ, ALLOW)
+
+    val acls = changeAclAndVerify(Set.empty, Set(readAcl), Set.empty, wildCardResource)
+
+    val host1Context = newRequestContext(user1, host1)
+    assertTrue("User1 should have READ access from host1", authorize(aclAuthorizer, host1Context, READ, resource))
+
+    //allow WRITE to specific topic.
+    val writeAcl = new AccessControlEntry(user1.toString, host1.getHostAddress, WRITE, ALLOW)
+    changeAclAndVerify(Set.empty, Set(writeAcl), Set.empty)
+
+    //deny WRITE to wild card topic.
+    val denyWriteOnWildCardResourceAcl = new AccessControlEntry(user1.toString, host1.getHostAddress, WRITE, DENY)
+    changeAclAndVerify(acls, Set(denyWriteOnWildCardResourceAcl), Set.empty, wildCardResource)
+
+    assertFalse("User1 should not have WRITE access from host1", authorize(aclAuthorizer, host1Context, WRITE, resource))
+  }
+
+  @Test
+  def testNoAclFound(): Unit = {
+    assertFalse("when acls = [], authorizer should deny op.", authorize(aclAuthorizer, requestContext, READ, resource))
+  }
+
+  @Test
+  def testNoAclFoundOverride(): Unit = {
+    val props = TestUtils.createBrokerConfig(1, zkConnect)
+    props.put(AclAuthorizer.AllowEveryoneIfNoAclIsFoundProp, "true")
+
+    val cfg = KafkaConfig.fromProps(props)
+    val testAuthorizer = new AclAuthorizer
+    try {
+      testAuthorizer.configure(cfg.originals)
+      assertTrue("when acls = null or [],  authorizer should allow op with allow.everyone = true.",
+        authorize(testAuthorizer, requestContext, READ, resource))
+    } finally {
+      testAuthorizer.close()
+    }
+  }
+
+  @Test
+  def testAclManagementAPIs(): Unit = {
+    val user1 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username)
+    val user2 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "bob")
+    val host1 = "host1"
+    val host2 = "host2"
+
+    val acl1 = new AccessControlEntry(user1.toString, host1, READ, ALLOW)
+    val acl2 = new AccessControlEntry(user1.toString, host1, WRITE, ALLOW)
+    val acl3 = new AccessControlEntry(user2.toString, host2, READ, ALLOW)
+    val acl4 = new AccessControlEntry(user2.toString, host2, WRITE, ALLOW)
+
+    var acls = changeAclAndVerify(Set.empty, Set(acl1, acl2, acl3, acl4), Set.empty)
+
+    //test addAcl is additive
+    val acl5 = new AccessControlEntry(user2.toString, WildcardHost, READ, ALLOW)
+    acls = changeAclAndVerify(acls, Set(acl5), Set.empty)
+
+    //test get by principal name.
+    TestUtils.waitUntilTrue(() => Set(acl1, acl2).map(acl => new AclBinding(resource, acl)) == getAcls(aclAuthorizer, user1),
+      "changes not propagated in timeout period")
+    TestUtils.waitUntilTrue(() => Set(acl3, acl4, acl5).map(acl => new AclBinding(resource, acl)) == getAcls(aclAuthorizer, user2),
+      "changes not propagated in timeout period")
+
+    val resourceToAcls = Map[ResourcePattern, Set[AccessControlEntry]](
+      new ResourcePattern(TOPIC, WILDCARD_RESOURCE, LITERAL) -> Set(new AccessControlEntry(user2.toString, WildcardHost, READ, ALLOW)),
+      new ResourcePattern(CLUSTER , WILDCARD_RESOURCE, LITERAL) -> Set(new AccessControlEntry(user2.toString, host1, READ, ALLOW)),
+      new ResourcePattern(GROUP, WILDCARD_RESOURCE, LITERAL) -> acls,
+      new ResourcePattern(GROUP, "test-ConsumerGroup", LITERAL) -> acls
+    )
+
+    resourceToAcls foreach { case (key, value) => changeAclAndVerify(Set.empty, value, Set.empty, key) }
+    val expectedAcls = (resourceToAcls + (resource -> acls)).flatMap {
+      case (res, resAcls) => resAcls.map { acl => new AclBinding(res, acl) }
+    }.toSet
+    TestUtils.waitUntilTrue(() => expectedAcls == getAcls(aclAuthorizer), "changes not propagated in timeout period.")
+
+    //test remove acl from existing acls.
+    acls = changeAclAndVerify(acls, Set.empty, Set(acl1, acl5))
+
+    //test remove all acls for resource
+    removeAcls(aclAuthorizer, Set.empty, resource)
+    TestUtils.waitAndVerifyAcls(Set.empty[AccessControlEntry], aclAuthorizer, resource)
+    assertTrue(!zkClient.resourceExists(AuthorizerUtils.convertToResource(resource)))
+
+    //test removing last acl also deletes ZooKeeper path
+    acls = changeAclAndVerify(Set.empty, Set(acl1), Set.empty)
+    changeAclAndVerify(acls, Set.empty, acls)
+    assertTrue(!zkClient.resourceExists(AuthorizerUtils.convertToResource(resource)))
+  }
+
+  @Test
+  def testLoadCache(): Unit = {
+    val user1 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username)
+    val acl1 = new AccessControlEntry(user1.toString, "host-1", READ, ALLOW)
+    val acls = Set(acl1)
+    addAcls(aclAuthorizer, acls, resource)
+
+    val user2 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "bob")
+    val resource1 = new ResourcePattern(TOPIC, "test-2", LITERAL)
+    val acl2 = new AccessControlEntry(user2.toString, "host3", READ, DENY)
+    val acls1 = Set(acl2)
+    addAcls(aclAuthorizer, acls1, resource1)
+
+    zkClient.deleteAclChangeNotifications
+    val authorizer = new AclAuthorizer
+    try {
+      authorizer.configure(config.originals)
+
+      assertEquals(acls, getAcls(authorizer, resource))
+      assertEquals(acls1, getAcls(authorizer, resource1))
+    } finally {
+      authorizer.close()
+    }
+  }
+
+  /**
+   * Verify that there is no timing window between loading ACL cache and setting
+   * up ZK change listener. Cache must be loaded before creating change listener
+   * in the authorizer to avoid the timing window.
+   */
+  @Test
+  def testChangeListenerTiming(): Unit = {
+    val configureSemaphore = new Semaphore(0)
+    val listenerSemaphore = new Semaphore(0)
+    val executor = Executors.newSingleThreadExecutor
+    val aclAuthorizer3 = new AclAuthorizer {
+      override private[authorizer] def startZkChangeListeners(): Unit = {
+        configureSemaphore.release()
+        listenerSemaphore.acquireUninterruptibly()
+        super.startZkChangeListeners()
+      }
+    }
+    try {
+      val future = executor.submit(CoreUtils.runnable(aclAuthorizer3.configure(config.originals)))
+      configureSemaphore.acquire()
+      val user1 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username)
+      val acls = Set(new AccessControlEntry(user1.toString, "host-1", READ, DENY))
+      addAcls(aclAuthorizer, acls, resource)
+
+      listenerSemaphore.release()
+      future.get(10, TimeUnit.SECONDS)
+
+      assertEquals(acls, getAcls(aclAuthorizer3, resource))
+    } finally {
+      aclAuthorizer3.close()
+      executor.shutdownNow()
+    }
+  }
+
+  @Test
+  def testLocalConcurrentModificationOfResourceAcls(): Unit = {
+    val commonResource = new ResourcePattern(TOPIC, "test", LITERAL)
+
+    val user1 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username)
+    val acl1 = new AccessControlEntry(user1.toString, WildcardHost, READ, ALLOW)
+
+    val user2 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "bob")
+    val acl2 = new AccessControlEntry(user2.toString, WildcardHost, READ, DENY)
+
+    addAcls(aclAuthorizer, Set(acl1), commonResource)
+    addAcls(aclAuthorizer, Set(acl2), commonResource)
+
+    TestUtils.waitAndVerifyAcls(Set(acl1, acl2), aclAuthorizer, commonResource)
+  }
+
+  @Test
+  def testDistributedConcurrentModificationOfResourceAcls(): Unit = {
+    val commonResource = new ResourcePattern(TOPIC, "test", LITERAL)
+
+    val user1 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username)
+    val acl1 = new AccessControlEntry(user1.toString, WildcardHost, READ, ALLOW)
+
+    val user2 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "bob")
+    val acl2 = new AccessControlEntry(user2.toString, WildcardHost, READ, DENY)
+
+    // Add on each instance
+    addAcls(aclAuthorizer, Set(acl1), commonResource)
+    addAcls(aclAuthorizer2, Set(acl2), commonResource)
+
+    TestUtils.waitAndVerifyAcls(Set(acl1, acl2), aclAuthorizer, commonResource)
+    TestUtils.waitAndVerifyAcls(Set(acl1, acl2), aclAuthorizer2, commonResource)
+
+    val user3 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "joe")
+    val acl3 = new AccessControlEntry(user3.toString, WildcardHost, READ, DENY)
+
+    // Add on one instance and delete on another
+    addAcls(aclAuthorizer, Set(acl3), commonResource)
+    val deleted = removeAcls(aclAuthorizer2, Set(acl3), commonResource)
+
+    assertTrue("The authorizer should see a value that needs to be deleted", deleted)
+
+    TestUtils.waitAndVerifyAcls(Set(acl1, acl2), aclAuthorizer, commonResource)
+    TestUtils.waitAndVerifyAcls(Set(acl1, acl2), aclAuthorizer2, commonResource)
+  }
+
+  @Test
+  def testHighConcurrencyModificationOfResourceAcls(): Unit = {
+    val commonResource = new ResourcePattern(TOPIC, "test", LITERAL)
+
+    val acls= (0 to 50).map { i =>
+      val useri = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, i.toString)
+      (new AccessControlEntry(useri.toString, WildcardHost, READ, ALLOW), i)
+    }
+
+    // Alternate authorizer, Remove all acls that end in 0
+    val concurrentFuctions = acls.map { case (acl, aclId) =>
+      () => {
+        if (aclId % 2 == 0) {
+          addAcls(aclAuthorizer, Set(acl), commonResource)
+        } else {
+          addAcls(aclAuthorizer2, Set(acl), commonResource)
+        }
+        if (aclId % 10 == 0) {
+          removeAcls(aclAuthorizer2, Set(acl), commonResource)
+        }
+      }
+    }
+
+    val expectedAcls = acls.filter { case (acl, aclId) =>
+      aclId % 10 != 0
+    }.map(_._1).toSet
+
+    TestUtils.assertConcurrent("Should support many concurrent calls", concurrentFuctions, 30 * 1000)
+
+    TestUtils.waitAndVerifyAcls(expectedAcls, aclAuthorizer, commonResource)
+    TestUtils.waitAndVerifyAcls(expectedAcls, aclAuthorizer2, commonResource)
+  }
+
+  /**
+    * Test ACL inheritance, as described in #{org.apache.kafka.common.acl.AclOperation}
+    */
+  @Test
+  def testAclInheritance(): Unit = {
+    testImplicationsOfAllow(AclOperation.ALL, Set(READ, WRITE, CREATE, DELETE, ALTER, DESCRIBE,
+      CLUSTER_ACTION, DESCRIBE_CONFIGS, ALTER_CONFIGS, IDEMPOTENT_WRITE))
+    testImplicationsOfDeny(AclOperation.ALL, Set(READ, WRITE, CREATE, DELETE, ALTER, DESCRIBE,
+      CLUSTER_ACTION, DESCRIBE_CONFIGS, ALTER_CONFIGS, IDEMPOTENT_WRITE))
+    testImplicationsOfAllow(READ, Set(DESCRIBE))
+    testImplicationsOfAllow(WRITE, Set(DESCRIBE))
+    testImplicationsOfAllow(DELETE, Set(DESCRIBE))
+    testImplicationsOfAllow(ALTER, Set(DESCRIBE))
+    testImplicationsOfDeny(DESCRIBE, Set())
+    testImplicationsOfAllow(ALTER_CONFIGS, Set(DESCRIBE_CONFIGS))
+    testImplicationsOfDeny(DESCRIBE_CONFIGS, Set())
+  }
+
+  private def testImplicationsOfAllow(parentOp: AclOperation, allowedOps: Set[AclOperation]): Unit = {
+    val user = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username)
+    val host = InetAddress.getByName("192.168.3.1")
+    val hostContext = newRequestContext(user, host)
+    val acl = new AccessControlEntry(user.toString, WildcardHost, parentOp, ALLOW)
+    addAcls(aclAuthorizer, Set(acl), clusterResource)
+    AclOperation.values.filter(validOp).foreach { op =>
+      val authorized = authorize(aclAuthorizer, hostContext, op, clusterResource)
+      if (allowedOps.contains(op) || op == parentOp)
+        assertTrue(s"ALLOW $parentOp should imply ALLOW $op", authorized)
+      else
+        assertFalse(s"ALLOW $parentOp should not imply ALLOW $op", authorized)
+    }
+    removeAcls(aclAuthorizer, Set(acl), clusterResource)
+  }
+
+  private def testImplicationsOfDeny(parentOp: AclOperation, deniedOps: Set[AclOperation]): Unit = {
+    val user1 = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username)
+    val host1 = InetAddress.getByName("192.168.3.1")
+    val host1Context = newRequestContext(user1, host1)
+    val acls = Set(new AccessControlEntry(user1.toString, WildcardHost, parentOp, DENY),
+      new AccessControlEntry(user1.toString, WildcardHost, AclOperation.ALL, ALLOW))
+    addAcls(aclAuthorizer, acls, clusterResource)
+    AclOperation.values.filter(validOp).foreach { op =>
+      val authorized = authorize(aclAuthorizer, host1Context, op, clusterResource)
+      if (deniedOps.contains(op) || op == parentOp)
+        assertFalse(s"DENY $parentOp should imply DENY $op", authorized)
+      else
+        assertTrue(s"DENY $parentOp should not imply DENY $op", authorized)
+    }
+    removeAcls(aclAuthorizer, acls, clusterResource)
+  }
+
+  @Test
+  def testHighConcurrencyDeletionOfResourceAcls(): Unit = {
+    val acl = new AccessControlEntry(new KafkaPrincipal(KafkaPrincipal.USER_TYPE, username).toString, WildcardHost, AclOperation.ALL, ALLOW)
+
+    // Alternate authorizer to keep adding and removing ZooKeeper path
+    val concurrentFuctions = (0 to 50).map { _ =>
+      () => {
+        addAcls(aclAuthorizer, Set(acl), resource)
+        removeAcls(aclAuthorizer2, Set(acl), resource)
+      }
+    }
+
+    TestUtils.assertConcurrent("Should support many concurrent calls", concurrentFuctions, 30 * 1000)
+
+    TestUtils.waitAndVerifyAcls(Set.empty[AccessControlEntry], aclAuthorizer, resource)
+    TestUtils.waitAndVerifyAcls(Set.empty[AccessControlEntry], aclAuthorizer2, resource)
+  }
+
+  @Test
+  def testAccessAllowedIfAllowAclExistsOnWildcardResource(): Unit = {
+    addAcls(aclAuthorizer, Set(allowReadAcl), wildCardResource)
+
+    assertTrue(authorize(aclAuthorizer, requestContext, READ, resource))
+  }
+
+  @Test
+  def testDeleteAclOnWildcardResource(): Unit = {
+    addAcls(aclAuthorizer, Set(allowReadAcl, allowWriteAcl), wildCardResource)
+
+    removeAcls(aclAuthorizer, Set(allowReadAcl), wildCardResource)
+
+    assertEquals(Set(allowWriteAcl), getAcls(aclAuthorizer, wildCardResource))
+  }
+
+  @Test
+  def testDeleteAllAclOnWildcardResource(): Unit = {
+    addAcls(aclAuthorizer, Set(allowReadAcl), wildCardResource)
+
+    removeAcls(aclAuthorizer, Set.empty, wildCardResource)
+
+    assertEquals(Set.empty, getAcls(aclAuthorizer))
+  }
+
+  @Test
+  def testAccessAllowedIfAllowAclExistsOnPrefixedResource(): Unit = {
+    addAcls(aclAuthorizer, Set(allowReadAcl), prefixedResource)
+
+    assertTrue(authorize(aclAuthorizer, requestContext, READ, resource))
+  }
+
+  @Test
+  def testDeleteAclOnPrefixedResource(): Unit = {
+    addAcls(aclAuthorizer, Set(allowReadAcl, allowWriteAcl), prefixedResource)
+
+    removeAcls(aclAuthorizer, Set(allowReadAcl), prefixedResource)
+
+    assertEquals(Set(allowWriteAcl), getAcls(aclAuthorizer, prefixedResource))
+  }
+
+  @Test
+  def testDeleteAllAclOnPrefixedResource(): Unit = {
+    addAcls(aclAuthorizer, Set(allowReadAcl, allowWriteAcl), prefixedResource)
+
+    removeAcls(aclAuthorizer, Set.empty, prefixedResource)
+
+    assertEquals(Set.empty, getAcls(aclAuthorizer))
+  }
+
+  @Test
+  def testAddAclsOnLiteralResource(): Unit = {
+    addAcls(aclAuthorizer, Set(allowReadAcl, allowWriteAcl), resource)
+    addAcls(aclAuthorizer, Set(allowWriteAcl, denyReadAcl), resource)
+
+    assertEquals(Set(allowReadAcl, allowWriteAcl, denyReadAcl), getAcls(aclAuthorizer, resource))
+    assertEquals(Set.empty, getAcls(aclAuthorizer, wildCardResource))
+    assertEquals(Set.empty, getAcls(aclAuthorizer, prefixedResource))
+  }
+
+  @Test
+  def testAddAclsOnWildcardResource(): Unit = {
+    addAcls(aclAuthorizer, Set(allowReadAcl, allowWriteAcl), wildCardResource)
+    addAcls(aclAuthorizer, Set(allowWriteAcl, denyReadAcl), wildCardResource)
+
+    assertEquals(Set(allowReadAcl, allowWriteAcl, denyReadAcl), getAcls(aclAuthorizer, wildCardResource))
+    assertEquals(Set.empty, getAcls(aclAuthorizer, resource))
+    assertEquals(Set.empty, getAcls(aclAuthorizer, prefixedResource))
+  }
+
+  @Test
+  def testAddAclsOnPrefixedResource(): Unit = {
+    addAcls(aclAuthorizer, Set(allowReadAcl, allowWriteAcl), prefixedResource)
+    addAcls(aclAuthorizer, Set(allowWriteAcl, denyReadAcl), prefixedResource)
+
+    assertEquals(Set(allowReadAcl, allowWriteAcl, denyReadAcl), getAcls(aclAuthorizer, prefixedResource))
+    assertEquals(Set.empty, getAcls(aclAuthorizer, wildCardResource))
+    assertEquals(Set.empty, getAcls(aclAuthorizer, resource))
+  }
+
+  @Test
+  def testAuthorizeWithPrefixedResource(): Unit = {
+    addAcls(aclAuthorizer, Set(denyReadAcl), new ResourcePattern(TOPIC, "a_other", LITERAL))
+    addAcls(aclAuthorizer, Set(denyReadAcl), new ResourcePattern(TOPIC, "a_other", PREFIXED))
+    addAcls(aclAuthorizer, Set(denyReadAcl), new ResourcePattern(TOPIC, "foo-" + UUID.randomUUID(), PREFIXED))
+    addAcls(aclAuthorizer, Set(denyReadAcl), new ResourcePattern(TOPIC, "foo-" + UUID.randomUUID(), PREFIXED))
+    addAcls(aclAuthorizer, Set(denyReadAcl), new ResourcePattern(TOPIC, "foo-" + UUID.randomUUID() + "-zzz", PREFIXED))
+    addAcls(aclAuthorizer, Set(denyReadAcl), new ResourcePattern(TOPIC, "fooo-" + UUID.randomUUID(), PREFIXED))
+    addAcls(aclAuthorizer, Set(denyReadAcl), new ResourcePattern(TOPIC, "fo-" + UUID.randomUUID(), PREFIXED))
+    addAcls(aclAuthorizer, Set(denyReadAcl), new ResourcePattern(TOPIC, "fop-" + UUID.randomUUID(), PREFIXED))
+    addAcls(aclAuthorizer, Set(denyReadAcl), new ResourcePattern(TOPIC, "fon-" + UUID.randomUUID(), PREFIXED))
+    addAcls(aclAuthorizer, Set(denyReadAcl), new ResourcePattern(TOPIC, "fon-", PREFIXED))
+    addAcls(aclAuthorizer, Set(denyReadAcl), new ResourcePattern(TOPIC, "z_other", PREFIXED))
+    addAcls(aclAuthorizer, Set(denyReadAcl), new ResourcePattern(TOPIC, "z_other", LITERAL))
+
+    addAcls(aclAuthorizer, Set(allowReadAcl), prefixedResource)
+
+    assertTrue(authorize(aclAuthorizer, requestContext, READ, resource))
+  }
+
+  @Test
+  def testSingleCharacterResourceAcls(): Unit = {
+    addAcls(aclAuthorizer, Set(allowReadAcl), new ResourcePattern(TOPIC, "f", LITERAL))
+    assertTrue(authorize(aclAuthorizer, requestContext, READ, new ResourcePattern(TOPIC, "f", LITERAL)))
+    assertFalse(authorize(aclAuthorizer, requestContext, READ, new ResourcePattern(TOPIC, "foo", LITERAL)))
+
+    addAcls(aclAuthorizer, Set(allowReadAcl), new ResourcePattern(TOPIC, "_", PREFIXED))
+    assertTrue(authorize(aclAuthorizer, requestContext, READ, new ResourcePattern(TOPIC, "_foo", LITERAL)))
+    assertTrue(authorize(aclAuthorizer, requestContext, READ, new ResourcePattern(TOPIC, "_", LITERAL)))
+    assertFalse(authorize(aclAuthorizer, requestContext, READ, new ResourcePattern(TOPIC, "foo_", LITERAL)))
+  }
+
+  @Test
+  def testGetAclsPrincipal(): Unit = {
+    val aclOnSpecificPrincipal = new AccessControlEntry(principal.toString, WildcardHost, WRITE, ALLOW)
+    addAcls(aclAuthorizer, Set(aclOnSpecificPrincipal), resource)
+
+    assertEquals("acl on specific should not be returned for wildcard request",
+      0, getAcls(aclAuthorizer, wildcardPrincipal).size)
+    assertEquals("acl on specific should be returned for specific request",
+      1, getAcls(aclAuthorizer, principal).size)
+    assertEquals("acl on specific should be returned for different principal instance",
+      1, getAcls(aclAuthorizer, new KafkaPrincipal(principal.getPrincipalType, principal.getName)).size)
+
+    removeAcls(aclAuthorizer, Set.empty, resource)
+    val aclOnWildcardPrincipal = new AccessControlEntry(WildcardPrincipal, WildcardHost, WRITE, ALLOW)
+    addAcls(aclAuthorizer, Set(aclOnWildcardPrincipal), resource)
+
+    assertEquals("acl on wildcard should be returned for wildcard request",
+      1, getAcls(aclAuthorizer, wildcardPrincipal).size)
+    assertEquals("acl on wildcard should not be returned for specific request",
+      0, getAcls(aclAuthorizer, principal).size)
+  }
+
+  @Test
+  def testAclsFilter(): Unit = {
+    val resource1 = new ResourcePattern(TOPIC, "foo-" + UUID.randomUUID(), LITERAL)
+    val resource2 = new ResourcePattern(TOPIC, "bar-" + UUID.randomUUID(), LITERAL)
+    val prefixedResource = new ResourcePattern(TOPIC, "bar-", PREFIXED)
+
+    val acl1 = new AclBinding(resource1, new AccessControlEntry(principal.toString, WildcardHost, READ, ALLOW))
+    val acl2 = new AclBinding(resource1, new AccessControlEntry(principal.toString, "192.168.0.1", WRITE, ALLOW))
+    val acl3 = new AclBinding(resource2, new AccessControlEntry(principal.toString, WildcardHost, DESCRIBE, ALLOW))
+    val acl4 = new AclBinding(prefixedResource, new AccessControlEntry(wildcardPrincipal.toString, WildcardHost, READ, ALLOW))
+
+    aclAuthorizer.createAcls(requestContext, List(acl1, acl2, acl3, acl4).asJava)
+    assertEquals(Set(acl1, acl2, acl3, acl4), aclAuthorizer.acls(AclBindingFilter.ANY).asScala.toSet)
+    assertEquals(Set(acl1, acl2), aclAuthorizer.acls(new AclBindingFilter(resource1.toFilter, AccessControlEntryFilter.ANY)).asScala.toSet)
+    assertEquals(Set(acl4), aclAuthorizer.acls(new AclBindingFilter(prefixedResource.toFilter, AccessControlEntryFilter.ANY)).asScala.toSet)
+    val matchingFilter = new AclBindingFilter(new ResourcePatternFilter(ResourceType.ANY, resource2.name, MATCH), AccessControlEntryFilter.ANY)
+    assertEquals(Set(acl3, acl4), aclAuthorizer.acls(matchingFilter).asScala.toSet)
+
+    val filters = List(matchingFilter,
+      acl1.toFilter,
+      new AclBindingFilter(resource2.toFilter, AccessControlEntryFilter.ANY),
+      new AclBindingFilter(new ResourcePatternFilter(TOPIC, "baz", PatternType.ANY), AccessControlEntryFilter.ANY))
+    val deleteResults = aclAuthorizer.deleteAcls(requestContext, filters.asJava).asScala
+    assertEquals(List.empty, deleteResults.filter(_.exception != null))
+    filters.indices.foreach { i =>
+      assertEquals(Set.empty, deleteResults(i).aclBindingDeleteResults.asScala.toSet.filter(_.exception != null))
+    }
+    assertEquals(Set(acl3, acl4), deleteResults(0).aclBindingDeleteResults.asScala.map(_.aclBinding).toSet)
+    assertEquals(Set(acl1), deleteResults(1).aclBindingDeleteResults.asScala.map(_.aclBinding).toSet)
+    assertEquals(Set.empty, deleteResults(2).aclBindingDeleteResults.asScala.map(_.aclBinding).toSet)
+    assertEquals(Set.empty, deleteResults(3).aclBindingDeleteResults.asScala.map(_.aclBinding).toSet)
+  }
+
+  @Test(expected = classOf[UnsupportedVersionException])
+  def testThrowsOnAddPrefixedAclIfInterBrokerProtocolVersionTooLow(): Unit = {
+    givenAuthorizerWithProtocolVersion(Option(KAFKA_2_0_IV0))
+    addAcls(aclAuthorizer, Set(denyReadAcl), new ResourcePattern(TOPIC, "z_other", PREFIXED))
+  }
+
+  @Test
+  def testWritesExtendedAclChangeEventIfInterBrokerProtocolNotSet(): Unit = {
+    givenAuthorizerWithProtocolVersion(Option.empty)
+    val resource = new ResourcePattern(TOPIC, "z_other", PREFIXED)
+    val expected = new String(ZkAclStore(PREFIXED).changeStore
+      .createChangeNode(AuthorizerUtils.convertToResource(resource)).bytes, UTF_8)
+
+    addAcls(aclAuthorizer, Set(denyReadAcl), resource)
+
+    val actual = getAclChangeEventAsString(PREFIXED)
+
+    assertEquals(expected, actual)
+  }
+
+  @Test
+  def testWritesExtendedAclChangeEventWhenInterBrokerProtocolAtLeastKafkaV2(): Unit = {
+    givenAuthorizerWithProtocolVersion(Option(KAFKA_2_0_IV1))
+    val resource = new ResourcePattern(TOPIC, "z_other", PREFIXED)
+    val expected = new String(ZkAclStore(PREFIXED).changeStore
+      .createChangeNode(AuthorizerUtils.convertToResource(resource)).bytes, UTF_8)
+
+    addAcls(aclAuthorizer, Set(denyReadAcl), resource)
+
+    val actual = getAclChangeEventAsString(PREFIXED)
+
+    assertEquals(expected, actual)
+  }
+
+  @Test
+  def testWritesLiteralWritesLiteralAclChangeEventWhenInterBrokerProtocolLessThanKafkaV2eralAclChangesForOlderProtocolVersions(): Unit = {
+    givenAuthorizerWithProtocolVersion(Option(KAFKA_2_0_IV0))
+    val resource = new ResourcePattern(TOPIC, "z_other", LITERAL)
+    val expected = new String(ZkAclStore(LITERAL).changeStore
+      .createChangeNode(AuthorizerUtils.convertToResource(resource)).bytes, UTF_8)
+
+    addAcls(aclAuthorizer, Set(denyReadAcl), resource)
+
+    val actual = getAclChangeEventAsString(LITERAL)
+
+    assertEquals(expected, actual)
+  }
+
+  @Test
+  def testWritesLiteralAclChangeEventWhenInterBrokerProtocolIsKafkaV2(): Unit = {
+    givenAuthorizerWithProtocolVersion(Option(KAFKA_2_0_IV1))
+    val resource = new ResourcePattern(TOPIC, "z_other", LITERAL)
+    val expected = new String(ZkAclStore(LITERAL).changeStore
+      .createChangeNode(AuthorizerUtils.convertToResource(resource)).bytes, UTF_8)
+
+    addAcls(aclAuthorizer, Set(denyReadAcl), resource)
+
+    val actual = getAclChangeEventAsString(LITERAL)
+
+    assertEquals(expected, actual)
+  }
+
+  private def givenAuthorizerWithProtocolVersion(protocolVersion: Option[ApiVersion]): Unit = {
+    aclAuthorizer.close()
+
+    val props = TestUtils.createBrokerConfig(0, zkConnect)
+    props.put(AclAuthorizer.SuperUsersProp, superUsers)
+    protocolVersion.foreach(version => props.put(KafkaConfig.InterBrokerProtocolVersionProp, version.toString))
+
+    config = KafkaConfig.fromProps(props)
+
+    aclAuthorizer.configure(config.originals)
+  }
+
+  private def getAclChangeEventAsString(patternType: PatternType) = {
+    val store = ZkAclStore(patternType)
+    val children = zooKeeperClient.handleRequest(GetChildrenRequest(store.changeStore.aclChangePath))
+    children.maybeThrow()
+    assertEquals("Expecting 1 change event", 1, children.children.size)
+
+    val data = zooKeeperClient.handleRequest(GetDataRequest(s"${store.changeStore.aclChangePath}/${children.children.head}"))
+    data.maybeThrow()
+
+    new String(data.data, UTF_8)
+  }
+
+  private def changeAclAndVerify(originalAcls: Set[AccessControlEntry],
+                                 addedAcls: Set[AccessControlEntry],
+                                 removedAcls: Set[AccessControlEntry],
+                                 resource: ResourcePattern = resource): Set[AccessControlEntry] = {
+    var acls = originalAcls
+
+    if(addedAcls.nonEmpty) {
+      addAcls(aclAuthorizer, addedAcls, resource)
+      acls ++= addedAcls
+    }
+
+    if(removedAcls.nonEmpty) {
+      removeAcls(aclAuthorizer, removedAcls, resource)
+      acls --=removedAcls
+    }
+
+    TestUtils.waitAndVerifyAcls(acls, aclAuthorizer, resource)
+
+    acls
+  }
+
+  private def newRequestContext(principal: KafkaPrincipal, clientAddress: InetAddress, apiKey: ApiKeys = ApiKeys.PRODUCE): RequestContext = {
+    val securityProtocol = SecurityProtocol.SASL_PLAINTEXT
+    val header = new RequestHeader(apiKey, 2, "", 1) //ApiKeys apiKey, short version, String clientId, int correlation
+    new RequestContext(header, "", clientAddress, principal, ListenerName.forSecurityProtocol(securityProtocol), securityProtocol)
+  }
+
+  private def authorize(authorizer: AclAuthorizer, requestContext: RequestContext, operation: AclOperation, resource: ResourcePattern): Boolean = {
+    val action = new Action(operation, resource, 1, true, true)
+    authorizer.authorize(requestContext, List(action).asJava).asScala.head == AuthorizationResult.ALLOWED
+  }
+
+  private def addAcls(authorizer: AclAuthorizer, aces: Set[AccessControlEntry], resourcePattern: ResourcePattern): Unit = {
+    val bindings = aces.map { ace => new AclBinding(resourcePattern, ace) }
+    authorizer.createAcls(requestContext, bindings.toList.asJava).asScala.foreach { result =>
+      if (result.exception != null)
+        throw result.exception
+    }
+  }
+
+  private def removeAcls(authorizer: AclAuthorizer, aces: Set[AccessControlEntry], resourcePattern: ResourcePattern): Boolean = {
+    val bindings = if (aces.isEmpty)
+      Set(new AclBindingFilter(resourcePattern.toFilter, AccessControlEntryFilter.ANY) )
+    else
+      aces.map { ace => new AclBinding(resourcePattern, ace).toFilter }
+    authorizer.deleteAcls(requestContext, bindings.toList.asJava).asScala.forall { result =>
+      if (result.exception != null)
+        throw result.exception
+      result.aclBindingDeleteResults.asScala.foreach { r =>
+        if (r.exception != null)
+          throw r.exception
+      }
+      result.aclBindingDeleteResults.asScala.exists(_.deleted)
+    }
+  }
+
+  private def getAcls(authorizer: AclAuthorizer, resourcePattern: ResourcePattern): Set[AccessControlEntry] = {
+    val acls = authorizer.acls(new AclBindingFilter(resourcePattern.toFilter, AccessControlEntryFilter.ANY)).asScala.toSet
+    acls.map(_.entry)
+  }
+
+  private def getAcls(authorizer: AclAuthorizer, principal: KafkaPrincipal): Set[AclBinding] = {
+    val filter = new AclBindingFilter(ResourcePatternFilter.ANY,
+      new AccessControlEntryFilter(principal.toString, null, AclOperation.ANY, AclPermissionType.ANY))
+    authorizer.acls(filter).asScala.toSet
+  }
+
+  private def getAcls(authorizer: AclAuthorizer): Set[AclBinding] = {
+    authorizer.acls(AclBindingFilter.ANY).asScala.toSet
+  }
+
+  private def validOp(op: AclOperation): Boolean = {
+    op != AclOperation.ANY && op != AclOperation.UNKNOWN
+  }
+}
diff --git a/core/src/test/scala/unit/kafka/security/token/delegation/DelegationTokenManagerTest.scala b/core/src/test/scala/unit/kafka/security/token/delegation/DelegationTokenManagerTest.scala
index 30e9e6b537752..7398a6db6b551 100644
--- a/core/src/test/scala/unit/kafka/security/token/delegation/DelegationTokenManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/security/token/delegation/DelegationTokenManagerTest.scala
@@ -23,17 +23,23 @@ import java.util.{Base64, Properties}
 
 import kafka.network.RequestChannel.Session
 import kafka.security.auth.Acl.WildCardHost
-import kafka.security.auth._
+import kafka.security.authorizer.{AclAuthorizer, AuthorizerUtils}
 import kafka.server.{CreateTokenResult, Defaults, DelegationTokenManager, KafkaConfig}
 import kafka.utils.TestUtils
 import kafka.zk.{KafkaZkClient, ZooKeeperTestHarness}
+import org.apache.kafka.common.acl.{AccessControlEntry, AclBinding, AclOperation}
+import org.apache.kafka.common.acl.AclOperation._
+import org.apache.kafka.common.acl.AclPermissionType.ALLOW
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.resource.PatternType.LITERAL
+import org.apache.kafka.common.resource.ResourcePattern
+import org.apache.kafka.common.resource.ResourceType.DELEGATION_TOKEN
 import org.apache.kafka.common.security.auth.KafkaPrincipal
 import org.apache.kafka.common.security.scram.internals.ScramMechanism
 import org.apache.kafka.common.security.token.delegation.internals.DelegationTokenCache
 import org.apache.kafka.common.security.token.delegation.{DelegationToken, TokenInformation}
 import org.apache.kafka.common.utils.{MockTime, SecurityUtils, Time}
+import org.apache.kafka.server.authorizer._
 import org.junit.Assert._
 import org.junit.{After, Before, Test}
 
@@ -228,8 +234,8 @@ class DelegationTokenManagerTest extends ZooKeeperTestHarness  {
     val renewer3 = SecurityUtils.parseKafkaPrincipal("User:renewer3")
     val renewer4 = SecurityUtils.parseKafkaPrincipal("User:renewer4")
 
-    val simpleAclAuthorizer = new SimpleAclAuthorizer
-    simpleAclAuthorizer.configure(config.originals)
+    val aclAuthorizer = new AclAuthorizer
+    aclAuthorizer.configure(config.originals)
 
     var hostSession = new Session(owner1, InetAddress.getByName("192.168.1.1"))
 
@@ -250,61 +256,72 @@ class DelegationTokenManagerTest extends ZooKeeperTestHarness  {
     assert(tokenManager.getAllTokenInformation().size == 4 )
 
     //get tokens non-exiting owner
-    var  tokens = getTokens(tokenManager, simpleAclAuthorizer, hostSession, owner1, List(SecurityUtils.parseKafkaPrincipal("User:unknown")))
+    var  tokens = getTokens(tokenManager, aclAuthorizer, hostSession, owner1, List(SecurityUtils.parseKafkaPrincipal("User:unknown")))
     assert(tokens.size == 0)
 
     //get all tokens for  empty owner list
-    tokens = getTokens(tokenManager, simpleAclAuthorizer, hostSession, owner1, List())
+    tokens = getTokens(tokenManager, aclAuthorizer, hostSession, owner1, List())
     assert(tokens.size == 0)
 
     //get all tokens for owner1
-    tokens = getTokens(tokenManager, simpleAclAuthorizer, hostSession, owner1, List(owner1))
+    tokens = getTokens(tokenManager, aclAuthorizer, hostSession, owner1, List(owner1))
     assert(tokens.size == 2)
 
     //get all tokens for owner1
-    tokens = getTokens(tokenManager, simpleAclAuthorizer, hostSession, owner1, null)
+    tokens = getTokens(tokenManager, aclAuthorizer, hostSession, owner1, null)
     assert(tokens.size == 2)
 
     //get all tokens for unknown owner
-    tokens = getTokens(tokenManager, simpleAclAuthorizer, hostSession, SecurityUtils.parseKafkaPrincipal("User:unknown"), null)
+    tokens = getTokens(tokenManager, aclAuthorizer, hostSession, SecurityUtils.parseKafkaPrincipal("User:unknown"), null)
     assert(tokens.size == 0)
 
     //get all tokens for multiple owners (owner1, renewer4) and without permission for renewer4
-    tokens = getTokens(tokenManager, simpleAclAuthorizer, hostSession, owner1, List(owner1, renewer4))
+    tokens = getTokens(tokenManager, aclAuthorizer, hostSession, owner1, List(owner1, renewer4))
     assert(tokens.size == 2)
 
+    def createAcl(aclBinding: AclBinding): Unit = {
+      val result = aclAuthorizer.createAcls(null, List(aclBinding).asJava).get(0)
+      if (result.failed)
+        throw result.exception
+    }
+
     //get all tokens for multiple owners (owner1, renewer4) and with permission
-    var acl = new Acl(owner1, Allow, WildCardHost, Describe)
-    simpleAclAuthorizer.addAcls(Set(acl), Resource(kafka.security.auth.DelegationToken, tokenId3, LITERAL))
-    tokens = getTokens(tokenManager, simpleAclAuthorizer, hostSession, owner1, List(owner1, renewer4))
+    createAcl(new AclBinding(new ResourcePattern(DELEGATION_TOKEN, tokenId3, LITERAL),
+      new AccessControlEntry(owner1.toString, WildCardHost, DESCRIBE, ALLOW)))
+    tokens = getTokens(tokenManager, aclAuthorizer, hostSession, owner1, List(owner1, renewer4))
     assert(tokens.size == 3)
 
     //get all tokens for renewer4 which is a renewer principal for some tokens
-    tokens = getTokens(tokenManager, simpleAclAuthorizer, hostSession,  renewer4, List(renewer4))
+    tokens = getTokens(tokenManager, aclAuthorizer, hostSession,  renewer4, List(renewer4))
     assert(tokens.size == 2)
 
     //get all tokens for multiple owners (renewer2, renewer3) which are token renewers principals and without permissions for renewer3
-    tokens = getTokens(tokenManager, simpleAclAuthorizer, hostSession,  renewer2, List(renewer2, renewer3))
+    tokens = getTokens(tokenManager, aclAuthorizer, hostSession,  renewer2, List(renewer2, renewer3))
     assert(tokens.size == 1)
 
     //get all tokens for multiple owners (renewer2, renewer3) which are token renewers principals and with permissions
-    hostSession = new Session(renewer2, InetAddress.getByName("192.168.1.1"))
-    acl = new Acl(renewer2, Allow, WildCardHost, Describe)
-    simpleAclAuthorizer.addAcls(Set(acl), Resource(kafka.security.auth.DelegationToken, tokenId2, LITERAL))
-    tokens = getTokens(tokenManager, simpleAclAuthorizer, hostSession,  renewer2, List(renewer2, renewer3))
+    hostSession = Session(renewer2, InetAddress.getByName("192.168.1.1"))
+    createAcl(new AclBinding(new ResourcePattern(DELEGATION_TOKEN, tokenId2, LITERAL),
+      new AccessControlEntry(renewer2.toString, WildCardHost, DESCRIBE, ALLOW)))
+    tokens = getTokens(tokenManager, aclAuthorizer, hostSession,  renewer2, List(renewer2, renewer3))
     assert(tokens.size == 2)
 
-    simpleAclAuthorizer.close()
+    aclAuthorizer.close()
   }
 
-  private def getTokens(tokenManager: DelegationTokenManager, simpleAclAuthorizer: SimpleAclAuthorizer, hostSession: Session,
+  private def getTokens(tokenManager: DelegationTokenManager, aclAuthorizer: AclAuthorizer, hostSession: Session,
                         requestPrincipal: KafkaPrincipal, requestedOwners: List[KafkaPrincipal]): List[DelegationToken] = {
 
     if (requestedOwners != null && requestedOwners.isEmpty) {
       List()
     }
     else {
-      def authorizeToken(tokenId: String) = simpleAclAuthorizer.authorize(hostSession, Describe, Resource(kafka.security.auth.DelegationToken, tokenId, LITERAL))
+      def authorizeToken(tokenId: String) = {
+        val requestContext = AuthorizerUtils.sessionToRequestContext(hostSession)
+        val action = new Action(AclOperation.DESCRIBE,
+          new ResourcePattern(DELEGATION_TOKEN, tokenId, LITERAL), 1, true, true)
+        aclAuthorizer.authorize(requestContext, List(action).asJava).asScala.head == AuthorizationResult.ALLOWED
+      }
       def eligible(token: TokenInformation) = DelegationTokenManager.filterToken(requestPrincipal, Option(requestedOwners), token, authorizeToken)
       tokenManager.getTokens(eligible)
     }
diff --git a/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala b/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala
index e9472ae7c5255..2c3ec5f43c21f 100755
--- a/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala
+++ b/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala
@@ -17,14 +17,17 @@
 
 package kafka.server
 
-import java.util
+import java.{lang, util}
 import java.util.Properties
+import java.util.concurrent.CompletableFuture
 
 import kafka.utils.TestUtils
 import kafka.zk.KafkaZkClient
-import org.apache.kafka.common.Reconfigurable
+import org.apache.kafka.common.{Endpoint, Reconfigurable}
+import org.apache.kafka.common.acl.{AclBinding, AclBindingFilter}
 import org.apache.kafka.common.config.types.Password
 import org.apache.kafka.common.config.{ConfigException, SslConfigs}
+import org.apache.kafka.server.authorizer._
 import org.easymock.EasyMock
 import org.junit.Assert._
 import org.junit.Test
@@ -321,6 +324,43 @@ class DynamicBrokerConfigTest {
     dynamicListenerConfig.validateReconfiguration(newConfig)
   }
 
+  @Test
+  def testAuthorizerConfig(): Unit = {
+    val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 9092)
+    val oldConfig =  KafkaConfig.fromProps(props)
+    val kafkaServer: KafkaServer = EasyMock.createMock(classOf[kafka.server.KafkaServer])
+
+    class TestAuthorizer extends Authorizer with Reconfigurable {
+      @volatile var superUsers = ""
+      override def acls(filter: AclBindingFilter): lang.Iterable[AclBinding] = null
+      override def start(serverInfo: AuthorizerServerInfo): util.Map[Endpoint, CompletableFuture[Void]] = Map.empty.asJava
+      override def deleteAcls(requestContext: AuthorizableRequestContext, aclBindingFilters: util.List[AclBindingFilter]): util.List[AclDeleteResult] = null
+      override def createAcls(requestContext: AuthorizableRequestContext, aclBindings: util.List[AclBinding]): util.List[AclCreateResult] = null
+      override def authorize(requestContext: AuthorizableRequestContext, actions: util.List[Action]): util.List[AuthorizationResult] = null
+      override def configure(configs: util.Map[String, _]): Unit = {}
+      override def close(): Unit = {}
+      override def reconfigurableConfigs(): util.Set[String] = Set("super.users").asJava
+      override def validateReconfiguration(configs: util.Map[String, _]): Unit = {}
+      override def reconfigure(configs: util.Map[String, _]): Unit = {
+        superUsers = configs.get("super.users").toString
+      }
+    }
+
+    val authorizer = new TestAuthorizer
+    EasyMock.expect(kafkaServer.config).andReturn(oldConfig).anyTimes()
+    EasyMock.expect(kafkaServer.authorizer).andReturn(Some(authorizer)).anyTimes()
+    EasyMock.replay(kafkaServer)
+    try {
+      kafkaServer.config.dynamicConfig.addReconfigurables(kafkaServer)
+    } catch {
+      case _: Throwable => // We are only testing authorizer reconfiguration, ignore any exceptions due to incomplete mock
+    }
+
+    props.put("super.users", "User:admin")
+    kafkaServer.config.dynamicConfig.updateBrokerConfig(0, props)
+    assertEquals("User:admin", authorizer.superUsers)
+  }
+
   @Test
   def testSynonyms(): Unit = {
     assertEquals(List("listener.name.secure.ssl.keystore.type", "ssl.keystore.type"),
diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
index 37610403a3925..1e0f9aa15b247 100644
--- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
@@ -29,7 +29,6 @@ import kafka.coordinator.group.GroupCoordinator
 import kafka.coordinator.transaction.TransactionCoordinator
 import kafka.network.RequestChannel
 import kafka.network.RequestChannel.SendResponse
-import kafka.security.auth.Authorizer
 import kafka.server.QuotaFactory.QuotaManagers
 import kafka.utils.{MockTime, TestUtils}
 import kafka.zk.KafkaZkClient
@@ -52,6 +51,7 @@ import org.apache.kafka.common.message.{HeartbeatRequestData, JoinGroupRequestDa
 import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocol
 import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
 import org.apache.kafka.common.replica.ClientMetadata
+import org.apache.kafka.server.authorizer.Authorizer
 import org.junit.Assert.{assertEquals, assertNull, assertTrue}
 import org.junit.{After, Test}
 
diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
index 882a7c2702b15..f302105f97724 100644
--- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
+++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
@@ -14,12 +14,13 @@
 
 package kafka.server
 
+import java.util
 import java.util.{Collections, LinkedHashMap, Optional, Properties}
 import java.util.concurrent.{Executors, Future, TimeUnit}
 
 import kafka.log.LogConfig
 import kafka.network.RequestChannel.Session
-import kafka.security.auth._
+import kafka.security.authorizer.AclAuthorizer
 import kafka.utils.TestUtils
 import org.apache.kafka.common.{ElectionType, Node, TopicPartition}
 import org.apache.kafka.common.acl._
@@ -38,6 +39,7 @@ import org.apache.kafka.common.requests._
 import org.apache.kafka.common.resource.{PatternType, ResourcePattern, ResourcePatternFilter, ResourceType => AdminResourceType}
 import org.apache.kafka.common.security.auth.{AuthenticationContext, KafkaPrincipal, KafkaPrincipalBuilder, SecurityProtocol}
 import org.apache.kafka.common.utils.{Sanitizer, SecurityUtils}
+import org.apache.kafka.server.authorizer.{Action, AuthorizableRequestContext, AuthorizationResult}
 import org.junit.Assert._
 import org.junit.{After, Before, Test}
 
@@ -541,7 +543,7 @@ class RequestQuotaTest extends BaseRequestTest {
       case ApiKeys.API_VERSIONS => new ApiVersionsResponse(response).throttleTimeMs
       case ApiKeys.CREATE_TOPICS =>
         new CreateTopicsResponse(response, ApiKeys.CREATE_TOPICS.latestVersion).throttleTimeMs
-      case ApiKeys.DELETE_TOPICS => 
+      case ApiKeys.DELETE_TOPICS =>
         new DeleteTopicsResponse(response, ApiKeys.DELETE_TOPICS.latestVersion).throttleTimeMs
       case ApiKeys.DELETE_RECORDS => new DeleteRecordsResponse(response).throttleTimeMs
       case ApiKeys.INIT_PRODUCER_ID => new InitProducerIdResponse(response, ApiKeys.INIT_PRODUCER_ID.latestVersion).throttleTimeMs
@@ -649,9 +651,11 @@ object RequestQuotaTest {
   // Principal used for all client connections. This is modified by tests which
   // check unauthorized code path
   var principal = KafkaPrincipal.ANONYMOUS
-  class TestAuthorizer extends SimpleAclAuthorizer {
-    override def authorize(session: Session, operation: Operation, resource: Resource): Boolean = {
-      session.principal != UnauthorizedPrincipal
+  class TestAuthorizer extends AclAuthorizer {
+    override def authorize(requestContext: AuthorizableRequestContext, actions: util.List[Action]): util.List[AuthorizationResult] = {
+      actions.asScala.map { _ =>
+        if (requestContext.principal != UnauthorizedPrincipal) AuthorizationResult.ALLOWED else AuthorizationResult.DENIED
+      }.asJava
     }
   }
   class TestPrincipalBuilder extends KafkaPrincipalBuilder {
diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index 2a27ed4e52cb4..074c107f5bb10 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -28,8 +28,8 @@ import java.util.Arrays
 import java.util.Collections
 import java.util.Properties
 import java.util.concurrent.{Callable, ExecutionException, Executors, TimeUnit}
-
 import javax.net.ssl.X509TrustManager
+
 import kafka.api._
 import kafka.cluster.{Broker, EndPoint}
 import kafka.log._
@@ -45,17 +45,19 @@ import org.apache.kafka.clients.admin.AlterConfigOp.OpType
 import org.apache.kafka.clients.admin._
 import org.apache.kafka.clients.consumer._
 import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig, ProducerRecord}
+import org.apache.kafka.common.acl.{AccessControlEntry, AccessControlEntryFilter, AclBindingFilter}
 import org.apache.kafka.common.{KafkaFuture, TopicPartition}
 import org.apache.kafka.common.config.ConfigResource
-import org.apache.kafka.common.errors.RetriableException
 import org.apache.kafka.common.header.Header
 import org.apache.kafka.common.internals.Topic
 import org.apache.kafka.common.network.{ListenerName, Mode}
 import org.apache.kafka.common.record._
+import org.apache.kafka.common.resource.ResourcePattern
 import org.apache.kafka.common.security.auth.SecurityProtocol
 import org.apache.kafka.common.serialization.{ByteArrayDeserializer, ByteArraySerializer, Deserializer, IntegerSerializer, Serializer}
 import org.apache.kafka.common.utils.Time
 import org.apache.kafka.common.utils.Utils._
+import org.apache.kafka.server.authorizer.{Authorizer => JAuthorizer}
 import org.apache.kafka.test.{TestSslUtils, TestUtils => JTestUtils}
 import org.apache.zookeeper.KeeperException.SessionExpiredException
 import org.apache.zookeeper.ZooDefs._
@@ -1171,6 +1173,15 @@ object TestUtils extends Logging {
     trustManager
   }
 
+  def waitAndVerifyAcls(expected: Set[AccessControlEntry], authorizer: JAuthorizer, resource: ResourcePattern) = {
+    val newLine = scala.util.Properties.lineSeparator
+
+    val filter = new AclBindingFilter(resource.toFilter, AccessControlEntryFilter.ANY)
+    waitUntilTrue(() => authorizer.acls(filter).asScala.map(_.entry).toSet == expected,
+      s"expected acls:${expected.mkString(newLine + "\t", newLine + "\t", newLine)}" +
+        s"but got:${authorizer.acls(filter).asScala.map(_.entry).mkString(newLine + "\t", newLine + "\t", newLine)}", waitTimeMs = JTestUtils.DEFAULT_MAX_WAIT_MS)
+  }
+
   def waitAndVerifyAcls(expected: Set[Acl], authorizer: Authorizer, resource: Resource) = {
     val newLine = scala.util.Properties.lineSeparator
 

From 88d1b6de1f0e5b5228743f4f54ac01a2a153adf9 Mon Sep 17 00:00:00 2001
From: teebee 
Date: Mon, 2 Sep 2019 23:32:49 +0530
Subject: [PATCH 0577/1071] KAFKA-8860: Let SslPrincipalMapper split SSL
 principal mapping rules

Author: teebee 

Reviewers: Reviewers: Manikumar Reddy 

Closes #7140 from teebee/teebee/ssl-principal-mapping-rules-handling
---
 .../internals/BrokerSecurityConfigs.java      |   2 +-
 .../common/network/SslChannelBuilder.java     |   4 +-
 .../security/ssl/SslPrincipalMapper.java      |  62 +++++-----
 .../DefaultKafkaPrincipalBuilderTest.java     |   4 +-
 .../security/ssl/SslPrincipalMapperTest.java  | 114 ++++++++++--------
 .../main/scala/kafka/server/KafkaConfig.scala |   2 +-
 6 files changed, 98 insertions(+), 90 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/config/internals/BrokerSecurityConfigs.java b/clients/src/main/java/org/apache/kafka/common/config/internals/BrokerSecurityConfigs.java
index 98f6467a48429..de54f164be598 100644
--- a/clients/src/main/java/org/apache/kafka/common/config/internals/BrokerSecurityConfigs.java
+++ b/clients/src/main/java/org/apache/kafka/common/config/internals/BrokerSecurityConfigs.java
@@ -55,7 +55,7 @@ public class BrokerSecurityConfigs {
             " see  security authorization and acls. Note that this configuration is ignored" +
             " if an extension of KafkaPrincipalBuilder is provided by the " + PRINCIPAL_BUILDER_CLASS_CONFIG + "" +
            " configuration.";
-    public static final List DEFAULT_SSL_PRINCIPAL_MAPPING_RULES = Collections.singletonList("DEFAULT");
+    public static final String DEFAULT_SSL_PRINCIPAL_MAPPING_RULES = "DEFAULT";
 
     public static final String SASL_KERBEROS_PRINCIPAL_TO_LOCAL_RULES_DOC = "A list of rules for mapping from principal " +
             "names to short names (typically operating system usernames). The rules are evaluated in order and the " +
diff --git a/clients/src/main/java/org/apache/kafka/common/network/SslChannelBuilder.java b/clients/src/main/java/org/apache/kafka/common/network/SslChannelBuilder.java
index 50081999a27cb..aade37c296973 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/SslChannelBuilder.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/SslChannelBuilder.java
@@ -35,7 +35,6 @@
 import java.net.InetSocketAddress;
 import java.nio.channels.SelectionKey;
 import java.nio.channels.SocketChannel;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Supplier;
@@ -63,8 +62,7 @@ public SslChannelBuilder(Mode mode, ListenerName listenerName, boolean isInterBr
     public void configure(Map configs) throws KafkaException {
         try {
             this.configs = configs;
-            @SuppressWarnings("unchecked")
-            List sslPrincipalMappingRules = (List) configs.get(BrokerSecurityConfigs.SSL_PRINCIPAL_MAPPING_RULES_CONFIG);
+            String sslPrincipalMappingRules = (String) configs.get(BrokerSecurityConfigs.SSL_PRINCIPAL_MAPPING_RULES_CONFIG);
             if (sslPrincipalMappingRules != null)
                 sslPrincipalMapper = SslPrincipalMapper.fromRules(sslPrincipalMappingRules);
             this.sslFactory = new SslFactory(mode, null, isInterBrokerListener);
diff --git a/clients/src/main/java/org/apache/kafka/common/security/ssl/SslPrincipalMapper.java b/clients/src/main/java/org/apache/kafka/common/security/ssl/SslPrincipalMapper.java
index 3b95e1a5f4d41..977fdc332cbfa 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/ssl/SslPrincipalMapper.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/ssl/SslPrincipalMapper.java
@@ -18,53 +18,44 @@
 
 import java.io.IOException;
 import java.util.List;
-import java.util.Collections;
 import java.util.ArrayList;
 import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import static org.apache.kafka.common.config.internals.BrokerSecurityConfigs.DEFAULT_SSL_PRINCIPAL_MAPPING_RULES;
+
 public class SslPrincipalMapper {
 
-    private static final Pattern RULE_PARSER = Pattern.compile("((DEFAULT)|(RULE:(([^/]*)/([^/]*))/([LU])?))");
+    private static final String RULE_PATTERN = "(DEFAULT)|RULE:((\\\\.|[^\\\\/])*)/((\\\\.|[^\\\\/])*)/([LU]?).*?|(.*?)";
+    private static final Pattern RULE_SPLITTER = Pattern.compile("\\s*(" + RULE_PATTERN + ")\\s*(,\\s*|$)");
+    private static final Pattern RULE_PARSER = Pattern.compile(RULE_PATTERN);
 
     private final List rules;
 
-    public SslPrincipalMapper(List sslPrincipalMappingRules) {
-        this.rules = sslPrincipalMappingRules;
+    public SslPrincipalMapper(String sslPrincipalMappingRules) {
+        this.rules = parseRules(splitRules(sslPrincipalMappingRules));
     }
 
-    public static SslPrincipalMapper fromRules(List sslPrincipalMappingRules) {
-        List rules = sslPrincipalMappingRules == null ? Collections.singletonList("DEFAULT") : sslPrincipalMappingRules;
-        return new SslPrincipalMapper(parseRules(rules));
+    public static SslPrincipalMapper fromRules(String sslPrincipalMappingRules) {
+        return new SslPrincipalMapper(sslPrincipalMappingRules);
     }
 
-    private static List joinSplitRules(List rules) {
-        String rule = "RULE:";
-        String defaultRule = "DEFAULT";
-        List retVal = new ArrayList<>();
-        StringBuilder currentRule = new StringBuilder();
-        for (String r : rules) {
-            if (currentRule.length() > 0) {
-                if (r.startsWith(rule) || r.equals(defaultRule)) {
-                    retVal.add(currentRule.toString());
-                    currentRule.setLength(0);
-                    currentRule.append(r);
-                } else {
-                    currentRule.append(String.format(",%s", r));
-                }
-            } else {
-                currentRule.append(r);
-            }
+    private static List splitRules(String sslPrincipalMappingRules) {
+        if (sslPrincipalMappingRules == null) {
+            sslPrincipalMappingRules = DEFAULT_SSL_PRINCIPAL_MAPPING_RULES;
         }
-        if (currentRule.length() > 0) {
-            retVal.add(currentRule.toString());
+
+        List result = new ArrayList<>();
+        Matcher matcher = RULE_SPLITTER.matcher(sslPrincipalMappingRules.trim());
+        while (matcher.find()) {
+            result.add(matcher.group(1));
         }
-        return retVal;
+
+        return result;
     }
 
     private static List parseRules(List rules) {
-        rules = joinSplitRules(rules);
         List result = new ArrayList<>();
         for (String rule : rules) {
             Matcher matcher = RULE_PARSER.matcher(rule);
@@ -74,15 +65,18 @@ private static List parseRules(List rules) {
             if (rule.length() != matcher.end()) {
                 throw new IllegalArgumentException("Invalid rule: `" + rule + "`, unmatched substring: `" + rule.substring(matcher.end()) + "`");
             }
-            if (matcher.group(2) != null) {
+
+            // empty rules are ignored
+            if (matcher.group(1) != null) {
                 result.add(new Rule());
-            } else {
-                result.add(new Rule(matcher.group(5),
-                                    matcher.group(6),
-                                    "L".equals(matcher.group(7)),
-                                    "U".equals(matcher.group(7))));
+            } else if (matcher.group(2) != null) {
+                result.add(new Rule(matcher.group(2),
+                                    matcher.group(4),
+                                    "L".equals(matcher.group(6)),
+                                    "U".equals(matcher.group(6))));
             }
         }
+
         return result;
     }
 
diff --git a/clients/src/test/java/org/apache/kafka/common/security/auth/DefaultKafkaPrincipalBuilderTest.java b/clients/src/test/java/org/apache/kafka/common/security/auth/DefaultKafkaPrincipalBuilderTest.java
index dd5087a84b387..dc1d6225e66d9 100644
--- a/clients/src/test/java/org/apache/kafka/common/security/auth/DefaultKafkaPrincipalBuilderTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/security/auth/DefaultKafkaPrincipalBuilderTest.java
@@ -30,8 +30,6 @@
 import javax.security.sasl.SaslServer;
 import java.net.InetAddress;
 import java.security.Principal;
-import java.util.Arrays;
-import java.util.List;
 
 import static org.junit.Assert.assertEquals;
 import static org.mockito.ArgumentMatchers.any;
@@ -143,7 +141,7 @@ public void testPrincipalWithSslPrincipalMapper() throws Exception {
                                         .thenReturn(new X500Principal("CN=duke, OU=JavaSoft, O=Sun Microsystems"))
                                         .thenReturn(new X500Principal("OU=JavaSoft, O=Sun Microsystems, C=US"));
 
-        List rules = Arrays.asList(
+        String rules = String.join(", ",
             "RULE:^CN=(.*),OU=ServiceUsers.*$/$1/L",
             "RULE:^CN=(.*),OU=(.*),O=(.*),L=(.*),ST=(.*),C=(.*)$/$1@$2/L",
             "RULE:^.*[Cc][Nn]=([a-zA-Z0-9.]*).*$/$1/U",
diff --git a/clients/src/test/java/org/apache/kafka/common/security/ssl/SslPrincipalMapperTest.java b/clients/src/test/java/org/apache/kafka/common/security/ssl/SslPrincipalMapperTest.java
index 56ef977a1120a..7b7b67b1dbc3a 100644
--- a/clients/src/test/java/org/apache/kafka/common/security/ssl/SslPrincipalMapperTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/security/ssl/SslPrincipalMapperTest.java
@@ -18,9 +18,6 @@
 
 import org.junit.Test;
 
-import java.util.Arrays;
-import java.util.List;
-
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
@@ -28,59 +25,37 @@ public class SslPrincipalMapperTest {
 
     @Test
     public void testValidRules() {
-        testValidRule(Arrays.asList("DEFAULT"));
-        testValidRule(Arrays.asList("RULE:^CN=(.*?),OU=ServiceUsers.*$/$1/"));
-        testValidRule(Arrays.asList("RULE:^CN=(.*?),OU=ServiceUsers.*$/$1/L", "DEFAULT"));
-        testValidRule(Arrays.asList("RULE:^CN=(.*?),OU=(.*?),O=(.*?),L=(.*?),ST=(.*?),C=(.*?)$/$1@$2/"));
-        testValidRule(Arrays.asList("RULE:^.*[Cc][Nn]=([a-zA-Z0-9.]*).*$/$1/L"));
-        testValidRule(Arrays.asList("RULE:^cn=(.?),ou=(.?),dc=(.?),dc=(.?)$/$1@$2/U"));
+        testValidRule("DEFAULT");
+        testValidRule("RULE:^CN=(.*?),OU=ServiceUsers.*$/$1/");
+        testValidRule("RULE:^CN=(.*?),OU=ServiceUsers.*$/$1/L, DEFAULT");
+        testValidRule("RULE:^CN=(.*?),OU=(.*?),O=(.*?),L=(.*?),ST=(.*?),C=(.*?)$/$1@$2/");
+        testValidRule("RULE:^.*[Cc][Nn]=([a-zA-Z0-9.]*).*$/$1/L");
+        testValidRule("RULE:^cn=(.?),ou=(.?),dc=(.?),dc=(.?)$/$1@$2/U");
+
+        testValidRule("RULE:^CN=([^,ADEFLTU,]+)(,.*|$)/$1/");
+        testValidRule("RULE:^CN=([^,DEFAULT,]+)(,.*|$)/$1/");
     }
 
-    @Test
-    public void testValidSplitRules() {
-        testValidRule(Arrays.asList("DEFAULT"));
-        testValidRule(Arrays.asList("RULE:^CN=(.*?)", "OU=ServiceUsers.*$/$1/"));
-        testValidRule(Arrays.asList("RULE:^CN=(.*?)", "OU=ServiceUsers.*$/$1/L", "DEFAULT"));
-        testValidRule(Arrays.asList("RULE:^CN=(.*?)", "OU=(.*?),O=(.*?),L=(.*?)", "ST=(.*?)", "C=(.*?)$/$1@$2/"));
-        testValidRule(Arrays.asList("RULE:^.*[Cc][Nn]=([a-zA-Z0-9.]*).*$/$1/L"));
-        testValidRule(Arrays.asList("RULE:^cn=(.?)", "ou=(.?)", "dc=(.?)", "dc=(.?)$/$1@$2/U"));
-    }
-
-    private void testValidRule(List rules) {
+    private void testValidRule(String rules) {
         SslPrincipalMapper.fromRules(rules);
     }
 
     @Test
     public void testInvalidRules() {
-        testInvalidRule(Arrays.asList("default"));
-        testInvalidRule(Arrays.asList("DEFAUL"));
-        testInvalidRule(Arrays.asList("DEFAULT/L"));
-        testInvalidRule(Arrays.asList("DEFAULT/U"));
-
-        testInvalidRule(Arrays.asList("RULE:CN=(.*?),OU=ServiceUsers.*/$1"));
-        testInvalidRule(Arrays.asList("rule:^CN=(.*?),OU=ServiceUsers.*$/$1/"));
-        testInvalidRule(Arrays.asList("RULE:^CN=(.*?),OU=ServiceUsers.*$/$1/L/U"));
-        testInvalidRule(Arrays.asList("RULE:^CN=(.*?),OU=ServiceUsers.*$/L"));
-        testInvalidRule(Arrays.asList("RULE:^CN=(.*?),OU=ServiceUsers.*$/U"));
-        testInvalidRule(Arrays.asList("RULE:^CN=(.*?),OU=ServiceUsers.*$/LU"));
+        testInvalidRule("default");
+        testInvalidRule("DEFAUL");
+        testInvalidRule("DEFAULT/L");
+        testInvalidRule("DEFAULT/U");
+
+        testInvalidRule("RULE:CN=(.*?),OU=ServiceUsers.*/$1");
+        testInvalidRule("rule:^CN=(.*?),OU=ServiceUsers.*$/$1/");
+        testInvalidRule("RULE:^CN=(.*?),OU=ServiceUsers.*$/$1/L/U");
+        testInvalidRule("RULE:^CN=(.*?),OU=ServiceUsers.*$/L");
+        testInvalidRule("RULE:^CN=(.*?),OU=ServiceUsers.*$/U");
+        testInvalidRule("RULE:^CN=(.*?),OU=ServiceUsers.*$/LU");
     }
 
-    @Test
-    public void testInvalidSplitRules() {
-        testInvalidRule(Arrays.asList("default"));
-        testInvalidRule(Arrays.asList("DEFAUL"));
-        testInvalidRule(Arrays.asList("DEFAULT/L"));
-        testInvalidRule(Arrays.asList("DEFAULT/U"));
-
-        testInvalidRule(Arrays.asList("RULE:CN=(.*?)", "OU=ServiceUsers.*/$1"));
-        testInvalidRule(Arrays.asList("rule:^CN=(.*?)", "OU=ServiceUsers.*$/$1/"));
-        testInvalidRule(Arrays.asList("RULE:^CN=(.*?)", "OU=ServiceUsers.*$/$1/L/U"));
-        testInvalidRule(Arrays.asList("RULE:^CN=(.*?)", "OU=ServiceUsers.*$/L"));
-        testInvalidRule(Arrays.asList("RULE:^CN=(.*?)", "OU=ServiceUsers.*$/U"));
-        testInvalidRule(Arrays.asList("RULE:^CN=(.*?)", "OU=ServiceUsers.*$/LU"));
-    }
-
-    private void testInvalidRule(List rules) {
+    private void testInvalidRule(String rules) {
         try {
             System.out.println(SslPrincipalMapper.fromRules(rules));
             fail("should have thrown IllegalArgumentException");
@@ -90,7 +65,7 @@ private void testInvalidRule(List rules) {
 
     @Test
     public void testSslPrincipalMapper() throws Exception {
-        List rules = Arrays.asList(
+        String rules = String.join(", ",
             "RULE:^CN=(.*?),OU=ServiceUsers.*$/$1/L",
             "RULE:^CN=(.*?),OU=(.*?),O=(.*?),L=(.*?),ST=(.*?),C=(.*?)$/$1@$2/L",
             "RULE:^cn=(.*?),ou=(.*?),dc=(.*?),dc=(.*?)$/$1@$2/U",
@@ -107,4 +82,47 @@ public void testSslPrincipalMapper() throws Exception {
         assertEquals("OU=JavaSoft,O=Sun Microsystems,C=US", mapper.getName("OU=JavaSoft,O=Sun Microsystems,C=US"));
     }
 
+    private void testRulesSplitting(String expected, String rules) {
+        SslPrincipalMapper mapper = SslPrincipalMapper.fromRules(rules);
+        assertEquals(String.format("SslPrincipalMapper(rules = %s)", expected), mapper.toString());
+    }
+
+    @Test
+    public void testRulesSplitting() {
+        // seeing is believing
+        testRulesSplitting("[]", "");
+        testRulesSplitting("[DEFAULT]", "DEFAULT");
+        testRulesSplitting("[RULE:/]", "RULE://");
+        testRulesSplitting("[RULE:/.*]", "RULE:/.*/");
+        testRulesSplitting("[RULE:/.*/L]", "RULE:/.*/L");
+        testRulesSplitting("[RULE:/, DEFAULT]", "RULE://,DEFAULT");
+        testRulesSplitting("[RULE:/, DEFAULT]", "  RULE:// ,  DEFAULT  ");
+        testRulesSplitting("[RULE:   /     , DEFAULT]", "  RULE:   /     / ,  DEFAULT  ");
+        testRulesSplitting("[RULE:  /     /U, DEFAULT]", "  RULE:  /     /U   ,DEFAULT  ");
+        testRulesSplitting("[RULE:([A-Z]*)/$1/U, RULE:([a-z]+)/$1, DEFAULT]", "  RULE:([A-Z]*)/$1/U   ,RULE:([a-z]+)/$1/,   DEFAULT  ");
+
+        // empty rules are ignored
+        testRulesSplitting("[]", ",   , , ,      , , ,   ");
+        testRulesSplitting("[RULE:/, DEFAULT]", ",,RULE://,,,DEFAULT,,");
+        testRulesSplitting("[RULE: /   , DEFAULT]", ",  , RULE: /   /    ,,,   DEFAULT, ,   ");
+        testRulesSplitting("[RULE:   /  /U, DEFAULT]", "     ,  , RULE:   /  /U    ,,  ,DEFAULT, ,");
+
+        // escape sequences
+        testRulesSplitting("[RULE:\\/\\\\\\(\\)\\n\\t/\\/\\/]", "RULE:\\/\\\\\\(\\)\\n\\t/\\/\\//");
+        testRulesSplitting("[RULE:\\**\\/+/*/L, RULE:\\/*\\**/**]", "RULE:\\**\\/+/*/L,RULE:\\/*\\**/**/");
+
+        // rules rule
+        testRulesSplitting(
+            "[RULE:,RULE:,/,RULE:,\\//U, RULE:,/RULE:,, RULE:,RULE:,/L,RULE:,/L, RULE:, DEFAULT, /DEFAULT, DEFAULT]",
+            "RULE:,RULE:,/,RULE:,\\//U,RULE:,/RULE:,/,RULE:,RULE:,/L,RULE:,/L,RULE:, DEFAULT, /DEFAULT/,DEFAULT"
+        );
+    }
+
+    @Test
+    public void testCommaWithWhitespace() throws Exception {
+        String rules = "RULE:^CN=((\\\\, *|\\w)+)(,.*|$)/$1/,DEFAULT";
+
+        SslPrincipalMapper mapper = SslPrincipalMapper.fromRules(rules);
+        assertEquals("Tkac\\, Adam", mapper.getName("CN=Tkac\\, Adam,OU=ITZ,DC=geodis,DC=cz"));
+    }
 }
diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index 97f894cb5a600..4d6a65d79a13d 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -1059,7 +1059,7 @@ object KafkaConfig {
       .define(SslSecureRandomImplementationProp, STRING, null, LOW, SslSecureRandomImplementationDoc)
       .define(SslClientAuthProp, STRING, Defaults.SslClientAuthentication, in(Defaults.SslClientAuthenticationValidValues:_*), MEDIUM, SslClientAuthDoc)
       .define(SslCipherSuitesProp, LIST, Collections.emptyList(), MEDIUM, SslCipherSuitesDoc)
-      .define(SslPrincipalMappingRulesProp, LIST, Defaults.SslPrincipalMappingRules, LOW, SslPrincipalMappingRulesDoc)
+      .define(SslPrincipalMappingRulesProp, STRING, Defaults.SslPrincipalMappingRules, LOW, SslPrincipalMappingRulesDoc)
 
       /** ********* Sasl Configuration ****************/
       .define(SaslMechanismInterBrokerProtocolProp, STRING, Defaults.SaslMechanismInterBrokerProtocol, MEDIUM, SaslMechanismInterBrokerProtocolDoc)

From 2c0157765f3fa174c48520fc8cf7dc542cbf7273 Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Tue, 3 Sep 2019 07:27:03 -0700
Subject: [PATCH 0578/1071] MINOR: Use new `Admin` interface instead of
 `KafkaAdminClient` class (#7232)

Reviewers: Guozhang Wang , Ismael Juma , Bill Bejeck 
---
 .../scala/kafka/tools/StreamsResetter.java    | 48 +++++++++----------
 1 file changed, 22 insertions(+), 26 deletions(-)

diff --git a/core/src/main/scala/kafka/tools/StreamsResetter.java b/core/src/main/scala/kafka/tools/StreamsResetter.java
index e61b096ab495b..597b9d34c239d 100644
--- a/core/src/main/scala/kafka/tools/StreamsResetter.java
+++ b/core/src/main/scala/kafka/tools/StreamsResetter.java
@@ -26,7 +26,6 @@
 import org.apache.kafka.clients.admin.DeleteTopicsResult;
 import org.apache.kafka.clients.admin.DescribeConsumerGroupsOptions;
 import org.apache.kafka.clients.admin.DescribeConsumerGroupsResult;
-import org.apache.kafka.clients.admin.KafkaAdminClient;
 import org.apache.kafka.clients.admin.MemberDescription;
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
@@ -136,8 +135,7 @@ public int run(final String[] args,
                    final Properties config) {
         int exitCode;
 
-        KafkaAdminClient kafkaAdminClient = null;
-
+        Admin adminClient = null;
         try {
             parseArguments(args);
 
@@ -150,11 +148,11 @@ public int run(final String[] args,
             }
             properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, options.valueOf(bootstrapServerOption));
 
-            kafkaAdminClient = (KafkaAdminClient) Admin.create(properties);
-            validateNoActiveConsumers(groupId, kafkaAdminClient);
+            adminClient = Admin.create(properties);
+            validateNoActiveConsumers(groupId, adminClient);
 
             allTopics.clear();
-            allTopics.addAll(kafkaAdminClient.listTopics().names().get(60, TimeUnit.SECONDS));
+            allTopics.addAll(adminClient.listTopics().names().get(60, TimeUnit.SECONDS));
 
             if (dryRun) {
                 System.out.println("----Dry run displays the actions which will be performed when running Streams Reset Tool----");
@@ -163,15 +161,15 @@ public int run(final String[] args,
             final HashMap consumerConfig = new HashMap<>(config);
             consumerConfig.putAll(properties);
             exitCode = maybeResetInputAndSeekToEndIntermediateTopicOffsets(consumerConfig, dryRun);
-            maybeDeleteInternalTopics(kafkaAdminClient, dryRun);
+            maybeDeleteInternalTopics(adminClient, dryRun);
 
         } catch (final Throwable e) {
             exitCode = EXIT_CODE_ERROR;
             System.err.println("ERROR: " + e);
             e.printStackTrace(System.err);
         } finally {
-            if (kafkaAdminClient != null) {
-                kafkaAdminClient.close(Duration.ofSeconds(60));
+            if (adminClient != null) {
+                adminClient.close(Duration.ofSeconds(60));
             }
         }
 
@@ -182,8 +180,9 @@ private void validateNoActiveConsumers(final String groupId,
                                            final Admin adminClient)
         throws ExecutionException, InterruptedException {
 
-        final DescribeConsumerGroupsResult describeResult = adminClient.describeConsumerGroups(Collections.singleton(groupId),
-                (new DescribeConsumerGroupsOptions()).timeoutMs(10 * 1000));
+        final DescribeConsumerGroupsResult describeResult = adminClient.describeConsumerGroups(
+            Collections.singleton(groupId),
+            new DescribeConsumerGroupsOptions().timeoutMs(10 * 1000));
         final List members =
             new ArrayList<>(describeResult.describedGroups().get(groupId).get().members());
         if (!members.isEmpty()) {
@@ -193,8 +192,7 @@ private void validateNoActiveConsumers(final String groupId,
         }
     }
 
-    private void parseArguments(final String[] args) throws IOException {
-
+    private void parseArguments(final String[] args) {
         final OptionParser optionParser = new OptionParser(false);
         applicationIdOption = optionParser.accepts("application-id", "The Kafka Streams application ID (application.id).")
             .withRequiredArg()
@@ -280,11 +278,17 @@ private void parseArguments(final String[] args) throws IOException {
         checkInvalidArgs(optionParser, options, allScenarioOptions, shiftByOption);
     }
 
-    private  void checkInvalidArgs(OptionParser optionParser, OptionSet options, Set> allOptions,
-                                  OptionSpec option) {
-        Set> invalidOptions = new HashSet<>(allOptions);
+    private  void checkInvalidArgs(final OptionParser optionParser,
+                                      final OptionSet options,
+                                      final Set> allOptions,
+                                      final OptionSpec option) {
+        final Set> invalidOptions = new HashSet<>(allOptions);
         invalidOptions.remove(option);
-        CommandLineUtils.checkInvalidArgs(optionParser, options, option, JavaConverters.asScalaSetConverter(invalidOptions).asScala());
+        CommandLineUtils.checkInvalidArgs(
+            optionParser,
+            options,
+            option,
+            JavaConverters.asScalaSetConverter(invalidOptions).asScala());
     }
 
     private int maybeResetInputAndSeekToEndIntermediateTopicOffsets(final Map consumerConfig,
@@ -408,7 +412,6 @@ public void maybeSeekToEnd(final String groupId,
                     System.out.println("Topic: " + topicPartition.topic());
                 }
             }
-
             client.seekToEnd(intermediateTopicPartitions);
         }
     }
@@ -626,8 +629,7 @@ private boolean isIntermediateTopic(final String topic) {
         return options.valuesOf(intermediateTopicsOption).contains(topic);
     }
 
-    private void maybeDeleteInternalTopics(final KafkaAdminClient adminClient, final boolean dryRun) {
-
+    private void maybeDeleteInternalTopics(final Admin adminClient, final boolean dryRun) {
         System.out.println("Deleting all internal/auto-created topics for application " + options.valueOf(applicationIdOption));
         final List topicsToDelete = new ArrayList<>();
         for (final String listing : allTopics) {
@@ -666,7 +668,6 @@ public void doDelete(final List topicsToDelete,
         }
     }
 
-
     private boolean isInternalTopic(final String topicName) {
         // Specified input/intermediate topics might be named like internal topics (by chance).
         // Even is this is not expected in general, we need to exclude those topics here
@@ -677,11 +678,6 @@ private boolean isInternalTopic(final String topicName) {
             && (topicName.endsWith("-changelog") || topicName.endsWith("-repartition"));
     }
 
-    private void printHelp(final OptionParser parser) throws IOException {
-        System.err.println(usage);
-        parser.printHelpOn(System.err);
-    }
-
     public static void main(final String[] args) {
         Exit.exit(new StreamsResetter().run(args));
     }

From 31a5f92b9f657d3b0f65ad2abfc2a085958621c9 Mon Sep 17 00:00:00 2001
From: LuyingLiu <41963486+LuyingLiu@users.noreply.github.com>
Date: Wed, 4 Sep 2019 04:47:17 +0800
Subject: [PATCH 0579/1071] Changed for updatedTasks, avoids stopping and
 starting of unnecessary tasks (#7097)

Corrected the `KafkaConfigBackingStore` logic to notify of only the changed tasks, rather than all tasks. This was not noticed before because Connect always stopped and restarted all tasks during a rebalanced, but since 2.3 the incremental rebalance logic exposed this bug.

Author: Luying Liu 
Reviewers: Konstantine Karantasis , Randall Hauch 
---
 .../apache/kafka/connect/storage/KafkaConfigBackingStore.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java
index 8cf47fe04799f..3572d8cf8c5c9 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java
@@ -620,7 +620,7 @@ public void onCompletion(Throwable error, ConsumerRecord record)
                     } else {
                         if (deferred != null) {
                             taskConfigs.putAll(deferred);
-                            updatedTasks.addAll(taskConfigs.keySet());
+                            updatedTasks.addAll(deferred.keySet());
                         }
                         inconsistent.remove(connectorName);
                     }

From 74fc3323a5815fad80318e96a88dc2164c700809 Mon Sep 17 00:00:00 2001
From: Konstantine Karantasis 
Date: Tue, 3 Sep 2019 13:48:19 -0700
Subject: [PATCH 0580/1071] MINOR: Add unit test for KAFKA-8676 to guard
 against unrequired task restarts (#7287)

Added unit test for recent fix of `KafkaConfigBackingStore`.

Author: Konstantine Karantasis 
Reviewer: Randall Hauch 
---
 .../storage/KafkaConfigBackingStoreTest.java  | 90 +++++++++++++++++++
 1 file changed, 90 insertions(+)

diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java
index 68c447a1aebdd..8e358e0670958 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java
@@ -302,6 +302,96 @@ public void testPutTaskConfigs() throws Exception {
         PowerMock.verifyAll();
     }
 
+    @Test
+    public void testPutTaskConfigsStartsOnlyReconfiguredTasks() throws Exception {
+        expectConfigure();
+        expectStart(Collections.emptyList(), Collections.emptyMap());
+
+        // Task configs should read to end, write to the log, read to end, write root, then read to end again
+        expectReadToEnd(new LinkedHashMap<>());
+        expectConvertWriteRead(
+                TASK_CONFIG_KEYS.get(0), KafkaConfigBackingStore.TASK_CONFIGURATION_V0, CONFIGS_SERIALIZED.get(0),
+                "properties", SAMPLE_CONFIGS.get(0));
+        expectConvertWriteRead(
+                TASK_CONFIG_KEYS.get(1), KafkaConfigBackingStore.TASK_CONFIGURATION_V0, CONFIGS_SERIALIZED.get(1),
+                "properties", SAMPLE_CONFIGS.get(1));
+        expectReadToEnd(new LinkedHashMap());
+        expectConvertWriteRead(
+                COMMIT_TASKS_CONFIG_KEYS.get(0), KafkaConfigBackingStore.CONNECTOR_TASKS_COMMIT_V0, CONFIGS_SERIALIZED.get(2),
+                "tasks", 2); // Starts with 0 tasks, after update has 2
+        // As soon as root is rewritten, we should see a callback notifying us that we reconfigured some tasks
+        configUpdateListener.onTaskConfigUpdate(Arrays.asList(TASK_IDS.get(0), TASK_IDS.get(1)));
+        EasyMock.expectLastCall();
+
+        // Records to be read by consumer as it reads to the end of the log
+        LinkedHashMap serializedConfigs = new LinkedHashMap<>();
+        serializedConfigs.put(TASK_CONFIG_KEYS.get(0), CONFIGS_SERIALIZED.get(0));
+        serializedConfigs.put(TASK_CONFIG_KEYS.get(1), CONFIGS_SERIALIZED.get(1));
+        serializedConfigs.put(COMMIT_TASKS_CONFIG_KEYS.get(0), CONFIGS_SERIALIZED.get(2));
+        expectReadToEnd(serializedConfigs);
+
+        // Task configs should read to end, write to the log, read to end, write root, then read to end again
+        expectReadToEnd(new LinkedHashMap<>());
+        expectConvertWriteRead(
+                TASK_CONFIG_KEYS.get(2), KafkaConfigBackingStore.TASK_CONFIGURATION_V0, CONFIGS_SERIALIZED.get(3),
+                "properties", SAMPLE_CONFIGS.get(2));
+        expectReadToEnd(new LinkedHashMap<>());
+        expectConvertWriteRead(
+                COMMIT_TASKS_CONFIG_KEYS.get(1), KafkaConfigBackingStore.CONNECTOR_TASKS_COMMIT_V0, CONFIGS_SERIALIZED.get(4),
+                "tasks", 1); // Starts with 2 tasks, after update has 3
+
+        // As soon as root is rewritten, we should see a callback notifying us that we reconfigured some tasks
+        configUpdateListener.onTaskConfigUpdate(Arrays.asList(TASK_IDS.get(2)));
+        EasyMock.expectLastCall();
+
+        // Records to be read by consumer as it reads to the end of the log
+        serializedConfigs = new LinkedHashMap<>();
+        serializedConfigs.put(TASK_CONFIG_KEYS.get(2), CONFIGS_SERIALIZED.get(3));
+        serializedConfigs.put(COMMIT_TASKS_CONFIG_KEYS.get(1), CONFIGS_SERIALIZED.get(4));
+        expectReadToEnd(serializedConfigs);
+
+        expectStop();
+
+        PowerMock.replayAll();
+
+        configStorage.setupAndCreateKafkaBasedLog(TOPIC, DEFAULT_DISTRIBUTED_CONFIG);
+        configStorage.start();
+
+        // Bootstrap as if we had already added the connector, but no tasks had been added yet
+        whiteboxAddConnector(CONNECTOR_IDS.get(0), SAMPLE_CONFIGS.get(0), Collections.emptyList());
+        whiteboxAddConnector(CONNECTOR_IDS.get(1), SAMPLE_CONFIGS.get(1), Collections.emptyList());
+
+        // Null before writing
+        ClusterConfigState configState = configStorage.snapshot();
+        assertEquals(-1, configState.offset());
+        assertNull(configState.taskConfig(TASK_IDS.get(0)));
+        assertNull(configState.taskConfig(TASK_IDS.get(1)));
+
+        // Writing task configs should block until all the writes have been performed and the root record update
+        // has completed
+        List> taskConfigs = Arrays.asList(SAMPLE_CONFIGS.get(0), SAMPLE_CONFIGS.get(1));
+        configStorage.putTaskConfigs("connector1", taskConfigs);
+        taskConfigs = Collections.singletonList(SAMPLE_CONFIGS.get(2));
+        configStorage.putTaskConfigs("connector2", taskConfigs);
+
+        // Validate root config by listing all connectors and tasks
+        configState = configStorage.snapshot();
+        assertEquals(5, configState.offset());
+        String connectorName1 = CONNECTOR_IDS.get(0);
+        String connectorName2 = CONNECTOR_IDS.get(1);
+        assertEquals(Arrays.asList(connectorName1, connectorName2), new ArrayList<>(configState.connectors()));
+        assertEquals(Arrays.asList(TASK_IDS.get(0), TASK_IDS.get(1)), configState.tasks(connectorName1));
+        assertEquals(Collections.singletonList(TASK_IDS.get(2)), configState.tasks(connectorName2));
+        assertEquals(SAMPLE_CONFIGS.get(0), configState.taskConfig(TASK_IDS.get(0)));
+        assertEquals(SAMPLE_CONFIGS.get(1), configState.taskConfig(TASK_IDS.get(1)));
+        assertEquals(SAMPLE_CONFIGS.get(2), configState.taskConfig(TASK_IDS.get(2)));
+        assertEquals(Collections.EMPTY_SET, configState.inconsistentConnectors());
+
+        configStorage.stop();
+
+        PowerMock.verifyAll();
+    }
+
     @Test
     public void testPutTaskConfigsZeroTasks() throws Exception {
         expectConfigure();

From 3b8d7a661cc993b298385fd7c726d0ae0463ee12 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Tue, 3 Sep 2019 23:16:33 -0700
Subject: [PATCH 0581/1071] MINOR: Add system configuration to zk security
 exception messages (#7280)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This patch ensures that relevant system configurations are included in exception messages when zk security validation fails.

Reviewers: Vikas Singh ,  José Armando García Sancio , Manikumar Reddy 
---
 .../kafka/common/security/JaasUtils.java      | 41 ++++++++++++++-----
 .../main/scala/kafka/server/KafkaServer.scala |  3 +-
 2 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/security/JaasUtils.java b/clients/src/main/java/org/apache/kafka/common/security/JaasUtils.java
index c2e7fbdc74a2a..593f104e300fa 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/JaasUtils.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/JaasUtils.java
@@ -16,12 +16,12 @@
  */
 package org.apache.kafka.common.security;
 
-import javax.security.auth.login.Configuration;
-
 import org.apache.kafka.common.KafkaException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.security.auth.login.Configuration;
+
 public final class JaasUtils {
     private static final Logger LOG = LoggerFactory.getLogger(JaasUtils.class);
     public static final String JAVA_LOGIN_CONFIG_PARAM = "java.security.auth.login.config";
@@ -31,27 +31,48 @@ public final class JaasUtils {
     public static final String ZK_SASL_CLIENT = "zookeeper.sasl.client";
     public static final String ZK_LOGIN_CONTEXT_NAME_KEY = "zookeeper.sasl.clientconfig";
 
+    private static final String DEFAULT_ZK_LOGIN_CONTEXT_NAME = "Client";
+    private static final String DEFAULT_ZK_SASL_CLIENT = "true";
+
     private JaasUtils() {}
 
+    public static String zkSecuritySysConfigString() {
+        String loginConfig = System.getProperty(JAVA_LOGIN_CONFIG_PARAM);
+        String clientEnabled = System.getProperty(ZK_SASL_CLIENT, "default:" + DEFAULT_ZK_SASL_CLIENT);
+        String contextName = System.getProperty(ZK_LOGIN_CONTEXT_NAME_KEY, "default:" + DEFAULT_ZK_LOGIN_CONTEXT_NAME);
+        return "[" +
+                JAVA_LOGIN_CONFIG_PARAM + "=" + loginConfig +
+                ", " +
+                ZK_SASL_CLIENT + "=" + clientEnabled +
+                ", " +
+                ZK_LOGIN_CONTEXT_NAME_KEY + "=" + contextName +
+                "]";
+    }
+
     public static boolean isZkSecurityEnabled() {
-        boolean zkSaslEnabled = Boolean.parseBoolean(System.getProperty(ZK_SASL_CLIENT, "true"));
-        String zkLoginContextName = System.getProperty(ZK_LOGIN_CONTEXT_NAME_KEY, "Client");
+        boolean zkSaslEnabled = Boolean.parseBoolean(System.getProperty(ZK_SASL_CLIENT, DEFAULT_ZK_SASL_CLIENT));
+        String zkLoginContextName = System.getProperty(ZK_LOGIN_CONTEXT_NAME_KEY, DEFAULT_ZK_LOGIN_CONTEXT_NAME);
 
-        boolean isSecurityEnabled;
+        LOG.debug("Checking login config for Zookeeper JAAS context {}", zkSecuritySysConfigString());
+
+        boolean foundLoginConfigEntry;
         try {
             Configuration loginConf = Configuration.getConfiguration();
-            isSecurityEnabled = loginConf.getAppConfigurationEntry(zkLoginContextName) != null;
+            foundLoginConfigEntry = loginConf.getAppConfigurationEntry(zkLoginContextName) != null;
         } catch (Exception e) {
-            throw new KafkaException("Exception while loading Zookeeper JAAS login context '" + zkLoginContextName + "'", e);
+            throw new KafkaException("Exception while loading Zookeeper JAAS login context " +
+                    zkSecuritySysConfigString(), e);
         }
-        if (isSecurityEnabled && !zkSaslEnabled) {
+
+        if (foundLoginConfigEntry && !zkSaslEnabled) {
             LOG.error("JAAS configuration is present, but system property " +
                         ZK_SASL_CLIENT + " is set to false, which disables " +
                         "SASL in the ZooKeeper client");
-            throw new KafkaException("Exception while determining if ZooKeeper is secure");
+            throw new KafkaException("Exception while determining if ZooKeeper is secure " +
+                    zkSecuritySysConfigString());
         }
 
-        return isSecurityEnabled;
+        return foundLoginConfigEntry;
     }
 }
 
diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala
index 353b66a32d161..10677150aa418 100755
--- a/core/src/main/scala/kafka/server/KafkaServer.scala
+++ b/core/src/main/scala/kafka/server/KafkaServer.scala
@@ -384,7 +384,8 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
     val isZkSecurityEnabled = JaasUtils.isZkSecurityEnabled()
 
     if (secureAclsEnabled && !isZkSecurityEnabled)
-      throw new java.lang.SecurityException(s"${KafkaConfig.ZkEnableSecureAclsProp} is true, but the verification of the JAAS login file failed.")
+      throw new java.lang.SecurityException(s"${KafkaConfig.ZkEnableSecureAclsProp} is true, but the " +
+        s"verification of the JAAS login file failed ${JaasUtils.zkSecuritySysConfigString}")
 
     // make sure chroot path exists
     chrootOption.foreach { chroot =>

From 8dc80e22973d89090833cb791d9269e9a0598059 Mon Sep 17 00:00:00 2001
From: Omar Al-Safi 
Date: Wed, 4 Sep 2019 08:24:00 +0200
Subject: [PATCH 0582/1071] KAFKA-7849: Fix the warning when using GlobalKTable
 (#7104)

Reviewers: Matthias J. Sax , Guozhang Wang 
---
 .../internals/InternalTopologyBuilder.java    |  1 +
 .../InternalTopologyBuilderTest.java          | 20 +++++++++++++++++++
 2 files changed, 21 insertions(+)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java
index e5f404032659a..3a7bd55668d48 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java
@@ -1196,6 +1196,7 @@ synchronized Pattern sourceTopicPattern() {
                 for (final List topics : nodeToSourceTopics.values()) {
                     allSourceTopics.addAll(maybeDecorateInternalSourceTopics(topics));
                 }
+                allSourceTopics.removeAll(globalTopics);
             }
             Collections.sort(allSourceTopics);
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java
index b86211fbcd238..185cd908a704e 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java
@@ -243,6 +243,26 @@ public void testSourceTopics() {
         assertEquals(expectedPattern.pattern(), builder.sourceTopicPattern().pattern());
     }
 
+    @Test
+    public void testSourceTopicsWithGlobalTopics() {
+        builder.setApplicationId("X");
+        builder.addSource(null, "source-1", null, null, null, "topic-1");
+        builder.addSource(null, "source-2", null, null, null, "topic-2");
+        builder.addGlobalStore(
+                new MockKeyValueStoreBuilder("global-store", false).withLoggingDisabled(),
+                "globalSource",
+                null,
+                null,
+                null,
+                "globalTopic",
+                "global-processor",
+                new MockProcessorSupplier());
+
+        final Pattern expectedPattern = Pattern.compile("topic-1|topic-2");
+
+        assertThat(builder.sourceTopicPattern().pattern(), equalTo(expectedPattern.pattern()));
+    }
+
     @Test
     public void testPatternSourceTopic() {
         final Pattern expectedPattern = Pattern.compile("topic-\\d");

From caaf253b637d8fa2ad90b84a5b55103e192b37d6 Mon Sep 17 00:00:00 2001
From: khairy 
Date: Wed, 4 Sep 2019 15:12:18 +0100
Subject: [PATCH 0583/1071] MINOR: remove unnecessary nulllity check (#7282)

Minor code enhancement: remove unnecessary check of nullity.

Reviewers: Bill Bejeck 
---
 .../kafka/streams/state/internals/RocksDBWindowStore.java      | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStore.java
index 3b634ebea200a..49eefa14db2bc 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStore.java
@@ -63,9 +63,6 @@ public void put(final Bytes key, final byte[] value, final long windowStartTimes
     @Override
     public byte[] fetch(final Bytes key, final long timestamp) {
         final byte[] bytesValue = wrapped().get(WindowKeySchema.toStoreKeyBinary(key, timestamp, seqnum));
-        if (bytesValue == null) {
-            return null;
-        }
         return bytesValue;
     }
 

From 18e6bb251b6637d49ec53d790ea8e8ef7ab808c4 Mon Sep 17 00:00:00 2001
From: Chia-Ping Tsai 
Date: Wed, 4 Sep 2019 22:13:56 +0800
Subject: [PATCH 0584/1071] KAFKA-8861 Fix flaky
 RegexSourceIntegrationTest.testMultipleConsumersCanReadFromPartitionedTopic
 (#7281)

similar to https://issues.apache.org/jira/browse/KAFKA-8011 and https://issues.apache.org/jira/browse/KAFKA-8026

Reviewers:  Matthias J. Sax ,  Bill Bejeck 
---
 .../kafka/streams/integration/RegexSourceIntegrationTest.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/RegexSourceIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/RegexSourceIntegrationTest.java
index 10e06508b900f..8c00ee4f4829f 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/RegexSourceIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/RegexSourceIntegrationTest.java
@@ -318,8 +318,8 @@ public void testMultipleConsumersCanReadFromPartitionedTopic() throws Exception
             partitionedStreamLeader.to(outputTopic, Produced.with(stringSerde, stringSerde));
             partitionedStreamFollower.to(outputTopic, Produced.with(stringSerde, stringSerde));
 
-            final List leaderAssignment = new ArrayList<>();
-            final List followerAssignment = new ArrayList<>();
+            final List leaderAssignment = new CopyOnWriteArrayList<>();
+            final List followerAssignment = new CopyOnWriteArrayList<>();
 
             partitionedStreamsLeader  = new KafkaStreams(builderLeader.build(), streamsConfiguration, new DefaultKafkaClientSupplier() {
                 @Override

From 8d8e2fb5c94a2287d62fe010c87fe0c8728e88c0 Mon Sep 17 00:00:00 2001
From: "Tu V. Tran" 
Date: Wed, 4 Sep 2019 12:46:30 -0700
Subject: [PATCH 0585/1071] KAFKA-8729, pt 1: Add 4 new metrics to keep track
 of various types of invalid record rejections (#7142)

Right now we only have very generic FailedProduceRequestsPerSec and FailedFetchRequestsPerSec metrics that mark whenever a record is failed on the broker side. To improve the debugging UX, I added 4 new metrics in BrokerTopicStats to log various scenarios when an InvalidRecordException is thrown when LogValidator fails to validate a record:

-- NoKeyCompactedTopicRecordsPerSec: counter of failures by compacted records with no key
-- InvalidMagicNumberRecordsPerSec: counter of failures by records with invalid magic number
-- InvalidMessageCrcRecordsPerSec: counter of failures by records with crc corruption
-- NonIncreasingOffsetRecordsPerSec: counter of failures by records with invalid offset

Reviewers: Robert Yokota , Guozhang Wang 
---
 core/src/main/scala/kafka/log/Log.scala       |   9 +-
 .../main/scala/kafka/log/LogValidator.scala   | 119 +++++++----
 .../kafka/server/KafkaRequestHandler.scala    |  21 +-
 .../test/scala/unit/kafka/log/LogTest.scala   |  12 +-
 .../unit/kafka/log/LogValidatorTest.scala     | 197 ++++++++++++++----
 .../unit/kafka/metrics/MetricsTest.scala      |  32 ++-
 .../kafka/server/ProduceRequestTest.scala     |   5 +
 .../scala/unit/kafka/utils/TestUtils.scala    |  11 +
 8 files changed, 300 insertions(+), 106 deletions(-)

diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index 09cffa6d7d42b..34a89dc43165f 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -1034,6 +1034,7 @@ class Log(@volatile var dir: File,
           val now = time.milliseconds
           val validateAndOffsetAssignResult = try {
             LogValidator.validateMessagesAndAssignOffsets(validRecords,
+              topicPartition,
               offset,
               time,
               now,
@@ -1045,7 +1046,8 @@ class Log(@volatile var dir: File,
               config.messageTimestampDifferenceMaxMs,
               leaderEpoch,
               isFromClient,
-              interBrokerProtocolVersion)
+              interBrokerProtocolVersion,
+              brokerTopicStats)
           } catch {
             case e: IOException =>
               throw new KafkaException(s"Error validating messages while appending to log $name", e)
@@ -1350,7 +1352,10 @@ class Log(@volatile var dir: File,
       }
 
       // check the validity of the message by checking CRC
-      batch.ensureValid()
+      if (!batch.isValid) {
+        brokerTopicStats.allTopicsStats.invalidMessageCrcRecordsPerSec.mark()
+        throw new InvalidRecordException(s"Record is corrupt (stored crc = ${batch.checksum()}) in topic partition $topicPartition.")
+      }
 
       if (batch.maxTimestamp > maxTimestamp) {
         maxTimestamp = batch.maxTimestamp
diff --git a/core/src/main/scala/kafka/log/LogValidator.scala b/core/src/main/scala/kafka/log/LogValidator.scala
index b0412bf89147b..d317738a9f6be 100644
--- a/core/src/main/scala/kafka/log/LogValidator.scala
+++ b/core/src/main/scala/kafka/log/LogValidator.scala
@@ -21,9 +21,11 @@ import java.nio.ByteBuffer
 import kafka.api.{ApiVersion, KAFKA_2_1_IV0}
 import kafka.common.LongRef
 import kafka.message.{CompressionCodec, NoCompressionCodec, ZStdCompressionCodec}
+import kafka.server.BrokerTopicStats
 import kafka.utils.Logging
+import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.{InvalidTimestampException, UnsupportedCompressionTypeException, UnsupportedForMessageFormatException}
-import org.apache.kafka.common.record.{AbstractRecords, CompressionType, InvalidRecordException, MemoryRecords, Record, RecordBatch, RecordConversionStats, TimestampType, BufferSupplier}
+import org.apache.kafka.common.record.{AbstractRecords, BufferSupplier, CompressionType, InvalidRecordException, MemoryRecords, Record, RecordBatch, RecordConversionStats, TimestampType}
 import org.apache.kafka.common.utils.Time
 
 import scala.collection.{Seq, mutable}
@@ -47,6 +49,7 @@ private[kafka] object LogValidator extends Logging {
    * of the shallow message with the max timestamp and a boolean indicating whether the message sizes may have changed.
    */
   private[kafka] def validateMessagesAndAssignOffsets(records: MemoryRecords,
+                                                      topicPartition: TopicPartition,
                                                       offsetCounter: LongRef,
                                                       time: Time,
                                                       now: Long,
@@ -58,19 +61,20 @@ private[kafka] object LogValidator extends Logging {
                                                       timestampDiffMaxMs: Long,
                                                       partitionLeaderEpoch: Int,
                                                       isFromClient: Boolean,
-                                                      interBrokerProtocolVersion: ApiVersion): ValidationAndOffsetAssignResult = {
+                                                      interBrokerProtocolVersion: ApiVersion,
+                                                      brokerTopicStats: BrokerTopicStats): ValidationAndOffsetAssignResult = {
     if (sourceCodec == NoCompressionCodec && targetCodec == NoCompressionCodec) {
       // check the magic value
       if (!records.hasMatchingMagic(magic))
-        convertAndAssignOffsetsNonCompressed(records, offsetCounter, compactedTopic, time, now, timestampType,
-          timestampDiffMaxMs, magic, partitionLeaderEpoch, isFromClient)
+        convertAndAssignOffsetsNonCompressed(records, topicPartition, offsetCounter, compactedTopic, time, now, timestampType,
+          timestampDiffMaxMs, magic, partitionLeaderEpoch, isFromClient, brokerTopicStats)
       else
         // Do in-place validation, offset assignment and maybe set timestamp
-        assignOffsetsNonCompressed(records, offsetCounter, now, compactedTopic, timestampType, timestampDiffMaxMs,
-          partitionLeaderEpoch, isFromClient, magic)
+        assignOffsetsNonCompressed(records, topicPartition, offsetCounter, now, compactedTopic, timestampType, timestampDiffMaxMs,
+          partitionLeaderEpoch, isFromClient, magic, brokerTopicStats)
     } else {
-      validateMessagesAndAssignOffsetsCompressed(records, offsetCounter, time, now, sourceCodec, targetCodec, compactedTopic,
-        magic, timestampType, timestampDiffMaxMs, partitionLeaderEpoch, isFromClient, interBrokerProtocolVersion)
+      validateMessagesAndAssignOffsetsCompressed(records, topicPartition, offsetCounter, time, now, sourceCodec, targetCodec, compactedTopic,
+        magic, timestampType, timestampDiffMaxMs, partitionLeaderEpoch, isFromClient, interBrokerProtocolVersion, brokerTopicStats)
     }
   }
 
@@ -93,33 +97,45 @@ private[kafka] object LogValidator extends Logging {
     batch
   }
 
-  private def validateBatch(firstBatch: RecordBatch, batch: RecordBatch, isFromClient: Boolean, toMagic: Byte): Unit = {
+  private def validateBatch(topicPartition: TopicPartition, firstBatch: RecordBatch, batch: RecordBatch, isFromClient: Boolean, toMagic: Byte, brokerTopicStats: BrokerTopicStats): Unit = {
     // batch magic byte should have the same magic as the first batch
-    if (firstBatch.magic() != batch.magic())
-      throw new InvalidRecordException(s"Batch magic ${batch.magic()} is not the same as the first batch'es magic byte ${firstBatch.magic()}")
+    if (firstBatch.magic() != batch.magic()) {
+      brokerTopicStats.allTopicsStats.invalidMagicNumberRecordsPerSec.mark()
+      throw new InvalidRecordException(s"Batch magic ${batch.magic()} is not the same as the first batch'es magic byte ${firstBatch.magic()} in topic partition $topicPartition.")
+    }
 
     if (isFromClient) {
       if (batch.magic >= RecordBatch.MAGIC_VALUE_V2) {
         val countFromOffsets = batch.lastOffset - batch.baseOffset + 1
-        if (countFromOffsets <= 0)
-          throw new InvalidRecordException(s"Batch has an invalid offset range: [${batch.baseOffset}, ${batch.lastOffset}]")
+        if (countFromOffsets <= 0) {
+          brokerTopicStats.allTopicsStats.invalidOffsetOrSequenceRecordsPerSec.mark()
+          throw new InvalidRecordException(s"Batch has an invalid offset range: [${batch.baseOffset}, ${batch.lastOffset}] in topic partition $topicPartition.")
+        }
 
         // v2 and above messages always have a non-null count
         val count = batch.countOrNull
-        if (count <= 0)
-          throw new InvalidRecordException(s"Invalid reported count for record batch: $count")
+        if (count <= 0) {
+          brokerTopicStats.allTopicsStats.invalidOffsetOrSequenceRecordsPerSec.mark()
+          throw new InvalidRecordException(s"Invalid reported count for record batch: $count in topic partition $topicPartition.")
+        }
 
-        if (countFromOffsets != batch.countOrNull)
+        if (countFromOffsets != batch.countOrNull) {
+          brokerTopicStats.allTopicsStats.invalidOffsetOrSequenceRecordsPerSec.mark()
           throw new InvalidRecordException(s"Inconsistent batch offset range [${batch.baseOffset}, ${batch.lastOffset}] " +
-            s"and count of records $count")
+            s"and count of records $count in topic partition $topicPartition.")
+        }
       }
 
-      if (batch.hasProducerId && batch.baseSequence < 0)
+      if (batch.hasProducerId && batch.baseSequence < 0) {
+        brokerTopicStats.allTopicsStats.invalidOffsetOrSequenceRecordsPerSec.mark()
         throw new InvalidRecordException(s"Invalid sequence number ${batch.baseSequence} in record batch " +
-          s"with producerId ${batch.producerId}")
+          s"with producerId ${batch.producerId} in topic partition $topicPartition.")
+      }
 
-      if (batch.isControlBatch)
-        throw new InvalidRecordException("Clients are not allowed to write control records")
+      if (batch.isControlBatch) {
+        brokerTopicStats.allTopicsStats.invalidOffsetOrSequenceRecordsPerSec.mark()
+        throw new InvalidRecordException(s"Clients are not allowed to write control records in topic partition $topicPartition.")
+      }
     }
 
     if (batch.isTransactional && toMagic < RecordBatch.MAGIC_VALUE_V2)
@@ -129,23 +145,33 @@ private[kafka] object LogValidator extends Logging {
       throw new UnsupportedForMessageFormatException(s"Idempotent records cannot be used with magic version $toMagic")
   }
 
-  private def validateRecord(batch: RecordBatch, record: Record, now: Long, timestampType: TimestampType,
-                             timestampDiffMaxMs: Long, compactedTopic: Boolean): Unit = {
-    if (!record.hasMagic(batch.magic))
-      throw new InvalidRecordException(s"Log record $record's magic does not match outer magic ${batch.magic}")
+  private def validateRecord(batch: RecordBatch, topicPartition: TopicPartition, record: Record, now: Long, timestampType: TimestampType,
+                             timestampDiffMaxMs: Long, compactedTopic: Boolean, brokerTopicStats: BrokerTopicStats): Unit = {
+    if (!record.hasMagic(batch.magic)) {
+      brokerTopicStats.allTopicsStats.invalidMagicNumberRecordsPerSec.mark()
+      throw new InvalidRecordException(s"Log record $record's magic does not match outer magic ${batch.magic} in topic partition $topicPartition.")
+    }
 
     // verify the record-level CRC only if this is one of the deep entries of a compressed message
     // set for magic v0 and v1. For non-compressed messages, there is no inner record for magic v0 and v1,
     // so we depend on the batch-level CRC check in Log.analyzeAndValidateRecords(). For magic v2 and above,
     // there is no record-level CRC to check.
-    if (batch.magic <= RecordBatch.MAGIC_VALUE_V1 && batch.isCompressed)
-      record.ensureValid()
+    if (batch.magic <= RecordBatch.MAGIC_VALUE_V1 && batch.isCompressed) {
+      try {
+        record.ensureValid()
+      } catch {
+        case e: InvalidRecordException =>
+          brokerTopicStats.allTopicsStats.invalidMessageCrcRecordsPerSec.mark()
+          throw new InvalidRecordException(e.getMessage + s" in topic partition $topicPartition.")
+      }
+    }
 
-    validateKey(record, compactedTopic)
+    validateKey(record, topicPartition, compactedTopic, brokerTopicStats)
     validateTimestamp(batch, record, now, timestampType, timestampDiffMaxMs)
   }
 
   private def convertAndAssignOffsetsNonCompressed(records: MemoryRecords,
+                                                   topicPartition: TopicPartition,
                                                    offsetCounter: LongRef,
                                                    compactedTopic: Boolean,
                                                    time: Time,
@@ -154,7 +180,8 @@ private[kafka] object LogValidator extends Logging {
                                                    timestampDiffMaxMs: Long,
                                                    toMagicValue: Byte,
                                                    partitionLeaderEpoch: Int,
-                                                   isFromClient: Boolean): ValidationAndOffsetAssignResult = {
+                                                   isFromClient: Boolean,
+                                                   brokerTopicStats: BrokerTopicStats): ValidationAndOffsetAssignResult = {
     val startNanos = time.nanoseconds
     val sizeInBytesAfterConversion = AbstractRecords.estimateSizeInBytes(toMagicValue, offsetCounter.value,
       CompressionType.NONE, records.records)
@@ -171,10 +198,10 @@ private[kafka] object LogValidator extends Logging {
     val firstBatch = getFirstBatchAndMaybeValidateNoMoreBatches(records, NoCompressionCodec)
 
     for (batch <- records.batches.asScala) {
-      validateBatch(firstBatch, batch, isFromClient, toMagicValue)
+      validateBatch(topicPartition, firstBatch, batch, isFromClient, toMagicValue, brokerTopicStats)
 
       for (record <- batch.asScala) {
-        validateRecord(batch, record, now, timestampType, timestampDiffMaxMs, compactedTopic)
+        validateRecord(batch, topicPartition, record, now, timestampType, timestampDiffMaxMs, compactedTopic, brokerTopicStats)
         builder.appendWithOffset(offsetCounter.getAndIncrement(), record)
       }
     }
@@ -193,6 +220,7 @@ private[kafka] object LogValidator extends Logging {
   }
 
   private def assignOffsetsNonCompressed(records: MemoryRecords,
+                                         topicPartition: TopicPartition,
                                          offsetCounter: LongRef,
                                          now: Long,
                                          compactedTopic: Boolean,
@@ -200,7 +228,8 @@ private[kafka] object LogValidator extends Logging {
                                          timestampDiffMaxMs: Long,
                                          partitionLeaderEpoch: Int,
                                          isFromClient: Boolean,
-                                         magic: Byte): ValidationAndOffsetAssignResult = {
+                                         magic: Byte,
+                                         brokerTopicStats: BrokerTopicStats): ValidationAndOffsetAssignResult = {
     var maxTimestamp = RecordBatch.NO_TIMESTAMP
     var offsetOfMaxTimestamp = -1L
     val initialOffset = offsetCounter.value
@@ -208,13 +237,13 @@ private[kafka] object LogValidator extends Logging {
     val firstBatch = getFirstBatchAndMaybeValidateNoMoreBatches(records, NoCompressionCodec)
 
     for (batch <- records.batches.asScala) {
-      validateBatch(firstBatch, batch, isFromClient, magic)
+      validateBatch(topicPartition, firstBatch, batch, isFromClient, magic, brokerTopicStats)
 
       var maxBatchTimestamp = RecordBatch.NO_TIMESTAMP
       var offsetOfMaxBatchTimestamp = -1L
 
       for (record <- batch.asScala) {
-        validateRecord(batch, record, now, timestampType, timestampDiffMaxMs, compactedTopic)
+        validateRecord(batch, topicPartition, record, now, timestampType, timestampDiffMaxMs, compactedTopic, brokerTopicStats)
         val offset = offsetCounter.getAndIncrement()
         if (batch.magic > RecordBatch.MAGIC_VALUE_V0 && record.timestamp > maxBatchTimestamp) {
           maxBatchTimestamp = record.timestamp
@@ -263,6 +292,7 @@ private[kafka] object LogValidator extends Logging {
    * 3. When the target magic is equal to V0, meaning absolute offsets need to be re-assigned.
    */
   def validateMessagesAndAssignOffsetsCompressed(records: MemoryRecords,
+                                                 topicPartition: TopicPartition,
                                                  offsetCounter: LongRef,
                                                  time: Time,
                                                  now: Long,
@@ -274,7 +304,8 @@ private[kafka] object LogValidator extends Logging {
                                                  timestampDiffMaxMs: Long,
                                                  partitionLeaderEpoch: Int,
                                                  isFromClient: Boolean,
-                                                 interBrokerProtocolVersion: ApiVersion): ValidationAndOffsetAssignResult = {
+                                                 interBrokerProtocolVersion: ApiVersion,
+                                                 brokerTopicStats: BrokerTopicStats): ValidationAndOffsetAssignResult = {
 
     if (targetCodec == ZStdCompressionCodec && interBrokerProtocolVersion < KAFKA_2_1_IV0)
       throw new UnsupportedCompressionTypeException("Produce requests to inter.broker.protocol.version < 2.1 broker " +
@@ -306,7 +337,7 @@ private[kafka] object LogValidator extends Logging {
 
     val batches = records.batches.asScala
     for (batch <- batches) {
-      validateBatch(firstBatch, batch, isFromClient, toMagic)
+      validateBatch(topicPartition, firstBatch, batch, isFromClient, toMagic, brokerTopicStats)
       uncompressedSizeInBytes += AbstractRecords.recordBatchHeaderSizeInBytes(toMagic, batch.compressionType())
 
       // if we are on version 2 and beyond, and we know we are going for in place assignment,
@@ -321,14 +352,16 @@ private[kafka] object LogValidator extends Logging {
           if (sourceCodec != NoCompressionCodec && record.isCompressed)
             throw new InvalidRecordException("Compressed outer record should not have an inner record with a " +
               s"compression attribute set: $record")
-          validateRecord(batch, record, now, timestampType, timestampDiffMaxMs, compactedTopic)
+          validateRecord(batch, topicPartition, record, now, timestampType, timestampDiffMaxMs, compactedTopic, brokerTopicStats)
 
           uncompressedSizeInBytes += record.sizeInBytes()
           if (batch.magic > RecordBatch.MAGIC_VALUE_V0 && toMagic > RecordBatch.MAGIC_VALUE_V0) {
             // inner records offset should always be continuous
             val expectedOffset = expectedInnerOffset.getAndIncrement()
-            if (record.offset != expectedOffset)
-              throw new InvalidRecordException(s"Inner record $record inside the compressed record batch does not have incremental offsets, expected offset is $expectedOffset")
+            if (record.offset != expectedOffset) {
+              brokerTopicStats.allTopicsStats.invalidOffsetOrSequenceRecordsPerSec.mark()
+              throw new InvalidRecordException(s"Inner record $record inside the compressed record batch does not have incremental offsets, expected offset is $expectedOffset in topic partition $topicPartition.")
+            }
             if (record.timestamp > maxTimestamp)
               maxTimestamp = record.timestamp
           }
@@ -422,9 +455,11 @@ private[kafka] object LogValidator extends Logging {
       recordConversionStats = recordConversionStats)
   }
 
-  private def validateKey(record: Record, compactedTopic: Boolean): Unit = {
-    if (compactedTopic && !record.hasKey)
-      throw new InvalidRecordException("Compacted topic cannot accept message without key.")
+  private def validateKey(record: Record, topicPartition: TopicPartition, compactedTopic: Boolean, brokerTopicStats: BrokerTopicStats) {
+    if (compactedTopic && !record.hasKey) {
+      brokerTopicStats.allTopicsStats.noKeyCompactedTopicRecordsPerSec.mark()
+      throw new InvalidRecordException(s"Compacted topic cannot accept message without key in topic partition $topicPartition.")
+    }
   }
 
   /**
diff --git a/core/src/main/scala/kafka/server/KafkaRequestHandler.scala b/core/src/main/scala/kafka/server/KafkaRequestHandler.scala
index 27962f65f8915..639627fc022c9 100755
--- a/core/src/main/scala/kafka/server/KafkaRequestHandler.scala
+++ b/core/src/main/scala/kafka/server/KafkaRequestHandler.scala
@@ -188,7 +188,11 @@ class BrokerTopicMetrics(name: Option[String]) extends KafkaMetricsGroup {
     BrokerTopicStats.TotalProduceRequestsPerSec -> MeterWrapper(BrokerTopicStats.TotalProduceRequestsPerSec, "requests"),
     BrokerTopicStats.TotalFetchRequestsPerSec -> MeterWrapper(BrokerTopicStats.TotalFetchRequestsPerSec, "requests"),
     BrokerTopicStats.FetchMessageConversionsPerSec -> MeterWrapper(BrokerTopicStats.FetchMessageConversionsPerSec, "requests"),
-    BrokerTopicStats.ProduceMessageConversionsPerSec -> MeterWrapper(BrokerTopicStats.ProduceMessageConversionsPerSec, "requests")
+    BrokerTopicStats.ProduceMessageConversionsPerSec -> MeterWrapper(BrokerTopicStats.ProduceMessageConversionsPerSec, "requests"),
+    BrokerTopicStats.NoKeyCompactedTopicRecordsPerSec -> MeterWrapper(BrokerTopicStats.NoKeyCompactedTopicRecordsPerSec, "requests"),
+    BrokerTopicStats.InvalidMagicNumberRecordsPerSec -> MeterWrapper(BrokerTopicStats.InvalidMagicNumberRecordsPerSec, "requests"),
+    BrokerTopicStats.InvalidMessageCrcRecordsPerSec -> MeterWrapper(BrokerTopicStats.InvalidMessageCrcRecordsPerSec, "requests"),
+    BrokerTopicStats.InvalidOffsetOrSequenceRecordsPerSec -> MeterWrapper(BrokerTopicStats.InvalidOffsetOrSequenceRecordsPerSec, "requests")
   ).asJava)
   if (name.isEmpty) {
     metricTypeMap.put(BrokerTopicStats.ReplicationBytesInPerSec, MeterWrapper(BrokerTopicStats.ReplicationBytesInPerSec, "bytes"))
@@ -226,6 +230,14 @@ class BrokerTopicMetrics(name: Option[String]) extends KafkaMetricsGroup {
 
   def produceMessageConversionsRate = metricTypeMap.get(BrokerTopicStats.ProduceMessageConversionsPerSec).meter()
 
+  def noKeyCompactedTopicRecordsPerSec = metricTypeMap.get(BrokerTopicStats.NoKeyCompactedTopicRecordsPerSec).meter()
+
+  def invalidMagicNumberRecordsPerSec = metricTypeMap.get(BrokerTopicStats.InvalidMagicNumberRecordsPerSec).meter()
+
+  def invalidMessageCrcRecordsPerSec = metricTypeMap.get(BrokerTopicStats.InvalidMessageCrcRecordsPerSec).meter()
+
+  def invalidOffsetOrSequenceRecordsPerSec = metricTypeMap.get(BrokerTopicStats.InvalidOffsetOrSequenceRecordsPerSec).meter()
+
   def closeMetric(metricType: String): Unit = {
     val meter = metricTypeMap.get(metricType)
     if (meter != null)
@@ -248,6 +260,13 @@ object BrokerTopicStats {
   val TotalFetchRequestsPerSec = "TotalFetchRequestsPerSec"
   val FetchMessageConversionsPerSec = "FetchMessageConversionsPerSec"
   val ProduceMessageConversionsPerSec = "ProduceMessageConversionsPerSec"
+
+  // These following topics are for LogValidator for better debugging on failed records
+  val NoKeyCompactedTopicRecordsPerSec = "NoKeyCompactedTopicRecordsPerSec"
+  val InvalidMagicNumberRecordsPerSec = "InvalidMagicNumberRecordsPerSec"
+  val InvalidMessageCrcRecordsPerSec = "InvalidMessageCrcRecordsPerSec"
+  val InvalidOffsetOrSequenceRecordsPerSec = "InvalidOffsetOrSequenceRecordsPerSec"
+
   private val valueFactory = (k: String) => new BrokerTopicMetrics(Some(k))
 }
 
diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala
index 10ac4ad7b560f..0e3634e55cfda 100755
--- a/core/src/test/scala/unit/kafka/log/LogTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogTest.scala
@@ -23,6 +23,7 @@ import java.nio.file.{Files, Paths}
 import java.util.regex.Pattern
 import java.util.{Collections, Optional, Properties}
 
+import com.yammer.metrics.Metrics
 import kafka.api.{ApiVersion, KAFKA_0_11_0_IV0}
 import kafka.common.{OffsetsOutOfOrderException, UnexpectedAppendOffsetException}
 import kafka.log.Log.DeleteDirSuffix
@@ -55,6 +56,7 @@ class LogTest {
   val tmpDir = TestUtils.tempDir()
   val logDir = TestUtils.randomPartitionLogDir(tmpDir)
   val mockTime = new MockTime()
+  def metricsKeySet = Metrics.defaultRegistry.allMetrics.keySet.asScala
 
   @Before
   def setUp(): Unit = {
@@ -1806,21 +1808,25 @@ class LogTest {
       log.appendAsLeader(messageSetWithUnkeyedMessage, leaderEpoch = 0)
       fail("Compacted topics cannot accept a message without a key.")
     } catch {
-      case _: CorruptRecordException => // this is good
+      case _: InvalidRecordException => // this is good
     }
     try {
       log.appendAsLeader(messageSetWithOneUnkeyedMessage, leaderEpoch = 0)
       fail("Compacted topics cannot accept a message without a key.")
     } catch {
-      case _: CorruptRecordException => // this is good
+      case _: InvalidRecordException => // this is good
     }
     try {
       log.appendAsLeader(messageSetWithCompressedUnkeyedMessage, leaderEpoch = 0)
       fail("Compacted topics cannot accept a message without a key.")
     } catch {
-      case _: CorruptRecordException => // this is good
+      case _: InvalidRecordException => // this is good
     }
 
+    // check if metric for NoKeyCompactedTopicRecordsPerSec is logged
+    assertEquals(metricsKeySet.count(_.getMBeanName.endsWith(s"${BrokerTopicStats.NoKeyCompactedTopicRecordsPerSec}")), 1)
+    assertTrue(TestUtils.meterCount(s"${BrokerTopicStats.NoKeyCompactedTopicRecordsPerSec}") > 0)
+
     // the following should succeed without any InvalidMessageException
     log.appendAsLeader(messageSetWithKeyedMessage, leaderEpoch = 0)
     log.appendAsLeader(messageSetWithKeyedMessages, leaderEpoch = 0)
diff --git a/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala b/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
index abbb4a4308fc6..e37c85bc74189 100644
--- a/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
@@ -19,9 +19,13 @@ package kafka.log
 import java.nio.ByteBuffer
 import java.util.concurrent.TimeUnit
 
+import com.yammer.metrics.Metrics
 import kafka.api.{ApiVersion, KAFKA_2_0_IV1, KAFKA_2_3_IV1}
 import kafka.common.LongRef
 import kafka.message._
+import kafka.server.BrokerTopicStats
+import kafka.utils.TestUtils.meterCount
+import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.{InvalidTimestampException, UnsupportedCompressionTypeException, UnsupportedForMessageFormatException}
 import org.apache.kafka.common.record._
 import org.apache.kafka.common.utils.Time
@@ -35,6 +39,9 @@ import scala.collection.JavaConverters._
 class LogValidatorTest {
 
   val time = Time.SYSTEM
+  val topicPartition = new TopicPartition("topic", 0)
+  val brokerTopicStats = new BrokerTopicStats
+  val metricsKeySet = Metrics.defaultRegistry.allMetrics.keySet.asScala
 
   @Test
   def testOnlyOneBatch(): Unit = {
@@ -76,10 +83,13 @@ class LogValidatorTest {
     assertThrows[InvalidRecordException] {
       validateMessages(recordsWithInvalidInnerMagic(batchMagic, recordMagic, compressionType), batchMagic, compressionType, compressionType)
     }
+    assertEquals(metricsKeySet.count(_.getMBeanName.endsWith(s"${BrokerTopicStats.InvalidMagicNumberRecordsPerSec}")), 1)
+    assertTrue(meterCount(s"${BrokerTopicStats.InvalidMagicNumberRecordsPerSec}") > 0)
   }
 
   private def validateMessages(records: MemoryRecords, magic: Byte, sourceCompressionType: CompressionType, targetCompressionType: CompressionType): Unit = {
     LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       new LongRef(0L),
       time,
       now = 0L,
@@ -91,7 +101,8 @@ class LogValidatorTest {
       1000L,
       RecordBatch.NO_PRODUCER_EPOCH,
       isFromClient = true,
-      KAFKA_2_3_IV1
+      KAFKA_2_3_IV1,
+      brokerTopicStats
     )
   }
 
@@ -105,6 +116,7 @@ class LogValidatorTest {
     // The timestamps should be overwritten
     val records = createRecords(magicValue = magic, timestamp = 1234L, codec = CompressionType.NONE)
     val validatedResults = LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(0),
       time= time,
       now = now,
@@ -116,7 +128,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats)
     val validatedRecords = validatedResults.validatedRecords
     assertEquals("message set size should not change", records.records.asScala.size, validatedRecords.records.asScala.size)
     validatedRecords.batches.asScala.foreach(batch => validateLogAppendTime(now, 1234L, batch))
@@ -143,6 +156,7 @@ class LogValidatorTest {
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V0, codec = CompressionType.GZIP)
     val validatedResults = LogValidator.validateMessagesAndAssignOffsets(
       records,
+      topicPartition,
       offsetCounter = new LongRef(0),
       time = time,
       now = now,
@@ -154,7 +168,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats)
     val validatedRecords = validatedResults.validatedRecords
 
     assertEquals("message set size should not change", records.records.asScala.size, validatedRecords.records.asScala.size)
@@ -185,6 +200,7 @@ class LogValidatorTest {
     val records = createRecords(magicValue = magic, timestamp = 1234L, codec = CompressionType.GZIP)
     val validatedResults = LogValidator.validateMessagesAndAssignOffsets(
       records,
+      topicPartition,
       offsetCounter = new LongRef(0),
       time = time,
       now = now,
@@ -196,7 +212,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats)
     val validatedRecords = validatedResults.validatedRecords
 
     assertEquals("message set size should not change", records.records.asScala.size,
@@ -243,6 +260,7 @@ class LogValidatorTest {
     records.buffer.putInt(DefaultRecordBatch.LAST_OFFSET_DELTA_OFFSET, lastOffsetDelta)
     LogValidator.validateMessagesAndAssignOffsets(
       records,
+      topicPartition,
       offsetCounter = new LongRef(0),
       time = time,
       now = time.milliseconds(),
@@ -254,7 +272,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats)
   }
 
   @Test
@@ -285,6 +304,7 @@ class LogValidatorTest {
       new SimpleRecord(timestampSeq(2), "beautiful".getBytes))
 
     val validatingResults = LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(0),
       time = time,
       now = System.currentTimeMillis(),
@@ -296,7 +316,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = partitionLeaderEpoch,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats)
     val validatedRecords = validatingResults.validatedRecords
 
     var i = 0
@@ -352,6 +373,7 @@ class LogValidatorTest {
       new SimpleRecord(timestampSeq(2), "beautiful".getBytes))
 
     val validatingResults = LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(0),
       time = time,
       now = System.currentTimeMillis(),
@@ -363,7 +385,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = partitionLeaderEpoch,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats)
     val validatedRecords = validatingResults.validatedRecords
 
     var i = 0
@@ -403,6 +426,7 @@ class LogValidatorTest {
   private def checkCreateTimeUpConversionFromV0(toMagic: Byte): Unit = {
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V0, codec = CompressionType.GZIP)
     val validatedResults = LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(0),
       time = time,
       now = System.currentTimeMillis(),
@@ -414,7 +438,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats)
     val validatedRecords = validatedResults.validatedRecords
 
     for (batch <- validatedRecords.batches.asScala) {
@@ -445,6 +470,7 @@ class LogValidatorTest {
     val timestamp = System.currentTimeMillis()
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V1, codec = CompressionType.GZIP, timestamp = timestamp)
     val validatedResults = LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(0),
       time = time,
       now = timestamp,
@@ -456,7 +482,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats)
     val validatedRecords = validatedResults.validatedRecords
 
     for (batch <- validatedRecords.batches.asScala) {
@@ -500,6 +527,7 @@ class LogValidatorTest {
       new SimpleRecord(timestampSeq(2), "beautiful".getBytes))
 
     val validatedResults = LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(0),
       time = time,
       now = System.currentTimeMillis(),
@@ -511,7 +539,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = partitionLeaderEpoch,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats)
     val validatedRecords = validatedResults.validatedRecords
 
     var i = 0
@@ -551,6 +580,7 @@ class LogValidatorTest {
       codec = CompressionType.NONE)
     LogValidator.validateMessagesAndAssignOffsets(
       records,
+      topicPartition,
       offsetCounter = new LongRef(0),
       time = time,
       now = System.currentTimeMillis(),
@@ -562,7 +592,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats)
   }
 
   @Test(expected = classOf[InvalidTimestampException])
@@ -572,6 +603,7 @@ class LogValidatorTest {
       codec = CompressionType.NONE)
     LogValidator.validateMessagesAndAssignOffsets(
       records,
+      topicPartition,
       offsetCounter = new LongRef(0),
       time = time,
       now = System.currentTimeMillis(),
@@ -583,7 +615,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats)
   }
 
   @Test(expected = classOf[InvalidTimestampException])
@@ -593,6 +626,7 @@ class LogValidatorTest {
       codec = CompressionType.GZIP)
     LogValidator.validateMessagesAndAssignOffsets(
       records,
+      topicPartition,
       offsetCounter = new LongRef(0),
       time = time,
       now = System.currentTimeMillis(),
@@ -604,7 +638,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats)
   }
 
   @Test(expected = classOf[InvalidTimestampException])
@@ -614,6 +649,7 @@ class LogValidatorTest {
       codec = CompressionType.GZIP)
     LogValidator.validateMessagesAndAssignOffsets(
       records,
+      topicPartition,
       offsetCounter = new LongRef(0),
       time = time,
       now = System.currentTimeMillis(),
@@ -625,7 +661,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats)
   }
 
   @Test
@@ -634,6 +671,7 @@ class LogValidatorTest {
     val offset = 1234567
     checkOffsets(records, 0)
     checkOffsets(LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -645,7 +683,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion).validatedRecords, offset)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
 
   @Test
@@ -654,6 +693,7 @@ class LogValidatorTest {
     val offset = 1234567
     checkOffsets(records, 0)
     checkOffsets(LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -665,7 +705,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion).validatedRecords, offset)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
 
   @Test
@@ -675,6 +716,7 @@ class LogValidatorTest {
     val offset = 1234567
     checkOffsets(records, 0)
     val messageWithOffset = LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -686,7 +728,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion).validatedRecords
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats).validatedRecords
     checkOffsets(messageWithOffset, offset)
   }
 
@@ -697,6 +740,7 @@ class LogValidatorTest {
     val offset = 1234567
     checkOffsets(records, 0)
     val messageWithOffset = LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -708,7 +752,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion).validatedRecords
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats).validatedRecords
     checkOffsets(messageWithOffset, offset)
   }
 
@@ -720,6 +765,7 @@ class LogValidatorTest {
     checkOffsets(records, 0)
     val compressedMessagesWithOffset = LogValidator.validateMessagesAndAssignOffsets(
       records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -731,7 +777,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion).validatedRecords
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats).validatedRecords
     checkOffsets(compressedMessagesWithOffset, offset)
   }
 
@@ -743,6 +790,7 @@ class LogValidatorTest {
     checkOffsets(records, 0)
     val compressedMessagesWithOffset = LogValidator.validateMessagesAndAssignOffsets(
       records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -754,7 +802,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion).validatedRecords
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats).validatedRecords
     checkOffsets(compressedMessagesWithOffset, offset)
   }
 
@@ -764,6 +813,7 @@ class LogValidatorTest {
     checkOffsets(records, 0)
     val offset = 1234567
     val validatedResults = LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -775,7 +825,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats)
     checkOffsets(validatedResults.validatedRecords, offset)
     verifyRecordConversionStats(validatedResults.recordConversionStats, numConvertedRecords = 3, records,
       compressed = false)
@@ -787,6 +838,7 @@ class LogValidatorTest {
     checkOffsets(records, 0)
     val offset = 1234567
     val validatedResults = LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -798,7 +850,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats)
     checkOffsets(validatedResults.validatedRecords, offset)
     verifyRecordConversionStats(validatedResults.recordConversionStats, numConvertedRecords = 3, records,
       compressed = false)
@@ -810,6 +863,7 @@ class LogValidatorTest {
     val offset = 1234567
     checkOffsets(records, 0)
     val validatedResults = LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -821,7 +875,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats)
     checkOffsets(validatedResults.validatedRecords, offset)
     verifyRecordConversionStats(validatedResults.recordConversionStats, numConvertedRecords = 3, records,
       compressed = true)
@@ -833,6 +888,7 @@ class LogValidatorTest {
     val offset = 1234567
     checkOffsets(records, 0)
     val validatedResults = LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -844,7 +900,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats)
     checkOffsets(validatedResults.validatedRecords, offset)
     verifyRecordConversionStats(validatedResults.recordConversionStats, numConvertedRecords = 3, records,
       compressed = true)
@@ -856,6 +913,7 @@ class LogValidatorTest {
     val endTxnMarker = new EndTransactionMarker(ControlRecordType.COMMIT, 0)
     val records = MemoryRecords.withEndTransactionMarker(23423L, 5, endTxnMarker)
     LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -867,7 +925,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats)
   }
 
   @Test
@@ -876,6 +935,7 @@ class LogValidatorTest {
     val endTxnMarker = new EndTransactionMarker(ControlRecordType.COMMIT, 0)
     val records = MemoryRecords.withEndTransactionMarker(23423L, 5, endTxnMarker)
     val result = LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -887,7 +947,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = false,
-      interBrokerProtocolVersion = ApiVersion.latestVersion)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats)
     val batches = TestUtils.toList(result.validatedRecords.batches)
     assertEquals(1, batches.size)
     val batch = batches.get(0)
@@ -901,6 +962,7 @@ class LogValidatorTest {
     val records = createRecords(RecordBatch.MAGIC_VALUE_V1, now, codec = CompressionType.NONE)
     checkOffsets(records, 0)
     checkOffsets(LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -912,7 +974,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion).validatedRecords, offset)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
 
   @Test
@@ -922,6 +985,7 @@ class LogValidatorTest {
     val records = createRecords(RecordBatch.MAGIC_VALUE_V1, now, CompressionType.GZIP)
     checkOffsets(records, 0)
     checkOffsets(LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -933,7 +997,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion).validatedRecords, offset)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
 
   @Test
@@ -942,6 +1007,7 @@ class LogValidatorTest {
     checkOffsets(records, 0)
     val offset = 1234567
     checkOffsets(LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -953,7 +1019,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion).validatedRecords, offset)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
 
   @Test
@@ -962,6 +1029,7 @@ class LogValidatorTest {
     val offset = 1234567
     checkOffsets(records, 0)
     checkOffsets(LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -973,7 +1041,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion).validatedRecords, offset)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
 
   @Test
@@ -983,6 +1052,7 @@ class LogValidatorTest {
     val records = createRecords(RecordBatch.MAGIC_VALUE_V2, now, codec = CompressionType.NONE)
     checkOffsets(records, 0)
     checkOffsets(LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -994,7 +1064,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion).validatedRecords, offset)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
 
   @Test
@@ -1004,6 +1075,7 @@ class LogValidatorTest {
     val records = createRecords(RecordBatch.MAGIC_VALUE_V2, now, CompressionType.GZIP)
     checkOffsets(records, 0)
     checkOffsets(LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -1015,7 +1087,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion).validatedRecords, offset)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
 
   @Test(expected = classOf[UnsupportedForMessageFormatException])
@@ -1027,6 +1100,7 @@ class LogValidatorTest {
     val records = MemoryRecords.withTransactionalRecords(CompressionType.NONE, producerId, producerEpoch, sequence,
       new SimpleRecord("hello".getBytes), new SimpleRecord("there".getBytes), new SimpleRecord("beautiful".getBytes))
     checkOffsets(LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -1038,7 +1112,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion).validatedRecords, offset)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
 
   @Test(expected = classOf[UnsupportedForMessageFormatException])
@@ -1050,6 +1125,7 @@ class LogValidatorTest {
     val records = MemoryRecords.withIdempotentRecords(CompressionType.NONE, producerId, producerEpoch, sequence,
       new SimpleRecord("hello".getBytes), new SimpleRecord("there".getBytes), new SimpleRecord("beautiful".getBytes))
     checkOffsets(LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -1061,7 +1137,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion).validatedRecords, offset)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
 
   @Test
@@ -1071,6 +1148,7 @@ class LogValidatorTest {
     val records = createRecords(RecordBatch.MAGIC_VALUE_V2, now, codec = CompressionType.NONE)
     checkOffsets(records, 0)
     checkOffsets(LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -1082,7 +1160,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion).validatedRecords, offset)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
 
   @Test
@@ -1092,6 +1171,7 @@ class LogValidatorTest {
     val records = createRecords(RecordBatch.MAGIC_VALUE_V2, now, CompressionType.GZIP)
     checkOffsets(records, 0)
     checkOffsets(LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -1103,7 +1183,33 @@ class LogValidatorTest {
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion).validatedRecords, offset)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats).validatedRecords, offset)
+  }
+
+  @Test
+  def testNonIncreasingOffsetRecordBatchHasMetricsLogged(): Unit = {
+    val records = createNonIncreasingOffsetRecords(RecordBatch.MAGIC_VALUE_V2)
+    records.batches().asScala.head.setLastOffset(2)
+    assertThrows[InvalidRecordException] {
+      LogValidator.validateMessagesAndAssignOffsets(records,
+        topicPartition,
+        offsetCounter = new LongRef(0L),
+        time = time,
+        now = System.currentTimeMillis(),
+        sourceCodec = DefaultCompressionCodec,
+        targetCodec = DefaultCompressionCodec,
+        compactedTopic = false,
+        magic = RecordBatch.MAGIC_VALUE_V0,
+        timestampType = TimestampType.CREATE_TIME,
+        timestampDiffMaxMs = 5000L,
+        partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
+        isFromClient = true,
+        interBrokerProtocolVersion = ApiVersion.latestVersion,
+        brokerTopicStats = brokerTopicStats)
+    }
+    assertEquals(metricsKeySet.count(_.getMBeanName.endsWith(s"${BrokerTopicStats.InvalidOffsetOrSequenceRecordsPerSec}")), 1)
+    assertTrue(meterCount(s"${BrokerTopicStats.InvalidOffsetOrSequenceRecordsPerSec}") > 0)
   }
 
   @Test(expected = classOf[InvalidRecordException])
@@ -1117,6 +1223,7 @@ class LogValidatorTest {
     // The timestamps should be overwritten
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V2, timestamp = 1234L, codec = CompressionType.NONE)
     LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(0),
       time= time,
       now = now,
@@ -1128,7 +1235,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = KAFKA_2_0_IV1)
+      interBrokerProtocolVersion = KAFKA_2_0_IV1,
+      brokerTopicStats = brokerTopicStats)
   }
 
   @Test(expected = classOf[InvalidRecordException])
@@ -1152,6 +1260,7 @@ class LogValidatorTest {
     buffer.flip()
     val records = MemoryRecords.readableRecords(buffer)
     LogValidator.validateMessagesAndAssignOffsets(records,
+      topicPartition,
       offsetCounter = new LongRef(offset),
       time = time,
       now = System.currentTimeMillis(),
@@ -1163,7 +1272,8 @@ class LogValidatorTest {
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
       isFromClient = true,
-      interBrokerProtocolVersion = ApiVersion.latestVersion)
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      brokerTopicStats = brokerTopicStats)
   }
 
   private def createRecords(magicValue: Byte,
@@ -1177,6 +1287,17 @@ class LogValidatorTest {
     builder.build()
   }
 
+  private def createNonIncreasingOffsetRecords(magicValue: Byte,
+                                               timestamp: Long = RecordBatch.NO_TIMESTAMP,
+                                               codec: CompressionType = CompressionType.NONE): MemoryRecords = {
+    val buf = ByteBuffer.allocate(512)
+    val builder = MemoryRecords.builder(buf, magicValue, codec, TimestampType.CREATE_TIME, 0L)
+    builder.appendWithOffset(0, timestamp, null, "hello".getBytes)
+    builder.appendWithOffset(2, timestamp, null, "there".getBytes)
+    builder.appendWithOffset(3, timestamp, null, "beautiful".getBytes)
+    builder.build()
+  }
+
   private def createTwoBatchedRecords(magicValue: Byte,
                                       timestamp: Long,
                                       codec: CompressionType): MemoryRecords = {
diff --git a/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala b/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
index 7a3b3a9ac5246..875e15e805f3b 100644
--- a/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
+++ b/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
@@ -18,9 +18,10 @@
 package kafka.metrics
 
 import java.util.Properties
+
 import javax.management.ObjectName
 import com.yammer.metrics.Metrics
-import com.yammer.metrics.core.{Meter, MetricPredicate}
+import com.yammer.metrics.core.MetricPredicate
 import org.junit.Test
 import org.junit.Assert._
 import kafka.integration.KafkaServerTestHarness
@@ -32,6 +33,7 @@ import scala.collection.JavaConverters._
 import scala.util.matching.Regex
 import kafka.log.LogConfig
 import org.apache.kafka.common.TopicPartition
+import org.scalatest.Assertions
 
 class MetricsTest extends KafkaServerTestHarness with Logging {
   val numNodes = 2
@@ -125,25 +127,25 @@ class MetricsTest extends KafkaServerTestHarness with Logging {
 
     // Consume messages to make bytesOut tick
     TestUtils.consumeTopicRecords(servers, topic, nMessages)
-    val initialReplicationBytesIn = meterCount(replicationBytesIn)
-    val initialReplicationBytesOut = meterCount(replicationBytesOut)
-    val initialBytesIn = meterCount(bytesIn)
-    val initialBytesOut = meterCount(bytesOut)
+    val initialReplicationBytesIn = TestUtils.meterCount(replicationBytesIn)
+    val initialReplicationBytesOut = TestUtils.meterCount(replicationBytesOut)
+    val initialBytesIn = TestUtils.meterCount(bytesIn)
+    val initialBytesOut = TestUtils.meterCount(bytesOut)
 
     // BytesOut doesn't include replication, so it shouldn't have changed
-    assertEquals(initialBytesOut, meterCount(bytesOut))
+    assertEquals(initialBytesOut, TestUtils.meterCount(bytesOut))
 
     // Produce a few messages to make the metrics tick
     TestUtils.generateAndProduceMessages(servers, topic, nMessages)
 
-    assertTrue(meterCount(replicationBytesIn) > initialReplicationBytesIn)
-    assertTrue(meterCount(replicationBytesOut) > initialReplicationBytesOut)
-    assertTrue(meterCount(bytesIn) > initialBytesIn)
+    assertTrue(TestUtils.meterCount(replicationBytesIn) > initialReplicationBytesIn)
+    assertTrue(TestUtils.meterCount(replicationBytesOut) > initialReplicationBytesOut)
+    assertTrue(TestUtils.meterCount(bytesIn) > initialBytesIn)
 
     // Consume messages to make bytesOut tick
     TestUtils.consumeTopicRecords(servers, topic, nMessages)
 
-    assertTrue(meterCount(bytesOut) > initialBytesOut)
+    assertTrue(TestUtils.meterCount(bytesOut) > initialBytesOut)
   }
 
   @Test
@@ -174,16 +176,6 @@ class MetricsTest extends KafkaServerTestHarness with Logging {
     assertEquals(metrics.keySet.asScala.count(_.getMBeanName == "kafka.server:type=SessionExpireListener,name=ZooKeeperDisconnectsPerSec"), 1)
   }
 
-  private def meterCount(metricName: String): Long = {
-    Metrics.defaultRegistry.allMetrics.asScala
-      .filterKeys(_.getMBeanName.endsWith(metricName))
-      .values
-      .headOption
-      .getOrElse(fail(s"Unable to find metric $metricName"))
-      .asInstanceOf[Meter]
-      .count
-  }
-
   private def topicMetrics(topic: Option[String]): Set[String] = {
     val metricNames = Metrics.defaultRegistry.allMetrics().keySet.asScala.map(_.getMBeanName)
     filterByTopicMetricRegex(metricNames, topic)
diff --git a/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala b/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala
index a02e6315a3f7d..32e49e5f85d5d 100644
--- a/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala
@@ -19,6 +19,7 @@ package kafka.server
 
 import java.util.Properties
 
+import com.yammer.metrics.Metrics
 import kafka.log.LogConfig
 import kafka.message.ZStdCompressionCodec
 import kafka.utils.TestUtils
@@ -38,6 +39,8 @@ import scala.collection.JavaConverters._
   */
 class ProduceRequestTest extends BaseRequestTest {
 
+  val metricsKeySet = Metrics.defaultRegistry.allMetrics.keySet.asScala
+
   @Test
   def testSimpleProduceRequest(): Unit = {
     val (partition, leader) = createTopicAndFindPartitionWithLeader("topic")
@@ -114,6 +117,8 @@ class ProduceRequestTest extends BaseRequestTest {
     assertEquals(Errors.CORRUPT_MESSAGE, partitionResponse.error)
     assertEquals(-1, partitionResponse.baseOffset)
     assertEquals(-1, partitionResponse.logAppendTime)
+    assertEquals(metricsKeySet.count(_.getMBeanName.endsWith(s"${BrokerTopicStats.InvalidMessageCrcRecordsPerSec}")), 1)
+    assertTrue(TestUtils.meterCount(s"${BrokerTopicStats.InvalidMessageCrcRecordsPerSec}") > 0)
   }
 
   @Test
diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index 074c107f5bb10..d28bd99c55382 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -38,6 +38,7 @@ import kafka.server._
 import kafka.server.checkpoints.OffsetCheckpointFile
 import Implicits._
 import com.yammer.metrics.Metrics
+import com.yammer.metrics.core.Meter
 import kafka.controller.LeaderIsrAndControllerEpoch
 import kafka.zk._
 import org.apache.kafka.clients.CommonClientConfigs
@@ -1585,6 +1586,16 @@ object TestUtils extends Logging {
     total.toLong
   }
 
+  def meterCount(metricName: String): Long = {
+    Metrics.defaultRegistry.allMetrics.asScala
+      .filterKeys(_.getMBeanName.endsWith(metricName))
+      .values
+      .headOption
+      .getOrElse(fail(s"Unable to find metric $metricName"))
+      .asInstanceOf[Meter]
+      .count
+  }
+
   def clearYammerMetrics(): Unit = {
     for (metricName <- Metrics.defaultRegistry.allMetrics.keySet.asScala)
       Metrics.defaultRegistry.removeMetric(metricName)

From c5f2bd64d1f21a44bc9832f80f4ae76d3c065b96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=BA=B7=E6=99=BA=E5=86=AC?= 
Date: Thu, 5 Sep 2019 11:37:23 +0800
Subject: [PATCH 0586/1071] MINOR: Fix few typos in the javadocs/docs

---
 .../org/apache/kafka/common/network/SslChannelBuilder.java    | 2 +-
 .../org/apache/kafka/common/network/SslTransportLayer.java    | 4 ++--
 .../java/org/apache/kafka/common/network/TransportLayer.java  | 2 +-
 core/src/main/scala/kafka/server/KafkaApis.scala              | 2 +-
 docs/protocol.html                                            | 2 +-
 docs/security.html                                            | 2 +-
 6 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/network/SslChannelBuilder.java b/clients/src/main/java/org/apache/kafka/common/network/SslChannelBuilder.java
index aade37c296973..9c6816124132e 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/SslChannelBuilder.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/SslChannelBuilder.java
@@ -50,7 +50,7 @@ public class SslChannelBuilder implements ChannelBuilder, ListenerReconfigurable
     private SslPrincipalMapper sslPrincipalMapper;
 
     /**
-     * Constructs a SSL channel builder. ListenerName is provided only
+     * Constructs an SSL channel builder. ListenerName is provided only
      * for server channel builder and will be null for client channel builder.
      */
     public SslChannelBuilder(Mode mode, ListenerName listenerName, boolean isInterBrokerListener) {
diff --git a/clients/src/main/java/org/apache/kafka/common/network/SslTransportLayer.java b/clients/src/main/java/org/apache/kafka/common/network/SslTransportLayer.java
index 1ea7e6e5b6031..6ed2cea80632d 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/SslTransportLayer.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/SslTransportLayer.java
@@ -152,7 +152,7 @@ public boolean isConnected() {
 
 
     /**
-    * Sends a SSL close message and closes socketChannel.
+    * Sends an SSL close message and closes socketChannel.
     */
     @Override
     public void close() throws IOException {
@@ -737,7 +737,7 @@ public Principal peerPrincipal() {
     }
 
     /**
-     * returns a SSL Session after the handshake is established
+     * returns an SSL Session after the handshake is established
      * throws IllegalStateException if the handshake is not established
      */
     public SSLSession sslSession() throws IllegalStateException {
diff --git a/clients/src/main/java/org/apache/kafka/common/network/TransportLayer.java b/clients/src/main/java/org/apache/kafka/common/network/TransportLayer.java
index a8a4b8730283c..b196c5be96c65 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/TransportLayer.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/TransportLayer.java
@@ -81,7 +81,7 @@ public interface TransportLayer extends ScatteringByteChannel, GatheringByteChan
     boolean hasPendingWrites();
 
     /**
-     * Returns `SSLSession.getPeerPrincipal()` if this is a SslTransportLayer and there is an authenticated peer,
+     * Returns `SSLSession.getPeerPrincipal()` if this is an SslTransportLayer and there is an authenticated peer,
      * `KafkaPrincipal.ANONYMOUS` is returned otherwise.
      */
     Principal peerPrincipal() throws IOException;
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 300003fc76628..0d2e03d032167 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -1600,7 +1600,7 @@ class KafkaApis(val requestChannel: RequestChannel,
   def handleApiVersionsRequest(request: RequestChannel.Request): Unit = {
     // Note that broker returns its full list of supported ApiKeys and versions regardless of current
     // authentication state (e.g., before SASL authentication on an SASL listener, do note that no
-    // Kafka protocol requests may take place on a SSL listener before the SSL handshake is finished).
+    // Kafka protocol requests may take place on an SSL listener before the SSL handshake is finished).
     // If this is considered to leak information about the broker version a workaround is to use SSL
     // with client authentication which is performed at an earlier stage of the connection where the
     // ApiVersionRequest is not available.
diff --git a/docs/protocol.html b/docs/protocol.html
index b884992a7bcdd..30a04ac70dd48 100644
--- a/docs/protocol.html
+++ b/docs/protocol.html
@@ -130,7 +130,7 @@ 
            Retrieving Supported API versions<
         this happens after SSL connection has been established.
     
          1. On receiving ApiVersionsRequest, a broker returns its full list of supported ApiKeys and
         versions regardless of current authentication state (e.g., before SASL authentication on an SASL listener, do note that no
-        Kafka protocol requests may take place on a SSL listener before the SSL handshake is finished). If this is considered to
+        Kafka protocol requests may take place on an SSL listener before the SSL handshake is finished). If this is considered to
         leak information about the broker version a workaround is to use SSL with client authentication which is performed at an
         earlier stage of the connection where the ApiVersionRequest is not available. Also, note that broker versions older
         than 0.10.0.0 do not support this API and will either ignore the request or close connection in response to the request.
            2. 
diff --git a/docs/security.html b/docs/security.html
index ea103b70e42ef..b4c86546f0806 100644
--- a/docs/security.html
+++ b/docs/security.html
@@ -1755,7 +1755,7 @@ 
            7.5 Incorp
 
         When performing an incremental bounce stop the brokers cleanly via a SIGTERM. It's also good practice to wait for restarted replicas to return to the ISR list before moving onto the next node.
         
            
-        As an example, say we wish to encrypt both broker-client and broker-broker communication with SSL. In the first incremental bounce, a SSL port is opened on each node:
+        As an example, say we wish to encrypt both broker-client and broker-broker communication with SSL. In the first incremental bounce, an SSL port is opened on each node:
             
                         listeners=PLAINTEXT://broker1:9091,SSL://broker1:9092
             
            

From b66ea874406c3b4d59b23e47e036cbdd4c843c7a Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Thu, 5 Sep 2019 09:21:01 +0100
Subject: [PATCH 0587/1071] KAFKA-8866; Return exceptions as
 Optional in authorizer API (#7294)

Reviewers: Ismael Juma , Manikumar Reddy 
---
 .../server/authorizer/AclCreateResult.java    | 17 +++++-------
 .../server/authorizer/AclDeleteResult.java    | 26 +++++++++----------
 .../main/scala/kafka/admin/AclCommand.scala   | 19 +++++++-------
 .../security/auth/SimpleAclAuthorizer.scala   |  9 ++++---
 .../authorizer/AuthorizerWrapper.scala        |  2 +-
 .../main/scala/kafka/server/KafkaApis.scala   |  9 +++----
 .../kafka/api/AuthorizerIntegrationTest.scala |  4 +--
 .../DescribeAuthorizedOperationsTest.scala    |  4 +--
 .../api/SslAdminClientIntegrationTest.scala   |  4 +--
 .../authorizer/AclAuthorizerTest.scala        | 16 +++++-------
 .../DelegationTokenManagerTest.scala          |  4 +--
 11 files changed, 52 insertions(+), 62 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/server/authorizer/AclCreateResult.java b/clients/src/main/java/org/apache/kafka/server/authorizer/AclCreateResult.java
index 7255df5f06373..70b9c00b3646c 100644
--- a/clients/src/main/java/org/apache/kafka/server/authorizer/AclCreateResult.java
+++ b/clients/src/main/java/org/apache/kafka/server/authorizer/AclCreateResult.java
@@ -20,13 +20,15 @@
 import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.common.errors.ApiException;
 
+import java.util.Optional;
+
 @InterfaceStability.Evolving
 public class AclCreateResult {
     public static final AclCreateResult SUCCESS = new AclCreateResult();
 
     private final ApiException exception;
 
-    public AclCreateResult() {
+    private AclCreateResult() {
         this(null);
     }
 
@@ -35,16 +37,9 @@ public AclCreateResult(ApiException exception) {
     }
 
     /**
-     * Returns any exception during create. If exception is null, the request has succeeded.
-     */
-    public ApiException exception() {
-        return exception;
-    }
-
-    /**
-     * Returns true if the request failed.
+     * Returns any exception during create. If exception is empty, the request has succeeded.
      */
-    public boolean failed() {
-        return exception != null;
+    public Optional exception() {
+        return exception == null ? Optional.empty() : Optional.of(exception);
     }
 }
diff --git a/clients/src/main/java/org/apache/kafka/server/authorizer/AclDeleteResult.java b/clients/src/main/java/org/apache/kafka/server/authorizer/AclDeleteResult.java
index ed4449a6ff992..994d6fde74f1c 100644
--- a/clients/src/main/java/org/apache/kafka/server/authorizer/AclDeleteResult.java
+++ b/clients/src/main/java/org/apache/kafka/server/authorizer/AclDeleteResult.java
@@ -19,6 +19,8 @@
 
 import java.util.Collections;
 import java.util.Collection;
+import java.util.Optional;
+
 import org.apache.kafka.common.acl.AclBinding;
 import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.common.errors.ApiException;
@@ -43,9 +45,11 @@ private AclDeleteResult(Collection deleteResults, ApiExc
 
     /**
      * Returns any exception while attempting to match ACL filter to delete ACLs.
+     * If exception is empty, filtering has succeeded. See {@link #aclBindingDeleteResults()}
+     * for deletion results for each filter.
      */
-    public ApiException exception() {
-        return exception;
+    public Optional exception() {
+        return exception == null ? Optional.empty() : Optional.of(exception);
     }
 
     /**
@@ -73,25 +77,19 @@ public AclBindingDeleteResult(AclBinding aclBinding, ApiException exception) {
         }
 
         /**
-         * Returns ACL binding that matched the delete filter. {@link #deleted()} indicates if
-         * the binding was deleted.
+         * Returns ACL binding that matched the delete filter. If {@link #exception()} is
+         * empty, the ACL binding was successfully deleted.
          */
         public AclBinding aclBinding() {
             return aclBinding;
         }
 
         /**
-         * Returns exception that resulted in failure to delete ACL binding.
-         */
-        public ApiException exception() {
-            return exception;
-        }
-
-        /**
-         * Returns true if ACL binding was deleted, false otherwise.
+         * Returns any exception that resulted in failure to delete ACL binding.
+         * If exception is empty, the ACL binding was successfully deleted.
          */
-        public boolean deleted() {
-            return exception == null;
+        public Optional exception() {
+            return exception == null ? Optional.empty() : Optional.of(exception);
         }
     }
 }
diff --git a/core/src/main/scala/kafka/admin/AclCommand.scala b/core/src/main/scala/kafka/admin/AclCommand.scala
index d4d95a9b79407..090ac7fab6f5a 100644
--- a/core/src/main/scala/kafka/admin/AclCommand.scala
+++ b/core/src/main/scala/kafka/admin/AclCommand.scala
@@ -36,6 +36,7 @@ import org.apache.kafka.common.utils.{Utils, SecurityUtils => JSecurityUtils}
 import org.apache.kafka.server.authorizer.{Authorizer => JAuthorizer}
 
 import scala.collection.JavaConverters._
+import scala.compat.java8.OptionConverters._
 import scala.collection.mutable
 import scala.io.StdIn
 
@@ -315,9 +316,9 @@ object AclCommand extends Logging {
           println(s"Adding ACLs for resource `$resource`: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline")
           val aclBindings = acls.map(acl => new AclBinding(resource, acl))
           authorizer.createAcls(null, aclBindings.toList.asJava).asScala.foreach { result =>
-            if (result.failed) {
-              println(s"Error while adding ACLs: ${result.exception.getMessage}")
-              println(Utils.stackTrace(result.exception))
+            result.exception.asScala.foreach { exception =>
+              println(s"Error while adding ACLs: ${exception.getMessage}")
+              println(Utils.stackTrace(exception))
             }
           }
         }
@@ -374,14 +375,14 @@ object AclCommand extends Logging {
         authorizer.deleteAcls(null, aclBindingFilters)
       }
       result.asScala.foreach { result =>
-        if (result.exception != null) {
-          println(s"Error while removing ACLs: ${result.exception.getMessage}")
-          println(Utils.stackTrace(result.exception))
+        result.exception.asScala.foreach { exception =>
+          println(s"Error while removing ACLs: ${exception.getMessage}")
+          println(Utils.stackTrace(exception))
         }
         result.aclBindingDeleteResults.asScala.foreach { deleteResult =>
-          if (deleteResult.exception != null) {
-            println(s"Error while removing ACLs: ${deleteResult.exception.getMessage}")
-            println(Utils.stackTrace(deleteResult.exception))
+          deleteResult.exception.asScala.foreach { exception =>
+            println(s"Error while removing ACLs: ${exception.getMessage}")
+            println(Utils.stackTrace(exception))
           }
         }
       }
diff --git a/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala b/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala
index bdbcf9366a63d..6e04021c80281 100644
--- a/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala
+++ b/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala
@@ -29,6 +29,7 @@ import org.apache.kafka.server.authorizer.{Action, AuthorizableRequestContext, A
 
 import scala.collection.mutable
 import scala.collection.JavaConverters._
+import scala.compat.java8.OptionConverters._
 
 @deprecated("Use kafka.security.authorizer.AclAuthorizer", "Since 2.4")
 object SimpleAclAuthorizer {
@@ -125,15 +126,15 @@ class SimpleAclAuthorizer extends Authorizer with Logging {
   private def createAcls(bindings: Set[AclBinding]): Unit = {
     aclAuthorizer.maxUpdateRetries = maxUpdateRetries
     val results = aclAuthorizer.createAcls(null, bindings.toList.asJava).asScala
-    results.find(_.exception != null).foreach { r => throw r.exception }
+    results.foreach { result => result.exception.asScala.foreach(e => throw e) }
   }
 
   private def deleteAcls(filters: Set[AclBindingFilter]): Boolean = {
     aclAuthorizer.maxUpdateRetries = maxUpdateRetries
     val results = aclAuthorizer.deleteAcls(null, filters.toList.asJava).asScala
-    results.find(_.exception != null).foreach { r => throw r.exception }
-    results.flatMap(_.aclBindingDeleteResults.asScala).find(_.exception != null).foreach { r => throw r.exception }
-    results.exists(r => r.aclBindingDeleteResults.asScala.exists(_.deleted))
+    results.foreach { result => result.exception.asScala.foreach(e => throw e) }
+    results.flatMap(_.aclBindingDeleteResults.asScala).foreach { result => result.exception.asScala.foreach(e => throw e) }
+    results.exists(r => r.aclBindingDeleteResults.asScala.exists(d => !d.exception.isPresent))
   }
 
   private def acls(filter: AclBindingFilter): Map[Resource, Set[Acl]] = {
diff --git a/core/src/main/scala/kafka/security/authorizer/AuthorizerWrapper.scala b/core/src/main/scala/kafka/security/authorizer/AuthorizerWrapper.scala
index 86ee341786a57..df10ff45eaaee 100644
--- a/core/src/main/scala/kafka/security/authorizer/AuthorizerWrapper.scala
+++ b/core/src/main/scala/kafka/security/authorizer/AuthorizerWrapper.scala
@@ -65,7 +65,7 @@ class AuthorizerWrapper(private[kafka] val baseAuthorizer: kafka.security.auth.A
           case Right((resource, acl)) =>
             try {
               baseAuthorizer.addAcls(Set(acl), resource)
-              new AclCreateResult
+              AclCreateResult.SUCCESS
             } catch {
               case e: ApiException => new AclCreateResult(e)
               case e: Throwable => new AclCreateResult(new InvalidRequestException("Failed to create ACL", e))
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 0d2e03d032167..fd8fa4de428af 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -2197,10 +2197,7 @@ class KafkaApis(val requestChannel: RequestChannel,
 
         val aclCreationResults = aclBindings.map { acl =>
           val result = errorResults.getOrElse(acl, createResults.get(validBindings.indexOf(acl)))
-          if (result.failed)
-            new AclCreationResponse(ApiError.fromThrowable(result.exception))
-          else
-            new AclCreationResponse(ApiError.NONE)
+          new AclCreationResponse(result.exception.asScala.map(ApiError.fromThrowable).getOrElse(ApiError.NONE))
         }
 
         sendResponseMaybeThrottle(request, requestThrottleMs =>
@@ -2219,8 +2216,8 @@ class KafkaApis(val requestChannel: RequestChannel,
       case Some(auth) =>
 
         val results = auth.deleteAcls(request.context, deleteAclsRequest.filters)
-        def toErrorCode(exception: ApiException): ApiError = {
-          if (exception != null) ApiError.fromThrowable(exception) else ApiError.NONE
+        def toErrorCode(exception: Optional[ApiException]): ApiError = {
+          exception.asScala.map(ApiError.fromThrowable).getOrElse(ApiError.NONE)
         }
         val filterResponses = results.asScala.map { result =>
           val deletions = result.aclBindingDeleteResults().asScala.toList.map { deletionResult =>
diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
index d3fccf5fec249..af80a99ba0c16 100644
--- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
@@ -70,6 +70,7 @@ import org.junit.{After, Assert, Before, Test}
 import org.scalatest.Assertions.intercept
 
 import scala.collection.JavaConverters._
+import scala.compat.java8.OptionConverters._
 import scala.collection.mutable
 import scala.collection.mutable.Buffer
 
@@ -1695,8 +1696,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
     val aclBindings = acls.map { acl => new AclBinding(resource, acl) }
     servers.head.dataPlaneRequestProcessor.authorizer.get
       .createAcls(null, aclBindings.toList.asJava).asScala.foreach {result =>
-        if (result.failed())
-          throw result.exception()
+        result.exception.asScala.foreach { e => throw e }
       }
     val aclFilter = new AclBindingFilter(resource.toFilter, AccessControlEntryFilter.ANY)
     TestUtils.waitAndVerifyAcls(
diff --git a/core/src/test/scala/integration/kafka/api/DescribeAuthorizedOperationsTest.scala b/core/src/test/scala/integration/kafka/api/DescribeAuthorizedOperationsTest.scala
index a4f76ce91f123..9fa1dbc62eebd 100644
--- a/core/src/test/scala/integration/kafka/api/DescribeAuthorizedOperationsTest.scala
+++ b/core/src/test/scala/integration/kafka/api/DescribeAuthorizedOperationsTest.scala
@@ -28,7 +28,7 @@ import org.apache.kafka.common.resource.{PatternType, ResourcePattern, ResourceT
 import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
 import org.apache.kafka.common.utils.Utils
 import org.apache.kafka.server.authorizer.Authorizer
-import org.junit.Assert.{assertEquals, assertNull}
+import org.junit.Assert.{assertEquals, assertFalse}
 import org.junit.{After, Before, Test}
 
 import scala.collection.JavaConverters._
@@ -60,7 +60,7 @@ class DescribeAuthorizedOperationsTest extends IntegrationTestHarness with SaslS
         new AclBinding(clusterResource, accessControlEntry(JaasTestUtils.KafkaServerPrincipalUnqualifiedName.toString, ALLOW, CLUSTER_ACTION)),
         new AclBinding(clusterResource, accessControlEntry(JaasTestUtils.KafkaClientPrincipalUnqualifiedName2.toString, ALLOW, ALTER)),
         new AclBinding(topicResource, accessControlEntry(JaasTestUtils.KafkaClientPrincipalUnqualifiedName2.toString, ALLOW, DESCRIBE))).asJava)
-      result.asScala.foreach { result => assertNull(result.exception) }
+      result.asScala.foreach { result => assertFalse(result.exception.isPresent) }
 
     } finally {
       authorizer.close()
diff --git a/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala
index 4ad2e5329f00f..64f44410ced02 100644
--- a/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala
@@ -78,8 +78,8 @@ class SslAdminClientIntegrationTest extends SaslSslAdminClientIntegrationTest {
     val clusterFilter = new AclBindingFilter(clusterResourcePattern.toFilter, AccessControlEntryFilter.ANY)
     val prevAcls = authorizer.acls(clusterFilter).asScala.map(_.entry).toSet
     val deleteFilter = new AclBindingFilter(clusterResourcePattern.toFilter, ace.toFilter)
-    Assert.assertTrue(authorizer.deleteAcls(null, Collections.singletonList(deleteFilter))
-      .get(0).aclBindingDeleteResults().asScala.head.deleted)
+    Assert.assertFalse(authorizer.deleteAcls(null, Collections.singletonList(deleteFilter))
+      .get(0).aclBindingDeleteResults().asScala.head.exception.isPresent)
     TestUtils.waitAndVerifyAcls(prevAcls -- Set(ace), authorizer, clusterResourcePattern)
   }
 
diff --git a/core/src/test/scala/unit/kafka/security/authorizer/AclAuthorizerTest.scala b/core/src/test/scala/unit/kafka/security/authorizer/AclAuthorizerTest.scala
index aac9594f39a05..3cba4ed441758 100644
--- a/core/src/test/scala/unit/kafka/security/authorizer/AclAuthorizerTest.scala
+++ b/core/src/test/scala/unit/kafka/security/authorizer/AclAuthorizerTest.scala
@@ -46,6 +46,7 @@ import org.junit.Assert._
 import org.junit.{After, Before, Test}
 
 import scala.collection.JavaConverters._
+import scala.compat.java8.OptionConverters._
 
 class AclAuthorizerTest extends ZooKeeperTestHarness {
 
@@ -704,9 +705,9 @@ class AclAuthorizerTest extends ZooKeeperTestHarness {
       new AclBindingFilter(resource2.toFilter, AccessControlEntryFilter.ANY),
       new AclBindingFilter(new ResourcePatternFilter(TOPIC, "baz", PatternType.ANY), AccessControlEntryFilter.ANY))
     val deleteResults = aclAuthorizer.deleteAcls(requestContext, filters.asJava).asScala
-    assertEquals(List.empty, deleteResults.filter(_.exception != null))
+    assertEquals(List.empty, deleteResults.filter(_.exception.isPresent))
     filters.indices.foreach { i =>
-      assertEquals(Set.empty, deleteResults(i).aclBindingDeleteResults.asScala.toSet.filter(_.exception != null))
+      assertEquals(Set.empty, deleteResults(i).aclBindingDeleteResults.asScala.toSet.filter(_.exception.isPresent))
     }
     assertEquals(Set(acl3, acl4), deleteResults(0).aclBindingDeleteResults.asScala.map(_.aclBinding).toSet)
     assertEquals(Set(acl1), deleteResults(1).aclBindingDeleteResults.asScala.map(_.aclBinding).toSet)
@@ -835,8 +836,7 @@ class AclAuthorizerTest extends ZooKeeperTestHarness {
   private def addAcls(authorizer: AclAuthorizer, aces: Set[AccessControlEntry], resourcePattern: ResourcePattern): Unit = {
     val bindings = aces.map { ace => new AclBinding(resourcePattern, ace) }
     authorizer.createAcls(requestContext, bindings.toList.asJava).asScala.foreach { result =>
-      if (result.exception != null)
-        throw result.exception
+      result.exception.asScala.foreach { e => throw e }
     }
   }
 
@@ -846,13 +846,11 @@ class AclAuthorizerTest extends ZooKeeperTestHarness {
     else
       aces.map { ace => new AclBinding(resourcePattern, ace).toFilter }
     authorizer.deleteAcls(requestContext, bindings.toList.asJava).asScala.forall { result =>
-      if (result.exception != null)
-        throw result.exception
+      result.exception.asScala.foreach { e => throw e }
       result.aclBindingDeleteResults.asScala.foreach { r =>
-        if (r.exception != null)
-          throw r.exception
+        r.exception.asScala.foreach { e => throw e }
       }
-      result.aclBindingDeleteResults.asScala.exists(_.deleted)
+      result.aclBindingDeleteResults.asScala.exists(_.exception.asScala.isEmpty)
     }
   }
 
diff --git a/core/src/test/scala/unit/kafka/security/token/delegation/DelegationTokenManagerTest.scala b/core/src/test/scala/unit/kafka/security/token/delegation/DelegationTokenManagerTest.scala
index 7398a6db6b551..8550f157357c9 100644
--- a/core/src/test/scala/unit/kafka/security/token/delegation/DelegationTokenManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/security/token/delegation/DelegationTokenManagerTest.scala
@@ -44,6 +44,7 @@ import org.junit.Assert._
 import org.junit.{After, Before, Test}
 
 import scala.collection.JavaConverters._
+import scala.compat.java8.OptionConverters._
 import scala.collection.mutable.Buffer
 
 class DelegationTokenManagerTest extends ZooKeeperTestHarness  {
@@ -281,8 +282,7 @@ class DelegationTokenManagerTest extends ZooKeeperTestHarness  {
 
     def createAcl(aclBinding: AclBinding): Unit = {
       val result = aclAuthorizer.createAcls(null, List(aclBinding).asJava).get(0)
-      if (result.failed)
-        throw result.exception
+      result.exception.asScala.foreach { e => throw e }
     }
 
     //get all tokens for multiple owners (owner1, renewer4) and with permission

From c9318ced19c812c90066af468732d8af8d12f890 Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Thu, 5 Sep 2019 09:23:11 +0100
Subject: [PATCH 0588/1071] KAFKA-8857; Don't check synonyms while determining
 if config is readOnly (#7278)

Reviewers: Manikumar Reddy 
---
 .../scala/kafka/server/AdminManager.scala     |  2 +-
 .../DynamicBrokerReconfigurationTest.scala    | 41 +++++++++++++++++--
 2 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/core/src/main/scala/kafka/server/AdminManager.scala b/core/src/main/scala/kafka/server/AdminManager.scala
index fc3c93432f366..3ed365e43c7b3 100644
--- a/core/src/main/scala/kafka/server/AdminManager.scala
+++ b/core/src/main/scala/kafka/server/AdminManager.scala
@@ -661,7 +661,7 @@ class AdminManager(val config: KafkaConfig,
         .filter(perBrokerConfig || _.source == ConfigSource.DYNAMIC_DEFAULT_BROKER_CONFIG)
     val synonyms = if (!includeSynonyms) List.empty else allSynonyms
     val source = if (allSynonyms.isEmpty) ConfigSource.DEFAULT_CONFIG else allSynonyms.head.source
-    val readOnly = !allNames.exists(DynamicBrokerConfig.AllDynamicConfigs.contains)
+    val readOnly = !DynamicBrokerConfig.AllDynamicConfigs.contains(name)
     new DescribeConfigsResponse.ConfigEntry(name, valueAsString, source, isSensitive, readOnly, synonyms.asJava)
   }
 }
diff --git a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
index 3a9920e6eb294..f58046ad48917 100644
--- a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
+++ b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
@@ -119,6 +119,8 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
       props.put(KafkaConfig.NumReplicaFetchersProp, "2") // greater than one to test reducing threads
       props.put(KafkaConfig.ProducerQuotaBytesPerSecondDefaultProp, "10000000") // non-default value to trigger a new metric
       props.put(KafkaConfig.PasswordEncoderSecretProp, "dynamic-config-secret")
+      props.put(KafkaConfig.LogRetentionTimeMillisProp, 1680000000.toString)
+      props.put(KafkaConfig.LogRetentionTimeHoursProp, 168.toString)
 
       props ++= sslProperties1
       props ++= securityProps(sslProperties1, KEYSTORE_PROPS, listenerPrefix(SecureInternal))
@@ -159,9 +161,10 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
   }
 
   @Test
-  def testKeyStoreDescribeUsingAdminClient(): Unit = {
+  def testConfigDescribeUsingAdminClient(): Unit = {
 
-    def verifyConfig(configName: String, configEntry: ConfigEntry, isSensitive: Boolean, expectedProps: Properties): Unit = {
+    def verifyConfig(configName: String, configEntry: ConfigEntry, isSensitive: Boolean, isReadOnly: Boolean,
+                     expectedProps: Properties): Unit = {
       if (isSensitive) {
         assertTrue(s"Value is sensitive: $configName", configEntry.isSensitive)
         assertNull(s"Sensitive value returned for $configName", configEntry.value)
@@ -169,6 +172,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
         assertFalse(s"Config is not sensitive: $configName", configEntry.isSensitive)
         assertEquals(expectedProps.getProperty(configName), configEntry.value)
       }
+      assertEquals(s"isReadOnly incorrect for $configName: $configEntry", isReadOnly, configEntry.isReadOnly)
     }
 
     def verifySynonym(configName: String, synonym: ConfigSynonym, isSensitive: Boolean,
@@ -203,7 +207,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
       KEYSTORE_PROPS.asScala.foreach { configName =>
         val desc = configEntry(configDesc, s"$prefix$configName")
         val isSensitive = configName.contains("password")
-        verifyConfig(configName, desc, isSensitive, if (prefix.isEmpty) invalidSslProperties else sslProperties1)
+        verifyConfig(configName, desc, isSensitive, isReadOnly = prefix.nonEmpty, if (prefix.isEmpty) invalidSslProperties else sslProperties1)
         val defaultValue = if (configName == SSL_KEYSTORE_TYPE_CONFIG) Some("JKS") else None
         verifySynonyms(configName, desc.synonyms, isSensitive, prefix, defaultValue)
       }
@@ -215,6 +219,37 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
     val configDesc = describeConfig(adminClient)
     verifySslConfig("listener.name.external.", sslProperties1, configDesc)
     verifySslConfig("", invalidSslProperties, configDesc)
+
+    // Verify a few log configs with and without synonyms
+    val expectedProps = new Properties
+    expectedProps.setProperty(KafkaConfig.LogRetentionTimeMillisProp, "1680000000")
+    expectedProps.setProperty(KafkaConfig.LogRetentionTimeHoursProp, "168")
+    expectedProps.setProperty(KafkaConfig.LogRollTimeHoursProp, "168")
+    expectedProps.setProperty(KafkaConfig.LogCleanerThreadsProp, "1")
+    val logRetentionMs = configEntry(configDesc, KafkaConfig.LogRetentionTimeMillisProp)
+    verifyConfig(KafkaConfig.LogRetentionTimeMillisProp, logRetentionMs,
+      isSensitive = false, isReadOnly = false, expectedProps)
+    val logRetentionHours = configEntry(configDesc, KafkaConfig.LogRetentionTimeHoursProp)
+    verifyConfig(KafkaConfig.LogRetentionTimeHoursProp, logRetentionHours,
+      isSensitive = false, isReadOnly = true, expectedProps)
+    val logRollHours = configEntry(configDesc, KafkaConfig.LogRollTimeHoursProp)
+    verifyConfig(KafkaConfig.LogRollTimeHoursProp, logRollHours,
+      isSensitive = false, isReadOnly = true, expectedProps)
+    val logCleanerThreads = configEntry(configDesc, KafkaConfig.LogCleanerThreadsProp)
+    verifyConfig(KafkaConfig.LogCleanerThreadsProp, logCleanerThreads,
+      isSensitive = false, isReadOnly = false, expectedProps)
+
+    def synonymsList(configEntry: ConfigEntry): List[(String, ConfigSource)] =
+      configEntry.synonyms.asScala.map(s => (s.name, s.source)).toList
+    assertEquals(List((KafkaConfig.LogRetentionTimeMillisProp, ConfigSource.STATIC_BROKER_CONFIG),
+      (KafkaConfig.LogRetentionTimeHoursProp, ConfigSource.STATIC_BROKER_CONFIG),
+      (KafkaConfig.LogRetentionTimeHoursProp, ConfigSource.DEFAULT_CONFIG)),
+      synonymsList(logRetentionMs))
+    assertEquals(List((KafkaConfig.LogRetentionTimeHoursProp, ConfigSource.STATIC_BROKER_CONFIG),
+      (KafkaConfig.LogRetentionTimeHoursProp, ConfigSource.DEFAULT_CONFIG)),
+      synonymsList(logRetentionHours))
+    assertEquals(List((KafkaConfig.LogRollTimeHoursProp, ConfigSource.DEFAULT_CONFIG)), synonymsList(logRollHours))
+    assertEquals(List((KafkaConfig.LogCleanerThreadsProp, ConfigSource.DEFAULT_CONFIG)), synonymsList(logCleanerThreads))
   }
 
   @Test

From deac5d93cef000f408332a7648a3062e4e25e388 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Thu, 5 Sep 2019 09:10:52 -0700
Subject: [PATCH 0589/1071] KAFKA-8724; Improve range checking when computing
 cleanable partitions (#7264)

This patch contains a few improvements on the offset range handling when computing the cleanable range of offsets.

1. It adds bounds checking to ensure the dirty offset cannot be larger than the log end offset. If it is, we reset to the log start offset.
2. It adds a method to get the non-active segments in the log while holding the lock. This ensures that a truncation cannot lead to an invalid segment range.
3. It improves exception messages in the case that an inconsistent segment range is provided so that we have more information to find the root cause.

The patch also fixes a few problems in `LogCleanerManagerTest` due to unintended reuse of the underlying log directory.

Reviewers: Vikas Singh , Jun Rao 
---
 core/src/main/scala/kafka/log/Log.scala       |  12 ++
 .../src/main/scala/kafka/log/LogCleaner.scala |  11 +-
 .../scala/kafka/log/LogCleanerManager.scala   |  38 ++--
 .../kafka/log/LogCleanerManagerTest.scala     | 193 ++++++++++--------
 .../scala/unit/kafka/log/LogCleanerTest.scala |  46 ++---
 .../test/scala/unit/kafka/log/LogTest.scala   |  37 ++++
 6 files changed, 208 insertions(+), 129 deletions(-)

diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index 34a89dc43165f..370074929ca3a 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -2106,12 +2106,24 @@ class Log(@volatile var dir: File,
   def logSegments(from: Long, to: Long): Iterable[LogSegment] = {
     lock synchronized {
       val view = Option(segments.floorKey(from)).map { floor =>
+        if (to < floor)
+          throw new IllegalArgumentException(s"Invalid log segment range: requested segments from offset $from " +
+            s"mapping to segment with base offset $floor, which is greater than limit offset $to")
         segments.subMap(floor, to)
       }.getOrElse(segments.headMap(to))
       view.values.asScala
     }
   }
 
+  def nonActiveLogSegmentsFrom(from: Long): Iterable[LogSegment] = {
+    lock synchronized {
+      if (from > activeSegment.baseOffset)
+        throw new IllegalArgumentException("Illegal request for non-active segments beginning at " +
+          s"offset $from, which is larger than the active segment's base offset ${activeSegment.baseOffset}")
+      logSegments(from, activeSegment.baseOffset)
+    }
+  }
+
   /**
    * Get the largest log segment with a base offset less than or equal to the given offset, if one exists.
    * @return the optional log segment
diff --git a/core/src/main/scala/kafka/log/LogCleaner.scala b/core/src/main/scala/kafka/log/LogCleaner.scala
index 955733b402f35..47c1fa4f001f4 100644
--- a/core/src/main/scala/kafka/log/LogCleaner.scala
+++ b/core/src/main/scala/kafka/log/LogCleaner.scala
@@ -440,7 +440,7 @@ object LogCleaner {
     * @return the biggest uncleanable offset and the total amount of cleanable bytes
     */
   def calculateCleanableBytes(log: Log, firstDirtyOffset: Long, uncleanableOffset: Long): (Long, Long) = {
-    val firstUncleanableSegment = log.logSegments(uncleanableOffset, log.activeSegment.baseOffset).headOption.getOrElse(log.activeSegment)
+    val firstUncleanableSegment = log.nonActiveLogSegmentsFrom(uncleanableOffset).headOption.getOrElse(log.activeSegment)
     val firstUncleanableOffset = firstUncleanableSegment.baseOffset
     val cleanableBytes = log.logSegments(firstDirtyOffset, math.max(firstDirtyOffset, firstUncleanableOffset)).map(_.size.toLong).sum
 
@@ -466,7 +466,7 @@ private[log] class Cleaner(val id: Int,
                            dupBufferLoadFactor: Double,
                            throttler: Throttler,
                            time: Time,
-                           checkDone: (TopicPartition) => Unit) extends Logging {
+                           checkDone: TopicPartition => Unit) extends Logging {
 
   protected override def loggerName = classOf[LogCleaner].getName
 
@@ -1042,8 +1042,11 @@ private class CleanerStats(time: Time = Time.SYSTEM) {
   * Helper class for a log, its topic/partition, the first cleanable position, the first uncleanable dirty position,
   * and whether it needs compaction immediately.
   */
-private case class LogToClean(topicPartition: TopicPartition, log: Log, firstDirtyOffset: Long,
-                              uncleanableOffset: Long, needCompactionNow: Boolean = false) extends Ordered[LogToClean] {
+private case class LogToClean(topicPartition: TopicPartition,
+                              log: Log,
+                              firstDirtyOffset: Long,
+                              uncleanableOffset: Long,
+                              needCompactionNow: Boolean = false) extends Ordered[LogToClean] {
   val cleanBytes = log.logSegments(-1, firstDirtyOffset).map(_.size.toLong).sum
   val (firstUncleanableOffset, cleanableBytes) = LogCleaner.calculateCleanableBytes(log, firstDirtyOffset, uncleanableOffset)
   val totalBytes = cleanBytes + cleanableBytes
diff --git a/core/src/main/scala/kafka/log/LogCleanerManager.scala b/core/src/main/scala/kafka/log/LogCleanerManager.scala
index 9e215d2a88798..bde30b011418a 100755
--- a/core/src/main/scala/kafka/log/LogCleanerManager.scala
+++ b/core/src/main/scala/kafka/log/LogCleanerManager.scala
@@ -58,6 +58,8 @@ private[log] case class LogCleaningPaused(pausedCount: Int) extends LogCleaningS
 private[log] class LogCleanerManager(val logDirs: Seq[File],
                                      val logs: Pool[TopicPartition, Log],
                                      val logDirFailureChannel: LogDirFailureChannel) extends Logging with KafkaMetricsGroup {
+  import LogCleanerManager._
+
 
   protected override def loggerName = classOf[LogCleaner].getName
 
@@ -103,7 +105,7 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
                   val now = Time.SYSTEM.milliseconds
                   partitions.map { tp =>
                     val log = logs.get(tp)
-                    val (firstDirtyOffset, firstUncleanableDirtyOffset) = LogCleanerManager.cleanableOffsets(log, tp, lastClean, now)
+                    val (firstDirtyOffset, firstUncleanableDirtyOffset) = cleanableOffsets(log, tp, lastClean, now)
                     val (_, uncleanableBytes) = LogCleaner.calculateCleanableBytes(log, firstDirtyOffset, firstUncleanableDirtyOffset)
                     uncleanableBytes
                   }.sum
@@ -178,10 +180,8 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
           inProgress.contains(topicPartition) || isUncleanablePartition(log, topicPartition)
       }.map {
         case (topicPartition, log) => // create a LogToClean instance for each
-          val (firstDirtyOffset, firstUncleanableDirtyOffset) =
-            LogCleanerManager.cleanableOffsets(log, topicPartition, lastClean, now)
-
-          val compactionDelayMs = LogCleanerManager.getMaxCompactionDelay(log, firstDirtyOffset, now)
+          val (firstDirtyOffset, firstUncleanableDirtyOffset) = cleanableOffsets(log, topicPartition, lastClean, now)
+          val compactionDelayMs = maxCompactionDelay(log, firstDirtyOffset, now)
           preCleanStats.updateMaxCompactionDelay(compactionDelayMs)
 
           LogToClean(topicPartition, log, firstDirtyOffset, firstUncleanableDirtyOffset, compactionDelayMs > 0)
@@ -487,10 +487,8 @@ private[log] object LogCleanerManager extends Logging {
     * get max delay between the time when log is required to be compacted as determined
     * by maxCompactionLagMs and the current time.
     */
-  def getMaxCompactionDelay(log: Log, firstDirtyOffset: Long, now: Long) : Long = {
-
-    val dirtyNonActiveSegments = log.logSegments(firstDirtyOffset, log.activeSegment.baseOffset)
-
+  def maxCompactionDelay(log: Log, firstDirtyOffset: Long, now: Long) : Long = {
+    val dirtyNonActiveSegments = log.nonActiveLogSegmentsFrom(firstDirtyOffset)
     val firstBatchTimestamps = log.getFirstBatchTimestampForSegments(dirtyNonActiveSegments).filter(_ > 0)
 
     val earliestDirtySegmentTimestamp = {
@@ -523,16 +521,24 @@ private[log] object LogCleanerManager extends Logging {
 
     // If the log segments are abnormally truncated and hence the checkpointed offset is no longer valid;
     // reset to the log starting offset and log the error
-    val logStartOffset = log.logSegments.head.baseOffset
     val firstDirtyOffset = {
-      val offset = lastCleanOffset.getOrElse(logStartOffset)
-      if (offset < logStartOffset) {
-        // don't bother with the warning if compact and delete are enabled.
+      val logStartOffset = log.logStartOffset
+      val checkpointDirtyOffset = lastCleanOffset.getOrElse(logStartOffset)
+
+      if (checkpointDirtyOffset < logStartOffset) {
+        // Don't bother with the warning if compact and delete are enabled.
         if (!isCompactAndDelete(log))
-          warn(s"Resetting first dirty offset of ${log.name} to log start offset $logStartOffset since the checkpointed offset $offset is invalid.")
+          warn(s"Resetting first dirty offset of ${log.name} to log start offset $logStartOffset " +
+            s"since the checkpointed offset $checkpointDirtyOffset is invalid.")
+        logStartOffset
+      } else if (checkpointDirtyOffset > log.logEndOffset) {
+        // The dirty offset has gotten ahead of the log end offset. This could happen if there was data
+        // corruption at the end of the log. We conservatively assume that the full log needs cleaning.
+        warn(s"The last checkpoint dirty offset for partition $topicPartition is $checkpointDirtyOffset, " +
+          s"which is larger than the log end offset ${log.logEndOffset}. Resetting to the log start offset $logStartOffset.")
         logStartOffset
       } else {
-        offset
+        checkpointDirtyOffset
       }
     }
 
@@ -552,7 +558,7 @@ private[log] object LogCleanerManager extends Logging {
       // the first segment whose largest message timestamp is within a minimum time lag from now
       if (minCompactionLagMs > 0) {
         // dirty log segments
-        val dirtyNonActiveSegments = log.logSegments(firstDirtyOffset, log.activeSegment.baseOffset)
+        val dirtyNonActiveSegments = log.nonActiveLogSegmentsFrom(firstDirtyOffset)
         dirtyNonActiveSegments.find { s =>
           val isUncleanable = s.largestTimestamp > now - minCompactionLagMs
           debug(s"Checking if log segment may be cleaned: log='${log.name}' segment.baseOffset=${s.baseOffset} segment.largestTimestamp=${s.largestTimestamp}; now - compactionLag=${now - minCompactionLagMs}; is uncleanable=$isUncleanable")
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
index 3935338192bf7..6a74874be8d94 100644
--- a/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
@@ -61,107 +61,120 @@ class LogCleanerManagerTest extends Logging {
     Utils.delete(tmpDir)
   }
 
+  private def setupIncreasinglyFilthyLogs(partitions: Seq[TopicPartition],
+                                          startNumBatches: Int,
+                                          batchIncrement: Int): Pool[TopicPartition, Log] = {
+    val logs = new Pool[TopicPartition, Log]()
+    var numBatches = startNumBatches
+
+    for (tp <- partitions) {
+      val log = createLog(2048, LogConfig.Compact, topicPartition = tp)
+      logs.put(tp, log)
+
+      writeRecords(log, numBatches = numBatches, recordsPerBatch = 1, batchesPerSegment = 5)
+      numBatches += batchIncrement
+    }
+    logs
+  }
+
   @Test
   def testGrabFilthiestCompactedLogReturnsLogWithDirtiestRatio(): Unit = {
-    val records = TestUtils.singletonRecords("test".getBytes)
-    val log1: Log = createLog(records.sizeInBytes * 5, LogConfig.Compact, 1)
-    val log2: Log = createLog(records.sizeInBytes * 10, LogConfig.Compact, 2)
-    val log3: Log = createLog(records.sizeInBytes * 15, LogConfig.Compact, 3)
+    val tp0 = new TopicPartition("wishing-well", 0)
+    val tp1 = new TopicPartition("wishing-well", 1)
+    val tp2 = new TopicPartition("wishing-well", 2)
+    val partitions = Seq(tp0, tp1, tp2)
 
-    val logs = new Pool[TopicPartition, Log]()
-    val tp1 = new TopicPartition("wishing well", 0) // active segment starts at 0
-    logs.put(tp1, log1)
-    val tp2 = new TopicPartition("wishing well", 1) // active segment starts at 10
-    logs.put(tp2, log2)
-    val tp3 = new TopicPartition("wishing well", 2) // // active segment starts at 20
-    logs.put(tp3, log3)
-    val cleanerManager: LogCleanerManagerMock = createCleanerManager(logs, toMock = true).asInstanceOf[LogCleanerManagerMock]
-    cleanerCheckpoints.put(tp1, 0) // all clean
-    cleanerCheckpoints.put(tp2, 1) // dirtiest - 9 dirty messages
-    cleanerCheckpoints.put(tp3, 15) // 5 dirty messages
+    // setup logs with cleanable range: [20, 20], [20, 25], [20, 30]
+    val logs = setupIncreasinglyFilthyLogs(partitions, startNumBatches = 20, batchIncrement = 5)
+    val cleanerManager = createCleanerManagerMock(logs)
+    partitions.foreach(partition => cleanerCheckpoints.put(partition, 20))
 
     val filthiestLog: LogToClean = cleanerManager.grabFilthiestCompactedLog(time).get
-
-    assertEquals(log2, filthiestLog.log)
     assertEquals(tp2, filthiestLog.topicPartition)
+    assertEquals(tp2, filthiestLog.log.topicPartition)
   }
 
   @Test
   def testGrabFilthiestCompactedLogIgnoresUncleanablePartitions(): Unit = {
-    val records = TestUtils.singletonRecords("test".getBytes)
-    val log1: Log = createLog(records.sizeInBytes * 5, LogConfig.Compact, 1)
-    val log2: Log = createLog(records.sizeInBytes * 10, LogConfig.Compact, 2)
-    val log3: Log = createLog(records.sizeInBytes * 15, LogConfig.Compact, 3)
+    val tp0 = new TopicPartition("wishing-well", 0)
+    val tp1 = new TopicPartition("wishing-well", 1)
+    val tp2 = new TopicPartition("wishing-well", 2)
+    val partitions = Seq(tp0, tp1, tp2)
 
-    val logs = new Pool[TopicPartition, Log]()
-    val tp1 = new TopicPartition("wishing well", 0) // active segment starts at 0
-    logs.put(tp1, log1)
-    val tp2 = new TopicPartition("wishing well", 1) // active segment starts at 10
-    logs.put(tp2, log2)
-    val tp3 = new TopicPartition("wishing well", 2) // // active segment starts at 20
-    logs.put(tp3, log3)
-    val cleanerManager: LogCleanerManagerMock = createCleanerManager(logs, toMock = true).asInstanceOf[LogCleanerManagerMock]
-    cleanerCheckpoints.put(tp1, 0) // all clean
-    cleanerCheckpoints.put(tp2, 1) // dirtiest - 9 dirty messages
-    cleanerCheckpoints.put(tp3, 15) // 5 dirty messages
-    cleanerManager.markPartitionUncleanable(log2.dir.getParent, tp2)
+    // setup logs with cleanable range: [20, 20], [20, 25], [20, 30]
+    val logs = setupIncreasinglyFilthyLogs(partitions, startNumBatches = 20, batchIncrement = 5)
+    val cleanerManager = createCleanerManagerMock(logs)
+    partitions.foreach(partition => cleanerCheckpoints.put(partition, 20))
 
-    val filthiestLog: LogToClean = cleanerManager.grabFilthiestCompactedLog(time).get
+    cleanerManager.markPartitionUncleanable(logs.get(tp2).dir.getParent, tp2)
 
-    assertEquals(log3, filthiestLog.log)
-    assertEquals(tp3, filthiestLog.topicPartition)
+    val filthiestLog: LogToClean = cleanerManager.grabFilthiestCompactedLog(time).get
+    assertEquals(tp1, filthiestLog.topicPartition)
+    assertEquals(tp1, filthiestLog.log.topicPartition)
   }
 
   @Test
   def testGrabFilthiestCompactedLogIgnoresInProgressPartitions(): Unit = {
-    val records = TestUtils.singletonRecords("test".getBytes)
-    val log1: Log = createLog(records.sizeInBytes * 5, LogConfig.Compact, 1)
-    val log2: Log = createLog(records.sizeInBytes * 10, LogConfig.Compact, 2)
-    val log3: Log = createLog(records.sizeInBytes * 15, LogConfig.Compact, 3)
+    val tp0 = new TopicPartition("wishing-well", 0)
+    val tp1 = new TopicPartition("wishing-well", 1)
+    val tp2 = new TopicPartition("wishing-well", 2)
+    val partitions = Seq(tp0, tp1, tp2)
+
+    // setup logs with cleanable range: [20, 20], [20, 25], [20, 30]
+    val logs = setupIncreasinglyFilthyLogs(partitions, startNumBatches = 20, batchIncrement = 5)
+    val cleanerManager = createCleanerManagerMock(logs)
+    partitions.foreach(partition => cleanerCheckpoints.put(partition, 20))
 
-    val logs = new Pool[TopicPartition, Log]()
-    val tp1 = new TopicPartition("wishing well", 0) // active segment starts at 0
-    logs.put(tp1, log1)
-    val tp2 = new TopicPartition("wishing well", 1) // active segment starts at 10
-    logs.put(tp2, log2)
-    val tp3 = new TopicPartition("wishing well", 2) // // active segment starts at 20
-    logs.put(tp3, log3)
-    val cleanerManager: LogCleanerManagerMock = createCleanerManager(logs, toMock = true).asInstanceOf[LogCleanerManagerMock]
-    cleanerCheckpoints.put(tp1, 0) // all clean
-    cleanerCheckpoints.put(tp2, 1) // dirtiest - 9 dirty messages
-    cleanerCheckpoints.put(tp3, 15) // 5 dirty messages
     cleanerManager.setCleaningState(tp2, LogCleaningInProgress)
 
     val filthiestLog: LogToClean = cleanerManager.grabFilthiestCompactedLog(time).get
 
-    assertEquals(log3, filthiestLog.log)
-    assertEquals(tp3, filthiestLog.topicPartition)
+    assertEquals(tp1, filthiestLog.topicPartition)
+    assertEquals(tp1, filthiestLog.log.topicPartition)
   }
 
   @Test
   def testGrabFilthiestCompactedLogIgnoresBothInProgressPartitionsAndUncleanablePartitions(): Unit = {
-    val records = TestUtils.singletonRecords("test".getBytes)
-    val log1: Log = createLog(records.sizeInBytes * 5, LogConfig.Compact, 1)
-    val log2: Log = createLog(records.sizeInBytes * 10, LogConfig.Compact, 2)
-    val log3: Log = createLog(records.sizeInBytes * 15, LogConfig.Compact, 3)
+    val tp0 = new TopicPartition("wishing-well", 0)
+    val tp1 = new TopicPartition("wishing-well", 1)
+    val tp2 = new TopicPartition("wishing-well", 2)
+    val partitions = Seq(tp0, tp1, tp2)
+
+    // setup logs with cleanable range: [20, 20], [20, 25], [20, 30]
+    val logs = setupIncreasinglyFilthyLogs(partitions, startNumBatches = 20, batchIncrement = 5)
+    val cleanerManager = createCleanerManagerMock(logs)
+    partitions.foreach(partition => cleanerCheckpoints.put(partition, 20))
 
-    val logs = new Pool[TopicPartition, Log]()
-    val tp1 = new TopicPartition("wishing well", 0) // active segment starts at 0
-    logs.put(tp1, log1)
-    val tp2 = new TopicPartition("wishing well", 1) // active segment starts at 10
-    logs.put(tp2, log2)
-    val tp3 = new TopicPartition("wishing well", 2) // // active segment starts at 20
-    logs.put(tp3, log3)
-    val cleanerManager: LogCleanerManagerMock = createCleanerManager(logs, toMock = true).asInstanceOf[LogCleanerManagerMock]
-    cleanerCheckpoints.put(tp1, 0) // all clean
-    cleanerCheckpoints.put(tp2, 1) // dirtiest - 9 dirty messages
-    cleanerCheckpoints.put(tp3, 15) // 5 dirty messages
     cleanerManager.setCleaningState(tp2, LogCleaningInProgress)
-    cleanerManager.markPartitionUncleanable(log3.dir.getParent, tp3)
+    cleanerManager.markPartitionUncleanable(logs.get(tp1).dir.getParent, tp1)
 
     val filthiestLog: Option[LogToClean] = cleanerManager.grabFilthiestCompactedLog(time)
+    assertEquals(None, filthiestLog)
+  }
+
+  @Test
+  def testDirtyOffsetResetIfLargerThanEndOffset(): Unit = {
+    val tp = new TopicPartition("foo", 0)
+    val logs = setupIncreasinglyFilthyLogs(Seq(tp), startNumBatches = 20, batchIncrement = 5)
+    val cleanerManager = createCleanerManagerMock(logs)
+    cleanerCheckpoints.put(tp, 200)
+
+    val filthiestLog = cleanerManager.grabFilthiestCompactedLog(time).get
+    assertEquals(0L, filthiestLog.firstDirtyOffset)
+  }
+
+  @Test
+  def testDirtyOffsetResetIfSmallerThanStartOffset(): Unit = {
+    val tp = new TopicPartition("foo", 0)
+    val logs = setupIncreasinglyFilthyLogs(Seq(tp), startNumBatches = 20, batchIncrement = 5)
+
+    logs.get(tp).maybeIncrementLogStartOffset(10L)
 
-    assertTrue(filthiestLog.isEmpty)
+    val cleanerManager = createCleanerManagerMock(logs)
+    cleanerCheckpoints.put(tp, 0L)
+
+    val filthiestLog = cleanerManager.grabFilthiestCompactedLog(time).get
+    assertEquals(10L, filthiestLog.firstDirtyOffset)
   }
 
   /**
@@ -482,17 +495,16 @@ class LogCleanerManagerTest extends Logging {
   private def createCleanerManager(log: Log): LogCleanerManager = {
     val logs = new Pool[TopicPartition, Log]()
     logs.put(topicPartition, log)
-    createCleanerManager(logs)
+    new LogCleanerManager(Array(logDir), logs, null)
   }
 
-  private def createCleanerManager(pool: Pool[TopicPartition, Log], toMock: Boolean = false): LogCleanerManager = {
-    if (toMock)
-      new LogCleanerManagerMock(Array(logDir), pool, null)
-    else
-      new LogCleanerManager(Array(logDir), pool, null)
+  private def createCleanerManagerMock(pool: Pool[TopicPartition, Log]): LogCleanerManagerMock = {
+    new LogCleanerManagerMock(Array(logDir), pool, null)
   }
 
-  private def createLog(segmentSize: Int, cleanupPolicy: String, segmentsCount: Int = 0): Log = {
+  private def createLog(segmentSize: Int,
+                        cleanupPolicy: String,
+                        topicPartition: TopicPartition = new TopicPartition("log", 0)): Log = {
     val logProps = new Properties()
     logProps.put(LogConfig.SegmentBytesProp, segmentSize: Integer)
     logProps.put(LogConfig.RetentionMsProp, 1: Integer)
@@ -500,8 +512,9 @@ class LogCleanerManagerTest extends Logging {
     logProps.put(LogConfig.MinCleanableDirtyRatioProp, 0.05: java.lang.Double) // small for easier and clearer tests
 
     val config = LogConfig(logProps)
-    val partitionDir = new File(logDir, "log-0")
-    val log = Log(partitionDir,
+    val partitionDir = new File(logDir, Log.logDirName(topicPartition))
+
+    Log(partitionDir,
       config,
       logStartOffset = 0L,
       recoveryPoint = 0L,
@@ -511,10 +524,15 @@ class LogCleanerManagerTest extends Logging {
       maxProducerIdExpirationMs = 60 * 60 * 1000,
       producerIdExpirationCheckIntervalMs = LogManager.ProducerIdExpirationCheckIntervalMs,
       logDirFailureChannel = new LogDirFailureChannel(10))
-    for (i <- 0 until segmentsCount) {
-      val startOffset = i * 10
-      val endOffset = startOffset + 10
-      val segment = LogUtils.createSegment(startOffset, logDir)
+  }
+
+  private def writeRecords(log: Log,
+                           numBatches: Int,
+                           recordsPerBatch: Int,
+                           batchesPerSegment: Int): Unit = {
+    for (i <- 0 until numBatches) {
+      val startOffset = i * recordsPerBatch
+      val endOffset = startOffset + recordsPerBatch
       var lastTimestamp = 0L
       val records = (startOffset until endOffset).map { offset =>
         val currentTimestamp = time.milliseconds()
@@ -524,10 +542,13 @@ class LogCleanerManagerTest extends Logging {
         new SimpleRecord(currentTimestamp, s"key-$offset".getBytes, s"value-$offset".getBytes)
       }
 
-      segment.append(endOffset, lastTimestamp, endOffset, MemoryRecords.withRecords(CompressionType.NONE, records:_*))
-      log.addSegment(segment)
+      log.appendAsLeader(MemoryRecords.withRecords(CompressionType.NONE, records:_*), leaderEpoch = 1)
+      log.maybeIncrementHighWatermark(log.logEndOffsetMetadata)
+
+      if (i % batchesPerSegment == 0)
+        log.roll()
     }
-    log
+    log.roll()
   }
 
   private def makeLog(dir: File = logDir, config: LogConfig) =
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala
index caa985d18bde5..bb30287bb12bd 100755
--- a/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala
@@ -347,7 +347,7 @@ class LogCleanerTest {
     log.roll()
 
     // cannot remove the marker in this pass because there are still valid records
-    var dirtyOffset = cleaner.doClean(LogToClean(tp, log, 0L, 100L), deleteHorizonMs = Long.MaxValue)._1
+    var dirtyOffset = cleaner.doClean(LogToClean(tp, log, 0L, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)._1
     assertEquals(List(1, 3, 2), LogTest.keysInLog(log))
     assertEquals(List(0, 2, 3, 4, 5), offsetsInLog(log))
 
@@ -356,17 +356,17 @@ class LogCleanerTest {
     log.roll()
 
     // the first cleaning preserves the commit marker (at offset 3) since there were still records for the transaction
-    dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1
+    dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)._1
     assertEquals(List(2, 1, 3), LogTest.keysInLog(log))
     assertEquals(List(3, 4, 5, 6, 7, 8), offsetsInLog(log))
 
     // delete horizon forced to 0 to verify marker is not removed early
-    dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = 0L)._1
+    dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, log.activeSegment.baseOffset), deleteHorizonMs = 0L)._1
     assertEquals(List(2, 1, 3), LogTest.keysInLog(log))
     assertEquals(List(3, 4, 5, 6, 7, 8), offsetsInLog(log))
 
     // clean again with large delete horizon and verify the marker is removed
-    dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1
+    dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)._1
     assertEquals(List(2, 1, 3), LogTest.keysInLog(log))
     assertEquals(List(4, 5, 6, 7, 8), offsetsInLog(log))
   }
@@ -395,11 +395,11 @@ class LogCleanerTest {
     log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, isFromClient = false)
     log.roll()
 
-    cleaner.doClean(LogToClean(tp, log, 0L, 100L), deleteHorizonMs = Long.MaxValue)
+    cleaner.doClean(LogToClean(tp, log, 0L, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)
     assertEquals(List(2), LogTest.keysInLog(log))
     assertEquals(List(1, 3, 4), offsetsInLog(log))
 
-    cleaner.doClean(LogToClean(tp, log, 0L, 100L), deleteHorizonMs = Long.MaxValue)
+    cleaner.doClean(LogToClean(tp, log, 0L, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)
     assertEquals(List(2), LogTest.keysInLog(log))
     assertEquals(List(3, 4), offsetsInLog(log))
   }
@@ -434,14 +434,14 @@ class LogCleanerTest {
 
     // first time through the records are removed
     // Expected State: [{Producer1: EmptyBatch}, {Producer2: EmptyBatch}, {Producer2: Commit}, {2}, {3}, {Producer1: Commit}]
-    var dirtyOffset = cleaner.doClean(LogToClean(tp, log, 0L, 100L), deleteHorizonMs = Long.MaxValue)._1
+    var dirtyOffset = cleaner.doClean(LogToClean(tp, log, 0L, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)._1
     assertEquals(List(2, 3), LogTest.keysInLog(log))
     assertEquals(List(4, 5, 6, 7), offsetsInLog(log))
     assertEquals(List(1, 3, 4, 5, 6, 7), lastOffsetsPerBatchInLog(log))
 
     // the empty batch remains if cleaned again because it still holds the last sequence
     // Expected State: [{Producer1: EmptyBatch}, {Producer2: EmptyBatch}, {Producer2: Commit}, {2}, {3}, {Producer1: Commit}]
-    dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1
+    dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)._1
     assertEquals(List(2, 3), LogTest.keysInLog(log))
     assertEquals(List(4, 5, 6, 7), offsetsInLog(log))
     assertEquals(List(1, 3, 4, 5, 6, 7), lastOffsetsPerBatchInLog(log))
@@ -454,13 +454,13 @@ class LogCleanerTest {
     log.roll()
 
     // Expected State: [{Producer1: EmptyBatch}, {Producer2: Commit}, {2}, {3}, {Producer1: Commit}, {Producer2: 1}, {Producer2: Commit}]
-    dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1
+    dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)._1
     assertEquals(List(2, 3, 1), LogTest.keysInLog(log))
     assertEquals(List(4, 5, 6, 7, 8, 9), offsetsInLog(log))
     assertEquals(List(1, 4, 5, 6, 7, 8, 9), lastOffsetsPerBatchInLog(log))
 
     // Expected State: [{Producer1: EmptyBatch}, {2}, {3}, {Producer1: Commit}, {Producer2: 1}, {Producer2: Commit}]
-    dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1
+    dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)._1
     assertEquals(List(2, 3, 1), LogTest.keysInLog(log))
     assertEquals(List(5, 6, 7, 8, 9), offsetsInLog(log))
     assertEquals(List(1, 5, 6, 7, 8, 9), lastOffsetsPerBatchInLog(log))
@@ -484,14 +484,14 @@ class LogCleanerTest {
 
     // first time through the control batch is retained as an empty batch
     // Expected State: [{Producer1: EmptyBatch}], [{2}, {3}]
-    var dirtyOffset = cleaner.doClean(LogToClean(tp, log, 0L, 100L), deleteHorizonMs = Long.MaxValue)._1
+    var dirtyOffset = cleaner.doClean(LogToClean(tp, log, 0L, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)._1
     assertEquals(List(2, 3), LogTest.keysInLog(log))
     assertEquals(List(1, 2), offsetsInLog(log))
     assertEquals(List(0, 1, 2), lastOffsetsPerBatchInLog(log))
 
     // the empty control batch does not cause an exception when cleaned
     // Expected State: [{Producer1: EmptyBatch}], [{2}, {3}]
-    dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1
+    dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)._1
     assertEquals(List(2, 3), LogTest.keysInLog(log))
     assertEquals(List(1, 2), offsetsInLog(log))
     assertEquals(List(0, 1, 2), lastOffsetsPerBatchInLog(log))
@@ -515,7 +515,7 @@ class LogCleanerTest {
     log.roll()
 
     // Both the record and the marker should remain after cleaning
-    cleaner.doClean(LogToClean(tp, log, 0L, 100L), deleteHorizonMs = Long.MaxValue)
+    cleaner.doClean(LogToClean(tp, log, 0L, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)
     assertEquals(List(0, 1), offsetsInLog(log))
     assertEquals(List(0, 1), lastOffsetsPerBatchInLog(log))
   }
@@ -540,12 +540,12 @@ class LogCleanerTest {
     // Both the batch and the marker should remain after cleaning. The batch is retained
     // because it is the last entry for this producerId. The marker is retained because
     // there are still batches remaining from this transaction.
-    cleaner.doClean(LogToClean(tp, log, 0L, 100L), deleteHorizonMs = Long.MaxValue)
+    cleaner.doClean(LogToClean(tp, log, 0L, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)
     assertEquals(List(1), offsetsInLog(log))
     assertEquals(List(0, 1), lastOffsetsPerBatchInLog(log))
 
     // The empty batch and the marker is still retained after a second cleaning.
-    cleaner.doClean(LogToClean(tp, log, 0L, 100L), deleteHorizonMs = Long.MaxValue)
+    cleaner.doClean(LogToClean(tp, log, 0L, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)
     assertEquals(List(1), offsetsInLog(log))
     assertEquals(List(0, 1), lastOffsetsPerBatchInLog(log))
   }
@@ -570,12 +570,12 @@ class LogCleanerTest {
     log.roll()
 
     // delete horizon set to 0 to verify marker is not removed early
-    val dirtyOffset = cleaner.doClean(LogToClean(tp, log, 0L, 100L), deleteHorizonMs = 0L)._1
+    val dirtyOffset = cleaner.doClean(LogToClean(tp, log, 0L, log.activeSegment.baseOffset), deleteHorizonMs = 0L)._1
     assertEquals(List(3), LogTest.keysInLog(log))
     assertEquals(List(3, 4, 5), offsetsInLog(log))
 
     // clean again with large delete horizon and verify the marker is removed
-    cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)
+    cleaner.doClean(LogToClean(tp, log, dirtyOffset, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)
     assertEquals(List(3), LogTest.keysInLog(log))
     assertEquals(List(4, 5), offsetsInLog(log))
   }
@@ -609,12 +609,12 @@ class LogCleanerTest {
 
     // Both transactional batches will be cleaned. The last one will remain in the log
     // as an empty batch in order to preserve the producer sequence number and epoch
-    cleaner.doClean(LogToClean(tp, log, 0L, 100L), deleteHorizonMs = Long.MaxValue)
+    cleaner.doClean(LogToClean(tp, log, 0L, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)
     assertEquals(List(1, 3, 4, 5), offsetsInLog(log))
     assertEquals(List(1, 2, 3, 4, 5), lastOffsetsPerBatchInLog(log))
 
     // On the second round of cleaning, the marker from the first transaction should be removed.
-    cleaner.doClean(LogToClean(tp, log, 0L, 100L), deleteHorizonMs = Long.MaxValue)
+    cleaner.doClean(LogToClean(tp, log, 0L, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)
     assertEquals(List(3, 4, 5), offsetsInLog(log))
     assertEquals(List(2, 3, 4, 5), lastOffsetsPerBatchInLog(log))
   }
@@ -646,14 +646,14 @@ class LogCleanerTest {
     assertAbortedTransactionIndexed()
 
     // first time through the records are removed
-    var dirtyOffset = cleaner.doClean(LogToClean(tp, log, 0L, 100L), deleteHorizonMs = Long.MaxValue)._1
+    var dirtyOffset = cleaner.doClean(LogToClean(tp, log, 0L, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)._1
     assertAbortedTransactionIndexed()
     assertEquals(List(), LogTest.keysInLog(log))
     assertEquals(List(2), offsetsInLog(log)) // abort marker is retained
     assertEquals(List(1, 2), lastOffsetsPerBatchInLog(log)) // empty batch is retained
 
     // the empty batch remains if cleaned again because it still holds the last sequence
-    dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1
+    dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)._1
     assertAbortedTransactionIndexed()
     assertEquals(List(), LogTest.keysInLog(log))
     assertEquals(List(2), offsetsInLog(log)) // abort marker is still retained
@@ -663,13 +663,13 @@ class LogCleanerTest {
     appendProducer(Seq(1))
     log.roll()
 
-    dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1
+    dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)._1
     assertAbortedTransactionIndexed()
     assertEquals(List(1), LogTest.keysInLog(log))
     assertEquals(List(2, 3), offsetsInLog(log)) // abort marker is not yet gone because we read the empty batch
     assertEquals(List(2, 3), lastOffsetsPerBatchInLog(log)) // but we do not preserve the empty batch
 
-    dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1
+    dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)._1
     assertEquals(List(1), LogTest.keysInLog(log))
     assertEquals(List(3), offsetsInLog(log)) // abort marker is gone
     assertEquals(List(3), lastOffsetsPerBatchInLog(log))
diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala
index 0e3634e55cfda..2234366575c42 100755
--- a/core/src/test/scala/unit/kafka/log/LogTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogTest.scala
@@ -516,6 +516,43 @@ class LogTest {
     assertEquals(startOffset, log.logEndOffset)
   }
 
+  @Test
+  def testNonActiveSegmentsFrom(): Unit = {
+    val logConfig = LogTest.createLogConfig()
+    val log = createLog(logDir, logConfig)
+
+    for (i <- 0 until 5) {
+      val record = new SimpleRecord(mockTime.milliseconds, i.toString.getBytes)
+      log.appendAsLeader(TestUtils.records(List(record)), leaderEpoch = 0)
+      log.roll()
+    }
+
+    def nonActiveBaseOffsetsFrom(startOffset: Long): Seq[Long] = {
+      log.nonActiveLogSegmentsFrom(startOffset).map(_.baseOffset).toSeq
+    }
+
+    assertEquals(5L, log.activeSegment.baseOffset)
+    assertEquals(0 until 5, nonActiveBaseOffsetsFrom(0L))
+    assertEquals(Seq.empty, nonActiveBaseOffsetsFrom(5L))
+    assertEquals(2 until 5, nonActiveBaseOffsetsFrom(2L))
+  }
+
+  @Test
+  def testInconsistentLogSegmentRange(): Unit = {
+    val logConfig = LogTest.createLogConfig()
+    val log = createLog(logDir, logConfig)
+
+    for (i <- 0 until 5) {
+      val record = new SimpleRecord(mockTime.milliseconds, i.toString.getBytes)
+      log.appendAsLeader(TestUtils.records(List(record)), leaderEpoch = 0)
+      log.roll()
+    }
+
+    assertThrows[IllegalArgumentException] {
+      log.logSegments(5, 1)
+    }
+  }
+
   @Test
   def testLogDelete(): Unit = {
     val logConfig = LogTest.createLogConfig()

From ffef0871c2d64bcbc171b129c2057b572c2f41b2 Mon Sep 17 00:00:00 2001
From: vinoth chandar 
Date: Thu, 5 Sep 2019 13:50:55 -0700
Subject: [PATCH 0590/1071] KAFKA-7149 : Reducing streams assignment data size 
 (#7185)

* Leader instance uses dictionary encoding on the wire to send topic partitions
* Topic names (most expensive component) are mapped to an integer using the dictionary
* Follower instances receive the dictionary, decode topic names back
* Purely an on-the-wire optimization, no in-memory structures changed
* Test case added for version 5 AssignmentInfo

Reviewers: Guozhang Wang 
---
 .../internals/StreamsPartitionAssignor.java   |  45 ++++---
 .../internals/assignment/AssignmentInfo.java  | 118 +++++++++++++++---
 .../assignment/SubscriptionInfo.java          |  34 ++---
 .../assignment/AssignmentInfoTest.java        |   7 ++
 .../streams/tests/StreamsUpgradeTest.java     |   2 +-
 .../tests/streams/streams_upgrade_test.py     |  14 +--
 6 files changed, 163 insertions(+), 57 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
index 36196a68ddba1..d2f2b3a8e10d9 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
@@ -65,6 +65,7 @@ public class StreamsPartitionAssignor implements ConsumerPartitionAssignor, Conf
     private final static int VERSION_TWO = 2;
     private final static int VERSION_THREE = 3;
     private final static int VERSION_FOUR = 4;
+    private final static int VERSION_FIVE = 5;
     private final static int EARLIEST_PROBEABLE_VERSION = VERSION_THREE;
     protected final Set supportedVersions = new HashSet<>();
 
@@ -446,7 +447,7 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
             for (final String topic : topicsInfo.sourceTopics) {
                 if (!topicsInfo.repartitionSourceTopics.keySet().contains(topic) &&
                     !metadata.topics().contains(topic)) {
-                    log.error("Missing source topic {} durign assignment. Returning error {}.",
+                    log.error("Missing source topic {} during assignment. Returning error {}.",
                               topic, Error.INCOMPLETE_SOURCE_TOPIC_METADATA.name());
                     return new GroupAssignment(errorAssignment(clientMetadataMap, topic, Error.INCOMPLETE_SOURCE_TOPIC_METADATA.code));
                 }
@@ -625,6 +626,7 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
             for (final Map.Entry entry : clientMetadataMap.entrySet()) {
                 final HostInfo hostInfo = entry.getValue().hostInfo;
 
+                // if application server is configured, also include host state map
                 if (hostInfo != null) {
                     final Set topicPartitions = new HashSet<>();
                     final ClientState state = entry.getValue().state;
@@ -775,6 +777,17 @@ List> interleaveTasksByGroupId(final Collection taskIds, fi
         return taskIdsForConsumerAssignment;
     }
 
+    private void upgradeSubscriptionVersionIfNeeded(final int leaderSupportedVersion) {
+        if (leaderSupportedVersion > usedSubscriptionMetadataVersion) {
+            log.info("Sent a version {} subscription and group leader's latest supported version is {}. " +
+                    "Upgrading subscription metadata version to {} for next rebalance.",
+                usedSubscriptionMetadataVersion,
+                leaderSupportedVersion,
+                leaderSupportedVersion);
+            usedSubscriptionMetadataVersion = leaderSupportedVersion;
+        }
+    }
+
     /**
      * @throws TaskAssignmentException if there is no task id for one of the partitions specified
      */
@@ -835,29 +848,20 @@ public void onAssignment(final Assignment assignment, final ConsumerGroupMetadat
                 partitionsByHost = info.partitionsByHost();
                 break;
             case VERSION_THREE:
-                if (leaderSupportedVersion > usedSubscriptionMetadataVersion) {
-                    log.info("Sent a version {} subscription and group leader's latest supported version is {}. " +
-                        "Upgrading subscription metadata version to {} for next rebalance.",
-                        usedSubscriptionMetadataVersion,
-                        leaderSupportedVersion,
-                        leaderSupportedVersion);
-                    usedSubscriptionMetadataVersion = leaderSupportedVersion;
-                }
+                upgradeSubscriptionVersionIfNeeded(leaderSupportedVersion);
                 processVersionThreeAssignment(info, partitions, activeTasks, topicToPartitionInfo);
                 partitionsByHost = info.partitionsByHost();
                 break;
             case VERSION_FOUR:
-                if (leaderSupportedVersion > usedSubscriptionMetadataVersion) {
-                    log.info("Sent a version {} subscription and group leader's latest supported version is {}. " +
-                        "Upgrading subscription metadata version to {} for next rebalance.",
-                        usedSubscriptionMetadataVersion,
-                        leaderSupportedVersion,
-                        leaderSupportedVersion);
-                    usedSubscriptionMetadataVersion = leaderSupportedVersion;
-                }
+                upgradeSubscriptionVersionIfNeeded(leaderSupportedVersion);
                 processVersionFourAssignment(info, partitions, activeTasks, topicToPartitionInfo);
                 partitionsByHost = info.partitionsByHost();
                 break;
+            case VERSION_FIVE:
+                upgradeSubscriptionVersionIfNeeded(leaderSupportedVersion);
+                processVersionFiveAssignment(info, partitions, activeTasks, topicToPartitionInfo);
+                partitionsByHost = info.partitionsByHost();
+                break;
             default:
                 throw new IllegalStateException("This code should never be reached. Please file a bug report at https://issues.apache.org/jira/projects/KAFKA/");
         }
@@ -918,6 +922,13 @@ private void processVersionFourAssignment(final AssignmentInfo info,
         processVersionThreeAssignment(info, partitions, activeTasks, topicToPartitionInfo);
     }
 
+    private void processVersionFiveAssignment(final AssignmentInfo info,
+                                              final List partitions,
+                                              final Map> activeTasks,
+                                              final Map topicToPartitionInfo) {
+        processVersionFourAssignment(info, partitions, activeTasks, topicToPartitionInfo);
+    }
+
     // for testing
     protected void processLatestVersionAssignment(final AssignmentInfo info,
                                                   final List partitions,
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfo.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfo.java
index 8ad4036f63355..4b2dae2608898 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfo.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfo.java
@@ -41,7 +41,7 @@ public class AssignmentInfo {
 
     private static final Logger log = LoggerFactory.getLogger(AssignmentInfo.class);
 
-    public static final int LATEST_SUPPORTED_VERSION = 4;
+    public static final int LATEST_SUPPORTED_VERSION = 5;
     static final int UNKNOWN = -1;
 
     private final int usedVersion;
@@ -59,12 +59,14 @@ private AssignmentInfo(final int version,
         this.errCode = 0;
     }
 
+    // for testing only
     public AssignmentInfo(final List activeTasks,
                           final Map> standbyTasks,
-                          final Map> hostState) {
-        this(LATEST_SUPPORTED_VERSION, activeTasks, standbyTasks, hostState, 0);
+                          final Map> partitionsByHost) {
+        this(LATEST_SUPPORTED_VERSION, activeTasks, standbyTasks, partitionsByHost, 0);
     }
 
+    // creates an empty assignment
     public AssignmentInfo() {
         this(LATEST_SUPPORTED_VERSION,
             Collections.emptyList(),
@@ -76,9 +78,9 @@ public AssignmentInfo() {
     public AssignmentInfo(final int version,
                           final List activeTasks,
                           final Map> standbyTasks,
-                          final Map> hostState,
+                          final Map> partitionsByHost,
                           final int errCode) {
-        this(version, LATEST_SUPPORTED_VERSION, activeTasks, standbyTasks, hostState, errCode);
+        this(version, LATEST_SUPPORTED_VERSION, activeTasks, standbyTasks, partitionsByHost, errCode);
         if (version < 1 || version > LATEST_SUPPORTED_VERSION) {
             throw new IllegalArgumentException("version must be between 1 and " + LATEST_SUPPORTED_VERSION
                 + "; was: " + version);
@@ -90,13 +92,13 @@ public AssignmentInfo(final int version,
                    final int latestSupportedVersion,
                    final List activeTasks,
                    final Map> standbyTasks,
-                   final Map> hostState,
+                   final Map> partitionsByHost,
                    final int errCode) {
         this.usedVersion = version;
         this.latestSupportedVersion = latestSupportedVersion;
         this.activeTasks = activeTasks;
         this.standbyTasks = standbyTasks;
-        this.partitionsByHost = hostState;
+        this.partitionsByHost = partitionsByHost;
         this.errCode = errCode;
     }
 
@@ -145,6 +147,9 @@ public ByteBuffer encode() {
                 case 4:
                     encodeVersionFour(out);
                     break;
+                case 5:
+                    encodeVersionFive(out);
+                    break;
                 default:
                     throw new IllegalStateException("Unknown metadata version: " + usedVersion
                         + "; latest supported version: " + LATEST_SUPPORTED_VERSION);
@@ -192,13 +197,50 @@ private void encodePartitionsByHost(final DataOutputStream out) throws IOExcepti
         // encode partitions by host
         out.writeInt(partitionsByHost.size());
         for (final Map.Entry> entry : partitionsByHost.entrySet()) {
-            final HostInfo hostInfo = entry.getKey();
-            out.writeUTF(hostInfo.host());
-            out.writeInt(hostInfo.port());
+            writeHostInfo(out, entry.getKey());
             writeTopicPartitions(out, entry.getValue());
         }
     }
 
+    private void encodePartitionsByHostAsDictionary(final DataOutputStream out) throws IOException {
+
+        // Build a dictionary to encode topicNames
+        int topicIndex = 0;
+        final Map topicNameDict = new HashMap<>();
+        for (final Map.Entry> entry : partitionsByHost.entrySet()) {
+            for (final TopicPartition topicPartition : entry.getValue()) {
+                if (!topicNameDict.containsKey(topicPartition.topic())) {
+                    topicNameDict.put(topicPartition.topic(), topicIndex++);
+                }
+            }
+        }
+
+        // write the topic name dictionary out
+        out.writeInt(topicNameDict.size());
+        for (final Map.Entry entry : topicNameDict.entrySet()) {
+            out.writeInt(entry.getValue());
+            out.writeUTF(entry.getKey());
+        }
+
+        // encode partitions by host
+        out.writeInt(partitionsByHost.size());
+
+        // Write the topic index, partition
+        for (final Map.Entry> entry : partitionsByHost.entrySet()) {
+            writeHostInfo(out, entry.getKey());
+            out.writeInt(entry.getValue().size());
+            for (final TopicPartition partition : entry.getValue()) {
+                out.writeInt(topicNameDict.get(partition.topic()));
+                out.writeInt(partition.partition());
+            }
+        }
+    }
+
+    private void writeHostInfo(final DataOutputStream out, final HostInfo hostInfo) throws IOException {
+        out.writeUTF(hostInfo.host());
+        out.writeInt(hostInfo.port());
+    }
+
     private void writeTopicPartitions(final DataOutputStream out,
                                       final Set partitions) throws IOException {
         out.writeInt(partitions.size());
@@ -223,6 +265,14 @@ private void encodeVersionFour(final DataOutputStream out) throws IOException {
         out.writeInt(errCode);
     }
 
+    private void encodeVersionFive(final DataOutputStream out) throws IOException {
+        out.writeInt(5);
+        out.writeInt(LATEST_SUPPORTED_VERSION);
+        encodeActiveAndStandbyTaskAssignment(out);
+        encodePartitionsByHostAsDictionary(out);
+        out.writeInt(errCode);
+    }
+
     /**
      * @throws TaskAssignmentException if method fails to decode the data or if the data version is unknown
      */
@@ -254,6 +304,11 @@ public static AssignmentInfo decode(final ByteBuffer data) {
                     assignmentInfo = new AssignmentInfo(usedVersion, latestSupportedVersion);
                     decodeVersionFourData(assignmentInfo, in);
                     break;
+                case 5:
+                    latestSupportedVersion = in.readInt();
+                    assignmentInfo = new AssignmentInfo(usedVersion, latestSupportedVersion);
+                    decodeVersionFiveData(assignmentInfo, in);
+                    break;
                 default:
                     final TaskAssignmentException fatalException = new TaskAssignmentException("Unable to decode assignment data: " +
                         "used version: " + usedVersion + "; latest supported version: " + LATEST_SUPPORTED_VERSION);
@@ -297,10 +352,10 @@ private static void decodeVersionTwoData(final AssignmentInfo assignmentInfo,
                                              final DataInputStream in) throws IOException {
         decodeActiveTasks(assignmentInfo, in);
         decodeStandbyTasks(assignmentInfo, in);
-        decodeGlobalAssignmentData(assignmentInfo, in);
+        decodePartitionsByHost(assignmentInfo, in);
     }
 
-    private static void decodeGlobalAssignmentData(final AssignmentInfo assignmentInfo,
+    private static void decodePartitionsByHost(final AssignmentInfo assignmentInfo,
                                                    final DataInputStream in) throws IOException {
         assignmentInfo.partitionsByHost = new HashMap<>();
         final int numEntries = in.readInt();
@@ -319,11 +374,37 @@ private static Set readTopicPartitions(final DataInputStream in)
         return partitions;
     }
 
+    private static void decodePartitionsByHostUsingDictionary(final AssignmentInfo assignmentInfo,
+        final DataInputStream in) throws IOException {
+        assignmentInfo.partitionsByHost = new HashMap<>();
+        final int dictSize = in.readInt();
+        final Map topicIndexDict = new HashMap<>(dictSize);
+        for (int i = 0; i < dictSize; i++) {
+            topicIndexDict.put(in.readInt(), in.readUTF());
+        }
+
+        final int numEntries = in.readInt();
+        for (int i = 0; i < numEntries; i++) {
+            final HostInfo hostInfo = new HostInfo(in.readUTF(), in.readInt());
+            assignmentInfo.partitionsByHost.put(hostInfo, readTopicPartitions(in, topicIndexDict));
+        }
+    }
+
+    private static Set readTopicPartitions(final DataInputStream in,
+        final Map topicIndexDict) throws IOException {
+        final int numPartitions = in.readInt();
+        final Set partitions = new HashSet<>(numPartitions);
+        for (int j = 0; j < numPartitions; j++) {
+            partitions.add(new TopicPartition(topicIndexDict.get(in.readInt()), in.readInt()));
+        }
+        return partitions;
+    }
+
     private static void decodeVersionThreeData(final AssignmentInfo assignmentInfo,
                                                final DataInputStream in) throws IOException {
         decodeActiveTasks(assignmentInfo, in);
         decodeStandbyTasks(assignmentInfo, in);
-        decodeGlobalAssignmentData(assignmentInfo, in);
+        decodePartitionsByHost(assignmentInfo, in);
     }
 
     private static void decodeVersionFourData(final AssignmentInfo assignmentInfo,
@@ -332,6 +413,14 @@ private static void decodeVersionFourData(final AssignmentInfo assignmentInfo,
         assignmentInfo.errCode = in.readInt();
     }
 
+    private static void decodeVersionFiveData(final AssignmentInfo assignmentInfo,
+                                              final DataInputStream in) throws IOException {
+        decodeActiveTasks(assignmentInfo, in);
+        decodeStandbyTasks(assignmentInfo, in);
+        decodePartitionsByHostUsingDictionary(assignmentInfo, in);
+        assignmentInfo.errCode = in.readInt();
+    }
+
     @Override
     public int hashCode() {
         return usedVersion ^ latestSupportedVersion ^ activeTasks.hashCode() ^ standbyTasks.hashCode()
@@ -359,7 +448,6 @@ public String toString() {
             + ", supported version=" + latestSupportedVersion
             + ", active tasks=" + activeTasks
             + ", standby tasks=" + standbyTasks
-            + ", global assignment=" + partitionsByHost + "]";
+            + ", partitions by host=" + partitionsByHost + "]";
     }
-
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfo.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfo.java
index 03b9e2d06b20d..9161527fc50b4 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfo.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfo.java
@@ -32,7 +32,7 @@ public class SubscriptionInfo {
 
     private static final Logger log = LoggerFactory.getLogger(SubscriptionInfo.class);
 
-    public static final int LATEST_SUPPORTED_VERSION = 4;
+    public static final int LATEST_SUPPORTED_VERSION = 5;
     static final int UNKNOWN = -1;
 
     private final int usedVersion;
@@ -127,6 +127,9 @@ public ByteBuffer encode() {
             case 4:
                 buf = encodeVersionFour();
                 break;
+            case 5:
+                buf = encodeVersionFive();
+                break;
             default:
                 throw new IllegalStateException("Unknown metadata version: " + usedVersion
                     + "; latest supported version: " + LATEST_SUPPORTED_VERSION);
@@ -205,12 +208,11 @@ protected void encodeUserEndPoint(final ByteBuffer buf,
         }
     }
 
-    private ByteBuffer encodeVersionThree() {
+    private ByteBuffer encodeVersionThreeFourAndFive(final int usedVersion) {
         final byte[] endPointBytes = prepareUserEndPoint();
+        final ByteBuffer buf = ByteBuffer.allocate(getVersionThreeFourAndFiveByteLength(endPointBytes));
 
-        final ByteBuffer buf = ByteBuffer.allocate(getVersionThreeAndFourByteLength(endPointBytes));
-
-        buf.putInt(3); // used version
+        buf.putInt(usedVersion); // used version
         buf.putInt(LATEST_SUPPORTED_VERSION); // supported version
         encodeClientUUID(buf);
         encodeTasks(buf, prevTasks);
@@ -220,22 +222,19 @@ private ByteBuffer encodeVersionThree() {
         return buf;
     }
 
-    private ByteBuffer encodeVersionFour() {
-        final byte[] endPointBytes = prepareUserEndPoint();
-
-        final ByteBuffer buf = ByteBuffer.allocate(getVersionThreeAndFourByteLength(endPointBytes));
+    private ByteBuffer encodeVersionThree() {
+        return encodeVersionThreeFourAndFive(3);
+    }
 
-        buf.putInt(4); // used version
-        buf.putInt(LATEST_SUPPORTED_VERSION); // supported version
-        encodeClientUUID(buf);
-        encodeTasks(buf, prevTasks);
-        encodeTasks(buf, standbyTasks);
-        encodeUserEndPoint(buf, endPointBytes);
+    private ByteBuffer encodeVersionFour() {
+        return encodeVersionThreeFourAndFive(4);
+    }
 
-        return buf;
+    private ByteBuffer encodeVersionFive() {
+        return encodeVersionThreeFourAndFive(5);
     }
 
-    protected int getVersionThreeAndFourByteLength(final byte[] endPointBytes) {
+    protected int getVersionThreeFourAndFiveByteLength(final byte[] endPointBytes) {
         return 4 + // used version
                4 + // latest supported version version
                16 + // client ID
@@ -266,6 +265,7 @@ public static SubscriptionInfo decode(final ByteBuffer data) {
                 break;
             case 3:
             case 4:
+            case 5:
                 latestSupportedVersion = data.getInt();
                 subscriptionInfo = new SubscriptionInfo(usedVersion, latestSupportedVersion);
                 decodeVersionThreeData(subscriptionInfo, data);
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfoTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfoTest.java
index 8b990659da15f..0551e1d3ea952 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfoTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfoTest.java
@@ -94,4 +94,11 @@ public void shouldEncodeAndDecodeVersion4() {
         final AssignmentInfo expectedInfo = new AssignmentInfo(4, AssignmentInfo.LATEST_SUPPORTED_VERSION, activeTasks, standbyTasks, globalAssignment, 2);
         assertEquals(expectedInfo, AssignmentInfo.decode(info.encode()));
     }
+
+    @Test
+    public void shouldEncodeAndDecodeVersion5() {
+        final AssignmentInfo info = new AssignmentInfo(5, activeTasks, standbyTasks, globalAssignment, 2);
+        final AssignmentInfo expectedInfo = new AssignmentInfo(5, AssignmentInfo.LATEST_SUPPORTED_VERSION, activeTasks, standbyTasks, globalAssignment, 2);
+        assertEquals(expectedInfo, AssignmentInfo.decode(info.encode()));
+    }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
index 0b2d0b319aa5a..50a0a1148f930 100644
--- a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
@@ -271,7 +271,7 @@ public ByteBuffer encode() {
         private ByteBuffer encodeFutureVersion() {
             final byte[] endPointBytes = prepareUserEndPoint();
 
-            final ByteBuffer buf = ByteBuffer.allocate(getVersionThreeAndFourByteLength(endPointBytes));
+            final ByteBuffer buf = ByteBuffer.allocate(getVersionThreeFourAndFiveByteLength(endPointBytes));
 
             buf.putInt(LATEST_SUPPORTED_VERSION + 1); // used version
             buf.putInt(LATEST_SUPPORTED_VERSION + 1); // supported version
diff --git a/tests/kafkatest/tests/streams/streams_upgrade_test.py b/tests/kafkatest/tests/streams/streams_upgrade_test.py
index d382c97006f96..fb85732818940 100644
--- a/tests/kafkatest/tests/streams/streams_upgrade_test.py
+++ b/tests/kafkatest/tests/streams/streams_upgrade_test.py
@@ -571,24 +571,24 @@ def do_rolling_bounce(self, processor, counter, current_generation):
                     monitors[first_other_processor] = first_other_monitor
                     monitors[second_other_processor] = second_other_monitor
 
-                    leader_monitor.wait_until("Received a future (version probing) subscription (version: 5). Sending empty assignment back (with supported version 4).",
+                    leader_monitor.wait_until("Received a future (version probing) subscription (version: 6). Sending empty assignment back (with supported version 5).",
                                               timeout_sec=60,
                                               err_msg="Could not detect 'version probing' attempt at leader " + str(self.leader.node.account))
 
                     if len(self.old_processors) > 0:
-                        log_monitor.wait_until("Sent a version 5 subscription and got version 4 assignment back (successful version probing). Downgrading subscription metadata to received version and trigger new rebalance.",
+                        log_monitor.wait_until("Sent a version 6 subscription and got version 5 assignment back (successful version probing). Downgrading subscription metadata to received version and trigger new rebalance.",
                                                timeout_sec=60,
                                                err_msg="Could not detect 'successful version probing' at upgrading node " + str(node.account))
                     else:
-                        log_monitor.wait_until("Sent a version 5 subscription and got version 4 assignment back (successful version probing). Setting subscription metadata to leaders supported version 5 and trigger new rebalance.",
+                        log_monitor.wait_until("Sent a version 6 subscription and got version 5 assignment back (successful version probing). Setting subscription metadata to leaders supported version 6 and trigger new rebalance.",
                                                timeout_sec=60,
                                                err_msg="Could not detect 'successful version probing with upgraded leader' at upgrading node " + str(node.account))
-                        first_other_monitor.wait_until("Sent a version 4 subscription and group leader.s latest supported version is 5. Upgrading subscription metadata version to 5 for next rebalance.",
+                        first_other_monitor.wait_until("Sent a version 5 subscription and group leader.s latest supported version is 6. Upgrading subscription metadata version to 6 for next rebalance.",
                                                        timeout_sec=60,
-                                                       err_msg="Never saw output 'Upgrade metadata to version 4' on" + str(first_other_node.account))
-                        second_other_monitor.wait_until("Sent a version 4 subscription and group leader.s latest supported version is 5. Upgrading subscription metadata version to 5 for next rebalance.",
+                                                       err_msg="Never saw output 'Upgrade metadata to version 5' on" + str(first_other_node.account))
+                        second_other_monitor.wait_until("Sent a version 5 subscription and group leader.s latest supported version is 6. Upgrading subscription metadata version to 6 for next rebalance.",
                                                         timeout_sec=60,
-                                                        err_msg="Never saw output 'Upgrade metadata to version 4' on" + str(second_other_node.account))
+                                                        err_msg="Never saw output 'Upgrade metadata to version 5' on" + str(second_other_node.account))
 
                     log_monitor.wait_until("Version probing detected. Triggering new rebalance.",
                                            timeout_sec=60,

From ad3ccf8f317bd28b6527fd4d5089ce8f0a37479f Mon Sep 17 00:00:00 2001
From: Sean Glover 
Date: Fri, 6 Sep 2019 02:03:13 -0400
Subject: [PATCH 0591/1071] KAFKA-8822; Consolidate `PartitionRecords` and
 `CompletedFetch` types in Fetcher (#7228)

The `CompletedFetch` and `PartitionRecords` types are somewhat redundant. This PR consolidates the two types.

Reviewers: Jason Gustafson 
---
 .../clients/consumer/internals/Fetcher.java   | 192 ++++++++----------
 1 file changed, 82 insertions(+), 110 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
index c1c42bd06ca4f..72aed9d867866 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
@@ -142,7 +142,7 @@ public class Fetcher implements Closeable {
     private final ConsumerMetadata metadata;
     private final FetchManagerMetrics sensors;
     private final SubscriptionState subscriptions;
-    private final ConcurrentLinkedQueue completedFetches;
+    private final ConcurrentLinkedQueue completedFetches;
     private final BufferSupplier decompressionBufferSupplier = BufferSupplier.create();
     private final Deserializer keyDeserializer;
     private final Deserializer valueDeserializer;
@@ -154,7 +154,7 @@ public class Fetcher implements Closeable {
     private final Set nodesWithPendingFetchRequests;
     private final ApiVersions apiVersions;
 
-    private PartitionRecords nextInLineRecords = null;
+    private CompletedFetch nextInLineFetch = null;
 
     public Fetcher(LogContext logContext,
                    ConsumerNetworkClient client,
@@ -298,15 +298,16 @@ public void onSuccess(ClientResponse resp) {
                                             throw new IllegalStateException(message);
                                         } else {
                                             long fetchOffset = requestData.fetchOffset;
-                                            FetchResponse.PartitionData fetchData = entry.getValue();
+                                            FetchResponse.PartitionData partitionData = entry.getValue();
 
                                             log.debug("Fetch {} at offset {} for partition {} returned fetch data {}",
-                                                    isolationLevel, fetchOffset, partition, fetchData);
+                                                    isolationLevel, fetchOffset, partition, partitionData);
 
-                                            CompletedFetch completedFetch = new CompletedFetch(partition, fetchOffset,
-                                                    fetchData, metricAggregator, resp.requestHeader().apiVersion());
+                                            Iterator batches = partitionData.records.batches().iterator();
+                                            short responseVersion = resp.requestHeader().apiVersion();
 
-                                            completedFetches.add(parseCompletedFetch(completedFetch));
+                                            completedFetches.add(new CompletedFetch(partition, partitionData,
+                                                    metricAggregator, batches, fetchOffset, responseVersion));
                                         }
                                     }
 
@@ -591,45 +592,45 @@ private Map beginningOrEndOffset(Collection>> fetchedRecords() {
         Map>> fetched = new HashMap<>();
-        Queue pausedCompletedFetches = new ArrayDeque<>();
+        Queue pausedCompletedFetches = new ArrayDeque<>();
         int recordsRemaining = maxPollRecords;
 
         try {
             while (recordsRemaining > 0) {
-                if (nextInLineRecords == null || nextInLineRecords.isFetched) {
-                    PartitionRecords records = completedFetches.peek();
+                if (nextInLineFetch == null || nextInLineFetch.isConsumed) {
+                    CompletedFetch records = completedFetches.peek();
                     if (records == null) break;
 
                     if (records.notInitialized()) {
                         try {
-                            nextInLineRecords = initializePartitionRecords(records);
+                            nextInLineFetch = initializeCompletedFetch(records);
                         } catch (Exception e) {
                             // Remove a completedFetch upon a parse with exception if (1) it contains no records, and
                             // (2) there are no fetched records with actual content preceding this exception.
                             // The first condition ensures that the completedFetches is not stuck with the same completedFetch
                             // in cases such as the TopicAuthorizationException, and the second condition ensures that no
                             // potential data loss due to an exception in a following record.
-                            FetchResponse.PartitionData partition = records.completedFetch.partitionData;
+                            FetchResponse.PartitionData partition = records.partitionData;
                             if (fetched.isEmpty() && (partition.records == null || partition.records.sizeInBytes() == 0)) {
                                 completedFetches.poll();
                             }
                             throw e;
                         }
                     } else {
-                        nextInLineRecords = records;
+                        nextInLineFetch = records;
                     }
                     completedFetches.poll();
-                } else if (subscriptions.isPaused(nextInLineRecords.partition)) {
+                } else if (subscriptions.isPaused(nextInLineFetch.partition)) {
                     // when the partition is paused we add the records back to the completedFetches queue instead of draining
                     // them so that they can be returned on a subsequent poll if the partition is resumed at that time
-                    log.debug("Skipping fetching records for assigned partition {} because it is paused", nextInLineRecords.partition);
-                    pausedCompletedFetches.add(nextInLineRecords);
-                    nextInLineRecords = null;
+                    log.debug("Skipping fetching records for assigned partition {} because it is paused", nextInLineFetch.partition);
+                    pausedCompletedFetches.add(nextInLineFetch);
+                    nextInLineFetch = null;
                 } else {
-                    List> records = fetchRecords(nextInLineRecords, recordsRemaining);
+                    List> records = fetchRecords(nextInLineFetch, recordsRemaining);
 
                     if (!records.isEmpty()) {
-                        TopicPartition partition = nextInLineRecords.partition;
+                        TopicPartition partition = nextInLineFetch.partition;
                         List> currentRecords = fetched.get(partition);
                         if (currentRecords == null) {
                             fetched.put(partition, records);
@@ -658,38 +659,38 @@ public Map>> fetchedRecords() {
         return fetched;
     }
 
-    private List> fetchRecords(PartitionRecords partitionRecords, int maxRecords) {
-        if (!subscriptions.isAssigned(partitionRecords.partition)) {
+    private List> fetchRecords(CompletedFetch completedFetch, int maxRecords) {
+        if (!subscriptions.isAssigned(completedFetch.partition)) {
             // this can happen when a rebalance happened before fetched records are returned to the consumer's poll call
             log.debug("Not returning fetched records for partition {} since it is no longer assigned",
-                    partitionRecords.partition);
-        } else if (!subscriptions.isFetchable(partitionRecords.partition)) {
+                    completedFetch.partition);
+        } else if (!subscriptions.isFetchable(completedFetch.partition)) {
             // this can happen when a partition is paused before fetched records are returned to the consumer's
             // poll call or if the offset is being reset
             log.debug("Not returning fetched records for assigned partition {} since it is no longer fetchable",
-                    partitionRecords.partition);
+                    completedFetch.partition);
         } else {
-            SubscriptionState.FetchPosition position = subscriptions.position(partitionRecords.partition);
-            if (partitionRecords.nextFetchOffset == position.offset) {
-                List> partRecords = partitionRecords.fetchRecords(maxRecords);
+            SubscriptionState.FetchPosition position = subscriptions.position(completedFetch.partition);
+            if (completedFetch.nextFetchOffset == position.offset) {
+                List> partRecords = completedFetch.fetchRecords(maxRecords);
 
-                if (partitionRecords.nextFetchOffset > position.offset) {
+                if (completedFetch.nextFetchOffset > position.offset) {
                     SubscriptionState.FetchPosition nextPosition = new SubscriptionState.FetchPosition(
-                            partitionRecords.nextFetchOffset,
-                            partitionRecords.lastEpoch,
+                            completedFetch.nextFetchOffset,
+                            completedFetch.lastEpoch,
                             position.currentLeader);
                     log.trace("Returning fetched records at offset {} for assigned partition {} and update " +
-                            "position to {}", position, partitionRecords.partition, nextPosition);
-                    subscriptions.position(partitionRecords.partition, nextPosition);
+                            "position to {}", position, completedFetch.partition, nextPosition);
+                    subscriptions.position(completedFetch.partition, nextPosition);
                 }
 
-                Long partitionLag = subscriptions.partitionLag(partitionRecords.partition, isolationLevel);
+                Long partitionLag = subscriptions.partitionLag(completedFetch.partition, isolationLevel);
                 if (partitionLag != null)
-                    this.sensors.recordPartitionLag(partitionRecords.partition, partitionLag);
+                    this.sensors.recordPartitionLag(completedFetch.partition, partitionLag);
 
-                Long lead = subscriptions.partitionLead(partitionRecords.partition);
+                Long lead = subscriptions.partitionLead(completedFetch.partition);
                 if (lead != null) {
-                    this.sensors.recordPartitionLead(partitionRecords.partition, lead);
+                    this.sensors.recordPartitionLead(completedFetch.partition, lead);
                 }
 
                 return partRecords;
@@ -697,12 +698,12 @@ private List> fetchRecords(PartitionRecords partitionRecord
                 // these records aren't next in line based on the last consumed position, ignore them
                 // they must be from an obsolete request
                 log.debug("Ignoring fetched records for {} at offset {} since the current position is {}",
-                        partitionRecords.partition, partitionRecords.nextFetchOffset, position);
+                        completedFetch.partition, completedFetch.nextFetchOffset, position);
             }
         }
 
-        log.trace("Draining fetched records for partition {}", partitionRecords.partition);
-        partitionRecords.drain();
+        log.trace("Draining fetched records for partition {}", completedFetch.partition);
+        completedFetch.drain();
 
         return emptyList();
     }
@@ -1055,10 +1056,10 @@ public ListOffsetResult() {
 
     private List fetchablePartitions() {
         Set exclude = new HashSet<>();
-        if (nextInLineRecords != null && !nextInLineRecords.isFetched) {
-            exclude.add(nextInLineRecords.partition);
+        if (nextInLineFetch != null && !nextInLineFetch.isConsumed) {
+            exclude.add(nextInLineFetch.partition);
         }
-        for (PartitionRecords completedFetch : completedFetches) {
+        for (CompletedFetch completedFetch : completedFetches) {
             exclude.add(completedFetch.partition);
         }
         return subscriptions.fetchablePartitions(tp -> !exclude.contains(tp));
@@ -1158,25 +1159,13 @@ private  Map> regroupPartitionMapByNode(Map partition = completedFetch.partitionData;
-
-        Iterator batches = partition.records.batches().iterator();
-        return new PartitionRecords(tp, completedFetch, batches);
-    }
-
-    /**
-     * Initialize a PartitionRecords object.
-     */
-    private PartitionRecords initializePartitionRecords(PartitionRecords partitionRecordsToInitialize) {
-        CompletedFetch completedFetch = partitionRecordsToInitialize.completedFetch;
-        TopicPartition tp = completedFetch.partition;
-        FetchResponse.PartitionData partition = completedFetch.partitionData;
-        long fetchOffset = completedFetch.fetchedOffset;
-        PartitionRecords partitionRecords = null;
+    private CompletedFetch initializeCompletedFetch(CompletedFetch nextCompletedFetch) {
+        TopicPartition tp = nextCompletedFetch.partition;
+        FetchResponse.PartitionData partition = nextCompletedFetch.partitionData;
+        long fetchOffset = nextCompletedFetch.nextFetchOffset;
+        CompletedFetch completedFetch = null;
         Errors error = partition.error;
 
         try {
@@ -1196,7 +1185,7 @@ private PartitionRecords initializePartitionRecords(PartitionRecords partitionRe
                 log.trace("Preparing to read {} bytes of data for partition {} with offset {}",
                         partition.records.sizeInBytes(), tp, position);
                 Iterator batches = partition.records.batches().iterator();
-                partitionRecords = partitionRecordsToInitialize;
+                completedFetch = nextCompletedFetch;
 
                 if (!batches.hasNext() && partition.records.sizeInBytes() > 0) {
                     if (completedFetch.responseVersion < 3) {
@@ -1232,7 +1221,7 @@ private PartitionRecords initializePartitionRecords(PartitionRecords partitionRe
                 }
 
                 if (partition.preferredReadReplica.isPresent()) {
-                    subscriptions.updatePreferredReadReplica(partitionRecords.partition, partition.preferredReadReplica.get(), () -> {
+                    subscriptions.updatePreferredReadReplica(completedFetch.partition, partition.preferredReadReplica.get(), () -> {
                         long expireTimeMs = time.milliseconds() + metadata.metadataExpireMs();
                         log.debug("Updating preferred read replica for partition {} to {}, set to expire at {}",
                                 tp, partition.preferredReadReplica.get(), expireTimeMs);
@@ -1241,7 +1230,7 @@ private PartitionRecords initializePartitionRecords(PartitionRecords partitionRe
                 }
 
 
-                partitionRecordsToInitialize.initialized = true;
+                nextCompletedFetch.initialized = true;
             } else if (error == Errors.NOT_LEADER_FOR_PARTITION ||
                        error == Errors.REPLICA_NOT_AVAILABLE ||
                        error == Errors.KAFKA_STORAGE_ERROR ||
@@ -1281,8 +1270,8 @@ private PartitionRecords initializePartitionRecords(PartitionRecords partitionRe
                 throw new IllegalStateException("Unexpected error code " + error.code() + " while fetching from partition " + tp);
             }
         } finally {
-            if (partitionRecords == null)
-                completedFetch.metricAggregator.record(tp, 0, 0);
+            if (completedFetch == null)
+                nextCompletedFetch.metricAggregator.record(tp, 0, 0);
 
             if (error != Errors.NONE)
                 // we move the partition to the end if there was an error. This way, it's more likely that partitions for
@@ -1290,7 +1279,7 @@ private PartitionRecords initializePartitionRecords(PartitionRecords partitionRe
                 subscriptions.movePartitionToEnd(tp);
         }
 
-        return partitionRecords;
+        return completedFetch;
     }
 
     /**
@@ -1332,9 +1321,9 @@ private Optional maybeLeaderEpoch(int leaderEpoch) {
      * @param assignedPartitions  newly assigned {@link TopicPartition}
      */
     public void clearBufferedDataForUnassignedPartitions(Collection assignedPartitions) {
-        Iterator completedFetchesItr = completedFetches.iterator();
+        Iterator completedFetchesItr = completedFetches.iterator();
         while (completedFetchesItr.hasNext()) {
-            PartitionRecords records = completedFetchesItr.next();
+            CompletedFetch records = completedFetchesItr.next();
             TopicPartition tp = records.partition;
             if (!assignedPartitions.contains(tp)) {
                 records.drain();
@@ -1342,9 +1331,9 @@ public void clearBufferedDataForUnassignedPartitions(Collection
             }
         }
 
-        if (nextInLineRecords != null && !assignedPartitions.contains(nextInLineRecords.partition)) {
-            nextInLineRecords.drain();
-            nextInLineRecords = null;
+        if (nextInLineFetch != null && !assignedPartitions.contains(nextInLineFetch.partition)) {
+            nextInLineFetch.drain();
+            nextInLineFetch = null;
         }
     }
 
@@ -1377,12 +1366,14 @@ public static Sensor throttleTimeSensor(Metrics metrics, FetcherMetricsRegistry
         return fetchThrottleTimeSensor;
     }
 
-    private class PartitionRecords {
+    private class CompletedFetch {
         private final TopicPartition partition;
-        private final CompletedFetch completedFetch;
         private final Iterator batches;
         private final Set abortedProducerIds;
         private final PriorityQueue abortedTransactions;
+        private final FetchResponse.PartitionData partitionData;
+        private final FetchResponseMetricAggregator metricAggregator;
+        private final short responseVersion;
 
         private int recordsRead;
         private int bytesRead;
@@ -1391,29 +1382,34 @@ private class PartitionRecords {
         private CloseableIterator records;
         private long nextFetchOffset;
         private Optional lastEpoch;
-        private boolean isFetched = false;
+        private boolean isConsumed = false;
         private Exception cachedRecordException = null;
         private boolean corruptLastRecord = false;
         private boolean initialized = false;
 
-        private PartitionRecords(TopicPartition partition,
-                                 CompletedFetch completedFetch,
-                                 Iterator batches) {
+        private CompletedFetch(TopicPartition partition,
+                               FetchResponse.PartitionData partitionData,
+                               FetchResponseMetricAggregator metricAggregator,
+                               Iterator batches,
+                               Long fetchOffset,
+                               short responseVersion) {
             this.partition = partition;
-            this.completedFetch = completedFetch;
+            this.partitionData = partitionData;
+            this.metricAggregator = metricAggregator;
             this.batches = batches;
-            this.nextFetchOffset = completedFetch.fetchedOffset;
+            this.nextFetchOffset = fetchOffset;
+            this.responseVersion = responseVersion;
             this.lastEpoch = Optional.empty();
             this.abortedProducerIds = new HashSet<>();
-            this.abortedTransactions = abortedTransactions(completedFetch.partitionData);
+            this.abortedTransactions = abortedTransactions(partitionData);
         }
 
         private void drain() {
-            if (!isFetched) {
+            if (!isConsumed) {
                 maybeCloseRecordStream();
                 cachedRecordException = null;
-                this.isFetched = true;
-                this.completedFetch.metricAggregator.record(partition, bytesRead, recordsRead);
+                this.isConsumed = true;
+                this.metricAggregator.record(partition, bytesRead, recordsRead);
 
                 // we move the partition to the end if we received some bytes. This way, it's more likely that partitions
                 // for the same topic can remain together (allowing for more efficient serialization).
@@ -1422,10 +1418,6 @@ private void drain() {
             }
         }
 
-        private Optional preferredReadReplica() {
-            return completedFetch.partitionData.preferredReadReplica;
-        }
-
         private void maybeEnsureValid(RecordBatch batch) {
             if (checkCrcs && currentBatch.magic() >= RecordBatch.MAGIC_VALUE_V2) {
                 try {
@@ -1523,7 +1515,7 @@ private List> fetchRecords(int maxRecords) {
                                              + ". If needed, please seek past the record to "
                                              + "continue consumption.", cachedRecordException);
 
-            if (isFetched)
+            if (isConsumed)
                 return Collections.emptyList();
 
             List> records = new ArrayList<>();
@@ -1602,26 +1594,6 @@ private boolean notInitialized() {
         }
     }
 
-    private static class CompletedFetch {
-        private final TopicPartition partition;
-        private final long fetchedOffset;
-        private final FetchResponse.PartitionData partitionData;
-        private final FetchResponseMetricAggregator metricAggregator;
-        private final short responseVersion;
-
-        private CompletedFetch(TopicPartition partition,
-                               long fetchedOffset,
-                               FetchResponse.PartitionData partitionData,
-                               FetchResponseMetricAggregator metricAggregator,
-                               short responseVersion) {
-            this.partition = partition;
-            this.fetchedOffset = fetchedOffset;
-            this.partitionData = partitionData;
-            this.metricAggregator = metricAggregator;
-            this.responseVersion = responseVersion;
-        }
-    }
-
     /**
      * Since we parse the message data for each partition from each fetch response lazily, fetch-level
      * metrics need to be aggregated as the messages from each partition are parsed. This class is used
@@ -1837,8 +1809,8 @@ private Map topicPartitionTags(TopicPartition tp) {
 
     @Override
     public void close() {
-        if (nextInLineRecords != null)
-            nextInLineRecords.drain();
+        if (nextInLineFetch != null)
+            nextInLineFetch.drain();
         decompressionBufferSupplier.close();
     }
 

From c0019e653891182d7a95464175c9b4ef63f8bae1 Mon Sep 17 00:00:00 2001
From: Boyang Chen 
Date: Thu, 5 Sep 2019 23:07:42 -0700
Subject: [PATCH 0592/1071] KAFKA-8590; Use automated TxnOffsetCommit type and
 add tests for OffsetCommit  (#6994)

This PR changes the TxnOffsetCommit protocol to auto-generated types, and add more unit test coverage to the plain OffsetCommit protocol.

Reviewers: Jason Gustafson 
---
 checkstyle/suppressions.xml                   |   2 +-
 .../internals/TransactionManager.java         |  19 +-
 .../apache/kafka/common/protocol/ApiKeys.java |   8 +-
 .../common/requests/AbstractResponse.java     |   2 +-
 .../common/requests/OffsetCommitRequest.java  |  28 +-
 .../common/requests/OffsetCommitResponse.java |  46 ++-
 .../requests/TxnOffsetCommitRequest.java      | 266 ++++++------------
 .../requests/TxnOffsetCommitResponse.java     | 150 ++++------
 .../internals/TransactionManagerTest.java     |   6 +-
 .../kafka/common/message/MessageTest.java     | 178 ++++++++++++
 .../requests/OffsetCommitRequestTest.java     | 128 ++++++++-
 .../requests/OffsetCommitResponseTest.java    |  99 +++++++
 .../common/requests/RequestResponseTest.java  |  10 +-
 .../requests/TxnOffsetCommitRequestTest.java  | 109 +++++++
 .../requests/TxnOffsetCommitResponseTest.java |  67 +++++
 .../main/scala/kafka/server/KafkaApis.scala   |  19 +-
 .../unit/kafka/server/KafkaApisTest.scala     |  13 +-
 .../unit/kafka/server/RequestQuotaTest.scala  |  14 +-
 18 files changed, 820 insertions(+), 344 deletions(-)
 create mode 100644 clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java
 create mode 100644 clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitRequestTest.java
 create mode 100644 clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java

diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index 475b9b0cd6087..e2cafa9be2e9c 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -65,7 +65,7 @@
               files="(Sender|Fetcher|KafkaConsumer|Metrics|RequestResponse|TransactionManager|KafkaAdminClient|Message)Test.java"/>
 
     
+              files="(ConsumerCoordinator|KafkaConsumer|RequestResponse|Fetcher|KafkaAdminClient|Message)Test.java"/>
 
     
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java
index cd091542dc8db..944eb8c5c381c 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java
@@ -32,6 +32,7 @@
 import org.apache.kafka.common.errors.UnsupportedVersionException;
 import org.apache.kafka.common.message.FindCoordinatorRequestData;
 import org.apache.kafka.common.message.InitProducerIdRequestData;
+import org.apache.kafka.common.message.TxnOffsetCommitRequestData;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.record.DefaultRecordBatch;
 import org.apache.kafka.common.record.RecordBatch;
@@ -986,8 +987,14 @@ private TxnOffsetCommitHandler txnOffsetCommitHandler(TransactionalRequestResult
                     offsetAndMetadata.metadata(), offsetAndMetadata.leaderEpoch());
             pendingTxnOffsetCommits.put(entry.getKey(), committedOffset);
         }
-        TxnOffsetCommitRequest.Builder builder = new TxnOffsetCommitRequest.Builder(transactionalId, consumerGroupId,
-                producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, pendingTxnOffsetCommits);
+        TxnOffsetCommitRequest.Builder builder = new TxnOffsetCommitRequest.Builder(
+            new TxnOffsetCommitRequestData()
+                .setTransactionalId(transactionalId)
+                .setGroupId(consumerGroupId)
+                .setProducerId(producerIdAndEpoch.producerId)
+                .setProducerEpoch(producerIdAndEpoch.epoch)
+                .setTopics(TxnOffsetCommitRequest.getTopics(pendingTxnOffsetCommits))
+        );
         return new TxnOffsetCommitHandler(result, builder);
     }
 
@@ -1432,7 +1439,7 @@ FindCoordinatorRequest.CoordinatorType coordinatorType() {
 
         @Override
         String coordinatorKey() {
-            return builder.consumerGroupId();
+            return builder.data.groupId();
         }
 
         @Override
@@ -1441,7 +1448,7 @@ public void handleResponse(AbstractResponse response) {
             boolean coordinatorReloaded = false;
             Map errors = txnOffsetCommitResponse.errors();
 
-            log.debug("Received TxnOffsetCommit response for consumer group {}: {}", builder.consumerGroupId(),
+            log.debug("Received TxnOffsetCommit response for consumer group {}: {}", builder.data.groupId(),
                     errors);
 
             for (Map.Entry entry : errors.entrySet()) {
@@ -1454,14 +1461,14 @@ public void handleResponse(AbstractResponse response) {
                         || error == Errors.REQUEST_TIMED_OUT) {
                     if (!coordinatorReloaded) {
                         coordinatorReloaded = true;
-                        lookupCoordinator(FindCoordinatorRequest.CoordinatorType.GROUP, builder.consumerGroupId());
+                        lookupCoordinator(FindCoordinatorRequest.CoordinatorType.GROUP, builder.data.groupId());
                     }
                 } else if (error == Errors.UNKNOWN_TOPIC_OR_PARTITION
                         || error == Errors.COORDINATOR_LOAD_IN_PROGRESS) {
                     // If the topic is unknown or the coordinator is loading, retry with the current coordinator
                     continue;
                 } else if (error == Errors.GROUP_AUTHORIZATION_FAILED) {
-                    abortableError(GroupAuthorizationException.forGroupId(builder.consumerGroupId()));
+                    abortableError(GroupAuthorizationException.forGroupId(builder.data.groupId()));
                     break;
                 } else if (error == Errors.TRANSACTIONAL_ID_AUTHORIZATION_FAILED
                         || error == Errors.INVALID_PRODUCER_EPOCH
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
index 3b45e5882f4e0..2393da202369c 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
@@ -66,6 +66,8 @@
 import org.apache.kafka.common.message.SaslHandshakeResponseData;
 import org.apache.kafka.common.message.SyncGroupRequestData;
 import org.apache.kafka.common.message.SyncGroupResponseData;
+import org.apache.kafka.common.message.TxnOffsetCommitRequestData;
+import org.apache.kafka.common.message.TxnOffsetCommitResponseData;
 import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.SchemaException;
 import org.apache.kafka.common.protocol.types.Struct;
@@ -109,8 +111,6 @@
 import org.apache.kafka.common.requests.ProduceResponse;
 import org.apache.kafka.common.requests.StopReplicaRequest;
 import org.apache.kafka.common.requests.StopReplicaResponse;
-import org.apache.kafka.common.requests.TxnOffsetCommitRequest;
-import org.apache.kafka.common.requests.TxnOffsetCommitResponse;
 import org.apache.kafka.common.requests.UpdateMetadataRequest;
 import org.apache.kafka.common.requests.UpdateMetadataResponse;
 import org.apache.kafka.common.requests.WriteTxnMarkersRequest;
@@ -172,8 +172,8 @@ public Struct parseResponse(short version, ByteBuffer buffer) {
             EndTxnResponse.schemaVersions()),
     WRITE_TXN_MARKERS(27, "WriteTxnMarkers", true, RecordBatch.MAGIC_VALUE_V2, WriteTxnMarkersRequest.schemaVersions(),
             WriteTxnMarkersResponse.schemaVersions()),
-    TXN_OFFSET_COMMIT(28, "TxnOffsetCommit", false, RecordBatch.MAGIC_VALUE_V2, TxnOffsetCommitRequest.schemaVersions(),
-            TxnOffsetCommitResponse.schemaVersions()),
+    TXN_OFFSET_COMMIT(28, "TxnOffsetCommit", false, RecordBatch.MAGIC_VALUE_V2, TxnOffsetCommitRequestData.SCHEMAS,
+                      TxnOffsetCommitResponseData.SCHEMAS),
     DESCRIBE_ACLS(29, "DescribeAcls", DescribeAclsRequest.schemaVersions(), DescribeAclsResponse.schemaVersions()),
     CREATE_ACLS(30, "CreateAcls", CreateAclsRequest.schemaVersions(), CreateAclsResponse.schemaVersions()),
     DELETE_ACLS(31, "DeleteAcls", DeleteAclsRequest.schemaVersions(), DeleteAclsResponse.schemaVersions()),
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
index b84a2c03d1eaa..11466d7249867 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
@@ -127,7 +127,7 @@ public static AbstractResponse parseResponse(ApiKeys apiKey, Struct struct, shor
             case WRITE_TXN_MARKERS:
                 return new WriteTxnMarkersResponse(struct);
             case TXN_OFFSET_COMMIT:
-                return new TxnOffsetCommitResponse(struct);
+                return new TxnOffsetCommitResponse(struct, version);
             case DESCRIBE_ACLS:
                 return new DescribeAclsResponse(struct);
             case CREATE_ACLS:
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java
index ff53a2dae1a3a..72ebef261f4df 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java
@@ -19,7 +19,10 @@
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.errors.UnsupportedVersionException;
 import org.apache.kafka.common.message.OffsetCommitRequestData;
+import org.apache.kafka.common.message.OffsetCommitRequestData.OffsetCommitRequestTopic;
 import org.apache.kafka.common.message.OffsetCommitResponseData;
+import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponsePartition;
+import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponseTopic;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.protocol.types.Struct;
@@ -88,7 +91,7 @@ public OffsetCommitRequestData data() {
 
     public Map offsets() {
         Map offsets = new HashMap<>();
-        for (OffsetCommitRequestData.OffsetCommitRequestTopic topic : data.topics()) {
+        for (OffsetCommitRequestTopic topic : data.topics()) {
             for (OffsetCommitRequestData.OffsetCommitRequestPartition partition : topic.partitions()) {
                 offsets.put(new TopicPartition(topic.name(), partition.partitionIndex()),
                         partition.committedOffset());
@@ -97,20 +100,19 @@ public Map offsets() {
         return offsets;
     }
 
-    public static List getErrorResponseTopics(
-            List requestTopics,
+    public static List getErrorResponseTopics(
+            List requestTopics,
             Errors e) {
-        List
-                responseTopicData = new ArrayList<>();
-        for (OffsetCommitRequestData.OffsetCommitRequestTopic entry : requestTopics) {
-            List responsePartitions =
+        List responseTopicData = new ArrayList<>();
+        for (OffsetCommitRequestTopic entry : requestTopics) {
+            List responsePartitions =
                     new ArrayList<>();
             for (OffsetCommitRequestData.OffsetCommitRequestPartition requestPartition : entry.partitions()) {
-                responsePartitions.add(new OffsetCommitResponseData.OffsetCommitResponsePartition()
-                        .setPartitionIndex(requestPartition.partitionIndex())
-                        .setErrorCode(e.code()));
+                responsePartitions.add(new OffsetCommitResponsePartition()
+                                           .setPartitionIndex(requestPartition.partitionIndex())
+                                           .setErrorCode(e.code()));
             }
-            responseTopicData.add(new OffsetCommitResponseData.OffsetCommitResponseTopic()
+            responseTopicData.add(new OffsetCommitResponseTopic()
                     .setName(entry.name())
                     .setPartitions(responsePartitions)
             );
@@ -119,8 +121,8 @@ public static List getErrorR
     }
 
     @Override
-    public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
-        List
+    public OffsetCommitResponse getErrorResponse(int throttleTimeMs, Throwable e) {
+        List
                 responseTopicData = getErrorResponseTopics(data.topics(), Errors.forException(e));
 
         short versionId = version();
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
index fee2e472cc1af..9f30d3b911d0a 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
@@ -18,6 +18,8 @@
 
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.message.OffsetCommitResponseData;
+import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponsePartition;
+import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponseTopic;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.protocol.types.Struct;
@@ -30,18 +32,18 @@
 /**
  * Possible error codes:
  *
- * UNKNOWN_TOPIC_OR_PARTITION (3)
- * REQUEST_TIMED_OUT (7)
- * OFFSET_METADATA_TOO_LARGE (12)
- * COORDINATOR_LOAD_IN_PROGRESS (14)
- * GROUP_COORDINATOR_NOT_AVAILABLE (15)
- * NOT_COORDINATOR (16)
- * ILLEGAL_GENERATION (22)
- * UNKNOWN_MEMBER_ID (25)
- * REBALANCE_IN_PROGRESS (27)
- * INVALID_COMMIT_OFFSET_SIZE (28)
- * TOPIC_AUTHORIZATION_FAILED (29)
- * GROUP_AUTHORIZATION_FAILED (30)
+ *   - {@link Errors#UNKNOWN_TOPIC_OR_PARTITION}
+ *   - {@link Errors#REQUEST_TIMED_OUT}
+ *   - {@link Errors#OFFSET_METADATA_TOO_LARGE}
+ *   - {@link Errors#COORDINATOR_LOAD_IN_PROGRESS}
+ *   - {@link Errors#COORDINATOR_NOT_AVAILABLE}
+ *   - {@link Errors#NOT_COORDINATOR}
+ *   - {@link Errors#ILLEGAL_GENERATION}
+ *   - {@link Errors#UNKNOWN_MEMBER_ID}
+ *   - {@link Errors#REBALANCE_IN_PROGRESS}
+ *   - {@link Errors#INVALID_COMMIT_OFFSET_SIZE}
+ *   - {@link Errors#TOPIC_AUTHORIZATION_FAILED}
+ *   - {@link Errors#GROUP_AUTHORIZATION_FAILED}
  */
 public class OffsetCommitResponse extends AbstractResponse {
 
@@ -52,23 +54,19 @@ public OffsetCommitResponse(OffsetCommitResponseData data) {
     }
 
     public OffsetCommitResponse(int requestThrottleMs, Map responseData) {
-        Map
+        Map
                 responseTopicDataMap = new HashMap<>();
 
         for (Map.Entry entry : responseData.entrySet()) {
             TopicPartition topicPartition = entry.getKey();
             String topicName = topicPartition.topic();
 
-            OffsetCommitResponseData.OffsetCommitResponseTopic topic = responseTopicDataMap
-                    .getOrDefault(topicName, new OffsetCommitResponseData.OffsetCommitResponseTopic());
+            OffsetCommitResponseTopic topic = responseTopicDataMap.getOrDefault(
+                topicName, new OffsetCommitResponseTopic().setName(topicName));
 
-            if (topic.name().equals("")) {
-                topic.setName(topicName);
-            }
-            topic.partitions().add(new OffsetCommitResponseData.OffsetCommitResponsePartition()
-                    .setErrorCode(entry.getValue().code())
-                    .setPartitionIndex(topicPartition.partition())
-            );
+            topic.partitions().add(new OffsetCommitResponsePartition()
+                                       .setErrorCode(entry.getValue().code())
+                                       .setPartitionIndex(topicPartition.partition()));
             responseTopicDataMap.put(topicName, topic);
         }
 
@@ -97,8 +95,8 @@ public OffsetCommitResponseData data() {
     @Override
     public Map errorCounts() {
         Map errorMap = new HashMap<>();
-        for (OffsetCommitResponseData.OffsetCommitResponseTopic topic : data.topics()) {
-            for (OffsetCommitResponseData.OffsetCommitResponsePartition partition : topic.partitions()) {
+        for (OffsetCommitResponseTopic topic : data.topics()) {
+            for (OffsetCommitResponsePartition partition : topic.partitions()) {
                 errorMap.put(new TopicPartition(topic.name(), partition.partitionIndex()),
                         Errors.forCode(partition.errorCode()));
             }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java
index 1c922e1dd525c..279fc9cb070a7 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java
@@ -17,225 +17,129 @@
 package org.apache.kafka.common.requests;
 
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.message.TxnOffsetCommitRequestData;
+import org.apache.kafka.common.message.TxnOffsetCommitRequestData.TxnOffsetCommitRequestPartition;
+import org.apache.kafka.common.message.TxnOffsetCommitRequestData.TxnOffsetCommitRequestTopic;
+import org.apache.kafka.common.message.TxnOffsetCommitResponseData;
+import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponsePartition;
+import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
-import org.apache.kafka.common.utils.CollectionUtils;
+import org.apache.kafka.common.record.RecordBatch;
 
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
-
-import static org.apache.kafka.common.protocol.CommonFields.COMMITTED_LEADER_EPOCH;
-import static org.apache.kafka.common.protocol.CommonFields.COMMITTED_METADATA;
-import static org.apache.kafka.common.protocol.CommonFields.COMMITTED_OFFSET;
-import static org.apache.kafka.common.protocol.CommonFields.GROUP_ID;
-import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
-import static org.apache.kafka.common.protocol.CommonFields.PRODUCER_EPOCH;
-import static org.apache.kafka.common.protocol.CommonFields.PRODUCER_ID;
-import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
-import static org.apache.kafka.common.protocol.CommonFields.TRANSACTIONAL_ID;
+import java.util.stream.Collectors;
 
 public class TxnOffsetCommitRequest extends AbstractRequest {
-    // top level fields
-    private static final Field.ComplexArray TOPICS = new Field.ComplexArray("topics",
-            "Topics to commit offsets");
-
-    // topic level fields
-    private static final Field.ComplexArray PARTITIONS = new Field.ComplexArray("partitions",
-            "Partitions to commit offsets");
-
-    private static final Field PARTITIONS_V0 = PARTITIONS.withFields(
-            PARTITION_ID,
-            COMMITTED_OFFSET,
-            COMMITTED_METADATA);
-
-    private static final Field TOPICS_V0 = TOPICS.withFields(
-            TOPIC_NAME,
-            PARTITIONS_V0);
-
-    private static final Schema TXN_OFFSET_COMMIT_REQUEST_V0 = new Schema(
-            TRANSACTIONAL_ID,
-            GROUP_ID,
-            PRODUCER_ID,
-            PRODUCER_EPOCH,
-            TOPICS_V0);
-
-    // V1 bump used to indicate that on quota violation brokers send out responses before throttling.
-    private static final Schema TXN_OFFSET_COMMIT_REQUEST_V1 = TXN_OFFSET_COMMIT_REQUEST_V0;
 
-    // V2 adds the leader epoch to the partition data
-    private static final Field PARTITIONS_V2 = PARTITIONS.withFields(
-            PARTITION_ID,
-            COMMITTED_OFFSET,
-            COMMITTED_LEADER_EPOCH,
-            COMMITTED_METADATA);
-
-    private static final Field TOPICS_V2 = TOPICS.withFields(
-            TOPIC_NAME,
-            PARTITIONS_V2);
-
-    private static final Schema TXN_OFFSET_COMMIT_REQUEST_V2 = new Schema(
-            TRANSACTIONAL_ID,
-            GROUP_ID,
-            PRODUCER_ID,
-            PRODUCER_EPOCH,
-            TOPICS_V2);
-
-    public static Schema[] schemaVersions() {
-        return new Schema[]{TXN_OFFSET_COMMIT_REQUEST_V0, TXN_OFFSET_COMMIT_REQUEST_V1, TXN_OFFSET_COMMIT_REQUEST_V2};
-    }
+    public final TxnOffsetCommitRequestData data;
 
     public static class Builder extends AbstractRequest.Builder {
-        private final String transactionalId;
-        private final String consumerGroupId;
-        private final long producerId;
-        private final short producerEpoch;
-        private final Map offsets;
 
-        public Builder(String transactionalId, String consumerGroupId, long producerId, short producerEpoch,
-                       Map offsets) {
-            super(ApiKeys.TXN_OFFSET_COMMIT);
-            this.transactionalId = transactionalId;
-            this.consumerGroupId = consumerGroupId;
-            this.producerId = producerId;
-            this.producerEpoch = producerEpoch;
-            this.offsets = offsets;
-        }
-
-        public String consumerGroupId() {
-            return consumerGroupId;
-        }
+        public final TxnOffsetCommitRequestData data;
 
-        public Map offsets() {
-            return offsets;
+        public Builder(TxnOffsetCommitRequestData data) {
+            super(ApiKeys.TXN_OFFSET_COMMIT);
+            this.data = data;
         }
 
         @Override
         public TxnOffsetCommitRequest build(short version) {
-            return new TxnOffsetCommitRequest(version, transactionalId, consumerGroupId, producerId, producerEpoch, offsets);
+            return new TxnOffsetCommitRequest(data, version);
         }
 
         @Override
         public String toString() {
-            StringBuilder bld = new StringBuilder();
-            bld.append("(type=TxnOffsetCommitRequest").
-                    append(", transactionalId=").append(transactionalId).
-                    append(", producerId=").append(producerId).
-                    append(", producerEpoch=").append(producerEpoch).
-                    append(", consumerGroupId=").append(consumerGroupId).
-                    append(", offsets=").append(offsets).
-                    append(")");
-            return bld.toString();
+            return data.toString();
         }
     }
 
-    private final String transactionalId;
-    private final String consumerGroupId;
-    private final long producerId;
-    private final short producerEpoch;
-    private final Map offsets;
-
-    public TxnOffsetCommitRequest(short version, String transactionalId, String consumerGroupId, long producerId,
-                                  short producerEpoch, Map offsets) {
+    public TxnOffsetCommitRequest(TxnOffsetCommitRequestData data, short version) {
         super(ApiKeys.TXN_OFFSET_COMMIT, version);
-        this.transactionalId = transactionalId;
-        this.consumerGroupId = consumerGroupId;
-        this.producerId = producerId;
-        this.producerEpoch = producerEpoch;
-        this.offsets = offsets;
+        this.data = data;
     }
 
     public TxnOffsetCommitRequest(Struct struct, short version) {
         super(ApiKeys.TXN_OFFSET_COMMIT, version);
-        this.transactionalId = struct.get(TRANSACTIONAL_ID);
-        this.consumerGroupId = struct.get(GROUP_ID);
-        this.producerId = struct.get(PRODUCER_ID);
-        this.producerEpoch = struct.get(PRODUCER_EPOCH);
+        this.data = new TxnOffsetCommitRequestData(struct, version);
+    }
 
-        Map offsets = new HashMap<>();
-        Object[] topicPartitionsArray = struct.get(TOPICS);
-        for (Object topicPartitionObj : topicPartitionsArray) {
-            Struct topicPartitionStruct = (Struct) topicPartitionObj;
-            String topic = topicPartitionStruct.get(TOPIC_NAME);
-            for (Object partitionObj : topicPartitionStruct.get(PARTITIONS)) {
-                Struct partitionStruct = (Struct) partitionObj;
-                TopicPartition partition = new TopicPartition(topic, partitionStruct.get(PARTITION_ID));
-                long offset = partitionStruct.get(COMMITTED_OFFSET);
-                String metadata = partitionStruct.get(COMMITTED_METADATA);
-                Optional leaderEpoch = RequestUtils.getLeaderEpoch(partitionStruct, COMMITTED_LEADER_EPOCH);
-                offsets.put(partition, new CommittedOffset(offset, metadata, leaderEpoch));
+    public Map offsets() {
+        List topics = data.topics();
+        Map offsetMap = new HashMap<>();
+        for (TxnOffsetCommitRequestTopic topic : topics) {
+            for (TxnOffsetCommitRequestPartition partition : topic.partitions()) {
+                offsetMap.put(new TopicPartition(topic.name(), partition.partitionIndex()),
+                              new CommittedOffset(partition.committedOffset(),
+                                                  partition.committedMetadata(),
+                                                  RequestUtils.getLeaderEpoch(partition.committedLeaderEpoch()))
+                );
             }
         }
-        this.offsets = offsets;
+        return offsetMap;
     }
 
-    public String transactionalId() {
-        return transactionalId;
-    }
-
-    public String consumerGroupId() {
-        return consumerGroupId;
-    }
-
-    public long producerId() {
-        return producerId;
-    }
-
-    public short producerEpoch() {
-        return producerEpoch;
-    }
-
-    public Map offsets() {
-        return offsets;
+    public static List getTopics(Map pendingTxnOffsetCommits) {
+        Map> topicPartitionMap = new HashMap<>();
+        for (Map.Entry entry : pendingTxnOffsetCommits.entrySet()) {
+            TopicPartition topicPartition = entry.getKey();
+            CommittedOffset offset = entry.getValue();
+
+            List partitions =
+                topicPartitionMap.getOrDefault(topicPartition.topic(), new ArrayList<>());
+            partitions.add(new TxnOffsetCommitRequestPartition()
+                               .setPartitionIndex(topicPartition.partition())
+                               .setCommittedOffset(offset.offset)
+                               .setCommittedLeaderEpoch(offset.leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH))
+                               .setCommittedMetadata(offset.metadata)
+            );
+            topicPartitionMap.put(topicPartition.topic(), partitions);
+        }
+        return topicPartitionMap.entrySet().stream()
+                   .map(entry -> new TxnOffsetCommitRequestTopic()
+                                     .setName(entry.getKey())
+                                     .setPartitions(entry.getValue()))
+                   .collect(Collectors.toList());
     }
 
     @Override
     protected Struct toStruct() {
-        Struct struct = new Struct(ApiKeys.TXN_OFFSET_COMMIT.requestSchema(version()));
-        struct.set(TRANSACTIONAL_ID, transactionalId);
-        struct.set(GROUP_ID, consumerGroupId);
-        struct.set(PRODUCER_ID, producerId);
-        struct.set(PRODUCER_EPOCH, producerEpoch);
-
-        Map> mappedPartitionOffsets = CollectionUtils.groupPartitionDataByTopic(offsets);
-        Object[] partitionsArray = new Object[mappedPartitionOffsets.size()];
-        int i = 0;
-        for (Map.Entry> topicAndPartitions : mappedPartitionOffsets.entrySet()) {
-            Struct topicPartitionsStruct = struct.instance(TOPICS);
-            topicPartitionsStruct.set(TOPIC_NAME, topicAndPartitions.getKey());
+        return data.toStruct(version());
+    }
 
-            Map partitionOffsets = topicAndPartitions.getValue();
-            Object[] partitionOffsetsArray = new Object[partitionOffsets.size()];
-            int j = 0;
-            for (Map.Entry partitionOffset : partitionOffsets.entrySet()) {
-                Struct partitionOffsetStruct = topicPartitionsStruct.instance(PARTITIONS);
-                partitionOffsetStruct.set(PARTITION_ID, partitionOffset.getKey());
-                CommittedOffset committedOffset = partitionOffset.getValue();
-                partitionOffsetStruct.set(COMMITTED_OFFSET, committedOffset.offset);
-                partitionOffsetStruct.set(COMMITTED_METADATA, committedOffset.metadata);
-                RequestUtils.setLeaderEpochIfExists(partitionOffsetStruct, COMMITTED_LEADER_EPOCH,
-                        committedOffset.leaderEpoch);
-                partitionOffsetsArray[j++] = partitionOffsetStruct;
+    static List getErrorResponseTopics(List requestTopics,
+                                                                     Errors e) {
+        List responseTopicData = new ArrayList<>();
+        for (TxnOffsetCommitRequestTopic entry : requestTopics) {
+            List responsePartitions = new ArrayList<>();
+            for (TxnOffsetCommitRequestPartition requestPartition : entry.partitions()) {
+                responsePartitions.add(new TxnOffsetCommitResponsePartition()
+                                           .setPartitionIndex(requestPartition.partitionIndex())
+                                           .setErrorCode(e.code()));
             }
-            topicPartitionsStruct.set(PARTITIONS, partitionOffsetsArray);
-            partitionsArray[i++] = topicPartitionsStruct;
+            responseTopicData.add(new TxnOffsetCommitResponseTopic()
+                                      .setName(entry.name())
+                                      .setPartitions(responsePartitions)
+            );
         }
-
-        struct.set(TOPICS, partitionsArray);
-        return struct;
+        return responseTopicData;
     }
 
     @Override
     public TxnOffsetCommitResponse getErrorResponse(int throttleTimeMs, Throwable e) {
-        Errors error = Errors.forException(e);
-        Map errors = new HashMap<>(offsets.size());
-        for (TopicPartition partition : offsets.keySet())
-            errors.put(partition, error);
-        return new TxnOffsetCommitResponse(throttleTimeMs, errors);
+        List responseTopicData =
+            getErrorResponseTopics(data.topics(), Errors.forException(e));
+
+        return new TxnOffsetCommitResponse(new TxnOffsetCommitResponseData()
+                                               .setThrottleTimeMs(throttleTimeMs)
+                                               .setTopics(responseTopicData));
     }
 
     public static TxnOffsetCommitRequest parse(ByteBuffer buffer, short version) {
@@ -260,6 +164,22 @@ public String toString() {
                     ", leaderEpoch=" + leaderEpoch +
                     ", metadata='" + metadata + "')";
         }
-    }
 
+        @Override
+        public boolean equals(Object other) {
+            if (!(other instanceof CommittedOffset)) {
+                return false;
+            }
+            CommittedOffset otherOffset = (CommittedOffset) other;
+
+            return this.offset == otherOffset.offset
+                       && this.leaderEpoch.equals(otherOffset.leaderEpoch)
+                       && Objects.equals(this.metadata, otherOffset.metadata);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(offset, leaderEpoch, metadata);
+        }
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java
index ba8f7d6be2eaf..1851f89f831d8 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java
@@ -17,142 +17,98 @@
 package org.apache.kafka.common.requests;
 
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.message.TxnOffsetCommitResponseData;
+import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponsePartition;
+import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
-import org.apache.kafka.common.utils.CollectionUtils;
 
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
-import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
-import static org.apache.kafka.common.protocol.CommonFields.THROTTLE_TIME_MS;
-import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
-
 /**
- *
  * Possible error codes:
- *   InvalidProducerEpoch
- *   NotCoordinator
- *   CoordinatorNotAvailable
- *   CoordinatorLoadInProgress
- *   OffsetMetadataTooLarge
- *   GroupAuthorizationFailed
- *   InvalidCommitOffsetSize
- *   TransactionalIdAuthorizationFailed
- *   RequestTimedOut
+ *
+ *   - {@link Errors#INVALID_PRODUCER_EPOCH}
+ *   - {@link Errors#NOT_COORDINATOR}
+ *   - {@link Errors#COORDINATOR_NOT_AVAILABLE}
+ *   - {@link Errors#COORDINATOR_LOAD_IN_PROGRESS}
+ *   - {@link Errors#OFFSET_METADATA_TOO_LARGE}
+ *   - {@link Errors#GROUP_AUTHORIZATION_FAILED}
+ *   - {@link Errors#INVALID_COMMIT_OFFSET_SIZE}
+ *   - {@link Errors#TRANSACTIONAL_ID_AUTHORIZATION_FAILED}
+ *   - {@link Errors#REQUEST_TIMED_OUT}
  */
 public class TxnOffsetCommitResponse extends AbstractResponse {
-    private static final Field.ComplexArray TOPICS = new Field.ComplexArray("topics",
-            "Responses by topic for committed offsets");
-
-    // topic level fields
-    private static final Field.ComplexArray PARTITIONS = new Field.ComplexArray("partitions",
-            "Responses by partition for committed offsets");
-
-    private static final Field PARTITIONS_V0 = PARTITIONS.withFields(
-            PARTITION_ID,
-            ERROR_CODE);
-
-    private static final Field TOPICS_V0 = TOPICS.withFields(
-            TOPIC_NAME,
-            PARTITIONS_V0);
 
-    private static final Schema TXN_OFFSET_COMMIT_RESPONSE_V0 = new Schema(
-            THROTTLE_TIME_MS,
-            TOPICS_V0);
+    public final TxnOffsetCommitResponseData data;
 
-    // V1 bump used to indicate that on quota violation brokers send out responses before throttling.
-    private static final Schema TXN_OFFSET_COMMIT_RESPONSE_V1 = TXN_OFFSET_COMMIT_RESPONSE_V0;
-
-    // V2 adds the leader epoch to the partition data
-    private static final Schema TXN_OFFSET_COMMIT_RESPONSE_V2 = TXN_OFFSET_COMMIT_RESPONSE_V1;
+    public TxnOffsetCommitResponse(TxnOffsetCommitResponseData data) {
+        this.data = data;
+    }
 
-    public static Schema[] schemaVersions() {
-        return new Schema[]{TXN_OFFSET_COMMIT_RESPONSE_V0, TXN_OFFSET_COMMIT_RESPONSE_V1, TXN_OFFSET_COMMIT_RESPONSE_V2};
+    public TxnOffsetCommitResponse(Struct struct, short version) {
+        this.data = new TxnOffsetCommitResponseData(struct, version);
     }
 
-    private final Map errors;
-    private final int throttleTimeMs;
+    public TxnOffsetCommitResponse(int requestThrottleMs, Map responseData) {
+        Map responseTopicDataMap = new HashMap<>();
 
-    public TxnOffsetCommitResponse(int throttleTimeMs, Map errors) {
-        this.throttleTimeMs = throttleTimeMs;
-        this.errors = errors;
-    }
+        for (Map.Entry entry : responseData.entrySet()) {
+            TopicPartition topicPartition = entry.getKey();
+            String topicName = topicPartition.topic();
 
-    public TxnOffsetCommitResponse(Struct struct) {
-        this.throttleTimeMs = struct.get(THROTTLE_TIME_MS);
-        Map errors = new HashMap<>();
-        Object[] topicPartitionsArray = struct.get(TOPICS);
-        for (Object topicPartitionObj : topicPartitionsArray) {
-            Struct topicPartitionStruct = (Struct) topicPartitionObj;
-            String topic = topicPartitionStruct.get(TOPIC_NAME);
-            for (Object partitionObj : topicPartitionStruct.get(PARTITIONS)) {
-                Struct partitionStruct = (Struct) partitionObj;
-                Integer partition = partitionStruct.get(PARTITION_ID);
-                Errors error = Errors.forCode(partitionStruct.get(ERROR_CODE));
-                errors.put(new TopicPartition(topic, partition), error);
-            }
+            TxnOffsetCommitResponseTopic topic = responseTopicDataMap.getOrDefault(
+                topicName, new TxnOffsetCommitResponseTopic().setName(topicName));
+
+            topic.partitions().add(new TxnOffsetCommitResponsePartition()
+                                       .setErrorCode(entry.getValue().code())
+                                       .setPartitionIndex(topicPartition.partition())
+            );
+            responseTopicDataMap.put(topicName, topic);
         }
-        this.errors = errors;
+
+        data = new TxnOffsetCommitResponseData()
+                   .setTopics(new ArrayList<>(responseTopicDataMap.values()))
+                   .setThrottleTimeMs(requestThrottleMs);
     }
 
     @Override
     protected Struct toStruct(short version) {
-        Struct struct = new Struct(ApiKeys.TXN_OFFSET_COMMIT.responseSchema(version));
-        struct.set(THROTTLE_TIME_MS, throttleTimeMs);
-        Map> mappedPartitions = CollectionUtils.groupPartitionDataByTopic(errors);
-        Object[] partitionsArray = new Object[mappedPartitions.size()];
-        int i = 0;
-        for (Map.Entry> topicAndPartitions : mappedPartitions.entrySet()) {
-            Struct topicPartitionsStruct = struct.instance(TOPICS);
-            topicPartitionsStruct.set(TOPIC_NAME, topicAndPartitions.getKey());
-            Map partitionAndErrors = topicAndPartitions.getValue();
-
-            Object[] partitionAndErrorsArray = new Object[partitionAndErrors.size()];
-            int j = 0;
-            for (Map.Entry partitionAndError : partitionAndErrors.entrySet()) {
-                Struct partitionAndErrorStruct = topicPartitionsStruct.instance(PARTITIONS);
-                partitionAndErrorStruct.set(PARTITION_ID, partitionAndError.getKey());
-                partitionAndErrorStruct.set(ERROR_CODE, partitionAndError.getValue().code());
-                partitionAndErrorsArray[j++] = partitionAndErrorStruct;
-            }
-            topicPartitionsStruct.set(PARTITIONS, partitionAndErrorsArray);
-            partitionsArray[i++] = topicPartitionsStruct;
-        }
-
-        struct.set(TOPICS, partitionsArray);
-        return struct;
+        return data.toStruct(version);
     }
 
     @Override
     public int throttleTimeMs() {
-        return throttleTimeMs;
-    }
-
-    public Map errors() {
-        return errors;
+        return data.throttleTimeMs();
     }
 
     @Override
     public Map errorCounts() {
-        return errorCounts(errors);
+        return errorCounts(errors());
+    }
+
+    public Map errors() {
+        Map errorMap = new HashMap<>();
+        for (TxnOffsetCommitResponseTopic topic : data.topics()) {
+            for (TxnOffsetCommitResponsePartition partition : topic.partitions()) {
+                errorMap.put(new TopicPartition(topic.name(), partition.partitionIndex()),
+                             Errors.forCode(partition.errorCode()));
+            }
+        }
+        return errorMap;
     }
 
     public static TxnOffsetCommitResponse parse(ByteBuffer buffer, short version) {
-        return new TxnOffsetCommitResponse(ApiKeys.TXN_OFFSET_COMMIT.parseResponse(version, buffer));
+        return new TxnOffsetCommitResponse(ApiKeys.TXN_OFFSET_COMMIT.parseResponse(version, buffer), version);
     }
 
     @Override
     public String toString() {
-        return "TxnOffsetCommitResponse(" +
-                "errors=" + errors +
-                ", throttleTimeMs=" + throttleTimeMs +
-                ')';
+        return data.toString();
     }
 
     @Override
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java
index 24869aa7f73ac..3d33a43e60d56 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java
@@ -2826,9 +2826,9 @@ private void prepareTxnOffsetCommitResponse(final String consumerGroupId,
                                                 Map txnOffsetCommitResponse) {
         client.prepareResponse(request -> {
             TxnOffsetCommitRequest txnOffsetCommitRequest = (TxnOffsetCommitRequest) request;
-            assertEquals(consumerGroupId, txnOffsetCommitRequest.consumerGroupId());
-            assertEquals(producerId, txnOffsetCommitRequest.producerId());
-            assertEquals(producerEpoch, txnOffsetCommitRequest.producerEpoch());
+            assertEquals(consumerGroupId, txnOffsetCommitRequest.data.groupId());
+            assertEquals(producerId, txnOffsetCommitRequest.data.producerId());
+            assertEquals(producerEpoch, txnOffsetCommitRequest.data.producerEpoch());
             return true;
         }, new TxnOffsetCommitResponse(0, txnOffsetCommitResponse));
     }
diff --git a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
index 473912ceda345..bf060cdd161ec 100644
--- a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
@@ -22,9 +22,17 @@
 import org.apache.kafka.common.message.AddPartitionsToTxnRequestData.AddPartitionsToTxnTopicCollection;
 import org.apache.kafka.common.message.JoinGroupResponseData.JoinGroupResponseMember;
 import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse;
+import org.apache.kafka.common.message.OffsetCommitRequestData.OffsetCommitRequestPartition;
+import org.apache.kafka.common.message.OffsetCommitRequestData.OffsetCommitRequestTopic;
+import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponsePartition;
+import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponseTopic;
 import org.apache.kafka.common.message.OffsetFetchRequestData.OffsetFetchRequestTopic;
 import org.apache.kafka.common.message.OffsetFetchResponseData.OffsetFetchResponsePartition;
 import org.apache.kafka.common.message.OffsetFetchResponseData.OffsetFetchResponseTopic;
+import org.apache.kafka.common.message.TxnOffsetCommitRequestData.TxnOffsetCommitRequestPartition;
+import org.apache.kafka.common.message.TxnOffsetCommitRequestData.TxnOffsetCommitRequestTopic;
+import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponsePartition;
+import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.ByteBufferAccessor;
 import org.apache.kafka.common.protocol.Errors;
@@ -271,6 +279,176 @@ public void testLeaderAndIsrVersions() throws Exception {
         testAllMessageRoundTripsFromVersion((short) 3, new LeaderAndIsrRequestData().setTopicStates(Collections.singletonList(partitionStateWithAddingRemovingReplicas)));
     }
 
+    @Test
+    public void testOffsetCommitRequestVersions() throws Exception {
+        String groupId = "groupId";
+        String topicName = "topic";
+        String metadata = "metadata";
+        int partition = 2;
+        int offset = 100;
+
+        testAllMessageRoundTrips(new OffsetCommitRequestData()
+                                     .setGroupId(groupId)
+                                     .setTopics(Collections.singletonList(
+                                         new OffsetCommitRequestTopic()
+                                             .setName(topicName)
+                                             .setPartitions(Collections.singletonList(
+                                                 new OffsetCommitRequestPartition()
+                                                     .setPartitionIndex(partition)
+                                                     .setCommittedMetadata(metadata)
+                                                     .setCommittedOffset(offset)
+                                             )))));
+
+        Supplier request =
+            () -> new OffsetCommitRequestData()
+                      .setGroupId(groupId)
+                      .setMemberId("memberId")
+                      .setGroupInstanceId("instanceId")
+                      .setTopics(Collections.singletonList(
+                          new OffsetCommitRequestTopic()
+                              .setName(topicName)
+                              .setPartitions(Collections.singletonList(
+                                  new OffsetCommitRequestPartition()
+                                      .setPartitionIndex(partition)
+                                      .setCommittedLeaderEpoch(10)
+                                      .setCommittedMetadata(metadata)
+                                      .setCommittedOffset(offset)
+                                      .setCommitTimestamp(20)
+                            ))))
+                    .setRetentionTimeMs(20);
+
+        for (short version = 0; version <= ApiKeys.OFFSET_COMMIT.latestVersion(); version++) {
+            OffsetCommitRequestData requestData = request.get();
+            if (version < 1) {
+                requestData.setMemberId("");
+                requestData.setGenerationId(-1);
+            }
+
+            if (version != 1) {
+                requestData.topics().get(0).partitions().get(0).setCommitTimestamp(-1);
+            }
+
+            if (version < 2 || version > 4) {
+                requestData.setRetentionTimeMs(-1);
+            }
+
+            if (version < 6) {
+                requestData.topics().get(0).partitions().get(0).setCommittedLeaderEpoch(-1);
+
+            }
+
+            if (version < 7) {
+                requestData.setGroupInstanceId(null);
+            }
+
+            if (version == 1) {
+                testEquivalentMessageRoundTrip(version, requestData);
+            } else if (version >= 2 && version <= 4) {
+                testAllMessageRoundTripsBetweenVersions(version, (short) 4, requestData, requestData);
+            } else {
+                testAllMessageRoundTripsFromVersion(version, requestData);
+            }
+        }
+    }
+
+    @Test
+    public void testOffsetCommitResponseVersions() throws Exception {
+        Supplier response =
+            () -> new OffsetCommitResponseData()
+                      .setTopics(
+                          singletonList(
+                              new OffsetCommitResponseTopic()
+                                  .setName("topic")
+                                  .setPartitions(singletonList(
+                                      new OffsetCommitResponsePartition()
+                                          .setPartitionIndex(1)
+                                          .setErrorCode(Errors.UNKNOWN_MEMBER_ID.code())
+                                  ))
+                          )
+                      )
+                      .setThrottleTimeMs(20);
+
+        for (short version = 0; version <= ApiKeys.OFFSET_COMMIT.latestVersion(); version++) {
+            OffsetCommitResponseData responseData = response.get();
+            if (version < 3) {
+                responseData.setThrottleTimeMs(0);
+            }
+            testAllMessageRoundTripsFromVersion(version, responseData);
+        }
+    }
+
+    @Test
+    public void testTxnOffsetCommitRequestVersions() throws Exception {
+        String groupId = "groupId";
+        String topicName = "topic";
+        String metadata = "metadata";
+        String txnId = "transactionalId";
+        int producerId = 25;
+        short producerEpoch = 10;
+
+        int partition = 2;
+        int offset = 100;
+
+        testAllMessageRoundTrips(new TxnOffsetCommitRequestData()
+                                     .setGroupId(groupId)
+                                     .setTransactionalId(txnId)
+                                     .setProducerId(producerId)
+                                     .setProducerEpoch(producerEpoch)
+                                     .setTopics(Collections.singletonList(
+                                         new TxnOffsetCommitRequestTopic()
+                                             .setName(topicName)
+                                             .setPartitions(Collections.singletonList(
+                                                 new TxnOffsetCommitRequestPartition()
+                                                     .setPartitionIndex(partition)
+                                                     .setCommittedMetadata(metadata)
+                                                     .setCommittedOffset(offset)
+                                             )))));
+
+        Supplier request =
+            () -> new TxnOffsetCommitRequestData()
+                      .setGroupId(groupId)
+                      .setTransactionalId(txnId)
+                      .setProducerId(producerId)
+                      .setProducerEpoch(producerEpoch)
+                      .setTopics(Collections.singletonList(
+                          new TxnOffsetCommitRequestTopic()
+                              .setName(topicName)
+                              .setPartitions(Collections.singletonList(
+                                  new TxnOffsetCommitRequestPartition()
+                                      .setPartitionIndex(partition)
+                                      .setCommittedLeaderEpoch(10)
+                                      .setCommittedMetadata(metadata)
+                                      .setCommittedOffset(offset)
+                              ))));
+
+        for (short version = 0; version <= ApiKeys.TXN_OFFSET_COMMIT.latestVersion(); version++) {
+            TxnOffsetCommitRequestData requestData = request.get();
+            if (version < 6) {
+                requestData.topics().get(0).partitions().get(0).setCommittedLeaderEpoch(-1);
+            }
+
+            testAllMessageRoundTripsFromVersion(version, requestData);
+        }
+    }
+
+    @Test
+    public void testTxnOffsetCommitResponseVersions() throws Exception {
+        testAllMessageRoundTrips(
+            new TxnOffsetCommitResponseData()
+                .setTopics(
+                   singletonList(
+                       new TxnOffsetCommitResponseTopic()
+                           .setName("topic")
+                           .setPartitions(singletonList(
+                               new TxnOffsetCommitResponsePartition()
+                                   .setPartitionIndex(1)
+                                   .setErrorCode(Errors.UNKNOWN_MEMBER_ID.code())
+                           ))
+                   )
+               )
+               .setThrottleTimeMs(20));
+    }
+
     @Test
     public void testOffsetFetchVersions() throws Exception {
         String groupId = "groupId";
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitRequestTest.java
index 21c3317e7bc1c..c6c5a7bea24c4 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitRequestTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitRequestTest.java
@@ -16,19 +16,131 @@
  */
 package org.apache.kafka.common.requests;
 
+import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.errors.UnsupportedVersionException;
 import org.apache.kafka.common.message.OffsetCommitRequestData;
+import org.apache.kafka.common.message.OffsetCommitRequestData.OffsetCommitRequestPartition;
+import org.apache.kafka.common.message.OffsetCommitRequestData.OffsetCommitRequestTopic;
+import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponsePartition;
+import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponseTopic;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.junit.Before;
 import org.junit.Test;
 
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.kafka.common.requests.AbstractResponse.DEFAULT_THROTTLE_TIME;
+import static org.apache.kafka.common.requests.OffsetCommitRequest.getErrorResponseTopics;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+
 public class OffsetCommitRequestTest {
 
-    @Test(expected = UnsupportedVersionException.class)
-    public void testRequestVersionCompatibilityFailBuild() {
-        new OffsetCommitRequest.Builder(
-                new OffsetCommitRequestData()
-                        .setGroupId("groupId")
-                        .setMemberId("consumerId")
-                        .setGroupInstanceId("groupInstanceId")
-        ).build((short) 6);
+    protected static String groupId = "groupId";
+    protected static String topicOne = "topicOne";
+    protected static String topicTwo = "topicTwo";
+    protected static int partitionOne = 1;
+    protected static int partitionTwo = 2;
+    protected static long offset = 100L;
+    protected static short leaderEpoch = 20;
+    protected static String metadata = "metadata";
+
+    protected static int throttleTimeMs = 10;
+
+    private static OffsetCommitRequestData data;
+    private static List topics;
+
+    @Before
+    public void setUp() {
+        topics = Arrays.asList(
+            new OffsetCommitRequestTopic()
+                .setName(topicOne)
+                .setPartitions(Collections.singletonList(
+                    new OffsetCommitRequestPartition()
+                        .setPartitionIndex(partitionOne)
+                        .setCommittedOffset(offset)
+                        .setCommittedLeaderEpoch(leaderEpoch)
+                        .setCommittedMetadata(metadata)
+                )),
+            new OffsetCommitRequestTopic()
+                .setName(topicTwo)
+                .setPartitions(Collections.singletonList(
+                    new OffsetCommitRequestPartition()
+                        .setPartitionIndex(partitionTwo)
+                        .setCommittedOffset(offset)
+                        .setCommittedLeaderEpoch(leaderEpoch)
+                        .setCommittedMetadata(metadata)
+                ))
+        );
+        data = new OffsetCommitRequestData()
+                   .setGroupId(groupId)
+                   .setTopics(topics);
+    }
+
+    @Test
+    public void testConstructor() {
+        Map expectedOffsets = new HashMap<>();
+        expectedOffsets.put(new TopicPartition(topicOne, partitionOne), offset);
+        expectedOffsets.put(new TopicPartition(topicTwo, partitionTwo), offset);
+
+        OffsetCommitRequest.Builder builder = new OffsetCommitRequest.Builder(data);
+
+        for (short version = 0; version <= ApiKeys.TXN_OFFSET_COMMIT.latestVersion(); version++) {
+            OffsetCommitRequest request = builder.build(version);
+            assertEquals(expectedOffsets, request.offsets());
+
+            OffsetCommitResponse response = request.getErrorResponse(throttleTimeMs, Errors.NOT_COORDINATOR.exception());
+
+            assertEquals(Collections.singletonMap(Errors.NOT_COORDINATOR, 2), response.errorCounts());
+
+            if (version >= 3) {
+                assertEquals(throttleTimeMs, response.throttleTimeMs());
+            } else {
+                assertEquals(DEFAULT_THROTTLE_TIME, response.throttleTimeMs());
+            }
+        }
+    }
+
+    @Test
+    public void testGetErrorResponseTopics() {
+        List expectedTopics = Arrays.asList(
+            new OffsetCommitResponseTopic()
+                .setName(topicOne)
+                .setPartitions(Collections.singletonList(
+                    new OffsetCommitResponsePartition()
+                        .setErrorCode(Errors.UNKNOWN_MEMBER_ID.code())
+                        .setPartitionIndex(partitionOne))),
+            new OffsetCommitResponseTopic()
+                .setName(topicTwo)
+                .setPartitions(Collections.singletonList(
+                    new OffsetCommitResponsePartition()
+                        .setErrorCode(Errors.UNKNOWN_MEMBER_ID.code())
+                        .setPartitionIndex(partitionTwo)))
+        );
+        assertEquals(expectedTopics, getErrorResponseTopics(topics, Errors.UNKNOWN_MEMBER_ID));
+    }
+
+    @Test
+    public void testVersionSupportForGroupInstanceId() {
+        OffsetCommitRequest.Builder builder = new OffsetCommitRequest.Builder(
+            new OffsetCommitRequestData()
+                .setGroupId("groupId")
+                .setMemberId("consumerId")
+                .setGroupInstanceId("groupInstanceId")
+        );
+
+        for (short version = 0; version <= ApiKeys.OFFSET_COMMIT.latestVersion(); version++) {
+            if (version >= 7) {
+                builder.build(version);
+            } else {
+                final short finalVersion = version;
+                assertThrows(UnsupportedVersionException.class, () -> builder.build(finalVersion));
+            }
+        }
     }
 }
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java
new file mode 100644
index 0000000000000..62229168b2405
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.requests;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.message.OffsetCommitResponseData;
+import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponsePartition;
+import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponseTopic;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.kafka.common.requests.AbstractResponse.DEFAULT_THROTTLE_TIME;
+import static org.junit.Assert.assertEquals;
+
+public class OffsetCommitResponseTest {
+
+    protected final int throttleTimeMs = 10;
+
+    protected final String topicOne = "topic1";
+    protected final int partitionOne = 1;
+    protected final Errors errorOne = Errors.COORDINATOR_NOT_AVAILABLE;
+    protected final Errors errorTwo = Errors.NOT_COORDINATOR;
+    protected final String topicTwo = "topic2";
+    protected final int partitionTwo = 2;
+
+    protected TopicPartition tp1 = new TopicPartition(topicOne, partitionOne);
+    protected TopicPartition tp2 = new TopicPartition(topicTwo, partitionTwo);
+    protected Map expectedErrorCounts;
+    protected Map errorsMap;
+
+    @Before
+    public void setUp() {
+        expectedErrorCounts = new HashMap<>();
+        expectedErrorCounts.put(errorOne, 1);
+        expectedErrorCounts.put(errorTwo, 1);
+
+        errorsMap = new HashMap<>();
+        errorsMap.put(tp1, errorOne);
+        errorsMap.put(tp2, errorTwo);
+    }
+
+    @Test
+    public void testConstructorWithErrorResponse() {
+        OffsetCommitResponse response = new OffsetCommitResponse(throttleTimeMs, errorsMap);
+
+        assertEquals(expectedErrorCounts, response.errorCounts());
+        assertEquals(throttleTimeMs, response.throttleTimeMs());
+    }
+
+    @Test
+    public void testConstructorWithStruct() {
+        OffsetCommitResponseData data = new OffsetCommitResponseData()
+                                            .setTopics(Arrays.asList(
+                                                new OffsetCommitResponseTopic().setPartitions(
+                                                    Collections.singletonList(new OffsetCommitResponsePartition()
+                                                                                  .setPartitionIndex(partitionOne)
+                                                                                  .setErrorCode(errorOne.code()))),
+                                                new OffsetCommitResponseTopic().setPartitions(
+                                                    Collections.singletonList(new OffsetCommitResponsePartition()
+                                                                                  .setPartitionIndex(partitionTwo)
+                                                                                  .setErrorCode(errorTwo.code())))
+                                            ))
+                                            .setThrottleTimeMs(throttleTimeMs);
+
+        for (short version = 0; version <= ApiKeys.OFFSET_COMMIT.latestVersion(); version++) {
+            OffsetCommitResponse response = new OffsetCommitResponse(data.toStruct(version), version);
+            assertEquals(expectedErrorCounts, response.errorCounts());
+
+            if (version >= 3) {
+                assertEquals(throttleTimeMs, response.throttleTimeMs());
+            } else {
+                assertEquals(DEFAULT_THROTTLE_TIME, response.throttleTimeMs());
+            }
+
+            assertEquals(version >= 4, response.shouldClientThrottle(version));
+        }
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index 8645d795b8463..ddf574937e3fb 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -88,6 +88,7 @@
 import org.apache.kafka.common.message.SaslAuthenticateResponseData;
 import org.apache.kafka.common.message.SaslHandshakeRequestData;
 import org.apache.kafka.common.message.SaslHandshakeResponseData;
+import org.apache.kafka.common.message.TxnOffsetCommitRequestData;
 import org.apache.kafka.common.network.ListenerName;
 import org.apache.kafka.common.network.Send;
 import org.apache.kafka.common.protocol.ApiKeys;
@@ -1384,7 +1385,14 @@ private TxnOffsetCommitRequest createTxnOffsetCommitRequest() {
                     new TxnOffsetCommitRequest.CommittedOffset(100, null, Optional.empty()));
         offsets.put(new TopicPartition("topic", 74),
                 new TxnOffsetCommitRequest.CommittedOffset(100, "blah", Optional.of(27)));
-        return new TxnOffsetCommitRequest.Builder("transactionalId", "groupId", 21L, (short) 42, offsets).build();
+        return new TxnOffsetCommitRequest.Builder(
+            new TxnOffsetCommitRequestData()
+                .setTransactionalId("transactionalId")
+                .setGroupId("groupId")
+                .setProducerId(21L)
+                .setProducerEpoch((short) 42)
+                .setTopics(TxnOffsetCommitRequest.getTopics(offsets))
+            ).build();
     }
 
     private TxnOffsetCommitResponse createTxnOffsetCommitResponse() {
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitRequestTest.java
new file mode 100644
index 0000000000000..ff1d1ab872b6c
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitRequestTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.requests;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.message.TxnOffsetCommitRequestData;
+import org.apache.kafka.common.message.TxnOffsetCommitRequestData.TxnOffsetCommitRequestPartition;
+import org.apache.kafka.common.message.TxnOffsetCommitRequestData.TxnOffsetCommitRequestTopic;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.TxnOffsetCommitRequest.CommittedOffset;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.junit.Assert.assertEquals;
+
+public class TxnOffsetCommitRequestTest extends OffsetCommitRequestTest {
+
+    private static String transactionalId = "transactionalId";
+    private static int producerId = 10;
+    private static short producerEpoch = 1;
+
+    private static TxnOffsetCommitRequestData data;
+
+    @Before
+    @Override
+    public void setUp() {
+        super.setUp();
+        data = new TxnOffsetCommitRequestData()
+                   .setGroupId(groupId)
+                   .setTransactionalId(transactionalId)
+                   .setProducerId(producerId)
+                   .setProducerEpoch(producerEpoch)
+                   .setTopics(Arrays.asList(
+                       new TxnOffsetCommitRequestTopic()
+                           .setName(topicOne)
+                           .setPartitions(Collections.singletonList(
+                               new TxnOffsetCommitRequestPartition()
+                                   .setPartitionIndex(partitionOne)
+                                   .setCommittedOffset(offset)
+                                   .setCommittedLeaderEpoch(leaderEpoch)
+                                   .setCommittedMetadata(metadata)
+                           )),
+                       new TxnOffsetCommitRequestTopic()
+                           .setName(topicTwo)
+                           .setPartitions(Collections.singletonList(
+                               new TxnOffsetCommitRequestPartition()
+                                   .setPartitionIndex(partitionTwo)
+                                   .setCommittedOffset(offset)
+                                   .setCommittedLeaderEpoch(leaderEpoch)
+                                   .setCommittedMetadata(metadata)
+                           ))
+                   ));
+    }
+
+    @Test
+    @Override
+    public void testConstructor() {
+        Map expectedOffsets = new HashMap<>();
+        expectedOffsets.put(new TopicPartition(topicOne, partitionOne),
+                            new CommittedOffset(
+                                offset,
+                                metadata,
+                                Optional.of((int) leaderEpoch)));
+        expectedOffsets.put(new TopicPartition(topicTwo, partitionTwo),
+                            new CommittedOffset(
+                                offset,
+                                metadata,
+                                Optional.of((int) leaderEpoch)));
+
+        TxnOffsetCommitRequest.Builder builder = new TxnOffsetCommitRequest.Builder(data);
+        Map errorsMap = new HashMap<>();
+        errorsMap.put(new TopicPartition(topicOne, partitionOne), Errors.NOT_COORDINATOR);
+        errorsMap.put(new TopicPartition(topicTwo, partitionTwo), Errors.NOT_COORDINATOR);
+
+        for (short version = 0; version <= ApiKeys.TXN_OFFSET_COMMIT.latestVersion(); version++) {
+            TxnOffsetCommitRequest request = builder.build(version);
+            assertEquals(expectedOffsets, request.offsets());
+            assertEquals(data.topics(), TxnOffsetCommitRequest.getTopics(request.offsets()));
+
+            TxnOffsetCommitResponse response =
+                request.getErrorResponse(throttleTimeMs, Errors.NOT_COORDINATOR.exception());
+
+            assertEquals(errorsMap, response.errors());
+            assertEquals(Collections.singletonMap(Errors.NOT_COORDINATOR, 2), response.errorCounts());
+            assertEquals(throttleTimeMs, response.throttleTimeMs());
+        }
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java
new file mode 100644
index 0000000000000..be4f5df359a05
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.requests;
+
+import org.apache.kafka.common.message.TxnOffsetCommitResponseData;
+import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponsePartition;
+import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic;
+import org.apache.kafka.common.protocol.ApiKeys;
+
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+
+import static org.junit.Assert.assertEquals;
+
+public class TxnOffsetCommitResponseTest extends OffsetCommitResponseTest {
+
+    @Test
+    @Override
+    public void testConstructorWithErrorResponse() {
+        TxnOffsetCommitResponse response = new TxnOffsetCommitResponse(throttleTimeMs, errorsMap);
+
+        assertEquals(errorsMap, response.errors());
+        assertEquals(expectedErrorCounts, response.errorCounts());
+        assertEquals(throttleTimeMs, response.throttleTimeMs());
+    }
+
+    @Test
+    @Override
+    public void testConstructorWithStruct() {
+        TxnOffsetCommitResponseData data = new TxnOffsetCommitResponseData()
+            .setThrottleTimeMs(throttleTimeMs)
+            .setTopics(Arrays.asList(
+                new TxnOffsetCommitResponseTopic().setPartitions(
+                    Collections.singletonList(new TxnOffsetCommitResponsePartition()
+                                                  .setPartitionIndex(partitionOne)
+                                                  .setErrorCode(errorOne.code()))),
+                new TxnOffsetCommitResponseTopic().setPartitions(
+                    Collections.singletonList(new TxnOffsetCommitResponsePartition()
+                                                  .setPartitionIndex(partitionTwo)
+                                                  .setErrorCode(errorTwo.code()))
+                    )
+            ));
+
+        for (short version = 0; version <= ApiKeys.TXN_OFFSET_COMMIT.latestVersion(); version++) {
+            TxnOffsetCommitResponse response = new TxnOffsetCommitResponse(data.toStruct(version), version);
+            assertEquals(expectedErrorCounts, response.errorCounts());
+            assertEquals(throttleTimeMs, response.throttleTimeMs());
+            assertEquals(version >= 1, response.shouldClientThrottle(version));
+        }
+    }
+}
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index fd8fa4de428af..ac68f1a8f0ded 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -422,11 +422,16 @@ class KafkaApis(val requestChannel: RequestChannel,
           val metadata = if (partitionData.committedMetadata() == null)
             OffsetAndMetadata.NoMetadata
           else
-            partitionData.committedMetadata()
+            partitionData.committedMetadata
+
+          val leaderEpochOpt = if (partitionData.committedLeaderEpoch == RecordBatch.NO_PARTITION_LEADER_EPOCH)
+            Optional.empty[Integer]
+          else
+            Optional.of[Integer](partitionData.committedLeaderEpoch)
 
           k -> new OffsetAndMetadata(
             offset = partitionData.committedOffset(),
-            leaderEpoch = Optional.ofNullable[Integer](partitionData.committedLeaderEpoch),
+            leaderEpoch = leaderEpochOpt,
             metadata = metadata,
             commitTimestamp = partitionData.commitTimestamp() match {
               case OffsetCommitRequest.DEFAULT_TIMESTAMP => currentTimestamp
@@ -2087,9 +2092,9 @@ class KafkaApis(val requestChannel: RequestChannel,
 
     // authorize for the transactionalId and the consumer group. Note that we skip producerId authorization
     // since it is implied by transactionalId authorization
-    if (!authorize(request, WRITE, TRANSACTIONAL_ID, txnOffsetCommitRequest.transactionalId))
+    if (!authorize(request, WRITE, TRANSACTIONAL_ID, txnOffsetCommitRequest.data.transactionalId))
       sendErrorResponseMaybeThrottle(request, Errors.TRANSACTIONAL_ID_AUTHORIZATION_FAILED.exception)
-    else if (!authorize(request, READ, GROUP, txnOffsetCommitRequest.consumerGroupId))
+    else if (!authorize(request, READ, GROUP, txnOffsetCommitRequest.data.groupId))
       sendErrorResponseMaybeThrottle(request, Errors.GROUP_AUTHORIZATION_FAILED.exception)
     else {
       val unauthorizedTopicErrors = mutable.Map[TopicPartition, Errors]()
@@ -2125,9 +2130,9 @@ class KafkaApis(val requestChannel: RequestChannel,
       else {
         val offsetMetadata = convertTxnOffsets(authorizedTopicCommittedOffsets.toMap)
         groupCoordinator.handleTxnCommitOffsets(
-          txnOffsetCommitRequest.consumerGroupId,
-          txnOffsetCommitRequest.producerId,
-          txnOffsetCommitRequest.producerEpoch,
+          txnOffsetCommitRequest.data.groupId,
+          txnOffsetCommitRequest.data.producerId,
+          txnOffsetCommitRequest.data.producerEpoch,
           offsetMetadata,
           sendResponseCallback)
       }
diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
index 1e0f9aa15b247..06fc534d2b78a 100644
--- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
@@ -47,7 +47,7 @@ import org.apache.kafka.common.requests.{FetchMetadata => JFetchMetadata, _}
 import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
 import org.easymock.{Capture, EasyMock, IAnswer}
 import EasyMock._
-import org.apache.kafka.common.message.{HeartbeatRequestData, JoinGroupRequestData, OffsetCommitRequestData, OffsetCommitResponseData, SyncGroupRequestData}
+import org.apache.kafka.common.message.{HeartbeatRequestData, JoinGroupRequestData, OffsetCommitRequestData, OffsetCommitResponseData, SyncGroupRequestData, TxnOffsetCommitRequestData}
 import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocol
 import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
 import org.apache.kafka.common.replica.ClientMetadata
@@ -161,8 +161,15 @@ class KafkaApisTest {
 
       val invalidTopicPartition = new TopicPartition(topic, invalidPartitionId)
       val partitionOffsetCommitData = new TxnOffsetCommitRequest.CommittedOffset(15L, "", Optional.empty())
-      val (offsetCommitRequest, request) = buildRequest(new TxnOffsetCommitRequest.Builder("txnlId", "groupId",
-        15L, 0.toShort, Map(invalidTopicPartition -> partitionOffsetCommitData).asJava))
+      val (offsetCommitRequest, request) = buildRequest(new TxnOffsetCommitRequest.Builder(
+        new TxnOffsetCommitRequestData()
+          .setTransactionalId("txnlId")
+          .setGroupId("groupId")
+          .setProducerId(15L)
+          .setProducerEpoch(0.toShort)
+          .setTopics(TxnOffsetCommitRequest.getTopics(
+            Map(invalidTopicPartition -> partitionOffsetCommitData).asJava))
+      ))
 
       val capturedResponse = expectNoThrottling()
       EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel)
diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
index f302105f97724..4a93ef9f67a0f 100644
--- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
+++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
@@ -379,8 +379,16 @@ class RequestQuotaTest extends BaseRequestTest {
           new WriteTxnMarkersRequest.Builder(List.empty.asJava)
 
         case ApiKeys.TXN_OFFSET_COMMIT =>
-          new TxnOffsetCommitRequest.Builder("test-transactional-id", "test-txn-group", 2, 0,
-            Map.empty[TopicPartition, TxnOffsetCommitRequest.CommittedOffset].asJava)
+          new TxnOffsetCommitRequest.Builder(
+            new TxnOffsetCommitRequestData()
+              .setTransactionalId("test-transactional-id")
+              .setGroupId("test-txn-group")
+              .setProducerId(2)
+              .setProducerEpoch(0)
+              .setTopics(TxnOffsetCommitRequest.getTopics(
+                Map.empty[TopicPartition, TxnOffsetCommitRequest.CommittedOffset].asJava
+              ))
+          )
 
         case ApiKeys.DESCRIBE_ACLS =>
           new DescribeAclsRequest.Builder(AclBindingFilter.ANY)
@@ -550,7 +558,7 @@ class RequestQuotaTest extends BaseRequestTest {
       case ApiKeys.ADD_PARTITIONS_TO_TXN => new AddPartitionsToTxnResponse(response).throttleTimeMs
       case ApiKeys.ADD_OFFSETS_TO_TXN => new AddOffsetsToTxnResponse(response).throttleTimeMs
       case ApiKeys.END_TXN => new EndTxnResponse(response).throttleTimeMs
-      case ApiKeys.TXN_OFFSET_COMMIT => new TxnOffsetCommitResponse(response).throttleTimeMs
+      case ApiKeys.TXN_OFFSET_COMMIT => new TxnOffsetCommitResponse(response, ApiKeys.TXN_OFFSET_COMMIT.latestVersion).throttleTimeMs
       case ApiKeys.DESCRIBE_ACLS => new DescribeAclsResponse(response).throttleTimeMs
       case ApiKeys.CREATE_ACLS => new CreateAclsResponse(response).throttleTimeMs
       case ApiKeys.DELETE_ACLS => new DeleteAclsResponse(response).throttleTimeMs

From c65e19ae95549f7dc49eec3e5fd3cc2de84687d9 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Fri, 6 Sep 2019 00:54:53 -0700
Subject: [PATCH 0593/1071] MINOR: Use toString in Records implementations only
 for summary metadata (#7290)

This patch fixes a couple problems with logging of request/response objects which include records data. First, it adds a missing `toString` to `LazyDownConversionRecords`. Second, it changes the `toString` of `MemoryRecords` to not print record-level information. This was always a dangerous practice, but it was especially bad when these objects ended up in request logs. With this patch, implementations use `toString` only to print summary details.

Reviewers: Guozhang Wang 
---
 .../kafka/common/record/FileRecords.java      |  3 +-
 .../record/LazyDownConversionRecords.java     |  9 ++++
 .../kafka/common/record/MemoryRecords.java    | 33 ++-------------
 .../common/record/MemoryRecordsTest.java      | 35 +---------------
 .../record/SimpleMemoryRecordsTest.java       | 41 -------------------
 5 files changed, 15 insertions(+), 106 deletions(-)
 delete mode 100644 clients/src/test/java/org/apache/kafka/common/record/SimpleMemoryRecordsTest.java

diff --git a/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java b/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java
index a02fb4135b526..9b312d9e7f73b 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java
@@ -370,7 +370,8 @@ public Iterable batches() {
 
     @Override
     public String toString() {
-        return "FileRecords(file= " + file +
+        return "FileRecords(size=" + sizeInBytes() +
+                ", file=" + file +
                 ", start=" + start +
                 ", end=" + end +
                 ")";
diff --git a/clients/src/main/java/org/apache/kafka/common/record/LazyDownConversionRecords.java b/clients/src/main/java/org/apache/kafka/common/record/LazyDownConversionRecords.java
index 113e1f2efe316..56c6809b45463 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/LazyDownConversionRecords.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/LazyDownConversionRecords.java
@@ -106,6 +106,15 @@ public int hashCode() {
         return result;
     }
 
+    @Override
+    public String toString() {
+        return "LazyDownConversionRecords(size=" + sizeInBytes +
+                ", underlying=" + records +
+                ", toMagic=" + toMagic +
+                ", firstOffset=" + firstOffset +
+                ")";
+    }
+
     public java.util.Iterator> iterator(long maximumReadSize) {
         // We typically expect only one iterator instance to be created, so null out the first converted batch after
         // first use to make it available for GC.
diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java
index c6db18d419162..8f73565d1b40f 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.common.record;
 
-import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.errors.CorruptRecordException;
 import org.apache.kafka.common.record.MemoryRecords.RecordFilter.BatchRetention;
@@ -32,7 +31,6 @@
 import java.nio.channels.GatheringByteChannel;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
 
@@ -279,34 +277,9 @@ public Iterable batches() {
 
     @Override
     public String toString() {
-        StringBuilder builder = new StringBuilder();
-        builder.append('[');
-
-        Iterator batchIterator = batches.iterator();
-        while (batchIterator.hasNext()) {
-            RecordBatch batch = batchIterator.next();
-            try (CloseableIterator recordsIterator = batch.streamingIterator(BufferSupplier.create())) {
-                while (recordsIterator.hasNext()) {
-                    Record record = recordsIterator.next();
-                    appendRecordToStringBuilder(builder, record.toString());
-                    if (recordsIterator.hasNext())
-                        builder.append(", ");
-                }
-            } catch (KafkaException e) {
-                appendRecordToStringBuilder(builder, "CORRUPTED");
-            }
-            if (batchIterator.hasNext())
-                builder.append(", ");
-        }
-        builder.append(']');
-        return builder.toString();
-    }
-
-    private void appendRecordToStringBuilder(StringBuilder builder, String recordAsString) {
-        builder.append('(')
-            .append("record=")
-            .append(recordAsString)
-            .append(")");
+        return "MemoryRecords(size=" + sizeInBytes() +
+                ", buffer=" + buffer +
+                ")";
     }
 
     @Override
diff --git a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsTest.java b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsTest.java
index 3d5a4f1a498b3..b8824d3a8276c 100644
--- a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsTest.java
@@ -30,8 +30,8 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.function.Supplier;
 import java.util.List;
+import java.util.function.Supplier;
 
 import static java.util.Arrays.asList;
 import static org.apache.kafka.common.record.RecordBatch.MAGIC_VALUE_V2;
@@ -41,7 +41,6 @@
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 import static org.junit.Assume.assumeTrue;
 
 @RunWith(value = Parameterized.class)
@@ -688,38 +687,6 @@ public void testFilterToWithUndersizedBuffer() {
             assertNotNull(record.key());
     }
 
-    @Test
-    public void testToString() {
-        assumeAtLeastV2OrNotZstd();
-
-        long timestamp = 1000000;
-        MemoryRecords memoryRecords = MemoryRecords.withRecords(magic, compression,
-                new SimpleRecord(timestamp, "key1".getBytes(), "value1".getBytes()),
-                new SimpleRecord(timestamp + 1, "key2".getBytes(), "value2".getBytes()));
-        switch (magic) {
-            case RecordBatch.MAGIC_VALUE_V0:
-                assertEquals("[(record=LegacyRecordBatch(offset=0, Record(magic=0, attributes=0, compression=NONE, " +
-                                "crc=1978725405, key=4 bytes, value=6 bytes))), (record=LegacyRecordBatch(offset=1, Record(magic=0, " +
-                                "attributes=0, compression=NONE, crc=1964753830, key=4 bytes, value=6 bytes)))]",
-                        memoryRecords.toString());
-                break;
-            case RecordBatch.MAGIC_VALUE_V1:
-                assertEquals("[(record=LegacyRecordBatch(offset=0, Record(magic=1, attributes=0, compression=NONE, " +
-                                "crc=97210616, CreateTime=1000000, key=4 bytes, value=6 bytes))), (record=LegacyRecordBatch(offset=1, " +
-                                "Record(magic=1, attributes=0, compression=NONE, crc=3535988507, CreateTime=1000001, key=4 bytes, " +
-                                "value=6 bytes)))]",
-                        memoryRecords.toString());
-                break;
-            case RecordBatch.MAGIC_VALUE_V2:
-                assertEquals("[(record=DefaultRecord(offset=0, timestamp=1000000, key=4 bytes, value=6 bytes)), " +
-                                "(record=DefaultRecord(offset=1, timestamp=1000001, key=4 bytes, value=6 bytes))]",
-                        memoryRecords.toString());
-                break;
-            default:
-                fail("Unexpected magic " + magic);
-        }
-    }
-
     @Test
     public void testFilterTo() {
         assumeAtLeastV2OrNotZstd();
diff --git a/clients/src/test/java/org/apache/kafka/common/record/SimpleMemoryRecordsTest.java b/clients/src/test/java/org/apache/kafka/common/record/SimpleMemoryRecordsTest.java
deleted file mode 100644
index 2909566d90efd..0000000000000
--- a/clients/src/test/java/org/apache/kafka/common/record/SimpleMemoryRecordsTest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kafka.common.record;
-
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Non-parameterized MemoryRecords tests.
- */
-public class SimpleMemoryRecordsTest {
-
-    @Test
-    public void testToStringIfLz4ChecksumIsCorrupted() {
-        long timestamp = 1000000;
-        MemoryRecords memoryRecords = MemoryRecords.withRecords(CompressionType.LZ4,
-                new SimpleRecord(timestamp, "key1".getBytes(), "value1".getBytes()),
-                new SimpleRecord(timestamp + 1, "key2".getBytes(), "value2".getBytes()));
-        // Change the lz4 checksum value (not the kafka record crc) so that it doesn't match the contents
-        int lz4ChecksumOffset = 6;
-        memoryRecords.buffer().array()[DefaultRecordBatch.RECORD_BATCH_OVERHEAD + lz4ChecksumOffset] = 0;
-        assertEquals("[(record=CORRUPTED)]", memoryRecords.toString());
-    }
-
-}

From 7b62248c6219937727e059cd4ecf020d9188894e Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Fri, 6 Sep 2019 09:44:15 -0700
Subject: [PATCH 0594/1071] MINOR: Add cause to thrown exception when deleting
 topic in TopicCommand (#7301)

Unexpected exceptions are caught during topic deletion in `TopicCommand`. The caught exception is currently lost and we raise `AdminOperationException`. This patch fixes the problem by chaining the caught exception.

Reviewers: Guozhang Wang 
---
 core/src/main/scala/kafka/admin/TopicCommand.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/kafka/admin/TopicCommand.scala b/core/src/main/scala/kafka/admin/TopicCommand.scala
index 986e555cc83e5..9123d49295343 100755
--- a/core/src/main/scala/kafka/admin/TopicCommand.scala
+++ b/core/src/main/scala/kafka/admin/TopicCommand.scala
@@ -453,8 +453,8 @@ object TopicCommand extends Logging {
             println(s"Topic $topic is already marked for deletion.")
           case e: AdminOperationException =>
             throw e
-          case _: Throwable =>
-            throw new AdminOperationException(s"Error while deleting topic $topic")
+          case e: Throwable =>
+            throw new AdminOperationException(s"Error while deleting topic $topic", e)
         }
       }
     }

From 0f177ea6b826cd96f7a4a687da3691fb786a0a89 Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Sun, 8 Sep 2019 19:06:06 -0500
Subject: [PATCH 0595/1071] MINOR: Clean up partition assignment logic (#7249)

These are just some "tidying up" changes I made when I was preparing to start working on KIP-441.

Reviewers: Guozhang Wang 
---
 .../streams/internals/QuietStreamsConfig.java |   0
 .../internals/InternalTopicConfig.java        |  10 +-
 .../internals/InternalTopicManager.java       |  13 +-
 .../processor/internals/StreamThread.java     |   9 +-
 .../internals/StreamsPartitionAssignor.java   | 548 +++++++-----------
 .../processor/internals/TaskManager.java      |   6 +-
 .../internals/assignment/AssignmentInfo.java  |   7 +-
 .../assignment/AssignorConfiguration.java     | 188 ++++++
 .../internals/assignment/AssignorError.java   |  34 ++
 .../CopartitionedTopicsEnforcer.java          | 110 ++++
 .../StreamsAssignmentProtocolVersions.java    |  30 +
 .../assignment/SubscriptionInfo.java          |   7 +-
 ...a => CopartitionedTopicsEnforcerTest.java} |  43 +-
 .../StreamsPartitionAssignorTest.java         | 174 +++---
 .../assignment/AssignmentInfoTest.java        |  16 +-
 .../assignment/SubscriptionInfoTest.java      |  22 +-
 .../streams/tests/StreamsUpgradeTest.java     |  35 +-
 .../kafka/test/MockInternalTopicManager.java  |   2 +-
 18 files changed, 749 insertions(+), 505 deletions(-)
 rename streams/{test-utils => }/src/main/java/org/apache/kafka/streams/internals/QuietStreamsConfig.java (100%)
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorConfiguration.java
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorError.java
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/StreamsAssignmentProtocolVersions.java
 rename streams/src/test/java/org/apache/kafka/streams/processor/internals/{CopartitionedTopicsValidatorTest.java => CopartitionedTopicsEnforcerTest.java} (73%)

diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/internals/QuietStreamsConfig.java b/streams/src/main/java/org/apache/kafka/streams/internals/QuietStreamsConfig.java
similarity index 100%
rename from streams/test-utils/src/main/java/org/apache/kafka/streams/internals/QuietStreamsConfig.java
rename to streams/src/main/java/org/apache/kafka/streams/internals/QuietStreamsConfig.java
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java
index 9005311c7af31..bfe6a73f75eb7 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java
@@ -20,17 +20,17 @@
 
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * InternalTopicConfig captures the properties required for configuring
  * the internal topics we create for change-logs and repartitioning etc.
  */
 public abstract class InternalTopicConfig {
-
     final String name;
     final Map topicConfigs;
 
-    private int numberOfPartitions = StreamsPartitionAssignor.UNKNOWN;
+    private Optional numberOfPartitions = Optional.empty();
 
     InternalTopicConfig(final String name, final Map topicConfigs) {
         Objects.requireNonNull(name, "name can't be null");
@@ -53,15 +53,15 @@ public String name() {
         return name;
     }
 
-    public int numberOfPartitions() {
+    public Optional numberOfPartitions() {
         return numberOfPartitions;
     }
 
-    void setNumberOfPartitions(final int numberOfPartitions) {
+    public void setNumberOfPartitions(final int numberOfPartitions) {
         if (numberOfPartitions < 1) {
             throw new IllegalArgumentException("Number of partitions must be at least 1.");
         }
-        this.numberOfPartitions = numberOfPartitions;
+        this.numberOfPartitions = Optional.of(numberOfPartitions);
     }
 
     @Override
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java
index ffd744bc7df02..621ce7e9094d5 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java
@@ -36,6 +36,7 @@
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 
@@ -104,7 +105,7 @@ public void makeReady(final Map topics) {
         while (!topicsNotReady.isEmpty() && remainingRetries >= 0) {
             topicsNotReady = validateTopics(topicsNotReady, topics);
 
-            if (topicsNotReady.size() > 0) {
+            if (!topicsNotReady.isEmpty()) {
                 final Set newTopics = new HashSet<>();
 
                 for (final String topicName : topicsNotReady) {
@@ -120,7 +121,7 @@ public void makeReady(final Map topics) {
                         new NewTopic(
                             internalTopicConfig.name(),
                             internalTopicConfig.numberOfPartitions(),
-                            replicationFactor)
+                            Optional.of(replicationFactor))
                             .configs(topicConfig));
                 }
 
@@ -224,13 +225,13 @@ private Set validateTopics(final Set topicsToValidate,
         final Set topicsToCreate = new HashSet<>();
         for (final Map.Entry entry : topicsMap.entrySet()) {
             final String topicName = entry.getKey();
-            final int numberOfPartitions = entry.getValue().numberOfPartitions();
-            if (existedTopicPartition.containsKey(topicName)) {
-                if (!existedTopicPartition.get(topicName).equals(numberOfPartitions)) {
+            final Optional numberOfPartitions = entry.getValue().numberOfPartitions();
+            if (existedTopicPartition.containsKey(topicName) && numberOfPartitions.isPresent()) {
+                if (!existedTopicPartition.get(topicName).equals(numberOfPartitions.get())) {
                     final String errorMsg = String.format("Existing internal topic %s has invalid partitions: " +
                             "expected: %d; actual: %d. " +
                             "Use 'kafka.tools.StreamsResetter' tool to clean up invalid topics before processing.",
-                        topicName, numberOfPartitions, existedTopicPartition.get(topicName));
+                        topicName, numberOfPartitions.get(), existedTopicPartition.get(topicName));
                     log.error(errorMsg);
                     throw new StreamsException(errorMsg);
                 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index 5fa3a33c54913..1c88054d23247 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -42,6 +42,7 @@
 import org.apache.kafka.streams.processor.TaskId;
 import org.apache.kafka.streams.processor.TaskMetadata;
 import org.apache.kafka.streams.processor.ThreadMetadata;
+import org.apache.kafka.streams.processor.internals.assignment.AssignorError;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.apache.kafka.streams.state.internals.ThreadCache;
@@ -269,7 +270,7 @@ public void onPartitionsAssigned(final Collection assignment) {
                 taskManager.suspendedActiveTaskIds(),
                 taskManager.suspendedStandbyTaskIds());
 
-            if (streamThread.assignmentErrorCode.get() == StreamsPartitionAssignor.Error.INCOMPLETE_SOURCE_TOPIC_METADATA.code()) {
+            if (streamThread.assignmentErrorCode.get() == AssignorError.INCOMPLETE_SOURCE_TOPIC_METADATA.code()) {
                 log.error("Received error code {} - shutdown", streamThread.assignmentErrorCode.get());
                 streamThread.shutdown();
                 return;
@@ -281,7 +282,7 @@ public void onPartitionsAssigned(final Collection assignment) {
                         "Skipping task creation in rebalance because we are already in {} state.",
                         streamThread.state()
                     );
-                } else if (streamThread.assignmentErrorCode.get() != StreamsPartitionAssignor.Error.NONE.code()) {
+                } else if (streamThread.assignmentErrorCode.get() != AssignorError.NONE.code()) {
                     log.debug(
                         "Encountered assignment error during partition assignment: {}. Skipping task initialization",
                         streamThread.assignmentErrorCode
@@ -790,9 +791,9 @@ private void runLoop() {
         while (isRunning()) {
             try {
                 runOnce();
-                if (assignmentErrorCode.get() == StreamsPartitionAssignor.Error.VERSION_PROBING.code()) {
+                if (assignmentErrorCode.get() == AssignorError.VERSION_PROBING.code()) {
                     log.info("Version probing detected. Triggering new rebalance.");
-                    assignmentErrorCode.set(StreamsPartitionAssignor.Error.NONE.code());
+                    assignmentErrorCode.set(AssignorError.NONE.code());
                     enforceRebalance();
                 }
             } catch (final TaskMigratedException ignoreAndRejoinGroup) {
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
index d2f2b3a8e10d9..2d46778124af0 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
@@ -16,8 +16,6 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
-import java.nio.ByteBuffer;
-import org.apache.kafka.clients.CommonClientConfigs;
 import org.apache.kafka.clients.consumer.ConsumerGroupMetadata;
 import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor;
 import org.apache.kafka.common.Cluster;
@@ -28,21 +26,22 @@
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.config.ConfigException;
 import org.apache.kafka.common.utils.LogContext;
-import org.apache.kafka.common.utils.Utils;
-import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.errors.StreamsException;
 import org.apache.kafka.streams.errors.TaskAssignmentException;
 import org.apache.kafka.streams.processor.PartitionGrouper;
 import org.apache.kafka.streams.processor.TaskId;
 import org.apache.kafka.streams.processor.internals.assignment.AssignmentInfo;
+import org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration;
+import org.apache.kafka.streams.processor.internals.assignment.AssignorError;
 import org.apache.kafka.streams.processor.internals.assignment.ClientState;
+import org.apache.kafka.streams.processor.internals.assignment.CopartitionedTopicsEnforcer;
 import org.apache.kafka.streams.processor.internals.assignment.StickyTaskAssignor;
 import org.apache.kafka.streams.processor.internals.assignment.SubscriptionInfo;
 import org.apache.kafka.streams.state.HostInfo;
 import org.slf4j.Logger;
 
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
@@ -51,58 +50,29 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.apache.kafka.common.utils.Utils.getHost;
 import static org.apache.kafka.common.utils.Utils.getPort;
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.EARLIEST_PROBEABLE_VERSION;
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.LATEST_SUPPORTED_VERSION;
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.UNKNOWN;
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.VERSION_FIVE;
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.VERSION_FOUR;
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.VERSION_ONE;
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.VERSION_THREE;
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.VERSION_TWO;
 
 public class StreamsPartitionAssignor implements ConsumerPartitionAssignor, Configurable {
-
-    final static int UNKNOWN = -1;
-    private final static int VERSION_ONE = 1;
-    private final static int VERSION_TWO = 2;
-    private final static int VERSION_THREE = 3;
-    private final static int VERSION_FOUR = 4;
-    private final static int VERSION_FIVE = 5;
-    private final static int EARLIEST_PROBEABLE_VERSION = VERSION_THREE;
-    protected final Set supportedVersions = new HashSet<>();
-
     private Logger log;
     private String logPrefix;
-    public enum Error {
-        NONE(0),
-        INCOMPLETE_SOURCE_TOPIC_METADATA(1),
-        VERSION_PROBING(2);
-
-        private final int code;
-
-        Error(final int code) {
-            this.code = code;
-        }
-
-        public int code() {
-            return code;
-        }
-
-        public static Error fromCode(final int code) {
-            switch (code) {
-                case 0:
-                    return NONE;
-                case 1:
-                    return INCOMPLETE_SOURCE_TOPIC_METADATA;
-                case 2:
-                    return VERSION_PROBING;
-                default:
-                    throw new IllegalArgumentException("Unknown error code: " + code);
-            }
-        }
-    }
 
     private static class AssignedPartition implements Comparable {
-        public final TaskId taskId;
-        public final TopicPartition partition;
+        private final TaskId taskId;
+        private final TopicPartition partition;
 
         AssignedPartition(final TaskId taskId,
                           final TopicPartition partition) {
@@ -112,7 +82,7 @@ private static class AssignedPartition implements Comparable
 
         @Override
         public int compareTo(final AssignedPartition that) {
-            return PARTITION_COMPARATOR.compare(this.partition, that.partition);
+            return PARTITION_COMPARATOR.compare(partition, that.partition);
         }
 
         @Override
@@ -132,9 +102,9 @@ public int hashCode() {
     }
 
     private static class ClientMetadata {
-        final HostInfo hostInfo;
-        final Set consumers;
-        final ClientState state;
+        private final HostInfo hostInfo;
+        private final Set consumers;
+        private final ClientState state;
 
         ClientMetadata(final String endPoint) {
 
@@ -144,7 +114,9 @@ private static class ClientMetadata {
                 final Integer port = getPort(endPoint);
 
                 if (host == null || port == null) {
-                    throw new ConfigException(String.format("Error parsing host address %s. Expected format host:port.", endPoint));
+                    throw new ConfigException(
+                        String.format("Error parsing host address %s. Expected format host:port.", endPoint)
+                    );
                 }
 
                 hostInfo = new HostInfo(host, port);
@@ -170,28 +142,16 @@ void addConsumer(final String consumerMemberId,
         @Override
         public String toString() {
             return "ClientMetadata{" +
-                    "hostInfo=" + hostInfo +
-                    ", consumers=" + consumers +
-                    ", state=" + state +
-                    '}';
+                "hostInfo=" + hostInfo +
+                ", consumers=" + consumers +
+                ", state=" + state +
+                '}';
         }
     }
 
-    private static final class InternalStreamsConfig extends StreamsConfig {
-        private InternalStreamsConfig(final Map props) {
-            super(props, false);
-        }
-    }
 
-    protected static final Comparator PARTITION_COMPARATOR = (p1, p2) -> {
-        final int result = p1.topic().compareTo(p2.topic());
-
-        if (result != 0) {
-            return result;
-        } else {
-            return Integer.compare(p1.partition(), p2.partition());
-        }
-    };
+    protected static final Comparator PARTITION_COMPARATOR =
+        Comparator.comparing(TopicPartition::topic).thenComparingInt(TopicPartition::partition);
 
     private String userEndPoint;
     private int numStandbyReplicas;
@@ -200,10 +160,10 @@ private InternalStreamsConfig(final Map props) {
     private PartitionGrouper partitionGrouper;
     private AtomicInteger assignmentErrorCode;
 
-    protected int usedSubscriptionMetadataVersion = SubscriptionInfo.LATEST_SUPPORTED_VERSION;
+    protected int usedSubscriptionMetadataVersion = LATEST_SUPPORTED_VERSION;
 
     private InternalTopicManager internalTopicManager;
-    private CopartitionedTopicsValidator copartitionedTopicsValidator;
+    private CopartitionedTopicsEnforcer copartitionedTopicsEnforcer;
 
     protected String userEndPoint() {
         return userEndPoint;
@@ -217,95 +177,26 @@ protected TaskManager taskManger() {
      * We need to have the PartitionAssignor and its StreamThread to be mutually accessible
      * since the former needs later's cached metadata while sending subscriptions,
      * and the latter needs former's returned assignment when adding tasks.
+     *
      * @throws KafkaException if the stream thread is not specified
      */
     @Override
     public void configure(final Map configs) {
-        final StreamsConfig streamsConfig = new InternalStreamsConfig(configs);
-
-        // Setting the logger with the passed in client thread name
-        logPrefix = String.format("stream-thread [%s] ", streamsConfig.getString(CommonClientConfigs.CLIENT_ID_CONFIG));
-        final LogContext logContext = new LogContext(logPrefix);
-        log = logContext.logger(getClass());
-
-        final String upgradeFrom = streamsConfig.getString(StreamsConfig.UPGRADE_FROM_CONFIG);
-        if (upgradeFrom != null) {
-            switch (upgradeFrom) {
-                case StreamsConfig.UPGRADE_FROM_0100:
-                    log.info("Downgrading metadata version from {} to 1 for upgrade from 0.10.0.x.", SubscriptionInfo.LATEST_SUPPORTED_VERSION);
-                    usedSubscriptionMetadataVersion = VERSION_ONE;
-                    break;
-                case StreamsConfig.UPGRADE_FROM_0101:
-                case StreamsConfig.UPGRADE_FROM_0102:
-                case StreamsConfig.UPGRADE_FROM_0110:
-                case StreamsConfig.UPGRADE_FROM_10:
-                case StreamsConfig.UPGRADE_FROM_11:
-                    log.info("Downgrading metadata version from {} to 2 for upgrade from {}.x.", SubscriptionInfo.LATEST_SUPPORTED_VERSION, upgradeFrom);
-                    usedSubscriptionMetadataVersion = VERSION_TWO;
-                    break;
-                default:
-                    throw new IllegalArgumentException("Unknown configuration value for parameter 'upgrade.from': " + upgradeFrom);
-            }
-        }
-
-        final Object o = configs.get(StreamsConfig.InternalConfig.TASK_MANAGER_FOR_PARTITION_ASSIGNOR);
-        if (o == null) {
-            final KafkaException fatalException = new KafkaException("TaskManager is not specified");
-            log.error(fatalException.getMessage(), fatalException);
-            throw fatalException;
-        }
-
-        if (!(o instanceof TaskManager)) {
-            final KafkaException fatalException = new KafkaException(String.format("%s is not an instance of %s", o.getClass().getName(), TaskManager.class.getName()));
-            log.error(fatalException.getMessage(), fatalException);
-            throw fatalException;
-        }
-
-        taskManager = (TaskManager) o;
-
-        final Object ai = configs.get(StreamsConfig.InternalConfig.ASSIGNMENT_ERROR_CODE);
-        if (ai == null) {
-            final KafkaException fatalException = new KafkaException("assignmentErrorCode is not specified");
-            log.error(fatalException.getMessage(), fatalException);
-            throw fatalException;
-        }
-
-        if (!(ai instanceof AtomicInteger)) {
-            final KafkaException fatalException = new KafkaException(String.format("%s is not an instance of %s",
-                ai.getClass().getName(), AtomicInteger.class.getName()));
-            log.error(fatalException.getMessage(), fatalException);
-            throw fatalException;
-        }
-        assignmentErrorCode = (AtomicInteger) ai;
-
-        numStandbyReplicas = streamsConfig.getInt(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG);
-
-        partitionGrouper = streamsConfig.getConfiguredInstance(StreamsConfig.PARTITION_GROUPER_CLASS_CONFIG, PartitionGrouper.class);
-
-        final String userEndPoint = streamsConfig.getString(StreamsConfig.APPLICATION_SERVER_CONFIG);
-        if (userEndPoint != null && !userEndPoint.isEmpty()) {
-            try {
-                final String host = getHost(userEndPoint);
-                final Integer port = getPort(userEndPoint);
-
-                if (host == null || port == null) {
-                    throw new ConfigException(String.format("%s Config %s isn't in the correct format. Expected a host:port pair" +
-                            " but received %s",
-                        logPrefix, StreamsConfig.APPLICATION_SERVER_CONFIG, userEndPoint));
-                }
-            } catch (final NumberFormatException nfe) {
-                throw new ConfigException(String.format("%s Invalid port supplied in %s for config %s",
-                        logPrefix, userEndPoint, StreamsConfig.APPLICATION_SERVER_CONFIG));
-            }
-
-            this.userEndPoint = userEndPoint;
-        }
-
-        internalTopicManager = new InternalTopicManager(taskManager.adminClient, streamsConfig);
-
-        copartitionedTopicsValidator = new CopartitionedTopicsValidator(logPrefix);
+        final AssignorConfiguration assignorConfiguration = new AssignorConfiguration(configs);
+
+        logPrefix = assignorConfiguration.logPrefix();
+        log = new LogContext(logPrefix).logger(getClass());
+        usedSubscriptionMetadataVersion = assignorConfiguration.configuredMetadataVersion(usedSubscriptionMetadataVersion);
+        taskManager = assignorConfiguration.getTaskManager();
+        assignmentErrorCode = assignorConfiguration.getAssignmentErrorCode(configs);
+        numStandbyReplicas = assignorConfiguration.getNumStandbyReplicas();
+        partitionGrouper = assignorConfiguration.getPartitionGrouper();
+        userEndPoint = assignorConfiguration.getUserEndPoint();
+        internalTopicManager = assignorConfiguration.getInternalTopicManager();
+        copartitionedTopicsEnforcer = assignorConfiguration.getCopartitionedTopicsEnforcer();
     }
 
+
     @Override
     public String name() {
         return "stream";
@@ -326,7 +217,7 @@ public ByteBuffer subscriptionUserData(final Set topics) {
             taskManager.processId(),
             previousActiveTasks,
             standbyTasks,
-            this.userEndPoint);
+            userEndPoint);
 
         taskManager.updateSubscriptionsFromMetadata(topics);
 
@@ -337,22 +228,23 @@ private Map errorAssignment(final Map
                                                     final String topic,
                                                     final int errorCode) {
         log.error("{} is unknown yet during rebalance," +
-            " please make sure they have been pre-created before starting the Streams application.", topic);
+                      " please make sure they have been pre-created before starting the Streams application.", topic);
         final Map assignment = new HashMap<>();
         for (final ClientMetadata clientMetadata : clientsMetadata.values()) {
             for (final String consumerId : clientMetadata.consumers) {
                 assignment.put(consumerId, new Assignment(
                     Collections.emptyList(),
-                    new AssignmentInfo(AssignmentInfo.LATEST_SUPPORTED_VERSION,
-                        Collections.emptyList(),
-                        Collections.emptyMap(),
-                        Collections.emptyMap(),
-                        errorCode).encode()
+                    new AssignmentInfo(LATEST_SUPPORTED_VERSION,
+                                       Collections.emptyList(),
+                                       Collections.emptyMap(),
+                                       Collections.emptyMap(),
+                                       errorCode).encode()
                 ));
             }
         }
         return assignment;
     }
+
     /*
      * This assigns tasks to consumer clients in the following steps.
      *
@@ -380,9 +272,8 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
         final Map clientMetadataMap = new HashMap<>();
         final Set futureConsumers = new HashSet<>();
 
-        int minReceivedMetadataVersion = SubscriptionInfo.LATEST_SUPPORTED_VERSION;
+        int minReceivedMetadataVersion = LATEST_SUPPORTED_VERSION;
 
-        supportedVersions.clear();
         int futureMetadataVersion = UNKNOWN;
         for (final Map.Entry entry : subscriptions.entrySet()) {
             final String consumerId = entry.getKey();
@@ -390,8 +281,7 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
 
             final SubscriptionInfo info = SubscriptionInfo.decode(subscription.userData());
             final int usedVersion = info.version();
-            supportedVersions.add(info.latestSupportedVersion());
-            if (usedVersion > SubscriptionInfo.LATEST_SUPPORTED_VERSION) {
+            if (usedVersion > LATEST_SUPPORTED_VERSION) {
                 futureMetadataVersion = usedVersion;
                 futureConsumers.add(consumerId);
                 continue;
@@ -413,24 +303,28 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
         }
 
         final boolean versionProbing;
-        if (futureMetadataVersion != UNKNOWN) {
+        if (futureMetadataVersion == UNKNOWN) {
+            versionProbing = false;
+        } else {
             if (minReceivedMetadataVersion >= EARLIEST_PROBEABLE_VERSION) {
-                log.info("Received a future (version probing) subscription (version: {}). Sending empty assignment back (with supported version {}).",
-                    futureMetadataVersion,
-                    SubscriptionInfo.LATEST_SUPPORTED_VERSION);
+                log.info("Received a future (version probing) subscription (version: {})."
+                             + " Sending empty assignment back (with supported version {}).",
+                         futureMetadataVersion,
+                         LATEST_SUPPORTED_VERSION);
                 versionProbing = true;
             } else {
-                throw new IllegalStateException("Received a future (version probing) subscription (version: " + futureMetadataVersion
-                    + ") and an incompatible pre Kafka 2.0 subscription (version: " + minReceivedMetadataVersion + ") at the same time.");
+                throw new IllegalStateException(
+                    "Received a future (version probing) subscription (version: " + futureMetadataVersion
+                        + ") and an incompatible pre Kafka 2.0 subscription (version: " + minReceivedMetadataVersion
+                        + ") at the same time."
+                );
             }
-        } else {
-            versionProbing = false;
         }
 
-        if (minReceivedMetadataVersion < SubscriptionInfo.LATEST_SUPPORTED_VERSION) {
+        if (minReceivedMetadataVersion < LATEST_SUPPORTED_VERSION) {
             log.info("Downgrading metadata to version {}. Latest supported version is {}.",
-                minReceivedMetadataVersion,
-                SubscriptionInfo.LATEST_SUPPORTED_VERSION);
+                     minReceivedMetadataVersion,
+                     LATEST_SUPPORTED_VERSION);
         }
 
         log.debug("Constructed client metadata {} from the member subscriptions.", clientMetadataMap);
@@ -448,11 +342,13 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
                 if (!topicsInfo.repartitionSourceTopics.keySet().contains(topic) &&
                     !metadata.topics().contains(topic)) {
                     log.error("Missing source topic {} during assignment. Returning error {}.",
-                              topic, Error.INCOMPLETE_SOURCE_TOPIC_METADATA.name());
-                    return new GroupAssignment(errorAssignment(clientMetadataMap, topic, Error.INCOMPLETE_SOURCE_TOPIC_METADATA.code));
+                              topic, AssignorError.INCOMPLETE_SOURCE_TOPIC_METADATA.name());
+                    return new GroupAssignment(
+                        errorAssignment(clientMetadataMap, topic, AssignorError.INCOMPLETE_SOURCE_TOPIC_METADATA.code())
+                    );
                 }
             }
-            for (final InternalTopicConfig topic: topicsInfo.repartitionSourceTopics.values()) {
+            for (final InternalTopicConfig topic : topicsInfo.repartitionSourceTopics.values()) {
                 repartitionTopicMetadata.put(topic.name(), topic);
             }
         }
@@ -463,9 +359,10 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
 
             for (final InternalTopologyBuilder.TopicsInfo topicsInfo : topicGroups.values()) {
                 for (final String topicName : topicsInfo.repartitionSourceTopics.keySet()) {
-                    int numPartitions = repartitionTopicMetadata.get(topicName).numberOfPartitions();
+                    final Optional maybeNumPartitions = repartitionTopicMetadata.get(topicName).numberOfPartitions();
+                    Integer numPartitions = null;
 
-                    if (numPartitions == UNKNOWN) {
+                    if (!maybeNumPartitions.isPresent()) {
                         // try set the number of partitions for this repartition topic if it is not set yet
                         for (final InternalTopologyBuilder.TopicsInfo otherTopicsInfo : topicGroups.values()) {
                             final Set otherSinkTopics = otherTopicsInfo.sinkTopics;
@@ -474,16 +371,25 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
                                 // if this topic is one of the sink topics of this topology,
                                 // use the maximum of all its source topic partitions as the number of partitions
                                 for (final String sourceTopicName : otherTopicsInfo.sourceTopics) {
-                                    final Integer numPartitionsCandidate;
+                                    final int numPartitionsCandidate;
                                     // It is possible the sourceTopic is another internal topic, i.e,
                                     // map().join().join(map())
-                                    if (repartitionTopicMetadata.containsKey(sourceTopicName)) {
-                                        numPartitionsCandidate = repartitionTopicMetadata.get(sourceTopicName).numberOfPartitions();
+                                    if (repartitionTopicMetadata.containsKey(sourceTopicName)
+                                        && repartitionTopicMetadata.get(sourceTopicName).numberOfPartitions().isPresent()) {
+                                        numPartitionsCandidate = repartitionTopicMetadata.get(sourceTopicName).numberOfPartitions().get();
                                     } else {
-                                        numPartitionsCandidate = metadata.partitionCountForTopic(sourceTopicName);
+                                        final Integer count = metadata.partitionCountForTopic(sourceTopicName);
+                                        if (count == null) {
+                                            throw new IllegalStateException(
+                                                "No partition count found for source topic "
+                                                    + sourceTopicName
+                                                    + ", but it should have been."
+                                            );
+                                        }
+                                        numPartitionsCandidate = count;
                                     }
 
-                                    if (numPartitionsCandidate > numPartitions) {
+                                    if (numPartitions == null || numPartitionsCandidate > numPartitions) {
                                         numPartitions = numPartitionsCandidate;
                                     }
                                 }
@@ -491,7 +397,7 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
                         }
                         // if we still have not find the right number of partitions,
                         // another iteration is needed
-                        if (numPartitions == UNKNOWN) {
+                        if (numPartitions == null) {
                             numPartitionsNeeded = true;
                         } else {
                             repartitionTopicMetadata.get(topicName).setNumberOfPartitions(numPartitions);
@@ -516,11 +422,13 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
         final Map allRepartitionTopicPartitions = new HashMap<>();
         for (final Map.Entry entry : repartitionTopicMetadata.entrySet()) {
             final String topic = entry.getKey();
-            final int numPartitions = entry.getValue().numberOfPartitions();
+            final int numPartitions = entry.getValue().numberOfPartitions().orElse(-1);
 
             for (int partition = 0; partition < numPartitions; partition++) {
-                allRepartitionTopicPartitions.put(new TopicPartition(topic, partition),
-                        new PartitionInfo(topic, partition, null, new Node[0], new Node[0]));
+                allRepartitionTopicPartitions.put(
+                    new TopicPartition(topic, partition),
+                    new PartitionInfo(topic, partition, null, new Node[0], new Node[0])
+                );
             }
         }
 
@@ -539,7 +447,8 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
             sourceTopicsByGroup.put(entry.getKey(), entry.getValue().sourceTopics);
         }
 
-        final Map> partitionsForTask = partitionGrouper.partitionGroups(sourceTopicsByGroup, fullMetadata);
+        final Map> partitionsForTask =
+            partitionGrouper.partitionGroups(sourceTopicsByGroup, fullMetadata);
 
         // check if all partitions are assigned, and there are no duplicates of partitions in multiple tasks
         final Set allAssignedPartitions = new HashSet<>();
@@ -558,19 +467,19 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
         }
         for (final String topic : allSourceTopics) {
             final List partitionInfoList = fullMetadata.partitionsForTopic(topic);
-            if (!partitionInfoList.isEmpty()) {
+            if (partitionInfoList.isEmpty()) {
+                log.warn("No partitions found for topic {}", topic);
+            } else {
                 for (final PartitionInfo partitionInfo : partitionInfoList) {
                     final TopicPartition partition = new TopicPartition(partitionInfo.topic(), partitionInfo.partition());
                     if (!allAssignedPartitions.contains(partition)) {
                         log.warn("Partition {} is not assigned to any tasks: {}"
-                                 + " Possible causes of a partition not getting assigned"
-                                 + " is that another topic defined in the topology has not been"
-                                 + " created when starting your streams application,"
-                                 + " resulting in no tasks created for this topology at all.", partition, partitionsForTask);
+                                     + " Possible causes of a partition not getting assigned"
+                                     + " is that another topic defined in the topology has not been"
+                                     + " created when starting your streams application,"
+                                     + " resulting in no tasks created for this topology at all.", partition, partitionsForTask);
                     }
                 }
-            } else {
-                log.warn("No partitions found for topic {}", topic);
             }
         }
 
@@ -611,7 +520,7 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
         }
 
         log.debug("Assigning tasks {} to clients {} with number of replicas {}",
-                partitionsForTask.keySet(), states, numStandbyReplicas);
+                  partitionsForTask.keySet(), states, numStandbyReplicas);
 
         final StickyTaskAssignor taskAssignor = new StickyTaskAssignor<>(states, partitionsForTask.keySet());
         taskAssignor.assign(numStandbyReplicas);
@@ -643,18 +552,29 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
 
         final Map assignment;
         if (versionProbing) {
-            assignment = versionProbingAssignment(clientMetadataMap, partitionsForTask, partitionsByHostState, futureConsumers, minReceivedMetadataVersion);
+            assignment = versionProbingAssignment(
+                clientMetadataMap,
+                partitionsForTask,
+                partitionsByHostState,
+                futureConsumers,
+                minReceivedMetadataVersion
+            );
         } else {
-            assignment = computeNewAssignment(clientMetadataMap, partitionsForTask, partitionsByHostState, minReceivedMetadataVersion);
+            assignment = computeNewAssignment(
+                clientMetadataMap,
+                partitionsForTask,
+                partitionsByHostState,
+                minReceivedMetadataVersion
+            );
         }
 
         return new GroupAssignment(assignment);
     }
 
-    private Map computeNewAssignment(final Map clientsMetadata,
-                                                         final Map> partitionsForTask,
-                                                         final Map> partitionsByHostState,
-                                                         final int minUserMetadataVersion) {
+    private static Map computeNewAssignment(final Map clientsMetadata,
+                                                                final Map> partitionsForTask,
+                                                                final Map> partitionsByHostState,
+                                                                final int minUserMetadataVersion) {
         final Map assignment = new HashMap<>();
 
         // within the client, distribute tasks to its owned consumers
@@ -662,14 +582,16 @@ private Map computeNewAssignment(final Map consumers = entry.getValue().consumers;
             final ClientState state = entry.getValue().state;
 
-            final List> interleavedActive = interleaveTasksByGroupId(state.activeTasks(), consumers.size());
-            final List> interleavedStandby = interleaveTasksByGroupId(state.standbyTasks(), consumers.size());
+            final List> interleavedActive =
+                interleaveTasksByGroupId(state.activeTasks(), consumers.size());
+            final List> interleavedStandby =
+                interleaveTasksByGroupId(state.standbyTasks(), consumers.size());
 
             int consumerTaskIndex = 0;
 
             for (final String consumer : consumers) {
                 final Map> standby = new HashMap<>();
-                final ArrayList assignedPartitions = new ArrayList<>();
+                final List assignedPartitions = new ArrayList<>();
 
                 final List assignedActiveList = interleavedActive.get(consumerTaskIndex);
 
@@ -697,20 +619,30 @@ private Map computeNewAssignment(final Map versionProbingAssignment(final Map clientsMetadata,
-                                                             final Map> partitionsForTask,
-                                                             final Map> partitionsByHostState,
-                                                             final Set futureConsumers,
-                                                             final int minUserMetadataVersion) {
+    private static Map versionProbingAssignment(final Map clientsMetadata,
+                                                                    final Map> partitionsForTask,
+                                                                    final Map> partitionsByHostState,
+                                                                    final Set futureConsumers,
+                                                                    final int minUserMetadataVersion) {
         final Map assignment = new HashMap<>();
 
         // assign previously assigned tasks to "old consumers"
@@ -758,7 +690,7 @@ private Map versionProbingAssignment(final Map> interleaveTasksByGroupId(final Collection taskIds, final int numberThreads) {
+    static List> interleaveTasksByGroupId(final Collection taskIds, final int numberThreads) {
         final LinkedList sortedTasks = new LinkedList<>(taskIds);
         Collections.sort(sortedTasks);
         final List> taskIdsForConsumerAssignment = new ArrayList<>(numberThreads);
@@ -780,10 +712,10 @@ List> interleaveTasksByGroupId(final Collection taskIds, fi
     private void upgradeSubscriptionVersionIfNeeded(final int leaderSupportedVersion) {
         if (leaderSupportedVersion > usedSubscriptionMetadataVersion) {
             log.info("Sent a version {} subscription and group leader's latest supported version is {}. " +
-                    "Upgrading subscription metadata version to {} for next rebalance.",
-                usedSubscriptionMetadataVersion,
-                leaderSupportedVersion,
-                leaderSupportedVersion);
+                         "Upgrading subscription metadata version to {} for next rebalance.",
+                     usedSubscriptionMetadataVersion,
+                     leaderSupportedVersion,
+                     leaderSupportedVersion);
             usedSubscriptionMetadataVersion = leaderSupportedVersion;
         }
     }
@@ -797,7 +729,7 @@ public void onAssignment(final Assignment assignment, final ConsumerGroupMetadat
         partitions.sort(PARTITION_COMPARATOR);
 
         final AssignmentInfo info = AssignmentInfo.decode(assignment.userData());
-        if (info.errCode() != Error.NONE.code) {
+        if (info.errCode() != AssignorError.NONE.code()) {
             // set flag to shutdown streams app
             assignmentErrorCode.set(info.errCode());
             return;
@@ -806,29 +738,36 @@ public void onAssignment(final Assignment assignment, final ConsumerGroupMetadat
         final int leaderSupportedVersion = info.latestSupportedVersion();
 
         if (receivedAssignmentMetadataVersion > usedSubscriptionMetadataVersion) {
-            throw new IllegalStateException("Sent a version " + usedSubscriptionMetadataVersion
-                + " subscription but got an assignment with higher version " + receivedAssignmentMetadataVersion + ".");
+            throw new IllegalStateException(
+                "Sent a version " + usedSubscriptionMetadataVersion
+                    + " subscription but got an assignment with higher version "
+                    + receivedAssignmentMetadataVersion + "."
+            );
         }
 
         if (receivedAssignmentMetadataVersion < usedSubscriptionMetadataVersion
             && receivedAssignmentMetadataVersion >= EARLIEST_PROBEABLE_VERSION) {
 
             if (receivedAssignmentMetadataVersion == leaderSupportedVersion) {
-                log.info("Sent a version {} subscription and got version {} assignment back (successful version probing). " +
+                log.info(
+                    "Sent a version {} subscription and got version {} assignment back (successful version probing). " +
                         "Downgrading subscription metadata to received version and trigger new rebalance.",
                     usedSubscriptionMetadataVersion,
-                    receivedAssignmentMetadataVersion);
+                    receivedAssignmentMetadataVersion
+                );
                 usedSubscriptionMetadataVersion = receivedAssignmentMetadataVersion;
             } else {
-                log.info("Sent a version {} subscription and got version {} assignment back (successful version probing). " +
-                    "Setting subscription metadata to leaders supported version {} and trigger new rebalance.",
+                log.info(
+                    "Sent a version {} subscription and got version {} assignment back (successful version probing). " +
+                        "Setting subscription metadata to leaders supported version {} and trigger new rebalance.",
                     usedSubscriptionMetadataVersion,
                     receivedAssignmentMetadataVersion,
-                    leaderSupportedVersion);
+                    leaderSupportedVersion
+                );
                 usedSubscriptionMetadataVersion = leaderSupportedVersion;
             }
 
-            assignmentErrorCode.set(Error.VERSION_PROBING.code);
+            assignmentErrorCode.set(AssignorError.VERSION_PROBING.code());
             return;
         }
 
@@ -840,30 +779,25 @@ public void onAssignment(final Assignment assignment, final ConsumerGroupMetadat
 
         switch (receivedAssignmentMetadataVersion) {
             case VERSION_ONE:
-                processVersionOneAssignment(info, partitions, activeTasks);
+                processVersionOneAssignment(logPrefix, info, partitions, activeTasks);
                 partitionsByHost = Collections.emptyMap();
                 break;
             case VERSION_TWO:
-                processVersionTwoAssignment(info, partitions, activeTasks, topicToPartitionInfo);
+                processVersionTwoAssignment(logPrefix, info, partitions, activeTasks, topicToPartitionInfo);
                 partitionsByHost = info.partitionsByHost();
                 break;
             case VERSION_THREE:
-                upgradeSubscriptionVersionIfNeeded(leaderSupportedVersion);
-                processVersionThreeAssignment(info, partitions, activeTasks, topicToPartitionInfo);
-                partitionsByHost = info.partitionsByHost();
-                break;
             case VERSION_FOUR:
-                upgradeSubscriptionVersionIfNeeded(leaderSupportedVersion);
-                processVersionFourAssignment(info, partitions, activeTasks, topicToPartitionInfo);
-                partitionsByHost = info.partitionsByHost();
-                break;
             case VERSION_FIVE:
                 upgradeSubscriptionVersionIfNeeded(leaderSupportedVersion);
-                processVersionFiveAssignment(info, partitions, activeTasks, topicToPartitionInfo);
+                processVersionTwoAssignment(logPrefix, info, partitions, activeTasks, topicToPartitionInfo);
                 partitionsByHost = info.partitionsByHost();
                 break;
             default:
-                throw new IllegalStateException("This code should never be reached. Please file a bug report at https://issues.apache.org/jira/projects/KAFKA/");
+                throw new IllegalStateException(
+                    "This code should never be reached."
+                        + " Please file a bug report at https://issues.apache.org/jira/projects/KAFKA/"
+                );
         }
 
         taskManager.setClusterMetadata(Cluster.empty().withPartitions(topicToPartitionInfo));
@@ -872,15 +806,20 @@ public void onAssignment(final Assignment assignment, final ConsumerGroupMetadat
         taskManager.updateSubscriptionsFromAssignment(partitions);
     }
 
-    private void processVersionOneAssignment(final AssignmentInfo info,
-                                             final List partitions,
-                                             final Map> activeTasks) {
+    private static void processVersionOneAssignment(final String logPrefix,
+                                                    final AssignmentInfo info,
+                                                    final List partitions,
+                                                    final Map> activeTasks) {
         // the number of assigned partitions should be the same as number of active tasks, which
         // could be duplicated if one task has more than one assigned partitions
         if (partitions.size() != info.activeTasks().size()) {
             throw new TaskAssignmentException(
-                String.format("%sNumber of assigned partitions %d is not equal to the number of active taskIds %d" +
-                    ", assignmentInfo=%s", logPrefix, partitions.size(), info.activeTasks().size(), info.toString())
+                String.format(
+                    "%sNumber of assigned partitions %d is not equal to "
+                        + "the number of active taskIds %d, assignmentInfo=%s",
+                    logPrefix, partitions.size(),
+                    info.activeTasks().size(), info.toString()
+                )
             );
         }
 
@@ -891,11 +830,12 @@ private void processVersionOneAssignment(final AssignmentInfo info,
         }
     }
 
-    private void processVersionTwoAssignment(final AssignmentInfo info,
-                                             final List partitions,
-                                             final Map> activeTasks,
-                                             final Map topicToPartitionInfo) {
-        processVersionOneAssignment(info, partitions, activeTasks);
+    public static void processVersionTwoAssignment(final String logPrefix,
+                                            final AssignmentInfo info,
+                                            final List partitions,
+                                            final Map> activeTasks,
+                                            final Map topicToPartitionInfo) {
+        processVersionOneAssignment(logPrefix, info, partitions, activeTasks);
 
         // process partitions by host
         final Map> partitionsByHost = info.partitionsByHost();
@@ -903,40 +843,18 @@ private void processVersionTwoAssignment(final AssignmentInfo info,
             for (final TopicPartition topicPartition : value) {
                 topicToPartitionInfo.put(
                     topicPartition,
-                    new PartitionInfo(topicPartition.topic(), topicPartition.partition(), null, new Node[0], new Node[0]));
+                    new PartitionInfo(
+                        topicPartition.topic(),
+                        topicPartition.partition(),
+                        null,
+                        new Node[0],
+                        new Node[0]
+                    )
+                );
             }
         }
     }
 
-    private void processVersionThreeAssignment(final AssignmentInfo info,
-                                               final List partitions,
-                                               final Map> activeTasks,
-                                               final Map topicToPartitionInfo) {
-        processVersionTwoAssignment(info, partitions, activeTasks, topicToPartitionInfo);
-    }
-
-    private void processVersionFourAssignment(final AssignmentInfo info,
-                                              final List partitions,
-                                              final Map> activeTasks,
-                                              final Map topicToPartitionInfo) {
-        processVersionThreeAssignment(info, partitions, activeTasks, topicToPartitionInfo);
-    }
-
-    private void processVersionFiveAssignment(final AssignmentInfo info,
-                                              final List partitions,
-                                              final Map> activeTasks,
-                                              final Map topicToPartitionInfo) {
-        processVersionFourAssignment(info, partitions, activeTasks, topicToPartitionInfo);
-    }
-
-    // for testing
-    protected void processLatestVersionAssignment(final AssignmentInfo info,
-                                                  final List partitions,
-                                                  final Map> activeTasks,
-                                                  final Map topicToPartitionInfo) {
-        processVersionThreeAssignment(info, partitions, activeTasks, topicToPartitionInfo);
-    }
-
     /**
      * Internal helper function that creates a Kafka topic
      *
@@ -949,12 +867,15 @@ private void prepareTopic(final Map topicPartitions
         final Map topicsToMakeReady = new HashMap<>();
 
         for (final InternalTopicConfig topic : topicPartitions.values()) {
-            final int numPartitions = topic.numberOfPartitions();
-            if (numPartitions == UNKNOWN) {
-                throw new StreamsException(String.format("%sTopic [%s] number of partitions not defined", logPrefix, topic.name()));
+            final Optional numPartitions = topic.numberOfPartitions();
+            if (!numPartitions.isPresent()) {
+                throw new StreamsException(
+                    String.format("%sTopic [%s] number of partitions not defined",
+                                  logPrefix, topic.name())
+                );
             }
 
-            topic.setNumberOfPartitions(numPartitions);
+            topic.setNumberOfPartitions(numPartitions.get());
             topicsToMakeReady.put(topic.name(), topic);
         }
 
@@ -969,65 +890,10 @@ private void ensureCopartitioning(final Collection> copartitionGroup
                                       final Map allRepartitionTopicsNumPartitions,
                                       final Cluster metadata) {
         for (final Set copartitionGroup : copartitionGroups) {
-            copartitionedTopicsValidator.validate(copartitionGroup, allRepartitionTopicsNumPartitions, metadata);
+            copartitionedTopicsEnforcer.enforce(copartitionGroup, allRepartitionTopicsNumPartitions, metadata);
         }
     }
 
-    static class CopartitionedTopicsValidator {
-        private final String logPrefix;
-        private final Logger log;
-
-        CopartitionedTopicsValidator(final String logPrefix) {
-            this.logPrefix = logPrefix;
-            final LogContext logContext = new LogContext(logPrefix);
-            log = logContext.logger(getClass());
-        }
-
-        void validate(final Set copartitionGroup,
-                      final Map allRepartitionTopicsNumPartitions,
-                      final Cluster metadata) {
-            int numPartitions = UNKNOWN;
-
-            for (final String topic : copartitionGroup) {
-                if (!allRepartitionTopicsNumPartitions.containsKey(topic)) {
-                    final Integer partitions = metadata.partitionCountForTopic(topic);
-                    if (partitions == null) {
-                        final String str = String.format("%sTopic not found: %s", logPrefix, topic);
-                        log.error(str);
-                        throw new IllegalStateException(str);
-                    }
-
-                    if (numPartitions == UNKNOWN) {
-                        numPartitions = partitions;
-                    } else if (numPartitions != partitions) {
-                        final String[] topics = copartitionGroup.toArray(new String[0]);
-                        Arrays.sort(topics);
-                        throw new org.apache.kafka.streams.errors.TopologyException(String.format("%sTopics not co-partitioned: [%s]", logPrefix, Utils.join(Arrays.asList(topics), ",")));
-                    }
-                }
-            }
-
-            // if all topics for this co-partition group is repartition topics,
-            // then set the number of partitions to be the maximum of the number of partitions.
-            if (numPartitions == UNKNOWN) {
-                for (final Map.Entry entry: allRepartitionTopicsNumPartitions.entrySet()) {
-                    if (copartitionGroup.contains(entry.getKey())) {
-                        final int partitions = entry.getValue().numberOfPartitions();
-                        if (partitions > numPartitions) {
-                            numPartitions = partitions;
-                        }
-                    }
-                }
-            }
-            // enforce co-partitioning restrictions to repartition topics by updating their number of partitions
-            for (final Map.Entry entry : allRepartitionTopicsNumPartitions.entrySet()) {
-                if (copartitionGroup.contains(entry.getKey())) {
-                    entry.getValue().setNumberOfPartitions(numPartitions);
-                }
-            }
-
-        }
-    }
 
     // following functions are for test only
     void setInternalTopicManager(final InternalTopicManager internalTopicManager) {
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
index a7fa1fa96706c..cc448a153b877 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
@@ -57,7 +57,7 @@ public class TaskManager {
     private final StreamThread.AbstractTaskCreator standbyTaskCreator;
     private final StreamsMetadataState streamsMetadataState;
 
-    final Admin adminClient;
+    private final Admin adminClient;
     private DeleteRecordsResult deleteRecordsResult;
 
     // following information is updated during rebalance phase by the partition assignor
@@ -94,6 +94,10 @@ public class TaskManager {
         this.adminClient = adminClient;
     }
 
+    public Admin adminClient() {
+        return adminClient;
+    }
+
     void createTasks(final Collection assignment) {
         if (consumer == null) {
             throw new IllegalStateException(logPrefix + "consumer has not been initialized while adding stream tasks. This should not happen.");
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfo.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfo.java
index 4b2dae2608898..7d812309ddcf4 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfo.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfo.java
@@ -37,13 +37,12 @@
 import java.util.Map;
 import java.util.Set;
 
-public class AssignmentInfo {
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.LATEST_SUPPORTED_VERSION;
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.UNKNOWN;
 
+public class AssignmentInfo {
     private static final Logger log = LoggerFactory.getLogger(AssignmentInfo.class);
 
-    public static final int LATEST_SUPPORTED_VERSION = 5;
-    static final int UNKNOWN = -1;
-
     private final int usedVersion;
     private final int latestSupportedVersion;
     private int errCode;
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorConfiguration.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorConfiguration.java
new file mode 100644
index 0000000000000..5c668cbd8f768
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorConfiguration.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.config.ConfigException;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.internals.QuietStreamsConfig;
+import org.apache.kafka.streams.processor.PartitionGrouper;
+import org.apache.kafka.streams.processor.internals.InternalTopicManager;
+import org.apache.kafka.streams.processor.internals.TaskManager;
+import org.slf4j.Logger;
+
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.kafka.common.utils.Utils.getHost;
+import static org.apache.kafka.common.utils.Utils.getPort;
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.LATEST_SUPPORTED_VERSION;
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.VERSION_ONE;
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.VERSION_TWO;
+
+public final class AssignorConfiguration {
+    private final String logPrefix;
+    private final Logger log;
+    private final Integer numStandbyReplicas;
+    private final PartitionGrouper partitionGrouper;
+    private final String userEndPoint;
+    private final TaskManager taskManager;
+    private final InternalTopicManager internalTopicManager;
+    private final CopartitionedTopicsEnforcer copartitionedTopicsEnforcer;
+    private final StreamsConfig streamsConfig;
+
+    public AssignorConfiguration(final Map configs) {
+        streamsConfig = new QuietStreamsConfig(configs);
+
+        // Setting the logger with the passed in client thread name
+        logPrefix = String.format("stream-thread [%s] ", streamsConfig.getString(CommonClientConfigs.CLIENT_ID_CONFIG));
+        final LogContext logContext = new LogContext(logPrefix);
+        log = logContext.logger(getClass());
+
+        numStandbyReplicas = streamsConfig.getInt(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG);
+
+        partitionGrouper = streamsConfig.getConfiguredInstance(
+            StreamsConfig.PARTITION_GROUPER_CLASS_CONFIG,
+            PartitionGrouper.class
+        );
+
+        final String configuredUserEndpoint = streamsConfig.getString(StreamsConfig.APPLICATION_SERVER_CONFIG);
+        if (configuredUserEndpoint != null && !configuredUserEndpoint.isEmpty()) {
+            try {
+                final String host = getHost(configuredUserEndpoint);
+                final Integer port = getPort(configuredUserEndpoint);
+
+                if (host == null || port == null) {
+                    throw new ConfigException(
+                        String.format(
+                            "%s Config %s isn't in the correct format. Expected a host:port pair but received %s",
+                            logPrefix, StreamsConfig.APPLICATION_SERVER_CONFIG, configuredUserEndpoint
+                        )
+                    );
+                }
+            } catch (final NumberFormatException nfe) {
+                throw new ConfigException(
+                    String.format("%s Invalid port supplied in %s for config %s: %s",
+                                  logPrefix, configuredUserEndpoint, StreamsConfig.APPLICATION_SERVER_CONFIG, nfe)
+                );
+            }
+            userEndPoint = configuredUserEndpoint;
+        } else {
+            userEndPoint = null;
+        }
+
+        final Object o = configs.get(StreamsConfig.InternalConfig.TASK_MANAGER_FOR_PARTITION_ASSIGNOR);
+        if (o == null) {
+            final KafkaException fatalException = new KafkaException("TaskManager is not specified");
+            log.error(fatalException.getMessage(), fatalException);
+            throw fatalException;
+        }
+
+        if (!(o instanceof TaskManager)) {
+            final KafkaException fatalException = new KafkaException(
+                String.format("%s is not an instance of %s", o.getClass().getName(), TaskManager.class.getName())
+            );
+            log.error(fatalException.getMessage(), fatalException);
+            throw fatalException;
+        }
+
+        taskManager = (TaskManager) o;
+
+        internalTopicManager = new InternalTopicManager(taskManager.adminClient(), streamsConfig);
+
+        copartitionedTopicsEnforcer = new CopartitionedTopicsEnforcer(logPrefix);
+    }
+
+    public AtomicInteger getAssignmentErrorCode(final Map configs) {
+        final Object ai = configs.get(StreamsConfig.InternalConfig.ASSIGNMENT_ERROR_CODE);
+        if (ai == null) {
+            final KafkaException fatalException = new KafkaException("assignmentErrorCode is not specified");
+            log.error(fatalException.getMessage(), fatalException);
+            throw fatalException;
+        }
+
+        if (!(ai instanceof AtomicInteger)) {
+            final KafkaException fatalException = new KafkaException(
+                String.format("%s is not an instance of %s", ai.getClass().getName(), AtomicInteger.class.getName())
+            );
+            log.error(fatalException.getMessage(), fatalException);
+            throw fatalException;
+        }
+        return (AtomicInteger) ai;
+    }
+
+    public TaskManager getTaskManager() {
+        return taskManager;
+    }
+
+    public String logPrefix() {
+        return logPrefix;
+    }
+
+    public int configuredMetadataVersion(final int priorVersion) {
+        final String upgradeFrom = streamsConfig.getString(StreamsConfig.UPGRADE_FROM_CONFIG);
+        if (upgradeFrom != null) {
+            switch (upgradeFrom) {
+                case StreamsConfig.UPGRADE_FROM_0100:
+                    log.info(
+                        "Downgrading metadata version from {} to 1 for upgrade from 0.10.0.x.",
+                        LATEST_SUPPORTED_VERSION
+                    );
+                    return VERSION_ONE;
+                case StreamsConfig.UPGRADE_FROM_0101:
+                case StreamsConfig.UPGRADE_FROM_0102:
+                case StreamsConfig.UPGRADE_FROM_0110:
+                case StreamsConfig.UPGRADE_FROM_10:
+                case StreamsConfig.UPGRADE_FROM_11:
+                    log.info(
+                        "Downgrading metadata version from {} to 2 for upgrade from {}.x.",
+                        LATEST_SUPPORTED_VERSION,
+                        upgradeFrom
+                    );
+                    return VERSION_TWO;
+                default:
+                    throw new IllegalArgumentException(
+                        "Unknown configuration value for parameter 'upgrade.from': " + upgradeFrom
+                    );
+            }
+        } else {
+            return priorVersion;
+        }
+    }
+
+    public int getNumStandbyReplicas() {
+        return numStandbyReplicas;
+    }
+
+    public PartitionGrouper getPartitionGrouper() {
+        return partitionGrouper;
+    }
+
+    public String getUserEndPoint() {
+        return userEndPoint;
+    }
+
+    public InternalTopicManager getInternalTopicManager() {
+        return internalTopicManager;
+    }
+
+    public CopartitionedTopicsEnforcer getCopartitionedTopicsEnforcer() {
+        return copartitionedTopicsEnforcer;
+    }
+}
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorError.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorError.java
new file mode 100644
index 0000000000000..3baf4f55d4077
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorError.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+public enum AssignorError {
+    NONE(0),
+    INCOMPLETE_SOURCE_TOPIC_METADATA(1),
+    VERSION_PROBING(2);
+
+    private final int code;
+
+    AssignorError(final int code) {
+        this.code = code;
+    }
+
+    public int code() {
+        return code;
+    }
+
+}
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java
new file mode 100644
index 0000000000000..c277cc6f7cc2c
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import org.apache.kafka.common.Cluster;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.streams.errors.TopologyException;
+import org.apache.kafka.streams.processor.internals.InternalTopicConfig;
+import org.slf4j.Logger;
+
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+
+public class CopartitionedTopicsEnforcer {
+    private final String logPrefix;
+    private final Logger log;
+
+    public CopartitionedTopicsEnforcer(final String logPrefix) {
+        this.logPrefix = logPrefix;
+        final LogContext logContext = new LogContext(logPrefix);
+        log = logContext.logger(getClass());
+    }
+
+    public void enforce(final Set copartitionGroup,
+                        final Map allRepartitionTopicsNumPartitions,
+                        final Cluster metadata) {
+        if (copartitionGroup.isEmpty()) {
+            return;
+        }
+
+        final Map repartitionTopicConfigs =
+            copartitionGroup.stream()
+                            .filter(allRepartitionTopicsNumPartitions::containsKey)
+                            .collect(Collectors.toMap(topic -> topic, allRepartitionTopicsNumPartitions::get));
+
+        final Map nonRepartitionTopicPartitions =
+            copartitionGroup.stream().filter(topic -> !allRepartitionTopicsNumPartitions.containsKey(topic))
+                            .collect(Collectors.toMap(topic -> topic, topic -> {
+                                final Integer partitions = metadata.partitionCountForTopic(topic);
+                                if (partitions == null) {
+                                    final String str = String.format("%sTopic not found: %s", logPrefix, topic);
+                                    log.error(str);
+                                    throw new IllegalStateException(str);
+                                } else {
+                                    return partitions;
+                                }
+                            }));
+
+        final int numPartitionsToUseForRepartitionTopics;
+        if (copartitionGroup.equals(repartitionTopicConfigs.keySet())) {
+            // If all topics for this co-partition group is repartition topics,
+            // then set the number of partitions to be the maximum of the number of partitions.
+            numPartitionsToUseForRepartitionTopics = getMaxPartitions(repartitionTopicConfigs);
+        } else {
+            // Otherwise, use the number of partitions from external topics (which must all be the same)
+            numPartitionsToUseForRepartitionTopics = getSamePartitions(nonRepartitionTopicPartitions);
+
+        }
+
+        // coerce all the repartition topics to use the decided number of partitions.
+        for (final InternalTopicConfig config : repartitionTopicConfigs.values()) {
+            config.setNumberOfPartitions(numPartitionsToUseForRepartitionTopics);
+        }
+    }
+
+    private int getSamePartitions(final Map nonRepartitionTopicsInCopartitionGroup) {
+        final int partitions = nonRepartitionTopicsInCopartitionGroup.values().iterator().next();
+        for (final Map.Entry entry : nonRepartitionTopicsInCopartitionGroup.entrySet()) {
+            if (entry.getValue() != partitions) {
+                final TreeMap sorted = new TreeMap<>(nonRepartitionTopicsInCopartitionGroup);
+                throw new TopologyException(
+                    String.format("%sTopics not co-partitioned: [%s]",
+                                  logPrefix, sorted)
+                );
+            }
+        }
+        return partitions;
+    }
+
+    private int getMaxPartitions(final Map repartitionTopicsInCopartitionGroup) {
+        int maxPartitions = 0;
+
+        for (final InternalTopicConfig config : repartitionTopicsInCopartitionGroup.values()) {
+            final Optional partitions = config.numberOfPartitions();
+            maxPartitions = Integer.max(maxPartitions, partitions.orElse(maxPartitions));
+        }
+        if (maxPartitions <= 0) {
+            throw new IllegalStateException(logPrefix + "Could not validate the copartitioning of topics: " + repartitionTopicsInCopartitionGroup.keySet());
+        }
+        return maxPartitions;
+    }
+
+}
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/StreamsAssignmentProtocolVersions.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/StreamsAssignmentProtocolVersions.java
new file mode 100644
index 0000000000000..b82e24e73d862
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/StreamsAssignmentProtocolVersions.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+public final class StreamsAssignmentProtocolVersions {
+    public static final int UNKNOWN = -1;
+    public static final int VERSION_ONE = 1;
+    public static final int VERSION_TWO = 2;
+    public static final int VERSION_THREE = 3;
+    public static final int VERSION_FOUR = 4;
+    public static final int VERSION_FIVE = 5;
+    public static final int EARLIEST_PROBEABLE_VERSION = VERSION_THREE;
+    public static final int LATEST_SUPPORTED_VERSION = VERSION_FIVE;
+
+    private StreamsAssignmentProtocolVersions() {}
+}
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfo.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfo.java
index 9161527fc50b4..81430ff605311 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfo.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfo.java
@@ -28,13 +28,12 @@
 import java.util.Set;
 import java.util.UUID;
 
-public class SubscriptionInfo {
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.LATEST_SUPPORTED_VERSION;
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.UNKNOWN;
 
+public class SubscriptionInfo {
     private static final Logger log = LoggerFactory.getLogger(SubscriptionInfo.class);
 
-    public static final int LATEST_SUPPORTED_VERSION = 5;
-    static final int UNKNOWN = -1;
-
     private final int usedVersion;
     private final int latestSupportedVersion;
     private UUID processId;
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsValidatorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsEnforcerTest.java
similarity index 73%
rename from streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsValidatorTest.java
rename to streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsEnforcerTest.java
index a272aa5407b79..87177f24f0b5c 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsValidatorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsEnforcerTest.java
@@ -21,20 +21,21 @@
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.streams.errors.TopologyException;
+import org.apache.kafka.streams.processor.internals.assignment.CopartitionedTopicsEnforcer;
 import org.junit.Before;
 import org.junit.Test;
 
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.MatcherAssert.assertThat;
 
-public class CopartitionedTopicsValidatorTest {
+public class CopartitionedTopicsEnforcerTest {
 
-    private final StreamsPartitionAssignor.CopartitionedTopicsValidator validator =
-        new StreamsPartitionAssignor.CopartitionedTopicsValidator("thread");
+    private final CopartitionedTopicsEnforcer validator = new CopartitionedTopicsEnforcer("thread");
     private final Map partitions = new HashMap<>();
     private final Cluster cluster = Cluster.empty();
 
@@ -56,17 +57,17 @@ public void before() {
 
     @Test(expected = IllegalStateException.class)
     public void shouldThrowTopologyBuilderExceptionIfNoPartitionsFoundForCoPartitionedTopic() {
-        validator.validate(Collections.singleton("topic"),
-                           Collections.emptyMap(),
-                           cluster);
+        validator.enforce(Collections.singleton("topic"),
+                          Collections.emptyMap(),
+                          cluster);
     }
 
     @Test(expected = TopologyException.class)
     public void shouldThrowTopologyBuilderExceptionIfPartitionCountsForCoPartitionedTopicsDontMatch() {
         partitions.remove(new TopicPartition("second", 0));
-        validator.validate(Utils.mkSet("first", "second"),
-                           Collections.emptyMap(),
-                           cluster.withPartitions(partitions));
+        validator.enforce(Utils.mkSet("first", "second"),
+                          Collections.emptyMap(),
+                          cluster.withPartitions(partitions));
     }
 
 
@@ -74,11 +75,11 @@ public void shouldThrowTopologyBuilderExceptionIfPartitionCountsForCoPartitioned
     public void shouldEnforceCopartitioningOnRepartitionTopics() {
         final InternalTopicConfig config = createTopicConfig("repartitioned", 10);
 
-        validator.validate(Utils.mkSet("first", "second", config.name()),
-                           Collections.singletonMap(config.name(), config),
-                           cluster.withPartitions(partitions));
+        validator.enforce(Utils.mkSet("first", "second", config.name()),
+                          Collections.singletonMap(config.name(), config),
+                          cluster.withPartitions(partitions));
 
-        assertThat(config.numberOfPartitions(), equalTo(2));
+        assertThat(config.numberOfPartitions(), equalTo(Optional.of(2)));
     }
 
 
@@ -93,16 +94,16 @@ public void shouldSetNumPartitionsToMaximumPartitionsWhenAllTopicsAreRepartition
         repartitionTopicConfig.put(two.name(), two);
         repartitionTopicConfig.put(three.name(), three);
 
-        validator.validate(Utils.mkSet(one.name(),
-                                       two.name(),
-                                       three.name()),
-                           repartitionTopicConfig,
-                           cluster
+        validator.enforce(Utils.mkSet(one.name(),
+                                      two.name(),
+                                      three.name()),
+                          repartitionTopicConfig,
+                          cluster
         );
 
-        assertThat(one.numberOfPartitions(), equalTo(15));
-        assertThat(two.numberOfPartitions(), equalTo(15));
-        assertThat(three.numberOfPartitions(), equalTo(15));
+        assertThat(one.numberOfPartitions(), equalTo(Optional.of(15)));
+        assertThat(two.numberOfPartitions(), equalTo(Optional.of(15)));
+        assertThat(three.numberOfPartitions(), equalTo(Optional.of(15)));
     }
 
     private InternalTopicConfig createTopicConfig(final String repartitionTopic,
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
index 616deaf39040c..2886f51fb5d6c 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
@@ -61,6 +61,7 @@
 
 import static java.time.Duration.ofMillis;
 import static java.util.Arrays.asList;
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.LATEST_SUPPORTED_VERSION;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.not;
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -119,7 +120,7 @@ public class StreamsPartitionAssignorTest {
     private final String userEndPoint = "localhost:8080";
     private final String applicationId = "stream-partition-assignor-test";
 
-    private final TaskManager taskManager = EasyMock.createNiceMock(TaskManager.class);
+    private TaskManager taskManager;
 
     private Map configProps() {
         final Map configurationMap = new HashMap<>();
@@ -136,15 +137,24 @@ private void configurePartitionAssignor(final Map props) {
         partitionAssignor.configure(configurationMap);
     }
 
-    private void mockTaskManager(final Set prevTasks,
-                                 final Set cachedTasks,
-                                 final UUID processId,
-                                 final InternalTopologyBuilder builder) {
+    private void createMockTaskManager() {
+        final StreamsBuilder builder = new StreamsBuilder();
+        final InternalTopologyBuilder internalTopologyBuilder = TopologyWrapper.getInternalTopologyBuilder(builder.build());
+        internalTopologyBuilder.setApplicationId(applicationId);
+
+        createMockTaskManager(emptyTasks, emptyTasks, UUID.randomUUID(), internalTopologyBuilder);
+    }
+
+    private void createMockTaskManager(final Set prevTasks,
+                                       final Set cachedTasks,
+                                       final UUID processId,
+                                       final InternalTopologyBuilder builder) {
+        taskManager = EasyMock.createNiceMock(TaskManager.class);
+        EasyMock.expect(taskManager.adminClient()).andReturn(null).anyTimes();
         EasyMock.expect(taskManager.builder()).andReturn(builder).anyTimes();
         EasyMock.expect(taskManager.prevActiveTaskIds()).andReturn(prevTasks).anyTimes();
         EasyMock.expect(taskManager.cachedTasksIds()).andReturn(cachedTasks).anyTimes();
         EasyMock.expect(taskManager.processId()).andReturn(processId).anyTimes();
-        EasyMock.replay(taskManager);
     }
 
     private Map subscriptions;
@@ -180,7 +190,7 @@ public void shouldInterleaveTasksByGroupId() {
         final List tasks = asList(taskIdC0, taskIdC1, taskIdB0, taskIdB1, taskIdB2, taskIdA0, taskIdA1, taskIdA2, taskIdA3);
         Collections.shuffle(tasks);
 
-        final List> interleavedTaskIds = partitionAssignor.interleaveTasksByGroupId(tasks, 3);
+        final List> interleavedTaskIds = StreamsPartitionAssignor.interleaveTasksByGroupId(tasks, 3);
 
         assertThat(interleavedTaskIds, equalTo(embeddedList));
     }
@@ -198,7 +208,8 @@ public void testSubscription() {
                 new TaskId(0, 2), new TaskId(1, 2), new TaskId(2, 2));
 
         final UUID processId = UUID.randomUUID();
-        mockTaskManager(prevTasks, cachedTasks, processId, builder);
+        createMockTaskManager(prevTasks, cachedTasks, processId, builder);
+        EasyMock.replay(taskManager);
 
         configurePartitionAssignor(Collections.emptyMap());
 
@@ -233,7 +244,8 @@ public void testAssignBasic() {
         final UUID uuid1 = UUID.randomUUID();
         final UUID uuid2 = UUID.randomUUID();
 
-        mockTaskManager(prevTasks10, standbyTasks10, uuid1, builder);
+        createMockTaskManager(prevTasks10, standbyTasks10, uuid1, builder);
+        EasyMock.replay(taskManager);
         configurePartitionAssignor(Collections.emptyMap());
 
         partitionAssignor.setInternalTopicManager(new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer));
@@ -317,7 +329,8 @@ public void shouldAssignEvenlyAcrossConsumersOneClientMultipleThreads() {
 
         final UUID uuid1 = UUID.randomUUID();
 
-        mockTaskManager(new HashSet<>(), new HashSet<>(), uuid1, builder);
+        createMockTaskManager(new HashSet<>(), new HashSet<>(), uuid1, builder);
+        EasyMock.replay(taskManager);
         configurePartitionAssignor(Collections.emptyMap());
 
         partitionAssignor.setInternalTopicManager(new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer));
@@ -361,7 +374,8 @@ public void testAssignWithPartialTopology() {
 
         final UUID uuid1 = UUID.randomUUID();
 
-        mockTaskManager(emptyTasks, emptyTasks, uuid1, builder);
+        createMockTaskManager(emptyTasks, emptyTasks, uuid1, builder);
+        EasyMock.replay(taskManager);
         configurePartitionAssignor(Collections.singletonMap(StreamsConfig.PARTITION_GROUPER_CLASS_CONFIG, SingleGroupPartitionGrouperStub.class));
 
         partitionAssignor.setInternalTopicManager(new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer));
@@ -398,7 +412,8 @@ public void testAssignEmptyMetadata() {
             Collections.emptySet());
         final UUID uuid1 = UUID.randomUUID();
 
-        mockTaskManager(prevTasks10, standbyTasks10, uuid1, builder);
+        createMockTaskManager(prevTasks10, standbyTasks10, uuid1, builder);
+        EasyMock.replay(taskManager);
         configurePartitionAssignor(Collections.emptyMap());
 
         subscriptions.put("consumer10",
@@ -452,7 +467,8 @@ public void testAssignWithNewTasks() {
 
         final UUID uuid1 = UUID.randomUUID();
         final UUID uuid2 = UUID.randomUUID();
-        mockTaskManager(prevTasks10, emptyTasks, uuid1, builder);
+        createMockTaskManager(prevTasks10, emptyTasks, uuid1, builder);
+        EasyMock.replay(taskManager);
         configurePartitionAssignor(Collections.emptyMap());
 
         partitionAssignor.setInternalTopicManager(new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer));
@@ -514,11 +530,8 @@ public void testAssignWithStates() {
         final UUID uuid1 = UUID.randomUUID();
         final UUID uuid2 = UUID.randomUUID();
 
-        mockTaskManager(
-            emptyTasks,
-            emptyTasks,
-            uuid1,
-            builder);
+        createMockTaskManager(emptyTasks, emptyTasks, uuid1, builder);
+        EasyMock.replay(taskManager);
         configurePartitionAssignor(Collections.emptyMap());
 
         partitionAssignor.setInternalTopicManager(new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer));
@@ -605,7 +618,8 @@ public void testAssignWithStandbyReplicas() {
         final UUID uuid1 = UUID.randomUUID();
         final UUID uuid2 = UUID.randomUUID();
 
-        mockTaskManager(prevTasks00, standbyTasks01, uuid1, builder);
+        createMockTaskManager(prevTasks00, standbyTasks01, uuid1, builder);
+        EasyMock.replay(taskManager);
 
         configurePartitionAssignor(Collections.singletonMap(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 1));
 
@@ -655,31 +669,33 @@ public void testAssignWithStandbyReplicas() {
 
     @Test
     public void testOnAssignment() {
-        configurePartitionAssignor(Collections.emptyMap());
+        createMockTaskManager();
+        final Map> hostState = Collections.singletonMap(
+            new HostInfo("localhost", 9090),
+            Utils.mkSet(t3p0, t3p3));
+        taskManager.setPartitionsByHostState(hostState);
+        EasyMock.expectLastCall();
 
-        final List activeTaskList = asList(task0, task3);
         final Map> activeTasks = new HashMap<>();
-        final Map> standbyTasks = new HashMap<>();
-        final Map> hostState = Collections.singletonMap(
-                new HostInfo("localhost", 9090),
-                Utils.mkSet(t3p0, t3p3));
         activeTasks.put(task0, Utils.mkSet(t3p0));
         activeTasks.put(task3, Utils.mkSet(t3p3));
+        final Map> standbyTasks = new HashMap<>();
         standbyTasks.put(task1, Utils.mkSet(t3p1));
         standbyTasks.put(task2, Utils.mkSet(t3p2));
-
-        final AssignmentInfo info = new AssignmentInfo(activeTaskList, standbyTasks, hostState);
-        final ConsumerPartitionAssignor.Assignment assignment = new ConsumerPartitionAssignor.Assignment(asList(t3p0, t3p3), info.encode());
-
-        final Capture capturedCluster = EasyMock.newCapture();
-        taskManager.setPartitionsByHostState(hostState);
-        EasyMock.expectLastCall();
         taskManager.setAssignmentMetadata(activeTasks, standbyTasks);
         EasyMock.expectLastCall();
+
+        final Capture capturedCluster = EasyMock.newCapture();
         taskManager.setClusterMetadata(EasyMock.capture(capturedCluster));
         EasyMock.expectLastCall();
+
         EasyMock.replay(taskManager);
 
+        configurePartitionAssignor(Collections.emptyMap());
+        final List activeTaskList = asList(task0, task3);
+        final AssignmentInfo info = new AssignmentInfo(activeTaskList, standbyTasks, hostState);
+        final ConsumerPartitionAssignor.Assignment assignment = new ConsumerPartitionAssignor.Assignment(asList(t3p0, t3p3), info.encode());
+
         partitionAssignor.onAssignment(assignment, null);
 
         EasyMock.verify(taskManager);
@@ -701,7 +717,8 @@ public void testAssignWithInternalTopics() {
         final Set allTasks = Utils.mkSet(task0, task1, task2);
 
         final UUID uuid1 = UUID.randomUUID();
-        mockTaskManager(emptyTasks, emptyTasks, uuid1, builder);
+        createMockTaskManager(emptyTasks, emptyTasks, uuid1, builder);
+        EasyMock.replay(taskManager);
         configurePartitionAssignor(Collections.emptyMap());
         final MockInternalTopicManager internalTopicManager = new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer);
         partitionAssignor.setInternalTopicManager(internalTopicManager);
@@ -734,7 +751,8 @@ public void testAssignWithInternalTopicThatsSourceIsAnotherInternalTopic() {
         final Set allTasks = Utils.mkSet(task0, task1, task2);
 
         final UUID uuid1 = UUID.randomUUID();
-        mockTaskManager(emptyTasks, emptyTasks, uuid1, builder);
+        createMockTaskManager(emptyTasks, emptyTasks, uuid1, builder);
+        EasyMock.replay(taskManager);
 
         configurePartitionAssignor(Collections.emptyMap());
         final MockInternalTopicManager internalTopicManager = new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer);
@@ -780,11 +798,8 @@ public void shouldGenerateTasksForAllCreatedPartitions() {
         final InternalTopologyBuilder internalTopologyBuilder = TopologyWrapper.getInternalTopologyBuilder(builder.build());
         internalTopologyBuilder.setApplicationId(applicationId);
 
-        mockTaskManager(
-            emptyTasks,
-            emptyTasks,
-            UUID.randomUUID(),
-            internalTopologyBuilder);
+        createMockTaskManager(emptyTasks, emptyTasks, UUID.randomUUID(), internalTopologyBuilder);
+        EasyMock.replay(taskManager);
         configurePartitionAssignor(Collections.emptyMap());
 
         final MockInternalTopicManager mockInternalTopicManager = new MockInternalTopicManager(
@@ -797,7 +812,9 @@ public void shouldGenerateTasksForAllCreatedPartitions() {
                         asList("topic1", "topic3"),
                         new SubscriptionInfo(uuid, emptyTasks, emptyTasks, userEndPoint).encode())
         );
-        final Map assignment = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
+        final Map assignment =
+            partitionAssignor.assign(metadata, new GroupSubscription(subscriptions))
+                             .groupAssignment();
 
         final Map expectedCreatedInternalTopics = new HashMap<>();
         expectedCreatedInternalTopics.put(applicationId + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 4);
@@ -838,11 +855,8 @@ public void shouldAddUserDefinedEndPointToSubscription() {
         builder.addSink("sink", "output", null, null, null, "processor");
 
         final UUID uuid1 = UUID.randomUUID();
-        mockTaskManager(
-            emptyTasks,
-            emptyTasks,
-            uuid1,
-            builder);
+        createMockTaskManager(emptyTasks, emptyTasks, uuid1, builder);
+        EasyMock.replay(taskManager);
         configurePartitionAssignor(Collections.singletonMap(StreamsConfig.APPLICATION_SERVER_CONFIG, userEndPoint));
         final Set topics = Utils.mkSet("input");
         final ConsumerPartitionAssignor.Subscription subscription = new ConsumerPartitionAssignor.Subscription(new ArrayList<>(topics), partitionAssignor.subscriptionUserData(topics));
@@ -861,7 +875,8 @@ public void shouldMapUserEndPointToTopicPartitions() {
 
         final UUID uuid1 = UUID.randomUUID();
 
-        mockTaskManager(emptyTasks, emptyTasks, uuid1, builder);
+        createMockTaskManager(emptyTasks, emptyTasks, uuid1, builder);
+        EasyMock.replay(taskManager);
         configurePartitionAssignor(Collections.singletonMap(StreamsConfig.APPLICATION_SERVER_CONFIG, userEndPoint));
 
         partitionAssignor.setInternalTopicManager(new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer));
@@ -886,7 +901,8 @@ public void shouldMapUserEndPointToTopicPartitions() {
     public void shouldThrowExceptionIfApplicationServerConfigIsNotHostPortPair() {
         builder.setApplicationId(applicationId);
 
-        mockTaskManager(emptyTasks, emptyTasks, UUID.randomUUID(), builder);
+        createMockTaskManager(emptyTasks, emptyTasks, UUID.randomUUID(), builder);
+        EasyMock.replay(taskManager);
         partitionAssignor.setInternalTopicManager(new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer));
 
         try {
@@ -952,11 +968,8 @@ public void shouldNotLoopInfinitelyOnMissingMetadataAndShouldNotCreateRelatedTas
         final InternalTopologyBuilder internalTopologyBuilder = TopologyWrapper.getInternalTopologyBuilder(builder.build());
         internalTopologyBuilder.setApplicationId(applicationId);
 
-        mockTaskManager(
-            emptyTasks,
-            emptyTasks,
-            UUID.randomUUID(),
-            internalTopologyBuilder);
+        createMockTaskManager(emptyTasks, emptyTasks, UUID.randomUUID(), internalTopologyBuilder);
+        EasyMock.replay(taskManager);
         configurePartitionAssignor(Collections.emptyMap());
 
         final MockInternalTopicManager mockInternalTopicManager = new MockInternalTopicManager(
@@ -983,11 +996,12 @@ public void shouldUpdateClusterMetadataAndHostInfoOnAssignment() {
         final Map> hostState = Collections.singletonMap(
                 new HostInfo("localhost", 9090), Utils.mkSet(partitionOne, partitionTwo));
 
-        configurePartitionAssignor(Collections.emptyMap());
-
-        taskManager.setPartitionsByHostState(hostState);
-        EasyMock.expectLastCall();
+        final StreamsBuilder builder = new StreamsBuilder();
+        final InternalTopologyBuilder internalTopologyBuilder = TopologyWrapper.getInternalTopologyBuilder(builder.build());
+        internalTopologyBuilder.setApplicationId(applicationId);
+        createMockTaskManager(emptyTasks, emptyTasks, UUID.randomUUID(), internalTopologyBuilder);
         EasyMock.replay(taskManager);
+        configurePartitionAssignor(Collections.emptyMap());
 
         partitionAssignor.onAssignment(createAssignment(hostState), null);
 
@@ -1004,11 +1018,8 @@ public void shouldNotAddStandbyTaskPartitionsToPartitionsForHost() {
 
 
         final UUID uuid = UUID.randomUUID();
-        mockTaskManager(
-            emptyTasks,
-            emptyTasks,
-            uuid,
-            internalTopologyBuilder);
+        createMockTaskManager(emptyTasks, emptyTasks, uuid, internalTopologyBuilder);
+        EasyMock.replay(taskManager);
 
         final Map props = new HashMap<>();
         props.put(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 1);
@@ -1070,6 +1081,7 @@ public void shouldThrowKafkaExceptionIfTaskMangerConfigIsNotTaskManagerInstance(
 
     @Test
     public void shouldThrowKafkaExceptionAssignmentErrorCodeNotConfigured() {
+        createMockTaskManager();
         final Map config = configProps();
         config.remove(StreamsConfig.InternalConfig.ASSIGNMENT_ERROR_CODE);
 
@@ -1083,6 +1095,7 @@ public void shouldThrowKafkaExceptionAssignmentErrorCodeNotConfigured() {
 
     @Test
     public void shouldThrowKafkaExceptionIfVersionProbingFlagConfigIsNotAtomicInteger() {
+        createMockTaskManager();
         final Map config = configProps();
         config.put(StreamsConfig.InternalConfig.ASSIGNMENT_ERROR_CODE, "i am not an AtomicInteger");
 
@@ -1124,11 +1137,8 @@ private void shouldReturnLowestAssignmentVersionForDifferentSubscriptionVersions
                 )
         );
 
-        mockTaskManager(
-            emptyTasks,
-            emptyTasks,
-            UUID.randomUUID(),
-            builder);
+        createMockTaskManager(emptyTasks, emptyTasks, UUID.randomUUID(), builder);
+        EasyMock.replay(taskManager);
         partitionAssignor.configure(configProps());
         final Map assignment = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
 
@@ -1139,11 +1149,8 @@ private void shouldReturnLowestAssignmentVersionForDifferentSubscriptionVersions
 
     @Test
     public void shouldDownGradeSubscriptionToVersion1() {
-        mockTaskManager(
-            emptyTasks,
-            emptyTasks,
-            UUID.randomUUID(),
-            builder);
+        createMockTaskManager(emptyTasks, emptyTasks, UUID.randomUUID(), builder);
+        EasyMock.replay(taskManager);
         configurePartitionAssignor(Collections.singletonMap(StreamsConfig.UPGRADE_FROM_CONFIG, StreamsConfig.UPGRADE_FROM_0100));
 
         final Set topics = Utils.mkSet("topic1");
@@ -1178,11 +1185,8 @@ public void shouldDownGradeSubscriptionToVersion2For11() {
     }
 
     private void shouldDownGradeSubscriptionToVersion2(final Object upgradeFromValue) {
-        mockTaskManager(
-            emptyTasks,
-            emptyTasks,
-            UUID.randomUUID(),
-            builder);
+        createMockTaskManager(emptyTasks, emptyTasks, UUID.randomUUID(), builder);
+        EasyMock.replay(taskManager);
         configurePartitionAssignor(Collections.singletonMap(StreamsConfig.UPGRADE_FROM_CONFIG, upgradeFromValue));
 
         final Set topics = Utils.mkSet("topic1");
@@ -1216,11 +1220,8 @@ public void shouldReturnUnchangedAssignmentForOldInstancesAndEmptyAssignmentForF
                         encodeFutureSubscription())
         );
 
-        mockTaskManager(
-            allTasks,
-            allTasks,
-            UUID.randomUUID(),
-            builder);
+        createMockTaskManager(allTasks, allTasks, UUID.randomUUID(), builder);
+        EasyMock.replay(taskManager);
         partitionAssignor.configure(configProps());
         final Map assignment = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
 
@@ -1251,8 +1252,8 @@ public void shouldThrowIfV2SubscriptionAndFutureSubscriptionIsMixed() {
     private ByteBuffer encodeFutureSubscription() {
         final ByteBuffer buf = ByteBuffer.allocate(4 /* used version */
                                                    + 4 /* supported version */);
-        buf.putInt(SubscriptionInfo.LATEST_SUPPORTED_VERSION + 1);
-        buf.putInt(SubscriptionInfo.LATEST_SUPPORTED_VERSION + 1);
+        buf.putInt(LATEST_SUPPORTED_VERSION + 1);
+        buf.putInt(LATEST_SUPPORTED_VERSION + 1);
         return buf;
     }
 
@@ -1268,11 +1269,8 @@ private void shouldThrowIfPreVersionProbingSubscriptionAndFutureSubscriptionIsMi
                         encodeFutureSubscription())
         );
 
-        mockTaskManager(
-            emptyTasks,
-            emptyTasks,
-            UUID.randomUUID(),
-            builder);
+        createMockTaskManager(emptyTasks, emptyTasks, UUID.randomUUID(), builder);
+        EasyMock.replay(taskManager);
         partitionAssignor.configure(configProps());
 
         try {
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfoTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfoTest.java
index 0551e1d3ea952..b65994ef3c184 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfoTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfoTest.java
@@ -29,6 +29,8 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.LATEST_SUPPORTED_VERSION;
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.UNKNOWN;
 import static org.junit.Assert.assertEquals;
 
 public class AssignmentInfoTest {
@@ -54,7 +56,7 @@ public class AssignmentInfoTest {
     @Test
     public void shouldUseLatestSupportedVersionByDefault() {
         final AssignmentInfo info = new AssignmentInfo(activeTasks, standbyTasks, globalAssignment);
-        assertEquals(AssignmentInfo.LATEST_SUPPORTED_VERSION, info.version());
+        assertEquals(LATEST_SUPPORTED_VERSION, info.version());
     }
 
     @Test(expected = IllegalArgumentException.class)
@@ -64,41 +66,41 @@ public void shouldThrowForUnknownVersion1() {
 
     @Test(expected = IllegalArgumentException.class)
     public void shouldThrowForUnknownVersion2() {
-        new AssignmentInfo(AssignmentInfo.LATEST_SUPPORTED_VERSION + 1, activeTasks, standbyTasks, globalAssignment, 0);
+        new AssignmentInfo(LATEST_SUPPORTED_VERSION + 1, activeTasks, standbyTasks, globalAssignment, 0);
     }
 
     @Test
     public void shouldEncodeAndDecodeVersion1() {
         final AssignmentInfo info = new AssignmentInfo(1, activeTasks, standbyTasks, globalAssignment, 0);
-        final AssignmentInfo expectedInfo = new AssignmentInfo(1, AssignmentInfo.UNKNOWN, activeTasks, standbyTasks, Collections.>emptyMap(), 0);
+        final AssignmentInfo expectedInfo = new AssignmentInfo(1, UNKNOWN, activeTasks, standbyTasks, Collections.>emptyMap(), 0);
         assertEquals(expectedInfo, AssignmentInfo.decode(info.encode()));
     }
 
     @Test
     public void shouldEncodeAndDecodeVersion2() {
         final AssignmentInfo info = new AssignmentInfo(2, activeTasks, standbyTasks, globalAssignment, 0);
-        final AssignmentInfo expectedInfo = new AssignmentInfo(2, AssignmentInfo.UNKNOWN, activeTasks, standbyTasks, globalAssignment, 0);
+        final AssignmentInfo expectedInfo = new AssignmentInfo(2, UNKNOWN, activeTasks, standbyTasks, globalAssignment, 0);
         assertEquals(expectedInfo, AssignmentInfo.decode(info.encode()));
     }
 
     @Test
     public void shouldEncodeAndDecodeVersion3() {
         final AssignmentInfo info = new AssignmentInfo(3, activeTasks, standbyTasks, globalAssignment, 0);
-        final AssignmentInfo expectedInfo = new AssignmentInfo(3, AssignmentInfo.LATEST_SUPPORTED_VERSION, activeTasks, standbyTasks, globalAssignment, 0);
+        final AssignmentInfo expectedInfo = new AssignmentInfo(3, LATEST_SUPPORTED_VERSION, activeTasks, standbyTasks, globalAssignment, 0);
         assertEquals(expectedInfo, AssignmentInfo.decode(info.encode()));
     }
 
     @Test
     public void shouldEncodeAndDecodeVersion4() {
         final AssignmentInfo info = new AssignmentInfo(4, activeTasks, standbyTasks, globalAssignment, 2);
-        final AssignmentInfo expectedInfo = new AssignmentInfo(4, AssignmentInfo.LATEST_SUPPORTED_VERSION, activeTasks, standbyTasks, globalAssignment, 2);
+        final AssignmentInfo expectedInfo = new AssignmentInfo(4, LATEST_SUPPORTED_VERSION, activeTasks, standbyTasks, globalAssignment, 2);
         assertEquals(expectedInfo, AssignmentInfo.decode(info.encode()));
     }
 
     @Test
     public void shouldEncodeAndDecodeVersion5() {
         final AssignmentInfo info = new AssignmentInfo(5, activeTasks, standbyTasks, globalAssignment, 2);
-        final AssignmentInfo expectedInfo = new AssignmentInfo(5, AssignmentInfo.LATEST_SUPPORTED_VERSION, activeTasks, standbyTasks, globalAssignment, 2);
+        final AssignmentInfo expectedInfo = new AssignmentInfo(5, LATEST_SUPPORTED_VERSION, activeTasks, standbyTasks, globalAssignment, 2);
         assertEquals(expectedInfo, AssignmentInfo.decode(info.encode()));
     }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfoTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfoTest.java
index 2a75c57b237e4..a6150c054c0bd 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfoTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfoTest.java
@@ -25,6 +25,8 @@
 import java.util.Set;
 import java.util.UUID;
 
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.LATEST_SUPPORTED_VERSION;
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.UNKNOWN;
 import static org.junit.Assert.assertEquals;
 
 public class SubscriptionInfoTest {
@@ -42,7 +44,7 @@ public class SubscriptionInfoTest {
     @Test
     public void shouldUseLatestSupportedVersionByDefault() {
         final SubscriptionInfo info = new SubscriptionInfo(processId, activeTasks, standbyTasks, "localhost:80");
-        assertEquals(SubscriptionInfo.LATEST_SUPPORTED_VERSION, info.version());
+        assertEquals(LATEST_SUPPORTED_VERSION, info.version());
     }
 
     @Test(expected = IllegalArgumentException.class)
@@ -52,49 +54,49 @@ public void shouldThrowForUnknownVersion1() {
 
     @Test(expected = IllegalArgumentException.class)
     public void shouldThrowForUnknownVersion2() {
-        new SubscriptionInfo(SubscriptionInfo.LATEST_SUPPORTED_VERSION + 1, processId, activeTasks, standbyTasks, "localhost:80");
+        new SubscriptionInfo(LATEST_SUPPORTED_VERSION + 1, processId, activeTasks, standbyTasks, "localhost:80");
     }
 
     @Test
     public void shouldEncodeAndDecodeVersion1() {
         final SubscriptionInfo info = new SubscriptionInfo(1, processId, activeTasks, standbyTasks, IGNORED_USER_ENDPOINT);
-        final SubscriptionInfo expectedInfo = new SubscriptionInfo(1, SubscriptionInfo.UNKNOWN, processId, activeTasks, standbyTasks, null);
+        final SubscriptionInfo expectedInfo = new SubscriptionInfo(1, UNKNOWN, processId, activeTasks, standbyTasks, null);
         assertEquals(expectedInfo, SubscriptionInfo.decode(info.encode()));
     }
 
     @Test
     public void shouldEncodeAndDecodeVersion2() {
         final SubscriptionInfo info = new SubscriptionInfo(2, processId, activeTasks, standbyTasks, "localhost:80");
-        final SubscriptionInfo expectedInfo = new SubscriptionInfo(2, SubscriptionInfo.UNKNOWN, processId, activeTasks, standbyTasks, "localhost:80");
+        final SubscriptionInfo expectedInfo = new SubscriptionInfo(2, UNKNOWN, processId, activeTasks, standbyTasks, "localhost:80");
         assertEquals(expectedInfo, SubscriptionInfo.decode(info.encode()));
     }
 
     @Test
     public void shouldEncodeAndDecodeVersion3() {
         final SubscriptionInfo info = new SubscriptionInfo(3, processId, activeTasks, standbyTasks, "localhost:80");
-        final SubscriptionInfo expectedInfo = new SubscriptionInfo(3, SubscriptionInfo.LATEST_SUPPORTED_VERSION, processId, activeTasks, standbyTasks, "localhost:80");
+        final SubscriptionInfo expectedInfo = new SubscriptionInfo(3, LATEST_SUPPORTED_VERSION, processId, activeTasks, standbyTasks, "localhost:80");
         assertEquals(expectedInfo, SubscriptionInfo.decode(info.encode()));
     }
 
     @Test
     public void shouldEncodeAndDecodeVersion4() {
         final SubscriptionInfo info = new SubscriptionInfo(4, processId, activeTasks, standbyTasks, "localhost:80");
-        final SubscriptionInfo expectedInfo = new SubscriptionInfo(4, SubscriptionInfo.LATEST_SUPPORTED_VERSION, processId, activeTasks, standbyTasks, "localhost:80");
+        final SubscriptionInfo expectedInfo = new SubscriptionInfo(4, LATEST_SUPPORTED_VERSION, processId, activeTasks, standbyTasks, "localhost:80");
         assertEquals(expectedInfo, SubscriptionInfo.decode(info.encode()));
     }
 
     @Test
     public void shouldAllowToDecodeFutureSupportedVersion() {
         final SubscriptionInfo info = SubscriptionInfo.decode(encodeFutureVersion());
-        assertEquals(SubscriptionInfo.LATEST_SUPPORTED_VERSION + 1, info.version());
-        assertEquals(SubscriptionInfo.LATEST_SUPPORTED_VERSION + 1, info.latestSupportedVersion());
+        assertEquals(LATEST_SUPPORTED_VERSION + 1, info.version());
+        assertEquals(LATEST_SUPPORTED_VERSION + 1, info.latestSupportedVersion());
     }
 
     private ByteBuffer encodeFutureVersion() {
         final ByteBuffer buf = ByteBuffer.allocate(4 /* used version */
                                                    + 4 /* supported version */);
-        buf.putInt(SubscriptionInfo.LATEST_SUPPORTED_VERSION + 1);
-        buf.putInt(SubscriptionInfo.LATEST_SUPPORTED_VERSION + 1);
+        buf.putInt(LATEST_SUPPORTED_VERSION + 1);
+        buf.putInt(LATEST_SUPPORTED_VERSION + 1);
         return buf;
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
index 50a0a1148f930..a4735869c515c 100644
--- a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
@@ -49,12 +49,15 @@
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 import java.util.UUID;
 
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.LATEST_SUPPORTED_VERSION;
+
 public class StreamsUpgradeTest {
 
     @SuppressWarnings("unchecked")
@@ -110,7 +113,7 @@ public Consumer getConsumer(final Map config) {
     public static class FutureStreamsPartitionAssignor extends StreamsPartitionAssignor {
 
         public FutureStreamsPartitionAssignor() {
-            usedSubscriptionMetadataVersion = SubscriptionInfo.LATEST_SUPPORTED_VERSION + 1;
+            usedSubscriptionMetadataVersion = LATEST_SUPPORTED_VERSION + 1;
         }
 
         @Override
@@ -126,7 +129,7 @@ public ByteBuffer subscriptionUserData(final Set topics) {
             standbyTasks.removeAll(previousActiveTasks);
             final FutureSubscriptionInfo data = new FutureSubscriptionInfo(
                 usedSubscriptionMetadataVersion,
-                SubscriptionInfo.LATEST_SUPPORTED_VERSION + 1,
+                LATEST_SUPPORTED_VERSION + 1,
                 taskManager.processId(),
                 previousActiveTasks,
                 standbyTasks,
@@ -156,13 +159,13 @@ public void onAssignment(final ConsumerPartitionAssignor.Assignment assignment,
                 throw new TaskAssignmentException("Failed to decode AssignmentInfo", ex);
             }
 
-            if (usedVersion > AssignmentInfo.LATEST_SUPPORTED_VERSION + 1) {
+            if (usedVersion > LATEST_SUPPORTED_VERSION + 1) {
                 throw new IllegalStateException("Unknown metadata version: " + usedVersion
-                    + "; latest supported version: " + AssignmentInfo.LATEST_SUPPORTED_VERSION + 1);
+                                                    + "; latest supported version: " + LATEST_SUPPORTED_VERSION + 1);
             }
 
             final AssignmentInfo info = AssignmentInfo.decode(
-                assignment.userData().putInt(0, AssignmentInfo.LATEST_SUPPORTED_VERSION));
+                assignment.userData().putInt(0, LATEST_SUPPORTED_VERSION));
 
             final List partitions = new ArrayList<>(assignment.partitions());
             partitions.sort(PARTITION_COMPARATOR);
@@ -173,7 +176,7 @@ public void onAssignment(final ConsumerPartitionAssignor.Assignment assignment,
             final Map topicToPartitionInfo = new HashMap<>();
             final Map> partitionsByHost;
 
-            processLatestVersionAssignment(info, partitions, activeTasks, topicToPartitionInfo);
+            processVersionTwoAssignment("test ", info, partitions, activeTasks, topicToPartitionInfo);
             partitionsByHost = info.partitionsByHost();
 
             final TaskManager taskManager = taskManger();
@@ -186,12 +189,18 @@ public void onAssignment(final ConsumerPartitionAssignor.Assignment assignment,
         @Override
         public GroupAssignment assign(final Cluster metadata, final GroupSubscription groupSubscription) {
             final Map subscriptions = groupSubscription.groupSubscription();
+            final Set supportedVersions = new HashSet<>();
+            for (final Map.Entry entry : subscriptions.entrySet()) {
+                final Subscription subscription = entry.getValue();
+                final SubscriptionInfo info = SubscriptionInfo.decode(subscription.userData());
+                supportedVersions.add(info.latestSupportedVersion());
+            }
             Map assignment = null;
 
             final Map downgradedSubscriptions = new HashMap<>();
             for (final Subscription subscription : subscriptions.values()) {
                 final SubscriptionInfo info = SubscriptionInfo.decode(subscription.userData());
-                if (info.version() < SubscriptionInfo.LATEST_SUPPORTED_VERSION + 1) {
+                if (info.version() < LATEST_SUPPORTED_VERSION + 1) {
                     assignment = super.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
                     break;
                 }
@@ -200,14 +209,14 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
             boolean bumpUsedVersion = false;
             final boolean bumpSupportedVersion;
             if (assignment != null) {
-                bumpSupportedVersion = supportedVersions.size() == 1 && supportedVersions.iterator().next() == SubscriptionInfo.LATEST_SUPPORTED_VERSION + 1;
+                bumpSupportedVersion = supportedVersions.size() == 1 && supportedVersions.iterator().next() == LATEST_SUPPORTED_VERSION + 1;
             } else {
                 for (final Map.Entry entry : subscriptions.entrySet()) {
                     final Subscription subscription = entry.getValue();
 
                     final SubscriptionInfo info = SubscriptionInfo.decode(subscription.userData()
-                        .putInt(0, SubscriptionInfo.LATEST_SUPPORTED_VERSION)
-                        .putInt(4, SubscriptionInfo.LATEST_SUPPORTED_VERSION));
+                        .putInt(0, LATEST_SUPPORTED_VERSION)
+                        .putInt(4, LATEST_SUPPORTED_VERSION));
 
                     downgradedSubscriptions.put(
                         entry.getKey(),
@@ -255,7 +264,7 @@ private static class FutureSubscriptionInfo extends SubscriptionInfo {
         }
 
         public ByteBuffer encode() {
-            if (version() <= SubscriptionInfo.LATEST_SUPPORTED_VERSION) {
+            if (version() <= LATEST_SUPPORTED_VERSION) {
                 final ByteBuffer buf = super.encode();
                 // super.encode() always encodes `LATEST_SUPPORTED_VERSION` as "latest supported version"
                 // need to update to future version
@@ -307,13 +316,13 @@ public ByteBuffer encode() {
             try (final DataOutputStream out = new DataOutputStream(baos)) {
                 if (bumpUsedVersion) {
                     originalUserMetadata.getInt(); // discard original used version
-                    out.writeInt(AssignmentInfo.LATEST_SUPPORTED_VERSION + 1);
+                    out.writeInt(LATEST_SUPPORTED_VERSION + 1);
                 } else {
                     out.writeInt(originalUserMetadata.getInt());
                 }
                 if (bumpSupportedVersion) {
                     originalUserMetadata.getInt(); // discard original supported version
-                    out.writeInt(AssignmentInfo.LATEST_SUPPORTED_VERSION + 1);
+                    out.writeInt(LATEST_SUPPORTED_VERSION + 1);
                 }
 
                 try {
diff --git a/streams/src/test/java/org/apache/kafka/test/MockInternalTopicManager.java b/streams/src/test/java/org/apache/kafka/test/MockInternalTopicManager.java
index 1eebf2479d701..548657ad96a9c 100644
--- a/streams/src/test/java/org/apache/kafka/test/MockInternalTopicManager.java
+++ b/streams/src/test/java/org/apache/kafka/test/MockInternalTopicManager.java
@@ -46,7 +46,7 @@ public MockInternalTopicManager(final StreamsConfig streamsConfig,
     public void makeReady(final Map topics) {
         for (final InternalTopicConfig topic : topics.values()) {
             final String topicName = topic.name();
-            final int numberOfPartitions = topic.numberOfPartitions();
+            final int numberOfPartitions = topic.numberOfPartitions().get();
             readyTopics.put(topicName, numberOfPartitions);
 
             final List partitions = new ArrayList<>();

From 39afe9fe0eea057a53d85ec645ea2dcba54b29f7 Mon Sep 17 00:00:00 2001
From: Scott Hendricks 
Date: Sun, 8 Sep 2019 19:49:13 -0700
Subject: [PATCH 0596/1071] KAFKA-8853; Create sustained connections test for
 Trogdor

This creates a test that generates sustained connections against Kafka.  There
are three different components we can stress with this, KafkaConsumer,
KafkaProducer, and AdminClient.  This test tries use minimal bandwidth per
connection to reduce overhead impacts.

This test works by creating a threadpool that creates connections and then
maintains a central pool of connections at a specified keepalive rate.  The
keepalive action varies by which component is being stressed:

  * KafkaProducer:  Sends one single produce record.  The configuration for
    the produce request uses the same key/value generator as the ProduceBench
    test.

  * KafkaConsumer: Subscribes to a single partition, seeks to the end, and
    then polls a minimal number of records.  Each consumer connection is its
    own consumer group, and defaults to 1024 bytes as FETCH_MAX_BYTES to keep
    traffic to a minimum.

  * AdminClient: Makes an API call to get the nodes in the cluster.

NOTE: This test is designed to be run alongside a ProduceBench test for a
specific topic, due to the way the Consumer test polls a single partition.
There may be no data returned by the consumer test if this is run on its own.
The connection should still be kept alive, but with no data returned.

Author: Scott Hendricks 

Reviewers: Stanislav Kozlovski, Gwen Shapira

Closes #7289 from scott-hendricks/trunk
---
 checkstyle/suppressions.xml                   |   2 +
 .../workload/SustainedConnectionSpec.java     | 196 +++++++
 .../workload/SustainedConnectionWorker.java   | 531 ++++++++++++++++++
 3 files changed, 729 insertions(+)
 create mode 100644 tools/src/main/java/org/apache/kafka/trogdor/workload/SustainedConnectionSpec.java
 create mode 100644 tools/src/main/java/org/apache/kafka/trogdor/workload/SustainedConnectionWorker.java

diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index e2cafa9be2e9c..63eedae14e3c1 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -214,6 +214,8 @@
               files="SignalLogger.java"/>
     
+    
 
     
      producerConf;
+    private final Map consumerConf;
+    private final Map adminClientConf;
+    private final Map commonClientConf;
+    private final PayloadGenerator keyGenerator;
+    private final PayloadGenerator valueGenerator;
+    private final int producerConnectionCount;
+    private final int consumerConnectionCount;
+    private final int metadataConnectionCount;
+    private final String topicName;
+    private final int numThreads;
+    private final int refreshRateMs;
+
+    @JsonCreator
+    public SustainedConnectionSpec(
+            @JsonProperty("startMs") long startMs,
+            @JsonProperty("durationMs") long durationMs,
+            @JsonProperty("clientNode") String clientNode,
+            @JsonProperty("bootstrapServers") String bootstrapServers,
+            @JsonProperty("producerConf") Map producerConf,
+            @JsonProperty("consumerConf") Map consumerConf,
+            @JsonProperty("adminClientConf") Map adminClientConf,
+            @JsonProperty("commonClientConf") Map commonClientConf,
+            @JsonProperty("keyGenerator") PayloadGenerator keyGenerator,
+            @JsonProperty("valueGenerator") PayloadGenerator valueGenerator,
+            @JsonProperty("producerConnectionCount") int producerConnectionCount,
+            @JsonProperty("consumerConnectionCount") int consumerConnectionCount,
+            @JsonProperty("metadataConnectionCount") int metadataConnectionCount,
+            @JsonProperty("topicName") String topicName,
+            @JsonProperty("numThreads") int numThreads,
+            @JsonProperty("refreshRateMs") int refreshRateMs) {
+        super(startMs, durationMs);
+        this.clientNode = clientNode == null ? "" : clientNode;
+        this.bootstrapServers = (bootstrapServers == null) ? "" : bootstrapServers;
+        this.producerConf = configOrEmptyMap(producerConf);
+        this.consumerConf = configOrEmptyMap(consumerConf);
+        this.adminClientConf = configOrEmptyMap(adminClientConf);
+        this.commonClientConf = configOrEmptyMap(commonClientConf);
+        this.keyGenerator = keyGenerator;
+        this.valueGenerator = valueGenerator;
+        this.producerConnectionCount = producerConnectionCount;
+        this.consumerConnectionCount = consumerConnectionCount;
+        this.metadataConnectionCount = metadataConnectionCount;
+        this.topicName = topicName;
+        this.numThreads = numThreads < 1 ? 1 : numThreads;
+        this.refreshRateMs = refreshRateMs < 1 ? 1 : refreshRateMs;
+    }
+
+    @JsonProperty
+    public String clientNode() {
+        return clientNode;
+    }
+
+    @JsonProperty
+    public String bootstrapServers() {
+        return bootstrapServers;
+    }
+
+    @JsonProperty
+    public Map producerConf() {
+        return producerConf;
+    }
+
+    @JsonProperty
+    public Map consumerConf() {
+        return consumerConf;
+    }
+
+    @JsonProperty
+    public Map adminClientConf() {
+        return adminClientConf;
+    }
+
+    @JsonProperty
+    public Map commonClientConf() {
+        return commonClientConf;
+    }
+
+    @JsonProperty
+    public PayloadGenerator keyGenerator() {
+        return keyGenerator;
+    }
+
+    @JsonProperty
+    public PayloadGenerator valueGenerator() {
+        return valueGenerator;
+    }
+
+    @JsonProperty
+    public int producerConnectionCount() {
+        return producerConnectionCount;
+    }
+
+    @JsonProperty
+    public int consumerConnectionCount() {
+        return consumerConnectionCount;
+    }
+
+    @JsonProperty
+    public int metadataConnectionCount() {
+        return metadataConnectionCount;
+    }
+
+    @JsonProperty
+    public String topicName() {
+        return topicName;
+    }
+
+    @JsonProperty
+    public int numThreads() {
+        return numThreads;
+    }
+
+    @JsonProperty
+    public int refreshRateMs() {
+        return refreshRateMs;
+    }
+
+    public TaskController newController(String id) {
+        return topology -> Collections.singleton(clientNode);
+    }
+
+    @Override
+    public TaskWorker newTaskWorker(String id) {
+        return new SustainedConnectionWorker(id, this);
+    }
+}
diff --git a/tools/src/main/java/org/apache/kafka/trogdor/workload/SustainedConnectionWorker.java b/tools/src/main/java/org/apache/kafka/trogdor/workload/SustainedConnectionWorker.java
new file mode 100644
index 0000000000000..6ba9916e6d0f0
--- /dev/null
+++ b/tools/src/main/java/org/apache/kafka/trogdor/workload/SustainedConnectionWorker.java
@@ -0,0 +1,531 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.trogdor.workload;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.JsonNode;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.internals.KafkaFutureImpl;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.apache.kafka.common.utils.SystemTime;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.trogdor.common.JsonUtil;
+import org.apache.kafka.trogdor.common.Platform;
+import org.apache.kafka.trogdor.common.ThreadUtils;
+import org.apache.kafka.trogdor.common.WorkerUtils;
+import org.apache.kafka.trogdor.task.TaskWorker;
+import org.apache.kafka.trogdor.task.WorkerStatusTracker;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+
+public class SustainedConnectionWorker implements TaskWorker {
+    private static final Logger log = LoggerFactory.getLogger(SustainedConnectionWorker.class);
+    private static final SystemTime SYSTEM_TIME = new SystemTime();
+
+    // This is the metadata for the test itself.
+    private final String id;
+    private final SustainedConnectionSpec spec;
+
+    // These variables are used to maintain the connections.
+    private static final int BACKOFF_PERIOD_MS = 10;
+    private ExecutorService workerExecutor;
+    private final AtomicBoolean running = new AtomicBoolean(false);
+    private KafkaFutureImpl doneFuture;
+    private ArrayList connections;
+
+    // These variables are used when tracking the reported status of the worker.
+    private static final int REPORT_INTERVAL_MS = 5000;
+    private WorkerStatusTracker status;
+    private AtomicLong totalProducerConnections;
+    private AtomicLong totalProducerFailedConnections;
+    private AtomicLong totalConsumerConnections;
+    private AtomicLong totalConsumerFailedConnections;
+    private AtomicLong totalMetadataConnections;
+    private AtomicLong totalMetadataFailedConnections;
+    private AtomicLong totalAbortedThreads;
+    private Future statusUpdaterFuture;
+    private ScheduledExecutorService statusUpdaterExecutor;
+
+    public SustainedConnectionWorker(String id, SustainedConnectionSpec spec) {
+        this.id = id;
+        this.spec = spec;
+    }
+
+    @Override
+    public void start(Platform platform, WorkerStatusTracker status,
+                      KafkaFutureImpl doneFuture) throws Exception {
+        if (!running.compareAndSet(false, true)) {
+            throw new IllegalStateException("SustainedConnectionWorker is already running.");
+        }
+        log.info("{}: Activating SustainedConnectionWorker with {}", this.id, this.spec);
+        this.doneFuture = doneFuture;
+        this.status = status;
+        this.connections = new ArrayList<>();
+
+        // Initialize all status reporting metrics to 0.
+        this.totalProducerConnections = new AtomicLong(0);
+        this.totalProducerFailedConnections = new AtomicLong(0);
+        this.totalConsumerConnections = new AtomicLong(0);
+        this.totalConsumerFailedConnections = new AtomicLong(0);
+        this.totalMetadataConnections = new AtomicLong(0);
+        this.totalMetadataFailedConnections = new AtomicLong(0);
+        this.totalAbortedThreads = new AtomicLong(0);
+
+        // Create the worker classes and add them to the list of items to act on.
+        for (int i = 0; i < this.spec.producerConnectionCount(); i++) {
+            this.connections.add(new ProducerSustainedConnection());
+        }
+        for (int i = 0; i < this.spec.consumerConnectionCount(); i++) {
+            this.connections.add(new ConsumerSustainedConnection());
+        }
+        for (int i = 0; i < this.spec.metadataConnectionCount(); i++) {
+            this.connections.add(new MetadataSustainedConnection());
+        }
+
+        // Create the status reporter thread and schedule it.
+        this.statusUpdaterExecutor = Executors.newScheduledThreadPool(1,
+            ThreadUtils.createThreadFactory("StatusUpdaterWorkerThread%d", false));
+        this.statusUpdaterFuture = this.statusUpdaterExecutor.scheduleAtFixedRate(
+            new StatusUpdater(), 0, REPORT_INTERVAL_MS, TimeUnit.MILLISECONDS);
+
+        // Create the maintainer pool, add all the maintainer threads, then start it.
+        this.workerExecutor = Executors.newFixedThreadPool(spec.numThreads(),
+            ThreadUtils.createThreadFactory("SustainedConnectionWorkerThread%d", false));
+        for (int i = 0; i < this.spec.numThreads(); i++) {
+            this.workerExecutor.submit(new MaintainLoop());
+        }
+    }
+
+    private interface SustainedConnection extends AutoCloseable {
+        boolean needsRefresh(long milliseconds);
+        void refresh();
+        void claim();
+    }
+
+    private abstract class ClaimableConnection implements SustainedConnection {
+
+        protected long nextUpdate = 0;
+        protected boolean inUse = false;
+        protected long refreshRate;
+
+        @Override
+        public boolean needsRefresh(long milliseconds) {
+            return !this.inUse && (milliseconds > this.nextUpdate);
+        }
+
+        @Override
+        public void claim() {
+            this.inUse = true;
+        }
+
+        @Override
+        public void close() throws Exception {
+            this.closeQuietly();
+        }
+
+        protected void completeRefresh() {
+            this.nextUpdate = SustainedConnectionWorker.SYSTEM_TIME.milliseconds() + this.refreshRate;
+            this.inUse = false;
+        }
+
+        protected abstract void closeQuietly();
+
+    }
+
+    private class MetadataSustainedConnection extends ClaimableConnection {
+
+        private AdminClient client;
+        private final Properties props;
+
+        MetadataSustainedConnection() {
+
+            // These variables are used to maintain the connection itself.
+            this.client = null;
+            this.refreshRate = SustainedConnectionWorker.this.spec.refreshRateMs();
+
+            // This variable is used to maintain the connection properties.
+            this.props = new Properties();
+            this.props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, SustainedConnectionWorker.this.spec.bootstrapServers());
+            WorkerUtils.addConfigsToProperties(
+                    this.props, SustainedConnectionWorker.this.spec.commonClientConf(), SustainedConnectionWorker.this.spec.commonClientConf());
+        }
+
+        @Override
+        public void refresh() {
+            try {
+                if (this.client == null) {
+                    // Housekeeping to track the number of opened connections.
+                    SustainedConnectionWorker.this.totalMetadataConnections.incrementAndGet();
+
+                    // Create the admin client connection.
+                    this.client = AdminClient.create(this.props);
+                }
+
+                // Fetch some metadata to keep the connection alive.
+                this.client.describeCluster().nodes().get();
+
+            } catch (Throwable e) {
+                // Set the admin client to be recreated on the next cycle.
+                this.closeQuietly();
+
+                // Housekeeping to track the number of opened connections and failed connection attempts.
+                SustainedConnectionWorker.this.totalMetadataConnections.decrementAndGet();
+                SustainedConnectionWorker.this.totalMetadataFailedConnections.incrementAndGet();
+                SustainedConnectionWorker.log.error("Error while refreshing sustained AdminClient connection", e);
+            }
+
+            // Schedule this again and set to not in use.
+            this.completeRefresh();
+        }
+
+        @Override
+        protected void closeQuietly() {
+            Utils.closeQuietly(this.client, "AdminClient");
+            this.client = null;
+        }
+    }
+
+    private class ProducerSustainedConnection extends ClaimableConnection {
+
+        private KafkaProducer producer;
+        private List partitions;
+        private Iterator partitionsIterator;
+        private final String topicName;
+        private final PayloadIterator keys;
+        private final PayloadIterator values;
+        private final Properties props;
+
+        ProducerSustainedConnection() {
+
+            // These variables are used to maintain the connection itself.
+            this.producer = null;
+            this.partitions = null;
+            this.topicName = SustainedConnectionWorker.this.spec.topicName();
+            this.partitionsIterator = null;
+            this.keys = new PayloadIterator(SustainedConnectionWorker.this.spec.keyGenerator());
+            this.values = new PayloadIterator(SustainedConnectionWorker.this.spec.valueGenerator());
+            this.refreshRate = SustainedConnectionWorker.this.spec.refreshRateMs();
+
+            // This variable is used to maintain the connection properties.
+            this.props = new Properties();
+            this.props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, SustainedConnectionWorker.this.spec.bootstrapServers());
+            WorkerUtils.addConfigsToProperties(
+                    this.props, SustainedConnectionWorker.this.spec.commonClientConf(), SustainedConnectionWorker.this.spec.producerConf());
+        }
+
+        @Override
+        public void refresh() {
+            try {
+                if (this.producer == null) {
+                    // Housekeeping to track the number of opened connections.
+                    SustainedConnectionWorker.this.totalProducerConnections.incrementAndGet();
+
+                    // Create the producer, fetch the specified topic's partitions and randomize them.
+                    this.producer = new KafkaProducer<>(this.props, new ByteArraySerializer(), new ByteArraySerializer());
+                    this.partitions = this.producer.partitionsFor(this.topicName).stream()
+                             .map(partitionInfo -> new TopicPartition(partitionInfo.topic(), partitionInfo.partition()))
+                             .collect(Collectors.toList());
+                    Collections.shuffle(this.partitions);
+                }
+
+                // Create a new iterator over the partitions if the current one doesn't exist or is exhausted.
+                if (this.partitionsIterator == null || !this.partitionsIterator.hasNext()) {
+                    this.partitionsIterator = this.partitions.iterator();
+                }
+
+                // Produce a single record and send it synchronously.
+                TopicPartition partition = this.partitionsIterator.next();
+                ProducerRecord record = new ProducerRecord<>(
+                        partition.topic(), partition.partition(), keys.next(), values.next());
+                producer.send(record).get();
+
+            } catch (Throwable e) {
+                // Set the producer to be recreated on the next cycle.
+                this.closeQuietly();
+
+                // Housekeeping to track the number of opened connections and failed connection attempts.
+                SustainedConnectionWorker.this.totalProducerConnections.decrementAndGet();
+                SustainedConnectionWorker.this.totalProducerFailedConnections.incrementAndGet();
+                SustainedConnectionWorker.log.error("Error while refreshing sustained KafkaProducer connection", e);
+
+            }
+
+            // Schedule this again and set to not in use.
+            this.completeRefresh();
+        }
+
+        @Override
+        protected void closeQuietly() {
+            Utils.closeQuietly(this.producer, "KafkaProducer");
+            this.producer = null;
+            this.partitions = null;
+            this.partitionsIterator = null;
+        }
+    }
+
+    private class ConsumerSustainedConnection extends ClaimableConnection {
+
+        private KafkaConsumer consumer;
+        private TopicPartition activePartition;
+        private final String topicName;
+        private final Random rand;
+        private final Properties props;
+
+        ConsumerSustainedConnection() {
+            // These variables are used to maintain the connection itself.
+            this.topicName = SustainedConnectionWorker.this.spec.topicName();
+            this.consumer = null;
+            this.activePartition = null;
+            this.rand = new Random();
+            this.refreshRate = SustainedConnectionWorker.this.spec.refreshRateMs();
+
+            // This variable is used to maintain the connection properties.
+            this.props = new Properties();
+            this.props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, SustainedConnectionWorker.this.spec.bootstrapServers());
+            this.props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");
+            this.props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 1);
+            this.props.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, 1024);
+            WorkerUtils.addConfigsToProperties(
+                    this.props, SustainedConnectionWorker.this.spec.commonClientConf(), SustainedConnectionWorker.this.spec.consumerConf());
+        }
+
+        @Override
+        public void refresh() {
+            try {
+
+                if (this.consumer == null) {
+
+                    // Housekeeping to track the number of opened connections.
+                    SustainedConnectionWorker.this.totalConsumerConnections.incrementAndGet();
+
+                    // Create the consumer and fetch the partitions for the specified topic.
+                    this.consumer = new KafkaConsumer<>(this.props, new ByteArrayDeserializer(), new ByteArrayDeserializer());
+                    List partitions = this.consumer.partitionsFor(this.topicName).stream()
+                            .map(partitionInfo -> new TopicPartition(partitionInfo.topic(), partitionInfo.partition()))
+                            .collect(Collectors.toList());
+
+                    // Select a random partition and assign it.
+                    this.activePartition = partitions.get(this.rand.nextInt(partitions.size()));
+                    this.consumer.assign(Collections.singletonList(this.activePartition));
+                }
+
+                // The behavior when passing in an empty list is to seek to the end of all subscribed partitions.
+                this.consumer.seekToEnd(Collections.emptyList());
+
+                // Poll to keep the connection alive, ignoring any records returned.
+                this.consumer.poll(Duration.ofMillis(50));
+
+            } catch (Throwable e) {
+
+                // Set the consumer to be recreated on the next cycle.
+                this.closeQuietly();
+
+                // Housekeeping to track the number of opened connections and failed connection attempts.
+                SustainedConnectionWorker.this.totalConsumerConnections.decrementAndGet();
+                SustainedConnectionWorker.this.totalConsumerFailedConnections.incrementAndGet();
+                SustainedConnectionWorker.log.error("Error while refreshing sustained KafkaConsumer connection", e);
+
+            }
+
+            // Schedule this again and set to not in use.
+            this.completeRefresh();
+        }
+
+        @Override
+        protected void closeQuietly() {
+            Utils.closeQuietly(this.consumer, "KafkaConsumer");
+            this.consumer = null;
+            this.activePartition = null;
+        }
+    }
+
+    public class MaintainLoop implements Runnable {
+        @Override
+        public void run() {
+            try {
+                while (!doneFuture.isDone()) {
+                    Optional currentConnection = SustainedConnectionWorker.this.findConnectionToMaintain();
+                    if (currentConnection.isPresent()) {
+                        currentConnection.get().refresh();
+                    } else {
+                        SustainedConnectionWorker.SYSTEM_TIME.sleep(SustainedConnectionWorker.BACKOFF_PERIOD_MS);
+                    }
+                }
+            } catch (Exception e) {
+                SustainedConnectionWorker.this.totalAbortedThreads.incrementAndGet();
+                SustainedConnectionWorker.log.error("Aborted thread while maintaining sustained connections", e);
+            }
+        }
+    }
+
+    private synchronized Optional findConnectionToMaintain() {
+        final long milliseconds = SustainedConnectionWorker.SYSTEM_TIME.milliseconds();
+        for (SustainedConnection connection : this.connections) {
+            if (connection.needsRefresh(milliseconds)) {
+                connection.claim();
+                return Optional.of(connection);
+            }
+        }
+        return Optional.empty();
+    }
+
+    private class StatusUpdater implements Runnable {
+        @Override
+        public void run() {
+            try {
+                JsonNode node = JsonUtil.JSON_SERDE.valueToTree(
+                    new StatusData(
+                            SustainedConnectionWorker.this.totalProducerConnections.get(),
+                            SustainedConnectionWorker.this.totalProducerFailedConnections.get(),
+                            SustainedConnectionWorker.this.totalConsumerConnections.get(),
+                            SustainedConnectionWorker.this.totalConsumerFailedConnections.get(),
+                            SustainedConnectionWorker.this.totalMetadataConnections.get(),
+                            SustainedConnectionWorker.this.totalMetadataFailedConnections.get(),
+                            SustainedConnectionWorker.this.totalAbortedThreads.get(),
+                            SustainedConnectionWorker.SYSTEM_TIME.milliseconds()));
+                status.update(node);
+            } catch (Exception e) {
+                SustainedConnectionWorker.log.error("Aborted test while running StatusUpdater", e);
+                WorkerUtils.abort(log, "StatusUpdater", e, doneFuture);
+            }
+        }
+    }
+
+    public static class StatusData {
+        private final long totalProducerConnections;
+        private final long totalProducerFailedConnections;
+        private final long totalConsumerConnections;
+        private final long totalConsumerFailedConnections;
+        private final long totalMetadataConnections;
+        private final long totalMetadataFailedConnections;
+        private final long totalAbortedThreads;
+        private final long updatedMs;
+
+        @JsonCreator
+        StatusData(@JsonProperty("totalProducerConnections") long totalProducerConnections,
+                   @JsonProperty("totalProducerFailedConnections") long totalProducerFailedConnections,
+                   @JsonProperty("totalConsumerConnections") long totalConsumerConnections,
+                   @JsonProperty("totalConsumerFailedConnections") long totalConsumerFailedConnections,
+                   @JsonProperty("totalMetadataConnections") long totalMetadataConnections,
+                   @JsonProperty("totalMetadataFailedConnections") long totalMetadataFailedConnections,
+                   @JsonProperty("totalAbortedThreads") long totalAbortedThreads,
+                   @JsonProperty("updatedMs") long updatedMs) {
+            this.totalProducerConnections = totalProducerConnections;
+            this.totalProducerFailedConnections = totalProducerFailedConnections;
+            this.totalConsumerConnections = totalConsumerConnections;
+            this.totalConsumerFailedConnections = totalConsumerFailedConnections;
+            this.totalMetadataConnections = totalMetadataConnections;
+            this.totalMetadataFailedConnections = totalMetadataFailedConnections;
+            this.totalAbortedThreads = totalAbortedThreads;
+            this.updatedMs = updatedMs;
+        }
+
+        @JsonProperty
+        public long totalProducerConnections() {
+            return totalProducerConnections;
+        }
+
+        @JsonProperty
+        public long totalProducerFailedConnections() {
+            return totalProducerFailedConnections;
+        }
+
+        @JsonProperty
+        public long totalConsumerConnections() {
+            return totalConsumerConnections;
+        }
+
+        @JsonProperty
+        public long totalConsumerFailedConnections() {
+            return totalConsumerFailedConnections;
+        }
+
+        @JsonProperty
+        public long totalMetadataConnections() {
+            return totalMetadataConnections;
+        }
+
+        @JsonProperty
+        public long totalMetadataFailedConnections() {
+            return totalMetadataFailedConnections;
+        }
+
+        @JsonProperty
+        public long totalAbortedThreads() {
+            return totalAbortedThreads;
+        }
+
+        @JsonProperty
+        public long updatedMs() {
+            return updatedMs;
+        }
+    }
+
+    @Override
+    public void stop(Platform platform) throws Exception {
+        if (!running.compareAndSet(true, false)) {
+            throw new IllegalStateException("SustainedConnectionWorker is not running.");
+        }
+        log.info("{}: Deactivating SustainedConnectionWorker.", this.id);
+
+        // Shut down the periodic status updater and perform a final update on the
+        // statistics.  We want to do this first, before deactivating any threads.
+        // Otherwise, if some threads take a while to terminate, this could lead
+        // to a misleading rate getting reported.
+        this.statusUpdaterFuture.cancel(false);
+        this.statusUpdaterExecutor.shutdown();
+        this.statusUpdaterExecutor.awaitTermination(1, TimeUnit.HOURS);
+        this.statusUpdaterExecutor = null;
+        new StatusUpdater().run();
+
+        doneFuture.complete("");
+        for (SustainedConnection connection : this.connections) {
+            connection.close();
+        }
+        workerExecutor.shutdownNow();
+        workerExecutor.awaitTermination(1, TimeUnit.HOURS);
+        this.workerExecutor = null;
+        this.status = null;
+        this.connections = null;
+    }
+}

From 312e4db5902a2d18d7e1df4932608fe789cec6d0 Mon Sep 17 00:00:00 2001
From: Vikas Singh <50422828+soondenana@users.noreply.github.com>
Date: Mon, 9 Sep 2019 07:57:29 -0700
Subject: [PATCH 0597/1071] MINOR. implement --expose-ports option in ducker-ak
 (#7269)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This change adds a command line option to the `ducker-ak up' command to enable exposing ports from docker containers. The exposed ports will be mapped to the ephemeral ports on the host. The option is called `expose-ports' and can take either a single value (like 5005) or a range (like 5005-5009). This port will then exposed from each docker container that ducker-ak sets up.

Reviewers: Colin P. McCabe , José Armando García Sancio 
---
 tests/README.md        | 35 +++++++++++++++++++++++++++++++++++
 tests/docker/ducker-ak | 21 ++++++++++++++++++---
 2 files changed, 53 insertions(+), 3 deletions(-)

diff --git a/tests/README.md b/tests/README.md
index cf355e5624c70..2fce5c31c5912 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -47,6 +47,41 @@ bash tests/docker/ducker-ak up -j 'openjdk:11'; tests/docker/run_tests.sh
   - The docker containers are named knode01, knode02 etc.
    These nodes can't be used for any other purpose.
 
+* Exposing ports using --expose-ports option of `ducker-ak up` command
+
+    If `--expose-ports` is specified then we will expose those ports to random ephemeral ports
+    on the host. The argument can be a single port (like 5005), a port range like (5005-5009)
+    or a combination of port/port-range separated by comma (like 2181,9092 or 2181,5005-5008).
+    By default no port is exposed.
+    
+    The exposed port mapping can be seen by executing `docker ps` command. The PORT column
+    of the output shows the mapping like this (maps port 33891 on host to port 2182 in container):
+
+    0.0.0.0:33891->2182/tcp
+
+    Behind the scene Docker is setting up a DNAT rule for the mapping and it is visible in
+    the DOCKER section of iptables command (`sudo iptables -t nat -L -n`), something like:
+
+    
            DNAT       tcp  --  0.0.0.0/0      0.0.0.0/0      tcp       dpt:33882       to:172.22.0.2:9092
            
+
+    The exposed port(s) are useful to attach a remote debugger to the process running
+    in the docker image. For example if port 5005 was exposed and is mapped to an ephemeral
+    port (say 33891), then a debugger attaching to port 33891 on host will be connecting to
+    a debug session started at port 5005 in the docker image. As an example, for above port
+    numbers, run following commands in the docker image (say by ssh using `./docker/ducker-ak ssh ducker02`):
+
+    > $ export KAFKA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005"
+    
+    > $ /opt/kafka-dev/bin/kafka-topics.sh --bootstrap-server ducker03:9095 --topic __consumer_offsets --describe
+
+    This will run the TopicCommand to describe the __consumer-offset topic. The java process
+    will stop and wait for debugger to attach as `suspend=y` option was specified. Now starting
+    a debugger on host with host `localhost` and following parameter as JVM setting:
+
+    `-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=33891`
+
+    will attach it to the TopicCommand process running in the docker image.
+
 Examining CI run
 ----------------
 * Set BUILD_ID is travis ci's build id. E.g. build id is 169519874 for the following build
diff --git a/tests/docker/ducker-ak b/tests/docker/ducker-ak
index 2a4e5f6b84d16..a29395ee8c737 100755
--- a/tests/docker/ducker-ak
+++ b/tests/docker/ducker-ak
@@ -61,7 +61,7 @@ help|-h|--help
     Display this help message
 
 up [-n|--num-nodes NUM_NODES] [-f|--force] [docker-image]
-        [-C|--custom-ducktape DIR]
+        [-C|--custom-ducktape DIR] [-e|--expose-ports ports]
     Bring up a cluster with the specified amount of nodes (defaults to ${default_num_nodes}).
     The docker image name defaults to ${default_image_name}.  If --force is specified, we will
     attempt to bring up an image even some parameters are not valid.
@@ -70,6 +70,11 @@ up [-n|--num-nodes NUM_NODES] [-f|--force] [docker-image]
     ducktape source code directory before bringing up the nodes.  The provided
     directory should be the ducktape git repo, not the ducktape installed module directory.
 
+    if --expose-ports is specified then we will expose those ports to random ephemeral ports
+    on the host. The argument can be a single port (like 5005), a port range like (5005-5009)
+    or a combination of port/port-range separated by comma (like 2181,9092 or 2181,5005-5008).
+    By default no port is exposed. See README.md for more detail on this option.
+
 test [test-name(s)]
     Run a test or set of tests inside the currently active Ducker nodes.
     For example, to run the system test produce_bench_test, you would run:
@@ -235,12 +240,21 @@ $((${duration} % 60))s.  See ${build_log} for details."
 docker_run() {
     local node=${1}
     local image_name=${2}
+    local ports_option=${3}
+
+    local expose_ports=""
+    if [[ -n ${ports_option} ]]; then
+        expose_ports="-P"
+        for expose_port in ${ports_option//,/ }; do
+            expose_ports="${expose_ports} --expose ${expose_port}"
+        done
+    fi
 
     # Invoke docker-run. We need privileged mode to be able to run iptables
     # and mount FUSE filesystems inside the container.  We also need it to
     # run iptables inside the container.
     must_do -v docker run --privileged \
-        -d -t -h "${node}" --network ducknet \
+        -d -t -h "${node}" --network ducknet "${expose_ports}" \
         --memory=${docker_run_memory_limit} --memory-swappiness=1 \
         -v "${kafka_dir}:/opt/kafka-dev" --name "${node}" -- "${image_name}"
 }
@@ -269,6 +283,7 @@ ducker_up() {
             -f|--force) force=1; shift;;
             -n|--num-nodes) set_once num_nodes "${2}" "number of nodes"; shift 2;;
             -j|--jdk) set_once jdk_version "${2}" "the OpenJDK base image"; shift 2;;
+            -e|--expose-ports) set_once expose_ports "${2}" "the ports to expose"; shift 2;;
             *) set_once image_name "${1}" "docker image name"; shift;;
         esac
     done
@@ -322,7 +337,7 @@ attempting to start new ones."
     fi
     for n in $(seq -f %02g 1 ${num_nodes}); do
         local node="ducker${n}"
-        docker_run "${node}" "${image_name}"
+        docker_run "${node}" "${image_name}" "${expose_ports}"
     done
     mkdir -p "${ducker_dir}/build"
     exec 3<> "${ducker_dir}/build/node_hosts"

From e59e4caadc48ee3ba3325d98cbc1fc714ced2b8e Mon Sep 17 00:00:00 2001
From: Boyang Chen 
Date: Mon, 9 Sep 2019 13:06:47 -0700
Subject: [PATCH 0598/1071] KAFKA-8222 & KIP-345 part 5: admin request to batch
 remove members (#7122)

This PR adds supporting features for static membership. It could batch remove consumers from the group with provided group.instance.id list.

Reviewers: Guozhang Wang 
---
 .../org/apache/kafka/clients/admin/Admin.java |  12 ++
 .../kafka/clients/admin/KafkaAdminClient.java |  67 +++++++-
 .../clients/admin/MembershipChangeResult.java |  49 ++++++
 .../RemoveMemberFromConsumerGroupOptions.java |  50 ++++++
 .../admin/RemoveMemberFromGroupResult.java    |  85 ++++++++++
 .../common/requests/LeaveGroupResponse.java   |   4 +
 .../common/message/LeaveGroupResponse.json    |   2 +-
 .../clients/admin/KafkaAdminClientTest.java   | 133 +++++++++++++--
 .../admin/MembershipChangeResultTest.java     |  50 ++++++
 .../kafka/clients/admin/MockAdminClient.java  |   5 +
 ...oveMemberFromConsumerGroupOptionsTest.java |  38 +++++
 .../RemoveMemberFromGroupResultTest.java      | 154 ++++++++++++++++++
 .../api/AdminClientIntegrationTest.scala      |  85 +++++++++-
 13 files changed, 711 insertions(+), 23 deletions(-)
 create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/MembershipChangeResult.java
 create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/RemoveMemberFromConsumerGroupOptions.java
 create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/RemoveMemberFromGroupResult.java
 create mode 100644 clients/src/test/java/org/apache/kafka/clients/admin/MembershipChangeResultTest.java
 create mode 100644 clients/src/test/java/org/apache/kafka/clients/admin/RemoveMemberFromConsumerGroupOptionsTest.java
 create mode 100644 clients/src/test/java/org/apache/kafka/clients/admin/RemoveMemberFromGroupResultTest.java

diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java b/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java
index 8fdd21b1ad54c..48630d46bb161 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java
@@ -34,6 +34,7 @@
 import org.apache.kafka.common.acl.AclBindingFilter;
 import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.common.config.ConfigResource;
+import org.apache.kafka.common.requests.LeaveGroupResponse;
 
 /**
  * The administrative client for Kafka, which supports managing and inspecting topics, brokers, configurations and ACLs.
@@ -1045,6 +1046,17 @@ default ListPartitionReassignmentsResult listPartitionReassignments(ListPartitio
     ListPartitionReassignmentsResult listPartitionReassignments(Optional> partitions,
                                                                 ListPartitionReassignmentsOptions options);
 
+    /**
+     * Remove members from the consumer group by given member identities.
+     * 
            
+     * For possible error codes, refer to {@link LeaveGroupResponse}.
+     *
+     * @param groupId The ID of the group to remove member from.
+     * @param options The options to carry removing members' information.
+     * @return The MembershipChangeResult.
+     */
+    MembershipChangeResult removeMemberFromConsumerGroup(String groupId, RemoveMemberFromConsumerGroupOptions options);
+
     /**
      * Get the metrics kept by the adminClient
      */
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
index 3ed981909576b..c7ea97005a516 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
@@ -143,6 +143,8 @@
 import org.apache.kafka.common.requests.FindCoordinatorResponse;
 import org.apache.kafka.common.requests.IncrementalAlterConfigsRequest;
 import org.apache.kafka.common.requests.IncrementalAlterConfigsResponse;
+import org.apache.kafka.common.requests.LeaveGroupRequest;
+import org.apache.kafka.common.requests.LeaveGroupResponse;
 import org.apache.kafka.common.requests.ListGroupsRequest;
 import org.apache.kafka.common.requests.ListGroupsResponse;
 import org.apache.kafka.common.requests.ListPartitionReassignmentsRequest;
@@ -2043,9 +2045,9 @@ void handleFailure(Throwable throwable) {
         return futures;
     }
 
-    private  IncrementalAlterConfigsRequestData toIncrementalAlterConfigsRequestData(final Collection resources,
-                                                                   final Map> configs,
-                                                                   final boolean validateOnly) {
+    private IncrementalAlterConfigsRequestData toIncrementalAlterConfigsRequestData(final Collection resources,
+                                                                                    final Map> configs,
+                                                                                    final boolean validateOnly) {
         IncrementalAlterConfigsRequestData requestData = new IncrementalAlterConfigsRequestData();
         requestData.setValidateOnly(validateOnly);
         for (ConfigResource resource : resources) {
@@ -3339,4 +3341,63 @@ private boolean dependsOnSpecificNode(ConfigResource resource) {
         return (resource.type() == ConfigResource.Type.BROKER && !resource.isDefault())
                 || resource.type() == ConfigResource.Type.BROKER_LOGGER;
     }
+
+    @Override
+    public MembershipChangeResult removeMemberFromConsumerGroup(String groupId,
+                                                                RemoveMemberFromConsumerGroupOptions options) {
+        final long startFindCoordinatorMs = time.milliseconds();
+        final long deadline = calcDeadlineMs(startFindCoordinatorMs, options.timeoutMs());
+
+        KafkaFutureImpl future = new KafkaFutureImpl<>();
+
+        ConsumerGroupOperationContext context =
+            new ConsumerGroupOperationContext<>(groupId, options, deadline, future);
+
+        Call findCoordinatorCall = getFindCoordinatorCall(context,
+            () -> KafkaAdminClient.this.getRemoveMembersFromGroupCall(context));
+        runnable.call(findCoordinatorCall, startFindCoordinatorMs);
+
+        return new MembershipChangeResult(future);
+    }
+
+
+    private Call getRemoveMembersFromGroupCall(ConsumerGroupOperationContext
+                                                    context) {
+        return new Call("leaveGroup",
+                        context.getDeadline(),
+                        new ConstantNodeIdProvider(context.getNode().get().id())) {
+            @Override
+            AbstractRequest.Builder createRequest(int timeoutMs) {
+                return new LeaveGroupRequest.Builder(context.getGroupId(),
+                                                     context.getOptions().getMembers());
+            }
+
+            @Override
+            void handleResponse(AbstractResponse abstractResponse) {
+                final LeaveGroupResponse response = (LeaveGroupResponse) abstractResponse;
+
+                // If coordinator changed since we fetched it, retry
+                if (context.hasCoordinatorMoved(response)) {
+                    rescheduleTask(context, () -> getRemoveMembersFromGroupCall(context));
+                    return;
+                }
+
+                // If error is transient coordinator error, retry
+                Errors error = response.error();
+                if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS || error == Errors.COORDINATOR_NOT_AVAILABLE) {
+                    throw error.exception();
+                }
+
+                final RemoveMemberFromGroupResult membershipChangeResult =
+                    new RemoveMemberFromGroupResult(response, context.getOptions().getMembers());
+
+                context.getFuture().complete(membershipChangeResult);
+            }
+
+            @Override
+            void handleFailure(Throwable throwable) {
+                context.getFuture().completeExceptionally(throwable);
+            }
+        };
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/MembershipChangeResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/MembershipChangeResult.java
new file mode 100644
index 0000000000000..e704ed9a816da
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/MembershipChangeResult.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.admin;
+
+import org.apache.kafka.common.KafkaFuture;
+import org.apache.kafka.common.annotation.InterfaceStability;
+
+import java.util.concurrent.ExecutionException;
+
+/**
+ * The result of the {@link KafkaAdminClient#removeMemberFromConsumerGroup(String, RemoveMemberFromConsumerGroupOptions)} call.
+ *
+ * The API of this class is evolving, see {@link AdminClient} for details.
+ */
+@InterfaceStability.Evolving
+public class MembershipChangeResult {
+
+    private KafkaFuture future;
+
+    MembershipChangeResult(KafkaFuture future) {
+        this.future = future;
+    }
+
+    /**
+     * Return a future which contains the member removal results.
+     */
+    public RemoveMemberFromGroupResult all() throws ExecutionException, InterruptedException {
+        return future.get();
+    }
+
+    // Visible for testing
+    public KafkaFuture future() {
+        return future;
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/RemoveMemberFromConsumerGroupOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/RemoveMemberFromConsumerGroupOptions.java
new file mode 100644
index 0000000000000..ed1fdab0be197
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/RemoveMemberFromConsumerGroupOptions.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.admin;
+
+import org.apache.kafka.common.annotation.InterfaceStability;
+import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Options for {@link AdminClient#removeMemberFromConsumerGroup(String, RemoveMemberFromConsumerGroupOptions)}.
+ * It carries the members to be removed from the consumer group.
+ *
+ * The API of this class is evolving, see {@link AdminClient} for details.
+ */
+@InterfaceStability.Evolving
+public class RemoveMemberFromConsumerGroupOptions extends AbstractOptions {
+
+    private List members;
+
+    public RemoveMemberFromConsumerGroupOptions(Collection groupInstanceIds) {
+        members = groupInstanceIds.stream().map(
+            instanceId -> new MemberIdentity()
+                              .setGroupInstanceId(instanceId)
+                              .setMemberId(JoinGroupRequest.UNKNOWN_MEMBER_ID)
+        ).collect(Collectors.toList());
+    }
+
+    public List getMembers() {
+        return members;
+    }
+}
+
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/RemoveMemberFromGroupResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/RemoveMemberFromGroupResult.java
new file mode 100644
index 0000000000000..1bd9a8be56f7e
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/RemoveMemberFromGroupResult.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.admin;
+
+import org.apache.kafka.common.KafkaFuture;
+import org.apache.kafka.common.internals.KafkaFutureImpl;
+import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity;
+import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.LeaveGroupResponse;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Result of a batch member removal operation.
+ */
+public class RemoveMemberFromGroupResult {
+
+    private final Errors topLevelError;
+    private final Map> memberFutures;
+    private boolean hasError = false;
+
+    RemoveMemberFromGroupResult(LeaveGroupResponse response,
+                                List membersToRemove) {
+        this.topLevelError = response.topLevelError();
+        this.memberFutures = new HashMap<>(membersToRemove.size());
+
+        if (this.topLevelError != Errors.NONE) {
+            // If the populated error is a top-level error, fail every member's future.
+            for (MemberIdentity memberIdentity : membersToRemove) {
+                KafkaFutureImpl future = new KafkaFutureImpl<>();
+                future.completeExceptionally(topLevelError.exception());
+                memberFutures.put(memberIdentity, future);
+            }
+            hasError = true;
+        } else {
+            for (MemberResponse memberResponse : response.memberResponses()) {
+                KafkaFutureImpl future = new KafkaFutureImpl<>();
+                Errors memberError = Errors.forCode(memberResponse.errorCode());
+                if (memberError != Errors.NONE) {
+                    future.completeExceptionally(memberError.exception());
+                    hasError = true;
+                } else {
+                    future.complete(null);
+                }
+                memberFutures.put(new MemberIdentity()
+                                      .setMemberId(memberResponse.memberId())
+                                      .setGroupInstanceId(memberResponse.groupInstanceId()), future);
+            }
+        }
+    }
+
+    public Errors topLevelError() {
+        return topLevelError;
+    }
+
+    public boolean hasError() {
+        return hasError;
+    }
+
+    /**
+     * Futures of members with corresponding errors when they leave the group.
+     *
+     * @return list of members who failed to be removed
+     */
+    public Map> memberFutures() {
+        return memberFutures;
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java
index 97a583f3d226f..e64fde168980b 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java
@@ -97,6 +97,10 @@ public Errors error() {
         return getError(Errors.forCode(data.errorCode()), data.members());
     }
 
+    public Errors topLevelError() {
+        return Errors.forCode(data.errorCode());
+    }
+
     private static Errors getError(Errors topLevelError, List memberResponses) {
         if (topLevelError != Errors.NONE) {
             return topLevelError;
diff --git a/clients/src/main/resources/common/message/LeaveGroupResponse.json b/clients/src/main/resources/common/message/LeaveGroupResponse.json
index 2bbf63df236de..495d5c39ba181 100644
--- a/clients/src/main/resources/common/message/LeaveGroupResponse.json
+++ b/clients/src/main/resources/common/message/LeaveGroupResponse.json
@@ -19,7 +19,7 @@
   "name": "LeaveGroupResponse",
   // Version 1 adds the throttle time.
   // Starting in version 2, on quota violation, brokers send out responses before throttling.
-  // Starting in version 3, we will make leave group request into batch mode.
+  // Starting in version 3, we will make leave group request into batch mode and add group.instance.id.
   "validVersions": "0-3",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "1+", "ignorable": true,
diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
index ff9e27206b83b..5d4d84b2d47c9 100644
--- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
@@ -50,6 +50,7 @@
 import org.apache.kafka.common.errors.SecurityDisabledException;
 import org.apache.kafka.common.errors.TimeoutException;
 import org.apache.kafka.common.errors.TopicDeletionDisabledException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
 import org.apache.kafka.common.errors.UnknownServerException;
 import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
 import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData;
@@ -66,6 +67,9 @@
 import org.apache.kafka.common.message.ElectLeadersResponseData.ReplicaElectionResult;
 import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData.AlterConfigsResourceResponse;
 import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData;
+import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity;
+import org.apache.kafka.common.message.LeaveGroupResponseData;
+import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse;
 import org.apache.kafka.common.message.ListGroupsResponseData;
 import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData;
 import org.apache.kafka.common.protocol.Errors;
@@ -89,6 +93,7 @@
 import org.apache.kafka.common.requests.ElectLeadersResponse;
 import org.apache.kafka.common.requests.FindCoordinatorResponse;
 import org.apache.kafka.common.requests.IncrementalAlterConfigsResponse;
+import org.apache.kafka.common.requests.LeaveGroupResponse;
 import org.apache.kafka.common.requests.ListGroupsResponse;
 import org.apache.kafka.common.requests.ListPartitionReassignmentsResponse;
 import org.apache.kafka.common.requests.MetadataRequest;
@@ -224,7 +229,7 @@ private static Cluster mockCluster(int controllerIndex) {
 
     private static Cluster mockBootstrapCluster() {
         return Cluster.bootstrap(ClientUtils.parseAndValidateAddresses(
-                Collections.singletonList("localhost:8121"), ClientDnsLookup.DEFAULT));
+                singletonList("localhost:8121"), ClientDnsLookup.DEFAULT));
     }
 
     private static AdminClientUnitTestEnv mockClientEnv(String... configVals) {
@@ -301,7 +306,7 @@ public void testUnreachableBootstrapServer() throws Exception {
         // This tests the scenario in which the bootstrap server is unreachable for a short while,
         // which prevents AdminClient from being able to send the initial metadata request
 
-        Cluster cluster = Cluster.bootstrap(Collections.singletonList(new InetSocketAddress("localhost", 8121)));
+        Cluster cluster = Cluster.bootstrap(singletonList(new InetSocketAddress("localhost", 8121)));
         Map unreachableNodes = Collections.singletonMap(cluster.nodes().get(0), 200L);
         try (final AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(Time.SYSTEM, cluster,
                 AdminClientUnitTestEnv.clientConfigs(), unreachableNodes)) {
@@ -429,19 +434,19 @@ public void testDeleteTopics() throws Exception {
 
             env.kafkaClient().prepareResponse(body -> body instanceof DeleteTopicsRequest,
                     prepareDeleteTopicsResponse("myTopic", Errors.NONE));
-            KafkaFuture future = env.adminClient().deleteTopics(Collections.singletonList("myTopic"),
+            KafkaFuture future = env.adminClient().deleteTopics(singletonList("myTopic"),
                     new DeleteTopicsOptions()).all();
             future.get();
 
             env.kafkaClient().prepareResponse(body -> body instanceof DeleteTopicsRequest,
                     prepareDeleteTopicsResponse("myTopic", Errors.TOPIC_DELETION_DISABLED));
-            future = env.adminClient().deleteTopics(Collections.singletonList("myTopic"),
+            future = env.adminClient().deleteTopics(singletonList("myTopic"),
                     new DeleteTopicsOptions()).all();
             TestUtils.assertFutureError(future, TopicDeletionDisabledException.class);
 
             env.kafkaClient().prepareResponse(body -> body instanceof DeleteTopicsRequest,
                     prepareDeleteTopicsResponse("myTopic", Errors.UNKNOWN_TOPIC_OR_PARTITION));
-            future = env.adminClient().deleteTopics(Collections.singletonList("myTopic"),
+            future = env.adminClient().deleteTopics(singletonList("myTopic"),
                     new DeleteTopicsOptions()).all();
             TestUtils.assertFutureError(future, UnknownTopicOrPartitionException.class);
         }
@@ -1550,8 +1555,8 @@ public void testIncrementalAlterConfigs()  throws Exception {
                     AlterConfigOp.OpType.APPEND);
 
             final Map> configs = new HashMap<>();
-            configs.put(brokerResource, Collections.singletonList(alterConfigOp1));
-            configs.put(topicResource, Collections.singletonList(alterConfigOp2));
+            configs.put(brokerResource, singletonList(alterConfigOp1));
+            configs.put(topicResource, singletonList(alterConfigOp2));
 
             AlterConfigsResult result = env.adminClient().incrementalAlterConfigs(configs);
             TestUtils.assertFutureError(result.values().get(brokerResource), ClusterAuthorizationException.class);
@@ -1566,7 +1571,117 @@ public void testIncrementalAlterConfigs()  throws Exception {
                     .setErrorMessage(ApiError.NONE.message()));
 
             env.kafkaClient().prepareResponse(new IncrementalAlterConfigsResponse(responseData));
-            env.adminClient().incrementalAlterConfigs(Collections.singletonMap(brokerResource, asList(alterConfigOp1))).all().get();
+            env.adminClient().incrementalAlterConfigs(Collections.singletonMap(brokerResource, singletonList(alterConfigOp1))).all().get();
+        }
+    }
+
+    @Test
+    public void testRemoveMembersFromGroup() throws Exception {
+        try (AdminClientUnitTestEnv env = mockClientEnv()) {
+            final String instanceOne = "instance-1";
+            final String instanceTwo = "instance-2";
+            env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
+
+            MemberResponse responseOne = new MemberResponse()
+                                             .setGroupInstanceId(instanceOne)
+                                             .setErrorCode(Errors.UNKNOWN_MEMBER_ID.code());
+
+            MemberResponse responseTwo = new MemberResponse()
+                                             .setGroupInstanceId(instanceTwo)
+                                             .setErrorCode(Errors.NONE.code());
+
+            List memberResponses = Arrays.asList(responseOne, responseTwo);
+
+            // Retriable FindCoordinatorResponse errors should be retried
+            env.kafkaClient().prepareResponse(prepareFindCoordinatorResponse(Errors.COORDINATOR_NOT_AVAILABLE,  Node.noNode()));
+            env.kafkaClient().prepareResponse(prepareFindCoordinatorResponse(Errors.COORDINATOR_LOAD_IN_PROGRESS,  Node.noNode()));
+            env.kafkaClient().prepareResponse(prepareFindCoordinatorResponse(Errors.NONE, env.cluster().controller()));
+
+            // Retriable errors should be retried
+            env.kafkaClient().prepareResponse(null, true);
+            env.kafkaClient().prepareResponse(new LeaveGroupResponse(new LeaveGroupResponseData()
+                                                                         .setErrorCode(Errors.COORDINATOR_NOT_AVAILABLE.code())));
+            env.kafkaClient().prepareResponse(new LeaveGroupResponse(new LeaveGroupResponseData()
+                                                                         .setErrorCode(Errors.COORDINATOR_LOAD_IN_PROGRESS.code())));
+
+            // Inject a top-level non-retriable error
+            env.kafkaClient().prepareResponse(new LeaveGroupResponse(new LeaveGroupResponseData()
+                                                                         .setErrorCode(Errors.UNKNOWN_SERVER_ERROR.code())));
+
+            String groupId = "groupId";
+            List membersToRemove = Arrays.asList(instanceOne, instanceTwo);
+            final MembershipChangeResult unknownErrorResult = env.adminClient().removeMemberFromConsumerGroup(
+                groupId,
+                new RemoveMemberFromConsumerGroupOptions(membersToRemove)
+            );
+
+            RemoveMemberFromGroupResult result = unknownErrorResult.all();
+            assertTrue(result.hasError());
+            assertEquals(Errors.UNKNOWN_SERVER_ERROR, result.topLevelError());
+
+            Map> memberFutures = result.memberFutures();
+            assertEquals(2, memberFutures.size());
+            for (Map.Entry> entry : memberFutures.entrySet()) {
+                KafkaFuture memberFuture = entry.getValue();
+                assertTrue(memberFuture.isCompletedExceptionally());
+                try {
+                    memberFuture.get();
+                    fail("get() should throw exception");
+                } catch (ExecutionException | InterruptedException e0) {
+                    assertTrue(e0.getCause() instanceof UnknownServerException);
+                }
+            }
+
+            // Inject one member level error.
+            env.kafkaClient().prepareResponse(prepareFindCoordinatorResponse(Errors.NONE, env.cluster().controller()));
+            env.kafkaClient().prepareResponse(new LeaveGroupResponse(new LeaveGroupResponseData()
+                                                                         .setErrorCode(Errors.NONE.code())
+                                                                         .setMembers(memberResponses)));
+
+            final MembershipChangeResult memberLevelErrorResult = env.adminClient().removeMemberFromConsumerGroup(
+                groupId,
+                new RemoveMemberFromConsumerGroupOptions(membersToRemove)
+            );
+
+            result = memberLevelErrorResult.all();
+            assertTrue(result.hasError());
+            assertEquals(Errors.NONE, result.topLevelError());
+
+            memberFutures = result.memberFutures();
+            assertEquals(2, memberFutures.size());
+            for (Map.Entry> entry : memberFutures.entrySet()) {
+                KafkaFuture memberFuture = entry.getValue();
+                if (entry.getKey().groupInstanceId().equals(instanceOne)) {
+                    try {
+                        memberFuture.get();
+                        fail("get() should throw ExecutionException");
+                    } catch (ExecutionException | InterruptedException e0) {
+                        assertTrue(e0.getCause() instanceof UnknownMemberIdException);
+                    }
+                } else {
+                    assertFalse(memberFuture.isCompletedExceptionally());
+                }
+            }
+
+            // Return success.
+            env.kafkaClient().prepareResponse(prepareFindCoordinatorResponse(Errors.NONE, env.cluster().controller()));
+            env.kafkaClient().prepareResponse(new LeaveGroupResponse(new LeaveGroupResponseData()
+                                                                         .setErrorCode(Errors.NONE.code())
+                                                                         .setMembers(Collections.singletonList(responseTwo))));
+
+            final MembershipChangeResult noErrorResult = env.adminClient().removeMemberFromConsumerGroup(
+                groupId,
+                new RemoveMemberFromConsumerGroupOptions(membersToRemove)
+            );
+            result = noErrorResult.all();
+            assertFalse(result.hasError());
+            assertEquals(Errors.NONE, result.topLevelError());
+
+            memberFutures = result.memberFutures();
+            assertEquals(1, memberFutures.size());
+            for (Map.Entry> entry : memberFutures.entrySet()) {
+                assertFalse(entry.getValue().isCompletedExceptionally());
+            }
         }
     }
 
@@ -1838,7 +1953,5 @@ boolean callHasExpired(KafkaAdminClient.Call call) {
                 }
             }
         }
-
     }
-
 }
diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/MembershipChangeResultTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/MembershipChangeResultTest.java
new file mode 100644
index 0000000000000..5a1357296b199
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/clients/admin/MembershipChangeResultTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.admin;
+
+import org.apache.kafka.common.internals.KafkaFutureImpl;
+import org.apache.kafka.common.message.LeaveGroupResponseData;
+import org.apache.kafka.common.requests.LeaveGroupResponse;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class MembershipChangeResultTest {
+
+    @Test
+    public void testConstructor() {
+        KafkaFutureImpl removeMemberFuture = new KafkaFutureImpl<>();
+
+        MembershipChangeResult changeResult = new MembershipChangeResult(removeMemberFuture);
+        assertEquals(removeMemberFuture, changeResult.future());
+        RemoveMemberFromGroupResult removeMemberFromGroupResult = new RemoveMemberFromGroupResult(
+            new LeaveGroupResponse(new LeaveGroupResponseData()),
+            Collections.emptyList()
+        );
+
+        removeMemberFuture.complete(removeMemberFromGroupResult);
+        try {
+            assertEquals(removeMemberFromGroupResult, changeResult.all());
+        } catch (ExecutionException | InterruptedException e) {
+            fail("Unexpected exception " + e + " when trying to get remove member result");
+        }
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java b/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java
index baaf6613841e3..86ba572f532d9 100644
--- a/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java
+++ b/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java
@@ -357,6 +357,11 @@ public ElectLeadersResult electLeaders(
         throw new UnsupportedOperationException("Not implemented yet");
     }
 
+    @Override
+    public MembershipChangeResult removeMemberFromConsumerGroup(String groupId, RemoveMemberFromConsumerGroupOptions options) {
+        throw new UnsupportedOperationException("Not implemented yet");
+    }
+
     @Override
     public CreateAclsResult createAcls(Collection acls, CreateAclsOptions options) {
         throw new UnsupportedOperationException("Not implemented yet");
diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/RemoveMemberFromConsumerGroupOptionsTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/RemoveMemberFromConsumerGroupOptionsTest.java
new file mode 100644
index 0000000000000..b36461efbcd95
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/clients/admin/RemoveMemberFromConsumerGroupOptionsTest.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.admin;
+
+import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class RemoveMemberFromConsumerGroupOptionsTest {
+
+    @Test
+    public void testConstructor() {
+        List groupInstanceIds = Collections.singletonList("instance-1");
+
+        RemoveMemberFromConsumerGroupOptions options = new RemoveMemberFromConsumerGroupOptions(groupInstanceIds);
+
+        assertEquals(Collections.singletonList(
+            new MemberIdentity().setGroupInstanceId("instance-1")), options.getMembers());
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/RemoveMemberFromGroupResultTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/RemoveMemberFromGroupResultTest.java
new file mode 100644
index 0000000000000..f4a6ebe948f88
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/clients/admin/RemoveMemberFromGroupResultTest.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.admin;
+
+import org.apache.kafka.common.KafkaFuture;
+import org.apache.kafka.common.errors.FencedInstanceIdException;
+import org.apache.kafka.common.errors.GroupAuthorizationException;
+import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity;
+import org.apache.kafka.common.message.LeaveGroupResponseData;
+import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.LeaveGroupResponse;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class RemoveMemberFromGroupResultTest {
+
+    private String instanceOne = "instance-1";
+    private String instanceTwo = "instance-2";
+    private List membersToRemove;
+
+    private List memberResponses;
+
+    @Before
+    public void setUp() {
+        membersToRemove = Arrays.asList(
+            new MemberIdentity()
+                .setGroupInstanceId(instanceOne),
+            new MemberIdentity()
+                .setGroupInstanceId(instanceTwo)
+        );
+
+        memberResponses = Arrays.asList(
+            new MemberResponse()
+                .setGroupInstanceId(instanceOne),
+            new MemberResponse()
+                .setGroupInstanceId(instanceTwo)
+        );
+    }
+
+    @Test
+    public void testTopLevelErrorConstructor() {
+        RemoveMemberFromGroupResult topLevelErrorResult =
+            new RemoveMemberFromGroupResult(new LeaveGroupResponse(
+                new LeaveGroupResponseData()
+                    .setErrorCode(Errors.GROUP_AUTHORIZATION_FAILED.code())
+                    .setMembers(memberResponses)), membersToRemove);
+
+        assertTrue(topLevelErrorResult.hasError());
+        assertEquals(Errors.GROUP_AUTHORIZATION_FAILED, topLevelErrorResult.topLevelError());
+
+        Map> memberFutures = topLevelErrorResult.memberFutures();
+        assertEquals(2, memberFutures.size());
+        for (Map.Entry> entry : memberFutures.entrySet()) {
+            KafkaFuture memberFuture = entry.getValue();
+            assertTrue(memberFuture.isCompletedExceptionally());
+            try {
+                memberFuture.get();
+                fail("get() should throw ExecutionException");
+            } catch (ExecutionException | InterruptedException e0) {
+                assertTrue(e0.getCause() instanceof GroupAuthorizationException);
+            }
+        }
+    }
+
+    @Test
+    public void testMemberLevelErrorConstructor() {
+        MemberResponse responseOne = new MemberResponse()
+                                         .setGroupInstanceId(instanceOne)
+                                         .setErrorCode(Errors.FENCED_INSTANCE_ID.code());
+        MemberResponse responseTwo = new MemberResponse()
+                                         .setGroupInstanceId(instanceTwo)
+                                         .setErrorCode(Errors.NONE.code());
+
+        RemoveMemberFromGroupResult memberLevelErrorResult = new RemoveMemberFromGroupResult(
+            new LeaveGroupResponse(new LeaveGroupResponseData()
+                                       .setMembers(Arrays.asList(responseOne, responseTwo))),
+            membersToRemove);
+        assertTrue(memberLevelErrorResult.hasError());
+        assertEquals(Errors.NONE, memberLevelErrorResult.topLevelError());
+
+        Map> memberFutures = memberLevelErrorResult.memberFutures();
+        assertEquals(2, memberFutures.size());
+        for (Map.Entry> entry : memberFutures.entrySet()) {
+            KafkaFuture memberFuture = entry.getValue();
+            if (entry.getKey().groupInstanceId().equals(instanceOne)) {
+                assertTrue(memberFuture.isCompletedExceptionally());
+                try {
+                    memberFuture.get();
+                    fail("get() should throw ExecutionException");
+                } catch (ExecutionException | InterruptedException e0) {
+                    assertTrue(e0.getCause() instanceof FencedInstanceIdException);
+                }
+            } else {
+                assertFalse(memberFuture.isCompletedExceptionally());
+                try {
+                    memberFuture.get();
+                } catch (ExecutionException | InterruptedException e0) {
+                    fail("get() shouldn't throw exception");
+                }
+            }
+        }
+    }
+
+    @Test
+    public void testNoErrorConstructor() {
+        MemberResponse responseOne = new MemberResponse()
+                                         .setGroupInstanceId(instanceOne)
+                                         .setErrorCode(Errors.NONE.code());
+        MemberResponse responseTwo = new MemberResponse()
+                                         .setGroupInstanceId(instanceTwo)
+                                         .setErrorCode(Errors.NONE.code());
+        // If no error is specified, failed members are not visible.
+        RemoveMemberFromGroupResult noErrorResult = new RemoveMemberFromGroupResult(
+            new LeaveGroupResponse(new LeaveGroupResponseData()
+                                       .setMembers(Arrays.asList(responseOne, responseTwo))),
+            membersToRemove);
+        assertFalse(noErrorResult.hasError());
+        assertEquals(Errors.NONE, noErrorResult.topLevelError());
+        Map> memberFutures = noErrorResult.memberFutures();
+        assertEquals(2, memberFutures.size());
+        for (Map.Entry> entry : memberFutures.entrySet()) {
+            try {
+                entry.getValue().get();
+            } catch (ExecutionException | InterruptedException e0) {
+                fail("get() shouldn't throw exception");
+            }
+        }
+    }
+}
diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index 0484f6a5f78da..e2e0ca71a03aa 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -43,7 +43,11 @@ import org.apache.kafka.common.TopicPartitionReplica
 import org.apache.kafka.common.acl._
 import org.apache.kafka.common.config.{ConfigResource, LogLevelConfig}
 import org.apache.kafka.common.errors._
-import org.apache.kafka.common.requests.{DeleteRecordsRequest, MetadataResponse}
+import org.apache.kafka.common.internals.KafkaFutureImpl
+import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
+import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.requests.{DeleteRecordsRequest, JoinGroupRequest, MetadataResponse}
 import org.apache.kafka.common.resource.{PatternType, Resource, ResourcePattern, ResourceType}
 import org.apache.kafka.common.utils.{Time, Utils}
 import org.junit.Assert._
@@ -1168,10 +1172,12 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
       }
       val testGroupId = "test_group_id"
       val testClientId = "test_client_id"
+      val testInstanceId = "test_instance_id"
       val fakeGroupId = "fake_group_id"
       val newConsumerConfig = new Properties(consumerConfig)
       newConsumerConfig.setProperty(ConsumerConfig.GROUP_ID_CONFIG, testGroupId)
       newConsumerConfig.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, testClientId)
+      newConsumerConfig.setProperty(ConsumerConfig.GROUP_INSTANCE_ID_CONFIG, testInstanceId)
       val consumer = createConsumer(configOverrides = newConsumerConfig)
       val latch = new CountDownLatch(1)
       try {
@@ -1201,13 +1207,13 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
             !matching.isEmpty
           }, s"Expected to be able to list $testGroupId")
 
-          val result = client.describeConsumerGroups(Seq(testGroupId, fakeGroupId).asJava,
+          val describeWithFakeGroupResult = client.describeConsumerGroups(Seq(testGroupId, fakeGroupId).asJava,
             new DescribeConsumerGroupsOptions().includeAuthorizedOperations(true))
-          assertEquals(2, result.describedGroups().size())
+          assertEquals(2, describeWithFakeGroupResult.describedGroups().size())
 
           // Test that we can get information about the test consumer group.
-          assertTrue(result.describedGroups().containsKey(testGroupId))
-          val testGroupDescription = result.describedGroups().get(testGroupId).get()
+          assertTrue(describeWithFakeGroupResult.describedGroups().containsKey(testGroupId))
+          var testGroupDescription = describeWithFakeGroupResult.describedGroups().get(testGroupId).get()
 
           assertEquals(testGroupId, testGroupDescription.groupId())
           assertFalse(testGroupDescription.isSimpleConsumerGroup())
@@ -1223,8 +1229,8 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
           assertEquals(expectedOperations, testGroupDescription.authorizedOperations())
 
           // Test that the fake group is listed as dead.
-          assertTrue(result.describedGroups().containsKey(fakeGroupId))
-          val fakeGroupDescription = result.describedGroups().get(fakeGroupId).get()
+          assertTrue(describeWithFakeGroupResult.describedGroups().containsKey(fakeGroupId))
+          val fakeGroupDescription = describeWithFakeGroupResult.describedGroups().get(fakeGroupId).get()
 
           assertEquals(fakeGroupId, fakeGroupDescription.groupId())
           assertEquals(0, fakeGroupDescription.members().size())
@@ -1233,7 +1239,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
           assertEquals(expectedOperations, fakeGroupDescription.authorizedOperations())
 
           // Test that all() returns 2 results
-          assertEquals(2, result.all().get().size())
+          assertEquals(2, describeWithFakeGroupResult.all().get().size())
 
           // Test listConsumerGroupOffsets
           TestUtils.waitUntilTrue(() => {
@@ -1242,8 +1248,30 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
             parts.containsKey(part) && (parts.get(part).offset() == 1)
           }, s"Expected the offset for partition 0 to eventually become 1.")
 
+          // Test delete non-exist consumer instance
+          val invalidInstanceId = "invalid-instance-id"
+          var removeMemberResult = client.removeMemberFromConsumerGroup(testGroupId, new RemoveMemberFromConsumerGroupOptions(
+            Collections.singletonList(invalidInstanceId)
+          )).all()
+
+          assertTrue(removeMemberResult.hasError)
+          assertEquals(Errors.NONE, removeMemberResult.topLevelError)
+
+          val firstMemberFutures = removeMemberResult.memberFutures()
+          assertEquals(1, firstMemberFutures.size)
+          firstMemberFutures.values.asScala foreach { case value =>
+            try {
+              value.get()
+            } catch {
+              case e: ExecutionException =>
+                assertTrue(e.getCause.isInstanceOf[UnknownMemberIdException])
+              case _ =>
+                fail("Should have caught exception in getting member future")
+            }
+          }
+
           // Test consumer group deletion
-          val deleteResult = client.deleteConsumerGroups(Seq(testGroupId, fakeGroupId).asJava)
+          var deleteResult = client.deleteConsumerGroups(Seq(testGroupId, fakeGroupId).asJava)
           assertEquals(2, deleteResult.deletedGroups().size())
 
           // Deleting the fake group ID should get GroupIdNotFoundException.
@@ -1255,6 +1283,45 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
           assertTrue(deleteResult.deletedGroups().containsKey(testGroupId))
           assertFutureExceptionTypeEquals(deleteResult.deletedGroups().get(testGroupId),
             classOf[GroupNotEmptyException])
+
+          // Test delete correct member
+          removeMemberResult = client.removeMemberFromConsumerGroup(testGroupId, new RemoveMemberFromConsumerGroupOptions(
+            Collections.singletonList(testInstanceId)
+          )).all()
+
+          assertFalse(removeMemberResult.hasError)
+          assertEquals(Errors.NONE, removeMemberResult.topLevelError)
+
+          val deletedMemberFutures = removeMemberResult.memberFutures()
+          assertEquals(1, firstMemberFutures.size)
+          deletedMemberFutures.values.asScala foreach { case value =>
+            try {
+              value.get()
+            } catch {
+              case e: ExecutionException =>
+                assertTrue(e.getCause.isInstanceOf[UnknownMemberIdException])
+              case _ =>
+                fail("Should have caught exception in getting member future")
+            }
+          }
+
+          // The group should contain no member now.
+          val describeTestGroupResult = client.describeConsumerGroups(Seq(testGroupId).asJava,
+            new DescribeConsumerGroupsOptions().includeAuthorizedOperations(true))
+          assertEquals(1, describeTestGroupResult.describedGroups().size())
+
+          testGroupDescription = describeTestGroupResult.describedGroups().get(testGroupId).get()
+
+          assertEquals(testGroupId, testGroupDescription.groupId)
+          assertFalse(testGroupDescription.isSimpleConsumerGroup)
+          assertTrue(testGroupDescription.members().isEmpty)
+
+          // Consumer group deletion on empty group should succeed
+          deleteResult = client.deleteConsumerGroups(Seq(testGroupId).asJava)
+          assertEquals(1, deleteResult.deletedGroups().size())
+
+          assertTrue(deleteResult.deletedGroups().containsKey(testGroupId))
+          assertNull(deleteResult.deletedGroups().get(testGroupId).get())
         } finally {
           consumerThread.interrupt()
           consumerThread.join()

From d54285f0c0f1095bab0fa860e73f9258def397c2 Mon Sep 17 00:00:00 2001
From: wineandcheeze 
Date: Tue, 10 Sep 2019 07:26:53 +0200
Subject: [PATCH 0599/1071] KAFKA-8889: Log the details about error (#7317)

We need stacktrace of the error to understand the root cause and to trouble shoot the underlying problem.

Reviewers: Guozhang Wang 
---
 .../main/java/org/apache/kafka/clients/FetchSessionHandler.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java b/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java
index 5b402bf5b3b2a..494bbbe4769e1 100644
--- a/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java
+++ b/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java
@@ -439,7 +439,7 @@ public boolean handleResponse(FetchResponse response) {
      * @param t     The exception.
      */
     public void handleError(Throwable t) {
-        log.info("Error sending fetch request {} to node {}: {}.", nextMetadata, node, t.toString());
+        log.info("Error sending fetch request {} to node {}: {}.", nextMetadata, node, t);
         nextMetadata = nextMetadata.nextCloseExisting();
     }
 }

From 18246e509ee39736d38694b98e0c98ee73c6675f Mon Sep 17 00:00:00 2001
From: cpettitt-confluent
 <53191309+cpettitt-confluent@users.noreply.github.com>
Date: Tue, 10 Sep 2019 14:37:34 -0600
Subject: [PATCH 0600/1071] KAFKA-8878: Fix flaky test
 AssignedStreamsTasksTest#shouldCloseCleanlyWithSuspendedTaskAndEOS (#7302)

The previous approach to testing KAFKA-8412 was to look at the logs and
determine if an error occurred during close. There was no direct way to
detect than an exception occurred because the exception was eaten in
AssignedTasks.close. In the PR for that ticket (#7207) it was
acknowledged that this was a brittle way to test for the exception. We
now see occasional failures because an unrelated ERROR level log entry
is made while closing the task.

This change eliminates the brittle log checking by rethrowing any time
an exception occurs in close, even when a subsequent unclean close
succeeds. This has the potential benefit of uncovering other supressed
exceptions down the road.

I've verified that even with us rethrowing on closeUnclean that all
tests pass.

Reviewers: Matthias J. Sax ,  Bill Bejeck 
---
 .../processor/internals/AssignedTasks.java    |  7 ++--
 .../internals/AssignedStreamsTasksTest.java   | 33 +------------------
 2 files changed, 3 insertions(+), 37 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
index 85d8307e7beed..48933a41e1d59 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
@@ -349,12 +349,9 @@ void close(final boolean clean) {
                     task.id(),
                     t);
                 if (clean) {
-                    if (!closeUnclean(task)) {
-                        firstException.compareAndSet(null, t);
-                    }
-                } else {
-                    firstException.compareAndSet(null, t);
+                    closeUnclean(task);
                 }
+                firstException.compareAndSet(null, t);
             }
         }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasksTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasksTest.java
index ca51a3b1b17c7..a00969debd3d9 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasksTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasksTest.java
@@ -29,7 +29,6 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Set;
-import kafka.utils.LogCaptureAppender;
 import org.apache.kafka.clients.consumer.MockConsumer;
 import org.apache.kafka.clients.consumer.OffsetResetStrategy;
 import org.apache.kafka.clients.producer.MockProducer;
@@ -47,8 +46,6 @@
 import org.apache.kafka.streams.processor.TaskId;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.apache.kafka.test.MockSourceNode;
-import org.apache.log4j.Level;
-import org.apache.log4j.spi.LoggingEvent;
 import org.easymock.EasyMock;
 import org.junit.Before;
 import org.junit.Test;
@@ -525,35 +522,7 @@ public void shouldCloseCleanlyWithSuspendedTaskAndEOS() {
         assignedTasks.initializeNewTasks();
         assertNull(assignedTasks.suspend());
 
-        // We have to test for close failure by looking at the logs because the current close
-        // logic suppresses the raised exception in AssignedTasks.close. It's not clear if this
-        // is the intended behavior.
-        //
-        // Also note that capturing the failure through this side effect is very brittle.
-        final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
-        final Level previousLevel =
-            LogCaptureAppender.setClassLoggerLevel(AssignedStreamsTasks.class, Level.ERROR);
-        try {
-            assignedTasks.close(true);
-        } finally {
-            LogCaptureAppender.setClassLoggerLevel(AssignedStreamsTasks.class, previousLevel);
-            LogCaptureAppender.unregister(appender);
-        }
-        if (!appender.getMessages().isEmpty()) {
-            final LoggingEvent firstError = appender.getMessages().head();
-            final String firstErrorCause =
-                firstError.getThrowableStrRep() != null
-                    ? String.join("\n", firstError.getThrowableStrRep())
-                    : "N/A";
-
-            final String failMsg =
-                String.format("Expected no ERROR message while closing assignedTasks, but got %d. " +
-                    "First error: %s. Cause: %s",
-                    appender.getMessages().size(),
-                    firstError.getMessage(),
-                    firstErrorCause);
-            fail(failMsg);
-        }
+        assignedTasks.close(true);
     }
 
     private void addAndInitTask() {

From 7012fa3262d63cb15125ea45c378128786899b49 Mon Sep 17 00:00:00 2001
From: Stanislav Kozlovski 
Date: Tue, 10 Sep 2019 17:29:15 -0700
Subject: [PATCH 0601/1071] KAFKA-8747; Add atomic counter to fix flaky
 testEventQueueTime test (#7320)

This patch adds an atomic counter in the test to ensure we have processed all the events before we assert the metrics. There was a race condition with the previous assertion, which asserted that the event queue is empty before checking the metrics.

Reviewers: Jason Gustafson 
---
 .../kafka/controller/ControllerEventManagerTest.scala     | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/core/src/test/scala/unit/kafka/controller/ControllerEventManagerTest.scala b/core/src/test/scala/unit/kafka/controller/ControllerEventManagerTest.scala
index fc8f3240b7126..83bc38fddcfee 100644
--- a/core/src/test/scala/unit/kafka/controller/ControllerEventManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/controller/ControllerEventManagerTest.scala
@@ -69,11 +69,13 @@ class ControllerEventManagerTest {
     val controllerStats = new ControllerStats
     val time = new MockTime()
     val latch = new CountDownLatch(1)
+    val processedEvents = new AtomicInteger()
 
     val eventProcessor = new ControllerEventProcessor {
       override def process(event: ControllerEvent): Unit = {
         latch.await()
         time.sleep(500)
+        processedEvents.incrementAndGet()
       }
       override def preempt(event: ControllerEvent): Unit = {}
     }
@@ -89,12 +91,12 @@ class ControllerEventManagerTest {
     controllerEventManager.put(TopicChange)
     latch.countDown()
 
+    TestUtils.waitUntilTrue(() => processedEvents.get() == 2,
+      "Timed out waiting for processing of all events")
+
     val queueTimeHistogram = Metrics.defaultRegistry.allMetrics.asScala.filterKeys(_.getMBeanName == metricName).values.headOption
       .getOrElse(fail(s"Unable to find metric $metricName")).asInstanceOf[Histogram]
 
-    TestUtils.waitUntilTrue(() => controllerEventManager.isEmpty,
-      "Timed out waiting for processing of all events")
-
     assertEquals(2, queueTimeHistogram.count)
     assertEquals(0, queueTimeHistogram.min, 0.01)
     assertEquals(500, queueTimeHistogram.max, 0.01)

From a043edb559a20501660ffe593b4c14bc572dc16e Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Tue, 10 Sep 2019 19:01:10 -0700
Subject: [PATCH 0602/1071] KAFKA-8817: Remove timeout for the whole test
 (#7313)

I bumped into this flaky test while working on another PR. It's a bit different from the reported PR, where it actually timed out at parsing localhost:port already. I think what we should really test is that the closing call can complete, so I removed the whole test timeout and add the timeout for the shutdown latch instead.

Reviewers: Jason Gustafson , cpettitt-confluent <53191309+cpettitt-confluent@users.noreply.github.com>
---
 .../kafka/clients/producer/KafkaProducerTest.java    | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java
index 170b75e4bfbff..a209a923b36c0 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java
@@ -809,7 +809,7 @@ public void testTransactionalMethodThrowsWhenSenderClosed() {
         assertThrows(IllegalStateException.class, producer::initTransactions);
     }
 
-    @Test(timeout = 5000)
+    @Test
     public void testCloseIsForcedOnPendingFindCoordinator() throws InterruptedException {
         Map configs = new HashMap<>();
         configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9000");
@@ -834,10 +834,10 @@ public void testCloseIsForcedOnPendingFindCoordinator() throws InterruptedExcept
 
         client.waitForRequests(1, 2000);
         producer.close(Duration.ofMillis(1000));
-        assertionDoneLatch.await();
+        assertionDoneLatch.await(5000, TimeUnit.MILLISECONDS);
     }
 
-    @Test(timeout = 5000)
+    @Test
     public void testCloseIsForcedOnPendingInitProducerId() throws InterruptedException {
         Map configs = new HashMap<>();
         configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9000");
@@ -863,10 +863,10 @@ public void testCloseIsForcedOnPendingInitProducerId() throws InterruptedExcepti
 
         client.waitForRequests(1, 2000);
         producer.close(Duration.ofMillis(1000));
-        assertionDoneLatch.await();
+        assertionDoneLatch.await(5000, TimeUnit.MILLISECONDS);
     }
 
-    @Test(timeout = 5000)
+    @Test
     public void testCloseIsForcedOnPendingAddOffsetRequest() throws InterruptedException {
         Map configs = new HashMap<>();
         configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9000");
@@ -892,7 +892,7 @@ public void testCloseIsForcedOnPendingAddOffsetRequest() throws InterruptedExcep
 
         client.waitForRequests(1, 2000);
         producer.close(Duration.ofMillis(1000));
-        assertionDoneLatch.await();
+        assertionDoneLatch.await(5000, TimeUnit.MILLISECONDS);
     }
 
     private ProducerMetadata newMetadata(long refreshBackoffMs, long expirationMs) {

From 41b89a6ecf2aef6ee01b8bbf9a38c366d242b0ca Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Tue, 10 Sep 2019 19:08:42 -0700
Subject: [PATCH 0603/1071] MINOR: Add api version to uncaught exception
 message (#7311)

When we have an unhandled exception in the request handler, we print some details about the request such as the api key and payload. It is also useful to see the version of the request which is not always apparent from the request payload.

Reviewers: Guozhang Wang 
---
 core/src/main/scala/kafka/server/KafkaApis.scala | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index ac68f1a8f0ded..82b0616f1407c 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -2683,6 +2683,7 @@ class KafkaApis(val requestChannel: RequestChannel,
       s"clientId=${request.header.clientId}, " +
       s"correlationId=${request.header.correlationId}, " +
       s"api=${request.header.apiKey}, " +
+      s"version=${request.header.apiVersion}, " +
       s"body=${request.body[AbstractRequest]}", e)
     if (mayThrottle)
       sendErrorResponseMaybeThrottle(request, e)

From 18d4e57f6e8c67ffa7937fc855707d3a03cc165a Mon Sep 17 00:00:00 2001
From: Stanislav Kozlovski 
Date: Tue, 10 Sep 2019 22:19:44 -0700
Subject: [PATCH 0604/1071] KAFKA-8345 (KIP-455): Controller and KafkaApi
 changes (part 3/4) (#7128)

Implement the revisions to the controller state machine and reassignment logic needed for KIP-455.

Add the addingReplicas and removingReplicas field to the topics ZNode.

Deprecate the methods initiating a reassignment via direct ZK access in KafkaZkClient.

Add ControllerContextTest, and add some test cases to ReassignPartitionsClusterTest.

Add a note to upgrade.html recommending not initiating reassignments during an upgrade.

Reviewers: Colin P. McCabe , Viktor Somogyi 
---
 .../main/scala/kafka/admin/TopicCommand.scala |   9 +-
 .../controller/ControllerChannelManager.scala |   6 +-
 .../kafka/controller/ControllerContext.scala  | 106 ++-
 .../kafka/controller/ControllerState.scala    |  10 +-
 .../kafka/controller/KafkaController.scala    | 608 +++++++++++----
 .../controller/PartitionStateMachine.scala    |   9 +-
 .../controller/ReplicaStateMachine.scala      |   6 +-
 .../scala/kafka/server/AdminManager.scala     |   8 +-
 .../main/scala/kafka/server/KafkaApis.scala   |  86 ++-
 .../main/scala/kafka/zk/AdminZkClient.scala   |  20 +-
 .../main/scala/kafka/zk/KafkaZkClient.scala   |  37 +-
 core/src/main/scala/kafka/zk/ZkData.scala     |  62 +-
 .../api/AdminClientIntegrationTest.scala      |  52 ++
 .../kafka/api/BaseProducerSendTest.scala      |   4 +-
 .../kafka/ReplicationQuotasTestRig.scala      |   4 +-
 .../unit/kafka/admin/AddPartitionsTest.scala  |  19 +-
 .../unit/kafka/admin/DeleteTopicTest.scala    |   9 +-
 .../admin/ReassignPartitionsClusterTest.scala | 726 ++++++++++++++++--
 .../ControllerChannelManagerTest.scala        |  12 +-
 .../controller/ControllerContextTest.scala    | 189 +++++
 .../ControllerIntegrationTest.scala           |  22 +-
 .../PartitionStateMachineTest.scala           |  13 +-
 .../controller/ReplicaStateMachineTest.scala  |  11 +-
 .../security/auth/ZkAuthorizationTest.scala   |   3 +-
 .../unit/kafka/zk/AdminZkClientTest.scala     |   8 +-
 .../unit/kafka/zk/KafkaZkClientTest.scala     |   6 +-
 docs/upgrade.html                             |   2 +
 27 files changed, 1721 insertions(+), 326 deletions(-)
 create mode 100644 core/src/test/scala/unit/kafka/controller/ControllerContextTest.scala

diff --git a/core/src/main/scala/kafka/admin/TopicCommand.scala b/core/src/main/scala/kafka/admin/TopicCommand.scala
index 9123d49295343..0ceb9aaa03605 100755
--- a/core/src/main/scala/kafka/admin/TopicCommand.scala
+++ b/core/src/main/scala/kafka/admin/TopicCommand.scala
@@ -371,8 +371,8 @@ object TopicCommand extends Logging {
           }
           println("WARNING: If partitions are increased for a topic that has a key, the partition " +
             "logic or ordering of the messages will be affected")
-          val existingAssignment = zkClient.getReplicaAssignmentForTopics(immutable.Set(topic)).map {
-            case (topicPartition, replicas) => topicPartition.partition -> replicas
+          val existingAssignment = zkClient.getFullReplicaAssignmentForTopics(immutable.Set(topic)).map {
+            case (topicPartition, assignment) => topicPartition.partition -> assignment
           }
           if (existingAssignment.isEmpty)
             throw new InvalidTopicException(s"The topic $topic does not exist")
@@ -401,14 +401,15 @@ object TopicCommand extends Logging {
               val configs = adminZkClient.fetchEntityConfig(ConfigType.Topic, topic).asScala
               if (!opts.reportOverriddenConfigs || configs.nonEmpty) {
                 val numPartitions = topicPartitionAssignment.size
-                val replicationFactor = topicPartitionAssignment.head._2.size
+                val replicationFactor = topicPartitionAssignment.head._2.replicas.size
                 val config = new JConfig(configs.map{ case (k, v) => new ConfigEntry(k, v) }.asJavaCollection)
                 val topicDesc = TopicDescription(topic, numPartitions, replicationFactor, config, markedForDeletion)
                 topicDesc.printDescription()
               }
             }
             if (describeOptions.describePartitions) {
-              for ((partitionId, assignedReplicas) <- topicPartitionAssignment.toSeq.sortBy(_._1)) {
+              for ((partitionId, replicaAssignment) <- topicPartitionAssignment.toSeq.sortBy(_._1)) {
+                val assignedReplicas = replicaAssignment.replicas
                 val tp = new TopicPartition(topic, partitionId)
                 val (leaderOpt, isr) =  zkClient.getTopicPartitionState(tp).map(_.leaderAndIsr) match {
                   case Some(leaderAndIsr) => (leaderAndIsr.leaderOpt, leaderAndIsr.isr)
diff --git a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
index 7bd494ef028aa..e91e48d80cf15 100755
--- a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
+++ b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
@@ -376,7 +376,7 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
   def addLeaderAndIsrRequestForBrokers(brokerIds: Seq[Int],
                                        topicPartition: TopicPartition,
                                        leaderIsrAndControllerEpoch: LeaderIsrAndControllerEpoch,
-                                       replicas: Seq[Int],
+                                       replicaAssignment: PartitionReplicaAssignment,
                                        isNew: Boolean): Unit = {
 
     brokerIds.filter(_ >= 0).foreach { brokerId =>
@@ -387,7 +387,9 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
         leaderIsrAndControllerEpoch.leaderAndIsr.leaderEpoch,
         leaderIsrAndControllerEpoch.leaderAndIsr.isr.map(Integer.valueOf).asJava,
         leaderIsrAndControllerEpoch.leaderAndIsr.zkVersion,
-        replicas.map(Integer.valueOf).asJava,
+        replicaAssignment.replicas.map(Integer.valueOf).asJava,
+        replicaAssignment.addingReplicas.map(Integer.valueOf).asJava,
+        replicaAssignment.removingReplicas.map(Integer.valueOf).asJava,
         isNew || alreadyNew))
     }
 
diff --git a/core/src/main/scala/kafka/controller/ControllerContext.scala b/core/src/main/scala/kafka/controller/ControllerContext.scala
index 85a4abab0012a..be7f54ad47778 100644
--- a/core/src/main/scala/kafka/controller/ControllerContext.scala
+++ b/core/src/main/scala/kafka/controller/ControllerContext.scala
@@ -22,6 +22,45 @@ import org.apache.kafka.common.TopicPartition
 
 import scala.collection.{Map, Seq, Set, mutable}
 
+object PartitionReplicaAssignment {
+  def fromOldAndNewReplicas(oldReplicas: Seq[Int], newReplicas: Seq[Int]): PartitionReplicaAssignment = {
+    val fullReplicaSet = (newReplicas ++ oldReplicas).distinct
+    PartitionReplicaAssignment(
+      fullReplicaSet,
+      fullReplicaSet.filterNot(oldReplicas.contains(_)),
+      fullReplicaSet.filterNot(newReplicas.contains(_))
+    )
+  }
+}
+
+case class PartitionReplicaAssignment(replicas: Seq[Int], addingReplicas: Seq[Int], removingReplicas: Seq[Int]) {
+  def isBeingReassigned: Boolean = {
+    addingReplicas.nonEmpty || removingReplicas.nonEmpty
+  }
+
+  /**
+    * Returns the partition replica assignment previous to this one.
+    * It is different than this one only when the partition is undergoing reassignment
+    * Note that this will not preserve the original ordering
+    */
+  def previousAssignment: PartitionReplicaAssignment = {
+    PartitionReplicaAssignment(
+      replicas.filterNot(addingReplicas.contains(_)),
+      Seq(),
+      Seq()
+    )
+  }
+
+  /**
+    * Returns the target replica assignment for this partition.
+    * This is different than the `replicas` variable only when there is a reassignment going on
+    */
+  def targetReplicas: Seq[Int] = replicas.filterNot(removingReplicas.contains(_))
+
+  override def toString: String = s"PartitionReplicaAssignment(replicas: ${replicas.mkString(",")}, " +
+    s"addingReplicas: ${addingReplicas.mkString(",")}, removingReplicas: ${removingReplicas.mkString(",")})"
+}
+
 class ControllerContext {
   val stats = new ControllerStats
   var offlinePartitionCount = 0
@@ -32,7 +71,7 @@ class ControllerContext {
   var epochZkVersion: Int = KafkaController.InitialControllerEpochZkVersion
 
   var allTopics: Set[String] = Set.empty
-  val partitionAssignments = mutable.Map.empty[String, mutable.Map[Int, Seq[Int]]]
+  val partitionAssignments = mutable.Map.empty[String, mutable.Map[Int, PartitionReplicaAssignment]]
   val partitionLeadershipInfo = mutable.Map.empty[TopicPartition, LeaderIsrAndControllerEpoch]
   val partitionsBeingReassigned = mutable.Map.empty[TopicPartition, ReassignedPartitionsContext]
   val partitionStates = mutable.Map.empty[TopicPartition, PartitionState]
@@ -63,12 +102,6 @@ class ControllerContext {
   val topicsWithDeletionStarted = mutable.Set.empty[String]
   val topicsIneligibleForDeletion = mutable.Set.empty[String]
 
-
-  def partitionReplicaAssignment(topicPartition: TopicPartition): Seq[Int] = {
-    partitionAssignments.getOrElse(topicPartition.topic, mutable.Map.empty)
-      .getOrElse(topicPartition.partition, Seq.empty)
-  }
-
   private def clearTopicsState(): Unit = {
     allTopics = Set.empty
     partitionAssignments.clear()
@@ -80,14 +113,55 @@ class ControllerContext {
     replicaStates.clear()
   }
 
+  def partitionReplicaAssignment(topicPartition: TopicPartition): Seq[Int] = {
+    partitionAssignments.getOrElse(topicPartition.topic, mutable.Map.empty)
+      .get(topicPartition.partition) match {
+        case Some(partitionAssignment) => partitionAssignment.replicas
+        case None => Seq.empty
+      }
+  }
+
+  def partitionFullReplicaAssignment(topicPartition: TopicPartition): PartitionReplicaAssignment = {
+    partitionAssignments.getOrElse(topicPartition.topic, mutable.Map.empty)
+      .get(topicPartition.partition) match {
+      case Some(partitionAssignment) => partitionAssignment
+      case None => PartitionReplicaAssignment(Seq(), Seq(), Seq())
+    }
+  }
+
   def updatePartitionReplicaAssignment(topicPartition: TopicPartition, newReplicas: Seq[Int]): Unit = {
-    partitionAssignments.getOrElseUpdate(topicPartition.topic, mutable.Map.empty)
-      .put(topicPartition.partition, newReplicas)
+    val assignments = partitionAssignments.getOrElseUpdate(topicPartition.topic, mutable.Map.empty)
+    val newAssignment = assignments.get(topicPartition.partition) match {
+      case Some(partitionAssignment) =>
+        PartitionReplicaAssignment(
+          newReplicas,
+          partitionAssignment.addingReplicas,
+          partitionAssignment.removingReplicas
+        )
+      case None =>
+        PartitionReplicaAssignment(
+          newReplicas,
+          Seq.empty,
+          Seq.empty
+        )
+    }
+    updatePartitionFullReplicaAssignment(topicPartition, newAssignment)
+  }
+
+  def updatePartitionFullReplicaAssignment(topicPartition: TopicPartition, newAssignment: PartitionReplicaAssignment): Unit = {
+    val assignments = partitionAssignments.getOrElseUpdate(topicPartition.topic, mutable.Map.empty)
+    assignments.put(topicPartition.partition, newAssignment)
   }
 
   def partitionReplicaAssignmentForTopic(topic : String): Map[TopicPartition, Seq[Int]] = {
     partitionAssignments.getOrElse(topic, Map.empty).map {
-      case (partition, replicas) => (new TopicPartition(topic, partition), replicas)
+      case (partition, assignment) => (new TopicPartition(topic, partition), assignment.replicas)
+    }.toMap
+  }
+
+  def partitionFullReplicaAssignmentForTopic(topic : String): Map[TopicPartition, PartitionReplicaAssignment] = {
+    partitionAssignments.getOrElse(topic, Map.empty).map {
+      case (partition, assignment) => (new TopicPartition(topic, partition), assignment)
     }.toMap
   }
 
@@ -131,7 +205,7 @@ class ControllerContext {
   def partitionsOnBroker(brokerId: Int): Set[TopicPartition] = {
     partitionAssignments.flatMap {
       case (topic, topicReplicaAssignment) => topicReplicaAssignment.filter {
-        case (_, replicas) => replicas.contains(brokerId)
+        case (_, partitionAssignment) => partitionAssignment.replicas.contains(brokerId)
       }.map {
         case (partition, _) => new TopicPartition(topic, partition)
       }
@@ -150,7 +224,7 @@ class ControllerContext {
     brokerIds.flatMap { brokerId =>
       partitionAssignments.flatMap {
         case (topic, topicReplicaAssignment) => topicReplicaAssignment.collect {
-          case (partition, replicas)  if replicas.contains(brokerId) =>
+          case (partition, partitionAssignment) if partitionAssignment.replicas.contains(brokerId) =>
             PartitionAndReplica(new TopicPartition(topic, partition), brokerId)
         }
       }
@@ -159,7 +233,7 @@ class ControllerContext {
 
   def replicasForTopic(topic: String): Set[PartitionAndReplica] = {
     partitionAssignments.getOrElse(topic, mutable.Map.empty).flatMap {
-      case (partition, replicas) => replicas.map(r => PartitionAndReplica(new TopicPartition(topic, partition), r))
+      case (partition, assignment) => assignment.replicas.map(r => PartitionAndReplica(new TopicPartition(topic, partition), r))
     }.toSet
   }
 
@@ -183,10 +257,10 @@ class ControllerContext {
   def onlineAndOfflineReplicas: (Set[PartitionAndReplica], Set[PartitionAndReplica]) = {
     val onlineReplicas = mutable.Set.empty[PartitionAndReplica]
     val offlineReplicas = mutable.Set.empty[PartitionAndReplica]
-    for ((topic, partitionReplicas) <- partitionAssignments;
-         (partitionId, replicas) <- partitionReplicas) {
+    for ((topic, partitionAssignments) <- partitionAssignments;
+         (partitionId, assignment) <- partitionAssignments) {
       val partition = new TopicPartition(topic, partitionId)
-      for (replica <- replicas) {
+      for (replica <- assignment.replicas) {
         val partitionAndReplica = PartitionAndReplica(partition, replica)
         if (isReplicaOnline(replica, partition))
           onlineReplicas.add(partitionAndReplica)
diff --git a/core/src/main/scala/kafka/controller/ControllerState.scala b/core/src/main/scala/kafka/controller/ControllerState.scala
index aa41c7f54586a..dc0df2f3e4623 100644
--- a/core/src/main/scala/kafka/controller/ControllerState.scala
+++ b/core/src/main/scala/kafka/controller/ControllerState.scala
@@ -58,7 +58,7 @@ object ControllerState {
     def value = 4
   }
 
-  case object PartitionReassignment extends ControllerState {
+  case object AlterPartitionReassignment extends ControllerState {
     def value = 5
   }
 
@@ -98,7 +98,11 @@ object ControllerState {
     def value = 14
   }
 
+  case object ListPartitionReassignment extends ControllerState {
+    def value = 15
+  }
+
   val values: Seq[ControllerState] = Seq(Idle, ControllerChange, BrokerChange, TopicChange, TopicDeletion,
-    PartitionReassignment, AutoLeaderBalance, ManualLeaderBalance, ControlledShutdown, IsrChange, LeaderAndIsrResponseReceived,
-    LogDirChange, ControllerShutdown, UncleanLeaderElectionEnable, TopicUncleanLeaderElectionEnable)
+    AlterPartitionReassignment, AutoLeaderBalance, ManualLeaderBalance, ControlledShutdown, IsrChange, LeaderAndIsrResponseReceived,
+    LogDirChange, ControllerShutdown, UncleanLeaderElectionEnable, TopicUncleanLeaderElectionEnable, ListPartitionReassignment)
 }
diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala
index b9bd731087d46..04703e68aa51d 100644
--- a/core/src/main/scala/kafka/controller/KafkaController.scala
+++ b/core/src/main/scala/kafka/controller/KafkaController.scala
@@ -22,7 +22,7 @@ import com.yammer.metrics.core.Gauge
 import kafka.admin.AdminOperationException
 import kafka.api._
 import kafka.common._
-import kafka.controller.KafkaController.ElectLeadersCallback
+import kafka.controller.KafkaController.{AlterReassignmentsCallback, ElectLeadersCallback, ListReassignmentsCallback}
 import kafka.metrics.{KafkaMetricsGroup, KafkaTimer}
 import kafka.server._
 import kafka.utils._
@@ -54,6 +54,8 @@ object KafkaController extends Logging {
   val InitialControllerEpochZkVersion = 0
 
   type ElectLeadersCallback = Map[TopicPartition, Either[ApiError, Int]] => Unit
+  type ListReassignmentsCallback = Either[Map[TopicPartition, PartitionReplicaAssignment], ApiError] => Unit
+  type AlterReassignmentsCallback = Either[Map[TopicPartition, ApiError], ApiError] => Unit
 }
 
 class KafkaController(val config: KafkaConfig,
@@ -534,104 +536,195 @@ class KafkaController(val config: KafkaConfig,
   }
 
   /**
-   * This callback is invoked by the reassigned partitions listener. When an admin command initiates a partition
-   * reassignment, it creates the /admin/reassign_partitions path that triggers the zookeeper listener.
+   * This callback is invoked:
+   * 1. By the AlterPartitionReassignments API
+   * 2. By the reassigned partitions listener which is triggered when the /admin/reassign/partitions znode is created
+   * 3. When an ongoing reassignment finishes - this is detected by a change in the partition's ISR znode
+   * 4. Whenever a new broker comes up which is part of an ongoing reassignment
+   * 5. On controller startup/failover
+   *
+   *
    * Reassigning replicas for a partition goes through a few steps listed in the code.
-   * RAR = Reassigned replicas
-   * OAR = Original list of replicas for partition
-   * AR = current assigned replicas
+   * RS = current assigned replica set
+   * ORS = Original replica set for partition
+   * TRS = Reassigned (target) replica set
+   * AR = The replicas we are adding as part of this reassignment
+   * RR = The replicas we are removing as part of this reassignment
+   *
+   * A reassignment may have up to three phases, each with its own steps:
+   *
+   *
+   * Cleanup Phase: In the cases where this reassignment has to override an ongoing reassignment.
+   *   . The ongoing reassignment is in phase A
+   *   . ORS denotes the original replica set, prior to the ongoing reassignment
+   *   . URS denotes the unnecessary replicas, ones which are currently part of the AR of the ongoing reassignment but will not be part of the new one
+   *   . OVRS denotes the overlapping replica set - replicas which are part of the AR of the ongoing reassignment and will be part of the overriding reassignment
+   *       (it is essentially (RS - ORS) - URS)
+   *
+   *   1 Set RS = ORS + OVRS, AR = OVRS, RS = [] in memory
+   *   2 Send LeaderAndIsr request with RS = ORS + OVRS, AR = [], RS = [] to all brokers in ORS + OVRS
+   *     (because the ongoing reassignment is in phase A, we know we wouldn't have a leader in URS
+   *      unless a preferred leader election was triggered while the reassignment was happening)
+   *   3 Replicas in URS -> Offline (force those replicas out of ISR)
+   *   4 Replicas in URS -> NonExistentReplica (force those replicas to be deleted)
+   *
+   * Phase A: Initial trigger (when TRS != ISR)
+   *   A1. Update ZK with RS = ORS + TRS,
+   *                      AR = TRS - ORS and
+   *                      RR = ORS - TRS.
+   *   A2. Update memory with RS = ORS + TRS, AR = TRS - ORS and RR = ORS - TRS
+   *   A3. Send LeaderAndIsr request to every replica in ORS + TRS (with the new RS, AR and RR).
+   *       We do this by forcing an update of the leader epoch in zookeeper.
+   *   A4. Start new replicas AR by moving replicas in AR to NewReplica state.
    *
-   * 1. Update AR in ZK with OAR + RAR.
-   * 2. Send LeaderAndIsr request to every replica in OAR + RAR (with AR as OAR + RAR). We do this by forcing an update
-   *    of the leader epoch in zookeeper.
-   * 3. Start new replicas RAR - OAR by moving replicas in RAR - OAR to NewReplica state.
-   * 4. Wait until all replicas in RAR are in sync with the leader.
-   * 5  Move all replicas in RAR to OnlineReplica state.
-   * 6. Set AR to RAR in memory.
-   * 7. If the leader is not in RAR, elect a new leader from RAR. If new leader needs to be elected from RAR, a LeaderAndIsr
-   *    will be sent. If not, then leader epoch will be incremented in zookeeper and a LeaderAndIsr request will be sent.
-   *    In any case, the LeaderAndIsr request will have AR = RAR. This will prevent the leader from adding any replica in
-   *    RAR - OAR back in the isr.
-   * 8. Move all replicas in OAR - RAR to OfflineReplica state. As part of OfflineReplica state change, we shrink the
-   *    isr to remove OAR - RAR in zookeeper and send a LeaderAndIsr ONLY to the Leader to notify it of the shrunk isr.
-   *    After that, we send a StopReplica (delete = false) to the replicas in OAR - RAR.
-   * 9. Move all replicas in OAR - RAR to NonExistentReplica state. This will send a StopReplica (delete = true) to
-   *    the replicas in OAR - RAR to physically delete the replicas on disk.
-   * 10. Update AR in ZK with RAR.
-   * 11. Update the /admin/reassign_partitions path in ZK to remove this partition.
-   * 12. After electing leader, the replicas and isr information changes. So resend the update metadata request to every broker.
+   * Phase B: All of TRS have caught up with the leaders and are in ISR
+   *   B1. Move all replicas in TRS to OnlineReplica state.
+   *   B2. Set RS = TRS, AR = [], RR = [] in memory.
+   *   B3. Send a LeaderAndIsr request with RS = TRS. This will prevent the leader from adding any replica in TRS - ORS back in the isr.
+   *       If the current leader is not in TRS or isn't alive, we move the leader to a new replica in TRS.
+   *       We may send the LeaderAndIsr to more than the TRS replicas due to the
+   *       way the partition state machine works (it reads replicas from ZK)
+   *   B4. Move all replicas in RR to OfflineReplica state. As part of OfflineReplica state change, we shrink the
+   *       isr to remove RR in ZooKeeper and send a LeaderAndIsr ONLY to the Leader to notify it of the shrunk isr.
+   *       After that, we send a StopReplica (delete = false) to the replicas in RR.
+   *   B5. Move all replicas in RR to NonExistentReplica state. This will send a StopReplica (delete = true) to
+   *       the replicas in RR to physically delete the replicas on disk.
+   *   B6. Update ZK with RS=TRS, AR=[], RR=[].
+   *   B7. Remove the ISR reassign listener and maybe update the /admin/reassign_partitions path in ZK to remove this partition from it if present.
+   *   B8. After electing leader, the replicas and isr information changes. So resend the update metadata request to every broker.
    *
-   * For example, if OAR = {1, 2, 3} and RAR = {4,5,6}, the values in the assigned replica (AR) and leader/isr path in ZK
-   * may go through the following transition.
-   * AR                 leader/isr
-   * {1,2,3}            1/{1,2,3}           (initial state)
-   * {1,2,3,4,5,6}      1/{1,2,3}           (step 2)
-   * {1,2,3,4,5,6}      1/{1,2,3,4,5,6}     (step 4)
-   * {1,2,3,4,5,6}      4/{1,2,3,4,5,6}     (step 7)
-   * {1,2,3,4,5,6}      4/{4,5,6}           (step 8)
-   * {4,5,6}            4/{4,5,6}           (step 10)
+   * In general, there are two goals we want to aim for:
+   * 1. Every replica present in the replica set of a LeaderAndIsrRequest gets the request sent to it
+   * 2. Replicas that are removed from a partition's assignment get StopReplica sent to them
    *
-   * Note that we have to update AR in ZK with RAR last since it's the only place where we store OAR persistently.
+   * For example, if ORS = {1,2,3} and TRS = {4,5,6}, the values in the topic and leader/isr paths in ZK
+   * may go through the following transitions.
+   * RS                AR          RR          leader     isr
+   * {1,2,3}           {}          {}          1          {1,2,3}           (initial state)
+   * {4,5,6,1,2,3}     {4,5,6}     {1,2,3}     1          {1,2,3}           (step A2)
+   * {4,5,6,1,2,3}     {4,5,6}     {1,2,3}     1          {1,2,3,4,5,6}     (phase B)
+   * {4,5,6,1,2,3}     {4,5,6}     {1,2,3}     4          {1,2,3,4,5,6}     (step B3)
+   * {4,5,6,1,2,3}     {4,5,6}     {1,2,3}     4          {4,5,6}           (step B4)
+   * {4,5,6}           {}          {}          4          {4,5,6}           (step B6)
+   *
+   * Note that we have to update RS in ZK with TRS last since it's the only place where we store ORS persistently.
    * This way, if the controller crashes before that step, we can still recover.
    */
   private def onPartitionReassignment(topicPartition: TopicPartition, reassignedPartitionContext: ReassignedPartitionsContext): Unit = {
-    val reassignedReplicas = reassignedPartitionContext.newReplicas
-    if (!areReplicasInIsr(topicPartition, reassignedReplicas)) {
-      info(s"New replicas ${reassignedReplicas.mkString(",")} for partition $topicPartition being reassigned not yet " +
+    val originalAssignmentOpt = maybeRevertOngoingReassignment(topicPartition, reassignedPartitionContext)
+    val oldReplicas = originalAssignmentOpt match {
+      case Some(originalReplicas) => originalReplicas
+      case None => controllerContext.partitionFullReplicaAssignment(topicPartition).previousAssignment.replicas
+    }
+    // RS = ORS + TRS, AR = TRS - ORS and RR = ORS - TRS
+    val partitionAssignment = PartitionReplicaAssignment.fromOldAndNewReplicas(
+      oldReplicas = oldReplicas,
+      newReplicas = reassignedPartitionContext.newReplicas)
+    assert(reassignedPartitionContext.newReplicas == partitionAssignment.targetReplicas,
+      s"newReplicas ${reassignedPartitionContext.newReplicas} were not equal to the expected targetReplicas ${partitionAssignment.targetReplicas}")
+    val targetReplicas = partitionAssignment.targetReplicas
+
+    if (!areReplicasInIsr(topicPartition, targetReplicas)) {
+      info(s"New replicas ${targetReplicas.mkString(",")} for partition $topicPartition being reassigned not yet " +
         "caught up with the leader")
-      val newReplicasNotInOldReplicaList = reassignedReplicas.toSet -- controllerContext.partitionReplicaAssignment(topicPartition).toSet
-      val newAndOldReplicas = (reassignedPartitionContext.newReplicas ++ controllerContext.partitionReplicaAssignment(topicPartition)).toSet
-      //1. Update AR in ZK with OAR + RAR.
-      updateAssignedReplicasForPartition(topicPartition, newAndOldReplicas.toSeq)
-      //2. Send LeaderAndIsr request to every replica in OAR + RAR (with AR as OAR + RAR).
-      updateLeaderEpochAndSendRequest(topicPartition, controllerContext.partitionReplicaAssignment(topicPartition),
-        newAndOldReplicas.toSeq)
-      //3. replicas in RAR - OAR -> NewReplica
-      startNewReplicasForReassignedPartition(topicPartition, reassignedPartitionContext, newReplicasNotInOldReplicaList)
-      info(s"Waiting for new replicas ${reassignedReplicas.mkString(",")} for partition ${topicPartition} being " +
-        "reassigned to catch up with the leader")
+
+      // A1. Update ZK with RS = ORS + TRS, AR = TRS - ORS and RR = ORS - TRS.
+      updateReplicaAssignmentForPartition(topicPartition, partitionAssignment)
+      // A2. Update memory with RS = ORS + TRS, AR = TRS - ORS and RR = ORS - TRS
+      controllerContext.updatePartitionFullReplicaAssignment(topicPartition, partitionAssignment)
+      // A3. Send LeaderAndIsr request to every replica in ORS + TRS (with the new RS, AR and RR).
+      val updatedAssignment = controllerContext.partitionFullReplicaAssignment(topicPartition)
+      updateLeaderEpochAndSendRequest(topicPartition, updatedAssignment.replicas, updatedAssignment)
+      // A4. replicas in AR -> NewReplica
+      startNewReplicasForReassignedPartition(topicPartition, updatedAssignment.addingReplicas)
+      info(s"Waiting for new replicas ${updatedAssignment.addingReplicas.mkString(",")} for partition $topicPartition being " +
+        s"reassigned to catch up with the leader (target replicas ${updatedAssignment.targetReplicas})")
     } else {
-      //4. Wait until all replicas in RAR are in sync with the leader.
-      val oldReplicas = controllerContext.partitionReplicaAssignment(topicPartition).toSet -- reassignedReplicas.toSet
-      //5. replicas in RAR -> OnlineReplica
-      reassignedReplicas.foreach { replica =>
-        replicaStateMachine.handleStateChanges(Seq(new PartitionAndReplica(topicPartition, replica)), OnlineReplica)
+      // B1. replicas in TRS -> OnlineReplica
+      targetReplicas.foreach { replica =>
+        replicaStateMachine.handleStateChanges(Seq(PartitionAndReplica(topicPartition, replica)), OnlineReplica)
       }
-      //6. Set AR to RAR in memory.
-      //7. Send LeaderAndIsr request with a potential new leader (if current leader not in RAR) and
-      //   a new AR (using RAR) and same isr to every broker in RAR
-      moveReassignedPartitionLeaderIfRequired(topicPartition, reassignedPartitionContext)
-      //8. replicas in OAR - RAR -> Offline (force those replicas out of isr)
-      //9. replicas in OAR - RAR -> NonExistentReplica (force those replicas to be deleted)
-      stopOldReplicasOfReassignedPartition(topicPartition, reassignedPartitionContext, oldReplicas)
-      //10. Update AR in ZK with RAR.
-      updateAssignedReplicasForPartition(topicPartition, reassignedReplicas)
-      //11. Update the /admin/reassign_partitions path in ZK to remove this partition.
-      removePartitionsFromReassignedPartitions(Set(topicPartition))
-      //12. After electing leader, the replicas and isr information changes, so resend the update metadata request to every broker
+      // B2. Set RS = TRS, AR = [], RR = [] in memory.
+      // B3. Send LeaderAndIsr request with a potential new leader (if current leader not in TRS) and
+      //   a new RS (using TRS) and same isr to every broker in ORS + TRS or TRS
+      moveReassignedPartitionLeaderIfRequired(topicPartition, reassignedPartitionContext, partitionAssignment)
+      // B4. replicas in RR -> Offline (force those replicas out of isr)
+      // B5. replicas in RR -> NonExistentReplica (force those replicas to be deleted)
+      stopRemovedReplicasOfReassignedPartition(topicPartition, partitionAssignment.removingReplicas)
+      // B6. Update ZK with RS = TRS, AR = [], RR = [].
+      updateReplicaAssignmentForPartition(topicPartition, controllerContext.partitionFullReplicaAssignment(topicPartition))
+      // B7. Remove the ISR reassign listener and maybe update the /admin/reassign_partitions path in ZK to remove this partition from it.
+      removePartitionFromReassignedPartitions(topicPartition)
+      // B8. After electing a leader in B3, the replicas and isr information changes, so resend the update metadata request to every broker
       sendUpdateMetadataRequest(controllerContext.liveOrShuttingDownBrokerIds.toSeq, Set(topicPartition))
       // signal delete topic thread if reassignment for some partitions belonging to topics being deleted just completed
       topicDeletionManager.resumeDeletionForTopics(Set(topicPartition.topic))
     }
   }
 
+  /**
+    * This is called in case we need to override/revert an ongoing reassignment.
+    * Note that due to the way we compute the original replica set, we have no guarantee that a revert would put it in the same order.
+    * @return An option of the original replicas prior to the ongoing reassignment. None if there is no ongoing reassignment
+    */
+  private def maybeRevertOngoingReassignment(topicPartition: TopicPartition, reassignedPartitionContext: ReassignedPartitionsContext): Option[Seq[Int]] = {
+    reassignedPartitionContext.ongoingReassignmentOpt match {
+      case Some(ongoingAssignment) =>
+        val originalAssignment = ongoingAssignment.previousAssignment
+        assert(ongoingAssignment.isBeingReassigned)
+        assert(!originalAssignment.isBeingReassigned)
+
+        val unnecessaryReplicas = ongoingAssignment.replicas.filterNot(originalAssignment.replicas.contains(_))
+        val (overlappingReplicas, replicasToRemove) = unnecessaryReplicas.partition(reassignedPartitionContext.newReplicas.contains(_))
+        // RS = ORS + OVRS, AR = OVRS, RR = []
+        val intermediateReplicaAssignment = PartitionReplicaAssignment(originalAssignment.replicas ++ overlappingReplicas, overlappingReplicas, Seq())
+
+        if (isDebugEnabled)
+          debug(s"Reverting previous reassignment $originalAssignment (we were in the " +
+            s"process of an ongoing reassignment with target replicas ${ongoingAssignment.targetReplicas.mkString(",")} (" +
+            s"Overlapping replicas: ${overlappingReplicas.mkString(",")}, Replicas to remove: ${replicasToRemove.mkString(",")})")
+
+        // Set RS = ORS + OVRS, AR = OVRS, RR = [] in memory.
+        controllerContext.updatePartitionFullReplicaAssignment(topicPartition, intermediateReplicaAssignment)
+
+        // Increment leader epoch and send LeaderAndIsr with new RS (using old replicas and overlapping replicas) and same ISR to every broker in ORS + OVRS
+        // This will prevent the leader from adding any replica outside RS back in the ISR
+        updateLeaderEpochAndSendRequest(topicPartition, intermediateReplicaAssignment.replicas, intermediateReplicaAssignment)
+
+        // replicas in URS -> Offline (force those replicas out of isr)
+        // replicas in URS -> NonExistentReplica (force those replicas to be deleted)
+        stopRemovedReplicasOfReassignedPartition(topicPartition, replicasToRemove)
+        reassignedPartitionContext.ongoingReassignmentOpt = None
+        Some(originalAssignment.replicas)
+      case None => None // nothing to revert
+    }
+  }
+
   /**
    * Trigger partition reassignment for the provided partitions if the assigned replicas are not the same as the
    * reassigned replicas (as defined in `ControllerContext.partitionsBeingReassigned`) and if the topic has not been
    * deleted.
    *
+   * Called when:
+   * 1. zNode is first created
+   * 2. Controller fail over
+   * 3. AlterPartitionReassignments API is called
+   *
    * `partitionsBeingReassigned` must be populated with all partitions being reassigned before this method is invoked
    * as explained in the method documentation of `removePartitionFromReassignedPartitions` (which is invoked by this
    * method).
    *
    * @throws IllegalStateException if a partition is not in `partitionsBeingReassigned`
    */
-  private def maybeTriggerPartitionReassignment(topicPartitions: Set[TopicPartition]): Unit = {
+  private def maybeTriggerPartitionReassignment(topicPartitions: Set[TopicPartition]): Map[TopicPartition, ApiError] = {
+    val reassignmentResults: mutable.Map[TopicPartition, ApiError] = mutable.Map.empty
     val partitionsToBeRemovedFromReassignment = scala.collection.mutable.Set.empty[TopicPartition]
+
     topicPartitions.foreach { tp =>
       if (topicDeletionManager.isTopicQueuedUpForDeletion(tp.topic)) {
         info(s"Skipping reassignment of $tp since the topic is currently being deleted")
         partitionsToBeRemovedFromReassignment.add(tp)
+        reassignmentResults.put(tp, new ApiError(Errors.UNKNOWN_TOPIC_OR_PARTITION, "The partition does not exist."))
       } else {
         val reassignedPartitionContext = controllerContext.partitionsBeingReassigned.get(tp).getOrElse {
           throw new IllegalStateException(s"Initiating reassign replicas for partition $tp not present in " +
@@ -645,32 +738,42 @@ class KafkaController(val config: KafkaConfig,
             info(s"Partition $tp to be reassigned is already assigned to replicas " +
               s"${newReplicas.mkString(",")}. Ignoring request for partition reassignment.")
             partitionsToBeRemovedFromReassignment.add(tp)
+            reassignmentResults.put(tp, ApiError.NONE)
           } else {
             try {
-              info(s"Handling reassignment of partition $tp to new replicas ${newReplicas.mkString(",")}")
+              info(s"Handling reassignment of partition $tp from current replicas ${assignedReplicas.mkString(",")}" +
+                s"to new replicas ${newReplicas.mkString(",")}")
               // first register ISR change listener
               reassignedPartitionContext.registerReassignIsrChangeHandler(zkClient)
               // mark topic ineligible for deletion for the partitions being reassigned
               topicDeletionManager.markTopicIneligibleForDeletion(Set(topic),
                 reason = "topic reassignment in progress")
               onPartitionReassignment(tp, reassignedPartitionContext)
+              reassignmentResults.put(tp, ApiError.NONE)
             } catch {
               case e: ControllerMovedException =>
                 error(s"Error completing reassignment of partition $tp because controller has moved to another broker", e)
                 throw e
               case e: Throwable =>
                 error(s"Error completing reassignment of partition $tp", e)
-                // remove the partition from the admin path to unblock the admin client
                 partitionsToBeRemovedFromReassignment.add(tp)
+                zkClient.getFullReplicaAssignmentForTopics(immutable.Set(tp.topic())).find(_._1 == tp) match {
+                  case Some(persistedAssignment) =>
+                    controllerContext.updatePartitionFullReplicaAssignment(tp, persistedAssignment._2)
+                  case None =>
+                }
+                reassignmentResults.put(tp, new ApiError(Errors.UNKNOWN_SERVER_ERROR))
             }
           }
         } else {
             error(s"Ignoring request to reassign partition $tp that doesn't exist.")
             partitionsToBeRemovedFromReassignment.add(tp)
+            reassignmentResults.put(tp, new ApiError(Errors.UNKNOWN_TOPIC_OR_PARTITION, "The partition does not exist."))
         }
       }
     }
     removePartitionsFromReassignedPartitions(partitionsToBeRemovedFromReassignment)
+    reassignmentResults
   }
 
   /**
@@ -730,8 +833,14 @@ class KafkaController(val config: KafkaConfig,
     info(s"Initialized broker epochs cache: ${controllerContext.liveBrokerIdAndEpochs}")
     controllerContext.allTopics = zkClient.getAllTopicsInCluster.toSet
     registerPartitionModificationsHandlers(controllerContext.allTopics.toSeq)
-    zkClient.getReplicaAssignmentForTopics(controllerContext.allTopics.toSet).foreach {
-      case (topicPartition, assignedReplicas) => controllerContext.updatePartitionReplicaAssignment(topicPartition, assignedReplicas)
+    zkClient.getFullReplicaAssignmentForTopics(controllerContext.allTopics.toSet).foreach {
+      case (topicPartition, replicaAssignment) =>
+        controllerContext.updatePartitionFullReplicaAssignment(topicPartition, replicaAssignment)
+        if (replicaAssignment.isBeingReassigned) {
+          val reassignIsrChangeHandler = new PartitionReassignmentIsrChangeHandler(eventManager, topicPartition)
+          controllerContext.partitionsBeingReassigned.put(topicPartition, ReassignedPartitionsContext(replicaAssignment.targetReplicas,
+            reassignIsrChangeHandler, persistedInZk = false, ongoingReassignmentOpt = None))
+        }
     }
     controllerContext.partitionLeadershipInfo.clear()
     controllerContext.shuttingDownBrokerIds = mutable.Set.empty[Int]
@@ -767,14 +876,27 @@ class KafkaController(val config: KafkaConfig,
     pendingPreferredReplicaElections
   }
 
+  /**
+    * Initializes the partitions being reassigned by reading them from the /admin/reassign_partitions znode
+    * This will overwrite any reassignments that were set by the AlterPartitionReassignments API
+    */
   private def initializePartitionReassignment(): Unit = {
-    // read the partitions being reassigned from zookeeper path /admin/reassign_partitions
     val partitionsBeingReassigned = zkClient.getPartitionReassignment
-    info(s"Partitions being reassigned: $partitionsBeingReassigned")
+    info(s"DEPRECATED: Partitions being reassigned through ZooKeeper: $partitionsBeingReassigned")
+
+    partitionsBeingReassigned.foreach {
+      case (tp, newReplicas) =>
+        val reassignIsrChangeHandler = new PartitionReassignmentIsrChangeHandler(eventManager, tp)
+        val assignment = controllerContext.partitionFullReplicaAssignment(tp)
+        val ongoingReassignmentOption = if (assignment.isBeingReassigned)
+          Some(assignment)
+        else
+          None
 
-    controllerContext.partitionsBeingReassigned ++= partitionsBeingReassigned.iterator.map { case (tp, newReplicas) =>
-      val reassignIsrChangeHandler = new PartitionReassignmentIsrChangeHandler(eventManager, tp)
-      tp -> ReassignedPartitionsContext(newReplicas, reassignIsrChangeHandler)
+        controllerContext.partitionsBeingReassigned += (
+          tp -> ReassignedPartitionsContext(newReplicas, reassignIsrChangeHandler,
+            persistedInZk = true,
+            ongoingReassignmentOpt = ongoingReassignmentOption))
     }
   }
 
@@ -805,13 +927,19 @@ class KafkaController(val config: KafkaConfig,
   }
 
   private def moveReassignedPartitionLeaderIfRequired(topicPartition: TopicPartition,
-                                                      reassignedPartitionContext: ReassignedPartitionsContext): Unit = {
+                                                      reassignedPartitionContext: ReassignedPartitionsContext,
+                                                      currentAssignment: PartitionReplicaAssignment): Unit = {
     val reassignedReplicas = reassignedPartitionContext.newReplicas
     val currentLeader = controllerContext.partitionLeadershipInfo(topicPartition).leaderAndIsr.leader
+
     // change the assigned replica list to just the reassigned replicas in the cache so it gets sent out on the LeaderAndIsr
     // request to the current or new leader. This will prevent it from adding the old replicas to the ISR
-    val oldAndNewReplicas = controllerContext.partitionReplicaAssignment(topicPartition)
-    controllerContext.updatePartitionReplicaAssignment(topicPartition, reassignedReplicas)
+    val newAssignment = PartitionReplicaAssignment(replicas = reassignedReplicas, addingReplicas = Seq(), removingReplicas = Seq())
+    controllerContext.updatePartitionFullReplicaAssignment(
+      topicPartition,
+      newAssignment
+    )
+
     if (!reassignedPartitionContext.newReplicas.contains(currentLeader)) {
       info(s"Leader $currentLeader for partition $topicPartition being reassigned, " +
         s"is not in the new list of replicas ${reassignedReplicas.mkString(",")}. Re-electing leader")
@@ -823,7 +951,7 @@ class KafkaController(val config: KafkaConfig,
         info(s"Leader $currentLeader for partition $topicPartition being reassigned, " +
           s"is already in the new list of replicas ${reassignedReplicas.mkString(",")} and is alive")
         // shrink replication factor and update the leader epoch in zookeeper to use on the next LeaderAndIsrRequest
-        updateLeaderEpochAndSendRequest(topicPartition, oldAndNewReplicas, reassignedReplicas)
+        updateLeaderEpochAndSendRequest(topicPartition, newAssignment.replicas, newAssignment)
       } else {
         info(s"Leader $currentLeader for partition $topicPartition being reassigned, " +
           s"is already in the new list of replicas ${reassignedReplicas.mkString(",")} but is dead")
@@ -832,45 +960,46 @@ class KafkaController(val config: KafkaConfig,
     }
   }
 
-  private def stopOldReplicasOfReassignedPartition(topicPartition: TopicPartition,
-                                                   reassignedPartitionContext: ReassignedPartitionsContext,
-                                                   oldReplicas: Set[Int]): Unit = {
+  private def stopRemovedReplicasOfReassignedPartition(topicPartition: TopicPartition,
+                                                       removedReplicas: Seq[Int]): Unit = {
     // first move the replica to offline state (the controller removes it from the ISR)
-    val replicasToBeDeleted = oldReplicas.map(PartitionAndReplica(topicPartition, _))
-    replicaStateMachine.handleStateChanges(replicasToBeDeleted.toSeq, OfflineReplica)
+    val replicasToBeDeleted = removedReplicas.map(PartitionAndReplica(topicPartition, _))
+    replicaStateMachine.handleStateChanges(replicasToBeDeleted, OfflineReplica)
     // send stop replica command to the old replicas
-    replicaStateMachine.handleStateChanges(replicasToBeDeleted.toSeq, ReplicaDeletionStarted)
+    replicaStateMachine.handleStateChanges(replicasToBeDeleted, ReplicaDeletionStarted)
     // TODO: Eventually partition reassignment could use a callback that does retries if deletion failed
-    replicaStateMachine.handleStateChanges(replicasToBeDeleted.toSeq, ReplicaDeletionSuccessful)
-    replicaStateMachine.handleStateChanges(replicasToBeDeleted.toSeq, NonExistentReplica)
+    replicaStateMachine.handleStateChanges(replicasToBeDeleted, ReplicaDeletionSuccessful)
+    replicaStateMachine.handleStateChanges(replicasToBeDeleted, NonExistentReplica)
   }
 
-  private def updateAssignedReplicasForPartition(partition: TopicPartition,
-                                                 replicas: Seq[Int]): Unit = {
-    controllerContext.updatePartitionReplicaAssignment(partition, replicas)
-    val setDataResponse = zkClient.setTopicAssignmentRaw(partition.topic, controllerContext.partitionReplicaAssignmentForTopic(partition.topic), controllerContext.epochZkVersion)
+  private def updateReplicaAssignmentForPartition(partition: TopicPartition,
+                                                  assignment: PartitionReplicaAssignment): Unit = {
+    var topicAssignment = controllerContext.partitionFullReplicaAssignmentForTopic(partition.topic)
+    topicAssignment += partition -> assignment
+
+    val setDataResponse = zkClient.setTopicAssignmentRaw(partition.topic, topicAssignment, controllerContext.epochZkVersion)
     setDataResponse.resultCode match {
       case Code.OK =>
-        info(s"Updated assigned replicas for partition $partition being reassigned to ${replicas.mkString(",")}")
-        // update the assigned replica list after a successful zookeeper write
-        controllerContext.updatePartitionReplicaAssignment(partition, replicas)
+        info(s"Updated assigned replicas for partition $partition being reassigned to ${assignment.targetReplicas.mkString(",")}" +
+          s" (addingReplicas: ${assignment.addingReplicas.mkString(",")}, removingReplicas: ${assignment.removingReplicas.mkString(",")})")
       case Code.NONODE => throw new IllegalStateException(s"Topic ${partition.topic} doesn't exist")
       case _ => throw new KafkaException(setDataResponse.resultException.get)
     }
   }
 
-  private def startNewReplicasForReassignedPartition(topicPartition: TopicPartition,
-                                                     reassignedPartitionContext: ReassignedPartitionsContext,
-                                                     newReplicas: Set[Int]): Unit = {
+  private def startNewReplicasForReassignedPartition(topicPartition: TopicPartition, newReplicas: Seq[Int]): Unit = {
     // send the start replica request to the brokers in the reassigned replicas list that are not in the assigned
     // replicas list
     newReplicas.foreach { replica =>
-      replicaStateMachine.handleStateChanges(Seq(new PartitionAndReplica(topicPartition, replica)), NewReplica)
+      replicaStateMachine.handleStateChanges(Seq(PartitionAndReplica(topicPartition, replica)), NewReplica)
     }
   }
 
-  private def updateLeaderEpochAndSendRequest(partition: TopicPartition, replicasToReceiveRequest: Seq[Int], newAssignedReplicas: Seq[Int]): Unit = {
+  private def updateLeaderEpochAndSendRequest(partition: TopicPartition, replicasToReceiveRequest: Seq[Int],
+                                              newAssignedReplicas: PartitionReplicaAssignment): Unit = {
     val stateChangeLog = stateChangeLogger.withControllerEpoch(controllerContext.epoch)
+    val replicaSetStr = s"replica set ${newAssignedReplicas.replicas.mkString(",")} " +
+      s"(addingReplicas: ${newAssignedReplicas.addingReplicas.mkString(",")}, removingReplicas: ${newAssignedReplicas.removingReplicas.mkString(",")})"
     updateLeaderEpoch(partition) match {
       case Some(updatedLeaderIsrAndControllerEpoch) =>
         try {
@@ -882,12 +1011,12 @@ class KafkaController(val config: KafkaConfig,
           case e: IllegalStateException =>
             handleIllegalState(e)
         }
-        stateChangeLog.trace(s"Sent LeaderAndIsr request $updatedLeaderIsrAndControllerEpoch with new assigned replica " +
-          s"list ${newAssignedReplicas.mkString(",")} to leader ${updatedLeaderIsrAndControllerEpoch.leaderAndIsr.leader} " +
+        stateChangeLog.trace(s"Sent LeaderAndIsr request $updatedLeaderIsrAndControllerEpoch with new assigned $replicaSetStr" +
+          s"to leader ${updatedLeaderIsrAndControllerEpoch.leaderAndIsr.leader} " +
           s"for partition being reassigned $partition")
       case None => // fail the reassignment
-        stateChangeLog.error("Failed to send LeaderAndIsr request with new assigned replica list " +
-          s"${newAssignedReplicas.mkString( ",")} to leader for partition being reassigned $partition")
+        stateChangeLog.error(s"Failed to send LeaderAndIsr request with new assigned $replicaSetStr " +
+          s"to leader for partition being reassigned $partition")
     }
   }
 
@@ -914,7 +1043,7 @@ class KafkaController(val config: KafkaConfig,
    * is complete (i.e. there is no other partition with a reassignment in progress), the reassign_partitions znode
    * is deleted.
    *
-   * `ControllerContext.partitionsBeingReassigned` must be populated with all partitions being reassigned before this
+   * `ControllerContext.partitionsBeingReassigned` must be populated with all the zk-persisted partition reassignments before this
    * method is invoked to avoid premature deletion of the `reassign_partitions` znode.
    */
   private def removePartitionsFromReassignedPartitions(partitionsToBeRemoved: Set[TopicPartition]): Unit = {
@@ -922,7 +1051,33 @@ class KafkaController(val config: KafkaConfig,
       reassignContext.unregisterReassignIsrChangeHandler(zkClient)
     }
 
-    val updatedPartitionsBeingReassigned = controllerContext.partitionsBeingReassigned -- partitionsToBeRemoved
+    removePartitionsFromZkReassignment(partitionsToBeRemoved)
+
+    controllerContext.partitionsBeingReassigned --= partitionsToBeRemoved
+  }
+
+  private def removePartitionFromReassignedPartitions(partitionToBeRemoved: TopicPartition) {
+    controllerContext.partitionsBeingReassigned.get(partitionToBeRemoved) match {
+      case Some(reassignContext) =>
+        reassignContext.unregisterReassignIsrChangeHandler(zkClient)
+
+        if (reassignContext.persistedInZk) {
+          removePartitionsFromZkReassignment(Set(partitionToBeRemoved))
+        }
+
+        controllerContext.partitionsBeingReassigned.remove(partitionToBeRemoved)
+      case None =>
+        throw new IllegalStateException("Cannot remove a reassigning partition because it is not present in memory")
+    }
+  }
+
+  private def removePartitionsFromZkReassignment(partitionsToBeRemoved: Set[TopicPartition]): Unit = {
+    if (!zkClient.reassignPartitionsInProgress()) {
+      debug(s"Cannot remove partitions $partitionsToBeRemoved from ZooKeeper because there is no ZooKeeper reassignment present")
+      return
+    }
+
+    val updatedPartitionsBeingReassigned = controllerContext.partitionsBeingReassigned.filter(_._2.persistedInZk).toMap -- partitionsToBeRemoved
 
     info(s"Removing partitions $partitionsToBeRemoved from the list of reassigned partitions in zookeeper")
 
@@ -931,7 +1086,7 @@ class KafkaController(val config: KafkaConfig,
       info(s"No more partitions need to be reassigned. Deleting zk path ${ReassignPartitionsZNode.path}")
       zkClient.deletePartitionReassignment(controllerContext.epochZkVersion)
       // Ensure we detect future reassignments
-      eventManager.put(PartitionReassignment)
+      eventManager.put(PartitionReassignment(None, None))
     } else {
       val reassignment = updatedPartitionsBeingReassigned.map { case (k, v) => k -> v.newReplicas }
       try zkClient.setOrCreatePartitionReassignment(reassignment, controllerContext.epochZkVersion)
@@ -939,8 +1094,6 @@ class KafkaController(val config: KafkaConfig,
         case e: KeeperException => throw new AdminOperationException(e)
       }
     }
-
-    controllerContext.partitionsBeingReassigned --= partitionsToBeRemoved
   }
 
   private def removePartitionsFromPreferredReplicaElection(partitionsToBeRemoved: Set[TopicPartition],
@@ -1216,11 +1369,21 @@ class KafkaController(val config: KafkaConfig,
         0
       } else {
         controllerContext.allPartitions.count { topicPartition =>
-          val replicas = controllerContext.partitionReplicaAssignment(topicPartition)
+          val replicaAssignment: PartitionReplicaAssignment = controllerContext.partitionFullReplicaAssignment(topicPartition)
+          val replicas = replicaAssignment.replicas
           val preferredReplica = replicas.head
-          val leadershipInfo = controllerContext.partitionLeadershipInfo.get(topicPartition)
-          leadershipInfo.map(_.leaderAndIsr.leader != preferredReplica).getOrElse(false) &&
-            !topicDeletionManager.isTopicQueuedUpForDeletion(topicPartition.topic)
+
+          val isImbalanced = controllerContext.partitionLeadershipInfo.get(topicPartition) match {
+            case Some(leadershipInfo) =>
+              if (replicaAssignment.isBeingReassigned && replicaAssignment.addingReplicas.contains(preferredReplica))
+                // reassigning partitions are not counted as imbalanced until the new replica joins the ISR (completes reassignment)
+                leadershipInfo.leaderAndIsr.isr.contains(preferredReplica)
+              else
+                leadershipInfo.leaderAndIsr.leader != preferredReplica
+            case None => false
+          }
+
+          isImbalanced && !topicDeletionManager.isTopicQueuedUpForDeletion(topicPartition.topic)
         }
       }
 
@@ -1388,10 +1551,10 @@ class KafkaController(val config: KafkaConfig,
     controllerContext.allTopics = topics
 
     registerPartitionModificationsHandlers(newTopics.toSeq)
-    val addedPartitionReplicaAssignment = zkClient.getReplicaAssignmentForTopics(newTopics)
+    val addedPartitionReplicaAssignment = zkClient.getFullReplicaAssignmentForTopics(newTopics)
     deletedTopics.foreach(controllerContext.removeTopic)
     addedPartitionReplicaAssignment.foreach {
-      case (topicAndPartition, newReplicas) => controllerContext.updatePartitionReplicaAssignment(topicAndPartition, newReplicas)
+      case (topicAndPartition, newReplicaAssignment) => controllerContext.updatePartitionFullReplicaAssignment(topicAndPartition, newReplicaAssignment)
     }
     info(s"New topics: [$newTopics], deleted topics: [$deletedTopics], new partition replica assignment " +
       s"[$addedPartitionReplicaAssignment]")
@@ -1416,10 +1579,15 @@ class KafkaController(val config: KafkaConfig,
       info("Restoring the partition replica assignment for topic %s".format(topic))
 
       val existingPartitions = zkClient.getChildren(TopicPartitionsZNode.path(topic))
-      val existingPartitionReplicaAssignment = newPartitionReplicaAssignment.filter(p =>
-        existingPartitions.contains(p._1.partition.toString))
+      val existingPartitionReplicaAssignment = newPartitionReplicaAssignment
+        .filter(p => existingPartitions.contains(p._1.partition.toString))
+        .map { case (tp, _) =>
+          tp -> controllerContext.partitionFullReplicaAssignment(tp)
+      }.toMap
 
-      zkClient.setTopicAssignment(topic, existingPartitionReplicaAssignment, controllerContext.epochZkVersion)
+      zkClient.setTopicAssignment(topic,
+        existingPartitionReplicaAssignment,
+        controllerContext.epochZkVersion)
     }
 
     if (!isActive) return
@@ -1479,25 +1647,134 @@ class KafkaController(val config: KafkaConfig,
     }
   }
 
-  private def processPartitionReassignment(): Unit = {
-    if (!isActive) return
+  /**
+   * Process a partition reassignment.
+   * A partition reassignment can be triggered through the AlterPartitionReassignment API (in which case reassignmentsOpt is present)
+   *   or through the /admin/reassign_partitions znode (in which case reassignmentsOpt is None).
+   * In both cases, we need to populate `partitionsBeingReassigned` with all partitions being reassigned
+   *   before invoking `maybeTriggerPartitionReassignment` (see method documentation for the reason)
+   *
+   * @param reassignmentsOpt - optional map of reassignments, expected when an API reassignment is issued
+   *                           The map consists of topic partitions to an optional sequence of target replicas.
+   *                           An empty target replicas option denotes a revert of an on-going reassignment.
+   * @param callback - optional callback, expected when an API reassignment is issued
+   */
+  private def processPartitionReassignment(reassignmentsOpt: Option[Map[TopicPartition, Option[Seq[Int]]]],
+                                           callback: Option[AlterReassignmentsCallback]): Unit = {
+    if (!isActive) {
+      callback match {
+        case Some(cb) => cb(Right(new ApiError(Errors.NOT_CONTROLLER)))
+        case None =>
+      }
+      return
+    }
+
+    val reassignmentResults: mutable.Map[TopicPartition, ApiError] = mutable.Map.empty
+    val partitionsToBeReassigned = reassignmentsOpt match {
+      case Some(reassignments) => // API triggered
+        val (savedReassignments, _) = reassignments.partition { case (tp, targetReplicas) =>
+          if (replicasAreValid(targetReplicas)) {
+            savePartitionReassignment(reassignmentResults, tp, targetReplicas, zkTriggered = false)
+          } else {
+            reassignmentResults.put(tp, new ApiError(Errors.INVALID_REPLICA_ASSIGNMENT, "The partition assignment is not valid."))
+            false
+          }
+        }
+        savedReassignments.keySet
+
+      case None => // ZK triggered
+        // We need to register the watcher if the path doesn't exist in order to detect future reassignments and we get
+        // the `path exists` check for free
+        if (zkClient.registerZNodeChangeHandlerAndCheckExistence(partitionReassignmentHandler)) {
+          val partitionReassignment = zkClient.getPartitionReassignment
+          val (savedReassignments, _) = partitionReassignment.partition { case (tp, targetReplicas) =>
+            savePartitionReassignment(reassignmentResults, tp, Some(targetReplicas), zkTriggered = true)
+          }
+          savedReassignments.keySet
+        } else {
+          Set.empty[TopicPartition]
+        }
+    }
 
-    // We need to register the watcher if the path doesn't exist in order to detect future reassignments and we get
-    // the `path exists` check for free
-    if (zkClient.registerZNodeChangeHandlerAndCheckExistence(partitionReassignmentHandler)) {
-      val partitionReassignment = zkClient.getPartitionReassignment
+    reassignmentResults ++= maybeTriggerPartitionReassignment(partitionsToBeReassigned)
+    callback match {
+      case Some(cb) => cb(Left(reassignmentResults))
+      case None =>
+    }
+  }
 
-      // Populate `partitionsBeingReassigned` with all partitions being reassigned before invoking
-      // `maybeTriggerPartitionReassignment` (see method documentation for the reason)
-      partitionReassignment.foreach { case (tp, newReplicas) =>
-        val reassignIsrChangeHandler = new PartitionReassignmentIsrChangeHandler(eventManager, tp)
-        controllerContext.partitionsBeingReassigned.put(tp, ReassignedPartitionsContext(newReplicas, reassignIsrChangeHandler))
-      }
+  private def replicasAreValid(replicasOpt: Option[Seq[Int]]): Boolean = {
+    replicasOpt match {
+      case Some(replicas) =>
+        val replicaSet = replicas.toSet
+
+        if (replicas.size != replicaSet.size)
+          false
+        else if (replicas.exists(_ < 0))
+          false
+        else
+          replicaSet.subsetOf(controllerContext.liveBrokerIds)
+      case None => true
+    }
+  }
+
+  private def savePartitionReassignment(reassignmentResults: mutable.Map[TopicPartition, ApiError], partition: TopicPartition,
+                                        targetReplicasOpt: Option[Seq[Int]], zkTriggered: Boolean): Boolean = {
+    val reassignIsrChangeHandler = new PartitionReassignmentIsrChangeHandler(eventManager, partition)
+    val replicaAssignment = controllerContext.partitionFullReplicaAssignment(partition)
+    val reassignmentIsInProgress = controllerContext.partitionsBeingReassigned.contains(partition)
+
+    val newContextOpt = targetReplicasOpt match {
+      case Some(targetReplicas) =>
+        if (reassignmentIsInProgress) {
+          info(s"Overriding old reassignment for partition $partition " +
+            s"(with target replicas ${replicaAssignment.targetReplicas.mkString(",")}) " +
+            s"to new target replicas (${targetReplicas.mkString(",")})")
+          assert(replicaAssignment.isBeingReassigned)
+
+          val oldContext = controllerContext.partitionsBeingReassigned(partition)
+          oldContext.unregisterReassignIsrChangeHandler(zkClient)
 
-      maybeTriggerPartitionReassignment(partitionReassignment.keySet)
+          Some(ReassignedPartitionsContext(targetReplicas, reassignIsrChangeHandler,
+            persistedInZk = zkTriggered || oldContext.persistedInZk,
+            ongoingReassignmentOpt = Some(replicaAssignment))
+          )
+        } else {
+          Some(ReassignedPartitionsContext(targetReplicas, reassignIsrChangeHandler,
+            persistedInZk = zkTriggered,
+            ongoingReassignmentOpt = None)
+          )
+        }
+      case None => // revert
+        if (reassignmentIsInProgress) {
+          val originalAssignment = replicaAssignment.previousAssignment
+          info(s"Reverting old reassignment for partition $partition " +
+            s"(with target replicas ${replicaAssignment.targetReplicas.mkString(",")}) " +
+            s"to original replicas (${originalAssignment.replicas.mkString(",")})")
+          assert(replicaAssignment.isBeingReassigned)
+
+          val oldContext = controllerContext.partitionsBeingReassigned(partition)
+          oldContext.unregisterReassignIsrChangeHandler(zkClient)
+
+          Some(ReassignedPartitionsContext(originalAssignment.replicas, reassignIsrChangeHandler,
+            persistedInZk = oldContext.persistedInZk,
+            ongoingReassignmentOpt = Some(replicaAssignment)
+          ))
+        } else {
+          reassignmentResults.put(partition, new ApiError(Errors.NO_REASSIGNMENT_IN_PROGRESS))
+          None
+        }
+    }
+
+    newContextOpt match {
+      case Some(newContext) =>
+        controllerContext.partitionsBeingReassigned.put(partition, newContext)
+        true
+      case None => false
     }
   }
 
+
   private def processPartitionReassignmentIsrChange(partition: TopicPartition): Unit = {
     if (!isActive) return
     // check if this partition is still being reassigned or not
@@ -1524,6 +1801,30 @@ class KafkaController(val config: KafkaConfig,
     }
   }
 
+  private def processListPartitionReassignments(partitionsOpt: Option[Set[TopicPartition]], callback: ListReassignmentsCallback): Unit = {
+    if (!isActive) {
+      callback(Right(new ApiError(Errors.NOT_CONTROLLER)))
+    } else {
+      val results: mutable.Map[TopicPartition, PartitionReplicaAssignment] = mutable.Map.empty
+      val partitionsToList = partitionsOpt match {
+        case Some(partitions) => partitions
+        case None => controllerContext.partitionsBeingReassigned.keys
+      }
+
+      partitionsToList.foreach { tp =>
+        val assignment = controllerContext.partitionFullReplicaAssignment(tp)
+        if (assignment.replicas.isEmpty) {
+          callback(Right(new ApiError(Errors.UNKNOWN_TOPIC_OR_PARTITION)))
+          return
+        } else if (assignment.isBeingReassigned) {
+          results += tp -> assignment
+        }
+      }
+
+      callback(Left(results))
+    }
+  }
+
   private def processIsrChangeNotification(): Unit = {
     def processUpdateNotifications(partitions: Seq[TopicPartition]): Unit = {
       val liveBrokers: Seq[Int] = controllerContext.liveOrShuttingDownBrokerIds.toSeq
@@ -1553,6 +1854,16 @@ class KafkaController(val config: KafkaConfig,
     eventManager.put(ReplicaLeaderElection(Some(partitions), electionType, AdminClientTriggered, callback))
   }
 
+  def listPartitionReassignments(partitions: Option[Set[TopicPartition]],
+                                 callback: ListReassignmentsCallback): Unit = {
+    eventManager.put(ListPartitionReassignments(partitions, callback))
+  }
+
+  def alterPartitionReassignments(partitions: Map[TopicPartition, Option[Seq[Int]]],
+                                  callback: AlterReassignmentsCallback): Unit = {
+    eventManager.put(PartitionReassignment(Some(partitions), Some(callback)))
+  }
+
   private def preemptReplicaLeaderElection(
     partitionsFromAdminClientOpt: Option[Set[TopicPartition]],
     callback: ElectLeadersCallback
@@ -1700,8 +2011,10 @@ class KafkaController(val config: KafkaConfig,
           processPartitionModifications(topic)
         case TopicDeletion =>
           processTopicDeletion()
-        case PartitionReassignment =>
-          processPartitionReassignment()
+        case PartitionReassignment(reassignments, callback) =>
+          processPartitionReassignment(reassignments, callback)
+        case ListPartitionReassignments(partitions, callback) =>
+          processListPartitionReassignments(partitions, callback)
         case PartitionReassignmentIsrChange(partition) =>
           processPartitionReassignmentIsrChange(partition)
         case IsrChangeNotification =>
@@ -1781,7 +2094,7 @@ class PartitionReassignmentHandler(eventManager: ControllerEventManager) extends
   // Note that the event is also enqueued when the znode is deleted, but we do it explicitly instead of relying on
   // handleDeletion(). This approach is more robust as it doesn't depend on the watcher being re-registered after
   // it's consumed during data changes (we ensure re-registration when the znode is deleted).
-  override def handleCreation(): Unit = eventManager.put(PartitionReassignment)
+  override def handleCreation(): Unit = eventManager.put(PartitionReassignment(None, None))
 }
 
 class PartitionReassignmentIsrChangeHandler(eventManager: ControllerEventManager, partition: TopicPartition) extends ZNodeChangeHandler {
@@ -1814,8 +2127,17 @@ class ControllerChangeHandler(eventManager: ControllerEventManager) extends ZNod
   override def handleDataChange(): Unit = eventManager.put(ControllerChange)
 }
 
+/**
+  * @param newReplicas - the target replicas for this partition
+  * @param reassignIsrChangeHandler - a handler for tracking ISR changes in this partition
+  * @param persistedInZk - a boolean indicating whether this partition is stored in the /admin/reassign_partitions znode
+  *                        it is needed to ensure that an API reassignment that overrides a znode reassignment still cleans up after itself
+  * @param ongoingReassignmentOpt - the ongoing reassignment for this partition, if one is present -- it will be reverted.
+  */
 case class ReassignedPartitionsContext(var newReplicas: Seq[Int] = Seq.empty,
-                                       reassignIsrChangeHandler: PartitionReassignmentIsrChangeHandler) {
+                                       reassignIsrChangeHandler: PartitionReassignmentIsrChangeHandler,
+                                       persistedInZk: Boolean,
+                                       var ongoingReassignmentOpt: Option[PartitionReplicaAssignment]) {
 
   def registerReassignIsrChangeHandler(zkClient: KafkaZkClient): Unit =
     zkClient.registerZNodeChangeHandler(reassignIsrChangeHandler)
@@ -1934,12 +2256,13 @@ case object TopicDeletion extends ControllerEvent {
   override def state: ControllerState = ControllerState.TopicDeletion
 }
 
-case object PartitionReassignment extends ControllerEvent {
-  override def state: ControllerState = ControllerState.PartitionReassignment
+case class PartitionReassignment(reassignments: Option[Map[TopicPartition, Option[Seq[Int]]]],
+                                 callback: Option[AlterReassignmentsCallback]) extends ControllerEvent {
+  override def state: ControllerState = ControllerState.AlterPartitionReassignment
 }
 
 case class PartitionReassignmentIsrChange(partition: TopicPartition) extends ControllerEvent {
-  override def state: ControllerState = ControllerState.PartitionReassignment
+  override def state: ControllerState = ControllerState.AlterPartitionReassignment
 }
 
 case object IsrChangeNotification extends ControllerEvent {
@@ -1955,6 +2278,15 @@ case class ReplicaLeaderElection(
   override def state: ControllerState = ControllerState.ManualLeaderBalance
 }
 
+/**
+  * @param partitionsOpt - an Optional set of partitions. If not present, all reassigning partitions are to be listed
+  */
+case class ListPartitionReassignments(partitionsOpt: Option[Set[TopicPartition]],
+                                      callback: ListReassignmentsCallback) extends ControllerEvent {
+  override def state: ControllerState = ControllerState.ListPartitionReassignment
+}
+
+
 // Used only in test cases
 abstract class MockEvent(val state: ControllerState) extends ControllerEvent {
   def process(): Unit
diff --git a/core/src/main/scala/kafka/controller/PartitionStateMachine.scala b/core/src/main/scala/kafka/controller/PartitionStateMachine.scala
index 94e8b7e0aa35e..c2dc835fdc0e5 100755
--- a/core/src/main/scala/kafka/controller/PartitionStateMachine.scala
+++ b/core/src/main/scala/kafka/controller/PartitionStateMachine.scala
@@ -308,7 +308,7 @@ class ZkPartitionStateMachine(config: KafkaConfig,
       if (code == Code.OK) {
         controllerContext.partitionLeadershipInfo.put(partition, leaderIsrAndControllerEpoch)
         controllerBrokerRequestBatch.addLeaderAndIsrRequestForBrokers(leaderIsrAndControllerEpoch.leaderAndIsr.isr,
-          partition, leaderIsrAndControllerEpoch, controllerContext.partitionReplicaAssignment(partition), isNew = true)
+          partition, leaderIsrAndControllerEpoch, controllerContext.partitionFullReplicaAssignment(partition), isNew = true)
         successfulInitializations += partition
       } else {
         logFailedStateChange(partition, NewPartition, OnlinePartition, code)
@@ -342,6 +342,9 @@ class ZkPartitionStateMachine(config: KafkaConfig,
       }
 
       finishedElections ++= finished
+
+      if (remaining.nonEmpty)
+        logger.info(s"Retrying leader election with strategy $partitionLeaderElectionStrategy for partitions $remaining")
     }
 
     finishedElections.toMap
@@ -429,11 +432,11 @@ class ZkPartitionStateMachine(config: KafkaConfig,
       adjustedLeaderAndIsrs, controllerContext.epoch, controllerContext.epochZkVersion)
     finishedUpdates.foreach { case (partition, result) =>
       result.right.foreach { leaderAndIsr =>
-        val replicas = controllerContext.partitionReplicaAssignment(partition)
+        val replicaAssignment = controllerContext.partitionFullReplicaAssignment(partition)
         val leaderIsrAndControllerEpoch = LeaderIsrAndControllerEpoch(leaderAndIsr, controllerContext.epoch)
         controllerContext.partitionLeadershipInfo.put(partition, leaderIsrAndControllerEpoch)
         controllerBrokerRequestBatch.addLeaderAndIsrRequestForBrokers(recipientsPerPartition(partition), partition,
-          leaderIsrAndControllerEpoch, replicas, isNew = false)
+          leaderIsrAndControllerEpoch, replicaAssignment, isNew = false)
       }
     }
 
diff --git a/core/src/main/scala/kafka/controller/ReplicaStateMachine.scala b/core/src/main/scala/kafka/controller/ReplicaStateMachine.scala
index 610315498194d..5fada1b326b99 100644
--- a/core/src/main/scala/kafka/controller/ReplicaStateMachine.scala
+++ b/core/src/main/scala/kafka/controller/ReplicaStateMachine.scala
@@ -175,7 +175,7 @@ class ZkReplicaStateMachine(config: KafkaConfig,
                 controllerBrokerRequestBatch.addLeaderAndIsrRequestForBrokers(Seq(replicaId),
                   replica.topicPartition,
                   leaderIsrAndControllerEpoch,
-                  controllerContext.partitionReplicaAssignment(replica.topicPartition),
+                  controllerContext.partitionFullReplicaAssignment(replica.topicPartition),
                   isNew = true)
                 logSuccessfulTransition(replicaId, partition, currentState, NewReplica)
                 controllerContext.putReplicaState(replica, NewReplica)
@@ -202,7 +202,7 @@ class ZkReplicaStateMachine(config: KafkaConfig,
                   controllerBrokerRequestBatch.addLeaderAndIsrRequestForBrokers(Seq(replicaId),
                     replica.topicPartition,
                     leaderIsrAndControllerEpoch,
-                    controllerContext.partitionReplicaAssignment(partition), isNew = false)
+                    controllerContext.partitionFullReplicaAssignment(partition), isNew = false)
                 case None =>
               }
           }
@@ -223,7 +223,7 @@ class ZkReplicaStateMachine(config: KafkaConfig,
             controllerBrokerRequestBatch.addLeaderAndIsrRequestForBrokers(recipients,
               partition,
               leaderIsrAndControllerEpoch,
-              controllerContext.partitionReplicaAssignment(partition), isNew = false)
+              controllerContext.partitionFullReplicaAssignment(partition), isNew = false)
           }
           val replica = PartitionAndReplica(partition, replicaId)
           val currentState = controllerContext.replicaState(replica)
diff --git a/core/src/main/scala/kafka/server/AdminManager.scala b/core/src/main/scala/kafka/server/AdminManager.scala
index 3ed365e43c7b3..7b71e35a88e0f 100644
--- a/core/src/main/scala/kafka/server/AdminManager.scala
+++ b/core/src/main/scala/kafka/server/AdminManager.scala
@@ -245,8 +245,8 @@ class AdminManager(val config: KafkaConfig,
         if (reassignPartitionsInProgress)
           throw new ReassignmentInProgressException("A partition reassignment is in progress.")
 
-        val existingAssignment = zkClient.getReplicaAssignmentForTopics(immutable.Set(topic)).map {
-          case (topicPartition, replicas) => topicPartition.partition -> replicas
+        val existingAssignment = zkClient.getFullReplicaAssignmentForTopics(immutable.Set(topic)).map {
+          case (topicPartition, assignment) => topicPartition.partition -> assignment
         }
         if (existingAssignment.isEmpty)
           throw new UnknownTopicOrPartitionException(s"The topic '$topic' does not exist.")
@@ -261,7 +261,7 @@ class AdminManager(val config: KafkaConfig,
           throw new InvalidPartitionsException(s"Topic already has $oldNumPartitions partitions.")
         }
 
-        val reassignment = Option(newPartition.newAssignments).map(_.asScala.map(_.asScala.map(_.toInt))).map { assignments =>
+        val newPartitionsAssignment = Option(newPartition.newAssignments).map(_.asScala.map(_.asScala.map(_.toInt))).map { assignments =>
           val unknownBrokers = assignments.flatten.toSet -- allBrokerIds
           if (unknownBrokers.nonEmpty)
             throw new InvalidReplicaAssignmentException(
@@ -278,7 +278,7 @@ class AdminManager(val config: KafkaConfig,
         }
 
         val updatedReplicaAssignment = adminZkClient.addPartitions(topic, existingAssignment, allBrokers,
-          newPartition.totalCount, reassignment, validateOnly = validateOnly)
+          newPartition.totalCount, newPartitionsAssignment, validateOnly = validateOnly)
         CreatePartitionsMetadata(topic, updatedReplicaAssignment, ApiError.NONE)
       } catch {
         case e: AdminOperationException =>
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 82b0616f1407c..810711de64339 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -30,7 +30,7 @@ import kafka.api.ElectLeadersRequestOps
 import kafka.api.{ApiVersion, KAFKA_0_11_0_IV0, KAFKA_2_3_IV0}
 import kafka.cluster.Partition
 import kafka.common.OffsetAndMetadata
-import kafka.controller.KafkaController
+import kafka.controller.{KafkaController, PartitionReplicaAssignment}
 import kafka.coordinator.group.{GroupCoordinator, JoinGroupResult, LeaveGroupResult, SyncGroupResult}
 import kafka.coordinator.transaction.{InitProducerIdResult, TransactionCoordinator}
 import kafka.message.ZStdCompressionCodec
@@ -53,6 +53,7 @@ import org.apache.kafka.common.message.CreateTopicsResponseData.{CreatableTopicR
 import org.apache.kafka.common.message.DeleteGroupsResponseData
 import org.apache.kafka.common.message.DeleteGroupsResponseData.{DeletableGroupResult, DeletableGroupResultCollection}
 import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData
+import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.{ReassignablePartitionResponse, ReassignableTopicResponse}
 import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData
 import org.apache.kafka.common.message.DeleteTopicsResponseData
 import org.apache.kafka.common.message.DeleteTopicsResponseData.{DeletableTopicResult, DeletableTopicResultCollection}
@@ -2285,26 +2286,85 @@ class KafkaApis(val requestChannel: RequestChannel,
     authorizeClusterOperation(request, ALTER)
     val alterPartitionReassignmentsRequest = request.body[AlterPartitionReassignmentsRequest]
 
-    sendResponseMaybeThrottle(request, requestThrottleMs =>
-      new AlterPartitionReassignmentsResponse(
-        new AlterPartitionReassignmentsResponseData().setThrottleTimeMs(requestThrottleMs)
-          .setErrorCode(Errors.UNSUPPORTED_VERSION.code()).setErrorMessage(Errors.UNSUPPORTED_VERSION.message())
-          .toStruct(0)
+    def sendResponseCallback(result: Either[Map[TopicPartition, ApiError], ApiError]): Unit = {
+      val responseData = result match {
+        case Right(topLevelError) =>
+          new AlterPartitionReassignmentsResponseData().setErrorMessage(topLevelError.message()).setErrorCode(topLevelError.error().code())
+
+        case Left(assignments) =>
+          val topicResponses = assignments.groupBy(_._1.topic()).map {
+            case (topic, reassignmentsByTp) =>
+              val partitionResponses = reassignmentsByTp.map {
+                case (topicPartition, error) =>
+                  new ReassignablePartitionResponse().setPartitionIndex(topicPartition.partition())
+                    .setErrorCode(error.error().code()).setErrorMessage(error.message())
+              }
+              new ReassignableTopicResponse().setName(topic).setPartitions(partitionResponses.toList.asJava)
+          }
+          new AlterPartitionReassignmentsResponseData().setResponses(topicResponses.toList.asJava)
+      }
+
+      sendResponseMaybeThrottle(request, requestThrottleMs =>
+        new AlterPartitionReassignmentsResponse(responseData.setThrottleTimeMs(requestThrottleMs))
       )
-    )
+    }
+
+    val reassignments = alterPartitionReassignmentsRequest.data().topics().asScala.flatMap {
+      reassignableTopic => reassignableTopic.partitions().asScala.map {
+        reassignablePartition =>
+          val tp = new TopicPartition(reassignableTopic.name(), reassignablePartition.partitionIndex())
+          if (reassignablePartition.replicas() == null)
+            tp -> None // revert call
+          else
+            tp -> Some(reassignablePartition.replicas().asScala.map(_.toInt))
+      }
+    }.toMap
+
+    controller.alterPartitionReassignments(reassignments, sendResponseCallback)
   }
 
   def handleListPartitionReassignmentsRequest(request: RequestChannel.Request): Unit = {
     authorizeClusterOperation(request, DESCRIBE)
     val listPartitionReassignmentsRequest = request.body[ListPartitionReassignmentsRequest]
 
-    sendResponseMaybeThrottle(request, requestThrottleMs =>
-      new ListPartitionReassignmentsResponse(
-        new ListPartitionReassignmentsResponseData().setThrottleTimeMs(requestThrottleMs)
-          .setErrorCode(Errors.UNSUPPORTED_VERSION.code()).setErrorMessage(Errors.UNSUPPORTED_VERSION.message())
-          .toStruct(0)
+    def sendResponseCallback(result: Either[Map[TopicPartition, PartitionReplicaAssignment], ApiError]): Unit = {
+      val responseData = result match {
+        case Right(error) => new ListPartitionReassignmentsResponseData().setErrorMessage(error.message()).setErrorCode(error.error().code())
+
+        case Left(assignments) =>
+          val topicReassignments = assignments.groupBy(_._1.topic()).map {
+            case (topic, reassignmentsByTp) =>
+              val partitionReassignments = reassignmentsByTp.map {
+                case (topicPartition, assignment) =>
+                  new ListPartitionReassignmentsResponseData.OngoingPartitionReassignment()
+                    .setPartitionIndex(topicPartition.partition())
+                    .setAddingReplicas(assignment.addingReplicas.toList.asJava.asInstanceOf[java.util.List[java.lang.Integer]])
+                    .setRemovingReplicas(assignment.removingReplicas.toList.asJava.asInstanceOf[java.util.List[java.lang.Integer]])
+                    .setReplicas(assignment.replicas.toList.asJava.asInstanceOf[java.util.List[java.lang.Integer]])
+              }.toList
+
+              new ListPartitionReassignmentsResponseData.OngoingTopicReassignment().setName(topic)
+                .setPartitions(partitionReassignments.asJava)
+          }.toList
+
+          new ListPartitionReassignmentsResponseData().setTopics(topicReassignments.asJava)
+      }
+
+      sendResponseMaybeThrottle(request, requestThrottleMs =>
+        new ListPartitionReassignmentsResponse(responseData.setThrottleTimeMs(requestThrottleMs))
       )
-    )
+    }
+
+    val partitionsOpt = listPartitionReassignmentsRequest.data().topics() match {
+      case topics: Any =>
+        Some(topics.iterator().asScala.flatMap { topic =>
+          topic.partitionIndexes().iterator().asScala
+            .map { tp => new TopicPartition(topic.name(), tp) }
+        }.toSet)
+      case _ => None
+    }
+
+    controller.listPartitionReassignments(partitionsOpt, sendResponseCallback)
   }
 
   private def configsAuthorizationApiError(resource: ConfigResource): ApiError = {
diff --git a/core/src/main/scala/kafka/zk/AdminZkClient.scala b/core/src/main/scala/kafka/zk/AdminZkClient.scala
index f0e57991c78a9..de6337b8d139f 100644
--- a/core/src/main/scala/kafka/zk/AdminZkClient.scala
+++ b/core/src/main/scala/kafka/zk/AdminZkClient.scala
@@ -20,6 +20,7 @@ import java.util.Properties
 
 import kafka.admin.{AdminOperationException, AdminUtils, BrokerMetadata, RackAwareMode}
 import kafka.common.TopicAlreadyMarkedForDeletionException
+import kafka.controller.PartitionReplicaAssignment
 import kafka.log.LogConfig
 import kafka.server.{ConfigEntityName, ConfigType, DynamicConfig}
 import kafka.utils._
@@ -92,7 +93,8 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
     zkClient.setOrCreateEntityConfigs(ConfigType.Topic, topic, config)
 
     // create the partition assignment
-    writeTopicPartitionAssignment(topic, partitionReplicaAssignment, isUpdate = false)
+    writeTopicPartitionAssignment(topic, partitionReplicaAssignment.mapValues(PartitionReplicaAssignment(_, List(), List())).toMap,
+      isUpdate = false)
   }
 
   /**
@@ -134,12 +136,12 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
     LogConfig.validate(config)
   }
 
-  private def writeTopicPartitionAssignment(topic: String, replicaAssignment: Map[Int, Seq[Int]], isUpdate: Boolean): Unit = {
+  private def writeTopicPartitionAssignment(topic: String, replicaAssignment: Map[Int, PartitionReplicaAssignment], isUpdate: Boolean): Unit = {
     try {
       val assignment = replicaAssignment.map { case (partitionId, replicas) => (new TopicPartition(topic,partitionId), replicas) }.toMap
 
       if (!isUpdate) {
-        zkClient.createTopicAssignment(topic, assignment)
+        zkClient.createTopicAssignment(topic, assignment.mapValues(_.replicas).toMap)
       } else {
         zkClient.setTopicAssignment(topic, assignment)
       }
@@ -172,7 +174,7 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
   * Add partitions to existing topic with optional replica assignment
   *
   * @param topic Topic for adding partitions to
-  * @param existingAssignment A map from partition id to its assigned replicas
+  * @param existingAssignment A map from partition id to its assignment
   * @param allBrokers All brokers in the cluster
   * @param numPartitions Number of partitions to be set
   * @param replicaAssignment Manual replica assignment, or none
@@ -180,7 +182,7 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
   * @return the updated replica assignment
   */
   def addPartitions(topic: String,
-                    existingAssignment: Map[Int, Seq[Int]],
+                    existingAssignment: Map[Int, PartitionReplicaAssignment],
                     allBrokers: Seq[BrokerMetadata],
                     numPartitions: Int = 1,
                     replicaAssignment: Option[Map[Int, Seq[Int]]] = None,
@@ -188,7 +190,7 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
     val existingAssignmentPartition0 = existingAssignment.getOrElse(0,
       throw new AdminOperationException(
         s"Unexpected existing replica assignment for topic '$topic', partition id 0 is missing. " +
-          s"Assignment: $existingAssignment"))
+          s"Assignment: $existingAssignment")).replicas
 
     val partitionsToAdd = numPartitions - existingAssignment.size
     if (partitionsToAdd <= 0)
@@ -208,14 +210,16 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
         startIndex, existingAssignment.size)
     }
 
-    val proposedAssignment = existingAssignment ++ proposedAssignmentForNewPartitions
+    val proposedAssignment = existingAssignment ++ proposedAssignmentForNewPartitions.map { case (tp, replicas) =>
+      tp -> PartitionReplicaAssignment(replicas, List(), List())
+    }
     if (!validateOnly) {
       info(s"Creating $partitionsToAdd partitions for '$topic' with the following replica assignment: " +
         s"$proposedAssignmentForNewPartitions.")
 
       writeTopicPartitionAssignment(topic, proposedAssignment, isUpdate = true)
     }
-    proposedAssignment
+    proposedAssignment.mapValues(_.replicas).toMap
   }
 
   private def validateReplicaAssignment(replicaAssignment: Map[Int, Seq[Int]],
diff --git a/core/src/main/scala/kafka/zk/KafkaZkClient.scala b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
index 21ebe607b09b4..73f83fe71f0b6 100644
--- a/core/src/main/scala/kafka/zk/KafkaZkClient.scala
+++ b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
@@ -21,7 +21,7 @@ import java.util.Properties
 import com.yammer.metrics.core.MetricName
 import kafka.api.LeaderAndIsr
 import kafka.cluster.Broker
-import kafka.controller.{KafkaController, LeaderIsrAndControllerEpoch}
+import kafka.controller.{KafkaController, LeaderIsrAndControllerEpoch, PartitionReplicaAssignment}
 import kafka.log.LogConfig
 import kafka.metrics.KafkaMetricsGroup
 import kafka.security.authorizer.AclAuthorizer.{NoAcls, VersionedAcls}
@@ -484,7 +484,7 @@ class KafkaZkClient private[zk] (zooKeeperClient: ZooKeeperClient, isSecure: Boo
    * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.
    * @return SetDataResponse
    */
-  def setTopicAssignmentRaw(topic: String, assignment: collection.Map[TopicPartition, Seq[Int]], expectedControllerEpochZkVersion: Int): SetDataResponse = {
+  def setTopicAssignmentRaw(topic: String, assignment: collection.Map[TopicPartition, PartitionReplicaAssignment], expectedControllerEpochZkVersion: Int): SetDataResponse = {
     val setDataRequest = SetDataRequest(TopicZNode.path(topic), TopicZNode.encode(assignment), ZkVersion.MatchAnyVersion)
     retryRequestUntilConnected(setDataRequest, expectedControllerEpochZkVersion)
   }
@@ -496,7 +496,7 @@ class KafkaZkClient private[zk] (zooKeeperClient: ZooKeeperClient, isSecure: Boo
    * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.
    * @throws KeeperException if there is an error while setting assignment
    */
-  def setTopicAssignment(topic: String, assignment: Map[TopicPartition, Seq[Int]], expectedControllerEpochZkVersion: Int = ZkVersion.MatchAnyVersion) = {
+  def setTopicAssignment(topic: String, assignment: Map[TopicPartition, PartitionReplicaAssignment], expectedControllerEpochZkVersion: Int = ZkVersion.MatchAnyVersion) = {
     val setDataResponse = setTopicAssignmentRaw(topic, assignment, expectedControllerEpochZkVersion)
     setDataResponse.maybeThrow
   }
@@ -508,7 +508,8 @@ class KafkaZkClient private[zk] (zooKeeperClient: ZooKeeperClient, isSecure: Boo
    * @throws KeeperException if there is an error while creating assignment
    */
   def createTopicAssignment(topic: String, assignment: Map[TopicPartition, Seq[Int]]) = {
-    createRecursive(TopicZNode.path(topic), TopicZNode.encode(assignment))
+    val persistedAssignments = assignment.mapValues(PartitionReplicaAssignment(_, List(), List())).toMap
+    createRecursive(TopicZNode.path(topic), TopicZNode.encode(persistedAssignments))
   }
 
   /**
@@ -569,18 +570,28 @@ class KafkaZkClient private[zk] (zooKeeperClient: ZooKeeperClient, isSecure: Boo
   }
 
   /**
-   * Gets the assignments for the given topics.
+   * Gets the replica assignments for the given topics.
+   * This function does not return information about which replicas are being added or removed from the assignment.
    * @param topics the topics whose partitions we wish to get the assignments for.
    * @return the replica assignment for each partition from the given topics.
    */
   def getReplicaAssignmentForTopics(topics: Set[String]): Map[TopicPartition, Seq[Int]] = {
+    getFullReplicaAssignmentForTopics(topics).mapValues(_.replicas).toMap
+  }
+
+  /**
+    * Gets the replica assignments for the given topics.
+    * @param topics the topics whose partitions we wish to get the assignments for.
+    * @return the full replica assignment for each partition from the given topics.
+    */
+  def getFullReplicaAssignmentForTopics(topics: Set[String]): Map[TopicPartition, PartitionReplicaAssignment] = {
     val getDataRequests = topics.map(topic => GetDataRequest(TopicZNode.path(topic), ctx = Some(topic)))
     val getDataResponses = retryRequestsUntilConnected(getDataRequests.toSeq)
     getDataResponses.flatMap { getDataResponse =>
       val topic = getDataResponse.ctx.get.asInstanceOf[String]
       getDataResponse.resultCode match {
         case Code.OK => TopicZNode.decode(topic, getDataResponse.data)
-        case Code.NONODE => Map.empty[TopicPartition, Seq[Int]]
+        case Code.NONODE => Map.empty[TopicPartition, PartitionReplicaAssignment]
         case _ => throw getDataResponse.resultException.get
       }
     }.toMap
@@ -591,7 +602,7 @@ class KafkaZkClient private[zk] (zooKeeperClient: ZooKeeperClient, isSecure: Boo
    * @param topics the topics whose partitions we wish to get the assignments for.
    * @return the partition assignment for each partition from the given topics.
    */
-  def getPartitionAssignmentForTopics(topics: Set[String]): Map[String, Map[Int, Seq[Int]]] = {
+  def getPartitionAssignmentForTopics(topics: Set[String]): Map[String, Map[Int, PartitionReplicaAssignment]] = {
     val getDataRequests = topics.map(topic => GetDataRequest(TopicZNode.path(topic), ctx = Some(topic)))
     val getDataResponses = retryRequestsUntilConnected(getDataRequests.toSeq)
     getDataResponses.flatMap { getDataResponse =>
@@ -600,7 +611,7 @@ class KafkaZkClient private[zk] (zooKeeperClient: ZooKeeperClient, isSecure: Boo
         val partitionMap = TopicZNode.decode(topic, getDataResponse.data).map { case (k, v) => (k.partition, v) }
         Map(topic -> partitionMap)
       } else if (getDataResponse.resultCode == Code.NONODE) {
-        Map.empty[String, Map[Int, Seq[Int]]]
+        Map.empty[String, Map[Int, PartitionReplicaAssignment]]
       } else {
         throw getDataResponse.resultException.get
       }
@@ -791,7 +802,9 @@ class KafkaZkClient private[zk] (zooKeeperClient: ZooKeeperClient, isSecure: Boo
   /**
    * Returns all reassignments.
    * @return the reassignments for each partition.
+   * @deprecated Use the PartitionReassignment Kafka API instead
    */
+  @Deprecated
   def getPartitionReassignment: collection.Map[TopicPartition, Seq[Int]] = {
     val getDataRequest = GetDataRequest(ReassignPartitionsZNode.path)
     val getDataResponse = retryRequestUntilConnected(getDataRequest)
@@ -815,7 +828,9 @@ class KafkaZkClient private[zk] (zooKeeperClient: ZooKeeperClient, isSecure: Boo
    * @param reassignment the reassignment to set on the reassignment znode
    * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.
    * @throws KeeperException if there is an error while setting or creating the znode
+   * @deprecated Use the PartitionReassignment Kafka API instead
    */
+  @Deprecated
   def setOrCreatePartitionReassignment(reassignment: collection.Map[TopicPartition, Seq[Int]], expectedControllerEpochZkVersion: Int): Unit = {
 
     def set(reassignmentData: Array[Byte]): SetDataResponse = {
@@ -843,7 +858,9 @@ class KafkaZkClient private[zk] (zooKeeperClient: ZooKeeperClient, isSecure: Boo
    * Creates the partition reassignment znode with the given reassignment.
    * @param reassignment the reassignment to set on the reassignment znode.
    * @throws KeeperException if there is an error while creating the znode
+   * @deprecated Use the PartitionReassignment Kafka API instead
    */
+  @Deprecated
   def createPartitionReassignment(reassignment: Map[TopicPartition, Seq[Int]])  = {
     createRecursive(ReassignPartitionsZNode.path, ReassignPartitionsZNode.encode(reassignment))
   }
@@ -851,7 +868,9 @@ class KafkaZkClient private[zk] (zooKeeperClient: ZooKeeperClient, isSecure: Boo
   /**
    * Deletes the partition reassignment znode.
    * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.
+   * @deprecated Use the PartitionReassignment Kafka API instead
    */
+  @Deprecated
   def deletePartitionReassignment(expectedControllerEpochZkVersion: Int): Unit = {
     deletePath(ReassignPartitionsZNode.path, expectedControllerEpochZkVersion)
   }
@@ -859,7 +878,9 @@ class KafkaZkClient private[zk] (zooKeeperClient: ZooKeeperClient, isSecure: Boo
   /**
    * Checks if reassign partitions is in progress
    * @return true if reassign partitions is in progress, else false
+   * @deprecated Use the PartitionReassignment Kafka API instead
    */
+  @Deprecated
   def reassignPartitionsInProgress(): Boolean = {
     pathExists(ReassignPartitionsZNode.path)
   }
diff --git a/core/src/main/scala/kafka/zk/ZkData.scala b/core/src/main/scala/kafka/zk/ZkData.scala
index cefa94589de89..43769ad7deb61 100644
--- a/core/src/main/scala/kafka/zk/ZkData.scala
+++ b/core/src/main/scala/kafka/zk/ZkData.scala
@@ -17,6 +17,7 @@
 package kafka.zk
 
 import java.nio.charset.StandardCharsets.UTF_8
+import java.util
 import java.util.Properties
 
 import com.fasterxml.jackson.annotation.JsonProperty
@@ -24,12 +25,13 @@ import com.fasterxml.jackson.core.JsonProcessingException
 import kafka.api.{ApiVersion, KAFKA_0_10_0_IV1, LeaderAndIsr}
 import kafka.cluster.{Broker, EndPoint}
 import kafka.common.{NotificationHandler, ZkNodeChangeNotificationListener}
-import kafka.controller.{IsrChangeNotificationHandler, LeaderIsrAndControllerEpoch}
+import kafka.controller.{IsrChangeNotificationHandler, LeaderIsrAndControllerEpoch, PartitionReplicaAssignment}
 import kafka.security.auth.Resource.Separator
 import kafka.security.authorizer.AclAuthorizer.VersionedAcls
 import kafka.security.auth.{Acl, Resource, ResourceType}
 import kafka.server.{ConfigType, DelegationTokenManager}
 import kafka.utils.Json
+import kafka.utils.json.JsonObject
 import org.apache.kafka.common.{KafkaException, TopicPartition}
 import org.apache.kafka.common.errors.UnsupportedVersionException
 import org.apache.kafka.common.network.ListenerName
@@ -43,7 +45,7 @@ import org.apache.zookeeper.data.{ACL, Stat}
 import scala.beans.BeanProperty
 import scala.collection.JavaConverters._
 import scala.collection.mutable.ArrayBuffer
-import scala.collection.{Map, Seq}
+import scala.collection.{Map, Seq, mutable}
 import scala.util.{Failure, Success, Try}
 
 // This file contains objects for encoding/decoding data stored in ZooKeeper nodes (znodes).
@@ -238,19 +240,49 @@ object TopicsZNode {
 
 object TopicZNode {
   def path(topic: String) = s"${TopicsZNode.path}/$topic"
-  def encode(assignment: collection.Map[TopicPartition, Seq[Int]]): Array[Byte] = {
-    val assignmentJson = assignment.map { case (partition, replicas) =>
-      partition.partition.toString -> replicas.asJava
+  def encode(assignment: collection.Map[TopicPartition, PartitionReplicaAssignment]): Array[Byte] = {
+    val replicaAssignmentJson = mutable.Map[String, util.List[Int]]()
+    val addingReplicasAssignmentJson = mutable.Map[String, util.List[Int]]()
+    val removingReplicasAssignmentJson = mutable.Map[String, util.List[Int]]()
+
+    for ((partition, replicaAssignment) <- assignment) {
+      replicaAssignmentJson += (partition.partition.toString -> replicaAssignment.replicas.asJava)
+      if (replicaAssignment.addingReplicas.nonEmpty)
+        addingReplicasAssignmentJson += (partition.partition.toString -> replicaAssignment.addingReplicas.asJava)
+      if (replicaAssignment.removingReplicas.nonEmpty)
+        removingReplicasAssignmentJson += (partition.partition.toString -> replicaAssignment.removingReplicas.asJava)
     }
-    Json.encodeAsBytes(Map("version" -> 1, "partitions" -> assignmentJson.asJava).asJava)
+
+    Json.encodeAsBytes(Map(
+      "version" -> 2,
+      "partitions" -> replicaAssignmentJson.asJava,
+      "addingReplicas" -> addingReplicasAssignmentJson.asJava,
+      "removingReplicas" -> removingReplicasAssignmentJson.asJava
+    ).asJava)
   }
-  def decode(topic: String, bytes: Array[Byte]): Map[TopicPartition, Seq[Int]] = {
+  def decode(topic: String, bytes: Array[Byte]): Map[TopicPartition, PartitionReplicaAssignment] = {
+    def getReplicas(replicasJsonOpt: Option[JsonObject], partition: String): Seq[Int] = {
+      replicasJsonOpt match {
+        case Some(replicasJson) => replicasJson.get(partition) match {
+          case Some(ar) => ar.to[Seq[Int]]
+          case None => Seq.empty[Int]
+        }
+        case None => Seq.empty[Int]
+      }
+    }
+
     Json.parseBytes(bytes).flatMap { js =>
       val assignmentJson = js.asJsonObject
       val partitionsJsonOpt = assignmentJson.get("partitions").map(_.asJsonObject)
+      val addingReplicasJsonOpt = assignmentJson.get("addingReplicas").map(_.asJsonObject)
+      val removingReplicasJsonOpt = assignmentJson.get("removingReplicas").map(_.asJsonObject)
       partitionsJsonOpt.map { partitionsJson =>
         partitionsJson.iterator.map { case (partition, replicas) =>
-          new TopicPartition(topic, partition.toInt) -> replicas.to[Seq[Int]]
+          new TopicPartition(topic, partition.toInt) -> PartitionReplicaAssignment(
+            replicas.to[Seq[Int]],
+            getReplicas(addingReplicasJsonOpt, partition),
+            getReplicas(addingReplicasJsonOpt, partition)
+          )
         }
       }
     }.map(_.toMap).getOrElse(Map.empty)
@@ -373,6 +405,10 @@ object DeleteTopicsTopicZNode {
   def path(topic: String) = s"${DeleteTopicsZNode.path}/$topic"
 }
 
+/**
+ * The znode for initiating a partition reassignment.
+ * @deprecated Since 2.4, use the PartitionReassignment Kafka API instead.
+ */
 object ReassignPartitionsZNode {
 
   /**
@@ -388,14 +424,16 @@ object ReassignPartitionsZNode {
   /**
     * An assignment consists of a `version` and a list of `partitions`, which represent the
     * assignment of topic-partitions to brokers.
+    * @deprecated Use the PartitionReassignment Kafka API instead
     */
-  case class PartitionAssignment(@BeanProperty @JsonProperty("version") version: Int,
-                                 @BeanProperty @JsonProperty("partitions") partitions: java.util.List[ReplicaAssignment])
+  @Deprecated
+  case class LegacyPartitionAssignment(@BeanProperty @JsonProperty("version") version: Int,
+                                       @BeanProperty @JsonProperty("partitions") partitions: java.util.List[ReplicaAssignment])
 
   def path = s"${AdminZNode.path}/reassign_partitions"
 
   def encode(reassignmentMap: collection.Map[TopicPartition, Seq[Int]]): Array[Byte] = {
-    val reassignment = PartitionAssignment(1,
+    val reassignment = LegacyPartitionAssignment(1,
       reassignmentMap.toSeq.map { case (tp, replicas) =>
         ReplicaAssignment(tp.topic, tp.partition, replicas.asJava)
       }.asJava
@@ -404,7 +442,7 @@ object ReassignPartitionsZNode {
   }
 
   def decode(bytes: Array[Byte]): Either[JsonProcessingException, collection.Map[TopicPartition, Seq[Int]]] =
-    Json.parseBytesAs[PartitionAssignment](bytes).right.map { partitionAssignment =>
+    Json.parseBytesAs[LegacyPartitionAssignment](bytes).right.map { partitionAssignment =>
       partitionAssignment.partitions.asScala.iterator.map { replicaAssignment =>
         new TopicPartition(replicaAssignment.topic, replicaAssignment.partition) -> replicaAssignment.replicas.asScala
       }.toMap
diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index e2e0ca71a03aa..02b8f1afc7046 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -1699,6 +1699,22 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     assertEquals(Option(broker3), TestUtils.currentLeader(client, partition2))
   }
 
+  @Test
+  def testListReassignmentsDoesNotShowNonReassigningPartitions(): Unit = {
+    client = AdminClient.create(createConfig())
+
+    // Create topics
+    val topic = "list-reassignments-no-reassignments"
+    createTopic(topic, numPartitions = 1, replicationFactor = 3)
+    val tp = new TopicPartition(topic, 0)
+
+    val reassignmentsMap = client.listPartitionReassignments(Set(tp).asJava).reassignments().get()
+    assertEquals(0, reassignmentsMap.size())
+
+    val allReassignmentsMap = client.listPartitionReassignments().reassignments().get()
+    assertEquals(0, allReassignmentsMap.size())
+  }
+
   @Test
   def testValidIncrementalAlterConfigs(): Unit = {
     client = AdminClient.create(createConfig)
@@ -1877,6 +1893,42 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
       Some("Invalid config value for resource"))
   }
 
+  @Test
+  def testInvalidAlterPartitionReassignments(): Unit = {
+    client = AdminClient.create(createConfig)
+    val topic = "alter-reassignments-topic-1"
+    val tp1 = new TopicPartition(topic, 0)
+    val tp2 = new TopicPartition(topic, 1)
+    val tp3 = new TopicPartition(topic, 2)
+    createTopic(topic, numPartitions = 3)
+
+    val validAssignment = new NewPartitionReassignment((0 until brokerCount).map(_.asInstanceOf[Integer]).asJava)
+
+    val nonExistentTp1 = new TopicPartition("topicA", 0)
+    val nonExistentTp2 = new TopicPartition(topic, 3)
+    val nonExistentPartitionsResult = client.alterPartitionReassignments(Map(
+      tp1 -> java.util.Optional.of(validAssignment),
+      tp2 -> java.util.Optional.of(validAssignment),
+      tp3 -> java.util.Optional.of(validAssignment),
+      nonExistentTp1 -> java.util.Optional.of(validAssignment),
+      nonExistentTp2 -> java.util.Optional.of(validAssignment)
+    ).asJava).values()
+    assertFutureExceptionTypeEquals(nonExistentPartitionsResult.get(nonExistentTp1), classOf[UnknownTopicOrPartitionException])
+    assertFutureExceptionTypeEquals(nonExistentPartitionsResult.get(nonExistentTp2), classOf[UnknownTopicOrPartitionException])
+
+    val extraNonExistentReplica = new NewPartitionReassignment((0 until brokerCount + 1).map(_.asInstanceOf[Integer]).asJava)
+    val negativeIdReplica = new NewPartitionReassignment(Seq(-3, -2, -1).map(_.asInstanceOf[Integer]).asJava)
+    val duplicateReplica = new NewPartitionReassignment(Seq(0, 1, 1).map(_.asInstanceOf[Integer]).asJava)
+    val invalidReplicaResult = client.alterPartitionReassignments(Map(
+      tp1 -> java.util.Optional.of(extraNonExistentReplica),
+      tp2 -> java.util.Optional.of(negativeIdReplica),
+      tp3 -> java.util.Optional.of(duplicateReplica)
+    ).asJava).values()
+    assertFutureExceptionTypeEquals(invalidReplicaResult.get(tp1), classOf[InvalidReplicaAssignmentException])
+    assertFutureExceptionTypeEquals(invalidReplicaResult.get(tp2), classOf[InvalidReplicaAssignmentException])
+    assertFutureExceptionTypeEquals(invalidReplicaResult.get(tp3), classOf[InvalidReplicaAssignmentException])
+  }
+
   @Test
   def testLongTopicNames(): Unit = {
     val client = AdminClient.create(createConfig)
diff --git a/core/src/test/scala/integration/kafka/api/BaseProducerSendTest.scala b/core/src/test/scala/integration/kafka/api/BaseProducerSendTest.scala
index 357dc44aa31fe..9ae095372aa40 100644
--- a/core/src/test/scala/integration/kafka/api/BaseProducerSendTest.scala
+++ b/core/src/test/scala/integration/kafka/api/BaseProducerSendTest.scala
@@ -381,8 +381,8 @@ abstract class BaseProducerSendTest extends KafkaServerTestHarness {
       }
     }
 
-    val existingAssignment = zkClient.getReplicaAssignmentForTopics(Set(topic)).map {
-      case (topicPartition, replicas) => topicPartition.partition -> replicas
+    val existingAssignment = zkClient.getFullReplicaAssignmentForTopics(Set(topic)).map {
+      case (topicPartition, assignment) => topicPartition.partition -> assignment
     }
     adminZkClient.addPartitions(topic, existingAssignment, adminZkClient.getBrokerMetadatas(), 2)
     // read metadata from a broker and verify the new topic partitions exist
diff --git a/core/src/test/scala/other/kafka/ReplicationQuotasTestRig.scala b/core/src/test/scala/other/kafka/ReplicationQuotasTestRig.scala
index a9e241ead5b37..5f31818da182b 100644
--- a/core/src/test/scala/other/kafka/ReplicationQuotasTestRig.scala
+++ b/core/src/test/scala/other/kafka/ReplicationQuotasTestRig.scala
@@ -173,9 +173,9 @@ object ReplicationQuotasTestRig {
 
       //Long stats
       println("The replicas are " + replicas.toSeq.sortBy(_._1).map("\n" + _))
-      println("This is the current replica assignment:\n" + actual.toSeq)
+      println("This is the current replica assignment:\n" + actual.mapValues(_.replicas).toMap.toSeq)
       println("proposed assignment is: \n" + newAssignment)
-      println("This is the assignment we ended up with" + actual)
+      println("This is the assignment we ended up with" + actual.mapValues(_.replicas).toMap)
 
       //Test Stats
       println(s"numBrokers: ${config.brokers}")
diff --git a/core/src/test/scala/unit/kafka/admin/AddPartitionsTest.scala b/core/src/test/scala/unit/kafka/admin/AddPartitionsTest.scala
index 69f0e41e5ae7f..13216e824eaf7 100755
--- a/core/src/test/scala/unit/kafka/admin/AddPartitionsTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/AddPartitionsTest.scala
@@ -17,6 +17,7 @@
 
 package kafka.admin
 
+import kafka.controller.PartitionReplicaAssignment
 import kafka.network.SocketServer
 import org.junit.Assert._
 import kafka.utils.TestUtils._
@@ -38,24 +39,24 @@ class AddPartitionsTest extends BaseRequestTest {
   val partitionId = 0
 
   val topic1 = "new-topic1"
-  val topic1Assignment = Map(0->Seq(0,1))
+  val topic1Assignment = Map(0 -> PartitionReplicaAssignment(Seq(0,1), List(), List()))
   val topic2 = "new-topic2"
-  val topic2Assignment = Map(0->Seq(1,2))
+  val topic2Assignment = Map(0 -> PartitionReplicaAssignment(Seq(1,2), List(), List()))
   val topic3 = "new-topic3"
-  val topic3Assignment = Map(0->Seq(2,3,0,1))
+  val topic3Assignment = Map(0 -> PartitionReplicaAssignment(Seq(2,3,0,1), List(), List()))
   val topic4 = "new-topic4"
-  val topic4Assignment = Map(0->Seq(0,3))
+  val topic4Assignment = Map(0 -> PartitionReplicaAssignment(Seq(0,3), List(), List()))
   val topic5 = "new-topic5"
-  val topic5Assignment = Map(1->Seq(0,1))
+  val topic5Assignment = Map(1 -> PartitionReplicaAssignment(Seq(0,1), List(), List()))
 
   @Before
   override def setUp(): Unit = {
     super.setUp()
 
-    createTopic(topic1, partitionReplicaAssignment = topic1Assignment)
-    createTopic(topic2, partitionReplicaAssignment = topic2Assignment)
-    createTopic(topic3, partitionReplicaAssignment = topic3Assignment)
-    createTopic(topic4, partitionReplicaAssignment = topic4Assignment)
+    createTopic(topic1, partitionReplicaAssignment = topic1Assignment.mapValues(_.replicas).toMap)
+    createTopic(topic2, partitionReplicaAssignment = topic2Assignment.mapValues(_.replicas).toMap)
+    createTopic(topic3, partitionReplicaAssignment = topic3Assignment.mapValues(_.replicas).toMap)
+    createTopic(topic4, partitionReplicaAssignment = topic4Assignment.mapValues(_.replicas).toMap)
   }
 
   @Test
diff --git a/core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala b/core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala
index fa28be915bd1f..34c8c85f931dc 100644
--- a/core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala
@@ -29,7 +29,7 @@ import org.junit.{After, Test}
 
 import kafka.admin.TopicCommand.ZookeeperTopicService
 import kafka.common.TopicAlreadyMarkedForDeletionException
-import kafka.controller.{OfflineReplica, PartitionAndReplica, ReplicaDeletionSuccessful}
+import kafka.controller.{OfflineReplica, PartitionAndReplica, PartitionReplicaAssignment, ReplicaDeletionSuccessful}
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.UnknownTopicOrPartitionException
 import org.scalatest.Assertions.fail
@@ -39,6 +39,7 @@ class DeleteTopicTest extends ZooKeeperTestHarness {
   var servers: Seq[KafkaServer] = Seq()
 
   val expectedReplicaAssignment = Map(0 -> List(0, 1, 2))
+  val expectedReplicaFullAssignment = expectedReplicaAssignment.mapValues(PartitionReplicaAssignment(_, List(), List())).toMap
 
   @After
   override def tearDown(): Unit = {
@@ -107,7 +108,6 @@ class DeleteTopicTest extends ZooKeeperTestHarness {
 
   @Test
   def testPartitionReassignmentDuringDeleteTopic(): Unit = {
-    val expectedReplicaAssignment = Map(0 -> List(0, 1, 2))
     val topic = "test"
     val topicPartition = new TopicPartition(topic, 0)
     val brokerConfigs = TestUtils.createBrokerConfigs(4, zkConnect, false)
@@ -178,7 +178,6 @@ class DeleteTopicTest extends ZooKeeperTestHarness {
 
   @Test
   def testIncreasePartitionCountDuringDeleteTopic(): Unit = {
-    val expectedReplicaAssignment = Map(0 -> List(0, 1, 2))
     val topic = "test"
     val topicPartition = new TopicPartition(topic, 0)
     val brokerConfigs = TestUtils.createBrokerConfigs(4, zkConnect, false)
@@ -248,7 +247,7 @@ class DeleteTopicTest extends ZooKeeperTestHarness {
     TestUtils.waitUntilTrue(() => zkClient.getBroker(follower.config.brokerId).isEmpty,
       s"Follower ${follower.config.brokerId} was not removed from ZK")
     // add partitions to topic
-    adminZkClient.addPartitions(topic, expectedReplicaAssignment, brokers, 2,
+    adminZkClient.addPartitions(topic, expectedReplicaFullAssignment, brokers, 2,
       Some(Map(1 -> Seq(0, 1, 2), 2 -> Seq(0, 1, 2))))
     // start topic deletion
     adminZkClient.deleteTopic(topic)
@@ -271,7 +270,7 @@ class DeleteTopicTest extends ZooKeeperTestHarness {
     adminZkClient.deleteTopic(topic)
     // add partitions to topic
     val newPartition = new TopicPartition(topic, 1)
-    adminZkClient.addPartitions(topic, expectedReplicaAssignment, brokers, 2,
+    adminZkClient.addPartitions(topic, expectedReplicaFullAssignment, brokers, 2,
       Some(Map(1 -> Seq(0, 1, 2), 2 -> Seq(0, 1, 2))))
     TestUtils.verifyTopicDeletion(zkClient, topic, 1, servers)
     // verify that new partition doesn't exist on any broker either
diff --git a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
index 957ab9b6ae67d..0c2bdd1b7a7c5 100644
--- a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
@@ -12,33 +12,37 @@
   */
 package kafka.admin
 
-import java.util.Collections
-import java.util.Properties
+import java.util.{Collections, Properties}
 
 import kafka.admin.ReassignPartitionsCommand._
 import kafka.common.AdminCommandFailedException
-import kafka.server.{KafkaConfig, KafkaServer}
+import kafka.server.{DynamicConfig, KafkaConfig, KafkaServer}
 import kafka.utils.TestUtils._
 import kafka.utils.{Logging, TestUtils}
 import kafka.zk.{ReassignPartitionsZNode, ZkVersion, ZooKeeperTestHarness}
-import org.junit.Assert.{assertEquals, assertTrue}
+import org.junit.Assert.{assertEquals, assertFalse, assertTrue}
 import org.junit.{After, Before, Test}
 import kafka.admin.ReplicationQuotaUtils._
-import org.apache.kafka.clients.admin.{Admin, AdminClient, AdminClientConfig}
+import org.apache.kafka.clients.admin.{Admin, AdminClientConfig, AlterConfigOp, ConfigEntry, NewPartitionReassignment, PartitionReassignment, AdminClient => JAdminClient}
 import org.apache.kafka.common.{TopicPartition, TopicPartitionReplica}
 
 import scala.collection.JavaConverters._
-import scala.collection.Map
-import scala.collection.Seq
+import scala.collection.{Map, Seq}
 import scala.util.Random
 import java.io.File
 
+import kafka.controller.PartitionReplicaAssignment
+import kafka.log.LogConfig
 import org.apache.kafka.clients.producer.ProducerRecord
+import org.apache.kafka.common.config.ConfigResource
+import org.apache.kafka.common.errors.NoReassignmentInProgressException
 
 class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
-  val partitionId = 0
   var servers: Seq[KafkaServer] = null
+  var brokerIds: Seq[Int] = null
   val topicName = "my-topic"
+  val tp0 = new TopicPartition(topicName, 0)
+  val tp1 = new TopicPartition(topicName, 1)
   val delayMs = 1000
   var adminClient: Admin = null
 
@@ -49,8 +53,9 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     super.setUp()
   }
 
-  def startBrokers(brokerIds: Seq[Int]): Unit = {
-    servers = brokerIds.map { i =>
+  def startBrokers(ids: Seq[Int]): Unit = {
+    brokerIds = ids
+    servers = ids.map { i =>
       val props = createBrokerConfig(i, zkConnect, enableControlledShutdown = false, logDirCount = 3)
       // shorter backoff to reduce test durations when no active partitions are eligible for fetching due to throttling
       props.put(KafkaConfig.ReplicaFetchBackoffMsProp, "100")
@@ -62,7 +67,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     val props = new Properties()
     props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, TestUtils.getBrokerListStrFromServers(servers))
     props.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000")
-    AdminClient.create(props)
+    JAdminClient.create(props)
   }
 
   def getRandomLogDirAssignment(brokerId: Int): String = {
@@ -86,26 +91,27 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     //Given a single replica on server 100
     startBrokers(Seq(100, 101, 102))
     adminClient = createAdminClient(servers)
-    createTopic(zkClient, topicName, Map(0 -> Seq(100)), servers = servers)
+    createTopic(zkClient, topicName, Map(tp0.partition() -> Seq(100)), servers = servers)
 
-    val topicPartition = new TopicPartition(topicName, 0)
     val leaderServer = servers.find(_.config.brokerId == 100).get
-    leaderServer.replicaManager.logManager.truncateFullyAndStartAt(topicPartition, 100L, false)
+    leaderServer.replicaManager.logManager.truncateFullyAndStartAt(tp0, 100L, false)
 
-    val topicJson: String = s"""{"version":1,"partitions":[{"topic":"$topicName","partition":0,"replicas":[101, 102]}]}"""
+    val topicJson = executeAssignmentJson(Seq(
+      PartitionAssignmentJson(tp0, replicas=Seq(101, 102))
+    ))
     ReassignPartitionsCommand.executeAssignment(zkClient, Some(adminClient), topicJson, NoThrottle)
 
     val newLeaderServer = servers.find(_.config.brokerId == 101).get
 
-    TestUtils.waitUntilTrue (
-      () => newLeaderServer.replicaManager.nonOfflinePartition(topicPartition).flatMap(_.leaderLogIfLocal).isDefined,
+    waitUntilTrue (
+      () => newLeaderServer.replicaManager.nonOfflinePartition(tp0).flatMap(_.leaderLogIfLocal).isDefined,
       "broker 101 should be the new leader", pause = 1L
     )
 
-    assertEquals(100, newLeaderServer.replicaManager.localLogOrException(topicPartition)
+    assertEquals(100, newLeaderServer.replicaManager.localLogOrException(tp0)
       .highWatermark)
     val newFollowerServer = servers.find(_.config.brokerId == 102).get
-    TestUtils.waitUntilTrue(() => newFollowerServer.replicaManager.localLogOrException(topicPartition)
+    waitUntilTrue(() => newFollowerServer.replicaManager.localLogOrException(tp0)
       .highWatermark == 100,
       "partition follower's highWatermark should be 100")
   }
@@ -115,18 +121,20 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     //Given a single replica on server 100
     startBrokers(Seq(100, 101))
     adminClient = createAdminClient(servers)
-    val partition = 0
     // Get a random log directory on broker 101
     val expectedLogDir = getRandomLogDirAssignment(101)
-    createTopic(zkClient, topicName, Map(partition -> Seq(100)), servers = servers)
+    createTopic(zkClient, topicName, Map(tp0.partition() -> Seq(100)), servers = servers)
 
     //When we move the replica on 100 to broker 101
-    val topicJson: String = s"""{"version":1,"partitions":[{"topic":"$topicName","partition":0,"replicas":[101],"log_dirs":["$expectedLogDir"]}]}"""
+    val topicJson = executeAssignmentJson(Seq(
+      PartitionAssignmentJson(tp0, replicas = Seq(101), logDirectories = Some(Seq(expectedLogDir)))
+    ))
     ReassignPartitionsCommand.executeAssignment(zkClient, Some(adminClient), topicJson, NoThrottle)
-    waitForReassignmentToComplete()
+    waitForZkReassignmentToComplete()
 
     //Then the replica should be on 101
-    assertEquals(Seq(101), zkClient.getPartitionAssignmentForTopics(Set(topicName)).get(topicName).get(partition))
+    val partitionAssignment = zkClient.getPartitionAssignmentForTopics(Set(topicName)).get(topicName).get(tp0.partition())
+    assertMoveForPartitionOccurred(Seq(101), partitionAssignment)
     // The replica should be in the expected log directory on broker 101
     val replica = new TopicPartitionReplica(topicName, 0, 101)
     assertEquals(expectedLogDir, adminClient.describeReplicaLogDirs(Collections.singleton(replica)).all().get.get(replica).getCurrentReplicaLogDir)
@@ -138,13 +146,15 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     startBrokers(Seq(100, 101))
     adminClient = createAdminClient(servers)
     val expectedLogDir = getRandomLogDirAssignment(100)
-    createTopic(zkClient, topicName, Map(0 -> Seq(100)), servers = servers)
+    createTopic(zkClient, topicName, Map(tp0.partition() -> Seq(100)), servers = servers)
 
     // When we execute an assignment that moves an existing replica to another log directory on the same broker
-    val topicJson: String = s"""{"version":1,"partitions":[{"topic":"$topicName","partition":0,"replicas":[100],"log_dirs":["$expectedLogDir"]}]}"""
+    val topicJson = executeAssignmentJson(Seq(
+      PartitionAssignmentJson(tp0, replicas = Seq(100), logDirectories = Some(Seq(expectedLogDir)))
+    ))
     ReassignPartitionsCommand.executeAssignment(zkClient, Some(adminClient), topicJson, NoThrottle)
     val replica = new TopicPartitionReplica(topicName, 0, 100)
-    TestUtils.waitUntilTrue(() => {
+    waitUntilTrue(() => {
       expectedLogDir == adminClient.describeReplicaLogDirs(Collections.singleton(replica)).all().get.get(replica).getCurrentReplicaLogDir
     }, "Partition should have been moved to the expected log directory", 1000)
   }
@@ -161,7 +171,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     ), servers = servers)
 
     //When rebalancing
-    val newAssignment = generateAssignment(zkClient, brokers, json(topicName), true)._1
+    val newAssignment = generateAssignment(zkClient, brokers, generateAssignmentJson(topicName), true)._1
     // Find a partition in the new assignment on broker 102 and a random log directory on broker 102,
     // which currently does not have any partition for this topic
     val partition1 = newAssignment.find { case (_, brokerIds) => brokerIds.contains(102) }.get._1.partition
@@ -177,11 +187,11 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     val newReplicaAssignment = Map(replica1 -> expectedLogDir1, replica2 -> expectedLogDir2)
     ReassignPartitionsCommand.executeAssignment(zkClient, Some(adminClient),
       ReassignPartitionsCommand.formatAsReassignmentJson(newAssignment, newReplicaAssignment), NoThrottle)
-    waitForReassignmentToComplete()
+    waitForZkReassignmentToComplete()
 
     // Then the replicas should span all three brokers
     val actual = zkClient.getPartitionAssignmentForTopics(Set(topicName))(topicName)
-    assertEquals(Seq(100, 101, 102), actual.values.flatten.toSeq.distinct.sorted)
+    assertMoveForTopicOccurred(Seq(100, 101, 102), actual)
     // The replica should be in the expected log directory on broker 102 and 100
     waitUntilTrue(() => {
       expectedLogDir1 == adminClient.describeReplicaLogDirs(Collections.singleton(replica1)).all().get.get(replica1).getCurrentReplicaLogDir
@@ -203,14 +213,14 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     ), servers = servers)
 
     //When rebalancing
-    val newAssignment = generateAssignment(zkClient, Array(100, 101), json(topicName), true)._1
+    val newAssignment = generateAssignment(zkClient, Array(100, 101), generateAssignmentJson(topicName), true)._1
     ReassignPartitionsCommand.executeAssignment(zkClient, None,
       ReassignPartitionsCommand.formatAsReassignmentJson(newAssignment, Map.empty), NoThrottle)
-    waitForReassignmentToComplete()
+    waitForZkReassignmentToComplete()
 
     //Then replicas should only span the first two brokers
     val actual = zkClient.getPartitionAssignmentForTopics(Set(topicName))(topicName)
-    assertEquals(Seq(100, 101), actual.values.flatten.toSeq.distinct.sorted)
+    assertMoveForTopicOccurred(Seq(100, 101), actual)
   }
 
   @Test
@@ -247,16 +257,16 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     //When rebalancing
     ReassignPartitionsCommand.executeAssignment(zkClient, Some(adminClient),
       ReassignPartitionsCommand.formatAsReassignmentJson(proposed, proposedReplicaAssignment), NoThrottle)
-    waitForReassignmentToComplete()
+    waitForZkReassignmentToComplete()
 
     //Then the proposed changes should have been made
     val actual = zkClient.getPartitionAssignmentForTopics(Set("topic1", "topic2"))
-    assertEquals(Seq(100, 102), actual("topic1")(0))//changed
-    assertEquals(Seq(101, 102), actual("topic1")(1))
-    assertEquals(Seq(100, 102), actual("topic1")(2))//changed
-    assertEquals(Seq(100, 101), actual("topic2")(0))
-    assertEquals(Seq(101, 100), actual("topic2")(1))//changed
-    assertEquals(Seq(100, 102), actual("topic2")(2))//changed
+    assertMoveForPartitionOccurred(Seq(100, 102), actual("topic1")(0)) //changed
+    assertMoveForPartitionOccurred(Seq(101, 102), actual("topic1")(1))
+    assertMoveForPartitionOccurred(Seq(100, 102), actual("topic1")(2)) //changed
+    assertMoveForPartitionOccurred(Seq(100, 101), actual("topic2")(0))
+    assertMoveForPartitionOccurred(Seq(101, 100), actual("topic2")(1)) //changed
+    assertMoveForPartitionOccurred(Seq(100, 102), actual("topic2")(2)) //changed
 
     // The replicas should be in the expected log directories
     val replicaDirs = adminClient.describeReplicaLogDirs(List(replica1, replica2).asJava).all().get()
@@ -283,7 +293,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     assertEquals(expectedDurationSecs, numMessages * msgSize / initialThrottle.interBrokerLimit)
 
     //Start rebalance which will move replica on 100 -> replica on 102
-    val newAssignment = generateAssignment(zkClient, Array(101, 102), json(topicName), true)._1
+    val newAssignment = generateAssignment(zkClient, Array(101, 102), generateAssignmentJson(topicName), true)._1
 
     val start = System.currentTimeMillis()
     ReassignPartitionsCommand.executeAssignment(zkClient, None,
@@ -293,12 +303,12 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     checkThrottleConfigAddedToZK(adminZkClient, initialThrottle.interBrokerLimit, servers, topicName, Set("0:100","0:101"), Set("0:102"))
 
     //Await completion
-    waitForReassignmentToComplete()
+    waitForZkReassignmentToComplete()
     val took = System.currentTimeMillis() - start - delayMs
 
     //Check move occurred
     val actual = zkClient.getPartitionAssignmentForTopics(Set(topicName))(topicName)
-    assertEquals(Seq(101, 102), actual.values.flatten.toSeq.distinct.sorted)
+    assertMoveForTopicOccurred(Seq(101, 102), actual)
 
     //Then command should have taken longer than the throttle rate
     assertTrue(s"Expected replication to be > ${expectedDurationSecs * 0.9 * 1000} but was $took",
@@ -367,7 +377,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     produceMessages(topicName, numMessages = 200, acks = 0, valueLength = 100 * 1000)
 
     //Start rebalance
-    val newAssignment = generateAssignment(zkClient, Array(101, 102), json(topicName), true)._1
+    val newAssignment = generateAssignment(zkClient, Array(101, 102), generateAssignmentJson(topicName), true)._1
 
     ReassignPartitionsCommand.executeAssignment(zkClient, None,
       ReassignPartitionsCommand.formatAsReassignmentJson(newAssignment, Map.empty), Throttle(initialThrottle))
@@ -391,7 +401,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     checkThrottleConfigAddedToZK(adminZkClient, newThrottle, servers, topicName, Set("0:100","0:101"), Set("0:102"))
 
     //Await completion
-    waitForReassignmentToComplete()
+    waitForZkReassignmentToComplete()
 
     //Verify should remove the throttle
     verifyAssignment(zkClient, None, ReassignPartitionsCommand.formatAsReassignmentJson(newAssignment, Map.empty))
@@ -401,17 +411,17 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
 
     //Check move occurred
     val actual = zkClient.getPartitionAssignmentForTopics(Set(topicName))(topicName)
-    assertEquals(Seq(101, 102), actual.values.flatten.toSeq.distinct.sorted)
+    assertMoveForTopicOccurred(Seq(101, 102), actual)
   }
 
   @Test(expected = classOf[AdminCommandFailedException])
   def shouldFailIfProposedDoesNotMatchExisting(): Unit = {
     //Given a single replica on server 100
     startBrokers(Seq(100, 101))
-    createTopic(zkClient, topicName, Map(0 -> Seq(100)), servers = servers)
+    createTopic(zkClient, topicName, Map(tp0.partition() -> Seq(100)), servers = servers)
 
     //When we execute an assignment that includes an invalid partition (1:101 in this case)
-    val topicJson = s"""{"version":1,"partitions":[{"topic":"$topicName","partition":1,"replicas":[101]}]}"""
+    val topicJson = executeAssignmentJson(Seq(PartitionAssignmentJson(tp1, Seq(101))))
     ReassignPartitionsCommand.executeAssignment(zkClient, None, topicJson, NoThrottle)
   }
 
@@ -419,10 +429,10 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
   def shouldFailIfProposedHasEmptyReplicaList(): Unit = {
     //Given a single replica on server 100
     startBrokers(Seq(100, 101))
-    createTopic(zkClient, topicName, Map(0 -> Seq(100)), servers = servers)
+    createTopic(zkClient, topicName, Map(tp0.partition() -> Seq(100)), servers = servers)
 
     //When we execute an assignment that specifies an empty replica list (0: empty list in this case)
-    val topicJson = s"""{"version":1,"partitions":[{"topic":"$topicName","partition":0,"replicas":[]}]}"""
+    val topicJson = executeAssignmentJson(Seq(PartitionAssignmentJson(tp0, Seq())))
     ReassignPartitionsCommand.executeAssignment(zkClient, None, topicJson, NoThrottle)
   }
 
@@ -430,10 +440,10 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
   def shouldFailIfProposedHasInvalidBrokerID(): Unit = {
     //Given a single replica on server 100
     startBrokers(Seq(100, 101))
-    createTopic(zkClient, topicName, Map(0 -> Seq(100)), servers = servers)
+    createTopic(zkClient, topicName, Map(tp0.partition() -> Seq(100)), servers = servers)
 
     //When we execute an assignment that specifies an invalid brokerID (102: invalid broker ID in this case)
-    val topicJson = s"""{"version":1,"partitions":[{"topic":"$topicName","partition":0,"replicas":[101, 102]}]}"""
+    val topicJson = executeAssignmentJson(Seq(PartitionAssignmentJson(tp0, Seq(101, 102))))
     ReassignPartitionsCommand.executeAssignment(zkClient, None, topicJson, NoThrottle)
   }
 
@@ -442,10 +452,10 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     // Given a single replica on server 100
     startBrokers(Seq(100, 101))
     adminClient = createAdminClient(servers)
-    createTopic(zkClient, topicName, Map(0 -> Seq(100)), servers = servers)
+    createTopic(zkClient, topicName, Map(tp0.partition() -> Seq(100)), servers = servers)
 
     // When we execute an assignment that specifies an invalid log directory
-    val topicJson: String = s"""{"version":1,"partitions":[{"topic":"$topicName","partition":0,"replicas":[101],"log_dirs":["invalidDir"]}]}"""
+    val topicJson = executeAssignmentJson(Seq(PartitionAssignmentJson(tp0, Seq(101), logDirectories = Some(Seq("invalidDir")))))
     ReassignPartitionsCommand.executeAssignment(zkClient, Some(adminClient), topicJson, NoThrottle)
   }
 
@@ -455,10 +465,10 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     startBrokers(Seq(100, 101))
     adminClient = createAdminClient(servers)
     val logDir = getRandomLogDirAssignment(100)
-    createTopic(zkClient, topicName, Map(0 -> Seq(100)), servers = servers)
+    createTopic(zkClient, topicName, Map(tp0.partition() -> Seq(100)), servers = servers)
 
-    // When we execute an assignment whose length of replicas doesn't match that of replicas
-    val topicJson: String = s"""{"version":1,"partitions":[{"topic":"$topicName","partition":0,"replicas":[101],"log_dirs":["$logDir", "$logDir"]}]}"""
+    // When we execute an assignment whose length of replicas doesn't match that of log dirs
+    val topicJson = executeAssignmentJson(Seq(PartitionAssignmentJson(tp0, Seq(101), logDirectories = Some(Seq(logDir, logDir)))))
     ReassignPartitionsCommand.executeAssignment(zkClient, Some(adminClient), topicJson, NoThrottle)
   }
 
@@ -485,7 +495,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     //When we run a throttled reassignment
     new ReassignPartitionsCommand(zkClient, None, move, adminZkClient = adminZkClient).reassignPartitions(throttle)
 
-    waitForReassignmentToComplete()
+    waitForZkReassignmentToComplete()
 
     //Check moved replicas did move
     assertEquals(Seq(0, 2, 3), zkClient.getReplicasForPartition(new TopicPartition("orders", 0)))
@@ -525,7 +535,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
 
     new ReassignPartitionsCommand(zkClient, None, firstMove, adminZkClient = adminZkClient).reassignPartitions()
     // Low pause to detect deletion of the reassign_partitions znode before the reassignment is complete
-    waitForReassignmentToComplete(pause = 1L)
+    waitForZkReassignmentToComplete(pause = 1L)
 
     // Check moved replicas did move
     assertEquals(Seq(0, 2, 3), zkClient.getReplicasForPartition(new TopicPartition("orders", 0)))
@@ -550,7 +560,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
 
     new ReassignPartitionsCommand(zkClient, None, secondMove, adminZkClient = adminZkClient).reassignPartitions()
     // Low pause to detect deletion of the reassign_partitions znode before the reassignment is complete
-    waitForReassignmentToComplete(pause = 1L)
+    waitForZkReassignmentToComplete(pause = 1L)
 
     // Check moved replicas did move
     assertEquals(Seq(0, 2, 3), zkClient.getReplicasForPartition(new TopicPartition("orders", 0)))
@@ -584,7 +594,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     }.exists(identity)
 
     // Low pause to detect deletion of the reassign_partitions znode before the reassignment is complete
-    waitForReassignmentToComplete(pause = 1L)
+    waitForZkReassignmentToComplete(pause = 1L)
 
     // Check moved replicas for thirdMove and fourthMove
     assertEquals(Seq(1, 2, 3), zkClient.getReplicasForPartition(new TopicPartition("orders", 0)))
@@ -620,23 +630,611 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     zkClient.setOrCreatePartitionReassignment(firstMove, ZkVersion.MatchAnyVersion)
 
     servers.foreach(_.startup())
-    waitForReassignmentToComplete()
+    waitForZkReassignmentToComplete()
 
     assertEquals(Seq(2, 1), zkClient.getReplicasForPartition(new TopicPartition("orders", 0)))
     assertEquals(Seq(1, 2), zkClient.getReplicasForPartition(new TopicPartition("orders", 1)))
     assertEquals(Seq.empty, zkClient.getReplicasForPartition(new TopicPartition("customers", 0)))
   }
 
-  def waitForReassignmentToComplete(pause: Long = 100L): Unit = {
+  /**
+    * Set a reassignment through the `/topics/` znode and set the `reassign_partitions` znode while the brokers are down.
+    * Verify that the reassignment is triggered by the Controller during start-up with the `reassign_partitions` znode taking precedence
+    */
+  @Test
+  def shouldTriggerReassignmentWithZnodePrecedenceOnControllerStartup(): Unit = {
+    startBrokers(Seq(0, 1, 2))
+    adminClient = createAdminClient(servers)
+    createTopic(zkClient, "orders", Map(0 -> List(0, 1), 1 -> List(1, 2), 2 -> List(0, 1), 3 -> List(0, 1)), servers)
+    val sameMoveTp = new TopicPartition("orders", 2)
+
+    // Throttle to ensure we minimize race conditions and test flakiness
+    throttle(Seq("orders"), throttleSettingForSeconds(10), Map(
+      sameMoveTp -> Seq(0, 1, 2)
+    ))
+
+    servers.foreach(_.shutdown())
+    adminClient.close()
+
+    zkClient.setTopicAssignment("orders", Map(
+      new TopicPartition("orders", 0) -> PartitionReplicaAssignment(List(0, 1), List(2), List(0)), // should be overwritten
+      new TopicPartition("orders", 1) -> PartitionReplicaAssignment(List(1, 2), List(3), List(1)), // should be overwritten
+      // should be overwritten (so we know to remove it from ZK) even though we do the exact same move
+      sameMoveTp -> PartitionReplicaAssignment(List(0, 1, 2), List(2), List(0)),
+      new TopicPartition("orders", 3) -> PartitionReplicaAssignment(List(0, 1, 2), List(2), List(0)) // moves
+    ))
+    val move = Map(
+      new TopicPartition("orders", 0) -> Seq(2, 1), // moves
+      new TopicPartition("orders", 1) -> Seq(1, 2), // stays
+      sameMoveTp -> Seq(1, 2), // same reassignment
+      // orders-3 intentionally left for API
+      new TopicPartition("customers", 0) -> Seq(1, 2) // non-existent topic, triggers topic deleted path
+    )
+
+    // Set znode directly to avoid non-existent topic validation
+    zkClient.setOrCreatePartitionReassignment(move, ZkVersion.MatchAnyVersion)
+
+    servers.foreach(_.startup())
+    TestUtils.waitUntilBrokerMetadataIsPropagated(servers)
+    adminClient = createAdminClient(servers)
+    resetBrokersThrottle()
+
+    waitForZkReassignmentToComplete()
+
+    assertEquals(Seq(2, 1), zkClient.getReplicasForPartition(new TopicPartition("orders", 0)))
+    assertEquals(Seq(1, 2), zkClient.getReplicasForPartition(new TopicPartition("orders", 1)))
+    assertEquals(Seq(1, 2), zkClient.getReplicasForPartition(sameMoveTp))
+    assertEquals(Seq.empty, zkClient.getReplicasForPartition(new TopicPartition("customers", 0)))
+  }
+
+  @Test
+  def shouldListReassignmentsTriggeredByZk(): Unit = {
+    // Given a single replica on server 100
+    startBrokers(Seq(100, 101))
+    adminClient = createAdminClient(servers)
+    // Get a random log directory on broker 101
+    val expectedLogDir = getRandomLogDirAssignment(101)
+    createTopic(zkClient, topicName, Map(tp0.partition() -> Seq(100)), servers = servers)
+    // Given throttle set so replication will take at least 2 sec (to ensure we don't minimize race condition and test flakiness
+    val throttle: Long = 1000 * 1000
+    produceMessages(topicName, numMessages = 20, acks = 0, valueLength = 100 * 1000)
+
+    // When we move the replica on 100 to broker 101
+    val topicJson = executeAssignmentJson(Seq(
+      PartitionAssignmentJson(tp0, replicas = Seq(101), Some(Seq(expectedLogDir)))))
+    ReassignPartitionsCommand.executeAssignment(zkClient, Some(adminClient), topicJson, Throttle(throttle))
+    // Then the replica should be removing
+    val reassigningPartitionsResult = adminClient.listPartitionReassignments(Set(tp0).asJava).reassignments().get().get(tp0)
+    assertIsReassigning(from = Seq(100), to = Seq(101), reassigningPartitionsResult)
+
+    waitForZkReassignmentToComplete()
+
+    // Then the replica should be on 101
+    val partitionAssignment = zkClient.getPartitionAssignmentForTopics(Set(topicName)).get(topicName).get(tp0.partition())
+    assertMoveForPartitionOccurred(Seq(101), partitionAssignment)
+  }
+
+  @Test
+  def shouldReassignThroughApi(): Unit = {
+    startBrokers(Seq(100, 101))
+    adminClient = createAdminClient(servers)
+    createTopic(zkClient, topicName, Map(tp0.partition() -> Seq(100)), servers = servers)
+
+    assertTrue(adminClient.listPartitionReassignments(Set(tp0).asJava).reassignments().get().isEmpty)
+    assertEquals(Seq(100), zkClient.getReplicasForPartition(tp0))
+    adminClient.alterPartitionReassignments(
+      Map(reassignmentEntry(tp0, Seq(101))).asJava
+    ).all().get()
+
+    waitForAllReassignmentsToComplete()
+    assertEquals(Seq(101), zkClient.getReplicasForPartition(tp0))
+  }
+
+  @Test
+  def shouldListMovingPartitionsThroughApi(): Unit = {
+    startBrokers(Seq(100, 101))
+    adminClient = createAdminClient(servers)
+    val topic2 = "topic2"
+    val tp2 = new TopicPartition(topic2, 0)
+
+    createTopic(zkClient, topicName,
+      Map(tp0.partition() -> Seq(100),
+          tp1.partition() -> Seq(101)),
+      servers = servers)
+    createTopic(zkClient, topic2,
+      Map(tp2.partition() -> Seq(100)),
+      servers = servers)
+    assertTrue(adminClient.listPartitionReassignments().reassignments().get().isEmpty)
+
+    // Throttle to ensure we minimize race conditions and test flakiness
+    throttle(Seq(topicName), throttleSettingForSeconds(10), Map(
+      tp0 -> Seq(100, 101),
+      tp2 -> Seq(100, 101)
+    ))
+    adminClient.alterPartitionReassignments(
+      Map(reassignmentEntry(tp0, Seq(101)),
+          reassignmentEntry(tp2, Seq(101))).asJava
+    ).all().get()
+
+    val reassignmentsInProgress = adminClient.listPartitionReassignments(Set(tp0, tp1, tp2).asJava).reassignments().get()
+    assertFalse(reassignmentsInProgress.containsKey(tp1)) // tp1 is not reassigning
+    assertIsReassigning(from = Seq(100), to = Seq(101), reassignmentsInProgress.get(tp0))
+    assertIsReassigning(from = Seq(100), to = Seq(101), reassignmentsInProgress.get(tp2))
+
+    resetBrokersThrottle()
+    waitForAllReassignmentsToComplete()
+    assertEquals(Seq(101), zkClient.getReplicasForPartition(tp0))
+    assertEquals(Seq(101), zkClient.getReplicasForPartition(tp2))
+  }
+
+  @Test
+  def shouldUseLatestOrderingIfTwoConsecutiveReassignmentsHaveSameSetButDifferentOrdering(): Unit = {
+    startBrokers(Seq(100, 101, 102))
+    adminClient = createAdminClient(servers)
+    createTopic(zkClient, topicName,
+      Map(tp0.partition() -> Seq(100, 101),
+          tp1.partition() -> Seq(100, 101)),
+      servers = servers)
+
+    // Throttle to ensure we minimize race conditions and test flakiness
+    throttle(Seq(topicName), throttleSettingForSeconds(10), Map(
+      tp0 -> Seq(100, 101, 102),
+      tp1 -> Seq(100, 101, 102)
+    ))
+
+    adminClient.alterPartitionReassignments(
+      Map(reassignmentEntry(tp0, Seq(100, 101, 102)),
+          reassignmentEntry(tp1, Seq(100, 101, 102))).asJava
+    ).all().get()
+    val apiReassignmentsInProgress = adminClient.listPartitionReassignments(Set(tp0, tp1).asJava).reassignments().get()
+    assertIsReassigning(from = Seq(100, 101), to = Seq(100, 101, 102), apiReassignmentsInProgress.get(tp0))
+    assertIsReassigning(from = Seq(100, 101), to = Seq(100, 101, 102), apiReassignmentsInProgress.get(tp1))
+
+    // API reassignment to the same replicas but a different order
+    adminClient.alterPartitionReassignments(
+      Map(reassignmentEntry(tp0, Seq(102, 101, 100)),
+          reassignmentEntry(tp1, Seq(102, 101, 100))).asJava
+    ).all().get()
+    val apiReassignmentsInProgress2 = adminClient.listPartitionReassignments(Set(tp0, tp1).asJava).reassignments().get()
+    // assert same replicas, ignoring ordering
+    assertIsReassigning(from = Seq(100, 101), to = Seq(100, 101, 102), apiReassignmentsInProgress2.get(tp0))
+    assertIsReassigning(from = Seq(100, 101), to = Seq(100, 101, 102), apiReassignmentsInProgress2.get(tp1))
+
+    resetBrokersThrottle()
+    waitForAllReassignmentsToComplete()
+
+    //Check move occurred
+    val actual = zkClient.getPartitionAssignmentForTopics(Set(topicName))(topicName)
+    assertMoveForPartitionOccurred(Seq(102, 101, 100), actual(tp0.partition()))
+    assertMoveForPartitionOccurred(Seq(102, 101, 100), actual(tp1.partition()))
+  }
+
+  /**
+    * 1. Trigger API reassignment for partitions
+    * 2. Trigger ZK reassignment for partitions
+    * Ensure ZK reassignment overrides API reassignment and znode is deleted
+    */
+  @Test
+  def znodeReassignmentShouldOverrideApiTriggeredReassignment(): Unit = {
+    startBrokers(Seq(100, 101, 102))
+    adminClient = createAdminClient(servers)
+    createTopic(zkClient, topicName,
+      Map(tp0.partition() -> Seq(100),
+          tp1.partition() -> Seq(100)),
+      servers = servers)
+
+    // Throttle to avoid race conditions
+    val throttleSetting = throttleSettingForSeconds(10)
+    throttle(Seq(topicName), throttleSetting, Map(
+      tp0 -> Seq(100, 101, 102),
+      tp1 -> Seq(100, 101, 102)
+    ))
+
+    // API reassignment to 101 for both partitions
+    adminClient.alterPartitionReassignments(
+      Map(reassignmentEntry(tp0, Seq(101)),
+          reassignmentEntry(tp1, Seq(101))).asJava
+    ).all().get()
+    val apiReassignmentsInProgress = adminClient.listPartitionReassignments(Set(tp0, tp1).asJava).reassignments().get()
+    assertIsReassigning(from = Seq(100), to = Seq(101), apiReassignmentsInProgress.get(tp0))
+    assertIsReassigning(from = Seq(100), to = Seq(101), apiReassignmentsInProgress.get(tp1))
+
+    // znode reassignment to 102 for both partitions
+    val topicJson = executeAssignmentJson(Seq(
+      PartitionAssignmentJson(tp0, Seq(102)),
+      PartitionAssignmentJson(tp1, Seq(102))
+    ))
+    ReassignPartitionsCommand.executeAssignment(zkClient, Some(adminClient), topicJson, Throttle(throttleSetting.throttleBytes.toLong))
+    waitUntilTrue(() => {
+      !adminClient.listPartitionReassignments().reassignments().get().isEmpty
+    }, "Controller should have picked up on znode creation", 1000)
+
+    val zkReassignmentsInProgress = adminClient.listPartitionReassignments(Set(tp0, tp1).asJava).reassignments().get()
+    assertIsReassigning(from = Seq(100), to = Seq(102), zkReassignmentsInProgress.get(tp0))
+    assertIsReassigning(from = Seq(100), to = Seq(102), zkReassignmentsInProgress.get(tp1))
+
+    resetBrokersThrottle()
+    waitForZkReassignmentToComplete()
+    assertTrue(adminClient.listPartitionReassignments(Set(tp0, tp1).asJava).reassignments().get().isEmpty)
+    assertEquals(Seq(102), zkClient.getReplicasForPartition(tp0))
+    assertEquals(Seq(102), zkClient.getReplicasForPartition(tp1))
+  }
+
+  /**
+    * 1. Trigger ZK reassignment for TP A-0, A-1
+    * 2. Trigger API reassignment for partitions TP A-1, B-0
+    * 3. Unthrottle A-0, A-1 so the ZK reassignment finishes quickly
+    * 4. Ensure ZK node is emptied out after the API reassignment of 1 finishes
+    */
+  @Test
+  def shouldDeleteReassignmentZnodeAfterApiReassignmentForPartitionCompletes(): Unit = {
+    startBrokers(Seq(100, 101, 102))
+    adminClient = createAdminClient(servers)
+    val tpA0 = new TopicPartition("A", 0)
+    val tpA1 = new TopicPartition("A", 1)
+    val tpB0 = new TopicPartition("B", 0)
+
+    createTopic(zkClient, "A",
+      Map(tpA0.partition() -> Seq(100),
+          tpA1.partition() -> Seq(100)),
+      servers = servers)
+    createTopic(zkClient, "B",
+      Map(tpB0.partition() -> Seq(100)),
+      servers = servers)
+
+    // Throttle to avoid race conditions
+    throttle(Seq("A", "B"), throttleSettingForSeconds(10), Map(
+      tpA0 -> Seq(100, 101, 102),
+      tpA1 -> Seq(100, 101, 102),
+      tpB0 -> Seq(100, 101, 102)
+    ))
+
+    // 1. znode reassignment to 101 for TP A-0, A-1
+    val topicJson = executeAssignmentJson(Seq(
+      PartitionAssignmentJson(tpA0, replicas=Seq(101)),
+      PartitionAssignmentJson(tpA1, replicas=Seq(101))
+    ))
+    ReassignPartitionsCommand.executeAssignment(zkClient, Some(adminClient), topicJson, NoThrottle)
+    waitUntilTrue(() => {
+      !adminClient.listPartitionReassignments().reassignments().get().isEmpty
+    }, "Controller should have picked up on znode creation", 1000)
+    val zkReassignmentsInProgress = adminClient.listPartitionReassignments(Set(tpA0, tpA1).asJava).reassignments().get()
+    assertIsReassigning(from = Seq(100), to = Seq(101), zkReassignmentsInProgress.get(tpA0))
+    assertIsReassigning(from = Seq(100), to = Seq(101), zkReassignmentsInProgress.get(tpA1))
+
+    // 2. API reassignment to 102 for TP A-1, B-0
+    adminClient.alterPartitionReassignments(
+      Map(reassignmentEntry(tpA1, Seq(102)), reassignmentEntry(tpB0, Seq(102))).asJava
+    ).all().get()
+    val apiReassignmentsInProgress = adminClient.listPartitionReassignments(Set(tpA1, tpB0).asJava).reassignments().get()
+    assertIsReassigning(from = Seq(100), to = Seq(102), apiReassignmentsInProgress.get(tpA1))
+    assertIsReassigning(from = Seq(100), to = Seq(102), apiReassignmentsInProgress.get(tpB0))
+
+    // 3. Unthrottle topic A
+    removePartitionReplicaThrottles(Set(tpA0, tpA1))
+    waitForZkReassignmentToComplete()
+    // 4. Ensure the API reassignment not part of the znode is still in progress
+    val leftoverReassignments = adminClient.listPartitionReassignments(Set(tpA0, tpA1, tpB0).asJava).reassignments().get()
+    assertEquals(1, leftoverReassignments.size())
+    val tpB0LeftoverReassignment = leftoverReassignments.get(tpB0)
+    assertIsReassigning(from = Seq(100), to = Seq(102), tpB0LeftoverReassignment)
+
+    resetBrokersThrottle()
+    waitForAllReassignmentsToComplete()
+    assertEquals(Seq(101), zkClient.getReplicasForPartition(tpA0))
+    assertEquals(Seq(102), zkClient.getReplicasForPartition(tpA1))
+    assertEquals(Seq(102), zkClient.getReplicasForPartition(tpB0))
+  }
+
+  @Test
+  def shouldBeAbleToCancelThroughApi(): Unit = {
+    startBrokers(Seq(100, 101, 102))
+    adminClient = createAdminClient(servers)
+    createTopic(zkClient, topicName, Map(tp0.partition() -> Seq(100, 101)), servers = servers)
+    // Throttle to ensure we minimize race conditions and test flakiness
+    throttle(Seq(topicName), throttleSettingForSeconds(10), Map(
+      tp0 -> Seq(100, 101, 102)
+    ))
+
+    // move to [102, 101]
+    adminClient.alterPartitionReassignments(
+      Map(reassignmentEntry(tp0, Seq(102, 101))).asJava
+    ).all().get()
+    val apiReassignmentsInProgress = adminClient.listPartitionReassignments().reassignments().get()
+    val tpReassignment = apiReassignmentsInProgress.get(tp0)
+    assertIsReassigning(from = Seq(100, 101), to = Seq(101, 102), tpReassignment)
+
+    adminClient.alterPartitionReassignments(
+      Map(cancelReassignmentEntry(tp0)).asJava
+    ).all().get()
+
+    resetBrokersThrottle()
+    waitForAllReassignmentsToComplete()
+    assertEquals(Seq(100, 101), zkClient.getReplicasForPartition(tp0).sorted) // revert ordering is not guaranteed
+  }
+
+  @Test
+  def shouldBeAbleToCancelZkTriggeredReassignmentThroughApi(): Unit = {
+    startBrokers(Seq(100, 101))
+    adminClient = createAdminClient(servers)
+    createTopic(zkClient, topicName,
+      Map(tp0.partition() -> Seq(100),
+          tp1.partition() -> Seq(100)),
+      servers = servers)
+
+    // Throttle to avoid race conditions
+    throttle(Seq(topicName), throttleSettingForSeconds(10), Map(
+      tp0 -> Seq(100, 101),
+      tp1 -> Seq(100, 101)
+    ))
+
+    val move = Map(
+      tp0 -> Seq(101),
+      tp1 -> Seq(101)
+    )
+    zkClient.setOrCreatePartitionReassignment(move, ZkVersion.MatchAnyVersion)
+    waitUntilTrue(() => {
+      !adminClient.listPartitionReassignments().reassignments().get().isEmpty
+    }, "Controller should have picked up on znode creation", 1000)
+    var reassignmentIsOngoing = adminClient.listPartitionReassignments().reassignments().get().size() > 0
+    assertTrue(reassignmentIsOngoing)
+
+    adminClient.alterPartitionReassignments(
+      Map(cancelReassignmentEntry(tp0), cancelReassignmentEntry(tp1)).asJava
+    ).all().get()
+
+    resetBrokersThrottle()
+    waitForZkReassignmentToComplete()
+    reassignmentIsOngoing = adminClient.listPartitionReassignments().reassignments().get().size() > 0
+    assertFalse(reassignmentIsOngoing)
+    assertEquals(Seq(100), zkClient.getReplicasForPartition(tp0))
+    assertEquals(Seq(100), zkClient.getReplicasForPartition(tp1))
+  }
+
+  /**
+    * Cancel and set reassignments in the same API call.
+    * Even though one cancellation is invalid, ensure the other entries in the request pass
+    */
+  @Test
+  def testCancelAndSetSomeReassignments(): Unit = {
+    startBrokers(Seq(100, 101, 102))
+    adminClient = createAdminClient(servers)
+    val tp2 = new TopicPartition(topicName, 2)
+    val tp3 = new TopicPartition(topicName, 3)
+
+    createTopic(zkClient, topicName,
+      Map(tp0.partition() -> Seq(100), tp1.partition() -> Seq(100), tp2.partition() -> Seq(100), tp3.partition() -> Seq(100)),
+      servers = servers)
+
+    // Throttle to avoid race conditions
+    throttle(Seq(topicName), throttleSettingForSeconds(10), Map(
+      tp0 -> Seq(100, 101, 102),
+      tp1 -> Seq(100, 101, 102),
+      tp2 -> Seq(100, 101, 102),
+      tp3 -> Seq(100, 101, 102)
+    ))
+
+    // API reassignment to 101 for tp0 and tp1
+    adminClient.alterPartitionReassignments(
+      Map(reassignmentEntry(tp0, Seq(101)), reassignmentEntry(tp1, Seq(101))).asJava
+    ).all().get()
+
+    // cancel tp0, reassign tp1 to 102 (override), assign tp2 to 101 (new reassignment) and cancel tp3 (it is not moving)
+    val alterResults = adminClient.alterPartitionReassignments(
+      Map(cancelReassignmentEntry(tp0), reassignmentEntry(tp1, Seq(102)),
+          reassignmentEntry(tp2, Seq(101)), cancelReassignmentEntry(tp3)).asJava
+    ).values()
+    alterResults.get(tp0).get()
+    alterResults.get(tp1).get()
+    alterResults.get(tp2).get()
+    try {
+      alterResults.get(tp3).get()
+    } catch {
+      case exception: Exception =>
+        assertEquals(exception.getCause.getClass, classOf[NoReassignmentInProgressException])
+    }
+
+    resetBrokersThrottle()
+    waitForAllReassignmentsToComplete()
+    assertEquals(Seq(100), zkClient.getReplicasForPartition(tp0))
+    assertEquals(Seq(102), zkClient.getReplicasForPartition(tp1))
+    assertEquals(Seq(101), zkClient.getReplicasForPartition(tp2))
+    assertEquals(Seq(100), zkClient.getReplicasForPartition(tp3))
+  }
+
+  /**
+    * Three different Alter Reassignment calls should all create reassignments
+    */
+  @Test
+  def shouldBeAbleToIncrementallyStackDifferentReassignments(): Unit = {
+    startBrokers(Seq(100, 101))
+    adminClient = createAdminClient(servers)
+    val tpA0 = new TopicPartition("A", 0)
+    val tpA1 = new TopicPartition("A", 1)
+    val tpB0 = new TopicPartition("B", 0)
+
+    createTopic(zkClient, "A",
+      Map(tpA0.partition() -> Seq(100),
+          tpA1.partition() -> Seq(100)),
+      servers = servers)
+    createTopic(zkClient, "B",
+      Map(tpB0.partition() -> Seq(100)),
+      servers = servers)
+
+    // Throttle to avoid race conditions
+    throttle(Seq("A", "B"), throttleSettingForSeconds(10), Map(
+      tpA0 -> Seq(100, 101, 102),
+      tpA1 -> Seq(100, 101, 102),
+      tpB0 -> Seq(100, 101, 102)
+    ))
+
+    adminClient.alterPartitionReassignments(Map(reassignmentEntry(tpA0, Seq(101))).asJava).all().get()
+    val apiReassignmentsInProgress1 = adminClient.listPartitionReassignments().reassignments().get()
+    assertEquals(1, apiReassignmentsInProgress1.size())
+    assertIsReassigning(
+      from = Seq(100), to = Seq(101),
+      apiReassignmentsInProgress1.get(tpA0)
+    )
+
+    adminClient.alterPartitionReassignments(Map(reassignmentEntry(tpA1, Seq(101))).asJava).all().get()
+    val apiReassignmentsInProgress2 = adminClient.listPartitionReassignments().reassignments().get()
+    assertEquals(2, apiReassignmentsInProgress2.size())
+    assertIsReassigning(from = Seq(100), to = Seq(101), apiReassignmentsInProgress2.get(tpA0))
+    assertIsReassigning(
+      from = Seq(100), to = Seq(101),
+      apiReassignmentsInProgress2.get(tpA1)
+    )
+
+    adminClient.alterPartitionReassignments(Map(reassignmentEntry(tpB0, Seq(101))).asJava).all().get()
+    val apiReassignmentsInProgress3 = adminClient.listPartitionReassignments().reassignments().get()
+    assertEquals(s"${apiReassignmentsInProgress3}", 3, apiReassignmentsInProgress3.size())
+    assertIsReassigning(from = Seq(100), to = Seq(101), apiReassignmentsInProgress3.get(tpA0))
+    assertIsReassigning(from = Seq(100), to = Seq(101), apiReassignmentsInProgress3.get(tpA1))
+    assertIsReassigning(
+      from = Seq(100), to = Seq(101),
+      apiReassignmentsInProgress3.get(tpB0)
+    )
+
+    resetBrokersThrottle()
+    waitForAllReassignmentsToComplete()
+    assertEquals(Seq(101), zkClient.getReplicasForPartition(tpA0))
+    assertEquals(Seq(101), zkClient.getReplicasForPartition(tpA1))
+    assertEquals(Seq(101), zkClient.getReplicasForPartition(tpB0))
+  }
+
+  /**
+    * Asserts that a replica is being reassigned from the given replicas to the target replicas
+    */
+  def assertIsReassigning(from: Seq[Int], to: Seq[Int], reassignment: PartitionReassignment): Unit = {
+    assertReplicas((from ++ to).distinct, reassignment.replicas())
+    assertReplicas(to.filterNot(from.contains(_)), reassignment.addingReplicas())
+    assertReplicas(from.filterNot(to.contains(_)), reassignment.removingReplicas())
+  }
+
+  /**
+   * Asserts that a topic's reassignments completed and span across the expected replicas
+   */
+  def assertMoveForTopicOccurred(expectedReplicas: Seq[Int],
+                                 partitionAssignments: Map[Int, PartitionReplicaAssignment]): Unit = {
+    assertEquals(expectedReplicas, partitionAssignments.values.flatMap(_.replicas).toSeq.distinct.sorted)
+    assertTrue(partitionAssignments.values.flatMap(_.addingReplicas).isEmpty)
+    assertTrue(partitionAssignments.values.flatMap(_.removingReplicas).isEmpty)
+  }
+
+  /**
+   * Asserts that a partition moved to the exact expected replicas in the specific order
+   */
+  def assertMoveForPartitionOccurred(expectedReplicas: Seq[Int],
+                                     partitionAssignment: PartitionReplicaAssignment): Unit = {
+    assertEquals(expectedReplicas, partitionAssignment.replicas)
+    assertTrue(partitionAssignment.addingReplicas.isEmpty)
+    assertTrue(partitionAssignment.removingReplicas.isEmpty)
+  }
+
+  /**
+   * Asserts that two replica sets are equal, ignoring ordering
+   */
+  def assertReplicas(expectedReplicas: Seq[Int], receivedReplicas: java.util.List[Integer]): Unit = {
+    assertEquals(expectedReplicas.sorted, receivedReplicas.asScala.map(_.toInt).sorted)
+  }
+
+  def throttleAllBrokersReplication(throttleBytes: String): Unit = {
+    val throttleConfigs = Seq(
+      new AlterConfigOp(new ConfigEntry(DynamicConfig.Broker.LeaderReplicationThrottledRateProp, throttleBytes), AlterConfigOp.OpType.SET),
+      new AlterConfigOp(new ConfigEntry(DynamicConfig.Broker.FollowerReplicationThrottledRateProp, throttleBytes), AlterConfigOp.OpType.SET)
+    ).asJavaCollection
+
+    adminClient.incrementalAlterConfigs(
+      brokerIds.map { brokerId =>
+        new ConfigResource(ConfigResource.Type.BROKER, brokerId.toString) -> throttleConfigs
+      }.toMap.asJava
+    ).all().get()
+  }
+
+  def resetBrokersThrottle(): Unit = throttleAllBrokersReplication(Int.MaxValue.toString)
+
+  def assignThrottledPartitionReplicas(allReplicasByPartition: Map[TopicPartition, Seq[Int]]): Unit = {
+    val throttles = allReplicasByPartition.groupBy(_._1.topic()).map {
+      case (topic, replicasByPartition) =>
+        new ConfigResource(ConfigResource.Type.TOPIC, topic) -> Seq(
+          new AlterConfigOp(new ConfigEntry(LogConfig.LeaderReplicationThrottledReplicasProp, formatReplicaThrottles(replicasByPartition)), AlterConfigOp.OpType.SET),
+          new AlterConfigOp(new ConfigEntry(LogConfig.FollowerReplicationThrottledReplicasProp, formatReplicaThrottles(replicasByPartition)), AlterConfigOp.OpType.SET)
+        ).asJavaCollection
+    }
+    adminClient.incrementalAlterConfigs(throttles.asJava).all().get()
+  }
+
+  def removePartitionReplicaThrottles(partitions: Set[TopicPartition]): Unit = {
+    val throttles = partitions.map {
+      tp =>
+        new ConfigResource(ConfigResource.Type.TOPIC, tp.topic()) -> Seq(
+          new AlterConfigOp(new ConfigEntry(LogConfig.LeaderReplicationThrottledReplicasProp, ""), AlterConfigOp.OpType.DELETE),
+          new AlterConfigOp(new ConfigEntry(LogConfig.FollowerReplicationThrottledReplicasProp, ""), AlterConfigOp.OpType.DELETE)
+        ).asJavaCollection
+    }.toMap
+    adminClient.incrementalAlterConfigs(throttles.asJava).all().get()
+  }
+
+  def formatReplicaThrottles(moves: Map[TopicPartition, Seq[Int]]): String =
+    moves.flatMap { case (tp, assignment) =>
+      assignment.map(replicaId => s"${tp.partition}:$replicaId")
+    }.mkString(",")
+
+  def reassignmentEntry(tp: TopicPartition, replicas: Seq[Int]): (TopicPartition, java.util.Optional[NewPartitionReassignment]) =
+    tp -> java.util.Optional.of(new NewPartitionReassignment(replicas.map(_.asInstanceOf[Integer]).asJava))
+
+  def cancelReassignmentEntry(tp: TopicPartition): (TopicPartition, java.util.Optional[NewPartitionReassignment]) =
+    tp -> java.util.Optional.empty()
+
+  def waitForZkReassignmentToComplete(pause: Long = 100L): Unit = {
     waitUntilTrue(() => !zkClient.reassignPartitionsInProgress,
       s"Znode ${ReassignPartitionsZNode.path} wasn't deleted", pause = pause)
   }
 
-  def json(topic: String*): String = {
+  def waitForAllReassignmentsToComplete(pause: Long = 100L): Unit = {
+    waitUntilTrue(() => adminClient.listPartitionReassignments().reassignments().get().isEmpty,
+      s"There still are ongoing reassignments", pause = pause)
+  }
+
+  def generateAssignmentJson(topic: String*): String = {
     val topicStr = topic.map { t => "{\"topic\": \"" + t + "\"}" }.mkString(",")
     s"""{"topics": [$topicStr],"version":1}"""
   }
 
+  def executeAssignmentJson(partitions: Seq[PartitionAssignmentJson]): String =
+    s"""{"version":1,"partitions":[${partitions.map(_.toJson).mkString(",")}]}"""
+
+  case class PartitionAssignmentJson(topicPartition: TopicPartition, replicas: Seq[Int],
+                                     logDirectories: Option[Seq[String]] = None) {
+    def toJson: String = {
+      val logDirsSuffix = logDirectories match {
+        case Some(dirs) => s""","log_dirs":[${dirs.map("\"" + _ + "\"").mkString(",")}]"""
+        case None => ""
+      }
+      s"""{"topic":"${topicPartition.topic()}","partition":${topicPartition.partition()}""" +
+        s""","replicas":[${replicas.mkString(",")}]""" +
+        s"$logDirsSuffix}"
+    }
+  }
+
+  case class ThrottleSetting(throttleBytes: String, numMessages: Int, messageSizeBytes: Int)
+
+  def throttleSettingForSeconds(secondsDuration: Int): ThrottleSetting = {
+    val throttle = 1000 * 1000 // 1 MB/s throttle
+    val messageSize = 100 * 100 // 0.01 MB message size
+    val messagesPerSecond = throttle / messageSize
+    ThrottleSetting(throttle.toString, messagesPerSecond * secondsDuration, messageSize)
+  }
+
+  def throttle(topics: Seq[String], throttle: ThrottleSetting, replicasToThrottle: Map[TopicPartition, Seq[Int]]): Unit = {
+    val messagesPerTopic = throttle.numMessages / topics.size
+    for (topic <- topics) {
+      produceMessages(topic, numMessages = messagesPerTopic, acks = 0, valueLength = throttle.messageSizeBytes)
+    }
+    throttleAllBrokersReplication(throttle.throttleBytes)
+    assignThrottledPartitionReplicas(replicasToThrottle)
+  }
+
   private def produceMessages(topic: String, numMessages: Int, acks: Int, valueLength: Int): Unit = {
     val records = (0 until numMessages).map(_ => new ProducerRecord[Array[Byte], Array[Byte]](topic,
       new Array[Byte](valueLength)))
diff --git a/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala b/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala
index afe530898d6a3..9c2941ae2fd33 100644
--- a/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala
@@ -58,7 +58,7 @@ class ControllerChannelManagerTest {
     partitions.foreach { case (partition, leaderAndIsr) =>
       val leaderIsrAndControllerEpoch = LeaderIsrAndControllerEpoch(leaderAndIsr, controllerEpoch)
       context.partitionLeadershipInfo.put(partition, leaderIsrAndControllerEpoch)
-      batch.addLeaderAndIsrRequestForBrokers(Seq(2), partition, leaderIsrAndControllerEpoch, Seq(1, 2, 3), isNew = false)
+      batch.addLeaderAndIsrRequestForBrokers(Seq(2), partition, leaderIsrAndControllerEpoch, replicaAssignment(Seq(1, 2, 3)), isNew = false)
     }
     batch.sendRequestsToBrokers(controllerEpoch)
 
@@ -96,8 +96,8 @@ class ControllerChannelManagerTest {
     context.partitionLeadershipInfo.put(partition, leaderIsrAndControllerEpoch)
 
     batch.newBatch()
-    batch.addLeaderAndIsrRequestForBrokers(Seq(2), partition, leaderIsrAndControllerEpoch, Seq(1, 2, 3), isNew = true)
-    batch.addLeaderAndIsrRequestForBrokers(Seq(2), partition, leaderIsrAndControllerEpoch, Seq(1, 2, 3), isNew = false)
+    batch.addLeaderAndIsrRequestForBrokers(Seq(2), partition, leaderIsrAndControllerEpoch, replicaAssignment(Seq(1, 2, 3)), isNew = true)
+    batch.addLeaderAndIsrRequestForBrokers(Seq(2), partition, leaderIsrAndControllerEpoch, replicaAssignment(Seq(1, 2, 3)), isNew = false)
     batch.sendRequestsToBrokers(controllerEpoch)
 
     val leaderAndIsrRequests = batch.collectLeaderAndIsrRequestsFor(2)
@@ -126,7 +126,7 @@ class ControllerChannelManagerTest {
     context.partitionLeadershipInfo.put(partition, leaderIsrAndControllerEpoch)
 
     batch.newBatch()
-    batch.addLeaderAndIsrRequestForBrokers(Seq(1, 2, 3), partition, leaderIsrAndControllerEpoch, Seq(1, 2, 3), isNew = false)
+    batch.addLeaderAndIsrRequestForBrokers(Seq(1, 2, 3), partition, leaderIsrAndControllerEpoch, replicaAssignment(Seq(1, 2, 3)), isNew = false)
     batch.sendRequestsToBrokers(controllerEpoch)
 
     assertEquals(0, batch.sentEvents.size)
@@ -170,7 +170,7 @@ class ControllerChannelManagerTest {
     context.partitionLeadershipInfo.put(partition, leaderIsrAndControllerEpoch)
 
     batch.newBatch()
-    batch.addLeaderAndIsrRequestForBrokers(Seq(2), partition, leaderIsrAndControllerEpoch, Seq(1, 2, 3), isNew = false)
+    batch.addLeaderAndIsrRequestForBrokers(Seq(2), partition, leaderIsrAndControllerEpoch, replicaAssignment(Seq(1, 2, 3)), isNew = false)
     batch.sendRequestsToBrokers(controllerEpoch)
 
     val leaderAndIsrRequests = batch.collectLeaderAndIsrRequestsFor(2, expectedLeaderAndIsrVersion)
@@ -635,6 +635,8 @@ class ControllerChannelManagerTest {
     KafkaConfig.fromProps(props)
   }
 
+  private def replicaAssignment(replicas: Seq[Int]): PartitionReplicaAssignment = PartitionReplicaAssignment(replicas, Seq(), Seq())
+
   private def initContext(brokers: Seq[Int],
                           topics: Set[String],
                           numPartitions: Int,
diff --git a/core/src/test/scala/unit/kafka/controller/ControllerContextTest.scala b/core/src/test/scala/unit/kafka/controller/ControllerContextTest.scala
new file mode 100644
index 0000000000000..eeddad00fef51
--- /dev/null
+++ b/core/src/test/scala/unit/kafka/controller/ControllerContextTest.scala
@@ -0,0 +1,189 @@
+/**
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *    http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+
+package unit.kafka.controller
+
+import kafka.cluster.{Broker, EndPoint}
+import kafka.controller.{ControllerContext, PartitionReplicaAssignment}
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.network.ListenerName
+import org.apache.kafka.common.security.auth.SecurityProtocol
+import org.junit.{Before, Test}
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertTrue
+import org.junit.Assert.assertFalse
+
+
+class ControllerContextTest {
+
+  var context: ControllerContext = null
+  val brokers: Seq[Int] = Seq(1, 2, 3)
+  val tp1 = new TopicPartition("A", 0)
+  val tp2 = new TopicPartition("A", 1)
+  val tp3 = new TopicPartition("B", 0)
+
+  @Before
+  def setUp(): Unit = {
+    context = new ControllerContext
+
+    val brokerEpochs = Seq(1,2,3).map { brokerId =>
+      val endpoint = new EndPoint("localhost", 9900 + brokerId, new ListenerName("PLAINTEXT"),
+        SecurityProtocol.PLAINTEXT)
+      Broker(brokerId, Seq(endpoint), rack = None) -> 1L
+    }.toMap
+
+    context.setLiveBrokerAndEpochs(brokerEpochs)
+
+    // Simple round-robin replica assignment
+    var leaderIndex = 0
+    Seq(tp1, tp2, tp3).foreach {
+      partition =>
+        val replicas = brokers.indices.map { i =>
+          val replica = brokers((i + leaderIndex) % brokers.size)
+          replica
+        }
+        context.updatePartitionReplicaAssignment(partition, replicas)
+        leaderIndex += 1
+    }
+  }
+
+  @Test
+  def testUpdatePartitionReplicaAssignmentUpdatesReplicaAssignmentOnly(): Unit = {
+    val expectedReplicas = Seq(4)
+    context.updatePartitionReplicaAssignment(tp1, expectedReplicas)
+    val assignment = context.partitionReplicaAssignment(tp1)
+    val fullAssignment = context.partitionFullReplicaAssignment(tp1)
+
+    assertEquals(expectedReplicas, assignment)
+    assertEquals(expectedReplicas, fullAssignment.replicas)
+    assertEquals(Seq(), fullAssignment.addingReplicas)
+    assertEquals(Seq(), fullAssignment.removingReplicas)
+  }
+
+  @Test
+  def testUpdatePartitionReplicaAssignmentUpdatesReplicaAssignmentOnlyAndDoesNotOverwrite(): Unit = {
+    val expectedReplicas = Seq(4)
+    val expectedFullAssignment = PartitionReplicaAssignment(Seq(3), Seq(1), Seq(2))
+    context.updatePartitionFullReplicaAssignment(tp1, expectedFullAssignment)
+
+    context.updatePartitionReplicaAssignment(tp1, expectedReplicas) // update only the replicas
+
+    val assignment = context.partitionReplicaAssignment(tp1)
+    val fullAssignment = context.partitionFullReplicaAssignment(tp1)
+    assertEquals(expectedReplicas, assignment)
+    assertEquals(expectedReplicas, fullAssignment.replicas)
+    // adding/removing replicas preserved
+    assertEquals(Seq(1), fullAssignment.addingReplicas)
+    assertEquals(Seq(2), fullAssignment.removingReplicas)
+  }
+
+  @Test
+  def testUpdatePartitionFullReplicaAssignmentUpdatesReplicaAssignment(): Unit = {
+    val initialReplicas = Seq(4)
+    context.updatePartitionReplicaAssignment(tp1, initialReplicas) // update only the replicas
+    val fullAssignment = context.partitionFullReplicaAssignment(tp1)
+    assertEquals(initialReplicas, fullAssignment.replicas)
+    assertEquals(Seq(), fullAssignment.addingReplicas)
+    assertEquals(Seq(), fullAssignment.removingReplicas)
+
+    val expectedFullAssignment = PartitionReplicaAssignment(Seq(3), Seq(1), Seq(2))
+    context.updatePartitionFullReplicaAssignment(tp1, expectedFullAssignment)
+    val updatedFullAssignment = context.partitionFullReplicaAssignment(tp1)
+    assertEquals(expectedFullAssignment.replicas, updatedFullAssignment.replicas)
+    assertEquals(expectedFullAssignment.addingReplicas, updatedFullAssignment.addingReplicas)
+    assertEquals(expectedFullAssignment.removingReplicas, updatedFullAssignment.removingReplicas)
+  }
+
+  @Test
+  def testPartitionReplicaAssignmentReturnsEmptySeqIfTopicOrPartitionDoesNotExist(): Unit = {
+    val noTopicReplicas = context.partitionReplicaAssignment(new TopicPartition("NONEXISTENT", 0))
+    assertEquals(Seq.empty, noTopicReplicas)
+    val noPartitionReplicas = context.partitionReplicaAssignment(new TopicPartition("A", 100))
+    assertEquals(Seq.empty, noPartitionReplicas)
+  }
+
+  @Test
+  def testPartitionFullReplicaAssignmentReturnsEmptyAssignmentIfTopicOrPartitionDoesNotExist(): Unit = {
+    val expectedEmptyAssignment = PartitionReplicaAssignment(Seq.empty, Seq.empty, Seq.empty)
+
+    val noTopicAssignment = context.partitionFullReplicaAssignment(new TopicPartition("NONEXISTENT", 0))
+    assertEquals(expectedEmptyAssignment, noTopicAssignment)
+    val noPartitionAssignment = context.partitionFullReplicaAssignment(new TopicPartition("A", 100))
+    assertEquals(expectedEmptyAssignment, noPartitionAssignment)
+  }
+
+  @Test
+  def testPartitionReplicaAssignmentForTopicReturnsEmptyMapIfTopicDoesNotExist(): Unit = {
+    assertEquals(Map.empty, context.partitionReplicaAssignmentForTopic("NONEXISTENT"))
+  }
+
+  @Test
+  def testPartitionReplicaAssignmentForTopicReturnsExpectedReplicaAssignments(): Unit = {
+    val expectedAssignments = Map(
+      tp1 -> context.partitionReplicaAssignment(tp1),
+      tp2 -> context.partitionReplicaAssignment(tp2)
+    )
+    val receivedAssignments = context.partitionReplicaAssignmentForTopic("A")
+    assertEquals(expectedAssignments, receivedAssignments)
+  }
+
+  @Test
+  def testPartitionReplicaAssignment(): Unit = {
+    val reassigningPartition = PartitionReplicaAssignment(List(1, 2, 3, 4, 5, 6), List(2, 3, 4), List(1, 5, 6))
+    assertTrue(reassigningPartition.isBeingReassigned)
+    assertEquals(List(2, 3, 4), reassigningPartition.targetReplicas)
+
+    val reassigningPartition2 = PartitionReplicaAssignment(List(1, 2, 3, 4), List(), List(1, 4))
+    assertTrue(reassigningPartition2.isBeingReassigned)
+    assertEquals(List(2, 3), reassigningPartition2.targetReplicas)
+
+    val reassigningPartition3 = PartitionReplicaAssignment(List(1, 2, 3, 4), List(4), List(2))
+    assertTrue(reassigningPartition3.isBeingReassigned)
+    assertEquals(List(1, 3, 4), reassigningPartition3.targetReplicas)
+
+    val partition = PartitionReplicaAssignment(List(1, 2, 3, 4, 5, 6), List(), List())
+    assertFalse(partition.isBeingReassigned)
+    assertEquals(List(1, 2, 3, 4, 5, 6), partition.targetReplicas)
+
+    val reassigningPartition4 = PartitionReplicaAssignment.fromOldAndNewReplicas(
+      List(1, 2, 3, 4), List(4, 2, 5, 3)
+    )
+    assertEquals(List(4, 2, 5, 3, 1), reassigningPartition4.replicas)
+    assertEquals(List(4, 2, 5, 3), reassigningPartition4.targetReplicas)
+    assertEquals(List(5), reassigningPartition4.addingReplicas)
+    assertEquals(List(1), reassigningPartition4.removingReplicas)
+    assertTrue(reassigningPartition4.isBeingReassigned)
+
+    val reassigningPartition5 = PartitionReplicaAssignment.fromOldAndNewReplicas(
+      List(1, 2, 3), List(4, 5, 6)
+    )
+    assertEquals(List(4, 5, 6, 1, 2, 3), reassigningPartition5.replicas)
+    assertEquals(List(4, 5, 6), reassigningPartition5.targetReplicas)
+    assertEquals(List(4, 5, 6), reassigningPartition5.addingReplicas)
+    assertEquals(List(1, 2, 3), reassigningPartition5.removingReplicas)
+    assertTrue(reassigningPartition5.isBeingReassigned)
+
+    val nonReassigningPartition = PartitionReplicaAssignment.fromOldAndNewReplicas(
+      List(1, 2, 3), List(3, 1, 2)
+    )
+    assertEquals(List(3, 1, 2), nonReassigningPartition.replicas)
+    assertEquals(List(3, 1, 2), nonReassigningPartition.targetReplicas)
+    assertEquals(List(), nonReassigningPartition.addingReplicas)
+    assertEquals(List(), nonReassigningPartition.removingReplicas)
+    assertFalse(nonReassigningPartition.isBeingReassigned)
+  }
+}
diff --git a/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala b/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala
index 70d26b8a49803..867f4c6fa1511 100644
--- a/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala
@@ -249,7 +249,9 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     val tp0 = new TopicPartition("t", 0)
     val tp1 = new TopicPartition("t", 1)
     val assignment = Map(tp0.partition -> Seq(0))
-    val expandedAssignment = Map(tp0 -> Seq(0), tp1 -> Seq(0))
+    val expandedAssignment = Map(
+      tp0 -> PartitionReplicaAssignment(Seq(0), Seq(), Seq()),
+      tp1 -> PartitionReplicaAssignment(Seq(0), Seq(), Seq()))
     TestUtils.createTopic(zkClient, tp0.topic, partitionReplicaAssignment = assignment, servers = servers)
     zkClient.setTopicAssignment(tp0.topic, expandedAssignment, firstControllerEpochZkVersion)
     waitForPartitionState(tp1, firstControllerEpoch, 0, LeaderAndIsr.initialLeaderEpoch,
@@ -265,7 +267,9 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     val tp0 = new TopicPartition("t", 0)
     val tp1 = new TopicPartition("t", 1)
     val assignment = Map(tp0.partition -> Seq(otherBrokerId, controllerId))
-    val expandedAssignment = Map(tp0 -> Seq(otherBrokerId, controllerId), tp1 -> Seq(otherBrokerId, controllerId))
+    val expandedAssignment = Map(
+      tp0 -> PartitionReplicaAssignment(Seq(otherBrokerId, controllerId), Seq(), Seq()),
+      tp1 -> PartitionReplicaAssignment(Seq(otherBrokerId, controllerId), Seq(), Seq()))
     TestUtils.createTopic(zkClient, tp0.topic, partitionReplicaAssignment = assignment, servers = servers)
     servers(otherBrokerId).shutdown()
     servers(otherBrokerId).awaitShutdown()
@@ -280,18 +284,18 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     servers = makeServers(2)
     val controllerId = TestUtils.waitUntilControllerElected(zkClient)
 
-    val metricName = s"kafka.controller:type=ControllerStats,name=${ControllerState.PartitionReassignment.rateAndTimeMetricName.get}"
+    val metricName = s"kafka.controller:type=ControllerStats,name=${ControllerState.AlterPartitionReassignment.rateAndTimeMetricName.get}"
     val timerCount = timer(metricName).count
 
     val otherBrokerId = servers.map(_.config.brokerId).filter(_ != controllerId).head
     val tp = new TopicPartition("t", 0)
     val assignment = Map(tp.partition -> Seq(controllerId))
-    val reassignment = Map(tp -> Seq(otherBrokerId))
+    val reassignment = Map(tp -> PartitionReplicaAssignment(Seq(otherBrokerId), List(), List()))
     TestUtils.createTopic(zkClient, tp.topic, partitionReplicaAssignment = assignment, servers = servers)
-    zkClient.createPartitionReassignment(reassignment)
+    zkClient.createPartitionReassignment(reassignment.mapValues(_.replicas).toMap)
     waitForPartitionState(tp, firstControllerEpoch, otherBrokerId, LeaderAndIsr.initialLeaderEpoch + 3,
       "failed to get expected partition state after partition reassignment")
-    TestUtils.waitUntilTrue(() =>  zkClient.getReplicaAssignmentForTopics(Set(tp.topic)) == reassignment,
+    TestUtils.waitUntilTrue(() =>  zkClient.getFullReplicaAssignmentForTopics(Set(tp.topic)) == reassignment,
       "failed to get updated partition assignment on topic znode after partition reassignment")
     TestUtils.waitUntilTrue(() => !zkClient.reassignPartitionsInProgress(),
       "failed to remove reassign partitions path after completion")
@@ -326,17 +330,17 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     val otherBrokerId = servers.map(_.config.brokerId).filter(_ != controllerId).head
     val tp = new TopicPartition("t", 0)
     val assignment = Map(tp.partition -> Seq(controllerId))
-    val reassignment = Map(tp -> Seq(otherBrokerId))
+    val reassignment = Map(tp -> PartitionReplicaAssignment(Seq(otherBrokerId), List(), List()))
     TestUtils.createTopic(zkClient, tp.topic, partitionReplicaAssignment = assignment, servers = servers)
     servers(otherBrokerId).shutdown()
     servers(otherBrokerId).awaitShutdown()
-    zkClient.createPartitionReassignment(reassignment)
+    zkClient.createPartitionReassignment(reassignment.mapValues(_.replicas).toMap)
     waitForPartitionState(tp, firstControllerEpoch, controllerId, LeaderAndIsr.initialLeaderEpoch + 1,
       "failed to get expected partition state during partition reassignment with offline replica")
     servers(otherBrokerId).startup()
     waitForPartitionState(tp, firstControllerEpoch, otherBrokerId, LeaderAndIsr.initialLeaderEpoch + 4,
       "failed to get expected partition state after partition reassignment")
-    TestUtils.waitUntilTrue(() => zkClient.getReplicaAssignmentForTopics(Set(tp.topic)) == reassignment,
+    TestUtils.waitUntilTrue(() => zkClient.getFullReplicaAssignmentForTopics(Set(tp.topic)) == reassignment,
       "failed to get updated partition assignment on topic znode after partition reassignment")
     TestUtils.waitUntilTrue(() => !zkClient.reassignPartitionsInProgress(),
       "failed to remove reassign partitions path after completion")
diff --git a/core/src/test/scala/unit/kafka/controller/PartitionStateMachineTest.scala b/core/src/test/scala/unit/kafka/controller/PartitionStateMachineTest.scala
index 9ca3bd273be53..99dc6ed50e37a 100644
--- a/core/src/test/scala/unit/kafka/controller/PartitionStateMachineTest.scala
+++ b/core/src/test/scala/unit/kafka/controller/PartitionStateMachineTest.scala
@@ -89,7 +89,7 @@ class PartitionStateMachineTest {
     EasyMock.expect(mockZkClient.createTopicPartitionStatesRaw(Map(partition -> leaderIsrAndControllerEpoch), controllerContext.epochZkVersion))
       .andReturn(Seq(CreateResponse(Code.OK, null, Some(partition), null, ResponseMetadata(0, 0))))
     EasyMock.expect(mockControllerBrokerRequestBatch.addLeaderAndIsrRequestForBrokers(Seq(brokerId),
-      partition, leaderIsrAndControllerEpoch, Seq(brokerId), isNew = true))
+      partition, leaderIsrAndControllerEpoch, replicaAssignment(Seq(brokerId)), isNew = true))
     EasyMock.expect(mockControllerBrokerRequestBatch.sendRequestsToBrokers(controllerEpoch))
     EasyMock.replay(mockZkClient, mockControllerBrokerRequestBatch)
     partitionStateMachine.handleStateChanges(
@@ -175,7 +175,7 @@ class PartitionStateMachineTest {
     EasyMock.expect(mockZkClient.updateLeaderAndIsr(Map(partition -> leaderAndIsrAfterElection), controllerEpoch, controllerContext.epochZkVersion))
       .andReturn(UpdateLeaderAndIsrResult(Map(partition -> Right(updatedLeaderAndIsr)), Seq.empty))
     EasyMock.expect(mockControllerBrokerRequestBatch.addLeaderAndIsrRequestForBrokers(Seq(brokerId),
-      partition, LeaderIsrAndControllerEpoch(updatedLeaderAndIsr, controllerEpoch), Seq(brokerId), isNew = false))
+      partition, LeaderIsrAndControllerEpoch(updatedLeaderAndIsr, controllerEpoch), replicaAssignment(Seq(brokerId)), isNew = false))
     EasyMock.expect(mockControllerBrokerRequestBatch.sendRequestsToBrokers(controllerEpoch))
     EasyMock.replay(mockZkClient, mockControllerBrokerRequestBatch)
 
@@ -210,7 +210,7 @@ class PartitionStateMachineTest {
 
     // The leaderAndIsr request should be sent to both brokers, including the shutting down one
     EasyMock.expect(mockControllerBrokerRequestBatch.addLeaderAndIsrRequestForBrokers(Seq(brokerId, otherBrokerId),
-      partition, LeaderIsrAndControllerEpoch(updatedLeaderAndIsr, controllerEpoch), Seq(brokerId, otherBrokerId),
+      partition, LeaderIsrAndControllerEpoch(updatedLeaderAndIsr, controllerEpoch), replicaAssignment(Seq(brokerId, otherBrokerId)),
       isNew = false))
     EasyMock.expect(mockControllerBrokerRequestBatch.sendRequestsToBrokers(controllerEpoch))
     EasyMock.replay(mockZkClient, mockControllerBrokerRequestBatch)
@@ -263,7 +263,7 @@ class PartitionStateMachineTest {
     EasyMock.expect(mockZkClient.updateLeaderAndIsr(Map(partition -> leaderAndIsrAfterElection), controllerEpoch, controllerContext.epochZkVersion))
       .andReturn(UpdateLeaderAndIsrResult(Map(partition -> Right(updatedLeaderAndIsr)), Seq.empty))
     EasyMock.expect(mockControllerBrokerRequestBatch.addLeaderAndIsrRequestForBrokers(Seq(brokerId),
-      partition, LeaderIsrAndControllerEpoch(updatedLeaderAndIsr, controllerEpoch), Seq(brokerId), isNew = false))
+      partition, LeaderIsrAndControllerEpoch(updatedLeaderAndIsr, controllerEpoch), replicaAssignment(Seq(brokerId)), isNew = false))
     EasyMock.expect(mockControllerBrokerRequestBatch.sendRequestsToBrokers(controllerEpoch))
     EasyMock.replay(mockZkClient, mockControllerBrokerRequestBatch)
 
@@ -327,7 +327,7 @@ class PartitionStateMachineTest {
         Seq(brokerId),
         partition,
         LeaderIsrAndControllerEpoch(updatedLeaderAndIsr, controllerEpoch),
-        Seq(leaderBrokerId, brokerId),
+        replicaAssignment(Seq(leaderBrokerId, brokerId)),
         false
       )
     )
@@ -516,4 +516,7 @@ class PartitionStateMachineTest {
     topicDeletionManager.enqueueTopicsForDeletion(Set(topic))
     assertEquals(s"There should be no offline partition(s)", 0, controllerContext.offlinePartitionCount)
   }
+
+  private def replicaAssignment(replicas: Seq[Int]): PartitionReplicaAssignment = PartitionReplicaAssignment(replicas, Seq(), Seq())
+
 }
diff --git a/core/src/test/scala/unit/kafka/controller/ReplicaStateMachineTest.scala b/core/src/test/scala/unit/kafka/controller/ReplicaStateMachineTest.scala
index 6afa5b64974e9..c43fd0902f0a7 100644
--- a/core/src/test/scala/unit/kafka/controller/ReplicaStateMachineTest.scala
+++ b/core/src/test/scala/unit/kafka/controller/ReplicaStateMachineTest.scala
@@ -194,7 +194,7 @@ class ReplicaStateMachineTest {
     controllerContext.partitionLeadershipInfo.put(partition, leaderIsrAndControllerEpoch)
     EasyMock.expect(mockControllerBrokerRequestBatch.newBatch())
     EasyMock.expect(mockControllerBrokerRequestBatch.addLeaderAndIsrRequestForBrokers(Seq(brokerId),
-      partition, leaderIsrAndControllerEpoch, Seq(brokerId), isNew = false))
+      partition, leaderIsrAndControllerEpoch, replicaAssignment(Seq(brokerId)), isNew = false))
     EasyMock.expect(mockControllerBrokerRequestBatch.sendRequestsToBrokers(controllerEpoch))
     EasyMock.replay(mockZkClient, mockControllerBrokerRequestBatch)
     replicaStateMachine.handleStateChanges(replicas, OnlineReplica)
@@ -224,7 +224,7 @@ class ReplicaStateMachineTest {
     EasyMock.expect(mockZkClient.updateLeaderAndIsr(Map(partition -> adjustedLeaderAndIsr), controllerEpoch, controllerContext.epochZkVersion))
       .andReturn(UpdateLeaderAndIsrResult(Map(partition -> Right(updatedLeaderAndIsr)), Seq.empty))
     EasyMock.expect(mockControllerBrokerRequestBatch.addLeaderAndIsrRequestForBrokers(Seq(otherBrokerId),
-      partition, updatedLeaderIsrAndControllerEpoch, replicaIds, isNew = false))
+      partition, updatedLeaderIsrAndControllerEpoch, replicaAssignment(replicaIds), isNew = false))
     EasyMock.expect(mockControllerBrokerRequestBatch.sendRequestsToBrokers(controllerEpoch))
 
     EasyMock.replay(mockZkClient, mockControllerBrokerRequestBatch)
@@ -267,7 +267,7 @@ class ReplicaStateMachineTest {
     controllerContext.partitionLeadershipInfo.put(partition, leaderIsrAndControllerEpoch)
     EasyMock.expect(mockControllerBrokerRequestBatch.newBatch())
     EasyMock.expect(mockControllerBrokerRequestBatch.addLeaderAndIsrRequestForBrokers(Seq(brokerId),
-      partition, leaderIsrAndControllerEpoch, Seq(brokerId), isNew = false))
+      partition, leaderIsrAndControllerEpoch, replicaAssignment(Seq(brokerId)), isNew = false))
     EasyMock.expect(mockControllerBrokerRequestBatch.sendRequestsToBrokers(controllerEpoch))
     EasyMock.replay(mockZkClient, mockControllerBrokerRequestBatch)
     replicaStateMachine.handleStateChanges(replicas, OnlineReplica)
@@ -385,7 +385,7 @@ class ReplicaStateMachineTest {
     controllerContext.partitionLeadershipInfo.put(partition, leaderIsrAndControllerEpoch)
     EasyMock.expect(mockControllerBrokerRequestBatch.newBatch())
     EasyMock.expect(mockControllerBrokerRequestBatch.addLeaderAndIsrRequestForBrokers(Seq(brokerId),
-      partition, leaderIsrAndControllerEpoch, Seq(brokerId), isNew = false))
+      partition, leaderIsrAndControllerEpoch, replicaAssignment(Seq(brokerId)), isNew = false))
     EasyMock.expect(mockControllerBrokerRequestBatch.sendRequestsToBrokers(controllerEpoch))
     EasyMock.replay(mockZkClient, mockControllerBrokerRequestBatch)
     replicaStateMachine.handleStateChanges(replicas, OnlineReplica)
@@ -408,4 +408,7 @@ class ReplicaStateMachineTest {
     replicaStateMachine.handleStateChanges(replicas, toState)
     assertEquals(fromState, replicaState(replica))
   }
+
+  private def replicaAssignment(replicas: Seq[Int]): PartitionReplicaAssignment = PartitionReplicaAssignment(replicas, Seq(), Seq())
+
 }
diff --git a/core/src/test/scala/unit/kafka/security/auth/ZkAuthorizationTest.scala b/core/src/test/scala/unit/kafka/security/auth/ZkAuthorizationTest.scala
index 429e1899a67cb..f1647ce4a820e 100644
--- a/core/src/test/scala/unit/kafka/security/auth/ZkAuthorizationTest.scala
+++ b/core/src/test/scala/unit/kafka/security/auth/ZkAuthorizationTest.scala
@@ -32,6 +32,7 @@ import scala.util.{Failure, Success, Try}
 import javax.security.auth.login.Configuration
 import kafka.api.ApiVersion
 import kafka.cluster.{Broker, EndPoint}
+import kafka.controller.PartitionReplicaAssignment
 import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.security.auth.SecurityProtocol
 import org.apache.kafka.common.utils.Time
@@ -130,7 +131,7 @@ class ZkAuthorizationTest extends ZooKeeperTestHarness with Logging {
 
     // Test that can update persistent nodes
     val updatedAssignment = assignment - new TopicPartition(topic1, 2)
-    zkClient.setTopicAssignment(topic1, updatedAssignment)
+    zkClient.setTopicAssignment(topic1, updatedAssignment.mapValues { case (v) => PartitionReplicaAssignment(v, List(), List()) }.toMap)
     assertEquals(updatedAssignment.size, zkClient.getTopicPartitionCount(topic1).get)
   }
 
diff --git a/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala b/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala
index 7e2416ebda6e8..07405f77f3982 100644
--- a/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala
+++ b/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala
@@ -19,6 +19,7 @@ package kafka.admin
 import java.util
 import java.util.Properties
 
+import kafka.controller.PartitionReplicaAssignment
 import kafka.log._
 import kafka.server.DynamicConfig.Broker._
 import kafka.server.KafkaConfig._
@@ -87,7 +88,7 @@ class AdminZkClientTest extends ZooKeeperTestHarness with Logging with RackAware
                          1 -> List(1, 2, 3))
     adminZkClient.createTopicWithAssignment("test", topicConfig, assignment)
     val found = zkClient.getPartitionAssignmentForTopics(Set("test"))
-    assertEquals(assignment, found("test"))
+    assertEquals(assignment.mapValues(PartitionReplicaAssignment(_, List(), List())).toMap, found("test"))
   }
 
   @Test
@@ -179,8 +180,9 @@ class AdminZkClientTest extends ZooKeeperTestHarness with Logging with RackAware
       catch { case _: TopicExistsException => () }
       val (_, partitionAssignment) = zkClient.getPartitionAssignmentForTopics(Set(topic)).head
       assertEquals(3, partitionAssignment.size)
-      partitionAssignment.foreach { case (partition, replicas) =>
-        assertEquals(s"Unexpected replication factor for $partition", 1, replicas.size)
+      partitionAssignment.foreach { case (partition, partitionReplicaAssignment) =>
+        assertEquals(s"Unexpected replication factor for $partition",
+          1, partitionReplicaAssignment.replicas.size)
       }
       val savedProps = zkClient.getEntityConfigs(ConfigType.Topic, topic)
       assertEquals(props, savedProps)
diff --git a/core/src/test/scala/unit/kafka/zk/KafkaZkClientTest.scala b/core/src/test/scala/unit/kafka/zk/KafkaZkClientTest.scala
index ba8dd7e54dda4..f9961d886bc2a 100644
--- a/core/src/test/scala/unit/kafka/zk/KafkaZkClientTest.scala
+++ b/core/src/test/scala/unit/kafka/zk/KafkaZkClientTest.scala
@@ -40,7 +40,7 @@ import scala.collection.JavaConverters._
 import scala.collection.mutable.ArrayBuffer
 import scala.collection.{Seq, mutable}
 import scala.util.Random
-import kafka.controller.LeaderIsrAndControllerEpoch
+import kafka.controller.{LeaderIsrAndControllerEpoch, PartitionReplicaAssignment}
 import kafka.zk.KafkaZkClient.UpdateLeaderAndIsrResult
 import kafka.zookeeper._
 import org.apache.kafka.common.errors.ControllerMovedException
@@ -169,7 +169,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
     val expectedAssignment = assignment map { topicAssignment =>
       val partition = topicAssignment._1.partition
       val assignment = topicAssignment._2
-      partition -> assignment
+      partition -> PartitionReplicaAssignment(assignment, List(), List())
     }
 
     assertEquals(assignment.size, zkClient.getTopicPartitionCount(topic1).get)
@@ -179,7 +179,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
 
     val updatedAssignment = assignment - new TopicPartition(topic1, 2)
 
-    zkClient.setTopicAssignment(topic1, updatedAssignment)
+    zkClient.setTopicAssignment(topic1, updatedAssignment.mapValues { case v => PartitionReplicaAssignment(v, List(), List()) }.toMap)
     assertEquals(updatedAssignment.size, zkClient.getTopicPartitionCount(topic1).get)
 
     // add second topic
diff --git a/docs/upgrade.html b/docs/upgrade.html
index eed3ad803e8f9..f95adfcf00fdf 100644
--- a/docs/upgrade.html
+++ b/docs/upgrade.html
@@ -21,6 +21,8 @@
 
 
            Notable changes in 2.4.0
            
 
              
+    
            • A new Admin API has been added for partition reassignments. Due to changing the way Kafka propagates reassignment information,
+        it is possible to lose reassignment state in failure edge cases while upgrading to the new version. It is not recommended to start reassignments while upgrading.
              • 
     
            • ZooKeeper has been upgraded from 3.4.14 to 3.5.5. TLS and dynamic reconfiguration are supported by the new version.
              • 
     
            • The bin/kafka-preferred-replica-election.sh command line tool has been deprecated. It has been replaced by bin/kafka-leader-election.sh.
              • 
     
            • The methods electPreferredLeaders in the Java AdminClient class have been deprecated in favor of the methods electLeaders.
              • 

From 6882b3b760de3208ee0cca86d6e1a651cc5e5a1f Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Wed, 11 Sep 2019 16:56:56 +0100
Subject: [PATCH 0605/1071] KAFKA-8886; Make Authorizer create/delete API
 asynchronous (#7316)

Reviewers: Manikumar Reddy 
---
 .../kafka/server/authorizer/Authorizer.java   | 46 +++++++++--
 .../main/scala/kafka/admin/AclCommand.scala   |  4 +-
 .../security/auth/SimpleAclAuthorizer.scala   | 18 ++++-
 .../security/authorizer/AclAuthorizer.scala   | 69 +++++++++++------
 .../authorizer/AuthorizerWrapper.scala        | 12 +--
 .../main/scala/kafka/server/KafkaApis.scala   |  4 +-
 .../main/scala/kafka/server/KafkaServer.scala |  2 +-
 .../kafka/api/AuthorizerIntegrationTest.scala |  4 +-
 .../DescribeAuthorizedOperationsTest.scala    |  2 +-
 .../api/SslAdminClientIntegrationTest.scala   | 77 ++++++++++++++++++-
 .../authorizer/AclAuthorizerTest.scala        | 41 ++++++----
 .../DelegationTokenManagerTest.scala          |  2 +-
 .../server/DynamicBrokerConfigTest.scala      | 12 +--
 13 files changed, 218 insertions(+), 75 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/server/authorizer/Authorizer.java b/clients/src/main/java/org/apache/kafka/server/authorizer/Authorizer.java
index 654bcd04912d7..45bd6d939928d 100644
--- a/clients/src/main/java/org/apache/kafka/server/authorizer/Authorizer.java
+++ b/clients/src/main/java/org/apache/kafka/server/authorizer/Authorizer.java
@@ -20,7 +20,8 @@
 import java.io.Closeable;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionStage;
+
 import org.apache.kafka.common.Configurable;
 import org.apache.kafka.common.Endpoint;
 import org.apache.kafka.common.acl.AclBinding;
@@ -47,7 +48,18 @@
  * Authorizer implementation class may optionally implement @{@link org.apache.kafka.common.Reconfigurable}
  * to enable dynamic reconfiguration without restarting the broker.
  * 
              
- * Thread safety: All authorizer operations including authorization and ACL updates must be thread-safe.
+ * Threading model:
+ * 
                
+ *   
              • All authorizer operations including authorization and ACL updates must be thread-safe.
                • 
+ *   
              • ACL update methods are asynchronous. Implementations with low update latency may return a
+ *       completed future using {@link java.util.concurrent.CompletableFuture#completedFuture(Object)}.
+ *       This ensures that the request will be handled synchronously by the caller without using a
+ *       purgatory to wait for the result. If ACL updates require remote communication which may block,
+ *       return a future that is completed asynchronously when the remote operation completes. This enables
+ *       the caller to process other requests on the request threads without blocking.
                • 
+ *   
              • Any threads or thread pools used for processing remote operations asynchronously can be started during
+ *       {@link #start(AuthorizerServerInfo)}. These threads must be shutdown during {@link Authorizer#close()}.
                • 
+ * 
              
  * 
              
  */
 @InterfaceStability.Evolving
@@ -60,14 +72,18 @@ public interface Authorizer extends Configurable, Closeable {
      * requests on that listener.
      *
      * @param serverInfo Metadata for the broker including broker id and listener endpoints
-     * @return CompletableFutures for each endpoint that completes when authorizer is ready to
+     * @return CompletionStage for each endpoint that completes when authorizer is ready to
      *         start authorizing requests on that listener.
      */
-    Map> start(AuthorizerServerInfo serverInfo);
+    Map> start(AuthorizerServerInfo serverInfo);
 
     /**
      * Authorizes the specified action. Additional metadata for the action is specified
      * in `requestContext`.
+     * 
              
+     * This is a synchronous API designed for use with locally cached ACLs. Since this method is invoked on the
+     * request thread while processing each request, implementations of this method should avoid time-consuming
+     * remote communication that may block request threads.
      *
      * @param requestContext Request context including request type, security protocol and listener name
      * @param actions Actions being authorized including resource and operation for each action
@@ -77,18 +93,27 @@ public interface Authorizer extends Configurable, Closeable {
 
     /**
      * Creates new ACL bindings.
+     * 
              
+     * This is an asynchronous API that enables the caller to avoid blocking during the update. Implementations of this
+     * API can return completed futures using {@link java.util.concurrent.CompletableFuture#completedFuture(Object)}
+     * to process the update synchronously on the request thread.
      *
      * @param requestContext Request context if the ACL is being created by a broker to handle
      *        a client request to create ACLs. This may be null if ACLs are created directly in ZooKeeper
      *        using AclCommand.
      * @param aclBindings ACL bindings to create
      *
-     * @return Create result for each ACL binding in the same order as in the input list
+     * @return Create result for each ACL binding in the same order as in the input list. Each result
+     *         is returned as a CompletionStage that completes when the result is available.
      */
-    List createAcls(AuthorizableRequestContext requestContext, List aclBindings);
+    List> createAcls(AuthorizableRequestContext requestContext, List aclBindings);
 
     /**
      * Deletes all ACL bindings that match the provided filters.
+     * 
              
+     * This is an asynchronous API that enables the caller to avoid blocking during the update. Implementations of this
+     * API can return completed futures using {@link java.util.concurrent.CompletableFuture#completedFuture(Object)}
+     * to process the update synchronously on the request thread.
      *
      * @param requestContext Request context if the ACL is being deleted by a broker to handle
      *        a client request to delete ACLs. This may be null if ACLs are deleted directly in ZooKeeper
@@ -97,12 +122,17 @@ public interface Authorizer extends Configurable, Closeable {
      *
      * @return Delete result for each filter in the same order as in the input list.
      *         Each result indicates which ACL bindings were actually deleted as well as any
-     *         bindings that matched but could not be deleted.
+     *         bindings that matched but could not be deleted. Each result is returned as a
+     *         CompletionStage that completes when the result is available.
      */
-    List deleteAcls(AuthorizableRequestContext requestContext, List aclBindingFilters);
+    List> deleteAcls(AuthorizableRequestContext requestContext, List aclBindingFilters);
 
     /**
      * Returns ACL bindings which match the provided filter.
+     * 
              
+     * This is a synchronous API designed for use with locally cached ACLs. This method is invoked on the request
+     * thread while processing DescribeAcls requests and should avoid time-consuming remote communication that may
+     * block request threads.
      *
      * @return Iterator for ACL bindings, which may be populated lazily.
      */
diff --git a/core/src/main/scala/kafka/admin/AclCommand.scala b/core/src/main/scala/kafka/admin/AclCommand.scala
index 090ac7fab6f5a..0ed0703a95698 100644
--- a/core/src/main/scala/kafka/admin/AclCommand.scala
+++ b/core/src/main/scala/kafka/admin/AclCommand.scala
@@ -315,7 +315,7 @@ object AclCommand extends Logging {
         for ((resource, acls) <- resourceToAcl) {
           println(s"Adding ACLs for resource `$resource`: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline")
           val aclBindings = acls.map(acl => new AclBinding(resource, acl))
-          authorizer.createAcls(null, aclBindings.toList.asJava).asScala.foreach { result =>
+          authorizer.createAcls(null,aclBindings.toList.asJava).asScala.map(_.toCompletableFuture.get).foreach { result =>
             result.exception.asScala.foreach { exception =>
               println(s"Error while adding ACLs: ${exception.getMessage}")
               println(Utils.stackTrace(exception))
@@ -374,7 +374,7 @@ object AclCommand extends Logging {
         val aclBindingFilters = acls.map(acl => new AclBindingFilter(filter, acl.toFilter)).toList.asJava
         authorizer.deleteAcls(null, aclBindingFilters)
       }
-      result.asScala.foreach { result =>
+      result.asScala.map(_.toCompletableFuture.get).foreach { result =>
         result.exception.asScala.foreach { exception =>
           println(s"Error while removing ACLs: ${exception.getMessage}")
           println(Utils.stackTrace(exception))
diff --git a/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala b/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala
index 6e04021c80281..222fbbcf5b802 100644
--- a/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala
+++ b/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala
@@ -23,6 +23,7 @@ import kafka.security.authorizer.{AclAuthorizer, AuthorizerUtils}
 import kafka.utils._
 import kafka.zk.ZkVersion
 import org.apache.kafka.common.acl.{AccessControlEntryFilter, AclBinding, AclBindingFilter, AclOperation, AclPermissionType}
+import org.apache.kafka.common.errors.ApiException
 import org.apache.kafka.common.resource.{PatternType, ResourcePatternFilter}
 import org.apache.kafka.common.security.auth.KafkaPrincipal
 import org.apache.kafka.server.authorizer.{Action, AuthorizableRequestContext, AuthorizationResult}
@@ -125,14 +126,14 @@ class SimpleAclAuthorizer extends Authorizer with Logging {
 
   private def createAcls(bindings: Set[AclBinding]): Unit = {
     aclAuthorizer.maxUpdateRetries = maxUpdateRetries
-    val results = aclAuthorizer.createAcls(null, bindings.toList.asJava).asScala
-    results.foreach { result => result.exception.asScala.foreach(e => throw e) }
+    val results = aclAuthorizer.createAcls(null, bindings.toList.asJava).asScala.map(_.toCompletableFuture.get)
+    results.foreach { result => result.exception.asScala.foreach(throwException) }
   }
 
   private def deleteAcls(filters: Set[AclBindingFilter]): Boolean = {
     aclAuthorizer.maxUpdateRetries = maxUpdateRetries
-    val results = aclAuthorizer.deleteAcls(null, filters.toList.asJava).asScala
-    results.foreach { result => result.exception.asScala.foreach(e => throw e) }
+    val results = aclAuthorizer.deleteAcls(null, filters.toList.asJava).asScala.map(_.toCompletableFuture.get)
+    results.foreach { result => result.exception.asScala.foreach(throwException) }
     results.flatMap(_.aclBindingDeleteResults.asScala).foreach { result => result.exception.asScala.foreach(e => throw e) }
     results.exists(r => r.aclBindingDeleteResults.asScala.exists(d => !d.exception.isPresent))
   }
@@ -147,6 +148,15 @@ class SimpleAclAuthorizer extends Authorizer with Logging {
     result.mapValues(_.toSet).toMap
   }
 
+  // To retain the same exceptions as in previous versions, throw the underlying exception when the exception
+  // was wrapped by AclAuthorizer in an ApiException
+  private def throwException(e: ApiException): Unit = {
+    if (e.getCause != null)
+      throw e.getCause
+    else
+      throw e
+  }
+
   class BaseAuthorizer extends AclAuthorizer {
     override def logAuditMessage(requestContext: AuthorizableRequestContext, action: Action, authorized: Boolean): Unit = {
       val principal = requestContext.principal
diff --git a/core/src/main/scala/kafka/security/authorizer/AclAuthorizer.scala b/core/src/main/scala/kafka/security/authorizer/AclAuthorizer.scala
index 5d1880b41374b..84a641d60dd7e 100644
--- a/core/src/main/scala/kafka/security/authorizer/AclAuthorizer.scala
+++ b/core/src/main/scala/kafka/security/authorizer/AclAuthorizer.scala
@@ -17,7 +17,7 @@
 package kafka.security.authorizer
 
 import java.{lang, util}
-import java.util.concurrent.CompletableFuture
+import java.util.concurrent.{CompletableFuture, CompletionStage}
 import java.util.concurrent.locks.ReentrantReadWriteLock
 
 import com.typesafe.scalalogging.Logger
@@ -118,7 +118,7 @@ class AclAuthorizer extends Authorizer with Logging {
     loadCache()
   }
 
-  override def start(serverInfo: AuthorizerServerInfo): util.Map[Endpoint, CompletableFuture[Void]] = {
+  override def start(serverInfo: AuthorizerServerInfo): util.Map[Endpoint, _ <: CompletionStage[Void]] = {
     serverInfo.endpoints.asScala.map { endpoint =>
       endpoint -> CompletableFuture.completedFuture[Void](null) }.toMap.asJava
   }
@@ -127,7 +127,8 @@ class AclAuthorizer extends Authorizer with Logging {
     actions.asScala.map { action => authorizeAction(requestContext, action) }.asJava
   }
 
-  override def createAcls(requestContext: AuthorizableRequestContext, aclBindings: util.List[AclBinding]): util.List[AclCreateResult] = {
+  override def createAcls(requestContext: AuthorizableRequestContext,
+                          aclBindings: util.List[AclBinding]): util.List[_ <: CompletionStage[AclCreateResult]] = {
     val results = new Array[AclCreateResult](aclBindings.size)
     val aclsToCreate = aclBindings.asScala.zipWithIndex
       .filter { case (aclBinding, i) =>
@@ -139,11 +140,8 @@ class AclAuthorizer extends Authorizer with Logging {
           AuthorizerUtils.validateAclBinding(aclBinding)
           true
         } catch {
-          case e: ApiException =>
-            results(i) = new AclCreateResult(e)
-            false
           case e: Throwable =>
-            results(i) = new AclCreateResult(new InvalidRequestException("Failed to create ACL", e))
+            results(i) = new AclCreateResult(new InvalidRequestException("Failed to create ACL", apiException(e)))
             false
         }
       }.groupBy(_._1.pattern)
@@ -151,19 +149,26 @@ class AclAuthorizer extends Authorizer with Logging {
     if (aclsToCreate.nonEmpty) {
       inWriteLock(lock) {
         aclsToCreate.foreach { case (resource, aclsWithIndex) =>
-          updateResourceAcls(AuthorizerUtils.convertToResource(resource)) { currentAcls =>
-            val newAcls = aclsWithIndex.map { case (acl, index) => AuthorizerUtils.convertToAcl(acl.entry) }
-            currentAcls ++ newAcls
+          try {
+            updateResourceAcls(AuthorizerUtils.convertToResource(resource)) { currentAcls =>
+              val newAcls = aclsWithIndex.map { case (acl, index) => AuthorizerUtils.convertToAcl(acl.entry) }
+              currentAcls ++ newAcls
+            }
+            aclsWithIndex.foreach { case (_, index) => results(index) = AclCreateResult.SUCCESS }
+          } catch {
+            case e: Throwable =>
+              aclsWithIndex.foreach { case (_, index) => results(index) = new AclCreateResult(apiException(e)) }
           }
-          aclsWithIndex.foreach { case (_, index) => results(index) = AclCreateResult.SUCCESS }
         }
       }
     }
-    results.toList.asJava
+    results.toList.map(CompletableFuture.completedFuture[AclCreateResult]).asJava
   }
 
-  override def deleteAcls(requestContext: AuthorizableRequestContext, aclBindingFilters: util.List[AclBindingFilter]): util.List[AclDeleteResult] = {
+  override def deleteAcls(requestContext: AuthorizableRequestContext,
+                          aclBindingFilters: util.List[AclBindingFilter]): util.List[_ <: CompletionStage[AclDeleteResult]] = {
     val deletedBindings = new mutable.HashMap[AclBinding, Int]()
+    val deleteExceptions = new mutable.HashMap[AclBinding, ApiException]()
     val filters = aclBindingFilters.asScala.zipWithIndex
     inWriteLock(lock) {
       val resourcesToUpdate = aclCache.keys.map { resource =>
@@ -174,24 +179,35 @@ class AclAuthorizer extends Authorizer with Logging {
       }.toMap.filter(_._2.nonEmpty)
 
       resourcesToUpdate.foreach { case (resource, matchingFilters) =>
-        updateResourceAcls(resource) { currentAcls =>
-          val aclsToRemove = currentAcls.filter { acl =>
-            matchingFilters.exists { case (filter, index) =>
-              val matches = filter.entryFilter.matches(AuthorizerUtils.convertToAccessControlEntry(acl))
-              if (matches)
-                deletedBindings.getOrElseUpdate(AuthorizerUtils.convertToAclBinding(resource, acl), index)
-              matches
+        val resourceBindingsBeingDeleted = new mutable.HashMap[AclBinding, Int]()
+        try {
+          updateResourceAcls(resource) { currentAcls =>
+            val aclsToRemove = currentAcls.filter { acl =>
+              matchingFilters.exists { case (filter, index) =>
+                val matches = filter.entryFilter.matches(AuthorizerUtils.convertToAccessControlEntry(acl))
+                if (matches) {
+                  val binding = AuthorizerUtils.convertToAclBinding(resource, acl)
+                  deletedBindings.getOrElseUpdate(binding, index)
+                  resourceBindingsBeingDeleted.getOrElseUpdate(binding, index)
+                }
+                matches
+              }
             }
+            currentAcls -- aclsToRemove
           }
-          currentAcls -- aclsToRemove
+        } catch {
+          case e: Exception =>
+            resourceBindingsBeingDeleted.foreach { case (binding, index) =>
+                deleteExceptions.getOrElseUpdate(binding, apiException(e))
+            }
         }
       }
     }
     val deletedResult = deletedBindings.groupBy(_._2)
-      .mapValues(_.map{ case (binding, _) => new AclBindingDeleteResult(binding) })
+      .mapValues(_.map{ case (binding, _) => new AclBindingDeleteResult(binding, deleteExceptions.getOrElse(binding, null)) })
     (0 until aclBindingFilters.size).map { i =>
       new AclDeleteResult(deletedResult.getOrElse(i, Set.empty[AclBindingDeleteResult]).toSet.asJava)
-    }.asJava
+    }.map(CompletableFuture.completedFuture[AclDeleteResult]).asJava
   }
 
   override def acls(filter: AclBindingFilter): lang.Iterable[AclBinding] = {
@@ -448,6 +464,13 @@ class AclAuthorizer extends Authorizer with Logging {
     retryBackoffMs + Random.nextInt(retryBackoffJitterMs)
   }
 
+  private def apiException(e: Throwable): ApiException = {
+    e match {
+      case e1: ApiException => e1
+      case e1 => new ApiException(e1)
+    }
+  }
+
   object AclChangedNotificationHandler extends AclChangeNotificationHandler {
     override def processNotification(resource: Resource): Unit = {
       inWriteLock(lock) {
diff --git a/core/src/main/scala/kafka/security/authorizer/AuthorizerWrapper.scala b/core/src/main/scala/kafka/security/authorizer/AuthorizerWrapper.scala
index df10ff45eaaee..9556c8c6b9136 100644
--- a/core/src/main/scala/kafka/security/authorizer/AuthorizerWrapper.scala
+++ b/core/src/main/scala/kafka/security/authorizer/AuthorizerWrapper.scala
@@ -17,7 +17,7 @@
 
 package kafka.security.authorizer
 
-import java.util.concurrent.CompletableFuture
+import java.util.concurrent.{CompletableFuture, CompletionStage}
 import java.{lang, util}
 
 import kafka.network.RequestChannel.Session
@@ -40,7 +40,7 @@ class AuthorizerWrapper(private[kafka] val baseAuthorizer: kafka.security.auth.A
     baseAuthorizer.configure(configs)
   }
 
-  override def start(serverInfo: AuthorizerServerInfo): util.Map[Endpoint, CompletableFuture[Void]] = {
+  override def start(serverInfo: AuthorizerServerInfo): util.Map[Endpoint, _ <: CompletionStage[Void]] = {
     serverInfo.endpoints.asScala.map { endpoint =>
       endpoint -> CompletableFuture.completedFuture[Void](null) }.toMap.asJava
   }
@@ -57,7 +57,7 @@ class AuthorizerWrapper(private[kafka] val baseAuthorizer: kafka.security.auth.A
   }
 
   override def createAcls(requestContext: AuthorizableRequestContext,
-                          aclBindings: util.List[AclBinding]): util.List[AclCreateResult] = {
+                          aclBindings: util.List[AclBinding]): util.List[_ <: CompletionStage[AclCreateResult]] = {
     aclBindings.asScala
       .map { aclBinding =>
         AuthorizerUtils.convertToResourceAndAcl(aclBinding.toFilter) match {
@@ -71,11 +71,11 @@ class AuthorizerWrapper(private[kafka] val baseAuthorizer: kafka.security.auth.A
               case e: Throwable => new AclCreateResult(new InvalidRequestException("Failed to create ACL", e))
             }
         }
-      }.toList.asJava
+      }.toList.map(CompletableFuture.completedFuture[AclCreateResult]).asJava
   }
 
   override def deleteAcls(requestContext: AuthorizableRequestContext,
-                          aclBindingFilters: util.List[AclBindingFilter]): util.List[AclDeleteResult] = {
+                          aclBindingFilters: util.List[AclBindingFilter]): util.List[_ <: CompletionStage[AclDeleteResult]] = {
     val filters = aclBindingFilters.asScala
     val results = mutable.Map[Int, AclDeleteResult]()
     val toDelete = mutable.Map[Int, ArrayBuffer[(Resource, Acl)]]()
@@ -121,7 +121,7 @@ class AuthorizerWrapper(private[kafka] val baseAuthorizer: kafka.security.auth.A
 
     filters.indices.map { i =>
       results.getOrElse(i, new AclDeleteResult(Seq.empty[AclBindingDeleteResult].asJava))
-    }.asJava
+    }.map(CompletableFuture.completedFuture[AclDeleteResult]).asJava
   }
 
   override def acls(filter: AclBindingFilter): lang.Iterable[AclBinding] = {
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 810711de64339..de0397cc783e2 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -2202,7 +2202,7 @@ class KafkaApis(val requestChannel: RequestChannel,
         val createResults = auth.createAcls(request.context, validBindings.asJava)
 
         val aclCreationResults = aclBindings.map { acl =>
-          val result = errorResults.getOrElse(acl, createResults.get(validBindings.indexOf(acl)))
+          val result = errorResults.getOrElse(acl, createResults.get(validBindings.indexOf(acl)).toCompletableFuture.get)
           new AclCreationResponse(result.exception.asScala.map(ApiError.fromThrowable).getOrElse(ApiError.NONE))
         }
 
@@ -2225,7 +2225,7 @@ class KafkaApis(val requestChannel: RequestChannel,
         def toErrorCode(exception: Optional[ApiException]): ApiError = {
           exception.asScala.map(ApiError.fromThrowable).getOrElse(ApiError.NONE)
         }
-        val filterResponses = results.asScala.map { result =>
+        val filterResponses = results.asScala.map(_.toCompletableFuture.get).map { result =>
           val deletions = result.aclBindingDeleteResults().asScala.toList.map { deletionResult =>
             new AclDeletionResult(toErrorCode(deletionResult.exception), deletionResult.aclBinding)
           }.asJava
diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala
index 10677150aa418..0028e4a581987 100755
--- a/core/src/main/scala/kafka/server/KafkaServer.scala
+++ b/core/src/main/scala/kafka/server/KafkaServer.scala
@@ -297,7 +297,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
         authorizer.foreach(_.configure(config.originals))
         val authorizerFutures: Map[Endpoint, CompletableFuture[Void]] = authorizer match {
           case Some(authZ) =>
-            authZ.start(brokerInfo.broker.toServerInfo(clusterId, config)).asScala
+            authZ.start(brokerInfo.broker.toServerInfo(clusterId, config)).asScala.mapValues(_.toCompletableFuture).toMap
           case None =>
             brokerInfo.broker.endPoints.map{ ep => (ep.asInstanceOf[Endpoint], CompletableFuture.completedFuture[Void](null)) }.toMap
         }
diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
index af80a99ba0c16..9896f75c6873f 100644
--- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
@@ -1636,7 +1636,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   def removeAllAcls(): Unit = {
     val authorizer = servers.head.dataPlaneRequestProcessor.authorizer.get
     val aclFilter = AclBindingFilter.ANY
-    authorizer.deleteAcls(null, List(aclFilter).asJava).asScala.flatMap { deletion =>
+    authorizer.deleteAcls(null, List(aclFilter).asJava).asScala.map(_.toCompletableFuture.get).flatMap { deletion =>
       deletion.aclBindingDeleteResults().asScala.map(_.aclBinding.pattern).toSet
     }.foreach { resource =>
       TestUtils.waitAndVerifyAcls(Set.empty[AccessControlEntry], authorizer, resource)
@@ -1695,7 +1695,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   private def addAndVerifyAcls(acls: Set[AccessControlEntry], resource: ResourcePattern): Unit = {
     val aclBindings = acls.map { acl => new AclBinding(resource, acl) }
     servers.head.dataPlaneRequestProcessor.authorizer.get
-      .createAcls(null, aclBindings.toList.asJava).asScala.foreach {result =>
+      .createAcls(null, aclBindings.toList.asJava).asScala.map(_.toCompletableFuture.get).foreach {result =>
         result.exception.asScala.foreach { e => throw e }
       }
     val aclFilter = new AclBindingFilter(resource.toFilter, AccessControlEntryFilter.ANY)
diff --git a/core/src/test/scala/integration/kafka/api/DescribeAuthorizedOperationsTest.scala b/core/src/test/scala/integration/kafka/api/DescribeAuthorizedOperationsTest.scala
index 9fa1dbc62eebd..bda827127dbbc 100644
--- a/core/src/test/scala/integration/kafka/api/DescribeAuthorizedOperationsTest.scala
+++ b/core/src/test/scala/integration/kafka/api/DescribeAuthorizedOperationsTest.scala
@@ -60,7 +60,7 @@ class DescribeAuthorizedOperationsTest extends IntegrationTestHarness with SaslS
         new AclBinding(clusterResource, accessControlEntry(JaasTestUtils.KafkaServerPrincipalUnqualifiedName.toString, ALLOW, CLUSTER_ACTION)),
         new AclBinding(clusterResource, accessControlEntry(JaasTestUtils.KafkaClientPrincipalUnqualifiedName2.toString, ALLOW, ALTER)),
         new AclBinding(topicResource, accessControlEntry(JaasTestUtils.KafkaClientPrincipalUnqualifiedName2.toString, ALLOW, DESCRIBE))).asJava)
-      result.asScala.foreach { result => assertFalse(result.exception.isPresent) }
+      result.asScala.map(_.toCompletableFuture.get).foreach { result => assertFalse(result.exception.isPresent) }
 
     } finally {
       authorizer.close()
diff --git a/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala
index 64f44410ced02..b3e412bd96fcd 100644
--- a/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala
@@ -13,28 +13,61 @@
 package kafka.api
 
 import java.io.File
+import java.util
 import java.util.Collections
+import java.util.concurrent.{CompletionStage, Semaphore}
 
 import kafka.security.authorizer.AclAuthorizer
 import kafka.security.authorizer.AuthorizerUtils.{WildcardHost, WildcardPrincipal}
 import kafka.security.auth.{Operation, PermissionType}
 import kafka.server.KafkaConfig
 import kafka.utils.{CoreUtils, TestUtils}
+import org.apache.kafka.clients.admin.AdminClient
 import org.apache.kafka.common.acl._
 import org.apache.kafka.common.acl.AclOperation._
 import org.apache.kafka.common.acl.AclPermissionType._
+import org.apache.kafka.common.protocol.ApiKeys
 import org.apache.kafka.common.resource.ResourcePattern
 import org.apache.kafka.common.resource.PatternType._
 import org.apache.kafka.common.resource.ResourceType._
 import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
 import org.apache.kafka.server.authorizer._
-import org.junit.Assert
+import org.junit.Assert.{assertEquals, assertFalse, assertTrue}
+import org.junit.{Assert, Test}
 
 import scala.collection.JavaConverters._
 
+object SslAdminClientIntegrationTest {
+  @volatile var semaphore: Option[Semaphore] = None
+  @volatile var lastUpdateRequestContext: Option[AuthorizableRequestContext] = None
+  class TestableAclAuthorizer extends AclAuthorizer {
+    override def createAcls(requestContext: AuthorizableRequestContext,
+                            aclBindings: util.List[AclBinding]): util.List[_ <: CompletionStage[AclCreateResult]] = {
+      lastUpdateRequestContext = Some(requestContext)
+      semaphore.foreach(_.acquire())
+      try {
+        super.createAcls(requestContext, aclBindings)
+      } finally {
+        semaphore.foreach(_.release())
+      }
+    }
+
+    override def deleteAcls(requestContext: AuthorizableRequestContext,
+                            aclBindingFilters: util.List[AclBindingFilter]): util.List[_ <: CompletionStage[AclDeleteResult]] = {
+      lastUpdateRequestContext = Some(requestContext)
+      semaphore.foreach(_.acquire())
+      try {
+        super.deleteAcls(requestContext, aclBindingFilters)
+      } finally {
+        semaphore.foreach(_.release())
+      }
+    }
+  }
+}
+
 class SslAdminClientIntegrationTest extends SaslSslAdminClientIntegrationTest {
   this.serverConfig.setProperty(KafkaConfig.ZkEnableSecureAclsProp, "true")
-  this.serverConfig.setProperty(KafkaConfig.AuthorizerClassNameProp, classOf[AclAuthorizer].getName)
+  this.serverConfig.setProperty(KafkaConfig.AuthorizerClassNameProp, classOf[SslAdminClientIntegrationTest.TestableAclAuthorizer].getName)
 
   override protected def securityProtocol = SecurityProtocol.SSL
   override protected lazy val trustStoreFile = Some(File.createTempFile("truststore", ".jks"))
@@ -79,7 +112,7 @@ class SslAdminClientIntegrationTest extends SaslSslAdminClientIntegrationTest {
     val prevAcls = authorizer.acls(clusterFilter).asScala.map(_.entry).toSet
     val deleteFilter = new AclBindingFilter(clusterResourcePattern.toFilter, ace.toFilter)
     Assert.assertFalse(authorizer.deleteAcls(null, Collections.singletonList(deleteFilter))
-      .get(0).aclBindingDeleteResults().asScala.head.exception.isPresent)
+      .get(0).toCompletableFuture.get.aclBindingDeleteResults().asScala.head.exception.isPresent)
     TestUtils.waitAndVerifyAcls(prevAcls -- Set(ace), authorizer, clusterResourcePattern)
   }
 
@@ -87,4 +120,42 @@ class SslAdminClientIntegrationTest extends SaslSslAdminClientIntegrationTest {
     new AccessControlEntry(new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "*").toString,
       WildcardHost, operation, permissionType)
   }
+
+  @Test
+  def testAsyncAclUpdates(): Unit = {
+    client = AdminClient.create(createConfig())
+
+    def validateRequestContext(context: AuthorizableRequestContext, apiKey: ApiKeys): Unit = {
+      assertEquals(SecurityProtocol.SSL, context.securityProtocol)
+      assertEquals("SSL", context.listener)
+      assertEquals(KafkaPrincipal.ANONYMOUS, context.principal)
+      assertEquals(apiKey.id.toInt, context.requestType)
+      assertEquals(apiKey.latestVersion.toInt, context.requestVersion)
+      assertTrue(s"Invalid correlation id: ${context.correlationId}", context.correlationId > 0)
+      assertTrue(s"Invalid client id: ${context.clientId}", context.clientId.startsWith("adminclient"))
+      assertTrue(s"Invalid host address: ${context.clientAddress}", context.clientAddress.isLoopbackAddress)
+    }
+
+    val testSemaphore = new Semaphore(0)
+    SslAdminClientIntegrationTest.semaphore = Some(testSemaphore)
+    val results = client.createAcls(List(acl2, acl3).asJava).values
+    assertEquals(Set(acl2, acl3), results.keySet().asScala)
+    assertFalse(results.values().asScala.exists(_.isDone))
+    TestUtils.waitUntilTrue(() => testSemaphore.hasQueuedThreads, "Authorizer not blocked in createAcls")
+    testSemaphore.release()
+    results.values().asScala.foreach(_.get)
+    validateRequestContext(SslAdminClientIntegrationTest.lastUpdateRequestContext.get, ApiKeys.CREATE_ACLS)
+
+    testSemaphore.acquire()
+    val results2 = client.deleteAcls(List(ACL1.toFilter, acl2.toFilter, acl3.toFilter).asJava).values
+    assertEquals(Set(ACL1.toFilter, acl2.toFilter, acl3.toFilter), results2.keySet.asScala)
+    assertFalse(results2.values().asScala.exists(_.isDone))
+    TestUtils.waitUntilTrue(() => testSemaphore.hasQueuedThreads, "Authorizer not blocked in deleteAcls")
+    testSemaphore.release()
+    results.values().asScala.foreach(_.get)
+    assertEquals(0, results2.get(ACL1.toFilter).get.values.size())
+    assertEquals(Set(acl2), results2.get(acl2.toFilter).get.values.asScala.map(_.binding).toSet)
+    assertEquals(Set(acl3), results2.get(acl3.toFilter).get.values.asScala.map(_.binding).toSet)
+    validateRequestContext(SslAdminClientIntegrationTest.lastUpdateRequestContext.get, ApiKeys.DELETE_ACLS)
+  }
 }
diff --git a/core/src/test/scala/unit/kafka/security/authorizer/AclAuthorizerTest.scala b/core/src/test/scala/unit/kafka/security/authorizer/AclAuthorizerTest.scala
index 3cba4ed441758..6250958596620 100644
--- a/core/src/test/scala/unit/kafka/security/authorizer/AclAuthorizerTest.scala
+++ b/core/src/test/scala/unit/kafka/security/authorizer/AclAuthorizerTest.scala
@@ -31,7 +31,7 @@ import kafka.zookeeper.{GetChildrenRequest, GetDataRequest, ZooKeeperClient}
 import org.apache.kafka.common.acl._
 import org.apache.kafka.common.acl.AclOperation._
 import org.apache.kafka.common.acl.AclPermissionType.{ALLOW, DENY}
-import org.apache.kafka.common.errors.UnsupportedVersionException
+import org.apache.kafka.common.errors.{ApiException, UnsupportedVersionException}
 import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.protocol.ApiKeys
 import org.apache.kafka.common.requests.{RequestContext, RequestHeader}
@@ -41,9 +41,10 @@ import org.apache.kafka.common.resource.ResourceType._
 import org.apache.kafka.common.resource.PatternType.{LITERAL, MATCH, PREFIXED}
 import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
 import org.apache.kafka.server.authorizer._
-import org.apache.kafka.common.utils.{SecurityUtils => JSecurityUtils, Time}
+import org.apache.kafka.common.utils.{Time, SecurityUtils => JSecurityUtils}
 import org.junit.Assert._
 import org.junit.{After, Before, Test}
+import org.scalatest.Assertions.intercept
 
 import scala.collection.JavaConverters._
 import scala.compat.java8.OptionConverters._
@@ -114,9 +115,12 @@ class AclAuthorizerTest extends ZooKeeperTestHarness {
   }
 
   // Authorizing the empty resource is not supported because we create a znode with the resource name.
-  @Test(expected = classOf[IllegalArgumentException])
+  @Test
   def testEmptyAclThrowsException(): Unit = {
-    addAcls(aclAuthorizer, Set(allowReadAcl), new ResourcePattern(GROUP, "", LITERAL))
+    val e = intercept[ApiException] {
+      addAcls(aclAuthorizer, Set(allowReadAcl), new ResourcePattern(GROUP, "", LITERAL))
+    }
+    assertTrue(s"Unexpected exception $e", e.getCause.isInstanceOf[IllegalArgumentException])
   }
 
   @Test
@@ -704,7 +708,7 @@ class AclAuthorizerTest extends ZooKeeperTestHarness {
       acl1.toFilter,
       new AclBindingFilter(resource2.toFilter, AccessControlEntryFilter.ANY),
       new AclBindingFilter(new ResourcePatternFilter(TOPIC, "baz", PatternType.ANY), AccessControlEntryFilter.ANY))
-    val deleteResults = aclAuthorizer.deleteAcls(requestContext, filters.asJava).asScala
+    val deleteResults = aclAuthorizer.deleteAcls(requestContext, filters.asJava).asScala.map(_.toCompletableFuture.get)
     assertEquals(List.empty, deleteResults.filter(_.exception.isPresent))
     filters.indices.foreach { i =>
       assertEquals(Set.empty, deleteResults(i).aclBindingDeleteResults.asScala.toSet.filter(_.exception.isPresent))
@@ -715,10 +719,13 @@ class AclAuthorizerTest extends ZooKeeperTestHarness {
     assertEquals(Set.empty, deleteResults(3).aclBindingDeleteResults.asScala.map(_.aclBinding).toSet)
   }
 
-  @Test(expected = classOf[UnsupportedVersionException])
+  @Test
   def testThrowsOnAddPrefixedAclIfInterBrokerProtocolVersionTooLow(): Unit = {
     givenAuthorizerWithProtocolVersion(Option(KAFKA_2_0_IV0))
-    addAcls(aclAuthorizer, Set(denyReadAcl), new ResourcePattern(TOPIC, "z_other", PREFIXED))
+    val e = intercept[ApiException] {
+      addAcls(aclAuthorizer, Set(denyReadAcl), new ResourcePattern(TOPIC, "z_other", PREFIXED))
+    }
+    assertTrue(s"Unexpected exception $e", e.getCause.isInstanceOf[UnsupportedVersionException])
   }
 
   @Test
@@ -835,9 +842,9 @@ class AclAuthorizerTest extends ZooKeeperTestHarness {
 
   private def addAcls(authorizer: AclAuthorizer, aces: Set[AccessControlEntry], resourcePattern: ResourcePattern): Unit = {
     val bindings = aces.map { ace => new AclBinding(resourcePattern, ace) }
-    authorizer.createAcls(requestContext, bindings.toList.asJava).asScala.foreach { result =>
-      result.exception.asScala.foreach { e => throw e }
-    }
+    authorizer.createAcls(requestContext, bindings.toList.asJava).asScala
+      .map(_.toCompletableFuture.get)
+      .foreach { result => result.exception.asScala.foreach { e => throw e } }
   }
 
   private def removeAcls(authorizer: AclAuthorizer, aces: Set[AccessControlEntry], resourcePattern: ResourcePattern): Boolean = {
@@ -845,13 +852,15 @@ class AclAuthorizerTest extends ZooKeeperTestHarness {
       Set(new AclBindingFilter(resourcePattern.toFilter, AccessControlEntryFilter.ANY) )
     else
       aces.map { ace => new AclBinding(resourcePattern, ace).toFilter }
-    authorizer.deleteAcls(requestContext, bindings.toList.asJava).asScala.forall { result =>
-      result.exception.asScala.foreach { e => throw e }
-      result.aclBindingDeleteResults.asScala.foreach { r =>
-        r.exception.asScala.foreach { e => throw e }
+    authorizer.deleteAcls(requestContext, bindings.toList.asJava).asScala
+      .map(_.toCompletableFuture.get)
+      .forall { result =>
+        result.exception.asScala.foreach { e => throw e }
+        result.aclBindingDeleteResults.asScala.foreach { r =>
+          r.exception.asScala.foreach { e => throw e }
+        }
+        result.aclBindingDeleteResults.asScala.exists(_.exception.asScala.isEmpty)
       }
-      result.aclBindingDeleteResults.asScala.exists(_.exception.asScala.isEmpty)
-    }
   }
 
   private def getAcls(authorizer: AclAuthorizer, resourcePattern: ResourcePattern): Set[AccessControlEntry] = {
diff --git a/core/src/test/scala/unit/kafka/security/token/delegation/DelegationTokenManagerTest.scala b/core/src/test/scala/unit/kafka/security/token/delegation/DelegationTokenManagerTest.scala
index 8550f157357c9..90a3b93a941af 100644
--- a/core/src/test/scala/unit/kafka/security/token/delegation/DelegationTokenManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/security/token/delegation/DelegationTokenManagerTest.scala
@@ -281,7 +281,7 @@ class DelegationTokenManagerTest extends ZooKeeperTestHarness  {
     assert(tokens.size == 2)
 
     def createAcl(aclBinding: AclBinding): Unit = {
-      val result = aclAuthorizer.createAcls(null, List(aclBinding).asJava).get(0)
+      val result = aclAuthorizer.createAcls(null, List(aclBinding).asJava).get(0).toCompletableFuture.get
       result.exception.asScala.foreach { e => throw e }
     }
 
diff --git a/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala b/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala
index 2c3ec5f43c21f..31df6c772b322 100755
--- a/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala
+++ b/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala
@@ -19,7 +19,7 @@ package kafka.server
 
 import java.{lang, util}
 import java.util.Properties
-import java.util.concurrent.CompletableFuture
+import java.util.concurrent.CompletionStage
 
 import kafka.utils.TestUtils
 import kafka.zk.KafkaZkClient
@@ -332,13 +332,13 @@ class DynamicBrokerConfigTest {
 
     class TestAuthorizer extends Authorizer with Reconfigurable {
       @volatile var superUsers = ""
-      override def acls(filter: AclBindingFilter): lang.Iterable[AclBinding] = null
-      override def start(serverInfo: AuthorizerServerInfo): util.Map[Endpoint, CompletableFuture[Void]] = Map.empty.asJava
-      override def deleteAcls(requestContext: AuthorizableRequestContext, aclBindingFilters: util.List[AclBindingFilter]): util.List[AclDeleteResult] = null
-      override def createAcls(requestContext: AuthorizableRequestContext, aclBindings: util.List[AclBinding]): util.List[AclCreateResult] = null
+      override def start(serverInfo: AuthorizerServerInfo): util.Map[Endpoint, _ <: CompletionStage[Void]] = Map.empty.asJava
       override def authorize(requestContext: AuthorizableRequestContext, actions: util.List[Action]): util.List[AuthorizationResult] = null
-      override def configure(configs: util.Map[String, _]): Unit = {}
+      override def createAcls(requestContext: AuthorizableRequestContext, aclBindings: util.List[AclBinding]): util.List[_ <: CompletionStage[AclCreateResult]] = null
+      override def deleteAcls(requestContext: AuthorizableRequestContext, aclBindingFilters: util.List[AclBindingFilter]): util.List[_ <: CompletionStage[AclDeleteResult]] = null
+      override def acls(filter: AclBindingFilter): lang.Iterable[AclBinding] = null
       override def close(): Unit = {}
+      override def configure(configs: util.Map[String, _]): Unit = {}
       override def reconfigurableConfigs(): util.Set[String] = Set("super.users").asJava
       override def validateReconfiguration(configs: util.Map[String, _]): Unit = {}
       override def reconfigure(configs: util.Map[String, _]): Unit = {

From 6a3a58039943aa047fbd07abfc7eff6431c367a5 Mon Sep 17 00:00:00 2001
From: Bruno Cadonna 
Date: Wed, 11 Sep 2019 20:01:39 +0200
Subject: [PATCH 0606/1071] KAFKA-8856: Add Streams config for
 backward-compatible metrics (#7279)

Reviewers: John Roesler , Guozhang Wang 
---
 .../apache/kafka/streams/StreamsConfig.java   | 23 ++++++++++++
 .../kafka/streams/StreamsConfigTest.java      | 36 ++++++++++++++++++-
 2 files changed, 58 insertions(+), 1 deletion(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
index f08eecaac4743..c0eda800e6a56 100644
--- a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
+++ b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
@@ -263,6 +263,16 @@ public class StreamsConfig extends AbstractConfig {
     @SuppressWarnings("WeakerAccess")
     public static final String EXACTLY_ONCE = "exactly_once";
 
+    /**
+     * Config value for parameter {@link #BUILT_IN_METRICS_VERSION_CONFIG "built.in.metrics.version"} for built-in metrics from version 0.10.0. to 2.3
+     */
+    public static final String METRICS_0100_TO_23 = "0.10.0-2.3";
+
+    /**
+     * Config value for parameter {@link #BUILT_IN_METRICS_VERSION_CONFIG "built.in.metrics.version"} for the latest built-in metrics version.
+     */
+    public static final String METRICS_LATEST = "latest";
+
     /** {@code application.id} */
     @SuppressWarnings("WeakerAccess")
     public static final String APPLICATION_ID_CONFIG = "application.id";
@@ -282,6 +292,10 @@ public class StreamsConfig extends AbstractConfig {
     public static final String BUFFERED_RECORDS_PER_PARTITION_CONFIG = "buffered.records.per.partition";
     private static final String BUFFERED_RECORDS_PER_PARTITION_DOC = "Maximum number of records to buffer per partition.";
 
+    /** {@code built.in.metrics.version} */
+    public static final String BUILT_IN_METRICS_VERSION_CONFIG = "built.in.metrics.version";
+    private static final String BUILT_IN_METRICS_VERSION_DOC = "Version of the built-in metrics to use.";
+
     /** {@code cache.max.bytes.buffering} */
     @SuppressWarnings("WeakerAccess")
     public static final String CACHE_MAX_BYTES_BUFFERING_CONFIG = "cache.max.bytes.buffering";
@@ -581,6 +595,15 @@ public class StreamsConfig extends AbstractConfig {
                     1000,
                     Importance.LOW,
                     BUFFERED_RECORDS_PER_PARTITION_DOC)
+            .define(BUILT_IN_METRICS_VERSION_CONFIG,
+                    Type.STRING,
+                    METRICS_LATEST,
+                    in(
+                        METRICS_0100_TO_23,
+                        METRICS_LATEST
+                    ),
+                    Importance.LOW,
+                    BUILT_IN_METRICS_VERSION_DOC)
             .define(COMMIT_INTERVAL_MS_CONFIG,
                     Type.LONG,
                     DEFAULT_COMMIT_INTERVAL_MS,
diff --git a/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java b/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
index 27e225597b2fc..c9a168912a0a5 100644
--- a/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
@@ -50,10 +50,13 @@
 import static org.apache.kafka.streams.StreamsConfig.consumerPrefix;
 import static org.apache.kafka.streams.StreamsConfig.producerPrefix;
 import static org.apache.kafka.test.StreamsTestUtils.getStreamsConfig;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.core.IsEqual.equalTo;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -450,11 +453,42 @@ public void shouldAcceptExactlyOnce() {
     }
 
     @Test(expected = ConfigException.class)
-    public void shouldThrowExceptionIfNotAtLestOnceOrExactlyOnce() {
+    public void shouldThrowExceptionIfNotAtLeastOnceOrExactlyOnce() {
         props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, "bad_value");
         new StreamsConfig(props);
     }
 
+    @Test
+    public void shouldAcceptBuiltInMetricsVersion0100To23() {
+        // don't use `StreamsConfig.METRICS_0100_TO_23` to actually do a useful test
+        props.put(StreamsConfig.BUILT_IN_METRICS_VERSION_CONFIG, "0.10.0-2.3");
+        new StreamsConfig(props);
+    }
+
+    @Test
+    public void shouldAcceptBuiltInMetricsLatestVersion() {
+        // don't use `StreamsConfig.METRICS_LATEST` to actually do a useful test
+        props.put(StreamsConfig.BUILT_IN_METRICS_VERSION_CONFIG, "latest");
+        new StreamsConfig(props);
+    }
+
+    @Test
+    public void shouldSetDefaultBuiltInMetricsVersionIfNoneIsSpecified() {
+        final StreamsConfig config = new StreamsConfig(props);
+        assertThat(config.getString(StreamsConfig.BUILT_IN_METRICS_VERSION_CONFIG), is(StreamsConfig.METRICS_LATEST));
+    }
+
+    @Test
+    public void shouldThrowIfBuiltInMetricsVersionInvalid() {
+        final String invalidVersion = "0.0.1";
+        props.put(StreamsConfig.BUILT_IN_METRICS_VERSION_CONFIG, invalidVersion);
+        final Exception exception = assertThrows(ConfigException.class, () -> new StreamsConfig(props));
+        assertThat(
+            exception.getMessage(),
+            containsString("Invalid value " + invalidVersion + " for configuration built.in.metrics.version")
+        );
+    }
+
     @Test
     public void shouldResetToDefaultIfConsumerIsolationLevelIsOverriddenIfEosEnabled() {
         props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE);

From d3559f628b2ccb23a9faf531796675376ac06abb Mon Sep 17 00:00:00 2001
From: huxi 
Date: Thu, 12 Sep 2019 05:24:27 +0800
Subject: [PATCH 0607/1071] KAFKA-8875; CreateTopic API should check topic
 existence before replication factor (#7298)

If the topic already exists, `handleCreateTopicsRequest` should return TopicExistsException even given an invalid config (replication factor for instance).

Reviewers: Rajini Sivaram , Jason Gustafson 
---
 .../main/scala/kafka/server/AdminManager.scala  |  5 ++++-
 .../kafka/api/AdminClientIntegrationTest.scala  | 17 +++++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/kafka/server/AdminManager.scala b/core/src/main/scala/kafka/server/AdminManager.scala
index 7b71e35a88e0f..1c4115e20068c 100644
--- a/core/src/main/scala/kafka/server/AdminManager.scala
+++ b/core/src/main/scala/kafka/server/AdminManager.scala
@@ -29,7 +29,7 @@ import org.apache.kafka.clients.admin.AlterConfigOp
 import org.apache.kafka.clients.admin.AlterConfigOp.OpType
 import org.apache.kafka.common.config.ConfigDef.ConfigKey
 import org.apache.kafka.common.config.{AbstractConfig, ConfigDef, ConfigException, ConfigResource, LogLevelConfig}
-import org.apache.kafka.common.errors.{ApiException, InvalidConfigurationException, InvalidPartitionsException, InvalidReplicaAssignmentException, InvalidRequestException, ReassignmentInProgressException, UnknownTopicOrPartitionException}
+import org.apache.kafka.common.errors.{ApiException, InvalidConfigurationException, InvalidPartitionsException, InvalidReplicaAssignmentException, InvalidRequestException, ReassignmentInProgressException, TopicExistsException, UnknownTopicOrPartitionException}
 import org.apache.kafka.common.internals.Topic
 import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic
 import org.apache.kafka.common.metrics.Metrics
@@ -88,6 +88,9 @@ class AdminManager(val config: KafkaConfig,
     val brokers = metadataCache.getAliveBrokers.map { b => kafka.admin.BrokerMetadata(b.id, b.rack) }
     val metadata = toCreate.values.map(topic =>
       try {
+        if (metadataCache.contains(topic.name))
+          throw new TopicExistsException(s"Topic '${topic.name}' already exists.")
+
         val configs = new Properties()
         topic.configs.asScala.foreach { entry =>
           configs.setProperty(entry.name, entry.value)
diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index 02b8f1afc7046..27c435be3be2b 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -229,6 +229,23 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     waitForTopics(client, List(), topics)
   }
 
+  @Test
+  def testCreateExistingTopicsThrowTopicExistsException(): Unit = {
+    client = AdminClient.create(createConfig())
+    val topic = "mytopic"
+    val topics = Seq(topic)
+    val newTopics = Seq(new NewTopic(topic, 1, 1.toShort))
+
+    client.createTopics(newTopics.asJava).all.get()
+    waitForTopics(client, topics, List())
+
+    val newTopicsWithInvalidRF = Seq(new NewTopic(topic, 1, (servers.size + 1).toShort))
+    val e = intercept[ExecutionException] {
+      client.createTopics(newTopicsWithInvalidRF.asJava, new CreateTopicsOptions().validateOnly(true)).all.get()
+    }
+    assertTrue(e.getCause.isInstanceOf[TopicExistsException])
+  }
+
   @Test
   def testMetadataRefresh(): Unit = {
     client = AdminClient.create(createConfig())

From 23708b77db110bdb20bf4a01b656d1b0ccc0f864 Mon Sep 17 00:00:00 2001
From: Boyang Chen 
Date: Thu, 12 Sep 2019 09:57:22 -0700
Subject: [PATCH 0608/1071] KAFKA-8355: add static membership to range assignor
 (#7014)

The purpose of this PR is to add static membership support for range assignor. More details for the motivation in here.

Similar to round robin assignor, if we are capable of persisting member identity across generations, we will reach a much more stable assignment.

Reviewers: John Roesler , Guozhang Wang , Bruno Cadonna 
---
 .../kafka/clients/consumer/RangeAssignor.java |  45 +++-
 .../clients/consumer/RoundRobinAssignor.java  |   3 +-
 .../clients/consumer/RangeAssignorTest.java   | 252 ++++++++++++++----
 .../consumer/RoundRobinAssignorTest.java      | 119 ++++-----
 4 files changed, 283 insertions(+), 136 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/RangeAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/RangeAssignor.java
index 78956abb2182d..ea642885c0623 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/RangeAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/RangeAssignor.java
@@ -40,6 +40,29 @@
  * 
            • C0: [t0p0, t0p1, t1p0, t1p1]
              • 
  * 
            • C1: [t0p2, t1p2]
              • 
  * 
            
+ *
+ * Since the introduction of static membership, we could leverage group.instance.id to make the assignment behavior more sticky.
+ * For the above example, after one rolling bounce, group coordinator will attempt to assign new member.id towards consumers,
+ * for example C0 -> C3 C1 -> C2.
+ *
+ * 
            The assignment could be completely shuffled to:
+ * 
              
+ * 
            • C3 (was C0): [t0p2, t1p2] (before was [t0p0, t0p1, t1p0, t1p1])
+ * 
            • C2 (was C1): [t0p0, t0p1, t1p0, t1p1] (before was [t0p2, t1p2])
+ * 
            
+ *
+ * The assignment change was caused by the change of member.id relative order, and
+ * can be avoided by setting the group.instance.id.
+ * Consumers will have individual instance ids I1, I2. As long as
+ * 1. Number of members remain the same across generation
+ * 2. Static members' identities persist across generation
+ * 3. Subscription pattern doesn't change for any member
+ *
+ * 
            The assignment will always be:
+ * 
              
+ * 
            • I0: [t0p0, t0p1, t1p0, t1p1]
+ * 
            • I1: [t0p2, t1p2]
+ * 
            
  */
 public class RangeAssignor extends AbstractPartitionAssignor {
 
@@ -48,27 +71,30 @@ public String name() {
         return "range";
     }
 
-    private Map> consumersPerTopic(Map consumerMetadata) {
-        Map> res = new HashMap<>();
+    private Map> consumersPerTopic(Map consumerMetadata) {
+        Map> topicToConsumers = new HashMap<>();
         for (Map.Entry subscriptionEntry : consumerMetadata.entrySet()) {
             String consumerId = subscriptionEntry.getKey();
-            for (String topic : subscriptionEntry.getValue().topics())
-                put(res, topic, consumerId);
+            MemberInfo memberInfo = new MemberInfo(consumerId, subscriptionEntry.getValue().groupInstanceId());
+            for (String topic : subscriptionEntry.getValue().topics()) {
+                put(topicToConsumers, topic, memberInfo);
+            }
         }
-        return res;
+        return topicToConsumers;
     }
 
     @Override
     public Map> assign(Map partitionsPerTopic,
                                                     Map subscriptions) {
-        Map> consumersPerTopic = consumersPerTopic(subscriptions);
+        Map> consumersPerTopic = consumersPerTopic(subscriptions);
+
         Map> assignment = new HashMap<>();
         for (String memberId : subscriptions.keySet())
             assignment.put(memberId, new ArrayList<>());
 
-        for (Map.Entry> topicEntry : consumersPerTopic.entrySet()) {
+        for (Map.Entry> topicEntry : consumersPerTopic.entrySet()) {
             String topic = topicEntry.getKey();
-            List consumersForTopic = topicEntry.getValue();
+            List consumersForTopic = topicEntry.getValue();
 
             Integer numPartitionsForTopic = partitionsPerTopic.get(topic);
             if (numPartitionsForTopic == null)
@@ -83,10 +109,9 @@ public Map> assign(Map partitionsP
             for (int i = 0, n = consumersForTopic.size(); i < n; i++) {
                 int start = numPartitionsPerConsumer * i + Math.min(i, consumersWithExtraPartition);
                 int length = numPartitionsPerConsumer + (i + 1 > consumersWithExtraPartition ? 0 : 1);
-                assignment.get(consumersForTopic.get(i)).addAll(partitions.subList(start, start + length));
+                assignment.get(consumersForTopic.get(i).memberId).addAll(partitions.subList(start, start + length));
             }
         }
         return assignment;
     }
-
 }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/RoundRobinAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/RoundRobinAssignor.java
index 0981400a6e5af..0ac90b386834e 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/RoundRobinAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/RoundRobinAssignor.java
@@ -77,7 +77,7 @@
  * After one rolling bounce, group coordinator will attempt to assign new member.id towards consumers,
  * for example C0 -> C5 C1 -> C3, C2 -> C4.
  *
- * 
            the assignment could be completely shuffled to:
+ * 
            The assignment could be completely shuffled to:
  * 
              
  * 
            • C3 (was C1): [t0p0, t1p0] (before was [t0p1, t1p1])
  * 
            • C4 (was C2): [t0p1, t1p1] (before was [t0p2, t1p2])
@@ -88,6 +88,7 @@
  * I1, I2, I3. As long as
  * 1. Number of members remain the same across generation
  * 2. Static members' identities persist across generation
+ * 3. Subscription pattern doesn't change for any member
  *
  * 
              The assignment will always be:
  * 
                
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/RangeAssignorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/RangeAssignorTest.java
index 118e60a3d1287..e5c5073afee00 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/RangeAssignorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/RangeAssignorTest.java
@@ -17,15 +17,20 @@
 package org.apache.kafka.clients.consumer;
 
 import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription;
+import org.apache.kafka.clients.consumer.internals.AbstractPartitionAssignor;
+import org.apache.kafka.clients.consumer.internals.AbstractPartitionAssignor.MemberInfo;
 import org.apache.kafka.common.TopicPartition;
+import org.junit.Before;
 import org.junit.Test;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -33,39 +38,57 @@
 public class RangeAssignorTest {
 
     private RangeAssignor assignor = new RangeAssignor();
-    private String consumerId = "consumer";
-    private String topic = "topic";
+
+    // For plural tests
+    private String topic1 = "topic1";
+    private String topic2 = "topic2";
+    private final String consumer1 = "consumer1";
+    private final String instance1 = "instance1";
+    private final String consumer2 = "consumer2";
+    private final String instance2 = "instance2";
+    private final String consumer3 = "consumer3";
+    private final String instance3 = "instance3";
+
+    private List staticMemberInfos;
+
+    @Before
+    public void setUp() {
+        staticMemberInfos = new ArrayList<>();
+        staticMemberInfos.add(new MemberInfo(consumer1, Optional.of(instance1)));
+        staticMemberInfos.add(new MemberInfo(consumer2, Optional.of(instance2)));
+        staticMemberInfos.add(new MemberInfo(consumer3, Optional.of(instance3)));
+    }
 
     @Test
     public void testOneConsumerNoTopic() {
         Map partitionsPerTopic = new HashMap<>();
 
         Map> assignment = assignor.assign(partitionsPerTopic,
-                Collections.singletonMap(consumerId, new Subscription(Collections.emptyList())));
+                Collections.singletonMap(consumer1, new Subscription(Collections.emptyList())));
 
-        assertEquals(Collections.singleton(consumerId), assignment.keySet());
-        assertTrue(assignment.get(consumerId).isEmpty());
+        assertEquals(Collections.singleton(consumer1), assignment.keySet());
+        assertTrue(assignment.get(consumer1).isEmpty());
     }
 
     @Test
     public void testOneConsumerNonexistentTopic() {
         Map partitionsPerTopic = new HashMap<>();
         Map> assignment = assignor.assign(partitionsPerTopic,
-                Collections.singletonMap(consumerId, new Subscription(topics(topic))));
-        assertEquals(Collections.singleton(consumerId), assignment.keySet());
-        assertTrue(assignment.get(consumerId).isEmpty());
+                Collections.singletonMap(consumer1, new Subscription(topics(topic1))));
+        assertEquals(Collections.singleton(consumer1), assignment.keySet());
+        assertTrue(assignment.get(consumer1).isEmpty());
     }
 
     @Test
     public void testOneConsumerOneTopic() {
         Map partitionsPerTopic = new HashMap<>();
-        partitionsPerTopic.put(topic, 3);
+        partitionsPerTopic.put(topic1, 3);
 
         Map> assignment = assignor.assign(partitionsPerTopic,
-                Collections.singletonMap(consumerId, new Subscription(topics(topic))));
+                Collections.singletonMap(consumer1, new Subscription(topics(topic1))));
 
-        assertEquals(Collections.singleton(consumerId), assignment.keySet());
-        assertAssignment(partitions(tp(topic, 0), tp(topic, 1), tp(topic, 2)), assignment.get(consumerId));
+        assertEquals(Collections.singleton(consumer1), assignment.keySet());
+        assertAssignment(partitions(tp(topic1, 0), tp(topic1, 1), tp(topic1, 2)), assignment.get(consumer1));
     }
 
     @Test
@@ -73,79 +96,58 @@ public void testOnlyAssignsPartitionsFromSubscribedTopics() {
         String otherTopic = "other";
 
         Map partitionsPerTopic = new HashMap<>();
-        partitionsPerTopic.put(topic, 3);
+        partitionsPerTopic.put(topic1, 3);
         partitionsPerTopic.put(otherTopic, 3);
 
         Map> assignment = assignor.assign(partitionsPerTopic,
-                Collections.singletonMap(consumerId, new Subscription(topics(topic))));
-        assertEquals(Collections.singleton(consumerId), assignment.keySet());
-        assertAssignment(partitions(tp(topic, 0), tp(topic, 1), tp(topic, 2)), assignment.get(consumerId));
+                Collections.singletonMap(consumer1, new Subscription(topics(topic1))));
+        assertEquals(Collections.singleton(consumer1), assignment.keySet());
+        assertAssignment(partitions(tp(topic1, 0), tp(topic1, 1), tp(topic1, 2)), assignment.get(consumer1));
     }
 
     @Test
     public void testOneConsumerMultipleTopics() {
-        String topic1 = "topic1";
-        String topic2 = "topic2";
-
-        Map partitionsPerTopic = new HashMap<>();
-        partitionsPerTopic.put(topic1, 1);
-        partitionsPerTopic.put(topic2, 2);
+        Map partitionsPerTopic = setupPartitionsPerTopicWithTwoTopics(1, 2);
 
         Map> assignment = assignor.assign(partitionsPerTopic,
-                Collections.singletonMap(consumerId, new Subscription(topics(topic1, topic2))));
+                Collections.singletonMap(consumer1, new Subscription(topics(topic1, topic2))));
 
-        assertEquals(Collections.singleton(consumerId), assignment.keySet());
-        assertAssignment(partitions(tp(topic1, 0), tp(topic2, 0), tp(topic2, 1)), assignment.get(consumerId));
+        assertEquals(Collections.singleton(consumer1), assignment.keySet());
+        assertAssignment(partitions(tp(topic1, 0), tp(topic2, 0), tp(topic2, 1)), assignment.get(consumer1));
     }
 
     @Test
     public void testTwoConsumersOneTopicOnePartition() {
-        String topic = "topic";
-        String consumer1 = "consumer1";
-        String consumer2 = "consumer2";
-
         Map partitionsPerTopic = new HashMap<>();
-        partitionsPerTopic.put(topic, 1);
+        partitionsPerTopic.put(topic1, 1);
 
         Map consumers = new HashMap<>();
-        consumers.put(consumer1, new Subscription(topics(topic)));
-        consumers.put(consumer2, new Subscription(topics(topic)));
+        consumers.put(consumer1, new Subscription(topics(topic1)));
+        consumers.put(consumer2, new Subscription(topics(topic1)));
 
         Map> assignment = assignor.assign(partitionsPerTopic, consumers);
-        assertAssignment(partitions(tp(topic, 0)), assignment.get(consumer1));
+        assertAssignment(partitions(tp(topic1, 0)), assignment.get(consumer1));
         assertAssignment(Collections.emptyList(), assignment.get(consumer2));
     }
 
 
     @Test
     public void testTwoConsumersOneTopicTwoPartitions() {
-        String topic = "topic";
-        String consumer1 = "consumer1";
-        String consumer2 = "consumer2";
-
         Map partitionsPerTopic = new HashMap<>();
-        partitionsPerTopic.put(topic, 2);
+        partitionsPerTopic.put(topic1, 2);
 
         Map consumers = new HashMap<>();
-        consumers.put(consumer1, new Subscription(topics(topic)));
-        consumers.put(consumer2, new Subscription(topics(topic)));
+        consumers.put(consumer1, new Subscription(topics(topic1)));
+        consumers.put(consumer2, new Subscription(topics(topic1)));
 
         Map> assignment = assignor.assign(partitionsPerTopic, consumers);
-        assertAssignment(partitions(tp(topic, 0)), assignment.get(consumer1));
-        assertAssignment(partitions(tp(topic, 1)), assignment.get(consumer2));
+        assertAssignment(partitions(tp(topic1, 0)), assignment.get(consumer1));
+        assertAssignment(partitions(tp(topic1, 1)), assignment.get(consumer2));
     }
 
     @Test
     public void testMultipleConsumersMixedTopics() {
-        String topic1 = "topic1";
-        String topic2 = "topic2";
-        String consumer1 = "consumer1";
-        String consumer2 = "consumer2";
-        String consumer3 = "consumer3";
-
-        Map partitionsPerTopic = new HashMap<>();
-        partitionsPerTopic.put(topic1, 3);
-        partitionsPerTopic.put(topic2, 2);
+        Map partitionsPerTopic = setupPartitionsPerTopicWithTwoTopics(3, 2);
 
         Map consumers = new HashMap<>();
         consumers.put(consumer1, new Subscription(topics(topic1)));
@@ -165,9 +167,7 @@ public void testTwoConsumersTwoTopicsSixPartitions() {
         String consumer1 = "consumer1";
         String consumer2 = "consumer2";
 
-        Map partitionsPerTopic = new HashMap<>();
-        partitionsPerTopic.put(topic1, 3);
-        partitionsPerTopic.put(topic2, 3);
+        Map partitionsPerTopic = setupPartitionsPerTopicWithTwoTopics(3, 3);
 
         Map consumers = new HashMap<>();
         consumers.put(consumer1, new Subscription(topics(topic1, topic2)));
@@ -178,11 +178,155 @@ public void testTwoConsumersTwoTopicsSixPartitions() {
         assertAssignment(partitions(tp(topic1, 2), tp(topic2, 2)), assignment.get(consumer2));
     }
 
+    @Test
+    public void testTwoStaticConsumersTwoTopicsSixPartitions() {
+        // although consumer high has a higher rank than consumer low, the comparison happens on
+        // instance id level.
+        String consumerIdLow = "consumer-b";
+        String consumerIdHigh = "consumer-a";
+
+        Map partitionsPerTopic = setupPartitionsPerTopicWithTwoTopics(3, 3);
+
+        Map consumers = new HashMap<>();
+        Subscription consumerLowSubscription = new Subscription(topics(topic1, topic2),
+                                                              null,
+                                                              Collections.emptyList());
+        consumerLowSubscription.setGroupInstanceId(Optional.of(instance1));
+        consumers.put(consumerIdLow, consumerLowSubscription);
+        Subscription consumerHighSubscription = new Subscription(topics(topic1, topic2),
+                                                              null,
+                                                              Collections.emptyList());
+        consumerHighSubscription.setGroupInstanceId(Optional.of(instance2));
+        consumers.put(consumerIdHigh, consumerHighSubscription);
+        Map> assignment = assignor.assign(partitionsPerTopic, consumers);
+        assertAssignment(partitions(tp(topic1, 0), tp(topic1, 1), tp(topic2, 0), tp(topic2, 1)), assignment.get(consumerIdLow));
+        assertAssignment(partitions(tp(topic1, 2), tp(topic2, 2)), assignment.get(consumerIdHigh));
+    }
+
+    @Test
+    public void testOneStaticConsumerAndOneDynamicConsumerTwoTopicsSixPartitions() {
+        // although consumer high has a higher rank than low, consumer low will win the comparison
+        // because it has instance id while consumer 2 doesn't.
+        String consumerIdLow = "consumer-b";
+        String consumerIdHigh = "consumer-a";
+
+        Map partitionsPerTopic = setupPartitionsPerTopicWithTwoTopics(3, 3);
+
+        Map consumers = new HashMap<>();
+
+        Subscription consumerLowSubscription = new Subscription(topics(topic1, topic2),
+                                                              null,
+                                                              Collections.emptyList());
+        consumerLowSubscription.setGroupInstanceId(Optional.of(instance1));
+        consumers.put(consumerIdLow, consumerLowSubscription);
+        consumers.put(consumerIdHigh, new Subscription(topics(topic1, topic2)));
+
+        Map> assignment = assignor.assign(partitionsPerTopic, consumers);
+        assertAssignment(partitions(tp(topic1, 0), tp(topic1, 1), tp(topic2, 0), tp(topic2, 1)), assignment.get(consumerIdLow));
+        assertAssignment(partitions(tp(topic1, 2), tp(topic2, 2)), assignment.get(consumerIdHigh));
+    }
+
+    @Test
+    public void testStaticMemberRangeAssignmentPersistent() {
+        Map partitionsPerTopic = setupPartitionsPerTopicWithTwoTopics(5, 4);
+
+        Map consumers = new HashMap<>();
+        for (MemberInfo m : staticMemberInfos) {
+            Subscription subscription = new Subscription(topics(topic1, topic2),
+                                                         null,
+                                                         Collections.emptyList());
+            subscription.setGroupInstanceId(m.groupInstanceId);
+            consumers.put(m.memberId, subscription);
+        }
+        // Consumer 4 is a dynamic member.
+        String consumer4 = "consumer4";
+        consumers.put(consumer4, new Subscription(topics(topic1, topic2)));
+
+        Map> expectedAssignment = new HashMap<>();
+        // Have 3 static members instance1, instance2, instance3 to be persistent
+        // across generations. Their assignment shall be the same.
+        expectedAssignment.put(consumer1, partitions(tp(topic1, 0), tp(topic1, 1), tp(topic2, 0)));
+        expectedAssignment.put(consumer2, partitions(tp(topic1, 2), tp(topic2, 1)));
+        expectedAssignment.put(consumer3, partitions(tp(topic1, 3), tp(topic2, 2)));
+        expectedAssignment.put(consumer4, partitions(tp(topic1, 4), tp(topic2, 3)));
+
+        Map> assignment = assignor.assign(partitionsPerTopic, consumers);
+        assertEquals(expectedAssignment, assignment);
+
+        // Replace dynamic member 4 with a new dynamic member 5.
+        consumers.remove(consumer4);
+        String consumer5 = "consumer5";
+        consumers.put(consumer5, new Subscription(topics(topic1, topic2)));
+
+        expectedAssignment.remove(consumer4);
+        expectedAssignment.put(consumer5, partitions(tp(topic1, 4), tp(topic2, 3)));
+        assignment = assignor.assign(partitionsPerTopic, consumers);
+        assertEquals(expectedAssignment, assignment);
+    }
+
+    @Test
+    public void testStaticMemberRangeAssignmentPersistentAfterMemberIdChanges() {
+        Map partitionsPerTopic = setupPartitionsPerTopicWithTwoTopics(5, 5);
+
+        Map consumers = new HashMap<>();
+        for (MemberInfo m : staticMemberInfos) {
+            Subscription subscription = new Subscription(topics(topic1, topic2),
+                                                         null,
+                                                         Collections.emptyList());
+            subscription.setGroupInstanceId(m.groupInstanceId);
+            consumers.put(m.memberId, subscription);
+        }
+        Map> expectedInstanceAssignment = new HashMap<>();
+        expectedInstanceAssignment.put(instance1,
+                                       partitions(tp(topic1, 0), tp(topic1, 1), tp(topic2, 0), tp(topic2, 1)));
+        expectedInstanceAssignment.put(instance2,
+                                       partitions(tp(topic1, 2), tp(topic1, 3), tp(topic2, 2), tp(topic2, 3)));
+        expectedInstanceAssignment.put(instance3,
+                                       partitions(tp(topic1, 4), tp(topic2, 4)));
+
+        Map> staticAssignment =
+            checkStaticAssignment(assignor, partitionsPerTopic, consumers);
+        assertEquals(expectedInstanceAssignment, staticAssignment);
+
+        // Now switch the member.id fields for each member info, the assignment should
+        // stay the same as last time.
+        String consumer4 = "consumer4";
+        String consumer5 = "consumer5";
+        consumers.put(consumer4, consumers.get(consumer3));
+        consumers.remove(consumer3);
+        consumers.put(consumer5, consumers.get(consumer2));
+        consumers.remove(consumer2);
+
+        Map> newStaticAssignment =
+            checkStaticAssignment(assignor, partitionsPerTopic, consumers);
+        assertEquals(staticAssignment, newStaticAssignment);
+    }
+
+    static Map> checkStaticAssignment(AbstractPartitionAssignor assignor,
+                                                                   Map partitionsPerTopic,
+                                                                   Map consumers) {
+        Map> assignmentByMemberId = assignor.assign(partitionsPerTopic, consumers);
+        Map> assignmentByInstanceId = new HashMap<>();
+        for (Map.Entry entry : consumers.entrySet()) {
+            String memberId = entry.getKey();
+            Optional instanceId = entry.getValue().groupInstanceId();
+            instanceId.ifPresent(id -> assignmentByInstanceId.put(id, assignmentByMemberId.get(memberId)));
+        }
+        return assignmentByInstanceId;
+    }
+
     private void assertAssignment(List expected, List actual) {
         // order doesn't matter for assignment, so convert to a set
         assertEquals(new HashSet<>(expected), new HashSet<>(actual));
     }
 
+    private Map setupPartitionsPerTopicWithTwoTopics(int numberOfPartitions1, int numberOfPartitions2) {
+        Map partitionsPerTopic = new HashMap<>();
+        partitionsPerTopic.put(topic1, numberOfPartitions1);
+        partitionsPerTopic.put(topic2, numberOfPartitions2);
+        return partitionsPerTopic;
+    }
+
     private static List topics(String... topics) {
         return Arrays.asList(topics);
     }
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/RoundRobinAssignorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/RoundRobinAssignorTest.java
index e7622807834ad..5358a814d4305 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/RoundRobinAssignorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/RoundRobinAssignorTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.clients.consumer;
 
+import org.apache.kafka.clients.consumer.internals.AbstractPartitionAssignor;
 import org.apache.kafka.clients.consumer.internals.AbstractPartitionAssignor.MemberInfo;
 import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription;
 import org.apache.kafka.common.TopicPartition;
@@ -29,6 +30,7 @@
 import java.util.Map;
 import java.util.Optional;
 
+import static org.apache.kafka.clients.consumer.RangeAssignorTest.checkStaticAssignment;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -38,6 +40,9 @@ public class RoundRobinAssignorTest {
     private String topic = "topic";
     private String consumerId = "consumer";
 
+    private String topic1 = "topic1";
+    private String topic2 = "topic2";
+
     @Test
     public void testOneConsumerNoTopic() {
         Map partitionsPerTopic = new HashMap<>();
@@ -83,12 +88,7 @@ public void testOnlyAssignsPartitionsFromSubscribedTopics() {
 
     @Test
     public void testOneConsumerMultipleTopics() {
-        String topic1 = "topic1";
-        String topic2 = "topic2";
-
-        Map partitionsPerTopic = new HashMap<>();
-        partitionsPerTopic.put(topic1, 1);
-        partitionsPerTopic.put(topic2, 2);
+        Map partitionsPerTopic = setupPartitionsPerTopicWithTwoTopics(1, 2);
 
         Map> assignment = assignor.assign(partitionsPerTopic,
                 Collections.singletonMap(consumerId, new Subscription(topics(topic1, topic2))));
@@ -137,9 +137,7 @@ public void testMultipleConsumersMixedTopics() {
         String consumer2 = "consumer2";
         String consumer3 = "consumer3";
 
-        Map partitionsPerTopic = new HashMap<>();
-        partitionsPerTopic.put(topic1, 3);
-        partitionsPerTopic.put(topic2, 2);
+        Map partitionsPerTopic = setupPartitionsPerTopicWithTwoTopics(3, 2);
 
         Map consumers = new HashMap<>();
         consumers.put(consumer1, new Subscription(topics(topic1)));
@@ -159,9 +157,7 @@ public void testTwoDynamicConsumersTwoTopicsSixPartitions() {
         String consumer1 = "consumer1";
         String consumer2 = "consumer2";
 
-        Map partitionsPerTopic = new HashMap<>();
-        partitionsPerTopic.put(topic1, 3);
-        partitionsPerTopic.put(topic2, 3);
+        Map partitionsPerTopic = setupPartitionsPerTopicWithTwoTopics(3, 3);
 
         Map consumers = new HashMap<>();
         consumers.put(consumer1, new Subscription(topics(topic1, topic2)));
@@ -183,9 +179,7 @@ public void testTwoStaticConsumersTwoTopicsSixPartitions() {
         String consumer2 = "consumer-a";
         String instance2 = "instance2";
 
-        Map partitionsPerTopic = new HashMap<>();
-        partitionsPerTopic.put(topic1, 3);
-        partitionsPerTopic.put(topic2, 3);
+        Map partitionsPerTopic = setupPartitionsPerTopicWithTwoTopics(3, 3);
 
         Map consumers = new HashMap<>();
         Subscription consumer1Subscription = new Subscription(topics(topic1, topic2), null);
@@ -203,15 +197,11 @@ public void testTwoStaticConsumersTwoTopicsSixPartitions() {
     public void testOneStaticConsumerAndOneDynamicConsumerTwoTopicsSixPartitions() {
         // although consumer 2 has a higher rank than 1, consumer 1 will win the comparison
         // because it has instance id while consumer 2 doesn't.
-        String topic1 = "topic1";
-        String topic2 = "topic2";
         String consumer1 = "consumer-b";
         String instance1 = "instance1";
         String consumer2 = "consumer-a";
 
-        Map partitionsPerTopic = new HashMap<>();
-        partitionsPerTopic.put(topic1, 3);
-        partitionsPerTopic.put(topic2, 3);
+        Map partitionsPerTopic = setupPartitionsPerTopicWithTwoTopics(3, 3);
 
         Map consumers = new HashMap<>();
 
@@ -226,11 +216,9 @@ public void testOneStaticConsumerAndOneDynamicConsumerTwoTopicsSixPartitions() {
     }
 
     @Test
-    public void testStaticMemberAssignmentPersistent() {
+    public void testStaticMemberRoundRobinAssignmentPersistent() {
         // Have 3 static members instance1, instance2, instance3 to be persistent
         // across generations. Their assignment shall be the same.
-        String topic1 = "topic1";
-        String topic2 = "topic2";
         String consumer1 = "consumer1";
         String instance1 = "instance1";
         String consumer2 = "consumer2";
@@ -246,9 +234,8 @@ public void testStaticMemberAssignmentPersistent() {
         // Consumer 4 is a dynamic member.
         String consumer4 = "consumer4";
 
-        Map partitionsPerTopic = new HashMap<>();
-        partitionsPerTopic.put(topic1, 3);
-        partitionsPerTopic.put(topic2, 3);
+        Map partitionsPerTopic = setupPartitionsPerTopicWithTwoTopics(3, 3);
+
         Map consumers = new HashMap<>();
         for (MemberInfo m : staticMemberInfos) {
             Subscription subscription = new Subscription(topics(topic1, topic2), null);
@@ -278,9 +265,7 @@ public void testStaticMemberAssignmentPersistent() {
     }
 
     @Test
-    public void testStaticMemberAssignmentPersistentAfterMemberIdChanges() {
-        String topic1 = "topic1";
-        String topic2 = "topic2";
+    public void testStaticMemberRoundRobinAssignmentPersistentAfterMemberIdChanges() {
         String consumer1 = "consumer1";
         String instance1 = "instance1";
         String consumer2 = "consumer2";
@@ -292,37 +277,20 @@ public void testStaticMemberAssignmentPersistentAfterMemberIdChanges() {
         memberIdToInstanceId.put(consumer2, instance2);
         memberIdToInstanceId.put(consumer3, instance3);
 
-        List memberIdList = Arrays.asList(consumer1, consumer2, consumer3);
-        Map> staticAssignment =
-            checkStaticAssignment(topic1, topic2, memberIdList, memberIdToInstanceId);
-        memberIdToInstanceId.clear();
+        Map partitionsPerTopic = setupPartitionsPerTopicWithTwoTopics(5, 5);
 
-        // Now switch the member.id fields for each member info, the assignment should
-        // stay the same as last time.
-        String consumer4 = "consumer4";
-        String consumer5 = "consumer5";
-        memberIdToInstanceId.put(consumer4, instance1);
-        memberIdToInstanceId.put(consumer5, instance2);
-        memberIdToInstanceId.put(consumer1, instance3);
-        memberIdList = Arrays.asList(consumer4, consumer5, consumer1);
-        Map> newStaticAssignment =
-            checkStaticAssignment(topic1, topic2, memberIdList, memberIdToInstanceId);
-
-        assertEquals(staticAssignment, newStaticAssignment);
-    }
-
-    private Map> checkStaticAssignment(String topic1,
-                                                                    String topic2,
-                                                                    List memberIdList,
-                                                                    Map memberIdToInstanceId) {
-        List staticMemberInfos = new ArrayList<>();
+        Map> expectedInstanceAssignment = new HashMap<>();
+        expectedInstanceAssignment.put(instance1,
+                                       partitions(tp(topic1, 0), tp(topic1, 3), tp(topic2, 1), tp(topic2, 4)));
+        expectedInstanceAssignment.put(instance2,
+                                       partitions(tp(topic1, 1), tp(topic1, 4), tp(topic2, 2)));
+        expectedInstanceAssignment.put(instance3,
+                                       partitions(tp(topic1, 2), tp(topic2, 0), tp(topic2, 3)));
+
+        List staticMemberInfos = new ArrayList<>();
         for (Map.Entry entry : memberIdToInstanceId.entrySet()) {
-            staticMemberInfos.add(new MemberInfo(entry.getKey(), Optional.of(entry.getValue())));
+            staticMemberInfos.add(new AbstractPartitionAssignor.MemberInfo(entry.getKey(), Optional.of(entry.getValue())));
         }
-
-        Map partitionsPerTopic = new HashMap<>();
-        partitionsPerTopic.put(topic1, 3);
-        partitionsPerTopic.put(topic2, 3);
         Map consumers = new HashMap<>();
         for (MemberInfo m : staticMemberInfos) {
             Subscription subscription = new Subscription(topics(topic1, topic2), null);
@@ -330,21 +298,23 @@ private Map> checkStaticAssignment(String topic1,
             consumers.put(m.memberId, subscription);
         }
 
-        Map> expectedInstanceAssignment = new HashMap<>();
-        for (int i = 0; i < memberIdList.size(); i++) {
-            expectedInstanceAssignment.put(memberIdToInstanceId.get(memberIdList.get(i)),
-                                           partitions(tp(topic1, i), tp(topic2, i)));
-        }
+        Map> staticAssignment =
+            checkStaticAssignment(assignor, partitionsPerTopic, consumers);
+        assertEquals(expectedInstanceAssignment, staticAssignment);
 
-        Map> assignmentByMemberId =
-            assignor.assign(partitionsPerTopic, consumers);
-        Map> assignmentByInstanceId = new HashMap<>();
-        for (String memberId : memberIdList) {
-            assignmentByInstanceId.put(memberIdToInstanceId.get(memberId),
-                                       assignmentByMemberId.get(memberId));
-        }
-        assertEquals(expectedInstanceAssignment, assignmentByInstanceId);
-        return assignmentByInstanceId;
+        memberIdToInstanceId.clear();
+
+        // Now switch the member.id fields for each member info, the assignment should
+        // stay the same as last time.
+        String consumer4 = "consumer4";
+        String consumer5 = "consumer5";
+        consumers.put(consumer4, consumers.get(consumer3));
+        consumers.remove(consumer3);
+        consumers.put(consumer5, consumers.get(consumer2));
+        consumers.remove(consumer2);
+        Map> newStaticAssignment =
+            checkStaticAssignment(assignor, partitionsPerTopic, consumers);
+        assertEquals(staticAssignment, newStaticAssignment);
     }
 
     private static List topics(String... topics) {
@@ -358,4 +328,11 @@ private static List partitions(TopicPartition... partitions) {
     private static TopicPartition tp(String topic, int partition) {
         return new TopicPartition(topic, partition);
     }
+
+    private Map setupPartitionsPerTopicWithTwoTopics(int numberOfPartitions1, int numberOfPartitions2) {
+        Map partitionsPerTopic = new HashMap<>();
+        partitionsPerTopic.put(topic1, numberOfPartitions1);
+        partitionsPerTopic.put(topic2, numberOfPartitions2);
+        return partitionsPerTopic;
+    }
 }

From 6530600e6b314565f100b65ae774f3f409ed9828 Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Fri, 13 Sep 2019 13:36:58 -0500
Subject: [PATCH 0609/1071] MINOR: Add UUID type to Kafka API code generation
 (#7291)

Reviewers: Colin P. McCabe 
---
 .gitignore                                    |  1 +
 build.gradle                                  | 12 ++-
 .../kafka/common/protocol/MessageUtil.java    |  3 +
 .../kafka/common/protocol/Readable.java       |  8 ++
 .../kafka/common/protocol/Writable.java       |  9 ++
 .../kafka/common/protocol/types/Field.java    | 10 +++
 .../kafka/common/protocol/types/Struct.java   | 23 +++++
 .../kafka/common/protocol/types/Type.java     | 39 +++++++++
 .../common/message/TestUUIDDataTest.java      | 86 +++++++++++++++++++
 .../resources/common/message/TestUUID.json    | 26 ++++++
 .../message/ApiMessageTypeGenerator.java      |  4 +
 .../org/apache/kafka/message/FieldType.java   | 41 +++++++++
 .../kafka/message/MessageDataGenerator.java   | 44 +++++++---
 .../kafka/message/MessageGenerator.java       | 16 ++--
 .../apache/kafka/message/SchemaGenerator.java |  6 ++
 15 files changed, 311 insertions(+), 17 deletions(-)
 create mode 100644 clients/src/test/java/org/apache/kafka/common/message/TestUUIDDataTest.java
 create mode 100644 clients/src/test/resources/common/message/TestUUID.json

diff --git a/.gitignore b/.gitignore
index a31643f00af6e..3daa2b8b6c8c7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -53,3 +53,4 @@ kafkatest.egg-info/
 systest/
 *.swp
 clients/src/generated
+clients/src/generated-test
diff --git a/build.gradle b/build.gradle
index 323a3ca89e4a9..083cbdfc8485f 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1017,6 +1017,14 @@ project(':clients') {
     outputs.dir("src/generated/java/org/apache/kafka/common/message")
   }
 
+  task processTestMessages(type:JavaExec) {
+    main = "org.apache.kafka.message.MessageGenerator"
+    classpath = project(':generator').sourceSets.main.runtimeClasspath
+    args = [ "src/generated-test/java/org/apache/kafka/common/message", "src/test/resources/common/message" ]
+    inputs.dir("src/test/resources/common/message")
+    outputs.dir("src/generated-test/java/org/apache/kafka/common/message")
+  }
+
   sourceSets {
     main {
       java {
@@ -1025,13 +1033,15 @@ project(':clients') {
     }
     test {
       java {
-        srcDirs = ["src/generated/java", "src/test/java"]
+        srcDirs = ["src/generated/java", "src/generated-test/java", "src/test/java"]
       }
     }
   }
 
   compileJava.dependsOn 'processMessages'
 
+  compileTestJava.dependsOn 'processTestMessages'
+
   javadoc {
     include "**/org/apache/kafka/clients/admin/*"
     include "**/org/apache/kafka/clients/consumer/*"
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java b/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java
index e3fdea7feae3b..a19059f5660a4 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java
@@ -20,8 +20,11 @@
 import org.apache.kafka.common.utils.Utils;
 
 import java.util.Iterator;
+import java.util.UUID;
 
 public final class MessageUtil {
+    public static final UUID ZERO_UUID = new UUID(0L, 0L);
+
     /**
      * Get the length of the UTF8 representation of a string, without allocating
      * a byte buffer for the string.
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Readable.java b/clients/src/main/java/org/apache/kafka/common/protocol/Readable.java
index a527239ca486c..e966bde1662a7 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/Readable.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/Readable.java
@@ -18,6 +18,7 @@
 package org.apache.kafka.common.protocol;
 
 import java.nio.charset.StandardCharsets;
+import java.util.UUID;
 
 public interface Readable {
     byte readByte();
@@ -54,4 +55,11 @@ default byte[] readNullableBytes() {
         readArray(arr);
         return arr;
     }
+
+    /**
+     * Read a UUID with the most significant digits first.
+     */
+    default UUID readUUID() {
+        return new UUID(readLong(), readLong());
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Writable.java b/clients/src/main/java/org/apache/kafka/common/protocol/Writable.java
index 9478ed3ec6225..e929e53bc6244 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/Writable.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/Writable.java
@@ -18,6 +18,7 @@
 package org.apache.kafka.common.protocol;
 
 import java.nio.charset.StandardCharsets;
+import java.util.UUID;
 
 public interface Writable {
     void writeByte(byte val);
@@ -68,4 +69,12 @@ default void writeString(String string) {
         writeShort((short) arr.length);
         writeArray(arr);
     }
+
+    /**
+     * Write a UUID with the most significant digits first.
+     */
+    default void writeUUID(UUID uuid) {
+        writeLong(uuid.getMostSignificantBits());
+        writeLong(uuid.getLeastSignificantBits());
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Field.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Field.java
index 72e051c179e73..b79bc932b4203 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Field.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Field.java
@@ -75,6 +75,16 @@ public Int64(String name, String docString, long defaultValue) {
         }
     }
 
+    public static class UUID extends Field {
+        public UUID(String name, String docString) {
+            super(name, Type.UUID, docString, false, null);
+        }
+
+        public UUID(String name, String docString, UUID defaultValue) {
+            super(name, Type.UUID, docString, true, defaultValue);
+        }
+    }
+
     public static class Int16 extends Field {
         public Int16(String name, String docString) {
             super(name, Type.INT16, docString, false, null);
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
index e47a2cdce2a6a..350497cf4fc84 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
@@ -21,6 +21,7 @@
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.Objects;
+import java.util.UUID;
 
 /**
  * A record that can be serialized and deserialized according to a pre-defined schema
@@ -88,6 +89,10 @@ public Long get(Field.Int64 field) {
         return getLong(field.name);
     }
 
+    public UUID get(Field.UUID field) {
+        return getUUID(field.name);
+    }
+
     public Short get(Field.Int16 field) {
         return getShort(field.name);
     }
@@ -118,6 +123,12 @@ public Long getOrElse(Field.Int64 field, long alternative) {
         return alternative;
     }
 
+    public UUID getOrElse(Field.UUID field, UUID alternative) {
+        if (hasField(field.name))
+            return getUUID(field.name);
+        return alternative;
+    }
+
     public Short getOrElse(Field.Int16 field, short alternative) {
         if (hasField(field.name))
             return getShort(field.name);
@@ -245,6 +256,14 @@ public Long getLong(String name) {
         return (Long) get(name);
     }
 
+    public UUID getUUID(BoundField field) {
+        return (UUID) get(field);
+    }
+
+    public UUID getUUID(String name) {
+        return (UUID) get(name);
+    }
+
     public Object[] getArray(BoundField field) {
         return (Object[]) get(field);
     }
@@ -342,6 +361,10 @@ public Struct set(Field.Int64 def, long value) {
         return set(def.name, value);
     }
 
+    public Struct set(Field.UUID def, UUID value) {
+        return set(def.name, value);
+    }
+
     public Struct set(Field.Int16 def, short value) {
         return set(def.name, value);
     }
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java
index 4bd508b77989f..df457463f06c8 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java
@@ -23,6 +23,7 @@
 import org.apache.kafka.common.utils.Utils;
 
 import java.nio.ByteBuffer;
+import java.util.UUID;
 
 /**
  * A serializable type
@@ -313,6 +314,44 @@ public String documentation() {
         }
     };
 
+    public static final DocumentedType UUID = new DocumentedType() {
+        @Override
+        public void write(ByteBuffer buffer, Object o) {
+            final java.util.UUID uuid = (java.util.UUID) o;
+            buffer.putLong(uuid.getMostSignificantBits());
+            buffer.putLong(uuid.getLeastSignificantBits());
+        }
+
+        @Override
+        public Object read(ByteBuffer buffer) {
+            return new java.util.UUID(buffer.getLong(), buffer.getLong());
+        }
+
+        @Override
+        public int sizeOf(Object o) {
+            return 16;
+        }
+
+        @Override
+        public String typeName() {
+            return "UUID";
+        }
+
+        @Override
+        public UUID validate(Object item) {
+            if (item instanceof UUID)
+                return (UUID) item;
+            else
+                throw new SchemaException(item + " is not a UUID.");
+        }
+
+        @Override
+        public String documentation() {
+            return "Represents a java.util.UUID. " +
+                    "The values are encoded using sixteen bytes in network byte order (big-endian).";
+        }
+    };
+
     public static final DocumentedType STRING = new DocumentedType() {
         @Override
         public void write(ByteBuffer buffer, Object o) {
diff --git a/clients/src/test/java/org/apache/kafka/common/message/TestUUIDDataTest.java b/clients/src/test/java/org/apache/kafka/common/message/TestUUIDDataTest.java
new file mode 100644
index 0000000000000..4b8e36b127190
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/message/TestUUIDDataTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.message;
+
+import org.apache.kafka.common.protocol.ByteBufferAccessor;
+import org.apache.kafka.common.protocol.types.Struct;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.util.UUID;
+
+public class TestUUIDDataTest {
+
+    @Test
+    public void shouldStoreField() {
+        final UUID uuid = UUID.randomUUID();
+        final TestUUIDData out = new TestUUIDData();
+        out.setProcessId(uuid);
+        Assert.assertEquals(uuid, out.processId());
+    }
+
+    @Test
+    public void shouldDefaultField() {
+        final TestUUIDData out = new TestUUIDData();
+        Assert.assertEquals(UUID.fromString("00000000-0000-0000-0000-000000000000"), out.processId());
+    }
+
+    @Test
+    public void shouldRoundTripFieldThroughStruct() {
+        final UUID uuid = UUID.randomUUID();
+        final TestUUIDData out = new TestUUIDData();
+        out.setProcessId(uuid);
+
+        final Struct struct = out.toStruct((short) 1);
+        final TestUUIDData in = new TestUUIDData();
+        in.fromStruct(struct, (short) 1);
+
+        Assert.assertEquals(uuid, in.processId());
+    }
+
+    @Test
+    public void shouldRoundTripFieldThroughBuffer() {
+        final UUID uuid = UUID.randomUUID();
+        final TestUUIDData out = new TestUUIDData();
+        out.setProcessId(uuid);
+
+        final ByteBuffer buffer = ByteBuffer.allocate(out.size((short) 1));
+        out.write(new ByteBufferAccessor(buffer), (short) 1);
+        buffer.rewind();
+
+        final TestUUIDData in = new TestUUIDData();
+        in.read(new ByteBufferAccessor(buffer), (short) 1);
+
+        Assert.assertEquals(uuid, in.processId());
+    }
+
+    @Test
+    public void shouldImplementJVMMethods() {
+        final UUID uuid = UUID.randomUUID();
+        final TestUUIDData a = new TestUUIDData();
+        a.setProcessId(uuid);
+
+        final TestUUIDData b = new TestUUIDData();
+        b.setProcessId(uuid);
+
+        Assert.assertEquals(a, b);
+        Assert.assertEquals(a.hashCode(), b.hashCode());
+        // just tagging this on here
+        Assert.assertEquals(a.toString(), b.toString());
+    }
+}
diff --git a/clients/src/test/resources/common/message/TestUUID.json b/clients/src/test/resources/common/message/TestUUID.json
new file mode 100644
index 0000000000000..8c1c7bd8f06ec
--- /dev/null
+++ b/clients/src/test/resources/common/message/TestUUID.json
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+{
+  "name": "TestUUID",
+  "validVersions": "1",
+  "fields": [
+    {
+      "name": "processId",
+      "versions": "1",
+      "type": "uuid"
+    }
+  ],
+  "type": "header"
+}
\ No newline at end of file
diff --git a/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java b/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java
index f5d00a697008c..c5439aa2e81c8 100644
--- a/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java
+++ b/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java
@@ -73,6 +73,10 @@ public ApiMessageTypeGenerator() {
         this.buffer = new CodeBuffer();
     }
 
+    public boolean hasRegisteredTypes() {
+        return !apis.isEmpty();
+    }
+
     public void registerMessageType(MessageSpec spec) {
         switch (spec.type()) {
             case REQUEST: {
diff --git a/generator/src/main/java/org/apache/kafka/message/FieldType.java b/generator/src/main/java/org/apache/kafka/message/FieldType.java
index d98b8f39c0500..8f0c3634ace69 100644
--- a/generator/src/main/java/org/apache/kafka/message/FieldType.java
+++ b/generator/src/main/java/org/apache/kafka/message/FieldType.java
@@ -41,6 +41,11 @@ final class Int8FieldType implements FieldType {
         static final Int8FieldType INSTANCE = new Int8FieldType();
         private static final String NAME = "int8";
 
+        @Override
+        public boolean isInteger() {
+            return true;
+        }
+
         @Override
         public Optional fixedLength() {
             return Optional.of(1);
@@ -56,6 +61,11 @@ final class Int16FieldType implements FieldType {
         static final Int16FieldType INSTANCE = new Int16FieldType();
         private static final String NAME = "int16";
 
+        @Override
+        public boolean isInteger() {
+            return true;
+        }
+
         @Override
         public Optional fixedLength() {
             return Optional.of(2);
@@ -71,6 +81,11 @@ final class Int32FieldType implements FieldType {
         static final Int32FieldType INSTANCE = new Int32FieldType();
         private static final String NAME = "int32";
 
+        @Override
+        public boolean isInteger() {
+            return true;
+        }
+
         @Override
         public Optional fixedLength() {
             return Optional.of(4);
@@ -86,6 +101,11 @@ final class Int64FieldType implements FieldType {
         static final Int64FieldType INSTANCE = new Int64FieldType();
         private static final String NAME = "int64";
 
+        @Override
+        public boolean isInteger() {
+            return true;
+        }
+
         @Override
         public Optional fixedLength() {
             return Optional.of(8);
@@ -97,6 +117,21 @@ public String toString() {
         }
     }
 
+    final class UUIDFieldType implements FieldType {
+        static final UUIDFieldType INSTANCE = new UUIDFieldType();
+        private static final String NAME = "uuid";
+
+        @Override
+        public Optional fixedLength() {
+            return Optional.of(16);
+        }
+
+        @Override
+        public String toString() {
+            return NAME;
+        }
+    }
+
     final class StringFieldType implements FieldType {
         static final StringFieldType INSTANCE = new StringFieldType();
         private static final String NAME = "string";
@@ -204,6 +239,8 @@ static FieldType parse(String string) {
                 return Int32FieldType.INSTANCE;
             case Int64FieldType.NAME:
                 return Int64FieldType.INSTANCE;
+            case UUIDFieldType.NAME:
+                return UUIDFieldType.INSTANCE;
             case StringFieldType.NAME:
                 return StringFieldType.INSTANCE;
             case BytesFieldType.NAME:
@@ -229,6 +266,10 @@ static FieldType parse(String string) {
         }
     }
 
+    default boolean isInteger() {
+        return false;
+    }
+
     /**
      * Returns true if this is an array type.
      */
diff --git a/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java b/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java
index d0e5044f1f292..6a2056077c6f8 100644
--- a/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java
+++ b/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java
@@ -22,6 +22,7 @@
 import java.util.Iterator;
 import java.util.Optional;
 import java.util.Set;
+import java.util.UUID;
 import java.util.stream.Collectors;
 
 /**
@@ -311,6 +312,8 @@ private String fieldAbstractJavaType(FieldSpec field) {
             return "int";
         } else if (field.type() instanceof FieldType.Int64FieldType) {
             return "long";
+        } else if (field.type() instanceof FieldType.UUIDFieldType) {
+            return "UUID";
         } else if (field.type().isString()) {
             return "String";
         } else if (field.type().isBytes()) {
@@ -475,6 +478,8 @@ private String readFieldFromReadable(FieldType type) {
             return "readable.readInt()";
         } else if (type instanceof FieldType.Int64FieldType) {
             return "readable.readLong()";
+        } else if (type instanceof FieldType.UUIDFieldType) {
+            return "readable.readUUID()";
         } else if (type.isString()) {
             return "readable.readNullableString()";
         } else if (type.isBytes()) {
@@ -581,6 +586,8 @@ private String getBoxedJavaType(FieldType type) {
             return "Integer";
         } else if (type instanceof FieldType.Int64FieldType) {
             return "Long";
+        } else if (type instanceof FieldType.UUIDFieldType) {
+            return "UUID";
         } else if (type.isString()) {
             return "String";
         } else if (type.isStruct()) {
@@ -601,6 +608,8 @@ private String readFieldFromStruct(FieldType type, String name) {
             return String.format("struct.getInt(\"%s\")", name);
         } else if (type instanceof FieldType.Int64FieldType) {
             return String.format("struct.getLong(\"%s\")", name);
+        } else if (type instanceof FieldType.UUIDFieldType) {
+            return String.format("struct.getUUID(\"%s\")", name);
         } else if (type.isString()) {
             return String.format("struct.getString(\"%s\")", name);
         } else if (type.isBytes()) {
@@ -649,6 +658,8 @@ private String writeFieldToWritable(FieldType type, boolean nullable, String nam
             return String.format("writable.writeInt(%s)", name);
         } else if (type instanceof FieldType.Int64FieldType) {
             return String.format("writable.writeLong(%s)", name);
+        } else if (type instanceof FieldType.UUIDFieldType) {
+            return String.format("writable.writeUUID(%s)", name);
         } else if (type instanceof FieldType.StringFieldType) {
             if (nullable) {
                 return String.format("writable.writeNullableString(%s)", name);
@@ -746,10 +757,8 @@ private void generateFieldToStruct(FieldSpec field, Versions curVersions) {
                 " cannot be nullable.");
         }
         if ((field.type() instanceof FieldType.BoolFieldType) ||
-                (field.type() instanceof FieldType.Int8FieldType) ||
-                (field.type() instanceof FieldType.Int16FieldType) ||
-                (field.type() instanceof FieldType.Int32FieldType) ||
-                (field.type() instanceof FieldType.Int64FieldType) ||
+                (field.type().isInteger()) ||
+                (field.type() instanceof FieldType.UUIDFieldType) ||
                 (field.type() instanceof FieldType.StringFieldType)) {
             boolean maybeAbsent =
                 generateVersionCheck(curVersions, field.versions());
@@ -1039,16 +1048,17 @@ private void generateFieldHashCode(FieldSpec field) {
         } else if (field.type() instanceof FieldType.Int64FieldType) {
             buffer.printf("hashCode = 31 * hashCode + ((int) (%s >> 32) ^ (int) %s);%n",
                 field.camelCaseName(), field.camelCaseName());
-        } else if (field.type().isString()) {
-            buffer.printf("hashCode = 31 * hashCode + (%s == null ? 0 : %s.hashCode());%n",
-                field.camelCaseName(), field.camelCaseName());
         } else if (field.type().isBytes()) {
             headerGenerator.addImport(MessageGenerator.ARRAYS_CLASS);
             buffer.printf("hashCode = 31 * hashCode + Arrays.hashCode(%s);%n",
                 field.camelCaseName());
-        } else if (field.type().isStruct() || field.type().isArray()) {
+        } else if (field.type().isStruct()
+                   || field.type().isArray()
+                   || field.type().isString()
+                   || field.type() instanceof  FieldType.UUIDFieldType
+        ) {
             buffer.printf("hashCode = 31 * hashCode + (%s == null ? 0 : %s.hashCode());%n",
-                field.camelCaseName(), field.camelCaseName());
+                          field.camelCaseName(), field.camelCaseName());
         } else {
             throw new RuntimeException("Unsupported field type " + field.type());
         }
@@ -1078,7 +1088,8 @@ private void generateFieldToString(String prefix, FieldSpec field) {
         } else if ((field.type() instanceof FieldType.Int8FieldType) ||
                 (field.type() instanceof FieldType.Int16FieldType) ||
                 (field.type() instanceof FieldType.Int32FieldType) ||
-                (field.type() instanceof FieldType.Int64FieldType)) {
+                (field.type() instanceof FieldType.Int64FieldType) ||
+                (field.type() instanceof FieldType.UUIDFieldType)) {
             buffer.printf("+ \"%s%s=\" + %s%n",
                 prefix, field.camelCaseName(), field.camelCaseName());
         } else if (field.type().isString()) {
@@ -1227,6 +1238,19 @@ private String fieldDefault(FieldSpec field) {
                 }
                 return field.defaultString() + "L";
             }
+        } else if (field.type() instanceof FieldType.UUIDFieldType) {
+            headerGenerator.addImport(MessageGenerator.UUID_CLASS);
+            if (field.defaultString().isEmpty()) {
+                return "org.apache.kafka.common.protocol.MessageUtil.ZERO_UUID";
+            } else {
+                try {
+                    UUID.fromString(field.defaultString());
+                } catch (IllegalArgumentException e) {
+                    throw new RuntimeException("Invalid default for uuid field " +
+                        field.name() + ": " + field.defaultString(), e);
+                }
+                return "UUID.fromString(\"" + field.defaultString() + "\")";
+            }
         } else if (field.type() instanceof FieldType.StringFieldType) {
             if (field.defaultString().equals("null")) {
                 validateNullDefault(field);
diff --git a/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java b/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java
index e227ded0572c3..df0c5f886b12a 100644
--- a/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java
+++ b/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java
@@ -79,6 +79,8 @@ public final class MessageGenerator {
 
     static final String BYTES_CLASS = "org.apache.kafka.common.utils.Bytes";
 
+    static final String UUID_CLASS = "java.util.UUID";
+
     static final String REQUEST_SUFFIX = "Request";
 
     static final String RESPONSE_SUFFIX = "Response";
@@ -122,13 +124,15 @@ public static void processDirectories(String outputDir, String inputDir) throws
                 }
             }
         }
-        Path factoryOutputPath = Paths.get(outputDir, API_MESSAGE_TYPE_JAVA);
-        outputFileNames.add(API_MESSAGE_TYPE_JAVA);
-        try (BufferedWriter writer = Files.newBufferedWriter(factoryOutputPath)) {
-            messageTypeGenerator.generate();
-            messageTypeGenerator.write(writer);
+        if (messageTypeGenerator.hasRegisteredTypes()) {
+            Path factoryOutputPath = Paths.get(outputDir, API_MESSAGE_TYPE_JAVA);
+            outputFileNames.add(API_MESSAGE_TYPE_JAVA);
+            try (BufferedWriter writer = Files.newBufferedWriter(factoryOutputPath)) {
+                messageTypeGenerator.generate();
+                messageTypeGenerator.write(writer);
+            }
+            numProcessed++;
         }
-        numProcessed++;
         try (DirectoryStream directoryStream = Files.
                 newDirectoryStream(Paths.get(outputDir))) {
             for (Path outputPath : directoryStream) {
diff --git a/generator/src/main/java/org/apache/kafka/message/SchemaGenerator.java b/generator/src/main/java/org/apache/kafka/message/SchemaGenerator.java
index 2ef386867f7e0..3b04b7b11a735 100644
--- a/generator/src/main/java/org/apache/kafka/message/SchemaGenerator.java
+++ b/generator/src/main/java/org/apache/kafka/message/SchemaGenerator.java
@@ -188,6 +188,12 @@ private String fieldTypeToSchemaType(FieldType type, boolean nullable, short ver
                 throw new RuntimeException("Type " + type + " cannot be nullable.");
             }
             return "Type.INT64";
+        } else if (type instanceof FieldType.UUIDFieldType) {
+            headerGenerator.addImport(MessageGenerator.TYPE_CLASS);
+            if (nullable) {
+                throw new RuntimeException("Type " + type + " cannot be nullable.");
+            }
+            return "Type.UUID";
         } else if (type instanceof FieldType.StringFieldType) {
             headerGenerator.addImport(MessageGenerator.TYPE_CLASS);
             return nullable ? "Type.NULLABLE_STRING" : "Type.STRING";

From 83c7c0158fc62c294267f0ae60e9f52945e6a678 Mon Sep 17 00:00:00 2001
From: cpettitt-confluent
 <53191309+cpettitt-confluent@users.noreply.github.com>
Date: Fri, 13 Sep 2019 16:45:29 -0600
Subject: [PATCH 0610/1071] KAFKA-8755: Fix state restore for standby tasks
 with optimized topology (#7238)

Key changes include:

1. Moves general offset limit updates down to StandbyTask.
2. Updates offsets for StandbyTask at most once per commit and only when we need and updated offset limit to make progress.
3. Avoids writing an 0 checkpoint when StandbyTask.update is called but we cannot apply any of the records.
4. Avoids going into a restoring state in the case that the last checkpoint is greater or equal to the offset limit (consumer committed offset). This needs special attention please. Code is in
StoreChangelogReader.
5. Does update offset limits initially for StreamTask because it provides a way to prevent playing to many records from the changelog (also the input topic with optimized topology).

NOTE: this PR depends on KAFKA-8816, which is under review separately. Fortunately the changes involved are few. You can focus just on the KAFKA-8755 commit if you prefer.

Reviewers: Matthias J. Sax , Guozhang Wang 
---
 .../processor/internals/AbstractTask.java     |  37 +-
 .../internals/AssignedStandbyTasks.java       |  10 +
 .../processor/internals/StandbyTask.java      |  67 +++-
 .../internals/StoreChangelogReader.java       |  60 ++--
 .../processor/internals/StreamTask.java       |  42 ++-
 .../OptimizedKTableIntegrationTest.java       | 335 ++++++++++++++++++
 .../processor/internals/AbstractTaskTest.java |  69 +---
 .../processor/internals/StandbyTaskTest.java  |  90 +++++
 .../internals/StoreChangelogReaderTest.java   |  58 +++
 .../processor/internals/StreamTaskTest.java   |  63 +++-
 .../StreamThreadStateStoreProviderTest.java   |   5 +-
 11 files changed, 698 insertions(+), 138 deletions(-)
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/integration/OptimizedKTableIntegrationTest.java

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java
index 5656c914f7bd3..798f0b089a032 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java
@@ -171,26 +171,6 @@ public String toString(final String indent) {
         return sb.toString();
     }
 
-    protected void updateOffsetLimits() {
-        for (final TopicPartition partition : partitions) {
-            try {
-                final OffsetAndMetadata metadata = consumer.committed(partition); // TODO: batch API?
-                final long offset = metadata != null ? metadata.offset() : 0L;
-                stateMgr.putOffsetLimit(partition, offset);
-
-                if (log.isTraceEnabled()) {
-                    log.trace("Updating store offset limits {} for changelog {}", offset, partition);
-                }
-            } catch (final AuthorizationException e) {
-                throw new ProcessorStateException(String.format("task [%s] AuthorizationException when initializing offsets for %s", id, partition), e);
-            } catch (final WakeupException e) {
-                throw e;
-            } catch (final KafkaException e) {
-                throw new ProcessorStateException(String.format("task [%s] Failed to initialize offsets for %s", id, partition), e);
-            }
-        }
-    }
-
     /**
      * Flush all state stores owned by this task
      */
@@ -219,9 +199,6 @@ void registerStateStores() {
         }
         log.trace("Initializing state stores");
 
-        // set initial offset limits
-        updateOffsetLimits();
-
         for (final StateStore store : topology.stateStores()) {
             log.trace("Initializing store {}", store.name());
             processorContext.uninitialize();
@@ -272,4 +249,18 @@ public boolean hasStateStores() {
     public Collection changelogPartitions() {
         return stateMgr.changelogPartitions();
     }
+
+    long committedOffsetForPartition(final TopicPartition partition) {
+        try {
+            final OffsetAndMetadata metadata = consumer.committed(partition);
+            return metadata != null ? metadata.offset() : 0L;
+        } catch (final AuthorizationException e) {
+            throw new ProcessorStateException(String.format("task [%s] AuthorizationException when initializing offsets for %s", id, partition), e);
+        } catch (final WakeupException e) {
+            throw e;
+        } catch (final KafkaException e) {
+            throw new ProcessorStateException(String.format("task [%s] Failed to initialize offsets for %s", id, partition), e);
+        }
+    }
+
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStandbyTasks.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStandbyTasks.java
index a99e45147b9ec..6025e885a1324 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStandbyTasks.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStandbyTasks.java
@@ -24,4 +24,14 @@ class AssignedStandbyTasks extends AssignedTasks {
         super(logContext, "standby task");
     }
 
+    @Override
+    int commit() {
+        final int committed = super.commit();
+        // TODO: this contortion would not be necessary if we got rid of the two-step
+        // task.commitNeeded and task.commit and instead just had task.commitIfNeeded. Currently
+        // we only call commit if commitNeeded is true, which means that we need a way to indicate
+        // that we are eligible for updating the offset limit outside of commit.
+        running.forEach((id, task) -> task.allowUpdateOfOffsetLimit());
+        return committed;
+    }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
index 424ded4381c80..c758ccd2e7b23 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
@@ -16,6 +16,14 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.common.TopicPartition;
@@ -25,20 +33,14 @@
 import org.apache.kafka.streams.processor.TaskId;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
 /**
  * A StandbyTask
  */
 public class StandbyTask extends AbstractTask {
-
     private Map checkpointedOffsets = new HashMap<>();
     private final Sensor closeTaskSensor;
+    private final Map offsetLimits = new HashMap<>();
+    private final Set updateableOffsetLimits = new HashSet<>();
 
     /**
      * Create {@link StandbyTask} with its assigned partitions
@@ -63,6 +65,14 @@ public class StandbyTask extends AbstractTask {
 
         closeTaskSensor = metrics.threadLevelSensor("task-closed", Sensor.RecordingLevel.INFO);
         processorContext = new StandbyContextImpl(id, config, stateMgr, metrics);
+
+        final Set changelogTopicNames = new HashSet<>(topology.storeToChangelogTopic().values());
+        partitions.stream()
+            .filter(tp -> changelogTopicNames.contains(tp.topic()))
+            .forEach(tp -> {
+                offsetLimits.put(tp, 0L);
+                updateableOffsetLimits.add(tp);
+            });
     }
 
     @Override
@@ -88,7 +98,7 @@ public void initializeTopology() {
     @Override
     public void resume() {
         log.debug("Resuming");
-        updateOffsetLimits();
+        allowUpdateOfOffsetLimit();
     }
 
     /**
@@ -102,9 +112,7 @@ public void resume() {
     public void commit() {
         log.trace("Committing");
         flushAndCheckpointState();
-        // reinitialize offset limits
-        updateOffsetLimits();
-
+        allowUpdateOfOffsetLimit();
         commitNeeded = false;
     }
 
@@ -165,14 +173,25 @@ public void closeSuspended(final boolean clean,
      */
     public List> update(final TopicPartition partition,
                                                        final List> records) {
+        if (records.isEmpty()) {
+            return Collections.emptyList();
+        }
+
         log.trace("Updating standby replicas of its state store for partition [{}]", partition);
-        final long limit = stateMgr.offsetLimit(partition);
+        long limit = offsetLimits.getOrDefault(partition, Long.MAX_VALUE);
 
         long lastOffset = -1L;
         final List> restoreRecords = new ArrayList<>(records.size());
         final List> remainingRecords = new ArrayList<>();
 
         for (final ConsumerRecord record : records) {
+            // Check if we're unable to process records due to an offset limit (e.g. when our
+            // partition is both a source and a changelog). If we're limited then try to refresh
+            // the offset limit if possible.
+            if (record.offset() >= limit && updateableOffsetLimits.contains(partition)) {
+                limit = updateOffsetLimits(partition);
+            }
+
             if (record.offset() < limit) {
                 restoreRecords.add(record);
                 lastOffset = record.offset();
@@ -181,9 +200,8 @@ public List> update(final TopicPartition partitio
             }
         }
 
-        stateMgr.updateStandbyStates(partition, restoreRecords, lastOffset);
-
         if (!restoreRecords.isEmpty()) {
+            stateMgr.updateStandbyStates(partition, restoreRecords, lastOffset);
             commitNeeded = true;
         }
 
@@ -194,4 +212,23 @@ Map checkpointedOffsets() {
         return checkpointedOffsets;
     }
 
+    private long updateOffsetLimits(final TopicPartition partition) {
+        if (!offsetLimits.containsKey(partition)) {
+            throw new IllegalArgumentException("Topic is not both a source and a changelog: " + partition);
+        }
+
+        updateableOffsetLimits.remove(partition);
+
+        final long newLimit = committedOffsetForPartition(partition);
+        final long previousLimit = offsetLimits.put(partition, newLimit);
+        if (previousLimit > newLimit) {
+            throw new IllegalStateException("Offset limit should monotonically increase, but was reduced. " +
+                "New limit: " + newLimit + ". Previous limit: " + previousLimit);
+        }
+        return newLimit;
+    }
+
+    void allowUpdateOfOffsetLimit() {
+        updateableOffsetLimits.addAll(offsetLimits.keySet());
+    }
 }
\ No newline at end of file
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java
index c8a9842ff9a97..adb886969aecc 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
+import java.util.Map.Entry;
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.clients.consumer.ConsumerRecords;
@@ -44,7 +45,7 @@ public class StoreChangelogReader implements ChangelogReader {
     private final Logger log;
     private final Consumer restoreConsumer;
     private final StateRestoreListener userStateRestoreListener;
-    private final Map endOffsets = new HashMap<>();
+    private final Map restoreToOffsets = new HashMap<>();
     private final Map> partitionInfo = new HashMap<>();
     private final Map stateRestorers = new HashMap<>();
     private final Set needsRestoring = new HashSet<>();
@@ -89,11 +90,11 @@ public Collection restore(final RestoringTasks active) {
 
             for (final TopicPartition partition : needsRestoring) {
                 final StateRestorer restorer = stateRestorers.get(partition);
-                final long pos = processNext(records.records(partition), restorer, endOffsets.get(partition));
+                final long pos = processNext(records.records(partition), restorer, restoreToOffsets.get(partition));
                 restorer.setRestoredOffset(pos);
-                if (restorer.hasCompleted(pos, endOffsets.get(partition))) {
+                if (restorer.hasCompleted(pos, restoreToOffsets.get(partition))) {
                     restorer.restoreDone();
-                    endOffsets.remove(partition);
+                    restoreToOffsets.remove(partition);
                     completedRestorers.add(partition);
                 }
             }
@@ -141,39 +142,44 @@ private void initialize(final RestoringTasks active) {
 
         // try to fetch end offsets for the initializable restorers and remove any partitions
         // where we already have all of the data
+        final Map endOffsets;
         try {
-            endOffsets.putAll(restoreConsumer.endOffsets(initializable));
+            endOffsets = restoreConsumer.endOffsets(initializable);
         } catch (final TimeoutException e) {
             // if timeout exception gets thrown we just give up this time and retry in the next run loop
             log.debug("Could not fetch end offset for {}; will fall back to partition by partition fetching", initializable);
             return;
         }
 
+        endOffsets.forEach((partition, endOffset) -> {
+            if (endOffset != null) {
+                final StateRestorer restorer = stateRestorers.get(partition);
+                final long offsetLimit = restorer.offsetLimit();
+                restoreToOffsets.put(partition, Math.min(endOffset, offsetLimit));
+            } else {
+                log.info("End offset cannot be found form the returned metadata; removing this partition from the current run loop");
+                initializable.remove(partition);
+            }
+        });
+
         final Iterator iter = initializable.iterator();
         while (iter.hasNext()) {
             final TopicPartition topicPartition = iter.next();
-            final Long endOffset = endOffsets.get(topicPartition);
+            final Long restoreOffset = restoreToOffsets.get(topicPartition);
+            final StateRestorer restorer = stateRestorers.get(topicPartition);
 
-            // offset should not be null; but since the consumer API does not guarantee it
-            // we add this check just in case
-            if (endOffset != null) {
-                final StateRestorer restorer = stateRestorers.get(topicPartition);
-                if (restorer.checkpoint() >= endOffset) {
-                    restorer.setRestoredOffset(restorer.checkpoint());
-                    iter.remove();
-                    completedRestorers.add(topicPartition);
-                } else if (restorer.offsetLimit() == 0 || endOffset == 0) {
-                    restorer.setRestoredOffset(0);
-                    iter.remove();
-                    completedRestorers.add(topicPartition);
-                } else {
-                    restorer.setEndingOffset(endOffset);
-                }
-                needsInitializing.remove(topicPartition);
-            } else {
-                log.info("End offset cannot be found form the returned metadata; removing this partition from the current run loop");
+            if (restorer.checkpoint() >= restoreOffset) {
+                restorer.setRestoredOffset(restorer.checkpoint());
+                iter.remove();
+                completedRestorers.add(topicPartition);
+            } else if (restoreOffset == 0) {
+                restorer.setRestoredOffset(0);
                 iter.remove();
+                completedRestorers.add(topicPartition);
+            } else {
+                restorer.setEndingOffset(restoreOffset);
             }
+            needsInitializing.remove(topicPartition);
         }
 
         // set up restorer for those initializable
@@ -200,7 +206,7 @@ private void startRestoration(final Set initialized,
                 restoreConsumer.seek(partition, restorer.checkpoint());
                 logRestoreOffsets(partition,
                         restorer.checkpoint(),
-                        endOffsets.get(partition));
+                        restoreToOffsets.get(partition));
                 restorer.setStartingOffset(restoreConsumer.position(partition));
                 restorer.restoreStarted();
             } else {
@@ -232,7 +238,7 @@ private void startRestoration(final Set initialized,
                 final long position = restoreConsumer.position(restorer.partition());
                 logRestoreOffsets(restorer.partition(),
                         position,
-                        endOffsets.get(restorer.partition()));
+                        restoreToOffsets.get(restorer.partition()));
                 restorer.setStartingOffset(position);
                 restorer.restoreStarted();
             }
@@ -279,7 +285,7 @@ public void reset() {
         partitionInfo.clear();
         stateRestorers.clear();
         needsRestoring.clear();
-        endOffsets.clear();
+        restoreToOffsets.clear();
         needsInitializing.clear();
         completedRestorers.clear();
     }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
index d6ee2dff735ac..3dd83b5b00cd4 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
@@ -16,6 +16,19 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
+import static java.lang.String.format;
+import static java.util.Collections.singleton;
+import static org.apache.kafka.streams.kstream.internals.metrics.Sensors.recordLatenessSensor;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import org.apache.kafka.clients.consumer.CommitFailedException;
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
@@ -48,18 +61,6 @@
 import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.apache.kafka.streams.state.internals.ThreadCache;
 
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import static java.lang.String.format;
-import static java.util.Collections.singleton;
-import static org.apache.kafka.streams.kstream.internals.metrics.Sensors.recordLatenessSensor;
-
 /**
  * A StreamTask is associated with a {@link PartitionGroup}, and is assigned to a StreamThread for processing.
  */
@@ -236,6 +237,22 @@ public StreamTask(final TaskId id,
     @Override
     public boolean initializeStateStores() {
         log.trace("Initializing state stores");
+
+        // Currently there is no easy way to tell the ProcessorStateManager to only restore up to
+        // a specific offset. In most cases this is fine. However, in optimized topologies we can
+        // have a source topic that also serves as a changelog, and in this case we want our active
+        // stream task to only play records up to the last consumer committed offset. Here we find
+        // partitions of topics that are both sources and changelogs and set the consumer committed
+        // offset via stateMgr as there is not a more direct route.
+        final Set changelogTopicNames = new HashSet<>(topology.storeToChangelogTopic().values());
+        partitions.stream()
+            .filter(tp -> changelogTopicNames.contains(tp.topic()))
+            .forEach(tp -> {
+                final long offset = committedOffsetForPartition(tp);
+                stateMgr.putOffsetLimit(tp, offset);
+                log.trace("Updating store offset limits {} for changelog {}", offset, tp);
+            });
+
         registerStateStores();
 
         return changelogPartitions().isEmpty();
@@ -460,7 +477,6 @@ void commit(final boolean startNewTransaction) {
             final TopicPartition partition = entry.getKey();
             final long offset = entry.getValue() + 1;
             consumedOffsetsAndMetadata.put(partition, new OffsetAndMetadata(offset));
-            stateMgr.putOffsetLimit(partition, offset);
         }
 
         try {
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/OptimizedKTableIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/OptimizedKTableIntegrationTest.java
new file mode 100644
index 0000000000000..22083c117ab43
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/OptimizedKTableIntegrationTest.java
@@ -0,0 +1,335 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.integration;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.Assert.fail;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.serialization.IntegerSerializer;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.KafkaStreams.State;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
+import org.apache.kafka.streams.integration.utils.IntegrationTestUtils;
+import org.apache.kafka.streams.kstream.Consumed;
+import org.apache.kafka.streams.kstream.Materialized;
+import org.apache.kafka.streams.processor.StateRestoreListener;
+import org.apache.kafka.streams.state.KeyValueStore;
+import org.apache.kafka.streams.state.QueryableStoreTypes;
+import org.apache.kafka.streams.state.ReadOnlyKeyValueStore;
+import org.apache.kafka.test.IntegrationTest;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(IntegrationTest.class)
+public class OptimizedKTableIntegrationTest {
+    private static final int NUM_BROKERS = 1;
+
+    private static final String INPUT_TOPIC_NAME = "input-topic";
+    private static final String TABLE_NAME = "source-table";
+
+    @Rule
+    public final EmbeddedKafkaCluster cluster = new EmbeddedKafkaCluster(NUM_BROKERS);
+
+    private final Map kafkaStreamsStates = new HashMap<>();
+    private final Lock kafkaStreamsStatesLock = new ReentrantLock();
+    private final Condition kafkaStreamsStateUpdate = kafkaStreamsStatesLock.newCondition();
+    private final MockTime mockTime = cluster.time;
+
+    @Before
+    public void before() throws InterruptedException {
+        cluster.createTopic(INPUT_TOPIC_NAME, 2, 1);
+    }
+
+    @After
+    public void after() {
+        for (final KafkaStreams kafkaStreams : kafkaStreamsStates.keySet()) {
+            kafkaStreams.close();
+        }
+    }
+
+    @Test
+    public void standbyShouldNotPerformRestoreAtStartup() throws Exception {
+        final int numMessages = 10;
+        final int key = 1;
+        final Semaphore semaphore = new Semaphore(0);
+
+        final StreamsBuilder builder = new StreamsBuilder();
+        builder
+            .table(INPUT_TOPIC_NAME, Consumed.with(Serdes.Integer(), Serdes.Integer()),
+                Materialized.>as(TABLE_NAME)
+                    .withCachingDisabled())
+            .toStream()
+            .peek((k, v) -> semaphore.release());
+
+        final KafkaStreams kafkaStreams1 = createKafkaStreams(builder, streamsConfiguration());
+        final KafkaStreams kafkaStreams2 = createKafkaStreams(builder, streamsConfiguration());
+        final List kafkaStreamsList = Arrays.asList(kafkaStreams1, kafkaStreams2);
+
+        produceValueRange(key, 0, 10);
+
+        final AtomicLong restoreStartOffset = new AtomicLong(-1);
+        kafkaStreamsList.forEach(kafkaStreams -> {
+            kafkaStreams.setGlobalStateRestoreListener(createTrackingRestoreListener(restoreStartOffset, new AtomicLong()));
+            kafkaStreams.start();
+        });
+        waitForKafkaStreamssToEnterRunningState(kafkaStreamsList, 60, TimeUnit.SECONDS);
+
+        // Assert that all messages in the first batch were processed in a timely manner
+        assertThat(semaphore.tryAcquire(numMessages, 60, TimeUnit.SECONDS), is(equalTo(true)));
+
+        // Assert that no restore occurred
+        assertThat(restoreStartOffset.get(), is(equalTo(-1L)));
+    }
+
+    @Test
+    public void shouldApplyUpdatesToStandbyStore() throws Exception {
+        final int batch1NumMessages = 100;
+        final int batch2NumMessages = 100;
+        final int key = 1;
+        final Semaphore semaphore = new Semaphore(0);
+
+        final StreamsBuilder builder = new StreamsBuilder();
+        builder
+            .table(INPUT_TOPIC_NAME, Consumed.with(Serdes.Integer(), Serdes.Integer()),
+                Materialized.>as(TABLE_NAME)
+                    .withCachingDisabled())
+            .toStream()
+            .peek((k, v) -> semaphore.release());
+
+        final KafkaStreams kafkaStreams1 = createKafkaStreams(builder, streamsConfiguration());
+        final KafkaStreams kafkaStreams2 = createKafkaStreams(builder, streamsConfiguration());
+        final List kafkaStreamsList = Arrays.asList(kafkaStreams1, kafkaStreams2);
+
+        final AtomicLong restoreStartOffset = new AtomicLong(-1L);
+        final AtomicLong restoreEndOffset = new AtomicLong(-1L);
+        kafkaStreamsList.forEach(kafkaStreams -> {
+            kafkaStreams.setGlobalStateRestoreListener(createTrackingRestoreListener(restoreStartOffset, restoreEndOffset));
+            kafkaStreams.start();
+        });
+        waitForKafkaStreamssToEnterRunningState(kafkaStreamsList, 60, TimeUnit.SECONDS);
+
+        produceValueRange(key, 0, batch1NumMessages);
+
+        // Assert that all messages in the first batch were processed in a timely manner
+        assertThat(semaphore.tryAcquire(batch1NumMessages, 60, TimeUnit.SECONDS), is(equalTo(true)));
+
+        final ReadOnlyKeyValueStore store1 = kafkaStreams1
+            .store(TABLE_NAME, QueryableStoreTypes.keyValueStore());
+
+        final ReadOnlyKeyValueStore store2 = kafkaStreams2
+            .store(TABLE_NAME, QueryableStoreTypes.keyValueStore());
+
+        final boolean kafkaStreams1WasFirstActive;
+        if (store1.get(key) != null) {
+            kafkaStreams1WasFirstActive = true;
+        } else {
+            // Assert that data from the job was sent to the store
+            assertThat(store2.get(key), is(notNullValue()));
+            kafkaStreams1WasFirstActive = false;
+        }
+
+        // Assert that no restore has occurred, ensures that when we check later that the restore
+        // notification actually came from after the rebalance.
+        assertThat(restoreStartOffset.get(), is(equalTo(-1L)));
+
+        // Assert that the current value in store reflects all messages being processed
+        assertThat(kafkaStreams1WasFirstActive ? store1.get(key) : store2.get(key), is(equalTo(batch1NumMessages - 1)));
+
+        if (kafkaStreams1WasFirstActive) {
+            kafkaStreams1.close();
+        } else {
+            kafkaStreams2.close();
+        }
+
+        final ReadOnlyKeyValueStore newActiveStore =
+            kafkaStreams1WasFirstActive ? store2 : store1;
+        retryOnExceptionWithTimeout(100, 60 * 1000, TimeUnit.MILLISECONDS, () -> {
+            // Assert that after failover we have recovered to the last store write
+            assertThat(newActiveStore.get(key), is(equalTo(batch1NumMessages - 1)));
+        });
+
+        final int totalNumMessages = batch1NumMessages + batch2NumMessages;
+
+        produceValueRange(key, batch1NumMessages, totalNumMessages);
+
+        // Assert that all messages in the second batch were processed in a timely manner
+        assertThat(semaphore.tryAcquire(batch2NumMessages, 60, TimeUnit.SECONDS), is(equalTo(true)));
+
+        // Assert that either restore was unnecessary or we restored from an offset later than 0
+        assertThat(restoreStartOffset.get(), is(anyOf(greaterThan(0L), equalTo(-1L))));
+
+        // Assert that either restore was unnecessary or we restored to the last offset before we closed the kafkaStreams
+        assertThat(restoreEndOffset.get(), is(anyOf(equalTo(batch1NumMessages - 1), equalTo(-1L))));
+
+        // Assert that the current value in store reflects all messages being processed
+        assertThat(newActiveStore.get(key), is(equalTo(totalNumMessages - 1)));
+    }
+
+    private void produceValueRange(final int key, final int start, final int endExclusive) throws Exception {
+        final Properties producerProps = new Properties();
+        producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, cluster.bootstrapServers());
+        producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class);
+        producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class);
+
+        IntegrationTestUtils.produceKeyValuesSynchronously(
+            INPUT_TOPIC_NAME,
+            IntStream.range(start, endExclusive)
+                .mapToObj(i -> KeyValue.pair(key, i))
+                .collect(Collectors.toList()),
+            producerProps,
+            mockTime);
+    }
+
+    private void retryOnExceptionWithTimeout(final long pollInterval,
+                                             final long timeout,
+                                             final TimeUnit timeUnit,
+                                             final Runnable runnable) throws InterruptedException {
+        final long expectedEnd = System.currentTimeMillis() + timeUnit.toMillis(timeout);
+
+        while (true) {
+            try {
+                runnable.run();
+                return;
+            } catch (final Throwable t) {
+                if (expectedEnd <= System.currentTimeMillis()) {
+                    throw new AssertionError(t);
+                }
+                Thread.sleep(timeUnit.toMillis(pollInterval));
+            }
+        }
+    }
+
+    private void waitForKafkaStreamssToEnterRunningState(final Collection kafkaStreamss,
+                                                         final long time,
+                                                         final TimeUnit timeUnit) throws InterruptedException {
+
+        final long expectedEnd = System.currentTimeMillis() + timeUnit.toMillis(time);
+
+        kafkaStreamsStatesLock.lock();
+        try {
+            while (!kafkaStreamss.stream().allMatch(kafkaStreams -> kafkaStreamsStates.get(kafkaStreams) == State.RUNNING)) {
+                if (expectedEnd <= System.currentTimeMillis()) {
+                    fail("one or more kafkaStreamss did not enter RUNNING in a timely manner");
+                }
+                final long millisRemaining = Math.max(1, expectedEnd - System.currentTimeMillis());
+                kafkaStreamsStateUpdate.await(millisRemaining, TimeUnit.MILLISECONDS);
+            }
+        } finally {
+            kafkaStreamsStatesLock.unlock();
+        }
+    }
+
+    private KafkaStreams createKafkaStreams(final StreamsBuilder builder, final Properties config) {
+        final KafkaStreams kafkaStreams = new KafkaStreams(builder.build(config), config);
+        kafkaStreamsStatesLock.lock();
+        try {
+            kafkaStreamsStates.put(kafkaStreams, kafkaStreams.state());
+        } finally {
+            kafkaStreamsStatesLock.unlock();
+        }
+
+        kafkaStreams.setStateListener((newState, oldState) -> {
+            kafkaStreamsStatesLock.lock();
+            try {
+                kafkaStreamsStates.put(kafkaStreams, newState);
+                if (newState == State.RUNNING) {
+                    if (kafkaStreamsStates.values().stream().allMatch(state -> state == State.RUNNING)) {
+                        kafkaStreamsStateUpdate.signalAll();
+                    }
+                }
+            } finally {
+                kafkaStreamsStatesLock.unlock();
+            }
+        });
+        return kafkaStreams;
+    }
+
+    private StateRestoreListener createTrackingRestoreListener(final AtomicLong restoreStartOffset,
+                                                               final AtomicLong restoreEndOffset) {
+        return new StateRestoreListener() {
+            @Override
+            public void onRestoreStart(final TopicPartition topicPartition,
+                                       final String storeName,
+                                       final long startingOffset,
+                                       final long endingOffset) {
+                restoreStartOffset.set(startingOffset);
+                restoreEndOffset.set(endingOffset);
+            }
+
+            @Override
+            public void onBatchRestored(final TopicPartition topicPartition, final String storeName,
+                final long batchEndOffset, final long numRestored) {
+
+            }
+
+            @Override
+            public void onRestoreEnd(final TopicPartition topicPartition, final String storeName,
+                final long totalRestored) {
+
+            }
+        };
+    }
+
+    private Properties streamsConfiguration() {
+        final String applicationId = "streamsApp";
+        final Properties config = new Properties();
+        config.put(StreamsConfig.TOPOLOGY_OPTIMIZATION, StreamsConfig.OPTIMIZE);
+        config.put(StreamsConfig.APPLICATION_ID_CONFIG, applicationId);
+        config.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, cluster.bootstrapServers());
+        config.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory(applicationId).getPath());
+        config.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass());
+        config.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.Integer().getClass());
+        config.put(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 1);
+        config.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 100);
+        config.put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, 200);
+        config.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 1000);
+        config.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 100);
+        return config;
+    }
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractTaskTest.java
index c5080d7803107..0a79e8459afd3 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractTaskTest.java
@@ -16,20 +16,28 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
+import static org.apache.kafka.streams.processor.internals.ProcessorTopologyFactories.withLocalStores;
+import static org.easymock.EasyMock.expect;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
 import org.apache.kafka.clients.consumer.Consumer;
-import org.apache.kafka.clients.consumer.MockConsumer;
-import org.apache.kafka.clients.consumer.OffsetAndMetadata;
-import org.apache.kafka.clients.consumer.OffsetResetStrategy;
-import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.errors.AuthorizationException;
-import org.apache.kafka.common.errors.WakeupException;
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.common.utils.MockTime;
 import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.errors.LockException;
-import org.apache.kafka.streams.errors.ProcessorStateException;
 import org.apache.kafka.streams.processor.StateStore;
 import org.apache.kafka.streams.processor.TaskId;
 import org.apache.kafka.test.InternalMockProcessorContext;
@@ -40,22 +48,6 @@
 import org.junit.Before;
 import org.junit.Test;
 
-import java.io.File;
-import java.io.IOException;
-import java.time.Duration;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-
-import static org.apache.kafka.streams.processor.internals.ProcessorTopologyFactories.withLocalStores;
-import static org.easymock.EasyMock.expect;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
 public class AbstractTaskTest {
 
     private final TaskId id = new TaskId(0, 0);
@@ -72,27 +64,6 @@ public void before() {
         expect(stateDirectory.directoryForTask(id)).andReturn(TestUtils.tempDirectory());
     }
 
-    @Test(expected = ProcessorStateException.class)
-    public void shouldThrowProcessorStateExceptionOnInitializeOffsetsWhenAuthorizationException() {
-        final Consumer consumer = mockConsumer(new AuthorizationException("blah"));
-        final AbstractTask task = createTask(consumer, Collections.emptyMap());
-        task.updateOffsetLimits();
-    }
-
-    @Test(expected = ProcessorStateException.class)
-    public void shouldThrowProcessorStateExceptionOnInitializeOffsetsWhenKafkaException() {
-        final Consumer consumer = mockConsumer(new KafkaException("blah"));
-        final AbstractTask task = createTask(consumer, Collections.emptyMap());
-        task.updateOffsetLimits();
-    }
-
-    @Test(expected = WakeupException.class)
-    public void shouldThrowWakeupExceptionOnInitializeOffsetsWhenWakeupException() {
-        final Consumer consumer = mockConsumer(new WakeupException());
-        final AbstractTask task = createTask(consumer, Collections.emptyMap());
-        task.updateOffsetLimits();
-    }
-
     @Test
     public void shouldThrowLockExceptionIfFailedToLockStateDirectoryWhenTopologyHasStores() throws IOException {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
@@ -270,14 +241,4 @@ public boolean initializeStateStores() {
             public void initializeTopology() {}
         };
     }
-
-    private Consumer mockConsumer(final RuntimeException toThrow) {
-        return new MockConsumer(OffsetResetStrategy.EARLIEST) {
-            @Override
-            public OffsetAndMetadata committed(final TopicPartition partition) {
-                throw toThrow;
-            }
-        };
-    }
-
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
index 2faa078d6751e..df889798a86b6 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
@@ -16,6 +16,8 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.clients.consumer.MockConsumer;
 import org.apache.kafka.clients.consumer.OffsetAndMetadata;
@@ -554,6 +556,94 @@ private ConsumerRecord makeConsumerRecord(final TopicPartition t
         );
     }
 
+    @Test
+    public void shouldNotGetConsumerCommittedOffsetIfThereAreNoRecordUpdates() throws IOException {
+        final AtomicInteger committedCallCount = new AtomicInteger();
+
+        final Consumer consumer = new MockConsumer(OffsetResetStrategy.EARLIEST) {
+            @Override
+            public synchronized OffsetAndMetadata committed(final TopicPartition partition) {
+                committedCallCount.getAndIncrement();
+                return super.committed(partition);
+            }
+        };
+
+        consumer.assign(Collections.singletonList(globalTopicPartition));
+        consumer.commitSync(mkMap(mkEntry(globalTopicPartition, new OffsetAndMetadata(0L))));
+
+        task = new StandbyTask(
+            taskId,
+            ktablePartitions,
+            ktableTopology,
+            consumer,
+            changelogReader,
+            createConfig(baseDir),
+            streamsMetrics,
+            stateDirectory
+        );
+        task.initializeStateStores();
+        assertThat(committedCallCount.get(), equalTo(0));
+
+        task.update(globalTopicPartition, Collections.emptyList());
+        // We should not make a consumer.committed() call because there are no new records.
+        assertThat(committedCallCount.get(), equalTo(0));
+    }
+
+    @Test
+    public void shouldGetConsumerCommittedOffsetsOncePerCommit() throws IOException {
+        final AtomicInteger committedCallCount = new AtomicInteger();
+
+        final Consumer consumer = new MockConsumer(OffsetResetStrategy.EARLIEST) {
+            @Override
+            public synchronized OffsetAndMetadata committed(final TopicPartition partition) {
+                committedCallCount.getAndIncrement();
+                return super.committed(partition);
+            }
+        };
+
+        consumer.assign(Collections.singletonList(globalTopicPartition));
+        consumer.commitSync(mkMap(mkEntry(globalTopicPartition, new OffsetAndMetadata(0L))));
+
+        task = new StandbyTask(
+            taskId,
+            ktablePartitions,
+            ktableTopology,
+            consumer,
+            changelogReader,
+            createConfig(baseDir),
+            streamsMetrics,
+            stateDirectory
+        );
+        task.initializeStateStores();
+
+        task.update(
+            globalTopicPartition,
+            Collections.singletonList(
+                makeConsumerRecord(globalTopicPartition, 1, 1)
+            )
+        );
+        assertThat(committedCallCount.get(), equalTo(1));
+
+        task.update(
+            globalTopicPartition,
+            Collections.singletonList(
+                makeConsumerRecord(globalTopicPartition, 1, 1)
+            )
+        );
+        // We should not make another consumer.committed() call until we commit
+        assertThat(committedCallCount.get(), equalTo(1));
+
+        task.commit();
+        task.update(
+            globalTopicPartition,
+            Collections.singletonList(
+                makeConsumerRecord(globalTopicPartition, 1, 1)
+            )
+        );
+        // We committed so we're allowed to make another consumer.committed() call
+        assertThat(committedCallCount.get(), equalTo(2));
+    }
+
     @Test
     public void shouldInitializeStateStoreWithoutException() throws IOException {
         final InternalStreamsBuilder builder = new InternalStreamsBuilder(new InternalTopologyBuilder());
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java
index bf59c21f12c15..9ae9d798cb810 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java
@@ -750,6 +750,64 @@ public void shouldNotThrowTaskMigratedExceptionIfEndOffsetNotExceededDuringResto
         assertThat(callback.restored.size(), equalTo(10));
     }
 
+    @Test
+    public void shouldRestoreUpToOffsetLimit() {
+        setupConsumer(10, topicPartition);
+        changelogReader.register(new StateRestorer(
+            topicPartition,
+            restoreListener,
+            2L,
+            5,
+            true,
+            "storeName1",
+            identity()));
+        expect(active.restoringTaskFor(topicPartition)).andStubReturn(task);
+        replay(active, task);
+        changelogReader.restore(active);
+
+        assertThat(callback.restored.size(), equalTo(3));
+        assertAllCallbackStatesExecuted(callback, "storeName1");
+        assertCorrectOffsetsReportedByListener(callback, 2L, 4L, 3L);
+    }
+
+    @Test
+    public void shouldNotRestoreIfCheckpointIsEqualToOffsetLimit() {
+        setupConsumer(10, topicPartition);
+        changelogReader.register(new StateRestorer(
+            topicPartition,
+            restoreListener,
+            5L,
+            5,
+            true,
+            "storeName1",
+            identity()));
+        expect(active.restoringTaskFor(topicPartition)).andStubReturn(task);
+        replay(active, task);
+        changelogReader.restore(active);
+
+        assertThat(callback.storeNameCalledStates.size(), equalTo(0));
+        assertThat(callback.restored.size(), equalTo(0));
+    }
+
+    @Test
+    public void shouldNotRestoreIfCheckpointIsGreaterThanOffsetLimit() {
+        setupConsumer(10, topicPartition);
+        changelogReader.register(new StateRestorer(
+            topicPartition,
+            restoreListener,
+            10L,
+            5,
+            true,
+            "storeName1",
+            identity()));
+        expect(active.restoringTaskFor(topicPartition)).andStubReturn(task);
+        replay(active, task);
+        changelogReader.restore(active);
+
+        assertThat(callback.storeNameCalledStates.size(), equalTo(0));
+        assertThat(callback.restored.size(), equalTo(0));
+    }
+
     private void setupConsumer(final long messages,
                                final TopicPartition topicPartition) {
         assignPartition(messages, topicPartition);
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
index fb196fdcda319..cc0af23eed37d 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
@@ -16,15 +16,19 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
+import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.clients.consumer.MockConsumer;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
 import org.apache.kafka.clients.consumer.OffsetResetStrategy;
 import org.apache.kafka.clients.producer.MockProducer;
 import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.AuthorizationException;
 import org.apache.kafka.common.errors.ProducerFencedException;
 import org.apache.kafka.common.errors.TimeoutException;
+import org.apache.kafka.common.errors.WakeupException;
 import org.apache.kafka.common.metrics.JmxReporter;
 import org.apache.kafka.common.metrics.KafkaMetric;
 import org.apache.kafka.common.metrics.MetricConfig;
@@ -40,6 +44,7 @@
 import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.errors.DefaultProductionExceptionHandler;
+import org.apache.kafka.streams.errors.ProcessorStateException;
 import org.apache.kafka.streams.errors.StreamsException;
 import org.apache.kafka.streams.errors.TaskMigratedException;
 import org.apache.kafka.streams.processor.PunctuationType;
@@ -77,6 +82,7 @@
 import static org.apache.kafka.common.utils.Utils.mkEntry;
 import static org.apache.kafka.common.utils.Utils.mkMap;
 import static org.apache.kafka.common.utils.Utils.mkProperties;
+import static org.apache.kafka.common.utils.Utils.mkSet;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.nullValue;
@@ -98,7 +104,7 @@ public class StreamTaskTest {
     private final String topic2 = "topic2";
     private final TopicPartition partition1 = new TopicPartition(topic1, 1);
     private final TopicPartition partition2 = new TopicPartition(topic2, 1);
-    private final Set partitions = Utils.mkSet(partition1, partition2);
+    private final Set partitions = mkSet(partition1, partition2);
 
     private final MockSourceNode source1 = new MockSourceNode<>(new String[]{topic1}, intDeserializer, intDeserializer);
     private final MockSourceNode source2 = new MockSourceNode<>(new String[]{topic2}, intDeserializer, intDeserializer);
@@ -1417,7 +1423,7 @@ public void shouldReturnOffsetsForRepartitionTopicsForPurging() {
 
         task = new StreamTask(
             taskId00,
-            Utils.mkSet(partition1, repartition),
+            mkSet(partition1, repartition),
             topology,
             consumer,
             changelogReader,
@@ -1478,6 +1484,59 @@ public void punctuate(final long timestamp) {
         assertEquals(1, producer.history().size());
     }
 
+    @Test(expected = ProcessorStateException.class)
+    public void shouldThrowProcessorStateExceptionOnInitializeOffsetsWhenAuthorizationException() {
+        final Consumer consumer = mockConsumerWithCommittedException(new AuthorizationException("message"));
+        final StreamTask task = createOptimizedStatefulTask(createConfig(false), consumer);
+        task.initializeStateStores();
+    }
+
+    @Test(expected = ProcessorStateException.class)
+    public void shouldThrowProcessorStateExceptionOnInitializeOffsetsWhenKafkaException() {
+        final Consumer consumer = mockConsumerWithCommittedException(new KafkaException("message"));
+        final AbstractTask task = createOptimizedStatefulTask(createConfig(false), consumer);
+        task.initializeStateStores();
+    }
+
+    @Test(expected = WakeupException.class)
+    public void shouldThrowWakeupExceptionOnInitializeOffsetsWhenWakeupException() {
+        final Consumer consumer = mockConsumerWithCommittedException(new WakeupException());
+        final AbstractTask task = createOptimizedStatefulTask(createConfig(false), consumer);
+        task.initializeStateStores();
+    }
+
+    private Consumer mockConsumerWithCommittedException(final RuntimeException toThrow) {
+        return new MockConsumer(OffsetResetStrategy.EARLIEST) {
+            @Override
+            public OffsetAndMetadata committed(final TopicPartition partition) {
+                throw toThrow;
+            }
+        };
+    }
+
+    private StreamTask createOptimizedStatefulTask(final StreamsConfig config, final Consumer consumer) {
+        final StateStore stateStore = new MockKeyValueStore(storeName, true);
+
+        final ProcessorTopology topology = ProcessorTopologyFactories.with(
+            asList(source1),
+            mkMap(mkEntry(topic1, source1)),
+            singletonList(stateStore),
+            Collections.singletonMap(storeName, topic1));
+
+        return new StreamTask(
+            taskId00,
+            mkSet(partition1),
+            topology,
+            consumer,
+            changelogReader,
+            config,
+            streamsMetrics,
+            stateDirectory,
+            null,
+            time,
+            () -> producer = new MockProducer<>(false, bytesSerializer, bytesSerializer));
+    }
+
     private StreamTask createStatefulTask(final StreamsConfig config, final boolean logged) {
         final StateStore stateStore = new MockKeyValueStore(storeName, logged);
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/StreamThreadStateStoreProviderTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/StreamThreadStateStoreProviderTest.java
index f48c31c1d91c2..a2e4557c27ea3 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/StreamThreadStateStoreProviderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/StreamThreadStateStoreProviderTest.java
@@ -317,10 +317,7 @@ private StreamTask createStreamsTask(final StreamsConfig streamsConfig,
             stateDirectory,
             null,
             new MockTime(),
-            () -> clientSupplier.getProducer(new HashMap<>())) {
-            @Override
-            protected void updateOffsetLimits() {}
-        };
+            () -> clientSupplier.getProducer(new HashMap<>()));
     }
 
     private void mockThread(final boolean initialized) {

From ac385c4c3a770728848438f28f4acb8854ffc868 Mon Sep 17 00:00:00 2001
From: Mickael Maison 
Date: Fri, 13 Sep 2019 23:48:36 +0100
Subject: [PATCH 0611/1071] KAFKA-8474; Use HTML lists for config layout
 (#6870)

Replace the `` elements by `
                  ` so the full page width can be used for the configuration descriptions instead of only a very narrow column. I moved the other fields (Type, Default Value, etc) below each entry.

Reviewers: Boyang Chen , Jason Gustafson 
---
 .../kafka/clients/CommonClientConfigs.java    |  6 +--
 .../clients/admin/AdminClientConfig.java      |  2 +-
 .../clients/consumer/ConsumerConfig.java      |  6 +--
 .../clients/producer/ProducerConfig.java      |  7 +--
 .../apache/kafka/common/config/ConfigDef.java | 50 +++++++++++++++++++
 .../kafka/common/config/TopicConfig.java      |  8 +--
 .../internals/BrokerSecurityConfigs.java      |  3 +-
 .../kafka/common/config/ConfigDefTest.java    |  2 +-
 .../connect/runtime/SinkConnectorConfig.java  |  2 +-
 .../runtime/SourceConnectorConfig.java        |  2 +-
 .../distributed/DistributedConfig.java        |  2 +-
 .../connect/tools/TransformationDoc.java      |  2 +-
 core/src/main/scala/kafka/log/LogConfig.scala |  2 +-
 .../main/scala/kafka/server/KafkaConfig.scala |  6 +--
 .../scala/unit/kafka/log/LogConfigTest.scala  | 12 ++++-
 .../apache/kafka/streams/StreamsConfig.java   |  2 +-
 16 files changed, 87 insertions(+), 27 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java b/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java
index 54c3891ec26bc..af47e55d75a6a 100644
--- a/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java
+++ b/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java
@@ -42,9 +42,9 @@ public class CommonClientConfigs {
                                                        + "servers (you may want more than one, though, in case a server is down).";
 
     public static final String CLIENT_DNS_LOOKUP_CONFIG = "client.dns.lookup";
-    public static final String CLIENT_DNS_LOOKUP_DOC = "
                  Controls how the client uses DNS lookups.
                  If set to use_all_dns_ips then, when the lookup returns multiple IP addresses for a hostname,"
-                                                       + " they will all be attempted to connect to before failing the connection. Applies to both bootstrap and advertised servers.
                  "
-                                                       + "
                  If the value is resolve_canonical_bootstrap_servers_only each entry will be resolved and expanded into a list of canonical names.
                  ";
+    public static final String CLIENT_DNS_LOOKUP_DOC = "Controls how the client uses DNS lookups. If set to use_all_dns_ips then, when the lookup returns multiple IP addresses for a hostname,"
+                                                       + " they will all be attempted to connect to before failing the connection. Applies to both bootstrap and advertised servers."
+                                                       + " If the value is resolve_canonical_bootstrap_servers_only each entry will be resolved and expanded into a list of canonical names.";
 
     public static final String METADATA_MAX_AGE_CONFIG = "metadata.max.age.ms";
     public static final String METADATA_MAX_AGE_DOC = "The period of time in milliseconds after which we force a refresh of metadata even if we haven't seen any partition leadership changes to proactively discover any new brokers or partitions.";
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/AdminClientConfig.java b/clients/src/main/java/org/apache/kafka/clients/admin/AdminClientConfig.java
index 35a781e1deb50..107eb56d63aed 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/AdminClientConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/AdminClientConfig.java
@@ -217,7 +217,7 @@ public static ConfigDef configDef() {
     }
 
     public static void main(String[] args) {
-        System.out.println(CONFIG.toHtmlTable());
+        System.out.println(CONFIG.toHtml());
     }
 
 }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
index a4f3003cc5008..e0f7c4d160d5e 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
@@ -251,9 +251,9 @@ public class ConsumerConfig extends AbstractConfig {
 
     /** isolation.level */
     public static final String ISOLATION_LEVEL_CONFIG = "isolation.level";
-    public static final String ISOLATION_LEVEL_DOC = "
                  Controls how to read messages written transactionally. If set to read_committed, consumer.poll() will only return" +
+    public static final String ISOLATION_LEVEL_DOC = "Controls how to read messages written transactionally. If set to read_committed, consumer.poll() will only return" +
             " transactional messages which have been committed. If set to read_uncommitted' (the default), consumer.poll() will return all messages, even transactional messages" +
-            " which have been aborted. Non-transactional messages will be returned unconditionally in either mode.
                   
                  Messages will always be returned in offset order. Hence, in " +
+            " which have been aborted. Non-transactional messages will be returned unconditionally in either mode. 
                  Messages will always be returned in offset order. Hence, in " +
             " read_committed mode, consumer.poll() will only return messages up to the last stable offset (LSO), which is the one less than the offset of the first open transaction." +
             " In particular any messages appearing after messages belonging to ongoing transactions will be withheld until the relevant transaction has been completed. As a result, read_committed" +
             " consumers will not be able to read up to the high watermark when there are in flight transactions.
                   Further, when in read_committed the seekToEnd method will" +
@@ -556,7 +556,7 @@ public static ConfigDef configDef() {
     }
 
     public static void main(String[] args) {
-        System.out.println(CONFIG.toHtmlTable());
+        System.out.println(CONFIG.toHtml());
     }
 
 }
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
index 00767eb29d97f..f81480433df11 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
@@ -83,13 +83,14 @@ public class ProducerConfig extends AbstractConfig {
                                            + " server at all. The record will be immediately added to the socket buffer and considered sent. No guarantee can be"
                                            + " made that the server has received the record in this case, and the retries configuration will not"
                                            + " take effect (as the client won't generally know of any failures). The offset given back for each record will"
-                                           + " always be set to -1."
+                                           + " always be set to -1."
                                            + " 
                • acks=1 This will mean the leader will write the record to its local log but will respond"
                                            + " without awaiting full acknowledgement from all followers. In this case should the leader fail immediately after"
                                            + " acknowledging the record but before the followers have replicated it then the record will be lost."
                                            + " 
                • acks=all This means the leader will wait for the full set of in-sync replicas to"
                                            + " acknowledge the record. This guarantees that the record will not be lost as long as at least one in-sync replica"
-                                           + " remains alive. This is the strongest available guarantee. This is equivalent to the acks=-1 setting.";
+                                           + " remains alive. This is the strongest available guarantee. This is equivalent to the acks=-1 setting."
+                                           + "
                ";
 
     /** linger.ms */
     public static final String LINGER_MS_CONFIG = "linger.ms";
@@ -421,7 +422,7 @@ public static ConfigDef configDef() {
     }
 
     public static void main(String[] args) {
-        System.out.println(CONFIG.toHtmlTable());
+        System.out.println(CONFIG.toHtml());
     }
 
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java b/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java
index fa99931783e00..c22b9c77dac93 100644
--- a/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java
+++ b/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java
@@ -1409,4 +1409,54 @@ public boolean visible(String name, Map parsedConfig) {
         };
     }
 
+    public String toHtml() {
+        return toHtml(Collections.emptyMap());
+    }
+
+    /**
+     * Converts this config into an HTML list that can be embedded into docs.
+     * If dynamicUpdateModes is non-empty, a "Dynamic Update Mode" label
+     * will be included in the config details with the value of the update mode. Default
+     * mode is "read-only".
+     * @param dynamicUpdateModes Config name -> update mode mapping
+     */
+    public String toHtml(Map dynamicUpdateModes) {
+        boolean hasUpdateModes = !dynamicUpdateModes.isEmpty();
+        List configs = sortedConfigs();
+        StringBuilder b = new StringBuilder();
+        b.append("
                  \n");
+
+        for (ConfigKey key : configs) {
+            if (key.internalConfig) {
+                continue;
+            }
+            b.append("
                • ");
+            b.append("");
+            b.append(key.name);
+            b.append(": ");
+            b.append(key.documentation);
+            b.append("
                  ");
+            // details
+            b.append("
                    ");
+            for (String detail : headers()) {
+                if (detail.equals("Name") || detail.equals("Description")) continue;
+                addConfigDetail(b, detail, getConfigValue(key, detail));
+            }
+            if (hasUpdateModes) {
+                String updateMode = dynamicUpdateModes.get(key.name);
+                if (updateMode == null)
+                    updateMode = "read-only";
+                addConfigDetail(b, "Update Mode", updateMode);
+            }
+            b.append("
                  ");
+            b.append("
                  • \n");
+        }
+        b.append("
                \n");
+        return b.toString();
+    }
+
+    private static void addConfigDetail(StringBuilder builder, String name, String value) {
+        builder.append("
              • " + name + ": " + value + "
                • ");
+    }
+
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/config/TopicConfig.java b/clients/src/main/java/org/apache/kafka/common/config/TopicConfig.java
index 24f8a5acbf463..9b36736524f34 100755
--- a/clients/src/main/java/org/apache/kafka/common/config/TopicConfig.java
+++ b/clients/src/main/java/org/apache/kafka/common/config/TopicConfig.java
@@ -76,12 +76,12 @@ public class TopicConfig {
         "their data. If set to -1, no time limit is applied.";
 
     public static final String MAX_MESSAGE_BYTES_CONFIG = "max.message.bytes";
-    public static final String MAX_MESSAGE_BYTES_DOC = "
                The largest record batch size allowed by Kafka. If this " +
+    public static final String MAX_MESSAGE_BYTES_DOC = "The largest record batch size allowed by Kafka. If this " +
         "is increased and there are consumers older than 0.10.2, the consumers' fetch size must also be increased so that " +
-        "the they can fetch record batches this large.
                " +
-        "
                In the latest message format version, records are always grouped into batches for efficiency. In previous " +
+        "the they can fetch record batches this large. " +
+        "In the latest message format version, records are always grouped into batches for efficiency. In previous " +
         "message format versions, uncompressed records are not grouped into batches and this limit only applies to a " +
-        "single record in that case.
                ";
+        "single record in that case.";
 
     public static final String INDEX_INTERVAL_BYTES_CONFIG = "index.interval.bytes";
     public static final String INDEX_INTERVAL_BYTES_DOCS = "This setting controls how frequently " +
diff --git a/clients/src/main/java/org/apache/kafka/common/config/internals/BrokerSecurityConfigs.java b/clients/src/main/java/org/apache/kafka/common/config/internals/BrokerSecurityConfigs.java
index de54f164be598..8a4a51738dedd 100644
--- a/clients/src/main/java/org/apache/kafka/common/config/internals/BrokerSecurityConfigs.java
+++ b/clients/src/main/java/org/apache/kafka/common/config/internals/BrokerSecurityConfigs.java
@@ -73,7 +73,8 @@ public class BrokerSecurityConfigs {
             + " client authentication is required."
             + " 
              • ssl.client.auth=requested This means client authentication is optional."
             + " unlike requested , if this option is set client can choose not to provide authentication information about itself"
-            + " 
              • ssl.client.auth=none This means client authentication is not needed.";
+            + " 
              • ssl.client.auth=none This means client authentication is not needed."
+            + "";
 
     public static final String SASL_ENABLED_MECHANISMS_DOC = "The list of SASL mechanisms enabled in the Kafka server. "
             + "The list may contain any mechanism for which a security provider is available. "
diff --git a/clients/src/test/java/org/apache/kafka/common/config/ConfigDefTest.java b/clients/src/test/java/org/apache/kafka/common/config/ConfigDefTest.java
index 701e9f44919bd..9baa0345b4c54 100644
--- a/clients/src/test/java/org/apache/kafka/common/config/ConfigDefTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/config/ConfigDefTest.java
@@ -648,7 +648,7 @@ public void testClassWithAlias() {
             // of the aliasing logic to suffice for this test.
             Thread.currentThread().setContextClassLoader(new ClassLoader(originalClassLoader) {
                 @Override
-                public Class loadClass(String name, boolean resolve) throws ClassNotFoundException {
+                public Class loadClass(String name, boolean resolve) throws ClassNotFoundException {
                     if (alias.equals(name)) {
                         return NestedClass.class;
                     } else {
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SinkConnectorConfig.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SinkConnectorConfig.java
index d8912a1350213..0672f4e9c7f7f 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SinkConnectorConfig.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SinkConnectorConfig.java
@@ -121,6 +121,6 @@ public boolean isDlqContextHeadersEnabled() {
     }
 
     public static void main(String[] args) {
-        System.out.println(config.toHtmlTable());
+        System.out.println(config.toHtml());
     }
 }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SourceConnectorConfig.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SourceConnectorConfig.java
index ad891b6945033..a0ee46893acaa 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SourceConnectorConfig.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SourceConnectorConfig.java
@@ -34,6 +34,6 @@ public SourceConnectorConfig(Plugins plugins, Map props) {
     }
 
     public static void main(String[] args) {
-        System.out.println(config.toHtmlTable());
+        System.out.println(config.toHtml());
     }
 }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java
index 0d66c35f68179..f383482d9fdd6 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java
@@ -317,6 +317,6 @@ public DistributedConfig(Map props) {
     }
 
     public static void main(String[] args) {
-        System.out.println(CONFIG.toHtmlTable());
+        System.out.println(CONFIG.toHtml());
     }
 }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/tools/TransformationDoc.java b/connect/runtime/src/main/java/org/apache/kafka/connect/tools/TransformationDoc.java
index b76e7d49b5d5c..77d7728e2434b 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/tools/TransformationDoc.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/tools/TransformationDoc.java
@@ -74,7 +74,7 @@ private static void printTransformationHtml(PrintStream out, DocInfo docInfo) {
 
         out.println("
                ");
 
-        out.println(docInfo.configDef.toHtmlTable());
+        out.println(docInfo.configDef.toHtml());
 
         out.println("");
     }
diff --git a/core/src/main/scala/kafka/log/LogConfig.scala b/core/src/main/scala/kafka/log/LogConfig.scala
index 6d93d7a276e76..4f2671614bb06 100755
--- a/core/src/main/scala/kafka/log/LogConfig.scala
+++ b/core/src/main/scala/kafka/log/LogConfig.scala
@@ -113,7 +113,7 @@ case class LogConfig(props: java.util.Map[_, _], overriddenConfigs: Set[String]
 object LogConfig {
 
   def main(args: Array[String]): Unit = {
-    println(configDef.toHtmlTable)
+    println(configDef.toHtml)
   }
 
   val SegmentBytesProp = TopicConfig.SEGMENT_BYTES_CONFIG
diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index 4d6a65d79a13d..6d106c7b7594f 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -258,7 +258,7 @@ object KafkaConfig {
   private val LogConfigPrefix = "log."
 
   def main(args: Array[String]): Unit = {
-    System.out.println(configDef.toHtmlTable(DynamicBrokerConfig.dynamicConfigUpdateModes))
+    System.out.println(configDef.toHtml(DynamicBrokerConfig.dynamicConfigUpdateModes))
   }
 
   /** ********* Zookeeper Configuration ***********/
@@ -508,7 +508,7 @@ object KafkaConfig {
   "To avoid conflicts between zookeeper generated broker id's and user configured broker id's, generated broker ids " +
   "start from " + MaxReservedBrokerIdProp + " + 1."
   val MessageMaxBytesDoc = TopicConfig.MAX_MESSAGE_BYTES_DOC +
-    s"
                This can be set per topic with the topic level ${TopicConfig.MAX_MESSAGE_BYTES_CONFIG} config.
                "
+    s"This can be set per topic with the topic level ${TopicConfig.MAX_MESSAGE_BYTES_CONFIG} config."
   val NumNetworkThreadsDoc = "The number of threads that the server uses for receiving requests from the network and sending responses to the network"
   val NumIoThreadsDoc = "The number of threads that the server uses for processing requests, which may include disk I/O"
   val NumReplicaAlterLogDirsThreadsDoc = "The number of threads that can move replicas between log directories, which may include disk I/O"
@@ -741,7 +741,7 @@ object KafkaConfig {
   val TransactionsTopicPartitionsDoc = "The number of partitions for the transaction topic (should not change after deployment)."
   val TransactionsTopicSegmentBytesDoc = "The transaction topic segment bytes should be kept relatively small in order to facilitate faster log compaction and cache loads"
   val TransactionsAbortTimedOutTransactionsIntervalMsDoc = "The interval at which to rollback transactions that have timed out"
-  val TransactionsRemoveExpiredTransactionsIntervalMsDoc = "The interval at which to remove transactions that have expired due to transactional.id.expiration.ms passing"
+  val TransactionsRemoveExpiredTransactionsIntervalMsDoc = "The interval at which to remove transactions that have expired due to transactional.id.expiration.ms passing"
 
   /** ********* Fetch Session Configuration **************/
   val MaxIncrementalFetchSessionCacheSlotsDoc = "The maximum number of incremental fetch sessions that we will maintain."
diff --git a/core/src/test/scala/unit/kafka/log/LogConfigTest.scala b/core/src/test/scala/unit/kafka/log/LogConfigTest.scala
index e39f697efcc13..556810eff6fbf 100644
--- a/core/src/test/scala/unit/kafka/log/LogConfigTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogConfigTest.scala
@@ -115,14 +115,22 @@ class LogConfigTest {
     assertFalse(isValid("100:0,10 :   "))
   }
 
-  /* Sanity check that toHtml produces one of the expected configs */
+  /* Sanity check that toHtmlTable produces one of the expected configs */
   @Test
-  def testToHtml(): Unit = {
+  def testToHtmlTable(): Unit = {
     val html = LogConfig.configDefCopy.toHtmlTable
     val expectedConfig = "
                • "
     assertTrue(s"Could not find `$expectedConfig` in:\n $html", html.contains(expectedConfig))
   }
 
+  /* Sanity check that toHtml produces one of the expected configs */
+  @Test
+  def testToHtml(): Unit = {
+    val html = LogConfig.configDefCopy.toHtml
+    val expectedConfig = "
              • file.delete.delay.ms"
+    assertTrue(s"Could not find `$expectedConfig` in:\n $html", html.contains(expectedConfig))
+  }
+
   /* Sanity check that toEnrichedRst produces one of the expected configs */
   @Test
   def testToEnrichedRst(): Unit = {
diff --git a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
index c0eda800e6a56..bbce717324ee6 100644
--- a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
+++ b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
@@ -1259,6 +1259,6 @@ private Map clientProps(final Set configNames,
     }
 
     public static void main(final String[] args) {
-        System.out.println(CONFIG.toHtmlTable());
+        System.out.println(CONFIG.toHtml());
     }
 }

From e24d0e22abb0fb3e4cb3974284a3dad126544584 Mon Sep 17 00:00:00 2001
From: David Jacot 
Date: Sat, 14 Sep 2019 01:49:25 +0200
Subject: [PATCH 0612/1071] KAFKA-8730; Add API to delete consumer offsets
 (KIP-496) (#7276)

This adds an administrative API to delete consumer offsets of a group as well as extends the mechanism to expire offsets of consumer groups.

It makes the group coordinator aware of the set of topics a consumer group (protocol type == 'consumer') is actively subscribed to, allowing offsets of topics which are not actively subscribed to by the group to be either expired or administratively deleted. The expiration rules remain the same.

For the other groups (non-consumer), the API allows to delete offsets when the group is empty and the expiration remains the same.

Reviewers: Stanislav Kozlovski , Jason Gustafson 
---
 .../org/apache/kafka/clients/admin/Admin.java |  23 ++
 .../DeleteConsumerGroupOffsetsOptions.java    |  30 +++
 .../DeleteConsumerGroupOffsetsResult.java     |  84 ++++++
 .../kafka/clients/admin/KafkaAdminClient.java |  93 +++++++
 .../consumer/internals/ConsumerProtocol.java  |  17 +-
 .../GroupSubscribedToTopicException.java      |  23 ++
 .../apache/kafka/common/protocol/ApiKeys.java |   5 +-
 .../apache/kafka/common/protocol/Errors.java  |   5 +-
 .../common/requests/AbstractRequest.java      |   2 +
 .../common/requests/AbstractResponse.java     |   2 +
 .../common/requests/OffsetDeleteRequest.java  |  83 ++++++
 .../common/requests/OffsetDeleteResponse.java |  95 +++++++
 .../common/message/OffsetDeleteRequest.json   |  37 +++
 .../common/message/OffsetDeleteResponse.json  |  41 +++
 .../clients/admin/KafkaAdminClientTest.java   | 253 ++++++++++++++++++
 .../kafka/clients/admin/MockAdminClient.java  |   5 +
 .../common/requests/RequestResponseTest.java  |  51 ++++
 .../coordinator/group/GroupCoordinator.scala  |  87 ++++++
 .../coordinator/group/GroupMetadata.scala     |  81 +++++-
 .../group/GroupMetadataManager.scala          |  25 ++
 .../main/scala/kafka/server/KafkaApis.scala   |  52 ++++
 .../api/AdminClientIntegrationTest.scala      |  82 +++++-
 .../kafka/api/AuthorizerIntegrationTest.scala |  74 ++++-
 .../group/GroupCoordinatorTest.scala          | 215 ++++++++++++++-
 .../group/GroupMetadataManagerTest.scala      | 206 ++++++++++++++
 .../coordinator/group/GroupMetadataTest.scala |  55 ++++
 .../unit/kafka/server/RequestQuotaTest.scala  |  12 +
 27 files changed, 1717 insertions(+), 21 deletions(-)
 create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupOffsetsOptions.java
 create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupOffsetsResult.java
 create mode 100644 clients/src/main/java/org/apache/kafka/common/errors/GroupSubscribedToTopicException.java
 create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java
 create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java
 create mode 100644 clients/src/main/resources/common/message/OffsetDeleteRequest.json
 create mode 100644 clients/src/main/resources/common/message/OffsetDeleteResponse.json

diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java b/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java
index 48630d46bb161..915bd728e18c5 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java
@@ -837,6 +837,29 @@ default DeleteConsumerGroupsResult deleteConsumerGroups(Collection group
         return deleteConsumerGroups(groupIds, new DeleteConsumerGroupsOptions());
     }
 
+    /**
+     * Delete committed offsets for a set of partitions in a consumer group. This will
+     * succeed at the partition level only if the group is not actively subscribed
+     * to the corresponding topic.
+     *
+     * @param options The options to use when deleting offsets in a consumer group.
+     * @return The DeleteConsumerGroupOffsetsResult.
+     */
+    DeleteConsumerGroupOffsetsResult deleteConsumerGroupOffsets(String groupId,
+        Set partitions,
+        DeleteConsumerGroupOffsetsOptions options);
+
+    /**
+     * Delete committed offsets for a set of partitions in a consumer group with the default
+     * options. This will succeed at the partition level only if the group is not actively
+     * subscribed to the corresponding topic.
+     *
+     * @return The DeleteConsumerGroupOffsetsResult.
+     */
+    default DeleteConsumerGroupOffsetsResult deleteConsumerGroupOffsets(String groupId, Set partitions) {
+        return deleteConsumerGroupOffsets(groupId, partitions, new DeleteConsumerGroupOffsetsOptions());
+    }
+
     /**
      * Elect the preferred replica as leader for topic partitions.
      * 
                
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupOffsetsOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupOffsetsOptions.java
new file mode 100644
index 0000000000000..63e6b4be84bae
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupOffsetsOptions.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.admin;
+
+import java.util.Set;
+import org.apache.kafka.common.annotation.InterfaceStability;
+
+/**
+ * Options for the {@link Admin#deleteConsumerGroupOffsets(String, Set)} call.
+ *
+ * The API of this class is evolving, see {@link Admin} for details.
+ */
+@InterfaceStability.Evolving
+public class DeleteConsumerGroupOffsetsOptions extends AbstractOptions {
+
+}
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupOffsetsResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupOffsetsResult.java
new file mode 100644
index 0000000000000..433f478c38623
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupOffsetsResult.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.admin;
+
+import java.util.Set;
+import org.apache.kafka.common.KafkaFuture;
+import org.apache.kafka.common.KafkaFuture.BaseFunction;
+import org.apache.kafka.common.KafkaFuture.BiConsumer;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.annotation.InterfaceStability;
+
+import java.util.Map;
+import org.apache.kafka.common.internals.KafkaFutureImpl;
+import org.apache.kafka.common.protocol.Errors;
+
+/**
+ * The result of the {@link Admin#deleteConsumerGroupOffsets(String, Set)} call.
+ *
+ * The API of this class is evolving, see {@link Admin} for details.
+ */
+@InterfaceStability.Evolving
+public class DeleteConsumerGroupOffsetsResult {
+    private final KafkaFuture> future;
+
+    DeleteConsumerGroupOffsetsResult(KafkaFuture> future) {
+        this.future = future;
+    }
+
+    /**
+     * Return a future which can be used to check the result for a given partition.
+     */
+    public KafkaFuture partitionResult(final TopicPartition partition) {
+        final KafkaFutureImpl result = new KafkaFutureImpl<>();
+
+        this.future.whenComplete(new BiConsumer, Throwable>() {
+            @Override
+            public void accept(final Map topicPartitions, final Throwable throwable) {
+                if (throwable != null) {
+                    result.completeExceptionally(throwable);
+                } else if (!topicPartitions.containsKey(partition)) {
+                    result.completeExceptionally(new IllegalArgumentException(
+                        "Group offset deletion for partition \"" + partition +
+                        "\" was not attempted"));
+                } else {
+                    final Errors error = topicPartitions.get(partition);
+                    if (error == Errors.NONE) {
+                        result.complete(null);
+                    } else {
+                        result.completeExceptionally(error.exception());
+                    }
+                }
+
+            }
+        });
+
+        return result;
+    }
+
+    /**
+     * Return a future which succeeds only if all the deletions succeed.
+     */
+    public KafkaFuture all() {
+        return this.future.thenApply(new BaseFunction, Void>() {
+            @Override
+            public Void apply(final Map topicPartitionErrorsMap) {
+                return null;
+            }
+        });
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
index c7ea97005a516..ce9df22fef9b3 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
@@ -85,6 +85,10 @@
 import org.apache.kafka.common.message.ListGroupsResponseData;
 import org.apache.kafka.common.message.ListPartitionReassignmentsRequestData;
 import org.apache.kafka.common.message.MetadataRequestData;
+import org.apache.kafka.common.message.OffsetDeleteRequestData;
+import org.apache.kafka.common.message.OffsetDeleteRequestData.OffsetDeleteRequestPartition;
+import org.apache.kafka.common.message.OffsetDeleteRequestData.OffsetDeleteRequestTopic;
+import org.apache.kafka.common.message.OffsetDeleteRequestData.OffsetDeleteRequestTopicCollection;
 import org.apache.kafka.common.message.RenewDelegationTokenRequestData;
 import org.apache.kafka.common.metrics.JmxReporter;
 import org.apache.kafka.common.metrics.MetricConfig;
@@ -151,6 +155,9 @@
 import org.apache.kafka.common.requests.ListPartitionReassignmentsResponse;
 import org.apache.kafka.common.requests.MetadataRequest;
 import org.apache.kafka.common.requests.MetadataResponse;
+import org.apache.kafka.common.requests.OffsetDeleteRequest;
+import org.apache.kafka.common.requests.OffsetDeleteRequest.Builder;
+import org.apache.kafka.common.requests.OffsetDeleteResponse;
 import org.apache.kafka.common.requests.OffsetFetchRequest;
 import org.apache.kafka.common.requests.OffsetFetchResponse;
 import org.apache.kafka.common.requests.RenewDelegationTokenRequest;
@@ -3055,6 +3062,92 @@ void handleFailure(Throwable throwable) {
         };
     }
 
+    @Override
+    public DeleteConsumerGroupOffsetsResult deleteConsumerGroupOffsets(
+            String groupId,
+            Set partitions,
+            DeleteConsumerGroupOffsetsOptions options) {
+        final KafkaFutureImpl> future = new KafkaFutureImpl<>();
+
+        if (groupIdIsUnrepresentable(groupId)) {
+            future.completeExceptionally(new InvalidGroupIdException("The given group id '" +
+                groupId + "' cannot be represented in a request."));
+            return new DeleteConsumerGroupOffsetsResult(future);
+        }
+
+        final long startFindCoordinatorMs = time.milliseconds();
+        final long deadline = calcDeadlineMs(startFindCoordinatorMs, options.timeoutMs());
+        ConsumerGroupOperationContext, DeleteConsumerGroupOffsetsOptions> context =
+            new ConsumerGroupOperationContext<>(groupId, options, deadline, future);
+
+        Call findCoordinatorCall = getFindCoordinatorCall(context,
+            () -> KafkaAdminClient.this.getDeleteConsumerGroupOffsetsCall(context, partitions));
+        runnable.call(findCoordinatorCall, startFindCoordinatorMs);
+
+        return new DeleteConsumerGroupOffsetsResult(future);
+    }
+
+    private Call getDeleteConsumerGroupOffsetsCall(
+            ConsumerGroupOperationContext, DeleteConsumerGroupOffsetsOptions> context,
+            Set partitions) {
+        return new Call("deleteConsumerGroupOffsets", context.getDeadline(), new ConstantNodeIdProvider(context.getNode().get().id())) {
+
+            @Override
+            AbstractRequest.Builder createRequest(int timeoutMs) {
+                final OffsetDeleteRequestTopicCollection topics = new OffsetDeleteRequestTopicCollection();
+
+                partitions.stream().collect(Collectors.groupingBy(TopicPartition::topic)).forEach((topic, topicPartitions) -> {
+                    topics.add(
+                        new OffsetDeleteRequestTopic()
+                        .setName(topic)
+                        .setPartitions(topicPartitions.stream()
+                            .map(tp -> new OffsetDeleteRequestPartition().setPartitionIndex(tp.partition()))
+                            .collect(Collectors.toList())
+                        )
+                    );
+                });
+
+                return new OffsetDeleteRequest.Builder(
+                    new OffsetDeleteRequestData()
+                        .setGroupId(context.groupId)
+                        .setTopics(topics)
+                );
+            }
+
+            @Override
+            void handleResponse(AbstractResponse abstractResponse) {
+                final OffsetDeleteResponse response = (OffsetDeleteResponse) abstractResponse;
+
+                // If coordinator changed since we fetched it, retry
+                if (context.hasCoordinatorMoved(response)) {
+                    rescheduleTask(context, () -> getDeleteConsumerGroupOffsetsCall(context, partitions));
+                    return;
+                }
+
+                // If the error is an error at the group level, the future is failed with it
+                final Errors groupError = Errors.forCode(response.data.errorCode());
+                if (handleGroupRequestError(groupError, context.getFuture()))
+                    return;
+
+                final Map partitions = new HashMap<>();
+                response.data.topics().forEach(topic -> {
+                    topic.partitions().forEach(partition -> {
+                        partitions.put(
+                            new TopicPartition(topic.name(), partition.partitionIndex()),
+                            Errors.forCode(partition.errorCode()));
+                    });
+                });
+
+                context.getFuture().complete(partitions);
+            }
+
+            @Override
+            void handleFailure(Throwable throwable) {
+                context.getFuture().completeExceptionally(throwable);
+            }
+        };
+    }
+
     @Override
     public Map metrics() {
         return Collections.unmodifiableMap(this.metrics.metrics());
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java
index e852e62cd46b6..8b29835d62968 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java
@@ -90,7 +90,7 @@ public class ConsumerProtocol {
     public static final short CONSUMER_PROTOCOL_V0 = 0;
     public static final short CONSUMER_PROTOCOL_V1 = 1;
 
-    public static final short CONSUMER_PROTOCL_LATEST_VERSION = CONSUMER_PROTOCOL_V1;
+    public static final short CONSUMER_PROTOCOL_LATEST_VERSION = CONSUMER_PROTOCOL_V1;
 
     public static final Schema CONSUMER_PROTOCOL_HEADER_SCHEMA = new Schema(
             new Field(VERSION_KEY_NAME, Type.INT16));
@@ -120,6 +120,11 @@ public class ConsumerProtocol {
         new Field(TOPIC_PARTITIONS_KEY_NAME, new ArrayOf(TOPIC_ASSIGNMENT_V0)),
         new Field(USER_DATA_KEY_NAME, Type.NULLABLE_BYTES));
 
+    public static Short deserializeVersion(ByteBuffer buffer) {
+        Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer);
+        return header.getShort(VERSION_KEY_NAME);
+    }
+
     public static ByteBuffer serializeSubscriptionV0(Subscription subscription) {
         Struct struct = new Struct(SUBSCRIPTION_V0);
         struct.set(USER_DATA_KEY_NAME, subscription.userData());
@@ -154,7 +159,7 @@ public static ByteBuffer serializeSubscriptionV1(Subscription subscription) {
     }
 
     public static ByteBuffer serializeSubscription(Subscription subscription) {
-        return serializeSubscription(subscription, CONSUMER_PROTOCL_LATEST_VERSION);
+        return serializeSubscription(subscription, CONSUMER_PROTOCOL_LATEST_VERSION);
     }
 
     public static ByteBuffer serializeSubscription(Subscription subscription, short version) {
@@ -201,8 +206,7 @@ public static Subscription deserializeSubscriptionV1(ByteBuffer buffer) {
     }
 
     public static Subscription deserializeSubscription(ByteBuffer buffer) {
-        Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer);
-        Short version = header.getShort(VERSION_KEY_NAME);
+        Short version = deserializeVersion(buffer);
 
         if (version < CONSUMER_PROTOCOL_V0)
             throw new SchemaException("Unsupported subscription version: " + version);
@@ -261,7 +265,7 @@ public static ByteBuffer serializeAssignmentV1(Assignment assignment) {
     }
 
     public static ByteBuffer serializeAssignment(Assignment assignment) {
-        return serializeAssignment(assignment, CONSUMER_PROTOCL_LATEST_VERSION);
+        return serializeAssignment(assignment, CONSUMER_PROTOCOL_LATEST_VERSION);
     }
 
     public static ByteBuffer serializeAssignment(Assignment assignment, short version) {
@@ -297,8 +301,7 @@ public static Assignment deserializeAssignmentV1(ByteBuffer buffer) {
     }
 
     public static Assignment deserializeAssignment(ByteBuffer buffer) {
-        Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer);
-        Short version = header.getShort(VERSION_KEY_NAME);
+        Short version = deserializeVersion(buffer);
 
         if (version < CONSUMER_PROTOCOL_V0)
             throw new SchemaException("Unsupported assignment version: " + version);
diff --git a/clients/src/main/java/org/apache/kafka/common/errors/GroupSubscribedToTopicException.java b/clients/src/main/java/org/apache/kafka/common/errors/GroupSubscribedToTopicException.java
new file mode 100644
index 0000000000000..a62fe325d8136
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/errors/GroupSubscribedToTopicException.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.errors;
+
+public class GroupSubscribedToTopicException extends ApiException {
+    public GroupSubscribedToTopicException(String message) {
+        super(message);
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
index 2393da202369c..b5c1e0dbd6d8d 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
@@ -56,6 +56,8 @@
 import org.apache.kafka.common.message.MetadataResponseData;
 import org.apache.kafka.common.message.OffsetCommitRequestData;
 import org.apache.kafka.common.message.OffsetCommitResponseData;
+import org.apache.kafka.common.message.OffsetDeleteRequestData;
+import org.apache.kafka.common.message.OffsetDeleteResponseData;
 import org.apache.kafka.common.message.OffsetFetchRequestData;
 import org.apache.kafka.common.message.OffsetFetchResponseData;
 import org.apache.kafka.common.message.RenewDelegationTokenRequestData;
@@ -201,7 +203,8 @@ public Struct parseResponse(short version, ByteBuffer buffer) {
     ALTER_PARTITION_REASSIGNMENTS(45, "AlterPartitionReassignments", AlterPartitionReassignmentsRequestData.SCHEMAS,
                                   AlterPartitionReassignmentsResponseData.SCHEMAS),
     LIST_PARTITION_REASSIGNMENTS(46, "ListPartitionReassignments", ListPartitionReassignmentsRequestData.SCHEMAS,
-                                 ListPartitionReassignmentsResponseData.SCHEMAS);
+                                 ListPartitionReassignmentsResponseData.SCHEMAS),
+    OFFSET_DELETE(47, "OffsetDelete", OffsetDeleteRequestData.SCHEMAS, OffsetDeleteResponseData.SCHEMAS);
 
     private static final ApiKeys[] ID_TO_TYPE;
     private static final int MIN_API_KEY = 0;
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java
index 9f11fc89adf05..0425ef1a4d313 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java
@@ -20,6 +20,7 @@
 import org.apache.kafka.common.errors.BrokerNotAvailableException;
 import org.apache.kafka.common.errors.ClusterAuthorizationException;
 import org.apache.kafka.common.errors.ConcurrentTransactionsException;
+import org.apache.kafka.common.errors.GroupSubscribedToTopicException;
 import org.apache.kafka.common.errors.ControllerMovedException;
 import org.apache.kafka.common.errors.CoordinatorLoadInProgressException;
 import org.apache.kafka.common.errors.CoordinatorNotAvailableException;
@@ -312,7 +313,9 @@ public enum Errors {
             EligibleLeadersNotAvailableException::new),
     ELECTION_NOT_NEEDED(84, "Leader election not needed for topic partition", ElectionNotNeededException::new),
     NO_REASSIGNMENT_IN_PROGRESS(85, "No partition reassignment is in progress.",
-            NoReassignmentInProgressException::new);
+            NoReassignmentInProgressException::new),
+    GROUP_SUBSCRIBED_TO_TOPIC(86, "Deleting offsets of a topic is forbidden while the consumer group is actively subscribed to it.",
+        GroupSubscribedToTopicException::new);
 
     private static final Logger log = LoggerFactory.getLogger(Errors.class);
 
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
index 58bf1283a77a1..b737b49f4a5c1 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
@@ -237,6 +237,8 @@ public static AbstractRequest parseRequest(ApiKeys apiKey, short apiVersion, Str
                 return new AlterPartitionReassignmentsRequest(struct, apiVersion);
             case LIST_PARTITION_REASSIGNMENTS:
                 return new ListPartitionReassignmentsRequest(struct, apiVersion);
+            case OFFSET_DELETE:
+                return new OffsetDeleteRequest(struct, apiVersion);
             default:
                 throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseRequest`, the " +
                         "code should be updated to do so.", apiKey));
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
index 11466d7249867..64002aac211e9 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
@@ -164,6 +164,8 @@ public static AbstractResponse parseResponse(ApiKeys apiKey, Struct struct, shor
                 return new AlterPartitionReassignmentsResponse(struct, version);
             case LIST_PARTITION_REASSIGNMENTS:
                 return new ListPartitionReassignmentsResponse(struct, version);
+            case OFFSET_DELETE:
+                return new OffsetDeleteResponse(struct, version);
             default:
                 throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseResponse`, the " +
                         "code should be updated to do so.", apiKey));
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java
new file mode 100644
index 0000000000000..293efc2f6f2db
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.requests;
+
+import org.apache.kafka.common.message.OffsetDeleteRequestData;
+import org.apache.kafka.common.message.OffsetDeleteResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.protocol.types.Struct;
+
+import java.nio.ByteBuffer;
+
+public class OffsetDeleteRequest extends AbstractRequest {
+
+    public static class Builder extends AbstractRequest.Builder {
+
+        private final OffsetDeleteRequestData data;
+
+        public Builder(OffsetDeleteRequestData data) {
+            super(ApiKeys.OFFSET_DELETE);
+            this.data = data;
+        }
+
+        @Override
+        public OffsetDeleteRequest build(short version) {
+            return new OffsetDeleteRequest(data, version);
+        }
+
+        @Override
+        public String toString() {
+            return data.toString();
+        }
+    }
+
+    public final OffsetDeleteRequestData data;
+
+    public OffsetDeleteRequest(OffsetDeleteRequestData data, short version) {
+        super(ApiKeys.OFFSET_DELETE, version);
+        this.data = data;
+    }
+
+    public OffsetDeleteRequest(Struct struct, short version) {
+        super(ApiKeys.OFFSET_DELETE, version);
+        this.data = new OffsetDeleteRequestData(struct, version);
+    }
+
+    public AbstractResponse getErrorResponse(int throttleTimeMs, Errors error) {
+        return new OffsetDeleteResponse(
+            new OffsetDeleteResponseData()
+                .setThrottleTimeMs(throttleTimeMs)
+                .setErrorCode(error.code())
+        );
+    }
+
+    @Override
+    public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
+        return getErrorResponse(throttleTimeMs, Errors.forException(e));
+    }
+
+    public static OffsetDeleteRequest parse(ByteBuffer buffer, short version) {
+        return new OffsetDeleteRequest(ApiKeys.OFFSET_DELETE.parseRequest(version, buffer),
+            version);
+    }
+
+    @Override
+    protected Struct toStruct() {
+        return data.toStruct(version());
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java
new file mode 100644
index 0000000000000..ff0c2967be890
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.requests;
+
+import org.apache.kafka.common.message.OffsetDeleteResponseData;
+import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponsePartition;
+import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopic;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.protocol.types.Struct;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Possible error codes:
+ *
+ * - Partition errors:
+ *   - {@link Errors#GROUP_SUBSCRIBED_TO_TOPIC}
+ *   - {@link Errors#TOPIC_AUTHORIZATION_FAILED}
+ *   - {@link Errors#UNKNOWN_TOPIC_OR_PARTITION}
+ *
+ * - Group or coordinator errors:
+ *   - {@link Errors#COORDINATOR_LOAD_IN_PROGRESS}
+ *   - {@link Errors#COORDINATOR_NOT_AVAILABLE}
+ *   - {@link Errors#NOT_COORDINATOR}
+ *   - {@link Errors#GROUP_AUTHORIZATION_FAILED}
+ *   - {@link Errors#INVALID_GROUP_ID}
+ *   - {@link Errors#GROUP_ID_NOT_FOUND}
+ *   - {@link Errors#NON_EMPTY_GROUP}
+ */
+public class OffsetDeleteResponse extends AbstractResponse {
+
+    public final OffsetDeleteResponseData data;
+
+    public OffsetDeleteResponse(OffsetDeleteResponseData data) {
+        this.data = data;
+    }
+
+    public OffsetDeleteResponse(Struct struct) {
+        short latestVersion = (short) (OffsetDeleteResponseData.SCHEMAS.length - 1);
+        this.data = new OffsetDeleteResponseData(struct, latestVersion);
+    }
+
+    public OffsetDeleteResponse(Struct struct, short version) {
+        this.data = new OffsetDeleteResponseData(struct, version);
+    }
+
+    @Override
+    protected Struct toStruct(short version) {
+        return data.toStruct(version);
+    }
+
+    @Override
+    public Map errorCounts() {
+        Map counts = new HashMap<>();
+        counts.put(Errors.forCode(data.errorCode()), 1);
+        for (OffsetDeleteResponseTopic topic : data.topics()) {
+            for (OffsetDeleteResponsePartition partition : topic.partitions()) {
+                Errors error = Errors.forCode(partition.errorCode());
+                counts.put(error, counts.getOrDefault(error, 0) + 1);
+            }
+        }
+        return counts;
+    }
+
+    public static OffsetDeleteResponse parse(ByteBuffer buffer, short version) {
+        return new OffsetDeleteResponse(ApiKeys.OFFSET_DELETE.parseResponse(version, buffer));
+    }
+
+    @Override
+    public int throttleTimeMs() {
+        return data.throttleTimeMs();
+    }
+
+    @Override
+    public boolean shouldClientThrottle(short version) {
+        return version >= 0;
+    }
+}
diff --git a/clients/src/main/resources/common/message/OffsetDeleteRequest.json b/clients/src/main/resources/common/message/OffsetDeleteRequest.json
new file mode 100644
index 0000000000000..563594997097c
--- /dev/null
+++ b/clients/src/main/resources/common/message/OffsetDeleteRequest.json
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+{
+  "apiKey": 47,
+  "type": "request",
+  "name": "OffsetDeleteRequest",
+  "validVersions": "0",
+  "fields": [
+    { "name": "GroupId", "type": "string", "versions": "0+",
+      "about": "The unique group identifier." },
+    { "name": "Topics", "type": "[]OffsetDeleteRequestTopic", "versions": "0+",
+      "about": "The topics to delete offsets for", "fields": [
+        { "name": "Name",  "type": "string",  "versions": "0+", "mapKey": true,
+          "about": "The topic name." },
+        { "name": "Partitions", "type": "[]OffsetDeleteRequestPartition", "versions": "0+",
+          "about": "Each partition to delete offsets for.", "fields": [
+            { "name": "PartitionIndex", "type": "int32", "versions": "0+",
+              "about": "The partition index." }
+          ]
+        }
+      ]
+    }
+  ]
+}
diff --git a/clients/src/main/resources/common/message/OffsetDeleteResponse.json b/clients/src/main/resources/common/message/OffsetDeleteResponse.json
new file mode 100644
index 0000000000000..f2be321f5d8fd
--- /dev/null
+++ b/clients/src/main/resources/common/message/OffsetDeleteResponse.json
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+{
+  "apiKey": 47,
+  "type": "response",
+  "name": "OffsetDeleteResponse",
+  "validVersions": "0",
+  "fields": [
+    { "name": "ErrorCode", "type": "int16", "versions": "0+",
+      "about": "The top-level error code, or 0 if there was no error." },
+    { "name": "ThrottleTimeMs",  "type": "int32",  "versions": "0+", "ignorable": true,
+      "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
+    { "name": "Topics", "type": "[]OffsetDeleteResponseTopic", "versions": "0+",
+      "about": "The responses for each topic.", "fields": [
+        { "name": "Name", "type": "string", "versions": "0+", "mapKey": true,
+          "about": "The topic name." },
+        { "name": "Partitions", "type": "[]OffsetDeleteResponsePartition", "versions": "0+",
+          "about": "The responses for each partition in the topic.", "fields": [
+            { "name": "PartitionIndex", "type": "int32", "versions": "0+", "mapKey": true,
+              "about": "The partition index." },
+            { "name": "ErrorCode", "type": "int16", "versions": "0+",
+              "about": "The error code, or 0 if there was no error." }
+          ]
+        }
+      ]
+    }
+  ]
+}
diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
index 5d4d84b2d47c9..b6fece991d247 100644
--- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
@@ -16,6 +16,8 @@
  */
 package org.apache.kafka.clients.admin;
 
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import org.apache.kafka.clients.ClientDnsLookup;
 import org.apache.kafka.clients.ClientUtils;
 import org.apache.kafka.clients.MockClient;
@@ -41,6 +43,7 @@
 import org.apache.kafka.common.errors.AuthenticationException;
 import org.apache.kafka.common.errors.ClusterAuthorizationException;
 import org.apache.kafka.common.errors.GroupAuthorizationException;
+import org.apache.kafka.common.errors.GroupSubscribedToTopicException;
 import org.apache.kafka.common.errors.InvalidRequestException;
 import org.apache.kafka.common.errors.InvalidTopicException;
 import org.apache.kafka.common.errors.LeaderNotAvailableException;
@@ -72,6 +75,11 @@
 import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse;
 import org.apache.kafka.common.message.ListGroupsResponseData;
 import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData;
+import org.apache.kafka.common.message.OffsetDeleteResponseData;
+import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponsePartition;
+import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponsePartitionCollection;
+import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopic;
+import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopicCollection;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.requests.AlterPartitionReassignmentsResponse;
 import org.apache.kafka.common.requests.ApiError;
@@ -98,6 +106,7 @@
 import org.apache.kafka.common.requests.ListPartitionReassignmentsResponse;
 import org.apache.kafka.common.requests.MetadataRequest;
 import org.apache.kafka.common.requests.MetadataResponse;
+import org.apache.kafka.common.requests.OffsetDeleteResponse;
 import org.apache.kafka.common.requests.OffsetFetchResponse;
 import org.apache.kafka.common.resource.PatternType;
 import org.apache.kafka.common.resource.ResourcePattern;
@@ -242,6 +251,30 @@ public void testCloseAdminClient() {
         }
     }
 
+    private static OffsetDeleteResponse prepareOffsetDeleteResponse(Errors error) {
+        return new OffsetDeleteResponse(
+            new OffsetDeleteResponseData()
+                .setErrorCode(error.code())
+                .setTopics(new OffsetDeleteResponseTopicCollection())
+        );
+    }
+
+    private static OffsetDeleteResponse prepareOffsetDeleteResponse(String topic, int partition, Errors error) {
+        return new OffsetDeleteResponse(
+            new OffsetDeleteResponseData()
+                .setErrorCode(Errors.NONE.code())
+                .setTopics(new OffsetDeleteResponseTopicCollection(Stream.of(
+                    new OffsetDeleteResponseTopic()
+                        .setName(topic)
+                        .setPartitions(new OffsetDeleteResponsePartitionCollection(Collections.singletonList(
+                            new OffsetDeleteResponsePartition()
+                                .setPartitionIndex(partition)
+                                .setErrorCode(error.code())
+                        ).iterator()))
+                ).collect(Collectors.toList()).iterator()))
+        );
+    }
+
     private static CreateTopicsResponse prepareCreateTopicsResponse(String topicName, Errors error) {
         CreateTopicsResponseData data = new CreateTopicsResponseData();
         data.topics().add(new CreatableTopicResult().
@@ -1522,6 +1555,226 @@ public void testDeleteConsumerGroups() throws Exception {
         }
     }
 
+    @Test
+    public void testDeleteConsumerGroupOffsets() throws Exception {
+        // Happy path
+
+        final Map nodes = new HashMap<>();
+        nodes.put(0, new Node(0, "localhost", 8121));
+
+        final Cluster cluster =
+            new Cluster(
+                "mockClusterId",
+                nodes.values(),
+                Collections.emptyList(),
+                Collections.emptySet(),
+                Collections.emptySet(), nodes.get(0));
+
+        final String groupId = "group-0";
+        final TopicPartition tp1 = new TopicPartition("foo", 0);
+        final TopicPartition tp2 = new TopicPartition("bar", 0);
+        final TopicPartition tp3 = new TopicPartition("foobar", 0);
+
+        try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster)) {
+            env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
+
+            env.kafkaClient().prepareResponse(
+                prepareFindCoordinatorResponse(Errors.NONE, env.cluster().controller()));
+
+            env.kafkaClient().prepareResponse(new OffsetDeleteResponse(
+                new OffsetDeleteResponseData()
+                    .setTopics(new OffsetDeleteResponseTopicCollection(Stream.of(
+                        new OffsetDeleteResponseTopic()
+                            .setName("foo")
+                            .setPartitions(new OffsetDeleteResponsePartitionCollection(Collections.singletonList(
+                                new OffsetDeleteResponsePartition()
+                                    .setPartitionIndex(0)
+                                    .setErrorCode(Errors.NONE.code())
+                            ).iterator())),
+                        new OffsetDeleteResponseTopic()
+                            .setName("bar")
+                            .setPartitions(new OffsetDeleteResponsePartitionCollection(Collections.singletonList(
+                                new OffsetDeleteResponsePartition()
+                                    .setPartitionIndex(0)
+                                    .setErrorCode(Errors.GROUP_SUBSCRIBED_TO_TOPIC.code())
+                            ).iterator()))
+                    ).collect(Collectors.toList()).iterator()))
+                )
+            );
+
+            final DeleteConsumerGroupOffsetsResult errorResult = env.adminClient().deleteConsumerGroupOffsets(
+                groupId, Stream.of(tp1, tp2).collect(Collectors.toSet()));
+
+            assertNull(errorResult.all().get());
+            assertNull(errorResult.partitionResult(tp1).get());
+            TestUtils.assertFutureError(errorResult.partitionResult(tp2), GroupSubscribedToTopicException.class);
+            TestUtils.assertFutureError(errorResult.partitionResult(tp3), IllegalArgumentException.class);
+        }
+    }
+
+    @Test
+    public void testDeleteConsumerGroupOffsetsRetriableErrors() throws Exception {
+        // Retriable errors should be retried
+
+        final Map nodes = new HashMap<>();
+        nodes.put(0, new Node(0, "localhost", 8121));
+
+        final Cluster cluster =
+            new Cluster(
+                "mockClusterId",
+                nodes.values(),
+                Collections.emptyList(),
+                Collections.emptySet(),
+                Collections.emptySet(), nodes.get(0));
+
+        final String groupId = "group-0";
+        final TopicPartition tp1 = new TopicPartition("foo", 0);
+
+        try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster)) {
+            env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
+
+            env.kafkaClient().prepareResponse(
+                FindCoordinatorResponse.prepareResponse(Errors.NONE, env.cluster().controller()));
+
+            env.kafkaClient().prepareResponse(
+                prepareOffsetDeleteResponse(Errors.COORDINATOR_NOT_AVAILABLE));
+
+            env.kafkaClient().prepareResponse(
+                prepareOffsetDeleteResponse(Errors.COORDINATOR_LOAD_IN_PROGRESS));
+
+            /*
+             * We need to return two responses here, one for NOT_COORDINATOR call when calling delete a consumer group
+             * api using coordinator that has moved. This will retry whole operation. So we need to again respond with a
+             * FindCoordinatorResponse.
+             */
+            env.kafkaClient().prepareResponse(
+                prepareOffsetDeleteResponse(Errors.NOT_COORDINATOR));
+
+            env.kafkaClient().prepareResponse(
+                prepareFindCoordinatorResponse(Errors.NONE, env.cluster().controller()));
+
+            env.kafkaClient().prepareResponse(
+                prepareOffsetDeleteResponse("foo", 0, Errors.NONE));
+
+            final DeleteConsumerGroupOffsetsResult errorResult1 = env.adminClient()
+                .deleteConsumerGroupOffsets(groupId, Stream.of(tp1).collect(Collectors.toSet()));
+
+            assertNull(errorResult1.all().get());
+            assertNull(errorResult1.partitionResult(tp1).get());
+        }
+    }
+
+    @Test
+    public void testDeleteConsumerGroupOffsetsNonRetriableErrors() throws Exception {
+        // Non-retriable errors throw an exception
+
+        final Map nodes = new HashMap<>();
+        nodes.put(0, new Node(0, "localhost", 8121));
+
+        final Cluster cluster =
+            new Cluster(
+                "mockClusterId",
+                nodes.values(),
+                Collections.emptyList(),
+                Collections.emptySet(),
+                Collections.emptySet(), nodes.get(0));
+
+        final String groupId = "group-0";
+        final TopicPartition tp1 = new TopicPartition("foo", 0);
+        final List retriableErrors = Arrays.asList(
+            Errors.GROUP_AUTHORIZATION_FAILED, Errors.INVALID_GROUP_ID, Errors.GROUP_ID_NOT_FOUND);
+
+        try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster)) {
+            env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
+
+            for (Errors error : retriableErrors) {
+                env.kafkaClient().prepareResponse(FindCoordinatorResponse
+                    .prepareResponse(Errors.NONE, env.cluster().controller()));
+
+                env.kafkaClient().prepareResponse(
+                    prepareOffsetDeleteResponse(error));
+
+                DeleteConsumerGroupOffsetsResult errorResult = env.adminClient()
+                    .deleteConsumerGroupOffsets(groupId, Stream.of(tp1).collect(Collectors.toSet()));
+
+                TestUtils.assertFutureError(errorResult.all(), error.exception().getClass());
+                TestUtils.assertFutureError(errorResult.partitionResult(tp1), error.exception().getClass());
+            }
+        }
+    }
+
+    @Test
+    public void testDeleteConsumerGroupOffsetsFindCoordinatorRetriableErrors() throws Exception {
+        // Retriable FindCoordinatorResponse errors should be retried
+
+        final Map nodes = new HashMap<>();
+        nodes.put(0, new Node(0, "localhost", 8121));
+
+        final Cluster cluster =
+            new Cluster(
+                "mockClusterId",
+                nodes.values(),
+                Collections.emptyList(),
+                Collections.emptySet(),
+                Collections.emptySet(), nodes.get(0));
+
+        final String groupId = "group-0";
+        final TopicPartition tp1 = new TopicPartition("foo", 0);
+
+        try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster)) {
+            env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
+
+            env.kafkaClient().prepareResponse(
+                prepareFindCoordinatorResponse(Errors.COORDINATOR_NOT_AVAILABLE,  Node.noNode()));
+            env.kafkaClient().prepareResponse(
+                prepareFindCoordinatorResponse(Errors.COORDINATOR_LOAD_IN_PROGRESS, Node.noNode()));
+
+            env.kafkaClient().prepareResponse(
+                prepareFindCoordinatorResponse(Errors.NONE, env.cluster().controller()));
+
+            env.kafkaClient().prepareResponse(
+                prepareOffsetDeleteResponse("foo", 0, Errors.NONE));
+
+            final DeleteConsumerGroupOffsetsResult result = env.adminClient()
+                .deleteConsumerGroupOffsets(groupId, Stream.of(tp1).collect(Collectors.toSet()));
+
+            assertNull(result.all().get());
+            assertNull(result.partitionResult(tp1).get());
+        }
+    }
+
+    @Test
+    public void testDeleteConsumerGroupOffsetsFindCoordinatorNonRetriableErrors() throws Exception {
+        // Non-retriable FindCoordinatorResponse errors throw an exception
+
+        final Map nodes = new HashMap<>();
+        nodes.put(0, new Node(0, "localhost", 8121));
+
+        final Cluster cluster =
+            new Cluster(
+                "mockClusterId",
+                nodes.values(),
+                Collections.emptyList(),
+                Collections.emptySet(),
+                Collections.emptySet(), nodes.get(0));
+
+        final String groupId = "group-0";
+        final TopicPartition tp1 = new TopicPartition("foo", 0);
+
+        try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster)) {
+            env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
+
+            env.kafkaClient().prepareResponse(
+                prepareFindCoordinatorResponse(Errors.GROUP_AUTHORIZATION_FAILED,  Node.noNode()));
+
+            final DeleteConsumerGroupOffsetsResult errorResult = env.adminClient()
+                .deleteConsumerGroupOffsets(groupId, Stream.of(tp1).collect(Collectors.toSet()));
+
+            TestUtils.assertFutureError(errorResult.all(), GroupAuthorizationException.class);
+            TestUtils.assertFutureError(errorResult.partitionResult(tp1), GroupAuthorizationException.class);
+        }
+    }
+
     @Test
     public void testIncrementalAlterConfigs()  throws Exception {
         try (AdminClientUnitTestEnv env = mockClientEnv()) {
diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java b/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java
index 86ba572f532d9..bc06468a648eb 100644
--- a/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java
+++ b/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java
@@ -343,6 +343,11 @@ public DeleteConsumerGroupsResult deleteConsumerGroups(Collection groupI
         throw new UnsupportedOperationException("Not implemented yet");
     }
 
+    @Override
+    public DeleteConsumerGroupOffsetsResult deleteConsumerGroupOffsets(String groupId, Set partitions, DeleteConsumerGroupOffsetsOptions options) {
+        throw new UnsupportedOperationException("Not implemented yet");
+    }
+
     @Deprecated
     @Override
     public ElectPreferredLeadersResult electPreferredLeaders(Collection partitions, ElectPreferredLeadersOptions options) {
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index ddf574937e3fb..d813cbdd9b1ce 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -82,6 +82,15 @@
 import org.apache.kafka.common.message.ListGroupsResponseData;
 import org.apache.kafka.common.message.OffsetCommitRequestData;
 import org.apache.kafka.common.message.OffsetCommitResponseData;
+import org.apache.kafka.common.message.OffsetDeleteRequestData;
+import org.apache.kafka.common.message.OffsetDeleteRequestData.OffsetDeleteRequestPartition;
+import org.apache.kafka.common.message.OffsetDeleteRequestData.OffsetDeleteRequestTopic;
+import org.apache.kafka.common.message.OffsetDeleteRequestData.OffsetDeleteRequestTopicCollection;
+import org.apache.kafka.common.message.OffsetDeleteResponseData;
+import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponsePartition;
+import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponsePartitionCollection;
+import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopic;
+import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopicCollection;
 import org.apache.kafka.common.message.RenewDelegationTokenRequestData;
 import org.apache.kafka.common.message.RenewDelegationTokenResponseData;
 import org.apache.kafka.common.message.SaslAuthenticateRequestData;
@@ -377,6 +386,9 @@ public void testSerialization() throws Exception {
         checkRequest(createListPartitionReassignmentsRequest(), true);
         checkErrorResponse(createListPartitionReassignmentsRequest(), new UnknownServerException(), true);
         checkResponse(createListPartitionReassignmentsResponse(), 0, true);
+        checkRequest(createOffsetDeleteRequest(), true);
+        checkErrorResponse(createOffsetDeleteRequest(), new UnknownServerException(), true);
+        checkResponse(createOffsetDeleteResponse(), 0, true);
     }
 
     @Test
@@ -1724,4 +1736,43 @@ private ListPartitionReassignmentsResponse createListPartitionReassignmentsRespo
         ));
         return new ListPartitionReassignmentsResponse(data);
     }
+
+    private OffsetDeleteRequest createOffsetDeleteRequest() {
+        OffsetDeleteRequestTopicCollection topics = new OffsetDeleteRequestTopicCollection();
+        topics.add(new OffsetDeleteRequestTopic()
+            .setName("topic1")
+            .setPartitions(Collections.singletonList(
+                new OffsetDeleteRequestPartition()
+                    .setPartitionIndex(0)
+                )
+            )
+        );
+
+        OffsetDeleteRequestData data = new OffsetDeleteRequestData();
+        data.setGroupId("group1");
+        data.setTopics(topics);
+
+        return new OffsetDeleteRequest.Builder(data).build((short) 0);
+    }
+
+    private OffsetDeleteResponse createOffsetDeleteResponse() {
+        OffsetDeleteResponsePartitionCollection partitions = new OffsetDeleteResponsePartitionCollection();
+        partitions.add(new OffsetDeleteResponsePartition()
+            .setPartitionIndex(0)
+            .setErrorCode(Errors.NONE.code())
+        );
+
+        OffsetDeleteResponseTopicCollection topics = new OffsetDeleteResponseTopicCollection();
+        topics.add(new OffsetDeleteResponseTopic()
+            .setName("topic1")
+            .setPartitions(partitions)
+        );
+
+        OffsetDeleteResponseData data = new OffsetDeleteResponseData();
+        data.setErrorCode(Errors.NONE.code());
+        data.setTopics(topics);
+
+        return new OffsetDeleteResponse(data);
+    }
+
 }
diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
index 9e52e516b05b8..0be2a48867e30 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
@@ -30,6 +30,7 @@ import org.apache.kafka.common.internals.Topic
 import org.apache.kafka.common.message.JoinGroupResponseData.JoinGroupResponseMember
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
+import org.apache.kafka.common.metrics.stats.Meter
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
 import org.apache.kafka.common.record.RecordBatch.{NO_PRODUCER_EPOCH, NO_PRODUCER_ID}
 import org.apache.kafka.common.requests._
@@ -63,6 +64,27 @@ class GroupCoordinator(val brokerId: Int,
   type JoinCallback = JoinGroupResult => Unit
   type SyncCallback = SyncGroupResult => Unit
 
+  /* setup metrics */
+  val offsetDeletionSensor = metrics.sensor("OffsetDeletions")
+
+  offsetDeletionSensor.add(new Meter(
+    metrics.metricName("offset-deletion-rate",
+      "group-coordinator-metrics",
+      "The rate of administrative deleted offsets"),
+    metrics.metricName("offset-deletion-count",
+      "group-coordinator-metrics",
+      "The total number of administrative deleted offsets")))
+
+  val groupCompletedRebalanceSensor = metrics.sensor("OffsetDeletions")
+
+  groupCompletedRebalanceSensor.add(new Meter(
+    metrics.metricName("group-completed-rebalance-rate",
+      "group-coordinator-metrics",
+      "The rate of completed rebalance"),
+    metrics.metricName("group-completed-rebalance-count",
+      "group-coordinator-metrics",
+      "The total number of completed rebalance")))
+
   this.logIdent = "[GroupCoordinator " + brokerId + "]: "
 
   private val isActive = new AtomicBoolean(false)
@@ -407,6 +429,7 @@ class GroupCoordinator(val brokerId: Int,
                   }
                 }
               })
+              groupCompletedRebalanceSensor.record()
             }
 
           case Stable =>
@@ -519,6 +542,70 @@ class GroupCoordinator(val brokerId: Int,
     groupErrors
   }
 
+  def handleDeleteOffsets(groupId: String, partitions: Seq[TopicPartition]): (Errors, Map[TopicPartition, Errors]) = {
+    var groupError: Errors = Errors.NONE
+    var partitionErrors: Map[TopicPartition, Errors] = Map()
+    var partitionEligibleForDeletion: Seq[TopicPartition] = Seq()
+
+    validateGroupStatus(groupId, ApiKeys.OFFSET_DELETE) match {
+      case Some(error) =>
+        groupError = error
+
+      case None =>
+        groupManager.getGroup(groupId) match {
+          case None =>
+            groupError = if (groupManager.groupNotExists(groupId))
+              Errors.GROUP_ID_NOT_FOUND else Errors.NOT_COORDINATOR
+
+          case Some(group) =>
+            group.inLock {
+              group.currentState match {
+                case Dead =>
+                  groupError = if (groupManager.groupNotExists(groupId))
+                    Errors.GROUP_ID_NOT_FOUND else Errors.NOT_COORDINATOR
+
+                case Empty =>
+                  val (knownPartitions, unknownPartitions) =
+                    partitions.partition(tp => group.offset(tp).nonEmpty)
+
+                  partitionEligibleForDeletion = knownPartitions
+                  partitionErrors = unknownPartitions.map(_ -> Errors.UNKNOWN_TOPIC_OR_PARTITION).toMap
+
+                case PreparingRebalance | CompletingRebalance | Stable if group.isConsumerGroup =>
+                  val (knownPartitions, unknownPartitions) =
+                    partitions.partition(tp => group.offset(tp).nonEmpty)
+
+                  val (consumed, notConsumed) =
+                    knownPartitions.partition(tp => group.isSubscribedToTopic(tp.topic()))
+
+                  partitionEligibleForDeletion = notConsumed
+                  partitionErrors = consumed.map(_ -> Errors.GROUP_SUBSCRIBED_TO_TOPIC).toMap ++
+                    unknownPartitions.map(_ -> Errors.UNKNOWN_TOPIC_OR_PARTITION).toMap
+
+                case _ =>
+                  groupError = Errors.NON_EMPTY_GROUP
+              }
+            }
+
+            if (partitionEligibleForDeletion.nonEmpty) {
+              val offsetsRemoved = groupManager.cleanupGroupMetadata(Seq(group), group => {
+                group.removeOffsets(partitionEligibleForDeletion)
+              })
+
+              partitionErrors ++= partitionEligibleForDeletion.map(_ -> Errors.NONE).toMap
+
+              offsetDeletionSensor.record(offsetsRemoved)
+
+              info(s"The following offsets of the group $groupId were deleted: ${partitionEligibleForDeletion.mkString(", ")}. " +
+                s"A total of $offsetsRemoved offsets were removed.")
+            }
+        }
+    }
+
+    // If there is a group error, the partition errors is empty
+    groupError -> partitionErrors
+  }
+
   def handleHeartbeat(groupId: String,
                       memberId: String,
                       groupInstanceId: Option[String],
diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala
index ca42b0f098c0b..b8237686fecb5 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala
@@ -16,17 +16,21 @@
  */
 package kafka.coordinator.group
 
+import java.nio.ByteBuffer
 import java.util.UUID
 import java.util.concurrent.locks.ReentrantLock
 
 import kafka.common.OffsetAndMetadata
 import kafka.utils.{CoreUtils, Logging, nonthreadsafe}
+import org.apache.kafka.clients.consumer.internals.ConsumerProtocol
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.message.JoinGroupResponseData.JoinGroupResponseMember
 import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.protocol.types.SchemaException
 import org.apache.kafka.common.utils.Time
 
 import scala.collection.{Seq, immutable, mutable}
+import scala.collection.JavaConverters._
 
 private[group] sealed trait GroupState
 
@@ -136,6 +140,7 @@ private object GroupMetadata {
         group.addStaticMember(member.groupInstanceId, member.memberId)
       }
     })
+    group.subscribedTopics = group.computeSubscribedTopics()
     group
   }
 
@@ -204,6 +209,10 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
   private var receivedTransactionalOffsetCommits = false
   private var receivedConsumerOffsetCommits = false
 
+  // When protocolType == `consumer`, a set of subscribed topics is maintained. The set is
+  // computed when a new generation is created or when the group is restored from the log.
+  private var subscribedTopics: Option[Set[String]] = None
+
   var newMemberAdded: Boolean = false
 
   def inLock[T](fun: => T): T = CoreUtils.inLock(lock)(fun)
@@ -219,6 +228,8 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
   def protocolOrNull: String = protocol.orNull
   def currentStateTimestampOrDefault: Long = currentStateTimestamp.getOrElse(-1)
 
+  def isConsumerGroup: Boolean = protocolType.contains(ConsumerProtocol.PROTOCOL_TYPE)
+
   def add(member: MemberMetadata, callback: JoinCallback = null): Unit = {
     if (members.isEmpty)
       this.protocolType = Some(member.protocolType)
@@ -423,6 +434,54 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
       protocolType.contains(memberProtocolType) && memberProtocols.exists(supportedProtocols(_) == members.size)
   }
 
+  def getSubscribedTopics: Option[Set[String]] = subscribedTopics
+
+  /**
+   * Returns true if the consumer group is actively subscribed to the topic. When the consumer
+   * group does not know, because the information is not available yet or because the it has
+   * failed to parse the Consumer Protocol, it returns true to be safe.
+   */
+  def isSubscribedToTopic(topic: String): Boolean = subscribedTopics match {
+    case Some(topics) => topics.contains(topic)
+    case None => true
+  }
+
+  /**
+   * Collects the set of topics that the members are subscribed to when the Protocol Type is equal
+   * to 'consumer'. None is returned if
+   * - the protocol type is not equal to 'consumer';
+   * - the protocol is not defined yet; or
+   * - the protocol metadata does not comply with the schema.
+   */
+  private[group] def computeSubscribedTopics(): Option[Set[String]] = {
+    protocolType match {
+      case Some(ConsumerProtocol.PROTOCOL_TYPE) if members.nonEmpty && protocol.isDefined =>
+        try {
+          Some(
+            members.map { case (_, member) =>
+              // The consumer protocol is parsed with V0 which is the based prefix of all versions.
+              // This way the consumer group manager does not depend on any specific existing or
+              // future versions of the consumer protocol. VO must prefix all new versions.
+              val buffer = ByteBuffer.wrap(member.metadata(protocol.get))
+              ConsumerProtocol.deserializeVersion(buffer)
+              ConsumerProtocol.deserializeSubscriptionV0(buffer).topics.asScala.toSet
+            }.reduceLeft(_ ++ _)
+          )
+        } catch {
+          case e: SchemaException => {
+            warn(s"Failed to parse Consumer Protocol ${ConsumerProtocol.PROTOCOL_TYPE}:${protocol.get} " +
+              s"of group $groupId. Consumer group coordinator is not aware of the subscribed topics.", e)
+            None
+          }
+        }
+
+      case Some(ConsumerProtocol.PROTOCOL_TYPE) if members.isEmpty =>
+        Option(Set.empty)
+
+      case _ => None
+    }
+  }
+
   def updateMember(member: MemberMetadata,
                    protocols: List[(String, Array[Byte])],
                    callback: JoinCallback) = {
@@ -465,10 +524,12 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
     if (members.nonEmpty) {
       generationId += 1
       protocol = Some(selectProtocol)
+      subscribedTopics = computeSubscribedTopics()
       transitionTo(CompletingRebalance)
     } else {
       generationId += 1
       protocol = None
+      subscribedTopics = computeSubscribedTopics()
       transitionTo(Empty)
     }
     receivedConsumerOffsetCommits = false
@@ -630,9 +691,11 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
 
   def removeExpiredOffsets(currentTimestamp: Long, offsetRetentionMs: Long): Map[TopicPartition, OffsetAndMetadata] = {
 
-    def getExpiredOffsets(baseTimestamp: CommitRecordMetadataAndOffset => Long): Map[TopicPartition, OffsetAndMetadata] = {
+    def getExpiredOffsets(baseTimestamp: CommitRecordMetadataAndOffset => Long,
+                          subscribedTopics: Set[String] = Set.empty): Map[TopicPartition, OffsetAndMetadata] = {
       offsets.filter {
         case (topicPartition, commitRecordMetadataAndOffset) =>
+          !subscribedTopics.contains(topicPartition.topic()) &&
           !pendingOffsetCommits.contains(topicPartition) && {
             commitRecordMetadataAndOffset.offsetAndMetadata.expireTimestamp match {
               case None =>
@@ -656,8 +719,20 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
         //   expire all offsets with no pending offset commit;
         // - if there is no current state timestamp (old group metadata schema) and retention period has passed
         //   since the last commit timestamp, expire the offset
-        getExpiredOffsets(commitRecordMetadataAndOffset =>
-          currentStateTimestamp.getOrElse(commitRecordMetadataAndOffset.offsetAndMetadata.commitTimestamp))
+        getExpiredOffsets(
+          commitRecordMetadataAndOffset => currentStateTimestamp
+            .getOrElse(commitRecordMetadataAndOffset.offsetAndMetadata.commitTimestamp)
+        )
+
+      case Some(ConsumerProtocol.PROTOCOL_TYPE) if subscribedTopics.isDefined =>
+        // consumers exist in the group =>
+        // - if the group is aware of the subscribed topics and retention period had passed since the
+        //   the last commit timestamp, expire the offset. offset with pending offset commit are not
+        //   expired
+        getExpiredOffsets(
+          _.offsetAndMetadata.commitTimestamp,
+          subscribedTopics.get
+        )
 
       case None =>
         // protocolType is None => standalone (simple) consumer, that uses Kafka for offset storage only
diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
index 330ccd0668463..ef4fea91cc0c9 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
@@ -36,6 +36,7 @@ import kafka.zk.KafkaZkClient
 import org.apache.kafka.clients.consumer.ConsumerRecord
 import org.apache.kafka.common.{KafkaException, TopicPartition}
 import org.apache.kafka.common.internals.Topic
+import org.apache.kafka.common.metrics.stats.Meter
 import org.apache.kafka.common.metrics.stats.{Avg, Max}
 import org.apache.kafka.common.metrics.{MetricConfig, Metrics}
 import org.apache.kafka.common.protocol.Errors
@@ -95,6 +96,26 @@ class GroupMetadataManager(brokerId: Int,
     "group-coordinator-metrics",
     "The avg time it took to load the partitions in the last 30sec"), new Avg())
 
+  val offsetCommitsSensor = metrics.sensor("OffsetCommits")
+
+  offsetCommitsSensor.add(new Meter(
+    metrics.metricName("offset-commit-rate",
+      "group-coordinator-metrics",
+      "The rate of committed offsets"),
+    metrics.metricName("offset-commit-count",
+      "group-coordinator-metrics",
+      "The total number of committed offsets")))
+
+  val offsetExpiredSensor = metrics.sensor("OffsetExpired")
+
+  offsetExpiredSensor.add(new Meter(
+    metrics.metricName("offset-expiration-rate",
+      "group-coordinator-metrics",
+      "The rate of expired offsets"),
+    metrics.metricName("offset-expiration-count",
+      "group-coordinator-metrics",
+      "The total number of expired offsets")))
+
   this.logIdent = s"[GroupMetadataManager brokerId=$brokerId] "
 
   private def recreateGauge[T](name: String, gauge: Gauge[T]): Gauge[T] = {
@@ -359,6 +380,9 @@ class GroupMetadataManager(brokerId: Int,
               throw new IllegalStateException("Append status %s should only have one partition %s"
                 .format(responseStatus, offsetTopicPartition))
 
+            // record the number of offsets committed to the log
+            offsetCommitsSensor.record(records.size)
+
             // construct the commit response status and insert
             // the offset and metadata to cache if the append status has no error
             val status = responseStatus(offsetTopicPartition)
@@ -742,6 +766,7 @@ class GroupMetadataManager(brokerId: Int,
     val numOffsetsRemoved = cleanupGroupMetadata(groupMetadataCache.values, group => {
       group.removeExpiredOffsets(currentTimestamp, config.offsetsRetentionMs)
     })
+    offsetCommitsSensor.record(numOffsetsRemoved)
     info(s"Removed $numOffsetsRemoved expired offsets in ${time.milliseconds() - currentTimestamp} milliseconds.")
   }
 
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index de0397cc783e2..49beab9d0d914 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -70,6 +70,7 @@ import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse
 import org.apache.kafka.common.message.ListGroupsResponseData
 import org.apache.kafka.common.message.OffsetCommitRequestData
 import org.apache.kafka.common.message.OffsetCommitResponseData
+import org.apache.kafka.common.message.OffsetDeleteResponseData
 import org.apache.kafka.common.message.RenewDelegationTokenResponseData
 import org.apache.kafka.common.message.SaslAuthenticateResponseData
 import org.apache.kafka.common.message.SaslHandshakeResponseData
@@ -186,6 +187,7 @@ class KafkaApis(val requestChannel: RequestChannel,
         case ApiKeys.INCREMENTAL_ALTER_CONFIGS => handleIncrementalAlterConfigsRequest(request)
         case ApiKeys.ALTER_PARTITION_REASSIGNMENTS => handleAlterPartitionReassignmentsRequest(request)
         case ApiKeys.LIST_PARTITION_REASSIGNMENTS => handleListPartitionReassignmentsRequest(request)
+        case ApiKeys.OFFSET_DELETE => handleOffsetDeleteRequest(request)
       }
     } catch {
       case e: FatalExitError => throw e
@@ -2660,6 +2662,56 @@ class KafkaApis(val requestChannel: RequestChannel,
     }
   }
 
+  def handleOffsetDeleteRequest(request: RequestChannel.Request): Unit = {
+    val offsetDeleteRequest = request.body[OffsetDeleteRequest]
+    val groupId = offsetDeleteRequest.data.groupId
+
+    if (authorize(request, DELETE, GROUP, groupId)) {
+      val topicPartitions = offsetDeleteRequest.data.topics.asScala.flatMap { topic =>
+        topic.partitions.asScala.map { partition =>
+          new TopicPartition(topic.name, partition.partitionIndex)
+        }
+      }.toSeq
+
+      val authorizedTopics = filterAuthorized(request, READ, TOPIC, topicPartitions.map(_.topic))
+      val (authorizedTopicPartitions, unauthorizedTopicPartitions) = topicPartitions.partition { topicPartition =>
+        authorizedTopics.contains(topicPartition.topic)
+      }
+
+      val unauthorizedTopicPartitionsErrors = unauthorizedTopicPartitions.map(_ -> Errors.TOPIC_AUTHORIZATION_FAILED)
+      val (groupError, authorizedTopicPartitionsErrors) = groupCoordinator.handleDeleteOffsets(groupId, authorizedTopicPartitions)
+      val topicPartitionsErrors = unauthorizedTopicPartitionsErrors ++ authorizedTopicPartitionsErrors
+
+      sendResponseMaybeThrottle(request, requestThrottleMs => {
+        if (groupError != Errors.NONE)
+          offsetDeleteRequest.getErrorResponse(requestThrottleMs, groupError)
+        else {
+          val topics = new OffsetDeleteResponseData.OffsetDeleteResponseTopicCollection
+          topicPartitionsErrors.groupBy(_._1.topic).map { case (topic, topicPartitions) =>
+            val partitions = new OffsetDeleteResponseData.OffsetDeleteResponsePartitionCollection
+            topicPartitions.map { case (topicPartition, error) =>
+              partitions.add(
+                new OffsetDeleteResponseData.OffsetDeleteResponsePartition()
+                  .setPartitionIndex(topicPartition.partition)
+                  .setErrorCode(error.code)
+              )
+              topics.add(new OffsetDeleteResponseData.OffsetDeleteResponseTopic()
+                .setName(topic)
+                .setPartitions(partitions))
+            }
+          }
+
+          new OffsetDeleteResponse(new OffsetDeleteResponseData()
+            .setTopics(topics)
+            .setThrottleTimeMs(requestThrottleMs))
+        }
+      })
+    } else {
+      sendResponseMaybeThrottle(request, requestThrottleMs =>
+        offsetDeleteRequest.getErrorResponse(requestThrottleMs, Errors.GROUP_AUTHORIZATION_FAILED))
+    }
+  }
+
   private def authorize(request: RequestChannel.Request,
                         operation: AclOperation,
                         resourceType: ResourceType,
diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index 27c435be3be2b..50ac94426af17 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -33,6 +33,7 @@ import kafka.utils.TestUtils._
 import kafka.utils.{Log4jController, Logging, TestUtils}
 import kafka.zk.KafkaZkClient
 import org.apache.kafka.clients.admin._
+import org.apache.kafka.clients.consumer.ConsumerRecords
 import org.apache.kafka.clients.consumer.{ConsumerConfig, KafkaConsumer}
 import org.apache.kafka.clients.producer.KafkaProducer
 import org.apache.kafka.clients.producer.ProducerRecord
@@ -1056,6 +1057,14 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     TestUtils.pollUntilTrue(consumer, () => !consumer.assignment.isEmpty, "Expected non-empty assignment")
   }
 
+  private def subscribeAndWaitForRecords(topic: String, consumer: KafkaConsumer[Array[Byte], Array[Byte]]): Unit = {
+    consumer.subscribe(Collections.singletonList(topic))
+    TestUtils.pollRecordsUntilTrue(
+      consumer,
+      (records: ConsumerRecords[Array[Byte], Array[Byte]]) => !records.isEmpty,
+      "Expected records" )
+  }
+
   private def sendRecords(producer: KafkaProducer[Array[Byte], Array[Byte]],
                           numRecords: Int,
                           topicPartition: TopicPartition): Unit = {
@@ -1264,7 +1273,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
             val part = new TopicPartition(testTopicName, 0)
             parts.containsKey(part) && (parts.get(part).offset() == 1)
           }, s"Expected the offset for partition 0 to eventually become 1.")
-
+          
           // Test delete non-exist consumer instance
           val invalidInstanceId = "invalid-instance-id"
           var removeMemberResult = client.removeMemberFromConsumerGroup(testGroupId, new RemoveMemberFromConsumerGroupOptions(
@@ -1351,6 +1360,77 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     }
   }
 
+  @Test
+  def testDeleteConsumerGroupOffsets(): Unit = {
+    val config = createConfig()
+    client = AdminClient.create(config)
+    try {
+      val testTopicName = "test_topic"
+      val testGroupId = "test_group_id"
+      val testClientId = "test_client_id"
+      val fakeGroupId = "fake_group_id"
+
+      val tp1 = new TopicPartition(testTopicName, 0)
+      val tp2 = new TopicPartition("foo", 0)
+
+      client.createTopics(Collections.singleton(
+        new NewTopic(testTopicName, 1, 1.toShort))).all().get()
+      waitForTopics(client, List(testTopicName), List())
+
+      val producer = createProducer()
+      try {
+        producer.send(new ProducerRecord(testTopicName, 0, null, null)).get()
+      } finally {
+        Utils.closeQuietly(producer, "producer")
+      }
+
+      val newConsumerConfig = new Properties(consumerConfig)
+      newConsumerConfig.setProperty(ConsumerConfig.GROUP_ID_CONFIG, testGroupId)
+      newConsumerConfig.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, testClientId)
+      // Increase timeouts to avoid having a rebalance during the test
+      newConsumerConfig.setProperty(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, Integer.MAX_VALUE.toString)
+      newConsumerConfig.setProperty(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, Defaults.GroupMaxSessionTimeoutMs.toString)
+      val consumer = createConsumer(configOverrides = newConsumerConfig)
+
+      try {
+        subscribeAndWaitForRecords(testTopicName, consumer)
+        consumer.commitSync()
+
+        // Test offset deletion while consuming
+        val offsetDeleteResult = client.deleteConsumerGroupOffsets(testGroupId, Set(tp1, tp2).asJava)
+
+        assertNull(offsetDeleteResult.all().get())
+        assertFutureExceptionTypeEquals(offsetDeleteResult.partitionResult(tp1),
+          classOf[GroupSubscribedToTopicException])
+        assertFutureExceptionTypeEquals(offsetDeleteResult.partitionResult(tp2),
+          classOf[UnknownTopicOrPartitionException])
+
+        // Test the fake group ID
+        val fakeDeleteResult = client.deleteConsumerGroupOffsets(fakeGroupId, Set(tp1, tp2).asJava)
+
+        assertFutureExceptionTypeEquals(fakeDeleteResult.all(), classOf[GroupIdNotFoundException])
+        assertFutureExceptionTypeEquals(fakeDeleteResult.partitionResult(tp1),
+          classOf[GroupIdNotFoundException])
+        assertFutureExceptionTypeEquals(fakeDeleteResult.partitionResult(tp2),
+          classOf[GroupIdNotFoundException])
+
+      } finally {
+        Utils.closeQuietly(consumer, "consumer")
+      }
+
+      // Test offset deletion when group is empty
+      val offsetDeleteResult = client.deleteConsumerGroupOffsets(testGroupId, Set(tp1, tp2).asJava)
+
+      assertNull(offsetDeleteResult.all().get())
+      assertNull(offsetDeleteResult.partitionResult(tp1).get())
+      assertFutureExceptionTypeEquals(offsetDeleteResult.partitionResult(tp2),
+        classOf[UnknownTopicOrPartitionException])
+
+    } finally {
+      Utils.closeQuietly(client, "adminClient")
+    }
+  }
+
   @Test
   def testElectPreferredLeaders(): Unit = {
     client = AdminClient.create(createConfig)
diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
index 9896f75c6873f..af007da962d23 100644
--- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
@@ -37,6 +37,7 @@ import org.apache.kafka.common.acl.AclPermissionType.{ALLOW, DENY}
 import org.apache.kafka.common.config.{ConfigResource, LogLevelConfig}
 import org.apache.kafka.common.errors._
 import org.apache.kafka.common.internals.Topic.GROUP_METADATA_TOPIC_NAME
+import org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData
 import org.apache.kafka.common.message.ControlledShutdownRequestData
 import org.apache.kafka.common.message.CreateTopicsRequestData
 import org.apache.kafka.common.message.CreateTopicsRequestData.{CreatableTopic, CreatableTopicCollection}
@@ -47,13 +48,12 @@ import org.apache.kafka.common.message.FindCoordinatorRequestData
 import org.apache.kafka.common.message.HeartbeatRequestData
 import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData
 import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData.{AlterConfigsResource, AlterableConfig, AlterableConfigCollection}
-import org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData
-import org.apache.kafka.common.message.ListPartitionReassignmentsRequestData
 import org.apache.kafka.common.message.JoinGroupRequestData
-import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection
-import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
+import org.apache.kafka.common.message.ListPartitionReassignmentsRequestData
 import org.apache.kafka.common.message.OffsetCommitRequestData
 import org.apache.kafka.common.message.SyncGroupRequestData
+import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection
+import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
 import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
 import org.apache.kafka.common.record.{CompressionType, MemoryRecords, RecordBatch, Records, SimpleRecord}
@@ -176,7 +176,8 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
       ApiKeys.ELECT_LEADERS -> classOf[ElectLeadersResponse],
       ApiKeys.INCREMENTAL_ALTER_CONFIGS -> classOf[IncrementalAlterConfigsResponse],
       ApiKeys.ALTER_PARTITION_REASSIGNMENTS -> classOf[AlterPartitionReassignmentsResponse],
-      ApiKeys.LIST_PARTITION_REASSIGNMENTS -> classOf[ListPartitionReassignmentsResponse]
+      ApiKeys.LIST_PARTITION_REASSIGNMENTS -> classOf[ListPartitionReassignmentsResponse],
+      ApiKeys.OFFSET_DELETE -> classOf[OffsetDeleteResponse]
     )
 
   val requestKeyToError = Map[ApiKeys, Nothing => Errors](
@@ -231,7 +232,15 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
         topicResourceError.error()
     }),
     ApiKeys.ALTER_PARTITION_REASSIGNMENTS -> ((resp: AlterPartitionReassignmentsResponse) => Errors.forCode(resp.data().errorCode())),
-    ApiKeys.LIST_PARTITION_REASSIGNMENTS -> ((resp: ListPartitionReassignmentsResponse) => Errors.forCode(resp.data().errorCode()))
+    ApiKeys.LIST_PARTITION_REASSIGNMENTS -> ((resp: ListPartitionReassignmentsResponse) => Errors.forCode(resp.data().errorCode())),
+    ApiKeys.OFFSET_DELETE -> ((resp: OffsetDeleteResponse) => {
+      Errors.forCode(
+        resp.data
+          .topics().asScala.find(_.name() == topic).get
+          .partitions().asScala.find(_.partitionIndex() == part).get
+          .errorCode()
+      )
+    })
   )
 
   val requestKeysToAcls = Map[ApiKeys, Map[ResourcePattern, Set[AccessControlEntry]]](
@@ -273,7 +282,8 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
     ApiKeys.ELECT_LEADERS -> clusterAlterAcl,
     ApiKeys.INCREMENTAL_ALTER_CONFIGS -> topicAlterConfigsAcl,
     ApiKeys.ALTER_PARTITION_REASSIGNMENTS -> clusterAlterAcl,
-    ApiKeys.LIST_PARTITION_REASSIGNMENTS -> clusterDescribeAcl
+    ApiKeys.LIST_PARTITION_REASSIGNMENTS -> clusterDescribeAcl,
+    ApiKeys.OFFSET_DELETE -> groupReadAcl
   )
 
   @Before
@@ -1322,6 +1332,56 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
     TestUtils.assertFutureExceptionTypeEquals(result.deletedGroups().get(group), classOf[GroupAuthorizationException])
   }
 
+  @Test
+  def testDeleteGroupOffsetsWithAcl(): Unit = {
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DELETE, ALLOW)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), topicResource)
+    val consumer = createConsumer()
+    consumer.assign(List(tp).asJava)
+    consumer.commitSync(Map(tp -> new OffsetAndMetadata(5, "")).asJava)
+    consumer.close()
+    val result = createAdminClient().deleteConsumerGroupOffsets(group, Set(tp).asJava)
+    assertNull(result.partitionResult(tp).get())
+  }
+
+  @Test
+  def testDeleteGroupOffsetsWithoutDeleteAcl(): Unit = {
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), topicResource)
+    val consumer = createConsumer()
+    consumer.assign(List(tp).asJava)
+    consumer.commitSync(Map(tp -> new OffsetAndMetadata(5, "")).asJava)
+    consumer.close()
+    val result = createAdminClient().deleteConsumerGroupOffsets(group, Set(tp).asJava)
+    TestUtils.assertFutureExceptionTypeEquals(result.all(), classOf[GroupAuthorizationException])
+  }
+
+  @Test
+  def testDeleteGroupOffsetsWithDeleteAclWithoutTopicAcl(): Unit = {
+    // Create the consumer group
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), topicResource)
+    val consumer = createConsumer()
+    consumer.assign(List(tp).asJava)
+    consumer.commitSync(Map(tp -> new OffsetAndMetadata(5, "")).asJava)
+    consumer.close()
+
+    // Remove the topic ACL & Check that it does not work without it
+    removeAllAcls()
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, DELETE, ALLOW)), groupResource)
+    addAndVerifyAcls(Set(new AccessControlEntry(userPrincipalStr, WildcardHost, READ, ALLOW)), groupResource)
+    val result = createAdminClient().deleteConsumerGroupOffsets(group, Set(tp).asJava)
+    assertNull(result.all().get())
+    TestUtils.assertFutureExceptionTypeEquals(result.partitionResult(tp), classOf[TopicAuthorizationException])
+  }
+
+  @Test
+  def testDeleteGroupOffsetsWithNoAcl(): Unit = {
+    val result = createAdminClient().deleteConsumerGroupOffsets(group, Set(tp).asJava)
+    TestUtils.assertFutureExceptionTypeEquals(result.all(), classOf[GroupAuthorizationException])
+  }
+
   @Test
   def testUnauthorizedDeleteTopicsWithoutDescribe(): Unit = {
     val response = connectAndSend(deleteTopicsRequest, ApiKeys.DELETE_TOPICS)
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
index 1b7b68f528d36..9cc0d3a18ab29 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
@@ -34,6 +34,8 @@ import java.util.concurrent.locks.ReentrantLock
 
 import kafka.cluster.Partition
 import kafka.zk.KafkaZkClient
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription
+import org.apache.kafka.clients.consumer.internals.ConsumerProtocol
 import org.apache.kafka.common.internals.Topic
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
@@ -41,7 +43,8 @@ import org.junit.Assert._
 import org.junit.{After, Assert, Before, Test}
 import org.scalatest.Assertions.intercept
 
-import scala.collection.mutable
+import scala.collection.JavaConverters._
+import scala.collection.{Seq, mutable}
 import scala.collection.mutable.ArrayBuffer
 import scala.concurrent.duration.Duration
 import scala.concurrent.{Await, Future, Promise, TimeoutException}
@@ -2818,6 +2821,216 @@ class GroupCoordinatorTest {
     assertEquals(Dead.toString, groupCoordinator.handleDescribeGroup(groupId)._2.state)
   }
 
+  @Test
+  def testDeleteOffsetOfNonExistingGroup(): Unit = {
+    val tp = new TopicPartition("foo", 0)
+    val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(tp))
+
+    assertEquals(Errors.GROUP_ID_NOT_FOUND, groupError)
+    assert(topics.isEmpty)
+  }
+
+  @Test
+  def testDeleteOffsetOfNonEmptyNonConsumerGroup(): Unit = {
+    val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
+    dynamicJoinGroup(groupId, memberId, "My Protocol", protocols)
+    val tp = new TopicPartition("foo", 0)
+    val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(tp))
+
+    assertEquals(Errors.NON_EMPTY_GROUP, groupError)
+    assert(topics.isEmpty)
+  }
+
+  @Test
+  def testDeleteOffsetOfEmptyNonConsumerGroup(): Unit = {
+    // join the group
+    val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
+
+    val joinGroupResult = dynamicJoinGroup(groupId, memberId, "My Protocol", protocols)
+    assertEquals(Errors.NONE, joinGroupResult.error)
+
+    EasyMock.reset(replicaManager)
+    val syncGroupResult = syncGroupLeader(groupId, joinGroupResult.generationId, joinGroupResult.leaderId, Map.empty)
+    assertEquals(Errors.NONE, syncGroupResult._2)
+
+    val t1p0 = new TopicPartition("foo", 0)
+    val t2p0 = new TopicPartition("bar", 0)
+    val offset = offsetAndMetadata(37)
+
+    EasyMock.reset(replicaManager)
+    val validOffsetCommitResult = commitOffsets(groupId, joinGroupResult.memberId, joinGroupResult.generationId,
+      Map(t1p0 -> offset, t2p0 -> offset))
+    assertEquals(Errors.NONE, validOffsetCommitResult(t1p0))
+    assertEquals(Errors.NONE, validOffsetCommitResult(t2p0))
+
+    // and leaves.
+    EasyMock.reset(replicaManager)
+    val leaveGroupResults = singleLeaveGroup(groupId, joinGroupResult.memberId)
+    verifyLeaveGroupResult(leaveGroupResults)
+
+    assert(groupCoordinator.groupManager.getGroup(groupId).exists(_.is(Empty)))
+
+    val groupTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupPartitionId)
+    val partition: Partition = EasyMock.niceMock(classOf[Partition])
+
+    EasyMock.reset(replicaManager)
+    EasyMock.expect(replicaManager.getMagic(EasyMock.anyObject())).andStubReturn(Some(RecordBatch.CURRENT_MAGIC_VALUE))
+    EasyMock.expect(replicaManager.getPartition(groupTopicPartition)).andStubReturn(HostedPartition.Online(partition))
+    EasyMock.expect(replicaManager.nonOfflinePartition(groupTopicPartition)).andStubReturn(Some(partition))
+    EasyMock.replay(replicaManager, partition)
+
+    val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(t1p0))
+
+    assertEquals(Errors.NONE, groupError)
+    assert(topics.size == 1)
+    assertEquals(Some(Errors.NONE), topics.get(t1p0))
+
+    val cachedOffsets = groupCoordinator.groupManager.getOffsets(groupId, Some(Seq(t1p0, t2p0)))
+
+    assertEquals(Some(OffsetFetchResponse.INVALID_OFFSET), cachedOffsets.get(t1p0).map(_.offset))
+    assertEquals(Some(offset.offset), cachedOffsets.get(t2p0).map(_.offset))
+  }
+
+  @Test
+  def testDeleteOffsetOfConsumerGroupWithUnparsableProtocol(): Unit = {
+    val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
+    val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols)
+
+    EasyMock.reset(replicaManager)
+    val syncGroupResult = syncGroupLeader(groupId, joinGroupResult.generationId, joinGroupResult.leaderId, Map.empty)
+    assertEquals(Errors.NONE, syncGroupResult._2)
+
+    val t1p0 = new TopicPartition("foo", 0)
+    val t2p0 = new TopicPartition("bar", 0)
+    val offset = offsetAndMetadata(37)
+
+    EasyMock.reset(replicaManager)
+    val validOffsetCommitResult = commitOffsets(groupId, joinGroupResult.memberId, joinGroupResult.generationId,
+      Map(t1p0 -> offset))
+    assertEquals(Errors.NONE, validOffsetCommitResult(t1p0))
+
+    val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(t1p0, t2p0))
+
+    assertEquals(Errors.NONE, groupError)
+    assert(topics.size == 2)
+    assertEquals(Some(Errors.GROUP_SUBSCRIBED_TO_TOPIC), topics.get(t1p0))
+    assertEquals(Some(Errors.UNKNOWN_TOPIC_OR_PARTITION), topics.get(t2p0))
+  }
+
+  @Test
+  def testDeleteOffsetOfDeadConsumerGroup(): Unit = {
+    val group = new GroupMetadata(groupId, Dead, new MockTime())
+    group.protocolType = Some(protocolType)
+    groupCoordinator.groupManager.addGroup(group)
+
+    val tp = new TopicPartition("foo", 0)
+    val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(tp))
+
+    assertEquals(Errors.GROUP_ID_NOT_FOUND, groupError)
+    assert(topics.size == 0)
+  }
+
+  @Test
+  def testDeleteOffsetOfEmptyConsumerGroup(): Unit = {
+    // join the group
+    val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
+    val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType, protocols)
+    assertEquals(Errors.NONE, joinGroupResult.error)
+
+    EasyMock.reset(replicaManager)
+    val syncGroupResult = syncGroupLeader(groupId, joinGroupResult.generationId, joinGroupResult.leaderId, Map.empty)
+    assertEquals(Errors.NONE, syncGroupResult._2)
+
+    val t1p0 = new TopicPartition("foo", 0)
+    val t2p0 = new TopicPartition("bar", 0)
+    val t3p0 = new TopicPartition("unknown", 0)
+    val offset = offsetAndMetadata(37)
+
+    EasyMock.reset(replicaManager)
+    val validOffsetCommitResult = commitOffsets(groupId, joinGroupResult.memberId, joinGroupResult.generationId,
+      Map(t1p0 -> offset, t2p0 -> offset))
+    assertEquals(Errors.NONE, validOffsetCommitResult(t1p0))
+    assertEquals(Errors.NONE, validOffsetCommitResult(t2p0))
+
+    // and leaves.
+    EasyMock.reset(replicaManager)
+    val leaveGroupResults = singleLeaveGroup(groupId, joinGroupResult.memberId)
+    verifyLeaveGroupResult(leaveGroupResults)
+
+    assert(groupCoordinator.groupManager.getGroup(groupId).exists(_.is(Empty)))
+
+    val groupTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupPartitionId)
+    val partition: Partition = EasyMock.niceMock(classOf[Partition])
+
+    EasyMock.reset(replicaManager)
+    EasyMock.expect(replicaManager.getMagic(EasyMock.anyObject())).andStubReturn(Some(RecordBatch.CURRENT_MAGIC_VALUE))
+    EasyMock.expect(replicaManager.getPartition(groupTopicPartition)).andStubReturn(HostedPartition.Online(partition))
+    EasyMock.expect(replicaManager.nonOfflinePartition(groupTopicPartition)).andStubReturn(Some(partition))
+    EasyMock.replay(replicaManager, partition)
+
+    val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(t1p0, t3p0))
+
+    assertEquals(Errors.NONE, groupError)
+    assert(topics.size == 2)
+    assertEquals(Some(Errors.NONE), topics.get(t1p0))
+    assertEquals(Some(Errors.UNKNOWN_TOPIC_OR_PARTITION), topics.get(t3p0))
+
+    val cachedOffsets = groupCoordinator.groupManager.getOffsets(groupId, Some(Seq(t1p0, t2p0)))
+
+    assertEquals(Some(OffsetFetchResponse.INVALID_OFFSET), cachedOffsets.get(t1p0).map(_.offset))
+    assertEquals(Some(offset.offset), cachedOffsets.get(t2p0).map(_.offset))
+  }
+
+  @Test
+  def testDeleteOffsetOfStableConsumerGroup(): Unit = {
+    // join the group
+    val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
+    val subscription = new Subscription(List("bar").asJava)
+
+    val joinGroupResult = dynamicJoinGroup(groupId, memberId, protocolType,
+      List(("protocol", ConsumerProtocol.serializeSubscription(subscription).array())))
+    assertEquals(Errors.NONE, joinGroupResult.error)
+
+    EasyMock.reset(replicaManager)
+    val syncGroupResult = syncGroupLeader(groupId, joinGroupResult.generationId, joinGroupResult.leaderId, Map.empty)
+    assertEquals(Errors.NONE, syncGroupResult._2)
+
+    val t1p0 = new TopicPartition("foo", 0)
+    val t2p0 = new TopicPartition("bar", 0)
+    val t3p0 = new TopicPartition("unknown", 0)
+    val offset = offsetAndMetadata(37)
+
+    EasyMock.reset(replicaManager)
+    val validOffsetCommitResult = commitOffsets(groupId, joinGroupResult.memberId, joinGroupResult.generationId,
+      Map(t1p0 -> offset, t2p0 -> offset))
+    assertEquals(Errors.NONE, validOffsetCommitResult(t1p0))
+    assertEquals(Errors.NONE, validOffsetCommitResult(t2p0))
+
+    assert(groupCoordinator.groupManager.getGroup(groupId).exists(_.is(Stable)))
+
+    val groupTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupPartitionId)
+    val partition: Partition = EasyMock.niceMock(classOf[Partition])
+
+    EasyMock.reset(replicaManager)
+    EasyMock.expect(replicaManager.getMagic(EasyMock.anyObject())).andStubReturn(Some(RecordBatch.CURRENT_MAGIC_VALUE))
+    EasyMock.expect(replicaManager.getPartition(groupTopicPartition)).andStubReturn(HostedPartition.Online(partition))
+    EasyMock.expect(replicaManager.nonOfflinePartition(groupTopicPartition)).andStubReturn(Some(partition))
+    EasyMock.replay(replicaManager, partition)
+
+    val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(t1p0, t2p0, t3p0))
+
+    assertEquals(Errors.NONE, groupError)
+    assert(topics.size == 3)
+    assertEquals(Some(Errors.NONE), topics.get(t1p0))
+    assertEquals(Some(Errors.GROUP_SUBSCRIBED_TO_TOPIC), topics.get(t2p0))
+    assertEquals(Some(Errors.UNKNOWN_TOPIC_OR_PARTITION), topics.get(t3p0))
+
+    val cachedOffsets = groupCoordinator.groupManager.getOffsets(groupId, Some(Seq(t1p0, t2p0)))
+
+    assertEquals(Some(OffsetFetchResponse.INVALID_OFFSET), cachedOffsets.get(t1p0).map(_.offset))
+    assertEquals(Some(offset.offset), cachedOffsets.get(t2p0).map(_.offset))
+  }
+
   @Test
   def shouldDelayInitialRebalanceByGroupInitialRebalanceDelayOnEmptyGroup(): Unit = {
     val firstJoinFuture = sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols)
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
index f961d8e18d53c..516180d5c733f 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
@@ -804,6 +804,74 @@ class GroupMetadataManagerTest {
     assertEquals(staticMemberId, group.getStaticMemberId(groupInstanceId))
   }
 
+  @Test
+  def testLoadConsumerGroup(): Unit = {
+    val generation = 27
+    val protocolType = "consumer"
+    val protocol = "protocol"
+    val memberId = "member1"
+    val topic = "foo"
+
+    val subscriptions = List(
+      ("protocol", ConsumerProtocol.serializeSubscription(new Subscription(List(topic).asJava)).array())
+    )
+
+    val member = new MemberMetadata(memberId, groupId, groupInstanceId, "", "", rebalanceTimeout,
+      sessionTimeout, protocolType, subscriptions)
+
+    val members = Seq(member)
+
+    val group = GroupMetadata.loadGroup(groupId, Stable, generation, protocolType, protocol, null, None,
+      members, time)
+
+    assertTrue(group.is(Stable))
+    assertEquals(generation, group.generationId)
+    assertEquals(Some(protocolType), group.protocolType)
+    assertEquals(protocol, group.protocolOrNull)
+    assertEquals(Some(Set(topic)), group.getSubscribedTopics)
+    assertTrue(group.has(memberId))
+  }
+
+  @Test
+  def testLoadEmptyConsumerGroup(): Unit = {
+    val generation = 27
+    val protocolType = "consumer"
+
+    val group = GroupMetadata.loadGroup(groupId, Empty, generation, protocolType, null, null, None,
+      Seq(), time)
+
+    assertTrue(group.is(Empty))
+    assertEquals(generation, group.generationId)
+    assertEquals(Some(protocolType), group.protocolType)
+    assertNull(group.protocolOrNull)
+    assertEquals(Some(Set.empty), group.getSubscribedTopics)
+  }
+
+  @Test
+  def testLoadConsumerGroupWithFaultyConsumerProtocol(): Unit = {
+    val generation = 27
+    val protocolType = "consumer"
+    val protocol = "protocol"
+    val memberId = "member1"
+
+    val subscriptions = List(("protocol", Array[Byte]()))
+
+    val member = new MemberMetadata(memberId, groupId, groupInstanceId, "", "", rebalanceTimeout,
+      sessionTimeout, protocolType, subscriptions)
+
+    val members = Seq(member)
+
+    val group = GroupMetadata.loadGroup(groupId, Stable, generation, protocolType, protocol, null, None,
+      members, time)
+
+    assertTrue(group.is(Stable))
+    assertEquals(generation, group.generationId)
+    assertEquals(Some(protocolType), group.protocolType)
+    assertEquals(protocol, group.protocolOrNull)
+    assertEquals(None, group.getSubscribedTopics)
+    assertTrue(group.has(memberId))
+  }
+
   @Test
   def testReadFromOldGroupMetadata(): Unit = {
     val generation = 1
@@ -1673,6 +1741,144 @@ class GroupMetadataManagerTest {
     assert(group.is(Dead))
   }
 
+  @Test
+  def testOffsetExpirationOfActiveGroupSemantics(): Unit = {
+    val memberId = "memberId"
+    val clientId = "clientId"
+    val clientHost = "localhost"
+
+    val topic1 = "foo"
+    val topic1Partition0 = new TopicPartition(topic1, 0)
+    val topic1Partition1 = new TopicPartition(topic1, 1)
+
+    val topic2 = "bar"
+    val topic2Partition0 = new TopicPartition(topic2, 0)
+    val topic2Partition1 = new TopicPartition(topic2, 1)
+
+    val offset = 37
+
+    groupMetadataManager.addPartitionOwnership(groupPartitionId)
+
+    val group = new GroupMetadata(groupId, Empty, time)
+    groupMetadataManager.addGroup(group)
+
+    // Subscribe to topic1 and topic2
+    val subscriptionTopic1AndTopic2 = new Subscription(List(topic1, topic2).asJava)
+
+    val member = new MemberMetadata(
+      memberId,
+      groupId,
+      groupInstanceId,
+      clientId,
+      clientHost,
+      rebalanceTimeout,
+      sessionTimeout,
+      ConsumerProtocol.PROTOCOL_TYPE,
+      List(("protocol", ConsumerProtocol.serializeSubscription(subscriptionTopic1AndTopic2).array()))
+    )
+
+    group.add(member, _ => ())
+    group.transitionTo(PreparingRebalance)
+    group.initNextGeneration()
+    group.transitionTo(Stable)
+
+    val startMs = time.milliseconds
+
+    val t1p0OffsetAndMetadata = OffsetAndMetadata(offset, "", startMs)
+    val t1p1OffsetAndMetadata = OffsetAndMetadata(offset, "", startMs)
+
+    val t2p0OffsetAndMetadata = OffsetAndMetadata(offset, "", startMs)
+    val t2p1OffsetAndMetadata = OffsetAndMetadata(offset, "", startMs)
+
+    val offsets = immutable.Map(
+      topic1Partition0 -> t1p0OffsetAndMetadata,
+      topic1Partition1 -> t1p1OffsetAndMetadata,
+      topic2Partition0 -> t2p0OffsetAndMetadata,
+      topic2Partition1 -> t2p1OffsetAndMetadata)
+
+    mockGetPartition()
+    expectAppendMessage(Errors.NONE)
+    EasyMock.replay(replicaManager)
+
+    var commitErrors: Option[immutable.Map[TopicPartition, Errors]] = None
+    def callback(errors: immutable.Map[TopicPartition, Errors]): Unit = {
+      commitErrors = Some(errors)
+    }
+
+    groupMetadataManager.storeOffsets(group, memberId, offsets, callback)
+    assertTrue(group.hasOffsets)
+
+    assertFalse(commitErrors.isEmpty)
+    assertEquals(Some(Errors.NONE), commitErrors.get.get(topic1Partition0))
+
+    // advance time to just after the offset of last partition is to be expired
+    time.sleep(defaultOffsetRetentionMs + 2)
+
+    // no offset should expire because all topics are actively consumed
+    groupMetadataManager.cleanupGroupMetadata()
+
+    assertEquals(Some(group), groupMetadataManager.getGroup(groupId))
+    assert(group.is(Stable))
+
+    assertEquals(Some(t1p0OffsetAndMetadata), group.offset(topic1Partition0))
+    assertEquals(Some(t1p1OffsetAndMetadata), group.offset(topic1Partition1))
+    assertEquals(Some(t2p0OffsetAndMetadata), group.offset(topic2Partition0))
+    assertEquals(Some(t2p1OffsetAndMetadata), group.offset(topic2Partition1))
+
+    var cachedOffsets = groupMetadataManager.getOffsets(groupId,
+      Some(Seq(topic1Partition0, topic1Partition1, topic2Partition0, topic2Partition1)))
+
+    assertEquals(Some(offset), cachedOffsets.get(topic1Partition0).map(_.offset))
+    assertEquals(Some(offset), cachedOffsets.get(topic1Partition1).map(_.offset))
+    assertEquals(Some(offset), cachedOffsets.get(topic2Partition0).map(_.offset))
+    assertEquals(Some(offset), cachedOffsets.get(topic2Partition1).map(_.offset))
+
+    EasyMock.verify(replicaManager)
+
+    group.transitionTo(PreparingRebalance)
+
+    // Subscribe to topic1, offsets of topic2 should be removed
+    val subscriptionTopic1 = new Subscription(List(topic1).asJava)
+
+    group.updateMember(
+      member,
+      List(("protocol", ConsumerProtocol.serializeSubscription(subscriptionTopic1).array())),
+      null
+    )
+
+    group.initNextGeneration()
+    group.transitionTo(Stable)
+
+    // expect the offset tombstone
+    EasyMock.expect(partition.appendRecordsToLeader(EasyMock.anyObject(classOf[MemoryRecords]),
+      isFromClient = EasyMock.eq(false), requiredAcks = EasyMock.anyInt()))
+      .andReturn(LogAppendInfo.UnknownLogAppendInfo)
+    EasyMock.expectLastCall().times(1)
+
+    EasyMock.replay(partition)
+
+    groupMetadataManager.cleanupGroupMetadata()
+
+    EasyMock.verify(partition)
+    EasyMock.verify(replicaManager)
+
+    assertEquals(Some(group), groupMetadataManager.getGroup(groupId))
+    assert(group.is(Stable))
+
+    assertEquals(Some(t1p0OffsetAndMetadata), group.offset(topic1Partition0))
+    assertEquals(Some(t1p1OffsetAndMetadata), group.offset(topic1Partition1))
+    assertEquals(None, group.offset(topic2Partition0))
+    assertEquals(None, group.offset(topic2Partition1))
+
+    cachedOffsets = groupMetadataManager.getOffsets(groupId,
+      Some(Seq(topic1Partition0, topic1Partition1, topic2Partition0, topic2Partition1)))
+
+    assertEquals(Some(offset), cachedOffsets.get(topic1Partition0).map(_.offset))
+    assertEquals(Some(offset), cachedOffsets.get(topic1Partition1).map(_.offset))
+    assertEquals(Some(OffsetFetchResponse.INVALID_OFFSET), cachedOffsets.get(topic2Partition0).map(_.offset))
+    assertEquals(Some(OffsetFetchResponse.INVALID_OFFSET), cachedOffsets.get(topic2Partition1).map(_.offset))
+  }
+
   @Test
   def testLoadOffsetFromOldCommit() = {
     val groupMetadataTopicPartition = groupTopicPartition
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala
index 6436e959bc98a..82f151c361739 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala
@@ -18,12 +18,16 @@
 package kafka.coordinator.group
 
 import kafka.common.OffsetAndMetadata
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription
+import org.apache.kafka.clients.consumer.internals.ConsumerProtocol
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.utils.Time
 import org.junit.Assert._
 import org.junit.{Before, Test}
 
+import scala.collection.JavaConverters._
+
 /**
  * Test group state transitions and other GroupMetadata functionality
  */
@@ -262,6 +266,57 @@ class GroupMetadataTest {
     assertFalse(group.supportsProtocols(protocolType, Set("range", "foo")))
   }
 
+  @Test
+  def testSubscribedTopics(): Unit = {
+    // not able to compute it for a newly created group
+    assertEquals(None, group.getSubscribedTopics)
+
+    val memberId = "memberId"
+    val member = new MemberMetadata(memberId, groupId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs,
+      sessionTimeoutMs, protocolType, List(("range", ConsumerProtocol.serializeSubscription(new Subscription(List("foo").asJava)).array())))
+
+    group.transitionTo(PreparingRebalance)
+    group.add(member)
+
+    group.initNextGeneration()
+
+    assertEquals(Some(Set("foo")), group.getSubscribedTopics)
+
+    group.transitionTo(PreparingRebalance)
+    group.remove(memberId)
+
+    group.initNextGeneration()
+
+    assertEquals(Some(Set.empty), group.getSubscribedTopics)
+
+    val memberWithFaultyProtocol  = new MemberMetadata(memberId, groupId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs,
+      sessionTimeoutMs, protocolType, List(("range", Array.empty[Byte])))
+
+    group.transitionTo(PreparingRebalance)
+    group.add(memberWithFaultyProtocol)
+
+    group.initNextGeneration()
+
+    assertEquals(None, group.getSubscribedTopics)
+  }
+
+  @Test
+  def testSubscribedTopicsNonConsumerGroup(): Unit = {
+    // not able to compute it for a newly created group
+    assertEquals(None, group.getSubscribedTopics)
+
+    val memberId = "memberId"
+    val member = new MemberMetadata(memberId, groupId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs,
+      sessionTimeoutMs, "My Protocol", List(("range", Array.empty[Byte])))
+
+    group.transitionTo(PreparingRebalance)
+    group.add(member)
+
+    group.initNextGeneration()
+
+    assertEquals(None, group.getSubscribedTopics)
+  }
+
   @Test
   def testInitNextGeneration(): Unit = {
     member.supportedProtocols = List(("roundrobin", Array.empty[Byte]))
diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
index 4a93ef9f67a0f..34c293e01973c 100644
--- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
+++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
@@ -473,6 +473,17 @@ class RequestQuotaTest extends BaseRequestTest {
             new ListPartitionReassignmentsRequestData()
           )
 
+        case ApiKeys.OFFSET_DELETE =>
+          new OffsetDeleteRequest.Builder(
+            new OffsetDeleteRequestData()
+              .setGroupId("test-group")
+              .setTopics(new OffsetDeleteRequestData.OffsetDeleteRequestTopicCollection(
+                Collections.singletonList(new OffsetDeleteRequestData.OffsetDeleteRequestTopic()
+                  .setName("test-topic")
+                  .setPartitions(Collections.singletonList(
+                    new OffsetDeleteRequestData.OffsetDeleteRequestPartition()
+                      .setPartitionIndex(0)))).iterator())))
+
         case _ =>
           throw new IllegalArgumentException("Unsupported API key " + apiKey)
     }
@@ -578,6 +589,7 @@ class RequestQuotaTest extends BaseRequestTest {
         new IncrementalAlterConfigsResponse(response, ApiKeys.INCREMENTAL_ALTER_CONFIGS.latestVersion()).throttleTimeMs
       case ApiKeys.ALTER_PARTITION_REASSIGNMENTS => new AlterPartitionReassignmentsResponse(response).throttleTimeMs
       case ApiKeys.LIST_PARTITION_REASSIGNMENTS => new ListPartitionReassignmentsResponse(response).throttleTimeMs
+      case ApiKeys.OFFSET_DELETE => new OffsetDeleteResponse(response).throttleTimeMs()
       case requestId => throw new IllegalArgumentException(s"No throttle time for $requestId")
     }
   }

From fbd06ec00c42a937ba06943431ae99ea2da31c4b Mon Sep 17 00:00:00 2001
From: Colin Patrick McCabe 
Date: Mon, 16 Sep 2019 13:13:06 -0700
Subject: [PATCH 0613/1071] MINOR: add unsigned varint support (#7338)

Support reading and writing unsigned varints.

Reviewers: Jason Gustafson 
---
 .../apache/kafka/common/utils/ByteUtils.java  | 106 ++++++++++++++----
 .../kafka/common/utils/ByteUtilsTest.java     |  40 +++++++
 2 files changed, 123 insertions(+), 23 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/utils/ByteUtils.java b/clients/src/main/java/org/apache/kafka/common/utils/ByteUtils.java
index 6efe3112cdb43..ae93075065710 100644
--- a/clients/src/main/java/org/apache/kafka/common/utils/ByteUtils.java
+++ b/clients/src/main/java/org/apache/kafka/common/utils/ByteUtils.java
@@ -129,7 +129,7 @@ public static void writeUnsignedIntLE(byte[] buffer, int offset, int value) {
     }
 
     /**
-     * Read an integer stored in variable-length format using zig-zag decoding from
+     * Read an integer stored in variable-length format using unsigned decoding from
      *  Google Protocol Buffers.
      *
      * @param buffer The buffer to read from
@@ -137,7 +137,7 @@ public static void writeUnsignedIntLE(byte[] buffer, int offset, int value) {
      *
      * @throws IllegalArgumentException if variable-length value does not terminate after 5 bytes have been read
      */
-    public static int readVarint(ByteBuffer buffer) {
+    public static int readUnsignedVarint(ByteBuffer buffer) {
         int value = 0;
         int i = 0;
         int b;
@@ -148,11 +148,11 @@ public static int readVarint(ByteBuffer buffer) {
                 throw illegalVarintException(value);
         }
         value |= b << i;
-        return (value >>> 1) ^ -(value & 1);
+        return value;
     }
 
     /**
-     * Read an integer stored in variable-length format using zig-zag decoding from
+     * Read an integer stored in variable-length format using unsigned decoding from
      *  Google Protocol Buffers.
      *
      * @param in The input to read from
@@ -161,7 +161,7 @@ public static int readVarint(ByteBuffer buffer) {
      * @throws IllegalArgumentException if variable-length value does not terminate after 5 bytes have been read
      * @throws IOException              if {@link DataInput} throws {@link IOException}
      */
-    public static int readVarint(DataInput in) throws IOException {
+    public static int readUnsignedVarint(DataInput in) throws IOException {
         int value = 0;
         int i = 0;
         int b;
@@ -172,6 +172,35 @@ public static int readVarint(DataInput in) throws IOException {
                 throw illegalVarintException(value);
         }
         value |= b << i;
+        return value;
+    }
+
+    /**
+     * Read an integer stored in variable-length format using zig-zag decoding from
+     *  Google Protocol Buffers.
+     *
+     * @param buffer The buffer to read from
+     * @return The integer read
+     *
+     * @throws IllegalArgumentException if variable-length value does not terminate after 5 bytes have been read
+     */
+    public static int readVarint(ByteBuffer buffer) {
+        int value = readUnsignedVarint(buffer);
+        return (value >>> 1) ^ -(value & 1);
+    }
+
+    /**
+     * Read an integer stored in variable-length format using zig-zag decoding from
+     *  Google Protocol Buffers.
+     *
+     * @param in The input to read from
+     * @return The integer read
+     *
+     * @throws IllegalArgumentException if variable-length value does not terminate after 5 bytes have been read
+     * @throws IOException              if {@link DataInput} throws {@link IOException}
+     */
+    public static int readVarint(DataInput in) throws IOException {
+        int value = readUnsignedVarint(in);
         return (value >>> 1) ^ -(value & 1);
     }
 
@@ -222,6 +251,40 @@ public static long readVarlong(ByteBuffer buffer)  {
         return (value >>> 1) ^ -(value & 1);
     }
 
+    /**
+     * Write the given integer following the variable-length unsigned encoding from
+     *  Google Protocol Buffers
+     * into the buffer.
+     *
+     * @param value The value to write
+     * @param buffer The output to write to
+     */
+    public static void writeUnsignedVarint(int value, ByteBuffer buffer) {
+        while ((value & 0xffffff80) != 0L) {
+            byte b = (byte) ((value & 0x7f) | 0x80);
+            buffer.put(b);
+            value >>>= 7;
+        }
+        buffer.put((byte) value);
+    }
+
+    /**
+     * Write the given integer following the variable-length unsigned encoding from
+     *  Google Protocol Buffers
+     * into the buffer.
+     *
+     * @param value The value to write
+     * @param out The output to write to
+     */
+    public static void writeUnsignedVarint(int value, DataOutput out) throws IOException {
+        while ((value & 0xffffff80) != 0L) {
+            byte b = (byte) ((value & 0x7f) | 0x80);
+            out.writeByte(b);
+            value >>>= 7;
+        }
+        out.writeByte((byte) value);
+    }
+
     /**
      * Write the given integer following the variable-length zig-zag encoding from
      *  Google Protocol Buffers
@@ -231,12 +294,7 @@ public static long readVarlong(ByteBuffer buffer)  {
      * @param out The output to write to
      */
     public static void writeVarint(int value, DataOutput out) throws IOException {
-        int v = (value << 1) ^ (value >> 31);
-        while ((v & 0xffffff80) != 0L) {
-            out.writeByte((v & 0x7f) | 0x80);
-            v >>>= 7;
-        }
-        out.writeByte((byte) v);
+        writeUnsignedVarint((value << 1) ^ (value >> 31), out);
     }
 
     /**
@@ -248,13 +306,7 @@ public static void writeVarint(int value, DataOutput out) throws IOException {
      * @param buffer The output to write to
      */
     public static void writeVarint(int value, ByteBuffer buffer) {
-        int v = (value << 1) ^ (value >> 31);
-        while ((v & 0xffffff80) != 0L) {
-            byte b = (byte) ((v & 0x7f) | 0x80);
-            buffer.put(b);
-            v >>>= 7;
-        }
-        buffer.put((byte) v);
+        writeUnsignedVarint((value << 1) ^ (value >> 31), buffer);
     }
 
     /**
@@ -293,20 +345,28 @@ public static void writeVarlong(long value, ByteBuffer buffer) {
     }
 
     /**
-     * Number of bytes needed to encode an integer in variable-length format.
+     * Number of bytes needed to encode an integer in unsigned variable-length format.
      *
      * @param value The signed value
      */
-    public static int sizeOfVarint(int value) {
-        int v = (value << 1) ^ (value >> 31);
+    public static int sizeOfUnsignedVarint(int value) {
         int bytes = 1;
-        while ((v & 0xffffff80) != 0L) {
+        while ((value & 0xffffff80) != 0L) {
             bytes += 1;
-            v >>>= 7;
+            value >>>= 7;
         }
         return bytes;
     }
 
+    /**
+     * Number of bytes needed to encode an integer in variable-length format.
+     *
+     * @param value The signed value
+     */
+    public static int sizeOfVarint(int value) {
+        return sizeOfUnsignedVarint((value << 1) ^ (value >> 31));
+    }
+
     /**
      * Number of bytes needed to encode a long in variable-length format.
      *
diff --git a/clients/src/test/java/org/apache/kafka/common/utils/ByteUtilsTest.java b/clients/src/test/java/org/apache/kafka/common/utils/ByteUtilsTest.java
index ce23a3378a857..f15e26d222b05 100644
--- a/clients/src/test/java/org/apache/kafka/common/utils/ByteUtilsTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/utils/ByteUtilsTest.java
@@ -33,11 +33,17 @@ public class ByteUtilsTest {
     private final byte x01 = 0x01;
     private final byte x02 = 0x02;
     private final byte x0F = 0x0f;
+    private final byte x07 = 0x07;
+    private final byte x08 = 0x08;
+    private final byte x3F = 0x3f;
+    private final byte x40 = 0x40;
     private final byte x7E = 0x7E;
     private final byte x7F = 0x7F;
     private final byte xFF = (byte) 0xff;
     private final byte x80 = (byte) 0x80;
     private final byte x81 = (byte) 0x81;
+    private final byte xBF = (byte) 0xbf;
+    private final byte xC0 = (byte) 0xc0;
     private final byte xFE = (byte) 0xfe;
 
     @Test
@@ -111,6 +117,24 @@ public void testWriteUnsignedIntLEToOutputStream() throws IOException {
         assertArrayEquals(new byte[] {(byte) 0xf1, (byte) 0xf2, (byte) 0xf3, (byte) 0xf4}, os2.toByteArray());
     }
 
+    @Test
+    public void testUnsignedVarintSerde() throws Exception {
+        assertUnsignedVarintSerde(0, new byte[] {x00});
+        assertUnsignedVarintSerde(-1, new byte[] {xFF, xFF, xFF, xFF, x0F});
+        assertUnsignedVarintSerde(1, new byte[] {x01});
+        assertUnsignedVarintSerde(63, new byte[] {x3F});
+        assertUnsignedVarintSerde(-64, new byte[] {xC0, xFF, xFF, xFF, x0F});
+        assertUnsignedVarintSerde(64, new byte[] {x40});
+        assertUnsignedVarintSerde(8191, new byte[] {xFF, x3F});
+        assertUnsignedVarintSerde(-8192, new byte[] {x80, xC0, xFF, xFF, x0F});
+        assertUnsignedVarintSerde(8192, new byte[] {x80, x40});
+        assertUnsignedVarintSerde(-8193, new byte[] {xFF, xBF, xFF, xFF, x0F});
+        assertUnsignedVarintSerde(1048575, new byte[] {xFF, xFF, x3F});
+        assertUnsignedVarintSerde(1048576, new byte[] {x80, x80, x40});
+        assertUnsignedVarintSerde(Integer.MAX_VALUE, new byte[] {xFF, xFF, xFF, xFF, x07});
+        assertUnsignedVarintSerde(Integer.MIN_VALUE, new byte[] {x80, x80, x80, x80, x08});
+    }
+
     @Test
     public void testVarintSerde() throws Exception {
         assertVarintSerde(0, new byte[] {x00});
@@ -197,6 +221,22 @@ public void testInvalidVarlong() {
         ByteUtils.readVarlong(buf);
     }
 
+    private void assertUnsignedVarintSerde(int value, byte[] expectedEncoding) throws IOException {
+        ByteBuffer buf = ByteBuffer.allocate(32);
+        ByteUtils.writeUnsignedVarint(value, buf);
+        buf.flip();
+        assertArrayEquals(expectedEncoding, Utils.toArray(buf));
+        assertEquals(value, ByteUtils.readUnsignedVarint(buf.duplicate()));
+
+        buf.rewind();
+        DataOutputStream out = new DataOutputStream(new ByteBufferOutputStream(buf));
+        ByteUtils.writeUnsignedVarint(value, out);
+        buf.flip();
+        assertArrayEquals(expectedEncoding, Utils.toArray(buf));
+        DataInputStream in = new DataInputStream(new ByteBufferInputStream(buf));
+        assertEquals(value, ByteUtils.readUnsignedVarint(in));
+    }
+
     private void assertVarintSerde(int value, byte[] expectedEncoding) throws IOException {
         ByteBuffer buf = ByteBuffer.allocate(32);
         ByteUtils.writeVarint(value, buf);

From c5dfb90b46d52112963066295dcd38e10cb61a9d Mon Sep 17 00:00:00 2001
From: David Jacot 
Date: Mon, 16 Sep 2019 22:26:20 +0200
Subject: [PATCH 0614/1071] MINOR: Cleanup scala warnings (#7335)

This patch removes a few warnings: mainly unused imports or vars.

Reviewers: Jason Gustafson 
---
 .../kafka/coordinator/group/GroupMetadataManager.scala   | 2 +-
 .../kafka/api/AdminClientIntegrationTest.scala           | 9 +++------
 .../test/scala/unit/kafka/admin/ConfigCommandTest.scala  | 2 --
 core/src/test/scala/unit/kafka/metrics/MetricsTest.scala | 2 --
 .../unit/kafka/server/ReplicationQuotaManagerTest.scala  | 2 +-
 5 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
index ef4fea91cc0c9..2cbf7c8af29bc 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
@@ -38,7 +38,7 @@ import org.apache.kafka.common.{KafkaException, TopicPartition}
 import org.apache.kafka.common.internals.Topic
 import org.apache.kafka.common.metrics.stats.Meter
 import org.apache.kafka.common.metrics.stats.{Avg, Max}
-import org.apache.kafka.common.metrics.{MetricConfig, Metrics}
+import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.protocol.types.Type._
 import org.apache.kafka.common.protocol.types._
diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index 50ac94426af17..23de704530a7e 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -44,11 +44,8 @@ import org.apache.kafka.common.TopicPartitionReplica
 import org.apache.kafka.common.acl._
 import org.apache.kafka.common.config.{ConfigResource, LogLevelConfig}
 import org.apache.kafka.common.errors._
-import org.apache.kafka.common.internals.KafkaFutureImpl
-import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
-import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse
 import org.apache.kafka.common.protocol.Errors
-import org.apache.kafka.common.requests.{DeleteRecordsRequest, JoinGroupRequest, MetadataResponse}
+import org.apache.kafka.common.requests.{DeleteRecordsRequest, MetadataResponse}
 import org.apache.kafka.common.resource.{PatternType, Resource, ResourcePattern, ResourceType}
 import org.apache.kafka.common.utils.{Time, Utils}
 import org.junit.Assert._
@@ -1291,7 +1288,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
             } catch {
               case e: ExecutionException =>
                 assertTrue(e.getCause.isInstanceOf[UnknownMemberIdException])
-              case _ =>
+              case _: Throwable =>
                 fail("Should have caught exception in getting member future")
             }
           }
@@ -1326,7 +1323,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
             } catch {
               case e: ExecutionException =>
                 assertTrue(e.getCause.isInstanceOf[UnknownMemberIdException])
-              case _ =>
+              case _: Throwable =>
                 fail("Should have caught exception in getting member future")
             }
           }
diff --git a/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala b/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala
index 3da7a35352c58..4fd110aab6a13 100644
--- a/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala
@@ -278,7 +278,6 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging {
 
   @Test(expected = classOf[IllegalArgumentException])
   def testEntityDefaultOptionWithDescribeBrokerLoggerIsNotAllowed(): Unit = {
-    val node = new Node(1, "localhost", 9092)
     val optsList = List("--bootstrap-server", "localhost:9092",
       "--entity-type", ConfigCommand.BrokerLoggerConfigType,
       "--entity-default",
@@ -290,7 +289,6 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging {
 
   @Test(expected = classOf[IllegalArgumentException])
   def testEntityDefaultOptionWithAlterBrokerLoggerIsNotAllowed(): Unit = {
-    val node = new Node(1, "localhost", 9092)
     val optsList = List("--bootstrap-server", "localhost:9092",
       "--entity-type", ConfigCommand.BrokerLoggerConfigType,
       "--entity-default",
diff --git a/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala b/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
index 875e15e805f3b..62040d5e262f9 100644
--- a/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
+++ b/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
@@ -30,10 +30,8 @@ import kafka.utils._
 
 import scala.collection._
 import scala.collection.JavaConverters._
-import scala.util.matching.Regex
 import kafka.log.LogConfig
 import org.apache.kafka.common.TopicPartition
-import org.scalatest.Assertions
 
 class MetricsTest extends KafkaServerTestHarness with Logging {
   val numNodes = 2
diff --git a/core/src/test/scala/unit/kafka/server/ReplicationQuotaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicationQuotaManagerTest.scala
index da9c5340cbd33..16c1ab7e2c708 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicationQuotaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicationQuotaManagerTest.scala
@@ -32,7 +32,7 @@ class ReplicationQuotaManagerTest {
   private val metrics = new Metrics(new MetricConfig(), Collections.emptyList(), time)
 
   @After
-  def tearDown: Unit = {
+  def tearDown(): Unit = {
     metrics.close()
   }
 

From bab3e082dc48bc3db68692694bd114a39b41fa68 Mon Sep 17 00:00:00 2001
From: Bruno Cadonna 
Date: Tue, 17 Sep 2019 04:48:25 +0000
Subject: [PATCH 0615/1071] KAFKA-8859: Expose built-in streams metrics version
 in `StreamsMetricsImpl` (#7323)

The streams config built.in.metrics.version is needed to add metrics in
a backward-compatible way. However, not in every location where metrics are
added a streams config is available to check built.in.metrics.version. Thus,
the config value needs to be exposed through the StreamsMetricsImpl object.

Reviewers: John Roesler , Guozhang Wang 
---
 .../internals/GlobalStreamThread.java         |  6 ++-
 .../processor/internals/StreamThread.java     |  6 ++-
 .../internals/metrics/StreamsMetricsImpl.java | 24 ++++++++++-
 .../internals/MockStreamsMetrics.java         |  3 +-
 .../processor/internals/StandbyTaskTest.java  |  3 +-
 .../processor/internals/StreamThreadTest.java | 26 +++++++-----
 .../metrics/StreamsMetricsImplTest.java       | 42 ++++++++++++-------
 .../GlobalStateStoreProviderTest.java         |  5 ++-
 .../MeteredTimestampedWindowStoreTest.java    |  3 +-
 .../internals/MeteredWindowStoreTest.java     |  3 +-
 .../test/InternalMockProcessorContext.java    | 26 +++++++++---
 .../kafka/streams/TopologyTestDriver.java     |  6 ++-
 .../processor/MockProcessorContext.java       |  2 +-
 13 files changed, 116 insertions(+), 39 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStreamThread.java
index 0b6539a3c7def..f0aaab64b56e5 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStreamThread.java
@@ -189,7 +189,11 @@ public GlobalStreamThread(final ProcessorTopology topology,
         this.topology = topology;
         this.globalConsumer = globalConsumer;
         this.stateDirectory = stateDirectory;
-        this.streamsMetrics = new StreamsMetricsImpl(metrics, threadClientId);
+        this.streamsMetrics = new StreamsMetricsImpl(
+            metrics,
+            threadClientId,
+            config.getString(StreamsConfig.BUILT_IN_METRICS_VERSION_CONFIG)
+        );
         this.logPrefix = String.format("global-stream-thread [%s] ", threadClientId);
         this.logContext = new LogContext(logPrefix);
         this.log = logContext.logger(getClass());
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index 1c88054d23247..00b376d3a10de 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -596,7 +596,11 @@ public static StreamThread create(final InternalTopologyBuilder builder,
             threadProducer = clientSupplier.getProducer(producerConfigs);
         }
 
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, threadClientId);
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(
+            metrics,
+            threadClientId,
+            config.getString(StreamsConfig.BUILT_IN_METRICS_VERSION_CONFIG)
+        );
 
         final ThreadCache cache = new ThreadCache(logContext, cacheSizeBytes, streamsMetrics);
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
index 34338e1502d98..1e7c1aee480bd 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
@@ -29,6 +29,7 @@
 import org.apache.kafka.common.metrics.stats.Value;
 import org.apache.kafka.common.metrics.stats.WindowedCount;
 import org.apache.kafka.common.metrics.stats.WindowedSum;
+import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.StreamsMetrics;
 
 import java.util.Arrays;
@@ -42,10 +43,17 @@
 import java.util.concurrent.TimeUnit;
 
 public class StreamsMetricsImpl implements StreamsMetrics {
+
+    public enum Version {
+        LATEST,
+        FROM_100_TO_23
+    }
+
     private final Metrics metrics;
     private final Map parentSensors;
     private final String threadName;
 
+    private final Version version;
     private final Deque threadLevelSensors = new LinkedList<>();
     private final Map> taskLevelSensors = new HashMap<>();
     private final Map> nodeLevelSensors = new HashMap<>();
@@ -83,14 +91,28 @@ public class StreamsMetricsImpl implements StreamsMetrics {
     public static final String EXPIRED_WINDOW_RECORD_DROP = "expired-window-record-drop";
     public static final String LATE_RECORD_DROP = "late-record-drop";
 
-    public StreamsMetricsImpl(final Metrics metrics, final String threadName) {
+    public StreamsMetricsImpl(final Metrics metrics, final String threadName, final String builtInMetricsVersion) {
         Objects.requireNonNull(metrics, "Metrics cannot be null");
+        Objects.requireNonNull(builtInMetricsVersion, "Built-in metrics version cannot be null");
         this.metrics = metrics;
         this.threadName = threadName;
+        this.version = parseBuiltInMetricsVersion(builtInMetricsVersion);
 
         this.parentSensors = new HashMap<>();
     }
 
+    private static Version parseBuiltInMetricsVersion(final String builtInMetricsVersion) {
+        if (builtInMetricsVersion.equals(StreamsConfig.METRICS_LATEST)) {
+            return Version.LATEST;
+        } else {
+            return Version.FROM_100_TO_23;
+        }
+    }
+
+    public Version version() {
+        return version;
+    }
+
     public final Sensor threadLevelSensor(final String sensorName,
                                           final RecordingLevel recordingLevel,
                                           final Sensor... parents) {
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/MockStreamsMetrics.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/MockStreamsMetrics.java
index bd3553083ff2c..273f4b9e45ada 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/MockStreamsMetrics.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/MockStreamsMetrics.java
@@ -17,11 +17,12 @@
 package org.apache.kafka.streams.processor.internals;
 
 import org.apache.kafka.common.metrics.Metrics;
+import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 
 public class MockStreamsMetrics extends StreamsMetricsImpl {
 
     public MockStreamsMetrics(final Metrics metrics) {
-        super(metrics, "test");
+        super(metrics, "test", StreamsConfig.METRICS_LATEST);
     }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
index df889798a86b6..033aeeefd7fdd 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
@@ -160,7 +160,8 @@ private StreamsConfig createConfig(final File baseDir) throws IOException {
     private final byte[] recordKey = intSerializer.serialize(null, 1);
 
     private final String threadName = "threadName";
-    private final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(new Metrics(), threadName);
+    private final StreamsMetricsImpl streamsMetrics =
+        new StreamsMetricsImpl(new Metrics(), threadName, StreamsConfig.METRICS_LATEST);
 
     @Before
     public void setup() throws Exception {
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
index 5f7d9dd56daae..d60c0069039c4 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
@@ -340,7 +340,7 @@ public void shouldNotCommitBeforeTheCommitInterval() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
         final TaskManager taskManager = mockTaskManagerCommit(consumer, 1, 1);
 
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId);
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -466,7 +466,7 @@ public void shouldNotCauseExceptionIfNothingCommitted() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
         final TaskManager taskManager = mockTaskManagerCommit(consumer, 1, 0);
 
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId);
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -501,7 +501,7 @@ public void shouldCommitAfterTheCommitInterval() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
         final TaskManager taskManager = mockTaskManagerCommit(consumer, 2, 1);
 
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId);
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -651,7 +651,8 @@ public void shouldShutdownTaskManagerOnClose() {
         EasyMock.expectLastCall();
         EasyMock.replay(taskManager, consumer);
 
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId);
+        final StreamsMetricsImpl streamsMetrics =
+            new StreamsMetricsImpl(metrics, clientId, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -684,7 +685,8 @@ public void shouldShutdownTaskManagerOnCloseWithoutStart() {
         EasyMock.expectLastCall();
         EasyMock.replay(taskManager, consumer);
 
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId);
+        final StreamsMetricsImpl streamsMetrics =
+            new StreamsMetricsImpl(metrics, clientId, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -756,7 +758,8 @@ private void setStreamThread(final StreamThread streamThread) {
         taskManager.setConsumer(mockStreamThreadConsumer);
         taskManager.setAssignmentMetadata(Collections.emptyMap(), Collections.emptyMap());
 
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId);
+        final StreamsMetricsImpl streamsMetrics =
+            new StreamsMetricsImpl(metrics, clientId, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -790,7 +793,8 @@ public void shouldOnlyShutdownOnce() {
         EasyMock.expectLastCall();
         EasyMock.replay(taskManager, consumer);
 
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId);
+        final StreamsMetricsImpl streamsMetrics =
+            new StreamsMetricsImpl(metrics, clientId, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -1184,7 +1188,8 @@ private void setupInternalTopologyWithoutState() {
     private StandbyTask createStandbyTask() {
         final LogContext logContext = new LogContext("test");
         final Logger log = logContext.logger(StreamThreadTest.class);
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId);
+        final StreamsMetricsImpl streamsMetrics =
+            new StreamsMetricsImpl(metrics, clientId, StreamsConfig.METRICS_LATEST);
         final StreamThread.StandbyTaskCreator standbyTaskCreator = new StreamThread.StandbyTaskCreator(
             internalTopologyBuilder,
             config,
@@ -1621,7 +1626,8 @@ public void producerMetricsVerificationWithoutEOS() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
         final TaskManager taskManager = mockTaskManagerCommit(consumer, 1, 0);
 
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId);
+        final StreamsMetricsImpl streamsMetrics =
+            new StreamsMetricsImpl(metrics, clientId, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
                 mockTime,
                 config,
@@ -1660,7 +1666,7 @@ public void adminClientMetricsVerification() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
         final TaskManager taskManager = EasyMock.createNiceMock(TaskManager.class);
 
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId);
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
                 mockTime,
                 config,
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java
index 4fd6f88a95df0..f1d951bab968a 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java
@@ -23,6 +23,8 @@
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.Sensor.RecordingLevel;
 import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.Version;
 import org.easymock.EasyMock;
 import org.easymock.EasyMockSupport;
 import org.junit.Test;
@@ -50,6 +52,8 @@ public class StreamsMetricsImplTest extends EasyMockSupport {
     private final static String SENSOR_PREFIX_DELIMITER = ".";
     private final static String SENSOR_NAME_DELIMITER = ".s.";
     private final static String INTERNAL_PREFIX = "internal";
+    private final static String THREAD_NAME = "test-thread";
+    private final static String VERSION = StreamsConfig.METRICS_LATEST;
 
     private final Metrics metrics = new Metrics();
     private final Sensor sensor = metrics.sensor("dummy");
@@ -59,21 +63,21 @@ public class StreamsMetricsImplTest extends EasyMockSupport {
     private final String description1 = "description number one";
     private final String description2 = "description number two";
     private final MockTime time = new MockTime(0);
+    private final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, THREAD_NAME, VERSION);
 
     @Test
     public void shouldGetThreadLevelSensor() {
         final Metrics metrics = mock(Metrics.class);
-        final String threadName = "thread1";
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, THREAD_NAME, VERSION);
         final String sensorName = "sensor1";
         final String expectedFullSensorName =
-            INTERNAL_PREFIX + SENSOR_PREFIX_DELIMITER + threadName + SENSOR_NAME_DELIMITER + sensorName;
+            INTERNAL_PREFIX + SENSOR_PREFIX_DELIMITER + THREAD_NAME + SENSOR_NAME_DELIMITER + sensorName;
         final RecordingLevel recordingLevel = RecordingLevel.DEBUG;
         final Sensor[] parents = {};
         EasyMock.expect(metrics.sensor(expectedFullSensorName, recordingLevel, parents)).andReturn(null);
 
         replayAll();
 
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, threadName);
         final Sensor sensor = streamsMetrics.threadLevelSensor(sensorName, recordingLevel);
 
         verifyAll();
@@ -83,12 +87,11 @@ public void shouldGetThreadLevelSensor() {
 
     @Test(expected = NullPointerException.class)
     public void testNullMetrics() {
-        new StreamsMetricsImpl(null, "");
+        new StreamsMetricsImpl(null, "", VERSION);
     }
 
     @Test(expected = NullPointerException.class)
     public void testRemoveNullSensor() {
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(new Metrics(), "");
         streamsMetrics.removeSensor(null);
     }
 
@@ -98,7 +101,6 @@ public void testRemoveSensor() {
         final String scope = "scope";
         final String entity = "entity";
         final String operation = "put";
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(new Metrics(), "");
 
         final Sensor sensor1 = streamsMetrics.addSensor(sensorName, Sensor.RecordingLevel.DEBUG);
         streamsMetrics.removeSensor(sensor1);
@@ -116,9 +118,9 @@ public void testRemoveSensor() {
     }
 
     @Test
-    public void testMutiLevelSensorRemoval() {
+    public void testMultiLevelSensorRemoval() {
         final Metrics registry = new Metrics();
-        final StreamsMetricsImpl metrics = new StreamsMetricsImpl(registry, "");
+        final StreamsMetricsImpl metrics = new StreamsMetricsImpl(registry, THREAD_NAME, VERSION);
         for (final MetricName defaultMetric : registry.metrics().keySet()) {
             registry.removeMetric(defaultMetric);
         }
@@ -169,7 +171,6 @@ public void testMutiLevelSensorRemoval() {
 
     @Test
     public void testLatencyMetrics() {
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(new Metrics(), "");
         final int defaultMetrics = streamsMetrics.metrics().size();
 
         final String scope = "scope";
@@ -189,7 +190,6 @@ public void testLatencyMetrics() {
 
     @Test
     public void testThroughputMetrics() {
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(new Metrics(), "");
         final int defaultMetrics = streamsMetrics.metrics().size();
 
         final String scope = "scope";
@@ -211,7 +211,7 @@ public void testTotalMetricDoesntDecrease() {
         final MockTime time = new MockTime(1);
         final MetricConfig config = new MetricConfig().timeWindow(1, TimeUnit.MILLISECONDS);
         final Metrics metrics = new Metrics(config, time);
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, "");
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, "", VERSION);
 
         final String scope = "scope";
         final String entity = "entity";
@@ -245,8 +245,6 @@ public void testTotalMetricDoesntDecrease() {
 
     @Test
     public void shouldGetStoreLevelTagMap() {
-        final String threadName = "test-thread";
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, threadName);
         final String taskName = "test-task";
         final String storeType = "remote-window";
         final String storeName = "window-keeper";
@@ -254,7 +252,7 @@ public void shouldGetStoreLevelTagMap() {
         final Map tagMap = streamsMetrics.storeLevelTagMap(taskName, storeType, storeName);
 
         assertThat(tagMap.size(), equalTo(3));
-        assertThat(tagMap.get(StreamsMetricsImpl.THREAD_ID_TAG), equalTo(threadName));
+        assertThat(tagMap.get(StreamsMetricsImpl.THREAD_ID_TAG), equalTo(THREAD_NAME));
         assertThat(tagMap.get(StreamsMetricsImpl.TASK_ID_TAG), equalTo(taskName));
         assertThat(tagMap.get(storeType + "-" + StreamsMetricsImpl.STORE_ID_TAG), equalTo(storeName));
     }
@@ -327,6 +325,22 @@ public void shouldAddAvgAndTotalMetricsToSensor() {
         assertThat(metrics.metrics().size(), equalTo(2 + 1)); // one metric is added automatically in the constructor of Metrics
     }
 
+    @Test
+    public void shouldReturnMetricsVersionCurrent() {
+        assertThat(
+            new StreamsMetricsImpl(metrics, THREAD_NAME, StreamsConfig.METRICS_LATEST).version(),
+            equalTo(Version.LATEST)
+        );
+    }
+
+    @Test
+    public void shouldReturnMetricsVersionFrom100To23() {
+        assertThat(
+            new StreamsMetricsImpl(metrics, THREAD_NAME, StreamsConfig.METRICS_0100_TO_23).version(),
+            equalTo(Version.FROM_100_TO_23)
+        );
+    }
+
     private void verifyMetric(final String name,
                               final String description,
                               final double valueToRecord1,
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/GlobalStateStoreProviderTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/GlobalStateStoreProviderTest.java
index 9c76e14d253b8..1ab8684e9052b 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/GlobalStateStoreProviderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/GlobalStateStoreProviderTest.java
@@ -18,6 +18,7 @@
 
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.errors.InvalidStateStoreException;
 import org.apache.kafka.streams.processor.StateStore;
 import org.apache.kafka.streams.processor.TaskId;
@@ -90,7 +91,9 @@ public void before() {
 
         final ProcessorContextImpl mockContext = mock(ProcessorContextImpl.class);
         expect(mockContext.applicationId()).andReturn("appId").anyTimes();
-        expect(mockContext.metrics()).andReturn(new StreamsMetricsImpl(new Metrics(), "threadName")).anyTimes();
+        expect(mockContext.metrics())
+            .andReturn(new StreamsMetricsImpl(new Metrics(), "threadName", StreamsConfig.METRICS_LATEST))
+            .anyTimes();
         expect(mockContext.taskId()).andReturn(new TaskId(0, 0)).anyTimes();
         expect(mockContext.recordCollector()).andReturn(null).anyTimes();
         replay(mockContext);
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedWindowStoreTest.java
index e3378faf885f0..53096d2bf13c7 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedWindowStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedWindowStoreTest.java
@@ -59,7 +59,8 @@ public class MeteredTimestampedWindowStoreTest {
 
     @Before
     public void setUp() {
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, "test");
+        final StreamsMetricsImpl streamsMetrics =
+            new StreamsMetricsImpl(metrics, "test", StreamsConfig.METRICS_LATEST);
 
         context = new InternalMockProcessorContext(
             TestUtils.tempDirectory(),
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java
index bd5bce1632f17..e62ab3a8a424a 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java
@@ -75,7 +75,8 @@ public class MeteredWindowStoreTest {
 
     @Before
     public void setUp() {
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, "test");
+        final StreamsMetricsImpl streamsMetrics =
+            new StreamsMetricsImpl(metrics, "test", StreamsConfig.METRICS_LATEST);
 
         context = new InternalMockProcessorContext(
             TestUtils.tempDirectory(),
diff --git a/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java b/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java
index 4b9267959ac8b..0935ae300f7ec 100644
--- a/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java
+++ b/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java
@@ -69,7 +69,7 @@ public InternalMockProcessorContext() {
         this(null,
             null,
             null,
-            new StreamsMetricsImpl(new Metrics(), "mock"),
+            new StreamsMetricsImpl(new Metrics(), "mock", StreamsConfig.METRICS_LATEST),
             new StreamsConfig(StreamsTestUtils.getStreamsConfig()),
             null,
             null
@@ -78,14 +78,30 @@ public InternalMockProcessorContext() {
 
     public InternalMockProcessorContext(final File stateDir,
                                         final StreamsConfig config) {
-        this(stateDir, null, null, new StreamsMetricsImpl(new Metrics(), "mock"), config, null, null);
+        this(
+            stateDir,
+            null,
+            null,
+            new StreamsMetricsImpl(new Metrics(), "mock", StreamsConfig.METRICS_LATEST),
+            config,
+            null,
+            null
+        );
     }
 
     public InternalMockProcessorContext(final File stateDir,
                                         final Serde keySerde,
                                         final Serde valSerde,
                                         final StreamsConfig config) {
-        this(stateDir, keySerde, valSerde, new StreamsMetricsImpl(new Metrics(), "mock"), config, null, null);
+        this(
+            stateDir,
+            keySerde,
+            valSerde,
+            new StreamsMetricsImpl(new Metrics(), "mock", StreamsConfig.METRICS_LATEST),
+            config,
+            null,
+            null
+        );
     }
 
     public InternalMockProcessorContext(final StateSerdes serdes,
@@ -100,7 +116,7 @@ public InternalMockProcessorContext(final StateSerdes serdes,
             null,
             serdes.keySerde(),
             serdes.valueSerde(),
-            new StreamsMetricsImpl(metrics, "mock"),
+            new StreamsMetricsImpl(metrics, "mock", StreamsConfig.METRICS_LATEST),
             new StreamsConfig(StreamsTestUtils.getStreamsConfig()),
             () -> collector,
             null
@@ -115,7 +131,7 @@ public InternalMockProcessorContext(final File stateDir,
         this(stateDir,
             keySerde,
             valSerde,
-            new StreamsMetricsImpl(new Metrics(), "mock"),
+            new StreamsMetricsImpl(new Metrics(), "mock", StreamsConfig.METRICS_LATEST),
             new StreamsConfig(StreamsTestUtils.getStreamsConfig()),
             () -> collector,
             cache
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
index d0ca6eae253d3..0a0351caededa 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
@@ -280,7 +280,11 @@ public List partitionsFor(final String topic) {
         metrics = new Metrics(metricConfig, mockWallClockTime);
 
         final String threadName = "topology-test-driver-virtual-thread";
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, threadName);
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(
+            metrics,
+            threadName,
+            StreamsConfig.METRICS_LATEST
+        );
         final Sensor skippedRecordsSensor = streamsMetrics.threadLevelSensor("skipped-records", Sensor.RecordingLevel.INFO);
         final String threadLevelGroup = "stream-metrics";
         skippedRecordsSensor.add(new MetricName("skipped-records-rate",
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java
index c478b6ba2bf08..041bf0d65d8b8 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java
@@ -214,7 +214,7 @@ public MockProcessorContext(final Properties config, final TaskId taskId, final
         final MetricConfig metricConfig = new MetricConfig();
         metricConfig.recordLevel(Sensor.RecordingLevel.DEBUG);
         final String threadName = "mock-processor-context-virtual-thread";
-        this.metrics = new StreamsMetricsImpl(new Metrics(metricConfig), threadName);
+        this.metrics = new StreamsMetricsImpl(new Metrics(metricConfig), threadName, StreamsConfig.METRICS_LATEST);
         ThreadMetrics.skipRecordSensor(metrics);
     }
 

From 935b280540913ade0b2dc1549ba22c4262ee2669 Mon Sep 17 00:00:00 2001
From: Stanislav Kozlovski 
Date: Tue, 17 Sep 2019 17:43:39 +0100
Subject: [PATCH 0616/1071] MINOR: Default to 5 partitions of the
 __consumer_offsets topic in Streams integration tests (#7331)

Given that the tests do not create clusters larger than 3, we do not gain much by creating 50 partitions for that topic. Reducing it should slightly increase test startup and shutdown speed.

Reviewers: Matthias J. Sax , Guozhang Wang , Bill Bejeck 
---
 .../kafka/streams/integration/utils/EmbeddedKafkaCluster.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/utils/EmbeddedKafkaCluster.java b/streams/src/test/java/org/apache/kafka/streams/integration/utils/EmbeddedKafkaCluster.java
index d4d2e1c021171..f30ecedc3aaa1 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/utils/EmbeddedKafkaCluster.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/utils/EmbeddedKafkaCluster.java
@@ -94,6 +94,7 @@ public void start() throws IOException, InterruptedException {
         putIfAbsent(brokerConfig, KafkaConfig$.MODULE$.GroupMinSessionTimeoutMsProp(), 0);
         putIfAbsent(brokerConfig, KafkaConfig$.MODULE$.GroupInitialRebalanceDelayMsProp(), 0);
         putIfAbsent(brokerConfig, KafkaConfig$.MODULE$.OffsetsTopicReplicationFactorProp(), (short) 1);
+        putIfAbsent(brokerConfig, KafkaConfig$.MODULE$.OffsetsTopicPartitionsProp(), 5);
         putIfAbsent(brokerConfig, KafkaConfig$.MODULE$.AutoCreateTopicsEnableProp(), true);
 
         for (int i = 0; i < brokers.length; i++) {

From 4962c8193e2faa589914be52962c701aba0980d1 Mon Sep 17 00:00:00 2001
From: vinoth chandar 
Date: Tue, 17 Sep 2019 09:46:40 -0700
Subject: [PATCH 0617/1071] KAFKA-8839 : Improve streams debug logging (#7258)

* log lock acquistion failures on the state store
* Document required uniqueness of state.dir path
* Move bunch of log calls around task state changes to DEBUG
* More readable log messages during partition assignment

Reviewers: Matthias J. Sax , A. Sophie Blee-Goldman , Guozhang Wang 
---
 docs/streams/developer-guide/config-streams.html            | 4 +++-
 .../main/java/org/apache/kafka/streams/StreamsConfig.java   | 2 +-
 .../streams/processor/internals/AssignedStreamsTasks.java   | 2 +-
 .../kafka/streams/processor/internals/AssignedTasks.java    | 5 +++--
 .../streams/processor/internals/InternalTopicManager.java   | 6 +++---
 .../kafka/streams/processor/internals/StreamTask.java       | 3 +--
 .../kafka/streams/processor/internals/StreamThread.java     | 4 ++--
 .../processor/internals/StreamsPartitionAssignor.java       | 5 ++++-
 .../kafka/streams/processor/internals/TaskManager.java      | 2 +-
 9 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/docs/streams/developer-guide/config-streams.html b/docs/streams/developer-guide/config-streams.html
index 21f488b549196..0c6f4b2fdfe27 100644
--- a/docs/streams/developer-guide/config-streams.html
+++ b/docs/streams/developer-guide/config-streams.html
@@ -474,7 +474,9 @@ 
                state.dir
             
                The state directory. Kafka Streams persists local states under the state directory. Each application has a subdirectory on its hosting
               machine that is located under the state directory. The name of the subdirectory is the application ID. The state stores associated
-              with the application are created under this subdirectory.
                
+              with the application are created under this subdirectory. When running multiple instances of the same application on a single machine,
+              this path must be unique for each such instance.
+          
         
         
                
           
                timestamp.extractor
                
diff --git a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
index bbce717324ee6..f9ce18b34bce9 100644
--- a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
+++ b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
@@ -465,7 +465,7 @@ public class StreamsConfig extends AbstractConfig {
     /** {@code state.dir} */
     @SuppressWarnings("WeakerAccess")
     public static final String STATE_DIR_CONFIG = "state.dir";
-    private static final String STATE_DIR_DOC = "Directory location for state store.";
+    private static final String STATE_DIR_DOC = "Directory location for state store. This path must be unique for each streams instance sharing the same underlying filesystem.";
 
     /** {@code topology.optimization} */
     public static final String TOPOLOGY_OPTIMIZATION = "topology.optimization";
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasks.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasks.java
index f0019ec96722c..0dd40ce92b40e 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasks.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasks.java
@@ -103,7 +103,7 @@ void updateRestored(final Collection restored) {
             if (restoredPartitions.containsAll(task.changelogPartitions())) {
                 transitionToRunning(task);
                 it.remove();
-                log.trace("Stream task {} completed restoration as all its changelog partitions {} have been applied to restore state",
+                log.debug("Stream task {} completed restoration as all its changelog partitions {} have been applied to restore state",
                     task.id(),
                     task.changelogPartitions());
             } else {
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
index 48933a41e1d59..8053bc792cfb5 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
@@ -77,8 +77,9 @@ void initializeNewTasks() {
                 }
                 it.remove();
             } catch (final LockException e) {
-                // made this trace as it will spam the logs in the poll loop.
-                log.trace("Could not create {} {} due to {}; will retry", taskTypeName, entry.getKey(), e.toString());
+                // If this is a permanent error, then we could spam the log since this is in the run loop. But, other related
+                // messages show up anyway. So keeping in debug for sake of faster discoverability of problem
+                log.debug("Could not create {} {} due to {}; will retry", taskTypeName, entry.getKey(), e.toString());
             }
         }
     }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java
index 621ce7e9094d5..ab214931ede39 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java
@@ -73,9 +73,9 @@ public InternalTopicManager(final Admin adminClient,
         retries = dummyAdmin.getInt(AdminClientConfig.RETRIES_CONFIG);
         retryBackOffMs = dummyAdmin.getLong(AdminClientConfig.RETRY_BACKOFF_MS_CONFIG);
 
-        log.debug("Configs:" + Utils.NL,
-            "\t{} = {}" + Utils.NL,
-            "\t{} = {}" + Utils.NL,
+        log.debug("Configs:" + Utils.NL +
+            "\t{} = {}" + Utils.NL +
+            "\t{} = {}" + Utils.NL +
             "\t{} = {}",
             AdminClientConfig.RETRIES_CONFIG, retries,
             StreamsConfig.REPLICATION_FACTOR_CONFIG, replicationFactor,
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
index 3dd83b5b00cd4..b822cb3eca53e 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
@@ -236,7 +236,7 @@ public StreamTask(final TaskId id,
 
     @Override
     public boolean initializeStateStores() {
-        log.trace("Initializing state stores");
+        log.debug("Initializing state stores");
 
         // Currently there is no easy way to tell the ProcessorStateManager to only restore up to
         // a specific offset. In most cases this is fine. However, in optimized topologies we can
@@ -252,7 +252,6 @@ public boolean initializeStateStores() {
                 stateMgr.putOffsetLimit(tp, offset);
                 log.trace("Updating store offset limits {} for changelog {}", offset, tp);
             });
-
         registerStateStores();
 
         return changelogPartitions().isEmpty();
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index 00b376d3a10de..d17768b0886d7 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -262,7 +262,7 @@ static class RebalanceListener implements ConsumerRebalanceListener {
 
         @Override
         public void onPartitionsAssigned(final Collection assignment) {
-            log.debug("at state {}: partitions {} assigned at the end of consumer rebalance.\n" +
+            log.debug("Current state {}: assigned partitions {} at the end of consumer rebalance.\n" +
                     "\tcurrent suspended active tasks: {}\n" +
                     "\tcurrent suspended standby tasks: {}\n",
                 streamThread.state,
@@ -310,7 +310,7 @@ public void onPartitionsAssigned(final Collection assignment) {
 
         @Override
         public void onPartitionsRevoked(final Collection assignment) {
-            log.debug("at state {}: partitions {} revoked at the beginning of consumer rebalance.\n" +
+            log.debug("Current state {}: revoked partitions {} at the beginning of consumer rebalance.\n" +
                     "\tcurrent assigned active tasks: {}\n" +
                     "\tcurrent assigned standby tasks: {}\n",
                 streamThread.state,
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
index 2d46778124af0..869c42a882fb3 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
@@ -26,6 +26,7 @@
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.config.ConfigException;
 import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.streams.errors.StreamsException;
 import org.apache.kafka.streams.errors.TaskAssignmentException;
 import org.apache.kafka.streams.processor.PartitionGrouper;
@@ -52,6 +53,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.stream.Collectors;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -525,7 +527,8 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
         final StickyTaskAssignor taskAssignor = new StickyTaskAssignor<>(states, partitionsForTask.keySet());
         taskAssignor.assign(numStandbyReplicas);
 
-        log.info("Assigned tasks to clients as {}.", states);
+        log.info("Assigned tasks to clients as {}{}.", Utils.NL, states.entrySet().stream()
+            .map(Map.Entry::toString).collect(Collectors.joining(Utils.NL)));
 
         // ---------------- Step Three ---------------- //
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
index cc448a153b877..00509fcbd662f 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
@@ -147,7 +147,7 @@ private void addStreamTasks(final Collection assignment) {
         // CANNOT FIND RETRY AND BACKOFF LOGIC
         // create all newly assigned tasks (guard against race condition with other thread via backoff and retry)
         // -> other thread will call removeSuspendedTasks(); eventually
-        log.trace("New active tasks to be created: {}", newTasks);
+        log.debug("New active tasks to be created: {}", newTasks);
 
         for (final StreamTask task : taskCreator.createTasks(consumer, newTasks)) {
             active.addNewTask(task);

From dcfd31c5520008dfc1518ac142192df57eaabcd5 Mon Sep 17 00:00:00 2001
From: Stanislav Kozlovski 
Date: Tue, 17 Sep 2019 22:28:22 +0100
Subject: [PATCH 0618/1071] MINOR: Remove duplicate definition of
 transactional.id.expiration.ms config (#7245)

Reviewers: Bob Barrett , Apurva Mehta , Ismael Juma 
---
 core/src/main/scala/kafka/log/LogManager.scala     |  2 +-
 core/src/main/scala/kafka/server/KafkaConfig.scala | 12 +++---------
 2 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/core/src/main/scala/kafka/log/LogManager.scala b/core/src/main/scala/kafka/log/LogManager.scala
index f01a74b8c14d6..c90ea7aabe4fc 100755
--- a/core/src/main/scala/kafka/log/LogManager.scala
+++ b/core/src/main/scala/kafka/log/LogManager.scala
@@ -1036,7 +1036,7 @@ object LogManager {
       flushRecoveryOffsetCheckpointMs = config.logFlushOffsetCheckpointIntervalMs,
       flushStartOffsetCheckpointMs = config.logFlushStartOffsetCheckpointIntervalMs,
       retentionCheckMs = config.logCleanupIntervalMs,
-      maxPidExpirationMs = config.transactionIdExpirationMs,
+      maxPidExpirationMs = config.transactionalIdExpirationMs,
       scheduler = kafkaScheduler,
       brokerState = brokerState,
       brokerTopicStats = brokerTopicStats,
diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index 6d106c7b7594f..17ebd278f19a3 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -731,7 +731,9 @@ object KafkaConfig {
   "or this timeout is reached. This is similar to the producer request timeout."
   val OffsetCommitRequiredAcksDoc = "The required acks before the commit can be accepted. In general, the default (-1) should not be overridden"
   /** ********* Transaction management configuration ***********/
-  val TransactionalIdExpirationMsDoc = "The maximum amount of time in ms that the transaction coordinator will wait before proactively expire a producer's transactional id without receiving any transaction status updates from it."
+  val TransactionalIdExpirationMsDoc = "The time in ms that the transaction coordinator will wait without receiving any transaction status updates " +
+    "for the current transaction before expiring its transactional id. This setting also influences producer id expiration - producer ids are expired " + 
+    "once this time has elapsed after the last write with the given producer id. Note that producer ids may expire sooner if the last write from the producer id is deleted due to the topic's retention settings."
   val TransactionsMaxTimeoutMsDoc = "The maximum allowed timeout for transactions. " +
     "If a client’s requested transaction time exceed this, then the broker will return an error in InitProducerIdRequest. This prevents a client from too large of a timeout, which can stall consumers reading from topics included in the transaction."
   val TransactionsTopicMinISRDoc = "Overridden " + MinInSyncReplicasProp + " config for the transaction topic."
@@ -761,11 +763,6 @@ object KafkaConfig {
     "which is used to determine quota limits applied to client requests. By default, ,  or  " +
     "quotas stored in ZooKeeper are applied. For any given request, the most specific quota that matches the user principal " +
     "of the session and the client-id of the request is applied."
-  /** ********* Transaction Configuration ***********/
-  val TransactionIdExpirationMsDoc = "The maximum time of inactivity before a transactional id is expired by the " +
-    "transaction coordinator. Note that this also influences producer id expiration: Producer ids are guaranteed to expire " +
-    "after expiration of this timeout from the last write by the producer id (they may expire sooner if the last write " +
-    "from the producer id is deleted due to the topic's retention settings)."
 
   val DeleteTopicEnableDoc = "Enables delete topic. Delete topic through the admin tool will have no effect if this config is turned off"
   val CompressionTypeDoc = "Specify the final compression type for a given topic. This configuration accepts the standard compression codecs " +
@@ -1361,9 +1358,6 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO
   val numAlterLogDirsReplicationQuotaSamples = getInt(KafkaConfig.NumAlterLogDirsReplicationQuotaSamplesProp)
   val alterLogDirsReplicationQuotaWindowSizeSeconds = getInt(KafkaConfig.AlterLogDirsReplicationQuotaWindowSizeSecondsProp)
 
-  /** ********* Transaction Configuration **************/
-  val transactionIdExpirationMs = getInt(KafkaConfig.TransactionalIdExpirationMsProp)
-
   /** ********* Fetch Session Configuration **************/
   val maxIncrementalFetchSessionCacheSlots = getInt(KafkaConfig.MaxIncrementalFetchSessionCacheSlots)
 

From 9ba898edc7c834259c3676402337f74b62aaba99 Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Tue, 17 Sep 2019 14:29:46 -0700
Subject: [PATCH 0619/1071] remove unused import (#7345)

Remove unused import that's slipping past checkstyle somehow

Reviewers: Matthias J. Sax , Christopher Pettitt 
---
 .../kafka/streams/processor/internals/StoreChangelogReader.java  | 1 -
 1 file changed, 1 deletion(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java
index adb886969aecc..3b45c5efc55db 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
-import java.util.Map.Entry;
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.clients.consumer.ConsumerRecords;

From 15d69b78561a7912e2257c1dcc3909c35563b4ff Mon Sep 17 00:00:00 2001
From: vinoth chandar 
Date: Tue, 17 Sep 2019 17:36:59 -0700
Subject: [PATCH 0620/1071] KAFKA-8913: Document topic based configs & ISR
 settings for Streams apps (#7346)

Reviewer: Matthias J. Sax 
---
 docs/streams/developer-guide/config-streams.html | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/docs/streams/developer-guide/config-streams.html b/docs/streams/developer-guide/config-streams.html
index 0c6f4b2fdfe27..221e3c0f89cb6 100644
--- a/docs/streams/developer-guide/config-streams.html
+++ b/docs/streams/developer-guide/config-streams.html
@@ -617,6 +617,14 @@ 
                Namingmain.consumer. and main.consumer., if you only want to specify one consumer type config.
                
+          
                 Additionally, to configure the internal repartition/changelog topics, you could use the topic. prefix, followed by any of the standard topic configs.
                
+            
                Properties streamsSettings = new Properties();
+// Override default for both changelog and repartition topics
+streamsSettings.put("topic.PARAMETER_NAME", "topic-value");
+// alternatively, you can use
+streamsSettings.put(StreamsConfig.topicPrefix("PARAMETER_NAME"), "topic-value");
+
                
+            
                
           
                
         
         
           
                Properties streamsSettings = new Properties();
 streamsSettings.put(StreamsConfig.REPLICATION_FACTOR_CONFIG, 3);
+streamsSettings.put(StreamsConfig.topicPrefix(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG), 2);
 streamsSettings.put(StreamsConfig.producerPrefix(ProducerConfig.ACKS_CONFIG), "all");
 
                
           
                

From f3ded39a0556f9e43a66b238c89c6a60f3ef33df Mon Sep 17 00:00:00 2001
From: Lucas Bradstreet 
Date: Wed, 18 Sep 2019 09:11:39 -0700
Subject: [PATCH 0621/1071] KAFKA-8841; Reduce overhead of
 ReplicaManager.updateFollowerFetchState (#7324)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR makes two changes to code in the ReplicaManager.updateFollowerFetchState path, which is in the hot path for follower fetches. Although calling ReplicaManager.updateFollowerFetch state is inexpensive on its own, it is called once for each partition every time a follower fetch occurs.

1. updateFollowerFetchState no longer calls maybeExpandIsr when the follower is already in the ISR. This avoid repeated expansion checks.
2. Partition.maybeIncrementLeaderHW is also in the hot path for ReplicaManager.updateFollowerFetchState. Partition.maybeIncrementLeaderHW calls Partition.remoteReplicas four times each iteration, and it performs a toSet conversion. maybeIncrementLeaderHW now avoids generating any intermediate collections when updating the HWM.

**Benchmark results for Partition.updateFollowerFetchState on a r5.xlarge:**
Old:
```
  1288.633 ±(99.9%) 1.170 ns/op [Average]
  (min, avg, max) = (1287.343, 1288.633, 1290.398), stdev = 1.037
  CI (99.9%): [1287.463, 1289.802] (assumes normal distribution)
```

New (when follower fetch offset is updated):
```
  261.727 ±(99.9%) 0.122 ns/op [Average]
  (min, avg, max) = (261.565, 261.727, 261.937), stdev = 0.114
  CI (99.9%): [261.605, 261.848] (assumes normal distribution)
```

New (when follower fetch offset is the same):
```
  68.484 ±(99.9%) 0.025 ns/op [Average]
  (min, avg, max) = (68.446, 68.484, 68.520), stdev = 0.023
  CI (99.9%): [68.460, 68.509] (assumes normal distribution)
```

Reviewers: Ismael Juma , Jason Gustafson 
---
 build.gradle                                  |   5 +
 checkstyle/import-control-jmh-benchmarks.xml  |  45 +++++
 .../main/scala/kafka/cluster/Partition.scala  | 112 ++++++-----
 core/src/main/scala/kafka/log/Log.scala       |  22 ++-
 .../scala/kafka/server/ReplicaManager.scala   |   4 +-
 gradle/spotbugs-exclude.xml                   |   1 +
 .../UpdateFollowerFetchStateBenchmark.java    | 175 ++++++++++++++++++
 7 files changed, 306 insertions(+), 58 deletions(-)
 create mode 100644 checkstyle/import-control-jmh-benchmarks.xml
 create mode 100644 jmh-benchmarks/src/main/java/org/apache/kafka/jmh/partition/UpdateFollowerFetchStateBenchmark.java

diff --git a/build.gradle b/build.gradle
index 083cbdfc8485f..dc7ce681d4d69 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1418,9 +1418,11 @@ project(':jmh-benchmarks') {
   }
 
   dependencies {
+    compile project(':core')
     compile project(':clients')
     compile project(':streams')
     compile libs.jmhCore
+    compile libs.mockitoCore
     annotationProcessor libs.jmhGeneratorAnnProcess
     compile libs.jmhCoreBenchmarks
   }
@@ -1431,6 +1433,9 @@ project(':jmh-benchmarks') {
     }
   }
 
+  checkstyle {
+    configProperties = checkstyleConfigProperties("import-control-jmh-benchmarks.xml")
+  }
 
   task jmh(type: JavaExec, dependsOn: [':jmh-benchmarks:clean', ':jmh-benchmarks:shadowJar']) {
 
diff --git a/checkstyle/import-control-jmh-benchmarks.xml b/checkstyle/import-control-jmh-benchmarks.xml
new file mode 100644
index 0000000000000..49bd8c786920d
--- /dev/null
+++ b/checkstyle/import-control-jmh-benchmarks.xml
@@ -0,0 +1,45 @@
+
+
+
+
+
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+
+
+    
+    
+
diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala
index 03395a4e4e568..655bdeae008a1 100755
--- a/core/src/main/scala/kafka/cluster/Partition.scala
+++ b/core/src/main/scala/kafka/cluster/Partition.scala
@@ -404,8 +404,9 @@ class Partition(val topicPartition: TopicPartition,
       this.log = Some(log)
   }
 
-  def remoteReplicas: Set[Replica] =
-    remoteReplicasMap.values.toSet
+  // remoteReplicas will be called in the hot path, and must be inexpensive
+  def remoteReplicas: Iterable[Replica] =
+    remoteReplicasMap.values
 
   def futureReplicaDirChanged(newDestinationDir: String): Boolean = {
     inReadLock(leaderIsrUpdateLock) {
@@ -585,31 +586,41 @@ class Partition(val topicPartition: TopicPartition,
                                followerStartOffset: Long,
                                followerFetchTimeMs: Long,
                                leaderEndOffset: Long): Boolean = {
-
     getReplica(followerId) match {
       case Some(followerReplica) =>
         // No need to calculate low watermark if there is no delayed DeleteRecordsRequest
         val oldLeaderLW = if (delayedOperations.numDelayedDelete > 0) lowWatermarkIfLeader else -1L
+        val prevFollowerEndOffset = followerReplica.logEndOffset
         followerReplica.updateFetchState(
           followerFetchOffsetMetadata,
           followerStartOffset,
           followerFetchTimeMs,
           leaderEndOffset)
+
         val newLeaderLW = if (delayedOperations.numDelayedDelete > 0) lowWatermarkIfLeader else -1L
         // check if the LW of the partition has incremented
         // since the replica's logStartOffset may have incremented
         val leaderLWIncremented = newLeaderLW > oldLeaderLW
+
         // check if we need to expand ISR to include this replica
         // if it is not in the ISR yet
-        val followerFetchOffset = followerFetchOffsetMetadata.messageOffset
-        val leaderHWIncremented = maybeExpandIsr(followerReplica, followerFetchTimeMs)
+        if (!inSyncReplicaIds(followerId))
+          maybeExpandIsr(followerReplica, followerFetchTimeMs)
+
+        // check if the HW of the partition can now be incremented
+        // since the replica may already be in the ISR and its LEO has just incremented
+        val leaderHWIncremented = if (prevFollowerEndOffset != followerReplica.logEndOffset) {
+          leaderLogIfLocal.exists(leaderLog => maybeIncrementLeaderHW(leaderLog, followerFetchTimeMs))
+        } else {
+          false
+        }
 
         // some delayed operations may be unblocked after HW or LW changed
         if (leaderLWIncremented || leaderHWIncremented)
           tryCompleteDelayedRequests()
 
         debug(s"Recorded replica $followerId log end offset (LEO) position " +
-          s"$followerFetchOffset and log start offset $followerStartOffset.")
+          s"${followerFetchOffsetMetadata.messageOffset} and log start offset $followerStartOffset.")
         true
 
       case None =>
@@ -654,27 +665,20 @@ class Partition(val topicPartition: TopicPartition,
    * whether a replica is in-sync, we only check HW.
    *
    * This function can be triggered when a replica's LEO has incremented.
-   *
-   * @return true if the high watermark has been updated
    */
-  private def maybeExpandIsr(followerReplica: Replica, followerFetchTimeMs: Long): Boolean = {
+  private def maybeExpandIsr(followerReplica: Replica, followerFetchTimeMs: Long): Unit = {
     inWriteLock(leaderIsrUpdateLock) {
       // check if this replica needs to be added to the ISR
-      leaderLogIfLocal match {
-        case Some(leaderLog) =>
-          val leaderHighwatermark = leaderLog.highWatermark
-          if (!inSyncReplicaIds.contains(followerReplica.brokerId) && isFollowerInSync(followerReplica, leaderHighwatermark)) {
-            val newInSyncReplicaIds = inSyncReplicaIds + followerReplica.brokerId
-            info(s"Expanding ISR from ${inSyncReplicaIds.mkString(",")} " +
-              s"to ${newInSyncReplicaIds.mkString(",")}")
-
-            // update ISR in ZK and cache
-            expandIsr(newInSyncReplicaIds)
-          }
-          // check if the HW of the partition can now be incremented
-          // since the replica may already be in the ISR and its LEO has just incremented
-          maybeIncrementLeaderHW(leaderLog, followerFetchTimeMs)
-        case None => false // nothing to do if no longer leader
+      leaderLogIfLocal.foreach { leaderLog =>
+        val leaderHighwatermark = leaderLog.highWatermark
+        if (!inSyncReplicaIds.contains(followerReplica.brokerId) && isFollowerInSync(followerReplica, leaderHighwatermark)) {
+          val newInSyncReplicaIds = inSyncReplicaIds + followerReplica.brokerId
+          info(s"Expanding ISR from ${inSyncReplicaIds.mkString(",")} " +
+            s"to ${newInSyncReplicaIds.mkString(",")}")
+
+          // update ISR in ZK and cache
+          expandIsr(newInSyncReplicaIds)
+        }
       }
     }
   }
@@ -749,25 +753,35 @@ class Partition(val topicPartition: TopicPartition,
    * since all callers of this private API acquire that lock
    */
   private def maybeIncrementLeaderHW(leaderLog: Log, curTime: Long = time.milliseconds): Boolean = {
-    val replicaLogEndOffsets = remoteReplicas.filter { replica =>
-      curTime - replica.lastCaughtUpTimeMs <= replicaLagTimeMaxMs || inSyncReplicaIds.contains(replica.brokerId)
-    }.map(_.logEndOffsetMetadata)
-    val newHighWatermark = (replicaLogEndOffsets + leaderLog.logEndOffsetMetadata).min(new LogOffsetMetadata.OffsetOrdering)
-    leaderLog.maybeIncrementHighWatermark(newHighWatermark) match {
-      case Some(oldHighWatermark) =>
-        debug(s"High watermark updated from $oldHighWatermark to $newHighWatermark")
-        true
-
-      case None =>
-        def logEndOffsetString: ((Int, LogOffsetMetadata)) => String = {
-          case (brokerId, logEndOffsetMetadata) => s"replica $brokerId: $logEndOffsetMetadata"
+    inReadLock(leaderIsrUpdateLock) {
+      // maybeIncrementLeaderHW is in the hot path, the following code is written to
+      // avoid unnecessary collection generation
+      var newHighWatermark = leaderLog.logEndOffsetMetadata
+      remoteReplicasMap.values.foreach { replica =>
+        if (replica.logEndOffsetMetadata.messageOffset < newHighWatermark.messageOffset &&
+          (curTime - replica.lastCaughtUpTimeMs <= replicaLagTimeMaxMs || inSyncReplicaIds.contains(replica.brokerId))) {
+          newHighWatermark = replica.logEndOffsetMetadata
         }
+      }
 
-        val replicaInfo = remoteReplicas.map(replica => (replica.brokerId, replica.logEndOffsetMetadata))
-        val localLogInfo = (localBrokerId, localLogOrException.logEndOffsetMetadata)
-        trace(s"Skipping update high watermark since new hw $newHighWatermark is not larger than old value. " +
-          s"All current LEOs are ${(replicaInfo + localLogInfo).map(logEndOffsetString)}")
-        false
+      leaderLog.maybeIncrementHighWatermark(newHighWatermark) match {
+        case Some(oldHighWatermark) =>
+          debug(s"High watermark updated from $oldHighWatermark to $newHighWatermark")
+          true
+
+        case None =>
+          def logEndOffsetString: ((Int, LogOffsetMetadata)) => String = {
+            case (brokerId, logEndOffsetMetadata) => s"replica $brokerId: $logEndOffsetMetadata"
+          }
+
+          if (isTraceEnabled) {
+            val replicaInfo = remoteReplicas.map(replica => (replica.brokerId, replica.logEndOffsetMetadata)).toSet
+            val localLogInfo = (localBrokerId, localLogOrException.logEndOffsetMetadata)
+            trace(s"Skipping update high watermark since new hw $newHighWatermark is not larger than old value. " +
+              s"All current LEOs are ${(replicaInfo + localLogInfo).map(logEndOffsetString)}")
+          }
+          false
+      }
     }
   }
 
@@ -779,15 +793,21 @@ class Partition(val topicPartition: TopicPartition,
   def lowWatermarkIfLeader: Long = {
     if (!isLeader)
       throw new NotLeaderForPartitionException(s"Leader not local for partition $topicPartition on broker $localBrokerId")
-    val logStartOffsets = remoteReplicas.collect {
-      case replica if metadataCache.getAliveBroker(replica.brokerId).nonEmpty => replica.logStartOffset
-    } + localLogOrException.logStartOffset
+
+    // lowWatermarkIfLeader may be called many times when a DeleteRecordsRequest is outstanding,
+    // care has been taken to avoid generating unnecessary collections in this code
+    var lowWaterMark = localLogOrException.logStartOffset
+    remoteReplicas.foreach { replica =>
+      if (metadataCache.getAliveBroker(replica.brokerId).nonEmpty && replica.logStartOffset < lowWaterMark) {
+        lowWaterMark = replica.logStartOffset
+      }
+    }
 
     futureLog match {
       case Some(partitionFutureLog) =>
-        CoreUtils.min(logStartOffsets + partitionFutureLog.logStartOffset, 0L)
+        Math.min(lowWaterMark, partitionFutureLog.logStartOffset)
       case None =>
-        CoreUtils.min(logStartOffsets, 0L)
+        lowWaterMark
     }
   }
 
diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index 370074929ca3a..48fc9e9a7d5a8 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -341,16 +341,18 @@ class Log(@volatile var dir: File,
       throw new IllegalArgumentException(s"High watermark $newHighWatermark update exceeds current " +
         s"log end offset $logEndOffsetMetadata")
 
-    val oldHighWatermark = fetchHighWatermarkMetadata
-
-    // Ensure that the high watermark increases monotonically. We also update the high watermark when the new
-    // offset metadata is on a newer segment, which occurs whenever the log is rolled to a new segment.
-    if (oldHighWatermark.messageOffset < newHighWatermark.messageOffset ||
-      (oldHighWatermark.messageOffset == newHighWatermark.messageOffset && oldHighWatermark.onOlderSegment(newHighWatermark))) {
-      updateHighWatermarkMetadata(newHighWatermark)
-      Some(oldHighWatermark)
-    } else {
-      None
+    lock.synchronized {
+      val oldHighWatermark = fetchHighWatermarkMetadata
+
+      // Ensure that the high watermark increases monotonically. We also update the high watermark when the new
+      // offset metadata is on a newer segment, which occurs whenever the log is rolled to a new segment.
+      if (oldHighWatermark.messageOffset < newHighWatermark.messageOffset ||
+        (oldHighWatermark.messageOffset == newHighWatermark.messageOffset && oldHighWatermark.onOlderSegment(newHighWatermark))) {
+        updateHighWatermarkMetadata(newHighWatermark)
+        Some(oldHighWatermark)
+      } else {
+        None
+      }
     }
   }
 
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index 12cb507d35005..9579b1f56ec38 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -1085,8 +1085,8 @@ class ReplicaManager(val config: KafkaConfig,
             .map(replica => new DefaultReplicaView(
               replicaEndpoints.getOrElse(replica.brokerId, Node.noNode()),
               replica.logEndOffset,
-              currentTimeMs - replica.lastCaughtUpTimeMs
-            ))
+              currentTimeMs - replica.lastCaughtUpTimeMs))
+            .toSet
 
           if (partition.leaderReplicaIdOpt.isDefined) {
             val leaderReplica: ReplicaView = partition.leaderReplicaIdOpt
diff --git a/gradle/spotbugs-exclude.xml b/gradle/spotbugs-exclude.xml
index e843d9f3118ac..7854a9beb24af 100644
--- a/gradle/spotbugs-exclude.xml
+++ b/gradle/spotbugs-exclude.xml
@@ -156,6 +156,7 @@ For a detailed description of spotbugs bug categories, see https://spotbugs.read
             
             
             
+            
         
     
 
diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/partition/UpdateFollowerFetchStateBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/partition/UpdateFollowerFetchStateBenchmark.java
new file mode 100644
index 0000000000000..ee45b6419faec
--- /dev/null
+++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/partition/UpdateFollowerFetchStateBenchmark.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.jmh.partition;
+
+import kafka.api.ApiVersion$;
+import kafka.cluster.DelayedOperations;
+import kafka.cluster.Partition;
+import kafka.cluster.PartitionStateStore;
+import kafka.log.CleanerConfig;
+import kafka.log.Defaults;
+import kafka.log.LogConfig;
+import kafka.log.LogManager;
+import kafka.server.BrokerState;
+import kafka.server.BrokerTopicStats;
+import kafka.server.LogDirFailureChannel;
+import kafka.server.LogOffsetMetadata;
+import kafka.server.MetadataCache;
+import kafka.server.checkpoints.OffsetCheckpoints;
+import kafka.utils.KafkaScheduler;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.requests.LeaderAndIsrRequest;
+import org.apache.kafka.common.utils.Time;
+import org.mockito.Mockito;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+import scala.Option;
+import scala.collection.JavaConverters;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Properties;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+@State(Scope.Benchmark)
+@Fork(value = 1)
+@Warmup(iterations = 5)
+@Measurement(iterations = 15)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+public class UpdateFollowerFetchStateBenchmark {
+    private TopicPartition topicPartition = new TopicPartition(UUID.randomUUID().toString(), 0);
+    private File logDir = new File(System.getProperty("java.io.tmpdir"), topicPartition.toString());
+    private KafkaScheduler scheduler = new KafkaScheduler(1, "scheduler", true);
+    private BrokerTopicStats brokerTopicStats = new BrokerTopicStats();
+    private LogDirFailureChannel logDirFailureChannel = Mockito.mock(LogDirFailureChannel.class);
+    private long nextOffset = 0;
+    private LogManager logManager;
+    private Partition partition;
+
+    @Setup(Level.Trial)
+    public void setUp() {
+        scheduler.startup();
+        LogConfig logConfig = createLogConfig();
+        List logDirs = Collections.singletonList(logDir);
+        logManager = new LogManager(JavaConverters.asScalaIteratorConverter(logDirs.iterator()).asScala().toSeq(),
+                JavaConverters.asScalaIteratorConverter(new ArrayList().iterator()).asScala().toSeq(),
+                new scala.collection.mutable.HashMap<>(),
+                logConfig,
+                new CleanerConfig(0, 0, 0, 0, 0, 0.0, 0, false, "MD5"),
+                1,
+                1000L,
+                10000L,
+                10000L,
+                1000L,
+                60000,
+                scheduler,
+                new BrokerState(),
+                brokerTopicStats,
+                logDirFailureChannel,
+                Time.SYSTEM);
+        OffsetCheckpoints offsetCheckpoints = Mockito.mock(OffsetCheckpoints.class);
+        Mockito.when(offsetCheckpoints.fetch(logDir.getAbsolutePath(), topicPartition)).thenReturn(Option.apply(0L));
+        DelayedOperations delayedOperations = new DelayedOperationsMock();
+
+        // one leader, plus two followers
+        List replicas = new ArrayList<>();
+        replicas.add(0);
+        replicas.add(1);
+        replicas.add(2);
+        LeaderAndIsrRequest.PartitionState partitionState = new LeaderAndIsrRequest.PartitionState(
+                0, 0, 0, replicas, 1, replicas, true);
+        PartitionStateStore partitionStateStore = Mockito.mock(PartitionStateStore.class);
+        Mockito.when(partitionStateStore.fetchTopicConfig()).thenReturn(new Properties());
+        partition = new Partition(topicPartition, 100,
+                ApiVersion$.MODULE$.latestVersion(), 0, Time.SYSTEM,
+                partitionStateStore, delayedOperations,
+                Mockito.mock(MetadataCache.class), logManager);
+        partition.makeLeader(0, partitionState, 0, offsetCheckpoints);
+    }
+
+    // avoid mocked DelayedOperations to avoid mocked class affecting benchmark results
+    private class DelayedOperationsMock extends DelayedOperations {
+        DelayedOperationsMock() {
+            super(topicPartition, null, null, null);
+        }
+
+        @Override
+        public int numDelayedDelete() {
+            return 0;
+        }
+    }
+
+    @TearDown(Level.Trial)
+    public void tearDown() {
+        logManager.shutdown();
+        scheduler.shutdown();
+    }
+
+    private LogConfig createLogConfig() {
+        Properties logProps = new Properties();
+        logProps.put(LogConfig.SegmentMsProp(), Defaults.SegmentMs());
+        logProps.put(LogConfig.SegmentBytesProp(), Defaults.SegmentSize());
+        logProps.put(LogConfig.RetentionMsProp(), Defaults.RetentionMs());
+        logProps.put(LogConfig.RetentionBytesProp(), Defaults.RetentionSize());
+        logProps.put(LogConfig.SegmentJitterMsProp(), Defaults.SegmentJitterMs());
+        logProps.put(LogConfig.CleanupPolicyProp(), Defaults.CleanupPolicy());
+        logProps.put(LogConfig.MaxMessageBytesProp(), Defaults.MaxMessageSize());
+        logProps.put(LogConfig.IndexIntervalBytesProp(), Defaults.IndexInterval());
+        logProps.put(LogConfig.SegmentIndexBytesProp(), Defaults.MaxIndexSize());
+        logProps.put(LogConfig.MessageFormatVersionProp(), Defaults.MessageFormatVersion());
+        logProps.put(LogConfig.FileDeleteDelayMsProp(), Defaults.FileDeleteDelayMs());
+        return LogConfig.apply(logProps, new scala.collection.immutable.HashSet<>());
+    }
+
+    @Benchmark
+    @OutputTimeUnit(TimeUnit.NANOSECONDS)
+    public void updateFollowerFetchStateBench() {
+        // measure the impact of two follower fetches on the leader
+        partition.updateFollowerFetchState(1, new LogOffsetMetadata(nextOffset, nextOffset, 0),
+                0, 1, nextOffset);
+        partition.updateFollowerFetchState(2, new LogOffsetMetadata(nextOffset, nextOffset, 0),
+                0, 1, nextOffset);
+        nextOffset++;
+    }
+
+    @Benchmark
+    @OutputTimeUnit(TimeUnit.NANOSECONDS)
+    public void updateFollowerFetchStateBenchNoChange() {
+        // measure the impact of two follower fetches on the leader when the follower didn't
+        // end up fetching anything
+        partition.updateFollowerFetchState(1, new LogOffsetMetadata(nextOffset, nextOffset, 0),
+                0, 1, 100);
+        partition.updateFollowerFetchState(2, new LogOffsetMetadata(nextOffset, nextOffset, 0),
+                0, 1, 100);
+    }
+}

From ada35d5ff45500d9d77ff64bc6017305e1cac70d Mon Sep 17 00:00:00 2001
From: Randall Hauch 
Date: Wed, 18 Sep 2019 12:21:21 -0500
Subject: [PATCH 0622/1071] Add recent versions of Kafka to the matrix of
 ConnectDistributedTest (#7024)

Reviewers: Arjun Satish , Konstantine Karantasis 
---
 .../tests/connect/connect_distributed_test.py      | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tests/kafkatest/tests/connect/connect_distributed_test.py b/tests/kafkatest/tests/connect/connect_distributed_test.py
index 329d9f9404df6..ada6ca27b4875 100644
--- a/tests/kafkatest/tests/connect/connect_distributed_test.py
+++ b/tests/kafkatest/tests/connect/connect_distributed_test.py
@@ -24,7 +24,7 @@
 from kafkatest.services.connect import ConnectDistributedService, VerifiableSource, VerifiableSink, ConnectRestError, MockSink, MockSource
 from kafkatest.services.console_consumer import ConsoleConsumer
 from kafkatest.services.security.security_config import SecurityConfig
-from kafkatest.version import DEV_BRANCH, LATEST_0_11_0, LATEST_0_10_2, LATEST_0_10_1, LATEST_0_10_0, LATEST_0_9, LATEST_0_8_2, KafkaVersion
+from kafkatest.version import DEV_BRANCH, LATEST_2_3, LATEST_2_2, LATEST_2_1, LATEST_2_0, LATEST_1_1, LATEST_1_0, LATEST_0_11_0, LATEST_0_10_2, LATEST_0_10_1, LATEST_0_10_0, LATEST_0_9, LATEST_0_8_2, KafkaVersion
 
 from collections import Counter, namedtuple
 import itertools
@@ -528,11 +528,23 @@ def test_transformations(self, connect_protocol):
 
     @cluster(num_nodes=5)
     @parametrize(broker_version=str(DEV_BRANCH), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible')
+    @parametrize(broker_version=str(LATEST_2_3), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible')
+    @parametrize(broker_version=str(LATEST_2_2), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible')
+    @parametrize(broker_version=str(LATEST_2_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible')
+    @parametrize(broker_version=str(LATEST_2_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible')
+    @parametrize(broker_version=str(LATEST_1_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible')
+    @parametrize(broker_version=str(LATEST_1_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible')
     @parametrize(broker_version=str(LATEST_0_11_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible')
     @parametrize(broker_version=str(LATEST_0_10_2), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible')
     @parametrize(broker_version=str(LATEST_0_10_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible')
     @parametrize(broker_version=str(LATEST_0_10_0), auto_create_topics=True, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible')
     @parametrize(broker_version=str(DEV_BRANCH), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager')
+    @parametrize(broker_version=str(LATEST_2_3), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager')
+    @parametrize(broker_version=str(LATEST_2_2), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager')
+    @parametrize(broker_version=str(LATEST_2_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager')
+    @parametrize(broker_version=str(LATEST_2_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager')
+    @parametrize(broker_version=str(LATEST_1_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager')
+    @parametrize(broker_version=str(LATEST_1_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager')
     @parametrize(broker_version=str(LATEST_0_11_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager')
     @parametrize(broker_version=str(LATEST_0_10_2), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager')
     @parametrize(broker_version=str(LATEST_0_10_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager')

From f718c5e1e10d07253d2024d221768434a731fd17 Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Wed, 18 Sep 2019 12:51:47 -0700
Subject: [PATCH 0623/1071] MINOR: Update dependencies for Kafka 2.4 (part 2)
 (#7333)

Upgrade to Gradle 5.6.2 as a step towards Gradle 6.0 (necessary
for Java 13 support).

https://docs.gradle.org/5.5.1/release-notes.html
https://docs.gradle.org/5.6.2/release-notes.html

The other updates are mostly bug fixes:

* Scala 2.13.1: https://github.com/scala/scala/releases/tag/v2.13.1
* Scala 2.12.10: https://github.com/scala/scala/releases/tag/v2.12.10
* Jetty 9.4.20: https://www.eclipse.org/lists/jetty-announce/msg00133.html
* SLF4J 1.7.28: adds Automatic-Module-Name in MANIFEST.MF
* Bouncy castle 1.63: https://www.bouncycastle.org/releasenotes.html
* zstd 1.4.3: https://github.com/facebook/zstd/releases/tag/v1.4.3

Reviewers: Manikumar Reddy 
---
 bin/kafka-run-class.sh          |  2 +-
 bin/windows/kafka-run-class.bat |  2 +-
 gradle.properties               |  2 +-
 gradle/dependencies.gradle      | 14 +++++++-------
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/bin/kafka-run-class.sh b/bin/kafka-run-class.sh
index dc0235e1cbece..1221860cb1116 100755
--- a/bin/kafka-run-class.sh
+++ b/bin/kafka-run-class.sh
@@ -48,7 +48,7 @@ should_include_file() {
 base_dir=$(dirname $0)/..
 
 if [ -z "$SCALA_VERSION" ]; then
-  SCALA_VERSION=2.12.9
+  SCALA_VERSION=2.12.10
 fi
 
 if [ -z "$SCALA_BINARY_VERSION" ]; then
diff --git a/bin/windows/kafka-run-class.bat b/bin/windows/kafka-run-class.bat
index cc9f3d6a26021..ad7cbb882ce08 100755
--- a/bin/windows/kafka-run-class.bat
+++ b/bin/windows/kafka-run-class.bat
@@ -27,7 +27,7 @@ set BASE_DIR=%CD%
 popd
 
 IF ["%SCALA_VERSION%"] EQU [""] (
-  set SCALA_VERSION=2.12.9
+  set SCALA_VERSION=2.12.10
 )
 
 IF ["%SCALA_BINARY_VERSION%"] EQU [""] (
diff --git a/gradle.properties b/gradle.properties
index 9c4949629c2fe..20344ed6c4e00 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -21,6 +21,6 @@ group=org.apache.kafka
 #  - tests/kafkatest/version.py (variable DEV_VERSION)
 #  - kafka-merge-pr.py
 version=2.4.0-SNAPSHOT
-scalaVersion=2.12.9
+scalaVersion=2.12.10
 task=build
 org.gradle.jvmargs=-Xmx1024m -Xss2m
diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle
index d49e44e1cbbf2..045abbc0263ae 100644
--- a/gradle/dependencies.gradle
+++ b/gradle/dependencies.gradle
@@ -30,8 +30,8 @@ ext {
 
 // Add Scala version
 def defaultScala211Version = '2.11.12'
-def defaultScala212Version = '2.12.9'
-def defaultScala213Version = '2.13.0'
+def defaultScala212Version = '2.12.10'
+def defaultScala213Version = '2.13.1'
 if (hasProperty('scalaVersion')) {
   if (scalaVersion == '2.11') {
     versions["scala"] = defaultScala211Version
@@ -63,10 +63,10 @@ versions += [
   apacheda: "1.0.2",
   apacheds: "2.0.0-M24",
   argparse4j: "0.7.0",
-  bcpkix: "1.62",
+  bcpkix: "1.63",
   checkstyle: "8.20",
   commonsCli: "1.4",
-  gradle: "5.4.1",
+  gradle: "5.6.2",
   gradleVersionsPlugin: "0.21.0",
   grgit: "3.1.1",
   httpclient: "4.5.9",
@@ -74,7 +74,7 @@ versions += [
   jackson: "2.9.9",
   jacksonDatabind: "2.9.9.3",
   jacoco: "0.8.3",
-  jetty: "9.4.19.v20190610",
+  jetty: "9.4.20.v20190813",
   jersey: "2.28",
   jmh: "1.21",
   hamcrest: "2.1",
@@ -110,13 +110,13 @@ versions += [
   scoverage: "1.4.0",
   scoveragePlugin: "2.5.0",
   shadowPlugin: "4.0.4",
-  slf4j: "1.7.27",
+  slf4j: "1.7.28",
   snappy: "1.1.7.3",
   spotbugs: "3.1.12",
   spotbugsPlugin: "1.6.9",
   spotlessPlugin: "3.23.1",
   zookeeper: "3.5.5",
-  zstd: "1.4.2-1"
+  zstd: "1.4.3-1"
 ]
 
 libs += [

From fe2797a9fa9f344dbbdd73a46ec48cf9894a9e80 Mon Sep 17 00:00:00 2001
From: Manikumar Reddy 
Date: Thu, 19 Sep 2019 20:43:28 +0530
Subject: [PATCH 0624/1071] MINOR: Update authorizer start-up check to handle
 end point with ephemeral port

Author: Manikumar Reddy 

Reviewers: Rajini Sivaram , Ismael Juma 

Closes #7350 from omkreddy/checkstartup
---
 .../main/scala/kafka/network/SocketServer.scala   | 15 +++++++++++----
 .../unit/kafka/network/SocketServerTest.scala     |  6 +++++-
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala
index adaaabf946f25..e705d41b16d11 100644
--- a/core/src/main/scala/kafka/network/SocketServer.scala
+++ b/core/src/main/scala/kafka/network/SocketServer.scala
@@ -210,9 +210,7 @@ class SocketServer(val config: KafkaConfig,
     orderedAcceptors.foreach { acceptor =>
       val endpoint = acceptor.endPoint
       debug(s"Wait for authorizer to complete start up on listener ${endpoint.listener}")
-      authorizerFutures.get(endpoint).foreach { future =>
-        future.join()
-      }
+      waitForAuthorizerFuture(acceptor, authorizerFutures)
       debug(s"Start processors on listener ${endpoint.listener}")
       acceptor.startProcessors(DataPlaneThreadPrefix)
     }
@@ -226,7 +224,7 @@ class SocketServer(val config: KafkaConfig,
    */
   def startControlPlaneProcessor(authorizerFutures: Map[Endpoint, CompletableFuture[Void]] = Map.empty): Unit = synchronized {
     controlPlaneAcceptorOpt.foreach { controlPlaneAcceptor =>
-      authorizerFutures.get(controlPlaneAcceptor.endPoint).foreach(_.get)
+      waitForAuthorizerFuture(controlPlaneAcceptor, authorizerFutures)
       controlPlaneAcceptor.startProcessors(ControlPlaneThreadPrefix)
       info(s"Started control-plane processor for the control-plane acceptor")
     }
@@ -380,6 +378,15 @@ class SocketServer(val config: KafkaConfig,
     }
   }
 
+  private def waitForAuthorizerFuture(acceptor: Acceptor,
+                                      authorizerFutures: Map[Endpoint, CompletableFuture[Void]]): Unit = {
+    //we can't rely on authorizerFutures.get() due to ephemeral ports. Get the future using listener name
+    authorizerFutures.foreach { case (endpoint, future) =>
+      if (endpoint.listener() == acceptor.endPoint.listener())
+        future.join()
+    }
+  }
+
   // `protected` for test usage
   protected[network] def newProcessor(id: Int, requestChannel: RequestChannel, connectionQuotas: ConnectionQuotas, listenerName: ListenerName,
                                       securityProtocol: SecurityProtocol, memoryPool: MemoryPool): Processor = {
diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
index 7072c94f7653c..507e117f5d22e 100644
--- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
+++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
@@ -202,6 +202,10 @@ class SocketServerTest {
     val config = KafkaConfig.fromProps(testProps)
     val testableServer = new TestableSocketServer(config)
     testableServer.startup(startupProcessors = false)
+    val updatedEndPoints = config.advertisedListeners.map { endpoint =>
+      endpoint.copy(port = testableServer.boundPort(endpoint.listenerName))
+    }.map(_.asInstanceOf[Endpoint])
+
     val externalReadyFuture = new CompletableFuture[Void]()
     val executor = Executors.newSingleThreadExecutor()
 
@@ -221,7 +225,7 @@ class SocketServerTest {
       sendAndReceiveControllerRequest(socket1, testableServer)
 
       val externalListener = new ListenerName("EXTERNAL")
-      val externalEndpoint = new Endpoint(externalListener.value, SecurityProtocol.PLAINTEXT, "localhost", 0)
+      val externalEndpoint = updatedEndPoints.find(e => e.listener() == externalListener.value).get
       val futures =  Map(externalEndpoint -> externalReadyFuture)
       val startFuture = executor.submit(CoreUtils.runnable(testableServer.startDataPlaneProcessors(futures)))
       TestUtils.waitUntilTrue(() => listenerStarted(config.interBrokerListenerName), "Inter-broker listener not started")

From d9d3d3b710dd627533175f5142a5957e6b3cc2cc Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Thu, 19 Sep 2019 08:50:59 -0700
Subject: [PATCH 0625/1071] MINOR: Add last modified time and deletion horizon
 to clear log message (#7357)

It's useful to know when the cleaner runs what the last modified time
of the segment and the deletion horizon is. The current log message
only allows you to infer that one is greater than the other.

Reviewers: Jun Rao 
---
 core/src/main/scala/kafka/log/LogCleaner.scala | 17 +++++++++--------
 core/src/main/scala/kafka/log/LogSegment.scala |  6 +++++-
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/core/src/main/scala/kafka/log/LogCleaner.scala b/core/src/main/scala/kafka/log/LogCleaner.scala
index 47c1fa4f001f4..e37eacf8c06bf 100644
--- a/core/src/main/scala/kafka/log/LogCleaner.scala
+++ b/core/src/main/scala/kafka/log/LogCleaner.scala
@@ -571,12 +571,13 @@ private[log] class Cleaner(val id: Int,
         val abortedTransactions = log.collectAbortedTransactions(startOffset, upperBoundOffset)
         transactionMetadata.addAbortedTransactions(abortedTransactions)
 
-        val retainDeletes = currentSegment.lastModified > deleteHorizonMs
-        info(s"Cleaning segment $startOffset in log ${log.name} (largest timestamp ${new Date(currentSegment.largestTimestamp)}) " +
-          s"into ${cleaned.baseOffset}, ${if(retainDeletes) "retaining" else "discarding"} deletes.")
+        val retainDeletesAndTxnMarkers = currentSegment.lastModified > deleteHorizonMs
+        info(s"Cleaning $currentSegment in log ${log.name} into ${cleaned.baseOffset} " +
+          s"with deletion horizon $deleteHorizonMs, " +
+          s"${if(retainDeletesAndTxnMarkers) "retaining" else "discarding"} deletes.")
 
         try {
-          cleanInto(log.topicPartition, currentSegment.log, cleaned, map, retainDeletes, log.config.maxMessageSize,
+          cleanInto(log.topicPartition, currentSegment.log, cleaned, map, retainDeletesAndTxnMarkers, log.config.maxMessageSize,
             transactionMetadata, lastOffsetOfActiveProducers, stats)
         } catch {
           case e: LogSegmentOffsetOverflowException =>
@@ -618,7 +619,7 @@ private[log] class Cleaner(val id: Int,
    * @param sourceRecords The dirty log segment
    * @param dest The cleaned log segment
    * @param map The key=>offset mapping
-   * @param retainDeletes Should delete tombstones be retained while cleaning this segment
+   * @param retainDeletesAndTxnMarkers Should tombstones and markers be retained while cleaning this segment
    * @param maxLogMessageSize The maximum message size of the corresponding topic
    * @param stats Collector for cleaning statistics
    */
@@ -626,7 +627,7 @@ private[log] class Cleaner(val id: Int,
                              sourceRecords: FileRecords,
                              dest: LogSegment,
                              map: OffsetMap,
-                             retainDeletes: Boolean,
+                             retainDeletesAndTxnMarkers: Boolean,
                              maxLogMessageSize: Int,
                              transactionMetadata: CleanedTransactionMetadata,
                              lastRecordsOfActiveProducers: Map[Long, LastRecord],
@@ -637,7 +638,7 @@ private[log] class Cleaner(val id: Int,
       override def checkBatchRetention(batch: RecordBatch): BatchRetention = {
         // we piggy-back on the tombstone retention logic to delay deletion of transaction markers.
         // note that we will never delete a marker until all the records from that transaction are removed.
-        discardBatchRecords = shouldDiscardBatch(batch, transactionMetadata, retainTxnMarkers = retainDeletes)
+        discardBatchRecords = shouldDiscardBatch(batch, transactionMetadata, retainTxnMarkers = retainDeletesAndTxnMarkers)
 
         def isBatchLastRecordOfProducer: Boolean = {
           // We retain the batch in order to preserve the state of active producers. There are three cases:
@@ -667,7 +668,7 @@ private[log] class Cleaner(val id: Int,
           // The batch is only retained to preserve producer sequence information; the records can be removed
           false
         else
-          Cleaner.this.shouldRetainRecord(map, retainDeletes, batch, record, stats)
+          Cleaner.this.shouldRetainRecord(map, retainDeletesAndTxnMarkers, batch, record, stats)
       }
     }
 
diff --git a/core/src/main/scala/kafka/log/LogSegment.scala b/core/src/main/scala/kafka/log/LogSegment.scala
index f5d740be4ffee..38a46e8504102 100755
--- a/core/src/main/scala/kafka/log/LogSegment.scala
+++ b/core/src/main/scala/kafka/log/LogSegment.scala
@@ -409,7 +409,11 @@ class LogSegment private[log] (val log: FileRecords,
   def collectAbortedTxns(fetchOffset: Long, upperBoundOffset: Long): TxnIndexSearchResult =
     txnIndex.collectAbortedTxns(fetchOffset, upperBoundOffset)
 
-  override def toString = "LogSegment(baseOffset=" + baseOffset + ", size=" + size + ")"
+  override def toString: String = "LogSegment(baseOffset=" + baseOffset +
+    ", size=" + size +
+    ", lastModifiedTime=" + lastModified +
+    ", largestTime=" + largestTimestamp
+    ")"
 
   /**
    * Truncate off all index and log entries with offsets >= the given offset.

From 73c6bd8ac9a63de83162370a20f5364581fc9576 Mon Sep 17 00:00:00 2001
From: Richard Yu 
Date: Thu, 19 Sep 2019 11:50:25 -0700
Subject: [PATCH 0626/1071] [KAFKA-7994] Improve Stream time accuracy for
 restarts and rebalances  (#6694)

Reviewers: Bruno Cadonna , A. Sophie Blee-Goldman , Boyang Chen , Matthias J. Sax 
---
 .../processor/internals/AssignedTasks.java    |   1 +
 .../processor/internals/PartitionGroup.java   |  20 ++
 .../processor/internals/RecordQueue.java      |  21 +-
 .../processor/internals/StandbyTask.java      |   5 +
 .../processor/internals/StreamTask.java       |  83 +++++++-
 .../streams/processor/internals/Task.java     |   1 +
 .../ResetPartitionTimeIntegrationTest.java    | 190 ++++++++++++++++++
 .../processor/internals/AbstractTaskTest.java |   3 +
 .../internals/AssignedStreamsTasksTest.java   |  10 +
 .../internals/PartitionGroupTest.java         |  55 ++++-
 .../processor/internals/RecordQueueTest.java  |  27 +++
 .../processor/internals/StreamTaskTest.java   |  79 +++++++-
 12 files changed, 477 insertions(+), 18 deletions(-)
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/integration/ResetPartitionTimeIntegrationTest.java

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
index 8053bc792cfb5..8d2c3596dfc90 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
@@ -200,6 +200,7 @@ boolean maybeResumeSuspendedTask(final TaskId taskId, final Set
     void transitionToRunning(final T task) {
         log.debug("Transitioning {} {} to running", taskTypeName, task.id());
         running.put(task.id(), task);
+        task.initializeTaskTime();
         task.initializeTopology();
         for (final TopicPartition topicPartition : task.partitions()) {
             runningByPartition.put(topicPartition, task);
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/PartitionGroup.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/PartitionGroup.java
index 83b3673464713..9160468847cfb 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/PartitionGroup.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/PartitionGroup.java
@@ -84,6 +84,26 @@ RecordQueue queue() {
         streamTime = RecordQueue.UNKNOWN;
     }
 
+    // visible for testing
+    long partitionTimestamp(final TopicPartition partition) {
+        final RecordQueue queue = partitionQueues.get(partition);
+        if (queue == null) {
+            throw new NullPointerException("Partition " + partition + " not found.");
+        }
+        return queue.partitionTime();
+    }
+
+    void setPartitionTime(final TopicPartition partition, final long partitionTime) {
+        final RecordQueue queue = partitionQueues.get(partition);
+        if (queue == null) {
+            throw new NullPointerException("Partition " + partition + " not found.");
+        }
+        if (streamTime < partitionTime) {
+            streamTime = partitionTime;
+        }
+        queue.setPartitionTime(partitionTime);
+    }
+
     /**
      * Get the next record and queue
      *
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordQueue.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordQueue.java
index de1d9a26bb35f..70eb770a283bf 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordQueue.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordQueue.java
@@ -71,6 +71,10 @@ public class RecordQueue {
         );
         this.log = logContext.logger(RecordQueue.class);
     }
+ 
+    void setPartitionTime(final long partitionTime) {
+        this.partitionTime = partitionTime;
+    }
 
     /**
      * Returns the corresponding source node in the topology
@@ -148,15 +152,6 @@ public long headRecordTimestamp() {
         return headRecord == null ? UNKNOWN : headRecord.timestamp;
     }
 
-    /**
-     * Returns the tracked partition time
-     *
-     * @return partition time
-     */
-    long partitionTime() {
-        return partitionTime;
-    }
-
     /**
      * Clear the fifo queue of its elements, also clear the time tracker's kept stamped elements
      */
@@ -198,10 +193,16 @@ private void updateHead() {
                 skipRecordsSensor.record();
                 continue;
             }
-
             headRecord = new StampedRecord(deserialized, timestamp);
 
             partitionTime = Math.max(partitionTime, timestamp);
         }
     }
+
+    /**
+     * @return the local partitionTime for this particular RecordQueue
+     */
+    long partitionTime() {
+        return partitionTime;
+    }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
index c758ccd2e7b23..17b6e665d46af 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
@@ -90,6 +90,11 @@ public void initializeTopology() {
         //no-op
     }
 
+    @Override
+    public void initializeTaskTime() {
+        //no-op
+    }
+
     /**
      * 
                      * - update offset limits
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
index b822cb3eca53e..3eb1e55908991 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
@@ -23,6 +23,8 @@
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.io.StringWriter;
+import java.nio.ByteBuffer;
+import java.util.Base64;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -67,6 +69,8 @@
 public class StreamTask extends AbstractTask implements ProcessorNodePunctuator {
 
     private static final ConsumerRecord DUMMY_RECORD = new ConsumerRecord<>(ProcessorContextImpl.NONEXIST_TOPIC, -1, -1L, null, null);
+    // visible for testing
+    static final byte LATEST_MAGIC_BYTE = 1;
 
     private final Time time;
     private final long maxTaskIdleMs;
@@ -453,7 +457,7 @@ private void updateProcessorContext(final StampedRecord record, final ProcessorN
      */
     @Override
     public void commit() {
-        commit(true);
+        commit(true, extractPartitionTimes());
     }
 
     /**
@@ -461,7 +465,7 @@ public void commit() {
      *                               or if the task producer got fenced (EOS)
      */
     // visible for testing
-    void commit(final boolean startNewTransaction) {
+    void commit(final boolean startNewTransaction, final Map partitionTimes) {
         final long startNs = time.nanoseconds();
         log.debug("Committing");
 
@@ -475,7 +479,9 @@ void commit(final boolean startNewTransaction) {
         for (final Map.Entry entry : consumedOffsets.entrySet()) {
             final TopicPartition partition = entry.getKey();
             final long offset = entry.getValue() + 1;
-            consumedOffsetsAndMetadata.put(partition, new OffsetAndMetadata(offset));
+            final long partitionTime = partitionTimes.get(partition);
+            consumedOffsetsAndMetadata.put(partition, new OffsetAndMetadata(offset, encodeTimestamp(partitionTime)));
+            stateMgr.putOffsetLimit(partition, offset);
         }
 
         try {
@@ -576,6 +582,9 @@ public void suspend() {
     // visible for testing
     void suspend(final boolean clean,
                  final boolean isZombie) {
+        // this is necessary because all partition times are reset to -1 during close
+        // we need to preserve the original partitions times before calling commit
+        final Map partitionTimes = extractPartitionTimes();
         try {
             closeTopology(); // should we call this only on clean suspend?
         } catch (final RuntimeException fatal) {
@@ -587,10 +596,9 @@ void suspend(final boolean clean,
         if (clean) {
             TaskMigratedException taskMigratedException = null;
             try {
-                commit(false);
+                commit(false, partitionTimes);
             } finally {
                 if (eosEnabled) {
-
                     stateMgr.checkpoint(activeTaskCheckpointableOffsets());
 
                     try {
@@ -727,6 +735,28 @@ public void close(boolean clean,
         taskClosed = true;
     }
 
+    private void initializeCommittedTimestamp(final TopicPartition partition) {
+        final OffsetAndMetadata metadata = consumer.committed(partition);
+
+        if (metadata != null) {
+            final long committedTimestamp = decodeTimestamp(metadata.metadata());
+            partitionGroup.setPartitionTime(partition, committedTimestamp);
+            log.debug("A committed timestamp was detected: setting the partition time of partition {}"
+                      + " to {} in stream task {}", partition, committedTimestamp, this);
+        } else {
+            log.debug("No committed timestamp was found in metadata for partition {}", partition);
+        }
+    }
+
+    /**
+     * Retrieves formerly committed timestamps and updates the local queue's partition time.
+     */
+    public void initializeTaskTime() {
+        for (final TopicPartition partition : partitionGroup.partitions()) {
+            initializeCommittedTimestamp(partition);
+        }
+    }
+
     /**
      * Adds records to queues. If a record has an invalid (i.e., negative) timestamp, the record is skipped
      * and not added to the queue for processing
@@ -866,6 +896,16 @@ RecordCollector recordCollector() {
         return recordCollector;
     }
 
+    // used for testing
+    long streamTime() {
+        return partitionGroup.streamTime();
+    }
+
+    // used for testing
+    long partitionTime(final TopicPartition partition) {
+        return partitionGroup.partitionTimestamp(partition);
+    }
+
     Producer getProducer() {
         return producer;
     }
@@ -888,4 +928,37 @@ private void initializeTransactions() {
             );
         }
     }
+
+    // visible for testing
+    String encodeTimestamp(final long partitionTime) {
+        final ByteBuffer buffer = ByteBuffer.allocate(9);
+        buffer.put(LATEST_MAGIC_BYTE);
+        buffer.putLong(partitionTime);
+        return Base64.getEncoder().encodeToString(buffer.array());
+    }
+
+    // visible for testing
+    long decodeTimestamp(final String encryptedString) {
+        if (encryptedString.length() == 0) {
+            return RecordQueue.UNKNOWN;
+        }
+        final ByteBuffer buffer = ByteBuffer.wrap(Base64.getDecoder().decode(encryptedString));
+        final byte version = buffer.get();
+        switch (version) {
+            case LATEST_MAGIC_BYTE:
+                return buffer.getLong();
+            default: 
+                log.warn("Unsupported offset metadata version found. Supported version {}. Found version {}.", 
+                         LATEST_MAGIC_BYTE, version);
+                return RecordQueue.UNKNOWN;
+        }
+    }
+
+    private Map extractPartitionTimes() {
+        final Map partitionTimes = new HashMap<>();
+        for (final TopicPartition partition : partitionGroup.partitions()) {
+            partitionTimes.put(partition, partitionTime(partition));
+        }
+        return partitionTimes;
+    }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/Task.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/Task.java
index 812e7e1131ca2..da9e656ad9e2b 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/Task.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/Task.java
@@ -72,4 +72,5 @@ void close(final boolean clean,
 
     String toString(final String indent);
 
+    void initializeTaskTime();
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/ResetPartitionTimeIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/ResetPartitionTimeIntegrationTest.java
new file mode 100644
index 0000000000000..75fa48233a061
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/ResetPartitionTimeIntegrationTest.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.integration;
+
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.serialization.Deserializer;
+import org.apache.kafka.common.serialization.Serde;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.Serializer;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.KeyValueTimestamp;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
+import org.apache.kafka.streams.integration.utils.IntegrationTestUtils;
+import org.apache.kafka.streams.kstream.Consumed;
+import org.apache.kafka.streams.processor.TimestampExtractor;
+import org.apache.kafka.test.IntegrationTest;
+import org.apache.kafka.test.TestUtils;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+import java.util.Properties;
+
+import static java.util.Arrays.asList;
+import static org.apache.kafka.common.utils.Utils.mkEntry;
+import static org.apache.kafka.common.utils.Utils.mkMap;
+import static org.apache.kafka.common.utils.Utils.mkProperties;
+import static org.apache.kafka.streams.StreamsConfig.AT_LEAST_ONCE;
+import static org.apache.kafka.streams.StreamsConfig.EXACTLY_ONCE;
+import static org.apache.kafka.streams.integration.utils.IntegrationTestUtils.cleanStateAfterTest;
+import static org.apache.kafka.streams.integration.utils.IntegrationTestUtils.cleanStateBeforeTest;
+import static org.apache.kafka.streams.integration.utils.IntegrationTestUtils.getStartedStreams;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+@RunWith(Parameterized.class)
+@Category({IntegrationTest.class})
+public class ResetPartitionTimeIntegrationTest {
+    private static final int NUM_BROKERS = 1;
+    private static final Properties BROKER_CONFIG;
+    static {
+        BROKER_CONFIG = new Properties();
+        BROKER_CONFIG.put("transaction.state.log.replication.factor", (short) 1);
+        BROKER_CONFIG.put("transaction.state.log.min.isr", 1);
+    }
+    @ClassRule
+    public static final EmbeddedKafkaCluster CLUSTER =
+        new EmbeddedKafkaCluster(NUM_BROKERS, BROKER_CONFIG, 0L);
+
+    private static final StringDeserializer STRING_DESERIALIZER = new StringDeserializer();
+    private static final StringSerializer STRING_SERIALIZER = new StringSerializer();
+    private static final Serde STRING_SERDE = Serdes.String();
+    private static final int DEFAULT_TIMEOUT = 100;
+    private final boolean eosEnabled;
+    private static long lastRecordedTimestamp = -2L;
+
+    @Parameters(name = "{index}: eosEnabled={0}")
+    public static Collection parameters() {
+        return asList(
+            new Object[] {false},
+            new Object[] {true}
+        );
+    }
+
+    public ResetPartitionTimeIntegrationTest(final boolean eosEnabled) {
+        this.eosEnabled = eosEnabled;
+    }
+
+    @Test
+    public void shouldPreservePartitionTimeOnKafkaStreamRestart() throws Exception {
+        final String appId = "appId";
+        final String input = "input";
+        final String outputRaw = "output-raw";
+
+        cleanStateBeforeTest(CLUSTER, 2, input, outputRaw);
+
+        final StreamsBuilder builder = new StreamsBuilder();
+        builder.stream(
+                input,
+                Consumed.with(STRING_SERDE, STRING_SERDE))
+               .to(outputRaw);
+
+        final Properties streamsConfig = new Properties();
+        streamsConfig.put(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, MaxTimestampExtractor.class);
+        streamsConfig.put(StreamsConfig.APPLICATION_ID_CONFIG, appId);
+        streamsConfig.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
+        streamsConfig.put(StreamsConfig.POLL_MS_CONFIG, Integer.toString(DEFAULT_TIMEOUT));
+        streamsConfig.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, Integer.toString(DEFAULT_TIMEOUT));
+        streamsConfig.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, eosEnabled ? EXACTLY_ONCE : AT_LEAST_ONCE);
+        streamsConfig.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath());
+
+        KafkaStreams kafkaStreams = getStartedStreams(streamsConfig, builder, true);
+        try {
+            // start sending some records to have partition time committed 
+            produceSynchronouslyToPartitionZero(
+                input,
+                asList(
+                    new KeyValueTimestamp<>("k3", "v3", 5000)
+                )
+            );
+            verifyOutput(
+                outputRaw,
+                asList(
+                    new KeyValueTimestamp<>("k3", "v3", 5000)
+                )
+            );
+            assertThat(lastRecordedTimestamp, is(-1L));
+            lastRecordedTimestamp = -2L;
+
+            kafkaStreams.close();
+            assertThat(kafkaStreams.state(), is(KafkaStreams.State.NOT_RUNNING));
+
+            kafkaStreams = getStartedStreams(streamsConfig, builder, true);
+
+            // resend some records and retrieve the last committed timestamp
+            produceSynchronouslyToPartitionZero(
+                input,
+                asList(
+                    new KeyValueTimestamp<>("k5", "v5", 4999)
+                )
+            );
+            verifyOutput(
+                outputRaw,
+                asList(
+                    new KeyValueTimestamp<>("k5", "v5", 4999)
+                )
+            );
+            assertThat(lastRecordedTimestamp, is(5000L));
+        } finally {
+            kafkaStreams.close();
+            cleanStateAfterTest(CLUSTER, kafkaStreams);
+        }
+    }
+
+    public static final class MaxTimestampExtractor implements TimestampExtractor {
+        @Override
+        public long extract(final ConsumerRecord record, final long partitionTime) {
+            lastRecordedTimestamp = partitionTime;
+            return record.timestamp();
+        }
+    }
+
+    private void verifyOutput(final String topic, final List> keyValueTimestamps) {
+        final Properties properties = mkProperties(
+            mkMap(
+                mkEntry(ConsumerConfig.GROUP_ID_CONFIG, "test-group"),
+                mkEntry(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()),
+                mkEntry(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ((Deserializer) STRING_DESERIALIZER).getClass().getName()),
+                mkEntry(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ((Deserializer) STRING_DESERIALIZER).getClass().getName())
+            )
+        );
+        IntegrationTestUtils.verifyKeyValueTimestamps(properties, topic, keyValueTimestamps);
+    }
+
+    private static void produceSynchronouslyToPartitionZero(final String topic, final List> toProduce) {
+        final Properties producerConfig = mkProperties(mkMap(
+            mkEntry(ProducerConfig.CLIENT_ID_CONFIG, "anything"),
+            mkEntry(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ((Serializer) STRING_SERIALIZER).getClass().getName()),
+            mkEntry(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ((Serializer) STRING_SERIALIZER).getClass().getName()),
+            mkEntry(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers())
+        ));
+        IntegrationTestUtils.produceSynchronously(producerConfig, false, topic, Optional.of(0), toProduce);
+    }
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractTaskTest.java
index 0a79e8459afd3..a8526bf82067e 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractTaskTest.java
@@ -217,6 +217,9 @@ private AbstractTask createTask(final Consumer consumer,
                                 stateDirectory,
                                 config) {
 
+            @Override
+            public void initializeTaskTime() {}
+
             @Override
             public void resume() {}
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasksTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasksTest.java
index a00969debd3d9..0aaeef9fde178 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasksTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasksTest.java
@@ -84,16 +84,20 @@ public void shouldInitializeNewTasks() {
     @Test
     public void shouldMoveInitializedTasksNeedingRestoreToRestoring() {
         EasyMock.expect(t1.initializeStateStores()).andReturn(false);
+        t1.initializeTaskTime();
         t1.initializeTopology();
         EasyMock.expectLastCall().once();
         EasyMock.expect(t1.partitions()).andReturn(Collections.singleton(tp1));
         EasyMock.expect(t1.changelogPartitions()).andReturn(Collections.emptySet());
         EasyMock.expect(t2.initializeStateStores()).andReturn(true);
+        t1.initializeTaskTime();
         t2.initializeTopology();
         EasyMock.expectLastCall().once();
         final Set t2partitions = Collections.singleton(tp2);
         EasyMock.expect(t2.partitions()).andReturn(t2partitions);
         EasyMock.expect(t2.changelogPartitions()).andReturn(Collections.emptyList());
+        t1.initializeTaskTime();
+        t2.initializeTaskTime();
 
         EasyMock.replay(t1, t2);
 
@@ -110,6 +114,7 @@ public void shouldMoveInitializedTasksNeedingRestoreToRestoring() {
     @Test
     public void shouldMoveInitializedTasksThatDontNeedRestoringToRunning() {
         EasyMock.expect(t2.initializeStateStores()).andReturn(true);
+        t2.initializeTaskTime();
         t2.initializeTopology();
         EasyMock.expectLastCall().once();
         EasyMock.expect(t2.partitions()).andReturn(Collections.singleton(tp2));
@@ -130,6 +135,7 @@ public void shouldTransitionFullyRestoredTasksToRunning() {
         EasyMock.expect(t1.partitions()).andReturn(task1Partitions).anyTimes();
         EasyMock.expect(t1.changelogPartitions()).andReturn(Utils.mkSet(changeLog1, changeLog2)).anyTimes();
         EasyMock.expect(t1.hasStateStores()).andReturn(true).anyTimes();
+        t1.initializeTaskTime();
         t1.initializeTopology();
         EasyMock.expectLastCall().once();
         EasyMock.replay(t1);
@@ -224,6 +230,7 @@ public void shouldResumeMatchingSuspendedTasks() {
         mockRunningTaskSuspension();
         t1.resume();
         EasyMock.expectLastCall();
+        t1.initializeTaskTime();
         t1.initializeTopology();
         EasyMock.expectLastCall().once();
         EasyMock.replay(t1);
@@ -239,6 +246,7 @@ public void shouldResumeMatchingSuspendedTasks() {
     public void shouldCloseTaskOnResumeSuspendedIfTaskMigratedException() {
         mockRunningTaskSuspension();
         t1.resume();
+        t1.initializeTaskTime();
         t1.initializeTopology();
         EasyMock.expectLastCall().andThrow(new TaskMigratedException());
         t1.close(false, true);
@@ -258,6 +266,7 @@ public void shouldCloseTaskOnResumeSuspendedIfTaskMigratedException() {
 
     private void mockTaskInitialization() {
         EasyMock.expect(t1.initializeStateStores()).andReturn(true);
+        t1.initializeTaskTime();
         t1.initializeTopology();
         EasyMock.expectLastCall().once();
         EasyMock.expect(t1.partitions()).andReturn(Collections.singleton(tp1));
@@ -537,6 +546,7 @@ private RuntimeException suspendTask() {
 
     private void mockRunningTaskSuspension() {
         EasyMock.expect(t1.initializeStateStores()).andReturn(true);
+        t1.initializeTaskTime();
         t1.initializeTopology();
         EasyMock.expectLastCall().once();
         EasyMock.expect(t1.hasStateStores()).andReturn(false).anyTimes();
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/PartitionGroupTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/PartitionGroupTest.java
index cfc814f2e42c5..3584f9c9a30d1 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/PartitionGroupTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/PartitionGroupTest.java
@@ -40,12 +40,15 @@
 import static org.apache.kafka.common.utils.Utils.mkEntry;
 import static org.apache.kafka.common.utils.Utils.mkMap;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
 
 public class PartitionGroupTest {
     private final LogContext logContext = new LogContext();
     private final Serializer intSerializer = new IntegerSerializer();
     private final Deserializer intDeserializer = new IntegerDeserializer();
     private final TimestampExtractor timestampExtractor = new MockTimestampExtractor();
+    private final TopicPartition randomPartition = new TopicPartition("random-partition", 0);
+    private final String errMessage = "Partition " + randomPartition + " not found.";
     private final String[] topics = {"topic"};
     private final TopicPartition partition1 = new TopicPartition(topics[0], 1);
     private final TopicPartition partition2 = new TopicPartition(topics[0], 2);
@@ -86,7 +89,6 @@ private static Sensor getValueSensor(final Metrics metrics, final MetricName met
     @Test
     public void testTimeTracking() {
         assertEquals(0, group.numBuffered());
-
         // add three 3 records with timestamp 1, 3, 5 to partition-1
         final List> list1 = Arrays.asList(
             new ConsumerRecord<>("topic", 1, 1L, recordKey, recordValue),
@@ -119,6 +121,9 @@ record = group.nextRecord(info);
         // 2:[2, 4, 6]
         // st: 1
         assertEquals(partition1, info.partition());
+        assertEquals(3L, group.partitionTimestamp(partition1));
+        assertEquals(2L, group.partitionTimestamp(partition2));
+        assertEquals(1L, group.streamTime());
         verifyTimes(record, 1L, 1L);
         verifyBuffered(5, 2, 3);
         assertEquals(0.0, metrics.metric(lastLatenessValue).metricValue());
@@ -129,6 +134,9 @@ record = group.nextRecord(info);
         // 2:[4, 6]
         // st: 2
         assertEquals(partition2, info.partition());
+        assertEquals(3L, group.partitionTimestamp(partition1));
+        assertEquals(4L, group.partitionTimestamp(partition2));
+        assertEquals(2L, group.streamTime());
         verifyTimes(record, 2L, 2L);
         verifyBuffered(4, 2, 2);
         assertEquals(0.0, metrics.metric(lastLatenessValue).metricValue());
@@ -143,6 +151,8 @@ record = group.nextRecord(info);
         // 2:[4, 6]
         // st: 2 (just adding records shouldn't change it)
         verifyBuffered(6, 4, 2);
+        assertEquals(3L, group.partitionTimestamp(partition1));
+        assertEquals(4L, group.partitionTimestamp(partition2));
         assertEquals(2L, group.streamTime());
         assertEquals(0.0, metrics.metric(lastLatenessValue).metricValue());
 
@@ -152,6 +162,9 @@ record = group.nextRecord(info);
         // 2:[4, 6]
         // st: 3
         assertEquals(partition1, info.partition());
+        assertEquals(5L, group.partitionTimestamp(partition1));
+        assertEquals(4L, group.partitionTimestamp(partition2));
+        assertEquals(3L, group.streamTime());
         verifyTimes(record, 3L, 3L);
         verifyBuffered(5, 3, 2);
         assertEquals(0.0, metrics.metric(lastLatenessValue).metricValue());
@@ -162,6 +175,9 @@ record = group.nextRecord(info);
         // 2:[6]
         // st: 4
         assertEquals(partition2, info.partition());
+        assertEquals(5L, group.partitionTimestamp(partition1));
+        assertEquals(6L, group.partitionTimestamp(partition2));
+        assertEquals(4L, group.streamTime());
         verifyTimes(record, 4L, 4L);
         verifyBuffered(4, 3, 1);
         assertEquals(0.0, metrics.metric(lastLatenessValue).metricValue());
@@ -172,6 +188,9 @@ record = group.nextRecord(info);
         // 2:[6]
         // st: 5
         assertEquals(partition1, info.partition());
+        assertEquals(5L, group.partitionTimestamp(partition1));
+        assertEquals(6L, group.partitionTimestamp(partition2));
+        assertEquals(5L, group.streamTime());
         verifyTimes(record, 5L, 5L);
         verifyBuffered(3, 2, 1);
         assertEquals(0.0, metrics.metric(lastLatenessValue).metricValue());
@@ -182,6 +201,9 @@ record = group.nextRecord(info);
         // 2:[6]
         // st: 5
         assertEquals(partition1, info.partition());
+        assertEquals(5L, group.partitionTimestamp(partition1));
+        assertEquals(6L, group.partitionTimestamp(partition2));
+        assertEquals(5L, group.streamTime());
         verifyTimes(record, 2L, 5L);
         verifyBuffered(2, 1, 1);
         assertEquals(3.0, metrics.metric(lastLatenessValue).metricValue());
@@ -192,6 +214,9 @@ record = group.nextRecord(info);
         // 2:[6]
         // st: 5
         assertEquals(partition1, info.partition());
+        assertEquals(5L, group.partitionTimestamp(partition1));
+        assertEquals(6L, group.partitionTimestamp(partition2));
+        assertEquals(5L, group.streamTime());
         verifyTimes(record, 4L, 5L);
         verifyBuffered(1, 0, 1);
         assertEquals(1.0, metrics.metric(lastLatenessValue).metricValue());
@@ -202,10 +227,12 @@ record = group.nextRecord(info);
         // 2:[]
         // st: 6
         assertEquals(partition2, info.partition());
+        assertEquals(5L, group.partitionTimestamp(partition1));
+        assertEquals(6L, group.partitionTimestamp(partition2));
+        assertEquals(6L, group.streamTime());
         verifyTimes(record, 6L, 6L);
         verifyBuffered(0, 0, 0);
         assertEquals(0.0, metrics.metric(lastLatenessValue).metricValue());
-
     }
 
     @Test
@@ -266,4 +293,28 @@ private void verifyBuffered(final int totalBuffered, final int partitionOneBuffe
         assertEquals(partitionOneBuffered, group.numBuffered(partition1));
         assertEquals(partitionTwoBuffered, group.numBuffered(partition2));
     }
+
+    @Test
+    public void shouldSetPartitionTimestampAndStreamTime() {
+        group.setPartitionTime(partition1, 100L);
+        assertEquals(100L, group.partitionTimestamp(partition1));
+        assertEquals(100L, group.streamTime());
+        group.setPartitionTime(partition2, 50L);
+        assertEquals(50L, group.partitionTimestamp(partition2));
+        assertEquals(100L, group.streamTime());
+    }
+
+    @Test
+    public void shouldThrowNullpointerUponSetPartitionTimestampFailure() {
+        assertThrows(errMessage, NullPointerException.class, () -> {
+            group.setPartitionTime(randomPartition, 0L);
+        });
+    }
+
+    @Test
+    public void shouldThrowNullpointerUponGetPartitionTimestampFailure() {
+        assertThrows(errMessage, NullPointerException.class, () -> {
+            group.partitionTimestamp(randomPartition);
+        });
+    }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/RecordQueueTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RecordQueueTest.java
index 6dadb49b8ab82..c5f1a76f41309 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/RecordQueueTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RecordQueueTest.java
@@ -209,6 +209,33 @@ public void shouldTrackPartitionTimeAsMaxSeenTimestamp() {
         assertEquals(queue.partitionTime(), 3L);
     }
 
+    @Test
+    public void shouldSetTimestampAndRespectMaxTimestampPolicy() {
+        assertTrue(queue.isEmpty());
+        assertEquals(0, queue.size());
+        assertEquals(RecordQueue.UNKNOWN, queue.headRecordTimestamp());
+        queue.setPartitionTime(150L);
+
+        final List> list1 = Arrays.asList(
+            new ConsumerRecord<>("topic", 1, 200, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, recordKey, recordValue),
+            new ConsumerRecord<>("topic", 1, 100, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, recordKey, recordValue),
+            new ConsumerRecord<>("topic", 1, 300, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, recordKey, recordValue),
+            new ConsumerRecord<>("topic", 1, 400, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, recordKey, recordValue));
+
+        assertEquals(150L, queue.partitionTime());
+
+        queue.addRawRecords(list1);
+
+        assertEquals(200L, queue.partitionTime());
+
+        queue.setPartitionTime(500L);
+        queue.poll();
+        assertEquals(500L, queue.partitionTime());
+
+        queue.poll();
+        assertEquals(500L, queue.partitionTime());
+    }
+
     @Test(expected = StreamsException.class)
     public void shouldThrowStreamsExceptionWhenKeyDeserializationFails() {
         final byte[] key = Serdes.Long().serializer().serialize("foo", 1L);
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
index cc0af23eed37d..e90151f282270 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
@@ -70,7 +70,9 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.time.Duration;
+import java.util.Base64;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -152,6 +154,9 @@ public Map restoredOffsets() {
     private StreamTask task;
     private long punctuatedAt;
 
+    private static final String APPLICATION_ID = "stream-task-test";
+    private static final long DEFAULT_TIMESTAMP = 1000;
+
     private final Punctuator punctuator = new Punctuator() {
         @Override
         public void punctuate(final long timestamp) {
@@ -191,7 +196,7 @@ static StreamsConfig createConfig(final boolean enableEoS) {
             throw new RuntimeException(e);
         }
         return new StreamsConfig(mkProperties(mkMap(
-            mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, "stream-task-test"),
+            mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, APPLICATION_ID),
             mkEntry(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:2171"),
             mkEntry(StreamsConfig.BUFFERED_RECORDS_PER_PARTITION_CONFIG, "3"),
             mkEntry(StreamsConfig.STATE_DIR_CONFIG, canonicalPath),
@@ -648,6 +653,78 @@ public void shouldRespectCommitNeeded() {
         assertFalse(task.commitNeeded());
     }
 
+    @Test
+    public void shouldRestorePartitionTimeAfterRestartWithEosDisabled() {
+        createTaskWithProcessAndCommit(false);
+
+        assertEquals(DEFAULT_TIMESTAMP, task.decodeTimestamp(consumer.committed(partition1).metadata()));
+        // reset times here by creating a new task
+        task = createStatelessTask(createConfig(false));
+
+        task.initializeTaskTime();
+        assertEquals(DEFAULT_TIMESTAMP, task.partitionTime(partition1));
+        assertEquals(DEFAULT_TIMESTAMP, task.streamTime());
+    }
+
+    @Test
+    public void shouldRestorePartitionTimeAfterRestartWithEosEnabled() {
+        createTaskWithProcessAndCommit(true);
+
+        // extract the committed metadata from MockProducer
+        final List>> metadataList = 
+            producer.consumerGroupOffsetsHistory();
+        final String storedMetadata = metadataList.get(0).get(APPLICATION_ID).get(partition1).metadata();
+        final long partitionTime = task.decodeTimestamp(storedMetadata);
+        assertEquals(DEFAULT_TIMESTAMP, partitionTime);
+
+        // since producer and consumer is mocked, we need to "connect" producer and consumer
+        // so we should manually commit offsets here to simulate this "connection"
+        final Map offsetMap = new HashMap<>();
+        final String encryptedMetadata = task.encodeTimestamp(partitionTime);
+        offsetMap.put(partition1, new OffsetAndMetadata(partitionTime, encryptedMetadata));
+        consumer.commitSync(offsetMap);
+
+        // reset times here by creating a new task
+        task = createStatelessTask(createConfig(true));
+
+        task.initializeTaskTime();
+        assertEquals(DEFAULT_TIMESTAMP, task.partitionTime(partition1));
+        assertEquals(DEFAULT_TIMESTAMP, task.streamTime());
+    }
+
+    private void createTaskWithProcessAndCommit(final boolean eosEnabled) {
+        task = createStatelessTask(createConfig(eosEnabled));
+        task.initializeStateStores();
+        task.initializeTopology();
+
+        task.addRecords(partition1, singletonList(getConsumerRecord(partition1, DEFAULT_TIMESTAMP)));
+
+        task.process();
+        task.commit();
+    }
+
+    @Test
+    public void shouldEncodeAndDecodeMetadata() {
+        task = createStatelessTask(createConfig(false));
+        assertEquals(DEFAULT_TIMESTAMP, task.decodeTimestamp(task.encodeTimestamp(DEFAULT_TIMESTAMP)));
+    }
+
+    @Test
+    public void shouldReturnUnknownTimestampIfUnknownVersion() {
+        task = createStatelessTask(createConfig(false));
+
+        final byte[] emptyMessage = {StreamTask.LATEST_MAGIC_BYTE + 1};
+        final String encodedString = Base64.getEncoder().encodeToString(emptyMessage);
+        assertEquals(RecordQueue.UNKNOWN, task.decodeTimestamp(encodedString));
+    }
+
+    @Test
+    public void shouldReturnUnknownTimestampIfEmptyMessage() {
+        task = createStatelessTask(createConfig(false));
+
+        assertEquals(RecordQueue.UNKNOWN, task.decodeTimestamp(""));
+    }
+
     @Test
     public void shouldRespectCommitRequested() {
         task = createStatelessTask(createConfig(false));

From 3825ff3866325dcf0e9bff3e557eacb66450a856 Mon Sep 17 00:00:00 2001
From: Stanislav Kozlovski 
Date: Thu, 19 Sep 2019 22:42:14 +0100
Subject: [PATCH 0627/1071] MINOR: Send latest LeaderAndIsr version (#7351)

KIP-455 (18d4e57f6e8c67ffa7937fc855707d3a03cc165a) bumped the LeaderAndIsr version to 3 but did not change the Controller code to actually send the new version. The ControllerChannelManagerTest had a bug which made it assert wrongly, hence why it did not catch it. This patch fixes said test.
Because the new fields in LeaderAndIsr are not used yet, the gap was not caught by integration tests either.

Reviewers: Jason Gustafson 
---
 .../src/main/scala/kafka/api/ApiVersion.scala | 11 ++-
 .../controller/ControllerChannelManager.scala |  3 +-
 .../scala/unit/kafka/api/ApiVersionTest.scala | 22 ++++++
 .../ControllerChannelManagerTest.scala        | 72 ++++++++++---------
 4 files changed, 73 insertions(+), 35 deletions(-)

diff --git a/core/src/main/scala/kafka/api/ApiVersion.scala b/core/src/main/scala/kafka/api/ApiVersion.scala
index 8e8ca0b749302..1a3362508c100 100644
--- a/core/src/main/scala/kafka/api/ApiVersion.scala
+++ b/core/src/main/scala/kafka/api/ApiVersion.scala
@@ -90,7 +90,9 @@ object ApiVersion {
     // Introduced static membership.
     KAFKA_2_3_IV0,
     // Add rack_id to FetchRequest, preferred_read_replica to FetchResponse, and replica_id to OffsetsForLeaderRequest
-    KAFKA_2_3_IV1
+    KAFKA_2_3_IV1,
+    // Add adding_replicas and removing_replicas fields to LeaderAndIsrRequest
+    KAFKA_2_4_IV0
   )
 
   // Map keys are the union of the short and full versions
@@ -316,6 +318,13 @@ case object KAFKA_2_3_IV1 extends DefaultApiVersion {
   val id: Int = 23
 }
 
+case object KAFKA_2_4_IV0 extends DefaultApiVersion {
+  val shortVersion: String = "2.4"
+  val subVersion = "IV0"
+  val recordVersion = RecordVersion.V2
+  val id: Int = 24
+}
+
 object ApiVersionValidator extends Validator {
 
   override def ensureValid(name: String, value: Any): Unit = {
diff --git a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
index e91e48d80cf15..d4d37d2849917 100755
--- a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
+++ b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
@@ -443,7 +443,8 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
 
   private def sendLeaderAndIsrRequest(controllerEpoch: Int, stateChangeLog: StateChangeLogger): Unit = {
     val leaderAndIsrRequestVersion: Short =
-      if (config.interBrokerProtocolVersion >= KAFKA_2_2_IV0) 2
+      if (config.interBrokerProtocolVersion >= KAFKA_2_4_IV0) 3
+      else if (config.interBrokerProtocolVersion >= KAFKA_2_2_IV0) 2
       else if (config.interBrokerProtocolVersion >= KAFKA_1_0_IV0) 1
       else 0
 
diff --git a/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala b/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala
index 571a0775bd3a4..dadef1d00d5d7 100644
--- a/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala
+++ b/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala
@@ -88,6 +88,13 @@ class ApiVersionTest {
     assertEquals(KAFKA_2_2_IV1, ApiVersion("2.2"))
     assertEquals(KAFKA_2_2_IV0, ApiVersion("2.2-IV0"))
     assertEquals(KAFKA_2_2_IV1, ApiVersion("2.2-IV1"))
+
+    assertEquals(KAFKA_2_3_IV1, ApiVersion("2.3"))
+    assertEquals(KAFKA_2_3_IV0, ApiVersion("2.3-IV0"))
+    assertEquals(KAFKA_2_3_IV1, ApiVersion("2.3-IV1"))
+
+    assertEquals(KAFKA_2_4_IV0, ApiVersion("2.4"))
+    assertEquals(KAFKA_2_4_IV0, ApiVersion("2.4-IV0"))
   }
 
   @Test
@@ -116,7 +123,22 @@ class ApiVersionTest {
   def testShortVersion(): Unit = {
     assertEquals("0.8.0", KAFKA_0_8_0.shortVersion)
     assertEquals("0.10.0", KAFKA_0_10_0_IV0.shortVersion)
+    assertEquals("0.10.0", KAFKA_0_10_0_IV1.shortVersion)
     assertEquals("0.11.0", KAFKA_0_11_0_IV0.shortVersion)
+    assertEquals("0.11.0", KAFKA_0_11_0_IV1.shortVersion)
+    assertEquals("0.11.0", KAFKA_0_11_0_IV2.shortVersion)
+    assertEquals("1.0", KAFKA_1_0_IV0.shortVersion)
+    assertEquals("1.1", KAFKA_1_1_IV0.shortVersion)
+    assertEquals("2.0", KAFKA_2_0_IV0.shortVersion)
+    assertEquals("2.0", KAFKA_2_0_IV1.shortVersion)
+    assertEquals("2.1", KAFKA_2_1_IV0.shortVersion)
+    assertEquals("2.1", KAFKA_2_1_IV1.shortVersion)
+    assertEquals("2.1", KAFKA_2_1_IV2.shortVersion)
+    assertEquals("2.2", KAFKA_2_2_IV0.shortVersion)
+    assertEquals("2.2", KAFKA_2_2_IV1.shortVersion)
+    assertEquals("2.3", KAFKA_2_3_IV0.shortVersion)
+    assertEquals("2.3", KAFKA_2_3_IV1.shortVersion)
+    assertEquals("2.4", KAFKA_2_4_IV0.shortVersion)
   }
 
   @Test
diff --git a/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala b/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala
index 9c2941ae2fd33..172c45b30fdde 100644
--- a/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala
@@ -18,7 +18,7 @@ package kafka.controller
 
 import java.util.Properties
 
-import kafka.api.{ApiVersion, KAFKA_0_10_0_IV1, KAFKA_0_10_2_IV0, KAFKA_0_9_0, KAFKA_1_0_IV0, KAFKA_2_2_IV0, LeaderAndIsr}
+import kafka.api.{ApiVersion, KAFKA_0_10_0_IV1, KAFKA_0_10_2_IV0, KAFKA_0_9_0, KAFKA_1_0_IV0, KAFKA_2_2_IV0, KAFKA_2_4_IV0, LeaderAndIsr}
 import kafka.cluster.{Broker, EndPoint}
 import kafka.server.KafkaConfig
 import kafka.utils.TestUtils
@@ -149,8 +149,9 @@ class ControllerChannelManagerTest {
 
     for (apiVersion <- ApiVersion.allVersions) {
       val leaderAndIsrRequestVersion: Short =
-        if (config.interBrokerProtocolVersion >= KAFKA_2_2_IV0) 2
-        else if (config.interBrokerProtocolVersion >= KAFKA_1_0_IV0) 1
+        if (apiVersion >= KAFKA_2_4_IV0) 3
+        else if (apiVersion >= KAFKA_2_2_IV0) 2
+        else if (apiVersion >= KAFKA_1_0_IV0) 1
         else 0
 
       testLeaderAndIsrRequestFollowsInterBrokerProtocolVersion(apiVersion, leaderAndIsrRequestVersion)
@@ -173,9 +174,10 @@ class ControllerChannelManagerTest {
     batch.addLeaderAndIsrRequestForBrokers(Seq(2), partition, leaderIsrAndControllerEpoch, replicaAssignment(Seq(1, 2, 3)), isNew = false)
     batch.sendRequestsToBrokers(controllerEpoch)
 
-    val leaderAndIsrRequests = batch.collectLeaderAndIsrRequestsFor(2, expectedLeaderAndIsrVersion)
+    val leaderAndIsrRequests = batch.collectLeaderAndIsrRequestsFor(2)
     assertEquals(1, leaderAndIsrRequests.size)
-    assertEquals(expectedLeaderAndIsrVersion, leaderAndIsrRequests.head.version)
+    assertEquals(s"IBP $interBrokerProtocolVersion should use version $expectedLeaderAndIsrVersion",
+      expectedLeaderAndIsrVersion, leaderAndIsrRequests.head.version)
   }
 
   @Test
@@ -316,11 +318,11 @@ class ControllerChannelManagerTest {
 
     for (apiVersion <- ApiVersion.allVersions) {
       val updateMetadataRequestVersion: Short =
-        if (config.interBrokerProtocolVersion >= KAFKA_2_2_IV0) 5
-        else if (config.interBrokerProtocolVersion >= KAFKA_1_0_IV0) 4
-        else if (config.interBrokerProtocolVersion >= KAFKA_0_10_2_IV0) 3
-        else if (config.interBrokerProtocolVersion >= KAFKA_0_10_0_IV1) 2
-        else if (config.interBrokerProtocolVersion >= KAFKA_0_9_0) 1
+        if (apiVersion >= KAFKA_2_2_IV0) 5
+        else if (apiVersion >= KAFKA_1_0_IV0) 4
+        else if (apiVersion >= KAFKA_0_10_2_IV0) 3
+        else if (apiVersion >= KAFKA_0_10_0_IV1) 2
+        else if (apiVersion >= KAFKA_0_9_0) 1
         else 0
 
       testUpdateMetadataFollowsInterBrokerProtocolVersion(apiVersion, updateMetadataRequestVersion)
@@ -341,8 +343,11 @@ class ControllerChannelManagerTest {
     assertEquals(1, batch.sentRequests.size)
     assertTrue(batch.sentRequests.contains(2))
 
-    val requests = batch.collectUpdateMetadataRequestsFor(2, expectedUpdateMetadataVersion)
-    assertTrue(requests.forall(_.version == expectedUpdateMetadataVersion))
+    val requests = batch.collectUpdateMetadataRequestsFor(2)
+    val allVersions = requests.map(_.version)
+    assertTrue(s"IBP $interBrokerProtocolVersion should use version $expectedUpdateMetadataVersion, " +
+      s"but found versions $allVersions",
+      allVersions.forall(_ == expectedUpdateMetadataVersion))
   }
 
   @Test
@@ -369,7 +374,7 @@ class ControllerChannelManagerTest {
     val sentRequests = batch.sentRequests(2)
     assertEquals(1, sentRequests.size)
 
-    val sentStopReplicaRequests = batch.collectStopReplicRequestsFor(2)
+    val sentStopReplicaRequests = batch.collectStopReplicaRequestsFor(2)
     assertEquals(1, sentStopReplicaRequests.size)
 
     val stopReplicaRequest = sentStopReplicaRequests.head
@@ -407,7 +412,7 @@ class ControllerChannelManagerTest {
     val sentRequests = batch.sentRequests(2)
     assertEquals(1, sentRequests.size)
 
-    val sentStopReplicaRequests = batch.collectStopReplicRequestsFor(2)
+    val sentStopReplicaRequests = batch.collectStopReplicaRequestsFor(2)
     assertEquals(1, sentStopReplicaRequests.size)
     assertEquals(partitions, sentStopReplicaRequests.flatMap(_.partitions.asScala).toSet)
     assertTrue(sentStopReplicaRequests.forall(_.deletePartitions()))
@@ -444,7 +449,7 @@ class ControllerChannelManagerTest {
     val sentRequests = batch.sentRequests(2)
     assertEquals(1, sentRequests.size)
 
-    val sentStopReplicaRequests = batch.collectStopReplicRequestsFor(2)
+    val sentStopReplicaRequests = batch.collectStopReplicaRequestsFor(2)
     assertEquals(1, sentStopReplicaRequests.size)
     assertEquals(partitions, sentStopReplicaRequests.flatMap(_.partitions.asScala).toSet)
     assertTrue(sentStopReplicaRequests.forall(_.deletePartitions()))
@@ -493,7 +498,7 @@ class ControllerChannelManagerTest {
     val sentRequests = batch.sentRequests(2)
     assertEquals(2, sentRequests.size)
 
-    val sentStopReplicaRequests = batch.collectStopReplicRequestsFor(2)
+    val sentStopReplicaRequests = batch.collectStopReplicaRequestsFor(2)
     assertEquals(2, sentStopReplicaRequests.size)
 
     val (deleteRequests, nonDeleteRequests) = sentStopReplicaRequests.partition(_.deletePartitions())
@@ -529,7 +534,7 @@ class ControllerChannelManagerTest {
     assertEquals(1, sentRequests.size)
 
     for (brokerId <- Set(2, 3)) {
-      val sentStopReplicaRequests = batch.collectStopReplicRequestsFor(brokerId)
+      val sentStopReplicaRequests = batch.collectStopReplicaRequestsFor(brokerId)
       assertEquals(1, sentStopReplicaRequests.size)
 
       val stopReplicaRequest = sentStopReplicaRequests.head
@@ -569,7 +574,7 @@ class ControllerChannelManagerTest {
     val sentRequests = batch.sentRequests(2)
     assertEquals(1, sentRequests.size)
 
-    val sentStopReplicaRequests = batch.collectStopReplicRequestsFor(2)
+    val sentStopReplicaRequests = batch.collectStopReplicaRequestsFor(2)
     assertEquals(1, sentStopReplicaRequests.size)
 
     val stopReplicaRequest = sentStopReplicaRequests.head
@@ -583,14 +588,14 @@ class ControllerChannelManagerTest {
 
     for (apiVersion <- ApiVersion.allVersions) {
       if (apiVersion < KAFKA_2_2_IV0)
-        testStopReplicaFollowsInterBrokerProtocolVersion(ApiVersion.latestVersion, 0.toShort)
+        testStopReplicaFollowsInterBrokerProtocolVersion(apiVersion, 0.toShort)
       else
-        testStopReplicaFollowsInterBrokerProtocolVersion(ApiVersion.latestVersion, 1.toShort)
+        testStopReplicaFollowsInterBrokerProtocolVersion(apiVersion, 1.toShort)
     }
   }
 
   private def testStopReplicaFollowsInterBrokerProtocolVersion(interBrokerProtocolVersion: ApiVersion,
-                                                       expectedStopReplicaRequestVersion: Short): Unit = {
+                                                               expectedStopReplicaRequestVersion: Short): Unit = {
     val context = initContext(Seq(1, 2, 3), Set("foo"), 2, 3)
     val config = createConfig(interBrokerProtocolVersion)
     val batch = new MockControllerBrokerRequestBatch(context, config)
@@ -605,8 +610,11 @@ class ControllerChannelManagerTest {
     assertEquals(1, batch.sentRequests.size)
     assertTrue(batch.sentRequests.contains(2))
 
-    val requests = batch.collectStopReplicRequestsFor(2, expectedStopReplicaRequestVersion)
-    assertTrue(requests.forall(_.version() == expectedStopReplicaRequestVersion))
+    val requests = batch.collectStopReplicaRequestsFor(2)
+    val allVersions = requests.map(_.version)
+    assertTrue(s"IBP $interBrokerProtocolVersion should use version $expectedStopReplicaRequestVersion, " +
+      s"but found versions $allVersions",
+      allVersions.forall(_ == expectedStopReplicaRequestVersion))
   }
 
   private def applyStopReplicaResponseCallbacks(error: Errors, sentRequests: List[SentRequest]): Unit = {
@@ -631,7 +639,8 @@ class ControllerChannelManagerTest {
     val props = new Properties()
     props.put(KafkaConfig.BrokerIdProp, controllerId.toString)
     props.put(KafkaConfig.ZkConnectProp, "zkConnect")
-    props.put(KafkaConfig.InterBrokerProtocolVersionProp, ApiVersion.latestVersion.version)
+    props.put(KafkaConfig.InterBrokerProtocolVersionProp, interBrokerVersion.version)
+    props.put(KafkaConfig.LogMessageFormatVersionProp, interBrokerVersion.version)
     KafkaConfig.fromProps(props)
   }
 
@@ -680,32 +689,29 @@ class ControllerChannelManagerTest {
       sentRequests(brokerId).append(SentRequest(request, callback))
     }
 
-    def collectStopReplicRequestsFor(brokerId: Int,
-                                     version: Short = ApiKeys.STOP_REPLICA.latestVersion): List[StopReplicaRequest] = {
+    def collectStopReplicaRequestsFor(brokerId: Int): List[StopReplicaRequest] = {
       sentRequests.get(brokerId) match {
         case Some(requests) => requests
           .filter(_.request.apiKey == ApiKeys.STOP_REPLICA)
-          .map(_.request.build(version).asInstanceOf[StopReplicaRequest]).toList
+          .map(_.request.build().asInstanceOf[StopReplicaRequest]).toList
         case None => List.empty[StopReplicaRequest]
       }
     }
 
-    def collectUpdateMetadataRequestsFor(brokerId: Int,
-                                         version: Short = ApiKeys.UPDATE_METADATA.latestVersion): List[UpdateMetadataRequest] = {
+    def collectUpdateMetadataRequestsFor(brokerId: Int): List[UpdateMetadataRequest] = {
       sentRequests.get(brokerId) match {
         case Some(requests) => requests
           .filter(_.request.apiKey == ApiKeys.UPDATE_METADATA)
-          .map(_.request.build(version).asInstanceOf[UpdateMetadataRequest]).toList
+          .map(_.request.build().asInstanceOf[UpdateMetadataRequest]).toList
         case None => List.empty[UpdateMetadataRequest]
       }
     }
 
-    def collectLeaderAndIsrRequestsFor(brokerId: Int,
-                                       version: Short = ApiKeys.LEADER_AND_ISR.latestVersion): List[LeaderAndIsrRequest] = {
+    def collectLeaderAndIsrRequestsFor(brokerId: Int): List[LeaderAndIsrRequest] = {
       sentRequests.get(brokerId) match {
         case Some(requests) => requests
           .filter(_.request.apiKey == ApiKeys.LEADER_AND_ISR)
-          .map(_.request.build(version).asInstanceOf[LeaderAndIsrRequest]).toList
+          .map(_.request.build().asInstanceOf[LeaderAndIsrRequest]).toList
         case None => List.empty[LeaderAndIsrRequest]
       }
     }

From 8aff871b0da5604fe981561467dff7359fb56942 Mon Sep 17 00:00:00 2001
From: Stanislav Kozlovski 
Date: Thu, 19 Sep 2019 22:50:30 +0100
Subject: [PATCH 0628/1071] MINOR: Fix bug where we would incorrectly load
 partition reassignment info from ZK (#7334)

Reviewers: Ismael Juma , Colin P. McCabe 
---
 core/src/main/scala/kafka/zk/ZkData.scala     |  2 +-
 .../admin/ReassignPartitionsClusterTest.scala |  1 +
 .../unit/kafka/zk/KafkaZkClientTest.scala     | 21 +++++++++++++++++++
 3 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/kafka/zk/ZkData.scala b/core/src/main/scala/kafka/zk/ZkData.scala
index 43769ad7deb61..75d169d0e2f5d 100644
--- a/core/src/main/scala/kafka/zk/ZkData.scala
+++ b/core/src/main/scala/kafka/zk/ZkData.scala
@@ -281,7 +281,7 @@ object TopicZNode {
           new TopicPartition(topic, partition.toInt) -> PartitionReplicaAssignment(
             replicas.to[Seq[Int]],
             getReplicas(addingReplicasJsonOpt, partition),
-            getReplicas(addingReplicasJsonOpt, partition)
+            getReplicas(removingReplicasJsonOpt, partition)
           )
         }
       }
diff --git a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
index 0c2bdd1b7a7c5..10ed6bb98ad4c 100644
--- a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
@@ -684,6 +684,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     assertEquals(Seq(2, 1), zkClient.getReplicasForPartition(new TopicPartition("orders", 0)))
     assertEquals(Seq(1, 2), zkClient.getReplicasForPartition(new TopicPartition("orders", 1)))
     assertEquals(Seq(1, 2), zkClient.getReplicasForPartition(sameMoveTp))
+    assertEquals(Seq(1, 2), zkClient.getReplicasForPartition(new TopicPartition("orders", 3)))
     assertEquals(Seq.empty, zkClient.getReplicasForPartition(new TopicPartition("customers", 0)))
   }
 
diff --git a/core/src/test/scala/unit/kafka/zk/KafkaZkClientTest.scala b/core/src/test/scala/unit/kafka/zk/KafkaZkClientTest.scala
index f9961d886bc2a..135092dd8fa0c 100644
--- a/core/src/test/scala/unit/kafka/zk/KafkaZkClientTest.scala
+++ b/core/src/test/scala/unit/kafka/zk/KafkaZkClientTest.scala
@@ -811,6 +811,27 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
       expectedSuccessfulPartitions, successfulPartitions)
   }
 
+  @Test
+  def testTopicAssignments(): Unit = {
+    assertEquals(0, zkClient.getPartitionAssignmentForTopics(Set(topicPartition.topic())).size)
+    zkClient.createTopicAssignment(topicPartition.topic(),
+      Map(topicPartition -> Seq()))
+
+    val expectedAssignment = PartitionReplicaAssignment(Seq(1,2,3), Seq(1), Seq(3))
+    val response = zkClient.setTopicAssignmentRaw(topicPartition.topic(),
+      Map(topicPartition -> expectedAssignment), controllerEpochZkVersion)
+    assertEquals(Code.OK, response.resultCode)
+
+    val topicPartitionAssignments = zkClient.getPartitionAssignmentForTopics(Set(topicPartition.topic()))
+    assertEquals(1, topicPartitionAssignments.size)
+    assertTrue(topicPartitionAssignments.contains(topicPartition.topic()))
+    val partitionAssignments = topicPartitionAssignments(topicPartition.topic())
+    assertEquals(1, partitionAssignments.size)
+    assertTrue(partitionAssignments.contains(topicPartition.partition()))
+    val assignment = partitionAssignments(topicPartition.partition())
+    assertEquals(expectedAssignment, assignment)
+  }
+
   @Test
   def testUpdateLeaderAndIsr(): Unit = {
     zkClient.createRecursive(TopicZNode.path(topic1))

From 2d0cd2ef54029cfb592ab5762f16214ef5d0101f Mon Sep 17 00:00:00 2001
From: Adam Bellemare 
Date: Thu, 19 Sep 2019 15:36:32 -0700
Subject: [PATCH 0629/1071] MINOR: Murmur3 Hash with Guava dependency

Part of supporting KIP-213 ( https://cwiki.apache.org/confluence/display/KAFKA/KIP-213+Support+non-key+joining+in+KTable ). Murmur3 hash is used as a hashing mechanism in KIP-213 for the large range of uniqueness. The Murmur3 class and tests are ported directly from Apache Hive, with no alterations to the code or dependencies.

Author: Adam Bellemare 

Reviewers: John Roesler , Ismael Juma , Guozhang Wang 

Closes #7271 from bellemare/murmur3hash
---
 checkstyle/suppressions.xml                   |   6 +
 .../apache/kafka/common/utils/Murmur3.java    | 548 ++++++++++++++++++
 .../kafka/common/utils/Murmur3Test.java       |  70 +++
 gradle/spotbugs-exclude.xml                   |  19 +
 4 files changed, 643 insertions(+)
 create mode 100644 clients/src/main/java/org/apache/kafka/common/utils/Murmur3.java
 create mode 100644 clients/src/test/java/org/apache/kafka/common/utils/Murmur3Test.java

diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index 63eedae14e3c1..000dc50b02ac8 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -60,6 +60,9 @@
     
 
+    
+
     
     
@@ -76,6 +79,9 @@
     
 
+    
+
     
     
diff --git a/clients/src/main/java/org/apache/kafka/common/utils/Murmur3.java b/clients/src/main/java/org/apache/kafka/common/utils/Murmur3.java
new file mode 100644
index 0000000000000..a2c780279974a
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/utils/Murmur3.java
@@ -0,0 +1,548 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.utils;
+
+/**
+ * This class was taken from Hive org.apache.hive.common.util;
+ * https://github.com/apache/hive/blob/master/storage-api/src/java/org/apache/hive/common/util/Murmur3.java
+ * Commit: dffa3a16588bc8e95b9d0ab5af295a74e06ef702
+ *
+ *
+ * Murmur3 is successor to Murmur2 fast non-crytographic hash algorithms.
+ *
+ * Murmur3 32 and 128 bit variants.
+ * 32-bit Java port of https://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp#94
+ * 128-bit Java port of https://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp#255
+ *
+ * This is a public domain code with no copyrights.
+ * From homepage of MurmurHash (https://code.google.com/p/smhasher/),
+ * "All MurmurHash versions are public domain software, and the author disclaims all copyright
+ * to their code."
+ */
+@SuppressWarnings("fallthrough")
+public class Murmur3 {
+    // from 64-bit linear congruential generator
+    public static final long NULL_HASHCODE = 2862933555777941757L;
+
+    // Constants for 32 bit variant
+    private static final int C1_32 = 0xcc9e2d51;
+    private static final int C2_32 = 0x1b873593;
+    private static final int R1_32 = 15;
+    private static final int R2_32 = 13;
+    private static final int M_32 = 5;
+    private static final int N_32 = 0xe6546b64;
+
+    // Constants for 128 bit variant
+    private static final long C1 = 0x87c37b91114253d5L;
+    private static final long C2 = 0x4cf5ad432745937fL;
+    private static final int R1 = 31;
+    private static final int R2 = 27;
+    private static final int R3 = 33;
+    private static final int M = 5;
+    private static final int N1 = 0x52dce729;
+    private static final int N2 = 0x38495ab5;
+
+    public static final int DEFAULT_SEED = 104729;
+
+    public static int hash32(long l0, long l1) {
+        return hash32(l0, l1, DEFAULT_SEED);
+    }
+
+    public static int hash32(long l0) {
+        return hash32(l0, DEFAULT_SEED);
+    }
+
+    /**
+     * Murmur3 32-bit variant.
+     */
+    public static int hash32(long l0, int seed) {
+        int hash = seed;
+        final long r0 = Long.reverseBytes(l0);
+
+        hash = mix32((int) r0, hash);
+        hash = mix32((int) (r0 >>> 32), hash);
+
+        return fmix32(Long.BYTES, hash);
+    }
+
+    /**
+     * Murmur3 32-bit variant.
+     */
+    public static int hash32(long l0, long l1, int seed) {
+        int hash = seed;
+        final long r0 = Long.reverseBytes(l0);
+        final long r1 = Long.reverseBytes(l1);
+
+        hash = mix32((int) r0, hash);
+        hash = mix32((int) (r0 >>> 32), hash);
+        hash = mix32((int) (r1), hash);
+        hash = mix32((int) (r1 >>> 32), hash);
+
+        return fmix32(Long.BYTES * 2, hash);
+    }
+
+    /**
+     * Murmur3 32-bit variant.
+     *
+     * @param data - input byte array
+     * @return - hashcode
+     */
+    public static int hash32(byte[] data) {
+        return hash32(data, 0, data.length, DEFAULT_SEED);
+    }
+
+    /**
+     * Murmur3 32-bit variant.
+     *
+     * @param data - input byte array
+     * @param length - length of array
+     * @return - hashcode
+     */
+    public static int hash32(byte[] data, int length) {
+        return hash32(data, 0, length, DEFAULT_SEED);
+    }
+
+    /**
+     * Murmur3 32-bit variant.
+     *
+     * @param data   - input byte array
+     * @param length - length of array
+     * @param seed   - seed. (default 0)
+     * @return - hashcode
+     */
+    public static int hash32(byte[] data, int length, int seed) {
+        return hash32(data, 0, length, seed);
+    }
+
+    /**
+     * Murmur3 32-bit variant.
+     *
+     * @param data   - input byte array
+     * @param offset - offset of data
+     * @param length - length of array
+     * @param seed   - seed. (default 0)
+     * @return - hashcode
+     */
+    public static int hash32(byte[] data, int offset, int length, int seed) {
+        int hash = seed;
+        final int nblocks = length >> 2;
+
+        // body
+        for (int i = 0; i < nblocks; i++) {
+            int i_4 = i << 2;
+            int k = (data[offset + i_4] & 0xff)
+                    | ((data[offset + i_4 + 1] & 0xff) << 8)
+                    | ((data[offset + i_4 + 2] & 0xff) << 16)
+                    | ((data[offset + i_4 + 3] & 0xff) << 24);
+
+            hash = mix32(k, hash);
+        }
+
+        // tail
+        int idx = nblocks << 2;
+        int k1 = 0;
+        switch (length - idx) {
+            case 3:
+                k1 ^= data[offset + idx + 2] << 16;
+            case 2:
+                k1 ^= data[offset + idx + 1] << 8;
+            case 1:
+                k1 ^= data[offset + idx];
+
+                // mix functions
+                k1 *= C1_32;
+                k1 = Integer.rotateLeft(k1, R1_32);
+                k1 *= C2_32;
+                hash ^= k1;
+        }
+
+        return fmix32(length, hash);
+    }
+
+    private static int mix32(int k, int hash) {
+        k *= C1_32;
+        k = Integer.rotateLeft(k, R1_32);
+        k *= C2_32;
+        hash ^= k;
+        return Integer.rotateLeft(hash, R2_32) * M_32 + N_32;
+    }
+
+    private static int fmix32(int length, int hash) {
+        hash ^= length;
+        hash ^= (hash >>> 16);
+        hash *= 0x85ebca6b;
+        hash ^= (hash >>> 13);
+        hash *= 0xc2b2ae35;
+        hash ^= (hash >>> 16);
+
+        return hash;
+    }
+
+    /**
+     * Murmur3 64-bit variant. This is essentially MSB 8 bytes of Murmur3 128-bit variant.
+     *
+     * @param data - input byte array
+     * @return - hashcode
+     */
+    public static long hash64(byte[] data) {
+        return hash64(data, 0, data.length, DEFAULT_SEED);
+    }
+
+    public static long hash64(long data) {
+        long hash = DEFAULT_SEED;
+        long k = Long.reverseBytes(data);
+        int length = Long.BYTES;
+        // mix functions
+        k *= C1;
+        k = Long.rotateLeft(k, R1);
+        k *= C2;
+        hash ^= k;
+        hash = Long.rotateLeft(hash, R2) * M + N1;
+        // finalization
+        hash ^= length;
+        hash = fmix64(hash);
+        return hash;
+    }
+
+    public static long hash64(int data) {
+        long k1 = Integer.reverseBytes(data) & (-1L >>> 32);
+        int length = Integer.BYTES;
+        long hash = DEFAULT_SEED;
+        k1 *= C1;
+        k1 = Long.rotateLeft(k1, R1);
+        k1 *= C2;
+        hash ^= k1;
+        // finalization
+        hash ^= length;
+        hash = fmix64(hash);
+        return hash;
+    }
+
+    public static long hash64(short data) {
+        long hash = DEFAULT_SEED;
+        long k1 = 0;
+        k1 ^= ((long) data & 0xff) << 8;
+        k1 ^= ((long)((data & 0xFF00) >> 8) & 0xff);
+        k1 *= C1;
+        k1 = Long.rotateLeft(k1, R1);
+        k1 *= C2;
+        hash ^= k1;
+
+        // finalization
+        hash ^= Short.BYTES;
+        hash = fmix64(hash);
+        return hash;
+    }
+
+    public static long hash64(byte[] data, int offset, int length) {
+        return hash64(data, offset, length, DEFAULT_SEED);
+    }
+
+    /**
+     * Murmur3 64-bit variant. This is essentially MSB 8 bytes of Murmur3 128-bit variant.
+     *
+     * @param data   - input byte array
+     * @param length - length of array
+     * @param seed   - seed. (default is 0)
+     * @return - hashcode
+     */
+    public static long hash64(byte[] data, int offset, int length, int seed) {
+        long hash = seed;
+        final int nblocks = length >> 3;
+
+        // body
+        for (int i = 0; i < nblocks; i++) {
+            final int i8 = i << 3;
+            long k = ((long) data[offset + i8] & 0xff)
+                    | (((long) data[offset + i8 + 1] & 0xff) << 8)
+                    | (((long) data[offset + i8 + 2] & 0xff) << 16)
+                    | (((long) data[offset + i8 + 3] & 0xff) << 24)
+                    | (((long) data[offset + i8 + 4] & 0xff) << 32)
+                    | (((long) data[offset + i8 + 5] & 0xff) << 40)
+                    | (((long) data[offset + i8 + 6] & 0xff) << 48)
+                    | (((long) data[offset + i8 + 7] & 0xff) << 56);
+
+            // mix functions
+            k *= C1;
+            k = Long.rotateLeft(k, R1);
+            k *= C2;
+            hash ^= k;
+            hash = Long.rotateLeft(hash, R2) * M + N1;
+        }
+
+        // tail
+        long k1 = 0;
+        int tailStart = nblocks << 3;
+        switch (length - tailStart) {
+            case 7:
+                k1 ^= ((long) data[offset + tailStart + 6] & 0xff) << 48;
+            case 6:
+                k1 ^= ((long) data[offset + tailStart + 5] & 0xff) << 40;
+            case 5:
+                k1 ^= ((long) data[offset + tailStart + 4] & 0xff) << 32;
+            case 4:
+                k1 ^= ((long) data[offset + tailStart + 3] & 0xff) << 24;
+            case 3:
+                k1 ^= ((long) data[offset + tailStart + 2] & 0xff) << 16;
+            case 2:
+                k1 ^= ((long) data[offset + tailStart + 1] & 0xff) << 8;
+            case 1:
+                k1 ^= ((long) data[offset + tailStart] & 0xff);
+                k1 *= C1;
+                k1 = Long.rotateLeft(k1, R1);
+                k1 *= C2;
+                hash ^= k1;
+        }
+
+        // finalization
+        hash ^= length;
+        hash = fmix64(hash);
+
+        return hash;
+    }
+
+    /**
+     * Murmur3 128-bit variant.
+     *
+     * @param data - input byte array
+     * @return - hashcode (2 longs)
+     */
+    public static long[] hash128(byte[] data) {
+        return hash128(data, 0, data.length, DEFAULT_SEED);
+    }
+
+    /**
+     * Murmur3 128-bit variant.
+     *
+     * @param data   - input byte array
+     * @param offset - the first element of array
+     * @param length - length of array
+     * @param seed   - seed. (default is 0)
+     * @return - hashcode (2 longs)
+     */
+    public static long[] hash128(byte[] data, int offset, int length, int seed) {
+        long h1 = seed;
+        long h2 = seed;
+        final int nblocks = length >> 4;
+
+        // body
+        for (int i = 0; i < nblocks; i++) {
+            final int i16 = i << 4;
+            long k1 = ((long) data[offset + i16] & 0xff)
+                    | (((long) data[offset + i16 + 1] & 0xff) << 8)
+                    | (((long) data[offset + i16 + 2] & 0xff) << 16)
+                    | (((long) data[offset + i16 + 3] & 0xff) << 24)
+                    | (((long) data[offset + i16 + 4] & 0xff) << 32)
+                    | (((long) data[offset + i16 + 5] & 0xff) << 40)
+                    | (((long) data[offset + i16 + 6] & 0xff) << 48)
+                    | (((long) data[offset + i16 + 7] & 0xff) << 56);
+
+            long k2 = ((long) data[offset + i16 + 8] & 0xff)
+                    | (((long) data[offset + i16 + 9] & 0xff) << 8)
+                    | (((long) data[offset + i16 + 10] & 0xff) << 16)
+                    | (((long) data[offset + i16 + 11] & 0xff) << 24)
+                    | (((long) data[offset + i16 + 12] & 0xff) << 32)
+                    | (((long) data[offset + i16 + 13] & 0xff) << 40)
+                    | (((long) data[offset + i16 + 14] & 0xff) << 48)
+                    | (((long) data[offset + i16 + 15] & 0xff) << 56);
+
+            // mix functions for k1
+            k1 *= C1;
+            k1 = Long.rotateLeft(k1, R1);
+            k1 *= C2;
+            h1 ^= k1;
+            h1 = Long.rotateLeft(h1, R2);
+            h1 += h2;
+            h1 = h1 * M + N1;
+
+            // mix functions for k2
+            k2 *= C2;
+            k2 = Long.rotateLeft(k2, R3);
+            k2 *= C1;
+            h2 ^= k2;
+            h2 = Long.rotateLeft(h2, R1);
+            h2 += h1;
+            h2 = h2 * M + N2;
+        }
+
+        // tail
+        long k1 = 0;
+        long k2 = 0;
+        int tailStart = nblocks << 4;
+        switch (length - tailStart) {
+            case 15:
+                k2 ^= (long) (data[offset + tailStart + 14] & 0xff) << 48;
+            case 14:
+                k2 ^= (long) (data[offset + tailStart + 13] & 0xff) << 40;
+            case 13:
+                k2 ^= (long) (data[offset + tailStart + 12] & 0xff) << 32;
+            case 12:
+                k2 ^= (long) (data[offset + tailStart + 11] & 0xff) << 24;
+            case 11:
+                k2 ^= (long) (data[offset + tailStart + 10] & 0xff) << 16;
+            case 10:
+                k2 ^= (long) (data[offset + tailStart + 9] & 0xff) << 8;
+            case 9:
+                k2 ^= (long) (data[offset + tailStart + 8] & 0xff);
+                k2 *= C2;
+                k2 = Long.rotateLeft(k2, R3);
+                k2 *= C1;
+                h2 ^= k2;
+
+            case 8:
+                k1 ^= (long) (data[offset + tailStart + 7] & 0xff) << 56;
+            case 7:
+                k1 ^= (long) (data[offset + tailStart + 6] & 0xff) << 48;
+            case 6:
+                k1 ^= (long) (data[offset + tailStart + 5] & 0xff) << 40;
+            case 5:
+                k1 ^= (long) (data[offset + tailStart + 4] & 0xff) << 32;
+            case 4:
+                k1 ^= (long) (data[offset + tailStart + 3] & 0xff) << 24;
+            case 3:
+                k1 ^= (long) (data[offset + tailStart + 2] & 0xff) << 16;
+            case 2:
+                k1 ^= (long) (data[offset + tailStart + 1] & 0xff) << 8;
+            case 1:
+                k1 ^= (long) (data[offset + tailStart] & 0xff);
+                k1 *= C1;
+                k1 = Long.rotateLeft(k1, R1);
+                k1 *= C2;
+                h1 ^= k1;
+        }
+
+        // finalization
+        h1 ^= length;
+        h2 ^= length;
+
+        h1 += h2;
+        h2 += h1;
+
+        h1 = fmix64(h1);
+        h2 = fmix64(h2);
+
+        h1 += h2;
+        h2 += h1;
+
+        return new long[]{h1, h2};
+    }
+
+    private static long fmix64(long h) {
+        h ^= (h >>> 33);
+        h *= 0xff51afd7ed558ccdL;
+        h ^= (h >>> 33);
+        h *= 0xc4ceb9fe1a85ec53L;
+        h ^= (h >>> 33);
+        return h;
+    }
+
+    public static class IncrementalHash32 {
+        byte[] tail = new byte[3];
+        int tailLen;
+        int totalLen;
+        int hash;
+
+        public final void start(int hash) {
+            tailLen = totalLen = 0;
+            this.hash = hash;
+        }
+
+        public final void add(byte[] data, int offset, int length) {
+            if (length == 0) return;
+            totalLen += length;
+            if (tailLen + length < 4) {
+                System.arraycopy(data, offset, tail, tailLen, length);
+                tailLen += length;
+                return;
+            }
+            int offset2 = 0;
+            if (tailLen > 0) {
+                offset2 = (4 - tailLen);
+                int k = -1;
+                switch (tailLen) {
+                    case 1:
+                        k = orBytes(tail[0], data[offset], data[offset + 1], data[offset + 2]);
+                        break;
+                    case 2:
+                        k = orBytes(tail[0], tail[1], data[offset], data[offset + 1]);
+                        break;
+                    case 3:
+                        k = orBytes(tail[0], tail[1], tail[2], data[offset]);
+                        break;
+                    default: throw new AssertionError(tailLen);
+                }
+                // mix functions
+                k *= C1_32;
+                k = Integer.rotateLeft(k, R1_32);
+                k *= C2_32;
+                hash ^= k;
+                hash = Integer.rotateLeft(hash, R2_32) * M_32 + N_32;
+            }
+            int length2 = length - offset2;
+            offset += offset2;
+            final int nblocks = length2 >> 2;
+
+            for (int i = 0; i < nblocks; i++) {
+                int i_4 = (i << 2) + offset;
+                int k = orBytes(data[i_4], data[i_4 + 1], data[i_4 + 2], data[i_4 + 3]);
+
+                // mix functions
+                k *= C1_32;
+                k = Integer.rotateLeft(k, R1_32);
+                k *= C2_32;
+                hash ^= k;
+                hash = Integer.rotateLeft(hash, R2_32) * M_32 + N_32;
+            }
+
+            int consumed = (nblocks << 2);
+            tailLen = length2 - consumed;
+            if (consumed == length2) return;
+            System.arraycopy(data, offset + consumed, tail, 0, tailLen);
+        }
+
+        public final int end() {
+            int k1 = 0;
+            switch (tailLen) {
+                case 3:
+                    k1 ^= tail[2] << 16;
+                case 2:
+                    k1 ^= tail[1] << 8;
+                case 1:
+                    k1 ^= tail[0];
+
+                    // mix functions
+                    k1 *= C1_32;
+                    k1 = Integer.rotateLeft(k1, R1_32);
+                    k1 *= C2_32;
+                    hash ^= k1;
+            }
+
+            // finalization
+            hash ^= totalLen;
+            hash ^= (hash >>> 16);
+            hash *= 0x85ebca6b;
+            hash ^= (hash >>> 13);
+            hash *= 0xc2b2ae35;
+            hash ^= (hash >>> 16);
+            return hash;
+        }
+    }
+
+    private static int orBytes(byte b1, byte b2, byte b3, byte b4) {
+        return (b1 & 0xff) | ((b2 & 0xff) << 8) | ((b3 & 0xff) << 16) | ((b4 & 0xff) << 24);
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/common/utils/Murmur3Test.java b/clients/src/test/java/org/apache/kafka/common/utils/Murmur3Test.java
new file mode 100644
index 0000000000000..b850947d2519c
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/utils/Murmur3Test.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.utils;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+import java.util.Map;
+
+/**
+ * This class was taken from Hive org.apache.hive.common.util;
+ * https://github.com/apache/hive/blob/master/storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java
+ * Commit: dffa3a16588bc8e95b9d0ab5af295a74e06ef702
+ *
+ *
+ * Tests for Murmur3 variants.
+ */
+public class Murmur3Test {
+
+    @Test
+    public void testMurmur3_32() {
+        Map cases = new java.util.HashMap<>();
+        cases.put("21".getBytes(), 896581614);
+        cases.put("foobar".getBytes(), -328928243);
+        cases.put("a-little-bit-long-string".getBytes(), -1479816207);
+        cases.put("a-little-bit-longer-string".getBytes(), -153232333);
+        cases.put("lkjh234lh9fiuh90y23oiuhsafujhadof229phr9h19h89h8".getBytes(), 13417721);
+        cases.put(new byte[]{'a', 'b', 'c'}, 461137560);
+
+        int seed = 123;
+        for (Map.Entry c : cases.entrySet()) {
+            byte[] b = (byte[]) c.getKey();
+            assertEquals(c.getValue(), Murmur3.hash32(b, b.length, seed));
+        }
+    }
+
+    @Test
+    public void testMurmur3_128() {
+        Map cases = new java.util.HashMap<>();
+        cases.put("21".getBytes(), new long[]{5857341059704281894L, -5288187638297930763L});
+        cases.put("foobar".getBytes(), new long[]{-351361463397418609L, 8959716011862540668L});
+        cases.put("a-little-bit-long-string".getBytes(), new long[]{8836256500583638442L, -198172363548498523L});
+        cases.put("a-little-bit-longer-string".getBytes(), new long[]{1838346159335108511L, 8794688210320490705L});
+        cases.put("lkjh234lh9fiuh90y23oiuhsafujhadof229phr9h19h89h8".getBytes(), new long[]{-4024021876037397259L, -1482317706335141238L});
+        cases.put(new byte[]{'a', 'b', 'c'}, new long[]{1489494923063836066L, -5440978547625122829L});
+
+        int seed = 123;
+
+        for (Map.Entry c : cases.entrySet()) {
+            byte[] b = (byte[]) c.getKey();
+            long[] result = Murmur3.hash128(b, 0, b.length, seed);
+            assertArrayEquals((long[]) c.getValue(), result);
+        }
+    }
+}
\ No newline at end of file
diff --git a/gradle/spotbugs-exclude.xml b/gradle/spotbugs-exclude.xml
index 7854a9beb24af..61bac20b56f5a 100644
--- a/gradle/spotbugs-exclude.xml
+++ b/gradle/spotbugs-exclude.xml
@@ -221,6 +221,25 @@ For a detailed description of spotbugs bug categories, see https://spotbugs.read
         
     
 
+    
+        
+        
+        
+            
+            
+        
+    
+
+
+    
+        
+        
+        
+            
+            
+        
+    
+
     
         
     
@@ -79,9 +76,6 @@
     
 
-    
-
     
     
@@ -161,6 +155,9 @@
     
 
+    
+
     
     
@@ -199,6 +196,10 @@
     
 
+    
+
+
     
     
diff --git a/gradle/spotbugs-exclude.xml b/gradle/spotbugs-exclude.xml
index 61bac20b56f5a..07ca2b450e753 100644
--- a/gradle/spotbugs-exclude.xml
+++ b/gradle/spotbugs-exclude.xml
@@ -223,7 +223,7 @@ For a detailed description of spotbugs bug categories, see https://spotbugs.read
 
     
         
-        
+        
         
             
             
@@ -233,7 +233,7 @@ For a detailed description of spotbugs bug categories, see https://spotbugs.read
 
     
         
-        
+        
         
             
             
diff --git a/clients/src/main/java/org/apache/kafka/common/utils/Murmur3.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/Murmur3.java
similarity index 99%
rename from clients/src/main/java/org/apache/kafka/common/utils/Murmur3.java
rename to streams/src/main/java/org/apache/kafka/streams/state/internals/Murmur3.java
index a2c780279974a..5581a03648312 100644
--- a/clients/src/main/java/org/apache/kafka/common/utils/Murmur3.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/Murmur3.java
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.kafka.common.utils;
+package org.apache.kafka.streams.state.internals;
 
 /**
  * This class was taken from Hive org.apache.hive.common.util;
diff --git a/clients/src/test/java/org/apache/kafka/common/utils/Murmur3Test.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/Murmur3Test.java
similarity index 98%
rename from clients/src/test/java/org/apache/kafka/common/utils/Murmur3Test.java
rename to streams/src/test/java/org/apache/kafka/streams/state/internals/Murmur3Test.java
index b850947d2519c..d0759bfba14c5 100644
--- a/clients/src/test/java/org/apache/kafka/common/utils/Murmur3Test.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/Murmur3Test.java
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.kafka.common.utils;
+package org.apache.kafka.streams.state.internals;
 
 import static org.junit.Assert.*;
 

From e85d671dee3ac03a1b4623a8b65330df50fe2990 Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Thu, 19 Sep 2019 18:28:24 -0700
Subject: [PATCH 0631/1071] MINOR: replace `late` with `out-of-order` in
 JavaDocs and docs (#7274)

Reviewers: Bill Bejeck , John Roesler 
---
 docs/streams/core-concepts.html                       | 10 +++++-----
 docs/streams/developer-guide/dsl-api.html             |  8 ++++----
 .../org/apache/kafka/streams/kstream/JoinWindows.java |  8 ++++----
 .../apache/kafka/streams/kstream/SessionWindows.java  |  8 ++++----
 .../org/apache/kafka/streams/kstream/Suppressed.java  |  2 +-
 .../org/apache/kafka/streams/kstream/TimeWindows.java |  8 ++++----
 .../org/apache/kafka/streams/kstream/Windows.java     | 11 ++++++-----
 .../processor/UsePreviousTimeOnInvalidTimestamp.java  |  4 ++--
 8 files changed, 30 insertions(+), 29 deletions(-)

diff --git a/docs/streams/core-concepts.html b/docs/streams/core-concepts.html
index 474cac9bb0215..4ee23a2494175 100644
--- a/docs/streams/core-concepts.html
+++ b/docs/streams/core-concepts.html
@@ -49,7 +49,7 @@ 
                Core Concepts
                
         
              • Has no external dependencies on systems other than Apache Kafka itself as the internal messaging layer; notably, it uses Kafka's partitioning model to horizontally scale processing while maintaining strong ordering guarantees.
                • 
         
              • Supports fault-tolerant local state, which enables very fast and efficient stateful operations like windowed joins and aggregations.
                • 
         
              • Supports exactly-once processing semantics to guarantee that each record will be processed once and only once even when there is a failure on either Streams clients or Kafka brokers in the middle of processing.
                • 
-        
              • Employs one-record-at-a-time processing to achieve millisecond processing latency, and supports event-time based windowing operations with late arrival of records.
                • 
+        
              • Employs one-record-at-a-time processing to achieve millisecond processing latency, and supports event-time based windowing operations with out-of-order arrival of records.
                • 
         
              • Offers necessary stream processing primitives, along with a high-level Streams DSL and a low-level Processor API.
                • 
 
     
@@ -124,7 +124,7 @@ 
                Time
                
     
                  
         
                •  When new output records are generated via processing some input record, for example, context.forward() triggered in the process() function call, output record timestamps are inherited from input record timestamps directly.
                  • 
         
                •  When new output records are generated via periodic functions such as Punctuator#punctuate(), the output record timestamp is defined as the current internal time (obtained through context.timestamp()) of the stream task.
                  • 
-        
                •  For aggregations, the timestamp of a resulting aggregate update record will be that of the latest arrived input record that triggered the update.
                  • 
+        
                •  For aggregations, the timestamp of a result update record will be the maximum timestamp of all input records contributing to the result.
                  • 
     
                
 
     
                
@@ -137,7 +137,7 @@ 
                
     
                
 
     
                
-        In the Kafka Streams DSL, an input stream of an aggregation can be a KStream or a KTable, but the output stream will always be a KTable. This allows Kafka Streams to update an aggregate value upon the late arrival of further records after the value was produced and emitted. When such late arrival happens, the aggregating KStream or KTable emits a new aggregate value. Because the output is a KTable, the new value is considered to overwrite the old value with the same key in subsequent processing steps.
+        In the Kafka Streams DSL, an input stream of an aggregation can be a KStream or a KTable, but the output stream will always be a KTable. This allows Kafka Streams to update an aggregate value upon the out-of-order arrival of further records after the value was produced and emitted. When such out-of-order arrival happens, the aggregating KStream or KTable emits a new aggregate value. Because the output is a KTable, the new value is considered to overwrite the old value with the same key in subsequent processing steps.
     
                
 
     
                 Windowing
                
@@ -145,10 +145,10 @@ 
                 Windo
         Windowing lets you control how to group records that have the same key for stateful operations such as aggregations or joins into so-called windows. Windows are tracked per record key.
     
                
     
                
-        Windowing operations are available in the Kafka Streams DSL. When working with windows, you can specify a retention period for the window. This retention period controls how long Kafka Streams will wait for out-of-order or late-arriving data records for a given window. If a record arrives after the retention period of a window has passed, the record is discarded and will not be processed in that window.
+        Windowing operations are available in the Kafka Streams DSL. When working with windows, you can specify a grace period for the window. This grace period controls how long Kafka Streams will wait for out-of-order data records for a given window. If a record arrives after the grace period of a window has passed, the record is discarded and will not be processed in that window. Specifically, a record is discarded if its timestamp dictates it belongs to a window, but the current stream time is greater than the end of the window plus the grace period.
     
                
     
                
-        Late-arriving records are always possible in the real world and should be properly accounted for in your applications. It depends on the effective time semantics  how late records are handled. In the case of processing-time, the semantics are "when the record is being processed", which means that the notion of late records is not applicable as, by definition, no record can be late. Hence, late-arriving records can only be considered as such (i.e. as arriving "late") for event-time or ingestion-time semantics. In both cases, Kafka Streams is able to properly handle late-arriving records.
+        Out-of-order records are always possible in the real world and should be properly accounted for in your applications. It depends on the effective time semantics  how out-of-order records are handled. In the case of processing-time, the semantics are "when the record is being processed", which means that the notion of out-of-order records is not applicable as, by definition, no record can be out-of-order. Hence, out-of-order records can only be considered as such for event-time. In both cases, Kafka Streams is able to properly handle out-of-order records.
     
                
 
     
                Duality of Streams and Tables
                
diff --git a/docs/streams/developer-guide/dsl-api.html b/docs/streams/developer-guide/dsl-api.html
index f5c3df95eff10..c99b03e6c5042 100644
--- a/docs/streams/developer-guide/dsl-api.html
+++ b/docs/streams/developer-guide/dsl-api.html
@@ -335,7 +335,7 @@ 
                
             
                Some KStream transformations may generate one or more KStream objects, for example:
                 - filter and map on a KStream will generate another KStream
                 - branch on KStream can generate multiple KStreams
                
-            
                Some others may generate a KTable object, for example an aggregation of a KStream also yields a KTable. This allows Kafka Streams to continuously update the computed value upon arrivals of late records after it
+            
                Some others may generate a KTable object, for example an aggregation of a KStream also yields a KTable. This allows Kafka Streams to continuously update the computed value upon arrivals of out-of-order records after it
                 has already been produced to the downstream transformation operators.
                
             
                All KTable transformation operations can only generate another KTable. However, the Kafka Streams DSL does provide a special function
                 that converts a KTable representation into a KStream. All of these transformation methods can be chained together to compose
@@ -2961,14 +2961,14 @@ 
                
 of time in Kafka Streams is milliseconds, which means the time numbers would need to be multiplied with 60 * 1,000 to
 convert from minutes to milliseconds (e.g. t=5 would become t=300,000).
                
                         
-                        
                If we then receive three additional records (including two late-arriving records), what would happen is that the two
+                        
                If we then receive three additional records (including two out-of-order records), what would happen is that the two
                             existing sessions for the green record key will be merged into a single session starting at time 0 and ending at time 6,
                             consisting of a total of three records.  The existing session for the blue record key will be extended to end at time 5,
                             consisting of a total of two records.  And, finally, there will be a new session for the blue key starting and ending at
                             time 11.
                
                         
                
                             
-                            
                Detected sessions after having received six input records.  Note the two late-arriving data records at t=4 (green) and
+                            
                Detected sessions after having received six input records.  Note the two out-of-order data records at t=4 (green) and
 t=5 (blue), which lead to a merge of sessions and an extension of a session, respectively.
                
                         
                
                     
@@ -3007,7 +3007,7 @@ 
                
 			    
                
 				    
                grace(ofMinutes(10))
                
 				    
                This allows us to bound the lateness of events the window will accept.
-				        For example, the 09:00 to 10:00 window will accept late-arriving records until 10:10, at which point, the window is closed.
+				        For example, the 09:00 to 10:00 window will accept out-of-order records until 10:10, at which point, the window is closed.
 				    
                
 				    
                .suppress(Suppressed.untilWindowCloses(...))
                
 				    
                This configures the suppression operator to emit nothing for a window until it closes, and then emit the final result.
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/JoinWindows.java b/streams/src/main/java/org/apache/kafka/streams/kstream/JoinWindows.java
index 4798183297c9b..f7e3585d1376f 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/JoinWindows.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/JoinWindows.java
@@ -215,12 +215,12 @@ public long size() {
     }
 
     /**
-     * Reject late events that arrive more than {@code afterWindowEnd}
+     * Reject out-of-order events that are delayed more than {@code afterWindowEnd}
      * after the end of its window.
+     * 
                
+     * Delay is defined as (stream_time - record_timestamp).
      *
-     * Lateness is defined as (stream_time - record_timestamp).
-     *
-     * @param afterWindowEnd The grace period to admit late-arriving events to a window.
+     * @param afterWindowEnd The grace period to admit out-of-order events to a window.
      * @return this updated builder
      * @throws IllegalArgumentException if the {@code afterWindowEnd} is negative of can't be represented as {@code long milliseconds}
      */
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindows.java b/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindows.java
index c0153a30fbe94..a67d001a38c26 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindows.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindows.java
@@ -134,14 +134,14 @@ public SessionWindows until(final long durationMs) throws IllegalArgumentExcepti
     }
 
     /**
-     * Reject late events that arrive more than {@code afterWindowEnd}
+     * Reject out-of-order events that arrive more than {@code afterWindowEnd}
      * after the end of its window.
-     *
+     * 
                
      * Note that new events may change the boundaries of session windows, so aggressive
-     * close times can lead to surprising results in which a too-late event is rejected and then
+     * close times can lead to surprising results in which an out-of-order event is rejected and then
      * a subsequent event moves the window boundary forward.
      *
-     * @param afterWindowEnd The grace period to admit late-arriving events to a window.
+     * @param afterWindowEnd The grace period to admit out-of-order events to a window.
      * @return this updated builder
      * @throws IllegalArgumentException if the {@code afterWindowEnd} is negative of can't be represented as {@code long milliseconds}
      */
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Suppressed.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Suppressed.java
index c80eaecc70fd4..2e0301ce4d090 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/Suppressed.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Suppressed.java
@@ -132,7 +132,7 @@ static StrictBufferConfig unbounded() {
      *
      * To accomplish this, the operator will buffer events from the window until the window close (that is,
      * until the end-time passes, and additionally until the grace period expires). Since windowed operators
-     * are required to reject late events for a window whose grace period is expired, there is an additional
+     * are required to reject out-of-order events for a window whose grace period is expired, there is an additional
      * guarantee that the final results emitted from this suppression will match any queriable state upstream.
      *
      * @param bufferConfig A configuration specifying how much space to use for buffering intermediate results.
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindows.java b/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindows.java
index a87dbf366bb2d..ba90fb292a22d 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindows.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindows.java
@@ -188,12 +188,12 @@ public long size() {
     }
 
     /**
-     * Reject late events that arrive more than {@code millisAfterWindowEnd}
+     * Reject out-of-order events that arrive more than {@code millisAfterWindowEnd}
      * after the end of its window.
+     * 
                
+     * Delay is defined as (stream_time - record_timestamp).
      *
-     * Lateness is defined as (stream_time - record_timestamp).
-     *
-     * @param afterWindowEnd The grace period to admit late-arriving events to a window.
+     * @param afterWindowEnd The grace period to admit out-of-order events to a window.
      * @return this updated builder
      * @throws IllegalArgumentException if {@code afterWindowEnd} is negative or can't be represented as {@code long milliseconds}
      */
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Windows.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Windows.java
index e122b4a5dcaab..45b63bc1f5611 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/Windows.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Windows.java
@@ -26,9 +26,10 @@
 
 /**
  * The window specification for fixed size windows that is used to define window boundaries and grace period.
- *
- * Grace period defines how long to wait on late events, where lateness is defined as (stream_time - record_timestamp).
- *
+ * 
                
+ * Grace period defines how long to wait on out-of-order events. That is, windows will continue to accept new records until {@code stream_time >= window_end + grace_period}.
+ * Records that arrive after the grace period passed are considered late and will not be processed but are dropped.
+ * 
                
  * Warning: It may be unsafe to use objects of this class in set- or map-like collections,
  * since the equals and hashCode methods depend on mutable fields.
  *
@@ -118,9 +119,9 @@ protected Windows segments(final int segments) throws IllegalArgumentExceptio
 
     /**
      * Return the window grace period (the time to admit
-     * late-arriving events after the end of the window.)
+     * out-of-order events after the end of the window.)
      *
-     * Lateness is defined as (stream_time - record_timestamp).
+     * Delay is defined as (stream_time - record_timestamp).
      */
     public abstract long gracePeriodMs();
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/UsePreviousTimeOnInvalidTimestamp.java b/streams/src/main/java/org/apache/kafka/streams/processor/UsePreviousTimeOnInvalidTimestamp.java
index 6807c6d257176..2ec937300b4c1 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/UsePreviousTimeOnInvalidTimestamp.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/UsePreviousTimeOnInvalidTimestamp.java
@@ -50,8 +50,8 @@ public class UsePreviousTimeOnInvalidTimestamp extends ExtractRecordMetadataTime
      * @param record a data record
      * @param recordTimestamp the timestamp extractor from the record
      * @param partitionTime the highest extracted valid timestamp of the current record's partition˙ (could be -1 if unknown)
-     * @return the provided latest extracted valid timestamp as new timestamp for the record
-     * @throws StreamsException if latest extracted valid timestamp is unknown
+     * @return the provided highest extracted valid timestamp as new timestamp for the record
+     * @throws StreamsException if highest extracted valid timestamp is unknown
      */
     @Override
     public long onInvalidTimestamp(final ConsumerRecord record,

From c955828095b28d9533b98fc979e4a768c28263d8 Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Fri, 20 Sep 2019 09:25:19 -0700
Subject: [PATCH 0632/1071] MINOR: the code generator should be able to set the
 java package (#7355)

Reviewers: Colin P. McCabe 
---
 build.gradle                                         |  8 ++++++--
 .../kafka/message/ApiMessageTypeGenerator.java       |  4 ++--
 .../org/apache/kafka/message/HeaderGenerator.java    |  9 ++++++---
 .../apache/kafka/message/MessageDataGenerator.java   |  4 ++--
 .../org/apache/kafka/message/MessageGenerator.java   | 12 ++++++------
 .../kafka/message/MessageDataGeneratorTest.java      |  6 +++---
 6 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/build.gradle b/build.gradle
index dc7ce681d4d69..d80118330d611 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1012,7 +1012,9 @@ project(':clients') {
   task processMessages(type:JavaExec) {
     main = "org.apache.kafka.message.MessageGenerator"
     classpath = project(':generator').sourceSets.main.runtimeClasspath
-    args = [ "src/generated/java/org/apache/kafka/common/message", "src/main/resources/common/message" ]
+    args = [ "org.apache.kafka.common.message",
+             "src/generated/java/org/apache/kafka/common/message",
+             "src/main/resources/common/message" ]
     inputs.dir("src/main/resources/common/message")
     outputs.dir("src/generated/java/org/apache/kafka/common/message")
   }
@@ -1020,7 +1022,9 @@ project(':clients') {
   task processTestMessages(type:JavaExec) {
     main = "org.apache.kafka.message.MessageGenerator"
     classpath = project(':generator').sourceSets.main.runtimeClasspath
-    args = [ "src/generated-test/java/org/apache/kafka/common/message", "src/test/resources/common/message" ]
+    args = [ "org.apache.kafka.common.message",
+             "src/generated-test/java/org/apache/kafka/common/message",
+             "src/test/resources/common/message" ]
     inputs.dir("src/test/resources/common/message")
     outputs.dir("src/generated-test/java/org/apache/kafka/common/message")
   }
diff --git a/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java b/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java
index c5439aa2e81c8..5a2cf117fea50 100644
--- a/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java
+++ b/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java
@@ -67,8 +67,8 @@ String responseSchema() {
         }
     }
 
-    public ApiMessageTypeGenerator() {
-        this.headerGenerator = new HeaderGenerator();
+    public ApiMessageTypeGenerator(String packageName) {
+        this.headerGenerator = new HeaderGenerator(packageName);
         this.apis = new TreeMap<>();
         this.buffer = new CodeBuffer();
     }
diff --git a/generator/src/main/java/org/apache/kafka/message/HeaderGenerator.java b/generator/src/main/java/org/apache/kafka/message/HeaderGenerator.java
index 6de9736af818c..1223c00a6c08c 100644
--- a/generator/src/main/java/org/apache/kafka/message/HeaderGenerator.java
+++ b/generator/src/main/java/org/apache/kafka/message/HeaderGenerator.java
@@ -17,6 +17,7 @@
 
 package org.apache.kafka.message;
 
+import java.util.Objects;
 import java.util.TreeSet;
 
 /**
@@ -45,15 +46,16 @@ public final class HeaderGenerator {
         ""
     };
 
-    static final String PACKAGE = "org.apache.kafka.common.message";
 
     private final CodeBuffer buffer;
 
     private final TreeSet imports;
+    private final String packageName;
 
-    public HeaderGenerator() {
+    public HeaderGenerator(String packageName) {
         this.buffer = new CodeBuffer();
         this.imports = new TreeSet<>();
+        this.packageName = packageName;
     }
 
     public void addImport(String newImport) {
@@ -61,10 +63,11 @@ public void addImport(String newImport) {
     }
 
     public void generate() {
+        Objects.requireNonNull(packageName);
         for (int i = 0; i < HEADER.length; i++) {
             buffer.printf("%s%n", HEADER[i]);
         }
-        buffer.printf("package %s;%n", PACKAGE);
+        buffer.printf("package %s;%n", packageName);
         buffer.printf("%n");
         for (String newImport : imports) {
             buffer.printf("import %s;%n", newImport);
diff --git a/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java b/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java
index 6a2056077c6f8..2e946e7bfb458 100644
--- a/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java
+++ b/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java
@@ -34,9 +34,9 @@ public final class MessageDataGenerator {
     private final SchemaGenerator schemaGenerator;
     private final CodeBuffer buffer;
 
-    MessageDataGenerator() {
+    MessageDataGenerator(String packageName) {
         this.structRegistry = new StructRegistry();
-        this.headerGenerator = new HeaderGenerator();
+        this.headerGenerator = new HeaderGenerator(packageName);
         this.schemaGenerator = new SchemaGenerator(headerGenerator, structRegistry);
         this.buffer = new CodeBuffer();
     }
diff --git a/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java b/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java
index df0c5f886b12a..9b5d9ed2da5ce 100644
--- a/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java
+++ b/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java
@@ -98,10 +98,10 @@ public final class MessageGenerator {
         JSON_SERDE.setSerializationInclusion(JsonInclude.Include.NON_EMPTY);
     }
 
-    public static void processDirectories(String outputDir, String inputDir) throws Exception {
+    public static void processDirectories(String packageName, String outputDir, String inputDir) throws Exception {
         Files.createDirectories(Paths.get(outputDir));
         int numProcessed = 0;
-        ApiMessageTypeGenerator messageTypeGenerator = new ApiMessageTypeGenerator();
+        ApiMessageTypeGenerator messageTypeGenerator = new ApiMessageTypeGenerator(packageName);
         HashSet outputFileNames = new HashSet<>();
         try (DirectoryStream directoryStream = Files
                 .newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) {
@@ -113,7 +113,7 @@ public static void processDirectories(String outputDir, String inputDir) throws
                     outputFileNames.add(javaName);
                     Path outputPath = Paths.get(outputDir, javaName);
                     try (BufferedWriter writer = Files.newBufferedWriter(outputPath)) {
-                        MessageDataGenerator generator = new MessageDataGenerator();
+                        MessageDataGenerator generator = new MessageDataGenerator(packageName);
                         generator.generate(spec);
                         generator.write(writer);
                     }
@@ -198,16 +198,16 @@ static String stripSuffix(String str, String suffix) {
         }
     }
 
-    private final static String USAGE = "MessageGenerator: [output Java file] [input JSON file]";
+    private final static String USAGE = "MessageGenerator: [output Java package] [output Java file] [input JSON file]";
 
     public static void main(String[] args) throws Exception {
         if (args.length == 0) {
             System.out.println(USAGE);
             System.exit(0);
-        } else if (args.length != 2) {
+        } else if (args.length != 3) {
             System.out.println(USAGE);
             System.exit(1);
         }
-        processDirectories(args[0], args[1]);
+        processDirectories(args[0], args[1], args[2]);
     }
 }
diff --git a/generator/src/test/java/org/apache/kafka/message/MessageDataGeneratorTest.java b/generator/src/test/java/org/apache/kafka/message/MessageDataGeneratorTest.java
index e8fcaf227000f..a772988974924 100644
--- a/generator/src/test/java/org/apache/kafka/message/MessageDataGeneratorTest.java
+++ b/generator/src/test/java/org/apache/kafka/message/MessageDataGeneratorTest.java
@@ -47,7 +47,7 @@ public void testNullDefaults() throws Exception {
                 "      \"nullableVersions\": \"2+\", \"default\": \"null\" }",
                 "  ]",
                 "}")), MessageSpec.class);
-        new MessageDataGenerator().generate(testMessageSpec);
+        new MessageDataGenerator("org.apache.kafka.common.message").generate(testMessageSpec);
     }
 
     @Test
@@ -62,7 +62,7 @@ public void testInvalidNullDefaultForInt() throws Exception {
             "  ]",
             "}")), MessageSpec.class);
         try {
-            new MessageDataGenerator().generate(testMessageSpec);
+            new MessageDataGenerator("org.apache.kafka.common.message").generate(testMessageSpec);
             fail("Expected MessageDataGenerator#generate to fail");
         } catch (Throwable e) {
             assertTrue("Invalid error message: " + e.getMessage(),
@@ -83,7 +83,7 @@ public void testInvalidNullDefaultForPotentiallyNonNullableArray() throws Except
                 "  ]",
                 "}")), MessageSpec.class);
         try {
-            new MessageDataGenerator().generate(testMessageSpec);
+            new MessageDataGenerator("org.apache.kafka.common.message").generate(testMessageSpec);
             fail("Expected MessageDataGenerator#generate to fail");
         } catch (RuntimeException e) {
             assertTrue("Invalid error message: " + e.getMessage(),

From 8001aff30429879401cdd8cb44c12ea9d773dcfd Mon Sep 17 00:00:00 2001
From: Kamal Chandraprakash 
Date: Fri, 20 Sep 2019 23:51:06 +0530
Subject: [PATCH 0633/1071] KAFKA-8892: Display the sorted configs in Kafka
 Configs Help Command.

Author: Kamal Chandraprakash 

Reviewers: Reviewers: Manikumar Reddy 

Closes #7319 from kamalcph/KAFKA-8892
---
 core/src/main/scala/kafka/admin/ConfigCommand.scala | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/core/src/main/scala/kafka/admin/ConfigCommand.scala b/core/src/main/scala/kafka/admin/ConfigCommand.scala
index 1b74eb6fc6e35..9203334e4615e 100644
--- a/core/src/main/scala/kafka/admin/ConfigCommand.scala
+++ b/core/src/main/scala/kafka/admin/ConfigCommand.scala
@@ -128,7 +128,7 @@ object ConfigCommand extends Config {
       if (dynamicBrokerConfigs.nonEmpty) {
         val perBrokerConfig = entityName != ConfigEntityName.Default
         val errorMessage = s"--bootstrap-server option must be specified to update broker configs $dynamicBrokerConfigs."
-        val info = "Broker configuraton updates using ZooKeeper are supported for bootstrapping before brokers" +
+        val info = "Broker configuration updates using ZooKeeper are supported for bootstrapping before brokers" +
           " are started to enable encrypted password configs to be stored in ZooKeeper."
         if (perBrokerConfig) {
           adminZkClient.parseBroker(entityName).foreach { brokerId =>
@@ -522,9 +522,9 @@ object ConfigCommand extends Config {
     val nl = System.getProperty("line.separator")
     val addConfig = parser.accepts("add-config", "Key Value pairs of configs to add. Square brackets can be used to group values which contain commas: 'k1=v1,k2=[v1,v2,v2],k3=v3'. The following is a list of valid configurations: " +
             "For entity-type '" + ConfigType.Topic + "': " + LogConfig.configNames.map("\t" + _).mkString(nl, nl, nl) +
-            "For entity-type '" + ConfigType.Broker + "': " + DynamicConfig.Broker.names.asScala.map("\t" + _).mkString(nl, nl, nl) +
-            "For entity-type '" + ConfigType.User + "': " + DynamicConfig.User.names.asScala.map("\t" + _).mkString(nl, nl, nl) +
-            "For entity-type '" + ConfigType.Client + "': " + DynamicConfig.Client.names.asScala.map("\t" + _).mkString(nl, nl, nl) +
+            "For entity-type '" + ConfigType.Broker + "': " + DynamicConfig.Broker.names.asScala.toSeq.sorted.map("\t" + _).mkString(nl, nl, nl) +
+            "For entity-type '" + ConfigType.User + "': " + DynamicConfig.User.names.asScala.toSeq.sorted.map("\t" + _).mkString(nl, nl, nl) +
+            "For entity-type '" + ConfigType.Client + "': " + DynamicConfig.Client.names.asScala.toSeq.sorted.map("\t" + _).mkString(nl, nl, nl) +
             s"Entity types '${ConfigType.User}' and '${ConfigType.Client}' may be specified together to update config for clients of a specific user.")
             .withRequiredArg
             .ofType(classOf[String])

From d91a94e7bff7a2ffe94b562ef1108307afce1bc7 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Fri, 20 Sep 2019 17:50:12 -0700
Subject: [PATCH 0634/1071] KAFKA-8609: Add consumer rebalance metrics (#7347)

Adding the following metrics in:

1. AbstractCoordinator (for both consumer and connect)
* rebalance-latency-avg
* rebalance-latency-max
* rebalance-total
* rebalance-rate-per-hour
* failed-rebalance-total
* failed-rebalance-rate-per-hour
* last-rebalance-seconds-ago

2. ConsumerCoordinator
* partition-revoked-latency-avg
* partition-revoked-latency-max
* partition-assigned-latency-avg
* partition-assigned-latency-max
* partition-lost-latency-avg
* partition-lost-latency-max

Reviewers: Bruno Cadonna , A. Sophie Blee-Goldman , Matthias J. Sax 
---
 .../internals/AbstractCoordinator.java        | 136 +++++++++++++-----
 .../internals/ConsumerCoordinator.java        |  45 +++++-
 .../clients/consumer/internals/Heartbeat.java |   2 +-
 .../kafka/common/metrics/JmxReporter.java     |   1 +
 .../internals/AbstractCoordinatorTest.java    |  93 +++++++++++-
 .../internals/ConsumerCoordinatorTest.java    |  54 +++++++
 6 files changed, 287 insertions(+), 44 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index 4c71a89f3bdcd..fbd15d8e905fe 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -39,12 +39,13 @@
 import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse;
 import org.apache.kafka.common.message.SyncGroupRequestData;
 import org.apache.kafka.common.metrics.Measurable;
-import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
+import org.apache.kafka.common.metrics.stats.CumulativeCount;
 import org.apache.kafka.common.metrics.stats.Max;
 import org.apache.kafka.common.metrics.stats.Meter;
+import org.apache.kafka.common.metrics.stats.Rate;
 import org.apache.kafka.common.metrics.stats.WindowedCount;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
@@ -122,13 +123,15 @@ private enum MemberState {
     protected final ConsumerNetworkClient client;
     protected final Time time;
 
-    private HeartbeatThread heartbeatThread = null;
+    private Node coordinator = null;
     private boolean rejoinNeeded = true;
     private boolean needsJoinPrepare = true;
     private MemberState state = MemberState.UNJOINED;
+    private HeartbeatThread heartbeatThread = null;
     private RequestFuture joinFuture = null;
-    private Node coordinator = null;
     private Generation generation = Generation.NO_GENERATION;
+    private long lastRebalanceStartMs = -1L;
+    private long lastRebalanceEndMs = -1L;
 
     private RequestFuture findCoordinatorFuture = null;
 
@@ -440,6 +443,10 @@ private synchronized RequestFuture initiateJoinGroup() {
             disableHeartbeatThread();
 
             state = MemberState.REBALANCING;
+            // a rebalance can be triggered consecutively if the previous one failed,
+            // in this case we would not update the start time.
+            if (lastRebalanceStartMs == -1L)
+                lastRebalanceStartMs = time.milliseconds();
             joinFuture = sendJoinGroupRequest();
             joinFuture.addListener(new RequestFutureListener() {
                 @Override
@@ -450,6 +457,10 @@ public void onSuccess(ByteBuffer value) {
                         log.info("Successfully joined group with generation {}", generation.generationId);
                         state = MemberState.STABLE;
                         rejoinNeeded = false;
+                        // record rebalance latency
+                        lastRebalanceEndMs = time.milliseconds();
+                        sensors.successfulRebalanceSensor.record(lastRebalanceEndMs - lastRebalanceStartMs);
+                        lastRebalanceStartMs = -1L;
 
                         if (heartbeatThread != null)
                             heartbeatThread.enable();
@@ -462,6 +473,7 @@ public void onFailure(RuntimeException e) {
                     // after having been woken up, the exception is ignored and we will rejoin
                     synchronized (AbstractCoordinator.this) {
                         state = MemberState.UNJOINED;
+                        sensors.failedRebalanceSensor.record();
                     }
                 }
             });
@@ -511,7 +523,7 @@ public void handle(JoinGroupResponse joinResponse, RequestFuture fut
             Errors error = joinResponse.error();
             if (error == Errors.NONE) {
                 log.debug("Received successful JoinGroup response: {}", joinResponse);
-                sensors.joinLatency.record(response.requestLatencyMs());
+                sensors.joinSensor.record(response.requestLatencyMs());
 
                 synchronized (AbstractCoordinator.this) {
                     if (state != MemberState.REBALANCING) {
@@ -641,7 +653,7 @@ public void handle(SyncGroupResponse syncResponse,
                            RequestFuture future) {
             Errors error = syncResponse.error();
             if (error == Errors.NONE) {
-                sensors.syncLatency.record(response.requestLatencyMs());
+                sensors.syncSensor.record(response.requestLatencyMs());
                 future.complete(ByteBuffer.wrap(syncResponse.data.assignment()));
             } else {
                 requestRejoin();
@@ -931,7 +943,7 @@ synchronized RequestFuture sendHeartbeatRequest() {
     private class HeartbeatResponseHandler extends CoordinatorResponseHandler {
         @Override
         public void handle(HeartbeatResponse heartbeatResponse, RequestFuture future) {
-            sensors.heartbeatLatency.record(response.requestLatencyMs());
+            sensors.heartbeatSensor.record(response.requestLatencyMs());
             Errors error = heartbeatResponse.error();
             if (error == Errors.NONE) {
                 log.debug("Received successful Heartbeat response");
@@ -1005,44 +1017,98 @@ protected Meter createMeter(Metrics metrics, String groupName, String baseName,
     private class GroupCoordinatorMetrics {
         public final String metricGrpName;
 
-        public final Sensor heartbeatLatency;
-        public final Sensor joinLatency;
-        public final Sensor syncLatency;
+        public final Sensor heartbeatSensor;
+        public final Sensor joinSensor;
+        public final Sensor syncSensor;
+        public final Sensor successfulRebalanceSensor;
+        public final Sensor failedRebalanceSensor;
 
         public GroupCoordinatorMetrics(Metrics metrics, String metricGrpPrefix) {
             this.metricGrpName = metricGrpPrefix + "-coordinator-metrics";
 
-            this.heartbeatLatency = metrics.sensor("heartbeat-latency");
-            this.heartbeatLatency.add(metrics.metricName("heartbeat-response-time-max",
-                    this.metricGrpName,
-                    "The max time taken to receive a response to a heartbeat request"), new Max());
-            this.heartbeatLatency.add(createMeter(metrics, metricGrpName, "heartbeat", "heartbeats"));
+            this.heartbeatSensor = metrics.sensor("heartbeat-latency");
+            this.heartbeatSensor.add(metrics.metricName("heartbeat-response-time-max",
+                this.metricGrpName,
+                "The max time taken to receive a response to a heartbeat request"), new Max());
+            this.heartbeatSensor.add(createMeter(metrics, metricGrpName, "heartbeat", "heartbeats"));
 
-            this.joinLatency = metrics.sensor("join-latency");
-            this.joinLatency.add(metrics.metricName("join-time-avg",
+            this.joinSensor = metrics.sensor("join-latency");
+            this.joinSensor.add(metrics.metricName("join-time-avg",
+                this.metricGrpName,
+                "The average time taken for a group rejoin"), new Avg());
+            this.joinSensor.add(metrics.metricName("join-time-max",
+                this.metricGrpName,
+                "The max time taken for a group rejoin"), new Max());
+            this.joinSensor.add(createMeter(metrics, metricGrpName, "join", "group joins"));
+
+            this.syncSensor = metrics.sensor("sync-latency");
+            this.syncSensor.add(metrics.metricName("sync-time-avg",
+                this.metricGrpName,
+                "The average time taken for a group sync"), new Avg());
+            this.syncSensor.add(metrics.metricName("sync-time-max",
+                this.metricGrpName,
+                "The max time taken for a group sync"), new Max());
+            this.syncSensor.add(createMeter(metrics, metricGrpName, "sync", "group syncs"));
+
+            this.successfulRebalanceSensor = metrics.sensor("rebalance-latency");
+            this.successfulRebalanceSensor.add(metrics.metricName("rebalance-latency-avg",
+                this.metricGrpName,
+                "The average time taken for a group to complete a successful rebalance, which may be composed of " +
+                    "several failed re-trials until it succeeded"), new Avg());
+            this.successfulRebalanceSensor.add(metrics.metricName("rebalance-latency-max",
+                this.metricGrpName,
+                "The max time taken for a group to complete a successful rebalance, which may be composed of " +
+                    "several failed re-trials until it succeeded"), new Max());
+            this.successfulRebalanceSensor.add(
+                metrics.metricName("rebalance-total",
                     this.metricGrpName,
-                    "The average time taken for a group rejoin"), new Avg());
-            this.joinLatency.add(metrics.metricName("join-time-max",
+                    "The total number of successful rebalance events, each event is composed of " +
+                        "several failed re-trials until it succeeded"),
+                new CumulativeCount()
+            );
+            this.successfulRebalanceSensor.add(
+                metrics.metricName(
+                    "rebalance-rate-per-hour",
                     this.metricGrpName,
-                    "The max time taken for a group rejoin"), new Max());
-            this.joinLatency.add(createMeter(metrics, metricGrpName, "join", "group joins"));
-
+                    "The number of successful rebalance events per hour, each event is composed of " +
+                        "several failed re-trials until it succeeded"),
+                new Rate(TimeUnit.HOURS, new WindowedCount())
+            );
 
-            this.syncLatency = metrics.sensor("sync-latency");
-            this.syncLatency.add(metrics.metricName("sync-time-avg",
+            this.failedRebalanceSensor = metrics.sensor("failed-rebalance");
+            this.failedRebalanceSensor.add(
+                metrics.metricName("failed-rebalance-total",
                     this.metricGrpName,
-                    "The average time taken for a group sync"), new Avg());
-            this.syncLatency.add(metrics.metricName("sync-time-max",
+                    "The total number of failed rebalance events"),
+                new CumulativeCount()
+            );
+            this.failedRebalanceSensor.add(
+                metrics.metricName(
+                    "failed-rebalance-rate-per-hour",
                     this.metricGrpName,
-                    "The max time taken for a group sync"), new Max());
-            this.syncLatency.add(createMeter(metrics, metricGrpName, "sync", "group syncs"));
+                    "The number of failed rebalance events per hour"),
+                new Rate(TimeUnit.HOURS, new WindowedCount())
+            );
 
-            Measurable lastHeartbeat =
-                new Measurable() {
-                    public double measure(MetricConfig config, long now) {
-                        return TimeUnit.SECONDS.convert(now - heartbeat.lastHeartbeatSend(), TimeUnit.MILLISECONDS);
-                    }
-                };
+            Measurable lastRebalance = (config, now) -> {
+                if (lastRebalanceEndMs == -1L)
+                    // if no rebalance is ever triggered, we just return -1.
+                    return -1d;
+                else
+                    return TimeUnit.SECONDS.convert(now - lastRebalanceEndMs, TimeUnit.MILLISECONDS);
+            };
+            metrics.addMetric(metrics.metricName("last-rebalance-seconds-ago",
+                this.metricGrpName,
+                "The number of seconds since the last successful rebalance event"),
+                lastRebalance);
+
+            Measurable lastHeartbeat = (config, now) -> {
+                if (heartbeat.lastHeartbeatSend() == 0L)
+                    // if no heartbeat is ever triggered, just return -1.
+                    return -1d;
+                else
+                    return TimeUnit.SECONDS.convert(now - heartbeat.lastHeartbeatSend(), TimeUnit.MILLISECONDS);
+            };
             metrics.addMetric(metrics.metricName("last-heartbeat-seconds-ago",
                 this.metricGrpName,
                 "The number of seconds since the last coordinator heartbeat was sent"),
@@ -1251,4 +1317,8 @@ private static class UnjoinedGroupException extends RetriableException {
     public Heartbeat heartbeat() {
         return heartbeat;
     }
+
+    public void setLastRebalanceTime(final long timestamp) {
+        lastRebalanceEndMs = timestamp;
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index a3aaface7822d..f361735a6e010 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -268,7 +268,9 @@ private Exception invokePartitionsAssigned(final Set assignedPar
 
         ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
         try {
+            final long startMs = time.milliseconds();
             listener.onPartitionsAssigned(assignedPartitions);
+            sensors.assignCallbackSensor.record(time.milliseconds() - startMs);
         } catch (WakeupException | InterruptException e) {
             throw e;
         } catch (Exception e) {
@@ -285,7 +287,9 @@ private Exception invokePartitionsRevoked(final Set revokedParti
 
         ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
         try {
+            final long startMs = time.milliseconds();
             listener.onPartitionsRevoked(revokedPartitions);
+            sensors.revokeCallbackSensor.record(time.milliseconds() - startMs);
         } catch (WakeupException | InterruptException e) {
             throw e;
         } catch (Exception e) {
@@ -302,7 +306,9 @@ private Exception invokePartitionsLost(final Set lostPartitions)
 
         ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
         try {
+            final long startMs = time.milliseconds();
             listener.onPartitionsLost(lostPartitions);
+            sensors.loseCallbackSensor.record(time.milliseconds() - startMs);
         } catch (WakeupException | InterruptException e) {
             throw e;
         } catch (Exception e) {
@@ -1078,7 +1084,7 @@ private OffsetCommitResponseHandler(Map offse
 
         @Override
         public void handle(OffsetCommitResponse commitResponse, RequestFuture future) {
-            sensors.commitLatency.record(response.requestLatencyMs());
+            sensors.commitSensor.record(response.requestLatencyMs());
             Set unauthorizedTopics = new HashSet<>();
 
             for (OffsetCommitResponseData.OffsetCommitResponseTopic topic : commitResponse.data().topics()) {
@@ -1241,19 +1247,46 @@ public void handle(OffsetFetchResponse response, RequestFuture subscriptions.numAssignedPartitions();
             metrics.addMetric(metrics.metricName("assigned-partitions",
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Heartbeat.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Heartbeat.java
index 3bf8c92c2edb8..4d19ef4a0141e 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Heartbeat.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Heartbeat.java
@@ -31,7 +31,7 @@ public final class Heartbeat {
     private final Timer sessionTimer;
     private final Timer pollTimer;
 
-    private volatile long lastHeartbeatSend;
+    private volatile long lastHeartbeatSend = 0L;
 
     public Heartbeat(GroupRebalanceConfig config,
                      Time time) {
diff --git a/clients/src/main/java/org/apache/kafka/common/metrics/JmxReporter.java b/clients/src/main/java/org/apache/kafka/common/metrics/JmxReporter.java
index 8d7bdbe85a036..3f852cf1a5387 100644
--- a/clients/src/main/java/org/apache/kafka/common/metrics/JmxReporter.java
+++ b/clients/src/main/java/org/apache/kafka/common/metrics/JmxReporter.java
@@ -78,6 +78,7 @@ public void init(List metrics) {
     public boolean containsMbean(String mbeanName) {
         return mbeans.containsKey(mbeanName);
     }
+
     @Override
     public void metricChange(KafkaMetric metric) {
         synchronized (LOCK) {
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
index e0264b3f35917..1936fcc3c3106 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
@@ -32,6 +32,7 @@
 import org.apache.kafka.common.message.LeaveGroupResponseData;
 import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse;
 import org.apache.kafka.common.message.SyncGroupResponseData;
+import org.apache.kafka.common.metrics.KafkaMetric;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
@@ -88,12 +89,13 @@ public class AbstractCoordinatorTest {
     private static final String GROUP_ID = "dummy-group";
     private static final String METRIC_GROUP_PREFIX = "consumer";
 
-    private MockClient mockClient;
-    private MockTime mockTime;
     private Node node;
+    private Metrics metrics;
+    private MockTime mockTime;
     private Node coordinatorNode;
-    private ConsumerNetworkClient consumerClient;
+    private MockClient mockClient;
     private DummyCoordinator coordinator;
+    private ConsumerNetworkClient consumerClient;
 
     private final String memberId = "memberId";
     private final String leaderId = "leaderId";
@@ -124,7 +126,7 @@ false, false, new SubscriptionState(logContext, OffsetResetStrategy.EARLIEST),
                                                         retryBackoffMs,
                                                         REQUEST_TIMEOUT_MS,
                                                         HEARTBEAT_INTERVAL_MS);
-        Metrics metrics = new Metrics();
+        metrics = new Metrics(mockTime);
 
         mockClient.updateMetadata(TestUtils.metadataUpdateWith(1, emptyMap()));
         this.node = metadata.fetch().nodes().get(0);
@@ -143,6 +145,89 @@ false, false, new SubscriptionState(logContext, OffsetResetStrategy.EARLIEST),
                                                 mockTime);
     }
 
+    @Test
+    public void testMetrics() {
+        setupCoordinator();
+
+        assertNotNull(getMetric("heartbeat-response-time-max"));
+        assertNotNull(getMetric("heartbeat-rate"));
+        assertNotNull(getMetric("heartbeat-total"));
+        assertNotNull(getMetric("last-heartbeat-seconds-ago"));
+        assertNotNull(getMetric("join-time-avg"));
+        assertNotNull(getMetric("join-time-max"));
+        assertNotNull(getMetric("join-rate"));
+        assertNotNull(getMetric("join-total"));
+        assertNotNull(getMetric("sync-time-avg"));
+        assertNotNull(getMetric("sync-time-max"));
+        assertNotNull(getMetric("sync-rate"));
+        assertNotNull(getMetric("sync-total"));
+        assertNotNull(getMetric("rebalance-latency-avg"));
+        assertNotNull(getMetric("rebalance-latency-max"));
+        assertNotNull(getMetric("rebalance-rate-per-hour"));
+        assertNotNull(getMetric("rebalance-total"));
+        assertNotNull(getMetric("last-rebalance-seconds-ago"));
+        assertNotNull(getMetric("failed-rebalance-rate-per-hour"));
+        assertNotNull(getMetric("failed-rebalance-total"));
+
+        metrics.sensor("heartbeat-latency").record(1.0d);
+        metrics.sensor("heartbeat-latency").record(6.0d);
+        metrics.sensor("heartbeat-latency").record(2.0d);
+
+        assertEquals(6.0d, getMetric("heartbeat-response-time-max").metricValue());
+        assertEquals(0.1d, getMetric("heartbeat-rate").metricValue());
+        assertEquals(3.0d, getMetric("heartbeat-total").metricValue());
+
+        assertEquals(-1.0d, getMetric("last-heartbeat-seconds-ago").metricValue());
+        coordinator.heartbeat().sentHeartbeat(mockTime.milliseconds());
+        assertEquals(0.0d, getMetric("last-heartbeat-seconds-ago").metricValue());
+        mockTime.sleep(10 * 1000L);
+        assertEquals(10.0d, getMetric("last-heartbeat-seconds-ago").metricValue());
+
+        metrics.sensor("join-latency").record(1.0d);
+        metrics.sensor("join-latency").record(6.0d);
+        metrics.sensor("join-latency").record(2.0d);
+
+        assertEquals(3.0d, getMetric("join-time-avg").metricValue());
+        assertEquals(6.0d, getMetric("join-time-max").metricValue());
+        assertEquals(0.1d, getMetric("join-rate").metricValue());
+        assertEquals(3.0d, getMetric("join-total").metricValue());
+
+        metrics.sensor("sync-latency").record(1.0d);
+        metrics.sensor("sync-latency").record(6.0d);
+        metrics.sensor("sync-latency").record(2.0d);
+
+        assertEquals(3.0d, getMetric("sync-time-avg").metricValue());
+        assertEquals(6.0d, getMetric("sync-time-max").metricValue());
+        assertEquals(0.1d, getMetric("sync-rate").metricValue());
+        assertEquals(3.0d, getMetric("sync-total").metricValue());
+
+        metrics.sensor("rebalance-latency").record(1.0d);
+        metrics.sensor("rebalance-latency").record(6.0d);
+        metrics.sensor("rebalance-latency").record(2.0d);
+
+        assertEquals(3.0d, getMetric("rebalance-latency-avg").metricValue());
+        assertEquals(6.0d, getMetric("rebalance-latency-max").metricValue());
+        assertEquals(360.0d, getMetric("rebalance-rate-per-hour").metricValue());
+        assertEquals(3.0d, getMetric("rebalance-total").metricValue());
+
+        metrics.sensor("failed-rebalance").record(1.0d);
+        metrics.sensor("failed-rebalance").record(6.0d);
+        metrics.sensor("failed-rebalance").record(2.0d);
+
+        assertEquals(360.0d, getMetric("failed-rebalance-rate-per-hour").metricValue());
+        assertEquals(3.0d, getMetric("failed-rebalance-total").metricValue());
+
+        assertEquals(-1.0d, getMetric("last-rebalance-seconds-ago").metricValue());
+        coordinator.setLastRebalanceTime(mockTime.milliseconds());
+        assertEquals(0.0d, getMetric("last-rebalance-seconds-ago").metricValue());
+        mockTime.sleep(10 * 1000L);
+        assertEquals(10.0d, getMetric("last-rebalance-seconds-ago").metricValue());
+    }
+
+    private KafkaMetric getMetric(final String name) {
+        return metrics.metrics().get(metrics.metricName(name, "consumer-coordinator-metrics"));
+    }
+
     @Test
     public void testCoordinatorDiscoveryBackoff() {
         setupCoordinator();
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index 91d3529ea10e4..068f7bd8809e3 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -49,7 +49,9 @@
 import org.apache.kafka.common.message.OffsetCommitRequestData;
 import org.apache.kafka.common.message.OffsetCommitResponseData;
 import org.apache.kafka.common.message.SyncGroupResponseData;
+import org.apache.kafka.common.metrics.KafkaMetric;
 import org.apache.kafka.common.metrics.Metrics;
+import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.record.RecordBatch;
 import org.apache.kafka.common.requests.AbstractRequest;
@@ -200,6 +202,58 @@ public void teardown() {
         this.coordinator.close(time.timer(0));
     }
 
+    @Test
+    public void testMetrics() {
+        assertNotNull(getMetric("commit-latency-avg"));
+        assertNotNull(getMetric("commit-latency-max"));
+        assertNotNull(getMetric("commit-rate"));
+        assertNotNull(getMetric("commit-total"));
+        assertNotNull(getMetric("partition-revoked-latency-avg"));
+        assertNotNull(getMetric("partition-revoked-latency-max"));
+        assertNotNull(getMetric("partition-assigned-latency-avg"));
+        assertNotNull(getMetric("partition-assigned-latency-max"));
+        assertNotNull(getMetric("partition-lost-latency-avg"));
+        assertNotNull(getMetric("partition-lost-latency-max"));
+        assertNotNull(getMetric("assigned-partitions"));
+
+        metrics.sensor("commit-latency").record(1.0d);
+        metrics.sensor("commit-latency").record(6.0d);
+        metrics.sensor("commit-latency").record(2.0d);
+
+        assertEquals(3.0d, getMetric("commit-latency-avg").metricValue());
+        assertEquals(6.0d, getMetric("commit-latency-max").metricValue());
+        assertEquals(0.1d, getMetric("commit-rate").metricValue());
+        assertEquals(3.0d, getMetric("commit-total").metricValue());
+
+        metrics.sensor("partition-revoked-latency").record(1.0d);
+        metrics.sensor("partition-revoked-latency").record(2.0d);
+        metrics.sensor("partition-assigned-latency").record(1.0d);
+        metrics.sensor("partition-assigned-latency").record(2.0d);
+        metrics.sensor("partition-lost-latency").record(1.0d);
+        metrics.sensor("partition-lost-latency").record(2.0d);
+
+        assertEquals(1.5d, getMetric("partition-revoked-latency-avg").metricValue());
+        assertEquals(2.0d, getMetric("partition-revoked-latency-max").metricValue());
+        assertEquals(1.5d, getMetric("partition-assigned-latency-avg").metricValue());
+        assertEquals(2.0d, getMetric("partition-assigned-latency-max").metricValue());
+        assertEquals(1.5d, getMetric("partition-lost-latency-avg").metricValue());
+        assertEquals(2.0d, getMetric("partition-lost-latency-max").metricValue());
+
+        assertEquals(0.0d, getMetric("assigned-partitions").metricValue());
+        subscriptions.assignFromUser(Collections.singleton(t1p));
+        assertEquals(1.0d, getMetric("assigned-partitions").metricValue());
+        subscriptions.assignFromUser(Utils.mkSet(t1p, t2p));
+        assertEquals(2.0d, getMetric("assigned-partitions").metricValue());
+    }
+
+    private KafkaMetric getMetric(final String name) {
+        return metrics.metrics().get(metrics.metricName(name, "consumer" + groupId + "-coordinator-metrics"));
+    }
+
+    private Sensor getSensor(final String name) {
+        return metrics.sensor(name);
+    }
+
     @Test
     public void testSelectRebalanceProtcol() {
         List assignors = new ArrayList<>();

From e98e239a0c7fd9a500de606b7a05582f1a384da4 Mon Sep 17 00:00:00 2001
From: Bruno Cadonna 
Date: Mon, 23 Sep 2019 07:12:45 -0700
Subject: [PATCH 0635/1071] KAFKA-8859: Refactor cache-level metrics (#7367)

Cache-level metrics are refactor according to KIP-444:

tag client-id changed to thread-id
name hitRatio changed to hit-ratio
made backward compatible by using streams config built.in.metrics.version

Reviewers: Guozhang Wang , Bill Bejeck 
---
 .../internals/metrics/StreamsMetricsImpl.java |  83 ++++++++++---
 .../streams/state/internals/NamedCache.java   |  88 +++----------
 .../internals/metrics/NamedCacheMetrics.java  |  86 +++++++++++++
 .../integration/MetricsIntegrationTest.java   |  44 +++++--
 .../metrics/StreamsMetricsImplTest.java       |  46 ++++++-
 .../state/internals/NamedCacheTest.java       |  33 +----
 .../metrics/NamedCacheMetricsTest.java        | 117 ++++++++++++++++++
 7 files changed, 367 insertions(+), 130 deletions(-)
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/NamedCacheMetrics.java
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/NamedCacheMetricsTest.java

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
index 1e7c1aee480bd..961bec80044fd 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
@@ -25,6 +25,7 @@
 import org.apache.kafka.common.metrics.stats.CumulativeCount;
 import org.apache.kafka.common.metrics.stats.CumulativeSum;
 import org.apache.kafka.common.metrics.stats.Max;
+import org.apache.kafka.common.metrics.stats.Min;
 import org.apache.kafka.common.metrics.stats.Rate;
 import org.apache.kafka.common.metrics.stats.Value;
 import org.apache.kafka.common.metrics.stats.WindowedCount;
@@ -63,9 +64,11 @@ public enum Version {
     private static final String SENSOR_PREFIX_DELIMITER = ".";
     private static final String SENSOR_NAME_DELIMITER = ".s.";
 
-    public static final String THREAD_ID_TAG = "client-id";
+    public static final String THREAD_ID_TAG = "thread-id";
+    public static final String THREAD_ID_TAG_0100_TO_23 = "client-id";
     public static final String TASK_ID_TAG = "task-id";
     public static final String STORE_ID_TAG = "state-id";
+    public static final String RECORD_CACHE_ID_TAG = "record-cache-id";
 
     public static final String ROLLUP_VALUE = "all";
 
@@ -77,6 +80,11 @@ public enum Version {
     public static final String TOTAL_SUFFIX = "-total";
     public static final String RATIO_SUFFIX = "-ratio";
 
+    public static final String AVG_VALUE_DOC = "The average value of ";
+    public static final String MAX_VALUE_DOC = "The maximum value of ";
+    public static final String AVG_LATENCY_DOC = "The average latency of ";
+    public static final String MAX_LATENCY_DOC = "The maximum latency of ";
+
     public static final String GROUP_PREFIX_WO_DELIMITER = "stream";
     public static final String GROUP_PREFIX = GROUP_PREFIX_WO_DELIMITER + "-";
     public static final String GROUP_SUFFIX = "-metrics";
@@ -84,6 +92,7 @@ public enum Version {
     public static final String THREAD_LEVEL_GROUP = GROUP_PREFIX_WO_DELIMITER + GROUP_SUFFIX;
     public static final String TASK_LEVEL_GROUP = GROUP_PREFIX + "task" + GROUP_SUFFIX;
     public static final String STATE_LEVEL_GROUP = GROUP_PREFIX + "state" + GROUP_SUFFIX;
+    public static final String CACHE_LEVEL_GROUP = GROUP_PREFIX + "record-cache" + GROUP_SUFFIX;
 
     public static final String PROCESSOR_NODE_METRICS_GROUP = "stream-processor-node-metrics";
     public static final String PROCESSOR_NODE_ID_TAG = "processor-node-id";
@@ -130,7 +139,7 @@ private String threadSensorPrefix() {
 
     public Map threadLevelTagMap() {
         final Map tagMap = new LinkedHashMap<>();
-        tagMap.put(THREAD_ID_TAG, threadName);
+        tagMap.put(THREAD_ID_TAG_0100_TO_23, threadName);
         return tagMap;
     }
 
@@ -237,12 +246,12 @@ private String nodeSensorPrefix(final String taskName, final String processorNod
         return taskSensorPrefix(taskName) + SENSOR_PREFIX_DELIMITER + "node" + SENSOR_PREFIX_DELIMITER + processorNodeName;
     }
 
-    public final Sensor cacheLevelSensor(final String taskName,
-                                         final String cacheName,
-                                         final String sensorName,
-                                         final Sensor.RecordingLevel recordingLevel,
-                                         final Sensor... parents) {
-        final String key = cacheSensorPrefix(taskName, cacheName);
+    public Sensor cacheLevelSensor(final String taskName,
+                                   final String storeName,
+                                   final String sensorName,
+                                   final Sensor.RecordingLevel recordingLevel,
+                                   final Sensor... parents) {
+        final String key = cacheSensorPrefix(taskName, storeName);
         synchronized (cacheLevelSensors) {
             if (!cacheLevelSensors.containsKey(key)) {
                 cacheLevelSensors.put(key, new LinkedList<>());
@@ -258,6 +267,18 @@ public final Sensor cacheLevelSensor(final String taskName,
         }
     }
 
+    public Map cacheLevelTagMap(final String taskName, final String storeName) {
+        final Map tagMap = new LinkedHashMap<>();
+        tagMap.put(TASK_ID_TAG, taskName);
+        tagMap.put(RECORD_CACHE_ID_TAG, storeName);
+        if (version == Version.FROM_100_TO_23) {
+            tagMap.put(THREAD_ID_TAG_0100_TO_23, Thread.currentThread().getName());
+        } else {
+            tagMap.put(THREAD_ID_TAG, Thread.currentThread().getName());
+        }
+        return tagMap;
+    }
+
     public final void removeAllCacheLevelSensors(final String taskName, final String cacheName) {
         final String key = cacheSensorPrefix(taskName, cacheName);
         synchronized (cacheLevelSensors) {
@@ -423,15 +444,17 @@ private String externalParentSensorName(final String operationName) {
     }
 
 
-    public static void addAvgAndMaxToSensor(final Sensor sensor,
+    private static void addAvgAndMaxToSensor(final Sensor sensor,
                                             final String group,
                                             final Map tags,
-                                            final String operation) {
+                                            final String operation,
+                                            final String descriptionOfAvg,
+                                            final String descriptionOfMax) {
         sensor.add(
             new MetricName(
                 operation + AVG_SUFFIX,
                 group,
-                "The average value of " + operation + ".",
+                descriptionOfAvg,
                 tags),
             new Avg()
         );
@@ -439,12 +462,26 @@ public static void addAvgAndMaxToSensor(final Sensor sensor,
             new MetricName(
                 operation + MAX_SUFFIX,
                 group,
-                "The max value of " + operation + ".",
+                descriptionOfMax,
                 tags),
             new Max()
         );
     }
 
+    public static void addAvgAndMaxToSensor(final Sensor sensor,
+                                            final String group,
+                                            final Map tags,
+                                            final String operation) {
+        addAvgAndMaxToSensor(
+            sensor,
+            group,
+            tags,
+            operation,
+            AVG_VALUE_DOC + operation + ".",
+            MAX_VALUE_DOC + operation + "."
+        );
+    }
+
     public static void addAvgAndMaxLatencyToSensor(final Sensor sensor,
                                                    final String group,
                                                    final Map tags,
@@ -453,7 +490,7 @@ public static void addAvgAndMaxLatencyToSensor(final Sensor sensor,
             new MetricName(
                 operation + "-latency-avg",
                 group,
-                "The average latency of " + operation + " operation.",
+                AVG_LATENCY_DOC + operation + " operation.",
                 tags),
             new Avg()
         );
@@ -461,12 +498,30 @@ public static void addAvgAndMaxLatencyToSensor(final Sensor sensor,
             new MetricName(
                 operation + "-latency-max",
                 group,
-                "The max latency of " + operation + " operation.",
+                MAX_LATENCY_DOC + operation + " operation.",
                 tags),
             new Max()
         );
     }
 
+    public static void addAvgAndMinAndMaxToSensor(final Sensor sensor,
+                                                  final String group,
+                                                  final Map tags,
+                                                  final String operation,
+                                                  final String descriptionOfAvg,
+                                                  final String descriptionOfMin,
+                                                  final String descriptionOfMax) {
+        addAvgAndMaxToSensor(sensor, group, tags, operation, descriptionOfAvg, descriptionOfMax);
+        sensor.add(
+            new MetricName(
+                operation + MIN_SUFFIX,
+                group,
+                descriptionOfMin,
+                tags),
+            new Min()
+        );
+    }
+
     public static void addInvocationRateAndCountToSensor(final Sensor sensor,
                                                          final String group,
                                                          final Map tags,
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
index a1d0aab277052..2d53440ef4724 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
@@ -19,14 +19,12 @@
 import java.util.NavigableMap;
 import java.util.TreeMap;
 import java.util.TreeSet;
-import org.apache.kafka.common.MetricName;
+
 import org.apache.kafka.common.metrics.Sensor;
-import org.apache.kafka.common.metrics.stats.Avg;
-import org.apache.kafka.common.metrics.stats.Max;
-import org.apache.kafka.common.metrics.stats.Min;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.state.internals.metrics.NamedCacheMetrics;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -34,19 +32,22 @@
 import java.util.Iterator;
 import java.util.LinkedHashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 class NamedCache {
     private static final Logger log = LoggerFactory.getLogger(NamedCache.class);
     private final String name;
+    private final String storeName;
+    private final String taskName;
     private final NavigableMap cache = new TreeMap<>();
     private final Set dirtyKeys = new LinkedHashSet<>();
     private ThreadCache.DirtyEntryFlushListener listener;
     private LRUNode tail;
     private LRUNode head;
     private long currentSizeBytes;
-    private final NamedCacheMetrics namedCacheMetrics;
+
+    private final StreamsMetricsImpl streamsMetrics;
+    private final Sensor hitRatioSensor;
 
     // internal stats
     private long numReadHits = 0;
@@ -54,9 +55,12 @@ class NamedCache {
     private long numOverwrites = 0;
     private long numFlushes = 0;
 
-    NamedCache(final String name, final StreamsMetricsImpl metrics) {
+    NamedCache(final String name, final StreamsMetricsImpl streamsMetrics) {
         this.name = name;
-        this.namedCacheMetrics = new NamedCacheMetrics(metrics, name);
+        this.streamsMetrics = streamsMetrics;
+        storeName = ThreadCache.underlyingStoreNamefromCacheName(name);
+        taskName = ThreadCache.taskIDfromCacheName(name);
+        hitRatioSensor = NamedCacheMetrics.hitRatioSensor(streamsMetrics, taskName, storeName);
     }
 
     synchronized final String name() {
@@ -187,7 +191,7 @@ private LRUNode getInternal(final Bytes key) {
             return null;
         } else {
             numReadHits++;
-            namedCacheMetrics.hitRatioSensor.record((double) numReadHits / (double) (numReadHits + numReadMisses));
+            hitRatioSensor.record((double) numReadHits / (double) (numReadHits + numReadMisses));
         }
         return node;
     }
@@ -311,7 +315,7 @@ synchronized void close() {
         currentSizeBytes = 0;
         dirtyKeys.clear();
         cache.clear();
-        namedCacheMetrics.removeAllSensors();
+        streamsMetrics.removeAllCacheLevelSensors(taskName, storeName);
     }
 
     /**
@@ -357,68 +361,4 @@ private void update(final LRUCacheEntry entry) {
         }
     }
 
-    private static class NamedCacheMetrics {
-        private final StreamsMetricsImpl metrics;
-
-        private final Sensor hitRatioSensor;
-        private final String taskName;
-        private final String cacheName;
-
-        private NamedCacheMetrics(final StreamsMetricsImpl metrics, final String cacheName) {
-            taskName = ThreadCache.taskIDfromCacheName(cacheName);
-            this.cacheName = cacheName;
-            this.metrics = metrics;
-            final String group = "stream-record-cache-metrics";
-
-            // add parent
-            final Map allMetricTags = metrics.tagMap(
-                 "task-id", taskName,
-                "record-cache-id", "all"
-            );
-            final Sensor taskLevelHitRatioSensor = metrics.taskLevelSensor(taskName, "hitRatio", Sensor.RecordingLevel.DEBUG);
-            taskLevelHitRatioSensor.add(
-                new MetricName("hitRatio-avg", group, "The average cache hit ratio.", allMetricTags),
-                new Avg()
-            );
-            taskLevelHitRatioSensor.add(
-                new MetricName("hitRatio-min", group, "The minimum cache hit ratio.", allMetricTags),
-                new Min()
-            );
-            taskLevelHitRatioSensor.add(
-                new MetricName("hitRatio-max", group, "The maximum cache hit ratio.", allMetricTags),
-                new Max()
-            );
-
-            // add child
-            final Map metricTags = metrics.tagMap(
-                 "task-id", taskName,
-                "record-cache-id", ThreadCache.underlyingStoreNamefromCacheName(cacheName)
-            );
-
-            hitRatioSensor = metrics.cacheLevelSensor(
-                taskName,
-                cacheName,
-                "hitRatio",
-                Sensor.RecordingLevel.DEBUG,
-                taskLevelHitRatioSensor
-            );
-            hitRatioSensor.add(
-                new MetricName("hitRatio-avg", group, "The average cache hit ratio.", metricTags),
-                new Avg()
-            );
-            hitRatioSensor.add(
-                new MetricName("hitRatio-min", group, "The minimum cache hit ratio.", metricTags),
-                new Min()
-            );
-            hitRatioSensor.add(
-                new MetricName("hitRatio-max", group, "The maximum cache hit ratio.", metricTags),
-                new Max()
-            );
-
-        }
-
-        private void removeAllSensors() {
-            metrics.removeAllCacheLevelSensors(taskName, cacheName);
-        }
-    }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/NamedCacheMetrics.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/NamedCacheMetrics.java
new file mode 100644
index 0000000000000..3744428ba5828
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/NamedCacheMetrics.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals.metrics;
+
+import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.CACHE_LEVEL_GROUP;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.ROLLUP_VALUE;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.Version.FROM_100_TO_23;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addAvgAndMinAndMaxToSensor;
+
+public class NamedCacheMetrics {
+    private NamedCacheMetrics() {}
+
+    private static final String HIT_RATIO_0100_TO_23 = "hitRatio";
+    private static final String HIT_RATIO = "hit-ratio";
+    private static final String HIT_RATIO_AVG_DESCRIPTION = "The average cache hit ratio";
+    private static final String HIT_RATIO_MIN_DESCRIPTION = "The minimum cache hit ratio";
+    private static final String HIT_RATIO_MAX_DESCRIPTION = "The maximum cache hit ratio";
+
+
+    public static Sensor hitRatioSensor(final StreamsMetricsImpl streamsMetrics,
+                                        final String taskName,
+                                        final String storeName) {
+
+        final Sensor hitRatioSensor;
+        final String hitRatioName;
+        if (streamsMetrics.version() == FROM_100_TO_23) {
+            hitRatioName = HIT_RATIO_0100_TO_23;
+            final Sensor taskLevelHitRatioSensor = streamsMetrics.taskLevelSensor(
+                taskName,
+                hitRatioName,
+                Sensor.RecordingLevel.DEBUG
+            );
+            addAvgAndMinAndMaxToSensor(
+                taskLevelHitRatioSensor,
+                CACHE_LEVEL_GROUP,
+                streamsMetrics.cacheLevelTagMap(taskName, ROLLUP_VALUE),
+                hitRatioName,
+                HIT_RATIO_AVG_DESCRIPTION,
+                HIT_RATIO_MIN_DESCRIPTION,
+                HIT_RATIO_MAX_DESCRIPTION
+            );
+            hitRatioSensor = streamsMetrics.cacheLevelSensor(
+                taskName,
+                storeName,
+                hitRatioName,
+                Sensor.RecordingLevel.DEBUG,
+                taskLevelHitRatioSensor
+            );
+        } else {
+            hitRatioName = HIT_RATIO;
+            hitRatioSensor = streamsMetrics.cacheLevelSensor(
+                taskName,
+                storeName,
+                hitRatioName,
+                Sensor.RecordingLevel.DEBUG
+            );
+        }
+        addAvgAndMinAndMaxToSensor(
+            hitRatioSensor,
+            CACHE_LEVEL_GROUP,
+            streamsMetrics.cacheLevelTagMap(taskName, storeName),
+            hitRatioName,
+            HIT_RATIO_AVG_DESCRIPTION,
+            HIT_RATIO_MIN_DESCRIPTION,
+            HIT_RATIO_MAX_DESCRIPTION
+        );
+        return hitRatioSensor;
+    }
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
index 0104d024e986c..cac33c980f369 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
@@ -151,9 +151,12 @@ public class MetricsIntegrationTest {
     private static final String SKIPPED_RECORDS_TOTAL = "skipped-records-total";
     private static final String RECORD_LATENESS_AVG = "record-lateness-avg";
     private static final String RECORD_LATENESS_MAX = "record-lateness-max";
-    private static final String HIT_RATIO_AVG = "hitRatio-avg";
-    private static final String HIT_RATIO_MIN = "hitRatio-min";
-    private static final String HIT_RATIO_MAX = "hitRatio-max";
+    private static final String HIT_RATIO_AVG_BEFORE_24 = "hitRatio-avg";
+    private static final String HIT_RATIO_MIN_BEFORE_24 = "hitRatio-min";
+    private static final String HIT_RATIO_MAX_BEFORE_24 = "hitRatio-max";
+    private static final String HIT_RATIO_AVG = "hit-ratio-avg";
+    private static final String HIT_RATIO_MIN = "hit-ratio-min";
+    private static final String HIT_RATIO_MAX = "hit-ratio-max";
 
     // RocksDB metrics
     private static final String BYTES_WRITTEN_RATE = "bytes-written-rate";
@@ -275,7 +278,18 @@ private void closeApplication() throws Exception {
     }
 
     @Test
-    public void shouldAddMetricsOnAllLevels() throws Exception {
+    public void shouldAddMetricsOnAllLevelsWithBuiltInMetricsLatestVersion() throws Exception {
+        shouldAddMetricsOnAllLevels(StreamsConfig.METRICS_LATEST);
+    }
+
+    @Test
+    public void shouldAddMetricsOnAllLevelsWithBuiltInMetricsVersion0100To23() throws Exception {
+        shouldAddMetricsOnAllLevels(StreamsConfig.METRICS_0100_TO_23);
+    }
+
+    private void shouldAddMetricsOnAllLevels(final String builtInMetricsVersion) throws Exception {
+        streamsConfiguration.put(StreamsConfig.BUILT_IN_METRICS_VERSION_CONFIG, builtInMetricsVersion);
+
         builder.stream(STREAM_INPUT, Consumed.with(Serdes.Integer(), Serdes.String()))
             .to(STREAM_OUTPUT_1, Produced.with(Serdes.Integer(), Serdes.String()));
         builder.table(STREAM_OUTPUT_1,
@@ -299,7 +313,7 @@ public void shouldAddMetricsOnAllLevels() throws Exception {
         checkKeyValueStoreMetricsByGroup(STREAM_STORE_ROCKSDB_STATE_METRICS);
         checkKeyValueStoreMetricsByGroup(STREAM_STORE_IN_MEMORY_LRU_STATE_METRICS);
         checkRocksDBMetricsByTag("rocksdb-state-id");
-        checkCacheMetrics();
+        checkCacheMetrics(builtInMetricsVersion);
 
         closeApplication();
 
@@ -526,13 +540,25 @@ private void checkMetricsDeregistration() {
         assertThat(listMetricAfterClosingApp.size(), is(0));
     }
 
-    private void checkCacheMetrics() {
+    private void checkCacheMetrics(final String builtInMetricsVersion) {
         final List listMetricCache = new ArrayList(kafkaStreams.metrics().values()).stream()
             .filter(m -> m.metricName().group().equals(STREAM_CACHE_NODE_METRICS))
             .collect(Collectors.toList());
-        checkMetricByName(listMetricCache, HIT_RATIO_AVG, 6);
-        checkMetricByName(listMetricCache, HIT_RATIO_MIN, 6);
-        checkMetricByName(listMetricCache, HIT_RATIO_MAX, 6);
+        checkMetricByName(
+            listMetricCache,
+            builtInMetricsVersion.equals(StreamsConfig.METRICS_LATEST) ? HIT_RATIO_AVG : HIT_RATIO_AVG_BEFORE_24,
+            builtInMetricsVersion.equals(StreamsConfig.METRICS_LATEST) ? 3 : 6 /* includes parent sensors */
+        );
+        checkMetricByName(
+            listMetricCache,
+            builtInMetricsVersion.equals(StreamsConfig.METRICS_LATEST) ? HIT_RATIO_MIN : HIT_RATIO_MIN_BEFORE_24,
+            builtInMetricsVersion.equals(StreamsConfig.METRICS_LATEST) ? 3 : 6 /* includes parent sensors */
+        );
+        checkMetricByName(
+            listMetricCache,
+            builtInMetricsVersion.equals(StreamsConfig.METRICS_LATEST) ? HIT_RATIO_MAX : HIT_RATIO_MAX_BEFORE_24,
+            builtInMetricsVersion.equals(StreamsConfig.METRICS_LATEST) ? 3 : 6 /* includes parent sensors */
+        );
     }
 
     private void checkWindowStoreMetrics() {
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java
index f1d951bab968a..b589119766166 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java
@@ -62,6 +62,7 @@ public class StreamsMetricsImplTest extends EasyMockSupport {
     private final Map tags = mkMap(mkEntry("tag", "value"));
     private final String description1 = "description number one";
     private final String description2 = "description number two";
+    private final String description3 = "description number three";
     private final MockTime time = new MockTime(0);
     private final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, THREAD_NAME, VERSION);
 
@@ -252,11 +253,40 @@ public void shouldGetStoreLevelTagMap() {
         final Map tagMap = streamsMetrics.storeLevelTagMap(taskName, storeType, storeName);
 
         assertThat(tagMap.size(), equalTo(3));
-        assertThat(tagMap.get(StreamsMetricsImpl.THREAD_ID_TAG), equalTo(THREAD_NAME));
+        assertThat(tagMap.get(StreamsMetricsImpl.THREAD_ID_TAG_0100_TO_23), equalTo(THREAD_NAME));
         assertThat(tagMap.get(StreamsMetricsImpl.TASK_ID_TAG), equalTo(taskName));
         assertThat(tagMap.get(storeType + "-" + StreamsMetricsImpl.STORE_ID_TAG), equalTo(storeName));
     }
 
+    @Test
+    public void shouldGetCacheLevelTagMapForBuiltInMetricsLatestVersion() {
+        shouldGetCacheLevelTagMap(StreamsConfig.METRICS_LATEST);
+    }
+
+    @Test
+    public void shouldGetCacheLevelTagMapForBuiltInMetricsVersion0100To23() {
+        shouldGetCacheLevelTagMap(StreamsConfig.METRICS_0100_TO_23);
+    }
+
+    private void shouldGetCacheLevelTagMap(final String builtInMetricsVersion) {
+        final StreamsMetricsImpl streamsMetrics =
+            new StreamsMetricsImpl(metrics, THREAD_NAME, builtInMetricsVersion);
+        final String taskName = "taskName";
+        final String storeName = "storeName";
+
+        final Map tagMap = streamsMetrics.cacheLevelTagMap(taskName, storeName);
+
+        assertThat(tagMap.size(), equalTo(3));
+        assertThat(
+            tagMap.get(
+                builtInMetricsVersion.equals(StreamsConfig.METRICS_LATEST) ? StreamsMetricsImpl.THREAD_ID_TAG
+                    : StreamsMetricsImpl.THREAD_ID_TAG_0100_TO_23),
+            equalTo(Thread.currentThread().getName())
+        );
+        assertThat(tagMap.get(StreamsMetricsImpl.TASK_ID_TAG), equalTo(taskName));
+        assertThat(tagMap.get(StreamsMetricsImpl.RECORD_CACHE_ID_TAG), equalTo(storeName));
+    }
+
     @Test
     public void shouldAddAmountRateAndSum() {
         StreamsMetricsImpl
@@ -325,6 +355,20 @@ public void shouldAddAvgAndTotalMetricsToSensor() {
         assertThat(metrics.metrics().size(), equalTo(2 + 1)); // one metric is added automatically in the constructor of Metrics
     }
 
+    @Test
+    public void shouldAddAvgAndMinAndMaxMetricsToSensor() {
+        StreamsMetricsImpl
+            .addAvgAndMinAndMaxToSensor(sensor, group, tags, metricNamePrefix, description1, description2, description3);
+
+        final double valueToRecord1 = 18.0;
+        final double valueToRecord2 = 42.0;
+        final double expectedAvgMetricValue = (valueToRecord1 + valueToRecord2) / 2;
+        verifyMetric(metricNamePrefix + "-avg", description1, valueToRecord1, valueToRecord2, expectedAvgMetricValue);
+        verifyMetric(metricNamePrefix + "-min", description2, valueToRecord1, valueToRecord2, valueToRecord1);
+        verifyMetric(metricNamePrefix + "-max", description3, valueToRecord1, valueToRecord2, valueToRecord2);
+        assertThat(metrics.metrics().size(), equalTo(3 + 1)); // one metric is added automatically in the constructor of Metrics
+    }
+
     @Test
     public void shouldReturnMetricsVersionCurrent() {
         assertThat(
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/NamedCacheTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/NamedCacheTest.java
index 6c82209c33aca..ea52b4e82d860 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/NamedCacheTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/NamedCacheTest.java
@@ -20,7 +20,6 @@
 import org.apache.kafka.common.header.Headers;
 import org.apache.kafka.common.header.internals.RecordHeader;
 import org.apache.kafka.common.header.internals.RecordHeaders;
-import org.apache.kafka.common.metrics.JmxReporter;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.streams.KeyValue;
@@ -29,20 +28,15 @@
 import org.junit.Before;
 import org.junit.Test;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Map;
 
-import static org.apache.kafka.test.StreamsTestUtils.getMetricByNameFilterByTags;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertTrue;
 
 public class NamedCacheTest {
 
@@ -61,7 +55,7 @@ public void setUp() {
     }
 
     @Test
-    public void shouldKeepTrackOfMostRecentlyAndLeastRecentlyUsed() throws IOException {
+    public void shouldKeepTrackOfMostRecentlyAndLeastRecentlyUsed() {
         final List> toInsert = Arrays.asList(
                 new KeyValue<>("K1", "V1"),
                 new KeyValue<>("K2", "V2"),
@@ -83,31 +77,6 @@ public void shouldKeepTrackOfMostRecentlyAndLeastRecentlyUsed() throws IOExcepti
         }
     }
 
-    @Test
-    public void testMetrics() {
-        final Map metricTags = new LinkedHashMap<>();
-        metricTags.put("record-cache-id", underlyingStoreName);
-        metricTags.put("task-id", taskIDString);
-        metricTags.put("client-id", "test");
-
-        getMetricByNameFilterByTags(metrics.metrics(), "hitRatio-avg", "stream-record-cache-metrics", metricTags);
-        getMetricByNameFilterByTags(metrics.metrics(), "hitRatio-min", "stream-record-cache-metrics", metricTags);
-        getMetricByNameFilterByTags(metrics.metrics(), "hitRatio-max", "stream-record-cache-metrics", metricTags);
-
-        // test "all"
-        metricTags.put("record-cache-id", "all");
-        getMetricByNameFilterByTags(metrics.metrics(), "hitRatio-avg", "stream-record-cache-metrics", metricTags);
-        getMetricByNameFilterByTags(metrics.metrics(), "hitRatio-min", "stream-record-cache-metrics", metricTags);
-        getMetricByNameFilterByTags(metrics.metrics(), "hitRatio-max", "stream-record-cache-metrics", metricTags);
-
-        final JmxReporter reporter = new JmxReporter("kafka.streams");
-        innerMetrics.addReporter(reporter);
-        assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-record-cache-metrics,client-id=test,task-id=%s,record-cache-id=%s",
-                taskIDString, underlyingStoreName)));
-        assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-record-cache-metrics,client-id=test,task-id=%s,record-cache-id=%s",
-                taskIDString, "all")));
-    }
-
     @Test
     public void shouldKeepTrackOfSize() {
         final LRUCacheEntry value = new LRUCacheEntry(new byte[]{0});
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/NamedCacheMetricsTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/NamedCacheMetricsTest.java
new file mode 100644
index 0000000000000..46ea302738bea
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/NamedCacheMetricsTest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals.metrics;
+
+import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.metrics.Sensor.RecordingLevel;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.Version;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.util.Map;
+
+import static org.apache.kafka.common.utils.Utils.mkEntry;
+import static org.apache.kafka.common.utils.Utils.mkMap;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.mock;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.powermock.api.easymock.PowerMock.createMock;
+import static org.powermock.api.easymock.PowerMock.mockStatic;
+import static org.powermock.api.easymock.PowerMock.replay;
+import static org.powermock.api.easymock.PowerMock.verify;
+
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({StreamsMetricsImpl.class, Sensor.class})
+public class NamedCacheMetricsTest {
+
+    private static final String TASK_NAME = "taskName";
+    private static final String STORE_NAME = "storeName";
+    private static final String HIT_RATIO_AVG_DESCRIPTION = "The average cache hit ratio";
+    private static final String HIT_RATIO_MIN_DESCRIPTION = "The minimum cache hit ratio";
+    private static final String HIT_RATIO_MAX_DESCRIPTION = "The maximum cache hit ratio";
+
+    private final StreamsMetricsImpl streamsMetrics = createMock(StreamsMetricsImpl.class);
+    private final Sensor expectedSensor = mock(Sensor.class);
+    private final Map tagMap = mkMap(mkEntry("key", "value"));
+
+    @Test
+    public void shouldGetHitRatioSensorWithBuiltInMetricsVersionCurrent() {
+        final String hitRatio = "hit-ratio";
+        mockStatic(StreamsMetricsImpl.class);
+        setUpStreamsMetrics(Version.LATEST, hitRatio);
+        replay(streamsMetrics);
+        replay(StreamsMetricsImpl.class);
+
+        final Sensor sensor = NamedCacheMetrics.hitRatioSensor(streamsMetrics, TASK_NAME, STORE_NAME);
+
+        verifyResult(sensor);
+    }
+
+    @Test
+    public void shouldGetHitRatioSensorWithBuiltInMetricsVersionBefore24() {
+        final Map parentTagMap = mkMap(mkEntry("key", "all"));
+        final String hitRatio = "hitRatio";
+        final RecordingLevel recordingLevel = RecordingLevel.DEBUG;
+        mockStatic(StreamsMetricsImpl.class);
+        final Sensor parentSensor = mock(Sensor.class);
+        expect(streamsMetrics.taskLevelSensor(TASK_NAME, hitRatio, recordingLevel)).andReturn(parentSensor);
+        expect(streamsMetrics.cacheLevelTagMap(TASK_NAME, StreamsMetricsImpl.ROLLUP_VALUE)).andReturn(parentTagMap);
+        StreamsMetricsImpl.addAvgAndMinAndMaxToSensor(
+            parentSensor,
+            StreamsMetricsImpl.CACHE_LEVEL_GROUP,
+            parentTagMap,
+            hitRatio,
+            HIT_RATIO_AVG_DESCRIPTION,
+            HIT_RATIO_MIN_DESCRIPTION,
+            HIT_RATIO_MAX_DESCRIPTION);
+        setUpStreamsMetrics(Version.FROM_100_TO_23, hitRatio, parentSensor);
+        replay(streamsMetrics);
+        replay(StreamsMetricsImpl.class);
+
+        final Sensor sensor = NamedCacheMetrics.hitRatioSensor(streamsMetrics, TASK_NAME, STORE_NAME);
+
+        verifyResult(sensor);
+    }
+
+    private void setUpStreamsMetrics(final Version builtInMetricsVersion,
+                                     final String hitRatio,
+                                     final Sensor... parents) {
+        expect(streamsMetrics.version()).andReturn(builtInMetricsVersion);
+        expect(streamsMetrics.cacheLevelSensor(TASK_NAME, STORE_NAME, hitRatio, RecordingLevel.DEBUG, parents))
+            .andReturn(expectedSensor);
+        expect(streamsMetrics.cacheLevelTagMap(TASK_NAME, STORE_NAME)).andReturn(tagMap);
+        StreamsMetricsImpl.addAvgAndMinAndMaxToSensor(
+            expectedSensor,
+            StreamsMetricsImpl.CACHE_LEVEL_GROUP,
+            tagMap,
+            hitRatio,
+            HIT_RATIO_AVG_DESCRIPTION,
+            HIT_RATIO_MIN_DESCRIPTION,
+            HIT_RATIO_MAX_DESCRIPTION);
+    }
+
+    private void verifyResult(final Sensor sensor) {
+        verify(streamsMetrics);
+        verify(StreamsMetricsImpl.class);
+        assertThat(sensor, is(expectedSensor));
+    }
+}
\ No newline at end of file

From b4035610b8ec77ab152c89a655f3ba0453c39cdd Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Mon, 23 Sep 2019 09:24:32 -0700
Subject: [PATCH 0636/1071] KAFKA-8086: Use 1 partition for offset topic when
 possible (#7356)

I realized some flaky tests failed at setup or calls that tries to create offset topics, and I think using one partition and one replica would be sufficient in these cases.

Reviewers: Bill Bejeck 
---
 .../integration/kafka/api/AuthorizerIntegrationTest.scala     | 1 +
 .../kafka/network/DynamicConnectionQuotaTest.scala            | 4 ----
 .../test/scala/unit/kafka/server/MetadataRequestTest.scala    | 1 +
 3 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
index af007da962d23..01021fce16a31 100644
--- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
@@ -132,6 +132,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
     properties.put(KafkaConfig.AuthorizerClassNameProp, classOf[SimpleAclAuthorizer].getName)
     properties.put(KafkaConfig.BrokerIdProp, brokerId.toString)
     properties.put(KafkaConfig.OffsetsTopicPartitionsProp, "1")
+    properties.put(KafkaConfig.OffsetsTopicReplicationFactorProp, "1")
     properties.put(KafkaConfig.TransactionsTopicPartitionsProp, "1")
     properties.put(KafkaConfig.TransactionsTopicReplicationFactorProp, "1")
     properties.put(KafkaConfig.TransactionsTopicMinISRProp, "1")
diff --git a/core/src/test/scala/integration/kafka/network/DynamicConnectionQuotaTest.scala b/core/src/test/scala/integration/kafka/network/DynamicConnectionQuotaTest.scala
index 871953addd3d8..16e3cbc9260c7 100644
--- a/core/src/test/scala/integration/kafka/network/DynamicConnectionQuotaTest.scala
+++ b/core/src/test/scala/integration/kafka/network/DynamicConnectionQuotaTest.scala
@@ -65,10 +65,6 @@ class DynamicConnectionQuotaTest extends BaseRequestTest {
     }
   }
 
-  override protected def brokerPropertyOverrides(properties: Properties): Unit = {
-    super.brokerPropertyOverrides(properties)
-  }
-
   @Test
   def testDynamicConnectionQuota(): Unit = {
     val maxConnectionsPerIP = 5
diff --git a/core/src/test/scala/unit/kafka/server/MetadataRequestTest.scala b/core/src/test/scala/unit/kafka/server/MetadataRequestTest.scala
index c51444cbe9d07..2b3f321819085 100644
--- a/core/src/test/scala/unit/kafka/server/MetadataRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/MetadataRequestTest.scala
@@ -36,6 +36,7 @@ import scala.collection.Seq
 class MetadataRequestTest extends BaseRequestTest {
 
   override def brokerPropertyOverrides(properties: Properties): Unit = {
+    properties.setProperty(KafkaConfig.OffsetsTopicPartitionsProp, "1")
     properties.setProperty(KafkaConfig.DefaultReplicationFactorProp, "2")
     properties.setProperty(KafkaConfig.RackProp, s"rack/${properties.getProperty(KafkaConfig.BrokerIdProp)}")
   }

From beac4c7534ef0cd7be92514a8d92639bf417221c Mon Sep 17 00:00:00 2001
From: Florian Hussonnois 
Date: Mon, 23 Sep 2019 19:11:56 +0200
Subject: [PATCH 0637/1071] KAFKA-6958: Overload methods for group and windowed
 stream to allow to name operation name using the new Named class (#6413)

This is the last PR for the KIP-307.
NOTE : PR 6412 should be merge first
Thanks a lot for the review.

Reviewers: John Roesler , Bill Bejeck 
---
 .../kafka/streams/kstream/KGroupedStream.java | 200 ++++++++++++
 .../kafka/streams/kstream/KGroupedTable.java  | 307 ++++++++++++++++++
 .../kstream/SessionWindowedKStream.java       | 259 +++++++++++++++
 .../streams/kstream/TimeWindowedKStream.java  | 278 ++++++++++++++++
 .../GroupedStreamAggregateBuilder.java        |   4 +-
 .../internals/InternalStreamsBuilder.java     |  26 +-
 .../kstream/internals/KGroupedStreamImpl.java |  46 ++-
 .../kstream/internals/KGroupedTableImpl.java  |  77 ++++-
 .../internals/SessionWindowedKStreamImpl.java |  63 +++-
 .../internals/TimeWindowedKStreamImpl.java    |  63 +++-
 .../kafka/streams/StreamsBuilderTest.java     |  24 +-
 .../SessionWindowedKStreamImplTest.java       |  13 +-
 .../TimeWindowedKStreamImplTest.java          |  14 +-
 13 files changed, 1320 insertions(+), 54 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java
index 7675bcb051d62..6b3c42302a1ab 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java
@@ -68,6 +68,35 @@ public interface KGroupedStream {
      */
     KTable count();
 
+    /**
+     * Count the number of records in this stream by the grouped key.
+     * Records with {@code null} key or value are ignored.
+     * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view).
+     * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream.
+     * 
                
+     * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
+     * the same key.
+     * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+     * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
+     * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
+     * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}.
+     * 
                
+     * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+     * The changelog topic will be named "${applicationId}-${internalStoreName}-changelog", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name
+     * and "-changelog" is a fixed suffix.
+     * Note that the internal store name may not be queriable through Interactive Queries.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
+     * @param named  a {@link Named} config used to name the processor in the topology
+     *
+     * @return a {@link KTable} that contains "update" records with unmodified keys and {@link Long} values that
+     * represent the latest (rolling) count (i.e., number of records) for each key
+     */
+    KTable count(final Named named);
+
     /**
      * Count the number of records in this stream by the grouped key.
      * Records with {@code null} key or value are ignored.
@@ -113,6 +142,53 @@ public interface KGroupedStream {
      */
     KTable count(final Materialized> materialized);
 
+    /**
+     * Count the number of records in this stream by the grouped key.
+     * Records with {@code null} key or value are ignored.
+     * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view)
+     * provided by the given store name in {@code materialized}.
+     * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream.
+     * 
                
+     * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
+     * the same key.
+     * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+     * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
+     * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
+     * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}.
+     * 
                
+     * To query the local {@link KeyValueStore} it must be obtained via
+     * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}.
+     * 
                {@code
+     * KafkaStreams streams = ... // counting words
+     * String queryableStoreName = "storeName"; // the store name should be the name of the store as defined by the Materialized instance
+     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
+     * String key = "some-word";
+     * Long countForWord = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
+     * }
                
+     * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
+     * query the value of the key on a parallel running instance of your Kafka Streams application.
+     *
+     * 
                
+     * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+     * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII
+     * alphanumerics, '.', '_' and '-'.
+     * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the
+     * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
+     * @param named         a {@link Named} config used to name the processor in the topology
+     * @param materialized  an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}.
+     *                      Note: the valueSerde will be automatically set to {@link org.apache.kafka.common.serialization.Serdes#Long() Serdes#Long()}
+     *                      if there is no valueSerde provided
+     * @return a {@link KTable} that contains "update" records with unmodified keys and {@link Long} values that
+     * represent the latest (rolling) count (i.e., number of records) for each key
+     */
+    KTable count(final Named named,
+                          final Materialized> materialized);
+
     /**
      * Combine the values of records in this stream by the grouped key.
      * Records with {@code null} key or value are ignored.
@@ -150,6 +226,68 @@ public interface KGroupedStream {
      */
     KTable reduce(final Reducer reducer);
 
+    /**
+     * Combine the value of records in this stream by the grouped key.
+     * Records with {@code null} key or value are ignored.
+     * Combining implies that the type of the aggregate result is the same as the type of the input value
+     * (c.f. {@link #aggregate(Initializer, Aggregator, Materialized)}).
+     * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view)
+     * provided by the given store name in {@code materialized}.
+     * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream.
+     * 
                
+     * The specified {@link Reducer} is applied for each input record and computes a new aggregate using the current
+     * aggregate (first argument) and the record's value (second argument):
+     * 
                {@code
+     * // At the example of a Reducer
+     * new Reducer() {
+     *   public Long apply(Long aggValue, Long currValue) {
+     *     return aggValue + currValue;
+     *   }
+     * }
+     * }
                
+     * 
                
+     * If there is no current aggregate the {@link Reducer} is not applied and the new aggregate will be the record's
+     * value as-is.
+     * Thus, {@code reduce(Reducer, Materialized)} can be used to compute aggregate functions like sum, min, or
+     * max.
+     * 
                
+     * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
+     * the same key.
+     * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+     * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
+     * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
+     * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}.
+     * 
                
+     * To query the local {@link KeyValueStore} it must be obtained via
+     * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}.
+     * 
                {@code
+     * KafkaStreams streams = ... // compute sum
+     * String queryableStoreName = "storeName" // the store name should be the name of the store as defined by the Materialized instance
+     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
+     * String key = "some-key";
+     * Long sumForKey = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
+     * }
                
+     * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
+     * query the value of the key on a parallel running instance of your Kafka Streams application.
+     *
+     * 
                
+     * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+     * The changelog topic will be named "${applicationId}-${internalStoreName}-changelog", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name
+     * and "-changelog" is a fixed suffix.
+     * Note that the internal store name may not be queriable through Interactive Queries.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
+     * @param reducer       a {@link Reducer} that computes a new aggregate result. Cannot be {@code null}.
+     * @param materialized  an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}.
+     * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the
+     * latest (rolling) aggregate for each key
+     */
+    KTable reduce(final Reducer reducer,
+                        final Materialized> materialized);
+
 
     /**
      * Combine the value of records in this stream by the grouped key.
@@ -206,6 +344,7 @@ public interface KGroupedStream {
      * You can retrieve all generated internal topic names via {@link Topology#describe()}.
      *
      * @param reducer       a {@link Reducer} that computes a new aggregate result. Cannot be {@code null}.
+     * @param named         a {@link Named} config used to name the processor in the topology.
      * @param materialized  an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}.
      * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the
      * latest (rolling) aggregate for each key. If the reduce function returns {@code null}, it is then interpreted as
@@ -213,6 +352,7 @@ public interface KGroupedStream {
      * will be handled as newly initialized value.
      */
     KTable reduce(final Reducer reducer,
+                        final Named named,
                         final Materialized> materialized);
 
     /**
@@ -312,12 +452,72 @@  KTable aggregate(final Initializer initializer,
      * @param materialized  an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}.
      * @param           the value type of the resulting {@link KTable}
      * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the
+     * latest (rolling) aggregate for each key
+     */
+     KTable aggregate(final Initializer initializer,
+                                 final Aggregator aggregator,
+                                 final Materialized> materialized);
+
+    /**
+     * Aggregate the values of records in this stream by the grouped key.
+     * Records with {@code null} key or value are ignored.
+     * Aggregating is a generalization of {@link #reduce(Reducer) combining via reduce(...)} as it, for example,
+     * allows the result to have a different type than the input values.
+     * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view)
+     * that can be queried by the given store name in {@code materialized}.
+     * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream.
+     * 
                
+     * The specified {@link Initializer} is applied once directly before the first input record is processed to
+     * provide an initial intermediate aggregation result that is used to process the first record.
+     * The specified {@link Aggregator} is applied for each input record and computes a new aggregate using the current
+     * aggregate (or for the very first record using the intermediate aggregation result provided via the
+     * {@link Initializer}) and the record's value.
+     * Thus, {@code aggregate(Initializer, Aggregator, Materialized)} can be used to compute aggregate functions like
+     * count (c.f. {@link #count()}).
+     * 
                
+     * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
+     * the same key.
+     * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+     * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
+     * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
+     * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}.
+     * 
                
+     * To query the local {@link KeyValueStore} it must be obtained via
+     * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}:
+     * 
                {@code
+     * KafkaStreams streams = ... // some aggregation on value type double
+     * String queryableStoreName = "storeName" // the store name should be the name of the store as defined by the Materialized instance
+     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
+     * String key = "some-key";
+     * Long aggForKey = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
+     * }
                
+     * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
+     * query the value of the key on a parallel running instance of your Kafka Streams application.
+     *
+     * 
                
+     * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+     * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII
+     * alphanumerics, '.', '_' and '-'.
+     * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the
+     * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
+     * @param initializer   an {@link Initializer} that computes an initial intermediate aggregation result
+     * @param aggregator    an {@link Aggregator} that computes a new aggregate result
+     * @param named         a {@link Named} config used to name the processor in the topology
+     * @param materialized  an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}.
+     * @param           the value type of the resulting {@link KTable}
+     * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the
      * latest (rolling) aggregate for each key. If the aggregate function returns {@code null}, it is then interpreted as
      * deletion for the key, and future messages of the same key coming from upstream operators
      * will be handled as newly initialized value.
      */
      KTable aggregate(final Initializer initializer,
                                  final Aggregator aggregator,
+                                 final Named named,
                                  final Materialized> materialized);
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java
index 74fb88055bec8..6eb70d9f0b9ea 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java
@@ -82,6 +82,50 @@ public interface KGroupedTable {
      */
     KTable count(final Materialized> materialized);
 
+    /**
+     * Count number of records of the original {@link KTable} that got {@link KTable#groupBy(KeyValueMapper) mapped} to
+     * the same key into a new instance of {@link KTable}.
+     * Records with {@code null} key are ignored.
+     * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view)
+     * that can be queried using the provided {@code queryableStoreName}.
+     * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream.
+     * 
                
+     * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
+     * the same key.
+     * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+     * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
+     * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
+     * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}.
+     * 
                
+     * To query the local {@link KeyValueStore} it must be obtained via
+     * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}:
+     * 
                {@code
+     * KafkaStreams streams = ... // counting words
+     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
+     * String key = "some-word";
+     * Long countForWord = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
+     * }
                
+     * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
+     * query the value of the key on a parallel running instance of your Kafka Streams application.
+     *
+     * 
                
+     * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+     * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII
+     * alphanumerics, '.', '_' and '-'.
+     * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the
+     * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
+     * @param named        the {@link Named} config used to name the processor in the topology
+     * @param materialized the instance of {@link Materialized} used to materialize the state store. Cannot be {@code null}
+     * @return a {@link KTable} that contains "update" records with unmodified keys and {@link Long} values that
+     * represent the latest (rolling) count (i.e., number of records) for each key
+     */
+    KTable count(final Named named, final Materialized> materialized);
+
     /**
      * Count number of records of the original {@link KTable} that got {@link KTable#groupBy(KeyValueMapper) mapped} to
      * the same key into a new instance of {@link KTable}.
@@ -110,6 +154,36 @@ public interface KGroupedTable {
      */
     KTable count();
 
+
+    /**
+     * Count number of records of the original {@link KTable} that got {@link KTable#groupBy(KeyValueMapper) mapped} to
+     * the same key into a new instance of {@link KTable}.
+     * Records with {@code null} key are ignored.
+     * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view)
+     * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream.
+     * 
                
+     * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
+     * the same key.
+     * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+     * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
+     * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
+     * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}.
+     * 
                
+     * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+     * The changelog topic will be named "${applicationId}-${internalStoreName}-changelog", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name
+     * and "-changelog" is a fixed suffix.
+     * Note that the internal store name may not be queriable through Interactive Queries.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
+     * @param named        the {@link Named} config used to name the processor in the topology
+     * @return a {@link KTable} that contains "update" records with unmodified keys and {@link Long} values that
+     * represent the latest (rolling) count (i.e., number of records) for each key
+     */
+    KTable count(final Named named);
+
     /**
      * Combine the value of records of the original {@link KTable} that got {@link KTable#groupBy(KeyValueMapper)
      * mapped} to the same key into a new instance of {@link KTable}.
@@ -181,6 +255,82 @@ public interface KGroupedTable {
     KTable reduce(final Reducer adder,
                         final Reducer subtractor,
                         final Materialized> materialized);
+
+
+    /**
+     * Combine the value of records of the original {@link KTable} that got {@link KTable#groupBy(KeyValueMapper)
+     * mapped} to the same key into a new instance of {@link KTable}.
+     * Records with {@code null} key are ignored.
+     * Combining implies that the type of the aggregate result is the same as the type of the input value
+     * (c.f. {@link #aggregate(Initializer, Aggregator, Aggregator, Materialized)}).
+     * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view)
+     * that can be queried using the provided {@code queryableStoreName}.
+     * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream.
+     * 
                
+     * Each update to the original {@link KTable} results in a two step update of the result {@link KTable}.
+     * The specified {@link Reducer adder} is applied for each update record and computes a new aggregate using the
+     * current aggregate (first argument) and the record's value (second argument) by adding the new record to the
+     * aggregate.
+     * The specified {@link Reducer subtractor} is applied for each "replaced" record of the original {@link KTable}
+     * and computes a new aggregate using the current aggregate (first argument) and the record's value (second
+     * argument) by "removing" the "replaced" record from the aggregate.
+     * If there is no current aggregate the {@link Reducer} is not applied and the new aggregate will be the record's
+     * value as-is.
+     * Thus, {@code reduce(Reducer, Reducer, String)} can be used to compute aggregate functions like sum.
+     * For sum, the adder and subtractor would work as follows:
+     * 
                {@code
+     * public class SumAdder implements Reducer {
+     *   public Integer apply(Integer currentAgg, Integer newValue) {
+     *     return currentAgg + newValue;
+     *   }
+     * }
+     *
+     * public class SumSubtractor implements Reducer {
+     *   public Integer apply(Integer currentAgg, Integer oldValue) {
+     *     return currentAgg - oldValue;
+     *   }
+     * }
+     * }
                
+     * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
+     * the same key.
+     * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+     * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
+     * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
+     * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}.
+     * 
                
+     * To query the local {@link KeyValueStore} it must be obtained via
+     * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}:
+     * 
                {@code
+     * KafkaStreams streams = ... // counting words
+     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
+     * String key = "some-word";
+     * Long countForWord = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
+     * }
                
+     * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
+     * query the value of the key on a parallel running instance of your Kafka Streams application.
+     * 
                
+     * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+     * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII
+     * alphanumerics, '.', '_' and '-'.
+     * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the
+     * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
+     * @param adder         a {@link Reducer} that adds a new value to the aggregate result
+     * @param subtractor    a {@link Reducer} that removed an old value from the aggregate result
+     * @param named         a {@link Named} config used to name the processor in the topology
+     * @param materialized  the instance of {@link Materialized} used to materialize the state store. Cannot be {@code null}
+     * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the
+     * latest (rolling) aggregate for each key
+     */
+    KTable reduce(final Reducer adder,
+                        final Reducer subtractor,
+                        final Named named,
+                        final Materialized> materialized);
+
     /**
      * Combine the value of records of the original {@link KTable} that got {@link KTable#groupBy(KeyValueMapper)
      * mapped} to the same key into a new instance of {@link KTable}.
@@ -320,6 +470,92 @@  KTable aggregate(final Initializer initializer,
                                  final Aggregator subtractor,
                                  final Materialized> materialized);
 
+
+    /**
+     * Aggregate the value of records of the original {@link KTable} that got {@link KTable#groupBy(KeyValueMapper)
+     * mapped} to the same key into a new instance of {@link KTable}.
+     * Records with {@code null} key are ignored.
+     * Aggregating is a generalization of {@link #reduce(Reducer, Reducer, Materialized) combining via reduce(...)} as it,
+     * for example, allows the result to have a different type than the input values.
+     * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view)
+     * that can be queried using the provided {@code queryableStoreName}.
+     * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream.
+     * 
                
+     * The specified {@link Initializer} is applied once directly before the first input record is processed to
+     * provide an initial intermediate aggregation result that is used to process the first record.
+     * Each update to the original {@link KTable} results in a two step update of the result {@link KTable}.
+     * The specified {@link Aggregator adder} is applied for each update record and computes a new aggregate using the
+     * current aggregate (or for the very first record using the intermediate aggregation result provided via the
+     * {@link Initializer}) and the record's value by adding the new record to the aggregate.
+     * The specified {@link Aggregator subtractor} is applied for each "replaced" record of the original {@link KTable}
+     * and computes a new aggregate using the current aggregate and the record's value by "removing" the "replaced"
+     * record from the aggregate.
+     * Thus, {@code aggregate(Initializer, Aggregator, Aggregator, Materialized)} can be used to compute aggregate functions
+     * like sum.
+     * For sum, the initializer, adder, and subtractor would work as follows:
+     * 
                {@code
+     * // in this example, LongSerde.class must be set as value serde in Materialized#withValueSerde
+     * public class SumInitializer implements Initializer {
+     *   public Long apply() {
+     *     return 0L;
+     *   }
+     * }
+     *
+     * public class SumAdder implements Aggregator {
+     *   public Long apply(String key, Integer newValue, Long aggregate) {
+     *     return aggregate + newValue;
+     *   }
+     * }
+     *
+     * public class SumSubtractor implements Aggregator {
+     *   public Long apply(String key, Integer oldValue, Long aggregate) {
+     *     return aggregate - oldValue;
+     *   }
+     * }
+     * }
                
+     * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
+     * the same key.
+     * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+     * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
+     * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
+     * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}.
+     * 
                
+     * To query the local {@link KeyValueStore} it must be obtained via
+     * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}:
+     * 
                {@code
+     * KafkaStreams streams = ... // counting words
+     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
+     * String key = "some-word";
+     * Long countForWord = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
+     * }
                
+     * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
+     * query the value of the key on a parallel running instance of your Kafka Streams application.
+     * 
                
+     * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+     * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII
+     * alphanumerics, '.', '_' and '-'.
+     * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the
+     * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
+     * @param initializer   an {@link Initializer} that provides an initial aggregate result value
+     * @param adder         an {@link Aggregator} that adds a new record to the aggregate result
+     * @param subtractor    an {@link Aggregator} that removed an old record from the aggregate result
+     * @param named         a {@link Named} config used to name the processor in the topology
+     * @param materialized  the instance of {@link Materialized} used to materialize the state store. Cannot be {@code null}
+     * @param           the value type of the aggregated {@link KTable}
+     * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the
+     * latest (rolling) aggregate for each key
+     */
+     KTable aggregate(final Initializer initializer,
+                                 final Aggregator adder,
+                                 final Aggregator subtractor,
+                                 final Named named,
+                                 final Materialized> materialized);
+
     /**
      * Aggregate the value of records of the original {@link KTable} that got {@link KTable#groupBy(KeyValueMapper)
      * mapped} to the same key into a new instance of {@link KTable} using default serializers and deserializers.
@@ -389,4 +625,75 @@  KTable aggregate(final Initializer initializer,
                                  final Aggregator adder,
                                  final Aggregator subtractor);
 
+
+    /**
+     * Aggregate the value of records of the original {@link KTable} that got {@link KTable#groupBy(KeyValueMapper)
+     * mapped} to the same key into a new instance of {@link KTable} using default serializers and deserializers.
+     * Records with {@code null} key are ignored.
+     * Aggregating is a generalization of {@link #reduce(Reducer, Reducer) combining via reduce(...)} as it,
+     * for example, allows the result to have a different type than the input values.
+     * If the result value type does not match the {@link StreamsConfig#DEFAULT_VALUE_SERDE_CLASS_CONFIG default value
+     * serde} you should use {@link #aggregate(Initializer, Aggregator, Aggregator, Materialized)}.
+     * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view)
+     * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream.
+     * 
                
+     * The specified {@link Initializer} is applied once directly before the first input record is processed to
+     * provide an initial intermediate aggregation result that is used to process the first record.
+     * Each update to the original {@link KTable} results in a two step update of the result {@link KTable}.
+     * The specified {@link Aggregator adder} is applied for each update record and computes a new aggregate using the
+     * current aggregate (or for the very first record using the intermediate aggregation result provided via the
+     * {@link Initializer}) and the record's value by adding the new record to the aggregate.
+     * The specified {@link Aggregator subtractor} is applied for each "replaced" record of the original {@link KTable}
+     * and computes a new aggregate using the current aggregate and the record's value by "removing" the "replaced"
+     * record from the aggregate.
+     * Thus, {@code aggregate(Initializer, Aggregator, Aggregator, String)} can be used to compute aggregate functions
+     * like sum.
+     * For sum, the initializer, adder, and subtractor would work as follows:
+     * 
                {@code
+     * // in this example, LongSerde.class must be set as default value serde in StreamsConfig
+     * public class SumInitializer implements Initializer {
+     *   public Long apply() {
+     *     return 0L;
+     *   }
+     * }
+     *
+     * public class SumAdder implements Aggregator {
+     *   public Long apply(String key, Integer newValue, Long aggregate) {
+     *     return aggregate + newValue;
+     *   }
+     * }
+     *
+     * public class SumSubtractor implements Aggregator {
+     *   public Long apply(String key, Integer oldValue, Long aggregate) {
+     *     return aggregate - oldValue;
+     *   }
+     * }
+     * }
                
+     * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
+     * the same key.
+     * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+     * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
+     * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
+     * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}.
+     * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+     * The changelog topic will be named "${applicationId}-${internalStoreName}-changelog", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name
+     * and "-changelog" is a fixed suffix.
+     * Note that the internal store name may not be queriable through Interactive Queries.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
+     * @param initializer a {@link Initializer} that provides an initial aggregate result value
+     * @param adder       a {@link Aggregator} that adds a new record to the aggregate result
+     * @param subtractor  a {@link Aggregator} that removed an old record from the aggregate result
+     * @param named       a {@link Named} config used to name the processor in the topology
+     * @param         the value type of the aggregated {@link KTable}
+     * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the
+     * latest (rolling) aggregate for each key
+     */
+     KTable aggregate(final Initializer initializer,
+                                 final Aggregator adder,
+                                 final Aggregator subtractor,
+                                 final Named named);
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindowedKStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindowedKStream.java
index c5c44c6ff983c..67df8059356fd 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindowedKStream.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindowedKStream.java
@@ -69,6 +69,24 @@ public interface SessionWindowedKStream {
      */
     KTable, Long> count();
 
+    /**
+     * Count the number of records in this stream by the grouped key into {@link SessionWindows}.
+     * Records with {@code null} key or value are ignored.
+     * 
                
+     * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
+     * the same window and key.
+     * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+     * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
+     * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
+     * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}.
+     *
+     * @param named       a {@link Named} config used for the filter processor
+     *
+     * @return a windowed {@link KTable} that contains "update" records with unmodified keys and {@link Long} values
+     * that represent the latest (rolling) count (i.e., number of records) for each key within a window
+     */
+    KTable, Long> count(final Named named);
+
     /**
      * Count the number of records in this stream by the grouped key into {@link SessionWindows}.
      * Records with {@code null} key or value are ignored.
@@ -112,6 +130,51 @@ public interface SessionWindowedKStream {
      */
     KTable, Long> count(final Materialized> materialized);
 
+    /**
+     * Count the number of records in this stream by the grouped key into {@link SessionWindows}.
+     * Records with {@code null} key or value are ignored.
+     * The result is written into a local {@link SessionStore} (which is basically an ever-updating
+     * materialized view) that can be queried using the name provided with {@link Materialized}.
+     * 
                
+     * Not all updates might get sent downstream, as an internal cache will be used to deduplicate consecutive updates to
+     * the same window and key if caching is enabled on the {@link Materialized} instance.
+     * When caching is enabled the rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+     * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
+     * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
+     * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}
+     * 
                
+     * To query the local windowed {@link KeyValueStore} it must be obtained via
+     * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}.
+     * 
                {@code
+     * KafkaStreams streams = ... // compute sum
+     * Sting queryableStoreName = ... // the queryableStoreName should be the name of the store as defined by the Materialized instance
+     * ReadOnlySessionStore localWindowStore = streams.store(queryableStoreName, QueryableStoreTypes.ReadOnlySessionStore);
+     * String key = "some-key";
+     * KeyValueIterator, Long> sumForKeyForWindows = localWindowStore.fetch(key); // key must be local (application state is shared over all running Kafka Streams instances)
+     * }
                
+     * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
+     * query the value of the key on a parallel running instance of your Kafka Streams application.
+     *
+     * 
                
+     * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+     * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII
+     * alphanumerics, '.', '_' and '-'.
+     * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the
+     * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
+     * @param named         a {@link Named} config used to name the processor in the topology
+     * @param materialized  an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}.
+     *                      Note: the valueSerde will be automatically set to {@link Serdes#Long()} if there is no valueSerde provided
+     * @return a windowed {@link KTable} that contains "update" records with unmodified keys and {@link Long} values
+     * that represent the latest (rolling) count (i.e., number of records) for each key within a window
+     */
+    KTable, Long> count(final Named named,
+                                    final Materialized> materialized);
+
     /**
      * Aggregate the values of records in this stream by the grouped key and defined {@link SessionWindows}.
      * Records with {@code null} key or value are ignored.
@@ -149,6 +212,46 @@  KTable, VR> aggregate(final Initializer initializer,
                                            final Aggregator aggregator,
                                            final Merger sessionMerger);
 
+
+    /**
+     * Aggregate the values of records in this stream by the grouped key and defined {@link SessionWindows}.
+     * Records with {@code null} key or value are ignored.
+     * Aggregating is a generalization of {@link #reduce(Reducer) combining via
+     * reduce(...)} as it, for example, allows the result to have a different type than the input values.
+     * 
                
+     * The specified {@link Initializer} is applied once per session directly before the first input record is
+     * processed to provide an initial intermediate aggregation result that is used to process the first record.
+     * The specified {@link Aggregator} is applied for each input record and computes a new aggregate using the current
+     * aggregate (or for the very first record using the intermediate aggregation result provided via the
+     * {@link Initializer}) and the record's value.
+     * The specified {@link Merger} is used to merge 2 existing sessions into one, i.e., when the windows overlap,
+     * they are merged into a single session and the old sessions are discarded.
+     * Thus, {@code aggregate(Initializer, Aggregator, Merger)} can be used to compute
+     * aggregate functions like count (c.f. {@link #count()})
+     * 
                
+     * The default value serde from config will be used for serializing the result.
+     * If a different serde is required then you should use {@link #aggregate(Initializer, Aggregator, Merger, Materialized)}.
+     * 
                
+     * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
+     * the same window and key.
+     * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+     * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
+     * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
+     * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}.
+     * 
                
+     * @param initializer    the instance of {@link Initializer}
+     * @param aggregator     the instance of {@link Aggregator}
+     * @param sessionMerger  the instance of {@link Merger}
+     * @param named          a {@link Named} config used to name the processor in the topology
+     * @param            the value type of the resulting {@link KTable}
+     * @return a windowed {@link KTable} that contains "update" records with unmodified keys, and values that represent
+     * the latest (rolling) aggregate for each key within a window
+     */
+     KTable, VR> aggregate(final Initializer initializer,
+                                           final Aggregator aggregator,
+                                           final Merger sessionMerger,
+                                           final Named named);
+
     /**
      * Aggregate the values of records in this stream by the grouped key and defined {@link SessionWindows}.
      * Records with {@code null} key or value are ignored.
@@ -207,6 +310,67 @@  KTable, VR> aggregate(final Initializer initializer,
                                            final Merger sessionMerger,
                                            final Materialized> materialized);
 
+
+    /**
+     * Aggregate the values of records in this stream by the grouped key and defined {@link SessionWindows}.
+     * Records with {@code null} key or value are ignored.
+     * The result is written into a local {@link SessionStore} (which is basically an ever-updating
+     * materialized view) that can be queried using the name provided with {@link Materialized}.
+     * Aggregating is a generalization of {@link #reduce(Reducer) combining via
+     * reduce(...)} as it, for example, allows the result to have a different type than the input values.
+     * 
                
+     * The specified {@link Initializer} is applied once per session directly before the first input record is
+     * processed to provide an initial intermediate aggregation result that is used to process the first record.
+     * The specified {@link Aggregator} is applied for each input record and computes a new aggregate using the current
+     * aggregate (or for the very first record using the intermediate aggregation result provided via the
+     * {@link Initializer}) and the record's value.
+     * * The specified {@link Merger} is used to merge 2 existing sessions into one, i.e., when the windows overlap,
+     * they are merged into a single session and the old sessions are discarded.
+     * Thus, {@code aggregate(Initializer, Aggregator, Merger)} can be used to compute
+     * aggregate functions like count (c.f. {@link #count()})
+     * 
                
+     * Not all updates might get sent downstream, as an internal cache will be used to deduplicate consecutive updates to
+     * the same window and key if caching is enabled on the {@link Materialized} instance.
+     * When caching is enabled the rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+     * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
+     * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
+     * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}
+     * 
                
+     * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}.
+     * 
                {@code
+     * KafkaStreams streams = ... // some windowed aggregation on value type double
+     * Sting queryableStoreName = ... // the queryableStoreName should be the name of the store as defined by the Materialized instance
+     * ReadOnlySessionStore sessionStore = streams.store(queryableStoreName, QueryableStoreTypes.sessionStore());
+     * String key = "some-key";
+     * KeyValueIterator, Long> aggForKeyForSession = localWindowStore.fetch(key); // key must be local (application state is shared over all running Kafka Streams instances)
+     * }
                
+     *
+     * 
                
+     * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+     * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII
+     * alphanumerics, '.', '_' and '-'.
+     * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the
+     * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
+     * @param initializer    the instance of {@link Initializer}
+     * @param aggregator     the instance of {@link Aggregator}
+     * @param sessionMerger  the instance of {@link Merger}
+     * @param named          a {@link Named} config used to name the processor in the topology
+     * @param materialized   an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}
+     * @param            the value type of the resulting {@link KTable}
+     * @return a windowed {@link KTable} that contains "update" records with unmodified keys, and values that represent
+     * the latest (rolling) aggregate for each key within a window
+     */
+     KTable, VR> aggregate(final Initializer initializer,
+                                           final Aggregator aggregator,
+                                           final Merger sessionMerger,
+                                           final Named named,
+                                           final Materialized> materialized);
+
     /**
      * Combine values of this stream by the grouped key into {@link SessionWindows}.
      * Records with {@code null} key or value are ignored.
@@ -235,6 +399,99 @@  KTable, VR> aggregate(final Initializer initializer,
      */
     KTable, V> reduce(final Reducer reducer);
 
+
+    /**
+     * Combine values of this stream by the grouped key into {@link SessionWindows}.
+     * Records with {@code null} key or value are ignored.
+     * Combining implies that the type of the aggregate result is the same as the type of the input value
+     * (c.f. {@link #aggregate(Initializer, Aggregator, Merger)}).
+     * The result is written into a local {@link SessionStore} (which is basically an ever-updating
+     * materialized view).
+     * 
                
+     * The specified {@link Reducer} is applied for each input record and computes a new aggregate using the current
+     * aggregate and the record's value.
+     * If there is no current aggregate the {@link Reducer} is not applied and the new aggregate will be the record's
+     * value as-is.
+     * Thus, {@code reduce(Reducer)} can be used to compute aggregate functions like sum, min,
+     * or max.
+     * 
                
+     * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
+     * the same window and key.
+     * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+     * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
+     * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
+     * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}.
+     *
+     * @param reducer       a {@link Reducer} that computes a new aggregate result. Cannot be {@code null}.
+     * @param named         a {@link Named} config used to name the processor in the topology
+     * @return a windowed {@link KTable} that contains "update" records with unmodified keys, and values that represent
+     * the latest (rolling) aggregate for each key within a window
+     */
+    KTable, V> reduce(final Reducer reducer, final Named named);
+
+    /**
+     * Combine values of this stream by the grouped key into {@link SessionWindows}.
+     * Records with {@code null} key or value are ignored.
+     * Combining implies that the type of the aggregate result is the same as the type of the input value
+     * (c.f. {@link #aggregate(Initializer, Aggregator, Merger)}).
+     * The result is written into a local {@link SessionStore} (which is basically an ever-updating materialized view)
+     * provided by the given {@link Materialized} instance.
+     * 
                
+     * The specified {@link Reducer} is applied for each input record and computes a new aggregate using the current
+     * aggregate (first argument) and the record's value (second argument):
+     * 
                {@code
+     * // At the example of a Reducer
+     * new Reducer() {
+     *   public Long apply(Long aggValue, Long currValue) {
+     *     return aggValue + currValue;
+     *   }
+     * }
+     * }
                
+     * 
                
+     * If there is no current aggregate the {@link Reducer} is not applied and the new aggregate will be the record's
+     * value as-is.
+     * Thus, {@code reduce(Reducer, Materialized)} can be used to compute aggregate functions like
+     * sum, min, or max.
+     * 
                
+     * Not all updates might get sent downstream, as an internal cache will be used to deduplicate consecutive updates to
+     * the same window and key if caching is enabled on the {@link Materialized} instance.
+     * When caching is enabled the rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+     * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
+     * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
+     * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}
+     * 
                
+     * To query the local windowed {@link KeyValueStore} it must be obtained via
+     * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}.
+     * 
                {@code
+     * KafkaStreams streams = ... // compute sum
+     * Sting queryableStoreName = ... // the queryableStoreName should be the name of the store as defined by the Materialized instance
+     * ReadOnlySessionStore localWindowStore = streams.store(queryableStoreName, QueryableStoreTypes.ReadOnlySessionStore);
+     * String key = "some-key";
+     * KeyValueIterator, Long> sumForKeyForWindows = localWindowStore.fetch(key); // key must be local (application state is shared over all running Kafka Streams instances)
+     * }
                
+     * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
+     * query the value of the key on a parallel running instance of your Kafka Streams application.
+     *
+     * 
                
+     * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+     * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII
+     * alphanumerics, '.', '_' and '-'.
+     * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the
+     * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix.
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
+     *
+     * @param reducer a {@link Reducer} that computes a new aggregate result. Cannot be {@code null}.
+     * @param materializedAs an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}
+     * @return a windowed {@link KTable} that contains "update" records with unmodified keys, and values that represent
+     * the latest (rolling) aggregate for each key within a window
+     */
+    KTable, V> reduce(final Reducer reducer,
+                                  final Materialized> materializedAs);
+
+
     /**
      * Combine values of this stream by the grouped key into {@link SessionWindows}.
      * Records with {@code null} key or value are ignored.
@@ -290,10 +547,12 @@  KTable, VR> aggregate(final Initializer initializer,
      *
      *
      * @param reducer a {@link Reducer} that computes a new aggregate result. Cannot be {@code null}.
+     * @param named   a {@link Named} config used to name the processor in the topology
      * @param materializedAs an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}
      * @return a windowed {@link KTable} that contains "update" records with unmodified keys, and values that represent
      * the latest (rolling) aggregate for each key within a window
      */
     KTable, V> reduce(final Reducer reducer,
+                                  final Named named,
                                   final Materialized> materializedAs);
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindowedKStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindowedKStream.java
index d209b6dea6c75..57f5e2b9dc343 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindowedKStream.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindowedKStream.java
@@ -80,6 +80,32 @@ public interface TimeWindowedKStream {
      */
     KTable, Long> count();
 
+    /**
+     * Count the number of records in this stream by the grouped key and the defined windows.
+     * Records with {@code null} key or value are ignored.
+     * 
                
+     * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
+     * the same window and key.
+     * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+     * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
+     * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
+     * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}.
+     * 
                
+     * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+     * The changelog topic will be named "${applicationId}-${internalStoreName}-changelog", where "applicationId" is
+     * user-specified in {@link StreamsConfig StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name
+     * and "-changelog" is a fixed suffix.
+     * Note that the internal store name may not be queriable through Interactive Queries.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
+     * @param named         a {@link Named} config used to name the processor in the topology
+     * @return a {@link KTable} that contains "update" records with unmodified keys and {@link Long} values that
+     * represent the latest (rolling) count (i.e., number of records) for each key
+     */
+    KTable, Long> count(final Named named);
+
     /**
      * Count the number of records in this stream by the grouped key and the defined windows.
      * Records with {@code null} key or value are ignored.
@@ -126,6 +152,54 @@ public interface TimeWindowedKStream {
      */
     KTable, Long> count(final Materialized> materialized);
 
+
+    /**
+     * Count the number of records in this stream by the grouped key and the defined windows.
+     * Records with {@code null} key or value are ignored.
+     * 
                
+     * Not all updates might get sent downstream, as an internal cache will be used to deduplicate consecutive updates to
+     * the same window and key if caching is enabled on the {@link Materialized} instance.
+     * When caching is enabled the rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+     * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
+     * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
+     * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}
+     *
+     * 
                
+     * To query the local windowed {@link KeyValueStore} it must be obtained via
+     * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}:
+     * 
                {@code
+     * KafkaStreams streams = ... // counting words
+     * Store queryableStoreName = ... // the queryableStoreName should be the name of the store as defined by the Materialized instance
+     * ReadOnlyWindowStore localWindowStore = streams.store(queryableStoreName, QueryableStoreTypes.windowStore());
+     *
+     * String key = "some-word";
+     * long fromTime = ...;
+     * long toTime = ...;
+     * WindowStoreIterator countForWordsForWindows = localWindowStore.fetch(key, timeFrom, timeTo); // key must be local (application state is shared over all running Kafka Streams instances)
+     * }
                
+     * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
+     * query the value of the key on a parallel running instance of your Kafka Streams application.
+     *
+     * 
                
+     * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+     * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII
+     * alphanumerics, '.', '_' and '-'.
+     * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the
+     * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
+     * @param named         a {@link Named} config used to name the processor in the topology
+     * @param materialized  an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}.
+     *                       Note: the valueSerde will be automatically set to {@link org.apache.kafka.common.serialization.Serdes#Long() Serdes#Long()}
+     *                       if there is no valueSerde provided
+     * @return a {@link KTable} that contains "update" records with unmodified keys and {@link Long} values that
+     * represent the latest (rolling) count (i.e., number of records) for each key
+     */
+    KTable, Long> count(final Named named, final Materialized> materialized);
+
     /**
      * Aggregate the values of records in this stream by the grouped key.
      * Records with {@code null} key or value are ignored.
@@ -172,6 +246,115 @@ public interface TimeWindowedKStream {
      KTable, VR> aggregate(final Initializer initializer,
                                            final Aggregator aggregator);
 
+
+    /**
+     * Aggregate the values of records in this stream by the grouped key.
+     * Records with {@code null} key or value are ignored.
+     * Aggregating is a generalization of {@link #reduce(Reducer) combining via reduce(...)} as it, for example,
+     * allows the result to have a different type than the input values.
+     * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view)
+     * that can be queried using the provided {@code queryableStoreName}.
+     * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream.
+     * 
                
+     * The specified {@link Initializer} is applied once directly before the first input record is processed to
+     * provide an initial intermediate aggregation result that is used to process the first record.
+     * The specified {@link Aggregator} is applied for each input record and computes a new aggregate using the current
+     * aggregate (or for the very first record using the intermediate aggregation result provided via the
+     * {@link Initializer}) and the record's value.
+     * Thus, {@code aggregate(Initializer, Aggregator)} can be used to compute aggregate functions like
+     * count (c.f. {@link #count()}).
+     * 
                
+     * The default value serde from config will be used for serializing the result.
+     * If a different serde is required then you should use {@link #aggregate(Initializer, Aggregator, Materialized)}.
+     * 
                
+     * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
+     * the same key.
+     * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+     * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
+     * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
+     * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}.
+     * 
                
+     * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+     * The changelog topic will be named "${applicationId}-${internalStoreName}-changelog", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name
+     * and "-changelog" is a fixed suffix.
+     * Note that the internal store name may not be queriable through Interactive Queries.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
+     *
+     * @param           the value type of the resulting {@link KTable}
+     * @param initializer   an {@link Initializer} that computes an initial intermediate aggregation result
+     * @param aggregator    an {@link Aggregator} that computes a new aggregate result
+     * @param named         a {@link Named} config used to name the processor in the topology
+     * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the
+     * latest (rolling) aggregate for each key
+     */
+     KTable, VR> aggregate(final Initializer initializer,
+                                           final Aggregator aggregator,
+                                           final Named named);
+
+    /**
+     * Aggregate the values of records in this stream by the grouped key.
+     * Records with {@code null} key or value are ignored.
+     * Aggregating is a generalization of {@link #reduce(Reducer) combining via reduce(...)} as it, for example,
+     * allows the result to have a different type than the input values.
+     * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view)
+     * that can be queried using the store name as provided with {@link Materialized}.
+     * 
                
+     * The specified {@link Initializer} is applied once directly before the first input record is processed to
+     * provide an initial intermediate aggregation result that is used to process the first record.
+     * The specified {@link Aggregator} is applied for each input record and computes a new aggregate using the current
+     * aggregate (or for the very first record using the intermediate aggregation result provided via the
+     * {@link Initializer}) and the record's value.
+     * Thus, {@code aggregate(Initializer, Aggregator, Materialized)} can be used to compute aggregate functions like
+     * count (c.f. {@link #count()}).
+     * 
                
+     * Not all updates might get sent downstream, as an internal cache will be used to deduplicate consecutive updates to
+     * the same window and key if caching is enabled on the {@link Materialized} instance.
+     * When caching is enable the rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+     * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
+     * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
+     * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}
+     *
+     * 
                
+     * To query the local windowed {@link KeyValueStore} it must be obtained via
+     * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}:
+     * 
                {@code
+     * KafkaStreams streams = ... // counting words
+     * Store queryableStoreName = ... // the queryableStoreName should be the name of the store as defined by the Materialized instance
+     * ReadOnlyWindowStore localWindowStore = streams.store(queryableStoreName, QueryableStoreTypes.windowStore());
+     *
+     * String key = "some-word";
+     * long fromTime = ...;
+     * long toTime = ...;
+     * WindowStoreIterator aggregateStore = localWindowStore.fetch(key, timeFrom, timeTo); // key must be local (application state is shared over all running Kafka Streams instances)
+     * }
                
+     *
+     * 
                
+     * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+     * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII
+     * alphanumerics, '.', '_' and '-'.
+     * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the
+     * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
+     * @param initializer   an {@link Initializer} that computes an initial intermediate aggregation result
+     * @param aggregator    an {@link Aggregator} that computes a new aggregate result
+     * @param materialized  an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}.
+     * @param           the value type of the resulting {@link KTable}
+     * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the
+     * latest (rolling) aggregate for each key
+     */
+     KTable, VR> aggregate(final Initializer initializer,
+                                           final Aggregator aggregator,
+                                           final Materialized> materialized);
+
+
     /**
      * Aggregate the values of records in this stream by the grouped key.
      * Records with {@code null} key or value are ignored.
@@ -222,6 +405,7 @@  KTable, VR> aggregate(final Initializer initializer,
      *
      * @param initializer   an {@link Initializer} that computes an initial intermediate aggregation result
      * @param aggregator    an {@link Aggregator} that computes a new aggregate result
+     * @param named         a {@link Named} config used to name the processor in the topology
      * @param materialized  an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}.
      * @param           the value type of the resulting {@link KTable}
      * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the
@@ -229,6 +413,7 @@  KTable, VR> aggregate(final Initializer initializer,
      */
      KTable, VR> aggregate(final Initializer initializer,
                                            final Aggregator aggregator,
+                                           final Named named,
                                            final Materialized> materialized);
 
     /**
@@ -266,6 +451,97 @@  KTable, VR> aggregate(final Initializer initializer,
      */
     KTable, V> reduce(final Reducer reducer);
 
+
+    /**
+     * Combine the values of records in this stream by the grouped key.
+     * Records with {@code null} key or value are ignored.
+     * Combining implies that the type of the aggregate result is the same as the type of the input value.
+     * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view)
+     * that can be queried using the provided {@code queryableStoreName}.
+     * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream.
+     * 
                
+     * The specified {@link Reducer} is applied for each input record and computes a new aggregate using the current
+     * aggregate and the record's value.
+     * If there is no current aggregate the {@link Reducer} is not applied and the new aggregate will be the record's
+     * value as-is.
+     * Thus, {@code reduce(Reducer, String)} can be used to compute aggregate functions like sum, min, or max.
+     * 
                
+     * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
+     * the same key.
+     * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+     * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
+     * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
+     * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}.
+     * 
                
+     * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+     * The changelog topic will be named "${applicationId}-${internalStoreName}-changelog", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name
+     * and "-changelog" is a fixed suffix.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
+     * @param named     a {@link Named} config used to name the processor in the topology
+     * @param reducer   a {@link Reducer} that computes a new aggregate result
+     * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the
+     * latest (rolling) aggregate for each key
+     */
+    KTable, V> reduce(final Reducer reducer, final Named named);
+
+    /**
+     * Combine the values of records in this stream by the grouped key.
+     * Records with {@code null} key or value are ignored.
+     * Combining implies that the type of the aggregate result is the same as the type of the input value.
+     * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view)
+     * that can be queried using the store name as provided with {@link Materialized}.
+     * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream.
+     * 
                
+     * The specified {@link Reducer} is applied for each input record and computes a new aggregate using the current
+     * aggregate and the record's value.
+     * If there is no current aggregate the {@link Reducer} is not applied and the new aggregate will be the record's
+     * value as-is.
+     * Thus, {@code reduce(Reducer, String)} can be used to compute aggregate functions like sum, min, or max.
+     * 
                
+     * Not all updates might get sent downstream, as an internal cache will be used to deduplicate consecutive updates to
+     * the same window and key if caching is enabled on the {@link Materialized} instance.
+     * When caching is enable the rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+     * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
+     * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
+     * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}
+     * 
                
+     * To query the local windowed {@link KeyValueStore} it must be obtained via
+     * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}:
+     * 
                {@code
+     * KafkaStreams streams = ... // counting words
+     * Store queryableStoreName = ... // the queryableStoreName should be the name of the store as defined by the Materialized instance
+     * ReadOnlyWindowStore localWindowStore = streams.store(queryableStoreName, QueryableStoreTypes.windowStore());
+     *
+     * String key = "some-word";
+     * long fromTime = ...;
+     * long toTime = ...;
+     * WindowStoreIterator reduceStore = localWindowStore.fetch(key, timeFrom, timeTo); // key must be local (application state is shared over all running Kafka Streams instances)
+     * }
                
+     *
+     * 
                
+     * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+     * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII
+     * alphanumerics, '.', '_' and '-'.
+     * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the
+     * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
+     * @param reducer       a {@link Reducer} that computes a new aggregate result
+     * @param materialized  an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}.
+     * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the
+     * latest (rolling) aggregate for each key
+     */
+    KTable, V> reduce(final Reducer reducer,
+                                  final Materialized> materialized);
+
+
     /**
      * Combine the values of records in this stream by the grouped key.
      * Records with {@code null} key or value are ignored.
@@ -312,10 +588,12 @@  KTable, VR> aggregate(final Initializer initializer,
      * You can retrieve all generated internal topic names via {@link Topology#describe()}.
      *
      * @param reducer       a {@link Reducer} that computes a new aggregate result
+     * @param named         a {@link Named} config used to name the processor in the topology
      * @param materialized  an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}.
      * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the
      * latest (rolling) aggregate for each key
      */
     KTable, V> reduce(final Reducer reducer,
+                                  final Named named,
                                   final Materialized> materialized);
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/GroupedStreamAggregateBuilder.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/GroupedStreamAggregateBuilder.java
index cd5155f4149c6..b77a2303d9d29 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/GroupedStreamAggregateBuilder.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/GroupedStreamAggregateBuilder.java
@@ -68,7 +68,7 @@ class GroupedStreamAggregateBuilder {
         this.userProvidedRepartitionTopicName = groupedInternal.name();
     }
 
-     KTable build(final String functionName,
+     KTable build(final NamedInternal functionName,
                                   final StoreBuilder storeBuilder,
                                   final KStreamAggProcessorSupplier aggregateSupplier,
                                   final String queryableStoreName,
@@ -76,7 +76,7 @@  KTable build(final String functionName,
                                   final Serde valSerde) {
         assert queryableStoreName == null || queryableStoreName.equals(storeBuilder.name());
 
-        final String aggFunctionName = builder.newProcessorName(functionName);
+        final String aggFunctionName = functionName.name();
 
         String sourceName = this.name;
         StreamsGraphNode parentNode = streamsGraphNode;
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java
index 3a90fd222d484..ca43c566c1a14 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java
@@ -57,6 +57,8 @@
 
 public class InternalStreamsBuilder implements InternalNameProvider {
 
+    private static final String TABLE_SOURCE_SUFFIX = "-source";
+
     final InternalTopologyBuilder internalTopologyBuilder;
     private final AtomicInteger index = new AtomicInteger(0);
 
@@ -115,10 +117,15 @@ public  KStream stream(final Pattern topicPattern,
     public  KTable table(final String topic,
                                      final ConsumedInternal consumed,
                                      final MaterializedInternal> materialized) {
-        final String sourceName = new NamedInternal(consumed.name())
-                .orElseGenerateWithPrefix(this, KStreamImpl.SOURCE_NAME);
-        final String tableSourceName = new NamedInternal(consumed.name())
-                .suffixWithOrElseGet("-table-source", this, KTableImpl.SOURCE_NAME);
+
+        final NamedInternal named = new NamedInternal(consumed.name());
+
+        final String sourceName = named
+            .suffixWithOrElseGet(TABLE_SOURCE_SUFFIX, this, KStreamImpl.SOURCE_NAME);
+
+        final String tableSourceName = named
+            .orElseGenerateWithPrefix(this, KTableImpl.SOURCE_NAME);
+
         final KTableSource tableSource = new KTableSource<>(materialized.storeName(), materialized.queryableStoreName());
         final ProcessorParameters processorParameters = new ProcessorParameters<>(tableSource, tableSourceName);
 
@@ -150,8 +157,15 @@ public  GlobalKTable globalTable(final String topic,
         Objects.requireNonNull(materialized, "materialized can't be null");
         // explicitly disable logging for global stores
         materialized.withLoggingDisabled();
-        final String sourceName = newProcessorName(KTableImpl.SOURCE_NAME);
-        final String processorName = newProcessorName(KTableImpl.SOURCE_NAME);
+
+        final NamedInternal named = new NamedInternal(consumed.name());
+
+        final String sourceName = named
+                .suffixWithOrElseGet(TABLE_SOURCE_SUFFIX, this, KStreamImpl.SOURCE_NAME);
+
+        final String processorName = named
+                .orElseGenerateWithPrefix(this, KTableImpl.SOURCE_NAME);
+
         // enforce store name as queryable name to always materialize global table stores
         final String storeName = materialized.storeName();
         final KTableSource tableSource = new KTableSource<>(storeName, storeName);
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java
index 6f6521cceac54..57511056a9032 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java
@@ -23,6 +23,7 @@
 import org.apache.kafka.streams.kstream.KGroupedStream;
 import org.apache.kafka.streams.kstream.KTable;
 import org.apache.kafka.streams.kstream.Materialized;
+import org.apache.kafka.streams.kstream.Named;
 import org.apache.kafka.streams.kstream.Reducer;
 import org.apache.kafka.streams.kstream.SessionWindowedKStream;
 import org.apache.kafka.streams.kstream.SessionWindows;
@@ -67,8 +68,16 @@ public KTable reduce(final Reducer reducer) {
     @Override
     public KTable reduce(final Reducer reducer,
                                final Materialized> materialized) {
+        return reduce(reducer, NamedInternal.empty(), materialized);
+    }
+
+    @Override
+    public KTable reduce(final Reducer reducer,
+                               final Named named,
+                               final Materialized> materialized) {
         Objects.requireNonNull(reducer, "reducer can't be null");
         Objects.requireNonNull(materialized, "materialized can't be null");
+        Objects.requireNonNull(named, "name can't be null");
 
         final MaterializedInternal> materializedInternal =
             new MaterializedInternal<>(materialized, builder, REDUCE_NAME);
@@ -80,9 +89,10 @@ public KTable reduce(final Reducer reducer,
             materializedInternal.withValueSerde(valSerde);
         }
 
+        final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, REDUCE_NAME);
         return doAggregate(
             new KStreamReduce<>(materializedInternal.storeName(), reducer),
-            REDUCE_NAME,
+            name,
             materializedInternal
         );
     }
@@ -91,9 +101,18 @@ public KTable reduce(final Reducer reducer,
     public  KTable aggregate(final Initializer initializer,
                                         final Aggregator aggregator,
                                         final Materialized> materialized) {
+        return aggregate(initializer, aggregator, NamedInternal.empty(), materialized);
+    }
+
+    @Override
+    public  KTable aggregate(final Initializer initializer,
+                                        final Aggregator aggregator,
+                                        final Named named,
+                                        final Materialized> materialized) {
         Objects.requireNonNull(initializer, "initializer can't be null");
         Objects.requireNonNull(aggregator, "aggregator can't be null");
         Objects.requireNonNull(materialized, "materialized can't be null");
+        Objects.requireNonNull(named, "named can't be null");
 
         final MaterializedInternal> materializedInternal =
             new MaterializedInternal<>(materialized, builder, AGGREGATE_NAME);
@@ -102,9 +121,10 @@ public  KTable aggregate(final Initializer initializer,
             materializedInternal.withKeySerde(keySerde);
         }
 
+        final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, AGGREGATE_NAME);
         return doAggregate(
             new KStreamAggregate<>(materializedInternal.storeName(), initializer, aggregator),
-            AGGREGATE_NAME,
+            name,
             materializedInternal
         );
     }
@@ -117,11 +137,22 @@ public  KTable aggregate(final Initializer initializer,
 
     @Override
     public KTable count() {
-        return doCount(Materialized.with(keySerde, Serdes.Long()));
+        return doCount(NamedInternal.empty(), Materialized.with(keySerde, Serdes.Long()));
+    }
+
+    @Override
+    public KTable count(final Named named) {
+        Objects.requireNonNull(named, "named can't be null");
+        return doCount(named, Materialized.with(keySerde, Serdes.Long()));
     }
 
     @Override
     public KTable count(final Materialized> materialized) {
+        return count(NamedInternal.empty(), materialized);
+    }
+
+    @Override
+    public KTable count(final Named named, final Materialized> materialized) {
         Objects.requireNonNull(materialized, "materialized can't be null");
 
         // TODO: remove this when we do a topology-incompatible release
@@ -130,10 +161,10 @@ public KTable count(final Materialized doCount(final Materialized> materialized) {
+    private KTable doCount(final Named named, final Materialized> materialized) {
         final MaterializedInternal> materializedInternal =
             new MaterializedInternal<>(materialized, builder, AGGREGATE_NAME);
 
@@ -144,9 +175,10 @@ private KTable doCount(final Materialized(materializedInternal.storeName(), aggregateBuilder.countInitializer, aggregateBuilder.countAggregator),
-            AGGREGATE_NAME,
+            name,
             materializedInternal);
     }
 
@@ -184,7 +216,7 @@ private  KTable doAggregate(final KStreamAggProcessorSupplier> materializedInternal) {
         return aggregateBuilder.build(
-            functionName,
+            new NamedInternal(functionName),
             new TimestampedKeyValueStoreMaterializer<>(materializedInternal).materialize(),
             aggregateSupplier,
             materializedInternal.queryableStoreName(),
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImpl.java
index 75c998a231b9b..e3d03a5bc2e59 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImpl.java
@@ -23,6 +23,7 @@
 import org.apache.kafka.streams.kstream.KGroupedTable;
 import org.apache.kafka.streams.kstream.KTable;
 import org.apache.kafka.streams.kstream.Materialized;
+import org.apache.kafka.streams.kstream.Named;
 import org.apache.kafka.streams.kstream.Reducer;
 import org.apache.kafka.streams.kstream.internals.graph.GroupedTableOperationRepartitionNode;
 import org.apache.kafka.streams.kstream.internals.graph.ProcessorParameters;
@@ -68,12 +69,13 @@ public class KGroupedTableImpl extends AbstractStream implements KGr
     }
 
     private  KTable doAggregate(final ProcessorSupplier> aggregateSupplier,
+                                         final NamedInternal named,
                                          final String functionName,
                                          final MaterializedInternal> materialized) {
 
-        final String sinkName = builder.newProcessorName(KStreamImpl.SINK_NAME);
-        final String sourceName = builder.newProcessorName(KStreamImpl.SOURCE_NAME);
-        final String funcName = builder.newProcessorName(functionName);
+        final String sinkName = named.suffixWithOrElseGet("-sink", builder, KStreamImpl.SINK_NAME);
+        final String sourceName = named.suffixWithOrElseGet("-source", builder, KStreamImpl.SOURCE_NAME);
+        final String funcName = named.orElseGenerateWithPrefix(builder, functionName);
         final String repartitionTopic = (userProvidedRepartitionTopicName != null ? userProvidedRepartitionTopicName : materialized.storeName())
             + KStreamImpl.REPARTITION_TOPIC_SUFFIX;
 
@@ -122,8 +124,17 @@ private GroupedTableOperationRepartitionNode createRepartitionNode(final S
     public KTable reduce(final Reducer adder,
                                final Reducer subtractor,
                                final Materialized> materialized) {
+        return reduce(adder, subtractor, NamedInternal.empty(), materialized);
+    }
+
+    @Override
+    public KTable reduce(final Reducer adder,
+                               final Reducer subtractor,
+                               final Named named,
+                               final Materialized> materialized) {
         Objects.requireNonNull(adder, "adder can't be null");
         Objects.requireNonNull(subtractor, "subtractor can't be null");
+        Objects.requireNonNull(named, "named can't be null");
         Objects.requireNonNull(materialized, "materialized can't be null");
         final MaterializedInternal> materializedInternal =
             new MaterializedInternal<>(materialized, builder, AGGREGATE_NAME);
@@ -134,10 +145,11 @@ public KTable reduce(final Reducer adder,
         if (materializedInternal.valueSerde() == null) {
             materializedInternal.withValueSerde(valSerde);
         }
-        final ProcessorSupplier> aggregateSupplier = new KTableReduce<>(materializedInternal.storeName(),
-                                                                                     adder,
-                                                                                     subtractor);
-        return doAggregate(aggregateSupplier, REDUCE_NAME, materializedInternal);
+        final ProcessorSupplier> aggregateSupplier = new KTableReduce<>(
+            materializedInternal.storeName(),
+            adder,
+            subtractor);
+        return doAggregate(aggregateSupplier, new NamedInternal(named), REDUCE_NAME, materializedInternal);
     }
 
     @Override
@@ -146,8 +158,14 @@ public KTable reduce(final Reducer adder,
         return reduce(adder, subtractor, Materialized.with(keySerde, valSerde));
     }
 
+
     @Override
     public KTable count(final Materialized> materialized) {
+        return count(NamedInternal.empty(), materialized);
+    }
+
+    @Override
+    public KTable count(final Named named, final Materialized> materialized) {
         final MaterializedInternal> materializedInternal =
             new MaterializedInternal<>(materialized, builder, AGGREGATE_NAME);
 
@@ -158,12 +176,13 @@ public KTable count(final Materialized> aggregateSupplier = new KTableAggregate<>(materializedInternal.storeName(),
-                                                                                        countInitializer,
-                                                                                        countAdder,
-                                                                                        countSubtractor);
+        final ProcessorSupplier> aggregateSupplier = new KTableAggregate<>(
+            materializedInternal.storeName(),
+            countInitializer,
+            countAdder,
+            countSubtractor);
 
-        return doAggregate(aggregateSupplier, AGGREGATE_NAME, materializedInternal);
+        return doAggregate(aggregateSupplier, new NamedInternal(named), AGGREGATE_NAME, materializedInternal);
     }
 
     @Override
@@ -171,14 +190,29 @@ public KTable count() {
         return count(Materialized.with(keySerde, Serdes.Long()));
     }
 
+    @Override
+    public KTable count(final Named named) {
+        return count(named, Materialized.with(keySerde, Serdes.Long()));
+    }
+
     @Override
     public  KTable aggregate(final Initializer initializer,
                                         final Aggregator adder,
                                         final Aggregator subtractor,
                                         final Materialized> materialized) {
+        return aggregate(initializer, adder, subtractor, NamedInternal.empty(), materialized);
+    }
+
+    @Override
+    public  KTable aggregate(final Initializer initializer,
+                                        final Aggregator adder,
+                                        final Aggregator subtractor,
+                                        final Named named,
+                                        final Materialized> materialized) {
         Objects.requireNonNull(initializer, "initializer can't be null");
         Objects.requireNonNull(adder, "adder can't be null");
         Objects.requireNonNull(subtractor, "subtractor can't be null");
+        Objects.requireNonNull(named, "named can't be null");
         Objects.requireNonNull(materialized, "materialized can't be null");
 
         final MaterializedInternal> materializedInternal =
@@ -187,11 +221,20 @@ public  KTable aggregate(final Initializer initializer,
         if (materializedInternal.keySerde() == null) {
             materializedInternal.withKeySerde(keySerde);
         }
-        final ProcessorSupplier> aggregateSupplier = new KTableAggregate<>(materializedInternal.storeName(),
-                                                                                        initializer,
-                                                                                        adder,
-                                                                                        subtractor);
-        return doAggregate(aggregateSupplier, AGGREGATE_NAME, materializedInternal);
+        final ProcessorSupplier> aggregateSupplier = new KTableAggregate<>(
+            materializedInternal.storeName(),
+            initializer,
+            adder,
+            subtractor);
+        return doAggregate(aggregateSupplier, new NamedInternal(named), AGGREGATE_NAME, materializedInternal);
+    }
+
+    @Override
+    public  KTable aggregate(final Initializer initializer,
+                                      final Aggregator adder,
+                                      final Aggregator subtractor,
+                                      final Named named) {
+        return aggregate(initializer, adder, subtractor, named, Materialized.with(keySerde, null));
     }
 
     @Override
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImpl.java
index 2106b1a28e3ec..ac49f9187cf76 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImpl.java
@@ -24,6 +24,7 @@
 import org.apache.kafka.streams.kstream.KTable;
 import org.apache.kafka.streams.kstream.Materialized;
 import org.apache.kafka.streams.kstream.Merger;
+import org.apache.kafka.streams.kstream.Named;
 import org.apache.kafka.streams.kstream.Reducer;
 import org.apache.kafka.streams.kstream.SessionWindowedKStream;
 import org.apache.kafka.streams.kstream.SessionWindows;
@@ -63,11 +64,21 @@ public class SessionWindowedKStreamImpl extends AbstractStream imple
 
     @Override
     public KTable, Long> count() {
-        return doCount(Materialized.with(keySerde, Serdes.Long()));
+        return count(NamedInternal.empty());
+    }
+
+    @Override
+    public KTable, Long> count(final Named named) {
+        return doCount(named, Materialized.with(keySerde, Serdes.Long()));
     }
 
     @Override
     public KTable, Long> count(final Materialized> materialized) {
+        return count(NamedInternal.empty(), materialized);
+    }
+
+    @Override
+    public KTable, Long> count(final Named named, final Materialized> materialized) {
         Objects.requireNonNull(materialized, "materialized can't be null");
 
         // TODO: remove this when we do a topology-incompatible release
@@ -76,12 +87,14 @@ public KTable, Long> count(final Materialized, Long> doCount(final Materialized> materialized) {
+    private KTable, Long> doCount(final Named named,
+                                              final Materialized> materialized) {
         final MaterializedInternal> materializedInternal =
             new MaterializedInternal<>(materialized, builder, AGGREGATE_NAME);
+
         if (materializedInternal.keySerde() == null) {
             materializedInternal.withKeySerde(keySerde);
         }
@@ -89,8 +102,9 @@ private KTable, Long> doCount(final Materialized(
                 windows,
@@ -105,13 +119,26 @@ private KTable, Long> doCount(final Materialized, V> reduce(final Reducer reducer) {
-        return reduce(reducer, Materialized.with(keySerde, valSerde));
+        return reduce(reducer, NamedInternal.empty());
+    }
+
+    @Override
+    public KTable, V> reduce(final Reducer reducer, final Named named) {
+        return reduce(reducer, named, Materialized.with(keySerde, valSerde));
+    }
+
+    @Override
+    public KTable, V> reduce(final Reducer reducer,
+                                         final Materialized> materialized) {
+        return reduce(reducer, NamedInternal.empty(), materialized);
     }
 
     @Override
     public KTable, V> reduce(final Reducer reducer,
+                                         final Named named,
                                          final Materialized> materialized) {
         Objects.requireNonNull(reducer, "reducer can't be null");
+        Objects.requireNonNull(named, "named can't be null");
         Objects.requireNonNull(materialized, "materialized can't be null");
         final Aggregator reduceAggregator = aggregatorForReducer(reducer);
         final MaterializedInternal> materializedInternal =
@@ -123,8 +150,9 @@ public KTable, V> reduce(final Reducer reducer,
             materializedInternal.withValueSerde(valSerde);
         }
 
+        final String reduceName = new NamedInternal(named).orElseGenerateWithPrefix(builder, REDUCE_NAME);
         return aggregateBuilder.build(
-            REDUCE_NAME,
+            new NamedInternal(reduceName),
             materialize(materializedInternal),
             new KStreamSessionWindowAggregate<>(
                 windows,
@@ -142,13 +170,30 @@ public KTable, V> reduce(final Reducer reducer,
     public  KTable, T> aggregate(final Initializer initializer,
                                                 final Aggregator aggregator,
                                                 final Merger sessionMerger) {
-        return aggregate(initializer, aggregator, sessionMerger, Materialized.with(keySerde, null));
+        return aggregate(initializer, aggregator, sessionMerger, NamedInternal.empty());
+    }
+
+    @Override
+    public  KTable, T> aggregate(final Initializer initializer,
+                                                final Aggregator aggregator,
+                                                final Merger sessionMerger,
+                                                final Named named) {
+        return aggregate(initializer, aggregator, sessionMerger, named, Materialized.with(keySerde, null));
+    }
+
+    @Override
+    public  KTable, VR> aggregate(final Initializer initializer,
+                                                  final Aggregator aggregator,
+                                                  final Merger sessionMerger,
+                                                  final Materialized> materialized) {
+        return aggregate(initializer, aggregator, sessionMerger, NamedInternal.empty(), materialized);
     }
 
     @Override
     public  KTable, VR> aggregate(final Initializer initializer,
                                                   final Aggregator aggregator,
                                                   final Merger sessionMerger,
+                                                  final Named named,
                                                   final Materialized> materialized) {
         Objects.requireNonNull(initializer, "initializer can't be null");
         Objects.requireNonNull(aggregator, "aggregator can't be null");
@@ -161,8 +206,10 @@ public  KTable, VR> aggregate(final Initializer initializer,
             materializedInternal.withKeySerde(keySerde);
         }
 
+        final String aggregateName = new NamedInternal(named).orElseGenerateWithPrefix(builder, AGGREGATE_NAME);
+
         return aggregateBuilder.build(
-            AGGREGATE_NAME,
+            new NamedInternal(aggregateName),
             materialize(materializedInternal),
             new KStreamSessionWindowAggregate<>(
                 windows,
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImpl.java
index a257603564bf6..84316cbe9ccf4 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImpl.java
@@ -23,6 +23,7 @@
 import org.apache.kafka.streams.kstream.Initializer;
 import org.apache.kafka.streams.kstream.KTable;
 import org.apache.kafka.streams.kstream.Materialized;
+import org.apache.kafka.streams.kstream.Named;
 import org.apache.kafka.streams.kstream.Reducer;
 import org.apache.kafka.streams.kstream.TimeWindowedKStream;
 import org.apache.kafka.streams.kstream.Window;
@@ -63,11 +64,22 @@ public class TimeWindowedKStreamImpl extends AbstractStr
 
     @Override
     public KTable, Long> count() {
-        return doCount(Materialized.with(keySerde, Serdes.Long()));
+        return count(NamedInternal.empty());
     }
 
+    @Override
+    public KTable, Long> count(final Named named) {
+        return doCount(named, Materialized.with(keySerde, Serdes.Long()));
+    }
+
+
     @Override
     public KTable, Long> count(final Materialized> materialized) {
+        return count(NamedInternal.empty(), materialized);
+    }
+
+    @Override
+    public KTable, Long> count(final Named named, final Materialized> materialized) {
         Objects.requireNonNull(materialized, "materialized can't be null");
 
         // TODO: remove this when we do a topology-incompatible release
@@ -76,10 +88,11 @@ public KTable, Long> count(final Materialized, Long> doCount(final Materialized> materialized) {
+    private KTable, Long> doCount(final Named named,
+                                              final Materialized> materialized) {
         final MaterializedInternal> materializedInternal =
             new MaterializedInternal<>(materialized, builder, AGGREGATE_NAME);
 
@@ -90,8 +103,9 @@ private KTable, Long> doCount(final Materialized(windows, materializedInternal.storeName(), aggregateBuilder.countInitializer, aggregateBuilder.countAggregator),
             materializedInternal.queryableStoreName(),
@@ -108,6 +122,22 @@ public  KTable, VR> aggregate(final Initializer initializer,
     @Override
     public  KTable, VR> aggregate(final Initializer initializer,
                                                   final Aggregator aggregator,
+                                                  final Named named) {
+        return aggregate(initializer, aggregator, named, Materialized.with(keySerde, null));
+    }
+
+
+    @Override
+    public  KTable, VR> aggregate(final Initializer initializer,
+                                                  final Aggregator aggregator,
+                                                  final Materialized> materialized) {
+        return aggregate(initializer, aggregator, NamedInternal.empty(), materialized);
+    }
+
+    @Override
+    public  KTable, VR> aggregate(final Initializer initializer,
+                                                  final Aggregator aggregator,
+                                                  final Named named,
                                                   final Materialized> materialized) {
         Objects.requireNonNull(initializer, "initializer can't be null");
         Objects.requireNonNull(aggregator, "aggregator can't be null");
@@ -117,8 +147,10 @@ public  KTable, VR> aggregate(final Initializer initializer,
         if (materializedInternal.keySerde() == null) {
             materializedInternal.withKeySerde(keySerde);
         }
+
+        final String aggregateName = new NamedInternal(named).orElseGenerateWithPrefix(builder, AGGREGATE_NAME);
         return aggregateBuilder.build(
-            AGGREGATE_NAME,
+            new NamedInternal(aggregateName),
             materialize(materializedInternal),
             new KStreamWindowAggregate<>(windows, materializedInternal.storeName(), initializer, aggregator),
             materializedInternal.queryableStoreName(),
@@ -128,12 +160,26 @@ public  KTable, VR> aggregate(final Initializer initializer,
 
     @Override
     public KTable, V> reduce(final Reducer reducer) {
-        return reduce(reducer, Materialized.with(keySerde, valSerde));
+        return reduce(reducer, NamedInternal.empty());
+    }
+
+    @Override
+    public KTable, V> reduce(final Reducer reducer, final Named named) {
+        return reduce(reducer, named, Materialized.with(keySerde, valSerde));
+    }
+
+    @Override
+    public KTable, V> reduce(final Reducer reducer,
+                                         final Materialized> materialized) {
+        return reduce(reducer, NamedInternal.empty(), materialized);
     }
 
     @Override
-    public KTable, V> reduce(final Reducer reducer, final Materialized> materialized) {
+    public KTable, V> reduce(final Reducer reducer,
+                                         final Named named,
+                                         final Materialized> materialized) {
         Objects.requireNonNull(reducer, "reducer can't be null");
+        Objects.requireNonNull(named, "named can't be null");
         Objects.requireNonNull(materialized, "materialized can't be null");
 
         final MaterializedInternal> materializedInternal =
@@ -146,8 +192,9 @@ public KTable, V> reduce(final Reducer reducer, final Materialize
             materializedInternal.withValueSerde(valSerde);
         }
 
+        final String reduceName = new NamedInternal(named).orElseGenerateWithPrefix(builder, REDUCE_NAME);
         return aggregateBuilder.build(
-            REDUCE_NAME,
+            new NamedInternal(reduceName),
             materialize(materializedInternal),
             new KStreamWindowAggregate<>(windows, materializedInternal.storeName(), aggregateBuilder.reduceInitializer, aggregatorForReducer(reducer)),
             materializedInternal.queryableStoreName(),
diff --git a/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java
index 92b471c77ad53..197e2344153dc 100644
--- a/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java
@@ -24,6 +24,7 @@
 import org.apache.kafka.streams.errors.TopologyException;
 import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.ForeachAction;
+import org.apache.kafka.streams.kstream.Grouped;
 import org.apache.kafka.streams.kstream.JoinWindows;
 import org.apache.kafka.streams.kstream.Joined;
 import org.apache.kafka.streams.kstream.KStream;
@@ -450,8 +451,8 @@ public void shouldUseSpecifiedNameForTableSourceProcessor() {
 
         assertSpecifiedNameForOperation(
                 topology,
+                expected + "-source",
                 expected,
-                expected + "-table-source",
                 "KSTREAM-SOURCE-0000000004",
                 "KTABLE-SOURCE-0000000005");
     }
@@ -738,6 +739,27 @@ public void shouldUseSpecifiedNameForToStreamWithMapper() {
                 "KSTREAM-KEY-SELECT-0000000004");
     }
 
+    @Test
+    public void shouldUseSpecifiedNameForAggregateOperationGivenTable() {
+        builder.table(STREAM_TOPIC).groupBy(KeyValue::pair, Grouped.as("group-operation")).count(Named.as(STREAM_OPERATION_NAME));
+        builder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertSpecifiedNameForStateStore(
+            topology.stateStores(),
+            STREAM_TOPIC + "-STATE-STORE-0000000000",
+             "KTABLE-AGGREGATE-STATE-STORE-0000000004");
+
+        assertSpecifiedNameForOperation(
+            topology,
+            "KSTREAM-SOURCE-0000000001",
+            "KTABLE-SOURCE-0000000002",
+            "group-operation",
+            STREAM_OPERATION_NAME + "-sink",
+            STREAM_OPERATION_NAME + "-source",
+            STREAM_OPERATION_NAME);
+    }
+
+
     private static void assertSpecifiedNameForOperation(final ProcessorTopology topology, final String... expected) {
         final List processors = topology.processors();
         assertEquals("Invalid number of expected processors", expected.length, processors.size());
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImplTest.java
index d1e5448fad069..b693dc7640fb5 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImplTest.java
@@ -29,6 +29,7 @@
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.Materialized;
 import org.apache.kafka.streams.kstream.Merger;
+import org.apache.kafka.streams.kstream.Named;
 import org.apache.kafka.streams.kstream.SessionWindowedKStream;
 import org.apache.kafka.streams.kstream.SessionWindows;
 import org.apache.kafka.streams.kstream.Windowed;
@@ -277,14 +278,20 @@ public void shouldThrowNullPointerOnMaterializedReduceIfReducerIsNull() {
     }
 
     @Test(expected = NullPointerException.class)
+    @SuppressWarnings("unchecked")
     public void shouldThrowNullPointerOnMaterializedReduceIfMaterializedIsNull() {
-        stream.reduce(MockReducer.STRING_ADDER,
-                      null);
+        stream.reduce(MockReducer.STRING_ADDER, (Materialized) null);
+    }
+
+    @Test(expected = NullPointerException.class)
+    @SuppressWarnings("unchecked")
+    public void shouldThrowNullPointerOnMaterializedReduceIfNamedIsNull() {
+        stream.reduce(MockReducer.STRING_ADDER, (Named) null);
     }
 
     @Test(expected = NullPointerException.class)
     public void shouldThrowNullPointerOnCountIfMaterializedIsNull() {
-        stream.count(null);
+        stream.count((Materialized>) null);
     }
 
     private void processData(final TopologyTestDriver driver) {
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImplTest.java
index 0c4685a45e3a4..0327ebd20cadb 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImplTest.java
@@ -27,6 +27,7 @@
 import org.apache.kafka.streams.kstream.Grouped;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.Materialized;
+import org.apache.kafka.streams.kstream.Named;
 import org.apache.kafka.streams.kstream.TimeWindowedKStream;
 import org.apache.kafka.streams.kstream.TimeWindows;
 import org.apache.kafka.streams.kstream.Windowed;
@@ -292,15 +293,24 @@ public void shouldThrowNullPointerOnMaterializedReduceIfReducerIsNull() {
     }
 
     @Test(expected = NullPointerException.class)
+    @SuppressWarnings("unchecked")
     public void shouldThrowNullPointerOnMaterializedReduceIfMaterializedIsNull() {
         windowedStream.reduce(
             MockReducer.STRING_ADDER,
-            null);
+            (Materialized) null);
+    }
+
+    @Test(expected = NullPointerException.class)
+    @SuppressWarnings("unchecked")
+    public void shouldThrowNullPointerOnMaterializedReduceIfNamedIsNull() {
+        windowedStream.reduce(
+            MockReducer.STRING_ADDER,
+            (Named) null);
     }
 
     @Test(expected = NullPointerException.class)
     public void shouldThrowNullPointerOnCountIfMaterializedIsNull() {
-        windowedStream.count(null);
+        windowedStream.count((Materialized>) null);
     }
 
     private void processData(final TopologyTestDriver driver) {

From bc4fd676e5ea0e3cf44a8684dca0db46f912c828 Mon Sep 17 00:00:00 2001
From: khairy 
Date: Tue, 24 Sep 2019 05:04:26 +0100
Subject: [PATCH 0638/1071] MINOR: remove unnecessary null check (#7299)

Reviewers: Matthias J. Sax 

From 0d31272b350adf40bda09709304721545553d7f8 Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Tue, 24 Sep 2019 10:30:17 +0100
Subject: [PATCH 0639/1071] KAFKA-8848; Update system tests to use new
 AclAuthorizer (#7374)

Reviewers: Manikumar Reddy 
---
 tests/kafkatest/services/kafka/kafka.py                   | 2 ++
 .../kafkatest/tests/core/security_rolling_upgrade_test.py | 8 ++++----
 .../tests/core/zookeeper_security_upgrade_test.py         | 2 +-
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/tests/kafkatest/services/kafka/kafka.py b/tests/kafkatest/services/kafka/kafka.py
index eb0ddfdde6d04..4256bbc21f0bc 100644
--- a/tests/kafkatest/services/kafka/kafka.py
+++ b/tests/kafkatest/services/kafka/kafka.py
@@ -65,6 +65,8 @@ class KafkaService(KafkaPathResolverMixin, JmxMixin, Service):
     DATA_LOG_DIR_2 = "%s-2" % (DATA_LOG_DIR_PREFIX)
     CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "kafka.properties")
     # Kafka Authorizer
+    ACL_AUTHORIZER = "kafka.security.authorizer.AclAuthorizer"
+    # Old Kafka Authorizer. This is deprecated but still supported.
     SIMPLE_AUTHORIZER = "kafka.security.auth.SimpleAclAuthorizer"
     HEAP_DUMP_FILE = os.path.join(PERSISTENT_ROOT, "kafka_heap_dump.bin")
     INTERBROKER_LISTENER_NAME = 'INTERNAL'
diff --git a/tests/kafkatest/tests/core/security_rolling_upgrade_test.py b/tests/kafkatest/tests/core/security_rolling_upgrade_test.py
index a64363c266a78..b4c92fe5f508a 100644
--- a/tests/kafkatest/tests/core/security_rolling_upgrade_test.py
+++ b/tests/kafkatest/tests/core/security_rolling_upgrade_test.py
@@ -73,8 +73,8 @@ def roll_in_secured_settings(self, client_protocol, broker_protocol):
         self.kafka.close_port(SecurityConfig.PLAINTEXT)
         self.set_authorizer_and_bounce(client_protocol, broker_protocol)
 
-    def set_authorizer_and_bounce(self, client_protocol, broker_protocol):
-        self.kafka.authorizer_class_name = KafkaService.SIMPLE_AUTHORIZER
+    def set_authorizer_and_bounce(self, client_protocol, broker_protocol, authorizer_class_name = KafkaService.ACL_AUTHORIZER):
+        self.kafka.authorizer_class_name = authorizer_class_name
         self.acls.set_acls(client_protocol, self.kafka, self.topic, self.group)
         self.acls.set_acls(broker_protocol, self.kafka, self.topic, self.group)
         self.bounce()
@@ -95,8 +95,8 @@ def roll_in_sasl_mechanism(self, security_protocol, new_sasl_mechanism):
         self.kafka.interbroker_sasl_mechanism = new_sasl_mechanism
         self.bounce()
 
-        # Bounce again with ACLs for new mechanism
-        self.set_authorizer_and_bounce(security_protocol, security_protocol)
+        # Bounce again with ACLs for new mechanism. Use old SimpleAclAuthorizer here to ensure that is also tested.
+        self.set_authorizer_and_bounce(security_protocol, security_protocol, KafkaService.SIMPLE_AUTHORIZER)
 
     def add_separate_broker_listener(self, broker_security_protocol, broker_sasl_mechanism):
         self.kafka.setup_interbroker_listener(broker_security_protocol, True)
diff --git a/tests/kafkatest/tests/core/zookeeper_security_upgrade_test.py b/tests/kafkatest/tests/core/zookeeper_security_upgrade_test.py
index 235f2fabecf98..d44bd4fd8f62d 100644
--- a/tests/kafkatest/tests/core/zookeeper_security_upgrade_test.py
+++ b/tests/kafkatest/tests/core/zookeeper_security_upgrade_test.py
@@ -102,7 +102,7 @@ def test_zk_security_upgrade(self, security_protocol):
 
         # set acls
         if self.is_secure:
-            self.kafka.authorizer_class_name = KafkaService.SIMPLE_AUTHORIZER
+            self.kafka.authorizer_class_name = KafkaService.ACL_AUTHORIZER
             self.acls.set_acls(security_protocol, self.kafka, self.topic, self.group)
 
         if self.no_sasl:

From 1ae0956892cf7fb6e2337dcdd8cd07d56a22ed54 Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Tue, 24 Sep 2019 09:54:30 -0700
Subject: [PATCH 0640/1071] HOTFIX: fix Kafka Streams upgrade note for broker
 backward compatibility (#7363)

Reviewer: Guozhang Wang 
---
 docs/streams/upgrade-guide.html | 12 ++++++++++--
 docs/upgrade.html               |  6 ++++++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/docs/streams/upgrade-guide.html b/docs/streams/upgrade-guide.html
index 0d10442230e08..bad37c70541f6 100644
--- a/docs/streams/upgrade-guide.html
+++ b/docs/streams/upgrade-guide.html
@@ -53,8 +53,10 @@ 
                Upgrade Guide and API Changes
                
     
 
     
                
-        Note, that a brokers must be on version 0.10.1 or higher to run a Kafka Streams application version 0.10.1 or higher;
-        On-disk message format must be 0.10 or higher to run a Kafka Streams application version 1.0 or higher.
+        To run a Kafka Streams application version 2.2.1, 2.3.0, or higher a broker version 0.11.0 or higher is required
+        and the on-disk message format must be 0.11 or higher.
+        Brokers must be on version 0.10.1 or higher to run a Kafka Streams application version 0.10.1 to 2.2.0.
+        Additionally, on-disk message format must be 0.10 or higher to run a Kafka Streams application version 1.0 to 2.2.0.
         For Kafka Streams 0.10.0, broker version 0.10.0 or higher is required.
     
                
 
@@ -139,6 +141,12 @@ 
                Streams API
         For more details please read KAFKA-8215.
     
                
 
+    
                Notable changes in Kafka Streams 2.2.1
                
+    
                
+        As of Kafka Streams 2.2.1 a message format 0.11 or higher is required;
+        this implies that brokers must be on version 0.11.0 or higher.
+    
                
+
     
                Streams API changes in 2.2.0
                
     
                
         We've simplified the KafkaStreams#state transition diagram during the starting up phase a bit in 2.2.0: in older versions the state will transit from CREATED to RUNNING, and then to REBALANCING to get the first
diff --git a/docs/upgrade.html b/docs/upgrade.html
index f95adfcf00fdf..757243b568ed2 100644
--- a/docs/upgrade.html
+++ b/docs/upgrade.html
@@ -99,6 +99,7 @@ 
                Notable changes in 2
     
              •  Kafka Streams DSL switches its used store types. While this change is mainly transparent to users, there are some corner cases that may require code changes.
         See the Kafka Streams upgrade section for more details.
     
                • 
+    
              • Kafka Streams 2.3.0 requires 0.11 message format or higher and does not work with older message format.
                • 
 
 
 
                Upgrading from 0.8.x, 0.9.x, 0.10.0.x, 0.10.1.x, 0.10.2.x, 0.11.0.x, 1.0.x, 1.1.x, 2.0.x or 2.1.x to 2.2.0
                
@@ -143,6 +144,11 @@ 
                Upgrading from 0.8.x, 0.9.x, 0.1
     
                • 
 
 
+
                Notable changes in 2.2.1
                
+
                  
+    
                • Kafka Streams 2.2.1 requires 0.11 message format or higher and does not work with older message format.
                  • 
+
                
+
 
                Notable changes in 2.2.0
                
 
                  
     
                • The default consumer group id has been changed from the empty string ("") to null. Consumers who use the new default group id will not be able to subscribe to topics,

From bcc023773ff0f5b68f0e808bc7c7aba457a4c472 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Tue, 24 Sep 2019 13:23:27 -0700
Subject: [PATCH 0641/1071] KAFKA-8880: Add overloaded function of
 Consumer.committed (#7304)

1. Add the overloaded functions.
2. Update the code in Streams to use the batch API for better latency (this applies to both active StreamsTask for initialize the offsets, as well as the StandbyTasks for updating offset limits).
3. Also update all unit test to replace the deprecated APIs.

Reviewers: Christopher Pettitt , Kamal Chandraprakash  , Bill Bejeck 
---
 .../kafka/clients/consumer/Consumer.java      | 12 +++
 .../kafka/clients/consumer/KafkaConsumer.java | 80 +++++++++++++++++--
 .../kafka/clients/consumer/MockConsumer.java  | 28 +++++--
 .../clients/consumer/KafkaConsumerTest.java   | 20 ++---
 .../clients/consumer/MockConsumerTest.java    | 10 ++-
 .../kafka/api/ConsumerBounceTest.scala        |  6 +-
 .../kafka/api/PlaintextConsumerTest.scala     | 46 +++++------
 .../kafka/api/TransactionsTest.scala          |  2 +-
 .../admin/ConsumerGroupCommandTest.scala      | 14 ++--
 .../scala/unit/kafka/utils/TestUtils.scala    |  9 ++-
 .../processor/internals/AbstractTask.java     | 20 +++--
 .../internals/ProcessorStateManager.java      |  9 +--
 .../processor/internals/StandbyTask.java      | 34 ++++----
 .../processor/internals/StreamTask.java       | 50 ++++++------
 .../processor/internals/StandbyTaskTest.java  |  8 +-
 .../processor/internals/StreamTaskTest.java   |  4 +-
 .../tools/TransactionalMessageCopier.java     | 10 +--
 17 files changed, 235 insertions(+), 127 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/Consumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/Consumer.java
index 7fdf4398b9add..5136d6886dd2d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/Consumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/Consumer.java
@@ -154,13 +154,25 @@ public interface Consumer extends Closeable {
     /**
      * @see KafkaConsumer#committed(TopicPartition)
      */
+    @Deprecated
     OffsetAndMetadata committed(TopicPartition partition);
 
     /**
      * @see KafkaConsumer#committed(TopicPartition, Duration)
      */
+    @Deprecated
     OffsetAndMetadata committed(TopicPartition partition, final Duration timeout);
 
+    /**
+     * @see KafkaConsumer#committed(Set)
+     */
+    Map committed(Set partitions);
+
+    /**
+     * @see KafkaConsumer#committed(Set, Duration)
+     */
+    Map committed(Set partitions, final Duration timeout);
+
     /**
      * @see KafkaConsumer#metrics()
      */
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index be3e176c2eeb9..7e023415dc3dd 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -1735,7 +1735,10 @@ public long position(TopicPartition partition, final Duration timeout) {
      * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors
      * @throws org.apache.kafka.common.errors.TimeoutException if the committed offset cannot be found before
      *             the timeout specified by {@code default.api.timeout.ms} expires.
+     *
+     * @deprecated since 2.4 Use {@link #committed(Set)} instead
      */
+    @Deprecated
     @Override
     public OffsetAndMetadata committed(TopicPartition partition) {
         return committed(partition, Duration.ofMillis(defaultApiTimeoutMs));
@@ -1745,7 +1748,8 @@ public OffsetAndMetadata committed(TopicPartition partition) {
      * Get the last committed offset for the given partition (whether the commit happened by this process or
      * another). This offset will be used as the position for the consumer in the event of a failure.
      * 
                  
-     * This call will block to do a remote call to get the latest committed offsets from the server.
+     * This call will block until the position can be determined, an unrecoverable error is
+     * encountered (in which case it is thrown to the caller), or the timeout expires.
      *
      * @param partition The partition to check
      * @param timeout  The maximum amount of time to await the current committed offset
@@ -1760,21 +1764,85 @@ public OffsetAndMetadata committed(TopicPartition partition) {
      * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors
      * @throws org.apache.kafka.common.errors.TimeoutException if the committed offset cannot be found before
      *             expiration of the timeout
+     *
+     * @deprecated since 2.4 Use {@link #committed(Set, Duration)} instead
      */
+    @Deprecated
     @Override
     public OffsetAndMetadata committed(TopicPartition partition, final Duration timeout) {
+        return committed(Collections.singleton(partition), timeout).get(partition);
+    }
+
+    /**
+     * Get the last committed offsets for the given partitions (whether the commit happened by this process or
+     * another). The returned offsets will be used as the position for the consumer in the event of a failure.
+     * 
                  
+     * Partitions that do not have a committed offset would not be included in the returned map.
+     * 
                  
+     * If any of the partitions requested do not exist, an exception would be thrown.
+     * 
                  
+     * This call will do a remote call to get the latest committed offsets from the server, and will block until the
+     * committed offsets are gotten successfully, an unrecoverable error is encountered (in which case it is thrown to
+     * the caller), or the timeout specified by {@code default.api.timeout.ms} expires (in which case a
+     * {@link org.apache.kafka.common.errors.TimeoutException} is thrown to the caller).
+     *
+     * @param partitions The partitions to check
+     * @return The latest committed offsets for the given partitions; partitions that do not have any committed offsets
+     *         would not be included in the returned result
+     * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this
+     *             function is called
+     * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while
+     *             this function is called
+     * @throws org.apache.kafka.common.errors.AuthenticationException if authentication fails. See the exception for more details
+     * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the
+     *             configured groupId. See the exception for more details
+     * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors
+     * @throws org.apache.kafka.common.errors.TimeoutException if the committed offset cannot be found before
+     *             the timeout specified by {@code default.api.timeout.ms} expires.
+     */
+    @Override
+    public Map committed(final Set partitions) {
+        return committed(partitions, Duration.ofMillis(defaultApiTimeoutMs));
+    }
+
+    /**
+     * Get the last committed offsets for the given partitions (whether the commit happened by this process or
+     * another). The returned offsets will be used as the position for the consumer in the event of a failure.
+     * 
                  
+     * Partitions that do not have a committed offset would not be included in the returned map.
+     * 
                  
+     * If any of the partitions requested do not exist, an exception would be thrown.
+     * 
                  
+     * This call will block to do a remote call to get the latest committed offsets from the server.
+     *
+     * @param partitions The partitions to check
+     * @param timeout  The maximum amount of time to await the latest committed offsets
+     * @return The latest committed offsets for the given partitions; partitions that do not have any committed offsets
+     *         would not be included in the returned result
+     * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this
+     *             function is called
+     * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while
+     *             this function is called
+     * @throws org.apache.kafka.common.errors.AuthenticationException if authentication fails. See the exception for more details
+     * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the
+     *             configured groupId. See the exception for more details
+     * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors
+     * @throws org.apache.kafka.common.errors.TimeoutException if the committed offset cannot be found before
+     *             expiration of the timeout
+     */
+    @Override
+    public Map committed(final Set partitions, final Duration timeout) {
         acquireAndEnsureOpen();
         try {
             maybeThrowInvalidGroupIdException();
-            Map offsets = coordinator.fetchCommittedOffsets(
-                    Collections.singleton(partition), time.timer(timeout));
+            Map offsets = coordinator.fetchCommittedOffsets(partitions, time.timer(timeout));
             if (offsets == null) {
                 throw new TimeoutException("Timeout of " + timeout.toMillis() + "ms expired before the last " +
-                        "committed offset for partition " + partition + " could be determined. Try tuning default.api.timeout.ms " +
-                        "larger to relax the threshold.");
+                    "committed offset for partitions " + partitions + " could be determined. Try tuning default.api.timeout.ms " +
+                    "larger to relax the threshold.");
             } else {
                 offsets.forEach(this::updateLastSeenEpochIfNewer);
-                return offsets.get(partition);
+                return offsets;
             }
         } finally {
             release();
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java
index 947746f3ce1af..b20ee9783fe90 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java
@@ -42,6 +42,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 
 /**
@@ -290,20 +291,33 @@ public void seek(TopicPartition partition, OffsetAndMetadata offsetAndMetadata)
         subscriptions.seek(partition, offsetAndMetadata.offset());
     }
 
+    @Deprecated
     @Override
-    public synchronized OffsetAndMetadata committed(TopicPartition partition) {
-        ensureNotClosed();
-        if (subscriptions.isAssigned(partition)) {
-            return committed.get(partition);
-        }
-        return new OffsetAndMetadata(0);
+    public synchronized OffsetAndMetadata committed(final TopicPartition partition) {
+        return committed(Collections.singleton(partition)).get(partition);
     }
 
+    @Deprecated
     @Override
-    public OffsetAndMetadata committed(TopicPartition partition, final Duration timeout) {
+    public OffsetAndMetadata committed(final TopicPartition partition, final Duration timeout) {
         return committed(partition);
     }
 
+    @Override
+    public synchronized Map committed(final Set partitions) {
+        ensureNotClosed();
+
+        return partitions.stream()
+            .filter(committed::containsKey)
+            .collect(Collectors.toMap(tp -> tp, tp -> subscriptions.isAssigned(tp) ?
+                committed.get(tp) : new OffsetAndMetadata(0)));
+    }
+
+    @Override
+    public synchronized Map committed(final Set partitions, final Duration timeout) {
+        return committed(partitions);
+    }
+
     @Override
     public synchronized long position(TopicPartition partition) {
         ensureNotClosed();
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
index 7892d6d1c92db..a9c4d259aa4ea 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
@@ -696,7 +696,7 @@ public void testCommitsFetchedDuringAssign() {
 
         // fetch offset for one topic
         client.prepareResponseFrom(offsetResponse(Collections.singletonMap(tp0, offset1), Errors.NONE), coordinator);
-        assertEquals(offset1, consumer.committed(tp0).offset());
+        assertEquals(offset1, consumer.committed(Collections.singleton(tp0)).get(tp0).offset());
 
         consumer.assign(Arrays.asList(tp0, tp1));
 
@@ -704,12 +704,12 @@ public void testCommitsFetchedDuringAssign() {
         Map offsets = new HashMap<>();
         offsets.put(tp0, offset1);
         client.prepareResponseFrom(offsetResponse(offsets, Errors.NONE), coordinator);
-        assertEquals(offset1, consumer.committed(tp0).offset());
+        assertEquals(offset1, consumer.committed(Collections.singleton(tp0)).get(tp0).offset());
 
         offsets.remove(tp0);
         offsets.put(tp1, offset2);
         client.prepareResponseFrom(offsetResponse(offsets, Errors.NONE), coordinator);
-        assertEquals(offset2, consumer.committed(tp1).offset());
+        assertEquals(offset2, consumer.committed(Collections.singleton(tp1)).get(tp1).offset());
         consumer.close(Duration.ofMillis(0));
     }
 
@@ -1137,7 +1137,7 @@ public void testManualAssignmentChangeWithAutoCommitEnabled() {
 
         // fetch offset for one topic
         client.prepareResponseFrom(offsetResponse(Collections.singletonMap(tp0, 0L), Errors.NONE), coordinator);
-        assertEquals(0, consumer.committed(tp0).offset());
+        assertEquals(0, consumer.committed(Collections.singleton(tp0)).get(tp0).offset());
 
         // verify that assignment immediately changes
         assertTrue(consumer.assignment().equals(singleton(tp0)));
@@ -1195,7 +1195,7 @@ public void testManualAssignmentChangeWithAutoCommitDisabled() {
         client.prepareResponseFrom(
                 offsetResponse(Collections.singletonMap(tp0, 0L), Errors.NONE),
                 coordinator);
-        assertEquals(0, consumer.committed(tp0).offset());
+        assertEquals(0, consumer.committed(Collections.singleton(tp0)).get(tp0).offset());
 
         // verify that assignment immediately changes
         assertTrue(consumer.assignment().equals(singleton(tp0)));
@@ -1256,12 +1256,12 @@ public void testOffsetOfPausedPartitions() {
         offsets.put(tp1, 0L);
 
         client.prepareResponseFrom(offsetResponse(offsets, Errors.NONE), coordinator);
-        assertEquals(0, consumer.committed(tp0).offset());
+        assertEquals(0, consumer.committed(Collections.singleton(tp0)).get(tp0).offset());
 
         offsets.remove(tp0);
         offsets.put(tp1, 0L);
         client.prepareResponseFrom(offsetResponse(offsets, Errors.NONE), coordinator);
-        assertEquals(0, consumer.committed(tp1).offset());
+        assertEquals(0, consumer.committed(Collections.singleton(tp1)).get(tp1).offset());
 
         // fetch and verify consumer's position in the two partitions
         final Map offsetResponse = new HashMap<>();
@@ -1356,7 +1356,7 @@ public void testOperationsBySubscribingConsumerWithDefaultGroupId() {
         }
 
         try {
-            newConsumer((String) null).committed(tp0);
+            newConsumer((String) null).committed(Collections.singleton(tp0)).get(tp0);
             fail("Expected an InvalidGroupIdException");
         } catch (InvalidGroupIdException e) {
             // OK, expected
@@ -1383,7 +1383,7 @@ public void testOperationsByAssigningConsumerWithDefaultGroupId() {
         consumer.assign(singleton(tp0));
 
         try {
-            consumer.committed(tp0);
+            consumer.committed(Collections.singleton(tp0)).get(tp0);
             fail("Expected an InvalidGroupIdException");
         } catch (InvalidGroupIdException e) {
             // OK, expected
@@ -1636,7 +1636,7 @@ public void testCommitSyncAuthenticationFailure() {
     @Test(expected = AuthenticationException.class)
     public void testCommittedAuthenticationFaiure() {
         final KafkaConsumer consumer = consumerWithPendingAuthenticationError();
-        consumer.committed(tp0);
+        consumer.committed(Collections.singleton(tp0)).get(tp0);
     }
 
     @Test
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/MockConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/MockConsumerTest.java
index aad4d2973a032..5a012b2cf67a9 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/MockConsumerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/MockConsumerTest.java
@@ -55,9 +55,10 @@ public void testSimpleMock() {
         assertEquals(rec1, iter.next());
         assertEquals(rec2, iter.next());
         assertFalse(iter.hasNext());
-        assertEquals(2L, consumer.position(new TopicPartition("test", 0)));
+        final TopicPartition tp = new TopicPartition("test", 0);
+        assertEquals(2L, consumer.position(tp));
         consumer.commitSync();
-        assertEquals(2L, consumer.committed(new TopicPartition("test", 0)).offset());
+        assertEquals(2L, consumer.committed(Collections.singleton(tp)).get(tp).offset());
     }
 
     @SuppressWarnings("deprecation")
@@ -81,9 +82,10 @@ public void testSimpleMockDeprecated() {
         assertEquals(rec1, iter.next());
         assertEquals(rec2, iter.next());
         assertFalse(iter.hasNext());
-        assertEquals(2L, consumer.position(new TopicPartition("test", 0)));
+        final TopicPartition tp = new TopicPartition("test", 0);
+        assertEquals(2L, consumer.position(tp));
         consumer.commitSync();
-        assertEquals(2L, consumer.committed(new TopicPartition("test", 0)).offset());
+        assertEquals(2L, consumer.committed(Collections.singleton(tp)).get(tp).offset());
     }
 
     @Test
diff --git a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
index 209eac0f506be..e355400fccc58 100644
--- a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
+++ b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
@@ -108,7 +108,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
 
       if (records.nonEmpty) {
         consumer.commitSync()
-        assertEquals(consumer.position(tp), consumer.committed(tp).offset)
+        assertEquals(consumer.position(tp), consumer.committed(Set(tp).asJava).get(tp).offset)
 
         if (consumer.position(tp) == numRecords) {
           consumer.seekToBeginning(Collections.emptyList())
@@ -153,7 +153,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
       } else if (coin == 2) {
         info("Committing offset.")
         consumer.commitSync()
-        assertEquals(consumer.position(tp), consumer.committed(tp).offset)
+        assertEquals(consumer.position(tp), consumer.committed(Set(tp).asJava).get(tp).offset)
       }
     }
   }
@@ -485,7 +485,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
     consumer.poll(time.Duration.ofSeconds(3L))
     assertTrue("Assignment did not complete on time", assignSemaphore.tryAcquire(1, TimeUnit.SECONDS))
     if (committedRecords > 0)
-      assertEquals(committedRecords, consumer.committed(tp).offset)
+      assertEquals(committedRecords, consumer.committed(Set(tp).asJava).get(tp).offset)
     consumer.close()
   }
 
diff --git a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
index acb0d6b17ba76..62b358e6fa24d 100644
--- a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
+++ b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
@@ -278,8 +278,8 @@ class PlaintextConsumerTest extends BaseConsumerTest {
 
     // now we should see the committed positions from another consumer
     val anotherConsumer = createConsumer()
-    assertEquals(300, anotherConsumer.committed(tp).offset)
-    assertEquals(500, anotherConsumer.committed(tp2).offset)
+    assertEquals(300, anotherConsumer.committed(Set(tp).asJava).get(tp).offset)
+    assertEquals(500, anotherConsumer.committed(Set(tp2).asJava).get(tp2).offset)
   }
 
   @Test
@@ -305,8 +305,8 @@ class PlaintextConsumerTest extends BaseConsumerTest {
 
     // now we should see the committed positions from another consumer
     val anotherConsumer = createConsumer()
-    assertEquals(300, anotherConsumer.committed(tp).offset)
-    assertEquals(500, anotherConsumer.committed(tp2).offset)
+    assertEquals(300, anotherConsumer.committed(Set(tp).asJava).get(tp).offset)
+    assertEquals(500, anotherConsumer.committed(Set(tp2).asJava).get(tp2).offset)
   }
 
   @Test
@@ -480,17 +480,17 @@ class PlaintextConsumerTest extends BaseConsumerTest {
     // sync commit
     val syncMetadata = new OffsetAndMetadata(5, Optional.of(15), "foo")
     consumer.commitSync(Map((tp, syncMetadata)).asJava)
-    assertEquals(syncMetadata, consumer.committed(tp))
+    assertEquals(syncMetadata, consumer.committed(Set(tp).asJava).get(tp))
 
     // async commit
     val asyncMetadata = new OffsetAndMetadata(10, "bar")
     sendAndAwaitAsyncCommit(consumer, Some(Map(tp -> asyncMetadata)))
-    assertEquals(asyncMetadata, consumer.committed(tp))
+    assertEquals(asyncMetadata, consumer.committed(Set(tp).asJava).get(tp))
 
     // handle null metadata
     val nullMetadata = new OffsetAndMetadata(5, null)
     consumer.commitSync(Map(tp -> nullMetadata).asJava)
-    assertEquals(nullMetadata, consumer.committed(tp))
+    assertEquals(nullMetadata, consumer.committed(Set(tp).asJava).get(tp))
   }
 
   @Test
@@ -509,7 +509,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
 
     assertEquals(None, callback.lastError)
     assertEquals(count, callback.successCount)
-    assertEquals(new OffsetAndMetadata(count), consumer.committed(tp))
+    assertEquals(new OffsetAndMetadata(count), consumer.committed(Set(tp).asJava).get(tp))
   }
 
   @Test
@@ -623,7 +623,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
     sendRecords(producer, numRecords = 5, tp)
 
     val consumer = createConsumer()
-    assertNull(consumer.committed(new TopicPartition(topic, 15)))
+    assertTrue(consumer.committed(Set(new TopicPartition(topic, 15)).asJava).isEmpty)
 
     // position() on a partition that we aren't subscribed to throws an exception
     intercept[IllegalStateException] {
@@ -634,12 +634,12 @@ class PlaintextConsumerTest extends BaseConsumerTest {
 
     assertEquals("position() on a partition that we are subscribed to should reset the offset", 0L, consumer.position(tp))
     consumer.commitSync()
-    assertEquals(0L, consumer.committed(tp).offset)
+    assertEquals(0L, consumer.committed(Set(tp).asJava).get(tp).offset)
 
     consumeAndVerifyRecords(consumer = consumer, numRecords = 5, startingOffset = 0)
     assertEquals("After consuming 5 records, position should be 5", 5L, consumer.position(tp))
     consumer.commitSync()
-    assertEquals("Committed offset should be returned", 5L, consumer.committed(tp).offset)
+    assertEquals("Committed offset should be returned", 5L, consumer.committed(Set(tp).asJava).get(tp).offset)
 
     sendRecords(producer, numRecords = 1, tp)
 
@@ -1024,12 +1024,12 @@ class PlaintextConsumerTest extends BaseConsumerTest {
     // commit sync and verify onCommit is called
     val commitCountBefore = MockConsumerInterceptor.ON_COMMIT_COUNT.intValue
     testConsumer.commitSync(Map[TopicPartition, OffsetAndMetadata]((tp, new OffsetAndMetadata(2L))).asJava)
-    assertEquals(2, testConsumer.committed(tp).offset)
+    assertEquals(2, testConsumer.committed(Set(tp).asJava).get(tp).offset)
     assertEquals(commitCountBefore + 1, MockConsumerInterceptor.ON_COMMIT_COUNT.intValue)
 
     // commit async and verify onCommit is called
     sendAndAwaitAsyncCommit(testConsumer, Some(Map(tp -> new OffsetAndMetadata(5L))))
-    assertEquals(5, testConsumer.committed(tp).offset)
+    assertEquals(5, testConsumer.committed(Set(tp).asJava).get(tp).offset)
     assertEquals(commitCountBefore + 2, MockConsumerInterceptor.ON_COMMIT_COUNT.intValue)
 
     testConsumer.close()
@@ -1076,8 +1076,8 @@ class PlaintextConsumerTest extends BaseConsumerTest {
                                                     rebalanceListener)
 
     // after rebalancing, we should have reset to the committed positions
-    assertEquals(10, testConsumer.committed(tp).offset)
-    assertEquals(20, testConsumer.committed(tp2).offset)
+    assertEquals(10, testConsumer.committed(Set(tp).asJava).get(tp).offset)
+    assertEquals(20, testConsumer.committed(Set(tp2).asJava).get(tp2).offset)
     assertTrue(MockConsumerInterceptor.ON_COMMIT_COUNT.intValue() > commitCountBeforeRebalance)
 
     // verify commits are intercepted on close
@@ -1321,19 +1321,19 @@ class PlaintextConsumerTest extends BaseConsumerTest {
     val pos1 = consumer.position(tp)
     val pos2 = consumer.position(tp2)
     consumer.commitSync(Map[TopicPartition, OffsetAndMetadata]((tp, new OffsetAndMetadata(3L))).asJava)
-    assertEquals(3, consumer.committed(tp).offset)
-    assertNull(consumer.committed(tp2))
+    assertEquals(3, consumer.committed(Set(tp).asJava).get(tp).offset)
+    assertNull(consumer.committed(Set(tp2).asJava).get(tp2))
 
     // Positions should not change
     assertEquals(pos1, consumer.position(tp))
     assertEquals(pos2, consumer.position(tp2))
     consumer.commitSync(Map[TopicPartition, OffsetAndMetadata]((tp2, new OffsetAndMetadata(5L))).asJava)
-    assertEquals(3, consumer.committed(tp).offset)
-    assertEquals(5, consumer.committed(tp2).offset)
+    assertEquals(3, consumer.committed(Set(tp).asJava).get(tp).offset)
+    assertEquals(5, consumer.committed(Set(tp2).asJava).get(tp2).offset)
 
     // Using async should pick up the committed changes after commit completes
     sendAndAwaitAsyncCommit(consumer, Some(Map(tp2 -> new OffsetAndMetadata(7L))))
-    assertEquals(7, consumer.committed(tp2).offset)
+    assertEquals(7, consumer.committed(Set(tp2).asJava).get(tp2).offset)
   }
 
   @Test
@@ -1371,8 +1371,8 @@ class PlaintextConsumerTest extends BaseConsumerTest {
     awaitAssignment(consumer, newAssignment)
 
     // after rebalancing, we should have reset to the committed positions
-    assertEquals(300, consumer.committed(tp).offset)
-    assertEquals(500, consumer.committed(tp2).offset)
+    assertEquals(300, consumer.committed(Set(tp).asJava).get(tp).offset)
+    assertEquals(500, consumer.committed(Set(tp2).asJava).get(tp2).offset)
   }
 
   @Test
@@ -1808,7 +1808,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
     }
 
     try {
-      consumer2.committed(tp)
+      consumer2.committed(Set(tp).asJava)
       fail("Expected committed offset fetch to fail due to null group id")
     } catch {
       case e: InvalidGroupIdException => // OK
diff --git a/core/src/test/scala/integration/kafka/api/TransactionsTest.scala b/core/src/test/scala/integration/kafka/api/TransactionsTest.scala
index 254ccadc89096..e1bf3db4ff9ef 100644
--- a/core/src/test/scala/integration/kafka/api/TransactionsTest.scala
+++ b/core/src/test/scala/integration/kafka/api/TransactionsTest.scala
@@ -389,7 +389,7 @@ class TransactionsTest extends KafkaServerTestHarness {
     val producer2 = transactionalProducers(1)
     producer2.initTransactions()
 
-    assertEquals(offsetAndMetadata, consumer.committed(tp))
+    assertEquals(offsetAndMetadata, consumer.committed(Set(tp).asJava).get(tp))
   }
 
   @Test
diff --git a/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala b/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
index fdadec41e3727..3ee83e25eeab9 100644
--- a/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
@@ -27,7 +27,7 @@ import kafka.server.KafkaConfig
 import kafka.utils.TestUtils
 import org.apache.kafka.clients.admin.AdminClientConfig
 import org.apache.kafka.clients.consumer.{KafkaConsumer, RangeAssignor}
-import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.{PartitionInfo, TopicPartition}
 import org.apache.kafka.common.errors.WakeupException
 import org.apache.kafka.common.serialization.StringDeserializer
 import org.junit.{After, Before}
@@ -70,14 +70,10 @@ class ConsumerGroupCommandTest extends KafkaServerTestHarness {
     props.put("group.id", group)
     val consumer = new KafkaConsumer(props, new StringDeserializer, new StringDeserializer)
     try {
-      consumer.partitionsFor(topic).asScala.flatMap { partitionInfo =>
-        val tp = new TopicPartition(partitionInfo.topic, partitionInfo.partition)
-        val committed = consumer.committed(tp)
-        if (committed == null)
-          None
-        else
-          Some(tp -> committed.offset)
-      }.toMap
+      val partitions: Set[TopicPartition] = consumer.partitionsFor(topic)
+        .asScala.toSet.map {partitionInfo : PartitionInfo => new TopicPartition(partitionInfo.topic, partitionInfo.partition)}
+
+      consumer.committed(partitions.asJava).asScala.mapValues(_.offset()).toMap
     } finally {
       consumer.close()
     }
diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index d28bd99c55382..c1951c010c53f 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -1432,11 +1432,12 @@ object TestUtils extends Logging {
     offsetsToCommit.toMap
   }
 
-  def resetToCommittedPositions(consumer: KafkaConsumer[Array[Byte], Array[Byte]]) = {
+  def resetToCommittedPositions(consumer: KafkaConsumer[Array[Byte], Array[Byte]]) {
+    val committed = consumer.committed(consumer.assignment).asScala.mapValues(_.offset)
+
     consumer.assignment.asScala.foreach { topicPartition =>
-      val offset = consumer.committed(topicPartition)
-      if (offset != null)
-        consumer.seek(topicPartition, offset.offset)
+      if (committed.contains(topicPartition))
+        consumer.seek(topicPartition, committed(topicPartition))
       else
         consumer.seekToBeginning(Collections.singletonList(topicPartition))
     }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java
index 798f0b089a032..03a002127b7f3 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java
@@ -17,7 +17,6 @@
 package org.apache.kafka.streams.processor.internals;
 
 import org.apache.kafka.clients.consumer.Consumer;
-import org.apache.kafka.clients.consumer.OffsetAndMetadata;
 import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.errors.AuthorizationException;
@@ -35,7 +34,9 @@
 import java.io.IOException;
 import java.util.Collection;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 public abstract class AbstractTask implements Task {
 
@@ -250,16 +251,23 @@ public Collection changelogPartitions() {
         return stateMgr.changelogPartitions();
     }
 
-    long committedOffsetForPartition(final TopicPartition partition) {
+    Map committedOffsetForPartitions(final Set partitions) {
         try {
-            final OffsetAndMetadata metadata = consumer.committed(partition);
-            return metadata != null ? metadata.offset() : 0L;
+            final Map results = consumer.committed(partitions)
+                .entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().offset()));
+
+            // those do not have a committed offset would default to 0
+            for (final TopicPartition tp : partitions) {
+                results.putIfAbsent(tp, 0L);
+            }
+
+            return results;
         } catch (final AuthorizationException e) {
-            throw new ProcessorStateException(String.format("task [%s] AuthorizationException when initializing offsets for %s", id, partition), e);
+            throw new ProcessorStateException(String.format("task [%s] AuthorizationException when initializing offsets for %s", id, partitions), e);
         } catch (final WakeupException e) {
             throw e;
         } catch (final KafkaException e) {
-            throw new ProcessorStateException(String.format("task [%s] Failed to initialize offsets for %s", id, partition), e);
+            throw new ProcessorStateException(String.format("task [%s] Failed to initialize offsets for %s", id, partitions), e);
         }
     }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorStateManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorStateManager.java
index 2e80c94e5f570..9f5f29fcdfe07 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorStateManager.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorStateManager.java
@@ -247,13 +247,12 @@ void updateStandbyStates(final TopicPartition storePartition,
         standbyRestoredOffsets.put(storePartition, lastOffset + 1);
     }
 
-    void putOffsetLimit(final TopicPartition partition,
-                        final long limit) {
-        log.trace("Updating store offset limit for partition {} to {}", partition, limit);
-        offsetLimits.put(partition, limit);
+    void putOffsetLimits(final Map offsets) {
+        log.trace("Updating store offset limit with {}", offsets);
+        offsetLimits.putAll(offsets);
     }
 
-    long offsetLimit(final TopicPartition partition) {
+    private long offsetLimit(final TopicPartition partition) {
         final Long limit = offsetLimits.get(partition);
         return limit != null ? limit : Long.MAX_VALUE;
     }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
index 17b6e665d46af..071b7d667bbbc 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
@@ -37,10 +37,10 @@
  * A StandbyTask
  */
 public class StandbyTask extends AbstractTask {
-    private Map checkpointedOffsets = new HashMap<>();
+    private boolean updateOffsetLimits;
     private final Sensor closeTaskSensor;
     private final Map offsetLimits = new HashMap<>();
-    private final Set updateableOffsetLimits = new HashSet<>();
+    private Map checkpointedOffsets = new HashMap<>();
 
     /**
      * Create {@link StandbyTask} with its assigned partitions
@@ -69,10 +69,8 @@ public class StandbyTask extends AbstractTask {
         final Set changelogTopicNames = new HashSet<>(topology.storeToChangelogTopic().values());
         partitions.stream()
             .filter(tp -> changelogTopicNames.contains(tp.topic()))
-            .forEach(tp -> {
-                offsetLimits.put(tp, 0L);
-                updateableOffsetLimits.add(tp);
-            });
+            .forEach(tp -> offsetLimits.put(tp, 0L));
+        updateOffsetLimits = true;
     }
 
     @Override
@@ -193,7 +191,7 @@ public List> update(final TopicPartition partitio
             // Check if we're unable to process records due to an offset limit (e.g. when our
             // partition is both a source and a changelog). If we're limited then try to refresh
             // the offset limit if possible.
-            if (record.offset() >= limit && updateableOffsetLimits.contains(partition)) {
+            if (record.offset() >= limit && updateOffsetLimits) {
                 limit = updateOffsetLimits(partition);
             }
 
@@ -222,18 +220,24 @@ private long updateOffsetLimits(final TopicPartition partition) {
             throw new IllegalArgumentException("Topic is not both a source and a changelog: " + partition);
         }
 
-        updateableOffsetLimits.remove(partition);
+        final Map newLimits = committedOffsetForPartitions(offsetLimits.keySet());
+
+        for (final Map.Entry newlimit : newLimits.entrySet()) {
+            final Long previousLimit = offsetLimits.get(newlimit.getKey());
+            if (previousLimit != null && previousLimit > newlimit.getValue()) {
+                throw new IllegalStateException("Offset limit should monotonically increase, but was reduced. " +
+                    "New limit: " + newlimit.getValue() + ". Previous limit: " + previousLimit);
+            }
 
-        final long newLimit = committedOffsetForPartition(partition);
-        final long previousLimit = offsetLimits.put(partition, newLimit);
-        if (previousLimit > newLimit) {
-            throw new IllegalStateException("Offset limit should monotonically increase, but was reduced. " +
-                "New limit: " + newLimit + ". Previous limit: " + previousLimit);
         }
-        return newLimit;
+
+        offsetLimits.putAll(newLimits);
+        updateOffsetLimits = false;
+
+        return offsetLimits.get(partition);
     }
 
     void allowUpdateOfOffsetLimit() {
-        updateableOffsetLimits.addAll(offsetLimits.keySet());
+        updateOffsetLimits = true;
     }
 }
\ No newline at end of file
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
index 3eb1e55908991..ccf522813f362 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
@@ -31,6 +31,8 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
 import org.apache.kafka.clients.consumer.CommitFailedException;
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
@@ -249,13 +251,11 @@ public boolean initializeStateStores() {
         // partitions of topics that are both sources and changelogs and set the consumer committed
         // offset via stateMgr as there is not a more direct route.
         final Set changelogTopicNames = new HashSet<>(topology.storeToChangelogTopic().values());
-        partitions.stream()
-            .filter(tp -> changelogTopicNames.contains(tp.topic()))
-            .forEach(tp -> {
-                final long offset = committedOffsetForPartition(tp);
-                stateMgr.putOffsetLimit(tp, offset);
-                log.trace("Updating store offset limits {} for changelog {}", offset, tp);
-            });
+        final Set sourcePartitionsAsChangelog = new HashSet<>(partitions)
+            .stream().filter(tp -> changelogTopicNames.contains(tp.topic())).collect(Collectors.toSet());
+        final Map committedOffsets = committedOffsetForPartitions(sourcePartitionsAsChangelog);
+        stateMgr.putOffsetLimits(committedOffsets);
+
         registerStateStores();
 
         return changelogPartitions().isEmpty();
@@ -481,7 +481,6 @@ void commit(final boolean startNewTransaction, final Map p
             final long offset = entry.getValue() + 1;
             final long partitionTime = partitionTimes.get(partition);
             consumedOffsetsAndMetadata.put(partition, new OffsetAndMetadata(offset, encodeTimestamp(partitionTime)));
-            stateMgr.putOffsetLimit(partition, offset);
         }
 
         try {
@@ -735,25 +734,30 @@ public void close(boolean clean,
         taskClosed = true;
     }
 
-    private void initializeCommittedTimestamp(final TopicPartition partition) {
-        final OffsetAndMetadata metadata = consumer.committed(partition);
-
-        if (metadata != null) {
-            final long committedTimestamp = decodeTimestamp(metadata.metadata());
-            partitionGroup.setPartitionTime(partition, committedTimestamp);
-            log.debug("A committed timestamp was detected: setting the partition time of partition {}"
-                      + " to {} in stream task {}", partition, committedTimestamp, this);
-        } else {
-            log.debug("No committed timestamp was found in metadata for partition {}", partition);
-        }
-    }
-
     /**
      * Retrieves formerly committed timestamps and updates the local queue's partition time.
      */
     public void initializeTaskTime() {
-        for (final TopicPartition partition : partitionGroup.partitions()) {
-            initializeCommittedTimestamp(partition);
+        final Map committed = consumer.committed(partitionGroup.partitions());
+
+        for (final Map.Entry entry : committed.entrySet()) {
+            final TopicPartition partition = entry.getKey();
+            final OffsetAndMetadata metadata = entry.getValue();
+
+            if (metadata != null) {
+                final long committedTimestamp = decodeTimestamp(metadata.metadata());
+                partitionGroup.setPartitionTime(partition, committedTimestamp);
+                log.debug("A committed timestamp was detected: setting the partition time of partition {}"
+                    + " to {} in stream task {}", partition, committedTimestamp, this);
+            } else {
+                log.debug("No committed timestamp was found in metadata for partition {}", partition);
+            }
+        }
+
+        final Set nonCommitted = new HashSet<>(partitionGroup.partitions());
+        nonCommitted.removeAll(committed.keySet());
+        for (final TopicPartition partition : nonCommitted) {
+            log.debug("No committed offset for partition {}, therefore no timestamp can be found for this partition", partition);
         }
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
index 033aeeefd7fdd..6f42fb26526e6 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
@@ -563,9 +563,9 @@ public void shouldNotGetConsumerCommittedOffsetIfThereAreNoRecordUpdates() throw
 
         final Consumer consumer = new MockConsumer(OffsetResetStrategy.EARLIEST) {
             @Override
-            public synchronized OffsetAndMetadata committed(final TopicPartition partition) {
+            public synchronized Map committed(final Set partitions) {
                 committedCallCount.getAndIncrement();
-                return super.committed(partition);
+                return super.committed(partitions);
             }
         };
 
@@ -596,9 +596,9 @@ public void shouldGetConsumerCommittedOffsetsOncePerCommit() throws IOException
 
         final Consumer consumer = new MockConsumer(OffsetResetStrategy.EARLIEST) {
             @Override
-            public synchronized OffsetAndMetadata committed(final TopicPartition partition) {
+            public synchronized Map committed(final Set partitions) {
                 committedCallCount.getAndIncrement();
-                return super.committed(partition);
+                return super.committed(partitions);
             }
         };
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
index e90151f282270..a4f529d8795f1 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
@@ -657,7 +657,7 @@ public void shouldRespectCommitNeeded() {
     public void shouldRestorePartitionTimeAfterRestartWithEosDisabled() {
         createTaskWithProcessAndCommit(false);
 
-        assertEquals(DEFAULT_TIMESTAMP, task.decodeTimestamp(consumer.committed(partition1).metadata()));
+        assertEquals(DEFAULT_TIMESTAMP, task.decodeTimestamp(consumer.committed(Collections.singleton(partition1)).get(partition1).metadata()));
         // reset times here by creating a new task
         task = createStatelessTask(createConfig(false));
 
@@ -1585,7 +1585,7 @@ public void shouldThrowWakeupExceptionOnInitializeOffsetsWhenWakeupException() {
     private Consumer mockConsumerWithCommittedException(final RuntimeException toThrow) {
         return new MockConsumer(OffsetResetStrategy.EARLIEST) {
             @Override
-            public OffsetAndMetadata committed(final TopicPartition partition) {
+            public Map committed(final Set partitions) {
                 throw toThrow;
             }
         };
diff --git a/tools/src/main/java/org/apache/kafka/tools/TransactionalMessageCopier.java b/tools/src/main/java/org/apache/kafka/tools/TransactionalMessageCopier.java
index a0ac1f188f21b..cfbac1a4e4c49 100644
--- a/tools/src/main/java/org/apache/kafka/tools/TransactionalMessageCopier.java
+++ b/tools/src/main/java/org/apache/kafka/tools/TransactionalMessageCopier.java
@@ -195,13 +195,13 @@ private static Map consumerPositions(KafkaCon
     }
 
     private static void resetToLastCommittedPositions(KafkaConsumer consumer) {
-        for (TopicPartition topicPartition : consumer.assignment()) {
-            OffsetAndMetadata offsetAndMetadata = consumer.committed(topicPartition);
+        final Map committed = consumer.committed(consumer.assignment());
+        committed.forEach((tp, offsetAndMetadata) -> {
             if (offsetAndMetadata != null)
-                consumer.seek(topicPartition, offsetAndMetadata.offset());
+                consumer.seek(tp, offsetAndMetadata.offset());
             else
-                consumer.seekToBeginning(singleton(topicPartition));
-        }
+                consumer.seekToBeginning(singleton(tp));
+        });
     }
 
     private static long messagesRemaining(KafkaConsumer consumer, TopicPartition partition) {

From ad3b8437fd9ecc18fcfa8204e43be9bccb1a6548 Mon Sep 17 00:00:00 2001
From: Bruno Cadonna 
Date: Tue, 24 Sep 2019 22:29:08 +0200
Subject: [PATCH 0642/1071] KAFKA-8580: Compute RocksDB metrics (#7263)

A metric recorder runs in it own thread and regularly records RocksDB metrics from
RocksDB's statistics. For segmented state stores the metrics are aggregated over the
segments.

Reviewers: John Roesler , A. Sophie Blee-Goldman , Guozhang Wang 
---
 .../apache/kafka/streams/KafkaStreams.java    |  72 ++--
 .../internals/GlobalStreamThread.java         |   7 +-
 .../processor/internals/StreamThread.java     |   5 +
 .../internals/metrics/StreamsMetricsImpl.java |  30 +-
 .../state/internals/KeyValueSegments.java     |   8 +-
 .../streams/state/internals/RocksDBStore.java |  34 +-
 .../state/internals/TimestampedSegments.java  |   6 -
 .../internals/metrics/RocksDBMetrics.java     |   3 +-
 .../metrics/RocksDBMetricsRecorder.java       | 182 ++++++----
 .../RocksDBMetricsRecordingTrigger.java       |  56 ++++
 .../integration/MetricsIntegrationTest.java   |   8 +-
 .../internals/GlobalStreamThreadTest.java     |  42 ++-
 .../processor/internals/StreamThreadTest.java |   3 +-
 .../state/internals/RocksDBStoreTest.java     | 142 +++++---
 .../state/internals/SegmentIteratorTest.java  |   5 +-
 .../metrics/RocksDBMetricsRecorderTest.java   | 315 +++++++++++++++---
 .../RocksDBMetricsRecordingTriggerTest.java   |  87 +++++
 .../internals/metrics/RocksDBMetricsTest.java |  22 +-
 .../test/InternalMockProcessorContext.java    |   2 +
 .../kafka/streams/TopologyTestDriver.java     |   2 +
 20 files changed, 782 insertions(+), 249 deletions(-)
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecordingTrigger.java
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecordingTriggerTest.java

diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
index 5f2780947e96c..e90caae3667d8 100644
--- a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
+++ b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
@@ -29,6 +29,7 @@
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.MetricsReporter;
 import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.metrics.Sensor.RecordingLevel;
 import org.apache.kafka.common.serialization.Serializer;
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.common.utils.Time;
@@ -59,6 +60,7 @@
 import org.apache.kafka.streams.state.internals.QueryableStoreProvider;
 import org.apache.kafka.streams.state.internals.StateStoreProvider;
 import org.apache.kafka.streams.state.internals.StreamThreadStateStoreProvider;
+import org.apache.kafka.streams.state.internals.metrics.RocksDBMetricsRecordingTrigger;
 import org.slf4j.Logger;
 
 import java.time.Duration;
@@ -80,6 +82,7 @@
 
 import static org.apache.kafka.common.utils.Utils.getHost;
 import static org.apache.kafka.common.utils.Utils.getPort;
+import static org.apache.kafka.streams.StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG;
 import static org.apache.kafka.streams.internals.ApiUtils.prepareMillisCheckFailMsgPrefix;
 
 /**
@@ -138,6 +141,7 @@ public class KafkaStreams implements AutoCloseable {
     private final StateDirectory stateDirectory;
     private final StreamsMetadataState streamsMetadataState;
     private final ScheduledExecutorService stateDirCleaner;
+    private final ScheduledExecutorService rocksDBMetricsRecordingTriggerThread;
     private final QueryableStoreProvider queryableStoreProvider;
     private final Admin adminClient;
 
@@ -145,6 +149,8 @@ public class KafkaStreams implements AutoCloseable {
     private KafkaStreams.StateListener stateListener;
     private StateRestoreListener globalStateRestoreListener;
 
+    private final RocksDBMetricsRecordingTrigger rocksDBMetricsRecordingTrigger = new RocksDBMetricsRecordingTrigger();
+
     // container states
     /**
      * Kafka Streams states are the possible state that a Kafka Streams instance can be in.
@@ -698,15 +704,18 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder,
         GlobalStreamThread.State globalThreadState = null;
         if (globalTaskTopology != null) {
             final String globalThreadId = clientId + "-GlobalStreamThread";
-            globalStreamThread = new GlobalStreamThread(globalTaskTopology,
-                                                        config,
-                                                        clientSupplier.getGlobalConsumer(config.getGlobalConsumerConfigs(clientId)),
-                                                        stateDirectory,
-                                                        cacheSizePerThread,
-                                                        metrics,
-                                                        time,
-                                                        globalThreadId,
-                                                        delegatingStateRestoreListener);
+            globalStreamThread = new GlobalStreamThread(
+                globalTaskTopology,
+                config,
+                clientSupplier.getGlobalConsumer(config.getGlobalConsumerConfigs(clientId)),
+                stateDirectory,
+                cacheSizePerThread,
+                metrics,
+                time,
+                globalThreadId,
+                delegatingStateRestoreListener,
+                rocksDBMetricsRecordingTrigger
+            );
             globalThreadState = globalStreamThread.state();
         }
 
@@ -716,19 +725,21 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder,
         final Map threadState = new HashMap<>(threads.length);
         final ArrayList storeProviders = new ArrayList<>();
         for (int i = 0; i < threads.length; i++) {
-            threads[i] = StreamThread.create(internalTopologyBuilder,
-                                             config,
-                                             clientSupplier,
-                                             adminClient,
-                                             processId,
-                                             clientId,
-                                             metrics,
-                                             time,
-                                             streamsMetadataState,
-                                             cacheSizePerThread,
-                                             stateDirectory,
-                                             delegatingStateRestoreListener,
-                                             i + 1);
+            threads[i] = StreamThread.create(
+                internalTopologyBuilder,
+                config,
+                clientSupplier,
+                adminClient,
+                processId,
+                clientId,
+                metrics,
+                time,
+                streamsMetadataState,
+                cacheSizePerThread,
+                stateDirectory,
+                delegatingStateRestoreListener,
+                i + 1);
+            threads[i].setRocksDBMetricsRecordingTrigger(rocksDBMetricsRecordingTrigger);
             threadState.put(threads[i].getId(), threads[i].state());
             storeProviders.add(new StreamThreadStateStoreProvider(threads[i]));
         }
@@ -749,6 +760,12 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder,
             thread.setDaemon(true);
             return thread;
         });
+
+        rocksDBMetricsRecordingTriggerThread = Executors.newSingleThreadScheduledExecutor(r -> {
+            final Thread thread = new Thread(r, clientId + "-RocksDBMetricsRecordingTrigger");
+            thread.setDaemon(true);
+            return thread;
+        });
     }
 
     private static HostInfo parseHostInfo(final String endPoint) {
@@ -803,6 +820,17 @@ public synchronized void start() throws IllegalStateException, StreamsException
                     stateDirectory.cleanRemovedTasks(cleanupDelay);
                 }
             }, cleanupDelay, cleanupDelay, TimeUnit.MILLISECONDS);
+
+            final long recordingDelay = 0;
+            final long recordingInterval = 1;
+            if (RecordingLevel.forName(config.getString(METRICS_RECORDING_LEVEL_CONFIG)) == RecordingLevel.DEBUG) {
+                rocksDBMetricsRecordingTriggerThread.scheduleAtFixedRate(
+                    rocksDBMetricsRecordingTrigger,
+                    recordingDelay,
+                    recordingInterval,
+                    TimeUnit.MINUTES
+                );
+            }
         } else {
             throw new IllegalStateException("The client is either already started or already stopped, cannot re-start");
         }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStreamThread.java
index f0aaab64b56e5..5248aa4f8ab9c 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStreamThread.java
@@ -33,6 +33,7 @@
 import org.apache.kafka.streams.processor.StateRestoreListener;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.apache.kafka.streams.state.internals.ThreadCache;
+import org.apache.kafka.streams.state.internals.metrics.RocksDBMetricsRecordingTrigger;
 import org.slf4j.Logger;
 
 import java.io.IOException;
@@ -182,18 +183,20 @@ public GlobalStreamThread(final ProcessorTopology topology,
                               final Metrics metrics,
                               final Time time,
                               final String threadClientId,
-                              final StateRestoreListener stateRestoreListener) {
+                              final StateRestoreListener stateRestoreListener,
+                              final RocksDBMetricsRecordingTrigger rocksDBMetricsRecordingTrigger) {
         super(threadClientId);
         this.time = time;
         this.config = config;
         this.topology = topology;
         this.globalConsumer = globalConsumer;
         this.stateDirectory = stateDirectory;
-        this.streamsMetrics = new StreamsMetricsImpl(
+        streamsMetrics = new StreamsMetricsImpl(
             metrics,
             threadClientId,
             config.getString(StreamsConfig.BUILT_IN_METRICS_VERSION_CONFIG)
         );
+        streamsMetrics.setRocksDBMetricsRecordingTrigger(rocksDBMetricsRecordingTrigger);
         this.logPrefix = String.format("global-stream-thread [%s] ", threadClientId);
         this.logContext = new LogContext(logPrefix);
         this.log = logContext.logger(getClass());
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index d17768b0886d7..b11b05c30f661 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -46,6 +46,7 @@
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.apache.kafka.streams.state.internals.ThreadCache;
+import org.apache.kafka.streams.state.internals.metrics.RocksDBMetricsRecordingTrigger;
 import org.slf4j.Logger;
 
 import java.time.Duration;
@@ -748,6 +749,10 @@ public static String getSharedAdminClientId(final String clientId) {
         return clientId + "-admin";
     }
 
+    public void setRocksDBMetricsRecordingTrigger(final RocksDBMetricsRecordingTrigger rocksDBMetricsRecordingTrigger) {
+        streamsMetrics.setRocksDBMetricsRecordingTrigger(rocksDBMetricsRecordingTrigger);
+    }
+
     /**
      * Execute the stream processors
      *
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
index 961bec80044fd..0341ab768cdc8 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
@@ -32,6 +32,7 @@
 import org.apache.kafka.common.metrics.stats.WindowedSum;
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.StreamsMetrics;
+import org.apache.kafka.streams.state.internals.metrics.RocksDBMetricsRecordingTrigger;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -61,6 +62,8 @@ public enum Version {
     private final Map> cacheLevelSensors = new HashMap<>();
     private final Map> storeLevelSensors = new HashMap<>();
 
+    private RocksDBMetricsRecordingTrigger rocksDBMetricsRecordingTrigger;
+
     private static final String SENSOR_PREFIX_DELIMITER = ".";
     private static final String SENSOR_NAME_DELIMITER = ".s.";
 
@@ -122,6 +125,14 @@ public Version version() {
         return version;
     }
 
+    public void setRocksDBMetricsRecordingTrigger(final RocksDBMetricsRecordingTrigger rocksDBMetricsRecordingTrigger) {
+        this.rocksDBMetricsRecordingTrigger = rocksDBMetricsRecordingTrigger;
+    }
+
+    public RocksDBMetricsRecordingTrigger rocksDBMetricsRecordingTrigger() {
+        return rocksDBMetricsRecordingTrigger;
+    }
+
     public final Sensor threadLevelSensor(final String sensorName,
                                           final RecordingLevel recordingLevel,
                                           final Sensor... parents) {
@@ -586,7 +597,24 @@ public static void addSumMetricToSensor(final Sensor sensor,
                                             final Map tags,
                                             final String operation,
                                             final String description) {
-        sensor.add(new MetricName(operation + TOTAL_SUFFIX, group, description, tags), new CumulativeSum());
+        addSumMetricToSensor(sensor, group, tags, operation, true, description);
+    }
+
+    public static void addSumMetricToSensor(final Sensor sensor,
+                                            final String group,
+                                            final Map tags,
+                                            final String operation,
+                                            final boolean withSuffix,
+                                            final String description) {
+        sensor.add(
+            new MetricName(
+                withSuffix ? operation + TOTAL_SUFFIX : operation,
+                group,
+                description,
+                tags
+            ),
+            new CumulativeSum()
+        );
     }
 
     public static void addValueMetricToSensor(final Sensor sensor,
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/KeyValueSegments.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/KeyValueSegments.java
index 7735893754621..01510c14790a5 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/KeyValueSegments.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/KeyValueSegments.java
@@ -24,7 +24,7 @@
  */
 class KeyValueSegments extends AbstractSegments {
 
-    final private RocksDBMetricsRecorder metricsRecorder;
+    private final RocksDBMetricsRecorder metricsRecorder;
 
     KeyValueSegments(final String name,
                      final String metricsScope,
@@ -51,10 +51,4 @@ public KeyValueSegment getOrCreateSegment(final long segmentId,
             return newSegment;
         }
     }
-
-    @Override
-    public void close() {
-        metricsRecorder.close();
-        super.close();
-    }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
index 4e56868e6b9ea..74ccd04becfb3 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
@@ -104,8 +104,7 @@ public class RocksDBStore implements KeyValueStore, BulkLoadingSt
     private RocksDBConfigSetter configSetter;
 
     private final RocksDBMetricsRecorder metricsRecorder;
-    private boolean closeMetricsRecorder = false;
-    private boolean removeStatisticsFromMetricsRecorder = false;
+    private boolean isStatisticsRegistered = false;
 
     private volatile boolean prepareForBulkload = false;
     ProcessorContext internalProcessorContext;
@@ -117,7 +116,6 @@ public class RocksDBStore implements KeyValueStore, BulkLoadingSt
     RocksDBStore(final String name,
                  final String metricsScope) {
         this(name, DB_FILE_DIR, new RocksDBMetricsRecorder(metricsScope, name));
-        closeMetricsRecorder = true;
     }
 
     RocksDBStore(final String name,
@@ -190,26 +188,21 @@ void openDB(final ProcessorContext context) {
             throw new ProcessorStateException(fatal);
         }
 
-        setUpMetrics(context, configs);
+        maybeSetUpMetricsRecorder(context, configs);
 
         openRocksDB(dbOptions, columnFamilyOptions);
         open = true;
     }
 
-    private void setUpMetrics(final ProcessorContext context, final Map configs) {
+    private void maybeSetUpMetricsRecorder(final ProcessorContext context, final Map configs) {
         if (userSpecifiedOptions.statistics() == null &&
             RecordingLevel.forName((String) configs.get(METRICS_RECORDING_LEVEL_CONFIG)) == RecordingLevel.DEBUG) {
 
+            isStatisticsRegistered = true;
             // metrics recorder will clean up statistics object
             final Statistics statistics = new Statistics();
             userSpecifiedOptions.setStatistics(statistics);
-            metricsRecorder.addStatistics(
-                name,
-                statistics,
-                (StreamsMetricsImpl) context.metrics(),
-                context.taskId()
-            );
-            removeStatisticsFromMetricsRecorder = true;
+            metricsRecorder.addStatistics(name, statistics, (StreamsMetricsImpl) context.metrics(), context.taskId());
         }
     }
 
@@ -434,7 +427,7 @@ public synchronized void close() {
             configSetter = null;
         }
 
-        closeOrUpdateMetricsRecorder();
+        maybeRemoveStatisticsFromMetricsRecorder();
 
         // Important: do not rearrange the order in which the below objects are closed!
         // Order of closing must follow: ColumnFamilyHandle > RocksDB > DBOptions > ColumnFamilyOptions
@@ -455,6 +448,13 @@ public synchronized void close() {
         cache = null;
     }
 
+    private void maybeRemoveStatisticsFromMetricsRecorder() {
+        if (isStatisticsRegistered) {
+            metricsRecorder.removeStatistics(name);
+            isStatisticsRegistered = false;
+        }
+    }
+
     private void closeOpenIterators() {
         final HashSet> iterators;
         synchronized (openIterators) {
@@ -468,14 +468,6 @@ private void closeOpenIterators() {
         }
     }
 
-    private void closeOrUpdateMetricsRecorder() {
-        if (closeMetricsRecorder) {
-            metricsRecorder.close();
-        } else if (removeStatisticsFromMetricsRecorder) {
-            metricsRecorder.removeStatistics(name);
-        }
-    }
-
     interface RocksDBAccessor {
 
         void put(final byte[] key,
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/TimestampedSegments.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/TimestampedSegments.java
index eb713390002ca..6545de0f691fc 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/TimestampedSegments.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/TimestampedSegments.java
@@ -51,10 +51,4 @@ public TimestampedSegment getOrCreateSegment(final long segmentId,
             return newSegment;
         }
     }
-
-    @Override
-    public void close() {
-        metricsRecorder.close();
-        super.close();
-    }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetrics.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetrics.java
index a0b8dc64a63d8..04da4bc53ab7b 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetrics.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetrics.java
@@ -366,12 +366,13 @@ public static Sensor compactionTimeMaxSensor(final StreamsMetricsImpl streamsMet
     public static Sensor numberOfOpenFilesSensor(final StreamsMetricsImpl streamsMetrics,
                                                  final RocksDBMetricContext metricContext) {
         final Sensor sensor = createSensor(streamsMetrics, metricContext, NUMBER_OF_OPEN_FILES);
-        addValueMetricToSensor(
+        addSumMetricToSensor(
             sensor,
             STATE_LEVEL_GROUP,
             streamsMetrics
                 .storeLevelTagMap(metricContext.taskName(), metricContext.metricsScope(), metricContext.storeName()),
             NUMBER_OF_OPEN_FILES,
+            false,
             NUMBER_OF_OPEN_FILES_DESCRIPTION
         );
         return sensor;
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorder.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorder.java
index 5649f4ae06dca..de2a37c07bec1 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorder.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorder.java
@@ -17,117 +17,175 @@
 package org.apache.kafka.streams.state.internals.metrics;
 
 import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.streams.processor.TaskId;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.apache.kafka.streams.state.internals.metrics.RocksDBMetrics.RocksDBMetricContext;
 import org.rocksdb.Statistics;
 import org.rocksdb.StatsLevel;
+import org.rocksdb.TickerType;
+import org.slf4j.Logger;
 
-import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 public class RocksDBMetricsRecorder {
+    private final Logger logger;
 
     private Sensor bytesWrittenToDatabaseSensor;
-    private Sensor bytesReadToDatabaseSensor;
+    private Sensor bytesReadFromDatabaseSensor;
     private Sensor memtableBytesFlushedSensor;
     private Sensor memtableHitRatioSensor;
-    private Sensor memtableAvgFlushTimeSensor;
-    private Sensor memtableMinFlushTimeSensor;
-    private Sensor memtableMaxFlushTimeSensor;
     private Sensor writeStallDurationSensor;
     private Sensor blockCacheDataHitRatioSensor;
     private Sensor blockCacheIndexHitRatioSensor;
     private Sensor blockCacheFilterHitRatioSensor;
     private Sensor bytesReadDuringCompactionSensor;
     private Sensor bytesWrittenDuringCompactionSensor;
-    private Sensor compactionTimeAvgSensor;
-    private Sensor compactionTimeMinSensor;
-    private Sensor compactionTimeMaxSensor;
     private Sensor numberOfOpenFilesSensor;
     private Sensor numberOfFileErrorsSensor;
 
-    final private Map statisticsToRecord = new HashMap<>();
-    final private String metricsScope;
-    final private String storeName;
+    private final Map statisticsToRecord = new ConcurrentHashMap<>();
+    private final String metricsScope;
+    private final String storeName;
+    private TaskId taskId;
+    private StreamsMetricsImpl streamsMetrics;
+    private boolean isInitialized = false;
 
     public RocksDBMetricsRecorder(final String metricsScope, final String storeName) {
         this.metricsScope = metricsScope;
         this.storeName = storeName;
+        final LogContext logContext = new LogContext(String.format("[RocksDB Metrics Recorder for %s] ", storeName));
+        logger = logContext.logger(RocksDBMetricsRecorder.class);
     }
 
-    private void init(final StreamsMetricsImpl streamsMetrics, final TaskId taskId) {
+    public String storeName() {
+        return storeName;
+    }
+
+    public TaskId taskId() {
+        return taskId;
+    }
+
+    public void addStatistics(final String segmentName,
+                              final Statistics statistics,
+                              final StreamsMetricsImpl streamsMetrics,
+                              final TaskId taskId) {
+        if (!isInitialized) {
+            initSensors(streamsMetrics, taskId);
+            this.taskId = taskId;
+            this.streamsMetrics = streamsMetrics;
+            isInitialized = true;
+        }
+        if (this.taskId != taskId) {
+            throw new IllegalStateException("Statistics of store \"" + segmentName + "\" for task " + taskId
+                + " cannot be added to metrics recorder for task " + this.taskId + ". This is a bug in Kafka Streams.");
+        }
+        if (statisticsToRecord.isEmpty()) {
+            logger.debug(
+                "Adding metrics recorder of task {} to metrics recording trigger",
+                taskId
+            );
+            streamsMetrics.rocksDBMetricsRecordingTrigger().addMetricsRecorder(this);
+        } else if (statisticsToRecord.containsKey(segmentName)) {
+            throw new IllegalStateException("Statistics for store \"" + segmentName + "\" of task " + taskId +
+                " has been already added. This is a bug in Kafka Streams.");
+        }
+        statistics.setStatsLevel(StatsLevel.EXCEPT_DETAILED_TIMERS);
+        logger.debug("Adding statistics for store {} of task {}", segmentName, taskId);
+        statisticsToRecord.put(segmentName, statistics);
+    }
+
+    private void initSensors(final StreamsMetricsImpl streamsMetrics, final TaskId taskId) {
         final RocksDBMetricContext metricContext = new RocksDBMetricContext(taskId.toString(), metricsScope, storeName);
         bytesWrittenToDatabaseSensor = RocksDBMetrics.bytesWrittenToDatabaseSensor(streamsMetrics, metricContext);
-        bytesReadToDatabaseSensor = RocksDBMetrics.bytesReadFromDatabaseSensor(streamsMetrics, metricContext);
+        bytesReadFromDatabaseSensor = RocksDBMetrics.bytesReadFromDatabaseSensor(streamsMetrics, metricContext);
         memtableBytesFlushedSensor = RocksDBMetrics.memtableBytesFlushedSensor(streamsMetrics, metricContext);
         memtableHitRatioSensor = RocksDBMetrics.memtableHitRatioSensor(streamsMetrics, metricContext);
-        memtableAvgFlushTimeSensor = RocksDBMetrics.memtableAvgFlushTimeSensor(streamsMetrics, metricContext);
-        memtableMinFlushTimeSensor = RocksDBMetrics.memtableMinFlushTimeSensor(streamsMetrics, metricContext);
-        memtableMaxFlushTimeSensor = RocksDBMetrics.memtableMaxFlushTimeSensor(streamsMetrics, metricContext);
         writeStallDurationSensor = RocksDBMetrics.writeStallDurationSensor(streamsMetrics, metricContext);
         blockCacheDataHitRatioSensor = RocksDBMetrics.blockCacheDataHitRatioSensor(streamsMetrics, metricContext);
         blockCacheIndexHitRatioSensor = RocksDBMetrics.blockCacheIndexHitRatioSensor(streamsMetrics, metricContext);
         blockCacheFilterHitRatioSensor = RocksDBMetrics.blockCacheFilterHitRatioSensor(streamsMetrics, metricContext);
-        bytesReadDuringCompactionSensor = RocksDBMetrics.bytesReadDuringCompactionSensor(streamsMetrics, metricContext);
         bytesWrittenDuringCompactionSensor =
             RocksDBMetrics.bytesWrittenDuringCompactionSensor(streamsMetrics, metricContext);
-        compactionTimeAvgSensor = RocksDBMetrics.compactionTimeAvgSensor(streamsMetrics, metricContext);
-        compactionTimeMinSensor = RocksDBMetrics.compactionTimeMinSensor(streamsMetrics, metricContext);
-        compactionTimeMaxSensor = RocksDBMetrics.compactionTimeMaxSensor(streamsMetrics, metricContext);
+        bytesReadDuringCompactionSensor = RocksDBMetrics.bytesReadDuringCompactionSensor(streamsMetrics, metricContext);
         numberOfOpenFilesSensor = RocksDBMetrics.numberOfOpenFilesSensor(streamsMetrics, metricContext);
         numberOfFileErrorsSensor = RocksDBMetrics.numberOfFileErrorsSensor(streamsMetrics, metricContext);
     }
 
-    public void addStatistics(final String storeName,
-                              final Statistics statistics,
-                              final StreamsMetricsImpl streamsMetrics,
-                              final TaskId taskId) {
-        if (statisticsToRecord.isEmpty()) {
-            init(streamsMetrics, taskId);
-        }
-        if (statisticsToRecord.containsKey(storeName)) {
-            throw new IllegalStateException("A statistics for store " + storeName + "has been already registered. "
-                + "This is a bug in Kafka Streams.");
+    public void removeStatistics(final String segmentName) {
+        logger.debug("Removing statistics for store {} of task {}", segmentName, taskId);
+        final Statistics removedStatistics = statisticsToRecord.remove(segmentName);
+        if (removedStatistics == null) {
+            throw new IllegalStateException("No statistics for store \"" + segmentName + "\" of task " + taskId
+                + " could be found. This is a bug in Kafka Streams.");
         }
-        statistics.setStatsLevel(StatsLevel.EXCEPT_DETAILED_TIMERS);
-        statisticsToRecord.put(storeName, statistics);
-    }
-
-    public void removeStatistics(final String storeName) {
-        final Statistics removedStatistics = statisticsToRecord.remove(storeName);
-        if (removedStatistics != null) {
-            removedStatistics.close();
+        removedStatistics.close();
+        if (statisticsToRecord.isEmpty()) {
+            logger.debug(
+                "Removing metrics recorder for store {} of task {} from metrics recording trigger",
+                storeName,
+                taskId
+            );
+            streamsMetrics.rocksDBMetricsRecordingTrigger().removeMetricsRecorder(this);
         }
     }
 
     public void record() {
-        // TODO: this block of record calls merely avoids compiler warnings.
-        //       The actual computations of the metrics will be implemented in the next PR
-        bytesWrittenToDatabaseSensor.record(0);
-        bytesReadToDatabaseSensor.record(0);
-        memtableBytesFlushedSensor.record(0);
-        memtableHitRatioSensor.record(0);
-        memtableAvgFlushTimeSensor.record(0);
-        memtableMinFlushTimeSensor.record(0);
-        memtableMaxFlushTimeSensor.record(0);
-        writeStallDurationSensor.record(0);
-        blockCacheDataHitRatioSensor.record(0);
-        blockCacheIndexHitRatioSensor.record(0);
-        blockCacheFilterHitRatioSensor.record(0);
-        bytesReadDuringCompactionSensor.record(0);
-        bytesWrittenDuringCompactionSensor.record(0);
-        compactionTimeAvgSensor.record(0);
-        compactionTimeMinSensor.record(0);
-        compactionTimeMaxSensor.record(0);
-        numberOfOpenFilesSensor.record(0);
-        numberOfFileErrorsSensor.record(0);
+        logger.debug("Recording metrics for store {}", storeName);
+        long bytesWrittenToDatabase = 0;
+        long bytesReadFromDatabase = 0;
+        long memtableBytesFlushed = 0;
+        long memtableHits = 0;
+        long memtableMisses = 0;
+        long blockCacheDataHits = 0;
+        long blockCacheDataMisses = 0;
+        long blockCacheIndexHits = 0;
+        long blockCacheIndexMisses = 0;
+        long blockCacheFilterHits = 0;
+        long blockCacheFilterMisses = 0;
+        long writeStallDuration = 0;
+        long bytesWrittenDuringCompaction = 0;
+        long bytesReadDuringCompaction = 0;
+        long numberOfOpenFiles = 0;
+        long numberOfFileErrors = 0;
+        for (final Statistics statistics : statisticsToRecord.values()) {
+            bytesWrittenToDatabase += statistics.getAndResetTickerCount(TickerType.BYTES_WRITTEN);
+            bytesReadFromDatabase += statistics.getAndResetTickerCount(TickerType.BYTES_READ);
+            memtableBytesFlushed += statistics.getAndResetTickerCount(TickerType.FLUSH_WRITE_BYTES);
+            memtableHits += statistics.getAndResetTickerCount(TickerType.MEMTABLE_HIT);
+            memtableMisses += statistics.getAndResetTickerCount(TickerType.MEMTABLE_MISS);
+            blockCacheDataHits += statistics.getAndResetTickerCount(TickerType.BLOCK_CACHE_DATA_HIT);
+            blockCacheDataMisses += statistics.getAndResetTickerCount(TickerType.BLOCK_CACHE_DATA_MISS);
+            blockCacheIndexHits += statistics.getAndResetTickerCount(TickerType.BLOCK_CACHE_INDEX_HIT);
+            blockCacheIndexMisses += statistics.getAndResetTickerCount(TickerType.BLOCK_CACHE_INDEX_MISS);
+            blockCacheFilterHits += statistics.getAndResetTickerCount(TickerType.BLOCK_CACHE_FILTER_HIT);
+            blockCacheFilterMisses += statistics.getAndResetTickerCount(TickerType.BLOCK_CACHE_FILTER_MISS);
+            writeStallDuration += statistics.getAndResetTickerCount(TickerType.STALL_MICROS);
+            bytesWrittenDuringCompaction += statistics.getAndResetTickerCount(TickerType.COMPACT_WRITE_BYTES);
+            bytesReadDuringCompaction += statistics.getAndResetTickerCount(TickerType.COMPACT_READ_BYTES);
+            numberOfOpenFiles += statistics.getAndResetTickerCount(TickerType.NO_FILE_OPENS)
+                - statistics.getAndResetTickerCount(TickerType.NO_FILE_CLOSES);
+            numberOfFileErrors += statistics.getAndResetTickerCount(TickerType.NO_FILE_ERRORS);
+        }
+        bytesWrittenToDatabaseSensor.record(bytesWrittenToDatabase);
+        bytesReadFromDatabaseSensor.record(bytesReadFromDatabase);
+        memtableBytesFlushedSensor.record(memtableBytesFlushed);
+        memtableHitRatioSensor.record(computeHitRatio(memtableHits, memtableMisses));
+        blockCacheDataHitRatioSensor.record(computeHitRatio(blockCacheDataHits, blockCacheDataMisses));
+        blockCacheIndexHitRatioSensor.record(computeHitRatio(blockCacheIndexHits, blockCacheIndexMisses));
+        blockCacheFilterHitRatioSensor.record(computeHitRatio(blockCacheFilterHits, blockCacheFilterMisses));
+        writeStallDurationSensor.record(writeStallDuration);
+        bytesWrittenDuringCompactionSensor.record(bytesWrittenDuringCompaction);
+        bytesReadDuringCompactionSensor.record(bytesReadDuringCompaction);
+        numberOfOpenFilesSensor.record(numberOfOpenFiles);
+        numberOfFileErrorsSensor.record(numberOfFileErrors);
     }
 
-    public void close() {
-        for (final Statistics statistics : statisticsToRecord.values()) {
-            statistics.close();
+    private double computeHitRatio(final long hits, final long misses) {
+        if (hits == 0) {
+            return 0;
         }
-        statisticsToRecord.clear();
+        return (double) hits / (hits + misses);
     }
 }
\ No newline at end of file
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecordingTrigger.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecordingTrigger.java
new file mode 100644
index 0000000000000..ce4c01af6fcb6
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecordingTrigger.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals.metrics;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class RocksDBMetricsRecordingTrigger implements Runnable {
+
+    private final Map metricsRecordersToTrigger = new ConcurrentHashMap<>();
+
+    public void addMetricsRecorder(final RocksDBMetricsRecorder metricsRecorder) {
+        final String metricsRecorderName = metricsRecorderName(metricsRecorder);
+        if (metricsRecordersToTrigger.containsKey(metricsRecorderName)) {
+            throw new IllegalStateException("RocksDB metrics recorder for store \"" + metricsRecorder.storeName() +
+                "\" of task " + metricsRecorder.taskId().toString() + " has already been added. "
+                + "This is a bug in Kafka Streams.");
+        }
+        metricsRecordersToTrigger.put(metricsRecorderName, metricsRecorder);
+    }
+
+    public void removeMetricsRecorder(final RocksDBMetricsRecorder metricsRecorder) {
+        final RocksDBMetricsRecorder removedMetricsRecorder =
+            metricsRecordersToTrigger.remove(metricsRecorderName(metricsRecorder));
+        if (removedMetricsRecorder == null) {
+            throw new IllegalStateException("No RocksDB metrics recorder for store "
+                + "\"" + metricsRecorder.storeName() + "\" of task " + metricsRecorder.taskId() + " could be found. "
+                + "This is a bug in Kafka Streams.");
+        }
+    }
+
+    private String metricsRecorderName(final RocksDBMetricsRecorder metricsRecorder) {
+        return metricsRecorder.taskId().toString() + "-" + metricsRecorder.storeName();
+    }
+
+    @Override
+    public void run() {
+        for (final RocksDBMetricsRecorder metricsRecorder : metricsRecordersToTrigger.values()) {
+            metricsRecorder.record();
+        }
+    }
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
index cac33c980f369..88659c3559886 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
@@ -475,9 +475,6 @@ private void checkRocksDBMetricsByTag(final String tag) {
         checkMetricByName(listMetricStore, MEMTABLE_BYTES_FLUSHED_RATE, 1);
         checkMetricByName(listMetricStore, MEMTABLE_BYTES_FLUSHED_TOTAL, 1);
         checkMetricByName(listMetricStore, MEMTABLE_HIT_RATIO, 1);
-        checkMetricByName(listMetricStore, MEMTABLE_FLUSH_TIME_AVG, 1);
-        checkMetricByName(listMetricStore, MEMTABLE_FLUSH_TIME_MIN, 1);
-        checkMetricByName(listMetricStore, MEMTABLE_FLUSH_TIME_MAX, 1);
         checkMetricByName(listMetricStore, WRITE_STALL_DURATION_AVG, 1);
         checkMetricByName(listMetricStore, WRITE_STALL_DURATION_TOTAL, 1);
         checkMetricByName(listMetricStore, BLOCK_CACHE_DATA_HIT_RATIO, 1);
@@ -485,9 +482,6 @@ private void checkRocksDBMetricsByTag(final String tag) {
         checkMetricByName(listMetricStore, BLOCK_CACHE_FILTER_HIT_RATIO, 1);
         checkMetricByName(listMetricStore, BYTES_READ_DURING_COMPACTION_RATE, 1);
         checkMetricByName(listMetricStore, BYTES_WRITTEN_DURING_COMPACTION_RATE, 1);
-        checkMetricByName(listMetricStore, COMPACTION_TIME_AVG, 1);
-        checkMetricByName(listMetricStore, COMPACTION_TIME_MIN, 1);
-        checkMetricByName(listMetricStore, COMPACTION_TIME_MAX, 1);
         checkMetricByName(listMetricStore, NUMBER_OF_OPEN_FILES, 1);
         checkMetricByName(listMetricStore, NUMBER_OF_FILE_ERRORS, 1);
     }
@@ -647,7 +641,7 @@ private void checkMetricByName(final List listMetric, final String metri
         final List metrics = listMetric.stream()
             .filter(m -> m.metricName().name().equals(metricName))
             .collect(Collectors.toList());
-        Assert.assertEquals("Size of metrics of type:'" + metricName + "' must be equal to:" + numMetric + " but it's equal to " + metrics.size(), numMetric, metrics.size());
+        Assert.assertEquals("Size of metrics of type:'" + metricName + "' must be equal to " + numMetric + " but it's equal to " + metrics.size(), numMetric, metrics.size());
         for (final Metric m : metrics) {
             Assert.assertNotNull("Metric:'" + m.metricName() + "' must be not null", m.metricValue());
         }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStreamThreadTest.java
index 508b000789ee6..ce08a6ac83e21 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStreamThreadTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStreamThreadTest.java
@@ -101,15 +101,18 @@ public String newStoreName(final String prefix) {
         properties.put(StreamsConfig.APPLICATION_ID_CONFIG, "blah");
         properties.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
         config = new StreamsConfig(properties);
-        globalStreamThread = new GlobalStreamThread(builder.rewriteTopology(config).buildGlobalStateTopology(),
-                                                    config,
-                                                    mockConsumer,
-                                                    new StateDirectory(config, time, true),
-                                                    0,
-                                                    new Metrics(),
-                                                    new MockTime(),
-                                                    "clientId",
-                                                     stateRestoreListener);
+        globalStreamThread = new GlobalStreamThread(
+            builder.rewriteTopology(config).buildGlobalStateTopology(),
+            config,
+            mockConsumer,
+            new StateDirectory(config, time, true),
+            0,
+            new Metrics(),
+            new MockTime(),
+            "clientId",
+            stateRestoreListener,
+            null
+        );
     }
 
     @Test
@@ -134,15 +137,18 @@ public List partitionsFor(final String topic) {
                 throw new RuntimeException("KABOOM!");
             }
         };
-        globalStreamThread = new GlobalStreamThread(builder.buildGlobalStateTopology(),
-                                                    config,
-                                                    mockConsumer,
-                                                    new StateDirectory(config, time, true),
-                                                    0,
-                                                    new Metrics(),
-                                                    new MockTime(),
-                                                    "clientId",
-                                                    stateRestoreListener);
+        globalStreamThread = new GlobalStreamThread(
+            builder.buildGlobalStateTopology(),
+            config,
+            mockConsumer,
+            new StateDirectory(config, time, true),
+            0,
+            new Metrics(),
+            new MockTime(),
+            "clientId",
+            stateRestoreListener,
+            null
+        );
 
         try {
             globalStreamThread.start();
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
index d60c0069039c4..2f293819bc533 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
@@ -255,7 +255,8 @@ private StreamThread createStreamThread(@SuppressWarnings("SameParameterValue")
             0,
             stateDirectory,
             new MockStateRestoreListener(),
-            threadIdx);
+            threadIdx
+        );
     }
 
     @Test
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java
index 70c44a5d5588b..e3b7274df1d4b 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java
@@ -16,6 +16,8 @@
  */
 package org.apache.kafka.streams.state.internals;
 
+import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.metrics.Sensor.RecordingLevel;
 import org.apache.kafka.common.serialization.Deserializer;
 import org.apache.kafka.common.serialization.Serdes;
 import org.apache.kafka.common.serialization.Serializer;
@@ -26,19 +28,21 @@
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.errors.ProcessorStateException;
-import org.apache.kafka.streams.processor.ProcessorContext;
 import org.apache.kafka.streams.processor.StateRestoreListener;
-import org.apache.kafka.streams.processor.TaskId;
-import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.apache.kafka.streams.state.KeyValueIterator;
 import org.apache.kafka.streams.state.RocksDBConfigSetter;
+import org.apache.kafka.streams.state.internals.metrics.RocksDBMetrics;
 import org.apache.kafka.streams.state.internals.metrics.RocksDBMetricsRecorder;
+import org.apache.kafka.streams.state.internals.metrics.RocksDBMetricsRecordingTrigger;
 import org.apache.kafka.test.InternalMockProcessorContext;
 import org.apache.kafka.test.StreamsTestUtils;
 import org.apache.kafka.test.TestUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
 import org.rocksdb.BlockBasedTableConfig;
 import org.rocksdb.BloomFilter;
 import org.rocksdb.Filter;
@@ -57,15 +61,10 @@
 import java.util.Set;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.kafka.common.utils.Utils.mkEntry;
-import static org.apache.kafka.common.utils.Utils.mkMap;
-import static org.apache.kafka.streams.StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG;
-import static org.apache.kafka.streams.StreamsConfig.ROCKSDB_CONFIG_SETTER_CLASS_CONFIG;
-import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.eq;
 import static org.easymock.EasyMock.mock;
-import static org.easymock.EasyMock.replay;
 import static org.easymock.EasyMock.reset;
-import static org.easymock.EasyMock.verify;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -73,7 +72,11 @@
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.powermock.api.easymock.PowerMock.replay;
+import static org.powermock.api.easymock.PowerMock.verify;
 
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({RocksDBMetrics.class, Sensor.class})
 public class RocksDBStoreTest {
     private static boolean enableBloomFilters = false;
     final static String DB_NAME = "db-name";
@@ -83,6 +86,8 @@ public class RocksDBStoreTest {
     private final Serializer stringSerializer = new StringSerializer();
     private final Deserializer stringDeserializer = new StringDeserializer();
 
+    private final RocksDBMetricsRecorder metricsRecorder = mock(RocksDBMetricsRecorder.class);
+
     InternalMockProcessorContext context;
     RocksDBStore rocksDBStore;
 
@@ -96,29 +101,79 @@ public void setUp() {
             Serdes.String(),
             Serdes.String(),
             new StreamsConfig(props));
+        context.metrics().setRocksDBMetricsRecordingTrigger(new RocksDBMetricsRecordingTrigger());
     }
 
     RocksDBStore getRocksDBStore() {
         return new RocksDBStore(DB_NAME, METRICS_SCOPE);
     }
 
+    private RocksDBStore getRocksDBStoreWithRocksDBMetricsRecorder() {
+        return new RocksDBStore(DB_NAME, METRICS_SCOPE, metricsRecorder);
+    }
+
+    private InternalMockProcessorContext getProcessorContext(final Properties streamsProps) {
+        return new InternalMockProcessorContext(
+            TestUtils.tempDirectory(),
+            new StreamsConfig(streamsProps)
+        );
+    }
+
+    private InternalMockProcessorContext getProcessorContext(
+        final RecordingLevel recordingLevel,
+        final Class rocksDBConfigSetterClass) {
+
+        final Properties streamsProps = StreamsTestUtils.getStreamsConfig();
+        streamsProps.setProperty(StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG, recordingLevel.name());
+        streamsProps.put(StreamsConfig.ROCKSDB_CONFIG_SETTER_CLASS_CONFIG, rocksDBConfigSetterClass);
+        return getProcessorContext(streamsProps);
+    }
+
+    private InternalMockProcessorContext getProcessorContext(final RecordingLevel recordingLevel) {
+        final Properties streamsProps = StreamsTestUtils.getStreamsConfig();
+        streamsProps.setProperty(StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG, recordingLevel.name());
+        return getProcessorContext(streamsProps);
+    }
+
     @After
     public void tearDown() {
         rocksDBStore.close();
     }
 
     @Test
-    public void shouldRemoveStatisticsFromInjectedMetricsRecorderOnClose() {
-        final RocksDBMetricsRecorder metricsRecorder = mock(RocksDBMetricsRecorder.class);
-        final RocksDBStore store = new RocksDBStore(DB_NAME, METRICS_SCOPE, metricsRecorder);
-        final ProcessorContext mockContext = mock(ProcessorContext.class);
-        expect(mockContext.appConfigs()).andReturn(mkMap(mkEntry(METRICS_RECORDING_LEVEL_CONFIG, "DEBUG")));
-        final String directoryPath = TestUtils.tempDirectory().getAbsolutePath();
-        final File directory = new File(directoryPath);
-        expect(mockContext.stateDir()).andReturn(directory);
-        expect(mockContext.metrics()).andReturn(mock(StreamsMetricsImpl.class));
-        expect(mockContext.taskId()).andReturn(new TaskId(0, 0));
-        replay(mockContext);
+    public void shouldAddStatisticsToInjectedMetricsRecorderWhenRecordingLevelIsDebug() {
+        final RocksDBStore store = getRocksDBStoreWithRocksDBMetricsRecorder();
+        final InternalMockProcessorContext mockContext = getProcessorContext(RecordingLevel.DEBUG);
+        reset(metricsRecorder);
+        metricsRecorder.addStatistics(
+            eq(DB_NAME),
+            anyObject(Statistics.class),
+            eq(mockContext.metrics()),
+            eq(mockContext.taskId())
+        );
+        replay(metricsRecorder);
+
+        store.openDB(mockContext);
+
+        verify(metricsRecorder);
+    }
+
+    @Test
+    public void shouldNotAddStatisticsToInjectedMetricsRecorderWhenRecordingLevelIsInfo() {
+        final RocksDBStore store = getRocksDBStoreWithRocksDBMetricsRecorder();
+        final InternalMockProcessorContext mockContext = getProcessorContext(RecordingLevel.INFO);
+        reset(metricsRecorder);
+        replay(metricsRecorder);
+
+        store.openDB(mockContext);
+
+        verify(metricsRecorder);
+    }
+
+    @Test
+    public void shouldRemoveStatisticsFromInjectedMetricsRecorderOnCloseWhenRecordingLevelIsDebug() {
+        final RocksDBStore store = getRocksDBStoreWithRocksDBMetricsRecorder();
+        final InternalMockProcessorContext mockContext = getProcessorContext(RecordingLevel.DEBUG);
         store.openDB(mockContext);
         reset(metricsRecorder);
         metricsRecorder.removeStatistics(DB_NAME);
@@ -131,24 +186,19 @@ public void shouldRemoveStatisticsFromInjectedMetricsRecorderOnClose() {
 
     @Test
     public void shouldNotRemoveStatisticsFromInjectedMetricsRecorderOnCloseWhenRecordingLevelIsInfo() {
-        final RocksDBMetricsRecorder metricsRecorder = mock(RocksDBMetricsRecorder.class);
-        replay(metricsRecorder);
-        final RocksDBStore store = new RocksDBStore(DB_NAME, METRICS_SCOPE, metricsRecorder);
-        final ProcessorContext mockContext = mock(ProcessorContext.class);
-        expect(mockContext.appConfigs()).andReturn(mkMap(mkEntry(METRICS_RECORDING_LEVEL_CONFIG, "INFO")));
-        final String directoryPath = TestUtils.tempDirectory().getAbsolutePath();
-        final File directory = new File(directoryPath);
-        expect(mockContext.stateDir()).andReturn(directory);
-        replay(mockContext);
+        final RocksDBStore store = getRocksDBStoreWithRocksDBMetricsRecorder();
+        final InternalMockProcessorContext mockContext = getProcessorContext(RecordingLevel.INFO);
         store.openDB(mockContext);
+        reset(metricsRecorder);
+        replay(metricsRecorder);
 
         store.close();
 
         verify(metricsRecorder);
     }
 
-    public static class TestRocksDBConfigSetter implements RocksDBConfigSetter {
-        public TestRocksDBConfigSetter(){}
+    public static class RocksDBConfigSetterWithUserProvidedStatistics implements RocksDBConfigSetter {
+        public RocksDBConfigSetterWithUserProvidedStatistics(){}
 
         public void setConfig(final String storeName, final Options options, final Map configs) {
             options.setStatistics(new Statistics());
@@ -160,19 +210,24 @@ public void close(final String storeName, final Options options) {
     }
 
     @Test
-    public void shouldNotRemoveStatisticsFromInjectedMetricsRecorderOnCloseWhenUserProvidsStatistics() {
-        final RocksDBMetricsRecorder metricsRecorder = mock(RocksDBMetricsRecorder.class);
+    public void shouldNotAddStatisticsToInjectedMetricsRecorderWhenUserProvidesStatistics() {
+        final RocksDBStore store = getRocksDBStoreWithRocksDBMetricsRecorder();
+        final InternalMockProcessorContext mockContext =
+            getProcessorContext(RecordingLevel.DEBUG, RocksDBConfigSetterWithUserProvidedStatistics.class);
         replay(metricsRecorder);
-        final RocksDBStore store = new RocksDBStore(DB_NAME, METRICS_SCOPE, metricsRecorder);
-        final ProcessorContext mockContext = mock(ProcessorContext.class);
-        expect(mockContext.appConfigs()).andReturn(mkMap(
-            mkEntry(METRICS_RECORDING_LEVEL_CONFIG, "DEBUG"),
-            mkEntry(ROCKSDB_CONFIG_SETTER_CLASS_CONFIG, TestRocksDBConfigSetter.class)));
-        final String directoryPath = TestUtils.tempDirectory().getAbsolutePath();
-        final File directory = new File(directoryPath);
-        expect(mockContext.stateDir()).andReturn(directory);
-        replay(mockContext);
+
         store.openDB(mockContext);
+        verify(metricsRecorder);
+    }
+
+    @Test
+    public void shouldNotRemoveStatisticsFromInjectedMetricsRecorderOnCloseWhenUserProvidesStatistics() {
+        final RocksDBStore store = getRocksDBStoreWithRocksDBMetricsRecorder();
+        final InternalMockProcessorContext mockContext =
+            getProcessorContext(RecordingLevel.DEBUG, RocksDBConfigSetterWithUserProvidedStatistics.class);
+        store.openDB(mockContext);
+        reset(metricsRecorder);
+        replay(metricsRecorder);
 
         store.close();
 
@@ -526,6 +581,7 @@ public void shouldHandleToggleOfEnablingBloomFilters() {
             Serdes.String(),
             Serdes.String(),
             new StreamsConfig(props));
+        context.metrics().setRocksDBMetricsRecordingTrigger(new RocksDBMetricsRecordingTrigger());
 
         enableBloomFilters = false;
         rocksDBStore.init(context, rocksDBStore);
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentIteratorTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentIteratorTest.java
index 85ec3379b2a92..2e305dd406b25 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentIteratorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentIteratorTest.java
@@ -41,10 +41,11 @@
 
 public class SegmentIteratorTest {
 
+    private final RocksDBMetricsRecorder rocksDBMetricsRecorder = new RocksDBMetricsRecorder("metrics-scope", "store-name");
     private final KeyValueSegment segmentOne =
-        new KeyValueSegment("one", "one", 0, new RocksDBMetricsRecorder("metrics-scope", "store-name"));
+        new KeyValueSegment("one", "one", 0, rocksDBMetricsRecorder);
     private final KeyValueSegment segmentTwo =
-        new KeyValueSegment("two", "window", 1, new RocksDBMetricsRecorder("metrics-scope", "store-name"));
+        new KeyValueSegment("two", "window", 1, rocksDBMetricsRecorder);
     private final HasNextCondition hasNextCondition = Iterator::hasNext;
 
     private SegmentIterator iterator = null;
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorderTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorderTest.java
index a5bd1da8db293..9bef2cc3b6f6b 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorderTest.java
@@ -20,16 +20,22 @@
 import org.apache.kafka.streams.processor.TaskId;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.apache.kafka.streams.state.internals.metrics.RocksDBMetrics.RocksDBMetricContext;
+import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.powermock.modules.junit4.PowerMockRunner;
 import org.rocksdb.Statistics;
 import org.rocksdb.StatsLevel;
+import org.rocksdb.TickerType;
 
+import static org.easymock.EasyMock.anyObject;
 import static org.easymock.EasyMock.eq;
 import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.mock;
+import static org.easymock.EasyMock.niceMock;
+import static org.easymock.EasyMock.resetToNice;
+import static org.junit.Assert.assertThrows;
 import static org.powermock.api.easymock.PowerMock.reset;
 import static org.powermock.api.easymock.PowerMock.createMock;
 import static org.powermock.api.easymock.PowerMock.mockStatic;
@@ -40,105 +46,312 @@
 @RunWith(PowerMockRunner.class)
 @PrepareForTest({RocksDBMetrics.class, Sensor.class})
 public class RocksDBMetricsRecorderTest {
-
     private final static String METRICS_SCOPE = "metrics-scope";
-    private final static String STORE_NAME = "store name";
-    private final static String SEGMENT_STORE_NAME_1 = "segment store name 1";
-    private final static String SEGMENT_STORE_NAME_2 = "segment  name 2";
+    private final static String STORE_NAME = "store-name";
+    private final static String SEGMENT_STORE_NAME_1 = "segment-store-name-1";
+    private final static String SEGMENT_STORE_NAME_2 = "segment-store-name-2";
 
     private final Statistics statisticsToAdd1 = mock(Statistics.class);
     private final Statistics statisticsToAdd2 = mock(Statistics.class);
-    private final Sensor sensor = createMock(Sensor.class);
-    private final StreamsMetricsImpl streamsMetrics = mock(StreamsMetricsImpl.class);
-    private final TaskId taskId = new TaskId(0, 0);
+
+    private final Sensor bytesWrittenToDatabaseSensor = createMock(Sensor.class);
+    private final Sensor bytesReadFromDatabaseSensor = createMock(Sensor.class);
+    private final Sensor memtableBytesFlushedSensor = createMock(Sensor.class);
+    private final Sensor memtableHitRatioSensor = createMock(Sensor.class);
+    private final Sensor writeStallDurationSensor = createMock(Sensor.class);
+    private final Sensor blockCacheDataHitRatioSensor = createMock(Sensor.class);
+    private final Sensor blockCacheIndexHitRatioSensor = createMock(Sensor.class);
+    private final Sensor blockCacheFilterHitRatioSensor = createMock(Sensor.class);
+    private final Sensor bytesReadDuringCompactionSensor = createMock(Sensor.class);
+    private final Sensor bytesWrittenDuringCompactionSensor = createMock(Sensor.class);
+    private final Sensor numberOfOpenFilesSensor = createMock(Sensor.class);
+    private final Sensor numberOfFileErrorsSensor = createMock(Sensor.class);
+
+    private final StreamsMetricsImpl streamsMetrics = niceMock(StreamsMetricsImpl.class);
+    private final RocksDBMetricsRecordingTrigger recordingTrigger = mock(RocksDBMetricsRecordingTrigger.class);
+    private final TaskId taskId1 = new TaskId(0, 0);
+    private final TaskId taskId2 = new TaskId(0, 2);
 
     private final RocksDBMetricsRecorder recorder = new RocksDBMetricsRecorder(METRICS_SCOPE, STORE_NAME);
 
+    @Before
+    public void setUp() {
+        expect(streamsMetrics.rocksDBMetricsRecordingTrigger()).andStubReturn(recordingTrigger);
+        replay(streamsMetrics);
+    }
+
     @Test
-    public void shouldSetStatsLevelToExceptDetailedTimers() {
+    public void shouldSetStatsLevelToExceptDetailedTimersWhenStatisticsIsAdded() {
         mockStaticNice(RocksDBMetrics.class);
         replay(RocksDBMetrics.class);
         statisticsToAdd1.setStatsLevel(StatsLevel.EXCEPT_DETAILED_TIMERS);
         replay(statisticsToAdd1);
 
-        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId);
+        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
 
         verify(statisticsToAdd1);
     }
 
     @Test
-    public void shouldInitMetricsOnlyWhenFirstStatisticsIsAdded() {
-        replayMetricsInitialization();
-        statisticsToAdd1.setStatsLevel(StatsLevel.EXCEPT_DETAILED_TIMERS);
-        replay(statisticsToAdd1);
-        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId);
+    public void shouldThrowIfTaskIdOfStatisticsToAddDiffersFromInitialisedOne() {
+        mockStaticNice(RocksDBMetrics.class);
+        replay(RocksDBMetrics.class);
+        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
+        assertThrows(
+            IllegalStateException.class,
+            () -> recorder.addStatistics(SEGMENT_STORE_NAME_2, statisticsToAdd2, streamsMetrics, taskId2)
+        );
+    }
+
+    @Test
+    public void shouldThrowIfStatisticsToAddHasBeenAlreadyAdded() {
+        mockStaticNice(RocksDBMetrics.class);
+        replay(RocksDBMetrics.class);
+        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
+
+        assertThrows(
+            IllegalStateException.class,
+            () -> recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1)
+        );
+    }
+
+    @Test
+    public void shouldInitMetricsAndAddItselfToRecordingTriggerOnlyWhenFirstStatisticsIsAdded() {
+        setUpMetricsMock();
+        recordingTrigger.addMetricsRecorder(recorder);
+        replay(recordingTrigger);
+
+        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
+
+        verify(recordingTrigger);
         verify(RocksDBMetrics.class);
 
         mockStatic(RocksDBMetrics.class);
         replay(RocksDBMetrics.class);
-        recorder.addStatistics(SEGMENT_STORE_NAME_2, statisticsToAdd2, streamsMetrics, taskId);
+        reset(recordingTrigger);
+        replay(recordingTrigger);
+
+        recorder.addStatistics(SEGMENT_STORE_NAME_2, statisticsToAdd2, streamsMetrics, taskId1);
+
+        verify(recordingTrigger);
         verify(RocksDBMetrics.class);
     }
 
     @Test
-    public void shouldCloseStatisticsWhenRecorderIsClosed() {
+    public void shouldAddItselfToRecordingTriggerWhenEmptyButInitialised() {
+        mockStaticNice(RocksDBMetrics.class);
+        replay(RocksDBMetrics.class);
+        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
+        recorder.removeStatistics(SEGMENT_STORE_NAME_1);
+        reset(recordingTrigger);
+        recordingTrigger.addMetricsRecorder(recorder);
+        replay(recordingTrigger);
+
+        recorder.addStatistics(SEGMENT_STORE_NAME_2, statisticsToAdd2, streamsMetrics, taskId1);
+
+        verify(recordingTrigger);
+    }
+
+    @Test
+    public void shouldNotAddItselfToRecordingTriggerWhenNotEmpty() {
+        mockStaticNice(RocksDBMetrics.class);
+        replay(RocksDBMetrics.class);
+        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
+        reset(recordingTrigger);
+        replay(recordingTrigger);
+
+        recorder.addStatistics(SEGMENT_STORE_NAME_2, statisticsToAdd2, streamsMetrics, taskId1);
+
+        verify(recordingTrigger);
+    }
+
+    @Test
+    public void shouldCloseStatisticsWhenStatisticsIsRemoved() {
         mockStaticNice(RocksDBMetrics.class);
         replay(RocksDBMetrics.class);
-        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId);
-        recorder.addStatistics(SEGMENT_STORE_NAME_2, statisticsToAdd2, streamsMetrics, taskId);
+        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
         reset(statisticsToAdd1);
-        reset(statisticsToAdd2);
         statisticsToAdd1.close();
-        statisticsToAdd2.close();
         replay(statisticsToAdd1);
-        replay(statisticsToAdd2);
 
-        recorder.close();
+        recorder.removeStatistics(SEGMENT_STORE_NAME_1);
 
         verify(statisticsToAdd1);
-        verify(statisticsToAdd2);
     }
 
     @Test
-    public void shouldCloseStatisticsWhenStatisticsIsRemoved() {
+    public void shouldRemoveItselfFromRecordingTriggerWhenLastStatisticsIsRemoved() {
+        mockStaticNice(RocksDBMetrics.class);
+        replay(RocksDBMetrics.class);
+        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
+        recorder.addStatistics(SEGMENT_STORE_NAME_2, statisticsToAdd2, streamsMetrics, taskId1);
+        reset(recordingTrigger);
+        replay(recordingTrigger);
+
+        recorder.removeStatistics(SEGMENT_STORE_NAME_1);
+
+        verify(recordingTrigger);
+
+        reset(recordingTrigger);
+        recordingTrigger.removeMetricsRecorder(recorder);
+        replay(recordingTrigger);
+
+        recorder.removeStatistics(SEGMENT_STORE_NAME_2);
+
+        verify(recordingTrigger);
+    }
+
+    @Test
+    public void shouldThrowIfStatisticsToRemoveNotFound() {
         mockStaticNice(RocksDBMetrics.class);
         replay(RocksDBMetrics.class);
-        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId);
+        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
+        assertThrows(
+            IllegalStateException.class,
+            () -> recorder.removeStatistics(SEGMENT_STORE_NAME_2)
+        );
+    }
+
+    @Test
+    public void shouldRecordMetrics() {
+        setUpMetricsMock();
+        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
+        recorder.addStatistics(SEGMENT_STORE_NAME_2, statisticsToAdd2, streamsMetrics, taskId1);
         reset(statisticsToAdd1);
-        statisticsToAdd1.close();
+        reset(statisticsToAdd2);
+
+        expect(statisticsToAdd1.getAndResetTickerCount(TickerType.BYTES_WRITTEN)).andReturn(1L);
+        expect(statisticsToAdd2.getAndResetTickerCount(TickerType.BYTES_WRITTEN)).andReturn(2L);
+        bytesWrittenToDatabaseSensor.record(1 + 2);
+        replay(bytesWrittenToDatabaseSensor);
+
+        expect(statisticsToAdd1.getAndResetTickerCount(TickerType.BYTES_READ)).andReturn(2L);
+        expect(statisticsToAdd2.getAndResetTickerCount(TickerType.BYTES_READ)).andReturn(3L);
+        bytesReadFromDatabaseSensor.record(2 + 3);
+        replay(bytesReadFromDatabaseSensor);
+
+        expect(statisticsToAdd1.getAndResetTickerCount(TickerType.FLUSH_WRITE_BYTES)).andReturn(3L);
+        expect(statisticsToAdd2.getAndResetTickerCount(TickerType.FLUSH_WRITE_BYTES)).andReturn(4L);
+        memtableBytesFlushedSensor.record(3 + 4);
+        replay(memtableBytesFlushedSensor);
+
+        expect(statisticsToAdd1.getAndResetTickerCount(TickerType.MEMTABLE_HIT)).andReturn(1L);
+        expect(statisticsToAdd1.getAndResetTickerCount(TickerType.MEMTABLE_MISS)).andReturn(2L);
+        expect(statisticsToAdd2.getAndResetTickerCount(TickerType.MEMTABLE_HIT)).andReturn(3L);
+        expect(statisticsToAdd2.getAndResetTickerCount(TickerType.MEMTABLE_MISS)).andReturn(4L);
+        memtableHitRatioSensor.record((double) 4 / (4 + 6));
+        replay(memtableHitRatioSensor);
+
+        expect(statisticsToAdd1.getAndResetTickerCount(TickerType.STALL_MICROS)).andReturn(4L);
+        expect(statisticsToAdd2.getAndResetTickerCount(TickerType.STALL_MICROS)).andReturn(5L);
+        writeStallDurationSensor.record(4 + 5);
+        replay(writeStallDurationSensor);
+
+        expect(statisticsToAdd1.getAndResetTickerCount(TickerType.BLOCK_CACHE_DATA_HIT)).andReturn(5L);
+        expect(statisticsToAdd1.getAndResetTickerCount(TickerType.BLOCK_CACHE_DATA_MISS)).andReturn(4L);
+        expect(statisticsToAdd2.getAndResetTickerCount(TickerType.BLOCK_CACHE_DATA_HIT)).andReturn(3L);
+        expect(statisticsToAdd2.getAndResetTickerCount(TickerType.BLOCK_CACHE_DATA_MISS)).andReturn(2L);
+        blockCacheDataHitRatioSensor.record((double) 8 / (8 + 6));
+        replay(blockCacheDataHitRatioSensor);
+
+        expect(statisticsToAdd1.getAndResetTickerCount(TickerType.BLOCK_CACHE_INDEX_HIT)).andReturn(4L);
+        expect(statisticsToAdd1.getAndResetTickerCount(TickerType.BLOCK_CACHE_INDEX_MISS)).andReturn(2L);
+        expect(statisticsToAdd2.getAndResetTickerCount(TickerType.BLOCK_CACHE_INDEX_HIT)).andReturn(2L);
+        expect(statisticsToAdd2.getAndResetTickerCount(TickerType.BLOCK_CACHE_INDEX_MISS)).andReturn(4L);
+        blockCacheIndexHitRatioSensor.record((double) 6 / (6 + 6));
+        replay(blockCacheIndexHitRatioSensor);
+
+        expect(statisticsToAdd1.getAndResetTickerCount(TickerType.BLOCK_CACHE_FILTER_HIT)).andReturn(2L);
+        expect(statisticsToAdd1.getAndResetTickerCount(TickerType.BLOCK_CACHE_FILTER_MISS)).andReturn(4L);
+        expect(statisticsToAdd2.getAndResetTickerCount(TickerType.BLOCK_CACHE_FILTER_HIT)).andReturn(3L);
+        expect(statisticsToAdd2.getAndResetTickerCount(TickerType.BLOCK_CACHE_FILTER_MISS)).andReturn(5L);
+        blockCacheFilterHitRatioSensor.record((double) 5 / (5 + 9));
+        replay(blockCacheFilterHitRatioSensor);
+
+        expect(statisticsToAdd1.getAndResetTickerCount(TickerType.COMPACT_WRITE_BYTES)).andReturn(2L);
+        expect(statisticsToAdd2.getAndResetTickerCount(TickerType.COMPACT_WRITE_BYTES)).andReturn(4L);
+        bytesWrittenDuringCompactionSensor.record(2 + 4);
+        replay(bytesWrittenDuringCompactionSensor);
+
+        expect(statisticsToAdd1.getAndResetTickerCount(TickerType.COMPACT_READ_BYTES)).andReturn(5L);
+        expect(statisticsToAdd2.getAndResetTickerCount(TickerType.COMPACT_READ_BYTES)).andReturn(6L);
+        bytesReadDuringCompactionSensor.record(5 + 6);
+        replay(bytesReadDuringCompactionSensor);
+
+        expect(statisticsToAdd1.getAndResetTickerCount(TickerType.NO_FILE_OPENS)).andReturn(5L);
+        expect(statisticsToAdd1.getAndResetTickerCount(TickerType.NO_FILE_CLOSES)).andReturn(3L);
+        expect(statisticsToAdd2.getAndResetTickerCount(TickerType.NO_FILE_OPENS)).andReturn(7L);
+        expect(statisticsToAdd2.getAndResetTickerCount(TickerType.NO_FILE_CLOSES)).andReturn(4L);
+        numberOfOpenFilesSensor.record((5 + 7) - (3 + 4));
+        replay(numberOfOpenFilesSensor);
+
+        expect(statisticsToAdd1.getAndResetTickerCount(TickerType.NO_FILE_ERRORS)).andReturn(34L);
+        expect(statisticsToAdd2.getAndResetTickerCount(TickerType.NO_FILE_ERRORS)).andReturn(11L);
+        numberOfFileErrorsSensor.record(11 + 34);
+        replay(numberOfFileErrorsSensor);
+
         replay(statisticsToAdd1);
+        replay(statisticsToAdd2);
 
-        recorder.removeStatistics(SEGMENT_STORE_NAME_1);
+        recorder.record();
 
         verify(statisticsToAdd1);
+        verify(statisticsToAdd2);
+        verify(bytesWrittenToDatabaseSensor);
+    }
+
+    @Test
+    public void shouldCorrectlyHandleHitRatioRecordingsWithZeroHitsAndMisses() {
+        setUpMetricsMock();
+        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
+        resetToNice(statisticsToAdd1);
+        expect(statisticsToAdd1.getTickerCount(anyObject())).andReturn(0L).anyTimes();
+        replay(statisticsToAdd1);
+        memtableHitRatioSensor.record(0);
+        blockCacheDataHitRatioSensor.record(0);
+        blockCacheIndexHitRatioSensor.record(0);
+        blockCacheFilterHitRatioSensor.record(0);
+        replay(memtableHitRatioSensor);
+        replay(blockCacheDataHitRatioSensor);
+        replay(blockCacheIndexHitRatioSensor);
+        replay(blockCacheFilterHitRatioSensor);
+
+        recorder.record();
+
+        verify(memtableHitRatioSensor);
+        verify(blockCacheDataHitRatioSensor);
+        verify(blockCacheIndexHitRatioSensor);
+        verify(blockCacheFilterHitRatioSensor);
     }
 
-    private void replayMetricsInitialization() {
+    private void setUpMetricsMock() {
         mockStatic(RocksDBMetrics.class);
         final RocksDBMetricContext metricsContext =
-            new RocksDBMetricContext(taskId.toString(), METRICS_SCOPE, STORE_NAME);
-        expect(RocksDBMetrics.bytesWrittenToDatabaseSensor(eq(streamsMetrics), eq(metricsContext))).andReturn(sensor);
-        expect(RocksDBMetrics.bytesReadFromDatabaseSensor(eq(streamsMetrics), eq(metricsContext))).andReturn(sensor);
-        expect(RocksDBMetrics.memtableBytesFlushedSensor(eq(streamsMetrics), eq(metricsContext))).andReturn(sensor);
-        expect(RocksDBMetrics.memtableHitRatioSensor(eq(streamsMetrics), eq(metricsContext))).andReturn(sensor);
-        expect(RocksDBMetrics.memtableAvgFlushTimeSensor(eq(streamsMetrics), eq(metricsContext))).andReturn(sensor);
-        expect(RocksDBMetrics.memtableMinFlushTimeSensor(eq(streamsMetrics), eq(metricsContext))).andReturn(sensor);
-        expect(RocksDBMetrics.memtableMaxFlushTimeSensor(eq(streamsMetrics), eq(metricsContext))).andReturn(sensor);
-        expect(RocksDBMetrics.writeStallDurationSensor(eq(streamsMetrics), eq(metricsContext))).andReturn(sensor);
-        expect(RocksDBMetrics.blockCacheDataHitRatioSensor(eq(streamsMetrics), eq(metricsContext))).andReturn(sensor);
-        expect(RocksDBMetrics.blockCacheIndexHitRatioSensor(eq(streamsMetrics), eq(metricsContext))).andReturn(sensor);
-        expect(RocksDBMetrics.blockCacheFilterHitRatioSensor(eq(streamsMetrics), eq(metricsContext))).andReturn(sensor);
-        expect(
-            RocksDBMetrics.bytesReadDuringCompactionSensor(eq(streamsMetrics), eq(metricsContext))
-        ).andReturn(sensor);
-        expect(
-            RocksDBMetrics.bytesWrittenDuringCompactionSensor(eq(streamsMetrics), eq(metricsContext))
-        ).andReturn(sensor);
-        expect(RocksDBMetrics.compactionTimeMinSensor(eq(streamsMetrics), eq(metricsContext))).andReturn(sensor);
-        expect(RocksDBMetrics.compactionTimeMaxSensor(eq(streamsMetrics), eq(metricsContext))).andReturn(sensor);
-        expect(RocksDBMetrics.compactionTimeAvgSensor(eq(streamsMetrics), eq(metricsContext))).andReturn(sensor);
-        expect(RocksDBMetrics.numberOfOpenFilesSensor(eq(streamsMetrics), eq(metricsContext))).andReturn(sensor);
-        expect(RocksDBMetrics.numberOfFileErrorsSensor(eq(streamsMetrics), eq(metricsContext))).andReturn(sensor);
+            new RocksDBMetricContext(taskId1.toString(), METRICS_SCOPE, STORE_NAME);
+        expect(RocksDBMetrics.bytesWrittenToDatabaseSensor(eq(streamsMetrics), eq(metricsContext)))
+            .andReturn(bytesWrittenToDatabaseSensor);
+        expect(RocksDBMetrics.bytesReadFromDatabaseSensor(eq(streamsMetrics), eq(metricsContext)))
+            .andReturn(bytesReadFromDatabaseSensor);
+        expect(RocksDBMetrics.memtableBytesFlushedSensor(eq(streamsMetrics), eq(metricsContext)))
+            .andReturn(memtableBytesFlushedSensor);
+        expect(RocksDBMetrics.memtableHitRatioSensor(eq(streamsMetrics), eq(metricsContext)))
+            .andReturn(memtableHitRatioSensor);
+        expect(RocksDBMetrics.writeStallDurationSensor(eq(streamsMetrics), eq(metricsContext)))
+            .andReturn(writeStallDurationSensor);
+        expect(RocksDBMetrics.blockCacheDataHitRatioSensor(eq(streamsMetrics), eq(metricsContext)))
+            .andReturn(blockCacheDataHitRatioSensor);
+        expect(RocksDBMetrics.blockCacheIndexHitRatioSensor(eq(streamsMetrics), eq(metricsContext)))
+            .andReturn(blockCacheIndexHitRatioSensor);
+        expect(RocksDBMetrics.blockCacheFilterHitRatioSensor(eq(streamsMetrics), eq(metricsContext)))
+            .andReturn(blockCacheFilterHitRatioSensor);
+        expect(RocksDBMetrics.bytesWrittenDuringCompactionSensor(eq(streamsMetrics), eq(metricsContext)))
+            .andReturn(bytesWrittenDuringCompactionSensor);
+        expect(RocksDBMetrics.bytesReadDuringCompactionSensor(eq(streamsMetrics), eq(metricsContext)))
+            .andReturn(bytesReadDuringCompactionSensor);
+        expect(RocksDBMetrics.numberOfOpenFilesSensor(eq(streamsMetrics), eq(metricsContext)))
+            .andReturn(numberOfOpenFilesSensor);
+        expect(RocksDBMetrics.numberOfFileErrorsSensor(eq(streamsMetrics), eq(metricsContext)))
+            .andReturn(numberOfFileErrorsSensor);
         replay(RocksDBMetrics.class);
     }
 }
\ No newline at end of file
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecordingTriggerTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecordingTriggerTest.java
new file mode 100644
index 0000000000000..c341055f3976f
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecordingTriggerTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals.metrics;
+
+import org.apache.kafka.streams.processor.TaskId;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.niceMock;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.resetToDefault;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertThrows;
+
+public class RocksDBMetricsRecordingTriggerTest {
+
+    private final static String STORE_NAME1 = "store-name1";
+    private final static String STORE_NAME2 = "store-name2";
+    private final static TaskId TASK_ID1 = new TaskId(1, 2);
+    private final static TaskId TASK_ID2 = new TaskId(2, 4);
+    private final RocksDBMetricsRecorder recorder1 = niceMock(RocksDBMetricsRecorder.class);
+    private final RocksDBMetricsRecorder recorder2 = niceMock(RocksDBMetricsRecorder.class);
+
+    private final RocksDBMetricsRecordingTrigger recordingTrigger = new RocksDBMetricsRecordingTrigger();
+
+    @Before
+    public void setUp() {
+        expect(recorder1.storeName()).andStubReturn(STORE_NAME1);
+        expect(recorder1.taskId()).andStubReturn(TASK_ID1);
+        replay(recorder1);
+        expect(recorder2.storeName()).andStubReturn(STORE_NAME2);
+        expect(recorder2.taskId()).andStubReturn(TASK_ID2);
+        replay(recorder2);
+    }
+
+    @Test
+    public void shouldTriggerAddedMetricsRecorders() {
+        recordingTrigger.addMetricsRecorder(recorder1);
+        recordingTrigger.addMetricsRecorder(recorder2);
+
+        resetToDefault(recorder1);
+        recorder1.record();
+        replay(recorder1);
+        resetToDefault(recorder2);
+        recorder2.record();
+        replay(recorder2);
+
+        recordingTrigger.run();
+
+        verify(recorder1);
+        verify(recorder2);
+    }
+
+    @Test
+    public void shouldThrowIfRecorderToAddHasBeenAlreadyAdded() {
+        recordingTrigger.addMetricsRecorder(recorder1);
+
+        assertThrows(
+            IllegalStateException.class,
+            () -> recordingTrigger.addMetricsRecorder(recorder1)
+        );
+    }
+
+    @Test
+    public void shouldThrowIfRecorderToRemoveCouldNotBeFound() {
+        recordingTrigger.addMetricsRecorder(recorder1);
+        assertThrows(
+            IllegalStateException.class,
+            () -> recordingTrigger.removeMetricsRecorder(recorder2)
+        );
+    }
+}
\ No newline at end of file
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsTest.java
index 52632d5360bdd..aae7afeeb7e21 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsTest.java
@@ -204,17 +204,14 @@ public void shouldGetCompactionTimeMaxSensor() {
     public void shouldGetNumberOfOpenFilesSensor() {
         final String metricNamePrefix = "number-open-files";
         final String description = "Number of currently open files";
-        verifyValueSensor(metricNamePrefix, description, RocksDBMetrics::numberOfOpenFilesSensor);
+        verifySumSensor(metricNamePrefix, false, description, RocksDBMetrics::numberOfOpenFilesSensor);
     }
 
     @Test
     public void shouldGetNumberOfFilesErrors() {
         final String metricNamePrefix = "number-file-errors";
         final String description = "Total number of file errors occurred";
-        setupStreamsMetricsMock(metricNamePrefix);
-        StreamsMetricsImpl.addSumMetricToSensor(sensor, STATE_LEVEL_GROUP, tags, metricNamePrefix, description);
-
-        replayCallAndVerify(RocksDBMetrics::numberOfFileErrorsSensor);
+        verifySumSensor(metricNamePrefix, true, description, RocksDBMetrics::numberOfFileErrorsSensor);
     }
 
     private void verifyRateAndTotalSensor(final String metricNamePrefix,
@@ -252,6 +249,21 @@ private void verifyValueSensor(final String metricNamePrefix,
         replayCallAndVerify(sensorCreator);
     }
 
+    private void verifySumSensor(final String metricNamePrefix,
+                                 final boolean withSuffix,
+                                 final String description,
+                                 final SensorCreator sensorCreator) {
+        setupStreamsMetricsMock(metricNamePrefix);
+        if (withSuffix) {
+            StreamsMetricsImpl.addSumMetricToSensor(sensor, STATE_LEVEL_GROUP, tags, metricNamePrefix, description);
+        } else {
+            StreamsMetricsImpl
+                .addSumMetricToSensor(sensor, STATE_LEVEL_GROUP, tags, metricNamePrefix, withSuffix, description);
+        }
+
+        replayCallAndVerify(sensorCreator);
+    }
+
     private void setupStreamsMetricsMock(final String metricNamePrefix) {
         mockStatic(StreamsMetricsImpl.class);
         expect(streamsMetrics.storeLevelSensor(
diff --git a/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java b/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java
index 0935ae300f7ec..f5cb014550a99 100644
--- a/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java
+++ b/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java
@@ -41,6 +41,7 @@
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.apache.kafka.streams.state.StateSerdes;
 import org.apache.kafka.streams.state.internals.ThreadCache;
+import org.apache.kafka.streams.state.internals.metrics.RocksDBMetricsRecordingTrigger;
 
 import java.io.File;
 import java.time.Duration;
@@ -155,6 +156,7 @@ public InternalMockProcessorContext(final File stateDir,
         this.keySerde = keySerde;
         this.valSerde = valSerde;
         this.recordCollectorSupplier = collectorSupplier;
+        this.metrics().setRocksDBMetricsRecordingTrigger(new RocksDBMetricsRecordingTrigger());
     }
 
     @Override
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
index 0a0351caededa..a0ca6b2cad2d4 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
@@ -74,6 +74,7 @@
 import org.apache.kafka.streams.state.ValueAndTimestamp;
 import org.apache.kafka.streams.state.WindowStore;
 import org.apache.kafka.streams.state.internals.ThreadCache;
+import org.apache.kafka.streams.state.internals.metrics.RocksDBMetricsRecordingTrigger;
 import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.streams.test.OutputVerifier;
 import org.slf4j.Logger;
@@ -285,6 +286,7 @@ public List partitionsFor(final String topic) {
             threadName,
             StreamsConfig.METRICS_LATEST
         );
+        streamsMetrics.setRocksDBMetricsRecordingTrigger(new RocksDBMetricsRecordingTrigger());
         final Sensor skippedRecordsSensor = streamsMetrics.threadLevelSensor("skipped-records", Sensor.RecordingLevel.INFO);
         final String threadLevelGroup = "stream-metrics";
         skippedRecordsSensor.add(new MetricName("skipped-records-rate",

From 74f8ae13034d90800da5573666dea3bea5d702ce Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Tue, 24 Sep 2019 15:33:03 -0700
Subject: [PATCH 0643/1071] KAFKA-8179: do not suspend standby tasks during
 rebalance (#7321)

Some work needs to be done in Streams before we can incorporate cooperative rebalancing.

This PR lays the groundwork for it by doing some refactoring, including a behavioral change that affects eager ("normal") rebalancing as well: will no longer suspend standbys in onPartitionsRevoked, instead we just close any that were reassigned in onPartitionsAssigned

Reviewers: Bruno Cadonna , Boyang Chen , John Roesler , Guozhang Wang 
---
 .../consumer/ConsumerRebalanceListener.java   |  16 +-
 .../internals/ConsumerCoordinator.java        |  74 ++---
 .../apache/kafka/streams/KafkaStreams.java    |  15 +-
 .../apache/kafka/streams/StreamsConfig.java   |  43 ++-
 .../internals/AssignedStandbyTasks.java       |  42 +++
 .../internals/AssignedStreamsTasks.java       | 293 +++++++++++++++--
 .../processor/internals/AssignedTasks.java    | 137 +-------
 .../processor/internals/ChangelogReader.java  |  15 +-
 .../internals/StoreChangelogReader.java       |  24 +-
 .../processor/internals/StreamThread.java     | 214 ++++---------
 .../internals/StreamsPartitionAssignor.java   |  39 ++-
 .../internals/StreamsRebalanceListener.java   | 169 ++++++++++
 .../processor/internals/TaskManager.java      | 297 ++++++++++++------
 .../assignment/AssignorConfiguration.java     |  24 ++
 .../RegexSourceIntegrationTest.java           |   4 +-
 .../internals/AssignedStreamsTasksTest.java   |  36 ++-
 .../internals/MockChangelogReader.java        |  10 +-
 .../processor/internals/StreamThreadTest.java |  66 +++-
 .../StreamsPartitionAssignorTest.java         |   2 +-
 .../processor/internals/TaskManagerTest.java  | 114 ++++---
 .../streams/tests/StreamsUpgradeTest.java     |   7 +-
 21 files changed, 1085 insertions(+), 556 deletions(-)
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsRebalanceListener.java

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
index 6046ef954682d..ec2ce39a2d9be 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
@@ -67,7 +67,9 @@
  * During a rebalance event, the {@link #onPartitionsAssigned(Collection) onPartitionsAssigned} function will always be triggered exactly once when
  * the rebalance completes. That is, even if there is no newly assigned partitions for a consumer member, its {@link #onPartitionsAssigned(Collection) onPartitionsAssigned}
  * will still be triggered with an empty collection of partitions. As a result this function can be used also to notify when a rebalance event has happened.
- * On the other hand, {@link #onPartitionsRevoked(Collection)} and {@link #onPartitionsLost(Collection)}
+ * With eager rebalancing, {@link #onPartitionsRevoked(Collection)} will always be called at the start of a rebalance. On the other hand, {@link #onPartitionsLost(Collection)}
+ * will only be called when there were non-empty partitions that were lost.
+ * With cooperative rebalancing, {@link #onPartitionsRevoked(Collection)} and {@link #onPartitionsLost(Collection)}
  * will only be triggered when there are non-empty partitions revoked or lost from this consumer member during a rebalance event.
  * 
                  
  * It is possible
@@ -117,16 +119,16 @@
 public interface ConsumerRebalanceListener {
 
     /**
-     * A callback method the user can implement to provide handling of offset commits to a customized store on the start
-     * of a rebalance operation. This method will be called before a rebalance operation starts and after the consumer
-     * stops fetching data. It can also be called when consumer is being closed ({@link KafkaConsumer#close(Duration)})
+     * A callback method the user can implement to provide handling of offset commits to a customized store.
+     * This method will be called during a rebalance operation when the consumer has to give up some partitions.
+     * It can also be called when consumer is being closed ({@link KafkaConsumer#close(Duration)})
      * or is unsubscribing ({@link KafkaConsumer#unsubscribe()}).
      * It is recommended that offsets should be committed in this callback to either Kafka or a
      * custom offset store to prevent duplicate data.
      * 
                  
-     * For examples on usage of this API, see Usage Examples section of {@link KafkaConsumer KafkaConsumer}
-     * 
                  
-     * NOTE: This method is only called before rebalances. It is not called prior to {@link KafkaConsumer#close()}.
+     * In eager rebalancing, it will always be called at the start of a rebalance and after the consumer stops fetching data.
+     * In cooperative rebalancing, it will be called at the end of a rebalance on the set of partitions being revoked iff the set is non-empty.
+     * For examples on usage of this API, see Usage Examples section of {@link KafkaConsumer KafkaConsumer}.
      * 
                  
      * It is common for the revocation callback to use the consumer instance in order to commit offsets. It is possible
      * for a {@link org.apache.kafka.common.errors.WakeupException} or {@link org.apache.kafka.common.errors.InterruptException}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index f361735a6e010..b5b5ce291292e 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -351,62 +351,50 @@ protected void onJoinComplete(int generation,
             return;
         }
 
-        // The leader may have assigned partitions which match our subscription pattern, but which
-        // were not explicitly requested, so we update the joined subscription here.
-        maybeUpdateJoinedSubscription(assignedPartitions);
-
-        // give the assignor a chance to update internal state based on the received assignment
-        ConsumerGroupMetadata metadata = new ConsumerGroupMetadata(rebalanceConfig.groupId, generation, memberId, rebalanceConfig.groupInstanceId);
-        assignor.onAssignment(assignment, metadata);
-
-        // reschedule the auto commit starting from now
-        if (autoCommitEnabled)
-            this.nextAutoCommitTimer.updateAndReset(autoCommitIntervalMs);
-
-        // execute the user's callback after rebalance
         final AtomicReference firstException = new AtomicReference<>(null);
         Set addedPartitions = new HashSet<>(assignedPartitions);
         addedPartitions.removeAll(ownedPartitions);
 
-        switch (protocol) {
-            case EAGER:
-                // assign partitions that are not yet owned
-                subscriptions.assignFromSubscribed(assignedPartitions);
-
-                firstException.compareAndSet(null, invokePartitionsAssigned(addedPartitions));
-
-                break;
+        // Invoke user's revocation callback before changing assignment or updating state
+        if (protocol == RebalanceProtocol.COOPERATIVE) {
+            Set revokedPartitions = new HashSet<>(ownedPartitions);
+            revokedPartitions.removeAll(assignedPartitions);
 
-            case COOPERATIVE:
-                Set revokedPartitions = new HashSet<>(ownedPartitions);
-                revokedPartitions.removeAll(assignedPartitions);
+            log.info("Updating with newly assigned partitions: {}, compare with already owned partitions: {}, " +
+                    "newly added partitions: {}, revoking partitions: {}",
+                Utils.join(assignedPartitions, ", "),
+                Utils.join(ownedPartitions, ", "),
+                Utils.join(addedPartitions, ", "),
+                Utils.join(revokedPartitions, ", "));
 
-                log.info("Updating with newly assigned partitions: {}, compare with already owned partitions: {}, " +
-                        "newly added partitions: {}, revoking partitions: {}",
-                    Utils.join(assignedPartitions, ", "),
-                    Utils.join(ownedPartitions, ", "),
-                    Utils.join(addedPartitions, ", "),
-                    Utils.join(revokedPartitions, ", "));
 
+            if (!revokedPartitions.isEmpty()) {
                 // revoke partitions that was previously owned but no longer assigned;
                 // note that we should only change the assignment AFTER we've triggered
                 // the revoke callback
-                if (!revokedPartitions.isEmpty()) {
-                    firstException.compareAndSet(null, invokePartitionsRevoked(revokedPartitions));
-                }
+                firstException.compareAndSet(null, invokePartitionsRevoked(revokedPartitions));
 
-                subscriptions.assignFromSubscribed(assignedPartitions);
+                // if revoked any partitions, need to re-join the group afterwards
+                requestRejoin();
+            }
+        }
 
-                // add partitions that were not previously owned but are now assigned
-                firstException.compareAndSet(null, invokePartitionsAssigned(addedPartitions));
+        // The leader may have assigned partitions which match our subscription pattern, but which
+        // were not explicitly requested, so we update the joined subscription here.
+        maybeUpdateJoinedSubscription(assignedPartitions);
 
-                // if revoked any partitions, need to re-join the group afterwards
-                if (!revokedPartitions.isEmpty()) {
-                    requestRejoin();
-                }
+        // give the assignor a chance to update internal state based on the received assignment
+        ConsumerGroupMetadata metadata = new ConsumerGroupMetadata(rebalanceConfig.groupId, generation, memberId, rebalanceConfig.groupInstanceId);
+        assignor.onAssignment(assignment, metadata);
 
-                break;
-        }
+        // reschedule the auto commit starting from now
+        if (autoCommitEnabled)
+            this.nextAutoCommitTimer.updateAndReset(autoCommitIntervalMs);
+
+        subscriptions.assignFromSubscribed(assignedPartitions);
+
+        // add partitions that were not previously owned but are now assigned
+        firstException.compareAndSet(null, invokePartitionsAssigned(addedPartitions));
 
         if (firstException.get() != null)
             throw new KafkaException("User rebalance callback throws an error", firstException.get());
@@ -566,7 +554,7 @@ protected Map performAssignment(String leaderId,
         // when these topics gets updated from metadata refresh.
         //
         // TODO: this is a hack and not something we want to support long-term unless we push regex into the protocol
-        //       we may need to modify the PartitionAssignor API to better support this case.
+        //       we may need to modify the ConsumerPartitionAssignor API to better support this case.
         Set assignedTopics = new HashSet<>();
         for (Assignment assigned : assignments.values()) {
             for (TopicPartition tp : assigned.partitions())
diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
index e90caae3667d8..a405c6f083368 100644
--- a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
+++ b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
@@ -51,6 +51,7 @@
 import org.apache.kafka.streams.processor.internals.ProcessorTopology;
 import org.apache.kafka.streams.processor.internals.StateDirectory;
 import org.apache.kafka.streams.processor.internals.StreamThread;
+import org.apache.kafka.streams.processor.internals.StreamThread.State;
 import org.apache.kafka.streams.processor.internals.StreamsMetadataState;
 import org.apache.kafka.streams.processor.internals.ThreadStateTransitionValidator;
 import org.apache.kafka.streams.state.HostInfo;
@@ -188,17 +189,19 @@ public class KafkaStreams implements AutoCloseable {
      *
      * 
      * Note the following:
-     * - RUNNING state will transit to REBALANCING if any of its threads is in PARTITION_REVOKED state
+     * - RUNNING state will transit to REBALANCING if any of its threads is in PARTITION_REVOKED or PARTITIONS_ASSIGNED state
      * - REBALANCING state will transit to RUNNING if all of its threads are in RUNNING state
      * - Any state except NOT_RUNNING can go to PENDING_SHUTDOWN (whenever close is called)
      * - Of special importance: If the global stream thread dies, or all stream threads die (or both) then
      *   the instance will be in the ERROR state. The user will need to close it.
      */
-    // TODO: the current transitions from other states directly to RUNNING is due to
-    //       the fact that onPartitionsRevoked may not be triggered. we need to refactor the
-    //       state diagram more thoroughly after we refactor StreamsPartitionAssignor to support COOPERATIVE
     public enum State {
-        CREATED(1, 2, 3), REBALANCING(2, 3, 5), RUNNING(1, 2, 3, 5), PENDING_SHUTDOWN(4), NOT_RUNNING, ERROR(3);
+        CREATED(1, 3),          // 0
+        REBALANCING(2, 3, 5),   // 1
+        RUNNING(1, 2, 3, 5),    // 2
+        PENDING_SHUTDOWN(4),    // 3
+        NOT_RUNNING,            // 4
+        ERROR(3);               // 5
 
         private final Set validTransitions = new HashSet<>();
 
@@ -462,7 +465,7 @@ public synchronized void onChange(final Thread thread,
                     final StreamThread.State newState = (StreamThread.State) abstractNewState;
                     threadState.put(thread.getId(), newState);
 
-                    if (newState == StreamThread.State.PARTITIONS_REVOKED) {
+                    if (newState == StreamThread.State.PARTITIONS_REVOKED || newState == StreamThread.State.PARTITIONS_ASSIGNED) {
                         setState(State.REBALANCING);
                     } else if (newState == StreamThread.State.RUNNING) {
                         maybeSetRunning();
diff --git a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
index f9ce18b34bce9..cfedee572f508 100644
--- a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
+++ b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
@@ -251,6 +251,30 @@ public class StreamsConfig extends AbstractConfig {
     @SuppressWarnings("WeakerAccess")
     public static final String UPGRADE_FROM_11 = "1.1";
 
+    /**
+     * Config value for parameter {@link #UPGRADE_FROM_CONFIG "upgrade.from"} for upgrading an application from version {@code 2.0.x}.
+     */
+    @SuppressWarnings("WeakerAccess")
+    public static final String UPGRADE_FROM_20 = "2.0";
+
+    /**
+     * Config value for parameter {@link #UPGRADE_FROM_CONFIG "upgrade.from"} for upgrading an application from version {@code 2.1.x}.
+     */
+    @SuppressWarnings("WeakerAccess")
+    public static final String UPGRADE_FROM_21 = "2.1";
+
+    /**
+     * Config value for parameter {@link #UPGRADE_FROM_CONFIG "upgrade.from"} for upgrading an application from version {@code 2.2.x}.
+     */
+    @SuppressWarnings("WeakerAccess")
+    public static final String UPGRADE_FROM_22 = "2.2";
+
+    /**
+     * Config value for parameter {@link #UPGRADE_FROM_CONFIG "upgrade.from"} for upgrading an application from version {@code 2.3.x}.
+     */
+    @SuppressWarnings("WeakerAccess")
+    public static final String UPGRADE_FROM_23 = "2.3";
+
     /**
      * Config value for parameter {@link #PROCESSING_GUARANTEE_CONFIG "processing.guarantee"} for at-least-once processing guarantees.
      */
@@ -474,9 +498,10 @@ public class StreamsConfig extends AbstractConfig {
     /** {@code upgrade.from} */
     @SuppressWarnings("WeakerAccess")
     public static final String UPGRADE_FROM_CONFIG = "upgrade.from";
-    private static final String UPGRADE_FROM_DOC = "Allows upgrading from versions 0.10.0/0.10.1/0.10.2/0.11.0/1.0/1.1 to version 1.2 (or newer) in a backward compatible way. " +
-        "When upgrading from 1.2 to a newer version it is not required to specify this config." +
-        "Default is null. Accepted values are \"" + UPGRADE_FROM_0100 + "\", \"" + UPGRADE_FROM_0101 + "\", \"" + UPGRADE_FROM_0102 + "\", \"" + UPGRADE_FROM_0110 + "\", \"" + UPGRADE_FROM_10 + "\", \"" + UPGRADE_FROM_11 + "\" (for upgrading from the corresponding old version).";
+    private static final String UPGRADE_FROM_DOC = "Allows upgrading in a backward compatible way. " +
+        "This is needed when upgrading from [0.10.0, 1.1] to 2.0+, or when upgrading from [2.0, 2.3] to 2.4+. " +
+        "When upgrading from 2.4 to a newer version it is not required to specify this config. " +
+        "Default is null. Accepted values are \"" + UPGRADE_FROM_0100 + "\", \"" + UPGRADE_FROM_0101 + "\", \"" + UPGRADE_FROM_0102 + "\", \"" + UPGRADE_FROM_0110 + "\", \"" + UPGRADE_FROM_10 + "\", \"" + UPGRADE_FROM_11 + "\", \"" + UPGRADE_FROM_20 + "\", \"" + UPGRADE_FROM_21 + "\", \"" + UPGRADE_FROM_22 + "\", \"" + UPGRADE_FROM_23 + "\" (for upgrading from the corresponding old version).";
 
     /** {@code windowstore.changelog.additional.retention.ms} */
     @SuppressWarnings("WeakerAccess")
@@ -709,7 +734,17 @@ public class StreamsConfig extends AbstractConfig {
             .define(UPGRADE_FROM_CONFIG,
                     ConfigDef.Type.STRING,
                     null,
-                    in(null, UPGRADE_FROM_0100, UPGRADE_FROM_0101, UPGRADE_FROM_0102, UPGRADE_FROM_0110, UPGRADE_FROM_10, UPGRADE_FROM_11),
+                    in(null,
+                       UPGRADE_FROM_0100,
+                       UPGRADE_FROM_0101,
+                       UPGRADE_FROM_0102,
+                       UPGRADE_FROM_0110,
+                       UPGRADE_FROM_10,
+                       UPGRADE_FROM_11,
+                       UPGRADE_FROM_20,
+                       UPGRADE_FROM_21,
+                       UPGRADE_FROM_22,
+                       UPGRADE_FROM_23),
                     Importance.LOW,
                     UPGRADE_FROM_DOC)
             .define(WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_CONFIG,
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStandbyTasks.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStandbyTasks.java
index 6025e885a1324..9783970d40788 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStandbyTasks.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStandbyTasks.java
@@ -16,7 +16,13 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.streams.processor.TaskId;
 
 class AssignedStandbyTasks extends AssignedTasks {
 
@@ -34,4 +40,40 @@ int commit() {
         running.forEach((id, task) -> task.allowUpdateOfOffsetLimit());
         return committed;
     }
+
+    /**
+     * Closes standby tasks that were reassigned elsewhere after a rebalance.
+     *
+     * @param revokedTasks the tasks which are no longer owned
+     * @return the changelogs of all standby tasks that were reassigned
+     */
+    List closeRevokedStandbyTasks(final Map> revokedTasks) {
+        log.debug("Closing revoked standby tasks {}", revokedTasks);
+
+        final List revokedChangelogs = new ArrayList<>();
+        for (final Map.Entry> entry : revokedTasks.entrySet()) {
+            final TaskId taskId = entry.getKey();
+            final Task task;
+
+            if (running.containsKey(taskId)) {
+                task = running.get(taskId);
+            } else if (created.containsKey(taskId)) {
+                task = created.get(taskId);
+            } else {
+                log.error("Could not find the standby task {} while closing it", taskId);
+                continue;
+            }
+
+            try {
+                task.close(true, false);
+            } catch (final RuntimeException e) {
+                log.error("Closing the {} {} failed due to the following error:", taskTypeName, task.id(), e);
+            } finally {
+                running.remove(taskId);
+                revokedChangelogs.addAll(task.changelogPartitions());
+            }
+        }
+        return revokedChangelogs;
+    }
+
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasks.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasks.java
index 0dd40ce92b40e..da0fc20939106 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasks.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasks.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
+import java.util.concurrent.atomic.AtomicReference;
 import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.utils.LogContext;
@@ -32,9 +33,11 @@
 import java.util.Set;
 
 class AssignedStreamsTasks extends AssignedTasks implements RestoringTasks {
+    private final Map suspended = new HashMap<>();
     private final Map restoring = new HashMap<>();
     private final Set restoredPartitions = new HashSet<>();
     private final Map restoringByPartition = new HashMap<>();
+    private final Set prevActiveTasks = new HashSet<>();
 
     AssignedStreamsTasks(final LogContext logContext) {
         super(logContext, "stream task");
@@ -49,6 +52,7 @@ public StreamTask restoringTaskFor(final TopicPartition partition) {
     List allTasks() {
         final List tasks = super.allTasks();
         tasks.addAll(restoring.values());
+        tasks.addAll(suspended.values());
         return tasks;
     }
 
@@ -56,39 +60,285 @@ List allTasks() {
     Set allAssignedTaskIds() {
         final Set taskIds = super.allAssignedTaskIds();
         taskIds.addAll(restoring.keySet());
+        taskIds.addAll(suspended.keySet());
         return taskIds;
     }
 
     @Override
     boolean allTasksRunning() {
-        return super.allTasksRunning() && restoring.isEmpty();
+        // If we have some tasks that are suspended but others are running, count this as all tasks are running
+        // since they will be closed soon anyway (eg if partitions are revoked at beginning of cooperative rebalance)
+        return super.allTasksRunning() && restoring.isEmpty() && (suspended.isEmpty() || !running.isEmpty());
     }
 
-    RuntimeException closeAllRestoringTasks() {
-        RuntimeException exception = null;
+    @Override
+    void closeTask(final StreamTask task, final boolean clean) {
+        if (suspended.containsKey(task.id())) {
+            task.closeSuspended(clean, false, null);
+        } else {
+            task.close(clean, false);
+        }
+    }
+    
+    Set suspendedTaskIds() {
+        return suspended.keySet();
+    }
+
+    Set previousRunningTaskIds() {
+        return prevActiveTasks;
+    }
+
+    RuntimeException suspendOrCloseTasks(final Set revokedTasks,
+                                         final List revokedTaskChangelogs) {
+        final AtomicReference firstException = new AtomicReference<>(null);
+        final Set revokedRunningTasks = new HashSet<>();
+        final Set revokedNonRunningTasks = new HashSet<>();
+        final Set revokedRestoringTasks = new HashSet<>();
+
+        // This set is used only for eager rebalancing, so we can just clear it and add any/all tasks that were running
+        prevActiveTasks.clear();
+        prevActiveTasks.addAll(runningTaskIds());
+
+        for (final TaskId task : revokedTasks) {
+            if (running.containsKey(task)) {
+                revokedRunningTasks.add(task);
+            } else if (created.containsKey(task)) {
+                revokedNonRunningTasks.add(task);
+            } else if (restoring.containsKey(task)) {
+                revokedRestoringTasks.add(task);
+            } else if (!suspended.containsKey(task)) {
+                log.warn("Task {} was revoked but cannot be found in the assignment", task);
+            }
+        }
+
+        firstException.compareAndSet(null, suspendRunningTasks(revokedRunningTasks, revokedTaskChangelogs));
+        firstException.compareAndSet(null, closeNonRunningTasks(revokedNonRunningTasks, revokedTaskChangelogs));
+        firstException.compareAndSet(null, closeRestoringTasks(revokedRestoringTasks, revokedTaskChangelogs));
+
+        return firstException.get();
+    }
+
+    private RuntimeException suspendRunningTasks(final Set runningTasksToSuspend,
+                                                 final List taskChangelogs) {
+
+        final AtomicReference firstException = new AtomicReference<>(null);
+        log.debug("Suspending running {} {}", taskTypeName, running.keySet());
+
+        for (final TaskId id : runningTasksToSuspend) {
+            final StreamTask task = running.get(id);
 
-        log.trace("Closing all restoring stream tasks {}", restoring.keySet());
-        final Iterator restoringTaskIterator = restoring.values().iterator();
-        while (restoringTaskIterator.hasNext()) {
-            final StreamTask task = restoringTaskIterator.next();
-            log.debug("Closing restoring task {}", task.id());
             try {
-                task.closeStateManager(true);
+                task.suspend();
+                suspended.put(id, task);
+            } catch (final TaskMigratedException closeAsZombieAndSwallow) {
+                // as we suspend a task, we are either shutting down or rebalancing, thus, we swallow and move on
+                log.info("Failed to suspend {} {} since it got migrated to another thread already. " +
+                    "Closing it as zombie and move on.", taskTypeName, id);
+                firstException.compareAndSet(null, closeZombieTask(task));
+                prevActiveTasks.remove(id);
             } catch (final RuntimeException e) {
-                log.error("Failed to remove restoring task {} due to the following error:", task.id(), e);
-                if (exception == null) {
-                    exception = e;
+                log.error("Suspending {} {} failed due to the following error:", taskTypeName, id, e);
+                firstException.compareAndSet(null, e);
+                try {
+                    prevActiveTasks.remove(id);
+                    task.close(false, false);
+                } catch (final RuntimeException f) {
+                    log.error(
+                        "After suspending failed, closing the same {} {} failed again due to the following error:",
+                        taskTypeName, id, f);
                 }
             } finally {
-                restoringTaskIterator.remove();
+                running.remove(id);
+                runningByPartition.keySet().removeAll(task.partitions());
+                runningByPartition.keySet().removeAll(task.changelogPartitions());
+                taskChangelogs.addAll(task.changelogPartitions());
             }
         }
 
-        restoring.clear();
-        restoredPartitions.clear();
-        restoringByPartition.clear();
+        log.trace("Successfully suspended the running {} {}", taskTypeName, suspended.keySet());
 
-        return exception;
+        return firstException.get();
+    }
+
+    private RuntimeException closeNonRunningTasks(final Set nonRunningTasksToClose,
+                                                  final List closedTaskChangelogs) {
+        log.debug("Closing the created but not initialized {} {}", taskTypeName, nonRunningTasksToClose);
+        final AtomicReference firstException = new AtomicReference<>();
+
+        for (final TaskId id : nonRunningTasksToClose) {
+            final StreamTask task = created.get(id);
+            firstException.compareAndSet(null, closeNonRunning(false, task, closedTaskChangelogs));
+        }
+
+        return firstException.get();
+    }
+
+    RuntimeException closeRestoringTasks(final Set restoringTasksToClose,
+                                         final List closedTaskChangelogs) {
+        log.debug("Closing restoring stream tasks {}", restoringTasksToClose);
+        final AtomicReference firstException = new AtomicReference<>();
+
+        for (final TaskId id : restoringTasksToClose) {
+            final StreamTask task = restoring.get(id);
+            firstException.compareAndSet(null, closeRestoring(false, task, closedTaskChangelogs));
+        }
+
+        return firstException.get();
+    }
+
+    private RuntimeException closeRunning(final boolean isZombie,
+                                          final StreamTask task,
+                                          final List closedTaskChangelogs) {
+        running.remove(task.id());
+        runningByPartition.keySet().removeAll(task.partitions());
+        runningByPartition.keySet().removeAll(task.changelogPartitions());
+        closedTaskChangelogs.addAll(task.changelogPartitions());
+
+        try {
+            final boolean clean = !isZombie;
+            task.close(clean, isZombie);
+        } catch (final RuntimeException e) {
+            log.error("Failed to close {}, {}", taskTypeName, task.id(), e);
+            return e;
+        }
+
+        return null;
+    }
+
+    private RuntimeException closeNonRunning(final boolean isZombie,
+                                             final StreamTask task,
+                                             final List closedTaskChangelogs) {
+        created.remove(task.id());
+        closedTaskChangelogs.addAll(task.changelogPartitions());
+
+        try {
+            task.close(false, isZombie);
+        } catch (final RuntimeException e) {
+            log.error("Failed to close {}, {}", taskTypeName, task.id(), e);
+            return e;
+        }
+
+        return null;
+    }
+
+    private RuntimeException closeRestoring(final boolean isZombie,
+                                            final StreamTask task,
+                                            final List closedTaskChangelogs) {
+        restoring.remove(task.id());
+        closedTaskChangelogs.addAll(task.changelogPartitions());
+        for (final TopicPartition tp : task.partitions()) {
+            restoredPartitions.remove(tp);
+            restoringByPartition.remove(tp);
+        }
+
+        try {
+            final boolean clean = !isZombie;
+            task.closeStateManager(clean);
+        } catch (final RuntimeException e) {
+            log.error("Failed to close restoring task {} due to the following error:", task.id(), e);
+            return e;
+        }
+
+        return null;
+    }
+
+    private RuntimeException closeSuspended(final boolean isZombie,
+                                            final StreamTask task) {
+        suspended.remove(task.id());
+
+        try {
+            final boolean clean = !isZombie;
+            task.closeSuspended(clean, isZombie, null);
+        } catch (final RuntimeException e) {
+            log.error("Failed to close suspended {} {} due to the following error:", taskTypeName, task.id(), e);
+            return e;
+        }
+
+        return null;
+    }
+
+    RuntimeException closeNotAssignedSuspendedTasks(final Set revokedTasks) {
+        log.debug("Closing the revoked active tasks {} {}", taskTypeName, revokedTasks);
+        final AtomicReference firstException = new AtomicReference<>(null);
+
+        for (final TaskId revokedTask : revokedTasks) {
+            final StreamTask suspendedTask = suspended.get(revokedTask);
+
+            // task may not be in the suspended tasks if it was closed due to some error
+            if (suspendedTask != null) {
+                firstException.compareAndSet(null, closeSuspended(false, suspendedTask));
+            } else {
+                log.debug("Revoked task {} could not be found in suspended, may have already been closed", revokedTask);
+            }
+        }
+        return firstException.get();
+    }
+
+    RuntimeException closeZombieTasks(final Set lostTasks, final List lostTaskChangelogs) {
+        final AtomicReference firstException = new AtomicReference<>(null);
+
+        for (final TaskId id : lostTasks) {
+            if (suspended.containsKey(id)) {
+                firstException.compareAndSet(null, closeSuspended(true, suspended.get(id)));
+            } else if (created.containsKey(id)) {
+                firstException.compareAndSet(null, closeNonRunning(true, created.get(id), lostTaskChangelogs));
+            } else if (restoring.containsKey(id)) {
+                firstException.compareAndSet(null, closeRestoring(true, created.get(id), lostTaskChangelogs));
+            } else if (running.containsKey(id)) {
+                firstException.compareAndSet(null, closeRunning(true, running.get(id), lostTaskChangelogs));
+            } else {
+                // task may have already been closed as a zombie and removed from all task maps
+            }
+        }
+
+        // We always clear the prevActiveTasks and replace with current set of running tasks to encode in subscription
+        // We should exclude any tasks that were lost however, they will be counted as standbys for assignment purposes
+        prevActiveTasks.clear();
+        prevActiveTasks.addAll(running.keySet());
+
+        // With the current rebalance protocol, there should not be any running tasks left as they were all lost
+        if (!prevActiveTasks.isEmpty()) {
+            log.error("Found still running {} after closing all tasks lost as zombies", taskTypeName);
+            firstException.compareAndSet(null, new IllegalStateException("Not all lost tasks were closed as zombies"));
+        }
+        return firstException.get();
+    }
+
+    /**
+     * @throws TaskMigratedException if the task producer got fenced (EOS only)
+     */
+    boolean maybeResumeSuspendedTask(final TaskId taskId,
+                                     final Set partitions) {
+        if (suspended.containsKey(taskId)) {
+            final StreamTask task = suspended.get(taskId);
+            log.trace("Found suspended {} {}", taskTypeName, taskId);
+            suspended.remove(taskId);
+
+            if (task.partitions().equals(partitions)) {
+                task.resume();
+                try {
+                    transitionToRunning(task);
+                } catch (final TaskMigratedException e) {
+                    // we need to catch migration exception internally since this function
+                    // is triggered in the rebalance callback
+                    log.info("Failed to resume {} {} since it got migrated to another thread already. " +
+                        "Closing it as zombie before triggering a new rebalance.", taskTypeName, task.id());
+                    final RuntimeException fatalException = closeZombieTask(task);
+                    running.remove(taskId);
+
+                    if (fatalException != null) {
+                        throw fatalException;
+                    }
+                    throw e;
+                }
+                log.trace("Resuming suspended {} {}", taskTypeName, task.id());
+                return true;
+            } else {
+                log.warn("Couldn't resume task {} assigned partitions {}, task partitions {}", taskId, partitions, task.partitions());
+                task.closeSuspended(true, false, null);
+            }
+        }
+        return false;
     }
 
     void updateRestored(final Collection restored) {
@@ -254,19 +504,24 @@ void clear() {
         restoring.clear();
         restoringByPartition.clear();
         restoredPartitions.clear();
+        suspended.clear();
     }
 
     public String toString(final String indent) {
         final StringBuilder builder = new StringBuilder();
         builder.append(super.toString(indent));
         describe(builder, restoring.values(), indent, "Restoring:");
+        describe(builder, suspended.values(), indent, "Suspended:");
         return builder.toString();
     }
 
-    // for testing only
-
+    // the following are for testing only
     Collection restoringTasks() {
         return Collections.unmodifiableCollection(restoring.values());
     }
 
+    Set restoringTaskIds() {
+        return new HashSet<>(restoring.keySet());
+    }
+
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
index 8d2c3596dfc90..21f700bb6f5d8 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
@@ -38,14 +38,12 @@
 
 abstract class AssignedTasks {
     final Logger log;
-    private final String taskTypeName;
-    private final Map created = new HashMap<>();
-    private final Map suspended = new HashMap<>();
-    private final Set previousActiveTasks = new HashSet<>();
+    final String taskTypeName;
+    final Map created = new HashMap<>();
 
     // IQ may access this map.
     final Map running = new ConcurrentHashMap<>();
-    private final Map runningByPartition = new HashMap<>();
+    final Map runningByPartition = new HashMap<>();
 
     AssignedTasks(final LogContext logContext,
                   final String taskTypeName) {
@@ -59,7 +57,7 @@ void addNewTask(final T task) {
 
     /**
      * @throws IllegalStateException If store gets registered after initialized is already finished
-     * @throws StreamsException if the store's change log does not contain the partition
+     * @throws StreamsException if the store's changelog does not contain the partition
      * @throws TaskMigratedException if the task producer got fenced (EOS only)
      */
     void initializeNewTasks() {
@@ -85,68 +83,13 @@ void initializeNewTasks() {
     }
 
     boolean allTasksRunning() {
-        return created.isEmpty() && suspended.isEmpty();
+        return created.isEmpty();
     }
 
     Collection running() {
         return running.values();
     }
 
-    RuntimeException suspend() {
-        final AtomicReference firstException = new AtomicReference<>(null);
-        log.trace("Suspending running {} {}", taskTypeName, runningTaskIds());
-        firstException.compareAndSet(null, suspendTasks(running.values()));
-        log.trace("Close created {} {}", taskTypeName, created.keySet());
-        firstException.compareAndSet(null, closeNonRunningTasks(created.values()));
-        previousActiveTasks.clear();
-        previousActiveTasks.addAll(running.keySet());
-        running.clear();
-        created.clear();
-        runningByPartition.clear();
-        return firstException.get();
-    }
-
-    private RuntimeException closeNonRunningTasks(final Collection tasks) {
-        RuntimeException exception = null;
-        for (final T task : tasks) {
-            try {
-                task.close(false, false);
-            } catch (final RuntimeException e) {
-                log.error("Failed to close {}, {}", taskTypeName, task.id(), e);
-                if (exception == null) {
-                    exception = e;
-                }
-            }
-        }
-        return exception;
-    }
-
-    private RuntimeException suspendTasks(final Collection tasks) {
-        final AtomicReference firstException = new AtomicReference<>(null);
-        for (final Iterator it = tasks.iterator(); it.hasNext(); ) {
-            final T task = it.next();
-            try {
-                task.suspend();
-                suspended.put(task.id(), task);
-            } catch (final TaskMigratedException closeAsZombieAndSwallow) {
-                // as we suspend a task, we are either shutting down or rebalancing, thus, we swallow and move on
-                log.info("Failed to suspend {} {} since it got migrated to another thread already. " +
-                        "Closing it as zombie and move on.", taskTypeName, task.id());
-                firstException.compareAndSet(null, closeZombieTask(task));
-                it.remove();
-            } catch (final RuntimeException e) {
-                log.error("Suspending {} {} failed due to the following error:", taskTypeName, task.id(), e);
-                firstException.compareAndSet(null, e);
-                try {
-                    task.close(false, false);
-                } catch (final RuntimeException f) {
-                    log.error("After suspending failed, closing the same {} {} failed again due to the following error:", taskTypeName, task.id(), f);
-                }
-            }
-        }
-        return firstException.get();
-    }
-
     RuntimeException closeZombieTask(final T task) {
         try {
             task.close(false, true);
@@ -161,39 +104,6 @@ boolean hasRunningTasks() {
         return !running.isEmpty();
     }
 
-    /**
-     * @throws TaskMigratedException if the task producer got fenced (EOS only)
-     */
-    boolean maybeResumeSuspendedTask(final TaskId taskId, final Set partitions) {
-        if (suspended.containsKey(taskId)) {
-            final T task = suspended.get(taskId);
-            log.trace("Found suspended {} {}", taskTypeName, taskId);
-            if (task.partitions().equals(partitions)) {
-                suspended.remove(taskId);
-                task.resume();
-                try {
-                    transitionToRunning(task);
-                } catch (final TaskMigratedException e) {
-                    // we need to catch migration exception internally since this function
-                    // is triggered in the rebalance callback
-                    log.info("Failed to resume {} {} since it got migrated to another thread already. " +
-                            "Closing it as zombie before triggering a new rebalance.", taskTypeName, task.id());
-                    final RuntimeException fatalException = closeZombieTask(task);
-                    running.remove(task.id());
-                    if (fatalException != null) {
-                        throw fatalException;
-                    }
-                    throw e;
-                }
-                log.trace("Resuming suspended {} {}", taskTypeName, task.id());
-                return true;
-            } else {
-                log.warn("Couldn't resume task {} assigned partitions {}, task partitions {}", taskId, partitions, task.partitions());
-            }
-        }
-        return false;
-    }
-
     /**
      * @throws TaskMigratedException if the task producer got fenced (EOS only)
      */
@@ -230,7 +140,6 @@ public String toString() {
     public String toString(final String indent) {
         final StringBuilder builder = new StringBuilder();
         describe(builder, running.values(), indent, "Running:");
-        describe(builder, suspended.values(), indent, "Suspended:");
         describe(builder, created.values(), indent, "New:");
         return builder.toString();
     }
@@ -249,7 +158,6 @@ void describe(final StringBuilder builder,
     List allTasks() {
         final List tasks = new ArrayList<>();
         tasks.addAll(running.values());
-        tasks.addAll(suspended.values());
         tasks.addAll(created.values());
         return tasks;
     }
@@ -257,7 +165,6 @@ List allTasks() {
     Set allAssignedTaskIds() {
         final Set taskIds = new HashSet<>();
         taskIds.addAll(running.keySet());
-        taskIds.addAll(suspended.keySet());
         taskIds.addAll(created.keySet());
         return taskIds;
     }
@@ -266,11 +173,6 @@ void clear() {
         runningByPartition.clear();
         running.clear();
         created.clear();
-        suspended.clear();
-    }
-
-    Set previousTaskIds() {
-        return previousActiveTasks;
     }
 
     /**
@@ -280,6 +182,7 @@ Set previousTaskIds() {
     int commit() {
         int committed = 0;
         RuntimeException firstException = null;
+
         for (final Iterator it = running().iterator(); it.hasNext(); ) {
             final T task = it.next();
             try {
@@ -314,33 +217,12 @@ int commit() {
         return committed;
     }
 
-    void closeNonAssignedSuspendedTasks(final Map> newAssignment) {
-        final Iterator standByTaskIterator = suspended.values().iterator();
-        while (standByTaskIterator.hasNext()) {
-            final T suspendedTask = standByTaskIterator.next();
-            if (!newAssignment.containsKey(suspendedTask.id()) || !suspendedTask.partitions().equals(newAssignment.get(suspendedTask.id()))) {
-                log.debug("Closing suspended and not re-assigned {} {}", taskTypeName, suspendedTask.id());
-                try {
-                    suspendedTask.closeSuspended(true, false, null);
-                } catch (final Exception e) {
-                    log.error("Failed to remove suspended {} {} due to the following error:", taskTypeName, suspendedTask.id(), e);
-                } finally {
-                    standByTaskIterator.remove();
-                }
-            }
-        }
-    }
-
     void close(final boolean clean) {
         final AtomicReference firstException = new AtomicReference<>(null);
 
         for (final T task: allTasks()) {
             try {
-                if (suspended.containsKey(task.id())) {
-                    task.closeSuspended(clean, false, null);
-                } else {
-                    task.close(clean, false);
-                }
+                closeTask(task, clean);
             } catch (final TaskMigratedException e) {
                 log.info("Failed to close {} {} since it got migrated to another thread already. " +
                     "Closing it as zombie and move on.", taskTypeName, task.id());
@@ -365,6 +247,10 @@ void close(final boolean clean) {
         }
     }
 
+    void closeTask(final T task, final boolean clean) {
+        task.close(clean, false);
+    }
+
     private boolean closeUnclean(final T task) {
         log.info("Try to close {} {} unclean.", task.getClass().getSimpleName(), task.id());
         try {
@@ -379,4 +265,5 @@ private boolean closeUnclean(final T task) {
 
         return true;
     }
+
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ChangelogReader.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ChangelogReader.java
index ed07aa7af34a3..782be15b90f98 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ChangelogReader.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ChangelogReader.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
+import java.util.List;
 import org.apache.kafka.common.TopicPartition;
 
 import java.util.Collection;
@@ -24,7 +25,7 @@
 /**
  * Performs bulk read operations from a set of partitions. Used to
  * restore  {@link org.apache.kafka.streams.processor.StateStore}s from their
- * change logs
+ * changelogs
  */
 public interface ChangelogReader {
     /**
@@ -44,5 +45,15 @@ public interface ChangelogReader {
      */
     Map restoredOffsets();
 
-    void reset();
+    /**
+     * Removes the passed in partitions from the set of changelogs
+     * @param revokedPartitions the set of partitions to remove
+     */
+    void remove(List revokedPartitions);
+
+    /**
+     * Clear all partitions
+     */
+    void clear();
+
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java
index 3b45c5efc55db..6c6a1c4d12e75 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java
@@ -79,9 +79,9 @@ public Collection restore(final RestoringTasks active) {
             initialize(active);
         }
 
-        if (needsRestoring.isEmpty()) {
+        if (needsRestoring.isEmpty() || restoreConsumer.assignment().isEmpty()) {
             restoreConsumer.unsubscribe();
-            return completed();
+            return completedRestorers;
         }
 
         try {
@@ -120,7 +120,7 @@ public Collection restore(final RestoringTasks active) {
             restoreConsumer.unsubscribe();
         }
 
-        return completed();
+        return completedRestorers;
     }
 
     private void initialize(final RestoringTasks active) {
@@ -255,10 +255,6 @@ private void logRestoreOffsets(final TopicPartition partition,
                   endOffset);
     }
 
-    private Collection completed() {
-        return completedRestorers;
-    }
-
     private void refreshChangelogInfo() {
         try {
             partitionInfo.putAll(restoreConsumer.listTopics());
@@ -280,7 +276,19 @@ public Map restoredOffsets() {
     }
 
     @Override
-    public void reset() {
+    public void remove(final List revokedPartitions) {
+        for (final TopicPartition partition : revokedPartitions) {
+            partitionInfo.remove(partition.topic());
+            stateRestorers.remove(partition);
+            needsRestoring.remove(partition);
+            restoreToOffsets.remove(partition);
+            needsInitializing.remove(partition);
+            completedRestorers.remove(partition);
+        }
+    }
+
+    @Override
+    public void clear() {
         partitionInfo.clear();
         stateRestorers.clear();
         needsRestoring.clear();
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index b11b05c30f661..e33c88724b0d0 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -73,43 +73,46 @@ public class StreamThread extends Thread {
      * The expected state transitions with the following defined states is:
      *
      * 
                  -     *                +-------------+
-     *          +<--- | Created (0) |
-     *          |     +-----+-------+
-     *          |           |
-     *          |           v
-     *          |     +-----+-------+
-     *          +<--- | Starting (1)|
-     *          |     +-----+-------+
-     *          |           |
-     *          |           |
-     *          |           v
-     *          |     +-----+-------+
-     *          +<--- | Partitions  |
-     *          |     | Revoked (2) | <----+
-     *          |     +-----+-------+      |
-     *          |           |              |
-     *          |           v              |
-     *          |     +-----+-------+      |
-     *          |     | Partitions  |      |
-     *          +<--- | Assigned (3)| ---->+
-     *          |     +-----+-------+      |
-     *          |           |              |
-     *          |           v              |
-     *          |     +-----+-------+      |
-     *          |     | Running (4) | ---->+
-     *          |     +-----+-------+
-     *          |           |
-     *          |           v
-     *          |     +-----+-------+
-     *          +---> | Pending     |
-     *                | Shutdown (5)|
-     *                +-----+-------+
-     *                      |
-     *                      v
-     *                +-----+-------+
-     *                | Dead (6)    |
-     *                +-------------+
+     *                 +-------------+
+     *          +<---- | Created (0) |
+     *          |      +-----+-------+
+     *          |            |
+     *          |            v
+     *          |      +-----+-------+
+     *          +<---- | Starting (1)|----->+
+     *          |      +-----+-------+      |
+     *          |            |              |
+     *          |            |              |
+     *          |            v              |
+     *          |      +-----+-------+      |
+     *          +<---- | Partitions  |      |
+     *          |      | Revoked (2) | <----+
+     *          |      +-----+-------+      |
+     *          |           |  ^            |
+     *          |           |  |            |
+     *          |           v  |            |
+     *          |      +-----+-------+      |
+     *          +<---- | Partitions  |      |
+     *          |      | Assigned (3)| <----+
+     *          |      +-----+-------+      |
+     *          |            |              |
+     *          |            |              |
+     *          |            v              |
+     *          |      +-----+-------+      |
+     *          |      | Running (4) | ---->+
+     *          |      +-----+-------+
+     *          |            |
+     *          |            |
+     *          |            v
+     *          |      +-----+-------+
+     *          +----> | Pending     |
+     *                 | Shutdown (5)|
+     *                 +-----+-------+
+     *                       |
+     *                       v
+     *                 +-----+-------+
+     *                 | Dead (6)    |
+     *                 +-------------+
      * 
                  
      *
      * Note the following:
@@ -124,15 +127,20 @@ public class StreamThread extends Thread {
      *         State PARTITIONS_REVOKED may want transit to itself indefinitely, in the corner case when
      *         the coordinator repeatedly fails in-between revoking partitions and assigning new partitions.
      *         Also during streams instance start up PARTITIONS_REVOKED may want to transit to itself as well.
-     *         In this case we will forbid the transition but will not treat as an error.
+     *         In this case we will allow the transition but it will be a no-op as the set of revoked partitions
+     *         should be empty.
      *     
                  • 
      * 
                
      */
     public enum State implements ThreadStateTransitionValidator {
-        // TODO: the current transitions from other states directly to PARTITIONS_REVOKED is due to
-        //       the fact that onPartitionsRevoked may not be triggered. we need to refactor the
-        //       state diagram more thoroughly after we refactor StreamsPartitionAssignor to support COOPERATIVE
-        CREATED(1, 5), STARTING(2, 3, 5), PARTITIONS_REVOKED(3, 5), PARTITIONS_ASSIGNED(2, 3, 4, 5), RUNNING(2, 3, 5), PENDING_SHUTDOWN(6), DEAD;
+
+        CREATED(1, 5),                   // 0
+        STARTING(2, 3, 5),               // 1
+        PARTITIONS_REVOKED(3, 5),        // 2
+        PARTITIONS_ASSIGNED(2, 3, 4, 5), // 3
+        RUNNING(2, 3, 5),                // 4
+        PENDING_SHUTDOWN(6),             // 5
+        DEAD;                            // 6
 
         private final Set validTransitions = new HashSet<>();
 
@@ -206,12 +214,6 @@ State setState(final State newState) {
                 // when the state is already in NOT_RUNNING, all its transitions
                 // will be refused but we do not throw exception here
                 return null;
-            } else if (state == State.PARTITIONS_REVOKED && newState == State.PARTITIONS_REVOKED) {
-                log.debug("Ignoring request to transit from PARTITIONS_REVOKED to PARTITIONS_REVOKED: " +
-                              "self transition is not allowed");
-                // when the state is already in PARTITIONS_REVOKED, its transition to itself will be
-                // refused but we do not throw exception here
-                return null;
             } else if (!state.isValidTransition(newState)) {
                 log.error("Unexpected state transition from {} to {}", oldState, newState);
                 throw new StreamsException(logPrefix + "Unexpected state transition from " + oldState + " to " + newState);
@@ -245,104 +247,12 @@ public boolean isRunning() {
         }
     }
 
-    static class RebalanceListener implements ConsumerRebalanceListener {
-        private final Time time;
-        private final TaskManager taskManager;
-        private final StreamThread streamThread;
-        private final Logger log;
-
-        RebalanceListener(final Time time,
-                          final TaskManager taskManager,
-                          final StreamThread streamThread,
-                          final Logger log) {
-            this.time = time;
-            this.taskManager = taskManager;
-            this.streamThread = streamThread;
-            this.log = log;
-        }
-
-        @Override
-        public void onPartitionsAssigned(final Collection assignment) {
-            log.debug("Current state {}: assigned partitions {} at the end of consumer rebalance.\n" +
-                    "\tcurrent suspended active tasks: {}\n" +
-                    "\tcurrent suspended standby tasks: {}\n",
-                streamThread.state,
-                assignment,
-                taskManager.suspendedActiveTaskIds(),
-                taskManager.suspendedStandbyTaskIds());
-
-            if (streamThread.assignmentErrorCode.get() == AssignorError.INCOMPLETE_SOURCE_TOPIC_METADATA.code()) {
-                log.error("Received error code {} - shutdown", streamThread.assignmentErrorCode.get());
-                streamThread.shutdown();
-                return;
-            }
-            final long start = time.milliseconds();
-            try {
-                if (streamThread.setState(State.PARTITIONS_ASSIGNED) == null) {
-                    log.debug(
-                        "Skipping task creation in rebalance because we are already in {} state.",
-                        streamThread.state()
-                    );
-                } else if (streamThread.assignmentErrorCode.get() != AssignorError.NONE.code()) {
-                    log.debug(
-                        "Encountered assignment error during partition assignment: {}. Skipping task initialization",
-                        streamThread.assignmentErrorCode
-                    );
-                } else {
-                    log.debug("Creating tasks based on assignment.");
-                    taskManager.createTasks(assignment);
-                }
-            } catch (final Throwable t) {
-                log.error(
-                    "Error caught during partition assignment, " +
-                        "will abort the current process and re-throw at the end of rebalance", t);
-                streamThread.setRebalanceException(t);
-            } finally {
-                log.info("partition assignment took {} ms.\n" +
-                        "\tcurrent active tasks: {}\n" +
-                        "\tcurrent standby tasks: {}\n" +
-                        "\tprevious active tasks: {}\n",
-                    time.milliseconds() - start,
-                    taskManager.activeTaskIds(),
-                    taskManager.standbyTaskIds(),
-                    taskManager.prevActiveTaskIds());
-            }
-        }
+    int getAssignmentErrorCode() {
+        return assignmentErrorCode.get();
+    }
 
-        @Override
-        public void onPartitionsRevoked(final Collection assignment) {
-            log.debug("Current state {}: revoked partitions {} at the beginning of consumer rebalance.\n" +
-                    "\tcurrent assigned active tasks: {}\n" +
-                    "\tcurrent assigned standby tasks: {}\n",
-                streamThread.state,
-                assignment,
-                taskManager.activeTaskIds(),
-                taskManager.standbyTaskIds());
-
-            if (streamThread.setState(State.PARTITIONS_REVOKED) != null) {
-                final long start = time.milliseconds();
-                try {
-                    // suspend active tasks
-                    taskManager.suspendTasksAndState();
-                } catch (final Throwable t) {
-                    log.error(
-                        "Error caught during partition revocation, " +
-                            "will abort the current process and re-throw at the end of rebalance: {}",
-                        t
-                    );
-                    streamThread.setRebalanceException(t);
-                } finally {
-                    streamThread.clearStandbyRecords();
-
-                    log.info("partition revocation took {} ms.\n" +
-                            "\tsuspended active tasks: {}\n" +
-                            "\tsuspended standby tasks: {}",
-                        time.milliseconds() - start,
-                        taskManager.suspendedActiveTaskIds(),
-                        taskManager.suspendedStandbyTaskIds());
-                }
-            }
-        }
+    void setRebalanceException(final Throwable rebalanceException) {
+        this.rebalanceException = rebalanceException;
     }
 
     static abstract class AbstractTaskCreator {
@@ -705,7 +615,7 @@ public StreamThread(final Time time,
         this.builder = builder;
         this.logPrefix = logContext.logPrefix();
         this.log = logContext.logger(StreamThread.class);
-        this.rebalanceListener = new RebalanceListener(time, taskManager, this, this.log);
+        this.rebalanceListener = new StreamsRebalanceListener(time, taskManager, this, this.log);
         this.taskManager = taskManager;
         this.producer = producer;
         this.restoreConsumer = restoreConsumer;
@@ -784,10 +694,6 @@ public void run() {
         }
     }
 
-    private void setRebalanceException(final Throwable rebalanceException) {
-        this.rebalanceException = rebalanceException;
-    }
-
     /**
      * Main event loop for polling, and processing records through topologies.
      *
@@ -1231,8 +1137,10 @@ private void completeShutdown(final boolean cleanRun) {
         log.info("Shutdown complete");
     }
 
-    private void clearStandbyRecords() {
-        standbyRecords.clear();
+    void clearStandbyRecords(final List partitions) {
+        for (final TopicPartition tp : partitions) {
+            standbyRecords.remove(tp);
+        }
     }
 
     /**
@@ -1339,7 +1247,7 @@ public Map consumerMetrics() {
     }
 
     public Map adminClientMetrics() {
-        final Map adminClientMetrics = taskManager.getAdminClient().metrics();
+        final Map adminClientMetrics = taskManager.adminClient().metrics();
         final LinkedHashMap result = new LinkedHashMap<>();
         result.putAll(adminClientMetrics);
         return result;
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
index 869c42a882fb3..b207e1de39458 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
@@ -166,6 +166,7 @@ public String toString() {
 
     private InternalTopicManager internalTopicManager;
     private CopartitionedTopicsEnforcer copartitionedTopicsEnforcer;
+    private RebalanceProtocol rebalanceProtocol;
 
     protected String userEndPoint() {
         return userEndPoint;
@@ -196,14 +197,24 @@ public void configure(final Map configs) {
         userEndPoint = assignorConfiguration.getUserEndPoint();
         internalTopicManager = assignorConfiguration.getInternalTopicManager();
         copartitionedTopicsEnforcer = assignorConfiguration.getCopartitionedTopicsEnforcer();
+        rebalanceProtocol = assignorConfiguration.rebalanceProtocol();
     }
 
-
     @Override
     public String name() {
         return "stream";
     }
 
+    @Override
+    public List supportedProtocols() {
+        final List supportedProtocols = new ArrayList<>();
+        supportedProtocols.add(RebalanceProtocol.EAGER);
+        if (rebalanceProtocol == RebalanceProtocol.COOPERATIVE) {
+            supportedProtocols.add(rebalanceProtocol);
+        }
+        return supportedProtocols;
+    }
+
     @Override
     public ByteBuffer subscriptionUserData(final Set topics) {
         // Adds the following information to subscription
@@ -211,7 +222,7 @@ public ByteBuffer subscriptionUserData(final Set topics) {
         // 2. Task ids of previously running tasks
         // 3. Task ids of valid local states on the client's state directory.
 
-        final Set previousActiveTasks = taskManager.prevActiveTaskIds();
+        final Set previousActiveTasks = taskManager.previousRunningTaskIds();
         final Set standbyTasks = taskManager.cachedTasksIds();
         standbyTasks.removeAll(previousActiveTasks);
         final SubscriptionInfo data = new SubscriptionInfo(
@@ -780,20 +791,22 @@ public void onAssignment(final Assignment assignment, final ConsumerGroupMetadat
         final Map topicToPartitionInfo = new HashMap<>();
         final Map> partitionsByHost;
 
+        final Map partitionsToTaskId = new HashMap<>();
+
         switch (receivedAssignmentMetadataVersion) {
             case VERSION_ONE:
-                processVersionOneAssignment(logPrefix, info, partitions, activeTasks);
+                processVersionOneAssignment(logPrefix, info, partitions, activeTasks, partitionsToTaskId);
                 partitionsByHost = Collections.emptyMap();
                 break;
             case VERSION_TWO:
-                processVersionTwoAssignment(logPrefix, info, partitions, activeTasks, topicToPartitionInfo);
+                processVersionTwoAssignment(logPrefix, info, partitions, activeTasks, topicToPartitionInfo, partitionsToTaskId);
                 partitionsByHost = info.partitionsByHost();
                 break;
             case VERSION_THREE:
             case VERSION_FOUR:
             case VERSION_FIVE:
                 upgradeSubscriptionVersionIfNeeded(leaderSupportedVersion);
-                processVersionTwoAssignment(logPrefix, info, partitions, activeTasks, topicToPartitionInfo);
+                processVersionTwoAssignment(logPrefix, info, partitions, activeTasks, topicToPartitionInfo, partitionsToTaskId);
                 partitionsByHost = info.partitionsByHost();
                 break;
             default:
@@ -805,6 +818,7 @@ public void onAssignment(final Assignment assignment, final ConsumerGroupMetadat
 
         taskManager.setClusterMetadata(Cluster.empty().withPartitions(topicToPartitionInfo));
         taskManager.setPartitionsByHostState(partitionsByHost);
+        taskManager.setPartitionsToTaskId(partitionsToTaskId);
         taskManager.setAssignmentMetadata(activeTasks, info.standbyTasks());
         taskManager.updateSubscriptionsFromAssignment(partitions);
     }
@@ -812,7 +826,8 @@ public void onAssignment(final Assignment assignment, final ConsumerGroupMetadat
     private static void processVersionOneAssignment(final String logPrefix,
                                                     final AssignmentInfo info,
                                                     final List partitions,
-                                                    final Map> activeTasks) {
+                                                    final Map> activeTasks,
+                                                    final Map partitionsToTaskId) {
         // the number of assigned partitions should be the same as number of active tasks, which
         // could be duplicated if one task has more than one assigned partitions
         if (partitions.size() != info.activeTasks().size()) {
@@ -830,15 +845,17 @@ private static void processVersionOneAssignment(final String logPrefix,
             final TopicPartition partition = partitions.get(i);
             final TaskId id = info.activeTasks().get(i);
             activeTasks.computeIfAbsent(id, k -> new HashSet<>()).add(partition);
+            partitionsToTaskId.put(partition, id);
         }
     }
 
     public static void processVersionTwoAssignment(final String logPrefix,
-                                            final AssignmentInfo info,
-                                            final List partitions,
-                                            final Map> activeTasks,
-                                            final Map topicToPartitionInfo) {
-        processVersionOneAssignment(logPrefix, info, partitions, activeTasks);
+                                                   final AssignmentInfo info,
+                                                   final List partitions,
+                                                   final Map> activeTasks,
+                                                   final Map topicToPartitionInfo,
+                                                   final Map partitionsToTaskId) {
+        processVersionOneAssignment(logPrefix, info, partitions, activeTasks, partitionsToTaskId);
 
         // process partitions by host
         final Map> partitionsByHost = info.partitionsByHost();
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsRebalanceListener.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsRebalanceListener.java
new file mode 100644
index 0000000000000..17563fd460e48
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsRebalanceListener.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.streams.processor.TaskId;
+import org.apache.kafka.streams.processor.internals.StreamThread.State;
+import org.apache.kafka.streams.processor.internals.assignment.AssignorError;
+import org.slf4j.Logger;
+
+public class StreamsRebalanceListener implements ConsumerRebalanceListener {
+
+    private final Time time;
+    private final TaskManager taskManager;
+    private final StreamThread streamThread;
+    private final Logger log;
+
+    StreamsRebalanceListener(final Time time,
+        final TaskManager taskManager,
+        final StreamThread streamThread,
+        final Logger log) {
+        this.time = time;
+        this.taskManager = taskManager;
+        this.streamThread = streamThread;
+        this.log = log;
+    }
+
+    @Override
+    public void onPartitionsAssigned(final Collection assignedPartitions) {
+        log.debug("Current state {}: assigned partitions {} at the end of consumer rebalance.\n" +
+                "\tpreviously assigned active tasks: {}\n" +
+                "\tpreviously assigned standby tasks: {}\n",
+            streamThread.state(),
+            assignedPartitions,
+            taskManager.previousActiveTaskIds(),
+            taskManager.previousStandbyTaskIds());
+
+        if (streamThread.getAssignmentErrorCode() == AssignorError.INCOMPLETE_SOURCE_TOPIC_METADATA.code()) {
+            log.error("Received error code {} - shutdown", streamThread.getAssignmentErrorCode());
+            streamThread.shutdown();
+            return;
+        }
+
+        final long start = time.milliseconds();
+        List revokedStandbyPartitions = null;
+
+        try {
+            if (streamThread.setState(State.PARTITIONS_ASSIGNED) == null) {
+                log.debug(
+                    "Skipping task creation in rebalance because we are already in {} state.",
+                    streamThread.state()
+                );
+            } else if (streamThread.getAssignmentErrorCode() != AssignorError.NONE.code()) {
+                log.debug(
+                    "Encountered assignment error during partition assignment: {}. Skipping task initialization",
+                    streamThread.getAssignmentErrorCode()
+                );
+            } else {
+                // Close non-reassigned tasks before initializing new ones as we may have suspended active
+                // tasks that become standbys or vice versa
+                revokedStandbyPartitions = taskManager.closeRevokedStandbyTasks();
+                taskManager.closeRevokedSuspendedTasks();
+                taskManager.createTasks(assignedPartitions);
+            }
+        } catch (final Throwable t) {
+            log.error(
+                "Error caught during partition assignment, " +
+                    "will abort the current process and re-throw at the end of rebalance", t);
+            streamThread.setRebalanceException(t);
+        } finally {
+            if (revokedStandbyPartitions != null) {
+                streamThread.clearStandbyRecords(revokedStandbyPartitions);
+            }
+            log.info("partition assignment took {} ms.\n" +
+                    "\tcurrently assigned active tasks: {}\n" +
+                    "\tcurrently assigned standby tasks: {}\n" +
+                    "\trevoked active tasks: {}\n" +
+                    "\trevoked standby tasks: {}\n",
+                time.milliseconds() - start,
+                taskManager.activeTaskIds(),
+                taskManager.standbyTaskIds(),
+                taskManager.revokedActiveTaskIds(),
+                taskManager.revokedStandbyTaskIds());
+        }
+    }
+
+    @Override
+    public void onPartitionsRevoked(final Collection revokedPartitions) {
+        log.debug("Current state {}: revoked partitions {} because of consumer rebalance.\n" +
+                "\tcurrently assigned active tasks: {}\n" +
+                "\tcurrently assigned standby tasks: {}\n",
+            streamThread.state(),
+            revokedPartitions,
+            taskManager.activeTaskIds(),
+            taskManager.standbyTaskIds());
+
+        Set suspendedTasks = new HashSet<>();
+        if (streamThread.setState(State.PARTITIONS_REVOKED) != null && !revokedPartitions.isEmpty()) {
+            final long start = time.milliseconds();
+            try {
+                // suspend only the active tasks, reassigned standby tasks will be closed in onPartitionsAssigned
+                suspendedTasks = taskManager.suspendActiveTasksAndState(revokedPartitions);
+            } catch (final Throwable t) {
+                log.error(
+                    "Error caught during partition revocation, " +
+                        "will abort the current process and re-throw at the end of rebalance: ",
+                    t
+                );
+                streamThread.setRebalanceException(t);
+            } finally {
+                log.info("partition revocation took {} ms.\n" +
+                        "\tcurrent suspended active tasks: {}\n",
+                    time.milliseconds() - start,
+                    suspendedTasks);
+            }
+        }
+    }
+
+    @Override
+    public void onPartitionsLost(final Collection lostPartitions) {
+        log.info("at state {}: partitions {} lost due to missed rebalance.\n" +
+                "\tlost active tasks: {}\n" +
+                "\tlost assigned standby tasks: {}\n",
+            streamThread.state(),
+            lostPartitions,
+            taskManager.activeTaskIds(),
+            taskManager.standbyTaskIds());
+
+        Set lostTasks = new HashSet<>();
+        final long start = time.milliseconds();
+        try {
+            // close lost active tasks but don't try to commit offsets as we no longer own them
+            lostTasks = taskManager.closeLostTasks(lostPartitions);
+        } catch (final Throwable t) {
+            log.error(
+                "Error caught during partitions lost, " +
+                    "will abort the current process and re-throw at the end of rebalance: ",
+                t
+            );
+            streamThread.setRebalanceException(t);
+        } finally {
+            log.info("partitions lost took {} ms.\n" +
+                    "\tsuspended lost active tasks: {}\n",
+                time.milliseconds() - start,
+                lostTasks);
+        }
+    }
+
+}
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
index 00509fcbd662f..0a7acf3336c84 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
+import java.util.ArrayList;
 import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.clients.admin.DeleteRecordsResult;
 import org.apache.kafka.clients.admin.RecordsToDelete;
@@ -60,10 +61,18 @@ public class TaskManager {
     private final Admin adminClient;
     private DeleteRecordsResult deleteRecordsResult;
 
+    // the restore consumer is only ever assigned changelogs from restoring tasks or standbys (but not both)
+    private boolean restoreConsumerAssignedStandbys = false;
+
     // following information is updated during rebalance phase by the partition assignor
     private Cluster cluster;
-    private Map> assignedActiveTasks;
-    private Map> assignedStandbyTasks;
+    private Map partitionsToTaskId = new HashMap<>();
+    private Map> assignedActiveTasks = new HashMap<>();
+    private Map> assignedStandbyTasks = new HashMap<>();
+    private Map> addedActiveTasks = new HashMap<>();
+    private Map> addedStandbyTasks = new HashMap<>();
+    private Map> revokedActiveTasks = new HashMap<>();
+    private Map> revokedStandbyTasks = new HashMap<>();
 
     private Consumer consumer;
 
@@ -103,78 +112,54 @@ void createTasks(final Collection assignment) {
             throw new IllegalStateException(logPrefix + "consumer has not been initialized while adding stream tasks. This should not happen.");
         }
 
-        // do this first as we may have suspended standby tasks that
-        // will become active or vice versa
-        standby.closeNonAssignedSuspendedTasks(assignedStandbyTasks);
-        active.closeNonAssignedSuspendedTasks(assignedActiveTasks);
+        if (!assignment.isEmpty() && !assignedActiveTasks.isEmpty()) {
+            resumeSuspended(assignment);
+        }
+        if (!addedActiveTasks.isEmpty()) {
+            addNewActiveTasks(addedActiveTasks);
+        }
+        if (!addedStandbyTasks.isEmpty()) {
+            addNewStandbyTasks(addedStandbyTasks);
+        }
+
+        // need to clear restore consumer if it was reading standbys but we have active tasks that may need restoring
+        if (!addedActiveTasks.isEmpty() && restoreConsumerAssignedStandbys) {
+            restoreConsumer.unsubscribe();
+            restoreConsumerAssignedStandbys = false;
+        }
 
-        addStreamTasks(assignment);
-        addStandbyTasks();
-        // Pause all the partitions until the underlying state store is ready for all the active tasks.
+        // Pause all the new partitions until the underlying state store is ready for all the active tasks.
         log.trace("Pausing partitions: {}", assignment);
         consumer.pause(assignment);
     }
 
-    private void addStreamTasks(final Collection assignment) {
-        if (assignedActiveTasks == null || assignedActiveTasks.isEmpty()) {
-            return;
-        }
-        final Map> newTasks = new HashMap<>();
-        // collect newly assigned tasks and reopen re-assigned tasks
-        log.debug("Adding assigned tasks as active: {}", assignedActiveTasks);
-        for (final Map.Entry> entry : assignedActiveTasks.entrySet()) {
-            final TaskId taskId = entry.getKey();
-            final Set partitions = entry.getValue();
+    private void resumeSuspended(final Collection assignment) {
+        final Set suspendedTasks = partitionsToTaskSet(assignment);
+        suspendedTasks.removeAll(addedActiveTasks.keySet());
 
-            if (assignment.containsAll(partitions)) {
-                try {
-                    if (!active.maybeResumeSuspendedTask(taskId, partitions)) {
-                        newTasks.put(taskId, partitions);
-                    }
-                } catch (final StreamsException e) {
-                    log.error("Failed to resume an active task {} due to the following error:", taskId, e);
-                    throw e;
+        for (final TaskId taskId : suspendedTasks) {
+            final Set partitions = assignedActiveTasks.get(taskId);
+            try {
+                if (!active.maybeResumeSuspendedTask(taskId, partitions)) {
+                    // recreate if resuming the suspended task failed because the associated partitions changed
+                    addedActiveTasks.put(taskId, partitions);
                 }
-            } else {
-                log.warn("Task {} owned partitions {} are not contained in the assignment {}", taskId, partitions, assignment);
+            } catch (final StreamsException e) {
+                log.error("Failed to resume an active task {} due to the following error:", taskId, e);
+                throw e;
             }
         }
+    }
 
-        if (newTasks.isEmpty()) {
-            return;
-        }
-
-        // CANNOT FIND RETRY AND BACKOFF LOGIC
-        // create all newly assigned tasks (guard against race condition with other thread via backoff and retry)
-        // -> other thread will call removeSuspendedTasks(); eventually
-        log.debug("New active tasks to be created: {}", newTasks);
+    private void addNewActiveTasks(final Map> newActiveTasks) {
+        log.debug("New active tasks to be created: {}", newActiveTasks);
 
-        for (final StreamTask task : taskCreator.createTasks(consumer, newTasks)) {
+        for (final StreamTask task : taskCreator.createTasks(consumer, newActiveTasks)) {
             active.addNewTask(task);
         }
     }
 
-    private void addStandbyTasks() {
-        if (assignedStandbyTasks == null || assignedStandbyTasks.isEmpty()) {
-            return;
-        }
-        log.debug("Adding assigned standby tasks {}", assignedStandbyTasks);
-        final Map> newStandbyTasks = new HashMap<>();
-        // collect newly assigned standby tasks and reopen re-assigned standby tasks
-        for (final Map.Entry> entry : assignedStandbyTasks.entrySet()) {
-            final TaskId taskId = entry.getKey();
-            final Set partitions = entry.getValue();
-            if (!standby.maybeResumeSuspendedTask(taskId, partitions)) {
-                newStandbyTasks.put(taskId, partitions);
-            }
-        }
-
-        if (newStandbyTasks.isEmpty()) {
-            return;
-        }
-
-        // create all newly assigned standby tasks (guard against race condition with other thread via backoff and retry)
-        // -> other thread will call removeSuspendedStandbyTasks(); eventually
+    private void addNewStandbyTasks(final Map> newStandbyTasks) {
         log.trace("New standby tasks to be created: {}", newStandbyTasks);
 
         for (final StandbyTask task : standbyTaskCreator.createTasks(consumer, newStandbyTasks)) {
@@ -182,18 +167,6 @@ private void addStandbyTasks() {
         }
     }
 
-    Set activeTaskIds() {
-        return active.allAssignedTaskIds();
-    }
-
-    Set standbyTaskIds() {
-        return standby.allAssignedTaskIds();
-    }
-
-    public Set prevActiveTaskIds() {
-        return active.previousTaskIds();
-    }
-
     /**
      * Returns ids of tasks whose states are kept on the local storage.
      */
@@ -224,47 +197,90 @@ public Set cachedTasksIds() {
         return tasks;
     }
 
-    public UUID processId() {
-        return processId;
+    /**
+     * Closes standby tasks that were not reassigned at the end of a rebalance.
+     *
+     * @return list of changelog topic partitions from revoked tasks
+     * @throws TaskMigratedException if the task producer got fenced (EOS only)
+     */
+    List closeRevokedStandbyTasks() {
+        final List revokedChangelogs = standby.closeRevokedStandbyTasks(revokedStandbyTasks);
+
+        // If the restore consumer is assigned any standby partitions they must be removed
+        removeChangelogsFromRestoreConsumer(revokedChangelogs, true);
+
+        return revokedChangelogs;
     }
 
-    InternalTopologyBuilder builder() {
-        return taskCreator.builder();
+    /**
+     * Closes suspended active tasks that were not reassigned at the end of a rebalance.
+     *
+     * @throws TaskMigratedException if the task producer got fenced (EOS only)
+     */
+    void closeRevokedSuspendedTasks() {
+        // changelogs should have already been removed during suspend
+        final RuntimeException exception = active.closeNotAssignedSuspendedTasks(revokedActiveTasks.keySet());
+
+        // At this point all revoked tasks should have been closed, we can just throw the exception
+        if (exception != null) {
+            throw exception;
+        }
     }
 
     /**
      * Similar to shutdownTasksAndState, however does not close the task managers, in the hope that
-     * soon the tasks will be assigned again
+     * soon the tasks will be assigned again.
+     * @return list of suspended tasks
      * @throws TaskMigratedException if the task producer got fenced (EOS only)
      */
-    void suspendTasksAndState()  {
-        log.debug("Suspending all active tasks {} and standby tasks {}", active.runningTaskIds(), standby.runningTaskIds());
-
+    Set suspendActiveTasksAndState(final Collection revokedPartitions)  {
         final AtomicReference firstException = new AtomicReference<>(null);
+        final List revokedChangelogs = new ArrayList<>();
 
-        firstException.compareAndSet(null, active.suspend());
-        // close all restoring tasks as well and then reset changelog reader;
-        // for those restoring and still assigned tasks, they will be re-created
-        // in addStreamTasks.
-        firstException.compareAndSet(null, active.closeAllRestoringTasks());
-        changelogReader.reset();
+        final Set revokedTasks = partitionsToTaskSet(revokedPartitions);
 
-        firstException.compareAndSet(null, standby.suspend());
+        firstException.compareAndSet(null, active.suspendOrCloseTasks(revokedTasks, revokedChangelogs));
 
-        // remove the changelog partitions from restore consumer
-        restoreConsumer.unsubscribe();
+        changelogReader.remove(revokedChangelogs);
+        removeChangelogsFromRestoreConsumer(revokedChangelogs, false);
 
         final Exception exception = firstException.get();
         if (exception != null) {
             throw new StreamsException(logPrefix + "failed to suspend stream tasks", exception);
         }
+        return active.suspendedTaskIds();
+    }
+
+    /**
+     * Closes active tasks as zombies, as these partitions have been lost and are no longer owned.
+     * @return list of lost tasks
+     */
+    Set closeLostTasks(final Collection lostPartitions) {
+        final Set zombieTasks = partitionsToTaskSet(lostPartitions);
+        log.debug("Closing lost tasks as zombies: {}", zombieTasks);
+
+        final List lostTaskChangelogs = new ArrayList<>();
+
+        final RuntimeException exception = active.closeZombieTasks(zombieTasks, lostTaskChangelogs);
+
+        assignedActiveTasks.keySet().removeAll(zombieTasks);
+        changelogReader.remove(lostTaskChangelogs);
+        removeChangelogsFromRestoreConsumer(lostTaskChangelogs, false);
+
+        if (exception != null) {
+            throw exception;
+        } else if (!assignedActiveTasks.isEmpty()) {
+            throw new IllegalStateException("TaskManager had leftover tasks after removing all zombies");
+        }
+
+        return zombieTasks;
     }
 
     void shutdown(final boolean clean) {
         final AtomicReference firstException = new AtomicReference<>(null);
 
-        log.debug("Shutting down all active tasks {}, standby tasks {}, suspended tasks {}, and suspended standby tasks {}", active.runningTaskIds(), standby.runningTaskIds(),
-                  active.previousTaskIds(), standby.previousTaskIds());
+        log.debug("Shutting down all active tasks {}, standby tasks {}, and suspended tasks {}", active.runningTaskIds(), standby.runningTaskIds(),
+                  active.suspendedTaskIds());
 
         try {
             active.close(clean);
@@ -288,16 +304,38 @@ void shutdown(final boolean clean) {
         }
     }
 
-    Admin getAdminClient() {
-        return adminClient;
+    Set activeTaskIds() {
+        return active.allAssignedTaskIds();
     }
 
-    Set suspendedActiveTaskIds() {
-        return active.previousTaskIds();
+    Set standbyTaskIds() {
+        return standby.allAssignedTaskIds();
     }
 
-    Set suspendedStandbyTaskIds() {
-        return standby.previousTaskIds();
+    Set revokedActiveTaskIds() {
+        return revokedActiveTasks.keySet();
+    }
+
+    Set revokedStandbyTaskIds() {
+        return revokedStandbyTasks.keySet();
+    }
+
+    public Set previousRunningTaskIds() {
+        return active.previousRunningTaskIds();
+    }
+
+    Set previousActiveTaskIds() {
+        final HashSet previousActiveTasks = new HashSet<>(assignedActiveTasks.keySet());
+        previousActiveTasks.addAll(revokedActiveTasks.keySet());
+        previousActiveTasks.removeAll(addedActiveTasks.keySet());
+        return previousActiveTasks;
+    }
+
+    Set previousStandbyTaskIds() {
+        final HashSet previousStandbyTasks = new HashSet<>(assignedStandbyTasks.keySet());
+        previousStandbyTasks.addAll(revokedStandbyTasks.keySet());
+        previousStandbyTasks.removeAll(addedStandbyTasks.keySet());
+        return previousStandbyTasks;
     }
 
     StreamTask activeTask(final TopicPartition partition) {
@@ -320,6 +358,14 @@ void setConsumer(final Consumer consumer) {
         this.consumer = consumer;
     }
 
+    public UUID processId() {
+        return processId;
+    }
+
+    InternalTopologyBuilder builder() {
+        return taskCreator.builder();
+    }
+
     /**
      * @throws IllegalStateException If store gets registered after initialized is already finished
      * @throws StreamsException if the store's change log does not contain the partition
@@ -329,8 +375,8 @@ boolean updateNewAndRestoringTasks() {
         standby.initializeNewTasks();
 
         final Collection restored = changelogReader.restore(active);
-
         active.updateRestored(restored);
+        removeChangelogsFromRestoreConsumer(restored, false);
 
         if (active.allTasksRunning()) {
             final Set assignment = consumer.assignment();
@@ -357,6 +403,7 @@ private void assignStandbyPartitions() {
             checkpointedOffsets.putAll(standbyTask.checkpointedOffsets());
         }
 
+        restoreConsumerAssignedStandbys = true;
         restoreConsumer.assign(checkpointedOffsets.keySet());
         for (final Map.Entry entry : checkpointedOffsets.entrySet()) {
             final TopicPartition partition = entry.getKey();
@@ -377,8 +424,40 @@ public void setPartitionsByHostState(final Map> pa
         this.streamsMetadataState.onChange(partitionsByHostState, cluster);
     }
 
+    public void setPartitionsToTaskId(final Map partitionsToTaskId) {
+        this.partitionsToTaskId = partitionsToTaskId;
+    }
+
     public void setAssignmentMetadata(final Map> activeTasks,
                                       final Map> standbyTasks) {
+        addedActiveTasks.clear();
+        for (final Map.Entry> entry : activeTasks.entrySet()) {
+            if (!assignedActiveTasks.containsKey(entry.getKey())) {
+                addedActiveTasks.put(entry.getKey(), entry.getValue());
+            }
+        }
+
+        addedStandbyTasks.clear();
+        for (final Map.Entry> entry : standbyTasks.entrySet()) {
+            if (!assignedStandbyTasks.containsKey(entry.getKey())) {
+                addedStandbyTasks.put(entry.getKey(), entry.getValue());
+            }
+        }
+
+        revokedActiveTasks.clear();
+        for (final Map.Entry> entry : assignedActiveTasks.entrySet()) {
+            if (!activeTasks.containsKey(entry.getKey())) {
+                revokedActiveTasks.put(entry.getKey(), entry.getValue());
+            }
+        }
+
+        revokedStandbyTasks.clear();
+        for (final Map.Entry> entry : assignedStandbyTasks.entrySet()) {
+            if (!standbyTasks.containsKey(entry.getKey())) {
+                revokedStandbyTasks.put(entry.getKey(), entry.getValue());
+            }
+        }
+
         this.assignedActiveTasks = activeTasks;
         this.assignedStandbyTasks = standbyTasks;
     }
@@ -482,6 +561,30 @@ public String toString(final String indent) {
         return builder.toString();
     }
 
+    // this should be safe to call whether the restore consumer is assigned standby or active restoring partitions
+    // as the removal will be a no-op
+    private void removeChangelogsFromRestoreConsumer(final Collection changelogs, final boolean areStandbyPartitions) {
+        if (!changelogs.isEmpty() && areStandbyPartitions == restoreConsumerAssignedStandbys) {
+            final Set updatedAssignment = new HashSet<>(restoreConsumer.assignment());
+            updatedAssignment.removeAll(changelogs);
+            restoreConsumer.assign(updatedAssignment);
+        }
+    }
+
+    private Set partitionsToTaskSet(final Collection partitions) {
+        final Set taskIds = new HashSet<>();
+        for (final TopicPartition tp : partitions) {
+            final TaskId id = partitionsToTaskId.get(tp);
+            if (id != null) {
+                taskIds.add(id);
+            } else {
+                log.error("Failed to lookup taskId for partition {}", tp);
+                throw new StreamsException("Found partition in assignment with no corresponding task");
+            }
+        }
+        return taskIds;
+    }
+
     // the following functions are for testing only
     Map> assignedActiveTasks() {
         return assignedActiveTasks;
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorConfiguration.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorConfiguration.java
index 5c668cbd8f768..080c6145f663a 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorConfiguration.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorConfiguration.java
@@ -17,6 +17,7 @@
 package org.apache.kafka.streams.processor.internals.assignment;
 
 import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.RebalanceProtocol;
 import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.config.ConfigException;
 import org.apache.kafka.common.utils.LogContext;
@@ -131,6 +132,29 @@ public TaskManager getTaskManager() {
         return taskManager;
     }
 
+    public RebalanceProtocol rebalanceProtocol() {
+        final String upgradeFrom = streamsConfig.getString(StreamsConfig.UPGRADE_FROM_CONFIG);
+        if (upgradeFrom != null) {
+            switch (upgradeFrom) {
+                case StreamsConfig.UPGRADE_FROM_0100:
+                case StreamsConfig.UPGRADE_FROM_0101:
+                case StreamsConfig.UPGRADE_FROM_0102:
+                case StreamsConfig.UPGRADE_FROM_0110:
+                case StreamsConfig.UPGRADE_FROM_10:
+                case StreamsConfig.UPGRADE_FROM_11:
+                case StreamsConfig.UPGRADE_FROM_20:
+                case StreamsConfig.UPGRADE_FROM_21:
+                case StreamsConfig.UPGRADE_FROM_22:
+                case StreamsConfig.UPGRADE_FROM_23:
+                    log.info("Turning off cooperative rebalancing for upgrade from {}.x", upgradeFrom);
+                    return RebalanceProtocol.EAGER;
+                default:
+                    throw new IllegalArgumentException("Unknown configuration value for parameter 'upgrade.from': " + upgradeFrom);
+            }
+        }
+        return RebalanceProtocol.EAGER;
+    }
+
     public String logPrefix() {
         return logPrefix;
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/RegexSourceIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/RegexSourceIntegrationTest.java
index 8c00ee4f4829f..6cecb5691897a 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/RegexSourceIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/RegexSourceIntegrationTest.java
@@ -413,7 +413,9 @@ private static class TheConsumerRebalanceListener implements ConsumerRebalanceLi
 
         @Override
         public void onPartitionsRevoked(final Collection partitions) {
-            assignedTopics.clear();
+            for (final TopicPartition partition : partitions) {
+                assignedTopics.remove(partition.topic());
+            }
             listener.onPartitionsRevoked(partitions);
         }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasksTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasksTest.java
index 0aaeef9fde178..94808b8043a75 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasksTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasksTest.java
@@ -26,8 +26,10 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 import java.util.Set;
 import org.apache.kafka.clients.consumer.MockConsumer;
 import org.apache.kafka.clients.consumer.OffsetResetStrategy;
@@ -62,11 +64,15 @@ public class AssignedStreamsTasksTest {
     private final TaskId taskId2 = new TaskId(1, 0);
     private AssignedStreamsTasks assignedTasks;
 
+    private final List revokedChangelogs = new ArrayList<>();
+
     @Before
     public void before() {
         assignedTasks = new AssignedStreamsTasks(new LogContext("log "));
         EasyMock.expect(t1.id()).andReturn(taskId1).anyTimes();
         EasyMock.expect(t2.id()).andReturn(taskId2).anyTimes();
+
+        revokedChangelogs.clear();
     }
 
     @Test
@@ -155,33 +161,35 @@ public void shouldSuspendRunningTasks() {
 
         assertThat(suspendTask(), nullValue());
 
-        assertThat(assignedTasks.previousTaskIds(), equalTo(Collections.singleton(taskId1)));
+        assertThat(assignedTasks.suspendedTaskIds(), equalTo(Collections.singleton(taskId1)));
         EasyMock.verify(t1);
     }
 
     @Test
     public void shouldCloseRestoringTasks() {
         EasyMock.expect(t1.initializeStateStores()).andReturn(false);
-        EasyMock.expect(t1.partitions()).andReturn(Collections.singleton(tp1));
-        EasyMock.expect(t1.changelogPartitions()).andReturn(Collections.emptySet());
+        EasyMock.expect(t1.partitions()).andReturn(Collections.singleton(tp1)).times(2);
+        EasyMock.expect(t1.changelogPartitions()).andReturn(Collections.emptySet()).times(2);
         t1.closeStateManager(true);
         EasyMock.expectLastCall();
         EasyMock.replay(t1);
 
         addAndInitTask();
-        assertThat(assignedTasks.closeAllRestoringTasks(), nullValue());
+        assertThat(assignedTasks.closeRestoringTasks(assignedTasks.restoringTaskIds(), revokedChangelogs), nullValue());
 
         EasyMock.verify(t1);
     }
 
     @Test
     public void shouldClosedUnInitializedTasksOnSuspend() {
+        EasyMock.expect(t1.changelogPartitions()).andAnswer(Collections::emptyList);
+
         t1.close(false, false);
         EasyMock.expectLastCall();
         EasyMock.replay(t1);
 
         assignedTasks.addNewTask(t1);
-        assertThat(assignedTasks.suspend(), nullValue());
+        assertThat(assignedTasks.suspendOrCloseTasks(assignedTasks.allAssignedTaskIds(), revokedChangelogs), nullValue());
 
         EasyMock.verify(t1);
     }
@@ -192,7 +200,7 @@ public void shouldNotSuspendSuspendedTasks() {
         EasyMock.replay(t1);
 
         assertThat(suspendTask(), nullValue());
-        assertThat(assignedTasks.suspend(), nullValue());
+        assertThat(assignedTasks.suspendOrCloseTasks(assignedTasks.allAssignedTaskIds(), revokedChangelogs), nullValue());
         EasyMock.verify(t1);
     }
 
@@ -200,20 +208,28 @@ public void shouldNotSuspendSuspendedTasks() {
     @Test
     public void shouldCloseTaskOnSuspendWhenRuntimeException() {
         mockTaskInitialization();
+        EasyMock.expect(t1.partitions()).andReturn(Collections.emptySet()).anyTimes();
+        EasyMock.expect(t1.changelogPartitions()).andReturn(Collections.emptySet()).anyTimes();
+
         t1.suspend();
         EasyMock.expectLastCall().andThrow(new RuntimeException("KABOOM!"));
         t1.close(false, false);
         EasyMock.expectLastCall();
+
         EasyMock.replay(t1);
 
         assertThat(suspendTask(), not(nullValue()));
-        assertThat(assignedTasks.previousTaskIds(), equalTo(Collections.singleton(taskId1)));
+        assertTrue(assignedTasks.runningTaskIds().isEmpty());
+        assertTrue(assignedTasks.suspendedTaskIds().isEmpty());
         EasyMock.verify(t1);
     }
 
     @Test
     public void shouldCloseTaskOnSuspendIfTaskMigratedException() {
         mockTaskInitialization();
+        EasyMock.expect(t1.partitions()).andReturn(Collections.emptySet()).anyTimes();
+        EasyMock.expect(t1.changelogPartitions()).andReturn(Collections.emptySet()).anyTimes();
+
         t1.suspend();
         EasyMock.expectLastCall().andThrow(new TaskMigratedException());
         t1.close(false, true);
@@ -221,7 +237,7 @@ public void shouldCloseTaskOnSuspendIfTaskMigratedException() {
         EasyMock.replay(t1);
 
         assertThat(suspendTask(), nullValue());
-        assertTrue(assignedTasks.previousTaskIds().isEmpty());
+        assertTrue(assignedTasks.runningTaskIds().isEmpty());
         EasyMock.verify(t1);
     }
 
@@ -529,7 +545,7 @@ public void shouldCloseCleanlyWithSuspendedTaskAndEOS() {
 
         assignedTasks.addNewTask(task);
         assignedTasks.initializeNewTasks();
-        assertNull(assignedTasks.suspend());
+        assertNull(assignedTasks.suspendOrCloseTasks(assignedTasks.allAssignedTaskIds(), revokedChangelogs));
 
         assignedTasks.close(true);
     }
@@ -541,7 +557,7 @@ private void addAndInitTask() {
 
     private RuntimeException suspendTask() {
         addAndInitTask();
-        return assignedTasks.suspend();
+        return assignedTasks.suspendOrCloseTasks(assignedTasks.allAssignedTaskIds(), revokedChangelogs);
     }
 
     private void mockRunningTaskSuspension() {
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/MockChangelogReader.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/MockChangelogReader.java
index 13309675fd6d3..9cc51cedb3046 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/MockChangelogReader.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/MockChangelogReader.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
+import java.util.List;
 import org.apache.kafka.common.TopicPartition;
 
 import java.util.Collection;
@@ -48,10 +49,17 @@ void setRestoredOffsets(final Map restoredOffsets) {
     }
 
     @Override
-    public void reset() {
+    public void clear() {
         registered.clear();
     }
 
+    @Override
+    public void remove(final List revokedPartitions) {
+        for (final TopicPartition partition : revokedPartitions) {
+            restoredOffsets.remove(partition);
+        }
+    }
+
     public boolean wasRegistered(final TopicPartition partition) {
         return registered.contains(partition);
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
index 2f293819bc533..3339080a90f35 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
@@ -384,12 +384,14 @@ public void shouldRespectNumIterationsInMainLoop() {
         thread.setState(StreamThread.State.STARTING);
         thread.setState(StreamThread.State.PARTITIONS_REVOKED);
 
+        final TaskId task1 = new TaskId(0, t1p1.partition());
         final Set assignedPartitions = Collections.singleton(t1p1);
         thread.taskManager().setAssignmentMetadata(
             Collections.singletonMap(
-                new TaskId(0, t1p1.partition()),
+                task1,
                 assignedPartitions),
             Collections.emptyMap());
+        thread.taskManager().setPartitionsToTaskId(Collections.singletonMap(t1p1, task1));
 
         final MockConsumer mockConsumer = (MockConsumer) thread.consumer;
         mockConsumer.assign(Collections.singleton(t1p1));
@@ -584,13 +586,17 @@ public void shouldInjectProducerPerTaskUsingClientSupplierOnCreateIfEosEnable()
 
         final Map> activeTasks = new HashMap<>();
         final List assignedPartitions = new ArrayList<>();
+        final Map partitionsToTaskId = new HashMap<>();
 
         // assign single partition
         assignedPartitions.add(t1p1);
         assignedPartitions.add(t1p2);
         activeTasks.put(task1, Collections.singleton(t1p1));
         activeTasks.put(task2, Collections.singleton(t1p2));
+        partitionsToTaskId.put(t1p1, task1);
+        partitionsToTaskId.put(t1p2, task2);
 
+        thread.taskManager().setPartitionsToTaskId(partitionsToTaskId);
         thread.taskManager().setAssignmentMetadata(activeTasks, Collections.emptyMap());
 
         final MockConsumer mockConsumer = (MockConsumer) thread.consumer;
@@ -720,6 +726,7 @@ public void shouldNotThrowWithoutPendingShutdownInRunOnce() {
 
     private void mockRunOnce(final boolean shutdownOnPoll) {
         final Collection assignedPartitions = Collections.singletonList(t1p1);
+        final Map partitionsToTaskId = Collections.singletonMap(t1p1, new TaskId(0, 1));
         class MockStreamThreadConsumer extends MockConsumer {
 
             private StreamThread streamThread;
@@ -734,6 +741,7 @@ public synchronized ConsumerRecords poll(final Duration timeout) {
                 if (shutdownOnPoll) {
                     streamThread.shutdown();
                 }
+                streamThread.taskManager().setPartitionsToTaskId(partitionsToTaskId);
                 streamThread.rebalanceListener.onPartitionsAssigned(assignedPartitions);
                 return super.poll(timeout);
             }
@@ -758,6 +766,7 @@ private void setStreamThread(final StreamThread streamThread) {
                                                         new AssignedStandbyTasks(new LogContext()));
         taskManager.setConsumer(mockStreamThreadConsumer);
         taskManager.setAssignmentMetadata(Collections.emptyMap(), Collections.emptyMap());
+        taskManager.setPartitionsToTaskId(Collections.emptyMap());
 
         final StreamsMetricsImpl streamsMetrics =
             new StreamsMetricsImpl(metrics, clientId, StreamsConfig.METRICS_LATEST);
@@ -849,15 +858,18 @@ public void shouldCloseTaskAsZombieAndRemoveFromActiveTasksIfProducerWasFencedWh
         consumer.updatePartitions(topic1, singletonList(new PartitionInfo(topic1, 1, null, null, null)));
 
         thread.setState(StreamThread.State.STARTING);
-        thread.rebalanceListener.onPartitionsRevoked(null);
+        thread.rebalanceListener.onPartitionsRevoked(Collections.emptySet());
 
         final Map> activeTasks = new HashMap<>();
+        final Map partitionsToTaskId = new HashMap<>();
         final List assignedPartitions = new ArrayList<>();
 
         // assign single partition
         assignedPartitions.add(t1p1);
         activeTasks.put(task1, Collections.singleton(t1p1));
+        partitionsToTaskId.put(t1p1, task1);
 
+        thread.taskManager().setPartitionsToTaskId(partitionsToTaskId);
         thread.taskManager().setAssignmentMetadata(activeTasks, Collections.emptyMap());
 
         final MockConsumer mockConsumer = (MockConsumer) thread.consumer;
@@ -907,15 +919,18 @@ public void shouldCloseTaskAsZombieAndRemoveFromActiveTasksIfProducerGotFencedIn
         internalTopologyBuilder.addSink("out", "output", null, null, null, "name");
 
         thread.setState(StreamThread.State.STARTING);
-        thread.rebalanceListener.onPartitionsRevoked(null);
+        thread.rebalanceListener.onPartitionsRevoked(Collections.emptySet());
 
         final Map> activeTasks = new HashMap<>();
+        final Map partitionsToTaskId = new HashMap<>();
         final List assignedPartitions = new ArrayList<>();
 
         // assign single partition
         assignedPartitions.add(t1p1);
         activeTasks.put(task1, Collections.singleton(t1p1));
+        partitionsToTaskId.put(t1p1, task1);
 
+        thread.taskManager().setPartitionsToTaskId(partitionsToTaskId);
         thread.taskManager().setAssignmentMetadata(activeTasks, Collections.emptyMap());
 
         final MockConsumer mockConsumer = (MockConsumer) thread.consumer;
@@ -928,7 +943,7 @@ public void shouldCloseTaskAsZombieAndRemoveFromActiveTasksIfProducerGotFencedIn
         assertThat(thread.tasks().size(), equalTo(1));
 
         clientSupplier.producers.get(0).fenceProducer();
-        thread.rebalanceListener.onPartitionsRevoked(null);
+        thread.rebalanceListener.onPartitionsRevoked(assignedPartitions);
         assertTrue(clientSupplier.producers.get(0).transactionInFlight());
         assertFalse(clientSupplier.producers.get(0).transactionCommitted());
         assertTrue(clientSupplier.producers.get(0).closed());
@@ -943,15 +958,18 @@ public void shouldCloseTaskAsZombieAndRemoveFromActiveTasksIfProducerGotFencedIn
         internalTopologyBuilder.addSink("out", "output", null, null, null, "name");
 
         thread.setState(StreamThread.State.STARTING);
-        thread.rebalanceListener.onPartitionsRevoked(null);
+        thread.rebalanceListener.onPartitionsRevoked(Collections.emptySet());
 
         final Map> activeTasks = new HashMap<>();
+        final Map partitionsToTaskId = new HashMap<>();
         final List assignedPartitions = new ArrayList<>();
 
         // assign single partition
         assignedPartitions.add(t1p1);
         activeTasks.put(task1, Collections.singleton(t1p1));
+        partitionsToTaskId.put(t1p1, task1);
 
+        thread.taskManager().setPartitionsToTaskId(partitionsToTaskId);
         thread.taskManager().setAssignmentMetadata(activeTasks, Collections.emptyMap());
 
         final MockConsumer mockConsumer = (MockConsumer) thread.consumer;
@@ -964,7 +982,7 @@ public void shouldCloseTaskAsZombieAndRemoveFromActiveTasksIfProducerGotFencedIn
         assertThat(thread.tasks().size(), equalTo(1));
 
         clientSupplier.producers.get(0).fenceProducerOnClose();
-        thread.rebalanceListener.onPartitionsRevoked(null);
+        thread.rebalanceListener.onPartitionsRevoked(assignedPartitions);
 
         assertFalse(clientSupplier.producers.get(0).transactionInFlight());
         assertTrue(clientSupplier.producers.get(0).transactionCommitted());
@@ -999,15 +1017,18 @@ public void shouldReturnActiveTaskMetadataWhileRunningState() {
         final StreamThread thread = createStreamThread(clientId, config, false);
 
         thread.setState(StreamThread.State.STARTING);
-        thread.rebalanceListener.onPartitionsRevoked(null);
+        thread.rebalanceListener.onPartitionsRevoked(Collections.emptySet());
 
         final Map> activeTasks = new HashMap<>();
         final List assignedPartitions = new ArrayList<>();
+        final Map partitionsToTaskId = new HashMap<>();
 
         // assign single partition
         assignedPartitions.add(t1p1);
         activeTasks.put(task1, Collections.singleton(t1p1));
+        partitionsToTaskId.put(t1p1, task1);
 
+        thread.taskManager().setPartitionsToTaskId(partitionsToTaskId);
         thread.taskManager().setAssignmentMetadata(activeTasks, Collections.emptyMap());
 
         final MockConsumer mockConsumer = (MockConsumer) thread.consumer;
@@ -1048,13 +1069,16 @@ public void shouldReturnStandbyTaskMetadataWhileRunningState() {
         restoreConsumer.updateBeginningOffsets(offsets);
 
         thread.setState(StreamThread.State.STARTING);
-        thread.rebalanceListener.onPartitionsRevoked(null);
+        thread.rebalanceListener.onPartitionsRevoked(Collections.emptySet());
 
         final Map> standbyTasks = new HashMap<>();
+        final Map partitionsToTaskId = new HashMap<>();
 
         // assign single partition
         standbyTasks.put(task1, Collections.singleton(t1p1));
+        partitionsToTaskId.put(t1p1, task1);
 
+        thread.taskManager().setPartitionsToTaskId(partitionsToTaskId);
         thread.taskManager().setAssignmentMetadata(Collections.emptyMap(), standbyTasks);
 
         thread.rebalanceListener.onPartitionsAssigned(Collections.emptyList());
@@ -1125,14 +1149,17 @@ public void shouldUpdateStandbyTask() throws Exception {
         }
 
         thread.setState(StreamThread.State.STARTING);
-        thread.rebalanceListener.onPartitionsRevoked(null);
+        thread.rebalanceListener.onPartitionsRevoked(Collections.emptySet());
 
         final Map> standbyTasks = new HashMap<>();
+        final Map partitionsToTaskId = new HashMap<>();
 
         // assign single partition
         standbyTasks.put(task1, Collections.singleton(t1p1));
         standbyTasks.put(task3, Collections.singleton(t2p1));
+        partitionsToTaskId.put(t1p1, task1);
 
+        thread.taskManager().setPartitionsToTaskId(partitionsToTaskId);
         thread.taskManager().setAssignmentMetadata(Collections.emptyMap(), standbyTasks);
 
         thread.rebalanceListener.onPartitionsAssigned(Collections.emptyList());
@@ -1230,15 +1257,18 @@ public void close() {}
         final StreamThread thread = createStreamThread(clientId, config, false);
 
         thread.setState(StreamThread.State.STARTING);
-        thread.rebalanceListener.onPartitionsRevoked(null);
+        thread.rebalanceListener.onPartitionsRevoked(Collections.emptySet());
         final List assignedPartitions = new ArrayList<>();
 
         final Map> activeTasks = new HashMap<>();
+        final Map partitionsToTaskId = new HashMap<>();
 
         // assign single partition
         assignedPartitions.add(t1p1);
         activeTasks.put(task1, Collections.singleton(t1p1));
+        partitionsToTaskId.put(t1p1, task1);
 
+        thread.taskManager().setPartitionsToTaskId(partitionsToTaskId);
         thread.taskManager().setAssignmentMetadata(activeTasks, Collections.emptyMap());
 
         clientSupplier.consumer.assign(assignedPartitions);
@@ -1357,7 +1387,9 @@ public void shouldRecoverFromInvalidOffsetExceptionOnRestoreAndFinishRestore() t
         final Set topicPartitionSet = Collections.singleton(topicPartition);
 
         final Map> activeTasks = new HashMap<>();
-        activeTasks.put(new TaskId(0, 0), topicPartitionSet);
+        final TaskId task0 = new TaskId(0, 0);
+        activeTasks.put(task0, topicPartitionSet);
+        thread.taskManager().setPartitionsToTaskId(Collections.singletonMap(topicPartition, task0));
         thread.taskManager().setAssignmentMetadata(activeTasks, Collections.emptyMap());
 
         mockConsumer.updatePartitions(
@@ -1463,11 +1495,11 @@ public void shouldRecordSkippedMetricForDeserializationException() {
         thread.setState(StreamThread.State.STARTING);
         thread.setState(StreamThread.State.PARTITIONS_REVOKED);
 
+        final TaskId task1 = new TaskId(0, t1p1.partition());
         final Set assignedPartitions = Collections.singleton(t1p1);
+        thread.taskManager().setPartitionsToTaskId(Collections.singletonMap(t1p1, task1));
         thread.taskManager().setAssignmentMetadata(
-            Collections.singletonMap(
-                new TaskId(0, t1p1.partition()),
-                assignedPartitions),
+            Collections.singletonMap(task1, assignedPartitions),
             Collections.emptyMap());
 
         final MockConsumer mockConsumer = (MockConsumer) thread.consumer;
@@ -1534,12 +1566,14 @@ public void shouldReportSkippedRecordsForInvalidTimestamps() {
         thread.setState(StreamThread.State.STARTING);
         thread.setState(StreamThread.State.PARTITIONS_REVOKED);
 
+        final TaskId task1 = new TaskId(0, t1p1.partition());
         final Set assignedPartitions = Collections.singleton(t1p1);
         thread.taskManager().setAssignmentMetadata(
             Collections.singletonMap(
-                new TaskId(0, t1p1.partition()),
+                task1,
                 assignedPartitions),
             Collections.emptyMap());
+        thread.taskManager().setPartitionsToTaskId(Collections.singletonMap(t1p1, task1));
 
         final MockConsumer mockConsumer = (MockConsumer) thread.consumer;
         mockConsumer.assign(Collections.singleton(t1p1));
@@ -1690,7 +1724,7 @@ public void adminClientMetricsVerification() {
             null,
             new MockTime());
 
-        EasyMock.expect(taskManager.getAdminClient()).andReturn(adminClient);
+        EasyMock.expect(taskManager.adminClient()).andReturn(adminClient);
         EasyMock.expectLastCall();
         EasyMock.replay(taskManager, consumer);
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
index 2886f51fb5d6c..2bc465b50099f 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
@@ -152,7 +152,7 @@ private void createMockTaskManager(final Set prevTasks,
         taskManager = EasyMock.createNiceMock(TaskManager.class);
         EasyMock.expect(taskManager.adminClient()).andReturn(null).anyTimes();
         EasyMock.expect(taskManager.builder()).andReturn(builder).anyTimes();
-        EasyMock.expect(taskManager.prevActiveTaskIds()).andReturn(prevTasks).anyTimes();
+        EasyMock.expect(taskManager.previousRunningTaskIds()).andReturn(prevTasks).anyTimes();
         EasyMock.expect(taskManager.cachedTasksIds()).andReturn(cachedTasks).anyTimes();
         EasyMock.expect(taskManager.processId()).andReturn(processId).anyTimes();
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java
index 77e0254b9f058..020ee363c05fc 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java
@@ -17,6 +17,8 @@
 
 package org.apache.kafka.streams.processor.internals;
 
+import java.util.ArrayList;
+import java.util.List;
 import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.clients.admin.DeleteRecordsResult;
 import org.apache.kafka.clients.admin.DeletedRecords;
@@ -67,6 +69,7 @@ public class TaskManagerTest {
     private final TopicPartition t1p0 = new TopicPartition("t1", 0);
     private final Set taskId0Partitions = Utils.mkSet(t1p0);
     private final Map> taskId0Assignment = Collections.singletonMap(taskId0, taskId0Partitions);
+    private final Map taskId0PartitionToTaskId = Collections.singletonMap(t1p0, taskId0);
 
     @Mock(type = MockType.STRICT)
     private InternalTopologyBuilder.SubscriptionUpdates subscriptionUpdates;
@@ -113,6 +116,10 @@ public class TaskManagerTest {
     private final TaskId task03 = new TaskId(0, 3);
     private final TaskId task11 = new TaskId(1, 1);
 
+    private final Set revokedTasks = new HashSet<>();
+    private final List revokedPartitions = new ArrayList<>();
+    private final List revokedChangelogs = Collections.emptyList();
+
     @Rule
     public final TemporaryFolder testFolder = new TemporaryFolder();
 
@@ -129,6 +136,7 @@ public void setUp() {
                                       active,
                                       standby);
         taskManager.setConsumer(consumer);
+        revokedChangelogs.clear();
     }
 
     private void replay() {
@@ -235,81 +243,77 @@ public void shouldReturnCachedTaskIdsFromDirectory() throws IOException {
     }
 
     @Test
-    public void shouldCloseActiveUnAssignedSuspendedTasksWhenCreatingNewTasks() {
+    public void shouldCloseActiveUnAssignedSuspendedTasksWhenClosingRevokedTasks() {
         mockSingleActiveTask();
-        active.closeNonAssignedSuspendedTasks(taskId0Assignment);
-        expectLastCall();
+        EasyMock.expect(active.closeNotAssignedSuspendedTasks(taskId0Assignment.keySet())).andReturn(null).once();
+        expect(restoreConsumer.assignment()).andReturn(Collections.emptySet());
+
         replay();
 
         taskManager.setAssignmentMetadata(taskId0Assignment, Collections.>emptyMap());
-        taskManager.createTasks(taskId0Partitions);
+        taskManager.setAssignmentMetadata(Collections.>emptyMap(), Collections.>emptyMap());
+
+        taskManager.closeRevokedSuspendedTasks();
 
         verify(active);
     }
 
     @Test
-    public void shouldCloseStandbyUnAssignedSuspendedTasksWhenCreatingNewTasks() {
+    public void shouldCloseStandbyUnassignedTasksWhenCreatingNewTasks() {
         mockSingleActiveTask();
-        standby.closeNonAssignedSuspendedTasks(taskId0Assignment);
-        expectLastCall();
+        EasyMock.expect(standby.closeRevokedStandbyTasks(taskId0Assignment)).andReturn(Collections.emptyList()).once();
         replay();
 
         taskManager.setAssignmentMetadata(taskId0Assignment, Collections.>emptyMap());
+        taskManager.setPartitionsToTaskId(taskId0PartitionToTaskId);
         taskManager.createTasks(taskId0Partitions);
 
         verify(active);
     }
 
     @Test
-    public void shouldAddNonResumedActiveTasks() {
+    public void shouldAddNonResumedSuspendedTasks() {
         mockSingleActiveTask();
         expect(active.maybeResumeSuspendedTask(taskId0, taskId0Partitions)).andReturn(false);
         active.addNewTask(EasyMock.same(streamTask));
         replay();
 
+        // Need to call this twice so task manager doesn't consider all partitions "new"
+        taskManager.setAssignmentMetadata(taskId0Assignment, Collections.>emptyMap());
         taskManager.setAssignmentMetadata(taskId0Assignment, Collections.>emptyMap());
+        taskManager.setPartitionsToTaskId(taskId0PartitionToTaskId);
         taskManager.createTasks(taskId0Partitions);
 
         verify(activeTaskCreator, active);
     }
 
     @Test
-    public void shouldNotAddResumedActiveTasks() {
-        checkOrder(active, true);
-        expect(active.maybeResumeSuspendedTask(taskId0, taskId0Partitions)).andReturn(true);
+    public void shouldAddNewActiveTasks() {
+        mockSingleActiveTask();
+        active.addNewTask(EasyMock.same(streamTask));
         replay();
 
         taskManager.setAssignmentMetadata(taskId0Assignment, Collections.>emptyMap());
+        taskManager.setPartitionsToTaskId(taskId0PartitionToTaskId);
         taskManager.createTasks(taskId0Partitions);
 
-        // should be no calls to activeTaskCreator and no calls to active.addNewTasks(..)
-        verify(active, activeTaskCreator);
-    }
-
-    @Test
-    public void shouldAddNonResumedStandbyTasks() {
-        mockStandbyTaskExpectations();
-        expect(standby.maybeResumeSuspendedTask(taskId0, taskId0Partitions)).andReturn(false);
-        standby.addNewTask(EasyMock.same(standbyTask));
-        replay();
-
-        taskManager.setAssignmentMetadata(Collections.>emptyMap(), taskId0Assignment);
-        taskManager.createTasks(taskId0Partitions);
-
-        verify(standbyTaskCreator, active);
+        verify(activeTaskCreator, active);
     }
 
     @Test
-    public void shouldNotAddResumedStandbyTasks() {
+    public void shouldNotAddResumedActiveTasks() {
         checkOrder(active, true);
-        expect(standby.maybeResumeSuspendedTask(taskId0, taskId0Partitions)).andReturn(true);
+        expect(active.maybeResumeSuspendedTask(taskId0, taskId0Partitions)).andReturn(true);
         replay();
 
-        taskManager.setAssignmentMetadata(Collections.>emptyMap(), taskId0Assignment);
+        // Need to call this twice so task manager doesn't consider all partitions "new"
+        taskManager.setAssignmentMetadata(taskId0Assignment, Collections.>emptyMap());
+        taskManager.setAssignmentMetadata(taskId0Assignment, Collections.>emptyMap());
+        taskManager.setPartitionsToTaskId(taskId0PartitionToTaskId);
         taskManager.createTasks(taskId0Partitions);
 
-        // should be no calls to standbyTaskCreator and no calls to standby.addNewTasks(..)
-        verify(standby, standbyTaskCreator);
+        // should be no calls to activeTaskCreator and no calls to active.addNewTasks(..)
+        verify(active, activeTaskCreator);
     }
 
     @Test
@@ -320,48 +324,46 @@ public void shouldPauseActivePartitions() {
         replay();
 
         taskManager.setAssignmentMetadata(taskId0Assignment, Collections.>emptyMap());
+        taskManager.setPartitionsToTaskId(taskId0PartitionToTaskId);
         taskManager.createTasks(taskId0Partitions);
         verify(consumer);
     }
 
     @Test
     public void shouldSuspendActiveTasks() {
-        expect(active.suspend()).andReturn(null);
+        expect(active.suspendOrCloseTasks(revokedTasks, revokedChangelogs)).andReturn(null);
+        expect(restoreConsumer.assignment()).andReturn(Collections.emptySet());
         replay();
 
-        taskManager.suspendTasksAndState();
+        taskManager.suspendActiveTasksAndState(revokedPartitions);
         verify(active);
     }
 
     @Test
-    public void shouldSuspendStandbyTasks() {
-        expect(standby.suspend()).andReturn(null);
-        replay();
-
-        taskManager.suspendTasksAndState();
-        verify(standby);
-    }
-
-    @Test
+    @SuppressWarnings("unchecked")
     public void shouldUnassignChangelogPartitionsOnSuspend() {
-        restoreConsumer.unsubscribe();
+        expect(active.suspendOrCloseTasks(revokedTasks, new ArrayList<>()))
+                .andAnswer(() -> {
+                    ((List) EasyMock.getCurrentArguments()[1]).add(t1p0);
+                    return null;
+                });
+        expect(restoreConsumer.assignment()).andReturn(Collections.singleton(t1p0));
+
+        restoreConsumer.assign(Collections.emptySet());
         expectLastCall();
         replay();
 
-        taskManager.suspendTasksAndState();
+        taskManager.suspendActiveTasksAndState(Collections.emptySet());
         verify(restoreConsumer);
     }
 
     @Test
     public void shouldThrowStreamsExceptionAtEndIfExceptionDuringSuspend() {
-        expect(active.suspend()).andReturn(new RuntimeException(""));
-        expect(standby.suspend()).andReturn(new RuntimeException(""));
-        expectLastCall();
-        restoreConsumer.unsubscribe();
+        expect(active.suspendOrCloseTasks(revokedTasks, revokedChangelogs)).andReturn(new RuntimeException(""));
 
         replay();
         try {
-            taskManager.suspendTasksAndState();
+            taskManager.suspendActiveTasksAndState(revokedPartitions);
             fail("Should have thrown streams exception");
         } catch (final StreamsException e) {
             // expected
@@ -401,6 +403,8 @@ public void shouldUnassignChangelogPartitionsOnShutdown() {
 
     @Test
     public void shouldInitializeNewActiveTasks() {
+        EasyMock.expect(restoreConsumer.assignment()).andReturn(Collections.emptySet()).once();
+        EasyMock.expect(changeLogReader.restore(active)).andReturn(taskId0Partitions).once();
         active.updateRestored(EasyMock.>anyObject());
         expectLastCall();
         replay();
@@ -411,6 +415,8 @@ public void shouldInitializeNewActiveTasks() {
 
     @Test
     public void shouldInitializeNewStandbyTasks() {
+        EasyMock.expect(restoreConsumer.assignment()).andReturn(Collections.emptySet()).once();
+        EasyMock.expect(changeLogReader.restore(active)).andReturn(taskId0Partitions).once();
         active.updateRestored(EasyMock.>anyObject());
         expectLastCall();
         replay();
@@ -421,6 +427,7 @@ public void shouldInitializeNewStandbyTasks() {
 
     @Test
     public void shouldRestoreStateFromChangeLogReader() {
+        EasyMock.expect(restoreConsumer.assignment()).andReturn(taskId0Partitions).once();
         expect(changeLogReader.restore(active)).andReturn(taskId0Partitions);
         active.updateRestored(taskId0Partitions);
         expectLastCall();
@@ -432,6 +439,7 @@ public void shouldRestoreStateFromChangeLogReader() {
 
     @Test
     public void shouldResumeRestoredPartitions() {
+        EasyMock.expect(restoreConsumer.assignment()).andReturn(taskId0Partitions).once();
         expect(changeLogReader.restore(active)).andReturn(taskId0Partitions);
         expect(active.allTasksRunning()).andReturn(true);
         expect(consumer.assignment()).andReturn(taskId0Partitions);
@@ -475,6 +483,7 @@ public void shouldReturnFalseWhenOnlyActiveTasksAreRunning() {
     @Test
     public void shouldReturnFalseWhenThereAreStillNonRunningTasks() {
         expect(active.allTasksRunning()).andReturn(false);
+        EasyMock.expect(changeLogReader.restore(active)).andReturn(Collections.emptySet()).once();
         replay();
 
         assertFalse(taskManager.updateNewAndRestoringTasks());
@@ -623,10 +632,12 @@ public void shouldPunctuateActiveTasks() {
 
     @Test
     public void shouldNotResumeConsumptionUntilAllStoresRestored() {
+        EasyMock.expect(changeLogReader.restore(active)).andReturn(Collections.emptySet()).once();
         expect(active.allTasksRunning()).andReturn(false);
+
         final Consumer consumer = EasyMock.createStrictMock(Consumer.class);
         taskManager.setConsumer(consumer);
-        EasyMock.replay(active, consumer);
+        EasyMock.replay(active, consumer, changeLogReader);
 
         // shouldn't invoke `resume` method in consumer
         taskManager.updateNewAndRestoringTasks();
@@ -662,6 +673,9 @@ private void mockAssignStandbyPartitions(final long offset) {
 
         expectLastCall();
         EasyMock.replay(task);
+
+        EasyMock.expect(restoreConsumer.assignment()).andReturn(taskId0Partitions).once();
+        EasyMock.expect(changeLogReader.restore(active)).andReturn(taskId0Partitions).once();
     }
 
     private void mockStandbyTaskExpectations() {
diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
index a4735869c515c..de5311cef92d3 100644
--- a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
@@ -124,7 +124,7 @@ public ByteBuffer subscriptionUserData(final Set topics) {
             // 3. Task ids of valid local states on the client's state directory.
 
             final TaskManager taskManager = taskManger();
-            final Set previousActiveTasks = taskManager.prevActiveTaskIds();
+            final Set previousActiveTasks = taskManager.previousRunningTaskIds();
             final Set standbyTasks = taskManager.cachedTasksIds();
             standbyTasks.removeAll(previousActiveTasks);
             final FutureSubscriptionInfo data = new FutureSubscriptionInfo(
@@ -176,12 +176,15 @@ public void onAssignment(final ConsumerPartitionAssignor.Assignment assignment,
             final Map topicToPartitionInfo = new HashMap<>();
             final Map> partitionsByHost;
 
-            processVersionTwoAssignment("test ", info, partitions, activeTasks, topicToPartitionInfo);
+            final Map partitionsToTaskId = new HashMap<>();
+
+            processVersionTwoAssignment("test ", info, partitions, activeTasks, topicToPartitionInfo, partitionsToTaskId);
             partitionsByHost = info.partitionsByHost();
 
             final TaskManager taskManager = taskManger();
             taskManager.setClusterMetadata(Cluster.empty().withPartitions(topicToPartitionInfo));
             taskManager.setPartitionsByHostState(partitionsByHost);
+            taskManager.setPartitionsToTaskId(partitionsToTaskId);
             taskManager.setAssignmentMetadata(activeTasks, info.standbyTasks());
             taskManager.updateSubscriptionsFromAssignment(partitions);
         }

From d112ffd8d85de2441a8ea32d080e584d7d16a9a5 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Tue, 24 Sep 2019 19:32:14 -0700
Subject: [PATCH 0644/1071] KAFKA-8880: Docs on upgrade-guide (#7385)

Reviewers: Matthias J. Sax 
---
 docs/upgrade.html | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/upgrade.html b/docs/upgrade.html
index 757243b568ed2..0c69336127ca5 100644
--- a/docs/upgrade.html
+++ b/docs/upgrade.html
@@ -40,6 +40,11 @@ 
                Notable changes in 2
         it may result in uneven distribution of records across partitions in edge cases. Generally users will not be impacted, but this difference may be noticeable in tests and
         other situations producing records for a very short amount of time.
     
+    
              • The blocking KafkaConsumer#committed methods have been extended to allow a list of partitions as input parameters rather than a single partition.
+        It enables fewer request/response iterations between clients and brokers fetching for the committed offsets for the consumer group.
+        The old overloaded functions are deprecated and we would recommend users to make their code changes to leverage the new methods (details
+        can be found in KIP-520).
+    
                • 
 
 
 
                Upgrading from 0.8.x, 0.9.x, 0.10.0.x, 0.10.1.x, 0.10.2.x, 0.11.0.x, 1.0.x, 1.1.x, 2.0.x or 2.1.x or 2.2.x to 2.3.0
                

From 1d7f0b7c58f212d555c4f2ce766c2ca38f93a1cd Mon Sep 17 00:00:00 2001
From: Colin Patrick McCabe 
Date: Wed, 25 Sep 2019 08:20:51 -0700
Subject: [PATCH 0645/1071] MINOR: Improve the org.apache.kafka.common.protocol
 code (#7344)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add UUID to the list of types documented in Type#toHtml.

Type, Protocol, ArrayOf: use Type#isArray and Type#arrayElementType rather than typecasting to handle arrays.  This is cleaner.  It will also make it easier for us to add compact arrays (as specified by KIP-482) as a new array type distinct from the old array type.

Add MessageUtil#byteBufferToArray, as well as a test for it.  We will need this for handling tagged fields of type "bytes".

Schema#Visitor: we don't need a separate function overload for visiting arrays. We can just call "visit(Type field)".

TestUUID.json: reformat the JSON file to match the others.

ProtocolSerializationTest: improve the error messages on failure.  Check that each type has the name we expect it to have.

Reviewers: David Arthur , José Armando García Sancio , Vikas Singh 
---
 .../kafka/common/protocol/MessageUtil.java    | 16 ++++++
 .../kafka/common/protocol/Protocol.java       | 16 +++---
 .../kafka/common/protocol/types/ArrayOf.java  |  6 +-
 .../kafka/common/protocol/types/Schema.java   |  8 +--
 .../kafka/common/protocol/types/Struct.java   | 11 ++--
 .../kafka/common/protocol/types/Type.java     | 18 +++++-
 .../kafka/common/message/MessageTest.java     |  7 +--
 .../common/protocol/MessageUtilTest.java      | 10 ++++
 .../types/ProtocolSerializationTest.java      | 56 ++++++++++---------
 .../resources/common/message/TestUUID.json    |  6 +-
 10 files changed, 95 insertions(+), 59 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java b/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java
index a19059f5660a4..6e31e50c47c23 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java
@@ -19,6 +19,7 @@
 
 import org.apache.kafka.common.utils.Utils;
 
+import java.nio.ByteBuffer;
 import java.util.Iterator;
 import java.util.UUID;
 
@@ -37,6 +38,21 @@ public static short serializedUtf8Length(CharSequence input) {
         return (short) count;
     }
 
+    /**
+     * Copy a byte buffer into an array.  This will not affect the buffer's
+     * position or mark.
+     */
+    public static byte[] byteBufferToArray(ByteBuffer buf) {
+        byte[] arr = new byte[buf.remaining()];
+        int prevPosition = buf.position();
+        try {
+            buf.get(arr);
+        } finally {
+            buf.position(prevPosition);
+        }
+        return arr;
+    }
+
     public static String deepToString(Iterator iter) {
         StringBuilder bld = new StringBuilder("[");
         String prefix = "";
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java b/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java
index b5042c3f92a8d..65b5966dc66a9 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.common.protocol;
 
-import org.apache.kafka.common.protocol.types.ArrayOf;
 import org.apache.kafka.common.protocol.types.BoundField;
 import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Type;
@@ -43,18 +42,19 @@ private static void schemaToBnfHtml(Schema schema, StringBuilder b, int indentSi
 
         // Top level fields
         for (BoundField field: schema.fields()) {
-            if (field.def.type instanceof ArrayOf) {
+            Type type = field.def.type;
+            if (type.isArray()) {
                 b.append("[");
                 b.append(field.def.name);
                 b.append("] ");
-                Type innerType = ((ArrayOf) field.def.type).type();
-                if (!subTypes.containsKey(field.def.name))
-                    subTypes.put(field.def.name, innerType);
+                if (!subTypes.containsKey(field.def.name)) {
+                    subTypes.put(field.def.name, type.arrayElementType().get());
+                }
             } else {
                 b.append(field.def.name);
                 b.append(" ");
                 if (!subTypes.containsKey(field.def.name))
-                    subTypes.put(field.def.name, field.def.type);
+                    subTypes.put(field.def.name, type);
             }
         }
         b.append("\n");
@@ -81,8 +81,8 @@ private static void schemaToBnfHtml(Schema schema, StringBuilder b, int indentSi
     private static void populateSchemaFields(Schema schema, Set fields) {
         for (BoundField field: schema.fields()) {
             fields.add(field);
-            if (field.def.type instanceof ArrayOf) {
-                Type innerType = ((ArrayOf) field.def.type).type();
+            if (field.def.type.isArray()) {
+                Type innerType = field.def.type.arrayElementType().get();
                 if (innerType instanceof Schema)
                     populateSchemaFields((Schema) innerType, fields);
             } else if (field.def.type instanceof Schema)
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/ArrayOf.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/ArrayOf.java
index 6609dfd519619..3333084ef663a 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/ArrayOf.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/ArrayOf.java
@@ -19,6 +19,7 @@
 import org.apache.kafka.common.protocol.types.Type.DocumentedType;
 
 import java.nio.ByteBuffer;
+import java.util.Optional;
 
 /**
  * Represents a type for an array of a particular type
@@ -91,8 +92,9 @@ public int sizeOf(Object o) {
         return size;
     }
 
-    public Type type() {
-        return type;
+    @Override
+    public Optional arrayElementType() {
+        return Optional.of(type);
     }
 
     @Override
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java
index 5b12eee6e7a80..721b8c63bd76e 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java
@@ -209,10 +209,9 @@ private static void handleNode(Type node, Visitor visitor) {
             visitor.visit(schema);
             for (BoundField f : schema.fields())
                 handleNode(f.def.type, visitor);
-        } else if (node instanceof ArrayOf) {
-            ArrayOf array = (ArrayOf) node;
-            visitor.visit(array);
-            handleNode(array.type(), visitor);
+        } else if (node.isArray()) {
+            visitor.visit(node);
+            handleNode(node.arrayElementType().get(), visitor);
         } else {
             visitor.visit(node);
         }
@@ -223,7 +222,6 @@ private static void handleNode(Type node, Visitor visitor) {
      */
     public static abstract class Visitor {
         public void visit(Schema schema) {}
-        public void visit(ArrayOf array) {}
         public void visit(Type field) {}
     }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
index 350497cf4fc84..4aaff74df4d84 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
@@ -418,9 +418,8 @@ public Struct instance(BoundField field) {
         validateField(field);
         if (field.def.type instanceof Schema) {
             return new Struct((Schema) field.def.type);
-        } else if (field.def.type instanceof ArrayOf) {
-            ArrayOf array = (ArrayOf) field.def.type;
-            return new Struct((Schema) array.type());
+        } else if (field.def.type.isArray()) {
+            return new Struct((Schema) field.def.type.arrayElementType().get());
         } else {
             throw new SchemaException("Field '" + field.def.name + "' is not a container type, it is of type " + field.def.type);
         }
@@ -495,7 +494,7 @@ public String toString() {
             BoundField f = this.schema.get(i);
             b.append(f.def.name);
             b.append('=');
-            if (f.def.type instanceof ArrayOf && this.values[i] != null) {
+            if (f.def.type.isArray() && this.values[i] != null) {
                 Object[] arrayValue = (Object[]) this.values[i];
                 b.append('[');
                 for (int j = 0; j < arrayValue.length; j++) {
@@ -519,7 +518,7 @@ public int hashCode() {
         int result = 1;
         for (int i = 0; i < this.values.length; i++) {
             BoundField f = this.schema.get(i);
-            if (f.def.type instanceof ArrayOf) {
+            if (f.def.type.isArray()) {
                 if (this.get(f) != null) {
                     Object[] arrayObject = (Object[]) this.get(f);
                     for (Object arrayItem: arrayObject)
@@ -549,7 +548,7 @@ public boolean equals(Object obj) {
         for (int i = 0; i < this.values.length; i++) {
             BoundField f = this.schema.get(i);
             boolean result;
-            if (f.def.type instanceof ArrayOf) {
+            if (f.def.type.isArray()) {
                 result = Arrays.equals((Object[]) this.get(f), (Object[]) other.get(f));
             } else {
                 Object thisField = this.get(f);
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java
index df457463f06c8..aa0ee40e68097 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java
@@ -23,6 +23,7 @@
 import org.apache.kafka.common.utils.Utils;
 
 import java.nio.ByteBuffer;
+import java.util.Optional;
 import java.util.UUID;
 
 /**
@@ -64,6 +65,20 @@ public boolean isNullable() {
         return false;
     }
 
+    /**
+     * If the type is an array, return the type of the array elements.  Otherwise, return empty.
+     */
+    public Optional arrayElementType() {
+        return Optional.empty();
+    }
+
+    /**
+     * Returns true if the type is an array.
+     */
+    public final boolean isArray() {
+        return arrayElementType().isPresent();
+    }
+
     /**
      * A Type that can return its description for documentation purposes.
      */
@@ -704,10 +719,9 @@ public String documentation() {
     };
 
     private static String toHtml() {
-
         DocumentedType[] types = {
             BOOLEAN, INT8, INT16, INT32, INT64,
-            UNSIGNED_INT32, VARINT, VARLONG,
+            UNSIGNED_INT32, VARINT, VARLONG, UUID,
             STRING, NULLABLE_STRING, BYTES, NULLABLE_BYTES,
             RECORDS, new ArrayOf(STRING)};
         final StringBuilder b = new StringBuilder();
diff --git a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
index bf060cdd161ec..ca8d63db8f28b 100644
--- a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
@@ -37,7 +37,6 @@
 import org.apache.kafka.common.protocol.ByteBufferAccessor;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.protocol.Message;
-import org.apache.kafka.common.protocol.types.ArrayOf;
 import org.apache.kafka.common.protocol.types.BoundField;
 import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.SchemaException;
@@ -699,9 +698,9 @@ private static void compareTypes(NamedType typeA, NamedType typeB) {
                     entryA, entryA.type.isNullable() ? "nullable" : "non-nullable",
                     entryB, entryB.type.isNullable() ? "nullable" : "non-nullable"));
             }
-            if (entryA.type instanceof ArrayOf) {
-                compareTypes(new NamedType(entryA.name, ((ArrayOf) entryA.type).type()),
-                             new NamedType(entryB.name, ((ArrayOf) entryB.type).type()));
+            if (entryA.type.isArray()) {
+                compareTypes(new NamedType(entryA.name, entryA.type.arrayElementType().get()),
+                             new NamedType(entryB.name, entryB.type.arrayElementType().get()));
             }
         }
     }
diff --git a/clients/src/test/java/org/apache/kafka/common/protocol/MessageUtilTest.java b/clients/src/test/java/org/apache/kafka/common/protocol/MessageUtilTest.java
index 8620a1b03dc8d..a28e15570e0b7 100755
--- a/clients/src/test/java/org/apache/kafka/common/protocol/MessageUtilTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/protocol/MessageUtilTest.java
@@ -21,9 +21,11 @@
 import org.junit.Test;
 import org.junit.rules.Timeout;
 
+import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 
 public final class MessageUtilTest {
@@ -56,4 +58,12 @@ public void testDeepToString() {
         assertEquals("[foo]",
             MessageUtil.deepToString(Arrays.asList("foo").iterator()));
     }
+
+    @Test
+    public void testByteBufferToArray() {
+        assertArrayEquals(new byte[] {1, 2, 3},
+                MessageUtil.byteBufferToArray(ByteBuffer.wrap(new byte[] {1, 2, 3})));
+        assertArrayEquals(new byte[] {},
+                MessageUtil.byteBufferToArray(ByteBuffer.wrap(new byte[] {})));
+    }
 }
diff --git a/clients/src/test/java/org/apache/kafka/common/protocol/types/ProtocolSerializationTest.java b/clients/src/test/java/org/apache/kafka/common/protocol/types/ProtocolSerializationTest.java
index 8a3b8ce01ac91..505772004a7c0 100644
--- a/clients/src/test/java/org/apache/kafka/common/protocol/types/ProtocolSerializationTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/protocol/types/ProtocolSerializationTest.java
@@ -68,31 +68,32 @@ public void setup() {
 
     @Test
     public void testSimple() {
-        check(Type.BOOLEAN, false);
-        check(Type.BOOLEAN, true);
-        check(Type.INT8, (byte) -111);
-        check(Type.INT16, (short) -11111);
-        check(Type.INT32, -11111111);
-        check(Type.INT64, -11111111111L);
-        check(Type.STRING, "");
-        check(Type.STRING, "hello");
-        check(Type.STRING, "A\u00ea\u00f1\u00fcC");
-        check(Type.NULLABLE_STRING, null);
-        check(Type.NULLABLE_STRING, "");
-        check(Type.NULLABLE_STRING, "hello");
-        check(Type.BYTES, ByteBuffer.allocate(0));
-        check(Type.BYTES, ByteBuffer.wrap("abcd".getBytes()));
-        check(Type.NULLABLE_BYTES, null);
-        check(Type.NULLABLE_BYTES, ByteBuffer.allocate(0));
-        check(Type.NULLABLE_BYTES, ByteBuffer.wrap("abcd".getBytes()));
-        check(Type.VARINT, Integer.MAX_VALUE);
-        check(Type.VARINT, Integer.MIN_VALUE);
-        check(Type.VARLONG, Long.MAX_VALUE);
-        check(Type.VARLONG, Long.MIN_VALUE);
-        check(new ArrayOf(Type.INT32), new Object[] {1, 2, 3, 4});
-        check(new ArrayOf(Type.STRING), new Object[] {});
-        check(new ArrayOf(Type.STRING), new Object[] {"hello", "there", "beautiful"});
-        check(ArrayOf.nullable(Type.STRING), null);
+        check(Type.BOOLEAN, false, "BOOLEAN");
+        check(Type.BOOLEAN, true, "BOOLEAN");
+        check(Type.INT8, (byte) -111, "INT8");
+        check(Type.INT16, (short) -11111, "INT16");
+        check(Type.INT32, -11111111, "INT32");
+        check(Type.INT64, -11111111111L, "INT64");
+        check(Type.STRING, "", "STRING");
+        check(Type.STRING, "hello", "STRING");
+        check(Type.STRING, "A\u00ea\u00f1\u00fcC", "STRING");
+        check(Type.NULLABLE_STRING, null, "NULLABLE_STRING");
+        check(Type.NULLABLE_STRING, "", "NULLABLE_STRING");
+        check(Type.NULLABLE_STRING, "hello", "NULLABLE_STRING");
+        check(Type.BYTES, ByteBuffer.allocate(0), "BYTES");
+        check(Type.BYTES, ByteBuffer.wrap("abcd".getBytes()), "BYTES");
+        check(Type.NULLABLE_BYTES, null, "NULLABLE_BYTES");
+        check(Type.NULLABLE_BYTES, ByteBuffer.allocate(0), "NULLABLE_BYTES");
+        check(Type.NULLABLE_BYTES, ByteBuffer.wrap("abcd".getBytes()), "NULLABLE_BYTES");
+        check(Type.VARINT, Integer.MAX_VALUE, "VARINT");
+        check(Type.VARINT, Integer.MIN_VALUE, "VARINT");
+        check(Type.VARLONG, Long.MAX_VALUE, "VARLONG");
+        check(Type.VARLONG, Long.MIN_VALUE, "VARLONG");
+        check(new ArrayOf(Type.INT32), new Object[] {1, 2, 3, 4}, "ARRAY(INT32)");
+        check(new ArrayOf(Type.STRING), new Object[] {}, "ARRAY(STRING)");
+        check(new ArrayOf(Type.STRING), new Object[] {"hello", "there", "beautiful"},
+                "ARRAY(STRING)");
+        check(ArrayOf.nullable(Type.STRING), null, "ARRAY(STRING)");
     }
 
     @Test
@@ -105,7 +106,7 @@ public void testNulls() {
                 if (!f.def.type.isNullable())
                     fail("Should not allow serialization of null value.");
             } catch (SchemaException e) {
-                assertFalse(f.def.type.isNullable());
+                assertFalse(f.toString() + " should not be nullable", f.def.type.isNullable());
             } finally {
                 this.struct.set(f, o);
             }
@@ -259,12 +260,13 @@ private Object roundtrip(Type type, Object obj) {
         return read;
     }
 
-    private void check(Type type, Object obj) {
+    private void check(Type type, Object obj, String expectedTypeName) {
         Object result = roundtrip(type, obj);
         if (obj instanceof Object[]) {
             obj = Arrays.asList((Object[]) obj);
             result = Arrays.asList((Object[]) result);
         }
+        assertEquals(expectedTypeName, type.toString());
         assertEquals("The object read back should be the same as what was written.", obj, result);
     }
 
diff --git a/clients/src/test/resources/common/message/TestUUID.json b/clients/src/test/resources/common/message/TestUUID.json
index 8c1c7bd8f06ec..843a0402bb568 100644
--- a/clients/src/test/resources/common/message/TestUUID.json
+++ b/clients/src/test/resources/common/message/TestUUID.json
@@ -16,11 +16,7 @@
   "name": "TestUUID",
   "validVersions": "1",
   "fields": [
-    {
-      "name": "processId",
-      "versions": "1",
-      "type": "uuid"
-    }
+    { "name": "processId", "versions": "1+", "type": "uuid" }
   ],
   "type": "header"
 }
\ No newline at end of file

From 92688ef82ccbeee6014ae9c1db64fb4289b7bdd7 Mon Sep 17 00:00:00 2001
From: Colin Patrick McCabe 
Date: Wed, 25 Sep 2019 08:58:54 -0700
Subject: [PATCH 0646/1071] MINOR: improve the Kafka RPC code generator (#7340)

Move the generator checkstyle suppressions to a special section, rather
than mixing them in with the other sections.  For generated code, do not
complain about variable names or cyclic complexity.

FieldType.java: remove isInteger since it isn't used anywhere.  This way, we
don't have to decide whether a UUID is an integer or not (there are arguments
for both choices).  Add FieldType#serializationIsDifferentInFlexibleVersions
and FieldType#isVariableLength.

HeaderGenerator: add the ability to generate static imports.  Add
IsNullConditional, VersionConditional, and ClauseGenerator as easier ways of
generating "if" statements.
---
 checkstyle/import-control.xml                 |   1 +
 checkstyle/suppressions.xml                   |  23 +-
 .../apache/kafka/message/ClauseGenerator.java |  25 ++
 .../org/apache/kafka/message/FieldType.java   |  55 ++--
 .../apache/kafka/message/HeaderGenerator.java |  13 +
 .../kafka/message/IsNullConditional.java      | 105 ++++++++
 .../kafka/message/MessageDataGenerator.java   |   5 +-
 .../kafka/message/VersionConditional.java     | 220 ++++++++++++++++
 .../org/apache/kafka/message/Versions.java    |  53 ++++
 .../apache/kafka/message/CodeBufferTest.java  |  73 ++++++
 .../kafka/message/IsNullConditionalTest.java  | 122 +++++++++
 .../kafka/message/VersionConditionalTest.java | 245 ++++++++++++++++++
 .../apache/kafka/message/VersionsTest.java    |  24 ++
 13 files changed, 931 insertions(+), 33 deletions(-)
 create mode 100644 generator/src/main/java/org/apache/kafka/message/ClauseGenerator.java
 create mode 100644 generator/src/main/java/org/apache/kafka/message/IsNullConditional.java
 create mode 100644 generator/src/main/java/org/apache/kafka/message/VersionConditional.java
 create mode 100644 generator/src/test/java/org/apache/kafka/message/CodeBufferTest.java
 create mode 100644 generator/src/test/java/org/apache/kafka/message/IsNullConditionalTest.java
 create mode 100644 generator/src/test/java/org/apache/kafka/message/VersionConditionalTest.java

diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml
index 44b06a7a381ca..066fc33239a94 100644
--- a/checkstyle/import-control.xml
+++ b/checkstyle/import-control.xml
@@ -118,6 +118,7 @@
     
       
       
+      
       
       
       
diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index ce1025a743e59..050108bd83efb 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -8,6 +8,14 @@
 
     
 
+    
+    
+    
+    
+
     
     
@@ -49,7 +57,7 @@
               files="(Utils|Topic|KafkaLZ4BlockOutputStream|AclData|JoinGroupRequest).java"/>
 
     
+              files="(ConsumerCoordinator|Fetcher|Sender|KafkaProducer|BufferPool|ConfigDef|RecordAccumulator|KerberosLogin|AbstractRequest|AbstractResponse|Selector|SslFactory|SslTransportLayer|SaslClientAuthenticator|SaslClientCallbackHandler|SaslServerAuthenticator|AbstractCoordinator).java"/>
 
     
@@ -60,6 +68,12 @@
     
 
+    
+
+    
+
     
     
@@ -233,11 +247,4 @@
     
 
-    
-    
-
-    
-
 
diff --git a/generator/src/main/java/org/apache/kafka/message/ClauseGenerator.java b/generator/src/main/java/org/apache/kafka/message/ClauseGenerator.java
new file mode 100644
index 0000000000000..5ef55f26e3835
--- /dev/null
+++ b/generator/src/main/java/org/apache/kafka/message/ClauseGenerator.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.message;
+
+/**
+ * Generates a clause.
+ */
+public interface ClauseGenerator {
+    void generate(Versions versions);
+}
diff --git a/generator/src/main/java/org/apache/kafka/message/FieldType.java b/generator/src/main/java/org/apache/kafka/message/FieldType.java
index 8f0c3634ace69..b14449ba9d97a 100644
--- a/generator/src/main/java/org/apache/kafka/message/FieldType.java
+++ b/generator/src/main/java/org/apache/kafka/message/FieldType.java
@@ -41,11 +41,6 @@ final class Int8FieldType implements FieldType {
         static final Int8FieldType INSTANCE = new Int8FieldType();
         private static final String NAME = "int8";
 
-        @Override
-        public boolean isInteger() {
-            return true;
-        }
-
         @Override
         public Optional fixedLength() {
             return Optional.of(1);
@@ -61,11 +56,6 @@ final class Int16FieldType implements FieldType {
         static final Int16FieldType INSTANCE = new Int16FieldType();
         private static final String NAME = "int16";
 
-        @Override
-        public boolean isInteger() {
-            return true;
-        }
-
         @Override
         public Optional fixedLength() {
             return Optional.of(2);
@@ -81,11 +71,6 @@ final class Int32FieldType implements FieldType {
         static final Int32FieldType INSTANCE = new Int32FieldType();
         private static final String NAME = "int32";
 
-        @Override
-        public boolean isInteger() {
-            return true;
-        }
-
         @Override
         public Optional fixedLength() {
             return Optional.of(4);
@@ -101,11 +86,6 @@ final class Int64FieldType implements FieldType {
         static final Int64FieldType INSTANCE = new Int64FieldType();
         private static final String NAME = "int64";
 
-        @Override
-        public boolean isInteger() {
-            return true;
-        }
-
         @Override
         public Optional fixedLength() {
             return Optional.of(8);
@@ -136,6 +116,11 @@ final class StringFieldType implements FieldType {
         static final StringFieldType INSTANCE = new StringFieldType();
         private static final String NAME = "string";
 
+        @Override
+        public boolean serializationIsDifferentInFlexibleVersions() {
+            return true;
+        }
+
         @Override
         public boolean isString() {
             return true;
@@ -156,6 +141,11 @@ final class BytesFieldType implements FieldType {
         static final BytesFieldType INSTANCE = new BytesFieldType();
         private static final String NAME = "bytes";
 
+        @Override
+        public boolean serializationIsDifferentInFlexibleVersions() {
+            return true;
+        }
+
         @Override
         public boolean isBytes() {
             return true;
@@ -179,6 +169,11 @@ final class StructType implements FieldType {
             this.type = type;
         }
 
+        @Override
+        public boolean serializationIsDifferentInFlexibleVersions() {
+            return true;
+        }
+
         @Override
         public boolean isStruct() {
             return true;
@@ -197,6 +192,11 @@ final class ArrayType implements FieldType {
             this.elementType = elementType;
         }
 
+        @Override
+        public boolean serializationIsDifferentInFlexibleVersions() {
+            return true;
+        }
+
         @Override
         public boolean isArray() {
             return true;
@@ -266,10 +266,6 @@ static FieldType parse(String string) {
         }
     }
 
-    default boolean isInteger() {
-        return false;
-    }
-
     /**
      * Returns true if this is an array type.
      */
@@ -284,6 +280,13 @@ default boolean isStructArray() {
         return false;
     }
 
+    /**
+     * Returns true if the serialization of this type is different in flexible versions.
+     */
+    default boolean serializationIsDifferentInFlexibleVersions() {
+        return false;
+    }
+
     /**
      * Returns true if this is a string type.
      */
@@ -319,6 +322,10 @@ default Optional fixedLength() {
         return Optional.empty();
     }
 
+    default boolean isVariableLength() {
+        return !fixedLength().isPresent();
+    }
+
     /**
      * Convert the field type to a JSON string.
      */
diff --git a/generator/src/main/java/org/apache/kafka/message/HeaderGenerator.java b/generator/src/main/java/org/apache/kafka/message/HeaderGenerator.java
index 1223c00a6c08c..e1b9c12935604 100644
--- a/generator/src/main/java/org/apache/kafka/message/HeaderGenerator.java
+++ b/generator/src/main/java/org/apache/kafka/message/HeaderGenerator.java
@@ -52,16 +52,23 @@ public final class HeaderGenerator {
     private final TreeSet imports;
     private final String packageName;
 
+    private final TreeSet staticImports;
+
     public HeaderGenerator(String packageName) {
         this.buffer = new CodeBuffer();
         this.imports = new TreeSet<>();
         this.packageName = packageName;
+        this.staticImports = new TreeSet<>();
     }
 
     public void addImport(String newImport) {
         this.imports.add(newImport);
     }
 
+    public void addStaticImport(String newImport) {
+        this.staticImports.add(newImport);
+    }
+
     public void generate() {
         Objects.requireNonNull(packageName);
         for (int i = 0; i < HEADER.length; i++) {
@@ -73,6 +80,12 @@ public void generate() {
             buffer.printf("import %s;%n", newImport);
         }
         buffer.printf("%n");
+        if (!staticImports.isEmpty()) {
+            for (String newImport : staticImports) {
+                buffer.printf("import static %s;%n", newImport);
+            }
+            buffer.printf("%n");
+        }
     }
 
     public CodeBuffer buffer() {
diff --git a/generator/src/main/java/org/apache/kafka/message/IsNullConditional.java b/generator/src/main/java/org/apache/kafka/message/IsNullConditional.java
new file mode 100644
index 0000000000000..5ff51dacc6652
--- /dev/null
+++ b/generator/src/main/java/org/apache/kafka/message/IsNullConditional.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.message;
+
+/**
+ * Creates an if statement based on whether or not a particular field is null.
+ */
+public final class IsNullConditional {
+    static IsNullConditional forName(String name) {
+        return new IsNullConditional(name);
+    }
+
+    static IsNullConditional forField(FieldSpec field) {
+        IsNullConditional cond = new IsNullConditional(field.camelCaseName());
+        cond.nullableVersions(field.nullableVersions());
+        return cond;
+    }
+
+    private final String name;
+    private Versions nullableVersions = Versions.ALL;
+    private Versions possibleVersions = Versions.ALL;
+    private Runnable ifNull = null;
+    private Runnable ifNotNull = null;
+    private boolean alwaysEmitBlockScope = false;
+
+    private IsNullConditional(String name) {
+        this.name = name;
+    }
+
+    IsNullConditional nullableVersions(Versions nullableVersions) {
+        this.nullableVersions = nullableVersions;
+        return this;
+    }
+
+    IsNullConditional possibleVersions(Versions possibleVersions) {
+        this.possibleVersions = possibleVersions;
+        return this;
+    }
+
+    IsNullConditional ifNull(Runnable ifNull) {
+        this.ifNull = ifNull;
+        return this;
+    }
+
+    IsNullConditional ifNotNull(Runnable ifNotNull) {
+        this.ifNotNull = ifNotNull;
+        return this;
+    }
+
+    IsNullConditional alwaysEmitBlockScope(boolean alwaysEmitBlockScope) {
+        this.alwaysEmitBlockScope = alwaysEmitBlockScope;
+        return this;
+    }
+
+    void generate(CodeBuffer buffer) {
+        if (nullableVersions.intersect(possibleVersions).empty()) {
+            if (ifNotNull != null) {
+                if (alwaysEmitBlockScope) {
+                    buffer.printf("{%n");
+                    buffer.incrementIndent();
+                }
+                ifNotNull.run();
+                if (alwaysEmitBlockScope) {
+                    buffer.decrementIndent();
+                    buffer.printf("}%n");
+                }
+            }
+        } else {
+            if (ifNull != null) {
+                buffer.printf("if (%s == null) {%n", name);
+                buffer.incrementIndent();
+                ifNull.run();
+                buffer.decrementIndent();
+                if (ifNotNull != null) {
+                    buffer.printf("} else {%n");
+                    buffer.incrementIndent();
+                    ifNotNull.run();
+                    buffer.decrementIndent();
+                }
+                buffer.printf("}%n");
+            } else if (ifNotNull != null) {
+                buffer.printf("if (%s != null) {%n", name);
+                buffer.incrementIndent();
+                ifNotNull.run();
+                buffer.decrementIndent();
+                buffer.printf("}%n");
+            }
+        }
+    }
+}
diff --git a/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java b/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java
index 2e946e7bfb458..848af660eedaa 100644
--- a/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java
+++ b/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java
@@ -757,7 +757,10 @@ private void generateFieldToStruct(FieldSpec field, Versions curVersions) {
                 " cannot be nullable.");
         }
         if ((field.type() instanceof FieldType.BoolFieldType) ||
-                (field.type().isInteger()) ||
+                (field.type() instanceof FieldType.Int8FieldType) ||
+                (field.type() instanceof FieldType.Int16FieldType) ||
+                (field.type() instanceof FieldType.Int32FieldType) ||
+                (field.type() instanceof FieldType.Int64FieldType) ||
                 (field.type() instanceof FieldType.UUIDFieldType) ||
                 (field.type() instanceof FieldType.StringFieldType)) {
             boolean maybeAbsent =
diff --git a/generator/src/main/java/org/apache/kafka/message/VersionConditional.java b/generator/src/main/java/org/apache/kafka/message/VersionConditional.java
new file mode 100644
index 0000000000000..51b6cb047a0f0
--- /dev/null
+++ b/generator/src/main/java/org/apache/kafka/message/VersionConditional.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.message;
+
+/**
+ * Creates an if statement based on whether or not the current version
+ * falls within a given range.
+ */
+public final class VersionConditional {
+    /**
+     * Create a version conditional.
+     *
+     * @param containingVersions    The versions for which the conditional is true.
+     * @param possibleVersions      The range of possible versions.
+     * @return                      The version conditional.
+     */
+    static VersionConditional forVersions(Versions containingVersions,
+                                          Versions possibleVersions) {
+        return new VersionConditional(containingVersions, possibleVersions);
+    }
+
+    private final Versions containingVersions;
+    private final Versions possibleVersions;
+    private ClauseGenerator ifMember = null;
+    private ClauseGenerator ifNotMember = null;
+    private boolean alwaysEmitBlockScope = false;
+    private boolean allowMembershipCheckAlwaysFalse = true;
+
+    private VersionConditional(Versions containingVersions, Versions possibleVersions) {
+        this.containingVersions = containingVersions;
+        this.possibleVersions = possibleVersions;
+    }
+
+    VersionConditional ifMember(ClauseGenerator ifMember) {
+        this.ifMember = ifMember;
+        return this;
+    }
+
+    VersionConditional ifNotMember(ClauseGenerator ifNotMember) {
+        this.ifNotMember = ifNotMember;
+        return this;
+    }
+
+    /**
+     * If this is set, we will always create a new block scope, even if there
+     * are no 'if' statements.  This is useful for cases where we want to
+     * declare variables in the clauses without worrying if they conflict with
+     * other variables of the same name.
+     */
+    VersionConditional alwaysEmitBlockScope(boolean alwaysEmitBlockScope) {
+        this.alwaysEmitBlockScope = alwaysEmitBlockScope;
+        return this;
+    }
+
+    /**
+     * If this is set, VersionConditional#generate will throw an exception if
+     * the 'ifMember' clause is never used.  This is useful as a sanity check
+     * in some cases where it doesn't make sense for the condition to always be
+     * false.  For example, when generating a Message#write function, 
+     * we might check that the version we're writing is supported.  It wouldn't
+     * make sense for this check to always be false, since that would mean that
+     * no versions at all were supported.
+     */
+    VersionConditional allowMembershipCheckAlwaysFalse(boolean allowMembershipCheckAlwaysFalse) {
+        this.allowMembershipCheckAlwaysFalse = allowMembershipCheckAlwaysFalse;
+        return this;
+    }
+
+    private void generateFullRangeCheck(Versions ifVersions,
+                                        Versions ifNotVersions,
+                                        CodeBuffer buffer) {
+        if (ifMember != null) {
+            buffer.printf("if ((version >= %d) && (version <= %d)) {%n",
+                    containingVersions.lowest(), containingVersions.highest());
+            buffer.incrementIndent();
+            ifMember.generate(ifVersions);
+            buffer.decrementIndent();
+            if (ifNotMember != null) {
+                buffer.printf("} else {%n");
+                buffer.incrementIndent();
+                ifNotMember.generate(ifNotVersions);
+                buffer.decrementIndent();
+            }
+            buffer.printf("}%n");
+        } else if (ifNotMember != null) {
+            buffer.printf("if ((version < %d) || (version > %d)) {%n",
+                    containingVersions.lowest(), containingVersions.highest());
+            buffer.incrementIndent();
+            ifNotMember.generate(ifNotVersions);
+            buffer.decrementIndent();
+            buffer.printf("}%n");
+        }
+    }
+
+    private void generateLowerRangeCheck(Versions ifVersions,
+                                         Versions ifNotVersions,
+                                         CodeBuffer buffer) {
+        if (ifMember != null) {
+            buffer.printf("if (version >= %d) {%n", containingVersions.lowest());
+            buffer.incrementIndent();
+            ifMember.generate(ifVersions);
+            buffer.decrementIndent();
+            if (ifNotMember != null) {
+                buffer.printf("} else {%n");
+                buffer.incrementIndent();
+                ifNotMember.generate(ifNotVersions);
+                buffer.decrementIndent();
+            }
+            buffer.printf("}%n");
+        } else if (ifNotMember != null) {
+            buffer.printf("if (version < %d) {%n", containingVersions.lowest());
+            buffer.incrementIndent();
+            ifNotMember.generate(ifNotVersions);
+            buffer.decrementIndent();
+            buffer.printf("}%n");
+        }
+    }
+
+    private void generateUpperRangeCheck(Versions ifVersions,
+                                         Versions ifNotVersions,
+                                         CodeBuffer buffer) {
+        if (ifMember != null) {
+            buffer.printf("if (version <= %d) {%n", containingVersions.highest());
+            buffer.incrementIndent();
+            ifMember.generate(ifVersions);
+            buffer.decrementIndent();
+            if (ifNotMember != null) {
+                buffer.printf("} else {%n");
+                buffer.incrementIndent();
+                ifNotMember.generate(ifNotVersions);
+                buffer.decrementIndent();
+            }
+            buffer.printf("}%n");
+        } else if (ifNotMember != null) {
+            buffer.printf("if (version > %d) {%n", containingVersions.highest());
+            buffer.incrementIndent();
+            ifNotMember.generate(ifNotVersions);
+            buffer.decrementIndent();
+            buffer.printf("}%n");
+        }
+    }
+
+    private void generateAlwaysTrueCheck(Versions ifVersions, CodeBuffer buffer) {
+        if (ifMember != null) {
+            if (alwaysEmitBlockScope) {
+                buffer.printf("{%n");
+                buffer.incrementIndent();
+            }
+            ifMember.generate(ifVersions);
+            if (alwaysEmitBlockScope) {
+                buffer.decrementIndent();
+                buffer.printf("}%n");
+            }
+        }
+    }
+
+    private void generateAlwaysFalseCheck(Versions ifNotVersions, CodeBuffer buffer) {
+        if (!allowMembershipCheckAlwaysFalse) {
+            throw new RuntimeException("Version ranges " + containingVersions +
+                " and " + possibleVersions + " have no versions in common.");
+        }
+        if (ifNotMember != null) {
+            if (alwaysEmitBlockScope) {
+                buffer.printf("{%n");
+                buffer.incrementIndent();
+            }
+            ifNotMember.generate(ifNotVersions);
+            if (alwaysEmitBlockScope) {
+                buffer.decrementIndent();
+                buffer.printf("}%n");
+            }
+        }
+    }
+
+    void generate(CodeBuffer buffer) {
+        Versions ifVersions = possibleVersions.intersect(containingVersions);
+        Versions ifNotVersions = possibleVersions.subtract(containingVersions);
+        // In the case where ifNotVersions would be two ranges rather than one,
+        // we just pass in the original possibleVersions instead.
+        // This is slightly less optimal, but allows us to avoid dealing with
+        // multiple ranges.
+        if (ifNotVersions == null) {
+            ifNotVersions = possibleVersions;
+        }
+
+        if (possibleVersions.lowest() < containingVersions.lowest()) {
+            if (possibleVersions.highest() > containingVersions.highest()) {
+                generateFullRangeCheck(ifVersions, ifNotVersions, buffer);
+            } else if (possibleVersions.highest() >= containingVersions.lowest()) {
+                generateLowerRangeCheck(ifVersions, ifNotVersions, buffer);
+            } else {
+                generateAlwaysFalseCheck(ifNotVersions, buffer);
+            }
+        } else if (possibleVersions.highest() >= containingVersions.lowest() &&
+                    (possibleVersions.lowest() <= containingVersions.highest())) {
+            if (possibleVersions.highest() > containingVersions.highest()) {
+                generateUpperRangeCheck(ifVersions, ifNotVersions, buffer);
+            } else {
+                generateAlwaysTrueCheck(ifVersions, buffer);
+            }
+        } else {
+            generateAlwaysFalseCheck(ifNotVersions, buffer);
+        }
+    }
+}
diff --git a/generator/src/main/java/org/apache/kafka/message/Versions.java b/generator/src/main/java/org/apache/kafka/message/Versions.java
index 3903680b422f4..649c065d043c0 100644
--- a/generator/src/main/java/org/apache/kafka/message/Versions.java
+++ b/generator/src/main/java/org/apache/kafka/message/Versions.java
@@ -109,6 +109,12 @@ public String toString() {
         }
     }
 
+    /**
+     * Return the intersection of two version ranges.
+     *
+     * @param other     The other version range.
+     * @return          A new version range.
+     */
     public Versions intersect(Versions other) {
         short newLowest = lowest > other.lowest ? lowest : other.lowest;
         short newHighest = highest < other.highest ? highest : other.highest;
@@ -118,6 +124,53 @@ public Versions intersect(Versions other) {
         return new Versions(newLowest, newHighest);
     }
 
+    /**
+     * Return a new version range that trims some versions from this range, if possible.
+     * We can't trim any versions if the resulting range would be disjoint.
+     *
+     * Some examples:
+     * 1-4.trim(1-2) = 3-4
+     * 3+.trim(4+) = 3
+     * 4+.trim(3+) = none
+     * 1-5.trim(2-4) = null
+     *
+     * @param other                 The other version range.
+     * @return                      A new version range.
+     */
+    public Versions subtract(Versions other) {
+        if (other.lowest() <= lowest) {
+            if (other.highest >= highest) {
+                // Case 1: other is a superset of this.  Trim everything.
+                return Versions.NONE;
+            } else if (other.highest < lowest) {
+                // Case 2: other is a disjoint version range that is lower than this.  Trim nothing.
+                return this;
+            } else {
+                // Case 3: trim some values from the beginning of this range.
+                //
+                // Note: it is safe to assume that other.highest() + 1 will not overflow.
+                // The reason is because if other.highest() were Short.MAX_VALUE,
+                // other.highest() < highest could not be true.
+                return new Versions((short) (other.highest() + 1), highest);
+            }
+        } else if (other.highest >= highest) {
+            int newHighest = other.lowest - 1;
+            if (newHighest < 0) {
+                // Case 4: other was NONE.  Trim nothing.
+                return this;
+            } else if (newHighest < highest) {
+                // Case 5: trim some values from the end of this range.
+                return new Versions(lowest, (short) newHighest);
+            } else {
+                // Case 6: other is a disjoint range that is higher than this.  Trim nothing.
+                return this;
+            }
+        } else {
+            // Case 7: the difference between this and other would be two ranges, not one.
+            return null;
+        }
+    }
+
     public boolean contains(short version) {
         return version >= lowest && version <= highest;
     }
diff --git a/generator/src/test/java/org/apache/kafka/message/CodeBufferTest.java b/generator/src/test/java/org/apache/kafka/message/CodeBufferTest.java
new file mode 100644
index 0000000000000..e896658bfb0e7
--- /dev/null
+++ b/generator/src/test/java/org/apache/kafka/message/CodeBufferTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.message;
+
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.StringWriter;
+
+public class CodeBufferTest {
+    @Rule
+    final public Timeout globalTimeout = Timeout.millis(120000);
+
+    @Test
+    public void testWrite() throws Exception {
+        CodeBuffer buffer = new CodeBuffer();
+        buffer.printf("public static void main(String[] args) throws Exception {%n");
+        buffer.incrementIndent();
+        buffer.printf("System.out.println(\"%s\");%n", "hello world");
+        buffer.decrementIndent();
+        buffer.printf("}%n");
+        StringWriter stringWriter = new StringWriter();
+        buffer.write(stringWriter);
+        Assert.assertEquals(
+            String.format("public static void main(String[] args) throws Exception {%n") +
+            String.format("    System.out.println(\"hello world\");%n") +
+            String.format("}%n"),
+            stringWriter.toString());
+    }
+
+    @Test
+    public void testEquals() {
+        CodeBuffer buffer1 = new CodeBuffer();
+        CodeBuffer buffer2 = new CodeBuffer();
+        Assert.assertEquals(buffer1, buffer2);
+        buffer1.printf("hello world");
+        Assert.assertNotEquals(buffer1, buffer2);
+        buffer2.printf("hello world");
+        Assert.assertEquals(buffer1, buffer2);
+        buffer1.printf("foo, bar, and baz");
+        buffer2.printf("foo, bar, and baz");
+        Assert.assertEquals(buffer1, buffer2);
+    }
+
+    @Test
+    public void testIndentMustBeNonNegative() {
+        CodeBuffer buffer = new CodeBuffer();
+        buffer.incrementIndent();
+        buffer.decrementIndent();
+        try {
+            buffer.decrementIndent();
+        } catch (RuntimeException e) {
+            Assert.assertTrue(e.getMessage().contains("Indent < 0"));
+        }
+    }
+}
diff --git a/generator/src/test/java/org/apache/kafka/message/IsNullConditionalTest.java b/generator/src/test/java/org/apache/kafka/message/IsNullConditionalTest.java
new file mode 100644
index 0000000000000..a8f7bc68c99a1
--- /dev/null
+++ b/generator/src/test/java/org/apache/kafka/message/IsNullConditionalTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.message;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+public class IsNullConditionalTest {
+    @Rule
+    final public Timeout globalTimeout = Timeout.millis(120000);
+
+    @Test
+    public void testNullCheck() throws Exception {
+        CodeBuffer buffer = new CodeBuffer();
+        IsNullConditional.
+            forName("foobar").
+            nullableVersions(Versions.parse("2+", null)).
+            possibleVersions(Versions.parse("0+", null)).
+            ifNull(() -> {
+                buffer.printf("System.out.println(\"null\");%n");
+            }).
+            generate(buffer);
+        VersionConditionalTest.assertEquals(buffer,
+            "if (foobar == null) {%n",
+            "    System.out.println(\"null\");%n",
+            "}%n");
+    }
+
+    @Test
+    public void testAnotherNullCheck() throws Exception {
+        CodeBuffer buffer = new CodeBuffer();
+        IsNullConditional.
+            forName("foobar").
+            nullableVersions(Versions.parse("0+", null)).
+            possibleVersions(Versions.parse("2+", null)).
+            ifNull(() -> {
+                buffer.printf("System.out.println(\"null\");%n");
+            }).
+            ifNotNull(() -> {
+                buffer.printf("System.out.println(\"not null\");%n");
+            }).
+            generate(buffer);
+        VersionConditionalTest.assertEquals(buffer,
+            "if (foobar == null) {%n",
+            "    System.out.println(\"null\");%n",
+            "} else {%n",
+            "    System.out.println(\"not null\");%n",
+            "}%n");
+    }
+
+    @Test
+    public void testNotNullCheck() throws Exception {
+        CodeBuffer buffer = new CodeBuffer();
+        IsNullConditional.
+            forName("foobar").
+            nullableVersions(Versions.parse("0+", null)).
+            possibleVersions(Versions.parse("2+", null)).
+            ifNotNull(() -> {
+                buffer.printf("System.out.println(\"not null\");%n");
+            }).
+            generate(buffer);
+        VersionConditionalTest.assertEquals(buffer,
+            "if (foobar != null) {%n",
+            "    System.out.println(\"not null\");%n",
+            "}%n");
+    }
+
+    @Test
+    public void testNeverNull() throws Exception {
+        CodeBuffer buffer = new CodeBuffer();
+        IsNullConditional.
+            forName("baz").
+            nullableVersions(Versions.parse("0-2", null)).
+            possibleVersions(Versions.parse("3+", null)).
+            ifNull(() -> {
+                buffer.printf("System.out.println(\"null\");%n");
+            }).
+            ifNotNull(() -> {
+                buffer.printf("System.out.println(\"not null\");%n");
+            }).
+            generate(buffer);
+        VersionConditionalTest.assertEquals(buffer,
+            "System.out.println(\"not null\");%n");
+    }
+
+    @Test
+    public void testNeverNullWithBlockScope() throws Exception {
+        CodeBuffer buffer = new CodeBuffer();
+        IsNullConditional.
+            forName("baz").
+            nullableVersions(Versions.parse("0-2", null)).
+            possibleVersions(Versions.parse("3+", null)).
+            ifNull(() -> {
+                buffer.printf("System.out.println(\"null\");%n");
+            }).
+            ifNotNull(() -> {
+                buffer.printf("System.out.println(\"not null\");%n");
+            }).
+            alwaysEmitBlockScope(true).
+            generate(buffer);
+        VersionConditionalTest.assertEquals(buffer,
+            "{%n",
+            "    System.out.println(\"not null\");%n",
+            "}%n");
+    }
+}
diff --git a/generator/src/test/java/org/apache/kafka/message/VersionConditionalTest.java b/generator/src/test/java/org/apache/kafka/message/VersionConditionalTest.java
new file mode 100644
index 0000000000000..97180f7810b4f
--- /dev/null
+++ b/generator/src/test/java/org/apache/kafka/message/VersionConditionalTest.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.message;
+
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.StringWriter;
+
+import static org.junit.Assert.assertTrue;
+
+public class VersionConditionalTest {
+    @Rule
+    final public Timeout globalTimeout = Timeout.millis(120000);
+
+    static void assertEquals(CodeBuffer buffer, String... lines) throws Exception {
+        StringWriter stringWriter = new StringWriter();
+        buffer.write(stringWriter);
+        StringBuilder expectedStringBuilder = new StringBuilder();
+        for (String line : lines) {
+            expectedStringBuilder.append(String.format(line));
+        }
+        Assert.assertEquals(expectedStringBuilder.toString(), stringWriter.toString());
+    }
+
+    @Test
+    public void testAlwaysFalseConditional() throws Exception {
+        CodeBuffer buffer = new CodeBuffer();
+        VersionConditional.
+            forVersions(Versions.parse("1-2", null), Versions.parse("3+", null)).
+            ifMember(__ -> {
+                buffer.printf("System.out.println(\"hello world\");%n");
+            }).
+            ifNotMember(__ -> {
+                buffer.printf("System.out.println(\"foobar\");%n");
+            }).
+            generate(buffer);
+        assertEquals(buffer,
+            "System.out.println(\"foobar\");%n");
+    }
+
+    @Test
+    public void testAnotherAlwaysFalseConditional() throws Exception {
+        CodeBuffer buffer = new CodeBuffer();
+        VersionConditional.
+            forVersions(Versions.parse("3+", null), Versions.parse("1-2", null)).
+            ifMember(__ -> {
+                buffer.printf("System.out.println(\"hello world\");%n");
+            }).
+            ifNotMember(__ -> {
+                buffer.printf("System.out.println(\"foobar\");%n");
+            }).
+            generate(buffer);
+        assertEquals(buffer,
+            "System.out.println(\"foobar\");%n");
+    }
+
+    @Test
+    public void testAllowMembershipCheckAlwaysFalseFails() throws Exception {
+        try {
+            CodeBuffer buffer = new CodeBuffer();
+            VersionConditional.
+                forVersions(Versions.parse("1-2", null), Versions.parse("3+", null)).
+                ifMember(__ -> {
+                    buffer.printf("System.out.println(\"hello world\");%n");
+                }).
+                ifNotMember(__ -> {
+                    buffer.printf("System.out.println(\"foobar\");%n");
+                }).
+                allowMembershipCheckAlwaysFalse(false).
+                generate(buffer);
+        } catch (RuntimeException e) {
+            assertTrue(e.getMessage().contains("no versions in common"));
+        }
+    }
+
+    @Test
+    public void testAlwaysTrueConditional() throws Exception {
+        CodeBuffer buffer = new CodeBuffer();
+        VersionConditional.
+            forVersions(Versions.parse("1-5", null), Versions.parse("2-4", null)).
+            ifMember(__ -> {
+                buffer.printf("System.out.println(\"hello world\");%n");
+            }).
+            ifNotMember(__ -> {
+                buffer.printf("System.out.println(\"foobar\");%n");
+            }).
+            allowMembershipCheckAlwaysFalse(false).
+            generate(buffer);
+        assertEquals(buffer,
+            "System.out.println(\"hello world\");%n");
+    }
+
+    @Test
+    public void testAlwaysTrueConditionalWithAlwaysEmitBlockScope() throws Exception {
+        CodeBuffer buffer = new CodeBuffer();
+        VersionConditional.
+            forVersions(Versions.parse("1-5", null), Versions.parse("2-4", null)).
+            ifMember(__ -> {
+                buffer.printf("System.out.println(\"hello world\");%n");
+            }).
+            ifNotMember(__ -> {
+                buffer.printf("System.out.println(\"foobar\");%n");
+            }).
+            alwaysEmitBlockScope(true).
+            generate(buffer);
+        assertEquals(buffer,
+            "{%n",
+            "    System.out.println(\"hello world\");%n",
+            "}%n");
+    }
+
+    @Test
+    public void testLowerRangeCheckWithElse() throws Exception {
+        CodeBuffer buffer = new CodeBuffer();
+        VersionConditional.
+            forVersions(Versions.parse("1+", null), Versions.parse("0-100", null)).
+            ifMember(__ -> {
+                buffer.printf("System.out.println(\"hello world\");%n");
+            }).
+            ifNotMember(__ -> {
+                buffer.printf("System.out.println(\"foobar\");%n");
+            }).
+            generate(buffer);
+        assertEquals(buffer,
+            "if (version >= 1) {%n",
+            "    System.out.println(\"hello world\");%n",
+            "} else {%n",
+            "    System.out.println(\"foobar\");%n",
+            "}%n");
+    }
+
+    @Test
+    public void testLowerRangeCheckWithIfMember() throws Exception {
+        CodeBuffer buffer = new CodeBuffer();
+        VersionConditional.
+            forVersions(Versions.parse("1+", null), Versions.parse("0-100", null)).
+            ifMember(__ -> {
+                buffer.printf("System.out.println(\"hello world\");%n");
+            }).
+            generate(buffer);
+        assertEquals(buffer,
+            "if (version >= 1) {%n",
+            "    System.out.println(\"hello world\");%n",
+            "}%n");
+    }
+
+    @Test
+    public void testLowerRangeCheckWithIfNotMember() throws Exception {
+        CodeBuffer buffer = new CodeBuffer();
+        VersionConditional.
+            forVersions(Versions.parse("1+", null), Versions.parse("0-100", null)).
+            ifNotMember(__ -> {
+                buffer.printf("System.out.println(\"hello world\");%n");
+            }).
+            generate(buffer);
+        assertEquals(buffer,
+            "if (version < 1) {%n",
+            "    System.out.println(\"hello world\");%n",
+            "}%n");
+    }
+
+    @Test
+    public void testUpperRangeCheckWithElse() throws Exception {
+        CodeBuffer buffer = new CodeBuffer();
+        VersionConditional.
+            forVersions(Versions.parse("0-10", null), Versions.parse("4+", null)).
+            ifMember(__ -> {
+                buffer.printf("System.out.println(\"hello world\");%n");
+            }).
+            ifNotMember(__ -> {
+                buffer.printf("System.out.println(\"foobar\");%n");
+            }).
+            generate(buffer);
+        assertEquals(buffer,
+            "if (version <= 10) {%n",
+            "    System.out.println(\"hello world\");%n",
+            "} else {%n",
+            "    System.out.println(\"foobar\");%n",
+            "}%n");
+    }
+
+    @Test
+    public void testUpperRangeCheckWithIfMember() throws Exception {
+        CodeBuffer buffer = new CodeBuffer();
+        VersionConditional.
+            forVersions(Versions.parse("0-10", null), Versions.parse("4+", null)).
+            ifMember(__ -> {
+                buffer.printf("System.out.println(\"hello world\");%n");
+            }).
+            generate(buffer);
+        assertEquals(buffer,
+            "if (version <= 10) {%n",
+            "    System.out.println(\"hello world\");%n",
+            "}%n");
+    }
+
+    @Test
+    public void testUpperRangeCheckWithIfNotMember() throws Exception {
+        CodeBuffer buffer = new CodeBuffer();
+        VersionConditional.
+            forVersions(Versions.parse("1+", null), Versions.parse("0-100", null)).
+            ifNotMember(__ -> {
+                buffer.printf("System.out.println(\"hello world\");%n");
+            }).
+            generate(buffer);
+        assertEquals(buffer,
+            "if (version < 1) {%n",
+            "    System.out.println(\"hello world\");%n",
+            "}%n");
+    }
+
+    @Test
+    public void testFullRangeCheck() throws Exception {
+        CodeBuffer buffer = new CodeBuffer();
+        VersionConditional.
+            forVersions(Versions.parse("5-10", null), Versions.parse("1+", null)).
+            ifMember(__ -> {
+                buffer.printf("System.out.println(\"hello world\");%n");
+            }).
+            allowMembershipCheckAlwaysFalse(false).
+            generate(buffer);
+        assertEquals(buffer,
+            "if ((version >= 5) && (version <= 10)) {%n",
+            "    System.out.println(\"hello world\");%n",
+            "}%n");
+    }
+}
diff --git a/generator/src/test/java/org/apache/kafka/message/VersionsTest.java b/generator/src/test/java/org/apache/kafka/message/VersionsTest.java
index 051202e46f343..f688299d18f09 100644
--- a/generator/src/test/java/org/apache/kafka/message/VersionsTest.java
+++ b/generator/src/test/java/org/apache/kafka/message/VersionsTest.java
@@ -72,6 +72,8 @@ public void testIntersections() {
         assertEquals(Versions.NONE,
             newVersions(9, Short.MAX_VALUE).intersect(
                 newVersions(2, 8)));
+        assertEquals(Versions.NONE,
+            Versions.NONE.intersect(Versions.NONE));
     }
 
     @Test
@@ -87,4 +89,26 @@ public void testContains() {
         assertTrue(newVersions(2, 3).contains(Versions.NONE));
         assertTrue(Versions.ALL.contains(newVersions(1, 2)));
     }
+
+    @Test
+    public void testSubtract() {
+        assertEquals(Versions.NONE,
+            Versions.NONE.subtract(Versions.NONE));
+        assertEquals(newVersions(0, 0),
+            newVersions(0, 0).subtract(Versions.NONE));
+        assertEquals(newVersions(1, 1),
+            newVersions(1, 2).subtract(newVersions(2, 2)));
+        assertEquals(newVersions(2, 2),
+            newVersions(1, 2).subtract(newVersions(1, 1)));
+        assertEquals(null,
+            newVersions(0, Short.MAX_VALUE).subtract(newVersions(1, 100)));
+        assertEquals(newVersions(10, 10),
+            newVersions(1, 10).subtract(newVersions(1, 9)));
+        assertEquals(newVersions(1, 1),
+            newVersions(1, 10).subtract(newVersions(2, 10)));
+        assertEquals(newVersions(2, 4),
+            newVersions(2, Short.MAX_VALUE).subtract(newVersions(5, Short.MAX_VALUE)));
+        assertEquals(newVersions(5, Short.MAX_VALUE),
+            newVersions(0, Short.MAX_VALUE).subtract(newVersions(0, 4)));
+    }
 }

From 70d1bb40d958918ac691ba906a35c4a158815d78 Mon Sep 17 00:00:00 2001
From: Yaroslav Tkachenko 
Date: Wed, 25 Sep 2019 09:23:03 -0700
Subject: [PATCH 0647/1071] KAFKA-7273: Extend Connect Converter to support
 headers (#6362)

Implemented KIP-440 to allow Connect converters to use record headers when serializing or deserializing keys and values. This change is backward compatible in that the new methods default to calling the older existing methods, so existing Converter implementations need not be changed. This changes the WorkerSinkTask and WorkerSourceTask to use the new converter methods, but Connect's existing Converter implementations and the use of converters for internal topics are intentionally not modified. Added unit tests.

Author: Yaroslav Tkachenko 
Reviewers: Ryanne Dolan , Ewen Cheslack-Postava , Randall Hauch 
---
 checkstyle/suppressions.xml                   |   3 +
 .../kafka/connect/storage/Converter.java      |  34 +++++
 .../kafka/connect/runtime/WorkerSinkTask.java |   4 +-
 .../connect/runtime/WorkerSourceTask.java     |   4 +-
 .../runtime/TestConverterWithHeaders.java     |  80 ++++++++++
 .../connect/runtime/WorkerSinkTaskTest.java   | 138 +++++++++++++++++-
 .../runtime/WorkerSinkTaskThreadedTest.java   |  17 ++-
 .../connect/runtime/WorkerSourceTaskTest.java | 128 ++++++++++++++--
 8 files changed, 384 insertions(+), 24 deletions(-)
 create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/runtime/TestConverterWithHeaders.java

diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index 050108bd83efb..4912a2593e146 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -141,6 +141,9 @@
     
 
+    
+
     
     
diff --git a/connect/api/src/main/java/org/apache/kafka/connect/storage/Converter.java b/connect/api/src/main/java/org/apache/kafka/connect/storage/Converter.java
index 507489345b9e9..29300c74eda49 100644
--- a/connect/api/src/main/java/org/apache/kafka/connect/storage/Converter.java
+++ b/connect/api/src/main/java/org/apache/kafka/connect/storage/Converter.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.connect.storage;
 
+import org.apache.kafka.common.header.Headers;
 import org.apache.kafka.connect.data.Schema;
 import org.apache.kafka.connect.data.SchemaAndValue;
 
@@ -44,6 +45,23 @@ public interface Converter {
      */
     byte[] fromConnectData(String topic, Schema schema, Object value);
 
+    /**
+     * Convert a Kafka Connect data object to a native object for serialization,
+     * potentially using the supplied topic and headers in the record as necessary.
+     *
+     * 
                Connect uses this method directly, and for backward compatibility reasons this method
+     * by default will call the {@link #fromConnectData(String, Schema, Object)} method.
+     * Override this method to make use of the supplied headers.
                
+     * @param topic the topic associated with the data
+     * @param headers the headers associated with the data
+     * @param schema the schema for the value
+     * @param value the value to convert
+     * @return the serialized value
+     */
+    default byte[] fromConnectData(String topic, Headers headers, Schema schema, Object value) {
+        return fromConnectData(topic, schema, value);
+    }
+
     /**
      * Convert a native object to a Kafka Connect data object.
      * @param topic the topic associated with the data
@@ -51,4 +69,20 @@ public interface Converter {
      * @return an object containing the {@link Schema} and the converted value
      */
     SchemaAndValue toConnectData(String topic, byte[] value);
+
+    /**
+     * Convert a native object to a Kafka Connect data object,
+     * potentially using the supplied topic and headers in the record as necessary.
+     *
+     * 
                Connect uses this method directly, and for backward compatibility reasons this method
+     * by default will call the {@link #toConnectData(String, byte[])} method.
+     * Override this method to make use of the supplied headers.
                
+     * @param topic the topic associated with the data
+     * @param headers the headers associated with the data
+     * @param value the value to convert
+     * @return an object containing the {@link Schema} and the converted value
+     */
+    default SchemaAndValue toConnectData(String topic, Headers headers, byte[] value) {
+        return toConnectData(topic, value);
+    }
 }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
index 395c93e0f772a..79760c0bf46ab 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
@@ -481,10 +481,10 @@ private void convertMessages(ConsumerRecords msgs) {
     }
 
     private SinkRecord convertAndTransformRecord(final ConsumerRecord msg) {
-        SchemaAndValue keyAndSchema = retryWithToleranceOperator.execute(() -> keyConverter.toConnectData(msg.topic(), msg.key()),
+        SchemaAndValue keyAndSchema = retryWithToleranceOperator.execute(() -> keyConverter.toConnectData(msg.topic(), msg.headers(), msg.key()),
                 Stage.KEY_CONVERTER, keyConverter.getClass());
 
-        SchemaAndValue valueAndSchema = retryWithToleranceOperator.execute(() -> valueConverter.toConnectData(msg.topic(), msg.value()),
+        SchemaAndValue valueAndSchema = retryWithToleranceOperator.execute(() -> valueConverter.toConnectData(msg.topic(), msg.headers(), msg.value()),
                 Stage.VALUE_CONVERTER, valueConverter.getClass());
 
         Headers headers = retryWithToleranceOperator.execute(() -> convertHeadersFor(msg), Stage.HEADER_CONVERTER, headerConverter.getClass());
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
index 9ca79753a4743..41b56c8900f6b 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
@@ -278,10 +278,10 @@ private ProducerRecord convertTransformedRecord(SourceRecord rec
 
         RecordHeaders headers = retryWithToleranceOperator.execute(() -> convertHeaderFor(record), Stage.HEADER_CONVERTER, headerConverter.getClass());
 
-        byte[] key = retryWithToleranceOperator.execute(() -> keyConverter.fromConnectData(record.topic(), record.keySchema(), record.key()),
+        byte[] key = retryWithToleranceOperator.execute(() -> keyConverter.fromConnectData(record.topic(), headers, record.keySchema(), record.key()),
                 Stage.KEY_CONVERTER, keyConverter.getClass());
 
-        byte[] value = retryWithToleranceOperator.execute(() -> valueConverter.fromConnectData(record.topic(), record.valueSchema(), record.value()),
+        byte[] value = retryWithToleranceOperator.execute(() -> valueConverter.fromConnectData(record.topic(), headers, record.valueSchema(), record.value()),
                 Stage.VALUE_CONVERTER, valueConverter.getClass());
 
         if (retryWithToleranceOperator.failed()) {
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/TestConverterWithHeaders.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/TestConverterWithHeaders.java
new file mode 100644
index 0000000000000..91e0999d2904e
--- /dev/null
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/TestConverterWithHeaders.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.runtime;
+
+import java.io.UnsupportedEncodingException;
+import java.util.Map;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.header.Headers;
+import org.apache.kafka.connect.data.Schema;
+import org.apache.kafka.connect.data.SchemaAndValue;
+import org.apache.kafka.connect.errors.DataException;
+import org.apache.kafka.connect.storage.Converter;
+
+/**
+ * This is a simple Converter implementation that uses "encoding" header to encode/decode strings via provided charset name
+ */
+public class TestConverterWithHeaders implements Converter {
+    private static final String HEADER_ENCODING = "encoding";
+
+    @Override
+    public void configure(Map configs, boolean isKey) {
+
+    }
+
+    @Override
+    public SchemaAndValue toConnectData(String topic, Headers headers, byte[] value) {
+        String encoding = extractEncoding(headers);
+
+        try {
+            return new SchemaAndValue(Schema.STRING_SCHEMA, new String(value, encoding));
+        } catch (UnsupportedEncodingException e) {
+            throw new DataException("Unsupported encoding: " + encoding, e);
+        }
+    }
+
+    @Override
+    public byte[] fromConnectData(String topic, Headers headers, Schema schema, Object value) {
+        String encoding = extractEncoding(headers);
+
+        try {
+            return ((String) value).getBytes(encoding);
+        } catch (UnsupportedEncodingException e) {
+            throw new DataException("Unsupported encoding: " + encoding, e);
+        }
+    }
+
+    private String extractEncoding(Headers headers) {
+        Header header = headers.lastHeader(HEADER_ENCODING);
+        if (header == null) {
+            throw new DataException("Header '" + HEADER_ENCODING + "' is required!");
+        }
+
+        return new String(header.value());
+    }
+
+
+    @Override
+    public SchemaAndValue toConnectData(String topic, byte[] value) {
+        throw new DataException("Headers are required for this converter!");
+    }
+
+    @Override
+    public byte[] fromConnectData(String topic, Schema schema, Object value) {
+        throw new DataException("Headers are required for this converter!");
+    }
+}
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskTest.java
index 4d629709efe8b..8c93528b76521 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskTest.java
@@ -16,6 +16,8 @@
  */
 package org.apache.kafka.connect.runtime;
 
+import java.util.Arrays;
+import java.util.Iterator;
 import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.clients.consumer.ConsumerRecords;
@@ -25,6 +27,9 @@
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.errors.WakeupException;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.header.Headers;
+import org.apache.kafka.common.header.internals.RecordHeaders;
 import org.apache.kafka.common.record.RecordBatch;
 import org.apache.kafka.common.record.TimestampType;
 import org.apache.kafka.common.utils.MockTime;
@@ -42,6 +47,7 @@
 import org.apache.kafka.connect.sink.SinkTask;
 import org.apache.kafka.connect.storage.Converter;
 import org.apache.kafka.connect.storage.HeaderConverter;
+import org.apache.kafka.connect.storage.StringConverter;
 import org.apache.kafka.connect.util.ConnectorTaskId;
 import org.easymock.Capture;
 import org.easymock.CaptureType;
@@ -162,6 +168,10 @@ public void setUp() {
     }
 
     private void createTask(TargetState initialState) {
+        createTask(initialState, keyConverter, valueConverter, headerConverter);
+    }
+
+    private void createTask(TargetState initialState, Converter keyConverter, Converter valueConverter, HeaderConverter headerConverter) {
         workerTask = new WorkerSinkTask(
             taskId, sinkTask, statusListener, initialState, workerConfig, ClusterConfigState.EMPTY, metrics,
             keyConverter, valueConverter, headerConverter,
@@ -1264,6 +1274,85 @@ public void testMetricsGroup() {
         assertEquals(30, metrics.currentMetricValueAsDouble(group1.metricGroup(), "put-batch-max-time-ms"), 0.001d);
     }
 
+    @Test
+    public void testHeaders() throws Exception {
+        Headers headers = new RecordHeaders();
+        headers.add("header_key", "header_value".getBytes());
+
+        createTask(initialState);
+
+        expectInitializeTask();
+        expectPollInitialAssignment();
+
+        expectConsumerPoll(1, headers);
+        expectConversionAndTransformation(1, null, headers);
+        sinkTask.put(EasyMock.>anyObject());
+        EasyMock.expectLastCall();
+
+        PowerMock.replayAll();
+
+        workerTask.initialize(TASK_CONFIG);
+        workerTask.initializeAndStart();
+        workerTask.iteration(); // iter 1 -- initial assignment
+        workerTask.iteration(); // iter 2 -- deliver 1 record
+
+        PowerMock.verifyAll();
+    }
+
+    @Test
+    public void testHeadersWithCustomConverter() throws Exception {
+        StringConverter stringConverter = new StringConverter();
+        TestConverterWithHeaders testConverter = new TestConverterWithHeaders();
+
+        createTask(initialState, stringConverter, testConverter, stringConverter);
+
+        expectInitializeTask();
+        expectPollInitialAssignment();
+
+        String keyA = "a";
+        String valueA = "Árvíztűrő tükörfúrógép";
+        Headers headersA = new RecordHeaders();
+        String encodingA = "latin2";
+        headersA.add("encoding", encodingA.getBytes());
+
+        String keyB = "b";
+        String valueB = "Тестовое сообщение";
+        Headers headersB = new RecordHeaders();
+        String encodingB = "koi8_r";
+        headersB.add("encoding", encodingB.getBytes());
+
+        expectConsumerPoll(Arrays.asList(
+            new ConsumerRecord<>(TOPIC, PARTITION, FIRST_OFFSET + recordsReturnedTp1 + 1, RecordBatch.NO_TIMESTAMP, TimestampType.NO_TIMESTAMP_TYPE,
+                0L, 0, 0, keyA.getBytes(), valueA.getBytes(encodingA), headersA),
+            new ConsumerRecord<>(TOPIC, PARTITION, FIRST_OFFSET + recordsReturnedTp1 + 2, RecordBatch.NO_TIMESTAMP, TimestampType.NO_TIMESTAMP_TYPE,
+                0L, 0, 0, keyB.getBytes(), valueB.getBytes(encodingB), headersB)
+        ));
+
+        expectTransformation(2, null);
+
+        Capture> records = EasyMock.newCapture(CaptureType.ALL);
+        sinkTask.put(EasyMock.capture(records));
+
+        PowerMock.replayAll();
+
+        workerTask.initialize(TASK_CONFIG);
+        workerTask.initializeAndStart();
+        workerTask.iteration(); // iter 1 -- initial assignment
+        workerTask.iteration(); // iter 2 -- deliver 1 record
+
+        Iterator iterator = records.getValue().iterator();
+
+        SinkRecord recordA = iterator.next();
+        assertEquals(keyA, recordA.key());
+        assertEquals(valueA, (String) recordA.value());
+
+        SinkRecord recordB = iterator.next();
+        assertEquals(keyB, recordB.key());
+        assertEquals(valueB, (String) recordB.value());
+
+        PowerMock.verifyAll();
+    }
+
     private void expectInitializeTask() throws Exception {
         consumer.subscribe(EasyMock.eq(asList(TOPIC)), EasyMock.capture(rebalanceListener));
         PowerMock.expectLastCall();
@@ -1346,17 +1435,25 @@ private void expectConsumerWakeup() {
     }
 
     private void expectConsumerPoll(final int numMessages) {
-        expectConsumerPoll(numMessages, RecordBatch.NO_TIMESTAMP, TimestampType.NO_TIMESTAMP_TYPE);
+        expectConsumerPoll(numMessages, RecordBatch.NO_TIMESTAMP, TimestampType.NO_TIMESTAMP_TYPE, emptyHeaders());
+    }
+
+    private void expectConsumerPoll(final int numMessages, Headers headers) {
+        expectConsumerPoll(numMessages, RecordBatch.NO_TIMESTAMP, TimestampType.NO_TIMESTAMP_TYPE, headers);
     }
 
     private void expectConsumerPoll(final int numMessages, final long timestamp, final TimestampType timestampType) {
+        expectConsumerPoll(numMessages, timestamp, timestampType, emptyHeaders());
+    }
+
+    private void expectConsumerPoll(final int numMessages, final long timestamp, final TimestampType timestampType, Headers headers) {
         EasyMock.expect(consumer.poll(Duration.ofMillis(EasyMock.anyLong()))).andAnswer(
                 new IAnswer>() {
                     @Override
                     public ConsumerRecords answer() throws Throwable {
                         List> records = new ArrayList<>();
                         for (int i = 0; i < numMessages; i++)
-                            records.add(new ConsumerRecord<>(TOPIC, PARTITION, FIRST_OFFSET + recordsReturnedTp1 + i, timestamp, timestampType, 0L, 0, 0, RAW_KEY, RAW_VALUE));
+                            records.add(new ConsumerRecord<>(TOPIC, PARTITION, FIRST_OFFSET + recordsReturnedTp1 + i, timestamp, timestampType, 0L, 0, 0, RAW_KEY, RAW_VALUE, headers));
                         recordsReturnedTp1 += numMessages;
                         return new ConsumerRecords<>(
                                 numMessages > 0 ?
@@ -1367,14 +1464,40 @@ public ConsumerRecords answer() throws Throwable {
                 });
     }
 
+    private void expectConsumerPoll(List> records) {
+        EasyMock.expect(consumer.poll(Duration.ofMillis(EasyMock.anyLong()))).andAnswer(
+            new IAnswer>() {
+                @Override
+                public ConsumerRecords answer() throws Throwable {
+                    return new ConsumerRecords<>(
+                        records.isEmpty() ?
+                            Collections.>>emptyMap() :
+                            Collections.singletonMap(new TopicPartition(TOPIC, PARTITION), records)
+                    );
+                }
+            });
+    }
+
     private void expectConversionAndTransformation(final int numMessages) {
         expectConversionAndTransformation(numMessages, null);
     }
 
     private void expectConversionAndTransformation(final int numMessages, final String topicPrefix) {
-        EasyMock.expect(keyConverter.toConnectData(TOPIC, RAW_KEY)).andReturn(new SchemaAndValue(KEY_SCHEMA, KEY)).times(numMessages);
-        EasyMock.expect(valueConverter.toConnectData(TOPIC, RAW_VALUE)).andReturn(new SchemaAndValue(VALUE_SCHEMA, VALUE)).times(numMessages);
+        expectConversionAndTransformation(numMessages, topicPrefix, emptyHeaders());
+    }
 
+    private void expectConversionAndTransformation(final int numMessages, final String topicPrefix, final Headers headers) {
+        EasyMock.expect(keyConverter.toConnectData(TOPIC, headers, RAW_KEY)).andReturn(new SchemaAndValue(KEY_SCHEMA, KEY)).times(numMessages);
+        EasyMock.expect(valueConverter.toConnectData(TOPIC, headers, RAW_VALUE)).andReturn(new SchemaAndValue(VALUE_SCHEMA, VALUE)).times(numMessages);
+
+        for (Header header : headers) {
+            EasyMock.expect(headerConverter.toConnectHeader(TOPIC, header.key(), header.value())).andReturn(new SchemaAndValue(VALUE_SCHEMA, new String(header.value()))).times(1);
+        }
+
+        expectTransformation(numMessages, topicPrefix);
+    }
+
+    private void expectTransformation(final int numMessages, final String topicPrefix) {
         final Capture recordCapture = EasyMock.newCapture();
         EasyMock.expect(transformationChain.apply(EasyMock.capture(recordCapture)))
                 .andAnswer(new IAnswer() {
@@ -1389,7 +1512,8 @@ public SinkRecord answer() {
                                        origRecord.key(),
                                        origRecord.valueSchema(),
                                        origRecord.value(),
-                                       origRecord.timestamp()
+                                       origRecord.timestamp(),
+                                       origRecord.headers()
                                )
                                : origRecord;
                     }
@@ -1472,6 +1596,10 @@ private void assertMetrics(int minimumPollCountExpected) {
         double sendTotal = metrics.currentMetricValueAsDouble(sinkTaskGroup, "sink-record-send-total");
     }
 
+    private RecordHeaders emptyHeaders() {
+        return new RecordHeaders();
+    }
+
     private abstract static class TestSinkTask extends SinkTask  {
     }
 }
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskThreadedTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskThreadedTest.java
index 6e2b01ce7bc19..ab20dce2d26bd 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskThreadedTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskThreadedTest.java
@@ -23,6 +23,7 @@
 import org.apache.kafka.clients.consumer.OffsetAndMetadata;
 import org.apache.kafka.clients.consumer.OffsetCommitCallback;
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.header.internals.RecordHeaders;
 import org.apache.kafka.common.record.TimestampType;
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.connect.data.Schema;
@@ -572,8 +573,8 @@ public ConsumerRecords answer() throws Throwable {
                         return records;
                     }
                 });
-        EasyMock.expect(keyConverter.toConnectData(TOPIC, RAW_KEY)).andReturn(new SchemaAndValue(KEY_SCHEMA, KEY)).anyTimes();
-        EasyMock.expect(valueConverter.toConnectData(TOPIC, RAW_VALUE)).andReturn(new SchemaAndValue(VALUE_SCHEMA, VALUE)).anyTimes();
+        EasyMock.expect(keyConverter.toConnectData(TOPIC, emptyHeaders(), RAW_KEY)).andReturn(new SchemaAndValue(KEY_SCHEMA, KEY)).anyTimes();
+        EasyMock.expect(valueConverter.toConnectData(TOPIC, emptyHeaders(), RAW_VALUE)).andReturn(new SchemaAndValue(VALUE_SCHEMA, VALUE)).anyTimes();
 
         final Capture recordCapture = EasyMock.newCapture();
         EasyMock.expect(transformationChain.apply(EasyMock.capture(recordCapture))).andAnswer(
@@ -606,8 +607,8 @@ public ConsumerRecords answer() throws Throwable {
                         return records;
                     }
                 });
-        EasyMock.expect(keyConverter.toConnectData(TOPIC, RAW_KEY)).andReturn(new SchemaAndValue(KEY_SCHEMA, KEY));
-        EasyMock.expect(valueConverter.toConnectData(TOPIC, RAW_VALUE)).andReturn(new SchemaAndValue(VALUE_SCHEMA, VALUE));
+        EasyMock.expect(keyConverter.toConnectData(TOPIC, emptyHeaders(), RAW_KEY)).andReturn(new SchemaAndValue(KEY_SCHEMA, KEY));
+        EasyMock.expect(valueConverter.toConnectData(TOPIC, emptyHeaders(), RAW_VALUE)).andReturn(new SchemaAndValue(VALUE_SCHEMA, VALUE));
         sinkTask.put(EasyMock.anyObject(Collection.class));
         return EasyMock.expectLastCall();
     }
@@ -651,8 +652,8 @@ public ConsumerRecords answer() throws Throwable {
         consumer.seek(TOPIC_PARTITION, startOffset);
         EasyMock.expectLastCall();
 
-        EasyMock.expect(keyConverter.toConnectData(TOPIC, RAW_KEY)).andReturn(new SchemaAndValue(KEY_SCHEMA, KEY));
-        EasyMock.expect(valueConverter.toConnectData(TOPIC, RAW_VALUE)).andReturn(new SchemaAndValue(VALUE_SCHEMA, VALUE));
+        EasyMock.expect(keyConverter.toConnectData(TOPIC, emptyHeaders(), RAW_KEY)).andReturn(new SchemaAndValue(KEY_SCHEMA, KEY));
+        EasyMock.expect(valueConverter.toConnectData(TOPIC, emptyHeaders(), RAW_VALUE)).andReturn(new SchemaAndValue(VALUE_SCHEMA, VALUE));
         sinkTask.put(EasyMock.anyObject(Collection.class));
         return EasyMock.expectLastCall();
     }
@@ -694,6 +695,10 @@ public Object answer() throws Throwable {
         return capturedCallback;
     }
 
+    private RecordHeaders emptyHeaders() {
+        return new RecordHeaders();
+    }
+
     private static abstract class TestSinkTask extends SinkTask {
     }
 }
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java
index 1f5cc438ef716..272a6d56e6a21 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java
@@ -16,15 +16,21 @@
  */
 package org.apache.kafka.connect.runtime;
 
+import java.nio.ByteBuffer;
 import org.apache.kafka.clients.producer.KafkaProducer;
 import org.apache.kafka.clients.producer.ProducerRecord;
 import org.apache.kafka.clients.producer.RecordMetadata;
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.header.Headers;
+import org.apache.kafka.common.header.internals.RecordHeaders;
 import org.apache.kafka.common.errors.TopicAuthorizationException;
 import org.apache.kafka.common.record.InvalidRecordException;
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.connect.data.Schema;
+import org.apache.kafka.connect.data.SchemaAndValue;
+import org.apache.kafka.connect.header.ConnectHeaders;
 import org.apache.kafka.connect.errors.ConnectException;
 import org.apache.kafka.connect.runtime.ConnectMetrics.MetricGroup;
 import org.apache.kafka.connect.runtime.WorkerSourceTask.SourceTaskMetricsGroup;
@@ -39,6 +45,7 @@
 import org.apache.kafka.connect.storage.HeaderConverter;
 import org.apache.kafka.connect.storage.OffsetStorageReader;
 import org.apache.kafka.connect.storage.OffsetStorageWriter;
+import org.apache.kafka.connect.storage.StringConverter;
 import org.apache.kafka.connect.util.Callback;
 import org.apache.kafka.connect.util.ConnectorTaskId;
 import org.apache.kafka.connect.util.ThreadedTest;
@@ -152,6 +159,10 @@ private void createWorkerTask() {
     }
 
     private void createWorkerTask(TargetState initialState) {
+        createWorkerTask(initialState, keyConverter, valueConverter, headerConverter);
+    }
+
+    private void createWorkerTask(TargetState initialState, Converter keyConverter, Converter valueConverter, HeaderConverter headerConverter) {
         workerTask = new WorkerSourceTask(taskId, sourceTask, statusListener, initialState, keyConverter, valueConverter, headerConverter,
                 transformationChain, producer, offsetReader, offsetWriter, config, clusterConfigState, metrics, plugins.delegatingLoader(), Time.SYSTEM,
                 RetryWithToleranceOperatorTest.NOOP_OPERATOR);
@@ -682,6 +693,80 @@ public void testMetricsGroup() {
         assertEquals(1800.0, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-active-count"), 0.001d);
     }
 
+    @Test
+    public void testHeaders() throws Exception {
+        Headers headers = new RecordHeaders();
+        headers.add("header_key", "header_value".getBytes());
+
+        org.apache.kafka.connect.header.Headers connectHeaders = new ConnectHeaders();
+        connectHeaders.add("header_key", new SchemaAndValue(Schema.STRING_SCHEMA, "header_value"));
+
+        createWorkerTask();
+
+        List records = new ArrayList<>();
+        records.add(new SourceRecord(PARTITION, OFFSET, "topic", null, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD, null, connectHeaders));
+
+        Capture> sent = expectSendRecord(true, false, true, true, true, headers);
+
+        PowerMock.replayAll();
+
+        Whitebox.setInternalState(workerTask, "toSend", records);
+        Whitebox.invokeMethod(workerTask, "sendRecords");
+        assertEquals(SERIALIZED_KEY, sent.getValue().key());
+        assertEquals(SERIALIZED_RECORD, sent.getValue().value());
+        assertEquals(headers, sent.getValue().headers());
+
+        PowerMock.verifyAll();
+    }
+
+    @Test
+    public void testHeadersWithCustomConverter() throws Exception {
+        StringConverter stringConverter = new StringConverter();
+        TestConverterWithHeaders testConverter = new TestConverterWithHeaders();
+
+        createWorkerTask(TargetState.STARTED, stringConverter, testConverter, stringConverter);
+
+        List records = new ArrayList<>();
+
+        String stringA = "Árvíztűrő tükörfúrógép";
+        org.apache.kafka.connect.header.Headers headersA = new ConnectHeaders();
+        String encodingA = "latin2";
+        headersA.addString("encoding", encodingA);
+
+        records.add(new SourceRecord(PARTITION, OFFSET, "topic", null, Schema.STRING_SCHEMA, "a", Schema.STRING_SCHEMA, stringA, null, headersA));
+
+        String stringB = "Тестовое сообщение";
+        org.apache.kafka.connect.header.Headers headersB = new ConnectHeaders();
+        String encodingB = "koi8_r";
+        headersB.addString("encoding", encodingB);
+
+        records.add(new SourceRecord(PARTITION, OFFSET, "topic", null, Schema.STRING_SCHEMA, "b", Schema.STRING_SCHEMA, stringB, null, headersB));
+
+        Capture> sentRecordA = expectSendRecord(false, false, true, true, false, null);
+        Capture> sentRecordB = expectSendRecord(false, false, true, true, false, null);
+
+        PowerMock.replayAll();
+
+        Whitebox.setInternalState(workerTask, "toSend", records);
+        Whitebox.invokeMethod(workerTask, "sendRecords");
+
+        assertEquals(ByteBuffer.wrap("a".getBytes()), ByteBuffer.wrap(sentRecordA.getValue().key()));
+        assertEquals(
+            ByteBuffer.wrap(stringA.getBytes(encodingA)),
+            ByteBuffer.wrap(sentRecordA.getValue().value())
+        );
+        assertEquals(encodingA, new String(sentRecordA.getValue().headers().lastHeader("encoding").value()));
+
+        assertEquals(ByteBuffer.wrap("b".getBytes()), ByteBuffer.wrap(sentRecordB.getValue().key()));
+        assertEquals(
+            ByteBuffer.wrap(stringB.getBytes(encodingB)),
+            ByteBuffer.wrap(sentRecordB.getValue().value())
+        );
+        assertEquals(encodingB, new String(sentRecordB.getValue().headers().lastHeader("encoding").value()));
+
+        PowerMock.verifyAll();
+    }
+
     private CountDownLatch expectPolls(int minimum, final AtomicInteger count) throws InterruptedException {
         final CountDownLatch latch = new CountDownLatch(minimum);
         // Note that we stub these to allow any number of calls because the thread will continue to
@@ -708,7 +793,7 @@ private CountDownLatch expectPolls(int count) throws InterruptedException {
 
     @SuppressWarnings("unchecked")
     private void expectSendRecordSyncFailure(Throwable error) throws InterruptedException {
-        expectConvertKeyValue(false);
+        expectConvertHeadersAndKeyValue(false);
         expectApplyTransformationChain(false);
 
         offsetWriter.offset(PARTITION, OFFSET);
@@ -729,24 +814,34 @@ private Capture> expectSendRecordOnce(boolean isR
     }
 
     private Capture> expectSendRecordProducerCallbackFail() throws InterruptedException {
-        return expectSendRecord(false, false, false, false);
+        return expectSendRecord(false, false, false, false, true, emptyHeaders());
     }
 
     private Capture> expectSendRecordTaskCommitRecordSucceed(boolean anyTimes, boolean isRetry) throws InterruptedException {
-        return expectSendRecord(anyTimes, isRetry, true, true);
+        return expectSendRecord(anyTimes, isRetry, true, true, true, emptyHeaders());
     }
 
     private Capture> expectSendRecordTaskCommitRecordFail(boolean anyTimes, boolean isRetry) throws InterruptedException {
-        return expectSendRecord(anyTimes, isRetry, true, false);
+        return expectSendRecord(anyTimes, isRetry, true, false, true, emptyHeaders());
+    }
+
+    private Capture> expectSendRecord(boolean anyTimes, boolean isRetry, boolean succeed) throws InterruptedException {
+        return expectSendRecord(anyTimes, isRetry, succeed, true, true, emptyHeaders());
     }
 
+    @SuppressWarnings("unchecked")
     private Capture> expectSendRecord(
         boolean anyTimes,
         boolean isRetry,
         boolean sendSuccess,
-        boolean commitSuccess
+        boolean commitSuccess,
+        boolean isMockedConverters,
+        Headers headers
     ) throws InterruptedException {
-        expectConvertKeyValue(anyTimes);
+        if (isMockedConverters) {
+            expectConvertHeadersAndKeyValue(anyTimes, headers);
+        }
+
         expectApplyTransformationChain(anyTimes);
 
         Capture> sent = EasyMock.newCapture();
@@ -794,13 +889,24 @@ public Future answer() throws Throwable {
         return sent;
     }
 
-    private void expectConvertKeyValue(boolean anyTimes) {
-        IExpectationSetters convertKeyExpect = EasyMock.expect(keyConverter.fromConnectData(TOPIC, KEY_SCHEMA, KEY));
+    private void expectConvertHeadersAndKeyValue(boolean anyTimes) {
+        expectConvertHeadersAndKeyValue(anyTimes, emptyHeaders());
+    }
+
+    private void expectConvertHeadersAndKeyValue(boolean anyTimes, Headers headers) {
+        for (Header header : headers) {
+            IExpectationSetters convertHeaderExpect = EasyMock.expect(headerConverter.fromConnectHeader(TOPIC, header.key(), Schema.STRING_SCHEMA, new String(header.value())));
+            if (anyTimes)
+                convertHeaderExpect.andStubReturn(header.value());
+            else
+                convertHeaderExpect.andReturn(header.value());
+        }
+        IExpectationSetters convertKeyExpect = EasyMock.expect(keyConverter.fromConnectData(TOPIC, headers, KEY_SCHEMA, KEY));
         if (anyTimes)
             convertKeyExpect.andStubReturn(SERIALIZED_KEY);
         else
             convertKeyExpect.andReturn(SERIALIZED_KEY);
-        IExpectationSetters convertValueExpect = EasyMock.expect(valueConverter.fromConnectData(TOPIC, RECORD_SCHEMA, RECORD));
+        IExpectationSetters convertValueExpect = EasyMock.expect(valueConverter.fromConnectData(TOPIC, headers, RECORD_SCHEMA, RECORD));
         if (anyTimes)
             convertValueExpect.andStubReturn(SERIALIZED_RECORD);
         else
@@ -902,6 +1008,10 @@ private void assertPollMetrics(int minimumPollCountExpected) {
         }
     }
 
+    private RecordHeaders emptyHeaders() {
+        return new RecordHeaders();
+    }
+
     private abstract static class TestSourceTask extends SourceTask {
     }
 

From 0566dfd3cedc13e2c0357378cc100c1ec9bd585a Mon Sep 17 00:00:00 2001
From: Colin Patrick McCabe 
Date: Wed, 25 Sep 2019 19:35:13 -0700
Subject: [PATCH 0648/1071] MINOR: add versioning to request and response
 headers (#7372)

Add a version number to request and response headers.  The header
version is determined by the first two 16 bit fields read (API key and
API version).  For now, ControlledShutdown v0 has header version 0, and
all other requests have v1.  Once KIP-482 is implemented, there will be
a v2 of the header which supports tagged fields.
---
 .../apache/kafka/clients/ClientRequest.java   |  10 +-
 .../apache/kafka/clients/NetworkClient.java   |   2 +-
 .../apache/kafka/common/protocol/ApiKeys.java |   8 +
 .../kafka/common/protocol/Protocol.java       |  30 ++--
 .../common/requests/AbstractResponse.java     |  14 +-
 .../requests/ControlledShutdownRequest.java   |  14 +-
 .../common/requests/DeleteGroupsRequest.java  |  30 ++--
 .../common/requests/DeleteGroupsResponse.java |   2 +-
 .../common/requests/HeartbeatRequest.java     |  19 +--
 .../common/requests/ListGroupsRequest.java    |  25 +--
 .../common/requests/OffsetFetchRequest.java   |  20 +--
 .../kafka/common/requests/RequestHeader.java  | 143 ++++++------------
 .../kafka/common/requests/ResponseHeader.java |  44 +++---
 .../SaslClientAuthenticator.java              |  10 +-
 .../common/message/RequestHeader.json         |   7 +-
 .../common/message/ResponseHeader.json        |   3 +-
 .../kafka/clients/NetworkClientTest.java      |  44 ++++--
 .../consumer/internals/FetcherTest.java       |   8 +-
 .../producer/internals/SenderTest.java        |   7 +-
 .../common/requests/RequestContextTest.java   |   2 +-
 .../common/requests/RequestHeaderTest.java    |  18 +--
 .../common/requests/RequestResponseTest.java  |  33 ++--
 .../authenticator/SaslAuthenticatorTest.java  |  10 +-
 .../unit/kafka/server/BaseRequestTest.scala   |  21 +--
 .../kafka/server/EdgeCaseRequestTest.scala    |   6 +-
 .../unit/kafka/server/FetchRequestTest.scala  |   4 +-
 .../unit/kafka/server/KafkaApisTest.scala     |   2 +-
 27 files changed, 251 insertions(+), 285 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/ClientRequest.java b/clients/src/main/java/org/apache/kafka/clients/ClientRequest.java
index 7b44ca3ad3c44..a160102db1dd8 100644
--- a/clients/src/main/java/org/apache/kafka/clients/ClientRequest.java
+++ b/clients/src/main/java/org/apache/kafka/clients/ClientRequest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.clients;
 
+import org.apache.kafka.common.message.RequestHeaderData;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.requests.AbstractRequest;
 import org.apache.kafka.common.requests.RequestHeader;
@@ -82,7 +83,14 @@ public ApiKeys apiKey() {
     }
 
     public RequestHeader makeHeader(short version) {
-        return new RequestHeader(apiKey(), version, clientId, correlationId);
+        short requestApiKey = requestBuilder.apiKey().id;
+        return new RequestHeader(
+            new RequestHeaderData().
+                setRequestApiKey(requestApiKey).
+                setRequestApiVersion(version).
+                setClientId(clientId).
+                setCorrelationId(correlationId),
+            ApiKeys.forId(requestApiKey).headerVersion(version));
     }
 
     public AbstractRequest.Builder requestBuilder() {
diff --git a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
index 047b03585c018..44c9fd3c07247 100644
--- a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
@@ -705,7 +705,7 @@ public static AbstractResponse parseResponse(ByteBuffer responseBuffer, RequestH
 
     private static Struct parseStructMaybeUpdateThrottleTimeMetrics(ByteBuffer responseBuffer, RequestHeader requestHeader,
                                                                     Sensor throttleTimeSensor, long now) {
-        ResponseHeader responseHeader = ResponseHeader.parse(responseBuffer);
+        ResponseHeader responseHeader = ResponseHeader.parse(responseBuffer, requestHeader.headerVersion());
         // Always expect the response version id to be the same as the request version id
         Struct responseBody = requestHeader.apiKey().parseResponse(requestHeader.apiVersion(), responseBuffer);
         correlate(requestHeader, responseHeader);
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
index b5c1e0dbd6d8d..cd0132723c47b 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
@@ -335,6 +335,14 @@ public boolean isVersionSupported(short apiVersion) {
         return apiVersion >= oldestVersion() && apiVersion <= latestVersion();
     }
 
+    public short headerVersion(short apiVersion) {
+        if ((this == CONTROLLED_SHUTDOWN) && (apiVersion == 0)) {
+            return 0;
+        } else {
+            return 1;
+        }
+    }
+
     private static String toHtml() {
         final StringBuilder b = new StringBuilder();
         b.append("
                file.delete.delay.ms
                \n");
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java b/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java
index 65b5966dc66a9..0e0975ccb8313 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java
@@ -16,11 +16,11 @@
  */
 package org.apache.kafka.common.protocol;
 
+import org.apache.kafka.common.message.RequestHeaderData;
+import org.apache.kafka.common.message.ResponseHeaderData;
 import org.apache.kafka.common.protocol.types.BoundField;
 import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Type;
-import org.apache.kafka.common.requests.RequestHeader;
-import org.apache.kafka.common.requests.ResponseHeader;
 
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
@@ -116,18 +116,20 @@ public static String toHtml() {
         final StringBuilder b = new StringBuilder();
         b.append("
                Headers:
                \n");
 
-        b.append("
                ");
-        b.append("Request Header => ");
-        schemaToBnfHtml(RequestHeader.SCHEMA, b, 2);
-        b.append("
                \n");
-        schemaToFieldTableHtml(RequestHeader.SCHEMA, b);
-
-        b.append("
                ");
-        b.append("Response Header => ");
-        schemaToBnfHtml(ResponseHeader.SCHEMA, b, 2);
-        b.append("
                \n");
-        schemaToFieldTableHtml(ResponseHeader.SCHEMA, b);
-
+        for (int i = 0; i < RequestHeaderData.SCHEMAS.length; i++) {
+            b.append("
                ");
+            b.append("Request Header v").append(i).append(" => ");
+            schemaToBnfHtml(RequestHeaderData.SCHEMAS[i], b, 2);
+            b.append("
                \n");
+            schemaToFieldTableHtml(RequestHeaderData.SCHEMAS[i], b);
+        }
+        for (int i = 0; i < ResponseHeaderData.SCHEMAS.length; i++) {
+            b.append("
                ");
+            b.append("Response Header v").append(i).append(" => ");
+            schemaToBnfHtml(ResponseHeaderData.SCHEMAS[i], b, 2);
+            b.append("
                \n");
+            schemaToFieldTableHtml(ResponseHeaderData.SCHEMAS[i], b);
+        }
         for (ApiKeys key : ApiKeys.values()) {
             // Key
             b.append("
                ");
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
index 64002aac211e9..9fbce91259946 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
@@ -31,14 +31,22 @@ public abstract class AbstractResponse extends AbstractRequestResponse {
     public static final int DEFAULT_THROTTLE_TIME = 0;
 
     protected Send toSend(String destination, ResponseHeader header, short apiVersion) {
-        return new NetworkSend(destination, serialize(apiVersion, header));
+        return new NetworkSend(destination, serialize(header.toStruct(), toStruct(apiVersion)));
     }
 
     /**
      * Visible for testing, typically {@link #toSend(String, ResponseHeader, short)} should be used instead.
      */
-    public ByteBuffer serialize(short version, ResponseHeader responseHeader) {
-        return serialize(responseHeader.toStruct(), toStruct(version));
+    public ByteBuffer serialize(ApiKeys apiKey, int correlationId) {
+        return serialize(apiKey, apiKey.latestVersion(), correlationId);
+    }
+
+    /**
+     * Visible for testing, typically {@link #toSend(String, ResponseHeader, short)} should be used instead.
+     */
+    public ByteBuffer serialize(ApiKeys apiKey, short version, int correlationId) {
+        ResponseHeader header = new ResponseHeader(correlationId, apiKey.headerVersion(version));
+        return serialize(header.toStruct(), toStruct(version));
     }
 
     public abstract Map errorCounts();
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java
index f89316aa30938..83e9444da0ad1 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java
@@ -64,18 +64,8 @@ public ControlledShutdownRequest(Struct struct, short version) {
 
     @Override
     public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
-        ControlledShutdownResponseData response = new ControlledShutdownResponseData();
-        response.setErrorCode(Errors.forException(e).code());
-        short versionId = version();
-        switch (versionId) {
-            case 0:
-            case 1:
-            case 2:
-                return new ControlledShutdownResponse(response);
-            default:
-                throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
-                    versionId, this.getClass().getSimpleName(), ApiKeys.CONTROLLED_SHUTDOWN.latestVersion()));
-        }
+        return new ControlledShutdownResponse(new ControlledShutdownResponseData().
+            setErrorCode(Errors.forException(e).code()));
     }
 
     public static ControlledShutdownRequest parse(ByteBuffer buffer, short version) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java
index d3061c4844891..2ad947c5904be 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java
@@ -61,26 +61,18 @@ public DeleteGroupsRequest(Struct struct, short version) {
     @Override
     public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
         Errors error = Errors.forException(e);
-
-        switch (version()) {
-            case 0:
-            case 1:
-                DeletableGroupResultCollection groupResults = new DeletableGroupResultCollection();
-                for (String groupId : data.groupsNames()) {
-                    groupResults.add(new DeletableGroupResult()
-                                         .setGroupId(groupId)
-                                         .setErrorCode(error.code()));
-                }
-
-                return new DeleteGroupsResponse(
-                    new DeleteGroupsResponseData()
-                        .setResults(groupResults)
-                        .setThrottleTimeMs(throttleTimeMs)
-                );
-            default:
-                throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
-                    version(), ApiKeys.DELETE_GROUPS.name, ApiKeys.DELETE_GROUPS.latestVersion()));
+        DeletableGroupResultCollection groupResults = new DeletableGroupResultCollection();
+        for (String groupId : data.groupsNames()) {
+            groupResults.add(new DeletableGroupResult()
+                                 .setGroupId(groupId)
+                                 .setErrorCode(error.code()));
         }
+
+        return new DeleteGroupsResponse(
+            new DeleteGroupsResponseData()
+                .setResults(groupResults)
+                .setThrottleTimeMs(throttleTimeMs)
+        );
     }
 
     public static DeleteGroupsRequest parse(ByteBuffer buffer, short version) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java
index 5fe86eb8f1649..a98a7c881d96e 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java
@@ -86,7 +86,7 @@ public Map errorCounts() {
     }
 
     public static DeleteGroupsResponse parse(ByteBuffer buffer, short version) {
-        return new DeleteGroupsResponse(ApiKeys.DELETE_GROUPS.parseResponse(version, buffer));
+        return new DeleteGroupsResponse(ApiKeys.DELETE_GROUPS.parseResponse(version, buffer), version);
     }
 
     @Override
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java
index ff034df34f13c..e5cc6c8860a58 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java
@@ -65,21 +65,12 @@ public HeartbeatRequest(Struct struct, short version) {
 
     @Override
     public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
-        HeartbeatResponseData response = new HeartbeatResponseData();
-        response.setErrorCode(Errors.forException(e).code());
-        short versionId = version();
-        switch (versionId) {
-            case 0:
-                return new HeartbeatResponse(response);
-            case 1:
-            case 2:
-            case 3:
-                response.setThrottleTimeMs(throttleTimeMs);
-                return new HeartbeatResponse(response);
-            default:
-                throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
-                        versionId, this.getClass().getSimpleName(), ApiKeys.HEARTBEAT.latestVersion()));
+        HeartbeatResponseData responseData = new HeartbeatResponseData().
+            setErrorCode(Errors.forException(e).code());
+        if (version() >= 1) {
+            responseData.setThrottleTimeMs(throttleTimeMs);
         }
+        return new HeartbeatResponse(responseData);
     }
 
     public static HeartbeatRequest parse(ByteBuffer buffer, short version) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java
index b0e8860f8c108..caa6f47906bfd 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java
@@ -68,24 +68,13 @@ public ListGroupsRequest(Struct struct, short version) {
 
     @Override
     public ListGroupsResponse getErrorResponse(int throttleTimeMs, Throwable e) {
-        short versionId = version();
-        switch (versionId) {
-            case 0:
-                return new ListGroupsResponse(new ListGroupsResponseData()
-                        .setGroups(Collections.emptyList())
-                        .setErrorCode(Errors.forException(e).code())
-                );
-            case 1:
-            case 2:
-                return new ListGroupsResponse(new ListGroupsResponseData()
-                        .setGroups(Collections.emptyList())
-                        .setErrorCode(Errors.forException(e).code())
-                        .setThrottleTimeMs(throttleTimeMs)
-                );
-            default:
-                throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
-                        versionId, this.getClass().getSimpleName(), ApiKeys.LIST_GROUPS.latestVersion()));
+        ListGroupsResponseData listGroupsResponseData = new ListGroupsResponseData().
+            setGroups(Collections.emptyList()).
+            setErrorCode(Errors.forException(e).code());
+        if (version() >= 1) {
+            listGroupsResponseData.setThrottleTimeMs(throttleTimeMs);
         }
+        return new ListGroupsResponse(listGroupsResponseData);
     }
 
     public static ListGroupsRequest parse(ByteBuffer buffer, short version) {
@@ -94,6 +83,6 @@ public static ListGroupsRequest parse(ByteBuffer buffer, short version) {
 
     @Override
     protected Struct toStruct() {
-        return new Struct(ApiKeys.LIST_GROUPS.requestSchema(version()));
+        return data.toStruct(version());
     }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java
index f616ac1df810a..2303f89f91984 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java
@@ -118,10 +118,8 @@ public OffsetFetchResponse getErrorResponse(Errors error) {
     }
 
     public OffsetFetchResponse getErrorResponse(int throttleTimeMs, Errors error) {
-        short versionId = version();
-
         Map responsePartitions = new HashMap<>();
-        if (versionId < 2) {
+        if (version() < 2) {
             OffsetFetchResponse.PartitionData partitionError = new OffsetFetchResponse.PartitionData(
                     OffsetFetchResponse.INVALID_OFFSET,
                     Optional.empty(),
@@ -136,18 +134,10 @@ public OffsetFetchResponse getErrorResponse(int throttleTimeMs, Errors error) {
             }
         }
 
-        switch (versionId) {
-            case 0:
-            case 1:
-            case 2:
-                return new OffsetFetchResponse(error, responsePartitions);
-            case 3:
-            case 4:
-            case 5:
-                return new OffsetFetchResponse(throttleTimeMs, error, responsePartitions);
-            default:
-                throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
-                        versionId, this.getClass().getSimpleName(), ApiKeys.OFFSET_FETCH.latestVersion()));
+        if (version() >= 3) {
+            return new OffsetFetchResponse(throttleTimeMs, error, responsePartitions);
+        } else {
+            return new OffsetFetchResponse(error, responsePartitions);
         }
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java
index e5ec43acebe8d..65bc39d230a24 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java
@@ -17,123 +17,94 @@
 package org.apache.kafka.common.requests;
 
 import org.apache.kafka.common.errors.InvalidRequestException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.RequestHeaderData;
 import org.apache.kafka.common.protocol.ApiKeys;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
+import org.apache.kafka.common.protocol.ByteBufferAccessor;
 import org.apache.kafka.common.protocol.types.Struct;
 
 import java.nio.ByteBuffer;
-import java.util.Objects;
-
-import static java.util.Objects.requireNonNull;
-import static org.apache.kafka.common.protocol.types.Type.INT16;
-import static org.apache.kafka.common.protocol.types.Type.INT32;
-import static org.apache.kafka.common.protocol.types.Type.NULLABLE_STRING;
 
 /**
  * The header for a request in the Kafka protocol
  */
 public class RequestHeader extends AbstractRequestResponse {
-    private static final String API_KEY_FIELD_NAME = "api_key";
-    private static final String API_VERSION_FIELD_NAME = "api_version";
-    private static final String CLIENT_ID_FIELD_NAME = "client_id";
-    private static final String CORRELATION_ID_FIELD_NAME = "correlation_id";
-
-    public static final Schema SCHEMA = new Schema(
-            new Field(API_KEY_FIELD_NAME, INT16, "The id of the request type."),
-            new Field(API_VERSION_FIELD_NAME, INT16, "The version of the API."),
-            new Field(CORRELATION_ID_FIELD_NAME, INT32, "A user-supplied integer value that will be passed back with the response"),
-            new Field(CLIENT_ID_FIELD_NAME, NULLABLE_STRING, "A user specified identifier for the client making the request.", ""));
-
-    // Version 0 of the controlled shutdown API used a non-standard request header (the clientId is missing).
-    // This can be removed once we drop support for that version.
-    private static final Schema CONTROLLED_SHUTDOWN_V0_SCHEMA = new Schema(
-            new Field(API_KEY_FIELD_NAME, INT16, "The id of the request type."),
-            new Field(API_VERSION_FIELD_NAME, INT16, "The version of the API."),
-            new Field(CORRELATION_ID_FIELD_NAME, INT32, "A user-supplied integer value that will be passed back with the response"));
-
-    private final ApiKeys apiKey;
-    private final short apiVersion;
-    private final String clientId;
-    private final int correlationId;
-
-    public RequestHeader(Struct struct) {
-        short apiKey = struct.getShort(API_KEY_FIELD_NAME);
-        if (!ApiKeys.hasId(apiKey))
-            throw new InvalidRequestException("Unknown API key " + apiKey);
-
-        this.apiKey = ApiKeys.forId(apiKey);
-        apiVersion = struct.getShort(API_VERSION_FIELD_NAME);
-
-        // only v0 of the controlled shutdown request is missing the clientId
-        if (struct.hasField(CLIENT_ID_FIELD_NAME))
-            clientId = struct.getString(CLIENT_ID_FIELD_NAME);
-        else
-            clientId = "";
-        correlationId = struct.getInt(CORRELATION_ID_FIELD_NAME);
+    private final RequestHeaderData data;
+    private final short headerVersion;
+
+    public RequestHeader(Struct struct, short headerVersion) {
+        this(new RequestHeaderData(struct, headerVersion), headerVersion);
+    }
+
+    public RequestHeader(ApiKeys requestApiKey, short requestVersion, String clientId, int correlationId) {
+        this(new RequestHeaderData().
+                setRequestApiKey(requestApiKey.id).
+                setRequestApiVersion(requestVersion).
+                setClientId(clientId).
+                setCorrelationId(correlationId),
+            ApiKeys.forId(requestApiKey.id).headerVersion(requestVersion));
     }
 
-    public RequestHeader(ApiKeys apiKey, short version, String clientId, int correlation) {
-        this.apiKey = requireNonNull(apiKey);
-        this.apiVersion = version;
-        this.clientId = clientId;
-        this.correlationId = correlation;
+    public RequestHeader(RequestHeaderData data, short headerVersion) {
+        this.data = data;
+        this.headerVersion = headerVersion;
     }
 
     public Struct toStruct() {
-        Schema schema = schema(apiKey.id, apiVersion);
-        Struct struct = new Struct(schema);
-        struct.set(API_KEY_FIELD_NAME, apiKey.id);
-        struct.set(API_VERSION_FIELD_NAME, apiVersion);
-
-        // only v0 of the controlled shutdown request is missing the clientId
-        if (struct.hasField(CLIENT_ID_FIELD_NAME))
-            struct.set(CLIENT_ID_FIELD_NAME, clientId);
-        struct.set(CORRELATION_ID_FIELD_NAME, correlationId);
-        return struct;
+        return this.data.toStruct(headerVersion);
     }
 
     public ApiKeys apiKey() {
-        return apiKey;
+        return ApiKeys.forId(data.requestApiKey());
     }
 
     public short apiVersion() {
-        return apiVersion;
+        return data.requestApiVersion();
+    }
+
+    public short headerVersion() {
+        return headerVersion;
     }
 
     public String clientId() {
-        return clientId;
+        return data.clientId();
     }
 
     public int correlationId() {
-        return correlationId;
+        return data.correlationId();
+    }
+
+    public RequestHeaderData data() {
+        return data;
     }
 
     public ResponseHeader toResponseHeader() {
-        return new ResponseHeader(correlationId);
+        return new ResponseHeader(data.correlationId(), headerVersion);
     }
 
     public static RequestHeader parse(ByteBuffer buffer) {
+        short apiKey = -1;
         try {
-            short apiKey = buffer.getShort();
+            apiKey = buffer.getShort();
             short apiVersion = buffer.getShort();
-            Schema schema = schema(apiKey, apiVersion);
+            short headerVersion = ApiKeys.forId(apiKey).headerVersion(apiVersion);
             buffer.rewind();
-            return new RequestHeader(schema.read(buffer));
-        } catch (InvalidRequestException e) {
-            throw e;
-        } catch (Throwable  ex) {
+            return new RequestHeader(new RequestHeaderData(
+                new ByteBufferAccessor(buffer), headerVersion), headerVersion);
+        } catch (UnsupportedVersionException e) {
+            throw new InvalidRequestException("Unknown API key " + apiKey, e);
+        } catch (Throwable ex) {
             throw new InvalidRequestException("Error parsing request header. Our best guess of the apiKey is: " +
-                    buffer.getShort(0), ex);
+                    apiKey, ex);
         }
     }
 
     @Override
     public String toString() {
-        return "RequestHeader(apiKey=" + apiKey +
-                ", apiVersion=" + apiVersion +
-                ", clientId=" + clientId +
-                ", correlationId=" + correlationId +
+        return "RequestHeader(apiKey=" + apiKey() +
+                ", apiVersion=" + apiVersion() +
+                ", clientId=" + clientId() +
+                ", correlationId=" + correlationId() +
                 ")";
     }
 
@@ -141,28 +112,12 @@ public String toString() {
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
-
         RequestHeader that = (RequestHeader) o;
-        return apiKey == that.apiKey &&
-                apiVersion == that.apiVersion &&
-                correlationId == that.correlationId &&
-                Objects.equals(clientId, that.clientId);
+        return this.data.equals(that.data);
     }
 
     @Override
     public int hashCode() {
-        int result = apiKey.hashCode();
-        result = 31 * result + (int) apiVersion;
-        result = 31 * result + (clientId != null ? clientId.hashCode() : 0);
-        result = 31 * result + correlationId;
-        return result;
-    }
-
-    private static Schema schema(short apiKey, short version) {
-        if (apiKey == ApiKeys.CONTROLLED_SHUTDOWN.id && version == 0)
-            // This will be removed once we remove support for v0 of ControlledShutdownRequest, which
-            // depends on a non-standard request header (it does not have a clientId)
-            return CONTROLLED_SHUTDOWN_V0_SCHEMA;
-        return SCHEMA;
+        return this.data.hashCode();
     }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java b/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java
index fe452a2dc0808..249b5d021081f 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java
@@ -16,31 +16,30 @@
  */
 package org.apache.kafka.common.requests;
 
-import org.apache.kafka.common.protocol.types.BoundField;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
+import org.apache.kafka.common.message.ResponseHeaderData;
+import org.apache.kafka.common.protocol.ByteBufferAccessor;
 import org.apache.kafka.common.protocol.types.Struct;
 
 import java.nio.ByteBuffer;
 
-import static org.apache.kafka.common.protocol.types.Type.INT32;
-
 /**
  * A response header in the kafka protocol.
  */
 public class ResponseHeader extends AbstractRequestResponse {
-    public static final Schema SCHEMA = new Schema(
-            new Field("correlation_id", INT32, "The user-supplied value passed in with the request"));
-    private static final BoundField CORRELATION_KEY_FIELD = SCHEMA.get("correlation_id");
+    private final ResponseHeaderData data;
+    private final short headerVersion;
 
-    private final int correlationId;
+    public ResponseHeader(Struct struct, short headerVersion) {
+        this(new ResponseHeaderData(struct, headerVersion), headerVersion);
+    }
 
-    public ResponseHeader(Struct struct) {
-        correlationId = struct.getInt(CORRELATION_KEY_FIELD);
+    public ResponseHeader(int correlationId, short headerVersion) {
+        this(new ResponseHeaderData().setCorrelationId(correlationId), headerVersion);
     }
 
-    public ResponseHeader(int correlationId) {
-        this.correlationId = correlationId;
+    public ResponseHeader(ResponseHeaderData data, short headerVersion) {
+        this.data = data;
+        this.headerVersion = headerVersion;
     }
 
     public int sizeOf() {
@@ -48,17 +47,24 @@ public int sizeOf() {
     }
 
     public Struct toStruct() {
-        Struct struct = new Struct(SCHEMA);
-        struct.set(CORRELATION_KEY_FIELD, correlationId);
-        return struct;
+        return data.toStruct(headerVersion);
     }
 
     public int correlationId() {
-        return correlationId;
+        return this.data.correlationId();
     }
 
-    public static ResponseHeader parse(ByteBuffer buffer) {
-        return new ResponseHeader(SCHEMA.read(buffer));
+    public short headerVersion() {
+        return headerVersion;
     }
 
+    public ResponseHeaderData data() {
+        return data;
+    }
+
+    public static ResponseHeader parse(ByteBuffer buffer, short headerVersion) {
+        return new ResponseHeader(
+            new ResponseHeaderData(new ByteBufferAccessor(buffer), headerVersion),
+                headerVersion);
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
index 3133f44a88f3f..4bf56e80b3f68 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
@@ -23,6 +23,7 @@
 import org.apache.kafka.common.errors.IllegalSaslStateException;
 import org.apache.kafka.common.errors.SaslAuthenticationException;
 import org.apache.kafka.common.errors.UnsupportedSaslMechanismException;
+import org.apache.kafka.common.message.RequestHeaderData;
 import org.apache.kafka.common.message.SaslAuthenticateRequestData;
 import org.apache.kafka.common.message.SaslHandshakeRequestData;
 import org.apache.kafka.common.network.Authenticator;
@@ -324,7 +325,14 @@ public Long reauthenticationLatencyMs() {
 
     private RequestHeader nextRequestHeader(ApiKeys apiKey, short version) {
         String clientId = (String) configs.get(CommonClientConfigs.CLIENT_ID_CONFIG);
-        currentRequestHeader = new RequestHeader(apiKey, version, clientId, correlationId++);
+        short requestApiKey = apiKey.id;
+        currentRequestHeader = new RequestHeader(
+            new RequestHeaderData().
+                setRequestApiKey(requestApiKey).
+                setRequestApiVersion(version).
+                setClientId(clientId).
+                setCorrelationId(correlationId++),
+            apiKey.headerVersion(version));
         return currentRequestHeader;
     }
 
diff --git a/clients/src/main/resources/common/message/RequestHeader.json b/clients/src/main/resources/common/message/RequestHeader.json
index d24dcf0e98755..a00658cc866de 100644
--- a/clients/src/main/resources/common/message/RequestHeader.json
+++ b/clients/src/main/resources/common/message/RequestHeader.json
@@ -16,7 +16,10 @@
 {
   "type": "header",
   "name": "RequestHeader",
-  "validVersions": "0",
+  // Version 0 of the RequestHeader is only used by v0 of ControlledShutdownRequest.
+  //
+  // Version 1 is the first version with ClientId.
+  "validVersions": "0-1",
   "fields": [
     { "name": "RequestApiKey", "type": "int16", "versions": "0+",
       "about": "The API key of this request." },
@@ -24,7 +27,7 @@
       "about": "The API version of this request." },
     { "name": "CorrelationId", "type": "int32", "versions": "0+",
       "about": "The correlation ID of this request." },
-    { "name": "ClientId", "type": "string", "versions": "0+",
+    { "name": "ClientId", "type": "string", "versions": "1+", "nullableVersions": "1+", "ignorable": true,
       "about": "The client ID string." }
   ]
 }
diff --git a/clients/src/main/resources/common/message/ResponseHeader.json b/clients/src/main/resources/common/message/ResponseHeader.json
index d44825977da7d..bc47b3f05f07c 100644
--- a/clients/src/main/resources/common/message/ResponseHeader.json
+++ b/clients/src/main/resources/common/message/ResponseHeader.json
@@ -16,7 +16,8 @@
 {
   "type": "header",
   "name": "ResponseHeader",
-  "validVersions": "0",
+  // Version 0 and version 1 are the same.
+  "validVersions": "0-1",
   "fields": [
     { "name": "CorrelationId", "type": "int32", "versions": "0+",
       "about": "The correlation ID of this response." }
diff --git a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
index b5bba716296b0..ab11b95520f98 100644
--- a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
@@ -45,6 +45,7 @@
 import java.util.Collections;
 import java.util.List;
 
+import static org.apache.kafka.common.protocol.ApiKeys.PRODUCE;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
@@ -155,16 +156,23 @@ public void testUnsupportedVersionDuringInternalMetadataRequest() {
 
     private void checkSimpleRequestResponse(NetworkClient networkClient) {
         awaitReady(networkClient, node); // has to be before creating any request, as it may send ApiVersionsRequest and its response is mocked with correlation id 0
-        ProduceRequest.Builder builder = ProduceRequest.Builder.forCurrentMagic((short) 1, 1000,
-                        Collections.emptyMap());
+        ProduceRequest.Builder builder = new ProduceRequest.Builder(
+                PRODUCE.latestVersion(),
+                PRODUCE.latestVersion(),
+                (short) 1,
+                1000,
+                Collections.emptyMap(),
+                null);
         TestCallbackHandler handler = new TestCallbackHandler();
         ClientRequest request = networkClient.newClientRequest(
                 node.idString(), builder, time.milliseconds(), true, defaultRequestTimeoutMs, handler);
         networkClient.send(request, time.milliseconds());
         networkClient.poll(1, time.milliseconds());
         assertEquals(1, networkClient.inFlightRequestCount());
-        ResponseHeader respHeader = new ResponseHeader(request.correlationId());
-        Struct resp = new Struct(ApiKeys.PRODUCE.responseSchema(ApiKeys.PRODUCE.latestVersion()));
+        ResponseHeader respHeader =
+            new ResponseHeader(request.correlationId(),
+                request.apiKey().headerVersion(PRODUCE.latestVersion()));
+        Struct resp = new Struct(PRODUCE.responseSchema(PRODUCE.latestVersion()));
         resp.set("responses", new Object[0]);
         Struct responseHeaderStruct = respHeader.toStruct();
         int size = responseHeaderStruct.sizeOf() + resp.sizeOf();
@@ -183,7 +191,7 @@ private void checkSimpleRequestResponse(NetworkClient networkClient) {
 
     private void setExpectedApiVersionsResponse(ApiVersionsResponse response) {
         short apiVersionsResponseVersion = response.apiVersion(ApiKeys.API_VERSIONS.id).maxVersion;
-        ByteBuffer buffer = response.serialize(apiVersionsResponseVersion, new ResponseHeader(0));
+        ByteBuffer buffer = response.serialize(ApiKeys.API_VERSIONS, apiVersionsResponseVersion, 0);
         selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer)));
     }
 
@@ -235,15 +243,22 @@ private void testRequestTimeout(ClientRequest request) {
     public void testConnectionThrottling() {
         // Instrument the test to return a response with a 100ms throttle delay.
         awaitReady(client, node);
-        ProduceRequest.Builder builder = ProduceRequest.Builder.forCurrentMagic((short) 1, 1000,
-            Collections.emptyMap());
+        ProduceRequest.Builder builder = new ProduceRequest.Builder(
+            PRODUCE.latestVersion(),
+            PRODUCE.latestVersion(),
+            (short) 1,
+            1000,
+            Collections.emptyMap(),
+            null);
         TestCallbackHandler handler = new TestCallbackHandler();
         ClientRequest request = client.newClientRequest(node.idString(), builder, time.milliseconds(), true,
                 defaultRequestTimeoutMs, handler);
         client.send(request, time.milliseconds());
         client.poll(1, time.milliseconds());
-        ResponseHeader respHeader = new ResponseHeader(request.correlationId());
-        Struct resp = new Struct(ApiKeys.PRODUCE.responseSchema(ApiKeys.PRODUCE.latestVersion()));
+        ResponseHeader respHeader =
+            new ResponseHeader(request.correlationId(),
+                request.apiKey().headerVersion(PRODUCE.latestVersion()));
+        Struct resp = new Struct(PRODUCE.responseSchema(PRODUCE.latestVersion()));
         resp.set("responses", new Object[0]);
         resp.set(CommonFields.THROTTLE_TIME_MS, 100);
         Struct responseHeaderStruct = respHeader.toStruct();
@@ -288,7 +303,7 @@ private ApiVersionsResponse createExpectedApiVersionsResponse(ApiKeys key, short
     public void testThrottlingNotEnabledForConnectionToOlderBroker() {
         // Instrument the test so that the max protocol version for PRODUCE returned from the node is 5 and thus
         // client-side throttling is not enabled. Also, return a response with a 100ms throttle delay.
-        setExpectedApiVersionsResponse(createExpectedApiVersionsResponse(ApiKeys.PRODUCE, (short) 5));
+        setExpectedApiVersionsResponse(createExpectedApiVersionsResponse(PRODUCE, (short) 5));
         while (!client.ready(node, time.milliseconds()))
             client.poll(1, time.milliseconds());
         selector.clear();
@@ -316,8 +331,7 @@ private int sendEmptyProduceRequest() {
     }
 
 
-    private void sendResponse(int correlationId, Struct response) {
-        ResponseHeader respHeader = new ResponseHeader(correlationId);
+    private void sendResponse(ResponseHeader respHeader, Struct response) {
         Struct responseHeaderStruct = respHeader.toStruct();
         int size = responseHeaderStruct.sizeOf() + response.sizeOf();
         ByteBuffer buffer = ByteBuffer.allocate(size);
@@ -328,10 +342,12 @@ private void sendResponse(int correlationId, Struct response) {
     }
 
     private void sendThrottledProduceResponse(int correlationId, int throttleMs) {
-        Struct resp = new Struct(ApiKeys.PRODUCE.responseSchema(ApiKeys.PRODUCE.latestVersion()));
+        Struct resp = new Struct(PRODUCE.responseSchema(PRODUCE.latestVersion()));
         resp.set("responses", new Object[0]);
         resp.set(CommonFields.THROTTLE_TIME_MS, throttleMs);
-        sendResponse(correlationId, resp);
+        sendResponse(new ResponseHeader(correlationId,
+            PRODUCE.headerVersion(PRODUCE.latestVersion())),
+            resp);
     }
 
     @Test
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
index 9e2621af53445..f2246d90d74b0 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
@@ -77,7 +77,6 @@
 import org.apache.kafka.common.requests.MetadataResponse;
 import org.apache.kafka.common.requests.OffsetsForLeaderEpochRequest;
 import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse;
-import org.apache.kafka.common.requests.ResponseHeader;
 import org.apache.kafka.common.serialization.ByteArrayDeserializer;
 import org.apache.kafka.common.serialization.BytesDeserializer;
 import org.apache.kafka.common.serialization.Deserializer;
@@ -1971,8 +1970,9 @@ public void testQuotaMetrics() {
                 1000, 1000, 64 * 1024, 64 * 1024, 1000,  ClientDnsLookup.DEFAULT,
                 time, true, new ApiVersions(), throttleTimeSensor, new LogContext());
 
-        short apiVersionsResponseVersion = ApiKeys.API_VERSIONS.latestVersion();
-        ByteBuffer buffer = ApiVersionsResponse.createApiVersionsResponse(400, RecordBatch.CURRENT_MAGIC_VALUE).serialize(apiVersionsResponseVersion, new ResponseHeader(0));
+        ByteBuffer buffer = ApiVersionsResponse.
+            createApiVersionsResponse(400, RecordBatch.CURRENT_MAGIC_VALUE).
+                serialize(ApiKeys.API_VERSIONS, 0);
         selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer)));
         while (!client.ready(node, time.milliseconds())) {
             client.poll(1, time.milliseconds());
@@ -1989,7 +1989,7 @@ public void testQuotaMetrics() {
             client.send(request, time.milliseconds());
             client.poll(1, time.milliseconds());
             FetchResponse response = fullFetchResponse(tp0, nextRecords, Errors.NONE, i, throttleTimeMs);
-            buffer = response.serialize(ApiKeys.FETCH.latestVersion(), new ResponseHeader(request.correlationId()));
+            buffer = response.serialize(ApiKeys.FETCH, request.correlationId());
             selector.completeReceive(new NetworkReceive(node.idString(), buffer));
             client.poll(1, time.milliseconds());
             // If a throttled response is received, advance the time to ensure progress.
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java
index 4df89455bdd63..653faf21aa0a4 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java
@@ -64,7 +64,6 @@
 import org.apache.kafka.common.requests.MetadataResponse;
 import org.apache.kafka.common.requests.ProduceRequest;
 import org.apache.kafka.common.requests.ProduceResponse;
-import org.apache.kafka.common.requests.ResponseHeader;
 import org.apache.kafka.common.requests.TransactionResult;
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.common.utils.MockTime;
@@ -277,8 +276,8 @@ public void testQuotaMetrics() throws Exception {
                 1000, 1000, 64 * 1024, 64 * 1024, 1000,  ClientDnsLookup.DEFAULT,
                 time, true, new ApiVersions(), throttleTimeSensor, logContext);
 
-        short apiVersionsResponseVersion = ApiKeys.API_VERSIONS.latestVersion();
-        ByteBuffer buffer = ApiVersionsResponse.createApiVersionsResponse(400, RecordBatch.CURRENT_MAGIC_VALUE).serialize(apiVersionsResponseVersion, new ResponseHeader(0));
+        ByteBuffer buffer = ApiVersionsResponse.createApiVersionsResponse(400, RecordBatch.CURRENT_MAGIC_VALUE).
+            serialize(ApiKeys.API_VERSIONS, 0);
         selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer)));
         while (!client.ready(node, time.milliseconds())) {
             client.poll(1, time.milliseconds());
@@ -295,7 +294,7 @@ public void testQuotaMetrics() throws Exception {
             client.send(request, time.milliseconds());
             client.poll(1, time.milliseconds());
             ProduceResponse response = produceResponse(tp0, i, Errors.NONE, throttleTimeMs);
-            buffer = response.serialize(ApiKeys.PRODUCE.latestVersion(), new ResponseHeader(request.correlationId()));
+            buffer = response.serialize(ApiKeys.PRODUCE, request.correlationId());
             selector.completeReceive(new NetworkReceive(node.idString(), buffer));
             client.poll(1, time.milliseconds());
             // If a throttled response is received, advance the time to ensure progress.
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java
index 7c24d1f2e1d04..d9ac2c2d0d77f 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java
@@ -64,7 +64,7 @@ public void testSerdeUnsupportedApiVersionRequest() throws Exception {
         responseBuffer.flip();
         responseBuffer.getInt(); // strip off the size
 
-        ResponseHeader responseHeader = ResponseHeader.parse(responseBuffer);
+        ResponseHeader responseHeader = ResponseHeader.parse(responseBuffer, header.headerVersion());
         assertEquals(correlationId, responseHeader.correlationId());
 
         Struct struct = ApiKeys.API_VERSIONS.parseResponse((short) 0, responseBuffer);
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java
index f73ee2f3823b1..dd9dc327fc93b 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java
@@ -43,6 +43,7 @@ public void testSerdeControlledShutdownV0() {
         assertEquals(0, deserialized.apiVersion());
         assertEquals(correlationId, deserialized.correlationId());
         assertEquals("", deserialized.clientId());
+        assertEquals(0, deserialized.headerVersion());
 
         Struct serialized = deserialized.toStruct();
         ByteBuffer serializedBuffer = toBuffer(serialized);
@@ -54,23 +55,12 @@ public void testSerdeControlledShutdownV0() {
     }
 
     @Test
-    public void testRequestHeader() {
+    public void testRequestHeaderV1() {
         RequestHeader header = new RequestHeader(ApiKeys.FIND_COORDINATOR, (short) 1, "", 10);
+        assertEquals(1, header.headerVersion());
         ByteBuffer buffer = toBuffer(header.toStruct());
+        assertEquals(10, buffer.remaining());
         RequestHeader deserialized = RequestHeader.parse(buffer);
         assertEquals(header, deserialized);
     }
-
-    @Test
-    public void testRequestHeaderWithNullClientId() {
-        RequestHeader header = new RequestHeader(ApiKeys.FIND_COORDINATOR, (short) 1, null, 10);
-        Struct headerStruct = header.toStruct();
-        ByteBuffer buffer = toBuffer(headerStruct);
-        RequestHeader deserialized = RequestHeader.parse(buffer);
-        assertEquals(header.apiKey(), deserialized.apiKey());
-        assertEquals(header.apiVersion(), deserialized.apiVersion());
-        assertEquals(header.correlationId(), deserialized.correlationId());
-        assertEquals("", deserialized.clientId()); // null defaults to ""
-    }
-
 }
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index d813cbdd9b1ce..52d918a991f90 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -144,6 +144,7 @@
 import java.util.Set;
 
 import static java.util.Arrays.asList;
+import static org.apache.kafka.common.protocol.ApiKeys.FETCH;
 import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID;
 import static org.apache.kafka.test.TestUtils.toBuffer;
 import static org.junit.Assert.assertEquals;
@@ -393,14 +394,14 @@ public void testSerialization() throws Exception {
 
     @Test
     public void testResponseHeader() {
-        ResponseHeader header = createResponseHeader();
+        ResponseHeader header = createResponseHeader((short) 1);
         ByteBuffer buffer = toBuffer(header.toStruct());
-        ResponseHeader deserialized = ResponseHeader.parse(buffer);
+        ResponseHeader deserialized = ResponseHeader.parse(buffer, header.headerVersion());
         assertEquals(header.correlationId(), deserialized.correlationId());
     }
 
     private void checkOlderFetchVersions() throws Exception {
-        int latestVersion = ApiKeys.FETCH.latestVersion();
+        int latestVersion = FETCH.latestVersion();
         for (int i = 0; i < latestVersion; ++i) {
             checkErrorResponse(createFetchRequest(i), new UnknownServerException(), true);
             checkRequest(createFetchRequest(i), true);
@@ -547,11 +548,12 @@ public void produceResponseV5Test() {
 
         ProduceResponse v5Response = new ProduceResponse(responseData, 10);
         short version = 5;
+        short headerVersion = ApiKeys.PRODUCE.headerVersion(version);
 
-        ByteBuffer buffer = v5Response.serialize(version, new ResponseHeader(0));
+        ByteBuffer buffer = v5Response.serialize(ApiKeys.PRODUCE, version, 0);
         buffer.rewind();
 
-        ResponseHeader.parse(buffer); // throw away.
+        ResponseHeader.parse(buffer, headerVersion); // throw away.
 
         Struct deserializedStruct = ApiKeys.PRODUCE.parseResponse(version, buffer);
 
@@ -602,9 +604,9 @@ public void fetchResponseVersionTest() {
         FetchResponse v1Response = new FetchResponse<>(Errors.NONE, responseData, 10, INVALID_SESSION_ID);
         assertEquals("Throttle time must be zero", 0, v0Response.throttleTimeMs());
         assertEquals("Throttle time must be 10", 10, v1Response.throttleTimeMs());
-        assertEquals("Should use schema version 0", ApiKeys.FETCH.responseSchema((short) 0),
+        assertEquals("Should use schema version 0", FETCH.responseSchema((short) 0),
                 v0Response.toStruct((short) 0).schema());
-        assertEquals("Should use schema version 1", ApiKeys.FETCH.responseSchema((short) 1),
+        assertEquals("Should use schema version 1", FETCH.responseSchema((short) 1),
                 v1Response.toStruct((short) 1).schema());
         assertEquals("Response data does not match", responseData, v0Response.responseData());
         assertEquals("Response data does not match", responseData, v1Response.responseData());
@@ -633,10 +635,10 @@ public void testFetchResponseV4() {
 
     @Test
     public void verifyFetchResponseFullWrites() throws Exception {
-        verifyFetchResponseFullWrite(ApiKeys.FETCH.latestVersion(), createFetchResponse(123));
-        verifyFetchResponseFullWrite(ApiKeys.FETCH.latestVersion(),
+        verifyFetchResponseFullWrite(FETCH.latestVersion(), createFetchResponse(123));
+        verifyFetchResponseFullWrite(FETCH.latestVersion(),
             createFetchResponse(Errors.FETCH_SESSION_ID_NOT_FOUND, 123));
-        for (short version = 0; version <= ApiKeys.FETCH.latestVersion(); version++) {
+        for (short version = 0; version <= FETCH.latestVersion(); version++) {
             verifyFetchResponseFullWrite(version, createFetchResponse());
         }
     }
@@ -644,7 +646,8 @@ public void verifyFetchResponseFullWrites() throws Exception {
     private void verifyFetchResponseFullWrite(short apiVersion, FetchResponse fetchResponse) throws Exception {
         int correlationId = 15;
 
-        Send send = fetchResponse.toSend("1", new ResponseHeader(correlationId), apiVersion);
+        short headerVersion = FETCH.headerVersion(apiVersion);
+        Send send = fetchResponse.toSend("1", new ResponseHeader(correlationId, headerVersion), apiVersion);
         ByteBufferChannel channel = new ByteBufferChannel(send.size());
         send.writeTo(channel);
         channel.close();
@@ -656,11 +659,11 @@ private void verifyFetchResponseFullWrite(short apiVersion, FetchResponse fetchR
         assertTrue(size > 0);
 
         // read the header
-        ResponseHeader responseHeader = ResponseHeader.parse(channel.buffer());
+        ResponseHeader responseHeader = ResponseHeader.parse(channel.buffer(), headerVersion);
         assertEquals(correlationId, responseHeader.correlationId());
 
         // read the body
-        Struct responseBody = ApiKeys.FETCH.responseSchema(apiVersion).read(buf);
+        Struct responseBody = FETCH.responseSchema(apiVersion).read(buf);
         assertEquals(fetchResponse.toStruct(apiVersion), responseBody);
 
         assertEquals(size, responseHeader.sizeOf() + responseBody.sizeOf());
@@ -757,8 +760,8 @@ public void testOffsetFetchRequestBuilderToString() {
         assertTrue(string.contains("group1"));
     }
 
-    private ResponseHeader createResponseHeader() {
-        return new ResponseHeader(10);
+    private ResponseHeader createResponseHeader(short headerVersion) {
+        return new ResponseHeader(10, headerVersion);
     }
 
     private FindCoordinatorRequest createFindCoordinatorRequest(int version) {
diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
index fc4eb031b35d6..82a1f97926472 100644
--- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
@@ -53,6 +53,7 @@
 import org.apache.kafka.common.config.internals.BrokerSecurityConfigs;
 import org.apache.kafka.common.config.types.Password;
 import org.apache.kafka.common.errors.SaslAuthenticationException;
+import org.apache.kafka.common.message.RequestHeaderData;
 import org.apache.kafka.common.message.SaslAuthenticateRequestData;
 import org.apache.kafka.common.message.SaslHandshakeRequestData;
 import org.apache.kafka.common.network.CertStores;
@@ -680,11 +681,16 @@ public void testApiVersionsRequestWithUnsupportedVersion() throws Exception {
         // Send ApiVersionsRequest with unsupported version and validate error response.
         String node = "1";
         createClientConnection(SecurityProtocol.PLAINTEXT, node);
-        RequestHeader header = new RequestHeader(ApiKeys.API_VERSIONS, Short.MAX_VALUE, "someclient", 1);
+
+        RequestHeader header = new RequestHeader(new RequestHeaderData().
+                setRequestApiKey(ApiKeys.API_VERSIONS.id).
+                setRequestApiVersion(Short.MAX_VALUE).
+                setClientId("someclient").
+                setCorrelationId(1), (short) 1);
         ApiVersionsRequest request = new ApiVersionsRequest.Builder().build();
         selector.send(request.toSend(node, header));
         ByteBuffer responseBuffer = waitForResponse();
-        ResponseHeader.parse(responseBuffer);
+        ResponseHeader.parse(responseBuffer, header.headerVersion());
         ApiVersionsResponse response = ApiVersionsResponse.parse(responseBuffer, (short) 0);
         assertEquals(Errors.UNSUPPORTED_VERSION, response.error());
 
diff --git a/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala b/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala
index faeb34df132a0..2062fb762028d 100644
--- a/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala
@@ -127,8 +127,8 @@ abstract class BaseRequestTest extends IntegrationTestHarness {
   /**
     * Serializes and sends the request to the given api.
     */
-  def send(request: AbstractRequest, apiKey: ApiKeys, socket: Socket, apiVersion: Option[Short] = None): Unit = {
-    val header = nextRequestHeader(apiKey, apiVersion.getOrElse(request.version))
+  def send(request: AbstractRequest, apiKey: ApiKeys, socket: Socket, apiVersion: Short): Unit = {
+    val header = nextRequestHeader(apiKey, apiVersion)
     val serializedBytes = request.serialize(header).array
     sendRequest(socket, serializedBytes)
   }
@@ -136,9 +136,9 @@ abstract class BaseRequestTest extends IntegrationTestHarness {
   /**
    * Receive response and return a ByteBuffer containing response without the header
    */
-  def receive(socket: Socket): ByteBuffer = {
+  def receive(socket: Socket, headerVersion: Short): ByteBuffer = {
     val response = receiveResponse(socket)
-    skipResponseHeader(response)
+    skipResponseHeader(response, headerVersion)
   }
 
   /**
@@ -146,9 +146,10 @@ abstract class BaseRequestTest extends IntegrationTestHarness {
     * A ByteBuffer containing the response is returned.
     */
   def sendAndReceive(request: AbstractRequest, apiKey: ApiKeys, socket: Socket, apiVersion: Option[Short] = None): ByteBuffer = {
-    send(request, apiKey, socket, apiVersion)
+    val version = apiVersion.getOrElse(request.version)
+    send(request, apiKey, socket, version)
     val response = receiveResponse(socket)
-    skipResponseHeader(response)
+    skipResponseHeader(response, apiKey.headerVersion(version))
   }
 
   /**
@@ -159,7 +160,7 @@ abstract class BaseRequestTest extends IntegrationTestHarness {
     val request = requestBuilder.build()
     val header = new RequestHeader(apiKey, request.version, clientId, correlationId)
     val response = requestAndReceive(socket, request.serialize(header).array)
-    val responseBuffer = skipResponseHeader(response)
+    val responseBuffer = skipResponseHeader(response, header.headerVersion())
     apiKey.parseResponse(request.version, responseBuffer)
   }
   
@@ -171,13 +172,13 @@ abstract class BaseRequestTest extends IntegrationTestHarness {
     val header = nextRequestHeader(apiKey, apiVersion)
     val serializedBytes = AbstractRequestResponse.serialize(header.toStruct, requestStruct).array
     val response = requestAndReceive(socket, serializedBytes)
-    skipResponseHeader(response)
+    skipResponseHeader(response, header.headerVersion())
   }
 
-  protected def skipResponseHeader(response: Array[Byte]): ByteBuffer = {
+  protected def skipResponseHeader(response: Array[Byte], headerVersion: Short): ByteBuffer = {
     val responseBuffer = ByteBuffer.wrap(response)
     // Parse the header to ensure its valid and move the buffer forward
-    ResponseHeader.parse(responseBuffer)
+    ResponseHeader.parse(responseBuffer, headerVersion)
     responseBuffer
   }
 
diff --git a/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala b/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala
index c267f04e8e116..42409db72c24d 100755
--- a/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala
@@ -116,7 +116,7 @@ class EdgeCaseRequestTest extends KafkaServerTestHarness {
     createTopic(topic, numPartitions = 1, replicationFactor = 1)
 
     val version = ApiKeys.PRODUCE.latestVersion: Short
-    val serializedBytes = {
+    val (serializedBytes, headerVersion) = {
       val headerBytes = requestHeaderBytes(ApiKeys.PRODUCE.id, version, null,
         correlationId)
       val records = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("message".getBytes))
@@ -124,13 +124,13 @@ class EdgeCaseRequestTest extends KafkaServerTestHarness {
       val byteBuffer = ByteBuffer.allocate(headerBytes.length + request.toStruct.sizeOf)
       byteBuffer.put(headerBytes)
       request.toStruct.writeTo(byteBuffer)
-      byteBuffer.array()
+      (byteBuffer.array(), request.api.headerVersion(version))
     }
 
     val response = requestAndReceive(serializedBytes)
 
     val responseBuffer = ByteBuffer.wrap(response)
-    val responseHeader = ResponseHeader.parse(responseBuffer)
+    val responseHeader = ResponseHeader.parse(responseBuffer, headerVersion)
     val produceResponse = ProduceResponse.parse(responseBuffer, version)
 
     assertEquals("The response should parse completely", 0, responseBuffer.remaining)
diff --git a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
index 0363963de7f53..23091c8b08e49 100644
--- a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
@@ -279,7 +279,7 @@ class FetchRequestTest extends BaseRequestTest {
 
       val socket = connect(brokerSocketServer(leaderId))
       try {
-        send(fetchRequest, ApiKeys.FETCH, socket)
+        send(fetchRequest, ApiKeys.FETCH, socket, fetchRequest.api.headerVersion(fetchRequest.version()))
         if (closeAfterPartialResponse) {
           // read some data to ensure broker has muted this channel and then close socket
           val size = new DataInputStream(socket.getInputStream).readInt()
@@ -290,7 +290,7 @@ class FetchRequestTest extends BaseRequestTest {
               size > maxPartitionBytes - batchSize)
           None
         } else {
-          Some(FetchResponse.parse(receive(socket), version))
+          Some(FetchResponse.parse(receive(socket, ApiKeys.FETCH.headerVersion(version)), version))
         }
       } finally {
         socket.close()
diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
index 06fc534d2b78a..3c37bc02b0ea4 100644
--- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
@@ -797,7 +797,7 @@ class KafkaApisTest {
     send.writeTo(channel)
     channel.close()
     channel.buffer.getInt() // read the size
-    ResponseHeader.parse(channel.buffer)
+    ResponseHeader.parse(channel.buffer, sendResponse.request.header.headerVersion())
     val struct = api.responseSchema(request.version).read(channel.buffer)
     AbstractResponse.parseResponse(api, struct, request.version)
   }

From 666505b72f1baaf5f37fac4f59fa26d93d833f63 Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Thu, 26 Sep 2019 08:28:05 +0100
Subject: [PATCH 0649/1071] MINOR: Address review comments for KIP-504
 authorizer changes (#7379)

Reviewers: Manikumar Reddy 
---
 .../org/apache/kafka/common/Endpoint.java     | 21 ++++++-----
 .../kafka/common/requests/RequestContext.java |  2 +-
 .../AuthorizableRequestContext.java           |  2 +-
 .../src/main/scala/kafka/cluster/Broker.scala | 20 +++++------
 .../main/scala/kafka/cluster/EndPoint.scala   |  9 +++--
 .../scala/kafka/network/SocketServer.scala    |  7 ++--
 .../security/auth/SimpleAclAuthorizer.scala   | 33 ++++++++---------
 .../security/authorizer/AclAuthorizer.scala   | 36 +++++++++----------
 .../security/authorizer/AuthorizerUtils.scala |  2 +-
 .../main/scala/kafka/server/KafkaServer.scala |  2 +-
 .../api/SslAdminClientIntegrationTest.scala   |  2 +-
 .../unit/kafka/network/SocketServerTest.scala |  4 +--
 12 files changed, 72 insertions(+), 68 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/Endpoint.java b/clients/src/main/java/org/apache/kafka/common/Endpoint.java
index 963d3aea987b7..2353de26ec4df 100644
--- a/clients/src/main/java/org/apache/kafka/common/Endpoint.java
+++ b/clients/src/main/java/org/apache/kafka/common/Endpoint.java
@@ -17,6 +17,8 @@
 package org.apache.kafka.common;
 
 import java.util.Objects;
+import java.util.Optional;
+
 import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.common.security.auth.SecurityProtocol;
 
@@ -27,23 +29,24 @@
 @InterfaceStability.Evolving
 public class Endpoint {
 
-    private final String listener;
+    private final String listenerName;
     private final SecurityProtocol securityProtocol;
     private final String host;
     private final int port;
 
-    public Endpoint(String listener, SecurityProtocol securityProtocol, String host, int port) {
-        this.listener = listener;
+    public Endpoint(String listenerName, SecurityProtocol securityProtocol, String host, int port) {
+        this.listenerName = listenerName;
         this.securityProtocol = securityProtocol;
         this.host = host;
         this.port = port;
     }
 
     /**
-     * Returns the listener name of this endpoint.
+     * Returns the listener name of this endpoint. This is non-empty for endpoints provided
+     * to broker plugins, but may be empty when used in clients.
      */
-    public String listener() {
-        return listener;
+    public Optional listenerName() {
+        return Optional.ofNullable(listenerName);
     }
 
     /**
@@ -77,7 +80,7 @@ public boolean equals(Object o) {
         }
 
         Endpoint that = (Endpoint) o;
-        return Objects.equals(this.listener, that.listener) &&
+        return Objects.equals(this.listenerName, that.listenerName) &&
             Objects.equals(this.securityProtocol, that.securityProtocol) &&
             Objects.equals(this.host, that.host) &&
             this.port == that.port;
@@ -86,13 +89,13 @@ public boolean equals(Object o) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(listener, securityProtocol, host, port);
+        return Objects.hash(listenerName, securityProtocol, host, port);
     }
 
     @Override
     public String toString() {
         return "Endpoint(" +
-            "listener='" + listener + '\'' +
+            "listenerName='" + listenerName + '\'' +
             ", securityProtocol=" + securityProtocol +
             ", host='" + host + '\'' +
             ", port=" + port +
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java
index 4002d2a5f44d0..c067444478b03 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java
@@ -91,7 +91,7 @@ public short apiVersion() {
     }
 
     @Override
-    public String listener() {
+    public String listenerName() {
         return listenerName.value();
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/server/authorizer/AuthorizableRequestContext.java b/clients/src/main/java/org/apache/kafka/server/authorizer/AuthorizableRequestContext.java
index 49c7290106af2..f68b9381ce1d9 100644
--- a/clients/src/main/java/org/apache/kafka/server/authorizer/AuthorizableRequestContext.java
+++ b/clients/src/main/java/org/apache/kafka/server/authorizer/AuthorizableRequestContext.java
@@ -32,7 +32,7 @@ public interface AuthorizableRequestContext {
     /**
      * Returns name of listener on which request was received.
      */
-    String listener();
+    String listenerName();
 
     /**
      * Returns the security protocol for the listener on which request was received.
diff --git a/core/src/main/scala/kafka/cluster/Broker.scala b/core/src/main/scala/kafka/cluster/Broker.scala
index 2ca8148153b18..e8e621f03555d 100755
--- a/core/src/main/scala/kafka/cluster/Broker.scala
+++ b/core/src/main/scala/kafka/cluster/Broker.scala
@@ -29,6 +29,13 @@ import org.apache.kafka.server.authorizer.AuthorizerServerInfo
 import scala.collection.Seq
 import scala.collection.JavaConverters._
 
+object Broker {
+  private[cluster] case class ServerInfo(clusterResource: ClusterResource,
+                                         brokerId: Int,
+                                         endpoints: util.List[Endpoint],
+                                         interBrokerEndpoint: Endpoint) extends AuthorizerServerInfo
+}
+
 /**
  * A Kafka broker.
  * A broker has an id, a collection of end-points, an optional rack and a listener to security protocol map.
@@ -75,15 +82,8 @@ case class Broker(id: Int, endPoints: Seq[EndPoint], rack: Option[String]) {
 
   def toServerInfo(clusterId: String, config: KafkaConfig): AuthorizerServerInfo = {
     val clusterResource: ClusterResource = new ClusterResource(clusterId)
-    val interBrokerEndpoint: Endpoint = endPoint(config.interBrokerListenerName)
-    val brokerEndpoints: util.List[Endpoint] = endPoints.toList.map(_.asInstanceOf[Endpoint]).asJava
-    BrokerEndpointInfo(clusterResource, id, brokerEndpoints, interBrokerEndpoint)
+    val interBrokerEndpoint: Endpoint = endPoint(config.interBrokerListenerName).toJava
+    val brokerEndpoints: util.List[Endpoint] = endPoints.toList.map(_.toJava).asJava
+    Broker.ServerInfo(clusterResource, id, brokerEndpoints, interBrokerEndpoint)
   }
-
-  case class BrokerEndpointInfo(clusterResource: ClusterResource,
-                                brokerId: Int,
-                                endpoints: util.List[Endpoint],
-                                interBrokerEndpoint: Endpoint)
-    extends AuthorizerServerInfo
-
 }
diff --git a/core/src/main/scala/kafka/cluster/EndPoint.scala b/core/src/main/scala/kafka/cluster/EndPoint.scala
index 4fbae70baadd5..2f8229a38865c 100644
--- a/core/src/main/scala/kafka/cluster/EndPoint.scala
+++ b/core/src/main/scala/kafka/cluster/EndPoint.scala
@@ -62,8 +62,7 @@ object EndPoint {
 /**
  * Part of the broker definition - matching host/port pair to a protocol
  */
-case class EndPoint(override val host: String, override val port: Int, listenerName: ListenerName, override val securityProtocol: SecurityProtocol)
-    extends JEndpoint(Option(listenerName).map(_.value).orNull, securityProtocol, host, port) {
+case class EndPoint(host: String, port: Int, listenerName: ListenerName, securityProtocol: SecurityProtocol) {
   def connectionString: String = {
     val hostport =
       if (host == null)
@@ -73,7 +72,7 @@ case class EndPoint(override val host: String, override val port: Int, listenerN
     listenerName.value + "://" + hostport
   }
 
-  // to keep spotbugs happy
-  override def equals(o: scala.Any): Boolean = super.equals(o)
-  override def hashCode(): Int = super.hashCode()
+  def toJava: JEndpoint = {
+    new JEndpoint(listenerName.value, securityProtocol, host, port)
+  }
 }
diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala
index e705d41b16d11..e50a6b0397211 100644
--- a/core/src/main/scala/kafka/network/SocketServer.scala
+++ b/core/src/main/scala/kafka/network/SocketServer.scala
@@ -22,6 +22,7 @@ import java.net._
 import java.nio.channels._
 import java.nio.channels.{Selector => NSelector}
 import java.util
+import java.util.Optional
 import java.util.concurrent._
 import java.util.concurrent.atomic._
 import java.util.function.Supplier
@@ -209,9 +210,9 @@ class SocketServer(val config: KafkaConfig,
       dataPlaneAcceptors.asScala.filterKeys(_ != interBrokerListener).values
     orderedAcceptors.foreach { acceptor =>
       val endpoint = acceptor.endPoint
-      debug(s"Wait for authorizer to complete start up on listener ${endpoint.listener}")
+      debug(s"Wait for authorizer to complete start up on listener ${endpoint.listenerName}")
       waitForAuthorizerFuture(acceptor, authorizerFutures)
-      debug(s"Start processors on listener ${endpoint.listener}")
+      debug(s"Start processors on listener ${endpoint.listenerName}")
       acceptor.startProcessors(DataPlaneThreadPrefix)
     }
     info(s"Started data-plane processors for ${dataPlaneAcceptors.size} acceptors")
@@ -382,7 +383,7 @@ class SocketServer(val config: KafkaConfig,
                                       authorizerFutures: Map[Endpoint, CompletableFuture[Void]]): Unit = {
     //we can't rely on authorizerFutures.get() due to ephemeral ports. Get the future using listener name
     authorizerFutures.foreach { case (endpoint, future) =>
-      if (endpoint.listener() == acceptor.endPoint.listener())
+      if (endpoint.listenerName == Optional.of(acceptor.endPoint.listenerName.value))
         future.join()
     }
   }
diff --git a/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala b/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala
index 222fbbcf5b802..56935d9b12e68 100644
--- a/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala
+++ b/core/src/main/scala/kafka/security/auth/SimpleAclAuthorizer.scala
@@ -19,6 +19,7 @@ package kafka.security.auth
 import java.util
 
 import kafka.network.RequestChannel.Session
+import kafka.security.auth.SimpleAclAuthorizer.BaseAuthorizer
 import kafka.security.authorizer.{AclAuthorizer, AuthorizerUtils}
 import kafka.utils._
 import kafka.zk.ZkVersion
@@ -50,6 +51,22 @@ object SimpleAclAuthorizer {
     def exists: Boolean = zkVersion != ZkVersion.UnknownVersion
   }
   val NoAcls = VersionedAcls(Set.empty, ZkVersion.UnknownVersion)
+
+  private[auth] class BaseAuthorizer extends AclAuthorizer {
+    override def logAuditMessage(requestContext: AuthorizableRequestContext, action: Action, authorized: Boolean): Unit = {
+      val principal = requestContext.principal
+      val host = requestContext.clientAddress.getHostAddress
+      val operation = Operation.fromJava(action.operation)
+      val resource = AuthorizerUtils.convertToResource(action.resourcePattern)
+      def logMessage: String = {
+        val authResult = if (authorized) "Allowed" else "Denied"
+        s"Principal = $principal is $authResult Operation = $operation from host = $host on resource = $resource"
+      }
+
+      if (authorized) authorizerLogger.debug(logMessage)
+      else authorizerLogger.info(logMessage)
+    }
+  }
 }
 
 @deprecated("Use kafka.security.authorizer.AclAuthorizer", "Since 2.4")
@@ -156,20 +173,4 @@ class SimpleAclAuthorizer extends Authorizer with Logging {
     else
       throw e
   }
-
-  class BaseAuthorizer extends AclAuthorizer {
-    override def logAuditMessage(requestContext: AuthorizableRequestContext, action: Action, authorized: Boolean): Unit = {
-      val principal = requestContext.principal
-      val host = requestContext.clientAddress.getHostAddress
-      val operation = Operation.fromJava(action.operation)
-      val resource = AuthorizerUtils.convertToResource(action.resourcePattern)
-      def logMessage: String = {
-        val authResult = if (authorized) "Allowed" else "Denied"
-        s"Principal = $principal is $authResult Operation = $operation from host = $host on resource = $resource"
-      }
-
-      if (authorized) authorizerLogger.debug(logMessage)
-      else authorizerLogger.info(logMessage)
-    }
-  }
 }
diff --git a/core/src/main/scala/kafka/security/authorizer/AclAuthorizer.scala b/core/src/main/scala/kafka/security/authorizer/AclAuthorizer.scala
index 84a641d60dd7e..8ba060d814921 100644
--- a/core/src/main/scala/kafka/security/authorizer/AclAuthorizer.scala
+++ b/core/src/main/scala/kafka/security/authorizer/AclAuthorizer.scala
@@ -61,6 +61,23 @@ object AclAuthorizer {
     def exists: Boolean = zkVersion != ZkVersion.UnknownVersion
   }
   val NoAcls = VersionedAcls(Set.empty, ZkVersion.UnknownVersion)
+
+  // Orders by resource type, then resource pattern type and finally reverse ordering by name.
+  private object ResourceOrdering extends Ordering[Resource] {
+
+    def compare(a: Resource, b: Resource): Int = {
+      val rt = a.resourceType compare b.resourceType
+      if (rt != 0)
+        rt
+      else {
+        val rnt = a.patternType compareTo b.patternType
+        if (rnt != 0)
+          rnt
+        else
+          (a.name compare b.name) * -1
+      }
+    }
+  }
 }
 
 class AclAuthorizer extends Authorizer with Logging {
@@ -72,7 +89,7 @@ class AclAuthorizer extends Authorizer with Logging {
   private var extendedAclSupport: Boolean = _
 
   @volatile
-  private var aclCache = new scala.collection.immutable.TreeMap[Resource, VersionedAcls]()(ResourceOrdering)
+  private var aclCache = new scala.collection.immutable.TreeMap[Resource, VersionedAcls]()(AclAuthorizer.ResourceOrdering)
   private val lock = new ReentrantReadWriteLock()
 
   // The maximum number of times we should try to update the resource acls in zookeeper before failing;
@@ -479,21 +496,4 @@ class AclAuthorizer extends Authorizer with Logging {
       }
     }
   }
-
-  // Orders by resource type, then resource pattern type and finally reverse ordering by name.
-  private object ResourceOrdering extends Ordering[Resource] {
-
-    def compare(a: Resource, b: Resource): Int = {
-      val rt = a.resourceType compare b.resourceType
-      if (rt != 0)
-        rt
-      else {
-        val rnt = a.patternType compareTo b.patternType
-        if (rnt != 0)
-          rnt
-        else
-          (a.name compare b.name) * -1
-      }
-    }
-  }
 }
diff --git a/core/src/main/scala/kafka/security/authorizer/AuthorizerUtils.scala b/core/src/main/scala/kafka/security/authorizer/AuthorizerUtils.scala
index e85e1063d0a3b..909f3a78bbb7e 100644
--- a/core/src/main/scala/kafka/security/authorizer/AuthorizerUtils.scala
+++ b/core/src/main/scala/kafka/security/authorizer/AuthorizerUtils.scala
@@ -84,7 +84,7 @@ object AuthorizerUtils {
     new AuthorizableRequestContext {
       override def clientId(): String = ""
       override def requestType(): Int = -1
-      override def listener(): String = ""
+      override def listenerName(): String = ""
       override def clientAddress(): InetAddress = session.clientAddress
       override def principal(): KafkaPrincipal = session.principal
       override def securityProtocol(): SecurityProtocol = null
diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala
index 0028e4a581987..57cb0b6d20ed1 100755
--- a/core/src/main/scala/kafka/server/KafkaServer.scala
+++ b/core/src/main/scala/kafka/server/KafkaServer.scala
@@ -299,7 +299,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
           case Some(authZ) =>
             authZ.start(brokerInfo.broker.toServerInfo(clusterId, config)).asScala.mapValues(_.toCompletableFuture).toMap
           case None =>
-            brokerInfo.broker.endPoints.map{ ep => (ep.asInstanceOf[Endpoint], CompletableFuture.completedFuture[Void](null)) }.toMap
+            brokerInfo.broker.endPoints.map { ep => ep.toJava -> CompletableFuture.completedFuture[Void](null) }.toMap
         }
 
         val fetchManager = new FetchManager(Time.SYSTEM,
diff --git a/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala
index b3e412bd96fcd..689cb0aef9f84 100644
--- a/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala
@@ -127,7 +127,7 @@ class SslAdminClientIntegrationTest extends SaslSslAdminClientIntegrationTest {
 
     def validateRequestContext(context: AuthorizableRequestContext, apiKey: ApiKeys): Unit = {
       assertEquals(SecurityProtocol.SSL, context.securityProtocol)
-      assertEquals("SSL", context.listener)
+      assertEquals("SSL", context.listenerName)
       assertEquals(KafkaPrincipal.ANONYMOUS, context.principal)
       assertEquals(apiKey.id.toInt, context.requestType)
       assertEquals(apiKey.latestVersion.toInt, context.requestVersion)
diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
index 507e117f5d22e..03135113f836b 100644
--- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
+++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
@@ -204,7 +204,7 @@ class SocketServerTest {
     testableServer.startup(startupProcessors = false)
     val updatedEndPoints = config.advertisedListeners.map { endpoint =>
       endpoint.copy(port = testableServer.boundPort(endpoint.listenerName))
-    }.map(_.asInstanceOf[Endpoint])
+    }.map(_.toJava)
 
     val externalReadyFuture = new CompletableFuture[Void]()
     val executor = Executors.newSingleThreadExecutor()
@@ -225,7 +225,7 @@ class SocketServerTest {
       sendAndReceiveControllerRequest(socket1, testableServer)
 
       val externalListener = new ListenerName("EXTERNAL")
-      val externalEndpoint = updatedEndPoints.find(e => e.listener() == externalListener.value).get
+      val externalEndpoint = updatedEndPoints.find(e => e.listenerName.get == externalListener.value).get
       val futures =  Map(externalEndpoint -> externalReadyFuture)
       val startFuture = executor.submit(CoreUtils.runnable(testableServer.startDataPlaneProcessors(futures)))
       TestUtils.waitUntilTrue(() => listenerStarted(config.interBrokerListenerName), "Inter-broker listener not started")

From 5d0052fe00c9071095b872b3b6ab6b1b455c9e20 Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Fri, 27 Sep 2019 13:28:00 +0100
Subject: [PATCH 0650/1071] KAFKA-8907; Return topic configs in CreateTopics
 response (KIP-525) (#7380)

Reviewers: Manikumar Reddy 
---
 .../clients/admin/CreateTopicsResult.java     | 91 ++++++++++++++++++-
 .../kafka/clients/admin/KafkaAdminClient.java | 34 +++++--
 .../common/message/CreateTopicsRequest.json   |  3 +-
 .../common/message/CreateTopicsResponse.json  | 26 +++++-
 .../kafka/clients/admin/MockAdminClient.java  |  6 +-
 .../scala/kafka/server/AdminManager.scala     | 23 +++++
 .../main/scala/kafka/server/KafkaApis.scala   | 19 +++-
 .../api/AdminClientIntegrationTest.scala      | 24 ++++-
 .../SaslSslAdminClientIntegrationTest.scala   | 66 +++++++++++++-
 9 files changed, 267 insertions(+), 25 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/CreateTopicsResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/CreateTopicsResult.java
index 4fada35c790c3..85c0a905240b3 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/CreateTopicsResult.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/CreateTopicsResult.java
@@ -18,9 +18,11 @@
 
 import org.apache.kafka.common.KafkaFuture;
 import org.apache.kafka.common.annotation.InterfaceStability;
+import org.apache.kafka.common.errors.ApiException;
 
 import java.util.Collection;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 /**
  * The result of {@link Admin#createTopics(Collection)}.
@@ -29,9 +31,11 @@
  */
 @InterfaceStability.Evolving
 public class CreateTopicsResult {
-    private final Map> futures;
+    final static int UNKNOWN = -1;
 
-    CreateTopicsResult(Map> futures) {
+    private final Map> futures;
+
+    CreateTopicsResult(Map> futures) {
         this.futures = futures;
     }
 
@@ -40,7 +44,8 @@ public class CreateTopicsResult {
      * topic creations.
      */
     public Map> values() {
-        return futures;
+        return futures.entrySet().stream()
+                .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().thenApply(v -> (Void) null)));
     }
 
     /**
@@ -49,4 +54,84 @@ public Map> values() {
     public KafkaFuture all() {
         return KafkaFuture.allOf(futures.values().toArray(new KafkaFuture[0]));
     }
+
+    /**
+     * Returns a future that provides topic configs for the topic when the request completes.
+     * 
                
+     * If broker version doesn't support replication factor in the response, throw
+     * {@link org.apache.kafka.common.errors.UnsupportedVersionException}.
+     * If broker returned an error for topic configs, throw appropriate exception. For example,
+     * {@link org.apache.kafka.common.errors.TopicAuthorizationException} is thrown if user does not
+     * have permission to describe topic configs.
+     */
+    public KafkaFuture config(String topic) {
+        return futures.get(topic).thenApply(TopicMetadataAndConfig::config);
+    }
+
+    /**
+     * Returns a future that provides number of partitions in the topic when the request completes.
+     * 
                
+     * If broker version doesn't support replication factor in the response, throw
+     * {@link org.apache.kafka.common.errors.UnsupportedVersionException}.
+     * If broker returned an error for topic configs, throw appropriate exception. For example,
+     * {@link org.apache.kafka.common.errors.TopicAuthorizationException} is thrown if user does not
+     * have permission to describe topic configs.
+     */
+    public KafkaFuture numPartitions(String topic) {
+        return futures.get(topic).thenApply(TopicMetadataAndConfig::numPartitions);
+    }
+
+    /**
+     * Returns a future that provides replication factor for the topic when the request completes.
+     * 
                
+     * If broker version doesn't support replication factor in the response, throw
+     * {@link org.apache.kafka.common.errors.UnsupportedVersionException}.
+     * If broker returned an error for topic configs, throw appropriate exception. For example,
+     * {@link org.apache.kafka.common.errors.TopicAuthorizationException} is thrown if user does not
+     * have permission to describe topic configs.
+     */
+    public KafkaFuture replicationFactor(String topic) {
+        return futures.get(topic).thenApply(TopicMetadataAndConfig::replicationFactor);
+    }
+
+    static class TopicMetadataAndConfig {
+        private final ApiException exception;
+        private final int numPartitions;
+        private final int replicationFactor;
+        private final Config config;
+
+        TopicMetadataAndConfig(int numPartitions, int replicationFactor, Config config) {
+            this.exception = null;
+            this.numPartitions = numPartitions;
+            this.replicationFactor = replicationFactor;
+            this.config = config;
+        }
+
+        TopicMetadataAndConfig(ApiException exception) {
+            this.exception = exception;
+            this.numPartitions = UNKNOWN;
+            this.replicationFactor = UNKNOWN;
+            this.config = null;
+        }
+
+        public int numPartitions() {
+            ensureSuccess();
+            return numPartitions;
+        }
+
+        public int replicationFactor() {
+            ensureSuccess();
+            return replicationFactor;
+        }
+
+        public Config config() {
+            ensureSuccess();
+            return config;
+        }
+
+        private void ensureSuccess() {
+            if (exception != null)
+                throw exception;
+        }
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
index ce9df22fef9b3..a7663da3789df 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
@@ -25,6 +25,7 @@
 import org.apache.kafka.clients.KafkaClient;
 import org.apache.kafka.clients.NetworkClient;
 import org.apache.kafka.clients.StaleMetadataException;
+import org.apache.kafka.clients.admin.CreateTopicsResult.TopicMetadataAndConfig;
 import org.apache.kafka.clients.admin.DeleteAclsResult.FilterResult;
 import org.apache.kafka.clients.admin.DeleteAclsResult.FilterResults;
 import org.apache.kafka.clients.admin.DescribeReplicaLogDirsResult.ReplicaLogDirInfo;
@@ -68,6 +69,7 @@
 import org.apache.kafka.common.message.CreateDelegationTokenResponseData;
 import org.apache.kafka.common.message.CreateTopicsRequestData;
 import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopicCollection;
+import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicConfigs;
 import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult;
 import org.apache.kafka.common.message.DeleteGroupsRequestData;
 import org.apache.kafka.common.message.DeleteTopicsRequestData;
@@ -1309,11 +1311,11 @@ int numPendingCalls() {
     @Override
     public CreateTopicsResult createTopics(final Collection newTopics,
                                            final CreateTopicsOptions options) {
-        final Map> topicFutures = new HashMap<>(newTopics.size());
+        final Map> topicFutures = new HashMap<>(newTopics.size());
         final CreatableTopicCollection topics = new CreatableTopicCollection();
         for (NewTopic newTopic : newTopics) {
             if (topicNameIsUnrepresentable(newTopic.name())) {
-                KafkaFutureImpl future = new KafkaFutureImpl<>();
+                KafkaFutureImpl future = new KafkaFutureImpl<>();
                 future.completeExceptionally(new InvalidTopicException("The given topic name '" +
                     newTopic.name() + "' cannot be represented in a request."));
                 topicFutures.put(newTopic.name(), future);
@@ -1348,7 +1350,7 @@ public void handleResponse(AbstractResponse abstractResponse) {
                 }
                 // Handle server responses for particular topics.
                 for (CreatableTopicResult result : response.data().topics()) {
-                    KafkaFutureImpl future = topicFutures.get(result.name());
+                    KafkaFutureImpl future = topicFutures.get(result.name());
                     if (future == null) {
                         log.warn("Server response mentioned unknown topic {}", result.name());
                     } else {
@@ -1358,13 +1360,33 @@ public void handleResponse(AbstractResponse abstractResponse) {
                         if (exception != null) {
                             future.completeExceptionally(exception);
                         } else {
-                            future.complete(null);
+                            TopicMetadataAndConfig topicMetadataAndConfig;
+                            if (result.topicConfigErrorCode() != Errors.NONE.code()) {
+                                topicMetadataAndConfig = new TopicMetadataAndConfig(Errors.forCode(result.topicConfigErrorCode()).exception());
+                            } else if (result.numPartitions() == CreateTopicsResult.UNKNOWN) {
+                                topicMetadataAndConfig = new TopicMetadataAndConfig(new UnsupportedVersionException(
+                                        "Topic metadata and configs in CreateTopics response not supported"));
+                            } else {
+                                List configs = result.configs();
+                                Config topicConfig = new Config(configs.stream()
+                                        .map(config -> new ConfigEntry(config.name(),
+                                                config.value(),
+                                                configSource(DescribeConfigsResponse.ConfigSource.forId(config.configSource())),
+                                                config.isSensitive(),
+                                                config.readOnly(),
+                                                Collections.emptyList()))
+                                        .collect(Collectors.toSet()));
+                                topicMetadataAndConfig = new TopicMetadataAndConfig(result.numPartitions(),
+                                        result.replicationFactor(),
+                                        topicConfig);
+                            }
+                            future.complete(topicMetadataAndConfig);
                         }
                     }
                 }
                 // The server should send back a response for every topic. But do a sanity check anyway.
-                for (Map.Entry> entry : topicFutures.entrySet()) {
-                    KafkaFutureImpl future = entry.getValue();
+                for (Map.Entry> entry : topicFutures.entrySet()) {
+                    KafkaFutureImpl future = entry.getValue();
                     if (!future.isDone()) {
                         future.completeExceptionally(new ApiException("The server response did not " +
                             "contain a reference to node " + entry.getKey()));
diff --git a/clients/src/main/resources/common/message/CreateTopicsRequest.json b/clients/src/main/resources/common/message/CreateTopicsRequest.json
index d2285668f4cba..e396ed2096436 100644
--- a/clients/src/main/resources/common/message/CreateTopicsRequest.json
+++ b/clients/src/main/resources/common/message/CreateTopicsRequest.json
@@ -19,7 +19,8 @@
   "name": "CreateTopicsRequest",
   // Version 1 adds validateOnly.
   // Version 4 makes partitions/replicationFactor optional even when assignments are not present (KIP-464)
-  "validVersions": "0-4",
+  // Version 5 returns topic configs in the response (KIP-525).
+  "validVersions": "0-5",
   "fields": [
     { "name": "Topics", "type": "[]CreatableTopic", "versions": "0+",
       "about": "The topics to create.", "fields": [
diff --git a/clients/src/main/resources/common/message/CreateTopicsResponse.json b/clients/src/main/resources/common/message/CreateTopicsResponse.json
index d56e642061e43..f646258bff5a7 100644
--- a/clients/src/main/resources/common/message/CreateTopicsResponse.json
+++ b/clients/src/main/resources/common/message/CreateTopicsResponse.json
@@ -20,8 +20,9 @@
   // Version 1 adds a per-topic error message string.
   // Version 2 adds the throttle time.
   // Starting in version 3, on quota violation, brokers send out responses before throttling.
-  // Version 4 makes partitions/replicationFactor optional even when assignments are not present (KIP-464)
-  "validVersions": "0-4",
+  // Version 4 makes partitions/replicationFactor optional even when assignments are not present (KIP-464).
+  // Version 5 returns topic configs in the response (KIP-525)
+  "validVersions": "0-5",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "2+", "ignorable": true,
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
@@ -32,7 +33,26 @@
       { "name": "ErrorCode", "type": "int16", "versions": "0+",
         "about": "The error code, or 0 if there was no error." },
       { "name": "ErrorMessage", "type": "string", "versions": "1+", "nullableVersions": "0+", "ignorable": true,
-        "about": "The error message, or null if there was no error." }
+        "about": "The error message, or null if there was no error." },
+      { "name": "TopicConfigErrorCode", "type": "int16", "versions": "5+", "default": "0",
+        "about": "Optional topic config error returned if configs are not returned in the response." },
+      { "name": "NumPartitions", "type": "int32", "versions": "5+", "default": "-1",
+        "about": "Number of partitions of the topic." },
+      { "name": "ReplicationFactor", "type": "int16", "versions": "5+", "default": "-1",
+        "about": "Replicator factor of the topic." },
+      { "name": "Configs", "type": "[]CreatableTopicConfigs", "versions": "5+", "nullableVersions": "5+",
+        "about": "Configuration of the topic.", "fields": [
+        { "name": "Name", "type": "string", "versions": "5+",
+          "about": "The configuration name." },
+        { "name": "Value", "type": "string", "versions": "5+", "nullableVersions": "0+",
+          "about": "The configuration value." },
+        { "name": "ReadOnly", "type": "bool", "versions": "5+",
+          "about": "True if the configuration is read-only." },
+        { "name": "ConfigSource", "type": "int8", "versions": "5+", "default": "-1", "ignorable": true,
+          "about": "The configuration source." },
+        { "name": "IsSensitive", "type": "bool", "versions": "5+",
+          "about": "True if this configuration is sensitive." }
+      ]}
     ]}
   ]
 }
diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java b/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java
index bc06468a648eb..017e447b74219 100644
--- a/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java
+++ b/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java
@@ -149,13 +149,13 @@ public DescribeClusterResult describeCluster(DescribeClusterOptions options) {
 
     @Override
     public CreateTopicsResult createTopics(Collection newTopics, CreateTopicsOptions options) {
-        Map> createTopicResult = new HashMap<>();
+        Map> createTopicResult = new HashMap<>();
 
         if (timeoutNextRequests > 0) {
             for (final NewTopic newTopic : newTopics) {
                 String topicName = newTopic.name();
 
-                KafkaFutureImpl future = new KafkaFutureImpl<>();
+                KafkaFutureImpl future = new KafkaFutureImpl<>();
                 future.completeExceptionally(new TimeoutException());
                 createTopicResult.put(topicName, future);
             }
@@ -165,7 +165,7 @@ public CreateTopicsResult createTopics(Collection newTopics, CreateTop
         }
 
         for (final NewTopic newTopic : newTopics) {
-            KafkaFutureImpl future = new KafkaFutureImpl<>();
+            KafkaFutureImpl future = new KafkaFutureImpl<>();
 
             String topicName = newTopic.name();
             if (allTopics.containsKey(topicName)) {
diff --git a/core/src/main/scala/kafka/server/AdminManager.scala b/core/src/main/scala/kafka/server/AdminManager.scala
index 1c4115e20068c..1b2a2c381522d 100644
--- a/core/src/main/scala/kafka/server/AdminManager.scala
+++ b/core/src/main/scala/kafka/server/AdminManager.scala
@@ -32,6 +32,7 @@ import org.apache.kafka.common.config.{AbstractConfig, ConfigDef, ConfigExceptio
 import org.apache.kafka.common.errors.{ApiException, InvalidConfigurationException, InvalidPartitionsException, InvalidReplicaAssignmentException, InvalidRequestException, ReassignmentInProgressException, TopicExistsException, UnknownTopicOrPartitionException}
 import org.apache.kafka.common.internals.Topic
 import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic
+import org.apache.kafka.common.message.CreateTopicsResponseData.{CreatableTopicConfigs, CreatableTopicResult}
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.protocol.Errors
@@ -82,6 +83,7 @@ class AdminManager(val config: KafkaConfig,
   def createTopics(timeout: Int,
                    validateOnly: Boolean,
                    toCreate: Map[String, CreatableTopic],
+                   includeConfigsAndMetatadata: Map[String, CreatableTopicResult],
                    responseCallback: Map[String, ApiError] => Unit): Unit = {
 
     // 1. map over topics creating assignment and calling zookeeper
@@ -153,6 +155,27 @@ class AdminManager(val config: KafkaConfig,
             else
               adminZkClient.createTopicWithAssignment(topic.name, configs, assignments)
         }
+
+        // For responses with DescribeConfigs permission, populate metadata and configs
+        includeConfigsAndMetatadata.get(topic.name).foreach { result =>
+          val logConfig = LogConfig.fromProps(KafkaServer.copyKafkaConfigToLog(config), configs)
+          val createEntry = createTopicConfigEntry(logConfig, configs, includeSynonyms = false)(_, _)
+          val topicConfigs = logConfig.values.asScala.map { case (k, v) =>
+            val entry = createEntry(k, v)
+            val source = ConfigSource.values.indices.map(_.toByte)
+              .find(i => ConfigSource.forId(i.toByte) == entry.source)
+              .getOrElse(0.toByte)
+            new CreatableTopicConfigs()
+                .setName(k)
+                .setValue(entry.value)
+                .setIsSensitive(entry.isSensitive)
+                .setReadOnly(entry.isReadOnly)
+                .setConfigSource(source)
+          }.toList.asJava
+          result.setConfigs(topicConfigs)
+          result.setNumPartitions(assignments.size)
+          result.setReplicationFactor(assignments(0).size.toShort)
+        }
         CreatePartitionsMetadata(topic.name, assignments, ApiError.NONE)
       } catch {
         // Log client errors at a lower level than unexpected exceptions
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 49beab9d0d914..6efcf29569116 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -1652,6 +1652,8 @@ class KafkaApis(val requestChannel: RequestChannel,
       val hasClusterAuthorization = authorize(request, CREATE, CLUSTER, CLUSTER_NAME, logIfDenied = false)
       val topics = createTopicsRequest.data.topics.asScala.map(_.name)
       val authorizedTopics = if (hasClusterAuthorization) topics.toSet else filterAuthorized(request, CREATE, TOPIC, topics.toSeq)
+      val authorizedForDescribeConfigs = filterAuthorized(request, DESCRIBE_CONFIGS, TOPIC, topics.toSeq)
+        .map(name => name -> results.find(name)).toMap
 
       results.asScala.foreach(topic => {
         if (results.findAll(topic.name()).size() > 1) {
@@ -1661,6 +1663,9 @@ class KafkaApis(val requestChannel: RequestChannel,
           topic.setErrorCode(Errors.TOPIC_AUTHORIZATION_FAILED.code())
           topic.setErrorMessage("Authorization failed.")
         }
+        if (!authorizedForDescribeConfigs.contains(topic.name)) {
+          topic.setTopicConfigErrorCode(Errors.TOPIC_AUTHORIZATION_FAILED.code)
+        }
       })
       val toCreate = mutable.Map[String, CreatableTopic]()
       createTopicsRequest.data.topics.asScala.foreach { topic =>
@@ -1670,15 +1675,23 @@ class KafkaApis(val requestChannel: RequestChannel,
       }
       def handleCreateTopicsResults(errors: Map[String, ApiError]): Unit = {
         errors.foreach { case (topicName, error) =>
-          results.find(topicName).
-            setErrorCode(error.error.code).
-            setErrorMessage(error.message)
+          val result = results.find(topicName)
+          result.setErrorCode(error.error.code)
+            .setErrorMessage(error.message)
+          // Reset any configs in the response if Create failed
+          if (error != ApiError.NONE) {
+            result.setConfigs(List.empty.asJava)
+              .setNumPartitions(-1)
+              .setReplicationFactor(-1)
+              .setTopicConfigErrorCode(0.toShort)
+          }
         }
         sendResponseCallback(results)
       }
       adminManager.createTopics(createTopicsRequest.data.timeoutMs,
           createTopicsRequest.data.validateOnly,
           toCreate,
+          authorizedForDescribeConfigs,
           handleCreateTopicsResults)
     }
   }
diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index 23de704530a7e..2841595137693 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -167,19 +167,37 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
       new NewTopic("mytopic2", 3, 3.toShort),
       new NewTopic("mytopic3", Option.empty[Integer].asJava, Option.empty[java.lang.Short].asJava)
     )
-    client.createTopics(newTopics.asJava, new CreateTopicsOptions().validateOnly(true)).all.get()
+    val validateResult = client.createTopics(newTopics.asJava, new CreateTopicsOptions().validateOnly(true))
+    validateResult.all.get()
     waitForTopics(client, List(), topics)
 
-    client.createTopics(newTopics.asJava).all.get()
+    def validateMetadataAndConfigs(result: CreateTopicsResult): Unit = {
+      assertEquals(2, result.numPartitions("mytopic").get())
+      assertEquals(2, result.replicationFactor("mytopic").get())
+      assertEquals(3, result.numPartitions("mytopic2").get())
+      assertEquals(3, result.replicationFactor("mytopic2").get())
+      assertEquals(configs.head.numPartitions, result.numPartitions("mytopic3").get())
+      assertEquals(configs.head.defaultReplicationFactor, result.replicationFactor("mytopic3").get())
+      assertFalse(result.config("mytopic").get().entries.isEmpty)
+    }
+    validateMetadataAndConfigs(validateResult)
+
+    val createResult = client.createTopics(newTopics.asJava)
+    createResult.all.get()
     waitForTopics(client, topics, List())
+    validateMetadataAndConfigs(createResult)
 
-    val results = client.createTopics(newTopics.asJava).values()
+    val failedCreateResult = client.createTopics(newTopics.asJava)
+    val results = failedCreateResult.values()
     assertTrue(results.containsKey("mytopic"))
     assertFutureExceptionTypeEquals(results.get("mytopic"), classOf[TopicExistsException])
     assertTrue(results.containsKey("mytopic2"))
     assertFutureExceptionTypeEquals(results.get("mytopic2"), classOf[TopicExistsException])
     assertTrue(results.containsKey("mytopic3"))
     assertFutureExceptionTypeEquals(results.get("mytopic3"), classOf[TopicExistsException])
+    assertFutureExceptionTypeEquals(failedCreateResult.numPartitions("mytopic3"), classOf[TopicExistsException])
+    assertFutureExceptionTypeEquals(failedCreateResult.replicationFactor("mytopic3"), classOf[TopicExistsException])
+    assertFutureExceptionTypeEquals(failedCreateResult.config("mytopic3"), classOf[TopicExistsException])
 
     val topicToDescription = client.describeTopics(topics.asJava).all.get()
     assertEquals(topics.toSet, topicToDescription.keySet.asScala)
diff --git a/core/src/test/scala/integration/kafka/api/SaslSslAdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/SaslSslAdminClientIntegrationTest.scala
index cad33e07a6100..2a8131e560f2b 100644
--- a/core/src/test/scala/integration/kafka/api/SaslSslAdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SaslSslAdminClientIntegrationTest.scala
@@ -15,20 +15,24 @@ package kafka.api
 import java.io.File
 import java.util
 
+import kafka.log.LogConfig
 import kafka.security.auth.{All, Allow, Alter, AlterConfigs, Authorizer, ClusterAction, Create, Delete, Deny, Describe, Group, Operation, PermissionType, SimpleAclAuthorizer, Topic, Acl => AuthAcl, Resource => AuthResource}
 import kafka.security.authorizer.AuthorizerWrapper
-import kafka.server.KafkaConfig
+import kafka.server.{Defaults, KafkaConfig}
 import kafka.utils.{CoreUtils, JaasTestUtils, TestUtils}
 import kafka.utils.TestUtils._
 import org.apache.kafka.clients.admin._
 import org.apache.kafka.common.acl._
-import org.apache.kafka.common.errors.{ClusterAuthorizationException, InvalidRequestException}
+import org.apache.kafka.common.config.ConfigResource
+import org.apache.kafka.common.errors.{ClusterAuthorizationException, InvalidRequestException, TopicAuthorizationException}
 import org.apache.kafka.common.resource.{PatternType, ResourcePattern, ResourcePatternFilter, ResourceType}
 import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
-import org.junit.Assert.assertEquals
+import org.junit.Assert.{assertEquals, assertTrue}
 import org.junit.{After, Assert, Before, Test}
 
 import scala.collection.JavaConverters._
+import scala.collection.Seq
+import scala.compat.java8.OptionConverters._
 import scala.util.{Failure, Success, Try}
 
 class SaslSslAdminClientIntegrationTest extends AdminClientIntegrationTest with SaslSetup {
@@ -401,6 +405,62 @@ class SaslSslAdminClientIntegrationTest extends AdminClientIntegrationTest with
     testAclCreateGetDelete(expectAuth = false)
   }
 
+  @Test
+  def testCreateTopicsResponseMetadataAndConfig(): Unit = {
+    val topic1 = "mytopic1"
+    val topic2 = "mytopic2"
+    val denyAcl = new AclBinding(new ResourcePattern(ResourceType.TOPIC, topic2, PatternType.LITERAL),
+      new AccessControlEntry("User:*", "*", AclOperation.DESCRIBE_CONFIGS, AclPermissionType.DENY))
+
+    client = AdminClient.create(createConfig())
+    client.createAcls(List(denyAcl).asJava, new CreateAclsOptions()).all().get()
+
+    val topics = Seq(topic1, topic2)
+    val configsOverride = Map(LogConfig.SegmentBytesProp -> "100000").asJava
+    val newTopics = Seq(
+      new NewTopic(topic1, 2, 3.toShort).configs(configsOverride),
+      new NewTopic(topic2, Option.empty[Integer].asJava, Option.empty[java.lang.Short].asJava).configs(configsOverride))
+    val validateResult = client.createTopics(newTopics.asJava, new CreateTopicsOptions().validateOnly(true))
+    validateResult.all.get()
+    waitForTopics(client, List(), topics)
+
+    def validateMetadataAndConfigs(result: CreateTopicsResult): Unit = {
+      assertEquals(2, result.numPartitions(topic1).get())
+      assertEquals(3, result.replicationFactor(topic1).get())
+      val topicConfigs = result.config(topic1).get().entries.asScala
+      assertTrue(topicConfigs.nonEmpty)
+      val segmentBytesConfig = topicConfigs.find(_.name == LogConfig.SegmentBytesProp).get
+      assertEquals(100000, segmentBytesConfig.value.toLong)
+      assertEquals(ConfigEntry.ConfigSource.DYNAMIC_TOPIC_CONFIG, segmentBytesConfig.source)
+      val compressionConfig = topicConfigs.find(_.name == LogConfig.CompressionTypeProp).get
+      assertEquals(Defaults.CompressionType, compressionConfig.value)
+      assertEquals(ConfigEntry.ConfigSource.DEFAULT_CONFIG, compressionConfig.source)
+
+      assertFutureExceptionTypeEquals(result.numPartitions(topic2), classOf[TopicAuthorizationException])
+      assertFutureExceptionTypeEquals(result.replicationFactor(topic2), classOf[TopicAuthorizationException])
+      assertFutureExceptionTypeEquals(result.config(topic2), classOf[TopicAuthorizationException])
+    }
+    validateMetadataAndConfigs(validateResult)
+
+    val createResult = client.createTopics(newTopics.asJava, new CreateTopicsOptions())
+    createResult.all.get()
+    waitForTopics(client, topics, List())
+    validateMetadataAndConfigs(createResult)
+    val createResponseConfig = createResult.config(topic1).get().entries.asScala
+
+    val topicResource = new ConfigResource(ConfigResource.Type.TOPIC, topic1)
+    val describeResponseConfig = client.describeConfigs(List(topicResource).asJava).values.get(topicResource).get().entries.asScala
+    assertEquals(describeResponseConfig.size, createResponseConfig.size)
+    describeResponseConfig.foreach { describeEntry =>
+      val name = describeEntry.name
+      val createEntry = createResponseConfig.find(_.name == name).get
+      assertEquals(s"Value mismatch for $name", describeEntry.value, createEntry.value)
+      assertEquals(s"isReadOnly mismatch for $name", describeEntry.isReadOnly, createEntry.isReadOnly)
+      assertEquals(s"isSensitive mismatch for $name", describeEntry.isSensitive, createEntry.isSensitive)
+      assertEquals(s"Source mismatch for $name", describeEntry.source, createEntry.source)
+    }
+  }
+
   private def waitForDescribeAcls(client: Admin, filter: AclBindingFilter, acls: Set[AclBinding]): Unit = {
     var lastResults: util.Collection[AclBinding] = null
     TestUtils.waitUntilTrue(() => {

From 30443af1c8d6c8b9ca327d046c4d4daffe5d43b2 Mon Sep 17 00:00:00 2001
From: Manikumar Reddy 
Date: Sat, 28 Sep 2019 00:13:45 +0530
Subject: [PATCH 0651/1071] KAFKA-6883: Add toUpperCase support to
 sasl.kerberos.principal.to.local rule (KIP-309)

Author: Manikumar Reddy 

Reviewers: Rajini Sivaram 

Closes #7375 from omkreddy/KAFKA-6883-KerberosShortNamer
---
 .../security/kerberos/KerberosRule.java       | 11 ++++++-
 .../security/kerberos/KerberosShortNamer.java |  5 +--
 .../security/kerberos/KerberosNameTest.java   | 32 +++++++++++++++++++
 docs/security.html                            |  4 ++-
 4 files changed, 48 insertions(+), 4 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosRule.java b/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosRule.java
index 92a70b1f19456..91280ca65a347 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosRule.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosRule.java
@@ -46,6 +46,7 @@ class KerberosRule {
     private final String toPattern;
     private final boolean repeat;
     private final boolean toLowerCase;
+    private final boolean toUpperCase;
 
     KerberosRule(String defaultRealm) {
         this.defaultRealm = defaultRealm;
@@ -57,10 +58,11 @@ class KerberosRule {
         toPattern = null;
         repeat = false;
         toLowerCase = false;
+        toUpperCase = false;
     }
 
     KerberosRule(String defaultRealm, int numOfComponents, String format, String match, String fromPattern,
-                 String toPattern, boolean repeat, boolean toLowerCase) {
+                 String toPattern, boolean repeat, boolean toLowerCase, boolean toUpperCase) {
         this.defaultRealm = defaultRealm;
         isDefault = false;
         this.numOfComponents = numOfComponents;
@@ -71,6 +73,7 @@ class KerberosRule {
         this.toPattern = toPattern;
         this.repeat = repeat;
         this.toLowerCase = toLowerCase;
+        this.toUpperCase = toUpperCase;
     }
 
     @Override
@@ -102,6 +105,9 @@ public String toString() {
             if (toLowerCase) {
                 buf.append("/L");
             }
+            if (toUpperCase) {
+                buf.append("/U");
+            }
         }
         return buf.toString();
     }
@@ -191,7 +197,10 @@ String apply(String[] params) throws IOException {
         }
         if (toLowerCase && result != null) {
             result = result.toLowerCase(Locale.ENGLISH);
+        } else if (toUpperCase && result != null) {
+            result = result.toUpperCase(Locale.ENGLISH);
         }
+
         return result;
     }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosShortNamer.java b/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosShortNamer.java
index 69b4689009baa..96e01f17942b4 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosShortNamer.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosShortNamer.java
@@ -33,7 +33,7 @@ public class KerberosShortNamer {
     /**
      * A pattern for parsing a auth_to_local rule.
      */
-    private static final Pattern RULE_PARSER = Pattern.compile("((DEFAULT)|((RULE:\\[(\\d*):([^\\]]*)](\\(([^)]*)\\))?(s/([^/]*)/([^/]*)/(g)?)?/?(L)?)))");
+    private static final Pattern RULE_PARSER = Pattern.compile("((DEFAULT)|((RULE:\\[(\\d*):([^\\]]*)](\\(([^)]*)\\))?(s/([^/]*)/([^/]*)/(g)?)?/?(L|U)?)))");
 
     /* Rules for the translation of the principal name into an operating system name */
     private final List principalToLocalRules;
@@ -66,7 +66,8 @@ private static List parseRules(String defaultRealm, List r
                         matcher.group(10),
                         matcher.group(11),
                         "g".equals(matcher.group(12)),
-                        "L".equals(matcher.group(13))));
+                        "L".equals(matcher.group(13)),
+                        "U".equals(matcher.group(13))));
 
             }
         }
diff --git a/clients/src/test/java/org/apache/kafka/common/security/kerberos/KerberosNameTest.java b/clients/src/test/java/org/apache/kafka/common/security/kerberos/KerberosNameTest.java
index 687f9370a6c9c..9dd44a14fb670 100644
--- a/clients/src/test/java/org/apache/kafka/common/security/kerberos/KerberosNameTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/security/kerberos/KerberosNameTest.java
@@ -86,6 +86,35 @@ public void testToLowerCase() throws Exception {
         assertEquals("user", shortNamer.shortName(name));
     }
 
+    @Test
+    public void testToUpperCase() throws Exception {
+        List rules = Arrays.asList(
+            "RULE:[1:$1]/U",
+            "RULE:[2:$1](Test.*)s/ABC///U",
+            "RULE:[2:$1](ABC.*)s/ABC/XYZ/g/U",
+            "RULE:[2:$1](App\\..*)s/App\\.(.*)/$1/g/U",
+            "RULE:[2:$1]/U",
+            "DEFAULT"
+        );
+
+        KerberosShortNamer shortNamer = KerberosShortNamer.fromUnparsedRules("REALM.COM", rules);
+
+        KerberosName name = KerberosName.parse("User@REALM.COM");
+        assertEquals("USER", shortNamer.shortName(name));
+
+        name = KerberosName.parse("TestABC/host@FOO.COM");
+        assertEquals("TEST", shortNamer.shortName(name));
+
+        name = KerberosName.parse("ABC_User_ABC/host@FOO.COM");
+        assertEquals("XYZ_USER_XYZ", shortNamer.shortName(name));
+
+        name = KerberosName.parse("App.SERVICE-name/example.com@REALM.COM");
+        assertEquals("SERVICE-NAME", shortNamer.shortName(name));
+
+        name = KerberosName.parse("User/root@REALM.COM");
+        assertEquals("USER", shortNamer.shortName(name));
+    }
+
     @Test
     public void testInvalidRules() {
         testInvalidRule(Arrays.asList("default"));
@@ -94,6 +123,9 @@ public void testInvalidRules() {
         testInvalidRule(Arrays.asList("DEFAULT/g"));
 
         testInvalidRule(Arrays.asList("rule:[1:$1]"));
+        testInvalidRule(Arrays.asList("rule:[1:$1]/L/U"));
+        testInvalidRule(Arrays.asList("rule:[1:$1]/U/L"));
+        testInvalidRule(Arrays.asList("rule:[1:$1]/LU"));
         testInvalidRule(Arrays.asList("RULE:[1:$1/L"));
         testInvalidRule(Arrays.asList("RULE:[1:$1]/l"));
         testInvalidRule(Arrays.asList("RULE:[2:$1](ABC.*)s/ABC/XYZ/L/g"));
diff --git a/docs/security.html b/docs/security.html
index b4c86546f0806..8467bd658fd7b 100644
--- a/docs/security.html
+++ b/docs/security.html
@@ -1052,13 +1052,15 @@ 
                Customizing SSL User N
     
                Customizing SASL User Name
                
 
     By default, the SASL user name will be the primary part of the Kerberos principal. One can change that by setting sasl.kerberos.principal.to.local.rules to a customized rule in server.properties.
-    The format of sasl.kerberos.principal.to.local.rules is a list where each rule works in the same way as the auth_to_local in Kerberos configuration file (krb5.conf). This also support additional lowercase rule, to force the translated result to be all lower case. This is done by adding a "/L" to the end of the rule. check below formats for syntax.
+    The format of sasl.kerberos.principal.to.local.rules is a list where each rule works in the same way as the auth_to_local in Kerberos configuration file (krb5.conf). This also support additional lowercase/uppercase rule, to force the translated result to be all lowercase/uppercase. This is done by adding a "/L" or "/U" to the end of the rule. check below formats for syntax.
     Each rules starts with RULE: and contains an expression as the following formats. See the kerberos documentation for more details.
     
                         RULE:[n:string](regexp)s/pattern/replacement/
         RULE:[n:string](regexp)s/pattern/replacement/g
         RULE:[n:string](regexp)s/pattern/replacement//L
         RULE:[n:string](regexp)s/pattern/replacement/g/L
+        RULE:[n:string](regexp)s/pattern/replacement//U
+        RULE:[n:string](regexp)s/pattern/replacement/g/U
     
                
 
     An example of adding a rule to properly translate user@MYDOMAIN.COM to user while also keeping the default rule in place is:

From 22434e6535c6471b8ac3e9cff1919e5ac15a50be Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Fri, 27 Sep 2019 12:08:15 -0700
Subject: [PATCH 0652/1071] KAFKA-8319: Make KafkaStreamsTest a non-integration
 test class (#7382)

Previous KafkaStreamsTest takes 2min20s on my local laptop, because lots of its integration test which is producing / consuming records, and checking state directory file system takes lots of time. On the other hand, these tests should be well simplified with mocks.

This test reduces the test from a clumsy integration test class into a unit tests with mocks of its internal modules. And some other test functions should not be in KafkaStreamsTest actually and have been moved to other modular test classes. Now it takes 2s.

Also it helps removing the potential flakiness of the following (some of them are claimed resolved only because we have not seen them recently, but after looking at the test code I can verify they are still flaky):

* KAFKA-5818 (the original JIRA ticket indeed exposed a real issue that has been fixed, but the test itself remains flaky)
* KAFKA-6215
* KAFKA-7921
* KAFKA-7990
* KAFKA-8319
* KAFKA-8427

Reviewers: Bill Bejeck , John Roesler , Bruno Cadonna 
---
 .../kafka/clients/consumer/MockConsumer.java  |   1 -
 .../apache/kafka/streams/KafkaStreams.java    |   3 +-
 .../kafka/streams/KafkaStreamsTest.java       | 878 +++++++++---------
 .../kafka/streams/StreamsConfigTest.java      |  26 +
 .../processor/internals/StreamThreadTest.java |  20 +-
 .../apache/kafka/test/MockClientSupplier.java |   2 +-
 6 files changed, 461 insertions(+), 469 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java
index b20ee9783fe90..0db883ed1b413 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java
@@ -460,7 +460,6 @@ public synchronized void close() {
     @SuppressWarnings("deprecation")
     @Override
     public synchronized void close(long timeout, TimeUnit unit) {
-        ensureNotClosed();
         this.closed = true;
     }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
index a405c6f083368..6a51d1ae83cb3 100644
--- a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
+++ b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
@@ -51,7 +51,6 @@
 import org.apache.kafka.streams.processor.internals.ProcessorTopology;
 import org.apache.kafka.streams.processor.internals.StateDirectory;
 import org.apache.kafka.streams.processor.internals.StreamThread;
-import org.apache.kafka.streams.processor.internals.StreamThread.State;
 import org.apache.kafka.streams.processor.internals.StreamsMetadataState;
 import org.apache.kafka.streams.processor.internals.ThreadStateTransitionValidator;
 import org.apache.kafka.streams.state.HostInfo;
@@ -146,7 +145,7 @@ public class KafkaStreams implements AutoCloseable {
     private final QueryableStoreProvider queryableStoreProvider;
     private final Admin adminClient;
 
-    private GlobalStreamThread globalStreamThread;
+    GlobalStreamThread globalStreamThread;
     private KafkaStreams.StateListener stateListener;
     private StateRestoreListener globalStateRestoreListener;
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java b/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java
index 2cd31ba736297..5bf21ea9ce1fd 100644
--- a/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java
@@ -16,286 +16,438 @@
  */
 package org.apache.kafka.streams;
 
-import org.apache.kafka.clients.CommonClientConfigs;
-import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.admin.Admin;
+import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.producer.MockProducer;
 import org.apache.kafka.common.Cluster;
-import org.apache.kafka.common.KafkaException;
-import org.apache.kafka.common.Node;
-import org.apache.kafka.common.config.ConfigException;
-import org.apache.kafka.common.metrics.Sensor;
-import org.apache.kafka.common.network.Selectable;
+import org.apache.kafka.common.metrics.MetricConfig;
+import org.apache.kafka.common.metrics.Metrics;
+import org.apache.kafka.common.metrics.MetricsReporter;
 import org.apache.kafka.common.serialization.Serdes;
-import org.apache.kafka.common.serialization.StringDeserializer;
 import org.apache.kafka.common.serialization.StringSerializer;
-import org.apache.kafka.common.utils.Utils;
-import org.apache.kafka.streams.errors.StreamsException;
-import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
-import org.apache.kafka.streams.integration.utils.IntegrationTestUtils;
-import org.apache.kafka.streams.kstream.Consumed;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.streams.kstream.Materialized;
 import org.apache.kafka.streams.processor.AbstractProcessor;
-import org.apache.kafka.streams.processor.ThreadMetadata;
+import org.apache.kafka.streams.processor.StateRestoreListener;
 import org.apache.kafka.streams.processor.internals.GlobalStreamThread;
+import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder;
+import org.apache.kafka.streams.processor.internals.ProcessorTopology;
+import org.apache.kafka.streams.processor.internals.StateDirectory;
 import org.apache.kafka.streams.processor.internals.StreamThread;
+import org.apache.kafka.streams.processor.internals.StreamsMetadataState;
 import org.apache.kafka.streams.state.KeyValueStore;
 import org.apache.kafka.streams.state.StoreBuilder;
 import org.apache.kafka.streams.state.Stores;
-import org.apache.kafka.test.IntegrationTest;
+import org.apache.kafka.streams.state.internals.metrics.RocksDBMetricsRecordingTrigger;
 import org.apache.kafka.test.MockClientSupplier;
 import org.apache.kafka.test.MockMetricsReporter;
 import org.apache.kafka.test.MockProcessorSupplier;
-import org.apache.kafka.test.MockStateRestoreListener;
 import org.apache.kafka.test.TestUtils;
-import org.junit.After;
+import org.easymock.Capture;
+import org.easymock.EasyMock;
 import org.junit.Assert;
 import org.junit.Before;
-import org.junit.ClassRule;
 import org.junit.Rule;
 import org.junit.Test;
-import org.junit.experimental.categories.Category;
 import org.junit.rules.TestName;
+import org.junit.runner.RunWith;
+import org.powermock.api.easymock.PowerMock;
+import org.powermock.api.easymock.annotation.Mock;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
 
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
+import java.net.InetSocketAddress;
 import java.time.Duration;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
+import java.util.UUID;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.stream.Collectors;
+import java.util.concurrent.atomic.AtomicReference;
 
-import static java.util.Arrays.asList;
+import static java.util.Collections.singletonList;
+import static org.easymock.EasyMock.anyInt;
+import static org.easymock.EasyMock.anyLong;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.anyString;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-@Category({IntegrationTest.class})
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({KafkaStreams.class, StreamThread.class})
 public class KafkaStreamsTest {
 
-    private static final int NUM_BROKERS = 1;
     private static final int NUM_THREADS = 2;
-    // We need this to avoid the KafkaConsumer hanging on poll
-    // (this may occur if the test doesn't complete quickly enough)
-    @ClassRule
-    public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(NUM_BROKERS);
-    private final StreamsBuilder builder = new StreamsBuilder();
-    private KafkaStreams globalStreams;
-    private Properties props;
 
     @Rule
     public TestName testName = new TestName();
 
+    private MockClientSupplier supplier;
+    private MockTime time;
+
+    private Properties props;
+
+    @Mock
+    private StateDirectory stateDirectory;
+    @Mock
+    private StreamThread streamThreadOne;
+    @Mock
+    private StreamThread streamThreadTwo;
+    @Mock
+    private GlobalStreamThread globalStreamThread;
+    @Mock
+    private ScheduledExecutorService cleanupSchedule;
+    @Mock
+    private Metrics metrics;
+
+    private StateListenerStub streamsStateListener;
+    private Capture> metricsReportersCapture;
+    private Capture threadStatelistenerCapture;
+
+    public static class StateListenerStub implements KafkaStreams.StateListener {
+        int numChanges = 0;
+        KafkaStreams.State oldState;
+        KafkaStreams.State newState;
+        public Map mapStates = new HashMap<>();
+
+        @Override
+        public void onChange(final KafkaStreams.State newState,
+                             final KafkaStreams.State oldState) {
+            final long prevCount = mapStates.containsKey(newState) ? mapStates.get(newState) : 0;
+            numChanges++;
+            this.oldState = oldState;
+            this.newState = newState;
+            mapStates.put(newState, prevCount + 1);
+        }
+    }
+
     @Before
-    public void before() {
+    public void before() throws Exception {
+        time = new MockTime();
+        supplier = new MockClientSupplier();
+        supplier.setClusterForAdminClient(Cluster.bootstrap(singletonList(new InetSocketAddress("localhost", 9999))));
+        streamsStateListener = new StateListenerStub();
+        threadStatelistenerCapture = EasyMock.newCapture();
+        metricsReportersCapture = EasyMock.newCapture();
+
         props = new Properties();
         props.put(StreamsConfig.APPLICATION_ID_CONFIG, "appId");
         props.put(StreamsConfig.CLIENT_ID_CONFIG, "clientId");
-        props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
+        props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:2018");
         props.put(StreamsConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName());
         props.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath());
         props.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, NUM_THREADS);
-        globalStreams = new KafkaStreams(builder.build(), props);
-    }
 
-    @After
-    public void cleanup() {
-        if (globalStreams != null) {
-            globalStreams.close();
-        }
+        prepareStreams();
     }
 
-    @Test
-    public void testOsDefaultSocketBufferSizes() {
-        props.put(CommonClientConfigs.SEND_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE);
-        props.put(CommonClientConfigs.RECEIVE_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE);
-        final KafkaStreams streams = new KafkaStreams(builder.build(), props);
-        streams.close();
-    }
-
-    @Test(expected = KafkaException.class)
-    public void testInvalidSocketSendBufferSize() {
-        props.put(CommonClientConfigs.SEND_BUFFER_CONFIG, -2);
-        final KafkaStreams streams = new KafkaStreams(builder.build(), props);
-        streams.close();
-    }
-
-    @Test(expected = KafkaException.class)
-    public void testInvalidSocketReceiveBufferSize() {
-        props.put(CommonClientConfigs.RECEIVE_BUFFER_CONFIG, -2);
-        final KafkaStreams streams = new KafkaStreams(builder.build(), props);
-        streams.close();
+    private void prepareStreams() throws Exception {
+        // setup metrics
+        PowerMock.expectNew(Metrics.class,
+            anyObject(MetricConfig.class),
+            EasyMock.capture(metricsReportersCapture),
+            EasyMock.anyObject(Time.class)
+        ).andAnswer(() -> {
+            for (final MetricsReporter reporter : metricsReportersCapture.getValue()) {
+                reporter.init(Collections.emptyList());
+            }
+            return metrics;
+        }).anyTimes();
+        metrics.close();
+        EasyMock.expectLastCall().andAnswer(() -> {
+            for (final MetricsReporter reporter : metricsReportersCapture.getValue()) {
+                reporter.close();
+            }
+            return null;
+        }).anyTimes();
+
+        // setup stream threads
+        PowerMock.mockStatic(StreamThread.class);
+        EasyMock.expect(StreamThread.create(
+            anyObject(InternalTopologyBuilder.class),
+            anyObject(StreamsConfig.class),
+            anyObject(KafkaClientSupplier.class),
+            anyObject(Admin.class),
+            anyObject(UUID.class),
+            anyObject(String.class),
+            anyObject(Metrics.class),
+            anyObject(Time.class),
+            anyObject(StreamsMetadataState.class),
+            anyLong(),
+            anyObject(StateDirectory.class),
+            anyObject(StateRestoreListener.class),
+            anyInt()
+        )).andReturn(streamThreadOne).andReturn(streamThreadTwo);
+        EasyMock.expect(StreamThread.getSharedAdminClientId(
+            anyString()
+        )).andReturn("admin").anyTimes();
+
+        EasyMock.expect(streamThreadOne.getId()).andReturn(0L).anyTimes();
+        EasyMock.expect(streamThreadTwo.getId()).andReturn(1L).anyTimes();
+        prepareStreamThread(streamThreadOne, true);
+        prepareStreamThread(streamThreadTwo, false);
+
+        // setup global threads
+        final AtomicReference globalThreadState = new AtomicReference<>(GlobalStreamThread.State.CREATED);
+        PowerMock.expectNew(GlobalStreamThread.class,
+            anyObject(ProcessorTopology.class),
+            anyObject(StreamsConfig.class),
+            anyObject(Consumer.class),
+            anyObject(StateDirectory.class),
+            anyLong(),
+            anyObject(Metrics.class),
+            anyObject(Time.class),
+            anyString(),
+            anyObject(StateRestoreListener.class),
+            anyObject(RocksDBMetricsRecordingTrigger.class)
+        ).andReturn(globalStreamThread).anyTimes();
+        EasyMock.expect(globalStreamThread.state()).andAnswer(globalThreadState::get).anyTimes();
+        globalStreamThread.setStateListener(EasyMock.capture(threadStatelistenerCapture));
+        EasyMock.expectLastCall().anyTimes();
+
+        globalStreamThread.start();
+        EasyMock.expectLastCall().andAnswer(() -> {
+            globalThreadState.set(GlobalStreamThread.State.RUNNING);
+            threadStatelistenerCapture.getValue().onChange(globalStreamThread,
+                GlobalStreamThread.State.RUNNING,
+                GlobalStreamThread.State.CREATED);
+            return null;
+        }).anyTimes();
+        globalStreamThread.shutdown();
+        EasyMock.expectLastCall().andAnswer(() -> {
+            supplier.restoreConsumer.close();
+            for (final MockProducer producer : supplier.producers) {
+                producer.close();
+            }
+            globalThreadState.set(GlobalStreamThread.State.DEAD);
+            threadStatelistenerCapture.getValue().onChange(globalStreamThread,
+                GlobalStreamThread.State.PENDING_SHUTDOWN,
+                GlobalStreamThread.State.RUNNING);
+            threadStatelistenerCapture.getValue().onChange(globalStreamThread,
+                GlobalStreamThread.State.DEAD,
+                GlobalStreamThread.State.PENDING_SHUTDOWN);
+            return null;
+        }).anyTimes();
+        EasyMock.expect(globalStreamThread.stillRunning()).andReturn(globalThreadState.get() == GlobalStreamThread.State.RUNNING).anyTimes();
+        globalStreamThread.join();
+        EasyMock.expectLastCall().anyTimes();
+
+        PowerMock.replay(StreamThread.class, Metrics.class, metrics, streamThreadOne, streamThreadTwo, GlobalStreamThread.class, globalStreamThread);
+    }
+
+    private void prepareStreamThread(final StreamThread thread, final boolean terminable) throws Exception {
+        final AtomicReference state = new AtomicReference<>(StreamThread.State.CREATED);
+        EasyMock.expect(thread.state()).andAnswer(state::get).anyTimes();
+
+        thread.setStateListener(EasyMock.capture(threadStatelistenerCapture));
+        EasyMock.expectLastCall().anyTimes();
+        thread.setRocksDBMetricsRecordingTrigger(EasyMock.anyObject(RocksDBMetricsRecordingTrigger.class));
+        EasyMock.expectLastCall().anyTimes();
+
+        thread.start();
+        EasyMock.expectLastCall().andAnswer(() -> {
+            state.set(StreamThread.State.STARTING);
+            threadStatelistenerCapture.getValue().onChange(thread,
+                StreamThread.State.STARTING,
+                StreamThread.State.CREATED);
+            threadStatelistenerCapture.getValue().onChange(thread,
+                StreamThread.State.PARTITIONS_REVOKED,
+                StreamThread.State.STARTING);
+            threadStatelistenerCapture.getValue().onChange(thread,
+                StreamThread.State.PARTITIONS_ASSIGNED,
+                StreamThread.State.PARTITIONS_REVOKED);
+            threadStatelistenerCapture.getValue().onChange(thread,
+                StreamThread.State.RUNNING,
+                StreamThread.State.PARTITIONS_ASSIGNED);
+            return null;
+        }).anyTimes();
+        thread.shutdown();
+        EasyMock.expectLastCall().andAnswer(() -> {
+            supplier.consumer.close();
+            supplier.restoreConsumer.close();
+            for (final MockProducer producer : supplier.producers) {
+                producer.close();
+            }
+            state.set(StreamThread.State.DEAD);
+            threadStatelistenerCapture.getValue().onChange(thread, StreamThread.State.PENDING_SHUTDOWN, StreamThread.State.RUNNING);
+            threadStatelistenerCapture.getValue().onChange(thread, StreamThread.State.DEAD, StreamThread.State.PENDING_SHUTDOWN);
+            return null;
+        }).anyTimes();
+        EasyMock.expect(thread.isRunning()).andReturn(state.get() == StreamThread.State.RUNNING).anyTimes();
+        thread.join();
+        if (terminable)
+            EasyMock.expectLastCall().anyTimes();
+        else
+            EasyMock.expectLastCall().andAnswer(() -> {
+                Thread.sleep(50L);
+                return null;
+            }).anyTimes();
     }
 
     @Test
-    public void stateShouldTransitToNotRunningIfCloseRightAfterCreated() {
-        globalStreams.close();
+    public void testShouldTransitToNotRunningIfCloseRightAfterCreated() {
+        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        streams.close();
 
-        Assert.assertEquals(KafkaStreams.State.NOT_RUNNING, globalStreams.state());
+        Assert.assertEquals(KafkaStreams.State.NOT_RUNNING, streams.state());
     }
 
     @Test
     public void stateShouldTransitToRunningIfNonDeadThreadsBackToRunning() throws InterruptedException {
-        final StateListenerStub stateListener = new StateListenerStub();
-        globalStreams.setStateListener(stateListener);
+        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        streams.setStateListener(streamsStateListener);
 
-        Assert.assertEquals(0, stateListener.numChanges);
-        Assert.assertEquals(KafkaStreams.State.CREATED, globalStreams.state());
+        Assert.assertEquals(0, streamsStateListener.numChanges);
+        Assert.assertEquals(KafkaStreams.State.CREATED, streams.state());
 
-        globalStreams.start();
+        streams.start();
 
         TestUtils.waitForCondition(
-            () -> stateListener.numChanges == 2,
+            () -> streamsStateListener.numChanges == 2,
             "Streams never started.");
-        Assert.assertEquals(KafkaStreams.State.RUNNING, globalStreams.state());
+        Assert.assertEquals(KafkaStreams.State.RUNNING, streams.state());
 
-        for (final StreamThread thread: globalStreams.threads) {
-            thread.stateListener().onChange(
+        for (final StreamThread thread: streams.threads) {
+            threadStatelistenerCapture.getValue().onChange(
                 thread,
                 StreamThread.State.PARTITIONS_REVOKED,
                 StreamThread.State.RUNNING);
         }
 
-        Assert.assertEquals(3, stateListener.numChanges);
-        Assert.assertEquals(KafkaStreams.State.REBALANCING, globalStreams.state());
+        Assert.assertEquals(3, streamsStateListener.numChanges);
+        Assert.assertEquals(KafkaStreams.State.REBALANCING, streams.state());
 
-        for (final StreamThread thread : globalStreams.threads) {
-            thread.stateListener().onChange(
+        for (final StreamThread thread : streams.threads) {
+            threadStatelistenerCapture.getValue().onChange(
                 thread,
                 StreamThread.State.PARTITIONS_ASSIGNED,
                 StreamThread.State.PARTITIONS_REVOKED);
         }
 
-        Assert.assertEquals(3, stateListener.numChanges);
-        Assert.assertEquals(KafkaStreams.State.REBALANCING, globalStreams.state());
+        Assert.assertEquals(3, streamsStateListener.numChanges);
+        Assert.assertEquals(KafkaStreams.State.REBALANCING, streams.state());
 
-        globalStreams.threads[NUM_THREADS - 1].stateListener().onChange(
-            globalStreams.threads[NUM_THREADS - 1],
+        threadStatelistenerCapture.getValue().onChange(
+            streams.threads[NUM_THREADS - 1],
             StreamThread.State.PENDING_SHUTDOWN,
             StreamThread.State.PARTITIONS_ASSIGNED);
 
-        globalStreams.threads[NUM_THREADS - 1].stateListener().onChange(
-            globalStreams.threads[NUM_THREADS - 1],
+        threadStatelistenerCapture.getValue().onChange(
+            streams.threads[NUM_THREADS - 1],
             StreamThread.State.DEAD,
             StreamThread.State.PENDING_SHUTDOWN);
 
-        Assert.assertEquals(3, stateListener.numChanges);
-        Assert.assertEquals(KafkaStreams.State.REBALANCING, globalStreams.state());
+        Assert.assertEquals(3, streamsStateListener.numChanges);
+        Assert.assertEquals(KafkaStreams.State.REBALANCING, streams.state());
 
-        for (final StreamThread thread : globalStreams.threads) {
-            if (thread != globalStreams.threads[NUM_THREADS - 1]) {
-                thread.stateListener().onChange(
+        for (final StreamThread thread : streams.threads) {
+            if (thread != streams.threads[NUM_THREADS - 1]) {
+                threadStatelistenerCapture.getValue().onChange(
                     thread,
                     StreamThread.State.RUNNING,
                     StreamThread.State.PARTITIONS_ASSIGNED);
             }
         }
 
-        Assert.assertEquals(4, stateListener.numChanges);
-        Assert.assertEquals(KafkaStreams.State.RUNNING, globalStreams.state());
+        Assert.assertEquals(4, streamsStateListener.numChanges);
+        Assert.assertEquals(KafkaStreams.State.RUNNING, streams.state());
 
-        globalStreams.close();
+        streams.close();
 
         TestUtils.waitForCondition(
-            () -> stateListener.numChanges == 6,
+            () -> streamsStateListener.numChanges == 6,
             "Streams never closed.");
-        Assert.assertEquals(KafkaStreams.State.NOT_RUNNING, globalStreams.state());
+        Assert.assertEquals(KafkaStreams.State.NOT_RUNNING, streams.state());
     }
 
     @Test
     public void stateShouldTransitToErrorIfAllThreadsDead() throws InterruptedException {
-        final StateListenerStub stateListener = new StateListenerStub();
-        globalStreams.setStateListener(stateListener);
+        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        streams.setStateListener(streamsStateListener);
 
-        Assert.assertEquals(0, stateListener.numChanges);
-        Assert.assertEquals(KafkaStreams.State.CREATED, globalStreams.state());
+        Assert.assertEquals(0, streamsStateListener.numChanges);
+        Assert.assertEquals(KafkaStreams.State.CREATED, streams.state());
 
-        globalStreams.start();
+        streams.start();
 
         TestUtils.waitForCondition(
-            () -> stateListener.numChanges == 2,
+            () -> streamsStateListener.numChanges == 2,
             "Streams never started.");
-        Assert.assertEquals(KafkaStreams.State.RUNNING, globalStreams.state());
+        Assert.assertEquals(KafkaStreams.State.RUNNING, streams.state());
 
-        for (final StreamThread thread : globalStreams.threads) {
-            thread.stateListener().onChange(
+        for (final StreamThread thread : streams.threads) {
+            threadStatelistenerCapture.getValue().onChange(
                 thread,
                 StreamThread.State.PARTITIONS_REVOKED,
                 StreamThread.State.RUNNING);
         }
 
-        Assert.assertEquals(3, stateListener.numChanges);
-        Assert.assertEquals(KafkaStreams.State.REBALANCING, globalStreams.state());
+        Assert.assertEquals(3, streamsStateListener.numChanges);
+        Assert.assertEquals(KafkaStreams.State.REBALANCING, streams.state());
 
-        globalStreams.threads[NUM_THREADS - 1].stateListener().onChange(
-            globalStreams.threads[NUM_THREADS - 1],
+        threadStatelistenerCapture.getValue().onChange(
+            streams.threads[NUM_THREADS - 1],
             StreamThread.State.PENDING_SHUTDOWN,
             StreamThread.State.PARTITIONS_REVOKED);
 
-        globalStreams.threads[NUM_THREADS - 1].stateListener().onChange(
-            globalStreams.threads[NUM_THREADS - 1],
+        threadStatelistenerCapture.getValue().onChange(
+            streams.threads[NUM_THREADS - 1],
             StreamThread.State.DEAD,
             StreamThread.State.PENDING_SHUTDOWN);
 
-        Assert.assertEquals(3, stateListener.numChanges);
-        Assert.assertEquals(KafkaStreams.State.REBALANCING, globalStreams.state());
+        Assert.assertEquals(3, streamsStateListener.numChanges);
+        Assert.assertEquals(KafkaStreams.State.REBALANCING, streams.state());
 
-        for (final StreamThread thread : globalStreams.threads) {
-            if (thread != globalStreams.threads[NUM_THREADS - 1]) {
-                thread.stateListener().onChange(
+        for (final StreamThread thread : streams.threads) {
+            if (thread != streams.threads[NUM_THREADS - 1]) {
+                threadStatelistenerCapture.getValue().onChange(
                     thread,
                     StreamThread.State.PENDING_SHUTDOWN,
                     StreamThread.State.PARTITIONS_REVOKED);
 
-                thread.stateListener().onChange(
+                threadStatelistenerCapture.getValue().onChange(
                     thread,
                     StreamThread.State.DEAD,
                     StreamThread.State.PENDING_SHUTDOWN);
             }
         }
 
-        Assert.assertEquals(4, stateListener.numChanges);
-        Assert.assertEquals(KafkaStreams.State.ERROR, globalStreams.state());
+        Assert.assertEquals(4, streamsStateListener.numChanges);
+        Assert.assertEquals(KafkaStreams.State.ERROR, streams.state());
 
-        globalStreams.close();
+        streams.close();
 
         // the state should not stuck with ERROR, but transit to NOT_RUNNING in the end
         TestUtils.waitForCondition(
-            () -> stateListener.numChanges == 6,
+            () -> streamsStateListener.numChanges == 6,
             "Streams never closed.");
-        Assert.assertEquals(KafkaStreams.State.NOT_RUNNING, globalStreams.state());
+        Assert.assertEquals(KafkaStreams.State.NOT_RUNNING, streams.state());
     }
 
     @Test
     public void shouldCleanupResourcesOnCloseWithoutPreviousStart() throws Exception {
+        final StreamsBuilder builder = new StreamsBuilder();
         builder.globalTable("anyTopic");
-        final List nodes = Collections.singletonList(new Node(0, "localhost", 8121));
-        final Cluster cluster = new Cluster("mockClusterId", nodes,
-                                            Collections.emptySet(), Collections.emptySet(),
-                                            Collections.emptySet(), nodes.get(0));
-        final MockClientSupplier clientSupplier = new MockClientSupplier();
-        clientSupplier.setClusterForAdminClient(cluster);
-        final KafkaStreams streams = new KafkaStreams(builder.build(), props, clientSupplier);
+
+        final KafkaStreams streams = new KafkaStreams(builder.build(), props, supplier, time);
         streams.close();
+
         TestUtils.waitForCondition(
             () -> streams.state() == KafkaStreams.State.NOT_RUNNING,
             "Streams never stopped.");
 
-        // Ensure that any created clients are closed
-        assertTrue(clientSupplier.consumer.closed());
-        assertTrue(clientSupplier.restoreConsumer.closed());
-        for (final MockProducer p : clientSupplier.producers) {
+        assertTrue(supplier.consumer.closed());
+        assertTrue(supplier.restoreConsumer.closed());
+        for (final MockProducer p : supplier.producers) {
             assertTrue(p.closed());
         }
     }
@@ -303,15 +455,12 @@ public void shouldCleanupResourcesOnCloseWithoutPreviousStart() throws Exception
     @Test
     public void testStateThreadClose() throws Exception {
         // make sure we have the global state thread running too
+        final StreamsBuilder builder = new StreamsBuilder();
         builder.globalTable("anyTopic");
-        final KafkaStreams streams = new KafkaStreams(builder.build(), props);
+        final KafkaStreams streams = new KafkaStreams(builder.build(), props, supplier, time);
 
         try {
-            final java.lang.reflect.Field threadsField = streams.getClass().getDeclaredField("threads");
-            threadsField.setAccessible(true);
-            final StreamThread[] threads = (StreamThread[]) threadsField.get(streams);
-
-            assertEquals(NUM_THREADS, threads.length);
+            assertEquals(NUM_THREADS, streams.threads.length);
             assertEquals(streams.state(), KafkaStreams.State.CREATED);
 
             streams.start();
@@ -320,12 +469,11 @@ public void testStateThreadClose() throws Exception {
                 "Streams never started.");
 
             for (int i = 0; i < NUM_THREADS; i++) {
-                final StreamThread tmpThread = threads[i];
+                final StreamThread tmpThread = streams.threads[i];
                 tmpThread.shutdown();
-                TestUtils.waitForCondition(
-                    () -> tmpThread.state() == StreamThread.State.DEAD,
+                TestUtils.waitForCondition(() -> tmpThread.state() == StreamThread.State.DEAD,
                     "Thread never stopped.");
-                threads[i].join();
+                streams.threads[i].join();
             }
             TestUtils.waitForCondition(
                 () -> streams.state() == KafkaStreams.State.ERROR,
@@ -338,26 +486,23 @@ public void testStateThreadClose() throws Exception {
             () -> streams.state() == KafkaStreams.State.NOT_RUNNING,
             "Streams never stopped.");
 
-        final java.lang.reflect.Field globalThreadField = streams.getClass().getDeclaredField("globalStreamThread");
-        globalThreadField.setAccessible(true);
-        final GlobalStreamThread globalStreamThread = (GlobalStreamThread) globalThreadField.get(streams);
-        assertNull(globalStreamThread);
+        assertNull(streams.globalStreamThread);
     }
 
     @Test
     public void testStateGlobalThreadClose() throws Exception {
         // make sure we have the global state thread running too
+        final StreamsBuilder builder = new StreamsBuilder();
         builder.globalTable("anyTopic");
-        final KafkaStreams streams = new KafkaStreams(builder.build(), props);
+        final KafkaStreams streams = new KafkaStreams(builder.build(), props, supplier, time);
 
         try {
             streams.start();
             TestUtils.waitForCondition(
                 () -> streams.state() == KafkaStreams.State.RUNNING,
                 "Streams never started.");
-            final java.lang.reflect.Field globalThreadField = streams.getClass().getDeclaredField("globalStreamThread");
-            globalThreadField.setAccessible(true);
-            final GlobalStreamThread globalStreamThread = (GlobalStreamThread) globalThreadField.get(streams);
+
+            final GlobalStreamThread globalStreamThread = streams.globalStreamThread;
             globalStreamThread.shutdown();
             TestUtils.waitForCondition(
                 () -> globalStreamThread.state() == GlobalStreamThread.State.DEAD,
@@ -371,51 +516,11 @@ public void testStateGlobalThreadClose() throws Exception {
         assertEquals(streams.state(), KafkaStreams.State.NOT_RUNNING);
     }
 
-    @Test
-    public void globalThreadShouldTimeoutWhenBrokerConnectionCannotBeEstablished() {
-        final Properties props = new Properties();
-        props.put(StreamsConfig.APPLICATION_ID_CONFIG, "appId");
-        props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:1");
-        props.put(StreamsConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName());
-        props.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath());
-        props.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, NUM_THREADS);
-
-        props.put(ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, 200);
-
-        // make sure we have the global state thread running too
-        builder.globalTable("anyTopic");
-        try (final KafkaStreams streams = new KafkaStreams(builder.build(), props)) {
-            streams.start();
-            fail("expected start() to time out and throw an exception.");
-        } catch (final StreamsException expected) {
-            // This is a result of not being able to connect to the broker.
-        }
-        // There's nothing to assert... We're testing that this operation actually completes.
-    }
-
-    @Test
-    public void testLocalThreadCloseWithoutConnectingToBroker() {
-        final Properties props = new Properties();
-        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "appId");
-        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:1");
-        props.setProperty(StreamsConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName());
-        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath());
-        props.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, NUM_THREADS);
-
-        // make sure we have the global state thread running too
-        builder.table("anyTopic");
-        try (final KafkaStreams streams = new KafkaStreams(builder.build(), props)) {
-            streams.start();
-        }
-        // There's nothing to assert... We're testing that this operation actually completes.
-    }
-
-
     @Test
     public void testInitializesAndDestroysMetricsReporters() {
         final int oldInitCount = MockMetricsReporter.INIT_COUNT.get();
 
-        try (final KafkaStreams streams = new KafkaStreams(builder.build(), props)) {
+        try (final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time)) {
             final int newInitCount = MockMetricsReporter.INIT_COUNT.get();
             final int initDiff = newInitCount - oldInitCount;
             assertTrue("some reporters should be initialized by calling on construction", initDiff > 0);
@@ -429,60 +534,50 @@ public void testInitializesAndDestroysMetricsReporters() {
 
     @Test
     public void testCloseIsIdempotent() {
-        globalStreams.close();
+        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        streams.close();
         final int closeCount = MockMetricsReporter.CLOSE_COUNT.get();
 
-        globalStreams.close();
+        streams.close();
         Assert.assertEquals("subsequent close() calls should do nothing",
             closeCount, MockMetricsReporter.CLOSE_COUNT.get());
     }
 
     @Test
     public void testCannotStartOnceClosed() {
-        globalStreams.start();
-        globalStreams.close();
+        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        streams.start();
+        streams.close();
         try {
-            globalStreams.start();
+            streams.start();
             fail("Should have throw IllegalStateException");
         } catch (final IllegalStateException expected) {
             // this is ok
         } finally {
-            globalStreams.close();
-        }
-    }
-
-    @Test
-    public void testCannotStartTwice() {
-        globalStreams.start();
-
-        try {
-            globalStreams.start();
-            fail("Should throw an IllegalStateException");
-        } catch (final IllegalStateException e) {
-            // this is ok
-        } finally {
-            globalStreams.close();
+            streams.close();
         }
     }
 
     @Test
     public void shouldNotSetGlobalRestoreListenerAfterStarting() {
-        globalStreams.start();
+        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        streams.start();
         try {
-            globalStreams.setGlobalStateRestoreListener(new MockStateRestoreListener());
+            streams.setGlobalStateRestoreListener(null);
             fail("Should throw an IllegalStateException");
         } catch (final IllegalStateException e) {
             // expected
         } finally {
-            globalStreams.close();
+            streams.close();
         }
     }
 
     @Test
     public void shouldThrowExceptionSettingUncaughtExceptionHandlerNotInCreateState() {
-        globalStreams.start();
+        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        streams.start();
         try {
-            globalStreams.setUncaughtExceptionHandler(null);
+            streams.setUncaughtExceptionHandler(null);
             fail("Should throw IllegalStateException");
         } catch (final IllegalStateException e) {
             // expected
@@ -491,9 +586,10 @@ public void shouldThrowExceptionSettingUncaughtExceptionHandlerNotInCreateState(
 
     @Test
     public void shouldThrowExceptionSettingStateListenerNotInCreateState() {
-        globalStreams.start();
+        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        streams.start();
         try {
-            globalStreams.setStateListener(null);
+            streams.setStateListener(null);
             fail("Should throw IllegalStateException");
         } catch (final IllegalStateException e) {
             // expected
@@ -501,193 +597,121 @@ public void shouldThrowExceptionSettingStateListenerNotInCreateState() {
     }
 
     @Test
-    public void testIllegalMetricsConfig() {
-        props.setProperty(StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG, "illegalConfig");
-
+    public void shouldAllowCleanupBeforeStartAndAfterClose() {
+        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
         try {
-            new KafkaStreams(builder.build(), props);
-            fail("Should have throw ConfigException");
-        } catch (final ConfigException expected) { /* expected */ }
+            streams.cleanUp();
+            streams.start();
+        } finally {
+            streams.close();
+            streams.cleanUp();
+        }
     }
 
     @Test
-    public void testLegalMetricsConfig() {
-        props.setProperty(StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG, Sensor.RecordingLevel.INFO.toString());
-        new KafkaStreams(builder.build(), props).close();
+    public void shouldThrowOnCleanupWhileRunning() throws InterruptedException {
+        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        streams.start();
+        TestUtils.waitForCondition(
+            () -> streams.state() == KafkaStreams.State.RUNNING,
+            "Streams never started.");
 
-        props.setProperty(StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG, Sensor.RecordingLevel.DEBUG.toString());
-        new KafkaStreams(builder.build(), props).close();
+        try {
+            streams.cleanUp();
+            fail("Should have thrown IllegalStateException");
+        } catch (final IllegalStateException expected) {
+            assertEquals("Cannot clean up while running.", expected.getMessage());
+        }
     }
 
     @Test(expected = IllegalStateException.class)
     public void shouldNotGetAllTasksWhenNotRunning() {
-        globalStreams.allMetadata();
+        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        streams.allMetadata();
     }
 
     @Test(expected = IllegalStateException.class)
     public void shouldNotGetAllTasksWithStoreWhenNotRunning() {
-        globalStreams.allMetadataForStore("store");
+        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        streams.allMetadataForStore("store");
     }
 
     @Test(expected = IllegalStateException.class)
     public void shouldNotGetTaskWithKeyAndSerializerWhenNotRunning() {
-        globalStreams.metadataForKey("store", "key", Serdes.String().serializer());
+        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        streams.metadataForKey("store", "key", Serdes.String().serializer());
     }
 
     @Test(expected = IllegalStateException.class)
     public void shouldNotGetTaskWithKeyAndPartitionerWhenNotRunning() {
-        globalStreams.metadataForKey("store", "key", (topic, key, value, numPartitions) -> 0);
+        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        streams.metadataForKey("store", "key", (topic, key, value, numPartitions) -> 0);
     }
 
     @Test
-    public void shouldReturnFalseOnCloseWhenThreadsHaventTerminated() throws Exception {
-        final AtomicBoolean keepRunning = new AtomicBoolean(true);
-        KafkaStreams streams = null;
-        try {
-            final StreamsBuilder builder = new StreamsBuilder();
-            final CountDownLatch latch = new CountDownLatch(1);
-            final String topic = "input";
-            CLUSTER.createTopics(topic);
-
-            builder.stream(topic, Consumed.with(Serdes.String(), Serdes.String()))
-                    .foreach((key, value) -> {
-                        try {
-                            latch.countDown();
-                            while (keepRunning.get()) {
-                                Thread.sleep(10);
-                            }
-                        } catch (final InterruptedException e) {
-                            // no-op
-                        }
-                    });
-            streams = new KafkaStreams(builder.build(), props);
-            streams.start();
-            IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(topic,
-                Collections.singletonList(new KeyValue<>("A", "A")),
-                TestUtils.producerConfig(
-                    CLUSTER.bootstrapServers(),
-                    StringSerializer.class,
-                    StringSerializer.class,
-                    new Properties()),
-                System.currentTimeMillis());
-
-            assertTrue("Timed out waiting to receive single message", latch.await(30, TimeUnit.SECONDS));
-            assertFalse(streams.close(Duration.ofMillis(10)));
-        } finally {
-            // stop the thread so we don't interfere with other tests etc
-            keepRunning.set(false);
-            if (streams != null) {
-                streams.close();
-            }
+    public void shouldReturnFalseOnCloseWhenThreadsHaventTerminated() {
+        // do not use mock time so that it can really elapse
+        try (final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier)) {
+            assertFalse(streams.close(Duration.ofMillis(10L)));
         }
     }
 
-    @Test
-    public void shouldReturnThreadMetadata() {
-        globalStreams.start();
-        final Set threadMetadata = globalStreams.localThreadsMetadata();
-        assertNotNull(threadMetadata);
-        assertEquals(2, threadMetadata.size());
-        for (final ThreadMetadata metadata : threadMetadata) {
-            assertTrue("#threadState() was: " + metadata.threadState() + "; expected either RUNNING, STARTING, PARTITIONS_REVOKED, PARTITIONS_ASSIGNED, or CREATED",
-                asList("RUNNING", "STARTING", "PARTITIONS_REVOKED", "PARTITIONS_ASSIGNED", "CREATED").contains(metadata.threadState()));
-            assertEquals(0, metadata.standbyTasks().size());
-            assertEquals(0, metadata.activeTasks().size());
-            final String threadName = metadata.threadName();
-            assertTrue(threadName.startsWith("clientId-StreamThread-"));
-            assertEquals(threadName + "-consumer", metadata.consumerClientId());
-            assertEquals(threadName + "-restore-consumer", metadata.restoreConsumerClientId());
-            assertEquals(Collections.singleton(threadName + "-producer"), metadata.producerClientIds());
-            assertEquals("clientId-admin", metadata.adminClientId());
+    @Test(expected = IllegalArgumentException.class)
+    public void shouldThrowOnNegativeTimeoutForClose() {
+        try (final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time)) {
+            streams.close(Duration.ofMillis(-1L));
         }
     }
 
     @Test
-    public void shouldAllowCleanupBeforeStartAndAfterClose() {
-        try {
-            globalStreams.cleanUp();
-            globalStreams.start();
-        } finally {
-            globalStreams.close();
+    public void shouldNotBlockInCloseForZeroDuration() {
+        try (final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time)) {
+            // with mock time that does not elapse, close would not return if it ever waits on the state transition
+            assertFalse(streams.close(Duration.ZERO));
         }
-        globalStreams.cleanUp();
     }
 
     @Test
-    public void shouldThrowOnCleanupWhileRunning() throws InterruptedException {
-        globalStreams.start();
-        TestUtils.waitForCondition(
-            () -> globalStreams.state() == KafkaStreams.State.RUNNING,
-            "Streams never started.");
-
-        try {
-            globalStreams.cleanUp();
-            fail("Should have thrown IllegalStateException");
-        } catch (final IllegalStateException expected) {
-            assertEquals("Cannot clean up while running.", expected.getMessage());
-        }
-    }
+    public void shouldCleanupOldStateDirs() throws Exception {
+        PowerMock.mockStatic(Executors.class);
+        EasyMock.expect(Executors.newSingleThreadScheduledExecutor(
+            anyObject(ThreadFactory.class)
+        )).andReturn(cleanupSchedule).anyTimes();
+
+        cleanupSchedule.scheduleAtFixedRate(
+            EasyMock.anyObject(Runnable.class),
+            EasyMock.eq(1L),
+            EasyMock.eq(1L),
+            EasyMock.eq(TimeUnit.MILLISECONDS)
+        );
+        EasyMock.expectLastCall().andReturn(null);
+        cleanupSchedule.shutdownNow();
+        EasyMock.expectLastCall().andReturn(null);
+
+        PowerMock.expectNew(StateDirectory.class,
+            anyObject(StreamsConfig.class),
+            anyObject(Time.class),
+            EasyMock.eq(true)
+        ).andReturn(stateDirectory);
+
+        PowerMock.replayAll(Executors.class, cleanupSchedule, stateDirectory);
 
-    @Test
-    public void shouldCleanupOldStateDirs() throws InterruptedException {
         props.setProperty(StreamsConfig.STATE_CLEANUP_DELAY_MS_CONFIG, "1");
 
-        final String topic = "topic";
-        CLUSTER.createTopic(topic);
         final StreamsBuilder builder = new StreamsBuilder();
+        builder.table("topic", Materialized.as("store"));
 
-        builder.table(topic, Materialized.as("store"));
-
-        try (final KafkaStreams streams = new KafkaStreams(builder.build(), props)) {
-            final CountDownLatch latch = new CountDownLatch(1);
-            streams.setStateListener((newState, oldState) -> {
-                if (newState == KafkaStreams.State.RUNNING && oldState == KafkaStreams.State.REBALANCING) {
-                    latch.countDown();
-                }
-            });
-            final String appDir = props.getProperty(StreamsConfig.STATE_DIR_CONFIG) + File.separator + props.getProperty(StreamsConfig.APPLICATION_ID_CONFIG);
-            final File oldTaskDir = new File(appDir, "10_1");
-            assertTrue(oldTaskDir.mkdirs());
-
+        try (final KafkaStreams streams = new KafkaStreams(builder.build(), props, supplier, time)) {
             streams.start();
-            latch.await(30, TimeUnit.SECONDS);
-            verifyCleanupStateDir(appDir, oldTaskDir);
-            assertTrue(oldTaskDir.mkdirs());
-            verifyCleanupStateDir(appDir, oldTaskDir);
-        }
-    }
-
-    @Test
-    public void shouldThrowOnNegativeTimeoutForClose() {
-        try (final KafkaStreams streams = new KafkaStreams(builder.build(), props)) {
-            streams.close(Duration.ofMillis(-1L));
-            fail("should not accept negative close parameter");
-        } catch (final IllegalArgumentException e) {
-            // expected
         }
-    }
 
-    @Test
-    public void shouldNotBlockInCloseForZeroDuration() throws InterruptedException {
-        final KafkaStreams streams = new KafkaStreams(builder.build(), props);
-        final Thread th = new Thread(() -> streams.close(Duration.ofMillis(0L)));
-
-        th.start();
-
-        try {
-            th.join(30_000L);
-            assertFalse(th.isAlive());
-        } finally {
-            streams.close();
-        }
+        PowerMock.verifyAll();
     }
 
     @Test
     public void statelessTopologyShouldNotCreateStateDirectory() throws Exception {
         final String inputTopic = testName.getMethodName() + "-input";
         final String outputTopic = testName.getMethodName() + "-output";
-        CLUSTER.createTopics(inputTopic, outputTopic);
-
         final Topology topology = new Topology();
         topology.addSource("source", Serdes.String().deserializer(), Serdes.String().deserializer(), inputTopic)
                 .addProcessor("process", () -> new AbstractProcessor() {
@@ -699,7 +723,7 @@ public void process(final String key, final String value) {
                     }
                 }, "source")
                 .addSink("sink", outputTopic, new StringSerializer(), new StringSerializer(), "process");
-        startStreamsAndCheckDirExists(topology, Collections.singleton(inputTopic), outputTopic, false);
+        startStreamsAndCheckDirExists(topology, false);
     }
 
     @Test
@@ -710,7 +734,7 @@ public void inMemoryStatefulTopologyShouldNotCreateStateDirectory() throws Excep
         final String storeName = testName.getMethodName() + "-counts";
         final String globalStoreName = testName.getMethodName() + "-globalStore";
         final Topology topology = getStatefulTopology(inputTopic, outputTopic, globalTopicName, storeName, globalStoreName, false);
-        startStreamsAndCheckDirExists(topology, asList(inputTopic, globalTopicName), outputTopic, false);
+        startStreamsAndCheckDirExists(topology, false);
     }
 
     @Test
@@ -721,7 +745,7 @@ public void statefulTopologyShouldCreateStateDirectory() throws Exception {
         final String storeName = testName.getMethodName() + "-counts";
         final String globalStoreName = testName.getMethodName() + "-globalStore";
         final Topology topology = getStatefulTopology(inputTopic, outputTopic, globalTopicName, storeName, globalStoreName, true);
-        startStreamsAndCheckDirExists(topology, asList(inputTopic, globalTopicName), outputTopic, true);
+        startStreamsAndCheckDirExists(topology, true);
     }
 
     @SuppressWarnings("unchecked")
@@ -730,8 +754,7 @@ private Topology getStatefulTopology(final String inputTopic,
                                          final String globalTopicName,
                                          final String storeName,
                                          final String globalStoreName,
-                                         final boolean isPersistentStore) throws Exception {
-        CLUSTER.createTopics(inputTopic, outputTopic, globalTopicName);
+                                         final boolean isPersistentStore) {
         final StoreBuilder> storeBuilder = Stores.keyValueStoreBuilder(
             isPersistentStore ?
                 Stores.persistentKeyValueStore(storeName)
@@ -740,110 +763,45 @@ private Topology getStatefulTopology(final String inputTopic,
             Serdes.Long());
         final Topology topology = new Topology();
         topology.addSource("source", Serdes.String().deserializer(), Serdes.String().deserializer(), inputTopic)
-                .addProcessor("process", () -> new AbstractProcessor() {
-                    @Override
-                    public void process(final String key, final String value) {
-                        final KeyValueStore kvStore =
-                                (KeyValueStore) context().getStateStore(storeName);
-                        kvStore.put(key, 5L);
-
-                        context().forward(key, "5");
-                        context().commit();
-                    }
-                }, "source")
-                .addStateStore(storeBuilder, "process")
-                .addSink("sink", outputTopic, new StringSerializer(), new StringSerializer(), "process");
+            .addProcessor("process", () -> new AbstractProcessor() {
+                @Override
+                public void process(final String key, final String value) {
+                    final KeyValueStore kvStore =
+                        (KeyValueStore) context().getStateStore(storeName);
+                    kvStore.put(key, 5L);
+
+                    context().forward(key, "5");
+                    context().commit();
+                }
+            }, "source")
+            .addStateStore(storeBuilder, "process")
+            .addSink("sink", outputTopic, new StringSerializer(), new StringSerializer(), "process");
 
         final StoreBuilder> globalStoreBuilder = Stores.keyValueStoreBuilder(
-                isPersistentStore ? Stores.persistentKeyValueStore(globalStoreName) : Stores.inMemoryKeyValueStore(globalStoreName),
-                Serdes.String(), Serdes.String()).withLoggingDisabled();
+            isPersistentStore ? Stores.persistentKeyValueStore(globalStoreName) : Stores.inMemoryKeyValueStore(globalStoreName),
+            Serdes.String(), Serdes.String()).withLoggingDisabled();
         topology.addGlobalStore(globalStoreBuilder,
-                "global",
-                Serdes.String().deserializer(),
-                Serdes.String().deserializer(),
-                globalTopicName,
-                globalTopicName + "-processor",
-                new MockProcessorSupplier());
+            "global",
+            Serdes.String().deserializer(),
+            Serdes.String().deserializer(),
+            globalTopicName,
+            globalTopicName + "-processor",
+            new MockProcessorSupplier());
         return topology;
     }
 
     private void startStreamsAndCheckDirExists(final Topology topology,
-                                               final Collection inputTopics,
-                                               final String outputTopic,
                                                final boolean shouldFilesExist) throws Exception {
-        final File baseDir = new File(TestUtils.IO_TMP_DIR + File.separator + "kafka-" + TestUtils.randomString(5));
-        final Path basePath = baseDir.toPath();
-        if (!baseDir.exists()) {
-            Files.createDirectory(basePath);
-        }
-        // changing the path of state directory to make sure that it should not clash with other test cases.
-        final Properties localProps = new Properties();
-        localProps.putAll(props);
-        localProps.put(StreamsConfig.STATE_DIR_CONFIG, baseDir.getAbsolutePath());
-
-        final KafkaStreams streams = new KafkaStreams(topology, localProps);
-        streams.start();
-
-        for (final String topic : inputTopics) {
-            IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(topic,
-                    Collections.singletonList(new KeyValue<>("A", "A")),
-                    TestUtils.producerConfig(
-                            CLUSTER.bootstrapServers(),
-                            StringSerializer.class,
-                            StringSerializer.class,
-                            new Properties()),
-                    System.currentTimeMillis());
-        }
-
-        IntegrationTestUtils.readKeyValues(outputTopic,
-                TestUtils.consumerConfig(
-                        CLUSTER.bootstrapServers(),
-                        outputTopic + "-group",
-                        StringDeserializer.class,
-                        StringDeserializer.class),
-                5000, 1);
-
-        try {
-            final List files = Files.find(basePath, 999, (p, bfa) -> !p.equals(basePath)).collect(Collectors.toList());
-            if (shouldFilesExist && files.isEmpty()) {
-                Assert.fail("Files should have existed, but it didn't: " + files);
-            }
-            if (!shouldFilesExist && !files.isEmpty()) {
-                Assert.fail("Files should not have existed, but it did: " + files);
-            }
-        } catch (final IOException e) {
-            Assert.fail("Couldn't read the state directory : " + baseDir.getPath());
-        } finally {
-            streams.close();
-            streams.cleanUp();
-            Utils.delete(baseDir);
-        }
-    }
+        PowerMock.expectNew(StateDirectory.class,
+            anyObject(StreamsConfig.class),
+            anyObject(Time.class),
+            EasyMock.eq(shouldFilesExist)
+        ).andReturn(stateDirectory);
 
-    private void verifyCleanupStateDir(final String appDir,
-                                       final File oldTaskDir) throws InterruptedException {
-        final File taskDir = new File(appDir, "0_0");
-        TestUtils.waitForCondition(
-            () -> !oldTaskDir.exists() && taskDir.exists(),
-            "cleanup has not successfully run");
-        assertTrue(taskDir.exists());
-    }
+        PowerMock.replayAll();
 
-    public static class StateListenerStub implements KafkaStreams.StateListener {
-        int numChanges = 0;
-        KafkaStreams.State oldState;
-        KafkaStreams.State newState;
-        public Map mapStates = new HashMap<>();
+        new KafkaStreams(topology, props, supplier, time);
 
-        @Override
-        public void onChange(final KafkaStreams.State newState,
-                             final KafkaStreams.State oldState) {
-            final long prevCount = mapStates.containsKey(newState) ? mapStates.get(newState) : 0;
-            numChanges++;
-            this.oldState = oldState;
-            this.newState = newState;
-            mapStates.put(newState, prevCount + 1);
-        }
+        PowerMock.verifyAll();
     }
-
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java b/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
index c9a168912a0a5..461500e529996 100644
--- a/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.streams;
 
+import org.apache.kafka.clients.CommonClientConfigs;
 import org.apache.kafka.clients.admin.AdminClientConfig;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
@@ -80,6 +81,31 @@ public void setUp() {
         streamsConfig = new StreamsConfig(props);
     }
 
+    @Test(expected = ConfigException.class)
+    public void testIllegalMetricsRecordingLevel() {
+        props.put(StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG, "illegalConfig");
+        new StreamsConfig(props);
+    }
+
+    @Test
+    public void testOsDefaultSocketBufferSizes() {
+        props.put(StreamsConfig.SEND_BUFFER_CONFIG, CommonClientConfigs.RECEIVE_BUFFER_LOWER_BOUND);
+        props.put(StreamsConfig.RECEIVE_BUFFER_CONFIG, CommonClientConfigs.RECEIVE_BUFFER_LOWER_BOUND);
+        new StreamsConfig(props);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void testInvalidSocketSendBufferSize() {
+        props.put(StreamsConfig.SEND_BUFFER_CONFIG, -2);
+        new StreamsConfig(props);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void testInvalidSocketReceiveBufferSize() {
+        props.put(StreamsConfig.RECEIVE_BUFFER_CONFIG, -2);
+        new StreamsConfig(props);
+    }
+
     @Test(expected = ConfigException.class)
     public void shouldThrowExceptionIfApplicationIdIsNotSet() {
         props.remove(StreamsConfig.APPLICATION_ID_CONFIG);
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
index 3339080a90f35..ef5b95b979afe 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
@@ -79,6 +79,7 @@
 import java.io.File;
 import java.time.Duration;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -222,7 +223,7 @@ public void testStateChangeStartClose() throws Exception {
     }
 
     private Cluster createCluster() {
-        final Node node = new Node(0, "localhost", 8121);
+        final Node node = new Node(-1, "localhost", 8121);
         return new Cluster(
             "mockClusterId",
             singletonList(node),
@@ -1038,10 +1039,19 @@ public void shouldReturnActiveTaskMetadataWhileRunningState() {
 
         thread.runOnce();
 
-        final ThreadMetadata threadMetadata = thread.threadMetadata();
-        assertEquals(StreamThread.State.RUNNING.name(), threadMetadata.threadState());
-        assertTrue(threadMetadata.activeTasks().contains(new TaskMetadata(task1.toString(), Utils.mkSet(t1p1))));
-        assertTrue(threadMetadata.standbyTasks().isEmpty());
+        final ThreadMetadata metadata = thread.threadMetadata();
+        assertEquals(StreamThread.State.RUNNING.name(), metadata.threadState());
+        assertTrue(metadata.activeTasks().contains(new TaskMetadata(task1.toString(), Utils.mkSet(t1p1))));
+        assertTrue(metadata.standbyTasks().isEmpty());
+
+        assertTrue("#threadState() was: " + metadata.threadState() + "; expected either RUNNING, STARTING, PARTITIONS_REVOKED, PARTITIONS_ASSIGNED, or CREATED",
+            Arrays.asList("RUNNING", "STARTING", "PARTITIONS_REVOKED", "PARTITIONS_ASSIGNED", "CREATED").contains(metadata.threadState()));
+        final String threadName = metadata.threadName();
+        assertTrue(threadName.startsWith("clientId-StreamThread-"));
+        assertEquals(threadName + "-consumer", metadata.consumerClientId());
+        assertEquals(threadName + "-restore-consumer", metadata.restoreConsumerClientId());
+        assertEquals(Collections.singleton(threadName + "-producer"), metadata.producerClientIds());
+        assertEquals("clientId-admin", metadata.adminClientId());
     }
 
     @Test
diff --git a/streams/src/test/java/org/apache/kafka/test/MockClientSupplier.java b/streams/src/test/java/org/apache/kafka/test/MockClientSupplier.java
index c199ac7a66e67..746dbd101a1a7 100644
--- a/streams/src/test/java/org/apache/kafka/test/MockClientSupplier.java
+++ b/streams/src/test/java/org/apache/kafka/test/MockClientSupplier.java
@@ -58,7 +58,7 @@ public void setClusterForAdminClient(final Cluster cluster) {
 
     @Override
     public Admin getAdmin(final Map config) {
-        return new MockAdminClient(cluster.nodes(), cluster.nodeById(0));
+        return new MockAdminClient(cluster.nodes(), cluster.nodeById(-1));
     }
 
     @Override

From 863210d172cacd8abccc4807f393116700bf19a9 Mon Sep 17 00:00:00 2001
From: Bruno Cadonna 
Date: Fri, 27 Sep 2019 21:12:30 +0200
Subject: [PATCH 0653/1071] KAFKA-8934: Create version file during build for
 Streams (#7397)

The Kafka Clients library includes a version file that contains its version and Git commit ID.
Since Kafka Streams wants to expose version and commit ID in the metrics it needs to read the version file. To enable the users to check during runtime for version mismatches between the Streams library and the Clients library, the version file is copied from Clients during build
time and during runtime only the Streams version file is read.

If Streams would read Clients' version file during runtime, it would read a wrong version and commit ID if the libraries where not build from repositories in different states.

Reviewers: John Roesler , Guozhang Wang 
---
 build.gradle | 40 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/build.gradle b/build.gradle
index d80118330d611..2b7083f2921e1 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1123,6 +1123,7 @@ project(':tools') {
 
 project(':streams') {
   archivesBaseName = "kafka-streams"
+  ext.buildStreamsVersionFileName = "kafka-streams-version.properties"
 
   dependencies {
     compile project(':clients')
@@ -1170,7 +1171,46 @@ project(':streams') {
     duplicatesStrategy 'exclude'
   }
 
+  task determineCommitId {
+    def takeFromHash = 16
+    if (commitId) {
+      commitId = commitId.take(takeFromHash)
+    } else if (file("$rootDir/.git/HEAD").exists()) {
+      def headRef = file("$rootDir/.git/HEAD").text
+      if (headRef.contains('ref: ')) {
+        headRef = headRef.replaceAll('ref: ', '').trim()
+        if (file("$rootDir/.git/$headRef").exists()) {
+          commitId = file("$rootDir/.git/$headRef").text.trim().take(takeFromHash)
+        }
+      } else {
+        commitId = headRef.trim().take(takeFromHash)
+      }
+    } else {
+      commitId = "unknown"
+    }
+  }
+
+  task createStreamsVersionFile(dependsOn: determineCommitId) {
+    ext.receiptFile = file("$buildDir/kafka/$buildStreamsVersionFileName")
+    outputs.file receiptFile
+    outputs.upToDateWhen { false }
+    doLast {
+      def data = [
+              commitId: commitId,
+              version: version,
+      ]
+
+      receiptFile.parentFile.mkdirs()
+      def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n")
+      receiptFile.setText(content, "ISO-8859-1")
+    }
+  }
+
   jar {
+    dependsOn 'createStreamsVersionFile'
+    from("$buildDir") {
+      include "kafka/$buildStreamsVersionFileName"
+    }
     dependsOn 'copyDependantLibs'
   }
 

From 8818a7037dff8b099b2516460b34353ca0c38a2b Mon Sep 17 00:00:00 2001
From: huxi 
Date: Sat, 28 Sep 2019 04:11:27 +0800
Subject: [PATCH 0654/1071] MINOR:fixed typo and removed outdated varilable
 name (#7402)

Reviewer: Matthias J. Sax 
---
 .../main/scala/kafka/server/epoch/LeaderEpochFileCache.scala  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/kafka/server/epoch/LeaderEpochFileCache.scala b/core/src/main/scala/kafka/server/epoch/LeaderEpochFileCache.scala
index e6dd0646b661e..9d14ae6f2bd09 100644
--- a/core/src/main/scala/kafka/server/epoch/LeaderEpochFileCache.scala
+++ b/core/src/main/scala/kafka/server/epoch/LeaderEpochFileCache.scala
@@ -84,7 +84,7 @@ class LeaderEpochFileCache(topicPartition: TopicPartition,
       debug(s"Appended new epoch entry $entryToAppend. Cache now contains ${epochs.size} entries.")
     } else if (removedEpochs.size > 1 || removedEpochs.head.startOffset != entryToAppend.startOffset) {
       // Only log a warning if there were non-trivial removals. If the start offset of the new entry
-      // matches the start offfset of the removed epoch, then no data has been written and the truncation
+      // matches the start offset of the removed epoch, then no data has been written and the truncation
       // is expected.
       warn(s"New epoch entry $entryToAppend caused truncation of conflicting entries $removedEpochs. " +
         s"Cache now contains ${epochs.size} entries.")
@@ -123,7 +123,7 @@ class LeaderEpochFileCache(topicPartition: TopicPartition,
     * Offset if the latest epoch was requested.
     *
     * During the upgrade phase, where there are existing messages may not have a leader epoch,
-    * if requestedEpoch is < the first epoch cached, UNSUPPORTED_EPOCH_OFFSET will be returned
+    * if requestedEpoch is < the first epoch cached, UNDEFINED_EPOCH_OFFSET will be returned
     * so that the follower falls back to High Water Mark.
     *
     * @param requestedEpoch requested leader epoch

From 1dc5063cde2101e4efb058a24379a7c7cb97db5e Mon Sep 17 00:00:00 2001
From: Lucas Bradstreet 
Date: Sat, 28 Sep 2019 17:03:06 -0700
Subject: [PATCH 0655/1071] MINOR: Don't generate unnecessary strings for debug
 logging in FetchSessionHandler (#7394)

Profiling while benchmarking shows unnecessary calls to
`responseDataToLogString` in FetchSessionHandler when logging was set to
INFO level. This leads to 1.47% of the JVM CPU time going to this method.
Fix it by checking if debug logging is enabled.

Reviewers: Ismael Juma 
---
 .../kafka/clients/FetchSessionHandler.java    | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java b/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java
index 494bbbe4769e1..0dc8943fdcbe6 100644
--- a/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java
+++ b/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java
@@ -397,14 +397,15 @@ public boolean handleResponse(FetchResponse response) {
                 nextMetadata = FetchMetadata.INITIAL;
                 return false;
             } else if (response.sessionId() == INVALID_SESSION_ID) {
-                log.debug("Node {} sent a full fetch response{}",
-                    node, responseDataToLogString(response));
+                if (log.isDebugEnabled())
+                    log.debug("Node {} sent a full fetch response{}", node, responseDataToLogString(response));
                 nextMetadata = FetchMetadata.INITIAL;
                 return true;
             } else {
                 // The server created a new incremental fetch session.
-                log.debug("Node {} sent a full fetch response that created a new incremental " +
-                    "fetch session {}{}", node, response.sessionId(), responseDataToLogString(response));
+                if (log.isDebugEnabled())
+                    log.debug("Node {} sent a full fetch response that created a new incremental " +
+                            "fetch session {}{}", node, response.sessionId(), responseDataToLogString(response));
                 nextMetadata = FetchMetadata.newIncremental(response.sessionId());
                 return true;
             }
@@ -416,14 +417,16 @@ public boolean handleResponse(FetchResponse response) {
                 return false;
             } else if (response.sessionId() == INVALID_SESSION_ID) {
                 // The incremental fetch session was closed by the server.
-                log.debug("Node {} sent an incremental fetch response closing session {}{}",
-                    node, nextMetadata.sessionId(), responseDataToLogString(response));
+                if (log.isDebugEnabled())
+                    log.debug("Node {} sent an incremental fetch response closing session {}{}",
+                            node, nextMetadata.sessionId(), responseDataToLogString(response));
                 nextMetadata = FetchMetadata.INITIAL;
                 return true;
             } else {
                 // The incremental fetch session was continued by the server.
-                log.debug("Node {} sent an incremental fetch response for session {}{}",
-                    node, response.sessionId(), responseDataToLogString(response));
+                if (log.isDebugEnabled())
+                    log.debug("Node {} sent an incremental fetch response for session {}{}",
+                            node, response.sessionId(), responseDataToLogString(response));
                 nextMetadata = nextMetadata.nextIncremental();
                 return true;
             }

From 66183f730feaa7dca7b4088053f44015a626a6e8 Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Sat, 28 Sep 2019 19:39:45 -0700
Subject: [PATCH 0656/1071] KAFKA-8471: Replace control requests/responses with
 automated protocol (#7353)

Replaced UpdateMetadata{Request, Response}, LeaderAndIsr{Request, Response}
and StopReplica{Request, Response} with the automated protocol classes.

Updated the JSON schema for the 3 request types to be more consistent and
less strict (if needed to avoid duplication).

The general approach is to avoid generating new collections in the request
classes. Normalization happens in the constructor to make this possible. Builders
still have to group by topic to maintain the external ungrouped view.

Introduced new tests for LeaderAndIsrRequest and UpdateMetadataRequest to
verify that the new logic is correct.

A few other clean-ups/fixes in code that was touched due to these changes:
* KAFKA-8956: Refactor DelayedCreatePartitions#updateWaiting to avoid modifying
collection in foreach.
* Avoid unnecessary allocation for state change trace logging if trace logging is not enabled
* Use `toBuffer` instead of `toList`, `toIndexedSeq` or `toSeq` as it generally performs
better and it matches the performance characteristics of `java.util.ArrayList`. This is
particularly important when passing such instances to Java code.
* Minor refactoring for clarity and readability.
* Removed usage of deprecated `/:`, unused imports and unnecessary `var`s.
* Include exception in `AdminClientIntegrationTest` failure message.
* Move StopReplicaRequest verification in `AuthorizerIntegrationTest` to the end
to match the comment.

Reviewers: Colin Patrick McCabe 
---
 .../apache/kafka/common/protocol/ApiKeys.java |  19 +-
 .../kafka/common/protocol/types/Struct.java   |   1 +
 .../requests/AbstractControlRequest.java      |  42 +-
 .../common/requests/AbstractResponse.java     |  11 +-
 .../requests/AddPartitionsToTxnResponse.java  |   2 +-
 .../requests/AlterReplicaLogDirsResponse.java |   2 +-
 .../common/requests/LeaderAndIsrRequest.java  | 430 ++++----------
 .../common/requests/LeaderAndIsrResponse.java |  89 +--
 .../common/requests/OffsetCommitResponse.java |   8 +-
 .../common/requests/StopReplicaRequest.java   | 190 +++----
 .../common/requests/StopReplicaResponse.java  |  85 +--
 .../requests/TxnOffsetCommitResponse.java     |   2 +-
 .../requests/UpdateMetadataRequest.java       | 527 +++++-------------
 .../requests/UpdateMetadataResponse.java      |  43 +-
 .../kafka/common/utils/FlattenedIterator.java |  45 ++
 .../kafka/common/utils/MappedIterator.java    |  44 ++
 .../common/message/LeaderAndIsrRequest.json   |  16 +-
 .../common/message/LeaderAndIsrResponse.json  |   2 +-
 .../common/message/StopReplicaRequest.json    |   4 +-
 .../common/message/StopReplicaResponse.json   |   2 +-
 .../common/message/UpdateMetadataRequest.json |  14 +-
 .../kafka/common/message/MessageTest.java     |  48 +-
 .../common/requests/ControlRequestTest.java   |  87 ---
 .../requests/LeaderAndIsrRequestTest.java     | 147 +++++
 .../requests/LeaderAndIsrResponseTest.java    |  80 ++-
 .../common/requests/RequestResponseTest.java  | 158 ++++--
 .../requests/StopReplicaRequestTest.java}     |  28 +-
 .../requests/StopReplicaResponseTest.java     |  38 +-
 .../requests/UpdateMetadataRequestTest.java   | 187 +++++++
 .../common/utils/FlattenedIteratorTest.java   | 115 ++++
 .../common/utils/MappedIteratorTest.java      |  61 ++
 .../java/org/apache/kafka/test/TestUtils.java |  11 +
 .../main/scala/kafka/cluster/Partition.scala  |  33 +-
 .../controller/ControllerChannelManager.scala | 115 ++--
 .../kafka/controller/KafkaController.scala    |  16 +-
 .../scala/kafka/server/AdminManager.scala     |   6 +-
 .../server/DelayedCreatePartitions.scala      |   2 +-
 .../kafka/server/DelayedElectLeader.scala     |  23 +-
 .../main/scala/kafka/server/KafkaApis.scala   |  51 +-
 .../scala/kafka/server/MetadataCache.scala    |  59 +-
 .../scala/kafka/server/ReplicaManager.scala   |  95 ++--
 .../main/scala/kafka/zk/KafkaZkClient.scala   |   2 +-
 .../kafka/api/AbstractConsumerTest.scala      |   6 +-
 .../api/AdminClientIntegrationTest.scala      |   8 +-
 .../kafka/api/AuthorizerIntegrationTest.scala |  54 +-
 .../DynamicBrokerReconfigurationTest.scala    |   2 +-
 ...rredReplicaLeaderElectionCommandTest.scala |   7 +-
 .../unit/kafka/cluster/PartitionTest.scala    | 255 +++++++--
 .../ControllerChannelManagerTest.scala        |  75 ++-
 .../ControllerIntegrationTest.scala           |  26 +-
 .../UncleanLeaderElectionTest.scala           |   2 +-
 .../unit/kafka/network/SocketServerTest.scala |   2 +-
 .../server/BrokerEpochIntegrationTest.scala   |  55 +-
 .../unit/kafka/server/FetchRequestTest.scala  |   5 +-
 .../unit/kafka/server/KafkaApisTest.scala     |  59 +-
 .../kafka/server/LeaderElectionTest.scala     |  26 +-
 .../unit/kafka/server/MetadataCacheTest.scala | 239 ++++++--
 .../kafka/server/ReplicaManagerTest.scala     | 269 +++++++--
 .../unit/kafka/server/RequestQuotaTest.scala  |  35 +-
 .../kafka/server/ServerShutdownTest.scala     |   2 +-
 .../kafka/server/StopReplicaRequestTest.scala |  11 +-
 .../epoch/LeaderEpochIntegrationTest.scala    |   5 +-
 .../scala/unit/kafka/utils/TestUtils.scala    |  18 +-
 .../UpdateFollowerFetchStateBenchmark.java    |  13 +-
 .../utils/IntegrationTestUtils.java           |   8 +-
 65 files changed, 2354 insertions(+), 1768 deletions(-)
 create mode 100644 clients/src/main/java/org/apache/kafka/common/utils/FlattenedIterator.java
 create mode 100644 clients/src/main/java/org/apache/kafka/common/utils/MappedIterator.java
 delete mode 100644 clients/src/test/java/org/apache/kafka/common/requests/ControlRequestTest.java
 create mode 100644 clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java
 rename clients/src/{main/java/org/apache/kafka/common/requests/BasePartitionState.java => test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java} (53%)
 create mode 100644 clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java
 create mode 100644 clients/src/test/java/org/apache/kafka/common/utils/FlattenedIteratorTest.java
 create mode 100644 clients/src/test/java/org/apache/kafka/common/utils/MappedIteratorTest.java

diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
index cd0132723c47b..ba59940175ea8 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
@@ -44,6 +44,8 @@
 import org.apache.kafka.common.message.InitProducerIdResponseData;
 import org.apache.kafka.common.message.JoinGroupRequestData;
 import org.apache.kafka.common.message.JoinGroupResponseData;
+import org.apache.kafka.common.message.LeaderAndIsrRequestData;
+import org.apache.kafka.common.message.LeaderAndIsrResponseData;
 import org.apache.kafka.common.message.LeaveGroupRequestData;
 import org.apache.kafka.common.message.LeaveGroupResponseData;
 import org.apache.kafka.common.message.ListGroupsRequestData;
@@ -66,8 +68,12 @@
 import org.apache.kafka.common.message.SaslAuthenticateResponseData;
 import org.apache.kafka.common.message.SaslHandshakeRequestData;
 import org.apache.kafka.common.message.SaslHandshakeResponseData;
+import org.apache.kafka.common.message.StopReplicaRequestData;
+import org.apache.kafka.common.message.StopReplicaResponseData;
 import org.apache.kafka.common.message.SyncGroupRequestData;
 import org.apache.kafka.common.message.SyncGroupResponseData;
+import org.apache.kafka.common.message.UpdateMetadataRequestData;
+import org.apache.kafka.common.message.UpdateMetadataResponseData;
 import org.apache.kafka.common.message.TxnOffsetCommitRequestData;
 import org.apache.kafka.common.message.TxnOffsetCommitResponseData;
 import org.apache.kafka.common.protocol.types.Schema;
@@ -103,18 +109,12 @@
 import org.apache.kafka.common.requests.EndTxnResponse;
 import org.apache.kafka.common.requests.FetchRequest;
 import org.apache.kafka.common.requests.FetchResponse;
-import org.apache.kafka.common.requests.LeaderAndIsrRequest;
-import org.apache.kafka.common.requests.LeaderAndIsrResponse;
 import org.apache.kafka.common.requests.ListOffsetRequest;
 import org.apache.kafka.common.requests.ListOffsetResponse;
 import org.apache.kafka.common.requests.OffsetsForLeaderEpochRequest;
 import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse;
 import org.apache.kafka.common.requests.ProduceRequest;
 import org.apache.kafka.common.requests.ProduceResponse;
-import org.apache.kafka.common.requests.StopReplicaRequest;
-import org.apache.kafka.common.requests.StopReplicaResponse;
-import org.apache.kafka.common.requests.UpdateMetadataRequest;
-import org.apache.kafka.common.requests.UpdateMetadataResponse;
 import org.apache.kafka.common.requests.WriteTxnMarkersRequest;
 import org.apache.kafka.common.requests.WriteTxnMarkersResponse;
 
@@ -133,10 +133,9 @@ public enum ApiKeys {
     FETCH(1, "Fetch", FetchRequest.schemaVersions(), FetchResponse.schemaVersions()),
     LIST_OFFSETS(2, "ListOffsets", ListOffsetRequest.schemaVersions(), ListOffsetResponse.schemaVersions()),
     METADATA(3, "Metadata", MetadataRequestData.SCHEMAS, MetadataResponseData.SCHEMAS),
-    LEADER_AND_ISR(4, "LeaderAndIsr", true, LeaderAndIsrRequest.schemaVersions(), LeaderAndIsrResponse.schemaVersions()),
-    STOP_REPLICA(5, "StopReplica", true, StopReplicaRequest.schemaVersions(), StopReplicaResponse.schemaVersions()),
-    UPDATE_METADATA(6, "UpdateMetadata", true, UpdateMetadataRequest.schemaVersions(),
-            UpdateMetadataResponse.schemaVersions()),
+    LEADER_AND_ISR(4, "LeaderAndIsr", true, LeaderAndIsrRequestData.SCHEMAS, LeaderAndIsrResponseData.SCHEMAS),
+    STOP_REPLICA(5, "StopReplica", true, StopReplicaRequestData.SCHEMAS, StopReplicaResponseData.SCHEMAS),
+    UPDATE_METADATA(6, "UpdateMetadata", true, UpdateMetadataRequestData.SCHEMAS, UpdateMetadataResponseData.SCHEMAS),
     CONTROLLED_SHUTDOWN(7, "ControlledShutdown", true, ControlledShutdownRequestData.SCHEMAS,
             ControlledShutdownResponseData.SCHEMAS),
     OFFSET_COMMIT(8, "OffsetCommit", OffsetCommitRequestData.SCHEMAS, OffsetCommitResponseData.SCHEMAS),
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
index 4aaff74df4d84..0b69851f16441 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
@@ -471,6 +471,7 @@ public void writeTo(ByteBuffer buffer) {
      * @throws SchemaException If validation fails
      */
     private void validateField(BoundField field) {
+        Objects.requireNonNull(field, "`field` must be non-null");
         if (this.schema != field.schema)
             throw new SchemaException("Attempt to access field '" + field.def.name + "' from a different schema instance.");
         if (field.index > values.length)
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractControlRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractControlRequest.java
index d1fae0502c438..dc4a1e21e8dd9 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractControlRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractControlRequest.java
@@ -17,20 +17,11 @@
 package org.apache.kafka.common.requests;
 
 import org.apache.kafka.common.protocol.ApiKeys;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Struct;
 
 // Abstract class for all control requests including UpdateMetadataRequest, LeaderAndIsrRequest and StopReplicaRequest
 public abstract class AbstractControlRequest extends AbstractRequest {
-    public static final long UNKNOWN_BROKER_EPOCH = -1L;
-
-    protected static final Field.Int32 CONTROLLER_ID = new Field.Int32("controller_id", "The controller id");
-    protected static final Field.Int32 CONTROLLER_EPOCH = new Field.Int32("controller_epoch", "The controller epoch");
-    protected static final Field.Int64 BROKER_EPOCH = new Field.Int64("broker_epoch", "The broker epoch");
 
-    protected final int controllerId;
-    protected final int controllerEpoch;
-    protected final long brokerEpoch;
+    public static final long UNKNOWN_BROKER_EPOCH = -1L;
 
     public static abstract class Builder extends AbstractRequest.Builder {
         protected final int controllerId;
@@ -46,35 +37,14 @@ protected Builder(ApiKeys api, short version, int controllerId, int controllerEp
 
     }
 
-    public int controllerId() {
-        return controllerId;
-    }
-
-    public int controllerEpoch() {
-        return controllerEpoch;
-    }
-
-    public long brokerEpoch() {
-        return brokerEpoch;
-    }
-
-    protected AbstractControlRequest(ApiKeys api, short version, int controllerId, int controllerEpoch, long brokerEpoch) {
+    protected AbstractControlRequest(ApiKeys api, short version) {
         super(api, version);
-        this.controllerId = controllerId;
-        this.controllerEpoch = controllerEpoch;
-        this.brokerEpoch = brokerEpoch;
     }
 
-    protected AbstractControlRequest(ApiKeys api, Struct struct, short version) {
-        super(api, version);
-        this.controllerId = struct.get(CONTROLLER_ID);
-        this.controllerEpoch = struct.get(CONTROLLER_EPOCH);
-        this.brokerEpoch = struct.getOrElse(BROKER_EPOCH, UNKNOWN_BROKER_EPOCH);
-    }
+    public abstract int controllerId();
 
-    // Used for test
-    long size() {
-        return toStruct().sizeOf();
-    }
+    public abstract int controllerEpoch();
+
+    public abstract long brokerEpoch();
 
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
index 9fbce91259946..eb11a4244215f 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
@@ -23,6 +23,7 @@
 import org.apache.kafka.common.protocol.types.Struct;
 
 import java.nio.ByteBuffer;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -55,9 +56,9 @@ protected Map errorCounts(Errors error) {
         return Collections.singletonMap(error, 1);
     }
 
-    protected Map errorCounts(Map errors) {
+    protected Map errorCounts(Collection errors) {
         Map errorCounts = new HashMap<>();
-        for (Errors error : errors.values())
+        for (Errors error : errors)
             updateErrorCounts(errorCounts, error);
         return errorCounts;
     }
@@ -101,13 +102,13 @@ public static AbstractResponse parseResponse(ApiKeys apiKey, Struct struct, shor
             case SYNC_GROUP:
                 return new SyncGroupResponse(struct, version);
             case STOP_REPLICA:
-                return new StopReplicaResponse(struct);
+                return new StopReplicaResponse(struct, version);
             case CONTROLLED_SHUTDOWN:
                 return new ControlledShutdownResponse(struct, version);
             case UPDATE_METADATA:
-                return new UpdateMetadataResponse(struct);
+                return new UpdateMetadataResponse(struct, version);
             case LEADER_AND_ISR:
-                return new LeaderAndIsrResponse(struct);
+                return new LeaderAndIsrResponse(struct, version);
             case DESCRIBE_GROUPS:
                 return new DescribeGroupsResponse(struct, version);
             case LIST_GROUPS:
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java
index ea8a073e01755..7f20f07b6a117 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java
@@ -102,7 +102,7 @@ public Map errors() {
 
     @Override
     public Map errorCounts() {
-        return errorCounts(errors);
+        return errorCounts(errors.values());
     }
 
     @Override
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java
index 7a73275d74577..2bbdce2122bc2 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java
@@ -132,7 +132,7 @@ public Map responses() {
 
     @Override
     public Map errorCounts() {
-        return errorCounts(responses);
+        return errorCounts(responses.values());
     }
 
     public static AlterReplicaLogDirsResponse parse(ByteBuffer buffer, short version) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
index 02c14e3274a99..b799e360bd4f4 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
@@ -17,152 +17,37 @@
 package org.apache.kafka.common.requests;
 
 import org.apache.kafka.common.Node;
-import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.message.LeaderAndIsrRequestData;
+import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrLiveLeader;
+import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrTopicState;
+import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState;
+import org.apache.kafka.common.message.LeaderAndIsrResponseData;
+import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError;
 import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.ByteBufferAccessor;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
-import org.apache.kafka.common.utils.CollectionUtils;
+import org.apache.kafka.common.utils.FlattenedIterator;
 import org.apache.kafka.common.utils.Utils;
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
-import java.util.Collections;
-
-import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
-import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
-import static org.apache.kafka.common.protocol.types.Type.INT32;
+import java.util.stream.Collectors;
 
 public class LeaderAndIsrRequest extends AbstractControlRequest {
-    private static final Field.ComplexArray TOPIC_STATES = new Field.ComplexArray("topic_states", "Topic states");
-    private static final Field.ComplexArray PARTITION_STATES = new Field.ComplexArray("partition_states", "Partition states");
-    private static final Field.ComplexArray LIVE_LEADERS = new Field.ComplexArray("live_leaders", "Live leaders");
-
-    // PartitionState fields
-    private static final Field.Int32 LEADER = new Field.Int32("leader", "The broker id for the leader.");
-    private static final Field.Int32 LEADER_EPOCH = new Field.Int32("leader_epoch", "The leader epoch.");
-    private static final Field.Array ISR = new Field.Array("isr", INT32, "The in sync replica ids.");
-    private static final Field.Int32 ZK_VERSION = new Field.Int32("zk_version", "The ZK version.");
-    private static final Field.Array REPLICAS = new Field.Array("replicas", INT32, "The replica ids.");
-    private static final Field.Array ADDING_REPLICAS = new Field.Array("adding_replicas", INT32,
-            "The replica ids we are in the process of adding to the replica set during a reassignment.");
-    private static final Field.Array REMOVING_REPLICAS = new Field.Array("removing_replicas", INT32,
-            "The replica ids we are in the process of removing from the replica set during a reassignment.");
-    private static final Field.Bool IS_NEW = new Field.Bool("is_new", "Whether the replica should have existed on the broker or not");
-
-    // live_leaders fields
-    private static final Field.Int32 END_POINT_ID = new Field.Int32("id", "The broker id");
-    private static final Field.Str HOST = new Field.Str("host", "The hostname of the broker.");
-    private static final Field.Int32  PORT = new Field.Int32("port", "The port on which the broker accepts requests.");
-
-    private static final Field PARTITION_STATES_V0  = PARTITION_STATES.withFields(
-            TOPIC_NAME,
-            PARTITION_ID,
-            CONTROLLER_EPOCH,
-            LEADER,
-            LEADER_EPOCH,
-            ISR,
-            ZK_VERSION,
-            REPLICAS);
-
-    // PARTITION_STATES_V1 added a per-partition is_new Field.
-    // This field specifies whether the replica should have existed on the broker or not.
-    private static final Field PARTITION_STATES_V1  = PARTITION_STATES.withFields(
-            TOPIC_NAME,
-            PARTITION_ID,
-            CONTROLLER_EPOCH,
-            LEADER,
-            LEADER_EPOCH,
-            ISR,
-            ZK_VERSION,
-            REPLICAS,
-            IS_NEW);
-
-    private static final Field PARTITION_STATES_V2  = PARTITION_STATES.withFields(
-            PARTITION_ID,
-            CONTROLLER_EPOCH,
-            LEADER,
-            LEADER_EPOCH,
-            ISR,
-            ZK_VERSION,
-            REPLICAS,
-            IS_NEW);
-
-    private static final Field PARTITION_STATES_V3 = PARTITION_STATES.withFields(
-            PARTITION_ID,
-            CONTROLLER_EPOCH,
-            LEADER,
-            LEADER_EPOCH,
-            ISR,
-            ZK_VERSION,
-            REPLICAS,
-            ADDING_REPLICAS,
-            REMOVING_REPLICAS,
-            IS_NEW);
-
-    // TOPIC_STATES_V2 normalizes TOPIC_STATES_V1 to make it more memory efficient
-    private static final Field TOPIC_STATES_V2 = TOPIC_STATES.withFields(
-            TOPIC_NAME,
-            PARTITION_STATES_V2);
-
-    // TOPIC_STATES_V3 adds two new fields - adding_replicas and removing_replicas
-    private static final Field TOPIC_STATES_V3 = TOPIC_STATES.withFields(
-            TOPIC_NAME,
-            PARTITION_STATES_V3);
-
-    private static final Field LIVE_LEADERS_V0 = LIVE_LEADERS.withFields(
-            END_POINT_ID,
-            HOST,
-            PORT);
-
-    private static final Schema LEADER_AND_ISR_REQUEST_V0 = new Schema(
-            CONTROLLER_ID,
-            CONTROLLER_EPOCH,
-            PARTITION_STATES_V0,
-            LIVE_LEADERS_V0);
-
-    // LEADER_AND_ISR_REQUEST_V1 added a per-partition is_new Field. This field specifies whether the replica should
-    // have existed on the broker or not.
-    private static final Schema LEADER_AND_ISR_REQUEST_V1 = new Schema(
-            CONTROLLER_ID,
-            CONTROLLER_EPOCH,
-            PARTITION_STATES_V1,
-            LIVE_LEADERS_V0);
-
-    // LEADER_AND_ISR_REQUEST_V2 added a broker_epoch Field. This field specifies the generation of the broker across
-    // bounces. It also normalizes partitions under each topic.
-    private static final Schema LEADER_AND_ISR_REQUEST_V2 = new Schema(
-            CONTROLLER_ID,
-            CONTROLLER_EPOCH,
-            BROKER_EPOCH,
-            TOPIC_STATES_V2,
-            LIVE_LEADERS_V0);
-
-    // LEADER_AND_ISR_REQUEST_V3 added two new fields - adding_replicas and removing_replicas.
-    // These fields respectively specify the replica IDs we want to add or remove as part of a reassignment
-    private static final Schema LEADER_AND_ISR_REQUEST_V3 = new Schema(
-            CONTROLLER_ID,
-            CONTROLLER_EPOCH,
-            BROKER_EPOCH,
-            TOPIC_STATES_V3,
-            LIVE_LEADERS_V0);
-
-    public static Schema[] schemaVersions() {
-        return new Schema[]{LEADER_AND_ISR_REQUEST_V0, LEADER_AND_ISR_REQUEST_V1, LEADER_AND_ISR_REQUEST_V2, LEADER_AND_ISR_REQUEST_V3};
-    }
 
     public static class Builder extends AbstractControlRequest.Builder {
-        private final Map partitionStates;
-        private final Set liveLeaders;
+
+        private final List partitionStates;
+        private final Collection liveLeaders;
 
         public Builder(short version, int controllerId, int controllerEpoch, long brokerEpoch,
-                       Map partitionStates, Set liveLeaders) {
+                       List partitionStates, Collection liveLeaders) {
             super(ApiKeys.LEADER_AND_ISR, version, controllerId, controllerEpoch, brokerEpoch);
             this.partitionStates = partitionStates;
             this.liveLeaders = liveLeaders;
@@ -170,7 +55,38 @@ public Builder(short version, int controllerId, int controllerEpoch, long broker
 
         @Override
         public LeaderAndIsrRequest build(short version) {
-            return new LeaderAndIsrRequest(controllerId, controllerEpoch, brokerEpoch, partitionStates, liveLeaders, version);
+            List leaders = liveLeaders.stream().map(n -> new LeaderAndIsrLiveLeader()
+                .setBrokerId(n.id())
+                .setHostName(n.host())
+                .setPort(n.port())
+            ).collect(Collectors.toList());
+
+            LeaderAndIsrRequestData data = new LeaderAndIsrRequestData()
+                .setControllerId(controllerId)
+                .setControllerEpoch(controllerEpoch)
+                .setBrokerEpoch(brokerEpoch)
+                .setLiveLeaders(leaders);
+
+            if (version >= 2) {
+                Map topicStatesMap = groupByTopic(partitionStates);
+                data.setTopicStates(new ArrayList<>(topicStatesMap.values()));
+            } else {
+                data.setUngroupedPartitionStates(partitionStates);
+            }
+
+            return new LeaderAndIsrRequest(data, version);
+        }
+
+        private static Map groupByTopic(List partitionStates) {
+            Map topicStates = new HashMap<>();
+            // We don't null out the topic name in LeaderAndIsrRequestPartition since it's ignored by
+            // the generated code if version >= 2
+            for (LeaderAndIsrPartitionState partition : partitionStates) {
+                LeaderAndIsrTopicState topicState = topicStates.computeIfAbsent(partition.topicName(),
+                    t -> new LeaderAndIsrTopicState().setTopicName(partition.topicName()));
+                topicState.partitionStates().add(partition);
+            }
+            return topicStates;
         }
 
         @Override
@@ -184,116 +100,60 @@ public String toString() {
                 .append(", liveLeaders=(").append(Utils.join(liveLeaders, ", ")).append(")")
                 .append(")");
             return bld.toString();
+
         }
     }
 
-    private final Map partitionStates;
-    private final Set liveLeaders;
+    private final LeaderAndIsrRequestData data;
 
-    private LeaderAndIsrRequest(int controllerId, int controllerEpoch, long brokerEpoch, Map partitionStates,
-                                Set liveLeaders, short version) {
-        super(ApiKeys.LEADER_AND_ISR, version, controllerId, controllerEpoch, brokerEpoch);
-        this.partitionStates = partitionStates;
-        this.liveLeaders = liveLeaders;
+    LeaderAndIsrRequest(LeaderAndIsrRequestData data, short version) {
+        super(ApiKeys.LEADER_AND_ISR, version);
+        this.data = data;
+        // Do this from the constructor to make it thread-safe (even though it's only needed when some methods are called)
+        normalize();
     }
 
-    public LeaderAndIsrRequest(Struct struct, short version) {
-        super(ApiKeys.LEADER_AND_ISR, struct, version);
-
-        Map partitionStates = new HashMap<>();
-        if (struct.hasField(TOPIC_STATES)) {
-            for (Object topicStatesDataObj : struct.get(TOPIC_STATES)) {
-                Struct topicStatesData = (Struct) topicStatesDataObj;
-                String topic = topicStatesData.get(TOPIC_NAME);
-                for (Object partitionStateDataObj : topicStatesData.get(PARTITION_STATES)) {
-                    Struct partitionStateData = (Struct) partitionStateDataObj;
-                    int partition = partitionStateData.get(PARTITION_ID);
-                    PartitionState partitionState = new PartitionState(partitionStateData);
-                    partitionStates.put(new TopicPartition(topic, partition), partitionState);
+    private void normalize() {
+        if (version() >= 2) {
+            for (LeaderAndIsrTopicState topicState : data.topicStates()) {
+                for (LeaderAndIsrPartitionState partitionState : topicState.partitionStates()) {
+                    // Set the topic name so that we can always present the ungrouped view to callers
+                    partitionState.setTopicName(topicState.topicName());
                 }
             }
-        } else {
-            for (Object partitionStateDataObj : struct.get(PARTITION_STATES)) {
-                Struct partitionStateData = (Struct) partitionStateDataObj;
-                String topic = partitionStateData.get(TOPIC_NAME);
-                int partition = partitionStateData.get(PARTITION_ID);
-                PartitionState partitionState = new PartitionState(partitionStateData);
-                partitionStates.put(new TopicPartition(topic, partition), partitionState);
-            }
-        }
-
-        Set leaders = new HashSet<>();
-        for (Object leadersDataObj : struct.get(LIVE_LEADERS)) {
-            Struct leadersData = (Struct) leadersDataObj;
-            int id = leadersData.get(END_POINT_ID);
-            String host = leadersData.get(HOST);
-            int port = leadersData.get(PORT);
-            leaders.add(new Node(id, host, port));
         }
+    }
 
-        this.partitionStates = partitionStates;
-        this.liveLeaders = leaders;
+    public LeaderAndIsrRequest(Struct struct, short version) {
+        this(new LeaderAndIsrRequestData(struct, version), version);
     }
 
     @Override
     protected Struct toStruct() {
-        short version = version();
-        Struct struct = new Struct(ApiKeys.LEADER_AND_ISR.requestSchema(version));
-        struct.set(CONTROLLER_ID, controllerId);
-        struct.set(CONTROLLER_EPOCH, controllerEpoch);
-        struct.setIfExists(BROKER_EPOCH, brokerEpoch);
-
-        if (struct.hasField(TOPIC_STATES)) {
-            Map> topicStates = CollectionUtils.groupPartitionDataByTopic(partitionStates);
-            List topicStatesData = new ArrayList<>(topicStates.size());
-            for (Map.Entry> entry : topicStates.entrySet()) {
-                Struct topicStateData = struct.instance(TOPIC_STATES);
-                topicStateData.set(TOPIC_NAME, entry.getKey());
-                Map partitionMap = entry.getValue();
-                List partitionStatesData = new ArrayList<>(partitionMap.size());
-                for (Map.Entry partitionEntry : partitionMap.entrySet()) {
-                    Struct partitionStateData = topicStateData.instance(PARTITION_STATES);
-                    partitionStateData.set(PARTITION_ID, partitionEntry.getKey());
-                    partitionEntry.getValue().setStruct(partitionStateData, version);
-                    partitionStatesData.add(partitionStateData);
-                }
-                topicStateData.set(PARTITION_STATES, partitionStatesData.toArray());
-                topicStatesData.add(topicStateData);
-            }
-            struct.set(TOPIC_STATES, topicStatesData.toArray());
-        } else {
-            List partitionStatesData = new ArrayList<>(partitionStates.size());
-            for (Map.Entry entry : partitionStates.entrySet()) {
-                Struct partitionStateData = struct.instance(PARTITION_STATES);
-                TopicPartition topicPartition = entry.getKey();
-                partitionStateData.set(TOPIC_NAME, topicPartition.topic());
-                partitionStateData.set(PARTITION_ID, topicPartition.partition());
-                entry.getValue().setStruct(partitionStateData, version);
-                partitionStatesData.add(partitionStateData);
-            }
-            struct.set(PARTITION_STATES, partitionStatesData.toArray());
-        }
+        return data.toStruct(version());
+    }
 
-        List leadersData = new ArrayList<>(liveLeaders.size());
-        for (Node leader : liveLeaders) {
-            Struct leaderData = struct.instance(LIVE_LEADERS);
-            leaderData.set(END_POINT_ID, leader.id());
-            leaderData.set(HOST, leader.host());
-            leaderData.set(PORT, leader.port());
-            leadersData.add(leaderData);
-        }
-        struct.set(LIVE_LEADERS, leadersData.toArray());
-        return struct;
+    protected ByteBuffer toBytes() {
+        ByteBuffer bytes = ByteBuffer.allocate(size());
+        data.write(new ByteBufferAccessor(bytes), version());
+        bytes.flip();
+        return bytes;
     }
 
     @Override
     public LeaderAndIsrResponse getErrorResponse(int throttleTimeMs, Throwable e) {
+        LeaderAndIsrResponseData responseData = new LeaderAndIsrResponseData();
         Errors error = Errors.forException(e);
-
-        Map responses = new HashMap<>(partitionStates.size());
-        for (TopicPartition partition : partitionStates.keySet()) {
-            responses.put(partition, error);
+        responseData.setErrorCode(error.code());
+
+        List partitions = new ArrayList<>();
+        for (LeaderAndIsrPartitionState partition : partitionStates()) {
+            partitions.add(new LeaderAndIsrPartitionError()
+                .setTopicName(partition.topicName())
+                .setPartitionIndex(partition.partitionIndex())
+                .setErrorCode(error.code()));
         }
+        responseData.setPartitionErrors(partitions);
 
         short versionId = version();
         switch (versionId) {
@@ -301,133 +161,45 @@ public LeaderAndIsrResponse getErrorResponse(int throttleTimeMs, Throwable e) {
             case 1:
             case 2:
             case 3:
-                return new LeaderAndIsrResponse(error, responses);
+                return new LeaderAndIsrResponse(responseData);
             default:
                 throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
                         versionId, this.getClass().getSimpleName(), ApiKeys.LEADER_AND_ISR.latestVersion()));
         }
     }
 
+    @Override
     public int controllerId() {
-        return controllerId;
+        return data.controllerId();
     }
 
+    @Override
     public int controllerEpoch() {
-        return controllerEpoch;
+        return data.controllerEpoch();
     }
 
-    public Map partitionStates() {
-        return partitionStates;
+    @Override
+    public long brokerEpoch() {
+        return data.brokerEpoch();
     }
 
-    public Set liveLeaders() {
-        return liveLeaders;
+    public Iterable partitionStates() {
+        if (version() >= 2)
+            return () -> new FlattenedIterator<>(data.topicStates().iterator(),
+                topicState -> topicState.partitionStates().iterator());
+        return data.ungroupedPartitionStates();
     }
 
-    public static LeaderAndIsrRequest parse(ByteBuffer buffer, short version) {
-        return new LeaderAndIsrRequest(ApiKeys.LEADER_AND_ISR.parseRequest(version, buffer), version);
+    public List liveLeaders() {
+        return Collections.unmodifiableList(data.liveLeaders());
     }
 
-    public static final class PartitionState {
-        public final BasePartitionState basePartitionState;
-        public final List addingReplicas;
-        public final List removingReplicas;
-        public final boolean isNew;
-
-        public PartitionState(int controllerEpoch,
-                              int leader,
-                              int leaderEpoch,
-                              List isr,
-                              int zkVersion,
-                              List replicas,
-                              boolean isNew) {
-            this(controllerEpoch,
-                 leader,
-                 leaderEpoch,
-                 isr,
-                 zkVersion,
-                 replicas,
-                 Collections.emptyList(),
-                 Collections.emptyList(),
-                 isNew);
-        }
-
-        public PartitionState(int controllerEpoch,
-                              int leader,
-                              int leaderEpoch,
-                              List isr,
-                              int zkVersion,
-                              List replicas,
-                              List addingReplicas,
-                              List removingReplicas,
-                              boolean isNew) {
-            this.basePartitionState = new BasePartitionState(controllerEpoch, leader, leaderEpoch, isr, zkVersion, replicas);
-            this.addingReplicas = addingReplicas;
-            this.removingReplicas = removingReplicas;
-            this.isNew = isNew;
-        }
-
-        private PartitionState(Struct struct) {
-            int controllerEpoch = struct.get(CONTROLLER_EPOCH);
-            int leader = struct.get(LEADER);
-            int leaderEpoch = struct.get(LEADER_EPOCH);
-
-            Object[] isrArray = struct.get(ISR);
-            List isr = new ArrayList<>(isrArray.length);
-            for (Object r : isrArray)
-                isr.add((Integer) r);
-
-            int zkVersion = struct.get(ZK_VERSION);
-
-            Object[] replicasArray = struct.get(REPLICAS);
-            List replicas = new ArrayList<>(replicasArray.length);
-            for (Object r : replicasArray)
-                replicas.add((Integer) r);
-
-            this.basePartitionState = new BasePartitionState(controllerEpoch, leader, leaderEpoch, isr, zkVersion, replicas);
-
-            List addingReplicas = new ArrayList<>();
-            if (struct.hasField(ADDING_REPLICAS)) {
-                for (Object r : struct.get(ADDING_REPLICAS))
-                    addingReplicas.add((Integer) r);
-            }
-            this.addingReplicas = addingReplicas;
-
-            List removingReplicas = new ArrayList<>();
-            if (struct.hasField(REMOVING_REPLICAS)) {
-                for (Object r : struct.get(REMOVING_REPLICAS))
-                    removingReplicas.add((Integer) r);
-            }
-            this.removingReplicas = removingReplicas;
-
-            this.isNew = struct.getOrElse(IS_NEW, false);
-        }
-
-        @Override
-        public String toString() {
-            return "PartitionState(controllerEpoch=" + basePartitionState.controllerEpoch +
-                ", leader=" + basePartitionState.leader +
-                ", leaderEpoch=" + basePartitionState.leaderEpoch +
-                ", isr=" + Utils.join(basePartitionState.isr, ",") +
-                ", zkVersion=" + basePartitionState.zkVersion +
-                ", replicas=" + Utils.join(basePartitionState.replicas, ",") +
-                ", addingReplicas=" + Utils.join(addingReplicas, ",") +
-                ", removingReplicas=" + Utils.join(removingReplicas, ",") +
-                ", isNew=" + isNew + ")";
-        }
+    protected int size() {
+        return data.size(version());
+    }
 
-        private void setStruct(Struct struct, short version) {
-            struct.set(CONTROLLER_EPOCH, basePartitionState.controllerEpoch);
-            struct.set(LEADER, basePartitionState.leader);
-            struct.set(LEADER_EPOCH, basePartitionState.leaderEpoch);
-            struct.set(ISR, basePartitionState.isr.toArray());
-            struct.set(ZK_VERSION, basePartitionState.zkVersion);
-            struct.set(REPLICAS, basePartitionState.replicas.toArray());
-            if (version >= 3) {
-                struct.set(ADDING_REPLICAS, addingReplicas.toArray());
-                struct.set(REMOVING_REPLICAS, removingReplicas.toArray());
-            }
-            struct.setIfExists(IS_NEW, isNew);
-        }
+    public static LeaderAndIsrRequest parse(ByteBuffer buffer, short version) {
+        return new LeaderAndIsrRequest(ApiKeys.LEADER_AND_ISR.parseRequest(version, buffer), version);
     }
+
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java
index 3b80222eff0e7..0329307e66693 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java
@@ -16,45 +16,19 @@
  */
 package org.apache.kafka.common.requests;
 
-import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.message.LeaderAndIsrResponseData;
+import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
 
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-
-import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
-import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
-import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
+import java.util.stream.Collectors;
 
 public class LeaderAndIsrResponse extends AbstractResponse {
-    private static final Field.ComplexArray PARTITIONS = new Field.ComplexArray("partitions", "Response for the requests partitions");
-
-    private static final Field PARTITIONS_V0 = PARTITIONS.withFields(
-            TOPIC_NAME,
-            PARTITION_ID,
-            ERROR_CODE);
-    private static final Schema LEADER_AND_ISR_RESPONSE_V0 = new Schema(
-            ERROR_CODE,
-            PARTITIONS_V0);
-
-    // LeaderAndIsrResponse V1 may receive KAFKA_STORAGE_ERROR in the response
-    private static final Schema LEADER_AND_ISR_RESPONSE_V1 = LEADER_AND_ISR_RESPONSE_V0;
-
-    private static final Schema LEADER_AND_ISR_RESPONSE_V2 = LEADER_AND_ISR_RESPONSE_V1;
-
-    private static final Schema LEADER_AND_ISR_RESPONSE_V3 = LEADER_AND_ISR_RESPONSE_V2;
-
-    public static Schema[] schemaVersions() {
-        return new Schema[]{LEADER_AND_ISR_RESPONSE_V0, LEADER_AND_ISR_RESPONSE_V1, LEADER_AND_ISR_RESPONSE_V2, LEADER_AND_ISR_RESPONSE_V3};
-    }
 
     /**
      * Possible error code:
@@ -62,74 +36,45 @@ public static Schema[] schemaVersions() {
      * STALE_CONTROLLER_EPOCH (11)
      * STALE_BROKER_EPOCH (77)
      */
-    private final Errors error;
-
-    private final Map responses;
+    private final LeaderAndIsrResponseData data;
 
-    public LeaderAndIsrResponse(Errors error, Map responses) {
-        this.responses = responses;
-        this.error = error;
+    public LeaderAndIsrResponse(LeaderAndIsrResponseData data) {
+        this.data = data;
     }
 
-    public LeaderAndIsrResponse(Struct struct) {
-        responses = new HashMap<>();
-        for (Object responseDataObj : struct.get(PARTITIONS)) {
-            Struct responseData = (Struct) responseDataObj;
-            String topic = responseData.get(TOPIC_NAME);
-            int partition = responseData.get(PARTITION_ID);
-            Errors error = Errors.forCode(responseData.get(ERROR_CODE));
-            responses.put(new TopicPartition(topic, partition), error);
-        }
-
-        error = Errors.forCode(struct.get(ERROR_CODE));
+    public LeaderAndIsrResponse(Struct struct, short version) {
+        this.data = new LeaderAndIsrResponseData(struct, version);
     }
 
-    public Map responses() {
-        return responses;
+    public List partitions() {
+        return data.partitionErrors();
     }
 
     public Errors error() {
-        return error;
+        return Errors.forCode(data.errorCode());
     }
 
     @Override
     public Map errorCounts() {
+        Errors error = error();
         if (error != Errors.NONE)
             // Minor optimization since the top-level error applies to all partitions
-            return Collections.singletonMap(error, responses.size());
-        return errorCounts(responses);
+            return Collections.singletonMap(error, data.partitionErrors().size());
+        return errorCounts(data.partitionErrors().stream().map(l -> Errors.forCode(l.errorCode())).collect(Collectors.toList()));
     }
 
     public static LeaderAndIsrResponse parse(ByteBuffer buffer, short version) {
-        return new LeaderAndIsrResponse(ApiKeys.LEADER_AND_ISR.parseResponse(version, buffer));
+        return new LeaderAndIsrResponse(ApiKeys.LEADER_AND_ISR.parseResponse(version, buffer), version);
     }
 
     @Override
     protected Struct toStruct(short version) {
-        Struct struct = new Struct(ApiKeys.LEADER_AND_ISR.responseSchema(version));
-
-        List responseDatas = new ArrayList<>(responses.size());
-        for (Map.Entry response : responses.entrySet()) {
-            Struct partitionData = struct.instance(PARTITIONS);
-            TopicPartition partition = response.getKey();
-            partitionData.set(TOPIC_NAME, partition.topic());
-            partitionData.set(PARTITION_ID, partition.partition());
-            partitionData.set(ERROR_CODE, response.getValue().code());
-            responseDatas.add(partitionData);
-        }
-
-        struct.set(PARTITIONS, responseDatas.toArray());
-        struct.set(ERROR_CODE, error.code());
-
-        return struct;
+        return data.toStruct(version);
     }
 
     @Override
     public String toString() {
-        return "LeaderAndIsrResponse(" +
-                "responses=" + responses +
-                ", error=" + error +
-                ")";
+        return data.toString();
     }
 
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
index 9f30d3b911d0a..3adeb157c188c 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
@@ -27,6 +27,7 @@
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -94,14 +95,13 @@ public OffsetCommitResponseData data() {
 
     @Override
     public Map errorCounts() {
-        Map errorMap = new HashMap<>();
+        List errors = new ArrayList<>();
         for (OffsetCommitResponseTopic topic : data.topics()) {
             for (OffsetCommitResponsePartition partition : topic.partitions()) {
-                errorMap.put(new TopicPartition(topic.name(), partition.partitionIndex()),
-                        Errors.forCode(partition.errorCode()));
+                errors.add(Errors.forCode(partition.errorCode()));
             }
         }
-        return errorCounts(errorMap);
+        return errorCounts(errors);
     }
 
     public static OffsetCommitResponse parse(ByteBuffer buffer, short version) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
index bad9573a15758..e79228917fb32 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
@@ -17,57 +17,27 @@
 package org.apache.kafka.common.requests;
 
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.message.StopReplicaRequestData;
+import org.apache.kafka.common.message.StopReplicaRequestData.StopReplicaPartitionV0;
+import org.apache.kafka.common.message.StopReplicaRequestData.StopReplicaTopic;
+import org.apache.kafka.common.message.StopReplicaResponseData;
+import org.apache.kafka.common.message.StopReplicaResponseData.StopReplicaPartitionError;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
 import org.apache.kafka.common.utils.CollectionUtils;
+import org.apache.kafka.common.utils.FlattenedIterator;
+import org.apache.kafka.common.utils.MappedIterator;
 import org.apache.kafka.common.utils.Utils;
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-
-import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
-import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
-import static org.apache.kafka.common.protocol.types.Type.INT32;
+import java.util.stream.Collectors;
 
 public class StopReplicaRequest extends AbstractControlRequest {
-    private static final Field.Bool DELETE_PARTITIONS = new Field.Bool("delete_partitions", "Boolean which indicates if replica's partitions must be deleted.");
-    private static final Field.ComplexArray PARTITIONS = new Field.ComplexArray("partitions", "The partitions");
-    private static final Field.Array PARTITION_IDS = new Field.Array("partition_ids", INT32, "The partition ids of a topic");
-
-    private static final Field PARTITIONS_V0 = PARTITIONS.withFields(
-            TOPIC_NAME,
-            PARTITION_ID);
-    private static final Field PARTITIONS_V1 = PARTITIONS.withFields(
-            TOPIC_NAME,
-            PARTITION_IDS);
-
-    private static final Schema STOP_REPLICA_REQUEST_V0 = new Schema(
-            CONTROLLER_ID,
-            CONTROLLER_EPOCH,
-            DELETE_PARTITIONS,
-            PARTITIONS_V0);
-
-    // STOP_REPLICA_REQUEST_V1 added a broker_epoch Field. This field specifies the generation of the broker across
-    // bounces. It also normalizes partitions under each topic.
-    private static final Schema STOP_REPLICA_REQUEST_V1 = new Schema(
-            CONTROLLER_ID,
-            CONTROLLER_EPOCH,
-            BROKER_EPOCH,
-            DELETE_PARTITIONS,
-            PARTITIONS_V1);
-
-
-    public static Schema[] schemaVersions() {
-        return new Schema[] {STOP_REPLICA_REQUEST_V0, STOP_REPLICA_REQUEST_V1};
-    }
 
     public static class Builder extends AbstractControlRequest.Builder {
         private final boolean deletePartitions;
@@ -80,10 +50,31 @@ public Builder(short version, int controllerId, int controllerEpoch, long broker
             this.partitions = partitions;
         }
 
-        @Override
         public StopReplicaRequest build(short version) {
-            return new StopReplicaRequest(controllerId, controllerEpoch, brokerEpoch,
-                    deletePartitions, partitions, version);
+            StopReplicaRequestData data = new StopReplicaRequestData()
+                .setControllerId(controllerId)
+                .setControllerEpoch(controllerEpoch)
+                .setBrokerEpoch(brokerEpoch)
+                .setDeletePartitions(deletePartitions);
+
+            if (version >= 1) {
+                Map> topicPartitionsMap = CollectionUtils.groupPartitionsByTopic(partitions);
+                List topics = topicPartitionsMap.entrySet().stream().map(entry ->
+                    new StopReplicaTopic()
+                        .setName(entry.getKey())
+                        .setPartitionIndexes(entry.getValue())
+                ).collect(Collectors.toList());
+                data.setTopics(topics);
+            } else {
+                List requestPartitions = partitions.stream().map(tp ->
+                    new StopReplicaPartitionV0()
+                        .setTopicName(tp.topic())
+                        .setPartitionIndex(tp.partition())
+                ).collect(Collectors.toList());
+                data.setUngroupedPartitions(requestPartitions);
+            }
+
+            return new StopReplicaRequest(data, version);
         }
 
         @Override
@@ -100,54 +91,37 @@ public String toString() {
         }
     }
 
-    private final boolean deletePartitions;
-    private final Collection partitions;
+    private final StopReplicaRequestData data;
 
-    private StopReplicaRequest(int controllerId, int controllerEpoch, long brokerEpoch, boolean deletePartitions,
-                               Collection partitions, short version) {
-        super(ApiKeys.STOP_REPLICA, version, controllerId, controllerEpoch, brokerEpoch);
-        this.deletePartitions = deletePartitions;
-        this.partitions = partitions;
+    private StopReplicaRequest(StopReplicaRequestData data, short version) {
+        super(ApiKeys.STOP_REPLICA, version);
+        this.data = data;
     }
 
     public StopReplicaRequest(Struct struct, short version) {
-        super(ApiKeys.STOP_REPLICA, struct, version);
-
-        partitions = new HashSet<>();
-        if (version > 0) { // V1
-            for (Object topicObj : struct.get(PARTITIONS)) {
-                Struct topicData = (Struct) topicObj;
-                String topic = topicData.get(TOPIC_NAME);
-                for (Object partitionObj : topicData.get(PARTITION_IDS)) {
-                    int partition = (Integer) partitionObj;
-                    partitions.add(new TopicPartition(topic, partition));
-                }
-            }
-        } else { // V0
-            for (Object partitionDataObj : struct.get(PARTITIONS)) {
-                Struct partitionData = (Struct) partitionDataObj;
-                String topic = partitionData.get(TOPIC_NAME);
-                int partition = partitionData.get(PARTITION_ID);
-                partitions.add(new TopicPartition(topic, partition));
-            }
-        }
-        deletePartitions = struct.get(DELETE_PARTITIONS);
+        this(new StopReplicaRequestData(struct, version), version);
     }
 
     @Override
     public StopReplicaResponse getErrorResponse(int throttleTimeMs, Throwable e) {
         Errors error = Errors.forException(e);
 
-        Map responses = new HashMap<>(partitions.size());
-        for (TopicPartition partition : partitions) {
-            responses.put(partition, error);
+        StopReplicaResponseData data = new StopReplicaResponseData();
+        data.setErrorCode(error.code());
+        List partitions = new ArrayList<>();
+        for (TopicPartition tp : partitions()) {
+            partitions.add(new StopReplicaPartitionError()
+                .setTopicName(tp.topic())
+                .setPartitionIndex(tp.partition())
+                .setErrorCode(error.code()));
         }
+        data.setPartitionErrors(partitions);
 
         short versionId = version();
         switch (versionId) {
             case 0:
             case 1:
-                return new StopReplicaResponse(error, responses);
+                return new StopReplicaResponse(data);
             default:
                 throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
                         versionId, this.getClass().getSimpleName(), ApiKeys.STOP_REPLICA.latestVersion()));
@@ -155,11 +129,40 @@ public StopReplicaResponse getErrorResponse(int throttleTimeMs, Throwable e) {
     }
 
     public boolean deletePartitions() {
-        return deletePartitions;
+        return data.deletePartitions();
+    }
+
+    /**
+     * Note that this method has allocation overhead per iterated element, so callers should copy the result into
+     * another collection if they need to iterate more than once.
+     *
+     * Implementation note: we should strive to avoid allocation overhead per element, see
+     * `UpdateMetadataRequest.partitionStates()` for the preferred approach. That's not possible in this case and
+     * StopReplicaRequest should be relatively rare in comparison to other request types.
+     */
+    public Iterable partitions() {
+        if (version() >= 1) {
+            return () -> new FlattenedIterator<>(data.topics().iterator(), topic ->
+                new MappedIterator<>(topic.partitionIndexes().iterator(), partition ->
+                    new TopicPartition(topic.name(), partition)));
+        }
+        return () -> new MappedIterator<>(data.ungroupedPartitions().iterator(),
+            partition -> new TopicPartition(partition.topicName(), partition.partitionIndex()));
+    }
+
+    @Override
+    public int controllerId() {
+        return data.controllerId();
+    }
+
+    @Override
+    public int controllerEpoch() {
+        return data.controllerEpoch();
     }
 
-    public Collection partitions() {
-        return partitions;
+    @Override
+    public long brokerEpoch() {
+        return data.brokerEpoch();
     }
 
     public static StopReplicaRequest parse(ByteBuffer buffer, short version) {
@@ -168,34 +171,11 @@ public static StopReplicaRequest parse(ByteBuffer buffer, short version) {
 
     @Override
     protected Struct toStruct() {
-        Struct struct = new Struct(ApiKeys.STOP_REPLICA.requestSchema(version()));
-
-        struct.set(CONTROLLER_ID, controllerId);
-        struct.set(CONTROLLER_EPOCH, controllerEpoch);
-        struct.setIfExists(BROKER_EPOCH, brokerEpoch);
-        struct.set(DELETE_PARTITIONS, deletePartitions);
-
-        if (version() > 0) { // V1
-            Map> topicPartitionsMap = CollectionUtils.groupPartitionsByTopic(partitions);
-            List topicsData = new ArrayList<>(topicPartitionsMap.size());
-            for (Map.Entry> entry : topicPartitionsMap.entrySet()) {
-                Struct topicData = struct.instance(PARTITIONS);
-                topicData.set(TOPIC_NAME, entry.getKey());
-                topicData.set(PARTITION_IDS, entry.getValue().toArray());
-                topicsData.add(topicData);
-            }
-            struct.set(PARTITIONS, topicsData.toArray());
-
-        } else { // V0
-            List partitionDatas = new ArrayList<>(partitions.size());
-            for (TopicPartition partition : partitions) {
-                Struct partitionData = struct.instance(PARTITIONS);
-                partitionData.set(TOPIC_NAME, partition.topic());
-                partitionData.set(PARTITION_ID, partition.partition());
-                partitionDatas.add(partitionData);
-            }
-            struct.set(PARTITIONS, partitionDatas.toArray());
-        }
-        return struct;
+        return data.toStruct(version());
     }
+
+    protected long size() {
+        return data.size(version());
+    }
+
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java
index 6089bea8d4f0a..7d6c7a0742899 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java
@@ -16,43 +16,19 @@
  */
 package org.apache.kafka.common.requests;
 
-import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.message.StopReplicaResponseData;
+import org.apache.kafka.common.message.StopReplicaResponseData.StopReplicaPartitionError;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
 
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-
-import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
-import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
-import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
+import java.util.stream.Collectors;
 
 public class StopReplicaResponse extends AbstractResponse {
-    private static final Field.ComplexArray PARTITIONS = new Field.ComplexArray("partitions", "Response for the requests partitions");
-
-    private static final Field PARTITIONS_V0 = PARTITIONS.withFields(
-            TOPIC_NAME,
-            PARTITION_ID,
-            ERROR_CODE);
-    private static final Schema STOP_REPLICA_RESPONSE_V0 = new Schema(
-            ERROR_CODE,
-            PARTITIONS_V0);
-
-    private static final Schema STOP_REPLICA_RESPONSE_V1 = STOP_REPLICA_RESPONSE_V0;
-
-
-    public static Schema[] schemaVersions() {
-        return new Schema[] {STOP_REPLICA_RESPONSE_V0, STOP_REPLICA_RESPONSE_V1};
-    }
-
-    private final Map responses;
 
     /**
      * Possible error code:
@@ -60,71 +36,44 @@ public static Schema[] schemaVersions() {
      * STALE_CONTROLLER_EPOCH (11)
      * STALE_BROKER_EPOCH (77)
      */
-    private final Errors error;
+    private final StopReplicaResponseData data;
 
-    public StopReplicaResponse(Errors error, Map responses) {
-        this.responses = responses;
-        this.error = error;
+    public StopReplicaResponse(StopReplicaResponseData data) {
+        this.data = data;
     }
 
-    public StopReplicaResponse(Struct struct) {
-        responses = new HashMap<>();
-        for (Object responseDataObj : struct.get(PARTITIONS)) {
-            Struct responseData = (Struct) responseDataObj;
-            String topic = responseData.get(TOPIC_NAME);
-            int partition = responseData.get(PARTITION_ID);
-            Errors error = Errors.forCode(responseData.get(ERROR_CODE));
-            responses.put(new TopicPartition(topic, partition), error);
-        }
-
-        error = Errors.forCode(struct.get(ERROR_CODE));
+    public StopReplicaResponse(Struct struct, short version) {
+        data = new StopReplicaResponseData(struct, version);
     }
 
-    public Map responses() {
-        return responses;
+    public List partitionErrors() {
+        return data.partitionErrors();
     }
 
     public Errors error() {
-        return error;
+        return Errors.forCode(data.errorCode());
     }
 
     @Override
     public Map errorCounts() {
-        if (error != Errors.NONE)
+        if (data.errorCode() != Errors.NONE.code())
             // Minor optimization since the top-level error applies to all partitions
-            return Collections.singletonMap(error, responses.size());
-        return errorCounts(responses);
+            return Collections.singletonMap(error(), data.partitionErrors().size());
+        return errorCounts(data.partitionErrors().stream().map(p -> Errors.forCode(p.errorCode())).collect(Collectors.toList()));
     }
 
     public static StopReplicaResponse parse(ByteBuffer buffer, short version) {
-        return new StopReplicaResponse(ApiKeys.STOP_REPLICA.parseResponse(version, buffer));
+        return new StopReplicaResponse(ApiKeys.STOP_REPLICA.parseResponse(version, buffer), version);
     }
 
     @Override
     protected Struct toStruct(short version) {
-        Struct struct = new Struct(ApiKeys.STOP_REPLICA.responseSchema(version));
-
-        List responseDatas = new ArrayList<>(responses.size());
-        for (Map.Entry response : responses.entrySet()) {
-            Struct partitionData = struct.instance(PARTITIONS);
-            TopicPartition partition = response.getKey();
-            partitionData.set(TOPIC_NAME, partition.topic());
-            partitionData.set(PARTITION_ID, partition.partition());
-            partitionData.set(ERROR_CODE, response.getValue().code());
-            responseDatas.add(partitionData);
-        }
-
-        struct.set(PARTITIONS, responseDatas.toArray());
-        struct.set(ERROR_CODE, error.code());
-        return struct;
+        return data.toStruct(version);
     }
 
     @Override
     public String toString() {
-        return "StopReplicaResponse(" +
-                "responses=" + responses +
-                ", error=" + error +
-                ")";
+        return data.toString();
     }
 
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java
index 1851f89f831d8..256b9983671e0 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java
@@ -88,7 +88,7 @@ public int throttleTimeMs() {
 
     @Override
     public Map errorCounts() {
-        return errorCounts(errors());
+        return errorCounts(errors().values());
     }
 
     public Map errors() {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
index b715c2aa65b6a..b73fa5b5c1a4b 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
@@ -16,177 +16,38 @@
  */
 package org.apache.kafka.common.requests;
 
-import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.UpdateMetadataRequestData;
+import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataBroker;
+import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataEndpoint;
+import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataPartitionState;
+import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataTopicState;
+import org.apache.kafka.common.message.UpdateMetadataResponseData;
 import org.apache.kafka.common.network.ListenerName;
 import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.ByteBufferAccessor;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
 import org.apache.kafka.common.security.auth.SecurityProtocol;
-import org.apache.kafka.common.utils.CollectionUtils;
+import org.apache.kafka.common.utils.FlattenedIterator;
 import org.apache.kafka.common.utils.Utils;
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
-import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
-import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
-import static org.apache.kafka.common.protocol.types.Type.INT32;
+import static java.util.Collections.singletonList;
 
 public class UpdateMetadataRequest extends AbstractControlRequest {
-    private static final Field.ComplexArray TOPIC_STATES = new Field.ComplexArray("topic_states", "Topic states");
-    private static final Field.ComplexArray PARTITION_STATES = new Field.ComplexArray("partition_states", "Partition states");
-    private static final Field.ComplexArray LIVE_BROKERS = new Field.ComplexArray("live_brokers", "Live broekrs");
-
-    // PartitionState fields
-    private static final Field.Int32 LEADER = new Field.Int32("leader", "The broker id for the leader.");
-    private static final Field.Int32 LEADER_EPOCH = new Field.Int32("leader_epoch", "The leader epoch.");
-    private static final Field.Array ISR = new Field.Array("isr", INT32, "The in sync replica ids.");
-    private static final Field.Int32 ZK_VERSION = new Field.Int32("zk_version", "The ZK version.");
-    private static final Field.Array REPLICAS = new Field.Array("replicas", INT32, "The replica ids.");
-    private static final Field.Array OFFLINE_REPLICAS = new Field.Array("offline_replicas", INT32, "The offline replica ids");
-
-    // Live brokers fields
-    private static final Field.Int32 BROKER_ID = new Field.Int32("id", "The broker id");
-    private static final Field.ComplexArray ENDPOINTS = new Field.ComplexArray("end_points", "The endpoints");
-    private static final Field.NullableStr RACK = new Field.NullableStr("rack", "The rack");
-
-    // EndPoint fields
-    private static final Field.Str HOST = new Field.Str("host", "The hostname of the broker.");
-    private static final Field.Int32  PORT = new Field.Int32("port", "The port on which the broker accepts requests.");
-    private static final Field.Str LISTENER_NAME = new Field.Str("listener_name", "The listener name.");
-    private static final Field.Int16  SECURITY_PROTOCOL_TYPE = new Field.Int16("security_protocol_type", "The security protocol type.");
-
-    private static final Field PARTITION_STATES_V0 = PARTITION_STATES.withFields(
-            TOPIC_NAME,
-            PARTITION_ID,
-            CONTROLLER_EPOCH,
-            LEADER,
-            LEADER_EPOCH,
-            ISR,
-            ZK_VERSION,
-            REPLICAS);
-
-    // PARTITION_STATES_V4 added a per-partition offline_replicas field. This field specifies
-    // the list of replicas that are offline.
-    private static final Field PARTITION_STATES_V4 = PARTITION_STATES.withFields(
-            TOPIC_NAME,
-            PARTITION_ID,
-            CONTROLLER_EPOCH,
-            LEADER,
-            LEADER_EPOCH,
-            ISR,
-            ZK_VERSION,
-            REPLICAS,
-            OFFLINE_REPLICAS);
-
-    private static final Field PARTITION_STATES_V5 = PARTITION_STATES.withFields(
-            PARTITION_ID,
-            CONTROLLER_EPOCH,
-            LEADER,
-            LEADER_EPOCH,
-            ISR,
-            ZK_VERSION,
-            REPLICAS,
-            OFFLINE_REPLICAS);
-
-    // TOPIC_STATES_V5 normalizes TOPIC_STATES_V4 to
-    // make it more memory efficient
-    private static final Field TOPIC_STATES_V5 = TOPIC_STATES.withFields(
-            TOPIC_NAME,
-            PARTITION_STATES_V5);
-
-    // for some reason, V1 sends `port` before `host` while V0 sends `host` before `port
-    private static final Field ENDPOINTS_V1 = ENDPOINTS.withFields(
-            PORT,
-            HOST,
-            SECURITY_PROTOCOL_TYPE);
-
-    private static final Field ENDPOINTS_V3 = ENDPOINTS.withFields(
-            PORT,
-            HOST,
-            LISTENER_NAME,
-            SECURITY_PROTOCOL_TYPE);
-
-    private static final Field LIVE_BROKERS_V0 = LIVE_BROKERS.withFields(
-            BROKER_ID,
-            HOST,
-            PORT);
-
-    private static final Field LIVE_BROKERS_V1 = LIVE_BROKERS.withFields(
-            BROKER_ID,
-            ENDPOINTS_V1);
-
-    private static final Field LIVE_BROKERS_V2 = LIVE_BROKERS.withFields(
-            BROKER_ID,
-            ENDPOINTS_V1,
-            RACK);
-
-    private static final Field LIVE_BROKERS_V3 = LIVE_BROKERS.withFields(
-            BROKER_ID,
-            ENDPOINTS_V3,
-            RACK);
-
-    private static final Schema UPDATE_METADATA_REQUEST_V0 = new Schema(
-            CONTROLLER_ID,
-            CONTROLLER_EPOCH,
-            PARTITION_STATES_V0,
-            LIVE_BROKERS_V0);
-
-    private static final Schema UPDATE_METADATA_REQUEST_V1 = new Schema(
-            CONTROLLER_ID,
-            CONTROLLER_EPOCH,
-            PARTITION_STATES_V0,
-            LIVE_BROKERS_V1);
-
-    private static final Schema UPDATE_METADATA_REQUEST_V2 = new Schema(
-            CONTROLLER_ID,
-            CONTROLLER_EPOCH,
-            PARTITION_STATES_V0,
-            LIVE_BROKERS_V2);
-
-
-    private static final Schema UPDATE_METADATA_REQUEST_V3 = new Schema(
-            CONTROLLER_ID,
-            CONTROLLER_EPOCH,
-            PARTITION_STATES_V0,
-            LIVE_BROKERS_V3);
-
-    // UPDATE_METADATA_REQUEST_V4 added a per-partition offline_replicas field. This field specifies the list of replicas that are offline.
-    private static final Schema UPDATE_METADATA_REQUEST_V4 = new Schema(
-            CONTROLLER_ID,
-            CONTROLLER_EPOCH,
-            PARTITION_STATES_V4,
-            LIVE_BROKERS_V3);
-
-    // UPDATE_METADATA_REQUEST_V5 added a broker_epoch Field. This field specifies the generation of the broker across
-    // bounces. It also normalizes partitions under each topic.
-    private static final Schema UPDATE_METADATA_REQUEST_V5 = new Schema(
-            CONTROLLER_ID,
-            CONTROLLER_EPOCH,
-            BROKER_EPOCH,
-            TOPIC_STATES_V5,
-            LIVE_BROKERS_V3);
-
-    public static Schema[] schemaVersions() {
-        return new Schema[] {UPDATE_METADATA_REQUEST_V0, UPDATE_METADATA_REQUEST_V1, UPDATE_METADATA_REQUEST_V2,
-            UPDATE_METADATA_REQUEST_V3, UPDATE_METADATA_REQUEST_V4, UPDATE_METADATA_REQUEST_V5};
-    }
 
     public static class Builder extends AbstractControlRequest.Builder {
-        private final Map partitionStates;
-        private final Set liveBrokers;
+        private final List partitionStates;
+        private final List liveBrokers;
 
         public Builder(short version, int controllerId, int controllerEpoch, long brokerEpoch,
-                       Map partitionStates, Set liveBrokers) {
+                       List partitionStates, List liveBrokers) {
             super(ApiKeys.UPDATE_METADATA, version, controllerId, controllerEpoch, brokerEpoch);
             this.partitionStates = partitionStates;
             this.liveBrokers = liveBrokers;
@@ -194,14 +55,53 @@ public Builder(short version, int controllerId, int controllerEpoch, long broker
 
         @Override
         public UpdateMetadataRequest build(short version) {
-            if (version == 0) {
-                for (Broker broker : liveBrokers) {
-                    if (broker.endPoints.size() != 1 || broker.endPoints.get(0).securityProtocol != SecurityProtocol.PLAINTEXT) {
-                        throw new UnsupportedVersionException("UpdateMetadataRequest v0 only handles PLAINTEXT endpoints");
+            if (version < 3) {
+                for (UpdateMetadataBroker broker : liveBrokers) {
+                    if (version == 0) {
+                        if (broker.endpoints().size() != 1)
+                            throw new UnsupportedVersionException("UpdateMetadataRequest v0 requires a single endpoint");
+                        if (broker.endpoints().get(0).securityProtocol() != SecurityProtocol.PLAINTEXT.id)
+                            throw new UnsupportedVersionException("UpdateMetadataRequest v0 only handles PLAINTEXT endpoints");
+                        // Don't null out `endpoints` since it's ignored by the generated code if version >= 1
+                        UpdateMetadataEndpoint endpoint = broker.endpoints().get(0);
+                        broker.setV0Host(endpoint.host());
+                        broker.setV0Port(endpoint.port());
+                    } else {
+                        if (broker.endpoints().stream().anyMatch(endpoint -> !endpoint.listener().isEmpty() &&
+                                !endpoint.listener().equals(listenerNameFromSecurityProtocol(endpoint)))) {
+                            throw new UnsupportedVersionException("UpdateMetadataRequest v0-v3 does not support custom " +
+                                "listeners, request version: " + version + ", endpoints: " + broker.endpoints());
+                        }
                     }
                 }
             }
-            return new UpdateMetadataRequest(version, controllerId, controllerEpoch, brokerEpoch, partitionStates, liveBrokers);
+
+            UpdateMetadataRequestData data = new UpdateMetadataRequestData()
+                    .setControllerId(controllerId)
+                    .setControllerEpoch(controllerEpoch)
+                    .setBrokerEpoch(brokerEpoch)
+                    .setLiveBrokers(liveBrokers);
+
+            if (version >= 5) {
+                Map topicStatesMap = groupByTopic(partitionStates);
+                data.setTopicStates(new ArrayList<>(topicStatesMap.values()));
+            } else {
+                data.setUngroupedPartitionStates(partitionStates);
+            }
+
+            return new UpdateMetadataRequest(data, version);
+        }
+
+        private static Map groupByTopic(List partitionStates) {
+            Map topicStates = new HashMap<>();
+            for (UpdateMetadataPartitionState partition : partitionStates) {
+                // We don't null out the topic name in UpdateMetadataTopicState since it's ignored by the generated
+                // code if version >= 5
+                UpdateMetadataTopicState topicState = topicStates.computeIfAbsent(partition.topicName(),
+                    t -> new UpdateMetadataTopicState().setTopicName(partition.topicName()));
+                topicState.partitionStates().add(partition);
+            }
+            return topicStates;
         }
 
         @Override
@@ -218,275 +118,110 @@ public String toString() {
         }
     }
 
-    public static final class PartitionState {
-        public final BasePartitionState basePartitionState;
-        public final List offlineReplicas;
-
-        public PartitionState(int controllerEpoch,
-                              int leader,
-                              int leaderEpoch,
-                              List isr,
-                              int zkVersion,
-                              List replicas,
-                              List offlineReplicas) {
-            this.basePartitionState = new BasePartitionState(controllerEpoch, leader, leaderEpoch, isr, zkVersion, replicas);
-            this.offlineReplicas = offlineReplicas;
-        }
-
-        private PartitionState(Struct struct) {
-            int controllerEpoch = struct.get(CONTROLLER_EPOCH);
-            int leader = struct.get(LEADER);
-            int leaderEpoch = struct.get(LEADER_EPOCH);
-
-            Object[] isrArray = struct.get(ISR);
-            List isr = new ArrayList<>(isrArray.length);
-            for (Object r : isrArray)
-                isr.add((Integer) r);
-
-            int zkVersion = struct.get(ZK_VERSION);
-
-            Object[] replicasArray = struct.get(REPLICAS);
-            List replicas = new ArrayList<>(replicasArray.length);
-            for (Object r : replicasArray)
-                replicas.add((Integer) r);
+    private final UpdateMetadataRequestData data;
 
-            this.basePartitionState = new BasePartitionState(controllerEpoch, leader, leaderEpoch, isr, zkVersion, replicas);
+    UpdateMetadataRequest(UpdateMetadataRequestData data, short version) {
+        super(ApiKeys.UPDATE_METADATA, version);
+        this.data = data;
+        // Do this from the constructor to make it thread-safe (even though it's only needed when some methods are called)
+        normalize();
+    }
 
-            this.offlineReplicas = new ArrayList<>();
-            if (struct.hasField(OFFLINE_REPLICAS)) {
-                Object[] offlineReplicasArray = struct.get(OFFLINE_REPLICAS);
-                for (Object r : offlineReplicasArray)
-                    offlineReplicas.add((Integer) r);
+    private void normalize() {
+        // Version 0 only supported a single host and port and the protocol was always plaintext
+        // Version 1 added support for multiple endpoints, each with its own security protocol
+        // Version 2 added support for rack
+        // Version 3 added support for listener name, which we can infer from the security protocol for older versions
+        if (version() < 3) {
+            for (UpdateMetadataBroker liveBroker : data.liveBrokers()) {
+                // Set endpoints so that callers can rely on it always being present
+                if (version() == 0 && liveBroker.endpoints().isEmpty()) {
+                    SecurityProtocol securityProtocol = SecurityProtocol.PLAINTEXT;
+                    liveBroker.setEndpoints(singletonList(new UpdateMetadataEndpoint()
+                        .setHost(liveBroker.v0Host())
+                        .setPort(liveBroker.v0Port())
+                        .setSecurityProtocol(securityProtocol.id)
+                        .setListener(ListenerName.forSecurityProtocol(securityProtocol).value())));
+                } else {
+                    for (UpdateMetadataEndpoint endpoint : liveBroker.endpoints()) {
+                        // Set listener so that callers can rely on it always being present
+                        if (endpoint.listener().isEmpty())
+                            endpoint.setListener(listenerNameFromSecurityProtocol(endpoint));
+                    }
+                }
             }
         }
 
-        @Override
-        public String toString() {
-            return "PartitionState(controllerEpoch=" + basePartitionState.controllerEpoch +
-                ", leader=" + basePartitionState.leader +
-                ", leaderEpoch=" + basePartitionState.leaderEpoch +
-                ", isr=" + Arrays.toString(basePartitionState.isr.toArray()) +
-                ", zkVersion=" + basePartitionState.zkVersion +
-                ", replicas=" + Arrays.toString(basePartitionState.replicas.toArray()) +
-                ", offlineReplicas=" + Arrays.toString(offlineReplicas.toArray()) + ")";
-        }
-
-        private void setStruct(Struct struct) {
-            struct.set(CONTROLLER_EPOCH, basePartitionState.controllerEpoch);
-            struct.set(LEADER, basePartitionState.leader);
-            struct.set(LEADER_EPOCH, basePartitionState.leaderEpoch);
-            struct.set(ISR, basePartitionState.isr.toArray());
-            struct.set(ZK_VERSION, basePartitionState.zkVersion);
-            struct.set(REPLICAS, basePartitionState.replicas.toArray());
-            struct.setIfExists(OFFLINE_REPLICAS, offlineReplicas.toArray());
+        if (version() >= 5) {
+            for (UpdateMetadataTopicState topicState : data.topicStates()) {
+                for (UpdateMetadataPartitionState partitionState : topicState.partitionStates()) {
+                    // Set the topic name so that we can always present the ungrouped view to callers
+                    partitionState.setTopicName(topicState.topicName());
+                }
+            }
         }
     }
 
-    public static final class Broker {
-        public final int id;
-        public final List endPoints;
-        public final String rack; // introduced in V2
-
-        public Broker(int id, List endPoints, String rack) {
-            this.id = id;
-            this.endPoints = endPoints;
-            this.rack = rack;
-        }
-
-        @Override
-        public String toString() {
-            StringBuilder bld = new StringBuilder();
-            bld.append("(id=").append(id);
-            bld.append(", endPoints=").append(Utils.join(endPoints, ","));
-            bld.append(", rack=").append(rack);
-            bld.append(")");
-            return bld.toString();
-        }
+    private static String listenerNameFromSecurityProtocol(UpdateMetadataEndpoint endpoint) {
+        SecurityProtocol securityProtocol = SecurityProtocol.forId(endpoint.securityProtocol());
+        return ListenerName.forSecurityProtocol(securityProtocol).value();
     }
 
-    public static final class EndPoint {
-        public final String host;
-        public final int port;
-        public final SecurityProtocol securityProtocol;
-        public final ListenerName listenerName; // introduced in V3
-
-        public EndPoint(String host, int port, SecurityProtocol securityProtocol, ListenerName listenerName) {
-            this.host = host;
-            this.port = port;
-            this.securityProtocol = securityProtocol;
-            this.listenerName = listenerName;
-        }
-
-        @Override
-        public String toString() {
-            return "(host=" + host + ", port=" + port + ", listenerName=" + listenerName +
-                    ", securityProtocol=" + securityProtocol + ")";
-        }
+    public UpdateMetadataRequest(Struct struct, short version) {
+        this(new UpdateMetadataRequestData(struct, version), version);
     }
 
-    private final Map partitionStates;
-    private final Set liveBrokers;
-
-    private UpdateMetadataRequest(short version, int controllerId, int controllerEpoch, long brokerEpoch,
-                                  Map partitionStates, Set liveBrokers) {
-        super(ApiKeys.UPDATE_METADATA, version, controllerId, controllerEpoch, brokerEpoch);
-        this.partitionStates = partitionStates;
-        this.liveBrokers = liveBrokers;
+    @Override
+    public int controllerId() {
+        return data.controllerId();
     }
 
-    public UpdateMetadataRequest(Struct struct, short versionId) {
-        super(ApiKeys.UPDATE_METADATA, struct, versionId);
-        Map partitionStates = new HashMap<>();
-        if (struct.hasField(TOPIC_STATES)) {
-            for (Object topicStatesDataObj : struct.get(TOPIC_STATES)) {
-                Struct topicStatesData = (Struct) topicStatesDataObj;
-                String topic = topicStatesData.get(TOPIC_NAME);
-                for (Object partitionStateDataObj : topicStatesData.get(PARTITION_STATES)) {
-                    Struct partitionStateData = (Struct) partitionStateDataObj;
-                    int partition = partitionStateData.get(PARTITION_ID);
-                    PartitionState partitionState = new PartitionState(partitionStateData);
-                    partitionStates.put(new TopicPartition(topic, partition), partitionState);
-                }
-            }
-        } else {
-            for (Object partitionStateDataObj : struct.get(PARTITION_STATES)) {
-                Struct partitionStateData = (Struct) partitionStateDataObj;
-                String topic = partitionStateData.get(TOPIC_NAME);
-                int partition = partitionStateData.get(PARTITION_ID);
-                PartitionState partitionState = new PartitionState(partitionStateData);
-                partitionStates.put(new TopicPartition(topic, partition), partitionState);
-            }
-        }
-
-        Set liveBrokers = new HashSet<>();
-
-        for (Object brokerDataObj : struct.get(LIVE_BROKERS)) {
-            Struct brokerData = (Struct) brokerDataObj;
-            int brokerId = brokerData.get(BROKER_ID);
+    @Override
+    public int controllerEpoch() {
+        return data.controllerEpoch();
+    }
 
-            // V0
-            if (brokerData.hasField(HOST)) {
-                String host = brokerData.get(HOST);
-                int port = brokerData.get(PORT);
-                List endPoints = new ArrayList<>(1);
-                SecurityProtocol securityProtocol = SecurityProtocol.PLAINTEXT;
-                endPoints.add(new EndPoint(host, port, securityProtocol, ListenerName.forSecurityProtocol(securityProtocol)));
-                liveBrokers.add(new Broker(brokerId, endPoints, null));
-            } else { // V1, V2 or V3
-                List endPoints = new ArrayList<>();
-                for (Object endPointDataObj : brokerData.get(ENDPOINTS)) {
-                    Struct endPointData = (Struct) endPointDataObj;
-                    int port = endPointData.get(PORT);
-                    String host = endPointData.get(HOST);
-                    short protocolTypeId = endPointData.get(SECURITY_PROTOCOL_TYPE);
-                    SecurityProtocol securityProtocol = SecurityProtocol.forId(protocolTypeId);
-                    String listenerName;
-                    if (endPointData.hasField(LISTENER_NAME)) // V3
-                        listenerName = endPointData.get(LISTENER_NAME);
-                    else
-                        listenerName = securityProtocol.name;
-                    endPoints.add(new EndPoint(host, port, securityProtocol, new ListenerName(listenerName)));
-                }
-                String rack = null;
-                if (brokerData.hasField(RACK)) { // V2
-                    rack = brokerData.get(RACK);
-                }
-                liveBrokers.add(new Broker(brokerId, endPoints, rack));
-            }
-        }
-        this.partitionStates = partitionStates;
-        this.liveBrokers = liveBrokers;
+    @Override
+    public long brokerEpoch() {
+        return data.brokerEpoch();
     }
 
     @Override
-    protected Struct toStruct() {
+    public UpdateMetadataResponse getErrorResponse(int throttleTimeMs, Throwable e) {
         short version = version();
-        Struct struct = new Struct(ApiKeys.UPDATE_METADATA.requestSchema(version));
-        struct.set(CONTROLLER_ID, controllerId);
-        struct.set(CONTROLLER_EPOCH, controllerEpoch);
-        struct.setIfExists(BROKER_EPOCH, brokerEpoch);
-
-        if (struct.hasField(TOPIC_STATES)) {
-            Map> topicStates = CollectionUtils.groupPartitionDataByTopic(partitionStates);
-            List topicStatesData = new ArrayList<>(topicStates.size());
-            for (Map.Entry> entry : topicStates.entrySet()) {
-                Struct topicStateData = struct.instance(TOPIC_STATES);
-                topicStateData.set(TOPIC_NAME, entry.getKey());
-                Map partitionMap = entry.getValue();
-                List partitionStatesData = new ArrayList<>(partitionMap.size());
-                for (Map.Entry partitionEntry : partitionMap.entrySet()) {
-                    Struct partitionStateData = topicStateData.instance(PARTITION_STATES);
-                    partitionStateData.set(PARTITION_ID, partitionEntry.getKey());
-                    partitionEntry.getValue().setStruct(partitionStateData);
-                    partitionStatesData.add(partitionStateData);
-                }
-                topicStateData.set(PARTITION_STATES, partitionStatesData.toArray());
-                topicStatesData.add(topicStateData);
-            }
-            struct.set(TOPIC_STATES, topicStatesData.toArray());
-        } else {
-            List partitionStatesData = new ArrayList<>(partitionStates.size());
-            for (Map.Entry entry : partitionStates.entrySet()) {
-                Struct partitionStateData = struct.instance(PARTITION_STATES);
-                TopicPartition topicPartition = entry.getKey();
-                partitionStateData.set(TOPIC_NAME, topicPartition.topic());
-                partitionStateData.set(PARTITION_ID, topicPartition.partition());
-                entry.getValue().setStruct(partitionStateData);
-                partitionStatesData.add(partitionStateData);
-            }
-            struct.set(PARTITION_STATES, partitionStatesData.toArray());
-        }
-
-        List brokersData = new ArrayList<>(liveBrokers.size());
-        for (Broker broker : liveBrokers) {
-            Struct brokerData = struct.instance(LIVE_BROKERS);
-            brokerData.set(BROKER_ID, broker.id);
-
-            if (version == 0) {
-                EndPoint endPoint = broker.endPoints.get(0);
-                brokerData.set(HOST, endPoint.host);
-                brokerData.set(PORT, endPoint.port);
-            } else {
-                List endPointsData = new ArrayList<>(broker.endPoints.size());
-                for (EndPoint endPoint : broker.endPoints) {
-                    Struct endPointData = brokerData.instance(ENDPOINTS);
-                    endPointData.set(PORT, endPoint.port);
-                    endPointData.set(HOST, endPoint.host);
-                    endPointData.set(SECURITY_PROTOCOL_TYPE, endPoint.securityProtocol.id);
-                    if (version >= 3)
-                        endPointData.set(LISTENER_NAME, endPoint.listenerName.value());
-                    endPointsData.add(endPointData);
-
-                }
-                brokerData.set(ENDPOINTS, endPointsData.toArray());
-                if (version >= 2) {
-                    brokerData.set(RACK, broker.rack);
-                }
-            }
+        if (version <= 5)
+            return new UpdateMetadataResponse(new UpdateMetadataResponseData().setErrorCode(Errors.forException(e).code()));
+        else
+            throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
+                version, this.getClass().getSimpleName(), ApiKeys.UPDATE_METADATA.latestVersion()));
+    }
 
-            brokersData.add(brokerData);
+    public Iterable partitionStates() {
+        if (version() >= 5) {
+            return () -> new FlattenedIterator<>(data.topicStates().iterator(),
+                topicState -> topicState.partitionStates().iterator());
         }
-        struct.set(LIVE_BROKERS, brokersData.toArray());
+        return data.ungroupedPartitionStates();
+    }
 
-        return struct;
+    public List liveBrokers() {
+        return data.liveBrokers();
     }
 
     @Override
-    public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
-        short versionId = version();
-        if (versionId <= 5)
-            return new UpdateMetadataResponse(Errors.forException(e));
-        else
-            throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
-                versionId, this.getClass().getSimpleName(), ApiKeys.UPDATE_METADATA.latestVersion()));
+    protected Struct toStruct() {
+        return data.toStruct(version());
     }
 
-    public Map partitionStates() {
-        return partitionStates;
+    protected ByteBuffer toBytes() {
+        ByteBuffer bytes = ByteBuffer.allocate(size());
+        data.write(new ByteBufferAccessor(bytes), version());
+        bytes.flip();
+        return bytes;
     }
 
-    public Set liveBrokers() {
-        return liveBrokers;
+    protected int size() {
+        return data.size(version());
     }
 
     public static UpdateMetadataRequest parse(ByteBuffer buffer, short version) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java
index 478265b395863..c7d803f560eb1 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java
@@ -16,62 +16,41 @@
  */
 package org.apache.kafka.common.requests;
 
+import org.apache.kafka.common.message.UpdateMetadataResponseData;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
 
 import java.nio.ByteBuffer;
 import java.util.Map;
 
-import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
-
 public class UpdateMetadataResponse extends AbstractResponse {
-    private static final Schema UPDATE_METADATA_RESPONSE_V0 = new Schema(ERROR_CODE);
-    private static final Schema UPDATE_METADATA_RESPONSE_V1 = UPDATE_METADATA_RESPONSE_V0;
-    private static final Schema UPDATE_METADATA_RESPONSE_V2 = UPDATE_METADATA_RESPONSE_V1;
-    private static final Schema UPDATE_METADATA_RESPONSE_V3 = UPDATE_METADATA_RESPONSE_V2;
-    private static final Schema UPDATE_METADATA_RESPONSE_V4 = UPDATE_METADATA_RESPONSE_V3;
-    private static final Schema UPDATE_METADATA_RESPONSE_V5 = UPDATE_METADATA_RESPONSE_V4;
-
-    public static Schema[] schemaVersions() {
-        return new Schema[]{UPDATE_METADATA_RESPONSE_V0, UPDATE_METADATA_RESPONSE_V1, UPDATE_METADATA_RESPONSE_V2,
-            UPDATE_METADATA_RESPONSE_V3, UPDATE_METADATA_RESPONSE_V4, UPDATE_METADATA_RESPONSE_V5};
-    }
 
-    /**
-     * Possible error code:
-     *
-     * STALE_CONTROLLER_EPOCH (11)
-     * STALE_BROKER_EPOCH (77)
-     */
-    private final Errors error;
+    private final UpdateMetadataResponseData data;
 
-    public UpdateMetadataResponse(Errors error) {
-        this.error = error;
+    public UpdateMetadataResponse(UpdateMetadataResponseData data) {
+        this.data = data;
     }
 
-    public UpdateMetadataResponse(Struct struct) {
-        error = Errors.forCode(struct.get(ERROR_CODE));
+    public UpdateMetadataResponse(Struct struct, short version) {
+        this(new UpdateMetadataResponseData(struct, version));
     }
 
     public Errors error() {
-        return error;
+        return Errors.forCode(data.errorCode());
     }
 
     @Override
     public Map errorCounts() {
-        return errorCounts(error);
+        return errorCounts(error());
     }
 
     public static UpdateMetadataResponse parse(ByteBuffer buffer, short version) {
-        return new UpdateMetadataResponse(ApiKeys.UPDATE_METADATA.parseResponse(version, buffer));
+        return new UpdateMetadataResponse(ApiKeys.UPDATE_METADATA.parseResponse(version, buffer), version);
     }
 
     @Override
     protected Struct toStruct(short version) {
-        Struct struct = new Struct(ApiKeys.UPDATE_METADATA.responseSchema(version));
-        struct.set(ERROR_CODE, error.code());
-        return struct;
+        return data.toStruct(version);
     }
-}
\ No newline at end of file
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/utils/FlattenedIterator.java b/clients/src/main/java/org/apache/kafka/common/utils/FlattenedIterator.java
new file mode 100644
index 0000000000000..48bf3b7199e14
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/utils/FlattenedIterator.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.utils;
+
+import java.util.Iterator;
+import java.util.function.Function;
+
+/**
+ * Provides a flattened iterator over the inner elements of an outer iterator.
+ */
+public final class FlattenedIterator extends AbstractIterator {
+    private final Iterator outerIterator;
+    private final Function> innerIteratorFunction;
+    private Iterator innerIterator;
+
+    public FlattenedIterator(Iterator outerIterator, Function> innerIteratorFunction) {
+        this.outerIterator = outerIterator;
+        this.innerIteratorFunction = innerIteratorFunction;
+    }
+
+    @Override
+    public I makeNext() {
+        while (innerIterator == null || !innerIterator.hasNext()) {
+            if (outerIterator.hasNext())
+                innerIterator = innerIteratorFunction.apply(outerIterator.next());
+            else
+                return allDone();
+        }
+        return innerIterator.next();
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/utils/MappedIterator.java b/clients/src/main/java/org/apache/kafka/common/utils/MappedIterator.java
new file mode 100644
index 0000000000000..f6eb270c56a2d
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/utils/MappedIterator.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.utils;
+
+import java.util.Iterator;
+import java.util.function.Function;
+
+/**
+ * An iterator that maps another iterator's elements from type `F` to type `T`.
+ */
+public final class MappedIterator implements Iterator {
+    private final Iterator underlyingIterator;
+    private final Function mapper;
+
+    public MappedIterator(Iterator underlyingIterator, Function mapper) {
+        this.underlyingIterator = underlyingIterator;
+        this.mapper = mapper;
+    }
+
+    @Override
+    public final boolean hasNext() {
+        return underlyingIterator.hasNext();
+    }
+
+    @Override
+    public final T next() {
+        return mapper.apply(underlyingIterator.next());
+    }
+
+}
diff --git a/clients/src/main/resources/common/message/LeaderAndIsrRequest.json b/clients/src/main/resources/common/message/LeaderAndIsrRequest.json
index c43d2f4b1633b..12f159a157401 100644
--- a/clients/src/main/resources/common/message/LeaderAndIsrRequest.json
+++ b/clients/src/main/resources/common/message/LeaderAndIsrRequest.json
@@ -30,16 +30,16 @@
       "about": "The current controller epoch." },
     { "name": "BrokerEpoch", "type": "int64", "versions": "2+", "ignorable": true, "default": "-1",
       "about": "The current broker epoch." },
-    { "name": "PartitionStatesV0", "type": "[]LeaderAndIsrRequestPartition", "versions": "0-1",
+    { "name": "UngroupedPartitionStates", "type": "[]LeaderAndIsrPartitionState", "versions": "0-1",
       "about": "The state of each partition, in a v0 or v1 message." },
     // In v0 or v1 requests, each partition is listed alongside its topic name.
     // In v2+ requests, partitions are organized by topic, so that each topic name
     // only needs to be listed once.
-    { "name": "TopicStates", "type": "[]LeaderAndIsrRequestTopicState", "versions": "2+",
+    { "name": "TopicStates", "type": "[]LeaderAndIsrTopicState", "versions": "2+",
       "about": "Each topic.", "fields": [
-      { "name": "Name", "type": "string", "versions": "2+", "entityType": "topicName",
+      { "name": "TopicName", "type": "string", "versions": "2+", "entityType": "topicName",
         "about": "The topic name." },
-      { "name": "PartitionStatesV0", "type": "[]LeaderAndIsrRequestPartition", "versions": "2+",
+      { "name": "PartitionStates", "type": "[]LeaderAndIsrPartitionState", "versions": "2+",
         "about": "The state of each partition" }
     ]},
     { "name": "LiveLeaders", "type": "[]LeaderAndIsrLiveLeader", "versions": "0+",
@@ -53,18 +53,18 @@
     ]}
   ],
   "commonStructs": [
-    { "name": "LeaderAndIsrRequestPartition", "versions": "0+", "fields": [
-      { "name": "TopicName", "type": "string", "versions": "0-1", "entityType": "topicName",
+    { "name": "LeaderAndIsrPartitionState", "versions": "0+", "fields": [
+      { "name": "TopicName", "type": "string", "versions": "0-1", "entityType": "topicName", "ignorable": true,
         "about": "The topic name.  This is only present in v0 or v1." },
       { "name": "PartitionIndex", "type": "int32", "versions": "0+",
         "about": "The partition index." },
       { "name": "ControllerEpoch", "type": "int32", "versions": "0+",
         "about": "The controller epoch." },
-      { "name": "LeaderKey", "type": "int32", "versions": "0+", "entityType": "brokerId",
+      { "name": "Leader", "type": "int32", "versions": "0+", "entityType": "brokerId",
         "about": "The broker ID of the leader." },
       { "name": "LeaderEpoch", "type": "int32", "versions": "0+",
         "about": "The leader epoch." },
-      { "name": "IsrReplicas", "type": "[]int32", "versions": "0+",
+      { "name": "Isr", "type": "[]int32", "versions": "0+",
         "about": "The in-sync replica IDs." },
       { "name": "ZkVersion", "type": "int32", "versions": "0+",
         "about": "The ZooKeeper version." },
diff --git a/clients/src/main/resources/common/message/LeaderAndIsrResponse.json b/clients/src/main/resources/common/message/LeaderAndIsrResponse.json
index 06bb088e179ba..f30055b97aac2 100644
--- a/clients/src/main/resources/common/message/LeaderAndIsrResponse.json
+++ b/clients/src/main/resources/common/message/LeaderAndIsrResponse.json
@@ -26,7 +26,7 @@
   "fields": [
     { "name": "ErrorCode", "type": "int16", "versions": "0+",
       "about": "The error code, or 0 if there was no error." },
-    { "name": "Partitions", "type": "[]LeaderAndIsrResponsePartition", "versions": "0+",
+    { "name": "PartitionErrors", "type": "[]LeaderAndIsrPartitionError", "versions": "0+",
       "about": "Each partition.", "fields": [
       { "name": "TopicName", "type": "string", "versions": "0+", "entityType": "topicName",
         "about": "The topic name." },
diff --git a/clients/src/main/resources/common/message/StopReplicaRequest.json b/clients/src/main/resources/common/message/StopReplicaRequest.json
index 12c3d7e8feb63..248faa58a1f86 100644
--- a/clients/src/main/resources/common/message/StopReplicaRequest.json
+++ b/clients/src/main/resources/common/message/StopReplicaRequest.json
@@ -29,14 +29,14 @@
       "about": "The broker epoch." },
     { "name": "DeletePartitions", "type": "bool", "versions": "0+",
       "about": "Whether these partitions should be deleted." },
-    { "name": "PartitionsV0", "type": "[]StopReplicaRequestPartitionV0", "versions": "0",
+    { "name": "UngroupedPartitions", "type": "[]StopReplicaPartitionV0", "versions": "0",
       "about": "The partitions to stop.", "fields": [
       { "name": "TopicName", "type": "string", "versions": "0", "entityType": "topicName",
         "about": "The topic name." },
       { "name": "PartitionIndex", "type": "int32", "versions": "0",
         "about": "The partition index." }
     ]},
-    { "name": "Topics", "type": "[]StopReplicaRequestTopic", "versions": "1+",
+    { "name": "Topics", "type": "[]StopReplicaTopic", "versions": "1+",
       "about": "The topics to stop.", "fields": [
       { "name": "Name", "type": "string", "versions": "1+", "entityType": "topicName",
         "about": "The topic name." },
diff --git a/clients/src/main/resources/common/message/StopReplicaResponse.json b/clients/src/main/resources/common/message/StopReplicaResponse.json
index 962cacbedd8b9..759bffecf07c3 100644
--- a/clients/src/main/resources/common/message/StopReplicaResponse.json
+++ b/clients/src/main/resources/common/message/StopReplicaResponse.json
@@ -22,7 +22,7 @@
   "fields": [
     { "name": "ErrorCode", "type": "int16", "versions": "0+",
       "about": "The top-level error code, or 0 if there was no top-level error." },
-    { "name": "Partitions", "type": "[]StopReplicaResponsePartition", "versions": "0+",
+    { "name": "PartitionErrors", "type": "[]StopReplicaPartitionError", "versions": "0+",
       "about": "The responses for each partition.", "fields": [
       { "name": "TopicName", "type": "string", "versions": "0+", "entityType": "topicName",
         "about": "The topic name." },
diff --git a/clients/src/main/resources/common/message/UpdateMetadataRequest.json b/clients/src/main/resources/common/message/UpdateMetadataRequest.json
index d0873d1fd097b..b1ac987b46c64 100644
--- a/clients/src/main/resources/common/message/UpdateMetadataRequest.json
+++ b/clients/src/main/resources/common/message/UpdateMetadataRequest.json
@@ -34,16 +34,16 @@
       "about": "The controller epoch." },
     { "name": "BrokerEpoch", "type": "int64", "versions": "5+", "ignorable": true, "default": "-1",
       "about": "The broker epoch." },
-    { "name": "LegacyPartitionStates", "type": "[]UpdateMetadataPartitionState", "versions": "0-4",
+    { "name": "UngroupedPartitionStates", "type": "[]UpdateMetadataPartitionState", "versions": "0-4",
       "about": "In older versions of this RPC, each partition that we would like to update." },
-    { "name": "TopicStates", "type": "[]UpdateMetadataRequestTopicState", "versions": "5+",
+    { "name": "TopicStates", "type": "[]UpdateMetadataTopicState", "versions": "5+",
       "about": "In newer versions of this RPC, each topic that we would like to update.", "fields": [
       { "name": "TopicName", "type": "string", "versions": "5+", "entityType": "topicName",
         "about": "The topic name." },
       { "name": "PartitionStates", "type": "[]UpdateMetadataPartitionState", "versions": "5+",
         "about": "The partition that we would like to update." }
     ]},
-    { "name": "Brokers", "type": "[]UpdateMetadataRequestBroker", "versions": "0+", "fields": [
+    { "name": "LiveBrokers", "type": "[]UpdateMetadataBroker", "versions": "0+", "fields": [
         { "name": "Id", "type": "int32", "versions": "0+", "entityType": "brokerId",
           "about": "The broker id." },
         // Version 0 of the protocol only allowed specifying a single host and
@@ -52,13 +52,13 @@
           "about": "The broker hostname." },
         { "name": "V0Port", "type": "int32", "versions": "0", "ignorable": true,
           "about": "The broker port." },
-        { "name": "Endpoints", "type": "[]UpdateMetadataRequestEndpoint", "versions": "1+",
+        { "name": "Endpoints", "type": "[]UpdateMetadataEndpoint", "versions": "1+", "ignorable": true,
           "about": "The broker endpoints.", "fields": [
           { "name": "Port", "type": "int32", "versions": "1+",
             "about": "The port of this endpoint" },
           { "name": "Host", "type": "string", "versions": "1+",
             "about": "The hostname of this endpoint" },
-          { "name": "Listener", "type": "string", "versions": "3+",
+          { "name": "Listener", "type": "string", "versions": "3+", "ignorable": true,
             "about": "The listener name." },
           { "name": "SecurityProtocol", "type": "int16", "versions": "1+",
             "about": "The security protocol type." }
@@ -69,7 +69,7 @@
   ],
   "commonStructs": [
     { "name": "UpdateMetadataPartitionState", "versions": "0+", "fields": [
-      { "name": "TopicName", "type": "string", "versions": "0-4", "entityType": "topicName",
+      { "name": "TopicName", "type": "string", "versions": "0-4", "entityType": "topicName", "ignorable": true,
         "about": "In older versions of this RPC, the topic name." },
       { "name": "PartitionIndex", "type": "int32", "versions": "0+",
         "about": "The partition index." },
@@ -85,7 +85,7 @@
         "about": "The Zookeeper version." },
       { "name": "Replicas", "type": "[]int32", "versions": "0+", "entityType": "brokerId",
         "about": "All the replicas of this partition." },
-      { "name": "OfflineReplicas", "type": "[]int32", "versions": "4+", "entityType": "brokerId",
+      { "name": "OfflineReplicas", "type": "[]int32", "versions": "4+", "entityType": "brokerId", "ignorable": true,
         "about": "The replicas of this partition which are offline." }
     ]}
   ]
diff --git a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
index ca8d63db8f28b..1af694cd55eaf 100644
--- a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
@@ -248,33 +248,29 @@ public void testOffsetForLeaderEpochVersions() throws Exception {
     @Test
     public void testLeaderAndIsrVersions() throws Exception {
         // Version 3 adds two new fields - AddingReplicas and RemovingReplicas
-        LeaderAndIsrRequestData.LeaderAndIsrRequestTopicState partitionStateNoAddingRemovingReplicas =
-                new LeaderAndIsrRequestData.LeaderAndIsrRequestTopicState()
-                        .setName("topic")
-                        .setPartitionStatesV0(
-                                Collections.singletonList(
-                                        new LeaderAndIsrRequestData.LeaderAndIsrRequestPartition()
-                                                .setPartitionIndex(0)
-                                                .setReplicas(Collections.singletonList(0))
-                                )
-                        );
-        LeaderAndIsrRequestData.LeaderAndIsrRequestTopicState partitionStateWithAddingRemovingReplicas =
-                new LeaderAndIsrRequestData.LeaderAndIsrRequestTopicState()
-                        .setName("topic")
-                        .setPartitionStatesV0(
-                                Collections.singletonList(
-                                        new LeaderAndIsrRequestData.LeaderAndIsrRequestPartition()
-                                                .setPartitionIndex(0)
-                                                .setReplicas(Collections.singletonList(0))
-                                                .setAddingReplicas(Collections.singletonList(1))
-                                                .setRemovingReplicas(Collections.singletonList(1))
-                                )
-                        );
+        LeaderAndIsrRequestData.LeaderAndIsrTopicState partitionStateNoAddingRemovingReplicas =
+            new LeaderAndIsrRequestData.LeaderAndIsrTopicState()
+                .setTopicName("topic")
+                .setPartitionStates(Collections.singletonList(
+                    new LeaderAndIsrRequestData.LeaderAndIsrPartitionState()
+                        .setPartitionIndex(0)
+                        .setReplicas(Collections.singletonList(0))
+                ));
+        LeaderAndIsrRequestData.LeaderAndIsrTopicState partitionStateWithAddingRemovingReplicas =
+            new LeaderAndIsrRequestData.LeaderAndIsrTopicState()
+                .setTopicName("topic")
+                .setPartitionStates(Collections.singletonList(
+                    new LeaderAndIsrRequestData.LeaderAndIsrPartitionState()
+                        .setPartitionIndex(0)
+                        .setReplicas(Collections.singletonList(0))
+                        .setAddingReplicas(Collections.singletonList(1))
+                        .setRemovingReplicas(Collections.singletonList(1))
+                ));
         testAllMessageRoundTripsBetweenVersions(
-                (short) 2,
-                (short) 3,
-                new LeaderAndIsrRequestData().setTopicStates(Collections.singletonList(partitionStateWithAddingRemovingReplicas)),
-                new LeaderAndIsrRequestData().setTopicStates(Collections.singletonList(partitionStateNoAddingRemovingReplicas)));
+            (short) 2,
+            (short) 3,
+            new LeaderAndIsrRequestData().setTopicStates(Collections.singletonList(partitionStateWithAddingRemovingReplicas)),
+            new LeaderAndIsrRequestData().setTopicStates(Collections.singletonList(partitionStateNoAddingRemovingReplicas)));
         testAllMessageRoundTripsFromVersion((short) 3, new LeaderAndIsrRequestData().setTopicStates(Collections.singletonList(partitionStateWithAddingRemovingReplicas)));
     }
 
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ControlRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ControlRequestTest.java
deleted file mode 100644
index 2f4689ed0a09c..0000000000000
--- a/clients/src/test/java/org/apache/kafka/common/requests/ControlRequestTest.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.common.requests;
-
-import org.apache.kafka.common.TopicPartition;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.Collections;
-import java.util.Set;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Random;
-
-
-public class ControlRequestTest {
-    @Test
-    public void testLeaderAndIsrRequestNormalization() {
-        Set tps = generateRandomTopicPartitions(10, 10);
-        Map partitionStates = new HashMap<>();
-        for (TopicPartition tp: tps) {
-            partitionStates.put(tp, new LeaderAndIsrRequest.PartitionState(0, 0, 0,
-                    Collections.emptyList(), 0, Collections.emptyList(), false));
-        }
-        LeaderAndIsrRequest.Builder builder = new LeaderAndIsrRequest.Builder((short) 2, 0, 0, 0,
-                partitionStates, Collections.emptySet());
-
-        Assert.assertTrue(builder.build((short) 2).size() <  builder.build((short) 1).size());
-    }
-
-    @Test
-    public void testUpdateMetadataRequestNormalization() {
-        Set tps = generateRandomTopicPartitions(10, 10);
-        Map partitionStates = new HashMap<>();
-        for (TopicPartition tp: tps) {
-            partitionStates.put(tp, new UpdateMetadataRequest.PartitionState(0, 0, 0,
-                    Collections.emptyList(), 0, Collections.emptyList(), Collections.emptyList()));
-        }
-        UpdateMetadataRequest.Builder builder = new UpdateMetadataRequest.Builder((short) 5, 0, 0, 0,
-                partitionStates, Collections.emptySet());
-
-        Assert.assertTrue(builder.build((short) 5).size() <  builder.build((short) 4).size());
-    }
-
-    @Test
-    public void testStopReplicaRequestNormalization() {
-        Set tps = generateRandomTopicPartitions(10, 10);
-        Map partitionStates = new HashMap<>();
-        for (TopicPartition tp: tps) {
-            partitionStates.put(tp, new UpdateMetadataRequest.PartitionState(0, 0, 0,
-                    Collections.emptyList(), 0, Collections.emptyList(), Collections.emptyList()));
-        }
-        StopReplicaRequest.Builder builder = new StopReplicaRequest.Builder((short) 5, 0, 0, 0, false, tps);
-
-        Assert.assertTrue(builder.build((short) 1).size() <  builder.build((short) 0).size());
-    }
-
-    private Set generateRandomTopicPartitions(int numTopic, int numPartitionPerTopic) {
-        Set tps = new HashSet<>();
-        Random r = new Random();
-        for (int i = 0; i < numTopic; i++) {
-            byte[] array = new byte[32];
-            r.nextBytes(array);
-            String topic = new String(array);
-            for (int j = 0; j < numPartitionPerTopic; j++) {
-                tps.add(new TopicPartition(topic, j));
-            }
-        }
-        return tps;
-    }
-
-}
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java
new file mode 100644
index 0000000000000..ffcc9cc09a8aa
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.requests;
+
+import org.apache.kafka.common.Node;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.message.LeaderAndIsrRequestData;
+import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrLiveLeader;
+import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState;
+import org.apache.kafka.common.protocol.ByteBufferAccessor;
+import org.apache.kafka.test.TestUtils;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.emptyList;
+import static org.apache.kafka.common.protocol.ApiKeys.LEADER_AND_ISR;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class LeaderAndIsrRequestTest {
+
+    /**
+     * Verifies the logic we have in LeaderAndIsrRequest to present a unified interface across the various versions
+     * works correctly. For example, `LeaderAndIsrPartitionState.topicName` is not serialiazed/deserialized in
+     * recent versions, but we set it manually so that we can always present the ungrouped partition states
+     * independently of the version.
+     */
+    @Test
+    public void testVersionLogic() {
+        for (short version = LEADER_AND_ISR.oldestVersion(); version <= LEADER_AND_ISR.latestVersion(); version++) {
+            List partitionStates = asList(
+                new LeaderAndIsrPartitionState()
+                    .setTopicName("topic0")
+                    .setPartitionIndex(0)
+                    .setControllerEpoch(2)
+                    .setLeader(0)
+                    .setLeaderEpoch(10)
+                    .setIsr(asList(0, 1))
+                    .setZkVersion(10)
+                    .setReplicas(asList(0, 1, 2))
+                    .setAddingReplicas(asList(3))
+                    .setRemovingReplicas(asList(2)),
+                new LeaderAndIsrPartitionState()
+                    .setTopicName("topic0")
+                    .setPartitionIndex(1)
+                    .setControllerEpoch(2)
+                    .setLeader(1)
+                    .setLeaderEpoch(11)
+                    .setIsr(asList(1, 2, 3))
+                    .setZkVersion(11)
+                    .setReplicas(asList(1, 2, 3))
+                    .setAddingReplicas(emptyList())
+                    .setRemovingReplicas(emptyList()),
+                new LeaderAndIsrPartitionState()
+                    .setTopicName("topic1")
+                    .setPartitionIndex(0)
+                    .setControllerEpoch(2)
+                    .setLeader(2)
+                    .setLeaderEpoch(11)
+                    .setIsr(asList(2, 3, 4))
+                    .setZkVersion(11)
+                    .setReplicas(asList(2, 3, 4))
+                    .setAddingReplicas(emptyList())
+                    .setRemovingReplicas(emptyList())
+            );
+
+            List liveNodes = asList(
+                new Node(0, "host0", 9090),
+                new Node(1, "host1", 9091)
+            );
+            LeaderAndIsrRequest request = new LeaderAndIsrRequest.Builder(version, 1, 2, 3, partitionStates,
+                liveNodes).build();
+
+            List liveLeaders = liveNodes.stream().map(n -> new LeaderAndIsrLiveLeader()
+                .setBrokerId(n.id())
+                .setHostName(n.host())
+                .setPort(n.port())).collect(Collectors.toList());
+            assertEquals(new HashSet<>(partitionStates), iterableToSet(request.partitionStates()));
+            assertEquals(liveLeaders, request.liveLeaders());
+            assertEquals(1, request.controllerId());
+            assertEquals(2, request.controllerEpoch());
+            assertEquals(3, request.brokerEpoch());
+
+            ByteBuffer byteBuffer = request.toBytes();
+            LeaderAndIsrRequest deserializedRequest = new LeaderAndIsrRequest(new LeaderAndIsrRequestData(
+                new ByteBufferAccessor(byteBuffer), version), version);
+
+            // Adding/removing replicas is only supported from version 3, so the deserialized request won't have
+            // them for earlier versions.
+            if (version < 3) {
+                partitionStates.get(0)
+                    .setAddingReplicas(emptyList())
+                    .setRemovingReplicas(emptyList());
+            }
+
+            assertEquals(new HashSet<>(partitionStates), iterableToSet(deserializedRequest.partitionStates()));
+            assertEquals(liveLeaders, deserializedRequest.liveLeaders());
+            assertEquals(1, request.controllerId());
+            assertEquals(2, request.controllerEpoch());
+            assertEquals(3, request.brokerEpoch());
+        }
+    }
+
+    @Test
+    public void testTopicPartitionGroupingSizeReduction() {
+        Set tps = TestUtils.generateRandomTopicPartitions(10, 10);
+        List partitionStates = new ArrayList<>();
+        for (TopicPartition tp : tps) {
+            partitionStates.add(new LeaderAndIsrPartitionState()
+                .setTopicName(tp.topic())
+                .setPartitionIndex(tp.partition()));
+        }
+        LeaderAndIsrRequest.Builder builder = new LeaderAndIsrRequest.Builder((short) 2, 0, 0, 0,
+            partitionStates, Collections.emptySet());
+
+        LeaderAndIsrRequest v2 = builder.build((short) 2);
+        LeaderAndIsrRequest v1 = builder.build((short) 1);
+        assertTrue("Expected v2 < v1: v2=" + v2.size() + ", v1=" + v1.size(), v2.size() < v1.size());
+    }
+
+    private  Set iterableToSet(Iterable iterable) {
+        return StreamSupport.stream(iterable.spliterator(), false).collect(Collectors.toSet());
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java
index 74e097191c118..edb962b2c5805 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java
@@ -16,16 +16,19 @@
  */
 package org.apache.kafka.common.requests;
 
-import org.apache.kafka.common.Node;
-import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState;
+import org.apache.kafka.common.message.LeaderAndIsrResponseData;
+import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
 import org.junit.Test;
 
+import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
+import static java.util.Arrays.asList;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -33,32 +36,50 @@ public class LeaderAndIsrResponseTest {
 
     @Test
     public void testErrorCountsFromGetErrorResponse() {
-        HashMap partitionStates = new HashMap<>();
-        partitionStates.put(new TopicPartition("foo", 0), new LeaderAndIsrRequest.PartitionState(15, 1, 10,
-                Collections.singletonList(10), 20, Collections.singletonList(10), false));
-        partitionStates.put(new TopicPartition("foo", 1), new LeaderAndIsrRequest.PartitionState(15, 1, 10,
-                Collections.singletonList(10), 20, Collections.singletonList(10), false));
+        List partitionStates = new ArrayList<>();
+        partitionStates.add(new LeaderAndIsrPartitionState()
+            .setTopicName("foo")
+            .setPartitionIndex(0)
+            .setControllerEpoch(15)
+            .setLeader(1)
+            .setLeaderEpoch(10)
+            .setIsr(Collections.singletonList(10))
+            .setZkVersion(20)
+            .setReplicas(Collections.singletonList(10))
+            .setIsNew(false));
+        partitionStates.add(new LeaderAndIsrPartitionState()
+            .setTopicName("foo")
+            .setPartitionIndex(1)
+            .setControllerEpoch(15)
+            .setLeader(1)
+            .setLeaderEpoch(10)
+            .setIsr(Collections.singletonList(10))
+            .setZkVersion(20)
+            .setReplicas(Collections.singletonList(10))
+            .setIsNew(false));
         LeaderAndIsrRequest request = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion(),
-                15, 20, 0, partitionStates, Collections.emptySet()).build();
+                15, 20, 0, partitionStates, Collections.emptySet()).build();
         LeaderAndIsrResponse response = request.getErrorResponse(0, Errors.CLUSTER_AUTHORIZATION_FAILED.exception());
         assertEquals(Collections.singletonMap(Errors.CLUSTER_AUTHORIZATION_FAILED, 2), response.errorCounts());
     }
 
     @Test
     public void testErrorCountsWithTopLevelError() {
-        Map errors = new HashMap<>();
-        errors.put(new TopicPartition("foo", 0), Errors.NONE);
-        errors.put(new TopicPartition("foo", 1), Errors.NOT_LEADER_FOR_PARTITION);
-        LeaderAndIsrResponse response = new LeaderAndIsrResponse(Errors.UNKNOWN_SERVER_ERROR, errors);
+        List partitions = createPartitions("foo",
+            asList(Errors.NONE, Errors.NOT_LEADER_FOR_PARTITION));
+        LeaderAndIsrResponse response = new LeaderAndIsrResponse(new LeaderAndIsrResponseData()
+            .setErrorCode(Errors.UNKNOWN_SERVER_ERROR.code())
+            .setPartitionErrors(partitions));
         assertEquals(Collections.singletonMap(Errors.UNKNOWN_SERVER_ERROR, 2), response.errorCounts());
     }
 
     @Test
     public void testErrorCountsNoTopLevelError() {
-        Map errors = new HashMap<>();
-        errors.put(new TopicPartition("foo", 0), Errors.NONE);
-        errors.put(new TopicPartition("foo", 1), Errors.CLUSTER_AUTHORIZATION_FAILED);
-        LeaderAndIsrResponse response = new LeaderAndIsrResponse(Errors.NONE, errors);
+        List partitions = createPartitions("foo",
+            asList(Errors.NONE, Errors.CLUSTER_AUTHORIZATION_FAILED));
+        LeaderAndIsrResponse response = new LeaderAndIsrResponse(new LeaderAndIsrResponseData()
+            .setErrorCode(Errors.NONE.code())
+            .setPartitionErrors(partitions));
         Map errorCounts = response.errorCounts();
         assertEquals(2, errorCounts.size());
         assertEquals(1, errorCounts.get(Errors.NONE).intValue());
@@ -67,14 +88,27 @@ public void testErrorCountsNoTopLevelError() {
 
     @Test
     public void testToString() {
-        Map errors = new HashMap<>();
-        errors.put(new TopicPartition("foo", 0), Errors.NONE);
-        errors.put(new TopicPartition("foo", 1), Errors.CLUSTER_AUTHORIZATION_FAILED);
-        LeaderAndIsrResponse response = new LeaderAndIsrResponse(Errors.NONE, errors);
+        List partitions = createPartitions("foo",
+            asList(Errors.NONE, Errors.CLUSTER_AUTHORIZATION_FAILED));
+        LeaderAndIsrResponse response = new LeaderAndIsrResponse(new LeaderAndIsrResponseData()
+            .setErrorCode(Errors.NONE.code())
+            .setPartitionErrors(partitions));
         String responseStr = response.toString();
         assertTrue(responseStr.contains(LeaderAndIsrResponse.class.getSimpleName()));
-        assertTrue(responseStr.contains(errors.toString()));
-        assertTrue(responseStr.contains(Errors.NONE.name()));
+        assertTrue(responseStr.contains(partitions.toString()));
+        assertTrue(responseStr.contains("errorCode=" + Errors.NONE.code()));
+    }
+
+    private List createPartitions(String topicName, List errors) {
+        List partitions = new ArrayList<>();
+        int partitionIndex = 0;
+        for (Errors error : errors) {
+            partitions.add(new LeaderAndIsrPartitionError()
+                .setTopicName(topicName)
+                .setPartitionIndex(partitionIndex++)
+                .setErrorCode(error.code()));
+        }
+        return partitions;
     }
 
 }
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index 52d918a991f90..6f9f7466ecdfe 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -76,6 +76,8 @@
 import org.apache.kafka.common.message.InitProducerIdResponseData;
 import org.apache.kafka.common.message.JoinGroupRequestData;
 import org.apache.kafka.common.message.JoinGroupResponseData;
+import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState;
+import org.apache.kafka.common.message.LeaderAndIsrResponseData;
 import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity;
 import org.apache.kafka.common.message.LeaveGroupResponseData;
 import org.apache.kafka.common.message.ListGroupsRequestData;
@@ -97,7 +99,12 @@
 import org.apache.kafka.common.message.SaslAuthenticateResponseData;
 import org.apache.kafka.common.message.SaslHandshakeRequestData;
 import org.apache.kafka.common.message.SaslHandshakeResponseData;
+import org.apache.kafka.common.message.StopReplicaResponseData;
 import org.apache.kafka.common.message.TxnOffsetCommitRequestData;
+import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataBroker;
+import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataEndpoint;
+import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataPartitionState;
+import org.apache.kafka.common.message.UpdateMetadataResponseData;
 import org.apache.kafka.common.network.ListenerName;
 import org.apache.kafka.common.network.Send;
 import org.apache.kafka.common.protocol.ApiKeys;
@@ -237,6 +244,8 @@ public void testSerialization() throws Exception {
         checkErrorResponse(createLeaderAndIsrRequest(0), new UnknownServerException(), false);
         checkRequest(createLeaderAndIsrRequest(1), true);
         checkErrorResponse(createLeaderAndIsrRequest(1), new UnknownServerException(), false);
+        checkRequest(createLeaderAndIsrRequest(2), true);
+        checkErrorResponse(createLeaderAndIsrRequest(2), new UnknownServerException(), false);
         checkResponse(createLeaderAndIsrResponse(), 0, true);
         checkRequest(createSaslHandshakeRequest(), true);
         checkErrorResponse(createSaslHandshakeRequest(), new UnknownServerException(), true);
@@ -1115,9 +1124,14 @@ private StopReplicaRequest createStopReplicaRequest(int version, boolean deleteP
     }
 
     private StopReplicaResponse createStopReplicaResponse() {
-        Map responses = new HashMap<>();
-        responses.put(new TopicPartition("test", 0), Errors.NONE);
-        return new StopReplicaResponse(Errors.NONE, responses);
+        List partitions = new ArrayList<>();
+        partitions.add(new StopReplicaResponseData.StopReplicaPartitionError()
+            .setTopicName("test")
+            .setPartitionIndex(0)
+            .setErrorCode(Errors.NONE.code()));
+        return new StopReplicaResponse(new StopReplicaResponseData()
+            .setErrorCode(Errors.NONE.code())
+            .setPartitionErrors(partitions));
     }
 
     private ControlledShutdownRequest createControlledShutdownRequest() {
@@ -1155,15 +1169,39 @@ private ControlledShutdownResponse createControlledShutdownResponse() {
     }
 
     private LeaderAndIsrRequest createLeaderAndIsrRequest(int version) {
-        Map partitionStates = new HashMap<>();
+        List partitionStates = new ArrayList<>();
         List isr = asList(1, 2);
         List replicas = asList(1, 2, 3, 4);
-        partitionStates.put(new TopicPartition("topic5", 105),
-                new LeaderAndIsrRequest.PartitionState(0, 2, 1, new ArrayList<>(isr), 2, replicas, false));
-        partitionStates.put(new TopicPartition("topic5", 1),
-                new LeaderAndIsrRequest.PartitionState(1, 1, 1, new ArrayList<>(isr), 2, replicas, false));
-        partitionStates.put(new TopicPartition("topic20", 1),
-                new LeaderAndIsrRequest.PartitionState(1, 0, 1, new ArrayList<>(isr), 2, replicas, false));
+        partitionStates.add(new LeaderAndIsrPartitionState()
+            .setTopicName("topic5")
+            .setPartitionIndex(105)
+            .setControllerEpoch(0)
+            .setLeader(2)
+            .setLeaderEpoch(1)
+            .setIsr(isr)
+            .setZkVersion(2)
+            .setReplicas(replicas)
+            .setIsNew(false));
+        partitionStates.add(new LeaderAndIsrPartitionState()
+            .setTopicName("topic5")
+            .setPartitionIndex(1)
+            .setControllerEpoch(1)
+            .setLeader(1)
+            .setLeaderEpoch(1)
+            .setIsr(isr)
+            .setZkVersion(2)
+            .setReplicas(replicas)
+            .setIsNew(false));
+        partitionStates.add(new LeaderAndIsrPartitionState()
+            .setTopicName("topic20")
+            .setPartitionIndex(1)
+            .setControllerEpoch(1)
+            .setLeader(0)
+            .setLeaderEpoch(1)
+            .setIsr(isr)
+            .setZkVersion(2)
+            .setReplicas(replicas)
+            .setIsNew(false));
 
         Set leaders = Utils.mkSet(
                 new Node(0, "test0", 1223),
@@ -1173,49 +1211,97 @@ private LeaderAndIsrRequest createLeaderAndIsrRequest(int version) {
     }
 
     private LeaderAndIsrResponse createLeaderAndIsrResponse() {
-        Map responses = new HashMap<>();
-        responses.put(new TopicPartition("test", 0), Errors.NONE);
-        return new LeaderAndIsrResponse(Errors.NONE, responses);
+        List partitions = new ArrayList<>();
+        partitions.add(new LeaderAndIsrResponseData.LeaderAndIsrPartitionError()
+            .setTopicName("test")
+            .setPartitionIndex(0)
+            .setErrorCode(Errors.NONE.code()));
+        return new LeaderAndIsrResponse(new LeaderAndIsrResponseData()
+            .setErrorCode(Errors.NONE.code())
+            .setPartitionErrors(partitions));
     }
 
     private UpdateMetadataRequest createUpdateMetadataRequest(int version, String rack) {
-        Map partitionStates = new HashMap<>();
+        List partitionStates = new ArrayList<>();
         List isr = asList(1, 2);
         List replicas = asList(1, 2, 3, 4);
         List offlineReplicas = asList();
-        partitionStates.put(new TopicPartition("topic5", 105),
-            new UpdateMetadataRequest.PartitionState(0, 2, 1, isr, 2, replicas, offlineReplicas));
-        partitionStates.put(new TopicPartition("topic5", 1),
-            new UpdateMetadataRequest.PartitionState(1, 1, 1, isr, 2, replicas, offlineReplicas));
-        partitionStates.put(new TopicPartition("topic20", 1),
-            new UpdateMetadataRequest.PartitionState(1, 0, 1, isr, 2, replicas, offlineReplicas));
+        partitionStates.add(new UpdateMetadataPartitionState()
+            .setTopicName("topic5")
+            .setPartitionIndex(105)
+            .setControllerEpoch(0)
+            .setLeader(2)
+            .setLeaderEpoch(1)
+            .setIsr(isr)
+            .setZkVersion(2)
+            .setReplicas(replicas)
+            .setOfflineReplicas(offlineReplicas));
+        partitionStates.add(new UpdateMetadataPartitionState()
+                .setTopicName("topic5")
+                .setPartitionIndex(1)
+                .setControllerEpoch(1)
+                .setLeader(1)
+                .setLeaderEpoch(1)
+                .setIsr(isr)
+                .setZkVersion(2)
+                .setReplicas(replicas)
+                .setOfflineReplicas(offlineReplicas));
+        partitionStates.add(new UpdateMetadataPartitionState()
+                .setTopicName("topic20")
+                .setPartitionIndex(1)
+                .setControllerEpoch(1)
+                .setLeader(0)
+                .setLeaderEpoch(1)
+                .setIsr(isr)
+                .setZkVersion(2)
+                .setReplicas(replicas)
+                .setOfflineReplicas(offlineReplicas));
 
         SecurityProtocol plaintext = SecurityProtocol.PLAINTEXT;
-        List endPoints1 = new ArrayList<>();
-        endPoints1.add(new UpdateMetadataRequest.EndPoint("host1", 1223, plaintext,
-                ListenerName.forSecurityProtocol(plaintext)));
-
-        List endPoints2 = new ArrayList<>();
-        endPoints2.add(new UpdateMetadataRequest.EndPoint("host1", 1244, plaintext,
-                ListenerName.forSecurityProtocol(plaintext)));
+        List endpoints1 = new ArrayList<>();
+        endpoints1.add(new UpdateMetadataEndpoint()
+            .setHost("host1")
+            .setPort(1223)
+            .setSecurityProtocol(plaintext.id)
+            .setListener(ListenerName.forSecurityProtocol(plaintext).value()));
+
+        List endpoints2 = new ArrayList<>();
+        endpoints2.add(new UpdateMetadataEndpoint()
+            .setHost("host1")
+            .setPort(1244)
+            .setSecurityProtocol(plaintext.id)
+            .setListener(ListenerName.forSecurityProtocol(plaintext).value()));
         if (version > 0) {
             SecurityProtocol ssl = SecurityProtocol.SSL;
-            endPoints2.add(new UpdateMetadataRequest.EndPoint("host2", 1234, ssl,
-                    ListenerName.forSecurityProtocol(ssl)));
-            endPoints2.add(new UpdateMetadataRequest.EndPoint("host2", 1334, ssl,
-                    new ListenerName("CLIENT")));
+            endpoints2.add(new UpdateMetadataEndpoint()
+                .setHost("host2")
+                .setPort(1234)
+                .setSecurityProtocol(ssl.id)
+                .setListener(ListenerName.forSecurityProtocol(ssl).value()));
+            endpoints2.add(new UpdateMetadataEndpoint()
+                .setHost("host2")
+                .setPort(1334)
+                .setSecurityProtocol(ssl.id));
+            if (version >= 3)
+                endpoints2.get(1).setListener("CLIENT");
         }
 
-        Set liveBrokers = Utils.mkSet(
-                new UpdateMetadataRequest.Broker(0, endPoints1, rack),
-                new UpdateMetadataRequest.Broker(1, endPoints2, rack)
+        List liveBrokers = Arrays.asList(
+            new UpdateMetadataBroker()
+                .setId(0)
+                .setEndpoints(endpoints1)
+                .setRack(rack),
+            new UpdateMetadataBroker()
+                .setId(1)
+                .setEndpoints(endpoints2)
+                .setRack(rack)
         );
         return new UpdateMetadataRequest.Builder((short) version, 1, 10, 0, partitionStates,
-                liveBrokers).build();
+            liveBrokers).build();
     }
 
     private UpdateMetadataResponse createUpdateMetadataResponse() {
-        return new UpdateMetadataResponse(Errors.NONE);
+        return new UpdateMetadataResponse(new UpdateMetadataResponseData().setErrorCode(Errors.NONE.code()));
     }
 
     private SaslHandshakeRequest createSaslHandshakeRequest() {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/BasePartitionState.java b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java
similarity index 53%
rename from clients/src/main/java/org/apache/kafka/common/requests/BasePartitionState.java
rename to clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java
index c9a461093caaf..c06c9c3c61646 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/BasePartitionState.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java
@@ -16,25 +16,21 @@
  */
 package org.apache.kafka.common.requests;
 
-import java.util.List;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.test.TestUtils;
+import org.junit.Test;
 
-// This class contains the common fields shared between LeaderAndIsrRequest.PartitionState and UpdateMetadataRequest.PartitionState
-public class BasePartitionState {
+import java.util.Set;
 
-    public final int controllerEpoch;
-    public final int leader;
-    public final int leaderEpoch;
-    public final List isr;
-    public final int zkVersion;
-    public final List replicas;
+import static org.junit.Assert.assertTrue;
 
-    BasePartitionState(int controllerEpoch, int leader, int leaderEpoch, List isr, int zkVersion, List replicas) {
-        this.controllerEpoch = controllerEpoch;
-        this.leader = leader;
-        this.leaderEpoch = leaderEpoch;
-        this.isr = isr;
-        this.zkVersion = zkVersion;
-        this.replicas = replicas;
+public class StopReplicaRequestTest {
+
+    @Test
+    public void testStopReplicaRequestNormalization() {
+        Set tps = TestUtils.generateRandomTopicPartitions(10, 10);
+        StopReplicaRequest.Builder builder = new StopReplicaRequest.Builder((short) 5, 0, 0, 0, false, tps);
+        assertTrue(builder.build((short) 1).size() <  builder.build((short) 0).size());
     }
 
 }
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaResponseTest.java
index b6d4bdafb9b44..1463a0bc41cc0 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaResponseTest.java
@@ -17,13 +17,16 @@
 package org.apache.kafka.common.requests;
 
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.message.StopReplicaResponseData;
+import org.apache.kafka.common.message.StopReplicaResponseData.StopReplicaPartitionError;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.utils.Utils;
 import org.junit.Test;
 
+import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
@@ -41,19 +44,25 @@ public void testErrorCountsFromGetErrorResponse() {
 
     @Test
     public void testErrorCountsWithTopLevelError() {
-        Map errors = new HashMap<>();
-        errors.put(new TopicPartition("foo", 0), Errors.NONE);
-        errors.put(new TopicPartition("foo", 1), Errors.NOT_LEADER_FOR_PARTITION);
-        StopReplicaResponse response = new StopReplicaResponse(Errors.UNKNOWN_SERVER_ERROR, errors);
+        List errors = new ArrayList<>();
+        errors.add(new StopReplicaPartitionError().setTopicName("foo").setPartitionIndex(0));
+        errors.add(new StopReplicaPartitionError().setTopicName("foo").setPartitionIndex(1)
+            .setErrorCode(Errors.NOT_LEADER_FOR_PARTITION.code()));
+        StopReplicaResponse response = new StopReplicaResponse(new StopReplicaResponseData()
+            .setErrorCode(Errors.UNKNOWN_SERVER_ERROR.code())
+            .setPartitionErrors(errors));
         assertEquals(Collections.singletonMap(Errors.UNKNOWN_SERVER_ERROR, 2), response.errorCounts());
     }
 
     @Test
     public void testErrorCountsNoTopLevelError() {
-        Map errors = new HashMap<>();
-        errors.put(new TopicPartition("foo", 0), Errors.NONE);
-        errors.put(new TopicPartition("foo", 1), Errors.CLUSTER_AUTHORIZATION_FAILED);
-        StopReplicaResponse response = new StopReplicaResponse(Errors.NONE, errors);
+        List errors = new ArrayList<>();
+        errors.add(new StopReplicaPartitionError().setTopicName("foo").setPartitionIndex(0));
+        errors.add(new StopReplicaPartitionError().setTopicName("foo").setPartitionIndex(1)
+            .setErrorCode(Errors.CLUSTER_AUTHORIZATION_FAILED.code()));
+        StopReplicaResponse response = new StopReplicaResponse(new StopReplicaResponseData()
+            .setErrorCode(Errors.NONE.code())
+            .setPartitionErrors(errors));
         Map errorCounts = response.errorCounts();
         assertEquals(2, errorCounts.size());
         assertEquals(1, errorCounts.get(Errors.NONE).intValue());
@@ -62,14 +71,15 @@ public void testErrorCountsNoTopLevelError() {
 
     @Test
     public void testToString() {
-        Map errors = new HashMap<>();
-        errors.put(new TopicPartition("foo", 0), Errors.NONE);
-        errors.put(new TopicPartition("foo", 1), Errors.CLUSTER_AUTHORIZATION_FAILED);
-        StopReplicaResponse response = new StopReplicaResponse(Errors.NONE, errors);
+        List errors = new ArrayList<>();
+        errors.add(new StopReplicaPartitionError().setTopicName("foo").setPartitionIndex(0));
+        errors.add(new StopReplicaPartitionError().setTopicName("foo").setPartitionIndex(1)
+            .setErrorCode(Errors.CLUSTER_AUTHORIZATION_FAILED.code()));
+        StopReplicaResponse response = new StopReplicaResponse(new StopReplicaResponseData().setPartitionErrors(errors));
         String responseStr = response.toString();
         assertTrue(responseStr.contains(StopReplicaResponse.class.getSimpleName()));
         assertTrue(responseStr.contains(errors.toString()));
-        assertTrue(responseStr.contains(Errors.NONE.name()));
+        assertTrue(responseStr.contains("errorCode=" + Errors.NONE.code()));
     }
 
 }
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java
new file mode 100644
index 0000000000000..9bc81fe86ed29
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.requests;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.message.UpdateMetadataRequestData;
+import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataBroker;
+import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataEndpoint;
+import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataPartitionState;
+import org.apache.kafka.common.network.ListenerName;
+import org.apache.kafka.common.protocol.ByteBufferAccessor;
+import org.apache.kafka.common.security.auth.SecurityProtocol;
+import org.apache.kafka.test.TestUtils;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.emptyList;
+import static org.apache.kafka.common.protocol.ApiKeys.UPDATE_METADATA;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class UpdateMetadataRequestTest {
+
+    /**
+     * Verifies the logic we have in UpdateMetadataRequest to present a unified interface across the various versions
+     * works correctly. For example, `UpdateMetadataPartitionState.topicName` is not serialiazed/deserialized in
+     * recent versions, but we set it manually so that we can always present the ungrouped partition states
+     * independently of the version.
+     */
+    @Test
+    public void testVersionLogic() {
+        for (short version = UPDATE_METADATA.oldestVersion(); version <= UPDATE_METADATA.latestVersion(); version++) {
+            List partitionStates = asList(
+                new UpdateMetadataPartitionState()
+                    .setTopicName("topic0")
+                    .setPartitionIndex(0)
+                    .setControllerEpoch(2)
+                    .setLeader(0)
+                    .setLeaderEpoch(10)
+                    .setIsr(asList(0, 1))
+                    .setZkVersion(10)
+                    .setReplicas(asList(0, 1, 2))
+                    .setOfflineReplicas(asList(2)),
+                new UpdateMetadataPartitionState()
+                    .setTopicName("topic0")
+                    .setPartitionIndex(1)
+                    .setControllerEpoch(2)
+                    .setLeader(1)
+                    .setLeaderEpoch(11)
+                    .setIsr(asList(1, 2, 3))
+                    .setZkVersion(11)
+                    .setReplicas(asList(1, 2, 3))
+                    .setOfflineReplicas(emptyList()),
+                new UpdateMetadataPartitionState()
+                    .setTopicName("topic1")
+                    .setPartitionIndex(0)
+                    .setControllerEpoch(2)
+                    .setLeader(2)
+                    .setLeaderEpoch(11)
+                    .setIsr(asList(2, 3))
+                    .setZkVersion(11)
+                    .setReplicas(asList(2, 3, 4))
+                    .setOfflineReplicas(emptyList())
+            );
+
+            List broker0Endpoints = new ArrayList<>();
+            broker0Endpoints.add(
+                new UpdateMetadataEndpoint()
+                    .setHost("host0")
+                    .setPort(9090)
+                    .setSecurityProtocol(SecurityProtocol.PLAINTEXT.id));
+
+            // Non plaintext endpoints are only supported from version 1
+            if (version >= 1) {
+                broker0Endpoints.add(new UpdateMetadataEndpoint()
+                    .setHost("host0")
+                    .setPort(9091)
+                    .setSecurityProtocol(SecurityProtocol.SSL.id));
+            }
+
+            // Custom listeners are only supported from version 3
+            if (version >= 3) {
+                broker0Endpoints.get(0).setListener("listener0");
+                broker0Endpoints.get(1).setListener("listener1");
+            }
+
+            List liveBrokers = asList(
+                new UpdateMetadataBroker()
+                    .setId(0)
+                    .setRack("rack0")
+                    .setEndpoints(broker0Endpoints),
+                new UpdateMetadataBroker()
+                    .setId(1)
+                    .setEndpoints(asList(
+                        new UpdateMetadataEndpoint()
+                            .setHost("host1")
+                            .setPort(9090)
+                            .setSecurityProtocol(SecurityProtocol.PLAINTEXT.id)
+                            .setListener("PLAINTEXT")
+                    ))
+            );
+
+            UpdateMetadataRequest request = new UpdateMetadataRequest.Builder(version, 1, 2, 3,
+                partitionStates, liveBrokers).build();
+
+            assertEquals(new HashSet<>(partitionStates), iterableToSet(request.partitionStates()));
+            assertEquals(liveBrokers, request.liveBrokers());
+            assertEquals(1, request.controllerId());
+            assertEquals(2, request.controllerEpoch());
+            assertEquals(3, request.brokerEpoch());
+
+            ByteBuffer byteBuffer = request.toBytes();
+            UpdateMetadataRequest deserializedRequest = new UpdateMetadataRequest(new UpdateMetadataRequestData(
+                    new ByteBufferAccessor(byteBuffer), version), version);
+
+            // Unset fields that are not supported in this version as the deserialized request won't have them
+
+            // Rack is only supported from version 2
+            if (version < 2) {
+                for (UpdateMetadataBroker liveBroker : liveBrokers)
+                    liveBroker.setRack("");
+            }
+
+            // Non plaintext listener name is only supported from version 3
+            if (version < 3) {
+                for (UpdateMetadataBroker liveBroker : liveBrokers) {
+                    for (UpdateMetadataEndpoint endpoint : liveBroker.endpoints()) {
+                        SecurityProtocol securityProtocol = SecurityProtocol.forId(endpoint.securityProtocol());
+                        endpoint.setListener(ListenerName.forSecurityProtocol(securityProtocol).value());
+                    }
+                }
+            }
+
+            // Offline replicas are only supported from version 4
+            if (version < 4)
+                partitionStates.get(0).setOfflineReplicas(emptyList());
+
+            assertEquals(new HashSet<>(partitionStates), iterableToSet(deserializedRequest.partitionStates()));
+            assertEquals(liveBrokers, deserializedRequest.liveBrokers());
+            assertEquals(1, request.controllerId());
+            assertEquals(2, request.controllerEpoch());
+            assertEquals(3, request.brokerEpoch());
+        }
+    }
+
+    @Test
+    public void testTopicPartitionGroupingSizeReduction() {
+        Set tps = TestUtils.generateRandomTopicPartitions(10, 10);
+        List partitionStates = new ArrayList<>();
+        for (TopicPartition tp : tps) {
+            partitionStates.add(new UpdateMetadataPartitionState()
+                .setTopicName(tp.topic())
+                .setPartitionIndex(tp.partition()));
+        }
+        UpdateMetadataRequest.Builder builder = new UpdateMetadataRequest.Builder((short) 5, 0, 0, 0,
+                partitionStates, Collections.emptyList());
+
+        assertTrue(builder.build((short) 5).size() <  builder.build((short) 4).size());
+    }
+
+    private  Set iterableToSet(Iterable iterable) {
+        return StreamSupport.stream(iterable.spliterator(), false).collect(Collectors.toSet());
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/common/utils/FlattenedIteratorTest.java b/clients/src/test/java/org/apache/kafka/common/utils/FlattenedIteratorTest.java
new file mode 100644
index 0000000000000..95206cae473a8
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/utils/FlattenedIteratorTest.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.utils;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.emptyList;
+import static org.junit.Assert.assertEquals;
+
+public class FlattenedIteratorTest {
+
+    @Test
+    public void testNestedLists() {
+        List> list = asList(
+            asList("foo", "a", "bc"),
+            asList("ddddd"),
+            asList("", "bar2", "baz45"));
+
+        Iterable flattenedIterable = () -> new FlattenedIterator<>(list.iterator(), l -> l.iterator());
+        List flattened = new ArrayList<>();
+        flattenedIterable.forEach(flattened::add);
+
+        assertEquals(list.stream().flatMap(l -> l.stream()).collect(Collectors.toList()), flattened);
+
+        // Ensure we can iterate multiple times
+        List flattened2 = new ArrayList<>();
+        flattenedIterable.forEach(flattened2::add);
+
+        assertEquals(flattened, flattened2);
+    }
+
+    @Test
+    public void testEmptyList() {
+        List> list = emptyList();
+
+        Iterable flattenedIterable = () -> new FlattenedIterator<>(list.iterator(), l -> l.iterator());
+        List flattened = new ArrayList<>();
+        flattenedIterable.forEach(flattened::add);
+
+        assertEquals(emptyList(), flattened);
+    }
+
+    @Test
+    public void testNestedSingleEmptyList() {
+        List> list = asList(emptyList());
+
+        Iterable flattenedIterable = () -> new FlattenedIterator<>(list.iterator(), l -> l.iterator());
+        List flattened = new ArrayList<>();
+        flattenedIterable.forEach(flattened::add);
+
+        assertEquals(emptyList(), flattened);
+    }
+
+    @Test
+    public void testEmptyListFollowedByNonEmpty() {
+        List> list = asList(
+            emptyList(),
+            asList("boo", "b", "de"));
+
+        Iterable flattenedIterable = () -> new FlattenedIterator<>(list.iterator(), l -> l.iterator());
+        List flattened = new ArrayList<>();
+        flattenedIterable.forEach(flattened::add);
+
+        assertEquals(list.stream().flatMap(l -> l.stream()).collect(Collectors.toList()), flattened);
+    }
+
+    @Test
+    public void testEmptyListInBetweenNonEmpty() {
+        List> list = asList(
+            asList("aadwdwdw"),
+            emptyList(),
+            asList("ee", "aa", "dd"));
+
+        Iterable flattenedIterable = () -> new FlattenedIterator<>(list.iterator(), l -> l.iterator());
+        List flattened = new ArrayList<>();
+        flattenedIterable.forEach(flattened::add);
+
+        assertEquals(list.stream().flatMap(l -> l.stream()).collect(Collectors.toList()), flattened);
+    }
+
+    @Test
+    public void testEmptyListAtTheEnd() {
+        List> list = asList(
+            asList("ee", "dd"),
+            asList("e"),
+            emptyList());
+
+        Iterable flattenedIterable = () -> new FlattenedIterator<>(list.iterator(), l -> l.iterator());
+        List flattened = new ArrayList<>();
+        flattenedIterable.forEach(flattened::add);
+
+        assertEquals(list.stream().flatMap(l -> l.stream()).collect(Collectors.toList()), flattened);
+    }
+
+}
+
diff --git a/clients/src/test/java/org/apache/kafka/common/utils/MappedIteratorTest.java b/clients/src/test/java/org/apache/kafka/common/utils/MappedIteratorTest.java
new file mode 100644
index 0000000000000..729bb25f70f8d
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/utils/MappedIteratorTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.utils;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.emptyList;
+import static org.junit.Assert.assertEquals;
+
+public class MappedIteratorTest {
+
+    @Test
+    public void testStringToInteger() {
+        List list = asList("foo", "", "bar2", "baz45");
+        Function mapper = s -> s.length();
+
+        Iterable mappedIterable = () -> new MappedIterator<>(list.iterator(), mapper);
+        List mapped = new ArrayList<>();
+        mappedIterable.forEach(mapped::add);
+
+        assertEquals(list.stream().map(mapper).collect(Collectors.toList()), mapped);
+
+        // Ensure that we can iterate a second time
+        List mapped2 = new ArrayList<>();
+        mappedIterable.forEach(mapped2::add);
+        assertEquals(mapped, mapped2);
+    }
+
+    @Test
+    public void testEmptyList() {
+        List list = emptyList();
+        Function mapper = s -> s.length();
+
+        Iterable mappedIterable = () -> new MappedIterator<>(list.iterator(), mapper);
+        List mapped = new ArrayList<>();
+        mappedIterable.forEach(mapped::add);
+
+        assertEquals(emptyList(), mapped);
+    }
+
+}
diff --git a/clients/src/test/java/org/apache/kafka/test/TestUtils.java b/clients/src/test/java/org/apache/kafka/test/TestUtils.java
index 5966cbe4a2f04..5858a16e4a50c 100644
--- a/clients/src/test/java/org/apache/kafka/test/TestUtils.java
+++ b/clients/src/test/java/org/apache/kafka/test/TestUtils.java
@@ -446,6 +446,17 @@ public static ByteBuffer toBuffer(Struct struct) {
         return buffer;
     }
 
+    public static Set generateRandomTopicPartitions(int numTopic, int numPartitionPerTopic) {
+        Set tps = new HashSet<>();
+        for (int i = 0; i < numTopic; i++) {
+            String topic = randomString(32);
+            for (int j = 0; j < numPartitionPerTopic; j++) {
+                tps.add(new TopicPartition(topic, j));
+            }
+        }
+        return tps;
+    }
+
     public static void assertFutureError(Future future, Class exceptionClass)
         throws InterruptedException {
         try {
diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala
index 655bdeae008a1..c3282b778f1ee 100755
--- a/core/src/main/scala/kafka/cluster/Partition.scala
+++ b/core/src/main/scala/kafka/cluster/Partition.scala
@@ -19,6 +19,7 @@ package kafka.cluster
 import com.yammer.metrics.core.Gauge
 import java.util.concurrent.locks.ReentrantReadWriteLock
 import java.util.{Optional, Properties}
+
 import kafka.api.{ApiVersion, LeaderAndIsr, Request}
 import kafka.common.UnexpectedAppendOffsetException
 import kafka.controller.KafkaController
@@ -31,6 +32,7 @@ import kafka.utils._
 import kafka.zk.{AdminZkClient, KafkaZkClient}
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors._
+import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.protocol.Errors._
 import org.apache.kafka.common.record.FileRecords.TimestampAndOffset
@@ -38,6 +40,7 @@ import org.apache.kafka.common.record.{MemoryRecords, RecordBatch}
 import org.apache.kafka.common.requests.EpochEndOffset._
 import org.apache.kafka.common.requests._
 import org.apache.kafka.common.utils.Time
+
 import scala.collection.JavaConverters._
 import scala.collection.{Map, Seq}
 
@@ -476,29 +479,29 @@ class Partition(val topicPartition: TopicPartition,
    * If the leader replica id does not change, return false to indicate the replica manager.
    */
   def makeLeader(controllerId: Int,
-                 partitionStateInfo: LeaderAndIsrRequest.PartitionState,
+                 partitionState: LeaderAndIsrPartitionState,
                  correlationId: Int,
                  highWatermarkCheckpoints: OffsetCheckpoints): Boolean = {
     val (leaderHWIncremented, isNewLeader) = inWriteLock(leaderIsrUpdateLock) {
       // record the epoch of the controller that made the leadership decision. This is useful while updating the isr
       // to maintain the decision maker controller's epoch in the zookeeper path
-      controllerEpoch = partitionStateInfo.basePartitionState.controllerEpoch
+      controllerEpoch = partitionState.controllerEpoch
 
       updateAssignmentAndIsr(
-        assignment = partitionStateInfo.basePartitionState.replicas.asScala.iterator.map(_.toInt).toSeq,
-        isr = partitionStateInfo.basePartitionState.isr.asScala.iterator.map(_.toInt).toSet
+        assignment = partitionState.replicas.asScala.iterator.map(_.toInt).toSeq,
+        isr = partitionState.isr.asScala.iterator.map(_.toInt).toSet
       )
-      createLogIfNotExists(localBrokerId, partitionStateInfo.isNew, isFutureReplica = false, highWatermarkCheckpoints)
+      createLogIfNotExists(localBrokerId, partitionState.isNew, isFutureReplica = false, highWatermarkCheckpoints)
 
       val leaderLog = localLogOrException
       val leaderEpochStartOffset = leaderLog.logEndOffset
-      info(s"$topicPartition starts at Leader Epoch ${partitionStateInfo.basePartitionState.leaderEpoch} from " +
+      info(s"$topicPartition starts at Leader Epoch ${partitionState.leaderEpoch} from " +
         s"offset $leaderEpochStartOffset. Previous Leader Epoch was: $leaderEpoch")
 
       //We cache the leader epoch here, persisting it only if it's local (hence having a log dir)
-      leaderEpoch = partitionStateInfo.basePartitionState.leaderEpoch
+      leaderEpoch = partitionState.leaderEpoch
       leaderEpochStartOffsetOpt = Some(leaderEpochStartOffset)
-      zkVersion = partitionStateInfo.basePartitionState.zkVersion
+      zkVersion = partitionState.zkVersion
 
       // In the case of successive leader elections in a short time period, a follower may have
       // entries in its log from a later epoch than any entry in the new leader's log. In order
@@ -546,25 +549,25 @@ class Partition(val topicPartition: TopicPartition,
     * replica manager that state is already correct and the become-follower steps can be skipped
    */
   def makeFollower(controllerId: Int,
-                   partitionStateInfo: LeaderAndIsrRequest.PartitionState,
+                   partitionState: LeaderAndIsrPartitionState,
                    correlationId: Int,
                    highWatermarkCheckpoints: OffsetCheckpoints): Boolean = {
     inWriteLock(leaderIsrUpdateLock) {
-      val newLeaderBrokerId = partitionStateInfo.basePartitionState.leader
+      val newLeaderBrokerId = partitionState.leader
       val oldLeaderEpoch = leaderEpoch
       // record the epoch of the controller that made the leadership decision. This is useful while updating the isr
       // to maintain the decision maker controller's epoch in the zookeeper path
-      controllerEpoch = partitionStateInfo.basePartitionState.controllerEpoch
+      controllerEpoch = partitionState.controllerEpoch
 
       updateAssignmentAndIsr(
-        assignment = partitionStateInfo.basePartitionState.replicas.asScala.iterator.map(_.toInt).toSeq,
+        assignment = partitionState.replicas.asScala.iterator.map(_.toInt).toSeq,
         isr = Set.empty[Int]
       )
-      createLogIfNotExists(localBrokerId, partitionStateInfo.isNew, isFutureReplica = false, highWatermarkCheckpoints)
+      createLogIfNotExists(localBrokerId, partitionState.isNew, isFutureReplica = false, highWatermarkCheckpoints)
 
-      leaderEpoch = partitionStateInfo.basePartitionState.leaderEpoch
+      leaderEpoch = partitionState.leaderEpoch
       leaderEpochStartOffsetOpt = None
-      zkVersion = partitionStateInfo.basePartitionState.zkVersion
+      zkVersion = partitionState.zkVersion
 
       if (leaderReplicaIdOpt.contains(newLeaderBrokerId) && leaderEpoch == oldLeaderEpoch) {
         false
diff --git a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
index d4d37d2849917..6c7450a0c7a99 100755
--- a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
+++ b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
@@ -26,10 +26,11 @@ import kafka.metrics.KafkaMetricsGroup
 import kafka.server.KafkaConfig
 import kafka.utils._
 import org.apache.kafka.clients._
+import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.network._
-import org.apache.kafka.common.protocol.ApiKeys
-import org.apache.kafka.common.requests.UpdateMetadataRequest.EndPoint
+import org.apache.kafka.common.protocol.{ApiKeys, Errors}
+import org.apache.kafka.common.message.UpdateMetadataRequestData.{UpdateMetadataBroker, UpdateMetadataEndpoint, UpdateMetadataPartitionState}
 import org.apache.kafka.common.requests._
 import org.apache.kafka.common.security.JaasContext
 import org.apache.kafka.common.security.auth.SecurityProtocol
@@ -341,10 +342,10 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
                                                     controllerContext: ControllerContext,
                                                     stateChangeLogger: StateChangeLogger) extends  Logging {
   val controllerId: Int = config.brokerId
-  val leaderAndIsrRequestMap = mutable.Map.empty[Int, mutable.Map[TopicPartition, LeaderAndIsrRequest.PartitionState]]
+  val leaderAndIsrRequestMap = mutable.Map.empty[Int, mutable.Map[TopicPartition, LeaderAndIsrPartitionState]]
   val stopReplicaRequestMap = mutable.Map.empty[Int, ListBuffer[StopReplicaRequestInfo]]
   val updateMetadataRequestBrokerSet = mutable.Set.empty[Int]
-  val updateMetadataRequestPartitionInfoMap = mutable.Map.empty[TopicPartition, UpdateMetadataRequest.PartitionState]
+  val updateMetadataRequestPartitionInfoMap = mutable.Map.empty[TopicPartition, UpdateMetadataPartitionState]
 
   def sendEvent(event: ControllerEvent): Unit
 
@@ -382,15 +383,19 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
     brokerIds.filter(_ >= 0).foreach { brokerId =>
       val result = leaderAndIsrRequestMap.getOrElseUpdate(brokerId, mutable.Map.empty)
       val alreadyNew = result.get(topicPartition).exists(_.isNew)
-      result.put(topicPartition, new LeaderAndIsrRequest.PartitionState(leaderIsrAndControllerEpoch.controllerEpoch,
-        leaderIsrAndControllerEpoch.leaderAndIsr.leader,
-        leaderIsrAndControllerEpoch.leaderAndIsr.leaderEpoch,
-        leaderIsrAndControllerEpoch.leaderAndIsr.isr.map(Integer.valueOf).asJava,
-        leaderIsrAndControllerEpoch.leaderAndIsr.zkVersion,
-        replicaAssignment.replicas.map(Integer.valueOf).asJava,
-        replicaAssignment.addingReplicas.map(Integer.valueOf).asJava,
-        replicaAssignment.removingReplicas.map(Integer.valueOf).asJava,
-        isNew || alreadyNew))
+      val leaderAndIsr = leaderIsrAndControllerEpoch.leaderAndIsr
+      result.put(topicPartition, new LeaderAndIsrPartitionState()
+        .setTopicName(topicPartition.topic)
+        .setPartitionIndex(topicPartition.partition)
+        .setControllerEpoch(leaderIsrAndControllerEpoch.controllerEpoch)
+        .setLeader(leaderAndIsr.leader)
+        .setLeaderEpoch(leaderAndIsr.leaderEpoch)
+        .setIsr(leaderAndIsr.isr.map(Integer.valueOf).asJava)
+        .setZkVersion(leaderAndIsr.zkVersion)
+        .setReplicas(replicaAssignment.replicas.map(Integer.valueOf).asJava)
+        .setAddingReplicas(replicaAssignment.addingReplicas.map(Integer.valueOf).asJava)
+        .setRemovingReplicas(replicaAssignment.removingReplicas.map(Integer.valueOf).asJava)
+        .setIsNew(isNew || alreadyNew))
     }
 
     addUpdateMetadataRequestForBrokers(controllerContext.liveOrShuttingDownBrokerIds.toSeq, Set(topicPartition))
@@ -410,25 +415,24 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
                                          partitions: collection.Set[TopicPartition]): Unit = {
 
     def updateMetadataRequestPartitionInfo(partition: TopicPartition, beingDeleted: Boolean): Unit = {
-      val leaderIsrAndControllerEpochOpt = controllerContext.partitionLeadershipInfo.get(partition)
-      leaderIsrAndControllerEpochOpt match {
-        case Some(l @ LeaderIsrAndControllerEpoch(leaderAndIsr, controllerEpoch)) =>
+      controllerContext.partitionLeadershipInfo.get(partition) match {
+        case Some(LeaderIsrAndControllerEpoch(leaderAndIsr, controllerEpoch)) =>
           val replicas = controllerContext.partitionReplicaAssignment(partition)
           val offlineReplicas = replicas.filter(!controllerContext.isReplicaOnline(_, partition))
-          val leaderIsrAndControllerEpoch = if (beingDeleted) {
-            val leaderDuringDelete = LeaderAndIsr.duringDelete(leaderAndIsr.isr)
-            LeaderIsrAndControllerEpoch(leaderDuringDelete, controllerEpoch)
-          } else {
-            l
-          }
-
-          val partitionStateInfo = new UpdateMetadataRequest.PartitionState(leaderIsrAndControllerEpoch.controllerEpoch,
-            leaderIsrAndControllerEpoch.leaderAndIsr.leader,
-            leaderIsrAndControllerEpoch.leaderAndIsr.leaderEpoch,
-            leaderIsrAndControllerEpoch.leaderAndIsr.isr.map(Integer.valueOf).asJava,
-            leaderIsrAndControllerEpoch.leaderAndIsr.zkVersion,
-            replicas.map(Integer.valueOf).asJava,
-            offlineReplicas.map(Integer.valueOf).asJava)
+          val updatedLeaderAndIsr =
+            if (beingDeleted) LeaderAndIsr.duringDelete(leaderAndIsr.isr)
+            else leaderAndIsr
+
+          val partitionStateInfo = new UpdateMetadataPartitionState()
+            .setTopicName(partition.topic)
+            .setPartitionIndex(partition.partition)
+            .setControllerEpoch(controllerEpoch)
+            .setLeader(updatedLeaderAndIsr.leader)
+            .setLeaderEpoch(updatedLeaderAndIsr.leaderEpoch)
+            .setIsr(updatedLeaderAndIsr.isr.map(Integer.valueOf).asJava)
+            .setZkVersion(updatedLeaderAndIsr.zkVersion)
+            .setReplicas(replicas.map(Integer.valueOf).asJava)
+            .setOfflineReplicas(offlineReplicas.map(Integer.valueOf).asJava)
           updateMetadataRequestPartitionInfoMap.put(partition, partitionStateInfo)
 
         case None =>
@@ -450,20 +454,21 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
 
     leaderAndIsrRequestMap.filterKeys(controllerContext.liveOrShuttingDownBrokerIds.contains).foreach {
       case (broker, leaderAndIsrPartitionStates) =>
-        leaderAndIsrPartitionStates.foreach {
-          case (topicPartition, state) =>
+        if (stateChangeLog.isTraceEnabled) {
+          leaderAndIsrPartitionStates.foreach { case (topicPartition, state) =>
             val typeOfRequest =
-              if (broker == state.basePartitionState.leader) "become-leader"
+              if (broker == state.leader) "become-leader"
               else "become-follower"
             stateChangeLog.trace(s"Sending $typeOfRequest LeaderAndIsr request $state to broker $broker for partition $topicPartition")
+          }
         }
-        val leaderIds = leaderAndIsrPartitionStates.map(_._2.basePartitionState.leader).toSet
+        val leaderIds = leaderAndIsrPartitionStates.map(_._2.leader).toSet
         val leaders = controllerContext.liveOrShuttingDownBrokers.filter(b => leaderIds.contains(b.id)).map {
           _.node(config.interBrokerListenerName)
         }
         val brokerEpoch = controllerContext.liveBrokerIdAndEpochs(broker)
         val leaderAndIsrRequestBuilder = new LeaderAndIsrRequest.Builder(leaderAndIsrRequestVersion, controllerId, controllerEpoch,
-          brokerEpoch, leaderAndIsrPartitionStates.asJava, leaders.asJava)
+          brokerEpoch, leaderAndIsrPartitionStates.values.toBuffer.asJava, leaders.asJava)
         sendRequest(broker, leaderAndIsrRequestBuilder, (r: AbstractResponse) => sendEvent(LeaderAndIsrResponseReceived(r, broker)))
 
     }
@@ -476,7 +481,7 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
         s"for partition $tp")
     }
 
-    val partitionStates = Map.empty ++ updateMetadataRequestPartitionInfoMap
+    val partitionStates = updateMetadataRequestPartitionInfoMap.values.toBuffer
     val updateMetadataRequestVersion: Short =
       if (config.interBrokerProtocolVersion >= KAFKA_2_2_IV0) 5
       else if (config.interBrokerProtocolVersion >= KAFKA_1_0_IV0) 4
@@ -485,23 +490,31 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
       else if (config.interBrokerProtocolVersion >= KAFKA_0_9_0) 1
       else 0
 
-    val liveBrokers = if (updateMetadataRequestVersion == 0) {
-      // Version 0 of UpdateMetadataRequest only supports PLAINTEXT.
-      controllerContext.liveOrShuttingDownBrokers.map { broker =>
+    val liveBrokers = controllerContext.liveOrShuttingDownBrokers.iterator.map { broker =>
+      val endpoints = if (updateMetadataRequestVersion == 0) {
+        // Version 0 of UpdateMetadataRequest only supports PLAINTEXT
         val securityProtocol = SecurityProtocol.PLAINTEXT
         val listenerName = ListenerName.forSecurityProtocol(securityProtocol)
         val node = broker.node(listenerName)
-        val endPoints = Seq(new EndPoint(node.host, node.port, securityProtocol, listenerName))
-        new UpdateMetadataRequest.Broker(broker.id, endPoints.asJava, broker.rack.orNull)
-      }
-    } else {
-      controllerContext.liveOrShuttingDownBrokers.map { broker =>
-        val endPoints = broker.endPoints.map { endPoint =>
-          new UpdateMetadataRequest.EndPoint(endPoint.host, endPoint.port, endPoint.securityProtocol, endPoint.listenerName)
+        Seq(new UpdateMetadataEndpoint()
+          .setHost(node.host)
+          .setPort(node.port)
+          .setSecurityProtocol(securityProtocol.id)
+          .setListener(listenerName.value))
+      } else {
+        broker.endPoints.map { endpoint =>
+          new UpdateMetadataEndpoint()
+            .setHost(endpoint.host)
+            .setPort(endpoint.port)
+            .setSecurityProtocol(endpoint.securityProtocol.id)
+            .setListener(endpoint.listenerName.value)
         }
-        new UpdateMetadataRequest.Broker(broker.id, endPoints.asJava, broker.rack.orNull)
       }
-    }
+      new UpdateMetadataBroker()
+        .setId(broker.id)
+        .setEndpoints(endpoints.asJava)
+        .setRack(broker.rack.orNull)
+    }.toBuffer
 
     updateMetadataRequestBrokerSet.intersect(controllerContext.liveOrShuttingDownBrokerIds).foreach { broker =>
       val brokerEpoch = controllerContext.liveBrokerIdAndEpochs(broker)
@@ -520,9 +533,9 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
 
     def stopReplicaPartitionDeleteResponseCallback(brokerId: Int)(response: AbstractResponse): Unit = {
       val stopReplicaResponse = response.asInstanceOf[StopReplicaResponse]
-      val partitionErrorsForDeletingTopics = stopReplicaResponse.responses.asScala.filterKeys { partition =>
-        controllerContext.isTopicDeletionInProgress(partition.topic)
-      }.toMap
+      val partitionErrorsForDeletingTopics = stopReplicaResponse.partitionErrors.asScala.iterator.filter { pe =>
+        controllerContext.isTopicDeletionInProgress(pe.topicName)
+      }.map(pe => new TopicPartition(pe.topicName, pe.partitionIndex) -> Errors.forCode(pe.errorCode)).toMap
 
       if (partitionErrorsForDeletingTopics.nonEmpty)
         sendEvent(TopicDeletionStopReplicaResponseReceived(brokerId, stopReplicaResponse.error, partitionErrorsForDeletingTopics))
diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala
index 04703e68aa51d..eae02feab7140 100644
--- a/core/src/main/scala/kafka/controller/KafkaController.scala
+++ b/core/src/main/scala/kafka/controller/KafkaController.scala
@@ -42,6 +42,7 @@ import org.apache.zookeeper.KeeperException.Code
 
 import scala.collection.JavaConverters._
 import scala.collection.{Map, Seq, Set, immutable, mutable}
+import scala.collection.mutable.ArrayBuffer
 import scala.util.{Failure, Try}
 
 sealed trait ElectionTrigger
@@ -1312,12 +1313,17 @@ class KafkaController(val config: KafkaConfig,
       return
     }
 
-    val offlineReplicas = leaderAndIsrResponse.responses.asScala.collect {
-      case (tp, error) if error == Errors.KAFKA_STORAGE_ERROR => tp
-    }
-    val onlineReplicas = leaderAndIsrResponse.responses.asScala.collect {
-      case (tp, error) if error == Errors.NONE => tp
+    val offlineReplicas = new ArrayBuffer[TopicPartition]()
+    val onlineReplicas = new ArrayBuffer[TopicPartition]()
+
+    leaderAndIsrResponse.partitions.asScala.foreach { partition =>
+      val tp = new TopicPartition(partition.topicName, partition.partitionIndex)
+      if (partition.errorCode == Errors.KAFKA_STORAGE_ERROR.code)
+        offlineReplicas += tp
+      else if (partition.errorCode == Errors.NONE.code)
+        onlineReplicas += tp
     }
+
     val previousOfflineReplicas = controllerContext.replicasOnOfflineDirs.getOrElse(brokerId, Set.empty[TopicPartition])
     val currentOfflineReplicas = previousOfflineReplicas -- onlineReplicas ++ offlineReplicas
     controllerContext.replicasOnOfflineDirs.put(brokerId, currentOfflineReplicas)
diff --git a/core/src/main/scala/kafka/server/AdminManager.scala b/core/src/main/scala/kafka/server/AdminManager.scala
index 1b2a2c381522d..5de1bcd97fba9 100644
--- a/core/src/main/scala/kafka/server/AdminManager.scala
+++ b/core/src/main/scala/kafka/server/AdminManager.scala
@@ -188,7 +188,7 @@ class AdminManager(val config: KafkaConfig,
         case e: Throwable =>
           error(s"Error processing create topic request $topic", e)
           CreatePartitionsMetadata(topic.name, Map(), ApiError.fromThrowable(e))
-      }).toIndexedSeq
+      }).toBuffer
 
     // 2. if timeout <= 0, validateOnly or no topics can proceed return immediately
     if (timeout <= 0 || validateOnly || !metadata.exists(_.error.is(Errors.NONE))) {
@@ -204,7 +204,7 @@ class AdminManager(val config: KafkaConfig,
     } else {
       // 3. else pass the assignments and errors to the delayed operation and set the keys
       val delayedCreate = new DelayedCreatePartitions(timeout, metadata, this, responseCallback)
-      val delayedCreateKeys = toCreate.values.map(topic => new TopicKey(topic.name)).toIndexedSeq
+      val delayedCreateKeys = toCreate.values.map(topic => new TopicKey(topic.name)).toBuffer
       // try to complete the request immediately, otherwise put it into the purgatory
       topicPurgatory.tryCompleteElseWatch(delayedCreate, delayedCreateKeys)
     }
@@ -345,7 +345,7 @@ class AdminManager(val config: KafkaConfig,
         val filteredConfigPairs = configs.filter { case (configName, _) =>
           /* Always returns true if configNames is None */
           configNames.forall(_.contains(configName))
-        }.toIndexedSeq
+        }.toBuffer
 
         val configEntries = filteredConfigPairs.map { case (name, value) => createConfigEntry(name, value) }
         new DescribeConfigsResponse.Config(ApiError.NONE, configEntries.asJava)
diff --git a/core/src/main/scala/kafka/server/DelayedCreatePartitions.scala b/core/src/main/scala/kafka/server/DelayedCreatePartitions.scala
index cfa017852d40f..8542bc2a2ecee 100644
--- a/core/src/main/scala/kafka/server/DelayedCreatePartitions.scala
+++ b/core/src/main/scala/kafka/server/DelayedCreatePartitions.scala
@@ -83,6 +83,6 @@ class DelayedCreatePartitions(delayMs: Long,
 
   private def isMissingLeader(topic: String, partition: Int): Boolean = {
     val partitionInfo = adminManager.metadataCache.getPartitionInfo(topic, partition)
-    partitionInfo.isEmpty || partitionInfo.get.basePartitionState.leader == LeaderAndIsr.NoLeader
+    partitionInfo.forall(_.leader == LeaderAndIsr.NoLeader)
   }
 }
diff --git a/core/src/main/scala/kafka/server/DelayedElectLeader.scala b/core/src/main/scala/kafka/server/DelayedElectLeader.scala
index 95b577a269273..d599f98411b3c 100644
--- a/core/src/main/scala/kafka/server/DelayedElectLeader.scala
+++ b/core/src/main/scala/kafka/server/DelayedElectLeader.scala
@@ -50,10 +50,10 @@ class DelayedElectLeader(
   override def onComplete(): Unit = {
     // This could be called to force complete, so I need the full list of partitions, so I can time them all out.
     updateWaiting()
-    val timedout = waitingPartitions.map {
+    val timedOut = waitingPartitions.map {
       case (tp, _) => tp -> new ApiError(Errors.REQUEST_TIMED_OUT, null)
     }.toMap
-    responseCallback(timedout ++ fullResults)
+    responseCallback(timedOut ++ fullResults)
   }
 
   /**
@@ -69,17 +69,14 @@ class DelayedElectLeader(
     waitingPartitions.isEmpty && forceComplete()
   }
 
-  private def updateWaiting() = {
-    waitingPartitions.foreach { case (tp, leader) =>
-      val ps = replicaManager.metadataCache.getPartitionInfo(tp.topic, tp.partition)
-      ps match {
-        case Some(ps) =>
-          if (leader == ps.basePartitionState.leader) {
-            waitingPartitions -= tp
-            fullResults += tp -> ApiError.NONE
-          }
-        case None =>
-      }
+  private def updateWaiting(): Unit = {
+    val metadataCache = replicaManager.metadataCache
+    val completedPartitions = waitingPartitions.collect {
+      case (tp, leader) if metadataCache.getPartitionInfo(tp.topic, tp.partition).exists(_.leader == leader) => tp
+    }
+    completedPartitions.foreach  { tp =>
+      waitingPartitions -= tp
+      fullResults += tp -> ApiError.NONE
     }
   }
 
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 6efcf29569116..109530c1b51d7 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -48,33 +48,14 @@ import org.apache.kafka.common.errors._
 import org.apache.kafka.common.internals.FatalExitError
 import org.apache.kafka.common.internals.Topic.{GROUP_METADATA_TOPIC_NAME, TRANSACTION_STATE_TOPIC_NAME, isInternal}
 import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic
-import org.apache.kafka.common.message.CreateTopicsResponseData
+import org.apache.kafka.common.message.{AlterPartitionReassignmentsResponseData, CreateTopicsResponseData, DeleteGroupsResponseData, DeleteTopicsResponseData, DescribeGroupsResponseData, ExpireDelegationTokenResponseData, FindCoordinatorResponseData, HeartbeatResponseData, InitProducerIdResponseData, JoinGroupResponseData, LeaveGroupResponseData, ListGroupsResponseData, ListPartitionReassignmentsResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteResponseData, RenewDelegationTokenResponseData, SaslAuthenticateResponseData, SaslHandshakeResponseData, StopReplicaResponseData, SyncGroupResponseData, UpdateMetadataResponseData}
 import org.apache.kafka.common.message.CreateTopicsResponseData.{CreatableTopicResult, CreatableTopicResultCollection}
-import org.apache.kafka.common.message.DeleteGroupsResponseData
 import org.apache.kafka.common.message.DeleteGroupsResponseData.{DeletableGroupResult, DeletableGroupResultCollection}
-import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData
 import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.{ReassignablePartitionResponse, ReassignableTopicResponse}
-import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData
-import org.apache.kafka.common.message.DeleteTopicsResponseData
 import org.apache.kafka.common.message.DeleteTopicsResponseData.{DeletableTopicResult, DeletableTopicResultCollection}
-import org.apache.kafka.common.message.DescribeGroupsResponseData
 import org.apache.kafka.common.message.ElectLeadersResponseData.PartitionResult
 import org.apache.kafka.common.message.ElectLeadersResponseData.ReplicaElectionResult
-import org.apache.kafka.common.message.ExpireDelegationTokenResponseData
-import org.apache.kafka.common.message.FindCoordinatorResponseData
-import org.apache.kafka.common.message.HeartbeatResponseData
-import org.apache.kafka.common.message.InitProducerIdResponseData
-import org.apache.kafka.common.message.JoinGroupResponseData
-import org.apache.kafka.common.message.LeaveGroupResponseData
 import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse
-import org.apache.kafka.common.message.ListGroupsResponseData
-import org.apache.kafka.common.message.OffsetCommitRequestData
-import org.apache.kafka.common.message.OffsetCommitResponseData
-import org.apache.kafka.common.message.OffsetDeleteResponseData
-import org.apache.kafka.common.message.RenewDelegationTokenResponseData
-import org.apache.kafka.common.message.SaslAuthenticateResponseData
-import org.apache.kafka.common.message.SaslHandshakeResponseData
-import org.apache.kafka.common.message.SyncGroupResponseData
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.network.{ListenerName, Send}
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
@@ -247,7 +228,7 @@ class KafkaApis(val requestChannel: RequestChannel,
       // for its previous generation so the broker should skip the stale request.
       info("Received stop replica request with broker epoch " +
         s"${stopReplicaRequest.brokerEpoch()} smaller than the current broker epoch ${controller.brokerEpoch}")
-      sendResponseExemptThrottle(request, new StopReplicaResponse(Errors.STALE_BROKER_EPOCH, Map.empty[TopicPartition, Errors].asJava))
+      sendResponseExemptThrottle(request, new StopReplicaResponse(new StopReplicaResponseData().setErrorCode(Errors.STALE_BROKER_EPOCH.code)))
     } else {
       val (result, error) = replicaManager.stopReplicas(stopReplicaRequest)
       // Clearing out the cache for groups that belong to an offsets topic partition for which this broker was the leader,
@@ -261,7 +242,16 @@ class KafkaApis(val requestChannel: RequestChannel,
           groupCoordinator.handleGroupEmigration(topicPartition.partition)
         }
       }
-      sendResponseExemptThrottle(request, new StopReplicaResponse(error, result.asJava))
+
+      def toStopReplicaPartition(tp: TopicPartition, error: Errors) =
+        new StopReplicaResponseData.StopReplicaPartitionError()
+          .setTopicName(tp.topic)
+          .setPartitionIndex(tp.partition)
+          .setErrorCode(error.code)
+
+      sendResponseExemptThrottle(request, new StopReplicaResponse(new StopReplicaResponseData()
+        .setErrorCode(error.code)
+        .setPartitionErrors(result.map { case (tp, error) => toStopReplicaPartition(tp, error) }.toBuffer.asJava)))
     }
 
     CoreUtils.swallow(replicaManager.replicaFetcherManager.shutdownIdleFetcherThreads(), this)
@@ -272,20 +262,21 @@ class KafkaApis(val requestChannel: RequestChannel,
     val updateMetadataRequest = request.body[UpdateMetadataRequest]
 
     authorizeClusterOperation(request, CLUSTER_ACTION)
-    if (isBrokerEpochStale(updateMetadataRequest.brokerEpoch())) {
+    if (isBrokerEpochStale(updateMetadataRequest.brokerEpoch)) {
       // When the broker restarts very quickly, it is possible for this broker to receive request intended
       // for its previous generation so the broker should skip the stale request.
       info("Received update metadata request with broker epoch " +
-        s"${updateMetadataRequest.brokerEpoch()} smaller than the current broker epoch ${controller.brokerEpoch}")
-      sendResponseExemptThrottle(request, new UpdateMetadataResponse(Errors.STALE_BROKER_EPOCH))
+        s"${updateMetadataRequest.brokerEpoch} smaller than the current broker epoch ${controller.brokerEpoch}")
+      sendResponseExemptThrottle(request,
+        new UpdateMetadataResponse(new UpdateMetadataResponseData().setErrorCode(Errors.STALE_BROKER_EPOCH.code)))
     } else {
       val deletedPartitions = replicaManager.maybeUpdateMetadataCache(correlationId, updateMetadataRequest)
       if (deletedPartitions.nonEmpty)
         groupCoordinator.handleDeletedPartitions(deletedPartitions)
 
       if (adminManager.hasDelayedTopicOperations) {
-        updateMetadataRequest.partitionStates.keySet.asScala.map(_.topic).foreach { topic =>
-          adminManager.tryCompleteDelayedTopicOperations(topic)
+        updateMetadataRequest.partitionStates.asScala.foreach { partitionState =>
+          adminManager.tryCompleteDelayedTopicOperations(partitionState.topicName)
         }
       }
       quotas.clientQuotaCallback.foreach { callback =>
@@ -296,11 +287,13 @@ class KafkaApis(val requestChannel: RequestChannel,
         }
       }
       if (replicaManager.hasDelayedElectionOperations) {
-        updateMetadataRequest.partitionStates.asScala.foreach { case (tp, _) =>
+        updateMetadataRequest.partitionStates.asScala.foreach { partitionState =>
+          val tp = new TopicPartition(partitionState.topicName, partitionState.partitionIndex)
           replicaManager.tryCompleteElection(TopicPartitionOperationKey(tp))
         }
       }
-      sendResponseExemptThrottle(request, new UpdateMetadataResponse(Errors.NONE))
+      sendResponseExemptThrottle(request, new UpdateMetadataResponse(
+        new UpdateMetadataResponseData().setErrorCode(Errors.NONE.code)))
     }
   }
 
diff --git a/core/src/main/scala/kafka/server/MetadataCache.scala b/core/src/main/scala/kafka/server/MetadataCache.scala
index e0203478e0776..1ad0e41352e5a 100755
--- a/core/src/main/scala/kafka/server/MetadataCache.scala
+++ b/core/src/main/scala/kafka/server/MetadataCache.scala
@@ -28,10 +28,12 @@ import kafka.controller.StateChangeLogger
 import kafka.utils.CoreUtils._
 import kafka.utils.Logging
 import org.apache.kafka.common.internals.Topic
+import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataPartitionState
 import org.apache.kafka.common.{Cluster, Node, PartitionInfo, TopicPartition}
 import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.requests.{MetadataResponse, UpdateMetadataRequest}
+import org.apache.kafka.common.security.auth.SecurityProtocol
 
 
 /**
@@ -75,14 +77,14 @@ class MetadataCache(brokerId: Int) extends Logging {
     snapshot.partitionStates.get(topic).map { partitions =>
       partitions.map { case (partitionId, partitionState) =>
         val topicPartition = new TopicPartition(topic, partitionId.toInt)
-        val leaderBrokerId = partitionState.basePartitionState.leader
-        val leaderEpoch = partitionState.basePartitionState.leaderEpoch
+        val leaderBrokerId = partitionState.leader
+        val leaderEpoch = partitionState.leaderEpoch
         val maybeLeader = getAliveEndpoint(snapshot, leaderBrokerId, listenerName)
-        val replicas = partitionState.basePartitionState.replicas.asScala
+        val replicas = partitionState.replicas.asScala
         val replicaInfo = getEndpoints(snapshot, replicas, listenerName, errorUnavailableEndpoints)
         val offlineReplicaInfo = getEndpoints(snapshot, partitionState.offlineReplicas.asScala, listenerName, errorUnavailableEndpoints)
 
-        val isr = partitionState.basePartitionState.isr.asScala
+        val isr = partitionState.isr.asScala
         val isrInfo = getEndpoints(snapshot, isr, listenerName, errorUnavailableEndpoints)
         maybeLeader match {
           case None =>
@@ -148,7 +150,7 @@ class MetadataCache(brokerId: Int) extends Logging {
     snapshot.partitionStates.keySet
   }
 
-  private def getAllPartitions(snapshot: MetadataSnapshot): Map[TopicPartition, UpdateMetadataRequest.PartitionState] = {
+  private def getAllPartitions(snapshot: MetadataSnapshot): Map[TopicPartition, UpdateMetadataPartitionState] = {
     snapshot.partitionStates.flatMap { case (topic, partitionStates) =>
       partitionStates.map { case (partition, state ) => (new TopicPartition(topic, partition.toInt), state) }
     }.toMap
@@ -166,15 +168,15 @@ class MetadataCache(brokerId: Int) extends Logging {
     metadataSnapshot.aliveBrokers.values.toBuffer
   }
 
-  private def addOrUpdatePartitionInfo(partitionStates: mutable.AnyRefMap[String, mutable.LongMap[UpdateMetadataRequest.PartitionState]],
+  private def addOrUpdatePartitionInfo(partitionStates: mutable.AnyRefMap[String, mutable.LongMap[UpdateMetadataPartitionState]],
                                        topic: String,
                                        partitionId: Int,
-                                       stateInfo: UpdateMetadataRequest.PartitionState): Unit = {
+                                       stateInfo: UpdateMetadataPartitionState): Unit = {
     val infos = partitionStates.getOrElseUpdate(topic, mutable.LongMap())
     infos(partitionId) = stateInfo
   }
 
-  def getPartitionInfo(topic: String, partitionId: Int): Option[UpdateMetadataRequest.PartitionState] = {
+  def getPartitionInfo(topic: String, partitionId: Int): Option[UpdateMetadataPartitionState] = {
     metadataSnapshot.partitionStates.get(topic).flatMap(_.get(partitionId))
   }
 
@@ -184,7 +186,7 @@ class MetadataCache(brokerId: Int) extends Logging {
   def getPartitionLeaderEndpoint(topic: String, partitionId: Int, listenerName: ListenerName): Option[Node] = {
     val snapshot = metadataSnapshot
     snapshot.partitionStates.get(topic).flatMap(_.get(partitionId)) map { partitionInfo =>
-      val leaderId = partitionInfo.basePartitionState.leader
+      val leaderId = partitionInfo.leader
 
       snapshot.aliveNodes.get(leaderId) match {
         case Some(nodeMap) =>
@@ -197,8 +199,8 @@ class MetadataCache(brokerId: Int) extends Logging {
 
   def getPartitionReplicaEndpoints(tp: TopicPartition, listenerName: ListenerName): Map[Int, Node] = {
     val snapshot = metadataSnapshot
-    snapshot.partitionStates.get(tp.topic()).flatMap(_.get(tp.partition())).map { partitionInfo =>
-      val replicaIds = partitionInfo.basePartitionState.replicas
+    snapshot.partitionStates.get(tp.topic).flatMap(_.get(tp.partition)).map { partitionInfo =>
+      val replicaIds = partitionInfo.replicas
       replicaIds.asScala
         .map(replicaId => replicaId.intValue() -> {
           snapshot.aliveBrokers.get(replicaId.longValue()) match {
@@ -220,17 +222,17 @@ class MetadataCache(brokerId: Int) extends Logging {
     val nodes = snapshot.aliveNodes.map { case (id, nodes) => (id, nodes.get(listenerName).orNull) }
     def node(id: Integer): Node = nodes.get(id.toLong).orNull
     val partitions = getAllPartitions(snapshot)
-      .filter { case (_, state) => state.basePartitionState.leader != LeaderAndIsr.LeaderDuringDelete }
+      .filter { case (_, state) => state.leader != LeaderAndIsr.LeaderDuringDelete }
       .map { case (tp, state) =>
-        new PartitionInfo(tp.topic, tp.partition, node(state.basePartitionState.leader),
-          state.basePartitionState.replicas.asScala.map(node).toArray,
-          state.basePartitionState.isr.asScala.map(node).toArray,
+        new PartitionInfo(tp.topic, tp.partition, node(state.leader),
+          state.replicas.asScala.map(node).toArray,
+          state.isr.asScala.map(node).toArray,
           state.offlineReplicas.asScala.map(node).toArray)
       }
     val unauthorizedTopics = Collections.emptySet[String]
     val internalTopics = getAllTopics(snapshot).filter(Topic.isInternal).asJava
-    new Cluster(clusterId, nodes.values.filter(_ != null).toList.asJava,
-      partitions.toList.asJava,
+    new Cluster(clusterId, nodes.values.filter(_ != null).toBuffer.asJava,
+      partitions.toBuffer.asJava,
       unauthorizedTopics, internalTopics,
       snapshot.controllerId.map(id => node(id)).orNull)
   }
@@ -252,9 +254,10 @@ class MetadataCache(brokerId: Int) extends Logging {
         // move to `AnyRefMap`, which has comparable performance.
         val nodes = new java.util.HashMap[ListenerName, Node]
         val endPoints = new mutable.ArrayBuffer[EndPoint]
-        broker.endPoints.asScala.foreach { ep =>
-          endPoints += EndPoint(ep.host, ep.port, ep.listenerName, ep.securityProtocol)
-          nodes.put(ep.listenerName, new Node(broker.id, ep.host, ep.port))
+        broker.endpoints.asScala.foreach { ep =>
+          val listenerName = new ListenerName(ep.listener)
+          endPoints += new EndPoint(ep.host, ep.port, listenerName, SecurityProtocol.forId(ep.securityProtocol))
+          nodes.put(listenerName, new Node(broker.id, ep.host, ep.port))
         }
         aliveBrokers(broker.id) = Broker(broker.id, endPoints, Option(broker.rack))
         aliveNodes(broker.id) = nodes.asScala
@@ -266,20 +269,21 @@ class MetadataCache(brokerId: Int) extends Logging {
       }
 
       val deletedPartitions = new mutable.ArrayBuffer[TopicPartition]
-      if (updateMetadataRequest.partitionStates().isEmpty) {
+      if (!updateMetadataRequest.partitionStates.iterator.hasNext) {
         metadataSnapshot = MetadataSnapshot(metadataSnapshot.partitionStates, controllerId, aliveBrokers, aliveNodes)
       } else {
         //since kafka may do partial metadata updates, we start by copying the previous state
-        val partitionStates = new mutable.AnyRefMap[String, mutable.LongMap[UpdateMetadataRequest.PartitionState]](metadataSnapshot.partitionStates.size)
+        val partitionStates = new mutable.AnyRefMap[String, mutable.LongMap[UpdateMetadataPartitionState]](metadataSnapshot.partitionStates.size)
         metadataSnapshot.partitionStates.foreach { case (topic, oldPartitionStates) =>
-          val copy = new mutable.LongMap[UpdateMetadataRequest.PartitionState](oldPartitionStates.size)
+          val copy = new mutable.LongMap[UpdateMetadataPartitionState](oldPartitionStates.size)
           copy ++= oldPartitionStates
           partitionStates += (topic -> copy)
         }
-        updateMetadataRequest.partitionStates.asScala.foreach { case (tp, info) =>
+        updateMetadataRequest.partitionStates.asScala.foreach { info =>
           val controllerId = updateMetadataRequest.controllerId
           val controllerEpoch = updateMetadataRequest.controllerEpoch
-          if (info.basePartitionState.leader == LeaderAndIsr.LeaderDuringDelete) {
+          val tp = new TopicPartition(info.topicName, info.partitionIndex)
+          if (info.leader == LeaderAndIsr.LeaderDuringDelete) {
             removePartitionInfo(partitionStates, tp.topic, tp.partition)
             stateChangeLogger.trace(s"Deleted partition $tp from metadata cache in response to UpdateMetadata " +
               s"request sent by controller $controllerId epoch $controllerEpoch with correlation id $correlationId")
@@ -302,7 +306,8 @@ class MetadataCache(brokerId: Int) extends Logging {
 
   def contains(tp: TopicPartition): Boolean = getPartitionInfo(tp.topic, tp.partition).isDefined
 
-  private def removePartitionInfo(partitionStates: mutable.AnyRefMap[String, mutable.LongMap[UpdateMetadataRequest.PartitionState]], topic: String, partitionId: Int): Boolean = {
+  private def removePartitionInfo(partitionStates: mutable.AnyRefMap[String, mutable.LongMap[UpdateMetadataPartitionState]],
+                                  topic: String, partitionId: Int): Boolean = {
     partitionStates.get(topic).exists { infos =>
       infos.remove(partitionId)
       if (infos.isEmpty) partitionStates.remove(topic)
@@ -310,7 +315,7 @@ class MetadataCache(brokerId: Int) extends Logging {
     }
   }
 
-  case class MetadataSnapshot(partitionStates: mutable.AnyRefMap[String, mutable.LongMap[UpdateMetadataRequest.PartitionState]],
+  case class MetadataSnapshot(partitionStates: mutable.AnyRefMap[String, mutable.LongMap[UpdateMetadataPartitionState]],
                               controllerId: Option[Int],
                               aliveBrokers: mutable.LongMap[Broker],
                               aliveNodes: mutable.LongMap[collection.Map[ListenerName, Node]])
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index 9579b1f56ec38..8a74aa7f8908e 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -37,6 +37,9 @@ import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.Node
 import org.apache.kafka.common.errors._
 import org.apache.kafka.common.internals.Topic
+import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
+import org.apache.kafka.common.message.LeaderAndIsrResponseData
+import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.protocol.Errors
@@ -1144,10 +1147,12 @@ class ReplicaManager(val config: KafkaConfig,
   def becomeLeaderOrFollower(correlationId: Int,
                              leaderAndIsrRequest: LeaderAndIsrRequest,
                              onLeadershipChange: (Iterable[Partition], Iterable[Partition]) => Unit): LeaderAndIsrResponse = {
-    leaderAndIsrRequest.partitionStates.asScala.foreach { case (topicPartition, stateInfo) =>
-      stateChangeLogger.trace(s"Received LeaderAndIsr request $stateInfo " +
-        s"correlation id $correlationId from controller ${leaderAndIsrRequest.controllerId} " +
-        s"epoch ${leaderAndIsrRequest.controllerEpoch} for partition $topicPartition")
+    if (stateChangeLogger.isTraceEnabled) {
+      leaderAndIsrRequest.partitionStates.asScala.foreach { partitionState =>
+        stateChangeLogger.trace(s"Received LeaderAndIsr request $partitionState " +
+          s"correlation id $correlationId from controller ${leaderAndIsrRequest.controllerId} " +
+          s"epoch ${leaderAndIsrRequest.controllerEpoch}")
+      }
     }
     replicaStateChangeLock synchronized {
       if (leaderAndIsrRequest.controllerEpoch < controllerEpoch) {
@@ -1161,10 +1166,11 @@ class ReplicaManager(val config: KafkaConfig,
         controllerEpoch = leaderAndIsrRequest.controllerEpoch
 
         // First check partition's leader epoch
-        val partitionState = new mutable.HashMap[Partition, LeaderAndIsrRequest.PartitionState]()
+        val partitionStates = new mutable.HashMap[Partition, LeaderAndIsrPartitionState]()
         val newPartitions = new mutable.HashSet[Partition]
 
-        leaderAndIsrRequest.partitionStates.asScala.foreach { case (topicPartition, stateInfo) =>
+        leaderAndIsrRequest.partitionStates.asScala.foreach { partitionState =>
+          val topicPartition = new TopicPartition(partitionState.topicName, partitionState.partitionIndex)
           val partitionOpt = getPartition(topicPartition) match {
             case HostedPartition.Offline =>
               stateChangeLogger.warn(s"Ignoring LeaderAndIsr request from " +
@@ -1185,16 +1191,16 @@ class ReplicaManager(val config: KafkaConfig,
 
           partitionOpt.foreach { partition =>
             val currentLeaderEpoch = partition.getLeaderEpoch
-            val requestLeaderEpoch = stateInfo.basePartitionState.leaderEpoch
+            val requestLeaderEpoch = partitionState.leaderEpoch
             if (requestLeaderEpoch > currentLeaderEpoch) {
               // If the leader epoch is valid record the epoch of the controller that made the leadership decision.
               // This is useful while updating the isr to maintain the decision maker controller's epoch in the zookeeper path
-              if (stateInfo.basePartitionState.replicas.contains(localBrokerId))
-                partitionState.put(partition, stateInfo)
+              if (partitionState.replicas.contains(localBrokerId))
+                partitionStates.put(partition, partitionState)
               else {
                 stateChangeLogger.warn(s"Ignoring LeaderAndIsr request from controller $controllerId with " +
                   s"correlation id $correlationId epoch $controllerEpoch for partition $topicPartition as itself is not " +
-                  s"in assigned replica list ${stateInfo.basePartitionState.replicas.asScala.mkString(",")}")
+                  s"in assigned replica list ${partitionState.replicas.asScala.mkString(",")}")
                 responseMap.put(topicPartition, Errors.UNKNOWN_TOPIC_OR_PARTITION)
               }
             } else if (requestLeaderEpoch < currentLeaderEpoch) {
@@ -1214,10 +1220,10 @@ class ReplicaManager(val config: KafkaConfig,
           }
         }
 
-        val partitionsTobeLeader = partitionState.filter { case (_, stateInfo) =>
-          stateInfo.basePartitionState.leader == localBrokerId
+        val partitionsTobeLeader = partitionStates.filter { case (_, partitionState) =>
+          partitionState.leader == localBrokerId
         }
-        val partitionsToBeFollower = partitionState -- partitionsTobeLeader.keys
+        val partitionsToBeFollower = partitionStates -- partitionsTobeLeader.keys
 
         val highWatermarkCheckpoints = new LazyOffsetCheckpoints(this.highWatermarkCheckpoints)
         val partitionsBecomeLeader = if (partitionsTobeLeader.nonEmpty)
@@ -1244,7 +1250,8 @@ class ReplicaManager(val config: KafkaConfig,
         // remove metrics for brokers which are not followers of a topic
         leaderTopicSet.diff(followerTopicSet).foreach(brokerTopicStats.removeOldFollowerMetrics)
 
-        leaderAndIsrRequest.partitionStates.asScala.keys.foreach { topicPartition =>
+        leaderAndIsrRequest.partitionStates.asScala.foreach { partitionState =>
+          val topicPartition = new TopicPartition(partitionState.topicName, partitionState.partitionIndex)
           /*
            * If there is offline log directory, a Partition object may have been created by getOrCreatePartition()
            * before getOrCreateReplica() failed to create local replica due to KafkaStorageException.
@@ -1284,7 +1291,15 @@ class ReplicaManager(val config: KafkaConfig,
         replicaFetcherManager.shutdownIdleFetcherThreads()
         replicaAlterLogDirsManager.shutdownIdleFetcherThreads()
         onLeadershipChange(partitionsBecomeLeader, partitionsBecomeFollower)
-        new LeaderAndIsrResponse(Errors.NONE, responseMap.asJava)
+        val responsePartitions = responseMap.iterator.map { case (tp, error) =>
+          new LeaderAndIsrPartitionError()
+            .setTopicName(tp.topic)
+            .setPartitionIndex(tp.partition)
+            .setErrorCode(error.code)
+        }.toBuffer
+        new LeaderAndIsrResponse(new LeaderAndIsrResponseData()
+          .setErrorCode(Errors.NONE.code)
+          .setPartitionErrors(responsePartitions.asJava))
       }
     }
   }
@@ -1304,42 +1319,42 @@ class ReplicaManager(val config: KafkaConfig,
    */
   private def makeLeaders(controllerId: Int,
                           controllerEpoch: Int,
-                          partitionState: Map[Partition, LeaderAndIsrRequest.PartitionState],
+                          partitionStates: Map[Partition, LeaderAndIsrPartitionState],
                           correlationId: Int,
                           responseMap: mutable.Map[TopicPartition, Errors],
                           highWatermarkCheckpoints: OffsetCheckpoints): Set[Partition] = {
-    partitionState.keys.foreach { partition =>
+    partitionStates.keys.foreach { partition =>
       stateChangeLogger.trace(s"Handling LeaderAndIsr request correlationId $correlationId from " +
         s"controller $controllerId epoch $controllerEpoch starting the become-leader transition for " +
         s"partition ${partition.topicPartition}")
     }
 
-    for (partition <- partitionState.keys)
+    for (partition <- partitionStates.keys)
       responseMap.put(partition.topicPartition, Errors.NONE)
 
     val partitionsToMakeLeaders = mutable.Set[Partition]()
 
     try {
       // First stop fetchers for all the partitions
-      replicaFetcherManager.removeFetcherForPartitions(partitionState.keySet.map(_.topicPartition))
+      replicaFetcherManager.removeFetcherForPartitions(partitionStates.keySet.map(_.topicPartition))
       // Update the partition information to be the leader
-      partitionState.foreach{ case (partition, partitionStateInfo) =>
+      partitionStates.foreach { case (partition, partitionState) =>
         try {
-          if (partition.makeLeader(controllerId, partitionStateInfo, correlationId, highWatermarkCheckpoints)) {
+          if (partition.makeLeader(controllerId, partitionState, correlationId, highWatermarkCheckpoints)) {
             partitionsToMakeLeaders += partition
             stateChangeLogger.trace(s"Stopped fetchers as part of become-leader request from " +
               s"controller $controllerId epoch $controllerEpoch with correlation id $correlationId for partition ${partition.topicPartition} " +
-              s"(last update controller epoch ${partitionStateInfo.basePartitionState.controllerEpoch})")
+              s"(last update controller epoch ${partitionState.controllerEpoch})")
           } else
             stateChangeLogger.info(s"Skipped the become-leader state change after marking its " +
               s"partition as leader with correlation id $correlationId from controller $controllerId epoch $controllerEpoch for " +
-              s"partition ${partition.topicPartition} (last update controller epoch ${partitionStateInfo.basePartitionState.controllerEpoch}) " +
+              s"partition ${partition.topicPartition} (last update controller epoch ${partitionState.controllerEpoch}) " +
               s"since it is already the leader for the partition.")
         } catch {
           case e: KafkaStorageException =>
             stateChangeLogger.error(s"Skipped the become-leader state change with " +
               s"correlation id $correlationId from controller $controllerId epoch $controllerEpoch for partition ${partition.topicPartition} " +
-              s"(last update controller epoch ${partitionStateInfo.basePartitionState.controllerEpoch}) since " +
+              s"(last update controller epoch ${partitionState.controllerEpoch}) since " +
               s"the replica for the partition is offline due to disk error $e")
             val dirOpt = getLogDir(partition.topicPartition)
             error(s"Error while making broker the leader for partition $partition in dir $dirOpt", e)
@@ -1349,7 +1364,7 @@ class ReplicaManager(val config: KafkaConfig,
 
     } catch {
       case e: Throwable =>
-        partitionState.keys.foreach { partition =>
+        partitionStates.keys.foreach { partition =>
           stateChangeLogger.error(s"Error while processing LeaderAndIsr request correlationId $correlationId received " +
             s"from controller $controllerId epoch $controllerEpoch for partition ${partition.topicPartition}", e)
         }
@@ -1357,7 +1372,7 @@ class ReplicaManager(val config: KafkaConfig,
         throw e
     }
 
-    partitionState.keys.foreach { partition =>
+    partitionStates.keys.foreach { partition =>
       stateChangeLogger.trace(s"Completed LeaderAndIsr request correlationId $correlationId from controller $controllerId " +
         s"epoch $controllerEpoch for the become-leader transition for partition ${partition.topicPartition}")
     }
@@ -1385,14 +1400,14 @@ class ReplicaManager(val config: KafkaConfig,
    */
   private def makeFollowers(controllerId: Int,
                             controllerEpoch: Int,
-                            partitionStates: Map[Partition, LeaderAndIsrRequest.PartitionState],
+                            partitionStates: Map[Partition, LeaderAndIsrPartitionState],
                             correlationId: Int,
                             responseMap: mutable.Map[TopicPartition, Errors],
                             highWatermarkCheckpoints: OffsetCheckpoints) : Set[Partition] = {
     partitionStates.foreach { case (partition, partitionState) =>
       stateChangeLogger.trace(s"Handling LeaderAndIsr request correlationId $correlationId from controller $controllerId " +
         s"epoch $controllerEpoch starting the become-follower transition for partition ${partition.topicPartition} with leader " +
-        s"${partitionState.basePartitionState.leader}")
+        s"${partitionState.leader}")
     }
 
     for (partition <- partitionStates.keys)
@@ -1401,37 +1416,37 @@ class ReplicaManager(val config: KafkaConfig,
     val partitionsToMakeFollower: mutable.Set[Partition] = mutable.Set()
     try {
       // TODO: Delete leaders from LeaderAndIsrRequest
-      partitionStates.foreach { case (partition, partitionStateInfo) =>
-        val newLeaderBrokerId = partitionStateInfo.basePartitionState.leader
+      partitionStates.foreach { case (partition, partitionState) =>
+        val newLeaderBrokerId = partitionState.leader
         try {
           metadataCache.getAliveBrokers.find(_.id == newLeaderBrokerId) match {
             // Only change partition state when the leader is available
             case Some(_) =>
-              if (partition.makeFollower(controllerId, partitionStateInfo, correlationId, highWatermarkCheckpoints))
+              if (partition.makeFollower(controllerId, partitionState, correlationId, highWatermarkCheckpoints))
                 partitionsToMakeFollower += partition
               else
                 stateChangeLogger.info(s"Skipped the become-follower state change after marking its partition as " +
                   s"follower with correlation id $correlationId from controller $controllerId epoch $controllerEpoch " +
                   s"for partition ${partition.topicPartition} (last update " +
-                  s"controller epoch ${partitionStateInfo.basePartitionState.controllerEpoch}) " +
+                  s"controller epoch ${partitionState.controllerEpoch}) " +
                   s"since the new leader $newLeaderBrokerId is the same as the old leader")
             case None =>
               // The leader broker should always be present in the metadata cache.
               // If not, we should record the error message and abort the transition process for this partition
               stateChangeLogger.error(s"Received LeaderAndIsrRequest with correlation id $correlationId from " +
                 s"controller $controllerId epoch $controllerEpoch for partition ${partition.topicPartition} " +
-                s"(last update controller epoch ${partitionStateInfo.basePartitionState.controllerEpoch}) " +
+                s"(last update controller epoch ${partitionState.controllerEpoch}) " +
                 s"but cannot become follower since the new leader $newLeaderBrokerId is unavailable.")
               // Create the local replica even if the leader is unavailable. This is required to ensure that we include
               // the partition's high watermark in the checkpoint file (see KAFKA-1647)
-              partition.createLogIfNotExists(localBrokerId, isNew = partitionStateInfo.isNew, isFutureReplica = false,
+              partition.createLogIfNotExists(localBrokerId, isNew = partitionState.isNew, isFutureReplica = false,
                 highWatermarkCheckpoints)
           }
         } catch {
           case e: KafkaStorageException =>
             stateChangeLogger.error(s"Skipped the become-follower state change with correlation id $correlationId from " +
               s"controller $controllerId epoch $controllerEpoch for partition ${partition.topicPartition} " +
-              s"(last update controller epoch ${partitionStateInfo.basePartitionState.controllerEpoch}) with leader " +
+              s"(last update controller epoch ${partitionState.controllerEpoch}) with leader " +
               s"$newLeaderBrokerId since the replica for the partition is offline due to disk error $e")
             val dirOpt = getLogDir(partition.topicPartition)
             error(s"Error while making broker the follower for partition $partition with leader " +
@@ -1444,7 +1459,7 @@ class ReplicaManager(val config: KafkaConfig,
       partitionsToMakeFollower.foreach { partition =>
         stateChangeLogger.trace(s"Stopped fetchers as part of become-follower request from controller $controllerId " +
           s"epoch $controllerEpoch with correlation id $correlationId for partition ${partition.topicPartition} with leader " +
-          s"${partitionStates(partition).basePartitionState.leader}")
+          s"${partitionStates(partition).leader}")
       }
 
       partitionsToMakeFollower.foreach { partition =>
@@ -1456,14 +1471,14 @@ class ReplicaManager(val config: KafkaConfig,
       partitionsToMakeFollower.foreach { partition =>
         stateChangeLogger.trace(s"Truncated logs and checkpointed recovery boundaries for partition " +
           s"${partition.topicPartition} as part of become-follower request with correlation id $correlationId from " +
-          s"controller $controllerId epoch $controllerEpoch with leader ${partitionStates(partition).basePartitionState.leader}")
+          s"controller $controllerId epoch $controllerEpoch with leader ${partitionStates(partition).leader}")
       }
 
       if (isShuttingDown.get()) {
         partitionsToMakeFollower.foreach { partition =>
           stateChangeLogger.trace(s"Skipped the adding-fetcher step of the become-follower state " +
             s"change with correlation id $correlationId from controller $controllerId epoch $controllerEpoch for " +
-            s"partition ${partition.topicPartition} with leader ${partitionStates(partition).basePartitionState.leader} " +
+            s"partition ${partition.topicPartition} with leader ${partitionStates(partition).leader} " +
             "since it is shutting down")
         }
       } else {
@@ -1493,7 +1508,7 @@ class ReplicaManager(val config: KafkaConfig,
     partitionStates.keys.foreach { partition =>
       stateChangeLogger.trace(s"Completed LeaderAndIsr request correlationId $correlationId from controller $controllerId " +
         s"epoch $controllerEpoch for the become-follower transition for partition ${partition.topicPartition} with leader " +
-        s"${partitionStates(partition).basePartitionState.leader}")
+        s"${partitionStates(partition).leader}")
     }
 
     partitionsToMakeFollower
@@ -1711,7 +1726,7 @@ class ReplicaManager(val config: KafkaConfig,
       if (expectedLeaders.nonEmpty) {
         val watchKeys = expectedLeaders.iterator.map {
           case (tp, _) => TopicPartitionOperationKey(tp)
-        }.toIndexedSeq
+        }.toBuffer
 
         delayedElectLeaderPurgatory.tryCompleteElseWatch(
           new DelayedElectLeader(
diff --git a/core/src/main/scala/kafka/zk/KafkaZkClient.scala b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
index 73f83fe71f0b6..864dcaba63952 100644
--- a/core/src/main/scala/kafka/zk/KafkaZkClient.scala
+++ b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
@@ -1656,7 +1656,7 @@ class KafkaZkClient private[zk] (zooKeeperClient: ZooKeeperClient, isSecure: Boo
   private def getTopicConfigs(topics: Set[String]): Seq[GetDataResponse] = {
     val getDataRequests: Seq[GetDataRequest] = topics.iterator.map { topic =>
       GetDataRequest(ConfigEntityZNode.path(ConfigType.Topic, topic), ctx = Some(topic))
-    }.toIndexedSeq
+    }.toBuffer
 
     retryRequestsUntilConnected(getDataRequests)
   }
diff --git a/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala b/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala
index 4853424ed35c1..fcd1402126ce9 100644
--- a/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala
@@ -334,7 +334,7 @@ abstract class AbstractConsumerTest extends BaseRequestTest {
     @volatile var thrownException: Option[Throwable] = None
     @volatile var receivedMessages = 0
 
-    @volatile private var partitionAssignment: mutable.Set[TopicPartition] = new mutable.HashSet[TopicPartition]()
+    private val partitionAssignment = mutable.Set[TopicPartition]()
     @volatile private var subscriptionChanged = false
     private var topicsSubscription = topicsToSubscribe
 
@@ -424,7 +424,7 @@ abstract class AbstractConsumerTest extends BaseRequestTest {
     }
 
     // make sure that sum of all partitions to all consumers equals total number of partitions
-    val totalPartitionsInAssignments = (0 /: assignments) (_ + _.size)
+    val totalPartitionsInAssignments = assignments.foldLeft(0)(_ + _.size)
     if (totalPartitionsInAssignments != partitions.size) {
       // either same partitions got assigned to more than one consumer or some
       // partitions were not assigned
@@ -434,7 +434,7 @@ abstract class AbstractConsumerTest extends BaseRequestTest {
     // The above checks could miss the case where one or more partitions were assigned to more
     // than one consumer and the same number of partitions were missing from assignments.
     // Make sure that all unique assignments are the same as 'partitions'
-    val uniqueAssignedPartitions = (Set[TopicPartition]() /: assignments) (_ ++ _)
+    val uniqueAssignedPartitions = assignments.foldLeft(Set.empty[TopicPartition])(_ ++ _)
     uniqueAssignedPartitions == partitions
   }
 
diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index 2841595137693..2b718584649ac 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -1306,8 +1306,8 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
             } catch {
               case e: ExecutionException =>
                 assertTrue(e.getCause.isInstanceOf[UnknownMemberIdException])
-              case _: Throwable =>
-                fail("Should have caught exception in getting member future")
+              case t: Throwable =>
+                fail(s"Should have caught exception in getting member future: $t")
             }
           }
 
@@ -1341,8 +1341,8 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
             } catch {
               case e: ExecutionException =>
                 assertTrue(e.getCause.isInstanceOf[UnknownMemberIdException])
-              case _: Throwable =>
-                fail("Should have caught exception in getting member future")
+              case t: Throwable =>
+                fail(s"Should have caught exception in getting member future: $t")
             }
           }
 
diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
index 01021fce16a31..38fe16598f704 100644
--- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
@@ -22,7 +22,7 @@ import java.time.Duration
 import kafka.admin.ConsumerGroupCommand.{ConsumerGroupCommandOptions, ConsumerGroupService}
 import kafka.log.LogConfig
 import kafka.network.SocketServer
-import kafka.security.auth.{ResourceType => AuthResourceType, SimpleAclAuthorizer, Topic}
+import kafka.security.auth.{SimpleAclAuthorizer, Topic, ResourceType => AuthResourceType}
 import kafka.security.authorizer.AuthorizerUtils.WildcardHost
 import kafka.server.{BaseRequestTest, KafkaConfig}
 import kafka.utils.TestUtils
@@ -49,11 +49,15 @@ import org.apache.kafka.common.message.HeartbeatRequestData
 import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData
 import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData.{AlterConfigsResource, AlterableConfig, AlterableConfigCollection}
 import org.apache.kafka.common.message.JoinGroupRequestData
+import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection
+import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
+import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
 import org.apache.kafka.common.message.ListPartitionReassignmentsRequestData
 import org.apache.kafka.common.message.OffsetCommitRequestData
 import org.apache.kafka.common.message.SyncGroupRequestData
 import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection
 import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
+import org.apache.kafka.common.message.UpdateMetadataRequestData.{UpdateMetadataBroker, UpdateMetadataEndpoint, UpdateMetadataPartitionState}
 import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
 import org.apache.kafka.common.record.{CompressionType, MemoryRecords, RecordBatch, Records, SimpleRecord}
@@ -200,8 +204,10 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
     ApiKeys.HEARTBEAT -> ((resp: HeartbeatResponse) => resp.error),
     ApiKeys.LEAVE_GROUP -> ((resp: LeaveGroupResponse) => resp.error),
     ApiKeys.DELETE_GROUPS -> ((resp: DeleteGroupsResponse) => resp.get(group)),
-    ApiKeys.LEADER_AND_ISR -> ((resp: requests.LeaderAndIsrResponse) => resp.responses.asScala.find(_._1 == tp).get._2),
-    ApiKeys.STOP_REPLICA -> ((resp: requests.StopReplicaResponse) => resp.responses.asScala.find(_._1 == tp).get._2),
+    ApiKeys.LEADER_AND_ISR -> ((resp: requests.LeaderAndIsrResponse) => Errors.forCode(
+      resp.partitions.asScala.find(p => p.topicName == tp.topic && p.partitionIndex == tp.partition).get.errorCode)),
+    ApiKeys.STOP_REPLICA -> ((resp: requests.StopReplicaResponse) => Errors.forCode(
+      resp.partitionErrors.asScala.find(pe => pe.topicName == tp.topic && pe.partitionIndex == tp.partition).get.errorCode)),
     ApiKeys.CONTROLLED_SHUTDOWN -> ((resp: requests.ControlledShutdownResponse) => resp.error),
     ApiKeys.CREATE_TOPICS -> ((resp: CreateTopicsResponse) => Errors.forCode(resp.data.topics.find(createTopic).errorCode())),
     ApiKeys.DELETE_TOPICS -> ((resp: requests.DeleteTopicsResponse) => Errors.forCode(resp.data.responses.find(deleteTopic).errorCode())),
@@ -353,14 +359,25 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   }
 
   private def createUpdateMetadataRequest = {
-    val partitionState = Map(tp -> new UpdateMetadataRequest.PartitionState(
-      Int.MaxValue, brokerId, Int.MaxValue, List(brokerId).asJava, 2, Seq(brokerId).asJava, Seq.empty[Integer].asJava)).asJava
+    val partitionStates = Seq(new UpdateMetadataPartitionState()
+      .setTopicName(tp.topic)
+      .setPartitionIndex(tp.partition)
+      .setControllerEpoch(Int.MaxValue)
+      .setLeader(brokerId)
+      .setLeaderEpoch(Int.MaxValue)
+      .setIsr(List(brokerId).asJava)
+      .setZkVersion(2)
+      .setReplicas(Seq(brokerId).asJava)).asJava
     val securityProtocol = SecurityProtocol.PLAINTEXT
-    val brokers = Set(new requests.UpdateMetadataRequest.Broker(brokerId,
-      Seq(new requests.UpdateMetadataRequest.EndPoint("localhost", 0, securityProtocol,
-        ListenerName.forSecurityProtocol(securityProtocol))).asJava, null)).asJava
+    val brokers = Seq(new UpdateMetadataBroker()
+      .setId(brokerId)
+      .setEndpoints(Seq(new UpdateMetadataEndpoint()
+        .setHost("localhost")
+        .setPort(0)
+        .setSecurityProtocol(securityProtocol.id)
+        .setListener(ListenerName.forSecurityProtocol(securityProtocol).value)).asJava)).asJava
     val version = ApiKeys.UPDATE_METADATA.latestVersion
-    new requests.UpdateMetadataRequest.Builder(version, brokerId, Int.MaxValue, Long.MaxValue, partitionState, brokers).build()
+    new requests.UpdateMetadataRequest.Builder(version, brokerId, Int.MaxValue, Long.MaxValue, partitionStates, brokers).build()
   }
 
   private def createJoinGroupRequest = {
@@ -442,7 +459,16 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   private def leaderAndIsrRequest = {
     new requests.LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, brokerId, Int.MaxValue, Long.MaxValue,
-      Map(tp -> new LeaderAndIsrRequest.PartitionState(Int.MaxValue, brokerId, Int.MaxValue, List(brokerId).asJava, 2, Seq(brokerId).asJava, false)).asJava,
+      Seq(new LeaderAndIsrPartitionState()
+        .setTopicName(tp.topic)
+        .setPartitionIndex(tp.partition)
+        .setControllerEpoch(Int.MaxValue)
+        .setLeader(brokerId)
+        .setLeaderEpoch(Int.MaxValue)
+        .setIsr(List(brokerId).asJava)
+        .setZkVersion(2)
+        .setReplicas(Seq(brokerId).asJava)
+        .setIsNew(false)).asJava,
       Set(new Node(brokerId, "localhost", 0)).asJava).build()
   }
 
@@ -532,7 +558,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
       List(new ListPartitionReassignmentsRequestData.ListPartitionReassignmentsTopics()
         .setName(topic)
         .setPartitionIndexes(
-          List(new Integer(tp.partition)).asJava
+          List(Integer.valueOf(tp.partition)).asJava
         )).asJava
     )
   ).build()
@@ -569,12 +595,12 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
       ApiKeys.CREATE_PARTITIONS -> createPartitionsRequest,
       ApiKeys.ADD_PARTITIONS_TO_TXN -> addPartitionsToTxnRequest,
       ApiKeys.ADD_OFFSETS_TO_TXN -> addOffsetsToTxnRequest,
-      // Check StopReplica last since some APIs depend on replica availability
-      ApiKeys.STOP_REPLICA -> stopReplicaRequest,
       ApiKeys.ELECT_LEADERS -> electLeadersRequest,
       ApiKeys.INCREMENTAL_ALTER_CONFIGS -> incrementalAlterConfigsRequest,
       ApiKeys.ALTER_PARTITION_REASSIGNMENTS -> alterPartitionReassignmentsRequest,
-      ApiKeys.LIST_PARTITION_REASSIGNMENTS -> listPartitionReassignmentsRequest
+      ApiKeys.LIST_PARTITION_REASSIGNMENTS -> listPartitionReassignmentsRequest,
+      // Check StopReplica last since some APIs depend on replica availability
+      ApiKeys.STOP_REPLICA -> stopReplicaRequest
     )
 
     for ((key, request) <- requestKeyToRequest) {
diff --git a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
index f58046ad48917..acab29bd161b9 100644
--- a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
+++ b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
@@ -260,7 +260,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
     val SslKeystorePasswordVal = "${file:ssl.keystore.password:password}"
 
     val configPrefix = listenerPrefix(SecureExternal)
-    var brokerConfigs = describeConfig(adminClients.head, servers).entries.asScala
+    val brokerConfigs = describeConfig(adminClients.head, servers).entries.asScala
     // the following are values before updated
     assertTrue("Initial value of polling interval", brokerConfigs.find(_.name == TestMetricsReporter.PollingIntervalProp) == None)
     assertTrue("Initial value of ssl truststore type", brokerConfigs.find(_.name == configPrefix+KafkaConfig.SslTruststoreTypeProp) == None)
diff --git a/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala b/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala
index 295fc3bd87fd2..37032f17501a8 100644
--- a/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala
@@ -93,8 +93,8 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
     debug(s"Starting server $targetServer now that a non-preferred replica is leader")
     servers(targetServer).startup()
     TestUtils.waitUntilTrue(() => servers.forall { server =>
-      server.metadataCache.getPartitionInfo(partition.topic(), partition.partition()).exists { partitionState =>
-        partitionState.basePartitionState.isr.contains(targetServer)
+      server.metadataCache.getPartitionInfo(partition.topic, partition.partition).exists { partitionState =>
+        partitionState.isr.contains(targetServer)
       }
     },
       s"Replicas for partition $partition not created")
@@ -106,8 +106,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit
 
   private def awaitLeader(topicPartition: TopicPartition, timeoutMs: Long = test.TestUtils.DEFAULT_MAX_WAIT_MS): Int = {
     TestUtils.awaitValue(() => {
-      servers.head.metadataCache.getPartitionInfo(topicPartition.topic, topicPartition.partition)
-          .map(_.basePartitionState.leader)
+      servers.head.metadataCache.getPartitionInfo(topicPartition.topic, topicPartition.partition).map(_.leader)
     }, s"Timed out waiting to find current leader of $topicPartition", timeoutMs)
   }
 
diff --git a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
index aeb9768fe735d..8e812ebd00318 100644
--- a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
+++ b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
@@ -32,11 +32,12 @@ import kafka.server.checkpoints.OffsetCheckpoints
 import kafka.utils._
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.{ApiException, OffsetNotAvailableException, ReplicaNotAvailableException}
+import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.record.FileRecords.TimestampAndOffset
 import org.apache.kafka.common.utils.Utils
 import org.apache.kafka.common.record._
-import org.apache.kafka.common.requests.{EpochEndOffset, IsolationLevel, LeaderAndIsrRequest, ListOffsetRequest}
+import org.apache.kafka.common.requests.{EpochEndOffset, IsolationLevel, ListOffsetRequest}
 import org.junit.{After, Before, Test}
 import org.junit.Assert._
 import org.mockito.Mockito.{doNothing, mock, when}
@@ -482,9 +483,14 @@ class PartitionTest {
       new SimpleRecord(20,"k4".getBytes, "v2".getBytes),
       new SimpleRecord(21,"k5".getBytes, "v3".getBytes)))
 
-    val leaderState = new LeaderAndIsrRequest.PartitionState(
-      controllerEpoch, leader, leaderEpoch, isr, 1, replicas.map(Int.box).asJava, true
-    )
+    val leaderState = new LeaderAndIsrPartitionState()
+      .setControllerEpoch(controllerEpoch)
+      .setLeader(leader)
+      .setLeaderEpoch(leaderEpoch)
+      .setIsr(isr)
+      .setZkVersion(1)
+      .setReplicas(replicas.map(Int.box).asJava)
+      .setIsNew(true)
 
     assertTrue("Expected first makeLeader() to return 'leader changed'",
       partition.makeLeader(controllerId, leaderState, 0, offsetCheckpoints))
@@ -552,15 +558,27 @@ class PartitionTest {
     assertEquals(Right(None), fetchOffsetsForTimestamp(30, Some(IsolationLevel.READ_UNCOMMITTED)))
 
     // Make into a follower
-    val followerState = new LeaderAndIsrRequest.PartitionState(
-      controllerEpoch, follower2, leaderEpoch + 1, isr, 4, replicas.map(Int.box).asJava, false
-    )
+    val followerState = new LeaderAndIsrPartitionState()
+      .setControllerEpoch(controllerEpoch)
+      .setLeader(follower2)
+      .setLeaderEpoch(leaderEpoch + 1)
+      .setIsr(isr)
+      .setZkVersion(4)
+      .setReplicas(replicas.map(Int.box).asJava)
+      .setIsNew(false)
+
     assertTrue(partition.makeFollower(controllerId, followerState, 1, offsetCheckpoints))
 
     // Back to leader, this resets the startLogOffset for this epoch (to 2), we're now in the fault condition
-    val newLeaderState = new LeaderAndIsrRequest.PartitionState(
-      controllerEpoch, leader, leaderEpoch + 2, isr, 5, replicas.map(Int.box).asJava, false
-    )
+    val newLeaderState = new LeaderAndIsrPartitionState()
+      .setControllerEpoch(controllerEpoch)
+      .setLeader(leader)
+      .setLeaderEpoch(leaderEpoch + 2)
+      .setIsr(isr)
+      .setZkVersion(5)
+      .setReplicas(replicas.map(Int.box).asJava)
+      .setIsNew(false)
+
     assertTrue(partition.makeLeader(controllerId, newLeaderState, 2, offsetCheckpoints))
 
     // Try to get offsets as a client
@@ -633,13 +651,25 @@ class PartitionTest {
 
     if (isLeader) {
       assertTrue("Expected become leader transition to succeed",
-        partition.makeLeader(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
-          leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
+        partition.makeLeader(controllerId, new LeaderAndIsrPartitionState()
+          .setControllerEpoch(controllerEpoch)
+          .setLeader(brokerId)
+          .setLeaderEpoch(leaderEpoch)
+          .setIsr(isr)
+          .setZkVersion(1)
+          .setReplicas(replicas)
+          .setIsNew(true), 0, offsetCheckpoints))
       assertEquals(leaderEpoch, partition.getLeaderEpoch)
     } else {
       assertTrue("Expected become follower transition to succeed",
-        partition.makeFollower(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId + 1,
-          leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
+        partition.makeFollower(controllerId, new LeaderAndIsrPartitionState()
+          .setControllerEpoch(controllerEpoch)
+          .setLeader(brokerId + 1)
+          .setLeaderEpoch(leaderEpoch)
+          .setIsr(isr)
+          .setZkVersion(1)
+          .setReplicas(replicas)
+          .setIsNew(true), 0, offsetCheckpoints))
       assertEquals(leaderEpoch, partition.getLeaderEpoch)
       assertEquals(None, partition.leaderLogIfLocal)
     }
@@ -710,8 +740,14 @@ class PartitionTest {
     partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
 
     assertTrue("Expected become leader transition to succeed",
-      partition.makeLeader(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
-        leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
+      partition.makeLeader(controllerId, new LeaderAndIsrPartitionState()
+        .setControllerEpoch(controllerEpoch)
+        .setLeader(brokerId)
+        .setLeaderEpoch(leaderEpoch)
+        .setIsr(isr)
+        .setZkVersion(1)
+        .setReplicas(replicas)
+        .setIsNew(true), 0, offsetCheckpoints))
     assertEquals(leaderEpoch, partition.getLeaderEpoch)
 
     val records = createTransactionalRecords(List(
@@ -773,19 +809,36 @@ class PartitionTest {
   @Test
   def testMakeFollowerWithNoLeaderIdChange(): Unit = {
     // Start off as follower
-    var partitionStateInfo = new LeaderAndIsrRequest.PartitionState(0, 1, 1,
-      List[Integer](0, 1, 2, brokerId).asJava, 1, List[Integer](0, 1, 2, brokerId).asJava, false)
-    partition.makeFollower(0, partitionStateInfo, 0, offsetCheckpoints)
+    var partitionState = new LeaderAndIsrPartitionState()
+      .setControllerEpoch(0)
+      .setLeader(1)
+      .setLeaderEpoch(1)
+      .setIsr(List[Integer](0, 1, 2, brokerId).asJava)
+      .setZkVersion(1)
+      .setReplicas(List[Integer](0, 1, 2, brokerId).asJava)
+      .setIsNew(false)
+    partition.makeFollower(0, partitionState, 0, offsetCheckpoints)
 
     // Request with same leader and epoch increases by only 1, do become-follower steps
-    partitionStateInfo = new LeaderAndIsrRequest.PartitionState(0, 1, 4,
-      List[Integer](0, 1, 2, brokerId).asJava, 1, List[Integer](0, 1, 2, brokerId).asJava, false)
-    assertTrue(partition.makeFollower(0, partitionStateInfo, 2, offsetCheckpoints))
+    partitionState = new LeaderAndIsrPartitionState()
+      .setControllerEpoch(0)
+      .setLeader(1)
+      .setLeaderEpoch(4)
+      .setIsr(List[Integer](0, 1, 2, brokerId).asJava)
+      .setZkVersion(1)
+      .setReplicas(List[Integer](0, 1, 2, brokerId).asJava)
+      .setIsNew(false)
+    assertTrue(partition.makeFollower(0, partitionState, 2, offsetCheckpoints))
 
     // Request with same leader and same epoch, skip become-follower steps
-    partitionStateInfo = new LeaderAndIsrRequest.PartitionState(0, 1, 4,
-      List[Integer](0, 1, 2, brokerId).asJava, 1, List[Integer](0, 1, 2, brokerId).asJava, false)
-    assertFalse(partition.makeFollower(0, partitionStateInfo, 2, offsetCheckpoints))
+    partitionState = new LeaderAndIsrPartitionState()
+      .setControllerEpoch(0)
+      .setLeader(1)
+      .setLeaderEpoch(4)
+      .setIsr(List[Integer](0, 1, 2, brokerId).asJava)
+      .setZkVersion(1)
+      .setReplicas(List[Integer](0, 1, 2, brokerId).asJava)
+    assertFalse(partition.makeFollower(0, partitionState, 2, offsetCheckpoints))
   }
 
   @Test
@@ -806,7 +859,14 @@ class PartitionTest {
     val batch3 = TestUtils.records(records = List(new SimpleRecord("k6".getBytes, "v1".getBytes),
                                                   new SimpleRecord("k7".getBytes, "v2".getBytes)))
 
-    val leaderState = new LeaderAndIsrRequest.PartitionState(controllerEpoch, leader, leaderEpoch, isr, 1, replicas, true)
+    val leaderState = new LeaderAndIsrPartitionState()
+      .setControllerEpoch(controllerEpoch)
+      .setLeader(leader)
+      .setLeaderEpoch(leaderEpoch)
+      .setIsr(isr)
+      .setZkVersion(1)
+      .setReplicas(replicas)
+      .setIsNew(true)
     assertTrue("Expected first makeLeader() to return 'leader changed'",
                partition.makeLeader(controllerId, leaderState, 0, offsetCheckpoints))
     assertEquals("Current leader epoch", leaderEpoch, partition.getLeaderEpoch)
@@ -834,12 +894,24 @@ class PartitionTest {
     assertEquals("Expected leader's HW", lastOffsetOfFirstBatch, partition.log.get.highWatermark)
 
     // current leader becomes follower and then leader again (without any new records appended)
-    val followerState = new LeaderAndIsrRequest.PartitionState(controllerEpoch, follower2, leaderEpoch + 1, isr, 1,
-      replicas, false)
+    val followerState = new LeaderAndIsrPartitionState()
+      .setControllerEpoch(controllerEpoch)
+      .setLeader(follower2)
+      .setLeaderEpoch(leaderEpoch + 1)
+      .setIsr(isr)
+      .setZkVersion(1)
+      .setReplicas(replicas)
+      .setIsNew(false)
     partition.makeFollower(controllerId, followerState, 1, offsetCheckpoints)
 
-    val newLeaderState = new LeaderAndIsrRequest.PartitionState(controllerEpoch, leader, leaderEpoch + 2, isr, 1,
-      replicas, false)
+    val newLeaderState = new LeaderAndIsrPartitionState()
+      .setControllerEpoch(controllerEpoch)
+      .setLeader(leader)
+      .setLeaderEpoch(leaderEpoch + 2)
+      .setIsr(isr)
+      .setZkVersion(1)
+      .setReplicas(replicas)
+      .setIsNew(false)
     assertTrue("Expected makeLeader() to return 'leader changed' after makeFollower()",
                partition.makeLeader(controllerEpoch, newLeaderState, 2, offsetCheckpoints))
     val currentLeaderEpochStartOffset = partition.localLogOrException.logEndOffset
@@ -904,8 +976,14 @@ class PartitionTest {
         })
 
       partition.setLog(log, isFutureLog = false)
-      val leaderState = new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
-        leaderEpoch, isr, 1, replicaIds, true)
+      val leaderState = new LeaderAndIsrPartitionState()
+        .setControllerEpoch(controllerEpoch)
+        .setLeader(brokerId)
+        .setLeaderEpoch(leaderEpoch)
+        .setIsr(isr)
+        .setZkVersion(1)
+        .setReplicas(replicaIds)
+        .setIsNew(true)
       partition.makeLeader(controllerId, leaderState, 0, offsetCheckpoints)
       partitions += partition
     }
@@ -989,7 +1067,14 @@ class PartitionTest {
 
     assertFalse(partition.isAtMinIsr)
     // Make isr set to only have leader to trigger AtMinIsr (default min isr config is 1)
-    val leaderState = new LeaderAndIsrRequest.PartitionState(controllerEpoch, leader, leaderEpoch, isr, 1, replicas, true)
+    val leaderState = new LeaderAndIsrPartitionState()
+      .setControllerEpoch(controllerEpoch)
+      .setLeader(leader)
+      .setLeaderEpoch(leaderEpoch)
+      .setIsr(isr)
+      .setZkVersion(1)
+      .setReplicas(replicas)
+      .setIsNew(true)
     partition.makeLeader(controllerId, leaderState, 0, offsetCheckpoints)
     assertTrue(partition.isAtMinIsr)
   }
@@ -1012,8 +1097,18 @@ class PartitionTest {
 
     val initializeTimeMs = time.milliseconds()
     assertTrue("Expected become leader transition to succeed",
-      partition.makeLeader(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
-        leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
+      partition.makeLeader(
+        controllerId,
+        new LeaderAndIsrPartitionState()
+          .setControllerEpoch(controllerEpoch)
+          .setLeader(brokerId)
+          .setLeaderEpoch(leaderEpoch)
+          .setIsr(isr)
+          .setZkVersion(1)
+          .setReplicas(replicas)
+          .setIsNew(true),
+        0,
+        offsetCheckpoints))
 
     val remoteReplica = partition.getReplica(remoteBrokerId).get
     assertEquals(initializeTimeMs, remoteReplica.lastCaughtUpTimeMs)
@@ -1065,11 +1160,16 @@ class PartitionTest {
       "Expected become leader transition to succeed",
       partition.makeLeader(
         controllerId,
-        new LeaderAndIsrRequest.PartitionState(
-          controllerEpoch, brokerId, leaderEpoch, isr, 1, replicas.map(Int.box).asJava, true),
+        new LeaderAndIsrPartitionState()
+          .setControllerEpoch(controllerEpoch)
+          .setLeader(brokerId)
+          .setLeaderEpoch(leaderEpoch)
+          .setIsr(isr)
+          .setZkVersion(1)
+          .setReplicas(replicas.map(Int.box).asJava)
+          .setIsNew(true),
         0,
-        offsetCheckpoints
-      )
+        offsetCheckpoints)
     )
     assertEquals(Set(brokerId), partition.inSyncReplicaIds)
 
@@ -1122,8 +1222,18 @@ class PartitionTest {
 
     partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
     assertTrue("Expected become leader transition to succeed",
-      partition.makeLeader(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
-        leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
+      partition.makeLeader(
+        controllerId,
+        new LeaderAndIsrPartitionState()
+          .setControllerEpoch(controllerEpoch)
+          .setLeader(brokerId)
+          .setLeaderEpoch(leaderEpoch)
+          .setIsr(isr)
+          .setZkVersion(1)
+          .setReplicas(replicas)
+          .setIsNew(true),
+        0,
+        offsetCheckpoints))
     assertEquals(Set(brokerId), partition.inSyncReplicaIds)
 
     val remoteReplica = partition.getReplica(remoteBrokerId).get
@@ -1170,11 +1280,16 @@ class PartitionTest {
       "Expected become leader transition to succeed",
       partition.makeLeader(
         controllerId,
-        new LeaderAndIsrRequest.PartitionState(
-          controllerEpoch, brokerId, leaderEpoch, isr, 1, replicas.map(Int.box).asJava, true),
+        new LeaderAndIsrPartitionState()
+          .setControllerEpoch(controllerEpoch)
+          .setLeader(brokerId)
+          .setLeaderEpoch(leaderEpoch)
+          .setIsr(isr)
+          .setZkVersion(1)
+          .setReplicas(replicas.map(Int.box).asJava)
+          .setIsNew(true),
         0,
-        offsetCheckpoints
-      )
+        offsetCheckpoints)
     )
     assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicaIds)
     assertEquals(0L, partition.localLogOrException.highWatermark)
@@ -1222,11 +1337,16 @@ class PartitionTest {
       "Expected become leader transition to succeed",
       partition.makeLeader(
         controllerId,
-        new LeaderAndIsrRequest.PartitionState(
-          controllerEpoch, brokerId, leaderEpoch, isr, 1, replicas.map(Int.box).asJava, true),
+        new LeaderAndIsrPartitionState()
+          .setControllerEpoch(controllerEpoch)
+          .setLeader(brokerId)
+          .setLeaderEpoch(leaderEpoch)
+          .setIsr(isr)
+          .setZkVersion(1)
+          .setReplicas(replicas.map(Int.box).asJava)
+          .setIsNew(true),
         0,
-        offsetCheckpoints
-      )
+        offsetCheckpoints)
     )
     assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicaIds)
     assertEquals(0L, partition.localLogOrException.highWatermark)
@@ -1289,11 +1409,16 @@ class PartitionTest {
       "Expected become leader transition to succeed",
       partition.makeLeader(
         controllerId,
-        new LeaderAndIsrRequest.PartitionState(
-          controllerEpoch, brokerId, leaderEpoch, isr, 1, replicas.map(Int.box).asJava, true),
+        new LeaderAndIsrPartitionState()
+          .setControllerEpoch(controllerEpoch)
+          .setLeader(brokerId)
+          .setLeaderEpoch(leaderEpoch)
+          .setIsr(isr)
+          .setZkVersion(1)
+          .setReplicas(replicas.map(Int.box).asJava)
+          .setIsNew(true),
         0,
-        offsetCheckpoints
-      )
+        offsetCheckpoints)
     )
     assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicaIds)
     assertEquals(0L, partition.localLogOrException.highWatermark)
@@ -1339,8 +1464,18 @@ class PartitionTest {
     val initializeTimeMs = time.milliseconds()
     partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
     assertTrue("Expected become leader transition to succeed",
-      partition.makeLeader(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
-        leaderEpoch, isr, 1, replicas, true), 0, offsetCheckpoints))
+      partition.makeLeader(
+        controllerId,
+        new LeaderAndIsrPartitionState()
+          .setControllerEpoch(controllerEpoch)
+          .setLeader(brokerId)
+          .setLeaderEpoch(leaderEpoch)
+          .setIsr(isr)
+          .setZkVersion(1)
+          .setReplicas(replicas)
+          .setIsNew(true),
+        0,
+        offsetCheckpoints))
     assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicaIds)
     assertEquals(0L, partition.localLogOrException.highWatermark)
 
@@ -1375,8 +1510,14 @@ class PartitionTest {
     val controllerId = 0
     val controllerEpoch = 3
     val replicas = List[Integer](brokerId, brokerId + 1).asJava
-    val leaderState = new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId,
-      6, replicas, 1, replicas, false)
+    val leaderState = new LeaderAndIsrPartitionState()
+      .setControllerEpoch(controllerEpoch)
+      .setLeader(brokerId)
+      .setLeaderEpoch(6)
+      .setIsr(replicas)
+      .setZkVersion(1)
+      .setReplicas(replicas)
+      .setIsNew(false)
     partition.makeLeader(controllerId, leaderState, 0, offsetCheckpoints)
     assertEquals(4, partition.localLogOrException.highWatermark)
   }
diff --git a/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala b/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala
index 172c45b30fdde..e603b25cfa3f4 100644
--- a/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala
@@ -23,6 +23,9 @@ import kafka.cluster.{Broker, EndPoint}
 import kafka.server.KafkaConfig
 import kafka.utils.TestUtils
 import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.message.{LeaderAndIsrResponseData, StopReplicaResponseData}
+import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError
+import org.apache.kafka.common.message.StopReplicaResponseData.StopReplicaPartitionError
 import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
 import org.apache.kafka.common.requests.{AbstractControlRequest, AbstractResponse, LeaderAndIsrRequest, LeaderAndIsrResponse, StopReplicaRequest, StopReplicaResponse, UpdateMetadataRequest}
@@ -70,18 +73,21 @@ class ControllerChannelManagerTest {
     val leaderAndIsrRequest = leaderAndIsrRequests.head
     assertEquals(controllerId, leaderAndIsrRequest.controllerId)
     assertEquals(controllerEpoch, leaderAndIsrRequest.controllerEpoch)
-    assertEquals(partitions.keySet, leaderAndIsrRequest.partitionStates.keySet.asScala)
+    assertEquals(partitions.keySet,
+      leaderAndIsrRequest.partitionStates.asScala.map(p => new TopicPartition(p.topicName, p.partitionIndex)).toSet)
     assertEquals(partitions.map { case (k, v) => (k, v.leader) },
-      leaderAndIsrRequest.partitionStates.asScala.mapValues(_.basePartitionState.leader).toMap)
+      leaderAndIsrRequest.partitionStates.asScala.map(p => new TopicPartition(p.topicName, p.partitionIndex) -> p.leader).toMap)
     assertEquals(partitions.map { case (k, v) => (k, v.isr) },
-      leaderAndIsrRequest.partitionStates.asScala.mapValues(_.basePartitionState.isr.asScala).toMap)
+      leaderAndIsrRequest.partitionStates.asScala.map(p => new TopicPartition(p.topicName, p.partitionIndex) -> p.isr.asScala).toMap)
 
     applyLeaderAndIsrResponseCallbacks(Errors.NONE, batch.sentRequests(2).toList)
     assertEquals(1, batch.sentEvents.size)
 
     val LeaderAndIsrResponseReceived(response, brokerId) = batch.sentEvents.head
+    val leaderAndIsrResponse = response.asInstanceOf[LeaderAndIsrResponse]
     assertEquals(2, brokerId)
-    assertEquals(partitions.keySet, response.asInstanceOf[LeaderAndIsrResponse].responses.keySet.asScala)
+    assertEquals(partitions.keySet,
+      leaderAndIsrResponse.partitions.asScala.map(p => new TopicPartition(p.topicName, p.partitionIndex)).toSet)
   }
 
   @Test
@@ -106,8 +112,10 @@ class ControllerChannelManagerTest {
     assertEquals(1, updateMetadataRequests.size)
 
     val leaderAndIsrRequest = leaderAndIsrRequests.head
-    assertEquals(Set(partition), leaderAndIsrRequest.partitionStates.keySet.asScala)
-    assertTrue(leaderAndIsrRequest.partitionStates.get(partition).isNew)
+    val partitionStates = leaderAndIsrRequest.partitionStates.asScala
+    assertEquals(Seq(partition), partitionStates.map(p => new TopicPartition(p.topicName, p.partitionIndex)))
+    val partitionState = partitionStates.find(p => p.topicName == partition.topic && p.partitionIndex == partition.partition)
+    assertEquals(Some(true), partitionState.map(_.isNew))
   }
 
   @Test
@@ -139,7 +147,7 @@ class ControllerChannelManagerTest {
       assertEquals(1, leaderAndIsrRequests.size)
       assertEquals(1, updateMetadataRequests.size)
       val leaderAndIsrRequest = leaderAndIsrRequests.head
-      assertEquals(Set(partition), leaderAndIsrRequest.partitionStates.keySet.asScala)
+      assertEquals(Seq(partition), leaderAndIsrRequest.partitionStates.asScala.map(p => new TopicPartition(p.topicName, p.partitionIndex)))
     }
   }
 
@@ -203,11 +211,12 @@ class ControllerChannelManagerTest {
     assertEquals(1, updateMetadataRequests.size)
 
     val updateMetadataRequest = updateMetadataRequests.head
-    assertEquals(3, updateMetadataRequest.partitionStates.size)
+    val partitionStates = updateMetadataRequest.partitionStates.asScala.toBuffer
+    assertEquals(3, partitionStates.size)
     assertEquals(partitions.map { case (k, v) => (k, v.leader) },
-      updateMetadataRequest.partitionStates.asScala.map { case (k, v) => (k, v.basePartitionState.leader) })
+      partitionStates.map(ps => (new TopicPartition(ps.topicName, ps.partitionIndex), ps.leader)).toMap)
     assertEquals(partitions.map { case (k, v) => (k, v.isr) },
-      updateMetadataRequest.partitionStates.asScala.map { case (k, v) => (k, v.basePartitionState.isr.asScala) })
+      partitionStates.map(ps => (new TopicPartition(ps.topicName, ps.partitionIndex), ps.isr.asScala)).toMap)
 
     assertEquals(controllerId, updateMetadataRequest.controllerId)
     assertEquals(controllerEpoch, updateMetadataRequest.controllerEpoch)
@@ -238,7 +247,7 @@ class ControllerChannelManagerTest {
     assertEquals(1, updateMetadataRequests.size)
 
     val updateMetadataRequest = updateMetadataRequests.head
-    assertEquals(0, updateMetadataRequest.partitionStates.size)
+    assertEquals(0, updateMetadataRequest.partitionStates.asScala.size)
     assertEquals(3, updateMetadataRequest.liveBrokers.size)
     assertEquals(Set(1, 2, 3), updateMetadataRequest.liveBrokers.asScala.map(_.id).toSet)
   }
@@ -268,19 +277,18 @@ class ControllerChannelManagerTest {
     assertEquals(1, updateMetadataRequests.size)
 
     val updateMetadataRequest = updateMetadataRequests.head
-    assertEquals(3, updateMetadataRequest.partitionStates.size)
+    assertEquals(3, updateMetadataRequest.partitionStates.asScala.size)
 
     assertTrue(updateMetadataRequest.partitionStates.asScala
-      .filterKeys(_.topic == "foo")
-      .values
-      .map(_.basePartitionState.leader)
+      .filter(_.topicName == "foo")
+      .map(_.leader)
       .forall(leaderId => leaderId == LeaderAndIsr.LeaderDuringDelete))
 
     assertEquals(partitions.filter { case (k, _) => k.topic == "bar" }.map { case (k, v) => (k, v.leader) },
-      updateMetadataRequest.partitionStates.asScala.filter { case (k, _) => k.topic == "bar" }.map { case (k, v) =>
-        (k, v.basePartitionState.leader) })
+      updateMetadataRequest.partitionStates.asScala.filter(ps => ps.topicName == "bar").map { ps =>
+        (new TopicPartition(ps.topicName, ps.partitionIndex), ps.leader) }.toMap)
     assertEquals(partitions.map { case (k, v) => (k, v.isr) },
-      updateMetadataRequest.partitionStates.asScala.map { case (k, v) => (k, v.basePartitionState.isr.asScala) })
+      updateMetadataRequest.partitionStates.asScala.map(ps => (new TopicPartition(ps.topicName, ps.partitionIndex), ps.isr.asScala)).toMap)
 
     assertEquals(3, updateMetadataRequest.liveBrokers.size)
     assertEquals(Set(1, 2, 3), updateMetadataRequest.liveBrokers.asScala.map(_.id).toSet)
@@ -306,7 +314,7 @@ class ControllerChannelManagerTest {
       assertEquals(1, updateMetadataRequests.size)
 
       val updateMetadataRequest = updateMetadataRequests.head
-      assertEquals(0, updateMetadataRequest.partitionStates.size)
+      assertEquals(0, updateMetadataRequest.partitionStates.asScala.size)
       assertEquals(2, updateMetadataRequest.liveBrokers.size)
       assertEquals(Set(1, 2), updateMetadataRequest.liveBrokers.asScala.map(_.id).toSet)
     }
@@ -415,7 +423,7 @@ class ControllerChannelManagerTest {
     val sentStopReplicaRequests = batch.collectStopReplicaRequestsFor(2)
     assertEquals(1, sentStopReplicaRequests.size)
     assertEquals(partitions, sentStopReplicaRequests.flatMap(_.partitions.asScala).toSet)
-    assertTrue(sentStopReplicaRequests.forall(_.deletePartitions()))
+    assertTrue(sentStopReplicaRequests.forall(_.deletePartitions))
 
     // No events will be sent after the response returns
     applyStopReplicaResponseCallbacks(Errors.NONE, batch.sentRequests(2).toList)
@@ -620,8 +628,18 @@ class ControllerChannelManagerTest {
   private def applyStopReplicaResponseCallbacks(error: Errors, sentRequests: List[SentRequest]): Unit = {
     sentRequests.filter(_.responseCallback != null).foreach { sentRequest =>
       val stopReplicaRequest = sentRequest.request.build().asInstanceOf[StopReplicaRequest]
-      val partitionErrorMap = stopReplicaRequest.partitions.asScala.map(_ -> error).toMap.asJava
-      val stopReplicaResponse = new StopReplicaResponse(error, partitionErrorMap)
+      val stopReplicaResponse =
+        if (error == Errors.NONE) {
+          val partitionErrors = stopReplicaRequest.partitions.asScala.map { tp =>
+            new StopReplicaPartitionError()
+              .setTopicName(tp.topic)
+              .setPartitionIndex(tp.partition)
+              .setErrorCode(error.code)
+          }.toBuffer.asJava
+          new StopReplicaResponse(new StopReplicaResponseData().setPartitionErrors(partitionErrors))
+        } else {
+          stopReplicaRequest.getErrorResponse(error.exception)
+        }
       sentRequest.responseCallback.apply(stopReplicaResponse)
     }
   }
@@ -629,9 +647,16 @@ class ControllerChannelManagerTest {
   private def applyLeaderAndIsrResponseCallbacks(error: Errors, sentRequests: List[SentRequest]): Unit = {
     sentRequests.filter(_.request.apiKey == ApiKeys.LEADER_AND_ISR).filter(_.responseCallback != null).foreach { sentRequest =>
       val leaderAndIsrRequest = sentRequest.request.build().asInstanceOf[LeaderAndIsrRequest]
-      val partitionErrorMap = leaderAndIsrRequest.partitionStates.asScala.keySet.map(_ -> error).toMap.asJava
-      val leaderAndIsrResponse = new LeaderAndIsrResponse(error, partitionErrorMap)
-      sentRequest.responseCallback.apply(leaderAndIsrResponse)
+      val partitionErrors = leaderAndIsrRequest.partitionStates.asScala.map(p =>
+        new LeaderAndIsrPartitionError()
+          .setTopicName(p.topicName)
+          .setPartitionIndex(p.partitionIndex)
+          .setErrorCode(error.code))
+      val leaderAndIsrResponse = new LeaderAndIsrResponse(
+        new LeaderAndIsrResponseData()
+          .setErrorCode(error.code)
+          .setPartitionErrors(partitionErrors.toBuffer.asJava))
+      sentRequest.responseCallback(leaderAndIsrResponse)
     }
   }
 
diff --git a/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala b/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala
index 867f4c6fa1511..dc09d7300abb5 100644
--- a/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala
@@ -143,10 +143,10 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
       val offlineReplicaPartitionInfo = server.metadataCache.getPartitionInfo(topic, 0).get
       assertEquals(1, offlineReplicaPartitionInfo.offlineReplicas.size())
       assertEquals(testBroker.config.brokerId, offlineReplicaPartitionInfo.offlineReplicas.get(0))
-      assertEquals(assignment(0).asJava, offlineReplicaPartitionInfo.basePartitionState.replicas)
-      assertEquals(Seq(remainingBrokers.head.config.brokerId).asJava, offlineReplicaPartitionInfo.basePartitionState.isr)
+      assertEquals(assignment(0).asJava, offlineReplicaPartitionInfo.replicas)
+      assertEquals(Seq(remainingBrokers.head.config.brokerId).asJava, offlineReplicaPartitionInfo.isr)
       val onlinePartitionInfo = server.metadataCache.getPartitionInfo(topic, 1).get
-      assertEquals(assignment(1).asJava, onlinePartitionInfo.basePartitionState.replicas)
+      assertEquals(assignment(1).asJava, onlinePartitionInfo.replicas)
       assertTrue(onlinePartitionInfo.offlineReplicas.isEmpty)
     }
 
@@ -158,7 +158,7 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
           val partitionInfoOpt = server.metadataCache.getPartitionInfo(topic, partitionId)
           if (partitionInfoOpt.isDefined) {
             val partitionInfo = partitionInfoOpt.get
-            !partitionInfo.offlineReplicas.isEmpty || !partitionInfo.basePartitionState.replicas.asScala.equals(replicas)
+            !partitionInfo.offlineReplicas.isEmpty || !partitionInfo.replicas.asScala.equals(replicas)
           } else {
             true
           }
@@ -204,8 +204,8 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
         val partitionInfoOpt = broker.metadataCache.getPartitionInfo(topic, 0)
         if (partitionInfoOpt.isDefined) {
           val partitionInfo = partitionInfoOpt.get
-          (!partitionInfo.offlineReplicas.isEmpty && partitionInfo.basePartitionState.leader == -1
-            && !partitionInfo.basePartitionState.replicas.isEmpty && !partitionInfo.basePartitionState.isr.isEmpty)
+          (!partitionInfo.offlineReplicas.isEmpty && partitionInfo.leader == -1
+            && !partitionInfo.replicas.isEmpty && !partitionInfo.isr.isEmpty)
         } else {
           false
         }
@@ -463,14 +463,14 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     var activeServers = servers.filter(s => s.config.brokerId != 2)
     // wait for the update metadata request to trickle to the brokers
     TestUtils.waitUntilTrue(() =>
-      activeServers.forall(_.dataPlaneRequestProcessor.metadataCache.getPartitionInfo(topic,partition).get.basePartitionState.isr.size != 3),
+      activeServers.forall(_.dataPlaneRequestProcessor.metadataCache.getPartitionInfo(topic,partition).get.isr.size != 3),
       "Topic test not created after timeout")
     assertEquals(0, partitionsRemaining.size)
     var partitionStateInfo = activeServers.head.dataPlaneRequestProcessor.metadataCache.getPartitionInfo(topic,partition).get
-    var leaderAfterShutdown = partitionStateInfo.basePartitionState.leader
+    var leaderAfterShutdown = partitionStateInfo.leader
     assertEquals(0, leaderAfterShutdown)
-    assertEquals(2, partitionStateInfo.basePartitionState.isr.size)
-    assertEquals(List(0,1), partitionStateInfo.basePartitionState.isr.asScala)
+    assertEquals(2, partitionStateInfo.isr.size)
+    assertEquals(List(0,1), partitionStateInfo.isr.asScala)
     controller.controlledShutdown(1, servers.find(_.config.brokerId == 1).get.kafkaController.brokerEpoch, controlledShutdownCallback)
     partitionsRemaining = resultQueue.take() match {
       case Success(partitions) => partitions
@@ -479,15 +479,15 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     assertEquals(0, partitionsRemaining.size)
     activeServers = servers.filter(s => s.config.brokerId == 0)
     partitionStateInfo = activeServers.head.dataPlaneRequestProcessor.metadataCache.getPartitionInfo(topic,partition).get
-    leaderAfterShutdown = partitionStateInfo.basePartitionState.leader
+    leaderAfterShutdown = partitionStateInfo.leader
     assertEquals(0, leaderAfterShutdown)
 
-    assertTrue(servers.forall(_.dataPlaneRequestProcessor.metadataCache.getPartitionInfo(topic,partition).get.basePartitionState.leader == 0))
+    assertTrue(servers.forall(_.dataPlaneRequestProcessor.metadataCache.getPartitionInfo(topic,partition).get.leader == 0))
     controller.controlledShutdown(0, servers.find(_.config.brokerId == 0).get.kafkaController.brokerEpoch, controlledShutdownCallback)
     partitionsRemaining = resultQueue.take().get
     assertEquals(1, partitionsRemaining.size)
     // leader doesn't change since all the replicas are shut down
-    assertTrue(servers.forall(_.dataPlaneRequestProcessor.metadataCache.getPartitionInfo(topic,partition).get.basePartitionState.leader == 0))
+    assertTrue(servers.forall(_.dataPlaneRequestProcessor.metadataCache.getPartitionInfo(topic,partition).get.leader == 0))
   }
 
   @Test
diff --git a/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala b/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala
index 4996eec523223..ac8b787a8331b 100755
--- a/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala
+++ b/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala
@@ -255,7 +255,7 @@ class UncleanLeaderElectionTest extends ZooKeeperTestHarness {
     //make sure follower server joins the ISR
     TestUtils.waitUntilTrue(() => {
       val partitionInfoOpt = followerServer.metadataCache.getPartitionInfo(topic, partitionId)
-      partitionInfoOpt.isDefined && partitionInfoOpt.get.basePartitionState.isr.contains(followerId)
+      partitionInfoOpt.isDefined && partitionInfoOpt.get.isr.contains(followerId)
     }, "Inconsistent metadata after first server startup")
 
     servers.filter(server => server.config.brokerId == leaderId).foreach(server => shutdownServer(server))
diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
index 03135113f836b..2cf637a263605 100644
--- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
+++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
@@ -32,7 +32,7 @@ import kafka.security.CredentialProvider
 import kafka.server.{KafkaConfig, ThrottledChannel}
 import kafka.utils.Implicits._
 import kafka.utils.{CoreUtils, TestUtils}
-import org.apache.kafka.common.{Endpoint, TopicPartition}
+import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.memory.MemoryPool
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.network.KafkaChannel.ChannelMuteState
diff --git a/core/src/test/scala/unit/kafka/server/BrokerEpochIntegrationTest.scala b/core/src/test/scala/unit/kafka/server/BrokerEpochIntegrationTest.scala
index d80b5268ac5c8..23836dd66d8c7 100755
--- a/core/src/test/scala/unit/kafka/server/BrokerEpochIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/server/BrokerEpochIntegrationTest.scala
@@ -24,10 +24,11 @@ import kafka.utils.TestUtils
 import kafka.utils.TestUtils.createTopic
 import kafka.zk.ZooKeeperTestHarness
 import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
+import org.apache.kafka.common.message.UpdateMetadataRequestData.{UpdateMetadataBroker, UpdateMetadataEndpoint, UpdateMetadataPartitionState}
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
-import org.apache.kafka.common.requests.UpdateMetadataRequest.EndPoint
 import org.apache.kafka.common.requests._
 import org.apache.kafka.common.security.auth.SecurityProtocol
 import org.apache.kafka.common.utils.Time
@@ -132,10 +133,17 @@ class BrokerEpochIntegrationTest extends ZooKeeperTestHarness {
     try {
       // Send LeaderAndIsr request with correct broker epoch
       {
-        val partitionStates = Map(
-          tp -> new LeaderAndIsrRequest.PartitionState(controllerEpoch, brokerId2, LeaderAndIsr.initialLeaderEpoch + 1,
-            Seq(brokerId1, brokerId2).map(Integer.valueOf).asJava, LeaderAndIsr.initialZKVersion,
-            Seq(0, 1).map(Integer.valueOf).asJava, false)
+        val partitionStates = Seq(
+          new LeaderAndIsrPartitionState()
+            .setTopicName(tp.topic)
+            .setPartitionIndex(tp.partition)
+            .setControllerEpoch(controllerEpoch)
+            .setLeader(brokerId2)
+            .setLeaderEpoch(LeaderAndIsr.initialLeaderEpoch + 1)
+            .setIsr(Seq(brokerId1, brokerId2).map(Integer.valueOf).asJava)
+            .setZkVersion(LeaderAndIsr.initialZKVersion)
+            .setReplicas(Seq(0, 1).map(Integer.valueOf).asJava)
+            .setIsNew(false)
         )
         val requestBuilder = new LeaderAndIsrRequest.Builder(
           ApiKeys.LEADER_AND_ISR.latestVersion, controllerId, controllerEpoch,
@@ -153,32 +161,43 @@ class BrokerEpochIntegrationTest extends ZooKeeperTestHarness {
 
       // Send UpdateMetadata request with correct broker epoch
       {
-        val partitionStates = Map(
-          tp -> new UpdateMetadataRequest.PartitionState(controllerEpoch, brokerId2, LeaderAndIsr.initialLeaderEpoch + 1,
-            Seq(brokerId1, brokerId2).map(Integer.valueOf).asJava, LeaderAndIsr.initialZKVersion,
-            Seq(0, 1).map(Integer.valueOf).asJava, Seq.empty.asJava)
-        )
-        val liverBrokers = brokerAndEpochs.map { brokerAndEpoch =>
-          val broker = brokerAndEpoch._1
+        val partitionStates = Seq(
+          new UpdateMetadataPartitionState()
+            .setTopicName(tp.topic)
+            .setPartitionIndex(tp.partition)
+            .setControllerEpoch(controllerEpoch)
+            .setLeader(brokerId2)
+            .setLeaderEpoch(LeaderAndIsr.initialLeaderEpoch + 1)
+            .setIsr(Seq(brokerId1, brokerId2).map(Integer.valueOf).asJava)
+            .setZkVersion(LeaderAndIsr.initialZKVersion)
+            .setReplicas(Seq(0, 1).map(Integer.valueOf).asJava))
+        val liveBrokers = brokerAndEpochs.map { case (broker, _) =>
           val securityProtocol = SecurityProtocol.PLAINTEXT
           val listenerName = ListenerName.forSecurityProtocol(securityProtocol)
           val node = broker.node(listenerName)
-          val endPoints = Seq(new EndPoint(node.host, node.port, securityProtocol, listenerName))
-          new UpdateMetadataRequest.Broker(broker.id, endPoints.asJava, broker.rack.orNull)
-        }
+          val endpoints = Seq(new UpdateMetadataEndpoint()
+            .setHost(node.host)
+            .setPort(node.port)
+            .setSecurityProtocol(securityProtocol.id)
+            .setListener(listenerName.value))
+          new UpdateMetadataBroker()
+            .setId(broker.id)
+            .setEndpoints(endpoints.asJava)
+            .setRack(broker.rack.orNull)
+        }.toBuffer
         val requestBuilder = new UpdateMetadataRequest.Builder(
           ApiKeys.UPDATE_METADATA.latestVersion, controllerId, controllerEpoch,
           epochInRequest,
-          partitionStates.asJava, liverBrokers.toSet.asJava)
+          partitionStates.asJava, liveBrokers.asJava)
 
         if (isEpochInRequestStale) {
           sendAndVerifyStaleBrokerEpochInResponse(controllerChannelManager, requestBuilder)
         }
         else {
           sendAndVerifySuccessfulResponse(controllerChannelManager, requestBuilder)
-          TestUtils.waitUntilMetadataIsPropagated(Seq(broker2), tp.topic(), tp.partition(), 10000)
+          TestUtils.waitUntilMetadataIsPropagated(Seq(broker2), tp.topic, tp.partition, 10000)
           assertEquals(brokerId2,
-            broker2.metadataCache.getPartitionInfo(tp.topic(), tp.partition()).get.basePartitionState.leader)
+            broker2.metadataCache.getPartitionInfo(tp.topic, tp.partition).get.leader)
         }
       }
 
diff --git a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
index 23091c8b08e49..daea624240433 100644
--- a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
@@ -34,6 +34,7 @@ import org.junit.Assert._
 import org.junit.Test
 
 import scala.collection.JavaConverters._
+import scala.collection.Seq
 import scala.util.Random
 
 /**
@@ -556,7 +557,7 @@ class FetchRequestTest extends BaseRequestTest {
   }
 
   private def records(partitionData: FetchResponse.PartitionData[MemoryRecords]): Seq[Record] = {
-    partitionData.records.records.asScala.toIndexedSeq
+    partitionData.records.records.asScala.toBuffer
   }
 
   private def checkFetchResponse(expectedPartitions: Seq[TopicPartition], fetchResponse: FetchResponse[MemoryRecords],
@@ -574,7 +575,7 @@ class FetchRequestTest extends BaseRequestTest {
       val records = partitionData.records
       responseBufferSize += records.sizeInBytes
 
-      val batches = records.batches.asScala.toIndexedSeq
+      val batches = records.batches.asScala.toBuffer
       assertTrue(batches.size < numMessagesPerPartition)
       val batchesSize = batches.map(_.sizeInBytes).sum
       responseSize += batchesSize
diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
index 3c37bc02b0ea4..48d21e29f26ae 100644
--- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
@@ -41,7 +41,6 @@ import org.apache.kafka.common.protocol.{ApiKeys, Errors}
 import org.apache.kafka.common.record.FileRecords.TimestampAndOffset
 import org.apache.kafka.common.record._
 import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse
-import org.apache.kafka.common.requests.UpdateMetadataRequest.{Broker, EndPoint}
 import org.apache.kafka.common.requests.WriteTxnMarkersRequest.TxnMarkerEntry
 import org.apache.kafka.common.requests.{FetchMetadata => JFetchMetadata, _}
 import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
@@ -50,6 +49,7 @@ import EasyMock._
 import org.apache.kafka.common.message.{HeartbeatRequestData, JoinGroupRequestData, OffsetCommitRequestData, OffsetCommitResponseData, SyncGroupRequestData, TxnOffsetCommitRequestData}
 import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocol
 import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
+import org.apache.kafka.common.message.UpdateMetadataRequestData.{UpdateMetadataBroker, UpdateMetadataEndpoint, UpdateMetadataPartitionState}
 import org.apache.kafka.common.replica.ClientMetadata
 import org.apache.kafka.server.authorizer.Authorizer
 import org.junit.Assert.{assertEquals, assertNull, assertTrue}
@@ -714,14 +714,34 @@ class KafkaApisTest {
   private def updateMetadataCacheWithInconsistentListeners(): (ListenerName, ListenerName) = {
     val plaintextListener = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)
     val anotherListener = new ListenerName("LISTENER2")
-    val brokers = Set(
-      new Broker(0, Seq(new EndPoint("broker0", 9092, SecurityProtocol.PLAINTEXT, plaintextListener),
-        new EndPoint("broker0", 9093, SecurityProtocol.PLAINTEXT, anotherListener)).asJava, "rack"),
-      new Broker(1, Seq(new EndPoint("broker1", 9092, SecurityProtocol.PLAINTEXT, plaintextListener)).asJava,
-        "rack")
+    val brokers = Seq(
+      new UpdateMetadataBroker()
+        .setId(0)
+        .setRack("rack")
+        .setEndpoints(Seq(
+          new UpdateMetadataEndpoint()
+            .setHost("broker0")
+            .setPort(9092)
+            .setSecurityProtocol(SecurityProtocol.PLAINTEXT.id)
+            .setListener(plaintextListener.value),
+          new UpdateMetadataEndpoint()
+            .setHost("broker0")
+            .setPort(9093)
+            .setSecurityProtocol(SecurityProtocol.PLAINTEXT.id)
+            .setListener(anotherListener.value)
+        ).asJava),
+      new UpdateMetadataBroker()
+        .setId(1)
+        .setRack("rack")
+        .setEndpoints(Seq(
+          new UpdateMetadataEndpoint()
+            .setHost("broker1")
+            .setPort(9092)
+            .setSecurityProtocol(SecurityProtocol.PLAINTEXT.id)
+            .setListener(plaintextListener.value)).asJava)
     )
     val updateMetadataRequest = new UpdateMetadataRequest.Builder(ApiKeys.UPDATE_METADATA.latestVersion, 0,
-      0, 0, Map.empty[TopicPartition, UpdateMetadataRequest.PartitionState].asJava, brokers.asJava).build()
+      0, 0, Seq.empty[UpdateMetadataPartitionState].asJava, brokers.asJava).build()
     metadataCache.updateMetadata(correlationId = 0, updateMetadataRequest)
     (plaintextListener, anotherListener)
   }
@@ -815,12 +835,29 @@ class KafkaApisTest {
 
   private def setupBasicMetadataCache(topic: String, numPartitions: Int): Unit = {
     val replicas = List(0.asInstanceOf[Integer]).asJava
-    val partitionState = new UpdateMetadataRequest.PartitionState(1, 0, 1, replicas, 0, replicas, Collections.emptyList())
+
+    def createPartitionState(partition: Int) = new UpdateMetadataPartitionState()
+      .setTopicName(topic)
+      .setPartitionIndex(partition)
+      .setControllerEpoch(1)
+      .setLeader(0)
+      .setLeaderEpoch(1)
+      .setReplicas(replicas)
+      .setZkVersion(0)
+      .setReplicas(replicas)
+
     val plaintextListener = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)
-    val broker = new Broker(0, Seq(new EndPoint("broker0", 9092, SecurityProtocol.PLAINTEXT, plaintextListener)).asJava, "rack")
-    val partitions = (0 until numPartitions).map(new TopicPartition(topic, _) -> partitionState).toMap
+    val broker = new UpdateMetadataBroker()
+      .setId(0)
+      .setRack("rack")
+      .setEndpoints(Seq(new UpdateMetadataEndpoint()
+        .setHost("broker0")
+        .setPort(9092)
+        .setSecurityProtocol(SecurityProtocol.PLAINTEXT.id)
+        .setListener(plaintextListener.value)).asJava)
+    val partitionStates = (0 until numPartitions).map(createPartitionState)
     val updateMetadataRequest = new UpdateMetadataRequest.Builder(ApiKeys.UPDATE_METADATA.latestVersion, 0,
-      0, 0, partitions.asJava, Set(broker).asJava).build()
+      0, 0, partitionStates.asJava, Seq(broker).asJava).build()
     metadataCache.updateMetadata(correlationId = 0, updateMetadataRequest)
   }
 }
diff --git a/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala b/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala
index 6aeb355832cf5..01423a39f61f0 100755
--- a/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala
+++ b/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala
@@ -28,6 +28,7 @@ import kafka.cluster.Broker
 import kafka.controller.{ControllerChannelManager, ControllerContext, StateChangeLogger}
 import kafka.utils.TestUtils._
 import kafka.zk.ZooKeeperTestHarness
+import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
@@ -93,12 +94,7 @@ class LeaderElectionTest extends ZooKeeperTestHarness {
     servers.head.startup()
     //make sure second server joins the ISR
     TestUtils.waitUntilTrue(() => {
-      val partitionInfoOpt = servers.last.metadataCache.getPartitionInfo(topic, partitionId)
-      if (partitionInfoOpt.isDefined) {
-        partitionInfoOpt.get.basePartitionState.isr.size() == 2
-      } else {
-        false
-      }
+      servers.last.metadataCache.getPartitionInfo(topic, partitionId).exists(_.isr.size == 2)
     }, "Inconsistent metadata after second broker startup")
 
     servers.last.shutdown()
@@ -145,13 +141,21 @@ class LeaderElectionTest extends ZooKeeperTestHarness {
     controllerChannelManager.startup()
     try {
       val staleControllerEpoch = 0
-      val partitionStates = Map(
-        new TopicPartition(topic, partitionId) -> new LeaderAndIsrRequest.PartitionState(2, brokerId2, LeaderAndIsr.initialLeaderEpoch,
-          Seq(brokerId1, brokerId2).map(Integer.valueOf).asJava, LeaderAndIsr.initialZKVersion,
-          Seq(0, 1).map(Integer.valueOf).asJava, false)
+      val partitionStates = Seq(
+        new LeaderAndIsrPartitionState()
+          .setTopicName(topic)
+          .setPartitionIndex(partitionId)
+          .setControllerEpoch(2)
+          .setLeader(brokerId2)
+          .setLeaderEpoch(LeaderAndIsr.initialLeaderEpoch)
+          .setIsr(Seq(brokerId1, brokerId2).map(Integer.valueOf).asJava)
+          .setZkVersion(LeaderAndIsr.initialZKVersion)
+          .setReplicas(Seq(0, 1).map(Integer.valueOf).asJava)
+          .setIsNew(false)
       )
       val requestBuilder = new LeaderAndIsrRequest.Builder(
-        ApiKeys.LEADER_AND_ISR.latestVersion, controllerId, staleControllerEpoch, servers(brokerId2).kafkaController.brokerEpoch ,partitionStates.asJava, nodes.toSet.asJava)
+        ApiKeys.LEADER_AND_ISR.latestVersion, controllerId, staleControllerEpoch,
+        servers(brokerId2).kafkaController.brokerEpoch, partitionStates.asJava, nodes.toSet.asJava)
 
       controllerChannelManager.sendRequest(brokerId2, requestBuilder, staleControllerEpochCallback)
       TestUtils.waitUntilTrue(() => staleControllerEpochDetected, "Controller epoch should be stale")
diff --git a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala
index 8d7b27a25d377..a42a86a5d03e5 100644
--- a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala
+++ b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala
@@ -20,14 +20,14 @@ import java.util
 import java.util.Optional
 import util.Arrays.asList
 
-import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.message.UpdateMetadataRequestData.{UpdateMetadataBroker, UpdateMetadataEndpoint, UpdateMetadataPartitionState}
 import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
 import org.apache.kafka.common.requests.UpdateMetadataRequest
-import org.apache.kafka.common.requests.UpdateMetadataRequest.{Broker, EndPoint}
 import org.apache.kafka.common.security.auth.SecurityProtocol
 import org.junit.Test
 import org.junit.Assert._
+import org.scalatest.Assertions
 
 import scala.collection.JavaConverters._
 
@@ -53,22 +53,57 @@ class MetadataCacheTest {
     val controllerId = 2
     val controllerEpoch = 1
 
-    def endPoints(brokerId: Int): Seq[EndPoint] = {
+    def endpoints(brokerId: Int): Seq[UpdateMetadataEndpoint] = {
       val host = s"foo-$brokerId"
       Seq(
-        new EndPoint(host, 9092, SecurityProtocol.PLAINTEXT, ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)),
-        new EndPoint(host, 9093, SecurityProtocol.SSL, ListenerName.forSecurityProtocol(SecurityProtocol.SSL))
+        new UpdateMetadataEndpoint()
+          .setHost(host)
+          .setPort(9092)
+          .setSecurityProtocol(SecurityProtocol.PLAINTEXT.id)
+          .setListener(ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT).value),
+        new UpdateMetadataEndpoint()
+          .setHost(host)
+          .setPort(9093)
+          .setSecurityProtocol(SecurityProtocol.SSL.id)
+          .setListener(ListenerName.forSecurityProtocol(SecurityProtocol.SSL).value)
       )
     }
 
     val brokers = (0 to 4).map { brokerId =>
-      new Broker(brokerId, endPoints(brokerId).asJava, "rack1")
-    }.toSet
+      new UpdateMetadataBroker()
+        .setId(brokerId)
+        .setEndpoints(endpoints(brokerId).asJava)
+        .setRack("rack1")
+    }
 
-    val partitionStates = Map(
-      new TopicPartition(topic0, 0) -> new UpdateMetadataRequest.PartitionState(controllerEpoch, 0, 0, asList(0, 1, 3), zkVersion, asList(0, 1, 3), asList()),
-      new TopicPartition(topic0, 1) -> new UpdateMetadataRequest.PartitionState(controllerEpoch, 1, 1, asList(1, 0), zkVersion, asList(1, 2, 0, 4), asList()),
-      new TopicPartition(topic1, 0) -> new UpdateMetadataRequest.PartitionState(controllerEpoch, 2, 2, asList(2, 1), zkVersion, asList(2, 1, 3), asList()))
+    val partitionStates = Seq(
+      new UpdateMetadataPartitionState()
+        .setTopicName(topic0)
+        .setPartitionIndex(0)
+        .setControllerEpoch(controllerEpoch)
+        .setLeader(0)
+        .setLeaderEpoch(0)
+        .setIsr(asList(0, 1, 3))
+        .setZkVersion(zkVersion)
+        .setReplicas(asList(0, 1, 3)),
+      new UpdateMetadataPartitionState()
+        .setTopicName(topic0)
+        .setPartitionIndex(1)
+        .setControllerEpoch(controllerEpoch)
+        .setLeader(1)
+        .setLeaderEpoch(1)
+        .setIsr(asList(1, 0))
+        .setZkVersion(zkVersion)
+        .setReplicas(asList(1, 2, 0, 4)),
+      new UpdateMetadataPartitionState()
+        .setTopicName(topic1)
+        .setPartitionIndex(0)
+        .setControllerEpoch(controllerEpoch)
+        .setLeader(2)
+        .setLeaderEpoch(2)
+        .setIsr(asList(2, 1))
+        .setZkVersion(zkVersion)
+        .setReplicas(asList(2, 1, 3)))
 
     val version = ApiKeys.UPDATE_METADATA.latestVersion
     val updateMetadataRequest = new UpdateMetadataRequest.Builder(version, controllerId, controllerEpoch, brokerEpoch,
@@ -86,7 +121,7 @@ class MetadataCacheTest {
         assertEquals(Errors.NONE, topicMetadata.error)
         assertEquals(topic, topicMetadata.topic)
 
-        val topicPartitionStates = partitionStates.filter { case (tp, _) => tp.topic ==  topic }
+        val topicPartitionStates = partitionStates.filter { ps => ps.topicName == topic }
         val partitionMetadatas = topicMetadata.partitionMetadata.asScala.sortBy(_.partition)
         assertEquals(s"Unexpected partition count for topic $topic", topicPartitionStates.size, partitionMetadatas.size)
 
@@ -94,14 +129,15 @@ class MetadataCacheTest {
           assertEquals(Errors.NONE, partitionMetadata.error)
           assertEquals(partitionId, partitionMetadata.partition)
           val leader = partitionMetadata.leader
-          val partitionState = topicPartitionStates(new TopicPartition(topic, partitionId))
-          assertEquals(partitionState.basePartitionState.leader, leader.id)
-          assertEquals(Optional.of(partitionState.basePartitionState.leaderEpoch), partitionMetadata.leaderEpoch)
-          assertEquals(partitionState.basePartitionState.isr, partitionMetadata.isr.asScala.map(_.id).asJava)
-          assertEquals(partitionState.basePartitionState.replicas, partitionMetadata.replicas.asScala.map(_.id).asJava)
-          val endPoint = endPoints(partitionMetadata.leader.id).find(_.listenerName == listenerName).get
-          assertEquals(endPoint.host, leader.host)
-          assertEquals(endPoint.port, leader.port)
+          val partitionState = topicPartitionStates.find(_.partitionIndex == partitionId).getOrElse(
+            Assertions.fail(s"Unable to find partition state for partition $partitionId"))
+          assertEquals(partitionState.leader, leader.id)
+          assertEquals(Optional.of(partitionState.leaderEpoch), partitionMetadata.leaderEpoch)
+          assertEquals(partitionState.isr, partitionMetadata.isr.asScala.map(_.id).asJava)
+          assertEquals(partitionState.replicas, partitionMetadata.replicas.asScala.map(_.id).asJava)
+          val endpoint = endpoints(partitionMetadata.leader.id).find(_.listener == listenerName.value).get
+          assertEquals(endpoint.host, leader.host)
+          assertEquals(endpoint.port, leader.port)
         }
       }
 
@@ -115,7 +151,13 @@ class MetadataCacheTest {
   def getTopicMetadataPartitionLeaderNotAvailable(): Unit = {
     val securityProtocol = SecurityProtocol.PLAINTEXT
     val listenerName = ListenerName.forSecurityProtocol(securityProtocol)
-    val brokers = Set(new Broker(0, Seq(new EndPoint("foo", 9092, securityProtocol, listenerName)).asJava, null))
+    val brokers = Seq(new UpdateMetadataBroker()
+      .setId(0)
+      .setEndpoints(Seq(new UpdateMetadataEndpoint()
+        .setHost("foo")
+        .setPort(9092)
+        .setSecurityProtocol(securityProtocol.id)
+        .setListener(listenerName.value)).asJava))
     verifyTopicMetadataPartitionLeaderOrEndpointNotAvailable(brokers, listenerName,
       leader = 1, Errors.LEADER_NOT_AVAILABLE, errorUnavailableListeners = false)
     verifyTopicMetadataPartitionLeaderOrEndpointNotAvailable(brokers, listenerName,
@@ -127,10 +169,28 @@ class MetadataCacheTest {
     val plaintextListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)
     val sslListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.SSL)
     val broker0Endpoints = Seq(
-      new EndPoint("host0", 9092, SecurityProtocol.PLAINTEXT, plaintextListenerName),
-      new EndPoint("host0", 9093, SecurityProtocol.SSL, sslListenerName))
-    val broker1Endpoints = Seq(new EndPoint("host1", 9092, SecurityProtocol.PLAINTEXT, plaintextListenerName))
-    val brokers = Set(new Broker(0, broker0Endpoints.asJava, null), new Broker(1, broker1Endpoints.asJava, null))
+      new UpdateMetadataEndpoint()
+        .setHost("host0")
+        .setPort(9092)
+        .setSecurityProtocol(SecurityProtocol.PLAINTEXT.id)
+        .setListener(plaintextListenerName.value),
+      new UpdateMetadataEndpoint()
+        .setHost("host0")
+        .setPort(9093)
+        .setSecurityProtocol(SecurityProtocol.SSL.id)
+        .setListener(sslListenerName.value))
+    val broker1Endpoints = Seq(new UpdateMetadataEndpoint()
+      .setHost("host1")
+      .setPort(9092)
+      .setSecurityProtocol(SecurityProtocol.PLAINTEXT.id)
+      .setListener(plaintextListenerName.value))
+    val brokers = Seq(
+      new UpdateMetadataBroker()
+        .setId(0)
+        .setEndpoints(broker0Endpoints.asJava),
+      new UpdateMetadataBroker()
+        .setId(1)
+        .setEndpoints(broker1Endpoints.asJava))
     verifyTopicMetadataPartitionLeaderOrEndpointNotAvailable(brokers, sslListenerName,
       leader = 1, Errors.LISTENER_NOT_FOUND, errorUnavailableListeners = true)
   }
@@ -140,15 +200,33 @@ class MetadataCacheTest {
     val plaintextListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)
     val sslListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.SSL)
     val broker0Endpoints = Seq(
-      new EndPoint("host0", 9092, SecurityProtocol.PLAINTEXT, plaintextListenerName),
-      new EndPoint("host0", 9093, SecurityProtocol.SSL, sslListenerName))
-    val broker1Endpoints = Seq(new EndPoint("host1", 9092, SecurityProtocol.PLAINTEXT, plaintextListenerName))
-    val brokers = Set(new Broker(0, broker0Endpoints.asJava, null), new Broker(1, broker1Endpoints.asJava, null))
+      new UpdateMetadataEndpoint()
+        .setHost("host0")
+        .setPort(9092)
+        .setSecurityProtocol(SecurityProtocol.PLAINTEXT.id)
+        .setListener(plaintextListenerName.value),
+      new UpdateMetadataEndpoint()
+        .setHost("host0")
+        .setPort(9093)
+        .setSecurityProtocol(SecurityProtocol.SSL.id)
+        .setListener(sslListenerName.value))
+    val broker1Endpoints = Seq(new UpdateMetadataEndpoint()
+      .setHost("host1")
+      .setPort(9092)
+      .setSecurityProtocol(SecurityProtocol.PLAINTEXT.id)
+      .setListener(plaintextListenerName.value))
+    val brokers = Seq(
+      new UpdateMetadataBroker()
+        .setId(0)
+        .setEndpoints(broker0Endpoints.asJava),
+      new UpdateMetadataBroker()
+        .setId(1)
+        .setEndpoints(broker1Endpoints.asJava))
     verifyTopicMetadataPartitionLeaderOrEndpointNotAvailable(brokers, sslListenerName,
       leader = 1, Errors.LEADER_NOT_AVAILABLE, errorUnavailableListeners = false)
   }
 
-  private def verifyTopicMetadataPartitionLeaderOrEndpointNotAvailable(brokers: Set[Broker],
+  private def verifyTopicMetadataPartitionLeaderOrEndpointNotAvailable(brokers: Seq[UpdateMetadataBroker],
                                                                        listenerName: ListenerName,
                                                                        leader: Int,
                                                                        expectedError: Errors,
@@ -162,8 +240,15 @@ class MetadataCacheTest {
     val controllerEpoch = 1
 
     val leaderEpoch = 1
-    val partitionStates = Map(
-      new TopicPartition(topic, 0) -> new UpdateMetadataRequest.PartitionState(controllerEpoch, leader, leaderEpoch, asList(0), zkVersion, asList(0), asList()))
+    val partitionStates = Seq(new UpdateMetadataPartitionState()
+      .setTopicName(topic)
+      .setPartitionIndex(0)
+      .setControllerEpoch(controllerEpoch)
+      .setLeader(leader)
+      .setLeaderEpoch(leaderEpoch)
+      .setIsr(asList(0))
+      .setZkVersion(zkVersion)
+      .setReplicas(asList(0)))
 
     val version = ApiKeys.UPDATE_METADATA.latestVersion
     val updateMetadataRequest = new UpdateMetadataRequest.Builder(version, controllerId, controllerEpoch, brokerEpoch,
@@ -198,7 +283,13 @@ class MetadataCacheTest {
     val controllerEpoch = 1
     val securityProtocol = SecurityProtocol.PLAINTEXT
     val listenerName = ListenerName.forSecurityProtocol(securityProtocol)
-    val brokers = Set(new Broker(0, Seq(new EndPoint("foo", 9092, securityProtocol, listenerName)).asJava, null))
+    val brokers = Seq(new UpdateMetadataBroker()
+      .setId(0)
+      .setEndpoints(Seq(new UpdateMetadataEndpoint()
+        .setHost("foo")
+        .setPort(9092)
+        .setSecurityProtocol(securityProtocol.id)
+        .setListener(listenerName.value)).asJava))
 
     // replica 1 is not available
     val leader = 0
@@ -206,8 +297,16 @@ class MetadataCacheTest {
     val replicas = asList[Integer](0, 1)
     val isr = asList[Integer](0)
 
-    val partitionStates = Map(
-      new TopicPartition(topic, 0) -> new UpdateMetadataRequest.PartitionState(controllerEpoch, leader, leaderEpoch, isr, zkVersion, replicas, asList()))
+    val partitionStates = Seq(
+      new UpdateMetadataPartitionState()
+        .setTopicName(topic)
+        .setPartitionIndex(0)
+        .setControllerEpoch(controllerEpoch)
+        .setLeader(leader)
+        .setLeaderEpoch(leaderEpoch)
+        .setIsr(isr)
+        .setZkVersion(zkVersion)
+        .setReplicas(replicas))
 
     val version = ApiKeys.UPDATE_METADATA.latestVersion
     val updateMetadataRequest = new UpdateMetadataRequest.Builder(version, controllerId, controllerEpoch, brokerEpoch,
@@ -258,7 +357,14 @@ class MetadataCacheTest {
     val controllerEpoch = 1
     val securityProtocol = SecurityProtocol.PLAINTEXT
     val listenerName = ListenerName.forSecurityProtocol(securityProtocol)
-    val brokers = Set(new Broker(0, Seq(new EndPoint("foo", 9092, securityProtocol, listenerName)).asJava, "rack1"))
+    val brokers = Seq(new UpdateMetadataBroker()
+      .setId(0)
+      .setRack("rack1")
+      .setEndpoints(Seq(new UpdateMetadataEndpoint()
+        .setHost("foo")
+        .setPort(9092)
+        .setSecurityProtocol(securityProtocol.id)
+        .setListener(listenerName.value)).asJava))
 
     // replica 1 is not available
     val leader = 0
@@ -266,8 +372,15 @@ class MetadataCacheTest {
     val replicas = asList[Integer](0)
     val isr = asList[Integer](0, 1)
 
-    val partitionStates = Map(
-      new TopicPartition(topic, 0) -> new UpdateMetadataRequest.PartitionState(controllerEpoch, leader, leaderEpoch, isr, zkVersion, replicas, asList()))
+    val partitionStates = Seq(new UpdateMetadataPartitionState()
+      .setTopicName(topic)
+      .setPartitionIndex(0)
+      .setControllerEpoch(controllerEpoch)
+      .setLeader(leader)
+      .setLeaderEpoch(leaderEpoch)
+      .setIsr(isr)
+      .setZkVersion(zkVersion)
+      .setReplicas(replicas))
 
     val version = ApiKeys.UPDATE_METADATA.latestVersion
     val updateMetadataRequest = new UpdateMetadataRequest.Builder(version, controllerId, controllerEpoch, brokerEpoch,
@@ -312,15 +425,28 @@ class MetadataCacheTest {
     val topic = "topic"
     val cache = new MetadataCache(1)
     val securityProtocol = SecurityProtocol.PLAINTEXT
-    val brokers = Set(new Broker(0,
-      Seq(new EndPoint("foo", 9092, securityProtocol, ListenerName.forSecurityProtocol(securityProtocol))).asJava, ""))
+    val brokers = Seq(new UpdateMetadataBroker()
+      .setId(0)
+      .setRack("")
+      .setEndpoints(Seq(new UpdateMetadataEndpoint()
+        .setHost("foo")
+        .setPort(9092)
+        .setSecurityProtocol(securityProtocol.id)
+        .setListener(ListenerName.forSecurityProtocol(securityProtocol).value)).asJava))
     val controllerEpoch = 1
     val leader = 0
     val leaderEpoch = 0
     val replicas = asList[Integer](0)
     val isr = asList[Integer](0, 1)
-    val partitionStates = Map(
-      new TopicPartition(topic, 0) -> new UpdateMetadataRequest.PartitionState(controllerEpoch, leader, leaderEpoch, isr, 3, replicas, asList()))
+    val partitionStates = Seq(new UpdateMetadataPartitionState()
+      .setTopicName(topic)
+      .setPartitionIndex(0)
+      .setControllerEpoch(controllerEpoch)
+      .setLeader(leader)
+      .setLeaderEpoch(leaderEpoch)
+      .setIsr(isr)
+      .setZkVersion(3)
+      .setReplicas(replicas))
     val version = ApiKeys.UPDATE_METADATA.latestVersion
     val updateMetadataRequest = new UpdateMetadataRequest.Builder(version, 2, controllerEpoch, brokerEpoch, partitionStates.asJava,
       brokers.asJava).build()
@@ -337,31 +463,44 @@ class MetadataCacheTest {
     val topic = "topic"
     val cache = new MetadataCache(1)
 
-    def updateCache(brokerIds: Set[Int]): Unit = {
+    def updateCache(brokerIds: Seq[Int]): Unit = {
       val brokers = brokerIds.map { brokerId =>
         val securityProtocol = SecurityProtocol.PLAINTEXT
-        new Broker(brokerId, Seq(
-          new EndPoint("foo", 9092, securityProtocol, ListenerName.forSecurityProtocol(securityProtocol))).asJava, "")
+        new UpdateMetadataBroker()
+          .setId(brokerId)
+          .setRack("")
+          .setEndpoints(Seq(new UpdateMetadataEndpoint()
+            .setHost("foo")
+            .setPort(9092)
+            .setSecurityProtocol(securityProtocol.id)
+            .setListener(ListenerName.forSecurityProtocol(securityProtocol).value)).asJava)
       }
       val controllerEpoch = 1
       val leader = 0
       val leaderEpoch = 0
       val replicas = asList[Integer](0)
       val isr = asList[Integer](0, 1)
-      val partitionStates = Map(
-        new TopicPartition(topic, 0) -> new UpdateMetadataRequest.PartitionState(controllerEpoch, leader, leaderEpoch, isr, 3, replicas, asList()))
+      val partitionStates = Seq(new UpdateMetadataPartitionState()
+        .setTopicName(topic)
+        .setPartitionIndex(0)
+        .setControllerEpoch(controllerEpoch)
+        .setLeader(leader)
+        .setLeaderEpoch(leaderEpoch)
+        .setIsr(isr)
+        .setZkVersion(3)
+        .setReplicas(replicas))
       val version = ApiKeys.UPDATE_METADATA.latestVersion
       val updateMetadataRequest = new UpdateMetadataRequest.Builder(version, 2, controllerEpoch, brokerEpoch, partitionStates.asJava,
         brokers.asJava).build()
       cache.updateMetadata(15, updateMetadataRequest)
     }
 
-    val initialBrokerIds = (0 to 2).toSet
+    val initialBrokerIds = (0 to 2)
     updateCache(initialBrokerIds)
     val aliveBrokersFromCache = cache.getAliveBrokers
     // This should not change `aliveBrokersFromCache`
-    updateCache((0 to 3).toSet)
-    assertEquals(initialBrokerIds, aliveBrokersFromCache.map(_.id).toSet)
+    updateCache((0 to 3))
+    assertEquals(initialBrokerIds.toSet, aliveBrokersFromCache.map(_.id).toSet)
   }
 
 }
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index 857d06ecc50c5..438dedf5b2836 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -36,6 +36,7 @@ import kafka.utils.TestUtils.createBroker
 import kafka.utils.timer.MockTimer
 import kafka.utils.{MockScheduler, MockTime, TestUtils}
 import kafka.zk.KafkaZkClient
+import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
 import org.apache.kafka.common.record._
@@ -170,8 +171,16 @@ class ReplicaManagerTest {
         new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints))
       // Make this replica the leader.
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
-        collection.immutable.Map(new TopicPartition(topic, 0) ->
-          new LeaderAndIsrRequest.PartitionState(0, 0, 0, brokerList, 0, brokerList, false)).asJava,
+        Seq(new LeaderAndIsrPartitionState()
+          .setTopicName(topic)
+          .setPartitionIndex(0)
+          .setControllerEpoch(0)
+          .setLeader(0)
+          .setLeaderEpoch(0)
+          .setIsr(brokerList)
+          .setZkVersion(0)
+          .setReplicas(brokerList)
+          .setIsNew(false)).asJava,
         Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
       rm.becomeLeaderOrFollower(0, leaderAndIsrRequest1, (_, _) => ())
       rm.getPartitionOrException(new TopicPartition(topic, 0), expectLeader = true)
@@ -184,8 +193,16 @@ class ReplicaManagerTest {
 
       // Make this replica the follower
       val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
-        collection.immutable.Map(new TopicPartition(topic, 0) ->
-          new LeaderAndIsrRequest.PartitionState(0, 1, 1, brokerList, 0, brokerList, false)).asJava,
+        Seq(new LeaderAndIsrPartitionState()
+          .setTopicName(topic)
+          .setPartitionIndex(0)
+          .setControllerEpoch(0)
+          .setLeader(1)
+          .setLeaderEpoch(1)
+          .setIsr(brokerList)
+          .setZkVersion(0)
+          .setReplicas(brokerList)
+          .setIsNew(false)).asJava,
         Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
       rm.becomeLeaderOrFollower(1, leaderAndIsrRequest2, (_, _) => ())
 
@@ -209,8 +226,16 @@ class ReplicaManagerTest {
 
       // Make this replica the leader.
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
-        collection.immutable.Map(new TopicPartition(topic, 0) ->
-          new LeaderAndIsrRequest.PartitionState(0, 0, 0, brokerList, 0, brokerList, true)).asJava,
+        Seq(new LeaderAndIsrPartitionState()
+          .setTopicName(topic)
+          .setPartitionIndex(0)
+          .setControllerEpoch(0)
+          .setLeader(0)
+          .setLeaderEpoch(0)
+          .setIsr(brokerList)
+          .setZkVersion(0)
+          .setReplicas(brokerList)
+          .setIsNew(true)).asJava,
         Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
       replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest1, (_, _) => ())
       replicaManager.getPartitionOrException(new TopicPartition(topic, 0), expectLeader = true)
@@ -261,8 +286,16 @@ class ReplicaManagerTest {
 
       // Make this replica the leader.
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
-        collection.immutable.Map(new TopicPartition(topic, 0) ->
-          new LeaderAndIsrRequest.PartitionState(0, 0, 0, brokerList, 0, brokerList, true)).asJava,
+        Seq(new LeaderAndIsrPartitionState()
+          .setTopicName(topic)
+          .setPartitionIndex(0)
+          .setControllerEpoch(0)
+          .setLeader(0)
+          .setLeaderEpoch(0)
+          .setIsr(brokerList)
+          .setZkVersion(0)
+          .setReplicas(brokerList)
+          .setIsNew(true)).asJava,
         Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
       replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest1, (_, _) => ())
       replicaManager.getPartitionOrException(new TopicPartition(topic, 0), expectLeader = true)
@@ -358,8 +391,16 @@ class ReplicaManagerTest {
 
       // Make this replica the leader.
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
-        collection.immutable.Map(new TopicPartition(topic, 0) ->
-          new LeaderAndIsrRequest.PartitionState(0, 0, 0, brokerList, 0, brokerList, true)).asJava,
+        Seq(new LeaderAndIsrPartitionState()
+          .setTopicName(topic)
+          .setPartitionIndex(0)
+          .setControllerEpoch(0)
+          .setLeader(0)
+          .setLeaderEpoch(0)
+          .setIsr(brokerList)
+          .setZkVersion(0)
+          .setReplicas(brokerList)
+          .setIsNew(true)).asJava,
         Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
       replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest1, (_, _) => ())
       replicaManager.getPartitionOrException(new TopicPartition(topic, 0), expectLeader = true)
@@ -425,8 +466,16 @@ class ReplicaManagerTest {
 
       // Make this replica the leader.
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
-        collection.immutable.Map(new TopicPartition(topic, 0) ->
-          new LeaderAndIsrRequest.PartitionState(0, 0, 0, brokerList, 0, brokerList, false)).asJava,
+        Seq(new LeaderAndIsrPartitionState()
+          .setTopicName(topic)
+          .setPartitionIndex(0)
+          .setControllerEpoch(0)
+          .setLeader(0)
+          .setLeaderEpoch(0)
+          .setIsr(brokerList)
+          .setZkVersion(0)
+          .setReplicas(brokerList)
+          .setIsNew(false)).asJava,
         Set(new Node(0, "host1", 0), new Node(1, "host2", 1), new Node(2, "host2", 2)).asJava).build()
       rm.becomeLeaderOrFollower(0, leaderAndIsrRequest1, (_, _) => ())
       rm.getPartitionOrException(new TopicPartition(topic, 0), expectLeader = true)
@@ -469,10 +518,18 @@ class ReplicaManagerTest {
       val replicas = aliveBrokersIds.toList.map(Int.box).asJava
 
       // Broker 0 becomes leader of the partition
-      val leaderAndIsrPartitionState = new LeaderAndIsrRequest.PartitionState(0, 0, leaderEpoch,
-        replicas, 0, replicas, true)
+      val leaderAndIsrPartitionState = new LeaderAndIsrPartitionState()
+        .setTopicName(topic)
+        .setPartitionIndex(0)
+        .setControllerEpoch(0)
+        .setLeader(0)
+        .setLeaderEpoch(leaderEpoch)
+        .setIsr(replicas)
+        .setZkVersion(0)
+        .setReplicas(replicas)
+        .setIsNew(true)
       val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
-        Map(tp -> leaderAndIsrPartitionState).asJava,
+        Seq(leaderAndIsrPartitionState).asJava,
         Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
       val leaderAndIsrResponse = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest, (_, _) => ())
       assertEquals(Errors.NONE, leaderAndIsrResponse.error)
@@ -565,9 +622,27 @@ class ReplicaManagerTest {
       val partition0Replicas = Seq[Integer](0, 1).asJava
       val partition1Replicas = Seq[Integer](0, 2).asJava
       val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
-        collection.immutable.Map(
-          tp0 -> new LeaderAndIsrRequest.PartitionState(0, 0, 0, partition0Replicas, 0, partition0Replicas, true),
-          tp1 -> new LeaderAndIsrRequest.PartitionState(0, 0, 0, partition1Replicas, 0, partition1Replicas, true)
+        Seq(
+          new LeaderAndIsrPartitionState()
+            .setTopicName(tp0.topic)
+            .setPartitionIndex(tp0.partition)
+            .setControllerEpoch(0)
+            .setLeader(0)
+            .setLeaderEpoch(0)
+            .setIsr(partition0Replicas)
+            .setZkVersion(0)
+            .setReplicas(partition0Replicas)
+            .setIsNew(true),
+          new LeaderAndIsrPartitionState()
+            .setTopicName(tp1.topic)
+            .setPartitionIndex(tp1.partition)
+            .setControllerEpoch(0)
+            .setLeader(0)
+            .setLeaderEpoch(0)
+            .setIsr(partition1Replicas)
+            .setZkVersion(0)
+            .setReplicas(partition1Replicas)
+            .setIsNew(true)
         ).asJava,
         Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
       replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest, (_, _) => ())
@@ -653,11 +728,12 @@ class ReplicaManagerTest {
       topicPartition, leaderEpoch + leaderEpochIncrement, followerBrokerId, leaderBrokerId, countDownLatch, expectTruncation = true)
 
     // Initialize partition state to follower, with leader = 1, leaderEpoch = 1
-    val partition = replicaManager.createPartition(new TopicPartition(topic, topicPartition))
+    val tp = new TopicPartition(topic, topicPartition)
+    val partition = replicaManager.createPartition(tp)
     val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
     partition.createLogIfNotExists(followerBrokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
     partition.makeFollower(controllerId,
-      leaderAndIsrPartitionState(leaderEpoch, leaderBrokerId, aliveBrokerIds),
+      leaderAndIsrPartitionState(tp, leaderEpoch, leaderBrokerId, aliveBrokerIds),
       correlationId, offsetCheckpoints)
 
     // Make local partition a follower - because epoch increased by more than 1, truncation should
@@ -665,8 +741,7 @@ class ReplicaManagerTest {
     leaderEpoch += leaderEpochIncrement
     val leaderAndIsrRequest0 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion,
       controllerId, controllerEpoch, brokerEpoch,
-      collection.immutable.Map(new TopicPartition(topic, topicPartition) ->
-        leaderAndIsrPartitionState(leaderEpoch, leaderBrokerId, aliveBrokerIds)).asJava,
+      Seq(leaderAndIsrPartitionState(tp, leaderEpoch, leaderBrokerId, aliveBrokerIds)).asJava,
       Set(new Node(followerBrokerId, "host1", 0),
         new Node(leaderBrokerId, "host2", 1)).asJava).build()
     replicaManager.becomeLeaderOrFollower(correlationId, leaderAndIsrRequest0,
@@ -693,13 +768,14 @@ class ReplicaManagerTest {
       topicPartition, leaderEpoch + leaderEpochIncrement, followerBrokerId,
       leaderBrokerId, countDownLatch, expectTruncation = true)
 
-    val partition = replicaManager.createPartition(new TopicPartition(topic, topicPartition))
+    val tp = new TopicPartition(topic, topicPartition)
+    val partition = replicaManager.createPartition(tp)
 
     val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
     partition.createLogIfNotExists(leaderBrokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
     partition.makeLeader(
       controllerId,
-      leaderAndIsrPartitionState(leaderEpoch, leaderBrokerId, aliveBrokerIds),
+      leaderAndIsrPartitionState(tp, leaderEpoch, leaderBrokerId, aliveBrokerIds),
       correlationId,
       offsetCheckpoints
     )
@@ -737,8 +813,16 @@ class ReplicaManagerTest {
 
     // Make this replica the follower
     val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
-      collection.immutable.Map(new TopicPartition(topic, 0) ->
-        new LeaderAndIsrRequest.PartitionState(0, 1, 1, brokerList, 0, brokerList, false)).asJava,
+      Seq(new LeaderAndIsrPartitionState()
+        .setTopicName(topic)
+        .setPartitionIndex(0)
+        .setControllerEpoch(0)
+        .setLeader(1)
+        .setLeaderEpoch(1)
+        .setIsr(brokerList)
+        .setZkVersion(0)
+        .setReplicas(brokerList)
+        .setIsNew(false)).asJava,
       Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
     replicaManager.becomeLeaderOrFollower(1, leaderAndIsrRequest2, (_, _) => ())
 
@@ -778,8 +862,16 @@ class ReplicaManagerTest {
 
     // Make this replica the follower
     val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
-      collection.immutable.Map(new TopicPartition(topic, 0) ->
-        new LeaderAndIsrRequest.PartitionState(0, 0, 1, brokerList, 0, brokerList, false)).asJava,
+      Seq(new LeaderAndIsrPartitionState()
+        .setTopicName(topic)
+        .setPartitionIndex(0)
+        .setControllerEpoch(0)
+        .setLeader(0)
+        .setLeaderEpoch(1)
+        .setIsr(brokerList)
+        .setZkVersion(0)
+        .setReplicas(brokerList)
+        .setIsNew(false)).asJava,
       Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
     replicaManager.becomeLeaderOrFollower(1, leaderAndIsrRequest2, (_, _) => ())
 
@@ -960,11 +1052,20 @@ class ReplicaManagerTest {
     (replicaManager, mockLogMgr)
   }
 
-  private def leaderAndIsrPartitionState(leaderEpoch: Int,
+  private def leaderAndIsrPartitionState(topicPartition: TopicPartition,
+                                         leaderEpoch: Int,
                                          leaderBrokerId: Int,
-                                         aliveBrokerIds: Seq[Integer]) : LeaderAndIsrRequest.PartitionState = {
-    new LeaderAndIsrRequest.PartitionState(controllerEpoch, leaderBrokerId, leaderEpoch, aliveBrokerIds.asJava,
-      zkVersion, aliveBrokerIds.asJava, false)
+                                         aliveBrokerIds: Seq[Integer]): LeaderAndIsrPartitionState = {
+    new LeaderAndIsrPartitionState()
+      .setTopicName(topic)
+      .setPartitionIndex(topicPartition.partition)
+      .setControllerEpoch(controllerEpoch)
+      .setLeader(leaderBrokerId)
+      .setLeaderEpoch(leaderEpoch)
+      .setIsr(aliveBrokerIds.asJava)
+      .setZkVersion(zkVersion)
+      .setReplicas(aliveBrokerIds.asJava)
+      .setIsNew(false)
   }
 
   private class CallbackResult[T] {
@@ -1117,11 +1218,27 @@ class ReplicaManagerTest {
 
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion,
         controllerId, 0, brokerEpoch,
-        collection.immutable.Map(
-          tp0 -> new LeaderAndIsrRequest.PartitionState(controllerEpoch, 0, leaderEpoch,
-            partition0Replicas, 0, partition0Replicas, true),
-          tp1 -> new LeaderAndIsrRequest.PartitionState(controllerEpoch, 1, leaderEpoch,
-            partition1Replicas, 0, partition1Replicas, true)
+        Seq(
+          new LeaderAndIsrPartitionState()
+            .setTopicName(tp0.topic)
+            .setPartitionIndex(tp0.partition)
+            .setControllerEpoch(controllerEpoch)
+            .setLeader(0)
+            .setLeaderEpoch(leaderEpoch)
+            .setIsr(partition0Replicas)
+            .setZkVersion(0)
+            .setReplicas(partition0Replicas)
+            .setIsNew(true),
+          new LeaderAndIsrPartitionState()
+            .setTopicName(tp1.topic)
+            .setPartitionIndex(tp1.partition)
+            .setControllerEpoch(controllerEpoch)
+            .setLeader(1)
+            .setLeaderEpoch(leaderEpoch)
+            .setIsr(partition1Replicas)
+            .setZkVersion(0)
+            .setReplicas(partition1Replicas)
+            .setIsNew(true)
         ).asJava,
         Set(new Node(0, "host0", 0), new Node(1, "host1", 1)).asJava).build()
 
@@ -1131,11 +1248,27 @@ class ReplicaManagerTest {
       // make broker 0 the leader of partition 1 so broker 1 loses its leadership position
       val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, controllerId,
         controllerEpoch, brokerEpoch,
-        collection.immutable.Map(
-          tp0 -> new LeaderAndIsrRequest.PartitionState(controllerEpoch, 0, leaderEpoch + leaderEpochIncrement,
-            partition0Replicas, 0, partition0Replicas, true),
-          tp1 -> new LeaderAndIsrRequest.PartitionState(controllerEpoch, 0, leaderEpoch + leaderEpochIncrement,
-            partition1Replicas, 0, partition1Replicas, true)
+        Seq(
+          new LeaderAndIsrPartitionState()
+            .setTopicName(tp0.topic)
+            .setPartitionIndex(tp0.partition)
+            .setControllerEpoch(controllerEpoch)
+            .setLeader(0)
+            .setLeaderEpoch(leaderEpoch + leaderEpochIncrement)
+            .setIsr(partition0Replicas)
+            .setZkVersion(0)
+            .setReplicas(partition0Replicas)
+            .setIsNew(true),
+          new LeaderAndIsrPartitionState()
+            .setTopicName(tp1.topic)
+            .setPartitionIndex(tp1.partition)
+            .setControllerEpoch(controllerEpoch)
+            .setLeader(0)
+            .setLeaderEpoch(leaderEpoch + leaderEpochIncrement)
+            .setIsr(partition1Replicas)
+            .setZkVersion(0)
+            .setReplicas(partition1Replicas)
+            .setIsNew(true)
         ).asJava,
         Set(new Node(0, "host0", 0), new Node(1, "host1", 1)).asJava).build()
 
@@ -1173,11 +1306,27 @@ class ReplicaManagerTest {
 
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion,
         controllerId, 0, brokerEpoch,
-        collection.immutable.Map(
-          tp0 -> new LeaderAndIsrRequest.PartitionState(controllerEpoch, 1, leaderEpoch,
-            partition0Replicas, 0, partition0Replicas, true),
-          tp1 -> new LeaderAndIsrRequest.PartitionState(controllerEpoch, 1, leaderEpoch,
-            partition1Replicas, 0, partition1Replicas, true)
+        Seq(
+          new LeaderAndIsrPartitionState()
+            .setTopicName(tp0.topic)
+            .setPartitionIndex(tp0.partition)
+            .setControllerEpoch(controllerEpoch)
+            .setLeader(1)
+            .setLeaderEpoch(leaderEpoch)
+            .setIsr(partition0Replicas)
+            .setZkVersion(0)
+            .setReplicas(partition0Replicas)
+            .setIsNew(true),
+          new LeaderAndIsrPartitionState()
+            .setTopicName(tp1.topic)
+            .setPartitionIndex(tp1.partition)
+            .setControllerEpoch(controllerEpoch)
+            .setLeader(1)
+            .setLeaderEpoch(leaderEpoch)
+            .setIsr(partition1Replicas)
+            .setZkVersion(0)
+            .setReplicas(partition1Replicas)
+            .setIsNew(true)
         ).asJava,
         Set(new Node(0, "host0", 0), new Node(1, "host1", 1)).asJava).build()
 
@@ -1187,11 +1336,27 @@ class ReplicaManagerTest {
       // make broker 0 the leader of partition 1 so broker 1 loses its leadership position
       val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, controllerId,
         controllerEpoch, brokerEpoch,
-        collection.immutable.Map(
-          tp0 -> new LeaderAndIsrRequest.PartitionState(controllerEpoch, 0, leaderEpoch + leaderEpochIncrement,
-            partition0Replicas, 0, partition0Replicas, true),
-          tp1 -> new LeaderAndIsrRequest.PartitionState(controllerEpoch, 0, leaderEpoch + leaderEpochIncrement,
-            partition1Replicas, 0, partition1Replicas, true)
+        Seq(
+          new LeaderAndIsrPartitionState()
+            .setTopicName(tp0.topic)
+            .setPartitionIndex(tp0.partition)
+            .setControllerEpoch(controllerEpoch)
+            .setLeader(0)
+            .setLeaderEpoch(leaderEpoch + leaderEpochIncrement)
+            .setIsr(partition0Replicas)
+            .setZkVersion(0)
+            .setReplicas(partition0Replicas)
+            .setIsNew(true),
+          new LeaderAndIsrPartitionState()
+            .setTopicName(tp1.topic)
+            .setPartitionIndex(tp1.partition)
+            .setControllerEpoch(controllerEpoch)
+            .setLeader(0)
+            .setLeaderEpoch(leaderEpoch + leaderEpochIncrement)
+            .setIsr(partition1Replicas)
+            .setZkVersion(0)
+            .setReplicas(partition1Replicas)
+            .setIsNew(true)
         ).asJava,
         Set(new Node(0, "host0", 0), new Node(1, "host1", 1)).asJava).build()
 
diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
index 34c293e01973c..4e0540b4f0585 100644
--- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
+++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
@@ -27,7 +27,9 @@ import org.apache.kafka.common.acl._
 import org.apache.kafka.common.config.ConfigResource
 import org.apache.kafka.common.message.CreateTopicsRequestData.{CreatableTopic, CreatableTopicCollection}
 import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection
+import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
 import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
+import org.apache.kafka.common.message.UpdateMetadataRequestData.{UpdateMetadataBroker, UpdateMetadataEndpoint, UpdateMetadataPartitionState}
 import org.apache.kafka.common.message._
 import org.apache.kafka.common.metrics.{KafkaMetric, Quota, Sensor}
 import org.apache.kafka.common.network.ListenerName
@@ -227,20 +229,39 @@ class RequestQuotaTest extends BaseRequestTest {
 
         case ApiKeys.LEADER_AND_ISR =>
           new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, brokerId, Int.MaxValue, Long.MaxValue,
-            Map(tp -> new LeaderAndIsrRequest.PartitionState(Int.MaxValue, brokerId, Int.MaxValue, List(brokerId).asJava,
-              2, Seq(brokerId).asJava, true)).asJava,
+            Seq(new LeaderAndIsrPartitionState()
+              .setTopicName(tp.topic)
+              .setPartitionIndex(tp.partition)
+              .setControllerEpoch(Int.MaxValue)
+              .setLeader(brokerId)
+              .setLeaderEpoch(Int.MaxValue)
+              .setIsr(List(brokerId).asJava)
+              .setZkVersion(2)
+              .setReplicas(Seq(brokerId).asJava)
+              .setIsNew(true)).asJava,
             Set(new Node(brokerId, "localhost", 0)).asJava)
 
         case ApiKeys.STOP_REPLICA =>
           new StopReplicaRequest.Builder(ApiKeys.STOP_REPLICA.latestVersion, brokerId, Int.MaxValue, Long.MaxValue, true, Set(tp).asJava)
 
         case ApiKeys.UPDATE_METADATA =>
-          val partitionState = Map(tp -> new UpdateMetadataRequest.PartitionState(
-            Int.MaxValue, brokerId, Int.MaxValue, List(brokerId).asJava, 2, Seq(brokerId).asJava, Seq.empty[Integer].asJava)).asJava
+          val partitionState = Seq(new UpdateMetadataPartitionState()
+            .setTopicName(tp.topic)
+            .setPartitionIndex(tp.partition)
+            .setControllerEpoch(Int.MaxValue)
+            .setLeader(brokerId)
+            .setLeaderEpoch(Int.MaxValue)
+            .setIsr(List(brokerId).asJava)
+            .setZkVersion(2)
+            .setReplicas(Seq(brokerId).asJava)).asJava
           val securityProtocol = SecurityProtocol.PLAINTEXT
-          val brokers = Set(new UpdateMetadataRequest.Broker(brokerId,
-            Seq(new UpdateMetadataRequest.EndPoint("localhost", 0, securityProtocol,
-            ListenerName.forSecurityProtocol(securityProtocol))).asJava, null)).asJava
+          val brokers = Seq(new UpdateMetadataBroker()
+            .setId(brokerId)
+            .setEndpoints(Seq(new UpdateMetadataEndpoint()
+              .setHost("localhost")
+              .setPort(0)
+              .setSecurityProtocol(securityProtocol.id)
+              .setListener(ListenerName.forSecurityProtocol(securityProtocol).value)).asJava)).asJava
           new UpdateMetadataRequest.Builder(ApiKeys.UPDATE_METADATA.latestVersion, brokerId, Int.MaxValue, Long.MaxValue, partitionState, brokers)
 
         case ApiKeys.CONTROLLED_SHUTDOWN =>
diff --git a/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala b/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala
index a5883ca0a1029..33e4566800380 100755
--- a/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala
@@ -233,7 +233,7 @@ class ServerShutdownTest extends ZooKeeperTestHarness {
 
       // Initiate a sendRequest and wait until connection is established and one byte is received by the peer
       val requestBuilder = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion,
-        controllerId, 1, 0L, Map.empty.asJava, brokerAndEpochs.keys.map(_.node(listenerName)).toSet.asJava)
+        controllerId, 1, 0L, Seq.empty.asJava, brokerAndEpochs.keys.map(_.node(listenerName)).toSet.asJava)
       controllerChannelManager.sendRequest(1, requestBuilder)
       receiveFuture.get(10, TimeUnit.SECONDS)
 
diff --git a/core/src/test/scala/unit/kafka/server/StopReplicaRequestTest.scala b/core/src/test/scala/unit/kafka/server/StopReplicaRequestTest.scala
index f246b253a2567..b1b43d069c0d5 100644
--- a/core/src/test/scala/unit/kafka/server/StopReplicaRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/StopReplicaRequestTest.scala
@@ -25,7 +25,6 @@ import org.junit.Assert._
 import org.junit.Test
 import collection.JavaConverters._
 
-
 class StopReplicaRequestTest extends BaseRequestTest {
   override val logDirCount = 2
   override val brokerCount: Int = 1
@@ -44,14 +43,16 @@ class StopReplicaRequestTest extends BaseRequestTest {
     val offlineDir = server.logManager.getLog(tp1).get.dir.getParent
     server.replicaManager.handleLogDirFailure(offlineDir, sendZkNotification = false)
 
-    for (i <- 1 to 2) {
+    for (_ <- 1 to 2) {
       val request1 = new StopReplicaRequest.Builder(1,
         server.config.brokerId, server.replicaManager.controllerEpoch, server.kafkaController.brokerEpoch,
         true, Set(tp0, tp1).asJava).build()
       val response1 = connectAndSend(request1, ApiKeys.STOP_REPLICA, controllerSocketServer)
-      val partitionErrors1 = StopReplicaResponse.parse(response1, request1.version).responses()
-      assertEquals(Errors.NONE, partitionErrors1.get(tp0))
-      assertEquals(Errors.KAFKA_STORAGE_ERROR, partitionErrors1.get(tp1))
+      val partitionErrors1 = StopReplicaResponse.parse(response1, request1.version).partitionErrors.asScala
+      assertEquals(Some(Errors.NONE.code),
+        partitionErrors1.find(pe => pe.topicName == tp0.topic && pe.partitionIndex == tp0.partition).map(_.errorCode))
+      assertEquals(Some(Errors.KAFKA_STORAGE_ERROR.code),
+        partitionErrors1.find(pe => pe.topicName == tp1.topic && pe.partitionIndex == tp1.partition).map(_.errorCode))
     }
   }
 
diff --git a/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala b/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala
index 4bf8fcca8e105..26de772a62fd2 100644
--- a/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala
@@ -231,10 +231,7 @@ class LeaderEpochIntegrationTest extends ZooKeeperTestHarness with Logging {
 
   private def waitForEpochChangeTo(topic: String, partition: Int, epoch: Int): Unit = {
     TestUtils.waitUntilTrue(() => {
-      brokers(0).metadataCache.getPartitionInfo(topic, partition) match {
-        case Some(m) => m.basePartitionState.leaderEpoch == epoch
-        case None => false
-      }
+      brokers(0).metadataCache.getPartitionInfo(topic, partition).exists(_.leaderEpoch == epoch)
     }, "Epoch didn't change")
   }
 
diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index c1951c010c53f..f3ccd16afdec3 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -910,16 +910,14 @@ object TestUtils extends Logging {
   def waitUntilMetadataIsPropagated(servers: Seq[KafkaServer], topic: String, partition: Int,
                                     timeout: Long = JTestUtils.DEFAULT_MAX_WAIT_MS): Int = {
     var leader: Int = -1
-    waitUntilTrue(() =>
-      servers.foldLeft(true) {
-        (result, server) =>
-          val partitionStateOpt = server.dataPlaneRequestProcessor.metadataCache.getPartitionInfo(topic, partition)
-          partitionStateOpt match {
-            case None => false
-            case Some(partitionState) =>
-              leader = partitionState.basePartitionState.leader
-              result && Request.isValidBrokerId(leader)
-          }
+    waitUntilTrue(
+      () => servers.forall { server =>
+        server.dataPlaneRequestProcessor.metadataCache.getPartitionInfo(topic, partition) match {
+          case Some(partitionState) if Request.isValidBrokerId(partitionState.leader) =>
+            leader = partitionState.leader
+            true
+          case _ => false
+        }
       },
       "Partition [%s,%d] metadata not propagated after %d ms".format(topic, partition, timeout),
       waitTimeMs = timeout)
diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/partition/UpdateFollowerFetchStateBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/partition/UpdateFollowerFetchStateBenchmark.java
index ee45b6419faec..35375a4b4f187 100644
--- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/partition/UpdateFollowerFetchStateBenchmark.java
+++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/partition/UpdateFollowerFetchStateBenchmark.java
@@ -33,7 +33,7 @@
 import kafka.server.checkpoints.OffsetCheckpoints;
 import kafka.utils.KafkaScheduler;
 import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.requests.LeaderAndIsrRequest;
+import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState;
 import org.apache.kafka.common.utils.Time;
 import org.mockito.Mockito;
 import org.openjdk.jmh.annotations.Benchmark;
@@ -54,7 +54,6 @@
 import java.io.File;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Properties;
 import java.util.UUID;
@@ -106,8 +105,14 @@ public void setUp() {
         replicas.add(0);
         replicas.add(1);
         replicas.add(2);
-        LeaderAndIsrRequest.PartitionState partitionState = new LeaderAndIsrRequest.PartitionState(
-                0, 0, 0, replicas, 1, replicas, true);
+        LeaderAndIsrPartitionState partitionState = new LeaderAndIsrPartitionState()
+            .setControllerEpoch(0)
+            .setLeader(0)
+            .setLeaderEpoch(0)
+            .setIsr(replicas)
+            .setZkVersion(1)
+            .setReplicas(replicas)
+            .setIsNew(true);
         PartitionStateStore partitionStateStore = Mockito.mock(PartitionStateStore.class);
         Mockito.when(partitionStateStore.fetchTopicConfig()).thenReturn(new Properties());
         partition = new Partition(topicPartition, 100,
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java b/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java
index a0a61b978e749..fad89b7d70133 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java
@@ -31,7 +31,7 @@
 import org.apache.kafka.common.Metric;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.header.Headers;
-import org.apache.kafka.common.requests.UpdateMetadataRequest;
+import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataPartitionState;
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.streams.KafkaStreams;
@@ -705,13 +705,13 @@ public static void waitUntilMetadataIsPropagated(final List servers
         TestUtils.waitForCondition(() -> {
             for (final KafkaServer server : servers) {
                 final MetadataCache metadataCache = server.dataPlaneRequestProcessor().metadataCache();
-                final Option partitionInfo =
+                final Option partitionInfo =
                         metadataCache.getPartitionInfo(topic, partition);
                 if (partitionInfo.isEmpty()) {
                     return false;
                 }
-                final UpdateMetadataRequest.PartitionState metadataPartitionState = partitionInfo.get();
-                if (!Request.isValidBrokerId(metadataPartitionState.basePartitionState.leader)) {
+                final UpdateMetadataPartitionState metadataPartitionState = partitionInfo.get();
+                if (!Request.isValidBrokerId(metadataPartitionState.leader())) {
                     return false;
                 }
             }

From 469f002810cbbd9f96400298dc853c33b7f4f750 Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Sat, 28 Sep 2019 23:34:16 -0700
Subject: [PATCH 0657/1071] MINOR: Add missing `+` in LogSegment.toString
 (#7408)

The closing `)` was previously being discarded.

Reviewers: Manikumar Reddy 
---
 core/src/main/scala/kafka/log/LogSegment.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/scala/kafka/log/LogSegment.scala b/core/src/main/scala/kafka/log/LogSegment.scala
index 38a46e8504102..3bdd96248d173 100755
--- a/core/src/main/scala/kafka/log/LogSegment.scala
+++ b/core/src/main/scala/kafka/log/LogSegment.scala
@@ -412,7 +412,7 @@ class LogSegment private[log] (val log: FileRecords,
   override def toString: String = "LogSegment(baseOffset=" + baseOffset +
     ", size=" + size +
     ", lastModifiedTime=" + lastModified +
-    ", largestTime=" + largestTimestamp
+    ", largestTime=" + largestTimestamp +
     ")"
 
   /**

From 9aa660786e46c1efbf5605a6a69136a1dac6edb9 Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Sun, 29 Sep 2019 08:53:41 -0700
Subject: [PATCH 0658/1071] MINOR: Update Jackson to 2.10.0 (#7411)

Guava hasn't been upgraded due to potential incompatibility with the reflections
library:

https://github.com/ronmamo/reflections/issues/194

Reviewers: Manikumar Reddy 
---
 gradle/dependencies.gradle | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle
index 045abbc0263ae..993273b9ff74d 100644
--- a/gradle/dependencies.gradle
+++ b/gradle/dependencies.gradle
@@ -71,8 +71,7 @@ versions += [
   grgit: "3.1.1",
   httpclient: "4.5.9",
   easymock: "4.0.2",
-  jackson: "2.9.9",
-  jacksonDatabind: "2.9.9.3",
+  jackson: "2.10.0",
   jacoco: "0.8.3",
   jetty: "9.4.20.v20190813",
   jersey: "2.28",
@@ -134,7 +133,7 @@ libs += [
   bcpkix: "org.bouncycastle:bcpkix-jdk15on:$versions.bcpkix",
   commonsCli: "commons-cli:commons-cli:$versions.commonsCli",
   easymock: "org.easymock:easymock:$versions.easymock",
-  jacksonDatabind: "com.fasterxml.jackson.core:jackson-databind:$versions.jacksonDatabind",
+  jacksonDatabind: "com.fasterxml.jackson.core:jackson-databind:$versions.jackson",
   jacksonDataformatCsv: "com.fasterxml.jackson.dataformat:jackson-dataformat-csv:$versions.jackson",
   jacksonModuleScala: "com.fasterxml.jackson.module:jackson-module-scala_$versions.baseScala:$versions.jackson",
   jacksonJDK8Datatypes: "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:$versions.jackson",

From 45c800ff0189bb8343f276aa1077fd01d0483860 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Siatkowski?= 
Date: Tue, 1 Oct 2019 00:53:25 +0200
Subject: [PATCH 0659/1071] KAFKA-8911: Using proper WindowSerdes constructors
 in their implicit definitions (#7352)

Detailed info is available in the ticket: https://issues.apache.org/jira/browse/KAFKA-8911

Briefly, implicit defs are calling empty constructors, which exists only for reflection object creation.
Therefore, while using the implicit definitons, a NPE occurs when Serde is called.

Reviewers: John Roesler , Bill Bejeck 
---
 .../scala/org/apache/kafka/streams/scala/Serdes.scala     | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/Serdes.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/Serdes.scala
index 02e5380e71079..92ff01ee953fe 100644
--- a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/Serdes.scala
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/Serdes.scala
@@ -37,9 +37,11 @@ object Serdes {
   implicit def Integer: Serde[Int] = JSerdes.Integer().asInstanceOf[Serde[Int]]
   implicit def JavaInteger: Serde[java.lang.Integer] = JSerdes.Integer()
 
-  implicit def timeWindowedSerde[T]: WindowedSerdes.TimeWindowedSerde[T] = new WindowedSerdes.TimeWindowedSerde[T]()
-  implicit def sessionWindowedSerde[T]: WindowedSerdes.SessionWindowedSerde[T] =
-    new WindowedSerdes.SessionWindowedSerde[T]()
+  implicit def timeWindowedSerde[T](implicit tSerde: Serde[T]): WindowedSerdes.TimeWindowedSerde[T] =
+    new WindowedSerdes.TimeWindowedSerde[T](tSerde)
+
+  implicit def sessionWindowedSerde[T](implicit tSerde: Serde[T]): WindowedSerdes.SessionWindowedSerde[T] =
+    new WindowedSerdes.SessionWindowedSerde[T](tSerde)
 
   def fromFn[T >: Null](serializer: T => Array[Byte], deserializer: Array[Byte] => Option[T]): Serde[T] =
     JSerdes.serdeFrom(

From d53eab16b28a44e24fa2f9d592f536835b29ea22 Mon Sep 17 00:00:00 2001
From: Bill Bejeck 
Date: Mon, 30 Sep 2019 19:28:41 -0400
Subject: [PATCH 0660/1071] MINOR: Adjust logic of conditions to set number of
 partitions in step zero of assignment. (#7419)

A minor change in logic to account for repartition topics where we might not have the num partitions yet in the metadata.

Ran all existing tests plus all streams system tests.

Reviewers: John Roesler , Guozhang Wang 
---
 .../processor/internals/StreamsPartitionAssignor.java    | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
index b207e1de39458..244544fa7e8a0 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
@@ -384,12 +384,13 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
                                 // if this topic is one of the sink topics of this topology,
                                 // use the maximum of all its source topic partitions as the number of partitions
                                 for (final String sourceTopicName : otherTopicsInfo.sourceTopics) {
-                                    final int numPartitionsCandidate;
+                                    int numPartitionsCandidate = 0;
                                     // It is possible the sourceTopic is another internal topic, i.e,
                                     // map().join().join(map())
-                                    if (repartitionTopicMetadata.containsKey(sourceTopicName)
-                                        && repartitionTopicMetadata.get(sourceTopicName).numberOfPartitions().isPresent()) {
-                                        numPartitionsCandidate = repartitionTopicMetadata.get(sourceTopicName).numberOfPartitions().get();
+                                    if (repartitionTopicMetadata.containsKey(sourceTopicName)) {
+                                        if (repartitionTopicMetadata.get(sourceTopicName).numberOfPartitions().isPresent()) {
+                                            numPartitionsCandidate = repartitionTopicMetadata.get(sourceTopicName).numberOfPartitions().get();
+                                        }
                                     } else {
                                         final Integer count = metadata.partitionCountForTopic(sourceTopicName);
                                         if (count == null) {

From ef2fbfda24115fbadc6a55542ec76d3bedff17d5 Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Mon, 30 Sep 2019 19:56:12 -0500
Subject: [PATCH 0661/1071] KAFKA-8609: Add rebalance-latency-total (#7401)

In addition to the existing metrics added in KAFKA-8609, add the total (cumulative) time spent during rebalances.

Reviewers: Guozhang Wang 
---
 .../clients/consumer/internals/AbstractCoordinator.java      | 5 +++++
 .../clients/consumer/internals/AbstractCoordinatorTest.java  | 2 ++
 2 files changed, 7 insertions(+)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index fbd15d8e905fe..a44e7db4367d3 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -43,6 +43,7 @@
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
 import org.apache.kafka.common.metrics.stats.CumulativeCount;
+import org.apache.kafka.common.metrics.stats.CumulativeSum;
 import org.apache.kafka.common.metrics.stats.Max;
 import org.apache.kafka.common.metrics.stats.Meter;
 import org.apache.kafka.common.metrics.stats.Rate;
@@ -1059,6 +1060,10 @@ public GroupCoordinatorMetrics(Metrics metrics, String metricGrpPrefix) {
                 this.metricGrpName,
                 "The max time taken for a group to complete a successful rebalance, which may be composed of " +
                     "several failed re-trials until it succeeded"), new Max());
+            this.successfulRebalanceSensor.add(metrics.metricName("rebalance-latency-total",
+                this.metricGrpName,
+                "The total number of milliseconds this consumer has spent in successful rebalances since creation"),
+                new CumulativeSum());
             this.successfulRebalanceSensor.add(
                 metrics.metricName("rebalance-total",
                     this.metricGrpName,
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
index 1936fcc3c3106..a8b22bdb42a17 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
@@ -163,6 +163,7 @@ public void testMetrics() {
         assertNotNull(getMetric("sync-total"));
         assertNotNull(getMetric("rebalance-latency-avg"));
         assertNotNull(getMetric("rebalance-latency-max"));
+        assertNotNull(getMetric("rebalance-latency-total"));
         assertNotNull(getMetric("rebalance-rate-per-hour"));
         assertNotNull(getMetric("rebalance-total"));
         assertNotNull(getMetric("last-rebalance-seconds-ago"));
@@ -207,6 +208,7 @@ public void testMetrics() {
 
         assertEquals(3.0d, getMetric("rebalance-latency-avg").metricValue());
         assertEquals(6.0d, getMetric("rebalance-latency-max").metricValue());
+        assertEquals(9.0d, getMetric("rebalance-latency-total").metricValue());
         assertEquals(360.0d, getMetric("rebalance-rate-per-hour").metricValue());
         assertEquals(3.0d, getMetric("rebalance-total").metricValue());
 

From 9fbb0de5fc493f9186d23ccdf03b772ccc01e1d4 Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Mon, 30 Sep 2019 18:11:37 -0700
Subject: [PATCH 0662/1071] KAFKA-8927: Deprecate PartitionGrouper interface
 (#7376)

Reviewers: Bruno Cadonna , Guozhang Wang 
---
 .../apache/kafka/streams/StreamsConfig.java   | 22 +++++++++++++------
 .../processor/DefaultPartitionGrouper.java    |  3 +++
 .../streams/processor/PartitionGrouper.java   |  3 +++
 .../internals/StreamsPartitionAssignor.java   |  6 ++---
 .../assignment/AssignorConfiguration.java     | 10 +++++----
 .../kafka/streams/StreamsConfigTest.java      | 22 +++++++++++++++++--
 .../DefaultPartitionGrouperTest.java          |  1 +
 .../SingleGroupPartitionGrouperStub.java      | 11 +++++-----
 .../StreamsPartitionAssignorTest.java         |  1 +
 9 files changed, 57 insertions(+), 22 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
index cfedee572f508..eb6faffd45b03 100644
--- a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
+++ b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
@@ -37,7 +37,6 @@
 import org.apache.kafka.streams.errors.LogAndFailExceptionHandler;
 import org.apache.kafka.streams.errors.ProductionExceptionHandler;
 import org.apache.kafka.streams.errors.StreamsException;
-import org.apache.kafka.streams.processor.DefaultPartitionGrouper;
 import org.apache.kafka.streams.processor.FailOnInvalidTimestamp;
 import org.apache.kafka.streams.processor.TimestampExtractor;
 import org.apache.kafka.streams.processor.internals.StreamsPartitionAssignor;
@@ -129,6 +128,7 @@
  * @see ConsumerConfig
  * @see ProducerConfig
  */
+@SuppressWarnings("deprecation")
 public class StreamsConfig extends AbstractConfig {
 
     private final static Logger log = LoggerFactory.getLogger(StreamsConfig.class);
@@ -422,11 +422,6 @@ public class StreamsConfig extends AbstractConfig {
     public static final String NUM_STREAM_THREADS_CONFIG = "num.stream.threads";
     private static final String NUM_STREAM_THREADS_DOC = "The number of threads to execute stream processing.";
 
-    /** {@code partition.grouper} */
-    @SuppressWarnings("WeakerAccess")
-    public static final String PARTITION_GROUPER_CLASS_CONFIG = "partition.grouper";
-    private static final String PARTITION_GROUPER_CLASS_DOC = "Partition grouper class that implements the org.apache.kafka.streams.processor.PartitionGrouper interface.";
-
     /** {@code poll.ms} */
     @SuppressWarnings("WeakerAccess")
     public static final String POLL_MS_CONFIG = "poll.ms";
@@ -508,6 +503,16 @@ public class StreamsConfig extends AbstractConfig {
     public static final String WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_CONFIG = "windowstore.changelog.additional.retention.ms";
     private static final String WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_DOC = "Added to a windows maintainMs to ensure data is not deleted from the log prematurely. Allows for clock drift. Default is 1 day";
 
+    // deprecated
+
+    /** {@code partition.grouper} */
+    @SuppressWarnings("WeakerAccess")
+    @Deprecated
+    public static final String PARTITION_GROUPER_CLASS_CONFIG = "partition.grouper";
+    private static final String PARTITION_GROUPER_CLASS_DOC = "Partition grouper class that implements the org.apache.kafka.streams.processor.PartitionGrouper interface." +
+        " WARNING: This config is deprecated and will be removed in 3.0.0 release.";
+
+
     private static final String[] NON_CONFIGURABLE_CONSUMER_DEFAULT_CONFIGS = new String[] {ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG};
     private static final String[] NON_CONFIGURABLE_CONSUMER_EOS_CONFIGS = new String[] {ConsumerConfig.ISOLATION_LEVEL_CONFIG};
     private static final String[] NON_CONFIGURABLE_PRODUCER_EOS_CONFIGS = new String[] {ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG,
@@ -671,7 +676,7 @@ public class StreamsConfig extends AbstractConfig {
                     CommonClientConfigs.METRICS_SAMPLE_WINDOW_MS_DOC)
             .define(PARTITION_GROUPER_CLASS_CONFIG,
                     Type.CLASS,
-                    DefaultPartitionGrouper.class.getName(),
+                    org.apache.kafka.streams.processor.DefaultPartitionGrouper.class.getName(),
                     Importance.LOW,
                     PARTITION_GROUPER_CLASS_DOC)
             .define(POLL_MS_CONFIG,
@@ -901,6 +906,9 @@ protected StreamsConfig(final Map props,
                             final boolean doLog) {
         super(CONFIG, props, doLog);
         eosEnabled = EXACTLY_ONCE.equals(getString(PROCESSING_GUARANTEE_CONFIG));
+        if (props.containsKey(PARTITION_GROUPER_CLASS_CONFIG)) {
+            log.warn("Configuration parameter `{}` is deprecated and will be removed in 3.0.0 release.", PARTITION_GROUPER_CLASS_CONFIG);
+        }
     }
 
     @Override
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/DefaultPartitionGrouper.java b/streams/src/main/java/org/apache/kafka/streams/processor/DefaultPartitionGrouper.java
index 5b735bb3263b9..c6dfdb9665c95 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/DefaultPartitionGrouper.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/DefaultPartitionGrouper.java
@@ -36,7 +36,10 @@
  * Join operations requires that topics of the joining entities are copartitoned, i.e., being partitioned by the same key and having the same
  * number of partitions. Copartitioning is ensured by having the same number of partitions on
  * joined topics, and by using the serialization and Producer's default partitioner.
+ *
+ * @deprecated since 2.4 release; will be removed in 3.0.0 via KAFKA-7785
  */
+@Deprecated
 public class DefaultPartitionGrouper implements PartitionGrouper {
 
     private static final Logger log = LoggerFactory.getLogger(DefaultPartitionGrouper.class);
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/PartitionGrouper.java b/streams/src/main/java/org/apache/kafka/streams/processor/PartitionGrouper.java
index 27db594bbad9b..ecbc3c6e74d3d 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/PartitionGrouper.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/PartitionGrouper.java
@@ -29,7 +29,10 @@
  * such that each generated partition group is assigned with a distinct {@link TaskId};
  * the created task ids will then be assigned to Kafka Streams instances that host the stream
  * processing application.
+ *
+ * @deprecated since 2.4 release; will be removed in 3.0.0 via KAFKA-7785
  */
+@Deprecated
 public interface PartitionGrouper {
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
index 244544fa7e8a0..11ea3209d4105 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
@@ -29,7 +29,6 @@
 import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.streams.errors.StreamsException;
 import org.apache.kafka.streams.errors.TaskAssignmentException;
-import org.apache.kafka.streams.processor.PartitionGrouper;
 import org.apache.kafka.streams.processor.TaskId;
 import org.apache.kafka.streams.processor.internals.assignment.AssignmentInfo;
 import org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration;
@@ -53,9 +52,9 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
-import java.util.stream.Collectors;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
 
 import static org.apache.kafka.common.utils.Utils.getHost;
 import static org.apache.kafka.common.utils.Utils.getPort;
@@ -159,7 +158,8 @@ public String toString() {
     private int numStandbyReplicas;
 
     private TaskManager taskManager;
-    private PartitionGrouper partitionGrouper;
+    @SuppressWarnings("deprecation")
+    private org.apache.kafka.streams.processor.PartitionGrouper partitionGrouper;
     private AtomicInteger assignmentErrorCode;
 
     protected int usedSubscriptionMetadataVersion = LATEST_SUPPORTED_VERSION;
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorConfiguration.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorConfiguration.java
index 080c6145f663a..ac88f2f78e4ba 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorConfiguration.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorConfiguration.java
@@ -23,7 +23,6 @@
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.internals.QuietStreamsConfig;
-import org.apache.kafka.streams.processor.PartitionGrouper;
 import org.apache.kafka.streams.processor.internals.InternalTopicManager;
 import org.apache.kafka.streams.processor.internals.TaskManager;
 import org.slf4j.Logger;
@@ -41,13 +40,15 @@ public final class AssignorConfiguration {
     private final String logPrefix;
     private final Logger log;
     private final Integer numStandbyReplicas;
-    private final PartitionGrouper partitionGrouper;
+    @SuppressWarnings("deprecation")
+    private final org.apache.kafka.streams.processor.PartitionGrouper partitionGrouper;
     private final String userEndPoint;
     private final TaskManager taskManager;
     private final InternalTopicManager internalTopicManager;
     private final CopartitionedTopicsEnforcer copartitionedTopicsEnforcer;
     private final StreamsConfig streamsConfig;
 
+    @SuppressWarnings("deprecation")
     public AssignorConfiguration(final Map configs) {
         streamsConfig = new QuietStreamsConfig(configs);
 
@@ -60,7 +61,7 @@ public AssignorConfiguration(final Map configs) {
 
         partitionGrouper = streamsConfig.getConfiguredInstance(
             StreamsConfig.PARTITION_GROUPER_CLASS_CONFIG,
-            PartitionGrouper.class
+            org.apache.kafka.streams.processor.PartitionGrouper.class
         );
 
         final String configuredUserEndpoint = streamsConfig.getString(StreamsConfig.APPLICATION_SERVER_CONFIG);
@@ -194,7 +195,8 @@ public int getNumStandbyReplicas() {
         return numStandbyReplicas;
     }
 
-    public PartitionGrouper getPartitionGrouper() {
+    @SuppressWarnings("deprecation")
+    public org.apache.kafka.streams.processor.PartitionGrouper getPartitionGrouper() {
         return partitionGrouper;
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java b/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
index 461500e529996..ad607086f2c96 100644
--- a/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
@@ -32,6 +32,7 @@
 import org.apache.kafka.streams.processor.FailOnInvalidTimestamp;
 import org.apache.kafka.streams.processor.TimestampExtractor;
 import org.apache.kafka.streams.processor.internals.StreamsPartitionAssignor;
+import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender;
 import org.hamcrest.CoreMatchers;
 import org.junit.Before;
 import org.junit.Test;
@@ -52,9 +53,10 @@
 import static org.apache.kafka.streams.StreamsConfig.producerPrefix;
 import static org.apache.kafka.test.StreamsTestUtils.getStreamsConfig;
 import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.hasItem;
 import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.core.IsEqual.equalTo;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.IsEqual.equalTo;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThrows;
@@ -632,7 +634,6 @@ public void shouldSpecifyCorrectKeySerdeClassOnError() {
         }
     }
 
-    @SuppressWarnings("deprecation")
     @Test
     public void shouldSpecifyCorrectValueSerdeClassOnError() {
         final Properties props = getStreamsConfig();
@@ -702,6 +703,23 @@ public void shouldThrowConfigExceptionWhenOptimizationConfigNotValueInRange() {
         new StreamsConfig(props);
     }
 
+    @SuppressWarnings("deprecation")
+    @Test
+    public void shouldLogWarningWhenPartitionGrouperIsUsed() {
+        props.put(StreamsConfig.PARTITION_GROUPER_CLASS_CONFIG, org.apache.kafka.streams.processor.DefaultPartitionGrouper.class);
+
+        LogCaptureAppender.setClassLoggerToDebug(StreamsConfig.class);
+        final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
+
+        new StreamsConfig(props);
+
+        LogCaptureAppender.unregister(appender);
+
+        assertThat(
+            appender.getMessages(),
+            hasItem("Configuration parameter `" + StreamsConfig.PARTITION_GROUPER_CLASS_CONFIG + "` is deprecated and will be removed in 3.0.0 release."));
+    }
+
     static class MisconfiguredSerde implements Serde {
         @Override
         public void configure(final Map configs, final boolean isKey) {
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/DefaultPartitionGrouperTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/DefaultPartitionGrouperTest.java
index b1c36843db6fe..ab88a08a06e40 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/DefaultPartitionGrouperTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/DefaultPartitionGrouperTest.java
@@ -32,6 +32,7 @@
 import static org.apache.kafka.common.utils.Utils.mkSet;
 import static org.junit.Assert.assertEquals;
 
+@SuppressWarnings("deprecation")
 public class DefaultPartitionGrouperTest {
 
     private final List infos = Arrays.asList(
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/SingleGroupPartitionGrouperStub.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/SingleGroupPartitionGrouperStub.java
index 2378ea7233fe2..43e26d0b7043f 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/SingleGroupPartitionGrouperStub.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/SingleGroupPartitionGrouperStub.java
@@ -18,8 +18,6 @@
 
 import org.apache.kafka.common.Cluster;
 import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.streams.processor.DefaultPartitionGrouper;
-import org.apache.kafka.streams.processor.PartitionGrouper;
 import org.apache.kafka.streams.processor.TaskId;
 
 import java.util.HashMap;
@@ -29,8 +27,10 @@
 /**
  * Used for testing the assignment of a subset of a topology group, not the entire topology
  */
-public class SingleGroupPartitionGrouperStub implements PartitionGrouper {
-    private PartitionGrouper defaultPartitionGrouper = new DefaultPartitionGrouper();
+@SuppressWarnings("deprecation")
+public class SingleGroupPartitionGrouperStub implements org.apache.kafka.streams.processor.PartitionGrouper {
+    private org.apache.kafka.streams.processor.PartitionGrouper defaultPartitionGrouper =
+        new org.apache.kafka.streams.processor.DefaultPartitionGrouper();
 
     @Override
     public Map> partitionGroups(final Map> topicGroups, final Cluster metadata) {
@@ -40,7 +40,6 @@ public Map> partitionGroups(final Map> result = defaultPartitionGrouper.partitionGroups(includedTopicGroups, metadata);
-        return result;
+        return defaultPartitionGrouper.partitionGroups(includedTopicGroups, metadata);
     }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
index 2bc465b50099f..5f67a9875db5f 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
@@ -361,6 +361,7 @@ public void shouldAssignEvenlyAcrossConsumersOneClientMultipleThreads() {
         assertEquals(expectedInfo11TaskIds, info11.activeTasks());
     }
 
+    @SuppressWarnings("deprecation")
     @Test
     public void testAssignWithPartialTopology() {
         builder.addSource(null, "source1", null, null, null, "topic1");

From f6f24c470004063d14fa33d8680d1eff75c4f598 Mon Sep 17 00:00:00 2001
From: "Tu V. Tran" 
Date: Mon, 30 Sep 2019 19:29:36 -0700
Subject: [PATCH 0663/1071] KAFKA-8729, pt 2: Add error_records and
 error_message to PartitionResponse (#7150)

As noted in the KIP-467, the updated ProduceResponse is

```
Produce Response (Version: 8) => [responses] throttle_time_ms
  responses => topic [partition_responses]
    topic => STRING
    partition_responses => partition error_code base_offset log_append_time log_start_offset
      partition => INT32
      error_code => INT16
      base_offset => INT64
      log_append_time => INT64
      log_start_offset => INT64
      error_records => [INT32]         // new field, encodes the relative offset of the records that caused error
      error_message => STRING          // new field, encodes the error message that client can use to log itself
    throttle_time_ms => INT32
with a new error code:
```

INVALID_RECORD(86, "Some record has failed the validation on broker and hence be rejected.", InvalidRecordException::new);

Reviewers: Jason Gustafson , Magnus Edenhill , Guozhang Wang 
---
 .../clients/consumer/internals/Fetcher.java   |   6 +-
 .../{record => }/InvalidRecordException.java  |   6 +-
 .../apache/kafka/common/protocol/Errors.java  |   4 +-
 .../record/AbstractLegacyRecordBatch.java     |   1 +
 .../common/record/ControlRecordType.java      |   1 +
 .../kafka/common/record/DefaultRecord.java    |   1 +
 .../common/record/DefaultRecordBatch.java     |   6 +-
 .../common/record/EndTransactionMarker.java   |   1 +
 .../kafka/common/record/LegacyRecord.java     |   5 +-
 .../kafka/common/requests/ProduceRequest.java |  12 +-
 .../common/requests/ProduceResponse.java      | 106 +++++++++++++++---
 .../common/message/ProduceRequest.json        |   4 +-
 .../common/message/ProduceResponse.json       |  16 ++-
 .../kafka/common/message/MessageTest.java     |  72 ++++++++++++
 .../record/AbstractLegacyRecordBatchTest.java |   1 +
 .../common/record/DefaultRecordBatchTest.java |   6 +-
 .../common/record/DefaultRecordTest.java      |   1 +
 .../record/EndTransactionMarkerTest.java      |   1 +
 .../kafka/common/record/LegacyRecordTest.java |   3 +-
 .../common/record/SimpleLegacyRecordTest.java |   6 +-
 .../common/requests/ProduceRequestTest.java   |   2 +-
 .../common/requests/RequestResponseTest.java  |   8 ++
 .../kafka/connect/util/ConnectUtils.java      |   2 +-
 .../connect/runtime/WorkerSourceTaskTest.java |   2 +-
 core/src/main/scala/kafka/log/Log.scala       |   4 +-
 .../src/main/scala/kafka/log/LogSegment.scala |   7 +-
 .../main/scala/kafka/log/LogValidator.scala   |   7 +-
 .../kafka/server/AbstractFetcherThread.scala  |   4 +-
 .../test/scala/unit/kafka/log/LogTest.scala   |   2 +-
 .../unit/kafka/log/LogValidatorTest.scala     |   1 +
 .../kafka/server/ProduceRequestTest.scala     |   1 +
 31 files changed, 248 insertions(+), 51 deletions(-)
 rename clients/src/main/java/org/apache/kafka/common/{record => }/InvalidRecordException.java (85%)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
index 72aed9d867866..6b4ae916e4bc0 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
@@ -35,6 +35,7 @@
 import org.apache.kafka.common.Node;
 import org.apache.kafka.common.PartitionInfo;
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.CorruptRecordException;
 import org.apache.kafka.common.errors.InvalidTopicException;
 import org.apache.kafka.common.errors.RecordTooLargeException;
 import org.apache.kafka.common.errors.RetriableException;
@@ -56,7 +57,6 @@
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.record.BufferSupplier;
 import org.apache.kafka.common.record.ControlRecordType;
-import org.apache.kafka.common.record.InvalidRecordException;
 import org.apache.kafka.common.record.Record;
 import org.apache.kafka.common.record.RecordBatch;
 import org.apache.kafka.common.record.Records;
@@ -1422,7 +1422,7 @@ private void maybeEnsureValid(RecordBatch batch) {
             if (checkCrcs && currentBatch.magic() >= RecordBatch.MAGIC_VALUE_V2) {
                 try {
                     batch.ensureValid();
-                } catch (InvalidRecordException e) {
+                } catch (CorruptRecordException e) {
                     throw new KafkaException("Record batch for partition " + partition + " at offset " +
                             batch.baseOffset() + " is invalid, cause: " + e.getMessage());
                 }
@@ -1433,7 +1433,7 @@ private void maybeEnsureValid(Record record) {
             if (checkCrcs) {
                 try {
                     record.ensureValid();
-                } catch (InvalidRecordException e) {
+                } catch (CorruptRecordException e) {
                     throw new KafkaException("Record for partition " + partition + " at offset " + record.offset()
                             + " is invalid, cause: " + e.getMessage());
                 }
diff --git a/clients/src/main/java/org/apache/kafka/common/record/InvalidRecordException.java b/clients/src/main/java/org/apache/kafka/common/InvalidRecordException.java
similarity index 85%
rename from clients/src/main/java/org/apache/kafka/common/record/InvalidRecordException.java
rename to clients/src/main/java/org/apache/kafka/common/InvalidRecordException.java
index 49f616611589e..4c2815bb3bda5 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/InvalidRecordException.java
+++ b/clients/src/main/java/org/apache/kafka/common/InvalidRecordException.java
@@ -14,11 +14,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.kafka.common.record;
+package org.apache.kafka.common;
 
-import org.apache.kafka.common.errors.CorruptRecordException;
+import org.apache.kafka.common.errors.ApiException;
 
-public class InvalidRecordException extends CorruptRecordException {
+public class InvalidRecordException extends ApiException {
 
     private static final long serialVersionUID = 1;
 
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java
index 0425ef1a4d313..13ce6684b1c16 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.common.protocol;
 
+import org.apache.kafka.common.InvalidRecordException;
 import org.apache.kafka.common.errors.ApiException;
 import org.apache.kafka.common.errors.BrokerNotAvailableException;
 import org.apache.kafka.common.errors.ClusterAuthorizationException;
@@ -315,7 +316,8 @@ public enum Errors {
     NO_REASSIGNMENT_IN_PROGRESS(85, "No partition reassignment is in progress.",
             NoReassignmentInProgressException::new),
     GROUP_SUBSCRIBED_TO_TOPIC(86, "Deleting offsets of a topic is forbidden while the consumer group is actively subscribed to it.",
-        GroupSubscribedToTopicException::new);
+        GroupSubscribedToTopicException::new),
+    INVALID_RECORD(87, "This record has failed the validation on broker and hence be rejected.", InvalidRecordException::new);
 
     private static final Logger log = LoggerFactory.getLogger(Errors.class);
 
diff --git a/clients/src/main/java/org/apache/kafka/common/record/AbstractLegacyRecordBatch.java b/clients/src/main/java/org/apache/kafka/common/record/AbstractLegacyRecordBatch.java
index cf38d3123067e..83637640af49d 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/AbstractLegacyRecordBatch.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/AbstractLegacyRecordBatch.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.common.record;
 
+import org.apache.kafka.common.InvalidRecordException;
 import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.errors.CorruptRecordException;
 import org.apache.kafka.common.header.Header;
diff --git a/clients/src/main/java/org/apache/kafka/common/record/ControlRecordType.java b/clients/src/main/java/org/apache/kafka/common/record/ControlRecordType.java
index d5ead14df9126..ad41f1d9cd8dd 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/ControlRecordType.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/ControlRecordType.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.common.record;
 
+import org.apache.kafka.common.InvalidRecordException;
 import org.apache.kafka.common.protocol.types.Field;
 import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
diff --git a/clients/src/main/java/org/apache/kafka/common/record/DefaultRecord.java b/clients/src/main/java/org/apache/kafka/common/record/DefaultRecord.java
index bf1320ea30c4a..976b5567a3709 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/DefaultRecord.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/DefaultRecord.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.common.record;
 
+import org.apache.kafka.common.InvalidRecordException;
 import org.apache.kafka.common.header.Header;
 import org.apache.kafka.common.header.internals.RecordHeader;
 import org.apache.kafka.common.utils.ByteUtils;
diff --git a/clients/src/main/java/org/apache/kafka/common/record/DefaultRecordBatch.java b/clients/src/main/java/org/apache/kafka/common/record/DefaultRecordBatch.java
index 5b53f19dd07b8..6d79b268575ab 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/DefaultRecordBatch.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/DefaultRecordBatch.java
@@ -16,7 +16,9 @@
  */
 package org.apache.kafka.common.record;
 
+import org.apache.kafka.common.InvalidRecordException;
 import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.errors.CorruptRecordException;
 import org.apache.kafka.common.header.Header;
 import org.apache.kafka.common.utils.ByteBufferOutputStream;
 import org.apache.kafka.common.utils.ByteUtils;
@@ -144,11 +146,11 @@ public byte magic() {
     @Override
     public void ensureValid() {
         if (sizeInBytes() < RECORD_BATCH_OVERHEAD)
-            throw new InvalidRecordException("Record batch is corrupt (the size " + sizeInBytes() +
+            throw new CorruptRecordException("Record batch is corrupt (the size " + sizeInBytes() +
                     " is smaller than the minimum allowed overhead " + RECORD_BATCH_OVERHEAD + ")");
 
         if (!isValid())
-            throw new InvalidRecordException("Record is corrupt (stored crc = " + checksum()
+            throw new CorruptRecordException("Record is corrupt (stored crc = " + checksum()
                     + ", computed crc = " + computeChecksum() + ")");
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/record/EndTransactionMarker.java b/clients/src/main/java/org/apache/kafka/common/record/EndTransactionMarker.java
index 726b52a197378..4bf1ebf94a098 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/EndTransactionMarker.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/EndTransactionMarker.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.common.record;
 
+import org.apache.kafka.common.InvalidRecordException;
 import org.apache.kafka.common.protocol.types.Field;
 import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
diff --git a/clients/src/main/java/org/apache/kafka/common/record/LegacyRecord.java b/clients/src/main/java/org/apache/kafka/common/record/LegacyRecord.java
index 482c4a65efc1e..32c5aa81d7530 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/LegacyRecord.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/LegacyRecord.java
@@ -17,6 +17,7 @@
 package org.apache.kafka.common.record;
 
 import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.errors.CorruptRecordException;
 import org.apache.kafka.common.utils.ByteBufferOutputStream;
 import org.apache.kafka.common.utils.ByteUtils;
 import org.apache.kafka.common.utils.Checksums;
@@ -135,11 +136,11 @@ public TimestampType wrapperRecordTimestampType() {
      */
     public void ensureValid() {
         if (sizeInBytes() < RECORD_OVERHEAD_V0)
-            throw new InvalidRecordException("Record is corrupt (crc could not be retrieved as the record is too "
+            throw new CorruptRecordException("Record is corrupt (crc could not be retrieved as the record is too "
                     + "small, size = " + sizeInBytes() + ")");
 
         if (!isValid())
-            throw new InvalidRecordException("Record is corrupt (stored crc = " + checksum()
+            throw new CorruptRecordException("Record is corrupt (stored crc = " + checksum()
                     + ", computed crc = " + computeChecksum() + ")");
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java
index ad23f3f61d83f..932473cb869ea 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.common.requests;
 
+import org.apache.kafka.common.InvalidRecordException;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.errors.UnsupportedCompressionTypeException;
 import org.apache.kafka.common.protocol.ApiKeys;
@@ -26,7 +27,6 @@
 import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
 import org.apache.kafka.common.record.CompressionType;
-import org.apache.kafka.common.record.InvalidRecordException;
 import org.apache.kafka.common.record.MemoryRecords;
 import org.apache.kafka.common.record.MutableRecordBatch;
 import org.apache.kafka.common.record.RecordBatch;
@@ -120,9 +120,15 @@ public class ProduceRequest extends AbstractRequest {
      */
     private static final Schema PRODUCE_REQUEST_V7 = PRODUCE_REQUEST_V6;
 
+    /**
+     * V8 bumped up to add two new fields error_records offset list and error_message to {@link org.apache.kafka.common.requests.ProduceResponse.PartitionResponse}
+     * (See KIP-467)
+     */
+    private static final Schema PRODUCE_REQUEST_V8 = PRODUCE_REQUEST_V7;
+
     public static Schema[] schemaVersions() {
         return new Schema[] {PRODUCE_REQUEST_V0, PRODUCE_REQUEST_V1, PRODUCE_REQUEST_V2, PRODUCE_REQUEST_V3,
-            PRODUCE_REQUEST_V4, PRODUCE_REQUEST_V5, PRODUCE_REQUEST_V6, PRODUCE_REQUEST_V7};
+            PRODUCE_REQUEST_V4, PRODUCE_REQUEST_V5, PRODUCE_REQUEST_V6, PRODUCE_REQUEST_V7, PRODUCE_REQUEST_V8};
     }
 
     public static class Builder extends AbstractRequest.Builder {
@@ -337,6 +343,7 @@ public ProduceResponse getErrorResponse(int throttleTimeMs, Throwable e) {
             case 5:
             case 6:
             case 7:
+            case 8:
                 return new ProduceResponse(responseMap, throttleTimeMs);
             default:
                 throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
@@ -434,6 +441,7 @@ public static byte requiredMagicForVersion(short produceRequestVersion) {
             case 5:
             case 6:
             case 7:
+            case 8:
                 return RecordBatch.MAGIC_VALUE_V2;
 
             default:
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java
index 7d3e4fed36280..a6df88002d582 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java
@@ -28,6 +28,7 @@
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -53,28 +54,37 @@ public class ProduceResponse extends AbstractResponse {
     /**
      * Possible error code:
      *
-     * CORRUPT_MESSAGE (2)
-     * UNKNOWN_TOPIC_OR_PARTITION (3)
-     * NOT_LEADER_FOR_PARTITION (6)
-     * MESSAGE_TOO_LARGE (10)
-     * INVALID_TOPIC (17)
-     * RECORD_LIST_TOO_LARGE (18)
-     * NOT_ENOUGH_REPLICAS (19)
-     * NOT_ENOUGH_REPLICAS_AFTER_APPEND (20)
-     * INVALID_REQUIRED_ACKS (21)
-     * TOPIC_AUTHORIZATION_FAILED (29)
-     * UNSUPPORTED_FOR_MESSAGE_FORMAT (43)
-     * INVALID_PRODUCER_EPOCH (47)
-     * CLUSTER_AUTHORIZATION_FAILED (31)
-     * TRANSACTIONAL_ID_AUTHORIZATION_FAILED (53)
+     * {@link Errors#CORRUPT_MESSAGE}
+     * {@link Errors#UNKNOWN_TOPIC_OR_PARTITION}
+     * {@link Errors#NOT_LEADER_FOR_PARTITION}
+     * {@link Errors#MESSAGE_TOO_LARGE}
+     * {@link Errors#INVALID_TOPIC_EXCEPTION}
+     * {@link Errors#RECORD_LIST_TOO_LARGE}
+     * {@link Errors#NOT_ENOUGH_REPLICAS}
+     * {@link Errors#NOT_ENOUGH_REPLICAS_AFTER_APPEND}
+     * {@link Errors#INVALID_REQUIRED_ACKS}
+     * {@link Errors#TOPIC_AUTHORIZATION_FAILED}
+     * {@link Errors#UNSUPPORTED_FOR_MESSAGE_FORMAT}
+     * {@link Errors#INVALID_PRODUCER_EPOCH}
+     * {@link Errors#CLUSTER_AUTHORIZATION_FAILED}
+     * {@link Errors#TRANSACTIONAL_ID_AUTHORIZATION_FAILED}
+     * {@link Errors#INVALID_RECORD}
      */
 
     private static final String BASE_OFFSET_KEY_NAME = "base_offset";
     private static final String LOG_APPEND_TIME_KEY_NAME = "log_append_time";
     private static final String LOG_START_OFFSET_KEY_NAME = "log_start_offset";
+    private static final String ERROR_RECORDS_KEY_NAME = "error_records";
+    private static final String RELATIVE_OFFSET_KEY_NAME = "relative_offset";
+    private static final String RELATIVE_OFFSET_ERROR_MESSAGE_KEY_NAME = "relative_offset_error_message";
+    private static final String ERROR_MESSAGE_KEY_NAME = "error_message";
 
     private static final Field.Int64 LOG_START_OFFSET_FIELD = new Field.Int64(LOG_START_OFFSET_KEY_NAME,
             "The start offset of the log at the time this produce response was created", INVALID_OFFSET);
+    private static final Field.NullableStr RELATIVE_OFFSET_ERROR_MESSAGE_FIELD = new Field.NullableStr(RELATIVE_OFFSET_ERROR_MESSAGE_KEY_NAME,
+            "The error message of the record that caused the batch to be dropped");
+    private static final Field.NullableStr ERROR_MESSAGE_FIELD = new Field.NullableStr(ERROR_MESSAGE_KEY_NAME,
+            "The global error message summarizing the common root cause of the records that caused the batch to be dropped");
 
     private static final Schema PRODUCE_RESPONSE_V0 = new Schema(
             new Field(RESPONSES_KEY_NAME, new ArrayOf(new Schema(
@@ -149,9 +159,32 @@ public class ProduceResponse extends AbstractResponse {
      */
     private static final Schema PRODUCE_RESPONSE_V7 = PRODUCE_RESPONSE_V6;
 
+    /**
+     * V8 adds error_records and error_message. (see KIP-467)
+     */
+    public static final Schema PRODUCE_RESPONSE_V8 = new Schema(
+            new Field(RESPONSES_KEY_NAME, new ArrayOf(new Schema(
+                    TOPIC_NAME,
+                    new Field(PARTITION_RESPONSES_KEY_NAME, new ArrayOf(new Schema(
+                            PARTITION_ID,
+                            ERROR_CODE,
+                            new Field(BASE_OFFSET_KEY_NAME, INT64),
+                            new Field(LOG_APPEND_TIME_KEY_NAME, INT64, "The timestamp returned by broker after appending " +
+                                    "the messages. If CreateTime is used for the topic, the timestamp will be -1. " +
+                                    "If LogAppendTime is used for the topic, the timestamp will be the broker local " +
+                                    "time when the messages are appended."),
+                            LOG_START_OFFSET_FIELD,
+                            new Field(ERROR_RECORDS_KEY_NAME, new ArrayOf(new Schema(
+                                    new Field.Int32(RELATIVE_OFFSET_KEY_NAME, "The relative offset of the record " +
+                                            "that caused the batch to be dropped"),
+                                    RELATIVE_OFFSET_ERROR_MESSAGE_FIELD
+                            )), "The relative offsets of records that caused the batch to be dropped"),
+                            ERROR_MESSAGE_FIELD)))))),
+            THROTTLE_TIME_MS);
+
     public static Schema[] schemaVersions() {
         return new Schema[]{PRODUCE_RESPONSE_V0, PRODUCE_RESPONSE_V1, PRODUCE_RESPONSE_V2, PRODUCE_RESPONSE_V3,
-            PRODUCE_RESPONSE_V4, PRODUCE_RESPONSE_V5, PRODUCE_RESPONSE_V6, PRODUCE_RESPONSE_V7};
+            PRODUCE_RESPONSE_V4, PRODUCE_RESPONSE_V5, PRODUCE_RESPONSE_V6, PRODUCE_RESPONSE_V7, PRODUCE_RESPONSE_V8};
     }
 
     private final Map responses;
@@ -183,6 +216,7 @@ public ProduceResponse(Struct struct) {
         for (Object topicResponse : struct.getArray(RESPONSES_KEY_NAME)) {
             Struct topicRespStruct = (Struct) topicResponse;
             String topic = topicRespStruct.get(TOPIC_NAME);
+
             for (Object partResponse : topicRespStruct.getArray(PARTITION_RESPONSES_KEY_NAME)) {
                 Struct partRespStruct = (Struct) partResponse;
                 int partition = partRespStruct.get(PARTITION_ID);
@@ -190,8 +224,20 @@ public ProduceResponse(Struct struct) {
                 long offset = partRespStruct.getLong(BASE_OFFSET_KEY_NAME);
                 long logAppendTime = partRespStruct.getLong(LOG_APPEND_TIME_KEY_NAME);
                 long logStartOffset = partRespStruct.getOrElse(LOG_START_OFFSET_FIELD, INVALID_OFFSET);
+
+                Map errorRecords = new HashMap<>();
+                if (partRespStruct.hasField(ERROR_RECORDS_KEY_NAME)) {
+                    for (Object recordOffsetAndMessage : partRespStruct.getArray(ERROR_RECORDS_KEY_NAME)) {
+                        Struct recordOffsetAndMessageStruct = (Struct) recordOffsetAndMessage;
+                        Integer relativeOffset = recordOffsetAndMessageStruct.getInt(RELATIVE_OFFSET_KEY_NAME);
+                        String relativeOffsetErrorMessage = recordOffsetAndMessageStruct.getOrElse(RELATIVE_OFFSET_ERROR_MESSAGE_FIELD, "");
+                        errorRecords.put(relativeOffset, relativeOffsetErrorMessage);
+                    }
+                }
+
+                String errorMessage = partRespStruct.getOrElse(ERROR_MESSAGE_FIELD, "");
                 TopicPartition tp = new TopicPartition(topic, partition);
-                responses.put(tp, new PartitionResponse(error, offset, logAppendTime, logStartOffset));
+                responses.put(tp, new PartitionResponse(error, offset, logAppendTime, logStartOffset, errorRecords, errorMessage));
             }
         }
         this.throttleTimeMs = struct.getOrElse(THROTTLE_TIME_MS, DEFAULT_THROTTLE_TIME);
@@ -223,6 +269,18 @@ protected Struct toStruct(short version) {
                 if (partStruct.hasField(LOG_APPEND_TIME_KEY_NAME))
                     partStruct.set(LOG_APPEND_TIME_KEY_NAME, part.logAppendTime);
                 partStruct.setIfExists(LOG_START_OFFSET_FIELD, part.logStartOffset);
+
+                List errorRecords = new ArrayList<>();
+                for (Map.Entry recordOffsetAndMessage : part.errorRecords.entrySet()) {
+                    Struct recordOffsetAndMessageStruct = partStruct.instance(ERROR_RECORDS_KEY_NAME)
+                            .set(RELATIVE_OFFSET_KEY_NAME, recordOffsetAndMessage.getKey())
+                            .setIfExists(RELATIVE_OFFSET_ERROR_MESSAGE_FIELD, recordOffsetAndMessage.getValue());
+                    errorRecords.add(recordOffsetAndMessageStruct);
+                }
+
+                partStruct.setIfExists(ERROR_RECORDS_KEY_NAME, errorRecords.toArray());
+
+                partStruct.setIfExists(ERROR_MESSAGE_FIELD, part.errorMessage);
                 partitionArray.add(partStruct);
             }
             topicData.set(PARTITION_RESPONSES_KEY_NAME, partitionArray.toArray());
@@ -256,16 +314,28 @@ public static final class PartitionResponse {
         public long baseOffset;
         public long logAppendTime;
         public long logStartOffset;
+        public Map errorRecords;
+        public String errorMessage;
 
         public PartitionResponse(Errors error) {
             this(error, INVALID_OFFSET, RecordBatch.NO_TIMESTAMP, INVALID_OFFSET);
         }
 
         public PartitionResponse(Errors error, long baseOffset, long logAppendTime, long logStartOffset) {
+            this(error, baseOffset, logAppendTime, logStartOffset, Collections.emptyMap(), null);
+        }
+
+        public PartitionResponse(Errors error, long baseOffset, long logAppendTime, long logStartOffset, Map errorRecords) {
+            this(error, baseOffset, logAppendTime, logStartOffset, errorRecords, "");
+        }
+
+        public PartitionResponse(Errors error, long baseOffset, long logAppendTime, long logStartOffset, Map errorRecords, String errorMessage) {
             this.error = error;
             this.baseOffset = baseOffset;
             this.logAppendTime = logAppendTime;
             this.logStartOffset = logStartOffset;
+            this.errorRecords = errorRecords;
+            this.errorMessage = errorMessage;
         }
 
         @Override
@@ -280,6 +350,10 @@ public String toString() {
             b.append(logAppendTime);
             b.append(", logStartOffset: ");
             b.append(logStartOffset);
+            b.append(", errorRecords: ");
+            b.append(errorRecords);
+            b.append(", errorMessage: ");
+            b.append(errorMessage);
             b.append('}');
             return b.toString();
         }
diff --git a/clients/src/main/resources/common/message/ProduceRequest.json b/clients/src/main/resources/common/message/ProduceRequest.json
index 2da4ed793b295..5a1556d2652d6 100644
--- a/clients/src/main/resources/common/message/ProduceRequest.json
+++ b/clients/src/main/resources/common/message/ProduceRequest.json
@@ -28,7 +28,9 @@
   // Version 5 and 6 are the same as version 3.
   //
   // Starting in version 7, records can be produced using ZStandard compression.  See KIP-110.
-  "validVersions": "0-7",
+  //
+  // Starting in Version 8, response has ErrorRecords and ErrorMEssage. See KIP-467.
+  "validVersions": "0-8",
   "fields": [
     { "name": "TransactionalId", "type": "string", "versions": "3+", "nullableVersions": "0+", "entityType": "transactionalId",
       "about": "The transactional ID, or null if the producer is not transactional." },
diff --git a/clients/src/main/resources/common/message/ProduceResponse.json b/clients/src/main/resources/common/message/ProduceResponse.json
index 0659b4c63db53..690880d29d17b 100644
--- a/clients/src/main/resources/common/message/ProduceResponse.json
+++ b/clients/src/main/resources/common/message/ProduceResponse.json
@@ -27,7 +27,10 @@
   //
   // Version 5 added LogStartOffset to filter out spurious
   // OutOfOrderSequenceExceptions on the client.
-  "validVersions": "0-7",
+  //
+  // Version 8 added ErrorRecords and ErrorMessage to include information about
+  // records that cause the whole batch to be dropped
+  "validVersions": "0-8",
   "fields": [
     { "name": "Responses", "type": "[]TopicProduceResponse", "versions": "0+",
       "about": "Each produce response", "fields": [
@@ -44,7 +47,16 @@
         { "name": "LogAppendTimeMs", "type": "int64", "versions": "2+", "default": "-1", "ignorable": true,
           "about": "The timestamp returned by broker after appending the messages. If CreateTime is used for the topic, the timestamp will be -1.  If LogAppendTime is used for the topic, the timestamp will be the broker local time when the messages are appended." },
         { "name": "LogStartOffset", "type": "int64", "versions": "5+", "default": "-1", "ignorable": true,
-          "about": "The log start offset." }
+          "about": "The log start offset." },
+        { "name": "ErrorRecords", "type": "[]RelativeOffsetAndErrorMessage", "versions": "8+", "ignorable": true,
+          "about": "The relative offsets of records that caused the batch to be dropped", "fields": [
+          { "name":  "RelativeOffset", "type": "int32", "versions":  "8+",
+            "about":  "The relative offset of the record that cause the batch to be dropped" },
+          { "name": "RelativeOffsetErrorMessage", "type": "string", "default": "null", "versions": "8+", "nullableVersions": "8+",
+            "about":  "The error message of the record that caused the batch to be dropped"}
+        ]},
+        { "name":  "ErrorMessage", "type": "string", "default": "null", "versions": "8+", "nullableVersions": "8+", "ignorable":  true,
+          "about":  "The global error message summarizing the common root cause of the records that caused the batch to be dropped"}
       ]}
     ]},
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "1+", "ignorable": true,
diff --git a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
index 1af694cd55eaf..c2d3acc7e9f24 100644
--- a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
@@ -505,6 +505,78 @@ public void testOffsetFetchVersions() throws Exception {
         }
     }
 
+    @Test
+    public void testProduceResponseVersions() throws Exception {
+        String topicName = "topic";
+        int partitionIndex = 0;
+        short errorCode = Errors.INVALID_TOPIC_EXCEPTION.code();
+        long baseOffset = 12L;
+        int throttleTimeMs = 1234;
+        long logAppendTimeMs = 1234L;
+        long logStartOffset = 1234L;
+        int relativeOffset = 0;
+        String relativeOffsetErrorMessage = "error message";
+        String errorMessage = "global error message";
+
+        testAllMessageRoundTrips(new ProduceResponseData()
+                                     .setResponses(singletonList(
+                                         new ProduceResponseData.TopicProduceResponse()
+                                             .setName(topicName)
+                                             .setPartitions(singletonList(
+                                                 new ProduceResponseData.PartitionProduceResponse()
+                                                     .setPartitionIndex(partitionIndex)
+                                                     .setErrorCode(errorCode)
+                                                     .setBaseOffset(baseOffset))))));
+
+        Supplier response =
+            () -> new ProduceResponseData()
+                      .setResponses(singletonList(
+                            new ProduceResponseData.TopicProduceResponse()
+                                .setName(topicName)
+                                .setPartitions(singletonList(
+                                     new ProduceResponseData.PartitionProduceResponse()
+                                         .setPartitionIndex(partitionIndex)
+                                         .setErrorCode(errorCode)
+                                         .setBaseOffset(baseOffset)
+                                         .setLogAppendTimeMs(logAppendTimeMs)
+                                         .setLogStartOffset(logStartOffset)
+                                         .setErrorRecords(singletonList(
+                                             new ProduceResponseData.RelativeOffsetAndErrorMessage()
+                                                 .setRelativeOffset(relativeOffset)
+                                                 .setRelativeOffsetErrorMessage(relativeOffsetErrorMessage)))
+                                         .setErrorMessage(errorMessage)))))
+                      .setThrottleTimeMs(throttleTimeMs);
+
+        for (short version = 0; version <= ApiKeys.PRODUCE.latestVersion(); version++) {
+            ProduceResponseData responseData = response.get();
+
+            if (version < 8) {
+                responseData.responses().get(0).partitions().get(0).setErrorRecords(Collections.emptyList());
+                responseData.responses().get(0).partitions().get(0).setErrorMessage(null);
+            }
+
+            if (version < 5) {
+                responseData.responses().get(0).partitions().get(0).setLogStartOffset(-1);
+            }
+
+            if (version < 2) {
+                responseData.responses().get(0).partitions().get(0).setLogAppendTimeMs(-1);
+            }
+
+            if (version < 1) {
+                responseData.setThrottleTimeMs(0);
+            }
+
+            if (version >= 3 && version <= 4) {
+                testAllMessageRoundTripsBetweenVersions(version, (short) 4, responseData, responseData);
+            } else if (version >= 6 && version <= 7) {
+                testAllMessageRoundTripsBetweenVersions(version, (short) 7, responseData, responseData);
+            } else {
+                testEquivalentMessageRoundTrip(version, responseData);
+            }
+        }
+    }
+
     private void testAllMessageRoundTrips(Message message) throws Exception {
         testAllMessageRoundTripsFromVersion(message.lowestSupportedVersion(), message);
     }
diff --git a/clients/src/test/java/org/apache/kafka/common/record/AbstractLegacyRecordBatchTest.java b/clients/src/test/java/org/apache/kafka/common/record/AbstractLegacyRecordBatchTest.java
index fe6ffabaf61eb..87811b8bb0d80 100644
--- a/clients/src/test/java/org/apache/kafka/common/record/AbstractLegacyRecordBatchTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/record/AbstractLegacyRecordBatchTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.common.record;
 
+import org.apache.kafka.common.InvalidRecordException;
 import org.apache.kafka.common.record.AbstractLegacyRecordBatch.ByteBufferLegacyRecordBatch;
 import org.apache.kafka.common.utils.Utils;
 import org.junit.Test;
diff --git a/clients/src/test/java/org/apache/kafka/common/record/DefaultRecordBatchTest.java b/clients/src/test/java/org/apache/kafka/common/record/DefaultRecordBatchTest.java
index 34e46adee125f..beee10ab77426 100644
--- a/clients/src/test/java/org/apache/kafka/common/record/DefaultRecordBatchTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/record/DefaultRecordBatchTest.java
@@ -16,6 +16,8 @@
  */
 package org.apache.kafka.common.record;
 
+import org.apache.kafka.common.InvalidRecordException;
+import org.apache.kafka.common.errors.CorruptRecordException;
 import org.apache.kafka.common.header.Header;
 import org.apache.kafka.common.header.internals.RecordHeader;
 import org.apache.kafka.common.utils.CloseableIterator;
@@ -173,7 +175,7 @@ public void testSizeInBytes() {
         assertEquals(actualSize, DefaultRecordBatch.sizeInBytes(Arrays.asList(records)));
     }
 
-    @Test(expected = InvalidRecordException.class)
+    @Test(expected = CorruptRecordException.class)
     public void testInvalidRecordSize() {
         MemoryRecords records = MemoryRecords.withRecords(RecordBatch.MAGIC_VALUE_V2, 0L,
                 CompressionType.NONE, TimestampType.CREATE_TIME,
@@ -233,7 +235,7 @@ public void testInvalidRecordCountTooLittleCompressedV2() {
         }
     }
 
-    @Test(expected = InvalidRecordException.class)
+    @Test(expected = CorruptRecordException.class)
     public void testInvalidCrc() {
         MemoryRecords records = MemoryRecords.withRecords(RecordBatch.MAGIC_VALUE_V2, 0L,
                 CompressionType.NONE, TimestampType.CREATE_TIME,
diff --git a/clients/src/test/java/org/apache/kafka/common/record/DefaultRecordTest.java b/clients/src/test/java/org/apache/kafka/common/record/DefaultRecordTest.java
index 198f9945d3f16..822b3b9c6aff0 100644
--- a/clients/src/test/java/org/apache/kafka/common/record/DefaultRecordTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/record/DefaultRecordTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.common.record;
 
+import org.apache.kafka.common.InvalidRecordException;
 import org.apache.kafka.common.header.Header;
 import org.apache.kafka.common.header.internals.RecordHeader;
 import org.apache.kafka.common.utils.ByteBufferInputStream;
diff --git a/clients/src/test/java/org/apache/kafka/common/record/EndTransactionMarkerTest.java b/clients/src/test/java/org/apache/kafka/common/record/EndTransactionMarkerTest.java
index 903f674ed1965..8698c7cfca86b 100644
--- a/clients/src/test/java/org/apache/kafka/common/record/EndTransactionMarkerTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/record/EndTransactionMarkerTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.common.record;
 
+import org.apache.kafka.common.InvalidRecordException;
 import org.junit.Test;
 
 import java.nio.ByteBuffer;
diff --git a/clients/src/test/java/org/apache/kafka/common/record/LegacyRecordTest.java b/clients/src/test/java/org/apache/kafka/common/record/LegacyRecordTest.java
index 9480c60ca940b..848f0a3c5f28e 100644
--- a/clients/src/test/java/org/apache/kafka/common/record/LegacyRecordTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/record/LegacyRecordTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.common.record;
 
+import org.apache.kafka.common.errors.CorruptRecordException;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -93,7 +94,7 @@ public void testChecksum() {
             try {
                 copy.ensureValid();
                 fail("Should fail the above test.");
-            } catch (InvalidRecordException e) {
+            } catch (CorruptRecordException e) {
                 // this is good
             }
         }
diff --git a/clients/src/test/java/org/apache/kafka/common/record/SimpleLegacyRecordTest.java b/clients/src/test/java/org/apache/kafka/common/record/SimpleLegacyRecordTest.java
index 5f578a873d0dc..cd287bbbf1c5b 100644
--- a/clients/src/test/java/org/apache/kafka/common/record/SimpleLegacyRecordTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/record/SimpleLegacyRecordTest.java
@@ -16,6 +16,8 @@
  */
 package org.apache.kafka.common.record;
 
+import org.apache.kafka.common.InvalidRecordException;
+import org.apache.kafka.common.errors.CorruptRecordException;
 import org.apache.kafka.common.utils.ByteBufferOutputStream;
 import org.apache.kafka.common.utils.Utils;
 import org.junit.Test;
@@ -66,7 +68,7 @@ public void testCompressedIterationWithEmptyRecords() throws Exception {
     }
 
     /* This scenario can happen if the record size field is corrupt and we end up allocating a buffer that is too small */
-    @Test(expected = InvalidRecordException.class)
+    @Test(expected = CorruptRecordException.class)
     public void testIsValidWithTooSmallBuffer() {
         ByteBuffer buffer = ByteBuffer.allocate(2);
         LegacyRecord record = new LegacyRecord(buffer);
@@ -74,7 +76,7 @@ public void testIsValidWithTooSmallBuffer() {
         record.ensureValid();
     }
 
-    @Test(expected = InvalidRecordException.class)
+    @Test(expected = CorruptRecordException.class)
     public void testIsValidWithChecksumMismatch() {
         ByteBuffer buffer = ByteBuffer.allocate(4);
         // set checksum
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ProduceRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ProduceRequestTest.java
index 809d64f1e327b..95d719adb3bb7 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/ProduceRequestTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/ProduceRequestTest.java
@@ -17,10 +17,10 @@
 
 package org.apache.kafka.common.requests;
 
+import org.apache.kafka.common.InvalidRecordException;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.record.CompressionType;
-import org.apache.kafka.common.record.InvalidRecordException;
 import org.apache.kafka.common.record.MemoryRecords;
 import org.apache.kafka.common.record.MemoryRecordsBuilder;
 import org.apache.kafka.common.record.RecordBatch;
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index 6f9f7466ecdfe..99172408d65ed 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -233,6 +233,7 @@ public void testSerialization() throws Exception {
         checkRequest(createProduceRequest(3), true);
         checkErrorResponse(createProduceRequest(3), new UnknownServerException(), true);
         checkResponse(createProduceResponse(), 2, true);
+        checkResponse(createProduceResponseWithErrorMessage(), 8, true);
         checkRequest(createStopReplicaRequest(0, true), true);
         checkRequest(createStopReplicaRequest(0, false), true);
         checkErrorResponse(createStopReplicaRequest(0, true), new UnknownServerException(), true);
@@ -1118,6 +1119,13 @@ private ProduceResponse createProduceResponse() {
         return new ProduceResponse(responseData, 0);
     }
 
+    private ProduceResponse createProduceResponseWithErrorMessage() {
+        Map responseData = new HashMap<>();
+        responseData.put(new TopicPartition("test", 0), new ProduceResponse.PartitionResponse(Errors.NONE,
+                10000, RecordBatch.NO_TIMESTAMP, 100, Collections.singletonMap(0, "error message"), "global error message"));
+        return new ProduceResponse(responseData, 0);
+    }
+
     private StopReplicaRequest createStopReplicaRequest(int version, boolean deletePartitions) {
         Set partitions = Utils.mkSet(new TopicPartition("test", 0));
         return new StopReplicaRequest.Builder((short) version, 0, 1, 0, deletePartitions, partitions).build();
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/util/ConnectUtils.java b/connect/runtime/src/main/java/org/apache/kafka/connect/util/ConnectUtils.java
index 86ed42e1a604c..e1e4a874035ab 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/util/ConnectUtils.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/util/ConnectUtils.java
@@ -18,7 +18,7 @@
 
 import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.common.KafkaFuture;
-import org.apache.kafka.common.record.InvalidRecordException;
+import org.apache.kafka.common.InvalidRecordException;
 import org.apache.kafka.common.record.RecordBatch;
 import org.apache.kafka.connect.errors.ConnectException;
 import org.apache.kafka.connect.runtime.WorkerConfig;
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java
index 272a6d56e6a21..8751f1cc38727 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java
@@ -22,11 +22,11 @@
 import org.apache.kafka.clients.producer.RecordMetadata;
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.InvalidRecordException;
 import org.apache.kafka.common.header.Header;
 import org.apache.kafka.common.header.Headers;
 import org.apache.kafka.common.header.internals.RecordHeaders;
 import org.apache.kafka.common.errors.TopicAuthorizationException;
-import org.apache.kafka.common.record.InvalidRecordException;
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.connect.data.Schema;
 import org.apache.kafka.connect.data.SchemaAndValue;
diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index 48fc9e9a7d5a8..8068c9ba31a26 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -42,7 +42,7 @@ import org.apache.kafka.common.record._
 import org.apache.kafka.common.requests.FetchResponse.AbortedTransaction
 import org.apache.kafka.common.requests.{EpochEndOffset, ListOffsetRequest}
 import org.apache.kafka.common.utils.{Time, Utils}
-import org.apache.kafka.common.{KafkaException, TopicPartition}
+import org.apache.kafka.common.{InvalidRecordException, KafkaException, TopicPartition}
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable.{ArrayBuffer, ListBuffer}
@@ -1356,7 +1356,7 @@ class Log(@volatile var dir: File,
       // check the validity of the message by checking CRC
       if (!batch.isValid) {
         brokerTopicStats.allTopicsStats.invalidMessageCrcRecordsPerSec.mark()
-        throw new InvalidRecordException(s"Record is corrupt (stored crc = ${batch.checksum()}) in topic partition $topicPartition.")
+        throw new CorruptRecordException(s"Record is corrupt (stored crc = ${batch.checksum()}) in topic partition $topicPartition.")
       }
 
       if (batch.maxTimestamp > maxTimestamp) {
diff --git a/core/src/main/scala/kafka/log/LogSegment.scala b/core/src/main/scala/kafka/log/LogSegment.scala
index 3bdd96248d173..6e336655957d5 100755
--- a/core/src/main/scala/kafka/log/LogSegment.scala
+++ b/core/src/main/scala/kafka/log/LogSegment.scala
@@ -26,6 +26,7 @@ import kafka.metrics.{KafkaMetricsGroup, KafkaTimer}
 import kafka.server.epoch.LeaderEpochFileCache
 import kafka.server.{FetchDataInfo, LogOffsetMetadata}
 import kafka.utils._
+import org.apache.kafka.common.InvalidRecordException
 import org.apache.kafka.common.errors.CorruptRecordException
 import org.apache.kafka.common.record.FileRecords.{LogOffsetPosition, TimestampAndOffset}
 import org.apache.kafka.common.record._
@@ -367,9 +368,9 @@ class LogSegment private[log] (val log: FileRecords,
         }
       }
     } catch {
-      case e: CorruptRecordException =>
-        warn("Found invalid messages in log segment %s at byte offset %d: %s."
-          .format(log.file.getAbsolutePath, validBytes, e.getMessage))
+      case e@ (_: CorruptRecordException | _: InvalidRecordException) =>
+        warn("Found invalid messages in log segment %s at byte offset %d: %s. %s"
+          .format(log.file.getAbsolutePath, validBytes, e.getMessage, e.getCause))
     }
     val truncated = log.sizeInBytes - validBytes
     if (truncated > 0)
diff --git a/core/src/main/scala/kafka/log/LogValidator.scala b/core/src/main/scala/kafka/log/LogValidator.scala
index d317738a9f6be..6bba8eb2d769e 100644
--- a/core/src/main/scala/kafka/log/LogValidator.scala
+++ b/core/src/main/scala/kafka/log/LogValidator.scala
@@ -23,9 +23,10 @@ import kafka.common.LongRef
 import kafka.message.{CompressionCodec, NoCompressionCodec, ZStdCompressionCodec}
 import kafka.server.BrokerTopicStats
 import kafka.utils.Logging
+import org.apache.kafka.common.errors.{CorruptRecordException, InvalidTimestampException, UnsupportedCompressionTypeException, UnsupportedForMessageFormatException}
+import org.apache.kafka.common.record.{AbstractRecords, BufferSupplier, CompressionType, MemoryRecords, Record, RecordBatch, RecordConversionStats, TimestampType}
+import org.apache.kafka.common.InvalidRecordException
 import org.apache.kafka.common.TopicPartition
-import org.apache.kafka.common.errors.{InvalidTimestampException, UnsupportedCompressionTypeException, UnsupportedForMessageFormatException}
-import org.apache.kafka.common.record.{AbstractRecords, BufferSupplier, CompressionType, InvalidRecordException, MemoryRecords, Record, RecordBatch, RecordConversionStats, TimestampType}
 import org.apache.kafka.common.utils.Time
 
 import scala.collection.{Seq, mutable}
@@ -162,7 +163,7 @@ private[kafka] object LogValidator extends Logging {
       } catch {
         case e: InvalidRecordException =>
           brokerTopicStats.allTopicsStats.invalidMessageCrcRecordsPerSec.mark()
-          throw new InvalidRecordException(e.getMessage + s" in topic partition $topicPartition.")
+          throw new CorruptRecordException(e.getMessage + s" in topic partition $topicPartition.")
       }
     }
 
diff --git a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
index 34da9e23780e4..6e2e5da4c9a5b 100755
--- a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
+++ b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
@@ -39,7 +39,7 @@ import java.util.function.Consumer
 
 import com.yammer.metrics.core.Gauge
 import kafka.log.LogAppendInfo
-import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.{InvalidRecordException, TopicPartition}
 import org.apache.kafka.common.internals.PartitionStates
 import org.apache.kafka.common.record.{FileRecords, MemoryRecords, Records}
 import org.apache.kafka.common.requests._
@@ -331,7 +331,7 @@ abstract class AbstractFetcherThread(name: String,
                       }
                     }
                   } catch {
-                    case ime: CorruptRecordException =>
+                    case ime@( _: CorruptRecordException | _: InvalidRecordException) =>
                       // we log the error and continue. This ensures two things
                       // 1. If there is a corrupt message in a topic partition, it does not bring the fetcher thread
                       //    down and cause other topic partition to also lag
diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala
index 2234366575c42..c446563570480 100755
--- a/core/src/test/scala/unit/kafka/log/LogTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogTest.scala
@@ -31,7 +31,7 @@ import kafka.server.checkpoints.LeaderEpochCheckpointFile
 import kafka.server.epoch.{EpochEntry, LeaderEpochFileCache}
 import kafka.server.{BrokerTopicStats, FetchDataInfo, FetchHighWatermark, FetchIsolation, FetchLogEnd, FetchTxnCommitted, KafkaConfig, LogDirFailureChannel, LogOffsetMetadata}
 import kafka.utils._
-import org.apache.kafka.common.{KafkaException, TopicPartition}
+import org.apache.kafka.common.{InvalidRecordException, KafkaException, TopicPartition}
 import org.apache.kafka.common.errors._
 import org.apache.kafka.common.record.FileRecords.TimestampAndOffset
 import org.apache.kafka.common.record.MemoryRecords.RecordFilter
diff --git a/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala b/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
index e37c85bc74189..7fd13210a8ccc 100644
--- a/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
@@ -25,6 +25,7 @@ import kafka.common.LongRef
 import kafka.message._
 import kafka.server.BrokerTopicStats
 import kafka.utils.TestUtils.meterCount
+import org.apache.kafka.common.InvalidRecordException
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.{InvalidTimestampException, UnsupportedCompressionTypeException, UnsupportedForMessageFormatException}
 import org.apache.kafka.common.record._
diff --git a/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala b/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala
index 32e49e5f85d5d..bedf6ff0f00e8 100644
--- a/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala
@@ -56,6 +56,7 @@ class ProduceRequestTest extends BaseRequestTest {
       assertEquals(Errors.NONE, partitionResponse.error)
       assertEquals(expectedOffset, partitionResponse.baseOffset)
       assertEquals(-1, partitionResponse.logAppendTime)
+      assertTrue(partitionResponse.errorRecords.isEmpty)
       partitionResponse
     }
 

From 422687148e01b607ef26d001fe1874a35e90d0ae Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Tue, 1 Oct 2019 06:11:05 -0700
Subject: [PATCH 0664/1071] MINOR: Mark RocksDBStoreTest as integration test
 (#7412)

shouldNotThrowExceptionOnRestoreWhenThereIsPreExistingRocksDbFiles takes
1m30s, which is too long for a unit test.

`RocksDBTimestampedStoreTest` inherits from `RocksDBStoreTest` and it's
implicitly considered an integration test too.

Reviewers: Guozhang Wang 
---
 .../apache/kafka/streams/state/internals/RocksDBStoreTest.java | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java
index e3b7274df1d4b..4c1932e2853f5 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java
@@ -34,12 +34,14 @@
 import org.apache.kafka.streams.state.internals.metrics.RocksDBMetrics;
 import org.apache.kafka.streams.state.internals.metrics.RocksDBMetricsRecorder;
 import org.apache.kafka.streams.state.internals.metrics.RocksDBMetricsRecordingTrigger;
+import org.apache.kafka.test.IntegrationTest;
 import org.apache.kafka.test.InternalMockProcessorContext;
 import org.apache.kafka.test.StreamsTestUtils;
 import org.apache.kafka.test.TestUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
 import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.powermock.modules.junit4.PowerMockRunner;
@@ -75,6 +77,7 @@
 import static org.powermock.api.easymock.PowerMock.replay;
 import static org.powermock.api.easymock.PowerMock.verify;
 
+@Category(IntegrationTest.class)
 @RunWith(PowerMockRunner.class)
 @PrepareForTest({RocksDBMetrics.class, Sensor.class})
 public class RocksDBStoreTest {

From 270e6109159f9288b4ed807462d8180a3c92e0da Mon Sep 17 00:00:00 2001
From: Boyang Chen 
Date: Tue, 1 Oct 2019 06:19:43 -0700
Subject: [PATCH 0665/1071] KAFKA-8896: Check group state before completing
 delayed heartbeat (#7377)

This is a defensive fix for KAFKA-8896, which would cause group coordinator crash when the heartbeat member was not found.
---
 .../coordinator/group/GroupCoordinator.scala  | 25 +++++++++++----
 .../group/GroupCoordinatorTest.scala          | 31 +++++++++++++++++--
 2 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
index 0be2a48867e30..64884ec5dbbd1 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
@@ -1110,24 +1110,37 @@ class GroupCoordinator(val brokerId: Int,
 
   def tryCompleteHeartbeat(group: GroupMetadata, memberId: String, isPending: Boolean, heartbeatDeadline: Long, forceComplete: () => Boolean) = {
     group.inLock {
-      if (isPending) {
+      // The group has been unloaded and invalid, we should complete the heartbeat.
+      if (group.is(Dead)) {
+        forceComplete()
+      } else if (isPending) {
         // complete the heartbeat if the member has joined the group
         if (group.has(memberId)) {
           forceComplete()
         } else false
-      }
-      else {
-        val member = group.get(memberId)
-        if (member.shouldKeepAlive(heartbeatDeadline) || member.isLeaving) {
+      } else {
+        if (shouldCompleteNonPendingHeartbeat(group, memberId, heartbeatDeadline)) {
           forceComplete()
         } else false
       }
     }
   }
 
+  def shouldCompleteNonPendingHeartbeat(group: GroupMetadata, memberId: String, heartbeatDeadline: Long): Boolean = {
+    if (group.has(memberId)) {
+      val member = group.get(memberId)
+      member.shouldKeepAlive(heartbeatDeadline) || member.isLeaving
+    } else {
+      info(s"Member id $memberId was not found in ${group.groupId} during heartbeat expiration.")
+      false
+    }
+  }
+
   def onExpireHeartbeat(group: GroupMetadata, memberId: String, isPending: Boolean, heartbeatDeadline: Long): Unit = {
     group.inLock {
-      if (isPending) {
+      if (group.is(Dead)) {
+        info(s"Received notification of heartbeat expiration for member $memberId after group ${group.groupId} had already been unloaded or deleted.")
+      } else if (isPending) {
         info(s"Pending member $memberId in group ${group.groupId} has been removed after session timeout expiration.")
         removePendingMemberAndUpdateGroup(group, memberId)
       } else if (!group.has(memberId)) {
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
index 9cc0d3a18ab29..91a963d7f14e1 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
@@ -1188,15 +1188,15 @@ class GroupCoordinatorTest {
 
     EasyMock.reset(replicaManager)
     val followerId = followerJoinGroupResult.memberId
-    val follwerSyncGroupResult = syncGroupFollower(groupId, leaderJoinGroupResult.generationId, followerId)
-    assertEquals(Errors.NONE, follwerSyncGroupResult._2)
+    val followerSyncGroupResult = syncGroupFollower(groupId, leaderJoinGroupResult.generationId, followerId)
+    assertEquals(Errors.NONE, followerSyncGroupResult._2)
     assertTrue(getGroup(groupId).is(Stable))
 
     new RebalanceResult(newGeneration,
       leaderId,
       leaderSyncGroupResult._1,
       followerId,
-      follwerSyncGroupResult._1)
+      followerSyncGroupResult._1)
   }
 
   private def checkJoinGroupResult(joinGroupResult: JoinGroupResult,
@@ -3101,6 +3101,31 @@ class GroupCoordinatorTest {
     assertEquals(Errors.NONE, thirdResult.error)
   }
 
+  @Test
+  def testCompleteHeartbeatWithGroupDead(): Unit = {
+    val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
+    EasyMock.reset(replicaManager)
+    heartbeat(groupId, rebalanceResult.leaderId, rebalanceResult.generation)
+    val group = getGroup(groupId)
+    group.transitionTo(Dead)
+    val leaderMemberId = rebalanceResult.leaderId
+    assertTrue(groupCoordinator.tryCompleteHeartbeat(group, leaderMemberId, false, DefaultSessionTimeout, () => true))
+    groupCoordinator.onExpireHeartbeat(group, leaderMemberId, false, DefaultSessionTimeout)
+    assertTrue(group.has(leaderMemberId))
+  }
+
+  @Test
+  def testCompleteHeartbeatWithMemberAlreadyRemoved(): Unit = {
+    val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId)
+    EasyMock.reset(replicaManager)
+    heartbeat(groupId, rebalanceResult.leaderId, rebalanceResult.generation)
+    val group = getGroup(groupId)
+    val leaderMemberId = rebalanceResult.leaderId
+    group.remove(leaderMemberId)
+    assertFalse(groupCoordinator.tryCompleteHeartbeat(group, leaderMemberId, false, DefaultSessionTimeout, () => true))
+    groupCoordinator.onExpireHeartbeat(group, leaderMemberId, false, DefaultSessionTimeout)
+  }
+
   private def getGroup(groupId: String): GroupMetadata = {
     val groupOpt = groupCoordinator.groupManager.getGroup(groupId)
     assertTrue(groupOpt.isDefined)

From 9e294cbca22e9da5f491792e8a8b0f9b27047fd0 Mon Sep 17 00:00:00 2001
From: Bill Bejeck 
Date: Tue, 1 Oct 2019 09:34:05 -0400
Subject: [PATCH 0666/1071] KAFKA-8807: Flaky GlobalStreamThread test (#7418)

A minor refactor to explicitly verify that Processor#close is only called once.

Reviewers: Guozhang Wang , Sophie Blee-Goldman ,  Bruno Cadonna ,
---
 .../integration/GlobalThreadShutDownOrderTest.java | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/GlobalThreadShutDownOrderTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/GlobalThreadShutDownOrderTest.java
index 7c2b009f642e9..8bd77d3572418 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/GlobalThreadShutDownOrderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/GlobalThreadShutDownOrderTest.java
@@ -49,14 +49,26 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Properties;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.junit.Assert.assertEquals;
 
+/**
+ * This test asserts that when Kafka Streams is closing and shuts
+ * down a StreamThread the closing of the GlobalStreamThread happens
+ * after all the StreamThreads are completely stopped.
+ *
+ * The test validates the Processor still has access to the GlobalStateStore while closing.
+ * Otherwise if the GlobalStreamThread were to close underneath the StreamThread
+ * an exception would be thrown as the GlobalStreamThread closes all global stores on closing.
+ */
 @Category({IntegrationTest.class})
 public class GlobalThreadShutDownOrderTest {
 
     private static final int NUM_BROKERS = 1;
     private static final Properties BROKER_CONFIG;
+    private final AtomicInteger closeCounter = new AtomicInteger(0);
+    private final int expectedCloseCount = 1;
 
     static {
         BROKER_CONFIG = new Properties();
@@ -136,6 +148,7 @@ public void shouldFinishGlobalStoreOperationOnShutDown() throws Exception {
 
         final List expectedRetrievedValues = Arrays.asList(1L, 2L, 3L, 4L);
         assertEquals(expectedRetrievedValues, retrievedValuesList);
+        assertEquals(expectedCloseCount, closeCounter.get());
     }
 
 
@@ -188,6 +201,7 @@ public void process(final String key, final Long value) {
 
         @Override
         public void close() {
+            closeCounter.getAndIncrement();
             final List keys = Arrays.asList("A", "B", "C", "D");
             for (final String key : keys) {
                 // need to simulate thread slow in closing

From bfcc17f211834710add79122d7ba3d267931cfb1 Mon Sep 17 00:00:00 2001
From: Almog Gavra 
Date: Tue, 1 Oct 2019 13:33:06 -0700
Subject: [PATCH 0667/1071] KAFKA-8595: Support deserialization of JSON
 decimals encoded in NUMERIC  (#7354)

Implemented KIP-481 by adding support for deserializing Connect DECIMAL values encoded in JSON as numbers, in addition to raw byte array (base64) format used previously.

Author: Almog Gavra 
Reviewers: Chris Egerton , Konstantine Karantasis , Randall Hauch 
---
 .../apache/kafka/common/config/ConfigDef.java |  28 ++++
 .../kafka/common/config/ConfigDefTest.java    |   5 +-
 .../kafka/connect/json/DecimalFormat.java     |  36 ++++
 .../kafka/connect/json/JsonConverter.java     | 157 ++++++++++--------
 .../connect/json/JsonConverterConfig.java     |  39 ++++-
 .../kafka/connect/json/JsonDeserializer.java  |  13 ++
 .../connect/json/JsonConverterConfigTest.java |  39 +++++
 .../kafka/connect/json/JsonConverterTest.java |  42 +++++
 8 files changed, 283 insertions(+), 76 deletions(-)
 create mode 100644 connect/json/src/main/java/org/apache/kafka/connect/json/DecimalFormat.java
 create mode 100644 connect/json/src/test/java/org/apache/kafka/connect/json/JsonConverterConfigTest.java

diff --git a/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java b/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java
index c22b9c77dac93..43ab70d9b7a4f 100644
--- a/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java
+++ b/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.common.config;
 
+import java.util.stream.Collectors;
 import org.apache.kafka.common.config.types.Password;
 import org.apache.kafka.common.utils.Utils;
 
@@ -947,6 +948,33 @@ public String toString() {
         }
     }
 
+    public static class CaseInsensitiveValidString implements Validator {
+
+        final Set validStrings;
+
+        private CaseInsensitiveValidString(List validStrings) {
+            this.validStrings = validStrings.stream()
+                .map(s -> s.toUpperCase(Locale.ROOT))
+                .collect(Collectors.toSet());
+        }
+
+        public static CaseInsensitiveValidString in(String... validStrings) {
+            return new CaseInsensitiveValidString(Arrays.asList(validStrings));
+        }
+
+        @Override
+        public void ensureValid(String name, Object o) {
+            String s = (String) o;
+            if (s == null || !validStrings.contains(s.toUpperCase(Locale.ROOT))) {
+                throw new ConfigException(name, o, "String must be one of (case insensitive): " + Utils.join(validStrings, ", "));
+            }
+        }
+
+        public String toString() {
+            return "(case insensitive) [" + Utils.join(validStrings, ", ") + "]";
+        }
+    }
+
     public static class NonNullValidator implements Validator {
         @Override
         public void ensureValid(String name, Object value) {
diff --git a/clients/src/test/java/org/apache/kafka/common/config/ConfigDefTest.java b/clients/src/test/java/org/apache/kafka/common/config/ConfigDefTest.java
index 9baa0345b4c54..9751545ca4de1 100644
--- a/clients/src/test/java/org/apache/kafka/common/config/ConfigDefTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/config/ConfigDefTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.common.config;
 
+import org.apache.kafka.common.config.ConfigDef.CaseInsensitiveValidString;
 import org.apache.kafka.common.config.ConfigDef.Importance;
 import org.apache.kafka.common.config.ConfigDef.Range;
 import org.apache.kafka.common.config.ConfigDef.Type;
@@ -157,7 +158,9 @@ public void testNestedClass() {
     public void testValidators() {
         testValidators(Type.INT, Range.between(0, 10), 5, new Object[]{1, 5, 9}, new Object[]{-1, 11, null});
         testValidators(Type.STRING, ValidString.in("good", "values", "default"), "default",
-                new Object[]{"good", "values", "default"}, new Object[]{"bad", "inputs", null});
+                new Object[]{"good", "values", "default"}, new Object[]{"bad", "inputs", "DEFAULT", null});
+        testValidators(Type.STRING, CaseInsensitiveValidString.in("good", "values", "default"), "default",
+            new Object[]{"gOOd", "VALUES", "default"}, new Object[]{"Bad", "iNPUts", null});
         testValidators(Type.LIST, ConfigDef.ValidList.in("1", "2", "3"), "1", new Object[]{"1", "2", "3"}, new Object[]{"4", "5", "6"});
         testValidators(Type.STRING, new ConfigDef.NonNullValidator(), "a", new Object[]{"abb"}, new Object[] {null});
         testValidators(Type.STRING, ConfigDef.CompositeValidator.of(new ConfigDef.NonNullValidator(), ValidString.in("a", "b")), "a", new Object[]{"a", "b"}, new Object[] {null, -1, "c"});
diff --git a/connect/json/src/main/java/org/apache/kafka/connect/json/DecimalFormat.java b/connect/json/src/main/java/org/apache/kafka/connect/json/DecimalFormat.java
new file mode 100644
index 0000000000000..b4a7fc55d4c61
--- /dev/null
+++ b/connect/json/src/main/java/org/apache/kafka/connect/json/DecimalFormat.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.json;
+
+/**
+ * Represents the valid {@link org.apache.kafka.connect.data.Decimal} serialization formats
+ * in a {@link JsonConverter}.
+ */
+public enum DecimalFormat {
+
+    /**
+     * Serializes the JSON Decimal as a base-64 string. For example, serializing the value
+     * `10.2345` with the BASE64 setting will result in `"D3J5"`.
+     */
+    BASE64,
+
+    /**
+     * Serializes the JSON Decimal as a JSON number. For example, serializing the value
+     * `10.2345` with the NUMERIC setting will result in `10.2345`.
+     */
+    NUMERIC
+}
diff --git a/connect/json/src/main/java/org/apache/kafka/connect/json/JsonConverter.java b/connect/json/src/main/java/org/apache/kafka/connect/json/JsonConverter.java
index 546fcf0daaa68..fe83ddf7e9ee6 100644
--- a/connect/json/src/main/java/org/apache/kafka/connect/json/JsonConverter.java
+++ b/connect/json/src/main/java/org/apache/kafka/connect/json/JsonConverter.java
@@ -16,10 +16,13 @@
  */
 package org.apache.kafka.connect.json;
 
+import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.util.HashSet;
+import java.util.Set;
 import org.apache.kafka.common.cache.Cache;
 import org.apache.kafka.common.cache.LRUCache;
 import org.apache.kafka.common.cache.SynchronizedCache;
@@ -185,94 +188,105 @@ public Object convert(Schema schema, JsonNode value) {
         });
     }
 
-    // Convert values in Kafka Connect form into their logical types. These logical converters are discovered by logical type
+    // Convert values in Kafka Connect form into/from their logical types. These logical converters are discovered by logical type
     // names specified in the field
-    private static final HashMap TO_CONNECT_LOGICAL_CONVERTERS = new HashMap<>();
+    private static final HashMap LOGICAL_CONVERTERS = new HashMap<>();
     static {
-        TO_CONNECT_LOGICAL_CONVERTERS.put(Decimal.LOGICAL_NAME, new LogicalTypeConverter() {
+        LOGICAL_CONVERTERS.put(Decimal.LOGICAL_NAME, new LogicalTypeConverter() {
             @Override
-            public Object convert(Schema schema, Object value) {
-                if (!(value instanceof byte[]))
-                    throw new DataException("Invalid type for Decimal, underlying representation should be bytes but was " + value.getClass());
-                return Decimal.toLogical(schema, (byte[]) value);
-            }
-        });
+            public JsonNode toJson(final Schema schema, final Object value, final JsonConverterConfig config) {
+                if (!(value instanceof BigDecimal))
+                    throw new DataException("Invalid type for Decimal, expected BigDecimal but was " + value.getClass());
 
-        TO_CONNECT_LOGICAL_CONVERTERS.put(Date.LOGICAL_NAME, new LogicalTypeConverter() {
-            @Override
-            public Object convert(Schema schema, Object value) {
-                if (!(value instanceof Integer))
-                    throw new DataException("Invalid type for Date, underlying representation should be int32 but was " + value.getClass());
-                return Date.toLogical(schema, (int) value);
+                final BigDecimal decimal = (BigDecimal) value;
+                switch (config.decimalFormat()) {
+                    case NUMERIC:
+                        return JsonNodeFactory.instance.numberNode(decimal);
+                    case BASE64:
+                        return JsonNodeFactory.instance.binaryNode(Decimal.fromLogical(schema, decimal));
+                    default:
+                        throw new DataException("Unexpected " + JsonConverterConfig.DECIMAL_FORMAT_CONFIG + ": " + config.decimalFormat());
+                }
             }
-        });
 
-        TO_CONNECT_LOGICAL_CONVERTERS.put(Time.LOGICAL_NAME, new LogicalTypeConverter() {
             @Override
-            public Object convert(Schema schema, Object value) {
-                if (!(value instanceof Integer))
-                    throw new DataException("Invalid type for Time, underlying representation should be int32 but was " + value.getClass());
-                return Time.toLogical(schema, (int) value);
+            public Object toConnect(final Schema schema, final JsonNode value) {
+                if (value.isNumber()) return value.decimalValue();
+                if (value.isBinary() || value.isTextual()) {
+                    try {
+                        return Decimal.toLogical(schema, value.binaryValue());
+                    } catch (Exception e) {
+                        throw new DataException("Invalid bytes for Decimal field", e);
+                    }
+                }
+
+                throw new DataException("Invalid type for Decimal, underlying representation should be numeric or bytes but was " + value.getNodeType());
             }
         });
 
-        TO_CONNECT_LOGICAL_CONVERTERS.put(Timestamp.LOGICAL_NAME, new LogicalTypeConverter() {
+        LOGICAL_CONVERTERS.put(Date.LOGICAL_NAME, new LogicalTypeConverter() {
             @Override
-            public Object convert(Schema schema, Object value) {
-                if (!(value instanceof Long))
-                    throw new DataException("Invalid type for Timestamp, underlying representation should be int64 but was " + value.getClass());
-                return Timestamp.toLogical(schema, (long) value);
+            public JsonNode toJson(final Schema schema, final Object value, final JsonConverterConfig config) {
+                if (!(value instanceof java.util.Date))
+                    throw new DataException("Invalid type for Date, expected Date but was " + value.getClass());
+                return JsonNodeFactory.instance.numberNode(Date.fromLogical(schema, (java.util.Date) value));
             }
-        });
-    }
 
-    private static final HashMap TO_JSON_LOGICAL_CONVERTERS = new HashMap<>();
-    static {
-        TO_JSON_LOGICAL_CONVERTERS.put(Decimal.LOGICAL_NAME, new LogicalTypeConverter() {
             @Override
-            public Object convert(Schema schema, Object value) {
-                if (!(value instanceof BigDecimal))
-                    throw new DataException("Invalid type for Decimal, expected BigDecimal but was " + value.getClass());
-                return Decimal.fromLogical(schema, (BigDecimal) value);
+            public Object toConnect(final Schema schema, final JsonNode value) {
+                if (!(value.isInt()))
+                    throw new DataException("Invalid type for Date, underlying representation should be integer but was " + value.getNodeType());
+                return Date.toLogical(schema, value.intValue());
             }
         });
 
-        TO_JSON_LOGICAL_CONVERTERS.put(Date.LOGICAL_NAME, new LogicalTypeConverter() {
+        LOGICAL_CONVERTERS.put(Time.LOGICAL_NAME, new LogicalTypeConverter() {
             @Override
-            public Object convert(Schema schema, Object value) {
+            public JsonNode toJson(final Schema schema, final Object value, final JsonConverterConfig config) {
                 if (!(value instanceof java.util.Date))
-                    throw new DataException("Invalid type for Date, expected Date but was " + value.getClass());
-                return Date.fromLogical(schema, (java.util.Date) value);
+                    throw new DataException("Invalid type for Time, expected Date but was " + value.getClass());
+                return JsonNodeFactory.instance.numberNode(Time.fromLogical(schema, (java.util.Date) value));
             }
-        });
 
-        TO_JSON_LOGICAL_CONVERTERS.put(Time.LOGICAL_NAME, new LogicalTypeConverter() {
             @Override
-            public Object convert(Schema schema, Object value) {
-                if (!(value instanceof java.util.Date))
-                    throw new DataException("Invalid type for Time, expected Date but was " + value.getClass());
-                return Time.fromLogical(schema, (java.util.Date) value);
+            public Object toConnect(final Schema schema, final JsonNode value) {
+                if (!(value.isInt()))
+                    throw new DataException("Invalid type for Time, underlying representation should be integer but was " + value.getNodeType());
+                return Time.toLogical(schema, value.intValue());
             }
         });
 
-        TO_JSON_LOGICAL_CONVERTERS.put(Timestamp.LOGICAL_NAME, new LogicalTypeConverter() {
+        LOGICAL_CONVERTERS.put(Timestamp.LOGICAL_NAME, new LogicalTypeConverter() {
             @Override
-            public Object convert(Schema schema, Object value) {
+            public JsonNode toJson(final Schema schema, final Object value, final JsonConverterConfig config) {
                 if (!(value instanceof java.util.Date))
                     throw new DataException("Invalid type for Timestamp, expected Date but was " + value.getClass());
-                return Timestamp.fromLogical(schema, (java.util.Date) value);
+                return JsonNodeFactory.instance.numberNode(Timestamp.fromLogical(schema, (java.util.Date) value));
+            }
+
+            @Override
+            public Object toConnect(final Schema schema, final JsonNode value) {
+                if (!(value.isIntegralNumber()))
+                    throw new DataException("Invalid type for Timestamp, underlying representation should be integral but was " + value.getNodeType());
+                return Timestamp.toLogical(schema, value.longValue());
             }
         });
     }
 
-
-    private boolean enableSchemas = JsonConverterConfig.SCHEMAS_ENABLE_DEFAULT;
-    private int cacheSize = JsonConverterConfig.SCHEMAS_CACHE_SIZE_DEFAULT;
+    private JsonConverterConfig config;
     private Cache fromConnectSchemaCache;
     private Cache toConnectSchemaCache;
 
     private final JsonSerializer serializer = new JsonSerializer();
-    private final JsonDeserializer deserializer = new JsonDeserializer();
+    private final JsonDeserializer deserializer;
+
+    public JsonConverter() {
+        // this ensures that the JsonDeserializer maintains full precision on
+        // floating point numbers that cannot fit into float64
+        final Set deserializationFeatures = new HashSet<>();
+        deserializationFeatures.add(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS);
+        deserializer = new JsonDeserializer(deserializationFeatures);
+    }
 
     @Override
     public ConfigDef config() {
@@ -281,16 +295,13 @@ public ConfigDef config() {
 
     @Override
     public void configure(Map configs) {
-        JsonConverterConfig config = new JsonConverterConfig(configs);
-        enableSchemas = config.schemasEnabled();
-        cacheSize = config.schemaCacheSize();
+        config = new JsonConverterConfig(configs);
 
-        boolean isKey = config.type() == ConverterType.KEY;
-        serializer.configure(configs, isKey);
-        deserializer.configure(configs, isKey);
+        serializer.configure(configs, config.type() == ConverterType.KEY);
+        deserializer.configure(configs, config.type() == ConverterType.KEY);
 
-        fromConnectSchemaCache = new SynchronizedCache<>(new LRUCache(cacheSize));
-        toConnectSchemaCache = new SynchronizedCache<>(new LRUCache(cacheSize));
+        fromConnectSchemaCache = new SynchronizedCache<>(new LRUCache<>(config.schemaCacheSize()));
+        toConnectSchemaCache = new SynchronizedCache<>(new LRUCache<>(config.schemaCacheSize()));
     }
 
     @Override
@@ -321,7 +332,7 @@ public byte[] fromConnectData(String topic, Schema schema, Object value) {
             return null;
         }
 
-        JsonNode jsonValue = enableSchemas ? convertToJsonWithEnvelope(schema, value) : convertToJsonWithoutEnvelope(schema, value);
+        JsonNode jsonValue = config.schemasEnabled() ? convertToJsonWithEnvelope(schema, value) : convertToJsonWithoutEnvelope(schema, value);
         try {
             return serializer.serialize(topic, jsonValue);
         } catch (SerializationException e) {
@@ -344,13 +355,13 @@ public SchemaAndValue toConnectData(String topic, byte[] value) {
             throw new DataException("Converting byte[] to Kafka Connect data failed due to serialization error: ", e);
         }
 
-        if (enableSchemas && (!jsonValue.isObject() || jsonValue.size() != 2 || !jsonValue.has(JsonSchema.ENVELOPE_SCHEMA_FIELD_NAME) || !jsonValue.has(JsonSchema.ENVELOPE_PAYLOAD_FIELD_NAME)))
+        if (config.schemasEnabled() && (!jsonValue.isObject() || jsonValue.size() != 2 || !jsonValue.has(JsonSchema.ENVELOPE_SCHEMA_FIELD_NAME) || !jsonValue.has(JsonSchema.ENVELOPE_PAYLOAD_FIELD_NAME)))
             throw new DataException("JsonConverter with schemas.enable requires \"schema\" and \"payload\" fields and may not contain additional fields." +
                     " If you are trying to deserialize plain JSON data, set schemas.enable=false in your converter configuration.");
 
         // The deserialized data should either be an envelope object containing the schema and the payload or the schema
         // was stripped during serialization and we need to fill in an all-encompassing schema.
-        if (!enableSchemas) {
+        if (!config.schemasEnabled()) {
             ObjectNode envelope = JsonNodeFactory.instance.objectNode();
             envelope.set(JsonSchema.ENVELOPE_SCHEMA_FIELD_NAME, null);
             envelope.set(JsonSchema.ENVELOPE_PAYLOAD_FIELD_NAME, jsonValue);
@@ -578,8 +589,8 @@ private JsonNode convertToJsonWithoutEnvelope(Schema schema, Object value) {
      * Convert this object, in the org.apache.kafka.connect.data format, into a JSON object, returning both the schema
      * and the converted object.
      */
-    private static JsonNode convertToJson(Schema schema, Object logicalValue) {
-        if (logicalValue == null) {
+    private JsonNode convertToJson(Schema schema, Object value) {
+        if (value == null) {
             if (schema == null) // Any schema is valid and we don't have a default, so treat this as an optional schema
                 return null;
             if (schema.defaultValue() != null)
@@ -589,11 +600,10 @@ private static JsonNode convertToJson(Schema schema, Object logicalValue) {
             throw new DataException("Conversion error: null value for field that is required and has no default value");
         }
 
-        Object value = logicalValue;
         if (schema != null && schema.name() != null) {
-            LogicalTypeConverter logicalConverter = TO_JSON_LOGICAL_CONVERTERS.get(schema.name());
+            LogicalTypeConverter logicalConverter = LOGICAL_CONVERTERS.get(schema.name());
             if (logicalConverter != null)
-                value = logicalConverter.convert(schema, logicalValue);
+                return logicalConverter.toJson(schema, value, config);
         }
 
         try {
@@ -742,13 +752,13 @@ private static Object convertToConnect(Schema schema, JsonNode jsonValue) {
         if (typeConverter == null)
             throw new DataException("Unknown schema type: " + String.valueOf(schemaType));
 
-        Object converted = typeConverter.convert(schema, jsonValue);
         if (schema != null && schema.name() != null) {
-            LogicalTypeConverter logicalConverter = TO_CONNECT_LOGICAL_CONVERTERS.get(schema.name());
+            LogicalTypeConverter logicalConverter = LOGICAL_CONVERTERS.get(schema.name());
             if (logicalConverter != null)
-                converted = logicalConverter.convert(schema, converted);
+                return logicalConverter.toConnect(schema, jsonValue);
         }
-        return converted;
+
+        return typeConverter.convert(schema, jsonValue);
     }
 
     private interface JsonToConnectTypeConverter {
@@ -756,6 +766,7 @@ private interface JsonToConnectTypeConverter {
     }
 
     private interface LogicalTypeConverter {
-        Object convert(Schema schema, Object value);
+        JsonNode toJson(Schema schema, Object value, JsonConverterConfig config);
+        Object toConnect(Schema schema, JsonNode value);
     }
 }
diff --git a/connect/json/src/main/java/org/apache/kafka/connect/json/JsonConverterConfig.java b/connect/json/src/main/java/org/apache/kafka/connect/json/JsonConverterConfig.java
index 7f1dda2c4c1b1..efb497991cbc9 100644
--- a/connect/json/src/main/java/org/apache/kafka/connect/json/JsonConverterConfig.java
+++ b/connect/json/src/main/java/org/apache/kafka/connect/json/JsonConverterConfig.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.connect.json;
 
+import java.util.Locale;
 import org.apache.kafka.common.config.ConfigDef;
 import org.apache.kafka.common.config.ConfigDef.Importance;
 import org.apache.kafka.common.config.ConfigDef.Type;
@@ -39,6 +40,12 @@ public class JsonConverterConfig extends ConverterConfig {
     private static final String SCHEMAS_CACHE_SIZE_DOC = "The maximum number of schemas that can be cached in this converter instance.";
     private static final String SCHEMAS_CACHE_SIZE_DISPLAY = "Schema Cache Size";
 
+    public static final String DECIMAL_FORMAT_CONFIG = "decimal.format";
+    public static final String DECIMAL_FORMAT_DEFAULT = DecimalFormat.BASE64.name();
+    private static final String DECIMAL_FORMAT_DOC = "Controls which format this converter will serialize decimals in."
+        + " This value is case insensitive and can be either 'BASE64' (default) or 'NUMERIC'";
+    private static final String DECIMAL_FORMAT_DISPLAY = "Decimal Format";
+
     private final static ConfigDef CONFIG;
 
     static {
@@ -49,14 +56,32 @@ public class JsonConverterConfig extends ConverterConfig {
                       orderInGroup++, Width.MEDIUM, SCHEMAS_ENABLE_DISPLAY);
         CONFIG.define(SCHEMAS_CACHE_SIZE_CONFIG, Type.INT, SCHEMAS_CACHE_SIZE_DEFAULT, Importance.HIGH, SCHEMAS_CACHE_SIZE_DOC, group,
                       orderInGroup++, Width.MEDIUM, SCHEMAS_CACHE_SIZE_DISPLAY);
+
+        group = "Serialization";
+        orderInGroup = 0;
+        CONFIG.define(
+            DECIMAL_FORMAT_CONFIG, Type.STRING, DECIMAL_FORMAT_DEFAULT,
+            ConfigDef.CaseInsensitiveValidString.in(
+                DecimalFormat.BASE64.name(),
+                DecimalFormat.NUMERIC.name()),
+            Importance.LOW, DECIMAL_FORMAT_DOC, group, orderInGroup++,
+            Width.MEDIUM, DECIMAL_FORMAT_DISPLAY);
     }
 
     public static ConfigDef configDef() {
         return CONFIG;
     }
 
+    // cached config values
+    private final boolean schemasEnabled;
+    private final int schemaCacheSize;
+    private final DecimalFormat decimalFormat;
+
     public JsonConverterConfig(Map props) {
         super(CONFIG, props);
+        this.schemasEnabled = getBoolean(SCHEMAS_ENABLE_CONFIG);
+        this.schemaCacheSize = getInt(SCHEMAS_CACHE_SIZE_CONFIG);
+        this.decimalFormat = DecimalFormat.valueOf(getString(DECIMAL_FORMAT_CONFIG).toUpperCase(Locale.ROOT));
     }
 
     /**
@@ -65,7 +90,7 @@ public JsonConverterConfig(Map props) {
      * @return true if enabled, or false otherwise
      */
     public boolean schemasEnabled() {
-        return getBoolean(SCHEMAS_ENABLE_CONFIG);
+        return schemasEnabled;
     }
 
     /**
@@ -74,6 +99,16 @@ public boolean schemasEnabled() {
      * @return the cache size
      */
     public int schemaCacheSize() {
-        return getInt(SCHEMAS_CACHE_SIZE_CONFIG);
+        return schemaCacheSize;
     }
+
+    /**
+     * Get the serialization format for decimal types.
+     *
+     * @return the decimal serialization format
+     */
+    public DecimalFormat decimalFormat() {
+        return decimalFormat;
+    }
+
 }
diff --git a/connect/json/src/main/java/org/apache/kafka/connect/json/JsonDeserializer.java b/connect/json/src/main/java/org/apache/kafka/connect/json/JsonDeserializer.java
index b006e22721519..a656e53283fea 100644
--- a/connect/json/src/main/java/org/apache/kafka/connect/json/JsonDeserializer.java
+++ b/connect/json/src/main/java/org/apache/kafka/connect/json/JsonDeserializer.java
@@ -16,8 +16,11 @@
  */
 package org.apache.kafka.connect.json;
 
+import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.Collections;
+import java.util.Set;
 import org.apache.kafka.common.errors.SerializationException;
 import org.apache.kafka.common.serialization.Deserializer;
 
@@ -32,8 +35,18 @@ public class JsonDeserializer implements Deserializer {
      * Default constructor needed by Kafka
      */
     public JsonDeserializer() {
+        this(Collections.emptySet());
     }
 
+    /**
+     * A constructor that additionally specifies some {@link DeserializationFeature}
+     * for the deserializer
+     *
+     * @param deserializationFeatures the specified deserialization features
+     */
+    JsonDeserializer(final Set deserializationFeatures) {
+        deserializationFeatures.forEach(objectMapper::enable);
+    }
 
     @Override
     public JsonNode deserialize(String topic, byte[] bytes) {
diff --git a/connect/json/src/test/java/org/apache/kafka/connect/json/JsonConverterConfigTest.java b/connect/json/src/test/java/org/apache/kafka/connect/json/JsonConverterConfigTest.java
new file mode 100644
index 0000000000000..590707bffab4b
--- /dev/null
+++ b/connect/json/src/test/java/org/apache/kafka/connect/json/JsonConverterConfigTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.json;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.kafka.connect.storage.ConverterConfig;
+import org.apache.kafka.connect.storage.ConverterType;
+import org.junit.Test;
+
+public class JsonConverterConfigTest {
+
+    @Test
+    public void shouldBeCaseInsensitiveForDecimalFormatConfig() {
+        final Map configValues = new HashMap<>();
+        configValues.put(ConverterConfig.TYPE_CONFIG, ConverterType.KEY.getName());
+        configValues.put(JsonConverterConfig.DECIMAL_FORMAT_CONFIG, "NuMeRiC");
+
+        final JsonConverterConfig config = new JsonConverterConfig(configValues);
+        assertEquals(config.decimalFormat(), DecimalFormat.NUMERIC);
+    }
+
+}
\ No newline at end of file
diff --git a/connect/json/src/test/java/org/apache/kafka/connect/json/JsonConverterTest.java b/connect/json/src/test/java/org/apache/kafka/connect/json/JsonConverterTest.java
index 003d7e610872c..fb9d8cb8ebb8a 100644
--- a/connect/json/src/test/java/org/apache/kafka/connect/json/JsonConverterTest.java
+++ b/connect/json/src/test/java/org/apache/kafka/connect/json/JsonConverterTest.java
@@ -58,6 +58,7 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -252,6 +253,27 @@ public void decimalToConnectOptionalWithDefaultValue() {
         assertEquals(reference, schemaAndValue.value());
     }
 
+    @Test
+    public void numericDecimalToConnect() {
+        BigDecimal reference = new BigDecimal(new BigInteger("156"), 2);
+        Schema schema = Decimal.schema(2);
+        String msg = "{ \"schema\": { \"type\": \"bytes\", \"name\": \"org.apache.kafka.connect.data.Decimal\", \"version\": 1, \"parameters\": { \"scale\": \"2\" } }, \"payload\": 1.56 }";
+        SchemaAndValue schemaAndValue = converter.toConnectData(TOPIC, msg.getBytes());
+        assertEquals(schema, schemaAndValue.schema());
+        assertEquals(reference, schemaAndValue.value());
+    }
+
+    @Test
+    public void highPrecisionNumericDecimalToConnect() {
+        // this number is too big to be kept in a float64!
+        BigDecimal reference = new BigDecimal("1.23456789123456789");
+        Schema schema = Decimal.schema(17);
+        String msg = "{ \"schema\": { \"type\": \"bytes\", \"name\": \"org.apache.kafka.connect.data.Decimal\", \"version\": 1, \"parameters\": { \"scale\": \"17\" } }, \"payload\": 1.23456789123456789 }";
+        SchemaAndValue schemaAndValue = converter.toConnectData(TOPIC, msg.getBytes());
+        assertEquals(schema, schemaAndValue.schema());
+        assertEquals(reference, schemaAndValue.value());
+    }
+
     @Test
     public void dateToConnect() {
         Schema schema = Date.SCHEMA;
@@ -587,9 +609,29 @@ public void decimalToJson() throws IOException {
         validateEnvelope(converted);
         assertEquals(parse("{ \"type\": \"bytes\", \"optional\": false, \"name\": \"org.apache.kafka.connect.data.Decimal\", \"version\": 1, \"parameters\": { \"scale\": \"2\" } }"),
                 converted.get(JsonSchema.ENVELOPE_SCHEMA_FIELD_NAME));
+        assertTrue("expected node to be base64 text", converted.get(JsonSchema.ENVELOPE_PAYLOAD_FIELD_NAME).isTextual());
         assertArrayEquals(new byte[]{0, -100}, converted.get(JsonSchema.ENVELOPE_PAYLOAD_FIELD_NAME).binaryValue());
     }
 
+    @Test
+    public void decimalToNumericJson() {
+        converter.configure(Collections.singletonMap(JsonConverterConfig.DECIMAL_FORMAT_CONFIG, DecimalFormat.NUMERIC.name()), false);
+        JsonNode converted = parse(converter.fromConnectData(TOPIC, Decimal.schema(2), new BigDecimal(new BigInteger("156"), 2)));
+        validateEnvelope(converted);
+        assertEquals(parse("{ \"type\": \"bytes\", \"optional\": false, \"name\": \"org.apache.kafka.connect.data.Decimal\", \"version\": 1, \"parameters\": { \"scale\": \"2\" } }"),
+            converted.get(JsonSchema.ENVELOPE_SCHEMA_FIELD_NAME));
+        assertTrue("expected node to be numeric", converted.get(JsonSchema.ENVELOPE_PAYLOAD_FIELD_NAME).isNumber());
+        assertEquals(new BigDecimal("1.56"), converted.get(JsonSchema.ENVELOPE_PAYLOAD_FIELD_NAME).decimalValue());
+    }
+
+    @Test
+    public void decimalToJsonWithoutSchema() throws IOException {
+        assertThrows(
+            "expected data exception when serializing BigDecimal without schema",
+            DataException.class,
+            () -> converter.fromConnectData(TOPIC, null, new BigDecimal(new BigInteger("156"), 2)));
+    }
+
     @Test
     public void dateToJson() {
         GregorianCalendar calendar = new GregorianCalendar(1970, Calendar.JANUARY, 1, 0, 0, 0);

From 3c87c9b5382891c924784828bad3d6a1b32940c6 Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Tue, 1 Oct 2019 22:25:56 +0100
Subject: [PATCH 0668/1071] KAFKA-8887; Use purgatory for ACL updates using
 async authorizers (#7404)

Reviewers: Manikumar Reddy 
---
 .../scala/kafka/server/DelayedFuture.scala    | 100 +++++++++++
 .../scala/kafka/server/DelayedOperation.scala |   2 +
 .../main/scala/kafka/server/KafkaApis.scala   |  41 +++--
 .../api/SslAdminClientIntegrationTest.scala   | 168 ++++++++++++++++--
 .../kafka/server/DelayedOperationTest.scala   |  65 +++++++
 5 files changed, 346 insertions(+), 30 deletions(-)
 create mode 100644 core/src/main/scala/kafka/server/DelayedFuture.scala

diff --git a/core/src/main/scala/kafka/server/DelayedFuture.scala b/core/src/main/scala/kafka/server/DelayedFuture.scala
new file mode 100644
index 0000000000000..f7e1e3fbf3d98
--- /dev/null
+++ b/core/src/main/scala/kafka/server/DelayedFuture.scala
@@ -0,0 +1,100 @@
+/**
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *    http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+
+package kafka.server
+
+import java.util.concurrent._
+import java.util.function.BiConsumer
+
+import org.apache.kafka.common.errors.TimeoutException
+import org.apache.kafka.common.utils.KafkaThread
+
+import scala.collection.Seq
+
+/**
+  * A delayed operation using CompletionFutures that can be created by KafkaApis and watched
+  * in a DelayedFuturePurgatory purgatory. This is used for ACL updates using async Authorizers.
+  */
+class DelayedFuture[T](timeoutMs: Long,
+                       futures: List[CompletableFuture[T]],
+                       responseCallback: () => Unit)
+  extends DelayedOperation(timeoutMs) {
+
+  /**
+   * The operation can be completed if all the futures have completed successfully
+   * or failed with exceptions.
+   */
+  override def tryComplete() : Boolean = {
+    trace(s"Trying to complete operation for ${futures.size} futures")
+
+    val pending = futures.count(future => !future.isDone)
+    if (pending == 0) {
+      trace("All futures have been completed or have errors, completing the delayed operation")
+      forceComplete()
+    } else {
+      trace(s"$pending future still pending, not completing the delayed operation")
+      false
+    }
+  }
+
+  /**
+   * Timeout any pending futures and invoke responseCallback. This is invoked when all
+   * futures have completed or the operation has timed out.
+   */
+  override def onComplete(): Unit = {
+    val pendingFutures = futures.filterNot(_.isDone)
+    trace(s"Completing operation for ${futures.size} futures, expired ${pendingFutures.size}")
+    pendingFutures.foreach(_.completeExceptionally(new TimeoutException(s"Request has been timed out after $timeoutMs ms")))
+    responseCallback.apply()
+  }
+
+  /**
+   * This is invoked after onComplete(), so no actions required.
+   */
+  override def onExpiration(): Unit = {
+  }
+}
+
+class DelayedFuturePurgatory(purgatoryName: String, brokerId: Int) {
+  private val purgatory = DelayedOperationPurgatory[DelayedFuture[_]](purgatoryName, brokerId)
+  private val executor = new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS,
+    new LinkedBlockingQueue[Runnable](),
+    new ThreadFactory {
+      override def newThread(r: Runnable): Thread = new KafkaThread(s"DelayedExecutor-$purgatoryName", r, true)
+    })
+  val purgatoryKey = new Object
+
+  def tryCompleteElseWatch[T](timeoutMs: Long,
+                              futures: List[CompletableFuture[T]],
+                              responseCallback: () => Unit): DelayedFuture[T] = {
+    val delayedFuture = new DelayedFuture[T](timeoutMs, futures, responseCallback)
+    val done = purgatory.tryCompleteElseWatch(delayedFuture, Seq(purgatoryKey))
+    if (!done) {
+      val callbackAction = new BiConsumer[Void, Throwable]() {
+        override def accept(result: Void, exception: Throwable): Unit = delayedFuture.forceComplete()
+      }
+      CompletableFuture.allOf(futures.toArray: _*).whenCompleteAsync(callbackAction, executor)
+    }
+    delayedFuture
+  }
+
+  def shutdown() {
+    executor.shutdownNow()
+    executor.awaitTermination(60, TimeUnit.SECONDS)
+    purgatory.shutdown()
+  }
+}
diff --git a/core/src/main/scala/kafka/server/DelayedOperation.scala b/core/src/main/scala/kafka/server/DelayedOperation.scala
index 5e965c4fb6362..51c0e1a891913 100644
--- a/core/src/main/scala/kafka/server/DelayedOperation.scala
+++ b/core/src/main/scala/kafka/server/DelayedOperation.scala
@@ -360,6 +360,8 @@ final class DelayedOperationPurgatory[T <: DelayedOperation](purgatoryName: Stri
     if (reaperEnabled)
       expirationReaper.shutdown()
     timeoutTimer.shutdown()
+    removeMetric("PurgatorySize", metricsTags)
+    removeMetric("NumDelayedOperations", metricsTags)
   }
 
   /**
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 109530c1b51d7..aa725930c71ea 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -22,7 +22,7 @@ import java.lang.{Long => JLong}
 import java.nio.ByteBuffer
 import java.util
 import java.util.{Collections, Optional}
-import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.{CompletableFuture, ConcurrentHashMap}
 import java.util.concurrent.atomic.AtomicInteger
 
 import kafka.admin.{AdminUtils, RackAwareMode}
@@ -108,8 +108,10 @@ class KafkaApis(val requestChannel: RequestChannel,
   type FetchResponseStats = Map[TopicPartition, RecordConversionStats]
   this.logIdent = "[KafkaApi-%d] ".format(brokerId)
   val adminZkClient = new AdminZkClient(zkClient)
+  private val alterAclsPurgatory = new DelayedFuturePurgatory(purgatoryName = "AlterAcls", brokerId = config.brokerId)
 
   def close(): Unit = {
+    alterAclsPurgatory.shutdown()
     info("Shutdown complete.")
   }
 
@@ -2207,15 +2209,19 @@ class KafkaApis(val requestChannel: RequestChannel,
             } else
               true
           }
-        val createResults = auth.createAcls(request.context, validBindings.asJava)
 
-        val aclCreationResults = aclBindings.map { acl =>
-          val result = errorResults.getOrElse(acl, createResults.get(validBindings.indexOf(acl)).toCompletableFuture.get)
-          new AclCreationResponse(result.exception.asScala.map(ApiError.fromThrowable).getOrElse(ApiError.NONE))
+        val createResults = auth.createAcls(request.context, validBindings.asJava)
+          .asScala.map(_.toCompletableFuture).toList
+        def sendResponseCallback(): Unit = {
+          val aclCreationResults = aclBindings.map { acl =>
+            val result = errorResults.getOrElse(acl, createResults(validBindings.indexOf(acl)).get)
+            new AclCreationResponse(result.exception.asScala.map(ApiError.fromThrowable).getOrElse(ApiError.NONE))
+          }
+          sendResponseMaybeThrottle(request, requestThrottleMs =>
+            new CreateAclsResponse(requestThrottleMs, aclCreationResults.asJava))
         }
 
-        sendResponseMaybeThrottle(request, requestThrottleMs =>
-          new CreateAclsResponse(requestThrottleMs, aclCreationResults.asJava))
+        alterAclsPurgatory.tryCompleteElseWatch(config.connectionsMaxIdleMs, createResults, sendResponseCallback)
     }
   }
 
@@ -2229,17 +2235,24 @@ class KafkaApis(val requestChannel: RequestChannel,
             new SecurityDisabledException("No Authorizer is configured on the broker.")))
       case Some(auth) =>
 
-        val results = auth.deleteAcls(request.context, deleteAclsRequest.filters)
+        val deleteResults = auth.deleteAcls(request.context, deleteAclsRequest.filters)
+          .asScala.map(_.toCompletableFuture).toList
+
         def toErrorCode(exception: Optional[ApiException]): ApiError = {
           exception.asScala.map(ApiError.fromThrowable).getOrElse(ApiError.NONE)
         }
-        val filterResponses = results.asScala.map(_.toCompletableFuture.get).map { result =>
-          val deletions = result.aclBindingDeleteResults().asScala.toList.map { deletionResult =>
-            new AclDeletionResult(toErrorCode(deletionResult.exception), deletionResult.aclBinding)
+
+        def sendResponseCallback(): Unit = {
+          val filterResponses = deleteResults.map(_.get).map { result =>
+            val deletions = result.aclBindingDeleteResults().asScala.toList.map { deletionResult =>
+              new AclDeletionResult(toErrorCode(deletionResult.exception), deletionResult.aclBinding)
+            }.asJava
+            new AclFilterResponse(toErrorCode(result.exception), deletions)
           }.asJava
-          new AclFilterResponse(toErrorCode(result.exception), deletions)
-        }.asJava
-        sendResponseMaybeThrottle(request, requestThrottleMs => new DeleteAclsResponse(requestThrottleMs, filterResponses))
+          sendResponseMaybeThrottle(request, requestThrottleMs =>
+            new DeleteAclsResponse(requestThrottleMs, filterResponses))
+        }
+        alterAclsPurgatory.tryCompleteElseWatch(config.connectionsMaxIdleMs, deleteResults, sendResponseCallback)
     }
   }
 
diff --git a/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala
index 689cb0aef9f84..071bd4c05acc1 100644
--- a/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala
@@ -15,14 +15,17 @@ package kafka.api
 import java.io.File
 import java.util
 import java.util.Collections
-import java.util.concurrent.{CompletionStage, Semaphore}
+import java.util.concurrent._
+import java.util.function.BiConsumer
 
+import com.yammer.metrics.Metrics
+import com.yammer.metrics.core.Gauge
 import kafka.security.authorizer.AclAuthorizer
 import kafka.security.authorizer.AuthorizerUtils.{WildcardHost, WildcardPrincipal}
 import kafka.security.auth.{Operation, PermissionType}
 import kafka.server.KafkaConfig
 import kafka.utils.{CoreUtils, TestUtils}
-import org.apache.kafka.clients.admin.AdminClient
+import org.apache.kafka.clients.admin.{AdminClient, AdminClientConfig, CreateAclsResult}
 import org.apache.kafka.common.acl._
 import org.apache.kafka.common.acl.AclOperation._
 import org.apache.kafka.common.acl.AclPermissionType._
@@ -32,35 +35,55 @@ import org.apache.kafka.common.resource.PatternType._
 import org.apache.kafka.common.resource.ResourceType._
 import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
 import org.apache.kafka.server.authorizer._
-import org.junit.Assert.{assertEquals, assertFalse, assertTrue}
+import org.junit.Assert.{assertEquals, assertFalse, assertNotNull, assertTrue}
 import org.junit.{Assert, Test}
 
 import scala.collection.JavaConverters._
+import scala.collection.mutable
 
 object SslAdminClientIntegrationTest {
   @volatile var semaphore: Option[Semaphore] = None
+  @volatile var executor: Option[ExecutorService] = None
   @volatile var lastUpdateRequestContext: Option[AuthorizableRequestContext] = None
   class TestableAclAuthorizer extends AclAuthorizer {
     override def createAcls(requestContext: AuthorizableRequestContext,
                             aclBindings: util.List[AclBinding]): util.List[_ <: CompletionStage[AclCreateResult]] = {
       lastUpdateRequestContext = Some(requestContext)
-      semaphore.foreach(_.acquire())
-      try {
-        super.createAcls(requestContext, aclBindings)
-      } finally {
-        semaphore.foreach(_.release())
-      }
+      execute[AclCreateResult](aclBindings.size, () => super.createAcls(requestContext, aclBindings))
     }
 
     override def deleteAcls(requestContext: AuthorizableRequestContext,
                             aclBindingFilters: util.List[AclBindingFilter]): util.List[_ <: CompletionStage[AclDeleteResult]] = {
       lastUpdateRequestContext = Some(requestContext)
-      semaphore.foreach(_.acquire())
-      try {
-        super.deleteAcls(requestContext, aclBindingFilters)
-      } finally {
-        semaphore.foreach(_.release())
+      execute[AclDeleteResult](aclBindingFilters.size, () => super.deleteAcls(requestContext, aclBindingFilters))
+    }
+
+    private def execute[T](batchSize: Int, action: () => util.List[_ <: CompletionStage[T]]): util.List[CompletableFuture[T]] = {
+      val futures = (0 until batchSize).map(_ => new CompletableFuture[T]).toList
+      val runnable = new Runnable {
+        override def run(): Unit = {
+          semaphore.foreach(_.acquire())
+          try {
+            action.apply().asScala.zip(futures).foreach { case (baseFuture, resultFuture) =>
+              baseFuture.whenComplete(new BiConsumer[T, Throwable]() {
+                override def accept(result: T, exception: Throwable): Unit = {
+                  if (exception != null)
+                    resultFuture.completeExceptionally(exception)
+                  else
+                    resultFuture.complete(result)
+                }
+              })
+            }
+          } finally {
+            semaphore.foreach(_.release())
+          }
+        }
       }
+      executor match {
+        case Some(executorService) => executorService.submit(runnable)
+        case None => runnable.run()
+      }
+      futures.asJava
     }
   }
 }
@@ -71,6 +94,7 @@ class SslAdminClientIntegrationTest extends SaslSslAdminClientIntegrationTest {
 
   override protected def securityProtocol = SecurityProtocol.SSL
   override protected lazy val trustStoreFile = Some(File.createTempFile("truststore", ".jks"))
+  private val adminClients = mutable.Buffer.empty[AdminClient]
 
   override def configureSecurityBeforeServersStart(): Unit = {
     val authorizer = CoreUtils.createObject[Authorizer](classOf[AclAuthorizer].getName)
@@ -92,9 +116,23 @@ class SslAdminClientIntegrationTest extends SaslSslAdminClientIntegrationTest {
   }
 
   override def setUpSasl(): Unit = {
+    SslAdminClientIntegrationTest.semaphore = None
+    SslAdminClientIntegrationTest.executor = None
+    SslAdminClientIntegrationTest.lastUpdateRequestContext = None
+
     startSasl(jaasSections(List.empty, None, ZkSasl))
   }
 
+  override def tearDown(): Unit = {
+    // Ensure semaphore doesn't block shutdown even if test has failed
+    val semaphore = SslAdminClientIntegrationTest.semaphore
+    SslAdminClientIntegrationTest.semaphore = None
+    semaphore.foreach(s => s.release(s.getQueueLength))
+
+    adminClients.foreach(_.close())
+    super.tearDown()
+  }
+
   override def addClusterAcl(permissionType: PermissionType, operation: Operation): Unit = {
     val ace = clusterAcl(permissionType.toJava, operation.toJava)
     val aclBinding = new AclBinding(clusterResourcePattern, ace)
@@ -122,9 +160,74 @@ class SslAdminClientIntegrationTest extends SaslSslAdminClientIntegrationTest {
   }
 
   @Test
-  def testAsyncAclUpdates(): Unit = {
-    client = AdminClient.create(createConfig())
+  def testAclUpdatesUsingSynchronousAuthorizer(): Unit = {
+    verifyAclUpdates()
+  }
 
+  @Test
+  def testAclUpdatesUsingAsynchronousAuthorizer(): Unit = {
+    SslAdminClientIntegrationTest.executor = Some(Executors.newSingleThreadExecutor)
+    verifyAclUpdates()
+  }
+
+  /**
+   * Verify that ACL updates using synchronous authorizer are performed synchronously
+   * on request threads without any performance overhead introduced by a purgatory.
+   */
+  @Test
+  def testSynchronousAuthorizerAclUpdatesBlockRequestThreads(): Unit = {
+    val testSemaphore = new Semaphore(0)
+    SslAdminClientIntegrationTest.semaphore = Some(testSemaphore)
+    waitForNoBlockedRequestThreads()
+
+    // Queue requests until all threads are blocked. ACL create requests are sent to least loaded
+    // node, so we may need more than `numRequestThreads` requests to block all threads.
+    val aclFutures = mutable.Buffer[CreateAclsResult]()
+    while (blockedRequestThreads.size < numRequestThreads) {
+      aclFutures += createAdminClient.createAcls(List(acl2).asJava)
+      assertTrue(s"Request threads not blocked numRequestThreads=$numRequestThreads blocked=$blockedRequestThreads",
+        aclFutures.size < numRequestThreads * 10)
+    }
+    assertEquals(0, purgatoryMetric("NumDelayedOperations"))
+    assertEquals(0, purgatoryMetric("PurgatorySize"))
+
+    // Verify that operations on other clients are blocked
+    val describeFuture = createAdminClient.describeCluster().clusterId()
+    assertFalse(describeFuture.isDone)
+
+    // Release the semaphore and verify that all requests complete
+    testSemaphore.release(aclFutures.size)
+    assertNotNull(describeFuture.get(10, TimeUnit.SECONDS))
+    aclFutures.foreach(_.all().get())
+  }
+
+  /**
+   * Verify that ACL updates using an asynchronous authorizer are completed asynchronously
+   * using a purgatory, enabling other requests to be processed even when ACL updates are blocked.
+   */
+  @Test
+  def testAsynchronousAuthorizerAclUpdatesDontBlockRequestThreads(): Unit = {
+    SslAdminClientIntegrationTest.executor = Some(Executors.newSingleThreadExecutor)
+    val testSemaphore = new Semaphore(0)
+    SslAdminClientIntegrationTest.semaphore = Some(testSemaphore)
+
+    waitForNoBlockedRequestThreads()
+
+    val aclFutures = (0 until numRequestThreads).map(_ => createAdminClient.createAcls(List(acl2).asJava))
+    waitForNoBlockedRequestThreads()
+    assertTrue(aclFutures.forall(future => !future.all.isDone))
+    // Other requests should succeed even though ACL updates are blocked
+    assertNotNull(createAdminClient.describeCluster().clusterId().get(10, TimeUnit.SECONDS))
+    TestUtils.waitUntilTrue(() => purgatoryMetric("PurgatorySize") > 0, "PurgatorySize metrics not updated")
+    TestUtils.waitUntilTrue(() => purgatoryMetric("NumDelayedOperations") > 0, "NumDelayedOperations metrics not updated")
+
+    // Release the semaphore and verify that ACL update requests complete
+    testSemaphore.release(aclFutures.size)
+    aclFutures.foreach(_.all.get())
+    assertEquals(0, purgatoryMetric("NumDelayedOperations"))
+  }
+
+  private def verifyAclUpdates(): Unit = {
     def validateRequestContext(context: AuthorizableRequestContext, apiKey: ApiKeys): Unit = {
       assertEquals(SecurityProtocol.SSL, context.securityProtocol)
       assertEquals("SSL", context.listenerName)
@@ -138,6 +241,8 @@ class SslAdminClientIntegrationTest extends SaslSslAdminClientIntegrationTest {
 
     val testSemaphore = new Semaphore(0)
     SslAdminClientIntegrationTest.semaphore = Some(testSemaphore)
+
+    client = AdminClient.create(createConfig())
     val results = client.createAcls(List(acl2, acl3).asJava).values
     assertEquals(Set(acl2, acl3), results.keySet().asScala)
     assertFalse(results.values().asScala.exists(_.isDone))
@@ -158,4 +263,35 @@ class SslAdminClientIntegrationTest extends SaslSslAdminClientIntegrationTest {
     assertEquals(Set(acl3), results2.get(acl3.toFilter).get.values.asScala.map(_.binding).toSet)
     validateRequestContext(SslAdminClientIntegrationTest.lastUpdateRequestContext.get, ApiKeys.DELETE_ACLS)
   }
+
+  private def createAdminClient: AdminClient = {
+    val config = createConfig()
+    config.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, "40000")
+    val client = AdminClient.create(config)
+    adminClients += client
+    client
+  }
+
+  private def blockedRequestThreads: List[Thread] = {
+    val requestThreads = Thread.getAllStackTraces.keySet.asScala
+      .filter(_.getName.contains("data-plane-kafka-request-handler"))
+    assertEquals(numRequestThreads, requestThreads.size)
+    requestThreads.filter(_.getState == Thread.State.WAITING).toList
+  }
+
+  private def numRequestThreads = servers.head.config.numIoThreads * servers.size
+
+  private def waitForNoBlockedRequestThreads(): Unit = {
+    val (blockedThreads, _) = TestUtils.computeUntilTrue(blockedRequestThreads)(_.isEmpty)
+    assertEquals(List.empty, blockedThreads)
+  }
+
+  private def purgatoryMetric(name: String): Int = {
+    val allMetrics = Metrics.defaultRegistry.allMetrics.asScala
+    val metrics = allMetrics.filter { case (metricName, _) =>
+      metricName.getMBeanName.contains("delayedOperation=AlterAcls") && metricName.getMBeanName.contains(s"name=$name")
+    }.values.toList
+    assertTrue(s"Unable to find metric $name: allMetrics: ${allMetrics.keySet.map(_.getMBeanName)}", metrics.nonEmpty)
+    metrics.map(_.asInstanceOf[Gauge[Int]].value).sum
+  }
 }
diff --git a/core/src/test/scala/unit/kafka/server/DelayedOperationTest.scala b/core/src/test/scala/unit/kafka/server/DelayedOperationTest.scala
index 0a81eb7bb90a6..212ef80e099dc 100644
--- a/core/src/test/scala/unit/kafka/server/DelayedOperationTest.scala
+++ b/core/src/test/scala/unit/kafka/server/DelayedOperationTest.scala
@@ -27,6 +27,9 @@ import kafka.utils.TestUtils
 import org.apache.kafka.common.utils.Time
 import org.junit.{After, Before, Test}
 import org.junit.Assert._
+import org.scalatest.Assertions.intercept
+
+import scala.collection.JavaConverters._
 
 class DelayedOperationTest {
 
@@ -77,6 +80,68 @@ class DelayedOperationTest {
     assertTrue(s"Time for expiration $elapsed should at least $expiration", elapsed >= expiration)
   }
 
+  @Test
+  def testDelayedFuture(): Unit = {
+    val purgatoryName = "testDelayedFuture"
+    val purgatory = new DelayedFuturePurgatory(purgatoryName, brokerId = 0)
+    val result = new AtomicInteger()
+
+    def hasExecutorThread: Boolean = Thread.getAllStackTraces.keySet.asScala.map(_.getName)
+      .exists(_.contains(s"DelayedExecutor-$purgatoryName"))
+    def updateResult(futures: List[CompletableFuture[Integer]]): Unit =
+      result.set(futures.filterNot(_.isCompletedExceptionally).map(_.get.intValue).sum)
+
+    assertFalse("Unnecessary thread created", hasExecutorThread)
+
+    // Two completed futures: callback should be executed immediately on the same thread
+    val futures1 = List(CompletableFuture.completedFuture(10.asInstanceOf[Integer]),
+      CompletableFuture.completedFuture(11.asInstanceOf[Integer]))
+    val r1 = purgatory.tryCompleteElseWatch[Integer](100000L, futures1, () => updateResult(futures1))
+    assertTrue("r1 not completed", r1.isCompleted)
+    assertEquals(21, result.get())
+    assertFalse("Unnecessary thread created", hasExecutorThread)
+
+    // Two delayed futures: callback should wait for both to complete
+    result.set(-1)
+    val futures2 = List(new CompletableFuture[Integer], new CompletableFuture[Integer])
+    val r2 = purgatory.tryCompleteElseWatch[Integer](100000L, futures2, () => updateResult(futures2))
+    assertFalse("r2 should be incomplete", r2.isCompleted)
+    futures2.head.complete(20)
+    assertFalse(r2.isCompleted)
+    assertEquals(-1, result.get())
+    futures2(1).complete(21)
+    TestUtils.waitUntilTrue(() => r2.isCompleted, "r2 not completed")
+    TestUtils.waitUntilTrue(() => result.get == 41, "callback not invoked")
+    assertTrue("Thread not created for executing delayed task", hasExecutorThread)
+
+    // One immediate and one delayed future: callback should wait for delayed task to complete
+    result.set(-1)
+    val futures3 = List(new CompletableFuture[Integer], CompletableFuture.completedFuture(31.asInstanceOf[Integer]))
+    val r3 = purgatory.tryCompleteElseWatch[Integer](100000L, futures3, () => updateResult(futures3))
+    assertFalse("r3 should be incomplete", r3.isCompleted)
+    assertEquals(-1, result.get())
+    futures3.head.complete(30)
+    TestUtils.waitUntilTrue(() => r3.isCompleted, "r3 not completed")
+    TestUtils.waitUntilTrue(() => result.get == 61, "callback not invoked")
+
+
+    // One future doesn't complete within timeout. Should expire and invoke callback after timeout.
+    result.set(-1)
+    val start = Time.SYSTEM.hiResClockMs
+    val expirationMs = 2000L
+    val futures4 = List(new CompletableFuture[Integer], new CompletableFuture[Integer])
+    val r4 = purgatory.tryCompleteElseWatch[Integer](expirationMs, futures4, () => updateResult(futures4))
+    futures4.head.complete(40)
+    TestUtils.waitUntilTrue(() => futures4(1).isDone, "r4 futures not expired")
+    assertTrue("r4 not completed after timeout", r4.isCompleted)
+    val elapsed = Time.SYSTEM.hiResClockMs - start
+    assertTrue(s"Time for expiration $elapsed should at least $expirationMs", elapsed >= expirationMs)
+    assertEquals(40, futures4.head.get)
+    assertEquals(classOf[org.apache.kafka.common.errors.TimeoutException],
+      intercept[ExecutionException](futures4(1).get).getCause.getClass)
+    assertEquals(40, result.get())
+  }
+
   @Test
   def testRequestPurge(): Unit = {
     val r1 = new MockDelayedOperation(100000L)

From 3ca204b427971af161fa3550c46d9ec796e8b401 Mon Sep 17 00:00:00 2001
From: Bruno Cadonna 
Date: Wed, 2 Oct 2019 16:54:08 +0200
Subject: [PATCH 0669/1071] MINOR: Shutdown RockDB metrics recording trigger
 thread (#7417)

added shutdown for thread that triggers recording of RocksDBMetrics
added unit tests to verify the start and shutdown of the thread
refactored a bit of code

Reviewers: Christopher Pettitt , Bill Bejeck 
---
 .../apache/kafka/streams/KafkaStreams.java    | 27 +++++--
 .../kafka/streams/KafkaStreamsTest.java       | 75 ++++++++++++++++---
 2 files changed, 82 insertions(+), 20 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
index 6a51d1ae83cb3..b9bc586c096e1 100644
--- a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
+++ b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
@@ -141,7 +141,7 @@ public class KafkaStreams implements AutoCloseable {
     private final StateDirectory stateDirectory;
     private final StreamsMetadataState streamsMetadataState;
     private final ScheduledExecutorService stateDirCleaner;
-    private final ScheduledExecutorService rocksDBMetricsRecordingTriggerThread;
+    private final ScheduledExecutorService rocksDBMetricsRecordingService;
     private final QueryableStoreProvider queryableStoreProvider;
     private final Admin adminClient;
 
@@ -763,11 +763,19 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder,
             return thread;
         });
 
-        rocksDBMetricsRecordingTriggerThread = Executors.newSingleThreadScheduledExecutor(r -> {
-            final Thread thread = new Thread(r, clientId + "-RocksDBMetricsRecordingTrigger");
-            thread.setDaemon(true);
-            return thread;
-        });
+        rocksDBMetricsRecordingService = maybeCreateRocksDBMetricsRecordingService(clientId, config);
+    }
+
+    private static ScheduledExecutorService maybeCreateRocksDBMetricsRecordingService(final String clientId,
+                                                                                      final StreamsConfig config) {
+        if (RecordingLevel.forName(config.getString(METRICS_RECORDING_LEVEL_CONFIG)) == RecordingLevel.DEBUG) {
+            return Executors.newSingleThreadScheduledExecutor(r -> {
+                final Thread thread = new Thread(r, clientId + "-RocksDBMetricsRecordingTrigger");
+                thread.setDaemon(true);
+                return thread;
+            });
+        }
+        return null;
     }
 
     private static HostInfo parseHostInfo(final String endPoint) {
@@ -825,8 +833,8 @@ public synchronized void start() throws IllegalStateException, StreamsException
 
             final long recordingDelay = 0;
             final long recordingInterval = 1;
-            if (RecordingLevel.forName(config.getString(METRICS_RECORDING_LEVEL_CONFIG)) == RecordingLevel.DEBUG) {
-                rocksDBMetricsRecordingTriggerThread.scheduleAtFixedRate(
+            if (rocksDBMetricsRecordingService != null) {
+                rocksDBMetricsRecordingService.scheduleAtFixedRate(
                     rocksDBMetricsRecordingTrigger,
                     recordingDelay,
                     recordingInterval,
@@ -881,6 +889,9 @@ private boolean close(final long timeoutMs) {
             log.info("Already in the pending shutdown state, wait to complete shutdown");
         } else {
             stateDirCleaner.shutdownNow();
+            if (rocksDBMetricsRecordingService != null) {
+                rocksDBMetricsRecordingService.shutdownNow();
+            }
 
             // wait for all threads to join in a separate thread;
             // save the current thread so that if it is a stream thread
diff --git a/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java b/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java
index 5bf21ea9ce1fd..fae9020ecac50 100644
--- a/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java
@@ -23,6 +23,7 @@
 import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.MetricsReporter;
+import org.apache.kafka.common.metrics.Sensor.RecordingLevel;
 import org.apache.kafka.common.serialization.Serdes;
 import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.common.utils.MockTime;
@@ -105,8 +106,6 @@ public class KafkaStreamsTest {
     @Mock
     private GlobalStreamThread globalStreamThread;
     @Mock
-    private ScheduledExecutorService cleanupSchedule;
-    @Mock
     private Metrics metrics;
 
     private StateListenerStub streamsStateListener;
@@ -671,33 +670,85 @@ public void shouldNotBlockInCloseForZeroDuration() {
         }
     }
 
+    @Test
+    public void shouldTriggerRecordingOfRocksDBMetricsIfRecordingLevelIsDebug() {
+        PowerMock.mockStatic(Executors.class);
+        final ScheduledExecutorService cleanupSchedule = EasyMock.niceMock(ScheduledExecutorService.class);
+        final ScheduledExecutorService rocksDBMetricsRecordingTriggerThread =
+            EasyMock.mock(ScheduledExecutorService.class);
+        EasyMock.expect(Executors.newSingleThreadScheduledExecutor(
+            anyObject(ThreadFactory.class)
+        )).andReturn(cleanupSchedule);
+        EasyMock.expect(Executors.newSingleThreadScheduledExecutor(
+            anyObject(ThreadFactory.class)
+        )).andReturn(rocksDBMetricsRecordingTriggerThread);
+        EasyMock.expect(rocksDBMetricsRecordingTriggerThread.scheduleAtFixedRate(
+            EasyMock.anyObject(RocksDBMetricsRecordingTrigger.class),
+            EasyMock.eq(0L),
+            EasyMock.eq(1L),
+            EasyMock.eq(TimeUnit.MINUTES)
+        )).andReturn(null);
+        EasyMock.expect(rocksDBMetricsRecordingTriggerThread.shutdownNow()).andReturn(null);
+        PowerMock.replay(Executors.class);
+        PowerMock.replay(rocksDBMetricsRecordingTriggerThread);
+        PowerMock.replay(cleanupSchedule);
+
+        final StreamsBuilder builder = new StreamsBuilder();
+        builder.table("topic", Materialized.as("store"));
+        props.setProperty(StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG, RecordingLevel.DEBUG.name());
+
+        try (final KafkaStreams streams = new KafkaStreams(builder.build(), props, supplier, time)) {
+            streams.start();
+        }
+
+        PowerMock.verify(Executors.class);
+        PowerMock.verify(rocksDBMetricsRecordingTriggerThread);
+    }
+
+    @Test
+    public void shouldNotTriggerRecordingOfRocksDBMetricsIfRecordingLevelIsInfo() {
+        PowerMock.mockStatic(Executors.class);
+        final ScheduledExecutorService cleanupSchedule = EasyMock.niceMock(ScheduledExecutorService.class);
+        final ScheduledExecutorService rocksDBMetricsRecordingTriggerThread =
+            EasyMock.mock(ScheduledExecutorService.class);
+        EasyMock.expect(Executors.newSingleThreadScheduledExecutor(
+            anyObject(ThreadFactory.class)
+        )).andReturn(cleanupSchedule);
+        PowerMock.replay(Executors.class, rocksDBMetricsRecordingTriggerThread, cleanupSchedule);
+
+        final StreamsBuilder builder = new StreamsBuilder();
+        builder.table("topic", Materialized.as("store"));
+        props.setProperty(StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG, RecordingLevel.INFO.name());
+
+        try (final KafkaStreams streams = new KafkaStreams(builder.build(), props, supplier, time)) {
+            streams.start();
+        }
+
+        PowerMock.verify(Executors.class, rocksDBMetricsRecordingTriggerThread);
+    }
+
     @Test
     public void shouldCleanupOldStateDirs() throws Exception {
         PowerMock.mockStatic(Executors.class);
+        final ScheduledExecutorService cleanupSchedule = EasyMock.mock(ScheduledExecutorService.class);
         EasyMock.expect(Executors.newSingleThreadScheduledExecutor(
             anyObject(ThreadFactory.class)
         )).andReturn(cleanupSchedule).anyTimes();
-
-        cleanupSchedule.scheduleAtFixedRate(
+        EasyMock.expect(cleanupSchedule.scheduleAtFixedRate(
             EasyMock.anyObject(Runnable.class),
             EasyMock.eq(1L),
             EasyMock.eq(1L),
             EasyMock.eq(TimeUnit.MILLISECONDS)
-        );
-        EasyMock.expectLastCall().andReturn(null);
-        cleanupSchedule.shutdownNow();
-        EasyMock.expectLastCall().andReturn(null);
-
+        )).andReturn(null);
+        EasyMock.expect(cleanupSchedule.shutdownNow()).andReturn(null);
         PowerMock.expectNew(StateDirectory.class,
             anyObject(StreamsConfig.class),
             anyObject(Time.class),
             EasyMock.eq(true)
         ).andReturn(stateDirectory);
-
         PowerMock.replayAll(Executors.class, cleanupSchedule, stateDirectory);
 
         props.setProperty(StreamsConfig.STATE_CLEANUP_DELAY_MS_CONFIG, "1");
-
         final StreamsBuilder builder = new StreamsBuilder();
         builder.table("topic", Materialized.as("store"));
 
@@ -705,7 +756,7 @@ public void shouldCleanupOldStateDirs() throws Exception {
             streams.start();
         }
 
-        PowerMock.verifyAll();
+        PowerMock.verify(Executors.class, cleanupSchedule);
     }
 
     @Test

From 157aadbbf38f6255208678058df506005cc066fd Mon Sep 17 00:00:00 2001
From: Jeff Huang <47870461+jeffhuang26@users.noreply.github.com>
Date: Wed, 2 Oct 2019 07:57:52 -0700
Subject: [PATCH 0670/1071] MINOR: Add provider name of SSLContext to debug
 log. (#7407)

---
 .../org/apache/kafka/common/security/ssl/SslEngineBuilder.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/security/ssl/SslEngineBuilder.java b/clients/src/main/java/org/apache/kafka/common/security/ssl/SslEngineBuilder.java
index dc49f5e9fff2c..ae76690496b7c 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/ssl/SslEngineBuilder.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/ssl/SslEngineBuilder.java
@@ -156,7 +156,8 @@ private SSLContext createSSLContext() {
             tmf.init(ts);
 
             sslContext.init(keyManagers, tmf.getTrustManagers(), this.secureRandomImplementation);
-            log.debug("Created SSL context with keystore {}, truststore {}", keystore, truststore);
+            log.debug("Created SSL context with keystore {}, truststore {}, provider {}.",
+                    keystore, truststore, sslContext.getProvider().getName());
             return sslContext;
         } catch (Exception e) {
             throw new KafkaException(e);

From c218aa63ca8cdcd9b0c2c02dd0dae600318fd47c Mon Sep 17 00:00:00 2001
From: David Jacot 
Date: Wed, 2 Oct 2019 08:16:41 -0700
Subject: [PATCH 0671/1071] KAFKA-8901; Extend consumer group command to use
 the new Admin API to delete consumer offsets (KIP-496)

It add support to delete offsets in the `kafka-consumer-group`.

*More detailed description of your change,
if necessary. The PR title and PR message become
the squashed commit message, so use a separate
comment to ping reviewers.*

*Summary of testing strategy (including rationale)
for the feature or bug fix. Unit and/or integration
tests are expected for any behaviour change and
system tests should be considered for larger changes.*

Author: David Jacot 

Reviewers: Gwen Shapira

Closes #7362 from dajac/KAFKA-8901-delete-offsets-command
---
 .../kafka/admin/ConsumerGroupCommand.scala    |  98 ++++++++-
 .../api/AdminClientIntegrationTest.scala      |  10 +-
 .../admin/ConsumerGroupCommandTest.scala      |  22 +-
 ...sConsumerGroupCommandIntegrationTest.scala | 199 ++++++++++++++++++
 .../scala/unit/kafka/utils/TestUtils.scala    |  11 +
 5 files changed, 320 insertions(+), 20 deletions(-)
 create mode 100644 core/src/test/scala/unit/kafka/admin/DeleteOffsetsConsumerGroupCommandIntegrationTest.scala

diff --git a/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala b/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala
index 73c54faf0b7e8..519b33535668a 100755
--- a/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala
+++ b/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala
@@ -20,6 +20,7 @@ package kafka.admin
 import java.text.{ParseException, SimpleDateFormat}
 import java.time.{Duration, Instant}
 import java.util.Properties
+import java.util.concurrent.ExecutionException
 
 import com.fasterxml.jackson.dataformat.csv.CsvMapper
 import com.fasterxml.jackson.module.scala.DefaultScalaModule
@@ -34,9 +35,11 @@ import org.apache.kafka.common.{KafkaException, Node, TopicPartition}
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable.ListBuffer
-import scala.collection.{immutable, Map, Seq, Set, mutable}
+import scala.collection.{Map, Seq, Set, immutable, mutable}
 import scala.util.{Failure, Success, Try}
 import joptsimple.OptionSpec
+import org.apache.kafka.common.protocol.Errors
+
 import scala.collection.immutable.TreeMap
 import scala.reflect.ClassTag
 
@@ -48,9 +51,9 @@ object ConsumerGroupCommand extends Logging {
     CommandLineUtils.printHelpAndExitIfNeeded(opts, "This tool helps to list all consumer groups, describe a consumer group, delete consumer group info, or reset consumer group offsets.")
 
     // should have exactly one action
-    val actions = Seq(opts.listOpt, opts.describeOpt, opts.deleteOpt, opts.resetOffsetsOpt).count(opts.options.has)
+    val actions = Seq(opts.listOpt, opts.describeOpt, opts.deleteOpt, opts.resetOffsetsOpt, opts.deleteOffsetsOpt).count(opts.options.has)
     if (actions != 1)
-      CommandLineUtils.printUsageAndDie(opts.parser, "Command must include exactly one action: --list, --describe, --delete, --reset-offsets")
+      CommandLineUtils.printUsageAndDie(opts.parser, "Command must include exactly one action: --list, --describe, --delete, --reset-offsets, --delete-offsets")
 
     opts.checkArgs()
 
@@ -71,6 +74,9 @@ object ConsumerGroupCommand extends Logging {
         } else
           printOffsetsToReset(offsetsToReset)
       }
+      else if (opts.options.has(opts.deleteOffsetsOpt)) {
+        consumerGroupService.deleteOffsets()
+      }
     } catch {
       case e: Throwable =>
         printError(s"Executing consumer group command failed due to ${e.getMessage}", Some(e))
@@ -395,6 +401,83 @@ object ConsumerGroupCommand extends Logging {
       result
     }
 
+    def deleteOffsets(groupId: String, topics: List[String]): Map[TopicPartition, Throwable] = {
+      var result: Map[TopicPartition, Throwable] = mutable.HashMap()
+
+      val (topicWithPartitions, topicWithoutPartitions) = topics.partition(_.contains(":"))
+
+      val knownPartitions = topicWithPartitions.flatMap { topicArg =>
+        val split = topicArg.split(":")
+        split(1).split(",").map { partition =>
+          new TopicPartition(split(0), partition.toInt)
+        }
+      }
+
+      // Get the partitions of topics that the user did not explicitly specify the partitions
+      val describeTopicsResult = adminClient.describeTopics(
+        topicWithoutPartitions.asJava,
+        withTimeoutMs(new DescribeTopicsOptions))
+
+      val unknownPartitions = describeTopicsResult.values().asScala.flatMap { case (topic, future) =>
+        Try(future.get()) match {
+          case Success(description) => description.partitions().asScala.map { partition =>
+            new TopicPartition(topic, partition.partition())
+          }
+          case Failure(e) =>
+            result += new TopicPartition(topic, -1) -> e
+            List.empty
+        }
+      }
+
+      val partitions = knownPartitions ++ unknownPartitions
+
+      val deleteResult = adminClient.deleteConsumerGroupOffsets(
+        groupId,
+        partitions.toSet.asJava,
+        withTimeoutMs(new DeleteConsumerGroupOffsetsOptions)
+      )
+
+      deleteResult.all().get()
+
+      partitions.foreach { partition =>
+        Try(deleteResult.partitionResult(partition).get()) match {
+          case Success(_) => result += partition -> null
+          case Failure(e) => result += partition -> e
+        }
+      }
+
+      result
+    }
+
+    def deleteOffsets(): Unit = {
+      val groupId = opts.options.valueOf(opts.groupOpt)
+      val topics = opts.options.valuesOf(opts.topicOpt).asScala.toList
+
+      try {
+        val result = deleteOffsets(groupId, topics)
+
+        println("\n%-30s %-15s %-15s".format("TOPIC", "PARTITION", "STATUS"))
+        result.toList.sortBy(t => t._1.topic + t._1.partition.toString).foreach { case (tp, error) =>
+          println("%-30s %-15s %-15s".format(
+            tp.topic,
+            if (tp.partition >= 0) tp.partition else "Not Provided",
+            if (error != null) s"Error: ${error.getMessage}" else "Successful"
+          ))
+        }
+      } catch {
+        case e: ExecutionException =>
+          Errors.forException(e.getCause) match {
+            case Errors.INVALID_GROUP_ID | Errors.GROUP_ID_NOT_FOUND =>
+              printError(s"'$groupId' is not valid or does not exist.")
+            case Errors.GROUP_AUTHORIZATION_FAILED =>
+              printError(s"Access to '$groupId' is not authorized.")
+            case Errors.NON_EMPTY_GROUP =>
+              printError(s"Deleting offsets of a non consumer group '$groupId' is forbidden if the group is not empty.")
+            case _ => throw e
+          }
+      }
+    }
+
     private[admin] def describeConsumerGroups(groupIds: Seq[String]): mutable.Map[String, ConsumerGroupDescription] = {
       adminClient.describeConsumerGroups(
         groupIds.asJava,
@@ -850,6 +933,7 @@ object ConsumerGroupCommand extends Logging {
       "Example: --bootstrap-server localhost:9092 --describe --group group1 --offsets"
     val StateDoc = "Describe the group state. This option may be used with '--describe' and '--bootstrap-server' options only." + nl +
       "Example: --bootstrap-server localhost:9092 --describe --group group1 --state"
+    val DeleteOffsetsDoc = "Delete offsets of consumer group. Supports one consumer group at the time, and multiple topics."
 
     val bootstrapServerOpt = parser.accepts("bootstrap-server", BootstrapServerDoc)
                                    .withRequiredArg
@@ -878,6 +962,7 @@ object ConsumerGroupCommand extends Logging {
                                   .describedAs("command config property file")
                                   .ofType(classOf[String])
     val resetOffsetsOpt = parser.accepts("reset-offsets", ResetOffsetsDoc)
+    val deleteOffsetsOpt = parser.accepts("delete-offsets", DeleteOffsetsDoc)
     val dryRunOpt = parser.accepts("dry-run", DryRunDoc)
     val executeOpt = parser.accepts("execute", ExecuteDoc)
     val exportOpt = parser.accepts("export", ExportDoc)
@@ -921,6 +1006,7 @@ object ConsumerGroupCommand extends Logging {
     val allConsumerGroupLevelOpts: Set[OptionSpec[_]]  = Set(listOpt, describeOpt, deleteOpt, resetOffsetsOpt)
     val allResetOffsetScenarioOpts: Set[OptionSpec[_]] = Set(resetToOffsetOpt, resetShiftByOpt,
       resetToDatetimeOpt, resetByDurationOpt, resetToEarliestOpt, resetToLatestOpt, resetToCurrentOpt, resetFromFileOpt)
+    val allDeleteOffsetsOpts: Set[OptionSpec[_]] = Set(groupOpt, topicOpt)
 
     def checkArgs(): Unit = {
 
@@ -944,6 +1030,12 @@ object ConsumerGroupCommand extends Logging {
             "deletion from a consumer group.")
       }
 
+      if (options.has(deleteOffsetsOpt)) {
+        if (!options.has(groupOpt) || !options.has(topicOpt))
+          CommandLineUtils.printUsageAndDie(parser,
+            s"Option $deleteOffsetsOpt takes the following options: ${allDeleteOffsetsOpts.mkString(", ")}")
+      }
+
       if (options.has(resetOffsetsOpt)) {
         if (options.has(dryRunOpt) && options.has(executeOpt))
           CommandLineUtils.printUsageAndDie(parser, s"Option $resetOffsetsOpt only accepts one of $executeOpt and $dryRunOpt")
diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index 2b718584649ac..c4194363b3053 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -1072,14 +1072,6 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     TestUtils.pollUntilTrue(consumer, () => !consumer.assignment.isEmpty, "Expected non-empty assignment")
   }
 
-  private def subscribeAndWaitForRecords(topic: String, consumer: KafkaConsumer[Array[Byte], Array[Byte]]): Unit = {
-    consumer.subscribe(Collections.singletonList(topic))
-    TestUtils.pollRecordsUntilTrue(
-      consumer,
-      (records: ConsumerRecords[Array[Byte], Array[Byte]]) => !records.isEmpty,
-      "Expected records" )
-  }
-
   private def sendRecords(producer: KafkaProducer[Array[Byte], Array[Byte]],
                           numRecords: Int,
                           topicPartition: TopicPartition): Unit = {
@@ -1408,7 +1400,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
       val consumer = createConsumer(configOverrides = newConsumerConfig)
 
       try {
-        subscribeAndWaitForRecords(testTopicName, consumer)
+        TestUtils.subscribeAndWaitForRecords(testTopicName, consumer)
         consumer.commitSync()
 
         // Test offset deletion while consuming
diff --git a/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala b/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
index 3ee83e25eeab9..a1f58d4d21331 100644
--- a/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
@@ -90,8 +90,9 @@ class ConsumerGroupCommandTest extends KafkaServerTestHarness {
                                topic: String = topic,
                                group: String = group,
                                strategy: String = classOf[RangeAssignor].getName,
-                               customPropsOpt: Option[Properties] = None): ConsumerGroupExecutor = {
-    val executor = new ConsumerGroupExecutor(brokerList, numConsumers, group, topic, strategy, customPropsOpt)
+                               customPropsOpt: Option[Properties] = None,
+                               syncCommit: Boolean = false): ConsumerGroupExecutor = {
+    val executor = new ConsumerGroupExecutor(brokerList, numConsumers, group, topic, strategy, customPropsOpt, syncCommit)
     addExecutor(executor)
     executor
   }
@@ -112,7 +113,8 @@ class ConsumerGroupCommandTest extends KafkaServerTestHarness {
 
 object ConsumerGroupCommandTest {
 
-  abstract class AbstractConsumerRunnable(broker: String, groupId: String, customPropsOpt: Option[Properties] = None) extends Runnable {
+  abstract class AbstractConsumerRunnable(broker: String, groupId: String, customPropsOpt: Option[Properties] = None,
+                                          syncCommit: Boolean = false) extends Runnable {
     val props = new Properties
     configure(props)
     customPropsOpt.foreach(props.asScala ++= _.asScala)
@@ -130,8 +132,11 @@ object ConsumerGroupCommandTest {
     def run(): Unit = {
       try {
         subscribe()
-        while (true)
+        while (true) {
           consumer.poll(Duration.ofMillis(Long.MaxValue))
+          if (syncCommit)
+            consumer.commitSync()
+        }
       } catch {
         case _: WakeupException => // OK
       } finally {
@@ -144,8 +149,9 @@ object ConsumerGroupCommandTest {
     }
   }
 
-  class ConsumerRunnable(broker: String, groupId: String, topic: String, strategy: String, customPropsOpt: Option[Properties] = None)
-    extends AbstractConsumerRunnable(broker, groupId, customPropsOpt) {
+  class ConsumerRunnable(broker: String, groupId: String, topic: String, strategy: String,
+                         customPropsOpt: Option[Properties] = None, syncCommit: Boolean = false)
+    extends AbstractConsumerRunnable(broker, groupId, customPropsOpt, syncCommit) {
 
     override def configure(props: Properties): Unit = {
       super.configure(props)
@@ -182,11 +188,11 @@ object ConsumerGroupCommandTest {
   }
 
   class ConsumerGroupExecutor(broker: String, numConsumers: Int, groupId: String, topic: String, strategy: String,
-                              customPropsOpt: Option[Properties] = None)
+                              customPropsOpt: Option[Properties] = None, syncCommit: Boolean = false)
     extends AbstractConsumerGroupExecutor(numConsumers) {
 
     for (_ <- 1 to numConsumers) {
-      submit(new ConsumerRunnable(broker, groupId, topic, strategy, customPropsOpt))
+      submit(new ConsumerRunnable(broker, groupId, topic, strategy, customPropsOpt, syncCommit))
     }
 
   }
diff --git a/core/src/test/scala/unit/kafka/admin/DeleteOffsetsConsumerGroupCommandIntegrationTest.scala b/core/src/test/scala/unit/kafka/admin/DeleteOffsetsConsumerGroupCommandIntegrationTest.scala
new file mode 100644
index 0000000000000..608e43610004b
--- /dev/null
+++ b/core/src/test/scala/unit/kafka/admin/DeleteOffsetsConsumerGroupCommandIntegrationTest.scala
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.admin
+
+import java.util.Properties
+import java.util.concurrent.ExecutionException
+
+import kafka.server.Defaults
+import kafka.utils.TestUtils
+import org.apache.kafka.clients.consumer.ConsumerConfig
+import org.apache.kafka.clients.consumer.KafkaConsumer
+import org.apache.kafka.clients.producer.KafkaProducer
+import org.apache.kafka.clients.producer.ProducerConfig
+import org.apache.kafka.clients.producer.ProducerRecord
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.serialization.ByteArrayDeserializer
+import org.apache.kafka.common.serialization.ByteArraySerializer
+import org.apache.kafka.common.utils.Utils
+import org.junit.Test
+import org.junit.Assert._
+
+class DeleteOffsetsConsumerGroupCommandIntegrationTest extends ConsumerGroupCommandTest {
+
+  def getArgs(group: String, topic: String): Array[String] = {
+    Array(
+      "--bootstrap-server", brokerList,
+      "--delete-offsets",
+      "--group", group,
+      "--topic", topic
+    )
+  }
+
+  @Test
+  def testDeleteOffsetsNonExistingGroup(): Unit = {
+    val group = "missing.group"
+    val topic = "foo:1"
+    val service = getConsumerGroupService(getArgs(group, topic))
+    try {
+      service.deleteOffsets(group, List(topic))
+      fail("GroupIdNotFoundException should have been raised")
+    } catch {
+      case e: ExecutionException =>
+        if (e.getCause != Errors.GROUP_ID_NOT_FOUND.exception())
+          throw e
+    }
+  }
+
+  @Test
+  def testDeleteOffsetsOfStableConsumerGroupWithTopicPartition(): Unit = {
+    testWithStableConsumerGroup(topic, 0, 0, Errors.GROUP_SUBSCRIBED_TO_TOPIC)
+  }
+
+  @Test
+  def testDeleteOffsetsOfStableConsumerGroupWithTopicOnly(): Unit = {
+    testWithStableConsumerGroup(topic, -1, 0, Errors.GROUP_SUBSCRIBED_TO_TOPIC)
+  }
+
+  @Test
+  def testDeleteOffsetsOfStableConsumerGroupWithUnknownTopicPartition(): Unit = {
+    testWithStableConsumerGroup("foobar", 0, 0, Errors.UNKNOWN_TOPIC_OR_PARTITION)
+  }
+
+  @Test
+  def testDeleteOffsetsOfStableConsumerGroupWithUnknownTopicOnly(): Unit = {
+    testWithStableConsumerGroup("foobar", -1, -1, Errors.UNKNOWN_TOPIC_OR_PARTITION)
+  }
+
+  @Test
+  def testDeleteOffsetsOfEmptyConsumerGroupWithTopicPartition(): Unit = {
+    testWithEmptyConsumerGroup(topic, 0, 0, Errors.NONE)
+  }
+
+  @Test
+  def testDeleteOffsetsOfEmptyConsumerGroupWithTopicOnly(): Unit = {
+    testWithEmptyConsumerGroup(topic, -1, 0, Errors.NONE)
+  }
+
+  @Test
+  def testDeleteOffsetsOfEmptyConsumerGroupWithUnknownTopicPartition(): Unit = {
+    testWithEmptyConsumerGroup("foobar", 0, 0, Errors.UNKNOWN_TOPIC_OR_PARTITION)
+  }
+
+  @Test
+  def testDeleteOffsetsOfEmptyConsumerGroupWithUnknownTopicOnly(): Unit = {
+    testWithEmptyConsumerGroup("foobar", -1, -1, Errors.UNKNOWN_TOPIC_OR_PARTITION)
+  }
+
+  private def testWithStableConsumerGroup(inputTopic: String,
+                                          inputPartition: Int,
+                                          expectedPartition: Int,
+                                          expectedError: Errors): Unit = {
+    testWithConsumerGroup(
+      withStableConsumerGroup,
+      inputTopic,
+      inputPartition,
+      expectedPartition,
+      expectedError)
+  }
+
+  private def testWithEmptyConsumerGroup(inputTopic: String,
+                                         inputPartition: Int,
+                                         expectedPartition: Int,
+                                         expectedError: Errors): Unit = {
+    testWithConsumerGroup(
+      withEmptyConsumerGroup,
+      inputTopic,
+      inputPartition,
+      expectedPartition,
+      expectedError)
+  }
+
+  private def testWithConsumerGroup(withConsumerGroup: (=> Unit) => Unit,
+                                    inputTopic: String,
+                                    inputPartition: Int,
+                                    expectedPartition: Int,
+                                    expectedError: Errors): Unit = {
+    produceRecord()
+    withConsumerGroup {
+      val topic = if (inputPartition >= 0) inputTopic + ":" + inputPartition else inputTopic
+      val service = getConsumerGroupService(getArgs(group, topic))
+      val partitions = service.deleteOffsets(group, List(topic))
+      val tp = new TopicPartition(inputTopic, expectedPartition)
+      if (expectedError == Errors.NONE)
+        assertNull(partitions(tp))
+      else
+        assertEquals(expectedError.exception, partitions(tp).getCause)
+    }
+  }
+
+  private def produceRecord(): Unit = {
+    val producer = createProducer()
+    try {
+      producer.send(new ProducerRecord(topic, 0, null, null)).get()
+    } finally {
+      Utils.closeQuietly(producer, "producer")
+    }
+  }
+
+  private def withStableConsumerGroup(body: => Unit): Unit = {
+    val consumer = createConsumer()
+    try {
+      TestUtils.subscribeAndWaitForRecords(this.topic, consumer)
+      consumer.commitSync()
+      body
+    } finally {
+      Utils.closeQuietly(consumer, "consumer")
+    }
+  }
+
+  private def withEmptyConsumerGroup(body: => Unit): Unit = {
+    val consumer = createConsumer()
+    try {
+      TestUtils.subscribeAndWaitForRecords(this.topic, consumer)
+      consumer.commitSync()
+    } finally {
+      Utils.closeQuietly(consumer, "consumer")
+    }
+    body
+  }
+
+  private def createProducer(config: Properties = new Properties()): KafkaProducer[Array[Byte], Array[Byte]] = {
+    config.putIfAbsent(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList)
+    config.putIfAbsent(ProducerConfig.ACKS_CONFIG, "-1")
+    config.putIfAbsent(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, classOf[ByteArraySerializer].getName)
+    config.putIfAbsent(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, classOf[ByteArraySerializer].getName)
+
+    new KafkaProducer(config)
+  }
+
+  private def createConsumer(config: Properties = new Properties()): KafkaConsumer[Array[Byte], Array[Byte]] = {
+    config.putIfAbsent(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList)
+    config.putIfAbsent(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
+    config.putIfAbsent(ConsumerConfig.GROUP_ID_CONFIG, group)
+    config.putIfAbsent(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, classOf[ByteArrayDeserializer].getName)
+    config.putIfAbsent(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, classOf[ByteArrayDeserializer].getName)
+    // Increase timeouts to avoid having a rebalance during the test
+    config.putIfAbsent(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, Integer.MAX_VALUE.toString)
+    config.putIfAbsent(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, Defaults.GroupMaxSessionTimeoutMs.toString)
+
+    new KafkaConsumer(config)
+  }
+
+}
diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index f3ccd16afdec3..6df853d8660b0 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -795,6 +795,17 @@ object TestUtils extends Logging {
     }, msg = msg, pause = 0L, waitTimeMs = waitTimeMs)
   }
 
+  def subscribeAndWaitForRecords(topic: String,
+                                 consumer: KafkaConsumer[Array[Byte], Array[Byte]],
+                                 waitTimeMs: Long = JTestUtils.DEFAULT_MAX_WAIT_MS): Unit = {
+    consumer.subscribe(Collections.singletonList(topic))
+    pollRecordsUntilTrue(
+      consumer,
+      (records: ConsumerRecords[Array[Byte], Array[Byte]]) => !records.isEmpty,
+      "Expected records",
+      waitTimeMs)
+  }
+
   /**
    * Wait for the presence of an optional value.
    *

From dc7dd5f4fe3d72fcf4388e9eef48b6fce05c56bb Mon Sep 17 00:00:00 2001
From: Stanislav Kozlovski 
Date: Wed, 2 Oct 2019 16:43:57 +0100
Subject: [PATCH 0672/1071] MINOR: Do not falsely log that partitions are being
 reassigned on controller startup (#7431)

Previously we would log the following on each controller startup:
```
[2019-09-13 22:40:10,272] INFO [Controller id=2] DEPRECATED: Partitions being reassigned through ZooKeeper: Map()
```
This patch only logs the message if the map is non-empty.

Reviewers: Jason Gustafson 
---
 .../kafka/controller/KafkaController.scala    | 32 ++++++++++---------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala
index eae02feab7140..642ebaa99345e 100644
--- a/core/src/main/scala/kafka/controller/KafkaController.scala
+++ b/core/src/main/scala/kafka/controller/KafkaController.scala
@@ -883,21 +883,23 @@ class KafkaController(val config: KafkaConfig,
     */
   private def initializePartitionReassignment(): Unit = {
     val partitionsBeingReassigned = zkClient.getPartitionReassignment
-    info(s"DEPRECATED: Partitions being reassigned through ZooKeeper: $partitionsBeingReassigned")
-
-    partitionsBeingReassigned.foreach {
-      case (tp, newReplicas) =>
-        val reassignIsrChangeHandler = new PartitionReassignmentIsrChangeHandler(eventManager, tp)
-        val assignment = controllerContext.partitionFullReplicaAssignment(tp)
-        val ongoingReassignmentOption = if (assignment.isBeingReassigned)
-          Some(assignment)
-        else
-          None
-
-        controllerContext.partitionsBeingReassigned += (
-          tp -> ReassignedPartitionsContext(newReplicas, reassignIsrChangeHandler,
-            persistedInZk = true,
-            ongoingReassignmentOpt = ongoingReassignmentOption))
+    if (partitionsBeingReassigned.nonEmpty) {
+      info(s"DEPRECATED: Partitions being reassigned through ZooKeeper: $partitionsBeingReassigned")
+
+      partitionsBeingReassigned.foreach {
+        case (tp, newReplicas) =>
+          val reassignIsrChangeHandler = new PartitionReassignmentIsrChangeHandler(eventManager, tp)
+          val assignment = controllerContext.partitionFullReplicaAssignment(tp)
+          val ongoingReassignmentOption = if (assignment.isBeingReassigned)
+            Some(assignment)
+          else
+            None
+
+          controllerContext.partitionsBeingReassigned += (
+            tp -> ReassignedPartitionsContext(newReplicas, reassignIsrChangeHandler,
+              persistedInZk = true,
+              ongoingReassignmentOpt = ongoingReassignmentOption))
+      }
     }
   }
 

From 8da69936a7e7c4da51e45cdf0fa392bcb0eacbbc Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Wed, 2 Oct 2019 08:54:32 -0700
Subject: [PATCH 0673/1071] KAFKA-8649: Send latest commonly supported version
 in assignment (#7423)

Instead of sending the leader's version and having older members try to blindly upgrade.

The only other real change here is that we will also set the VERSION_PROBING error code and return early from onAssignment when we are upgrading our used subscription version (not just downgrading it) since this implies the whole group has finished the rolling upgrade and all members should rejoin with the new subscription version.

Also piggy-backing on a fix for a potentially dangerous edge case, where every thread of an instance is assigned the same set of active tasks.

Reviewers: Guozhang Wang 
---
 .../internals/StreamsPartitionAssignor.java   | 216 +++++++++++-------
 .../internals/StreamsRebalanceListener.java   |   4 +-
 .../processor/internals/TaskManager.java      |   8 +-
 .../internals/assignment/AssignmentInfo.java  |  60 +++--
 .../internals/assignment/ClientState.java     |  36 ++-
 .../StreamsPartitionAssignorTest.java         |   2 +-
 .../processor/internals/TaskManagerTest.java  |   1 +
 .../internals/assignment/ClientStateTest.java |   4 +-
 .../assignment/StickyTaskAssignorTest.java    |  34 +--
 .../streams/tests/StreamsUpgradeTest.java     |   7 +
 .../tests/streams/streams_upgrade_test.py     |  27 +--
 11 files changed, 241 insertions(+), 158 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
index 11ea3209d4105..7afc95dda841b 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
@@ -27,6 +27,7 @@
 import org.apache.kafka.common.config.ConfigException;
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.errors.StreamsException;
 import org.apache.kafka.streams.errors.TaskAssignmentException;
 import org.apache.kafka.streams.processor.TaskId;
@@ -135,8 +136,8 @@ private static class ClientMetadata {
         void addConsumer(final String consumerMemberId,
                          final SubscriptionInfo info) {
             consumers.add(consumerMemberId);
-            state.addPreviousActiveTasks(info.prevTasks());
-            state.addPreviousStandbyTasks(info.standbyTasks());
+            state.addPreviousActiveTasks(consumerMemberId, info.prevTasks());
+            state.addPreviousStandbyTasks(consumerMemberId, info.standbyTasks());
             state.incrementCapacity();
         }
 
@@ -286,6 +287,7 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
         final Set futureConsumers = new HashSet<>();
 
         int minReceivedMetadataVersion = LATEST_SUPPORTED_VERSION;
+        int minSupportedMetadataVersion = LATEST_SUPPORTED_VERSION;
 
         int futureMetadataVersion = UNKNOWN;
         for (final Map.Entry entry : subscriptions.entrySet()) {
@@ -303,6 +305,11 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
                 minReceivedMetadataVersion = usedVersion;
             }
 
+            final int latestSupportedVersion = info.latestSupportedVersion();
+            if (latestSupportedVersion < minSupportedMetadataVersion) {
+                minSupportedMetadataVersion = latestSupportedVersion;
+            }
+
             // create the new client metadata if necessary
             ClientMetadata clientMetadata = clientMetadataMap.get(info.processId());
 
@@ -572,14 +579,16 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
                 partitionsForTask,
                 partitionsByHostState,
                 futureConsumers,
-                minReceivedMetadataVersion
+                minReceivedMetadataVersion,
+                minSupportedMetadataVersion
             );
         } else {
             assignment = computeNewAssignment(
                 clientMetadataMap,
                 partitionsForTask,
                 partitionsByHostState,
-                minReceivedMetadataVersion
+                minReceivedMetadataVersion,
+                minSupportedMetadataVersion
             );
         }
 
@@ -589,7 +598,8 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
     private static Map computeNewAssignment(final Map clientsMetadata,
                                                                 final Map> partitionsForTask,
                                                                 final Map> partitionsByHostState,
-                                                                final int minUserMetadataVersion) {
+                                                                final int minUserMetadataVersion,
+                                                                final int minSupportedMetadataVersion) {
         final Map assignment = new HashMap<>();
 
         // within the client, distribute tasks to its owned consumers
@@ -605,17 +615,15 @@ private static Map computeNewAssignment(final Map> standby = new HashMap<>();
-                final List assignedPartitions = new ArrayList<>();
+                final List activeTasks = interleavedActive.get(consumerTaskIndex);
 
-                final List assignedActiveList = interleavedActive.get(consumerTaskIndex);
+                // These will be filled in by buildAssignedActiveTaskAndPartitionsList below
+                final List activePartitionsList = new ArrayList<>();
+                final List assignedActiveList = new ArrayList<>();
 
-                for (final TaskId taskId : assignedActiveList) {
-                    for (final TopicPartition partition : partitionsForTask.get(taskId)) {
-                        assignedPartitions.add(new AssignedPartition(taskId, partition));
-                    }
-                }
+                buildAssignedActiveTaskAndPartitionsList(activeTasks, activePartitionsList, assignedActiveList, partitionsForTask);
 
+                final Map> standby = new HashMap<>();
                 if (!state.standbyTasks().isEmpty()) {
                     final List assignedStandbyList = interleavedStandby.get(consumerTaskIndex);
                     for (final TaskId taskId : assignedStandbyList) {
@@ -625,22 +633,15 @@ private static Map computeNewAssignment(final Map active = new ArrayList<>();
-                final List activePartitions = new ArrayList<>();
-                for (final AssignedPartition partition : assignedPartitions) {
-                    active.add(partition.taskId);
-                    activePartitions.add(partition.partition);
-                }
-
                 // finally, encode the assignment before sending back to coordinator
                 assignment.put(
                     consumer,
                     new Assignment(
-                        activePartitions,
+                        activePartitionsList,
                         new AssignmentInfo(
                             minUserMetadataVersion,
-                            active,
+                            minSupportedMetadataVersion,
+                            assignedActiveList,
                             standby,
                             partitionsByHostState,
                             0
@@ -657,7 +658,8 @@ private static Map versionProbingAssignment(final Map> partitionsForTask,
                                                                     final Map> partitionsByHostState,
                                                                     final Set futureConsumers,
-                                                                    final int minUserMetadataVersion) {
+                                                                    final int minUserMetadataVersion,
+                                                                    final int minSupportedMetadataVersion) {
         final Map assignment = new HashMap<>();
 
         // assign previously assigned tasks to "old consumers"
@@ -668,23 +670,27 @@ private static Map versionProbingAssignment(final Map activeTasks = new ArrayList<>(clientMetadata.state.prevActiveTasks());
+                // Return the same active tasks that were claimed in the subscription
+                final List activeTasks = new ArrayList<>(clientMetadata.state.prevActiveTasksForConsumer(consumerId));
 
-                final List assignedPartitions = new ArrayList<>();
-                for (final TaskId taskId : activeTasks) {
-                    assignedPartitions.addAll(partitionsForTask.get(taskId));
-                }
+                // These will be filled in by buildAssignedActiveTaskAndPartitionsList below
+                final List activePartitionsList = new ArrayList<>();
+                final List assignedActiveList = new ArrayList<>();
+
+                buildAssignedActiveTaskAndPartitionsList(activeTasks, activePartitionsList, assignedActiveList, partitionsForTask);
 
+                // Return the same standby tasks that were claimed in the subscription
                 final Map> standbyTasks = new HashMap<>();
-                for (final TaskId taskId : clientMetadata.state.prevStandbyTasks()) {
+                for (final TaskId taskId : clientMetadata.state.prevStandbyTasksForConsumer(consumerId)) {
                     standbyTasks.put(taskId, partitionsForTask.get(taskId));
                 }
 
                 assignment.put(consumerId, new Assignment(
-                    assignedPartitions,
+                    activePartitionsList,
                     new AssignmentInfo(
                         minUserMetadataVersion,
-                        activeTasks,
+                        minSupportedMetadataVersion,
+                        assignedActiveList,
                         standbyTasks,
                         partitionsByHostState,
                         0)
@@ -697,13 +703,34 @@ private static Map versionProbingAssignment(final Map activeTasks,
+                                                                 final List activePartitionsList,
+                                                                 final List assignedActiveList,
+                                                                 final Map> partitionsForTask) {
+        final List assignedPartitions = new ArrayList<>();
+
+        // Build up list of all assigned partition-task pairs
+        for (final TaskId taskId : activeTasks) {
+            for (final TopicPartition partition : partitionsForTask.get(taskId)) {
+                assignedPartitions.add(new AssignedPartition(taskId, partition));
+            }
+        }
+
+        // Add one copy of a task for each corresponding partition, so the receiver can determine the task <-> tp mapping
+        Collections.sort(assignedPartitions);
+        for (final AssignedPartition partition : assignedPartitions) {
+            assignedActiveList.add(partition.taskId);
+            activePartitionsList.add(partition.partition);
+        }
+    }
+
     // visible for testing
     static List> interleaveTasksByGroupId(final Collection taskIds, final int numberThreads) {
         final LinkedList sortedTasks = new LinkedList<>(taskIds);
@@ -724,17 +751,68 @@ static List> interleaveTasksByGroupId(final Collection task
         return taskIdsForConsumerAssignment;
     }
 
-    private void upgradeSubscriptionVersionIfNeeded(final int leaderSupportedVersion) {
-        if (leaderSupportedVersion > usedSubscriptionMetadataVersion) {
-            log.info("Sent a version {} subscription and group leader's latest supported version is {}. " +
-                         "Upgrading subscription metadata version to {} for next rebalance.",
-                     usedSubscriptionMetadataVersion,
-                     leaderSupportedVersion,
-                     leaderSupportedVersion);
-            usedSubscriptionMetadataVersion = leaderSupportedVersion;
+    private void validateMetadataVersions(final int receivedAssignmentMetadataVersion,
+                                          final int latestCommonlySupportedVersion) {
+
+        if (receivedAssignmentMetadataVersion > usedSubscriptionMetadataVersion) {
+            log.error("Leader sent back an assignment with version {} which was greater than our used version {}",
+                receivedAssignmentMetadataVersion, usedSubscriptionMetadataVersion);
+            throw new TaskAssignmentException(
+                "Sent a version " + usedSubscriptionMetadataVersion
+                    + " subscription but got an assignment with higher version "
+                    + receivedAssignmentMetadataVersion + "."
+            );
+        }
+
+        if (latestCommonlySupportedVersion > LATEST_SUPPORTED_VERSION) {
+            log.error("Leader sent back assignment with commonly supported version {} that is greater than our "
+                + "actual latest supported version {}", latestCommonlySupportedVersion, LATEST_SUPPORTED_VERSION);
+            throw new TaskAssignmentException("Can't upgrade to metadata version greater than we support");
+        }
+
+        if (receivedAssignmentMetadataVersion < EARLIEST_PROBEABLE_VERSION) {
+            log.error("Leader sent back assignment with version {} which is less than the earliest probeable version {}. " +
+                "To do a rolling upgrade from a version earlier than {} you must set the {} config",
+                receivedAssignmentMetadataVersion, EARLIEST_PROBEABLE_VERSION, EARLIEST_PROBEABLE_VERSION,
+                StreamsConfig.UPGRADE_FROM_CONFIG);
+            throw new IllegalStateException("Can't use version probing with versions less than " + EARLIEST_PROBEABLE_VERSION);
         }
     }
 
+    // Returns true if subscription version was changed, indicating version probing and need to rebalance again
+    protected boolean maybeUpdateSubscriptionVersion(final int receivedAssignmentMetadataVersion,
+                                                     final int latestCommonlySupportedVersion) {
+        // If the latest commonly supported version is now greater than our used version, this indicates we have just
+        // completed the rolling upgrade and can now update our subscription version for the final rebalance
+        if (latestCommonlySupportedVersion > usedSubscriptionMetadataVersion) {
+            log.info(
+                "Sent a version {} subscription and group's latest commonly supported version is {} (successful " +
+                    "version probing and end of rolling upgrade). Upgrading subscription metadata version to " +
+                    "{} for next rebalance.",
+                usedSubscriptionMetadataVersion,
+                latestCommonlySupportedVersion,
+                latestCommonlySupportedVersion
+            );
+            usedSubscriptionMetadataVersion = latestCommonlySupportedVersion;
+            return true;
+        }
+
+        // If we received a lower version than we sent, someone else in the group still hasn't upgraded. We
+        // should downgrade our subscription until everyone is on the latest version
+        if (receivedAssignmentMetadataVersion < usedSubscriptionMetadataVersion) {
+            log.info(
+                "Sent a version {} subscription and got version {} assignment back (successful version probing). " +
+                    "Downgrade subscription metadata to commonly supported version and trigger new rebalance.",
+                usedSubscriptionMetadataVersion,
+                receivedAssignmentMetadataVersion
+            );
+            usedSubscriptionMetadataVersion = latestCommonlySupportedVersion;
+            return true;
+        }
+
+        return false;
+    }
+
     /**
      * @throws TaskAssignmentException if there is no task id for one of the partitions specified
      */
@@ -746,43 +824,27 @@ public void onAssignment(final Assignment assignment, final ConsumerGroupMetadat
         final AssignmentInfo info = AssignmentInfo.decode(assignment.userData());
         if (info.errCode() != AssignorError.NONE.code()) {
             // set flag to shutdown streams app
-            assignmentErrorCode.set(info.errCode());
+            setAssignmentErrorCode(info.errCode());
             return;
         }
+        /*
+         * latestCommonlySupportedVersion belongs to [usedSubscriptionMetadataVersion, LATEST_SUPPORTED_VERSION]
+         * receivedAssignmentMetadataVersion belongs to [EARLIEST_PROBEABLE_VERSION, usedSubscriptionMetadataVersion]
+         *
+         * usedSubscriptionMetadataVersion will be downgraded to receivedAssignmentMetadataVersion during a rolling
+         * bounce upgrade with version probing.
+         *
+         * usedSubscriptionMetadataVersion will be upgraded to latestCommonlySupportedVersion when all members have
+         * been bounced and it is safe to use the latest version.
+         */
         final int receivedAssignmentMetadataVersion = info.version();
-        final int leaderSupportedVersion = info.latestSupportedVersion();
+        final int latestCommonlySupportedVersion = info.commonlySupportedVersion();
 
-        if (receivedAssignmentMetadataVersion > usedSubscriptionMetadataVersion) {
-            throw new IllegalStateException(
-                "Sent a version " + usedSubscriptionMetadataVersion
-                    + " subscription but got an assignment with higher version "
-                    + receivedAssignmentMetadataVersion + "."
-            );
-        }
-
-        if (receivedAssignmentMetadataVersion < usedSubscriptionMetadataVersion
-            && receivedAssignmentMetadataVersion >= EARLIEST_PROBEABLE_VERSION) {
-
-            if (receivedAssignmentMetadataVersion == leaderSupportedVersion) {
-                log.info(
-                    "Sent a version {} subscription and got version {} assignment back (successful version probing). " +
-                        "Downgrading subscription metadata to received version and trigger new rebalance.",
-                    usedSubscriptionMetadataVersion,
-                    receivedAssignmentMetadataVersion
-                );
-                usedSubscriptionMetadataVersion = receivedAssignmentMetadataVersion;
-            } else {
-                log.info(
-                    "Sent a version {} subscription and got version {} assignment back (successful version probing). " +
-                        "Setting subscription metadata to leaders supported version {} and trigger new rebalance.",
-                    usedSubscriptionMetadataVersion,
-                    receivedAssignmentMetadataVersion,
-                    leaderSupportedVersion
-                );
-                usedSubscriptionMetadataVersion = leaderSupportedVersion;
-            }
+        validateMetadataVersions(receivedAssignmentMetadataVersion, latestCommonlySupportedVersion);
 
-            assignmentErrorCode.set(AssignorError.VERSION_PROBING.code());
+        // Check if this was a version probing rebalance and check the error code to trigger another rebalance if so
+        if (maybeUpdateSubscriptionVersion(receivedAssignmentMetadataVersion, latestCommonlySupportedVersion)) {
+            setAssignmentErrorCode(AssignorError.VERSION_PROBING.code());
             return;
         }
 
@@ -800,13 +862,9 @@ public void onAssignment(final Assignment assignment, final ConsumerGroupMetadat
                 partitionsByHost = Collections.emptyMap();
                 break;
             case VERSION_TWO:
-                processVersionTwoAssignment(logPrefix, info, partitions, activeTasks, topicToPartitionInfo, partitionsToTaskId);
-                partitionsByHost = info.partitionsByHost();
-                break;
             case VERSION_THREE:
             case VERSION_FOUR:
             case VERSION_FIVE:
-                upgradeSubscriptionVersionIfNeeded(leaderSupportedVersion);
                 processVersionTwoAssignment(logPrefix, info, partitions, activeTasks, topicToPartitionInfo, partitionsToTaskId);
                 partitionsByHost = info.partitionsByHost();
                 break;
@@ -915,6 +973,10 @@ private void ensureCopartitioning(final Collection> copartitionGroup
         }
     }
 
+    protected void setAssignmentErrorCode(final Integer errorCode) {
+        assignmentErrorCode.set(errorCode);
+    }
+
 
     // following functions are for test only
     void setInternalTopicManager(final InternalTopicManager internalTopicManager) {
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsRebalanceListener.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsRebalanceListener.java
index 17563fd460e48..3adac44036ac7 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsRebalanceListener.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsRebalanceListener.java
@@ -72,9 +72,11 @@ public void onPartitionsAssigned(final Collection assignedPartit
                 );
             } else if (streamThread.getAssignmentErrorCode() != AssignorError.NONE.code()) {
                 log.debug(
-                    "Encountered assignment error during partition assignment: {}. Skipping task initialization",
+                    "Encountered assignment error during partition assignment: {}. Skipping task initialization and "
+                        + "pausing any partitions we may have been assigned.",
                     streamThread.getAssignmentErrorCode()
                 );
+                taskManager.pausePartitions();
             } else {
                 // Close non-reassigned tasks before initializing new ones as we may have suspended active
                 // tasks that become standbys or vice versa
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
index 0a7acf3336c84..cd90fad80b522 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
@@ -129,8 +129,7 @@ void createTasks(final Collection assignment) {
         }
 
         // Pause all the new partitions until the underlying state store is ready for all the active tasks.
-        log.trace("Pausing partitions: {}", assignment);
-        consumer.pause(assignment);
+        pausePartitions();
     }
 
     private void resumeSuspended(final Collection assignment) {
@@ -366,6 +365,11 @@ InternalTopologyBuilder builder() {
         return taskCreator.builder();
     }
 
+    void pausePartitions() {
+        log.trace("Pausing partitions: {}", consumer.assignment());
+        consumer.pause(consumer.assignment());
+    }
+
     /**
      * @throws IllegalStateException If store gets registered after initialized is already finished
      * @throws StreamsException if the store's change log does not contain the partition
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfo.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfo.java
index 7d812309ddcf4..dcaaa9a7fec39 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfo.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfo.java
@@ -44,18 +44,21 @@ public class AssignmentInfo {
     private static final Logger log = LoggerFactory.getLogger(AssignmentInfo.class);
 
     private final int usedVersion;
-    private final int latestSupportedVersion;
+    private final int commonlySupportedVersion;
     private int errCode;
     private List activeTasks;
     private Map> standbyTasks;
     private Map> partitionsByHost;
 
-    // used for decoding; don't apply version checks
-    private AssignmentInfo(final int version,
-                           final int latestSupportedVersion) {
-        this.usedVersion = version;
-        this.latestSupportedVersion = latestSupportedVersion;
-        this.errCode = 0;
+    // used for decoding and "future consumer" assignments during version probing
+    public AssignmentInfo(final int version,
+                          final int commonlySupportedVersion) {
+        this(version,
+            commonlySupportedVersion,
+            Collections.emptyList(),
+            Collections.emptyMap(),
+            Collections.emptyMap(),
+            0);
     }
 
     // for testing only
@@ -65,40 +68,31 @@ public AssignmentInfo(final List activeTasks,
         this(LATEST_SUPPORTED_VERSION, activeTasks, standbyTasks, partitionsByHost, 0);
     }
 
-    // creates an empty assignment
-    public AssignmentInfo() {
-        this(LATEST_SUPPORTED_VERSION,
-            Collections.emptyList(),
-            Collections.emptyMap(),
-            Collections.emptyMap(),
-            0);
-    }
-
     public AssignmentInfo(final int version,
                           final List activeTasks,
                           final Map> standbyTasks,
                           final Map> partitionsByHost,
                           final int errCode) {
         this(version, LATEST_SUPPORTED_VERSION, activeTasks, standbyTasks, partitionsByHost, errCode);
-        if (version < 1 || version > LATEST_SUPPORTED_VERSION) {
-            throw new IllegalArgumentException("version must be between 1 and " + LATEST_SUPPORTED_VERSION
-                + "; was: " + version);
-        }
     }
 
-    // for testing only; don't apply version checks
-    AssignmentInfo(final int version,
-                   final int latestSupportedVersion,
-                   final List activeTasks,
-                   final Map> standbyTasks,
-                   final Map> partitionsByHost,
-                   final int errCode) {
+    public AssignmentInfo(final int version,
+                          final int commonlySupportedVersion,
+                          final List activeTasks,
+                          final Map> standbyTasks,
+                          final Map> partitionsByHost,
+                          final int errCode) {
         this.usedVersion = version;
-        this.latestSupportedVersion = latestSupportedVersion;
+        this.commonlySupportedVersion = commonlySupportedVersion;
         this.activeTasks = activeTasks;
         this.standbyTasks = standbyTasks;
         this.partitionsByHost = partitionsByHost;
         this.errCode = errCode;
+
+        if (version < 1 || version > LATEST_SUPPORTED_VERSION) {
+            throw new IllegalArgumentException("version must be between 1 and " + LATEST_SUPPORTED_VERSION
+                + "; was: " + version);
+        }
     }
 
     public int version() {
@@ -109,8 +103,8 @@ public int errCode() {
         return errCode;
     }
 
-    public int latestSupportedVersion() {
-        return latestSupportedVersion;
+    public int commonlySupportedVersion() {
+        return commonlySupportedVersion;
     }
 
     public List activeTasks() {
@@ -422,7 +416,7 @@ private static void decodeVersionFiveData(final AssignmentInfo assignmentInfo,
 
     @Override
     public int hashCode() {
-        return usedVersion ^ latestSupportedVersion ^ activeTasks.hashCode() ^ standbyTasks.hashCode()
+        return usedVersion ^ commonlySupportedVersion ^ activeTasks.hashCode() ^ standbyTasks.hashCode()
             ^ partitionsByHost.hashCode() ^ errCode;
     }
 
@@ -431,7 +425,7 @@ public boolean equals(final Object o) {
         if (o instanceof AssignmentInfo) {
             final AssignmentInfo other = (AssignmentInfo) o;
             return usedVersion == other.usedVersion &&
-                    latestSupportedVersion == other.latestSupportedVersion &&
+                    commonlySupportedVersion == other.commonlySupportedVersion &&
                     errCode == other.errCode &&
                     activeTasks.equals(other.activeTasks) &&
                     standbyTasks.equals(other.standbyTasks) &&
@@ -444,7 +438,7 @@ public boolean equals(final Object o) {
     @Override
     public String toString() {
         return "[version=" + usedVersion
-            + ", supported version=" + latestSupportedVersion
+            + ", supported version=" + commonlySupportedVersion
             + ", active tasks=" + activeTasks
             + ", standby tasks=" + standbyTasks
             + ", partitions by host=" + partitionsByHost + "]";
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/ClientState.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/ClientState.java
index 6eff7bdc7ca19..ab213d520014a 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/ClientState.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/ClientState.java
@@ -16,6 +16,8 @@
  */
 package org.apache.kafka.streams.processor.internals.assignment;
 
+import java.util.HashMap;
+import java.util.Map;
 import org.apache.kafka.streams.processor.TaskId;
 
 import java.util.HashSet;
@@ -29,15 +31,25 @@ public class ClientState {
     private final Set prevStandbyTasks;
     private final Set prevAssignedTasks;
 
-    private int capacity;
+    private final Map> prevActiveTasksByConsumer;
+    private final Map> prevStandbyTasksByConsumer;
 
+    private int capacity;
 
     public ClientState() {
         this(0);
     }
 
     ClientState(final int capacity) {
-        this(new HashSet<>(), new HashSet<>(), new HashSet<>(), new HashSet<>(), new HashSet<>(), new HashSet<>(), capacity);
+        this(new HashSet<>(),
+             new HashSet<>(),
+             new HashSet<>(),
+             new HashSet<>(),
+             new HashSet<>(),
+             new HashSet<>(),
+             new HashMap<>(),
+             new HashMap<>(),
+             capacity);
     }
 
     private ClientState(final Set activeTasks,
@@ -46,6 +58,8 @@ private ClientState(final Set activeTasks,
                         final Set prevActiveTasks,
                         final Set prevStandbyTasks,
                         final Set prevAssignedTasks,
+                        final Map> prevActiveTasksByConsumer,
+                        final Map> prevStandbyTasksByConsumer,
                         final int capacity) {
         this.activeTasks = activeTasks;
         this.standbyTasks = standbyTasks;
@@ -53,6 +67,8 @@ private ClientState(final Set activeTasks,
         this.prevActiveTasks = prevActiveTasks;
         this.prevStandbyTasks = prevStandbyTasks;
         this.prevAssignedTasks = prevAssignedTasks;
+        this.prevActiveTasksByConsumer = prevActiveTasksByConsumer;
+        this.prevStandbyTasksByConsumer = prevStandbyTasksByConsumer;
         this.capacity = capacity;
     }
 
@@ -64,6 +80,8 @@ public ClientState copy() {
             new HashSet<>(prevActiveTasks),
             new HashSet<>(prevStandbyTasks),
             new HashSet<>(prevAssignedTasks),
+            new HashMap<>(prevActiveTasksByConsumer),
+            new HashMap<>(prevStandbyTasksByConsumer),
             capacity);
     }
 
@@ -107,14 +125,24 @@ public int activeTaskCount() {
         return activeTasks.size();
     }
 
-    public void addPreviousActiveTasks(final Set prevTasks) {
+    public void addPreviousActiveTasks(final String consumer, final Set prevTasks) {
         prevActiveTasks.addAll(prevTasks);
         prevAssignedTasks.addAll(prevTasks);
+        prevActiveTasksByConsumer.put(consumer, prevTasks);
     }
 
-    public void addPreviousStandbyTasks(final Set standbyTasks) {
+    public void addPreviousStandbyTasks(final String consumer, final Set standbyTasks) {
         prevStandbyTasks.addAll(standbyTasks);
         prevAssignedTasks.addAll(standbyTasks);
+        prevStandbyTasksByConsumer.put(consumer, standbyTasks);
+    }
+
+    public Set prevActiveTasksForConsumer(final String consumer) {
+        return prevActiveTasksByConsumer.get(consumer);
+    }
+
+    public Set prevStandbyTasksForConsumer(final String consumer) {
+        return prevStandbyTasksByConsumer.get(consumer);
     }
 
     @Override
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
index 5f67a9875db5f..e997f2e125c1d 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
@@ -1236,7 +1236,7 @@ public void shouldReturnUnchangedAssignmentForOldInstancesAndEmptyAssignmentForF
             )));
         assertThat(assignment.get("consumer1").partitions(), equalTo(asList(t1p0, t1p1)));
 
-        assertThat(AssignmentInfo.decode(assignment.get("future-consumer").userData()), equalTo(new AssignmentInfo()));
+        assertThat(AssignmentInfo.decode(assignment.get("future-consumer").userData()), equalTo(new AssignmentInfo(LATEST_SUPPORTED_VERSION, LATEST_SUPPORTED_VERSION)));
         assertThat(assignment.get("future-consumer").partitions().size(), equalTo(0));
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java
index 020ee363c05fc..7e1ca7f96a465 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java
@@ -319,6 +319,7 @@ public void shouldNotAddResumedActiveTasks() {
     @Test
     public void shouldPauseActivePartitions() {
         mockSingleActiveTask();
+        expect(consumer.assignment()).andReturn(taskId0Partitions).times(2);
         consumer.pause(taskId0Partitions);
         expectLastCall();
         replay();
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/ClientStateTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/ClientStateTest.java
index 034ae7b91d6c8..dc54c865d2b54 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/ClientStateTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/ClientStateTest.java
@@ -70,7 +70,7 @@ public void shouldAddPreviousActiveTasksToPreviousAssignedAndPreviousActive() {
         final TaskId tid1 = new TaskId(0, 1);
         final TaskId tid2 = new TaskId(0, 2);
 
-        client.addPreviousActiveTasks(Utils.mkSet(tid1, tid2));
+        client.addPreviousActiveTasks("consumer", Utils.mkSet(tid1, tid2));
         assertThat(client.previousActiveTasks(), equalTo(Utils.mkSet(tid1, tid2)));
         assertThat(client.previousAssignedTasks(), equalTo(Utils.mkSet(tid1, tid2)));
     }
@@ -80,7 +80,7 @@ public void shouldAddPreviousStandbyTasksToPreviousAssigned() {
         final TaskId tid1 = new TaskId(0, 1);
         final TaskId tid2 = new TaskId(0, 2);
 
-        client.addPreviousStandbyTasks(Utils.mkSet(tid1, tid2));
+        client.addPreviousStandbyTasks("consumer", Utils.mkSet(tid1, tid2));
         assertThat(client.previousActiveTasks().size(), equalTo(0));
         assertThat(client.previousAssignedTasks(), equalTo(Utils.mkSet(tid1, tid2)));
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/StickyTaskAssignorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/StickyTaskAssignorTest.java
index 17d403f13061a..19d7730787681 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/StickyTaskAssignorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/StickyTaskAssignorTest.java
@@ -207,11 +207,11 @@ public void shouldKeepActiveTaskStickynessWhenMoreClientThanActiveTasks() {
     @Test
     public void shouldAssignTasksToClientWithPreviousStandbyTasks() {
         final ClientState client1 = createClient(p1, 1);
-        client1.addPreviousStandbyTasks(Utils.mkSet(task02));
+        client1.addPreviousStandbyTasks("consumer", Utils.mkSet(task02));
         final ClientState client2 = createClient(p2, 1);
-        client2.addPreviousStandbyTasks(Utils.mkSet(task01));
+        client2.addPreviousStandbyTasks("consumer", Utils.mkSet(task01));
         final ClientState client3 = createClient(p3, 1);
-        client3.addPreviousStandbyTasks(Utils.mkSet(task00));
+        client3.addPreviousStandbyTasks("consumer", Utils.mkSet(task00));
 
         final StickyTaskAssignor taskAssignor = createTaskAssignor(task00, task01, task02);
 
@@ -225,9 +225,9 @@ public void shouldAssignTasksToClientWithPreviousStandbyTasks() {
     @Test
     public void shouldAssignBasedOnCapacityWhenMultipleClientHaveStandbyTasks() {
         final ClientState c1 = createClientWithPreviousActiveTasks(p1, 1, task00);
-        c1.addPreviousStandbyTasks(Utils.mkSet(task01));
+        c1.addPreviousStandbyTasks("consumer", Utils.mkSet(task01));
         final ClientState c2 = createClientWithPreviousActiveTasks(p2, 2, task02);
-        c2.addPreviousStandbyTasks(Utils.mkSet(task01));
+        c2.addPreviousStandbyTasks("consumer", Utils.mkSet(task01));
 
         final StickyTaskAssignor taskAssignor = createTaskAssignor(task00, task01, task02);
 
@@ -455,9 +455,9 @@ public void shouldNotHaveSameAssignmentOnAnyTwoHostsWhenThereArePreviousActiveTa
     @Test
     public void shouldNotHaveSameAssignmentOnAnyTwoHostsWhenThereArePreviousStandbyTasks() {
         final ClientState c1 = createClientWithPreviousActiveTasks(p1, 1, task01, task02);
-        c1.addPreviousStandbyTasks(Utils.mkSet(task03, task00));
+        c1.addPreviousStandbyTasks("consumer", Utils.mkSet(task03, task00));
         final ClientState c2 = createClientWithPreviousActiveTasks(p2, 1, task03, task00);
-        c2.addPreviousStandbyTasks(Utils.mkSet(task01, task02));
+        c2.addPreviousStandbyTasks("consumer", Utils.mkSet(task01, task02));
 
         createClient(p3, 1);
         createClient(p4, 1);
@@ -577,14 +577,14 @@ public void shouldAssignTasksNotPreviouslyActiveToNewClient() {
         final TaskId task23 = new TaskId(2, 3);
 
         final ClientState c1 = createClientWithPreviousActiveTasks(p1, 1, task01, task12, task13);
-        c1.addPreviousStandbyTasks(Utils.mkSet(task00, task11, task20, task21, task23));
+        c1.addPreviousStandbyTasks("consumer", Utils.mkSet(task00, task11, task20, task21, task23));
         final ClientState c2 = createClientWithPreviousActiveTasks(p2, 1, task00, task11, task22);
-        c2.addPreviousStandbyTasks(Utils.mkSet(task01, task10, task02, task20, task03, task12, task21, task13, task23));
+        c2.addPreviousStandbyTasks("consumer", Utils.mkSet(task01, task10, task02, task20, task03, task12, task21, task13, task23));
         final ClientState c3 = createClientWithPreviousActiveTasks(p3, 1, task20, task21, task23);
-        c3.addPreviousStandbyTasks(Utils.mkSet(task02, task12));
+        c3.addPreviousStandbyTasks("consumer", Utils.mkSet(task02, task12));
 
         final ClientState newClient = createClient(p4, 1);
-        newClient.addPreviousStandbyTasks(Utils.mkSet(task00, task10, task01, task02, task11, task20, task03, task12, task21, task13, task22, task23));
+        newClient.addPreviousStandbyTasks("consumer", Utils.mkSet(task00, task10, task01, task02, task11, task20, task03, task12, task21, task13, task22, task23));
 
         final StickyTaskAssignor taskAssignor = createTaskAssignor(task00, task10, task01, task02, task11, task20, task03, task12, task21, task13, task22, task23);
         taskAssignor.assign(0);
@@ -607,15 +607,15 @@ public void shouldAssignTasksNotPreviouslyActiveToMultipleNewClients() {
         final TaskId task23 = new TaskId(2, 3);
 
         final ClientState c1 = createClientWithPreviousActiveTasks(p1, 1, task01, task12, task13);
-        c1.addPreviousStandbyTasks(Utils.mkSet(task00, task11, task20, task21, task23));
+        c1.addPreviousStandbyTasks("c1onsumer", Utils.mkSet(task00, task11, task20, task21, task23));
         final ClientState c2 = createClientWithPreviousActiveTasks(p2, 1, task00, task11, task22);
-        c2.addPreviousStandbyTasks(Utils.mkSet(task01, task10, task02, task20, task03, task12, task21, task13, task23));
+        c2.addPreviousStandbyTasks("consumer", Utils.mkSet(task01, task10, task02, task20, task03, task12, task21, task13, task23));
 
         final ClientState bounce1 = createClient(p3, 1);
-        bounce1.addPreviousStandbyTasks(Utils.mkSet(task20, task21, task23));
+        bounce1.addPreviousStandbyTasks("consumer", Utils.mkSet(task20, task21, task23));
 
         final ClientState bounce2 = createClient(p4, 1);
-        bounce2.addPreviousStandbyTasks(Utils.mkSet(task02, task03, task10));
+        bounce2.addPreviousStandbyTasks("consumer", Utils.mkSet(task02, task03, task10));
 
         final StickyTaskAssignor taskAssignor = createTaskAssignor(task00, task10, task01, task02, task11, task20, task03, task12, task21, task13, task22, task23);
         taskAssignor.assign(0);
@@ -658,7 +658,7 @@ public void shouldAssignTasksToNewClientWithoutFlippingAssignmentBetweenExisting
         final TaskId task06 = new TaskId(0, 6);
         final ClientState c1 = createClientWithPreviousActiveTasks(p1, 1, task00, task01, task02, task06);
         final ClientState c2 = createClient(p2, 1);
-        c2.addPreviousStandbyTasks(Utils.mkSet(task03, task04, task05));
+        c2.addPreviousStandbyTasks("consumer", Utils.mkSet(task03, task04, task05));
         final ClientState newClient = createClient(p3, 1);
 
         final StickyTaskAssignor taskAssignor = createTaskAssignor(task00, task01, task02, task03, task04, task05, task06);
@@ -705,7 +705,7 @@ private ClientState createClient(final Integer processId, final int capacity) {
 
     private ClientState createClientWithPreviousActiveTasks(final Integer processId, final int capacity, final TaskId... taskIds) {
         final ClientState clientState = new ClientState(capacity);
-        clientState.addPreviousActiveTasks(Utils.mkSet(taskIds));
+        clientState.addPreviousActiveTasks("consumer", Utils.mkSet(taskIds));
         clients.put(processId, clientState);
         return clientState;
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
index de5311cef92d3..0e07cac971970 100644
--- a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
@@ -38,6 +38,7 @@
 import org.apache.kafka.streams.processor.internals.StreamsPartitionAssignor;
 import org.apache.kafka.streams.processor.internals.TaskManager;
 import org.apache.kafka.streams.processor.internals.assignment.AssignmentInfo;
+import org.apache.kafka.streams.processor.internals.assignment.AssignorError;
 import org.apache.kafka.streams.processor.internals.assignment.SubscriptionInfo;
 import org.apache.kafka.streams.state.HostInfo;
 
@@ -167,6 +168,11 @@ public void onAssignment(final ConsumerPartitionAssignor.Assignment assignment,
             final AssignmentInfo info = AssignmentInfo.decode(
                 assignment.userData().putInt(0, LATEST_SUPPORTED_VERSION));
 
+            if (super.maybeUpdateSubscriptionVersion(usedVersion, info.commonlySupportedVersion())) {
+                setAssignmentErrorCode(AssignorError.VERSION_PROBING.code());
+                return;
+            }
+
             final List partitions = new ArrayList<>(assignment.partitions());
             partitions.sort(PARTITION_COMPARATOR);
 
@@ -305,6 +311,7 @@ private static class FutureAssignmentInfo extends AssignmentInfo {
         private FutureAssignmentInfo(final boolean bumpUsedVersion,
                                      final boolean bumpSupportedVersion,
                                      final ByteBuffer bytes) {
+            super(LATEST_SUPPORTED_VERSION, LATEST_SUPPORTED_VERSION);
             this.bumpUsedVersion = bumpUsedVersion;
             this.bumpSupportedVersion = bumpSupportedVersion;
             originalUserMetadata = bytes;
diff --git a/tests/kafkatest/tests/streams/streams_upgrade_test.py b/tests/kafkatest/tests/streams/streams_upgrade_test.py
index fb85732818940..45e20ce26c4ac 100644
--- a/tests/kafkatest/tests/streams/streams_upgrade_test.py
+++ b/tests/kafkatest/tests/streams/streams_upgrade_test.py
@@ -355,8 +355,6 @@ def update_leader(self):
             for p in self.processors:
                 found = list(p.node.account.ssh_capture("grep \"Finished assignment for group\" %s" % p.LOG_FILE, allow_fail=True))
                 if len(found) >= self.leader_counter[p] + 1:
-                    if self.leader is not None:
-                        raise Exception("Could not uniquely identify leader")
                     self.leader = p
                     self.leader_counter[p] = self.leader_counter[p] + 1
 
@@ -557,38 +555,25 @@ def do_rolling_bounce(self, processor, counter, current_generation):
                     else:
                         self.leader_counter[self.leader] = self.leader_counter[self.leader] + 1
 
-                    if processor == self.leader:
-                        leader_monitor = log_monitor
-                    elif first_other_processor == self.leader:
-                        leader_monitor = first_other_monitor
-                    elif second_other_processor == self.leader:
-                        leader_monitor = second_other_monitor
-                    else:
-                        raise Exception("Could not identify leader.")
-
                     monitors = {}
                     monitors[processor] = log_monitor
                     monitors[first_other_processor] = first_other_monitor
                     monitors[second_other_processor] = second_other_monitor
 
-                    leader_monitor.wait_until("Received a future (version probing) subscription (version: 6). Sending empty assignment back (with supported version 5).",
-                                              timeout_sec=60,
-                                              err_msg="Could not detect 'version probing' attempt at leader " + str(self.leader.node.account))
-
                     if len(self.old_processors) > 0:
-                        log_monitor.wait_until("Sent a version 6 subscription and got version 5 assignment back (successful version probing). Downgrading subscription metadata to received version and trigger new rebalance.",
+                        log_monitor.wait_until("Sent a version 6 subscription and got version 5 assignment back (successful version probing). Downgrade subscription metadata to commonly supported version and trigger new rebalance.",
                                                timeout_sec=60,
                                                err_msg="Could not detect 'successful version probing' at upgrading node " + str(node.account))
                     else:
-                        log_monitor.wait_until("Sent a version 6 subscription and got version 5 assignment back (successful version probing). Setting subscription metadata to leaders supported version 6 and trigger new rebalance.",
+                        log_monitor.wait_until("Sent a version 6 subscription and got version 5 assignment back (successful version probing). Downgrade subscription metadata to commonly supported version and trigger new rebalance.",
                                                timeout_sec=60,
                                                err_msg="Could not detect 'successful version probing with upgraded leader' at upgrading node " + str(node.account))
-                        first_other_monitor.wait_until("Sent a version 5 subscription and group leader.s latest supported version is 6. Upgrading subscription metadata version to 6 for next rebalance.",
+                        first_other_monitor.wait_until("Sent a version 5 subscription and group.s latest commonly supported version is 6 (successful version probing and end of rolling upgrade). Upgrading subscription metadata version to 6 for next rebalance.",
                                                        timeout_sec=60,
-                                                       err_msg="Never saw output 'Upgrade metadata to version 5' on" + str(first_other_node.account))
-                        second_other_monitor.wait_until("Sent a version 5 subscription and group leader.s latest supported version is 6. Upgrading subscription metadata version to 6 for next rebalance.",
+                                                       err_msg="Never saw output 'Upgrade metadata to version 6' on" + str(first_other_node.account))
+                        second_other_monitor.wait_until("Sent a version 5 subscription and group.s latest commonly supported version is 6 (successful version probing and end of rolling upgrade). Upgrading subscription metadata version to 6 for next rebalance.",
                                                         timeout_sec=60,
-                                                        err_msg="Never saw output 'Upgrade metadata to version 5' on" + str(second_other_node.account))
+                                                        err_msg="Never saw output 'Upgrade metadata to version 6' on" + str(second_other_node.account))
 
                     log_monitor.wait_until("Version probing detected. Triggering new rebalance.",
                                            timeout_sec=60,

From c7efc3613c4a021ddaae314ca8226d04e6dd2080 Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Wed, 2 Oct 2019 14:32:07 -0700
Subject: [PATCH 0674/1071] HOTFIX: don't throw if upgrading from very old
 versions (#7436)

Reviewers: Guozhang Wang 
---
 .../internals/StreamsPartitionAssignor.java   | 69 ++++++++++---------
 1 file changed, 35 insertions(+), 34 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
index 7afc95dda841b..846617143a978 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
@@ -769,45 +769,46 @@ private void validateMetadataVersions(final int receivedAssignmentMetadataVersio
                 + "actual latest supported version {}", latestCommonlySupportedVersion, LATEST_SUPPORTED_VERSION);
             throw new TaskAssignmentException("Can't upgrade to metadata version greater than we support");
         }
-
-        if (receivedAssignmentMetadataVersion < EARLIEST_PROBEABLE_VERSION) {
-            log.error("Leader sent back assignment with version {} which is less than the earliest probeable version {}. " +
-                "To do a rolling upgrade from a version earlier than {} you must set the {} config",
-                receivedAssignmentMetadataVersion, EARLIEST_PROBEABLE_VERSION, EARLIEST_PROBEABLE_VERSION,
-                StreamsConfig.UPGRADE_FROM_CONFIG);
-            throw new IllegalStateException("Can't use version probing with versions less than " + EARLIEST_PROBEABLE_VERSION);
-        }
     }
 
     // Returns true if subscription version was changed, indicating version probing and need to rebalance again
     protected boolean maybeUpdateSubscriptionVersion(final int receivedAssignmentMetadataVersion,
                                                      final int latestCommonlySupportedVersion) {
-        // If the latest commonly supported version is now greater than our used version, this indicates we have just
-        // completed the rolling upgrade and can now update our subscription version for the final rebalance
-        if (latestCommonlySupportedVersion > usedSubscriptionMetadataVersion) {
-            log.info(
-                "Sent a version {} subscription and group's latest commonly supported version is {} (successful " +
-                    "version probing and end of rolling upgrade). Upgrading subscription metadata version to " +
-                    "{} for next rebalance.",
-                usedSubscriptionMetadataVersion,
-                latestCommonlySupportedVersion,
-                latestCommonlySupportedVersion
-            );
-            usedSubscriptionMetadataVersion = latestCommonlySupportedVersion;
-            return true;
-        }
-
-        // If we received a lower version than we sent, someone else in the group still hasn't upgraded. We
-        // should downgrade our subscription until everyone is on the latest version
-        if (receivedAssignmentMetadataVersion < usedSubscriptionMetadataVersion) {
-            log.info(
-                "Sent a version {} subscription and got version {} assignment back (successful version probing). " +
-                    "Downgrade subscription metadata to commonly supported version and trigger new rebalance.",
-                usedSubscriptionMetadataVersion,
-                receivedAssignmentMetadataVersion
-            );
-            usedSubscriptionMetadataVersion = latestCommonlySupportedVersion;
-            return true;
+        if (receivedAssignmentMetadataVersion >= EARLIEST_PROBEABLE_VERSION) {
+
+            // If the latest commonly supported version is now greater than our used version, this indicates we have just
+            // completed the rolling upgrade and can now update our subscription version for the final rebalance
+            if (latestCommonlySupportedVersion > usedSubscriptionMetadataVersion) {
+                log.info(
+                    "Sent a version {} subscription and group's latest commonly supported version is {} (successful "
+                        +
+                        "version probing and end of rolling upgrade). Upgrading subscription metadata version to " +
+                        "{} for next rebalance.",
+                    usedSubscriptionMetadataVersion,
+                    latestCommonlySupportedVersion,
+                    latestCommonlySupportedVersion
+                );
+                usedSubscriptionMetadataVersion = latestCommonlySupportedVersion;
+                return true;
+            }
+
+            // If we received a lower version than we sent, someone else in the group still hasn't upgraded. We
+            // should downgrade our subscription until everyone is on the latest version
+            if (receivedAssignmentMetadataVersion < usedSubscriptionMetadataVersion) {
+                log.info(
+                    "Sent a version {} subscription and got version {} assignment back (successful version probing). "
+                        +
+                        "Downgrade subscription metadata to commonly supported version and trigger new rebalance.",
+                    usedSubscriptionMetadataVersion,
+                    receivedAssignmentMetadataVersion
+                );
+                usedSubscriptionMetadataVersion = latestCommonlySupportedVersion;
+                return true;
+            }
+        } else {
+            log.debug("Received an assignment version {} that is less than the earliest version that allows version " +
+                "probing {}. If this is not during a rolling upgrade from version 2.0 or below, this is an error.",
+                receivedAssignmentMetadataVersion, EARLIEST_PROBEABLE_VERSION);
         }
 
         return false;

From 1c831c22e13fcc59e7e1a9e6c28fd38048eb3f62 Mon Sep 17 00:00:00 2001
From: Arjun Satish 
Date: Wed, 2 Oct 2019 15:00:37 -0700
Subject: [PATCH 0675/1071] KAFKA-7772: Dynamically Adjust Log Levels in
 Connect (#7403)

Implemented KIP-495 to expose a new `admin/loggers` endpoint for the Connect REST API that lists the current log levels and allows the caller to change log levels.

Author: Arjun Satish 
Reviewer: Randall Hauch 
---
 checkstyle/import-control.xml                 |   3 +
 .../apache/kafka/connect/runtime/Connect.java |   4 +
 .../kafka/connect/runtime/WorkerConfig.java   |  38 ++++
 .../connect/runtime/rest/RestServer.java      | 127 ++++++++++-
 .../rest/resources/LoggingResource.java       | 206 ++++++++++++++++++
 .../connect/runtime/rest/util/SSLUtils.java   |  14 +-
 .../connect/runtime/WorkerConfigTest.java     |  74 +++++++
 .../connect/runtime/rest/RestServerTest.java  | 140 ++++++++++++
 .../rest/resources/LoggingResourceTest.java   | 195 +++++++++++++++++
 .../util/clusters/EmbeddedConnectCluster.java |  10 +
 .../connect/util/clusters/WorkerHandle.java   |   9 +
 11 files changed, 808 insertions(+), 12 deletions(-)
 create mode 100644 connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/LoggingResource.java
 create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerConfigTest.java
 create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/LoggingResourceTest.java

diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml
index 066fc33239a94..7fd418aab4636 100644
--- a/checkstyle/import-control.xml
+++ b/checkstyle/import-control.xml
@@ -354,6 +354,9 @@
         
         
         
+        
+          
+        
       
 
       
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Connect.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Connect.java
index 4a0bcabea8982..773869a5ba21a 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Connect.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Connect.java
@@ -88,6 +88,10 @@ public URI restUrl() {
         return rest.serverUrl();
     }
 
+    public URI adminUrl() {
+        return rest.adminUrl();
+    }
+
     private class ShutdownHook extends Thread {
         @Override
         public void run() {
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java
index 8657510da2060..17d1d5fe0c89d 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java
@@ -22,6 +22,7 @@
 import org.apache.kafka.common.config.ConfigDef;
 import org.apache.kafka.common.config.ConfigDef.Importance;
 import org.apache.kafka.common.config.ConfigDef.Type;
+import org.apache.kafka.common.config.ConfigException;
 import org.apache.kafka.common.config.internals.BrokerSecurityConfigs;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.connect.json.JsonConverter;
@@ -186,6 +187,14 @@ public class WorkerConfig extends AbstractConfig {
         + "The default value of the Access-Control-Allow-Methods header allows cross origin requests for GET, POST and HEAD.";
     protected static final String ACCESS_CONTROL_ALLOW_METHODS_DEFAULT = "";
 
+    public static final String ADMIN_LISTENERS_CONFIG = "admin.listeners";
+    protected static final String ADMIN_LISTENERS_DOC = "List of comma-separated URIs the Admin REST API will listen on." +
+            " The supported protocols are HTTP and HTTPS." +
+            " An empty or blank string will disable this feature." +
+            " The default behavior is to use the regular listener (specified by the 'listeners' property).";
+    protected static final List ADMIN_LISTENERS_DEFAULT = null;
+    public static final String ADMIN_LISTENERS_HTTPS_CONFIGS_PREFIX = "admin.listeners.https.";
+
     public static final String PLUGIN_PATH_CONFIG = "plugin.path";
     protected static final String PLUGIN_PATH_DOC = "List of paths separated by commas (,) that "
             + "contain plugins (connectors, converters, transformations). The list should consist"
@@ -298,6 +307,8 @@ protected static ConfigDef baseConfigDef() {
                         Importance.LOW, CONFIG_PROVIDERS_DOC)
                 .define(REST_EXTENSION_CLASSES_CONFIG, Type.LIST, "",
                         Importance.LOW, REST_EXTENSION_CLASSES_DOC)
+                .define(ADMIN_LISTENERS_CONFIG, Type.LIST, null,
+                        new AdminListenersValidator(), Importance.LOW, ADMIN_LISTENERS_DOC)
                 .define(CONNECTOR_CLIENT_POLICY_CLASS_CONFIG, Type.STRING, CONNECTOR_CLIENT_POLICY_CLASS_DEFAULT,
                         Importance.MEDIUM, CONNECTOR_CLIENT_POLICY_CLASS_DOC);
     }
@@ -375,4 +386,31 @@ public WorkerConfig(ConfigDef definition, Map props) {
         logInternalConverterDeprecationWarnings(props);
     }
 
+    private static class AdminListenersValidator implements ConfigDef.Validator {
+        @Override
+        public void ensureValid(String name, Object value) {
+            if (value == null) {
+                return;
+            }
+
+            if (!(value instanceof List)) {
+                throw new ConfigException("Invalid value type (list expected).");
+            }
+
+            List items = (List) value;
+            if (items.isEmpty()) {
+                return;
+            }
+
+            for (Object item: items) {
+                if (!(item instanceof String)) {
+                    throw new ConfigException("Invalid type for admin listener (expected String).");
+                }
+                if (((String) item).trim().isEmpty()) {
+                    throw new ConfigException("Empty listener found when parsing list.");
+                }
+            }
+        }
+    }
+
 }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestServer.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestServer.java
index bab20f5862826..9ad24468ee564 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestServer.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestServer.java
@@ -29,6 +29,7 @@
 import org.apache.kafka.connect.runtime.rest.errors.ConnectExceptionMapper;
 import org.apache.kafka.connect.runtime.rest.resources.ConnectorPluginsResource;
 import org.apache.kafka.connect.runtime.rest.resources.ConnectorsResource;
+import org.apache.kafka.connect.runtime.rest.resources.LoggingResource;
 import org.apache.kafka.connect.runtime.rest.resources.RootResource;
 import org.apache.kafka.connect.runtime.rest.util.SSLUtils;
 import org.eclipse.jetty.server.Connector;
@@ -64,12 +65,17 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import static org.apache.kafka.connect.runtime.WorkerConfig.ADMIN_LISTENERS_HTTPS_CONFIGS_PREFIX;
+
 /**
  * Embedded server for the REST API that provides the control plane for Kafka Connect workers.
  */
 public class RestServer {
     private static final Logger log = LoggerFactory.getLogger(RestServer.class);
 
+    // Used to distinguish between Admin connectors and regular REST API connectors when binding admin handlers
+    private static final String ADMIN_SERVER_CONNECTOR_NAME = "Admin";
+
     private static final Pattern LISTENER_PATTERN = Pattern.compile("^(.*)://\\[?([0-9a-zA-Z\\-%._:]*)\\]?:(-?[0-9]+)");
     private static final long GRACEFUL_SHUTDOWN_TIMEOUT_MS = 60 * 1000;
 
@@ -89,11 +95,12 @@ public RestServer(WorkerConfig config) {
         this.config = config;
 
         List listeners = parseListeners();
+        List adminListeners = config.getList(WorkerConfig.ADMIN_LISTENERS_CONFIG);
 
         jettyServer = new Server();
         handlers = new ContextHandlerCollection();
 
-        createConnectors(listeners);
+        createConnectors(listeners, adminListeners);
     }
 
     @SuppressWarnings("deprecation")
@@ -114,24 +121,39 @@ List parseListeners() {
     /**
      * Adds Jetty connector for each configured listener
      */
-    public void createConnectors(List listeners) {
+    public void createConnectors(List listeners, List adminListeners) {
         List connectors = new ArrayList<>();
 
         for (String listener : listeners) {
             if (!listener.isEmpty()) {
                 Connector connector = createConnector(listener);
                 connectors.add(connector);
-                log.info("Added connector for " + listener);
+                log.info("Added connector for {}", listener);
             }
         }
 
         jettyServer.setConnectors(connectors.toArray(new Connector[connectors.size()]));
+
+        if (adminListeners != null && !adminListeners.isEmpty()) {
+            for (String adminListener : adminListeners) {
+                Connector conn = createConnector(adminListener, true);
+                jettyServer.addConnector(conn);
+                log.info("Added admin connector for {}", adminListener);
+            }
+        }
     }
 
     /**
-     * Creates Jetty connector according to configuration
+     * Creates regular (non-admin) Jetty connector according to configuration
      */
     public Connector createConnector(String listener) {
+        return createConnector(listener, false);
+    }
+
+    /**
+     * Creates Jetty connector according to configuration
+     */
+    public Connector createConnector(String listener, boolean isAdmin) {
         Matcher listenerMatcher = LISTENER_PATTERN.matcher(listener);
 
         if (!listenerMatcher.matches())
@@ -148,12 +170,25 @@ public Connector createConnector(String listener) {
         ServerConnector connector;
 
         if (PROTOCOL_HTTPS.equals(protocol)) {
-            SslContextFactory ssl = SSLUtils.createServerSideSslContextFactory(config);
+            SslContextFactory ssl;
+            if (isAdmin) {
+                ssl = SSLUtils.createServerSideSslContextFactory(config, ADMIN_LISTENERS_HTTPS_CONFIGS_PREFIX);
+            } else {
+                ssl = SSLUtils.createServerSideSslContextFactory(config);
+            }
             connector = new ServerConnector(jettyServer, ssl);
-            connector.setName(String.format("%s_%s%d", PROTOCOL_HTTPS, hostname, port));
+            if (!isAdmin) {
+                connector.setName(String.format("%s_%s%d", PROTOCOL_HTTPS, hostname, port));
+            }
         } else {
             connector = new ServerConnector(jettyServer);
-            connector.setName(String.format("%s_%s%d", PROTOCOL_HTTP, hostname, port));
+            if (!isAdmin) {
+                connector.setName(String.format("%s_%s%d", PROTOCOL_HTTP, hostname, port));
+            }
+        }
+
+        if (isAdmin) {
+            connector.setName(ADMIN_SERVER_CONNECTOR_NAME);
         }
 
         if (!hostname.isEmpty())
@@ -181,6 +216,7 @@ public void initializeServer() {
         }
 
         log.info("REST server listening at " + jettyServer.getURI() + ", advertising URL " + advertisedUrl());
+        log.info("REST admin endpoints at " + adminUrl());
     }
 
     public void initializeResources(Herder herder) {
@@ -198,12 +234,44 @@ public void initializeResources(Herder herder) {
 
         registerRestExtensions(herder, resourceConfig);
 
+        List adminListeners = config.getList(WorkerConfig.ADMIN_LISTENERS_CONFIG);
+        ResourceConfig adminResourceConfig;
+        if (adminListeners == null) {
+            log.info("Adding admin resources to main listener");
+            adminResourceConfig = resourceConfig;
+            adminResourceConfig.register(new LoggingResource());
+        } else if (adminListeners.size() > 0) {
+            // TODO: we need to check if these listeners are same as 'listeners'
+            // TODO: the following code assumes that they are different
+            log.info("Adding admin resources to admin listener");
+            adminResourceConfig = new ResourceConfig();
+            adminResourceConfig.register(new JacksonJsonProvider());
+            adminResourceConfig.register(new LoggingResource());
+            adminResourceConfig.register(ConnectExceptionMapper.class);
+        } else {
+            log.info("Skipping adding admin resources");
+            // set up adminResource but add no handlers to it
+            adminResourceConfig = resourceConfig;
+        }
+
         ServletContainer servletContainer = new ServletContainer(resourceConfig);
         ServletHolder servletHolder = new ServletHolder(servletContainer);
+        List contextHandlers = new ArrayList<>();
 
         ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
         context.setContextPath("/");
         context.addServlet(servletHolder, "/*");
+        contextHandlers.add(context);
+
+        ServletContextHandler adminContext = null;
+        if (adminResourceConfig != resourceConfig) {
+            adminContext = new ServletContextHandler(ServletContextHandler.SESSIONS);
+            ServletHolder adminServletHolder = new ServletHolder(new ServletContainer(adminResourceConfig));
+            adminContext.setContextPath("/");
+            adminContext.addServlet(adminServletHolder, "/*");
+            adminContext.setVirtualHosts(new String[]{"@" + ADMIN_SERVER_CONNECTOR_NAME});
+            contextHandlers.add(adminContext);
+        }
 
         String allowedOrigins = config.getString(WorkerConfig.ACCESS_CONTROL_ALLOW_ORIGIN_CONFIG);
         if (allowedOrigins != null && !allowedOrigins.trim().isEmpty()) {
@@ -223,13 +291,25 @@ public void initializeResources(Herder herder) {
         CustomRequestLog requestLog = new CustomRequestLog(slf4jRequestLogWriter, CustomRequestLog.EXTENDED_NCSA_FORMAT + " %msT");
         requestLogHandler.setRequestLog(requestLog);
 
-        handlers.setHandlers(new Handler[]{context, new DefaultHandler(), requestLogHandler});
+        contextHandlers.add(new DefaultHandler());
+        contextHandlers.add(requestLogHandler);
+
+        handlers.setHandlers(contextHandlers.toArray(new Handler[]{}));
         try {
             context.start();
         } catch (Exception e) {
             throw new ConnectException("Unable to initialize REST resources", e);
         }
 
+        if (adminResourceConfig != resourceConfig) {
+            try {
+                log.debug("Starting admin context");
+                adminContext.start();
+            } catch (Exception e) {
+                throw new ConnectException("Unable to initialize Admin REST resources", e);
+            }
+        }
+
         log.info("REST resources initialized; server is started and ready to handle requests");
     }
 
@@ -287,6 +367,34 @@ else if (serverConnector != null && serverConnector.getPort() > 0)
         return builder.build();
     }
 
+    /**
+     * @return the admin url for this worker. can be null if admin endpoints are disabled.
+     */
+    public URI adminUrl() {
+        ServerConnector adminConnector = null;
+        for (Connector connector : jettyServer.getConnectors()) {
+            if (ADMIN_SERVER_CONNECTOR_NAME.equals(connector.getName()))
+                adminConnector = (ServerConnector) connector;
+        }
+
+        if (adminConnector == null) {
+            List adminListeners = config.getList(WorkerConfig.ADMIN_LISTENERS_CONFIG);
+            if (adminListeners == null) {
+                return advertisedUrl();
+            } else if (adminListeners.isEmpty()) {
+                return null;
+            } else {
+                log.error("No admin connector found for listeners {}", adminListeners);
+                return null;
+            }
+        }
+
+        UriBuilder builder = UriBuilder.fromUri(jettyServer.getURI());
+        builder.port(adminConnector.getLocalPort());
+
+        return builder.build();
+    }
+
     String determineAdvertisedProtocol() {
         String advertisedSecurityProtocol = config.getString(WorkerConfig.REST_ADVERTISED_LISTENER_CONFIG);
         if (advertisedSecurityProtocol == null) {
@@ -310,7 +418,8 @@ else if (listeners.contains(String.format("%s://", PROTOCOL_HTTPS)))
 
     ServerConnector findConnector(String protocol) {
         for (Connector connector : jettyServer.getConnectors()) {
-            if (connector.getName().startsWith(protocol))
+            String connectorName = connector.getName();
+            if (connectorName.startsWith(protocol) && !ADMIN_SERVER_CONNECTOR_NAME.equals(connectorName))
                 return (ServerConnector) connector;
         }
 
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/LoggingResource.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/LoggingResource.java
new file mode 100644
index 0000000000000..05796600a936b
--- /dev/null
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/LoggingResource.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.runtime.rest.resources;
+
+import org.apache.kafka.connect.errors.NotFoundException;
+import org.apache.kafka.connect.runtime.rest.errors.BadRequestException;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.TreeMap;
+
+/**
+ * A set of endpoints to adjust the log levels of runtime loggers.
+ */
+@Path("/admin/loggers")
+@Produces(MediaType.APPLICATION_JSON)
+@Consumes(MediaType.APPLICATION_JSON)
+public class LoggingResource {
+
+    /**
+     * Log4j uses "root" (case insensitive) as name of the root logger.
+     */
+    private static final String ROOT_LOGGER_NAME = "root";
+
+    /**
+     * List the current loggers that have their levels explicitly set and their log levels.
+     *
+     * @return a list of current loggers and their levels.
+     */
+    @GET
+    @Path("/")
+    public Response listLoggers() {
+        Map> loggers = new TreeMap<>();
+        Enumeration enumeration = currentLoggers();
+        Collections.list(enumeration)
+                .stream()
+                .filter(logger -> logger.getLevel() != null)
+                .forEach(logger -> loggers.put(logger.getName(), levelToMap(logger)));
+
+        Logger root = rootLogger();
+        if (root.getLevel() != null) {
+            loggers.put(ROOT_LOGGER_NAME, levelToMap(root));
+        }
+
+        return Response.ok(loggers).build();
+    }
+
+    /**
+     * Get the log level of a named logger.
+     *
+     * @param namedLogger name of a logger
+     * @return level of the logger, effective level if the level was not explicitly set.
+     */
+    @GET
+    @Path("/{logger}")
+    public Response getLogger(final @PathParam("logger") String namedLogger) {
+        Objects.requireNonNull(namedLogger, "require non-null name");
+
+        Logger logger = null;
+        if (ROOT_LOGGER_NAME.equalsIgnoreCase(namedLogger)) {
+            logger = rootLogger();
+        } else {
+            Enumeration en = currentLoggers();
+            // search within existing loggers for the given name.
+            // using LogManger.getLogger() will create a logger if it doesn't exist
+            // (potential leak since these don't get cleaned up).
+            while (en.hasMoreElements()) {
+                Logger l = en.nextElement();
+                if (namedLogger.equals(l.getName())) {
+                    logger = l;
+                    break;
+                }
+            }
+        }
+        if (logger == null) {
+            throw new NotFoundException("Logger " + namedLogger + " not found.");
+        } else {
+            return Response.ok(effectiveLevelToMap(logger)).build();
+        }
+    }
+
+
+    /**
+     * Adjust level of a named logger. if name corresponds to an ancestor, then the log level is applied to all child loggers.
+     *
+     * @param namedLogger name of the logger
+     * @param levelMap a map that is expected to contain one key 'level', and a value that is one of the log4j levels:
+     *                 DEBUG, ERROR, FATAL, INFO, TRACE, WARN
+     * @return names of loggers whose levels were modified
+     */
+    @PUT
+    @Path("/{logger}")
+    public Response setLevel(final @PathParam("logger") String namedLogger,
+                             final Map levelMap) {
+        String desiredLevelStr = levelMap.get("level");
+        if (desiredLevelStr == null) {
+            throw new BadRequestException("Desired 'level' parameter was not specified in request.");
+        }
+
+        Level level = Level.toLevel(desiredLevelStr.toUpperCase(Locale.ROOT), null);
+        if (level == null) {
+            throw new NotFoundException("invalid log level '" + desiredLevelStr + "'.");
+        }
+
+        List childLoggers;
+        if (ROOT_LOGGER_NAME.equalsIgnoreCase(namedLogger)) {
+            childLoggers = Collections.list(currentLoggers());
+            childLoggers.add(rootLogger());
+        } else {
+            childLoggers = new ArrayList<>();
+            Logger ancestorLogger = lookupLogger(namedLogger);
+            Enumeration en = currentLoggers();
+            boolean present = false;
+            while (en.hasMoreElements()) {
+                Logger current = (Logger) en.nextElement();
+                if (current.getName().startsWith(namedLogger)) {
+                    childLoggers.add(current);
+                }
+                if (namedLogger.equals(current.getName())) {
+                    present = true;
+                }
+            }
+            if (!present) {
+                childLoggers.add(ancestorLogger);
+            }
+        }
+
+        List modifiedLoggerNames = new ArrayList<>();
+        for (Logger logger: childLoggers) {
+            logger.setLevel(level);
+            modifiedLoggerNames.add(logger.getName());
+        }
+        Collections.sort(modifiedLoggerNames);
+
+        return Response.ok(modifiedLoggerNames).build();
+    }
+
+    protected Logger lookupLogger(String namedLogger) {
+        return LogManager.getLogger(namedLogger);
+    }
+
+    @SuppressWarnings("unchecked")
+    protected Enumeration currentLoggers() {
+        return LogManager.getCurrentLoggers();
+    }
+
+    protected Logger rootLogger() {
+        return LogManager.getRootLogger();
+    }
+
+    /**
+     *
+     * Map representation of a logger's effective log level.
+     *
+     * @param logger a non-null log4j logger
+     * @return a singleton map whose key is level and the value is the string representation of the logger's effective log level.
+     */
+    private static Map effectiveLevelToMap(Logger logger) {
+        Level level = logger.getLevel();
+        if (level == null) {
+            level = logger.getEffectiveLevel();
+        }
+        return Collections.singletonMap("level", String.valueOf(level));
+    }
+
+    /**
+     *
+     * Map representation of a logger's log level.
+     *
+     * @param logger a non-null log4j logger
+     * @return a singleton map whose key is level and the value is the string representation of the logger's log level.
+     */
+    private static Map levelToMap(Logger logger) {
+        return Collections.singletonMap("level", String.valueOf(logger.getLevel()));
+    }
+}
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/util/SSLUtils.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/util/SSLUtils.java
index cfe9d0bc37f5b..6b391d96adaa7 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/util/SSLUtils.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/util/SSLUtils.java
@@ -34,11 +34,12 @@ public class SSLUtils {
 
     private static final Pattern COMMA_WITH_WHITESPACE = Pattern.compile("\\s*,\\s*");
 
+
     /**
-     * Configures SSL/TLS for HTTPS Jetty Server
+     * Configures SSL/TLS for HTTPS Jetty Server using configs with the given prefix
      */
-    public static SslContextFactory createServerSideSslContextFactory(WorkerConfig config) {
-        Map sslConfigValues = config.valuesWithPrefixAllOrNothing("listeners.https.");
+    public static SslContextFactory createServerSideSslContextFactory(WorkerConfig config, String prefix) {
+        Map sslConfigValues = config.valuesWithPrefixAllOrNothing(prefix);
 
         final SslContextFactory.Server ssl = new SslContextFactory.Server();
 
@@ -50,6 +51,13 @@ public static SslContextFactory createServerSideSslContextFactory(WorkerConfig c
         return ssl;
     }
 
+    /**
+     * Configures SSL/TLS for HTTPS Jetty Server
+     */
+    public static SslContextFactory createServerSideSslContextFactory(WorkerConfig config) {
+        return createServerSideSslContextFactory(config, "listeners.https.");
+    }
+
     /**
      * Configures SSL/TLS for HTTPS Jetty Client
      */
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerConfigTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerConfigTest.java
new file mode 100644
index 0000000000000..33416b99e04c5
--- /dev/null
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerConfigTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.runtime;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.common.config.ConfigException;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class WorkerConfigTest {
+
+    @Test
+    public void testAdminListenersConfigAllowedValues() {
+        Map props = baseProps();
+
+        // no value set for "admin.listeners"
+        WorkerConfig config = new WorkerConfig(WorkerConfig.baseConfigDef(), props);
+        assertNull("Default value should be null.", config.getList(WorkerConfig.ADMIN_LISTENERS_CONFIG));
+
+        props.put(WorkerConfig.ADMIN_LISTENERS_CONFIG, "");
+        config = new WorkerConfig(WorkerConfig.baseConfigDef(), props);
+        assertTrue(config.getList(WorkerConfig.ADMIN_LISTENERS_CONFIG).isEmpty());
+
+        props.put(WorkerConfig.ADMIN_LISTENERS_CONFIG, "http://a.b:9999, https://a.b:7812");
+        config = new WorkerConfig(WorkerConfig.baseConfigDef(), props);
+        assertEquals(config.getList(WorkerConfig.ADMIN_LISTENERS_CONFIG), Arrays.asList("http://a.b:9999", "https://a.b:7812"));
+
+        new WorkerConfig(WorkerConfig.baseConfigDef(), props);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void testAdminListenersNotAllowingEmptyStrings() {
+        Map props = baseProps();
+        props.put(WorkerConfig.ADMIN_LISTENERS_CONFIG, "http://a.b:9999,");
+        new WorkerConfig(WorkerConfig.baseConfigDef(), props);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void testAdminListenersNotAllowingBlankStrings() {
+        Map props = baseProps();
+        props.put(WorkerConfig.ADMIN_LISTENERS_CONFIG, "http://a.b:9999, ,https://a.b:9999");
+        new WorkerConfig(WorkerConfig.baseConfigDef(), props);
+    }
+
+    private Map baseProps() {
+        Map props = new HashMap<>();
+        props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
+        props.put(WorkerConfig.KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.json.JsonConverter");
+        props.put(WorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.json.JsonConverter");
+        return props;
+    }
+
+}
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/RestServerTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/RestServerTest.java
index a640b83f383bb..25c7a3c7a4544 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/RestServerTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/RestServerTest.java
@@ -16,11 +16,16 @@
  */
 package org.apache.kafka.connect.runtime.rest;
 
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.http.HttpHost;
 import org.apache.http.HttpRequest;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpOptions;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.BasicResponseHandler;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.kafka.clients.CommonClientConfigs;
@@ -39,6 +44,7 @@
 import org.powermock.api.easymock.annotation.MockStrict;
 import org.powermock.core.classloader.annotations.PowerMockIgnore;
 import org.powermock.modules.junit4.PowerMockRunner;
+import org.slf4j.LoggerFactory;
 
 import javax.ws.rs.core.MediaType;
 import java.io.ByteArrayOutputStream;
@@ -49,6 +55,13 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.apache.kafka.connect.runtime.WorkerConfig.ADMIN_LISTENERS_CONFIG;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
 @RunWith(PowerMockRunner.class)
 @PowerMockIgnore({"javax.net.ssl.*", "javax.security.*"})
 public class RestServerTest {
@@ -279,4 +292,131 @@ public void testStandaloneConfig() throws IOException  {
 
         Assert.assertEquals(200, response.getStatusLine().getStatusCode());
     }
+
+    @Test
+    public void testLoggersEndpointWithDefaults() throws IOException {
+        Map configMap = new HashMap<>(baseWorkerProps());
+        DistributedConfig workerConfig = new DistributedConfig(configMap);
+
+        EasyMock.expect(herder.kafkaClusterId()).andReturn(KAFKA_CLUSTER_ID);
+        EasyMock.expect(herder.plugins()).andStubReturn(plugins);
+        EasyMock.expect(plugins.newPlugins(Collections.emptyList(),
+                workerConfig,
+                ConnectRestExtension.class))
+                .andStubReturn(Collections.emptyList());
+        PowerMock.replayAll();
+
+        // create some loggers in the process
+        LoggerFactory.getLogger("a.b.c.s.W");
+
+        server = new RestServer(workerConfig);
+        server.initializeServer();
+        server.initializeResources(herder);
+
+        ObjectMapper mapper = new ObjectMapper();
+
+        String host = server.advertisedUrl().getHost();
+        int port = server.advertisedUrl().getPort();
+
+        executePut(host, port, "/admin/loggers/a.b.c.s.W", "{\"level\": \"INFO\"}");
+
+        String responseStr = executeGet(host, port, "/admin/loggers");
+        Map> loggers = mapper.readValue(responseStr, new TypeReference>>() {
+        });
+        assertNotNull("expected non null response for /admin/loggers" + prettyPrint(loggers), loggers);
+        assertTrue("expect at least 1 logger. instead found " + prettyPrint(loggers), loggers.size() >= 1);
+        assertEquals("expected to find logger a.b.c.s.W set to INFO level", loggers.get("a.b.c.s.W").get("level"), "INFO");
+    }
+
+    @Test
+    public void testIndependentAdminEndpoint() throws IOException {
+        Map configMap = new HashMap<>(baseWorkerProps());
+        configMap.put(ADMIN_LISTENERS_CONFIG, "http://localhost:0");
+
+        DistributedConfig workerConfig = new DistributedConfig(configMap);
+
+        EasyMock.expect(herder.kafkaClusterId()).andReturn(KAFKA_CLUSTER_ID);
+        EasyMock.expect(herder.plugins()).andStubReturn(plugins);
+        EasyMock.expect(plugins.newPlugins(Collections.emptyList(),
+                workerConfig,
+                ConnectRestExtension.class))
+                .andStubReturn(Collections.emptyList());
+        PowerMock.replayAll();
+
+        // create some loggers in the process
+        LoggerFactory.getLogger("a.b.c.s.W");
+        LoggerFactory.getLogger("a.b.c.p.X");
+        LoggerFactory.getLogger("a.b.c.p.Y");
+        LoggerFactory.getLogger("a.b.c.p.Z");
+
+        server = new RestServer(workerConfig);
+        server.initializeServer();
+        server.initializeResources(herder);
+
+        assertNotEquals(server.advertisedUrl(), server.adminUrl());
+
+        executeGet(server.adminUrl().getHost(), server.adminUrl().getPort(), "/admin/loggers");
+
+        HttpRequest request = new HttpGet("/admin/loggers");
+        CloseableHttpClient httpClient = HttpClients.createMinimal();
+        HttpHost httpHost = new HttpHost(server.advertisedUrl().getHost(), server.advertisedUrl().getPort());
+        CloseableHttpResponse response = httpClient.execute(httpHost, request);
+        Assert.assertEquals(404, response.getStatusLine().getStatusCode());
+    }
+
+    @Test
+    public void testDisableAdminEndpoint() throws IOException {
+        Map configMap = new HashMap<>(baseWorkerProps());
+        configMap.put(ADMIN_LISTENERS_CONFIG, "");
+
+        DistributedConfig workerConfig = new DistributedConfig(configMap);
+
+        EasyMock.expect(herder.kafkaClusterId()).andReturn(KAFKA_CLUSTER_ID);
+        EasyMock.expect(herder.plugins()).andStubReturn(plugins);
+        EasyMock.expect(plugins.newPlugins(Collections.emptyList(),
+                workerConfig,
+                ConnectRestExtension.class))
+                .andStubReturn(Collections.emptyList());
+        PowerMock.replayAll();
+
+        server = new RestServer(workerConfig);
+        server.initializeServer();
+        server.initializeResources(herder);
+
+        assertNull(server.adminUrl());
+
+        HttpRequest request = new HttpGet("/admin/loggers");
+        CloseableHttpClient httpClient = HttpClients.createMinimal();
+        HttpHost httpHost = new HttpHost(server.advertisedUrl().getHost(), server.advertisedUrl().getPort());
+        CloseableHttpResponse response = httpClient.execute(httpHost, request);
+        Assert.assertEquals(404, response.getStatusLine().getStatusCode());
+    }
+
+    private String executeGet(String host, int port, String endpoint) throws IOException {
+        HttpRequest request = new HttpGet(endpoint);
+        CloseableHttpClient httpClient = HttpClients.createMinimal();
+        HttpHost httpHost = new HttpHost(host, port);
+        CloseableHttpResponse response = httpClient.execute(httpHost, request);
+
+        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+        return new BasicResponseHandler().handleResponse(response);
+    }
+
+    private String executePut(String host, int port, String endpoint, String jsonBody) throws IOException {
+        HttpPut request = new HttpPut(endpoint);
+        StringEntity entity = new StringEntity(jsonBody, "UTF-8");
+        entity.setContentType("application/json");
+        request.setEntity(entity);
+        CloseableHttpClient httpClient = HttpClients.createMinimal();
+        HttpHost httpHost = new HttpHost(host, port);
+        CloseableHttpResponse response = httpClient.execute(httpHost, request);
+
+        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+        return new BasicResponseHandler().handleResponse(response);
+    }
+
+    private static String prettyPrint(Map map) throws IOException {
+        ObjectMapper mapper = new ObjectMapper();
+        return mapper.writerWithDefaultPrettyPrinter().writeValueAsString(map);
+    }
 }
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/LoggingResourceTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/LoggingResourceTest.java
new file mode 100644
index 0000000000000..155eae92781af
--- /dev/null
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/LoggingResourceTest.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.runtime.rest.resources;
+
+import org.apache.kafka.connect.errors.NotFoundException;
+import org.apache.kafka.connect.runtime.rest.errors.BadRequestException;
+import org.apache.log4j.Hierarchy;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.Map;
+import java.util.Vector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@SuppressWarnings("unchecked")
+public class LoggingResourceTest {
+
+    @Test
+    public void getLoggersIgnoresNullLevelsTest() {
+        LoggingResource loggingResource = mock(LoggingResource.class);
+        Logger root = new Logger("root") {
+        };
+        Logger a = new Logger("a") {
+        };
+        a.setLevel(null);
+        Logger b = new Logger("b") {
+        };
+        b.setLevel(Level.INFO);
+        when(loggingResource.currentLoggers()).thenReturn(loggers(a, b));
+        when(loggingResource.rootLogger()).thenReturn(root);
+        when(loggingResource.listLoggers()).thenCallRealMethod();
+        Map> loggers = (Map>) loggingResource.listLoggers().getEntity();
+        assertEquals(1, loggers.size());
+        assertEquals("INFO", loggers.get("b").get("level"));
+    }
+
+    @Test
+    public void getLoggerFallsbackToEffectiveLogLevelTest() {
+        LoggingResource loggingResource = mock(LoggingResource.class);
+        Logger root = new Logger("root") {
+        };
+        root.setLevel(Level.ERROR);
+        Hierarchy hierarchy = new Hierarchy(root);
+        Logger a = hierarchy.getLogger("a");
+        a.setLevel(null);
+        Logger b = hierarchy.getLogger("b");
+        b.setLevel(Level.INFO);
+        when(loggingResource.currentLoggers()).thenReturn(loggers(a, b));
+        when(loggingResource.rootLogger()).thenReturn(root);
+        when(loggingResource.getLogger(any())).thenCallRealMethod();
+        Map level = (Map) loggingResource.getLogger("a").getEntity();
+        assertEquals(1, level.size());
+        assertEquals("ERROR", level.get("level"));
+    }
+
+    @Test(expected = NotFoundException.class)
+    public void getUnknownLoggerTest() {
+        LoggingResource loggingResource = mock(LoggingResource.class);
+        Logger root = new Logger("root") {
+        };
+        root.setLevel(Level.ERROR);
+        Hierarchy hierarchy = new Hierarchy(root);
+        Logger a = hierarchy.getLogger("a");
+        a.setLevel(null);
+        Logger b = hierarchy.getLogger("b");
+        b.setLevel(Level.INFO);
+        when(loggingResource.currentLoggers()).thenReturn(loggers(a, b));
+        when(loggingResource.rootLogger()).thenReturn(root);
+        when(loggingResource.getLogger(any())).thenCallRealMethod();
+        loggingResource.getLogger("c");
+    }
+
+    @Test
+    public void setLevelTest() {
+        LoggingResource loggingResource = mock(LoggingResource.class);
+        Logger root = new Logger("root") {
+        };
+        root.setLevel(Level.ERROR);
+        Hierarchy hierarchy = new Hierarchy(root);
+        Logger p = hierarchy.getLogger("a.b.c.p");
+        Logger x = hierarchy.getLogger("a.b.c.p.X");
+        Logger y = hierarchy.getLogger("a.b.c.p.Y");
+        Logger z = hierarchy.getLogger("a.b.c.p.Z");
+        Logger w = hierarchy.getLogger("a.b.c.s.W");
+        x.setLevel(Level.INFO);
+        y.setLevel(Level.INFO);
+        z.setLevel(Level.INFO);
+        w.setLevel(Level.INFO);
+        when(loggingResource.currentLoggers()).thenReturn(loggers(x, y, z, w));
+        when(loggingResource.lookupLogger("a.b.c.p")).thenReturn(p);
+        when(loggingResource.rootLogger()).thenReturn(root);
+        when(loggingResource.setLevel(any(), any())).thenCallRealMethod();
+        List modified = (List) loggingResource.setLevel("a.b.c.p", Collections.singletonMap("level", "DEBUG")).getEntity();
+        assertEquals(4, modified.size());
+        assertEquals(Arrays.asList("a.b.c.p", "a.b.c.p.X", "a.b.c.p.Y", "a.b.c.p.Z"), modified);
+        assertEquals(p.getLevel(), Level.DEBUG);
+        assertEquals(x.getLevel(), Level.DEBUG);
+        assertEquals(y.getLevel(), Level.DEBUG);
+        assertEquals(z.getLevel(), Level.DEBUG);
+    }
+
+    @Test
+    public void setRootLevelTest() {
+        LoggingResource loggingResource = mock(LoggingResource.class);
+        Logger root = new Logger("root") {
+        };
+        root.setLevel(Level.ERROR);
+        Hierarchy hierarchy = new Hierarchy(root);
+        Logger p = hierarchy.getLogger("a.b.c.p");
+        Logger x = hierarchy.getLogger("a.b.c.p.X");
+        Logger y = hierarchy.getLogger("a.b.c.p.Y");
+        Logger z = hierarchy.getLogger("a.b.c.p.Z");
+        Logger w = hierarchy.getLogger("a.b.c.s.W");
+        x.setLevel(Level.INFO);
+        y.setLevel(Level.INFO);
+        z.setLevel(Level.INFO);
+        w.setLevel(Level.INFO);
+        when(loggingResource.currentLoggers()).thenReturn(loggers(x, y, z, w));
+        when(loggingResource.lookupLogger("a.b.c.p")).thenReturn(p);
+        when(loggingResource.rootLogger()).thenReturn(root);
+        when(loggingResource.setLevel(any(), any())).thenCallRealMethod();
+        List modified = (List) loggingResource.setLevel("root", Collections.singletonMap("level", "DEBUG")).getEntity();
+        assertEquals(5, modified.size());
+        assertEquals(Arrays.asList("a.b.c.p.X", "a.b.c.p.Y", "a.b.c.p.Z", "a.b.c.s.W", "root"), modified);
+        assertNull(p.getLevel());
+        assertEquals(root.getLevel(), Level.DEBUG);
+        assertEquals(w.getLevel(), Level.DEBUG);
+        assertEquals(x.getLevel(), Level.DEBUG);
+        assertEquals(y.getLevel(), Level.DEBUG);
+        assertEquals(z.getLevel(), Level.DEBUG);
+    }
+
+    @Test(expected = BadRequestException.class)
+    public void setLevelWithEmptyArgTest() {
+        LoggingResource loggingResource = mock(LoggingResource.class);
+        Logger root = new Logger("root") {
+        };
+        root.setLevel(Level.ERROR);
+        Hierarchy hierarchy = new Hierarchy(root);
+        Logger a = hierarchy.getLogger("a");
+        a.setLevel(null);
+        Logger b = hierarchy.getLogger("b");
+        b.setLevel(Level.INFO);
+        when(loggingResource.currentLoggers()).thenReturn(loggers(a, b));
+        when(loggingResource.rootLogger()).thenReturn(root);
+        when(loggingResource.setLevel(any(), any())).thenCallRealMethod();
+        loggingResource.setLevel("@root", Collections.emptyMap());
+    }
+
+    @Test(expected = NotFoundException.class)
+    public void setLevelWithInvalidArgTest() {
+        LoggingResource loggingResource = mock(LoggingResource.class);
+        Logger root = new Logger("root") {
+        };
+        root.setLevel(Level.ERROR);
+        Hierarchy hierarchy = new Hierarchy(root);
+        Logger a = hierarchy.getLogger("a");
+        a.setLevel(null);
+        Logger b = hierarchy.getLogger("b");
+        b.setLevel(Level.INFO);
+        when(loggingResource.currentLoggers()).thenReturn(loggers(a, b));
+        when(loggingResource.rootLogger()).thenReturn(root);
+        when(loggingResource.setLevel(any(), any())).thenCallRealMethod();
+        loggingResource.setLevel("@root", Collections.singletonMap("level", "HIGH"));
+    }
+
+    private Enumeration loggers(Logger... loggers) {
+        return new Vector<>(Arrays.asList(loggers)).elements();
+    }
+
+}
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java
index fc93ae0595c77..49a31ae009dc6 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java
@@ -333,6 +333,16 @@ public ConnectorStateInfo connectorStatus(String connectorName) {
         }
     }
 
+    public String adminEndpoint(String resource) throws IOException {
+        String url = connectCluster.stream()
+                .map(WorkerHandle::adminUrl)
+                .filter(Objects::nonNull)
+                .findFirst()
+                .orElseThrow(() -> new IOException("Admin endpoint is disabled."))
+                .toString();
+        return url + resource;
+    }
+
     public String endpointForResource(String resource) throws IOException {
         String url = connectCluster.stream()
                 .map(WorkerHandle::url)
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/WorkerHandle.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/WorkerHandle.java
index 7113f52a19b1f..64b2b1220f8ce 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/WorkerHandle.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/WorkerHandle.java
@@ -75,6 +75,15 @@ public URI url() {
         return worker.restUrl();
     }
 
+    /**
+     * Get the workers's url that accepts requests to its Admin REST endpoint.
+     *
+     * @return the worker's admin url
+     */
+    public URI adminUrl() {
+        return worker.adminUrl();
+    }
+
     @Override
     public String toString() {
         return "WorkerHandle{" +

From 16edc54048fec621c8bd428dee22f927903969b0 Mon Sep 17 00:00:00 2001
From: Konstantine Karantasis 
Date: Wed, 2 Oct 2019 15:04:10 -0700
Subject: [PATCH 0676/1071] KAFKA-5609: Connect log4j should also log to a file
 by default (KIP-521) (#7430)

Implemented KIP-521 to change the provided Connect Log4J configuration file to also write logs to files and to rotate them daily.

Author: Konstantine Karantasis 
Reviewer: Randall Hauch 
---
 config/connect-log4j.properties | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/config/connect-log4j.properties b/config/connect-log4j.properties
index d4b7adb39bcf5..f695e37eb348b 100644
--- a/config/connect-log4j.properties
+++ b/config/connect-log4j.properties
@@ -13,19 +13,31 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-log4j.rootLogger=INFO, stdout
-
+log4j.rootLogger=INFO, stdout, connectAppender
 
+# Send the logs to the console.
+#
 log4j.appender.stdout=org.apache.log4j.ConsoleAppender
 log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
 
+# Send the logs to a file, rolling the file at midnight local time. For example, the `File` option specifies the
+# location of the log files (e.g. ${kafka.logs.dir}/connect.log), and at midnight local time the file is closed
+# and copied in the same directory but with a filename that ends in the `DatePattern` option.
 #
+log4j.appender.connectAppender=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.connectAppender.DatePattern='.'yyyy-MM-dd-HH
+log4j.appender.connectAppender.File=${kafka.logs.dir}/connect.log
+log4j.appender.connectAppender.layout=org.apache.log4j.PatternLayout
+
 # The `%X{connector.context}` parameter in the layout includes connector-specific and task-specific information
 # in the log message, where appropriate. This makes it easier to identify those log messages that apply to a
 # specific connector. Simply add this parameter to the log layout configuration below to include the contextual information.
 #
-#log4j.appender.stdout.layout.ConversionPattern=[%d] %p %X{connector.context}%m (%c:%L)%n
-log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n
+connect.log.pattern=[%d] %p %m (%c:%L)%n
+#connect.log.pattern=[%d] %p %X{connector.context}%m (%c:%L)%n
+
+log4j.appender.stdout.layout.ConversionPattern=${connect.log.pattern}
+log4j.appender.connectAppender.layout.ConversionPattern=${connect.log.pattern}
 
 log4j.logger.org.apache.zookeeper=ERROR
 log4j.logger.org.reflections=ERROR

From 791d0d61bf5a2b7a4fda72d4c50075a1933a6af3 Mon Sep 17 00:00:00 2001
From: Chris Egerton 
Date: Wed, 2 Oct 2019 15:06:57 -0700
Subject: [PATCH 0677/1071] KAFKA-8804: Secure internal Connect REST endpoints
 (#7310)

Implemented KIP-507 to secure the internal Connect REST endpoints that are only for intra-cluster communication. A new V2 of the Connect subprotocol enables this feature, where the leader generates a new session key, shares it with the other workers via the configuration topic, and workers send and validate requests to these internal endpoints using the shared key.

Currently the internal `POST /connectors//tasks` endpoint is the only one that is secured.

This change adds unit tests and makes some small alterations to system tests to target the new `sessioned` Connect subprotocol. A new integration test ensures that the endpoint is actually secured (i.e., requests with missing/invalid signatures are rejected with a 400 BAD RESPONSE status).

Author: Chris Egerton 
Reviewed: Konstantine Karantasis , Randall Hauch 
---
 checkstyle/import-control.xml                 |   6 +
 checkstyle/suppressions.xml                   |   2 +-
 .../runtime/ConnectMetricsRegistry.java       |   2 +
 .../apache/kafka/connect/runtime/Herder.java  |   5 +-
 .../kafka/connect/runtime/SessionKey.java     |  73 +++
 .../distributed/ClusterConfigState.java       |  18 +
 .../ConnectProtocolCompatibility.java         |  64 ++-
 .../distributed/DistributedConfig.java        | 437 +++++++++++-------
 .../distributed/DistributedHerder.java        | 202 +++++++-
 .../IncrementalCooperativeAssignor.java       |  19 +-
 ...IncrementalCooperativeConnectProtocol.java |  49 +-
 .../distributed/WorkerCoordinator.java        |   6 +-
 .../rest/InternalRequestSignature.java        | 148 ++++++
 .../connect/runtime/rest/RestClient.java      |  43 +-
 .../rest/resources/ConnectorsResource.java    |  10 +-
 .../runtime/standalone/StandaloneHerder.java  |   9 +-
 .../connect/storage/ConfigBackingStore.java   |   9 +
 .../storage/KafkaConfigBackingStore.java      |  71 +++
 .../storage/MemoryConfigBackingStore.java     |   7 +
 .../SessionedProtocolIntegrationTest.java     | 168 +++++++
 .../connect/runtime/AbstractHerderTest.java   |   4 +-
 .../connect/runtime/WorkerTestUtils.java      |   1 +
 .../ConnectProtocolCompatibilityTest.java     |  27 +-
 .../distributed/DistributedConfigTest.java    | 108 +++++
 .../distributed/DistributedHerderTest.java    | 140 +++++-
 .../IncrementalCooperativeAssignorTest.java   |  72 +--
 .../WorkerCoordinatorIncrementalTest.java     |  13 +-
 .../distributed/WorkerCoordinatorTest.java    |   3 +
 .../rest/InternalRequestSignatureTest.java    | 151 ++++++
 .../connect/runtime/rest/RestServerTest.java  |   2 +-
 .../resources/ConnectorsResourceTest.java     |  71 ++-
 .../standalone/StandaloneHerderTest.java      |   6 +-
 .../storage/KafkaConfigBackingStoreTest.java  |   2 +-
 .../storage/KafkaOffsetBackingStoreTest.java  |   2 +-
 .../util/clusters/EmbeddedConnectCluster.java |  22 +
 .../tests/connect/connect_distributed_test.py |  23 +-
 .../templates/connect-distributed.properties  |   2 +-
 37 files changed, 1730 insertions(+), 267 deletions(-)
 create mode 100644 connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SessionKey.java
 create mode 100644 connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/InternalRequestSignature.java
 create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/integration/SessionedProtocolIntegrationTest.java
 create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedConfigTest.java
 create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/InternalRequestSignatureTest.java

diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml
index 7fd418aab4636..5f27dde64f091 100644
--- a/checkstyle/import-control.xml
+++ b/checkstyle/import-control.xml
@@ -346,6 +346,7 @@
       
       
       
+      
 
       
         
@@ -363,6 +364,10 @@
         
         
       
+
+      
+        
+      
     
 
     
@@ -376,6 +381,7 @@
     
       
       
+      
     
 
     
diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index 4912a2593e146..6a24ef94c29a7 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -132,7 +132,7 @@
               files="Values.java"/>
 
     
+              files="(DistributedHerder|RestClient|JsonConverter|KafkaConfigBackingStore|FileStreamSourceTask).java"/>
 
     
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectMetricsRegistry.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectMetricsRegistry.java
index 04699ea9ecbfe..44eba5ca5c044 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectMetricsRegistry.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectMetricsRegistry.java
@@ -87,6 +87,7 @@ public class ConnectMetricsRegistry {
     public final MetricNameTemplate taskStartupSuccessPercentage;
     public final MetricNameTemplate taskStartupFailureTotal;
     public final MetricNameTemplate taskStartupFailurePercentage;
+    public final MetricNameTemplate connectProtocol;
     public final MetricNameTemplate leaderName;
     public final MetricNameTemplate epoch;
     public final MetricNameTemplate rebalanceCompletedTotal;
@@ -291,6 +292,7 @@ public ConnectMetricsRegistry(Set tags) {
         /***** Worker rebalance level *****/
         Set rebalanceTags = new LinkedHashSet<>(tags);
 
+        connectProtocol = createTemplate("connect-protocol", WORKER_REBALANCE_GROUP_NAME, "The Connect protocol used by this cluster", rebalanceTags);
         leaderName = createTemplate("leader-name", WORKER_REBALANCE_GROUP_NAME, "The name of the group leader.", rebalanceTags);
         epoch = createTemplate("epoch", WORKER_REBALANCE_GROUP_NAME, "The epoch or generation number of this worker.", rebalanceTags);
         rebalanceCompletedTotal = createTemplate("completed-rebalances-total", WORKER_REBALANCE_GROUP_NAME,
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Herder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Herder.java
index 550348ff94f3e..7915682790317 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Herder.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Herder.java
@@ -17,6 +17,7 @@
 package org.apache.kafka.connect.runtime;
 
 import org.apache.kafka.connect.runtime.isolation.Plugins;
+import org.apache.kafka.connect.runtime.rest.InternalRequestSignature;
 import org.apache.kafka.connect.runtime.rest.entities.ConfigInfos;
 import org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo;
 import org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo;
@@ -120,8 +121,10 @@ public interface Herder {
      * @param connName connector to update
      * @param configs list of configurations
      * @param callback callback to invoke upon completion
+     * @param requestSignature the signature of the request made for this task (re-)configuration;
+     *                         may be null if no signature was provided
      */
-    void putTaskConfigs(String connName, List> configs, Callback callback);
+    void putTaskConfigs(String connName, List> configs, Callback callback, InternalRequestSignature requestSignature);
 
     /**
      * Get a list of connectors currently running in this cluster.
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SessionKey.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SessionKey.java
new file mode 100644
index 0000000000000..ab5476e4b9000
--- /dev/null
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SessionKey.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.runtime;
+
+import javax.crypto.SecretKey;
+import java.util.Objects;
+
+/**
+ * A session key, which can be used to validate internal REST requests between workers.
+ */
+public class SessionKey {
+
+    private final SecretKey key;
+    private final long creationTimestamp;
+
+    /**
+     * Create a new session key with the given key value and creation timestamp
+     * @param key the actual cryptographic key to use for request validation; may not be null
+     * @param creationTimestamp the time at which the key was generated
+     */
+    public SessionKey(SecretKey key, long creationTimestamp) {
+        this.key = Objects.requireNonNull(key, "Key may not be null");
+        this.creationTimestamp = creationTimestamp;
+    }
+
+    /**
+     * Get the cryptographic key to use for request validation.
+     *
+     * @return the cryptographic key; may not be null
+     */
+    public SecretKey key() {
+        return key;
+    }
+
+    /**
+     * Get the time at which the key was generated.
+     *
+     * @return the time at which the key was generated
+     */
+    public long creationTimestamp() {
+        return creationTimestamp;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        SessionKey that = (SessionKey) o;
+        return creationTimestamp == that.creationTimestamp
+            && key.equals(that.key);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(key, creationTimestamp);
+    }
+}
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/ClusterConfigState.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/ClusterConfigState.java
index fc6a50d2fc078..a85a8e69e9f99 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/ClusterConfigState.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/ClusterConfigState.java
@@ -17,6 +17,7 @@
 package org.apache.kafka.connect.runtime.distributed;
 
 import org.apache.kafka.common.config.provider.ConfigProvider;
+import org.apache.kafka.connect.runtime.SessionKey;
 import org.apache.kafka.connect.runtime.WorkerConfigTransformer;
 import org.apache.kafka.connect.runtime.TargetState;
 import org.apache.kafka.connect.util.ConnectorTaskId;
@@ -37,6 +38,7 @@ public class ClusterConfigState {
     public static final long NO_OFFSET = -1;
     public static final ClusterConfigState EMPTY = new ClusterConfigState(
             NO_OFFSET,
+            null,
             Collections.emptyMap(),
             Collections.>emptyMap(),
             Collections.emptyMap(),
@@ -44,6 +46,7 @@ public class ClusterConfigState {
             Collections.emptySet());
 
     private final long offset;
+    private final SessionKey sessionKey;
     private final Map connectorTaskCounts;
     private final Map> connectorConfigs;
     private final Map connectorTargetStates;
@@ -52,12 +55,14 @@ public class ClusterConfigState {
     private final WorkerConfigTransformer configTransformer;
 
     public ClusterConfigState(long offset,
+                              SessionKey sessionKey,
                               Map connectorTaskCounts,
                               Map> connectorConfigs,
                               Map connectorTargetStates,
                               Map> taskConfigs,
                               Set inconsistentConnectors) {
         this(offset,
+                sessionKey,
                 connectorTaskCounts,
                 connectorConfigs,
                 connectorTargetStates,
@@ -67,6 +72,7 @@ public ClusterConfigState(long offset,
     }
 
     public ClusterConfigState(long offset,
+                              SessionKey sessionKey,
                               Map connectorTaskCounts,
                               Map> connectorConfigs,
                               Map connectorTargetStates,
@@ -74,6 +80,7 @@ public ClusterConfigState(long offset,
                               Set inconsistentConnectors,
                               WorkerConfigTransformer configTransformer) {
         this.offset = offset;
+        this.sessionKey = sessionKey;
         this.connectorTaskCounts = connectorTaskCounts;
         this.connectorConfigs = connectorConfigs;
         this.connectorTargetStates = connectorTargetStates;
@@ -91,6 +98,14 @@ public long offset() {
         return offset;
     }
 
+    /**
+     * Get the latest session key from the config state
+     * @return the {@link SessionKey session key}; may be null if no key has been read yet
+     */
+    public SessionKey sessionKey() {
+        return sessionKey;
+    }
+
     /**
      * Check whether this snapshot contains configuration for a connector.
      * @param connector name of the connector
@@ -229,6 +244,7 @@ public Set inconsistentConnectors() {
     public String toString() {
         return "ClusterConfigState{" +
                 "offset=" + offset +
+                ", sessionKey=" + (sessionKey != null ? "[hidden]" : "null") +
                 ", connectorTaskCounts=" + connectorTaskCounts +
                 ", connectorConfigs=" + connectorConfigs +
                 ", taskConfigs=" + taskConfigs +
@@ -242,6 +258,7 @@ public boolean equals(Object o) {
         if (o == null || getClass() != o.getClass()) return false;
         ClusterConfigState that = (ClusterConfigState) o;
         return offset == that.offset &&
+                Objects.equals(sessionKey, that.sessionKey) &&
                 Objects.equals(connectorTaskCounts, that.connectorTaskCounts) &&
                 Objects.equals(connectorConfigs, that.connectorConfigs) &&
                 Objects.equals(connectorTargetStates, that.connectorTargetStates) &&
@@ -254,6 +271,7 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(
                 offset,
+                sessionKey,
                 connectorTaskCounts,
                 connectorConfigs,
                 connectorTargetStates,
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/ConnectProtocolCompatibility.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/ConnectProtocolCompatibility.java
index ccded8baaba20..d618fe255345d 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/ConnectProtocolCompatibility.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/ConnectProtocolCompatibility.java
@@ -19,6 +19,10 @@
 import java.util.Arrays;
 import java.util.Locale;
 
+import static org.apache.kafka.connect.runtime.distributed.ConnectProtocol.CONNECT_PROTOCOL_V0;
+import static org.apache.kafka.connect.runtime.distributed.IncrementalCooperativeConnectProtocol.CONNECT_PROTOCOL_V1;
+import static org.apache.kafka.connect.runtime.distributed.IncrementalCooperativeConnectProtocol.CONNECT_PROTOCOL_V2;
+
 /**
  * An enumeration of the modes available to the worker to signal which Connect protocols are
  * enabled at any time.
@@ -28,7 +32,11 @@
  *
  * {@code COMPATIBLE} signifies that this worker supports both eager and incremental cooperative
  * Connect protocols and will use the version that is elected by the Kafka broker coordinator
- * during rebalancing.
+ * during rebalance.
+ *
+ * {@code SESSIONED} signifies that this worker supports all of the above protocols in addition to
+ * a protocol that uses incremental cooperative rebalancing for worker assignment and uses session
+ * keys distributed via the config topic to verify internal REST requests
  */
 public enum ConnectProtocolCompatibility {
     EAGER {
@@ -36,6 +44,11 @@ public enum ConnectProtocolCompatibility {
         public String protocol() {
             return "default";
         }
+
+        @Override
+        public short protocolVersion() {
+            return CONNECT_PROTOCOL_V0;
+        }
     },
 
     COMPATIBLE {
@@ -43,11 +56,28 @@ public String protocol() {
         public String protocol() {
             return "compatible";
         }
+
+        @Override
+        public short protocolVersion() {
+            return CONNECT_PROTOCOL_V1;
+        }
+    },
+
+    SESSIONED {
+        @Override
+        public String protocol() {
+            return "sessioned";
+        }
+
+        @Override
+        public short protocolVersion() {
+            return CONNECT_PROTOCOL_V2;
+        }
     };
 
     /**
      * Return the enum that corresponds to the name that is given as an argument;
-     * if the no mapping is found {@code IllegalArgumentException} is thrown.
+     * if no mapping is found {@code IllegalArgumentException} is thrown.
      *
      * @param name the name of the protocol compatibility mode
      * @return the enum that corresponds to the protocol compatibility mode
@@ -60,11 +90,39 @@ public static ConnectProtocolCompatibility compatibility(String name) {
                         "Unknown Connect protocol compatibility mode: " + name));
     }
 
+    /**
+     * Return the enum that corresponds to the Connect protocol version that is given as an argument;
+     * if no mapping is found {@code IllegalArgumentException} is thrown.
+     *
+     * @param protocolVersion the version of the protocol; for example,
+     * {@link ConnectProtocol#CONNECT_PROTOCOL_V0 CONNECT_PROTOCOL_V0}. May not be null
+     * @return the enum that corresponds to the protocol compatibility mode
+     */
+    public static ConnectProtocolCompatibility fromProtocolVersion(short protocolVersion) {
+        switch (protocolVersion) {
+            case CONNECT_PROTOCOL_V0:
+                return EAGER;
+            case CONNECT_PROTOCOL_V1:
+                return COMPATIBLE;
+            case CONNECT_PROTOCOL_V2:
+                return SESSIONED;
+            default:
+                throw new IllegalArgumentException("Unknown Connect protocol version: " + protocolVersion);
+        }
+    }
+
     @Override
     public String toString() {
         return name().toLowerCase(Locale.ROOT);
     }
 
+    /**
+     * Return the version of the protocol for this mode.
+     *
+     * @return the protocol version
+     */
+    public abstract short protocolVersion();
+
     /**
      * Return the name of the protocol that this mode will use in {@code ProtocolMetadata}.
      *
@@ -74,7 +132,7 @@ public String toString() {
 
     /**
      * Return the enum that corresponds to the protocol name that is given as an argument;
-     * if the no mapping is found {@code IllegalArgumentException} is thrown.
+     * if no mapping is found {@code IllegalArgumentException} is thrown.
      *
      * @param protocolName the name of the connect protocol
      * @return the enum that corresponds to the protocol compatibility mode that supports the
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java
index f383482d9fdd6..68c7f61a39619 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java
@@ -22,15 +22,20 @@
 import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.connect.runtime.WorkerConfig;
 
+import javax.crypto.KeyGenerator;
+import javax.crypto.Mac;
+import java.security.InvalidParameterException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 
 import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
 import static org.apache.kafka.common.config.ConfigDef.Range.between;
 
 public class DistributedConfig extends WorkerConfig {
-    private static final ConfigDef CONFIG;
-
     /*
      * NOTE: DO NOT CHANGE EITHER CONFIG STRINGS OR THEIR JAVA VARIABLE NAMES AS
      * THESE ARE PART OF THE PUBLIC API AND CHANGE WILL BREAK USER CODE.
@@ -138,7 +143,7 @@ public class DistributedConfig extends WorkerConfig {
      */
     public static final String CONNECT_PROTOCOL_CONFIG = "connect.protocol";
     public static final String CONNECT_PROTOCOL_DOC = "Compatibility mode for Kafka Connect Protocol";
-    public static final String CONNECT_PROTOCOL_DEFAULT = ConnectProtocolCompatibility.COMPATIBLE.toString();
+    public static final String CONNECT_PROTOCOL_DEFAULT = ConnectProtocolCompatibility.SESSIONED.toString();
 
     /**
      * connect.protocol
@@ -150,162 +155,220 @@ public class DistributedConfig extends WorkerConfig {
             + "period the connectors and tasks of the departed workers remain unassigned";
     public static final int SCHEDULED_REBALANCE_MAX_DELAY_MS_DEFAULT = Math.toIntExact(TimeUnit.SECONDS.toMillis(300));
 
-    static {
-        CONFIG = baseConfigDef()
-                .define(GROUP_ID_CONFIG,
-                        ConfigDef.Type.STRING,
-                        ConfigDef.Importance.HIGH,
-                        GROUP_ID_DOC)
-                .define(SESSION_TIMEOUT_MS_CONFIG,
-                        ConfigDef.Type.INT,
-                        10000,
-                        ConfigDef.Importance.HIGH,
-                        SESSION_TIMEOUT_MS_DOC)
-                .define(REBALANCE_TIMEOUT_MS_CONFIG,
-                        ConfigDef.Type.INT,
-                        60000,
-                        ConfigDef.Importance.HIGH,
-                        REBALANCE_TIMEOUT_MS_DOC)
-                .define(HEARTBEAT_INTERVAL_MS_CONFIG,
-                        ConfigDef.Type.INT,
-                        3000,
-                        ConfigDef.Importance.HIGH,
-                        HEARTBEAT_INTERVAL_MS_DOC)
-                .define(CommonClientConfigs.METADATA_MAX_AGE_CONFIG,
-                        ConfigDef.Type.LONG,
-                        5 * 60 * 1000,
-                        atLeast(0),
-                        ConfigDef.Importance.LOW,
-                        CommonClientConfigs.METADATA_MAX_AGE_DOC)
-                .define(CommonClientConfigs.CLIENT_ID_CONFIG,
-                        ConfigDef.Type.STRING,
-                        "",
-                        ConfigDef.Importance.LOW,
-                        CommonClientConfigs.CLIENT_ID_DOC)
-                .define(CommonClientConfigs.SEND_BUFFER_CONFIG,
-                        ConfigDef.Type.INT,
-                        128 * 1024,
-                        atLeast(0),
-                        ConfigDef.Importance.MEDIUM,
-                        CommonClientConfigs.SEND_BUFFER_DOC)
-                .define(CommonClientConfigs.RECEIVE_BUFFER_CONFIG,
-                        ConfigDef.Type.INT,
-                        32 * 1024,
-                        atLeast(0),
-                        ConfigDef.Importance.MEDIUM,
-                        CommonClientConfigs.RECEIVE_BUFFER_DOC)
-                .define(CommonClientConfigs.RECONNECT_BACKOFF_MS_CONFIG,
-                        ConfigDef.Type.LONG,
-                        50L,
-                        atLeast(0L),
-                        ConfigDef.Importance.LOW,
-                        CommonClientConfigs.RECONNECT_BACKOFF_MS_DOC)
-                .define(CommonClientConfigs.RECONNECT_BACKOFF_MAX_MS_CONFIG,
-                        ConfigDef.Type.LONG,
-                        1000L,
-                        atLeast(0L),
-                        ConfigDef.Importance.LOW,
-                        CommonClientConfigs.RECONNECT_BACKOFF_MAX_MS_DOC)
-                .define(CommonClientConfigs.RETRY_BACKOFF_MS_CONFIG,
-                        ConfigDef.Type.LONG,
-                        100L,
-                        atLeast(0L),
-                        ConfigDef.Importance.LOW,
-                        CommonClientConfigs.RETRY_BACKOFF_MS_DOC)
-                .define(CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG,
-                        ConfigDef.Type.INT,
-                        40 * 1000,
-                        atLeast(0),
-                        ConfigDef.Importance.MEDIUM,
-                        CommonClientConfigs.REQUEST_TIMEOUT_MS_DOC)
-                        /* default is set to be a bit lower than the server default (10 min), to avoid both client and server closing connection at same time */
-                .define(CommonClientConfigs.CONNECTIONS_MAX_IDLE_MS_CONFIG,
-                        ConfigDef.Type.LONG,
-                        9 * 60 * 1000,
-                        ConfigDef.Importance.MEDIUM,
-                        CommonClientConfigs.CONNECTIONS_MAX_IDLE_MS_DOC)
-                // security support
-                .define(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG,
-                        ConfigDef.Type.STRING,
-                        CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL,
-                        ConfigDef.Importance.MEDIUM,
-                        CommonClientConfigs.SECURITY_PROTOCOL_DOC)
-                .withClientSslSupport()
-                .withClientSaslSupport()
-                .define(WORKER_SYNC_TIMEOUT_MS_CONFIG,
-                        ConfigDef.Type.INT,
-                        3000,
-                        ConfigDef.Importance.MEDIUM,
-                        WORKER_SYNC_TIMEOUT_MS_DOC)
-                .define(WORKER_UNSYNC_BACKOFF_MS_CONFIG,
-                        ConfigDef.Type.INT,
-                        WORKER_UNSYNC_BACKOFF_MS_DEFAULT,
-                        ConfigDef.Importance.MEDIUM,
-                        WORKER_UNSYNC_BACKOFF_MS_DOC)
-                .define(OFFSET_STORAGE_TOPIC_CONFIG,
-                        ConfigDef.Type.STRING,
-                        ConfigDef.Importance.HIGH,
-                        OFFSET_STORAGE_TOPIC_CONFIG_DOC)
-                .define(OFFSET_STORAGE_PARTITIONS_CONFIG,
-                        ConfigDef.Type.INT,
-                        25,
-                        atLeast(1),
-                        ConfigDef.Importance.LOW,
-                        OFFSET_STORAGE_PARTITIONS_CONFIG_DOC)
-                .define(OFFSET_STORAGE_REPLICATION_FACTOR_CONFIG,
-                        ConfigDef.Type.SHORT,
-                        (short) 3,
-                        atLeast(1),
-                        ConfigDef.Importance.LOW,
-                        OFFSET_STORAGE_REPLICATION_FACTOR_CONFIG_DOC)
-                .define(CONFIG_TOPIC_CONFIG,
-                        ConfigDef.Type.STRING,
-                        ConfigDef.Importance.HIGH,
-                        CONFIG_TOPIC_CONFIG_DOC)
-                .define(CONFIG_STORAGE_REPLICATION_FACTOR_CONFIG,
-                        ConfigDef.Type.SHORT,
-                        (short) 3,
-                        atLeast(1),
-                        ConfigDef.Importance.LOW,
-                        CONFIG_STORAGE_REPLICATION_FACTOR_CONFIG_DOC)
-                .define(STATUS_STORAGE_TOPIC_CONFIG,
-                        ConfigDef.Type.STRING,
-                        ConfigDef.Importance.HIGH,
-                        STATUS_STORAGE_TOPIC_CONFIG_DOC)
-                .define(STATUS_STORAGE_PARTITIONS_CONFIG,
-                        ConfigDef.Type.INT,
-                        5,
-                        atLeast(1),
-                        ConfigDef.Importance.LOW,
-                        STATUS_STORAGE_PARTITIONS_CONFIG_DOC)
-                .define(STATUS_STORAGE_REPLICATION_FACTOR_CONFIG,
-                        ConfigDef.Type.SHORT,
-                        (short) 3,
-                        atLeast(1),
-                        ConfigDef.Importance.LOW,
-                        STATUS_STORAGE_REPLICATION_FACTOR_CONFIG_DOC)
-                .define(CONNECT_PROTOCOL_CONFIG,
-                        ConfigDef.Type.STRING,
-                        CONNECT_PROTOCOL_DEFAULT,
-                        ConfigDef.LambdaValidator.with(
-                            (name, value) -> {
-                                try {
-                                    ConnectProtocolCompatibility.compatibility((String) value);
-                                } catch (Throwable t) {
-                                    throw new ConfigException(name, value, "Invalid Connect protocol "
-                                            + "compatibility");
-                                }
-                            },
-                            () -> "[" + Utils.join(ConnectProtocolCompatibility.values(), ", ") + "]"),
-                        ConfigDef.Importance.LOW,
-                        CONNECT_PROTOCOL_DOC)
-                .define(SCHEDULED_REBALANCE_MAX_DELAY_MS_CONFIG,
-                        ConfigDef.Type.INT,
-                        SCHEDULED_REBALANCE_MAX_DELAY_MS_DEFAULT,
-                        between(0, Integer.MAX_VALUE),
-                        ConfigDef.Importance.LOW,
-                        SCHEDULED_REBALANCE_MAX_DELAY_MS_DOC);
-    }
+    public static final String INTER_WORKER_KEY_GENERATION_ALGORITHM_CONFIG = "inter.worker.key.generation.algorithm";
+    public static final String INTER_WORKER_KEY_GENERATION_ALGORITHM_DOC = "The algorithm to use for generating internal request keys";
+    public static final String INTER_WORKER_KEY_GENERATION_ALGORITHM_DEFAULT = "HmacSHA256";
+
+    public static final String INTER_WORKER_KEY_SIZE_CONFIG = "inter.worker.key.size";
+    public static final String INTER_WORKER_KEY_SIZE_DOC = "The size of the key to use for signing internal requests, in bits. "
+        + "If null, the default key size for the key generation algorithm will be used.";
+    public static final Long INTER_WORKER_KEY_SIZE_DEFAULT = null;
+
+    public static final String INTER_WORKER_KEY_TTL_MS_CONFIG = "inter.worker.key.ttl.ms";
+    public static final String INTER_WORKER_KEY_TTL_MS_MS_DOC = "The TTL of generated session keys used for "
+        + "internal request validation (in milliseconds)";
+    public static final int INTER_WORKER_KEY_TTL_MS_MS_DEFAULT = Math.toIntExact(TimeUnit.HOURS.toMillis(1));
+
+    public static final String INTER_WORKER_SIGNATURE_ALGORITHM_CONFIG = "inter.worker.signature.algorithm";
+    public static final String INTER_WORKER_SIGNATURE_ALGORITHM_DOC = "The algorithm used to sign internal requests";
+    public static final String INTER_WORKER_SIGNATURE_ALGORITHM_DEFAULT = "HmacSHA256";
+
+    public static final String INTER_WORKER_VERIFICATION_ALGORITHMS_CONFIG = "inter.worker.verification.algorithms";
+    public static final String INTER_WORKER_VERIFICATION_ALGORITHMS_DOC = "A list of permitted algorithms for verifying internal requests";
+    public static final List INTER_WORKER_VERIFICATION_ALGORITHMS_DEFAULT = Collections.singletonList(INTER_WORKER_SIGNATURE_ALGORITHM_DEFAULT);
+
+    @SuppressWarnings("unchecked")
+    private static final ConfigDef CONFIG = baseConfigDef()
+            .define(GROUP_ID_CONFIG,
+                    ConfigDef.Type.STRING,
+                    ConfigDef.Importance.HIGH,
+                    GROUP_ID_DOC)
+            .define(SESSION_TIMEOUT_MS_CONFIG,
+                    ConfigDef.Type.INT,
+                    Math.toIntExact(TimeUnit.SECONDS.toMillis(10)),
+                    ConfigDef.Importance.HIGH,
+                    SESSION_TIMEOUT_MS_DOC)
+            .define(REBALANCE_TIMEOUT_MS_CONFIG,
+                    ConfigDef.Type.INT,
+                    Math.toIntExact(TimeUnit.MINUTES.toMillis(1)),
+                    ConfigDef.Importance.HIGH,
+                    REBALANCE_TIMEOUT_MS_DOC)
+            .define(HEARTBEAT_INTERVAL_MS_CONFIG,
+                    ConfigDef.Type.INT,
+                    Math.toIntExact(TimeUnit.SECONDS.toMillis(3)),
+                    ConfigDef.Importance.HIGH,
+                    HEARTBEAT_INTERVAL_MS_DOC)
+            .define(CommonClientConfigs.METADATA_MAX_AGE_CONFIG,
+                    ConfigDef.Type.LONG,
+                    TimeUnit.MINUTES.toMillis(5),
+                    atLeast(0),
+                    ConfigDef.Importance.LOW,
+                    CommonClientConfigs.METADATA_MAX_AGE_DOC)
+            .define(CommonClientConfigs.CLIENT_ID_CONFIG,
+                    ConfigDef.Type.STRING,
+                    "",
+                    ConfigDef.Importance.LOW,
+                    CommonClientConfigs.CLIENT_ID_DOC)
+            .define(CommonClientConfigs.SEND_BUFFER_CONFIG,
+                    ConfigDef.Type.INT,
+                    128 * 1024,
+                    atLeast(0),
+                    ConfigDef.Importance.MEDIUM,
+                    CommonClientConfigs.SEND_BUFFER_DOC)
+            .define(CommonClientConfigs.RECEIVE_BUFFER_CONFIG,
+                    ConfigDef.Type.INT,
+                    32 * 1024,
+                    atLeast(0),
+                    ConfigDef.Importance.MEDIUM,
+                    CommonClientConfigs.RECEIVE_BUFFER_DOC)
+            .define(CommonClientConfigs.RECONNECT_BACKOFF_MS_CONFIG,
+                    ConfigDef.Type.LONG,
+                    50L,
+                    atLeast(0L),
+                    ConfigDef.Importance.LOW,
+                    CommonClientConfigs.RECONNECT_BACKOFF_MS_DOC)
+            .define(CommonClientConfigs.RECONNECT_BACKOFF_MAX_MS_CONFIG,
+                    ConfigDef.Type.LONG,
+                    TimeUnit.SECONDS.toMillis(1),
+                    atLeast(0L),
+                    ConfigDef.Importance.LOW,
+                    CommonClientConfigs.RECONNECT_BACKOFF_MAX_MS_DOC)
+            .define(CommonClientConfigs.RETRY_BACKOFF_MS_CONFIG,
+                    ConfigDef.Type.LONG,
+                    100L,
+                    atLeast(0L),
+                    ConfigDef.Importance.LOW,
+                    CommonClientConfigs.RETRY_BACKOFF_MS_DOC)
+            .define(CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG,
+                    ConfigDef.Type.INT,
+                    Math.toIntExact(TimeUnit.SECONDS.toMillis(40)),
+                    atLeast(0),
+                    ConfigDef.Importance.MEDIUM,
+                    CommonClientConfigs.REQUEST_TIMEOUT_MS_DOC)
+                    /* default is set to be a bit lower than the server default (10 min), to avoid both client and server closing connection at same time */
+            .define(CommonClientConfigs.CONNECTIONS_MAX_IDLE_MS_CONFIG,
+                    ConfigDef.Type.LONG,
+                    TimeUnit.MINUTES.toMillis(9),
+                    ConfigDef.Importance.MEDIUM,
+                    CommonClientConfigs.CONNECTIONS_MAX_IDLE_MS_DOC)
+            // security support
+            .define(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG,
+                    ConfigDef.Type.STRING,
+                    CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL,
+                    ConfigDef.Importance.MEDIUM,
+                    CommonClientConfigs.SECURITY_PROTOCOL_DOC)
+            .withClientSslSupport()
+            .withClientSaslSupport()
+            .define(WORKER_SYNC_TIMEOUT_MS_CONFIG,
+                    ConfigDef.Type.INT,
+                    3000,
+                    ConfigDef.Importance.MEDIUM,
+                    WORKER_SYNC_TIMEOUT_MS_DOC)
+            .define(WORKER_UNSYNC_BACKOFF_MS_CONFIG,
+                    ConfigDef.Type.INT,
+                    WORKER_UNSYNC_BACKOFF_MS_DEFAULT,
+                    ConfigDef.Importance.MEDIUM,
+                    WORKER_UNSYNC_BACKOFF_MS_DOC)
+            .define(OFFSET_STORAGE_TOPIC_CONFIG,
+                    ConfigDef.Type.STRING,
+                    ConfigDef.Importance.HIGH,
+                    OFFSET_STORAGE_TOPIC_CONFIG_DOC)
+            .define(OFFSET_STORAGE_PARTITIONS_CONFIG,
+                    ConfigDef.Type.INT,
+                    25,
+                    atLeast(1),
+                    ConfigDef.Importance.LOW,
+                    OFFSET_STORAGE_PARTITIONS_CONFIG_DOC)
+            .define(OFFSET_STORAGE_REPLICATION_FACTOR_CONFIG,
+                    ConfigDef.Type.SHORT,
+                    (short) 3,
+                    atLeast(1),
+                    ConfigDef.Importance.LOW,
+                    OFFSET_STORAGE_REPLICATION_FACTOR_CONFIG_DOC)
+            .define(CONFIG_TOPIC_CONFIG,
+                    ConfigDef.Type.STRING,
+                    ConfigDef.Importance.HIGH,
+                    CONFIG_TOPIC_CONFIG_DOC)
+            .define(CONFIG_STORAGE_REPLICATION_FACTOR_CONFIG,
+                    ConfigDef.Type.SHORT,
+                    (short) 3,
+                    atLeast(1),
+                    ConfigDef.Importance.LOW,
+                    CONFIG_STORAGE_REPLICATION_FACTOR_CONFIG_DOC)
+            .define(STATUS_STORAGE_TOPIC_CONFIG,
+                    ConfigDef.Type.STRING,
+                    ConfigDef.Importance.HIGH,
+                    STATUS_STORAGE_TOPIC_CONFIG_DOC)
+            .define(STATUS_STORAGE_PARTITIONS_CONFIG,
+                    ConfigDef.Type.INT,
+                    5,
+                    atLeast(1),
+                    ConfigDef.Importance.LOW,
+                    STATUS_STORAGE_PARTITIONS_CONFIG_DOC)
+            .define(STATUS_STORAGE_REPLICATION_FACTOR_CONFIG,
+                    ConfigDef.Type.SHORT,
+                    (short) 3,
+                    atLeast(1),
+                    ConfigDef.Importance.LOW,
+                    STATUS_STORAGE_REPLICATION_FACTOR_CONFIG_DOC)
+            .define(CONNECT_PROTOCOL_CONFIG,
+                    ConfigDef.Type.STRING,
+                    CONNECT_PROTOCOL_DEFAULT,
+                    ConfigDef.LambdaValidator.with(
+                        (name, value) -> {
+                            try {
+                                ConnectProtocolCompatibility.compatibility((String) value);
+                            } catch (Throwable t) {
+                                throw new ConfigException(name, value, "Invalid Connect protocol "
+                                        + "compatibility");
+                            }
+                        },
+                        () -> "[" + Utils.join(ConnectProtocolCompatibility.values(), ", ") + "]"),
+                    ConfigDef.Importance.LOW,
+                    CONNECT_PROTOCOL_DOC)
+            .define(SCHEDULED_REBALANCE_MAX_DELAY_MS_CONFIG,
+                    ConfigDef.Type.INT,
+                    SCHEDULED_REBALANCE_MAX_DELAY_MS_DEFAULT,
+                    between(0, Integer.MAX_VALUE),
+                    ConfigDef.Importance.LOW,
+                    SCHEDULED_REBALANCE_MAX_DELAY_MS_DOC)
+            .define(INTER_WORKER_KEY_TTL_MS_CONFIG,
+                    ConfigDef.Type.INT,
+                    INTER_WORKER_KEY_TTL_MS_MS_DEFAULT,
+                    between(0, Integer.MAX_VALUE),
+                    ConfigDef.Importance.LOW,
+                    INTER_WORKER_KEY_TTL_MS_MS_DOC)
+            .define(INTER_WORKER_KEY_GENERATION_ALGORITHM_CONFIG,
+                    ConfigDef.Type.STRING,
+                    INTER_WORKER_KEY_GENERATION_ALGORITHM_DEFAULT,
+                    ConfigDef.LambdaValidator.with(
+                        (name, value) -> validateKeyAlgorithm(name, (String) value),
+                        () -> "Any KeyGenerator algorithm supported by the worker JVM"
+                    ),
+                    ConfigDef.Importance.LOW,
+                    INTER_WORKER_KEY_GENERATION_ALGORITHM_DOC)
+            .define(INTER_WORKER_KEY_SIZE_CONFIG,
+                    ConfigDef.Type.INT,
+                    INTER_WORKER_KEY_SIZE_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    INTER_WORKER_KEY_SIZE_DOC)
+            .define(INTER_WORKER_SIGNATURE_ALGORITHM_CONFIG,
+                    ConfigDef.Type.STRING,
+                    INTER_WORKER_SIGNATURE_ALGORITHM_DEFAULT,
+                    ConfigDef.LambdaValidator.with(
+                        (name, value) -> validateSignatureAlgorithm(name, (String) value),
+                        () -> "Any MAC algorithm supported by the worker JVM"),
+                    ConfigDef.Importance.LOW,
+                    INTER_WORKER_SIGNATURE_ALGORITHM_DOC)
+            .define(INTER_WORKER_VERIFICATION_ALGORITHMS_CONFIG,
+                    ConfigDef.Type.LIST,
+                    INTER_WORKER_VERIFICATION_ALGORITHMS_DEFAULT,
+                    ConfigDef.LambdaValidator.with(
+                        (name, value) -> validateSignatureAlgorithms(name, (List) value),
+                        () -> "A list of one or more MAC algorithms, each supported by the worker JVM"
+                    ),
+                    ConfigDef.Importance.LOW,
+                    INTER_WORKER_VERIFICATION_ALGORITHMS_DOC);
 
     @Override
     public Integer getRebalanceTimeout() {
@@ -314,9 +377,65 @@ public Integer getRebalanceTimeout() {
 
     public DistributedConfig(Map props) {
         super(CONFIG, props);
+        getInternalRequestKeyGenerator(); // Check here for a valid key size + key algorithm to fail fast if either are invalid
+        validateKeyAlgorithmAndVerificationAlgorithms();
     }
 
     public static void main(String[] args) {
         System.out.println(CONFIG.toHtml());
     }
+
+    public KeyGenerator getInternalRequestKeyGenerator() {
+        try {
+            KeyGenerator result = KeyGenerator.getInstance(getString(INTER_WORKER_KEY_GENERATION_ALGORITHM_CONFIG));
+            Optional.ofNullable(getInt(INTER_WORKER_KEY_SIZE_CONFIG)).ifPresent(result::init);
+            return result;
+        } catch (NoSuchAlgorithmException | InvalidParameterException e) {
+            throw new ConfigException(String.format(
+                "Unable to create key generator with algorithm %s and key size %d: %s",
+                getString(INTER_WORKER_KEY_GENERATION_ALGORITHM_CONFIG),
+                getInt(INTER_WORKER_KEY_SIZE_CONFIG),
+                e.getMessage()
+            ));
+        }
+    }
+
+    private void validateKeyAlgorithmAndVerificationAlgorithms() {
+        String keyAlgorithm = getString(INTER_WORKER_KEY_GENERATION_ALGORITHM_CONFIG);
+        List verificationAlgorithms = getList(INTER_WORKER_VERIFICATION_ALGORITHMS_CONFIG);
+        if (!verificationAlgorithms.contains(keyAlgorithm)) {
+            throw new ConfigException(
+                INTER_WORKER_KEY_GENERATION_ALGORITHM_CONFIG,
+                keyAlgorithm,
+                String.format("Key generation algorithm must be present in %s list", INTER_WORKER_VERIFICATION_ALGORITHMS_CONFIG)
+            );
+        }
+    }
+
+    private static void validateSignatureAlgorithms(String configName, List algorithms) {
+        if (algorithms.isEmpty()) {
+            throw new ConfigException(
+                configName,
+                algorithms,
+                "At least one signature verification algorithm must be provided"
+            );
+        }
+        algorithms.forEach(algorithm -> validateSignatureAlgorithm(configName, algorithm));
+    }
+
+    private static void validateSignatureAlgorithm(String configName, String algorithm) {
+        try {
+            Mac.getInstance(algorithm);
+        } catch (NoSuchAlgorithmException e) {
+            throw new ConfigException(configName, algorithm, e.getMessage());
+        }
+    }
+
+    private static void validateKeyAlgorithm(String configName, String algorithm) {
+        try {
+            KeyGenerator.getInstance(algorithm);
+        } catch (NoSuchAlgorithmException e) {
+            throw new ConfigException(configName, algorithm, e.getMessage());
+        }
+    }
 }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
index 86caeeb4af6f1..bce38d7a64a7b 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
@@ -42,14 +42,18 @@
 import org.apache.kafka.connect.runtime.ConnectorConfig;
 import org.apache.kafka.connect.runtime.HerderConnectorContext;
 import org.apache.kafka.connect.runtime.HerderRequest;
+import org.apache.kafka.connect.runtime.SessionKey;
 import org.apache.kafka.connect.runtime.SinkConnectorConfig;
 import org.apache.kafka.connect.runtime.SourceConnectorConfig;
 import org.apache.kafka.connect.runtime.TargetState;
 import org.apache.kafka.connect.runtime.Worker;
+import org.apache.kafka.connect.runtime.rest.InternalRequestSignature;
 import org.apache.kafka.connect.runtime.rest.RestClient;
 import org.apache.kafka.connect.runtime.rest.RestServer;
 import org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo;
 import org.apache.kafka.connect.runtime.rest.entities.TaskInfo;
+import org.apache.kafka.connect.runtime.rest.errors.BadRequestException;
+import org.apache.kafka.connect.runtime.rest.errors.ConnectRestException;
 import org.apache.kafka.connect.sink.SinkConnector;
 import org.apache.kafka.connect.storage.ConfigBackingStore;
 import org.apache.kafka.connect.storage.StatusBackingStore;
@@ -58,6 +62,9 @@
 import org.apache.kafka.connect.util.SinkUtils;
 import org.slf4j.Logger;
 
+import javax.crypto.KeyGenerator;
+import javax.crypto.SecretKey;
+import javax.ws.rs.core.Response;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -84,6 +91,7 @@
 import java.util.stream.Collectors;
 
 import static org.apache.kafka.connect.runtime.distributed.ConnectProtocol.CONNECT_PROTOCOL_V0;
+import static org.apache.kafka.connect.runtime.distributed.IncrementalCooperativeConnectProtocol.CONNECT_PROTOCOL_V2;
 
 /**
  * 
                
@@ -132,6 +140,10 @@ public class DistributedHerder extends AbstractHerder implements Runnable {
     private final int workerSyncTimeoutMs;
     private final long workerTasksShutdownTimeoutMs;
     private final int workerUnsyncBackoffMs;
+    private final int keyRotationIntervalMs;
+    private final String requestSignatureAlgorithm;
+    private final List keySignatureVerificationAlgorithms;
+    private final KeyGenerator keyGenerator;
 
     private final ExecutorService herderExecutor;
     private final ExecutorService forwardRequestExecutor;
@@ -161,6 +173,9 @@ public class DistributedHerder extends AbstractHerder implements Runnable {
     private boolean needsReconfigRebalance;
     private volatile int generation;
     private volatile long scheduledRebalance;
+    private SecretKey sessionKey;
+    private volatile long keyExpiration;
+    private short currentProtocolVersion;
 
     private final DistributedConfig config;
 
@@ -197,6 +212,10 @@ public DistributedHerder(DistributedConfig config,
         this.workerSyncTimeoutMs = config.getInt(DistributedConfig.WORKER_SYNC_TIMEOUT_MS_CONFIG);
         this.workerTasksShutdownTimeoutMs = config.getLong(DistributedConfig.TASK_SHUTDOWN_GRACEFUL_TIMEOUT_MS_CONFIG);
         this.workerUnsyncBackoffMs = config.getInt(DistributedConfig.WORKER_UNSYNC_BACKOFF_MS_CONFIG);
+        this.requestSignatureAlgorithm = config.getString(DistributedConfig.INTER_WORKER_SIGNATURE_ALGORITHM_CONFIG);
+        this.keyRotationIntervalMs = config.getInt(DistributedConfig.INTER_WORKER_KEY_TTL_MS_CONFIG);
+        this.keySignatureVerificationAlgorithms = config.getList(DistributedConfig.INTER_WORKER_VERIFICATION_ALGORITHMS_CONFIG);
+        this.keyGenerator = config.getInternalRequestKeyGenerator();
 
         String clientIdConfig = config.getString(CommonClientConfigs.CLIENT_ID_CONFIG);
         String clientId = clientIdConfig.length() <= 0 ? "connect-" + CONNECT_CLIENT_ID_SEQUENCE.getAndIncrement() : clientIdConfig;
@@ -225,6 +244,24 @@ public Thread newThread(Runnable herder) {
         needsReconfigRebalance = false;
         canReadConfigs = true; // We didn't try yet, but Configs are readable until proven otherwise
         scheduledRebalance = Long.MAX_VALUE;
+        keyExpiration = Long.MAX_VALUE;
+        sessionKey = null;
+
+        currentProtocolVersion = ConnectProtocolCompatibility.compatibility(
+            config.getString(DistributedConfig.CONNECT_PROTOCOL_CONFIG)
+        ).protocolVersion();
+        if (!internalRequestValidationEnabled(currentProtocolVersion)) {
+            log.warn(
+                "Internal request verification will be disabled for this cluster as this worker's {} configuration has been set to '{}'. "
+                    + "If this is not intentional, either remove the '{}' configuration from the worker config file or change its value "
+                    + "to '{}'. If this configuration is left as-is, the cluster will be insecure; for more information, see KIP-507: "
+                    + "https://cwiki.apache.org/confluence/display/KAFKA/KIP-507%3A+Securing+Internal+Connect+REST+Endpoints",
+                DistributedConfig.CONNECT_PROTOCOL_CONFIG,
+                config.getString(DistributedConfig.CONNECT_PROTOCOL_CONFIG),
+                DistributedConfig.CONNECT_PROTOCOL_CONFIG,
+                ConnectProtocolCompatibility.SESSIONED.name()
+            );
+        }
     }
 
     @Override
@@ -278,8 +315,17 @@ public void tick() {
             return;
         }
 
+        long now = time.milliseconds();
+
+        if (checkForKeyRotation(now)) {
+            keyExpiration = Long.MAX_VALUE;
+            configBackingStore.putSessionKey(new SessionKey(
+                keyGenerator.generateKey(),
+                now
+            ));
+        }
+
         // Process any external requests
-        final long now = time.milliseconds();
         long nextRequestTimeoutMs = Long.MAX_VALUE;
         while (true) {
             final DistributedHerderRequest next = peekWithoutException();
@@ -306,6 +352,11 @@ public void tick() {
             log.debug("Scheduled rebalance at: {} (now: {} nextRequestTimeoutMs: {}) ",
                     scheduledRebalance, now, nextRequestTimeoutMs);
         }
+        if (internalRequestValidationEnabled() && keyExpiration < Long.MAX_VALUE) {
+            nextRequestTimeoutMs = Math.min(nextRequestTimeoutMs, Math.max(keyExpiration - now, 0));
+            log.debug("Scheduled next key rotation at: {} (now: {} nextRequestTimeoutMs: {}) ",
+                    keyExpiration, now, nextRequestTimeoutMs);
+        }
 
         // Process any configuration updates
         AtomicReference> connectorConfigUpdatesCopy = new AtomicReference<>();
@@ -360,6 +411,31 @@ public void tick() {
         }
     }
 
+    private synchronized boolean checkForKeyRotation(long now) {
+        if (internalRequestValidationEnabled()) {
+            if (isLeader()) {
+                if (sessionKey == null) {
+                    log.debug("Internal request signing is enabled but no session key has been distributed yet. "
+                        + "Distributing new key now.");
+                    return true;
+                } else if (keyExpiration <= now) {
+                    log.debug("Existing key has expired. Distributing new key now.");
+                    return true;
+                } else if (!sessionKey.getAlgorithm().equals(keyGenerator.getAlgorithm())
+                        || sessionKey.getEncoded().length != keyGenerator.generateKey().getEncoded().length) {
+                    log.debug("Previously-distributed key uses different algorithm/key size "
+                        + "than required by current worker configuration. Distributing new key now.");
+                    return true;
+                }
+            } else if (sessionKey == null) {
+                // This happens on startup for follower workers; the snapshot contains the session key,
+                // but no callback in the config update listener has been fired for it yet.
+                sessionKey = configState.sessionKey().key();
+            }
+        }
+        return false;
+    }
+
     private synchronized boolean updateConfigsWithEager(AtomicReference> connectorConfigUpdatesCopy,
                                                         AtomicReference> connectorTargetStateChangesCopy) {
         // This branch is here to avoid creating a snapshot if not needed
@@ -481,7 +557,7 @@ private void processTargetStateChanges(Set connectorTargetStateChanges)
             // additionally, if the worker is running the connector itself, then we need to
             // request reconfiguration to ensure that config changes while paused take effect
             if (targetState == TargetState.STARTED)
-                reconfigureConnectorTasksWithRetry(connector);
+                reconfigureConnectorTasksWithRetry(time.milliseconds(), connector);
         }
     }
 
@@ -706,7 +782,7 @@ public void requestTaskReconfiguration(final String connName) {
                 new Callable() {
                     @Override
                     public Void call() throws Exception {
-                        reconfigureConnectorTasksWithRetry(connName);
+                        reconfigureConnectorTasksWithRetry(time.milliseconds(), connName);
                         return null;
                     }
                 },
@@ -751,8 +827,36 @@ public Void call() throws Exception {
     }
 
     @Override
-    public void putTaskConfigs(final String connName, final List> configs, final Callback callback) {
+    public void putTaskConfigs(final String connName, final List> configs, final Callback callback, InternalRequestSignature requestSignature) {
         log.trace("Submitting put task configuration request {}", connName);
+        if (internalRequestValidationEnabled()) {
+            ConnectRestException requestValidationError = null;
+            if (requestSignature == null) {
+                requestValidationError = new BadRequestException("Internal request missing required signature");
+            } else if (!keySignatureVerificationAlgorithms.contains(requestSignature.keyAlgorithm())) {
+                requestValidationError = new BadRequestException(String.format(
+                    "This worker does not support the '%s' key signing algorithm used by other workers. " 
+                        + "This worker is currently configured to use: %s. " 
+                        + "Check that all workers' configuration files permit the same set of signature algorithms, " 
+                        + "and correct any misconfigured worker and restart it.",
+                    requestSignature.keyAlgorithm(),
+                    keySignatureVerificationAlgorithms
+                ));
+            } else {
+                synchronized (this) {
+                    if (!requestSignature.isValid(sessionKey)) {
+                        requestValidationError = new ConnectRestException(
+                            Response.Status.FORBIDDEN,
+                            "Internal request contained invalid signature."
+                        );
+                    }
+                }
+            }
+            if (requestValidationError != null) {
+                callback.onCompletion(requestValidationError, null);
+                return;
+            }
+        }
 
         addRequest(
                 new Callable() {
@@ -1082,7 +1186,7 @@ private boolean startConnector(String connectorName) {
         // task configs if they are actually different from the existing ones to avoid unnecessary updates when this is
         // just restoring an existing connector.
         if (started && initialState == TargetState.STARTED)
-            reconfigureConnectorTasksWithRetry(connectorName);
+            reconfigureConnectorTasksWithRetry(time.milliseconds(), connectorName);
 
         return started;
     }
@@ -1117,7 +1221,7 @@ public Void call() throws Exception {
         };
     }
 
-    private void reconfigureConnectorTasksWithRetry(final String connName) {
+    private void reconfigureConnectorTasksWithRetry(long initialRequestTime, final String connName) {
         reconfigureConnector(connName, new Callback() {
             @Override
             public void onCompletion(Throwable error, Void result) {
@@ -1126,12 +1230,16 @@ public void onCompletion(Throwable error, Void result) {
                 // never makes progress. The retry has to run through a DistributedHerderRequest since this callback could be happening
                 // from the HTTP request forwarding thread.
                 if (error != null) {
-                    log.error("Failed to reconfigure connector's tasks, retrying after backoff:", error);
+                    if (isPossibleExpiredKeyException(initialRequestTime, error)) {
+                        log.debug("Failed to reconfigure connector's tasks, possibly due to expired session key. Retrying after backoff");
+                    } else {
+                        log.error("Failed to reconfigure connector's tasks, retrying after backoff:", error);
+                    }
                     addRequest(RECONFIGURE_CONNECTOR_TASKS_BACKOFF_MS,
                             new Callable() {
                                 @Override
                                 public Void call() throws Exception {
-                                    reconfigureConnectorTasksWithRetry(connName);
+                                    reconfigureConnectorTasksWithRetry(initialRequestTime, connName);
                                     return null;
                                 }
                             }, new Callback() {
@@ -1147,6 +1255,15 @@ public void onCompletion(Throwable error, Void result) {
         });
     }
 
+    boolean isPossibleExpiredKeyException(long initialRequestTime, Throwable error) {
+        if (error instanceof ConnectRestException) {
+            ConnectRestException connectError = (ConnectRestException) error;
+            return connectError.statusCode() == Response.Status.FORBIDDEN.getStatusCode()
+                && initialRequestTime + TimeUnit.MINUTES.toMillis(1) >= time.milliseconds();
+        }
+        return false;
+    }
+
     // Updates configurations for a connector by requesting them from the connector, filling in parameters provided
     // by the system, then checks whether any configs have actually changed before submitting the new configs to storage
     private void reconfigureConnector(final String connName, final Callback cb) {
@@ -1202,7 +1319,8 @@ public void run() {
                                     return;
                                 }
                                 String reconfigUrl = RestServer.urlJoin(leaderUrl, "/connectors/" + connName + "/tasks");
-                                RestClient.httpRequest(reconfigUrl, "POST", null, rawTaskProps, null, config);
+                                log.trace("Forwarding task configurations for connector {} to leader", connName);
+                                RestClient.httpRequest(reconfigUrl, "POST", null, rawTaskProps, null, config, sessionKey, requestSignatureAlgorithm);
                                 cb.onCompletion(null, null);
                             } catch (ConnectException e) {
                                 log.error("Request to leader to reconfigure connector tasks failed", e);
@@ -1239,6 +1357,14 @@ DistributedHerderRequest addRequest(long delayMs, Callable action, Callbac
         return req;
     }
 
+    private boolean internalRequestValidationEnabled() {
+        return internalRequestValidationEnabled(member.currentProtocolVersion());
+    }
+
+    private static boolean internalRequestValidationEnabled(short protocolVersion) {
+        return protocolVersion >= CONNECT_PROTOCOL_V2;
+    }
+
     private DistributedHerderRequest peekWithoutException() {
         try {
             return requests.isEmpty() ? null : requests.first();
@@ -1306,6 +1432,21 @@ public void onConnectorTargetStateChange(String connector) {
             }
             member.wakeup();
         }
+
+        @Override
+        public void onSessionKeyUpdate(SessionKey sessionKey) {
+            log.info("Session key updated");
+
+            synchronized (DistributedHerder.this) {
+                DistributedHerder.this.sessionKey = sessionKey.key();
+                // Track the expiration of the key if and only if this worker is the leader
+                // Followers will receive rotated keys from the follower and won't be responsible for
+                // tracking expiration and distributing new keys themselves
+                if (isLeader() && keyRotationIntervalMs > 0) {
+                    DistributedHerder.this.keyExpiration = sessionKey.creationTimestamp() + keyRotationIntervalMs;
+                }
+            }
+        }
     }
 
     class DistributedHerderRequest implements HerderRequest, Comparable {
@@ -1394,14 +1535,45 @@ public void onAssigned(ExtendedAssignment assignment, int generation) {
             // catch up (or backoff if we fail) not executed in a callback, and so we'll be able to invoke other
             // group membership actions (e.g., we may need to explicitly leave the group if we cannot handle the
             // assigned tasks).
-            log.info("Joined group at generation {} and got assignment: {}", generation, assignment);
+            short priorProtocolVersion = currentProtocolVersion;
+            DistributedHerder.this.currentProtocolVersion = member.currentProtocolVersion();
+            log.info(
+                "Joined group at generation {} with protocol version {} and got assignment: {}",
+                generation,
+                DistributedHerder.this.currentProtocolVersion,
+                assignment
+            );
             synchronized (DistributedHerder.this) {
                 DistributedHerder.this.assignment = assignment;
                 DistributedHerder.this.generation = generation;
                 int delay = assignment.delay();
                 DistributedHerder.this.scheduledRebalance = delay > 0
-                                                            ? time.milliseconds() + delay
-                                                            : Long.MAX_VALUE;
+                    ? time.milliseconds() + delay
+                    : Long.MAX_VALUE;
+
+                boolean requestValidationWasEnabled = internalRequestValidationEnabled(priorProtocolVersion);
+                boolean requestValidationNowEnabled = internalRequestValidationEnabled(currentProtocolVersion);
+                if (requestValidationNowEnabled != requestValidationWasEnabled) {
+                    // Internal request verification has been switched on or off; let the user know
+                    if (requestValidationNowEnabled) {
+                        log.info("Internal request validation has been re-enabled");
+                    } else {
+                        log.warn(
+                            "The protocol used by this Connect cluster has been downgraded from '{}' to '{}' and internal request "
+                                + "validation is now disabled. This is most likely caused by a new worker joining the cluster with an "
+                                + "older protocol specified for the {} configuration; if this is not intentional, either remove the {} "
+                                + "configuration from that worker's config file, or change its value to '{}'. If this configuration is "
+                                + "left as-is, the cluster will be insecure; for more information, see KIP-507: "
+                                + "https://cwiki.apache.org/confluence/display/KAFKA/KIP-507%3A+Securing+Internal+Connect+REST+Endpoints",
+                            ConnectProtocolCompatibility.fromProtocolVersion(priorProtocolVersion),
+                            ConnectProtocolCompatibility.fromProtocolVersion(DistributedHerder.this.currentProtocolVersion),
+                            DistributedConfig.CONNECT_PROTOCOL_CONFIG,
+                            DistributedConfig.CONNECT_PROTOCOL_CONFIG,
+                            ConnectProtocolCompatibility.SESSIONED.name()
+                        );
+                    }
+                }
+
                 rebalanceResolved = false;
                 herderMetrics.rebalanceStarted(time.milliseconds());
             }
@@ -1463,6 +1635,12 @@ public HerderMetrics(ConnectMetrics connectMetrics) {
             ConnectMetricsRegistry registry = connectMetrics.registry();
             metricGroup = connectMetrics.group(registry.workerRebalanceGroupName());
 
+            metricGroup.addValueMetric(registry.connectProtocol, new LiteralSupplier() {
+                @Override
+                public String metricValue(long now) {
+                    return ConnectProtocolCompatibility.fromProtocolVersion(member.currentProtocolVersion()).name();
+                }
+            });
             metricGroup.addValueMetric(registry.leaderName, new LiteralSupplier() {
                 @Override
                 public String metricValue(long now) {
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java
index 8e56ca8120f7e..91e1f7c8a73e9 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java
@@ -44,6 +44,7 @@
 import static org.apache.kafka.common.message.JoinGroupResponseData.JoinGroupResponseMember;
 import static org.apache.kafka.connect.runtime.distributed.ConnectProtocol.Assignment;
 import static org.apache.kafka.connect.runtime.distributed.IncrementalCooperativeConnectProtocol.CONNECT_PROTOCOL_V1;
+import static org.apache.kafka.connect.runtime.distributed.IncrementalCooperativeConnectProtocol.CONNECT_PROTOCOL_V2;
 import static org.apache.kafka.connect.runtime.distributed.WorkerCoordinator.LeaderState;
 
 /**
@@ -99,15 +100,20 @@ public Map performAssignment(String leaderId, String protoco
         log.debug("Max config offset root: {}, local snapshot config offsets root: {}",
                   maxOffset, coordinator.configSnapshot().offset());
 
+        short protocolVersion = memberConfigs.values().stream()
+            .allMatch(state -> state.assignment().version() == CONNECT_PROTOCOL_V2)
+                ? CONNECT_PROTOCOL_V2
+                : CONNECT_PROTOCOL_V1;
+
         Long leaderOffset = ensureLeaderConfig(maxOffset, coordinator);
         if (leaderOffset == null) {
             Map assignments = fillAssignments(
                     memberConfigs.keySet(), Assignment.CONFIG_MISMATCH,
                     leaderId, memberConfigs.get(leaderId).url(), maxOffset, Collections.emptyMap(),
-                    Collections.emptyMap(), Collections.emptyMap(), 0);
+                    Collections.emptyMap(), Collections.emptyMap(), 0, protocolVersion);
             return serializeAssignments(assignments);
         }
-        return performTaskAssignment(leaderId, leaderOffset, memberConfigs, coordinator);
+        return performTaskAssignment(leaderId, leaderOffset, memberConfigs, coordinator, protocolVersion);
     }
 
     private Long ensureLeaderConfig(long maxOffset, WorkerCoordinator coordinator) {
@@ -140,12 +146,13 @@ private Long ensureLeaderConfig(long maxOffset, WorkerCoordinator coordinator) {
      * round of rebalancing
      * @param coordinator the worker coordinator instance that provide the configuration snapshot
      * and get assigned the leader state during this assignment
+     * @param protocolVersion the Connect subprotocol version
      * @return the serialized assignment of tasks to the whole group, including assigned or
      * revoked tasks
      */
     protected Map performTaskAssignment(String leaderId, long maxOffset,
                                                             Map memberConfigs,
-                                                            WorkerCoordinator coordinator) {
+                                                            WorkerCoordinator coordinator, short protocolVersion) {
         // Base set: The previous assignment of connectors-and-tasks is a standalone snapshot that
         // can be used to calculate derived sets
         log.debug("Previous assignments: {}", previousAssignment);
@@ -281,7 +288,7 @@ protected Map performTaskAssignment(String leaderId, long ma
         Map assignments =
                 fillAssignments(memberConfigs.keySet(), Assignment.NO_ERROR, leaderId,
                                 memberConfigs.get(leaderId).url(), maxOffset, incrementalConnectorAssignments,
-                                incrementalTaskAssignments, toRevoke, delay);
+                                incrementalTaskAssignments, toRevoke, delay, protocolVersion);
 
         previousAssignment = computePreviousAssignment(toRevoke, connectorAssignments, taskAssignments, lostAssignments);
 
@@ -491,7 +498,7 @@ private Map fillAssignments(Collection membe
                                                             Map> connectorAssignments,
                                                             Map> taskAssignments,
                                                             Map revoked,
-                                                            int delay) {
+                                                            int delay, short protocolVersion) {
         Map groupAssignment = new HashMap<>();
         for (String member : members) {
             Collection connectorsToStart = connectorAssignments.getOrDefault(member, Collections.emptyList());
@@ -499,7 +506,7 @@ private Map fillAssignments(Collection membe
             Collection connectorsToStop = revoked.getOrDefault(member, ConnectorsAndTasks.EMPTY).connectors();
             Collection tasksToStop = revoked.getOrDefault(member, ConnectorsAndTasks.EMPTY).tasks();
             ExtendedAssignment assignment =
-                    new ExtendedAssignment(CONNECT_PROTOCOL_V1, error, leaderId, leaderUrl, maxOffset,
+                    new ExtendedAssignment(protocolVersion, error, leaderId, leaderUrl, maxOffset,
                             connectorsToStart, tasksToStart, connectorsToStop, tasksToStop, delay);
             log.debug("Filling assignment: {} -> {}", member, assignment);
             groupAssignment.put(member, assignment);
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeConnectProtocol.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeConnectProtocol.java
index 4e3c736ead27d..6bcf9be65eb64 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeConnectProtocol.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeConnectProtocol.java
@@ -24,7 +24,8 @@
 import org.apache.kafka.common.protocol.types.Type;
 
 import java.nio.ByteBuffer;
-import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.List;
 
 import static org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocol;
 import static org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection;
@@ -42,6 +43,7 @@
 import static org.apache.kafka.connect.runtime.distributed.ConnectProtocol.VERSION_KEY_NAME;
 import static org.apache.kafka.connect.runtime.distributed.ConnectProtocolCompatibility.COMPATIBLE;
 import static org.apache.kafka.connect.runtime.distributed.ConnectProtocolCompatibility.EAGER;
+import static org.apache.kafka.connect.runtime.distributed.ConnectProtocolCompatibility.SESSIONED;
 
 
 /**
@@ -55,6 +57,7 @@ public class IncrementalCooperativeConnectProtocol {
     public static final String REVOKED_KEY_NAME = "revoked";
     public static final String SCHEDULED_DELAY_KEY_NAME = "delay";
     public static final short CONNECT_PROTOCOL_V1 = 1;
+    public static final short CONNECT_PROTOCOL_V2 = 2;
     public static final boolean TOLERATE_MISSING_FIELDS_WITH_DEFAULTS = true;
 
     /**
@@ -66,6 +69,19 @@ public class IncrementalCooperativeConnectProtocol {
     private static final Struct CONNECT_PROTOCOL_HEADER_V1 = new Struct(CONNECT_PROTOCOL_HEADER_SCHEMA)
             .set(VERSION_KEY_NAME, CONNECT_PROTOCOL_V1);
 
+    /**
+     * Connect Protocol Header V2:
+     * 
                +     *   Version            => Int16
+     * 
                
+     * The V2 protocol is schematically identical to V1, but is used to signify that internal request
+     * verification and distribution of session keys is enabled (for more information, see KIP-507:
+     * https://cwiki.apache.org/confluence/display/KAFKA/KIP-507%3A+Securing+Internal+Connect+REST+Endpoints)
+     */
+    private static final Struct CONNECT_PROTOCOL_HEADER_V2 = new Struct(CONNECT_PROTOCOL_HEADER_SCHEMA)
+        .set(VERSION_KEY_NAME, CONNECT_PROTOCOL_V2);
+
+
     /**
      * Config State V1:
      * 
                @@ -132,17 +148,18 @@ public class IncrementalCooperativeConnectProtocol {
      *   Current Assignment => [Byte]
      * 
                
      */
-    public static ByteBuffer serializeMetadata(ExtendedWorkerState workerState) {
+    public static ByteBuffer serializeMetadata(ExtendedWorkerState workerState, boolean sessioned) {
         Struct configState = new Struct(CONFIG_STATE_V1)
                 .set(URL_KEY_NAME, workerState.url())
                 .set(CONFIG_OFFSET_KEY_NAME, workerState.offset());
         // Not a big issue if we embed the protocol version with the assignment in the metadata
         Struct allocation = new Struct(ALLOCATION_V1)
                 .set(ALLOCATION_KEY_NAME, serializeAssignment(workerState.assignment()));
-        ByteBuffer buffer = ByteBuffer.allocate(CONNECT_PROTOCOL_HEADER_V1.sizeOf()
+        Struct connectProtocolHeader = sessioned ? CONNECT_PROTOCOL_HEADER_V2 : CONNECT_PROTOCOL_HEADER_V1;
+        ByteBuffer buffer = ByteBuffer.allocate(connectProtocolHeader.sizeOf()
                                                 + CONFIG_STATE_V1.sizeOf(configState)
                                                 + ALLOCATION_V1.sizeOf(allocation));
-        CONNECT_PROTOCOL_HEADER_V1.writeTo(buffer);
+        connectProtocolHeader.writeTo(buffer);
         CONFIG_STATE_V1.write(buffer, configState);
         ALLOCATION_V1.write(buffer, allocation);
         buffer.flip();
@@ -154,18 +171,28 @@ public static ByteBuffer serializeMetadata(ExtendedWorkerState workerState) {
      * with their serialized metadata. The protocols are ordered by preference.
      *
      * @param workerState the current state of the worker metadata
+     * @param sessioned whether the {@link ConnectProtocolCompatibility#SESSIONED} protocol should
+     *                  be included in the collection of supported protocols
      * @return the collection of Connect protocol metadata
      */
-    public static JoinGroupRequestProtocolCollection metadataRequest(ExtendedWorkerState workerState) {
+    public static JoinGroupRequestProtocolCollection metadataRequest(ExtendedWorkerState workerState, boolean sessioned) {
         // Order matters in terms of protocol preference
-        return new JoinGroupRequestProtocolCollection(Arrays.asList(
-                new JoinGroupRequestProtocol()
+        List joinGroupRequestProtocols = new ArrayList<>();
+        if (sessioned) {
+            joinGroupRequestProtocols.add(new JoinGroupRequestProtocol()
+                .setName(SESSIONED.protocol())
+                .setMetadata(IncrementalCooperativeConnectProtocol.serializeMetadata(workerState, true).array())
+            );
+        }
+        joinGroupRequestProtocols.add(new JoinGroupRequestProtocol()
                         .setName(COMPATIBLE.protocol())
-                        .setMetadata(IncrementalCooperativeConnectProtocol.serializeMetadata(workerState).array()),
-                new JoinGroupRequestProtocol()
+                        .setMetadata(IncrementalCooperativeConnectProtocol.serializeMetadata(workerState, false).array())
+        );
+        joinGroupRequestProtocols.add(new JoinGroupRequestProtocol()
                         .setName(EAGER.protocol())
-                        .setMetadata(ConnectProtocol.serializeMetadata(workerState).array()))
-                .iterator());
+                        .setMetadata(ConnectProtocol.serializeMetadata(workerState).array())
+        );
+        return new JoinGroupRequestProtocolCollection(joinGroupRequestProtocols.iterator());
     }
 
     /**
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java
index 0b855a01e2dc2..ff47e36862ae6 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java
@@ -157,7 +157,9 @@ public JoinGroupRequestProtocolCollection metadata() {
             case EAGER:
                 return ConnectProtocol.metadataRequest(workerState);
             case COMPATIBLE:
-                return IncrementalCooperativeConnectProtocol.metadataRequest(workerState);
+                return IncrementalCooperativeConnectProtocol.metadataRequest(workerState, false);
+            case SESSIONED:
+                return IncrementalCooperativeConnectProtocol.metadataRequest(workerState, true);
             default:
                 throw new IllegalStateException("Unknown Connect protocol compatibility mode " + protocolCompatibility);
         }
@@ -292,7 +294,7 @@ public void leaderState(LeaderState update) {
      * @return the current connect protocol version
      */
     public short currentProtocolVersion() {
-        return currentConnectProtocol == EAGER ? (short) 0 : (short) 1;
+        return currentConnectProtocol.protocolVersion();
     }
 
     private class WorkerCoordinatorMetrics {
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/InternalRequestSignature.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/InternalRequestSignature.java
new file mode 100644
index 0000000000000..d59425b13f6e2
--- /dev/null
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/InternalRequestSignature.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.runtime.rest;
+
+import org.apache.kafka.connect.errors.ConnectException;
+import org.apache.kafka.connect.runtime.rest.errors.BadRequestException;
+import org.eclipse.jetty.client.api.Request;
+
+import javax.crypto.Mac;
+import javax.crypto.SecretKey;
+import javax.ws.rs.core.HttpHeaders;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Objects;
+
+public class InternalRequestSignature {
+
+    public static final String SIGNATURE_HEADER = "X-Connect-Authorization";
+    public static final String SIGNATURE_ALGORITHM_HEADER = "X-Connect-Request-Signature-Algorithm";
+
+    private final byte[] requestBody;
+    private final Mac mac;
+    private final byte[] requestSignature;
+
+    /**
+     * Add a signature to a request.
+     * @param key the key to sign the request with; may not be null
+     * @param requestBody the body of the request; may not be null
+     * @param signatureAlgorithm the algorithm to use to sign the request; may not be null
+     * @param request the request to add the signature to; may not be null
+     */
+    public static void addToRequest(SecretKey key, byte[] requestBody, String signatureAlgorithm, Request request) {
+        Mac mac;
+        try {
+            mac = mac(signatureAlgorithm);
+        }  catch (NoSuchAlgorithmException e) {
+            throw new ConnectException(e);
+        }
+        byte[] requestSignature = sign(mac, key, requestBody);
+        request.header(InternalRequestSignature.SIGNATURE_HEADER, Base64.getEncoder().encodeToString(requestSignature))
+               .header(InternalRequestSignature.SIGNATURE_ALGORITHM_HEADER, signatureAlgorithm);
+    }
+
+    /**
+     * Extract a signature from a request.
+     * @param requestBody the body of the request; may not be null
+     * @param headers the headers for the request; may be null
+     * @return the signature extracted from the request, or null if one or more request signature
+     * headers was not present
+     */
+    public static InternalRequestSignature fromHeaders(byte[] requestBody, HttpHeaders headers) {
+        if (headers == null) {
+            return null;
+        }
+
+        String signatureAlgorithm = headers.getHeaderString(SIGNATURE_ALGORITHM_HEADER);
+        String encodedSignature = headers.getHeaderString(SIGNATURE_HEADER);
+        if (signatureAlgorithm == null || encodedSignature == null) {
+            return null;
+        }
+
+        Mac mac;
+        try {
+            mac = mac(signatureAlgorithm);
+        } catch (NoSuchAlgorithmException e) {
+            throw new BadRequestException(e.getMessage());
+        }
+
+        byte[] decodedSignature;
+        try {
+            decodedSignature = Base64.getDecoder().decode(encodedSignature);
+        } catch (IllegalArgumentException e) {
+            throw new BadRequestException(e.getMessage());
+        }
+
+        return new InternalRequestSignature(
+            requestBody,
+            mac,
+            decodedSignature
+        );
+    }
+
+    // Public for testing
+    public InternalRequestSignature(byte[] requestBody, Mac mac, byte[] requestSignature) {
+        this.requestBody = requestBody;
+        this.mac = mac;
+        this.requestSignature = requestSignature;
+    }
+
+    public String keyAlgorithm() {
+        return mac.getAlgorithm();
+    }
+
+    public boolean isValid(SecretKey key) {
+        return Arrays.equals(sign(mac, key, requestBody), requestSignature);
+    }
+
+    private static Mac mac(String signatureAlgorithm) throws NoSuchAlgorithmException {
+        return Mac.getInstance(signatureAlgorithm);
+    }
+
+    private static byte[] sign(Mac mac, SecretKey key, byte[] requestBody) {
+        try {
+            mac.init(key);
+        } catch (InvalidKeyException e) {
+            throw new ConnectException(e);
+        }
+        return mac.doFinal(requestBody);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        InternalRequestSignature that = (InternalRequestSignature) o;
+        return Arrays.equals(requestBody, that.requestBody)
+            && mac.getAlgorithm().equals(that.mac.getAlgorithm())
+            && mac.getMacLength() == that.mac.getMacLength()
+            && mac.getProvider().equals(that.mac.getProvider())
+            && Arrays.equals(requestSignature, that.requestSignature);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = Objects.hash(mac);
+        result = 31 * result + Arrays.hashCode(requestBody);
+        result = 31 * result + Arrays.hashCode(requestSignature);
+        return result;
+    }
+}
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestClient.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestClient.java
index 7ef1c78cf0031..b38125971656c 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestClient.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestClient.java
@@ -19,7 +19,10 @@
 
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
+
+import javax.crypto.SecretKey;
 import javax.ws.rs.core.HttpHeaders;
+
 import org.apache.kafka.connect.runtime.WorkerConfig;
 import org.apache.kafka.connect.runtime.rest.entities.ErrorMessage;
 import org.apache.kafka.connect.runtime.rest.errors.ConnectRestException;
@@ -59,6 +62,27 @@ public class RestClient {
      */
     public static  HttpResponse httpRequest(String url, String method, HttpHeaders headers, Object requestBodyData,
                                                   TypeReference responseFormat, WorkerConfig config) {
+        return httpRequest(url, method, headers, requestBodyData, responseFormat, config, null, null);
+    }
+
+    /**
+     * Sends HTTP request to remote REST server
+     *
+     * @param url                       HTTP connection will be established with this url.
+     * @param method                    HTTP method ("GET", "POST", "PUT", etc.)
+     * @param headers                   HTTP headers from REST endpoint
+     * @param requestBodyData           Object to serialize as JSON and send in the request body.
+     * @param responseFormat            Expected format of the response to the HTTP request.
+     * @param                        The type of the deserialized response to the HTTP request.
+     * @param sessionKey                The key to sign the request with (intended for internal requests only);
+     *                                  may be null if the request doesn't need to be signed
+     * @param requestSignatureAlgorithm The algorithm to sign the request with (intended for internal requests only);
+     *                                  may be null if the request doesn't need to be signed
+     * @return The deserialized response to the HTTP request, or null if no data is expected.
+     */
+    public static  HttpResponse httpRequest(String url, String method, HttpHeaders headers, Object requestBodyData,
+                                                  TypeReference responseFormat, WorkerConfig config,
+                                                  SecretKey sessionKey, String requestSignatureAlgorithm) {
         HttpClient client;
 
         if (url.startsWith("https://")) {
@@ -88,6 +112,14 @@ public static  HttpResponse httpRequest(String url, String method, HttpHea
 
             if (serializedBody != null) {
                 req.content(new StringContentProvider(serializedBody, StandardCharsets.UTF_8), "application/json");
+                if (sessionKey != null && requestSignatureAlgorithm != null) {
+                    InternalRequestSignature.addToRequest(
+                        sessionKey,
+                        serializedBody.getBytes(StandardCharsets.UTF_8),
+                        requestSignatureAlgorithm,
+                        req
+                    );
+                }
             }
 
             ContentResponse res = req.send();
@@ -111,12 +143,11 @@ public static  HttpResponse httpRequest(String url, String method, HttpHea
             log.error("IO error forwarding REST request: ", e);
             throw new ConnectRestException(Response.Status.INTERNAL_SERVER_ERROR, "IO Error trying to forward REST request: " + e.getMessage(), e);
         } finally {
-            if (client != null)
-                try {
-                    client.stop();
-                } catch (Exception e) {
-                    log.error("Failed to stop HTTP client", e);
-                }
+            try {
+                client.stop();
+            } catch (Exception e) {
+                log.error("Failed to stop HTTP client", e);
+            }
         }
     }
 
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResource.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResource.java
index 96a4f51eec312..3b41fb7c143a2 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResource.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResource.java
@@ -19,12 +19,15 @@
 import com.fasterxml.jackson.core.type.TypeReference;
 
 import javax.ws.rs.core.HttpHeaders;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.kafka.connect.errors.NotFoundException;
 import org.apache.kafka.connect.runtime.ConnectorConfig;
 import org.apache.kafka.connect.runtime.Herder;
 import org.apache.kafka.connect.runtime.WorkerConfig;
 import org.apache.kafka.connect.runtime.distributed.RebalanceNeededException;
 import org.apache.kafka.connect.runtime.distributed.RequestTargetException;
+import org.apache.kafka.connect.runtime.rest.InternalRequestSignature;
 import org.apache.kafka.connect.runtime.rest.RestClient;
 import org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo;
 import org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo;
@@ -66,6 +69,8 @@
 @Consumes(MediaType.APPLICATION_JSON)
 public class ConnectorsResource {
     private static final Logger log = LoggerFactory.getLogger(ConnectorsResource.class);
+    private static final TypeReference>> TASK_CONFIGS_TYPE =
+        new TypeReference>>() { };
 
     // TODO: This should not be so long. However, due to potentially long rebalances that may have to wait a full
     // session timeout to complete, during which we cannot serve some requests. Ideally we could reduce this, but
@@ -230,9 +235,10 @@ public List getTaskConfigs(final @PathParam("connector") String connec
     public void putTaskConfigs(final @PathParam("connector") String connector,
                                final @Context HttpHeaders headers,
                                final @QueryParam("forward") Boolean forward,
-                               final List> taskConfigs) throws Throwable {
+                               final byte[] requestBody) throws Throwable {
+        List> taskConfigs = new ObjectMapper().readValue(requestBody, TASK_CONFIGS_TYPE);
         FutureCallback cb = new FutureCallback<>();
-        herder.putTaskConfigs(connector, taskConfigs, cb);
+        herder.putTaskConfigs(connector, taskConfigs, cb, InternalRequestSignature.fromHeaders(requestBody, headers));
         completeOrForwardRequest(cb, "/connectors/" + connector + "/tasks", "POST", headers, taskConfigs, forward);
     }
 
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java
index 00bda92574477..a3e75b5829b5f 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java
@@ -24,11 +24,13 @@
 import org.apache.kafka.connect.runtime.ConnectorConfig;
 import org.apache.kafka.connect.runtime.HerderConnectorContext;
 import org.apache.kafka.connect.runtime.HerderRequest;
+import org.apache.kafka.connect.runtime.SessionKey;
 import org.apache.kafka.connect.runtime.SinkConnectorConfig;
 import org.apache.kafka.connect.runtime.SourceConnectorConfig;
 import org.apache.kafka.connect.runtime.TargetState;
 import org.apache.kafka.connect.runtime.Worker;
 import org.apache.kafka.connect.runtime.distributed.ClusterConfigState;
+import org.apache.kafka.connect.runtime.rest.InternalRequestSignature;
 import org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo;
 import org.apache.kafka.connect.runtime.rest.entities.TaskInfo;
 import org.apache.kafka.connect.storage.ConfigBackingStore;
@@ -241,7 +243,7 @@ public synchronized void taskConfigs(String connName, Callback> c
     }
 
     @Override
-    public void putTaskConfigs(String connName, List> configs, Callback callback) {
+    public void putTaskConfigs(String connName, List> configs, Callback callback, InternalRequestSignature requestSignature) {
         throw new UnsupportedOperationException("Kafka Connect in standalone mode does not support externally setting task configurations.");
     }
 
@@ -379,6 +381,11 @@ public void onConnectorTargetStateChange(String connector) {
                     updateConnectorTasks(connector);
             }
         }
+
+        @Override
+        public void onSessionKeyUpdate(SessionKey sessionKey) {
+            // no-op
+        }
     }
 
     static class StandaloneHerderRequest implements HerderRequest {
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/ConfigBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/ConfigBackingStore.java
index b8fd64380cdb7..f8a6f70fd2a7f 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/ConfigBackingStore.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/ConfigBackingStore.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.connect.storage;
 
+import org.apache.kafka.connect.runtime.SessionKey;
 import org.apache.kafka.connect.runtime.TargetState;
 import org.apache.kafka.connect.runtime.distributed.ClusterConfigState;
 import org.apache.kafka.connect.util.ConnectorTaskId;
@@ -88,6 +89,8 @@ public interface ConfigBackingStore {
      */
     void putTargetState(String connector, TargetState state);
 
+    void putSessionKey(SessionKey sessionKey);
+
     /**
      * Set an update listener to get notifications when there are config/target state
      * changes.
@@ -119,6 +122,12 @@ interface UpdateListener {
          * @param connector name of the connector
          */
         void onConnectorTargetStateChange(String connector);
+
+        /**
+         * Invoked when the leader has distributed a new session key
+         * @param sessionKey the {@link SessionKey session key}
+         */
+        void onSessionKeyUpdate(SessionKey sessionKey);
     }
 
 }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java
index 3572d8cf8c5c9..1227da55f9f43 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java
@@ -33,6 +33,7 @@
 import org.apache.kafka.connect.data.Struct;
 import org.apache.kafka.connect.errors.ConnectException;
 import org.apache.kafka.connect.errors.DataException;
+import org.apache.kafka.connect.runtime.SessionKey;
 import org.apache.kafka.connect.runtime.TargetState;
 import org.apache.kafka.connect.runtime.WorkerConfig;
 import org.apache.kafka.connect.runtime.WorkerConfigTransformer;
@@ -45,8 +46,10 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.crypto.spec.SecretKeySpec;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Base64;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -176,6 +179,8 @@ public static String COMMIT_TASKS_KEY(String connectorName) {
         return COMMIT_TASKS_PREFIX + connectorName;
     }
 
+    public static final String SESSION_KEY_KEY = "session-key";
+
     // Note that while using real serialization for values as we have here, but ad hoc string serialization for keys,
     // isn't ideal, we use this approach because it avoids any potential problems with schema evolution or
     // converter/serializer changes causing keys to change. We need to absolutely ensure that the keys remain precisely
@@ -190,6 +195,13 @@ public static String COMMIT_TASKS_KEY(String connectorName) {
     public static final Schema TARGET_STATE_V0 = SchemaBuilder.struct()
             .field("state", Schema.STRING_SCHEMA)
             .build();
+    // The key is logically a byte array, but we can't use the JSON converter to (de-)serialize that without a schema.
+    // So instead, we base 64-encode it before serializing and decode it after deserializing.
+    public static final Schema SESSION_KEY_V0 = SchemaBuilder.struct()
+            .field("key", Schema.STRING_SCHEMA)
+            .field("algorithm", Schema.STRING_SCHEMA)
+            .field("creation-timestamp", Schema.INT64_SCHEMA)
+            .build();
 
     private static final long READ_TO_END_TIMEOUT_MS = 30000;
 
@@ -216,6 +228,8 @@ public static String COMMIT_TASKS_KEY(String connectorName) {
     // The most recently read offset. This does not take into account deferred task updates/commits, so we may have
     // outstanding data to be applied.
     private volatile long offset;
+    // The most recently read session key, to use for validating internal REST requests.
+    private volatile SessionKey sessionKey;
 
     // Connector -> Map[ConnectorTaskId -> Configs]
     private final Map>> deferredTaskUpdates = new HashMap<>();
@@ -270,6 +284,7 @@ public ClusterConfigState snapshot() {
             // immutable configs
             return new ClusterConfigState(
                     offset,
+                    sessionKey,
                     new HashMap<>(connectorTaskCounts),
                     new HashMap<>(connectorConfigs),
                     new HashMap<>(connectorTargetStates),
@@ -409,6 +424,23 @@ public void putTargetState(String connector, TargetState state) {
         configLog.send(TARGET_STATE_KEY(connector), serializedTargetState);
     }
 
+    @Override
+    public void putSessionKey(SessionKey sessionKey) {
+        log.debug("Distributing new session key");
+        Struct sessionKeyStruct = new Struct(SESSION_KEY_V0);
+        sessionKeyStruct.put("key", Base64.getEncoder().encodeToString(sessionKey.key().getEncoded()));
+        sessionKeyStruct.put("algorithm", sessionKey.key().getAlgorithm());
+        sessionKeyStruct.put("creation-timestamp", sessionKey.creationTimestamp());
+        byte[] serializedSessionKey = converter.fromConnectData(topic, SESSION_KEY_V0, sessionKeyStruct);
+        try {
+            configLog.send(SESSION_KEY_KEY, serializedSessionKey);
+            configLog.readToEnd().get(READ_TO_END_TIMEOUT_MS, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException | ExecutionException | TimeoutException e) {
+            log.error("Failed to write session key to Kafka: ", e);
+            throw new ConnectException("Error writing session key to Kafka", e);
+        }
+    }
+
     // package private for testing
     KafkaBasedLog setupAndCreateKafkaBasedLog(String topic, final WorkerConfig config) {
         Map originals = config.originals();
@@ -635,6 +667,45 @@ public void onCompletion(Throwable error, ConsumerRecord record)
 
                 if (started)
                     updateListener.onTaskConfigUpdate(updatedTasks);
+            } else if (record.key().equals(SESSION_KEY_KEY)) {
+                synchronized (lock) {
+                    if (value.value() == null) {
+                        log.error("Ignoring session key because it is unexpectedly null");
+                        return;
+                    }
+                    if (!(value.value() instanceof Map)) {
+                        log.error("Ignoring session key because the value is not a Map but is {}", value.value().getClass());
+                        return;
+                    }
+
+                    Map valueAsMap = (Map) value.value();
+
+                    Object sessionKey = valueAsMap.get("key");
+                    if (!(sessionKey instanceof String)) {
+                        log.error("Invalid data for session key 'key' field should be a String but is {}", sessionKey.getClass());
+                        return;
+                    }
+                    byte[] key = Base64.getDecoder().decode((String) sessionKey);
+
+                    Object keyAlgorithm = valueAsMap.get("algorithm");
+                    if (!(keyAlgorithm instanceof String)) {
+                        log.error("Invalid data for session key 'algorithm' field should be a String but it is {}", keyAlgorithm.getClass());
+                        return;
+                    }
+
+                    Object creationTimestamp = valueAsMap.get("creation-timestamp");
+                    if (!(creationTimestamp instanceof Long)) {
+                        log.error("Invalid data for session key 'creation-timestamp' field should be a long but it is {}", creationTimestamp.getClass());
+                        return;
+                    }
+                    KafkaConfigBackingStore.this.sessionKey = new SessionKey(
+                        new SecretKeySpec(key, (String) keyAlgorithm),
+                        (long) creationTimestamp
+                    );
+
+                    if (started)
+                        updateListener.onSessionKeyUpdate(KafkaConfigBackingStore.this.sessionKey);
+                }
             } else {
                 log.error("Discarding config update record with invalid key: {}", record.key());
             }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/MemoryConfigBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/MemoryConfigBackingStore.java
index 7e7d62ba3c982..90039b9c01618 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/MemoryConfigBackingStore.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/MemoryConfigBackingStore.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.connect.storage;
 
+import org.apache.kafka.connect.runtime.SessionKey;
 import org.apache.kafka.connect.runtime.TargetState;
 import org.apache.kafka.connect.runtime.WorkerConfigTransformer;
 import org.apache.kafka.connect.runtime.distributed.ClusterConfigState;
@@ -68,6 +69,7 @@ public synchronized ClusterConfigState snapshot() {
 
         return new ClusterConfigState(
                 ClusterConfigState.NO_OFFSET,
+                null,
                 connectorTaskCounts,
                 connectorConfigs,
                 connectorTargetStates,
@@ -143,6 +145,11 @@ public synchronized void putTargetState(String connector, TargetState state) {
             updateListener.onConnectorTargetStateChange(connector);
     }
 
+    @Override
+    public void putSessionKey(SessionKey sessionKey) {
+        // no-op
+    }
+
     @Override
     public synchronized void setUpdateListener(UpdateListener listener) {
         this.updateListener = listener;
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/SessionedProtocolIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/SessionedProtocolIntegrationTest.java
new file mode 100644
index 0000000000000..d01ab3b7c4a8d
--- /dev/null
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/SessionedProtocolIntegrationTest.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.integration;
+
+import org.apache.kafka.connect.runtime.distributed.ConnectProtocolCompatibility;
+import org.apache.kafka.connect.storage.StringConverter;
+import org.apache.kafka.connect.util.clusters.EmbeddedConnectCluster;
+import org.apache.kafka.test.IntegrationTest;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
+import static javax.ws.rs.core.Response.Status.FORBIDDEN;
+import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLASS_CONFIG;
+import static org.apache.kafka.connect.runtime.ConnectorConfig.KEY_CONVERTER_CLASS_CONFIG;
+import static org.apache.kafka.connect.runtime.ConnectorConfig.TASKS_MAX_CONFIG;
+import static org.apache.kafka.connect.runtime.ConnectorConfig.VALUE_CONVERTER_CLASS_CONFIG;
+import static org.apache.kafka.connect.runtime.SinkConnectorConfig.TOPICS_CONFIG;
+import static org.apache.kafka.connect.runtime.distributed.DistributedConfig.CONNECT_PROTOCOL_CONFIG;
+import static org.apache.kafka.connect.runtime.rest.InternalRequestSignature.SIGNATURE_ALGORITHM_HEADER;
+import static org.apache.kafka.connect.runtime.rest.InternalRequestSignature.SIGNATURE_HEADER;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * A simple integration test to ensure that internal request validation becomes enabled with the
+ * "sessioned" protocol.
+ */
+@Category(IntegrationTest.class)
+public class SessionedProtocolIntegrationTest {
+
+    private static final Logger log = LoggerFactory.getLogger(SessionedProtocolIntegrationTest.class);
+
+    private static final String CONNECTOR_NAME = "connector";
+    private static final long CONNECTOR_SETUP_DURATION_MS = 60000;
+
+    private EmbeddedConnectCluster connect;
+    private ConnectorHandle connectorHandle;
+
+    @Before
+    public void setup() throws IOException {
+        // setup Connect worker properties
+        Map workerProps = new HashMap<>();
+        workerProps.put(CONNECT_PROTOCOL_CONFIG, ConnectProtocolCompatibility.SESSIONED.protocol());
+
+        // build a Connect cluster backed by Kafka and Zk
+        connect = new EmbeddedConnectCluster.Builder()
+            .name("connect-cluster")
+            .numWorkers(2)
+            .numBrokers(1)
+            .workerProps(workerProps)
+            .build();
+
+        // start the clusters
+        connect.start();
+
+        // get a handle to the connector
+        connectorHandle = RuntimeHandles.get().connectorHandle(CONNECTOR_NAME);
+    }
+
+    @After
+    public void close() {
+        // stop all Connect, Kafka and Zk threads.
+        connect.stop();
+    }
+
+    @Test
+    @Ignore
+    // TODO: This test runs fine locally but fails on Jenkins. Ignoring for now, should revisit when
+    //       possible.
+    public void ensureInternalEndpointIsSecured() throws Throwable {
+        final String connectorTasksEndpoint = connect.endpointForResource(String.format(
+            "connectors/%s/tasks",
+            CONNECTOR_NAME
+        ));
+        final Map emptyHeaders = new HashMap<>();
+        final Map invalidSignatureHeaders = new HashMap<>();
+        invalidSignatureHeaders.put(SIGNATURE_HEADER, "S2Fma2Flc3F1ZQ==");
+        invalidSignatureHeaders.put(SIGNATURE_ALGORITHM_HEADER, "HmacSHA256");
+
+        // We haven't created the connector yet, but this should still return a 400 instead of a 404
+        // if the endpoint is secured
+        log.info(
+            "Making a POST request to the {} endpoint with no connector started and no signature header; " 
+                + "expecting 400 error response",
+            connectorTasksEndpoint
+        );
+        assertEquals(
+            BAD_REQUEST.getStatusCode(),
+            connect.executePost(connectorTasksEndpoint, "[]", emptyHeaders)
+        );
+
+        // Try again, but with an invalid signature
+        log.info(
+            "Making a POST request to the {} endpoint with no connector started and an invalid signature header; "
+                + "expecting 403 error response",
+            connectorTasksEndpoint
+        );
+        assertEquals(
+            FORBIDDEN.getStatusCode(),
+            connect.executePost(connectorTasksEndpoint, "[]", invalidSignatureHeaders)
+        );
+
+        // Create the connector now
+        // setup up props for the sink connector
+        Map connectorProps = new HashMap<>();
+        connectorProps.put(CONNECTOR_CLASS_CONFIG, MonitorableSinkConnector.class.getSimpleName());
+        connectorProps.put(TASKS_MAX_CONFIG, String.valueOf(1));
+        connectorProps.put(TOPICS_CONFIG, "test-topic");
+        connectorProps.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
+        connectorProps.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
+
+        // start a sink connector
+        log.info("Starting the {} connector", CONNECTOR_NAME);
+        StartAndStopLatch startLatch = connectorHandle.expectedStarts(1);
+        connect.configureConnector(CONNECTOR_NAME, connectorProps);
+        startLatch.await(CONNECTOR_SETUP_DURATION_MS, TimeUnit.MILLISECONDS);
+
+
+        // Verify the exact same behavior, after starting the connector
+
+        // We haven't created the connector yet, but this should still return a 400 instead of a 404
+        // if the endpoint is secured
+        log.info(
+            "Making a POST request to the {} endpoint with the connector started and no signature header; "
+                + "expecting 400 error response",
+            connectorTasksEndpoint
+        );
+        assertEquals(
+            BAD_REQUEST.getStatusCode(),
+            connect.executePost(connectorTasksEndpoint, "[]", emptyHeaders)
+        );
+
+        // Try again, but with an invalid signature
+        log.info(
+            "Making a POST request to the {} endpoint with the connector started and an invalid signature header; "
+                + "expecting 403 error response",
+            connectorTasksEndpoint
+        );
+        assertEquals(
+            FORBIDDEN.getStatusCode(),
+            connect.executePost(connectorTasksEndpoint, "[]", invalidSignatureHeaders)
+        );
+    }
+}
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java
index 801891675a313..48a79b1ca361d 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java
@@ -117,10 +117,10 @@ public class AbstractHerderTest {
         TASK_CONFIGS_MAP.put(TASK1, TASK_CONFIG);
         TASK_CONFIGS_MAP.put(TASK2, TASK_CONFIG);
     }
-    private static final ClusterConfigState SNAPSHOT = new ClusterConfigState(1, Collections.singletonMap(CONN1, 3),
+    private static final ClusterConfigState SNAPSHOT = new ClusterConfigState(1, null, Collections.singletonMap(CONN1, 3),
             Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.STARTED),
             TASK_CONFIGS_MAP, Collections.emptySet());
-    private static final ClusterConfigState SNAPSHOT_NO_TASKS = new ClusterConfigState(1, Collections.singletonMap(CONN1, 3),
+    private static final ClusterConfigState SNAPSHOT_NO_TASKS = new ClusterConfigState(1, null, Collections.singletonMap(CONN1, 3),
             Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.STARTED),
             Collections.emptyMap(), Collections.emptySet());
 
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTestUtils.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTestUtils.java
index a790574cfb098..ed77018f2883b 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTestUtils.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTestUtils.java
@@ -65,6 +65,7 @@ public static ClusterConfigState clusterConfigState(long offset,
                                                         int taskNum) {
         return new ClusterConfigState(
                 offset,
+                null,
                 connectorTaskCounts(1, connectorNum, taskNum),
                 connectorConfigs(1, connectorNum),
                 connectorTargetStates(1, connectorNum, TargetState.STARTED),
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/ConnectProtocolCompatibilityTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/ConnectProtocolCompatibilityTest.java
index 24e7cb9ba3459..a3144c04d3462 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/ConnectProtocolCompatibilityTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/ConnectProtocolCompatibilityTest.java
@@ -62,6 +62,7 @@ public void setup() {
         configStorage = mock(KafkaConfigBackingStore.class);
         configState = new ClusterConfigState(
                 1L,
+                null,
                 Collections.singletonMap(connectorId1, 1),
                 Collections.singletonMap(connectorId1, new HashMap<>()),
                 Collections.singletonMap(connectorId1, TargetState.STARTED),
@@ -89,18 +90,40 @@ public void testEagerToEagerMetadata() {
     public void testCoopToCoopMetadata() {
         when(configStorage.snapshot()).thenReturn(configState);
         ExtendedWorkerState workerState = new ExtendedWorkerState(LEADER_URL, configStorage.snapshot().offset(), null);
-        ByteBuffer metadata = IncrementalCooperativeConnectProtocol.serializeMetadata(workerState);
+        ByteBuffer metadata = IncrementalCooperativeConnectProtocol.serializeMetadata(workerState, false);
         ExtendedWorkerState state = IncrementalCooperativeConnectProtocol.deserializeMetadata(metadata);
         assertEquals(LEADER_URL, state.url());
         assertEquals(1, state.offset());
         verify(configStorage).snapshot();
     }
 
+    @Test
+    public void testSessionedToCoopMetadata() {
+        when(configStorage.snapshot()).thenReturn(configState);
+        ExtendedWorkerState workerState = new ExtendedWorkerState(LEADER_URL, configStorage.snapshot().offset(), null);
+        ByteBuffer metadata = IncrementalCooperativeConnectProtocol.serializeMetadata(workerState, true);
+        ExtendedWorkerState state = IncrementalCooperativeConnectProtocol.deserializeMetadata(metadata);
+        assertEquals(LEADER_URL, state.url());
+        assertEquals(1, state.offset());
+        verify(configStorage).snapshot();
+    }
+
+    @Test
+    public void testSessionedToEagerMetadata() {
+        when(configStorage.snapshot()).thenReturn(configState);
+        ExtendedWorkerState workerState = new ExtendedWorkerState(LEADER_URL, configStorage.snapshot().offset(), null);
+        ByteBuffer metadata = IncrementalCooperativeConnectProtocol.serializeMetadata(workerState, true);
+        ConnectProtocol.WorkerState state = ConnectProtocol.deserializeMetadata(metadata);
+        assertEquals(LEADER_URL, state.url());
+        assertEquals(1, state.offset());
+        verify(configStorage).snapshot();
+    }
+
     @Test
     public void testCoopToEagerMetadata() {
         when(configStorage.snapshot()).thenReturn(configState);
         ExtendedWorkerState workerState = new ExtendedWorkerState(LEADER_URL, configStorage.snapshot().offset(), null);
-        ByteBuffer metadata = IncrementalCooperativeConnectProtocol.serializeMetadata(workerState);
+        ByteBuffer metadata = IncrementalCooperativeConnectProtocol.serializeMetadata(workerState, false);
         ConnectProtocol.WorkerState state = ConnectProtocol.deserializeMetadata(metadata);
         assertEquals(LEADER_URL, state.url());
         assertEquals(1, state.offset());
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedConfigTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedConfigTest.java
new file mode 100644
index 0000000000000..630465b912f7b
--- /dev/null
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedConfigTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.connect.runtime.distributed;
+
+import org.apache.kafka.common.config.ConfigException;
+import org.junit.Test;
+
+import javax.crypto.KeyGenerator;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
+
+public class DistributedConfigTest {
+
+    public Map configs() {
+        Map result = new HashMap<>();
+        result.put(DistributedConfig.GROUP_ID_CONFIG, "connect-cluster");
+        result.put(DistributedConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
+        result.put(DistributedConfig.CONFIG_TOPIC_CONFIG, "connect-configs");
+        result.put(DistributedConfig.OFFSET_STORAGE_TOPIC_CONFIG, "connect-offsets");
+        result.put(DistributedConfig.STATUS_STORAGE_TOPIC_CONFIG, "connect-status");
+        result.put(DistributedConfig.KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.json.JsonConverter");
+        result.put(DistributedConfig.VALUE_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.json.JsonConverter");
+        return result;
+    }
+
+    @Test
+    public void shouldCreateKeyGeneratorWithDefaultSettings() {
+        DistributedConfig config = new DistributedConfig(configs());
+        assertNotNull(config.getInternalRequestKeyGenerator());
+    }
+
+    @Test
+    public void shouldCreateKeyGeneratorWithSpecificSettings() {
+        final String algorithm = "HmacSHA1";
+        Map configs = configs();
+        configs.put(DistributedConfig.INTER_WORKER_KEY_GENERATION_ALGORITHM_CONFIG, algorithm);
+        configs.put(DistributedConfig.INTER_WORKER_KEY_SIZE_CONFIG, "512");
+        configs.put(DistributedConfig.INTER_WORKER_VERIFICATION_ALGORITHMS_CONFIG, algorithm);
+        DistributedConfig config = new DistributedConfig(configs);
+        KeyGenerator keyGenerator = config.getInternalRequestKeyGenerator();
+        assertNotNull(keyGenerator);
+        assertEquals(algorithm, keyGenerator.getAlgorithm());
+        assertEquals(512 / 8, keyGenerator.generateKey().getEncoded().length);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void shouldFailWithEmptyListOfVerificationAlgorithms() {
+        Map configs = configs();
+        configs.put(DistributedConfig.INTER_WORKER_VERIFICATION_ALGORITHMS_CONFIG, "");
+        new DistributedConfig(configs);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void shouldFailIfKeyAlgorithmNotInVerificationAlgorithmsList() {
+        Map configs = configs();
+        configs.put(DistributedConfig.INTER_WORKER_KEY_GENERATION_ALGORITHM_CONFIG, "HmacSHA1");
+        configs.put(DistributedConfig.INTER_WORKER_VERIFICATION_ALGORITHMS_CONFIG, "HmacSHA256");
+        new DistributedConfig(configs);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void shouldFailWithInvalidKeyAlgorithm() {
+        Map configs = configs();
+        configs.put(DistributedConfig.INTER_WORKER_KEY_GENERATION_ALGORITHM_CONFIG, "not-actually-a-key-algorithm");
+        new DistributedConfig(configs);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void shouldFailWithInvalidKeySize() {
+        Map configs = configs();
+        configs.put(DistributedConfig.INTER_WORKER_KEY_SIZE_CONFIG, "0");
+        new DistributedConfig(configs);
+    }
+
+    @Test
+    public void shouldValidateAllVerificationAlgorithms() {
+        List algorithms =
+            new ArrayList<>(Arrays.asList("HmacSHA1", "HmacSHA256", "HmacMD5", "bad-algorithm"));
+        Map configs = configs();
+        for (int i = 0; i < algorithms.size(); i++) {
+            configs.put(DistributedConfig.INTER_WORKER_VERIFICATION_ALGORITHMS_CONFIG, String.join(",", algorithms));
+            assertThrows(ConfigException.class, () -> new DistributedConfig(configs));
+            algorithms.add(algorithms.remove(0));
+        }
+    }
+}
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java
index bc40a61785a69..ddbd560fa5c56 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java
@@ -41,10 +41,12 @@
 import org.apache.kafka.connect.runtime.isolation.DelegatingClassLoader;
 import org.apache.kafka.connect.runtime.isolation.PluginClassLoader;
 import org.apache.kafka.connect.runtime.isolation.Plugins;
+import org.apache.kafka.connect.runtime.rest.InternalRequestSignature;
 import org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo;
 import org.apache.kafka.connect.runtime.rest.entities.ConnectorType;
 import org.apache.kafka.connect.runtime.rest.entities.TaskInfo;
 import org.apache.kafka.connect.runtime.rest.errors.BadRequestException;
+import org.apache.kafka.connect.runtime.rest.errors.ConnectRestException;
 import org.apache.kafka.connect.sink.SinkConnector;
 import org.apache.kafka.connect.source.SourceConnector;
 import org.apache.kafka.connect.source.SourceTask;
@@ -78,16 +80,19 @@
 import java.util.concurrent.TimeoutException;
 
 import static java.util.Collections.singletonList;
+import static javax.ws.rs.core.Response.Status.FORBIDDEN;
 import static org.apache.kafka.connect.runtime.distributed.ConnectProtocol.CONNECT_PROTOCOL_V0;
 import static org.apache.kafka.connect.runtime.distributed.IncrementalCooperativeConnectProtocol.CONNECT_PROTOCOL_V1;
+import static org.apache.kafka.connect.runtime.distributed.IncrementalCooperativeConnectProtocol.CONNECT_PROTOCOL_V2;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 @SuppressWarnings("deprecation")
 @RunWith(PowerMockRunner.class)
 @PrepareForTest({DistributedHerder.class, Plugins.class})
-@PowerMockIgnore("javax.management.*")
+@PowerMockIgnore({"javax.management.*", "javax.crypto.*"})
 public class DistributedHerderTest {
     private static final Map HERDER_CONFIG = new HashMap<>();
     static {
@@ -144,13 +149,13 @@ public class DistributedHerderTest {
         TASK_CONFIGS_MAP.put(TASK1, TASK_CONFIG);
         TASK_CONFIGS_MAP.put(TASK2, TASK_CONFIG);
     }
-    private static final ClusterConfigState SNAPSHOT = new ClusterConfigState(1, Collections.singletonMap(CONN1, 3),
+    private static final ClusterConfigState SNAPSHOT = new ClusterConfigState(1, null, Collections.singletonMap(CONN1, 3),
             Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.STARTED),
             TASK_CONFIGS_MAP, Collections.emptySet());
-    private static final ClusterConfigState SNAPSHOT_PAUSED_CONN1 = new ClusterConfigState(1, Collections.singletonMap(CONN1, 3),
+    private static final ClusterConfigState SNAPSHOT_PAUSED_CONN1 = new ClusterConfigState(1, null, Collections.singletonMap(CONN1, 3),
             Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.PAUSED),
             TASK_CONFIGS_MAP, Collections.emptySet());
-    private static final ClusterConfigState SNAPSHOT_UPDATED_CONN1_CONFIG = new ClusterConfigState(1, Collections.singletonMap(CONN1, 3),
+    private static final ClusterConfigState SNAPSHOT_UPDATED_CONN1_CONFIG = new ClusterConfigState(1, null, Collections.singletonMap(CONN1, 3),
             Collections.singletonMap(CONN1, CONN1_CONFIG_UPDATED), Collections.singletonMap(CONN1, TargetState.STARTED),
             TASK_CONFIGS_MAP, Collections.emptySet());
 
@@ -1490,7 +1495,7 @@ public void testAccessors() throws Exception {
         EasyMock.expect(configTransformer.transform(EasyMock.eq(CONN1), EasyMock.anyObject()))
             .andThrow(new AssertionError("Config transformation should not occur when requesting connector or task info"));
         EasyMock.replay(configTransformer);
-        ClusterConfigState snapshotWithTransform = new ClusterConfigState(1, Collections.singletonMap(CONN1, 3),
+        ClusterConfigState snapshotWithTransform = new ClusterConfigState(1, null, Collections.singletonMap(CONN1, 3),
             Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.STARTED),
             TASK_CONFIGS_MAP, Collections.emptySet(), configTransformer);
 
@@ -1627,6 +1632,131 @@ public Object answer() throws Throwable {
         PowerMock.verifyAll();
     }
 
+
+    @Test
+    public void testPutTaskConfigsSignatureNotRequiredV0() {
+        Callback taskConfigCb = EasyMock.mock(Callback.class);
+
+        member.wakeup();
+        EasyMock.expectLastCall().once();
+        EasyMock.expect(member.currentProtocolVersion()).andReturn(CONNECT_PROTOCOL_V0).anyTimes();
+        PowerMock.replayAll(taskConfigCb);
+
+        herder.putTaskConfigs(CONN1, TASK_CONFIGS, taskConfigCb, null);
+
+        PowerMock.verifyAll();
+    }
+    @Test
+    public void testPutTaskConfigsSignatureNotRequiredV1() {
+        Callback taskConfigCb = EasyMock.mock(Callback.class);
+
+        member.wakeup();
+        EasyMock.expectLastCall().once();
+        EasyMock.expect(member.currentProtocolVersion()).andReturn(CONNECT_PROTOCOL_V1).anyTimes();
+        PowerMock.replayAll(taskConfigCb);
+
+        herder.putTaskConfigs(CONN1, TASK_CONFIGS, taskConfigCb, null);
+
+        PowerMock.verifyAll();
+    }
+
+    @Test
+    public void testPutTaskConfigsMissingRequiredSignature() {
+        Callback taskConfigCb = EasyMock.mock(Callback.class);
+        Capture errorCapture = Capture.newInstance();
+        taskConfigCb.onCompletion(EasyMock.capture(errorCapture), EasyMock.eq(null));
+        EasyMock.expectLastCall().once();
+
+        EasyMock.expect(member.currentProtocolVersion()).andReturn(CONNECT_PROTOCOL_V2).anyTimes();
+        PowerMock.replayAll(taskConfigCb);
+
+        herder.putTaskConfigs(CONN1, TASK_CONFIGS, taskConfigCb, null);
+
+        PowerMock.verifyAll();
+        assertTrue(errorCapture.getValue() instanceof BadRequestException);
+    }
+
+    @Test
+    public void testPutTaskConfigsDisallowedSignatureAlgorithm() {
+        Callback taskConfigCb = EasyMock.mock(Callback.class);
+        Capture errorCapture = Capture.newInstance();
+        taskConfigCb.onCompletion(EasyMock.capture(errorCapture), EasyMock.eq(null));
+        EasyMock.expectLastCall().once();
+
+        EasyMock.expect(member.currentProtocolVersion()).andReturn(CONNECT_PROTOCOL_V2).anyTimes();
+
+        InternalRequestSignature signature = EasyMock.mock(InternalRequestSignature.class);
+        EasyMock.expect(signature.keyAlgorithm()).andReturn("HmacSHA489").anyTimes();
+
+        PowerMock.replayAll(taskConfigCb, signature);
+
+        herder.putTaskConfigs(CONN1, TASK_CONFIGS, taskConfigCb, signature);
+
+        PowerMock.verifyAll();
+        assertTrue(errorCapture.getValue() instanceof BadRequestException);
+    }
+
+    @Test
+    public void testPutTaskConfigsInvalidSignature() {
+        Callback taskConfigCb = EasyMock.mock(Callback.class);
+        Capture errorCapture = Capture.newInstance();
+        taskConfigCb.onCompletion(EasyMock.capture(errorCapture), EasyMock.eq(null));
+        EasyMock.expectLastCall().once();
+
+        EasyMock.expect(member.currentProtocolVersion()).andReturn(CONNECT_PROTOCOL_V2).anyTimes();
+
+        InternalRequestSignature signature = EasyMock.mock(InternalRequestSignature.class);
+        EasyMock.expect(signature.keyAlgorithm()).andReturn("HmacSHA256").anyTimes();
+        EasyMock.expect(signature.isValid(EasyMock.anyObject())).andReturn(false).anyTimes();
+
+        PowerMock.replayAll(taskConfigCb, signature);
+
+        herder.putTaskConfigs(CONN1, TASK_CONFIGS, taskConfigCb, signature);
+
+        PowerMock.verifyAll();
+        assertTrue(errorCapture.getValue() instanceof ConnectRestException);
+        assertEquals(FORBIDDEN.getStatusCode(), ((ConnectRestException) errorCapture.getValue()).statusCode());
+    }
+
+    @Test
+    public void testPutTaskConfigsValidRequiredSignature() {
+        Callback taskConfigCb = EasyMock.mock(Callback.class);
+
+        member.wakeup();
+        EasyMock.expectLastCall().once();
+        EasyMock.expect(member.currentProtocolVersion()).andReturn(CONNECT_PROTOCOL_V2).anyTimes();
+
+        InternalRequestSignature signature = EasyMock.mock(InternalRequestSignature.class);
+        EasyMock.expect(signature.keyAlgorithm()).andReturn("HmacSHA256").anyTimes();
+        EasyMock.expect(signature.isValid(EasyMock.anyObject())).andReturn(true).anyTimes();
+
+        PowerMock.replayAll(taskConfigCb, signature);
+
+        herder.putTaskConfigs(CONN1, TASK_CONFIGS, taskConfigCb, signature);
+
+        PowerMock.verifyAll();
+    }
+
+    @Test
+    public void testKeyExceptionDetection() {
+        assertFalse(herder.isPossibleExpiredKeyException(
+            time.milliseconds(),
+            new RuntimeException()
+        ));
+        assertFalse(herder.isPossibleExpiredKeyException(
+            time.milliseconds(),
+            new BadRequestException("")
+        ));
+        assertFalse(herder.isPossibleExpiredKeyException(
+            time.milliseconds() - TimeUnit.MINUTES.toMillis(2),
+            new ConnectRestException(FORBIDDEN.getStatusCode(), "")
+        ));
+        assertTrue(herder.isPossibleExpiredKeyException(
+            time.milliseconds(),
+            new ConnectRestException(FORBIDDEN.getStatusCode(), "")
+        ));
+    }
+
     @Test
     public void testInconsistentConfigs() {
         // FIXME: if we have inconsistent configs, we need to request forced reconfig + write of the connector's task configs
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java
index 7085a7f15d429..c46d59bf8058a 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java
@@ -47,12 +47,15 @@
 import java.util.stream.IntStream;
 
 import static org.apache.kafka.connect.runtime.distributed.IncrementalCooperativeConnectProtocol.CONNECT_PROTOCOL_V1;
+import static org.apache.kafka.connect.runtime.distributed.IncrementalCooperativeConnectProtocol.CONNECT_PROTOCOL_V2;
 import static org.apache.kafka.connect.runtime.distributed.WorkerCoordinator.WorkerLoad;
 import static org.hamcrest.CoreMatchers.hasItems;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.runners.Parameterized.Parameter;
+import static org.junit.runners.Parameterized.Parameters;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.doThrow;
@@ -71,6 +74,14 @@ public class IncrementalCooperativeAssignorTest {
     @Captor
     ArgumentCaptor> assignmentsCapture;
 
+    @Parameters
+    public static Iterable mode() {
+        return Arrays.asList(new Object[][] {{CONNECT_PROTOCOL_V1, CONNECT_PROTOCOL_V2}});
+    }
+
+    @Parameter
+    public short protocolVersion;
+
     private ClusterConfigState configState;
     private Map memberConfigs;
     private Map expectedMemberConfigs;
@@ -115,7 +126,7 @@ public void testTaskAssignmentWhenWorkerJoins() {
         doReturn(Collections.EMPTY_MAP).when(assignor).serializeAssignments(assignmentsCapture.capture());
 
         // First assignment with 1 worker and 2 connectors configured but not yet assigned
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
@@ -127,7 +138,7 @@ public void testTaskAssignmentWhenWorkerJoins() {
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
         memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, null));
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
@@ -138,7 +149,7 @@ public void testTaskAssignmentWhenWorkerJoins() {
         // Third assignment after revocations
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
@@ -149,7 +160,7 @@ public void testTaskAssignmentWhenWorkerJoins() {
         // A fourth rebalance should not change assignments
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
@@ -172,7 +183,7 @@ public void testTaskAssignmentWhenWorkerLeavesPermanently() {
 
         // First assignment with 2 workers and 2 connectors configured but not yet assigned
         memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, null));
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
@@ -186,7 +197,7 @@ public void testTaskAssignmentWhenWorkerLeavesPermanently() {
         applyAssignments(returnedAssignments);
         assignments.remove("worker2");
         memberConfigs = memberConfigs(leader, offset, assignments);
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(rebalanceDelay, returnedAssignments);
@@ -200,7 +211,7 @@ public void testTaskAssignmentWhenWorkerLeavesPermanently() {
         // been reached yet
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(rebalanceDelay / 2, returnedAssignments);
@@ -213,7 +224,7 @@ public void testTaskAssignmentWhenWorkerLeavesPermanently() {
         // Fourth assignment after delay expired
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
@@ -236,7 +247,7 @@ public void testTaskAssignmentWhenWorkerBounces() {
 
         // First assignment with 2 workers and 2 connectors configured but not yet assigned
         memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, null));
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
@@ -250,7 +261,7 @@ public void testTaskAssignmentWhenWorkerBounces() {
         applyAssignments(returnedAssignments);
         assignments.remove("worker2");
         memberConfigs = memberConfigs(leader, offset, assignments);
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(rebalanceDelay, returnedAssignments);
@@ -264,7 +275,7 @@ public void testTaskAssignmentWhenWorkerBounces() {
         // been reached yet
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(rebalanceDelay / 2, returnedAssignments);
@@ -279,7 +290,7 @@ public void testTaskAssignmentWhenWorkerBounces() {
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
         memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, null));
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(rebalanceDelay / 4, returnedAssignments);
@@ -293,7 +304,7 @@ public void testTaskAssignmentWhenWorkerBounces() {
         // assignments ought to be assigned to the worker that has appeared as returned.
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
@@ -317,7 +328,7 @@ public void testTaskAssignmentWhenLeaderLeavesPermanently() {
         // First assignment with 3 workers and 2 connectors configured but not yet assigned
         memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, null));
         memberConfigs.put("worker3", new ExtendedWorkerState(leaderUrl, offset, null));
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
@@ -338,7 +349,7 @@ public void testTaskAssignmentWhenLeaderLeavesPermanently() {
 
         // Capture needs to be reset to point to the new assignor
         doReturn(Collections.EMPTY_MAP).when(assignor).serializeAssignments(assignmentsCapture.capture());
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
@@ -349,7 +360,7 @@ public void testTaskAssignmentWhenLeaderLeavesPermanently() {
         // Third (incidental) assignment with still only one worker in the group.
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
@@ -372,7 +383,7 @@ public void testTaskAssignmentWhenLeaderBounces() {
         // First assignment with 3 workers and 2 connectors configured but not yet assigned
         memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, null));
         memberConfigs.put("worker3", new ExtendedWorkerState(leaderUrl, offset, null));
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
@@ -393,7 +404,7 @@ public void testTaskAssignmentWhenLeaderBounces() {
 
         // Capture needs to be reset to point to the new assignor
         doReturn(Collections.EMPTY_MAP).when(assignor).serializeAssignments(assignmentsCapture.capture());
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
@@ -407,7 +418,7 @@ public void testTaskAssignmentWhenLeaderBounces() {
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
         memberConfigs.put("worker1", new ExtendedWorkerState(leaderUrl, offset, null));
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
@@ -417,7 +428,7 @@ public void testTaskAssignmentWhenLeaderBounces() {
         // Fourth assignment after revocations
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
@@ -442,7 +453,7 @@ public void testTaskAssignmentWhenFirstAssignmentAttemptFails() {
         // First assignment with 2 workers and 2 connectors configured but not yet assigned
         memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, null));
         try {
-            assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+            assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         } catch (RuntimeException e) {
             RequestFuture.failure(e);
         }
@@ -459,7 +470,7 @@ public void testTaskAssignmentWhenFirstAssignmentAttemptFails() {
         // or the workers that were bounced. Therefore it goes into assignment freeze for
         // the new assignments for a rebalance delay period
         doReturn(Collections.EMPTY_MAP).when(assignor).serializeAssignments(assignmentsCapture.capture());
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
@@ -473,7 +484,7 @@ public void testTaskAssignmentWhenFirstAssignmentAttemptFails() {
         // been reached yet
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(rebalanceDelay / 2, returnedAssignments);
@@ -486,7 +497,7 @@ public void testTaskAssignmentWhenFirstAssignmentAttemptFails() {
         // Fourth assignment after delay expired. Finally all the new assignments are assigned
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
@@ -509,7 +520,7 @@ public void testTaskAssignmentWhenSubsequentAssignmentAttemptFails() {
 
         // First assignment with 2 workers and 2 connectors configured but not yet assigned
         memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, null));
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
@@ -527,7 +538,7 @@ public void testTaskAssignmentWhenSubsequentAssignmentAttemptFails() {
         memberConfigs = memberConfigs(leader, offset, assignments);
         memberConfigs.put("worker3", new ExtendedWorkerState(leaderUrl, offset, null));
         try {
-            assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+            assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         } catch (RuntimeException e) {
             RequestFuture.failure(e);
         }
@@ -542,7 +553,7 @@ public void testTaskAssignmentWhenSubsequentAssignmentAttemptFails() {
         // Third assignment happens with members returning the same assignments (memberConfigs)
         // as the first time.
         doReturn(Collections.EMPTY_MAP).when(assignor).serializeAssignments(assignmentsCapture.capture());
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
@@ -562,7 +573,7 @@ public void testTaskAssignmentWhenConnectorsAreDeleted() {
 
         // First assignment with 1 worker and 2 connectors configured but not yet assigned
         memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, null));
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
@@ -575,7 +586,7 @@ public void testTaskAssignmentWhenConnectorsAreDeleted() {
         when(coordinator.configSnapshot()).thenReturn(configState);
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         assertDelay(0, returnedAssignments);
@@ -976,6 +987,7 @@ private static ClusterConfigState clusterConfigState(long offset,
                                                          int taskNum) {
         return new ClusterConfigState(
                 offset,
+                null,
                 connectorTaskCounts(1, connectorNum, taskNum),
                 connectorConfigs(1, connectorNum),
                 connectorTargetStates(1, connectorNum, TargetState.STARTED),
@@ -1052,7 +1064,7 @@ private void applyAssignments(Map newAssignments) {
 
     private ExtendedAssignment newExpandableAssignment() {
         return new ExtendedAssignment(
-                CONNECT_PROTOCOL_V1,
+                protocolVersion,
                 ConnectProtocol.Assignment.NO_ERROR,
                 leader,
                 leaderUrl,
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java
index b1d3ec6147d98..04c764cb89f1e 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java
@@ -56,6 +56,7 @@
 import static org.apache.kafka.connect.runtime.distributed.ConnectProtocol.WorkerState;
 import static org.apache.kafka.connect.runtime.distributed.ConnectProtocolCompatibility.COMPATIBLE;
 import static org.apache.kafka.connect.runtime.distributed.ConnectProtocolCompatibility.EAGER;
+import static org.apache.kafka.connect.runtime.distributed.ConnectProtocolCompatibility.SESSIONED;
 import static org.apache.kafka.connect.runtime.distributed.IncrementalCooperativeConnectProtocol.CONNECT_PROTOCOL_V1;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
@@ -119,7 +120,7 @@ public class WorkerCoordinatorIncrementalTest {
     // - Expected metadata size
     @Parameters
     public static Iterable mode() {
-        return Arrays.asList(new Object[][]{{COMPATIBLE, 2}});
+        return Arrays.asList(new Object[][]{{COMPATIBLE, 2}, {SESSIONED, 3}});
     }
 
     @Parameter
@@ -565,14 +566,18 @@ private static ExtendedAssignment deserializeAssignment(Map
         return IncrementalCooperativeConnectProtocol.deserializeAssignment(assignment.get(member));
     }
 
-    private static void addJoinGroupResponseMember(List responseMembers,
+    private void addJoinGroupResponseMember(List responseMembers,
                                                    String member,
                                                    long offset,
                                                    ExtendedAssignment assignment) {
         responseMembers.add(new JoinGroupResponseMember()
                 .setMemberId(member)
-                .setMetadata(IncrementalCooperativeConnectProtocol.serializeMetadata(
-                        new ExtendedWorkerState(expectedUrl(member), offset, assignment)).array())
+                .setMetadata(
+                    IncrementalCooperativeConnectProtocol.serializeMetadata(
+                        new ExtendedWorkerState(expectedUrl(member), offset, assignment),
+                        compatibility != COMPATIBLE
+                    ).array()
+                )
         );
     }
 }
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorTest.java
index eac13d158eb5c..05e83fb80231f 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorTest.java
@@ -149,6 +149,7 @@ public void setup() {
 
         configState1 = new ClusterConfigState(
                 1L,
+                null,
                 Collections.singletonMap(connectorId1, 1),
                 Collections.singletonMap(connectorId1, (Map) new HashMap()),
                 Collections.singletonMap(connectorId1, TargetState.STARTED),
@@ -171,6 +172,7 @@ public void setup() {
         configState2TaskConfigs.put(taskId2x0, new HashMap());
         configState2 = new ClusterConfigState(
                 2L,
+                null,
                 configState2ConnectorTaskCounts,
                 configState2ConnectorConfigs,
                 configState2TargetStates,
@@ -196,6 +198,7 @@ public void setup() {
         configStateSingleTaskConnectorsTaskConfigs.put(taskId3x0, new HashMap());
         configStateSingleTaskConnectors = new ClusterConfigState(
                 2L,
+                null,
                 configStateSingleTaskConnectorsConnectorTaskCounts,
                 configStateSingleTaskConnectorsConnectorConfigs,
                 configStateSingleTaskConnectorsTargetStates,
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/InternalRequestSignatureTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/InternalRequestSignatureTest.java
new file mode 100644
index 0000000000000..89012ee474ec6
--- /dev/null
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/InternalRequestSignatureTest.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.connect.runtime.rest;
+
+import org.apache.kafka.connect.errors.ConnectException;
+import org.apache.kafka.connect.runtime.rest.errors.BadRequestException;
+import org.eclipse.jetty.client.api.Request;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+import javax.crypto.Mac;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
+import javax.ws.rs.core.HttpHeaders;
+
+import java.util.Base64;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class InternalRequestSignatureTest {
+
+    private static final byte[] REQUEST_BODY =
+            "[{\"config\":\"value\"},{\"config\":\"other_value\"}]".getBytes();
+    private static final String SIGNATURE_ALGORITHM = "HmacSHA256";
+    private static final SecretKey KEY = new SecretKeySpec(
+        new byte[] {
+            109, 116, -111, 49, -94, 25, -103, 44, -99, -118, 53, -69, 87, -124, 5, 48,
+            89, -105, -2, 58, -92, 87, 67, 49, -125, -79, -39, -126, -51, -53, -85, 57
+        }, "HmacSHA256"
+    );
+    private static final byte[] SIGNATURE = new byte[] {
+        42, -3, 127, 57, 43, 49, -51, -43, 72, -62, -10, 120, 123, 125, 26, -65,
+        36, 72, 86, -71, -32, 13, -8, 115, 85, 73, -65, -112, 6, 68, 41, -50
+    };
+    private static final String ENCODED_SIGNATURE = Base64.getEncoder().encodeToString(SIGNATURE);
+
+    @Test
+    public void fromHeadersShouldReturnNullOnNullHeaders() {
+        assertNull(InternalRequestSignature.fromHeaders(REQUEST_BODY, null));
+    }
+
+    @Test
+    public void fromHeadersShouldReturnNullIfSignatureHeaderMissing() {
+        assertNull(InternalRequestSignature.fromHeaders(REQUEST_BODY, internalRequestHeaders(null, SIGNATURE_ALGORITHM)));
+    }
+
+    @Test
+    public void fromHeadersShouldReturnNullIfSignatureAlgorithmHeaderMissing() {
+        assertNull(InternalRequestSignature.fromHeaders(REQUEST_BODY, internalRequestHeaders(ENCODED_SIGNATURE, null)));
+    }
+
+    @Test(expected = BadRequestException.class)
+    public void fromHeadersShouldThrowExceptionOnInvalidSignatureAlgorithm() {
+        InternalRequestSignature.fromHeaders(REQUEST_BODY, internalRequestHeaders(ENCODED_SIGNATURE, "doesn'texist"));
+    }
+
+    @Test(expected = BadRequestException.class)
+    public void fromHeadersShouldThrowExceptionOnInvalidBase64Signature() {
+        InternalRequestSignature.fromHeaders(REQUEST_BODY, internalRequestHeaders("not valid base 64", SIGNATURE_ALGORITHM));
+    }
+
+    @Test
+    public void fromHeadersShouldReturnNonNullResultOnValidSignatureAndSignatureAlgorithm() {
+        InternalRequestSignature signature =
+                InternalRequestSignature.fromHeaders(REQUEST_BODY, internalRequestHeaders(ENCODED_SIGNATURE, SIGNATURE_ALGORITHM));
+        assertNotNull(signature);
+        assertNotNull(signature.keyAlgorithm());
+    }
+
+    @Test(expected = ConnectException.class)
+    public void addToRequestShouldThrowExceptionOnInvalidSignatureAlgorithm() {
+        Request request = mock(Request.class);
+        InternalRequestSignature.addToRequest(KEY, REQUEST_BODY, "doesn'texist", request);
+    }
+
+    @Test
+    public void addToRequestShouldAddHeadersOnValidSignatureAlgorithm() {
+        Request request = mock(Request.class);
+        ArgumentCaptor signatureCapture = ArgumentCaptor.forClass(String.class);
+        ArgumentCaptor signatureAlgorithmCapture = ArgumentCaptor.forClass(String.class);
+        when(request.header(
+                eq(InternalRequestSignature.SIGNATURE_HEADER),
+                signatureCapture.capture()
+            )).thenReturn(request);
+        when(request.header(
+                eq(InternalRequestSignature.SIGNATURE_ALGORITHM_HEADER),
+                signatureAlgorithmCapture.capture()
+            )).thenReturn(request);
+
+        InternalRequestSignature.addToRequest(KEY, REQUEST_BODY, SIGNATURE_ALGORITHM, request);
+
+        assertEquals(
+            "Request should have valid base 64-encoded signature added as header",
+            ENCODED_SIGNATURE,
+            signatureCapture.getValue()
+        );
+        assertEquals(
+            "Request should have provided signature algorithm added as header",
+            SIGNATURE_ALGORITHM,
+            signatureAlgorithmCapture.getValue()
+        );
+    }
+
+    @Test
+    public void testSignatureValidation() throws Exception {
+        Mac mac = Mac.getInstance(SIGNATURE_ALGORITHM);
+
+        InternalRequestSignature signature = new InternalRequestSignature(REQUEST_BODY, mac, SIGNATURE);
+        assertTrue(signature.isValid(KEY));
+
+        signature = InternalRequestSignature.fromHeaders(REQUEST_BODY, internalRequestHeaders(ENCODED_SIGNATURE, SIGNATURE_ALGORITHM));
+        assertTrue(signature.isValid(KEY));
+
+        signature = new InternalRequestSignature("[{\"different_config\":\"different_value\"}]".getBytes(), mac, SIGNATURE);
+        assertFalse(signature.isValid(KEY));
+
+        signature = new InternalRequestSignature(REQUEST_BODY, mac, "bad signature".getBytes());
+        assertFalse(signature.isValid(KEY));
+    }
+
+    private static HttpHeaders internalRequestHeaders(String signature, String signatureAlgorithm) {
+        HttpHeaders result = mock(HttpHeaders.class);
+        when(result.getHeaderString(eq(InternalRequestSignature.SIGNATURE_HEADER)))
+            .thenReturn(signature);
+        when(result.getHeaderString(eq(InternalRequestSignature.SIGNATURE_ALGORITHM_HEADER)))
+            .thenReturn(signatureAlgorithm);
+        return result;
+    }
+}
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/RestServerTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/RestServerTest.java
index 25c7a3c7a4544..ea6e98f01f7b9 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/RestServerTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/RestServerTest.java
@@ -63,7 +63,7 @@
 import static org.junit.Assert.assertTrue;
 
 @RunWith(PowerMockRunner.class)
-@PowerMockIgnore({"javax.net.ssl.*", "javax.security.*"})
+@PowerMockIgnore({"javax.net.ssl.*", "javax.security.*", "javax.crypto.*"})
 public class RestServerTest {
 
     @MockStrict
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java
index 5490df03afb6d..967b8fca07b19 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java
@@ -18,7 +18,10 @@
 
 import com.fasterxml.jackson.core.type.TypeReference;
 
+import javax.crypto.Mac;
 import javax.ws.rs.core.HttpHeaders;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.kafka.connect.errors.AlreadyExistsException;
 import org.apache.kafka.connect.errors.NotFoundException;
 import org.apache.kafka.connect.runtime.ConnectorConfig;
@@ -26,6 +29,7 @@
 import org.apache.kafka.connect.runtime.WorkerConfig;
 import org.apache.kafka.connect.runtime.distributed.NotAssignedException;
 import org.apache.kafka.connect.runtime.distributed.NotLeaderException;
+import org.apache.kafka.connect.runtime.rest.InternalRequestSignature;
 import org.apache.kafka.connect.runtime.rest.RestClient;
 import org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo;
 import org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo;
@@ -52,9 +56,11 @@
 import javax.ws.rs.core.MultivaluedMap;
 import javax.ws.rs.core.UriInfo;
 
+import java.io.IOException;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Base64;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -66,7 +72,7 @@
 
 @RunWith(PowerMockRunner.class)
 @PrepareForTest(RestClient.class)
-@PowerMockIgnore("javax.management.*")
+@PowerMockIgnore({"javax.management.*", "javax.crypto.*"})
 @SuppressWarnings("unchecked")
 public class ConnectorsResourceTest {
     // Note trailing / and that we do *not* use LEADER_URL to construct our reference values. This checks that we handle
@@ -622,27 +628,76 @@ public void testGetConnectorTaskConfigsConnectorNotFound() throws Throwable {
     }
 
     @Test
-    public void testPutConnectorTaskConfigs() throws Throwable {
+    public void testPutConnectorTaskConfigsNoInternalRequestSignature() throws Throwable {
         final Capture> cb = Capture.newInstance();
-        herder.putTaskConfigs(EasyMock.eq(CONNECTOR_NAME), EasyMock.eq(TASK_CONFIGS), EasyMock.capture(cb));
+        herder.putTaskConfigs(
+            EasyMock.eq(CONNECTOR_NAME),
+            EasyMock.eq(TASK_CONFIGS),
+            EasyMock.capture(cb),
+            EasyMock.anyObject(InternalRequestSignature.class)
+        );
         expectAndCallbackResult(cb, null);
 
         PowerMock.replayAll();
 
-        connectorsResource.putTaskConfigs(CONNECTOR_NAME, NULL_HEADERS, FORWARD, TASK_CONFIGS);
+        connectorsResource.putTaskConfigs(CONNECTOR_NAME, NULL_HEADERS, FORWARD, serializeAsBytes(TASK_CONFIGS));
+
+        PowerMock.verifyAll();
+    }
+
+    @Test
+    public void testPutConnectorTaskConfigsWithInternalRequestSignature() throws Throwable {
+        final String signatureAlgorithm = "HmacSHA256";
+        final String encodedSignature = "Kv1/OSsxzdVIwvZ4e30avyRIVrngDfhzVUm/kAZEKc4=";
+
+        final Capture> cb = Capture.newInstance();
+        final Capture signatureCapture = Capture.newInstance();
+        herder.putTaskConfigs(
+            EasyMock.eq(CONNECTOR_NAME),
+            EasyMock.eq(TASK_CONFIGS),
+            EasyMock.capture(cb),
+            EasyMock.capture(signatureCapture)
+        );
+        expectAndCallbackResult(cb, null);
+
+        HttpHeaders headers = EasyMock.mock(HttpHeaders.class);
+        EasyMock.expect(headers.getHeaderString(InternalRequestSignature.SIGNATURE_ALGORITHM_HEADER))
+            .andReturn(signatureAlgorithm)
+            .once();
+        EasyMock.expect(headers.getHeaderString(InternalRequestSignature.SIGNATURE_HEADER))
+            .andReturn(encodedSignature)
+            .once();
+
+        PowerMock.replayAll(headers);
+
+        connectorsResource.putTaskConfigs(CONNECTOR_NAME, headers, FORWARD, serializeAsBytes(TASK_CONFIGS));
 
         PowerMock.verifyAll();
+        InternalRequestSignature expectedSignature = new InternalRequestSignature(
+            serializeAsBytes(TASK_CONFIGS),
+            Mac.getInstance(signatureAlgorithm),
+            Base64.getDecoder().decode(encodedSignature)
+        );
+        assertEquals(
+            expectedSignature,
+            signatureCapture.getValue()
+        );
     }
 
     @Test(expected = NotFoundException.class)
     public void testPutConnectorTaskConfigsConnectorNotFound() throws Throwable {
         final Capture> cb = Capture.newInstance();
-        herder.putTaskConfigs(EasyMock.eq(CONNECTOR_NAME), EasyMock.eq(TASK_CONFIGS), EasyMock.capture(cb));
+        herder.putTaskConfigs(
+            EasyMock.eq(CONNECTOR_NAME),
+            EasyMock.eq(TASK_CONFIGS),
+            EasyMock.capture(cb),
+            EasyMock.anyObject(InternalRequestSignature.class)
+        );
         expectAndCallbackException(cb, new NotFoundException("not found"));
 
         PowerMock.replayAll();
 
-        connectorsResource.putTaskConfigs(CONNECTOR_NAME, NULL_HEADERS, FORWARD, TASK_CONFIGS);
+        connectorsResource.putTaskConfigs(CONNECTOR_NAME, NULL_HEADERS, FORWARD, serializeAsBytes(TASK_CONFIGS));
 
         PowerMock.verifyAll();
     }
@@ -748,6 +803,10 @@ public void testRestartTaskOwnerRedirect() throws Throwable {
         PowerMock.verifyAll();
     }
 
+    private  byte[] serializeAsBytes(final T value) throws IOException {
+        return new ObjectMapper().writeValueAsBytes(value);
+    }
+
     private   void expectAndCallbackResult(final Capture> cb, final T value) {
         PowerMock.expectLastCall().andAnswer(new IAnswer() {
             @Override
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java
index e020cee3016fd..3018677b82a9c 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java
@@ -361,6 +361,7 @@ public void testRestartTask() throws Exception {
 
         ClusterConfigState configState = new ClusterConfigState(
                 -1,
+                null,
                 Collections.singletonMap(CONNECTOR_NAME, 1),
                 Collections.singletonMap(CONNECTOR_NAME, connectorConfig),
                 Collections.singletonMap(CONNECTOR_NAME, TargetState.STARTED),
@@ -395,6 +396,7 @@ public void testRestartTaskFailureOnStart() throws Exception {
 
         ClusterConfigState configState = new ClusterConfigState(
                 -1,
+                null,
                 Collections.singletonMap(CONNECTOR_NAME, 1),
                 Collections.singletonMap(CONNECTOR_NAME, connectorConfig),
                 Collections.singletonMap(CONNECTOR_NAME, TargetState.STARTED),
@@ -570,7 +572,8 @@ public void testPutTaskConfigs() {
 
         herder.putTaskConfigs(CONNECTOR_NAME,
                 Arrays.asList(singletonMap("config", "value")),
-                cb);
+                cb,
+                null);
 
         PowerMock.verifyAll();
     }
@@ -645,6 +648,7 @@ private void expectAdd(SourceSink sourceSink) {
 
         ClusterConfigState configState = new ClusterConfigState(
                 -1,
+                null,
                 Collections.singletonMap(CONNECTOR_NAME, 1),
                 Collections.singletonMap(CONNECTOR_NAME, connectorConfig(sourceSink)),
                 Collections.singletonMap(CONNECTOR_NAME, TargetState.STARTED),
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java
index 8e358e0670958..03ce9eb95e394 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java
@@ -62,7 +62,7 @@
 
 @RunWith(PowerMockRunner.class)
 @PrepareForTest(KafkaConfigBackingStore.class)
-@PowerMockIgnore("javax.management.*")
+@PowerMockIgnore({"javax.management.*", "javax.crypto.*"})
 @SuppressWarnings({"unchecked", "deprecation"})
 public class KafkaConfigBackingStoreTest {
     private static final String TOPIC = "connect-configs";
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStoreTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStoreTest.java
index ff9f2c9d1ba2f..1df63f35e8a76 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStoreTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStoreTest.java
@@ -59,7 +59,7 @@
 
 @RunWith(PowerMockRunner.class)
 @PrepareForTest(KafkaOffsetBackingStore.class)
-@PowerMockIgnore("javax.management.*")
+@PowerMockIgnore({"javax.management.*", "javax.crypto.*"})
 @SuppressWarnings({"unchecked", "deprecation"})
 public class KafkaOffsetBackingStoreTest {
     private static final String TOPIC = "connect-offsets";
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java
index 49a31ae009dc6..94cc880234663 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java
@@ -384,6 +384,28 @@ public int executePut(String url, String body) throws IOException {
         return httpCon.getResponseCode();
     }
 
+    public int executePost(String url, String body, Map headers) throws IOException {
+        log.debug("Executing POST request to URL={}. Payload={}", url, body);
+        HttpURLConnection httpCon = (HttpURLConnection) new URL(url).openConnection();
+        httpCon.setDoOutput(true);
+        httpCon.setRequestProperty("Content-Type", "application/json");
+        headers.forEach(httpCon::setRequestProperty);
+        httpCon.setRequestMethod("POST");
+        try (OutputStreamWriter out = new OutputStreamWriter(httpCon.getOutputStream())) {
+            out.write(body);
+        }
+        if (httpCon.getResponseCode() < HttpURLConnection.HTTP_BAD_REQUEST) {
+            try (InputStream is = httpCon.getInputStream()) {
+                log.info("POST response for URL={} is {}", url, responseToString(is));
+            }
+        } else {
+            try (InputStream is = httpCon.getErrorStream()) {
+                log.info("POST error response for URL={} is {}", url, responseToString(is));
+            }
+        }
+        return httpCon.getResponseCode();
+    }
+
     /**
      * Execute a GET request on the given URL.
      *
diff --git a/tests/kafkatest/tests/connect/connect_distributed_test.py b/tests/kafkatest/tests/connect/connect_distributed_test.py
index ada6ca27b4875..04e98e9c1777a 100644
--- a/tests/kafkatest/tests/connect/connect_distributed_test.py
+++ b/tests/kafkatest/tests/connect/connect_distributed_test.py
@@ -54,7 +54,7 @@ class ConnectDistributedTest(Test):
     STATUS_REPLICATION_FACTOR = "1"
     STATUS_PARTITIONS = "1"
     SCHEDULED_REBALANCE_MAX_DELAY_MS = "60000"
-    CONNECT_PROTOCOL="compatible"
+    CONNECT_PROTOCOL="sessioned"
 
     # Since tasks can be assigned to any node and we're testing with files, we need to make sure the content is the same
     # across all nodes.
@@ -157,7 +157,7 @@ def task_is_running(self, connector, task_id, node=None):
         return self._task_has_state(task_id, status, 'RUNNING')
 
     @cluster(num_nodes=5)
-    @matrix(connect_protocol=['compatible', 'eager'])
+    @matrix(connect_protocol=['sessioned', 'compatible', 'eager'])
     def test_restart_failed_connector(self, connect_protocol):
         self.CONNECT_PROTOCOL = connect_protocol
         self.setup_services()
@@ -176,7 +176,7 @@ def test_restart_failed_connector(self, connect_protocol):
                    err_msg="Failed to see connector transition to the RUNNING state")
 
     @cluster(num_nodes=5)
-    @matrix(connector_type=['source', 'sink'], connect_protocol=['compatible', 'eager'])
+    @matrix(connector_type=['source', 'sink'], connect_protocol=['sessioned', 'compatible', 'eager'])
     def test_restart_failed_task(self, connector_type, connect_protocol):
         self.CONNECT_PROTOCOL = connect_protocol
         self.setup_services()
@@ -201,7 +201,7 @@ def test_restart_failed_task(self, connector_type, connect_protocol):
                    err_msg="Failed to see task transition to the RUNNING state")
 
     @cluster(num_nodes=5)
-    @matrix(connect_protocol=['compatible', 'eager'])
+    @matrix(connect_protocol=['sessioned', 'compatible', 'eager'])
     def test_pause_and_resume_source(self, connect_protocol):
         """
         Verify that source connectors stop producing records when paused and begin again after
@@ -242,7 +242,7 @@ def test_pause_and_resume_source(self, connect_protocol):
                    err_msg="Failed to produce messages after resuming source connector")
 
     @cluster(num_nodes=5)
-    @matrix(connect_protocol=['compatible', 'eager'])
+    @matrix(connect_protocol=['sessioned', 'compatible', 'eager'])
     def test_pause_and_resume_sink(self, connect_protocol):
         """
         Verify that sink connectors stop consuming records when paused and begin again after
@@ -290,7 +290,7 @@ def test_pause_and_resume_sink(self, connect_protocol):
                    err_msg="Failed to consume messages after resuming sink connector")
 
     @cluster(num_nodes=5)
-    @matrix(connect_protocol=['compatible', 'eager'])
+    @matrix(connect_protocol=['sessioned', 'compatible', 'eager'])
     def test_pause_state_persistent(self, connect_protocol):
         """
         Verify that paused state is preserved after a cluster restart.
@@ -322,7 +322,7 @@ def test_pause_state_persistent(self, connect_protocol):
                        err_msg="Failed to see connector startup in PAUSED state")
 
     @cluster(num_nodes=6)
-    @matrix(security_protocol=[SecurityConfig.PLAINTEXT, SecurityConfig.SASL_SSL], connect_protocol=['compatible', 'eager'])
+    @matrix(security_protocol=[SecurityConfig.PLAINTEXT, SecurityConfig.SASL_SSL], connect_protocol=['sessioned', 'compatible', 'eager'])
     def test_file_source_and_sink(self, security_protocol, connect_protocol):
         """
         Tests that a basic file connector works across clean rolling bounces. This validates that the connector is
@@ -360,7 +360,7 @@ def test_file_source_and_sink(self, security_protocol, connect_protocol):
         wait_until(lambda: self._validate_file_output(self.FIRST_INPUT_LIST + self.SECOND_INPUT_LIST), timeout_sec=timeout_sec, err_msg="Sink output file never converged to the same state as the input file")
 
     @cluster(num_nodes=6)
-    @matrix(clean=[True, False], connect_protocol=['compatible', 'eager'])
+    @matrix(clean=[True, False], connect_protocol=['sessioned', 'compatible', 'eager'])
     def test_bounce(self, clean, connect_protocol):
         """
         Validates that source and sink tasks that run continuously and produce a predictable sequence of messages
@@ -470,7 +470,7 @@ def test_bounce(self, clean, connect_protocol):
         assert success, "Found validation errors:\n" + "\n  ".join(errors)
 
     @cluster(num_nodes=6)
-    @matrix(connect_protocol=['compatible', 'eager'])
+    @matrix(connect_protocol=['sessioned', 'compatible', 'eager'])
     def test_transformations(self, connect_protocol):
         self.CONNECT_PROTOCOL = connect_protocol
         self.setup_services(timestamp_type='CreateTime')
@@ -527,6 +527,11 @@ def test_transformations(self, connect_protocol):
             assert obj['payload'][ts_fieldname] == ts
 
     @cluster(num_nodes=5)
+    @parametrize(broker_version=str(DEV_BRANCH), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='sessioned')
+    @parametrize(broker_version=str(LATEST_0_11_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='sessioned')
+    @parametrize(broker_version=str(LATEST_0_10_2), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='sessioned')
+    @parametrize(broker_version=str(LATEST_0_10_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='sessioned')
+    @parametrize(broker_version=str(LATEST_0_10_0), auto_create_topics=True, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='sessioned')
     @parametrize(broker_version=str(DEV_BRANCH), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible')
     @parametrize(broker_version=str(LATEST_2_3), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible')
     @parametrize(broker_version=str(LATEST_2_2), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible')
diff --git a/tests/kafkatest/tests/connect/templates/connect-distributed.properties b/tests/kafkatest/tests/connect/templates/connect-distributed.properties
index e31a54a27a14f..6d2d5e28d1035 100644
--- a/tests/kafkatest/tests/connect/templates/connect-distributed.properties
+++ b/tests/kafkatest/tests/connect/templates/connect-distributed.properties
@@ -20,7 +20,7 @@ bootstrap.servers={{ kafka.bootstrap_servers(kafka.security_config.security_prot
 
 group.id={{ group|default("connect-cluster") }}
 
-connect.protocol={{ CONNECT_PROTOCOL|default("compatible") }}
+connect.protocol={{ CONNECT_PROTOCOL|default("sessioned") }}
 scheduled.rebalance.max.delay.ms={{ SCHEDULED_REBALANCE_MAX_DELAY_MS|default(60000) }}
 
 key.converter={{ key_converter|default("org.apache.kafka.connect.json.JsonConverter") }}

From 11ab6e7d8fbdf213a22b7fa5dff04f374f55a1d2 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Wed, 2 Oct 2019 17:05:24 -0700
Subject: [PATCH 0678/1071] HOTFIX: remove unsued StreamsConfig from
 StreamsPartitionAssignor

---
 .../streams/processor/internals/StreamsPartitionAssignor.java    | 1 -
 1 file changed, 1 deletion(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
index 846617143a978..2e6c9c081d52f 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
@@ -27,7 +27,6 @@
 import org.apache.kafka.common.config.ConfigException;
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.common.utils.Utils;
-import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.errors.StreamsException;
 import org.apache.kafka.streams.errors.TaskAssignmentException;
 import org.apache.kafka.streams.processor.TaskId;

From 5b829f2b610d71d3176a7df523b34df807b47f22 Mon Sep 17 00:00:00 2001
From: Cyrus Vafadari 
Date: Wed, 2 Oct 2019 17:42:21 -0700
Subject: [PATCH 0679/1071] KAFKA-8447: New Metric to Measure Number of Tasks
 on a Connector (#6843)

Implemented KIP-475 to add new metrics for each worker to expose the number of current tasks per connector and per status.

Author: Cyrus Vafadari 
Reviewer: Randall Hauch , Boyang Chen 
---
 .../runtime/ConnectMetricsRegistry.java       |  35 ++++
 .../apache/kafka/connect/runtime/Worker.java  |  97 ++++++++--
 .../connect/runtime/WorkerConnector.java      |   8 +-
 .../kafka/connect/runtime/WorkerTest.java     | 179 ++++++++++++++++++
 4 files changed, 299 insertions(+), 20 deletions(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectMetricsRegistry.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectMetricsRegistry.java
index 44eba5ca5c044..6996dacb390a7 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectMetricsRegistry.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectMetricsRegistry.java
@@ -20,8 +20,10 @@
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 public class ConnectMetricsRegistry {
@@ -41,6 +43,12 @@ public class ConnectMetricsRegistry {
     public final MetricNameTemplate connectorType;
     public final MetricNameTemplate connectorClass;
     public final MetricNameTemplate connectorVersion;
+    public final MetricNameTemplate connectorTotalTaskCount;
+    public final MetricNameTemplate connectorRunningTaskCount;
+    public final MetricNameTemplate connectorPausedTaskCount;
+    public final MetricNameTemplate connectorFailedTaskCount;
+    public final MetricNameTemplate connectorUnassignedTaskCount;
+    public final MetricNameTemplate connectorDestroyedTaskCount;
     public final MetricNameTemplate taskStatus;
     public final MetricNameTemplate taskRunningRatio;
     public final MetricNameTemplate taskPauseRatio;
@@ -104,6 +112,8 @@ public class ConnectMetricsRegistry {
     public final MetricNameTemplate dlqProduceFailures;
     public final MetricNameTemplate lastErrorTimestamp;
 
+    public Map connectorStatusMetrics;
+
     public ConnectMetricsRegistry() {
         this(new LinkedHashSet());
     }
@@ -289,6 +299,31 @@ public ConnectMetricsRegistry(Set tags) {
         taskStartupFailurePercentage = createTemplate("task-startup-failure-percentage", WORKER_GROUP_NAME,
                                                       "The average percentage of this worker's tasks starts that failed.", workerTags);
 
+        Set workerConnectorTags = new LinkedHashSet<>(tags);
+        workerConnectorTags.add(CONNECTOR_TAG_NAME);
+        connectorTotalTaskCount = createTemplate("connector-total-task-count", WORKER_GROUP_NAME,
+            "The number of tasks of the connector on the worker.", workerConnectorTags);
+        connectorRunningTaskCount = createTemplate("connector-running-task-count", WORKER_GROUP_NAME,
+            "The number of running tasks of the connector on the worker.", workerConnectorTags);
+        connectorPausedTaskCount = createTemplate("connector-paused-task-count", WORKER_GROUP_NAME,
+            "The number of paused tasks of the connector on the worker.", workerConnectorTags);
+        connectorFailedTaskCount = createTemplate("connector-failed-task-count", WORKER_GROUP_NAME,
+            "The number of failed tasks of the connector on the worker.", workerConnectorTags);
+        connectorUnassignedTaskCount = createTemplate("connector-unassigned-task-count",
+            WORKER_GROUP_NAME,
+            "The number of unassigned tasks of the connector on the worker.", workerConnectorTags);
+        connectorDestroyedTaskCount = createTemplate("connector-destroyed-task-count",
+            WORKER_GROUP_NAME,
+            "The number of destroyed tasks of the connector on the worker.", workerConnectorTags);
+
+        connectorStatusMetrics = new HashMap<>();
+        connectorStatusMetrics.put(connectorRunningTaskCount, TaskStatus.State.RUNNING);
+        connectorStatusMetrics.put(connectorPausedTaskCount, TaskStatus.State.PAUSED);
+        connectorStatusMetrics.put(connectorFailedTaskCount, TaskStatus.State.FAILED);
+        connectorStatusMetrics.put(connectorUnassignedTaskCount, TaskStatus.State.UNASSIGNED);
+        connectorStatusMetrics.put(connectorDestroyedTaskCount, TaskStatus.State.DESTROYED);
+        connectorStatusMetrics = Collections.unmodifiableMap(connectorStatusMetrics);
+
         /***** Worker rebalance level *****/
         Set rebalanceTags = new LinkedHashSet<>(tags);
 
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
index 52b6f41cc77db..27955477fc573 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
@@ -21,6 +21,7 @@
 import org.apache.kafka.clients.producer.KafkaProducer;
 import org.apache.kafka.clients.producer.ProducerConfig;
 import org.apache.kafka.common.MetricName;
+import org.apache.kafka.common.MetricNameTemplate;
 import org.apache.kafka.common.config.ConfigValue;
 import org.apache.kafka.common.config.provider.ConfigProvider;
 import org.apache.kafka.common.metrics.Sensor;
@@ -35,7 +36,6 @@
 import org.apache.kafka.connect.connector.policy.ConnectorClientConfigRequest;
 import org.apache.kafka.connect.errors.ConnectException;
 import org.apache.kafka.connect.health.ConnectorType;
-import org.apache.kafka.connect.runtime.ConnectMetrics.LiteralSupplier;
 import org.apache.kafka.connect.runtime.ConnectMetrics.MetricGroup;
 import org.apache.kafka.connect.runtime.distributed.ClusterConfigState;
 import org.apache.kafka.connect.runtime.errors.DeadLetterQueueReporter;
@@ -94,6 +94,7 @@ public class Worker {
     private final Plugins plugins;
     private final ConnectMetrics metrics;
     private final WorkerMetricsGroup workerMetricsGroup;
+    private ConnectorStatusMetricsGroup connectorStatusMetricsGroup;
     private final WorkerConfig config;
     private final Converter internalKeyConverter;
     private final Converter internalValueConverter;
@@ -184,6 +185,8 @@ public void start() {
         offsetBackingStore.start();
         sourceTaskOffsetCommitter = new SourceTaskOffsetCommitter(config);
 
+        connectorStatusMetricsGroup = new ConnectorStatusMetricsGroup(metrics, tasks, herder);
+
         log.info("Worker started");
     }
 
@@ -215,6 +218,7 @@ public void stop() {
         log.info("Worker stopped");
 
         workerMetricsGroup.close();
+        connectorStatusMetricsGroup.close();
     }
 
     /**
@@ -416,6 +420,7 @@ public boolean startTask(
             if (tasks.containsKey(id))
                 throw new ConnectException("Task already exists in this worker: " + id);
 
+            connectorStatusMetricsGroup.recordTaskAdded(id);
             ClassLoader savedLoader = plugins.currentThreadLoader();
             try {
                 final ConnectorConfig connConfig = new ConnectorConfig(plugins, connProps);
@@ -465,6 +470,7 @@ public boolean startTask(
                 // Can't be put in a finally block because it needs to be swapped before the call on
                 // statusListener
                 Plugins.compareAndSwapLoaders(savedLoader);
+                connectorStatusMetricsGroup.recordTaskRemoved(id);
                 workerMetricsGroup.recordTaskFailure();
                 statusListener.onFailure(id, t);
                 return false;
@@ -706,6 +712,7 @@ private void stopTasks(Collection ids) {
     private void awaitStopTask(ConnectorTaskId taskId, long timeout) {
         try (LoggingContext loggingContext = LoggingContext.forTask(taskId)) {
             WorkerTask task = tasks.remove(taskId);
+            connectorStatusMetricsGroup.recordTaskRemoved(taskId);
             if (task == null) {
                 log.warn("Ignoring await stop request for non-present task {}", taskId);
                 return;
@@ -824,10 +831,84 @@ private void transitionTo(Object connectorOrTask, TargetState state, ClassLoader
         }
     }
 
+    ConnectorStatusMetricsGroup connectorStatusMetricsGroup() {
+        return connectorStatusMetricsGroup;
+    }
+
     WorkerMetricsGroup workerMetricsGroup() {
         return workerMetricsGroup;
     }
 
+    static class ConnectorStatusMetricsGroup {
+        private ConnectMetrics connectMetrics;
+        private ConnectMetricsRegistry registry;
+        private ConcurrentMap connectorStatusMetrics = new ConcurrentHashMap<>();
+        private Herder herder;
+        private ConcurrentMap tasks;
+
+
+        protected ConnectorStatusMetricsGroup(
+            ConnectMetrics connectMetrics, ConcurrentMap tasks, Herder herder) {
+            this.connectMetrics = connectMetrics;
+            this.registry = connectMetrics.registry();
+            this.tasks = tasks;
+            this.herder = herder;
+        }
+
+        protected ConnectMetrics.LiteralSupplier taskCounter(String connName) {
+            return now -> tasks.keySet()
+                .stream()
+                .filter(taskId -> taskId.connector().equals(connName))
+                .count();
+        }
+
+        protected ConnectMetrics.LiteralSupplier taskStatusCounter(String connName, TaskStatus.State state) {
+            return now -> tasks.values()
+                .stream()
+                .filter(task ->
+                    task.id().connector().equals(connName) &&
+                    herder.taskStatus(task.id()).state().equalsIgnoreCase(state.toString()))
+                .count();
+        }
+
+        protected synchronized void recordTaskAdded(ConnectorTaskId connectorTaskId) {
+            if (connectorStatusMetrics.containsKey(connectorTaskId.connector())) {
+                return;
+            }
+
+            String connName = connectorTaskId.connector();
+
+            MetricGroup metricGroup = connectMetrics.group(registry.workerGroupName(),
+                registry.connectorTagName(), connName);
+
+            metricGroup.addValueMetric(registry.connectorTotalTaskCount, taskCounter(connName));
+            for (Map.Entry statusMetric : registry.connectorStatusMetrics
+                .entrySet()) {
+                metricGroup.addValueMetric(statusMetric.getKey(), taskStatusCounter(connName,
+                    statusMetric.getValue()));
+            }
+            connectorStatusMetrics.put(connectorTaskId.connector(), metricGroup);
+        }
+
+        protected synchronized void recordTaskRemoved(ConnectorTaskId connectorTaskId) {
+            // Unregister connector task count metric if we remove the last task of the connector
+            if (tasks.keySet().stream().noneMatch(id -> id.connector().equals(connectorTaskId.connector()))) {
+                connectorStatusMetrics.get(connectorTaskId.connector()).close();
+                connectorStatusMetrics.remove(connectorTaskId.connector());
+            }
+        }
+
+        protected synchronized void close() {
+            for (MetricGroup metricGroup: connectorStatusMetrics.values()) {
+                metricGroup.close();
+            }
+        }
+
+        protected MetricGroup metricGroup(String connectorId) {
+            return connectorStatusMetrics.get(connectorId);
+        }
+    }
+
     class WorkerMetricsGroup {
         private final MetricGroup metricGroup;
         private final Sensor connectorStartupAttempts;
@@ -843,18 +924,8 @@ public WorkerMetricsGroup(ConnectMetrics connectMetrics) {
             ConnectMetricsRegistry registry = connectMetrics.registry();
             metricGroup = connectMetrics.group(registry.workerGroupName());
 
-            metricGroup.addValueMetric(registry.connectorCount, new LiteralSupplier() {
-                @Override
-                public Double metricValue(long now) {
-                    return (double) connectors.size();
-                }
-            });
-            metricGroup.addValueMetric(registry.taskCount, new LiteralSupplier() {
-                @Override
-                public Double metricValue(long now) {
-                    return (double) tasks.size();
-                }
-            });
+            metricGroup.addValueMetric(registry.connectorCount, now -> (double) connectors.size());
+            metricGroup.addValueMetric(registry.taskCount, now -> (double) tasks.size());
 
             MetricName connectorFailurePct = metricGroup.metricName(registry.connectorStartupFailurePercentage);
             MetricName connectorSuccessPct = metricGroup.metricName(registry.connectorStartupSuccessPercentage);
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java
index e5b990fc7abf7..8ec93cfe370be 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java
@@ -18,7 +18,6 @@
 
 import org.apache.kafka.connect.connector.Connector;
 import org.apache.kafka.connect.connector.ConnectorContext;
-import org.apache.kafka.connect.runtime.ConnectMetrics.LiteralSupplier;
 import org.apache.kafka.connect.runtime.ConnectMetrics.MetricGroup;
 import org.apache.kafka.connect.sink.SinkConnector;
 import org.apache.kafka.connect.source.SourceConnector;
@@ -257,12 +256,7 @@ public ConnectorMetricsGroup(ConnectMetrics connectMetrics, AbstractStatus.State
             metricGroup.addImmutableValueMetric(registry.connectorType, connectorType());
             metricGroup.addImmutableValueMetric(registry.connectorClass, connector.getClass().getName());
             metricGroup.addImmutableValueMetric(registry.connectorVersion, connector.version());
-            metricGroup.addValueMetric(registry.connectorStatus, new LiteralSupplier() {
-                @Override
-                public String metricValue(long now) {
-                    return state.toString().toLowerCase(Locale.getDefault());
-                }
-            });
+            metricGroup.addValueMetric(registry.connectorStatus, now -> state.toString().toLowerCase(Locale.getDefault()));
         }
 
         public void close() {
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java
index 9cb83eb5e8771..36a9b66d7cf04 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java
@@ -45,6 +45,7 @@
 import org.apache.kafka.connect.runtime.isolation.PluginClassLoader;
 import org.apache.kafka.connect.runtime.isolation.Plugins;
 import org.apache.kafka.connect.runtime.isolation.Plugins.ClassLoaderUsage;
+import org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo;
 import org.apache.kafka.connect.runtime.standalone.StandaloneConfig;
 import org.apache.kafka.connect.sink.SinkTask;
 import org.apache.kafka.connect.source.SourceRecord;
@@ -75,6 +76,8 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
 
 import static org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperatorTest.NOOP_OPERATOR;
@@ -83,6 +86,7 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.fail;
 
 @RunWith(PowerMockRunner.class)
@@ -116,6 +120,7 @@ public class WorkerTest extends ThreadedTest {
     @MockStrict
     private ConnectorStatus.Listener connectorStatusListener;
 
+    @Mock private Herder herder;
     @Mock private Connector connector;
     @Mock private ConnectorContext ctx;
     @Mock private TestSourceTask task;
@@ -598,6 +603,171 @@ public void testAddRemoveTask() throws Exception {
         PowerMock.verifyAll();
     }
 
+    @Test
+    public void testTaskStatusMetricsStatuses() throws Exception {
+        expectConverters();
+        expectStartStorage();
+
+        EasyMock.expect(workerTask.id()).andStubReturn(TASK_ID);
+
+        EasyMock.expect(plugins.currentThreadLoader()).andReturn(delegatingLoader).times(2);
+        PowerMock.expectNew(WorkerSourceTask.class, EasyMock.eq(TASK_ID),
+            EasyMock.eq(task),
+            anyObject(TaskStatus.Listener.class),
+            EasyMock.eq(TargetState.STARTED),
+            anyObject(JsonConverter.class),
+            anyObject(JsonConverter.class),
+            anyObject(JsonConverter.class),
+            EasyMock.eq(new TransformationChain<>(Collections.emptyList(), NOOP_OPERATOR)),
+            anyObject(KafkaProducer.class),
+            anyObject(OffsetStorageReader.class),
+            anyObject(OffsetStorageWriter.class),
+            EasyMock.eq(config),
+            anyObject(ClusterConfigState.class),
+            anyObject(ConnectMetrics.class),
+            anyObject(ClassLoader.class),
+            anyObject(Time.class),
+            anyObject(RetryWithToleranceOperator.class)).andReturn(workerTask);
+        Map origProps = new HashMap<>();
+        origProps.put(TaskConfig.TASK_CLASS_CONFIG, TestSourceTask.class.getName());
+
+        TaskConfig taskConfig = new TaskConfig(origProps);
+        // We should expect this call, but the pluginLoader being swapped in is only mocked.
+        // EasyMock.expect(pluginLoader.loadClass(TestSourceTask.class.getName()))
+        //        .andReturn((Class) TestSourceTask.class);
+        EasyMock.expect(plugins.newTask(TestSourceTask.class)).andReturn(task);
+        EasyMock.expect(task.version()).andReturn("1.0");
+
+        workerTask.initialize(taskConfig);
+        EasyMock.expectLastCall();
+
+        // Expect that the worker will create converters and will find them using the current classloader ...
+        assertNotNull(taskKeyConverter);
+        assertNotNull(taskValueConverter);
+        assertNotNull(taskHeaderConverter);
+        expectTaskKeyConverters(ClassLoaderUsage.CURRENT_CLASSLOADER, taskKeyConverter);
+        expectTaskValueConverters(ClassLoaderUsage.CURRENT_CLASSLOADER, taskValueConverter);
+        expectTaskHeaderConverter(ClassLoaderUsage.CURRENT_CLASSLOADER, taskHeaderConverter);
+
+        EasyMock.expect(executorService.submit(workerTask)).andReturn(null);
+
+        EasyMock.expect(plugins.delegatingLoader()).andReturn(delegatingLoader);
+        EasyMock.expect(delegatingLoader.connectorLoader(WorkerTestConnector.class.getName()))
+            .andReturn(pluginLoader);
+        EasyMock.expect(Plugins.compareAndSwapLoaders(pluginLoader)).andReturn(delegatingLoader)
+            .times(2);
+
+        EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader)
+            .times(2);
+        plugins.connectorClass(WorkerTestConnector.class.getName());
+        EasyMock.expectLastCall().andReturn(WorkerTestConnector.class);
+
+        EasyMock.expect(workerTask.awaitStop(EasyMock.anyLong())).andStubReturn(true);
+        EasyMock.expectLastCall();
+
+        // Each time we check the task metrics, the worker will call the herder
+        herder.taskStatus(TASK_ID);
+        EasyMock.expectLastCall()
+            .andReturn(new ConnectorStateInfo.TaskState(0, "RUNNING", "worker", "msg"));
+
+        herder.taskStatus(TASK_ID);
+        EasyMock.expectLastCall()
+            .andReturn(new ConnectorStateInfo.TaskState(0, "PAUSED", "worker", "msg"));
+
+        herder.taskStatus(TASK_ID);
+        EasyMock.expectLastCall()
+            .andReturn(new ConnectorStateInfo.TaskState(0, "FAILED", "worker", "msg"));
+
+        herder.taskStatus(TASK_ID);
+        EasyMock.expectLastCall()
+            .andReturn(new ConnectorStateInfo.TaskState(0, "DESTROYED", "worker", "msg"));
+
+        herder.taskStatus(TASK_ID);
+        EasyMock.expectLastCall()
+            .andReturn(new ConnectorStateInfo.TaskState(0, "UNASSIGNED", "worker", "msg"));
+
+        // Called when we stop the worker
+        EasyMock.expect(workerTask.loader()).andReturn(pluginLoader);
+        workerTask.stop();
+        EasyMock.expectLastCall();
+
+        PowerMock.replayAll();
+
+        worker = new Worker(WORKER_ID,
+            new MockTime(),
+            plugins,
+            config,
+            offsetBackingStore,
+            executorService,
+            noneConnectorClientConfigOverridePolicy);
+
+        worker.herder = herder;
+
+        worker.start();
+        assertStatistics(worker, 0, 0);
+        assertStartupStatistics(worker, 0, 0, 0, 0);
+        assertEquals(Collections.emptySet(), worker.taskIds());
+        worker.startTask(
+            TASK_ID,
+            ClusterConfigState.EMPTY,
+            anyConnectorConfigMap(),
+            origProps,
+            taskStatusListener,
+            TargetState.STARTED);
+
+        assertStatusMetrics(1L, "connector-running-task-count");
+        assertStatusMetrics(1L, "connector-paused-task-count");
+        assertStatusMetrics(1L, "connector-failed-task-count");
+        assertStatusMetrics(1L, "connector-destroyed-task-count");
+        assertStatusMetrics(1L, "connector-unassigned-task-count");
+
+        worker.stopAndAwaitTask(TASK_ID);
+        assertStatusMetrics(0L, "connector-running-task-count");
+        assertStatusMetrics(0L, "connector-paused-task-count");
+        assertStatusMetrics(0L, "connector-failed-task-count");
+        assertStatusMetrics(0L, "connector-destroyed-task-count");
+        assertStatusMetrics(0L, "connector-unassigned-task-count");
+
+        PowerMock.verifyAll();
+    }
+
+    @Test
+    public void testConnectorStatusMetricsGroup_taskStatusCounter() {
+        ConcurrentMap tasks = new ConcurrentHashMap<>();
+        tasks.put(new ConnectorTaskId("c1", 0), workerTask);
+        tasks.put(new ConnectorTaskId("c1", 1), workerTask);
+        tasks.put(new ConnectorTaskId("c2", 0), workerTask);
+
+
+        expectConverters();
+
+        expectStartStorage();
+
+        EasyMock.expect(Plugins.compareAndSwapLoaders(pluginLoader)).andReturn(delegatingLoader);
+        EasyMock.expect(Plugins.compareAndSwapLoaders(pluginLoader)).andReturn(delegatingLoader);
+
+        EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader);
+
+        taskStatusListener.onFailure(EasyMock.eq(TASK_ID), EasyMock.anyObject());
+        EasyMock.expectLastCall();
+
+        PowerMock.replayAll();
+
+        worker = new Worker(WORKER_ID,
+            new MockTime(),
+            plugins,
+            config,
+            offsetBackingStore,
+            noneConnectorClientConfigOverridePolicy);
+
+        Worker.ConnectorStatusMetricsGroup metricGroup = new Worker.ConnectorStatusMetricsGroup(
+            worker.metrics(), tasks, herder
+        );
+        assertEquals(2L, (long) metricGroup.taskCounter("c1").metricValue(0L));
+        assertEquals(1L, (long) metricGroup.taskCounter("c2").metricValue(0L));
+        assertEquals(0L, (long) metricGroup.taskCounter("fakeConnector").metricValue(0L));
+    }
+
     @Test
     public void testStartTaskFailure() {
         expectConverters();
@@ -1004,8 +1174,17 @@ public void testAdminConfigsClientOverridesWithNonePolicy() {
 
     }
 
+    private void assertStatusMetrics(long expected, String metricName) {
+        MetricGroup statusMetrics = worker.connectorStatusMetricsGroup().metricGroup(TASK_ID.connector());
+        if (expected == 0L) {
+            assertNull(statusMetrics);
+            return;
+        }
+        assertEquals(expected, (long) MockConnectMetrics.currentMetricValue(worker.metrics(), statusMetrics, metricName));
+    }
 
     private void assertStatistics(Worker worker, int connectors, int tasks) {
+        assertStatusMetrics((long) tasks, "connector-total-task-count");
         MetricGroup workerMetrics = worker.workerMetricsGroup().metricGroup();
         assertEquals(connectors, MockConnectMetrics.currentMetricValueAsDouble(worker.metrics(), workerMetrics, "connector-count"), 0.0001d);
         assertEquals(tasks, MockConnectMetrics.currentMetricValueAsDouble(worker.metrics(), workerMetrics, "task-count"), 0.0001d);

From 6925775e63fd33e6a44bbda671b2de7db41d150e Mon Sep 17 00:00:00 2001
From: Bill Bejeck 
Date: Wed, 2 Oct 2019 23:32:18 -0400
Subject: [PATCH 0680/1071] KAFKA-8558:  Add StreamJoined config object to join
 (#7285)

Reviewer: John Roesler , Matthias J. Sax 
---
 .../apache/kafka/streams/kstream/KStream.java | 279 ++++++++++++++++-
 .../kafka/streams/kstream/StreamJoined.java   | 287 ++++++++++++++++++
 .../kstream/internals/KStreamImpl.java        | 237 +++++----------
 .../kstream/internals/KStreamImplJoin.java    | 214 +++++++++++++
 .../internals/StreamJoinedInternal.java       |  60 ++++
 .../InMemoryWindowBytesStoreSupplier.java     |  10 +
 .../kafka/streams/StreamsBuilderTest.java     | 145 ++++++---
 .../RepartitionOptimizingIntegrationTest.java |   4 +-
 .../StreamStreamJoinIntegrationTest.java      |  38 +++
 .../kstream/RepartitionTopicNamingTest.java   |   2 +-
 .../kstream/internals/KStreamImplTest.java    |  28 +-
 .../internals/KStreamKStreamJoinTest.java     | 210 ++++++++++++-
 .../internals/KStreamKStreamLeftJoinTest.java |   8 +-
 .../streams/tests/StreamsOptimizedTest.java   |   4 +-
 .../streams/scala/ImplicitConversions.scala   |   5 +
 .../kafka/streams/scala/kstream/KStream.scala |  36 ++-
 .../streams/scala/kstream/StreamJoined.scala  |  90 ++++++
 .../kafka/streams/scala/kstream/package.scala |   1 +
 .../kafka/streams/scala/TopologyTest.scala    |  10 +-
 .../scala/kstream/StreamJoinedTest.scala      |  69 +++++
 20 files changed, 1480 insertions(+), 257 deletions(-)
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/StreamJoined.java
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImplJoin.java
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/StreamJoinedInternal.java
 create mode 100644 streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/StreamJoined.scala
 create mode 100644 streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/kstream/StreamJoinedTest.scala

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java
index b1d8422fd281a..b4e02253bf256 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java
@@ -76,7 +76,6 @@ public interface KStream {
      */
     KStream filter(final Predicate predicate, final Named named);
 
-
     /**
      * Create a new {@code KStream} that consists all records of this stream which do not satisfy the given
      * predicate.
@@ -280,7 +279,6 @@  KStream map(final KeyValueMapper KStream mapValues(final ValueMapper mapper);
 
-
     /**
      * Transform the value of each input record into a new value (with possible new type) of the output record.
      * The provided {@link ValueMapper} is applied to each input record value and computes a new value for it.
@@ -2128,7 +2126,7 @@ void process(final ProcessorSupplier processorSupplier,
      * Because a new key is selected, an internal repartitioning topic may need to be created in Kafka if a
      * later operator depends on the newly selected key.
      * This topic will be named "${applicationId}-<name>-repartition", where "applicationId" is user-specified in
-     * {@link  StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG},
+     * {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG},
      * "<name>" is an internally generated name, and "-repartition" is a fixed suffix.
      * 
                
      * You can retrieve all generated internal topic names via {@link Topology#describe()}.
@@ -2157,7 +2155,7 @@ void process(final ProcessorSupplier processorSupplier,
      * Because a new key is selected, an internal repartitioning topic may need to be created in Kafka if a
      * later operator depends on the newly selected key.
      * This topic will be as "${applicationId}-<name>-repartition", where "applicationId" is user-specified in
-     * {@link  StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG},
+     * {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG},
      * "<name>" is an internally generated name, and "-repartition" is a fixed suffix.
      * 
                
      * You can retrieve all generated internal topic names via {@link Topology#describe()}.
@@ -2189,7 +2187,7 @@  KGroupedStream groupBy(final KeyValueMapper
@@ -2253,7 +2251,7 @@  KGroupedStream groupBy(final KeyValueMapper
@@ -2264,7 +2262,7 @@  KGroupedStream groupBy(final KeyValueMapper
      * Both of the joining {@code KStream}s will be materialized in local state stores with auto-generated store names.
      * For failure and recovery each store will be backed by an internal changelog topic that will be created in Kafka.
-     * The changelog topic will be named "${applicationId}-storeName-changelog", where "applicationId" is user-specified
+     * The changelog topic will be named "${applicationId}-<storename>-changelog", where "applicationId" is user-specified
      * in {@link StreamsConfig} via parameter
      * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is an
      * internally generated name, and "-changelog" is a fixed suffix.
@@ -2330,7 +2328,7 @@  KStream join(final KStream otherStream,
      * If this requirement is not met, Kafka Streams will automatically repartition the data, i.e., it will create an
      * internal repartitioning topic in Kafka and write and re-read the data via this topic before the actual join.
      * The repartitioning topic will be named "${applicationId}-<name>-repartition", where "applicationId" is
-     * user-specified in {@link  StreamsConfig} via parameter
+     * user-specified in {@link StreamsConfig} via parameter
      * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "<name>" is an internally generated
      * name, and "-repartition" is a fixed suffix.
      * 
                
@@ -2341,7 +2339,7 @@  KStream join(final KStream otherStream,
      * 
                
      * Both of the joining {@code KStream}s will be materialized in local state stores with auto-generated store names.
      * For failure and recovery each store will be backed by an internal changelog topic that will be created in Kafka.
-     * The changelog topic will be named "${applicationId}-storeName-changelog", where "applicationId" is user-specified
+     * The changelog topic will be named "${applicationId}-<storename>-changelog", where "applicationId" is user-specified
      * in {@link StreamsConfig} via parameter
      * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is an
      * internally generated name, and "-changelog" is a fixed suffix.
@@ -2359,12 +2357,94 @@  KStream join(final KStream otherStream,
      * {@link ValueJoiner}, one for each matched record-pair with the same key and within the joining window intervals
      * @see #leftJoin(KStream, ValueJoiner, JoinWindows, Joined)
      * @see #outerJoin(KStream, ValueJoiner, JoinWindows, Joined)
+     * @deprecated since 2.4. Use {@link KStream#join(KStream, ValueJoiner, JoinWindows, StreamJoined)} instead.
      */
+    @Deprecated
      KStream join(final KStream otherStream,
                                  final ValueJoiner joiner,
                                  final JoinWindows windows,
                                  final Joined joined);
 
+    /**
+     * Join records of this stream with another {@code KStream}'s records using windowed inner equi join using the
+     * {@link Joined} instance for configuration of the {@link Serde key serde}, {@link Serde this stream's value serde},
+     * and {@link Serde the other stream's value serde}.
+     * The join is computed on the records' key with join attribute {@code thisKStream.key == otherKStream.key}.
+     * Furthermore, two records are only joined if their timestamps are close to each other as defined by the given
+     * {@link JoinWindows}, i.e., the window defines an additional join predicate on the record timestamps.
+     * 
                
+     * For each pair of records meeting both join predicates the provided {@link ValueJoiner} will be called to compute
+     * a value (with arbitrary type) for the result record.
+     * The key of the result record is the same as for both joining input records.
+     * If an input record key or value is {@code null} the record will not be included in the join operation and thus no
+     * output record will be added to the resulting {@code KStream}.
+     * 
                
+     * Example (assuming all input records belong to the correct windows):
+     * 
                
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
                thisotherresult
                <K1:A>
                <K2:B><K2:b><K2:ValueJoiner(B,b)>
                <K3:c>
                
+     * Both input streams (or to be more precise, their underlying source topics) need to have the same number of
+     * partitions.
+     * If this is not the case, you would need to call {@link #through(String)} (for one input stream) before doing the
+     * join, using a pre-created topic with the "correct" number of partitions.
+     * Furthermore, both input streams need to be co-partitioned on the join key (i.e., use the same partitioner).
+     * If this requirement is not met, Kafka Streams will automatically repartition the data, i.e., it will create an
+     * internal repartitioning topic in Kafka and write and re-read the data via this topic before the actual join.
+     * The repartitioning topic will be named "${applicationId}-<name>-repartition", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "<name>" is an internally generated
+     * name, and "-repartition" is a fixed suffix.
+     * 
                
+     * Repartitioning can happen for one or both of the joining {@code KStream}s.
+     * For this case, all data of the stream will be redistributed through the repartitioning topic by writing all
+     * records to it, and rereading all records from it, such that the join input {@code KStream} is partitioned
+     * correctly on its key.
+     * 
                
+     * Both of the joining {@code KStream}s will be materialized in local state stores with auto-generated store names,
+     * unless a name is provided via a {@code Materialized} instance.
+     * For failure and recovery each store will be backed by an internal changelog topic that will be created in Kafka.
+     * The changelog topic will be named "${applicationId}-<storename>-changelog", where "applicationId" is user-specified
+     * in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is an
+     * internally generated name, and "-changelog" is a fixed suffix.
+     * 
                
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
+     * @param            the value type of the other stream
+     * @param            the value type of the result stream
+     * @param otherStream    the {@code KStream} to be joined with this stream
+     * @param joiner         a {@link ValueJoiner} that computes the join result for a pair of matching records
+     * @param windows        the specification of the {@link JoinWindows}
+     * @param streamJoined   a {@link StreamJoined} used to configure join stores
+     * @return a {@code KStream} that contains join-records for each key and values computed by the given
+     * {@link ValueJoiner}, one for each matched record-pair with the same key and within the joining window intervals
+     * @see #leftJoin(KStream, ValueJoiner, JoinWindows, Joined)
+     * @see #outerJoin(KStream, ValueJoiner, JoinWindows, Joined)
+     */
+     KStream join(final KStream otherStream,
+                                 final ValueJoiner joiner,
+                                 final JoinWindows windows,
+                                 final StreamJoined streamJoined);
+
     /**
      * Join records of this stream with another {@code KStream}'s records using windowed left equi join with default
      * serializers and deserializers.
@@ -2424,7 +2504,7 @@  KStream join(final KStream otherStream,
      * 
                
      * Both of the joining {@code KStream}s will be materialized in local state stores with auto-generated store names.
      * For failure and recovery each store will be backed by an internal changelog topic that will be created in Kafka.
-     * The changelog topic will be named "${applicationId}-storeName-changelog", where "applicationId" is user-specified
+     * The changelog topic will be named "${applicationId}-<storename>-changelog", where "applicationId" is user-specified
      * in {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG},
      * "storeName" is an internally generated name, and "-changelog" is a fixed suffix.
      * 
                
@@ -2505,7 +2585,7 @@  KStream leftJoin(final KStream otherStream,
      * 
                
      * Both of the joining {@code KStream}s will be materialized in local state stores with auto-generated store names.
      * For failure and recovery each store will be backed by an internal changelog topic that will be created in Kafka.
-     * The changelog topic will be named "${applicationId}-storeName-changelog", where "applicationId" is user-specified
+     * The changelog topic will be named "${applicationId}-<storename>-changelog", where "applicationId" is user-specified
      * in {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG},
      * "storeName" is an internally generated name, and "-changelog" is a fixed suffix.
      * 
                
@@ -2523,12 +2603,98 @@  KStream leftJoin(final KStream otherStream,
      * this {@code KStream} and within the joining window intervals
      * @see #join(KStream, ValueJoiner, JoinWindows, Joined)
      * @see #outerJoin(KStream, ValueJoiner, JoinWindows, Joined)
+     * @deprecated since 2.4. Use {@link KStream#leftJoin(KStream, ValueJoiner, JoinWindows, StreamJoined)} instead.
      */
+    @Deprecated
      KStream leftJoin(final KStream otherStream,
                                      final ValueJoiner joiner,
                                      final JoinWindows windows,
                                      final Joined joined);
 
+    /**
+     * Join records of this stream with another {@code KStream}'s records using windowed left equi join using the
+     * {@link Joined} instance for configuration of the {@link Serde key serde}, {@link Serde this stream's value serde},
+     * and {@link Serde the other stream's value serde}.
+     * In contrast to {@link #join(KStream, ValueJoiner, JoinWindows) inner-join}, all records from this stream will
+     * produce at least one output record (cf. below).
+     * The join is computed on the records' key with join attribute {@code thisKStream.key == otherKStream.key}.
+     * Furthermore, two records are only joined if their timestamps are close to each other as defined by the given
+     * {@link JoinWindows}, i.e., the window defines an additional join predicate on the record timestamps.
+     * 
                
+     * For each pair of records meeting both join predicates the provided {@link ValueJoiner} will be called to compute
+     * a value (with arbitrary type) for the result record.
+     * The key of the result record is the same as for both joining input records.
+     * Furthermore, for each input record of this {@code KStream} that does not satisfy the join predicate the provided
+     * {@link ValueJoiner} will be called with a {@code null} value for the other stream.
+     * If an input record key or value is {@code null} the record will not be included in the join operation and thus no
+     * output record will be added to the resulting {@code KStream}.
+     * 
                
+     * Example (assuming all input records belong to the correct windows):
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
                thisotherresult
                <K1:A><K1:ValueJoiner(A,null)>
                <K2:B><K2:b><K2:ValueJoiner(B,b)>
                <K3:c>
                
+     * Both input streams (or to be more precise, their underlying source topics) need to have the same number of
+     * partitions.
+     * If this is not the case, you would need to call {@link #through(String)} (for one input stream) before doing the
+     * join, using a pre-created topic with the "correct" number of partitions.
+     * Furthermore, both input streams need to be co-partitioned on the join key (i.e., use the same partitioner).
+     * If this requirement is not met, Kafka Streams will automatically repartition the data, i.e., it will create an
+     * internal repartitioning topic in Kafka and write and re-read the data via this topic before the actual join.
+     * The repartitioning topic will be named "${applicationId}-<name>-repartition", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "<name>" is an internally generated
+     * name, and "-repartition" is a fixed suffix.
+     * 
                
+     * Repartitioning can happen for one or both of the joining {@code KStream}s.
+     * For this case, all data of the stream will be redistributed through the repartitioning topic by writing all
+     * records to it, and rereading all records from it, such that the join input {@code KStream} is partitioned
+     * correctly on its key.
+     * 
                
+     * Both of the joining {@code KStream}s will be materialized in local state stores with auto-generated store names,
+     * unless a name is provided via a {@code Materialized} instance.
+     * For failure and recovery each store will be backed by an internal changelog topic that will be created in Kafka.
+     * The changelog topic will be named "${applicationId}-<storename>-changelog", where "applicationId" is user-specified
+     * in {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG},
+     * "storeName" is an internally generated name, and "-changelog" is a fixed suffix.
+     * 
                
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
+     * @param         the value type of the other stream
+     * @param         the value type of the result stream
+     * @param otherStream the {@code KStream} to be joined with this stream
+     * @param joiner      a {@link ValueJoiner} that computes the join result for a pair of matching records
+     * @param windows     the specification of the {@link JoinWindows}
+     * @param streamJoined
+     * @return a {@code KStream} that contains join-records for each key and values computed by the given
+     * {@link ValueJoiner}, one for each matched record-pair with the same key plus one for each non-matching record of
+     * this {@code KStream} and within the joining window intervals
+     * @see #join(KStream, ValueJoiner, JoinWindows, Joined)
+     * @see #outerJoin(KStream, ValueJoiner, JoinWindows, Joined)
+     */
+     KStream leftJoin(final KStream otherStream,
+                                     final ValueJoiner joiner,
+                                     final JoinWindows windows,
+                                     final StreamJoined streamJoined);
+
     /**
      * Join records of this stream with another {@code KStream}'s records using windowed outer equi join with default
      * serializers and deserializers.
@@ -2589,7 +2755,7 @@  KStream leftJoin(final KStream otherStream,
      * 
                
      * Both of the joining {@code KStream}s will be materialized in local state stores with auto-generated store names.
      * For failure and recovery each store will be backed by an internal changelog topic that will be created in Kafka.
-     * The changelog topic will be named "${applicationId}-storeName-changelog", where "applicationId" is user-specified
+     * The changelog topic will be named "${applicationId}-<storename>-changelog", where "applicationId" is user-specified
      * in {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG},
      * "storeName" is an internally generated name, and "-changelog" is a fixed suffix.
      * 
                
@@ -2671,7 +2837,7 @@  KStream outerJoin(final KStream otherStream,
      * 
                
      * Both of the joining {@code KStream}s will be materialized in local state stores with auto-generated store names.
      * For failure and recovery each store will be backed by an internal changelog topic that will be created in Kafka.
-     * The changelog topic will be named "${applicationId}-storeName-changelog", where "applicationId" is user-specified
+     * The changelog topic will be named "${applicationId}-<storename>-changelog", where "applicationId" is user-specified
      * in {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG},
      * "storeName" is an internally generated name, and "-changelog" is a fixed suffix.
      * 
                
@@ -2689,12 +2855,99 @@  KStream outerJoin(final KStream otherStream,
      * both {@code KStream} and within the joining window intervals
      * @see #join(KStream, ValueJoiner, JoinWindows, Joined)
      * @see #leftJoin(KStream, ValueJoiner, JoinWindows, Joined)
+     * @deprecated since 2.4. Use {@link KStream#outerJoin(KStream, ValueJoiner, JoinWindows, StreamJoined)} instead.
      */
+    @Deprecated
      KStream outerJoin(final KStream otherStream,
                                       final ValueJoiner joiner,
                                       final JoinWindows windows,
                                       final Joined joined);
 
+    /**
+     * Join records of this stream with another {@code KStream}'s records using windowed outer equi join using the
+     * {@link Joined} instance for configuration of the {@link Serde key serde}, {@link Serde this stream's value serde},
+     * and {@link Serde the other stream's value serde}.
+     * In contrast to {@link #join(KStream, ValueJoiner, JoinWindows) inner-join} or
+     * {@link #leftJoin(KStream, ValueJoiner, JoinWindows) left-join}, all records from both streams will produce at
+     * least one output record (cf. below).
+     * The join is computed on the records' key with join attribute {@code thisKStream.key == otherKStream.key}.
+     * Furthermore, two records are only joined if their timestamps are close to each other as defined by the given
+     * {@link JoinWindows}, i.e., the window defines an additional join predicate on the record timestamps.
+     * 
                
+     * For each pair of records meeting both join predicates the provided {@link ValueJoiner} will be called to compute
+     * a value (with arbitrary type) for the result record.
+     * The key of the result record is the same as for both joining input records.
+     * Furthermore, for each input record of both {@code KStream}s that does not satisfy the join predicate the provided
+     * {@link ValueJoiner} will be called with a {@code null} value for this/other stream, respectively.
+     * If an input record key or value is {@code null} the record will not be included in the join operation and thus no
+     * output record will be added to the resulting {@code KStream}.
+     * 
                
+     * Example (assuming all input records belong to the correct windows):
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
+     * 
                thisotherresult
                <K1:A><K1:ValueJoiner(A,null)>
                <K2:B><K2:b><K2:ValueJoiner(null,b)>

                <K2:ValueJoiner(B,b)>
                <K3:c><K3:ValueJoiner(null,c)>
                
+     * Both input streams (or to be more precise, their underlying source topics) need to have the same number of
+     * partitions.
+     * If this is not the case, you would need to call {@link #through(String)} (for one input stream) before doing the
+     * join, using a pre-created topic with the "correct" number of partitions.
+     * Furthermore, both input streams need to be co-partitioned on the join key (i.e., use the same partitioner).
+     * If this requirement is not met, Kafka Streams will automatically repartition the data, i.e., it will create an
+     * internal repartitioning topic in Kafka and write and re-read the data via this topic before the actual join.
+     * The repartitioning topic will be named "${applicationId}-<name>-repartition", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "<name>" is an internally generated
+     * name, and "-repartition" is a fixed suffix.
+     * 
                
+     * Repartitioning can happen for one or both of the joining {@code KStream}s.
+     * For this case, all data of the stream will be redistributed through the repartitioning topic by writing all
+     * records to it, and rereading all records from it, such that the join input {@code KStream} is partitioned
+     * correctly on its key.
+     * 
                
+     * Both of the joining {@code KStream}s will be materialized in local state stores with auto-generated store names,
+     * unless a name is provided via a {@code Materialized} instance.
+     * For failure and recovery each store will be backed by an internal changelog topic that will be created in Kafka.
+     * The changelog topic will be named "${applicationId}-<storename>-changelog", where "applicationId" is user-specified
+     * in {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG},
+     * "storeName" is an internally generated name, and "-changelog" is a fixed suffix.
+     * 
                
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
+     * @param         the value type of the other stream
+     * @param         the value type of the result stream
+     * @param otherStream the {@code KStream} to be joined with this stream
+     * @param joiner      a {@link ValueJoiner} that computes the join result for a pair of matching records
+     * @param windows     the specification of the {@link JoinWindows}
+     * @param streamJoined
+     * @return a {@code KStream} that contains join-records for each key and values computed by the given
+     * {@link ValueJoiner}, one for each matched record-pair with the same key plus one for each non-matching record of
+     * both {@code KStream} and within the joining window intervals
+     * @see #join(KStream, ValueJoiner, JoinWindows, Joined)
+     * @see #leftJoin(KStream, ValueJoiner, JoinWindows, Joined)
+     */
+     KStream outerJoin(final KStream otherStream,
+                                      final ValueJoiner joiner,
+                                      final JoinWindows windows,
+                                      final StreamJoined streamJoined);
+
     /**
      * Join records of this stream with {@link KTable}'s records using non-windowed inner equi join with default
      * serializers and deserializers.
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/StreamJoined.java b/streams/src/main/java/org/apache/kafka/streams/kstream/StreamJoined.java
new file mode 100644
index 0000000000000..78bb48aa5c3ed
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/StreamJoined.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.kstream;
+
+import org.apache.kafka.common.serialization.Serde;
+import org.apache.kafka.streams.state.WindowBytesStoreSupplier;
+
+/**
+ * Class used to configure the name of the join processor, the repartition topic name,
+ * state stores or state store names in  Stream-Stream join.
+ * @param    the key type
+ * @param   this value type
+ * @param   other value type
+ */
+public class StreamJoined
+    implements NamedOperation> {
+
+    protected final Serde keySerde;
+    protected final Serde valueSerde;
+    protected final Serde otherValueSerde;
+    protected final WindowBytesStoreSupplier thisStoreSupplier;
+    protected final WindowBytesStoreSupplier otherStoreSupplier;
+    protected final String name;
+    protected final String storeName;
+
+    protected StreamJoined(final StreamJoined streamJoined) {
+        this(streamJoined.keySerde,
+            streamJoined.valueSerde,
+            streamJoined.otherValueSerde,
+            streamJoined.thisStoreSupplier,
+            streamJoined.otherStoreSupplier,
+            streamJoined.name,
+            streamJoined.storeName);
+    }
+
+    private StreamJoined(final Serde keySerde,
+                         final Serde valueSerde,
+                         final Serde otherValueSerde,
+                         final WindowBytesStoreSupplier thisStoreSupplier,
+                         final WindowBytesStoreSupplier otherStoreSupplier,
+                         final String name,
+                         final String storeName) {
+        this.keySerde = keySerde;
+        this.valueSerde = valueSerde;
+        this.otherValueSerde = otherValueSerde;
+        this.thisStoreSupplier = thisStoreSupplier;
+        this.otherStoreSupplier = otherStoreSupplier;
+        this.name = name;
+        this.storeName = storeName;
+    }
+
+    /**
+     * Creates a StreamJoined instance with the provided store suppliers. The store suppliers must implement
+     * the {@link WindowBytesStoreSupplier} interface.  The store suppliers must provide unique names or a
+     * {@link org.apache.kafka.streams.errors.StreamsException} is thrown.
+     *
+     * @param storeSupplier       this store supplier
+     * @param otherStoreSupplier  other store supplier
+     * @param                  the key type
+     * @param                 this value type
+     * @param                 other value type
+     * @return                    {@link StreamJoined} instance
+     */
+    public static  StreamJoined with(final WindowBytesStoreSupplier storeSupplier,
+                                                           final WindowBytesStoreSupplier otherStoreSupplier) {
+        return new StreamJoined<>(
+            null,
+            null,
+            null,
+            storeSupplier,
+            otherStoreSupplier,
+            null,
+            null
+        );
+    }
+
+    /**
+     * Creates a {@link StreamJoined} instance using the provided name for the state stores and hence the changelog
+     * topics for the join stores.  The name for the stores will be ${applicationId}-<storeName>-this-join and ${applicationId}-<storeName>-other-join
+     * or ${applicationId}-<storeName>-outer-this-join and ${applicationId}-<storeName>-outer-other-join depending if the join is an inner-join
+     * or an outer join. The changelog topics will have the -changelog suffix.  The user should note that even though the join stores will have a
+     * specified name, the stores will remain unavailable for querying.
+     *
+     * @param storeName  The name to use for the store
+     * @param         The key type
+     * @param        This value type
+     * @param        Other value type
+     * @return            {@link StreamJoined} instance
+     */
+    public static  StreamJoined as(final String storeName) {
+        return new StreamJoined<>(
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            storeName
+        );
+    }
+
+
+    /**
+     * Creates a {@link StreamJoined} instance with the provided serdes to configure the stores
+     * for the join.
+     * @param keySerde          The key serde
+     * @param valueSerde        This value serde
+     * @param otherValueSerde   Other value serde
+     * @param                The key type
+     * @param               This value type
+     * @param               Other value type
+     * @return                  {@link StreamJoined} instance
+     */
+    public static  StreamJoined with(final Serde keySerde,
+                                                           final Serde valueSerde,
+                                                           final Serde otherValueSerde
+    ) {
+        return new StreamJoined<>(
+            keySerde,
+            valueSerde,
+            otherValueSerde,
+            null,
+            null,
+            null,
+            null
+        );
+    }
+
+    /**
+     * Set the name to use for the join processor and the repartition topic(s) if required.
+     * @param name  the name to use
+     * @return      a new {@link StreamJoined} instance
+     */
+    @Override
+    public StreamJoined withName(final String name) {
+        return new StreamJoined<>(
+            keySerde,
+            valueSerde,
+            otherValueSerde,
+            thisStoreSupplier,
+            otherStoreSupplier,
+            name,
+            storeName
+        );
+    }
+
+    /**
+     * Sets the base store name to use for both sides of the join. The name for the state stores and hence the changelog
+     * topics for the join stores.  The name for the stores will be ${applicationId}-<storeName>-this-join and ${applicationId}-<storeName>-other-join
+     * or ${applicationId}-<storeName>-outer-this-join and ${applicationId}-<storeName>-outer-other-join depending if the join is an inner-join
+     * or an outer join. The changelog topics will have the -changelog suffix.  The user should note that even though the join stores will have a
+     * specified name, the stores will remain unavailable for querying.
+     *
+     * @param storeName the storeName to use
+     * @return
+     */
+    public StreamJoined withStoreName(final String storeName) {
+        return new StreamJoined<>(
+            keySerde,
+            valueSerde,
+            otherValueSerde,
+            thisStoreSupplier,
+            otherStoreSupplier,
+            name,
+            storeName
+        );
+    }
+
+    /**
+     * Configure with the provided {@link Serde Serde} for the key
+     * @param keySerde  the serde to use for the key
+     * @return          a new {@link StreamJoined} configured with the keySerde
+     */
+    public StreamJoined withKeySerde(final Serde keySerde) {
+        return new StreamJoined<>(
+            keySerde,
+            valueSerde,
+            otherValueSerde,
+            thisStoreSupplier,
+            otherStoreSupplier,
+            name,
+            storeName
+        );
+    }
+
+    /**
+     * Configure with the provided {@link Serde Serde} for this value
+     * @param valueSerde  the serde to use for this value (calling or left side of the join)
+     * @return            a new {@link StreamJoined} configured with the valueSerde
+     */
+    public StreamJoined withValueSerde(final Serde valueSerde) {
+        return new StreamJoined<>(
+            keySerde,
+            valueSerde,
+            otherValueSerde,
+            thisStoreSupplier,
+            otherStoreSupplier,
+            name,
+            storeName
+        );
+    }
+
+    /**
+     * Configure with the provided {@link Serde Serde} for the other value
+     * @param otherValueSerde  the serde to use for the other value (other or right side of the join)
+     * @return                 a new {@link StreamJoined} configured with the otherValueSerde
+     */
+    public StreamJoined withOtherValueSerde(final Serde otherValueSerde) {
+        return new StreamJoined<>(
+            keySerde,
+            valueSerde,
+            otherValueSerde,
+            thisStoreSupplier,
+            otherStoreSupplier,
+            name,
+            storeName
+        );
+    }
+
+    /**
+     * Configure with the provided {@link WindowBytesStoreSupplier} for this store supplier.  Please note
+     * this method only provides the store supplier for the left side of the join.  If you wish to also provide a
+     * store supplier for the right (i.e., other) side you must use the {@link StreamJoined#withOtherStoreSupplier(WindowBytesStoreSupplier)}
+     * method
+     * @param thisStoreSupplier  the store supplier to use for this store supplier (calling or left side of the join)
+     * @return            a new {@link StreamJoined} configured with thisStoreSupplier
+     */
+    public StreamJoined withThisStoreSupplier(final WindowBytesStoreSupplier thisStoreSupplier) {
+        return new StreamJoined<>(
+            keySerde,
+            valueSerde,
+            otherValueSerde,
+            thisStoreSupplier,
+            otherStoreSupplier,
+            name,
+            storeName
+        );
+    }
+
+    /**
+     * Configure with the provided {@link WindowBytesStoreSupplier} for the other store supplier.  Please note
+     * this method only provides the store supplier for the right side of the join.  If you wish to also provide a
+     * store supplier for the left side you must use the {@link StreamJoined#withThisStoreSupplier(WindowBytesStoreSupplier)}
+     * method
+     * @param otherStoreSupplier  the store supplier to use for the other store supplier (other or right side of the join)
+     * @return            a new {@link StreamJoined} configured with otherStoreSupplier
+     */
+    public StreamJoined withOtherStoreSupplier(final WindowBytesStoreSupplier otherStoreSupplier) {
+        return new StreamJoined<>(
+            keySerde,
+            valueSerde,
+            otherValueSerde,
+            thisStoreSupplier,
+            otherStoreSupplier,
+            name,
+            storeName
+        );
+    }
+
+    @Override
+    public String toString() {
+        return "StreamJoin{" +
+            "keySerde=" + keySerde +
+            ", valueSerde=" + valueSerde +
+            ", otherValueSerde=" + otherValueSerde +
+            ", thisStoreSupplier=" + thisStoreSupplier +
+            ", otherStoreSupplier=" + otherStoreSupplier +
+            ", name='" + name + '\'' +
+            ", storeName='" + storeName + '\'' +
+            '}';
+    }
+}
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java
index 9d76033e82f3d..45ff3a5a82e95 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java
@@ -31,6 +31,7 @@
 import org.apache.kafka.streams.kstream.Predicate;
 import org.apache.kafka.streams.kstream.Printed;
 import org.apache.kafka.streams.kstream.Produced;
+import org.apache.kafka.streams.kstream.StreamJoined;
 import org.apache.kafka.streams.kstream.TransformerSupplier;
 import org.apache.kafka.streams.kstream.ValueJoiner;
 import org.apache.kafka.streams.kstream.ValueMapper;
@@ -43,19 +44,14 @@
 import org.apache.kafka.streams.kstream.internals.graph.ProcessorParameters;
 import org.apache.kafka.streams.kstream.internals.graph.StatefulProcessorNode;
 import org.apache.kafka.streams.kstream.internals.graph.StreamSinkNode;
-import org.apache.kafka.streams.kstream.internals.graph.StreamStreamJoinNode;
 import org.apache.kafka.streams.kstream.internals.graph.StreamTableJoinNode;
 import org.apache.kafka.streams.kstream.internals.graph.StreamsGraphNode;
 import org.apache.kafka.streams.processor.FailOnInvalidTimestamp;
 import org.apache.kafka.streams.processor.ProcessorSupplier;
 import org.apache.kafka.streams.processor.TopicNameExtractor;
 import org.apache.kafka.streams.processor.internals.StaticTopicNameExtractor;
-import org.apache.kafka.streams.state.StoreBuilder;
-import org.apache.kafka.streams.state.Stores;
-import org.apache.kafka.streams.state.WindowStore;
 
 import java.lang.reflect.Array;
-import java.time.Duration;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
@@ -64,6 +60,23 @@
 
 public class KStreamImpl extends AbstractStream implements KStream {
 
+
+    static final String JOINTHIS_NAME = "KSTREAM-JOINTHIS-";
+
+    static final String JOINOTHER_NAME = "KSTREAM-JOINOTHER-";
+
+    static final String JOIN_NAME = "KSTREAM-JOIN-";
+
+    static final String LEFTJOIN_NAME = "KSTREAM-LEFTJOIN-";
+
+    static final String MERGE_NAME = "KSTREAM-MERGE-";
+
+    static final String OUTERTHIS_NAME = "KSTREAM-OUTERTHIS-";
+
+    static final String OUTEROTHER_NAME = "KSTREAM-OUTEROTHER-";
+
+    static final String WINDOWED_NAME = "KSTREAM-WINDOWED-";
+
     static final String SOURCE_NAME = "KSTREAM-SOURCE-";
 
     static final String SINK_NAME = "KSTREAM-SINK-";
@@ -82,24 +95,10 @@ public class KStreamImpl extends AbstractStream implements KStream extends AbstractStream implements KStream KStream join(final KStream other,
     }
 
     @Override
+    @Deprecated
     public  KStream join(final KStream otherStream,
                                         final ValueJoiner joiner,
                                         final JoinWindows windows,
                                         final Joined joined) {
+        Objects.requireNonNull(joined, "Joined can't be null");
+        final JoinedInternal joinedInternal = new JoinedInternal<>(joined);
+        final StreamJoined streamJoined =
+            StreamJoined.with(joinedInternal.keySerde(), joinedInternal.valueSerde(), joinedInternal.otherValueSerde()).withName(joinedInternal.name());
+
+        return join(otherStream, joiner, windows, streamJoined);
+
+    }
+
+    @Override
+    public  KStream join(final KStream otherStream,
+                                        final ValueJoiner joiner,
+                                        final JoinWindows windows,
+                                        final StreamJoined streamJoined) {
 
         return doJoin(otherStream,
                       joiner,
                       windows,
-                      joined,
-                      new KStreamImplJoin(false, false));
+                      streamJoined,
+                      new KStreamImplJoin(builder, false, false)
+                    );
 
     }
 
@@ -735,38 +748,52 @@ public  KStream outerJoin(final KStream other,
     }
 
     @Override
+    @Deprecated
     public  KStream outerJoin(final KStream other,
                                              final ValueJoiner joiner,
                                              final JoinWindows windows,
                                              final Joined joined) {
-        return doJoin(other, joiner, windows, joined, new KStreamImplJoin(true, true));
+        Objects.requireNonNull(joined, "Joined can't be null");
+        final JoinedInternal joinedInternal = new JoinedInternal<>(joined);
+        final StreamJoined streamJoined =
+            StreamJoined.with(joinedInternal.keySerde(), joinedInternal.valueSerde(), joinedInternal.otherValueSerde()).withName(joinedInternal.name());
+        return outerJoin(other, joiner, windows, streamJoined);
+    }
+
+    @Override
+    public  KStream outerJoin(final KStream other,
+                                             final ValueJoiner joiner,
+                                             final JoinWindows windows,
+                                             final StreamJoined streamJoined) {
+
+        return doJoin(other, joiner, windows, streamJoined, new KStreamImplJoin(builder, true, true));
     }
 
     private  KStream doJoin(final KStream other,
                                            final ValueJoiner joiner,
                                            final JoinWindows windows,
-                                           final Joined joined,
+                                           final StreamJoined streamJoined,
                                            final KStreamImplJoin join) {
         Objects.requireNonNull(other, "other KStream can't be null");
         Objects.requireNonNull(joiner, "joiner can't be null");
         Objects.requireNonNull(windows, "windows can't be null");
-        Objects.requireNonNull(joined, "joined can't be null");
+        Objects.requireNonNull(streamJoined, "streamJoined can't be null");
 
         KStreamImpl joinThis = this;
         KStreamImpl joinOther = (KStreamImpl) other;
 
-        final JoinedInternal joinedInternal = new JoinedInternal<>(joined);
-        final NamedInternal name = new NamedInternal(joinedInternal.name());
+        final StreamJoinedInternal streamJoinedInternal = new StreamJoinedInternal<>(streamJoined);
+        final NamedInternal name = new NamedInternal(streamJoinedInternal.name());
         if (joinThis.repartitionRequired) {
             final String joinThisName = joinThis.name;
             final String leftJoinRepartitionTopicName = name.suffixWithOrElseGet("-left", joinThisName);
-            joinThis = joinThis.repartitionForJoin(leftJoinRepartitionTopicName, joined.keySerde(), joined.valueSerde());
+            joinThis = joinThis.repartitionForJoin(leftJoinRepartitionTopicName, streamJoinedInternal.keySerde(), streamJoinedInternal.valueSerde());
         }
 
         if (joinOther.repartitionRequired) {
             final String joinOtherName = joinOther.name;
             final String rightJoinRepartitionTopicName = name.suffixWithOrElseGet("-right", joinOtherName);
-            joinOther = joinOther.repartitionForJoin(rightJoinRepartitionTopicName, joined.keySerde(), joined.otherValueSerde());
+            joinOther = joinOther.repartitionForJoin(rightJoinRepartitionTopicName, streamJoinedInternal.keySerde(), streamJoinedInternal.otherValueSerde());
         }
 
         joinThis.ensureJoinableWith(joinOther);
@@ -776,8 +803,7 @@ private  KStream doJoin(final KStream other,
             joinOther,
             joiner,
             windows,
-            joined
-        );
+            streamJoined);
     }
 
     /**
@@ -843,21 +869,33 @@ public  KStream leftJoin(final KStream other,
     }
 
     @Override
+    @Deprecated
     public  KStream leftJoin(final KStream other,
                                             final ValueJoiner joiner,
                                             final JoinWindows windows,
                                             final Joined joined) {
-        Objects.requireNonNull(joined, "joined can't be null");
-        return doJoin(
-            other,
-            joiner,
-            windows,
-            joined,
-            new KStreamImplJoin(true, false)
-        );
+        Objects.requireNonNull(joined, "Joined can't be null");
+        final JoinedInternal joinedInternal = new JoinedInternal<>(joined);
+        final StreamJoined streamJoined =
+            StreamJoined.with(joinedInternal.keySerde(), joinedInternal.valueSerde(), joinedInternal.otherValueSerde()).withName(joinedInternal.name());
+
+        return leftJoin(other, joiner, windows, streamJoined);
 
     }
 
+    @Override
+    public  KStream leftJoin(final KStream other,
+                                            final ValueJoiner joiner,
+                                            final JoinWindows windows,
+                                            final StreamJoined streamJoined) {
+        return doJoin(other,
+                     joiner,
+                     windows,
+                     streamJoined,
+                     new KStreamImplJoin(builder, true, false)
+        );
+    }
+
     @Override
     public  KStream join(final KTable other,
                                         final ValueJoiner joiner) {
@@ -1070,123 +1108,4 @@ public KGroupedStream groupByKey(final Grouped grouped) {
             builder);
     }
 
-    @SuppressWarnings("deprecation") // continuing to support Windows#maintainMs/segmentInterval in fallback mode
-    private static  StoreBuilder> joinWindowStoreBuilder(final String joinName,
-                                                                                 final JoinWindows windows,
-                                                                                 final Serde keySerde,
-                                                                                 final Serde valueSerde) {
-        return Stores.windowStoreBuilder(
-            Stores.persistentWindowStore(
-                joinName + "-store",
-                Duration.ofMillis(windows.size() + windows.gracePeriodMs()),
-                Duration.ofMillis(windows.size()),
-                true
-            ),
-            keySerde,
-            valueSerde
-        );
-    }
-
-    private class KStreamImplJoin {
-
-        private final boolean leftOuter;
-        private final boolean rightOuter;
-
-
-        KStreamImplJoin(final boolean leftOuter,
-                        final boolean rightOuter) {
-            this.leftOuter = leftOuter;
-            this.rightOuter = rightOuter;
-        }
-
-        public  KStream join(final KStream lhs,
-                                                   final KStream other,
-                                                   final ValueJoiner joiner,
-                                                   final JoinWindows windows,
-                                                   final Joined joined) {
-
-            final JoinedInternal  joinedInternal = new JoinedInternal<>(joined);
-            final NamedInternal renamed = new NamedInternal(joinedInternal.name());
-
-            final String thisWindowStreamName =  renamed.suffixWithOrElseGet(
-                    "-this-windowed", builder, WINDOWED_NAME);
-            final String otherWindowStreamName = renamed.suffixWithOrElseGet(
-                    "-other-windowed", builder, WINDOWED_NAME);
-
-            final String joinThisName = rightOuter ?
-                    renamed.suffixWithOrElseGet("-outer-this-join", builder, OUTERTHIS_NAME)
-                    : renamed.suffixWithOrElseGet("-this-join", builder, JOINTHIS_NAME);
-            final String joinOtherName = leftOuter ?
-                    renamed.suffixWithOrElseGet("-outer-other-join", builder, OUTEROTHER_NAME)
-                    : renamed.suffixWithOrElseGet("-other-join", builder, JOINOTHER_NAME);
-            final String joinMergeName = renamed.suffixWithOrElseGet(
-                    "-merge", builder, MERGE_NAME);
-            final StreamsGraphNode thisStreamsGraphNode = ((AbstractStream) lhs).streamsGraphNode;
-            final StreamsGraphNode otherStreamsGraphNode = ((AbstractStream) other).streamsGraphNode;
-
-
-            final StoreBuilder> thisWindowStore =
-                joinWindowStoreBuilder(joinThisName, windows, joined.keySerde(), joined.valueSerde());
-            final StoreBuilder> otherWindowStore =
-                joinWindowStoreBuilder(joinOtherName, windows, joined.keySerde(), joined.otherValueSerde());
-
-            final KStreamJoinWindow thisWindowedStream = new KStreamJoinWindow<>(thisWindowStore.name());
-
-            final ProcessorParameters thisWindowStreamProcessorParams = new ProcessorParameters<>(thisWindowedStream, thisWindowStreamName);
-            final ProcessorGraphNode thisWindowedStreamsNode = new ProcessorGraphNode<>(thisWindowStreamName, thisWindowStreamProcessorParams);
-            builder.addGraphNode(thisStreamsGraphNode, thisWindowedStreamsNode);
-
-            final KStreamJoinWindow otherWindowedStream = new KStreamJoinWindow<>(otherWindowStore.name());
-
-            final ProcessorParameters otherWindowStreamProcessorParams = new ProcessorParameters<>(otherWindowedStream, otherWindowStreamName);
-            final ProcessorGraphNode otherWindowedStreamsNode = new ProcessorGraphNode<>(otherWindowStreamName, otherWindowStreamProcessorParams);
-            builder.addGraphNode(otherStreamsGraphNode, otherWindowedStreamsNode);
-
-            final KStreamKStreamJoin joinThis = new KStreamKStreamJoin<>(
-                otherWindowStore.name(),
-                windows.beforeMs,
-                windows.afterMs,
-                joiner,
-                leftOuter
-            );
-
-            final KStreamKStreamJoin joinOther = new KStreamKStreamJoin<>(
-                thisWindowStore.name(),
-                windows.afterMs,
-                windows.beforeMs,
-                reverseJoiner(joiner),
-                rightOuter
-            );
-
-            final KStreamPassThrough joinMerge = new KStreamPassThrough<>();
-
-            final StreamStreamJoinNode.StreamStreamJoinNodeBuilder joinBuilder = StreamStreamJoinNode.streamStreamJoinNodeBuilder();
-
-            final ProcessorParameters joinThisProcessorParams = new ProcessorParameters<>(joinThis, joinThisName);
-            final ProcessorParameters joinOtherProcessorParams = new ProcessorParameters<>(joinOther, joinOtherName);
-            final ProcessorParameters joinMergeProcessorParams = new ProcessorParameters<>(joinMerge, joinMergeName);
-
-            joinBuilder.withJoinMergeProcessorParameters(joinMergeProcessorParams)
-                       .withJoinThisProcessorParameters(joinThisProcessorParams)
-                       .withJoinOtherProcessorParameters(joinOtherProcessorParams)
-                       .withThisWindowStoreBuilder(thisWindowStore)
-                       .withOtherWindowStoreBuilder(otherWindowStore)
-                       .withThisWindowedStreamProcessorParameters(thisWindowStreamProcessorParams)
-                       .withOtherWindowedStreamProcessorParameters(otherWindowStreamProcessorParams)
-                       .withValueJoiner(joiner)
-                       .withNodeName(joinMergeName);
-
-            final StreamsGraphNode joinGraphNode = joinBuilder.build();
-
-            builder.addGraphNode(Arrays.asList(thisStreamsGraphNode, otherStreamsGraphNode), joinGraphNode);
-
-            final Set allSourceNodes = new HashSet<>(((KStreamImpl) lhs).sourceNodes);
-            allSourceNodes.addAll(((KStreamImpl) other).sourceNodes);
-
-            // do not have serde for joined result;
-            // also for key serde we do not inherit from either since we cannot tell if these two serdes are different
-            return new KStreamImpl<>(joinMergeName, joined.keySerde(), null, allSourceNodes, false, joinGraphNode, builder);
-        }
-    }
-
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImplJoin.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImplJoin.java
new file mode 100644
index 0000000000000..b77e064ed1855
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImplJoin.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.kstream.internals;
+
+import org.apache.kafka.common.serialization.Serde;
+import org.apache.kafka.streams.errors.StreamsException;
+import org.apache.kafka.streams.kstream.JoinWindows;
+import org.apache.kafka.streams.kstream.KStream;
+import org.apache.kafka.streams.kstream.StreamJoined;
+import org.apache.kafka.streams.kstream.ValueJoiner;
+import org.apache.kafka.streams.kstream.internals.graph.ProcessorGraphNode;
+import org.apache.kafka.streams.kstream.internals.graph.ProcessorParameters;
+import org.apache.kafka.streams.kstream.internals.graph.StreamStreamJoinNode;
+import org.apache.kafka.streams.kstream.internals.graph.StreamsGraphNode;
+import org.apache.kafka.streams.state.StoreBuilder;
+import org.apache.kafka.streams.state.Stores;
+import org.apache.kafka.streams.state.WindowBytesStoreSupplier;
+import org.apache.kafka.streams.state.WindowStore;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+class KStreamImplJoin {
+
+    private final InternalStreamsBuilder builder;
+    private final boolean leftOuter;
+    private final boolean rightOuter;
+
+
+    KStreamImplJoin(final InternalStreamsBuilder builder,
+                    final boolean leftOuter,
+                    final boolean rightOuter) {
+        this.builder = builder;
+        this.leftOuter = leftOuter;
+        this.rightOuter = rightOuter;
+    }
+
+    public  KStream join(final KStream lhs,
+                                               final KStream other,
+                                               final ValueJoiner joiner,
+                                               final JoinWindows windows,
+                                               final StreamJoined streamJoined) {
+
+        final StreamJoinedInternal streamJoinedInternal = new StreamJoinedInternal<>(streamJoined);
+        final NamedInternal renamed = new NamedInternal(streamJoinedInternal.name());
+        final String joinThisSuffix = rightOuter ? "-outer-this-join" : "-this-join";
+        final String joinOtherSuffix = leftOuter ? "-outer-other-join" : "-other-join";
+
+        final String thisWindowStreamProcessorName = renamed.suffixWithOrElseGet(
+            "-this-windowed", builder, KStreamImpl.WINDOWED_NAME);
+        final String otherWindowStreamProcessorName = renamed.suffixWithOrElseGet(
+            "-other-windowed", builder, KStreamImpl.WINDOWED_NAME);
+
+        final String joinThisGeneratedName = rightOuter ? builder.newProcessorName(KStreamImpl.OUTERTHIS_NAME) : builder.newProcessorName(KStreamImpl.JOINTHIS_NAME);
+        final String joinOtherGeneratedName = leftOuter ? builder.newProcessorName(KStreamImpl.OUTEROTHER_NAME) : builder.newProcessorName(KStreamImpl.JOINOTHER_NAME);
+
+        final String joinThisName = renamed.suffixWithOrElseGet(joinThisSuffix, joinThisGeneratedName);
+        final String joinOtherName = renamed.suffixWithOrElseGet(joinOtherSuffix, joinOtherGeneratedName);
+
+        final String joinMergeName = renamed.suffixWithOrElseGet(
+            "-merge", builder, KStreamImpl.MERGE_NAME);
+
+        final StreamsGraphNode thisStreamsGraphNode = ((AbstractStream) lhs).streamsGraphNode;
+        final StreamsGraphNode otherStreamsGraphNode = ((AbstractStream) other).streamsGraphNode;
+
+        final StoreBuilder> thisWindowStore;
+        final StoreBuilder> otherWindowStore;
+        final String userProvidedBaseStoreName = streamJoinedInternal.storeName();
+
+        final WindowBytesStoreSupplier thisStoreSupplier = streamJoinedInternal.thisStoreSupplier();
+        final WindowBytesStoreSupplier otherStoreSupplier = streamJoinedInternal.otherStoreSupplier();
+
+        assertUniqueStoreNames(thisStoreSupplier, otherStoreSupplier);
+
+        if (thisStoreSupplier == null) {
+            final String thisJoinStoreName = userProvidedBaseStoreName == null ? joinThisGeneratedName : userProvidedBaseStoreName + joinThisSuffix;
+            thisWindowStore = joinWindowStoreBuilder(thisJoinStoreName, windows, streamJoinedInternal.keySerde(), streamJoinedInternal.valueSerde());
+        } else {
+            assertWindowSettings(thisStoreSupplier, windows);
+            thisWindowStore = joinWindowStoreBuilderFromSupplier(thisStoreSupplier, streamJoinedInternal.keySerde(), streamJoinedInternal.valueSerde());
+        }
+
+        if (otherStoreSupplier == null) {
+            final String otherJoinStoreName = userProvidedBaseStoreName == null ? joinOtherGeneratedName : userProvidedBaseStoreName + joinOtherSuffix;
+            otherWindowStore = joinWindowStoreBuilder(otherJoinStoreName, windows, streamJoinedInternal.keySerde(), streamJoinedInternal.otherValueSerde());
+        } else {
+            assertWindowSettings(otherStoreSupplier, windows);
+            otherWindowStore = joinWindowStoreBuilderFromSupplier(otherStoreSupplier, streamJoinedInternal.keySerde(), streamJoinedInternal.otherValueSerde());
+        }
+
+        final KStreamJoinWindow thisWindowedStream = new KStreamJoinWindow<>(thisWindowStore.name());
+
+        final ProcessorParameters thisWindowStreamProcessorParams = new ProcessorParameters<>(thisWindowedStream, thisWindowStreamProcessorName);
+        final ProcessorGraphNode thisWindowedStreamsNode = new ProcessorGraphNode<>(thisWindowStreamProcessorName, thisWindowStreamProcessorParams);
+        builder.addGraphNode(thisStreamsGraphNode, thisWindowedStreamsNode);
+
+        final KStreamJoinWindow otherWindowedStream = new KStreamJoinWindow<>(otherWindowStore.name());
+
+        final ProcessorParameters otherWindowStreamProcessorParams = new ProcessorParameters<>(otherWindowedStream, otherWindowStreamProcessorName);
+        final ProcessorGraphNode otherWindowedStreamsNode = new ProcessorGraphNode<>(otherWindowStreamProcessorName, otherWindowStreamProcessorParams);
+        builder.addGraphNode(otherStreamsGraphNode, otherWindowedStreamsNode);
+
+        final KStreamKStreamJoin joinThis = new KStreamKStreamJoin<>(
+            otherWindowStore.name(),
+            windows.beforeMs,
+            windows.afterMs,
+            joiner,
+            leftOuter
+        );
+
+        final KStreamKStreamJoin joinOther = new KStreamKStreamJoin<>(
+            thisWindowStore.name(),
+            windows.afterMs,
+            windows.beforeMs,
+            AbstractStream.reverseJoiner(joiner),
+            rightOuter
+        );
+
+        final KStreamPassThrough joinMerge = new KStreamPassThrough<>();
+
+        final StreamStreamJoinNode.StreamStreamJoinNodeBuilder joinBuilder = StreamStreamJoinNode.streamStreamJoinNodeBuilder();
+
+        final ProcessorParameters joinThisProcessorParams = new ProcessorParameters<>(joinThis, joinThisName);
+        final ProcessorParameters joinOtherProcessorParams = new ProcessorParameters<>(joinOther, joinOtherName);
+        final ProcessorParameters joinMergeProcessorParams = new ProcessorParameters<>(joinMerge, joinMergeName);
+
+        joinBuilder.withJoinMergeProcessorParameters(joinMergeProcessorParams)
+                   .withJoinThisProcessorParameters(joinThisProcessorParams)
+                   .withJoinOtherProcessorParameters(joinOtherProcessorParams)
+                   .withThisWindowStoreBuilder(thisWindowStore)
+                   .withOtherWindowStoreBuilder(otherWindowStore)
+                   .withThisWindowedStreamProcessorParameters(thisWindowStreamProcessorParams)
+                   .withOtherWindowedStreamProcessorParameters(otherWindowStreamProcessorParams)
+                   .withValueJoiner(joiner)
+                   .withNodeName(joinMergeName);
+
+        final StreamsGraphNode joinGraphNode = joinBuilder.build();
+
+        builder.addGraphNode(Arrays.asList(thisStreamsGraphNode, otherStreamsGraphNode), joinGraphNode);
+
+        final Set allSourceNodes = new HashSet<>(((KStreamImpl) lhs).sourceNodes);
+        allSourceNodes.addAll(((KStreamImpl) other).sourceNodes);
+
+        // do not have serde for joined result;
+        // also for key serde we do not inherit from either since we cannot tell if these two serdes are different
+        return new KStreamImpl<>(joinMergeName, streamJoinedInternal.keySerde(), null, allSourceNodes, false, joinGraphNode, builder);
+    }
+
+    private void assertWindowSettings(final WindowBytesStoreSupplier supplier, final JoinWindows joinWindows) {
+        if (!supplier.retainDuplicates()) {
+            throw new StreamsException("The StoreSupplier must set retainDuplicates=true, found retainDuplicates=false");
+        }
+        final boolean allMatch = supplier.retentionPeriod() == (joinWindows.size() + joinWindows.gracePeriodMs()) &&
+            supplier.windowSize() == joinWindows.size();
+        if (!allMatch) {
+            throw new StreamsException(String.format("Window settings mismatch. WindowBytesStoreSupplier settings %s must match JoinWindows settings %s", supplier, joinWindows));
+        }
+    }
+
+    private void assertUniqueStoreNames(final WindowBytesStoreSupplier supplier,
+                                        final WindowBytesStoreSupplier otherSupplier) {
+
+        if (supplier != null
+            && otherSupplier != null
+            && supplier.name().equals(otherSupplier.name())) {
+            throw new StreamsException("Both StoreSuppliers have the same name.  StoreSuppliers must provide unique names");
+        }
+    }
+
+    @SuppressWarnings("deprecation") // continuing to support Windows#maintainMs/segmentInterval in fallback mode
+    private static  StoreBuilder> joinWindowStoreBuilder(final String storeName,
+                                                                                 final JoinWindows windows,
+                                                                                 final Serde keySerde,
+                                                                                 final Serde valueSerde) {
+        return Stores.windowStoreBuilder(
+            Stores.persistentWindowStore(
+                storeName + "-store",
+                Duration.ofMillis(windows.size() + windows.gracePeriodMs()),
+                Duration.ofMillis(windows.size()),
+                true
+            ),
+            keySerde,
+            valueSerde
+        );
+    }
+
+    @SuppressWarnings("deprecation") // continuing to support Windows#maintainMs/segmentInterval in fallback mode
+    private static  StoreBuilder> joinWindowStoreBuilderFromSupplier(final WindowBytesStoreSupplier storeSupplier,
+                                                                                             final Serde keySerde,
+                                                                                             final Serde valueSerde) {
+        return Stores.windowStoreBuilder(
+            storeSupplier,
+            keySerde,
+            valueSerde
+        );
+    }
+}
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/StreamJoinedInternal.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/StreamJoinedInternal.java
new file mode 100644
index 0000000000000..c9efca16c704d
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/StreamJoinedInternal.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.kstream.internals;
+
+import org.apache.kafka.common.serialization.Serde;
+import org.apache.kafka.streams.kstream.StreamJoined;
+import org.apache.kafka.streams.state.WindowBytesStoreSupplier;
+
+public class StreamJoinedInternal extends StreamJoined {
+
+    //Needs to be public for testing
+    public StreamJoinedInternal(final StreamJoined streamJoined) {
+        super(streamJoined);
+    }
+
+    public Serde keySerde() {
+        return keySerde;
+    }
+
+    public Serde valueSerde() {
+        return valueSerde;
+    }
+
+    public Serde otherValueSerde() {
+        return otherValueSerde;
+    }
+
+    public String name() {
+        return name;
+    }
+
+    public String storeName() {
+        return storeName;
+    }
+
+    public WindowBytesStoreSupplier thisStoreSupplier() {
+        return thisStoreSupplier;
+    }
+
+    public WindowBytesStoreSupplier otherStoreSupplier() {
+        return otherStoreSupplier;
+    }
+
+
+}
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryWindowBytesStoreSupplier.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryWindowBytesStoreSupplier.java
index 042a5398b086a..52a2cd84e71bc 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryWindowBytesStoreSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryWindowBytesStoreSupplier.java
@@ -82,4 +82,14 @@ public long segmentIntervalMs() {
     public boolean retainDuplicates() {
         return retainDuplicates;
     }
+
+    @Override
+    public String toString() {
+        return "InMemoryWindowBytesStoreSupplier{" +
+            "name='" + name + '\'' +
+            ", retentionPeriod=" + retentionPeriod +
+            ", windowSize=" + windowSize +
+            ", retainDuplicates=" + retainDuplicates +
+            '}';
+    }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java
index 197e2344153dc..3536c016c45db 100644
--- a/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java
@@ -33,6 +33,7 @@
 import org.apache.kafka.streams.kstream.Named;
 import org.apache.kafka.streams.kstream.Printed;
 import org.apache.kafka.streams.kstream.Produced;
+import org.apache.kafka.streams.kstream.StreamJoined;
 import org.apache.kafka.streams.kstream.ValueTransformer;
 import org.apache.kafka.streams.kstream.ValueTransformerWithKey;
 import org.apache.kafka.streams.processor.StateStore;
@@ -437,7 +438,7 @@ public void shouldUseSpecifiedNameForStreamSourceProcessor() {
         builder.stream(STREAM_TOPIC_TWO);
         builder.build();
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForOperation(topology, expected, "KSTREAM-SOURCE-0000000001");
+        assertNamesForOperation(topology, expected, "KSTREAM-SOURCE-0000000001");
     }
 
     @Test
@@ -449,7 +450,7 @@ public void shouldUseSpecifiedNameForTableSourceProcessor() {
 
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
 
-        assertSpecifiedNameForOperation(
+        assertNamesForOperation(
                 topology,
                 expected + "-source",
                 expected,
@@ -465,7 +466,7 @@ public void shouldUseSpecifiedNameForGlobalTableSourceProcessor() {
         builder.build();
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
 
-        assertSpecifiedNameForStateStore(
+        assertNamesForStateStore(
                 topology.globalStateStores(),
                 "stream-topic-STATE-STORE-0000000000",
                 "stream-topic-two-STATE-STORE-0000000003"
@@ -480,7 +481,7 @@ public void shouldUseSpecifiedNameForSinkProcessor() {
         stream.to(STREAM_TOPIC_TWO);
         builder.build();
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", expected, "KSTREAM-SINK-0000000002");
+        assertNamesForOperation(topology, "KSTREAM-SOURCE-0000000000", expected, "KSTREAM-SINK-0000000002");
     }
 
     @Test
@@ -488,7 +489,7 @@ public void shouldUseSpecifiedNameForMapOperation() {
         builder.stream(STREAM_TOPIC).map(KeyValue::pair, Named.as(STREAM_OPERATION_NAME));
         builder.build();
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
+        assertNamesForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
     }
 
     @Test
@@ -496,7 +497,7 @@ public void shouldUseSpecifiedNameForMapValuesOperation() {
         builder.stream(STREAM_TOPIC).mapValues(v -> v, Named.as(STREAM_OPERATION_NAME));
         builder.build();
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
+        assertNamesForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
     }
 
     @Test
@@ -504,7 +505,7 @@ public void shouldUseSpecifiedNameForMapValuesWithKeyOperation() {
         builder.stream(STREAM_TOPIC).mapValues((k, v) -> v, Named.as(STREAM_OPERATION_NAME));
         builder.build();
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
+        assertNamesForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
     }
 
     @Test
@@ -512,7 +513,7 @@ public void shouldUseSpecifiedNameForFilterOperation() {
         builder.stream(STREAM_TOPIC).filter((k, v) -> true, Named.as(STREAM_OPERATION_NAME));
         builder.build();
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
+        assertNamesForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
     }
 
     @Test
@@ -520,7 +521,7 @@ public void shouldUseSpecifiedNameForForEachOperation() {
         builder.stream(STREAM_TOPIC).foreach((k, v) -> { }, Named.as(STREAM_OPERATION_NAME));
         builder.build();
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
+        assertNamesForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
     }
 
     @Test
@@ -528,7 +529,7 @@ public void shouldUseSpecifiedNameForTransform() {
         builder.stream(STREAM_TOPIC).transform(() -> null, Named.as(STREAM_OPERATION_NAME));
         builder.build();
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
+        assertNamesForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
     }
 
     @Test
@@ -537,7 +538,7 @@ public void shouldUseSpecifiedNameForTransformValues() {
         builder.stream(STREAM_TOPIC).transformValues(() -> (ValueTransformer) null, Named.as(STREAM_OPERATION_NAME));
         builder.build();
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
+        assertNamesForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
     }
 
     @Test
@@ -546,7 +547,7 @@ public void shouldUseSpecifiedNameForTransformValuesWithKey() {
         builder.stream(STREAM_TOPIC).transformValues(() -> (ValueTransformerWithKey) null, Named.as(STREAM_OPERATION_NAME));
         builder.build();
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
+        assertNamesForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
     }
 
     @Test
@@ -557,7 +558,7 @@ public void shouldUseSpecifiedNameForBranchOperation() {
 
         builder.build();
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForOperation(topology,
+        assertNamesForOperation(topology,
                 "KSTREAM-SOURCE-0000000000",
                 "branch-processor",
                 "branch-processor-predicate-0",
@@ -572,7 +573,7 @@ public void shouldUseSpecifiedNameForJoinOperationBetweenKStreamAndKTable() {
         builder.build();
 
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForOperation(topology,
+        assertNamesForOperation(topology,
                 "KSTREAM-SOURCE-0000000000",
                 "KSTREAM-SOURCE-0000000002",
                 "KTABLE-SOURCE-0000000003",
@@ -587,7 +588,7 @@ public void shouldUseSpecifiedNameForLeftJoinOperationBetweenKStreamAndKTable()
         builder.build();
 
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForOperation(topology,
+        assertNamesForOperation(topology,
                 "KSTREAM-SOURCE-0000000000",
                 "KSTREAM-SOURCE-0000000002",
                 "KTABLE-SOURCE-0000000003",
@@ -599,14 +600,14 @@ public void shouldUseSpecifiedNameForLeftJoinOperationBetweenKStreamAndKStream()
         final KStream streamOne = builder.stream(STREAM_TOPIC);
         final KStream streamTwo = builder.stream(STREAM_TOPIC_TWO);
 
-        streamOne.leftJoin(streamTwo, (value1, value2) -> value1, JoinWindows.of(Duration.ofHours(1)), Joined.as(STREAM_OPERATION_NAME));
+        streamOne.leftJoin(streamTwo, (value1, value2) -> value1, JoinWindows.of(Duration.ofHours(1)), StreamJoined.as(STREAM_OPERATION_NAME).withName(STREAM_OPERATION_NAME));
         builder.build();
 
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForStateStore(topology.stateStores(),
+        assertNamesForStateStore(topology.stateStores(),
                 STREAM_OPERATION_NAME + "-this-join-store", STREAM_OPERATION_NAME + "-outer-other-join-store"
         );
-        assertSpecifiedNameForOperation(topology,
+        assertNamesForOperation(topology,
                 "KSTREAM-SOURCE-0000000000",
                 "KSTREAM-SOURCE-0000000001",
                 STREAM_OPERATION_NAME + "-this-windowed",
@@ -616,20 +617,44 @@ public void shouldUseSpecifiedNameForLeftJoinOperationBetweenKStreamAndKStream()
                 STREAM_OPERATION_NAME + "-merge");
     }
 
+    @Test
+    @Deprecated
+    public void shouldUseGeneratedStoreNamesForLeftJoinOperationBetweenKStreamAndKStream() {
+        final KStream streamOne = builder.stream(STREAM_TOPIC);
+        final KStream streamTwo = builder.stream(STREAM_TOPIC_TWO);
+
+        streamOne.leftJoin(streamTwo, (value1, value2) -> value1, JoinWindows.of(Duration.ofHours(1)), Joined.as(STREAM_OPERATION_NAME));
+        builder.build();
+
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertNamesForStateStore(topology.stateStores(),
+            "KSTREAM-JOINTHIS-0000000004-store",
+            "KSTREAM-OUTEROTHER-0000000005-store"
+        );
+        assertNamesForOperation(topology,
+            "KSTREAM-SOURCE-0000000000",
+            "KSTREAM-SOURCE-0000000001",
+            STREAM_OPERATION_NAME + "-this-windowed",
+            STREAM_OPERATION_NAME + "-other-windowed",
+            STREAM_OPERATION_NAME + "-this-join",
+            STREAM_OPERATION_NAME + "-outer-other-join",
+            STREAM_OPERATION_NAME + "-merge");
+    }
+
     @Test
     public void shouldUseSpecifiedNameForJoinOperationBetweenKStreamAndKStream() {
         final KStream streamOne = builder.stream(STREAM_TOPIC);
         final KStream streamTwo = builder.stream(STREAM_TOPIC_TWO);
 
-        streamOne.join(streamTwo, (value1, value2) -> value1, JoinWindows.of(Duration.ofHours(1)), Joined.as(STREAM_OPERATION_NAME));
+        streamOne.join(streamTwo, (value1, value2) -> value1, JoinWindows.of(Duration.ofHours(1)), StreamJoined.as(STREAM_OPERATION_NAME).withName(STREAM_OPERATION_NAME));
         builder.build();
 
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForStateStore(topology.stateStores(),
+        assertNamesForStateStore(topology.stateStores(),
                 STREAM_OPERATION_NAME + "-this-join-store",
                 STREAM_OPERATION_NAME + "-other-join-store"
         );
-        assertSpecifiedNameForOperation(topology,
+        assertNamesForOperation(topology,
                 "KSTREAM-SOURCE-0000000000",
                 "KSTREAM-SOURCE-0000000001",
                 STREAM_OPERATION_NAME + "-this-windowed",
@@ -639,18 +664,42 @@ public void shouldUseSpecifiedNameForJoinOperationBetweenKStreamAndKStream() {
                 STREAM_OPERATION_NAME + "-merge");
     }
 
+    @Test
+    @Deprecated
+    public void shouldUseGeneratedNameForJoinOperationBetweenKStreamAndKStream() {
+        final KStream streamOne = builder.stream(STREAM_TOPIC);
+        final KStream streamTwo = builder.stream(STREAM_TOPIC_TWO);
+
+        streamOne.join(streamTwo, (value1, value2) -> value1, JoinWindows.of(Duration.ofHours(1)), Joined.as(STREAM_OPERATION_NAME));
+        builder.build();
+
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertNamesForStateStore(topology.stateStores(),
+            "KSTREAM-JOINTHIS-0000000004-store",
+            "KSTREAM-JOINOTHER-0000000005-store"
+        );
+        assertNamesForOperation(topology,
+            "KSTREAM-SOURCE-0000000000",
+            "KSTREAM-SOURCE-0000000001",
+            STREAM_OPERATION_NAME + "-this-windowed",
+            STREAM_OPERATION_NAME + "-other-windowed",
+            STREAM_OPERATION_NAME + "-this-join",
+            STREAM_OPERATION_NAME + "-other-join",
+            STREAM_OPERATION_NAME + "-merge");
+    }
+
     @Test
     public void shouldUseSpecifiedNameForOuterJoinOperationBetweenKStreamAndKStream() {
         final KStream streamOne = builder.stream(STREAM_TOPIC);
         final KStream streamTwo = builder.stream(STREAM_TOPIC_TWO);
 
-        streamOne.outerJoin(streamTwo, (value1, value2) -> value1, JoinWindows.of(Duration.ofHours(1)), Joined.as(STREAM_OPERATION_NAME));
+        streamOne.outerJoin(streamTwo, (value1, value2) -> value1, JoinWindows.of(Duration.ofHours(1)), StreamJoined.as(STREAM_OPERATION_NAME).withName(STREAM_OPERATION_NAME));
         builder.build();
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForStateStore(topology.stateStores(),
+        assertNamesForStateStore(topology.stateStores(),
                 STREAM_OPERATION_NAME + "-outer-this-join-store",
                 STREAM_OPERATION_NAME + "-outer-other-join-store");
-        assertSpecifiedNameForOperation(topology,
+        assertNamesForOperation(topology,
                 "KSTREAM-SOURCE-0000000000",
                 "KSTREAM-SOURCE-0000000001",
                 STREAM_OPERATION_NAME + "-this-windowed",
@@ -661,6 +710,31 @@ public void shouldUseSpecifiedNameForOuterJoinOperationBetweenKStreamAndKStream(
 
     }
 
+    @Test
+    @Deprecated
+    public void shouldUseGeneratedStoreNamesForOuterJoinOperationBetweenKStreamAndKStream() {
+        final KStream streamOne = builder.stream(STREAM_TOPIC);
+        final KStream streamTwo = builder.stream(STREAM_TOPIC_TWO);
+
+        streamOne.outerJoin(streamTwo, (value1, value2) -> value1, JoinWindows.of(Duration.ofHours(1)), Joined.as(STREAM_OPERATION_NAME));
+        builder.build();
+
+        final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
+        assertNamesForStateStore(topology.stateStores(),
+            "KSTREAM-OUTERTHIS-0000000004-store",
+            "KSTREAM-OUTEROTHER-0000000005-store"
+        );
+        assertNamesForOperation(topology,
+            "KSTREAM-SOURCE-0000000000",
+            "KSTREAM-SOURCE-0000000001",
+            STREAM_OPERATION_NAME + "-this-windowed",
+            STREAM_OPERATION_NAME + "-other-windowed",
+            STREAM_OPERATION_NAME + "-outer-this-join",
+            STREAM_OPERATION_NAME + "-outer-other-join",
+            STREAM_OPERATION_NAME + "-merge");
+    }
+
+
     @Test
     public void shouldUseSpecifiedNameForMergeOperation() {
         final String topic1 = "topic-1";
@@ -671,7 +745,7 @@ public void shouldUseSpecifiedNameForMergeOperation() {
         source1.merge(source2, Named.as("merge-processor"));
         builder.build();
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", "KSTREAM-SOURCE-0000000001", "merge-processor");
+        assertNamesForOperation(topology, "KSTREAM-SOURCE-0000000000", "KSTREAM-SOURCE-0000000001", "merge-processor");
     }
 
     @Test
@@ -681,7 +755,7 @@ public void shouldUseSpecifiedNameForProcessOperation() {
 
         builder.build();
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", "test-processor");
+        assertNamesForOperation(topology, "KSTREAM-SOURCE-0000000000", "test-processor");
     }
 
     @Test
@@ -689,7 +763,7 @@ public void shouldUseSpecifiedNameForPrintOperation() {
         builder.stream(STREAM_TOPIC).print(Printed.toSysOut().withName("print-processor"));
         builder.build();
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", "print-processor");
+        assertNamesForOperation(topology, "KSTREAM-SOURCE-0000000000", "print-processor");
     }
 
     @Test
@@ -698,7 +772,7 @@ public void shouldUseSpecifiedNameForFlatTransformValueOperation() {
         builder.stream(STREAM_TOPIC).flatTransformValues(() -> (ValueTransformer) null, Named.as(STREAM_OPERATION_NAME));
         builder.build();
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
+        assertNamesForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
     }
 
     @Test
@@ -707,7 +781,7 @@ public void shouldUseSpecifiedNameForFlatTransformValueWithKeyOperation() {
         builder.stream(STREAM_TOPIC).flatTransformValues(() -> (ValueTransformerWithKey) null, Named.as(STREAM_OPERATION_NAME));
         builder.build();
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
+        assertNamesForOperation(topology, "KSTREAM-SOURCE-0000000000", STREAM_OPERATION_NAME);
     }
 
     @Test
@@ -718,7 +792,7 @@ public void shouldUseSpecifiedNameForToStream() {
 
         builder.build();
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForOperation(topology,
+        assertNamesForOperation(topology,
                 "KSTREAM-SOURCE-0000000001",
                 "KTABLE-SOURCE-0000000002",
                 "to-stream");
@@ -732,7 +806,7 @@ public void shouldUseSpecifiedNameForToStreamWithMapper() {
 
         builder.build();
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForOperation(topology,
+        assertNamesForOperation(topology,
                 "KSTREAM-SOURCE-0000000001",
                 "KTABLE-SOURCE-0000000002",
                 "to-stream",
@@ -744,12 +818,12 @@ public void shouldUseSpecifiedNameForAggregateOperationGivenTable() {
         builder.table(STREAM_TOPIC).groupBy(KeyValue::pair, Grouped.as("group-operation")).count(Named.as(STREAM_OPERATION_NAME));
         builder.build();
         final ProcessorTopology topology = builder.internalTopologyBuilder.rewriteTopology(new StreamsConfig(props)).build();
-        assertSpecifiedNameForStateStore(
+        assertNamesForStateStore(
             topology.stateStores(),
             STREAM_TOPIC + "-STATE-STORE-0000000000",
              "KTABLE-AGGREGATE-STATE-STORE-0000000004");
 
-        assertSpecifiedNameForOperation(
+        assertNamesForOperation(
             topology,
             "KSTREAM-SOURCE-0000000001",
             "KTABLE-SOURCE-0000000002",
@@ -759,8 +833,7 @@ public void shouldUseSpecifiedNameForAggregateOperationGivenTable() {
             STREAM_OPERATION_NAME);
     }
 
-
-    private static void assertSpecifiedNameForOperation(final ProcessorTopology topology, final String... expected) {
+    private static void assertNamesForOperation(final ProcessorTopology topology, final String... expected) {
         final List processors = topology.processors();
         assertEquals("Invalid number of expected processors", expected.length, processors.size());
         for (int i = 0; i < expected.length; i++) {
@@ -768,7 +841,7 @@ private static void assertSpecifiedNameForOperation(final ProcessorTopology topo
         }
     }
 
-    private static void assertSpecifiedNameForStateStore(final List stores, final String... expected) {
+    private static void assertNamesForStateStore(final List stores, final String... expected) {
         assertEquals("Invalid number of expected state stores", expected.length, stores.size());
         for (int i = 0; i < expected.length; i++) {
             assertEquals(expected[i], stores.get(i).name());
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/RepartitionOptimizingIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/RepartitionOptimizingIntegrationTest.java
index 2d996b6a6864a..bea32f2197f47 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/RepartitionOptimizingIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/RepartitionOptimizingIntegrationTest.java
@@ -35,11 +35,11 @@
 import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.Initializer;
 import org.apache.kafka.streams.kstream.JoinWindows;
-import org.apache.kafka.streams.kstream.Joined;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.Materialized;
 import org.apache.kafka.streams.kstream.Produced;
 import org.apache.kafka.streams.kstream.Reducer;
+import org.apache.kafka.streams.kstream.StreamJoined;
 import org.apache.kafka.streams.processor.AbstractProcessor;
 import org.apache.kafka.test.IntegrationTest;
 import org.apache.kafka.test.StreamsTestUtils;
@@ -162,7 +162,7 @@ private void runIntegrationTest(final String optimizationConfig,
         mappedStream.filter((k, v) -> k.equals("A"))
             .join(countStream, (v1, v2) -> v1 + ":" + v2.toString(),
                   JoinWindows.of(ofMillis(5000)),
-                  Joined.with(Serdes.String(), Serdes.String(), Serdes.Long()))
+                  StreamJoined.with(Serdes.String(), Serdes.String(), Serdes.Long()))
             .to(JOINED_TOPIC);
 
         streamsConfiguration.setProperty(StreamsConfig.TOPOLOGY_OPTIMIZATION, optimizationConfig);
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/StreamStreamJoinIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/StreamStreamJoinIntegrationTest.java
index 4be14c22a98f6..95c231eb29703 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/StreamStreamJoinIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/StreamStreamJoinIntegrationTest.java
@@ -16,11 +16,18 @@
  */
 package org.apache.kafka.streams.integration;
 
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.KafkaStreams.State;
 import org.apache.kafka.streams.KeyValueTimestamp;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.errors.InvalidStateStoreException;
+import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.JoinWindows;
 import org.apache.kafka.streams.kstream.KStream;
+import org.apache.kafka.streams.kstream.StreamJoined;
+import org.apache.kafka.streams.state.QueryableStoreTypes;
 import org.apache.kafka.test.IntegrationTest;
 import org.apache.kafka.test.MockMapper;
 import org.junit.Before;
@@ -32,8 +39,11 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.CountDownLatch;
 
+import static java.time.Duration.ofMillis;
 import static java.time.Duration.ofSeconds;
+import static org.junit.Assert.assertThrows;
 
 /**
  * Tests all available joins of Kafka Streams DSL.
@@ -59,6 +69,34 @@ public void prepareTopology() throws InterruptedException {
         rightStream = builder.stream(INPUT_TOPIC_RIGHT);
     }
 
+    @Test
+    public void shouldNotAccessJoinStoresWhenGivingName() throws InterruptedException {
+        STREAMS_CONFIG.put(StreamsConfig.APPLICATION_ID_CONFIG, appID + "-no-store-access");
+        final StreamsBuilder builder = new StreamsBuilder();
+
+        final KStream left = builder.stream(INPUT_TOPIC_LEFT, Consumed.with(Serdes.String(), Serdes.Integer()));
+        final KStream right = builder.stream(INPUT_TOPIC_RIGHT, Consumed.with(Serdes.String(), Serdes.Integer()));
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        left.join(
+            right,
+            (value1, value2) -> value1 + value2,
+            JoinWindows.of(ofMillis(100)),
+            StreamJoined.with(Serdes.String(), Serdes.Integer(), Serdes.Integer()).withStoreName("join-store"));
+
+        try (final KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), STREAMS_CONFIG)) {
+            kafkaStreams.setStateListener((newState, oldState) -> {
+                if (newState == State.RUNNING) {
+                    latch.countDown();
+                }
+            });
+
+            kafkaStreams.start();
+            latch.await();
+            assertThrows(InvalidStateStoreException.class, () -> kafkaStreams.store("join-store", QueryableStoreTypes.keyValueStore()));
+        }
+    }
+
     @Test
     public void testInner() throws Exception {
         STREAMS_CONFIG.put(StreamsConfig.APPLICATION_ID_CONFIG, appID + "-inner");
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/RepartitionTopicNamingTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/RepartitionTopicNamingTest.java
index e621ffc337294..0f3e33d86f32a 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/RepartitionTopicNamingTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/RepartitionTopicNamingTest.java
@@ -496,7 +496,7 @@ private Topology buildTopology(final String optimizationConfig) {
         mappedStream.filter((k, v) -> k.equals("A"))
                 .join(countStream, (v1, v2) -> v1 + ":" + v2.toString(),
                         JoinWindows.of(Duration.ofMillis(5000L)),
-                        Joined.with(Serdes.String(), Serdes.String(), Serdes.Long(), fourthRepartitionTopicName))
+                        StreamJoined.with(Serdes.String(), Serdes.String(), Serdes.Long()).withStoreName(fourthRepartitionTopicName).withName(fourthRepartitionTopicName))
                 .to(JOINED_TOPIC);
 
         final Properties properties = new Properties();
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java
index 358be9b176a71..976ae1a0b7994 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java
@@ -34,6 +34,7 @@
 import org.apache.kafka.streams.kstream.KeyValueMapper;
 import org.apache.kafka.streams.kstream.Predicate;
 import org.apache.kafka.streams.kstream.Produced;
+import org.apache.kafka.streams.kstream.StreamJoined;
 import org.apache.kafka.streams.kstream.Transformer;
 import org.apache.kafka.streams.kstream.TransformerSupplier;
 import org.apache.kafka.streams.kstream.ValueJoiner;
@@ -69,8 +70,8 @@
 import static java.util.Arrays.asList;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.notNullValue;
-import static org.hamcrest.core.IsInstanceOf.instanceOf;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.IsInstanceOf.instanceOf;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThrows;
@@ -91,6 +92,7 @@ public class KStreamImplTest {
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String());
 
     private final Serde mySerde = new Serdes.StringSerde();
+    private final StreamJoined nullStreamJoinedForTest = null;
 
     @Before
     public void before() {
@@ -125,7 +127,7 @@ public void testNumProcesses() {
         );
 
         final int anyWindowSize = 1;
-        final Joined joined = Joined.with(Serdes.String(), Serdes.Integer(), Serdes.Integer());
+        final StreamJoined joined = StreamJoined.with(Serdes.String(), Serdes.Integer(), Serdes.Integer());
         final KStream stream4 = streams2[0].join(streams3[0],
             (value1, value2) -> value1 + value2, JoinWindows.of(ofMillis(anyWindowSize)), joined);
 
@@ -234,18 +236,18 @@ public void close() {}
 
         assertNull(((AbstractStream) stream1.join(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)))).keySerde());
         assertNull(((AbstractStream) stream1.join(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)))).valueSerde());
-        assertEquals(((AbstractStream) stream1.join(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)), Joined.with(mySerde, mySerde, mySerde))).keySerde(), mySerde);
-        assertNull(((AbstractStream) stream1.join(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)), Joined.with(mySerde, mySerde, mySerde))).valueSerde());
+        assertEquals(((AbstractStream) stream1.join(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)), StreamJoined.with(mySerde, mySerde, mySerde))).keySerde(), mySerde);
+        assertNull(((AbstractStream) stream1.join(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)), StreamJoined.with(mySerde, mySerde, mySerde))).valueSerde());
 
         assertNull(((AbstractStream) stream1.leftJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)))).keySerde());
         assertNull(((AbstractStream) stream1.leftJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)))).valueSerde());
-        assertEquals(((AbstractStream) stream1.leftJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)), Joined.with(mySerde, mySerde, mySerde))).keySerde(), mySerde);
-        assertNull(((AbstractStream) stream1.leftJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)), Joined.with(mySerde, mySerde, mySerde))).valueSerde());
+        assertEquals(((AbstractStream) stream1.leftJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)), StreamJoined.with(mySerde, mySerde, mySerde))).keySerde(), mySerde);
+        assertNull(((AbstractStream) stream1.leftJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)), StreamJoined.with(mySerde, mySerde, mySerde))).valueSerde());
 
         assertNull(((AbstractStream) stream1.outerJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)))).keySerde());
         assertNull(((AbstractStream) stream1.outerJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)))).valueSerde());
-        assertEquals(((AbstractStream) stream1.outerJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)), Joined.with(mySerde, mySerde, mySerde))).keySerde(), mySerde);
-        assertNull(((AbstractStream) stream1.outerJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)), Joined.with(mySerde, mySerde, mySerde))).valueSerde());
+        assertEquals(((AbstractStream) stream1.outerJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)), StreamJoined.with(mySerde, mySerde, mySerde))).keySerde(), mySerde);
+        assertNull(((AbstractStream) stream1.outerJoin(stream1, joiner, JoinWindows.of(Duration.ofMillis(100L)), StreamJoined.with(mySerde, mySerde, mySerde))).valueSerde());
 
         assertEquals(((AbstractStream) stream1.join(table1, joiner)).keySerde(), consumedInternal.keySerde());
         assertNull(((AbstractStream) stream1.join(table1, joiner)).valueSerde());
@@ -371,7 +373,7 @@ public void shouldUseRecordMetadataTimestampExtractorWhenInternalRepartitioningT
             kStream,
             valueJoiner,
             JoinWindows.of(ofMillis(windowSize)).grace(ofMillis(3L * windowSize)),
-            Joined.with(Serdes.String(), Serdes.String(), Serdes.String())
+            StreamJoined.with(Serdes.String(), Serdes.String(), Serdes.String())
         )
               .to("output-topic", Produced.with(Serdes.String(), Serdes.String()));
 
@@ -651,13 +653,13 @@ public void shouldThrowNullPointerOnJoinWithTableWhenJoinedIsNull() {
     }
 
     @Test(expected = NullPointerException.class)
-    public void shouldThrowNullPointerOnJoinWithStreamWhenJoinedIsNull() {
-        testStream.join(testStream, MockValueJoiner.TOSTRING_JOINER, JoinWindows.of(ofMillis(10)), null);
+    public void shouldThrowNullPointerOnJoinWithStreamWhenStreamJoinedIsNull() {
+        testStream.join(testStream, MockValueJoiner.TOSTRING_JOINER, JoinWindows.of(ofMillis(10)), nullStreamJoinedForTest);
     }
 
     @Test(expected = NullPointerException.class)
-    public void shouldThrowNullPointerOnOuterJoinJoinedIsNull() {
-        testStream.outerJoin(testStream, MockValueJoiner.TOSTRING_JOINER, JoinWindows.of(ofMillis(10)), null);
+    public void shouldThrowNullPointerOnOuterJoinStreamJoinedIsNull() {
+        testStream.outerJoin(testStream, MockValueJoiner.TOSTRING_JOINER, JoinWindows.of(ofMillis(10)), nullStreamJoinedForTest);
     }
 
     @Test
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoinTest.java
index 6925eba99144c..b66abd32c219e 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoinTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoinTest.java
@@ -23,11 +23,14 @@
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.TopologyWrapper;
+import org.apache.kafka.streams.errors.StreamsException;
 import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.JoinWindows;
-import org.apache.kafka.streams.kstream.Joined;
 import org.apache.kafka.streams.kstream.KStream;
+import org.apache.kafka.streams.kstream.StreamJoined;
 import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender;
+import org.apache.kafka.streams.state.Stores;
+import org.apache.kafka.streams.state.WindowBytesStoreSupplier;
 import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockProcessor;
 import org.apache.kafka.test.MockProcessorSupplier;
@@ -35,6 +38,7 @@
 import org.apache.kafka.test.StreamsTestUtils;
 import org.junit.Test;
 
+import java.time.Duration;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
@@ -46,6 +50,8 @@
 import static org.hamcrest.CoreMatchers.hasItem;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
 
 public class KStreamKStreamJoinTest {
     private final static KeyValueTimestamp[] EMPTY = new KeyValueTimestamp[0];
@@ -57,6 +63,10 @@ public class KStreamKStreamJoinTest {
         new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer(), 0L);
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String());
 
+    private final JoinWindows joinWindows = JoinWindows.of(ofMillis(50)).grace(Duration.ofMillis(50));
+    private final StreamJoined streamJoined = StreamJoined.with(Serdes.String(), Serdes.Integer(), Serdes.Integer());
+    private final String errorMessagePrefix = "Window settings mismatch. WindowBytesStoreSupplier settings";
+
     @Test
     public void shouldLogAndMeterOnSkippedRecordsWithNullValue() {
         final StreamsBuilder builder = new StreamsBuilder();
@@ -70,7 +80,7 @@ public void shouldLogAndMeterOnSkippedRecordsWithNullValue() {
             right,
             (value1, value2) -> value1 + value2,
             JoinWindows.of(ofMillis(100)),
-            Joined.with(Serdes.String(), Serdes.Integer(), Serdes.Integer())
+            StreamJoined.with(Serdes.String(), Serdes.Integer(), Serdes.Integer())
         );
 
         final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
@@ -84,6 +94,163 @@ public void shouldLogAndMeterOnSkippedRecordsWithNullValue() {
         }
     }
 
+    @Test
+    public void shouldThrowExceptionThisStoreSupplierRetentionDoNotMatchWindowsSizeAndGrace() {
+        // Case where retention of thisJoinStore doesn't match JoinWindows
+        final WindowBytesStoreSupplier thisStoreSupplier = buildWindowBytesStoreSupplier("in-memory-join-store", 500, 100, true);
+        final WindowBytesStoreSupplier otherStoreSupplier = buildWindowBytesStoreSupplier("in-memory-join-store-other", 150, 100, true);
+
+
+        buildStreamsJoinThatShouldThrow(streamJoined.withThisStoreSupplier(thisStoreSupplier).withOtherStoreSupplier(otherStoreSupplier),
+            joinWindows,
+            errorMessagePrefix);
+    }
+
+    @Test
+    public void shouldThrowExceptionThisStoreSupplierWindowSizeDoesNotMatchJoinWindowsWindowSize() {
+        //Case where window size of thisJoinStore doesn't match JoinWindows
+        final WindowBytesStoreSupplier thisStoreSupplier = buildWindowBytesStoreSupplier("in-memory-join-store", 150, 150, true);
+        final WindowBytesStoreSupplier otherStoreSupplier = buildWindowBytesStoreSupplier("in-memory-join-store-other", 150, 100, true);
+
+        buildStreamsJoinThatShouldThrow(streamJoined.withThisStoreSupplier(thisStoreSupplier).withOtherStoreSupplier(otherStoreSupplier),
+            joinWindows,
+            errorMessagePrefix);
+    }
+
+    @Test
+    public void shouldThrowExceptionWhenThisJoinStoreSetsRetainDuplicatesFalse() {
+        //Case where thisJoinStore retain duplicates false
+        final WindowBytesStoreSupplier thisStoreSupplier = buildWindowBytesStoreSupplier("in-memory-join-store", 150, 100, false);
+        final WindowBytesStoreSupplier otherStoreSupplier = buildWindowBytesStoreSupplier("in-memory-join-store-other", 150, 100, true);
+
+        buildStreamsJoinThatShouldThrow(streamJoined.withThisStoreSupplier(thisStoreSupplier).withOtherStoreSupplier(otherStoreSupplier),
+            joinWindows,
+            "The StoreSupplier must set retainDuplicates=true, found retainDuplicates=false");
+    }
+
+    @Test
+    public void shouldThrowExceptionOtherStoreSupplierRetentionDoNotMatchWindowsSizeAndGrace() {
+        //Case where retention size of otherJoinStore doesn't match JoinWindows
+        final WindowBytesStoreSupplier thisStoreSupplier = buildWindowBytesStoreSupplier("in-memory-join-store", 150, 100, true);
+        final WindowBytesStoreSupplier otherStoreSupplier = buildWindowBytesStoreSupplier("in-memory-join-store-other", 500, 100, true);
+
+        buildStreamsJoinThatShouldThrow(streamJoined.withThisStoreSupplier(thisStoreSupplier).withOtherStoreSupplier(otherStoreSupplier),
+            joinWindows,
+            errorMessagePrefix);
+    }
+
+    @Test
+    public void shouldThrowExceptionOtherStoreSupplierWindowSizeDoesNotMatchJoinWindowsWindowSize() {
+        //Case where window size of otherJoinStore doesn't match JoinWindows
+        final WindowBytesStoreSupplier thisStoreSupplier = buildWindowBytesStoreSupplier("in-memory-join-store", 150, 100, true);
+        final WindowBytesStoreSupplier otherStoreSupplier = buildWindowBytesStoreSupplier("in-memory-join-store-other", 150, 150, true);
+
+        buildStreamsJoinThatShouldThrow(streamJoined.withThisStoreSupplier(thisStoreSupplier).withOtherStoreSupplier(otherStoreSupplier),
+            joinWindows,
+            errorMessagePrefix);
+    }
+
+    @Test
+    public void shouldThrowExceptionWhenOtherJoinStoreSetsRetainDuplicatesFalse() {
+        //Case where otherJoinStore retain duplicates false
+        final WindowBytesStoreSupplier thisStoreSupplier = buildWindowBytesStoreSupplier("in-memory-join-store", 150, 100, true);
+        final WindowBytesStoreSupplier otherStoreSupplier = buildWindowBytesStoreSupplier("in-memory-join-store-other", 150, 100, false);
+
+        buildStreamsJoinThatShouldThrow(streamJoined.withThisStoreSupplier(thisStoreSupplier).withOtherStoreSupplier(otherStoreSupplier),
+            joinWindows,
+            "The StoreSupplier must set retainDuplicates=true, found retainDuplicates=false");
+    }
+
+    @Test
+    public void shouldBuildJoinWithCustomStoresAndCorrectWindowSettings() {
+        //Case where everything matches up
+        final WindowBytesStoreSupplier thisStoreSupplier = buildWindowBytesStoreSupplier("in-memory-join-store", 150, 100, true);
+        final WindowBytesStoreSupplier otherStoreSupplier = buildWindowBytesStoreSupplier("in-memory-join-store-other", 150, 100, true);
+
+        final StreamsBuilder builder = new StreamsBuilder();
+        final KStream left = builder.stream("left", Consumed.with(Serdes.String(), Serdes.Integer()));
+        final KStream right = builder.stream("right", Consumed.with(Serdes.String(), Serdes.Integer()));
+
+        left.join(right,
+            (value1, value2) -> value1 + value2,
+            joinWindows,
+            streamJoined);
+
+        builder.build();
+    }
+
+    @Test
+    public void shouldExceptionWhenJoinStoresDontHaveUniqueNames() {
+        final JoinWindows joinWindows = JoinWindows.of(ofMillis(100)).grace(Duration.ofMillis(50));
+        final StreamJoined streamJoined = StreamJoined.with(Serdes.String(), Serdes.Integer(), Serdes.Integer());
+        final WindowBytesStoreSupplier thisStoreSupplier = buildWindowBytesStoreSupplier("in-memory-join-store", 150, 100, true);
+        final WindowBytesStoreSupplier otherStoreSupplier = buildWindowBytesStoreSupplier("in-memory-join-store", 150, 100, true);
+
+        buildStreamsJoinThatShouldThrow(streamJoined.withThisStoreSupplier(thisStoreSupplier).withOtherStoreSupplier(otherStoreSupplier),
+            joinWindows,
+            "Both StoreSuppliers have the same name.  StoreSuppliers must provide unique names");
+    }
+
+    @Test
+    public void shouldJoinWithCustomStoreSuppliers() {
+
+        final JoinWindows joinWindows = JoinWindows.of(ofMillis(100));
+
+        final WindowBytesStoreSupplier thisStoreSupplier = Stores.inMemoryWindowStore("in-memory-join-store",
+            Duration.ofMillis(joinWindows.size() + joinWindows.gracePeriodMs()),
+            Duration.ofMillis(joinWindows.size()), true);
+
+        final WindowBytesStoreSupplier otherStoreSupplier = Stores.inMemoryWindowStore("in-memory-join-store-other",
+            Duration.ofMillis(joinWindows.size() + joinWindows.gracePeriodMs()),
+            Duration.ofMillis(joinWindows.size()), true);
+
+        final StreamJoined streamJoined = StreamJoined.with(Serdes.String(), Serdes.Integer(), Serdes.Integer());
+
+        //Case with 2 custom store suppliers
+        runJoin(streamJoined.withThisStoreSupplier(thisStoreSupplier).withOtherStoreSupplier(otherStoreSupplier), joinWindows);
+
+        //Case with this stream store supplier
+        runJoin(streamJoined.withThisStoreSupplier(thisStoreSupplier), joinWindows);
+
+        //Case with other stream store supplier
+        runJoin(streamJoined.withOtherStoreSupplier(otherStoreSupplier), joinWindows);
+
+
+    }
+
+    private void runJoin(final StreamJoined streamJoined,
+                         final JoinWindows joinWindows) {
+        final StreamsBuilder builder = new StreamsBuilder();
+
+        final KStream left = builder.stream("left", Consumed.with(Serdes.String(), Serdes.Integer()));
+        final KStream right = builder.stream("right", Consumed.with(Serdes.String(), Serdes.Integer()));
+        final ConsumerRecordFactory recordFactory =
+            new ConsumerRecordFactory<>(new StringSerializer(), new IntegerSerializer());
+        final MockProcessorSupplier supplier = new MockProcessorSupplier<>();
+        final KStream joinedStream;
+
+        joinedStream = left.join(
+            right,
+            (value1, value2) -> value1 + value2,
+            joinWindows,
+            streamJoined);
+
+        joinedStream.process(supplier);
+
+        try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final MockProcessor processor = supplier.theCapturedProcessor();
+
+            driver.pipeInput(recordFactory.create("left", "A", 1, 1L));
+            driver.pipeInput(recordFactory.create("left", "B", 1, 2L));
+
+            driver.pipeInput(recordFactory.create("right", "A", 1, 1L));
+            driver.pipeInput(recordFactory.create("right", "B", 2, 2L));
+
+            processor.checkAndClearProcessResult(new KeyValueTimestamp<>("A", 2, 1L),
+                new KeyValueTimestamp<>("B", 3, 2L));
+        }
+    }
+
     @Test
     public void testJoin() {
         final StreamsBuilder builder = new StreamsBuilder();
@@ -100,7 +267,7 @@ public void testJoin() {
             stream2,
             MockValueJoiner.TOSTRING_JOINER,
             JoinWindows.of(ofMillis(100)),
-            Joined.with(Serdes.Integer(), Serdes.String(), Serdes.String()));
+            StreamJoined.with(Serdes.Integer(), Serdes.String(), Serdes.String()));
         joined.process(supplier);
 
         final Collection> copartitionGroups =
@@ -208,7 +375,7 @@ public void testOuterJoin() {
             stream2,
             MockValueJoiner.TOSTRING_JOINER,
             JoinWindows.of(ofMillis(100)),
-            Joined.with(Serdes.Integer(), Serdes.String(), Serdes.String()));
+            StreamJoined.with(Serdes.Integer(), Serdes.String(), Serdes.String()));
         joined.process(supplier);
         final Collection> copartitionGroups =
             TopologyWrapper.getInternalTopologyBuilder(builder.build()).copartitionGroups();
@@ -318,7 +485,7 @@ public void testWindowing() {
             stream2,
             MockValueJoiner.TOSTRING_JOINER,
             JoinWindows.of(ofMillis(100)),
-            Joined.with(Serdes.Integer(), Serdes.String(), Serdes.String()));
+            StreamJoined.with(Serdes.Integer(), Serdes.String(), Serdes.String()));
         joined.process(supplier);
 
         final Collection> copartitionGroups =
@@ -842,7 +1009,7 @@ public void testAsymmetricWindowingAfter() {
             stream2,
             MockValueJoiner.TOSTRING_JOINER,
             JoinWindows.of(ofMillis(0)).after(ofMillis(100)),
-            Joined.with(Serdes.Integer(),
+            StreamJoined.with(Serdes.Integer(),
                 Serdes.String(),
                 Serdes.String()));
         joined.process(supplier);
@@ -1090,7 +1257,7 @@ public void testAsymmetricWindowingBefore() {
             stream2,
             MockValueJoiner.TOSTRING_JOINER,
             JoinWindows.of(ofMillis(0)).before(ofMillis(100)),
-            Joined.with(Serdes.Integer(), Serdes.String(), Serdes.String()));
+            StreamJoined.with(Serdes.Integer(), Serdes.String(), Serdes.String()));
         joined.process(supplier);
 
         final Collection> copartitionGroups =
@@ -1317,4 +1484,33 @@ public void testAsymmetricWindowingBefore() {
             processor.checkAndClearProcessResult(EMPTY);
         }
     }
+
+    private void buildStreamsJoinThatShouldThrow(final StreamJoined streamJoined,
+                                                 final JoinWindows joinWindows,
+                                                 final String expectedExceptionMessagePrefix) {
+
+        final StreamsBuilder builder = new StreamsBuilder();
+        final KStream left = builder.stream("left", Consumed.with(Serdes.String(), Serdes.Integer()));
+        final KStream right = builder.stream("right", Consumed.with(Serdes.String(), Serdes.Integer()));
+
+        final StreamsException streamsException = assertThrows(StreamsException.class, () -> left.join(
+            right,
+            (value1, value2) -> value1 + value2,
+            joinWindows,
+            streamJoined));
+
+        assertTrue(streamsException.getMessage().startsWith(expectedExceptionMessagePrefix));
+    }
+
+    private WindowBytesStoreSupplier buildWindowBytesStoreSupplier(final String name,
+                                                                   final long retentionPeriod,
+                                                                   final long windowSize,
+                                                                   final boolean retainDuplicates) {
+        return  Stores.inMemoryWindowStore(name,
+                                           Duration.ofMillis(retentionPeriod),
+                                           Duration.ofMillis(windowSize),
+                                           retainDuplicates);
+    }
+
+
 }
\ No newline at end of file
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamLeftJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamLeftJoinTest.java
index 8c29a14c06847..3fae39644b966 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamLeftJoinTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamLeftJoinTest.java
@@ -20,13 +20,13 @@
 import org.apache.kafka.common.serialization.Serdes;
 import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.streams.KeyValueTimestamp;
-import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.TopologyWrapper;
+import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.JoinWindows;
-import org.apache.kafka.streams.kstream.Joined;
 import org.apache.kafka.streams.kstream.KStream;
+import org.apache.kafka.streams.kstream.StreamJoined;
 import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockProcessor;
 import org.apache.kafka.test.MockProcessorSupplier;
@@ -70,7 +70,7 @@ public void testLeftJoin() {
             stream2,
             MockValueJoiner.TOSTRING_JOINER,
             JoinWindows.of(ofMillis(100)),
-            Joined.with(Serdes.Integer(), Serdes.String(), Serdes.String()));
+            StreamJoined.with(Serdes.Integer(), Serdes.String(), Serdes.String()));
         joined.process(supplier);
 
         final Collection> copartitionGroups =
@@ -159,7 +159,7 @@ public void testWindowing() {
             stream2,
             MockValueJoiner.TOSTRING_JOINER,
             JoinWindows.of(ofMillis(100)),
-            Joined.with(Serdes.Integer(), Serdes.String(), Serdes.String()));
+            StreamJoined.with(Serdes.Integer(), Serdes.String(), Serdes.String()));
         joined.process(supplier);
 
         final Collection> copartitionGroups =
diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsOptimizedTest.java b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsOptimizedTest.java
index d81edc53ef491..62f0e7461fce5 100644
--- a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsOptimizedTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsOptimizedTest.java
@@ -29,11 +29,11 @@
 import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.Initializer;
 import org.apache.kafka.streams.kstream.JoinWindows;
-import org.apache.kafka.streams.kstream.Joined;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.Materialized;
 import org.apache.kafka.streams.kstream.Produced;
 import org.apache.kafka.streams.kstream.Reducer;
+import org.apache.kafka.streams.kstream.StreamJoined;
 
 import java.time.Duration;
 import java.util.ArrayList;
@@ -101,7 +101,7 @@ public static void main(final String[] args) throws Exception {
 
         mappedStream.join(countStream, (v1, v2) -> v1 + ":" + v2.toString(),
             JoinWindows.of(ofMillis(500)),
-            Joined.with(Serdes.String(), Serdes.String(), Serdes.Long()))
+            StreamJoined.with(Serdes.String(), Serdes.String(), Serdes.Long()))
             .peek((k, v) -> System.out.println(String.format("JOINED key=%s value=%s", k, v)))
             .to(joinTopic, Produced.with(Serdes.String(), Serdes.String()));
 
diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/ImplicitConversions.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/ImplicitConversions.scala
index f62da2e410ed1..d4ebc34b24c90 100644
--- a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/ImplicitConversions.scala
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/ImplicitConversions.scala
@@ -80,4 +80,9 @@ object ImplicitConversions {
                                                       valueSerde: Serde[V],
                                                       otherValueSerde: Serde[VO]): Joined[K, V, VO] =
     Joined.`with`[K, V, VO]
+
+  implicit def streamJoinFromKeyValueOtherSerde[K, V, VO](implicit keySerde: Serde[K],
+                                                          valueSerde: Serde[V],
+                                                          otherValueSerde: Serde[VO]): StreamJoined[K, V, VO] =
+    StreamJoined.`with`[K, V, VO]
 }
diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KStream.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KStream.scala
index 382b040260658..9d19418a0a9b3 100644
--- a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KStream.scala
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KStream.scala
@@ -437,10 +437,12 @@ class KStream[K, V](val inner: KStreamJ[K, V]) {
    * @param otherStream the [[KStream]] to be joined with this stream
    * @param joiner      a function that computes the join result for a pair of matching records
    * @param windows     the specification of the `JoinWindows`
-   * @param joined      an implicit `Joined` instance that defines the serdes to be used to serialize/deserialize
-   *                    inputs and outputs of the joined streams. Instead of `Joined`, the user can also supply
+   * @param streamJoin  an implicit `StreamJoin` instance that defines the serdes to be used to serialize/deserialize
+   *                    inputs and outputs of the joined streams. Instead of `StreamJoin`, the user can also supply
    *                    key serde, value serde and other value serde in implicit scope and they will be
-   *                    converted to the instance of `Joined` through implicit conversion
+   *                    converted to the instance of `Stream` through implicit conversion.  The `StreamJoin` instance can
+   *                    also name the repartition topic (if required), the state stores for the join, and the join
+   *                    processor node.
    * @return a [[KStream]] that contains join-records for each key and values computed by the given `joiner`,
    * one for each matched record-pair with the same key and within the joining window intervals
    * @see `org.apache.kafka.streams.kstream.KStream#join`
@@ -448,8 +450,8 @@ class KStream[K, V](val inner: KStreamJ[K, V]) {
   def join[VO, VR](otherStream: KStream[K, VO])(
     joiner: (V, VO) => VR,
     windows: JoinWindows
-  )(implicit joined: Joined[K, V, VO]): KStream[K, VR] =
-    inner.join[VO, VR](otherStream.inner, joiner.asValueJoiner, windows, joined)
+  )(implicit streamJoin: StreamJoined[K, V, VO]): KStream[K, VR] =
+    inner.join[VO, VR](otherStream.inner, joiner.asValueJoiner, windows, streamJoin)
 
   /**
    * Join records of this stream with another [[KTable]]'s records using inner equi join with
@@ -496,10 +498,12 @@ class KStream[K, V](val inner: KStreamJ[K, V]) {
    * @param otherStream the [[KStream]] to be joined with this stream
    * @param joiner      a function that computes the join result for a pair of matching records
    * @param windows     the specification of the `JoinWindows`
-   * @param joined      an implicit `Joined` instance that defines the serdes to be used to serialize/deserialize
-   *                    inputs and outputs of the joined streams. Instead of `Joined`, the user can also supply
+   * @param streamJoin  an implicit `StreamJoin` instance that defines the serdes to be used to serialize/deserialize
+   *                    inputs and outputs of the joined streams. Instead of `StreamJoin`, the user can also supply
    *                    key serde, value serde and other value serde in implicit scope and they will be
-   *                    converted to the instance of `Joined` through implicit conversion
+   *                    converted to the instance of `Stream` through implicit conversion.  The `StreamJoin` instance can
+   *                    also name the repartition topic (if required), the state stores for the join, and the join
+   *                    processor node.
    * @return a [[KStream]] that contains join-records for each key and values computed by the given `joiner`,
    *                    one for each matched record-pair with the same key and within the joining window intervals
    * @see `org.apache.kafka.streams.kstream.KStream#leftJoin`
@@ -507,8 +511,8 @@ class KStream[K, V](val inner: KStreamJ[K, V]) {
   def leftJoin[VO, VR](otherStream: KStream[K, VO])(
     joiner: (V, VO) => VR,
     windows: JoinWindows
-  )(implicit joined: Joined[K, V, VO]): KStream[K, VR] =
-    inner.leftJoin[VO, VR](otherStream.inner, joiner.asValueJoiner, windows, joined)
+  )(implicit streamJoin: StreamJoined[K, V, VO]): KStream[K, VR] =
+    inner.leftJoin[VO, VR](otherStream.inner, joiner.asValueJoiner, windows, streamJoin)
 
   /**
    * Join records of this stream with another [[KTable]]'s records using left equi join with
@@ -551,10 +555,12 @@ class KStream[K, V](val inner: KStreamJ[K, V]) {
    * @param otherStream the [[KStream]] to be joined with this stream
    * @param joiner      a function that computes the join result for a pair of matching records
    * @param windows     the specification of the `JoinWindows`
-   * @param joined      an implicit `Joined` instance that defines the serdes to be used to serialize/deserialize
-   *                    inputs and outputs of the joined streams. Instead of `Joined`, the user can also supply
+   * @param streamJoin  an implicit `StreamJoin` instance that defines the serdes to be used to serialize/deserialize
+   *                    inputs and outputs of the joined streams. Instead of `StreamJoin`, the user can also supply
    *                    key serde, value serde and other value serde in implicit scope and they will be
-   *                    converted to the instance of `Joined` through implicit conversion
+   *                    converted to the instance of `Stream` through implicit conversion.  The `StreamJoin` instance can
+   *                    also name the repartition topic (if required), the state stores for the join, and the join
+   *                    processor node.
    * @return a [[KStream]] that contains join-records for each key and values computed by the given `joiner`,
    * one for each matched record-pair with the same key and within the joining window intervals
    * @see `org.apache.kafka.streams.kstream.KStream#outerJoin`
@@ -562,8 +568,8 @@ class KStream[K, V](val inner: KStreamJ[K, V]) {
   def outerJoin[VO, VR](otherStream: KStream[K, VO])(
     joiner: (V, VO) => VR,
     windows: JoinWindows
-  )(implicit joined: Joined[K, V, VO]): KStream[K, VR] =
-    inner.outerJoin[VO, VR](otherStream.inner, joiner.asValueJoiner, windows, joined)
+  )(implicit streamJoin: StreamJoined[K, V, VO]): KStream[K, VR] =
+    inner.outerJoin[VO, VR](otherStream.inner, joiner.asValueJoiner, windows, streamJoin)
 
   /**
    * Merge this stream and the given stream into one larger stream.
diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/StreamJoined.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/StreamJoined.scala
new file mode 100644
index 0000000000000..2240253a18f17
--- /dev/null
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/StreamJoined.scala
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.scala.kstream
+
+import org.apache.kafka.common.serialization.Serde
+import org.apache.kafka.streams.kstream.{StreamJoined => StreamJoinedJ}
+import org.apache.kafka.streams.state.WindowBytesStoreSupplier
+
+object StreamJoined {
+
+  /**
+   * Create an instance of [[StreamJoined]] with key, value, and otherValue [[Serde]]
+   * instances.
+   * `null` values are accepted and will be replaced by the default serdes as defined in config.
+   *
+   * @tparam K              key type
+   * @tparam V              value type
+   * @tparam VO             other value type
+   * @param keySerde        the key serde to use.
+   * @param valueSerde      the value serde to use.
+   * @param otherValueSerde the otherValue serde to use. If `null` the default value serde from config will be used
+   * @return new [[StreamJoined]] instance with the provided serdes
+   */
+  def `with`[K, V, VO](implicit keySerde: Serde[K],
+                       valueSerde: Serde[V],
+                       otherValueSerde: Serde[VO]): StreamJoinedJ[K, V, VO] =
+    StreamJoinedJ.`with`(keySerde, valueSerde, otherValueSerde)
+
+  /**
+   * Create an instance of [[StreamJoinJ]] with store suppliers for the calling stream
+   * and the other stream.  Also adds the key, value, and otherValue [[Serde]]
+   * instances.
+   * `null` values are accepted and will be replaced by the default serdes as defined in config.
+   *
+   * @tparam K key type
+   * @tparam V value type
+   * @tparam VO other value type
+   * @param supplier  store supplier to use
+   * @param otherSupplier other store supplier to use
+   * @param keySerde        the key serde to use.
+   * @param valueSerde      the value serde to use.
+   * @param otherValueSerde the otherValue serde to use. If `null` the default value serde from config will be used
+   * @return new [[StreamJoinJ]] instance with the provided store suppliers and serdes
+   */
+  def `with`[K, V, VO](
+    supplier: WindowBytesStoreSupplier,
+    otherSupplier: WindowBytesStoreSupplier
+  )(implicit keySerde: Serde[K], valueSerde: Serde[V], otherValueSerde: Serde[VO]): StreamJoinedJ[K, V, VO] =
+    StreamJoinedJ
+      .`with`(supplier, otherSupplier)
+      .withKeySerde(keySerde)
+      .withValueSerde(valueSerde)
+      .withOtherValueSerde(otherValueSerde)
+
+  /**
+   * Create an instance of [[StreamJoinJ]] with the name used for naming
+   * the state stores involved in the join.  Also adds the key, value, and otherValue [[Serde]]
+   * instances.
+   * `null` values are accepted and will be replaced by the default serdes as defined in config.
+   *
+   * @tparam K key type
+   * @tparam V value type
+   * @tparam VO other value type
+   * @param storeName       the name to use as a base name for the state stores of the join
+   * @param keySerde        the key serde to use.
+   * @param valueSerde      the value serde to use.
+   * @param otherValueSerde the otherValue serde to use. If `null` the default value serde from config will be used
+   * @return new [[StreamJoinJ]] instance with the provided store suppliers and serdes
+   */
+  def as[K, V, VO](
+    storeName: String
+  )(implicit keySerde: Serde[K], valueSerde: Serde[V], otherValueSerde: Serde[VO]): StreamJoinedJ[K, V, VO] =
+    StreamJoinedJ.as(storeName).withKeySerde(keySerde).withValueSerde(valueSerde).withOtherValueSerde(otherValueSerde)
+
+}
diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/package.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/package.scala
index db4463b8b0e51..5e640c547e560 100644
--- a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/package.scala
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/package.scala
@@ -24,4 +24,5 @@ package object kstream {
   type Consumed[K, V] = org.apache.kafka.streams.kstream.Consumed[K, V]
   type Produced[K, V] = org.apache.kafka.streams.kstream.Produced[K, V]
   type Joined[K, V, VO] = org.apache.kafka.streams.kstream.Joined[K, V, VO]
+  type StreamJoined[K, V, VO] = org.apache.kafka.streams.kstream.StreamJoined[K, V, VO]
 }
diff --git a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/TopologyTest.scala b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/TopologyTest.scala
index a265f1545c1aa..e57d88c73e127 100644
--- a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/TopologyTest.scala
+++ b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/TopologyTest.scala
@@ -37,7 +37,7 @@ import org.apache.kafka.streams.kstream.{
   TransformerSupplier,
   ValueJoiner,
   ValueMapper,
-  Joined => JoinedJ,
+  StreamJoined => StreamJoinedJ,
   KGroupedStream => KGroupedStreamJ,
   KStream => KStreamJ,
   KTable => KTableJ,
@@ -327,14 +327,14 @@ class TopologyTest {
       mappedStream
         .filter((k: String, _: String) => k == "A")
         .join(stream2)((v1: String, v2: Int) => v1 + ":" + v2.toString, JoinWindows.of(Duration.ofMillis(5000)))(
-          Joined.`with`(Serdes.String, Serdes.String, Serdes.Integer)
+          StreamJoined.`with`(Serdes.String, Serdes.String, Serdes.Integer)
         )
         .to(JOINED_TOPIC)
 
       mappedStream
         .filter((k: String, _: String) => k == "A")
         .join(stream3)((v1: String, v2: String) => v1 + ":" + v2.toString, JoinWindows.of(Duration.ofMillis(5000)))(
-          Joined.`with`(Serdes.String, Serdes.String, Serdes.String)
+          StreamJoined.`with`(Serdes.String, Serdes.String, Serdes.String)
         )
         .to(JOINED_TOPIC)
 
@@ -410,7 +410,7 @@ class TopologyTest {
         .join[Integer, String](stream2,
                                valueJoiner2,
                                JoinWindows.of(Duration.ofMillis(5000)),
-                               JoinedJ.`with`(Serdes.String, Serdes.String, SerdesJ.Integer))
+                               StreamJoinedJ.`with`(Serdes.String, Serdes.String, SerdesJ.Integer))
         .to(JOINED_TOPIC)
 
       mappedStream
@@ -420,7 +420,7 @@ class TopologyTest {
         .join(stream3,
               valueJoiner3,
               JoinWindows.of(Duration.ofMillis(5000)),
-              JoinedJ.`with`(Serdes.String, Serdes.String, SerdesJ.String))
+              StreamJoinedJ.`with`(Serdes.String, Serdes.String, SerdesJ.String))
         .to(JOINED_TOPIC)
 
       builder
diff --git a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/kstream/StreamJoinedTest.scala b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/kstream/StreamJoinedTest.scala
new file mode 100644
index 0000000000000..3d78d88aa8de8
--- /dev/null
+++ b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/kstream/StreamJoinedTest.scala
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.scala.kstream
+
+import java.time.Duration
+
+import org.apache.kafka.streams.kstream.internals.StreamJoinedInternal
+import org.apache.kafka.streams.scala.Serdes
+import org.apache.kafka.streams.scala.Serdes._
+import org.apache.kafka.streams.state.Stores
+import org.junit.runner.RunWith
+import org.scalatest.{FlatSpec, Matchers}
+import org.scalatest.junit.JUnitRunner
+
+@RunWith(classOf[JUnitRunner])
+class StreamJoinedTest extends FlatSpec with Matchers {
+
+  "Create a StreamJoined" should "create a StreamJoined with Serdes" in {
+    val streamJoined: StreamJoined[String, String, Long] = StreamJoined.`with`[String, String, Long]
+
+    val streamJoinedInternal = new StreamJoinedInternal[String, String, Long](streamJoined)
+    streamJoinedInternal.keySerde().getClass shouldBe Serdes.String.getClass
+    streamJoinedInternal.valueSerde().getClass shouldBe Serdes.String.getClass
+    streamJoinedInternal.otherValueSerde().getClass shouldBe Serdes.Long.getClass
+  }
+
+  "Create a StreamJoined" should "create a StreamJoined with Serdes and Store Suppliers" in {
+    val storeSupplier = Stores.inMemoryWindowStore("myStore", Duration.ofMillis(500), Duration.ofMillis(250), false)
+
+    val otherStoreSupplier =
+      Stores.inMemoryWindowStore("otherStore", Duration.ofMillis(500), Duration.ofMillis(250), false)
+
+    val streamJoined: StreamJoined[String, String, Long] =
+      StreamJoined.`with`[String, String, Long](storeSupplier, otherStoreSupplier)
+
+    val streamJoinedInternal = new StreamJoinedInternal[String, String, Long](streamJoined)
+    streamJoinedInternal.keySerde().getClass shouldBe Serdes.String.getClass
+    streamJoinedInternal.valueSerde().getClass shouldBe Serdes.String.getClass
+    streamJoinedInternal.otherValueSerde().getClass shouldBe Serdes.Long.getClass
+    streamJoinedInternal.otherStoreSupplier().equals(otherStoreSupplier)
+    streamJoinedInternal.thisStoreSupplier().equals(storeSupplier)
+  }
+
+  "Create a StreamJoined" should "create a StreamJoined with Serdes and a State Store name" in {
+    val streamJoined: StreamJoined[String, String, Long] = StreamJoined.as[String, String, Long]("myStoreName")
+
+    val streamJoinedInternal = new StreamJoinedInternal[String, String, Long](streamJoined)
+    streamJoinedInternal.keySerde().getClass shouldBe Serdes.String.getClass
+    streamJoinedInternal.valueSerde().getClass shouldBe Serdes.String.getClass
+    streamJoinedInternal.otherValueSerde().getClass shouldBe Serdes.Long.getClass
+    streamJoinedInternal.storeName().equals("myStoreName")
+  }
+
+}

From f99bb0466e9c8845963b350f3ad9b353db72fd5a Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Wed, 2 Oct 2019 21:02:33 -0700
Subject: [PATCH 0681/1071] MINOR: Start correlation id at 0 in
 SaslClientAuthenticator (#7432)

I tried to add a test for this, but it's actually pretty hard to verify what we want to
verify. I could add a test that checks the correlation field after the connection
has been established, but it would not catch this kind of bug where the issue is not
the value we store, but the value we create the request header with.

I have another PR that avoids intermediate structs during serialization/deserialization,
which has a test that fails without this change. So we'll get coverage that way.

Reviewers: Rajini Sivaram 
---
 .../common/security/authenticator/SaslClientAuthenticator.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
index 4bf56e80b3f68..266759000b12a 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
@@ -150,7 +150,7 @@ public SaslClientAuthenticator(Map configs,
         this.host = host;
         this.servicePrincipal = servicePrincipal;
         this.mechanism = mechanism;
-        this.correlationId = -1;
+        this.correlationId = 0;
         this.transportLayer = transportLayer;
         this.configs = configs;
         this.saslAuthenticateVersion = DISABLE_KAFKA_SASL_AUTHENTICATE_HEADER;

From c62dd1e92e44551c138c28e9cc628084481da8ee Mon Sep 17 00:00:00 2001
From: Andy Coates <8012398+big-andy-coates@users.noreply.github.com>
Date: Thu, 3 Oct 2019 10:11:28 -0700
Subject: [PATCH 0682/1071] MINOR: Make TopicDescription's other constructor
 public (#7405)

Reviewers: Colin P. McCabe 
---
 .../java/org/apache/kafka/clients/admin/TopicDescription.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/TopicDescription.java b/clients/src/main/java/org/apache/kafka/clients/admin/TopicDescription.java
index ea9bf057c49fe..dc18a0eb04f0e 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/TopicDescription.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/TopicDescription.java
@@ -33,7 +33,7 @@ public class TopicDescription {
     private final String name;
     private final boolean internal;
     private final List partitions;
-    private Set authorizedOperations;
+    private final Set authorizedOperations;
 
     @Override
     public boolean equals(final Object o) {
@@ -72,7 +72,7 @@ public TopicDescription(String name, boolean internal, List
      *                   leadership and replica information for that partition.
      * @param authorizedOperations authorized operations for this topic, or null if this is not known.
      */
-    TopicDescription(String name, boolean internal, List partitions,
+    public TopicDescription(String name, boolean internal, List partitions,
                             Set authorizedOperations) {
         this.name = name;
         this.internal = internal;

From e76770343c8a7b81d04c00d219381814190c8745 Mon Sep 17 00:00:00 2001
From: Gunnar Morling 
Date: Thu, 3 Oct 2019 20:51:00 +0200
Subject: [PATCH 0683/1071] KAFKA-8523 Enabling InsertField transform to be
 used with tombstone events (#6914)

* KAFKA-8523 Avoiding raw type usage

* KAFKA-8523 Gracefully handling tombstone events in InsertField SMT
---
 .../kafka/connect/transforms/InsertField.java |  8 ++-
 .../connect/transforms/InsertFieldTest.java   | 52 +++++++++++++++++--
 2 files changed, 54 insertions(+), 6 deletions(-)

diff --git a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/InsertField.java b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/InsertField.java
index 5e472a907c3f4..93ba79c0e3545 100644
--- a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/InsertField.java
+++ b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/InsertField.java
@@ -127,13 +127,19 @@ public void configure(Map props) {
 
     @Override
     public R apply(R record) {
-        if (operatingSchema(record) == null) {
+        if (isTombstoneRecord(record)) {
+            return record;
+        } else if (operatingSchema(record) == null) {
             return applySchemaless(record);
         } else {
             return applyWithSchema(record);
         }
     }
 
+    private boolean isTombstoneRecord(R record) {
+        return record.value() == null;
+    }
+
     private R applySchemaless(R record) {
         final Map value = requireMap(operatingValue(record), PURPOSE);
 
diff --git a/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/InsertFieldTest.java b/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/InsertFieldTest.java
index a0a09752a4ae9..b22872cacb236 100644
--- a/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/InsertFieldTest.java
+++ b/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/InsertFieldTest.java
@@ -104,11 +104,53 @@ public void schemalessInsertConfiguredFields() {
 
         final SourceRecord transformedRecord = xform.apply(record);
 
-        assertEquals(42L, ((Map) transformedRecord.value()).get("magic"));
-        assertEquals("test", ((Map) transformedRecord.value()).get("topic_field"));
-        assertEquals(0, ((Map) transformedRecord.value()).get("partition_field"));
-        assertEquals(null, ((Map) transformedRecord.value()).get("timestamp_field"));
-        assertEquals("my-instance-id", ((Map) transformedRecord.value()).get("instance_id"));
+        assertEquals(42L, ((Map) transformedRecord.value()).get("magic"));
+        assertEquals("test", ((Map) transformedRecord.value()).get("topic_field"));
+        assertEquals(0, ((Map) transformedRecord.value()).get("partition_field"));
+        assertEquals(null, ((Map) transformedRecord.value()).get("timestamp_field"));
+        assertEquals("my-instance-id", ((Map) transformedRecord.value()).get("instance_id"));
     }
 
+
+    @Test
+    public void insertConfiguredFieldsIntoTombstoneEventWithoutSchemaLeavesValueUnchanged() {
+        final Map props = new HashMap<>();
+        props.put("topic.field", "topic_field!");
+        props.put("partition.field", "partition_field");
+        props.put("timestamp.field", "timestamp_field?");
+        props.put("static.field", "instance_id");
+        props.put("static.value", "my-instance-id");
+
+        xform.configure(props);
+
+        final SourceRecord record = new SourceRecord(null, null, "test", 0,
+                null, null);
+
+        final SourceRecord transformedRecord = xform.apply(record);
+
+        assertEquals(null, transformedRecord.value());
+        assertEquals(null, transformedRecord.valueSchema());
+    }
+
+    @Test
+    public void insertConfiguredFieldsIntoTombstoneEventWithSchemaLeavesValueUnchanged() {
+        final Map props = new HashMap<>();
+        props.put("topic.field", "topic_field!");
+        props.put("partition.field", "partition_field");
+        props.put("timestamp.field", "timestamp_field?");
+        props.put("static.field", "instance_id");
+        props.put("static.value", "my-instance-id");
+
+        xform.configure(props);
+
+        final Schema simpleStructSchema = SchemaBuilder.struct().name("name").version(1).doc("doc").field("magic", Schema.OPTIONAL_INT64_SCHEMA).build();
+
+        final SourceRecord record = new SourceRecord(null, null, "test", 0,
+                simpleStructSchema, null);
+
+        final SourceRecord transformedRecord = xform.apply(record);
+
+        assertEquals(null, transformedRecord.value());
+        assertEquals(simpleStructSchema, transformedRecord.valueSchema());
+    }
 }

From ded1fb8c4d7448ffbc939aaacd2d9b475fd58f4e Mon Sep 17 00:00:00 2001
From: Nigel Liang 
Date: Thu, 3 Oct 2019 12:55:52 -0700
Subject: [PATCH 0684/1071] KAFKA-6290: Support casting from logical types in
 cast transform (#7371)

Adds support for the Connect Cast transforms to cast from Connect logical types, such as DATE, TIME, TIMESTAMP, and DECIMAL. Casting to numeric types will produce the underlying numeric value represented in the desired type. For logical types represented by underlying Java Date class, this means the milliseconds since EPOCH. For Decimal, this means the underlying value. If the value does not fit in the desired target type, it may overflow.

Casting to String from Date, Time, and Timestamp types will produce their ISO 8601 representation. Casting to String from Decimal will result in the value represented as a string. e.g. 1234 -> "1234".

Author: Nigel Liang 
Reviewer: Randall Hauch 
---
 .../apache/kafka/connect/transforms/Cast.java |  20 ++++
 .../kafka/connect/transforms/CastTest.java    | 101 +++++++++++++++++-
 2 files changed, 119 insertions(+), 2 deletions(-)

diff --git a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Cast.java b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Cast.java
index 3dc6dc7535717..475b98e88b3a7 100644
--- a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Cast.java
+++ b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Cast.java
@@ -24,10 +24,14 @@
 import org.apache.kafka.common.config.ConfigException;
 import org.apache.kafka.connect.connector.ConnectRecord;
 import org.apache.kafka.connect.data.ConnectSchema;
+import org.apache.kafka.connect.data.Date;
 import org.apache.kafka.connect.data.Field;
 import org.apache.kafka.connect.data.Schema;
+import org.apache.kafka.connect.data.Schema.Type;
 import org.apache.kafka.connect.data.SchemaBuilder;
 import org.apache.kafka.connect.data.Struct;
+import org.apache.kafka.connect.data.Time;
+import org.apache.kafka.connect.data.Timestamp;
 import org.apache.kafka.connect.data.Values;
 import org.apache.kafka.connect.errors.DataException;
 import org.apache.kafka.connect.transforms.util.SchemaUtil;
@@ -222,7 +226,18 @@ private SchemaBuilder convertFieldType(Schema.Type type) {
             default:
                 throw new DataException("Unexpected type in Cast transformation: " + type);
         }
+    }
 
+    private static Object encodeLogicalType(Schema schema, Object value) {
+        switch (schema.name()) {
+            case Date.LOGICAL_NAME:
+                return Date.fromLogical(schema, (java.util.Date) value);
+            case Time.LOGICAL_NAME:
+                return Time.fromLogical(schema, (java.util.Date) value);
+            case Timestamp.LOGICAL_NAME:
+                return Timestamp.fromLogical(schema, (java.util.Date) value);
+        }
+        return value;
     }
 
     private static Object castValueToType(Schema schema, Object value, Schema.Type targetType) {
@@ -238,6 +253,11 @@ private static Object castValueToType(Schema schema, Object value, Schema.Type t
             // Ensure the type we are trying to cast from is supported
             validCastType(inferredType, FieldType.INPUT);
 
+            // Perform logical type encoding to their internal representation.
+            if (schema != null && schema.name() != null && targetType != Type.STRING) {
+                value = encodeLogicalType(schema, value);
+            }
+
             switch (targetType) {
                 case INT8:
                     return castToInt8(value);
diff --git a/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/CastTest.java b/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/CastTest.java
index c568afb9ff452..a28aa28c6d72e 100644
--- a/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/CastTest.java
+++ b/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/CastTest.java
@@ -17,11 +17,16 @@
 
 package org.apache.kafka.connect.transforms;
 
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
 import org.apache.kafka.common.config.ConfigException;
 import org.apache.kafka.connect.data.Decimal;
 import org.apache.kafka.connect.data.Schema;
+import org.apache.kafka.connect.data.Schema.Type;
 import org.apache.kafka.connect.data.SchemaBuilder;
 import org.apache.kafka.connect.data.Struct;
+import org.apache.kafka.connect.data.Time;
 import org.apache.kafka.connect.data.Timestamp;
 import org.apache.kafka.connect.data.Values;
 import org.apache.kafka.connect.errors.DataException;
@@ -42,7 +47,8 @@
 public class CastTest {
     private final Cast xformKey = new Cast.Key<>();
     private final Cast xformValue = new Cast.Value<>();
-    private static final long MILLIS_PER_DAY = 24 * 60 * 60 * 1000;
+    private static final long MILLIS_PER_HOUR = TimeUnit.HOURS.toMillis(1);
+    private static final long MILLIS_PER_DAY = TimeUnit.DAYS.toMillis(1);
 
     @After
     public void teardown() {
@@ -320,6 +326,97 @@ public void castWholeRecordValueSchemalessUnsupportedType() {
         xformValue.apply(new SourceRecord(null, null, "topic", 0, null, Collections.singletonList("foo")));
     }
 
+    @Test
+    public void castLogicalToPrimitive() {
+        List specParts = Arrays.asList(
+            "date_to_int32:int32",  // Cast to underlying representation
+            "timestamp_to_int64:int64",  // Cast to underlying representation
+            "time_to_int64:int64",  // Cast to wider datatype than underlying representation
+            "decimal_to_int32:int32",  // Cast to narrower datatype with data loss
+            "timestamp_to_float64:float64",  // loss of precision casting to double
+            "null_timestamp_to_int32:int32"
+        );
+
+        Date day = new Date(MILLIS_PER_DAY);
+        xformValue.configure(Collections.singletonMap(Cast.SPEC_CONFIG,
+            String.join(",", specParts)));
+
+        SchemaBuilder builder = SchemaBuilder.struct();
+        builder.field("date_to_int32", org.apache.kafka.connect.data.Date.SCHEMA);
+        builder.field("timestamp_to_int64", Timestamp.SCHEMA);
+        builder.field("time_to_int64", Time.SCHEMA);
+        builder.field("decimal_to_int32", Decimal.schema(new BigDecimal((long) Integer.MAX_VALUE + 1).scale()));
+        builder.field("timestamp_to_float64", Timestamp.SCHEMA);
+        builder.field("null_timestamp_to_int32", Timestamp.builder().optional().build());
+
+        Schema supportedTypesSchema = builder.build();
+
+        Struct recordValue = new Struct(supportedTypesSchema);
+        recordValue.put("date_to_int32", day);
+        recordValue.put("timestamp_to_int64", new Date(0));
+        recordValue.put("time_to_int64", new Date(1));
+        recordValue.put("decimal_to_int32", new BigDecimal((long) Integer.MAX_VALUE + 1));
+        recordValue.put("timestamp_to_float64", new Date(Long.MAX_VALUE));
+        recordValue.put("null_timestamp_to_int32", null);
+
+        SourceRecord transformed = xformValue.apply(
+            new SourceRecord(null, null, "topic", 0,
+                supportedTypesSchema, recordValue));
+
+        assertEquals(1, ((Struct) transformed.value()).get("date_to_int32"));
+        assertEquals(0L, ((Struct) transformed.value()).get("timestamp_to_int64"));
+        assertEquals(1L, ((Struct) transformed.value()).get("time_to_int64"));
+        assertEquals(Integer.MIN_VALUE, ((Struct) transformed.value()).get("decimal_to_int32"));
+        assertEquals(9.223372036854776E18, ((Struct) transformed.value()).get("timestamp_to_float64"));
+        assertNull(((Struct) transformed.value()).get("null_timestamp_to_int32"));
+
+        Schema transformedSchema = ((Struct) transformed.value()).schema();
+        assertEquals(Type.INT32, transformedSchema.field("date_to_int32").schema().type());
+        assertEquals(Type.INT64, transformedSchema.field("timestamp_to_int64").schema().type());
+        assertEquals(Type.INT64, transformedSchema.field("time_to_int64").schema().type());
+        assertEquals(Type.INT32, transformedSchema.field("decimal_to_int32").schema().type());
+        assertEquals(Type.FLOAT64, transformedSchema.field("timestamp_to_float64").schema().type());
+        assertEquals(Type.INT32, transformedSchema.field("null_timestamp_to_int32").schema().type());
+    }
+
+    @Test
+    public void castLogicalToString() {
+        Date date = new Date(MILLIS_PER_DAY);
+        Date time = new Date(MILLIS_PER_HOUR);
+        Date timestamp = new Date();
+
+        xformValue.configure(Collections.singletonMap(Cast.SPEC_CONFIG,
+            "date:string,decimal:string,time:string,timestamp:string"));
+
+        SchemaBuilder builder = SchemaBuilder.struct();
+        builder.field("date", org.apache.kafka.connect.data.Date.SCHEMA);
+        builder.field("decimal", Decimal.schema(new BigDecimal(1982).scale()));
+        builder.field("time", Time.SCHEMA);
+        builder.field("timestamp", Timestamp.SCHEMA);
+
+        Schema supportedTypesSchema = builder.build();
+
+        Struct recordValue = new Struct(supportedTypesSchema);
+        recordValue.put("date", date);
+        recordValue.put("decimal", new BigDecimal(1982));
+        recordValue.put("time", time);
+        recordValue.put("timestamp", timestamp);
+
+        SourceRecord transformed = xformValue.apply(
+            new SourceRecord(null, null, "topic", 0,
+                supportedTypesSchema, recordValue));
+
+        assertEquals(Values.dateFormatFor(date).format(date), ((Struct) transformed.value()).get("date"));
+        assertEquals("1982", ((Struct) transformed.value()).get("decimal"));
+        assertEquals(Values.dateFormatFor(time).format(time), ((Struct) transformed.value()).get("time"));
+        assertEquals(Values.dateFormatFor(timestamp).format(timestamp), ((Struct) transformed.value()).get("timestamp"));
+
+        Schema transformedSchema = ((Struct) transformed.value()).schema();
+        assertEquals(Type.STRING, transformedSchema.field("date").schema().type());
+        assertEquals(Type.STRING, transformedSchema.field("decimal").schema().type());
+        assertEquals(Type.STRING, transformedSchema.field("time").schema().type());
+        assertEquals(Type.STRING, transformedSchema.field("timestamp").schema().type());
+    }
 
     @Test
     public void castFieldsWithSchema() {
@@ -338,7 +435,7 @@ public void castFieldsWithSchema() {
         builder.field("boolean", Schema.BOOLEAN_SCHEMA);
         builder.field("string", Schema.STRING_SCHEMA);
         builder.field("bigdecimal", Decimal.schema(new BigDecimal(42).scale()));
-        builder.field("date", Timestamp.SCHEMA);
+        builder.field("date", org.apache.kafka.connect.data.Date.SCHEMA);
         builder.field("optional", Schema.OPTIONAL_FLOAT32_SCHEMA);
         builder.field("timestamp", Timestamp.SCHEMA);
         Schema supportedTypesSchema = builder.build();

From c87fe9402cbebc460b42cd3dd7c268e5e6e659d9 Mon Sep 17 00:00:00 2001
From: Adam Bellemare 
Date: Thu, 3 Oct 2019 18:59:31 -0400
Subject: [PATCH 0685/1071] KAFKA-3705 Added a foreignKeyJoin implementation
 for KTable. (#5527)

https://issues.apache.org/jira/browse/KAFKA-3705

Allows for a KTable to map its value to a given foreign key and join on another KTable keyed on that foreign key. Applies the joiner, then returns the tuples keyed on the original key. This supports updates from both sides of the join.

Reviewers: Guozhang Wang , Matthias J. Sax ,  John Roesler , Boyang Chen , Christopher Pettitt , Bill Bejeck , Jan Filipiak , pgwhalen, Alexei Daniline
---
 build.gradle                                  |   6 +
 checkstyle/suppressions.xml                   |  12 +-
 .../org/apache/kafka/common/utils/Bytes.java  |  28 +-
 .../apache/kafka/common/utils/BytesTest.java  |  84 +++
 .../scala/kafka/tools/StreamsResetter.java    |   4 +-
 .../apache/kafka/streams/kstream/KTable.java  |  88 +++
 .../streams/kstream/internals/KTableImpl.java | 252 ++++++-
 .../KTableSourceValueGetterSupplier.java      |   6 +-
 .../internals/foreignkeyjoin/CombinedKey.java |  55 ++
 .../foreignkeyjoin/CombinedKeySchema.java     |  96 +++
 ...eignJoinSubscriptionProcessorSupplier.java | 114 +++
 ...JoinSubscriptionSendProcessorSupplier.java | 116 +++
 ...scriptionJoinForeignProcessorSupplier.java | 124 ++++
 ...criptionResolverJoinProcessorSupplier.java | 107 +++
 .../SubscriptionResponseWrapper.java          |  62 ++
 .../SubscriptionResponseWrapperSerde.java     | 124 ++++
 ...criptionStoreReceiveProcessorSupplier.java | 112 +++
 .../foreignkeyjoin/SubscriptionWrapper.java   | 111 +++
 .../SubscriptionWrapperSerde.java             | 119 +++
 .../internals/graph/BaseRepartitionNode.java  |  10 +-
 .../GroupedTableOperationRepartitionNode.java |   3 +-
 ...bleKTableForeignKeyJoinResolutionNode.java |  81 ++
 .../graph/OptimizableRepartitionNode.java     |  19 +-
 .../internals/graph/ProcessorGraphNode.java   |   7 +
 .../graph/StatefulProcessorNode.java          |  18 +-
 .../internals/graph/StreamSinkNode.java       |   4 +
 .../internals/InternalProcessorContext.java   |  13 +
 .../internals/InternalTopologyBuilder.java    |  45 +-
 .../internals/RocksDBPrefixIterator.java      |  54 ++
 .../integration/ForeignKeyJoinSuite.java      |  47 ++
 ...reignKeyInnerJoinMultiIntegrationTest.java | 254 +++++++
 ...leKTableForeignKeyJoinIntegrationTest.java | 699 ++++++++++++++++++
 .../foreignkeyjoin/CombinedKeySchemaTest.java |  73 ++
 .../SubscriptionResponseWrapperSerdeTest.java |  91 +++
 .../SubscriptionWrapperSerdeTest.java         |  86 +++
 .../internals/RocksDBKeyValueStoreTest.java   |   3 +-
 .../state/internals/RocksDBStoreTest.java     |   2 +-
 .../scala/FunctionsCompatConversions.scala    |   6 +
 .../kafka/streams/scala/kstream/KTable.scala  |  38 +-
 39 files changed, 3141 insertions(+), 32 deletions(-)
 create mode 100644 clients/src/test/java/org/apache/kafka/common/utils/BytesTest.java
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKey.java
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchema.java
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionProcessorSupplier.java
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionSendProcessorSupplier.java
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionJoinForeignProcessorSupplier.java
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplier.java
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapper.java
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapperSerde.java
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionStoreReceiveProcessorSupplier.java
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapper.java
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerde.java
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/KTableKTableForeignKeyJoinResolutionNode.java
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBPrefixIterator.java
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/integration/ForeignKeyJoinSuite.java
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyInnerJoinMultiIntegrationTest.java
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinIntegrationTest.java
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchemaTest.java
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapperSerdeTest.java
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerdeTest.java

diff --git a/build.gradle b/build.gradle
index 2b7083f2921e1..210112bb13df5 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1224,6 +1224,12 @@ project(':streams') {
     if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
     standardOutput = new File(generatedDocsDir, "streams_config.html").newOutputStream()
   }
+
+  test {
+    // The suites are for running sets of tests in IDEs.
+    // Gradle will run each test class, so we exclude the suites to avoid redundantly running the tests twice.
+    exclude '**/*Suite.class'
+  }
 }
 
 project(':streams:streams-scala') {
diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index 6a24ef94c29a7..7a04f8d5b94f5 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -1,4 +1,4 @@
-
+
 
 
+    
 
     
@@ -90,6 +92,9 @@
     
 
+    
+
     
     
@@ -149,7 +154,7 @@
               files="(TopologyBuilder|KafkaStreams|KStreamImpl|KTableImpl|StreamThread|StreamTask).java"/>
 
     
+              files="(KTableImpl|StreamsPartitionAssignor.java)"/>
 
     
@@ -229,7 +234,8 @@
 
     
-
+    
     
     = 0; i--) {
+            if (inputArr[i] == (byte) 0xFF && carry == 1) {
+                ret[i] = (byte) 0x00;
+            } else {
+                ret[i] = (byte) (inputArr[i] + carry);
+                carry = 0;
+            }
+        }
+        if (carry == 0) {
+            return wrap(ret);
+        } else {
+            throw new IndexOutOfBoundsException();
+        }
+    }
+
+    /**
+     * A byte array comparator based on lexicograpic ordering.
      */
     public final static ByteArrayComparator BYTES_LEXICO_COMPARATOR = new LexicographicByteArrayComparator();
 
diff --git a/clients/src/test/java/org/apache/kafka/common/utils/BytesTest.java b/clients/src/test/java/org/apache/kafka/common/utils/BytesTest.java
new file mode 100644
index 0000000000000..bf7ec712ddca2
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/utils/BytesTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.utils;
+
+import org.junit.Test;
+
+import java.util.Comparator;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertEquals;
+
+public class BytesTest {
+
+    @Test
+    public void testIncrement() {
+        byte[] input = new byte[]{(byte) 0xAB, (byte) 0xCD, (byte) 0xFF};
+        byte[] expected = new byte[]{(byte) 0xAB, (byte) 0xCE, (byte) 0x00};
+        Bytes output = Bytes.increment(Bytes.wrap(input));
+        assertArrayEquals(output.get(), expected);
+    }
+
+    @Test
+    public void testIncrementUpperBoundary() {
+        byte[] input = new byte[]{(byte) 0xFF, (byte) 0xFF, (byte) 0xFF};
+        assertThrows(IndexOutOfBoundsException.class, () -> Bytes.increment(Bytes.wrap(input)));
+    }
+
+    @Test
+    public void testIncrementWithSubmap() {
+        final NavigableMap map = new TreeMap<>();
+        Bytes key1 = Bytes.wrap(new byte[]{(byte) 0xAA});
+        byte[] val = new byte[]{(byte) 0x00};
+        map.put(key1, val);
+
+        Bytes key2 = Bytes.wrap(new byte[]{(byte) 0xAA, (byte) 0xAA});
+        map.put(key2, val);
+
+        Bytes key3 = Bytes.wrap(new byte[]{(byte) 0xAA, (byte) 0x00, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF});
+        map.put(key3, val);
+
+        Bytes key4 = Bytes.wrap(new byte[]{(byte) 0xAB, (byte) 0x00});
+        map.put(key4, val);
+
+        Bytes key5 = Bytes.wrap(new byte[]{(byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x01});
+        map.put(key5, val);
+
+        Bytes prefix = key1;
+        Bytes prefixEnd = Bytes.increment(prefix);
+
+        Comparator comparator = map.comparator();
+        final int result = comparator == null ? prefix.compareTo(prefixEnd) : comparator.compare(prefix, prefixEnd);
+        NavigableMap subMapResults;
+        if (result > 0) {
+            //Prefix increment would cause a wrap-around. Get the submap from toKey to the end of the map
+            subMapResults = map.tailMap(prefix, true);
+        } else {
+            subMapResults = map.subMap(prefix, true, prefixEnd, false);
+        }
+
+        NavigableMap subMapExpected = new TreeMap<>();
+        subMapExpected.put(key1, val);
+        subMapExpected.put(key2, val);
+        subMapExpected.put(key3, val);
+
+        assertEquals(subMapExpected.keySet(), subMapResults.keySet());
+    }
+}
diff --git a/core/src/main/scala/kafka/tools/StreamsResetter.java b/core/src/main/scala/kafka/tools/StreamsResetter.java
index 597b9d34c239d..574e9c66e291a 100644
--- a/core/src/main/scala/kafka/tools/StreamsResetter.java
+++ b/core/src/main/scala/kafka/tools/StreamsResetter.java
@@ -675,7 +675,9 @@ private boolean isInternalTopic(final String topicName) {
         // Cf. https://issues.apache.org/jira/browse/KAFKA-7930
         return !isInputTopic(topicName) && !isIntermediateTopic(topicName)
             && topicName.startsWith(options.valueOf(applicationIdOption) + "-")
-            && (topicName.endsWith("-changelog") || topicName.endsWith("-repartition"));
+            && (topicName.endsWith("-changelog") || topicName.endsWith("-repartition")
+                || topicName.endsWith("-subscription-registration-topic")
+                || topicName.endsWith("-subscription-response-topic"));
     }
 
     public static void main(final String[] args) {
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java
index a2b9bafd9a78a..21e42df688ee0 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java
@@ -30,6 +30,8 @@
 import org.apache.kafka.streams.state.QueryableStoreType;
 import org.apache.kafka.streams.state.ReadOnlyKeyValueStore;
 
+import java.util.function.Function;
+
 /**
  * {@code KTable} is an abstraction of a changelog stream from a primary-keyed table.
  * Each record in this changelog stream is an update on the primary-keyed table with the record key as the primary key.
@@ -2117,6 +2119,92 @@  KTable outerJoin(final KTable other,
                                      final Named named,
                                      final Materialized> materialized);
 
+    /**
+     *
+     * Join records of this [[KTable]] with another [[KTable]]'s records using non-windowed inner join. Records from this
+     * table are joined according to the result of keyExtractor on the other KTable.
+     *
+     * @param other  the other {@code KTable} to be joined with this {@code KTable}. Keyed by KO.
+     * @param foreignKeyExtractor a {@link Function} that extracts the key (KO) from this table's value (V)
+     * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records
+     * @param named     a {@link Named} config used to name the processor in the topology
+     * @param materialized  a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
+     *                      should be materialized. Cannot be {@code null}
+     * @param  the value type of the result {@code KTable}
+     * @param  the key type of the other {@code KTable}
+     * @param  the value type of the other {@code KTable}
+     * @return
+     */
+     KTable join(final KTable other,
+                                    final Function foreignKeyExtractor,
+                                    final ValueJoiner joiner,
+                                    final Named named,
+                                    final Materialized> materialized);
+
+    /**
+     *
+     * Join records of this [[KTable]] with another [[KTable]]'s records using non-windowed inner join. Records from this
+     * table are joined according to the result of keyExtractor on the other KTable.
+     *
+     * @param other  the other {@code KTable} to be joined with this {@code KTable}. Keyed by KO.
+     * @param foreignKeyExtractor a {@link Function} that extracts the key (KO) from this table's value (V)
+     * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records
+     * @param materialized  a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
+     *                      should be materialized. Cannot be {@code null}
+     * @param  the value type of the result {@code KTable}
+     * @param  the key type of the other {@code KTable}
+     * @param  the value type of the other {@code KTable}
+     * @return
+     */
+     KTable join(final KTable other,
+                                    final Function foreignKeyExtractor,
+                                    final ValueJoiner joiner,
+                                    final Materialized> materialized);
+
+    /**
+     *
+     * Join records of this [[KTable]] with another [[KTable]]'s records using non-windowed left join. Records from this
+     * table are joined according to the result of keyExtractor on the other KTable.
+     *
+     * @param other  the other {@code KTable} to be joined with this {@code KTable}. Keyed by KO.
+     * @param foreignKeyExtractor a {@link Function} that extracts the key (KO) from this table's value (V). If the
+     *      *                            resultant foreignKey is null, the record will not propagate to the output.
+     * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records
+     * @param named     a {@link Named} config used to name the processor in the topology
+     * @param materialized  a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
+     *                      should be materialized. Cannot be {@code null}
+     * @param  the value type of the result {@code KTable}
+     * @param  the key type of the other {@code KTable}
+     * @param  the value type of the other {@code KTable}
+     * @return a {@code KTable} that contains only those records that satisfy the given predicate
+     */
+     KTable leftJoin(final KTable other,
+                                        final Function foreignKeyExtractor,
+                                        final ValueJoiner joiner,
+                                        final Named named,
+                                        final Materialized> materialized);
+
+    /**
+     *
+     * Join records of this [[KTable]] with another [[KTable]]'s records using non-windowed left join. Records from this
+     * table are joined according to the result of keyExtractor on the other KTable.
+     *
+     * @param other  the other {@code KTable} to be joined with this {@code KTable}. Keyed by KO.
+     * @param foreignKeyExtractor a {@link Function} that extracts the key (KO) from this table's value (V). If the
+     *                            resultant foreignKey is null, the record will not propagate to the output.
+     * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records
+     * @param materialized  a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
+     *                      should be materialized. Cannot be {@code null}
+     * @param  the value type of the result {@code KTable}
+     * @param  the key type of the other {@code KTable}
+     * @param  the value type of the other {@code KTable}
+     * @return a {@code KTable} that contains only those records that satisfy the given predicate
+     */
+     KTable leftJoin(final KTable other,
+                                        final Function foreignKeyExtractor,
+                                        final ValueJoiner joiner,
+                                        final Materialized> materialized);
+
     /**
      * Get the name of the local state store used that can be used to query this {@code KTable}.
      *
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
index 4bc102a746d92..05e04e8c2912d 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
@@ -17,8 +17,10 @@
 package org.apache.kafka.streams.kstream.internals;
 
 import org.apache.kafka.common.serialization.Serde;
+import org.apache.kafka.common.serialization.Serdes;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.Grouped;
 import org.apache.kafka.streams.kstream.KGroupedTable;
 import org.apache.kafka.streams.kstream.KStream;
@@ -27,15 +29,29 @@
 import org.apache.kafka.streams.kstream.Materialized;
 import org.apache.kafka.streams.kstream.Named;
 import org.apache.kafka.streams.kstream.Predicate;
+import org.apache.kafka.streams.kstream.Produced;
 import org.apache.kafka.streams.kstream.Suppressed;
 import org.apache.kafka.streams.kstream.ValueJoiner;
 import org.apache.kafka.streams.kstream.ValueMapper;
 import org.apache.kafka.streams.kstream.ValueMapperWithKey;
 import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier;
+import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.CombinedKey;
+import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.CombinedKeySchema;
+import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.ForeignJoinSubscriptionProcessorSupplier;
+import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.ForeignJoinSubscriptionSendProcessorSupplier;
+import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionJoinForeignProcessorSupplier;
+import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionResolverJoinProcessorSupplier;
+import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionResponseWrapper;
+import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionResponseWrapperSerde;
+import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionStoreReceiveProcessorSupplier;
+import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionWrapper;
+import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionWrapperSerde;
 import org.apache.kafka.streams.kstream.internals.graph.KTableKTableJoinNode;
 import org.apache.kafka.streams.kstream.internals.graph.ProcessorGraphNode;
 import org.apache.kafka.streams.kstream.internals.graph.ProcessorParameters;
 import org.apache.kafka.streams.kstream.internals.graph.StatefulProcessorNode;
+import org.apache.kafka.streams.kstream.internals.graph.StreamSinkNode;
+import org.apache.kafka.streams.kstream.internals.graph.StreamSourceNode;
 import org.apache.kafka.streams.kstream.internals.graph.StreamsGraphNode;
 import org.apache.kafka.streams.kstream.internals.graph.TableProcessorNode;
 import org.apache.kafka.streams.kstream.internals.suppress.FinalResultsSuppressionBuilder;
@@ -43,17 +59,22 @@
 import org.apache.kafka.streams.kstream.internals.suppress.NamedSuppressed;
 import org.apache.kafka.streams.kstream.internals.suppress.SuppressedInternal;
 import org.apache.kafka.streams.processor.ProcessorSupplier;
+import org.apache.kafka.streams.processor.internals.StaticTopicNameExtractor;
 import org.apache.kafka.streams.state.KeyValueStore;
 import org.apache.kafka.streams.state.StoreBuilder;
+import org.apache.kafka.streams.state.Stores;
 import org.apache.kafka.streams.state.TimestampedKeyValueStore;
+import org.apache.kafka.streams.state.ValueAndTimestamp;
 import org.apache.kafka.streams.state.internals.InMemoryTimeOrderedKeyValueBuffer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.time.Duration;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.Objects;
 import java.util.Set;
+import java.util.function.Function;
 
 import static org.apache.kafka.streams.kstream.internals.graph.GraphGraceSearchUtil.findAndVerifyWindowGrace;
 
@@ -89,6 +110,15 @@ public class KTableImpl extends AbstractStream implements KTable<
 
     private static final String TRANSFORMVALUES_NAME = "KTABLE-TRANSFORMVALUES-";
 
+    private static final String FK_JOIN_STATE_STORE_NAME = "KTABLE-INTERNAL-SUBSCRIPTION-STATE-STORE-";
+    private static final String SUBSCRIPTION_REGISTRATION = "KTABLE-SUBSCRIPTION-REGISTRATION-";
+    private static final String SUBSCRIPTION_RESPONSE = "KTABLE-SUBSCRIPTION-RESPONSE-";
+    private static final String SUBSCRIPTION_PROCESSOR = "KTABLE-SUBSCRIPTION-PROCESSOR-";
+    private static final String SUBSCRIPTION_RESPONSE_RESOLVER_PROCESSOR = "KTABLE-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-";
+    private static final String FK_JOIN_OUTPUT_PROCESSOR = "KTABLE-OUTPUT-PROCESSOR-";
+    private static final String TOPIC_SUFFIX = "-topic";
+    private static final String SINK_NAME = "KTABLE-SINK-";
+
     private final ProcessorSupplier processorSupplier;
 
     private final String queryableStoreName;
@@ -495,7 +525,7 @@ public KTable suppress(final Suppressed suppressed) {
             storeName,
             this
         );
-        
+
         final ProcessorGraphNode> node = new StatefulProcessorNode<>(
             name,
             new ProcessorParameters<>(suppressionSupplier, name),
@@ -803,4 +833,224 @@ private  ProcessorParameters unsafeCastProcessorParametersToCompletel
         return (ProcessorParameters) kObjectProcessorParameters;
     }
 
+    @Override
+    public  KTable join(final KTable other,
+                                           final Function foreignKeyExtractor,
+                                           final ValueJoiner joiner,
+                                           final Named named,
+                                           final Materialized> materialized) {
+
+        return doJoinOnForeignKey(other, foreignKeyExtractor, joiner, named, new MaterializedInternal<>(materialized), false);
+    }
+
+    @Override
+    public  KTable join(final KTable other,
+                                           final Function foreignKeyExtractor,
+                                           final ValueJoiner joiner,
+                                           final Materialized> materialized) {
+
+        return doJoinOnForeignKey(other, foreignKeyExtractor, joiner, NamedInternal.empty(), new MaterializedInternal<>(materialized), false);
+    }
+
+    @Override
+    public  KTable leftJoin(final KTable other,
+                                               final Function foreignKeyExtractor,
+                                               final ValueJoiner joiner,
+                                               final Named named,
+                                               final Materialized> materialized) {
+        return doJoinOnForeignKey(other, foreignKeyExtractor, joiner, named, new MaterializedInternal<>(materialized), true);
+    }
+
+    @Override
+    public  KTable leftJoin(final KTable other,
+                                               final Function foreignKeyExtractor,
+                                               final ValueJoiner joiner,
+                                               final Materialized> materialized) {
+
+        return doJoinOnForeignKey(other, foreignKeyExtractor, joiner, NamedInternal.empty(), new MaterializedInternal<>(materialized), true);
+    }
+
+
+    @SuppressWarnings("unchecked")
+    private  KTable doJoinOnForeignKey(final KTable foreignKeyTable,
+                                                          final Function foreignKeyExtractor,
+                                                          final ValueJoiner joiner,
+                                                          final Named joinName,
+                                                          final MaterializedInternal> materializedInternal,
+                                                          final boolean leftJoin) {
+        Objects.requireNonNull(foreignKeyTable, "foreignKeyTable can't be null");
+        Objects.requireNonNull(foreignKeyExtractor, "foreignKeyExtractor can't be null");
+        Objects.requireNonNull(joiner, "joiner can't be null");
+        Objects.requireNonNull(joinName, "joinName can't be null");
+        Objects.requireNonNull(materializedInternal, "materialized can't be null");
+
+        //Old values are a useful optimization. The old values from the foreignKeyTable table are compared to the new values,
+        //such that identical values do not cause a prefixScan. PrefixScan and propagation can be expensive and should
+        //not be done needlessly.
+        ((KTableImpl) foreignKeyTable).enableSendingOldValues();
+
+        //Old values must be sent such that the ForeignJoinSubscriptionSendProcessorSupplier can propagate deletions to the correct node.
+        //This occurs whenever the extracted foreignKey changes values.
+        enableSendingOldValues();
+
+        final Serde foreignKeySerde = ((KTableImpl) foreignKeyTable).keySerde;
+        final Serde> subscriptionWrapperSerde = new SubscriptionWrapperSerde<>(keySerde);
+        final SubscriptionResponseWrapperSerde responseWrapperSerde =
+            new SubscriptionResponseWrapperSerde<>(((KTableImpl) foreignKeyTable).valSerde);
+
+
+        final NamedInternal renamed = new NamedInternal(joinName);
+        final String subscriptionTopicName = renamed.suffixWithOrElseGet("-subscription-registration", builder, SUBSCRIPTION_REGISTRATION) + TOPIC_SUFFIX;
+        builder.internalTopologyBuilder.addInternalTopic(subscriptionTopicName);
+        final CombinedKeySchema combinedKeySchema = new CombinedKeySchema<>(subscriptionTopicName, foreignKeySerde, keySerde);
+
+        final ProcessorGraphNode> subscriptionNode = new ProcessorGraphNode<>(
+            new ProcessorParameters<>(
+                new ForeignJoinSubscriptionSendProcessorSupplier<>(
+                    foreignKeyExtractor,
+                    foreignKeySerde,
+                    subscriptionTopicName,
+                    valSerde.serializer(),
+                    leftJoin
+                ),
+                renamed.suffixWithOrElseGet("-subscription-registration-processor", builder, SUBSCRIPTION_REGISTRATION)
+            )
+        );
+        builder.addGraphNode(streamsGraphNode, subscriptionNode);
+
+
+        final StreamSinkNode> subscriptionSink = new StreamSinkNode<>(
+            renamed.suffixWithOrElseGet("-subscription-registration-sink", builder, SINK_NAME),
+            new StaticTopicNameExtractor<>(subscriptionTopicName),
+            new ProducedInternal<>(Produced.with(foreignKeySerde, subscriptionWrapperSerde))
+        );
+        builder.addGraphNode(subscriptionNode, subscriptionSink);
+
+        final StreamSourceNode> subscriptionSource = new StreamSourceNode<>(
+            renamed.suffixWithOrElseGet("-subscription-registration-source", builder, SOURCE_NAME),
+            Collections.singleton(subscriptionTopicName),
+            new ConsumedInternal<>(Consumed.with(foreignKeySerde, subscriptionWrapperSerde))
+        );
+        builder.addGraphNode(subscriptionSink, subscriptionSource);
+
+        // The subscription source is the source node on the *receiving* end *after* the repartition.
+        // This topic needs to be copartitioned with the Foreign Key table.
+        final Set copartitionedRepartitionSources =
+            new HashSet<>(((KTableImpl) foreignKeyTable).sourceNodes);
+        copartitionedRepartitionSources.add(subscriptionSource.nodeName());
+        builder.internalTopologyBuilder.copartitionSources(copartitionedRepartitionSources);
+
+
+        final StoreBuilder>> subscriptionStore =
+            Stores.timestampedKeyValueStoreBuilder(
+                Stores.persistentTimestampedKeyValueStore(
+                    renamed.suffixWithOrElseGet("-subscription-store", builder, FK_JOIN_STATE_STORE_NAME)
+                ),
+                new Serdes.BytesSerde(),
+                subscriptionWrapperSerde
+            );
+        builder.addStateStore(subscriptionStore);
+
+        final StatefulProcessorNode> subscriptionReceiveNode =
+            new StatefulProcessorNode<>(
+                new ProcessorParameters<>(
+                    new SubscriptionStoreReceiveProcessorSupplier<>(subscriptionStore, combinedKeySchema),
+                    renamed.suffixWithOrElseGet("-subscription-receive", builder, SUBSCRIPTION_PROCESSOR)
+                ),
+                Collections.singleton(subscriptionStore),
+                Collections.emptySet()
+            );
+        builder.addGraphNode(subscriptionSource, subscriptionReceiveNode);
+
+        final StatefulProcessorNode, Change>>> subscriptionJoinForeignNode =
+            new StatefulProcessorNode<>(
+                new ProcessorParameters<>(
+                    new SubscriptionJoinForeignProcessorSupplier<>(
+                        ((KTableImpl) foreignKeyTable).valueGetterSupplier()
+                    ),
+                    renamed.suffixWithOrElseGet("-subscription-join-foreign", builder, SUBSCRIPTION_PROCESSOR)
+                ),
+                Collections.emptySet(),
+                Collections.singleton(((KTableImpl) foreignKeyTable).valueGetterSupplier())
+            );
+        builder.addGraphNode(subscriptionReceiveNode, subscriptionJoinForeignNode);
+
+        final StatefulProcessorNode> foreignJoinSubscriptionNode = new StatefulProcessorNode<>(
+            new ProcessorParameters<>(
+                new ForeignJoinSubscriptionProcessorSupplier<>(subscriptionStore, combinedKeySchema),
+                renamed.suffixWithOrElseGet("-foreign-join-subscription", builder, SUBSCRIPTION_PROCESSOR)
+            ),
+            Collections.singleton(subscriptionStore),
+            Collections.emptySet()
+        );
+        builder.addGraphNode(((KTableImpl) foreignKeyTable).streamsGraphNode, foreignJoinSubscriptionNode);
+
+
+        final String finalRepartitionTopicName = renamed.suffixWithOrElseGet("-subscription-response", builder, SUBSCRIPTION_RESPONSE) + TOPIC_SUFFIX;
+        builder.internalTopologyBuilder.addInternalTopic(finalRepartitionTopicName);
+
+        final StreamSinkNode> foreignResponseSink =
+            new StreamSinkNode<>(
+                renamed.suffixWithOrElseGet("-subscription-response-sink", builder, SINK_NAME),
+                new StaticTopicNameExtractor<>(finalRepartitionTopicName),
+                new ProducedInternal<>(Produced.with(keySerde, responseWrapperSerde))
+            );
+        builder.addGraphNode(subscriptionJoinForeignNode, foreignResponseSink);
+        builder.addGraphNode(foreignJoinSubscriptionNode, foreignResponseSink);
+
+        final StreamSourceNode> foreignResponseSource = new StreamSourceNode<>(
+            renamed.suffixWithOrElseGet("-subscription-response-source", builder, SOURCE_NAME),
+            Collections.singleton(finalRepartitionTopicName),
+            new ConsumedInternal<>(Consumed.with(keySerde, responseWrapperSerde))
+        );
+        builder.addGraphNode(foreignResponseSink, foreignResponseSource);
+
+        // the response topic has to be copartitioned with the left (primary) side of the join
+        final Set resultSourceNodes = new HashSet<>(this.sourceNodes);
+        resultSourceNodes.add(foreignResponseSource.nodeName());
+        builder.internalTopologyBuilder.copartitionSources(resultSourceNodes);
+
+        final KTableValueGetterSupplier primaryKeyValueGetter = valueGetterSupplier();
+        final StatefulProcessorNode> resolverNode = new StatefulProcessorNode<>(
+            new ProcessorParameters<>(
+                new SubscriptionResolverJoinProcessorSupplier<>(
+                    primaryKeyValueGetter,
+                    valueSerde().serializer(),
+                    joiner,
+                    leftJoin
+                ),
+                renamed.suffixWithOrElseGet("-subscription-response-resolver", builder, SUBSCRIPTION_RESPONSE_RESOLVER_PROCESSOR)
+            ),
+            Collections.emptySet(),
+            Collections.singleton(primaryKeyValueGetter)
+        );
+        builder.addGraphNode(foreignResponseSource, resolverNode);
+
+        final String resultProcessorName = renamed.suffixWithOrElseGet("-result", builder, FK_JOIN_OUTPUT_PROCESSOR);
+        final KTableSource resultProcessorSupplier = new KTableSource<>(materializedInternal.storeName(), materializedInternal.queryableStoreName());
+        final StoreBuilder> resultStore =
+            materializedInternal.queryableStoreName() == null
+                ? null
+                : new TimestampedKeyValueStoreMaterializer<>(materializedInternal).materialize();
+        final TableProcessorNode resultNode = new TableProcessorNode<>(
+            resultProcessorName,
+            new ProcessorParameters<>(
+                resultProcessorSupplier,
+                resultProcessorName
+            ),
+            resultStore
+        );
+        builder.addGraphNode(resolverNode, resultNode);
+
+        return new KTableImpl(
+            resultProcessorName,
+            keySerde,
+            materializedInternal.valueSerde(),
+            resultSourceNodes,
+            materializedInternal.storeName(),
+            resultProcessorSupplier,
+            resultNode,
+            builder
+        );
+    }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableSourceValueGetterSupplier.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableSourceValueGetterSupplier.java
index 7083b88cc5471..bed221387bd72 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableSourceValueGetterSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableSourceValueGetterSupplier.java
@@ -14,6 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.kafka.streams.kstream.internals;
 
 import org.apache.kafka.streams.processor.ProcessorContext;
@@ -23,7 +24,7 @@
 public class KTableSourceValueGetterSupplier implements KTableValueGetterSupplier {
     private final String storeName;
 
-    KTableSourceValueGetterSupplier(final String storeName) {
+    public KTableSourceValueGetterSupplier(final String storeName) {
         this.storeName = storeName;
     }
 
@@ -49,6 +50,7 @@ public ValueAndTimestamp get(final K key) {
         }
 
         @Override
-        public void close() {}
+        public void close() {
+        }
     }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKey.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKey.java
new file mode 100644
index 0000000000000..0dd60fc026d59
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKey.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.kstream.internals.foreignkeyjoin;
+
+import java.util.Objects;
+
+public class CombinedKey {
+    private final KF foreignKey;
+    private final KP primaryKey;
+
+    CombinedKey(final KF foreignKey, final KP primaryKey) {
+        Objects.requireNonNull(foreignKey, "foreignKey can't be null");
+        Objects.requireNonNull(primaryKey, "primaryKey can't be null");
+        this.foreignKey = foreignKey;
+        this.primaryKey = primaryKey;
+    }
+
+    public KF getForeignKey() {
+        return foreignKey;
+    }
+
+    public KP getPrimaryKey() {
+        return primaryKey;
+    }
+
+    public boolean equals(final KF foreignKey, final KP primaryKey) {
+        if (this.primaryKey == null) {
+            return false;
+        }
+        return this.foreignKey.equals(foreignKey) && this.primaryKey.equals(primaryKey);
+    }
+
+    @Override
+    public String toString() {
+        return "CombinedKey{" +
+                "foreignKey=" + foreignKey +
+                ", primaryKey=" + primaryKey +
+                '}';
+    }
+}
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchema.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchema.java
new file mode 100644
index 0000000000000..8abe583b0c002
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchema.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.kstream.internals.foreignkeyjoin;
+
+import org.apache.kafka.common.serialization.Deserializer;
+import org.apache.kafka.common.serialization.Serde;
+import org.apache.kafka.common.serialization.Serializer;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.processor.ProcessorContext;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Factory for creating CombinedKey serializers / deserializers.
+ */
+public class CombinedKeySchema {
+    private final String serdeTopic;
+    private Serializer primaryKeySerializer;
+    private Deserializer primaryKeyDeserializer;
+    private Serializer foreignKeySerializer;
+    private Deserializer foreignKeyDeserializer;
+
+    public CombinedKeySchema(final String serdeTopic, final Serde foreignKeySerde, final Serde primaryKeySerde) {
+        this.serdeTopic = serdeTopic;
+        primaryKeySerializer = primaryKeySerde.serializer();
+        primaryKeyDeserializer = primaryKeySerde.deserializer();
+        foreignKeyDeserializer = foreignKeySerde.deserializer();
+        foreignKeySerializer = foreignKeySerde.serializer();
+    }
+
+    @SuppressWarnings("unchecked")
+    public void init(final ProcessorContext context) {
+        primaryKeySerializer = primaryKeySerializer == null ? (Serializer) context.keySerde().serializer() : primaryKeySerializer;
+        primaryKeyDeserializer = primaryKeyDeserializer == null ? (Deserializer) context.keySerde().deserializer() : primaryKeyDeserializer;
+        foreignKeySerializer = foreignKeySerializer == null ? (Serializer) context.keySerde().serializer() : foreignKeySerializer;
+        foreignKeyDeserializer = foreignKeyDeserializer == null ? (Deserializer) context.keySerde().deserializer() : foreignKeyDeserializer;
+    }
+
+    Bytes toBytes(final KO foreignKey, final K primaryKey) {
+        //The serialization format - note that primaryKeySerialized may be null, such as when a prefixScan
+        //key is being created.
+        //{Integer.BYTES foreignKeyLength}{foreignKeySerialized}{Optional-primaryKeySerialized}
+        final byte[] foreignKeySerializedData = foreignKeySerializer.serialize(serdeTopic, foreignKey);
+
+        //? bytes
+        final byte[] primaryKeySerializedData = primaryKeySerializer.serialize(serdeTopic, primaryKey);
+
+        final ByteBuffer buf = ByteBuffer.allocate(Integer.BYTES + foreignKeySerializedData.length + primaryKeySerializedData.length);
+        buf.putInt(foreignKeySerializedData.length);
+        buf.put(foreignKeySerializedData);
+        buf.put(primaryKeySerializedData);
+        return Bytes.wrap(buf.array());
+    }
+
+
+    public CombinedKey fromBytes(final Bytes data) {
+        //{Integer.BYTES foreignKeyLength}{foreignKeySerialized}{Optional-primaryKeySerialized}
+        final byte[] dataArray = data.get();
+        final ByteBuffer dataBuffer = ByteBuffer.wrap(dataArray);
+        final int foreignKeyLength = dataBuffer.getInt();
+        final byte[] foreignKeyRaw = new byte[foreignKeyLength];
+        dataBuffer.get(foreignKeyRaw, 0, foreignKeyLength);
+        final KO foreignKey = foreignKeyDeserializer.deserialize(serdeTopic, foreignKeyRaw);
+
+        final byte[] primaryKeyRaw = new byte[dataArray.length - foreignKeyLength - Integer.BYTES];
+        dataBuffer.get(primaryKeyRaw, 0, primaryKeyRaw.length);
+        final K primaryKey = primaryKeyDeserializer.deserialize(serdeTopic, primaryKeyRaw);
+        return new CombinedKey<>(foreignKey, primaryKey);
+    }
+
+    Bytes prefixBytes(final KO key) {
+        //The serialization format. Note that primaryKeySerialized is not required/used in this function.
+        //{Integer.BYTES foreignKeyLength}{foreignKeySerialized}{Optional-primaryKeySerialized}
+
+        final byte[] foreignKeySerializedData = foreignKeySerializer.serialize(serdeTopic, key);
+
+        final ByteBuffer buf = ByteBuffer.allocate(Integer.BYTES + foreignKeySerializedData.length);
+        buf.putInt(foreignKeySerializedData.length);
+        buf.put(foreignKeySerializedData);
+        return Bytes.wrap(buf.array());
+    }
+}
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionProcessorSupplier.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionProcessorSupplier.java
new file mode 100644
index 0000000000000..614b91f071e5e
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionProcessorSupplier.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.kstream.internals.foreignkeyjoin;
+
+import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.errors.StreamsException;
+import org.apache.kafka.streams.kstream.internals.Change;
+import org.apache.kafka.streams.processor.AbstractProcessor;
+import org.apache.kafka.streams.processor.Processor;
+import org.apache.kafka.streams.processor.ProcessorContext;
+import org.apache.kafka.streams.processor.ProcessorSupplier;
+import org.apache.kafka.streams.processor.internals.InternalProcessorContext;
+import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
+import org.apache.kafka.streams.state.KeyValueIterator;
+import org.apache.kafka.streams.state.StoreBuilder;
+import org.apache.kafka.streams.state.TimestampedKeyValueStore;
+import org.apache.kafka.streams.state.ValueAndTimestamp;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+
+public class ForeignJoinSubscriptionProcessorSupplier implements ProcessorSupplier> {
+    private static final Logger LOG = LoggerFactory.getLogger(ForeignJoinSubscriptionProcessorSupplier.class);
+    private final StoreBuilder>> storeBuilder;
+    private final CombinedKeySchema keySchema;
+
+    public ForeignJoinSubscriptionProcessorSupplier(
+        final StoreBuilder>> storeBuilder,
+        final CombinedKeySchema keySchema) {
+
+        this.storeBuilder = storeBuilder;
+        this.keySchema = keySchema;
+    }
+
+    @Override
+    public Processor> get() {
+        return new KTableKTableJoinProcessor();
+    }
+
+
+    private final class KTableKTableJoinProcessor extends AbstractProcessor> {
+        private Sensor skippedRecordsSensor;
+        private TimestampedKeyValueStore> store;
+
+        @Override
+        public void init(final ProcessorContext context) {
+            super.init(context);
+            final InternalProcessorContext internalProcessorContext = (InternalProcessorContext) context;
+            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(internalProcessorContext.metrics());
+            store = internalProcessorContext.getStateStore(storeBuilder);
+        }
+
+        /**
+         * @throws StreamsException if key is null
+         */
+        @Override
+        public void process(final KO key, final Change value) {
+            // if the key is null, we do not need proceed aggregating
+            // the record with the table
+            if (key == null) {
+                LOG.warn(
+                    "Skipping record due to null key. value=[{}] topic=[{}] partition=[{}] offset=[{}]",
+                    value, context().topic(), context().partition(), context().offset()
+                );
+                skippedRecordsSensor.record();
+                return;
+            }
+
+            final Bytes prefixBytes = keySchema.prefixBytes(key);
+
+            //Perform the prefixScan and propagate the results
+            try (final KeyValueIterator>> prefixScanResults =
+                     store.range(prefixBytes, Bytes.increment(prefixBytes))) {
+
+                while (prefixScanResults.hasNext()) {
+                    final KeyValue>> next = prefixScanResults.next();
+                    // have to check the prefix because the range end is inclusive :(
+                    if (prefixEquals(next.key.get(), prefixBytes.get())) {
+                        final CombinedKey combinedKey = keySchema.fromBytes(next.key);
+                        context().forward(
+                            combinedKey.getPrimaryKey(),
+                            new SubscriptionResponseWrapper<>(next.value.value().getHash(), value.newValue)
+                        );
+                    }
+                }
+            }
+        }
+
+        private boolean prefixEquals(final byte[] x, final byte[] y) {
+            final int min = Math.min(x.length, y.length);
+            final ByteBuffer xSlice = ByteBuffer.wrap(x, 0, min);
+            final ByteBuffer ySlice = ByteBuffer.wrap(y, 0, min);
+            return xSlice.equals(ySlice);
+        }
+    }
+}
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionSendProcessorSupplier.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionSendProcessorSupplier.java
new file mode 100644
index 0000000000000..f122258d93049
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionSendProcessorSupplier.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.kstream.internals.foreignkeyjoin;
+
+import org.apache.kafka.common.serialization.Serde;
+import org.apache.kafka.common.serialization.Serializer;
+import org.apache.kafka.streams.kstream.internals.Change;
+import org.apache.kafka.streams.processor.AbstractProcessor;
+import org.apache.kafka.streams.processor.Processor;
+import org.apache.kafka.streams.processor.ProcessorContext;
+import org.apache.kafka.streams.processor.ProcessorSupplier;
+import org.apache.kafka.streams.state.internals.Murmur3;
+
+import java.util.function.Function;
+import java.util.Arrays;
+
+import static org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionWrapper.Instruction.PROPAGATE_ONLY_IF_FK_VAL_AVAILABLE;
+import static org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionWrapper.Instruction.PROPAGATE_NULL_IF_NO_FK_VAL_AVAILABLE;
+import static org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionWrapper.Instruction.DELETE_KEY_AND_PROPAGATE;
+import static org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionWrapper.Instruction.DELETE_KEY_NO_PROPAGATE;
+
+public class ForeignJoinSubscriptionSendProcessorSupplier implements ProcessorSupplier> {
+
+    private final Function foreignKeyExtractor;
+    private final String repartitionTopicName;
+    private final Serializer valueSerializer;
+    private final boolean leftJoin;
+    private Serializer foreignKeySerializer;
+
+    public ForeignJoinSubscriptionSendProcessorSupplier(final Function foreignKeyExtractor,
+                                                        final Serde foreignKeySerde,
+                                                        final String repartitionTopicName,
+                                                        final Serializer valueSerializer,
+                                                        final boolean leftJoin) {
+        this.foreignKeyExtractor = foreignKeyExtractor;
+        this.valueSerializer = valueSerializer;
+        this.leftJoin = leftJoin;
+        this.repartitionTopicName = repartitionTopicName;
+        foreignKeySerializer = foreignKeySerde == null ? null : foreignKeySerde.serializer();
+    }
+
+    @Override
+    public Processor> get() {
+        return new UnbindChangeProcessor();
+    }
+
+    private class UnbindChangeProcessor extends AbstractProcessor> {
+        
+        @SuppressWarnings("unchecked")
+        @Override
+        public void init(final ProcessorContext context) {
+            super.init(context);
+            // get default key serde if it wasn't supplied directly at construction
+            if (foreignKeySerializer == null) {
+                foreignKeySerializer = (Serializer) context.keySerde().serializer();
+            }
+        }
+
+        @Override
+        public void process(final K key, final Change change) {
+            final long[] currentHash = change.newValue == null ?
+                    null :
+                    Murmur3.hash128(valueSerializer.serialize(repartitionTopicName, change.newValue));
+
+            if (change.oldValue != null) {
+                final KO oldForeignKey = foreignKeyExtractor.apply(change.oldValue);
+                if (change.newValue != null) {
+                    final KO newForeignKey = foreignKeyExtractor.apply(change.newValue);
+
+                    final byte[] serialOldForeignKey = foreignKeySerializer.serialize(repartitionTopicName, oldForeignKey);
+                    final byte[] serialNewForeignKey = foreignKeySerializer.serialize(repartitionTopicName, newForeignKey);
+                    if (!Arrays.equals(serialNewForeignKey, serialOldForeignKey)) {
+                        //Different Foreign Key - delete the old key value and propagate the new one.
+                        //Delete it from the oldKey's state store
+                        context().forward(oldForeignKey, new SubscriptionWrapper<>(currentHash, DELETE_KEY_NO_PROPAGATE, key));
+                        //Add to the newKey's state store. Additionally, propagate null if no FK is found there,
+                        //since we must "unset" any output set by the previous FK-join. This is true for both INNER
+                        //and LEFT join.
+                    }
+                    context().forward(newForeignKey, new SubscriptionWrapper<>(currentHash, PROPAGATE_NULL_IF_NO_FK_VAL_AVAILABLE, key));
+                } else {
+                    //A simple propagatable delete. Delete from the state store and propagate the delete onwards.
+                    context().forward(oldForeignKey, new SubscriptionWrapper<>(currentHash, DELETE_KEY_AND_PROPAGATE, key));
+                }
+            } else if (change.newValue != null) {
+                //change.oldValue is null, which means it was deleted at least once before, or it is brand new.
+                //In either case, we only need to propagate if the FK_VAL is available, as the null from the delete would
+                //have been propagated otherwise.
+
+                final SubscriptionWrapper.Instruction instruction;
+                if (leftJoin) {
+                    //Want to send info even if RHS is null.
+                    instruction = PROPAGATE_NULL_IF_NO_FK_VAL_AVAILABLE;
+                } else {
+                    instruction = PROPAGATE_ONLY_IF_FK_VAL_AVAILABLE;
+                }
+                context().forward(foreignKeyExtractor.apply(change.newValue), new SubscriptionWrapper<>(currentHash, instruction, key));
+            }
+        }
+    }
+}
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionJoinForeignProcessorSupplier.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionJoinForeignProcessorSupplier.java
new file mode 100644
index 0000000000000..2544eb1856b7c
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionJoinForeignProcessorSupplier.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.kstream.internals.foreignkeyjoin;
+
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.streams.kstream.internals.Change;
+import org.apache.kafka.streams.kstream.internals.KTableValueGetter;
+import org.apache.kafka.streams.kstream.internals.KTableValueGetterSupplier;
+import org.apache.kafka.streams.processor.AbstractProcessor;
+import org.apache.kafka.streams.processor.Processor;
+import org.apache.kafka.streams.processor.ProcessorContext;
+import org.apache.kafka.streams.processor.ProcessorSupplier;
+import org.apache.kafka.streams.processor.To;
+import org.apache.kafka.streams.state.ValueAndTimestamp;
+
+import java.util.Objects;
+
+/**
+ * Receives {@code SubscriptionWrapper} events and processes them according to their Instruction.
+ * Depending on the results, {@code SubscriptionResponseWrapper}s are created, which will be propagated to
+ * the {@code SubscriptionResolverJoinProcessorSupplier} instance.
+ *
+ * @param  Type of primary keys
+ * @param  Type of foreign key
+ * @param  Type of foreign value
+ */
+public class SubscriptionJoinForeignProcessorSupplier
+    implements ProcessorSupplier, Change>>> {
+
+    private final KTableValueGetterSupplier foreignValueGetterSupplier;
+
+    public SubscriptionJoinForeignProcessorSupplier(final KTableValueGetterSupplier foreignValueGetterSupplier) {
+        this.foreignValueGetterSupplier = foreignValueGetterSupplier;
+    }
+
+    @Override
+    public Processor, Change>>> get() {
+
+        return new AbstractProcessor, Change>>>() {
+
+            private KTableValueGetter foreignValues;
+
+            @Override
+            public void init(final ProcessorContext context) {
+                super.init(context);
+                foreignValues = foreignValueGetterSupplier.get();
+                foreignValues.init(context);
+            }
+
+            @Override
+            public void process(final CombinedKey combinedKey, final Change>> change) {
+                Objects.requireNonNull(combinedKey, "This processor should never see a null key.");
+                Objects.requireNonNull(change, "This processor should never see a null value.");
+                final ValueAndTimestamp> valueAndTimestamp = change.newValue;
+                Objects.requireNonNull(valueAndTimestamp, "This processor should never see a null newValue.");
+                final SubscriptionWrapper value = valueAndTimestamp.value();
+
+                if (value.getVersion() != SubscriptionWrapper.CURRENT_VERSION) {
+                    //Guard against modifications to SubscriptionWrapper. Need to ensure that there is compatibility
+                    //with previous versions to enable rolling upgrades. Must develop a strategy for upgrading
+                    //from older SubscriptionWrapper versions to newer versions.
+                    throw new UnsupportedVersionException("SubscriptionWrapper is of an incompatible version.");
+                }
+
+                final ValueAndTimestamp foreignValueAndTime = foreignValues.get(combinedKey.getForeignKey());
+
+                final long resultTimestamp =
+                    foreignValueAndTime == null ?
+                        valueAndTimestamp.timestamp() :
+                        Math.max(valueAndTimestamp.timestamp(), foreignValueAndTime.timestamp());
+
+                switch (value.getInstruction()) {
+                    case DELETE_KEY_AND_PROPAGATE:
+                        context().forward(
+                            combinedKey.getPrimaryKey(),
+                            new SubscriptionResponseWrapper(value.getHash(), null),
+                            To.all().withTimestamp(resultTimestamp)
+                        );
+                        break;
+                    case PROPAGATE_NULL_IF_NO_FK_VAL_AVAILABLE:
+                        //This one needs to go through regardless of LEFT or INNER join, since the extracted FK was
+                        //changed and there is no match for it. We must propagate the (key, null) to ensure that the
+                        //downstream consumers are alerted to this fact.
+                        final VO valueToSend = foreignValueAndTime == null ? null : foreignValueAndTime.value();
+
+                        context().forward(
+                            combinedKey.getPrimaryKey(),
+                            new SubscriptionResponseWrapper<>(value.getHash(), valueToSend),
+                            To.all().withTimestamp(resultTimestamp)
+                        );
+                        break;
+                    case PROPAGATE_ONLY_IF_FK_VAL_AVAILABLE:
+                        if (foreignValueAndTime != null) {
+                            context().forward(
+                                combinedKey.getPrimaryKey(),
+                                new SubscriptionResponseWrapper<>(value.getHash(), foreignValueAndTime.value()),
+                                To.all().withTimestamp(resultTimestamp)
+                            );
+                        }
+                        break;
+                    case DELETE_KEY_NO_PROPAGATE:
+                        break;
+                    default:
+                        throw new IllegalStateException("Unhandled instruction: " + value.getInstruction());
+                }
+            }
+        };
+    }
+}
\ No newline at end of file
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplier.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplier.java
new file mode 100644
index 0000000000000..a188f15062c66
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplier.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.kstream.internals.foreignkeyjoin;
+
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.serialization.Serializer;
+import org.apache.kafka.streams.kstream.ValueJoiner;
+import org.apache.kafka.streams.kstream.internals.KTableValueGetter;
+import org.apache.kafka.streams.kstream.internals.KTableValueGetterSupplier;
+import org.apache.kafka.streams.processor.AbstractProcessor;
+import org.apache.kafka.streams.processor.Processor;
+import org.apache.kafka.streams.processor.ProcessorContext;
+import org.apache.kafka.streams.processor.ProcessorSupplier;
+import org.apache.kafka.streams.state.ValueAndTimestamp;
+import org.apache.kafka.streams.state.internals.Murmur3;
+
+/**
+ * Receives {@code SubscriptionResponseWrapper} events and filters out events which do not match the current hash
+ * of the primary key. This eliminates race-condition results for rapidly-changing foreign-keys for a given primary key.
+ * Applies the join and emits nulls according to LEFT/INNER rules.
+ *
+ * @param  Type of primary keys
+ * @param  Type of primary values
+ * @param  Type of foreign values
+ * @param  Type of joined result of primary and foreign values
+ */
+public class SubscriptionResolverJoinProcessorSupplier implements ProcessorSupplier> {
+    private final KTableValueGetterSupplier valueGetterSupplier;
+    private final Serializer valueSerializer;
+    private final ValueJoiner joiner;
+    private final boolean leftJoin;
+
+    public SubscriptionResolverJoinProcessorSupplier(final KTableValueGetterSupplier valueGetterSupplier,
+                                                     final Serializer valueSerializer,
+                                                     final ValueJoiner joiner,
+                                                     final boolean leftJoin) {
+        this.valueGetterSupplier = valueGetterSupplier;
+        this.valueSerializer = valueSerializer;
+        this.joiner = joiner;
+        this.leftJoin = leftJoin;
+    }
+
+    @Override
+    public Processor> get() {
+        return new AbstractProcessor>() {
+
+            private KTableValueGetter valueGetter;
+
+            @Override
+            public void init(final ProcessorContext context) {
+                super.init(context);
+                valueGetter = valueGetterSupplier.get();
+                valueGetter.init(context);
+            }
+
+            @Override
+            public void process(final K key, final SubscriptionResponseWrapper value) {
+                if (value.getVersion() != SubscriptionResponseWrapper.CURRENT_VERSION) {
+                    //Guard against modifications to SubscriptionResponseWrapper. Need to ensure that there is
+                    //compatibility with previous versions to enable rolling upgrades. Must develop a strategy for
+                    //upgrading from older SubscriptionWrapper versions to newer versions.
+                    throw new UnsupportedVersionException("SubscriptionResponseWrapper is of an incompatible version.");
+                }
+                final ValueAndTimestamp currentValueWithTimestamp = valueGetter.get(key);
+
+                //We are unable to access the actual source topic name for the valueSerializer at runtime, without
+                //tightly coupling to KTableRepartitionProcessorSupplier.
+                //While we can use the source topic from where the events came from, we shouldn't serialize against it
+                //as it causes problems with the confluent schema registry, which requires each topic have only a single
+                //registered schema.
+                final String dummySerializationTopic = context().topic() + "-join-resolver";
+                final long[] currentHash = currentValueWithTimestamp == null ?
+                    null :
+                    Murmur3.hash128(valueSerializer.serialize(dummySerializationTopic, currentValueWithTimestamp.value()));
+
+                final long[] messageHash = value.getOriginalValueHash();
+
+                //If this value doesn't match the current value from the original table, it is stale and should be discarded.
+                if (java.util.Arrays.equals(messageHash, currentHash)) {
+                    final VR result;
+
+                    if (value.getForeignValue() == null && (!leftJoin || currentValueWithTimestamp == null)) {
+                        result = null; //Emit tombstone
+                    } else {
+                        result = joiner.apply(currentValueWithTimestamp == null ? null : currentValueWithTimestamp.value(), value.getForeignValue());
+                    }
+                    context().forward(key, result);
+                }
+            }
+        };
+    }
+}
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapper.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapper.java
new file mode 100644
index 0000000000000..9c79e468213a0
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapper.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.kstream.internals.foreignkeyjoin;
+
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+
+import java.util.Arrays;
+
+public class SubscriptionResponseWrapper {
+    final static byte CURRENT_VERSION = 0x00;
+    private final long[] originalValueHash;
+    private final FV foreignValue;
+    private final byte version;
+
+    public SubscriptionResponseWrapper(final long[] originalValueHash, final FV foreignValue) {
+        this(originalValueHash, foreignValue, CURRENT_VERSION);
+    }
+
+    public SubscriptionResponseWrapper(final long[] originalValueHash, final FV foreignValue, final byte version) {
+        if (version != CURRENT_VERSION) {
+            throw new UnsupportedVersionException("SubscriptionWrapper does not support version " + version);
+        }
+        this.originalValueHash = originalValueHash;
+        this.foreignValue = foreignValue;
+        this.version = version;
+    }
+
+    public long[] getOriginalValueHash() {
+        return originalValueHash;
+    }
+
+    public FV getForeignValue() {
+        return foreignValue;
+    }
+
+    public byte getVersion() {
+        return version;
+    }
+
+    @Override
+    public String toString() {
+        return "SubscriptionResponseWrapper{" +
+            "version=" + version +
+            ", foreignValue=" + foreignValue +
+            ", originalValueHash=" + Arrays.toString(originalValueHash) +
+            '}';
+    }
+}
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapperSerde.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapperSerde.java
new file mode 100644
index 0000000000000..6524b4fc7647a
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapperSerde.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.kstream.internals.foreignkeyjoin;
+
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.serialization.Deserializer;
+import org.apache.kafka.common.serialization.Serde;
+import org.apache.kafka.common.serialization.Serializer;
+
+import java.nio.ByteBuffer;
+
+public class SubscriptionResponseWrapperSerde implements Serde> {
+    private final SubscriptionResponseWrapperSerializer serializer;
+    private final SubscriptionResponseWrapperDeserializer deserializer;
+
+    public SubscriptionResponseWrapperSerde(final Serde foreignValueSerde) {
+        serializer = new SubscriptionResponseWrapperSerializer<>(foreignValueSerde.serializer());
+        deserializer = new SubscriptionResponseWrapperDeserializer<>(foreignValueSerde.deserializer());
+    }
+
+    @Override
+    public Serializer> serializer() {
+        return serializer;
+    }
+
+    @Override
+    public Deserializer> deserializer() {
+        return deserializer;
+    }
+
+    private static final class SubscriptionResponseWrapperSerializer implements Serializer> {
+        private final Serializer serializer;
+
+        private SubscriptionResponseWrapperSerializer(final Serializer serializer) {
+            this.serializer = serializer;
+        }
+
+        @Override
+        public byte[] serialize(final String topic, final SubscriptionResponseWrapper data) {
+            //{1-bit-isHashNull}{7-bits-version}{Optional-16-byte-Hash}{n-bytes serialized data}
+
+            //7-bit (0x7F) maximum for data version.
+            if (Byte.compare((byte) 0x7F, data.getVersion()) < 0) {
+                throw new UnsupportedVersionException("SubscriptionResponseWrapper version is larger than maximum supported 0x7F");
+            }
+
+            final byte[] serializedData = serializer.serialize(topic, data.getForeignValue());
+            final int serializedDataLength = serializedData == null ? 0 : serializedData.length;
+            final long[] originalHash = data.getOriginalValueHash();
+            final int hashLength = originalHash == null ? 0 : 2 * Long.BYTES;
+
+            final ByteBuffer buf = ByteBuffer.allocate(1 + hashLength + serializedDataLength);
+
+            if (originalHash != null) {
+                buf.put(data.getVersion());
+                buf.putLong(originalHash[0]);
+                buf.putLong(originalHash[1]);
+            } else {
+                //Don't store hash as it's null.
+                buf.put((byte) (data.getVersion() | (byte) 0x80));
+            }
+
+            if (serializedData != null)
+                buf.put(serializedData);
+            return buf.array();
+        }
+
+    }
+
+    private static final class SubscriptionResponseWrapperDeserializer implements Deserializer> {
+        private final Deserializer deserializer;
+
+        private SubscriptionResponseWrapperDeserializer(final Deserializer deserializer) {
+            this.deserializer = deserializer;
+        }
+
+        @Override
+        public SubscriptionResponseWrapper deserialize(final String topic, final byte[] data) {
+            //{1-bit-isHashNull}{7-bits-version}{Optional-16-byte-Hash}{n-bytes serialized data}
+
+            final ByteBuffer buf = ByteBuffer.wrap(data);
+            final byte versionAndIsHashNull = buf.get();
+            final byte version = (byte) (0x7F & versionAndIsHashNull);
+            final boolean isHashNull = (0x80 & versionAndIsHashNull) == 0x80;
+
+            final long[] hash;
+            int lengthSum = 1; //The first byte
+            if (isHashNull) {
+                hash = null;
+            } else {
+                hash = new long[2];
+                hash[0] = buf.getLong();
+                hash[1] = buf.getLong();
+                lengthSum += 2 * Long.BYTES;
+            }
+
+            final byte[] serializedValue;
+            if (data.length - lengthSum > 0) {
+                serializedValue = new byte[data.length - lengthSum];
+                buf.get(serializedValue, 0, serializedValue.length);
+            } else
+                serializedValue = null;
+
+            final V value = deserializer.deserialize(topic, serializedValue);
+            return new SubscriptionResponseWrapper<>(hash, value, version);
+        }
+
+    }
+
+}
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionStoreReceiveProcessorSupplier.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionStoreReceiveProcessorSupplier.java
new file mode 100644
index 0000000000000..3d5f5160197a2
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionStoreReceiveProcessorSupplier.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.kstream.internals.foreignkeyjoin;
+
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.kstream.internals.Change;
+import org.apache.kafka.streams.processor.AbstractProcessor;
+import org.apache.kafka.streams.processor.Processor;
+import org.apache.kafka.streams.processor.ProcessorContext;
+import org.apache.kafka.streams.processor.ProcessorSupplier;
+import org.apache.kafka.streams.processor.To;
+import org.apache.kafka.streams.processor.internals.InternalProcessorContext;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
+import org.apache.kafka.streams.state.StoreBuilder;
+import org.apache.kafka.streams.state.TimestampedKeyValueStore;
+import org.apache.kafka.streams.state.ValueAndTimestamp;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SubscriptionStoreReceiveProcessorSupplier
+    implements ProcessorSupplier> {
+    private static final Logger LOG = LoggerFactory.getLogger(SubscriptionStoreReceiveProcessorSupplier.class);
+
+    private final StoreBuilder>> storeBuilder;
+    private final CombinedKeySchema keySchema;
+
+    public SubscriptionStoreReceiveProcessorSupplier(
+        final StoreBuilder>> storeBuilder,
+        final CombinedKeySchema keySchema) {
+
+        this.storeBuilder = storeBuilder;
+        this.keySchema = keySchema;
+    }
+
+    @Override
+    public Processor> get() {
+
+        return new AbstractProcessor>() {
+
+            private TimestampedKeyValueStore> store;
+            private StreamsMetricsImpl metrics;
+            private Sensor skippedRecordsSensor;
+
+            @Override
+            public void init(final ProcessorContext context) {
+                super.init(context);
+                final InternalProcessorContext internalProcessorContext = (InternalProcessorContext) context;
+
+                metrics = internalProcessorContext.metrics();
+                skippedRecordsSensor = ThreadMetrics.skipRecordSensor(metrics);
+                store = internalProcessorContext.getStateStore(storeBuilder);
+            }
+
+            @Override
+            public void process(final KO key, final SubscriptionWrapper value) {
+                if (key == null) {
+                    LOG.warn(
+                        "Skipping record due to null foreign key. value=[{}] topic=[{}] partition=[{}] offset=[{}]",
+                        value, context().topic(), context().partition(), context().offset()
+                    );
+                    skippedRecordsSensor.record();
+                    return;
+                }
+                if (value.getVersion() != SubscriptionWrapper.CURRENT_VERSION) {
+                    //Guard against modifications to SubscriptionWrapper. Need to ensure that there is compatibility
+                    //with previous versions to enable rolling upgrades. Must develop a strategy for upgrading
+                    //from older SubscriptionWrapper versions to newer versions.
+                    throw new UnsupportedVersionException("SubscriptionWrapper is of an incompatible version.");
+                }
+
+                final Bytes subscriptionKey = keySchema.toBytes(key, value.getPrimaryKey());
+
+                final ValueAndTimestamp> newValue = ValueAndTimestamp.make(value, context().timestamp());
+                final ValueAndTimestamp> oldValue = store.get(subscriptionKey);
+
+                //If the subscriptionWrapper hash indicates a null, must delete from statestore.
+                //This store is used by the prefix scanner in ForeignJoinSubscriptionProcessorSupplier
+                if (value.getHash() == null) {
+                    store.delete(subscriptionKey);
+                } else {
+                    store.put(subscriptionKey, newValue);
+                }
+                final Change>> change = new Change<>(newValue, oldValue);
+                // note: key is non-nullable
+                // note: newValue is non-nullable
+                context().forward(
+                    new CombinedKey<>(key, value.getPrimaryKey()),
+                    change,
+                    To.all().withTimestamp(newValue.timestamp())
+                );
+            }
+        };
+    }
+}
\ No newline at end of file
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapper.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapper.java
new file mode 100644
index 0000000000000..a757895aecf0d
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapper.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.kstream.internals.foreignkeyjoin;
+
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+
+import java.util.Arrays;
+import java.util.Objects;
+
+
+public class SubscriptionWrapper {
+    static final byte CURRENT_VERSION = 0;
+
+    private final long[] hash;
+    private final Instruction instruction;
+    private final byte version;
+    private final K primaryKey;
+
+    public enum Instruction {
+        //Send nothing. Do not propagate.
+        DELETE_KEY_NO_PROPAGATE((byte) 0x00),
+
+        //Send (k, null)
+        DELETE_KEY_AND_PROPAGATE((byte) 0x01),
+
+        //(changing foreign key, but FK+Val may not exist)
+        //Send (k, fk-val) OR
+        //Send (k, null) if fk-val does not exist
+        PROPAGATE_NULL_IF_NO_FK_VAL_AVAILABLE((byte) 0x02),
+
+        //(first time ever sending key)
+        //Send (k, fk-val) only if fk-val exists.
+        PROPAGATE_ONLY_IF_FK_VAL_AVAILABLE((byte) 0x03);
+
+        private final byte value;
+        Instruction(final byte value) {
+            this.value = value;
+        }
+
+        public byte getValue() {
+            return value;
+        }
+
+        public static Instruction fromValue(final byte value) {
+            for (final Instruction i: values()) {
+                if (i.value == value) {
+                    return i;
+                }
+            }
+            throw new IllegalArgumentException("Unknown instruction byte value = " + value);
+        }
+    }
+
+    public SubscriptionWrapper(final long[] hash, final Instruction instruction, final K primaryKey) {
+        this(hash, instruction, primaryKey, CURRENT_VERSION);
+    }
+
+    public SubscriptionWrapper(final long[] hash, final Instruction instruction, final K primaryKey, final byte version) {
+        Objects.requireNonNull(instruction, "instruction cannot be null. Required by downstream processor.");
+        Objects.requireNonNull(primaryKey, "primaryKey cannot be null. Required by downstream processor.");
+        if (version != CURRENT_VERSION) {
+            throw new UnsupportedVersionException("SubscriptionWrapper does not support version " + version);
+        }
+
+        this.instruction = instruction;
+        this.hash = hash;
+        this.primaryKey = primaryKey;
+        this.version = version;
+    }
+
+    public Instruction getInstruction() {
+        return instruction;
+    }
+
+    public long[] getHash() {
+        return hash;
+    }
+
+    public K getPrimaryKey() {
+        return primaryKey;
+    }
+
+    public byte getVersion() {
+        return version;
+    }
+
+    @Override
+    public String toString() {
+        return "SubscriptionWrapper{" +
+            "version=" + version +
+            ", primaryKey=" + primaryKey +
+            ", instruction=" + instruction +
+            ", hash=" + Arrays.toString(hash) +
+            '}';
+    }
+}
+
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerde.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerde.java
new file mode 100644
index 0000000000000..ae53ba8b3498a
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerde.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.kstream.internals.foreignkeyjoin;
+
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.serialization.Deserializer;
+import org.apache.kafka.common.serialization.Serde;
+import org.apache.kafka.common.serialization.Serializer;
+
+import java.nio.ByteBuffer;
+
+public class SubscriptionWrapperSerde implements Serde> {
+    private final SubscriptionWrapperSerializer serializer;
+    private final SubscriptionWrapperDeserializer deserializer;
+
+    public SubscriptionWrapperSerde(final Serde primaryKeySerde) {
+        serializer = new SubscriptionWrapperSerializer<>(primaryKeySerde.serializer());
+        deserializer = new SubscriptionWrapperDeserializer<>(primaryKeySerde.deserializer());
+    }
+
+    @Override
+    public Serializer> serializer() {
+        return serializer;
+    }
+
+    @Override
+    public Deserializer> deserializer() {
+        return deserializer;
+    }
+
+    private static class SubscriptionWrapperSerializer implements Serializer> {
+        private final Serializer primaryKeySerializer;
+        SubscriptionWrapperSerializer(final Serializer primaryKeySerializer) {
+            this.primaryKeySerializer = primaryKeySerializer;
+        }
+
+        @Override
+        public byte[] serialize(final String topic, final SubscriptionWrapper data) {
+            //{1-bit-isHashNull}{7-bits-version}{1-byte-instruction}{Optional-16-byte-Hash}{PK-serialized}
+
+            //7-bit (0x7F) maximum for data version.
+            if (Byte.compare((byte) 0x7F, data.getVersion()) < 0) {
+                throw new UnsupportedVersionException("SubscriptionWrapper version is larger than maximum supported 0x7F");
+            }
+
+            final byte[] primaryKeySerializedData = primaryKeySerializer.serialize(topic, data.getPrimaryKey());
+
+            final ByteBuffer buf;
+            if (data.getHash() != null) {
+                buf = ByteBuffer.allocate(2 + 2 * Long.BYTES + primaryKeySerializedData.length);
+                buf.put(data.getVersion());
+            } else {
+                //Don't store hash as it's null.
+                buf = ByteBuffer.allocate(2 + primaryKeySerializedData.length);
+                buf.put((byte) (data.getVersion() | (byte) 0x80));
+            }
+
+            buf.put(data.getInstruction().getValue());
+            final long[] elem = data.getHash();
+            if (data.getHash() != null) {
+                buf.putLong(elem[0]);
+                buf.putLong(elem[1]);
+            }
+            buf.put(primaryKeySerializedData);
+            return buf.array();
+        }
+
+    }
+
+    private static class SubscriptionWrapperDeserializer implements Deserializer> {
+        private final Deserializer primaryKeyDeserializer;
+        SubscriptionWrapperDeserializer(final Deserializer primaryKeyDeserializer) {
+            this.primaryKeyDeserializer = primaryKeyDeserializer;
+        }
+
+        @Override
+        public SubscriptionWrapper deserialize(final String topic, final byte[] data) {
+            //{7-bits-version}{1-bit-isHashNull}{1-byte-instruction}{Optional-16-byte-Hash}{PK-serialized}
+            final ByteBuffer buf = ByteBuffer.wrap(data);
+            final byte versionAndIsHashNull = buf.get();
+            final byte version = (byte) (0x7F & versionAndIsHashNull);
+            final boolean isHashNull = (0x80 & versionAndIsHashNull) == 0x80;
+            final SubscriptionWrapper.Instruction inst = SubscriptionWrapper.Instruction.fromValue(buf.get());
+
+            final long[] hash;
+            int lengthSum = 2; //The first 2 bytes
+            if (isHashNull) {
+                hash = null;
+            } else {
+                hash = new long[2];
+                hash[0] = buf.getLong();
+                hash[1] = buf.getLong();
+                lengthSum += 2 * Long.BYTES;
+            }
+
+            final byte[] primaryKeyRaw = new byte[data.length - lengthSum]; //The remaining data is the serialized pk
+            buf.get(primaryKeyRaw, 0, primaryKeyRaw.length);
+            final K primaryKey = primaryKeyDeserializer.deserialize(topic, primaryKeyRaw);
+
+            return new SubscriptionWrapper<>(hash, inst, primaryKey, version);
+        }
+
+    }
+
+}
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java
index 460f640b5c35b..2cc153975b706 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java
@@ -20,6 +20,7 @@
 import org.apache.kafka.common.serialization.Deserializer;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serializer;
+import org.apache.kafka.streams.processor.StreamPartitioner;
 
 public abstract class BaseRepartitionNode extends StreamsGraphNode {
 
@@ -29,7 +30,7 @@ public abstract class BaseRepartitionNode extends StreamsGraphNode {
     protected final String sourceName;
     protected final String repartitionTopic;
     protected final ProcessorParameters processorParameters;
-
+    protected final StreamPartitioner partitioner;
 
     BaseRepartitionNode(final String nodeName,
                         final String sourceName,
@@ -37,7 +38,8 @@ public abstract class BaseRepartitionNode extends StreamsGraphNode {
                         final Serde keySerde,
                         final Serde valueSerde,
                         final String sinkName,
-                        final String repartitionTopic) {
+                        final String repartitionTopic,
+                        final StreamPartitioner partitioner) {
 
         super(nodeName);
 
@@ -47,6 +49,7 @@ public abstract class BaseRepartitionNode extends StreamsGraphNode {
         this.sourceName = sourceName;
         this.repartitionTopic = repartitionTopic;
         this.processorParameters = processorParameters;
+        this.partitioner = partitioner;
     }
 
     abstract Serializer getValueSerializer();
@@ -61,7 +64,8 @@ public String toString() {
                ", sinkName='" + sinkName + '\'' +
                ", sourceName='" + sourceName + '\'' +
                ", repartitionTopic='" + repartitionTopic + '\'' +
-               ", processorParameters=" + processorParameters +
+               ", processorParameters=" + processorParameters + '\'' +
+               ", partitioner=" + partitioner +
                "} " + super.toString();
     }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java
index 4d1b67dbc33ed..a3f79c50ea9ab 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java
@@ -43,7 +43,8 @@ private GroupedTableOperationRepartitionNode(final String nodeName,
             keySerde,
             valueSerde,
             sinkName,
-            repartitionTopic
+            repartitionTopic,
+            null
         );
     }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/KTableKTableForeignKeyJoinResolutionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/KTableKTableForeignKeyJoinResolutionNode.java
new file mode 100644
index 0000000000000..672625cfc1cd2
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/KTableKTableForeignKeyJoinResolutionNode.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.kstream.internals.graph;
+
+import org.apache.kafka.common.serialization.Serde;
+import org.apache.kafka.streams.kstream.internals.Change;
+import org.apache.kafka.streams.kstream.internals.KTableValueGetterSupplier;
+import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionResponseWrapper;
+import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionWrapper;
+import org.apache.kafka.streams.processor.FailOnInvalidTimestamp;
+import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder;
+
+/**
+ * Too much specific information to generalize so the Foreign Key KTable-KTable join requires a specific node.
+ */
+public class KTableKTableForeignKeyJoinResolutionNode extends StreamsGraphNode {
+    private final ProcessorParameters> joinOneToOneProcessorParameters;
+    private final ProcessorParameters> joinByPrefixProcessorParameters;
+    private final ProcessorParameters> resolverProcessorParameters;
+    private final String finalRepartitionTopicName;
+    private final String finalRepartitionSinkName;
+    private final String finalRepartitionSourceName;
+    private final Serde keySerde;
+    private final Serde> subResponseSerde;
+    private final KTableValueGetterSupplier originalValueGetter;
+
+    public KTableKTableForeignKeyJoinResolutionNode(final String nodeName,
+                                                    final ProcessorParameters> joinOneToOneProcessorParameters,
+                                                    final ProcessorParameters> joinByPrefixProcessorParameters,
+                                                    final ProcessorParameters> resolverProcessorParameters,
+                                                    final String finalRepartitionTopicName,
+                                                    final String finalRepartitionSinkName,
+                                                    final String finalRepartitionSourceName,
+                                                    final Serde keySerde,
+                                                    final Serde> subResponseSerde,
+                                                    final KTableValueGetterSupplier originalValueGetter
+    ) {
+        super(nodeName);
+        this.joinOneToOneProcessorParameters = joinOneToOneProcessorParameters;
+        this.joinByPrefixProcessorParameters = joinByPrefixProcessorParameters;
+        this.resolverProcessorParameters = resolverProcessorParameters;
+        this.finalRepartitionTopicName = finalRepartitionTopicName;
+        this.finalRepartitionSinkName = finalRepartitionSinkName;
+        this.finalRepartitionSourceName = finalRepartitionSourceName;
+        this.keySerde = keySerde;
+        this.subResponseSerde = subResponseSerde;
+        this.originalValueGetter = originalValueGetter;
+    }
+
+    @Override
+    public void writeToTopology(final InternalTopologyBuilder topologyBuilder) {
+        topologyBuilder.addInternalTopic(finalRepartitionTopicName);
+        //Repartition back to the original partitioning structure
+        topologyBuilder.addSink(finalRepartitionSinkName, finalRepartitionTopicName,
+                keySerde.serializer(), subResponseSerde.serializer(),
+                null,
+                joinByPrefixProcessorParameters.processorName(), joinOneToOneProcessorParameters.processorName());
+
+        topologyBuilder.addSource(null, finalRepartitionSourceName, new FailOnInvalidTimestamp(),
+                keySerde.deserializer(), subResponseSerde.deserializer(), finalRepartitionTopicName);
+
+        //Connect highwaterProcessor to source, add the state store, and connect the statestore with the processor.
+        topologyBuilder.addProcessor(resolverProcessorParameters.processorName(), resolverProcessorParameters.processorSupplier(), finalRepartitionSourceName);
+        topologyBuilder.connectProcessorAndStateStores(resolverProcessorParameters.processorName(), originalValueGetter.storeNames());
+    }
+}
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java
index 4797a21399d5d..e3cf2b80d45ee 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java
@@ -22,6 +22,7 @@
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serializer;
 import org.apache.kafka.streams.processor.FailOnInvalidTimestamp;
+import org.apache.kafka.streams.processor.StreamPartitioner;
 import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder;
 
 public class OptimizableRepartitionNode extends BaseRepartitionNode {
@@ -32,7 +33,8 @@ public class OptimizableRepartitionNode extends BaseRepartitionNode
                                final Serde keySerde,
                                final Serde valueSerde,
                                final String sinkName,
-                               final String repartitionTopic) {
+                               final String repartitionTopic,
+                               final StreamPartitioner partitioner) {
 
         super(
             nodeName,
@@ -41,9 +43,9 @@ public class OptimizableRepartitionNode extends BaseRepartitionNode
             keySerde,
             valueSerde,
             sinkName,
-            repartitionTopic
+            repartitionTopic,
+            partitioner
         );
-
     }
 
     public Serde keySerde() {
@@ -91,7 +93,7 @@ public void writeToTopology(final InternalTopologyBuilder topologyBuilder) {
             repartitionTopic,
             keySerializer,
             getValueSerializer(),
-            null,
+            partitioner,
             processorParameters.processorName()
         );
 
@@ -120,6 +122,7 @@ public static final class OptimizableRepartitionNodeBuilder {
         private String sinkName;
         private String sourceName;
         private String repartitionTopic;
+        private StreamPartitioner partitioner;
 
         private OptimizableRepartitionNodeBuilder() {
         }
@@ -160,6 +163,11 @@ public OptimizableRepartitionNodeBuilder withNodeName(final String nodeNam
             return this;
         }
 
+        public OptimizableRepartitionNodeBuilder withPartitioner(final StreamPartitioner partitioner) {
+            this.partitioner = partitioner;
+            return this;
+        }
+
         public OptimizableRepartitionNode build() {
 
             return new OptimizableRepartitionNode<>(
@@ -169,7 +177,8 @@ public OptimizableRepartitionNode build() {
                 keySerde,
                 valueSerde,
                 sinkName,
-                repartitionTopic
+                repartitionTopic,
+                partitioner
             );
 
         }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/ProcessorGraphNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/ProcessorGraphNode.java
index 2cfe3ccb54c6f..5c75a09816aa6 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/ProcessorGraphNode.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/ProcessorGraphNode.java
@@ -28,6 +28,13 @@ public class ProcessorGraphNode extends StreamsGraphNode {
 
     private final ProcessorParameters processorParameters;
 
+    public ProcessorGraphNode(final ProcessorParameters processorParameters) {
+
+        super(processorParameters.processorName());
+
+        this.processorParameters = processorParameters;
+    }
+
     public ProcessorGraphNode(final String nodeName,
                               final ProcessorParameters processorParameters) {
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/StatefulProcessorNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/StatefulProcessorNode.java
index 1e910cea151ea..6ed2917da6711 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/StatefulProcessorNode.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/StatefulProcessorNode.java
@@ -18,21 +18,36 @@
 package org.apache.kafka.streams.kstream.internals.graph;
 
 
+import org.apache.kafka.streams.kstream.internals.KTableValueGetterSupplier;
 import org.apache.kafka.streams.processor.ProcessorSupplier;
 import org.apache.kafka.streams.processor.StateStore;
 import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder;
 import org.apache.kafka.streams.state.StoreBuilder;
 
 import java.util.Arrays;
+import java.util.Set;
+import java.util.stream.Stream;
 
 public class StatefulProcessorNode extends ProcessorGraphNode {
 
     private final String[] storeNames;
     private final StoreBuilder storeBuilder;
 
+    /**
+     * Create a node representing a stateful processor, where the named stores have already been registered.
+     */
+    public StatefulProcessorNode(final ProcessorParameters processorParameters,
+                                 final Set> preRegisteredStores,
+                                 final Set> valueGetterSuppliers) {
+        super(processorParameters.processorName(), processorParameters);
+        final Stream registeredStoreNames = preRegisteredStores.stream().map(StoreBuilder::name);
+        final Stream valueGetterStoreNames = valueGetterSuppliers.stream().flatMap(s -> Arrays.stream(s.storeNames()));
+        storeNames = Stream.concat(registeredStoreNames, valueGetterStoreNames).toArray(String[]::new);
+        storeBuilder = null;
+    }
 
     /**
-     * Create a node representing a stateful processor, where the named store has already been registered.
+     * Create a node representing a stateful processor, where the named stores have already been registered.
      */
     public StatefulProcessorNode(final String nodeName,
                                  final ProcessorParameters processorParameters,
@@ -80,5 +95,6 @@ public void writeToTopology(final InternalTopologyBuilder topologyBuilder) {
         if (storeBuilder != null) {
             topologyBuilder.addStateStore(storeBuilder, processorName);
         }
+
     }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/StreamSinkNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/StreamSinkNode.java
index dfe7f9e44b02a..40ce357600501 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/StreamSinkNode.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/StreamSinkNode.java
@@ -24,6 +24,7 @@
 import org.apache.kafka.streams.processor.StreamPartitioner;
 import org.apache.kafka.streams.processor.TopicNameExtractor;
 import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder;
+import org.apache.kafka.streams.processor.internals.StaticTopicNameExtractor;
 
 public class StreamSinkNode extends StreamsGraphNode {
 
@@ -60,6 +61,9 @@ public void writeToTopology(final InternalTopologyBuilder topologyBuilder) {
             @SuppressWarnings("unchecked")
             final StreamPartitioner windowedPartitioner = (StreamPartitioner) new WindowedStreamPartitioner((WindowedSerializer) keySerializer);
             topologyBuilder.addSink(nodeName(), topicNameExtractor, keySerializer, valSerializer, windowedPartitioner, parentNames);
+        } else if (topicNameExtractor instanceof StaticTopicNameExtractor) {
+            final String topicName = ((StaticTopicNameExtractor) topicNameExtractor).topicName;
+            topologyBuilder.addSink(nodeName(), topicName, keySerializer, valSerializer, partitioner, parentNames);
         } else {
             topologyBuilder.addSink(nodeName(), topicNameExtractor, keySerializer, valSerializer, partitioner,  parentNames);
         }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalProcessorContext.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalProcessorContext.java
index 2a1d05e2807db..23f7ef86ca0d6 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalProcessorContext.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalProcessorContext.java
@@ -18,7 +18,9 @@
 
 import org.apache.kafka.streams.processor.ProcessorContext;
 import org.apache.kafka.streams.processor.RecordContext;
+import org.apache.kafka.streams.processor.StateStore;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.state.StoreBuilder;
 import org.apache.kafka.streams.state.internals.ThreadCache;
 
 /**
@@ -66,4 +68,15 @@ public interface InternalProcessorContext extends ProcessorContext {
      * Mark this context as being uninitialized
      */
     void uninitialize();
+
+    /**
+     * Get a correctly typed state store, given a handle on the original builder.
+     * @param builder
+     * @param 
+     * @return
+     */
+    @SuppressWarnings("unchecked")
+    default  T getStateStore(final StoreBuilder builder) {
+        return (T) getStateStore(builder.name());
+    }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java
index 3a7bd55668d48..d34ff00c68629 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java
@@ -29,8 +29,8 @@
 import org.apache.kafka.streams.processor.TopicNameExtractor;
 import org.apache.kafka.streams.state.StoreBuilder;
 import org.apache.kafka.streams.state.internals.SessionStoreBuilder;
-import org.apache.kafka.streams.state.internals.WindowStoreBuilder;
 import org.apache.kafka.streams.state.internals.TimestampedWindowStoreBuilder;
+import org.apache.kafka.streams.state.internals.WindowStoreBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -50,6 +50,7 @@
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 public class InternalTopologyBuilder {
 
@@ -1149,21 +1150,45 @@ public Map> stateStoreNameToSourceTopics() {
     }
 
     public synchronized Collection> copartitionGroups() {
-        final List> list = new ArrayList<>(copartitionSourceGroups.size());
-        for (final Set nodeNames : copartitionSourceGroups) {
-            final Set copartitionGroup = new HashSet<>();
-            for (final String node : nodeNames) {
-                final List topics = nodeToSourceTopics.get(node);
-                if (topics != null) {
-                    copartitionGroup.addAll(maybeDecorateInternalSourceTopics(topics));
+        // compute transitive closures of copartitionGroups to relieve registering code to know all members
+        // of a copartitionGroup at the same time
+        final List> copartitionSourceTopics =
+            copartitionSourceGroups
+                .stream()
+                .map(sourceGroup ->
+                         sourceGroup
+                             .stream()
+                             .flatMap(node -> maybeDecorateInternalSourceTopics(nodeToSourceTopics.get(node)).stream())
+                             .collect(Collectors.toSet())
+                ).collect(Collectors.toList());
+
+        final Map> topicsToCopartitionGroup = new LinkedHashMap<>();
+        for (final Set topics : copartitionSourceTopics) {
+            if (topics != null) {
+                Set coparititonGroup = null;
+                for (final String topic : topics) {
+                    coparititonGroup = topicsToCopartitionGroup.get(topic);
+                    if (coparititonGroup != null) {
+                        break;
+                    }
+                }
+                if (coparititonGroup == null) {
+                    coparititonGroup = new HashSet<>();
+                }
+                coparititonGroup.addAll(maybeDecorateInternalSourceTopics(topics));
+                for (final String topic : topics) {
+                    topicsToCopartitionGroup.put(topic, coparititonGroup);
                 }
             }
-            list.add(Collections.unmodifiableSet(copartitionGroup));
         }
-        return Collections.unmodifiableList(list);
+        final Set> uniqueCopartitionGroups = new HashSet<>(topicsToCopartitionGroup.values());
+        return Collections.unmodifiableList(new ArrayList<>(uniqueCopartitionGroups));
     }
 
     private List maybeDecorateInternalSourceTopics(final Collection sourceTopics) {
+        if (sourceTopics == null) {
+            return Collections.emptyList();
+        }
         final List decoratedTopics = new ArrayList<>();
         for (final String topic : sourceTopics) {
             if (internalTopicNames.contains(topic)) {
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBPrefixIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBPrefixIterator.java
new file mode 100644
index 0000000000000..b84175e1ee0c6
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBPrefixIterator.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.state.KeyValueIterator;
+import org.rocksdb.RocksIterator;
+
+import java.util.Set;
+
+class RocksDBPrefixIterator extends RocksDbIterator {
+    private byte[] rawPrefix;
+
+    RocksDBPrefixIterator(final String name,
+                          final RocksIterator newIterator,
+                          final Set> openIterators,
+                          final Bytes prefix) {
+        super(name, newIterator, openIterators);
+        rawPrefix = prefix.get();
+        newIterator.seek(rawPrefix);
+    }
+
+    @Override
+    public synchronized boolean hasNext() {
+        if (!super.hasNext()) {
+            return false;
+        }
+
+        final byte[] rawNextKey = super.peekNextKey().get();
+        for (int i = 0; i < rawPrefix.length; i++) {
+            if (i == rawNextKey.length) {
+                throw new IllegalStateException("Unexpected RocksDB Key Value. Should have been skipped with seek.");
+            }
+            if (rawNextKey[i] != rawPrefix[i]) {
+                return false;
+            }
+        }
+        return true;
+    }
+}
\ No newline at end of file
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/ForeignKeyJoinSuite.java b/streams/src/test/java/org/apache/kafka/streams/integration/ForeignKeyJoinSuite.java
new file mode 100644
index 0000000000000..6245b11af6004
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/ForeignKeyJoinSuite.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.integration;
+
+import org.apache.kafka.common.utils.BytesTest;
+import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.CombinedKeySchemaTest;
+import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionResponseWrapperSerdeTest;
+import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionWrapperSerdeTest;
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+
+/**
+ * This suite runs all the tests related to the KTable-KTable foreign key join feature.
+ *
+ * It can be used from an IDE to selectively just run these tests when developing code related to KTable-KTable
+ * foreign key join.
+ *
+ * If desired, it can also be added to a Gradle build task, although this isn't strictly necessary, since all
+ * these tests are already included in the `:streams:test` task.
+ */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+    BytesTest.class,
+    KTableKTableForeignKeyInnerJoinMultiIntegrationTest.class,
+    KTableKTableForeignKeyJoinIntegrationTest.class,
+    CombinedKeySchemaTest.class,
+    SubscriptionWrapperSerdeTest.class,
+    SubscriptionResponseWrapperSerdeTest.class
+})
+public class ForeignKeyJoinSuite {
+}
+
+
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyInnerJoinMultiIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyInnerJoinMultiIntegrationTest.java
new file mode 100644
index 0000000000000..ad746d8f611c6
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyInnerJoinMultiIntegrationTest.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.integration;
+
+import kafka.utils.MockTime;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.serialization.FloatSerializer;
+import org.apache.kafka.common.serialization.IntegerDeserializer;
+import org.apache.kafka.common.serialization.IntegerSerializer;
+import org.apache.kafka.common.serialization.LongSerializer;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
+import org.apache.kafka.streams.integration.utils.IntegrationTestUtils;
+import org.apache.kafka.streams.kstream.Consumed;
+import org.apache.kafka.streams.kstream.KTable;
+import org.apache.kafka.streams.kstream.Materialized;
+import org.apache.kafka.streams.kstream.Produced;
+import org.apache.kafka.streams.kstream.ValueJoiner;
+import org.apache.kafka.streams.state.KeyValueStore;
+import org.apache.kafka.test.IntegrationTest;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+import java.util.function.Function;
+
+import static org.junit.Assert.assertEquals;
+
+@Category({IntegrationTest.class})
+public class KTableKTableForeignKeyInnerJoinMultiIntegrationTest {
+    private final static int NUM_BROKERS = 1;
+
+    @ClassRule
+    public final static EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(NUM_BROKERS);
+    private final static MockTime MOCK_TIME = CLUSTER.time;
+    private final static String TABLE_1 = "table1";
+    private final static String TABLE_2 = "table2";
+    private final static String TABLE_3 = "table3";
+    private final static String OUTPUT = "output-";
+    private static Properties streamsConfig;
+    private KafkaStreams streams;
+    private KafkaStreams streamsTwo;
+    private KafkaStreams streamsThree;
+    private final static Properties CONSUMER_CONFIG = new Properties();
+
+    private final static Properties PRODUCER_CONFIG_1 = new Properties();
+    private final static Properties PRODUCER_CONFIG_2 = new Properties();
+    private final static Properties PRODUCER_CONFIG_3 = new Properties();
+
+    @BeforeClass
+    public static void beforeTest() throws Exception {
+        //Use multiple partitions to ensure distribution of keys.
+
+        CLUSTER.createTopic(TABLE_1, 3, 1);
+        CLUSTER.createTopic(TABLE_2, 5, 1);
+        CLUSTER.createTopic(TABLE_3, 7, 1);
+        CLUSTER.createTopic(OUTPUT, 11, 1);
+
+        PRODUCER_CONFIG_1.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
+        PRODUCER_CONFIG_1.put(ProducerConfig.ACKS_CONFIG, "all");
+        PRODUCER_CONFIG_1.put(ProducerConfig.RETRIES_CONFIG, 0);
+        PRODUCER_CONFIG_1.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class);
+        PRODUCER_CONFIG_1.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, FloatSerializer.class);
+
+        PRODUCER_CONFIG_2.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
+        PRODUCER_CONFIG_2.put(ProducerConfig.ACKS_CONFIG, "all");
+        PRODUCER_CONFIG_2.put(ProducerConfig.RETRIES_CONFIG, 0);
+        PRODUCER_CONFIG_2.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
+        PRODUCER_CONFIG_2.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, LongSerializer.class);
+
+        PRODUCER_CONFIG_3.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
+        PRODUCER_CONFIG_3.put(ProducerConfig.ACKS_CONFIG, "all");
+        PRODUCER_CONFIG_3.put(ProducerConfig.RETRIES_CONFIG, 0);
+        PRODUCER_CONFIG_3.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class);
+        PRODUCER_CONFIG_3.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
+
+        streamsConfig = new Properties();
+        streamsConfig.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
+        streamsConfig.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+        streamsConfig.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath());
+        streamsConfig.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
+        streamsConfig.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 100);
+
+        final List> table1 = Arrays.asList(
+            new KeyValue<>(1, 1.33f),
+            new KeyValue<>(2, 2.22f),
+            new KeyValue<>(3, -1.22f), //Won't be joined in yet.
+            new KeyValue<>(4, -2.22f)  //Won't be joined in at all.
+        );
+
+        //Partitions pre-computed using the default Murmur2 hash, just to ensure that all 3 partitions will be exercised.
+        final List> table2 = Arrays.asList(
+            new KeyValue<>("0", 0L),  //partition 2
+            new KeyValue<>("1", 10L), //partition 0
+            new KeyValue<>("2", 20L), //partition 2
+            new KeyValue<>("3", 30L), //partition 2
+            new KeyValue<>("4", 40L), //partition 1
+            new KeyValue<>("5", 50L), //partition 0
+            new KeyValue<>("6", 60L), //partition 1
+            new KeyValue<>("7", 70L), //partition 0
+            new KeyValue<>("8", 80L), //partition 0
+            new KeyValue<>("9", 90L)  //partition 2
+        );
+
+        //Partitions pre-computed using the default Murmur2 hash, just to ensure that all 3 partitions will be exercised.
+        final List> table3 = Arrays.asList(
+                new KeyValue<>(10, "waffle")
+        );
+
+        IntegrationTestUtils.produceKeyValuesSynchronously(TABLE_1, table1, PRODUCER_CONFIG_1, MOCK_TIME);
+        IntegrationTestUtils.produceKeyValuesSynchronously(TABLE_2, table2, PRODUCER_CONFIG_2, MOCK_TIME);
+        IntegrationTestUtils.produceKeyValuesSynchronously(TABLE_3, table3, PRODUCER_CONFIG_3, MOCK_TIME);
+
+        CONSUMER_CONFIG.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
+        CONSUMER_CONFIG.put(ConsumerConfig.GROUP_ID_CONFIG, "ktable-ktable-consumer");
+        CONSUMER_CONFIG.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, IntegerDeserializer.class);
+        CONSUMER_CONFIG.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
+    }
+
+    @Before
+    public void before() throws IOException {
+        IntegrationTestUtils.purgeLocalStreamsState(streamsConfig);
+    }
+
+    @After
+    public void after() throws IOException {
+        if (streams != null) {
+            streams.close();
+            streams = null;
+        }
+        if (streamsTwo != null) {
+            streamsTwo.close();
+            streamsTwo = null;
+        }
+        if (streamsThree != null) {
+            streamsThree.close();
+            streamsThree = null;
+        }
+        IntegrationTestUtils.purgeLocalStreamsState(streamsConfig);
+    }
+
+    private enum JoinType {
+        INNER
+    }
+
+    @Test
+    public void shouldInnerJoinMultiPartitionQueryable() throws Exception {
+        final Set> expectedOne = new HashSet<>();
+        expectedOne.add(new KeyValue<>(1, "value1=1.33,value2=10,value3=waffle"));
+
+        verifyKTableKTableJoin(JoinType.INNER, expectedOne, true);
+    }
+
+    private void verifyKTableKTableJoin(final JoinType joinType,
+                                        final Set> expectedResult,
+                                        final boolean verifyQueryableState) throws Exception {
+        final String queryableName = verifyQueryableState ? joinType + "-store1" : null;
+        final String queryableNameTwo = verifyQueryableState ? joinType + "-store2" : null;
+        streamsConfig.put(StreamsConfig.APPLICATION_ID_CONFIG, joinType + queryableName);
+
+        streams = prepareTopology(queryableName, queryableNameTwo);
+        streamsTwo = prepareTopology(queryableName, queryableNameTwo);
+        streamsThree = prepareTopology(queryableName, queryableNameTwo);
+        streams.start();
+        streamsTwo.start();
+        streamsThree.start();
+
+        final Set> result = new HashSet<>(IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
+                CONSUMER_CONFIG,
+                OUTPUT,
+                expectedResult.size()));
+
+        assertEquals(expectedResult, result);
+    }
+
+    private KafkaStreams prepareTopology(final String queryableName, final String queryableNameTwo) {
+        final StreamsBuilder builder = new StreamsBuilder();
+
+        final KTable table1 = builder.table(TABLE_1, Consumed.with(Serdes.Integer(), Serdes.Float()));
+        final KTable table2 = builder.table(TABLE_2, Consumed.with(Serdes.String(), Serdes.Long()));
+        final KTable table3 = builder.table(TABLE_3, Consumed.with(Serdes.Integer(), Serdes.String()));
+
+        final Materialized> materialized;
+        if (queryableName != null) {
+            materialized = Materialized.>as(queryableName)
+                    .withKeySerde(Serdes.Integer())
+                    .withValueSerde(Serdes.String())
+                    .withCachingDisabled();
+        } else {
+            throw new RuntimeException("Current implementation of joinOnForeignKey requires a materialized store");
+        }
+
+        final Materialized> materializedTwo;
+        if (queryableNameTwo != null) {
+            materializedTwo = Materialized.>as(queryableNameTwo)
+                    .withKeySerde(Serdes.Integer())
+                    .withValueSerde(Serdes.String())
+                    .withCachingDisabled();
+        } else {
+            throw new RuntimeException("Current implementation of joinOnForeignKey requires a materialized store");
+        }
+
+        final Function tableOneKeyExtractor = value -> Integer.toString((int) value.floatValue());
+        final Function joinedTableKeyExtractor = value -> {
+            //Hardwired to return the desired foreign key as a test shortcut
+            if (value.contains("value2=10"))
+                return 10;
+            else
+                return 0;
+        };
+
+        final ValueJoiner joiner = (value1, value2) -> "value1=" + value1 + ",value2=" + value2;
+        final ValueJoiner joinerTwo = (value1, value2) -> value1 + ",value3=" + value2;
+
+        table1.join(table2, tableOneKeyExtractor, joiner, materialized)
+              .join(table3, joinedTableKeyExtractor, joinerTwo, materializedTwo)
+            .toStream()
+            .to(OUTPUT, Produced.with(Serdes.Integer(), Serdes.String()));
+
+        return new KafkaStreams(builder.build(streamsConfig), streamsConfig);
+    }
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinIntegrationTest.java
new file mode 100644
index 0000000000000..ae0f3c200fa19
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinIntegrationTest.java
@@ -0,0 +1,699 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.integration;
+
+import kafka.utils.MockTime;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.serialization.FloatSerializer;
+import org.apache.kafka.common.serialization.IntegerDeserializer;
+import org.apache.kafka.common.serialization.IntegerSerializer;
+import org.apache.kafka.common.serialization.LongSerializer;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.Topology;
+import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
+import org.apache.kafka.streams.integration.utils.IntegrationTestUtils;
+import org.apache.kafka.streams.kstream.Consumed;
+import org.apache.kafka.streams.kstream.KTable;
+import org.apache.kafka.streams.kstream.Materialized;
+import org.apache.kafka.streams.kstream.Named;
+import org.apache.kafka.streams.kstream.Produced;
+import org.apache.kafka.streams.kstream.ValueJoiner;
+import org.apache.kafka.streams.state.KeyValueIterator;
+import org.apache.kafka.streams.state.KeyValueStore;
+import org.apache.kafka.streams.state.QueryableStoreTypes;
+import org.apache.kafka.streams.state.ReadOnlyKeyValueStore;
+import org.apache.kafka.test.IntegrationTest;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.function.Function;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertArrayEquals;
+
+
+@Category({IntegrationTest.class})
+public class KTableKTableForeignKeyJoinIntegrationTest {
+    private final static int NUM_BROKERS = 1;
+
+    @ClassRule
+    public final static EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(NUM_BROKERS);
+    private final static MockTime MOCK_TIME = CLUSTER.time;
+    private final static String LEFT_TABLE = "left_table";
+    private final static String RIGHT_TABLE = "right_table";
+    private final static String OUTPUT = "output-topic";
+    private static Properties streamsConfig;
+    private KafkaStreams streams;
+    private KafkaStreams streamsTwo;
+    private KafkaStreams streamsThree;
+    private static final  Properties CONSUMER_CONFIG = new Properties();
+    private static final Properties LEFT_PROD_CONF = new Properties();
+    private static final Properties RIGHT_PROD_CONF = new Properties();
+
+    @BeforeClass
+    public static void beforeTest() {
+        //Use multiple partitions to ensure distribution of keys.
+        LEFT_PROD_CONF.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
+        LEFT_PROD_CONF.put(ProducerConfig.ACKS_CONFIG, "all");
+        LEFT_PROD_CONF.put(ProducerConfig.RETRIES_CONFIG, 0);
+        LEFT_PROD_CONF.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class);
+        LEFT_PROD_CONF.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, FloatSerializer.class);
+
+        RIGHT_PROD_CONF.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
+        RIGHT_PROD_CONF.put(ProducerConfig.ACKS_CONFIG, "all");
+        RIGHT_PROD_CONF.put(ProducerConfig.RETRIES_CONFIG, 0);
+        RIGHT_PROD_CONF.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
+        RIGHT_PROD_CONF.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, LongSerializer.class);
+
+        streamsConfig = new Properties();
+        streamsConfig.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
+        streamsConfig.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+        streamsConfig.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath());
+        streamsConfig.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
+        streamsConfig.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 100);
+        streamsConfig.put(StreamsConfig.TOPOLOGY_OPTIMIZATION, StreamsConfig.OPTIMIZE);
+
+        CONSUMER_CONFIG.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
+        CONSUMER_CONFIG.put(ConsumerConfig.GROUP_ID_CONFIG, "ktable-ktable-consumer");
+        CONSUMER_CONFIG.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, IntegerDeserializer.class);
+        CONSUMER_CONFIG.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
+    }
+
+    @Before
+    public void before() throws IOException, InterruptedException {
+        CLUSTER.deleteTopicsAndWait(LEFT_TABLE);
+        CLUSTER.deleteTopicsAndWait(RIGHT_TABLE);
+        CLUSTER.deleteTopicsAndWait(OUTPUT);
+
+        CLUSTER.createTopic(LEFT_TABLE, 3, 1);
+        CLUSTER.createTopic(RIGHT_TABLE, 3, 1);
+        CLUSTER.createTopic(OUTPUT, 3, 1);
+
+        IntegrationTestUtils.purgeLocalStreamsState(streamsConfig);
+    }
+
+    @After
+    public void after() throws IOException {
+        if (streams != null) {
+            streams.close();
+            streams = null;
+        }
+        if (streamsTwo != null) {
+            streamsTwo.close();
+            streamsTwo = null;
+        }
+        if (streamsThree != null) {
+            streamsThree.close();
+            streamsThree = null;
+        }
+        IntegrationTestUtils.purgeLocalStreamsState(streamsConfig);
+    }
+
+    @Test
+    public void doInnerJoinFromLeftThenDeleteLeftEntity() throws Exception {
+        final List> rightTableEvents = Arrays.asList(new KeyValue<>("1", 10L), new KeyValue<>("2", 20L)); //partition 0
+        IntegrationTestUtils.produceKeyValuesSynchronously(RIGHT_TABLE, rightTableEvents, RIGHT_PROD_CONF, MOCK_TIME);
+
+        final String currentMethodName = new Object() { }
+                .getClass()
+                .getEnclosingMethod()
+                .getName();
+        createAndStartStreamsApplication(currentMethodName, false);
+
+        final List> leftTableEvents = Arrays.asList(new KeyValue<>(1, 1.33f), new KeyValue<>(2, 2.77f));
+        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
+
+        final Set> expected = new HashSet<>();
+        expected.add(new KeyValue<>(1, "value1=1.33,value2=10"));
+        expected.add(new KeyValue<>(2, "value1=2.77,value2=20"));
+
+        final Set> result = new HashSet<>(IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
+                CONSUMER_CONFIG,
+                OUTPUT,
+                expected.size()));
+        assertEquals(expected, result);
+
+        //Now delete one LHS entity such that one delete is propagated down to the output.
+        final Set> expectedDeleted = new HashSet<>();
+        expectedDeleted.add(new KeyValue<>(1, null));
+
+        final List> rightTableDeleteEvents = Arrays.asList(new KeyValue<>(1, null));
+        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, rightTableDeleteEvents, LEFT_PROD_CONF, MOCK_TIME);
+        final Set> resultDeleted = new HashSet<>(IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
+                CONSUMER_CONFIG,
+                OUTPUT,
+                expectedDeleted.size()));
+        assertEquals(expectedDeleted, resultDeleted);
+
+        //Ensure the state stores have the correct values within:
+        final Set> expMatResults = new HashSet<>();
+        expMatResults.add(new KeyValue<>(2, "value1=2.77,value2=20"));
+        validateQueryableStoresContainExpectedKeyValues(expMatResults, currentMethodName);
+    }
+
+    @Test
+    public void doLeftJoinFromLeftThenDeleteLeftEntity() throws Exception {
+        final List> rightTableEvents = Arrays.asList(new KeyValue<>("1", 10L), new KeyValue<>("2", 20L)); //partition 0
+        IntegrationTestUtils.produceKeyValuesSynchronously(RIGHT_TABLE, rightTableEvents, RIGHT_PROD_CONF, MOCK_TIME);
+
+        final String currentMethodName = new Object() { }
+                .getClass()
+                .getEnclosingMethod()
+                .getName();
+        createAndStartStreamsApplication(currentMethodName, true);
+
+        final List> leftTableEvents = Arrays.asList(new KeyValue<>(1, 1.33f), new KeyValue<>(2, 2.77f));
+        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
+
+        final Set> expected = new HashSet<>();
+        expected.add(new KeyValue<>(1, "value1=1.33,value2=10"));
+        expected.add(new KeyValue<>(2, "value1=2.77,value2=20"));
+
+        final Set> result = new HashSet<>(IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
+                CONSUMER_CONFIG,
+                OUTPUT,
+                expected.size()));
+        assertEquals(expected, result);
+
+        //Now delete one LHS entity such that one delete is propagated down to the output.
+        final Set> expectedDeleted = new HashSet<>();
+        expectedDeleted.add(new KeyValue<>(1, null));
+
+        final List> rightTableDeleteEvents = Arrays.asList(new KeyValue<>(1, null));
+        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, rightTableDeleteEvents, LEFT_PROD_CONF, MOCK_TIME);
+        final Set> resultDeleted = new HashSet<>(IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
+                CONSUMER_CONFIG,
+                OUTPUT,
+                expectedDeleted.size()));
+        assertEquals(expectedDeleted, resultDeleted);
+
+        //Ensure the state stores have the correct values within:
+        final Set> expMatResults = new HashSet<>();
+        expMatResults.add(new KeyValue<>(2, "value1=2.77,value2=20"));
+        validateQueryableStoresContainExpectedKeyValues(expMatResults, currentMethodName);
+    }
+
+    @Test
+    public void doInnerJoinFromRightThenDeleteRightEntity() throws Exception {
+        final List> leftTableEvents = Arrays.asList(
+                new KeyValue<>(1, 1.33f),
+                new KeyValue<>(2, 1.77f),
+                new KeyValue<>(3, 3.77f));
+        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
+        final String currentMethodName = new Object() { }
+                .getClass()
+                .getEnclosingMethod()
+                .getName();
+        createAndStartStreamsApplication(currentMethodName, false);
+
+        final List> rightTableEvents = Arrays.asList(
+                new KeyValue<>("1", 10L),  //partition 0
+                new KeyValue<>("2", 20L),  //partition 2
+                new KeyValue<>("3", 30L)); //partition 2
+        IntegrationTestUtils.produceKeyValuesSynchronously(RIGHT_TABLE, rightTableEvents, RIGHT_PROD_CONF, MOCK_TIME);
+
+        //Ensure that the joined values exist in the output
+        final Set> expected = new HashSet<>();
+        expected.add(new KeyValue<>(1, "value1=1.33,value2=10"));   //Will be deleted.
+        expected.add(new KeyValue<>(2, "value1=1.77,value2=10"));   //Will be deleted.
+        expected.add(new KeyValue<>(3, "value1=3.77,value2=30"));   //Will not be deleted.
+
+        final Set> result = new HashSet<>(IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
+                CONSUMER_CONFIG,
+                OUTPUT,
+                expected.size()));
+        assertEquals(expected, result);
+
+        //Now delete the RHS entity such that all matching keys have deletes propagated.
+        final Set> expectedDeleted = new HashSet<>();
+        expectedDeleted.add(new KeyValue<>(1, null));
+        expectedDeleted.add(new KeyValue<>(2, null));
+
+        final List> rightTableDeleteEvents = Arrays.asList(new KeyValue<>("1", null));
+        IntegrationTestUtils.produceKeyValuesSynchronously(RIGHT_TABLE, rightTableDeleteEvents, RIGHT_PROD_CONF, MOCK_TIME);
+        final Set> resultDeleted = new HashSet<>(IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
+                CONSUMER_CONFIG,
+                OUTPUT,
+                expectedDeleted.size()));
+        assertEquals(expectedDeleted, resultDeleted);
+
+        //Ensure the state stores have the correct values within:
+        final Set> expMatResults = new HashSet<>();
+        expMatResults.add(new KeyValue<>(3, "value1=3.77,value2=30"));
+        validateQueryableStoresContainExpectedKeyValues(expMatResults, currentMethodName);
+    }
+
+    @Test
+    public void doLeftJoinFromRightThenDeleteRightEntity() throws Exception {
+        final List> leftTableEvents = Arrays.asList(
+                new KeyValue<>(1, 1.33f),
+                new KeyValue<>(2, 1.77f),
+                new KeyValue<>(3, 3.77f));
+        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
+        final String currentMethodName = new Object() { }
+                .getClass()
+                .getEnclosingMethod()
+                .getName();
+        createAndStartStreamsApplication(currentMethodName, true);
+
+        final List> rightTableEvents = Arrays.asList(
+                new KeyValue<>("1", 10L),  //partition 0
+                new KeyValue<>("2", 20L),  //partition 2
+                new KeyValue<>("3", 30L)); //partition 2
+        IntegrationTestUtils.produceKeyValuesSynchronously(RIGHT_TABLE, rightTableEvents, RIGHT_PROD_CONF, MOCK_TIME);
+
+        //Ensure that the joined values exist in the output
+        final Set> expected = new HashSet<>();
+        expected.add(new KeyValue<>(1, "value1=1.33,value2=10"));   //Will be deleted.
+        expected.add(new KeyValue<>(2, "value1=1.77,value2=10"));   //Will be deleted.
+        expected.add(new KeyValue<>(3, "value1=3.77,value2=30"));   //Will not be deleted.
+        //final HashSet> expected = new HashSet<>(buildExpectedResults(leftTableEvents, rightTableEvents, false));
+
+        final Set> result = new HashSet<>(IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
+                CONSUMER_CONFIG,
+                OUTPUT,
+                expected.size()));
+        assertEquals(expected, result);
+
+        //Now delete the RHS entity such that all matching keys have deletes propagated.
+        //This will exercise the joiner with the RHS value == null.
+        final Set> expectedDeleted = new HashSet<>();
+        expectedDeleted.add(new KeyValue<>(1, "value1=1.33,value2=null"));
+        expectedDeleted.add(new KeyValue<>(2, "value1=1.77,value2=null"));
+
+        final List> rightTableDeleteEvents = Arrays.asList(new KeyValue<>("1", null));
+        IntegrationTestUtils.produceKeyValuesSynchronously(RIGHT_TABLE, rightTableDeleteEvents, RIGHT_PROD_CONF, MOCK_TIME);
+        final Set> resultDeleted = new HashSet<>(IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
+                CONSUMER_CONFIG,
+                OUTPUT,
+                expectedDeleted.size()));
+        assertEquals(expectedDeleted, resultDeleted);
+
+        //Ensure the state stores have the correct values within:
+        final Set> expMatResults = new HashSet<>();
+        expMatResults.add(new KeyValue<>(1, "value1=1.33,value2=null"));
+        expMatResults.add(new KeyValue<>(2, "value1=1.77,value2=null"));
+        expMatResults.add(new KeyValue<>(3, "value1=3.77,value2=30"));
+        validateQueryableStoresContainExpectedKeyValues(expMatResults, currentMethodName);
+    }
+
+    @Test
+    public void doInnerJoinProduceNullsWhenValueHasNonMatchingForeignKey() throws Exception {
+        //There is no matching extracted foreign-key of 8 anywhere. Should not produce any output for INNER JOIN, only
+        //because the state is transitioning from oldValue=null -> newValue=8.33.
+        List> leftTableEvents = Arrays.asList(new KeyValue<>(1, 8.33f));
+        final List> rightTableEvents = Arrays.asList(new KeyValue<>("1", 10L)); //partition 0
+        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
+        IntegrationTestUtils.produceKeyValuesSynchronously(RIGHT_TABLE, rightTableEvents, RIGHT_PROD_CONF, MOCK_TIME);
+
+        final String currentMethodName = new Object() { }
+                .getClass()
+                .getEnclosingMethod()
+                .getName();
+        createAndStartStreamsApplication(currentMethodName, false);
+
+        //There is also no matching extracted foreign-key for 18 anywhere. This WILL produce a null output for INNER JOIN,
+        //since we cannot remember (maintain state) that the FK=8 also produced a null result.
+        leftTableEvents = Arrays.asList(new KeyValue<>(1, 18.00f));
+        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
+
+        final List> expected = new LinkedList<>();
+        expected.add(new KeyValue<>(1, null));
+
+        final List> result = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
+                CONSUMER_CONFIG,
+                OUTPUT,
+                expected.size());
+        assertEquals(result, expected);
+
+        //Another change to FK that has no match on the RHS will result in another null
+        leftTableEvents = Arrays.asList(new KeyValue<>(1, 100.00f));
+        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
+        //Consume the next event - note that we are using the same consumerGroupId, so this will consume a new event.
+        final List> result2 = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
+                CONSUMER_CONFIG,
+                OUTPUT,
+                expected.size());
+        assertEquals(result2, expected);
+
+        //Now set the LHS event FK to match the rightTableEvents key-value.
+        leftTableEvents = Arrays.asList(new KeyValue<>(1, 1.11f));
+
+        final List> expected3 = new LinkedList<>();
+        expected3.add(new KeyValue<>(1, "value1=1.11,value2=10"));
+
+        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
+        final List> result3 = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
+                CONSUMER_CONFIG,
+                OUTPUT,
+                expected3.size());
+        assertEquals(result3, expected3);
+
+        //Ensure the state stores have the correct values within:
+        final Set> expMatResults = new HashSet<>();
+        expMatResults.add(new KeyValue<>(1, "value1=1.11,value2=10"));
+        validateQueryableStoresContainExpectedKeyValues(expMatResults, currentMethodName);
+    }
+
+    @Test
+    public void doLeftJoinProduceJoinedResultsWhenValueHasNonMatchingForeignKey() throws Exception {
+        //There is no matching extracted foreign-key of 8 anywhere.
+        //However, it will still run the join function since this is LEFT join.
+        List> leftTableEvents = Arrays.asList(new KeyValue<>(1, 8.33f));
+        final List> rightTableEvents = Arrays.asList(new KeyValue<>("1", 10L)); //partition 0
+        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
+        IntegrationTestUtils.produceKeyValuesSynchronously(RIGHT_TABLE, rightTableEvents, RIGHT_PROD_CONF, MOCK_TIME);
+
+        final String currentMethodName = new Object() { }
+                .getClass()
+                .getEnclosingMethod()
+                .getName();
+        createAndStartStreamsApplication(currentMethodName, true);
+
+        final List> expected = new LinkedList<>();
+        expected.add(new KeyValue<>(1, "value1=8.33,value2=null"));
+        final List> result = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
+                CONSUMER_CONFIG,
+                OUTPUT,
+                expected.size());
+        assertEquals(expected, result);
+
+        //There is also no matching extracted foreign-key for 18 anywhere.
+        //However, it will still run the join function since this if LEFT join.
+        leftTableEvents = Arrays.asList(new KeyValue<>(1, 18.0f));
+        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
+
+        final List> expected2 = new LinkedList<>();
+        expected2.add(new KeyValue<>(1, "value1=18.0,value2=null"));
+        final List> result2 = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
+                CONSUMER_CONFIG,
+                OUTPUT,
+                expected2.size());
+        assertEquals(expected2, result2);
+
+
+        leftTableEvents = Arrays.asList(new KeyValue<>(1, 1.11f));
+        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
+
+        final List> expected3 = new LinkedList<>();
+        expected3.add(new KeyValue<>(1, "value1=1.11,value2=10"));
+        final List> result3 = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
+                CONSUMER_CONFIG,
+                OUTPUT,
+                expected3.size());
+        assertEquals(expected3, result3);
+
+        //Ensure the state stores have the correct values within:
+        final Set> expMatResults = new HashSet<>();
+        expMatResults.add(new KeyValue<>(1, "value1=1.11,value2=10"));
+        validateQueryableStoresContainExpectedKeyValues(expMatResults, currentMethodName);
+    }
+
+    @Test
+    public void doInnerJoinFilterOutRapidlyChangingForeignKeyValues() throws Exception {
+        final List> leftTableEvents = Arrays.asList(
+                new KeyValue<>(1, 1.33f),
+                new KeyValue<>(2, 2.22f),
+                new KeyValue<>(3, -1.22f), //Won't be joined in
+                new KeyValue<>(4, -2.22f), //Won't be joined in
+                new KeyValue<>(5, 2.22f)
+        );
+
+        //Partitions pre-computed using the default Murmur2 hash, just to ensure that all 3 partitions will be exercised.
+        final List> rightTableEvents = Arrays.asList(
+                new KeyValue<>("0", 0L),  //partition 2
+                new KeyValue<>("1", 10L), //partition 0
+                new KeyValue<>("2", 20L), //partition 2
+                new KeyValue<>("3", 30L), //partition 2
+                new KeyValue<>("4", 40L), //partition 1
+                new KeyValue<>("5", 50L), //partition 0
+                new KeyValue<>("6", 60L), //partition 1
+                new KeyValue<>("7", 70L), //partition 0
+                new KeyValue<>("8", 80L), //partition 0
+                new KeyValue<>("9", 90L)  //partition 2
+        );
+
+        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
+        IntegrationTestUtils.produceKeyValuesSynchronously(RIGHT_TABLE, rightTableEvents, RIGHT_PROD_CONF, MOCK_TIME);
+
+        final Set> expected = new HashSet<>();
+        expected.add(new KeyValue<>(1, "value1=1.33,value2=10"));
+        expected.add(new KeyValue<>(2, "value1=2.22,value2=20"));
+        expected.add(new KeyValue<>(5, "value1=2.22,value2=20"));
+
+        final String currentMethodName = new Object() { }
+                .getClass()
+                .getEnclosingMethod()
+                .getName();
+        createAndStartStreamsApplication(currentMethodName, false);
+
+        final Set> result = new HashSet<>(IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
+                CONSUMER_CONFIG,
+                OUTPUT,
+                expected.size()));
+
+        assertEquals(result, expected);
+
+        //Rapidly change the foreign key, to validate that the hashing prevents incorrect results from being output,
+        //and that eventually the correct value is output.
+        final List> table1ForeignKeyChange = Arrays.asList(
+                new KeyValue<>(3, 2.22f), //Partition 2
+                new KeyValue<>(3, 3.33f), //Partition 2
+                new KeyValue<>(3, 4.44f), //Partition 1
+                new KeyValue<>(3, 5.55f), //Partition 0
+                new KeyValue<>(3, 9.99f), //Partition 2
+                new KeyValue<>(3, 8.88f), //Partition 0
+                new KeyValue<>(3, 0.23f), //Partition 2
+                new KeyValue<>(3, 7.77f), //Partition 0
+                new KeyValue<>(3, 6.66f), //Partition 1
+                new KeyValue<>(3, 1.11f)  //Partition 0 - This will be the final result.
+        );
+
+        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, table1ForeignKeyChange, LEFT_PROD_CONF, MOCK_TIME);
+        final List> resultTwo = IntegrationTestUtils.readKeyValues(OUTPUT, CONSUMER_CONFIG, 15 * 1000L, Integer.MAX_VALUE);
+
+        final List> expectedTwo = new LinkedList<>();
+        expectedTwo.add(new KeyValue<>(3, "value1=1.11,value2=10"));
+        assertArrayEquals(resultTwo.toArray(), expectedTwo.toArray());
+
+        //Ensure the state stores have the correct values within:
+        final Set> expMatResults = new HashSet<>();
+        expMatResults.addAll(expected);
+        expMatResults.addAll(expectedTwo);
+        validateQueryableStoresContainExpectedKeyValues(expMatResults, currentMethodName);
+    }
+
+    @Test
+    public void doLeftJoinFilterOutRapidlyChangingForeignKeyValues() throws Exception {
+        final List> leftTableEvents = Arrays.asList(
+                new KeyValue<>(1, 1.33f),
+                new KeyValue<>(2, 2.22f)
+        );
+
+        //Partitions pre-computed using the default Murmur2 hash, just to ensure that all 3 partitions will be exercised.
+        final List> rightTableEvents = Arrays.asList(
+                new KeyValue<>("0", 0L),  //partition 2
+                new KeyValue<>("1", 10L), //partition 0
+                new KeyValue<>("2", 20L), //partition 2
+                new KeyValue<>("3", 30L), //partition 2
+                new KeyValue<>("4", 40L), //partition 1
+                new KeyValue<>("5", 50L), //partition 0
+                new KeyValue<>("6", 60L), //partition 1
+                new KeyValue<>("7", 70L), //partition 0
+                new KeyValue<>("8", 80L), //partition 0
+                new KeyValue<>("9", 90L)  //partition 2
+        );
+
+        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
+        IntegrationTestUtils.produceKeyValuesSynchronously(RIGHT_TABLE, rightTableEvents, RIGHT_PROD_CONF, MOCK_TIME);
+
+        final Set> expected = new HashSet<>();
+        expected.add(new KeyValue<>(1, "value1=1.33,value2=10"));
+        expected.add(new KeyValue<>(2, "value1=2.22,value2=20"));
+
+        final String currentMethodName = new Object() { }
+                .getClass()
+                .getEnclosingMethod()
+                .getName();
+        createAndStartStreamsApplication(currentMethodName, false);
+
+        final Set> result = new HashSet<>(IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
+                CONSUMER_CONFIG,
+                OUTPUT,
+                expected.size()));
+
+        assertEquals(result, expected);
+
+        //Rapidly change the foreign key, to validate that the hashing prevents incorrect results from being output,
+        //and that eventually the correct value is output.
+        final List> table1ForeignKeyChange = Arrays.asList(
+                new KeyValue<>(3, 2.22f), //Partition 2
+                new KeyValue<>(3, 3.33f), //Partition 2
+                new KeyValue<>(3, 4.44f), //Partition 1
+                new KeyValue<>(3, 5.55f), //Partition 0
+                new KeyValue<>(3, 9.99f), //Partition 2
+                new KeyValue<>(3, 8.88f), //Partition 0
+                new KeyValue<>(3, 0.23f), //Partition 2
+                new KeyValue<>(3, 7.77f), //Partition 0
+                new KeyValue<>(3, 6.66f), //Partition 1
+                new KeyValue<>(3, 1.11f)  //Partition 0 - This will be the final result.
+        );
+
+        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, table1ForeignKeyChange, LEFT_PROD_CONF, MOCK_TIME);
+        final List> resultTwo = IntegrationTestUtils.readKeyValues(OUTPUT, CONSUMER_CONFIG, 15 * 1000L, Integer.MAX_VALUE);
+
+        final List> expectedTwo = new LinkedList<>();
+        expectedTwo.add(new KeyValue<>(3, "value1=1.11,value2=10"));
+
+        assertArrayEquals(resultTwo.toArray(), expectedTwo.toArray());
+
+        //Ensure the state stores have the correct values within:
+        final Set> expMatResults = new HashSet<>();
+        expMatResults.addAll(expected);
+        expMatResults.addAll(expectedTwo);
+        validateQueryableStoresContainExpectedKeyValues(expMatResults, currentMethodName);
+    }
+
+    private void createAndStartStreamsApplication(final String queryableStoreName, final boolean leftJoin) {
+        streamsConfig.put(StreamsConfig.APPLICATION_ID_CONFIG, "ktable-ktable-joinOnForeignKey-" + queryableStoreName);
+        streams = prepareTopology(queryableStoreName, leftJoin);
+        streamsTwo = prepareTopology(queryableStoreName, leftJoin);
+        streamsThree = prepareTopology(queryableStoreName, leftJoin);
+        streams.start();
+        streamsTwo.start();
+        streamsThree.start();
+    }
+
+    // These are hardwired into the test logic for readability sake.
+    // Do not change unless you want to change all the test results as well.
+    private ValueJoiner joiner = (value1, value2) -> "value1=" + value1 + ",value2=" + value2;
+    //Do not change. See above comment.
+    private Function tableOneKeyExtractor = value -> Integer.toString((int) value.floatValue());
+
+    private void validateQueryableStoresContainExpectedKeyValues(final Set> expectedResult,
+                                                                 final String queryableStoreName) {
+        final ReadOnlyKeyValueStore myJoinStoreOne = streams.store(queryableStoreName,
+                QueryableStoreTypes.keyValueStore());
+
+        final ReadOnlyKeyValueStore myJoinStoreTwo = streamsTwo.store(queryableStoreName,
+                QueryableStoreTypes.keyValueStore());
+
+        final ReadOnlyKeyValueStore myJoinStoreThree = streamsThree.store(queryableStoreName,
+                QueryableStoreTypes.keyValueStore());
+
+        // store only keeps last set of values, not entire stream of value changes
+        final Map expectedInStore = new HashMap<>();
+        for (final KeyValue expected : expectedResult) {
+            expectedInStore.put(expected.key, expected.value);
+        }
+
+        // depending on partition assignment, the values will be in one of the three stream clients.
+        for (final Map.Entry expected : expectedInStore.entrySet()) {
+            final String one = myJoinStoreOne.get(expected.getKey());
+            final String two = myJoinStoreTwo.get(expected.getKey());
+            final String three = myJoinStoreThree.get(expected.getKey());
+
+            String result;
+            if (one != null)
+                result = one;
+            else if (two != null)
+                result = two;
+            else if (three != null)
+                result = three;
+            else
+                throw new RuntimeException("Cannot find key " + expected.getKey() + " in any of the state stores");
+            assertEquals(expected.getValue(), result);
+        }
+
+        //Merge all the iterators together to ensure that their sum equals the total set of expected elements.
+        final KeyValueIterator allOne = myJoinStoreOne.all();
+        final KeyValueIterator allTwo = myJoinStoreTwo.all();
+        final KeyValueIterator allThree = myJoinStoreThree.all();
+
+        final List> all = new LinkedList<>();
+
+        while (allOne.hasNext()) {
+            all.add(allOne.next());
+        }
+        while (allTwo.hasNext()) {
+            all.add(allTwo.next());
+        }
+        while (allThree.hasNext()) {
+            all.add(allThree.next());
+        }
+        allOne.close();
+        allTwo.close();
+        allThree.close();
+
+        for (final KeyValue elem : all) {
+            assertTrue(expectedResult.contains(elem));
+        }
+    }
+
+    private KafkaStreams prepareTopology(final String queryableStoreName, final boolean leftJoin) {
+        final StreamsBuilder builder = new StreamsBuilder();
+
+        final KTable left = builder.table(LEFT_TABLE, Consumed.with(Serdes.Integer(), Serdes.Float()));
+        final KTable right = builder.table(RIGHT_TABLE, Consumed.with(Serdes.String(), Serdes.Long()));
+
+        final Materialized> materialized;
+        if (queryableStoreName != null) {
+            materialized = Materialized.>as(queryableStoreName)
+                    .withKeySerde(Serdes.Integer())
+                    .withValueSerde(Serdes.String())
+                    .withCachingDisabled();
+        } else {
+            throw new RuntimeException("Current implementation of join on foreign key requires a materialized store");
+        }
+
+        if (leftJoin)
+            left.leftJoin(right, tableOneKeyExtractor, joiner, Named.as("customName"), materialized)
+                .toStream()
+                .to(OUTPUT, Produced.with(Serdes.Integer(), Serdes.String()));
+        else
+            left.join(right, tableOneKeyExtractor, joiner, materialized)
+                .toStream()
+                .to(OUTPUT, Produced.with(Serdes.Integer(), Serdes.String()));
+
+        final Topology topology = builder.build(streamsConfig);
+
+        return new KafkaStreams(topology, streamsConfig);
+    }
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchemaTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchemaTest.java
new file mode 100644
index 0000000000000..47348038521ca
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchemaTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.kstream.internals.foreignkeyjoin;
+
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Bytes;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertEquals;
+
+public class CombinedKeySchemaTest {
+
+    @Test
+    public void nonNullPrimaryKeySerdeTest() {
+        final CombinedKeySchema cks = new CombinedKeySchema<>("someTopic", Serdes.String(), Serdes.Integer());
+        final Integer primary = -999;
+        final Bytes result = cks.toBytes("foreignKey", primary);
+
+        final CombinedKey deserializedKey = cks.fromBytes(result);
+        assertEquals("foreignKey", deserializedKey.getForeignKey());
+        assertEquals(primary, deserializedKey.getPrimaryKey());
+    }
+
+    @Test(expected = NullPointerException.class)
+    public void nullPrimaryKeySerdeTest() {
+        final CombinedKeySchema cks = new CombinedKeySchema<>("someTopic", Serdes.String(), Serdes.Integer());
+        cks.toBytes("foreignKey", null);
+    }
+
+    @Test(expected = NullPointerException.class)
+    public void nullForeignKeySerdeTest() {
+        final CombinedKeySchema cks = new CombinedKeySchema<>("someTopic", Serdes.String(), Serdes.Integer());
+        cks.toBytes(null, 10);
+    }
+
+    @Test
+    public void prefixKeySerdeTest() {
+        final CombinedKeySchema cks = new CombinedKeySchema<>("someTopic", Serdes.String(), Serdes.Integer());
+        final String foreignKey = "someForeignKey";
+        final byte[] foreignKeySerializedData = Serdes.String().serializer().serialize("someTopic", foreignKey);
+        final Bytes prefix = cks.prefixBytes(foreignKey);
+
+        final ByteBuffer buf = ByteBuffer.allocate(Integer.BYTES + foreignKeySerializedData.length);
+        buf.putInt(foreignKeySerializedData.length);
+        buf.put(foreignKeySerializedData);
+        final Bytes expectedPrefixBytes = Bytes.wrap(buf.array());
+
+        assertEquals(expectedPrefixBytes, prefix);
+    }
+
+    @Test(expected = NullPointerException.class)
+    public void nullPrefixKeySerdeTest() {
+        final CombinedKeySchema cks = new CombinedKeySchema<>("someTopic", Serdes.String(), Serdes.Integer());
+        final String foreignKey = null;
+        cks.prefixBytes(foreignKey);
+    }
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapperSerdeTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapperSerdeTest.java
new file mode 100644
index 0000000000000..fde9bddfac37d
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapperSerdeTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.kstream.internals.foreignkeyjoin;
+
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.streams.state.internals.Murmur3;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertNull;
+
+public class SubscriptionResponseWrapperSerdeTest {
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void ShouldSerdeWithNonNullsTest() {
+        final long[] hashedValue = Murmur3.hash128(new byte[] {(byte) 0x01, (byte) 0x9A, (byte) 0xFF, (byte) 0x00});
+        final String foreignValue = "foreignValue";
+        final SubscriptionResponseWrapper srw = new SubscriptionResponseWrapper<>(hashedValue, foreignValue);
+        final SubscriptionResponseWrapperSerde srwSerde = new SubscriptionResponseWrapperSerde(Serdes.String());
+        final byte[] serResponse = srwSerde.serializer().serialize(null, srw);
+        final SubscriptionResponseWrapper result = (SubscriptionResponseWrapper) srwSerde.deserializer().deserialize(null, serResponse);
+
+        assertArrayEquals(hashedValue, result.getOriginalValueHash());
+        assertEquals(foreignValue, result.getForeignValue());
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void shouldSerdeWithNullForeignValueTest() {
+        final long[] hashedValue = Murmur3.hash128(new byte[] {(byte) 0x01, (byte) 0x9A, (byte) 0xFF, (byte) 0x00});
+        final SubscriptionResponseWrapper srw = new SubscriptionResponseWrapper<>(hashedValue, null);
+        final SubscriptionResponseWrapperSerde srwSerde = new SubscriptionResponseWrapperSerde(Serdes.String());
+        final byte[] serResponse = srwSerde.serializer().serialize(null, srw);
+        final SubscriptionResponseWrapper result = (SubscriptionResponseWrapper) srwSerde.deserializer().deserialize(null, serResponse);
+
+        assertArrayEquals(hashedValue, result.getOriginalValueHash());
+        assertNull(result.getForeignValue());
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void shouldSerdeWithNullHashTest() {
+        final long[] hashedValue = null;
+        final String foreignValue = "foreignValue";
+        final SubscriptionResponseWrapper srw = new SubscriptionResponseWrapper<>(hashedValue, foreignValue);
+        final SubscriptionResponseWrapperSerde srwSerde = new SubscriptionResponseWrapperSerde(Serdes.String());
+        final byte[] serResponse = srwSerde.serializer().serialize(null, srw);
+        final SubscriptionResponseWrapper result = (SubscriptionResponseWrapper) srwSerde.deserializer().deserialize(null, serResponse);
+
+        assertArrayEquals(hashedValue, result.getOriginalValueHash());
+        assertEquals(foreignValue, result.getForeignValue());
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void shouldSerdeWithNullsTest() {
+        final long[] hashedValue = null;
+        final String foreignValue = null;
+        final SubscriptionResponseWrapper srw = new SubscriptionResponseWrapper<>(hashedValue, foreignValue);
+        final SubscriptionResponseWrapperSerde srwSerde = new SubscriptionResponseWrapperSerde(Serdes.String());
+        final byte[] serResponse = srwSerde.serializer().serialize(null, srw);
+        final SubscriptionResponseWrapper result = (SubscriptionResponseWrapper) srwSerde.deserializer().deserialize(null, serResponse);
+
+        assertArrayEquals(hashedValue, result.getOriginalValueHash());
+        assertEquals(foreignValue, result.getForeignValue());
+    }
+
+    @Test (expected = UnsupportedVersionException.class)
+    @SuppressWarnings("unchecked")
+    public void shouldThrowExceptionWithBadVersionTest() {
+        final long[] hashedValue = null;
+        new SubscriptionResponseWrapper<>(hashedValue, "foreignValue", (byte) 0xFF);
+    }
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerdeTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerdeTest.java
new file mode 100644
index 0000000000000..d948d1f2a35fb
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerdeTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.kstream.internals.foreignkeyjoin;
+
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.streams.state.internals.Murmur3;
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+@SuppressWarnings({"unchecked", "rawtypes"})
+public class SubscriptionWrapperSerdeTest {
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void shouldSerdeTest() {
+        final String originalKey = "originalKey";
+        final SubscriptionWrapperSerde swSerde = new SubscriptionWrapperSerde<>(Serdes.String());
+        final long[] hashedValue = Murmur3.hash128(new byte[] {(byte) 0xFF, (byte) 0xAA, (byte) 0x00, (byte) 0x19});
+        final SubscriptionWrapper wrapper = new SubscriptionWrapper<>(hashedValue, SubscriptionWrapper.Instruction.DELETE_KEY_AND_PROPAGATE, originalKey);
+        final byte[] serialized = swSerde.serializer().serialize(null, wrapper);
+        final SubscriptionWrapper deserialized = (SubscriptionWrapper) swSerde.deserializer().deserialize(null, serialized);
+
+        assertEquals(SubscriptionWrapper.Instruction.DELETE_KEY_AND_PROPAGATE, deserialized.getInstruction());
+        assertArrayEquals(hashedValue, deserialized.getHash());
+        assertEquals(originalKey, deserialized.getPrimaryKey());
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void shouldSerdeNullHashTest() {
+        final String originalKey = "originalKey";
+        final SubscriptionWrapperSerde swSerde = new SubscriptionWrapperSerde<>(Serdes.String());
+        final long[] hashedValue = null;
+        final SubscriptionWrapper wrapper = new SubscriptionWrapper<>(hashedValue, SubscriptionWrapper.Instruction.PROPAGATE_ONLY_IF_FK_VAL_AVAILABLE, originalKey);
+        final byte[] serialized = swSerde.serializer().serialize(null, wrapper);
+        final SubscriptionWrapper deserialized = (SubscriptionWrapper) swSerde.deserializer().deserialize(null, serialized);
+
+        assertEquals(SubscriptionWrapper.Instruction.PROPAGATE_ONLY_IF_FK_VAL_AVAILABLE, deserialized.getInstruction());
+        assertArrayEquals(hashedValue, deserialized.getHash());
+        assertEquals(originalKey, deserialized.getPrimaryKey());
+    }
+
+    @Test (expected = NullPointerException.class)
+    @SuppressWarnings("unchecked")
+    public void shouldThrowExceptionOnNullKeyTest() {
+        final String originalKey = null;
+        final SubscriptionWrapperSerde swSerde = new SubscriptionWrapperSerde<>(Serdes.String());
+        final long[] hashedValue = Murmur3.hash128(new byte[] {(byte) 0xFF, (byte) 0xAA, (byte) 0x00, (byte) 0x19});
+        final SubscriptionWrapper wrapper = new SubscriptionWrapper<>(hashedValue, SubscriptionWrapper.Instruction.PROPAGATE_ONLY_IF_FK_VAL_AVAILABLE, originalKey);
+        swSerde.serializer().serialize(null, wrapper);
+    }
+
+    @Test (expected = NullPointerException.class)
+    @SuppressWarnings("unchecked")
+    public void shouldThrowExceptionOnNullInstructionTest() {
+        final String originalKey = "originalKey";
+        final SubscriptionWrapperSerde swSerde = new SubscriptionWrapperSerde<>(Serdes.String());
+        final long[] hashedValue = Murmur3.hash128(new byte[] {(byte) 0xFF, (byte) 0xAA, (byte) 0x00, (byte) 0x19});
+        final SubscriptionWrapper wrapper = new SubscriptionWrapper<>(hashedValue, null, originalKey);
+        swSerde.serializer().serialize(null, wrapper);
+    }
+
+    @Test (expected = UnsupportedVersionException.class)
+    public void shouldThrowExceptionOnUnsupportedVersionTest() {
+        final String originalKey = "originalKey";
+        final long[] hashedValue = null;
+        new SubscriptionWrapper<>(hashedValue, SubscriptionWrapper.Instruction.PROPAGATE_ONLY_IF_FK_VAL_AVAILABLE, originalKey, (byte) 0x80);
+    }
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBKeyValueStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBKeyValueStoreTest.java
index bdf0379ddb3fa..8db040ea4b76d 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBKeyValueStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBKeyValueStoreTest.java
@@ -87,7 +87,7 @@ public void shouldPerformAllQueriesWithCachingDisabled() {
     }
 
     @Test
-    public void shouldCloseOpenIteratorsWhenStoreClosedAndThrowInvalidStateStoreOnHasNextAndNext() {
+    public void shouldCloseOpenRangeIteratorsWhenStoreClosedAndThrowInvalidStateStoreOnHasNextAndNext() {
         context.setTime(1L);
         store.put(1, "hi");
         store.put(2, "goodbye");
@@ -127,5 +127,4 @@ public void shouldCloseOpenIteratorsWhenStoreClosedAndThrowInvalidStateStoreOnHa
             // ok
         }
     }
-
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java
index 4c1932e2853f5..0a28a773e9a90 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java
@@ -557,7 +557,7 @@ public void shouldThrowNullPointerExceptionOnRange() {
         rocksDBStore.init(context, rocksDBStore);
         try {
             rocksDBStore.range(null, new Bytes(stringSerializer.serialize(null, "2")));
-            fail("Should have thrown NullPointerException on deleting null key");
+            fail("Should have thrown NullPointerException on null range key");
         } catch (final NullPointerException e) {
             // this is good
         }
diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/FunctionsCompatConversions.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/FunctionsCompatConversions.scala
index 754f1cc1aad5d..7cd3ac832832d 100644
--- a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/FunctionsCompatConversions.scala
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/FunctionsCompatConversions.scala
@@ -61,6 +61,12 @@ private[scala] object FunctionsCompatConversions {
     }
   }
 
+  implicit class FunctionFromFunction[V, VR](val f: V => VR) extends AnyVal {
+    def asJavaFunction: java.util.function.Function[V, VR] = new java.util.function.Function[V, VR] {
+      override def apply(value: V): VR = f(value)
+    }
+  }
+
   implicit class ValueMapperFromFunction[V, VR](val f: V => VR) extends AnyVal {
     def asValueMapper: ValueMapper[V, VR] = new ValueMapper[V, VR] {
       override def apply(value: V): VR = f(value)
diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KTable.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KTable.scala
index 56549ad3875b0..288550bc47db2 100644
--- a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KTable.scala
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KTable.scala
@@ -21,7 +21,7 @@ package org.apache.kafka.streams.scala
 package kstream
 
 import org.apache.kafka.common.utils.Bytes
-import org.apache.kafka.streams.kstream.{Suppressed, ValueTransformerWithKeySupplier, KTable => KTableJ}
+import org.apache.kafka.streams.kstream.{Suppressed, ValueJoiner, ValueTransformerWithKeySupplier, KTable => KTableJ}
 import org.apache.kafka.streams.scala.FunctionsCompatConversions._
 import org.apache.kafka.streams.scala.ImplicitConversions._
 import org.apache.kafka.streams.state.KeyValueStore
@@ -318,6 +318,42 @@ class KTable[K, V](val inner: KTableJ[K, V]) {
   ): KTable[K, VR] =
     inner.outerJoin[VO, VR](other.inner, joiner.asValueJoiner, materialized)
 
+  /**
+   * Join records of this [[KTable]] with another [[KTable]]'s records using non-windowed inner join. Records from this
+   * table are joined according to the result of keyExtractor on the other KTable.
+   *
+   * @param other the other [[KTable]] to be joined with this [[KTable]], keyed on the value obtained from keyExtractor
+   * @param keyExtractor a function that extracts the foreign key from this table's value
+   * @param joiner       a function that computes the join result for a pair of matching records
+   * @param materialized a `Materialized` that describes how the `StateStore` for the resulting [[KTable]]
+   * should be materialized.
+   * @return a [[KTable]] that contains join-records for each key and values computed by the given joiner,
+   * one for each matched record-pair with the same key
+   */
+  def join[VR, KO, VO](other: KTable[KO, VO],
+                       keyExtractor: Function[V, KO],
+                       joiner: ValueJoiner[V, VO, VR],
+                       materialized: Materialized[K, VR, KeyValueStore[Bytes, Array[Byte]]]): KTable[K, VR] =
+    inner.join(other.inner, keyExtractor.asJavaFunction, joiner, materialized)
+
+  /**
+   * Join records of this [[KTable]] with another [[KTable]]'s records using non-windowed left join. Records from this
+   * table are joined according to the result of keyExtractor on the other KTable.
+   *
+   * @param other the other [[KTable]] to be joined with this [[KTable]], keyed on the value obtained from keyExtractor
+   * @param keyExtractor a function that extracts the foreign key from this table's value
+   * @param joiner       a function that computes the join result for a pair of matching records
+   * @param materialized a `Materialized` that describes how the `StateStore` for the resulting [[KTable]]
+   * should be materialized.
+   * @return a [[KTable]] that contains join-records for each key and values computed by the given joiner,
+   * one for each matched record-pair with the same key
+   */
+  def leftJoin[VR, KO, VO](other: KTable[KO, VO],
+                           keyExtractor: Function[V, KO],
+                           joiner: ValueJoiner[V, VO, VR],
+                           materialized: Materialized[K, VR, KeyValueStore[Bytes, Array[Byte]]]): KTable[K, VR] =
+    inner.leftJoin(other.inner, keyExtractor.asJavaFunction, joiner, materialized)
+
   /**
    * Get the name of the local state store used that can be used to query this [[KTable]].
    *

From f98013cc501ab30c43ca707a1fbf46c4e9dc1215 Mon Sep 17 00:00:00 2001
From: David Jacot 
Date: Fri, 4 Oct 2019 18:19:18 +0200
Subject: [PATCH 0686/1071] Part 1 of KIP-511: Collect and Expose Client's Name
 and Version in the Brokers #7381

Reviewers: Stanislav Kozlovski , David Arthur , Colin P. McCabe 
---
 .../org/apache/kafka/clients/ApiVersion.java  |  56 +++++
 .../apache/kafka/clients/NetworkClient.java   |  21 +-
 .../apache/kafka/clients/NodeApiVersions.java |  35 ++-
 .../clients/consumer/internals/Fetcher.java   |   4 +-
 .../apache/kafka/common/protocol/ApiKeys.java |   6 +-
 .../common/requests/AbstractResponse.java     |   2 +-
 .../common/requests/ApiVersionsRequest.java   |  82 ++++---
 .../common/requests/ApiVersionsResponse.java  | 203 +++++----------
 .../kafka/common/requests/RequestContext.java |   3 +-
 .../SaslClientAuthenticator.java              |  10 +-
 .../SaslServerAuthenticator.java              |   4 +-
 .../common/message/ApiVersionsRequest.json    |  10 +-
 .../common/message/ApiVersionsResponse.json   |  14 +-
 .../apache/kafka/clients/ApiVersionsTest.java |   6 +-
 .../kafka/clients/NetworkClientTest.java      | 232 +++++++++++++++++-
 .../kafka/clients/NodeApiVersionsTest.java    |  52 ++--
 .../internals/AbstractCoordinatorTest.java    |   4 +-
 .../consumer/internals/FetcherTest.java       |   7 +-
 .../internals/RecordAccumulatorTest.java      |   7 +-
 .../producer/internals/SenderTest.java        |   6 +-
 .../requests/ApiVersionsResponseTest.java     |  31 +--
 .../common/requests/RequestContextTest.java   |  13 +-
 .../common/requests/RequestResponseTest.java  | 112 ++++++++-
 .../authenticator/SaslAuthenticatorTest.java  | 108 ++++++--
 .../org/apache/kafka/test/MockSelector.java   |   7 +
 .../admin/BrokerApiVersionsCommand.scala      |  10 +-
 .../main/scala/kafka/server/KafkaApis.scala   |   2 +
 .../kafka/server/ApiVersionsRequestTest.scala |  31 ++-
 .../unit/kafka/server/RequestQuotaTest.scala  |   2 +-
 .../server/SaslApiVersionsRequestTest.scala   |   6 +-
 30 files changed, 782 insertions(+), 304 deletions(-)
 create mode 100644 clients/src/main/java/org/apache/kafka/clients/ApiVersion.java

diff --git a/clients/src/main/java/org/apache/kafka/clients/ApiVersion.java b/clients/src/main/java/org/apache/kafka/clients/ApiVersion.java
new file mode 100644
index 0000000000000..9d606bbfa7afc
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/ApiVersion.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients;
+
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKey;
+import org.apache.kafka.common.protocol.ApiKeys;
+
+/**
+ * Represents the min version and max version of an api key.
+ *
+ * NOTE: This class is intended for INTERNAL usage only within Kafka.
+ */
+public class ApiVersion {
+    public final short apiKey;
+    public final short minVersion;
+    public final short maxVersion;
+
+    public ApiVersion(ApiKeys apiKey) {
+        this(apiKey.id, apiKey.oldestVersion(), apiKey.latestVersion());
+    }
+
+    public ApiVersion(short apiKey, short minVersion, short maxVersion) {
+        this.apiKey = apiKey;
+        this.minVersion = minVersion;
+        this.maxVersion = maxVersion;
+    }
+
+    public ApiVersion(ApiVersionsResponseKey apiVersionsResponseKey) {
+        this.apiKey = apiVersionsResponseKey.apiKey();
+        this.minVersion = apiVersionsResponseKey.minVersion();
+        this.maxVersion = apiVersionsResponseKey.maxVersion();
+    }
+
+    @Override
+    public String toString() {
+        return "ApiVersion(" +
+            "apiKey=" + apiKey +
+            ", minVersion=" + minVersion +
+            ", maxVersion= " + maxVersion +
+            ")";
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
index 44c9fd3c07247..34e3a2f07b960 100644
--- a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
@@ -23,6 +23,7 @@
 import org.apache.kafka.common.errors.AuthenticationException;
 import org.apache.kafka.common.errors.DisconnectException;
 import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKey;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.network.ChannelState;
 import org.apache.kafka.common.network.NetworkReceive;
@@ -853,18 +854,28 @@ else if (req.isInternalRequest && body instanceof ApiVersionsResponse)
     private void handleApiVersionsResponse(List responses,
                                            InFlightRequest req, long now, ApiVersionsResponse apiVersionsResponse) {
         final String node = req.destination;
-        if (apiVersionsResponse.error() != Errors.NONE) {
-            if (req.request.version() == 0 || apiVersionsResponse.error() != Errors.UNSUPPORTED_VERSION) {
+        if (apiVersionsResponse.data.errorCode() != Errors.NONE.code()) {
+            if (req.request.version() == 0 || apiVersionsResponse.data.errorCode() != Errors.UNSUPPORTED_VERSION.code()) {
                 log.warn("Received error {} from node {} when making an ApiVersionsRequest with correlation id {}. Disconnecting.",
-                        apiVersionsResponse.error(), node, req.header.correlationId());
+                        Errors.forCode(apiVersionsResponse.data.errorCode()), node, req.header.correlationId());
                 this.selector.close(node);
                 processDisconnection(responses, node, now, ChannelState.LOCAL_CLOSE);
             } else {
-                nodesNeedingApiVersionsFetch.put(node, new ApiVersionsRequest.Builder((short) 0));
+                // Starting from Apache Kafka 2.4, ApiKeys field is populated with the supported versions of
+                // the ApiVersionsRequest when an UNSUPPORTED_VERSION error is returned.
+                // If not provided, the client falls back to version 0.
+                short maxApiVersion = 0;
+                if (apiVersionsResponse.data.apiKeys().size() > 0) {
+                    ApiVersionsResponseKey apiVersion = apiVersionsResponse.data.apiKeys().find(ApiKeys.API_VERSIONS.id);
+                    if (apiVersion != null) {
+                        maxApiVersion = apiVersion.maxVersion();
+                    }
+                }
+                nodesNeedingApiVersionsFetch.put(node, new ApiVersionsRequest.Builder(maxApiVersion));
             }
             return;
         }
-        NodeApiVersions nodeVersionInfo = new NodeApiVersions(apiVersionsResponse.apiVersions());
+        NodeApiVersions nodeVersionInfo = new NodeApiVersions(apiVersionsResponse.data.apiKeys());
         apiVersions.update(node, nodeVersionInfo);
         this.connectionStates.ready(node);
         log.debug("Recorded API versions for node {}: {}", node, nodeVersionInfo);
diff --git a/clients/src/main/java/org/apache/kafka/clients/NodeApiVersions.java b/clients/src/main/java/org/apache/kafka/clients/NodeApiVersions.java
index c8b3f44b77997..dcc5e5ffa2b2c 100644
--- a/clients/src/main/java/org/apache/kafka/clients/NodeApiVersions.java
+++ b/clients/src/main/java/org/apache/kafka/clients/NodeApiVersions.java
@@ -16,16 +16,17 @@
  */
 package org.apache.kafka.clients;
 
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
 import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKey;
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKeyCollection;
 import org.apache.kafka.common.protocol.ApiKeys;
-import org.apache.kafka.common.requests.ApiVersionsResponse.ApiVersion;
 import org.apache.kafka.common.utils.Utils;
 
 import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
 import java.util.EnumMap;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
@@ -34,6 +35,7 @@
  * An internal class which represents the API versions supported by a particular node.
  */
 public class NodeApiVersions {
+
     // A map of the usable versions of each API, keyed by the ApiKeys instance
     private final Map supportedVersions = new EnumMap<>(ApiKeys.class);
 
@@ -73,6 +75,31 @@ public static NodeApiVersions create(Collection overrides) {
         return new NodeApiVersions(apiVersions);
     }
 
+
+    /**
+     * Create a NodeApiVersions object with a single ApiKey. It is mainly used in tests.
+     *
+     * @param apiKey ApiKey's id.
+     * @param minVersion ApiKey's minimum version.
+     * @param maxVersion ApiKey's maximum version.
+     * @return A new NodeApiVersions object.
+     */
+    public static NodeApiVersions create(short apiKey, short minVersion, short maxVersion) {
+        return create(Collections.singleton(new ApiVersion(apiKey, minVersion, maxVersion)));
+    }
+
+    public NodeApiVersions(ApiVersionsResponseKeyCollection nodeApiVersions) {
+        for (ApiVersionsResponseKey nodeApiVersion : nodeApiVersions) {
+            if (ApiKeys.hasId(nodeApiVersion.apiKey())) {
+                ApiKeys nodeApiKey = ApiKeys.forId(nodeApiVersion.apiKey());
+                supportedVersions.put(nodeApiKey, new ApiVersion(nodeApiVersion));
+            } else {
+                // Newer brokers may support ApiKeys we don't know about
+                unknownApis.add(new ApiVersion(nodeApiVersion));
+            }
+        }
+    }
+
     public NodeApiVersions(Collection nodeApiVersions) {
         for (ApiVersion nodeApiVersion : nodeApiVersions) {
             if (ApiKeys.hasId(nodeApiVersion.apiKey)) {
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
index 6b4ae916e4bc0..9fd21cf121e07 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
@@ -21,6 +21,7 @@
 import org.apache.kafka.clients.FetchSessionHandler;
 import org.apache.kafka.clients.MetadataCache;
 import org.apache.kafka.clients.NodeApiVersions;
+import org.apache.kafka.clients.ApiVersion;
 import org.apache.kafka.clients.StaleMetadataException;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
@@ -61,7 +62,6 @@
 import org.apache.kafka.common.record.RecordBatch;
 import org.apache.kafka.common.record.Records;
 import org.apache.kafka.common.record.TimestampType;
-import org.apache.kafka.common.requests.ApiVersionsResponse;
 import org.apache.kafka.common.requests.FetchRequest;
 import org.apache.kafka.common.requests.FetchResponse;
 import org.apache.kafka.common.requests.IsolationLevel;
@@ -753,7 +753,7 @@ public void onFailure(RuntimeException e) {
     }
 
     private boolean hasUsableOffsetForLeaderEpochVersion(NodeApiVersions nodeApiVersions) {
-        ApiVersionsResponse.ApiVersion apiVersion = nodeApiVersions.apiVersion(ApiKeys.OFFSET_FOR_LEADER_EPOCH);
+        ApiVersion apiVersion = nodeApiVersions.apiVersion(ApiKeys.OFFSET_FOR_LEADER_EPOCH);
         if (apiVersion == null)
             return false;
 
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
index ba59940175ea8..31a98f5144158 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
@@ -16,6 +16,8 @@
  */
 package org.apache.kafka.common.protocol;
 
+import org.apache.kafka.common.message.ApiVersionsRequestData;
+import org.apache.kafka.common.message.ApiVersionsResponseData;
 import org.apache.kafka.common.message.ControlledShutdownRequestData;
 import org.apache.kafka.common.message.ControlledShutdownResponseData;
 import org.apache.kafka.common.message.CreateDelegationTokenRequestData;
@@ -89,8 +91,6 @@
 import org.apache.kafka.common.requests.AlterConfigsResponse;
 import org.apache.kafka.common.requests.AlterReplicaLogDirsRequest;
 import org.apache.kafka.common.requests.AlterReplicaLogDirsResponse;
-import org.apache.kafka.common.requests.ApiVersionsRequest;
-import org.apache.kafka.common.requests.ApiVersionsResponse;
 import org.apache.kafka.common.requests.CreateAclsRequest;
 import org.apache.kafka.common.requests.CreateAclsResponse;
 import org.apache.kafka.common.requests.CreatePartitionsRequest;
@@ -150,7 +150,7 @@ public enum ApiKeys {
             DescribeGroupsResponseData.SCHEMAS),
     LIST_GROUPS(16, "ListGroups", ListGroupsRequestData.SCHEMAS, ListGroupsResponseData.SCHEMAS),
     SASL_HANDSHAKE(17, "SaslHandshake", SaslHandshakeRequestData.SCHEMAS, SaslHandshakeResponseData.SCHEMAS),
-    API_VERSIONS(18, "ApiVersions", ApiVersionsRequest.schemaVersions(), ApiVersionsResponse.schemaVersions()) {
+    API_VERSIONS(18, "ApiVersions", ApiVersionsRequestData.SCHEMAS, ApiVersionsResponseData.SCHEMAS) {
         @Override
         public Struct parseResponse(short version, ByteBuffer buffer) {
             // Fallback to version 0 for ApiVersions response. If a client sends an ApiVersionsRequest
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
index eb11a4244215f..835541e3575c8 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
@@ -116,7 +116,7 @@ public static AbstractResponse parseResponse(ApiKeys apiKey, Struct struct, shor
             case SASL_HANDSHAKE:
                 return new SaslHandshakeResponse(struct, version);
             case API_VERSIONS:
-                return new ApiVersionsResponse(struct);
+                return ApiVersionsResponse.apiVersionsResponse(struct, version);
             case CREATE_TOPICS:
                 return new CreateTopicsResponse(struct, version);
             case DELETE_TOPICS:
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java
index 03f385b9ba586..87ffceaa26e1c 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java
@@ -16,30 +16,26 @@
  */
 package org.apache.kafka.common.requests;
 
+import java.util.regex.Pattern;
+import org.apache.kafka.common.message.ApiVersionsRequestData;
+import org.apache.kafka.common.message.ApiVersionsResponseData;
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKey;
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKeyCollection;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
+import org.apache.kafka.common.utils.AppInfoParser;
 
 import java.nio.ByteBuffer;
-import java.util.Collections;
 
 public class ApiVersionsRequest extends AbstractRequest {
-    private static final Schema API_VERSIONS_REQUEST_V0 = new Schema();
-
-    /* v1 request is the same as v0. Throttle time has been added to response */
-    private static final Schema API_VERSIONS_REQUEST_V1 = API_VERSIONS_REQUEST_V0;
-
-    /**
-     * The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
-     */
-    private static final Schema API_VERSIONS_REQUEST_V2 = API_VERSIONS_REQUEST_V1;
-
-    public static Schema[] schemaVersions() {
-        return new Schema[]{API_VERSIONS_REQUEST_V0, API_VERSIONS_REQUEST_V1, API_VERSIONS_REQUEST_V2};
-    }
 
     public static class Builder extends AbstractRequest.Builder {
+        private static final String DEFAULT_CLIENT_SOFTWARE_NAME = "apache-kafka-java";
+
+        private static final ApiVersionsRequestData DATA = new ApiVersionsRequestData()
+            .setClientSoftwareName(DEFAULT_CLIENT_SOFTWARE_NAME)
+            .setClientSoftwareVersion(AppInfoParser.getVersion());
 
         public Builder() {
             super(ApiKeys.API_VERSIONS);
@@ -51,23 +47,28 @@ public Builder(short version) {
 
         @Override
         public ApiVersionsRequest build(short version) {
-            return new ApiVersionsRequest(version);
+            return new ApiVersionsRequest(DATA, version);
         }
 
         @Override
         public String toString() {
-            return "(type=ApiVersionsRequest)";
+            return DATA.toString();
         }
     }
 
+    private static final Pattern SOFTWARE_NAME_VERSION_PATTERN = Pattern.compile("[a-zA-Z0-9](?:[a-zA-Z0-9\\-.]*[a-zA-Z0-9])?");
+
     private final Short unsupportedRequestVersion;
 
-    public ApiVersionsRequest(short version) {
-        this(version, null);
+    public final ApiVersionsRequestData data;
+
+    public ApiVersionsRequest(ApiVersionsRequestData data, short version) {
+        this(data, version, null);
     }
 
-    public ApiVersionsRequest(short version, Short unsupportedRequestVersion) {
+    public ApiVersionsRequest(ApiVersionsRequestData data, short version, Short unsupportedRequestVersion) {
         super(ApiKeys.API_VERSIONS, version);
+        this.data = data;
 
         // Unlike other request types, the broker handles ApiVersion requests with higher versions than
         // supported. It does so by treating the request as if it were v0 and returns a response using
@@ -78,31 +79,48 @@ public ApiVersionsRequest(short version, Short unsupportedRequestVersion) {
     }
 
     public ApiVersionsRequest(Struct struct, short version) {
-        this(version, null);
+        this(new ApiVersionsRequestData(struct, version), version);
     }
 
     public boolean hasUnsupportedRequestVersion() {
         return unsupportedRequestVersion != null;
     }
 
+    public boolean isValid() {
+        if (version() >= 3) {
+            return SOFTWARE_NAME_VERSION_PATTERN.matcher(data.clientSoftwareName()).matches() &&
+                SOFTWARE_NAME_VERSION_PATTERN.matcher(data.clientSoftwareVersion()).matches();
+        } else {
+            return true;
+        }
+    }
+
     @Override
     protected Struct toStruct() {
-        return new Struct(ApiKeys.API_VERSIONS.requestSchema(version()));
+        return data.toStruct(version());
     }
 
     @Override
     public ApiVersionsResponse getErrorResponse(int throttleTimeMs, Throwable e) {
-        short version = version();
-        switch (version) {
-            case 0:
-                return new ApiVersionsResponse(Errors.forException(e), Collections.emptyList());
-            case 1:
-            case 2:
-                return new ApiVersionsResponse(throttleTimeMs, Errors.forException(e), Collections.emptyList());
-            default:
-                throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
-                        version, this.getClass().getSimpleName(), ApiKeys.API_VERSIONS.latestVersion()));
+        ApiVersionsResponseData data = new ApiVersionsResponseData()
+            .setErrorCode(Errors.forException(e).code());
+
+        if (version() >= 1) {
+            data.setThrottleTimeMs(throttleTimeMs);
         }
+
+        // Starting from Apache Kafka 2.4 (KIP-511), ApiKeys field is populated with the supported
+        // versions of the ApiVersionsRequest when an UNSUPPORTED_VERSION error is returned.
+        if (Errors.forException(e) == Errors.UNSUPPORTED_VERSION) {
+            ApiVersionsResponseKeyCollection apiKeys = new ApiVersionsResponseKeyCollection();
+            apiKeys.add(new ApiVersionsResponseKey()
+                .setApiKey(ApiKeys.API_VERSIONS.id)
+                .setMinVersion(ApiKeys.API_VERSIONS.oldestVersion())
+                .setMaxVersion(ApiKeys.API_VERSIONS.latestVersion()));
+            data.setApiKeys(apiKeys);
+        }
+
+        return new ApiVersionsResponse(data);
     }
 
     public static ApiVersionsRequest parse(ByteBuffer buffer, short version) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java
index b3846d1bb3aa0..fd214043ebabc 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java
@@ -16,194 +16,109 @@
  */
 package org.apache.kafka.common.requests;
 
+import org.apache.kafka.common.message.ApiVersionsResponseData;
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKey;
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKeyCollection;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.ArrayOf;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
+import org.apache.kafka.common.protocol.types.SchemaException;
 import org.apache.kafka.common.protocol.types.Struct;
 import org.apache.kafka.common.record.RecordBatch;
 
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
-import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
-import static org.apache.kafka.common.protocol.CommonFields.THROTTLE_TIME_MS;
-import static org.apache.kafka.common.protocol.types.Type.INT16;
-
+/**
+ * Possible error codes:
+ * - {@link Errors#UNSUPPORTED_VERSION}
+ * - {@link Errors#INVALID_REQUEST}
+ */
 public class ApiVersionsResponse extends AbstractResponse {
-    private static final String API_VERSIONS_KEY_NAME = "api_versions";
-    private static final String API_KEY_NAME = "api_key";
-    private static final String MIN_VERSION_KEY_NAME = "min_version";
-    private static final String MAX_VERSION_KEY_NAME = "max_version";
-
-    private static final Schema API_VERSIONS_V0 = new Schema(
-            new Field(API_KEY_NAME, INT16, "API key."),
-            new Field(MIN_VERSION_KEY_NAME, INT16, "Minimum supported version."),
-            new Field(MAX_VERSION_KEY_NAME, INT16, "Maximum supported version."));
-
-    private static final Schema API_VERSIONS_RESPONSE_V0 = new Schema(
-            ERROR_CODE,
-            new Field(API_VERSIONS_KEY_NAME, new ArrayOf(API_VERSIONS_V0), "API versions supported by the broker."));
-    private static final Schema API_VERSIONS_RESPONSE_V1 = new Schema(
-            ERROR_CODE,
-            new Field(API_VERSIONS_KEY_NAME, new ArrayOf(API_VERSIONS_V0), "API versions supported by the broker."),
-            THROTTLE_TIME_MS);
-
-    /**
-     * The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
-     */
-    private static final Schema API_VERSIONS_RESPONSE_V2 = API_VERSIONS_RESPONSE_V1;
-
-    // initialized lazily to avoid circular initialization dependence with ApiKeys
-    private static volatile ApiVersionsResponse defaultApiVersionsResponse;
-
-    public static Schema[] schemaVersions() {
-        return new Schema[]{API_VERSIONS_RESPONSE_V0, API_VERSIONS_RESPONSE_V1, API_VERSIONS_RESPONSE_V2};
-    }
-
-    /**
-     * Possible error codes:
-     *
-     * UNSUPPORTED_VERSION (33)
-     */
-    private final Errors error;
-    private final int throttleTimeMs;
-    private final Map apiKeyToApiVersion;
-
-    public static final class ApiVersion {
-        public final short apiKey;
-        public final short minVersion;
-        public final short maxVersion;
-
-        public ApiVersion(ApiKeys apiKey) {
-            this(apiKey.id, apiKey.oldestVersion(), apiKey.latestVersion());
-        }
 
-        public ApiVersion(ApiKeys apiKey, short minVersion, short maxVersion) {
-            this(apiKey.id, minVersion, maxVersion);
-        }
-
-        public ApiVersion(short apiKey, short minVersion, short maxVersion) {
-            this.apiKey = apiKey;
-            this.minVersion = minVersion;
-            this.maxVersion = maxVersion;
-        }
+    public static final ApiVersionsResponse DEFAULT_API_VERSIONS_RESPONSE =
+        createApiVersionsResponse(DEFAULT_THROTTLE_TIME, RecordBatch.CURRENT_MAGIC_VALUE);
 
-        @Override
-        public String toString() {
-            return "ApiVersion(" +
-                    "apiKey=" + apiKey +
-                    ", minVersion=" + minVersion +
-                    ", maxVersion= " + maxVersion +
-                    ")";
-        }
-    }
+    public final ApiVersionsResponseData data;
 
-    public ApiVersionsResponse(Errors error, List apiVersions) {
-        this(DEFAULT_THROTTLE_TIME, error, apiVersions);
+    public ApiVersionsResponse(ApiVersionsResponseData data) {
+        this.data = data;
     }
 
-    public ApiVersionsResponse(int throttleTimeMs, Errors error, List apiVersions) {
-        this.throttleTimeMs = throttleTimeMs;
-        this.error = error;
-        this.apiKeyToApiVersion = buildApiKeyToApiVersion(apiVersions);
+    public ApiVersionsResponse(Struct struct) {
+        this(new ApiVersionsResponseData(struct, (short) (ApiVersionsResponseData.SCHEMAS.length - 1)));
     }
 
-    public ApiVersionsResponse(Struct struct) {
-        this.throttleTimeMs = struct.getOrElse(THROTTLE_TIME_MS, DEFAULT_THROTTLE_TIME);
-        this.error = Errors.forCode(struct.get(ERROR_CODE));
-        List tempApiVersions = new ArrayList<>();
-        for (Object apiVersionsObj : struct.getArray(API_VERSIONS_KEY_NAME)) {
-            Struct apiVersionStruct = (Struct) apiVersionsObj;
-            short apiKey = apiVersionStruct.getShort(API_KEY_NAME);
-            short minVersion = apiVersionStruct.getShort(MIN_VERSION_KEY_NAME);
-            short maxVersion = apiVersionStruct.getShort(MAX_VERSION_KEY_NAME);
-            tempApiVersions.add(new ApiVersion(apiKey, minVersion, maxVersion));
-        }
-        this.apiKeyToApiVersion = buildApiKeyToApiVersion(tempApiVersions);
+    public ApiVersionsResponse(Struct struct, short version) {
+        this(new ApiVersionsResponseData(struct, version));
     }
 
     @Override
     protected Struct toStruct(short version) {
-        Struct struct = new Struct(ApiKeys.API_VERSIONS.responseSchema(version));
-        struct.setIfExists(THROTTLE_TIME_MS, throttleTimeMs);
-        struct.set(ERROR_CODE, error.code());
-        List apiVersionList = new ArrayList<>();
-        for (ApiVersion apiVersion : apiKeyToApiVersion.values()) {
-            Struct apiVersionStruct = struct.instance(API_VERSIONS_KEY_NAME);
-            apiVersionStruct.set(API_KEY_NAME, apiVersion.apiKey);
-            apiVersionStruct.set(MIN_VERSION_KEY_NAME, apiVersion.minVersion);
-            apiVersionStruct.set(MAX_VERSION_KEY_NAME, apiVersion.maxVersion);
-            apiVersionList.add(apiVersionStruct);
-        }
-        struct.set(API_VERSIONS_KEY_NAME, apiVersionList.toArray());
-        return struct;
+        return this.data.toStruct(version);
     }
 
-    public static ApiVersionsResponse apiVersionsResponse(int throttleTimeMs, byte maxMagic) {
-        if (maxMagic == RecordBatch.CURRENT_MAGIC_VALUE && throttleTimeMs == DEFAULT_THROTTLE_TIME) {
-            return defaultApiVersionsResponse();
-        }
-        return createApiVersionsResponse(throttleTimeMs, maxMagic);
+    public ApiVersionsResponseKey apiVersion(short apiKey) {
+        return data.apiKeys().find(apiKey);
     }
 
     @Override
-    public int throttleTimeMs() {
-        return throttleTimeMs;
-    }
-
-    public Collection apiVersions() {
-        return apiKeyToApiVersion.values();
-    }
-
-    public ApiVersion apiVersion(short apiKey) {
-        return apiKeyToApiVersion.get(apiKey);
+    public Map errorCounts() {
+        return errorCounts(Errors.forCode(this.data.errorCode()));
     }
 
-    public Errors error() {
-        return error;
+    @Override
+    public int throttleTimeMs() {
+        return this.data.throttleTimeMs();
     }
 
     @Override
-    public Map errorCounts() {
-        return errorCounts(error);
+    public boolean shouldClientThrottle(short version) {
+        return version >= 2;
     }
 
     public static ApiVersionsResponse parse(ByteBuffer buffer, short version) {
-        return new ApiVersionsResponse(ApiKeys.API_VERSIONS.parseResponse(version, buffer));
+        return new ApiVersionsResponse(ApiKeys.API_VERSIONS.parseResponse(version, buffer), version);
+    }
+
+    public static ApiVersionsResponse apiVersionsResponse(Struct struct, short version) {
+        // Fallback to version 0 for ApiVersions response. If a client sends an ApiVersionsRequest
+        // using a version higher than that supported by the broker, a version 0 response is sent
+        // to the client indicating UNSUPPORTED_VERSION. When the client receives the response, it
+        // falls back while parsing it into a Struct which means that the version received by this
+        // method is not necessary the real one. It may be version 0 as well.
+        try {
+            return new ApiVersionsResponse(struct, version);
+        } catch (SchemaException e) {
+            if (version != 0)
+                return new ApiVersionsResponse(struct, (short) 0);
+            else
+                throw e;
+        }
     }
 
-    private Map buildApiKeyToApiVersion(List apiVersions) {
-        Map tempApiIdToApiVersion = new HashMap<>();
-        for (ApiVersion apiVersion : apiVersions) {
-            tempApiIdToApiVersion.put(apiVersion.apiKey, apiVersion);
+    public static ApiVersionsResponse apiVersionsResponse(int throttleTimeMs, byte maxMagic) {
+        if (maxMagic == RecordBatch.CURRENT_MAGIC_VALUE && throttleTimeMs == DEFAULT_THROTTLE_TIME) {
+            return DEFAULT_API_VERSIONS_RESPONSE;
         }
-        return tempApiIdToApiVersion;
+        return createApiVersionsResponse(throttleTimeMs, maxMagic);
     }
 
     public static ApiVersionsResponse createApiVersionsResponse(int throttleTimeMs, final byte minMagic) {
-        List versionList = new ArrayList<>();
+        ApiVersionsResponseKeyCollection apiKeys = new ApiVersionsResponseKeyCollection();
         for (ApiKeys apiKey : ApiKeys.values()) {
             if (apiKey.minRequiredInterBrokerMagic <= minMagic) {
-                versionList.add(new ApiVersionsResponse.ApiVersion(apiKey));
+                apiKeys.add(new ApiVersionsResponseKey()
+                    .setApiKey(apiKey.id)
+                    .setMinVersion(apiKey.oldestVersion())
+                    .setMaxVersion(apiKey.latestVersion()));
             }
         }
-        return new ApiVersionsResponse(throttleTimeMs, Errors.NONE, versionList);
-    }
 
-    public static ApiVersionsResponse defaultApiVersionsResponse() {
-        if (defaultApiVersionsResponse == null)
-            defaultApiVersionsResponse = createApiVersionsResponse(DEFAULT_THROTTLE_TIME, RecordBatch.CURRENT_MAGIC_VALUE);
-        return defaultApiVersionsResponse;
-    }
+        ApiVersionsResponseData data = new ApiVersionsResponseData();
+        data.setThrottleTimeMs(throttleTimeMs);
+        data.setErrorCode(Errors.NONE.code());
+        data.setApiKeys(apiKeys);
 
-    @Override
-    public boolean shouldClientThrottle(short version) {
-        return version >= 2;
+        return new ApiVersionsResponse(data);
     }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java
index c067444478b03..8e21f8e40be5e 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java
@@ -17,6 +17,7 @@
 package org.apache.kafka.common.requests;
 
 import org.apache.kafka.common.errors.InvalidRequestException;
+import org.apache.kafka.common.message.ApiVersionsRequestData;
 import org.apache.kafka.common.network.ListenerName;
 import org.apache.kafka.common.network.Send;
 import org.apache.kafka.common.protocol.ApiKeys;
@@ -55,7 +56,7 @@ public RequestContext(RequestHeader header,
     public RequestAndSize parseRequest(ByteBuffer buffer) {
         if (isUnsupportedApiVersionsRequest()) {
             // Unsupported ApiVersion requests are treated as v0 requests and are not parsed
-            ApiVersionsRequest apiVersionsRequest = new ApiVersionsRequest((short) 0, header.apiVersion());
+            ApiVersionsRequest apiVersionsRequest = new ApiVersionsRequest(new ApiVersionsRequestData(), (short) 0, header.apiVersion());
             return new RequestAndSize(apiVersionsRequest, 0);
         } else {
             ApiKeys apiKey = header.apiKey();
diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
index 266759000b12a..84f481f80e434 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
@@ -23,6 +23,7 @@
 import org.apache.kafka.common.errors.IllegalSaslStateException;
 import org.apache.kafka.common.errors.SaslAuthenticationException;
 import org.apache.kafka.common.errors.UnsupportedSaslMechanismException;
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKey;
 import org.apache.kafka.common.message.RequestHeaderData;
 import org.apache.kafka.common.message.SaslAuthenticateRequestData;
 import org.apache.kafka.common.message.SaslHandshakeRequestData;
@@ -38,7 +39,6 @@
 import org.apache.kafka.common.requests.AbstractResponse;
 import org.apache.kafka.common.requests.ApiVersionsRequest;
 import org.apache.kafka.common.requests.ApiVersionsResponse;
-import org.apache.kafka.common.requests.ApiVersionsResponse.ApiVersion;
 import org.apache.kafka.common.requests.RequestHeader;
 import org.apache.kafka.common.requests.SaslAuthenticateRequest;
 import org.apache.kafka.common.requests.SaslAuthenticateResponse;
@@ -202,7 +202,7 @@ public void authenticate() throws IOException {
         switch (saslState) {
             case SEND_APIVERSIONS_REQUEST:
                 // Always use version 0 request since brokers treat requests with schema exceptions as GSSAPI tokens
-                ApiVersionsRequest apiVersionsRequest = new ApiVersionsRequest((short) 0);
+                ApiVersionsRequest apiVersionsRequest = new ApiVersionsRequest.Builder().build((short) 0);
                 send(apiVersionsRequest.toSend(node, nextRequestHeader(ApiKeys.API_VERSIONS, apiVersionsRequest.version())));
                 setSaslState(SaslState.RECEIVE_APIVERSIONS_RESPONSE);
                 break;
@@ -285,7 +285,7 @@ public void authenticate() throws IOException {
 
     private void sendHandshakeRequest(ApiVersionsResponse apiVersionsResponse) throws IOException {
         SaslHandshakeRequest handshakeRequest = createSaslHandshakeRequest(
-                apiVersionsResponse.apiVersion(ApiKeys.SASL_HANDSHAKE.id).maxVersion);
+                apiVersionsResponse.apiVersion(ApiKeys.SASL_HANDSHAKE.id).maxVersion());
         send(handshakeRequest.toSend(node, nextRequestHeader(ApiKeys.SASL_HANDSHAKE, handshakeRequest.version())));
     }
 
@@ -344,9 +344,9 @@ protected SaslHandshakeRequest createSaslHandshakeRequest(short version) {
 
     // Visible to override for testing
     protected void saslAuthenticateVersion(ApiVersionsResponse apiVersionsResponse) {
-        ApiVersion authenticateVersion = apiVersionsResponse.apiVersion(ApiKeys.SASL_AUTHENTICATE.id);
+        ApiVersionsResponseKey authenticateVersion = apiVersionsResponse.apiVersion(ApiKeys.SASL_AUTHENTICATE.id);
         if (authenticateVersion != null)
-            this.saslAuthenticateVersion = (short) Math.min(authenticateVersion.maxVersion,
+            this.saslAuthenticateVersion = (short) Math.min(authenticateVersion.maxVersion(),
                     ApiKeys.SASL_AUTHENTICATE.latestVersion());
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java
index 7aca17716bb41..3953a10cde131 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java
@@ -568,7 +568,7 @@ private String handleHandshakeRequest(RequestContext context, SaslHandshakeReque
 
     // Visible to override for testing
     protected ApiVersionsResponse apiVersionsResponse() {
-        return ApiVersionsResponse.defaultApiVersionsResponse();
+        return ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE;
     }
 
     // Visible to override for testing
@@ -582,6 +582,8 @@ private void handleApiVersionsRequest(RequestContext context, ApiVersionsRequest
 
         if (apiVersionsRequest.hasUnsupportedRequestVersion())
             sendKafkaResponse(context, apiVersionsRequest.getErrorResponse(0, Errors.UNSUPPORTED_VERSION.exception()));
+        else if (!apiVersionsRequest.isValid())
+            sendKafkaResponse(context, apiVersionsRequest.getErrorResponse(0, Errors.INVALID_REQUEST.exception()));
         else {
             sendKafkaResponse(context, apiVersionsResponse());
             setSaslState(SaslState.HANDSHAKE_REQUEST);
diff --git a/clients/src/main/resources/common/message/ApiVersionsRequest.json b/clients/src/main/resources/common/message/ApiVersionsRequest.json
index d6ddb5475c7b8..59c3388703d07 100644
--- a/clients/src/main/resources/common/message/ApiVersionsRequest.json
+++ b/clients/src/main/resources/common/message/ApiVersionsRequest.json
@@ -18,7 +18,15 @@
   "type": "request",
   "name": "ApiVersionsRequest",
   // Versions 0 through 2 of ApiVersionsRequest are the same.
-  "validVersions": "0-2",
+  //
+  // Version 3 is the first flexible version and adds ClientSoftwareName and ClientSoftwareVersion.
+  "validVersions": "0-3",
+  // TODO(dajac) Enable this once KIP-482 is committed.
+  // "flexibleVersions": "3+",
   "fields": [
+    { "name": "ClientSoftwareName", "type": "string", "versions": "3+",
+      "about": "The name of the client." },
+    { "name": "ClientSoftwareVersion", "type": "string", "versions": "3+",
+      "about": "The version of the client." }
   ]
 }
diff --git a/clients/src/main/resources/common/message/ApiVersionsResponse.json b/clients/src/main/resources/common/message/ApiVersionsResponse.json
index 21fc84a9798c2..ccb44c671f723 100644
--- a/clients/src/main/resources/common/message/ApiVersionsResponse.json
+++ b/clients/src/main/resources/common/message/ApiVersionsResponse.json
@@ -18,14 +18,24 @@
   "type": "response",
   "name": "ApiVersionsResponse",
   // Version 1 adds throttle time to the response.
+  //
   // Starting in version 2, on quota violation, brokers send out responses before throttling.
-  "validVersions": "0-2",
+  //
+  // Version 3 is the first flexible version. Tagged fields are only supported in the body but
+  // not in the header. The length of the header must not change in order to guarantee the
+  // backward compatibility.
+  //
+  // Starting from Apache Kafka 2.4 (KIP-511), ApiKeys field is populated with the supported
+  // versions of the ApiVersionsRequest when an UNSUPPORTED_VERSION error is returned.
+  "validVersions": "0-3",
+  // TODO(dajac) Enable this once KIP-482 is committed.
+  // "flexibleVersions": "3+",
   "fields": [
     { "name": "ErrorCode", "type": "int16", "versions": "0+",
       "about": "The top-level error code." },
     { "name": "ApiKeys", "type": "[]ApiVersionsResponseKey", "versions": "0+",
       "about": "The APIs supported by the broker.", "fields": [
-      { "name": "Index", "type": "int16", "versions": "0+", "mapKey": true,
+      { "name": "ApiKey", "type": "int16", "versions": "0+", "mapKey": true,
         "about": "The API index." },
       { "name": "MinVersion", "type": "int16", "versions": "0+",
         "about": "The minimum supported version, inclusive." },
diff --git a/clients/src/test/java/org/apache/kafka/clients/ApiVersionsTest.java b/clients/src/test/java/org/apache/kafka/clients/ApiVersionsTest.java
index a66a86fb959b7..9458b04ec89e0 100644
--- a/clients/src/test/java/org/apache/kafka/clients/ApiVersionsTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/ApiVersionsTest.java
@@ -18,11 +18,8 @@
 
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.record.RecordBatch;
-import org.apache.kafka.common.requests.ApiVersionsResponse;
 import org.junit.Test;
 
-import java.util.Collections;
-
 import static org.junit.Assert.assertEquals;
 
 public class ApiVersionsTest {
@@ -35,8 +32,7 @@ public void testMaxUsableProduceMagic() {
         apiVersions.update("0", NodeApiVersions.create());
         assertEquals(RecordBatch.CURRENT_MAGIC_VALUE, apiVersions.maxUsableProduceMagic());
 
-        apiVersions.update("1", NodeApiVersions.create(Collections.singleton(
-                new ApiVersionsResponse.ApiVersion(ApiKeys.PRODUCE, (short) 0, (short) 2))));
+        apiVersions.update("1", NodeApiVersions.create(ApiKeys.PRODUCE.id, (short) 0, (short) 2));
         assertEquals(RecordBatch.MAGIC_VALUE_V1, apiVersions.maxUsableProduceMagic());
 
         apiVersions.remove("1");
diff --git a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
index ab11b95520f98..b5ecbc74c52ac 100644
--- a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
@@ -20,6 +20,9 @@
 import org.apache.kafka.common.Node;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ApiVersionsResponseData;
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKey;
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKeyCollection;
 import org.apache.kafka.common.network.NetworkReceive;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.CommonFields;
@@ -29,6 +32,7 @@
 import org.apache.kafka.common.requests.ApiVersionsResponse;
 import org.apache.kafka.common.requests.MetadataRequest;
 import org.apache.kafka.common.requests.ProduceRequest;
+import org.apache.kafka.common.requests.RequestHeader;
 import org.apache.kafka.common.requests.ResponseHeader;
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.common.utils.MockTime;
@@ -189,21 +193,219 @@ private void checkSimpleRequestResponse(NetworkClient networkClient) {
                 request.correlationId(), handler.response.requestHeader().correlationId());
     }
 
-    private void setExpectedApiVersionsResponse(ApiVersionsResponse response) {
-        short apiVersionsResponseVersion = response.apiVersion(ApiKeys.API_VERSIONS.id).maxVersion;
-        ByteBuffer buffer = response.serialize(ApiKeys.API_VERSIONS, apiVersionsResponseVersion, 0);
+    private void delayedApiVersionsResponse(int correlationId, short version, ApiVersionsResponse response) {
+        ByteBuffer buffer = response.serialize(ApiKeys.API_VERSIONS, version, correlationId);
         selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer)));
     }
 
+    private void setExpectedApiVersionsResponse(ApiVersionsResponse response) {
+        short apiVersionsResponseVersion = response.apiVersion(ApiKeys.API_VERSIONS.id).maxVersion();
+        delayedApiVersionsResponse(0, apiVersionsResponseVersion, response);
+    }
+
     private void awaitReady(NetworkClient client, Node node) {
         if (client.discoverBrokerVersions()) {
-            setExpectedApiVersionsResponse(ApiVersionsResponse.defaultApiVersionsResponse());
+            setExpectedApiVersionsResponse(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE);
         }
         while (!client.ready(node, time.milliseconds()))
             client.poll(1, time.milliseconds());
         selector.clear();
     }
 
+    @Test
+    public void testInvalidApiVersionsRequest() {
+        // initiate the connection
+        client.ready(node, time.milliseconds());
+
+        // handle the connection, send the ApiVersionsRequest
+        client.poll(0, time.milliseconds());
+
+        // check that the ApiVersionsRequest has been initiated
+        assertTrue(client.hasInFlightRequests(node.idString()));
+
+        // prepare response
+        delayedApiVersionsResponse(0, ApiKeys.API_VERSIONS.latestVersion(),
+            new ApiVersionsResponse(
+                new ApiVersionsResponseData()
+                    .setErrorCode(Errors.INVALID_REQUEST.code())
+                    .setThrottleTimeMs(0)
+            ));
+
+        // handle completed receives
+        client.poll(0, time.milliseconds());
+
+        // the ApiVersionsRequest is gone
+        assertFalse(client.hasInFlightRequests(node.idString()));
+
+        // various assertions
+        assertFalse(client.isReady(node, time.milliseconds()));
+    }
+
+    @Test
+    public void testApiVersionsRequest() {
+        // initiate the connection
+        client.ready(node, time.milliseconds());
+
+        // handle the connection, send the ApiVersionsRequest
+        client.poll(0, time.milliseconds());
+
+        // check that the ApiVersionsRequest has been initiated
+        assertTrue(client.hasInFlightRequests(node.idString()));
+
+        // prepare response
+        delayedApiVersionsResponse(0, ApiKeys.API_VERSIONS.latestVersion(),
+            ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE);
+
+        // handle completed receives
+        client.poll(0, time.milliseconds());
+
+        // the ApiVersionsRequest is gone
+        assertFalse(client.hasInFlightRequests(node.idString()));
+
+        // various assertions
+        assertTrue(client.isReady(node, time.milliseconds()));
+    }
+
+    @Test
+    public void testUnsupportedApiVersionsRequestWithVersionProvidedByTheBroker() {
+        // initiate the connection
+        client.ready(node, time.milliseconds());
+
+        // handle the connection, initiate first ApiVersionsRequest
+        client.poll(0, time.milliseconds());
+
+        // ApiVersionsRequest is in flight but not sent yet
+        assertTrue(client.hasInFlightRequests(node.idString()));
+
+        // completes initiated sends
+        client.poll(0, time.milliseconds());
+        assertEquals(1, selector.completedSends().size());
+
+        ByteBuffer buffer = selector.completedSendBuffers().get(0).buffer();
+        RequestHeader header = parseHeader(buffer);
+        assertEquals(ApiKeys.API_VERSIONS, header.apiKey());
+        assertEquals(3, header.apiVersion());
+
+        // prepare response
+        ApiVersionsResponseKeyCollection apiKeys = new ApiVersionsResponseKeyCollection();
+        apiKeys.add(new ApiVersionsResponseKey()
+            .setApiKey(ApiKeys.API_VERSIONS.id)
+            .setMinVersion((short) 0)
+            .setMaxVersion((short) 2));
+        delayedApiVersionsResponse(0, (short) 0,
+            new ApiVersionsResponse(
+                new ApiVersionsResponseData()
+                    .setErrorCode(Errors.UNSUPPORTED_VERSION.code())
+                    .setApiKeys(apiKeys)
+            ));
+
+        // handle ApiVersionResponse, initiate second ApiVersionRequest
+        client.poll(0, time.milliseconds());
+
+        // ApiVersionsRequest is in flight but not sent yet
+        assertTrue(client.hasInFlightRequests(node.idString()));
+
+        // ApiVersionsResponse has been received
+        assertEquals(1, selector.completedReceives().size());
+
+        // clean up the buffers
+        selector.completedSends().clear();
+        selector.completedSendBuffers().clear();
+        selector.completedReceives().clear();
+
+        // completes initiated sends
+        client.poll(0, time.milliseconds());
+
+        // ApiVersionsRequest has been sent
+        assertEquals(1, selector.completedSends().size());
+
+        buffer = selector.completedSendBuffers().get(0).buffer();
+        header = parseHeader(buffer);
+        assertEquals(ApiKeys.API_VERSIONS, header.apiKey());
+        assertEquals(2, header.apiVersion());
+
+        // prepare response
+        delayedApiVersionsResponse(1, (short) 0,
+            ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE);
+
+        // handle completed receives
+        client.poll(0, time.milliseconds());
+
+        // the ApiVersionsRequest is gone
+        assertFalse(client.hasInFlightRequests(node.idString()));
+        assertEquals(1, selector.completedReceives().size());
+
+        // the client is ready
+        assertTrue(client.isReady(node, time.milliseconds()));
+    }
+
+    @Test
+    public void testUnsupportedApiVersionsRequestWithoutVersionProvidedByTheBroker() {
+        // initiate the connection
+        client.ready(node, time.milliseconds());
+
+        // handle the connection, initiate first ApiVersionsRequest
+        client.poll(0, time.milliseconds());
+
+        // ApiVersionsRequest is in flight but not sent yet
+        assertTrue(client.hasInFlightRequests(node.idString()));
+
+        // completes initiated sends
+        client.poll(0, time.milliseconds());
+        assertEquals(1, selector.completedSends().size());
+
+        ByteBuffer buffer = selector.completedSendBuffers().get(0).buffer();
+        RequestHeader header = parseHeader(buffer);
+        assertEquals(ApiKeys.API_VERSIONS, header.apiKey());
+        assertEquals(3, header.apiVersion());
+
+        // prepare response
+        delayedApiVersionsResponse(0, (short) 0,
+            new ApiVersionsResponse(
+                new ApiVersionsResponseData()
+                    .setErrorCode(Errors.UNSUPPORTED_VERSION.code())
+            ));
+
+        // handle ApiVersionResponse, initiate second ApiVersionRequest
+        client.poll(0, time.milliseconds());
+
+        // ApiVersionsRequest is in flight but not sent yet
+        assertTrue(client.hasInFlightRequests(node.idString()));
+
+        // ApiVersionsResponse has been received
+        assertEquals(1, selector.completedReceives().size());
+
+        // clean up the buffers
+        selector.completedSends().clear();
+        selector.completedSendBuffers().clear();
+        selector.completedReceives().clear();
+
+        // completes initiated sends
+        client.poll(0, time.milliseconds());
+
+        // ApiVersionsRequest has been sent
+        assertEquals(1, selector.completedSends().size());
+        
+        buffer = selector.completedSendBuffers().get(0).buffer();
+        header = parseHeader(buffer);
+        assertEquals(ApiKeys.API_VERSIONS, header.apiKey());
+        assertEquals(0, header.apiVersion());
+
+        // prepare response
+        delayedApiVersionsResponse(1, (short) 0,
+            ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE);
+
+        // handle completed receives
+        client.poll(0, time.milliseconds());
+
+        // the ApiVersionsRequest is gone
+        assertFalse(client.hasInFlightRequests(node.idString()));
+        assertEquals(1, selector.completedReceives().size());
+
+        // the client is ready
+        assertTrue(client.isReady(node, time.milliseconds()));
+    }
+
     @Test
     public void testRequestTimeout() {
         awaitReady(client, node); // has to be before creating any request, as it may send ApiVersionsRequest and its response is mocked with correlation id 0
@@ -288,15 +490,24 @@ public void testConnectionThrottling() {
     // Creates expected ApiVersionsResponse from the specified node, where the max protocol version for the specified
     // key is set to the specified version.
     private ApiVersionsResponse createExpectedApiVersionsResponse(ApiKeys key, short maxVersion) {
-        List versionList = new ArrayList<>();
+        ApiVersionsResponseKeyCollection versionList = new ApiVersionsResponseKeyCollection();
         for (ApiKeys apiKey : ApiKeys.values()) {
             if (apiKey == key) {
-                versionList.add(new ApiVersionsResponse.ApiVersion(apiKey, (short) 0, maxVersion));
+                versionList.add(new ApiVersionsResponseKey()
+                    .setApiKey(apiKey.id)
+                    .setMinVersion((short) 0)
+                    .setMaxVersion(maxVersion));
             } else {
-                versionList.add(new ApiVersionsResponse.ApiVersion(apiKey));
+                versionList.add(new ApiVersionsResponseKey()
+                    .setApiKey(apiKey.id)
+                    .setMinVersion(apiKey.oldestVersion())
+                    .setMaxVersion(apiKey.latestVersion()));
             }
         }
-        return new ApiVersionsResponse(0, Errors.NONE, versionList);
+        return new ApiVersionsResponse(new ApiVersionsResponseData()
+            .setErrorCode(Errors.NONE.code())
+            .setThrottleTimeMs(0)
+            .setApiKeys(versionList));
     }
 
     @Test
@@ -593,6 +804,11 @@ public void testCallDisconnect() throws Exception {
         assertTrue(client.canConnect(node, time.milliseconds()));
     }
 
+    private RequestHeader parseHeader(ByteBuffer buffer) {
+        buffer.getInt(); // skip size
+        return RequestHeader.parse(buffer.slice());
+    }
+
     private void awaitInFlightApiVersionRequest() throws Exception {
         client.ready(node, time.milliseconds());
         TestUtils.waitForCondition(new TestCondition() {
diff --git a/clients/src/test/java/org/apache/kafka/clients/NodeApiVersionsTest.java b/clients/src/test/java/org/apache/kafka/clients/NodeApiVersionsTest.java
index 58fb363ad5378..99af49ef73551 100644
--- a/clients/src/test/java/org/apache/kafka/clients/NodeApiVersionsTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/NodeApiVersionsTest.java
@@ -16,17 +16,16 @@
  */
 package org.apache.kafka.clients;
 
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
 import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKey;
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKeyCollection;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.requests.ApiVersionsResponse;
-import org.apache.kafka.common.requests.ApiVersionsResponse.ApiVersion;
 import org.junit.Test;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -34,7 +33,7 @@ public class NodeApiVersionsTest {
 
     @Test
     public void testUnsupportedVersionsToString() {
-        NodeApiVersions versions = new NodeApiVersions(Collections.emptyList());
+        NodeApiVersions versions = new NodeApiVersions(new ApiVersionsResponseKeyCollection());
         StringBuilder bld = new StringBuilder();
         String prefix = "(";
         for (ApiKeys apiKey : ApiKeys.values()) {
@@ -48,8 +47,7 @@ public void testUnsupportedVersionsToString() {
 
     @Test
     public void testUnknownApiVersionsToString() {
-        ApiVersion unknownApiVersion = new ApiVersion((short) 337, (short) 0, (short) 1);
-        NodeApiVersions versions = new NodeApiVersions(Collections.singleton(unknownApiVersion));
+        NodeApiVersions versions = NodeApiVersions.create((short) 337, (short) 0, (short) 1);
         assertTrue(versions.toString().endsWith("UNKNOWN(337): 0 to 1)"));
     }
 
@@ -58,7 +56,7 @@ public void testVersionsToString() {
         List versionList = new ArrayList<>();
         for (ApiKeys apiKey : ApiKeys.values()) {
             if (apiKey == ApiKeys.DELETE_TOPICS) {
-                versionList.add(new ApiVersion(apiKey, (short) 10000, (short) 10001));
+                versionList.add(new ApiVersion(apiKey.id, (short) 10000, (short) 10001));
             } else {
                 versionList.add(new ApiVersion(apiKey));
             }
@@ -92,8 +90,7 @@ public void testVersionsToString() {
 
     @Test
     public void testLatestUsableVersion() {
-        NodeApiVersions apiVersions = NodeApiVersions.create(Collections.singleton(
-                new ApiVersion(ApiKeys.PRODUCE, (short) 1, (short) 3)));
+        NodeApiVersions apiVersions = NodeApiVersions.create(ApiKeys.PRODUCE.id, (short) 1, (short) 3);
         assertEquals(3, apiVersions.latestUsableVersion(ApiKeys.PRODUCE));
         assertEquals(1, apiVersions.latestUsableVersion(ApiKeys.PRODUCE, (short) 0, (short) 1));
         assertEquals(1, apiVersions.latestUsableVersion(ApiKeys.PRODUCE, (short) 1, (short) 1));
@@ -107,43 +104,52 @@ public void testLatestUsableVersion() {
 
     @Test(expected = UnsupportedVersionException.class)
     public void testLatestUsableVersionOutOfRangeLow() {
-        NodeApiVersions apiVersions = NodeApiVersions.create(Collections.singleton(
-                new ApiVersion(ApiKeys.PRODUCE, (short) 1, (short) 2)));
+        NodeApiVersions apiVersions = NodeApiVersions.create(ApiKeys.PRODUCE.id, (short) 1, (short) 2);
         apiVersions.latestUsableVersion(ApiKeys.PRODUCE, (short) 3, (short) 4);
     }
 
     @Test(expected = UnsupportedVersionException.class)
     public void testLatestUsableVersionOutOfRangeHigh() {
-        NodeApiVersions apiVersions = NodeApiVersions.create(Collections.singleton(
-                new ApiVersion(ApiKeys.PRODUCE, (short) 2, (short) 3)));
+        NodeApiVersions apiVersions = NodeApiVersions.create(ApiKeys.PRODUCE.id, (short) 2, (short) 3);
         apiVersions.latestUsableVersion(ApiKeys.PRODUCE, (short) 0, (short) 1);
     }
 
     @Test(expected = UnsupportedVersionException.class)
     public void testUsableVersionCalculationNoKnownVersions() {
-        List versionList = new ArrayList<>();
-        NodeApiVersions versions =  new NodeApiVersions(versionList);
+        NodeApiVersions versions = new NodeApiVersions(new ApiVersionsResponseKeyCollection());
         versions.latestUsableVersion(ApiKeys.FETCH);
     }
 
     @Test(expected = UnsupportedVersionException.class)
     public void testLatestUsableVersionOutOfRange() {
-        NodeApiVersions apiVersions = NodeApiVersions.create(Collections.singleton(
-                new ApiVersion(ApiKeys.PRODUCE, (short) 300, (short) 300)));
+        NodeApiVersions apiVersions = NodeApiVersions.create(ApiKeys.PRODUCE.id, (short) 300, (short) 300);
         apiVersions.latestUsableVersion(ApiKeys.PRODUCE);
     }
 
     @Test
     public void testUsableVersionLatestVersions() {
         List versionList = new LinkedList<>();
-        for (ApiVersion apiVersion: ApiVersionsResponse.defaultApiVersionsResponse().apiVersions()) {
-            versionList.add(apiVersion);
+        for (ApiVersionsResponseKey apiVersion: ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data.apiKeys()) {
+            versionList.add(new ApiVersion(apiVersion));
         }
         // Add an API key that we don't know about.
         versionList.add(new ApiVersion((short) 100, (short) 0, (short) 1));
-        NodeApiVersions versions =  new NodeApiVersions(versionList);
+        NodeApiVersions versions = new NodeApiVersions(versionList);
         for (ApiKeys apiKey: ApiKeys.values()) {
             assertEquals(apiKey.latestVersion(), versions.latestUsableVersion(apiKey));
         }
     }
+
+    @Test
+    public void testConstructionFromApiVersionsResponse() {
+        ApiVersionsResponse apiVersionsResponse = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE;
+        NodeApiVersions versions = new NodeApiVersions(apiVersionsResponse.data.apiKeys());
+
+        for (ApiVersionsResponseKey apiVersionKey : apiVersionsResponse.data.apiKeys()) {
+            ApiVersion apiVersion = versions.apiVersion(ApiKeys.forId(apiVersionKey.apiKey()));
+            assertEquals(apiVersionKey.apiKey(), apiVersion.apiKey);
+            assertEquals(apiVersionKey.minVersion(), apiVersion.minVersion);
+            assertEquals(apiVersionKey.maxVersion(), apiVersion.maxVersion);
+        }
+    }
 }
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
index a8b22bdb42a17..66786ca7273f1 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
@@ -37,7 +37,6 @@
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.requests.AbstractRequest;
-import org.apache.kafka.common.requests.ApiVersionsResponse;
 import org.apache.kafka.common.requests.FindCoordinatorResponse;
 import org.apache.kafka.common.requests.HeartbeatRequest;
 import org.apache.kafka.common.requests.HeartbeatResponse;
@@ -502,8 +501,7 @@ public void testHandleLeaveGroupResponseWithException() {
     @Test
     public void testHandleSingleLeaveGroupRequest() {
         setupCoordinator(RETRY_BACKOFF_MS, Integer.MAX_VALUE, Optional.empty());
-        mockClient.setNodeApiVersions(NodeApiVersions.create(Collections.singletonList(
-            new ApiVersionsResponse.ApiVersion(ApiKeys.LEAVE_GROUP, (short) 2, (short) 2))));
+        mockClient.setNodeApiVersions(NodeApiVersions.create(ApiKeys.LEAVE_GROUP.id, (short) 2, (short) 2));
 
         LeaveGroupResponse expectedResponse = leaveGroupResponse(Collections.singletonList(
             new MemberResponse()
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
index f2246d90d74b0..ff0afe92f7106 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
@@ -781,8 +781,7 @@ public void testFetchRequestWhenRecordTooLarge() {
         try {
             buildFetcher();
 
-            client.setNodeApiVersions(NodeApiVersions.create(Collections.singletonList(
-                new ApiVersionsResponse.ApiVersion(ApiKeys.FETCH, (short) 2, (short) 2))));
+            client.setNodeApiVersions(NodeApiVersions.create(ApiKeys.FETCH.id, (short) 2, (short) 2));
             makeFetchRequestWithIncompleteRecord();
             try {
                 fetcher.fetchedRecords();
@@ -3485,8 +3484,8 @@ public void testOffsetValidationSkippedForOldBroker() {
 
         // Offset validation requires OffsetForLeaderEpoch request v3 or higher
         Node node = metadata.fetch().nodes().get(0);
-        apiVersions.update(node.idString(), NodeApiVersions.create(singleton(
-                new ApiVersionsResponse.ApiVersion(ApiKeys.OFFSET_FOR_LEADER_EPOCH, (short) 0, (short) 2))));
+        apiVersions.update(node.idString(), NodeApiVersions.create(
+            ApiKeys.OFFSET_FOR_LEADER_EPOCH.id, (short) 0, (short) 2));
 
         // Seek with a position and leader+epoch
         Metadata.LeaderAndEpoch leaderAndEpoch = new Metadata.LeaderAndEpoch(
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java
index cdb164af5b2dd..b04e5ddcfe530 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java
@@ -38,7 +38,6 @@
 import org.apache.kafka.common.record.MutableRecordBatch;
 import org.apache.kafka.common.record.Record;
 import org.apache.kafka.common.record.TimestampType;
-import org.apache.kafka.common.requests.ApiVersionsResponse;
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.common.utils.MockTime;
 import org.apache.kafka.common.utils.Time;
@@ -188,8 +187,7 @@ private void testAppendLargeOldMessageFormat(CompressionType compressionType) th
         byte[] value = new byte[2 * batchSize];
 
         ApiVersions apiVersions = new ApiVersions();
-        apiVersions.update(node1.idString(), NodeApiVersions.create(Collections.singleton(
-                new ApiVersionsResponse.ApiVersion(ApiKeys.PRODUCE, (short) 0, (short) 2))));
+        apiVersions.update(node1.idString(), NodeApiVersions.create(ApiKeys.PRODUCE.id, (short) 0, (short) 2));
 
         RecordAccumulator accum = createTestRecordAccumulator(
                 batchSize + DefaultRecordBatch.RECORD_BATCH_OVERHEAD, 10 * 1024, compressionType, 0);
@@ -706,8 +704,7 @@ public void testIdempotenceWithOldMagic() throws InterruptedException {
         long totalSize = 10 * batchSize;
         String metricGrpName = "producer-metrics";
 
-        apiVersions.update("foobar", NodeApiVersions.create(Arrays.asList(new ApiVersionsResponse.ApiVersion(ApiKeys.PRODUCE,
-                (short) 0, (short) 2))));
+        apiVersions.update("foobar", NodeApiVersions.create(ApiKeys.PRODUCE.id, (short) 0, (short) 2));
         RecordAccumulator accum = new RecordAccumulator(logContext, batchSize + DefaultRecordBatch.RECORD_BATCH_OVERHEAD,
             CompressionType.NONE, lingerMs, retryBackoffMs, deliveryTimeoutMs, metrics, metricGrpName, time, apiVersions, new TransactionManager(),
             new BufferPool(totalSize, batchSize, metrics, time, metricGrpName));
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java
index 653faf21aa0a4..71180bdd7f7a7 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java
@@ -182,8 +182,7 @@ public void testMessageFormatDownConversion() throws Exception {
                 null, null, MAX_BLOCK_TIMEOUT, false).future;
 
         // now the partition leader supports only v2
-        apiVersions.update("0", NodeApiVersions.create(Collections.singleton(
-                new ApiVersionsResponse.ApiVersion(ApiKeys.PRODUCE, (short) 0, (short) 2))));
+        apiVersions.update("0", NodeApiVersions.create(ApiKeys.PRODUCE.id, (short) 0, (short) 2));
 
         client.prepareResponse(new MockClient.RequestMatcher() {
             @Override
@@ -222,8 +221,7 @@ public void testDownConversionForMismatchedMagicValues() throws Exception {
                 null, null, MAX_BLOCK_TIMEOUT, false).future;
 
         // now the partition leader supports only v2
-        apiVersions.update("0", NodeApiVersions.create(Collections.singleton(
-                new ApiVersionsResponse.ApiVersion(ApiKeys.PRODUCE, (short) 0, (short) 2))));
+        apiVersions.update("0", NodeApiVersions.create(ApiKeys.PRODUCE.id, (short) 0, (short) 2));
 
         Future future2 = accumulator.append(tp1, 0L, "key".getBytes(), "value".getBytes(),
                 null, null, MAX_BLOCK_TIMEOUT, false).future;
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ApiVersionsResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ApiVersionsResponseTest.java
index 2b526d12fe248..c7f1ecc001d94 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/ApiVersionsResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/ApiVersionsResponseTest.java
@@ -17,6 +17,7 @@
 
 package org.apache.kafka.common.requests;
 
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKey;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.record.RecordBatch;
 import org.apache.kafka.common.utils.Utils;
@@ -43,7 +44,7 @@ public void shouldCreateApiResponseOnlyWithKeysSupportedByMagicValue() {
 
     @Test
     public void shouldCreateApiResponseThatHasAllApiKeysSupportedByBroker() {
-        assertEquals(apiKeysInResponse(ApiVersionsResponse.defaultApiVersionsResponse()), Utils.mkSet(ApiKeys.values()));
+        assertEquals(apiKeysInResponse(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE), Utils.mkSet(ApiKeys.values()));
     }
 
     @Test
@@ -55,39 +56,39 @@ public void shouldReturnAllKeysWhenMagicIsCurrentValueAndThrottleMsIsDefaultThro
 
     @Test
     public void shouldHaveCorrectDefaultApiVersionsResponse() {
-        Collection apiVersions = ApiVersionsResponse.defaultApiVersionsResponse().apiVersions();
+        Collection apiVersions = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data.apiKeys();
         assertEquals("API versions for all API keys must be maintained.", apiVersions.size(), ApiKeys.values().length);
 
         for (ApiKeys key : ApiKeys.values()) {
-            ApiVersionsResponse.ApiVersion version = ApiVersionsResponse.defaultApiVersionsResponse().apiVersion(key.id);
+            ApiVersionsResponseKey version = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.apiVersion(key.id);
             assertNotNull("Could not find ApiVersion for API " + key.name, version);
-            assertEquals("Incorrect min version for Api " + key.name, version.minVersion, key.oldestVersion());
-            assertEquals("Incorrect max version for Api " + key.name, version.maxVersion, key.latestVersion());
+            assertEquals("Incorrect min version for Api " + key.name, version.minVersion(), key.oldestVersion());
+            assertEquals("Incorrect max version for Api " + key.name, version.maxVersion(), key.latestVersion());
 
             // Check if versions less than min version are indeed set as null, i.e., deprecated.
-            for (int i = 0; i < version.minVersion; ++i) {
-                assertNull("Request version " + i + " for API " + version.apiKey + " must be null", key.requestSchemas[i]);
-                assertNull("Response version " + i + " for API " + version.apiKey + " must be null", key.responseSchemas[i]);
+            for (int i = 0; i < version.minVersion(); ++i) {
+                assertNull("Request version " + i + " for API " + version.apiKey() + " must be null", key.requestSchemas[i]);
+                assertNull("Response version " + i + " for API " + version.apiKey() + " must be null", key.responseSchemas[i]);
             }
 
             // Check if versions between min and max versions are non null, i.e., valid.
-            for (int i = version.minVersion; i <= version.maxVersion; ++i) {
-                assertNotNull("Request version " + i + " for API " + version.apiKey + " must not be null", key.requestSchemas[i]);
-                assertNotNull("Response version " + i + " for API " + version.apiKey + " must not be null", key.responseSchemas[i]);
+            for (int i = version.minVersion(); i <= version.maxVersion(); ++i) {
+                assertNotNull("Request version " + i + " for API " + version.apiKey() + " must not be null", key.requestSchemas[i]);
+                assertNotNull("Response version " + i + " for API " + version.apiKey() + " must not be null", key.responseSchemas[i]);
             }
         }
     }
     
     private void verifyApiKeysForMagic(final ApiVersionsResponse response, final byte maxMagic) {
-        for (final ApiVersionsResponse.ApiVersion version : response.apiVersions()) {
-            assertTrue(ApiKeys.forId(version.apiKey).minRequiredInterBrokerMagic <= maxMagic);
+        for (final ApiVersionsResponseKey version : response.data.apiKeys()) {
+            assertTrue(ApiKeys.forId(version.apiKey()).minRequiredInterBrokerMagic <= maxMagic);
         }
     }
 
     private Set apiKeysInResponse(final ApiVersionsResponse apiVersions) {
         final Set apiKeys = new HashSet<>();
-        for (final ApiVersionsResponse.ApiVersion version : apiVersions.apiVersions()) {
-            apiKeys.add(ApiKeys.forId(version.apiKey));
+        for (final ApiVersionsResponseKey version : apiVersions.data.apiKeys()) {
+            apiKeys.add(ApiKeys.forId(version.apiKey()));
         }
         return apiKeys;
     }
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java
index d9ac2c2d0d77f..99ec0effa9cf3 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java
@@ -16,6 +16,8 @@
  */
 package org.apache.kafka.common.requests;
 
+import org.apache.kafka.common.message.ApiVersionsResponseData;
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKeyCollection;
 import org.apache.kafka.common.network.ListenerName;
 import org.apache.kafka.common.network.Send;
 import org.apache.kafka.common.protocol.ApiKeys;
@@ -27,7 +29,6 @@
 
 import java.net.InetAddress;
 import java.nio.ByteBuffer;
-import java.util.Collections;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -55,8 +56,10 @@ public void testSerdeUnsupportedApiVersionRequest() throws Exception {
         ApiVersionsRequest request = (ApiVersionsRequest) requestAndSize.request;
         assertTrue(request.hasUnsupportedRequestVersion());
 
-        Send send = context.buildResponse(new ApiVersionsResponse(0, Errors.UNSUPPORTED_VERSION,
-                Collections.emptyList()));
+        Send send = context.buildResponse(new ApiVersionsResponse(new ApiVersionsResponseData()
+            .setThrottleTimeMs(0)
+            .setErrorCode(Errors.UNSUPPORTED_VERSION.code())
+            .setApiKeys(new ApiVersionsResponseKeyCollection())));
         ByteBufferChannel channel = new ByteBufferChannel(256);
         send.writeTo(channel);
 
@@ -70,8 +73,8 @@ public void testSerdeUnsupportedApiVersionRequest() throws Exception {
         Struct struct = ApiKeys.API_VERSIONS.parseResponse((short) 0, responseBuffer);
         ApiVersionsResponse response = (ApiVersionsResponse)
             AbstractResponse.parseResponse(ApiKeys.API_VERSIONS, struct, (short) 0);
-        assertEquals(Errors.UNSUPPORTED_VERSION, response.error());
-        assertTrue(response.apiVersions().isEmpty());
+        assertEquals(Errors.UNSUPPORTED_VERSION.code(), response.data.errorCode());
+        assertTrue(response.data.apiKeys().isEmpty());
     }
 
 }
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index 99172408d65ed..a8cbfffec1b9d 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -33,6 +33,10 @@
 import org.apache.kafka.common.errors.SecurityDisabledException;
 import org.apache.kafka.common.errors.UnknownServerException;
 import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ApiVersionsRequestData;
+import org.apache.kafka.common.message.ApiVersionsResponseData;
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKey;
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKeyCollection;
 import org.apache.kafka.common.message.ControlledShutdownRequestData;
 import org.apache.kafka.common.message.ControlledShutdownResponseData.RemainingPartition;
 import org.apache.kafka.common.message.ControlledShutdownResponseData.RemainingPartitionCollection;
@@ -109,6 +113,7 @@
 import org.apache.kafka.common.network.Send;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.protocol.types.SchemaException;
 import org.apache.kafka.common.protocol.types.Struct;
 import org.apache.kafka.common.record.CompressionType;
 import org.apache.kafka.common.record.MemoryRecords;
@@ -156,6 +161,7 @@
 import static org.apache.kafka.test.TestUtils.toBuffer;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -257,7 +263,16 @@ public void testSerialization() throws Exception {
         checkResponse(createSaslAuthenticateResponse(), 1, true);
         checkRequest(createApiVersionRequest(), true);
         checkErrorResponse(createApiVersionRequest(), new UnknownServerException(), true);
+        checkErrorResponse(createApiVersionRequest(), new UnsupportedVersionException("Not Supported"), true);
         checkResponse(createApiVersionResponse(), 0, true);
+        checkResponse(createApiVersionResponse(), 1, true);
+        checkResponse(createApiVersionResponse(), 2, true);
+        checkResponse(createApiVersionResponse(), 3, true);
+        checkResponse(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE, 0, true);
+        checkResponse(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE, 1, true);
+        checkResponse(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE, 2, true);
+        checkResponse(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE, 3, true);
+
         checkRequest(createCreateTopicRequest(0), true);
         checkErrorResponse(createCreateTopicRequest(0), new UnknownServerException(), true);
         checkResponse(createCreateTopicResponse(), 0, true);
@@ -770,6 +785,91 @@ public void testOffsetFetchRequestBuilderToString() {
         assertTrue(string.contains("group1"));
     }
 
+    @Test
+    public void testApiVersionsRequestBeforeV3Validation() {
+        for (short version = 0; version < 3; version++) {
+            ApiVersionsRequest request = new ApiVersionsRequest(new ApiVersionsRequestData(), version);
+            assertTrue(request.isValid());
+        }
+    }
+
+    @Test
+    public void testValidApiVersionsRequest() {
+        ApiVersionsRequest request;
+
+        request = new ApiVersionsRequest.Builder().build();
+        assertTrue(request.isValid());
+
+        request = new ApiVersionsRequest(new ApiVersionsRequestData()
+            .setClientSoftwareName("apache-kafka.java")
+            .setClientSoftwareVersion("0.0.0-SNAPSHOT"),
+            ApiKeys.API_VERSIONS.latestVersion()
+        );
+        assertTrue(request.isValid());
+    }
+
+    @Test
+    public void testInvalidApiVersionsRequest() {
+        testInvalidCase("java@apache_kafka", "0.0.0-SNAPSHOT");
+        testInvalidCase("apache-kafka-java", "0.0.0@java");
+        testInvalidCase("-apache-kafka-java", "0.0.0");
+        testInvalidCase("apache-kafka-java.", "0.0.0");
+    }
+
+    private void testInvalidCase(String name, String version) {
+        ApiVersionsRequest request = new ApiVersionsRequest(new ApiVersionsRequestData()
+            .setClientSoftwareName(name)
+            .setClientSoftwareVersion(version),
+            ApiKeys.API_VERSIONS.latestVersion()
+        );
+        assertFalse(request.isValid());
+    }
+
+    @Test
+    public void testApiVersionResponseWithUnsupportedError() {
+        ApiVersionsRequest request = new ApiVersionsRequest.Builder().build();
+        ApiVersionsResponse response = request.getErrorResponse(0, Errors.UNSUPPORTED_VERSION.exception());
+
+        assertEquals(Errors.UNSUPPORTED_VERSION.code(), response.data.errorCode());
+
+        ApiVersionsResponseKey apiVersion = response.data.apiKeys().find(ApiKeys.API_VERSIONS.id);
+        assertNotNull(apiVersion);
+        assertEquals(ApiKeys.API_VERSIONS.id, apiVersion.apiKey());
+        assertEquals(ApiKeys.API_VERSIONS.oldestVersion(), apiVersion.minVersion());
+        assertEquals(ApiKeys.API_VERSIONS.latestVersion(), apiVersion.maxVersion());
+    }
+
+    @Test
+    public void testApiVersionResponseWithNotUnsupportedError() {
+        ApiVersionsRequest request = new ApiVersionsRequest.Builder().build();
+        ApiVersionsResponse response = request.getErrorResponse(0, Errors.INVALID_REQUEST.exception());
+
+        assertEquals(response.data.errorCode(), Errors.INVALID_REQUEST.code());
+        assertTrue(response.data.apiKeys().isEmpty());
+    }
+
+    @Test
+    public void testApiVersionResponseStructParsingFallback() {
+        Struct struct = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.toStruct((short) 0);
+        ApiVersionsResponse response = ApiVersionsResponse.apiVersionsResponse(struct, ApiKeys.API_VERSIONS.latestVersion());
+
+        assertEquals(Errors.NONE.code(), response.data.errorCode());
+    }
+
+    @Test(expected = SchemaException.class)
+    public void testApiVersionResponseStructParsingFallbackException() {
+        short version = 0;
+        ApiVersionsResponse.apiVersionsResponse(new Struct(ApiKeys.API_VERSIONS.requestSchema(version)), version);
+    }
+
+    @Test
+    public void testApiVersionResponseStructParsing() {
+        Struct struct = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.toStruct(ApiKeys.API_VERSIONS.latestVersion());
+        ApiVersionsResponse response = ApiVersionsResponse.apiVersionsResponse(struct, ApiKeys.API_VERSIONS.latestVersion());
+
+        assertEquals(Errors.NONE.code(), response.data.errorCode());
+    }
+
     private ResponseHeader createResponseHeader(short headerVersion) {
         return new ResponseHeader(10, headerVersion);
     }
@@ -1341,8 +1441,16 @@ private ApiVersionsRequest createApiVersionRequest() {
     }
 
     private ApiVersionsResponse createApiVersionResponse() {
-        List apiVersions = asList(new ApiVersionsResponse.ApiVersion((short) 0, (short) 0, (short) 2));
-        return new ApiVersionsResponse(Errors.NONE, apiVersions);
+        ApiVersionsResponseKeyCollection apiVersions = new ApiVersionsResponseKeyCollection();
+        apiVersions.add(new ApiVersionsResponseKey()
+            .setApiKey((short) 0)
+            .setMinVersion((short) 0)
+            .setMaxVersion((short) 2));
+
+        return new ApiVersionsResponse(new ApiVersionsResponseData()
+            .setErrorCode(Errors.NONE.code())
+            .setThrottleTimeMs(0)
+            .setApiKeys(apiVersions));
     }
 
     private CreateTopicsRequest createCreateTopicRequest(int version) {
diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
index 82a1f97926472..152290f360b62 100644
--- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
@@ -27,7 +27,6 @@
 import java.util.Base64;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -53,6 +52,10 @@
 import org.apache.kafka.common.config.internals.BrokerSecurityConfigs;
 import org.apache.kafka.common.config.types.Password;
 import org.apache.kafka.common.errors.SaslAuthenticationException;
+import org.apache.kafka.common.message.ApiVersionsRequestData;
+import org.apache.kafka.common.message.ApiVersionsResponseData;
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKey;
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKeyCollection;
 import org.apache.kafka.common.message.RequestHeaderData;
 import org.apache.kafka.common.message.SaslAuthenticateRequestData;
 import org.apache.kafka.common.message.SaslHandshakeRequestData;
@@ -77,7 +80,6 @@
 import org.apache.kafka.common.requests.AbstractResponse;
 import org.apache.kafka.common.requests.ApiVersionsRequest;
 import org.apache.kafka.common.requests.ApiVersionsResponse;
-import org.apache.kafka.common.requests.ApiVersionsResponse.ApiVersion;
 import org.apache.kafka.common.requests.MetadataRequest;
 import org.apache.kafka.common.requests.RequestHeader;
 import org.apache.kafka.common.requests.ResponseHeader;
@@ -116,6 +118,7 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -692,7 +695,44 @@ public void testApiVersionsRequestWithUnsupportedVersion() throws Exception {
         ByteBuffer responseBuffer = waitForResponse();
         ResponseHeader.parse(responseBuffer, header.headerVersion());
         ApiVersionsResponse response = ApiVersionsResponse.parse(responseBuffer, (short) 0);
-        assertEquals(Errors.UNSUPPORTED_VERSION, response.error());
+        assertEquals(Errors.UNSUPPORTED_VERSION.code(), response.data.errorCode());
+
+        ApiVersionsResponseKey apiVersion = response.data.apiKeys().find(ApiKeys.API_VERSIONS.id);
+        assertNotNull(apiVersion);
+        assertEquals(ApiKeys.API_VERSIONS.id, apiVersion.apiKey());
+        assertEquals(ApiKeys.API_VERSIONS.oldestVersion(), apiVersion.minVersion());
+        assertEquals(ApiKeys.API_VERSIONS.latestVersion(), apiVersion.maxVersion());
+
+        // Send ApiVersionsRequest with a supported version. This should succeed.
+        sendVersionRequestReceiveResponse(node);
+
+        // Test that client can authenticate successfully
+        sendHandshakeRequestReceiveResponse(node, handshakeVersion);
+        authenticateUsingSaslPlainAndCheckConnection(node, handshakeVersion > 0);
+    }
+
+    /**
+     * Tests that invalid ApiVersionRequest is handled by the server correctly and
+     * returns an INVALID_REQUEST error.
+     */
+    @Test
+    public void testInvalidApiVersionsRequest() throws Exception {
+        short handshakeVersion = ApiKeys.SASL_HANDSHAKE.latestVersion();
+        SecurityProtocol securityProtocol = SecurityProtocol.SASL_PLAINTEXT;
+        configureMechanisms("PLAIN", Arrays.asList("PLAIN"));
+        server = createEchoServer(securityProtocol);
+
+        // Send ApiVersionsRequest with invalid version and validate error response.
+        String node = "1";
+        short version = ApiKeys.API_VERSIONS.latestVersion();
+        createClientConnection(SecurityProtocol.PLAINTEXT, node);
+        RequestHeader header = new RequestHeader(ApiKeys.API_VERSIONS, version, "someclient", 1);
+        ApiVersionsRequest request = new ApiVersionsRequest(new ApiVersionsRequestData(), version);
+        selector.send(request.toSend(node, header));
+        ByteBuffer responseBuffer = waitForResponse();
+        ResponseHeader.parse(responseBuffer, header.headerVersion());
+        ApiVersionsResponse response = ApiVersionsResponse.parse(responseBuffer, version);
+        assertEquals(Errors.INVALID_REQUEST.code(), response.data.errorCode());
 
         // Send ApiVersionsRequest with a supported version. This should succeed.
         sendVersionRequestReceiveResponse(node);
@@ -702,6 +742,34 @@ public void testApiVersionsRequestWithUnsupportedVersion() throws Exception {
         authenticateUsingSaslPlainAndCheckConnection(node, handshakeVersion > 0);
     }
 
+    /**
+     * Tests that valid ApiVersionRequest is handled by the server correctly and
+     * returns an NONE error.
+     */
+    @Test
+    public void testValidApiVersionsRequest() throws Exception {
+        short handshakeVersion = ApiKeys.SASL_HANDSHAKE.latestVersion();
+        SecurityProtocol securityProtocol = SecurityProtocol.SASL_PLAINTEXT;
+        configureMechanisms("PLAIN", Arrays.asList("PLAIN"));
+        server = createEchoServer(securityProtocol);
+
+        // Send ApiVersionsRequest with valid version and validate error response.
+        String node = "1";
+        short version = ApiKeys.API_VERSIONS.latestVersion();
+        createClientConnection(SecurityProtocol.PLAINTEXT, node);
+        RequestHeader header = new RequestHeader(ApiKeys.API_VERSIONS, version, "someclient", 1);
+        ApiVersionsRequest request = new ApiVersionsRequest.Builder().build(version);
+        selector.send(request.toSend(node, header));
+        ByteBuffer responseBuffer = waitForResponse();
+        ResponseHeader.parse(responseBuffer, header.headerVersion());
+        ApiVersionsResponse response = ApiVersionsResponse.parse(responseBuffer, version);
+        assertEquals(Errors.NONE.code(), response.data.errorCode());
+
+        // Test that client can authenticate successfully
+        sendHandshakeRequestReceiveResponse(node, handshakeVersion);
+        authenticateUsingSaslPlainAndCheckConnection(node, handshakeVersion > 0);
+    }
+
     /**
      * Tests that unsupported version of SASL handshake request returns error
      * response and fails authentication. This test is similar to
@@ -1680,15 +1748,25 @@ protected SaslServerAuthenticator buildServerAuthenticator(Map config
 
                     @Override
                     protected ApiVersionsResponse apiVersionsResponse() {
-                        List apiVersions = new ArrayList<>(ApiVersionsResponse.defaultApiVersionsResponse().apiVersions());
-                        for (Iterator it = apiVersions.iterator(); it.hasNext(); ) {
-                            ApiVersion apiVersion = it.next();
-                            if (apiVersion.apiKey == ApiKeys.SASL_AUTHENTICATE.id) {
-                                it.remove();
-                                break;
+                        ApiVersionsResponse defaultApiVersionResponse = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE;
+                        ApiVersionsResponseKeyCollection apiVersions = new ApiVersionsResponseKeyCollection();
+                        for (ApiVersionsResponseKey apiVersion : defaultApiVersionResponse.data.apiKeys()) {
+                            if (apiVersion.apiKey() != ApiKeys.SASL_AUTHENTICATE.id) {
+                                // ApiVersionsResponseKey can NOT be reused in second ApiVersionsResponseKeyCollection
+                                // due to the internal pointers it contains.
+                                apiVersions.add(new ApiVersionsResponseKey()
+                                    .setApiKey(apiVersion.apiKey())
+                                    .setMinVersion(apiVersion.minVersion())
+                                    .setMaxVersion(apiVersion.maxVersion())
+                                );
                             }
+
                         }
-                        return new ApiVersionsResponse(0, Errors.NONE, apiVersions);
+                        ApiVersionsResponseData data = new ApiVersionsResponseData()
+                            .setErrorCode(Errors.NONE.code())
+                            .setThrottleTimeMs(0)
+                            .setApiKeys(apiVersions);
+                        return new ApiVersionsResponse(data);
                     }
 
                     @Override
@@ -1790,10 +1868,10 @@ private void testUnauthenticatedApiVersionsRequest(SecurityProtocol securityProt
 
         // Send ApiVersionsRequest and check response
         ApiVersionsResponse versionsResponse = sendVersionRequestReceiveResponse(node);
-        assertEquals(ApiKeys.SASL_HANDSHAKE.oldestVersion(), versionsResponse.apiVersion(ApiKeys.SASL_HANDSHAKE.id).minVersion);
-        assertEquals(ApiKeys.SASL_HANDSHAKE.latestVersion(), versionsResponse.apiVersion(ApiKeys.SASL_HANDSHAKE.id).maxVersion);
-        assertEquals(ApiKeys.SASL_AUTHENTICATE.oldestVersion(), versionsResponse.apiVersion(ApiKeys.SASL_AUTHENTICATE.id).minVersion);
-        assertEquals(ApiKeys.SASL_AUTHENTICATE.latestVersion(), versionsResponse.apiVersion(ApiKeys.SASL_AUTHENTICATE.id).maxVersion);
+        assertEquals(ApiKeys.SASL_HANDSHAKE.oldestVersion(), versionsResponse.apiVersion(ApiKeys.SASL_HANDSHAKE.id).minVersion());
+        assertEquals(ApiKeys.SASL_HANDSHAKE.latestVersion(), versionsResponse.apiVersion(ApiKeys.SASL_HANDSHAKE.id).maxVersion());
+        assertEquals(ApiKeys.SASL_AUTHENTICATE.oldestVersion(), versionsResponse.apiVersion(ApiKeys.SASL_AUTHENTICATE.id).minVersion());
+        assertEquals(ApiKeys.SASL_AUTHENTICATE.latestVersion(), versionsResponse.apiVersion(ApiKeys.SASL_AUTHENTICATE.id).maxVersion());
 
         // Send SaslHandshakeRequest and check response
         SaslHandshakeResponse handshakeResponse = sendHandshakeRequestReceiveResponse(node, saslHandshakeVersion);
@@ -1951,7 +2029,7 @@ private SaslHandshakeResponse sendHandshakeRequestReceiveResponse(String node, s
     private ApiVersionsResponse sendVersionRequestReceiveResponse(String node) throws Exception {
         ApiVersionsRequest handshakeRequest = createApiVersionsRequestV0();
         ApiVersionsResponse response =  (ApiVersionsResponse) sendKafkaRequestReceiveResponse(node, ApiKeys.API_VERSIONS, handshakeRequest);
-        assertEquals(Errors.NONE, response.error());
+        assertEquals(Errors.NONE.code(), response.data.errorCode());
         return response;
     }
 
diff --git a/clients/src/test/java/org/apache/kafka/test/MockSelector.java b/clients/src/test/java/org/apache/kafka/test/MockSelector.java
index 200d51115177c..9f3c432fa3461 100644
--- a/clients/src/test/java/org/apache/kafka/test/MockSelector.java
+++ b/clients/src/test/java/org/apache/kafka/test/MockSelector.java
@@ -41,6 +41,7 @@ public class MockSelector implements Selectable {
     private final Time time;
     private final List initiatedSends = new ArrayList<>();
     private final List completedSends = new ArrayList<>();
+    private final List completedSendBuffers = new ArrayList<>();
     private final List completedReceives = new ArrayList<>();
     private final Map disconnected = new HashMap<>();
     private final List connected = new ArrayList<>();
@@ -99,6 +100,7 @@ private void removeSendsForNode(String id, Collection sends) {
     public void clear() {
         this.completedSends.clear();
         this.completedReceives.clear();
+        this.completedSendBuffers.clear();
         this.disconnected.clear();
         this.connected.clear();
     }
@@ -129,6 +131,7 @@ private void completeSend(Send send) throws IOException {
                 send.writeTo(discardChannel);
             }
             completedSends.add(send);
+            completedSendBuffers.add(discardChannel);
         }
     }
 
@@ -154,6 +157,10 @@ public void completeSend(NetworkSend send) {
         this.completedSends.add(send);
     }
 
+    public List completedSendBuffers() {
+        return completedSendBuffers;
+    }
+
     @Override
     public List completedReceives() {
         return completedReceives;
diff --git a/core/src/main/scala/kafka/admin/BrokerApiVersionsCommand.scala b/core/src/main/scala/kafka/admin/BrokerApiVersionsCommand.scala
index f08385f090cb4..b97d858a2b113 100644
--- a/core/src/main/scala/kafka/admin/BrokerApiVersionsCommand.scala
+++ b/core/src/main/scala/kafka/admin/BrokerApiVersionsCommand.scala
@@ -36,10 +36,10 @@ import org.apache.kafka.common.internals.ClusterResourceListeners
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.network.Selector
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
-import org.apache.kafka.common.requests.ApiVersionsResponse.ApiVersion
 import org.apache.kafka.common.utils.LogContext
 import org.apache.kafka.common.utils.{KafkaThread, Time}
 import org.apache.kafka.common.Node
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKeyCollection
 import org.apache.kafka.common.requests.{AbstractRequest, AbstractResponse, ApiVersionsRequest, ApiVersionsResponse, MetadataRequest, MetadataResponse}
 
 import scala.collection.JavaConverters._
@@ -160,10 +160,10 @@ object BrokerApiVersionsCommand {
       throw new RuntimeException(s"Request $api failed on brokers $bootstrapBrokers")
     }
 
-    private def getApiVersions(node: Node): List[ApiVersion] = {
+    private def getApiVersions(node: Node): ApiVersionsResponseKeyCollection = {
       val response = send(node, ApiKeys.API_VERSIONS, new ApiVersionsRequest.Builder()).asInstanceOf[ApiVersionsResponse]
-      response.error.maybeThrow()
-      response.apiVersions.asScala.toList
+      Errors.forCode(response.data.errorCode()).maybeThrow()
+      response.data.apiKeys()
     }
 
     /**
@@ -189,7 +189,7 @@ object BrokerApiVersionsCommand {
 
     def listAllBrokerVersionInfo(): Map[Node, Try[NodeApiVersions]] =
       findAllBrokers().map { broker =>
-        broker -> Try[NodeApiVersions](new NodeApiVersions(getApiVersions(broker).asJava))
+        broker -> Try[NodeApiVersions](new NodeApiVersions(getApiVersions(broker)))
       }.toMap
 
     def close(): Unit = {
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index aa725930c71ea..c7747b60dd494 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -1611,6 +1611,8 @@ class KafkaApis(val requestChannel: RequestChannel,
       val apiVersionRequest = request.body[ApiVersionsRequest]
       if (apiVersionRequest.hasUnsupportedRequestVersion)
         apiVersionRequest.getErrorResponse(requestThrottleMs, Errors.UNSUPPORTED_VERSION.exception)
+      else if (!apiVersionRequest.isValid)
+        apiVersionRequest.getErrorResponse(requestThrottleMs, Errors.INVALID_REQUEST.exception)
       else
         ApiVersionsResponse.apiVersionsResponse(requestThrottleMs,
           config.interBrokerProtocolVersion.recordVersion.value)
diff --git a/core/src/test/scala/unit/kafka/server/ApiVersionsRequestTest.scala b/core/src/test/scala/unit/kafka/server/ApiVersionsRequestTest.scala
index 11ecfdef5a3a0..04e4f88d70798 100644
--- a/core/src/test/scala/unit/kafka/server/ApiVersionsRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ApiVersionsRequestTest.scala
@@ -17,8 +17,9 @@
 
 package kafka.server
 
+import org.apache.kafka.common.message.ApiVersionsRequestData
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKey
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
-import org.apache.kafka.common.requests.ApiVersionsResponse.ApiVersion
 import org.apache.kafka.common.requests.{ApiVersionsRequest, ApiVersionsResponse}
 import org.junit.Assert._
 import org.junit.Test
@@ -27,8 +28,8 @@ import scala.collection.JavaConverters._
 
 object ApiVersionsRequestTest {
   def validateApiVersionsResponse(apiVersionsResponse: ApiVersionsResponse): Unit = {
-    assertEquals("API keys in ApiVersionsResponse must match API keys supported by broker.", ApiKeys.values.length, apiVersionsResponse.apiVersions.size)
-    for (expectedApiVersion: ApiVersion <- ApiVersionsResponse.defaultApiVersionsResponse().apiVersions.asScala) {
+    assertEquals("API keys in ApiVersionsResponse must match API keys supported by broker.", ApiKeys.values.length, apiVersionsResponse.data.apiKeys().size())
+    for (expectedApiVersion: ApiVersionsResponseKey <- ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data.apiKeys().asScala) {
       val actualApiVersion = apiVersionsResponse.apiVersion(expectedApiVersion.apiKey)
       assertNotNull(s"API key ${actualApiVersion.apiKey} is supported by broker, but not received in ApiVersionsResponse.", actualApiVersion)
       assertEquals("API key must be supported by the broker.", expectedApiVersion.apiKey, actualApiVersion.apiKey)
@@ -50,9 +51,29 @@ class ApiVersionsRequestTest extends BaseRequestTest {
 
   @Test
   def testApiVersionsRequestWithUnsupportedVersion(): Unit = {
-    val apiVersionsRequest = new ApiVersionsRequest(0)
+    val apiVersionsRequest = new ApiVersionsRequest.Builder().build()
     val apiVersionsResponse = sendApiVersionsRequest(apiVersionsRequest, Some(Short.MaxValue), 0)
-    assertEquals(Errors.UNSUPPORTED_VERSION, apiVersionsResponse.error)
+    assertEquals(Errors.UNSUPPORTED_VERSION.code(), apiVersionsResponse.data.errorCode())
+    assertFalse(apiVersionsResponse.data.apiKeys().isEmpty)
+    val apiVersion = apiVersionsResponse.data.apiKeys().find(ApiKeys.API_VERSIONS.id)
+    assertEquals(ApiKeys.API_VERSIONS.id, apiVersion.apiKey())
+    assertEquals(ApiKeys.API_VERSIONS.oldestVersion(), apiVersion.minVersion())
+    assertEquals(ApiKeys.API_VERSIONS.latestVersion(), apiVersion.maxVersion())
+  }
+
+  @Test
+  def testApiVersionsRequestValidationV0(): Unit = {
+    val apiVersionsRequest = new ApiVersionsRequest.Builder().build( 0.asInstanceOf[Short])
+    val apiVersionsResponse = sendApiVersionsRequest(apiVersionsRequest, Some(0.asInstanceOf[Short]), 0)
+    ApiVersionsRequestTest.validateApiVersionsResponse(apiVersionsResponse)
+  }
+
+  @Test
+  def testApiVersionsRequestValidationV3(): Unit = {
+    // Invalid request because Name and Version are empty by default
+    val apiVersionsRequest = new ApiVersionsRequest(new ApiVersionsRequestData(), 3.asInstanceOf[Short])
+    val apiVersionsResponse = sendApiVersionsRequest(apiVersionsRequest, Some(3.asInstanceOf[Short]), 3)
+    assertEquals(Errors.INVALID_REQUEST.code(), apiVersionsResponse.data.errorCode())
   }
 
   private def sendApiVersionsRequest(request: ApiVersionsRequest, apiVersion: Option[Short] = None, responseVersion: Short = 1): ApiVersionsResponse = {
diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
index 4e0540b4f0585..f34eae45b1e4d 100644
--- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
+++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
@@ -358,7 +358,7 @@ class RequestQuotaTest extends BaseRequestTest {
           new SaslAuthenticateRequest.Builder(new SaslAuthenticateRequestData().setAuthBytes(new Array[Byte](0)))
 
         case ApiKeys.API_VERSIONS =>
-          new ApiVersionsRequest.Builder
+          new ApiVersionsRequest.Builder()
 
         case ApiKeys.CREATE_TOPICS => {
           new CreateTopicsRequest.Builder(
diff --git a/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala b/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala
index 675cc900805e9..b9510d483bd4f 100644
--- a/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala
@@ -68,7 +68,7 @@ class SaslApiVersionsRequestTest extends BaseRequestTest with SaslSetup {
     try {
       sendSaslHandshakeRequestValidateResponse(plaintextSocket)
       val response = sendApiVersionsRequest(plaintextSocket, new ApiVersionsRequest.Builder().build(0))
-      assertEquals(Errors.ILLEGAL_SASL_STATE, response.error)
+      assertEquals(Errors.ILLEGAL_SASL_STATE.code(), response.data.errorCode())
     } finally {
       plaintextSocket.close()
     }
@@ -78,9 +78,9 @@ class SaslApiVersionsRequestTest extends BaseRequestTest with SaslSetup {
   def testApiVersionsRequestWithUnsupportedVersion(): Unit = {
     val plaintextSocket = connect(protocol = securityProtocol)
     try {
-      val apiVersionsRequest = new ApiVersionsRequest(0)
+      val apiVersionsRequest = new ApiVersionsRequest.Builder().build(0)
       val apiVersionsResponse = sendApiVersionsRequest(plaintextSocket, apiVersionsRequest, Some(Short.MaxValue))
-      assertEquals(Errors.UNSUPPORTED_VERSION, apiVersionsResponse.error)
+      assertEquals(Errors.UNSUPPORTED_VERSION.code(), apiVersionsResponse.data.errorCode())
       val apiVersionsResponse2 = sendApiVersionsRequest(plaintextSocket, new ApiVersionsRequest.Builder().build(0))
       ApiVersionsRequestTest.validateApiVersionsResponse(apiVersionsResponse2)
       sendSaslHandshakeRequestValidateResponse(plaintextSocket)

From 586c587b3db0bd622d74a5d4da5734c50c698061 Mon Sep 17 00:00:00 2001
From: Magesh Nandakumar 
Date: Fri, 4 Oct 2019 11:12:32 -0700
Subject: [PATCH 0687/1071] KAFKA-8974: Trim whitespaces in topic names in sink
 connector configs (#7442)

Trim whitespaces in topic names specified in sink connector configs before subscribing to the consumer. Topic names don't allow whitespace characters, so trimming only will eliminate potential problems and will not place additional limits on topics specified in sink connectors.

Author: Magesh Nandakumar 
Reviewers: Arjun Satish , Randall Hauch 
---
 .../java/org/apache/kafka/connect/runtime/WorkerSinkTask.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
index 79760c0bf46ab..9a71a6658a3b3 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
@@ -287,6 +287,7 @@ protected void initializeAndStart() {
 
         if (SinkConnectorConfig.hasTopicsConfig(taskConfig)) {
             String[] topics = taskConfig.get(SinkTask.TOPICS_CONFIG).split(",");
+            Arrays.setAll(topics, i -> topics[i].trim());
             consumer.subscribe(Arrays.asList(topics), new HandleRebalance());
             log.debug("{} Initializing and starting task for topics {}", this, topics);
         } else {

From 3b05dc685b4bf7bafb8057b15c837fd5333789c5 Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Fri, 4 Oct 2019 15:58:11 -0700
Subject: [PATCH 0688/1071] MINOR: just remove leader on trunk like we did on
 2.3 (#7447)

Small follow-up to trunk PR #7423

While debugging the 2.3 VP PR we realized we should remove the leader-tracking from the VP system test altogether. We'd already merged the corresponding trunk PR so I made a quick new PR for trunk (also fixes a missed version bump in one of the log messages)

Reviewers: Guozhang Wang 
---
 .../tests/streams/streams_upgrade_test.py     | 44 ++-----------------
 1 file changed, 3 insertions(+), 41 deletions(-)

diff --git a/tests/kafkatest/tests/streams/streams_upgrade_test.py b/tests/kafkatest/tests/streams/streams_upgrade_test.py
index 45e20ce26c4ac..ab5709021b2ca 100644
--- a/tests/kafkatest/tests/streams/streams_upgrade_test.py
+++ b/tests/kafkatest/tests/streams/streams_upgrade_test.py
@@ -89,8 +89,6 @@ def __init__(self, test_context):
             'echo' : { 'partitions': 5 },
             'data' : { 'partitions': 5 },
         }
-        self.leader = None
-        self.leader_counter = {}
 
     processed_msg = "processed [0-9]* records"
     base_version_number = str(DEV_VERSION).split("-")[0]
@@ -317,13 +315,6 @@ def test_version_probing_upgrade(self):
         self.processors = [self.processor1, self.processor2, self.processor3]
         self.old_processors = [self.processor1, self.processor2, self.processor3]
         self.upgraded_processors = []
-        for p in self.processors:
-            self.leader_counter[p] = 2
-
-        self.update_leader()
-        for p in self.processors:
-            self.leader_counter[p] = 0
-        self.leader_counter[self.leader] = 3
 
         counter = 1
         current_generation = 3
@@ -348,25 +339,6 @@ def test_version_probing_upgrade(self):
                                    timeout_sec=60,
                                    err_msg="Never saw output 'UPGRADE-TEST-CLIENT-CLOSED' on" + str(node.account))
 
-    def update_leader(self):
-        self.leader = None
-        retries = 10
-        while retries > 0:
-            for p in self.processors:
-                found = list(p.node.account.ssh_capture("grep \"Finished assignment for group\" %s" % p.LOG_FILE, allow_fail=True))
-                if len(found) >= self.leader_counter[p] + 1:
-                    self.leader = p
-                    self.leader_counter[p] = self.leader_counter[p] + 1
-
-            if self.leader is None:
-                retries = retries - 1
-                time.sleep(5)
-            else:
-                break
-
-        if self.leader is None:
-            raise Exception("Could not identify leader")
-
     def get_version_string(self, version):
         if version.startswith("0") or version.startswith("1") \
           or version.startswith("2.0") or version.startswith("2.1"):
@@ -533,7 +505,6 @@ def do_rolling_bounce(self, processor, counter, current_generation):
                 node.account.ssh("mv " + processor.STDOUT_FILE + " " + processor.STDOUT_FILE + "." + str(counter), allow_fail=False)
                 node.account.ssh("mv " + processor.STDERR_FILE + " " + processor.STDERR_FILE + "." + str(counter), allow_fail=False)
                 node.account.ssh("mv " + processor.LOG_FILE + " " + processor.LOG_FILE + "." + str(counter), allow_fail=False)
-                self.leader_counter[processor] = 0
 
                 with node.account.monitor_log(processor.LOG_FILE) as log_monitor:
                     processor.set_upgrade_to("future_version")
@@ -550,11 +521,6 @@ def do_rolling_bounce(self, processor, counter, current_generation):
                                            timeout_sec=60,
                                            err_msg="Could not detect FutureStreamsPartitionAssignor in " + str(node.account))
 
-                    if processor == self.leader:
-                        self.update_leader()
-                    else:
-                        self.leader_counter[self.leader] = self.leader_counter[self.leader] + 1
-
                     monitors = {}
                     monitors[processor] = log_monitor
                     monitors[first_other_processor] = first_other_monitor
@@ -609,12 +575,8 @@ def do_rolling_bounce(self, processor, counter, current_generation):
                     if generation_synchronized == False:
                         raise Exception("Never saw all three processors have the synchronized generation number")
 
-                    if processor == self.leader:
-                        self.update_leader()
-                    else:
-                        self.leader_counter[self.leader] = self.leader_counter[self.leader] + 1
 
-                    if self.leader in self.old_processors or len(self.old_processors) > 0:
+                    if len(self.old_processors) > 0:
                         self.verify_metadata_no_upgraded_yet()
 
         return current_generation
@@ -624,9 +586,9 @@ def extract_highest_generation(self, found_generations):
 
     def verify_metadata_no_upgraded_yet(self):
         for p in self.processors:
-            found = list(p.node.account.ssh_capture("grep \"Sent a version 4 subscription and group leader.s latest supported version is 5. Upgrading subscription metadata version to 5 for next rebalance.\" " + p.LOG_FILE, allow_fail=True))
+            found = list(p.node.account.ssh_capture("grep \"Sent a version 5 subscription and group.s latest commonly supported version is 6 (successful version probing and end of rolling upgrade). Upgrading subscription metadata version to 6 for next rebalance.\" " + p.LOG_FILE, allow_fail=True))
             if len(found) > 0:
-                raise Exception("Kafka Streams failed with 'group member upgraded to metadata 4 too early'")
+                raise Exception("Kafka Streams failed with 'group member upgraded to metadata 6 too early'")
 
     def confirm_topics_on_all_brokers(self, expected_topic_set):
         for node in self.kafka.nodes:

From 52007e878aaac3f48d0d949dbe428a2ae5e56f57 Mon Sep 17 00:00:00 2001
From: Bruno Cadonna 
Date: Sat, 5 Oct 2019 02:07:30 +0200
Subject: [PATCH 0689/1071] KAFKA-8934: Introduce instance-level metrics for
 streams applications (#7416)

1. Moves StreamsMetricsImpl from StreamThread to KafkaStreams
2. Adds instance-level metrics as specified in KIP-444, i.e.:
-- version
-- commit-id
-- application-id
-- topology-description
-- state

Reviewers: Guozhang Wang , John Roesler , Bill Bejeck 
---
 .../apache/kafka/streams/KafkaStreams.java    |  22 +-
 .../internals/metrics/ClientMetrics.java      | 116 ++++++++
 .../kstream/internals/KStreamAggregate.java   |   2 +-
 .../kstream/internals/KStreamKStreamJoin.java |   2 +-
 .../internals/KStreamKTableJoinProcessor.java |   2 +-
 .../kstream/internals/KStreamReduce.java      |   2 +-
 .../KStreamSessionWindowAggregate.java        |   2 +-
 .../internals/KStreamWindowAggregate.java     |   2 +-
 .../internals/KTableKTableInnerJoin.java      |   2 +-
 .../internals/KTableKTableLeftJoin.java       |   2 +-
 .../internals/KTableKTableOuterJoin.java      |   2 +-
 .../internals/KTableKTableRightJoin.java      |   2 +-
 .../kstream/internals/KTableSource.java       |   2 +-
 ...eignJoinSubscriptionProcessorSupplier.java |   3 +-
 ...criptionStoreReceiveProcessorSupplier.java |   2 +-
 .../kstream/internals/metrics/Sensors.java    |  26 +-
 .../internals/GlobalStateUpdateTask.java      |   2 +-
 .../internals/GlobalStreamThread.java         |  20 +-
 .../processor/internals/ProcessorNode.java    |  48 ++-
 .../processor/internals/RecordQueue.java      |   3 +-
 .../processor/internals/StandbyTask.java      |   3 +-
 .../processor/internals/StreamTask.java       |  33 ++-
 .../processor/internals/StreamThread.java     |  64 ++--
 .../internals/metrics/StreamsMetricsImpl.java | 273 +++++++++++++-----
 .../internals/metrics/ThreadMetrics.java      | 170 ++++++-----
 .../AbstractRocksDBSegmentedBytesStore.java   |   4 +-
 .../state/internals/InMemorySessionStore.java |   4 +-
 .../state/internals/InMemoryWindowStore.java  |   4 +-
 .../state/internals/KeyValueSegments.java     |   2 +-
 .../state/internals/MeteredKeyValueStore.java | 119 +++++++-
 .../state/internals/MeteredSessionStore.java  |  70 ++++-
 .../state/internals/MeteredWindowStore.java   |  58 +++-
 .../streams/state/internals/NamedCache.java   |   9 +-
 .../streams/state/internals/RocksDBStore.java |   2 +-
 .../state/internals/TimestampedSegments.java  |   2 +-
 .../internals/metrics/NamedCacheMetrics.java  |   8 +-
 .../internals/metrics/RocksDBMetrics.java     | 159 +++++++---
 .../metrics/RocksDBMetricsRecorder.java       |   9 +-
 .../state/internals/metrics/Sensors.java      |  18 +-
 .../kafka/streams/KafkaStreamsTest.java       |  36 ++-
 .../integration/MetricsIntegrationTest.java   |  57 +++-
 .../internals/metrics/ClientMetricsTest.java  | 137 +++++++++
 ...amSessionWindowAggregateProcessorTest.java |  11 +-
 .../internals/KStreamWindowAggregateTest.java |   9 +-
 .../KTableSuppressProcessorMetricsTest.java   |  65 +++--
 .../internals/GlobalStreamThreadTest.java     |  11 +-
 .../internals/ProcessorNodeTest.java          |   7 +-
 .../processor/internals/StandbyTaskTest.java  |   3 +-
 .../processor/internals/StreamTaskTest.java   |  20 +-
 .../processor/internals/StreamThreadTest.java | 190 ++++++------
 .../metrics/StreamsMetricsImplTest.java       | 237 +++++++++++++--
 .../internals/metrics/ThreadMetricsTest.java  |  49 ++--
 ...bstractRocksDBSegmentedBytesStoreTest.java |   5 +-
 .../state/internals/KeyValueSegmentTest.java  |   3 +-
 .../internals/MeteredKeyValueStoreTest.java   |   6 +-
 .../internals/MeteredSessionStoreTest.java    |   7 +-
 .../MeteredTimestampedKeyValueStoreTest.java  |   7 +-
 .../internals/MeteredWindowStoreTest.java     |   5 +-
 .../state/internals/SegmentIteratorTest.java  |   3 +-
 .../internals/SessionBytesStoreTest.java      |   5 +-
 .../internals/TimestampedSegmentTest.java     |   3 +-
 .../state/internals/WindowBytesStoreTest.java |   5 +-
 .../metrics/NamedCacheMetricsTest.java        |  16 +-
 .../metrics/RocksDBMetricsRecorderTest.java   |   5 +-
 .../internals/metrics/RocksDBMetricsTest.java |  21 +-
 .../kafka/streams/TopologyTestDriver.java     |  11 +-
 .../processor/MockProcessorContext.java       |   6 +-
 67 files changed, 1625 insertions(+), 590 deletions(-)
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/internals/metrics/ClientMetrics.java
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/internals/metrics/ClientMetricsTest.java

diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
index b9bc586c096e1..5a00fe39f6cad 100644
--- a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
+++ b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
@@ -37,6 +37,7 @@
 import org.apache.kafka.streams.errors.ProcessorStateException;
 import org.apache.kafka.streams.errors.StreamsException;
 import org.apache.kafka.streams.internals.ApiUtils;
+import org.apache.kafka.streams.internals.metrics.ClientMetrics;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KTable;
 import org.apache.kafka.streams.kstream.Produced;
@@ -53,6 +54,7 @@
 import org.apache.kafka.streams.processor.internals.StreamThread;
 import org.apache.kafka.streams.processor.internals.StreamsMetadataState;
 import org.apache.kafka.streams.processor.internals.ThreadStateTransitionValidator;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.apache.kafka.streams.state.HostInfo;
 import org.apache.kafka.streams.state.QueryableStoreType;
 import org.apache.kafka.streams.state.StreamsMetadata;
@@ -144,6 +146,7 @@ public class KafkaStreams implements AutoCloseable {
     private final ScheduledExecutorService rocksDBMetricsRecordingService;
     private final QueryableStoreProvider queryableStoreProvider;
     private final Admin adminClient;
+    private final StreamsMetricsImpl streamsMetrics;
 
     GlobalStreamThread globalStreamThread;
     private KafkaStreams.StateListener stateListener;
@@ -672,6 +675,16 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder,
                 Collections.singletonMap(StreamsConfig.CLIENT_ID_CONFIG, clientId));
         reporters.add(new JmxReporter(JMX_PREFIX));
         metrics = new Metrics(metricConfig, reporters, time);
+        streamsMetrics =
+            new StreamsMetricsImpl(metrics, clientId, config.getString(StreamsConfig.BUILT_IN_METRICS_VERSION_CONFIG));
+        streamsMetrics.setRocksDBMetricsRecordingTrigger(rocksDBMetricsRecordingTrigger);
+        ClientMetrics.addVersionMetric(streamsMetrics);
+        ClientMetrics.addCommitIdMetric(streamsMetrics);
+        ClientMetrics.addApplicationIdMetric(streamsMetrics, config.getString(StreamsConfig.APPLICATION_ID_CONFIG));
+        ClientMetrics.addTopologyDescriptionMetric(streamsMetrics, internalTopologyBuilder.describe().toString());
+        ClientMetrics.addStateMetric(streamsMetrics, (metricsConfig, now) -> state);
+        log.info("Kafka Streams version: {}", ClientMetrics.version());
+        log.info("Kafka Streams commit ID: {}", ClientMetrics.commitId());
 
         // re-write the physical topology according to the config
         internalTopologyBuilder.rewriteTopology(config);
@@ -712,11 +725,10 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder,
                 clientSupplier.getGlobalConsumer(config.getGlobalConsumerConfigs(clientId)),
                 stateDirectory,
                 cacheSizePerThread,
-                metrics,
+                streamsMetrics,
                 time,
                 globalThreadId,
-                delegatingStateRestoreListener,
-                rocksDBMetricsRecordingTrigger
+                delegatingStateRestoreListener
             );
             globalThreadState = globalStreamThread.state();
         }
@@ -734,14 +746,13 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder,
                 adminClient,
                 processId,
                 clientId,
-                metrics,
+                streamsMetrics,
                 time,
                 streamsMetadataState,
                 cacheSizePerThread,
                 stateDirectory,
                 delegatingStateRestoreListener,
                 i + 1);
-            threads[i].setRocksDBMetricsRecordingTrigger(rocksDBMetricsRecordingTrigger);
             threadState.put(threads[i].getId(), threads[i].state());
             storeProviders.add(new StreamThreadStateStoreProvider(threads[i]));
         }
@@ -928,6 +939,7 @@ private boolean close(final long timeoutMs) {
 
                 adminClient.close();
 
+                streamsMetrics.removeAllClientLevelMetrics();
                 metrics.close();
                 setState(State.NOT_RUNNING);
             }, "kafka-streams-close-thread");
diff --git a/streams/src/main/java/org/apache/kafka/streams/internals/metrics/ClientMetrics.java b/streams/src/main/java/org/apache/kafka/streams/internals/metrics/ClientMetrics.java
new file mode 100644
index 0000000000000..564e06241883d
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/internals/metrics/ClientMetrics.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.internals.metrics;
+
+import org.apache.kafka.common.metrics.Gauge;
+import org.apache.kafka.common.metrics.Sensor.RecordingLevel;
+import org.apache.kafka.streams.KafkaStreams.State;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.InputStream;
+import java.util.Properties;
+
+public class ClientMetrics {
+    private ClientMetrics() {}
+
+    private static final Logger log = LoggerFactory.getLogger(ClientMetrics.class);
+    private static final String VERSION = "version";
+    private static final String COMMIT_ID = "commit-id";
+    private static final String APPLICATION_ID = "application-id";
+    private static final String TOPOLOGY_DESCRIPTION = "topology-description";
+    private static final String STATE = "state";
+    private static final String VERSION_FROM_FILE;
+    private static final String COMMIT_ID_FROM_FILE;
+    private static final String DEFAULT_VALUE = "unknown";
+
+    static {
+        final Properties props = new Properties();
+        try (InputStream resourceStream = ClientMetrics.class.getResourceAsStream(
+            "/kafka/kafka-streams-version.properties")) {
+
+            props.load(resourceStream);
+        } catch (final Exception exception) {
+            log.warn("Error while loading kafka-streams-version.properties", exception);
+        }
+        VERSION_FROM_FILE = props.getProperty("version", DEFAULT_VALUE).trim();
+        COMMIT_ID_FROM_FILE = props.getProperty("commitId", DEFAULT_VALUE).trim();
+    }
+
+    private static final String VERSION_DESCRIPTION = "The version of the Kafka Streams client";
+    private static final String COMMIT_ID_DESCRIPTION = "The version control commit ID of the Kafka Streams client";
+    private static final String APPLICATION_ID_DESCRIPTION = "The application ID of the Kafka Streams client";
+    private static final String TOPOLOGY_DESCRIPTION_DESCRIPTION =
+        "The description of the topology executed in the Kafka Streams client";
+    private static final String STATE_DESCRIPTION = "The state of the Kafka Streams client";
+
+    public static String version() {
+        return VERSION_FROM_FILE;
+    }
+
+    public static String commitId() {
+        return COMMIT_ID_FROM_FILE;
+    }
+
+    public static void addVersionMetric(final StreamsMetricsImpl streamsMetrics) {
+        streamsMetrics.addClientLevelImmutableMetric(
+            VERSION,
+            VERSION_DESCRIPTION,
+            RecordingLevel.INFO,
+            VERSION_FROM_FILE
+        );
+    }
+
+    public static void addCommitIdMetric(final StreamsMetricsImpl streamsMetrics) {
+        streamsMetrics.addClientLevelImmutableMetric(
+            COMMIT_ID,
+            COMMIT_ID_DESCRIPTION,
+            RecordingLevel.INFO,
+            COMMIT_ID_FROM_FILE
+        );
+    }
+
+    public static void addApplicationIdMetric(final StreamsMetricsImpl streamsMetrics, final String applicationId) {
+        streamsMetrics.addClientLevelImmutableMetric(
+            APPLICATION_ID,
+            APPLICATION_ID_DESCRIPTION,
+            RecordingLevel.INFO,
+            applicationId
+        );
+    }
+
+    public static void addTopologyDescriptionMetric(final StreamsMetricsImpl streamsMetrics,
+                                                    final String topologyDescription) {
+        streamsMetrics.addClientLevelImmutableMetric(
+            TOPOLOGY_DESCRIPTION,
+            TOPOLOGY_DESCRIPTION_DESCRIPTION,
+            RecordingLevel.INFO,
+            topologyDescription
+        );
+    }
+
+    public static void addStateMetric(final StreamsMetricsImpl streamsMetrics,
+                                      final Gauge stateProvider) {
+        streamsMetrics.addClientLevelMutableMetric(
+            STATE,
+            STATE_DESCRIPTION,
+            RecordingLevel.INFO,
+            stateProvider
+        );
+    }
+}
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamAggregate.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamAggregate.java
index ca7266f5aba35..7a0d1409466aa 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamAggregate.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamAggregate.java
@@ -67,7 +67,7 @@ private class KStreamAggregateProcessor extends AbstractProcessor {
         public void init(final ProcessorContext context) {
             super.init(context);
             metrics = (StreamsMetricsImpl) context.metrics();
-            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(metrics);
+            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(Thread.currentThread().getName(), metrics);
             store = (TimestampedKeyValueStore) context.getStateStore(storeName);
             tupleForwarder = new TimestampedTupleForwarder<>(
                 store,
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoin.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoin.java
index 97debbc802195..8f22b08e2f5c7 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoin.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoin.java
@@ -65,7 +65,7 @@ private class KStreamKStreamJoinProcessor extends AbstractProcessor {
         public void init(final ProcessorContext context) {
             super.init(context);
             metrics = (StreamsMetricsImpl) context.metrics();
-            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(metrics);
+            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(Thread.currentThread().getName(), metrics);
 
             otherWindow = (WindowStore) context.getStateStore(otherWindowName);
         }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamKTableJoinProcessor.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamKTableJoinProcessor.java
index 92fd4d52e5ae2..3123d770ccb20 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamKTableJoinProcessor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamKTableJoinProcessor.java
@@ -52,7 +52,7 @@ class KStreamKTableJoinProcessor extends AbstractProcessor {
         public void init(final ProcessorContext context) {
             super.init(context);
             metrics = (StreamsMetricsImpl) context.metrics();
-            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(metrics);
+            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(Thread.currentThread().getName(), metrics);
             store = (TimestampedKeyValueStore) context.getStateStore(storeName);
             tupleForwarder = new TimestampedTupleForwarder<>(
                 store,
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregate.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregate.java
index cb117b70ec92c..57557014a8fc5 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregate.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregate.java
@@ -94,7 +94,7 @@ public void init(final ProcessorContext context) {
             internalProcessorContext = (InternalProcessorContext) context;
             metrics = (StreamsMetricsImpl) context.metrics();
             lateRecordDropSensor = Sensors.lateRecordDropSensor(internalProcessorContext);
-            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(metrics);
+            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(Thread.currentThread().getName(), metrics);
 
             store = (SessionStore) context.getStateStore(storeName);
             tupleForwarder = new SessionTupleForwarder<>(store, context, new SessionCacheFlushListener<>(context), sendOldValues);
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregate.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregate.java
index 2983a3a9c7ea9..923237626c40f 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregate.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregate.java
@@ -92,7 +92,7 @@ public void init(final ProcessorContext context) {
             metrics = internalProcessorContext.metrics();
 
             lateRecordDropSensor = Sensors.lateRecordDropSensor(internalProcessorContext);
-            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(metrics);
+            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(Thread.currentThread().getName(), metrics);
             windowStore = (TimestampedWindowStore) context.getStateStore(storeName);
             tupleForwarder = new TimestampedTupleForwarder<>(
                 windowStore,
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableInnerJoin.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableInnerJoin.java
index 005ea809b3fe7..561fa4e30f006 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableInnerJoin.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableInnerJoin.java
@@ -78,7 +78,7 @@ private class KTableKTableJoinProcessor extends AbstractProcessor>
         public void init(final ProcessorContext context) {
             super.init(context);
             metrics = (StreamsMetricsImpl) context.metrics();
-            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(metrics);
+            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(Thread.currentThread().getName(), metrics);
             valueGetter.init(context);
         }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoin.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoin.java
index 4bd6af99d6db5..ff88bd6bb7f6b 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoin.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoin.java
@@ -77,7 +77,7 @@ private class KTableKTableLeftJoinProcessor extends AbstractProcessor {
         public void init(final ProcessorContext context) {
             super.init(context);
             metrics = (StreamsMetricsImpl) context.metrics();
-            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(metrics);
+            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(Thread.currentThread().getName(), metrics);
             if (queryableName != null) {
                 store = (TimestampedKeyValueStore) context.getStateStore(queryableName);
                 tupleForwarder = new TimestampedTupleForwarder<>(
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionProcessorSupplier.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionProcessorSupplier.java
index 614b91f071e5e..521e2f0ccaef5 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionProcessorSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionProcessorSupplier.java
@@ -64,7 +64,8 @@ private final class KTableKTableJoinProcessor extends AbstractProcessor tags = metrics.tagMap(
-            "task-id", context.taskId().toString()
+            threadId,
+            "task-id",
+            context.taskId().toString()
         );
         sensor.add(
             new MetricName(
@@ -86,8 +99,10 @@ public static Sensor recordLatenessSensor(final InternalProcessorContext context
 
     public static Sensor suppressionEmitSensor(final InternalProcessorContext context) {
         final StreamsMetricsImpl metrics = context.metrics();
+        final String threadId = Thread.currentThread().getName();
 
         final Sensor sensor = metrics.nodeLevelSensor(
+            threadId,
             context.taskId().toString(),
             context.currentNode().name(),
             "suppression-emit",
@@ -95,8 +110,11 @@ public static Sensor suppressionEmitSensor(final InternalProcessorContext contex
         );
 
         final Map tags = metrics.tagMap(
-            "task-id", context.taskId().toString(),
-            PROCESSOR_NODE_ID_TAG, context.currentNode().name()
+            threadId,
+            "task-id",
+            context.taskId().toString(),
+            PROCESSOR_NODE_ID_TAG,
+            context.currentNode().name()
         );
 
         sensor.add(
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStateUpdateTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStateUpdateTask.java
index d6f60e351334e..4d27e82ba2a59 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStateUpdateTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStateUpdateTask.java
@@ -70,7 +70,7 @@ public Map initialize() {
                     source,
                     deserializationExceptionHandler,
                     logContext,
-                    ThreadMetrics.skipRecordSensor(processorContext.metrics())
+                    ThreadMetrics.skipRecordSensor(Thread.currentThread().getName(), processorContext.metrics())
                 )
             );
         }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStreamThread.java
index 5248aa4f8ab9c..923480fa5b479 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStreamThread.java
@@ -23,7 +23,6 @@
 import org.apache.kafka.common.Metric;
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.common.utils.Utils;
@@ -33,7 +32,6 @@
 import org.apache.kafka.streams.processor.StateRestoreListener;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.apache.kafka.streams.state.internals.ThreadCache;
-import org.apache.kafka.streams.state.internals.metrics.RocksDBMetricsRecordingTrigger;
 import org.slf4j.Logger;
 
 import java.io.IOException;
@@ -180,27 +178,21 @@ public GlobalStreamThread(final ProcessorTopology topology,
                               final Consumer globalConsumer,
                               final StateDirectory stateDirectory,
                               final long cacheSizeBytes,
-                              final Metrics metrics,
+                              final StreamsMetricsImpl streamsMetrics,
                               final Time time,
                               final String threadClientId,
-                              final StateRestoreListener stateRestoreListener,
-                              final RocksDBMetricsRecordingTrigger rocksDBMetricsRecordingTrigger) {
+                              final StateRestoreListener stateRestoreListener) {
         super(threadClientId);
         this.time = time;
         this.config = config;
         this.topology = topology;
         this.globalConsumer = globalConsumer;
         this.stateDirectory = stateDirectory;
-        streamsMetrics = new StreamsMetricsImpl(
-            metrics,
-            threadClientId,
-            config.getString(StreamsConfig.BUILT_IN_METRICS_VERSION_CONFIG)
-        );
-        streamsMetrics.setRocksDBMetricsRecordingTrigger(rocksDBMetricsRecordingTrigger);
+        this.streamsMetrics = streamsMetrics;
         this.logPrefix = String.format("global-stream-thread [%s] ", threadClientId);
         this.logContext = new LogContext(logPrefix);
         this.log = logContext.logger(getClass());
-        this.cache = new ThreadCache(logContext, cacheSizeBytes, streamsMetrics);
+        this.cache = new ThreadCache(logContext, cacheSizeBytes, this.streamsMetrics);
         this.stateRestoreListener = stateRestoreListener;
     }
 
@@ -286,7 +278,7 @@ public void run() {
             setState(State.DEAD);
 
             log.warn("Error happened during initialization of the global state store; this thread has shutdown");
-            streamsMetrics.removeAllThreadLevelSensors();
+            streamsMetrics.removeAllThreadLevelSensors(getName());
 
             return;
         }
@@ -310,7 +302,7 @@ public void run() {
                 log.error("Failed to close state maintainer due to the following error:", e);
             }
 
-            streamsMetrics.removeAllThreadLevelSensors();
+            streamsMetrics.removeAllThreadLevelSensors(getName());
 
             setState(DEAD);
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorNode.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorNode.java
index bc66edee307c7..6e10e83830056 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorNode.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorNode.java
@@ -34,6 +34,7 @@
 import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.PROCESSOR_NODE_ID_TAG;
 import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.PROCESSOR_NODE_METRICS_GROUP;
 import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addAvgAndMaxLatencyToSensor;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addInvocationRateAndCountToSensor;
 
 public class ProcessorNode {
 
@@ -166,13 +167,27 @@ private static final class NodeMetrics {
         private NodeMetrics(final StreamsMetricsImpl metrics, final String processorNodeName, final ProcessorContext context) {
             this.metrics = metrics;
 
+            final String threadId = Thread.currentThread().getName();
             final String taskName = context.taskId().toString();
-            final Map tagMap = metrics.tagMap("task-id", context.taskId().toString(), PROCESSOR_NODE_ID_TAG, processorNodeName);
-            final Map allTagMap = metrics.tagMap("task-id", context.taskId().toString(), PROCESSOR_NODE_ID_TAG, "all");
+            final Map tagMap = metrics.tagMap(
+                threadId,
+                "task-id",
+                context.taskId().toString(),
+                PROCESSOR_NODE_ID_TAG,
+                processorNodeName
+            );
+            final Map allTagMap = metrics.tagMap(
+                threadId,
+                "task-id",
+                context.taskId().toString(),
+                PROCESSOR_NODE_ID_TAG,
+                "all"
+            );
 
             nodeProcessTimeSensor = createTaskAndNodeLatencyAndThroughputSensors(
                 "process",
                 metrics,
+                threadId,
                 taskName,
                 processorNodeName,
                 allTagMap,
@@ -182,6 +197,7 @@ private NodeMetrics(final StreamsMetricsImpl metrics, final String processorNode
             nodePunctuateTimeSensor = createTaskAndNodeLatencyAndThroughputSensors(
                 "punctuate",
                 metrics,
+                threadId,
                 taskName,
                 processorNodeName,
                 allTagMap,
@@ -191,6 +207,7 @@ private NodeMetrics(final StreamsMetricsImpl metrics, final String processorNode
             nodeCreationSensor = createTaskAndNodeLatencyAndThroughputSensors(
                 "create",
                 metrics,
+                threadId,
                 taskName,
                 processorNodeName,
                 allTagMap,
@@ -201,6 +218,7 @@ private NodeMetrics(final StreamsMetricsImpl metrics, final String processorNode
             nodeDestructionSensor = createTaskAndNodeLatencyAndThroughputSensors(
                 "destroy",
                 metrics,
+                threadId,
                 taskName,
                 processorNodeName,
                 allTagMap,
@@ -210,6 +228,7 @@ private NodeMetrics(final StreamsMetricsImpl metrics, final String processorNode
             sourceNodeForwardSensor = createTaskAndNodeLatencyAndThroughputSensors(
                 "forward",
                 metrics,
+                threadId,
                 taskName,
                 processorNodeName,
                 allTagMap,
@@ -221,24 +240,35 @@ private NodeMetrics(final StreamsMetricsImpl metrics, final String processorNode
         }
 
         private void removeAllSensors() {
-            metrics.removeAllNodeLevelSensors(taskName, processorNodeName);
+            metrics.removeAllNodeLevelSensors(Thread.currentThread().getName(), taskName, processorNodeName);
         }
 
         private static Sensor createTaskAndNodeLatencyAndThroughputSensors(final String operation,
                                                                            final StreamsMetricsImpl metrics,
+                                                                           final String threadId,
                                                                            final String taskName,
                                                                            final String processorNodeName,
                                                                            final Map taskTags,
                                                                            final Map nodeTags) {
-            final Sensor parent = metrics.taskLevelSensor(taskName, operation, Sensor.RecordingLevel.DEBUG);
+            final Sensor parent = metrics.taskLevelSensor(
+                threadId,
+                taskName,
+                operation,
+                Sensor.RecordingLevel.DEBUG
+            );
             addAvgAndMaxLatencyToSensor(parent, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation);
-            StreamsMetricsImpl
-                .addInvocationRateAndCountToSensor(parent, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation);
+            addInvocationRateAndCountToSensor(parent, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation);
 
-            final Sensor sensor = metrics.nodeLevelSensor(taskName, processorNodeName, operation, Sensor.RecordingLevel.DEBUG, parent);
+            final Sensor sensor = metrics.nodeLevelSensor(
+                threadId,
+                taskName,
+                processorNodeName,
+                operation,
+                Sensor.RecordingLevel.DEBUG,
+                parent
+            );
             addAvgAndMaxLatencyToSensor(sensor, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation);
-            StreamsMetricsImpl
-                .addInvocationRateAndCountToSensor(sensor, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation);
+            addInvocationRateAndCountToSensor(sensor, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation);
 
             return sensor;
         }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordQueue.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordQueue.java
index 70eb770a283bf..05849c57fd8c5 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordQueue.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordQueue.java
@@ -62,7 +62,8 @@ public class RecordQueue {
         this.fifoQueue = new ArrayDeque<>();
         this.timestampExtractor = timestampExtractor;
         this.processorContext = processorContext;
-        skipRecordsSensor = ThreadMetrics.skipRecordSensor(processorContext.metrics());
+        skipRecordsSensor =
+            ThreadMetrics.skipRecordSensor(Thread.currentThread().getName(), processorContext.metrics());
         recordDeserializer = new RecordDeserializer(
             source,
             deserializationExceptionHandler,
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
index 071b7d667bbbc..f10c25bc5a737 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
@@ -63,7 +63,8 @@ public class StandbyTask extends AbstractTask {
                 final StateDirectory stateDirectory) {
         super(id, partitions, topology, consumer, changelogReader, true, stateDirectory, config);
 
-        closeTaskSensor = metrics.threadLevelSensor("task-closed", Sensor.RecordingLevel.INFO);
+        closeTaskSensor = metrics
+            .threadLevelSensor(Thread.currentThread().getName(), "task-closed", Sensor.RecordingLevel.INFO);
         processorContext = new StandbyContextImpl(id, config, stateMgr, metrics);
 
         final Set changelogTopicNames = new HashSet<>(topology.storeToChangelogTopic().values());
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
index ccf522813f362..40466b388706d 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
@@ -96,19 +96,25 @@ protected static final class TaskMetrics {
         final StreamsMetricsImpl metrics;
         final Sensor taskCommitTimeSensor;
         final Sensor taskEnforcedProcessSensor;
+        private final String threadId;
         private final String taskName;
 
-        TaskMetrics(final TaskId id, final StreamsMetricsImpl metrics) {
-            taskName = id.toString();
+        TaskMetrics(final String threadId,
+                    final TaskId taskId,
+                    final StreamsMetricsImpl metrics) {
+            this.threadId = threadId;
+            taskName = taskId.toString();
             this.metrics = metrics;
             final String group = "stream-task-metrics";
 
             // first add the global operation metrics if not yet, with the global tags only
-            final Sensor parent = ThreadMetrics.commitOverTasksSensor(metrics);
+            final Sensor parent = ThreadMetrics.commitOverTasksSensor(threadId, metrics);
 
             // add the operation metrics with additional tags
-            final Map tagMap = metrics.tagMap("task-id", taskName);
-            taskCommitTimeSensor = metrics.taskLevelSensor(taskName, "commit", Sensor.RecordingLevel.DEBUG, parent);
+            final Map tagMap =
+                metrics.tagMap(Thread.currentThread().getName(), "task-id", taskName);
+            taskCommitTimeSensor =
+                metrics.taskLevelSensor(threadId, taskName, "commit", Sensor.RecordingLevel.DEBUG, parent);
             taskCommitTimeSensor.add(
                 new MetricName("commit-latency-avg", group, "The average latency of commit operation.", tagMap),
                 new Avg()
@@ -127,7 +133,13 @@ protected static final class TaskMetrics {
             );
 
             // add the metrics for enforced processing
-            taskEnforcedProcessSensor = metrics.taskLevelSensor(taskName, "enforced-processing", Sensor.RecordingLevel.DEBUG, parent);
+            taskEnforcedProcessSensor = metrics.taskLevelSensor(
+                threadId,
+                taskName,
+                "enforced-processing",
+                Sensor.RecordingLevel.DEBUG,
+                parent
+            );
             taskEnforcedProcessSensor.add(
                     new MetricName("enforced-processing-rate", group, "The average number of occurrence of enforced-processing operation per second.", tagMap),
                     new Rate(TimeUnit.SECONDS, new WindowedCount())
@@ -140,7 +152,7 @@ protected static final class TaskMetrics {
         }
 
         void removeAllSensors() {
-            metrics.removeAllTaskLevelSensors(taskName);
+            metrics.removeAllTaskLevelSensors(threadId, taskName);
         }
     }
 
@@ -179,9 +191,10 @@ public StreamTask(final TaskId id,
         this.time = time;
         this.producerSupplier = producerSupplier;
         this.producer = producerSupplier.get();
-        this.taskMetrics = new TaskMetrics(id, streamsMetrics);
+        final String threadId = Thread.currentThread().getName();
+        this.taskMetrics = new TaskMetrics(threadId, id, streamsMetrics);
 
-        closeTaskSensor = ThreadMetrics.closeTaskSensor(streamsMetrics);
+        closeTaskSensor = ThreadMetrics.closeTaskSensor(threadId, streamsMetrics);
 
         final ProductionExceptionHandler productionExceptionHandler = config.defaultProductionExceptionHandler();
 
@@ -190,7 +203,7 @@ public StreamTask(final TaskId id,
                 id.toString(),
                 logContext,
                 productionExceptionHandler,
-                ThreadMetrics.skipRecordSensor(streamsMetrics));
+                ThreadMetrics.skipRecordSensor(threadId, streamsMetrics));
         } else {
             this.recordCollector = recordCollector;
         }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index e33c88724b0d0..29e1bc7050ee4 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -29,7 +29,6 @@
 import org.apache.kafka.common.Metric;
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.serialization.ByteArrayDeserializer;
 import org.apache.kafka.common.utils.LogContext;
@@ -46,7 +45,6 @@
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.apache.kafka.streams.state.internals.ThreadCache;
-import org.apache.kafka.streams.state.internals.metrics.RocksDBMetricsRecordingTrigger;
 import org.slf4j.Logger;
 
 import java.time.Duration;
@@ -315,7 +313,7 @@ public void close() {}
     static class TaskCreator extends AbstractTaskCreator {
         private final ThreadCache cache;
         private final KafkaClientSupplier clientSupplier;
-        private final String threadClientId;
+        private final String threadId;
         private final Producer threadProducer;
         private final Sensor createTaskSensor;
 
@@ -328,7 +326,7 @@ static class TaskCreator extends AbstractTaskCreator {
                     final Time time,
                     final KafkaClientSupplier clientSupplier,
                     final Producer threadProducer,
-                    final String threadClientId,
+                    final String threadId,
                     final Logger log) {
             super(
                 builder,
@@ -341,8 +339,8 @@ static class TaskCreator extends AbstractTaskCreator {
             this.cache = cache;
             this.clientSupplier = clientSupplier;
             this.threadProducer = threadProducer;
-            this.threadClientId = threadClientId;
-            createTaskSensor = ThreadMetrics.createTaskSensor(streamsMetrics);
+            this.threadId = threadId;
+            createTaskSensor = ThreadMetrics.createTaskSensor(threadId, streamsMetrics);
         }
 
         @Override
@@ -368,7 +366,7 @@ StreamTask createTask(final Consumer consumer,
         private Producer createProducer(final TaskId id) {
             // eos
             if (threadProducer == null) {
-                final Map producerConfigs = config.getProducerConfigs(getTaskProducerClientId(threadClientId, id));
+                final Map producerConfigs = config.getProducerConfigs(getTaskProducerClientId(threadId, id));
                 log.info("Creating producer client for task {}", id);
                 producerConfigs.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, applicationId + "-" + id);
                 return clientSupplier.getProducer(producerConfigs);
@@ -398,6 +396,7 @@ static class StandbyTaskCreator extends AbstractTaskCreator {
                            final StateDirectory stateDirectory,
                            final ChangelogReader storeChangelogReader,
                            final Time time,
+                           final String threadId,
                            final Logger log) {
             super(
                 builder,
@@ -407,7 +406,7 @@ static class StandbyTaskCreator extends AbstractTaskCreator {
                 storeChangelogReader,
                 time,
                 log);
-            createTaskSensor = ThreadMetrics.createTaskSensor(streamsMetrics);
+            createTaskSensor = ThreadMetrics.createTaskSensor(threadId, streamsMetrics);
         }
 
         @Override
@@ -480,21 +479,21 @@ public static StreamThread create(final InternalTopologyBuilder builder,
                                       final Admin adminClient,
                                       final UUID processId,
                                       final String clientId,
-                                      final Metrics metrics,
+                                      final StreamsMetricsImpl streamsMetrics,
                                       final Time time,
                                       final StreamsMetadataState streamsMetadataState,
                                       final long cacheSizeBytes,
                                       final StateDirectory stateDirectory,
                                       final StateRestoreListener userStateRestoreListener,
                                       final int threadIdx) {
-        final String threadClientId = clientId + "-StreamThread-" + threadIdx;
+        final String threadId = clientId + "-StreamThread-" + threadIdx;
 
-        final String logPrefix = String.format("stream-thread [%s] ", threadClientId);
+        final String logPrefix = String.format("stream-thread [%s] ", threadId);
         final LogContext logContext = new LogContext(logPrefix);
         final Logger log = logContext.logger(StreamThread.class);
 
         log.info("Creating restore consumer client");
-        final Map restoreConsumerConfigs = config.getRestoreConsumerConfigs(getRestoreConsumerClientId(threadClientId));
+        final Map restoreConsumerConfigs = config.getRestoreConsumerConfigs(getRestoreConsumerClientId(threadId));
         final Consumer restoreConsumer = clientSupplier.getRestoreConsumer(restoreConsumerConfigs);
         final Duration pollTime = Duration.ofMillis(config.getLong(StreamsConfig.POLL_MS_CONFIG));
         final StoreChangelogReader changelogReader = new StoreChangelogReader(restoreConsumer, pollTime, userStateRestoreListener, logContext);
@@ -502,17 +501,11 @@ public static StreamThread create(final InternalTopologyBuilder builder,
         Producer threadProducer = null;
         final boolean eosEnabled = StreamsConfig.EXACTLY_ONCE.equals(config.getString(StreamsConfig.PROCESSING_GUARANTEE_CONFIG));
         if (!eosEnabled) {
-            final Map producerConfigs = config.getProducerConfigs(getThreadProducerClientId(threadClientId));
+            final Map producerConfigs = config.getProducerConfigs(getThreadProducerClientId(threadId));
             log.info("Creating shared producer client");
             threadProducer = clientSupplier.getProducer(producerConfigs);
         }
 
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(
-            metrics,
-            threadClientId,
-            config.getString(StreamsConfig.BUILT_IN_METRICS_VERSION_CONFIG)
-        );
-
         final ThreadCache cache = new ThreadCache(logContext, cacheSizeBytes, streamsMetrics);
 
         final AbstractTaskCreator activeTaskCreator = new TaskCreator(
@@ -525,7 +518,7 @@ public static StreamThread create(final InternalTopologyBuilder builder,
             time,
             clientSupplier,
             threadProducer,
-            threadClientId,
+            threadId,
             log);
         final AbstractTaskCreator standbyTaskCreator = new StandbyTaskCreator(
             builder,
@@ -534,6 +527,7 @@ public static StreamThread create(final InternalTopologyBuilder builder,
             stateDirectory,
             changelogReader,
             time,
+            threadId,
             log);
         final TaskManager taskManager = new TaskManager(
             changelogReader,
@@ -549,7 +543,7 @@ public static StreamThread create(final InternalTopologyBuilder builder,
 
         log.info("Creating consumer client");
         final String applicationId = config.getString(StreamsConfig.APPLICATION_ID_CONFIG);
-        final Map consumerConfigs = config.getMainConsumerConfigs(applicationId, getConsumerClientId(threadClientId), threadIdx);
+        final Map consumerConfigs = config.getMainConsumerConfigs(applicationId, getConsumerClientId(threadId), threadIdx);
         consumerConfigs.put(StreamsConfig.InternalConfig.TASK_MANAGER_FOR_PARTITION_ASSIGNOR, taskManager);
         final AtomicInteger assignmentErrorCode = new AtomicInteger();
         consumerConfigs.put(StreamsConfig.InternalConfig.ASSIGNMENT_ERROR_CODE, assignmentErrorCode);
@@ -572,7 +566,7 @@ public static StreamThread create(final InternalTopologyBuilder builder,
             taskManager,
             streamsMetrics,
             builder,
-            threadClientId,
+            threadId,
             logContext,
             assignmentErrorCode)
             .updateThreadMetadata(getSharedAdminClientId(clientId));
@@ -587,29 +581,29 @@ public StreamThread(final Time time,
                         final TaskManager taskManager,
                         final StreamsMetricsImpl streamsMetrics,
                         final InternalTopologyBuilder builder,
-                        final String threadClientId,
+                        final String threadId,
                         final LogContext logContext,
                         final AtomicInteger assignmentErrorCode) {
-        super(threadClientId);
+        super(threadId);
 
         this.stateLock = new Object();
         this.standbyRecords = new HashMap<>();
 
         this.streamsMetrics = streamsMetrics;
-        this.commitSensor = ThreadMetrics.commitSensor(streamsMetrics);
-        this.pollSensor = ThreadMetrics.pollSensor(streamsMetrics);
-        this.processSensor = ThreadMetrics.processSensor(streamsMetrics);
-        this.punctuateSensor = ThreadMetrics.punctuateSensor(streamsMetrics);
+        this.commitSensor = ThreadMetrics.commitSensor(threadId, streamsMetrics);
+        this.pollSensor = ThreadMetrics.pollSensor(threadId, streamsMetrics);
+        this.processSensor = ThreadMetrics.processSensor(threadId, streamsMetrics);
+        this.punctuateSensor = ThreadMetrics.punctuateSensor(threadId, streamsMetrics);
 
         // The following sensors are created here but their references are not stored in this object, since within
         // this object they are not recorded. The sensors are created here so that the stream threads starts with all
         // its metrics initialised. Otherwise, those sensors would have been created during processing, which could
         // lead to missing metrics. For instance, if no task were created, the metrics for created and closed
         // tasks would never be added to the metrics.
-        ThreadMetrics.createTaskSensor(streamsMetrics);
-        ThreadMetrics.closeTaskSensor(streamsMetrics);
-        ThreadMetrics.skipRecordSensor(streamsMetrics);
-        ThreadMetrics.commitOverTasksSensor(streamsMetrics);
+        ThreadMetrics.createTaskSensor(threadId, streamsMetrics);
+        ThreadMetrics.closeTaskSensor(threadId, streamsMetrics);
+        ThreadMetrics.skipRecordSensor(threadId, streamsMetrics);
+        ThreadMetrics.commitOverTasksSensor(threadId, streamsMetrics);
 
         this.time = time;
         this.builder = builder;
@@ -659,10 +653,6 @@ public static String getSharedAdminClientId(final String clientId) {
         return clientId + "-admin";
     }
 
-    public void setRocksDBMetricsRecordingTrigger(final RocksDBMetricsRecordingTrigger rocksDBMetricsRecordingTrigger) {
-        streamsMetrics.setRocksDBMetricsRecordingTrigger(rocksDBMetricsRecordingTrigger);
-    }
-
     /**
      * Execute the stream processors
      *
@@ -1131,7 +1121,7 @@ private void completeShutdown(final boolean cleanRun) {
         } catch (final Throwable e) {
             log.error("Failed to close restore consumer due to the following error:", e);
         }
-        streamsMetrics.removeAllThreadLevelSensors();
+        streamsMetrics.removeAllThreadLevelSensors(getName());
 
         setState(State.DEAD);
         log.info("Shutdown complete");
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
index 0341ab768cdc8..aea1f03341517 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
@@ -18,6 +18,8 @@
 
 import org.apache.kafka.common.Metric;
 import org.apache.kafka.common.MetricName;
+import org.apache.kafka.common.metrics.Gauge;
+import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.Sensor.RecordingLevel;
@@ -51,12 +53,43 @@ public enum Version {
         FROM_100_TO_23
     }
 
+    static class ImmutableMetricValue implements Gauge {
+        private final T value;
+
+        public ImmutableMetricValue(final T value) {
+            this.value = value;
+        }
+
+        @Override
+        public T value(final MetricConfig config, final long now) {
+            return value;
+        }
+
+        @Override
+        public boolean equals(final Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            final ImmutableMetricValue that = (ImmutableMetricValue) o;
+            return Objects.equals(value, that.value);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(value);
+        }
+    }
+
     private final Metrics metrics;
     private final Map parentSensors;
-    private final String threadName;
+    private final String clientId;
 
     private final Version version;
-    private final Deque threadLevelSensors = new LinkedList<>();
+    private final Deque clientLevelMetrics = new LinkedList<>();
+    private final Map> threadLevelSensors = new HashMap<>();
     private final Map> taskLevelSensors = new HashMap<>();
     private final Map> nodeLevelSensors = new HashMap<>();
     private final Map> cacheLevelSensors = new HashMap<>();
@@ -66,7 +99,15 @@ public enum Version {
 
     private static final String SENSOR_PREFIX_DELIMITER = ".";
     private static final String SENSOR_NAME_DELIMITER = ".s.";
-
+    private static final String SENSOR_TASK_LABEL = "task";
+    private static final String SENSOR_NODE_LABEL = "node";
+    private static final String SENSOR_CACHE_LABEL = "cache";
+    private static final String SENSOR_STORE_LABEL = "store";
+    private static final String SENSOR_ENTITY_LABEL = "entity";
+    private static final String SENSOR_EXTERNAL_LABEL = "external";
+    private static final String SENSOR_INTERNAL_LABEL = "internal";
+
+    public static final String CLIENT_ID_TAG = "client-id";
     public static final String THREAD_ID_TAG = "thread-id";
     public static final String THREAD_ID_TAG_0100_TO_23 = "client-id";
     public static final String TASK_ID_TAG = "task-id";
@@ -92,6 +133,7 @@ public enum Version {
     public static final String GROUP_PREFIX = GROUP_PREFIX_WO_DELIMITER + "-";
     public static final String GROUP_SUFFIX = "-metrics";
     public static final String STATE_LEVEL_GROUP_SUFFIX = "-state" + GROUP_SUFFIX;
+    public static final String CLIENT_LEVEL_GROUP = GROUP_PREFIX_WO_DELIMITER + GROUP_SUFFIX;
     public static final String THREAD_LEVEL_GROUP = GROUP_PREFIX_WO_DELIMITER + GROUP_SUFFIX;
     public static final String TASK_LEVEL_GROUP = GROUP_PREFIX + "task" + GROUP_SUFFIX;
     public static final String STATE_LEVEL_GROUP = GROUP_PREFIX + "state" + GROUP_SUFFIX;
@@ -103,12 +145,12 @@ public enum Version {
     public static final String EXPIRED_WINDOW_RECORD_DROP = "expired-window-record-drop";
     public static final String LATE_RECORD_DROP = "late-record-drop";
 
-    public StreamsMetricsImpl(final Metrics metrics, final String threadName, final String builtInMetricsVersion) {
+    public StreamsMetricsImpl(final Metrics metrics, final String clientId, final String builtInMetricsVersion) {
         Objects.requireNonNull(metrics, "Metrics cannot be null");
         Objects.requireNonNull(builtInMetricsVersion, "Built-in metrics version cannot be null");
         this.metrics = metrics;
-        this.threadName = threadName;
-        this.version = parseBuiltInMetricsVersion(builtInMetricsVersion);
+        this.clientId = clientId;
+        version = parseBuiltInMetricsVersion(builtInMetricsVersion);
 
         this.parentSensors = new HashMap<>();
     }
@@ -133,29 +175,62 @@ public RocksDBMetricsRecordingTrigger rocksDBMetricsRecordingTrigger() {
         return rocksDBMetricsRecordingTrigger;
     }
 
-    public final Sensor threadLevelSensor(final String sensorName,
+    public  void addClientLevelImmutableMetric(final String name,
+                                                  final String description,
+                                                  final RecordingLevel recordingLevel,
+                                                  final T value) {
+        final MetricName metricName = metrics.metricName(name, CLIENT_LEVEL_GROUP, description, clientLevelTagMap());
+        final MetricConfig metricConfig = new MetricConfig().recordLevel(recordingLevel);
+        synchronized (clientLevelMetrics) {
+            metrics.addMetric(metricName, metricConfig, new ImmutableMetricValue<>(value));
+            clientLevelMetrics.push(metricName);
+        }
+    }
+
+    public  void addClientLevelMutableMetric(final String name,
+                                                final String description,
+                                                final RecordingLevel recordingLevel,
+                                                final Gauge valueProvider) {
+        final MetricName metricName = metrics.metricName(name, CLIENT_LEVEL_GROUP, description, clientLevelTagMap());
+        final MetricConfig metricConfig = new MetricConfig().recordLevel(recordingLevel);
+        synchronized (clientLevelMetrics) {
+            metrics.addMetric(metricName, metricConfig, valueProvider);
+            clientLevelMetrics.push(metricName);
+        }
+    }
+
+    public final Sensor threadLevelSensor(final String threadId,
+                                          final String sensorName,
                                           final RecordingLevel recordingLevel,
                                           final Sensor... parents) {
+        final String key = threadSensorPrefix(threadId);
         synchronized (threadLevelSensors) {
-            final String fullSensorName = threadSensorPrefix() + SENSOR_NAME_DELIMITER + sensorName;
+            threadLevelSensors.putIfAbsent(key, new LinkedList<>());
+            final String fullSensorName = key + SENSOR_NAME_DELIMITER + sensorName;
             final Sensor sensor = metrics.sensor(fullSensorName, recordingLevel, parents);
-            threadLevelSensors.push(fullSensorName);
+            threadLevelSensors.get(key).push(fullSensorName);
             return sensor;
         }
     }
 
-    private String threadSensorPrefix() {
-        return "internal" + SENSOR_PREFIX_DELIMITER + threadName;
+    private String threadSensorPrefix(final String threadId) {
+        return SENSOR_INTERNAL_LABEL + SENSOR_PREFIX_DELIMITER + threadId;
     }
 
-    public Map threadLevelTagMap() {
+    public Map clientLevelTagMap() {
         final Map tagMap = new LinkedHashMap<>();
-        tagMap.put(THREAD_ID_TAG_0100_TO_23, threadName);
+        tagMap.put(CLIENT_ID_TAG, clientId);
         return tagMap;
     }
 
-    public Map threadLevelTagMap(final String... tags) {
-        final Map tagMap = threadLevelTagMap();
+    public Map threadLevelTagMap(final String threadId) {
+        final Map tagMap = new LinkedHashMap<>();
+        tagMap.put(THREAD_ID_TAG_0100_TO_23, threadId);
+        return tagMap;
+    }
+
+    public Map threadLevelTagMap(final String threadId, final String... tags) {
+        final Map tagMap = threadLevelTagMap(threadId);
         if (tags != null) {
             if ((tags.length % 2) != 0) {
                 throw new IllegalArgumentException("Tags needs to be specified in key-value pairs");
@@ -168,48 +243,58 @@ public Map threadLevelTagMap(final String... tags) {
         return tagMap;
     }
 
-    public final void removeAllThreadLevelSensors() {
+    public final void removeAllClientLevelMetrics() {
+        synchronized (clientLevelMetrics) {
+            while (!clientLevelMetrics.isEmpty()) {
+                metrics.removeMetric(clientLevelMetrics.pop());
+            }
+        }
+    }
+
+    public final void removeAllThreadLevelSensors(final String threadId) {
+        final String key = threadSensorPrefix(threadId);
         synchronized (threadLevelSensors) {
-            while (!threadLevelSensors.isEmpty()) {
-                metrics.removeSensor(threadLevelSensors.pop());
+            final Deque sensors = threadLevelSensors.remove(key);
+            while (sensors != null && !sensors.isEmpty()) {
+                metrics.removeSensor(sensors.pop());
             }
         }
     }
 
-    public Map taskLevelTagMap(final String taskName) {
-        final Map tagMap = threadLevelTagMap();
-        tagMap.put(TASK_ID_TAG, taskName);
+    public Map taskLevelTagMap(final String threadId, final String taskId) {
+        final Map tagMap = threadLevelTagMap(threadId);
+        tagMap.put(TASK_ID_TAG, taskId);
         return tagMap;
     }
 
-    public Map storeLevelTagMap(final String taskName, final String storeType, final String storeName) {
-        final Map tagMap = taskLevelTagMap(taskName);
+    public Map storeLevelTagMap(final String threadId,
+                                                final String taskName,
+                                                final String storeType,
+                                                final String storeName) {
+        final Map tagMap = taskLevelTagMap(threadId, taskName);
         tagMap.put(storeType + "-" + STORE_ID_TAG, storeName);
         return tagMap;
     }
 
-    public final Sensor taskLevelSensor(final String taskName,
+    public final Sensor taskLevelSensor(final String threadId,
+                                        final String taskId,
                                         final String sensorName,
                                         final RecordingLevel recordingLevel,
                                         final Sensor... parents) {
-        final String key = taskSensorPrefix(taskName);
+        final String key = taskSensorPrefix(threadId, taskId);
         synchronized (taskLevelSensors) {
             if (!taskLevelSensors.containsKey(key)) {
                 taskLevelSensors.put(key, new LinkedList<>());
             }
-
             final String fullSensorName = key + SENSOR_NAME_DELIMITER + sensorName;
-
             final Sensor sensor = metrics.sensor(fullSensorName, recordingLevel, parents);
-
             taskLevelSensors.get(key).push(fullSensorName);
-
             return sensor;
         }
     }
 
-    public final void removeAllTaskLevelSensors(final String taskName) {
-        final String key = taskSensorPrefix(taskName);
+    public final void removeAllTaskLevelSensors(final String threadId, final String taskId) {
+        final String key = taskSensorPrefix(threadId, taskId);
         synchronized (taskLevelSensors) {
             final Deque sensors = taskLevelSensors.remove(key);
             while (sensors != null && !sensors.isEmpty()) {
@@ -218,16 +303,18 @@ public final void removeAllTaskLevelSensors(final String taskName) {
         }
     }
 
-    private String taskSensorPrefix(final String taskName) {
-        return threadSensorPrefix() + SENSOR_PREFIX_DELIMITER + "task" + SENSOR_PREFIX_DELIMITER + taskName;
+    private String taskSensorPrefix(final String threadId, final String taskId) {
+        return threadSensorPrefix(threadId) + SENSOR_PREFIX_DELIMITER + SENSOR_TASK_LABEL + SENSOR_PREFIX_DELIMITER +
+            taskId;
     }
 
-    public Sensor nodeLevelSensor(final String taskName,
+    public Sensor nodeLevelSensor(final String threadId,
+                                  final String taskId,
                                   final String processorNodeName,
                                   final String sensorName,
                                   final Sensor.RecordingLevel recordingLevel,
                                   final Sensor... parents) {
-        final String key = nodeSensorPrefix(taskName, processorNodeName);
+        final String key = nodeSensorPrefix(threadId, taskId, processorNodeName);
         synchronized (nodeLevelSensors) {
             if (!nodeLevelSensors.containsKey(key)) {
                 nodeLevelSensors.put(key, new LinkedList<>());
@@ -243,8 +330,10 @@ public Sensor nodeLevelSensor(final String taskName,
         }
     }
 
-    public final void removeAllNodeLevelSensors(final String taskName, final String processorNodeName) {
-        final String key = nodeSensorPrefix(taskName, processorNodeName);
+    public final void removeAllNodeLevelSensors(final String threadId,
+                                                final String taskId,
+                                                final String processorNodeName) {
+        final String key = nodeSensorPrefix(threadId, taskId, processorNodeName);
         synchronized (nodeLevelSensors) {
             final Deque sensors = nodeLevelSensors.remove(key);
             while (sensors != null && !sensors.isEmpty()) {
@@ -253,16 +342,18 @@ public final void removeAllNodeLevelSensors(final String taskName, final String
         }
     }
 
-    private String nodeSensorPrefix(final String taskName, final String processorNodeName) {
-        return taskSensorPrefix(taskName) + SENSOR_PREFIX_DELIMITER + "node" + SENSOR_PREFIX_DELIMITER + processorNodeName;
+    private String nodeSensorPrefix(final String threadId, final String taskId, final String processorNodeName) {
+        return taskSensorPrefix(threadId, taskId)
+            + SENSOR_PREFIX_DELIMITER + SENSOR_NODE_LABEL + SENSOR_PREFIX_DELIMITER + processorNodeName;
     }
 
-    public Sensor cacheLevelSensor(final String taskName,
+    public Sensor cacheLevelSensor(final String threadId,
+                                   final String taskName,
                                    final String storeName,
                                    final String sensorName,
                                    final Sensor.RecordingLevel recordingLevel,
                                    final Sensor... parents) {
-        final String key = cacheSensorPrefix(taskName, storeName);
+        final String key = cacheSensorPrefix(threadId, taskName, storeName);
         synchronized (cacheLevelSensors) {
             if (!cacheLevelSensors.containsKey(key)) {
                 cacheLevelSensors.put(key, new LinkedList<>());
@@ -278,20 +369,22 @@ public Sensor cacheLevelSensor(final String taskName,
         }
     }
 
-    public Map cacheLevelTagMap(final String taskName, final String storeName) {
+    public Map cacheLevelTagMap(final String threadId,
+                                                final String taskId,
+                                                final String storeName) {
         final Map tagMap = new LinkedHashMap<>();
-        tagMap.put(TASK_ID_TAG, taskName);
-        tagMap.put(RECORD_CACHE_ID_TAG, storeName);
         if (version == Version.FROM_100_TO_23) {
-            tagMap.put(THREAD_ID_TAG_0100_TO_23, Thread.currentThread().getName());
+            tagMap.put(THREAD_ID_TAG_0100_TO_23, threadId);
         } else {
-            tagMap.put(THREAD_ID_TAG, Thread.currentThread().getName());
+            tagMap.put(THREAD_ID_TAG, threadId);
         }
+        tagMap.put(TASK_ID_TAG, taskId);
+        tagMap.put(RECORD_CACHE_ID_TAG, storeName);
         return tagMap;
     }
 
-    public final void removeAllCacheLevelSensors(final String taskName, final String cacheName) {
-        final String key = cacheSensorPrefix(taskName, cacheName);
+    public final void removeAllCacheLevelSensors(final String threadId, final String taskId, final String cacheName) {
+        final String key = cacheSensorPrefix(threadId, taskId, cacheName);
         synchronized (cacheLevelSensors) {
             final Deque strings = cacheLevelSensors.remove(key);
             while (strings != null && !strings.isEmpty()) {
@@ -300,16 +393,18 @@ public final void removeAllCacheLevelSensors(final String taskName, final String
         }
     }
 
-    private String cacheSensorPrefix(final String taskName, final String cacheName) {
-        return taskSensorPrefix(taskName) + SENSOR_PREFIX_DELIMITER + "cache" + SENSOR_PREFIX_DELIMITER + cacheName;
+    private String cacheSensorPrefix(final String threadId, final String taskId, final String cacheName) {
+        return taskSensorPrefix(threadId, taskId)
+            + SENSOR_PREFIX_DELIMITER + SENSOR_CACHE_LABEL + SENSOR_PREFIX_DELIMITER + cacheName;
     }
 
-    public final Sensor storeLevelSensor(final String taskName,
+    public final Sensor storeLevelSensor(final String threadId,
+                                         final String taskId,
                                          final String storeName,
                                          final String sensorName,
                                          final Sensor.RecordingLevel recordingLevel,
                                          final Sensor... parents) {
-        final String key = storeSensorPrefix(taskName, storeName);
+        final String key = storeSensorPrefix(threadId, taskId, storeName);
         synchronized (storeLevelSensors) {
             if (!storeLevelSensors.containsKey(key)) {
                 storeLevelSensors.put(key, new LinkedList<>());
@@ -323,8 +418,10 @@ public final Sensor storeLevelSensor(final String taskName,
         }
     }
 
-    public final void removeAllStoreLevelSensors(final String taskName, final String storeName) {
-        final String key = storeSensorPrefix(taskName, storeName);
+    public final void removeAllStoreLevelSensors(final String threadId,
+                                                 final String taskId,
+                                                 final String storeName) {
+        final String key = storeSensorPrefix(threadId, taskId, storeName);
         synchronized (storeLevelSensors) {
             final Deque sensors = storeLevelSensors.remove(key);
             while (sensors != null && !sensors.isEmpty()) {
@@ -333,8 +430,11 @@ public final void removeAllStoreLevelSensors(final String taskName, final String
         }
     }
 
-    private String storeSensorPrefix(final String taskName, final String storeName) {
-        return taskSensorPrefix(taskName) + SENSOR_PREFIX_DELIMITER + "store" + SENSOR_PREFIX_DELIMITER + storeName;
+    private String storeSensorPrefix(final String threadId,
+                                     final String taskId,
+                                     final String storeName) {
+        return taskSensorPrefix(threadId, taskId)
+            + SENSOR_PREFIX_DELIMITER + SENSOR_STORE_LABEL + SENSOR_PREFIX_DELIMITER + storeName;
     }
 
     @Override
@@ -362,9 +462,9 @@ public void recordThroughput(final Sensor sensor, final long value) {
         sensor.record(value);
     }
 
-    public final Map tagMap(final String... tags) {
+    public final Map tagMap(final String threadId, final String... tags) {
         final Map tagMap = new LinkedHashMap<>();
-        tagMap.put("client-id", threadName);
+        tagMap.put(THREAD_ID_TAG_0100_TO_23, threadId);
         if (tags != null) {
             if ((tags.length % 2) != 0) {
                 throw new IllegalArgumentException("Tags needs to be specified in key-value pairs");
@@ -378,11 +478,14 @@ public final Map tagMap(final String... tags) {
     }
 
 
-    private Map constructTags(final String scopeName, final String entityName, final String... tags) {
+    private Map constructTags(final String threadId,
+                                              final String scopeName,
+                                              final String entityName,
+                                              final String... tags) {
         final String[] updatedTags = Arrays.copyOf(tags, tags.length + 2);
         updatedTags[tags.length] = scopeName + "-id";
         updatedTags[tags.length + 1] = entityName;
-        return tagMap(updatedTags);
+        return tagMap(threadId, updatedTags);
     }
 
 
@@ -397,16 +500,21 @@ public Sensor addLatencyAndThroughputSensor(final String scopeName,
                                                 final String... tags) {
         final String group = groupNameFromScope(scopeName);
 
-        final Map tagMap = constructTags(scopeName, entityName, tags);
-        final Map allTagMap = constructTags(scopeName, "all", tags);
+        final String threadId = Thread.currentThread().getName();
+        final Map tagMap = constructTags(threadId, scopeName, entityName, tags);
+        final Map allTagMap = constructTags(threadId, scopeName, "all", tags);
 
         // first add the global operation metrics if not yet, with the global tags only
-        final Sensor parent = metrics.sensor(externalParentSensorName(operationName), recordingLevel);
+        final Sensor parent = metrics.sensor(externalParentSensorName(threadId, operationName), recordingLevel);
         addAvgAndMaxLatencyToSensor(parent, group, allTagMap, operationName);
         addInvocationRateAndCountToSensor(parent, group, allTagMap, operationName);
 
         // add the operation metrics with additional tags
-        final Sensor sensor = metrics.sensor(externalChildSensorName(operationName, entityName), recordingLevel, parent);
+        final Sensor sensor = metrics.sensor(
+            externalChildSensorName(threadId, operationName, entityName),
+            recordingLevel,
+            parent
+        );
         addAvgAndMaxLatencyToSensor(sensor, group, tagMap, operationName);
         addInvocationRateAndCountToSensor(sensor, group, tagMap, operationName);
 
@@ -427,15 +535,23 @@ public Sensor addThroughputSensor(final String scopeName,
                                       final String... tags) {
         final String group = groupNameFromScope(scopeName);
 
-        final Map tagMap = constructTags(scopeName, entityName, tags);
-        final Map allTagMap = constructTags(scopeName, "all", tags);
+        final String threadId = Thread.currentThread().getName();
+        final Map tagMap = constructTags(threadId, scopeName, entityName, tags);
+        final Map allTagMap = constructTags(threadId, scopeName, "all", tags);
 
         // first add the global operation metrics if not yet, with the global tags only
-        final Sensor parent = metrics.sensor(externalParentSensorName(operationName), recordingLevel);
+        final Sensor parent = metrics.sensor(
+            externalParentSensorName(threadId, operationName),
+            recordingLevel
+        );
         addInvocationRateAndCountToSensor(parent, group, allTagMap, operationName);
 
         // add the operation metrics with additional tags
-        final Sensor sensor = metrics.sensor(externalChildSensorName(operationName, entityName), recordingLevel, parent);
+        final Sensor sensor = metrics.sensor(
+            externalChildSensorName(threadId, operationName, entityName),
+            recordingLevel,
+            parent
+        );
         addInvocationRateAndCountToSensor(sensor, group, tagMap, operationName);
 
         parentSensors.put(sensor, parent);
@@ -444,23 +560,22 @@ public Sensor addThroughputSensor(final String scopeName,
 
     }
 
-    private String externalChildSensorName(final String operationName, final String entityName) {
-        return "external" + SENSOR_PREFIX_DELIMITER + threadName
-            + SENSOR_PREFIX_DELIMITER + "entity" + SENSOR_PREFIX_DELIMITER + entityName
+    private String externalChildSensorName(final String threadId, final String operationName, final String entityName) {
+        return SENSOR_EXTERNAL_LABEL + SENSOR_PREFIX_DELIMITER + threadId
+            + SENSOR_PREFIX_DELIMITER + SENSOR_ENTITY_LABEL + SENSOR_PREFIX_DELIMITER + entityName
             + SENSOR_NAME_DELIMITER + operationName;
     }
 
-    private String externalParentSensorName(final String operationName) {
-        return "external" + SENSOR_PREFIX_DELIMITER + threadName + SENSOR_NAME_DELIMITER + operationName;
+    private String externalParentSensorName(final String threadId, final String operationName) {
+        return SENSOR_EXTERNAL_LABEL + SENSOR_PREFIX_DELIMITER + threadId + SENSOR_NAME_DELIMITER + operationName;
     }
 
-
     private static void addAvgAndMaxToSensor(final Sensor sensor,
-                                            final String group,
-                                            final Map tags,
-                                            final String operation,
-                                            final String descriptionOfAvg,
-                                            final String descriptionOfMax) {
+                                             final String group,
+                                             final Map tags,
+                                             final String operation,
+                                             final String descriptionOfAvg,
+                                             final String descriptionOfMax) {
         sensor.add(
             new MetricName(
                 operation + AVG_SUFFIX,
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetrics.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetrics.java
index 8cf8a8b3772aa..9b83f71a73c0c 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetrics.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetrics.java
@@ -72,108 +72,124 @@ private ThreadMetrics() {}
     private static final String PROCESS_LATENCY = PROCESS + LATENCY_SUFFIX;
     private static final String PUNCTUATE_LATENCY = PUNCTUATE + LATENCY_SUFFIX;
 
-    public static Sensor createTaskSensor(final StreamsMetricsImpl streamsMetrics) {
-        final Sensor createTaskSensor = streamsMetrics.threadLevelSensor(CREATE_TASK, RecordingLevel.INFO);
-        addInvocationRateAndCountToSensor(createTaskSensor,
-                                  THREAD_LEVEL_GROUP,
-                                  streamsMetrics.threadLevelTagMap(),
-                                  CREATE_TASK,
-                                  CREATE_TASK_TOTAL_DESCRIPTION,
-                                  CREATE_TASK_RATE_DESCRIPTION);
+    public static Sensor createTaskSensor(final String threadId, final StreamsMetricsImpl streamsMetrics) {
+        final Sensor createTaskSensor = streamsMetrics.threadLevelSensor(threadId, CREATE_TASK, RecordingLevel.INFO);
+        addInvocationRateAndCountToSensor(
+            createTaskSensor,
+            THREAD_LEVEL_GROUP,
+            streamsMetrics.threadLevelTagMap(threadId),
+            CREATE_TASK,
+            CREATE_TASK_TOTAL_DESCRIPTION,
+            CREATE_TASK_RATE_DESCRIPTION
+        );
         return createTaskSensor;
     }
 
-    public static Sensor closeTaskSensor(final StreamsMetricsImpl streamsMetrics) {
-        final Sensor closeTaskSensor = streamsMetrics.threadLevelSensor(CLOSE_TASK, RecordingLevel.INFO);
-        addInvocationRateAndCountToSensor(closeTaskSensor,
-                                  THREAD_LEVEL_GROUP,
-                                  streamsMetrics.threadLevelTagMap(),
-                                  CLOSE_TASK,
-                                  CLOSE_TASK_TOTAL_DESCRIPTION,
-                                  CLOSE_TASK_RATE_DESCRIPTION);
+    public static Sensor closeTaskSensor(final String threadId, final StreamsMetricsImpl streamsMetrics) {
+        final Sensor closeTaskSensor = streamsMetrics.threadLevelSensor(threadId, CLOSE_TASK, RecordingLevel.INFO);
+        addInvocationRateAndCountToSensor(
+            closeTaskSensor,
+            THREAD_LEVEL_GROUP,
+            streamsMetrics.threadLevelTagMap(threadId),
+            CLOSE_TASK,
+            CLOSE_TASK_TOTAL_DESCRIPTION,
+            CLOSE_TASK_RATE_DESCRIPTION
+        );
         return closeTaskSensor;
     }
 
-    public static Sensor commitSensor(final StreamsMetricsImpl streamsMetrics) {
-        final Sensor commitSensor = streamsMetrics.threadLevelSensor(COMMIT, Sensor.RecordingLevel.INFO);
-        final Map tagMap = streamsMetrics.threadLevelTagMap();
+    public static Sensor commitSensor(final String threadId, final StreamsMetricsImpl streamsMetrics) {
+        final Sensor commitSensor = streamsMetrics.threadLevelSensor(threadId, COMMIT, Sensor.RecordingLevel.INFO);
+        final Map tagMap = streamsMetrics.threadLevelTagMap(threadId);
         addAvgAndMaxToSensor(commitSensor, THREAD_LEVEL_GROUP, tagMap, COMMIT_LATENCY);
-        addInvocationRateAndCountToSensor(commitSensor,
-                                  THREAD_LEVEL_GROUP,
-                                  tagMap,
-                                  COMMIT,
-                                  COMMIT_TOTAL_DESCRIPTION,
-                                  COMMIT_RATE_DESCRIPTION);
+        addInvocationRateAndCountToSensor(
+            commitSensor,
+            THREAD_LEVEL_GROUP,
+            tagMap,
+            COMMIT,
+            COMMIT_TOTAL_DESCRIPTION,
+            COMMIT_RATE_DESCRIPTION
+        );
         return commitSensor;
     }
 
-    public static Sensor pollSensor(final StreamsMetricsImpl streamsMetrics) {
-        final Sensor pollSensor = streamsMetrics.threadLevelSensor(POLL, Sensor.RecordingLevel.INFO);
-        final Map tagMap = streamsMetrics.threadLevelTagMap();
+    public static Sensor pollSensor(final String threadId, final StreamsMetricsImpl streamsMetrics) {
+        final Sensor pollSensor = streamsMetrics.threadLevelSensor(threadId, POLL, Sensor.RecordingLevel.INFO);
+        final Map tagMap = streamsMetrics.threadLevelTagMap(threadId);
         addAvgAndMaxToSensor(pollSensor, THREAD_LEVEL_GROUP, tagMap, POLL_LATENCY);
-        addInvocationRateAndCountToSensor(pollSensor,
-                                  THREAD_LEVEL_GROUP,
-                                  tagMap,
-                                  POLL,
-                                  POLL_TOTAL_DESCRIPTION,
-                                  POLL_RATE_DESCRIPTION);
+        addInvocationRateAndCountToSensor(
+            pollSensor,
+            THREAD_LEVEL_GROUP,
+            tagMap,
+            POLL,
+            POLL_TOTAL_DESCRIPTION,
+            POLL_RATE_DESCRIPTION
+        );
         return pollSensor;
     }
 
-    public static Sensor processSensor(final StreamsMetricsImpl streamsMetrics) {
-        final Sensor processSensor = streamsMetrics.threadLevelSensor(PROCESS, Sensor.RecordingLevel.INFO);
-        final Map tagMap = streamsMetrics.threadLevelTagMap();
+    public static Sensor processSensor(final String threadId, final StreamsMetricsImpl streamsMetrics) {
+        final Sensor processSensor = streamsMetrics.threadLevelSensor(threadId, PROCESS, Sensor.RecordingLevel.INFO);
+        final Map tagMap = streamsMetrics.threadLevelTagMap(threadId);
         addAvgAndMaxToSensor(processSensor, THREAD_LEVEL_GROUP, tagMap, PROCESS_LATENCY);
-        addInvocationRateAndCountToSensor(processSensor,
-                                  THREAD_LEVEL_GROUP,
-                                  tagMap,
-                                  PROCESS,
-                                  PROCESS_TOTAL_DESCRIPTION,
-                                  PROCESS_RATE_DESCRIPTION);
-
+        addInvocationRateAndCountToSensor(
+            processSensor,
+            THREAD_LEVEL_GROUP,
+            tagMap,
+            PROCESS,
+            PROCESS_TOTAL_DESCRIPTION,
+            PROCESS_RATE_DESCRIPTION
+        );
         return processSensor;
     }
 
-    public static Sensor punctuateSensor(final StreamsMetricsImpl streamsMetrics) {
-        final Sensor punctuateSensor = streamsMetrics.threadLevelSensor(PUNCTUATE, Sensor.RecordingLevel.INFO);
-        final Map tagMap = streamsMetrics.threadLevelTagMap();
+    public static Sensor punctuateSensor(final String threadId, final StreamsMetricsImpl streamsMetrics) {
+        final Sensor punctuateSensor = streamsMetrics.threadLevelSensor(threadId, PUNCTUATE, Sensor.RecordingLevel.INFO);
+        final Map tagMap = streamsMetrics.threadLevelTagMap(threadId);
         addAvgAndMaxToSensor(punctuateSensor, THREAD_LEVEL_GROUP, tagMap, PUNCTUATE_LATENCY);
-        addInvocationRateAndCountToSensor(punctuateSensor,
-                                  THREAD_LEVEL_GROUP,
-                                  tagMap,
-                                  PUNCTUATE,
-                                  PUNCTUATE_TOTAL_DESCRIPTION,
-                                  PUNCTUATE_RATE_DESCRIPTION);
-
+        addInvocationRateAndCountToSensor(
+            punctuateSensor,
+            THREAD_LEVEL_GROUP,
+            tagMap,
+            PUNCTUATE,
+            PUNCTUATE_TOTAL_DESCRIPTION,
+            PUNCTUATE_RATE_DESCRIPTION
+        );
         return punctuateSensor;
     }
 
-    public static Sensor skipRecordSensor(final StreamsMetricsImpl streamsMetrics) {
-        final Sensor skippedRecordsSensor = streamsMetrics.threadLevelSensor(SKIP_RECORD, Sensor.RecordingLevel.INFO);
-        addInvocationRateAndCountToSensor(skippedRecordsSensor,
-                                  THREAD_LEVEL_GROUP,
-                                  streamsMetrics.threadLevelTagMap(),
-                                  SKIP_RECORD,
-                                  SKIP_RECORD_TOTAL_DESCRIPTION,
-                                  SKIP_RECORD_RATE_DESCRIPTION);
-
+    public static Sensor skipRecordSensor(final String threadId, final StreamsMetricsImpl streamsMetrics) {
+        final Sensor skippedRecordsSensor =
+            streamsMetrics.threadLevelSensor(threadId, SKIP_RECORD, Sensor.RecordingLevel.INFO);
+        addInvocationRateAndCountToSensor(
+            skippedRecordsSensor,
+            THREAD_LEVEL_GROUP,
+            streamsMetrics.threadLevelTagMap(threadId),
+            SKIP_RECORD,
+            SKIP_RECORD_TOTAL_DESCRIPTION,
+            SKIP_RECORD_RATE_DESCRIPTION
+        );
         return skippedRecordsSensor;
     }
 
-    public static Sensor commitOverTasksSensor(final StreamsMetricsImpl streamsMetrics) {
-        final Sensor commitOverTasksSensor = streamsMetrics.threadLevelSensor(COMMIT, Sensor.RecordingLevel.DEBUG);
-        final Map tagMap = streamsMetrics.threadLevelTagMap(TASK_ID_TAG, ROLLUP_VALUE);
-        addAvgAndMaxToSensor(commitOverTasksSensor,
-                     TASK_LEVEL_GROUP,
-                     tagMap,
-                     COMMIT_LATENCY);
-        addInvocationRateAndCountToSensor(commitOverTasksSensor,
-                                  TASK_LEVEL_GROUP,
-                                  tagMap,
-                                  COMMIT,
-                                  COMMIT_OVER_TASKS_TOTAL_DESCRIPTION,
-                                  COMMIT_OVER_TASKS_RATE_DESCRIPTION);
-
+    public static Sensor commitOverTasksSensor(final String threadId, final StreamsMetricsImpl streamsMetrics) {
+        final Sensor commitOverTasksSensor =
+            streamsMetrics.threadLevelSensor(threadId, COMMIT, Sensor.RecordingLevel.DEBUG);
+        final Map tagMap = streamsMetrics.threadLevelTagMap(threadId, TASK_ID_TAG, ROLLUP_VALUE);
+        addAvgAndMaxToSensor(
+            commitOverTasksSensor,
+            TASK_LEVEL_GROUP,
+            tagMap,
+            COMMIT_LATENCY
+        );
+        addInvocationRateAndCountToSensor(
+            commitOverTasksSensor,
+            TASK_LEVEL_GROUP,
+            tagMap,
+            COMMIT,
+            COMMIT_OVER_TASKS_TOTAL_DESCRIPTION,
+            COMMIT_OVER_TASKS_RATE_DESCRIPTION
+        );
         return commitOverTasksSensor;
     }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/AbstractRocksDBSegmentedBytesStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/AbstractRocksDBSegmentedBytesStore.java
index 97dc8d5bc5f6b..65d2cecf1848c 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/AbstractRocksDBSegmentedBytesStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/AbstractRocksDBSegmentedBytesStore.java
@@ -174,9 +174,11 @@ public void init(final ProcessorContext context,
         this.context = (InternalProcessorContext) context;
 
         final StreamsMetricsImpl metrics = this.context.metrics();
+        final String threadId = Thread.currentThread().getName();
         final String taskName = context.taskId().toString();
 
         expiredRecordSensor = metrics.storeLevelSensor(
+            threadId,
             taskName,
             name(),
             EXPIRED_WINDOW_RECORD_DROP,
@@ -185,7 +187,7 @@ public void init(final ProcessorContext context,
         addInvocationRateAndCountToSensor(
             expiredRecordSensor,
             "stream-" + metricScope + "-metrics",
-            metrics.tagMap("task-id", taskName, metricScope + "-id", name()),
+            metrics.tagMap(threadId, "task-id", taskName, metricScope + "-id", name()),
             EXPIRED_WINDOW_RECORD_DROP
         );
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java
index 6c64b049ffb3c..85751c72902d6 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemorySessionStore.java
@@ -75,8 +75,10 @@ public String name() {
     @Override
     public void init(final ProcessorContext context, final StateStore root) {
         final StreamsMetricsImpl metrics = ((InternalProcessorContext) context).metrics();
+        final String threadId = Thread.currentThread().getName();
         final String taskName = context.taskId().toString();
         expiredRecordSensor = metrics.storeLevelSensor(
+            threadId,
             taskName,
             name(),
             EXPIRED_WINDOW_RECORD_DROP,
@@ -85,7 +87,7 @@ public void init(final ProcessorContext context, final StateStore root) {
         addInvocationRateAndCountToSensor(
             expiredRecordSensor,
             "stream-" + metricScope + "-metrics",
-            metrics.tagMap("task-id", taskName, metricScope + "-id", name()),
+            metrics.tagMap(threadId, "task-id", taskName, metricScope + "-id", name()),
             EXPIRED_WINDOW_RECORD_DROP
         );
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryWindowStore.java
index 1a3e26b7fd2a7..49dc566f0281d 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryWindowStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryWindowStore.java
@@ -91,8 +91,10 @@ public void init(final ProcessorContext context, final StateStore root) {
         this.context = (InternalProcessorContext) context;
 
         final StreamsMetricsImpl metrics = this.context.metrics();
+        final String threadId = Thread.currentThread().getName();
         final String taskName = context.taskId().toString();
         expiredRecordSensor = metrics.storeLevelSensor(
+            threadId,
             taskName,
             name(),
             EXPIRED_WINDOW_RECORD_DROP,
@@ -101,7 +103,7 @@ public void init(final ProcessorContext context, final StateStore root) {
         addInvocationRateAndCountToSensor(
             expiredRecordSensor,
             "stream-" + metricScope + "-metrics",
-            metrics.tagMap("task-id", taskName, metricScope + "-id", name()),
+            metrics.tagMap(threadId, "task-id", taskName, metricScope + "-id", name()),
             EXPIRED_WINDOW_RECORD_DROP
         );
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/KeyValueSegments.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/KeyValueSegments.java
index 01510c14790a5..fc49c129b998b 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/KeyValueSegments.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/KeyValueSegments.java
@@ -31,7 +31,7 @@ class KeyValueSegments extends AbstractSegments {
                      final long retentionPeriod,
                      final long segmentInterval) {
         super(name, retentionPeriod, segmentInterval);
-        metricsRecorder = new RocksDBMetricsRecorder(metricsScope, name);
+        metricsRecorder = new RocksDBMetricsRecorder(metricsScope, Thread.currentThread().getName(), name);
     }
 
     @Override
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStore.java
index 07e171f9eb813..2f50ba7a9022f 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStore.java
@@ -65,8 +65,10 @@ public class MeteredKeyValueStore
     private Sensor rangeTime;
     private Sensor flushTime;
     private StreamsMetricsImpl metrics;
+    private final String threadId;
     private String taskName;
 
+
     MeteredKeyValueStore(final KeyValueStore inner,
                          final String metricsScope,
                          final Time time,
@@ -74,6 +76,7 @@ public class MeteredKeyValueStore
                          final Serde valueSerde) {
         super(inner);
         this.metricsScope = metricsScope;
+        threadId = Thread.currentThread().getName();
         this.time = time != null ? time : Time.SYSTEM;
         this.keySerde = keySerde;
         this.valueSerde = valueSerde;
@@ -86,20 +89,112 @@ public void init(final ProcessorContext context,
 
         taskName = context.taskId().toString();
         final String metricsGroup = "stream-" + metricsScope + "-state-metrics";
-        final Map taskTags = metrics.storeLevelTagMap(taskName, metricsScope, ROLLUP_VALUE);
-        final Map storeTags = metrics.storeLevelTagMap(taskName, metricsScope, name());
+        final Map taskTags =
+            metrics.storeLevelTagMap(threadId, taskName, metricsScope, ROLLUP_VALUE);
+        final Map storeTags =
+            metrics.storeLevelTagMap(threadId, taskName, metricsScope, name());
 
         initStoreSerde(context);
 
-        putTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "put", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
-        putIfAbsentTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "put-if-absent", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
-        putAllTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "put-all", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
-        getTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "get", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
-        allTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "all", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
-        rangeTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "range", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
-        flushTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "flush", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
-        deleteTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "delete", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
-        final Sensor restoreTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "restore", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
+        putTime = createTaskAndStoreLatencyAndThroughputSensors(
+            DEBUG,
+            "put",
+            metrics,
+            metricsGroup,
+            threadId,
+            taskName,
+            name(),
+            taskTags,
+            storeTags
+        );
+        putIfAbsentTime = createTaskAndStoreLatencyAndThroughputSensors(
+            DEBUG,
+            "put-if-absent",
+            metrics,
+            metricsGroup,
+            threadId,
+            taskName,
+            name(),
+            taskTags,
+            storeTags
+        );
+        putAllTime = createTaskAndStoreLatencyAndThroughputSensors(
+            DEBUG,
+            "put-all",
+            metrics,
+            metricsGroup,
+            threadId,
+            taskName,
+            name(),
+            taskTags,
+            storeTags
+        );
+        getTime = createTaskAndStoreLatencyAndThroughputSensors(
+            DEBUG,
+            "get",
+            metrics,
+            metricsGroup,
+            threadId,
+            taskName,
+            name(),
+            taskTags,
+            storeTags
+        );
+        allTime = createTaskAndStoreLatencyAndThroughputSensors(
+            DEBUG,
+            "all",
+            metrics,
+            metricsGroup,
+            threadId,
+            taskName,
+            name(),
+            taskTags,
+            storeTags
+        );
+        rangeTime = createTaskAndStoreLatencyAndThroughputSensors(
+            DEBUG,
+            "range",
+            metrics,
+            metricsGroup,
+            threadId,
+            taskName,
+            name(),
+            taskTags,
+            storeTags
+        );
+        flushTime = createTaskAndStoreLatencyAndThroughputSensors(
+            DEBUG,
+            "flush",
+            metrics,
+            metricsGroup,
+            threadId,
+            taskName,
+            name(),
+            taskTags,
+            storeTags
+        );
+        deleteTime = createTaskAndStoreLatencyAndThroughputSensors(
+            DEBUG,
+            "delete",
+            metrics,
+            metricsGroup,
+            threadId,
+            taskName,
+            name(),
+            taskTags,
+            storeTags
+        );
+        final Sensor restoreTime = createTaskAndStoreLatencyAndThroughputSensors(
+            DEBUG,
+            "restore",
+            metrics,
+            metricsGroup,
+            threadId,
+            taskName,
+            name(),
+            taskTags,
+            storeTags
+        );
 
         // register and possibly restore the state from the logs
         if (restoreTime.shouldRecord()) {
@@ -247,7 +342,7 @@ public long approximateNumEntries() {
     @Override
     public void close() {
         super.close();
-        metrics.removeAllStoreLevelSensors(taskName, name());
+        metrics.removeAllStoreLevelSensors(threadId, taskName, name());
     }
 
     private interface Action {
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java
index 09030c23ce460..ebd3bbe930219 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java
@@ -52,6 +52,7 @@ public class MeteredSessionStore
     private Sensor flushTime;
     private Sensor removeTime;
     private String taskName;
+    private final String threadId;
 
     MeteredSessionStore(final SessionStore inner,
                         final String metricsScope,
@@ -59,6 +60,7 @@ public class MeteredSessionStore
                         final Serde valueSerde,
                         final Time time) {
         super(inner);
+        threadId = Thread.currentThread().getName();
         this.metricsScope = metricsScope;
         this.keySerde = keySerde;
         this.valueSerde = valueSerde;
@@ -78,14 +80,66 @@ public void init(final ProcessorContext context,
 
         taskName = context.taskId().toString();
         final String metricsGroup = "stream-" + metricsScope + "-state-metrics";
-        final Map taskTags = metrics.storeLevelTagMap(taskName, metricsScope, ROLLUP_VALUE);
-        final Map storeTags = metrics.storeLevelTagMap(taskName, metricsScope, name());
+        final Map taskTags =
+            metrics.storeLevelTagMap(threadId, taskName, metricsScope, ROLLUP_VALUE);
+        final Map storeTags =
+            metrics.storeLevelTagMap(threadId, taskName, metricsScope, name());
 
-        putTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "put", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
-        fetchTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "fetch", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
-        flushTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "flush", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
-        removeTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "remove", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
-        final Sensor restoreTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "restore", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
+        putTime = createTaskAndStoreLatencyAndThroughputSensors(
+            DEBUG,
+            "put",
+            metrics,
+            metricsGroup,
+            threadId,
+            taskName,
+            name(),
+            taskTags,
+            storeTags
+        );
+        fetchTime = createTaskAndStoreLatencyAndThroughputSensors(
+            DEBUG,
+            "fetch",
+            metrics,
+            metricsGroup,
+            threadId,
+            taskName,
+            name(),
+            taskTags,
+            storeTags
+        );
+        flushTime = createTaskAndStoreLatencyAndThroughputSensors(
+            DEBUG,
+            "flush",
+            metrics,
+            metricsGroup,
+            threadId,
+            taskName,
+            name(),
+            taskTags,
+            storeTags
+        );
+        removeTime = createTaskAndStoreLatencyAndThroughputSensors(
+            DEBUG,
+            "remove",
+            metrics,
+            metricsGroup,
+            threadId,
+            taskName,
+            name(),
+            taskTags,
+            storeTags
+        );
+        final Sensor restoreTime = createTaskAndStoreLatencyAndThroughputSensors(
+            DEBUG,
+            "restore",
+            metrics,
+            metricsGroup,
+            threadId,
+            taskName,
+            name(),
+            taskTags,
+            storeTags
+        );
 
         // register and possibly restore the state from the logs
         final long startNs = time.nanoseconds();
@@ -240,7 +294,7 @@ public void flush() {
     @Override
     public void close() {
         super.close();
-        metrics.removeAllStoreLevelSensors(taskName, name());
+        metrics.removeAllStoreLevelSensors(threadId, taskName, name());
     }
 
     private Bytes keyBytes(final K key) {
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java
index ac86679169af1..149f9f807819b 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java
@@ -54,6 +54,7 @@ public class MeteredWindowStore
     private Sensor fetchTime;
     private Sensor flushTime;
     private ProcessorContext context;
+    private final String threadId;
     private String taskName;
 
     MeteredWindowStore(final WindowStore inner,
@@ -64,6 +65,7 @@ public class MeteredWindowStore
                        final Serde valueSerde) {
         super(inner);
         this.windowSizeMs = windowSizeMs;
+        threadId = Thread.currentThread().getName();
         this.metricsScope = metricsScope;
         this.time = time;
         this.keySerde = keySerde;
@@ -79,13 +81,55 @@ public void init(final ProcessorContext context,
 
         taskName = context.taskId().toString();
         final String metricsGroup = GROUP_PREFIX + metricsScope + STATE_LEVEL_GROUP_SUFFIX;
-        final Map taskTags = metrics.storeLevelTagMap(taskName, metricsScope, ROLLUP_VALUE);
-        final Map storeTags = metrics.storeLevelTagMap(taskName, metricsScope, name());
+        final Map taskTags =
+            metrics.storeLevelTagMap(threadId, taskName, metricsScope, ROLLUP_VALUE);
+        final Map storeTags =
+            metrics.storeLevelTagMap(threadId, taskName, metricsScope, name());
 
-        putTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "put", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
-        fetchTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "fetch", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
-        flushTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "flush", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
-        final Sensor restoreTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "restore", metrics, metricsGroup, taskName, name(), taskTags, storeTags);
+        putTime = createTaskAndStoreLatencyAndThroughputSensors(
+            DEBUG,
+            "put",
+            metrics,
+            metricsGroup,
+            threadId,
+            taskName,
+            name(),
+            taskTags,
+            storeTags
+        );
+        fetchTime = createTaskAndStoreLatencyAndThroughputSensors(
+            DEBUG,
+            "fetch",
+            metrics,
+            metricsGroup,
+            threadId,
+            taskName,
+            name(),
+            taskTags,
+            storeTags
+        );
+        flushTime = createTaskAndStoreLatencyAndThroughputSensors(
+            DEBUG,
+            "flush",
+            metrics,
+            metricsGroup,
+            threadId,
+            taskName,
+            name(),
+            taskTags,
+            storeTags
+        );
+        final Sensor restoreTime = createTaskAndStoreLatencyAndThroughputSensors(
+            DEBUG,
+            "restore",
+            metrics,
+            metricsGroup,
+            threadId,
+            taskName,
+            name(),
+            taskTags,
+            storeTags
+        );
 
         // register and possibly restore the state from the logs
         final long startNs = time.nanoseconds();
@@ -218,7 +262,7 @@ public void flush() {
     @Override
     public void close() {
         super.close();
-        metrics.removeAllStoreLevelSensors(taskName, name());
+        metrics.removeAllStoreLevelSensors(threadId, taskName, name());
     }
 
     private Bytes keyBytes(final K key) {
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
index 2d53440ef4724..4693fbc7b6540 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
@@ -60,7 +60,12 @@ class NamedCache {
         this.streamsMetrics = streamsMetrics;
         storeName = ThreadCache.underlyingStoreNamefromCacheName(name);
         taskName = ThreadCache.taskIDfromCacheName(name);
-        hitRatioSensor = NamedCacheMetrics.hitRatioSensor(streamsMetrics, taskName, storeName);
+        hitRatioSensor = NamedCacheMetrics.hitRatioSensor(
+            streamsMetrics,
+            Thread.currentThread().getName(),
+            taskName,
+            storeName
+        );
     }
 
     synchronized final String name() {
@@ -315,7 +320,7 @@ synchronized void close() {
         currentSizeBytes = 0;
         dirtyKeys.clear();
         cache.clear();
-        streamsMetrics.removeAllCacheLevelSensors(taskName, storeName);
+        streamsMetrics.removeAllCacheLevelSensors(Thread.currentThread().getName(), taskName, storeName);
     }
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
index 74ccd04becfb3..d04b7c25eb543 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
@@ -115,7 +115,7 @@ public class RocksDBStore implements KeyValueStore, BulkLoadingSt
 
     RocksDBStore(final String name,
                  final String metricsScope) {
-        this(name, DB_FILE_DIR, new RocksDBMetricsRecorder(metricsScope, name));
+        this(name, DB_FILE_DIR, new RocksDBMetricsRecorder(metricsScope, Thread.currentThread().getName(), name));
     }
 
     RocksDBStore(final String name,
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/TimestampedSegments.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/TimestampedSegments.java
index 6545de0f691fc..400511f8ff4e0 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/TimestampedSegments.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/TimestampedSegments.java
@@ -31,7 +31,7 @@ class TimestampedSegments extends AbstractSegments {
                         final long retentionPeriod,
                         final long segmentInterval) {
         super(name, retentionPeriod, segmentInterval);
-        metricsRecorder = new RocksDBMetricsRecorder(metricsScope, name);
+        metricsRecorder = new RocksDBMetricsRecorder(metricsScope, Thread.currentThread().getName(), name);
     }
 
     @Override
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/NamedCacheMetrics.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/NamedCacheMetrics.java
index 3744428ba5828..e7e0e62e6e2cb 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/NamedCacheMetrics.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/NamedCacheMetrics.java
@@ -35,6 +35,7 @@ private NamedCacheMetrics() {}
 
 
     public static Sensor hitRatioSensor(final StreamsMetricsImpl streamsMetrics,
+                                        final String threadId,
                                         final String taskName,
                                         final String storeName) {
 
@@ -43,6 +44,7 @@ public static Sensor hitRatioSensor(final StreamsMetricsImpl streamsMetrics,
         if (streamsMetrics.version() == FROM_100_TO_23) {
             hitRatioName = HIT_RATIO_0100_TO_23;
             final Sensor taskLevelHitRatioSensor = streamsMetrics.taskLevelSensor(
+                threadId,
                 taskName,
                 hitRatioName,
                 Sensor.RecordingLevel.DEBUG
@@ -50,13 +52,14 @@ public static Sensor hitRatioSensor(final StreamsMetricsImpl streamsMetrics,
             addAvgAndMinAndMaxToSensor(
                 taskLevelHitRatioSensor,
                 CACHE_LEVEL_GROUP,
-                streamsMetrics.cacheLevelTagMap(taskName, ROLLUP_VALUE),
+                streamsMetrics.cacheLevelTagMap(threadId, taskName, ROLLUP_VALUE),
                 hitRatioName,
                 HIT_RATIO_AVG_DESCRIPTION,
                 HIT_RATIO_MIN_DESCRIPTION,
                 HIT_RATIO_MAX_DESCRIPTION
             );
             hitRatioSensor = streamsMetrics.cacheLevelSensor(
+                threadId,
                 taskName,
                 storeName,
                 hitRatioName,
@@ -66,6 +69,7 @@ public static Sensor hitRatioSensor(final StreamsMetricsImpl streamsMetrics,
         } else {
             hitRatioName = HIT_RATIO;
             hitRatioSensor = streamsMetrics.cacheLevelSensor(
+                threadId,
                 taskName,
                 storeName,
                 hitRatioName,
@@ -75,7 +79,7 @@ public static Sensor hitRatioSensor(final StreamsMetricsImpl streamsMetrics,
         addAvgAndMinAndMaxToSensor(
             hitRatioSensor,
             CACHE_LEVEL_GROUP,
-            streamsMetrics.cacheLevelTagMap(taskName, storeName),
+            streamsMetrics.cacheLevelTagMap(threadId, taskName, storeName),
             hitRatioName,
             HIT_RATIO_AVG_DESCRIPTION,
             HIT_RATIO_MIN_DESCRIPTION,
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetrics.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetrics.java
index 04da4bc53ab7b..e354cb82ea963 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetrics.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetrics.java
@@ -96,16 +96,24 @@ private RocksDBMetrics() {}
     private static final String NUMBER_OF_FILE_ERRORS_DESCRIPTION = "Total number of file errors occurred";
 
     public static class RocksDBMetricContext {
+        private final String threadId;
         private final String taskName;
         private final String metricsScope;
         private final String storeName;
 
-        public RocksDBMetricContext(final String taskName, final String metricsScope, final String storeName) {
+        public RocksDBMetricContext(final String threadId,
+                                    final String taskName,
+                                    final String metricsScope,
+                                    final String storeName) {
+            this.threadId = threadId;
             this.taskName = taskName;
             this.metricsScope = metricsScope;
             this.storeName = storeName;
         }
 
+        public String threadId() {
+            return threadId;
+        }
         public String taskName() {
             return taskName;
         }
@@ -125,14 +133,15 @@ public boolean equals(final Object o) {
                 return false;
             }
             final RocksDBMetricContext that = (RocksDBMetricContext) o;
-            return Objects.equals(taskName, that.taskName) &&
+            return Objects.equals(threadId, that.threadId) &&
+                Objects.equals(taskName, that.taskName) &&
                 Objects.equals(metricsScope, that.metricsScope) &&
                 Objects.equals(storeName, that.storeName);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(taskName, metricsScope, storeName);
+            return Objects.hash(threadId, taskName, metricsScope, storeName);
         }
     }
 
@@ -142,8 +151,12 @@ public static Sensor bytesWrittenToDatabaseSensor(final StreamsMetricsImpl strea
         addRateOfSumAndSumMetricsToSensor(
             sensor,
             STATE_LEVEL_GROUP,
-            streamsMetrics
-                .storeLevelTagMap(metricContext.taskName(), metricContext.metricsScope(), metricContext.storeName()),
+            streamsMetrics.storeLevelTagMap(
+                metricContext.threadId(),
+                metricContext.taskName(),
+                metricContext.metricsScope(),
+                metricContext.storeName()
+            ),
             BYTES_WRITTEN_TO_DB,
             BYTES_WRITTEN_TO_DB_RATE_DESCRIPTION,
             BYTES_WRITTEN_TO_DB_TOTAL_DESCRIPTION
@@ -157,8 +170,12 @@ public static Sensor bytesReadFromDatabaseSensor(final StreamsMetricsImpl stream
         addRateOfSumAndSumMetricsToSensor(
             sensor,
             STATE_LEVEL_GROUP,
-            streamsMetrics
-                .storeLevelTagMap(metricContext.taskName(), metricContext.metricsScope(), metricContext.storeName()),
+            streamsMetrics.storeLevelTagMap(
+                metricContext.threadId(),
+                metricContext.taskName(),
+                metricContext.metricsScope(),
+                metricContext.storeName()
+            ),
             BYTES_READ_FROM_DB,
             BYTES_READ_FROM_DB_RATE_DESCRIPTION,
             BYTES_READ_FROM_DB_TOTAL_DESCRIPTION
@@ -172,8 +189,12 @@ public static Sensor memtableBytesFlushedSensor(final StreamsMetricsImpl streams
         addRateOfSumAndSumMetricsToSensor(
             sensor,
             STATE_LEVEL_GROUP,
-            streamsMetrics
-                .storeLevelTagMap(metricContext.taskName(), metricContext.metricsScope(), metricContext.storeName()),
+            streamsMetrics.storeLevelTagMap(
+                metricContext.threadId(),
+                metricContext.taskName(),
+                metricContext.metricsScope(),
+                metricContext.storeName()
+            ),
             MEMTABLE_BYTES_FLUSHED,
             MEMTABLE_BYTES_FLUSHED_RATE_DESCRIPTION,
             MEMTABLE_BYTES_FLUSHED_TOTAL_DESCRIPTION
@@ -187,8 +208,12 @@ public static Sensor memtableHitRatioSensor(final StreamsMetricsImpl streamsMetr
         addValueMetricToSensor(
             sensor,
             STATE_LEVEL_GROUP,
-            streamsMetrics
-                .storeLevelTagMap(metricContext.taskName(), metricContext.metricsScope(), metricContext.storeName()),
+            streamsMetrics.storeLevelTagMap(
+                metricContext.threadId(),
+                metricContext.taskName(),
+                metricContext.metricsScope(),
+                metricContext.storeName()
+            ),
             MEMTABLE_HIT_RATIO,
             MEMTABLE_HIT_RATIO_DESCRIPTION
         );
@@ -201,8 +226,12 @@ public static Sensor memtableAvgFlushTimeSensor(final StreamsMetricsImpl streams
         addValueMetricToSensor(
             sensor,
             STATE_LEVEL_GROUP,
-            streamsMetrics
-                .storeLevelTagMap(metricContext.taskName(), metricContext.metricsScope(), metricContext.storeName()),
+            streamsMetrics.storeLevelTagMap(
+                metricContext.threadId(),
+                metricContext.taskName(),
+                metricContext.metricsScope(),
+                metricContext.storeName()
+            ),
             MEMTABLE_FLUSH_TIME_AVG,
             MEMTABLE_FLUSH_TIME_AVG_DESCRIPTION
         );
@@ -215,8 +244,12 @@ public static Sensor memtableMinFlushTimeSensor(final StreamsMetricsImpl streams
         addValueMetricToSensor(
             sensor,
             STATE_LEVEL_GROUP,
-            streamsMetrics
-                .storeLevelTagMap(metricContext.taskName(), metricContext.metricsScope(), metricContext.storeName()),
+            streamsMetrics.storeLevelTagMap(
+                metricContext.threadId(),
+                metricContext.taskName(),
+                metricContext.metricsScope(),
+                metricContext.storeName()
+            ),
             MEMTABLE_FLUSH_TIME_MIN,
             MEMTABLE_FLUSH_TIME_MIN_DESCRIPTION
         );
@@ -229,8 +262,12 @@ public static Sensor memtableMaxFlushTimeSensor(final StreamsMetricsImpl streams
         addValueMetricToSensor(
             sensor,
             STATE_LEVEL_GROUP,
-            streamsMetrics
-                .storeLevelTagMap(metricContext.taskName(), metricContext.metricsScope(), metricContext.storeName()),
+            streamsMetrics.storeLevelTagMap(
+                metricContext.threadId(),
+                metricContext.taskName(),
+                metricContext.metricsScope(),
+                metricContext.storeName()
+            ),
             MEMTABLE_FLUSH_TIME_MAX,
             MEMTABLE_FLUSH_TIME_MAX_DESCRIPTION
         );
@@ -243,8 +280,12 @@ public static Sensor writeStallDurationSensor(final StreamsMetricsImpl streamsMe
         addAvgAndSumMetricsToSensor(
             sensor,
             STATE_LEVEL_GROUP,
-            streamsMetrics
-                .storeLevelTagMap(metricContext.taskName(), metricContext.metricsScope(), metricContext.storeName()),
+            streamsMetrics.storeLevelTagMap(
+                metricContext.threadId(),
+                metricContext.taskName(),
+                metricContext.metricsScope(),
+                metricContext.storeName()
+            ),
             WRITE_STALL_DURATION,
             WRITE_STALL_DURATION_AVG_DESCRIPTION,
             WRITE_STALL_DURATION_TOTAL_DESCRIPTION
@@ -258,8 +299,12 @@ public static Sensor blockCacheDataHitRatioSensor(final StreamsMetricsImpl strea
         addValueMetricToSensor(
             sensor,
             STATE_LEVEL_GROUP,
-            streamsMetrics
-                .storeLevelTagMap(metricContext.taskName(), metricContext.metricsScope(), metricContext.storeName()),
+            streamsMetrics.storeLevelTagMap(
+                metricContext.threadId(),
+                metricContext.taskName(),
+                metricContext.metricsScope(),
+                metricContext.storeName()
+            ),
             BLOCK_CACHE_DATA_HIT_RATIO,
             BLOCK_CACHE_DATA_HIT_RATIO_DESCRIPTION
         );
@@ -272,7 +317,12 @@ public static Sensor blockCacheIndexHitRatioSensor(final StreamsMetricsImpl stre
         addValueMetricToSensor(
             sensor,
             STATE_LEVEL_GROUP,
-            streamsMetrics.storeLevelTagMap(metricContext.taskName(), metricContext.metricsScope(), metricContext.storeName()),
+            streamsMetrics.storeLevelTagMap(
+                metricContext.threadId(),
+                metricContext.taskName(),
+                metricContext.metricsScope(),
+                metricContext.storeName()
+            ),
             BLOCK_CACHE_INDEX_HIT_RATIO,
             BLOCK_CACHE_INDEX_HIT_RATIO_DESCRIPTION
         );
@@ -285,8 +335,12 @@ public static Sensor blockCacheFilterHitRatioSensor(final StreamsMetricsImpl str
         addValueMetricToSensor(
             sensor,
             STATE_LEVEL_GROUP,
-            streamsMetrics
-                .storeLevelTagMap(metricContext.taskName(), metricContext.metricsScope(), metricContext.storeName()),
+            streamsMetrics.storeLevelTagMap(
+                metricContext.threadId(),
+                metricContext.taskName(),
+                metricContext.metricsScope(),
+                metricContext.storeName()
+            ),
             BLOCK_CACHE_FILTER_HIT_RATIO,
             BLOCK_CACHE_FILTER_HIT_RATIO_DESCRIPTION
         );
@@ -299,8 +353,12 @@ public static Sensor bytesReadDuringCompactionSensor(final StreamsMetricsImpl st
         addRateOfSumMetricToSensor(
             sensor,
             STATE_LEVEL_GROUP,
-            streamsMetrics
-                .storeLevelTagMap(metricContext.taskName(), metricContext.metricsScope(), metricContext.storeName()),
+            streamsMetrics.storeLevelTagMap(
+                metricContext.threadId(),
+                metricContext.taskName(),
+                metricContext.metricsScope(),
+                metricContext.storeName()
+            ),
             BYTES_READ_DURING_COMPACTION,
             BYTES_READ_DURING_COMPACTION_DESCRIPTION
         );
@@ -313,8 +371,12 @@ public static Sensor bytesWrittenDuringCompactionSensor(final StreamsMetricsImpl
         addRateOfSumMetricToSensor(
             sensor,
             STATE_LEVEL_GROUP,
-            streamsMetrics
-                .storeLevelTagMap(metricContext.taskName(), metricContext.metricsScope(), metricContext.storeName()),
+            streamsMetrics.storeLevelTagMap(
+                metricContext.threadId(),
+                metricContext.taskName(),
+                metricContext.metricsScope(),
+                metricContext.storeName()
+            ),
             BYTES_WRITTEN_DURING_COMPACTION,
             BYTES_WRITTEN_DURING_COMPACTION_DESCRIPTION
         );
@@ -327,8 +389,12 @@ public static Sensor compactionTimeAvgSensor(final StreamsMetricsImpl streamsMet
         addValueMetricToSensor(
             sensor,
             STATE_LEVEL_GROUP,
-            streamsMetrics
-                .storeLevelTagMap(metricContext.taskName(), metricContext.metricsScope(), metricContext.storeName()),
+            streamsMetrics.storeLevelTagMap(
+                metricContext.threadId(),
+                metricContext.taskName(),
+                metricContext.metricsScope(),
+                metricContext.storeName()
+            ),
             COMPACTION_TIME_AVG,
             COMPACTION_TIME_AVG_DESCRIPTION
         );
@@ -341,8 +407,12 @@ public static Sensor compactionTimeMinSensor(final StreamsMetricsImpl streamsMet
         addValueMetricToSensor(
             sensor,
             STATE_LEVEL_GROUP,
-            streamsMetrics
-                .storeLevelTagMap(metricContext.taskName(), metricContext.metricsScope(), metricContext.storeName()),
+            streamsMetrics.storeLevelTagMap(
+                metricContext.threadId(),
+                metricContext.taskName(),
+                metricContext.metricsScope(),
+                metricContext.storeName()
+            ),
             COMPACTION_TIME_MIN,
             COMPACTION_TIME_MIN_DESCRIPTION
         );
@@ -355,8 +425,12 @@ public static Sensor compactionTimeMaxSensor(final StreamsMetricsImpl streamsMet
         addValueMetricToSensor(
             sensor,
             STATE_LEVEL_GROUP,
-            streamsMetrics
-                .storeLevelTagMap(metricContext.taskName(), metricContext.metricsScope(), metricContext.storeName()),
+            streamsMetrics.storeLevelTagMap(
+                metricContext.threadId(),
+                metricContext.taskName(),
+                metricContext.metricsScope(),
+                metricContext.storeName()
+            ),
             COMPACTION_TIME_MAX,
             COMPACTION_TIME_MAX_DESCRIPTION
         );
@@ -369,8 +443,12 @@ public static Sensor numberOfOpenFilesSensor(final StreamsMetricsImpl streamsMet
         addSumMetricToSensor(
             sensor,
             STATE_LEVEL_GROUP,
-            streamsMetrics
-                .storeLevelTagMap(metricContext.taskName(), metricContext.metricsScope(), metricContext.storeName()),
+            streamsMetrics.storeLevelTagMap(
+                metricContext.threadId(),
+                metricContext.taskName(),
+                metricContext.metricsScope(),
+                metricContext.storeName()
+            ),
             NUMBER_OF_OPEN_FILES,
             false,
             NUMBER_OF_OPEN_FILES_DESCRIPTION
@@ -384,8 +462,12 @@ public static Sensor numberOfFileErrorsSensor(final StreamsMetricsImpl streamsMe
         addSumMetricToSensor(
             sensor,
             STATE_LEVEL_GROUP,
-            streamsMetrics
-                .storeLevelTagMap(metricContext.taskName(), metricContext.metricsScope(), metricContext.storeName()),
+            streamsMetrics.storeLevelTagMap(
+                metricContext.threadId(),
+                metricContext.taskName(),
+                metricContext.metricsScope(),
+                metricContext.storeName()
+            ),
             NUMBER_OF_FILE_ERRORS,
             NUMBER_OF_FILE_ERRORS_DESCRIPTION
         );
@@ -396,6 +478,7 @@ private static Sensor createSensor(final StreamsMetricsImpl streamsMetrics,
                                        final RocksDBMetricContext metricContext,
                                        final String sensorName) {
         return streamsMetrics.storeLevelSensor(
+            metricContext.threadId(),
             metricContext.taskName(),
             metricContext.storeName(),
             sensorName,
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorder.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorder.java
index de2a37c07bec1..59ade54132ab1 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorder.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorder.java
@@ -48,12 +48,16 @@ public class RocksDBMetricsRecorder {
     private final Map statisticsToRecord = new ConcurrentHashMap<>();
     private final String metricsScope;
     private final String storeName;
+    private final String threadId;
     private TaskId taskId;
     private StreamsMetricsImpl streamsMetrics;
     private boolean isInitialized = false;
 
-    public RocksDBMetricsRecorder(final String metricsScope, final String storeName) {
+    public RocksDBMetricsRecorder(final String metricsScope,
+                                  final String threadId,
+                                  final String storeName) {
         this.metricsScope = metricsScope;
+        this.threadId = threadId;
         this.storeName = storeName;
         final LogContext logContext = new LogContext(String.format("[RocksDB Metrics Recorder for %s] ", storeName));
         logger = logContext.logger(RocksDBMetricsRecorder.class);
@@ -97,7 +101,8 @@ public void addStatistics(final String segmentName,
     }
 
     private void initSensors(final StreamsMetricsImpl streamsMetrics, final TaskId taskId) {
-        final RocksDBMetricContext metricContext = new RocksDBMetricContext(taskId.toString(), metricsScope, storeName);
+        final RocksDBMetricContext metricContext =
+            new RocksDBMetricContext(threadId, taskId.toString(), metricsScope, storeName);
         bytesWrittenToDatabaseSensor = RocksDBMetrics.bytesWrittenToDatabaseSensor(streamsMetrics, metricContext);
         bytesReadFromDatabaseSensor = RocksDBMetrics.bytesReadFromDatabaseSensor(streamsMetrics, metricContext);
         memtableBytesFlushedSensor = RocksDBMetrics.memtableBytesFlushedSensor(streamsMetrics, metricContext);
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/Sensors.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/Sensors.java
index 8ed4d47f24d53..ae967f38bc21a 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/Sensors.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/Sensors.java
@@ -18,6 +18,7 @@
 
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.metrics.Sensor.RecordingLevel;
 import org.apache.kafka.common.metrics.stats.Avg;
 import org.apache.kafka.common.metrics.stats.Max;
 import org.apache.kafka.common.metrics.stats.Value;
@@ -33,18 +34,20 @@
 public final class Sensors {
     private Sensors() {}
 
-    public static Sensor createTaskAndStoreLatencyAndThroughputSensors(final Sensor.RecordingLevel level,
+    public static Sensor createTaskAndStoreLatencyAndThroughputSensors(final RecordingLevel level,
                                                                        final String operation,
                                                                        final StreamsMetricsImpl metrics,
                                                                        final String metricsGroup,
+                                                                       final String threadId,
                                                                        final String taskName,
                                                                        final String storeName,
                                                                        final Map taskTags,
                                                                        final Map storeTags) {
-        final Sensor taskSensor = metrics.taskLevelSensor(taskName, operation, level);
+        final Sensor taskSensor = metrics.taskLevelSensor(threadId, taskName, operation, level);
         addAvgAndMaxLatencyToSensor(taskSensor, metricsGroup, taskTags, operation);
         addInvocationRateAndCountToSensor(taskSensor, metricsGroup, taskTags, operation);
-        final Sensor sensor = metrics.storeLevelSensor(taskName, storeName, operation, level, taskSensor);
+        final Sensor sensor = metrics
+            .storeLevelSensor(threadId, taskName, storeName, operation, level, taskSensor);
         addAvgAndMaxLatencyToSensor(sensor, metricsGroup, storeTags, operation);
         addInvocationRateAndCountToSensor(sensor, metricsGroup, storeTags, operation);
         return sensor;
@@ -64,10 +67,12 @@ private static Sensor getBufferSizeOrCountSensor(final StateStore store,
                                                      final InternalProcessorContext context,
                                                      final String property) {
         final StreamsMetricsImpl metrics = context.metrics();
+        final String threadId = Thread.currentThread().getName();
 
         final String sensorName = "suppression-buffer-" + property;
 
         final Sensor sensor = metrics.storeLevelSensor(
+            threadId,
             context.taskId().toString(),
             store.name(),
             sensorName,
@@ -77,8 +82,11 @@ private static Sensor getBufferSizeOrCountSensor(final StateStore store,
         final String metricsGroup = "stream-buffer-metrics";
 
         final Map tags = metrics.tagMap(
-            "task-id", context.taskId().toString(),
-            "buffer-id", store.name()
+            threadId,
+            "task-id",
+            context.taskId().toString(),
+            "buffer-id",
+            store.name()
         );
 
         sensor.add(
diff --git a/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java b/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java
index fae9020ecac50..d4850634a9129 100644
--- a/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java
@@ -28,6 +28,7 @@
 import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.common.utils.MockTime;
 import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.streams.internals.metrics.ClientMetrics;
 import org.apache.kafka.streams.kstream.Materialized;
 import org.apache.kafka.streams.processor.AbstractProcessor;
 import org.apache.kafka.streams.processor.StateRestoreListener;
@@ -37,6 +38,7 @@
 import org.apache.kafka.streams.processor.internals.StateDirectory;
 import org.apache.kafka.streams.processor.internals.StreamThread;
 import org.apache.kafka.streams.processor.internals.StreamsMetadataState;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.apache.kafka.streams.state.KeyValueStore;
 import org.apache.kafka.streams.state.StoreBuilder;
 import org.apache.kafka.streams.state.Stores;
@@ -84,10 +86,11 @@
 import static org.junit.Assert.fail;
 
 @RunWith(PowerMockRunner.class)
-@PrepareForTest({KafkaStreams.class, StreamThread.class})
+@PrepareForTest({KafkaStreams.class, StreamThread.class, ClientMetrics.class})
 public class KafkaStreamsTest {
 
     private static final int NUM_THREADS = 2;
+    private final static String APPLICATION_ID = "appId";
 
     @Rule
     public TestName testName = new TestName();
@@ -139,7 +142,7 @@ public void before() throws Exception {
         metricsReportersCapture = EasyMock.newCapture();
 
         props = new Properties();
-        props.put(StreamsConfig.APPLICATION_ID_CONFIG, "appId");
+        props.put(StreamsConfig.APPLICATION_ID_CONFIG, APPLICATION_ID);
         props.put(StreamsConfig.CLIENT_ID_CONFIG, "clientId");
         props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:2018");
         props.put(StreamsConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName());
@@ -169,6 +172,15 @@ private void prepareStreams() throws Exception {
             return null;
         }).anyTimes();
 
+        PowerMock.mockStatic(ClientMetrics.class);
+        EasyMock.expect(ClientMetrics.version()).andReturn("1.56");
+        EasyMock.expect(ClientMetrics.commitId()).andReturn("1a2b3c4d5e");
+        ClientMetrics.addVersionMetric(anyObject(StreamsMetricsImpl.class));
+        ClientMetrics.addCommitIdMetric(anyObject(StreamsMetricsImpl.class));
+        ClientMetrics.addApplicationIdMetric(anyObject(StreamsMetricsImpl.class), EasyMock.eq(APPLICATION_ID));
+        ClientMetrics.addTopologyDescriptionMetric(anyObject(StreamsMetricsImpl.class), anyString());
+        ClientMetrics.addStateMetric(anyObject(StreamsMetricsImpl.class), anyObject());
+
         // setup stream threads
         PowerMock.mockStatic(StreamThread.class);
         EasyMock.expect(StreamThread.create(
@@ -178,7 +190,7 @@ private void prepareStreams() throws Exception {
             anyObject(Admin.class),
             anyObject(UUID.class),
             anyObject(String.class),
-            anyObject(Metrics.class),
+            anyObject(StreamsMetricsImpl.class),
             anyObject(Time.class),
             anyObject(StreamsMetadataState.class),
             anyLong(),
@@ -203,11 +215,10 @@ private void prepareStreams() throws Exception {
             anyObject(Consumer.class),
             anyObject(StateDirectory.class),
             anyLong(),
-            anyObject(Metrics.class),
+            anyObject(StreamsMetricsImpl.class),
             anyObject(Time.class),
             anyString(),
-            anyObject(StateRestoreListener.class),
-            anyObject(RocksDBMetricsRecordingTrigger.class)
+            anyObject(StateRestoreListener.class)
         ).andReturn(globalStreamThread).anyTimes();
         EasyMock.expect(globalStreamThread.state()).andAnswer(globalThreadState::get).anyTimes();
         globalStreamThread.setStateListener(EasyMock.capture(threadStatelistenerCapture));
@@ -240,7 +251,16 @@ private void prepareStreams() throws Exception {
         globalStreamThread.join();
         EasyMock.expectLastCall().anyTimes();
 
-        PowerMock.replay(StreamThread.class, Metrics.class, metrics, streamThreadOne, streamThreadTwo, GlobalStreamThread.class, globalStreamThread);
+        PowerMock.replay(
+            StreamThread.class,
+            Metrics.class,
+            metrics,
+            ClientMetrics.class,
+            streamThreadOne,
+            streamThreadTwo,
+            GlobalStreamThread.class,
+            globalStreamThread
+        );
     }
 
     private void prepareStreamThread(final StreamThread thread, final boolean terminable) throws Exception {
@@ -249,8 +269,6 @@ private void prepareStreamThread(final StreamThread thread, final boolean termin
 
         thread.setStateListener(EasyMock.capture(threadStatelistenerCapture));
         EasyMock.expectLastCall().anyTimes();
-        thread.setRocksDBMetricsRecordingTrigger(EasyMock.anyObject(RocksDBMetricsRecordingTrigger.class));
-        EasyMock.expectLastCall().anyTimes();
 
         thread.start();
         EasyMock.expectLastCall().andAnswer(() -> {
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
index 88659c3559886..6030ac67f6778 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
@@ -29,6 +29,7 @@
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.Topology;
 import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
 import org.apache.kafka.streams.integration.utils.IntegrationTestUtils;
 import org.apache.kafka.streams.kstream.Consumed;
@@ -69,8 +70,10 @@ public class MetricsIntegrationTest {
     public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(NUM_BROKERS);
     private final long timeout = 60000;
 
+    private final static String APPLICATION_ID_VALUE = "stream-metrics-test";
 
     // Metric group
+    private static final String STREAM_CLIENT_NODE_METRICS = "stream-metrics";
     private static final String STREAM_THREAD_NODE_METRICS = "stream-metrics";
     private static final String STREAM_TASK_NODE_METRICS = "stream-task-metrics";
     private static final String STREAM_PROCESSOR_NODE_METRICS = "stream-processor-node-metrics";
@@ -82,6 +85,11 @@ public class MetricsIntegrationTest {
     private static final String STREAM_STORE_SESSION_ROCKSDB_STATE_METRICS = "stream-rocksdb-session-state-metrics";
 
     // Metrics name
+    private static final String VERSION = "version";
+    private static final String COMMIT_ID = "commit-id";
+    private static final String APPLICATION_ID = "application-id";
+    private static final String TOPOLOGY_DESCRIPTION = "topology-description";
+    private static final String STATE = "state";
     private static final String PUT_LATENCY_AVG = "put-latency-avg";
     private static final String PUT_LATENCY_MAX = "put-latency-max";
     private static final String PUT_IF_ABSENT_LATENCY_AVG = "put-if-absent-latency-avg";
@@ -205,7 +213,7 @@ public void before() throws InterruptedException {
         builder = new StreamsBuilder();
         CLUSTER.createTopics(STREAM_INPUT, STREAM_OUTPUT_1, STREAM_OUTPUT_2, STREAM_OUTPUT_3, STREAM_OUTPUT_4);
         streamsConfiguration = new Properties();
-        streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, "stream-metrics-test");
+        streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, APPLICATION_ID_VALUE);
         streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
         streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass());
         streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
@@ -220,7 +228,13 @@ public void after() throws InterruptedException {
     }
 
     private void startApplication() throws InterruptedException {
-        kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration);
+        final Topology topology = builder.build();
+        kafkaStreams = new KafkaStreams(topology, streamsConfiguration);
+
+        verifyStateMetric(State.CREATED);
+        verifyTopologyDescriptionMetric(topology.describe().toString());
+        verifyApplicationIdMetric(APPLICATION_ID_VALUE);
+
         kafkaStreams.start();
         TestUtils.waitForCondition(
             () -> kafkaStreams.state() == State.RUNNING,
@@ -306,6 +320,7 @@ private void shouldAddMetricsOnAllLevels(final String builtInMetricsVersion) thr
             .to(STREAM_OUTPUT_4);
         startApplication();
 
+        verifyStateMetric(State.RUNNING);
         checkThreadLevelMetrics();
         checkTaskLevelMetrics();
         checkProcessorLevelMetrics();
@@ -339,6 +354,8 @@ public void shouldAddMetricsForWindowStore() throws Exception {
 
         startApplication();
 
+        verifyStateMetric(State.RUNNING);
+
         waitUntilAllRecordsAreConsumed();
 
         checkWindowStoreMetrics();
@@ -369,6 +386,8 @@ public void shouldAddMetricsForSessionStore() throws Exception {
 
         startApplication();
 
+        verifyStateMetric(State.RUNNING);
+
         waitUntilAllRecordsAreConsumed();
 
         checkSessionStoreMetrics();
@@ -401,10 +420,44 @@ public void shouldNotAddRocksDBMetricsIfRecordingLevelIsInfo() throws Exception
         closeApplication();
     }
 
+    private void verifyStateMetric(final State state) {
+        final List metricsList = new ArrayList(kafkaStreams.metrics().values()).stream()
+            .filter(m -> m.metricName().name().equals(STATE) &&
+                m.metricName().group().equals(STREAM_CLIENT_NODE_METRICS))
+            .collect(Collectors.toList());
+        assertThat(metricsList.size(), is(1));
+        assertThat(metricsList.get(0).metricValue(), is(state));
+    }
+
+    private void verifyTopologyDescriptionMetric(final String topologyDescription) {
+        final List metricsList = new ArrayList(kafkaStreams.metrics().values()).stream()
+            .filter(m -> m.metricName().name().equals(TOPOLOGY_DESCRIPTION) &&
+                m.metricName().group().equals(STREAM_CLIENT_NODE_METRICS))
+            .collect(Collectors.toList());
+        assertThat(metricsList.size(), is(1));
+        assertThat(metricsList.get(0).metricValue(), is(topologyDescription));
+    }
+
+    private void verifyApplicationIdMetric(final String applicationId) {
+        final List metricsList = new ArrayList(kafkaStreams.metrics().values()).stream()
+            .filter(m -> m.metricName().name().equals(APPLICATION_ID) &&
+                m.metricName().group().equals(STREAM_CLIENT_NODE_METRICS))
+            .collect(Collectors.toList());
+        assertThat(metricsList.size(), is(1));
+        assertThat(metricsList.get(0).metricValue(), is(applicationId));
+    }
+
     private void checkThreadLevelMetrics() {
         final List listMetricThread = new ArrayList(kafkaStreams.metrics().values()).stream()
             .filter(m -> m.metricName().group().equals(STREAM_THREAD_NODE_METRICS))
             .collect(Collectors.toList());
+        // instance-level metrics start
+        checkMetricByName(listMetricThread, VERSION, 1);
+        checkMetricByName(listMetricThread, COMMIT_ID, 1);
+        checkMetricByName(listMetricThread, APPLICATION_ID, 1);
+        checkMetricByName(listMetricThread, TOPOLOGY_DESCRIPTION, 1);
+        checkMetricByName(listMetricThread, STATE, 1);
+        // instance-level metrics end
         checkMetricByName(listMetricThread, COMMIT_LATENCY_AVG, 1);
         checkMetricByName(listMetricThread, COMMIT_LATENCY_MAX, 1);
         checkMetricByName(listMetricThread, POLL_LATENCY_AVG, 1);
diff --git a/streams/src/test/java/org/apache/kafka/streams/internals/metrics/ClientMetricsTest.java b/streams/src/test/java/org/apache/kafka/streams/internals/metrics/ClientMetricsTest.java
new file mode 100644
index 0000000000000..20f5ac85e4d8b
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/internals/metrics/ClientMetricsTest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.internals.metrics;
+
+
+import org.apache.kafka.common.metrics.Gauge;
+import org.apache.kafka.common.metrics.Sensor.RecordingLevel;
+import org.apache.kafka.streams.KafkaStreams.State;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.junit.Test;
+
+import static org.easymock.EasyMock.and;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.mock;
+import static org.easymock.EasyMock.not;
+import static org.easymock.EasyMock.notNull;
+import static org.powermock.api.easymock.PowerMock.replay;
+import static org.powermock.api.easymock.PowerMock.verify;
+
+public class ClientMetricsTest {
+
+    private final StreamsMetricsImpl streamsMetrics = mock(StreamsMetricsImpl.class);
+
+    private interface OneParamMetricAdder {
+        void addMetric(final StreamsMetricsImpl streamsMetrics);
+    }
+
+    private interface TwoParamMetricAdder {
+        void addMetric(final StreamsMetricsImpl streamsMetrics, final String value);
+    }
+
+    /*
+     * This test may fail when executed from a IDE since it expects the /kafka/kafka-streams-version.properties on the
+     * class path.
+     */
+    @Test
+    public void shouldAddVersionMetric() {
+        final String name = "version";
+        final String description = "The version of the Kafka Streams client";
+        setUpAndVerifyMetricOneParam(name, description, ClientMetrics::addVersionMetric);
+    }
+
+    /*
+     * This test may fail when executed from a IDE since it expects the /kafka/kafka-streams-version.properties on the
+     * class path.
+     */
+    @Test
+    public void shouldAddCommitIdMetric() {
+        final String name = "commit-id";
+        final String description = "The version control commit ID of the Kafka Streams client";
+        setUpAndVerifyMetricOneParam(name, description, ClientMetrics::addCommitIdMetric);
+    }
+
+    @Test
+    public void shouldAddApplicationIdMetric() {
+        final String name = "application-id";
+        final String description = "The application ID of the Kafka Streams client";
+        setUpAndVerifyMetricTwoParam(name, description, "thisIsAnID", ClientMetrics::addApplicationIdMetric);
+    }
+
+    @Test
+    public void shouldAddTopologyDescriptionMetric() {
+        final String name = "topology-description";
+        final String description = "The description of the topology executed in the Kafka Streams client";
+        setUpAndVerifyMetricTwoParam(
+            name,
+            description,
+            "thisIsATopologyDescription",
+            ClientMetrics::addTopologyDescriptionMetric
+        );
+    }
+
+    @Test
+    public void shouldAddStateMetric() {
+        final String name = "state";
+        final String description = "The state of the Kafka Streams client";
+        final Gauge stateProvider = (config, now) -> State.RUNNING;
+        streamsMetrics.addClientLevelMutableMetric(
+            eq(name),
+            eq(description),
+            eq(RecordingLevel.INFO),
+            eq(stateProvider)
+        );
+        replay(streamsMetrics);
+
+        ClientMetrics.addStateMetric(streamsMetrics, stateProvider);
+
+        verify(streamsMetrics);
+    }
+
+    private void setUpAndVerifyMetricOneParam(final String name,
+                                              final String description,
+                                              final OneParamMetricAdder metricAdder) {
+        streamsMetrics.addClientLevelImmutableMetric(
+            eq(name),
+            eq(description),
+            eq(RecordingLevel.INFO),
+            and(not(eq("unknown")), notNull())
+        );
+        replay(streamsMetrics);
+
+        metricAdder.addMetric(streamsMetrics);
+
+        verify(streamsMetrics);
+    }
+
+    private void setUpAndVerifyMetricTwoParam(final String name,
+                                              final String description,
+                                              final String value,
+                                              final TwoParamMetricAdder metricAdder) {
+        streamsMetrics.addClientLevelImmutableMetric(
+            eq(name),
+            eq(description),
+            eq(RecordingLevel.INFO),
+            eq(value)
+        );
+        replay(streamsMetrics);
+
+        metricAdder.addMetric(streamsMetrics, value);
+
+        verify(streamsMetrics);
+    }
+}
\ No newline at end of file
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregateProcessorTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregateProcessorTest.java
index 5b207372964fc..4230958fc4a6d 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregateProcessorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregateProcessorTest.java
@@ -70,6 +70,7 @@ public class KStreamSessionWindowAggregateProcessorTest {
     private static final long GAP_MS = 5 * 60 * 1000L;
     private static final String STORE_NAME = "session-store";
 
+    private final String threadId = Thread.currentThread().getName();
     private final ToInternal toInternal = new ToInternal();
     private final Initializer initializer = () -> 0L;
     private final Aggregator aggregator = (aggKey, value, aggregate) -> aggregate + 1;
@@ -93,7 +94,7 @@ public void initializeStore() {
         final File stateDir = TestUtils.tempDirectory();
         metrics = new Metrics();
         final MockStreamsMetrics metrics = new MockStreamsMetrics(KStreamSessionWindowAggregateProcessorTest.this.metrics);
-        ThreadMetrics.skipRecordSensor(metrics);
+        ThreadMetrics.skipRecordSensor(threadId, metrics);
 
         context = new InternalMockProcessorContext(
             stateDir,
@@ -420,7 +421,7 @@ public void shouldLogAndMeterWhenSkippingLateRecordWithZeroGrace() {
             "stream-processor-node-metrics",
             "The total number of occurrence of late-record-drop operations.",
             mkMap(
-                mkEntry("client-id", "test"),
+                mkEntry("client-id", threadId),
                 mkEntry("task-id", "0_0"),
                 mkEntry("processor-node-id", "TESTING_NODE")
             )
@@ -433,7 +434,7 @@ public void shouldLogAndMeterWhenSkippingLateRecordWithZeroGrace() {
             "stream-processor-node-metrics",
             "The average number of occurrence of late-record-drop operations.",
             mkMap(
-                mkEntry("client-id", "test"),
+                mkEntry("client-id", threadId),
                 mkEntry("task-id", "0_0"),
                 mkEntry("processor-node-id", "TESTING_NODE")
             )
@@ -494,7 +495,7 @@ public void shouldLogAndMeterWhenSkippingLateRecordWithNonzeroGrace() {
             "stream-processor-node-metrics",
             "The total number of occurrence of late-record-drop operations.",
             mkMap(
-                mkEntry("client-id", "test"),
+                mkEntry("client-id", threadId),
                 mkEntry("task-id", "0_0"),
                 mkEntry("processor-node-id", "TESTING_NODE")
             )
@@ -507,7 +508,7 @@ public void shouldLogAndMeterWhenSkippingLateRecordWithNonzeroGrace() {
             "stream-processor-node-metrics",
             "The average number of occurrence of late-record-drop operations.",
             mkMap(
-                mkEntry("client-id", "test"),
+                mkEntry("client-id", threadId),
                 mkEntry("task-id", "0_0"),
                 mkEntry("processor-node-id", "TESTING_NODE")
             )
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java
index 61398201e5f56..9051ea69438a0 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java
@@ -65,6 +65,7 @@ public class KStreamWindowAggregateTest {
     private final ConsumerRecordFactory recordFactory =
         new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String());
+    private final String threadId = Thread.currentThread().getName();
 
     @Test
     public void testAggBasic() {
@@ -394,7 +395,7 @@ private void assertLatenessMetrics(final TopologyTestDriver driver,
             "stream-processor-node-metrics",
             "The total number of occurrence of late-record-drop operations.",
             mkMap(
-                mkEntry("client-id", "topology-test-driver-virtual-thread"),
+                mkEntry("client-id", threadId),
                 mkEntry("task-id", "0_0"),
                 mkEntry("processor-node-id", "KSTREAM-AGGREGATE-0000000001")
             )
@@ -406,7 +407,7 @@ private void assertLatenessMetrics(final TopologyTestDriver driver,
             "stream-processor-node-metrics",
             "The average number of occurrence of late-record-drop operations.",
             mkMap(
-                mkEntry("client-id", "topology-test-driver-virtual-thread"),
+                mkEntry("client-id", threadId),
                 mkEntry("task-id", "0_0"),
                 mkEntry("processor-node-id", "KSTREAM-AGGREGATE-0000000001")
             )
@@ -418,7 +419,7 @@ private void assertLatenessMetrics(final TopologyTestDriver driver,
             "stream-task-metrics",
             "The max observed lateness of records.",
             mkMap(
-                mkEntry("client-id", "topology-test-driver-virtual-thread"),
+                mkEntry("client-id", threadId),
                 mkEntry("task-id", "0_0")
             )
         );
@@ -429,7 +430,7 @@ private void assertLatenessMetrics(final TopologyTestDriver driver,
             "stream-task-metrics",
             "The average observed lateness of records.",
             mkMap(
-                mkEntry("client-id", "topology-test-driver-virtual-thread"),
+                mkEntry("client-id", threadId),
                 mkEntry("task-id", "0_0")
             )
         );
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorMetricsTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorMetricsTest.java
index ef46663309099..3fd75fffc2cc8 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorMetricsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/suppress/KTableSuppressProcessorMetricsTest.java
@@ -43,90 +43,91 @@
 
 public class KTableSuppressProcessorMetricsTest {
     private static final long ARBITRARY_LONG = 5L;
+    private final String threadId = Thread.currentThread().getName();
 
-    private static final MetricName EVICTION_TOTAL_METRIC = new MetricName(
+    private final MetricName evictionTotalMetric = new MetricName(
         "suppression-emit-total",
         "stream-processor-node-metrics",
         "The total number of occurrence of suppression-emit operations.",
         mkMap(
-            mkEntry("client-id", "mock-processor-context-virtual-thread"),
+            mkEntry("client-id", threadId),
             mkEntry("task-id", "0_0"),
             mkEntry("processor-node-id", "testNode")
         )
     );
 
-    private static final MetricName EVICTION_RATE_METRIC = new MetricName(
+    private final MetricName evictionRateMetric = new MetricName(
         "suppression-emit-rate",
         "stream-processor-node-metrics",
         "The average number of occurrence of suppression-emit operation per second.",
         mkMap(
-            mkEntry("client-id", "mock-processor-context-virtual-thread"),
+            mkEntry("client-id", threadId),
             mkEntry("task-id", "0_0"),
             mkEntry("processor-node-id", "testNode")
         )
     );
 
-    private static final MetricName BUFFER_SIZE_AVG_METRIC = new MetricName(
+    private final MetricName bufferSizeAvgMetric = new MetricName(
         "suppression-buffer-size-avg",
         "stream-buffer-metrics",
         "The average size of buffered records.",
         mkMap(
-            mkEntry("client-id", "mock-processor-context-virtual-thread"),
+            mkEntry("client-id", threadId),
             mkEntry("task-id", "0_0"),
             mkEntry("buffer-id", "test-store")
         )
     );
 
-    private static final MetricName BUFFER_SIZE_CURRENT_METRIC = new MetricName(
+    private final MetricName bufferSizeCurrentMetric = new MetricName(
         "suppression-buffer-size-current",
         "stream-buffer-metrics",
         "The current size of buffered records.",
         mkMap(
-            mkEntry("client-id", "mock-processor-context-virtual-thread"),
+            mkEntry("client-id", threadId),
             mkEntry("task-id", "0_0"),
             mkEntry("buffer-id", "test-store")
         )
     );
 
-    private static final MetricName BUFFER_SIZE_MAX_METRIC = new MetricName(
+    private final MetricName bufferSizeMaxMetric = new MetricName(
         "suppression-buffer-size-max",
         "stream-buffer-metrics",
         "The max size of buffered records.",
         mkMap(
-            mkEntry("client-id", "mock-processor-context-virtual-thread"),
+            mkEntry("client-id", threadId),
             mkEntry("task-id", "0_0"),
             mkEntry("buffer-id", "test-store")
         )
     );
 
-    private static final MetricName BUFFER_COUNT_AVG_METRIC = new MetricName(
+    private final MetricName bufferCountAvgMetric = new MetricName(
         "suppression-buffer-count-avg",
         "stream-buffer-metrics",
         "The average count of buffered records.",
         mkMap(
-            mkEntry("client-id", "mock-processor-context-virtual-thread"),
+            mkEntry("client-id", threadId),
             mkEntry("task-id", "0_0"),
             mkEntry("buffer-id", "test-store")
         )
     );
 
-    private static final MetricName BUFFER_COUNT_CURRENT_METRIC = new MetricName(
+    private final MetricName bufferCountCurrentMetric = new MetricName(
         "suppression-buffer-count-current",
         "stream-buffer-metrics",
         "The current count of buffered records.",
         mkMap(
-            mkEntry("client-id", "mock-processor-context-virtual-thread"),
+            mkEntry("client-id", threadId),
             mkEntry("task-id", "0_0"),
             mkEntry("buffer-id", "test-store")
         )
     );
 
-    private static final MetricName BUFFER_COUNT_MAX_METRIC = new MetricName(
+    private final MetricName bufferCountMaxMetric = new MetricName(
         "suppression-buffer-count-max",
         "stream-buffer-metrics",
         "The max count of buffered records.",
         mkMap(
-            mkEntry("client-id", "mock-processor-context-virtual-thread"),
+            mkEntry("client-id", threadId),
             mkEntry("task-id", "0_0"),
             mkEntry("buffer-id", "test-store")
         )
@@ -166,14 +167,14 @@ public void shouldRecordMetrics() {
         {
             final Map metrics = context.metrics().metrics();
 
-            verifyMetric(metrics, EVICTION_RATE_METRIC, is(0.0));
-            verifyMetric(metrics, EVICTION_TOTAL_METRIC, is(0.0));
-            verifyMetric(metrics, BUFFER_SIZE_AVG_METRIC, is(21.5));
-            verifyMetric(metrics, BUFFER_SIZE_CURRENT_METRIC, is(43.0));
-            verifyMetric(metrics, BUFFER_SIZE_MAX_METRIC, is(43.0));
-            verifyMetric(metrics, BUFFER_COUNT_AVG_METRIC, is(0.5));
-            verifyMetric(metrics, BUFFER_COUNT_CURRENT_METRIC, is(1.0));
-            verifyMetric(metrics, BUFFER_COUNT_MAX_METRIC, is(1.0));
+            verifyMetric(metrics, evictionRateMetric, is(0.0));
+            verifyMetric(metrics, evictionTotalMetric, is(0.0));
+            verifyMetric(metrics, bufferSizeAvgMetric, is(21.5));
+            verifyMetric(metrics, bufferSizeCurrentMetric, is(43.0));
+            verifyMetric(metrics, bufferSizeMaxMetric, is(43.0));
+            verifyMetric(metrics, bufferCountAvgMetric, is(0.5));
+            verifyMetric(metrics, bufferCountCurrentMetric, is(1.0));
+            verifyMetric(metrics, bufferCountMaxMetric, is(1.0));
         }
 
         context.setRecordMetadata("", 0, 1L, null, timestamp + 1);
@@ -182,14 +183,14 @@ public void shouldRecordMetrics() {
         {
             final Map metrics = context.metrics().metrics();
 
-            verifyMetric(metrics, EVICTION_RATE_METRIC, greaterThan(0.0));
-            verifyMetric(metrics, EVICTION_TOTAL_METRIC, is(1.0));
-            verifyMetric(metrics, BUFFER_SIZE_AVG_METRIC, is(41.0));
-            verifyMetric(metrics, BUFFER_SIZE_CURRENT_METRIC, is(39.0));
-            verifyMetric(metrics, BUFFER_SIZE_MAX_METRIC, is(82.0));
-            verifyMetric(metrics, BUFFER_COUNT_AVG_METRIC, is(1.0));
-            verifyMetric(metrics, BUFFER_COUNT_CURRENT_METRIC, is(1.0));
-            verifyMetric(metrics, BUFFER_COUNT_MAX_METRIC, is(2.0));
+            verifyMetric(metrics, evictionRateMetric, greaterThan(0.0));
+            verifyMetric(metrics, evictionTotalMetric, is(1.0));
+            verifyMetric(metrics, bufferSizeAvgMetric, is(41.0));
+            verifyMetric(metrics, bufferSizeCurrentMetric, is(39.0));
+            verifyMetric(metrics, bufferSizeMaxMetric, is(82.0));
+            verifyMetric(metrics, bufferCountAvgMetric, is(1.0));
+            verifyMetric(metrics, bufferCountCurrentMetric, is(1.0));
+            verifyMetric(metrics, bufferCountMaxMetric, is(2.0));
         }
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStreamThreadTest.java
index ce08a6ac83e21..ddb20c951ca47 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStreamThreadTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStreamThreadTest.java
@@ -34,6 +34,7 @@
 import org.apache.kafka.streams.kstream.internals.TimestampedKeyValueStoreMaterializer;
 import org.apache.kafka.streams.kstream.internals.MaterializedInternal;
 import org.apache.kafka.streams.processor.StateStore;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.apache.kafka.streams.state.KeyValueStore;
 import org.apache.kafka.test.MockStateRestoreListener;
 import org.apache.kafka.test.TestUtils;
@@ -107,11 +108,10 @@ public String newStoreName(final String prefix) {
             mockConsumer,
             new StateDirectory(config, time, true),
             0,
-            new Metrics(),
+            new StreamsMetricsImpl(new Metrics(), "test-client", StreamsConfig.METRICS_LATEST),
             new MockTime(),
             "clientId",
-            stateRestoreListener,
-            null
+            stateRestoreListener
         );
     }
 
@@ -143,11 +143,10 @@ public List partitionsFor(final String topic) {
             mockConsumer,
             new StateDirectory(config, time, true),
             0,
-            new Metrics(),
+            new StreamsMetricsImpl(new Metrics(), "test-client", StreamsConfig.METRICS_LATEST),
             new MockTime(),
             "clientId",
-            stateRestoreListener,
-            null
+            stateRestoreListener
         );
 
         try {
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorNodeTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorNodeTest.java
index c3d85832d5b18..42ad94e23cb0b 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorNodeTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorNodeTest.java
@@ -104,13 +104,14 @@ public void testMetrics() {
         final ProcessorNode node = new ProcessorNode<>("name", new NoOpProcessor(), Collections.emptySet());
         node.init(context);
 
+        final String threadId = Thread.currentThread().getName();
         final String[] latencyOperations = {"process", "punctuate", "create", "destroy"};
         final String throughputOperation = "forward";
         final String groupName = "stream-processor-node-metrics";
         final Map metricTags = new LinkedHashMap<>();
         metricTags.put("processor-node-id", node.name());
         metricTags.put("task-id", context.taskId().toString());
-        metricTags.put("client-id", "mock");
+        metricTags.put("client-id", threadId);
 
         for (final String opName : latencyOperations) {
             StreamsTestUtils.getMetricByNameFilterByTags(metrics.metrics(), opName + "-latency-avg", groupName, metricTags);
@@ -137,8 +138,8 @@ public void testMetrics() {
 
         final JmxReporter reporter = new JmxReporter("kafka.streams");
         metrics.addReporter(reporter);
-        assertTrue(reporter.containsMbean(String.format("kafka.streams:type=%s,client-id=mock,task-id=%s,processor-node-id=%s",
-                groupName, context.taskId().toString(), node.name())));
+        assertTrue(reporter.containsMbean(String.format("kafka.streams:type=%s,client-id=%s,task-id=%s,processor-node-id=%s",
+                groupName, threadId, context.taskId().toString(), node.name())));
     }
 
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
index 6f42fb26526e6..88d4d6f46cf7c 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
@@ -97,6 +97,7 @@
 
 public class StandbyTaskTest {
 
+    private final String threadId = Thread.currentThread().getName();
     private final TaskId taskId = new TaskId(0, 1);
     private StandbyTask task;
     private final Serializer intSerializer = new IntegerSerializer();
@@ -782,7 +783,7 @@ void closeStateManager(final boolean clean) throws ProcessorStateException {
 
     private MetricName setupCloseTaskMetric() {
         final MetricName metricName = new MetricName("name", "group", "description", Collections.emptyMap());
-        final Sensor sensor = streamsMetrics.threadLevelSensor("task-closed", Sensor.RecordingLevel.INFO);
+        final Sensor sensor = streamsMetrics.threadLevelSensor(threadId, "task-closed", Sensor.RecordingLevel.INFO);
         sensor.add(metricName, new CumulativeSum());
         return metricName;
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
index a4f529d8795f1..bda86c4423eaf 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
@@ -414,10 +414,18 @@ public void testMetrics() {
         assertNotNull(getMetric("%s-latency-max", "The max latency of %s operation.", "all"));
         assertNotNull(getMetric("%s-rate", "The average number of occurrence of %s operation per second.", "all"));
 
+        final String threadId = Thread.currentThread().getName();
         final JmxReporter reporter = new JmxReporter("kafka.streams");
         metrics.addReporter(reporter);
-        assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-task-metrics,client-id=test,task-id=%s", task.id.toString())));
-        assertTrue(reporter.containsMbean("kafka.streams:type=stream-task-metrics,client-id=test,task-id=all"));
+        assertTrue(reporter.containsMbean(String.format(
+            "kafka.streams:type=stream-task-metrics,client-id=%s,task-id=%s",
+            threadId,
+            task.id.toString()
+        )));
+        assertTrue(reporter.containsMbean(String.format(
+            "kafka.streams:type=stream-task-metrics,client-id=%s,task-id=all",
+            threadId
+        )));
     }
 
     private KafkaMetric getMetric(final String nameFormat, final String descriptionFormat, final String taskId) {
@@ -425,7 +433,7 @@ private KafkaMetric getMetric(final String nameFormat, final String descriptionF
             String.format(nameFormat, "commit"),
             "stream-task-metrics",
             String.format(descriptionFormat, "commit"),
-            mkMap(mkEntry("task-id", taskId), mkEntry("client-id", "test"))
+            mkMap(mkEntry("task-id", taskId), mkEntry("client-id", Thread.currentThread().getName()))
         ));
     }
 
@@ -760,7 +768,11 @@ public void shouldBeProcessableIfWaitedForTooLong() {
         task.initializeStateStores();
         task.initializeTopology();
 
-        final MetricName enforcedProcessMetric = metrics.metricName("enforced-processing-total", "stream-task-metrics", mkMap(mkEntry("client-id", "test"), mkEntry("task-id", taskId00.toString())));
+        final MetricName enforcedProcessMetric = metrics.metricName(
+            "enforced-processing-total",
+            "stream-task-metrics",
+            mkMap(mkEntry("client-id", Thread.currentThread().getName()), mkEntry("task-id", taskId00.toString()))
+        );
 
         assertFalse(task.isProcessable(0L));
         assertEquals(0.0, metrics.metric(enforcedProcessMetric).metricValue());
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
index ef5b95b979afe..8d10e4cf2c3aa 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
@@ -101,6 +101,7 @@
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.not;
 import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.CoreMatchers.startsWith;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -112,11 +113,15 @@
 
 public class StreamThreadTest {
 
-    private final String clientId = "clientId";
-    private final String applicationId = "stream-thread-test";
+    private final static String APPLICATION_ID = "stream-thread-test";
+    private final static UUID PROCESS_ID = UUID.fromString("87bf53a8-54f2-485f-a4b6-acdbec0a8b3d");
+    private final static String CLIENT_ID = APPLICATION_ID + "-" + PROCESS_ID;
+
     private final int threadIdx = 1;
     private final MockTime mockTime = new MockTime();
     private final Metrics metrics = new Metrics();
+    private final StreamsMetricsImpl streamsMetrics =
+        new StreamsMetricsImpl(metrics, APPLICATION_ID, StreamsConfig.METRICS_LATEST);
     private final MockClientSupplier clientSupplier = new MockClientSupplier();
     private final InternalStreamsBuilder internalStreamsBuilder = new InternalStreamsBuilder(new InternalTopologyBuilder());
     private final StreamsConfig config = new StreamsConfig(configProps(false));
@@ -124,16 +129,14 @@ public class StreamThreadTest {
     private final StateDirectory stateDirectory = new StateDirectory(config, mockTime, true);
     private final ConsumedInternal consumed = new ConsumedInternal<>();
 
-    private UUID processId = UUID.randomUUID();
     private InternalTopologyBuilder internalTopologyBuilder;
     private StreamsMetadataState streamsMetadataState;
 
     @Before
     public void setUp() {
-        processId = UUID.randomUUID();
-
+        Thread.currentThread().setName(CLIENT_ID + "-StreamThread-" + threadIdx);
         internalTopologyBuilder = InternalStreamsBuilderTest.internalTopologyBuilder(internalStreamsBuilder);
-        internalTopologyBuilder.setApplicationId(applicationId);
+        internalTopologyBuilder.setApplicationId(APPLICATION_ID);
         streamsMetadataState = new StreamsMetadataState(internalTopologyBuilder, StreamsMetadataState.UNKNOWN_HOST);
     }
 
@@ -151,7 +154,7 @@ public void setUp() {
 
     private Properties configProps(final boolean enableEoS) {
         return mkProperties(mkMap(
-            mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, applicationId),
+            mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, APPLICATION_ID),
             mkEntry(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:2171"),
             mkEntry(StreamsConfig.BUFFERED_RECORDS_PER_PARTITION_CONFIG, "3"),
             mkEntry(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, MockTimestampExtractor.class.getName()),
@@ -164,7 +167,7 @@ private Properties configProps(final boolean enableEoS) {
     public void testPartitionAssignmentChangeForSingleGroup() {
         internalTopologyBuilder.addSource(null, "source1", null, null, null, topic1);
 
-        final StreamThread thread = createStreamThread(clientId, config, false);
+        final StreamThread thread = createStreamThread(CLIENT_ID, config, false);
 
         final StateListenerStub stateListener = new StateListenerStub();
         thread.setStateListener(stateListener);
@@ -201,7 +204,7 @@ public void testPartitionAssignmentChangeForSingleGroup() {
 
     @Test
     public void testStateChangeStartClose() throws Exception {
-        final StreamThread thread = createStreamThread(clientId, config, false);
+        final StreamThread thread = createStreamThread(CLIENT_ID, config, false);
 
         final StateListenerStub stateListener = new StateListenerStub();
         thread.setStateListener(stateListener);
@@ -238,7 +241,7 @@ private StreamThread createStreamThread(@SuppressWarnings("SameParameterValue")
                                             final StreamsConfig config,
                                             final boolean eosEnabled) {
         if (eosEnabled) {
-            clientSupplier.setApplicationIdForProducer(applicationId);
+            clientSupplier.setApplicationIdForProducer(APPLICATION_ID);
         }
 
         clientSupplier.setClusterForAdminClient(createCluster());
@@ -248,9 +251,9 @@ private StreamThread createStreamThread(@SuppressWarnings("SameParameterValue")
             config,
             clientSupplier,
             clientSupplier.getAdmin(config.getAdminConfigs(clientId)),
-            processId,
+            PROCESS_ID,
             clientId,
-            metrics,
+            streamsMetrics,
             mockTime,
             streamsMetadataState,
             0,
@@ -262,7 +265,7 @@ private StreamThread createStreamThread(@SuppressWarnings("SameParameterValue")
 
     @Test
     public void testMetricsCreatedAtStartup() {
-        final StreamThread thread = createStreamThread(clientId, config, false);
+        final StreamThread thread = createStreamThread(CLIENT_ID, config, false);
         final String defaultGroupName = "stream-metrics";
         final Map defaultTags = Collections.singletonMap("client-id", thread.getName());
         final String descriptionIsNotVerified = "";
@@ -324,7 +327,7 @@ public void testMetricsCreatedAtStartup() {
 
         final JmxReporter reporter = new JmxReporter("kafka.streams");
         metrics.addReporter(reporter);
-        assertEquals(clientId + "-StreamThread-1", thread.getName());
+        assertEquals(CLIENT_ID + "-StreamThread-1", thread.getName());
         assertTrue(reporter.containsMbean(String.format("kafka.streams:type=%s,client-id=%s",
                    defaultGroupName, 
                    thread.getName())));
@@ -342,7 +345,7 @@ public void shouldNotCommitBeforeTheCommitInterval() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
         final TaskManager taskManager = mockTaskManagerCommit(consumer, 1, 1);
 
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId, StreamsConfig.METRICS_LATEST);
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -353,7 +356,7 @@ public void shouldNotCommitBeforeTheCommitInterval() {
             taskManager,
             streamsMetrics,
             internalTopologyBuilder,
-            clientId,
+            CLIENT_ID,
             new LogContext(""),
             new AtomicInteger()
         );
@@ -375,12 +378,12 @@ public void shouldRespectNumIterationsInMainLoop() {
 
         final Properties properties = new Properties();
         properties.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 100L);
-        final StreamsConfig config = new StreamsConfig(StreamsTestUtils.getStreamsConfig(applicationId,
+        final StreamsConfig config = new StreamsConfig(StreamsTestUtils.getStreamsConfig(APPLICATION_ID,
             "localhost:2171",
             Serdes.ByteArraySerde.class.getName(),
             Serdes.ByteArraySerde.class.getName(),
             properties));
-        final StreamThread thread = createStreamThread(clientId, config, false);
+        final StreamThread thread = createStreamThread(CLIENT_ID, config, false);
 
         thread.setState(StreamThread.State.STARTING);
         thread.setState(StreamThread.State.PARTITIONS_REVOKED);
@@ -470,7 +473,7 @@ public void shouldNotCauseExceptionIfNothingCommitted() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
         final TaskManager taskManager = mockTaskManagerCommit(consumer, 1, 0);
 
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId, StreamsConfig.METRICS_LATEST);
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -481,7 +484,7 @@ public void shouldNotCauseExceptionIfNothingCommitted() {
             taskManager,
             streamsMetrics,
             internalTopologyBuilder,
-            clientId,
+            CLIENT_ID,
             new LogContext(""),
             new AtomicInteger()
         );
@@ -505,7 +508,7 @@ public void shouldCommitAfterTheCommitInterval() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
         final TaskManager taskManager = mockTaskManagerCommit(consumer, 2, 1);
 
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId, StreamsConfig.METRICS_LATEST);
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -516,7 +519,7 @@ public void shouldCommitAfterTheCommitInterval() {
             taskManager,
             streamsMetrics,
             internalTopologyBuilder,
-            clientId,
+            CLIENT_ID,
             new LogContext(""),
             new AtomicInteger()
         );
@@ -543,7 +546,7 @@ public void shouldInjectSharedProducerForAllTasksUsingClientSupplierOnCreateIfEo
         internalTopologyBuilder.addSource(null, "source1", null, null, null, topic1);
         internalStreamsBuilder.buildAndOptimizeTopology();
 
-        final StreamThread thread = createStreamThread(clientId, config, false);
+        final StreamThread thread = createStreamThread(CLIENT_ID, config, false);
 
         thread.setState(StreamThread.State.STARTING);
         thread.rebalanceListener.onPartitionsRevoked(Collections.emptyList());
@@ -580,7 +583,7 @@ public void shouldInjectSharedProducerForAllTasksUsingClientSupplierOnCreateIfEo
     public void shouldInjectProducerPerTaskUsingClientSupplierOnCreateIfEosEnable() {
         internalTopologyBuilder.addSource(null, "source1", null, null, null, topic1);
 
-        final StreamThread thread = createStreamThread(clientId, new StreamsConfig(configProps(true)), true);
+        final StreamThread thread = createStreamThread(CLIENT_ID, new StreamsConfig(configProps(true)), true);
 
         thread.setState(StreamThread.State.STARTING);
         thread.rebalanceListener.onPartitionsRevoked(Collections.emptyList());
@@ -619,7 +622,7 @@ public void shouldInjectProducerPerTaskUsingClientSupplierOnCreateIfEosEnable()
     public void shouldCloseAllTaskProducersOnCloseIfEosEnabled() {
         internalTopologyBuilder.addSource(null, "source1", null, null, null, topic1);
 
-        final StreamThread thread = createStreamThread(clientId, new StreamsConfig(configProps(true)), true);
+        final StreamThread thread = createStreamThread(CLIENT_ID, new StreamsConfig(configProps(true)), true);
 
         thread.setState(StreamThread.State.STARTING);
         thread.rebalanceListener.onPartitionsRevoked(Collections.emptyList());
@@ -660,7 +663,7 @@ public void shouldShutdownTaskManagerOnClose() {
         EasyMock.replay(taskManager, consumer);
 
         final StreamsMetricsImpl streamsMetrics =
-            new StreamsMetricsImpl(metrics, clientId, StreamsConfig.METRICS_LATEST);
+            new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -671,10 +674,10 @@ public void shouldShutdownTaskManagerOnClose() {
             taskManager,
             streamsMetrics,
             internalTopologyBuilder,
-            clientId,
+            CLIENT_ID,
             new LogContext(""),
             new AtomicInteger()
-        ).updateThreadMetadata(getSharedAdminClientId(clientId));
+        ).updateThreadMetadata(getSharedAdminClientId(CLIENT_ID));
         thread.setStateListener(
             (t, newState, oldState) -> {
                 if (oldState == StreamThread.State.CREATED && newState == StreamThread.State.STARTING) {
@@ -694,7 +697,7 @@ public void shouldShutdownTaskManagerOnCloseWithoutStart() {
         EasyMock.replay(taskManager, consumer);
 
         final StreamsMetricsImpl streamsMetrics =
-            new StreamsMetricsImpl(metrics, clientId, StreamsConfig.METRICS_LATEST);
+            new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -705,10 +708,10 @@ public void shouldShutdownTaskManagerOnCloseWithoutStart() {
             taskManager,
             streamsMetrics,
             internalTopologyBuilder,
-            clientId,
+            CLIENT_ID,
             new LogContext(""),
             new AtomicInteger()
-        ).updateThreadMetadata(getSharedAdminClientId(clientId));
+        ).updateThreadMetadata(getSharedAdminClientId(CLIENT_ID));
         thread.shutdown();
         EasyMock.verify(taskManager);
     }
@@ -755,22 +758,24 @@ private void setStreamThread(final StreamThread streamThread) {
         final MockStreamThreadConsumer mockStreamThreadConsumer =
             new MockStreamThreadConsumer<>(OffsetResetStrategy.EARLIEST);
 
-        final TaskManager taskManager = new TaskManager(new MockChangelogReader(),
-                                                        processId,
-                                                        "log-prefix",
-                                                        mockStreamThreadConsumer,
-                                                        streamsMetadataState,
-                                                        null,
-                                                        null,
-                                                        null,
-                                                        new AssignedStreamsTasks(new LogContext()),
-                                                        new AssignedStandbyTasks(new LogContext()));
+        final TaskManager taskManager = new TaskManager(
+            new MockChangelogReader(),
+            PROCESS_ID,
+            "log-prefix",
+            mockStreamThreadConsumer,
+            streamsMetadataState,
+            null,
+            null,
+            null,
+            new AssignedStreamsTasks(new LogContext()),
+            new AssignedStandbyTasks(new LogContext())
+        );
         taskManager.setConsumer(mockStreamThreadConsumer);
         taskManager.setAssignmentMetadata(Collections.emptyMap(), Collections.emptyMap());
         taskManager.setPartitionsToTaskId(Collections.emptyMap());
 
         final StreamsMetricsImpl streamsMetrics =
-            new StreamsMetricsImpl(metrics, clientId, StreamsConfig.METRICS_LATEST);
+            new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -781,10 +786,10 @@ private void setStreamThread(final StreamThread streamThread) {
             taskManager,
             streamsMetrics,
             internalTopologyBuilder,
-            clientId,
+            CLIENT_ID,
             new LogContext(""),
             new AtomicInteger()
-        ).updateThreadMetadata(getSharedAdminClientId(clientId));
+        ).updateThreadMetadata(getSharedAdminClientId(CLIENT_ID));
 
         mockStreamThreadConsumer.setStreamThread(thread);
         mockStreamThreadConsumer.assign(assignedPartitions);
@@ -805,7 +810,7 @@ public void shouldOnlyShutdownOnce() {
         EasyMock.replay(taskManager, consumer);
 
         final StreamsMetricsImpl streamsMetrics =
-            new StreamsMetricsImpl(metrics, clientId, StreamsConfig.METRICS_LATEST);
+            new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -816,10 +821,10 @@ public void shouldOnlyShutdownOnce() {
             taskManager,
             streamsMetrics,
             internalTopologyBuilder,
-            clientId,
+            CLIENT_ID,
             new LogContext(""),
             new AtomicInteger()
-        ).updateThreadMetadata(getSharedAdminClientId(clientId));
+        ).updateThreadMetadata(getSharedAdminClientId(CLIENT_ID));
         thread.shutdown();
         // Execute the run method. Verification of the mock will check that shutdown was only done once
         thread.run();
@@ -831,7 +836,7 @@ public void shouldNotNullPointerWhenStandbyTasksAssignedAndNoStateStoresForTopol
         internalTopologyBuilder.addSource(null, "name", null, null, null, "topic");
         internalTopologyBuilder.addSink("out", "output", null, null, null, "name");
 
-        final StreamThread thread = createStreamThread(clientId, config, false);
+        final StreamThread thread = createStreamThread(CLIENT_ID, config, false);
 
         thread.setState(StreamThread.State.STARTING);
         thread.rebalanceListener.onPartitionsRevoked(Collections.emptyList());
@@ -852,7 +857,7 @@ public void shouldCloseTaskAsZombieAndRemoveFromActiveTasksIfProducerWasFencedWh
         internalTopologyBuilder.addSource(null, "source", null, null, null, topic1);
         internalTopologyBuilder.addSink("sink", "dummyTopic", null, null, null, "source");
 
-        final StreamThread thread = createStreamThread(clientId, new StreamsConfig(configProps(true)), true);
+        final StreamThread thread = createStreamThread(CLIENT_ID, new StreamsConfig(configProps(true)), true);
 
         final MockConsumer consumer = clientSupplier.consumer;
 
@@ -914,7 +919,7 @@ public void shouldCloseTaskAsZombieAndRemoveFromActiveTasksIfProducerWasFencedWh
 
     @Test
     public void shouldCloseTaskAsZombieAndRemoveFromActiveTasksIfProducerGotFencedInCommitTransactionWhenSuspendingTaks() {
-        final StreamThread thread = createStreamThread(clientId, new StreamsConfig(configProps(true)), true);
+        final StreamThread thread = createStreamThread(CLIENT_ID, new StreamsConfig(configProps(true)), true);
 
         internalTopologyBuilder.addSource(null, "name", null, null, null, topic1);
         internalTopologyBuilder.addSink("out", "output", null, null, null, "name");
@@ -953,7 +958,7 @@ public void shouldCloseTaskAsZombieAndRemoveFromActiveTasksIfProducerGotFencedIn
 
     @Test
     public void shouldCloseTaskAsZombieAndRemoveFromActiveTasksIfProducerGotFencedInCloseTransactionWhenSuspendingTasks() {
-        final StreamThread thread = createStreamThread(clientId, new StreamsConfig(configProps(true)), true);
+        final StreamThread thread = createStreamThread(CLIENT_ID, new StreamsConfig(configProps(true)), true);
 
         internalTopologyBuilder.addSource(null, "name", null, null, null, topic1);
         internalTopologyBuilder.addSink("out", "output", null, null, null, "name");
@@ -1015,7 +1020,7 @@ public void onChange(final Thread thread,
     public void shouldReturnActiveTaskMetadataWhileRunningState() {
         internalTopologyBuilder.addSource(null, "source", null, null, null, topic1);
 
-        final StreamThread thread = createStreamThread(clientId, config, false);
+        final StreamThread thread = createStreamThread(CLIENT_ID, config, false);
 
         thread.setState(StreamThread.State.STARTING);
         thread.rebalanceListener.onPartitionsRevoked(Collections.emptySet());
@@ -1047,11 +1052,11 @@ public void shouldReturnActiveTaskMetadataWhileRunningState() {
         assertTrue("#threadState() was: " + metadata.threadState() + "; expected either RUNNING, STARTING, PARTITIONS_REVOKED, PARTITIONS_ASSIGNED, or CREATED",
             Arrays.asList("RUNNING", "STARTING", "PARTITIONS_REVOKED", "PARTITIONS_ASSIGNED", "CREATED").contains(metadata.threadState()));
         final String threadName = metadata.threadName();
-        assertTrue(threadName.startsWith("clientId-StreamThread-"));
+        assertThat(threadName, startsWith(CLIENT_ID + "-StreamThread-" + threadIdx));
         assertEquals(threadName + "-consumer", metadata.consumerClientId());
         assertEquals(threadName + "-restore-consumer", metadata.restoreConsumerClientId());
         assertEquals(Collections.singleton(threadName + "-producer"), metadata.producerClientIds());
-        assertEquals("clientId-admin", metadata.adminClientId());
+        assertEquals(CLIENT_ID + "-admin", metadata.adminClientId());
     }
 
     @Test
@@ -1060,7 +1065,7 @@ public void shouldReturnStandbyTaskMetadataWhileRunningState() {
             .groupByKey().count(Materialized.as("count-one"));
 
         internalStreamsBuilder.buildAndOptimizeTopology();
-        final StreamThread thread = createStreamThread(clientId, config, false);
+        final StreamThread thread = createStreamThread(CLIENT_ID, config, false);
         final MockConsumer restoreConsumer = clientSupplier.restoreConsumer;
         restoreConsumer.updatePartitions(
             "stream-thread-test-count-one-changelog",
@@ -1106,8 +1111,8 @@ public void shouldReturnStandbyTaskMetadataWhileRunningState() {
     public void shouldUpdateStandbyTask() throws Exception {
         final String storeName1 = "count-one";
         final String storeName2 = "table-two";
-        final String changelogName1 = applicationId + "-" + storeName1 + "-changelog";
-        final String changelogName2 = applicationId + "-" + storeName2 + "-changelog";
+        final String changelogName1 = APPLICATION_ID + "-" + storeName1 + "-changelog";
+        final String changelogName2 = APPLICATION_ID + "-" + storeName2 + "-changelog";
         final TopicPartition partition1 = new TopicPartition(changelogName1, 1);
         final TopicPartition partition2 = new TopicPartition(changelogName2, 1);
         internalStreamsBuilder
@@ -1119,7 +1124,7 @@ public void shouldUpdateStandbyTask() throws Exception {
         internalStreamsBuilder.table(topic2, new ConsumedInternal<>(), materialized);
 
         internalStreamsBuilder.buildAndOptimizeTopology();
-        final StreamThread thread = createStreamThread(clientId, config, false);
+        final StreamThread thread = createStreamThread(CLIENT_ID, config, false);
         final MockConsumer restoreConsumer = clientSupplier.restoreConsumer;
         restoreConsumer.updatePartitions(changelogName1,
             singletonList(
@@ -1227,7 +1232,7 @@ private StandbyTask createStandbyTask() {
         final LogContext logContext = new LogContext("test");
         final Logger log = logContext.logger(StreamThreadTest.class);
         final StreamsMetricsImpl streamsMetrics =
-            new StreamsMetricsImpl(metrics, clientId, StreamsConfig.METRICS_LATEST);
+            new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST);
         final StreamThread.StandbyTaskCreator standbyTaskCreator = new StreamThread.StandbyTaskCreator(
             internalTopologyBuilder,
             config,
@@ -1235,6 +1240,7 @@ private StandbyTask createStandbyTask() {
             stateDirectory,
             new MockChangelogReader(),
             mockTime,
+            CLIENT_ID,
             log);
         return standbyTaskCreator.createTask(
             new MockConsumer<>(OffsetResetStrategy.EARLIEST),
@@ -1264,7 +1270,7 @@ public void close() {}
         internalStreamsBuilder.stream(Collections.singleton(topic1), consumed).process(punctuateProcessor);
         internalStreamsBuilder.buildAndOptimizeTopology();
 
-        final StreamThread thread = createStreamThread(clientId, config, false);
+        final StreamThread thread = createStreamThread(CLIENT_ID, config, false);
 
         thread.setState(StreamThread.State.STARTING);
         thread.rebalanceListener.onPartitionsRevoked(Collections.emptySet());
@@ -1321,7 +1327,7 @@ public void close() {}
 
     @Test
     public void shouldAlwaysUpdateTasksMetadataAfterChangingState() {
-        final StreamThread thread = createStreamThread(clientId, config, false);
+        final StreamThread thread = createStreamThread(CLIENT_ID, config, false);
         ThreadMetadata metadata = thread.threadMetadata();
         assertEquals(StreamThread.State.CREATED.name(), metadata.threadState());
 
@@ -1339,7 +1345,7 @@ public void shouldAlwaysReturnEmptyTasksMetadataWhileRebalancingStateAndTasksNot
             .groupByKey().count(Materialized.as("count-one"));
         internalStreamsBuilder.buildAndOptimizeTopology();
 
-        final StreamThread thread = createStreamThread(clientId, config, false);
+        final StreamThread thread = createStreamThread(CLIENT_ID, config, false);
         final MockConsumer restoreConsumer = clientSupplier.restoreConsumer;
         restoreConsumer.updatePartitions("stream-thread-test-count-one-changelog",
             asList(
@@ -1500,7 +1506,7 @@ public void shouldRecordSkippedMetricForDeserializationException() {
             StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG,
             LogAndContinueExceptionHandler.class.getName());
         config.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.Integer().getClass().getName());
-        final StreamThread thread = createStreamThread(clientId, new StreamsConfig(config), false);
+        final StreamThread thread = createStreamThread(CLIENT_ID, new StreamsConfig(config), false);
 
         thread.setState(StreamThread.State.STARTING);
         thread.setState(StreamThread.State.PARTITIONS_REVOKED);
@@ -1571,7 +1577,7 @@ public void shouldReportSkippedRecordsForInvalidTimestamps() {
         config.setProperty(
             StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG,
             LogAndSkipOnInvalidTimestamp.class.getName());
-        final StreamThread thread = createStreamThread(clientId, new StreamsConfig(config), false);
+        final StreamThread thread = createStreamThread(CLIENT_ID, new StreamsConfig(config), false);
 
         thread.setState(StreamThread.State.STARTING);
         thread.setState(StreamThread.State.PARTITIONS_REVOKED);
@@ -1672,21 +1678,21 @@ public void producerMetricsVerificationWithoutEOS() {
         final TaskManager taskManager = mockTaskManagerCommit(consumer, 1, 0);
 
         final StreamsMetricsImpl streamsMetrics =
-            new StreamsMetricsImpl(metrics, clientId, StreamsConfig.METRICS_LATEST);
+            new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
-                mockTime,
-                config,
-                producer,
-                consumer,
-                consumer,
-                null,
-                taskManager,
-                streamsMetrics,
-                internalTopologyBuilder,
-                clientId,
-                new LogContext(""),
-                new AtomicInteger()
-                );
+            mockTime,
+            config,
+            producer,
+            consumer,
+            consumer,
+            null,
+            taskManager,
+            streamsMetrics,
+            internalTopologyBuilder,
+            CLIENT_ID,
+            new LogContext(""),
+            new AtomicInteger()
+        );
         final MetricName testMetricName = new MetricName("test_metric", "", "", new HashMap<>());
         final Metric testMetric = new KafkaMetric(
             new Object(),
@@ -1711,21 +1717,21 @@ public void adminClientMetricsVerification() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
         final TaskManager taskManager = EasyMock.createNiceMock(TaskManager.class);
 
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, clientId, StreamsConfig.METRICS_LATEST);
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
-                mockTime,
-                config,
-                producer,
-                consumer,
-                consumer,
-                null,
-                taskManager,
-                streamsMetrics,
-                internalTopologyBuilder,
-                clientId,
-                new LogContext(""),
-                new AtomicInteger()
-                );
+            mockTime,
+            config,
+            producer,
+            consumer,
+            consumer,
+            null,
+            taskManager,
+            streamsMetrics,
+            internalTopologyBuilder,
+            CLIENT_ID,
+            new LogContext(""),
+            new AtomicInteger()
+        );
         final MetricName testMetricName = new MetricName("test_metric", "", "", new HashMap<>());
         final Metric testMetric = new KafkaMetric(
             new Object(),
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java
index b589119766166..69f7546d0e89e 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java
@@ -17,6 +17,7 @@
 package org.apache.kafka.streams.processor.internals.metrics;
 
 import org.apache.kafka.common.MetricName;
+import org.apache.kafka.common.metrics.Gauge;
 import org.apache.kafka.common.metrics.KafkaMetric;
 import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
@@ -24,9 +25,11 @@
 import org.apache.kafka.common.metrics.Sensor.RecordingLevel;
 import org.apache.kafka.common.utils.MockTime;
 import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.ImmutableMetricValue;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.Version;
 import org.easymock.EasyMock;
 import org.easymock.EasyMockSupport;
+import org.easymock.IArgumentMatcher;
 import org.junit.Test;
 
 import java.time.Duration;
@@ -36,54 +39,222 @@
 
 import static org.apache.kafka.common.utils.Utils.mkEntry;
 import static org.apache.kafka.common.utils.Utils.mkMap;
+import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.CLIENT_LEVEL_GROUP;
 import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.PROCESSOR_NODE_METRICS_GROUP;
 import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addAvgAndMaxLatencyToSensor;
 import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.addInvocationRateAndCountToSensor;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.anyString;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.resetToDefault;
+import static org.easymock.EasyMock.verify;
+import static org.hamcrest.CoreMatchers.equalToObject;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
 
 public class StreamsMetricsImplTest extends EasyMockSupport {
 
     private final static String SENSOR_PREFIX_DELIMITER = ".";
     private final static String SENSOR_NAME_DELIMITER = ".s.";
     private final static String INTERNAL_PREFIX = "internal";
-    private final static String THREAD_NAME = "test-thread";
     private final static String VERSION = StreamsConfig.METRICS_LATEST;
+    private final static String CLIENT_ID = "test-client";
+    private final static String THREAD_ID = "test-thread";
+    private final static String TASK_ID = "test-task";
+    private final static String METRIC_NAME1 = "test-metric1";
+    private final static String METRIC_NAME2 = "test-metric2";
 
     private final Metrics metrics = new Metrics();
     private final Sensor sensor = metrics.sensor("dummy");
+    private final String storeName = "store";
+    private final String sensorName1 = "sensor1";
+    private final String sensorName2 = "sensor2";
     private final String metricNamePrefix = "metric";
     private final String group = "group";
     private final Map tags = mkMap(mkEntry("tag", "value"));
     private final String description1 = "description number one";
     private final String description2 = "description number two";
     private final String description3 = "description number three";
+    private final Map clientLevelTags = mkMap(mkEntry("client-id", CLIENT_ID));
+    private final MetricName metricName1 =
+        new MetricName(METRIC_NAME1, CLIENT_LEVEL_GROUP, description1, clientLevelTags);
+    private final MetricName metricName2 =
+        new MetricName(METRIC_NAME1, CLIENT_LEVEL_GROUP, description2, clientLevelTags);
     private final MockTime time = new MockTime(0);
-    private final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, THREAD_NAME, VERSION);
+    private final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, VERSION);
+
+    private static MetricConfig eqMetricConfig(final MetricConfig metricConfig) {
+        EasyMock.reportMatcher(new IArgumentMatcher() {
+            private final StringBuffer message = new StringBuffer();
+
+            @Override
+            public boolean matches(final Object argument) {
+                if (argument instanceof MetricConfig) {
+                    final MetricConfig otherMetricConfig = (MetricConfig) argument;
+                    final boolean equalsComparisons =
+                        (otherMetricConfig.quota() == metricConfig.quota() ||
+                        otherMetricConfig.quota().equals(metricConfig.quota())) &&
+                        otherMetricConfig.tags().equals(metricConfig.tags());
+                    if (otherMetricConfig.eventWindow() == metricConfig.eventWindow() &&
+                        otherMetricConfig.recordLevel() == metricConfig.recordLevel() &&
+                        equalsComparisons &&
+                        otherMetricConfig.samples() == metricConfig.samples() &&
+                        otherMetricConfig.timeWindowMs() == metricConfig.timeWindowMs()) {
+
+                        return true;
+                    } else {
+                        message.append("{ ");
+                        message.append("eventWindow=");
+                        message.append(otherMetricConfig.eventWindow());
+                        message.append(", ");
+                        message.append("recordLevel=");
+                        message.append(otherMetricConfig.recordLevel());
+                        message.append(", ");
+                        message.append("quota=");
+                        message.append(otherMetricConfig.quota().toString());
+                        message.append(", ");
+                        message.append("samples=");
+                        message.append(otherMetricConfig.samples());
+                        message.append(", ");
+                        message.append("tags=");
+                        message.append(otherMetricConfig.tags().toString());
+                        message.append(", ");
+                        message.append("timeWindowMs=");
+                        message.append(otherMetricConfig.timeWindowMs());
+                        message.append(" }");
+                    }
+                }
+                message.append("not a MetricConfig object");
+                return false;
+            }
+
+            @Override
+            public void appendTo(final StringBuffer buffer) {
+                buffer.append(message);
+            }
+        });
+        return null;
+    }
+
+
+    private void addSensorsOnAllLevels(final Metrics metrics, final StreamsMetricsImpl streamsMetrics) {
+        expect(metrics.sensor(anyString(), anyObject(RecordingLevel.class), anyObject(Sensor[].class)))
+            .andStubReturn(sensor);
+        expect(metrics.metricName(METRIC_NAME1, CLIENT_LEVEL_GROUP, description1, clientLevelTags))
+            .andReturn(metricName1);
+        expect(metrics.metricName(METRIC_NAME2, CLIENT_LEVEL_GROUP, description2, clientLevelTags))
+            .andReturn(metricName2);
+        replay(metrics);
+        streamsMetrics.addClientLevelImmutableMetric(METRIC_NAME1, description1, RecordingLevel.INFO, "value");
+        streamsMetrics.addClientLevelImmutableMetric(METRIC_NAME2, description2, RecordingLevel.INFO, "value");
+        streamsMetrics.threadLevelSensor(THREAD_ID, sensorName1, RecordingLevel.INFO);
+        streamsMetrics.threadLevelSensor(THREAD_ID, sensorName2, RecordingLevel.INFO);
+        streamsMetrics.taskLevelSensor(THREAD_ID, TASK_ID, sensorName1, RecordingLevel.INFO);
+        streamsMetrics.taskLevelSensor(THREAD_ID, TASK_ID, sensorName2, RecordingLevel.INFO);
+        streamsMetrics.storeLevelSensor(THREAD_ID, TASK_ID, storeName, sensorName1, RecordingLevel.INFO);
+        streamsMetrics.storeLevelSensor(THREAD_ID, TASK_ID, storeName, sensorName2, RecordingLevel.INFO);
+    }
+
+    private void setupGetSensorTest(final Metrics metrics,
+                                    final String level,
+                                    final RecordingLevel recordingLevel) {
+        final String fullSensorName =
+            INTERNAL_PREFIX + SENSOR_PREFIX_DELIMITER + level + SENSOR_NAME_DELIMITER + sensorName1;
+        final Sensor[] parents = {};
+        expect(metrics.sensor(fullSensorName, recordingLevel, parents)).andReturn(sensor);
+        replay(metrics);
+    }
+
+    @Test
+    public void shouldAddClientLevelImmutableMetric() {
+        final Metrics metrics = mock(Metrics.class);
+        final RecordingLevel recordingLevel = RecordingLevel.INFO;
+        final MetricConfig metricConfig = new MetricConfig().recordLevel(recordingLevel);
+        final String value = "immutable-value";
+        final ImmutableMetricValue immutableValue = new ImmutableMetricValue<>(value);
+        expect(metrics.metricName(METRIC_NAME1, CLIENT_LEVEL_GROUP, description1, clientLevelTags))
+            .andReturn(metricName1);
+        metrics.addMetric(eq(metricName1), eqMetricConfig(metricConfig), eq(immutableValue));
+        replay(metrics);
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, VERSION);
+
+        streamsMetrics.addClientLevelImmutableMetric(METRIC_NAME1, description1, recordingLevel, value);
+
+        verify(metrics);
+    }
+
+    @Test
+    public void shouldAddClientLevelMutableMetric() {
+        final Metrics metrics = mock(Metrics.class);
+        final RecordingLevel recordingLevel = RecordingLevel.INFO;
+        final MetricConfig metricConfig = new MetricConfig().recordLevel(recordingLevel);
+        final Gauge valueProvider = (config, now) -> "mutable-value";
+        expect(metrics.metricName(METRIC_NAME1, CLIENT_LEVEL_GROUP, description1, clientLevelTags))
+            .andReturn(metricName1);
+        metrics.addMetric(eq(metricName1), eqMetricConfig(metricConfig), eq(valueProvider));
+        replay(metrics);
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, VERSION);
+
+        streamsMetrics.addClientLevelMutableMetric(METRIC_NAME1, description1, recordingLevel, valueProvider);
+
+        verify(metrics);
+    }
 
     @Test
     public void shouldGetThreadLevelSensor() {
         final Metrics metrics = mock(Metrics.class);
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, THREAD_NAME, VERSION);
-        final String sensorName = "sensor1";
-        final String expectedFullSensorName =
-            INTERNAL_PREFIX + SENSOR_PREFIX_DELIMITER + THREAD_NAME + SENSOR_NAME_DELIMITER + sensorName;
-        final RecordingLevel recordingLevel = RecordingLevel.DEBUG;
+        final RecordingLevel recordingLevel = RecordingLevel.INFO;
+        setupGetSensorTest(metrics, THREAD_ID, recordingLevel);
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, VERSION);
+
+        final Sensor actualSensor = streamsMetrics.threadLevelSensor(THREAD_ID, sensorName1, recordingLevel);
+
+        verify(metrics);
+        assertThat(actualSensor, is(equalToObject(sensor)));
+    }
+
+    private void setupRemoveSensorsTest(final Metrics metrics,
+                                        final String level,
+                                        final RecordingLevel recordingLevel) {
+        final String fullSensorNamePrefix = INTERNAL_PREFIX + SENSOR_PREFIX_DELIMITER + level + SENSOR_NAME_DELIMITER;
         final Sensor[] parents = {};
-        EasyMock.expect(metrics.sensor(expectedFullSensorName, recordingLevel, parents)).andReturn(null);
+        resetToDefault(metrics);
+        metrics.removeSensor(fullSensorNamePrefix + sensorName1);
+        metrics.removeSensor(fullSensorNamePrefix + sensorName2);
+        replay(metrics);
+    }
 
-        replayAll();
+    @Test
+    public void shouldRemoveClientLevelMetrics() {
+        final Metrics metrics = niceMock(Metrics.class);
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, VERSION);
+        addSensorsOnAllLevels(metrics, streamsMetrics);
+        resetToDefault(metrics);
+        expect(metrics.removeMetric(metricName1)).andStubReturn(null);
+        expect(metrics.removeMetric(metricName2)).andStubReturn(null);
+        replay(metrics);
+
+        streamsMetrics.removeAllClientLevelMetrics();
+
+        verify(metrics);
+    }
 
-        final Sensor sensor = streamsMetrics.threadLevelSensor(sensorName, recordingLevel);
+    @Test
+    public void shouldRemoveThreadLevelSensors() {
+        final Metrics metrics = niceMock(Metrics.class);
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, VERSION);
+        addSensorsOnAllLevels(metrics, streamsMetrics);
+        setupRemoveSensorsTest(metrics, THREAD_ID, RecordingLevel.INFO);
 
-        verifyAll();
+        streamsMetrics.removeAllThreadLevelSensors(THREAD_ID);
 
-        assertNull(sensor);
+        verify(metrics);
     }
 
     @Test(expected = NullPointerException.class)
@@ -121,7 +292,7 @@ public void testRemoveSensor() {
     @Test
     public void testMultiLevelSensorRemoval() {
         final Metrics registry = new Metrics();
-        final StreamsMetricsImpl metrics = new StreamsMetricsImpl(registry, THREAD_NAME, VERSION);
+        final StreamsMetricsImpl metrics = new StreamsMetricsImpl(registry, THREAD_ID, VERSION);
         for (final MetricName defaultMetric : registry.metrics().keySet()) {
             registry.removeMetric(defaultMetric);
         }
@@ -133,39 +304,39 @@ public void testMultiLevelSensorRemoval() {
         final String processorNodeName = "processorNodeName";
         final Map nodeTags = mkMap(mkEntry("nkey", "value"));
 
-        final Sensor parent1 = metrics.taskLevelSensor(taskName, operation, Sensor.RecordingLevel.DEBUG);
+        final Sensor parent1 = metrics.taskLevelSensor(THREAD_ID, taskName, operation, Sensor.RecordingLevel.DEBUG);
         addAvgAndMaxLatencyToSensor(parent1, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation);
         addInvocationRateAndCountToSensor(parent1, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation, "", "");
 
         final int numberOfTaskMetrics = registry.metrics().size();
 
-        final Sensor sensor1 = metrics.nodeLevelSensor(taskName, processorNodeName, operation, Sensor.RecordingLevel.DEBUG, parent1);
+        final Sensor sensor1 = metrics.nodeLevelSensor(THREAD_ID, taskName, processorNodeName, operation, Sensor.RecordingLevel.DEBUG, parent1);
         addAvgAndMaxLatencyToSensor(sensor1, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation);
         addInvocationRateAndCountToSensor(sensor1, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation, "", "");
 
         assertThat(registry.metrics().size(), greaterThan(numberOfTaskMetrics));
 
-        metrics.removeAllNodeLevelSensors(taskName, processorNodeName);
+        metrics.removeAllNodeLevelSensors(THREAD_ID, taskName, processorNodeName);
 
         assertThat(registry.metrics().size(), equalTo(numberOfTaskMetrics));
 
-        final Sensor parent2 = metrics.taskLevelSensor(taskName, operation, Sensor.RecordingLevel.DEBUG);
+        final Sensor parent2 = metrics.taskLevelSensor(THREAD_ID, taskName, operation, Sensor.RecordingLevel.DEBUG);
         addAvgAndMaxLatencyToSensor(parent2, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation);
         addInvocationRateAndCountToSensor(parent2, PROCESSOR_NODE_METRICS_GROUP, taskTags, operation, "", "");
 
         assertThat(registry.metrics().size(), equalTo(numberOfTaskMetrics));
 
-        final Sensor sensor2 = metrics.nodeLevelSensor(taskName, processorNodeName, operation, Sensor.RecordingLevel.DEBUG, parent2);
+        final Sensor sensor2 = metrics.nodeLevelSensor(THREAD_ID, taskName, processorNodeName, operation, Sensor.RecordingLevel.DEBUG, parent2);
         addAvgAndMaxLatencyToSensor(sensor2, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation);
         addInvocationRateAndCountToSensor(sensor2, PROCESSOR_NODE_METRICS_GROUP, nodeTags, operation, "", "");
 
         assertThat(registry.metrics().size(), greaterThan(numberOfTaskMetrics));
 
-        metrics.removeAllNodeLevelSensors(taskName, processorNodeName);
+        metrics.removeAllNodeLevelSensors(THREAD_ID, taskName, processorNodeName);
 
         assertThat(registry.metrics().size(), equalTo(numberOfTaskMetrics));
 
-        metrics.removeAllTaskLevelSensors(taskName);
+        metrics.removeAllTaskLevelSensors(THREAD_ID, taskName);
 
         assertThat(registry.metrics().size(), equalTo(0));
     }
@@ -231,7 +402,7 @@ public void testTotalMetricDoesntDecrease() {
             "stream-scope-metrics",
             "",
             "client-id",
-            "",
+            Thread.currentThread().getName(),
             "scope-id",
             "entity"
         );
@@ -244,16 +415,24 @@ public void testTotalMetricDoesntDecrease() {
         }
     }
 
+    @Test
+    public void shouldGetClientLevelTagMap() {
+        final Map tagMap = streamsMetrics.clientLevelTagMap();
+
+        assertThat(tagMap.size(), equalTo(1));
+        assertThat(tagMap.get(StreamsMetricsImpl.CLIENT_ID_TAG), equalTo(CLIENT_ID));
+    }
+
     @Test
     public void shouldGetStoreLevelTagMap() {
         final String taskName = "test-task";
         final String storeType = "remote-window";
         final String storeName = "window-keeper";
 
-        final Map tagMap = streamsMetrics.storeLevelTagMap(taskName, storeType, storeName);
+        final Map tagMap = streamsMetrics.storeLevelTagMap(THREAD_ID, taskName, storeType, storeName);
 
         assertThat(tagMap.size(), equalTo(3));
-        assertThat(tagMap.get(StreamsMetricsImpl.THREAD_ID_TAG_0100_TO_23), equalTo(THREAD_NAME));
+        assertThat(tagMap.get(StreamsMetricsImpl.THREAD_ID_TAG_0100_TO_23), equalTo(THREAD_ID));
         assertThat(tagMap.get(StreamsMetricsImpl.TASK_ID_TAG), equalTo(taskName));
         assertThat(tagMap.get(storeType + "-" + StreamsMetricsImpl.STORE_ID_TAG), equalTo(storeName));
     }
@@ -270,18 +449,18 @@ public void shouldGetCacheLevelTagMapForBuiltInMetricsVersion0100To23() {
 
     private void shouldGetCacheLevelTagMap(final String builtInMetricsVersion) {
         final StreamsMetricsImpl streamsMetrics =
-            new StreamsMetricsImpl(metrics, THREAD_NAME, builtInMetricsVersion);
+            new StreamsMetricsImpl(metrics, THREAD_ID, builtInMetricsVersion);
         final String taskName = "taskName";
         final String storeName = "storeName";
 
-        final Map tagMap = streamsMetrics.cacheLevelTagMap(taskName, storeName);
+        final Map tagMap = streamsMetrics.cacheLevelTagMap(THREAD_ID, taskName, storeName);
 
         assertThat(tagMap.size(), equalTo(3));
         assertThat(
             tagMap.get(
                 builtInMetricsVersion.equals(StreamsConfig.METRICS_LATEST) ? StreamsMetricsImpl.THREAD_ID_TAG
                     : StreamsMetricsImpl.THREAD_ID_TAG_0100_TO_23),
-            equalTo(Thread.currentThread().getName())
+            equalTo(THREAD_ID)
         );
         assertThat(tagMap.get(StreamsMetricsImpl.TASK_ID_TAG), equalTo(taskName));
         assertThat(tagMap.get(StreamsMetricsImpl.RECORD_CACHE_ID_TAG), equalTo(storeName));
@@ -372,7 +551,7 @@ public void shouldAddAvgAndMinAndMaxMetricsToSensor() {
     @Test
     public void shouldReturnMetricsVersionCurrent() {
         assertThat(
-            new StreamsMetricsImpl(metrics, THREAD_NAME, StreamsConfig.METRICS_LATEST).version(),
+            new StreamsMetricsImpl(metrics, THREAD_ID, StreamsConfig.METRICS_LATEST).version(),
             equalTo(Version.LATEST)
         );
     }
@@ -380,7 +559,7 @@ public void shouldReturnMetricsVersionCurrent() {
     @Test
     public void shouldReturnMetricsVersionFrom100To23() {
         assertThat(
-            new StreamsMetricsImpl(metrics, THREAD_NAME, StreamsConfig.METRICS_0100_TO_23).version(),
+            new StreamsMetricsImpl(metrics, THREAD_ID, StreamsConfig.METRICS_0100_TO_23).version(),
             equalTo(Version.FROM_100_TO_23)
         );
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetricsTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetricsTest.java
index b586258982510..67d5c226afc21 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetricsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/ThreadMetricsTest.java
@@ -43,6 +43,7 @@
 @PrepareForTest(StreamsMetricsImpl.class)
 public class ThreadMetricsTest {
 
+    private static final String THREAD_ID = "thread-id";
     private static final String THREAD_LEVEL_GROUP = "stream-metrics";
     private static final String TASK_LEVEL_GROUP = "stream-task-metrics";
 
@@ -57,15 +58,15 @@ public void shouldGetCreateTaskSensor() {
         final String totalDescription = "The total number of newly created tasks";
         final String rateDescription = "The average per-second number of newly created tasks";
         mockStatic(StreamsMetricsImpl.class);
-        expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.INFO)).andReturn(dummySensor);
-        expect(streamsMetrics.threadLevelTagMap()).andReturn(dummyTagMap);
+        expect(streamsMetrics.threadLevelSensor(THREAD_ID, operation, RecordingLevel.INFO)).andReturn(dummySensor);
+        expect(streamsMetrics.threadLevelTagMap(THREAD_ID)).andReturn(dummyTagMap);
         StreamsMetricsImpl.addInvocationRateAndCountToSensor(
             dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
 
         replayAll();
         replay(StreamsMetricsImpl.class);
 
-        final Sensor sensor = ThreadMetrics.createTaskSensor(streamsMetrics);
+        final Sensor sensor = ThreadMetrics.createTaskSensor(THREAD_ID, streamsMetrics);
 
         verifyAll();
         verify(StreamsMetricsImpl.class);
@@ -79,15 +80,15 @@ public void shouldGetCloseTaskSensor() {
         final String totalDescription = "The total number of closed tasks";
         final String rateDescription = "The average per-second number of closed tasks";
         mockStatic(StreamsMetricsImpl.class);
-        expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.INFO)).andReturn(dummySensor);
-        expect(streamsMetrics.threadLevelTagMap()).andReturn(dummyTagMap);
+        expect(streamsMetrics.threadLevelSensor(THREAD_ID, operation, RecordingLevel.INFO)).andReturn(dummySensor);
+        expect(streamsMetrics.threadLevelTagMap(THREAD_ID)).andReturn(dummyTagMap);
         StreamsMetricsImpl.addInvocationRateAndCountToSensor(
             dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
 
         replayAll();
         replay(StreamsMetricsImpl.class);
 
-        final Sensor sensor = ThreadMetrics.closeTaskSensor(streamsMetrics);
+        final Sensor sensor = ThreadMetrics.closeTaskSensor(THREAD_ID, streamsMetrics);
 
         verifyAll();
         verify(StreamsMetricsImpl.class);
@@ -102,8 +103,8 @@ public void shouldGetCommitSensor() {
         final String totalDescription = "The total number of commit calls";
         final String rateDescription = "The average per-second number of commit calls";
         mockStatic(StreamsMetricsImpl.class);
-        expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.INFO)).andReturn(dummySensor);
-        expect(streamsMetrics.threadLevelTagMap()).andReturn(dummyTagMap);
+        expect(streamsMetrics.threadLevelSensor(THREAD_ID, operation, RecordingLevel.INFO)).andReturn(dummySensor);
+        expect(streamsMetrics.threadLevelTagMap(THREAD_ID)).andReturn(dummyTagMap);
         StreamsMetricsImpl.addInvocationRateAndCountToSensor(
             dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
         StreamsMetricsImpl.addAvgAndMaxToSensor(
@@ -112,7 +113,7 @@ public void shouldGetCommitSensor() {
         replayAll();
         replay(StreamsMetricsImpl.class);
 
-        final Sensor sensor = ThreadMetrics.commitSensor(streamsMetrics);
+        final Sensor sensor = ThreadMetrics.commitSensor(THREAD_ID, streamsMetrics);
 
         verifyAll();
         verify(StreamsMetricsImpl.class);
@@ -127,8 +128,8 @@ public void shouldGetPollSensor() {
         final String totalDescription = "The total number of poll calls";
         final String rateDescription = "The average per-second number of poll calls";
         mockStatic(StreamsMetricsImpl.class);
-        expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.INFO)).andReturn(dummySensor);
-        expect(streamsMetrics.threadLevelTagMap()).andReturn(dummyTagMap);
+        expect(streamsMetrics.threadLevelSensor(THREAD_ID, operation, RecordingLevel.INFO)).andReturn(dummySensor);
+        expect(streamsMetrics.threadLevelTagMap(THREAD_ID)).andReturn(dummyTagMap);
         StreamsMetricsImpl.addInvocationRateAndCountToSensor(
             dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
         StreamsMetricsImpl.addAvgAndMaxToSensor(
@@ -137,7 +138,7 @@ public void shouldGetPollSensor() {
         replayAll();
         replay(StreamsMetricsImpl.class);
 
-        final Sensor sensor = ThreadMetrics.pollSensor(streamsMetrics);
+        final Sensor sensor = ThreadMetrics.pollSensor(THREAD_ID, streamsMetrics);
 
         verifyAll();
         verify(StreamsMetricsImpl.class);
@@ -152,8 +153,8 @@ public void shouldGetProcessSensor() {
         final String totalDescription = "The total number of process calls";
         final String rateDescription = "The average per-second number of process calls";
         mockStatic(StreamsMetricsImpl.class);
-        expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.INFO)).andReturn(dummySensor);
-        expect(streamsMetrics.threadLevelTagMap()).andReturn(dummyTagMap);
+        expect(streamsMetrics.threadLevelSensor(THREAD_ID, operation, RecordingLevel.INFO)).andReturn(dummySensor);
+        expect(streamsMetrics.threadLevelTagMap(THREAD_ID)).andReturn(dummyTagMap);
         StreamsMetricsImpl.addInvocationRateAndCountToSensor(
             dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
         StreamsMetricsImpl.addAvgAndMaxToSensor(
@@ -162,7 +163,7 @@ public void shouldGetProcessSensor() {
         replayAll();
         replay(StreamsMetricsImpl.class);
 
-        final Sensor sensor = ThreadMetrics.processSensor(streamsMetrics);
+        final Sensor sensor = ThreadMetrics.processSensor(THREAD_ID, streamsMetrics);
 
         verifyAll();
         verify(StreamsMetricsImpl.class);
@@ -177,8 +178,8 @@ public void shouldGetPunctuateSensor() {
         final String totalDescription = "The total number of punctuate calls";
         final String rateDescription = "The average per-second number of punctuate calls";
         mockStatic(StreamsMetricsImpl.class);
-        expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.INFO)).andReturn(dummySensor);
-        expect(streamsMetrics.threadLevelTagMap()).andReturn(dummyTagMap);
+        expect(streamsMetrics.threadLevelSensor(THREAD_ID, operation, RecordingLevel.INFO)).andReturn(dummySensor);
+        expect(streamsMetrics.threadLevelTagMap(THREAD_ID)).andReturn(dummyTagMap);
         StreamsMetricsImpl.addInvocationRateAndCountToSensor(
             dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
         StreamsMetricsImpl.addAvgAndMaxToSensor(
@@ -187,7 +188,7 @@ public void shouldGetPunctuateSensor() {
         replayAll();
         replay(StreamsMetricsImpl.class);
 
-        final Sensor sensor = ThreadMetrics.punctuateSensor(streamsMetrics);
+        final Sensor sensor = ThreadMetrics.punctuateSensor(THREAD_ID, streamsMetrics);
 
         verifyAll();
         verify(StreamsMetricsImpl.class);
@@ -201,15 +202,15 @@ public void shouldGetSkipRecordSensor() {
         final String totalDescription = "The total number of skipped records";
         final String rateDescription = "The average per-second number of skipped records";
         mockStatic(StreamsMetricsImpl.class);
-        expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.INFO)).andReturn(dummySensor);
-        expect(streamsMetrics.threadLevelTagMap()).andReturn(dummyTagMap);
+        expect(streamsMetrics.threadLevelSensor(THREAD_ID, operation, RecordingLevel.INFO)).andReturn(dummySensor);
+        expect(streamsMetrics.threadLevelTagMap(THREAD_ID)).andReturn(dummyTagMap);
         StreamsMetricsImpl.addInvocationRateAndCountToSensor(
             dummySensor, THREAD_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
 
         replayAll();
         replay(StreamsMetricsImpl.class);
 
-        final Sensor sensor = ThreadMetrics.skipRecordSensor(streamsMetrics);
+        final Sensor sensor = ThreadMetrics.skipRecordSensor(THREAD_ID, streamsMetrics);
 
         verifyAll();
         verify(StreamsMetricsImpl.class);
@@ -224,8 +225,8 @@ public void shouldGetCommitOverTasksSensor() {
         final String totalDescription = "The total number of commit calls over all tasks";
         final String rateDescription = "The average per-second number of commit calls over all tasks";
         mockStatic(StreamsMetricsImpl.class);
-        expect(streamsMetrics.threadLevelSensor(operation, RecordingLevel.DEBUG)).andReturn(dummySensor);
-        expect(streamsMetrics.threadLevelTagMap(TASK_ID_TAG, ROLLUP_VALUE)).andReturn(dummyTagMap);
+        expect(streamsMetrics.threadLevelSensor(THREAD_ID, operation, RecordingLevel.DEBUG)).andReturn(dummySensor);
+        expect(streamsMetrics.threadLevelTagMap(THREAD_ID, TASK_ID_TAG, ROLLUP_VALUE)).andReturn(dummyTagMap);
         StreamsMetricsImpl.addInvocationRateAndCountToSensor(
             dummySensor, TASK_LEVEL_GROUP, dummyTagMap, operation, totalDescription, rateDescription);
         StreamsMetricsImpl.addAvgAndMaxToSensor(
@@ -234,7 +235,7 @@ public void shouldGetCommitOverTasksSensor() {
         replayAll();
         replay(StreamsMetricsImpl.class);
 
-        final Sensor sensor = ThreadMetrics.commitOverTasksSensor(streamsMetrics);
+        final Sensor sensor = ThreadMetrics.commitOverTasksSensor(THREAD_ID, streamsMetrics);
 
         verifyAll();
         verify(StreamsMetricsImpl.class);
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractRocksDBSegmentedBytesStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractRocksDBSegmentedBytesStoreTest.java
index 004f181c5f613..19ea36d3e7c16 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractRocksDBSegmentedBytesStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractRocksDBSegmentedBytesStoreTest.java
@@ -415,13 +415,14 @@ public void shouldLogAndMeasureExpiredRecords() {
         LogCaptureAppender.unregister(appender);
 
         final Map metrics = context.metrics().metrics();
+        final String threadId = Thread.currentThread().getName();
 
         final Metric dropTotal = metrics.get(new MetricName(
             "expired-window-record-drop-total",
             "stream-metrics-scope-metrics",
             "The total number of occurrence of expired-window-record-drop operations.",
             mkMap(
-                mkEntry("client-id", "mock"),
+                mkEntry("client-id", threadId),
                 mkEntry("task-id", "0_0"),
                 mkEntry("metrics-scope-id", "bytes-store")
             )
@@ -432,7 +433,7 @@ public void shouldLogAndMeasureExpiredRecords() {
             "stream-metrics-scope-metrics",
             "The average number of occurrence of expired-window-record-drop operation per second.",
             mkMap(
-                mkEntry("client-id", "mock"),
+                mkEntry("client-id", threadId),
                 mkEntry("task-id", "0_0"),
                 mkEntry("metrics-scope-id", "bytes-store")
             )
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/KeyValueSegmentTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/KeyValueSegmentTest.java
index f8248220c9870..621fdd3afb72e 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/KeyValueSegmentTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/KeyValueSegmentTest.java
@@ -39,7 +39,8 @@
 
 public class KeyValueSegmentTest {
 
-    private final RocksDBMetricsRecorder metricsRecorder = new RocksDBMetricsRecorder("metrics-scope", "store-name");
+    private final RocksDBMetricsRecorder metricsRecorder =
+        new RocksDBMetricsRecorder("metrics-scope", "thread-id", "store-name");
 
     @Test
     public void shouldDeleteStateDirectoryOnDestroy() throws Exception {
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStoreTest.java
index e494532cfe8cf..533d58aea396b 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStoreTest.java
@@ -63,7 +63,7 @@ public class MeteredKeyValueStoreTest {
 
     private final TaskId taskId = new TaskId(0, 0);
     private final Map tags = mkMap(
-        mkEntry("client-id", "test"),
+        mkEntry("client-id", Thread.currentThread().getName()),
         mkEntry("task-id", taskId.toString()),
         mkEntry("scope-state-id", "metered")
     );
@@ -106,9 +106,9 @@ public void testMetrics() {
         final JmxReporter reporter = new JmxReporter("kafka.streams");
         metrics.addReporter(reporter);
         assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-state-metrics,client-id=%s,task-id=%s,%s-state-id=%s",
-                "scope", "test", taskId.toString(), "scope", "metered")));
+                "scope", Thread.currentThread().getName(), taskId.toString(), "scope", "metered")));
         assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-state-metrics,client-id=%s,task-id=%s,%s-state-id=%s",
-                "scope", "test", taskId.toString(), "scope", "all")));
+                "scope", Thread.currentThread().getName(), taskId.toString(), "scope", "all")));
     }
 
     @Test
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java
index e21fbc2052629..25f842c888ec4 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java
@@ -63,9 +63,10 @@
 @RunWith(EasyMockRunner.class)
 public class MeteredSessionStoreTest {
 
+    private final String threadId = Thread.currentThread().getName();
     private final TaskId taskId = new TaskId(0, 0);
     private final Map tags = mkMap(
-        mkEntry("client-id", "test"),
+        mkEntry("client-id", threadId),
         mkEntry("task-id", taskId.toString()),
         mkEntry("scope-state-id", "metered")
     );
@@ -105,9 +106,9 @@ public void testMetrics() {
         final JmxReporter reporter = new JmxReporter("kafka.streams");
         metrics.addReporter(reporter);
         assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-state-metrics,client-id=%s,task-id=%s,%s-state-id=%s",
-                "scope", "test", taskId.toString(), "scope", "metered")));
+                "scope", threadId, taskId.toString(), "scope", "metered")));
         assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-state-metrics,client-id=%s,task-id=%s,%s-state-id=%s",
-                "scope", "test", taskId.toString(), "scope", "all")));
+                "scope", threadId, taskId.toString(), "scope", "all")));
     }
 
     @Test
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedKeyValueStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedKeyValueStoreTest.java
index 47a49455cb3c0..837cc21009ad2 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedKeyValueStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedKeyValueStoreTest.java
@@ -64,9 +64,10 @@
 @RunWith(EasyMockRunner.class)
 public class MeteredTimestampedKeyValueStoreTest {
 
+    private final String threadId = Thread.currentThread().getName();
     private final TaskId taskId = new TaskId(0, 0);
     private final Map tags = mkMap(
-        mkEntry("client-id", "test"),
+        mkEntry("client-id", threadId),
         mkEntry("task-id", taskId.toString()),
         mkEntry("scope-state-id", "metered")
     );
@@ -111,9 +112,9 @@ public void testMetrics() {
         final JmxReporter reporter = new JmxReporter("kafka.streams");
         metrics.addReporter(reporter);
         assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-state-metrics,client-id=%s,task-id=%s,%s-state-id=%s",
-                "scope", "test", taskId.toString(), "scope", "metered")));
+                "scope", threadId, taskId.toString(), "scope", "metered")));
         assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-state-metrics,client-id=%s,task-id=%s,%s-state-id=%s",
-                "scope", "test", taskId.toString(), "scope", "all")));
+                "scope", threadId, taskId.toString(), "scope", "all")));
     }
     @Test
     public void shouldWriteBytesToInnerStoreAndRecordPutMetric() {
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java
index e62ab3a8a424a..588307ff37b23 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java
@@ -95,10 +95,11 @@ public void testMetrics() {
         store.init(context, store);
         final JmxReporter reporter = new JmxReporter("kafka.streams");
         metrics.addReporter(reporter);
+        final String threadId = Thread.currentThread().getName();
         assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-state-metrics,client-id=%s,task-id=%s,%s-state-id=%s",
-                "scope", "test", context.taskId().toString(), "scope", "mocked-store")));
+                "scope", threadId, context.taskId().toString(), "scope", "mocked-store")));
         assertTrue(reporter.containsMbean(String.format("kafka.streams:type=stream-%s-state-metrics,client-id=%s,task-id=%s,%s-state-id=%s",
-                "scope", "test", context.taskId().toString(), "scope", "all")));
+                "scope", threadId, context.taskId().toString(), "scope", "all")));
     }
 
     @Test
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentIteratorTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentIteratorTest.java
index 2e305dd406b25..7944e5f0f5188 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentIteratorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentIteratorTest.java
@@ -41,7 +41,8 @@
 
 public class SegmentIteratorTest {
 
-    private final RocksDBMetricsRecorder rocksDBMetricsRecorder = new RocksDBMetricsRecorder("metrics-scope", "store-name");
+    private final RocksDBMetricsRecorder rocksDBMetricsRecorder =
+        new RocksDBMetricsRecorder("metrics-scope", "thread-id", "store-name");
     private final KeyValueSegment segmentOne =
         new KeyValueSegment("one", "one", 0, rocksDBMetricsRecorder);
     private final KeyValueSegment segmentTwo =
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/SessionBytesStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/SessionBytesStoreTest.java
index 9b29e8b9ff607..fe2bd6e52743d 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/SessionBytesStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/SessionBytesStoreTest.java
@@ -443,13 +443,14 @@ public void shouldLogAndMeasureExpiredRecords() {
         final Map metrics = context.metrics().metrics();
 
         final String metricScope = getMetricsScope();
+        final String threadId = Thread.currentThread().getName();
 
         final Metric dropTotal = metrics.get(new MetricName(
             "expired-window-record-drop-total",
             "stream-" + metricScope + "-metrics",
             "The total number of occurrence of expired-window-record-drop operations.",
             mkMap(
-                mkEntry("client-id", "mock"),
+                mkEntry("client-id", threadId),
                 mkEntry("task-id", "0_0"),
                 mkEntry(metricScope + "-id", sessionStore.name())
             )
@@ -460,7 +461,7 @@ public void shouldLogAndMeasureExpiredRecords() {
             "stream-" + metricScope + "-metrics",
             "The average number of occurrence of expired-window-record-drop operation per second.",
             mkMap(
-                mkEntry("client-id", "mock"),
+                mkEntry("client-id", threadId),
                 mkEntry("task-id", "0_0"),
                 mkEntry(metricScope + "-id", sessionStore.name())
             )
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/TimestampedSegmentTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/TimestampedSegmentTest.java
index 7bd3e7ebe46eb..a0069c861fe10 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/TimestampedSegmentTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/TimestampedSegmentTest.java
@@ -39,7 +39,8 @@
 
 public class TimestampedSegmentTest {
 
-    private final RocksDBMetricsRecorder metricsRecorder = new RocksDBMetricsRecorder("metrics-scope", "store-name");
+    private final RocksDBMetricsRecorder metricsRecorder =
+        new RocksDBMetricsRecorder("metrics-scope", "thread-id", "store-name");
 
     @Test
     public void shouldDeleteStateDirectoryOnDestroy() throws Exception {
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/WindowBytesStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/WindowBytesStoreTest.java
index 5177079dd50ba..489c8254b3c70 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/WindowBytesStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/WindowBytesStoreTest.java
@@ -892,13 +892,14 @@ public void shouldLogAndMeasureExpiredRecords() {
         final Map metrics = context.metrics().metrics();
 
         final String metricScope = getMetricsScope();
+        final String threadId = Thread.currentThread().getName();
 
         final Metric dropTotal = metrics.get(new MetricName(
             "expired-window-record-drop-total",
             "stream-" + metricScope + "-metrics",
             "The total number of occurrence of expired-window-record-drop operations.",
             mkMap(
-                mkEntry("client-id", "mock"),
+                mkEntry("client-id", threadId),
                 mkEntry("task-id", "0_0"),
                 mkEntry(metricScope + "-id", windowStore.name())
             )
@@ -909,7 +910,7 @@ public void shouldLogAndMeasureExpiredRecords() {
             "stream-" + metricScope + "-metrics",
             "The average number of occurrence of expired-window-record-drop operation per second.",
             mkMap(
-                mkEntry("client-id", "mock"),
+                mkEntry("client-id", threadId),
                 mkEntry("task-id", "0_0"),
                 mkEntry(metricScope + "-id", windowStore.name())
             )
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/NamedCacheMetricsTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/NamedCacheMetricsTest.java
index 46ea302738bea..263947050978d 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/NamedCacheMetricsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/NamedCacheMetricsTest.java
@@ -43,7 +43,8 @@
 @PrepareForTest({StreamsMetricsImpl.class, Sensor.class})
 public class NamedCacheMetricsTest {
 
-    private static final String TASK_NAME = "taskName";
+    private static final String THREAD_ID = "test-thread";
+    private static final String TASK_ID = "test-task";
     private static final String STORE_NAME = "storeName";
     private static final String HIT_RATIO_AVG_DESCRIPTION = "The average cache hit ratio";
     private static final String HIT_RATIO_MIN_DESCRIPTION = "The minimum cache hit ratio";
@@ -61,7 +62,7 @@ public void shouldGetHitRatioSensorWithBuiltInMetricsVersionCurrent() {
         replay(streamsMetrics);
         replay(StreamsMetricsImpl.class);
 
-        final Sensor sensor = NamedCacheMetrics.hitRatioSensor(streamsMetrics, TASK_NAME, STORE_NAME);
+        final Sensor sensor = NamedCacheMetrics.hitRatioSensor(streamsMetrics, THREAD_ID, TASK_ID, STORE_NAME);
 
         verifyResult(sensor);
     }
@@ -73,8 +74,9 @@ public void shouldGetHitRatioSensorWithBuiltInMetricsVersionBefore24() {
         final RecordingLevel recordingLevel = RecordingLevel.DEBUG;
         mockStatic(StreamsMetricsImpl.class);
         final Sensor parentSensor = mock(Sensor.class);
-        expect(streamsMetrics.taskLevelSensor(TASK_NAME, hitRatio, recordingLevel)).andReturn(parentSensor);
-        expect(streamsMetrics.cacheLevelTagMap(TASK_NAME, StreamsMetricsImpl.ROLLUP_VALUE)).andReturn(parentTagMap);
+        expect(streamsMetrics.taskLevelSensor(THREAD_ID, TASK_ID, hitRatio, recordingLevel)).andReturn(parentSensor);
+        expect(streamsMetrics.cacheLevelTagMap(THREAD_ID, TASK_ID, StreamsMetricsImpl.ROLLUP_VALUE))
+            .andReturn(parentTagMap);
         StreamsMetricsImpl.addAvgAndMinAndMaxToSensor(
             parentSensor,
             StreamsMetricsImpl.CACHE_LEVEL_GROUP,
@@ -87,7 +89,7 @@ public void shouldGetHitRatioSensorWithBuiltInMetricsVersionBefore24() {
         replay(streamsMetrics);
         replay(StreamsMetricsImpl.class);
 
-        final Sensor sensor = NamedCacheMetrics.hitRatioSensor(streamsMetrics, TASK_NAME, STORE_NAME);
+        final Sensor sensor = NamedCacheMetrics.hitRatioSensor(streamsMetrics, THREAD_ID, TASK_ID, STORE_NAME);
 
         verifyResult(sensor);
     }
@@ -96,9 +98,9 @@ private void setUpStreamsMetrics(final Version builtInMetricsVersion,
                                      final String hitRatio,
                                      final Sensor... parents) {
         expect(streamsMetrics.version()).andReturn(builtInMetricsVersion);
-        expect(streamsMetrics.cacheLevelSensor(TASK_NAME, STORE_NAME, hitRatio, RecordingLevel.DEBUG, parents))
+        expect(streamsMetrics.cacheLevelSensor(THREAD_ID, TASK_ID, STORE_NAME, hitRatio, RecordingLevel.DEBUG, parents))
             .andReturn(expectedSensor);
-        expect(streamsMetrics.cacheLevelTagMap(TASK_NAME, STORE_NAME)).andReturn(tagMap);
+        expect(streamsMetrics.cacheLevelTagMap(THREAD_ID, TASK_ID, STORE_NAME)).andReturn(tagMap);
         StreamsMetricsImpl.addAvgAndMinAndMaxToSensor(
             expectedSensor,
             StreamsMetricsImpl.CACHE_LEVEL_GROUP,
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorderTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorderTest.java
index 9bef2cc3b6f6b..f22c02ed4d4c2 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorderTest.java
@@ -47,6 +47,7 @@
 @PrepareForTest({RocksDBMetrics.class, Sensor.class})
 public class RocksDBMetricsRecorderTest {
     private final static String METRICS_SCOPE = "metrics-scope";
+    private final static String THREAD_ID = "thread-id";
     private final static String STORE_NAME = "store-name";
     private final static String SEGMENT_STORE_NAME_1 = "segment-store-name-1";
     private final static String SEGMENT_STORE_NAME_2 = "segment-store-name-2";
@@ -72,7 +73,7 @@ public class RocksDBMetricsRecorderTest {
     private final TaskId taskId1 = new TaskId(0, 0);
     private final TaskId taskId2 = new TaskId(0, 2);
 
-    private final RocksDBMetricsRecorder recorder = new RocksDBMetricsRecorder(METRICS_SCOPE, STORE_NAME);
+    private final RocksDBMetricsRecorder recorder = new RocksDBMetricsRecorder(METRICS_SCOPE, THREAD_ID, STORE_NAME);
 
     @Before
     public void setUp() {
@@ -327,7 +328,7 @@ public void shouldCorrectlyHandleHitRatioRecordingsWithZeroHitsAndMisses() {
     private void setUpMetricsMock() {
         mockStatic(RocksDBMetrics.class);
         final RocksDBMetricContext metricsContext =
-            new RocksDBMetricContext(taskId1.toString(), METRICS_SCOPE, STORE_NAME);
+            new RocksDBMetricContext(THREAD_ID, taskId1.toString(), METRICS_SCOPE, STORE_NAME);
         expect(RocksDBMetrics.bytesWrittenToDatabaseSensor(eq(streamsMetrics), eq(metricsContext)))
             .andReturn(bytesWrittenToDatabaseSensor);
         expect(RocksDBMetrics.bytesReadFromDatabaseSensor(eq(streamsMetrics), eq(metricsContext)))
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsTest.java
index aae7afeeb7e21..d90a11624d1bc 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsTest.java
@@ -44,9 +44,10 @@
 public class RocksDBMetricsTest {
 
     private static final String STATE_LEVEL_GROUP = "stream-state-metrics";
-    private final String taskName = "task";
-    private final String storeType = "test-state-id";
-    private final String storeName = "store";
+    private static final String THREAD_ID = "test-thread";
+    private static final String TASK_ID = "test-task";
+    private static final String STORE_TYPE = "test-store-type";
+    private static final String STORE_NAME = "store";
     private final Metrics metrics = new Metrics();
     private final Sensor sensor = metrics.sensor("dummy");
     private final StreamsMetricsImpl streamsMetrics = createStrictMock(StreamsMetricsImpl.class);
@@ -267,15 +268,17 @@ private void verifySumSensor(final String metricNamePrefix,
     private void setupStreamsMetricsMock(final String metricNamePrefix) {
         mockStatic(StreamsMetricsImpl.class);
         expect(streamsMetrics.storeLevelSensor(
-            taskName,
-            storeName,
+            THREAD_ID,
+            TASK_ID,
+            STORE_NAME,
             metricNamePrefix,
             RecordingLevel.DEBUG
         )).andReturn(sensor);
         expect(streamsMetrics.storeLevelTagMap(
-            taskName,
-            storeType,
-            storeName
+            THREAD_ID,
+            TASK_ID,
+            STORE_TYPE,
+            STORE_NAME
         )).andReturn(tags);
     }
 
@@ -284,7 +287,7 @@ private void replayCallAndVerify(final SensorCreator sensorCreator) {
         replay(StreamsMetricsImpl.class);
 
         final Sensor sensor =
-            sensorCreator.sensor(streamsMetrics, new RocksDBMetricContext(taskName, storeType, storeName));
+            sensorCreator.sensor(streamsMetrics, new RocksDBMetricContext(THREAD_ID, TASK_ID, STORE_TYPE, STORE_NAME));
 
         verifyAll();
         verify(StreamsMetricsImpl.class);
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
index a0ca6b2cad2d4..9b3d0fdd7faf2 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
@@ -280,24 +280,25 @@ public List partitionsFor(final String topic) {
 
         metrics = new Metrics(metricConfig, mockWallClockTime);
 
-        final String threadName = "topology-test-driver-virtual-thread";
+        final String threadId = Thread.currentThread().getName();
         final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(
             metrics,
-            threadName,
+            "test-client",
             StreamsConfig.METRICS_LATEST
         );
         streamsMetrics.setRocksDBMetricsRecordingTrigger(new RocksDBMetricsRecordingTrigger());
-        final Sensor skippedRecordsSensor = streamsMetrics.threadLevelSensor("skipped-records", Sensor.RecordingLevel.INFO);
+        final Sensor skippedRecordsSensor =
+            streamsMetrics.threadLevelSensor(threadId, "skipped-records", Sensor.RecordingLevel.INFO);
         final String threadLevelGroup = "stream-metrics";
         skippedRecordsSensor.add(new MetricName("skipped-records-rate",
                                                 threadLevelGroup,
                                                 "The average per-second number of skipped records",
-                                                streamsMetrics.tagMap()),
+                                                streamsMetrics.tagMap(threadId)),
                                  new Rate(TimeUnit.SECONDS, new WindowedCount()));
         skippedRecordsSensor.add(new MetricName("skipped-records-total",
                                                 threadLevelGroup,
                                                 "The total number of skipped records",
-                                                streamsMetrics.tagMap()),
+                                                streamsMetrics.tagMap(threadId)),
                                  new CumulativeSum());
         final ThreadCache cache = new ThreadCache(
             new LogContext("topology-test-driver "),
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java
index 041bf0d65d8b8..09850e4ed5a42 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java
@@ -213,9 +213,9 @@ public MockProcessorContext(final Properties config, final TaskId taskId, final
         this.stateDir = stateDir;
         final MetricConfig metricConfig = new MetricConfig();
         metricConfig.recordLevel(Sensor.RecordingLevel.DEBUG);
-        final String threadName = "mock-processor-context-virtual-thread";
-        this.metrics = new StreamsMetricsImpl(new Metrics(metricConfig), threadName, StreamsConfig.METRICS_LATEST);
-        ThreadMetrics.skipRecordSensor(metrics);
+        final String threadId = Thread.currentThread().getName();
+        this.metrics = new StreamsMetricsImpl(new Metrics(metricConfig), threadId, StreamsConfig.METRICS_LATEST);
+        ThreadMetrics.skipRecordSensor(threadId, metrics);
     }
 
     @Override

From 4be2b0958840fcb229a260b4efde6c9eae1f9e9f Mon Sep 17 00:00:00 2001
From: David Jacot 
Date: Sun, 6 Oct 2019 00:28:59 +0200
Subject: [PATCH 0690/1071] MINOR: Rework error messages in
 ConsumerGroupCommand (#7445)

Distinguish invalid and not found groupId cases in consumer group command.

Reviewers: Jason Gustafson 
---
 core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala b/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala
index 519b33535668a..2d176591542d8 100755
--- a/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala
+++ b/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala
@@ -467,8 +467,10 @@ object ConsumerGroupCommand extends Logging {
       } catch {
         case e: ExecutionException =>
           Errors.forException(e.getCause) match {
-            case Errors.INVALID_GROUP_ID | Errors.GROUP_ID_NOT_FOUND =>
-              printError(s"'$groupId' is not valid or does not exist.")
+            case Errors.INVALID_GROUP_ID =>
+              printError(s"'$groupId' is not valid.")
+            case Errors.GROUP_ID_NOT_FOUND =>
+              printError(s"'$groupId' does not exist.")
             case Errors.GROUP_AUTHORIZATION_FAILED =>
               printError(s"Access to '$groupId' is not authorized.")
             case Errors.NON_EMPTY_GROUP =>

From 7349608009df3d231db154b9f5c15134b047e38d Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Sun, 6 Oct 2019 12:55:04 -0500
Subject: [PATCH 0691/1071] MINOR: Gracefully handle non-assigned case in
 fetcher metric (#7383)

Minor tweak to gracefully handle a possible IllegalStateException while checking a metric value.

Reviewers: Guozhang Wang , Jason Gustafson 
---
 .../apache/kafka/clients/consumer/internals/Fetcher.java   | 6 ++++--
 .../clients/consumer/internals/SubscriptionState.java      | 7 ++++++-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
index 9fd21cf121e07..6dd76048c72b3 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
@@ -1742,8 +1742,10 @@ private void maybeUpdateAssignment(SubscriptionState subscription) {
                     if (!this.assignedPartitions.contains(tp)) {
                         MetricName metricName = partitionPreferredReadReplicaMetricName(tp);
                         if (metrics.metric(metricName) == null) {
-                            metrics.addMetric(metricName, (Gauge) (config, now) ->
-                                subscription.preferredReadReplica(tp, 0L).orElse(-1));
+                            metrics.addMetric(
+                                metricName,
+                                (Gauge) (config, now) -> subscription.preferredReadReplica(tp, 0L).orElse(-1)
+                            );
                         }
                     }
                 }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
index 7c965ae58a55d..4641e5c4917e1 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
@@ -524,7 +524,12 @@ public synchronized void updatePreferredReadReplica(TopicPartition tp, int prefe
      * @return Returns the current preferred read replica, if it has been set and if it has not expired.
      */
     public synchronized Optional preferredReadReplica(TopicPartition tp, long timeMs) {
-        return assignedState(tp).preferredReadReplica(timeMs);
+        final TopicPartitionState topicPartitionState = assignedStateOrNull(tp);
+        if (topicPartitionState == null) {
+            return Optional.empty();
+        } else {
+            return topicPartitionState.preferredReadReplica(timeMs);
+        }
     }
 
     /**

From 0de61a4683b92bdee803c51211c3277578ab3edf Mon Sep 17 00:00:00 2001
From: Colin Patrick McCabe 
Date: Sun, 6 Oct 2019 21:13:23 -0700
Subject: [PATCH 0692/1071] KAFKA-8885; The Kafka Protocol should Support
 Optional Tagged Fields (#7325)

This patch implements support for optional (tagged) fields in the Kafka protocol as documented in KIP-482: https://cwiki.apache.org/confluence/display/KAFKA/KIP-482%3A+The+Kafka+Protocol+should+Support+Optional+Tagged+Fields#KIP-482:TheKafkaProtocolshouldSupportOptionalTaggedFields-TypeClasses.

Reviewers: David Jacot , Ismael Juma , Jason Gustafson 
---
 checkstyle/suppressions.xml                   |    2 +-
 .../apache/kafka/clients/ClientRequest.java   |    2 +-
 .../apache/kafka/clients/NetworkClient.java   |    3 +-
 .../apache/kafka/common/protocol/ApiKeys.java |   18 +-
 .../common/protocol/ByteBufferAccessor.java   |   14 +-
 .../apache/kafka/common/protocol/Message.java |   21 +-
 .../kafka/common/protocol/MessageUtil.java    |   14 -
 .../protocol/ObjectSerializationCache.java    |   57 +
 .../kafka/common/protocol/Readable.java       |   34 +-
 .../kafka/common/protocol/Writable.java       |   50 +-
 .../common/protocol/types/CompactArrayOf.java |  139 ++
 .../kafka/common/protocol/types/Field.java    |   38 +
 .../common/protocol/types/RawTaggedField.java |   56 +
 .../protocol/types/RawTaggedFieldWriter.java  |   80 +
 .../common/protocol/types/TaggedFields.java   |  181 ++
 .../kafka/common/protocol/types/Type.java     |  240 ++-
 .../common/requests/AbstractResponse.java     |    5 +-
 .../common/requests/ApiVersionsResponse.java  |   21 +-
 .../common/requests/LeaderAndIsrRequest.java  |   14 +-
 .../kafka/common/requests/RequestHeader.java  |    7 +-
 .../common/requests/StopReplicaRequest.java   |   10 +-
 .../requests/UpdateMetadataRequest.java       |   14 +-
 .../SaslClientAuthenticator.java              |    2 +-
 .../message/AddOffsetsToTxnRequest.json       |    1 +
 .../message/AddOffsetsToTxnResponse.json      |    1 +
 .../message/AddPartitionsToTxnRequest.json    |    1 +
 .../message/AddPartitionsToTxnResponse.json   |    1 +
 .../common/message/AlterConfigsRequest.json   |    1 +
 .../common/message/AlterConfigsResponse.json  |    1 +
 .../AlterPartitionReassignmentsRequest.json   |    1 +
 .../AlterPartitionReassignmentsResponse.json  |    1 +
 .../message/AlterReplicaLogDirsRequest.json   |    1 +
 .../message/AlterReplicaLogDirsResponse.json  |    1 +
 .../common/message/ApiVersionsRequest.json    |    3 +-
 .../common/message/ApiVersionsResponse.json   |    3 +-
 .../message/ControlledShutdownRequest.json    |    1 +
 .../message/ControlledShutdownResponse.json   |    1 +
 .../common/message/CreateAclsRequest.json     |    1 +
 .../common/message/CreateAclsResponse.json    |    1 +
 .../message/CreateDelegationTokenRequest.json |    5 +-
 .../CreateDelegationTokenResponse.json        |    5 +-
 .../message/CreatePartitionsRequest.json      |    1 +
 .../message/CreatePartitionsResponse.json     |    1 +
 .../common/message/CreateTopicsRequest.json   |    7 +-
 .../common/message/CreateTopicsResponse.json  |   11 +-
 .../common/message/DeleteAclsRequest.json     |    1 +
 .../common/message/DeleteAclsResponse.json    |    1 +
 .../common/message/DeleteGroupsRequest.json   |    5 +-
 .../common/message/DeleteGroupsResponse.json  |    5 +-
 .../common/message/DeleteRecordsRequest.json  |    1 +
 .../common/message/DeleteRecordsResponse.json |    1 +
 .../common/message/DeleteTopicsRequest.json   |    5 +-
 .../common/message/DeleteTopicsResponse.json  |    7 +-
 .../common/message/DescribeAclsRequest.json   |    1 +
 .../common/message/DescribeAclsResponse.json  |    1 +
 .../message/DescribeConfigsRequest.json       |    1 +
 .../message/DescribeConfigsResponse.json      |    1 +
 .../DescribeDelegationTokenRequest.json       |    1 +
 .../DescribeDelegationTokenResponse.json      |    1 +
 .../common/message/DescribeGroupsRequest.json |    7 +-
 .../message/DescribeGroupsResponse.json       |    8 +-
 .../message/DescribeLogDirsRequest.json       |    1 +
 .../message/DescribeLogDirsResponse.json      |    1 +
 .../common/message/ElectLeadersRequest.json   |    6 +-
 .../common/message/ElectLeadersResponse.json  |    6 +-
 .../common/message/EndTxnRequest.json         |    1 +
 .../common/message/EndTxnResponse.json        |    1 +
 .../message/ExpireDelegationTokenRequest.json |    1 +
 .../ExpireDelegationTokenResponse.json        |    1 +
 .../common/message/FetchRequest.json          |    1 +
 .../common/message/FetchResponse.json         |    1 +
 .../message/FindCoordinatorRequest.json       |    6 +-
 .../message/FindCoordinatorResponse.json      |    6 +-
 .../common/message/HeartbeatRequest.json      |    6 +-
 .../common/message/HeartbeatResponse.json     |    7 +-
 .../IncrementalAlterConfigsRequest.json       |    4 +-
 .../IncrementalAlterConfigsResponse.json      |    4 +-
 .../common/message/InitProducerIdRequest.json |    5 +-
 .../message/InitProducerIdResponse.json       |    5 +-
 .../common/message/JoinGroupRequest.json      |    8 +-
 .../common/message/JoinGroupResponse.json     |    9 +-
 .../common/message/LeaderAndIsrRequest.json   |    1 +
 .../common/message/LeaderAndIsrResponse.json  |    1 +
 .../common/message/LeaveGroupRequest.json     |    6 +-
 .../common/message/LeaveGroupResponse.json    |    7 +-
 .../common/message/ListGroupsRequest.json     |    5 +-
 .../common/message/ListGroupsResponse.json    |    6 +-
 .../common/message/ListOffsetRequest.json     |    5 +
 .../common/message/ListOffsetResponse.json    |    5 +
 .../ListPartitionReassignmentsRequest.json    |    1 +
 .../ListPartitionReassignmentsResponse.json   |    1 +
 .../common/message/MetadataRequest.json       |    6 +-
 .../common/message/MetadataResponse.json      |    6 +-
 .../common/message/OffsetCommitRequest.json   |    6 +-
 .../common/message/OffsetCommitResponse.json  |    6 +-
 .../common/message/OffsetFetchRequest.json    |    5 +-
 .../common/message/OffsetFetchResponse.json   |    5 +-
 .../message/OffsetForLeaderEpochRequest.json  |    3 +
 .../message/OffsetForLeaderEpochResponse.json |    1 +
 .../common/message/ProduceRequest.json        |    3 +-
 .../common/message/ProduceResponse.json       |    3 +-
 .../message/RenewDelegationTokenRequest.json  |    1 +
 .../message/RenewDelegationTokenResponse.json |    1 +
 .../common/message/RequestHeader.json         |    5 +-
 .../common/message/ResponseHeader.json        |    3 +-
 .../message/SaslAuthenticateRequest.json      |    1 +
 .../message/SaslAuthenticateResponse.json     |    1 +
 .../common/message/SaslHandshakeRequest.json  |    1 +
 .../common/message/SaslHandshakeResponse.json |    1 +
 .../common/message/StopReplicaRequest.json    |    1 +
 .../common/message/StopReplicaResponse.json   |    1 +
 .../common/message/SyncGroupRequest.json      |    6 +-
 .../common/message/SyncGroupResponse.json     |    7 +-
 .../message/TxnOffsetCommitRequest.json       |    1 +
 .../message/TxnOffsetCommitResponse.json      |    1 +
 .../common/message/UpdateMetadataRequest.json |    1 +
 .../message/UpdateMetadataResponse.json       |    1 +
 .../message/WriteTxnMarkersRequest.json       |    1 +
 .../message/WriteTxnMarkersResponse.json      |    1 +
 .../kafka/clients/NetworkClientTest.java      |    6 +-
 .../common/message/ApiMessageTypeTest.java    |   21 +
 .../kafka/common/message/MessageTest.java     |   98 +-
 .../message/SimpleExampleMessageTest.java     |  104 ++
 .../common/message/TestUUIDDataTest.java      |   86 -
 .../common/protocol/MessageTestUtil.java      |   39 +
 .../common/protocol/MessageUtilTest.java      |   20 -
 .../types/ProtocolSerializationTest.java      |   74 +-
 .../types/RawTaggedFieldWriterTest.java       |   99 ++
 .../requests/LeaderAndIsrRequestTest.java     |    8 +-
 .../common/requests/RequestContextTest.java   |    3 +-
 .../common/requests/RequestHeaderTest.java    |   10 +
 .../common/requests/RequestResponseTest.java  |   17 +-
 .../requests/StopReplicaRequestTest.java      |    4 +-
 .../requests/UpdateMetadataRequestTest.java   |    6 +-
 .../authenticator/SaslAuthenticatorTest.java  |   16 +-
 ...estUUID.json => SimpleExampleMessage.json} |   15 +-
 .../kafka/server/ApiVersionsRequestTest.scala |    6 +-
 .../unit/kafka/server/BaseRequestTest.scala   |   14 +-
 .../kafka/server/EdgeCaseRequestTest.scala    |    6 +-
 .../unit/kafka/server/FetchRequestTest.scala  |    4 +-
 .../unit/kafka/server/KafkaApisTest.scala     |    2 +-
 .../message/ApiMessageTypeGenerator.java      |   72 +
 .../org/apache/kafka/message/FieldSpec.java   |   95 +-
 .../kafka/message/MessageDataGenerator.java   | 1464 +++++++++++------
 .../kafka/message/MessageGenerator.java       |   34 +
 .../org/apache/kafka/message/MessageSpec.java |   25 +-
 .../apache/kafka/message/SchemaGenerator.java |  111 +-
 .../org/apache/kafka/message/StructSpec.java  |   29 +-
 .../kafka/message/VersionConditional.java     |   12 +-
 .../message/MessageDataGeneratorTest.java     |  194 ++-
 .../kafka/message/VersionConditionalTest.java |   14 +-
 151 files changed, 3139 insertions(+), 885 deletions(-)
 create mode 100644 clients/src/main/java/org/apache/kafka/common/protocol/ObjectSerializationCache.java
 create mode 100644 clients/src/main/java/org/apache/kafka/common/protocol/types/CompactArrayOf.java
 create mode 100644 clients/src/main/java/org/apache/kafka/common/protocol/types/RawTaggedField.java
 create mode 100644 clients/src/main/java/org/apache/kafka/common/protocol/types/RawTaggedFieldWriter.java
 create mode 100644 clients/src/main/java/org/apache/kafka/common/protocol/types/TaggedFields.java
 create mode 100644 clients/src/test/java/org/apache/kafka/common/message/SimpleExampleMessageTest.java
 delete mode 100644 clients/src/test/java/org/apache/kafka/common/message/TestUUIDDataTest.java
 create mode 100644 clients/src/test/java/org/apache/kafka/common/protocol/MessageTestUtil.java
 create mode 100644 clients/src/test/java/org/apache/kafka/common/protocol/types/RawTaggedFieldWriterTest.java
 rename clients/src/test/resources/common/message/{TestUUID.json => SimpleExampleMessage.json} (73%)

diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index 7a04f8d5b94f5..8927849784d03 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -70,7 +70,7 @@
     
 
-    
 
      requestBuilder() {
diff --git a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
index 34e3a2f07b960..d782df8652e5d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
@@ -706,7 +706,8 @@ public static AbstractResponse parseResponse(ByteBuffer responseBuffer, RequestH
 
     private static Struct parseStructMaybeUpdateThrottleTimeMetrics(ByteBuffer responseBuffer, RequestHeader requestHeader,
                                                                     Sensor throttleTimeSensor, long now) {
-        ResponseHeader responseHeader = ResponseHeader.parse(responseBuffer, requestHeader.headerVersion());
+        ResponseHeader responseHeader = ResponseHeader.parse(responseBuffer,
+            requestHeader.apiKey().responseHeaderVersion(requestHeader.apiVersion()));
         // Always expect the response version id to be the same as the request version id
         Struct responseBody = requestHeader.apiKey().parseResponse(requestHeader.apiVersion(), responseBuffer);
         correlate(requestHeader, responseHeader);
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
index 31a98f5144158..e2f7f99a082e7 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.common.protocol;
 
+import org.apache.kafka.common.message.ApiMessageType;
 import org.apache.kafka.common.message.ApiVersionsRequestData;
 import org.apache.kafka.common.message.ApiVersionsResponseData;
 import org.apache.kafka.common.message.ControlledShutdownRequestData;
@@ -122,6 +123,8 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.apache.kafka.common.protocol.types.Type.BYTES;
+import static org.apache.kafka.common.protocol.types.Type.COMPACT_BYTES;
+import static org.apache.kafka.common.protocol.types.Type.COMPACT_NULLABLE_BYTES;
 import static org.apache.kafka.common.protocol.types.Type.NULLABLE_BYTES;
 import static org.apache.kafka.common.protocol.types.Type.RECORDS;
 
@@ -334,12 +337,12 @@ public boolean isVersionSupported(short apiVersion) {
         return apiVersion >= oldestVersion() && apiVersion <= latestVersion();
     }
 
-    public short headerVersion(short apiVersion) {
-        if ((this == CONTROLLED_SHUTDOWN) && (apiVersion == 0)) {
-            return 0;
-        } else {
-            return 1;
-        }
+    public short requestHeaderVersion(short apiVersion) {
+        return ApiMessageType.fromApiKey(id).requestHeaderVersion(apiVersion);
+    }
+
+    public short responseHeaderVersion(short apiVersion) {
+        return ApiMessageType.fromApiKey(id).responseHeaderVersion(apiVersion);
     }
 
     private static String toHtml() {
@@ -372,7 +375,8 @@ private static boolean retainsBufferReference(Schema schema) {
         Schema.Visitor detector = new Schema.Visitor() {
             @Override
             public void visit(Type field) {
-                if (field == BYTES || field == NULLABLE_BYTES || field == RECORDS)
+                if (field == BYTES || field == NULLABLE_BYTES || field == RECORDS ||
+                    field == COMPACT_BYTES || field == COMPACT_NULLABLE_BYTES)
                     hasBuffer.set(true);
             }
         };
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ByteBufferAccessor.java b/clients/src/main/java/org/apache/kafka/common/protocol/ByteBufferAccessor.java
index 60a923a70362d..f3dbb87f45cd6 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/ByteBufferAccessor.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/ByteBufferAccessor.java
@@ -17,6 +17,8 @@
 
 package org.apache.kafka.common.protocol;
 
+import org.apache.kafka.common.utils.ByteUtils;
+
 import java.nio.ByteBuffer;
 
 public class ByteBufferAccessor implements Readable, Writable {
@@ -51,6 +53,11 @@ public void readArray(byte[] arr) {
         buf.get(arr);
     }
 
+    @Override
+    public int readUnsignedVarint() {
+        return ByteUtils.readUnsignedVarint(buf);
+    }
+
     @Override
     public void writeByte(byte val) {
         buf.put(val);
@@ -72,7 +79,12 @@ public void writeLong(long val) {
     }
 
     @Override
-    public void writeArray(byte[] arr) {
+    public void writeByteArray(byte[] arr) {
         buf.put(arr);
     }
+
+    @Override
+    public void writeUnsignedVarint(int i) {
+        ByteUtils.writeUnsignedVarint(i, buf);
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Message.java b/clients/src/main/java/org/apache/kafka/common/protocol/Message.java
index 885c69239c570..b31593f1e6f0a 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/Message.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/Message.java
@@ -17,8 +17,11 @@
 
 package org.apache.kafka.common.protocol;
 
+import org.apache.kafka.common.protocol.types.RawTaggedField;
 import org.apache.kafka.common.protocol.types.Struct;
 
+import java.util.List;
+
 /**
  * An object that can serialize itself.  The serialization protocol is versioned.
  * Messages also implement toString, equals, and hashCode.
@@ -37,28 +40,31 @@ public interface Message {
     /**
      * Returns the number of bytes it would take to write out this message.
      *
+     * @param cache         The serialization size cache to populate.
      * @param version       The version to use.
      *
      * @throws {@see org.apache.kafka.common.errors.UnsupportedVersionException}
      *                      If the specified version is too new to be supported
      *                      by this software.
      */
-    int size(short version);
+    int size(ObjectSerializationCache cache, short version);
 
     /**
-     * Writes out this message to the given ByteBuffer.
+     * Writes out this message to the given Writable.
      *
      * @param writable      The destination writable.
+     * @param cache         The object serialization cache to use.  You must have
+     *                      previously populated the size cache using #{Message#size()}.
      * @param version       The version to use.
      *
      * @throws {@see org.apache.kafka.common.errors.UnsupportedVersionException}
      *                      If the specified version is too new to be supported
      *                      by this software.
      */
-    void write(Writable writable, short version);
+    void write(Writable writable, ObjectSerializationCache cache, short version);
 
     /**
-     * Reads this message from the given ByteBuffer.  This will overwrite all
+     * Reads this message from the given Readable.  This will overwrite all
      * relevant fields with information from the byte buffer.
      *
      * @param readable      The source readable.
@@ -89,4 +95,11 @@ public interface Message {
      *                      by this software.
      */
     Struct toStruct(short version);
+
+    /**
+     * Returns a list of tagged fields which this software can't understand.
+     *
+     * @return              The raw tagged fields.
+     */
+    List unknownTaggedFields();
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java b/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java
index 6e31e50c47c23..6e473e780f3c0 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java
@@ -17,8 +17,6 @@
 
 package org.apache.kafka.common.protocol;
 
-import org.apache.kafka.common.utils.Utils;
-
 import java.nio.ByteBuffer;
 import java.util.Iterator;
 import java.util.UUID;
@@ -26,18 +24,6 @@
 public final class MessageUtil {
     public static final UUID ZERO_UUID = new UUID(0L, 0L);
 
-    /**
-     * Get the length of the UTF8 representation of a string, without allocating
-     * a byte buffer for the string.
-     */
-    public static short serializedUtf8Length(CharSequence input) {
-        int count = Utils.utf8Length(input);
-        if (count > Short.MAX_VALUE) {
-            throw new RuntimeException("String " + input + " is too long to serialize.");
-        }
-        return (short) count;
-    }
-
     /**
      * Copy a byte buffer into an array.  This will not affect the buffer's
      * position or mark.
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ObjectSerializationCache.java b/clients/src/main/java/org/apache/kafka/common/protocol/ObjectSerializationCache.java
new file mode 100644
index 0000000000000..98addf443ff0f
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/ObjectSerializationCache.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.common.protocol;
+
+import java.util.IdentityHashMap;
+
+/**
+ * The ObjectSerializationCache stores sizes and values computed during the
+ * first serialization pass.  This avoids recalculating and recomputing the same
+ * values during the second pass.
+ *
+ * It is intended to be used as part of a two-pass serialization process like:
+ * ObjectSerializationCache cache = new ObjectSerializationCache();
+ * message.size(version, cache);
+ * message.write(version, cache);
+ */
+public final class ObjectSerializationCache {
+    private final IdentityHashMap map;
+
+    public ObjectSerializationCache() {
+        this.map = new IdentityHashMap<>();
+    }
+
+    public void setArraySizeInBytes(Object o, int size) {
+        map.put(o, Integer.valueOf(size));
+    }
+
+    public int getArraySizeInBytes(Object o) {
+        Object value = map.get(o);
+        Integer sizeInBytes = (Integer) value;
+        return sizeInBytes;
+    }
+
+    public void cacheSerializedValue(Object o, byte[] val) {
+        map.put(o, val);
+    }
+
+    public byte[] getSerializedValue(Object o) {
+        Object value = map.get(o);
+        return (byte[]) value;
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Readable.java b/clients/src/main/java/org/apache/kafka/common/protocol/Readable.java
index e966bde1662a7..b3e2defd2186c 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/Readable.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/Readable.java
@@ -17,7 +17,11 @@
 
 package org.apache.kafka.common.protocol;
 
+import org.apache.kafka.common.protocol.types.RawTaggedField;
+
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.UUID;
 
 public interface Readable {
@@ -26,34 +30,22 @@ public interface Readable {
     int readInt();
     long readLong();
     void readArray(byte[] arr);
+    int readUnsignedVarint();
 
-    /**
-     * Read a Kafka-delimited string from a byte buffer.  The UTF-8 string
-     * length is stored in a two-byte short.  If the length is negative, the
-     * string is null.
-     */
-    default String readNullableString() {
-        int length = readShort();
-        if (length < 0) {
-            return null;
-        }
+    default String readString(int length) {
         byte[] arr = new byte[length];
         readArray(arr);
         return new String(arr, StandardCharsets.UTF_8);
     }
 
-    /**
-     * Read a Kafka-delimited array from a byte buffer.  The array length is
-     * stored in a four-byte short.
-     */
-    default byte[] readNullableBytes() {
-        int length = readInt();
-        if (length < 0) {
-            return null;
+    default List readUnknownTaggedField(List unknowns, int tag, int size) {
+        if (unknowns == null) {
+            unknowns = new ArrayList<>();
         }
-        byte[] arr = new byte[length];
-        readArray(arr);
-        return arr;
+        byte[] data = new byte[size];
+        readArray(data);
+        unknowns.add(new RawTaggedField(tag, data));
+        return unknowns;
     }
 
     /**
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Writable.java b/clients/src/main/java/org/apache/kafka/common/protocol/Writable.java
index e929e53bc6244..34f6db34bd68c 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/Writable.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/Writable.java
@@ -17,7 +17,6 @@
 
 package org.apache.kafka.common.protocol;
 
-import java.nio.charset.StandardCharsets;
 import java.util.UUID;
 
 public interface Writable {
@@ -25,54 +24,9 @@ public interface Writable {
     void writeShort(short val);
     void writeInt(int val);
     void writeLong(long val);
-    void writeArray(byte[] arr);
+    void writeByteArray(byte[] arr);
+    void writeUnsignedVarint(int i);
 
-    /**
-     * Write a nullable byte array delimited by a four-byte length prefix.
-     */
-    default void writeNullableBytes(byte[] arr) {
-        if (arr == null) {
-            writeInt(-1);
-        } else {
-            writeBytes(arr);
-        }
-    }
-
-    /**
-     * Write a byte array delimited by a four-byte length prefix.
-     */
-    default void writeBytes(byte[] arr) {
-        writeInt(arr.length);
-        writeArray(arr);
-    }
-
-    /**
-     * Write a nullable string delimited by a two-byte length prefix.
-     */
-    default void writeNullableString(String string) {
-        if (string == null) {
-            writeShort((short) -1);
-        } else {
-            writeString(string);
-        }
-    }
-
-    /**
-     * Write a string delimited by a two-byte length prefix.
-     */
-    default void writeString(String string) {
-        byte[] arr = string.getBytes(StandardCharsets.UTF_8);
-        if (arr.length > Short.MAX_VALUE) {
-            throw new RuntimeException("Can't store string longer than " +
-                Short.MAX_VALUE);
-        }
-        writeShort((short) arr.length);
-        writeArray(arr);
-    }
-
-    /**
-     * Write a UUID with the most significant digits first.
-     */
     default void writeUUID(UUID uuid) {
         writeLong(uuid.getMostSignificantBits());
         writeLong(uuid.getLeastSignificantBits());
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/CompactArrayOf.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/CompactArrayOf.java
new file mode 100644
index 0000000000000..4e9f8f8a98153
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/CompactArrayOf.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.protocol.types;
+
+import org.apache.kafka.common.protocol.types.Type.DocumentedType;
+import org.apache.kafka.common.utils.ByteUtils;
+
+import java.nio.ByteBuffer;
+import java.util.Optional;
+
+/**
+ * Represents a type for a compact array of a particular type.
+ * A compact array represents its length with a varint rather than a
+ * fixed-length field.
+ */
+public class CompactArrayOf extends DocumentedType {
+    private static final String COMPACT_ARRAY_TYPE_NAME = "COMPACT_ARRAY";
+
+    private final Type type;
+    private final boolean nullable;
+
+
+    public CompactArrayOf(Type type) {
+        this(type, false);
+    }
+
+    public static CompactArrayOf nullable(Type type) {
+        return new CompactArrayOf(type, true);
+    }
+
+    private CompactArrayOf(Type type, boolean nullable) {
+        this.type = type;
+        this.nullable = nullable;
+    }
+
+    @Override
+    public boolean isNullable() {
+        return nullable;
+    }
+
+    @Override
+    public void write(ByteBuffer buffer, Object o) {
+        if (o == null) {
+            ByteUtils.writeUnsignedVarint(0, buffer);
+            return;
+        }
+        Object[] objs = (Object[]) o;
+        int size = objs.length;
+        ByteUtils.writeUnsignedVarint(size + 1, buffer);
+
+        for (Object obj : objs)
+            type.write(buffer, obj);
+    }
+
+    @Override
+    public Object read(ByteBuffer buffer) {
+        int n = ByteUtils.readUnsignedVarint(buffer);
+        if (n == 0) {
+            if (isNullable()) {
+                return null;
+            } else {
+                throw new SchemaException("This array is not nullable.");
+            }
+        }
+        int size = n - 1;
+        if (size > buffer.remaining())
+            throw new SchemaException("Error reading array of size " + size + ", only " + buffer.remaining() + " bytes available");
+        Object[] objs = new Object[size];
+        for (int i = 0; i < size; i++)
+            objs[i] = type.read(buffer);
+        return objs;
+    }
+
+    @Override
+    public int sizeOf(Object o) {
+        if (o == null) {
+            return 1;
+        }
+        Object[] objs = (Object[]) o;
+        int size = ByteUtils.sizeOfUnsignedVarint(objs.length + 1);
+        for (Object obj : objs) {
+            size += type.sizeOf(obj);
+        }
+        return size;
+    }
+
+    @Override
+    public Optional arrayElementType() {
+        return Optional.of(type);
+    }
+
+    @Override
+    public String toString() {
+        return COMPACT_ARRAY_TYPE_NAME + "(" + type + ")";
+    }
+
+    @Override
+    public Object[] validate(Object item) {
+        try {
+            if (isNullable() && item == null)
+                return null;
+
+            Object[] array = (Object[]) item;
+            for (Object obj : array)
+                type.validate(obj);
+            return array;
+        } catch (ClassCastException e) {
+            throw new SchemaException("Not an Object[].");
+        }
+    }
+
+    @Override
+    public String typeName() {
+        return COMPACT_ARRAY_TYPE_NAME;
+    }
+
+    @Override
+    public String documentation() {
+        return "Represents a sequence of objects of a given type T. " +
+                "Type T can be either a primitive type (e.g. " + STRING + ") or a structure. " +
+                "First, the length N + 1 is given as an UNSIGNED_VARINT. Then N instances of type T follow. " +
+                "A null array is represented with a length of 0. " +
+                "In protocol documentation an array of T instances is referred to as [T].";
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Field.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Field.java
index b79bc932b4203..749b7f3dc961f 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Field.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Field.java
@@ -97,12 +97,24 @@ public Str(String name, String docString) {
         }
     }
 
+    public static class CompactStr extends Field {
+        public CompactStr(String name, String docString) {
+            super(name, Type.COMPACT_STRING, docString, false, null);
+        }
+    }
+
     public static class NullableStr extends Field {
         public NullableStr(String name, String docString) {
             super(name, Type.NULLABLE_STRING, docString, false, null);
         }
     }
 
+    public static class CompactNullableStr extends Field {
+        public CompactNullableStr(String name, String docString) {
+            super(name, Type.COMPACT_NULLABLE_STRING, docString, false, null);
+        }
+    }
+
     public static class Bool extends Field {
         public Bool(String name, String docString) {
             super(name, Type.BOOLEAN, docString, false, null);
@@ -115,6 +127,32 @@ public Array(String name, Type elementType, String docString) {
         }
     }
 
+    public static class CompactArray extends Field {
+        public CompactArray(String name, Type elementType, String docString) {
+            super(name, new CompactArrayOf(elementType), docString, false, null);
+        }
+    }
+
+    public static class TaggedFieldsSection extends Field {
+        private static final String NAME = "_tagged_fields";
+        private static final String DOC_STRING = "The tagged fields";
+
+        /**
+         * Create a new TaggedFieldsSection with the given tags and fields.
+         *
+         * @param fields    This is an array containing Integer tags followed
+         *                  by associated Field objects.
+         * @return          The new {@link TaggedFieldsSection}
+         */
+        public static TaggedFieldsSection of(Object... fields) {
+            return new TaggedFieldsSection(TaggedFields.of(fields));
+        }
+
+        public TaggedFieldsSection(Type type) {
+            super(NAME, type, DOC_STRING, false, null);
+        }
+    }
+
     public static class ComplexArray {
         public final String name;
         public final String docString;
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/RawTaggedField.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/RawTaggedField.java
new file mode 100644
index 0000000000000..60deb5ed5fc26
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/RawTaggedField.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.common.protocol.types;
+
+import java.util.Arrays;
+
+public class RawTaggedField {
+    private final int tag;
+    private final byte[] data;
+
+    public RawTaggedField(int tag, byte[] data) {
+        this.tag = tag;
+        this.data = data;
+    }
+
+    public int tag() {
+        return tag;
+    }
+
+    public byte[] data() {
+        return data;
+    }
+
+    public int size() {
+        return data.length;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if ((o == null) || (!o.getClass().equals(getClass()))) {
+            return false;
+        }
+        RawTaggedField other = (RawTaggedField) o;
+        return tag == other.tag && Arrays.equals(data, other.data);
+    }
+
+    @Override
+    public int hashCode() {
+        return tag ^ Arrays.hashCode(data);
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/RawTaggedFieldWriter.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/RawTaggedFieldWriter.java
new file mode 100644
index 0000000000000..7218d34032788
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/RawTaggedFieldWriter.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.common.protocol.types;
+
+import org.apache.kafka.common.protocol.Writable;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ListIterator;
+
+/**
+ * The RawTaggedFieldWriter is used by Message subclasses to serialize their
+ * lists of raw tags.
+ */
+public class RawTaggedFieldWriter {
+    private static final RawTaggedFieldWriter EMPTY_WRITER =
+        new RawTaggedFieldWriter(new ArrayList<>(0));
+
+    private final List fields;
+    private final ListIterator iter;
+    private int prevTag;
+
+    public static RawTaggedFieldWriter forFields(List fields) {
+        if (fields == null) {
+            return EMPTY_WRITER;
+        }
+        return new RawTaggedFieldWriter(fields);
+    }
+
+    private RawTaggedFieldWriter(List fields) {
+        this.fields = fields;
+        this.iter = this.fields.listIterator();
+        this.prevTag = -1;
+    }
+
+    public int numFields() {
+        return fields.size();
+    }
+
+    public void writeRawTags(Writable writable, int nextDefinedTag) {
+        while (iter.hasNext()) {
+            RawTaggedField field = iter.next();
+            int tag = field.tag();
+            if (tag >= nextDefinedTag) {
+                if (tag == nextDefinedTag) {
+                    // We must not have a raw tag field that duplicates the tag of another field.
+                    throw new RuntimeException("Attempted to use tag " + tag + " as an " +
+                        "undefined tag.");
+                }
+                iter.previous();
+                return;
+            }
+            if (tag <= prevTag) {
+                // The raw tag field list must be sorted by tag, and there must not be
+                // any duplicate tags.
+                throw new RuntimeException("Invalid raw tag field list: tag " + tag +
+                    " comes after tag " + prevTag + ", but is not higher than it.");
+            }
+            writable.writeUnsignedVarint(field.tag());
+            writable.writeUnsignedVarint(field.data().length);
+            writable.writeByteArray(field.data());
+            prevTag = tag;
+        }
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/TaggedFields.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/TaggedFields.java
new file mode 100644
index 0000000000000..4e1ab0d4d5add
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/TaggedFields.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.protocol.types;
+
+import org.apache.kafka.common.protocol.types.Type.DocumentedType;
+import org.apache.kafka.common.utils.ByteUtils;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+
+/**
+ * Represents a tagged fields section.
+ */
+public class TaggedFields extends DocumentedType {
+    private static final String TAGGED_FIELDS_TYPE_NAME = "TAGGED_FIELDS";
+
+    private final Map fields;
+
+    /**
+     * Create a new TaggedFields object with the given tags and fields.
+     *
+     * @param fields    This is an array containing Integer tags followed
+     *                  by associated Field objects.
+     * @return          The new {@link TaggedFields}
+     */
+    @SuppressWarnings("unchecked")
+    public static TaggedFields of(Object... fields) {
+        if (fields.length % 2 != 0) {
+            throw new RuntimeException("TaggedFields#of takes an even " +
+                "number of parameters.");
+        }
+        TreeMap newFields = new TreeMap<>();
+        for (int i = 0; i < fields.length; i += 2) {
+            Integer tag = (Integer) fields[i];
+            Field field = (Field) fields[i + 1];
+            newFields.put(tag, field);
+        }
+        return new TaggedFields(newFields);
+    }
+
+    public TaggedFields(Map fields) {
+        this.fields = fields;
+    }
+
+    @Override
+    public boolean isNullable() {
+        return false;
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public void write(ByteBuffer buffer, Object o) {
+        NavigableMap objects = (NavigableMap) o;
+        ByteUtils.writeUnsignedVarint(objects.size(), buffer);
+        for (Map.Entry entry : objects.entrySet()) {
+            Integer tag = entry.getKey();
+            Field field = fields.get(tag);
+            ByteUtils.writeUnsignedVarint(tag, buffer);
+            if (field == null) {
+                RawTaggedField value = (RawTaggedField) entry.getValue();
+                ByteUtils.writeUnsignedVarint(value.data().length, buffer);
+                buffer.put(value.data());
+            } else {
+                ByteUtils.writeUnsignedVarint(field.type.sizeOf(entry.getValue()), buffer);
+                field.type.write(buffer, entry.getValue());
+            }
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public NavigableMap read(ByteBuffer buffer) {
+        int numTaggedFields = ByteUtils.readUnsignedVarint(buffer);
+        if (numTaggedFields == 0) {
+            return Collections.emptyNavigableMap();
+        }
+        NavigableMap objects = new TreeMap<>();
+        int prevTag = -1;
+        for (int i = 0; i < numTaggedFields; i++) {
+            int tag = ByteUtils.readUnsignedVarint(buffer);
+            if (tag <= prevTag) {
+                throw new RuntimeException("Invalid or out-of-order tag " + tag);
+            }
+            prevTag = tag;
+            int size = ByteUtils.readUnsignedVarint(buffer);
+            Field field = fields.get(tag);
+            if (field == null) {
+                byte[] bytes = new byte[size];
+                buffer.get(bytes);
+                objects.put(tag, new RawTaggedField(tag, bytes));
+            } else {
+                objects.put(tag, field.type.read(buffer));
+            }
+        }
+        return objects;
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public int sizeOf(Object o) {
+        int size = 0;
+        NavigableMap objects = (NavigableMap) o;
+        size += ByteUtils.sizeOfUnsignedVarint(objects.size());
+        for (Map.Entry entry : objects.entrySet()) {
+            Integer tag = entry.getKey();
+            size += ByteUtils.sizeOfUnsignedVarint(tag);
+            Field field = fields.get(tag);
+            if (field == null) {
+                RawTaggedField value = (RawTaggedField) entry.getValue();
+                size += value.data().length + ByteUtils.sizeOfUnsignedVarint(value.data().length);
+            } else {
+                int valueSize = field.type.sizeOf(entry.getValue());
+                size += valueSize + ByteUtils.sizeOfUnsignedVarint(valueSize);
+            }
+        }
+        return size;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder bld = new StringBuilder("TAGGED_FIELDS_TYPE_NAME(");
+        String prefix = "";
+        for (Map.Entry field : fields.entrySet()) {
+            bld.append(prefix);
+            prefix = ", ";
+            bld.append(field.getKey()).append(" -> ").append(field.getValue().toString());
+        }
+        bld.append(")");
+        return bld.toString();
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public Map validate(Object item) {
+        try {
+            NavigableMap objects = (NavigableMap) item;
+            for (Map.Entry entry : objects.entrySet()) {
+                Integer tag = entry.getKey();
+                Field field = fields.get(tag);
+                if (field == null) {
+                    if (!(entry.getValue() instanceof RawTaggedField)) {
+                        throw new SchemaException("The value associated with tag " + tag +
+                            " must be a RawTaggedField in this version of the software.");
+                    }
+                } else {
+                    field.type.validate(entry.getValue());
+                }
+            }
+            return objects;
+        } catch (ClassCastException e) {
+            throw new SchemaException("Not a NavigableMap.");
+        }
+    }
+
+    @Override
+    public String typeName() {
+        return TAGGED_FIELDS_TYPE_NAME;
+    }
+
+    @Override
+    public String documentation() {
+        return "Represents a series of tagged fields.";
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java
index aa0ee40e68097..6183625a785f8 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java
@@ -415,6 +415,56 @@ public String documentation() {
         }
     };
 
+    public static final DocumentedType COMPACT_STRING = new DocumentedType() {
+        @Override
+        public void write(ByteBuffer buffer, Object o) {
+            byte[] bytes = Utils.utf8((String) o);
+            if (bytes.length > Short.MAX_VALUE)
+                throw new SchemaException("String length " + bytes.length + " is larger than the maximum string length.");
+            ByteUtils.writeUnsignedVarint(bytes.length + 1, buffer);
+            buffer.put(bytes);
+        }
+
+        @Override
+        public String read(ByteBuffer buffer) {
+            int length = ByteUtils.readUnsignedVarint(buffer) - 1;
+            if (length < 0)
+                throw new SchemaException("String length " + length + " cannot be negative");
+            if (length > Short.MAX_VALUE)
+                throw new SchemaException("String length " + length + " is larger than the maximum string length.");
+            if (length > buffer.remaining())
+                throw new SchemaException("Error reading string of length " + length + ", only " + buffer.remaining() + " bytes available");
+            String result = Utils.utf8(buffer, length);
+            buffer.position(buffer.position() + length);
+            return result;
+        }
+
+        @Override
+        public int sizeOf(Object o) {
+            int length = Utils.utf8Length((String) o);
+            return ByteUtils.sizeOfUnsignedVarint(length + 1) + length;
+        }
+
+        @Override
+        public String typeName() {
+            return "COMPACT_STRING";
+        }
+
+        @Override
+        public String validate(Object item) {
+            if (item instanceof String)
+                return (String) item;
+            else
+                throw new SchemaException(item + " is not a String.");
+        }
+
+        @Override
+        public String documentation() {
+            return "Represents a sequence of characters. First the length N + 1 is given as an UNSIGNED_VARINT " +
+                    ". Then N bytes follow which are the UTF-8 encoding of the character sequence.";
+        }
+    };
+
     public static final DocumentedType NULLABLE_STRING = new DocumentedType() {
         @Override
         public boolean isNullable() {
@@ -479,6 +529,74 @@ public String documentation() {
         }
     };
 
+    public static final DocumentedType COMPACT_NULLABLE_STRING = new DocumentedType() {
+        @Override
+        public boolean isNullable() {
+            return true;
+        }
+
+        @Override
+        public void write(ByteBuffer buffer, Object o) {
+            if (o == null) {
+                ByteUtils.writeUnsignedVarint(0, buffer);
+            } else {
+                byte[] bytes = Utils.utf8((String) o);
+                if (bytes.length > Short.MAX_VALUE)
+                    throw new SchemaException("String length " + bytes.length + " is larger than the maximum string length.");
+                ByteUtils.writeUnsignedVarint(bytes.length + 1, buffer);
+                buffer.put(bytes);
+            }
+        }
+
+        @Override
+        public String read(ByteBuffer buffer) {
+            int length = ByteUtils.readUnsignedVarint(buffer) - 1;
+            if (length < 0) {
+                return null;
+            } else if (length > Short.MAX_VALUE) {
+                throw new SchemaException("String length " + length + " is larger than the maximum string length.");
+            } else if (length > buffer.remaining()) {
+                throw new SchemaException("Error reading string of length " + length + ", only " + buffer.remaining() + " bytes available");
+            } else {
+                String result = Utils.utf8(buffer, length);
+                buffer.position(buffer.position() + length);
+                return result;
+            }
+        }
+
+        @Override
+        public int sizeOf(Object o) {
+            if (o == null) {
+                return 1;
+            }
+            int length = Utils.utf8Length((String) o);
+            return ByteUtils.sizeOfUnsignedVarint(length + 1) + length;
+        }
+
+        @Override
+        public String typeName() {
+            return "COMPACT_NULLABLE_STRING";
+        }
+
+        @Override
+        public String validate(Object item) {
+            if (item == null) {
+                return null;
+            } else if (item instanceof String) {
+                return (String) item;
+            } else {
+                throw new SchemaException(item + " is not a String.");
+            }
+        }
+
+        @Override
+        public String documentation() {
+            return "Represents a sequence of characters. First the length N + 1 is given as an UNSIGNED_VARINT " +
+                    ". Then N bytes follow which are the UTF-8 encoding of the character sequence. " +
+                    "A null string is represented with a length of 0.";
+        }
+    };
+
     public static final DocumentedType BYTES = new DocumentedType() {
         @Override
         public void write(ByteBuffer buffer, Object o) {
@@ -529,6 +647,57 @@ public String documentation() {
         }
     };
 
+    public static final DocumentedType COMPACT_BYTES = new DocumentedType() {
+        @Override
+        public void write(ByteBuffer buffer, Object o) {
+            ByteBuffer arg = (ByteBuffer) o;
+            int pos = arg.position();
+            ByteUtils.writeUnsignedVarint(arg.remaining() + 1, buffer);
+            buffer.put(arg);
+            arg.position(pos);
+        }
+
+        @Override
+        public Object read(ByteBuffer buffer) {
+            int size = ByteUtils.readUnsignedVarint(buffer) - 1;
+            if (size < 0)
+                throw new SchemaException("Bytes size " + size + " cannot be negative");
+            if (size > buffer.remaining())
+                throw new SchemaException("Error reading bytes of size " + size + ", only " + buffer.remaining() + " bytes available");
+
+            ByteBuffer val = buffer.slice();
+            val.limit(size);
+            buffer.position(buffer.position() + size);
+            return val;
+        }
+
+        @Override
+        public int sizeOf(Object o) {
+            ByteBuffer buffer = (ByteBuffer) o;
+            int remaining = buffer.remaining();
+            return ByteUtils.sizeOfUnsignedVarint(remaining + 1) + remaining;
+        }
+
+        @Override
+        public String typeName() {
+            return "COMPACT_BYTES";
+        }
+
+        @Override
+        public ByteBuffer validate(Object item) {
+            if (item instanceof ByteBuffer)
+                return (ByteBuffer) item;
+            else
+                throw new SchemaException(item + " is not a java.nio.ByteBuffer.");
+        }
+
+        @Override
+        public String documentation() {
+            return "Represents a raw sequence of bytes. First the length N+1 is given as an UNSIGNED_VARINT." +
+                    "Then N bytes follow.";
+        }
+    };
+
     public static final DocumentedType NULLABLE_BYTES = new DocumentedType() {
         @Override
         public boolean isNullable() {
@@ -595,6 +764,72 @@ public String documentation() {
         }
     };
 
+    public static final DocumentedType COMPACT_NULLABLE_BYTES = new DocumentedType() {
+        @Override
+        public boolean isNullable() {
+            return true;
+        }
+
+        @Override
+        public void write(ByteBuffer buffer, Object o) {
+            if (o == null) {
+                ByteUtils.writeUnsignedVarint(0, buffer);
+            } else {
+                ByteBuffer arg = (ByteBuffer) o;
+                int pos = arg.position();
+                ByteUtils.writeUnsignedVarint(arg.remaining() + 1, buffer);
+                buffer.put(arg);
+                arg.position(pos);
+            }
+        }
+
+        @Override
+        public Object read(ByteBuffer buffer) {
+            int size = ByteUtils.readUnsignedVarint(buffer) - 1;
+            if (size < 0)
+                return null;
+            if (size > buffer.remaining())
+                throw new SchemaException("Error reading bytes of size " + size + ", only " + buffer.remaining() + " bytes available");
+
+            ByteBuffer val = buffer.slice();
+            val.limit(size);
+            buffer.position(buffer.position() + size);
+            return val;
+        }
+
+        @Override
+        public int sizeOf(Object o) {
+            if (o == null) {
+                return 1;
+            }
+            ByteBuffer buffer = (ByteBuffer) o;
+            int remaining = buffer.remaining();
+            return ByteUtils.sizeOfUnsignedVarint(remaining + 1) + remaining;
+        }
+
+        @Override
+        public String typeName() {
+            return "COMPACT_NULLABLE_BYTES";
+        }
+
+        @Override
+        public ByteBuffer validate(Object item) {
+            if (item == null)
+                return null;
+
+            if (item instanceof ByteBuffer)
+                return (ByteBuffer) item;
+
+            throw new SchemaException(item + " is not a java.nio.ByteBuffer.");
+        }
+
+        @Override
+        public String documentation() {
+            return "Represents a raw sequence of bytes. First the length N+1 is given as an UNSIGNED_VARINT." +
+                    "Then N bytes follow. A null object is represented with a length of 0.";
+        }
+    };
+
     public static final DocumentedType RECORDS = new DocumentedType() {
         @Override
         public boolean isNullable() {
@@ -722,8 +957,9 @@ private static String toHtml() {
         DocumentedType[] types = {
             BOOLEAN, INT8, INT16, INT32, INT64,
             UNSIGNED_INT32, VARINT, VARLONG, UUID,
-            STRING, NULLABLE_STRING, BYTES, NULLABLE_BYTES,
-            RECORDS, new ArrayOf(STRING)};
+            STRING, COMPACT_STRING, NULLABLE_STRING, COMPACT_NULLABLE_STRING,
+            BYTES, COMPACT_BYTES, NULLABLE_BYTES, COMPACT_NULLABLE_BYTES,
+            RECORDS, new ArrayOf(STRING), new CompactArrayOf(COMPACT_STRING)};
         final StringBuilder b = new StringBuilder();
         b.append("\n");
         b.append("");
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
index 835541e3575c8..8a6edf14cbac0 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
@@ -46,7 +46,8 @@ public ByteBuffer serialize(ApiKeys apiKey, int correlationId) {
      * Visible for testing, typically {@link #toSend(String, ResponseHeader, short)} should be used instead.
      */
     public ByteBuffer serialize(ApiKeys apiKey, short version, int correlationId) {
-        ResponseHeader header = new ResponseHeader(correlationId, apiKey.headerVersion(version));
+        ResponseHeader header =
+            new ResponseHeader(correlationId, apiKey.responseHeaderVersion(version));
         return serialize(header.toStruct(), toStruct(version));
     }
 
@@ -116,7 +117,7 @@ public static AbstractResponse parseResponse(ApiKeys apiKey, Struct struct, shor
             case SASL_HANDSHAKE:
                 return new SaslHandshakeResponse(struct, version);
             case API_VERSIONS:
-                return ApiVersionsResponse.apiVersionsResponse(struct, version);
+                return ApiVersionsResponse.fromStruct(struct, version);
             case CREATE_TOPICS:
                 return new CreateTopicsResponse(struct, version);
             case DELETE_TOPICS:
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java
index fd214043ebabc..4604fedd0b0e3 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java
@@ -20,6 +20,7 @@
 import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKey;
 import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKeyCollection;
 import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.ByteBufferAccessor;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.protocol.types.SchemaException;
 import org.apache.kafka.common.protocol.types.Struct;
@@ -77,10 +78,26 @@ public boolean shouldClientThrottle(short version) {
     }
 
     public static ApiVersionsResponse parse(ByteBuffer buffer, short version) {
-        return new ApiVersionsResponse(ApiKeys.API_VERSIONS.parseResponse(version, buffer), version);
+        // Fallback to version 0 for ApiVersions response. If a client sends an ApiVersionsRequest
+        // using a version higher than that supported by the broker, a version 0 response is sent
+        // to the client indicating UNSUPPORTED_VERSION. When the client receives the response, it
+        // falls back while parsing it into a Struct which means that the version received by this
+        // method is not necessary the real one. It may be version 0 as well.
+        int prev = buffer.position();
+        try {
+            return new ApiVersionsResponse(
+                new ApiVersionsResponseData(new ByteBufferAccessor(buffer), version));
+        } catch (RuntimeException e) {
+            buffer.position(prev);
+            if (version != 0)
+                return new ApiVersionsResponse(
+                    new ApiVersionsResponseData(new ByteBufferAccessor(buffer), (short) 0));
+            else
+                throw e;
+        }
     }
 
-    public static ApiVersionsResponse apiVersionsResponse(Struct struct, short version) {
+    public static ApiVersionsResponse fromStruct(Struct struct, short version) {
         // Fallback to version 0 for ApiVersions response. If a client sends an ApiVersionsRequest
         // using a version higher than that supported by the broker, a version 0 response is sent
         // to the client indicating UNSUPPORTED_VERSION. When the client receives the response, it
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
index b799e360bd4f4..63013fff2b783 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
@@ -24,7 +24,6 @@
 import org.apache.kafka.common.message.LeaderAndIsrResponseData;
 import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError;
 import org.apache.kafka.common.protocol.ApiKeys;
-import org.apache.kafka.common.protocol.ByteBufferAccessor;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.protocol.types.Struct;
 import org.apache.kafka.common.utils.FlattenedIterator;
@@ -133,13 +132,6 @@ protected Struct toStruct() {
         return data.toStruct(version());
     }
 
-    protected ByteBuffer toBytes() {
-        ByteBuffer bytes = ByteBuffer.allocate(size());
-        data.write(new ByteBufferAccessor(bytes), version());
-        bytes.flip();
-        return bytes;
-    }
-
     @Override
     public LeaderAndIsrResponse getErrorResponse(int throttleTimeMs, Throwable e) {
         LeaderAndIsrResponseData responseData = new LeaderAndIsrResponseData();
@@ -194,12 +186,12 @@ public List liveLeaders() {
         return Collections.unmodifiableList(data.liveLeaders());
     }
 
-    protected int size() {
-        return data.size(version());
+    // Visible for testing
+    LeaderAndIsrRequestData data() {
+        return data;
     }
 
     public static LeaderAndIsrRequest parse(ByteBuffer buffer, short version) {
         return new LeaderAndIsrRequest(ApiKeys.LEADER_AND_ISR.parseRequest(version, buffer), version);
     }
-
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java
index 65bc39d230a24..8da1eb5fde9ed 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java
@@ -42,7 +42,7 @@ public RequestHeader(ApiKeys requestApiKey, short requestVersion, String clientI
                 setRequestApiVersion(requestVersion).
                 setClientId(clientId).
                 setCorrelationId(correlationId),
-            ApiKeys.forId(requestApiKey.id).headerVersion(requestVersion));
+            ApiKeys.forId(requestApiKey.id).requestHeaderVersion(requestVersion));
     }
 
     public RequestHeader(RequestHeaderData data, short headerVersion) {
@@ -79,7 +79,8 @@ public RequestHeaderData data() {
     }
 
     public ResponseHeader toResponseHeader() {
-        return new ResponseHeader(data.correlationId(), headerVersion);
+        return new ResponseHeader(data.correlationId(),
+            apiKey().responseHeaderVersion(apiVersion()));
     }
 
     public static RequestHeader parse(ByteBuffer buffer) {
@@ -87,7 +88,7 @@ public static RequestHeader parse(ByteBuffer buffer) {
         try {
             apiKey = buffer.getShort();
             short apiVersion = buffer.getShort();
-            short headerVersion = ApiKeys.forId(apiKey).headerVersion(apiVersion);
+            short headerVersion = ApiKeys.forId(apiKey).requestHeaderVersion(apiVersion);
             buffer.rewind();
             return new RequestHeader(new RequestHeaderData(
                 new ByteBufferAccessor(buffer), headerVersion), headerVersion);
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
index e79228917fb32..425f66a161900 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
@@ -169,13 +169,13 @@ public static StopReplicaRequest parse(ByteBuffer buffer, short version) {
         return new StopReplicaRequest(ApiKeys.STOP_REPLICA.parseRequest(version, buffer), version);
     }
 
+    // Visible for testing
+    StopReplicaRequestData data() {
+        return data;
+    }
+
     @Override
     protected Struct toStruct() {
         return data.toStruct(version());
     }
-
-    protected long size() {
-        return data.size(version());
-    }
-
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
index b73fa5b5c1a4b..90dbff3430256 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
@@ -25,7 +25,6 @@
 import org.apache.kafka.common.message.UpdateMetadataResponseData;
 import org.apache.kafka.common.network.ListenerName;
 import org.apache.kafka.common.protocol.ApiKeys;
-import org.apache.kafka.common.protocol.ByteBufferAccessor;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.protocol.types.Struct;
 import org.apache.kafka.common.security.auth.SecurityProtocol;
@@ -213,19 +212,12 @@ protected Struct toStruct() {
         return data.toStruct(version());
     }
 
-    protected ByteBuffer toBytes() {
-        ByteBuffer bytes = ByteBuffer.allocate(size());
-        data.write(new ByteBufferAccessor(bytes), version());
-        bytes.flip();
-        return bytes;
-    }
-
-    protected int size() {
-        return data.size(version());
+    // Visible for testing
+    UpdateMetadataRequestData data() {
+        return data;
     }
 
     public static UpdateMetadataRequest parse(ByteBuffer buffer, short version) {
         return new UpdateMetadataRequest(ApiKeys.UPDATE_METADATA.parseRequest(version, buffer), version);
     }
-
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
index 84f481f80e434..6aa68cd57dabd 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
@@ -332,7 +332,7 @@ private RequestHeader nextRequestHeader(ApiKeys apiKey, short version) {
                 setRequestApiVersion(version).
                 setClientId(clientId).
                 setCorrelationId(correlationId++),
-            apiKey.headerVersion(version));
+            apiKey.requestHeaderVersion(version));
         return currentRequestHeader;
     }
 
diff --git a/clients/src/main/resources/common/message/AddOffsetsToTxnRequest.json b/clients/src/main/resources/common/message/AddOffsetsToTxnRequest.json
index 604a9603f74d6..acf7301664aca 100644
--- a/clients/src/main/resources/common/message/AddOffsetsToTxnRequest.json
+++ b/clients/src/main/resources/common/message/AddOffsetsToTxnRequest.json
@@ -19,6 +19,7 @@
   "name": "AddOffsetsToTxnRequest",
   // Version 1 is the same as version 0.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "TransactionalId", "type": "string", "versions": "0+", "entityType": "transactionalId",
       "about": "The transactional id corresponding to the transaction."},
diff --git a/clients/src/main/resources/common/message/AddOffsetsToTxnResponse.json b/clients/src/main/resources/common/message/AddOffsetsToTxnResponse.json
index 080974226fa7a..4e44006aa9324 100644
--- a/clients/src/main/resources/common/message/AddOffsetsToTxnResponse.json
+++ b/clients/src/main/resources/common/message/AddOffsetsToTxnResponse.json
@@ -19,6 +19,7 @@
   "name": "AddOffsetsToTxnResponse",
   // Starting in version 1, on quota violation brokers send out responses before throttling.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
       "about": "Duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/AddPartitionsToTxnRequest.json b/clients/src/main/resources/common/message/AddPartitionsToTxnRequest.json
index 4d07b5bbcb0af..13a5a6e7e9b47 100644
--- a/clients/src/main/resources/common/message/AddPartitionsToTxnRequest.json
+++ b/clients/src/main/resources/common/message/AddPartitionsToTxnRequest.json
@@ -19,6 +19,7 @@
   "name": "AddPartitionsToTxnRequest",
   // Version 1 is the same as version 0.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "TransactionalId", "type": "string", "versions": "0+", "entityType": "transactionalId",
       "about": "The transactional id corresponding to the transaction."},
diff --git a/clients/src/main/resources/common/message/AddPartitionsToTxnResponse.json b/clients/src/main/resources/common/message/AddPartitionsToTxnResponse.json
index ec69da438b7f3..e143a3f1772c5 100644
--- a/clients/src/main/resources/common/message/AddPartitionsToTxnResponse.json
+++ b/clients/src/main/resources/common/message/AddPartitionsToTxnResponse.json
@@ -19,6 +19,7 @@
   "name": "AddPartitionsToTxnResponse",
   // Starting in version 1, on quota violation brokers send out responses before throttling.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
       "about": "Duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/AlterConfigsRequest.json b/clients/src/main/resources/common/message/AlterConfigsRequest.json
index 6d037654d0c50..6f8d72c95d13b 100644
--- a/clients/src/main/resources/common/message/AlterConfigsRequest.json
+++ b/clients/src/main/resources/common/message/AlterConfigsRequest.json
@@ -19,6 +19,7 @@
   "name": "AlterConfigsRequest",
   // Version 1 is the same as version 0.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "Resources", "type": "[]AlterConfigsResource", "versions": "0+", 
       "about": "The updates for each resource.", "fields": [
diff --git a/clients/src/main/resources/common/message/AlterConfigsResponse.json b/clients/src/main/resources/common/message/AlterConfigsResponse.json
index 5c559cfd3ec8a..288103df46237 100644
--- a/clients/src/main/resources/common/message/AlterConfigsResponse.json
+++ b/clients/src/main/resources/common/message/AlterConfigsResponse.json
@@ -19,6 +19,7 @@
   "name": "AlterConfigsResponse",
   // Starting in version 1, on quota violation brokers send out responses before throttling.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
       "about": "Duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json b/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json
index f962e1e65db37..ecf7eca6eae43 100644
--- a/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json
+++ b/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json
@@ -18,6 +18,7 @@
   "type": "request",
   "name": "AlterPartitionReassignmentsRequest",
   "validVersions": "0",
+  "flexibleVersions": "0+",
   "fields": [
     { "name": "TimeoutMs", "type": "int32", "versions": "0+", "default": "60000",
       "about": "The time in ms to wait for the request to complete." },
diff --git a/clients/src/main/resources/common/message/AlterPartitionReassignmentsResponse.json b/clients/src/main/resources/common/message/AlterPartitionReassignmentsResponse.json
index d04959678f64f..3fa08883d1bd5 100644
--- a/clients/src/main/resources/common/message/AlterPartitionReassignmentsResponse.json
+++ b/clients/src/main/resources/common/message/AlterPartitionReassignmentsResponse.json
@@ -18,6 +18,7 @@
   "type": "response",
   "name": "AlterPartitionReassignmentsResponse",
   "validVersions": "0",
+  "flexibleVersions": "0+",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/AlterReplicaLogDirsRequest.json b/clients/src/main/resources/common/message/AlterReplicaLogDirsRequest.json
index 826c3ca828a6c..d82ee3c975951 100644
--- a/clients/src/main/resources/common/message/AlterReplicaLogDirsRequest.json
+++ b/clients/src/main/resources/common/message/AlterReplicaLogDirsRequest.json
@@ -19,6 +19,7 @@
   "name": "AlterReplicaLogDirsRequest",
   // Version 1 is the same as version 0.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "Dirs", "type": "[]AlterReplicaLogDir", "versions": "0+", 
       "about": "The alterations to make for each directory.", "fields": [
diff --git a/clients/src/main/resources/common/message/AlterReplicaLogDirsResponse.json b/clients/src/main/resources/common/message/AlterReplicaLogDirsResponse.json
index 047baeec3a076..75e6f607822a8 100644
--- a/clients/src/main/resources/common/message/AlterReplicaLogDirsResponse.json
+++ b/clients/src/main/resources/common/message/AlterReplicaLogDirsResponse.json
@@ -19,6 +19,7 @@
   "name": "AlterReplicaLogDirsResponse",
   // Starting in version 1, on quota violation brokers send out responses before throttling.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
       "about": "Duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/ApiVersionsRequest.json b/clients/src/main/resources/common/message/ApiVersionsRequest.json
index 59c3388703d07..79fe7a735274f 100644
--- a/clients/src/main/resources/common/message/ApiVersionsRequest.json
+++ b/clients/src/main/resources/common/message/ApiVersionsRequest.json
@@ -21,8 +21,7 @@
   //
   // Version 3 is the first flexible version and adds ClientSoftwareName and ClientSoftwareVersion.
   "validVersions": "0-3",
-  // TODO(dajac) Enable this once KIP-482 is committed.
-  // "flexibleVersions": "3+",
+  "flexibleVersions": "3+",
   "fields": [
     { "name": "ClientSoftwareName", "type": "string", "versions": "3+",
       "about": "The name of the client." },
diff --git a/clients/src/main/resources/common/message/ApiVersionsResponse.json b/clients/src/main/resources/common/message/ApiVersionsResponse.json
index ccb44c671f723..ecbc448504918 100644
--- a/clients/src/main/resources/common/message/ApiVersionsResponse.json
+++ b/clients/src/main/resources/common/message/ApiVersionsResponse.json
@@ -28,8 +28,7 @@
   // Starting from Apache Kafka 2.4 (KIP-511), ApiKeys field is populated with the supported
   // versions of the ApiVersionsRequest when an UNSUPPORTED_VERSION error is returned.
   "validVersions": "0-3",
-  // TODO(dajac) Enable this once KIP-482 is committed.
-  // "flexibleVersions": "3+",
+  "flexibleVersions": "3+",
   "fields": [
     { "name": "ErrorCode", "type": "int16", "versions": "0+",
       "about": "The top-level error code." },
diff --git a/clients/src/main/resources/common/message/ControlledShutdownRequest.json b/clients/src/main/resources/common/message/ControlledShutdownRequest.json
index a6c3d9d6b8668..6592e78d870d9 100644
--- a/clients/src/main/resources/common/message/ControlledShutdownRequest.json
+++ b/clients/src/main/resources/common/message/ControlledShutdownRequest.json
@@ -25,6 +25,7 @@
   //
   // Version 2 adds BrokerEpoch.
   "validVersions": "0-2",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "BrokerId", "type": "int32", "versions": "0+", "entityType": "brokerId",
       "about": "The id of the broker for which controlled shutdown has been requested." },
diff --git a/clients/src/main/resources/common/message/ControlledShutdownResponse.json b/clients/src/main/resources/common/message/ControlledShutdownResponse.json
index 53ee8f78f7250..dc61d70e4d4c9 100644
--- a/clients/src/main/resources/common/message/ControlledShutdownResponse.json
+++ b/clients/src/main/resources/common/message/ControlledShutdownResponse.json
@@ -19,6 +19,7 @@
   "name": "ControlledShutdownResponse",
   // Versions 1 and 2 are the same as version 0.
   "validVersions": "0-2",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ErrorCode", "type": "int16", "versions": "0+",
       "about": "The top-level error code." },
diff --git a/clients/src/main/resources/common/message/CreateAclsRequest.json b/clients/src/main/resources/common/message/CreateAclsRequest.json
index 0e022a8954dc2..72d4d112c0d55 100644
--- a/clients/src/main/resources/common/message/CreateAclsRequest.json
+++ b/clients/src/main/resources/common/message/CreateAclsRequest.json
@@ -19,6 +19,7 @@
   "name": "CreateAclsRequest",
   // Version 1 adds resource pattern type.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "Creations", "type": "[]CreatableAcl", "versions": "0+", 
       "about": "The ACLs that we want to create.", "fields": [
diff --git a/clients/src/main/resources/common/message/CreateAclsResponse.json b/clients/src/main/resources/common/message/CreateAclsResponse.json
index ede3fef9058c4..d84b7235c3889 100644
--- a/clients/src/main/resources/common/message/CreateAclsResponse.json
+++ b/clients/src/main/resources/common/message/CreateAclsResponse.json
@@ -19,6 +19,7 @@
   "name": "CreateAclsResponse",
   // Starting in version 1, on quota violation, brokers send out responses before throttling.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/CreateDelegationTokenRequest.json b/clients/src/main/resources/common/message/CreateDelegationTokenRequest.json
index 2de43b9654261..8d881355aa9d2 100644
--- a/clients/src/main/resources/common/message/CreateDelegationTokenRequest.json
+++ b/clients/src/main/resources/common/message/CreateDelegationTokenRequest.json
@@ -18,7 +18,10 @@
   "type": "request",
   "name": "CreateDelegationTokenRequest",
   // Version 1 is the same as version 0.
-  "validVersions": "0-1",
+  //
+  // Version 2 is the first flexible version.
+  "validVersions": "0-2",
+  "flexibleVersions": "2+",
   "fields": [
     { "name": "Renewers", "type": "[]CreatableRenewers", "versions": "0+",
       "about": "A list of those who are allowed to renew this token before it expires.", "fields": [
diff --git a/clients/src/main/resources/common/message/CreateDelegationTokenResponse.json b/clients/src/main/resources/common/message/CreateDelegationTokenResponse.json
index 61367e4991c60..74ad905b94b26 100644
--- a/clients/src/main/resources/common/message/CreateDelegationTokenResponse.json
+++ b/clients/src/main/resources/common/message/CreateDelegationTokenResponse.json
@@ -18,7 +18,10 @@
   "type": "response",
   "name": "CreateDelegationTokenResponse",
   // Starting in version 1, on quota violation, brokers send out responses before throttling.
-  "validVersions": "0-1",
+  //
+  // Version 2 is the first flexible version.
+  "validVersions": "0-2",
+  "flexibleVersions": "2+",
   "fields": [
     { "name": "ErrorCode", "type": "int16", "versions": "0+",
       "about": "The top-level error, or zero if there was no error."},
diff --git a/clients/src/main/resources/common/message/CreatePartitionsRequest.json b/clients/src/main/resources/common/message/CreatePartitionsRequest.json
index 360e6f2aa7293..6db7049899876 100644
--- a/clients/src/main/resources/common/message/CreatePartitionsRequest.json
+++ b/clients/src/main/resources/common/message/CreatePartitionsRequest.json
@@ -19,6 +19,7 @@
   "name": "CreatePartitionsRequest",
   // Version 1 is the same as version 0.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "Topics", "type": "[]CreatePartitionsTopic", "versions": "0+",
       "about": "Each topic that we want to create new partitions inside.",  "fields": [
diff --git a/clients/src/main/resources/common/message/CreatePartitionsResponse.json b/clients/src/main/resources/common/message/CreatePartitionsResponse.json
index 2e79b1d3f4d6c..f6527266a3436 100644
--- a/clients/src/main/resources/common/message/CreatePartitionsResponse.json
+++ b/clients/src/main/resources/common/message/CreatePartitionsResponse.json
@@ -19,6 +19,7 @@
   "name": "CreatePartitionsResponse",
   // Starting in version 1, on quota violation, brokers send out responses before throttling.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/CreateTopicsRequest.json b/clients/src/main/resources/common/message/CreateTopicsRequest.json
index e396ed2096436..6299023a9a9c9 100644
--- a/clients/src/main/resources/common/message/CreateTopicsRequest.json
+++ b/clients/src/main/resources/common/message/CreateTopicsRequest.json
@@ -18,9 +18,14 @@
   "type": "request",
   "name": "CreateTopicsRequest",
   // Version 1 adds validateOnly.
+  //
   // Version 4 makes partitions/replicationFactor optional even when assignments are not present (KIP-464)
+  //
   // Version 5 returns topic configs in the response (KIP-525).
-  "validVersions": "0-5",
+  //
+  // Version 6 is the first flexible version.
+  "validVersions": "0-6",
+  "flexibleVersions": "6+",
   "fields": [
     { "name": "Topics", "type": "[]CreatableTopic", "versions": "0+",
       "about": "The topics to create.", "fields": [
diff --git a/clients/src/main/resources/common/message/CreateTopicsResponse.json b/clients/src/main/resources/common/message/CreateTopicsResponse.json
index f646258bff5a7..373258017961c 100644
--- a/clients/src/main/resources/common/message/CreateTopicsResponse.json
+++ b/clients/src/main/resources/common/message/CreateTopicsResponse.json
@@ -18,11 +18,18 @@
   "type": "response",
   "name": "CreateTopicsResponse",
   // Version 1 adds a per-topic error message string.
+  //
   // Version 2 adds the throttle time.
+  //
   // Starting in version 3, on quota violation, brokers send out responses before throttling.
+  //
   // Version 4 makes partitions/replicationFactor optional even when assignments are not present (KIP-464).
-  // Version 5 returns topic configs in the response (KIP-525)
-  "validVersions": "0-5",
+  //
+  // Version 5 returns topic configs in the response (KIP-525).
+  //
+  // Version 6 is the first flexible version.
+  "validVersions": "0-6",
+  "flexibleVersions": "6+",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "2+", "ignorable": true,
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/DeleteAclsRequest.json b/clients/src/main/resources/common/message/DeleteAclsRequest.json
index 3a4aed173349f..c9d6d4aaff21d 100644
--- a/clients/src/main/resources/common/message/DeleteAclsRequest.json
+++ b/clients/src/main/resources/common/message/DeleteAclsRequest.json
@@ -19,6 +19,7 @@
   "name": "DeleteAclsRequest",
   // Version 1 adds the pattern type.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "Filters", "type": "[]DeleteAclsFilter", "versions": "0+",
       "about": "The filters to use when deleting ACLs.", "fields": [
diff --git a/clients/src/main/resources/common/message/DeleteAclsResponse.json b/clients/src/main/resources/common/message/DeleteAclsResponse.json
index d752888866990..303fa2bc94dac 100644
--- a/clients/src/main/resources/common/message/DeleteAclsResponse.json
+++ b/clients/src/main/resources/common/message/DeleteAclsResponse.json
@@ -20,6 +20,7 @@
   // Version 1 adds the resource pattern type.
   // Starting in version 1, on quota violation, brokers send out responses before throttling.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/DeleteGroupsRequest.json b/clients/src/main/resources/common/message/DeleteGroupsRequest.json
index 5671f70f72ced..833ed7a4dc6d7 100644
--- a/clients/src/main/resources/common/message/DeleteGroupsRequest.json
+++ b/clients/src/main/resources/common/message/DeleteGroupsRequest.json
@@ -18,7 +18,10 @@
   "type": "request",
   "name": "DeleteGroupsRequest",
   // Version 1 is the same as version 0.
-  "validVersions": "0-1",
+  //
+  // Version 2 is the first flexible version.
+  "validVersions": "0-2",
+  "flexibleVersions": "2+",
   "fields": [
     { "name": "GroupsNames", "type": "[]string", "versions": "0+", "entityType": "groupId",
       "about": "The group names to delete." }
diff --git a/clients/src/main/resources/common/message/DeleteGroupsResponse.json b/clients/src/main/resources/common/message/DeleteGroupsResponse.json
index 693a1246fb8fd..37e06a55b9913 100644
--- a/clients/src/main/resources/common/message/DeleteGroupsResponse.json
+++ b/clients/src/main/resources/common/message/DeleteGroupsResponse.json
@@ -18,7 +18,10 @@
   "type": "response",
   "name": "DeleteGroupsResponse",
   // Starting in version 1, on quota violation, brokers send out responses before throttling.
-  "validVersions": "0-1",
+  //
+  // Version 2 is the first flexible version.
+  "validVersions": "0-2",
+  "flexibleVersions": "2+",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/DeleteRecordsRequest.json b/clients/src/main/resources/common/message/DeleteRecordsRequest.json
index 62e504f03347f..bfaac07900931 100644
--- a/clients/src/main/resources/common/message/DeleteRecordsRequest.json
+++ b/clients/src/main/resources/common/message/DeleteRecordsRequest.json
@@ -19,6 +19,7 @@
   "name": "DeleteRecordsRequest",
   // Version 1 is the same as version 0.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "Topics", "type": "[]DeleteRecordsTopic", "versions": "0+",
       "about": "Each topic that we want to delete records from.", "fields": [
diff --git a/clients/src/main/resources/common/message/DeleteRecordsResponse.json b/clients/src/main/resources/common/message/DeleteRecordsResponse.json
index 38de018a28536..140d3e75e4dfc 100644
--- a/clients/src/main/resources/common/message/DeleteRecordsResponse.json
+++ b/clients/src/main/resources/common/message/DeleteRecordsResponse.json
@@ -19,6 +19,7 @@
   "name": "DeleteRecordsResponse",
   // Starting in version 1, on quota violation, brokers send out responses before throttling.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/DeleteTopicsRequest.json b/clients/src/main/resources/common/message/DeleteTopicsRequest.json
index 12b202f8d0585..6e86fc26746bb 100644
--- a/clients/src/main/resources/common/message/DeleteTopicsRequest.json
+++ b/clients/src/main/resources/common/message/DeleteTopicsRequest.json
@@ -18,7 +18,10 @@
   "type": "request",
   "name": "DeleteTopicsRequest",
   // Versions 0, 1, 2, and 3 are the same.
-  "validVersions": "0-3",
+  //
+  // Version 4 is the first flexible version.
+  "validVersions": "0-4",
+  "flexibleVersions": "4+",
   "fields": [
     { "name": "TopicNames", "type": "[]string", "versions": "0+", "entityType": "topicName",
       "about": "The names of the topics to delete" },
diff --git a/clients/src/main/resources/common/message/DeleteTopicsResponse.json b/clients/src/main/resources/common/message/DeleteTopicsResponse.json
index 0d30a41c83931..814909a48d247 100644
--- a/clients/src/main/resources/common/message/DeleteTopicsResponse.json
+++ b/clients/src/main/resources/common/message/DeleteTopicsResponse.json
@@ -18,9 +18,14 @@
   "type": "response",
   "name": "DeleteTopicsResponse",
   // Version 1 adds the throttle time.
+  //
   // Starting in version 2, on quota violation, brokers send out responses before throttling.
+  //
   // Starting in version 3, a TOPIC_DELETION_DISABLED error code may be returned.
-  "validVersions": "0-3",
+  //
+  // Version 4 is the first flexible version.
+  "validVersions": "0-4",
+  "flexibleVersions": "4+",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "1+",
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/DescribeAclsRequest.json b/clients/src/main/resources/common/message/DescribeAclsRequest.json
index 9be6bbe24617f..258dddc1f51ef 100644
--- a/clients/src/main/resources/common/message/DescribeAclsRequest.json
+++ b/clients/src/main/resources/common/message/DescribeAclsRequest.json
@@ -19,6 +19,7 @@
   "name": "DescribeAclsRequest",
   // Version 1 adds resource pattern type.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ResourceType", "type": "int8", "versions": "0+",
       "about": "The resource type." },
diff --git a/clients/src/main/resources/common/message/DescribeAclsResponse.json b/clients/src/main/resources/common/message/DescribeAclsResponse.json
index 0fdf0c0335864..9f04de01b8757 100644
--- a/clients/src/main/resources/common/message/DescribeAclsResponse.json
+++ b/clients/src/main/resources/common/message/DescribeAclsResponse.json
@@ -20,6 +20,7 @@
   // Version 1 adds PatternType.
   // Starting in version 1, on quota violation, brokers send out responses before throttling.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/DescribeConfigsRequest.json b/clients/src/main/resources/common/message/DescribeConfigsRequest.json
index 5113d4d8dc4b9..5d4d13b3fdb8b 100644
--- a/clients/src/main/resources/common/message/DescribeConfigsRequest.json
+++ b/clients/src/main/resources/common/message/DescribeConfigsRequest.json
@@ -20,6 +20,7 @@
   // Version 1 adds IncludeSynoyms.
   // Version 2 is the same as version 1.
   "validVersions": "0-2",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "Resources", "type": "[]DescribeConfigsResource", "versions": "0+",
       "about": "The resources whose configurations we want to describe.", "fields": [
diff --git a/clients/src/main/resources/common/message/DescribeConfigsResponse.json b/clients/src/main/resources/common/message/DescribeConfigsResponse.json
index f1dc184e08958..82e44f7856ce8 100644
--- a/clients/src/main/resources/common/message/DescribeConfigsResponse.json
+++ b/clients/src/main/resources/common/message/DescribeConfigsResponse.json
@@ -20,6 +20,7 @@
   // Version 1 adds ConfigSource and the synonyms.
   // Starting in version 2, on quota violation, brokers send out responses before throttling.
   "validVersions": "0-2",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/DescribeDelegationTokenRequest.json b/clients/src/main/resources/common/message/DescribeDelegationTokenRequest.json
index e9ec3baffc5c2..2bc2eba721492 100644
--- a/clients/src/main/resources/common/message/DescribeDelegationTokenRequest.json
+++ b/clients/src/main/resources/common/message/DescribeDelegationTokenRequest.json
@@ -19,6 +19,7 @@
   "name": "DescribeDelegationTokenRequest",
   // Version 1 is the same as version 0.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "Owners", "type": "[]DescribeDelegationTokenOwner", "versions": "0+", "nullableVersions": "0+",
       "about": "Each owner that we want to describe delegation tokens for, or null to describe all tokens.", "fields": [
diff --git a/clients/src/main/resources/common/message/DescribeDelegationTokenResponse.json b/clients/src/main/resources/common/message/DescribeDelegationTokenResponse.json
index 627246d254edf..302ff24a5cc88 100644
--- a/clients/src/main/resources/common/message/DescribeDelegationTokenResponse.json
+++ b/clients/src/main/resources/common/message/DescribeDelegationTokenResponse.json
@@ -19,6 +19,7 @@
   "name": "DescribeDelegationTokenResponse",
   // Starting in version 1, on quota violation, brokers send out responses before throttling.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ErrorCode", "type": "int16", "versions": "0+",
       "about": "The error code, or 0 if there was no error." },
diff --git a/clients/src/main/resources/common/message/DescribeGroupsRequest.json b/clients/src/main/resources/common/message/DescribeGroupsRequest.json
index 87cad1ab9c037..8a5887a5680a8 100644
--- a/clients/src/main/resources/common/message/DescribeGroupsRequest.json
+++ b/clients/src/main/resources/common/message/DescribeGroupsRequest.json
@@ -18,9 +18,14 @@
   "type": "request",
   "name": "DescribeGroupsRequest",
   // Versions 1 and 2 are the same as version 0.
+  //
   // Starting in version 3, authorized operations can be requested.
+  //
   // Starting in version 4, the response will include group.instance.id info for members.
-  "validVersions": "0-4",
+  //
+  // Version 5 is the first flexible version.
+  "validVersions": "0-5",
+  "flexibleVersions": "5+",
   "fields": [
     { "name": "Groups", "type": "[]string", "versions": "0+", "entityType": "groupId",
       "about": "The names of the groups to describe" },
diff --git a/clients/src/main/resources/common/message/DescribeGroupsResponse.json b/clients/src/main/resources/common/message/DescribeGroupsResponse.json
index 9ec56e9c255c6..7e8a4d98b0727 100644
--- a/clients/src/main/resources/common/message/DescribeGroupsResponse.json
+++ b/clients/src/main/resources/common/message/DescribeGroupsResponse.json
@@ -18,10 +18,16 @@
   "type": "response",
   "name": "DescribeGroupsResponse",
   // Version 1 added throttle time.
+  //
   // Starting in version 2, on quota violation, brokers send out responses before throttling.
+  //
   // Starting in version 3, brokers can send authorized operations.
+  //
   // Starting in version 4, the response will include group.instance.id info for members.
-  "validVersions": "0-4",
+  //
+  // Version 5 is the first flexible version.
+  "validVersions": "0-5",
+  "flexibleVersions": "5+",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "1+", "ignorable": true,
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/DescribeLogDirsRequest.json b/clients/src/main/resources/common/message/DescribeLogDirsRequest.json
index f6023526c1a8e..333fa682b1dcb 100644
--- a/clients/src/main/resources/common/message/DescribeLogDirsRequest.json
+++ b/clients/src/main/resources/common/message/DescribeLogDirsRequest.json
@@ -19,6 +19,7 @@
   "name": "DescribeLogDirsRequest",
   // Version 1 is the same as version 0.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "Topics", "type": "[]DescribableLogDirTopic", "versions": "0+", "nullableVersions": "0+",
       "about": "Each topic that we want to describe log directories for, or null for all topics.", "fields": [
diff --git a/clients/src/main/resources/common/message/DescribeLogDirsResponse.json b/clients/src/main/resources/common/message/DescribeLogDirsResponse.json
index cd950a5d25a70..d75b348b298e2 100644
--- a/clients/src/main/resources/common/message/DescribeLogDirsResponse.json
+++ b/clients/src/main/resources/common/message/DescribeLogDirsResponse.json
@@ -19,6 +19,7 @@
   "name": "DescribeLogDirsResponse",
   // Starting in version 1, on quota violation, brokers send out responses before throttling.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/ElectLeadersRequest.json b/clients/src/main/resources/common/message/ElectLeadersRequest.json
index b7ed5dceed8f5..5b86c96b04d60 100644
--- a/clients/src/main/resources/common/message/ElectLeadersRequest.json
+++ b/clients/src/main/resources/common/message/ElectLeadersRequest.json
@@ -17,7 +17,11 @@
   "apiKey": 43,
   "type": "request",
   "name": "ElectLeadersRequest",
-  "validVersions": "0-1",
+  // Version 1 implements multiple leader election types, as described by KIP-460.
+  //
+  // Version 2 is the first flexible version.
+  "validVersions": "0-2",
+  "flexibleVersions": "2+",
   "fields": [
     { "name": "ElectionType", "type": "int8", "versions": "1+",
       "about": "Type of elections to conduct for the partition. A value of '0' elects the preferred replica. A value of '1' elects the first live replica if there are no in-sync replica." },
diff --git a/clients/src/main/resources/common/message/ElectLeadersResponse.json b/clients/src/main/resources/common/message/ElectLeadersResponse.json
index 09d0e1556c04f..15468c78d1f70 100644
--- a/clients/src/main/resources/common/message/ElectLeadersResponse.json
+++ b/clients/src/main/resources/common/message/ElectLeadersResponse.json
@@ -17,7 +17,11 @@
   "apiKey": 43,
   "type": "response",
   "name": "ElectLeadersResponse",
-  "validVersions": "0-1",
+  // Version 1 adds a top-level error code.
+  //
+  // Version 2 is the first flexible version.
+  "validVersions": "0-2",
+  "flexibleVersions": "2+",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/EndTxnRequest.json b/clients/src/main/resources/common/message/EndTxnRequest.json
index bf2db4c6e3538..4105479b69f29 100644
--- a/clients/src/main/resources/common/message/EndTxnRequest.json
+++ b/clients/src/main/resources/common/message/EndTxnRequest.json
@@ -19,6 +19,7 @@
   "name": "EndTxnRequest",
   // Version 1 is the same as version 0.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "TransactionalId", "type": "string", "versions": "0+", "entityType": "transactionalId",
       "about": "The ID of the transaction to end." },
diff --git a/clients/src/main/resources/common/message/EndTxnResponse.json b/clients/src/main/resources/common/message/EndTxnResponse.json
index b0d4010a7e408..247e9b974b963 100644
--- a/clients/src/main/resources/common/message/EndTxnResponse.json
+++ b/clients/src/main/resources/common/message/EndTxnResponse.json
@@ -19,6 +19,7 @@
   "name": "EndTxnResponse",
   // Starting in version 1, on quota violation, brokers send out responses before throttling.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/ExpireDelegationTokenRequest.json b/clients/src/main/resources/common/message/ExpireDelegationTokenRequest.json
index fdc50d0559f43..5192b3f1173cc 100644
--- a/clients/src/main/resources/common/message/ExpireDelegationTokenRequest.json
+++ b/clients/src/main/resources/common/message/ExpireDelegationTokenRequest.json
@@ -19,6 +19,7 @@
   "name": "ExpireDelegationTokenRequest",
   // Version 1 is the same as version 0.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "Hmac", "type": "bytes", "versions": "0+",
       "about": "The HMAC of the delegation token to be expired." },
diff --git a/clients/src/main/resources/common/message/ExpireDelegationTokenResponse.json b/clients/src/main/resources/common/message/ExpireDelegationTokenResponse.json
index 53fd5a0a7ea1c..4182ec0ca06d9 100644
--- a/clients/src/main/resources/common/message/ExpireDelegationTokenResponse.json
+++ b/clients/src/main/resources/common/message/ExpireDelegationTokenResponse.json
@@ -19,6 +19,7 @@
   "name": "ExpireDelegationTokenResponse",
   // Starting in version 1, on quota violation, brokers send out responses before throttling.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ErrorCode", "type": "int16", "versions": "0+",
       "about": "The error code, or 0 if there was no error." },
diff --git a/clients/src/main/resources/common/message/FetchRequest.json b/clients/src/main/resources/common/message/FetchRequest.json
index 5b834b883b088..6e0207831fbbe 100644
--- a/clients/src/main/resources/common/message/FetchRequest.json
+++ b/clients/src/main/resources/common/message/FetchRequest.json
@@ -45,6 +45,7 @@
   // described in KIP-110.
   //
   "validVersions": "0-11",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ReplicaId", "type": "int32", "versions": "0+",
       "about": "The broker ID of the follower, of -1 if this request is from a consumer." },
diff --git a/clients/src/main/resources/common/message/FetchResponse.json b/clients/src/main/resources/common/message/FetchResponse.json
index f6cc5828bca0b..0fe98ade2f5f7 100644
--- a/clients/src/main/resources/common/message/FetchResponse.json
+++ b/clients/src/main/resources/common/message/FetchResponse.json
@@ -39,6 +39,7 @@
   // algorithm, as described in KIP-110.
   //
   "validVersions": "0-11",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "1+", "ignorable": true,
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/FindCoordinatorRequest.json b/clients/src/main/resources/common/message/FindCoordinatorRequest.json
index 8ef3f7420e2f3..6a90887b1aa01 100644
--- a/clients/src/main/resources/common/message/FindCoordinatorRequest.json
+++ b/clients/src/main/resources/common/message/FindCoordinatorRequest.json
@@ -18,8 +18,12 @@
   "type": "request",
   "name": "FindCoordinatorRequest",
   // Version 1 adds KeyType.
+  //
   // Version 2 is the same as version 1.
-  "validVersions": "0-2",
+  //
+  // Version 3 is the first flexible version.
+  "validVersions": "0-3",
+  "flexibleVersions": "3+",
   "fields": [
     { "name": "Key", "type": "string", "versions": "0+",
       "about": "The coordinator key." },
diff --git a/clients/src/main/resources/common/message/FindCoordinatorResponse.json b/clients/src/main/resources/common/message/FindCoordinatorResponse.json
index b415a01132ed1..996e8fb5fba5c 100644
--- a/clients/src/main/resources/common/message/FindCoordinatorResponse.json
+++ b/clients/src/main/resources/common/message/FindCoordinatorResponse.json
@@ -18,8 +18,12 @@
   "type": "response",
   "name": "FindCoordinatorResponse",
   // Version 1 adds throttle time and error messages.
+  //
   // Starting in version 2, on quota violation, brokers send out responses before throttling.
-  "validVersions": "0-2",
+  //
+  // Version 3 is the first flexible version.
+  "validVersions": "0-3",
+  "flexibleVersions": "3+",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "1+", "ignorable": true,
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/HeartbeatRequest.json b/clients/src/main/resources/common/message/HeartbeatRequest.json
index aa7f33768cb47..4d799aa3c14c1 100644
--- a/clients/src/main/resources/common/message/HeartbeatRequest.json
+++ b/clients/src/main/resources/common/message/HeartbeatRequest.json
@@ -18,8 +18,12 @@
   "type": "request",
   "name": "HeartbeatRequest",
   // Version 1 and version 2 are the same as version 0.
+  //
   // Starting from version 3, we add a new field called groupInstanceId to indicate member identity across restarts.
-  "validVersions": "0-3",
+  //
+  // Version 4 is the first flexible version.
+  "validVersions": "0-4",
+  "flexibleVersions": "4+",
   "fields": [
     { "name": "GroupId", "type": "string", "versions": "0+", "entityType": "groupId",
       "about": "The group id." },
diff --git a/clients/src/main/resources/common/message/HeartbeatResponse.json b/clients/src/main/resources/common/message/HeartbeatResponse.json
index 525ba2677e637..280ba1103b437 100644
--- a/clients/src/main/resources/common/message/HeartbeatResponse.json
+++ b/clients/src/main/resources/common/message/HeartbeatResponse.json
@@ -18,9 +18,14 @@
   "type": "response",
   "name": "HeartbeatResponse",
   // Version 1 adds throttle time.
+  //
   // Starting in version 2, on quota violation, brokers send out responses before throttling.
+  //
   // Starting from version 3, heartbeatRequest supports a new field called groupInstanceId to indicate member identity across restarts.
-  "validVersions": "0-3",
+  //
+  // Version 4 is the first flexible version.
+  "validVersions": "0-4",
+  "flexibleVersions": "4+",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "1+", "ignorable": true,
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/IncrementalAlterConfigsRequest.json b/clients/src/main/resources/common/message/IncrementalAlterConfigsRequest.json
index d808c04599ed3..b1fb1e9481ac2 100644
--- a/clients/src/main/resources/common/message/IncrementalAlterConfigsRequest.json
+++ b/clients/src/main/resources/common/message/IncrementalAlterConfigsRequest.json
@@ -17,7 +17,9 @@
   "apiKey": 44,
   "type": "request",
   "name": "IncrementalAlterConfigsRequest",
-  "validVersions": "0",
+  // Version 1 is the first flexible version.
+  "validVersions": "0-1",
+  "flexibleVersions": "1+",
   "fields": [
     { "name": "Resources", "type": "[]AlterConfigsResource", "versions": "0+",
       "about": "The incremental updates for each resource.", "fields": [
diff --git a/clients/src/main/resources/common/message/IncrementalAlterConfigsResponse.json b/clients/src/main/resources/common/message/IncrementalAlterConfigsResponse.json
index beede24d12f69..d4dad294f5b7c 100644
--- a/clients/src/main/resources/common/message/IncrementalAlterConfigsResponse.json
+++ b/clients/src/main/resources/common/message/IncrementalAlterConfigsResponse.json
@@ -17,7 +17,9 @@
   "apiKey": 44,
   "type": "response",
   "name": "IncrementalAlterConfigsResponse",
-  "validVersions": "0",
+  // Version 1 is the first flexible version.
+  "validVersions": "0-1",
+  "flexibleVersions": "1+",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
       "about": "Duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/InitProducerIdRequest.json b/clients/src/main/resources/common/message/InitProducerIdRequest.json
index f6c5b35ef1822..6c0cbb420ca85 100644
--- a/clients/src/main/resources/common/message/InitProducerIdRequest.json
+++ b/clients/src/main/resources/common/message/InitProducerIdRequest.json
@@ -18,7 +18,10 @@
   "type": "request",
   "name": "InitProducerIdRequest",
   // Version 1 is the same as version 0.
-  "validVersions": "0-1",
+  //
+  // Version 2 is the first flexible version.
+  "validVersions": "0-2",
+  "flexibleVersions": "2+",
   "fields": [
     { "name": "TransactionalId", "type": "string", "versions": "0+", "nullableVersions": "0+", "entityType": "transactionalId",
       "about": "The transactional id, or null if the producer is not transactional." },
diff --git a/clients/src/main/resources/common/message/InitProducerIdResponse.json b/clients/src/main/resources/common/message/InitProducerIdResponse.json
index ea8812a50e212..0e8ad2e007e01 100644
--- a/clients/src/main/resources/common/message/InitProducerIdResponse.json
+++ b/clients/src/main/resources/common/message/InitProducerIdResponse.json
@@ -18,7 +18,10 @@
   "type": "response",
   "name": "InitProducerIdResponse",
   // Starting in version 1, on quota violation, brokers send out responses before throttling.
-  "validVersions": "0-1",
+  //
+  // Version 2 is the first flexible version.
+  "validVersions": "0-2",
+  "flexibleVersions": "2+",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", "ignorable": true,
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/JoinGroupRequest.json b/clients/src/main/resources/common/message/JoinGroupRequest.json
index 8e58b8764bd2c..707436f6e9ad4 100644
--- a/clients/src/main/resources/common/message/JoinGroupRequest.json
+++ b/clients/src/main/resources/common/message/JoinGroupRequest.json
@@ -18,11 +18,17 @@
   "type": "request",
   "name": "JoinGroupRequest",
   // Version 1 adds RebalanceTimeoutMs.
+  //
   // Version 2 and 3 are the same as version 1.
+  //
   // Starting from version 4, the client needs to issue a second request to join group
+  //
   // Starting from version 5, we add a new field called groupInstanceId to indicate member identity across restarts.
   // with assigned id.
-  "validVersions": "0-5",
+  //
+  // Version 6 is the first flexible version.
+  "validVersions": "0-6",
+  "flexibleVersions": "6+",
   "fields": [
     { "name": "GroupId", "type": "string", "versions": "0+", "entityType": "groupId",
       "about": "The group identifier." },
diff --git a/clients/src/main/resources/common/message/JoinGroupResponse.json b/clients/src/main/resources/common/message/JoinGroupResponse.json
index 14e6b1a9e4d16..2ab1e41cd8efc 100644
--- a/clients/src/main/resources/common/message/JoinGroupResponse.json
+++ b/clients/src/main/resources/common/message/JoinGroupResponse.json
@@ -18,12 +18,19 @@
   "type": "response",
   "name": "JoinGroupResponse",
   // Version 1 is the same as version 0.
+  //
   // Version 2 adds throttle time.
+  //
   // Starting in version 3, on quota violation, brokers send out responses before throttling.
+  //
   // Starting in version 4, the client needs to issue a second request to join group
   // with assigned id.
+  //
   // Version 5 is bumped to apply group.instance.id to identify member across restarts.
-  "validVersions": "0-5",
+  //
+  // Version 6 is the first flexible version.
+  "validVersions": "0-6",
+  "flexibleVersions": "6+",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "2+", "ignorable": true,
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/LeaderAndIsrRequest.json b/clients/src/main/resources/common/message/LeaderAndIsrRequest.json
index 12f159a157401..8af1df5f55c84 100644
--- a/clients/src/main/resources/common/message/LeaderAndIsrRequest.json
+++ b/clients/src/main/resources/common/message/LeaderAndIsrRequest.json
@@ -23,6 +23,7 @@
   //
   // Version 3 adds AddingReplicas and RemovingReplicas
   "validVersions": "0-3",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ControllerId", "type": "int32", "versions": "0+", "entityType": "brokerId",
       "about": "The current controller ID." },
diff --git a/clients/src/main/resources/common/message/LeaderAndIsrResponse.json b/clients/src/main/resources/common/message/LeaderAndIsrResponse.json
index f30055b97aac2..3b67c4cc9e073 100644
--- a/clients/src/main/resources/common/message/LeaderAndIsrResponse.json
+++ b/clients/src/main/resources/common/message/LeaderAndIsrResponse.json
@@ -23,6 +23,7 @@
   //
   // Version 3 is the same as version 2.
   "validVersions": "0-3",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ErrorCode", "type": "int16", "versions": "0+",
       "about": "The error code, or 0 if there was no error." },
diff --git a/clients/src/main/resources/common/message/LeaveGroupRequest.json b/clients/src/main/resources/common/message/LeaveGroupRequest.json
index 9f98f06501bca..acc7938c387b1 100644
--- a/clients/src/main/resources/common/message/LeaveGroupRequest.json
+++ b/clients/src/main/resources/common/message/LeaveGroupRequest.json
@@ -18,8 +18,12 @@
   "type": "request",
   "name": "LeaveGroupRequest",
   // Version 1 and 2 are the same as version 0.
+  //
   // Version 3 defines batch processing scheme with group.instance.id + member.id for identity
-  "validVersions": "0-3",
+  //
+  // Version 4 is the first flexible version.
+  "validVersions": "0-4",
+  "flexibleVersions": "4+",
   "fields": [
     { "name": "GroupId", "type": "string", "versions": "0+", "entityType": "groupId",
       "about": "The ID of the group to leave." },
diff --git a/clients/src/main/resources/common/message/LeaveGroupResponse.json b/clients/src/main/resources/common/message/LeaveGroupResponse.json
index 495d5c39ba181..0ddb4c6eb5323 100644
--- a/clients/src/main/resources/common/message/LeaveGroupResponse.json
+++ b/clients/src/main/resources/common/message/LeaveGroupResponse.json
@@ -18,9 +18,14 @@
   "type": "response",
   "name": "LeaveGroupResponse",
   // Version 1 adds the throttle time.
+  //
   // Starting in version 2, on quota violation, brokers send out responses before throttling.
+  //
   // Starting in version 3, we will make leave group request into batch mode and add group.instance.id.
-  "validVersions": "0-3",
+  //
+  // Version 4 is the first flexible version.
+  "validVersions": "0-4",
+  "flexibleVersions": "4+",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "1+", "ignorable": true,
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/ListGroupsRequest.json b/clients/src/main/resources/common/message/ListGroupsRequest.json
index b0c5aa0e3e81d..f0130e264902b 100644
--- a/clients/src/main/resources/common/message/ListGroupsRequest.json
+++ b/clients/src/main/resources/common/message/ListGroupsRequest.json
@@ -18,7 +18,10 @@
   "type": "request",
   "name": "ListGroupsRequest",
   // Version 1 and 2 are the same as version 0.
-  "validVersions": "0-2",
+  //
+  // Version 3 is the first flexible version.
+  "validVersions": "0-3",
+  "flexibleVersions": "3+",
   "fields": [
   ]
 }
diff --git a/clients/src/main/resources/common/message/ListGroupsResponse.json b/clients/src/main/resources/common/message/ListGroupsResponse.json
index 502ed349a4ab7..aa8bba6ebc738 100644
--- a/clients/src/main/resources/common/message/ListGroupsResponse.json
+++ b/clients/src/main/resources/common/message/ListGroupsResponse.json
@@ -18,8 +18,12 @@
   "type": "response",
   "name": "ListGroupsResponse",
   // Version 1 adds the throttle time.
+  //
   // Starting in version 2, on quota violation, brokers send out responses before throttling.
-  "validVersions": "0-2",
+  //
+  // Version 3 is the first flexible version.
+  "validVersions": "0-3",
+  "flexibleVersions": "3+",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "1+", "ignorable": true,
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/ListOffsetRequest.json b/clients/src/main/resources/common/message/ListOffsetRequest.json
index 194f6da5b2056..2e12b8b2b7d60 100644
--- a/clients/src/main/resources/common/message/ListOffsetRequest.json
+++ b/clients/src/main/resources/common/message/ListOffsetRequest.json
@@ -19,11 +19,16 @@
   "name": "ListOffsetRequest",
   // Version 1 removes MaxNumOffsets.  From this version forward, only a single
   // offset can be returned.
+  //
   // Version 2 adds the isolation level, which is used for transactional reads.
+  //
   // Version 3 is the same as version 2.
+  //
   // Version 4 adds the current leader epoch, which is used for fencing.
+  //
   // Version 5 is the same as version 5.
   "validVersions": "0-5",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ReplicaId", "type": "int32", "versions": "0+", "entityType": "brokerId",
       "about": "The broker ID of the requestor, or -1 if this request is being made by a normal consumer." },
diff --git a/clients/src/main/resources/common/message/ListOffsetResponse.json b/clients/src/main/resources/common/message/ListOffsetResponse.json
index eb691e397c764..4037e1af80bfb 100644
--- a/clients/src/main/resources/common/message/ListOffsetResponse.json
+++ b/clients/src/main/resources/common/message/ListOffsetResponse.json
@@ -19,11 +19,16 @@
   "name": "ListOffsetResponse",
   // Version 1 removes the offsets array in favor of returning a single offset.
   // Version 1 also adds the timestamp associated with the returned offset.
+  //
   // Version 2 adds the throttle time.
+  //
   // Starting in version 3, on quota violation, brokers send out responses before throttling.
+  //
   // Version 4 adds the leader epoch, which is used for fencing.
+  //
   // Version 5 adds a new error code, OFFSET_NOT_AVAILABLE.
   "validVersions": "0-5",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "2+", "ignorable": true,
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json b/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json
index ac871b2bc028f..7322f25bc456e 100644
--- a/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json
+++ b/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json
@@ -18,6 +18,7 @@
   "type": "request",
   "name": "ListPartitionReassignmentsRequest",
   "validVersions": "0",
+  "flexibleVersions": "0+",
   "fields": [
     { "name": "TimeoutMs", "type": "int32", "versions": "0+", "default": "60000",
       "about": "The time in ms to wait for the request to complete." },
diff --git a/clients/src/main/resources/common/message/ListPartitionReassignmentsResponse.json b/clients/src/main/resources/common/message/ListPartitionReassignmentsResponse.json
index b79e0523b32ee..36cf70cab80c9 100644
--- a/clients/src/main/resources/common/message/ListPartitionReassignmentsResponse.json
+++ b/clients/src/main/resources/common/message/ListPartitionReassignmentsResponse.json
@@ -18,6 +18,7 @@
   "type": "response",
   "name": "ListPartitionReassignmentsResponse",
   "validVersions": "0",
+  "flexibleVersions": "0+",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/MetadataRequest.json b/clients/src/main/resources/common/message/MetadataRequest.json
index cbb3489bf7dbd..6129d427b7eef 100644
--- a/clients/src/main/resources/common/message/MetadataRequest.json
+++ b/clients/src/main/resources/common/message/MetadataRequest.json
@@ -17,7 +17,8 @@
   "apiKey": 3,
   "type": "request",
   "name": "MetadataRequest",
-  "validVersions": "0-8",
+  "validVersions": "0-9",
+  "flexibleVersions": "9+",
   "fields": [
     // In version 0, an empty array indicates "request metadata for all topics."  In version 1 and
     // higher, an empty array indicates "request metadata for no topics," and a null array is used to
@@ -26,7 +27,10 @@
     // Version 2 and 3 are the same as version 1.
     //
     // Version 4 adds AllowAutoTopicCreation.
+    //
     // Starting in version 8, authorized operations can be requested for cluster and topic resource.
+    //
+    // Version 9 is the first flexible version.
     { "name": "Topics", "type": "[]MetadataRequestTopic", "versions": "0+", "nullableVersions": "1+",
       "about": "The topics to fetch metadata for.", "fields": [
       { "name": "Name", "type": "string", "versions": "0+", "entityType": "topicName",
diff --git a/clients/src/main/resources/common/message/MetadataResponse.json b/clients/src/main/resources/common/message/MetadataResponse.json
index bb09cdb4973a3..ce232b1233110 100644
--- a/clients/src/main/resources/common/message/MetadataResponse.json
+++ b/clients/src/main/resources/common/message/MetadataResponse.json
@@ -32,8 +32,12 @@
   // Starting in version 6, on quota violation, brokers send out responses before throttling.
   //
   // Version 7 adds the leader epoch to the partition metadata.
+  //
   // Starting in version 8, brokers can send authorized operations for topic and cluster.
-  "validVersions": "0-8",
+  //
+  // Version 9 is the first flexible version.
+  "validVersions": "0-9",
+  "flexibleVersions": "9+",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "3+",
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/OffsetCommitRequest.json b/clients/src/main/resources/common/message/OffsetCommitRequest.json
index adda0795c0ca8..096b61917ae0f 100644
--- a/clients/src/main/resources/common/message/OffsetCommitRequest.json
+++ b/clients/src/main/resources/common/message/OffsetCommitRequest.json
@@ -26,8 +26,12 @@
   // Version 5 removes the retention time, which is now controlled only by a broker configuration.
   //
   // Version 6 adds the leader epoch for fencing.
+  //
   // version 7 adds a new field called groupInstanceId to indicate member identity across restarts.
-  "validVersions": "0-7",
+  //
+  // Version 8 is the first flexible version.
+  "validVersions": "0-8",
+  "flexibleVersions": "8+",
   "fields": [
     { "name": "GroupId", "type": "string", "versions": "0+", "entityType": "groupId",
       "about": "The unique group identifier." },
diff --git a/clients/src/main/resources/common/message/OffsetCommitResponse.json b/clients/src/main/resources/common/message/OffsetCommitResponse.json
index 8698be10b3ec7..3d547794cfb42 100644
--- a/clients/src/main/resources/common/message/OffsetCommitResponse.json
+++ b/clients/src/main/resources/common/message/OffsetCommitResponse.json
@@ -24,8 +24,12 @@
   // Starting in version 4, on quota violation, brokers send out responses before throttling.
   //
   // Versions 5 and 6 are the same as version 4.
+  //
   // Version 7 offsetCommitRequest supports a new field called groupInstanceId to indicate member identity across restarts.
-  "validVersions": "0-7",
+  //
+  // Version 8 is the first flexible version.
+  "validVersions": "0-8",
+  "flexibleVersions": "8+",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "3+", "ignorable": true,
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/OffsetFetchRequest.json b/clients/src/main/resources/common/message/OffsetFetchRequest.json
index 0449fda7c0382..57306880df415 100644
--- a/clients/src/main/resources/common/message/OffsetFetchRequest.json
+++ b/clients/src/main/resources/common/message/OffsetFetchRequest.json
@@ -26,7 +26,10 @@
   // for group or coordinator level errors.
   //
   // Version 3, 4, and 5 are the same as version 2.
-  "validVersions": "0-5",
+  //
+  // Version 6 is the first flexible version.
+  "validVersions": "0-6",
+  "flexibleVersions": "6+",
   "fields": [
     { "name": "GroupId", "type": "string", "versions": "0+", "entityType": "groupId",
       "about": "The group to fetch offsets for." },
diff --git a/clients/src/main/resources/common/message/OffsetFetchResponse.json b/clients/src/main/resources/common/message/OffsetFetchResponse.json
index 69756cc523fca..e0d5e358a9173 100644
--- a/clients/src/main/resources/common/message/OffsetFetchResponse.json
+++ b/clients/src/main/resources/common/message/OffsetFetchResponse.json
@@ -26,7 +26,10 @@
   // Starting in version 4, on quota violation, brokers send out responses before throttling.
   //
   // Version 5 adds the leader epoch to the committed offset.
-  "validVersions": "0-5",
+  //
+  // Version 6 is the first flexible version.
+  "validVersions": "0-6",
+  "flexibleVersions": "6+",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "3+", "ignorable": true,
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/OffsetForLeaderEpochRequest.json b/clients/src/main/resources/common/message/OffsetForLeaderEpochRequest.json
index 53bef12d7ebf9..0d1b063b90cba 100644
--- a/clients/src/main/resources/common/message/OffsetForLeaderEpochRequest.json
+++ b/clients/src/main/resources/common/message/OffsetForLeaderEpochRequest.json
@@ -18,11 +18,14 @@
   "type": "request",
   "name": "OffsetForLeaderEpochRequest",
   // Version 1 is the same as version 0.
+  //
   // Version 2 adds the current leader epoch to support fencing.
+  //
   // Version 3 adds ReplicaId (the default is -2 which conventionally represents a
   //    "debug" consumer which is allowed to see offsets beyond the high watermark).
   //    Followers will use this replicaId when using an older version of the protocol.
   "validVersions": "0-3",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ReplicaId", "type": "int32", "versions": "3+", "default": -2, "ignorable": true,
       "about": "The broker ID of the follower, of -1 if this request is from a consumer." },
diff --git a/clients/src/main/resources/common/message/OffsetForLeaderEpochResponse.json b/clients/src/main/resources/common/message/OffsetForLeaderEpochResponse.json
index 169460624aef3..867b763a4214b 100644
--- a/clients/src/main/resources/common/message/OffsetForLeaderEpochResponse.json
+++ b/clients/src/main/resources/common/message/OffsetForLeaderEpochResponse.json
@@ -20,6 +20,7 @@
   // Version 1 added the leader epoch to the response.
   // Version 2 added the throttle time.
   "validVersions": "0-3",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "2+", "ignorable": true,
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/ProduceRequest.json b/clients/src/main/resources/common/message/ProduceRequest.json
index 5a1556d2652d6..4fcedf10da909 100644
--- a/clients/src/main/resources/common/message/ProduceRequest.json
+++ b/clients/src/main/resources/common/message/ProduceRequest.json
@@ -29,8 +29,9 @@
   //
   // Starting in version 7, records can be produced using ZStandard compression.  See KIP-110.
   //
-  // Starting in Version 8, response has ErrorRecords and ErrorMEssage. See KIP-467.
+  // Version 8 is the same as version 7 (but see KIP-467 for the response changes).
   "validVersions": "0-8",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "TransactionalId", "type": "string", "versions": "3+", "nullableVersions": "0+", "entityType": "transactionalId",
       "about": "The transactional ID, or null if the producer is not transactional." },
diff --git a/clients/src/main/resources/common/message/ProduceResponse.json b/clients/src/main/resources/common/message/ProduceResponse.json
index 690880d29d17b..c21b0aa93accd 100644
--- a/clients/src/main/resources/common/message/ProduceResponse.json
+++ b/clients/src/main/resources/common/message/ProduceResponse.json
@@ -29,8 +29,9 @@
   // OutOfOrderSequenceExceptions on the client.
   //
   // Version 8 added ErrorRecords and ErrorMessage to include information about
-  // records that cause the whole batch to be dropped
+  // records that cause the whole batch to be dropped.  See KIP-467 for details.
   "validVersions": "0-8",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "Responses", "type": "[]TopicProduceResponse", "versions": "0+",
       "about": "Each produce response", "fields": [
diff --git a/clients/src/main/resources/common/message/RenewDelegationTokenRequest.json b/clients/src/main/resources/common/message/RenewDelegationTokenRequest.json
index ba1db7ede5eb8..83fde2ee837e6 100644
--- a/clients/src/main/resources/common/message/RenewDelegationTokenRequest.json
+++ b/clients/src/main/resources/common/message/RenewDelegationTokenRequest.json
@@ -19,6 +19,7 @@
   "name": "RenewDelegationTokenRequest",
   // Version 1 is the same as version 0.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "Hmac", "type": "bytes", "versions": "0+",
       "about": "The HMAC of the delegation token to be renewed." },
diff --git a/clients/src/main/resources/common/message/RenewDelegationTokenResponse.json b/clients/src/main/resources/common/message/RenewDelegationTokenResponse.json
index aed734a879645..5c70521164563 100644
--- a/clients/src/main/resources/common/message/RenewDelegationTokenResponse.json
+++ b/clients/src/main/resources/common/message/RenewDelegationTokenResponse.json
@@ -19,6 +19,7 @@
   "name": "RenewDelegationTokenResponse",
   // Starting in version 1, on quota violation, brokers send out responses before throttling.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ErrorCode", "type": "int16", "versions": "0+",
       "about": "The error code, or 0 if there was no error." },
diff --git a/clients/src/main/resources/common/message/RequestHeader.json b/clients/src/main/resources/common/message/RequestHeader.json
index a00658cc866de..560b0190d5b9a 100644
--- a/clients/src/main/resources/common/message/RequestHeader.json
+++ b/clients/src/main/resources/common/message/RequestHeader.json
@@ -19,7 +19,10 @@
   // Version 0 of the RequestHeader is only used by v0 of ControlledShutdownRequest.
   //
   // Version 1 is the first version with ClientId.
-  "validVersions": "0-1",
+  //
+  // Version 2 is the first flexible version.
+  "validVersions": "0-2",
+  "flexibleVersions": "2+",
   "fields": [
     { "name": "RequestApiKey", "type": "int16", "versions": "0+",
       "about": "The API key of this request." },
diff --git a/clients/src/main/resources/common/message/ResponseHeader.json b/clients/src/main/resources/common/message/ResponseHeader.json
index bc47b3f05f07c..773673601a886 100644
--- a/clients/src/main/resources/common/message/ResponseHeader.json
+++ b/clients/src/main/resources/common/message/ResponseHeader.json
@@ -16,8 +16,9 @@
 {
   "type": "header",
   "name": "ResponseHeader",
-  // Version 0 and version 1 are the same.
+  // Version 1 is the first flexible version.
   "validVersions": "0-1",
+  "flexibleVersions": "1+",
   "fields": [
     { "name": "CorrelationId", "type": "int32", "versions": "0+",
       "about": "The correlation ID of this response." }
diff --git a/clients/src/main/resources/common/message/SaslAuthenticateRequest.json b/clients/src/main/resources/common/message/SaslAuthenticateRequest.json
index 96f6f3aa4efb4..27e97700304f4 100644
--- a/clients/src/main/resources/common/message/SaslAuthenticateRequest.json
+++ b/clients/src/main/resources/common/message/SaslAuthenticateRequest.json
@@ -19,6 +19,7 @@
   "name": "SaslAuthenticateRequest",
   // Version 1 is the same as version 0.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "AuthBytes", "type": "bytes", "versions": "0+",
       "about": "The SASL authentication bytes from the client, as defined by the SASL mechanism." }
diff --git a/clients/src/main/resources/common/message/SaslAuthenticateResponse.json b/clients/src/main/resources/common/message/SaslAuthenticateResponse.json
index f6644f6d3459d..66dfe74651fec 100644
--- a/clients/src/main/resources/common/message/SaslAuthenticateResponse.json
+++ b/clients/src/main/resources/common/message/SaslAuthenticateResponse.json
@@ -19,6 +19,7 @@
   "name": "SaslAuthenticateResponse",
   // Version 1 adds the session lifetime.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ErrorCode", "type": "int16", "versions": "0+",
       "about": "The error code, or 0 if there was no error." },
diff --git a/clients/src/main/resources/common/message/SaslHandshakeRequest.json b/clients/src/main/resources/common/message/SaslHandshakeRequest.json
index 7ad0a4716ef74..162448238dfe8 100644
--- a/clients/src/main/resources/common/message/SaslHandshakeRequest.json
+++ b/clients/src/main/resources/common/message/SaslHandshakeRequest.json
@@ -19,6 +19,7 @@
   "name": "SaslHandshakeRequest",
   // Version 1 supports SASL_AUTHENTICATE.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "Mechanism", "type": "string", "versions": "0+",
       "about": "The SASL mechanism chosen by the client." }
diff --git a/clients/src/main/resources/common/message/SaslHandshakeResponse.json b/clients/src/main/resources/common/message/SaslHandshakeResponse.json
index 821f3290833cf..2d26945f433f5 100644
--- a/clients/src/main/resources/common/message/SaslHandshakeResponse.json
+++ b/clients/src/main/resources/common/message/SaslHandshakeResponse.json
@@ -19,6 +19,7 @@
   "name": "SaslHandshakeResponse",
   // Version 1 is the same as version 0.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ErrorCode", "type": "int16", "versions": "0+",
       "about": "The error code, or 0 if there was no error." },
diff --git a/clients/src/main/resources/common/message/StopReplicaRequest.json b/clients/src/main/resources/common/message/StopReplicaRequest.json
index 248faa58a1f86..695c0c186d1de 100644
--- a/clients/src/main/resources/common/message/StopReplicaRequest.json
+++ b/clients/src/main/resources/common/message/StopReplicaRequest.json
@@ -20,6 +20,7 @@
   // Version 1 adds the broker epoch and reorganizes the partitions to be stored
   // per topic.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ControllerId", "type": "int32", "versions": "0+", "entityType": "brokerId",
       "about": "The controller id." },
diff --git a/clients/src/main/resources/common/message/StopReplicaResponse.json b/clients/src/main/resources/common/message/StopReplicaResponse.json
index 759bffecf07c3..d55a7b5e5a881 100644
--- a/clients/src/main/resources/common/message/StopReplicaResponse.json
+++ b/clients/src/main/resources/common/message/StopReplicaResponse.json
@@ -19,6 +19,7 @@
   "name": "StopReplicaResponse",
   // Version 1 is the same as version 0.
   "validVersions": "0-1",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ErrorCode", "type": "int16", "versions": "0+",
       "about": "The top-level error code, or 0 if there was no top-level error." },
diff --git a/clients/src/main/resources/common/message/SyncGroupRequest.json b/clients/src/main/resources/common/message/SyncGroupRequest.json
index 3ace70d22bc55..0e65d87f8db4a 100644
--- a/clients/src/main/resources/common/message/SyncGroupRequest.json
+++ b/clients/src/main/resources/common/message/SyncGroupRequest.json
@@ -18,8 +18,12 @@
   "type": "request",
   "name": "SyncGroupRequest",
   // Versions 1 and 2 are the same as version 0.
+  //
   // Starting from version 3, we add a new field called groupInstanceId to indicate member identity across restarts.
-  "validVersions": "0-3",
+  //
+  // Version 4 is the first flexible version.
+  "validVersions": "0-4",
+  "flexibleVersions": "4+",
   "fields": [
     { "name": "GroupId", "type": "string", "versions": "0+", "entityType": "groupId",
       "about": "The unique group identifier." },
diff --git a/clients/src/main/resources/common/message/SyncGroupResponse.json b/clients/src/main/resources/common/message/SyncGroupResponse.json
index 06e6bea29686d..ce60e3f562fa9 100644
--- a/clients/src/main/resources/common/message/SyncGroupResponse.json
+++ b/clients/src/main/resources/common/message/SyncGroupResponse.json
@@ -18,9 +18,14 @@
   "type": "response",
   "name": "SyncGroupResponse",
   // Version 1 adds throttle time.
+  //
   // Starting in version 2, on quota violation, brokers send out responses before throttling.
+  //
   // Starting from version 3, syncGroupRequest supports a new field called groupInstanceId to indicate member identity across restarts.
-  "validVersions": "0-3",
+  //
+  // Version 4 is the first flexible version.
+  "validVersions": "0-4",
+  "flexibleVersions": "4+",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "1+", "ignorable": true,
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/TxnOffsetCommitRequest.json b/clients/src/main/resources/common/message/TxnOffsetCommitRequest.json
index a7b7d05257d50..6ad3ceb113d84 100644
--- a/clients/src/main/resources/common/message/TxnOffsetCommitRequest.json
+++ b/clients/src/main/resources/common/message/TxnOffsetCommitRequest.json
@@ -21,6 +21,7 @@
   //
   // Version 2 adds the committed leader epoch.
   "validVersions": "0-2",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "TransactionalId", "type": "string", "versions": "0+",
       "about": "The ID of the transaction." },
diff --git a/clients/src/main/resources/common/message/TxnOffsetCommitResponse.json b/clients/src/main/resources/common/message/TxnOffsetCommitResponse.json
index 4a13800aeb8f6..7cfe0c3e75301 100644
--- a/clients/src/main/resources/common/message/TxnOffsetCommitResponse.json
+++ b/clients/src/main/resources/common/message/TxnOffsetCommitResponse.json
@@ -20,6 +20,7 @@
   // Starting in version 1, on quota violation, brokers send out responses before throttling.
   // Version 2 is the same as version 1.
   "validVersions": "0-2",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
diff --git a/clients/src/main/resources/common/message/UpdateMetadataRequest.json b/clients/src/main/resources/common/message/UpdateMetadataRequest.json
index b1ac987b46c64..203ee44cc6617 100644
--- a/clients/src/main/resources/common/message/UpdateMetadataRequest.json
+++ b/clients/src/main/resources/common/message/UpdateMetadataRequest.json
@@ -27,6 +27,7 @@
   //
   // Version 5 adds the broker epoch field and normalizes partitions by topic.
   "validVersions": "0-5",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "ControllerId", "type": "int32", "versions": "0+", "entityType": "brokerId",
       "about": "The controller id." },
diff --git a/clients/src/main/resources/common/message/UpdateMetadataResponse.json b/clients/src/main/resources/common/message/UpdateMetadataResponse.json
index 5069a63d77345..5990bebf39d7a 100644
--- a/clients/src/main/resources/common/message/UpdateMetadataResponse.json
+++ b/clients/src/main/resources/common/message/UpdateMetadataResponse.json
@@ -19,6 +19,7 @@
   "name": "UpdateMetadataResponse",
   // Versions 1, 2, 3, 4, and 5 are the same as version 0
   "validVersions": "0-5",
+  "flexibleVersions": "none",
   "fields": [
       { "name": "ErrorCode", "type": "int16", "versions": "0+",
         "about": "The error code, or 0 if there was no error." }
diff --git a/clients/src/main/resources/common/message/WriteTxnMarkersRequest.json b/clients/src/main/resources/common/message/WriteTxnMarkersRequest.json
index 63a73200af541..845a4ca3735f1 100644
--- a/clients/src/main/resources/common/message/WriteTxnMarkersRequest.json
+++ b/clients/src/main/resources/common/message/WriteTxnMarkersRequest.json
@@ -18,6 +18,7 @@
   "type": "request",
   "name": "WriteTxnMarkersRequest",
   "validVersions": "0",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "Markers", "type": "[]WritableTxnMarker", "versions": "0+",
       "about": "The transaction markers to be written.", "fields": [
diff --git a/clients/src/main/resources/common/message/WriteTxnMarkersResponse.json b/clients/src/main/resources/common/message/WriteTxnMarkersResponse.json
index a1abee8ffc9ae..67bb791541b3b 100644
--- a/clients/src/main/resources/common/message/WriteTxnMarkersResponse.json
+++ b/clients/src/main/resources/common/message/WriteTxnMarkersResponse.json
@@ -18,6 +18,7 @@
   "type": "response",
   "name": "WriteTxnMarkersResponse",
   "validVersions": "0",
+  "flexibleVersions": "none",
   "fields": [
     { "name": "Markers", "type": "[]WritableTxnMarkerResult", "versions": "0+",
       "about": "The results for writing makers.", "fields": [
diff --git a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
index b5ecbc74c52ac..e198f6c4a6be0 100644
--- a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
@@ -175,7 +175,7 @@ private void checkSimpleRequestResponse(NetworkClient networkClient) {
         assertEquals(1, networkClient.inFlightRequestCount());
         ResponseHeader respHeader =
             new ResponseHeader(request.correlationId(),
-                request.apiKey().headerVersion(PRODUCE.latestVersion()));
+                request.apiKey().responseHeaderVersion(PRODUCE.latestVersion()));
         Struct resp = new Struct(PRODUCE.responseSchema(PRODUCE.latestVersion()));
         resp.set("responses", new Object[0]);
         Struct responseHeaderStruct = respHeader.toStruct();
@@ -459,7 +459,7 @@ public void testConnectionThrottling() {
         client.poll(1, time.milliseconds());
         ResponseHeader respHeader =
             new ResponseHeader(request.correlationId(),
-                request.apiKey().headerVersion(PRODUCE.latestVersion()));
+                request.apiKey().responseHeaderVersion(PRODUCE.latestVersion()));
         Struct resp = new Struct(PRODUCE.responseSchema(PRODUCE.latestVersion()));
         resp.set("responses", new Object[0]);
         resp.set(CommonFields.THROTTLE_TIME_MS, 100);
@@ -557,7 +557,7 @@ private void sendThrottledProduceResponse(int correlationId, int throttleMs) {
         resp.set("responses", new Object[0]);
         resp.set(CommonFields.THROTTLE_TIME_MS, throttleMs);
         sendResponse(new ResponseHeader(correlationId,
-            PRODUCE.headerVersion(PRODUCE.latestVersion())),
+            PRODUCE.responseHeaderVersion(PRODUCE.latestVersion())),
             resp);
     }
 
diff --git a/clients/src/test/java/org/apache/kafka/common/message/ApiMessageTypeTest.java b/clients/src/test/java/org/apache/kafka/common/message/ApiMessageTypeTest.java
index 49a89699cd733..7acde182ddbea 100644
--- a/clients/src/test/java/org/apache/kafka/common/message/ApiMessageTypeTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/message/ApiMessageTypeTest.java
@@ -73,4 +73,25 @@ public void testUniqueness() {
         assertEquals(ApiMessageType.values().length, requestNames.size());
         assertEquals(ApiMessageType.values().length, responseNames.size());
     }
+
+    @Test
+    public void testHeaderVersion() {
+        assertEquals((short) 1, ApiMessageType.PRODUCE.requestHeaderVersion((short) 0));
+        assertEquals((short) 0, ApiMessageType.PRODUCE.responseHeaderVersion((short) 0));
+
+        assertEquals((short) 1, ApiMessageType.PRODUCE.requestHeaderVersion((short) 1));
+        assertEquals((short) 0, ApiMessageType.PRODUCE.responseHeaderVersion((short) 1));
+
+        assertEquals((short) 0, ApiMessageType.CONTROLLED_SHUTDOWN.requestHeaderVersion((short) 0));
+        assertEquals((short) 0, ApiMessageType.CONTROLLED_SHUTDOWN.responseHeaderVersion((short) 0));
+
+        assertEquals((short) 1, ApiMessageType.CONTROLLED_SHUTDOWN.requestHeaderVersion((short) 1));
+        assertEquals((short) 0, ApiMessageType.CONTROLLED_SHUTDOWN.responseHeaderVersion((short) 1));
+
+        assertEquals((short) 1, ApiMessageType.CREATE_TOPICS.requestHeaderVersion((short) 5));
+        assertEquals((short) 0, ApiMessageType.CREATE_TOPICS.responseHeaderVersion((short) 5));
+
+        assertEquals((short) 2, ApiMessageType.CREATE_TOPICS.requestHeaderVersion((short) 6));
+        assertEquals((short) 1, ApiMessageType.CREATE_TOPICS.responseHeaderVersion((short) 6));
+    }
 }
diff --git a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
index c2d3acc7e9f24..73bb2afd97ca4 100644
--- a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
@@ -37,7 +37,9 @@
 import org.apache.kafka.common.protocol.ByteBufferAccessor;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.protocol.Message;
+import org.apache.kafka.common.protocol.ObjectSerializationCache;
 import org.apache.kafka.common.protocol.types.BoundField;
+import org.apache.kafka.common.protocol.types.RawTaggedField;
 import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.SchemaException;
 import org.apache.kafka.common.protocol.types.Struct;
@@ -612,16 +614,20 @@ private void testEquivalentMessageRoundTrip(short version, Message message) thro
     }
 
     private void testByteBufferRoundTrip(short version, Message message, Message expected) throws Exception {
-        int size = message.size(version);
+        ObjectSerializationCache cache = new ObjectSerializationCache();
+        int size = message.size(cache, version);
         ByteBuffer buf = ByteBuffer.allocate(size);
         ByteBufferAccessor byteBufferAccessor = new ByteBufferAccessor(buf);
-        message.write(byteBufferAccessor, version);
-        assertEquals(size, buf.position());
+        message.write(byteBufferAccessor, cache, version);
+        assertEquals("The result of the size function does not match the number of bytes " +
+            "written for version " + version, size, buf.position());
         Message message2 = message.getClass().newInstance();
         buf.flip();
         message2.read(byteBufferAccessor, version);
-        assertEquals(size, buf.position());
-        assertEquals(expected, message2);
+        assertEquals("The result of the size function does not match the number of bytes " +
+            "read back in for version " + version, size, buf.position());
+        assertEquals("The message object created after a round trip did not match for " +
+            "version " + version, expected, message2);
         assertEquals(expected.hashCode(), message2.hashCode());
         assertEquals(expected.toString(), message2.toString());
     }
@@ -794,13 +800,13 @@ private static List flatten(NamedType type) {
 
     @Test
     public void testDefaultValues() throws Exception {
-        verifySizeRaisesUve((short) 0, "validateOnly",
+        verifyWriteRaisesUve((short) 0, "validateOnly",
             new CreateTopicsRequestData().setValidateOnly(true));
-        verifySizeSucceeds((short) 0,
+        verifyWriteSucceeds((short) 0,
             new CreateTopicsRequestData().setValidateOnly(false));
-        verifySizeSucceeds((short) 0,
+        verifyWriteSucceeds((short) 0,
             new OffsetCommitRequestData().setRetentionTimeMs(123));
-        verifySizeRaisesUve((short) 5, "forgotten",
+        verifyWriteRaisesUve((short) 5, "forgotten",
             new FetchRequestData().setForgotten(singletonList(
                 new FetchRequestData.ForgottenTopic().setName("foo"))));
     }
@@ -808,35 +814,81 @@ public void testDefaultValues() throws Exception {
     @Test
     public void testNonIgnorableFieldWithDefaultNull() throws Exception {
         // Test non-ignorable string field `groupInstanceId` with default null
-        verifySizeRaisesUve((short) 0, "groupInstanceId", new HeartbeatRequestData()
+        verifyWriteRaisesUve((short) 0, "groupInstanceId", new HeartbeatRequestData()
                 .setGroupId("groupId")
                 .setGenerationId(15)
                 .setMemberId(memberId)
                 .setGroupInstanceId(instanceId));
-        verifySizeSucceeds((short) 0, new HeartbeatRequestData()
+        verifyWriteSucceeds((short) 0, new HeartbeatRequestData()
                 .setGroupId("groupId")
                 .setGenerationId(15)
                 .setMemberId(memberId)
                 .setGroupInstanceId(null));
-        verifySizeSucceeds((short) 0, new HeartbeatRequestData()
+        verifyWriteSucceeds((short) 0, new HeartbeatRequestData()
                 .setGroupId("groupId")
                 .setGenerationId(15)
                 .setMemberId(memberId));
     }
 
-    private void verifySizeRaisesUve(short version, String problemFieldName,
-                                     Message message) throws Exception {
-        try {
-            message.size(version);
-            fail("Expected to see an UnsupportedVersionException when writing " +
-                message + " at version " + version);
-        } catch (UnsupportedVersionException e) {
-            assertTrue("Expected to get an error message about " + problemFieldName,
-                e.getMessage().contains(problemFieldName));
+    @Test
+    public void testWriteNullForNonNullableFieldRaisesException() throws Exception {
+        CreateTopicsRequestData createTopics = new CreateTopicsRequestData().setTopics(null);
+        for (short i = (short) 0; i <= createTopics.highestSupportedVersion(); i++) {
+            verifyWriteRaisesNpe(i, createTopics);
         }
+        MetadataRequestData metadata = new MetadataRequestData().setTopics(null);
+        verifyWriteRaisesNpe((short) 0, metadata);
     }
 
-    private void verifySizeSucceeds(short version, Message message) throws Exception {
-        message.size(version);
+    @Test
+    public void testUnknownTaggedFields() throws Exception {
+        CreateTopicsRequestData createTopics = new CreateTopicsRequestData();
+        verifyWriteSucceeds((short) 6, createTopics);
+        RawTaggedField field1000 = new RawTaggedField(1000, new byte[] {0x1, 0x2, 0x3});
+        createTopics.unknownTaggedFields().add(field1000);
+        verifyWriteRaisesUve((short) 0, "Tagged fields were set", createTopics);
+        verifyWriteSucceeds((short) 6, createTopics);
+    }
+
+    private void verifyWriteRaisesNpe(short version, Message message) throws Exception {
+        ObjectSerializationCache cache = new ObjectSerializationCache();
+        assertThrows(NullPointerException.class, () -> {
+            int size = message.size(cache, version);
+            ByteBuffer buf = ByteBuffer.allocate(size);
+            ByteBufferAccessor byteBufferAccessor = new ByteBufferAccessor(buf);
+            message.write(byteBufferAccessor, cache, version);
+        });
+    }
+
+    private void verifyWriteRaisesUve(short version,
+                                      String problemText,
+                                     Message message) throws Exception {
+        ObjectSerializationCache cache = new ObjectSerializationCache();
+        UnsupportedVersionException e =
+            assertThrows(UnsupportedVersionException.class, () -> {
+                int size = message.size(cache, version);
+                ByteBuffer buf = ByteBuffer.allocate(size);
+                ByteBufferAccessor byteBufferAccessor = new ByteBufferAccessor(buf);
+                message.write(byteBufferAccessor, cache, version);
+            });
+        assertTrue("Expected to get an error message about " + problemText +
+                ", but got: " + e.getMessage(),
+                e.getMessage().contains(problemText));
+    }
+
+    private void verifyWriteSucceeds(short version, Message message) throws Exception {
+        ObjectSerializationCache cache = new ObjectSerializationCache();
+        int size = message.size(cache, version);
+        ByteBuffer buf = ByteBuffer.allocate(size * 2);
+        ByteBufferAccessor byteBufferAccessor = new ByteBufferAccessor(buf);
+        message.write(byteBufferAccessor, cache, version);
+        ByteBuffer alt = buf.duplicate();
+        alt.flip();
+        StringBuilder bld = new StringBuilder();
+        while (alt.hasRemaining()) {
+            bld.append(String.format(" %02x", alt.get()));
+        }
+        assertEquals("Expected the serialized size to be " + size +
+            ", but it was " + buf.position(), size, buf.position());
     }
 }
diff --git a/clients/src/test/java/org/apache/kafka/common/message/SimpleExampleMessageTest.java b/clients/src/test/java/org/apache/kafka/common/message/SimpleExampleMessageTest.java
new file mode 100644
index 0000000000000..5a3026f1873c3
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/message/SimpleExampleMessageTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.message;
+
+import org.apache.kafka.common.protocol.ByteBufferAccessor;
+import org.apache.kafka.common.protocol.MessageTestUtil;
+import org.apache.kafka.common.protocol.ObjectSerializationCache;
+import org.apache.kafka.common.protocol.types.Struct;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.UUID;
+
+import static org.junit.Assert.assertEquals;
+
+public class SimpleExampleMessageTest {
+
+    @Test
+    public void shouldStoreField() {
+        final UUID uuid = UUID.randomUUID();
+        final SimpleExampleMessageData out = new SimpleExampleMessageData();
+        out.setProcessId(uuid);
+        assertEquals(uuid, out.processId());
+    }
+
+    @Test
+    public void shouldDefaultField() {
+        final SimpleExampleMessageData out = new SimpleExampleMessageData();
+        assertEquals(UUID.fromString("00000000-0000-0000-0000-000000000000"), out.processId());
+    }
+
+    @Test
+    public void shouldRoundTripFieldThroughStruct() {
+        final UUID uuid = UUID.randomUUID();
+        final SimpleExampleMessageData out = new SimpleExampleMessageData();
+        out.setProcessId(uuid);
+
+        final Struct struct = out.toStruct((short) 1);
+        final SimpleExampleMessageData in = new SimpleExampleMessageData();
+        in.fromStruct(struct, (short) 1);
+
+        assertEquals(uuid, in.processId());
+    }
+
+    @Test
+    public void shouldRoundTripFieldThroughBuffer() {
+        final UUID uuid = UUID.randomUUID();
+        final SimpleExampleMessageData out = new SimpleExampleMessageData();
+        out.setProcessId(uuid);
+
+        ObjectSerializationCache cache = new ObjectSerializationCache();
+        final ByteBuffer buffer = ByteBuffer.allocate(out.size(cache, (short) 1));
+        out.write(new ByteBufferAccessor(buffer), cache, (short) 1);
+        buffer.rewind();
+
+        final SimpleExampleMessageData in = new SimpleExampleMessageData();
+        in.read(new ByteBufferAccessor(buffer), (short) 1);
+
+        assertEquals(uuid, in.processId());
+    }
+
+    @Test
+    public void shouldImplementJVMMethods() {
+        final UUID uuid = UUID.randomUUID();
+        final SimpleExampleMessageData a = new SimpleExampleMessageData();
+        a.setProcessId(uuid);
+
+        final SimpleExampleMessageData b = new SimpleExampleMessageData();
+        b.setProcessId(uuid);
+
+        assertEquals(a, b);
+        assertEquals(a.hashCode(), b.hashCode());
+        // just tagging this on here
+        assertEquals(a.toString(), b.toString());
+    }
+
+    @Test
+    public void testMyTaggedIntArray() {
+        final SimpleExampleMessageData data = new SimpleExampleMessageData();
+        data.setMyTaggedIntArray(Arrays.asList(1, 2, 3));
+        short version = 1;
+        ByteBuffer buf = MessageTestUtil.messageToByteBuffer(data, version);
+        final SimpleExampleMessageData data2 = new SimpleExampleMessageData();
+        data2.read(new ByteBufferAccessor(buf.duplicate()), version);
+        assertEquals(Arrays.asList(1, 2, 3), data.myTaggedIntArray());
+        assertEquals(Arrays.asList(1, 2, 3), data2.myTaggedIntArray());
+        assertEquals(data, data2);
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/common/message/TestUUIDDataTest.java b/clients/src/test/java/org/apache/kafka/common/message/TestUUIDDataTest.java
deleted file mode 100644
index 4b8e36b127190..0000000000000
--- a/clients/src/test/java/org/apache/kafka/common/message/TestUUIDDataTest.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.common.message;
-
-import org.apache.kafka.common.protocol.ByteBufferAccessor;
-import org.apache.kafka.common.protocol.types.Struct;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.nio.ByteBuffer;
-import java.util.UUID;
-
-public class TestUUIDDataTest {
-
-    @Test
-    public void shouldStoreField() {
-        final UUID uuid = UUID.randomUUID();
-        final TestUUIDData out = new TestUUIDData();
-        out.setProcessId(uuid);
-        Assert.assertEquals(uuid, out.processId());
-    }
-
-    @Test
-    public void shouldDefaultField() {
-        final TestUUIDData out = new TestUUIDData();
-        Assert.assertEquals(UUID.fromString("00000000-0000-0000-0000-000000000000"), out.processId());
-    }
-
-    @Test
-    public void shouldRoundTripFieldThroughStruct() {
-        final UUID uuid = UUID.randomUUID();
-        final TestUUIDData out = new TestUUIDData();
-        out.setProcessId(uuid);
-
-        final Struct struct = out.toStruct((short) 1);
-        final TestUUIDData in = new TestUUIDData();
-        in.fromStruct(struct, (short) 1);
-
-        Assert.assertEquals(uuid, in.processId());
-    }
-
-    @Test
-    public void shouldRoundTripFieldThroughBuffer() {
-        final UUID uuid = UUID.randomUUID();
-        final TestUUIDData out = new TestUUIDData();
-        out.setProcessId(uuid);
-
-        final ByteBuffer buffer = ByteBuffer.allocate(out.size((short) 1));
-        out.write(new ByteBufferAccessor(buffer), (short) 1);
-        buffer.rewind();
-
-        final TestUUIDData in = new TestUUIDData();
-        in.read(new ByteBufferAccessor(buffer), (short) 1);
-
-        Assert.assertEquals(uuid, in.processId());
-    }
-
-    @Test
-    public void shouldImplementJVMMethods() {
-        final UUID uuid = UUID.randomUUID();
-        final TestUUIDData a = new TestUUIDData();
-        a.setProcessId(uuid);
-
-        final TestUUIDData b = new TestUUIDData();
-        b.setProcessId(uuid);
-
-        Assert.assertEquals(a, b);
-        Assert.assertEquals(a.hashCode(), b.hashCode());
-        // just tagging this on here
-        Assert.assertEquals(a.toString(), b.toString());
-    }
-}
diff --git a/clients/src/test/java/org/apache/kafka/common/protocol/MessageTestUtil.java b/clients/src/test/java/org/apache/kafka/common/protocol/MessageTestUtil.java
new file mode 100644
index 0000000000000..a175ccd7c14e1
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/protocol/MessageTestUtil.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.common.protocol;
+
+import java.nio.ByteBuffer;
+
+public final class MessageTestUtil {
+    public static int messageSize(Message message, short version) {
+        return message.size(new ObjectSerializationCache(), version);
+    }
+
+    public static ByteBuffer messageToByteBuffer(Message message, short version) {
+        ObjectSerializationCache cache = new ObjectSerializationCache();
+        int size = message.size(cache, version);
+        ByteBuffer bytes = ByteBuffer.allocate(size);
+        message.write(new ByteBufferAccessor(bytes), cache, version);
+        bytes.flip();
+        return bytes;
+    }
+
+    public static void messageFromByteBuffer(ByteBuffer bytes, Message message, short version) {
+        message.read(new ByteBufferAccessor(bytes.duplicate()), version);
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/common/protocol/MessageUtilTest.java b/clients/src/test/java/org/apache/kafka/common/protocol/MessageUtilTest.java
index a28e15570e0b7..7494d66268934 100755
--- a/clients/src/test/java/org/apache/kafka/common/protocol/MessageUtilTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/protocol/MessageUtilTest.java
@@ -22,7 +22,6 @@
 import org.junit.rules.Timeout;
 
 import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 
 import static org.junit.Assert.assertArrayEquals;
@@ -32,25 +31,6 @@ public final class MessageUtilTest {
     @Rule
     final public Timeout globalTimeout = Timeout.millis(120000);
 
-    @Test
-    public void testSimpleUtf8Lengths() {
-        validateUtf8Length("");
-        validateUtf8Length("abc");
-        validateUtf8Length("This is a test string.");
-    }
-
-    @Test
-    public void testMultibyteUtf8Lengths() {
-        validateUtf8Length("A\u00ea\u00f1\u00fcC");
-        validateUtf8Length("\ud801\udc00");
-        validateUtf8Length("M\u00fcO");
-    }
-
-    private void validateUtf8Length(String string) {
-        byte[] arr = string.getBytes(StandardCharsets.UTF_8);
-        assertEquals(arr.length, MessageUtil.serializedUtf8Length(string));
-    }
-
     @Test
     public void testDeepToString() {
         assertEquals("[1, 2, 3]",
diff --git a/clients/src/test/java/org/apache/kafka/common/protocol/types/ProtocolSerializationTest.java b/clients/src/test/java/org/apache/kafka/common/protocol/types/ProtocolSerializationTest.java
index 505772004a7c0..e1c85671c51f5 100644
--- a/clients/src/test/java/org/apache/kafka/common/protocol/types/ProtocolSerializationTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/protocol/types/ProtocolSerializationTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.common.protocol.types;
 
+import org.apache.kafka.common.utils.ByteUtils;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -44,11 +45,17 @@ public void setup() {
                                  new Field("varint", Type.VARINT),
                                  new Field("varlong", Type.VARLONG),
                                  new Field("string", Type.STRING),
+                                 new Field("compact_string", Type.COMPACT_STRING),
                                  new Field("nullable_string", Type.NULLABLE_STRING),
+                                 new Field("compact_nullable_string", Type.COMPACT_NULLABLE_STRING),
                                  new Field("bytes", Type.BYTES),
+                                 new Field("compact_bytes", Type.COMPACT_BYTES),
                                  new Field("nullable_bytes", Type.NULLABLE_BYTES),
+                                 new Field("compact_nullable_bytes", Type.COMPACT_NULLABLE_BYTES),
                                  new Field("array", new ArrayOf(Type.INT32)),
+                                 new Field("compact_array", new CompactArrayOf(Type.INT32)),
                                  new Field("null_array", ArrayOf.nullable(Type.INT32)),
+                                 new Field("compact_null_array", CompactArrayOf.nullable(Type.INT32)),
                                  new Field("struct", new Schema(new Field("field", new ArrayOf(Type.INT32)))));
         this.struct = new Struct(this.schema).set("boolean", true)
                                              .set("int8", (byte) 1)
@@ -58,11 +65,17 @@ public void setup() {
                                              .set("varint", 300)
                                              .set("varlong", 500L)
                                              .set("string", "1")
+                                             .set("compact_string", "1")
                                              .set("nullable_string", null)
+                                             .set("compact_nullable_string", null)
                                              .set("bytes", ByteBuffer.wrap("1".getBytes()))
+                                             .set("compact_bytes", ByteBuffer.wrap("1".getBytes()))
                                              .set("nullable_bytes", null)
+                                             .set("compact_nullable_bytes", null)
                                              .set("array", new Object[] {1})
-                                             .set("null_array", null);
+                                             .set("compact_array", new Object[] {1})
+                                             .set("null_array", null)
+                                             .set("compact_null_array", null);
         this.struct.set("struct", this.struct.instance("struct").set("field", new Object[] {1, 2, 3}));
     }
 
@@ -77,14 +90,26 @@ public void testSimple() {
         check(Type.STRING, "", "STRING");
         check(Type.STRING, "hello", "STRING");
         check(Type.STRING, "A\u00ea\u00f1\u00fcC", "STRING");
+        check(Type.COMPACT_STRING, "", "COMPACT_STRING");
+        check(Type.COMPACT_STRING, "hello", "COMPACT_STRING");
+        check(Type.COMPACT_STRING, "A\u00ea\u00f1\u00fcC", "COMPACT_STRING");
         check(Type.NULLABLE_STRING, null, "NULLABLE_STRING");
         check(Type.NULLABLE_STRING, "", "NULLABLE_STRING");
         check(Type.NULLABLE_STRING, "hello", "NULLABLE_STRING");
+        check(Type.COMPACT_NULLABLE_STRING, null, "COMPACT_NULLABLE_STRING");
+        check(Type.COMPACT_NULLABLE_STRING, "", "COMPACT_NULLABLE_STRING");
+        check(Type.COMPACT_NULLABLE_STRING, "hello", "COMPACT_NULLABLE_STRING");
         check(Type.BYTES, ByteBuffer.allocate(0), "BYTES");
         check(Type.BYTES, ByteBuffer.wrap("abcd".getBytes()), "BYTES");
+        check(Type.COMPACT_BYTES, ByteBuffer.allocate(0), "COMPACT_BYTES");
+        check(Type.COMPACT_BYTES, ByteBuffer.wrap("abcd".getBytes()), "COMPACT_BYTES");
         check(Type.NULLABLE_BYTES, null, "NULLABLE_BYTES");
         check(Type.NULLABLE_BYTES, ByteBuffer.allocate(0), "NULLABLE_BYTES");
         check(Type.NULLABLE_BYTES, ByteBuffer.wrap("abcd".getBytes()), "NULLABLE_BYTES");
+        check(Type.COMPACT_NULLABLE_BYTES, null, "COMPACT_NULLABLE_BYTES");
+        check(Type.COMPACT_NULLABLE_BYTES, ByteBuffer.allocate(0), "COMPACT_NULLABLE_BYTES");
+        check(Type.COMPACT_NULLABLE_BYTES, ByteBuffer.wrap("abcd".getBytes()),
+                "COMPACT_NULLABLE_BYTES");
         check(Type.VARINT, Integer.MAX_VALUE, "VARINT");
         check(Type.VARINT, Integer.MIN_VALUE, "VARINT");
         check(Type.VARLONG, Long.MAX_VALUE, "VARLONG");
@@ -93,7 +118,16 @@ public void testSimple() {
         check(new ArrayOf(Type.STRING), new Object[] {}, "ARRAY(STRING)");
         check(new ArrayOf(Type.STRING), new Object[] {"hello", "there", "beautiful"},
                 "ARRAY(STRING)");
+        check(new CompactArrayOf(Type.INT32), new Object[] {1, 2, 3, 4},
+                "COMPACT_ARRAY(INT32)");
+        check(new CompactArrayOf(Type.COMPACT_STRING), new Object[] {},
+                "COMPACT_ARRAY(COMPACT_STRING)");
+        check(new CompactArrayOf(Type.COMPACT_STRING),
+                new Object[] {"hello", "there", "beautiful"},
+                "COMPACT_ARRAY(COMPACT_STRING)");
         check(ArrayOf.nullable(Type.STRING), null, "ARRAY(STRING)");
+        check(CompactArrayOf.nullable(Type.COMPACT_STRING), null,
+                "COMPACT_ARRAY(COMPACT_STRING)");
     }
 
     @Test
@@ -124,7 +158,9 @@ public void testDefault() {
     @Test
     public void testNullableDefault() {
         checkNullableDefault(Type.NULLABLE_BYTES, ByteBuffer.allocate(0));
+        checkNullableDefault(Type.COMPACT_NULLABLE_BYTES, ByteBuffer.allocate(0));
         checkNullableDefault(Type.NULLABLE_STRING, "default");
+        checkNullableDefault(Type.COMPACT_NULLABLE_STRING, "default");
     }
 
     private void checkNullableDefault(Type type, Object defaultValue) {
@@ -152,6 +188,24 @@ public void testReadArraySizeTooLarge() {
         }
     }
 
+    @Test
+    public void testReadCompactArraySizeTooLarge() {
+        Type type = new CompactArrayOf(Type.INT8);
+        int size = 10;
+        ByteBuffer invalidBuffer = ByteBuffer.allocate(
+            ByteUtils.sizeOfUnsignedVarint(Integer.MAX_VALUE) + size);
+        ByteUtils.writeUnsignedVarint(Integer.MAX_VALUE, invalidBuffer);
+        for (int i = 0; i < size; i++)
+            invalidBuffer.put((byte) i);
+        invalidBuffer.rewind();
+        try {
+            type.read(invalidBuffer);
+            fail("Array size not validated");
+        } catch (SchemaException e) {
+            // Expected exception
+        }
+    }
+
     @Test
     public void testReadNegativeArraySize() {
         Type type = new ArrayOf(Type.INT8);
@@ -169,6 +223,24 @@ public void testReadNegativeArraySize() {
         }
     }
 
+    @Test
+    public void testReadZeroCompactArraySize() {
+        Type type = new CompactArrayOf(Type.INT8);
+        int size = 10;
+        ByteBuffer invalidBuffer = ByteBuffer.allocate(
+            ByteUtils.sizeOfUnsignedVarint(0) + size);
+        ByteUtils.writeUnsignedVarint(0, invalidBuffer);
+        for (int i = 0; i < size; i++)
+            invalidBuffer.put((byte) i);
+        invalidBuffer.rewind();
+        try {
+            type.read(invalidBuffer);
+            fail("Array size not validated");
+        } catch (SchemaException e) {
+            // Expected exception
+        }
+    }
+
     @Test
     public void testReadStringSizeTooLarge() {
         byte[] stringBytes = "foo".getBytes();
diff --git a/clients/src/test/java/org/apache/kafka/common/protocol/types/RawTaggedFieldWriterTest.java b/clients/src/test/java/org/apache/kafka/common/protocol/types/RawTaggedFieldWriterTest.java
new file mode 100644
index 0000000000000..9ea38f9b84805
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/protocol/types/RawTaggedFieldWriterTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.common.protocol.types;
+
+import org.apache.kafka.common.protocol.ByteBufferAccessor;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class RawTaggedFieldWriterTest {
+    @Rule
+    final public Timeout globalTimeout = Timeout.millis(120000);
+
+    @Test
+    public void testWritingZeroRawTaggedFields() {
+        RawTaggedFieldWriter writer = RawTaggedFieldWriter.forFields(null);
+        assertEquals(0, writer.numFields());
+        ByteBufferAccessor accessor = new ByteBufferAccessor(ByteBuffer.allocate(0));
+        writer.writeRawTags(accessor, Integer.MAX_VALUE);
+    }
+
+    @Test
+    public void testWritingSeveralRawTaggedFields() {
+        List tags = Arrays.asList(
+            new RawTaggedField(2, new byte[] {0x1, 0x2, 0x3}),
+            new RawTaggedField(5, new byte[] {0x4, 0x5})
+        );
+        RawTaggedFieldWriter writer = RawTaggedFieldWriter.forFields(tags);
+        assertEquals(2, writer.numFields());
+        byte[] arr = new byte[9];
+        ByteBufferAccessor accessor = new ByteBufferAccessor(ByteBuffer.wrap(arr));
+        writer.writeRawTags(accessor, 1);
+        assertArrayEquals(new byte[] {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, arr);
+        writer.writeRawTags(accessor, 3);
+        assertArrayEquals(new byte[] {0x2, 0x3, 0x1, 0x2, 0x3, 0x0, 0x0, 0x0, 0x0}, arr);
+        writer.writeRawTags(accessor, 7);
+        assertArrayEquals(new byte[] {0x2, 0x3, 0x1, 0x2, 0x3, 0x5, 0x2, 0x4, 0x5}, arr);
+        writer.writeRawTags(accessor, Integer.MAX_VALUE);
+        assertArrayEquals(new byte[] {0x2, 0x3, 0x1, 0x2, 0x3, 0x5, 0x2, 0x4, 0x5}, arr);
+    }
+
+    @Test
+    public void testInvalidNextDefinedTag() {
+        List tags = Arrays.asList(
+            new RawTaggedField(2, new byte[] {0x1, 0x2, 0x3}),
+            new RawTaggedField(5, new byte[] {0x4, 0x5, 0x6}),
+            new RawTaggedField(7, new byte[] {0x0})
+        );
+        RawTaggedFieldWriter writer = RawTaggedFieldWriter.forFields(tags);
+        assertEquals(3, writer.numFields());
+        try {
+            writer.writeRawTags(new ByteBufferAccessor(ByteBuffer.allocate(1024)), 2);
+            fail("expected to get RuntimeException");
+        } catch (RuntimeException e) {
+            assertEquals("Attempted to use tag 2 as an undefined tag.", e.getMessage());
+        }
+    }
+
+    @Test
+    public void testOutOfOrderTags() {
+        List tags = Arrays.asList(
+            new RawTaggedField(5, new byte[] {0x4, 0x5, 0x6}),
+            new RawTaggedField(2, new byte[] {0x1, 0x2, 0x3}),
+            new RawTaggedField(7, new byte[] {0x0 })
+        );
+        RawTaggedFieldWriter writer = RawTaggedFieldWriter.forFields(tags);
+        assertEquals(3, writer.numFields());
+        try {
+            writer.writeRawTags(new ByteBufferAccessor(ByteBuffer.allocate(1024)), 8);
+            fail("expected to get RuntimeException");
+        } catch (RuntimeException e) {
+            assertEquals("Invalid raw tag field list: tag 2 comes after tag 5, but is " +
+                "not higher than it.", e.getMessage());
+        }
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java
index ffcc9cc09a8aa..6ad6197b6acae 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java
@@ -22,6 +22,7 @@
 import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrLiveLeader;
 import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState;
 import org.apache.kafka.common.protocol.ByteBufferAccessor;
+import org.apache.kafka.common.protocol.MessageTestUtil;
 import org.apache.kafka.test.TestUtils;
 import org.junit.Test;
 
@@ -104,7 +105,7 @@ public void testVersionLogic() {
             assertEquals(2, request.controllerEpoch());
             assertEquals(3, request.brokerEpoch());
 
-            ByteBuffer byteBuffer = request.toBytes();
+            ByteBuffer byteBuffer = MessageTestUtil.messageToByteBuffer(request.data(), request.version());
             LeaderAndIsrRequest deserializedRequest = new LeaderAndIsrRequest(new LeaderAndIsrRequestData(
                 new ByteBufferAccessor(byteBuffer), version), version);
 
@@ -138,7 +139,10 @@ public void testTopicPartitionGroupingSizeReduction() {
 
         LeaderAndIsrRequest v2 = builder.build((short) 2);
         LeaderAndIsrRequest v1 = builder.build((short) 1);
-        assertTrue("Expected v2 < v1: v2=" + v2.size() + ", v1=" + v1.size(), v2.size() < v1.size());
+        int size2 = MessageTestUtil.messageSize(v2.data(), v2.version());
+        int size1 = MessageTestUtil.messageSize(v1.data(), v1.version());
+
+        assertTrue("Expected v2 < v1: v2=" + size2 + ", v1=" + size1, size2 < size1);
     }
 
     private  Set iterableToSet(Iterable iterable) {
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java
index 99ec0effa9cf3..7d066250d2f2b 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java
@@ -67,7 +67,8 @@ public void testSerdeUnsupportedApiVersionRequest() throws Exception {
         responseBuffer.flip();
         responseBuffer.getInt(); // strip off the size
 
-        ResponseHeader responseHeader = ResponseHeader.parse(responseBuffer, header.headerVersion());
+        ResponseHeader responseHeader = ResponseHeader.parse(responseBuffer,
+            ApiKeys.API_VERSIONS.responseHeaderVersion(header.apiVersion()));
         assertEquals(correlationId, responseHeader.correlationId());
 
         Struct struct = ApiKeys.API_VERSIONS.parseResponse((short) 0, responseBuffer);
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java
index dd9dc327fc93b..1a7693919769a 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java
@@ -63,4 +63,14 @@ public void testRequestHeaderV1() {
         RequestHeader deserialized = RequestHeader.parse(buffer);
         assertEquals(header, deserialized);
     }
+
+    @Test
+    public void testRequestHeaderV2() {
+        RequestHeader header = new RequestHeader(ApiKeys.CREATE_DELEGATION_TOKEN, (short) 2, "", 10);
+        assertEquals(2, header.headerVersion());
+        ByteBuffer buffer = toBuffer(header.toStruct());
+        assertEquals(10, buffer.remaining());
+        RequestHeader deserialized = RequestHeader.parse(buffer);
+        assertEquals(header, deserialized);
+    }
 }
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index a8cbfffec1b9d..32b0a00679eef 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -573,12 +573,11 @@ public void produceResponseV5Test() {
 
         ProduceResponse v5Response = new ProduceResponse(responseData, 10);
         short version = 5;
-        short headerVersion = ApiKeys.PRODUCE.headerVersion(version);
 
         ByteBuffer buffer = v5Response.serialize(ApiKeys.PRODUCE, version, 0);
         buffer.rewind();
 
-        ResponseHeader.parse(buffer, headerVersion); // throw away.
+        ResponseHeader.parse(buffer, ApiKeys.PRODUCE.responseHeaderVersion(version)); // throw away.
 
         Struct deserializedStruct = ApiKeys.PRODUCE.parseResponse(version, buffer);
 
@@ -671,8 +670,8 @@ public void verifyFetchResponseFullWrites() throws Exception {
     private void verifyFetchResponseFullWrite(short apiVersion, FetchResponse fetchResponse) throws Exception {
         int correlationId = 15;
 
-        short headerVersion = FETCH.headerVersion(apiVersion);
-        Send send = fetchResponse.toSend("1", new ResponseHeader(correlationId, headerVersion), apiVersion);
+        short responseHeaderVersion = FETCH.responseHeaderVersion(apiVersion);
+        Send send = fetchResponse.toSend("1", new ResponseHeader(correlationId, responseHeaderVersion), apiVersion);
         ByteBufferChannel channel = new ByteBufferChannel(send.size());
         send.writeTo(channel);
         channel.close();
@@ -684,7 +683,7 @@ private void verifyFetchResponseFullWrite(short apiVersion, FetchResponse fetchR
         assertTrue(size > 0);
 
         // read the header
-        ResponseHeader responseHeader = ResponseHeader.parse(channel.buffer(), headerVersion);
+        ResponseHeader responseHeader = ResponseHeader.parse(channel.buffer(), responseHeaderVersion);
         assertEquals(correlationId, responseHeader.correlationId());
 
         // read the body
@@ -851,7 +850,8 @@ public void testApiVersionResponseWithNotUnsupportedError() {
     @Test
     public void testApiVersionResponseStructParsingFallback() {
         Struct struct = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.toStruct((short) 0);
-        ApiVersionsResponse response = ApiVersionsResponse.apiVersionsResponse(struct, ApiKeys.API_VERSIONS.latestVersion());
+        ApiVersionsResponse response = ApiVersionsResponse.
+            fromStruct(struct, ApiKeys.API_VERSIONS.latestVersion());
 
         assertEquals(Errors.NONE.code(), response.data.errorCode());
     }
@@ -859,13 +859,14 @@ public void testApiVersionResponseStructParsingFallback() {
     @Test(expected = SchemaException.class)
     public void testApiVersionResponseStructParsingFallbackException() {
         short version = 0;
-        ApiVersionsResponse.apiVersionsResponse(new Struct(ApiKeys.API_VERSIONS.requestSchema(version)), version);
+        ApiVersionsResponse.fromStruct(new Struct(ApiKeys.API_VERSIONS.requestSchema(version)), version);
     }
 
     @Test
     public void testApiVersionResponseStructParsing() {
         Struct struct = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.toStruct(ApiKeys.API_VERSIONS.latestVersion());
-        ApiVersionsResponse response = ApiVersionsResponse.apiVersionsResponse(struct, ApiKeys.API_VERSIONS.latestVersion());
+        ApiVersionsResponse response = ApiVersionsResponse.
+            fromStruct(struct, ApiKeys.API_VERSIONS.latestVersion());
 
         assertEquals(Errors.NONE.code(), response.data.errorCode());
     }
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java
index c06c9c3c61646..a143ff3301a12 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java
@@ -17,6 +17,7 @@
 package org.apache.kafka.common.requests;
 
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.protocol.MessageTestUtil;
 import org.apache.kafka.test.TestUtils;
 import org.junit.Test;
 
@@ -30,7 +31,8 @@ public class StopReplicaRequestTest {
     public void testStopReplicaRequestNormalization() {
         Set tps = TestUtils.generateRandomTopicPartitions(10, 10);
         StopReplicaRequest.Builder builder = new StopReplicaRequest.Builder((short) 5, 0, 0, 0, false, tps);
-        assertTrue(builder.build((short) 1).size() <  builder.build((short) 0).size());
+        assertTrue(MessageTestUtil.messageSize(builder.build((short) 1).data(), (short) 1) <
+            MessageTestUtil.messageSize(builder.build((short) 0).data(), (short) 0));
     }
 
 }
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java
index 9bc81fe86ed29..a5d0d99d3d61d 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java
@@ -23,6 +23,7 @@
 import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataPartitionState;
 import org.apache.kafka.common.network.ListenerName;
 import org.apache.kafka.common.protocol.ByteBufferAccessor;
+import org.apache.kafka.common.protocol.MessageTestUtil;
 import org.apache.kafka.common.security.auth.SecurityProtocol;
 import org.apache.kafka.test.TestUtils;
 import org.junit.Test;
@@ -132,7 +133,7 @@ public void testVersionLogic() {
             assertEquals(2, request.controllerEpoch());
             assertEquals(3, request.brokerEpoch());
 
-            ByteBuffer byteBuffer = request.toBytes();
+            ByteBuffer byteBuffer = MessageTestUtil.messageToByteBuffer(request.data(), request.version());
             UpdateMetadataRequest deserializedRequest = new UpdateMetadataRequest(new UpdateMetadataRequestData(
                     new ByteBufferAccessor(byteBuffer), version), version);
 
@@ -178,7 +179,8 @@ public void testTopicPartitionGroupingSizeReduction() {
         UpdateMetadataRequest.Builder builder = new UpdateMetadataRequest.Builder((short) 5, 0, 0, 0,
                 partitionStates, Collections.emptyList());
 
-        assertTrue(builder.build((short) 5).size() <  builder.build((short) 4).size());
+        assertTrue(MessageTestUtil.messageSize(builder.build((short) 5).data(), (short) 5) <
+            MessageTestUtil.messageSize(builder.build((short) 4).data(), (short) 4));
     }
 
     private  Set iterableToSet(Iterable iterable) {
diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
index 152290f360b62..6d95d2e91d857 100644
--- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
@@ -689,11 +689,12 @@ public void testApiVersionsRequestWithUnsupportedVersion() throws Exception {
                 setRequestApiKey(ApiKeys.API_VERSIONS.id).
                 setRequestApiVersion(Short.MAX_VALUE).
                 setClientId("someclient").
-                setCorrelationId(1), (short) 1);
+                setCorrelationId(1),
+                (short) 2);
         ApiVersionsRequest request = new ApiVersionsRequest.Builder().build();
         selector.send(request.toSend(node, header));
         ByteBuffer responseBuffer = waitForResponse();
-        ResponseHeader.parse(responseBuffer, header.headerVersion());
+        ResponseHeader.parse(responseBuffer, ApiKeys.API_VERSIONS.responseHeaderVersion((short) 0));
         ApiVersionsResponse response = ApiVersionsResponse.parse(responseBuffer, (short) 0);
         assertEquals(Errors.UNSUPPORTED_VERSION.code(), response.data.errorCode());
 
@@ -727,11 +728,14 @@ public void testInvalidApiVersionsRequest() throws Exception {
         short version = ApiKeys.API_VERSIONS.latestVersion();
         createClientConnection(SecurityProtocol.PLAINTEXT, node);
         RequestHeader header = new RequestHeader(ApiKeys.API_VERSIONS, version, "someclient", 1);
-        ApiVersionsRequest request = new ApiVersionsRequest(new ApiVersionsRequestData(), version);
+        ApiVersionsRequest request = new ApiVersionsRequest(new ApiVersionsRequestData().
+                setClientSoftwareName("  ").
+                setClientSoftwareVersion("   "), version);
         selector.send(request.toSend(node, header));
         ByteBuffer responseBuffer = waitForResponse();
-        ResponseHeader.parse(responseBuffer, header.headerVersion());
-        ApiVersionsResponse response = ApiVersionsResponse.parse(responseBuffer, version);
+        ResponseHeader.parse(responseBuffer, ApiKeys.API_VERSIONS.responseHeaderVersion(version));
+        ApiVersionsResponse response =
+            ApiVersionsResponse.parse(responseBuffer, version);
         assertEquals(Errors.INVALID_REQUEST.code(), response.data.errorCode());
 
         // Send ApiVersionsRequest with a supported version. This should succeed.
@@ -761,7 +765,7 @@ public void testValidApiVersionsRequest() throws Exception {
         ApiVersionsRequest request = new ApiVersionsRequest.Builder().build(version);
         selector.send(request.toSend(node, header));
         ByteBuffer responseBuffer = waitForResponse();
-        ResponseHeader.parse(responseBuffer, header.headerVersion());
+        ResponseHeader.parse(responseBuffer, ApiKeys.API_VERSIONS.responseHeaderVersion(version));
         ApiVersionsResponse response = ApiVersionsResponse.parse(responseBuffer, version);
         assertEquals(Errors.NONE.code(), response.data.errorCode());
 
diff --git a/clients/src/test/resources/common/message/TestUUID.json b/clients/src/test/resources/common/message/SimpleExampleMessage.json
similarity index 73%
rename from clients/src/test/resources/common/message/TestUUID.json
rename to clients/src/test/resources/common/message/SimpleExampleMessage.json
index 843a0402bb568..b20c1c090cbc1 100644
--- a/clients/src/test/resources/common/message/TestUUID.json
+++ b/clients/src/test/resources/common/message/SimpleExampleMessage.json
@@ -13,10 +13,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 {
-  "name": "TestUUID",
-  "validVersions": "1",
+  "name": "SimpleExampleMessage",
+  "type": "header",
+  "validVersions": "0-1",
+  "flexibleVersions": "1+",
   "fields": [
-    { "name": "processId", "versions": "1+", "type": "uuid" }
-  ],
-  "type": "header"
-}
\ No newline at end of file
+    { "name": "processId", "versions": "1+", "type": "uuid" },
+    { "name": "myTaggedIntArray", "type": "[]int32",
+        "versions": "1+", "taggedVersions": "1+", "tag": 0 }
+  ]
+}
diff --git a/core/src/test/scala/unit/kafka/server/ApiVersionsRequestTest.scala b/core/src/test/scala/unit/kafka/server/ApiVersionsRequestTest.scala
index 04e4f88d70798..e915fcd357959 100644
--- a/core/src/test/scala/unit/kafka/server/ApiVersionsRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ApiVersionsRequestTest.scala
@@ -45,7 +45,8 @@ class ApiVersionsRequestTest extends BaseRequestTest {
 
   @Test
   def testApiVersionsRequest(): Unit = {
-    val apiVersionsResponse = sendApiVersionsRequest(new ApiVersionsRequest.Builder().build())
+    val request = new ApiVersionsRequest.Builder().build()
+    val apiVersionsResponse = sendApiVersionsRequest(request, None, request.version)
     ApiVersionsRequestTest.validateApiVersionsResponse(apiVersionsResponse)
   }
 
@@ -76,7 +77,8 @@ class ApiVersionsRequestTest extends BaseRequestTest {
     assertEquals(Errors.INVALID_REQUEST.code(), apiVersionsResponse.data.errorCode())
   }
 
-  private def sendApiVersionsRequest(request: ApiVersionsRequest, apiVersion: Option[Short] = None, responseVersion: Short = 1): ApiVersionsResponse = {
+  private def sendApiVersionsRequest(request: ApiVersionsRequest, apiVersion: Option[Short] = None,
+                                     responseVersion: Short): ApiVersionsResponse = {
     val response = connectAndSend(request, ApiKeys.API_VERSIONS, apiVersion = apiVersion)
     ApiVersionsResponse.parse(response, responseVersion)
   }
diff --git a/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala b/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala
index 2062fb762028d..5d00d540b149a 100644
--- a/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala
@@ -136,9 +136,9 @@ abstract class BaseRequestTest extends IntegrationTestHarness {
   /**
    * Receive response and return a ByteBuffer containing response without the header
    */
-  def receive(socket: Socket, headerVersion: Short): ByteBuffer = {
+  def receive(socket: Socket, responseHeaderVersion: Short): ByteBuffer = {
     val response = receiveResponse(socket)
-    skipResponseHeader(response, headerVersion)
+    skipResponseHeader(response, responseHeaderVersion)
   }
 
   /**
@@ -149,7 +149,7 @@ abstract class BaseRequestTest extends IntegrationTestHarness {
     val version = apiVersion.getOrElse(request.version)
     send(request, apiKey, socket, version)
     val response = receiveResponse(socket)
-    skipResponseHeader(response, apiKey.headerVersion(version))
+    skipResponseHeader(response, apiKey.responseHeaderVersion(version))
   }
 
   /**
@@ -160,7 +160,7 @@ abstract class BaseRequestTest extends IntegrationTestHarness {
     val request = requestBuilder.build()
     val header = new RequestHeader(apiKey, request.version, clientId, correlationId)
     val response = requestAndReceive(socket, request.serialize(header).array)
-    val responseBuffer = skipResponseHeader(response, header.headerVersion())
+    val responseBuffer = skipResponseHeader(response, apiKey.responseHeaderVersion(request.version()))
     apiKey.parseResponse(request.version, responseBuffer)
   }
   
@@ -172,13 +172,13 @@ abstract class BaseRequestTest extends IntegrationTestHarness {
     val header = nextRequestHeader(apiKey, apiVersion)
     val serializedBytes = AbstractRequestResponse.serialize(header.toStruct, requestStruct).array
     val response = requestAndReceive(socket, serializedBytes)
-    skipResponseHeader(response, header.headerVersion())
+    skipResponseHeader(response, apiKey.responseHeaderVersion(apiVersion))
   }
 
-  protected def skipResponseHeader(response: Array[Byte], headerVersion: Short): ByteBuffer = {
+  protected def skipResponseHeader(response: Array[Byte], responseHeaderVersion: Short): ByteBuffer = {
     val responseBuffer = ByteBuffer.wrap(response)
     // Parse the header to ensure its valid and move the buffer forward
-    ResponseHeader.parse(responseBuffer, headerVersion)
+    ResponseHeader.parse(responseBuffer, responseHeaderVersion)
     responseBuffer
   }
 
diff --git a/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala b/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala
index 42409db72c24d..b8374f2435d6c 100755
--- a/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala
@@ -116,7 +116,7 @@ class EdgeCaseRequestTest extends KafkaServerTestHarness {
     createTopic(topic, numPartitions = 1, replicationFactor = 1)
 
     val version = ApiKeys.PRODUCE.latestVersion: Short
-    val (serializedBytes, headerVersion) = {
+    val (serializedBytes, responseHeaderVersion) = {
       val headerBytes = requestHeaderBytes(ApiKeys.PRODUCE.id, version, null,
         correlationId)
       val records = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("message".getBytes))
@@ -124,13 +124,13 @@ class EdgeCaseRequestTest extends KafkaServerTestHarness {
       val byteBuffer = ByteBuffer.allocate(headerBytes.length + request.toStruct.sizeOf)
       byteBuffer.put(headerBytes)
       request.toStruct.writeTo(byteBuffer)
-      (byteBuffer.array(), request.api.headerVersion(version))
+      (byteBuffer.array(), request.api.responseHeaderVersion(version))
     }
 
     val response = requestAndReceive(serializedBytes)
 
     val responseBuffer = ByteBuffer.wrap(response)
-    val responseHeader = ResponseHeader.parse(responseBuffer, headerVersion)
+    val responseHeader = ResponseHeader.parse(responseBuffer, responseHeaderVersion)
     val produceResponse = ProduceResponse.parse(responseBuffer, version)
 
     assertEquals("The response should parse completely", 0, responseBuffer.remaining)
diff --git a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
index daea624240433..895d9e55401e3 100644
--- a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
@@ -280,7 +280,7 @@ class FetchRequestTest extends BaseRequestTest {
 
       val socket = connect(brokerSocketServer(leaderId))
       try {
-        send(fetchRequest, ApiKeys.FETCH, socket, fetchRequest.api.headerVersion(fetchRequest.version()))
+        send(fetchRequest, ApiKeys.FETCH, socket, fetchRequest.version())
         if (closeAfterPartialResponse) {
           // read some data to ensure broker has muted this channel and then close socket
           val size = new DataInputStream(socket.getInputStream).readInt()
@@ -291,7 +291,7 @@ class FetchRequestTest extends BaseRequestTest {
               size > maxPartitionBytes - batchSize)
           None
         } else {
-          Some(FetchResponse.parse(receive(socket, ApiKeys.FETCH.headerVersion(version)), version))
+          Some(FetchResponse.parse(receive(socket, ApiKeys.FETCH.responseHeaderVersion(version)), version))
         }
       } finally {
         socket.close()
diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
index 48d21e29f26ae..bf1a8c487b1cc 100644
--- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
@@ -817,7 +817,7 @@ class KafkaApisTest {
     send.writeTo(channel)
     channel.close()
     channel.buffer.getInt() // read the size
-    ResponseHeader.parse(channel.buffer, sendResponse.request.header.headerVersion())
+    ResponseHeader.parse(channel.buffer, api.responseHeaderVersion(request.version()))
     val struct = api.responseSchema(request.version).read(channel.buffer)
     AbstractResponse.parseResponse(api, struct, request.version)
   }
diff --git a/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java b/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java
index 5a2cf117fea50..14dc4e87480d2 100644
--- a/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java
+++ b/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java
@@ -135,6 +135,10 @@ public void generate() {
         generateAccessor("responseSchemas", "Schema[]");
         buffer.printf("%n");
         generateToString();
+        buffer.printf("%n");
+        generateHeaderVersion("request");
+        buffer.printf("%n");
+        generateHeaderVersion("response");
         buffer.decrementIndent();
         buffer.printf("}%n");
         headerGenerator.generate();
@@ -245,6 +249,74 @@ private void generateToString() {
         buffer.printf("}%n");
     }
 
+    private void generateHeaderVersion(String type) {
+        buffer.printf("public short %sHeaderVersion(short _version) {%n", type);
+        buffer.incrementIndent();
+        buffer.printf("switch (apiKey) {%n");
+        buffer.incrementIndent();
+        for (Map.Entry entry : apis.entrySet()) {
+            short apiKey = entry.getKey();
+            buffer.printf("case %d:%n", apiKey);
+            buffer.incrementIndent();
+            if (type.equals("response") && apiKey == 18) {
+                // ApiVersionsResponse always includes a v0 header.
+                // See KIP-511 for details.
+                buffer.printf("return (short) 0;%n");
+                buffer.decrementIndent();
+                continue;
+            }
+            if (type.equals("request") && apiKey == 7) {
+                // Version 0 of ControlledShutdownRequest has a non-standard request header
+                // which does not include clientId.  Version 1 of ControlledShutdownRequest
+                // and later use the standard request header.
+                buffer.printf("if (_version == 0) {%n");
+                buffer.incrementIndent();
+                buffer.printf("return (short) 0;%n");
+                buffer.decrementIndent();
+                buffer.printf("}%n");
+            }
+            ApiData data = entry.getValue();
+            MessageSpec spec = null;
+            if (type.equals("request")) {
+                spec = data.requestSpec;
+            } else if (type.equals("response")) {
+                spec = data.responseSpec;
+            } else {
+                throw new RuntimeException("Invalid type " + type + " for generateHeaderVersion");
+            }
+            if (spec == null) {
+                throw new RuntimeException("failed to find " + type + " for API key " + apiKey);
+            }
+            VersionConditional.forVersions(spec.flexibleVersions(),
+                spec.validVersions()).
+                ifMember(__ -> {
+                    if (type.equals("request")) {
+                        buffer.printf("return (short) 2;%n");
+                    } else {
+                        buffer.printf("return (short) 1;%n");
+                    }
+                }).
+                ifNotMember(__ -> {
+                    if (type.equals("request")) {
+                        buffer.printf("return (short) 1;%n");
+                    } else {
+                        buffer.printf("return (short) 0;%n");
+                    }
+                }).generate(buffer);
+            buffer.decrementIndent();
+        }
+        buffer.printf("default:%n");
+        buffer.incrementIndent();
+        headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS);
+        buffer.printf("throw new UnsupportedVersionException(\"Unsupported API key \"" +
+            " + apiKey);%n");
+        buffer.decrementIndent();
+        buffer.decrementIndent();
+        buffer.printf("}%n");
+        buffer.decrementIndent();
+        buffer.printf("}%n");
+    }
+
     public void write(BufferedWriter writer) throws IOException {
         headerGenerator.buffer().write(writer);
         buffer.write(writer);
diff --git a/generator/src/main/java/org/apache/kafka/message/FieldSpec.java b/generator/src/main/java/org/apache/kafka/message/FieldSpec.java
index 7e5b37c1dc3b4..653069787c571 100644
--- a/generator/src/main/java/org/apache/kafka/message/FieldSpec.java
+++ b/generator/src/main/java/org/apache/kafka/message/FieldSpec.java
@@ -24,8 +24,12 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
+import java.util.regex.Pattern;
 
 public final class FieldSpec {
+    private static final Pattern VALID_FIELD_NAMES = Pattern.compile("[A-Za-z]([A-Za-z0-9]*)");
+
     private final String name;
 
     private final Versions versions;
@@ -46,6 +50,12 @@ public final class FieldSpec {
 
     private final String about;
 
+    private final Versions taggedVersions;
+
+    private final Optional flexibleVersions;
+
+    private final Optional tag;
+
     @JsonCreator
     public FieldSpec(@JsonProperty("name") String name,
                      @JsonProperty("versions") String versions,
@@ -56,8 +66,14 @@ public FieldSpec(@JsonProperty("name") String name,
                      @JsonProperty("default") String fieldDefault,
                      @JsonProperty("ignorable") boolean ignorable,
                      @JsonProperty("entityType") EntityType entityType,
-                     @JsonProperty("about") String about) {
+                     @JsonProperty("about") String about,
+                     @JsonProperty("taggedVersions") String taggedVersions,
+                     @JsonProperty("flexibleVersions") String flexibleVersions,
+                     @JsonProperty("tag") Integer tag) {
         this.name = Objects.requireNonNull(name);
+        if (!VALID_FIELD_NAMES.matcher(this.name).matches()) {
+            throw new RuntimeException("Invalid field name " + this.name);
+        }
         this.versions = Versions.parse(versions, null);
         if (this.versions == null) {
             throw new RuntimeException("You must specify the version of the " +
@@ -84,6 +100,56 @@ public FieldSpec(@JsonProperty("name") String name,
                 throw new RuntimeException("Non-array field " + name + " cannot have fields");
             }
         }
+        this.taggedVersions = Versions.parse(taggedVersions, Versions.NONE);
+        if (flexibleVersions == null || flexibleVersions.isEmpty()) {
+            this.flexibleVersions = Optional.empty();
+        } else {
+            this.flexibleVersions = Optional.of(Versions.parse(flexibleVersions, null));
+            if (!(this.type.isString() || this.type.isBytes())) {
+                // For now, only allow flexibleVersions overrides for the string and bytes
+                // types.  Overrides are only needed to keep compatibility with some old formats,
+                // so there isn't any need to support them for all types.
+                throw new RuntimeException("Invalid flexibleVersions override for " + name +
+                    ".  Only fields of type string or bytes can specify a flexibleVersions " +
+                    "override.");
+            }
+        }
+        this.tag = Optional.ofNullable(tag);
+        checkTagInvariants();
+    }
+
+    private void checkTagInvariants() {
+        if (this.tag.isPresent()) {
+            if (this.tag.get() < 0) {
+                throw new RuntimeException("Field " + name + " specifies a tag of " + this.tag.get() +
+                    ".  Tags cannot be negative.");
+            }
+            if (this.taggedVersions.empty()) {
+                throw new RuntimeException("Field " + name + " specifies a tag of " + this.tag.get() +
+                    ", but has no tagged versions.  If a tag is specified, taggedVersions must " +
+                    "be specified as well.");
+            }
+            Versions nullableTaggedVersions = this.nullableVersions.intersect(this.taggedVersions);
+            if (!(nullableTaggedVersions.empty() || nullableTaggedVersions.equals(this.taggedVersions))) {
+                throw new RuntimeException("Field " + name + " specifies nullableVersions " +
+                    this.nullableVersions + " and taggedVersions " + this.taggedVersions + ".  " +
+                    "Either all tagged versions must be nullable, or none must be.");
+            }
+            if (this.taggedVersions.highest() < Short.MAX_VALUE) {
+                throw new RuntimeException("Field " + name + " specifies taggedVersions " +
+                    this.taggedVersions + ", which is not open-ended.  taggedVersions must " +
+                    "be either none, or an open-ended range (that ends with a plus sign).");
+            }
+            if (!this.taggedVersions.intersect(this.versions).equals(this.taggedVersions)) {
+                throw new RuntimeException("Field " + name + " specifies taggedVersions " +
+                    this.taggedVersions + ", and versions " + this.versions + ".  " +
+                    "taggedVersions must be a subset of versions.");
+            }
+        } else if (!this.taggedVersions.empty()) {
+            throw new RuntimeException("Field " + name + " does not specify a tag, " +
+                "but specifies tagged versions of " + this.taggedVersions + ".  " +
+                "Please specify a tag, or remove the taggedVersions.");
+        }
     }
 
     @JsonProperty("name")
@@ -154,4 +220,31 @@ public boolean ignorable() {
     public String about() {
         return about;
     }
+
+    @JsonProperty("taggedVersions")
+    public String taggedVersionsString() {
+        return taggedVersions.toString();
+    }
+
+    public Versions taggedVersions() {
+        return taggedVersions;
+    }
+
+    @JsonProperty("flexibleVersions")
+    public String flexibleVersionsString() {
+        return flexibleVersions.isPresent() ? flexibleVersions.get().toString() : null;
+    }
+
+    public Optional flexibleVersions() {
+        return flexibleVersions;
+    }
+
+    @JsonProperty("tag")
+    public Integer tagInteger() {
+        return tag.orElse(null);
+    }
+
+    public Optional tag() {
+        return tag;
+    }
 }
diff --git a/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java b/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java
index 848af660eedaa..4fd94ee88379c 100644
--- a/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java
+++ b/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java
@@ -22,6 +22,7 @@
 import java.util.Iterator;
 import java.util.Optional;
 import java.util.Set;
+import java.util.TreeMap;
 import java.util.UUID;
 import java.util.stream.Collectors;
 
@@ -29,10 +30,13 @@
  * Generates Kafka MessageData classes.
  */
 public final class MessageDataGenerator {
+    private final static String TAGGED_FIELDS_SECTION_NAME = "_tagged_fields";
+
     private final StructRegistry structRegistry;
     private final HeaderGenerator headerGenerator;
     private final SchemaGenerator schemaGenerator;
     private final CodeBuffer buffer;
+    private Versions messageFlexibleVersions;
 
     MessageDataGenerator(String packageName) {
         this.structRegistry = new StructRegistry();
@@ -48,6 +52,7 @@ void generate(MessageSpec message) throws Exception {
         }
         structRegistry.register(message);
         schemaGenerator.generateSchemas(message);
+        messageFlexibleVersions = message.flexibleVersions();
         generateClass(Optional.of(message),
             message.name() + "Data",
             message.struct(),
@@ -101,6 +106,8 @@ private void generateClass(Optional topLevelMessageSpec,
         buffer.printf("%n");
         generateClassToString(className, struct);
         generateFieldAccessors(struct, isSetElement);
+        buffer.printf("%n");
+        generateUnknownTaggedFieldsAccessor(struct);
         generateFieldMutators(struct, className, isSetElement);
 
         if (!isTopLevel) {
@@ -255,6 +262,9 @@ private void generateFieldDeclarations(StructSpec struct, boolean isSetElement)
         for (FieldSpec field : struct.fields()) {
             generateFieldDeclaration(field);
         }
+        headerGenerator.addImport(MessageGenerator.LIST_CLASS);
+        headerGenerator.addImport(MessageGenerator.RAW_TAGGED_FIELD_CLASS);
+        buffer.printf("private List _unknownTaggedFields;%n");
         if (isSetElement) {
             buffer.printf("private int next;%n");
             buffer.printf("private int prev;%n");
@@ -281,6 +291,26 @@ private void generateFieldAccessors(StructSpec struct, boolean isSetElement) {
         }
     }
 
+    private void generateUnknownTaggedFieldsAccessor(StructSpec struct) {
+        buffer.printf("@Override%n");
+        headerGenerator.addImport(MessageGenerator.LIST_CLASS);
+        headerGenerator.addImport(MessageGenerator.RAW_TAGGED_FIELD_CLASS);
+        buffer.printf("public List unknownTaggedFields() {%n");
+        buffer.incrementIndent();
+        // Optimize _unknownTaggedFields by not creating a new list object
+        // unless we need it.
+        buffer.printf("if (_unknownTaggedFields == null) {%n");
+        buffer.incrementIndent();
+        headerGenerator.addImport(MessageGenerator.ARRAYLIST_CLASS);
+        buffer.printf("_unknownTaggedFields = new ArrayList<>(0);%n");
+        buffer.decrementIndent();
+        buffer.printf("}%n");
+        buffer.printf("return _unknownTaggedFields;%n");
+        buffer.decrementIndent();
+        buffer.printf("}%n");
+
+    }
+
     private void generateFieldMutators(StructSpec struct, String className,
                                        boolean isSetElement) {
         for (FieldSpec field : struct.fields()) {
@@ -313,6 +343,7 @@ private String fieldAbstractJavaType(FieldSpec field) {
         } else if (field.type() instanceof FieldType.Int64FieldType) {
             return "long";
         } else if (field.type() instanceof FieldType.UUIDFieldType) {
+            headerGenerator.addImport(MessageGenerator.UUID_CLASS);
             return "UUID";
         } else if (field.type().isString()) {
             return "String";
@@ -351,18 +382,16 @@ private String fieldConcreteJavaType(FieldSpec field) {
 
     private void generateClassConstructors(String className, StructSpec struct) {
         headerGenerator.addImport(MessageGenerator.READABLE_CLASS);
-        buffer.printf("public %s(Readable readable, short version) {%n", className);
+        buffer.printf("public %s(Readable _readable, short _version) {%n", className);
         buffer.incrementIndent();
-        initializeArrayDefaults(struct);
-        buffer.printf("read(readable, version);%n");
+        buffer.printf("read(_readable, _version);%n");
         buffer.decrementIndent();
         buffer.printf("}%n");
         buffer.printf("%n");
         headerGenerator.addImport(MessageGenerator.STRUCT_CLASS);
-        buffer.printf("public %s(Struct struct, short version) {%n", className);
+        buffer.printf("public %s(Struct struct, short _version) {%n", className);
         buffer.incrementIndent();
-        initializeArrayDefaults(struct);
-        buffer.printf("fromStruct(struct, version);%n");
+        buffer.printf("fromStruct(struct, _version);%n");
         buffer.decrementIndent();
         buffer.printf("}%n");
         buffer.printf("%n");
@@ -376,15 +405,6 @@ private void generateClassConstructors(String className, StructSpec struct) {
         buffer.printf("}%n");
     }
 
-    private void initializeArrayDefaults(StructSpec struct) {
-        for (FieldSpec field : struct.fields()) {
-            if (field.type().isArray()) {
-                buffer.printf("this.%s = %s;%n",
-                    field.camelCaseName(), fieldDefault(field));
-            }
-        }
-    }
-
     private void generateShortAccessor(String name, short val) {
         buffer.printf("@Override%n");
         buffer.printf("public short %s() {%n", name);
@@ -398,181 +418,365 @@ private void generateClassReader(String className, StructSpec struct,
                                      Versions parentVersions) {
         headerGenerator.addImport(MessageGenerator.READABLE_CLASS);
         buffer.printf("@Override%n");
-        buffer.printf("public void read(Readable readable, short version) {%n");
+        buffer.printf("public void read(Readable _readable, short _version) {%n");
         buffer.incrementIndent();
-        if (generateInverseVersionCheck(parentVersions, struct.versions())) {
-            buffer.incrementIndent();
-            headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS);
-            buffer.printf("throw new UnsupportedVersionException(\"Can't read " +
-                "version \" + version + \" of %s\");%n", className);
-            buffer.decrementIndent();
-            buffer.printf("}%n");
-        }
+        VersionConditional.forVersions(parentVersions, struct.versions()).
+            allowMembershipCheckAlwaysFalse(false).
+            ifNotMember(__ -> {
+                headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS);
+                buffer.printf("throw new UnsupportedVersionException(\"Can't read " +
+                    "version \" + _version + \" of %s\");%n", className);
+            }).
+            generate(buffer);
         Versions curVersions = parentVersions.intersect(struct.versions());
-        if (curVersions.empty()) {
-            throw new RuntimeException("Version ranges " + parentVersions +
-                " and " + struct.versions() + " have no versions in common.");
-        }
         for (FieldSpec field : struct.fields()) {
-            generateFieldReader(field, curVersions);
+            Versions fieldFlexibleVersions = fieldFlexibleVersions(field);
+            if (!field.taggedVersions().intersect(fieldFlexibleVersions).equals(field.taggedVersions())) {
+                throw new RuntimeException("Field " + field.name() + " specifies tagged " +
+                    "versions " + field.taggedVersions() + " that are not a subset of the " +
+                    "flexible versions " + fieldFlexibleVersions);
+            }
+            Versions mandatoryVersions = field.versions().subtract(field.taggedVersions());
+            VersionConditional.forVersions(mandatoryVersions, curVersions).
+                alwaysEmitBlockScope(field.type().isVariableLength()).
+                ifNotMember(__ -> {
+                    // If the field is not present, or is tagged, set it to its default here.
+                    buffer.printf("this.%s = %s;%n", field.camelCaseName(), fieldDefault(field));
+                }).
+                ifMember(presentAndUntaggedVersions -> {
+                    if (field.type().isVariableLength()) {
+                        ClauseGenerator callGenerateVariableLengthReader = versions -> {
+                            generateVariableLengthReader(fieldFlexibleVersions(field),
+                                field.camelCaseName(),
+                                field.type(),
+                                versions,
+                                field.nullableVersions(),
+                                String.format("this.%s = ", field.camelCaseName()),
+                                String.format(";%n"),
+                                structRegistry.isStructArrayWithKeys(field));
+                        };
+                        // For arrays where the field type needs to be serialized differently in flexible
+                        // versions, lift the flexible version check outside of the array.
+                        // This may mean generating two separate 'for' loops-- one for flexible
+                        // versions, and one for regular versions.
+                        if (field.type().isArray() &&
+                            ((FieldType.ArrayType) field.type()).elementType().
+                                serializationIsDifferentInFlexibleVersions()) {
+                            VersionConditional.forVersions(fieldFlexibleVersions(field),
+                                    presentAndUntaggedVersions).
+                                ifMember(callGenerateVariableLengthReader).
+                                ifNotMember(callGenerateVariableLengthReader).
+                                generate(buffer);
+                        } else {
+                            callGenerateVariableLengthReader.generate(presentAndUntaggedVersions);
+                        }
+                    } else {
+                        buffer.printf("this.%s = %s;%n", field.camelCaseName(),
+                            primitiveReadExpression(field.type()));
+                    }
+                }).
+                generate(buffer);
         }
-        buffer.decrementIndent();
-        buffer.printf("}%n");
-    }
-
-    private void generateFieldReader(FieldSpec field, Versions curVersions) {
-        if (field.type().isArray()) {
-            boolean maybeAbsent =
-                generateVersionCheck(curVersions, field.versions());
-            if (!maybeAbsent) {
-                buffer.printf("{%n");
+        buffer.printf("this._unknownTaggedFields = null;%n");
+        VersionConditional.forVersions(messageFlexibleVersions, curVersions).
+            ifMember(curFlexibleVersions -> {
+                buffer.printf("int _numTaggedFields = _readable.readUnsignedVarint();%n");
+                buffer.printf("for (int _i = 0; _i < _numTaggedFields; _i++) {%n");
                 buffer.incrementIndent();
-            }
-            boolean hasKeys = structRegistry.isStructArrayWithKeys(field);
-            buffer.printf("int arrayLength = readable.readInt();%n");
-            buffer.printf("if (arrayLength < 0) {%n");
-            buffer.incrementIndent();
-            buffer.printf("this.%s = null;%n",
-                field.camelCaseName());
-            buffer.decrementIndent();
-            buffer.printf("} else {%n");
-            buffer.incrementIndent();
-            buffer.printf("this.%s.clear(%s);%n",
-                field.camelCaseName(),
-                hasKeys ? "arrayLength" : "");
-            buffer.printf("for (int i = 0; i < arrayLength; i++) {%n");
-            buffer.incrementIndent();
-            FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type();
-            buffer.printf("this.%s.add(%s);%n",
-                field.camelCaseName(), readFieldFromReadable(arrayType.elementType()));
-            buffer.decrementIndent();
-            buffer.printf("}%n");
-            buffer.decrementIndent();
-            buffer.printf("}%n");
-            if (maybeAbsent) {
-                generateSetDefault(field);
-            } else {
+                buffer.printf("int _tag = _readable.readUnsignedVarint();%n");
+                buffer.printf("int _size = _readable.readUnsignedVarint();%n");
+                buffer.printf("switch (_tag) {%n");
+                buffer.incrementIndent();
+                for (FieldSpec field : struct.fields()) {
+                    Versions validTaggedVersions = field.versions().intersect(field.taggedVersions());
+                    if (!validTaggedVersions.empty()) {
+                        if (!field.tag().isPresent()) {
+                            throw new RuntimeException("Field " + field.name() + " has tagged versions, but no tag.");
+                        }
+                        buffer.printf("case %d: {%n", field.tag().get());
+                        buffer.incrementIndent();
+                        VersionConditional.forVersions(validTaggedVersions, curFlexibleVersions).
+                            ifMember(presentAndTaggedVersions -> {
+                                if (field.type().isVariableLength()) {
+                                    // All tagged fields are serialized using the new-style
+                                    // flexible versions serialization.
+                                    generateVariableLengthReader(fieldFlexibleVersions(field),
+                                        field.camelCaseName(),
+                                        field.type(),
+                                        presentAndTaggedVersions,
+                                        field.nullableVersions(),
+                                        String.format("this.%s = ", field.camelCaseName()),
+                                        String.format(";%n"),
+                                        structRegistry.isStructArrayWithKeys(field));
+                                } else {
+                                    buffer.printf("this.%s = %s;%n", field.camelCaseName(),
+                                        primitiveReadExpression(field.type()));
+                                }
+                                buffer.printf("break;%n");
+                            }).
+                            ifNotMember(__ -> {
+                                buffer.printf("throw new RuntimeException(\"Tag %d is not " +
+                                    "valid for version \" + _version);%n", field.tag().get());
+                            }).
+                            generate(buffer);
+                        buffer.decrementIndent();
+                        buffer.printf("}%n");
+                    }
+                }
+                buffer.printf("default:%n");
+                buffer.incrementIndent();
+                buffer.printf("this._unknownTaggedFields = _readable.readUnknownTaggedField(this._unknownTaggedFields, _tag, _size);%n");
+                buffer.printf("break;%n");
+                buffer.decrementIndent();
                 buffer.decrementIndent();
                 buffer.printf("}%n");
-            }
-        } else {
-            boolean maybeAbsent =
-                generateVersionCheck(curVersions, field.versions());
-            buffer.printf("this.%s = %s;%n",
-                field.camelCaseName(),
-                readFieldFromReadable(field.type()));
-            if (maybeAbsent) {
-                generateSetDefault(field);
-            }
-        }
+                buffer.decrementIndent();
+                buffer.printf("}%n");
+            }).
+            generate(buffer);
+        buffer.decrementIndent();
+        buffer.printf("}%n");
     }
 
-    private String readFieldFromReadable(FieldType type) {
+    private String primitiveReadExpression(FieldType type) {
         if (type instanceof FieldType.BoolFieldType) {
-            return "readable.readByte() != 0";
+            return "_readable.readByte() != 0";
         } else if (type instanceof FieldType.Int8FieldType) {
-            return "readable.readByte()";
+            return "_readable.readByte()";
         } else if (type instanceof FieldType.Int16FieldType) {
-            return "readable.readShort()";
+            return "_readable.readShort()";
         } else if (type instanceof FieldType.Int32FieldType) {
-            return "readable.readInt()";
+            return "_readable.readInt()";
         } else if (type instanceof FieldType.Int64FieldType) {
-            return "readable.readLong()";
+            return "_readable.readLong()";
         } else if (type instanceof FieldType.UUIDFieldType) {
-            return "readable.readUUID()";
-        } else if (type.isString()) {
-            return "readable.readNullableString()";
-        } else if (type.isBytes()) {
-            return "readable.readNullableBytes()";
+            return "_readable.readUUID()";
         } else if (type.isStruct()) {
-            return String.format("new %s(readable, version)", type.toString());
+            return String.format("new %s(_readable, _version)", type.toString());
         } else {
             throw new RuntimeException("Unsupported field type " + type);
         }
     }
 
-    private void generateClassFromStruct(String className, StructSpec struct,
-                                         Versions parentVersions) {
-        headerGenerator.addImport(MessageGenerator.STRUCT_CLASS);
-        buffer.printf("@Override%n");
-        buffer.printf("public void fromStruct(Struct struct, short version) {%n");
+    private void generateVariableLengthReader(Versions fieldFlexibleVersions,
+                                              String name,
+                                              FieldType type,
+                                              Versions possibleVersions,
+                                              Versions nullableVersions,
+                                              String assignmentPrefix,
+                                              String assignmentSuffix,
+                                              boolean isStructArrayWithKeys) {
+        String lengthVar = type.isArray() ? "arrayLength" : "length";
+        buffer.printf("int %s;%n", lengthVar);
+        VersionConditional.forVersions(fieldFlexibleVersions, possibleVersions).
+            ifMember(__ -> {
+                buffer.printf("%s = _readable.readUnsignedVarint() - 1;%n", lengthVar);
+            }).
+            ifNotMember(__ -> {
+                if (type.isString()) {
+                    buffer.printf("%s = _readable.readShort();%n", lengthVar);
+                } else if (type.isBytes() || type.isArray()) {
+                    buffer.printf("%s = _readable.readInt();%n", lengthVar);
+                } else {
+                    throw new RuntimeException("Can't handle variable length type " + type);
+                }
+            }).
+            generate(buffer);
+        buffer.printf("if (%s < 0) {%n", lengthVar);
         buffer.incrementIndent();
-        if (generateInverseVersionCheck(parentVersions, struct.versions())) {
+        VersionConditional.forVersions(nullableVersions, possibleVersions).
+            ifNotMember(__ -> {
+                buffer.printf("throw new RuntimeException(\"non-nullable field %s " +
+                    "was serialized as null\");%n", name);
+            }).
+            ifMember(__ -> {
+                buffer.printf("%snull%s", assignmentPrefix, assignmentSuffix);
+            }).
+            generate(buffer);
+        buffer.decrementIndent();
+        if (type.isString()) {
+            buffer.printf("} else if (%s > 0x7fff) {%n", lengthVar);
             buffer.incrementIndent();
-            headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS);
-            buffer.printf("throw new UnsupportedVersionException(\"Can't read " +
-                "version \" + version + \" of %s\");%n", className);
+            buffer.printf("throw new RuntimeException(\"string field %s " +
+                "had invalid length \" + %s);%n", name, lengthVar);
+            buffer.decrementIndent();
+        }
+        buffer.printf("} else {%n");
+        buffer.incrementIndent();
+        if (type.isString()) {
+            buffer.printf("%s_readable.readString(%s)%s",
+                assignmentPrefix, lengthVar, assignmentSuffix);
+        } else if (type.isBytes()) {
+            buffer.printf("byte[] newBytes = new byte[%s];%n", lengthVar);
+            buffer.printf("_readable.readArray(newBytes);%n");
+            buffer.printf("%snewBytes%s", assignmentPrefix, assignmentSuffix);
+        } else if (type.isArray()) {
+            FieldType.ArrayType arrayType = (FieldType.ArrayType) type;
+            if (isStructArrayWithKeys) {
+                headerGenerator.addImport(MessageGenerator.IMPLICIT_LINKED_HASH_MULTI_COLLECTION_CLASS);
+                buffer.printf("%s newCollection = new %s(%s);%n",
+                    collectionType(arrayType.elementType().toString()),
+                        collectionType(arrayType.elementType().toString()), lengthVar);
+            } else {
+                headerGenerator.addImport(MessageGenerator.ARRAYLIST_CLASS);
+                buffer.printf("ArrayList<%s> newCollection = new ArrayList<%s>(%s);%n",
+                    getBoxedJavaType(arrayType.elementType()),
+                        getBoxedJavaType(arrayType.elementType()), lengthVar);
+            }
+            buffer.printf("for (int i = 0; i < %s; i++) {%n", lengthVar);
+            buffer.incrementIndent();
+            if (arrayType.elementType().isArray()) {
+                throw new RuntimeException("Nested arrays are not supported.  " +
+                    "Use an array of structures containing another array.");
+            } else if (arrayType.elementType().isBytes() || arrayType.elementType().isString()) {
+                generateVariableLengthReader(fieldFlexibleVersions,
+                    name + " element",
+                    arrayType.elementType(),
+                    possibleVersions,
+                    Versions.NONE,
+                    "newCollection.add(",
+                    String.format(");%n"),
+                    false);
+            } else {
+                buffer.printf("newCollection.add(%s);%n",
+                    primitiveReadExpression(arrayType.elementType()));
+            }
             buffer.decrementIndent();
             buffer.printf("}%n");
+            buffer.printf("%snewCollection%s", assignmentPrefix, assignmentSuffix);
+        } else {
+            throw new RuntimeException("Can't handle variable length type " + type);
         }
+        buffer.decrementIndent();
+        buffer.printf("}%n");
+    }
+
+    private void generateClassFromStruct(String className, StructSpec struct,
+                                         Versions parentVersions) {
+        headerGenerator.addImport(MessageGenerator.STRUCT_CLASS);
+        buffer.printf("@SuppressWarnings(\"unchecked\")%n");
+        buffer.printf("@Override%n");
+        buffer.printf("public void fromStruct(Struct struct, short _version) {%n");
+        buffer.incrementIndent();
+        VersionConditional.forVersions(parentVersions, struct.versions()).
+            allowMembershipCheckAlwaysFalse(false).
+            ifNotMember(__ -> {
+                headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS);
+                buffer.printf("throw new UnsupportedVersionException(\"Can't read " +
+                    "version \" + _version + \" of %s\");%n", className);
+            }).
+            generate(buffer);
         Versions curVersions = parentVersions.intersect(struct.versions());
-        if (curVersions.empty()) {
-            throw new RuntimeException("Version ranges " + parentVersions +
-                " and " + struct.versions() + " have no versions in common.");
+        if (!messageFlexibleVersions.intersect(struct.versions()).empty()) {
+            buffer.printf("NavigableMap _taggedFields = null;%n");
         }
+        buffer.printf("this._unknownTaggedFields = null;%n");
+        VersionConditional.forVersions(messageFlexibleVersions, struct.versions()).
+            ifMember(__ -> {
+                headerGenerator.addImport(MessageGenerator.NAVIGABLE_MAP_CLASS);
+                buffer.printf("_taggedFields = (NavigableMap) " +
+                    "struct.get(\"%s\");%n", TAGGED_FIELDS_SECTION_NAME);
+            }).
+            generate(buffer);
         for (FieldSpec field : struct.fields()) {
-            generateFieldFromStruct(field, curVersions);
+            VersionConditional.forVersions(field.versions(), curVersions).
+                alwaysEmitBlockScope(field.type().isArray()).
+                ifNotMember(__ -> {
+                    buffer.printf("this.%s = %s;%n", field.camelCaseName(), fieldDefault(field));
+                }).
+                ifMember(presentVersions -> {
+                    VersionConditional.forVersions(field.taggedVersions(), presentVersions).
+                        ifNotMember(presentAndUntaggedVersions -> {
+                            if (field.type().isArray()) {
+                                buffer.printf("Object[] _nestedObjects = struct.getArray(\"%s\");%n",
+                                    field.snakeCaseName());
+                                generateArrayFromStruct(field, presentAndUntaggedVersions);
+                            } else {
+                                buffer.printf("this.%s = %s;%n",
+                                    field.camelCaseName(),
+                                    readFieldFromStruct(field.type(), field.snakeCaseName()));
+                            }
+                        }).
+                        ifMember(presentAndTaggedVersions -> {
+                            buffer.printf("if (_taggedFields.containsKey(%d)) {%n", field.tag().get());
+                            buffer.incrementIndent();
+                            if (field.type().isArray()) {
+                                buffer.printf("Object[] _nestedObjects = " +
+                                    "(Object[]) _taggedFields.remove(%d);%n", field.tag().get());
+                                generateArrayFromStruct(field, presentAndTaggedVersions);
+                            } else if (field.type().isBytes()) {
+                                headerGenerator.addImport(MessageGenerator.BYTE_BUFFER_CLASS);
+                                headerGenerator.addImport(MessageGenerator.MESSAGE_UTIL_CLASS);
+                                buffer.printf("this.%s = MessageUtil.byteBufferToArray(" +
+                                    "(ByteBuffer) _taggedFields.remove(%d));%n",
+                                    field.camelCaseName(),
+                                    field.tag().get());
+                            } else {
+                                buffer.printf("this.%s = (%s) _taggedFields.remove(%d);%n",
+                                    field.camelCaseName(),
+                                    getBoxedJavaType(field.type()),
+                                    field.tag().get());
+                            }
+                            buffer.decrementIndent();
+                            buffer.printf("} else {%n");
+                            buffer.incrementIndent();
+                            buffer.printf("this.%s = %s;%n", field.camelCaseName(), fieldDefault(field));
+                            buffer.decrementIndent();
+                            buffer.printf("}%n");
+                        }).
+                        generate(buffer);
+                }).
+                generate(buffer);
         }
+        VersionConditional.forVersions(messageFlexibleVersions, struct.versions()).
+            ifMember(__ -> {
+                headerGenerator.addImport(MessageGenerator.NAVIGABLE_MAP_CLASS);
+                buffer.printf("if (!_taggedFields.isEmpty()) {%n");
+                buffer.incrementIndent();
+                headerGenerator.addImport(MessageGenerator.ARRAYLIST_CLASS);
+                buffer.printf("this._unknownTaggedFields = new ArrayList<>(_taggedFields.size());%n");
+                headerGenerator.addStaticImport(MessageGenerator.MAP_ENTRY_CLASS);
+                buffer.printf("for (Entry entry : _taggedFields.entrySet()) {%n");
+                buffer.incrementIndent();
+                headerGenerator.addImport(MessageGenerator.RAW_TAGGED_FIELD_CLASS);
+                buffer.printf("this._unknownTaggedFields.add((RawTaggedField) entry.getValue());%n");
+                buffer.decrementIndent();
+                buffer.printf("}%n");
+                buffer.decrementIndent();
+                buffer.printf("}%n");
+            }).
+            generate(buffer);
         buffer.decrementIndent();
         buffer.printf("}%n");
     }
 
-    private void generateFieldFromStruct(FieldSpec field, Versions curVersions) {
-        if (field.type().isArray()) {
-            boolean maybeAbsent =
-                generateVersionCheck(curVersions, field.versions());
-            if (!maybeAbsent) {
-                buffer.printf("{%n");
-                buffer.incrementIndent();
-            }
-            headerGenerator.addImport(MessageGenerator.STRUCT_CLASS);
-            buffer.printf("Object[] nestedObjects = struct.getArray(\"%s\");%n",
-                field.snakeCaseName());
-            boolean maybeNull = false;
-            if (!curVersions.intersect(field.nullableVersions()).empty()) {
-                maybeNull = true;
-                buffer.printf("if (nestedObjects == null) {%n", field.camelCaseName());
-                buffer.incrementIndent();
+
+    private void generateArrayFromStruct(FieldSpec field, Versions versions) {
+        IsNullConditional.forName("_nestedObjects").
+            possibleVersions(versions).
+            nullableVersions(field.nullableVersions()).
+            ifNull(() -> {
                 buffer.printf("this.%s = null;%n", field.camelCaseName());
-                buffer.decrementIndent();
-                buffer.printf("} else {%n");
+            }).
+            ifNotNull(() -> {
+                FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type();
+                FieldType elementType = arrayType.elementType();
+                buffer.printf("this.%s = new %s(_nestedObjects.length);%n",
+                    field.camelCaseName(), fieldConcreteJavaType(field));
+                buffer.printf("for (Object nestedObject : _nestedObjects) {%n");
                 buffer.incrementIndent();
-            }
-            FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type();
-            FieldType elementType = arrayType.elementType();
-            buffer.printf("this.%s = new %s(nestedObjects.length);%n",
-                field.camelCaseName(), fieldConcreteJavaType(field));
-            buffer.printf("for (Object nestedObject : nestedObjects) {%n");
-            buffer.incrementIndent();
-            if (elementType.isStruct()) {
-                buffer.printf("this.%s.add(new %s((Struct) nestedObject, version));%n",
-                    field.camelCaseName(), elementType.toString());
-            } else {
-                buffer.printf("this.%s.add((%s) nestedObject);%n",
-                    field.camelCaseName(), getBoxedJavaType(elementType));
-            }
-            buffer.decrementIndent();
-            buffer.printf("}%n");
-            if (maybeNull) {
-                buffer.decrementIndent();
-                buffer.printf("}%n");
-            }
-            if (maybeAbsent) {
-                generateSetDefault(field);
-            } else {
+                if (elementType.isStruct()) {
+                    headerGenerator.addImport(MessageGenerator.STRUCT_CLASS);
+                    buffer.printf("this.%s.add(new %s((Struct) nestedObject, _version));%n",
+                        field.camelCaseName(), elementType.toString());
+                } else {
+                    buffer.printf("this.%s.add((%s) nestedObject);%n",
+                        field.camelCaseName(), getBoxedJavaType(elementType));
+                }
                 buffer.decrementIndent();
                 buffer.printf("}%n");
-            }
-        } else {
-            boolean maybeAbsent =
-                generateVersionCheck(curVersions, field.versions());
-            buffer.printf("this.%s = %s;%n",
-                field.camelCaseName(),
-                readFieldFromStruct(field.type(), field.snakeCaseName()));
-            if (maybeAbsent) {
-                generateSetDefault(field);
-            }
-        }
+            }).
+            generate(buffer);
     }
 
     private String getBoxedJavaType(FieldType type) {
@@ -587,6 +791,7 @@ private String getBoxedJavaType(FieldType type) {
         } else if (type instanceof FieldType.Int64FieldType) {
             return "Long";
         } else if (type instanceof FieldType.UUIDFieldType) {
+            headerGenerator.addImport(MessageGenerator.UUID_CLASS);
             return "UUID";
         } else if (type.isString()) {
             return "String";
@@ -615,7 +820,7 @@ private String readFieldFromStruct(FieldType type, String name) {
         } else if (type.isBytes()) {
             return String.format("struct.getByteArray(\"%s\")", name);
         } else if (type.isStruct()) {
-            return String.format("new %s(struct, version)", type.toString());
+            return String.format("new %s(struct, _version)", type.toString());
         } else {
             throw new RuntimeException("Unsupported field type " + type);
         }
@@ -624,101 +829,301 @@ private String readFieldFromStruct(FieldType type, String name) {
     private void generateClassWriter(String className, StructSpec struct,
             Versions parentVersions) {
         headerGenerator.addImport(MessageGenerator.WRITABLE_CLASS);
+        headerGenerator.addImport(MessageGenerator.OBJECT_SERIALIZATION_CACHE_CLASS);
         buffer.printf("@Override%n");
-        buffer.printf("public void write(Writable writable, short version) {%n");
+        buffer.printf("public void write(Writable _writable, ObjectSerializationCache _cache, short _version) {%n");
         buffer.incrementIndent();
-        if (generateInverseVersionCheck(parentVersions, struct.versions())) {
-            buffer.incrementIndent();
-            headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS);
-            buffer.printf("throw new UnsupportedVersionException(\"Can't write " +
-                "version \" + version + \" of %s\");%n", className);
-            buffer.decrementIndent();
-            buffer.printf("}%n");
-        }
+        VersionConditional.forVersions(parentVersions, struct.versions()).
+            allowMembershipCheckAlwaysFalse(false).
+            ifNotMember(__ -> {
+                headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS);
+                buffer.printf("throw new UnsupportedVersionException(\"Can't write " +
+                    "version \" + _version + \" of %s\");%n", className);
+            }).
+            generate(buffer);
+        buffer.printf("int _numTaggedFields = 0;%n");
         Versions curVersions = parentVersions.intersect(struct.versions());
-        if (curVersions.empty()) {
-            throw new RuntimeException("Version ranges " + parentVersions +
-                " and " + struct.versions() + " have no versions in common.");
-        }
+        TreeMap taggedFields = new TreeMap<>();
         for (FieldSpec field : struct.fields()) {
-            generateFieldWriter(field, curVersions);
+            VersionConditional cond = VersionConditional.forVersions(field.versions(), curVersions).
+                ifMember(presentVersions -> {
+                    VersionConditional.forVersions(field.taggedVersions(), presentVersions).
+                        ifNotMember(presentAndUntaggedVersions -> {
+                            if (field.type().isVariableLength()) {
+                                ClauseGenerator callGenerateVariableLengthWriter = versions -> {
+                                    generateVariableLengthWriter(fieldFlexibleVersions(field),
+                                        field.camelCaseName(),
+                                        field.type(),
+                                        versions,
+                                        field.nullableVersions());
+                                };
+                                // For arrays where the field type needs to be serialized differently in flexible
+                                // versions, lift the flexible version check outside of the array.
+                                // This may mean generating two separate 'for' loops-- one for flexible
+                                // versions, and one for regular versions.
+                                if (field.type().isArray() &&
+                                    ((FieldType.ArrayType) field.type()).elementType().
+                                        serializationIsDifferentInFlexibleVersions()) {
+                                    VersionConditional.forVersions(fieldFlexibleVersions(field),
+                                            presentAndUntaggedVersions).
+                                        ifMember(callGenerateVariableLengthWriter).
+                                        ifNotMember(callGenerateVariableLengthWriter).
+                                        generate(buffer);
+                                } else {
+                                    callGenerateVariableLengthWriter.generate(presentAndUntaggedVersions);
+                                }
+                            } else {
+                                buffer.printf("%s;%n",
+                                    primitiveWriteExpression(field.type(), field.camelCaseName()));
+                            }
+                        }).
+                        ifMember(__ -> {
+                            generateNonDefaultValueCheck(field);
+                            buffer.incrementIndent();
+                            buffer.printf("_numTaggedFields++;%n");
+                            buffer.decrementIndent();
+                            buffer.printf("}%n");
+                            if (taggedFields.put(field.tag().get(), field) != null) {
+                                throw new RuntimeException("Field " + field.name() + " has tag " +
+                                    field.tag() + ", but another field already used that tag.");
+                            }
+                        }).
+                        generate(buffer);
+                });
+            if (!field.ignorable()) {
+                cond.ifNotMember(__ -> {
+                    generateNonDefaultValueCheck(field);
+                    buffer.incrementIndent();
+                    headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS);
+                    buffer.printf("throw new UnsupportedVersionException(" +
+                            "\"Attempted to write a non-default %s at version \" + _version);%n",
+                        field.camelCaseName());
+                    buffer.decrementIndent();
+                    buffer.printf("}%n");
+                });
+            }
+            cond.generate(buffer);
         }
+        headerGenerator.addImport(MessageGenerator.RAW_TAGGED_FIELD_WRITER_CLASS);
+        buffer.printf("RawTaggedFieldWriter _rawWriter = RawTaggedFieldWriter.forFields(_unknownTaggedFields);%n");
+        buffer.printf("_numTaggedFields += _rawWriter.numFields();%n");
+        VersionConditional.forVersions(messageFlexibleVersions, curVersions).
+            ifNotMember(__ -> {
+                generateCheckForUnsupportedNumTaggedFields("_numTaggedFields > 0");
+            }).
+            ifMember(__ -> {
+                buffer.printf("_writable.writeUnsignedVarint(_numTaggedFields);%n");
+                int prevTag = -1;
+                for (FieldSpec field : taggedFields.values()) {
+                    if (prevTag + 1 != field.tag().get()) {
+                        buffer.printf("_rawWriter.writeRawTags(_writable, %d);%n", field.tag().get());
+                    }
+                    VersionConditional.
+                        forVersions(field.versions(), field.taggedVersions().intersect(field.versions())).
+                        allowMembershipCheckAlwaysFalse(false).
+                        ifMember(presentAndTaggedVersions -> {
+                            generateNonDefaultValueCheck(field);
+                            buffer.incrementIndent();
+                            buffer.printf("_writable.writeUnsignedVarint(%d);%n", field.tag().get());
+                            if (field.type().isString()) {
+                                IsNullConditional.forName(field.camelCaseName()).
+                                    nullableVersions(field.nullableVersions()).
+                                    possibleVersions(presentAndTaggedVersions).
+                                    ifNull(() -> {
+                                        buffer.printf("_writable.writeUnsignedVarint(0);%n");
+                                    }).
+                                    ifNotNull(() -> {
+                                        buffer.printf("byte[] _stringBytes = _cache.getSerializedValue(this.%s);%n",
+                                            field.camelCaseName());
+                                        buffer.printf("_writable.writeUnsignedVarint(_stringBytes.length);%n");
+                                        buffer.printf("_writable.writeByteArray(_stringBytes);%n");
+                                    }).
+                                    generate(buffer);
+                            } else if (field.type().isVariableLength()) {
+                                if (field.type().isArray()) {
+                                    buffer.printf("_writable.writeUnsignedVarint(_cache.getArraySizeInBytes(this.%s) + 1);%n",
+                                        field.camelCaseName());
+                                } else if (field.type().isBytes()) {
+                                    buffer.printf("_writable.writeUnsignedVarint(this.%s.length + 1);%n",
+                                        field.camelCaseName());
+                                } else {
+                                    throw new RuntimeException("Unable to handle type " + field.type());
+                                }
+                                generateVariableLengthWriter(fieldFlexibleVersions(field),
+                                    field.camelCaseName(),
+                                    field.type(),
+                                    presentAndTaggedVersions,
+                                    field.nullableVersions());
+                            } else {
+                                buffer.printf("_writable.writeUnsignedVarint(%d);%n",
+                                    field.type().fixedLength().get());
+                                buffer.printf("%s;%n",
+                                    primitiveWriteExpression(field.type(), field.camelCaseName()));
+                            }
+                            buffer.decrementIndent();
+                            buffer.printf("}%n");
+                        }).
+                        generate(buffer);
+                    prevTag = field.tag().get();
+                }
+                if (prevTag < Integer.MAX_VALUE) {
+                    buffer.printf("_rawWriter.writeRawTags(_writable, Integer.MAX_VALUE);%n");
+                }
+            }).
+            generate(buffer);
         buffer.decrementIndent();
         buffer.printf("}%n");
     }
 
-    private String writeFieldToWritable(FieldType type, boolean nullable, String name) {
+    private void generateCheckForUnsupportedNumTaggedFields(String conditional) {
+        buffer.printf("if (%s) {%n", conditional);
+        buffer.incrementIndent();
+        headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS);
+        buffer.printf("throw new UnsupportedVersionException(\"Tagged fields were set, " +
+            "but version \" + _version + \" of this message does not support them.\");%n");
+        buffer.decrementIndent();
+        buffer.printf("}%n");
+    }
+
+    private String primitiveWriteExpression(FieldType type, String name) {
         if (type instanceof FieldType.BoolFieldType) {
-            return String.format("writable.writeByte(%s ? (byte) 1 : (byte) 0)", name);
+            return String.format("_writable.writeByte(%s ? (byte) 1 : (byte) 0)", name);
         } else if (type instanceof FieldType.Int8FieldType) {
-            return String.format("writable.writeByte(%s)", name);
+            return String.format("_writable.writeByte(%s)", name);
         } else if (type instanceof FieldType.Int16FieldType) {
-            return String.format("writable.writeShort(%s)", name);
+            return String.format("_writable.writeShort(%s)", name);
         } else if (type instanceof FieldType.Int32FieldType) {
-            return String.format("writable.writeInt(%s)", name);
+            return String.format("_writable.writeInt(%s)", name);
         } else if (type instanceof FieldType.Int64FieldType) {
-            return String.format("writable.writeLong(%s)", name);
+            return String.format("_writable.writeLong(%s)", name);
         } else if (type instanceof FieldType.UUIDFieldType) {
-            return String.format("writable.writeUUID(%s)", name);
-        } else if (type instanceof FieldType.StringFieldType) {
-            if (nullable) {
-                return String.format("writable.writeNullableString(%s)", name);
-            } else {
-                return String.format("writable.writeString(%s)", name);
-            }
-        } else if (type instanceof FieldType.BytesFieldType) {
-            if (nullable) {
-                return String.format("writable.writeNullableBytes(%s)", name);
-            } else {
-                return String.format("writable.writeBytes(%s)", name);
-            }
+            return String.format("_writable.writeUUID(%s)", name);
         } else if (type instanceof FieldType.StructType) {
-            return String.format("%s.write(writable, version)", name);
+            return String.format("%s.write(_writable, _cache, _version)", name);
         } else {
             throw new RuntimeException("Unsupported field type " + type);
         }
     }
 
-    private void generateFieldWriter(FieldSpec field, Versions curVersions) {
+    private void generateVariableLengthWriter(Versions fieldFlexibleVersions,
+                                              String name,
+                                              FieldType type,
+                                              Versions possibleVersions,
+                                              Versions nullableVersions) {
+        IsNullConditional.forName(name).
+            possibleVersions(possibleVersions).
+            nullableVersions(nullableVersions).
+            alwaysEmitBlockScope(type.isString()).
+            ifNull(() -> {
+                VersionConditional.forVersions(nullableVersions, possibleVersions).
+                    ifMember(__ -> {
+                        VersionConditional.forVersions(fieldFlexibleVersions, possibleVersions).
+                            ifMember(___ -> {
+                                buffer.printf("_writable.writeUnsignedVarint(0);%n");
+                            }).
+                            ifNotMember(___ -> {
+                                if (type.isString()) {
+                                    buffer.printf("_writable.writeShort((short) -1);%n");
+                                } else {
+                                    buffer.printf("_writable.writeInt(-1);%n");
+                                }
+                            }).
+                            generate(buffer);
+                    }).
+                    ifNotMember(__ -> {
+                        buffer.printf("throw new NullPointerException();%n");
+                    }).
+                    generate(buffer);
+            }).
+            ifNotNull(() -> {
+                final String lengthExpression;
+                if (type.isString()) {
+                    buffer.printf("byte[] _stringBytes = _cache.getSerializedValue(%s);%n",
+                        name);
+                    lengthExpression = "_stringBytes.length";
+                } else if (type.isBytes()) {
+                    lengthExpression = String.format("%s.length", name);
+                } else if (type.isArray()) {
+                    lengthExpression = String.format("%s.size()", name);
+                } else {
+                    throw new RuntimeException("Unhandled type " + type);
+                }
+                // Check whether we're dealing with a flexible version or not.  In a flexible
+                // version, the length is serialized differently.
+                //
+                // Note: for arrays, each branch of the if contains the loop for writing out
+                // the elements.  This allows us to lift the version check out of the loop.
+                // This is helpful for things like arrays of strings, where each element
+                // will be serialized differently based on whether the version is flexible.
+                VersionConditional.forVersions(fieldFlexibleVersions, possibleVersions).
+                    ifMember(ifMemberVersions -> {
+                        buffer.printf("_writable.writeUnsignedVarint(%s + 1);%n", lengthExpression);
+                    }).
+                    ifNotMember(ifNotMemberVersions -> {
+                        if (type.isString()) {
+                            buffer.printf("_writable.writeShort((short) %s);%n", lengthExpression);
+                        } else {
+                            buffer.printf("_writable.writeInt(%s);%n", lengthExpression);
+                        }
+                    }).
+                    generate(buffer);
+                if (type.isString()) {
+                    buffer.printf("_writable.writeByteArray(_stringBytes);%n");
+                } else if (type.isBytes()) {
+                    buffer.printf("_writable.writeByteArray(%s);%n", name);
+                } else if (type.isArray()) {
+                    FieldType.ArrayType arrayType = (FieldType.ArrayType) type;
+                    FieldType elementType = arrayType.elementType();
+                    String elementName = String.format("%sElement", name);
+                    buffer.printf("for (%s %s : %s) {%n",
+                        getBoxedJavaType(elementType),
+                        elementName,
+                        name);
+                    buffer.incrementIndent();
+                    if (elementType.isArray()) {
+                        throw new RuntimeException("Nested arrays are not supported.  " +
+                            "Use an array of structures containing another array.");
+                    } else if (elementType.isBytes() || elementType.isString()) {
+                        generateVariableLengthWriter(fieldFlexibleVersions,
+                            elementName,
+                            elementType,
+                            possibleVersions,
+                            Versions.NONE);
+                    } else {
+                        buffer.printf("%s;%n", primitiveWriteExpression(elementType, elementName));
+                    }
+                    buffer.decrementIndent();
+                    buffer.printf("}%n");
+                }
+            }).
+            generate(buffer);
+    }
+
+    private void generateNonDefaultValueCheck(FieldSpec field) {
         if (field.type().isArray()) {
-            boolean maybeAbsent =
-                generateVersionCheck(curVersions, field.versions());
-            boolean maybeNull = generateNullCheck(curVersions, field);
-            if (maybeNull) {
-                buffer.printf("writable.writeInt(-1);%n");
-                buffer.decrementIndent();
-                buffer.printf("} else {%n");
-                buffer.incrementIndent();
+            if (fieldDefault(field).equals("null")) {
+                buffer.printf("if (%s != null) {%n", field.camelCaseName());
+            } else {
+                buffer.printf("if (!%s.isEmpty()) {%n", field.camelCaseName());
             }
-            buffer.printf("writable.writeInt(%s.size());%n", field.camelCaseName());
-            FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type();
-            FieldType elementType = arrayType.elementType();
-            String nestedTypeName = elementType.isStruct() ?
-                elementType.toString() : getBoxedJavaType(elementType);
-            buffer.printf("for (%s element : %s) {%n",
-                nestedTypeName, field.camelCaseName());
-            buffer.incrementIndent();
-            buffer.printf("%s;%n", writeFieldToWritable(elementType, false, "element"));
-            buffer.decrementIndent();
-            buffer.printf("}%n");
-            if (maybeNull) {
-                buffer.decrementIndent();
-                buffer.printf("}%n");
+        } else if (field.type().isBytes()) {
+            if (fieldDefault(field).equals("null")) {
+                buffer.printf("if (%s != null) {%n", field.camelCaseName());
+            } else {
+                buffer.printf("if (%s.length != 0) {%n", field.camelCaseName());
             }
-            if (maybeAbsent) {
-                buffer.decrementIndent();
-                buffer.printf("}%n");
+        } else if (field.type().isString()) {
+            if (fieldDefault(field).equals("null")) {
+                buffer.printf("if (%s != null) {%n", field.camelCaseName());
+            } else {
+                buffer.printf("if (!%s.equals(%s)) {%n", field.camelCaseName(), fieldDefault(field));
             }
+        } else if (field.type() instanceof FieldType.BoolFieldType) {
+            buffer.printf("if (%s%s) {%n",
+                fieldDefault(field).equals("true") ? "!" : "",
+                field.camelCaseName());
         } else {
-            boolean maybeAbsent =
-                generateVersionCheck(curVersions, field.versions());
-            buffer.printf("%s;%n", writeFieldToWritable(field.type(),
-                !field.nullableVersions().empty(),
-                field.camelCaseName()));
-            if (maybeAbsent) {
-                buffer.decrementIndent();
-                buffer.printf("}%n");
-            }
+            buffer.printf("if (%s != %s) {%n", field.camelCaseName(), fieldDefault(field));
         }
     }
 
@@ -726,31 +1131,55 @@ private void generateClassToStruct(String className, StructSpec struct,
                                        Versions parentVersions) {
         headerGenerator.addImport(MessageGenerator.STRUCT_CLASS);
         buffer.printf("@Override%n");
-        buffer.printf("public Struct toStruct(short version) {%n");
+        buffer.printf("public Struct toStruct(short _version) {%n");
         buffer.incrementIndent();
-        if (generateInverseVersionCheck(parentVersions, struct.versions())) {
-            buffer.incrementIndent();
-            headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS);
-            buffer.printf("throw new UnsupportedVersionException(\"Can't write " +
-                "version \" + version + \" of %s\");%n", className);
-            buffer.decrementIndent();
-            buffer.printf("}%n");
-        }
+        VersionConditional.forVersions(parentVersions, struct.versions()).
+            allowMembershipCheckAlwaysFalse(false).
+            ifNotMember(__ -> {
+                headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS);
+                buffer.printf("throw new UnsupportedVersionException(\"Can't write " +
+                    "version \" + _version + \" of %s\");%n", className);
+            }).
+            generate(buffer);
         Versions curVersions = parentVersions.intersect(struct.versions());
-        if (curVersions.empty()) {
-            throw new RuntimeException("Version ranges " + parentVersions +
-                " and " + struct.versions() + " have no versions in common.");
-        }
-        buffer.printf("Struct struct = new Struct(SCHEMAS[version]);%n");
+        headerGenerator.addImport(MessageGenerator.TREE_MAP_CLASS);
+        buffer.printf("TreeMap _taggedFields = null;%n");
+        VersionConditional.forVersions(messageFlexibleVersions, struct.versions()).
+            ifMember(__ -> {
+                buffer.printf("_taggedFields = new TreeMap<>();%n");
+            }).
+            generate(buffer);
+        buffer.printf("Struct struct = new Struct(SCHEMAS[_version]);%n");
         for (FieldSpec field : struct.fields()) {
-            generateFieldToStruct(field, curVersions);
+            VersionConditional.forVersions(field.versions(), curVersions).
+                alwaysEmitBlockScope(field.type().isArray()).
+                ifMember(presentVersions -> {
+                    VersionConditional.forVersions(field.taggedVersions(), presentVersions).
+                        ifNotMember(presentAndUntaggedVersions -> {
+                            generateFieldToStruct(field, presentAndUntaggedVersions);
+                        }).
+                        ifMember(presentAndTaggedVersions -> {
+                            generateNonDefaultValueCheck(field);
+                            buffer.incrementIndent();
+                            generateTaggedFieldToMap(field, presentAndTaggedVersions);
+                            buffer.decrementIndent();
+                            buffer.printf("}%n");
+                        }).
+                        generate(buffer);
+                }).
+                generate(buffer);
         }
+        VersionConditional.forVersions(messageFlexibleVersions, curVersions).
+            ifMember(__ -> {
+                buffer.printf("struct.set(\"%s\", _taggedFields);%n", TAGGED_FIELDS_SECTION_NAME);
+            }).
+            generate(buffer);
         buffer.printf("return struct;%n");
         buffer.decrementIndent();
         buffer.printf("}%n");
     }
 
-    private void generateFieldToStruct(FieldSpec field, Versions curVersions) {
+    private void generateFieldToStruct(FieldSpec field, Versions versions) {
         if ((!field.type().canBeNullable()) &&
             (!field.nullableVersions().empty())) {
             throw new RuntimeException("Fields of type " + field.type() +
@@ -763,224 +1192,325 @@ private void generateFieldToStruct(FieldSpec field, Versions curVersions) {
                 (field.type() instanceof FieldType.Int64FieldType) ||
                 (field.type() instanceof FieldType.UUIDFieldType) ||
                 (field.type() instanceof FieldType.StringFieldType)) {
-            boolean maybeAbsent =
-                generateVersionCheck(curVersions, field.versions());
             buffer.printf("struct.set(\"%s\", this.%s);%n",
                 field.snakeCaseName(), field.camelCaseName());
-            if (maybeAbsent) {
-                buffer.decrementIndent();
-                buffer.printf("}%n");
-            }
         } else if (field.type().isBytes()) {
-            boolean maybeAbsent =
-                generateVersionCheck(curVersions, field.versions());
             buffer.printf("struct.setByteArray(\"%s\", this.%s);%n",
                 field.snakeCaseName(), field.camelCaseName());
-            if (maybeAbsent) {
-                buffer.decrementIndent();
-                buffer.printf("}%n");
-            }
         } else if (field.type().isArray()) {
-            boolean maybeAbsent =
-                generateVersionCheck(curVersions, field.versions());
-            if (!maybeAbsent) {
-                buffer.printf("{%n");
-                buffer.incrementIndent();
-            }
-            boolean maybeNull = generateNullCheck(curVersions, field);
-            if (maybeNull) {
-                buffer.printf("struct.set(\"%s\", null);%n", field.snakeCaseName());
-                buffer.decrementIndent();
-                buffer.printf("} else {%n");
-                buffer.incrementIndent();
-            }
-            FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type();
-            FieldType elementType = arrayType.elementType();
-            String boxdElementType = elementType.isStruct() ? "Struct" : getBoxedJavaType(elementType);
-            buffer.printf("%s[] nestedObjects = new %s[%s.size()];%n",
-                boxdElementType, boxdElementType, field.camelCaseName());
-            buffer.printf("int i = 0;%n");
-            buffer.printf("for (%s element : this.%s) {%n",
-                getBoxedJavaType(arrayType.elementType()), field.camelCaseName());
-            buffer.incrementIndent();
-            if (elementType.isStruct()) {
-                buffer.printf("nestedObjects[i++] = element.toStruct(version);%n");
+            IsNullConditional.forField(field).
+                possibleVersions(versions).
+                ifNull(() -> {
+                    buffer.printf("struct.set(\"%s\", null);%n", field.snakeCaseName());
+                }).
+                ifNotNull(() -> {
+                    generateFieldToObjectArray(field);
+                    buffer.printf("struct.set(\"%s\", (Object[]) _nestedObjects);%n",
+                        field.snakeCaseName());
+                }).generate(buffer);
+        } else {
+            throw new RuntimeException("Unsupported field type " + field.type());
+        }
+    }
+
+    private void generateTaggedFieldToMap(FieldSpec field, Versions versions) {
+        if ((!field.type().canBeNullable()) &&
+            (!field.nullableVersions().empty())) {
+            throw new RuntimeException("Fields of type " + field.type() +
+                " cannot be nullable.");
+        }
+        if ((field.type() instanceof FieldType.BoolFieldType) ||
+            (field.type() instanceof FieldType.Int8FieldType) ||
+            (field.type() instanceof FieldType.Int16FieldType) ||
+            (field.type() instanceof FieldType.Int32FieldType) ||
+            (field.type() instanceof FieldType.Int64FieldType) ||
+            (field.type() instanceof FieldType.UUIDFieldType) ||
+            (field.type() instanceof FieldType.StringFieldType)) {
+            buffer.printf("_taggedFields.put(%d, %s);%n",
+                field.tag().get(), field.camelCaseName());
+        } else if (field.type().isBytes()) {
+            headerGenerator.addImport(MessageGenerator.BYTE_BUFFER_CLASS);
+            if (field.taggedVersions().intersect(field.nullableVersions()).empty()) {
+                buffer.printf("_taggedFields.put(%d, ByteBuffer.wrap(%s));%n",
+                    field.tag().get(), field.camelCaseName());
             } else {
-                buffer.printf("nestedObjects[i++] = element;%n");
-            }
-            buffer.decrementIndent();
-            buffer.printf("}%n");
-            buffer.printf("struct.set(\"%s\", (Object[]) nestedObjects);%n",
-                field.snakeCaseName());
-            if (maybeNull) {
-                buffer.decrementIndent();
-                buffer.printf("}%n");
+                buffer.printf("_taggedFields.put(%d, (%s == null) ? null : ByteBuffer.wrap(%s));%n",
+                    field.tag().get(), field.camelCaseName());
             }
-            buffer.decrementIndent();
-            buffer.printf("}%n");
+        } else if (field.type().isArray()) {
+            IsNullConditional.forField(field).
+                possibleVersions(versions).
+                ifNull(() -> {
+                    buffer.printf("_taggedFields.put(%d, null);%n", field.tag().get());
+                }).
+                ifNotNull(() -> {
+                    generateFieldToObjectArray(field);
+                    buffer.printf("_taggedFields.put(%d, _nestedObjects);%n", field.tag().get());
+                }).generate(buffer);
         } else {
             throw new RuntimeException("Unsupported field type " + field.type());
         }
     }
 
+    private void generateFieldToObjectArray(FieldSpec field) {
+        FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type();
+        FieldType elementType = arrayType.elementType();
+        String boxdElementType = elementType.isStruct() ? "Struct" : getBoxedJavaType(elementType);
+        buffer.printf("%s[] _nestedObjects = new %s[%s.size()];%n",
+            boxdElementType, boxdElementType, field.camelCaseName());
+        buffer.printf("int i = 0;%n");
+        buffer.printf("for (%s element : this.%s) {%n",
+            getBoxedJavaType(arrayType.elementType()), field.camelCaseName());
+        buffer.incrementIndent();
+        if (elementType.isStruct()) {
+            buffer.printf("_nestedObjects[i++] = element.toStruct(_version);%n");
+        } else {
+            buffer.printf("_nestedObjects[i++] = element;%n");
+        }
+        buffer.decrementIndent();
+        buffer.printf("}%n");
+    }
+
     private void generateClassSize(String className, StructSpec struct,
                                    Versions parentVersions) {
+        headerGenerator.addImport(MessageGenerator.OBJECT_SERIALIZATION_CACHE_CLASS);
         buffer.printf("@Override%n");
-        buffer.printf("public int size(short version) {%n");
+        buffer.printf("public int size(ObjectSerializationCache _cache, short _version) {%n");
         buffer.incrementIndent();
-        buffer.printf("int size = 0;%n");
-        if (generateInverseVersionCheck(parentVersions, struct.versions())) {
-            buffer.incrementIndent();
-            headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS);
-            buffer.printf("throw new UnsupportedVersionException(\"Can't size " +
-                "version \" + version + \" of %s\");%n", className);
-            buffer.decrementIndent();
-            buffer.printf("}%n");
-        }
+        buffer.printf("int _size = 0, _numTaggedFields = 0;%n");
+        VersionConditional.forVersions(parentVersions, struct.versions()).
+            allowMembershipCheckAlwaysFalse(false).
+            ifNotMember(__ -> {
+                headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS);
+                buffer.printf("throw new UnsupportedVersionException(\"Can't size " +
+                    "version \" + _version + \" of %s\");%n", className);
+            }).
+            generate(buffer);
         Versions curVersions = parentVersions.intersect(struct.versions());
-        if (curVersions.empty()) {
-            throw new RuntimeException("Version ranges " + parentVersions +
-                " and " + struct.versions() + " have no versions in common.");
-        }
         for (FieldSpec field : struct.fields()) {
-            generateFieldSize(field, curVersions);
+            VersionConditional.forVersions(field.versions(), curVersions).
+                ifMember(presentVersions -> {
+                    VersionConditional.forVersions(field.taggedVersions(), presentVersions).
+                        ifMember(presentAndTaggedVersions -> {
+                            generateNonDefaultValueCheck(field);
+                            buffer.incrementIndent();
+                            buffer.printf("_numTaggedFields++;%n");
+                            buffer.printf("_size += %d;%n",
+                                MessageGenerator.sizeOfUnsignedVarint(field.tag().get()));
+                            generateFieldSize(field, presentAndTaggedVersions, true);
+                            buffer.decrementIndent();
+                            buffer.printf("}%n");
+                        }).
+                        ifNotMember(presentAndUntaggedVersions -> {
+                            generateFieldSize(field, presentAndUntaggedVersions, false);
+                        }).
+                        generate(buffer);
+                }).generate(buffer);
         }
-        buffer.printf("return size;%n");
+        buffer.printf("if (_unknownTaggedFields != null) {%n");
+        buffer.incrementIndent();
+        buffer.printf("_numTaggedFields += _unknownTaggedFields.size();%n");
+        buffer.printf("for (RawTaggedField _field : _unknownTaggedFields) {%n");
+        buffer.incrementIndent();
+        headerGenerator.addImport(MessageGenerator.BYTE_UTILS_CLASS);
+        buffer.printf("_size += ByteUtils.sizeOfUnsignedVarint(_field.tag());%n");
+        buffer.printf("_size += ByteUtils.sizeOfUnsignedVarint(_field.size());%n");
+        buffer.printf("_size += _field.size();%n");
+        buffer.decrementIndent();
+        buffer.printf("}%n");
+        buffer.decrementIndent();
+        buffer.printf("}%n");
+        VersionConditional.forVersions(messageFlexibleVersions, curVersions).
+            ifNotMember(__ -> {
+                generateCheckForUnsupportedNumTaggedFields("_numTaggedFields > 0");
+            }).
+            ifMember(__ -> {
+                headerGenerator.addImport(MessageGenerator.BYTE_UTILS_CLASS);
+                buffer.printf("_size += ByteUtils.sizeOfUnsignedVarint(_numTaggedFields);%n");
+            }).
+            generate(buffer);
+        buffer.printf("return _size;%n");
         buffer.decrementIndent();
         buffer.printf("}%n");
     }
 
-    private void generateVariableLengthFieldSize(String fieldName, FieldType type, boolean nullable) {
+    /**
+     * Generate the size calculator for a variable-length array element.
+     * Array elements cannot be null.
+     */
+    private void generateVariableLengthArrayElementSize(Versions flexibleVersions,
+                                                        String fieldName,
+                                                        FieldType type,
+                                                        Versions versions) {
         if (type instanceof FieldType.StringFieldType) {
-            buffer.printf("size += 2;%n");
-            if (nullable) {
-                buffer.printf("if (%s != null) {%n", fieldName);
-                buffer.incrementIndent();
-            }
-            headerGenerator.addImport(MessageGenerator.MESSAGE_UTIL_CLASS);
-            buffer.printf("size += MessageUtil.serializedUtf8Length(%s);%n", fieldName);
-            if (nullable) {
-                buffer.decrementIndent();
-                buffer.printf("}%n");
-            }
+            generateStringToBytes(fieldName);
+            VersionConditional.forVersions(flexibleVersions, versions).
+                ifNotMember(__ -> {
+                    buffer.printf("_arraySize += _stringBytes.length + 2;%n");
+                }).
+                ifMember(__ -> {
+                    headerGenerator.addImport(MessageGenerator.BYTE_UTILS_CLASS);
+                    buffer.printf("_arraySize += _stringBytes.length + " +
+                        "ByteUtils.sizeOfUnsignedVarint(_stringBytes.length + 1);%n");
+                }).
+                generate(buffer);
         } else if (type instanceof FieldType.BytesFieldType) {
-            buffer.printf("size += 4;%n");
-            if (nullable) {
-                buffer.printf("if (%s != null) {%n", fieldName);
-                buffer.incrementIndent();
-            }
-            buffer.printf("size += %s.length;%n", fieldName);
-            if (nullable) {
-                buffer.decrementIndent();
-                buffer.printf("}%n");
-            }
+            VersionConditional.forVersions(flexibleVersions, versions).
+                ifNotMember(__ -> {
+                    buffer.printf("_arraySize += %s.length + 4;%n", fieldName);
+                }).
+                ifMember(__ -> {
+                    headerGenerator.addImport(MessageGenerator.BYTE_UTILS_CLASS);
+                    buffer.printf("_arraySize += %s.length + " +
+                        "ByteUtils.sizeOfUnsignedVarint(%s.length + 1);%n",
+                        fieldName, fieldName);
+                }).
+                generate(buffer);
         } else if (type instanceof FieldType.StructType) {
-            buffer.printf("size += %s.size(version);%n", fieldName);
+            buffer.printf("_arraySize += %s.size(_cache, _version);%n", fieldName);
         } else {
             throw new RuntimeException("Unsupported type " + type);
         }
     }
 
-    private void generateFieldSize(FieldSpec field, Versions curVersions) {
+    private void generateFieldSize(FieldSpec field,
+                                   Versions possibleVersions,
+                                   boolean tagged) {
         if (field.type().fixedLength().isPresent()) {
-            boolean maybeAbsent =
-                generateVersionCheck(curVersions, field.versions());
-            buffer.printf("size += %d;%n", field.type().fixedLength().get());
-            if (maybeAbsent) {
-                buffer.decrementIndent();
-                generateAbsentValueCheck(field);
+            if (tagged) {
+                // Account for the tagged field prefix.
+                buffer.printf("_size += %d;%n",
+                    MessageGenerator.sizeOfUnsignedVarint(field.type().fixedLength().get()));
             }
-        } else if (field.type().isString() || field.type().isBytes() || field.type().isStruct()) {
-            boolean nullable = !curVersions.intersect(field.nullableVersions()).empty();
-            boolean maybeAbsent =
-                generateVersionCheck(curVersions, field.versions());
-            generateVariableLengthFieldSize(field.camelCaseName(), field.type(), nullable);
-            if (maybeAbsent) {
-                buffer.decrementIndent();
-                generateAbsentValueCheck(field);
-            }
-        } else if (field.type().isArray()) {
-            boolean maybeAbsent =
-                generateVersionCheck(curVersions, field.versions());
-            boolean maybeNull = generateNullCheck(curVersions, field);
-            if (maybeNull) {
-                buffer.printf("size += 4;%n");
-                buffer.decrementIndent();
-                buffer.printf("} else {%n");
-                buffer.incrementIndent();
-            }
-            buffer.printf("size += 4;%n");
-            FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type();
-            FieldType elementType = arrayType.elementType();
-            if (elementType.fixedLength().isPresent()) {
-                buffer.printf("size += %s.size() * %d;%n",
-                    field.camelCaseName(),
-                    elementType.fixedLength().get());
-            } else if (elementType instanceof FieldType.ArrayType) {
-                throw new RuntimeException("Arrays of arrays are not supported " +
-                    "(use a struct).");
-            } else {
-                buffer.printf("for (%s element : %s) {%n",
-                    getBoxedJavaType(elementType), field.camelCaseName());
-                buffer.incrementIndent();
-                generateVariableLengthFieldSize("element", elementType, false);
-                buffer.decrementIndent();
-                buffer.printf("}%n");
-            }
-            if (maybeNull) {
-                buffer.decrementIndent();
-                buffer.printf("}%n");
-            }
-            if (maybeAbsent) {
-                buffer.decrementIndent();
-                generateAbsentValueCheck(field);
-            }
-        } else {
-            throw new RuntimeException("Unsupported field type " + field.type());
+            buffer.printf("_size += %d;%n", field.type().fixedLength().get());
+            return;
         }
+        IsNullConditional.forField(field).
+            alwaysEmitBlockScope(true).
+            possibleVersions(possibleVersions).
+            ifNull(() -> {
+                VersionConditional.forVersions(fieldFlexibleVersions(field), possibleVersions).
+                    ifMember(__ -> {
+                        if (tagged) {
+                            // Account for the tagged field prefix.
+                            buffer.printf("_size += %d;%n", MessageGenerator.sizeOfUnsignedVarint(
+                                MessageGenerator.sizeOfUnsignedVarint(0)));
+                        }
+                        buffer.printf("_size += %d;%n", MessageGenerator.sizeOfUnsignedVarint(0));
+                    }).
+                    ifNotMember(__ -> {
+                        if (tagged) {
+                            throw new RuntimeException("Tagged field " + field.name() +
+                                " should not be present in non-flexible versions.");
+                        }
+                        if (field.type().isString()) {
+                            buffer.printf("_size += 2;%n");
+                        } else {
+                            buffer.printf("_size += 4;%n");
+                        }
+                    }).
+                    generate(buffer);
+            }).
+            ifNotNull(() -> {
+                if (field.type().isString()) {
+                    generateStringToBytes(field.camelCaseName());
+                    VersionConditional.forVersions(fieldFlexibleVersions(field), possibleVersions).
+                        ifMember(__ -> {
+                            headerGenerator.addImport(MessageGenerator.BYTE_UTILS_CLASS);
+                            if (tagged) {
+                                buffer.printf("int _stringPrefixSize = " +
+                                    "ByteUtils.sizeOfUnsignedVarint(_stringBytes.length + 1);%n");
+                                buffer.printf("_size += _stringBytes.length + _stringPrefixSize + " +
+                                    "ByteUtils.sizeOfUnsignedVarint(_stringPrefixSize);%n");
+                            } else {
+                                buffer.printf("_size += _stringBytes.length + " +
+                                    "ByteUtils.sizeOfUnsignedVarint(_stringBytes.length + 1);%n");
+                            }
+                        }).
+                        ifNotMember(__ -> {
+                            if (tagged) {
+                                throw new RuntimeException("Tagged field " + field.name() +
+                                    " should not be present in non-flexible versions.");
+                            }
+                            buffer.printf("_size += _stringBytes.length + 2;%n");
+                        }).
+                        generate(buffer);
+                } else if (field.type().isArray()) {
+                    buffer.printf("int _arraySize = 0;%n");
+                    VersionConditional.forVersions(fieldFlexibleVersions(field), possibleVersions).
+                        ifMember(__ -> {
+                            headerGenerator.addImport(MessageGenerator.BYTE_UTILS_CLASS);
+                            buffer.printf("_arraySize += ByteUtils.sizeOfUnsignedVarint(%s.size() + 1);%n",
+                                field.camelCaseName());
+                        }).
+                        ifNotMember(__ -> {
+                            buffer.printf("_arraySize += 4;%n");
+                        }).
+                        generate(buffer);
+                    FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type();
+                    FieldType elementType = arrayType.elementType();
+                    if (elementType.fixedLength().isPresent()) {
+                        buffer.printf("_arraySize += %s.size() * %d;%n",
+                            field.camelCaseName(),
+                            elementType.fixedLength().get());
+                    } else if (elementType instanceof FieldType.ArrayType) {
+                        throw new RuntimeException("Arrays of arrays are not supported " +
+                            "(use a struct).");
+                    } else {
+                        buffer.printf("for (%s %sElement : %s) {%n",
+                            getBoxedJavaType(elementType), field.camelCaseName(), field.camelCaseName());
+                        buffer.incrementIndent();
+                        generateVariableLengthArrayElementSize(fieldFlexibleVersions(field),
+                            String.format("%sElement", field.camelCaseName()),
+                            elementType,
+                            possibleVersions);
+                        buffer.decrementIndent();
+                        buffer.printf("}%n");
+                    }
+                    if (tagged) {
+                        headerGenerator.addImport(MessageGenerator.BYTE_UTILS_CLASS);
+                        buffer.printf("_cache.setArraySizeInBytes(%s, _arraySize);%n",
+                            field.camelCaseName());
+                        buffer.printf("_size += _arraySize + ByteUtils.sizeOfUnsignedVarint(_arraySize);%n");
+                    } else {
+                        buffer.printf("_size += _arraySize;%n");
+                    }
+                } else if (field.type().isBytes()) {
+                    buffer.printf("int _bytesSize = %s.length;%n", field.camelCaseName());
+                    VersionConditional.forVersions(fieldFlexibleVersions(field), possibleVersions).
+                        ifMember(__ -> {
+                            headerGenerator.addImport(MessageGenerator.BYTE_UTILS_CLASS);
+                            headerGenerator.addImport(MessageGenerator.BYTE_UTILS_CLASS);
+                            buffer.printf("_bytesSize += ByteUtils.sizeOfUnsignedVarint(%s.length + 1);%n",
+                                field.camelCaseName());
+                        }).
+                        ifNotMember(__ -> {
+                            buffer.printf("_bytesSize += 4;%n");
+                        }).
+                        generate(buffer);
+                    if (tagged) {
+                        headerGenerator.addImport(MessageGenerator.BYTE_UTILS_CLASS);
+                        buffer.printf("_size += _bytesSize + ByteUtils.sizeOfUnsignedVarint(_bytesSize);%n");
+                    } else {
+                        buffer.printf("_size += _bytesSize;%n");
+                    }
+                } else {
+                    throw new RuntimeException("unhandled type " + field.type());
+                }
+            }).
+            generate(buffer);
     }
 
-    private void generateAbsentValueCheck(FieldSpec field) {
-        if (field.ignorable()) {
-            buffer.printf("}%n");
-            return;
-        }
-        buffer.printf("} else {%n");
-        buffer.incrementIndent();
-        headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS);
-        if (field.type().isArray()) {
-            if (fieldDefault(field).equals("null")) {
-                buffer.printf("if (%s != null) {%n", field.camelCaseName());
-            } else {
-                buffer.printf("if (!%s.isEmpty()) {%n", field.camelCaseName());
-            }
-        } else if (field.type().isBytes()) {
-            if (fieldDefault(field).equals("null")) {
-                buffer.printf("if (%s != null) {%n", field.camelCaseName());
-            } else {
-                buffer.printf("if (%s.length != 0) {%n", field.camelCaseName());
-            }
-        } else if (field.type().isString()) {
-            if (fieldDefault(field).equals("null")) {
-                buffer.printf("if (%s != null) {%n", field.camelCaseName());
-            } else {
-                buffer.printf("if (!%s.equals(%s)) {%n", field.camelCaseName(), fieldDefault(field));
-            }
-        } else if (field.type() instanceof FieldType.BoolFieldType) {
-            buffer.printf("if (%s%s) {%n",
-                fieldDefault(field).equals("true") ? "!" : "",
-                field.camelCaseName());
-        } else {
-            buffer.printf("if (%s != %s) {%n", field.camelCaseName(), fieldDefault(field));
-        }
+    private void generateStringToBytes(String name) {
+        headerGenerator.addImport(MessageGenerator.STANDARD_CHARSETS);
+        buffer.printf("byte[] _stringBytes = %s.getBytes(StandardCharsets.UTF_8);%n", name);
+        buffer.printf("if (_stringBytes.length > 0x7fff) {%n");
         buffer.incrementIndent();
-        buffer.printf("throw new UnsupportedVersionException(" +
-                "\"Attempted to write a non-default %s at version \" + version);%n",
-            field.camelCaseName());
-        buffer.decrementIndent();
-        buffer.printf("}%n");
+        buffer.printf("throw new RuntimeException(\"'%s' field is too long to " +
+            "be serialized\");%n", name);
         buffer.decrementIndent();
         buffer.printf("}%n");
+        buffer.printf("_cache.cacheSerializedValue(%s, _stringBytes);%n", name);
     }
 
     private void generateClassEquals(String className, StructSpec struct, boolean onlyMapKeys) {
@@ -1002,7 +1532,10 @@ private void generateClassEquals(String className, StructSpec struct, boolean on
     }
 
     private void generateFieldEquals(FieldSpec field) {
-        if (field.type().isString() || field.type().isArray() || field.type().isStruct()) {
+        if (field.type() instanceof FieldType.UUIDFieldType) {
+            buffer.printf("if (!this.%s.equals(other.%s)) return false;%n",
+                field.camelCaseName(), field.camelCaseName());
+        } else if (field.type().isString() || field.type().isArray() || field.type().isStruct()) {
             buffer.printf("if (this.%s == null) {%n", field.camelCaseName());
             buffer.incrementIndent();
             buffer.printf("if (other.%s != null) return false;%n", field.camelCaseName());
@@ -1051,15 +1584,16 @@ private void generateFieldHashCode(FieldSpec field) {
         } else if (field.type() instanceof FieldType.Int64FieldType) {
             buffer.printf("hashCode = 31 * hashCode + ((int) (%s >> 32) ^ (int) %s);%n",
                 field.camelCaseName(), field.camelCaseName());
+        } else if (field.type() instanceof FieldType.UUIDFieldType) {
+            buffer.printf("hashCode = 31 * hashCode + %s.hashCode();%n",
+                field.camelCaseName());
         } else if (field.type().isBytes()) {
             headerGenerator.addImport(MessageGenerator.ARRAYS_CLASS);
             buffer.printf("hashCode = 31 * hashCode + Arrays.hashCode(%s);%n",
                 field.camelCaseName());
         } else if (field.type().isStruct()
                    || field.type().isArray()
-                   || field.type().isString()
-                   || field.type() instanceof  FieldType.UUIDFieldType
-        ) {
+                   || field.type().isString()) {
             buffer.printf("hashCode = 31 * hashCode + (%s == null ? 0 : %s.hashCode());%n",
                           field.camelCaseName(), field.camelCaseName());
         } else {
@@ -1091,18 +1625,18 @@ private void generateFieldToString(String prefix, FieldSpec field) {
         } else if ((field.type() instanceof FieldType.Int8FieldType) ||
                 (field.type() instanceof FieldType.Int16FieldType) ||
                 (field.type() instanceof FieldType.Int32FieldType) ||
-                (field.type() instanceof FieldType.Int64FieldType) ||
-                (field.type() instanceof FieldType.UUIDFieldType)) {
+                (field.type() instanceof FieldType.Int64FieldType)) {
             buffer.printf("+ \"%s%s=\" + %s%n",
                 prefix, field.camelCaseName(), field.camelCaseName());
         } else if (field.type().isString()) {
-            buffer.printf("+ \"%s%s='\" + %s + \"'\"%n",
-                prefix, field.camelCaseName(), field.camelCaseName());
+            buffer.printf("+ \"%s%s=\" + ((%s == null) ? \"null\" : \"'\" + %s.toString() + \"'\")%n",
+                prefix, field.camelCaseName(), field.camelCaseName(), field.camelCaseName());
         } else if (field.type().isBytes()) {
             headerGenerator.addImport(MessageGenerator.ARRAYS_CLASS);
             buffer.printf("+ \"%s%s=\" + Arrays.toString(%s)%n",
                 prefix, field.camelCaseName(), field.camelCaseName());
-        } else if (field.type().isStruct()) {
+        } else if (field.type().isStruct() ||
+            field.type() instanceof FieldType.UUIDFieldType) {
             buffer.printf("+ \"%s%s=\" + %s.toString()%n",
                 prefix, field.camelCaseName(), field.camelCaseName());
         } else if (field.type().isArray()) {
@@ -1120,67 +1654,6 @@ private void generateFieldToString(String prefix, FieldSpec field) {
         }
     }
 
-    private boolean generateNullCheck(Versions prevVersions, FieldSpec field) {
-        if (prevVersions.intersect(field.nullableVersions()).empty()) {
-            return false;
-        }
-        buffer.printf("if (%s == null) {%n", field.camelCaseName());
-        buffer.incrementIndent();
-        return true;
-    }
-
-    private boolean generateVersionCheck(Versions prev, Versions cur) {
-        if (cur.lowest() > prev.lowest()) {
-            if (cur.highest() < prev.highest()) {
-                buffer.printf("if ((version >= %d) && (version <= %d)) {%n",
-                    cur.lowest(), cur.highest());
-                buffer.incrementIndent();
-                return true;
-            } else {
-                buffer.printf("if (version >= %d) {%n", cur.lowest());
-                buffer.incrementIndent();
-                return true;
-            }
-        } else {
-            if (cur.highest() < prev.highest()) {
-                buffer.printf("if (version <= %d) {%n", cur.highest());
-                buffer.incrementIndent();
-                return true;
-            } else {
-                return false;
-            }
-        }
-    }
-
-    private boolean generateInverseVersionCheck(Versions prev, Versions cur) {
-        if (cur.lowest() > prev.lowest()) {
-            if (cur.highest() < prev.highest()) {
-                buffer.printf("if ((version < %d) || (version > %d)) {%n",
-                    cur.lowest(), cur.highest());
-                return true;
-            } else {
-                buffer.printf("if (version < %d) {%n", cur.lowest());
-                return true;
-            }
-        } else {
-            if (cur.highest() < prev.highest()) {
-                buffer.printf("if (version > %d) {%n", cur.highest());
-                return true;
-            } else {
-                return false;
-            }
-        }
-    }
-
-    private void generateSetDefault(FieldSpec field) {
-        buffer.decrementIndent();
-        buffer.printf("} else {%n");
-        buffer.incrementIndent();
-        buffer.printf("this.%s = %s;%n", field.camelCaseName(), fieldDefault(field));
-        buffer.decrementIndent();
-        buffer.printf("}%n");
-    }
-
     private String fieldDefault(FieldSpec field) {
         if (field.type() instanceof FieldType.BoolFieldType) {
             if (field.defaultString().isEmpty()) {
@@ -1244,7 +1717,8 @@ private String fieldDefault(FieldSpec field) {
         } else if (field.type() instanceof FieldType.UUIDFieldType) {
             headerGenerator.addImport(MessageGenerator.UUID_CLASS);
             if (field.defaultString().isEmpty()) {
-                return "org.apache.kafka.common.protocol.MessageUtil.ZERO_UUID";
+                headerGenerator.addImport(MessageGenerator.MESSAGE_UTIL_CLASS);
+                return "MessageUtil.ZERO_UUID";
             } else {
                 try {
                     UUID.fromString(field.defaultString());
@@ -1252,6 +1726,7 @@ private String fieldDefault(FieldSpec field) {
                     throw new RuntimeException("Invalid default for uuid field " +
                         field.name() + ": " + field.defaultString(), e);
                 }
+                headerGenerator.addImport(MessageGenerator.UUID_CLASS);
                 return "UUID.fromString(\"" + field.defaultString() + "\")";
             }
         } else if (field.type() instanceof FieldType.StringFieldType) {
@@ -1341,4 +1816,19 @@ private void generateSetter(String javaType, String functionName, String memberN
         buffer.decrementIndent();
         buffer.printf("}%n");
     }
+
+    private Versions fieldFlexibleVersions(FieldSpec field) {
+        if (field.flexibleVersions().isPresent()) {
+            if (!messageFlexibleVersions.intersect(field.flexibleVersions().get()).
+                    equals(field.flexibleVersions().get())) {
+                throw new RuntimeException("The flexible versions for field " +
+                    field.name() + " are " + field.flexibleVersions().get() +
+                    ", which are not a subset of the flexible versions for the " +
+                    "message as a whole, which are " + messageFlexibleVersions);
+            }
+            return field.flexibleVersions().get();
+        } else {
+            return messageFlexibleVersions;
+        }
+    }
 }
diff --git a/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java b/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java
index 9b5d9ed2da5ce..31a05f73fac95 100644
--- a/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java
+++ b/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java
@@ -75,6 +75,8 @@ public final class MessageGenerator {
 
     static final String ARRAYOF_CLASS = "org.apache.kafka.common.protocol.types.ArrayOf";
 
+    static final String COMPACT_ARRAYOF_CLASS = "org.apache.kafka.common.protocol.types.CompactArrayOf";
+
     static final String STRUCT_CLASS = "org.apache.kafka.common.protocol.types.Struct";
 
     static final String BYTES_CLASS = "org.apache.kafka.common.utils.Bytes";
@@ -85,6 +87,26 @@ public final class MessageGenerator {
 
     static final String RESPONSE_SUFFIX = "Response";
 
+    static final String BYTE_UTILS_CLASS = "org.apache.kafka.common.utils.ByteUtils";
+
+    static final String STANDARD_CHARSETS = "java.nio.charset.StandardCharsets";
+
+    static final String TAGGED_FIELDS_SECTION_CLASS = "org.apache.kafka.common.protocol.types.Field.TaggedFieldsSection";
+
+    static final String OBJECT_SERIALIZATION_CACHE_CLASS = "org.apache.kafka.common.protocol.ObjectSerializationCache";
+
+    static final String RAW_TAGGED_FIELD_CLASS = "org.apache.kafka.common.protocol.types.RawTaggedField";
+
+    static final String RAW_TAGGED_FIELD_WRITER_CLASS = "org.apache.kafka.common.protocol.types.RawTaggedFieldWriter";
+
+    static final String TREE_MAP_CLASS = "java.util.TreeMap";
+
+    static final String BYTE_BUFFER_CLASS = "java.nio.ByteBuffer";
+
+    static final String NAVIGABLE_MAP_CLASS = "java.util.NavigableMap";
+
+    static final String MAP_ENTRY_CLASS = "java.util.Map.Entry";
+
     /**
      * The Jackson serializer we use for JSON objects.
      */
@@ -198,6 +220,18 @@ static String stripSuffix(String str, String suffix) {
         }
     }
 
+    /**
+     * Return the number of bytes needed to encode an integer in unsigned variable-length format.
+     */
+    static int sizeOfUnsignedVarint(int value) {
+        int bytes = 1;
+        while ((value & 0xffffff80) != 0L) {
+            bytes += 1;
+            value >>>= 7;
+        }
+        return bytes;
+    }
+
     private final static String USAGE = "MessageGenerator: [output Java package] [output Java file] [input JSON file]";
 
     public static void main(String[] args) throws Exception {
diff --git a/generator/src/main/java/org/apache/kafka/message/MessageSpec.java b/generator/src/main/java/org/apache/kafka/message/MessageSpec.java
index 30b1fe4971d05..9dc05795294fb 100644
--- a/generator/src/main/java/org/apache/kafka/message/MessageSpec.java
+++ b/generator/src/main/java/org/apache/kafka/message/MessageSpec.java
@@ -35,18 +35,28 @@ public final class MessageSpec {
 
     private final List commonStructs;
 
+    private final Versions flexibleVersions;
+
     @JsonCreator
     public MessageSpec(@JsonProperty("name") String name,
                        @JsonProperty("validVersions") String validVersions,
                        @JsonProperty("fields") List fields,
                        @JsonProperty("apiKey") Short apiKey,
                        @JsonProperty("type") MessageSpecType type,
-                       @JsonProperty("commonStructs") List commonStructs) {
+                       @JsonProperty("commonStructs") List commonStructs,
+                       @JsonProperty("flexibleVersions") String flexibleVersions) {
         this.struct = new StructSpec(name, validVersions, fields);
         this.apiKey = apiKey == null ? Optional.empty() : Optional.of(apiKey);
         this.type = Objects.requireNonNull(type);
         this.commonStructs = commonStructs == null ? Collections.emptyList() :
                 Collections.unmodifiableList(new ArrayList<>(commonStructs));
+        this.flexibleVersions = Versions.parse(flexibleVersions, Versions.NONE);
+        if ((!this.flexibleVersions().empty()) &&
+                (this.flexibleVersions.highest() < Short.MAX_VALUE)) {
+            throw new RuntimeException("Field " + name + " specifies flexibleVersions " +
+                this.flexibleVersions + ", which is not open-ended.  flexibleVersions must " +
+                "be either none, or an open-ended range (that ends with a plus sign).");
+        }
     }
 
     public StructSpec struct() {
@@ -58,6 +68,10 @@ public String name() {
         return struct.name();
     }
 
+    public Versions validVersions() {
+        return struct.versions();
+    }
+
     @JsonProperty("validVersions")
     public String validVersionsString() {
         return struct.versionsString();
@@ -83,6 +97,15 @@ public List commonStructs() {
         return commonStructs;
     }
 
+    public Versions flexibleVersions() {
+        return flexibleVersions;
+    }
+
+    @JsonProperty("flexibleVersions")
+    public String flexibleVersionsString() {
+        return flexibleVersions.toString();
+    }
+
     public String generatedClassName() {
         return struct.name() + "Data";
     }
diff --git a/generator/src/main/java/org/apache/kafka/message/SchemaGenerator.java b/generator/src/main/java/org/apache/kafka/message/SchemaGenerator.java
index 3b04b7b11a735..3d54a588540e4 100644
--- a/generator/src/main/java/org/apache/kafka/message/SchemaGenerator.java
+++ b/generator/src/main/java/org/apache/kafka/message/SchemaGenerator.java
@@ -68,6 +68,11 @@ static class MessageInfo {
      */
     private final Map messages;
 
+    /**
+     * The versions that implement a KIP-482 flexible schema.
+     */
+    private Versions messageFlexibleVersions;
+
     SchemaGenerator(HeaderGenerator headerGenerator, StructRegistry structRegistry) {
         this.headerGenerator = headerGenerator;
         this.structRegistry = structRegistry;
@@ -75,6 +80,7 @@ static class MessageInfo {
     }
 
     void generateSchemas(MessageSpec message) throws Exception {
+        this.messageFlexibleVersions = message.flexibleVersions();
         // Generate schemas for inline structures
         generateSchemas(message.generatedClassName(), message.struct(),
             message.struct().versions());
@@ -117,7 +123,8 @@ void generateSchemas(String className, StructSpec struct,
         }
     }
 
-    private void generateSchemaForVersion(StructSpec struct, short version,
+    private void generateSchemaForVersion(StructSpec struct,
+                                          short version,
                                           CodeBuffer buffer) throws Exception {
         // Find the last valid field index.
         int lastValidIndex = struct.fields().size() - 1;
@@ -126,38 +133,95 @@ private void generateSchemaForVersion(StructSpec struct, short version,
                 break;
             }
             FieldSpec field = struct.fields().get(lastValidIndex);
-            if (field.versions().contains(version)) {
+            if ((!field.taggedVersions().contains(version)) &&
+                    field.versions().contains(version)) {
                 break;
             }
             lastValidIndex--;
         }
+        int finalLine = lastValidIndex;
+        if (messageFlexibleVersions.contains(version)) {
+            finalLine++;
+        }
 
         headerGenerator.addImport(MessageGenerator.SCHEMA_CLASS);
         buffer.printf("new Schema(%n");
         buffer.incrementIndent();
         for (int i = 0; i <= lastValidIndex; i++) {
             FieldSpec field = struct.fields().get(i);
-            if (!field.versions().contains(version)) {
+            if ((!field.versions().contains(version)) ||
+                    field.taggedVersions().contains(version)) {
                 continue;
             }
+            Versions fieldFlexibleVersions =
+                field.flexibleVersions().orElse(messageFlexibleVersions);
             headerGenerator.addImport(MessageGenerator.FIELD_CLASS);
             buffer.printf("new Field(\"%s\", %s, \"%s\")%s%n",
                 field.snakeCaseName(),
-                fieldTypeToSchemaType(field, version),
+                fieldTypeToSchemaType(field, version, fieldFlexibleVersions),
                 field.about(),
-                i == lastValidIndex ? "" : ",");
+                i == finalLine ? "" : ",");
+        }
+        if (messageFlexibleVersions.contains(version)) {
+            generateTaggedFieldsSchemaForVersion(struct, version, buffer);
         }
         buffer.decrementIndent();
         buffer.printf(");%n");
     }
 
-    private String fieldTypeToSchemaType(FieldSpec field, short version) {
+    private void generateTaggedFieldsSchemaForVersion(StructSpec struct,
+            short version, CodeBuffer buffer) throws Exception {
+        headerGenerator.addStaticImport(MessageGenerator.TAGGED_FIELDS_SECTION_CLASS);
+
+        // Find the last valid tagged field index.
+        int lastValidIndex = struct.fields().size() - 1;
+        while (true) {
+            if (lastValidIndex < 0) {
+                break;
+            }
+            FieldSpec field = struct.fields().get(lastValidIndex);
+            if ((field.taggedVersions().contains(version)) &&
+                field.versions().contains(version)) {
+                break;
+            }
+            lastValidIndex--;
+        }
+
+        buffer.printf("TaggedFieldsSection.of(%n");
+        buffer.incrementIndent();
+        for (int i = 0; i <= lastValidIndex; i++) {
+            FieldSpec field = struct.fields().get(i);
+            if ((!field.versions().contains(version)) ||
+                    (!field.taggedVersions().contains(version))) {
+                continue;
+            }
+            headerGenerator.addImport(MessageGenerator.FIELD_CLASS);
+            Versions fieldFlexibleVersions =
+                field.flexibleVersions().orElse(messageFlexibleVersions);
+            buffer.printf("%d, new Field(\"%s\", %s, \"%s\")%s%n",
+                field.tag().get(),
+                field.snakeCaseName(),
+                fieldTypeToSchemaType(field, version, fieldFlexibleVersions),
+                field.about(),
+                i == lastValidIndex ? "" : ",");
+        }
+        buffer.decrementIndent();
+        buffer.printf(")%n");
+    }
+
+    private String fieldTypeToSchemaType(FieldSpec field,
+                                         short version,
+                                         Versions fieldFlexibleVersions) {
         return fieldTypeToSchemaType(field.type(),
             field.nullableVersions().contains(version),
-            version);
+            version,
+            fieldFlexibleVersions);
     }
 
-    private String fieldTypeToSchemaType(FieldType type, boolean nullable, short version) {
+    private String fieldTypeToSchemaType(FieldType type,
+                                         boolean nullable,
+                                         short version,
+                                         Versions fieldFlexibleVersions) {
         if (type instanceof FieldType.BoolFieldType) {
             headerGenerator.addImport(MessageGenerator.TYPE_CLASS);
             if (nullable) {
@@ -196,16 +260,33 @@ private String fieldTypeToSchemaType(FieldType type, boolean nullable, short ver
             return "Type.UUID";
         } else if (type instanceof FieldType.StringFieldType) {
             headerGenerator.addImport(MessageGenerator.TYPE_CLASS);
-            return nullable ? "Type.NULLABLE_STRING" : "Type.STRING";
+            if (fieldFlexibleVersions.contains(version)) {
+                return nullable ? "Type.COMPACT_NULLABLE_STRING" : "Type.COMPACT_STRING";
+            } else {
+                return nullable ? "Type.NULLABLE_STRING" : "Type.STRING";
+            }
         } else if (type instanceof FieldType.BytesFieldType) {
             headerGenerator.addImport(MessageGenerator.TYPE_CLASS);
-            return nullable ? "Type.NULLABLE_BYTES" : "Type.BYTES";
+            if (fieldFlexibleVersions.contains(version)) {
+                return nullable ? "Type.COMPACT_NULLABLE_BYTES" : "Type.COMPACT_BYTES";
+            } else {
+                return nullable ? "Type.NULLABLE_BYTES" : "Type.BYTES";
+            }
         } else if (type.isArray()) {
-            headerGenerator.addImport(MessageGenerator.ARRAYOF_CLASS);
-            FieldType.ArrayType arrayType = (FieldType.ArrayType) type;
-            String prefix = nullable ? "ArrayOf.nullable" : "new ArrayOf";
-            return String.format("%s(%s)", prefix,
-                fieldTypeToSchemaType(arrayType.elementType(), false, version));
+            if (fieldFlexibleVersions.contains(version)) {
+                headerGenerator.addImport(MessageGenerator.COMPACT_ARRAYOF_CLASS);
+                FieldType.ArrayType arrayType = (FieldType.ArrayType) type;
+                String prefix = nullable ? "CompactArrayOf.nullable" : "new CompactArrayOf";
+                return String.format("%s(%s)", prefix,
+                        fieldTypeToSchemaType(arrayType.elementType(), false, version, fieldFlexibleVersions));
+
+            } else {
+                headerGenerator.addImport(MessageGenerator.ARRAYOF_CLASS);
+                FieldType.ArrayType arrayType = (FieldType.ArrayType) type;
+                String prefix = nullable ? "ArrayOf.nullable" : "new ArrayOf";
+                return String.format("%s(%s)", prefix,
+                        fieldTypeToSchemaType(arrayType.elementType(), false, version, fieldFlexibleVersions));
+            }
         } else if (type.isStruct()) {
             if (nullable) {
                 throw new RuntimeException("Type " + type + " cannot be nullable.");
diff --git a/generator/src/main/java/org/apache/kafka/message/StructSpec.java b/generator/src/main/java/org/apache/kafka/message/StructSpec.java
index a775e77ba27d8..b4094dab57845 100644
--- a/generator/src/main/java/org/apache/kafka/message/StructSpec.java
+++ b/generator/src/main/java/org/apache/kafka/message/StructSpec.java
@@ -22,6 +22,7 @@
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
 
@@ -44,8 +45,32 @@ public StructSpec(@JsonProperty("name") String name,
             throw new RuntimeException("You must specify the version of the " +
                     name + " structure.");
         }
-        this.fields = Collections.unmodifiableList(fields == null ?
-            Collections.emptyList() : new ArrayList<>(fields));
+        ArrayList newFields = new ArrayList<>();
+        if (fields != null) {
+            // Each field should have a unique tag ID (if the field has a tag ID).
+            HashSet tags = new HashSet<>();
+            for (FieldSpec field : fields) {
+                if (field.tag().isPresent()) {
+                    if (tags.contains(field.tag().get())) {
+                        throw new RuntimeException("In " + name + ", field " + field.name() +
+                            " has a duplicate tag ID " + field.tag().get() + ".  All tags IDs " +
+                            "must be unique.");
+                    }
+                    tags.add(field.tag().get());
+                }
+                newFields.add(field);
+            }
+            // Tag IDs should be contiguous and start at 0.  This optimizes space on the wire,
+            // since larger numbers take more space.
+            for (int i = 0; i < tags.size(); i++) {
+                if (!tags.contains(i)) {
+                    throw new RuntimeException("In " + name + ", the tag IDs are not " +
+                        "contiguous.  Make use of tag " + i + " before using any " +
+                        "higher tag IDs.");
+                }
+            }
+        }
+        this.fields = Collections.unmodifiableList(newFields);
         this.hasKeys = this.fields.stream().anyMatch(f -> f.mapKey());
     }
 
diff --git a/generator/src/main/java/org/apache/kafka/message/VersionConditional.java b/generator/src/main/java/org/apache/kafka/message/VersionConditional.java
index 51b6cb047a0f0..9a1abfa105f09 100644
--- a/generator/src/main/java/org/apache/kafka/message/VersionConditional.java
+++ b/generator/src/main/java/org/apache/kafka/message/VersionConditional.java
@@ -85,7 +85,7 @@ private void generateFullRangeCheck(Versions ifVersions,
                                         Versions ifNotVersions,
                                         CodeBuffer buffer) {
         if (ifMember != null) {
-            buffer.printf("if ((version >= %d) && (version <= %d)) {%n",
+            buffer.printf("if ((_version >= %d) && (_version <= %d)) {%n",
                     containingVersions.lowest(), containingVersions.highest());
             buffer.incrementIndent();
             ifMember.generate(ifVersions);
@@ -98,7 +98,7 @@ private void generateFullRangeCheck(Versions ifVersions,
             }
             buffer.printf("}%n");
         } else if (ifNotMember != null) {
-            buffer.printf("if ((version < %d) || (version > %d)) {%n",
+            buffer.printf("if ((_version < %d) || (_version > %d)) {%n",
                     containingVersions.lowest(), containingVersions.highest());
             buffer.incrementIndent();
             ifNotMember.generate(ifNotVersions);
@@ -111,7 +111,7 @@ private void generateLowerRangeCheck(Versions ifVersions,
                                          Versions ifNotVersions,
                                          CodeBuffer buffer) {
         if (ifMember != null) {
-            buffer.printf("if (version >= %d) {%n", containingVersions.lowest());
+            buffer.printf("if (_version >= %d) {%n", containingVersions.lowest());
             buffer.incrementIndent();
             ifMember.generate(ifVersions);
             buffer.decrementIndent();
@@ -123,7 +123,7 @@ private void generateLowerRangeCheck(Versions ifVersions,
             }
             buffer.printf("}%n");
         } else if (ifNotMember != null) {
-            buffer.printf("if (version < %d) {%n", containingVersions.lowest());
+            buffer.printf("if (_version < %d) {%n", containingVersions.lowest());
             buffer.incrementIndent();
             ifNotMember.generate(ifNotVersions);
             buffer.decrementIndent();
@@ -135,7 +135,7 @@ private void generateUpperRangeCheck(Versions ifVersions,
                                          Versions ifNotVersions,
                                          CodeBuffer buffer) {
         if (ifMember != null) {
-            buffer.printf("if (version <= %d) {%n", containingVersions.highest());
+            buffer.printf("if (_version <= %d) {%n", containingVersions.highest());
             buffer.incrementIndent();
             ifMember.generate(ifVersions);
             buffer.decrementIndent();
@@ -147,7 +147,7 @@ private void generateUpperRangeCheck(Versions ifVersions,
             }
             buffer.printf("}%n");
         } else if (ifNotMember != null) {
-            buffer.printf("if (version > %d) {%n", containingVersions.highest());
+            buffer.printf("if (_version > %d) {%n", containingVersions.highest());
             buffer.incrementIndent();
             ifNotMember.generate(ifNotVersions);
             buffer.decrementIndent();
diff --git a/generator/src/test/java/org/apache/kafka/message/MessageDataGeneratorTest.java b/generator/src/test/java/org/apache/kafka/message/MessageDataGeneratorTest.java
index a772988974924..6cbafb8b06ea2 100644
--- a/generator/src/test/java/org/apache/kafka/message/MessageDataGeneratorTest.java
+++ b/generator/src/test/java/org/apache/kafka/message/MessageDataGeneratorTest.java
@@ -23,6 +23,7 @@
 
 import java.util.Arrays;
 
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -50,6 +51,11 @@ public void testNullDefaults() throws Exception {
         new MessageDataGenerator("org.apache.kafka.common.message").generate(testMessageSpec);
     }
 
+    private void assertStringContains(String substring, String value) {
+        assertTrue("Expected string to contain '" + substring + "', but it was " + value,
+            value.contains(substring));
+    }
+
     @Test
     public void testInvalidNullDefaultForInt() throws Exception {
         MessageSpec testMessageSpec = MessageGenerator.JSON_SERDE.readValue(String.join("", Arrays.asList(
@@ -61,13 +67,10 @@ public void testInvalidNullDefaultForInt() throws Exception {
             "    { \"name\": \"field1\", \"type\": \"int32\", \"versions\": \"0+\", \"default\": \"null\" }",
             "  ]",
             "}")), MessageSpec.class);
-        try {
-            new MessageDataGenerator("org.apache.kafka.common.message").generate(testMessageSpec);
-            fail("Expected MessageDataGenerator#generate to fail");
-        } catch (Throwable e) {
-            assertTrue("Invalid error message: " + e.getMessage(),
-                    e.getMessage().contains("Invalid default for int32"));
-        }
+        assertStringContains("Invalid default for int32",
+            assertThrows(RuntimeException.class, () -> {
+                new MessageDataGenerator("org.apache.kafka.common.message").generate(testMessageSpec);
+            }).getMessage());
     }
 
     @Test
@@ -82,12 +85,175 @@ public void testInvalidNullDefaultForPotentiallyNonNullableArray() throws Except
                 "    \"default\": \"null\" }",
                 "  ]",
                 "}")), MessageSpec.class);
-        try {
-            new MessageDataGenerator("org.apache.kafka.common.message").generate(testMessageSpec);
-            fail("Expected MessageDataGenerator#generate to fail");
-        } catch (RuntimeException e) {
-            assertTrue("Invalid error message: " + e.getMessage(),
-                    e.getMessage().contains("not all versions of this field are nullable"));
-        }
+
+        assertStringContains("not all versions of this field are nullable",
+            assertThrows(RuntimeException.class, () -> {
+                new MessageDataGenerator("org.apache.kafka.common.message").generate(testMessageSpec);
+            }).getMessage());
+    }
+
+    /**
+     * Test attempting to create a field with an invalid name.  The name is
+     * invalid because it starts with an underscore.
+     */
+    @Test
+    public void testInvalidFieldName() {
+        assertStringContains("Invalid field name",
+            assertThrows(Throwable.class, () -> {
+                MessageGenerator.JSON_SERDE.readValue(String.join("", Arrays.asList(
+                    "{",
+                    "  \"type\": \"request\",",
+                    "  \"name\": \"FooBar\",",
+                    "  \"validVersions\": \"0-2\",",
+                    "  \"fields\": [",
+                    "    { \"name\": \"_badName\", \"type\": \"[]int32\", \"versions\": \"0+\" }",
+                    "  ]",
+                    "}")), MessageSpec.class);
+            }).getMessage());
+    }
+
+    @Test
+    public void testInvalidTagWithoutTaggedVersions() {
+        assertStringContains("If a tag is specified, taggedVersions must be specified as well.",
+            assertThrows(Throwable.class, () -> {
+                MessageGenerator.JSON_SERDE.readValue(String.join("", Arrays.asList(
+                    "{",
+                    "  \"type\": \"request\",",
+                    "  \"name\": \"FooBar\",",
+                    "  \"validVersions\": \"0-2\",",
+                    "  \"flexibleVersions\": \"0+\",",
+                    "  \"fields\": [",
+                    "    { \"name\": \"field1\", \"type\": \"int32\", \"versions\": \"0+\", \"tag\": 0 }",
+                    "  ]",
+                    "}")), MessageSpec.class);
+                fail("Expected the MessageSpec constructor to fail");
+            }).getMessage());
+    }
+
+    @Test
+    public void testInvalidNegativeTag() {
+        assertStringContains("Tags cannot be negative",
+            assertThrows(Throwable.class, () -> {
+                MessageGenerator.JSON_SERDE.readValue(String.join("", Arrays.asList(
+                    "{",
+                    "  \"type\": \"request\",",
+                    "  \"name\": \"FooBar\",",
+                    "  \"validVersions\": \"0-2\",",
+                    "  \"flexibleVersions\": \"0+\",",
+                    "  \"fields\": [",
+                    "    { \"name\": \"field1\", \"type\": \"int32\", \"versions\": \"0+\", ",
+                    "        \"tag\": -1, \"taggedVersions\": \"0+\" }",
+                    "  ]",
+                    "}")), MessageSpec.class);
+            }).getMessage());
+    }
+
+    @Test
+    public void testInvalidFlexibleVersionsRange() {
+        assertStringContains("flexibleVersions must be either none, or an open-ended range",
+            assertThrows(Throwable.class, () -> {
+                MessageGenerator.JSON_SERDE.readValue(String.join("", Arrays.asList(
+                    "{",
+                    "  \"type\": \"request\",",
+                    "  \"name\": \"FooBar\",",
+                    "  \"validVersions\": \"0-2\",",
+                    "  \"flexibleVersions\": \"0-2\",",
+                    "  \"fields\": [",
+                    "    { \"name\": \"field1\", \"type\": \"int32\", \"versions\": \"0+\" }",
+                    "  ]",
+                    "}")), MessageSpec.class);
+            }).getMessage());
+    }
+
+    @Test
+    public void testInvalidSometimesNullableTaggedField() {
+        assertStringContains("Either all tagged versions must be nullable, or none must be",
+            assertThrows(Throwable.class, () -> {
+                MessageGenerator.JSON_SERDE.readValue(String.join("", Arrays.asList(
+                    "{",
+                    "  \"type\": \"request\",",
+                    "  \"name\": \"FooBar\",",
+                    "  \"validVersions\": \"0-2\",",
+                    "  \"flexibleVersions\": \"0+\",",
+                    "  \"fields\": [",
+                    "    { \"name\": \"field1\", \"type\": \"string\", \"versions\": \"0+\", ",
+                    "        \"tag\": 0, \"taggedVersions\": \"0+\", \"nullableVersions\": \"1+\" }",
+                    "  ]",
+                    "}")), MessageSpec.class);
+            }).getMessage());
+    }
+
+    @Test
+    public void testInvalidTaggedVersionsNotASubetOfVersions() {
+        assertStringContains("taggedVersions must be a subset of versions",
+            assertThrows(Throwable.class, () -> {
+                MessageGenerator.JSON_SERDE.readValue(String.join("", Arrays.asList(
+                    "{",
+                    "  \"type\": \"request\",",
+                    "  \"name\": \"FooBar\",",
+                    "  \"validVersions\": \"0-2\",",
+                    "  \"flexibleVersions\": \"0+\",",
+                    "  \"fields\": [",
+                    "    { \"name\": \"field1\", \"type\": \"string\", \"versions\": \"0-2\", ",
+                    "        \"tag\": 0, \"taggedVersions\": \"1+\" }",
+                    "  ]",
+                    "}")), MessageSpec.class);
+            }).getMessage());
+    }
+
+    @Test
+    public void testInvalidTaggedVersionsWithoutTag() {
+        assertStringContains("Please specify a tag, or remove the taggedVersions",
+            assertThrows(Throwable.class, () -> {
+                MessageGenerator.JSON_SERDE.readValue(String.join("", Arrays.asList(
+                    "{",
+                    "  \"type\": \"request\",",
+                    "  \"name\": \"FooBar\",",
+                    "  \"validVersions\": \"0-2\",",
+                    "  \"flexibleVersions\": \"0+\",",
+                    "  \"fields\": [",
+                    "    { \"name\": \"field1\", \"type\": \"string\", \"versions\": \"0+\", ",
+                    "        \"taggedVersions\": \"1+\" }",
+                    "  ]",
+                    "}")), MessageSpec.class);
+            }).getMessage());
+    }
+
+    @Test
+    public void testInvalidTaggedVersionsRange() {
+        assertStringContains("taggedVersions must be either none, or an open-ended range",
+            assertThrows(Throwable.class, () -> {
+                MessageGenerator.JSON_SERDE.readValue(String.join("", Arrays.asList(
+                    "{",
+                    "  \"type\": \"request\",",
+                    "  \"name\": \"FooBar\",",
+                    "  \"validVersions\": \"0-2\",",
+                    "  \"flexibleVersions\": \"0+\",",
+                    "  \"fields\": [",
+                    "    { \"name\": \"field1\", \"type\": \"string\", \"versions\": \"0+\", ",
+                    "        \"tag\": 0, \"taggedVersions\": \"1-2\" }",
+                    "  ]",
+                    "}")), MessageSpec.class);
+            }).getMessage());
+    }
+
+    @Test
+    public void testDuplicateTags() {
+        assertStringContains("duplicate tag",
+            assertThrows(Throwable.class, () -> {
+                MessageGenerator.JSON_SERDE.readValue(String.join("", Arrays.asList(
+                    "{",
+                    "  \"type\": \"request\",",
+                    "  \"name\": \"FooBar\",",
+                    "  \"validVersions\": \"0-2\",",
+                    "  \"flexibleVersions\": \"0+\",",
+                    "  \"fields\": [",
+                    "    { \"name\": \"field1\", \"type\": \"string\", \"versions\": \"0+\", ",
+                    "        \"tag\": 0, \"taggedVersions\": \"0+\" },",
+                    "    { \"name\": \"field2\", \"type\": \"int64\", \"versions\": \"0+\", ",
+                    "        \"tag\": 0, \"taggedVersions\": \"0+\" }",
+                    "  ]",
+                    "}")), MessageSpec.class);
+            }).getMessage());
     }
 }
diff --git a/generator/src/test/java/org/apache/kafka/message/VersionConditionalTest.java b/generator/src/test/java/org/apache/kafka/message/VersionConditionalTest.java
index 97180f7810b4f..acf78c9deb53f 100644
--- a/generator/src/test/java/org/apache/kafka/message/VersionConditionalTest.java
+++ b/generator/src/test/java/org/apache/kafka/message/VersionConditionalTest.java
@@ -140,7 +140,7 @@ public void testLowerRangeCheckWithElse() throws Exception {
             }).
             generate(buffer);
         assertEquals(buffer,
-            "if (version >= 1) {%n",
+            "if (_version >= 1) {%n",
             "    System.out.println(\"hello world\");%n",
             "} else {%n",
             "    System.out.println(\"foobar\");%n",
@@ -157,7 +157,7 @@ public void testLowerRangeCheckWithIfMember() throws Exception {
             }).
             generate(buffer);
         assertEquals(buffer,
-            "if (version >= 1) {%n",
+            "if (_version >= 1) {%n",
             "    System.out.println(\"hello world\");%n",
             "}%n");
     }
@@ -172,7 +172,7 @@ public void testLowerRangeCheckWithIfNotMember() throws Exception {
             }).
             generate(buffer);
         assertEquals(buffer,
-            "if (version < 1) {%n",
+            "if (_version < 1) {%n",
             "    System.out.println(\"hello world\");%n",
             "}%n");
     }
@@ -190,7 +190,7 @@ public void testUpperRangeCheckWithElse() throws Exception {
             }).
             generate(buffer);
         assertEquals(buffer,
-            "if (version <= 10) {%n",
+            "if (_version <= 10) {%n",
             "    System.out.println(\"hello world\");%n",
             "} else {%n",
             "    System.out.println(\"foobar\");%n",
@@ -207,7 +207,7 @@ public void testUpperRangeCheckWithIfMember() throws Exception {
             }).
             generate(buffer);
         assertEquals(buffer,
-            "if (version <= 10) {%n",
+            "if (_version <= 10) {%n",
             "    System.out.println(\"hello world\");%n",
             "}%n");
     }
@@ -222,7 +222,7 @@ public void testUpperRangeCheckWithIfNotMember() throws Exception {
             }).
             generate(buffer);
         assertEquals(buffer,
-            "if (version < 1) {%n",
+            "if (_version < 1) {%n",
             "    System.out.println(\"hello world\");%n",
             "}%n");
     }
@@ -238,7 +238,7 @@ public void testFullRangeCheck() throws Exception {
             allowMembershipCheckAlwaysFalse(false).
             generate(buffer);
         assertEquals(buffer,
-            "if ((version >= 5) && (version <= 10)) {%n",
+            "if ((_version >= 5) && (_version <= 10)) {%n",
             "    System.out.println(\"hello world\");%n",
             "}%n");
     }

From a5a6938c69f4310f7ec519036f0df77d8022326a Mon Sep 17 00:00:00 2001
From: Jukka Karvanen <48978068+jukkakarvanen@users.noreply.github.com>
Date: Mon, 7 Oct 2019 11:01:58 +0300
Subject: [PATCH 0693/1071] KAFKA-8233: TopologyTestDriver test input and
 output usability improvements (#7378)

Implements KIP-470

Reviewers: Bill Bejeck , John Roesler , Matthias J. Sax 
---
 build.gradle                                  |   1 +
 docs/streams/developer-guide/testing.html     |  95 ++--
 .../examples/wordcount/WordCountDemo.java     |  20 +-
 .../examples/docs/DeveloperGuideTesting.java  | 187 ++++++++
 .../examples/wordcount/WordCountDemoTest.java | 112 +++++
 .../kafka/streams/StreamsBuilderTest.java     |  50 +--
 .../KStreamTransformIntegrationTest.java      |  10 +-
 .../kstream/internals/AbstractStreamTest.java |   6 +-
 .../internals/GlobalKTableJoinsTest.java      |  15 +-
 .../internals/KGroupedStreamImplTest.java     |  88 ++--
 .../internals/KGroupedTableImplTest.java      |  32 +-
 .../kstream/internals/KStreamBranchTest.java  |   6 +-
 .../kstream/internals/KStreamFilterTest.java  |   9 +-
 .../kstream/internals/KStreamFlatMapTest.java |  10 +-
 .../internals/KStreamFlatMapValuesTest.java   |  18 +-
 .../kstream/internals/KStreamForeachTest.java |   6 +-
 .../KStreamGlobalKTableJoinTest.java          |  22 +-
 .../KStreamGlobalKTableLeftJoinTest.java      |  22 +-
 .../kstream/internals/KStreamImplTest.java    | 100 +++--
 .../internals/KStreamKStreamJoinTest.java     | 161 ++++---
 .../internals/KStreamKStreamLeftJoinTest.java |  57 ++-
 .../internals/KStreamKTableJoinTest.java      |  23 +-
 .../internals/KStreamKTableLeftJoinTest.java  |  19 +-
 .../kstream/internals/KStreamMapTest.java     |  10 +-
 .../internals/KStreamMapValuesTest.java       |  12 +-
 .../kstream/internals/KStreamPeekTest.java    |   6 +-
 .../internals/KStreamSelectKeyTest.java       |  10 +-
 .../internals/KStreamTransformTest.java       |  27 +-
 .../internals/KStreamTransformValuesTest.java |  12 +-
 .../internals/KStreamWindowAggregateTest.java | 161 +++----
 .../internals/KTableAggregateTest.java        |  83 ++--
 .../kstream/internals/KTableFilterTest.java   |  86 ++--
 .../kstream/internals/KTableImplTest.java     |  18 +-
 .../internals/KTableKTableInnerJoinTest.java  | 170 ++++----
 .../internals/KTableKTableLeftJoinTest.java   | 192 ++++----
 .../internals/KTableKTableOuterJoinTest.java  | 185 ++++----
 .../kstream/internals/KTableMapKeysTest.java  |   8 +-
 .../internals/KTableMapValuesTest.java        |  64 +--
 .../kstream/internals/KTableSourceTest.java   |  80 ++--
 .../internals/KTableTransformValuesTest.java  |  38 +-
 .../SessionWindowedKStreamImplTest.java       |  16 +-
 .../internals/SuppressScenarioTest.java       | 197 ++++-----
 .../TimeWindowedKStreamImplTest.java          |  16 +-
 .../internals/ProcessorTopologyTest.java      | 339 +++++++-------
 .../internals/CachingWindowStoreTest.java     |  31 +-
 .../streams/scala/kstream/KStreamTest.scala   |  73 ++--
 .../streams/scala/kstream/KTableTest.scala    | 176 ++++----
 .../streams/scala/utils/TestDriver.scala      |  23 +-
 .../apache/kafka/streams/TestInputTopic.java  | 256 +++++++++++
 .../apache/kafka/streams/TestOutputTopic.java | 198 +++++++++
 .../kafka/streams/TopologyTestDriver.java     | 306 ++++++++++---
 .../streams/test/ConsumerRecordFactory.java   |   4 +
 .../kafka/streams/test/OutputVerifier.java    |   4 +
 .../apache/kafka/streams/test/TestRecord.java | 250 +++++++++++
 .../apache/kafka/streams/TestTopicsTest.java  | 412 ++++++++++++++++++
 .../kafka/streams/TopologyTestDriverTest.java | 338 +++++++++-----
 .../test/ConsumerRecordFactoryTest.java       |   1 +
 .../streams/test/OutputVerifierTest.java      |   1 +
 .../kafka/streams/test/TestRecordTest.java    | 168 +++++++
 59 files changed, 3595 insertions(+), 1445 deletions(-)
 create mode 100644 streams/examples/src/test/java/org/apache/kafka/streams/examples/docs/DeveloperGuideTesting.java
 create mode 100644 streams/examples/src/test/java/org/apache/kafka/streams/examples/wordcount/WordCountDemoTest.java
 create mode 100644 streams/test-utils/src/main/java/org/apache/kafka/streams/TestInputTopic.java
 create mode 100644 streams/test-utils/src/main/java/org/apache/kafka/streams/TestOutputTopic.java
 create mode 100644 streams/test-utils/src/main/java/org/apache/kafka/streams/test/TestRecord.java
 create mode 100644 streams/test-utils/src/test/java/org/apache/kafka/streams/TestTopicsTest.java
 create mode 100644 streams/test-utils/src/test/java/org/apache/kafka/streams/test/TestRecordTest.java

diff --git a/build.gradle b/build.gradle
index 210112bb13df5..1c567e65e88dc 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1284,6 +1284,7 @@ project(':streams:test-utils') {
     testCompile project(':clients').sourceSets.test.output
     testCompile libs.junit
     testCompile libs.easymock
+    testCompile libs.hamcrest
 
     testRuntime libs.slf4jlog4j
   }
diff --git a/docs/streams/developer-guide/testing.html b/docs/streams/developer-guide/testing.html
index 55a89b084abe0..5be2255dc8bd3 100644
--- a/docs/streams/developer-guide/testing.html
+++ b/docs/streams/developer-guide/testing.html
@@ -92,30 +92,23 @@ 
                
             
                
-                To verify the output, the test driver produces ProducerRecords with key and value type
-                byte[].
-                For result verification, you can specify corresponding deserializers when reading the output record from
-                the driver.
-            
                -ProducerRecord<String, Integer> outputRecord = testDriver.readOutput("output-topic", new StringDeserializer(), new LongDeserializer());
-        
                
-            
                
-                For result verification, you can use OutputVerifier.
-                It offers helper methods to compare only certain parts of the result record:
-                for example, you might only care about the key and value, but not the timestamp of the result record.
+                To verify the output, you can use TestOutputTopic
+                where you configure the topic and the corresponding deserializers during initialization.
+                It offers helper methods to read only certain parts of the result records or the collection of records.
+                For example, you can validate returned KeyValue with standard assertions
+                if you only care about the key and value, but not the timestamp of the result record.
             
                
             
                -OutputVerifier.compareKeyValue(outputRecord, "key", 42L); // throws AssertionError if key or value does not match
+TestOutputTopic<String, Long> outputTopic = testDriver.createOutputTopic("result-topic", stringSerde.deserializer(), longSerde.deserializer());
+assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>("key", 42L)));
         
                
             
                
                 TopologyTestDriver supports punctuations, too.
@@ -124,7 +117,7 @@ 
                Example
                
 
 @Test
 public void shouldFlushStoreForFirstInput() {
-    testDriver.pipeInput(recordFactory.create("input-topic", "a", 1L, 9999L));
-    OutputVerifier.compareKeyValue(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer), "a", 21L);
-    Assert.assertNull(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer));
+    inputTopic.pipeInput("a", 1L);
+    assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>("a", 21L)));
+    assertThat(outputTopic.isEmpty(), is(true));
 }
 
 @Test
 public void shouldNotUpdateStoreForSmallerValue() {
-    testDriver.pipeInput(recordFactory.create("input-topic", "a", 1L, 9999L));
-    Assert.assertThat(store.get("a"), equalTo(21L));
-    OutputVerifier.compareKeyValue(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer), "a", 21L);
-    Assert.assertNull(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer));
+    inputTopic.pipeInput("a", 1L);
+    assertThat(store.get("a"), equalTo(21L));
+    assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>("a", 21L)));
+    assertThat(outputTopic.isEmpty(), is(true));
 }
 
 @Test
 public void shouldNotUpdateStoreForLargerValue() {
-    testDriver.pipeInput(recordFactory.create("input-topic", "a", 42L, 9999L));
-    Assert.assertThat(store.get("a"), equalTo(42L));
-    OutputVerifier.compareKeyValue(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer), "a", 42L);
-    Assert.assertNull(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer));
+    inputTopic.pipeInput("a", 42L);
+    assertThat(store.get("a"), equalTo(42L));
+    assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>("a", 42L)));
+    assertThat(outputTopic.isEmpty(), is(true));
 }
 
 @Test
 public void shouldUpdateStoreForNewKey() {
-    testDriver.pipeInput(recordFactory.create("input-topic", "b", 21L, 9999L));
-    Assert.assertThat(store.get("b"), equalTo(21L));
-    OutputVerifier.compareKeyValue(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer), "a", 21L);
-    OutputVerifier.compareKeyValue(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer), "b", 21L);
-    Assert.assertNull(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer));
+    inputTopic.pipeInput("b", 21L);
+    assertThat(store.get("b"), equalTo(21L));
+    assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>("a", 21L)));
+    assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>("b", 21L)));
+    assertThat(outputTopic.isEmpty(), is(true));
 }
 
 @Test
 public void shouldPunctuateIfEvenTimeAdvances() {
-    testDriver.pipeInput(recordFactory.create("input-topic", "a", 1L, 9999L));
-    OutputVerifier.compareKeyValue(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer), "a", 21L);
+    final Instant recordTime = Instant.now();
+    inputTopic.pipeInput("a", 1L,  recordTime);
+    assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>("a", 21L)));
 
-    testDriver.pipeInput(recordFactory.create("input-topic", "a", 1L, 9999L));
-    Assert.assertNull(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer));
+    inputTopic.pipeInput("a", 1L,  recordTime);
+    assertThat(outputTopic.isEmpty(), is(true));
 
-    testDriver.pipeInput(recordFactory.create("input-topic", "a", 1L, 10000L));
-    OutputVerifier.compareKeyValue(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer), "a", 21L);
-    Assert.assertNull(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer));
+    inputTopic.pipeInput("a", 1L, recordTime.plusSeconds(10L));
+    assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>("a", 21L)));
+    assertThat(outputTopic.isEmpty(), is(true));
 }
 
 @Test
 public void shouldPunctuateIfWallClockTimeAdvances() {
-    testDriver.advanceWallClockTime(60000);
-    OutputVerifier.compareKeyValue(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer), "a", 21L);
-    Assert.assertNull(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer));
+    testDriver.advanceWallClockTime(Duration.ofSeconds(60));
+    assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>("a", 21L)));
+    assertThat(outputTopic.isEmpty(), is(true));
 }
 
 public class CustomMaxAggregatorSupplier implements ProcessorSupplier<String, Long> {
diff --git a/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountDemo.java b/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountDemo.java
index 931a8db11725d..e331b8a299660 100644
--- a/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountDemo.java
+++ b/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountDemo.java
@@ -44,7 +44,10 @@
  */
 public final class WordCountDemo {
 
-    public static void main(final String[] args) {
+    public static final String INPUT_TOPIC = "streams-plaintext-input";
+    public static final String OUTPUT_TOPIC = "streams-wordcount-output";
+
+    static Properties getStreamsConfig() {
         final Properties props = new Properties();
         props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-wordcount");
         props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
@@ -56,10 +59,11 @@ public static void main(final String[] args) {
         // Note: To re-run the demo, you need to use the offset reset tool:
         // https://cwiki.apache.org/confluence/display/KAFKA/Kafka+Streams+Application+Reset+Tool
         props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+        return props;
+    }
 
-        final StreamsBuilder builder = new StreamsBuilder();
-
-        final KStream source = builder.stream("streams-plaintext-input");
+    static void createWordCountStream(final StreamsBuilder builder) {
+        final KStream source = builder.stream(INPUT_TOPIC);
 
         final KTable counts = source
             .flatMapValues(value -> Arrays.asList(value.toLowerCase(Locale.getDefault()).split(" ")))
@@ -67,8 +71,14 @@ public static void main(final String[] args) {
             .count();
 
         // need to override value serde to Long type
-        counts.toStream().to("streams-wordcount-output", Produced.with(Serdes.String(), Serdes.Long()));
+        counts.toStream().to(OUTPUT_TOPIC, Produced.with(Serdes.String(), Serdes.Long()));
+    }
+
+    public static void main(final String[] args) {
+        final Properties props = getStreamsConfig();
 
+        final StreamsBuilder builder = new StreamsBuilder();
+        createWordCountStream(builder);
         final KafkaStreams streams = new KafkaStreams(builder.build(), props);
         final CountDownLatch latch = new CountDownLatch(1);
 
diff --git a/streams/examples/src/test/java/org/apache/kafka/streams/examples/docs/DeveloperGuideTesting.java b/streams/examples/src/test/java/org/apache/kafka/streams/examples/docs/DeveloperGuideTesting.java
new file mode 100644
index 0000000000000..e380f20cd1869
--- /dev/null
+++ b/streams/examples/src/test/java/org/apache/kafka/streams/examples/docs/DeveloperGuideTesting.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.examples.docs;
+
+import org.apache.kafka.common.serialization.Serde;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.Topology;
+import org.apache.kafka.streams.TopologyTestDriver;
+import org.apache.kafka.streams.processor.Processor;
+import org.apache.kafka.streams.processor.ProcessorContext;
+import org.apache.kafka.streams.processor.ProcessorSupplier;
+import org.apache.kafka.streams.processor.PunctuationType;
+import org.apache.kafka.streams.state.KeyValueIterator;
+import org.apache.kafka.streams.state.KeyValueStore;
+import org.apache.kafka.streams.state.Stores;
+import org.apache.kafka.streams.TestInputTopic;
+import org.apache.kafka.streams.TestOutputTopic;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Properties;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+
+/**
+ * This is code sample in docs/streams/developer-guide/testing.html
+ */
+
+public class DeveloperGuideTesting {
+    private TopologyTestDriver testDriver;
+    private TestInputTopic inputTopic;
+    private TestOutputTopic outputTopic;
+    private KeyValueStore store;
+
+    private Serde stringSerde = new Serdes.StringSerde();
+    private Serde longSerde = new Serdes.LongSerde();
+
+    @Before
+    public void setup() {
+        final Topology topology = new Topology();
+        topology.addSource("sourceProcessor", "input-topic");
+        topology.addProcessor("aggregator", new CustomMaxAggregatorSupplier(), "sourceProcessor");
+        topology.addStateStore(
+                Stores.keyValueStoreBuilder(
+                        Stores.inMemoryKeyValueStore("aggStore"),
+                        Serdes.String(),
+                        Serdes.Long()).withLoggingDisabled(), // need to disable logging to allow store pre-populating
+                "aggregator");
+        topology.addSink("sinkProcessor", "result-topic", "aggregator");
+
+        // setup test driver
+        final Properties props = new Properties();
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "maxAggregation");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy:1234");
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.Long().getClass().getName());
+        testDriver = new TopologyTestDriver(topology, props);
+
+        // setup test topics
+        inputTopic = testDriver.createInputTopic("input-topic", stringSerde.serializer(), longSerde.serializer());
+        outputTopic = testDriver.createOutputTopic("result-topic", stringSerde.deserializer(), longSerde.deserializer());
+
+        // pre-populate store
+        store = testDriver.getKeyValueStore("aggStore");
+        store.put("a", 21L);
+    }
+
+    @After
+    public void tearDown() {
+        testDriver.close();
+    }
+
+
+    @Test
+    public void shouldFlushStoreForFirstInput() {
+        inputTopic.pipeInput("a", 1L);
+        assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>("a", 21L)));
+        assertThat(outputTopic.isEmpty(), is(true));
+    }
+
+    @Test
+    public void shouldNotUpdateStoreForSmallerValue() {
+        inputTopic.pipeInput("a", 1L);
+        assertThat(store.get("a"), equalTo(21L));
+        assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>("a", 21L)));
+        assertThat(outputTopic.isEmpty(), is(true));
+    }
+
+    @Test
+    public void shouldNotUpdateStoreForLargerValue() {
+        inputTopic.pipeInput("a", 42L);
+        assertThat(store.get("a"), equalTo(42L));
+        assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>("a", 42L)));
+        assertThat(outputTopic.isEmpty(), is(true));
+    }
+
+    @Test
+    public void shouldUpdateStoreForNewKey() {
+        inputTopic.pipeInput("b", 21L);
+        assertThat(store.get("b"), equalTo(21L));
+        assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>("a", 21L)));
+        assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>("b", 21L)));
+        assertThat(outputTopic.isEmpty(), is(true));
+    }
+
+    @Test
+    public void shouldPunctuateIfEvenTimeAdvances() {
+        final Instant recordTime = Instant.now();
+        inputTopic.pipeInput("a", 1L,  recordTime);
+        assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>("a", 21L)));
+
+        inputTopic.pipeInput("a", 1L,  recordTime);
+        assertThat(outputTopic.isEmpty(), is(true));
+
+        inputTopic.pipeInput("a", 1L, recordTime.plusSeconds(10L));
+        assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>("a", 21L)));
+        assertThat(outputTopic.isEmpty(), is(true));
+    }
+
+    @Test
+    public void shouldPunctuateIfWallClockTimeAdvances() {
+        testDriver.advanceWallClockTime(Duration.ofSeconds(60));
+        assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>("a", 21L)));
+        assertThat(outputTopic.isEmpty(), is(true));
+    }
+
+    public class CustomMaxAggregatorSupplier implements ProcessorSupplier {
+        @Override
+        public Processor get() {
+            return new CustomMaxAggregator();
+        }
+    }
+
+    public class CustomMaxAggregator implements Processor {
+        ProcessorContext context;
+        private KeyValueStore store;
+
+        @SuppressWarnings("unchecked")
+        @Override
+        public void init(final ProcessorContext context) {
+            this.context = context;
+            context.schedule(Duration.ofSeconds(60), PunctuationType.WALL_CLOCK_TIME, time -> flushStore());
+            context.schedule(Duration.ofSeconds(10), PunctuationType.STREAM_TIME, time -> flushStore());
+            store = (KeyValueStore) context.getStateStore("aggStore");
+        }
+
+        @Override
+        public void process(final String key, final Long value) {
+            final Long oldValue = store.get(key);
+            if (oldValue == null || value > oldValue) {
+                store.put(key, value);
+            }
+        }
+
+        private void flushStore() {
+            final KeyValueIterator it = store.all();
+            while (it.hasNext()) {
+                final KeyValue next = it.next();
+                context.forward(next.key, next.value);
+            }
+        }
+
+        @Override
+        public void close() {}
+    }
+}
diff --git a/streams/examples/src/test/java/org/apache/kafka/streams/examples/wordcount/WordCountDemoTest.java b/streams/examples/src/test/java/org/apache/kafka/streams/examples/wordcount/WordCountDemoTest.java
new file mode 100644
index 0000000000000..05807371a61fa
--- /dev/null
+++ b/streams/examples/src/test/java/org/apache/kafka/streams/examples/wordcount/WordCountDemoTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.examples.wordcount;
+
+import org.apache.kafka.common.serialization.LongDeserializer;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.TopologyTestDriver;
+import org.apache.kafka.streams.TestInputTopic;
+import org.apache.kafka.streams.TestOutputTopic;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+/**
+ * Unit test of {@link WordCountDemo} stream using TopologyTestDriver.
+ */
+public class WordCountDemoTest {
+
+    private TopologyTestDriver testDriver;
+    private TestInputTopic inputTopic;
+    private TestOutputTopic outputTopic;
+
+    @Before
+    public void setup() {
+        final StreamsBuilder builder = new StreamsBuilder();
+        //Create Actual Stream Processing pipeline
+        WordCountDemo.createWordCountStream(builder);
+        testDriver = new TopologyTestDriver(builder.build(), WordCountDemo.getStreamsConfig());
+        inputTopic = testDriver.createInputTopic(WordCountDemo.INPUT_TOPIC, new StringSerializer(), new StringSerializer());
+        outputTopic = testDriver.createOutputTopic(WordCountDemo.OUTPUT_TOPIC, new StringDeserializer(), new LongDeserializer());
+    }
+
+    @After
+    public void tearDown() {
+        try {
+            testDriver.close();
+        } catch (final RuntimeException e) {
+            // https://issues.apache.org/jira/browse/KAFKA-6647 causes exception when executed in Windows, ignoring it
+            // Logged stacktrace cannot be avoided
+            System.out.println("Ignoring exception, test failing in Windows due this exception:" + e.getLocalizedMessage());
+        }
+    }
+
+
+    /**
+     * Simple test validating count of one word
+     */
+    @Test
+    public void testOneWord() {
+        //Feed word "Hello" to inputTopic and no kafka key, timestamp is irrelevant in this case
+        inputTopic.pipeInput("Hello");
+        //Read and validate output to match word as key and count as value
+        assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>("hello", 1L)));
+        //No more output in topic
+        assertThat(outputTopic.isEmpty(), is(true));
+    }
+
+    /**
+     * Test Word count of sentence list.
+     */
+    @Test
+    public void testCountListOfWords() {
+        final List inputValues = Arrays.asList(
+                "Apache Kafka Streams Example",
+                "Using Kafka Streams Test Utils",
+                "Reading and Writing Kafka Topic"
+        );
+        final Map expectedWordCounts = new HashMap<>();
+        expectedWordCounts.put("apache", 1L);
+        expectedWordCounts.put("kafka", 3L);
+        expectedWordCounts.put("streams", 2L);
+        expectedWordCounts.put("example", 1L);
+        expectedWordCounts.put("using", 1L);
+        expectedWordCounts.put("test", 1L);
+        expectedWordCounts.put("utils", 1L);
+        expectedWordCounts.put("reading", 1L);
+        expectedWordCounts.put("and", 1L);
+        expectedWordCounts.put("writing", 1L);
+        expectedWordCounts.put("topic", 1L);
+
+        inputTopic.pipeValueList(inputValues);
+        final Map actualWordCounts = outputTopic.readKeyValuesToMap();
+        assertThat(actualWordCounts, equalTo(expectedWordCounts));
+    }
+
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java
index 3536c016c45db..411829771ef77 100644
--- a/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java
@@ -41,7 +41,6 @@
 import org.apache.kafka.streams.processor.internals.ProcessorNode;
 import org.apache.kafka.streams.processor.internals.ProcessorTopology;
 import org.apache.kafka.streams.state.KeyValueStore;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockMapper;
 import org.apache.kafka.test.MockPredicate;
 import org.apache.kafka.test.MockProcessorSupplier;
@@ -50,6 +49,7 @@
 import org.junit.Test;
 
 import java.time.Duration;
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -254,11 +254,10 @@ public void shouldProcessingFromSinkTopic() {
         final MockProcessorSupplier processorSupplier = new MockProcessorSupplier<>();
         source.process(processorSupplier);
 
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L);
-
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create("topic-source", "A", "aa"));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic("topic-source", new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            inputTopic.pipeInput("A", "aa");
         }
 
         // no exception was thrown
@@ -277,11 +276,10 @@ public void shouldProcessViaThroughTopic() {
         final MockProcessorSupplier throughProcessorSupplier = new MockProcessorSupplier<>();
         through.process(throughProcessorSupplier);
 
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L);
-
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create("topic-source", "A", "aa"));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic("topic-source", new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            inputTopic.pipeInput("A", "aa");
         }
 
         assertEquals(Collections.singletonList(new KeyValueTimestamp<>("A", "aa", 0)), sourceProcessorSupplier.theCapturedProcessor().processed);
@@ -300,14 +298,16 @@ public void shouldMergeStreams() {
         final MockProcessorSupplier processorSupplier = new MockProcessorSupplier<>();
         merged.process(processorSupplier);
 
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L);
-
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(topic1, "A", "aa"));
-            driver.pipeInput(recordFactory.create(topic2, "B", "bb"));
-            driver.pipeInput(recordFactory.create(topic2, "C", "cc"));
-            driver.pipeInput(recordFactory.create(topic1, "D", "dd"));
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic2 =
+                    driver.createInputTopic(topic2, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+
+            inputTopic1.pipeInput("A", "aa");
+            inputTopic2.pipeInput("B", "bb");
+            inputTopic2.pipeInput("C", "cc");
+            inputTopic1.pipeInput("D", "dd");
         }
 
         assertEquals(asList(new KeyValueTimestamp<>("A", "aa", 0),
@@ -326,12 +326,11 @@ public void shouldUseSerdesDefinedInMaterializedToConsumeTable() {
                 .withValueSerde(Serdes.String()))
                 .toStream().foreach(action);
 
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(new LongSerializer(), new StringSerializer());
-
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(topic, 1L, "value1"));
-            driver.pipeInput(recordFactory.create(topic, 2L, "value2"));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(topic, new LongSerializer(), new StringSerializer());
+            inputTopic.pipeInput(1L, "value1");
+            inputTopic.pipeInput(2L, "value2");
 
             final KeyValueStore store = driver.getKeyValueStore("store");
             assertThat(store.get(1L), equalTo("value1"));
@@ -348,12 +347,11 @@ public void shouldUseSerdesDefinedInMaterializedToConsumeGlobalTable() {
                 .withKeySerde(Serdes.Long())
                 .withValueSerde(Serdes.String()));
 
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(new LongSerializer(), new StringSerializer());
-
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(topic, 1L, "value1"));
-            driver.pipeInput(recordFactory.create(topic, 2L, "value2"));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(topic, new LongSerializer(), new StringSerializer());
+            inputTopic.pipeInput(1L, "value1");
+            inputTopic.pipeInput(2L, "value2");
             final KeyValueStore store = driver.getKeyValueStore("store");
 
             assertThat(store.get(1L), equalTo("value1"));
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamTransformIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamTransformIntegrationTest.java
index fa4e33a700ebe..351b9f795a773 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamTransformIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamTransformIntegrationTest.java
@@ -30,7 +30,7 @@
 import org.apache.kafka.streams.state.KeyValueStore;
 import org.apache.kafka.streams.state.StoreBuilder;
 import org.apache.kafka.streams.state.Stores;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.test.IntegrationTest;
 import org.apache.kafka.test.StreamsTestUtils;
@@ -70,16 +70,16 @@ public void before() {
     }
 
     private void verifyResult(final List> expected) {
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(new IntegerSerializer(), new IntegerSerializer());
         final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.Integer());
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(topic, Arrays.asList(new KeyValue<>(1, 1),
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(topic, new IntegerSerializer(), new IntegerSerializer());
+            inputTopic.pipeKeyValueList(Arrays.asList(new KeyValue<>(1, 1),
                                                                        new KeyValue<>(2, 2),
                                                                        new KeyValue<>(3, 3),
                                                                        new KeyValue<>(2, 1),
                                                                        new KeyValue<>(2, 3),
-                                                                       new KeyValue<>(1, 3))));
+                                                                       new KeyValue<>(1, 3)));
         }
         assertThat(results, equalTo(expected));
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/AbstractStreamTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/AbstractStreamTest.java
index 89242eb91a227..f78eebbe7a231 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/AbstractStreamTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/AbstractStreamTest.java
@@ -31,7 +31,7 @@
 import org.apache.kafka.streams.processor.AbstractProcessor;
 import org.apache.kafka.streams.processor.Processor;
 import org.apache.kafka.streams.processor.ProcessorSupplier;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.TestUtils;
 import org.junit.Test;
@@ -88,10 +88,10 @@ public void testShouldBeExtensible() {
         props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
         props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
 
-        final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer());
         final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props);
+        final TestInputTopic inputTopic = driver.createInputTopic(topicName, new IntegerSerializer(), new StringSerializer());
         for (final int expectedKey : expectedKeys) {
-            driver.pipeInput(recordFactory.create(topicName, expectedKey, "V" + expectedKey));
+            inputTopic.pipeInput(expectedKey, "V" + expectedKey);
         }
 
         assertTrue(supplier.theCapturedProcessor().processed.size() <= expectedKeys.length);
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/GlobalKTableJoinsTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/GlobalKTableJoinsTest.java
index b29903f232d60..35c6c9cdc6489 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/GlobalKTableJoinsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/GlobalKTableJoinsTest.java
@@ -25,7 +25,7 @@
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KeyValueMapper;
 import org.apache.kafka.streams.state.ValueAndTimestamp;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockValueJoiner;
 import org.apache.kafka.test.StreamsTestUtils;
@@ -87,17 +87,18 @@ public void shouldInnerJoinWithStream() {
 
     private void verifyJoin(final Map> expected,
                             final MockProcessorSupplier supplier) {
-        final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
         final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String());
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic globalInputTopic = driver.createInputTopic(globalTopic, new StringSerializer(), new StringSerializer());
             // write some data to the global table
-            driver.pipeInput(recordFactory.create(globalTopic, "a", "A", 1L));
-            driver.pipeInput(recordFactory.create(globalTopic, "b", "B", 5L));
+            globalInputTopic.pipeInput("a", "A", 1L);
+            globalInputTopic.pipeInput("b", "B", 5L);
+            final TestInputTopic streamInputTopic = driver.createInputTopic(streamTopic, new StringSerializer(), new StringSerializer());
             //write some data to the stream
-            driver.pipeInput(recordFactory.create(streamTopic, "1", "a", 2L));
-            driver.pipeInput(recordFactory.create(streamTopic, "2", "b", 10L));
-            driver.pipeInput(recordFactory.create(streamTopic, "3", "c", 3L));
+            streamInputTopic.pipeInput("1", "a", 2L);
+            streamInputTopic.pipeInput("2", "b", 10L);
+            streamInputTopic.pipeInput("3", "c", 3L);
         }
 
         assertEquals(expected, supplier.theCapturedProcessor().lastValueAndTimestampPerKey);
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java
index d8c338942848c..ab2eb113bf8d0 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java
@@ -40,7 +40,7 @@
 import org.apache.kafka.streams.state.KeyValueStore;
 import org.apache.kafka.streams.state.SessionStore;
 import org.apache.kafka.streams.state.ValueAndTimestamp;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockAggregator;
 import org.apache.kafka.test.MockInitializer;
 import org.apache.kafka.test.MockProcessorSupplier;
@@ -72,8 +72,6 @@ public class KGroupedStreamImplTest {
     private final StreamsBuilder builder = new StreamsBuilder();
     private KGroupedStream groupedStream;
 
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String());
 
     @Before
@@ -157,12 +155,14 @@ public void shouldNotHaveInvalidStoreNameOnWindowedAggregate() {
 
     private void doAggregateSessionWindows(final MockProcessorSupplier, Integer> supplier) {
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 10));
-            driver.pipeInput(recordFactory.create(TOPIC, "2", "2", 15));
-            driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 30));
-            driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 70));
-            driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 100));
-            driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 90));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(TOPIC, new StringSerializer(), new StringSerializer());
+            inputTopic.pipeInput("1", "1", 10);
+            inputTopic.pipeInput("2", "2", 15);
+            inputTopic.pipeInput("1", "1", 30);
+            inputTopic.pipeInput("1", "1", 70);
+            inputTopic.pipeInput("1", "1", 100);
+            inputTopic.pipeInput("1", "1", 90);
         }
         final Map, ValueAndTimestamp> result
             = supplier.theCapturedProcessor().lastValueAndTimestampPerKey;
@@ -213,12 +213,14 @@ public void shouldAggregateSessionWindowsWithInternalStoreName() {
     private void doCountSessionWindows(final MockProcessorSupplier, Long> supplier) {
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 10));
-            driver.pipeInput(recordFactory.create(TOPIC, "2", "2", 15));
-            driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 30));
-            driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 70));
-            driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 100));
-            driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 90));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(TOPIC, new StringSerializer(), new StringSerializer());
+            inputTopic.pipeInput("1", "1", 10);
+            inputTopic.pipeInput("2", "2", 15);
+            inputTopic.pipeInput("1", "1", 30);
+            inputTopic.pipeInput("1", "1", 70);
+            inputTopic.pipeInput("1", "1", 100);
+            inputTopic.pipeInput("1", "1", 90);
         }
         final Map, ValueAndTimestamp> result =
             supplier.theCapturedProcessor().lastValueAndTimestampPerKey;
@@ -257,12 +259,14 @@ public void shouldCountSessionWindowsWithInternalStoreName() {
 
     private void doReduceSessionWindows(final MockProcessorSupplier, String> supplier) {
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(TOPIC, "1", "A", 10));
-            driver.pipeInput(recordFactory.create(TOPIC, "2", "Z", 15));
-            driver.pipeInput(recordFactory.create(TOPIC, "1", "B", 30));
-            driver.pipeInput(recordFactory.create(TOPIC, "1", "A", 70));
-            driver.pipeInput(recordFactory.create(TOPIC, "1", "B", 100));
-            driver.pipeInput(recordFactory.create(TOPIC, "1", "C", 90));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(TOPIC, new StringSerializer(), new StringSerializer());
+            inputTopic.pipeInput("1", "A", 10);
+            inputTopic.pipeInput("2", "Z", 15);
+            inputTopic.pipeInput("1", "B", 30);
+            inputTopic.pipeInput("1", "A", 70);
+            inputTopic.pipeInput("1", "B", 100);
+            inputTopic.pipeInput("1", "C", 90);
         }
         final Map, ValueAndTimestamp> result =
             supplier.theCapturedProcessor().lastValueAndTimestampPerKey;
@@ -551,29 +555,33 @@ public void shouldAggregateWithDefaultSerdes() {
     }
 
     private void processData(final TopologyTestDriver driver) {
-        driver.pipeInput(recordFactory.create(TOPIC, "1", "A", 5L));
-        driver.pipeInput(recordFactory.create(TOPIC, "2", "B", 1L));
-        driver.pipeInput(recordFactory.create(TOPIC, "1", "C", 3L));
-        driver.pipeInput(recordFactory.create(TOPIC, "1", "D", 10L));
-        driver.pipeInput(recordFactory.create(TOPIC, "3", "E", 8L));
-        driver.pipeInput(recordFactory.create(TOPIC, "3", "F", 9L));
-        driver.pipeInput(recordFactory.create(TOPIC, "3", (String) null));
+        final TestInputTopic inputTopic =
+                driver.createInputTopic(TOPIC, new StringSerializer(), new StringSerializer());
+        inputTopic.pipeInput("1", "A", 5L);
+        inputTopic.pipeInput("2", "B", 1L);
+        inputTopic.pipeInput("1", "C", 3L);
+        inputTopic.pipeInput("1", "D", 10L);
+        inputTopic.pipeInput("3", "E", 8L);
+        inputTopic.pipeInput("3", "F", 9L);
+        inputTopic.pipeInput("3", (String) null);
     }
 
     private void doCountWindowed(final  MockProcessorSupplier, Long> supplier) {
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(TOPIC, "1", "A", 0L));
-            driver.pipeInput(recordFactory.create(TOPIC, "1", "A", 499L));
-            driver.pipeInput(recordFactory.create(TOPIC, "1", "A", 100L));
-            driver.pipeInput(recordFactory.create(TOPIC, "2", "B", 0L));
-            driver.pipeInput(recordFactory.create(TOPIC, "2", "B", 100L));
-            driver.pipeInput(recordFactory.create(TOPIC, "2", "B", 200L));
-            driver.pipeInput(recordFactory.create(TOPIC, "3", "C", 1L));
-            driver.pipeInput(recordFactory.create(TOPIC, "1", "A", 500L));
-            driver.pipeInput(recordFactory.create(TOPIC, "1", "A", 500L));
-            driver.pipeInput(recordFactory.create(TOPIC, "2", "B", 500L));
-            driver.pipeInput(recordFactory.create(TOPIC, "2", "B", 500L));
-            driver.pipeInput(recordFactory.create(TOPIC, "3", "B", 100L));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(TOPIC, new StringSerializer(), new StringSerializer());
+            inputTopic.pipeInput("1", "A", 0L);
+            inputTopic.pipeInput("1", "A", 499L);
+            inputTopic.pipeInput("1", "A", 100L);
+            inputTopic.pipeInput("2", "B", 0L);
+            inputTopic.pipeInput("2", "B", 100L);
+            inputTopic.pipeInput("2", "B", 200L);
+            inputTopic.pipeInput("3", "C", 1L);
+            inputTopic.pipeInput("1", "A", 500L);
+            inputTopic.pipeInput("1", "A", 500L);
+            inputTopic.pipeInput("2", "B", 500L);
+            inputTopic.pipeInput("2", "B", 500L);
+            inputTopic.pipeInput("3", "B", 100L);
         }
         assertThat(supplier.theCapturedProcessor().processed, equalTo(Arrays.asList(
             new KeyValueTimestamp<>(new Windowed<>("1", new TimeWindow(0L, 500L)), 1L, 0L),
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImplTest.java
index a8fed6756f04d..2e2753bf71bf8 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImplTest.java
@@ -32,7 +32,7 @@
 import org.apache.kafka.streams.kstream.Materialized;
 import org.apache.kafka.streams.state.KeyValueStore;
 import org.apache.kafka.streams.state.ValueAndTimestamp;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockAggregator;
 import org.apache.kafka.test.MockInitializer;
 import org.apache.kafka.test.MockMapper;
@@ -136,18 +136,18 @@ private MockProcessorSupplier getReducedResults(final KTable> reducedResults,
                                final String topic,
                                final TopologyTestDriver driver) {
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(new StringSerializer(), new DoubleSerializer());
-        driver.pipeInput(recordFactory.create(topic, "A", 1.1, 10));
-        driver.pipeInput(recordFactory.create(topic, "B", 2.2, 11));
+        final TestInputTopic inputTopic =
+            driver.createInputTopic(topic, new StringSerializer(), new DoubleSerializer());
+        inputTopic.pipeInput("A", 1.1, 10);
+        inputTopic.pipeInput("B", 2.2, 11);
 
         assertEquals(ValueAndTimestamp.make(1, 10L), reducedResults.get("A"));
         assertEquals(ValueAndTimestamp.make(2, 11L), reducedResults.get("B"));
 
-        driver.pipeInput(recordFactory.create(topic, "A", 2.6, 30));
-        driver.pipeInput(recordFactory.create(topic, "B", 1.3, 30));
-        driver.pipeInput(recordFactory.create(topic, "A", 5.7, 50));
-        driver.pipeInput(recordFactory.create(topic, "B", 6.2, 20));
+        inputTopic.pipeInput("A", 2.6, 30);
+        inputTopic.pipeInput("B", 1.3, 30);
+        inputTopic.pipeInput("A", 5.7, 50);
+        inputTopic.pipeInput("B", 6.2, 20);
 
         assertEquals(ValueAndTimestamp.make(5, 50L), reducedResults.get("A"));
         assertEquals(ValueAndTimestamp.make(6, 30L), reducedResults.get("B"));
@@ -369,12 +369,12 @@ public void shouldThrowNullPointerOnAggregateWhenMaterializedIsNull() {
 
     private void processData(final String topic,
                              final TopologyTestDriver driver) {
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
-        driver.pipeInput(recordFactory.create(topic, "A", "1", 10L));
-        driver.pipeInput(recordFactory.create(topic, "B", "1", 50L));
-        driver.pipeInput(recordFactory.create(topic, "C", "1", 30L));
-        driver.pipeInput(recordFactory.create(topic, "D", "2", 40L));
-        driver.pipeInput(recordFactory.create(topic, "E", "2", 60L));
+        final TestInputTopic inputTopic =
+            driver.createInputTopic(topic, new StringSerializer(), new StringSerializer());
+        inputTopic.pipeInput("A", "1", 10L);
+        inputTopic.pipeInput("B", "1", 50L);
+        inputTopic.pipeInput("C", "1", 30L);
+        inputTopic.pipeInput("D", "2", 40L);
+        inputTopic.pipeInput("E", "2", 60L);
     }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamBranchTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamBranchTest.java
index 77d526963bd12..8909e77b5feef 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamBranchTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamBranchTest.java
@@ -24,7 +24,7 @@
 import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.Predicate;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockProcessor;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.StreamsTestUtils;
@@ -38,7 +38,6 @@
 public class KStreamBranchTest {
 
     private final String topicName = "topic";
-    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer());
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String());
 
     @SuppressWarnings("unchecked")
@@ -66,8 +65,9 @@ public void testKStreamBranch() {
         }
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic = driver.createInputTopic(topicName, new IntegerSerializer(), new StringSerializer());
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topicName, expectedKey, "V" + expectedKey));
+                inputTopic.pipeInput(expectedKey, "V" + expectedKey);
             }
         }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFilterTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFilterTest.java
index 3f8f83575e4b6..b988390e0c352 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFilterTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFilterTest.java
@@ -24,7 +24,7 @@
 import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.Predicate;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.StreamsTestUtils;
 import org.junit.Test;
@@ -36,7 +36,6 @@
 public class KStreamFilterTest {
 
     private final String topicName = "topic";
-    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer());
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String());
 
     private final Predicate isMultipleOfThree = (key, value) -> (key % 3) == 0;
@@ -53,8 +52,9 @@ public void testFilter() {
         stream.filter(isMultipleOfThree).process(supplier);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic = driver.createInputTopic(topicName, new IntegerSerializer(), new StringSerializer());
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topicName, expectedKey, "V" + expectedKey));
+                inputTopic.pipeInput(expectedKey, "V" + expectedKey);
             }
         }
 
@@ -74,7 +74,8 @@ public void testFilterNot() {
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topicName, expectedKey, "V" + expectedKey));
+                final TestInputTopic inputTopic = driver.createInputTopic(topicName, new IntegerSerializer(), new StringSerializer());
+                inputTopic.pipeInput(expectedKey, "V" + expectedKey);
             }
         }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapTest.java
index 990eb48e63310..a9219ff6600ee 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapTest.java
@@ -26,19 +26,19 @@
 import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KeyValueMapper;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.StreamsTestUtils;
 import org.junit.Test;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Properties;
 
 import static org.junit.Assert.assertEquals;
 
 public class KStreamFlatMapTest {
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer(), 0L);
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String());
 
     @Test
@@ -65,8 +65,10 @@ public void testFlatMap() {
         stream.flatMap(mapper).process(supplier);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(topicName, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0), Duration.ZERO);
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topicName, expectedKey, "V" + expectedKey));
+                inputTopic.pipeInput(expectedKey, "V" + expectedKey);
             }
         }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapValuesTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapValuesTest.java
index f92d80dbbe10a..62501d440283c 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapValuesTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapValuesTest.java
@@ -25,11 +25,13 @@
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.ValueMapper;
 import org.apache.kafka.streams.kstream.ValueMapperWithKey;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.StreamsTestUtils;
 import org.junit.Test;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Properties;
 
@@ -37,8 +39,6 @@
 
 public class KStreamFlatMapValuesTest {
     private final String topicName = "topic";
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(new IntegerSerializer(), new IntegerSerializer(), 0L);
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String());
 
     @Test
@@ -60,9 +60,11 @@ public void testFlatMapValues() {
         stream.flatMapValues(mapper).process(supplier);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(topicName, new IntegerSerializer(), new IntegerSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             for (final int expectedKey : expectedKeys) {
-                // passing the timestamp to recordFactory.create to disambiguate the call
-                driver.pipeInput(recordFactory.create(topicName, expectedKey, expectedKey, 0L));
+                // passing the timestamp to inputTopic.create to disambiguate the call
+                inputTopic.pipeInput(expectedKey, expectedKey, 0L);
             }
         }
 
@@ -95,9 +97,11 @@ public void testFlatMapValuesWithKeys() {
         stream.flatMapValues(mapper).process(supplier);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(topicName, new IntegerSerializer(), new IntegerSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             for (final int expectedKey : expectedKeys) {
-                // passing the timestamp to recordFactory.create to disambiguate the call
-                driver.pipeInput(recordFactory.create(topicName, expectedKey, expectedKey, 0L));
+                // passing the timestamp to inputTopic.create to disambiguate the call
+                inputTopic.pipeInput(expectedKey, expectedKey, 0L);
             }
         }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamForeachTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamForeachTest.java
index c6f26c89c9f4b..35db2453cfe6b 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamForeachTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamForeachTest.java
@@ -25,7 +25,7 @@
 import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.ForeachAction;
 import org.apache.kafka.streams.kstream.KStream;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.StreamsTestUtils;
 import org.junit.Test;
 
@@ -40,7 +40,6 @@
 public class KStreamForeachTest {
 
     private final String topicName = "topic";
-    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer());
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String());
 
     @Test
@@ -71,8 +70,9 @@ public void testForeach() {
 
         // Then
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic = driver.createInputTopic(topicName, new IntegerSerializer(), new StringSerializer());
             for (final KeyValue record : inputRecords) {
-                driver.pipeInput(recordFactory.create(topicName, record.key, record.value));
+                inputTopic.pipeInput(record.key, record.value);
             }
         }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableJoinTest.java
index d0348dfaada15..44fed0246a145 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableJoinTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableJoinTest.java
@@ -27,7 +27,7 @@
 import org.apache.kafka.streams.kstream.GlobalKTable;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KeyValueMapper;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockProcessor;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockValueJoiner;
@@ -36,6 +36,8 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.Collection;
 import java.util.Properties;
 import java.util.Set;
@@ -86,30 +88,30 @@ public void cleanup() {
     }
 
     private void pushToStream(final int messageCount, final String valuePrefix, final boolean includeForeignKey) {
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer(), 0L, 1L);
+        final TestInputTopic inputTopic =
+            driver.createInputTopic(streamTopic, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ofMillis(1L));
         for (int i = 0; i < messageCount; i++) {
             String value = valuePrefix + expectedKeys[i];
             if (includeForeignKey) {
                 value = value + ",FKey" + expectedKeys[i];
             }
-            driver.pipeInput(recordFactory.create(streamTopic, expectedKeys[i], value));
+            inputTopic.pipeInput(expectedKeys[i], value);
         }
     }
 
     private void pushToGlobalTable(final int messageCount, final String valuePrefix) {
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
+        final TestInputTopic inputTopic =
+            driver.createInputTopic(globalTableTopic, new StringSerializer(), new StringSerializer());
         for (int i = 0; i < messageCount; i++) {
-            driver.pipeInput(recordFactory.create(globalTableTopic, "FKey" + expectedKeys[i], valuePrefix + expectedKeys[i]));
+            inputTopic.pipeInput("FKey" + expectedKeys[i], valuePrefix + expectedKeys[i]);
         }
     }
 
     private void pushNullValueToGlobalTable(final int messageCount) {
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
+        final TestInputTopic inputTopic =
+            driver.createInputTopic(globalTableTopic, new StringSerializer(), new StringSerializer());
         for (int i = 0; i < messageCount; i++) {
-            driver.pipeInput(recordFactory.create(globalTableTopic, "FKey" + expectedKeys[i], (String) null));
+            inputTopic.pipeInput("FKey" + expectedKeys[i], (String) null);
         }
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableLeftJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableLeftJoinTest.java
index ce4454e5641a9..c6e32e4745495 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableLeftJoinTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableLeftJoinTest.java
@@ -27,7 +27,7 @@
 import org.apache.kafka.streams.kstream.GlobalKTable;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KeyValueMapper;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockProcessor;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockValueJoiner;
@@ -36,6 +36,8 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.Collection;
 import java.util.Properties;
 import java.util.Set;
@@ -86,30 +88,30 @@ public void cleanup() {
     }
 
     private void pushToStream(final int messageCount, final String valuePrefix, final boolean includeForeignKey) {
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer(), 0L, 1L);
+        final TestInputTopic inputTopic =
+            driver.createInputTopic(streamTopic, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ofMillis(1L));
         for (int i = 0; i < messageCount; i++) {
             String value = valuePrefix + expectedKeys[i];
             if (includeForeignKey) {
                 value = value + ",FKey" + expectedKeys[i];
             }
-            driver.pipeInput(recordFactory.create(streamTopic, expectedKeys[i], value));
+            inputTopic.pipeInput(expectedKeys[i], value);
         }
     }
 
     private void pushToGlobalTable(final int messageCount, final String valuePrefix) {
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L, 1L);
+        final TestInputTopic inputTopic =
+            driver.createInputTopic(globalTableTopic, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ofMillis(1L));
         for (int i = 0; i < messageCount; i++) {
-            driver.pipeInput(recordFactory.create(globalTableTopic, "FKey" + expectedKeys[i], valuePrefix + expectedKeys[i]));
+            inputTopic.pipeInput("FKey" + expectedKeys[i], valuePrefix + expectedKeys[i]);
         }
     }
 
     private void pushNullValueToGlobalTable(final int messageCount) {
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L, 1L);
+        final TestInputTopic inputTopic =
+            driver.createInputTopic(globalTableTopic, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ofMillis(1L));
         for (int i = 0; i < messageCount; i++) {
-            driver.pipeInput(recordFactory.create(globalTableTopic, "FKey" + expectedKeys[i], (String) null));
+            inputTopic.pipeInput("FKey" + expectedKeys[i], (String) null);
         }
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java
index 976ae1a0b7994..627584268087c 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java
@@ -48,7 +48,7 @@
 import org.apache.kafka.streams.processor.TopicNameExtractor;
 import org.apache.kafka.streams.processor.internals.ProcessorTopology;
 import org.apache.kafka.streams.processor.internals.SourceNode;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockMapper;
 import org.apache.kafka.test.MockProcessor;
 import org.apache.kafka.test.MockProcessorSupplier;
@@ -58,6 +58,7 @@
 import org.junit.Test;
 
 import java.time.Duration;
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -87,8 +88,6 @@ public class KStreamImplTest {
     private KStream testStream;
     private StreamsBuilder builder;
 
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L);
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String());
 
     private final Serde mySerde = new Serdes.StringSerde();
@@ -291,7 +290,9 @@ public void shouldSendDataThroughTopicUsingProduced() {
         stream.through("through-topic", Produced.with(Serdes.String(), Serdes.String())).process(processorSupplier);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(input, "a", "b"));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(input, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            inputTopic.pipeInput("a", "b");
         }
         assertThat(processorSupplier.theCapturedProcessor().processed, equalTo(Collections.singletonList(new KeyValueTimestamp<>("a", "b", 0))));
     }
@@ -305,7 +306,9 @@ public void shouldSendDataToTopicUsingProduced() {
         builder.stream("to-topic", stringConsumed).process(processorSupplier);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(input, "e", "f"));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(input, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            inputTopic.pipeInput("e", "f");
         }
         assertThat(processorSupplier.theCapturedProcessor().processed, equalTo(Collections.singletonList(new KeyValueTimestamp<>("e", "f", 0))));
     }
@@ -321,9 +324,11 @@ public void shouldSendDataToDynamicTopics() {
         builder.stream(input + "-b-v", stringConsumed).process(processorSupplier);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(input, "a", "v1"));
-            driver.pipeInput(recordFactory.create(input, "a", "v2"));
-            driver.pipeInput(recordFactory.create(input, "b", "v1"));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(input, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            inputTopic.pipeInput("a", "v1");
+            inputTopic.pipeInput("a", "v2");
+            inputTopic.pipeInput("b", "v1");
         }
         final List> mockProcessors = processorSupplier.capturedProcessors(2);
         assertThat(mockProcessors.get(0).processed, equalTo(asList(new KeyValueTimestamp<>("a", "v1", 0),
@@ -674,10 +679,14 @@ public void shouldMergeTwoStreams() {
         merged.process(processorSupplier);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(topic1, "A", "aa"));
-            driver.pipeInput(recordFactory.create(topic2, "B", "bb"));
-            driver.pipeInput(recordFactory.create(topic2, "C", "cc"));
-            driver.pipeInput(recordFactory.create(topic1, "D", "dd"));
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic2 =
+                    driver.createInputTopic(topic2, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            inputTopic1.pipeInput("A", "aa");
+            inputTopic2.pipeInput("B", "bb");
+            inputTopic2.pipeInput("C", "cc");
+            inputTopic1.pipeInput("D", "dd");
         }
 
         assertEquals(asList(new KeyValueTimestamp<>("A", "aa", 0),
@@ -702,14 +711,23 @@ public void shouldMergeMultipleStreams() {
         merged.process(processorSupplier);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(topic1, "A", "aa", 1L));
-            driver.pipeInput(recordFactory.create(topic2, "B", "bb", 9L));
-            driver.pipeInput(recordFactory.create(topic3, "C", "cc", 2L));
-            driver.pipeInput(recordFactory.create(topic4, "D", "dd", 8L));
-            driver.pipeInput(recordFactory.create(topic4, "E", "ee", 3L));
-            driver.pipeInput(recordFactory.create(topic3, "F", "ff", 7L));
-            driver.pipeInput(recordFactory.create(topic2, "G", "gg", 4L));
-            driver.pipeInput(recordFactory.create(topic1, "H", "hh", 6L));
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic2 =
+                    driver.createInputTopic(topic2, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic3 =
+                    driver.createInputTopic(topic3, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic4 =
+                    driver.createInputTopic(topic4, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+
+            inputTopic1.pipeInput("A", "aa", 1L);
+            inputTopic2.pipeInput("B", "bb", 9L);
+            inputTopic3.pipeInput("C", "cc", 2L);
+            inputTopic4.pipeInput("D", "dd", 8L);
+            inputTopic4.pipeInput("E", "ee", 3L);
+            inputTopic3.pipeInput("F", "ff", 7L);
+            inputTopic2.pipeInput("G", "gg", 4L);
+            inputTopic1.pipeInput("H", "hh", 6L);
         }
 
         assertEquals(asList(new KeyValueTimestamp<>("A", "aa", 1),
@@ -730,11 +748,22 @@ public void shouldProcessFromSourceThatMatchPattern() {
         pattern2Source.process(processorSupplier);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create("topic-3", "A", "aa", 1L));
-            driver.pipeInput(recordFactory.create("topic-4", "B", "bb", 5L));
-            driver.pipeInput(recordFactory.create("topic-5", "C", "cc", 10L));
-            driver.pipeInput(recordFactory.create("topic-6", "D", "dd", 8L));
-            driver.pipeInput(recordFactory.create("topic-7", "E", "ee", 3L));
+            final TestInputTopic inputTopic3 =
+                    driver.createInputTopic("topic-3", new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic4 =
+                    driver.createInputTopic("topic-4", new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic5 =
+                    driver.createInputTopic("topic-5", new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic6 =
+                    driver.createInputTopic("topic-6", new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic7 =
+                    driver.createInputTopic("topic-7", new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+
+            inputTopic3.pipeInput("A", "aa", 1L);
+            inputTopic4.pipeInput("B", "bb", 5L);
+            inputTopic5.pipeInput("C", "cc", 10L);
+            inputTopic6.pipeInput("D", "dd", 8L);
+            inputTopic7.pipeInput("E", "ee", 3L);
         }
 
         assertEquals(asList(new KeyValueTimestamp<>("A", "aa", 1),
@@ -757,11 +786,22 @@ public void shouldProcessFromSourcesThatMatchMultiplePattern() {
         merged.process(processorSupplier);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create("topic-3", "A", "aa", 1L));
-            driver.pipeInput(recordFactory.create("topic-4", "B", "bb", 5L));
-            driver.pipeInput(recordFactory.create("topic-A", "C", "cc", 10L));
-            driver.pipeInput(recordFactory.create("topic-Z", "D", "dd", 8L));
-            driver.pipeInput(recordFactory.create(topic3, "E", "ee", 3L));
+            final TestInputTopic inputTopic3 =
+                    driver.createInputTopic("topic-3", new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic4 =
+                    driver.createInputTopic("topic-4", new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopicA =
+                    driver.createInputTopic("topic-A", new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopicZ =
+                    driver.createInputTopic("topic-Z", new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(topic3, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+
+            inputTopic3.pipeInput("A", "aa", 1L);
+            inputTopic4.pipeInput("B", "bb", 5L);
+            inputTopicA.pipeInput("C", "cc", 10L);
+            inputTopicZ.pipeInput("D", "dd", 8L);
+            inputTopic.pipeInput("E", "ee", 3L);
         }
 
         assertEquals(asList(new KeyValueTimestamp<>("A", "aa", 1),
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoinTest.java
index b66abd32c219e..d7ce37e8bce29 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoinTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoinTest.java
@@ -29,9 +29,9 @@
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.StreamJoined;
 import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.streams.state.Stores;
 import org.apache.kafka.streams.state.WindowBytesStoreSupplier;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockProcessor;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockValueJoiner;
@@ -39,6 +39,7 @@
 import org.junit.Test;
 
 import java.time.Duration;
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
@@ -59,8 +60,6 @@ public class KStreamKStreamJoinTest {
     private final String topic1 = "topic1";
     private final String topic2 = "topic2";
     private final Consumed consumed = Consumed.with(Serdes.Integer(), Serdes.String());
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer(), 0L);
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String());
 
     private final JoinWindows joinWindows = JoinWindows.of(ofMillis(50)).grace(Duration.ofMillis(50));
@@ -73,8 +72,6 @@ public void shouldLogAndMeterOnSkippedRecordsWithNullValue() {
 
         final KStream left = builder.stream("left", Consumed.with(Serdes.String(), Serdes.Integer()));
         final KStream right = builder.stream("right", Consumed.with(Serdes.String(), Serdes.Integer()));
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(new StringSerializer(), new IntegerSerializer());
 
         left.join(
             right,
@@ -85,7 +82,9 @@ public void shouldLogAndMeterOnSkippedRecordsWithNullValue() {
 
         final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create("left", "A", null));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic("left", new StringSerializer(), new IntegerSerializer());
+            inputTopic.pipeInput("A", null);
             LogCaptureAppender.unregister(appender);
 
             assertThat(appender.getMessages(), hasItem("Skipping record due to null key or value. key=[A] value=[null] topic=[left] partition=[0] offset=[0]"));
@@ -224,8 +223,6 @@ private void runJoin(final StreamJoined streamJoined,
 
         final KStream left = builder.stream("left", Consumed.with(Serdes.String(), Serdes.Integer()));
         final KStream right = builder.stream("right", Consumed.with(Serdes.String(), Serdes.Integer()));
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(new StringSerializer(), new IntegerSerializer());
         final MockProcessorSupplier supplier = new MockProcessorSupplier<>();
         final KStream joinedStream;
 
@@ -238,13 +235,17 @@ private void runJoin(final StreamJoined streamJoined,
         joinedStream.process(supplier);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopicLeft =
+                    driver.createInputTopic("left", new StringSerializer(), new IntegerSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopicRight =
+                    driver.createInputTopic("right", new StringSerializer(), new IntegerSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             final MockProcessor processor = supplier.theCapturedProcessor();
 
-            driver.pipeInput(recordFactory.create("left", "A", 1, 1L));
-            driver.pipeInput(recordFactory.create("left", "B", 1, 2L));
+            inputTopicLeft.pipeInput("A", 1, 1L);
+            inputTopicLeft.pipeInput("B", 1, 2L);
 
-            driver.pipeInput(recordFactory.create("right", "A", 1, 1L));
-            driver.pipeInput(recordFactory.create("right", "B", 2, 2L));
+            inputTopicRight.pipeInput("A", 1, 1L);
+            inputTopicRight.pipeInput("B", 2, 2L);
 
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>("A", 2, 1L),
                 new KeyValueTimestamp<>("B", 3, 2L));
@@ -277,6 +278,10 @@ public void testJoin() {
         assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next());
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic2 =
+                    driver.createInputTopic(topic2, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             final MockProcessor processor = supplier.theCapturedProcessor();
 
             // push two items to the primary stream; the other window is empty
@@ -285,7 +290,7 @@ public void testJoin() {
             // --> w1 = { 0:A0, 1:A1 }
             //     w2 = {}
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "A" + expectedKeys[i]));
+                inputTopic1.pipeInput(expectedKeys[i], "A" + expectedKeys[i]);
             }
             processor.checkAndClearProcessResult(EMPTY);
 
@@ -295,7 +300,7 @@ public void testJoin() {
             // --> w1 = { 0:A0, 1:A1 }
             //     w2 = { 0:a0, 1:a1 }
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "a" + expectedKeys[i]));
+                inputTopic2.pipeInput(expectedKeys[i], "a" + expectedKeys[i]);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+a0", 0),
                 new KeyValueTimestamp<>(1, "A1+a1", 0));
@@ -306,7 +311,7 @@ public void testJoin() {
             // --> w1 = { 0:A0, 1:A1, 0:B0, 1:B1, 2:B2, 3:B3 }
             //     w2 = { 0:a0, 1:a1 }
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "B" + expectedKey));
+                inputTopic1.pipeInput(expectedKey, "B" + expectedKey);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "B0+a0", 0),
                 new KeyValueTimestamp<>(1, "B1+a1", 0));
@@ -317,7 +322,7 @@ public void testJoin() {
             // --> w1 = { 0:A0, 1:A1, 0:B0, 1:B1, 2:B2, 3:B3 }
             //     w2 = { 0:a0, 1:a1, 0:b0, 1:b1, 2:b2, 3:b3 }
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "b" + expectedKey));
+                inputTopic2.pipeInput(expectedKey, "b" + expectedKey);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+b0", 0),
                 new KeyValueTimestamp<>(0, "B0+b0", 0),
@@ -332,7 +337,7 @@ public void testJoin() {
             // --> w1 = { 0:A0, 1:A1, 0:B0, 1:B1, 2:B2, 3:B3, 0:C0, 1:C1, 2:C2, 3:C3 }
             //     w2 = { 0:a0, 1:a1, 0:b0, 1:b1, 2:b2, 3:b3 }
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "C" + expectedKey));
+                inputTopic1.pipeInput(expectedKey, "C" + expectedKey);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "C0+a0", 0),
                 new KeyValueTimestamp<>(0, "C0+b0", 0),
@@ -347,7 +352,7 @@ public void testJoin() {
             // --> w1 = { 0:A0, 1:A1, 0:B0, 1:B1, 2:B2, 3:B3, 0:C0, 1:C1, 2:C2, 3:C3 }
             //     w2 = { 0:a0, 1:a1, 0:b0, 1:b1, 2:b2, 3:b3, 0:c0, 1:c1 }
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "c" + expectedKeys[i]));
+                inputTopic2.pipeInput(expectedKeys[i], "c" + expectedKeys[i]);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+c0", 0),
                 new KeyValueTimestamp<>(0, "B0+c0", 0),
@@ -384,6 +389,10 @@ public void testOuterJoin() {
         assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next());
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic2 =
+                    driver.createInputTopic(topic2, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             final MockProcessor processor = supplier.theCapturedProcessor();
 
             // push two items to the primary stream; the other window is empty; this should produce two items
@@ -392,7 +401,7 @@ public void testOuterJoin() {
             // --> w1 = { 0:A0, 1:A1 }
             //     w2 = {}
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "A" + expectedKeys[i]));
+                inputTopic1.pipeInput(expectedKeys[i], "A" + expectedKeys[i]);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+null", 0),
                 new KeyValueTimestamp<>(1, "A1+null", 0));
@@ -403,7 +412,7 @@ public void testOuterJoin() {
             // --> w1 = { 0:A0, 1:A1 }
             //     w2 = { 0:a0, 1:a1 }
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "a" + expectedKeys[i]));
+                inputTopic2.pipeInput(expectedKeys[i], "a" + expectedKeys[i]);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+a0", 0),
                 new KeyValueTimestamp<>(1, "A1+a1", 0));
@@ -414,7 +423,7 @@ public void testOuterJoin() {
             // --> w1 = { 0:A0, 1:A1, 0:B0, 1:B1, 2:B2, 3:B3 }
             //     w2 = { 0:a0, 1:a1 }
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "B" + expectedKey));
+                inputTopic1.pipeInput(expectedKey, "B" + expectedKey);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "B0+a0", 0),
                 new KeyValueTimestamp<>(1, "B1+a1", 0),
@@ -427,7 +436,7 @@ public void testOuterJoin() {
             // --> w1 = { 0:A0, 1:A1, 0:B0, 1:B1, 2:B2, 3:B3 }
             //     w2 = { 0:a0, 1:a1, 0:b0, 0:b0, 1:b1, 2:b2, 3:b3 }
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "b" + expectedKey));
+                inputTopic2.pipeInput(expectedKey, "b" + expectedKey);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+b0", 0),
                 new KeyValueTimestamp<>(0, "B0+b0", 0),
@@ -442,7 +451,7 @@ public void testOuterJoin() {
             // --> w1 = { 0:A0, 1:A1, 0:B0, 1:B1, 2:B2, 3:B3, 0:C0, 1:C1, 2:C2, 3:C3 }
             //     w2 = { 0:a0, 1:a1, 0:b0, 0:b0, 1:b1, 2:b2, 3:b3 }
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "C" + expectedKey));
+                inputTopic1.pipeInput(expectedKey, "C" + expectedKey);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "C0+a0", 0),
                 new KeyValueTimestamp<>(0, "C0+b0", 0),
@@ -457,7 +466,7 @@ public void testOuterJoin() {
             // --> w1 = { 0:A0, 1:A1, 0:B0, 1:B1, 2:B2, 3:B3, 0:C0, 1:C1, 2:C2, 3:C3 }
             //     w2 = { 0:a0, 1:a1, 0:b0, 0:b0, 1:b1, 2:b2, 3:b3, 0:c0, 1:c1 }
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "c" + expectedKeys[i]));
+                inputTopic2.pipeInput(expectedKeys[i], "c" + expectedKeys[i]);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+c0", 0),
                 new KeyValueTimestamp<>(0, "B0+c0", 0),
@@ -495,6 +504,10 @@ public void testWindowing() {
         assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next());
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic2 =
+                    driver.createInputTopic(topic2, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             final MockProcessor processor = supplier.theCapturedProcessor();
             long time = 0L;
 
@@ -504,7 +517,7 @@ public void testWindowing() {
             // --> w1 = { 0:A0 (ts: 0), 1:A1 (ts: 0) }
             //     w2 = {}
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "A" + expectedKeys[i], time));
+                inputTopic1.pipeInput(expectedKeys[i], "A" + expectedKeys[i], time);
             }
             processor.checkAndClearProcessResult(EMPTY);
 
@@ -514,7 +527,7 @@ public void testWindowing() {
             // --> w1 = { 0:A0 (ts: 0), 1:A1 (ts: 0) }
             //     w2 = { 0:a0 (ts: 0), 1:a1 (ts: 0) }
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "a" + expectedKeys[i], time));
+                inputTopic2.pipeInput(expectedKeys[i], "a" + expectedKeys[i], time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+a0", 0),
                 new KeyValueTimestamp<>(1, "A1+a1", 0));
@@ -527,7 +540,7 @@ public void testWindowing() {
             //     w2 = { 0:a0 (ts: 0), 1:a1 (ts: 0) }
             time = 1000L;
             for (int i = 0; i < expectedKeys.length; i++) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "B" + expectedKeys[i], time + i));
+                inputTopic1.pipeInput(expectedKeys[i], "B" + expectedKeys[i], time + i);
             }
             processor.checkAndClearProcessResult(EMPTY);
 
@@ -541,7 +554,7 @@ public void testWindowing() {
             //            0:b0 (ts: 1100), 1:b1 (ts: 1100), 2:b2 (ts: 1100), 3:b3 (ts: 1100) }
             time += 100L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "b" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "b" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "B0+b0", 1100),
                 new KeyValueTimestamp<>(1, "B1+b1", 1100),
@@ -560,7 +573,7 @@ public void testWindowing() {
             //            0:c0 (ts: 1101), 1:c1 (ts: 1101), 2:c2 (ts: 1101), 3:c3 (ts: 1101) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "c" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "c" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(1, "B1+c1", 1101),
                 new KeyValueTimestamp<>(2, "B2+c2", 1101),
@@ -580,7 +593,7 @@ public void testWindowing() {
             //            0:d0 (ts: 1102), 1:d1 (ts: 1102), 2:d2 (ts: 1102), 3:d3 (ts: 1102) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "d" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "d" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(2, "B2+d2", 1102),
                 new KeyValueTimestamp<>(3, "B3+d3", 1102));
@@ -601,7 +614,7 @@ public void testWindowing() {
             //            0:e0 (ts: 1103), 1:e1 (ts: 1103), 2:e2 (ts: 1103), 3:e3 (ts: 1103) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "e" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "e" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(3, "B3+e3", 1103));
 
@@ -623,7 +636,7 @@ public void testWindowing() {
             //            0:f0 (ts: 1104), 1:f1 (ts: 1104), 2:f2 (ts: 1104), 3:f3 (ts: 1104) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "f" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "f" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(EMPTY);
 
@@ -647,7 +660,7 @@ public void testWindowing() {
             //            0:g0 (ts: 899), 1:g1 (ts: 899), 2:g2 (ts: 899), 3:g3 (ts: 899) }
             time = 1000L - 100L - 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "g" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "g" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(EMPTY);
 
@@ -673,7 +686,7 @@ public void testWindowing() {
             //            0:h0 (ts: 900), 1:h1 (ts: 900), 2:h2 (ts: 900), 3:h3 (ts: 900) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "h" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "h" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "B0+h0", 1000));
 
@@ -701,7 +714,7 @@ public void testWindowing() {
             //            0:i0 (ts: 901), 1:i1 (ts: 901), 2:i2 (ts: 901), 3:i3 (ts: 901) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "i" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "i" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "B0+i0", 1000),
                 new KeyValueTimestamp<>(1, "B1+i1", 1001));
@@ -732,7 +745,7 @@ public void testWindowing() {
             //            0:j0 (ts: 902), 1:j1 (ts: 902), 2:j2 (ts: 902), 3:j3 (ts: 902) }
             time += 1;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "j" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "j" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "B0+j0", 1000),
                 new KeyValueTimestamp<>(1, "B1+j1", 1001),
@@ -766,7 +779,7 @@ public void testWindowing() {
             //            0:k0 (ts: 903), 1:k1 (ts: 903), 2:k2 (ts: 903), 3:k3 (ts: 903) }
             time += 1;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "k" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "k" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "B0+k0", 1000),
                 new KeyValueTimestamp<>(1, "B1+k1", 1001),
@@ -783,7 +796,7 @@ public void testWindowing() {
             //     w2 = { 0:l0 (ts: 2000), 1:l1 (ts: 2001), 2:l2 (ts: 2002), 3:l3 (ts: 2003) }
             time = 2000L;
             for (int i = 0; i < expectedKeys.length; i++) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "l" + expectedKeys[i], time + i));
+                inputTopic2.pipeInput(expectedKeys[i], "l" + expectedKeys[i], time + i);
             }
             processor.checkAndClearProcessResult(EMPTY);
 
@@ -794,7 +807,7 @@ public void testWindowing() {
             //     w2 = { 0:l0 (ts: 2000), 1:l1 (ts: 2001), 2:l2 (ts: 2002), 3:l3 (ts: 2003) }
             time = 2000L + 100L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "C" + expectedKey, time));
+                inputTopic1.pipeInput(expectedKey, "C" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "C0+l0", 2100),
                 new KeyValueTimestamp<>(1, "C1+l1", 2100),
@@ -809,7 +822,7 @@ public void testWindowing() {
             //     w2 = { 0:l0 (ts: 2000), 1:l1 (ts: 2001), 2:l2 (ts: 2002), 3:l3 (ts: 2003) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "D" + expectedKey, time));
+                inputTopic1.pipeInput(expectedKey, "D" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(1, "D1+l1", 2101),
                 new KeyValueTimestamp<>(2, "D2+l2", 2101),
@@ -825,7 +838,7 @@ public void testWindowing() {
             //     w2 = { 0:l0 (ts: 2000), 1:l1 (ts: 2001), 2:l2 (ts: 2002), 3:l3 (ts: 2003) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "E" + expectedKey, time));
+                inputTopic1.pipeInput(expectedKey, "E" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(2, "E2+l2", 2102),
                 new KeyValueTimestamp<>(3, "E3+l3", 2102));
@@ -842,7 +855,7 @@ public void testWindowing() {
             //     w2 = { 0:l0 (ts: 2000), 1:l1 (ts: 2001), 2:l2 (ts: 2002), 3:l3 (ts: 2003) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "F" + expectedKey, time));
+                inputTopic1.pipeInput(expectedKey, "F" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(3, "F3+l3", 2103));
 
@@ -860,7 +873,7 @@ public void testWindowing() {
             //     w2 = { 0:l0 (ts: 2000), 1:l1 (ts: 2001), 2:l2 (ts: 2002), 3:l3 (ts: 2003) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "G" + expectedKey, time));
+                inputTopic1.pipeInput(expectedKey, "G" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(EMPTY);
 
@@ -880,7 +893,7 @@ public void testWindowing() {
             //     w2 = { 0:l0 (ts: 2000), 1:l1 (ts: 2001), 2:l2 (ts: 2002), 3:l3 (ts: 2003) }
             time = 2000L - 100L - 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "H" + expectedKey, time));
+                inputTopic1.pipeInput(expectedKey, "H" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(EMPTY);
 
@@ -902,7 +915,7 @@ public void testWindowing() {
             //     w2 = { 0:l0 (ts: 2000), 1:l1 (ts: 2001), 2:l2 (ts: 2002), 3:l3 (ts: 2003) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "I" + expectedKey, time));
+                inputTopic1.pipeInput(expectedKey, "I" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "I0+l0", 2000));
 
@@ -926,7 +939,7 @@ public void testWindowing() {
             //     w2 = { 0:l0 (ts: 2000), 1:l1 (ts: 2001), 2:l2 (ts: 2002), 3:l3 (ts: 2003) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "J" + expectedKey, time));
+                inputTopic1.pipeInput(expectedKey, "J" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "J0+l0", 2000),
                 new KeyValueTimestamp<>(1, "J1+l1", 2001));
@@ -953,7 +966,7 @@ public void testWindowing() {
             //     w2 = { 0:l0 (ts: 2000), 1:l1 (ts: 2001), 2:l2 (ts: 2002), 3:l3 (ts: 2003) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "K" + expectedKey, time));
+                inputTopic1.pipeInput(expectedKey, "K" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "K0+l0", 2000),
                 new KeyValueTimestamp<>(1, "K1+l1", 2001),
@@ -983,7 +996,7 @@ public void testWindowing() {
             //     w2 = { 0:l0 (ts: 2000), 1:l1 (ts: 2001), 2:l2 (ts: 2002), 3:l3 (ts: 2003) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "L" + expectedKey, time));
+                inputTopic1.pipeInput(expectedKey, "L" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "L0+l0", 2000),
                 new KeyValueTimestamp<>(1, "L1+l1", 2001),
@@ -1021,6 +1034,10 @@ public void testAsymmetricWindowingAfter() {
         assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next());
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic2 =
+                    driver.createInputTopic(topic2, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             final MockProcessor processor = supplier.theCapturedProcessor();
             long time = 1000L;
 
@@ -1030,7 +1047,7 @@ public void testAsymmetricWindowingAfter() {
             // --> w1 = { 0:A0 (ts: 1000), 1:A1 (ts: 1001), 2:A2 (ts: 1002), 3:A3 (ts: 1003) }
             //     w2 = {}
             for (int i = 0; i < expectedKeys.length; i++) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "A" + expectedKeys[i], time + i));
+                inputTopic1.pipeInput(expectedKeys[i], "A" + expectedKeys[i], time + i);
             }
             processor.checkAndClearProcessResult(EMPTY);
 
@@ -1041,7 +1058,7 @@ public void testAsymmetricWindowingAfter() {
             //     w2 = { 0:a0 (ts: 999), 1:a1 (ts: 999), 2:a2 (ts: 999), 3:a3 (ts: 999) }
             time = 1000L - 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "a" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "a" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(EMPTY);
 
@@ -1053,7 +1070,7 @@ public void testAsymmetricWindowingAfter() {
             //            0:b0 (ts: 1000), 1:b1 (ts: 1000), 2:b2 (ts: 1000), 3:b3 (ts: 1000) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "b" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "b" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+b0", 1000));
 
@@ -1067,7 +1084,7 @@ public void testAsymmetricWindowingAfter() {
             //            0:c0 (ts: 1001), 1:c1 (ts: 1001), 2:c2 (ts: 1001), 3:c3 (ts: 1001) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "c" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "c" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+c0", 1001),
                 new KeyValueTimestamp<>(1, "A1+c1", 1001));
@@ -1084,7 +1101,7 @@ public void testAsymmetricWindowingAfter() {
             //            0:d0 (ts: 1002), 1:d1 (ts: 1002), 2:d2 (ts: 1002), 3:d3 (ts: 1002) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "d" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "d" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+d0", 1002),
                 new KeyValueTimestamp<>(1, "A1+d1", 1002),
@@ -1104,7 +1121,7 @@ public void testAsymmetricWindowingAfter() {
             //            0:e0 (ts: 1003), 1:e1 (ts: 1003), 2:e2 (ts: 1003), 3:e3 (ts: 1003) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "e" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "e" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+e0", 1003),
                 new KeyValueTimestamp<>(1, "A1+e1", 1003),
@@ -1127,7 +1144,7 @@ public void testAsymmetricWindowingAfter() {
             //            0:f0 (ts: 1100), 1:f1 (ts: 1100), 2:f2 (ts: 1100), 3:f3 (ts: 1100) }
             time = 1000 + 100L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "f" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "f" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+f0", 1100),
                 new KeyValueTimestamp<>(1, "A1+f1", 1100),
@@ -1152,7 +1169,7 @@ public void testAsymmetricWindowingAfter() {
             //            0:g0 (ts: 1101), 1:g1 (ts: 1101), 2:g2 (ts: 1101), 3:g3 (ts: 1101) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "g" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "g" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(1, "A1+g1", 1101),
                 new KeyValueTimestamp<>(2, "A2+g2", 1101),
@@ -1178,7 +1195,7 @@ public void testAsymmetricWindowingAfter() {
             //            0:h0 (ts: 1102), 1:h1 (ts: 1102), 2:h2 (ts: 1102), 3:h3 (ts: 1102) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "h" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "h" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(2, "A2+h2", 1102),
                 new KeyValueTimestamp<>(3, "A3+h3", 1102));
@@ -1205,7 +1222,7 @@ public void testAsymmetricWindowingAfter() {
             //            0:i0 (ts: 1103), 1:i1 (ts: 1103), 2:i2 (ts: 1103), 3:i3 (ts: 1103) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "i" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "i" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(3, "A3+i3", 1103));
 
@@ -1233,7 +1250,7 @@ public void testAsymmetricWindowingAfter() {
             //            0:j0 (ts: 1104), 1:j1 (ts: 1104), 2:j2 (ts: 1104), 3:j3 (ts: 1104) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "j" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "j" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(EMPTY);
         }
@@ -1267,6 +1284,10 @@ public void testAsymmetricWindowingBefore() {
         assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next());
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic2 =
+                    driver.createInputTopic(topic2, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             final MockProcessor processor = supplier.theCapturedProcessor();
             long time = 1000L;
 
@@ -1276,7 +1297,7 @@ public void testAsymmetricWindowingBefore() {
             // --> w1 = { 0:A0 (ts: 1000), 1:A1 (ts: 1001), 2:A2 (ts: 1002), 3:A3 (ts: 1003) }
             //     w2 = {}
             for (int i = 0; i < expectedKeys.length; i++) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "A" + expectedKeys[i], time + i));
+                inputTopic1.pipeInput(expectedKeys[i], "A" + expectedKeys[i], time + i);
             }
             processor.checkAndClearProcessResult(EMPTY);
 
@@ -1287,7 +1308,7 @@ public void testAsymmetricWindowingBefore() {
             //     w2 = { 0:a0 (ts: 899), 1:a1 (ts: 899), 2:a2 (ts: 899), 3:a3 (ts: 899) }
             time = 1000L - 100L - 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "a" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "a" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(EMPTY);
 
@@ -1299,7 +1320,7 @@ public void testAsymmetricWindowingBefore() {
             //            0:b0 (ts: 900), 1:b1 (ts: 900), 2:b2 (ts: 900), 3:b3 (ts: 900) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "b" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "b" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+b0", 1000));
 
@@ -1313,7 +1334,7 @@ public void testAsymmetricWindowingBefore() {
             //            0:c0 (ts: 901), 1:c1 (ts: 901), 2:c2 (ts: 901), 3:c3 (ts: 901) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "c" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "c" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+c0", 1000),
                 new KeyValueTimestamp<>(1, "A1+c1", 1001));
@@ -1330,7 +1351,7 @@ public void testAsymmetricWindowingBefore() {
             //            0:d0 (ts: 902), 1:d1 (ts: 902), 2:d2 (ts: 902), 3:d3 (ts: 902) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "d" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "d" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+d0", 1000),
                 new KeyValueTimestamp<>(1, "A1+d1", 1001),
@@ -1350,7 +1371,7 @@ public void testAsymmetricWindowingBefore() {
             //            0:e0 (ts: 903), 1:e1 (ts: 903), 2:e2 (ts: 903), 3:e3 (ts: 903) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "e" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "e" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+e0", 1000),
                 new KeyValueTimestamp<>(1, "A1+e1", 1001),
@@ -1373,7 +1394,7 @@ public void testAsymmetricWindowingBefore() {
             //            0:f0 (ts: 1000), 1:f1 (ts: 1000), 2:f2 (ts: 1000), 3:f3 (ts: 1000) }
             time = 1000L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "f" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "f" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+f0", 1000),
                 new KeyValueTimestamp<>(1, "A1+f1", 1001),
@@ -1398,7 +1419,7 @@ public void testAsymmetricWindowingBefore() {
             //            0:g0 (ts: 1001), 1:g1 (ts: 1001), 2:g2 (ts: 1001), 3:g3 (ts: 1001) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "g" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "g" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(1, "A1+g1", 1001),
                 new KeyValueTimestamp<>(2, "A2+g2", 1002),
@@ -1424,7 +1445,7 @@ public void testAsymmetricWindowingBefore() {
             //            0:h0 (ts: 1002), 1:h1 (ts: 1002), 2:h2 (ts: 1002), 3:h3 (ts: 1002) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "h" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "h" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(2, "A2+h2", 1002),
                 new KeyValueTimestamp<>(3, "A3+h3", 1003));
@@ -1451,7 +1472,7 @@ public void testAsymmetricWindowingBefore() {
             //            0:i0 (ts: 1003), 1:i1 (ts: 1003), 2:i2 (ts: 1003), 3:i3 (ts: 1003) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "i" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "i" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(3, "A3+i3", 1003));
 
@@ -1479,7 +1500,7 @@ public void testAsymmetricWindowingBefore() {
             //            0:j0 (ts: 1004), 1:j1 (ts: 1004), 2:j2 (ts: 1004), 3:j3 (ts: 1004) }
             time += 1L;
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "j" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "j" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(EMPTY);
         }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamLeftJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamLeftJoinTest.java
index 3fae39644b966..fafda4ade89f2 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamLeftJoinTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamLeftJoinTest.java
@@ -26,14 +26,16 @@
 import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.JoinWindows;
 import org.apache.kafka.streams.kstream.KStream;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.streams.kstream.StreamJoined;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockProcessor;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockValueJoiner;
 import org.apache.kafka.test.StreamsTestUtils;
 import org.junit.Test;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
@@ -49,8 +51,6 @@ public class KStreamKStreamLeftJoinTest {
     private final String topic1 = "topic1";
     private final String topic2 = "topic2";
     private final Consumed consumed = Consumed.with(Serdes.Integer(), Serdes.String());
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer(), 0L);
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String());
 
     @Test
@@ -80,6 +80,10 @@ public void testLeftJoin() {
         assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next());
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic2 =
+                    driver.createInputTopic(topic2, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             final MockProcessor processor = supplier.theCapturedProcessor();
 
             // push two items to the primary stream; the other window is empty
@@ -88,7 +92,7 @@ public void testLeftJoin() {
             // --> w1 = { 0:A0, 1:A1 }
             // --> w2 = {}
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "A" + expectedKeys[i]));
+                inputTopic1.pipeInput(expectedKeys[i], "A" + expectedKeys[i]);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+null", 0),
                 new KeyValueTimestamp<>(1, "A1+null", 0));
@@ -98,7 +102,7 @@ public void testLeftJoin() {
             // --> w1 = { 0:A0, 1:A1 }
             // --> w2 = { 0:a0, 1:a1 }
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "a" + expectedKeys[i]));
+                inputTopic2.pipeInput(expectedKeys[i], "a" + expectedKeys[i]);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+a0", 0),
                 new KeyValueTimestamp<>(1, "A1+a1", 0));
@@ -108,7 +112,7 @@ public void testLeftJoin() {
             // --> w1 = { 0:A0, 1:A1, 0:B0, 1:B1, 2:B2 }
             // --> w2 = { 0:a0, 1:a1 }
             for (int i = 0; i < 3; i++) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "B" + expectedKeys[i]));
+                inputTopic1.pipeInput(expectedKeys[i], "B" + expectedKeys[i]);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "B0+a0", 0),
                 new KeyValueTimestamp<>(1, "B1+a1", 0),
@@ -119,7 +123,7 @@ public void testLeftJoin() {
             // --> w1 = { 0:A0, 1:A1, 0:B0, 1:B1, 2:B2 }
             // --> w2 = { 0:a0, 1:a1, 0:b0, 1:b1, 2:b2, 3:b3 }
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "b" + expectedKey));
+                inputTopic2.pipeInput(expectedKey, "b" + expectedKey);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+b0", 0),
                 new KeyValueTimestamp<>(0, "B0+b0", 0),
@@ -132,7 +136,7 @@ public void testLeftJoin() {
             // --> w1 = { 0:A0, 1:A1, 0:B0, 1:B1, 2:B2, 0:C0, 1:C1, 2:C2, 3:C3 }
             // --> w2 = { 0:a0, 1:a1, 0:b0, 1:b1, 2:b2, 3:b3 }
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "C" + expectedKey));
+                inputTopic1.pipeInput(expectedKey, "C" + expectedKey);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "C0+a0", 0),
                 new KeyValueTimestamp<>(0, "C0+b0", 0),
@@ -169,6 +173,10 @@ public void testWindowing() {
         assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next());
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic2 =
+                    driver.createInputTopic(topic2, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             final MockProcessor processor = supplier.theCapturedProcessor();
             final long time = 0L;
 
@@ -178,7 +186,7 @@ public void testWindowing() {
             // --> w1 = { 0:A0 (ts: 0), 1:A1 (ts: 0) }
             // --> w2 = {}
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "A" + expectedKeys[i], time));
+                inputTopic1.pipeInput(expectedKeys[i], "A" + expectedKeys[i], time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+null", 0),
                 new KeyValueTimestamp<>(1, "A1+null", 0));
@@ -188,7 +196,7 @@ public void testWindowing() {
             // --> w1 = { 0:A0 (ts: 0), 1:A1 (ts: 0) }
             // --> w2 = { 0:a0 (ts: 0), 1:a1 (ts: 0), 2:a2 (ts: 0), 3:a3 (ts: 0) }
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "a" + expectedKey, time));
+                inputTopic2.pipeInput(expectedKey, "a" + expectedKey, time);
             }
             processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "A0+a0", 0),
                 new KeyValueTimestamp<>(1, "A1+a1", 0));
@@ -202,6 +210,11 @@ private void testUpperWindowBound(final int[] expectedKeys,
                                       final MockProcessor processor) {
         long time;
 
+        final TestInputTopic inputTopic1 =
+                driver.createInputTopic(topic1, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+        final TestInputTopic inputTopic2 =
+                driver.createInputTopic(topic2, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+
         // push four items with larger and increasing timestamp (out of window) to the other stream; this should produce no items
         // w1 = { 0:A0 (ts: 0), 1:A1 (ts: 0) }
         // w2 = { 0:a0 (ts: 0), 1:a1 (ts: 0), 2:a2 (ts: 0), 3:a3 (ts: 0) }
@@ -210,7 +223,7 @@ private void testUpperWindowBound(final int[] expectedKeys,
         //            0:b0 (ts: 1000), 1:b1 (ts: 1001), 2:b2 (ts: 1002), 3:b3 (ts: 1003) }
         time = 1000L;
         for (int i = 0; i < expectedKeys.length; i++) {
-            driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "b" + expectedKeys[i], time + i));
+            inputTopic2.pipeInput(expectedKeys[i], "b" + expectedKeys[i], time + i);
         }
         processor.checkAndClearProcessResult(EMPTY);
 
@@ -224,7 +237,7 @@ private void testUpperWindowBound(final int[] expectedKeys,
         //            0:b0 (ts: 1000), 1:b1 (ts: 1001), 2:b2 (ts: 1002), 3:b3 (ts: 1003) }
         time = 1000L + 100L;
         for (final int expectedKey : expectedKeys) {
-            driver.pipeInput(recordFactory.create(topic1, expectedKey, "B" + expectedKey, time));
+            inputTopic1.pipeInput(expectedKey, "B" + expectedKey, time);
         }
         processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "B0+b0", 1100),
             new KeyValueTimestamp<>(1, "B1+b1", 1100),
@@ -242,7 +255,7 @@ private void testUpperWindowBound(final int[] expectedKeys,
         //            0:b0 (ts: 1000), 1:b1 (ts: 1001), 2:b2 (ts: 1002), 3:b3 (ts: 1003) }
         time += 1L;
         for (final int expectedKey : expectedKeys) {
-            driver.pipeInput(recordFactory.create(topic1, expectedKey, "C" + expectedKey, time));
+            inputTopic1.pipeInput(expectedKey, "C" + expectedKey, time);
         }
         processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "C0+null", 1101),
             new KeyValueTimestamp<>(1, "C1+b1", 1101),
@@ -262,7 +275,7 @@ private void testUpperWindowBound(final int[] expectedKeys,
         //            0:b0 (ts: 1000), 1:b1 (ts: 1001), 2:b2 (ts: 1002), 3:b3 (ts: 1003) }
         time += 1L;
         for (final int expectedKey : expectedKeys) {
-            driver.pipeInput(recordFactory.create(topic1, expectedKey, "D" + expectedKey, time));
+            inputTopic1.pipeInput(expectedKey, "D" + expectedKey, time);
         }
         processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "D0+null", 1102),
             new KeyValueTimestamp<>(1, "D1+null", 1102),
@@ -284,7 +297,7 @@ private void testUpperWindowBound(final int[] expectedKeys,
         //            0:b0 (ts: 1000), 1:b1 (ts: 1001), 2:b2 (ts: 1002), 3:b3 (ts: 1003) }
         time += 1L;
         for (final int expectedKey : expectedKeys) {
-            driver.pipeInput(recordFactory.create(topic1, expectedKey, "E" + expectedKey, time));
+            inputTopic1.pipeInput(expectedKey, "E" + expectedKey, time);
         }
         processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "E0+null", 1103),
             new KeyValueTimestamp<>(1, "E1+null", 1103),
@@ -308,7 +321,7 @@ private void testUpperWindowBound(final int[] expectedKeys,
         //            0:b0 (ts: 1000), 1:b1 (ts: 1001), 2:b2 (ts: 1002), 3:b3 (ts: 1003) }
         time += 1L;
         for (final int expectedKey : expectedKeys) {
-            driver.pipeInput(recordFactory.create(topic1, expectedKey, "F" + expectedKey, time));
+            inputTopic1.pipeInput(expectedKey, "F" + expectedKey, time);
         }
         processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "F0+null", 1104),
             new KeyValueTimestamp<>(1, "F1+null", 1104),
@@ -320,6 +333,8 @@ private void testLowerWindowBound(final int[] expectedKeys,
                                       final TopologyTestDriver driver,
                                       final MockProcessor processor) {
         long time;
+        final TestInputTopic inputTopic1 = driver.createInputTopic(topic1, new IntegerSerializer(), new StringSerializer());
+        final TestInputTopic inputTopic2 = driver.createInputTopic(topic2, new IntegerSerializer(), new StringSerializer());
         // push four items with smaller timestamp (before the window) to the primary stream; this should produce four left-join and no full-join items
         // w1 = { 0:A0 (ts: 0), 1:A1 (ts: 0),
         //        0:B0 (ts: 1100), 1:B1 (ts: 1100), 2:B2 (ts: 1100), 3:B3 (ts: 1100),
@@ -340,7 +355,7 @@ private void testLowerWindowBound(final int[] expectedKeys,
         //            0:b0 (ts: 1000), 1:b1 (ts: 1001), 2:b2 (ts: 1002), 3:b3 (ts: 1003) }
         time = 1000L - 100L - 1L;
         for (final int expectedKey : expectedKeys) {
-            driver.pipeInput(recordFactory.create(topic1, expectedKey, "G" + expectedKey, time));
+            inputTopic1.pipeInput(expectedKey, "G" + expectedKey, time);
         }
         processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "G0+null", 899),
             new KeyValueTimestamp<>(1, "G1+null", 899), new KeyValueTimestamp<>(2, "G2+null", 899),
@@ -367,7 +382,7 @@ private void testLowerWindowBound(final int[] expectedKeys,
         //            0:b0 (ts: 1000), 1:b1 (ts: 1001), 2:b2 (ts: 1002), 3:b3 (ts: 1003) }
         time += 1L;
         for (final int expectedKey : expectedKeys) {
-            driver.pipeInput(recordFactory.create(topic1, expectedKey, "H" + expectedKey, time));
+            inputTopic1.pipeInput(expectedKey, "H" + expectedKey, time);
         }
         processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "H0+b0", 1000),
             new KeyValueTimestamp<>(1, "H1+null", 900), new KeyValueTimestamp<>(2, "H2+null", 900),
@@ -396,7 +411,7 @@ private void testLowerWindowBound(final int[] expectedKeys,
         //            0:b0 (ts: 1000), 1:b1 (ts: 1001), 2:b2 (ts: 1002), 3:b3 (ts: 1003) }
         time += 1L;
         for (final int expectedKey : expectedKeys) {
-            driver.pipeInput(recordFactory.create(topic1, expectedKey, "I" + expectedKey, time));
+            inputTopic1.pipeInput(expectedKey, "I" + expectedKey, time);
         }
         processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "I0+b0", 1000),
             new KeyValueTimestamp<>(1, "I1+b1", 1001), new KeyValueTimestamp<>(2, "I2+null", 901),
@@ -427,7 +442,7 @@ private void testLowerWindowBound(final int[] expectedKeys,
         //            0:b0 (ts: 1000), 1:b1 (ts: 1001), 2:b2 (ts: 1002), 3:b3 (ts: 1003) }
         time += 1L;
         for (final int expectedKey : expectedKeys) {
-            driver.pipeInput(recordFactory.create(topic1, expectedKey, "J" + expectedKey, time));
+            inputTopic1.pipeInput(expectedKey, "J" + expectedKey, time);
         }
         processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "J0+b0", 1000),
             new KeyValueTimestamp<>(1, "J1+b1", 1001), new KeyValueTimestamp<>(2, "J2+b2", 1002),
@@ -460,7 +475,7 @@ private void testLowerWindowBound(final int[] expectedKeys,
         //            0:b0 (ts: 1000), 1:b1 (ts: 1001), 2:b2 (ts: 1002), 3:b3 (ts: 1003) }
         time += 1L;
         for (final int expectedKey : expectedKeys) {
-            driver.pipeInput(recordFactory.create(topic1, expectedKey, "K" + expectedKey, time));
+            inputTopic1.pipeInput(expectedKey, "K" + expectedKey, time);
         }
         processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "K0+b0", 1000),
             new KeyValueTimestamp<>(1, "K1+b1", 1001), new KeyValueTimestamp<>(2, "K2+b2", 1002),
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableJoinTest.java
index 78f326c657edf..7d71456395f94 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableJoinTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableJoinTest.java
@@ -27,7 +27,7 @@
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KTable;
 import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockProcessor;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockValueJoiner;
@@ -36,6 +36,8 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
@@ -53,8 +55,8 @@ public class KStreamKTableJoinTest {
 
     private final String streamTopic = "streamTopic";
     private final String tableTopic = "tableTopic";
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer(), 0L);
+    private TestInputTopic inputStreamTopic;
+    private TestInputTopic inputTableTopic;
     private final int[] expectedKeys = {0, 1, 2, 3};
 
     private MockProcessor processor;
@@ -76,6 +78,8 @@ public void setUp() {
 
         final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String());
         driver = new TopologyTestDriver(builder.build(), props);
+        inputStreamTopic = driver.createInputTopic(streamTopic, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+        inputTableTopic = driver.createInputTopic(tableTopic, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
 
         processor = supplier.theCapturedProcessor();
     }
@@ -87,24 +91,23 @@ public void cleanup() {
 
     private void pushToStream(final int messageCount, final String valuePrefix) {
         for (int i = 0; i < messageCount; i++) {
-            driver.pipeInput(recordFactory.create(streamTopic, expectedKeys[i], valuePrefix + expectedKeys[i], i));
+            inputStreamTopic.pipeInput(expectedKeys[i], valuePrefix + expectedKeys[i], i);
         }
     }
 
     private void pushToTable(final int messageCount, final String valuePrefix) {
         final Random r = new Random(System.currentTimeMillis());
         for (int i = 0; i < messageCount; i++) {
-            driver.pipeInput(recordFactory.create(
-                tableTopic,
+            inputTableTopic.pipeInput(
                 expectedKeys[i],
                 valuePrefix + expectedKeys[i],
-                r.nextInt(Integer.MAX_VALUE)));
+                r.nextInt(Integer.MAX_VALUE));
         }
     }
 
     private void pushNullValueToTable() {
         for (int i = 0; i < 2; i++) {
-            driver.pipeInput(recordFactory.create(tableTopic, expectedKeys[i], null));
+            inputTableTopic.pipeInput(expectedKeys[i], null);
         }
     }
 
@@ -194,7 +197,7 @@ public void shouldClearTableEntryOnNullValueUpdates() {
     @Test
     public void shouldLogAndMeterWhenSkippingNullLeftKey() {
         final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
-        driver.pipeInput(recordFactory.create(streamTopic, null, "A"));
+        inputStreamTopic.pipeInput(null, "A");
         LogCaptureAppender.unregister(appender);
 
         assertEquals(1.0, getMetricByName(driver.metrics(), "skipped-records-total", "stream-metrics").metricValue());
@@ -204,7 +207,7 @@ public void shouldLogAndMeterWhenSkippingNullLeftKey() {
     @Test
     public void shouldLogAndMeterWhenSkippingNullLeftValue() {
         final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
-        driver.pipeInput(recordFactory.create(streamTopic, 1, null));
+        inputStreamTopic.pipeInput(1, null);
         LogCaptureAppender.unregister(appender);
 
         assertEquals(1.0, getMetricByName(driver.metrics(), "skipped-records-total", "stream-metrics").metricValue());
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableLeftJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableLeftJoinTest.java
index d9ad07dfc4673..30e0ff86e11bc 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableLeftJoinTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableLeftJoinTest.java
@@ -26,7 +26,7 @@
 import org.apache.kafka.streams.TopologyWrapper;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KTable;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockProcessor;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockValueJoiner;
@@ -35,6 +35,8 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
@@ -49,8 +51,8 @@ public class KStreamKTableLeftJoinTest {
 
     private final String streamTopic = "streamTopic";
     private final String tableTopic = "tableTopic";
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer(), 0L);
+    private TestInputTopic inputStreamTopic;
+    private TestInputTopic inputTableTopic;
     private final int[] expectedKeys = {0, 1, 2, 3};
 
     private TopologyTestDriver driver;
@@ -72,6 +74,8 @@ public void setUp() {
 
         final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String());
         driver = new TopologyTestDriver(builder.build(), props);
+        inputStreamTopic = driver.createInputTopic(streamTopic, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+        inputTableTopic = driver.createInputTopic(tableTopic, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
 
         processor = supplier.theCapturedProcessor();
     }
@@ -83,24 +87,23 @@ public void cleanup() {
 
     private void pushToStream(final int messageCount, final String valuePrefix) {
         for (int i = 0; i < messageCount; i++) {
-            driver.pipeInput(recordFactory.create(streamTopic, expectedKeys[i], valuePrefix + expectedKeys[i], i));
+            inputStreamTopic.pipeInput(expectedKeys[i], valuePrefix + expectedKeys[i], i);
         }
     }
 
     private void pushToTable(final int messageCount, final String valuePrefix) {
         final Random r = new Random(System.currentTimeMillis());
         for (int i = 0; i < messageCount; i++) {
-            driver.pipeInput(recordFactory.create(
-                tableTopic,
+            inputTableTopic.pipeInput(
                 expectedKeys[i],
                 valuePrefix + expectedKeys[i],
-                r.nextInt(Integer.MAX_VALUE)));
+                r.nextInt(Integer.MAX_VALUE));
         }
     }
 
     private void pushNullValueToTable(final int messageCount) {
         for (int i = 0; i < messageCount; i++) {
-            driver.pipeInput(recordFactory.create(tableTopic, expectedKeys[i], null));
+            inputTableTopic.pipeInput(expectedKeys[i], null);
         }
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapTest.java
index 8020e6de37ae1..99b77f484ea75 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapTest.java
@@ -25,18 +25,18 @@
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.KStream;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.StreamsTestUtils;
 import org.junit.Test;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.Properties;
 
 import static org.junit.Assert.assertEquals;
 
 public class KStreamMapTest {
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer(), 0L);
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String());
 
     @Test
@@ -51,7 +51,9 @@ public void testMap() {
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topicName, expectedKey, "V" + expectedKey, 10L - expectedKey));
+                final TestInputTopic inputTopic =
+                        driver.createInputTopic(topicName, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+                inputTopic.pipeInput(expectedKey, "V" + expectedKey, 10L - expectedKey);
             }
         }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapValuesTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapValuesTest.java
index d01b63b2d8dcc..4990a3ee566de 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapValuesTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapValuesTest.java
@@ -25,7 +25,7 @@
 import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.ValueMapperWithKey;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.StreamsTestUtils;
 import org.junit.Test;
@@ -37,8 +37,6 @@
 public class KStreamMapValuesTest {
     private final String topicName = "topic";
     private final MockProcessorSupplier supplier = new MockProcessorSupplier<>();
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer(), 0L);
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String());
 
     @Test
@@ -52,7 +50,9 @@ public void testFlatMapValues() {
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topicName, expectedKey, Integer.toString(expectedKey), expectedKey / 2L));
+                final TestInputTopic inputTopic =
+                        driver.createInputTopic(topicName, new IntegerSerializer(), new StringSerializer());
+                inputTopic.pipeInput(expectedKey, Integer.toString(expectedKey), expectedKey / 2L);
             }
         }
         final KeyValueTimestamp[] expected = {new KeyValueTimestamp<>(1, 1, 0),
@@ -75,8 +75,10 @@ public void testMapValuesWithKeys() {
         stream.mapValues(mapper).process(supplier);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(topicName, new IntegerSerializer(), new StringSerializer());
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topicName, expectedKey, Integer.toString(expectedKey), expectedKey / 2L));
+                inputTopic.pipeInput(expectedKey, Integer.toString(expectedKey), expectedKey / 2L);
             }
         }
         final KeyValueTimestamp[] expected = {new KeyValueTimestamp<>(1, 2, 0),
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamPeekTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamPeekTest.java
index 4b7b04d1afd09..f04278ba9e272 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamPeekTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamPeekTest.java
@@ -25,7 +25,7 @@
 import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.ForeachAction;
 import org.apache.kafka.streams.kstream.KStream;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.StreamsTestUtils;
 import org.junit.Test;
 
@@ -39,7 +39,6 @@
 public class KStreamPeekTest {
 
     private final String topicName = "topic";
-    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer());
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String());
 
     @Test
@@ -50,10 +49,11 @@ public void shouldObserveStreamElements() {
         stream.peek(collect(peekObserved)).foreach(collect(streamObserved));
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic = driver.createInputTopic(topicName, new IntegerSerializer(), new StringSerializer());
             final List> expected = new ArrayList<>();
             for (int key = 0; key < 32; key++) {
                 final String value = "V" + key;
-                driver.pipeInput(recordFactory.create(topicName, key, value));
+                inputTopic.pipeInput(key, value);
                 expected.add(new KeyValue<>(key, value));
             }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSelectKeyTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSelectKeyTest.java
index 63f79e12a0fc0..e6d2eb60bfffa 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSelectKeyTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSelectKeyTest.java
@@ -24,11 +24,13 @@
 import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.KStream;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.StreamsTestUtils;
 import org.junit.Test;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
@@ -37,8 +39,6 @@
 
 public class KStreamSelectKeyTest {
     private final String topicName = "topic_key_select";
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(topicName, new StringSerializer(), new IntegerSerializer(), 0L);
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.Integer());
 
     @Test
@@ -61,8 +61,10 @@ public void testSelectKey() {
         stream.selectKey((key, value) -> keyMap.get(value)).process(supplier);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(topicName, new StringSerializer(), new IntegerSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             for (final int expectedValue : expectedValues) {
-                driver.pipeInput(recordFactory.create(expectedValue));
+                inputTopic.pipeInput(expectedValue);
             }
         }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformTest.java
index f7ba7bf4a7cf6..1f37b3c6ca25c 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformTest.java
@@ -29,12 +29,13 @@
 import org.apache.kafka.streams.kstream.TransformerSupplier;
 import org.apache.kafka.streams.processor.ProcessorContext;
 import org.apache.kafka.streams.processor.PunctuationType;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.StreamsTestUtils;
 import org.junit.Test;
 
 import java.time.Duration;
+import java.time.Instant;
 import java.util.Properties;
 
 import static org.apache.kafka.common.utils.Utils.mkEntry;
@@ -44,8 +45,6 @@
 
 public class KStreamTransformTest {
     private static final String TOPIC_NAME = "topic";
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(new IntegerSerializer(), new IntegerSerializer(), 0L);
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.Integer());
 
     @Test
@@ -87,16 +86,16 @@ public void close() { }
                 mkEntry(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy"),
                 mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, "test")
             )),
-            0L)) {
-            final ConsumerRecordFactory recordFactory =
-                new ConsumerRecordFactory<>(TOPIC_NAME, new IntegerSerializer(), new IntegerSerializer());
+            Instant.ofEpochMilli(0L))) {
+            final TestInputTopic inputTopic =
+                driver.createInputTopic(TOPIC_NAME, new IntegerSerializer(), new IntegerSerializer());
 
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(expectedKey, expectedKey * 10, expectedKey / 2L));
+                inputTopic.pipeInput(expectedKey, expectedKey * 10, expectedKey / 2L);
             }
 
-            driver.advanceWallClockTime(2);
-            driver.advanceWallClockTime(1);
+            driver.advanceWallClockTime(Duration.ofMillis(2));
+            driver.advanceWallClockTime(Duration.ofMillis(1));
 
             final KeyValueTimestamp[] expected = {
                 new KeyValueTimestamp<>(2, 10, 0),
@@ -146,15 +145,17 @@ public void close() { }
         final KStream stream = builder.stream(TOPIC_NAME, Consumed.with(Serdes.Integer(), Serdes.Integer()));
         stream.transform(transformerSupplier).process(processor);
 
-        try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, 0L)) {
+        try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, Instant.ofEpochMilli(0L))) {
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(TOPIC_NAME, new IntegerSerializer(), new IntegerSerializer());
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(TOPIC_NAME, expectedKey, expectedKey * 10, 0L));
+                inputTopic.pipeInput(expectedKey, expectedKey * 10, 0L);
             }
 
             // This tick yields the "-1:2" result
-            driver.advanceWallClockTime(2);
+            driver.advanceWallClockTime(Duration.ofMillis(2));
             // This tick further advances the clock to 3, which leads to the "-1:3" result
-            driver.advanceWallClockTime(1);
+            driver.advanceWallClockTime(Duration.ofMillis(1));
         }
 
         assertEquals(6, processor.theCapturedProcessor().processed.size());
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValuesTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValuesTest.java
index adb6d3589cf8b..196f71c9e0a95 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValuesTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValuesTest.java
@@ -30,7 +30,7 @@
 import org.apache.kafka.streams.processor.Processor;
 import org.apache.kafka.streams.processor.ProcessorContext;
 import org.apache.kafka.streams.processor.internals.ForwardingDisabledProcessorContext;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.SingletonNoOpValueTransformer;
 import org.apache.kafka.test.StreamsTestUtils;
@@ -50,8 +50,6 @@
 public class KStreamTransformValuesTest {
     private final String topicName = "topic";
     private final MockProcessorSupplier supplier = new MockProcessorSupplier<>();
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(new IntegerSerializer(), new IntegerSerializer(), 0L);
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.Integer());
     @Mock(MockType.NICE)
     private ProcessorContext context;
@@ -85,7 +83,9 @@ public void close() { }
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topicName, expectedKey, expectedKey * 10, expectedKey / 2L));
+                final TestInputTopic inputTopic =
+                        driver.createInputTopic(topicName, new IntegerSerializer(), new IntegerSerializer());
+                inputTopic.pipeInput(expectedKey, expectedKey * 10, expectedKey / 2L);
             }
         }
         final KeyValueTimestamp[] expected = {new KeyValueTimestamp<>(1, 10, 0),
@@ -124,8 +124,10 @@ public void close() { }
         stream.transformValues(valueTransformerSupplier).process(supplier);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(topicName, new IntegerSerializer(), new IntegerSerializer());
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topicName, expectedKey, expectedKey * 10, expectedKey / 2L));
+                inputTopic.pipeInput(expectedKey, expectedKey * 10, expectedKey / 2L);
             }
         }
         final KeyValueTimestamp[] expected = {new KeyValueTimestamp<>(1, 11, 0),
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java
index 9051ea69438a0..7e51567cd716a 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
-import org.apache.kafka.clients.producer.ProducerRecord;
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.serialization.Serdes;
 import org.apache.kafka.common.serialization.StringDeserializer;
@@ -25,6 +24,7 @@
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.KeyValueTimestamp;
 import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.TestOutputTopic;
 import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.Grouped;
@@ -35,8 +35,8 @@
 import org.apache.kafka.streams.kstream.Windowed;
 import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender;
 import org.apache.kafka.streams.state.WindowStore;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
-import org.apache.kafka.streams.test.OutputVerifier;
+import org.apache.kafka.streams.TestInputTopic;
+import org.apache.kafka.streams.test.TestRecord;
 import org.apache.kafka.test.MockAggregator;
 import org.apache.kafka.test.MockInitializer;
 import org.apache.kafka.test.MockProcessor;
@@ -53,17 +53,16 @@
 import static org.apache.kafka.common.utils.Utils.mkEntry;
 import static org.apache.kafka.common.utils.Utils.mkMap;
 import static org.apache.kafka.test.StreamsTestUtils.getMetricByName;
+import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.hasItem;
 import static org.hamcrest.CoreMatchers.hasItems;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.not;
-import static org.hamcrest.CoreMatchers.nullValue;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 public class KStreamWindowAggregateTest {
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String());
     private final String threadId = Thread.currentThread().getName();
 
@@ -82,27 +81,29 @@ public void testAggBasic() {
         table2.toStream().process(supplier);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(topic1, "A", "1", 0L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "2", 1L));
-            driver.pipeInput(recordFactory.create(topic1, "C", "3", 2L));
-            driver.pipeInput(recordFactory.create(topic1, "D", "4", 3L));
-            driver.pipeInput(recordFactory.create(topic1, "A", "1", 4L));
-
-            driver.pipeInput(recordFactory.create(topic1, "A", "1", 5L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "2", 6L));
-            driver.pipeInput(recordFactory.create(topic1, "D", "4", 7L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "2", 8L));
-            driver.pipeInput(recordFactory.create(topic1, "C", "3", 9L));
-
-            driver.pipeInput(recordFactory.create(topic1, "A", "1", 10L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "2", 11L));
-            driver.pipeInput(recordFactory.create(topic1, "D", "4", 12L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "2", 13L));
-            driver.pipeInput(recordFactory.create(topic1, "C", "3", 14L));
-
-            driver.pipeInput(recordFactory.create(topic1, "B", "1", 3L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "2", 2L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "3", 9L));
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer());
+            inputTopic1.pipeInput("A", "1", 0L);
+            inputTopic1.pipeInput("B", "2", 1L);
+            inputTopic1.pipeInput("C", "3", 2L);
+            inputTopic1.pipeInput("D", "4", 3L);
+            inputTopic1.pipeInput("A", "1", 4L);
+
+            inputTopic1.pipeInput("A", "1", 5L);
+            inputTopic1.pipeInput("B", "2", 6L);
+            inputTopic1.pipeInput("D", "4", 7L);
+            inputTopic1.pipeInput("B", "2", 8L);
+            inputTopic1.pipeInput("C", "3", 9L);
+
+            inputTopic1.pipeInput("A", "1", 10L);
+            inputTopic1.pipeInput("B", "2", 11L);
+            inputTopic1.pipeInput("D", "4", 12L);
+            inputTopic1.pipeInput("B", "2", 13L);
+            inputTopic1.pipeInput("C", "3", 14L);
+
+            inputTopic1.pipeInput("B", "1", 3L);
+            inputTopic1.pipeInput("B", "2", 2L);
+            inputTopic1.pipeInput("B", "3", 9L);
         }
 
         assertEquals(
@@ -167,11 +168,15 @@ public void testJoin() {
         table1.join(table2, (p1, p2) -> p1 + "%" + p2).toStream().process(supplier);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(topic1, "A", "1", 0L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "2", 1L));
-            driver.pipeInput(recordFactory.create(topic1, "C", "3", 2L));
-            driver.pipeInput(recordFactory.create(topic1, "D", "4", 3L));
-            driver.pipeInput(recordFactory.create(topic1, "A", "1", 9L));
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer());
+            final TestInputTopic inputTopic2 =
+                    driver.createInputTopic(topic2, new StringSerializer(), new StringSerializer());
+            inputTopic1.pipeInput("A", "1", 0L);
+            inputTopic1.pipeInput("B", "2", 1L);
+            inputTopic1.pipeInput("C", "3", 2L);
+            inputTopic1.pipeInput("D", "4", 3L);
+            inputTopic1.pipeInput("A", "1", 9L);
 
             final List, String>> processors = supplier.capturedProcessors(3);
 
@@ -186,11 +191,11 @@ public void testJoin() {
             processors.get(1).checkAndClearProcessResult(new KeyValueTimestamp[0]);
             processors.get(2).checkAndClearProcessResult(new KeyValueTimestamp[0]);
 
-            driver.pipeInput(recordFactory.create(topic1, "A", "1", 5L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "2", 6L));
-            driver.pipeInput(recordFactory.create(topic1, "D", "4", 7L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "2", 8L));
-            driver.pipeInput(recordFactory.create(topic1, "C", "3", 9L));
+            inputTopic1.pipeInput("A", "1", 5L);
+            inputTopic1.pipeInput("B", "2", 6L);
+            inputTopic1.pipeInput("D", "4", 7L);
+            inputTopic1.pipeInput("B", "2", 8L);
+            inputTopic1.pipeInput("C", "3", 9L);
 
             processors.get(0).checkAndClearProcessResult(
                 new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)),  "0+1+1+1",  9),
@@ -207,11 +212,11 @@ public void testJoin() {
             processors.get(1).checkAndClearProcessResult(new KeyValueTimestamp[0]);
             processors.get(2).checkAndClearProcessResult(new KeyValueTimestamp[0]);
 
-            driver.pipeInput(recordFactory.create(topic2, "A", "a", 0L));
-            driver.pipeInput(recordFactory.create(topic2, "B", "b", 1L));
-            driver.pipeInput(recordFactory.create(topic2, "C", "c", 2L));
-            driver.pipeInput(recordFactory.create(topic2, "D", "d", 20L));
-            driver.pipeInput(recordFactory.create(topic2, "A", "a", 20L));
+            inputTopic2.pipeInput("A", "a", 0L);
+            inputTopic2.pipeInput("B", "b", 1L);
+            inputTopic2.pipeInput("C", "c", 2L);
+            inputTopic2.pipeInput("D", "d", 20L);
+            inputTopic2.pipeInput("A", "a", 20L);
 
             processors.get(0).checkAndClearProcessResult(new KeyValueTimestamp[0]);
             processors.get(1).checkAndClearProcessResult(
@@ -228,11 +233,11 @@ public void testJoin() {
                 new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(0, 10)),  "0+2+2+2%0+b",  8),
                 new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(0, 10)),  "0+3+3%0+c",  9));
 
-            driver.pipeInput(recordFactory.create(topic2, "A", "a", 5L));
-            driver.pipeInput(recordFactory.create(topic2, "B", "b", 6L));
-            driver.pipeInput(recordFactory.create(topic2, "D", "d", 7L));
-            driver.pipeInput(recordFactory.create(topic2, "D", "d", 18L));
-            driver.pipeInput(recordFactory.create(topic2, "A", "a", 21L));
+            inputTopic2.pipeInput("A", "a", 5L);
+            inputTopic2.pipeInput("B", "b", 6L);
+            inputTopic2.pipeInput("D", "d", 7L);
+            inputTopic2.pipeInput("D", "d", 18L);
+            inputTopic2.pipeInput("A", "a", 21L);
 
             processors.get(0).checkAndClearProcessResult(new KeyValueTimestamp[0]);
             processors.get(1).checkAndClearProcessResult(
@@ -275,7 +280,9 @@ public void shouldLogAndMeterWhenSkippingNullKey() {
 
         final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(topic, null, "1"));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(topic, new StringSerializer(), new StringSerializer());
+            inputTopic.pipeInput(null, "1");
             LogCaptureAppender.unregister(appender);
 
             assertEquals(1.0, getMetricByName(driver.metrics(), "skipped-records-total", "stream-metrics").metricValue());
@@ -304,14 +311,16 @@ public void shouldLogAndMeterWhenSkippingExpiredWindow() {
         LogCaptureAppender.setClassLoggerToDebug(KStreamWindowAggregate.class);
         final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(topic, "k", "100", 100L));
-            driver.pipeInput(recordFactory.create(topic, "k", "0", 0L));
-            driver.pipeInput(recordFactory.create(topic, "k", "1", 1L));
-            driver.pipeInput(recordFactory.create(topic, "k", "2", 2L));
-            driver.pipeInput(recordFactory.create(topic, "k", "3", 3L));
-            driver.pipeInput(recordFactory.create(topic, "k", "4", 4L));
-            driver.pipeInput(recordFactory.create(topic, "k", "5", 5L));
-            driver.pipeInput(recordFactory.create(topic, "k", "6", 6L));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(topic, new StringSerializer(), new StringSerializer());
+            inputTopic.pipeInput("k", "100", 100L);
+            inputTopic.pipeInput("k", "0", 0L);
+            inputTopic.pipeInput("k", "1", 1L);
+            inputTopic.pipeInput("k", "2", 2L);
+            inputTopic.pipeInput("k", "3", 3L);
+            inputTopic.pipeInput("k", "4", 4L);
+            inputTopic.pipeInput("k", "5", 5L);
+            inputTopic.pipeInput("k", "6", 6L);
             LogCaptureAppender.unregister(appender);
 
             assertLatenessMetrics(
@@ -331,11 +340,14 @@ public void shouldLogAndMeterWhenSkippingExpiredWindow() {
                 "Skipping record for expired window. key=[k] topic=[topic] partition=[0] offset=[7] timestamp=[6] window=[0,10) expiration=[10] streamTime=[100]"
             ));
 
-            OutputVerifier.compareKeyValueTimestamp(getOutput(driver), "[k@95/105]", "+100", 100);
-            OutputVerifier.compareKeyValueTimestamp(getOutput(driver), "[k@100/110]", "+100", 100);
-            OutputVerifier.compareKeyValueTimestamp(getOutput(driver), "[k@5/15]", "+5", 5);
-            OutputVerifier.compareKeyValueTimestamp(getOutput(driver), "[k@5/15]", "+5+6", 6);
-            assertThat(driver.readOutput("output"), nullValue());
+            final TestOutputTopic outputTopic =
+                    driver.createOutputTopic("output", new StringDeserializer(), new StringDeserializer());
+
+            assertThat(outputTopic.readRecord(), equalTo(new TestRecord("[k@95/105]", "+100", null, 100L)));
+            assertThat(outputTopic.readRecord(), equalTo(new TestRecord("[k@100/110]", "+100", null, 100L)));
+            assertThat(outputTopic.readRecord(), equalTo(new TestRecord("[k@5/15]", "+5", null, 5L)));
+            assertThat(outputTopic.readRecord(), equalTo(new TestRecord("[k@5/15]", "+5+6", null, 6L)));
+            assertTrue(outputTopic.isEmpty());
         }
     }
 
@@ -359,14 +371,16 @@ public void shouldLogAndMeterWhenSkippingExpiredWindowByGrace() {
         LogCaptureAppender.setClassLoggerToDebug(KStreamWindowAggregate.class);
         final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(topic, "k", "100", 200L));
-            driver.pipeInput(recordFactory.create(topic, "k", "0", 100L));
-            driver.pipeInput(recordFactory.create(topic, "k", "1", 101L));
-            driver.pipeInput(recordFactory.create(topic, "k", "2", 102L));
-            driver.pipeInput(recordFactory.create(topic, "k", "3", 103L));
-            driver.pipeInput(recordFactory.create(topic, "k", "4", 104L));
-            driver.pipeInput(recordFactory.create(topic, "k", "5", 105L));
-            driver.pipeInput(recordFactory.create(topic, "k", "6", 6L));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(topic, new StringSerializer(), new StringSerializer());
+            inputTopic.pipeInput("k", "100", 200L);
+            inputTopic.pipeInput("k", "0", 100L);
+            inputTopic.pipeInput("k", "1", 101L);
+            inputTopic.pipeInput("k", "2", 102L);
+            inputTopic.pipeInput("k", "3", 103L);
+            inputTopic.pipeInput("k", "4", 104L);
+            inputTopic.pipeInput("k", "5", 105L);
+            inputTopic.pipeInput("k", "6", 6L);
             LogCaptureAppender.unregister(appender);
 
             assertLatenessMetrics(driver, is(7.0), is(194.0), is(97.375));
@@ -381,8 +395,10 @@ public void shouldLogAndMeterWhenSkippingExpiredWindowByGrace() {
                 "Skipping record for expired window. key=[k] topic=[topic] partition=[0] offset=[7] timestamp=[6] window=[0,10) expiration=[110] streamTime=[200]"
             ));
 
-            OutputVerifier.compareKeyValueTimestamp(getOutput(driver), "[k@200/210]", "+100", 200);
-            assertThat(driver.readOutput("output"), nullValue());
+            final TestOutputTopic outputTopic =
+                    driver.createOutputTopic("output", new StringDeserializer(), new StringDeserializer());
+            assertThat(outputTopic.readRecord(), equalTo(new TestRecord("[k@200/210]", "+100", null, 200L)));
+            assertTrue(outputTopic.isEmpty());
         }
     }
 
@@ -437,7 +453,4 @@ private void assertLatenessMetrics(final TopologyTestDriver driver,
         assertThat(driver.metrics().get(latenessAvgMetric).metricValue(), avgLateness);
     }
 
-    private ProducerRecord getOutput(final TopologyTestDriver driver) {
-        return driver.readOutput("output", new StringDeserializer(), new StringDeserializer());
-    }
 }
\ No newline at end of file
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableAggregateTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableAggregateTest.java
index b2b58a518723f..bc2a6074e69bc 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableAggregateTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableAggregateTest.java
@@ -30,7 +30,7 @@
 import org.apache.kafka.streams.kstream.KTable;
 import org.apache.kafka.streams.kstream.Materialized;
 import org.apache.kafka.streams.state.KeyValueStore;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockAggregator;
 import org.apache.kafka.test.MockInitializer;
 import org.apache.kafka.test.MockMapper;
@@ -39,6 +39,9 @@
 import org.apache.kafka.test.TestUtils;
 import org.junit.Test;
 
+import java.time.Duration;
+import java.time.Instant;
+
 import static java.util.Arrays.asList;
 import static org.apache.kafka.common.utils.Utils.mkEntry;
 import static org.apache.kafka.common.utils.Utils.mkMap;
@@ -78,18 +81,18 @@ public void testAggBasic() {
                     mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, "test"),
                     mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory("kafka-test").getAbsolutePath())
                 )),
-                0L)) {
-            final ConsumerRecordFactory recordFactory =
-                new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L, 0L);
-
-            driver.pipeInput(recordFactory.create(topic1, "A", "1", 10L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "2", 15L));
-            driver.pipeInput(recordFactory.create(topic1, "A", "3", 20L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "4", 18L));
-            driver.pipeInput(recordFactory.create(topic1, "C", "5", 5L));
-            driver.pipeInput(recordFactory.create(topic1, "D", "6", 25L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "7", 15L));
-            driver.pipeInput(recordFactory.create(topic1, "C", "8", 10L));
+                Instant.ofEpochMilli(0L))) {
+            final TestInputTopic inputTopic =
+                driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+
+            inputTopic.pipeInput("A", "1", 10L);
+            inputTopic.pipeInput("B", "2", 15L);
+            inputTopic.pipeInput("A", "3", 20L);
+            inputTopic.pipeInput("B", "4", 18L);
+            inputTopic.pipeInput("C", "5", 5L);
+            inputTopic.pipeInput("D", "6", 25L);
+            inputTopic.pipeInput("B", "7", 15L);
+            inputTopic.pipeInput("C", "8", 10L);
 
             assertEquals(
                 asList(
@@ -145,18 +148,18 @@ public void testAggRepartition() {
                     mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, "test"),
                     mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory("kafka-test").getAbsolutePath())
                 )),
-                0L)) {
-            final ConsumerRecordFactory recordFactory =
-                new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L, 0L);
-
-            driver.pipeInput(recordFactory.create(topic1, "A", "1", 10L));
-            driver.pipeInput(recordFactory.create(topic1, "A", (String) null, 15L));
-            driver.pipeInput(recordFactory.create(topic1, "A", "1", 12L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "2", 20L));
-            driver.pipeInput(recordFactory.create(topic1, "null", "3", 25L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "4", 23L));
-            driver.pipeInput(recordFactory.create(topic1, "NULL", "5", 24L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "7", 22L));
+                Instant.ofEpochMilli(0L))) {
+            final TestInputTopic inputTopic =
+                driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+
+            inputTopic.pipeInput("A", "1", 10L);
+            inputTopic.pipeInput("A", (String) null, 15L);
+            inputTopic.pipeInput("A", "1", 12L);
+            inputTopic.pipeInput("B", "2", 20L);
+            inputTopic.pipeInput("null", "3", 25L);
+            inputTopic.pipeInput("B", "4", 23L);
+            inputTopic.pipeInput("NULL", "5", 24L);
+            inputTopic.pipeInput("B", "7", 22L);
 
             assertEquals(
                 asList(
@@ -183,15 +186,15 @@ private static void testCountHelper(final StreamsBuilder builder,
                     mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, "test"),
                     mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory("kafka-test").getAbsolutePath())
                 )),
-                0L)) {
-            final ConsumerRecordFactory recordFactory =
-                new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L, 0L);
+                Instant.ofEpochMilli(0L))) {
+            final TestInputTopic inputTopic =
+                driver.createInputTopic(input, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
 
-            driver.pipeInput(recordFactory.create(input, "A", "green", 10L));
-            driver.pipeInput(recordFactory.create(input, "B", "green", 9L));
-            driver.pipeInput(recordFactory.create(input, "A", "blue", 12L));
-            driver.pipeInput(recordFactory.create(input, "C", "yellow", 15L));
-            driver.pipeInput(recordFactory.create(input, "D", "green", 11L));
+            inputTopic.pipeInput("A", "green", 10L);
+            inputTopic.pipeInput("B", "green", 9L);
+            inputTopic.pipeInput("A", "blue", 12L);
+            inputTopic.pipeInput("C", "yellow", 15L);
+            inputTopic.pipeInput("D", "green", 11L);
 
             assertEquals(
                 asList(
@@ -266,16 +269,16 @@ public void testRemoveOldBeforeAddNew() {
                     mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, "test"),
                     mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory("kafka-test").getAbsolutePath())
                 )),
-                0L)) {
-            final ConsumerRecordFactory recordFactory =
-                new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L, 0L);
+                Instant.ofEpochMilli(0L))) {
+            final TestInputTopic inputTopic =
+                driver.createInputTopic(input, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
 
             final MockProcessor proc = supplier.theCapturedProcessor();
 
-            driver.pipeInput(recordFactory.create(input, "11", "A", 10L));
-            driver.pipeInput(recordFactory.create(input, "12", "B", 8L));
-            driver.pipeInput(recordFactory.create(input, "11", (String) null, 12L));
-            driver.pipeInput(recordFactory.create(input, "12", "C", 6L));
+            inputTopic.pipeInput("11", "A", 10L);
+            inputTopic.pipeInput("12", "B", 8L);
+            inputTopic.pipeInput("11", (String) null, 12L);
+            inputTopic.pipeInput("12", "C", 6L);
 
             assertEquals(
                 asList(
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java
index 2d32878e5fd74..f4c18541bb564 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java
@@ -32,7 +32,7 @@
 import org.apache.kafka.streams.kstream.Predicate;
 import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder;
 import org.apache.kafka.streams.state.ValueAndTimestamp;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockMapper;
 import org.apache.kafka.test.MockProcessor;
 import org.apache.kafka.test.MockProcessorSupplier;
@@ -41,6 +41,8 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.List;
 import java.util.Properties;
 
@@ -50,8 +52,6 @@
 @SuppressWarnings("unchecked")
 public class KTableFilterTest {
     private final Consumed consumed = Consumed.with(Serdes.String(), Serdes.Integer());
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(new StringSerializer(), new IntegerSerializer(), 0L);
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.Integer());
 
     @Before
@@ -71,12 +71,14 @@ private void doTestKTable(final StreamsBuilder builder,
         table3.toStream().process(supplier);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(topic, "A", 1, 10L));
-            driver.pipeInput(recordFactory.create(topic, "B", 2, 5L));
-            driver.pipeInput(recordFactory.create(topic, "C", 3, 8L));
-            driver.pipeInput(recordFactory.create(topic, "D", 4, 14L));
-            driver.pipeInput(recordFactory.create(topic, "A", null, 18L));
-            driver.pipeInput(recordFactory.create(topic, "B", null, 15L));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(topic, new StringSerializer(), new IntegerSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            inputTopic.pipeInput("A", 1, 10L);
+            inputTopic.pipeInput("B", 2, 5L);
+            inputTopic.pipeInput("C", 3, 8L);
+            inputTopic.pipeInput("D", 4, 14L);
+            inputTopic.pipeInput("A", null, 18L);
+            inputTopic.pipeInput("B", null, 15L);
         }
 
         final List> processors = supplier.capturedProcessors(2);
@@ -142,15 +144,18 @@ private void doTestValueGetter(final StreamsBuilder builder,
         topologyBuilder.connectProcessorAndStateStores(table3.name, getterSupplier3.storeNames());
 
         try (final TopologyTestDriverWrapper driver = new TopologyTestDriverWrapper(topology, props)) {
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(topic1, new StringSerializer(), new IntegerSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+
             final KTableValueGetter getter2 = getterSupplier2.get();
             final KTableValueGetter getter3 = getterSupplier3.get();
 
             getter2.init(driver.setCurrentNodeForProcessorContext(table2.name));
             getter3.init(driver.setCurrentNodeForProcessorContext(table3.name));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", 1, 5L));
-            driver.pipeInput(recordFactory.create(topic1, "B", 1, 10L));
-            driver.pipeInput(recordFactory.create(topic1, "C", 1, 15L));
+            inputTopic.pipeInput("A", 1, 5L);
+            inputTopic.pipeInput("B", 1, 10L);
+            inputTopic.pipeInput("C", 1, 15L);
 
             assertNull(getter2.get("A"));
             assertNull(getter2.get("B"));
@@ -160,8 +165,8 @@ private void doTestValueGetter(final StreamsBuilder builder,
             assertEquals(ValueAndTimestamp.make(1, 10L), getter3.get("B"));
             assertEquals(ValueAndTimestamp.make(1, 15L), getter3.get("C"));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", 2, 10L));
-            driver.pipeInput(recordFactory.create(topic1, "B", 2, 5L));
+            inputTopic.pipeInput("A", 2, 10L);
+            inputTopic.pipeInput("B", 2, 5L);
 
             assertEquals(ValueAndTimestamp.make(2, 10L), getter2.get("A"));
             assertEquals(ValueAndTimestamp.make(2, 5L), getter2.get("B"));
@@ -171,7 +176,7 @@ private void doTestValueGetter(final StreamsBuilder builder,
             assertNull(getter3.get("B"));
             assertEquals(ValueAndTimestamp.make(1, 15L), getter3.get("C"));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", 3, 15L));
+            inputTopic.pipeInput("A", 3, 15L);
 
             assertNull(getter2.get("A"));
             assertEquals(ValueAndTimestamp.make(2, 5L), getter2.get("B"));
@@ -181,8 +186,8 @@ private void doTestValueGetter(final StreamsBuilder builder,
             assertNull(getter3.get("B"));
             assertEquals(ValueAndTimestamp.make(1, 15L), getter3.get("C"));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", null, 10L));
-            driver.pipeInput(recordFactory.create(topic1, "B", null, 20L));
+            inputTopic.pipeInput("A", null, 10L);
+            inputTopic.pipeInput("B", null, 20L);
 
             assertNull(getter2.get("A"));
             assertNull(getter2.get("B"));
@@ -226,10 +231,12 @@ private void doTestNotSendingOldValue(final StreamsBuilder builder,
         builder.build().addProcessor("proc2", supplier, table2.name);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(topic1, new StringSerializer(), new IntegerSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
 
-            driver.pipeInput(recordFactory.create(topic1, "A", 1, 5L));
-            driver.pipeInput(recordFactory.create(topic1, "B", 1, 10L));
-            driver.pipeInput(recordFactory.create(topic1, "C", 1, 15L));
+            inputTopic.pipeInput("A", 1, 5L);
+            inputTopic.pipeInput("B", 1, 10L);
+            inputTopic.pipeInput("C", 1, 15L);
 
             final List> processors = supplier.capturedProcessors(2);
 
@@ -240,20 +247,20 @@ private void doTestNotSendingOldValue(final StreamsBuilder builder,
                 new KeyValueTimestamp<>("B", new Change<>(null, null), 10),
                 new KeyValueTimestamp<>("C", new Change<>(null, null), 15));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", 2, 15L));
-            driver.pipeInput(recordFactory.create(topic1, "B", 2, 8L));
+            inputTopic.pipeInput("A", 2, 15L);
+            inputTopic.pipeInput("B", 2, 8L);
 
             processors.get(0).checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(2, null), 15),
                 new KeyValueTimestamp<>("B", new Change<>(2, null), 8));
             processors.get(1).checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(2, null), 15),
                 new KeyValueTimestamp<>("B", new Change<>(2, null), 8));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", 3, 20L));
+            inputTopic.pipeInput("A", 3, 20L);
 
             processors.get(0).checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(3, null), 20));
             processors.get(1).checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(null, null), 20));
-            driver.pipeInput(recordFactory.create(topic1, "A", null, 10L));
-            driver.pipeInput(recordFactory.create(topic1, "B", null, 20L));
+            inputTopic.pipeInput("A", null, 10L);
+            inputTopic.pipeInput("B", null, 20L);
 
             processors.get(0).checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(null, null), 10),
                 new KeyValueTimestamp<>("B", new Change<>(null, null), 20));
@@ -301,9 +308,12 @@ private void doTestSendingOldValue(final StreamsBuilder builder,
         topology.addProcessor("proc2", supplier, table2.name);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) {
-            driver.pipeInput(recordFactory.create(topic1, "A", 1, 5L));
-            driver.pipeInput(recordFactory.create(topic1, "B", 1, 10L));
-            driver.pipeInput(recordFactory.create(topic1, "C", 1, 15L));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(topic1, new StringSerializer(), new IntegerSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+
+            inputTopic.pipeInput("A", 1, 5L);
+            inputTopic.pipeInput("B", 1, 10L);
+            inputTopic.pipeInput("C", 1, 15L);
 
             final List> processors = supplier.capturedProcessors(2);
 
@@ -312,21 +322,21 @@ private void doTestSendingOldValue(final StreamsBuilder builder,
                 new KeyValueTimestamp<>("C", new Change<>(1, null), 15));
             processors.get(1).checkEmptyAndClearProcessResult();
 
-            driver.pipeInput(recordFactory.create(topic1, "A", 2, 15L));
-            driver.pipeInput(recordFactory.create(topic1, "B", 2, 8L));
+            inputTopic.pipeInput("A", 2, 15L);
+            inputTopic.pipeInput("B", 2, 8L);
 
             processors.get(0).checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(2, 1), 15),
                 new KeyValueTimestamp<>("B", new Change<>(2, 1), 8));
             processors.get(1).checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(2, null), 15),
                 new KeyValueTimestamp<>("B", new Change<>(2, null), 8));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", 3, 20L));
+            inputTopic.pipeInput("A", 3, 20L);
 
             processors.get(0).checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(3, 2), 20));
             processors.get(1).checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(null, 2), 20));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", null, 10L));
-            driver.pipeInput(recordFactory.create(topic1, "B", null, 20L));
+            inputTopic.pipeInput("A", null, 10L);
+            inputTopic.pipeInput("B", null, 20L);
 
             processors.get(0).checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(null, 3), 10),
                 new KeyValueTimestamp<>("B", new Change<>(null, 2), 20));
@@ -370,13 +380,13 @@ private void doTestSkipNullOnMaterialization(final StreamsBuilder builder,
         topology.addProcessor("proc1", supplier, table1.name);
         topology.addProcessor("proc2", supplier, table2.name);
 
-        final ConsumerRecordFactory stringRecordFactory =
-            new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L);
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) {
+            final TestInputTopic stringinputTopic =
+                    driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
 
-            driver.pipeInput(stringRecordFactory.create(topic1, "A", "reject", 5L));
-            driver.pipeInput(stringRecordFactory.create(topic1, "B", "reject", 10L));
-            driver.pipeInput(stringRecordFactory.create(topic1, "C", "reject", 20L));
+            stringinputTopic.pipeInput("A", "reject", 5L);
+            stringinputTopic.pipeInput("B", "reject", 10L);
+            stringinputTopic.pipeInput("C", "reject", 20L);
         }
 
         final List> processors = supplier.capturedProcessors(2);
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java
index 9e8a3b571bdd8..f782d454fee4e 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java
@@ -42,7 +42,7 @@
 import org.apache.kafka.streams.processor.internals.SinkNode;
 import org.apache.kafka.streams.processor.internals.SourceNode;
 import org.apache.kafka.streams.state.KeyValueStore;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockAggregator;
 import org.apache.kafka.test.MockInitializer;
 import org.apache.kafka.test.MockMapper;
@@ -70,8 +70,6 @@ public class KTableImplTest {
     private final Consumed consumed = Consumed.with(Serdes.String(), Serdes.String());
     private final Produced produced = Produced.with(Serdes.String(), Serdes.String());
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String());
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L);
     private final Serde mySerde = new Serdes.StringSerde();
 
     private KTable table;
@@ -104,12 +102,14 @@ public void testKTable() {
         table4.toStream().process(supplier);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(topic1, "A", "01", 5L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "02", 100L));
-            driver.pipeInput(recordFactory.create(topic1, "C", "03", 0L));
-            driver.pipeInput(recordFactory.create(topic1, "D", "04", 0L));
-            driver.pipeInput(recordFactory.create(topic1, "A", "05", 10L));
-            driver.pipeInput(recordFactory.create(topic1, "A", "06", 8L));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer());
+            inputTopic.pipeInput("A", "01", 5L);
+            inputTopic.pipeInput("B", "02", 100L);
+            inputTopic.pipeInput("C", "03", 0L);
+            inputTopic.pipeInput("D", "04", 0L);
+            inputTopic.pipeInput("A", "05", 10L);
+            inputTopic.pipeInput("A", "06", 8L);
         }
 
         final List> processors = supplier.capturedProcessors(4);
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableInnerJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableInnerJoinTest.java
index e18b5a1c6854d..44cc693f9fedc 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableInnerJoinTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableInnerJoinTest.java
@@ -20,6 +20,7 @@
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.streams.KeyValueTimestamp;
 import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.TestOutputTopic;
 import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.TopologyWrapper;
 import org.apache.kafka.streams.kstream.Consumed;
@@ -29,14 +30,16 @@
 import org.apache.kafka.streams.processor.Processor;
 import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender;
 import org.apache.kafka.streams.state.KeyValueStore;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
-import org.apache.kafka.streams.test.OutputVerifier;
+import org.apache.kafka.streams.TestInputTopic;
+import org.apache.kafka.streams.test.TestRecord;
 import org.apache.kafka.test.MockProcessor;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockValueJoiner;
 import org.apache.kafka.test.StreamsTestUtils;
 import org.junit.Test;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
@@ -44,11 +47,11 @@
 import java.util.Set;
 
 import static org.apache.kafka.test.StreamsTestUtils.getMetricByName;
+import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.hasItem;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 public class KTableKTableInnerJoinTest {
@@ -60,8 +63,6 @@ public class KTableKTableInnerJoinTest {
     private final Consumed consumed = Consumed.with(Serdes.Integer(), Serdes.String());
     private final Materialized> materialized =
         Materialized.with(Serdes.Integer(), Serdes.String());
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(Serdes.Integer().serializer(), Serdes.String().serializer(), 0L);
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String());
 
     @Test
@@ -156,6 +157,10 @@ public void testSendingOldValues() {
         builder.build().addProcessor("proc", supplier, ((KTableImpl) joined).name);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic2 =
+                    driver.createInputTopic(topic2, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             final MockProcessor proc = supplier.theCapturedProcessor();
 
             assertTrue(((KTableImpl) table1).sendingOldValueEnabled());
@@ -164,27 +169,27 @@ public void testSendingOldValues() {
 
             // push two items to the primary stream. the other table is empty
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i], 5L + i));
+                inputTopic1.pipeInput(expectedKeys[i], "X" + expectedKeys[i], 5L + i);
             }
             // pass tuple with null key, it will be discarded in join process
-            driver.pipeInput(recordFactory.create(topic1, null, "SomeVal", 42L));
+            inputTopic1.pipeInput(null, "SomeVal", 42L);
             // left: X0:0 (ts: 5), X1:1 (ts: 6)
             // right:
             proc.checkAndClearProcessResult(EMPTY);
 
             // push two items to the other stream. this should produce two items.
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i], 10L * i));
+                inputTopic2.pipeInput(expectedKeys[i], "Y" + expectedKeys[i], 10L * i);
             }
             // pass tuple with null key, it will be discarded in join process
-            driver.pipeInput(recordFactory.create(topic2, null, "AnotherVal", 73L));
+            inputTopic2.pipeInput(null, "AnotherVal", 73L);
             // left: X0:0 (ts: 5), X1:1 (ts: 6)
             // right: Y0:0 (ts: 0), Y1:1 (ts: 10)
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("X0+Y0", null), 5),
                 new KeyValueTimestamp<>(1, new Change<>("X1+Y1", null), 10));
             // push all four items to the primary stream. this should produce two items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, 7L));
+                inputTopic1.pipeInput(expectedKey, "XX" + expectedKey, 7L);
             }
             // left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
             // right: Y0:0 (ts: 0), Y1:1 (ts: 10)
@@ -192,7 +197,7 @@ public void testSendingOldValues() {
                 new KeyValueTimestamp<>(1, new Change<>("XX1+Y1", "X1+Y1"), 10));
             // push all items to the other stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, expectedKey * 5L));
+                inputTopic2.pipeInput(expectedKey, "YY" + expectedKey, expectedKey * 5L);
             }
             // left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
             // right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
@@ -204,7 +209,7 @@ public void testSendingOldValues() {
 
             // push all four items to the primary stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XXX" + expectedKey, 6L));
+                inputTopic1.pipeInput(expectedKey, "XXX" + expectedKey, 6L);
             }
             // left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
             // right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
@@ -215,25 +220,25 @@ public void testSendingOldValues() {
                 new KeyValueTimestamp<>(3, new Change<>("XXX3+YY3", "XX3+YY3"), 15));
 
             // push two items with null to the other stream as deletes. this should produce two item.
-            driver.pipeInput(recordFactory.create(topic2, expectedKeys[0], null, 5L));
-            driver.pipeInput(recordFactory.create(topic2, expectedKeys[1], null, 7L));
+            inputTopic2.pipeInput(expectedKeys[0], null, 5L);
+            inputTopic2.pipeInput(expectedKeys[1], null, 7L);
             // left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>(null, "XXX0+YY0"), 6),
                 new KeyValueTimestamp<>(1, new Change<>(null, "XXX1+YY1"), 7));
             // push all four items to the primary stream. this should produce two items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XXXX" + expectedKey, 13L));
+                inputTopic1.pipeInput(expectedKey, "XXXX" + expectedKey, 13L);
             }
             // left: XXXX0:0 (ts: 13), XXXX1:1 (ts: 13), XXXX2:2 (ts: 13), XXXX3:3 (ts: 13)
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(2, new Change<>("XXXX2+YY2", "XXX2+YY2"), 13),
                 new KeyValueTimestamp<>(3, new Change<>("XXXX3+YY3", "XXX3+YY3"), 15));
             // push four items to the primary stream with null. this should produce two items.
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[0], null, 0L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[1], null, 42L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[2], null, 5L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[3], null, 20L));
+            inputTopic1.pipeInput(expectedKeys[0], null, 0L);
+            inputTopic1.pipeInput(expectedKeys[1], null, 42L);
+            inputTopic1.pipeInput(expectedKeys[2], null, 5L);
+            inputTopic1.pipeInput(expectedKeys[3], null, 20L);
             // left:
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(2, new Change<>(null, "XXXX2+YY2"), 10),
@@ -271,6 +276,10 @@ private void doTestNotSendingOldValues(final StreamsBuilder builder,
                                            final KTable joined) {
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic2 =
+                    driver.createInputTopic(topic2, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             final MockProcessor proc = supplier.theCapturedProcessor();
 
             assertFalse(((KTableImpl) table1).sendingOldValueEnabled());
@@ -279,27 +288,27 @@ private void doTestNotSendingOldValues(final StreamsBuilder builder,
 
             // push two items to the primary stream. the other table is empty
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i], 5L + i));
+                inputTopic1.pipeInput(expectedKeys[i], "X" + expectedKeys[i], 5L + i);
             }
             // pass tuple with null key, it will be discarded in join process
-            driver.pipeInput(recordFactory.create(topic1, null, "SomeVal", 42L));
+            inputTopic1.pipeInput(null, "SomeVal", 42L);
             // left: X0:0 (ts: 5), X1:1 (ts: 6)
             // right:
             proc.checkAndClearProcessResult(EMPTY);
 
             // push two items to the other stream. this should produce two items.
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i], 10L * i));
+                inputTopic2.pipeInput(expectedKeys[i], "Y" + expectedKeys[i], 10L * i);
             }
             // pass tuple with null key, it will be discarded in join process
-            driver.pipeInput(recordFactory.create(topic2, null, "AnotherVal", 73L));
+            inputTopic2.pipeInput(null, "AnotherVal", 73L);
             // left: X0:0 (ts: 5), X1:1 (ts: 6)
             // right: Y0:0 (ts: 0), Y1:1 (ts: 10)
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("X0+Y0", null), 5),
                 new KeyValueTimestamp<>(1, new Change<>("X1+Y1", null), 10));
             // push all four items to the primary stream. this should produce two items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, 7L));
+                inputTopic1.pipeInput(expectedKey, "XX" + expectedKey, 7L);
             }
             // left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
             // right: Y0:0 (ts: 0), Y1:1 (ts: 10)
@@ -307,7 +316,7 @@ private void doTestNotSendingOldValues(final StreamsBuilder builder,
                 new KeyValueTimestamp<>(1, new Change<>("XX1+Y1", null), 10));
             // push all items to the other stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, expectedKey * 5L));
+                inputTopic2.pipeInput(expectedKey, "YY" + expectedKey, expectedKey * 5L);
             }
             // left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
             // right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
@@ -319,7 +328,7 @@ private void doTestNotSendingOldValues(final StreamsBuilder builder,
 
             // push all four items to the primary stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XXX" + expectedKey, 6L));
+                inputTopic1.pipeInput(expectedKey, "XXX" + expectedKey, 6L);
             }
             // left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
             // right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
@@ -330,25 +339,25 @@ private void doTestNotSendingOldValues(final StreamsBuilder builder,
                 new KeyValueTimestamp<>(3, new Change<>("XXX3+YY3", null), 15));
 
             // push two items with null to the other stream as deletes. this should produce two item.
-            driver.pipeInput(recordFactory.create(topic2, expectedKeys[0], null, 5L));
-            driver.pipeInput(recordFactory.create(topic2, expectedKeys[1], null, 7L));
+            inputTopic2.pipeInput(expectedKeys[0], null, 5L);
+            inputTopic2.pipeInput(expectedKeys[1], null, 7L);
             // left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>(null, null), 6),
                 new KeyValueTimestamp<>(1, new Change<>(null, null), 7));
             // push all four items to the primary stream. this should produce two items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XXXX" + expectedKey, 13L));
+                inputTopic1.pipeInput(expectedKey, "XXXX" + expectedKey, 13L);
             }
             // left: XXXX0:0 (ts: 13), XXXX1:1 (ts: 13), XXXX2:2 (ts: 13), XXXX3:3 (ts: 13)
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(2, new Change<>("XXXX2+YY2", null), 13),
                 new KeyValueTimestamp<>(3, new Change<>("XXXX3+YY3", null), 15));
             // push four items to the primary stream with null. this should produce two items.
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[0], null, 0L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[1], null, 42L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[2], null, 5L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[3], null, 20L));
+            inputTopic1.pipeInput(expectedKeys[0], null, 0L);
+            inputTopic1.pipeInput(expectedKeys[1], null, 42L);
+            inputTopic1.pipeInput(expectedKeys[2], null, 5L);
+            inputTopic1.pipeInput(expectedKeys[3], null, 20L);
             // left:
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(2, new Change<>(null, null), 10),
@@ -364,103 +373,106 @@ private void doTestJoin(final StreamsBuilder builder, final int[] expectedKeys)
         assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next());
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic2 =
+                    driver.createInputTopic(topic2, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestOutputTopic outputTopic =
+                    driver.createOutputTopic(output, Serdes.Integer().deserializer(), Serdes.String().deserializer());
+
             // push two items to the primary stream. the other table is empty
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i], 5L + i));
+                inputTopic1.pipeInput(expectedKeys[i], "X" + expectedKeys[i], 5L + i);
             }
             // pass tuple with null key, it will be discarded in join process
-            driver.pipeInput(recordFactory.create(topic1, null, "SomeVal", 42L));
+            inputTopic1.pipeInput(null, "SomeVal", 42L);
             // left: X0:0 (ts: 5), X1:1 (ts: 6)
             // right:
-            assertNull(driver.readOutput(output));
+            assertTrue(outputTopic.isEmpty());
 
             // push two items to the other stream. this should produce two items.
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i], 10L * i));
+                inputTopic2.pipeInput(expectedKeys[i], "Y" + expectedKeys[i], 10L * i);
             }
             // pass tuple with null key, it will be discarded in join process
-            driver.pipeInput(recordFactory.create(topic2, null, "AnotherVal", 73L));
+            inputTopic2.pipeInput(null, "AnotherVal", 73L);
             // left: X0:0 (ts: 5), X1:1 (ts: 6)
             // right: Y0:0 (ts: 0), Y1:1 (ts: 10)
-            assertOutputKeyValueTimestamp(driver, 0, "X0+Y0", 5L);
-            assertOutputKeyValueTimestamp(driver, 1, "X1+Y1", 10L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 0, "X0+Y0", 5L);
+            assertOutputKeyValueTimestamp(outputTopic, 1, "X1+Y1", 10L);
+            assertTrue(outputTopic.isEmpty());
 
             // push all four items to the primary stream. this should produce two items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, 7L));
+                inputTopic1.pipeInput(expectedKey, "XX" + expectedKey, 7L);
             }
             // left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
             // right: Y0:0 (ts: 0), Y1:1 (ts: 10)
-            assertOutputKeyValueTimestamp(driver, 0, "XX0+Y0", 7L);
-            assertOutputKeyValueTimestamp(driver, 1, "XX1+Y1", 10L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 0, "XX0+Y0", 7L);
+            assertOutputKeyValueTimestamp(outputTopic, 1, "XX1+Y1", 10L);
+            assertTrue(outputTopic.isEmpty());
 
             // push all items to the other stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, expectedKey * 5L));
+                inputTopic2.pipeInput(expectedKey, "YY" + expectedKey, expectedKey * 5L);
             }
             // left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
             // right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
-            assertOutputKeyValueTimestamp(driver, 0, "XX0+YY0", 7L);
-            assertOutputKeyValueTimestamp(driver, 1, "XX1+YY1", 7L);
-            assertOutputKeyValueTimestamp(driver, 2, "XX2+YY2", 10L);
-            assertOutputKeyValueTimestamp(driver, 3, "XX3+YY3", 15L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 0, "XX0+YY0", 7L);
+            assertOutputKeyValueTimestamp(outputTopic, 1, "XX1+YY1", 7L);
+            assertOutputKeyValueTimestamp(outputTopic, 2, "XX2+YY2", 10L);
+            assertOutputKeyValueTimestamp(outputTopic, 3, "XX3+YY3", 15L);
+            assertTrue(outputTopic.isEmpty());
 
             // push all four items to the primary stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XXX" + expectedKey, 6L));
+                inputTopic1.pipeInput(expectedKey, "XXX" + expectedKey, 6L);
             }
             // left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
             // right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
-            assertOutputKeyValueTimestamp(driver, 0, "XXX0+YY0", 6L);
-            assertOutputKeyValueTimestamp(driver, 1, "XXX1+YY1", 6L);
-            assertOutputKeyValueTimestamp(driver, 2, "XXX2+YY2", 10L);
-            assertOutputKeyValueTimestamp(driver, 3, "XXX3+YY3", 15L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 0, "XXX0+YY0", 6L);
+            assertOutputKeyValueTimestamp(outputTopic, 1, "XXX1+YY1", 6L);
+            assertOutputKeyValueTimestamp(outputTopic, 2, "XXX2+YY2", 10L);
+            assertOutputKeyValueTimestamp(outputTopic, 3, "XXX3+YY3", 15L);
+            assertTrue(outputTopic.isEmpty());
 
             // push two items with null to the other stream as deletes. this should produce two item.
-            driver.pipeInput(recordFactory.create(topic2, expectedKeys[0], null, 5L));
-            driver.pipeInput(recordFactory.create(topic2, expectedKeys[1], null, 7L));
+            inputTopic2.pipeInput(expectedKeys[0], null, 5L);
+            inputTopic2.pipeInput(expectedKeys[1], null, 7L);
             // left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
-            assertOutputKeyValueTimestamp(driver, 0, null, 6L);
-            assertOutputKeyValueTimestamp(driver, 1, null, 7L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 0, null, 6L);
+            assertOutputKeyValueTimestamp(outputTopic, 1, null, 7L);
+            assertTrue(outputTopic.isEmpty());
 
             // push all four items to the primary stream. this should produce two items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XXXX" + expectedKey, 13L));
+                inputTopic1.pipeInput(expectedKey, "XXXX" + expectedKey, 13L);
             }
             // left: XXXX0:0 (ts: 13), XXXX1:1 (ts: 13), XXXX2:2 (ts: 13), XXXX3:3 (ts: 13)
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
-            assertOutputKeyValueTimestamp(driver, 2, "XXXX2+YY2", 13L);
-            assertOutputKeyValueTimestamp(driver, 3, "XXXX3+YY3", 15L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 2, "XXXX2+YY2", 13L);
+            assertOutputKeyValueTimestamp(outputTopic, 3, "XXXX3+YY3", 15L);
+            assertTrue(outputTopic.isEmpty());
 
             // push fourt items to the primary stream with null. this should produce two items.
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[0], null, 0L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[1], null, 42L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[2], null, 5L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[3], null, 20L));
+            inputTopic1.pipeInput(expectedKeys[0], null, 0L);
+            inputTopic1.pipeInput(expectedKeys[1], null, 42L);
+            inputTopic1.pipeInput(expectedKeys[2], null, 5L);
+            inputTopic1.pipeInput(expectedKeys[3], null, 20L);
             // left:
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
-            assertOutputKeyValueTimestamp(driver, 2, null, 10L);
-            assertOutputKeyValueTimestamp(driver, 3, null, 20L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 2, null, 10L);
+            assertOutputKeyValueTimestamp(outputTopic, 3, null, 20L);
+            assertTrue(outputTopic.isEmpty());
         }
     }
 
-    private void assertOutputKeyValueTimestamp(final TopologyTestDriver driver,
+    private void assertOutputKeyValueTimestamp(final TestOutputTopic outputTopic,
                                                final Integer expectedKey,
                                                final String expectedValue,
                                                final long expectedTimestamp) {
-        OutputVerifier.compareKeyValueTimestamp(
-            driver.readOutput(output, Serdes.Integer().deserializer(), Serdes.String().deserializer()),
-            expectedKey,
-            expectedValue,
-            expectedTimestamp);
+        assertThat(outputTopic.readRecord(), equalTo(new TestRecord(expectedKey, expectedValue, null, expectedTimestamp)));
     }
 
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoinTest.java
index 36417e92ba988..958c8969c27d5 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoinTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoinTest.java
@@ -20,6 +20,7 @@
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.KeyValueTimestamp;
 import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.TestOutputTopic;
 import org.apache.kafka.streams.Topology;
 import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.TopologyTestDriverWrapper;
@@ -33,8 +34,8 @@
 import org.apache.kafka.streams.processor.Processor;
 import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender;
 import org.apache.kafka.streams.state.Stores;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
-import org.apache.kafka.streams.test.OutputVerifier;
+import org.apache.kafka.streams.TestInputTopic;
+import org.apache.kafka.streams.test.TestRecord;
 import org.apache.kafka.test.MockProcessor;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockReducer;
@@ -42,6 +43,8 @@
 import org.apache.kafka.test.StreamsTestUtils;
 import org.junit.Test;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
@@ -51,11 +54,11 @@
 import java.util.Set;
 
 import static org.apache.kafka.test.StreamsTestUtils.getMetricByName;
+import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.hasItem;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 public class KTableKTableLeftJoinTest {
@@ -63,8 +66,6 @@ public class KTableKTableLeftJoinTest {
     private final String topic2 = "topic2";
     private final String output = "output";
     private final Consumed consumed = Consumed.with(Serdes.Integer(), Serdes.String());
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(Serdes.Integer().serializer(), Serdes.String().serializer(), 0L);
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String());
 
     @Test
@@ -85,99 +86,106 @@ public void testJoin() {
         assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next());
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic2 =
+                    driver.createInputTopic(topic2, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestOutputTopic outputTopic =
+                    driver.createOutputTopic(output, Serdes.Integer().deserializer(), Serdes.String().deserializer());
+
             // push two items to the primary stream. the other table is empty
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i], 5L + i));
+                inputTopic1.pipeInput(expectedKeys[i], "X" + expectedKeys[i], 5L + i);
             }
             // pass tuple with null key, it will be discarded in join process
-            driver.pipeInput(recordFactory.create(topic1, null, "SomeVal", 42L));
+            inputTopic1.pipeInput(null, "SomeVal", 42L);
             // left: X0:0 (ts: 5), X1:1 (ts: 6)
             // right:
-            assertOutputKeyValueTimestamp(driver, 0, "X0+null", 5L);
-            assertOutputKeyValueTimestamp(driver, 1, "X1+null", 6L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 0, "X0+null", 5L);
+            assertOutputKeyValueTimestamp(outputTopic, 1, "X1+null", 6L);
+            assertTrue(outputTopic.isEmpty());
 
             // push two items to the other stream. this should produce two items.
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i], 10L * i));
+                inputTopic2.pipeInput(expectedKeys[i], "Y" + expectedKeys[i], 10L * i);
             }
             // pass tuple with null key, it will be discarded in join process
-            driver.pipeInput(recordFactory.create(topic2, null, "AnotherVal", 73L));
+            inputTopic2.pipeInput(null, "AnotherVal", 73L);
             // left: X0:0 (ts: 5), X1:1 (ts: 6)
             // right: Y0:0 (ts: 0), Y1:1 (ts: 10)
-            assertOutputKeyValueTimestamp(driver, 0, "X0+Y0", 5L);
-            assertOutputKeyValueTimestamp(driver, 1, "X1+Y1", 10L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 0, "X0+Y0", 5L);
+            assertOutputKeyValueTimestamp(outputTopic, 1, "X1+Y1", 10L);
+            assertTrue(outputTopic.isEmpty());
 
             // push all four items to the primary stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, 7L));
+                inputTopic1.pipeInput(expectedKey, "XX" + expectedKey, 7L);
             }
             // left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
             // right: Y0:0 (ts: 0), Y1:1 (ts: 10)
-            assertOutputKeyValueTimestamp(driver, 0, "XX0+Y0", 7L);
-            assertOutputKeyValueTimestamp(driver, 1, "XX1+Y1", 10L);
-            assertOutputKeyValueTimestamp(driver, 2, "XX2+null", 7L);
-            assertOutputKeyValueTimestamp(driver, 3, "XX3+null", 7L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 0, "XX0+Y0", 7L);
+            assertOutputKeyValueTimestamp(outputTopic, 1, "XX1+Y1", 10L);
+            assertOutputKeyValueTimestamp(outputTopic, 2, "XX2+null", 7L);
+            assertOutputKeyValueTimestamp(outputTopic, 3, "XX3+null", 7L);
+            assertTrue(outputTopic.isEmpty());
 
             // push all items to the other stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, expectedKey * 5L));
+                inputTopic2.pipeInput(expectedKey, "YY" + expectedKey, expectedKey * 5L);
             }
             // left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
             // right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
-            assertOutputKeyValueTimestamp(driver, 0, "XX0+YY0", 7L);
-            assertOutputKeyValueTimestamp(driver, 1, "XX1+YY1", 7L);
-            assertOutputKeyValueTimestamp(driver, 2, "XX2+YY2", 10L);
-            assertOutputKeyValueTimestamp(driver, 3, "XX3+YY3", 15L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 0, "XX0+YY0", 7L);
+            assertOutputKeyValueTimestamp(outputTopic, 1, "XX1+YY1", 7L);
+            assertOutputKeyValueTimestamp(outputTopic, 2, "XX2+YY2", 10L);
+            assertOutputKeyValueTimestamp(outputTopic, 3, "XX3+YY3", 15L);
+            assertTrue(outputTopic.isEmpty());
 
             // push all four items to the primary stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XXX" + expectedKey, 6L));
+                inputTopic1.pipeInput(expectedKey, "XXX" + expectedKey, 6L);
             }
             // left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
             // right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
-            assertOutputKeyValueTimestamp(driver, 0, "XXX0+YY0", 6L);
-            assertOutputKeyValueTimestamp(driver, 1, "XXX1+YY1", 6L);
-            assertOutputKeyValueTimestamp(driver, 2, "XXX2+YY2", 10L);
-            assertOutputKeyValueTimestamp(driver, 3, "XXX3+YY3", 15L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 0, "XXX0+YY0", 6L);
+            assertOutputKeyValueTimestamp(outputTopic, 1, "XXX1+YY1", 6L);
+            assertOutputKeyValueTimestamp(outputTopic, 2, "XXX2+YY2", 10L);
+            assertOutputKeyValueTimestamp(outputTopic, 3, "XXX3+YY3", 15L);
+            assertTrue(outputTopic.isEmpty());
 
             // push two items with null to the other stream as deletes. this should produce two item.
-            driver.pipeInput(recordFactory.create(topic2, expectedKeys[0], null, 5L));
-            driver.pipeInput(recordFactory.create(topic2, expectedKeys[1], null, 7L));
+            inputTopic2.pipeInput(expectedKeys[0], null, 5L);
+            inputTopic2.pipeInput(expectedKeys[1], null, 7L);
             // left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
-            assertOutputKeyValueTimestamp(driver, 0, "XXX0+null", 6L);
-            assertOutputKeyValueTimestamp(driver, 1, "XXX1+null", 7L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 0, "XXX0+null", 6L);
+            assertOutputKeyValueTimestamp(outputTopic, 1, "XXX1+null", 7L);
+            assertTrue(outputTopic.isEmpty());
 
             // push all four items to the primary stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XXXX" + expectedKey, 13L));
+                inputTopic1.pipeInput(expectedKey, "XXXX" + expectedKey, 13L);
             }
             // left: XXXX0:0 (ts: 13), XXXX1:1 (ts: 13), XXXX2:2 (ts: 13), XXXX3:3 (ts: 13)
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
-            assertOutputKeyValueTimestamp(driver, 0, "XXXX0+null", 13L);
-            assertOutputKeyValueTimestamp(driver, 1, "XXXX1+null", 13L);
-            assertOutputKeyValueTimestamp(driver, 2, "XXXX2+YY2", 13L);
-            assertOutputKeyValueTimestamp(driver, 3, "XXXX3+YY3", 15L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 0, "XXXX0+null", 13L);
+            assertOutputKeyValueTimestamp(outputTopic, 1, "XXXX1+null", 13L);
+            assertOutputKeyValueTimestamp(outputTopic, 2, "XXXX2+YY2", 13L);
+            assertOutputKeyValueTimestamp(outputTopic, 3, "XXXX3+YY3", 15L);
+            assertTrue(outputTopic.isEmpty());
 
             // push three items to the primary stream with null. this should produce four items.
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[0], null, 0L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[1], null, 42L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[2], null, 5L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[3], null, 20L));
+            inputTopic1.pipeInput(expectedKeys[0], null, 0L);
+            inputTopic1.pipeInput(expectedKeys[1], null, 42L);
+            inputTopic1.pipeInput(expectedKeys[2], null, 5L);
+            inputTopic1.pipeInput(expectedKeys[3], null, 20L);
             // left:
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
-            assertOutputKeyValueTimestamp(driver, 0, null, 0L);
-            assertOutputKeyValueTimestamp(driver, 1, null, 42L);
-            assertOutputKeyValueTimestamp(driver, 2, null, 10L);
-            assertOutputKeyValueTimestamp(driver, 3, null, 20L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 0, null, 0L);
+            assertOutputKeyValueTimestamp(outputTopic, 1, null, 42L);
+            assertOutputKeyValueTimestamp(outputTopic, 2, null, 10L);
+            assertOutputKeyValueTimestamp(outputTopic, 3, null, 20L);
+            assertTrue(outputTopic.isEmpty());
         }
     }
 
@@ -200,6 +208,10 @@ public void testNotSendingOldValue() {
         final Topology topology = builder.build().addProcessor("proc", supplier, ((KTableImpl) joined).name);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic2 =
+                    driver.createInputTopic(topic2, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             final MockProcessor proc = supplier.theCapturedProcessor();
 
             assertTrue(((KTableImpl) table1).sendingOldValueEnabled());
@@ -208,10 +220,10 @@ public void testNotSendingOldValue() {
 
             // push two items to the primary stream. the other table is empty
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i], 5L + i));
+                inputTopic1.pipeInput(expectedKeys[i], "X" + expectedKeys[i], 5L + i);
             }
             // pass tuple with null key, it will be discarded in join process
-            driver.pipeInput(recordFactory.create(topic1, null, "SomeVal", 42L));
+            inputTopic1.pipeInput(null, "SomeVal", 42L);
             // left: X0:0 (ts: 5), X1:1 (ts: 6)
             // right:
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("X0+null", null), 5),
@@ -219,10 +231,10 @@ public void testNotSendingOldValue() {
 
             // push two items to the other stream. this should produce two items.
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i], 10L * i));
+                inputTopic2.pipeInput(expectedKeys[i], "Y" + expectedKeys[i], 10L * i);
             }
             // pass tuple with null key, it will be discarded in join process
-            driver.pipeInput(recordFactory.create(topic2, null, "AnotherVal", 73L));
+            inputTopic2.pipeInput(null, "AnotherVal", 73L);
             // left: X0:0 (ts: 5), X1:1 (ts: 6)
             // right: Y0:0 (ts: 0), Y1:1 (ts: 10)
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("X0+Y0", null), 5),
@@ -230,7 +242,7 @@ public void testNotSendingOldValue() {
 
             // push all four items to the primary stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, 7L));
+                inputTopic1.pipeInput(expectedKey, "XX" + expectedKey, 7L);
             }
             // left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
             // right: Y0:0 (ts: 0), Y1:1 (ts: 10)
@@ -241,7 +253,7 @@ public void testNotSendingOldValue() {
 
             // push all items to the other stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, expectedKey * 5L));
+                inputTopic2.pipeInput(expectedKey, "YY" + expectedKey, expectedKey * 5L);
             }
             // left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
             // right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
@@ -252,7 +264,7 @@ public void testNotSendingOldValue() {
 
             // push all four items to the primary stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XXX" + expectedKey, 6L));
+                inputTopic1.pipeInput(expectedKey, "XXX" + expectedKey, 6L);
             }
             // left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
             // right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
@@ -262,8 +274,8 @@ public void testNotSendingOldValue() {
                 new KeyValueTimestamp<>(3, new Change<>("XXX3+YY3", null), 15));
 
             // push two items with null to the other stream as deletes. this should produce two item.
-            driver.pipeInput(recordFactory.create(topic2, expectedKeys[0], null, 5L));
-            driver.pipeInput(recordFactory.create(topic2, expectedKeys[1], null, 7L));
+            inputTopic2.pipeInput(expectedKeys[0], null, 5L);
+            inputTopic2.pipeInput(expectedKeys[1], null, 7L);
             // left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("XXX0+null", null), 6),
@@ -271,7 +283,7 @@ public void testNotSendingOldValue() {
 
             // push all four items to the primary stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XXXX" + expectedKey, 13L));
+                inputTopic1.pipeInput(expectedKey, "XXXX" + expectedKey, 13L);
             }
             // left: XXXX0:0 (ts: 13), XXXX1:1 (ts: 13), XXXX2:2 (ts: 13), XXXX3:3 (ts: 13)
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
@@ -281,10 +293,10 @@ public void testNotSendingOldValue() {
                 new KeyValueTimestamp<>(3, new Change<>("XXXX3+YY3", null), 15));
 
             // push four items to the primary stream with null. this should produce four items.
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[0], null, 0L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[1], null, 42L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[2], null, 5L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[3], null, 20L));
+            inputTopic1.pipeInput(expectedKeys[0], null, 0L);
+            inputTopic1.pipeInput(expectedKeys[1], null, 42L);
+            inputTopic1.pipeInput(expectedKeys[2], null, 5L);
+            inputTopic1.pipeInput(expectedKeys[3], null, 20L);
             // left:
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>(null, null), 0),
@@ -315,6 +327,10 @@ public void testSendingOldValue() {
         final Topology topology = builder.build().addProcessor("proc", supplier, ((KTableImpl) joined).name);
 
         try (final TopologyTestDriver driver = new TopologyTestDriverWrapper(topology, props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic2 =
+                    driver.createInputTopic(topic2, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             final MockProcessor proc = supplier.theCapturedProcessor();
 
             assertTrue(((KTableImpl) table1).sendingOldValueEnabled());
@@ -323,10 +339,10 @@ public void testSendingOldValue() {
 
             // push two items to the primary stream. the other table is empty
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i], 5L + i));
+                inputTopic1.pipeInput(expectedKeys[i], "X" + expectedKeys[i], 5L + i);
             }
             // pass tuple with null key, it will be discarded in join process
-            driver.pipeInput(recordFactory.create(topic1, null, "SomeVal", 42L));
+            inputTopic1.pipeInput(null, "SomeVal", 42L);
             // left: X0:0 (ts: 5), X1:1 (ts: 6)
             // right:
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("X0+null", null), 5),
@@ -334,10 +350,10 @@ public void testSendingOldValue() {
 
             // push two items to the other stream. this should produce two items.
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i], 10L * i));
+                inputTopic2.pipeInput(expectedKeys[i], "Y" + expectedKeys[i], 10L * i);
             }
             // pass tuple with null key, it will be discarded in join process
-            driver.pipeInput(recordFactory.create(topic2, null, "AnotherVal", 73L));
+            inputTopic2.pipeInput(null, "AnotherVal", 73L);
             // left: X0:0 (ts: 5), X1:1 (ts: 6)
             // right: Y0:0 (ts: 0), Y1:1 (ts: 10)
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("X0+Y0", "X0+null"), 5),
@@ -345,7 +361,7 @@ public void testSendingOldValue() {
 
             // push all four items to the primary stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, 7L));
+                inputTopic1.pipeInput(expectedKey, "XX" + expectedKey, 7L);
             }
             // left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
             // right: Y0:0 (ts: 0), Y1:1 (ts: 10)
@@ -356,7 +372,7 @@ public void testSendingOldValue() {
 
             // push all items to the other stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, expectedKey * 5L));
+                inputTopic2.pipeInput(expectedKey, "YY" + expectedKey, expectedKey * 5L);
             }
             // left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
             // right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
@@ -366,7 +382,7 @@ public void testSendingOldValue() {
                 new KeyValueTimestamp<>(3, new Change<>("XX3+YY3", "XX3+null"), 15));
             // push all four items to the primary stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XXX" + expectedKey, 6L));
+                inputTopic1.pipeInput(expectedKey, "XXX" + expectedKey, 6L);
             }
             // left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
             // right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
@@ -376,8 +392,8 @@ public void testSendingOldValue() {
                 new KeyValueTimestamp<>(3, new Change<>("XXX3+YY3", "XX3+YY3"), 15));
 
             // push two items with null to the other stream as deletes. this should produce two item.
-            driver.pipeInput(recordFactory.create(topic2, expectedKeys[0], null, 5L));
-            driver.pipeInput(recordFactory.create(topic2, expectedKeys[1], null, 7L));
+            inputTopic2.pipeInput(expectedKeys[0], null, 5L);
+            inputTopic2.pipeInput(expectedKeys[1], null, 7L);
             // left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("XXX0+null", "XXX0+YY0"), 6),
@@ -385,7 +401,7 @@ public void testSendingOldValue() {
 
             // push all four items to the primary stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XXXX" + expectedKey, 13L));
+                inputTopic1.pipeInput(expectedKey, "XXXX" + expectedKey, 13L);
             }
             // left: XXXX0:0 (ts: 13), XXXX1:1 (ts: 13), XXXX2:2 (ts: 13), XXXX3:3 (ts: 13)
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
@@ -394,10 +410,10 @@ public void testSendingOldValue() {
                 new KeyValueTimestamp<>(2, new Change<>("XXXX2+YY2", "XXX2+YY2"), 13),
                 new KeyValueTimestamp<>(3, new Change<>("XXXX3+YY3", "XXX3+YY3"), 15));
             // push four items to the primary stream with null. this should produce four items.
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[0], null, 0L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[1], null, 42L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[2], null, 5L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[3], null, 20L));
+            inputTopic1.pipeInput(expectedKeys[0], null, 0L);
+            inputTopic1.pipeInput(expectedKeys[1], null, 42L);
+            inputTopic1.pipeInput(expectedKeys[2], null, 5L);
+            inputTopic1.pipeInput(expectedKeys[3], null, 20L);
             // left:
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>(null, "XXXX0+null"), 0),
@@ -473,21 +489,21 @@ public void shouldNotThrowIllegalStateExceptionWhenMultiCacheEvictions() {
             .leftJoin(eight, MockValueJoiner.TOSTRING_JOINER)
             .mapValues(mapper);
 
-        final ConsumerRecordFactory factory = new ConsumerRecordFactory<>(Serdes.Long().serializer(), Serdes.String().serializer());
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-
             final String[] values = {
                 "a", "AA", "BBB", "CCCC", "DD", "EEEEEEEE", "F", "GGGGGGGGGGGGGGG", "HHH", "IIIIIIIIII",
                 "J", "KK", "LLLL", "MMMMMMMMMMMMMMMMMMMMMM", "NNNNN", "O", "P", "QQQQQ", "R", "SSSS",
                 "T", "UU", "VVVVVVVVVVVVVVVVVVV"
             };
 
+            TestInputTopic inputTopic;
             final Random random = new Random();
             for (int i = 0; i < 1000; i++) {
                 for (final String input : inputs) {
                     final Long key = (long) random.nextInt(1000);
                     final String value = values[random.nextInt(values.length)];
-                    driver.pipeInput(factory.create(input, key, value));
+                    inputTopic = driver.createInputTopic(input, Serdes.Long().serializer(), Serdes.String().serializer());
+                    inputTopic.pipeInput(key, value);
                 }
             }
         }
@@ -515,14 +531,10 @@ public void shouldLogAndMeterSkippedRecordsDueToNullLeftKey() {
         assertThat(appender.getMessages(), hasItem("Skipping record due to null key. change=[(new<-old)] topic=[left] partition=[-1] offset=[-2]"));
     }
 
-    private void assertOutputKeyValueTimestamp(final TopologyTestDriver driver,
+    private void assertOutputKeyValueTimestamp(final TestOutputTopic outputTopic,
                                                final Integer expectedKey,
                                                final String expectedValue,
                                                final long expectedTimestamp) {
-        OutputVerifier.compareKeyValueTimestamp(
-            driver.readOutput(output, Serdes.Integer().deserializer(), Serdes.String().deserializer()),
-            expectedKey,
-            expectedValue,
-            expectedTimestamp);
+        assertThat(outputTopic.readRecord(), equalTo(new TestRecord(expectedKey, expectedValue, null, expectedTimestamp)));
     }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableOuterJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableOuterJoinTest.java
index 9568c73d5b08e..0ab8418891a13 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableOuterJoinTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableOuterJoinTest.java
@@ -19,6 +19,7 @@
 import org.apache.kafka.common.serialization.Serdes;
 import org.apache.kafka.streams.KeyValueTimestamp;
 import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.TestOutputTopic;
 import org.apache.kafka.streams.Topology;
 import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.TopologyWrapper;
@@ -27,14 +28,16 @@
 import org.apache.kafka.streams.processor.MockProcessorContext;
 import org.apache.kafka.streams.processor.Processor;
 import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
-import org.apache.kafka.streams.test.OutputVerifier;
+import org.apache.kafka.streams.TestInputTopic;
+import org.apache.kafka.streams.test.TestRecord;
 import org.apache.kafka.test.MockProcessor;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockValueJoiner;
 import org.apache.kafka.test.StreamsTestUtils;
 import org.junit.Test;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
@@ -42,11 +45,11 @@
 import java.util.Set;
 
 import static org.apache.kafka.test.StreamsTestUtils.getMetricByName;
+import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.hasItem;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 public class KTableKTableOuterJoinTest {
@@ -54,8 +57,6 @@ public class KTableKTableOuterJoinTest {
     private final String topic2 = "topic2";
     private final String output = "output";
     private final Consumed consumed = Consumed.with(Serdes.Integer(), Serdes.String());
-    private final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(Serdes.Integer().serializer(), Serdes.String().serializer(), 0L);
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String());
 
     @Test
@@ -80,99 +81,105 @@ public void testJoin() {
         assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next());
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic2 =
+                    driver.createInputTopic(topic2, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestOutputTopic outputTopic =
+                    driver.createOutputTopic(output, Serdes.Integer().deserializer(), Serdes.String().deserializer());
             // push two items to the primary stream. the other table is empty
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i], 5L + i));
+                inputTopic1.pipeInput(expectedKeys[i], "X" + expectedKeys[i], 5L + i);
             }
             // pass tuple with null key, it will be discarded in join process
-            driver.pipeInput(recordFactory.create(topic1, null, "SomeVal", 42L));
+            inputTopic1.pipeInput(null, "SomeVal", 42L);
             // left: X0:0 (ts: 5), X1:1 (ts: 6)
             // right:
-            assertOutputKeyValueTimestamp(driver, 0, "X0+null", 5L);
-            assertOutputKeyValueTimestamp(driver, 1, "X1+null", 6L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 0, "X0+null", 5L);
+            assertOutputKeyValueTimestamp(outputTopic, 1, "X1+null", 6L);
+            assertTrue(outputTopic.isEmpty());
 
             // push two items to the other stream. this should produce two items.
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i], 10L * i));
+                inputTopic2.pipeInput(expectedKeys[i], "Y" + expectedKeys[i], 10L * i);
             }
             // pass tuple with null key, it will be discarded in join process
-            driver.pipeInput(recordFactory.create(topic2, null, "AnotherVal", 73L));
+            inputTopic2.pipeInput(null, "AnotherVal", 73L);
             // left: X0:0 (ts: 5), X1:1 (ts: 6)
             // right: Y0:0 (ts: 0), Y1:1 (ts: 10)
-            assertOutputKeyValueTimestamp(driver, 0, "X0+Y0", 5L);
-            assertOutputKeyValueTimestamp(driver, 1, "X1+Y1", 10L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 0, "X0+Y0", 5L);
+            assertOutputKeyValueTimestamp(outputTopic, 1, "X1+Y1", 10L);
+            assertTrue(outputTopic.isEmpty());
 
             // push all four items to the primary stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, 7L));
+                inputTopic1.pipeInput(expectedKey, "XX" + expectedKey, 7L);
             }
             // left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
             // right: Y0:0 (ts: 0), Y1:1 (ts: 10)
-            assertOutputKeyValueTimestamp(driver, 0, "XX0+Y0", 7L);
-            assertOutputKeyValueTimestamp(driver, 1, "XX1+Y1", 10L);
-            assertOutputKeyValueTimestamp(driver, 2, "XX2+null", 7L);
-            assertOutputKeyValueTimestamp(driver, 3, "XX3+null", 7L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 0, "XX0+Y0", 7L);
+            assertOutputKeyValueTimestamp(outputTopic, 1, "XX1+Y1", 10L);
+            assertOutputKeyValueTimestamp(outputTopic, 2, "XX2+null", 7L);
+            assertOutputKeyValueTimestamp(outputTopic, 3, "XX3+null", 7L);
+            assertTrue(outputTopic.isEmpty());
 
             // push all items to the other stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, expectedKey * 5L));
+                inputTopic2.pipeInput(expectedKey, "YY" + expectedKey, expectedKey * 5L);
             }
             // left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
             // right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
-            assertOutputKeyValueTimestamp(driver, 0, "XX0+YY0", 7L);
-            assertOutputKeyValueTimestamp(driver, 1, "XX1+YY1", 7L);
-            assertOutputKeyValueTimestamp(driver, 2, "XX2+YY2", 10L);
-            assertOutputKeyValueTimestamp(driver, 3, "XX3+YY3", 15L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 0, "XX0+YY0", 7L);
+            assertOutputKeyValueTimestamp(outputTopic, 1, "XX1+YY1", 7L);
+            assertOutputKeyValueTimestamp(outputTopic, 2, "XX2+YY2", 10L);
+            assertOutputKeyValueTimestamp(outputTopic, 3, "XX3+YY3", 15L);
+            assertTrue(outputTopic.isEmpty());
 
             // push all four items to the primary stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XXX" + expectedKey, 6L));
+                inputTopic1.pipeInput(expectedKey, "XXX" + expectedKey, 6L);
             }
             // left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
             // right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
-            assertOutputKeyValueTimestamp(driver, 0, "XXX0+YY0", 6L);
-            assertOutputKeyValueTimestamp(driver, 1, "XXX1+YY1", 6L);
-            assertOutputKeyValueTimestamp(driver, 2, "XXX2+YY2", 10L);
-            assertOutputKeyValueTimestamp(driver, 3, "XXX3+YY3", 15L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 0, "XXX0+YY0", 6L);
+            assertOutputKeyValueTimestamp(outputTopic, 1, "XXX1+YY1", 6L);
+            assertOutputKeyValueTimestamp(outputTopic, 2, "XXX2+YY2", 10L);
+            assertOutputKeyValueTimestamp(outputTopic, 3, "XXX3+YY3", 15L);
+            assertTrue(outputTopic.isEmpty());
 
             // push two items with null to the other stream as deletes. this should produce two item.
-            driver.pipeInput(recordFactory.create(topic2, expectedKeys[0], null, 5L));
-            driver.pipeInput(recordFactory.create(topic2, expectedKeys[1], null, 7L));
+            inputTopic2.pipeInput(expectedKeys[0], null, 5L);
+            inputTopic2.pipeInput(expectedKeys[1], null, 7L);
             // left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
-            assertOutputKeyValueTimestamp(driver, 0, "XXX0+null", 6L);
-            assertOutputKeyValueTimestamp(driver, 1, "XXX1+null", 7L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 0, "XXX0+null", 6L);
+            assertOutputKeyValueTimestamp(outputTopic, 1, "XXX1+null", 7L);
+            assertTrue(outputTopic.isEmpty());
 
             // push all four items to the primary stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XXXX" + expectedKey, 13L));
+                inputTopic1.pipeInput(expectedKey, "XXXX" + expectedKey, 13L);
             }
             // left: XXXX0:0 (ts: 13), XXXX1:1 (ts: 13), XXXX2:2 (ts: 13), XXXX3:3 (ts: 13)
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
-            assertOutputKeyValueTimestamp(driver, 0, "XXXX0+null", 13L);
-            assertOutputKeyValueTimestamp(driver, 1, "XXXX1+null", 13L);
-            assertOutputKeyValueTimestamp(driver, 2, "XXXX2+YY2", 13L);
-            assertOutputKeyValueTimestamp(driver, 3, "XXXX3+YY3", 15L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 0, "XXXX0+null", 13L);
+            assertOutputKeyValueTimestamp(outputTopic, 1, "XXXX1+null", 13L);
+            assertOutputKeyValueTimestamp(outputTopic, 2, "XXXX2+YY2", 13L);
+            assertOutputKeyValueTimestamp(outputTopic, 3, "XXXX3+YY3", 15L);
+            assertTrue(outputTopic.isEmpty());
 
             // push four items to the primary stream with null. this should produce four items.
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[0], null, 0L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[1], null, 42L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[2], null, 5L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[3], null, 20L));
+            inputTopic1.pipeInput(expectedKeys[0], null, 0L);
+            inputTopic1.pipeInput(expectedKeys[1], null, 42L);
+            inputTopic1.pipeInput(expectedKeys[2], null, 5L);
+            inputTopic1.pipeInput(expectedKeys[3], null, 20L);
             // left:
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
-            assertOutputKeyValueTimestamp(driver, 0, null, 0L);
-            assertOutputKeyValueTimestamp(driver, 1, null, 42L);
-            assertOutputKeyValueTimestamp(driver, 2, "null+YY2", 10L);
-            assertOutputKeyValueTimestamp(driver, 3, "null+YY3", 20L);
-            assertNull(driver.readOutput(output));
+            assertOutputKeyValueTimestamp(outputTopic, 0, null, 0L);
+            assertOutputKeyValueTimestamp(outputTopic, 1, null, 42L);
+            assertOutputKeyValueTimestamp(outputTopic, 2, "null+YY2", 10L);
+            assertOutputKeyValueTimestamp(outputTopic, 3, "null+YY3", 20L);
+            assertTrue(outputTopic.isEmpty());
         }
     }
 
@@ -195,6 +202,10 @@ public void testNotSendingOldValue() {
         final Topology topology = builder.build().addProcessor("proc", supplier, ((KTableImpl) joined).name);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic2 =
+                    driver.createInputTopic(topic2, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             final MockProcessor proc = supplier.theCapturedProcessor();
 
             assertTrue(((KTableImpl) table1).sendingOldValueEnabled());
@@ -203,27 +214,27 @@ public void testNotSendingOldValue() {
 
             // push two items to the primary stream. the other table is empty
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i], 5L + i));
+                inputTopic1.pipeInput(expectedKeys[i], "X" + expectedKeys[i], 5L + i);
             }
             // pass tuple with null key, it will be discarded in join process
-            driver.pipeInput(recordFactory.create(topic1, null, "SomeVal", 42L));
+            inputTopic1.pipeInput(null, "SomeVal", 42L);
             // left: X0:0 (ts: 5), X1:1 (ts: 6)
             // right:
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("X0+null", null), 5),
                     new KeyValueTimestamp<>(1, new Change<>("X1+null", null), 6));
             // push two items to the other stream. this should produce two items.
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i], 10L * i));
+                inputTopic2.pipeInput(expectedKeys[i], "Y" + expectedKeys[i], 10L * i);
             }
             // pass tuple with null key, it will be discarded in join process
-            driver.pipeInput(recordFactory.create(topic2, null, "AnotherVal", 73L));
+            inputTopic2.pipeInput(null, "AnotherVal", 73L);
             // left: X0:0 (ts: 5), X1:1 (ts: 6)
             // right: Y0:0 (ts: 0), Y1:1 (ts: 10)
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("X0+Y0", null), 5),
                     new KeyValueTimestamp<>(1, new Change<>("X1+Y1", null), 10));
             // push all four items to the primary stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, 7L));
+                inputTopic1.pipeInput(expectedKey, "XX" + expectedKey, 7L);
             }
             // left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
             // right: Y0:0 (ts: 0), Y1:1 (ts: 10)
@@ -233,7 +244,7 @@ public void testNotSendingOldValue() {
                     new KeyValueTimestamp<>(3, new Change<>("XX3+null", null), 7));
             // push all items to the other stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, expectedKey * 5L));
+                inputTopic2.pipeInput(expectedKey, "YY" + expectedKey, expectedKey * 5L);
             }
             // left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
             // right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
@@ -243,7 +254,7 @@ public void testNotSendingOldValue() {
                     new KeyValueTimestamp<>(3, new Change<>("XX3+YY3", null), 15));
             // push all four items to the primary stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XXX" + expectedKey, 6L));
+                inputTopic1.pipeInput(expectedKey, "XXX" + expectedKey, 6L);
             }
             // left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
             // right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
@@ -252,15 +263,15 @@ public void testNotSendingOldValue() {
                     new KeyValueTimestamp<>(2, new Change<>("XXX2+YY2", null), 10),
                     new KeyValueTimestamp<>(3, new Change<>("XXX3+YY3", null), 15));
             // push two items with null to the other stream as deletes. this should produce two item.
-            driver.pipeInput(recordFactory.create(topic2, expectedKeys[0], null, 5L));
-            driver.pipeInput(recordFactory.create(topic2, expectedKeys[1], null, 7L));
+            inputTopic2.pipeInput(expectedKeys[0], null, 5L);
+            inputTopic2.pipeInput(expectedKeys[1], null, 7L);
             // left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("XXX0+null", null), 6),
                     new KeyValueTimestamp<>(1, new Change<>("XXX1+null", null), 7));
             // push all four items to the primary stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XXXX" + expectedKey, 13L));
+                inputTopic1.pipeInput(expectedKey, "XXXX" + expectedKey, 13L);
             }
             // left: XXXX0:0 (ts: 13), XXXX1:1 (ts: 13), XXXX2:2 (ts: 13), XXXX3:3 (ts: 13)
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
@@ -269,10 +280,10 @@ public void testNotSendingOldValue() {
                     new KeyValueTimestamp<>(2, new Change<>("XXXX2+YY2", null), 13),
                     new KeyValueTimestamp<>(3, new Change<>("XXXX3+YY3", null), 15));
             // push four items to the primary stream with null. this should produce four items.
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[0], null, 0L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[1], null, 42L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[2], null, 5L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[3], null, 20L));
+            inputTopic1.pipeInput(expectedKeys[0], null, 0L);
+            inputTopic1.pipeInput(expectedKeys[1], null, 42L);
+            inputTopic1.pipeInput(expectedKeys[2], null, 5L);
+            inputTopic1.pipeInput(expectedKeys[3], null, 20L);
             // left:
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>(null, null), 0),
@@ -303,6 +314,10 @@ public void testSendingOldValue() {
         final Topology topology = builder.build().addProcessor("proc", supplier, ((KTableImpl) joined).name);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            final TestInputTopic inputTopic2 =
+                    driver.createInputTopic(topic2, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             final MockProcessor proc = supplier.theCapturedProcessor();
 
             assertTrue(((KTableImpl) table1).sendingOldValueEnabled());
@@ -311,27 +326,27 @@ public void testSendingOldValue() {
 
             // push two items to the primary stream. the other table is empty
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i], 5L + i));
+                inputTopic1.pipeInput(expectedKeys[i], "X" + expectedKeys[i], 5L + i);
             }
             // pass tuple with null key, it will be discarded in join process
-            driver.pipeInput(recordFactory.create(topic1, null, "SomeVal", 42L));
+            inputTopic1.pipeInput(null, "SomeVal", 42L);
             // left: X0:0 (ts: 5), X1:1 (ts: 6)
             // right:
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("X0+null", null), 5),
                     new KeyValueTimestamp<>(1, new Change<>("X1+null", null), 6));
             // push two items to the other stream. this should produce two items.
             for (int i = 0; i < 2; i++) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i], 10L * i));
+                inputTopic2.pipeInput(expectedKeys[i], "Y" + expectedKeys[i], 10L * i);
             }
             // pass tuple with null key, it will be discarded in join process
-            driver.pipeInput(recordFactory.create(topic2, null, "AnotherVal", 73L));
+            inputTopic2.pipeInput(null, "AnotherVal", 73L);
             // left: X0:0 (ts: 5), X1:1 (ts: 6)
             // right: Y0:0 (ts: 0), Y1:1 (ts: 10)
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("X0+Y0", "X0+null"), 5),
                     new KeyValueTimestamp<>(1, new Change<>("X1+Y1", "X1+null"), 10));
             // push all four items to the primary stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, 7L));
+                inputTopic1.pipeInput(expectedKey, "XX" + expectedKey, 7L);
             }
             // left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
             // right: Y0:0 (ts: 0), Y1:1 (ts: 10)
@@ -341,7 +356,7 @@ public void testSendingOldValue() {
                     new KeyValueTimestamp<>(3, new Change<>("XX3+null", null), 7));
             // push all items to the other stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, expectedKey * 5L));
+                inputTopic2.pipeInput(expectedKey, "YY" + expectedKey, expectedKey * 5L);
             }
             // left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
             // right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
@@ -351,7 +366,7 @@ public void testSendingOldValue() {
                     new KeyValueTimestamp<>(3, new Change<>("XX3+YY3", "XX3+null"), 15));
             // push all four items to the primary stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XXX" + expectedKey, 6L));
+                inputTopic1.pipeInput(expectedKey, "XXX" + expectedKey, 6L);
             }
             // left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
             // right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
@@ -360,15 +375,15 @@ public void testSendingOldValue() {
                     new KeyValueTimestamp<>(2, new Change<>("XXX2+YY2", "XX2+YY2"), 10),
                     new KeyValueTimestamp<>(3, new Change<>("XXX3+YY3", "XX3+YY3"), 15));
             // push two items with null to the other stream as deletes. this should produce two item.
-            driver.pipeInput(recordFactory.create(topic2, expectedKeys[0], null, 5L));
-            driver.pipeInput(recordFactory.create(topic2, expectedKeys[1], null, 7L));
+            inputTopic2.pipeInput(expectedKeys[0], null, 5L);
+            inputTopic2.pipeInput(expectedKeys[1], null, 7L);
             // left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("XXX0+null", "XXX0+YY0"), 6),
                     new KeyValueTimestamp<>(1, new Change<>("XXX1+null", "XXX1+YY1"), 7));
             // push all four items to the primary stream. this should produce four items.
             for (final int expectedKey : expectedKeys) {
-                driver.pipeInput(recordFactory.create(topic1, expectedKey, "XXXX" + expectedKey, 13L));
+                inputTopic1.pipeInput(expectedKey, "XXXX" + expectedKey, 13L);
             }
             // left: XXXX0:0 (ts: 13), XXXX1:1 (ts: 13), XXXX2:2 (ts: 13), XXXX3:3 (ts: 13)
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
@@ -377,10 +392,10 @@ public void testSendingOldValue() {
                     new KeyValueTimestamp<>(2, new Change<>("XXXX2+YY2", "XXX2+YY2"), 13),
                     new KeyValueTimestamp<>(3, new Change<>("XXXX3+YY3", "XXX3+YY3"), 15));
             // push four items to the primary stream with null. this should produce four items.
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[0], null, 0L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[1], null, 42L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[2], null, 5L));
-            driver.pipeInput(recordFactory.create(topic1, expectedKeys[3], null, 20L));
+            inputTopic1.pipeInput(expectedKeys[0], null, 0L);
+            inputTopic1.pipeInput(expectedKeys[1], null, 42L);
+            inputTopic1.pipeInput(expectedKeys[2], null, 5L);
+            inputTopic1.pipeInput(expectedKeys[3], null, 20L);
             // left:
             // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>(null, "XXXX0+null"), 0),
@@ -412,15 +427,11 @@ public void shouldLogAndMeterSkippedRecordsDueToNullLeftKey() {
         assertThat(appender.getMessages(), hasItem("Skipping record due to null key. change=[(new<-old)] topic=[left] partition=[-1] offset=[-2]"));
     }
 
-    private void assertOutputKeyValueTimestamp(final TopologyTestDriver driver,
+    private void assertOutputKeyValueTimestamp(final TestOutputTopic outputTopic,
                                                final Integer expectedKey,
                                                final String expectedValue,
                                                final long expectedTimestamp) {
-        OutputVerifier.compareKeyValueTimestamp(
-                driver.readOutput(output, Serdes.Integer().deserializer(), Serdes.String().deserializer()),
-                expectedKey,
-                expectedValue,
-                expectedTimestamp);
+        assertThat(outputTopic.readRecord(), equalTo(new TestRecord(expectedKey, expectedValue, null, expectedTimestamp)));
     }
 
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableMapKeysTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableMapKeysTest.java
index b0d81f1de4d03..ac74119553515 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableMapKeysTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableMapKeysTest.java
@@ -25,7 +25,7 @@
 import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KTable;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.StreamsTestUtils;
 import org.junit.Test;
@@ -37,8 +37,6 @@
 import static org.junit.Assert.assertEquals;
 
 public class KTableMapKeysTest {
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer(), 0L);
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String());
 
     @Test
@@ -66,7 +64,9 @@ public void testMapKeysConvertingToStream() {
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
             for (int i = 0; i < originalKeys.length; i++) {
-                driver.pipeInput(recordFactory.create(topic1, originalKeys[i], values[i], 5 + i * 5));
+                final TestInputTopic inputTopic =
+                        driver.createInputTopic(topic1, new IntegerSerializer(), new StringSerializer());
+                inputTopic.pipeInput(originalKeys[i], values[i], 5 + i * 5);
             }
         }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableMapValuesTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableMapValuesTest.java
index a00318e20e81a..7df93ec148c08 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableMapValuesTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableMapValuesTest.java
@@ -31,12 +31,14 @@
 import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder;
 import org.apache.kafka.streams.state.KeyValueStore;
 import org.apache.kafka.streams.state.ValueAndTimestamp;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockProcessor;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.StreamsTestUtils;
 import org.junit.Test;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.Properties;
 
 import static java.util.Arrays.asList;
@@ -48,18 +50,18 @@
 @SuppressWarnings("unchecked")
 public class KTableMapValuesTest {
     private final Consumed consumed = Consumed.with(Serdes.String(), Serdes.String());
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L);
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String());
 
     private void doTestKTable(final StreamsBuilder builder,
                               final String topic1,
                               final MockProcessorSupplier supplier) {
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(topic1, "A", "1", 5L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "2", 25L));
-            driver.pipeInput(recordFactory.create(topic1, "C", "3", 20L));
-            driver.pipeInput(recordFactory.create(topic1, "D", "4", 10L));
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            inputTopic1.pipeInput("A", "1", 5L);
+            inputTopic1.pipeInput("B", "2", 25L);
+            inputTopic1.pipeInput("C", "3", 20L);
+            inputTopic1.pipeInput("D", "4", 10L);
             assertEquals(asList(new KeyValueTimestamp<>("A", 1, 5),
                     new KeyValueTimestamp<>("B", 2, 25),
                     new KeyValueTimestamp<>("C", 3, 20),
@@ -114,15 +116,17 @@ private void doTestValueGetter(final StreamsBuilder builder,
         topologyBuilder.connectProcessorAndStateStores(table3.name, getterSupplier3.storeNames());
 
         try (final TopologyTestDriverWrapper driver = new TopologyTestDriverWrapper(builder.build(), props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             final KTableValueGetter getter2 = getterSupplier2.get();
             final KTableValueGetter getter3 = getterSupplier3.get();
 
             getter2.init(driver.setCurrentNodeForProcessorContext(table2.name));
             getter3.init(driver.setCurrentNodeForProcessorContext(table3.name));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", "01", 50L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "01", 10L));
-            driver.pipeInput(recordFactory.create(topic1, "C", "01", 30L));
+            inputTopic1.pipeInput("A", "01", 50L);
+            inputTopic1.pipeInput("B", "01", 10L);
+            inputTopic1.pipeInput("C", "01", 30L);
 
             assertEquals(ValueAndTimestamp.make(1, 50L), getter2.get("A"));
             assertEquals(ValueAndTimestamp.make(1, 10L), getter2.get("B"));
@@ -132,8 +136,8 @@ private void doTestValueGetter(final StreamsBuilder builder,
             assertEquals(ValueAndTimestamp.make(-1, 10L), getter3.get("B"));
             assertEquals(ValueAndTimestamp.make(-1, 30L), getter3.get("C"));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", "02", 25L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "02", 20L));
+            inputTopic1.pipeInput("A", "02", 25L);
+            inputTopic1.pipeInput("B", "02", 20L);
 
             assertEquals(ValueAndTimestamp.make(2, 25L), getter2.get("A"));
             assertEquals(ValueAndTimestamp.make(2, 20L), getter2.get("B"));
@@ -143,7 +147,7 @@ private void doTestValueGetter(final StreamsBuilder builder,
             assertEquals(ValueAndTimestamp.make(-2, 20L), getter3.get("B"));
             assertEquals(ValueAndTimestamp.make(-1, 30L), getter3.get("C"));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", "03", 35L));
+            inputTopic1.pipeInput("A", "03", 35L);
 
             assertEquals(ValueAndTimestamp.make(3, 35L), getter2.get("A"));
             assertEquals(ValueAndTimestamp.make(2, 20L), getter2.get("B"));
@@ -153,7 +157,7 @@ private void doTestValueGetter(final StreamsBuilder builder,
             assertEquals(ValueAndTimestamp.make(-2, 20L), getter3.get("B"));
             assertEquals(ValueAndTimestamp.make(-1, 30L), getter3.get("C"));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", (String) null, 1L));
+            inputTopic1.pipeInput("A", (String) null, 1L);
 
             assertNull(getter2.get("A"));
             assertEquals(ValueAndTimestamp.make(2, 20L), getter2.get("B"));
@@ -208,27 +212,29 @@ public void testNotSendingOldValue() {
         final Topology topology = builder.build().addProcessor("proc", supplier, table2.name);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             final MockProcessor proc = supplier.theCapturedProcessor();
 
             assertFalse(table1.sendingOldValueEnabled());
             assertFalse(table2.sendingOldValueEnabled());
 
-            driver.pipeInput(recordFactory.create(topic1, "A", "01", 5L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "01", 10L));
-            driver.pipeInput(recordFactory.create(topic1, "C", "01", 15L));
+            inputTopic1.pipeInput("A", "01", 5L);
+            inputTopic1.pipeInput("B", "01", 10L);
+            inputTopic1.pipeInput("C", "01", 15L);
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(1, null), 5),
                     new KeyValueTimestamp<>("B", new Change<>(1, null), 10),
                     new KeyValueTimestamp<>("C", new Change<>(1, null), 15));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", "02", 10L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "02", 8L));
+            inputTopic1.pipeInput("A", "02", 10L);
+            inputTopic1.pipeInput("B", "02", 8L);
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(2, null), 10),
                     new KeyValueTimestamp<>("B", new Change<>(2, null), 8));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", "03", 20L));
+            inputTopic1.pipeInput("A", "03", 20L);
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(3, null), 20));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", (String) null, 30L));
+            inputTopic1.pipeInput("A", (String) null, 30L);
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(null, null), 30));
         }
     }
@@ -248,27 +254,29 @@ public void testSendingOldValue() {
         builder.build().addProcessor("proc", supplier, table2.name);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             final MockProcessor proc = supplier.theCapturedProcessor();
 
             assertTrue(table1.sendingOldValueEnabled());
             assertTrue(table2.sendingOldValueEnabled());
 
-            driver.pipeInput(recordFactory.create(topic1, "A", "01", 5L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "01", 10L));
-            driver.pipeInput(recordFactory.create(topic1, "C", "01", 15L));
+            inputTopic1.pipeInput("A", "01", 5L);
+            inputTopic1.pipeInput("B", "01", 10L);
+            inputTopic1.pipeInput("C", "01", 15L);
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(1, null), 5),
                     new KeyValueTimestamp<>("B", new Change<>(1, null), 10),
                     new KeyValueTimestamp<>("C", new Change<>(1, null), 15));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", "02", 10L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "02", 8L));
+            inputTopic1.pipeInput("A", "02", 10L);
+            inputTopic1.pipeInput("B", "02", 8L);
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(2, 1), 10),
                     new KeyValueTimestamp<>("B", new Change<>(2, 1), 8));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", "03", 20L));
+            inputTopic1.pipeInput("A", "03", 20L);
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(3, 2), 20));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", (String) null, 30L));
+            inputTopic1.pipeInput("A", (String) null, 30L);
             proc.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(null, 3), 30));
         }
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableSourceTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableSourceTest.java
index a936dc084282a..1611efe865ec9 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableSourceTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableSourceTest.java
@@ -31,12 +31,14 @@
 import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder;
 import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender;
 import org.apache.kafka.streams.state.ValueAndTimestamp;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockProcessor;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.StreamsTestUtils;
 import org.junit.Test;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.Properties;
 
 import static java.util.Arrays.asList;
@@ -49,8 +51,6 @@
 
 public class KTableSourceTest {
     private final Consumed stringConsumed = Consumed.with(Serdes.String(), Serdes.String());
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L);
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String());
 
     @Test
@@ -63,15 +63,15 @@ public void testKTable() {
         final MockProcessorSupplier supplier = new MockProcessorSupplier<>();
         table1.toStream().process(supplier);
 
-        final ConsumerRecordFactory integerFactory =
-            new ConsumerRecordFactory<>(new StringSerializer(), new IntegerSerializer(), 0L);
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(integerFactory.create(topic1, "A", 1, 10L));
-            driver.pipeInput(integerFactory.create(topic1, "B", 2, 11L));
-            driver.pipeInput(integerFactory.create(topic1, "C", 3, 12L));
-            driver.pipeInput(integerFactory.create(topic1, "D", 4, 13L));
-            driver.pipeInput(integerFactory.create(topic1, "A", null, 14L));
-            driver.pipeInput(integerFactory.create(topic1, "B", null, 15L));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(topic1, new StringSerializer(), new IntegerSerializer());
+            inputTopic.pipeInput("A", 1, 10L);
+            inputTopic.pipeInput("B", 2, 11L);
+            inputTopic.pipeInput("C", 3, 12L);
+            inputTopic.pipeInput("D", 4, 13L);
+            inputTopic.pipeInput("A", null, 14L);
+            inputTopic.pipeInput("B", null, 15L);
         }
 
         assertEquals(
@@ -92,7 +92,9 @@ public void kTableShouldLogAndMeterOnSkippedRecords() {
 
         final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
-            driver.pipeInput(recordFactory.create(topic, null, "value"));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic(topic, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
+            inputTopic.pipeInput(null, "value");
             LogCaptureAppender.unregister(appender);
 
             assertEquals(1.0, getMetricByName(driver.metrics(), "skipped-records-total", "stream-metrics").metricValue());
@@ -116,32 +118,34 @@ public void testValueGetter() {
         topologyBuilder.connectProcessorAndStateStores(table1.name, getterSupplier1.storeNames());
 
         try (final TopologyTestDriverWrapper driver = new TopologyTestDriverWrapper(builder.build(), props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             final KTableValueGetter getter1 = getterSupplier1.get();
             getter1.init(driver.setCurrentNodeForProcessorContext(table1.name));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", "01", 10L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "01", 20L));
-            driver.pipeInput(recordFactory.create(topic1, "C", "01", 15L));
+            inputTopic1.pipeInput("A", "01", 10L);
+            inputTopic1.pipeInput("B", "01", 20L);
+            inputTopic1.pipeInput("C", "01", 15L);
 
             assertEquals(ValueAndTimestamp.make("01", 10L), getter1.get("A"));
             assertEquals(ValueAndTimestamp.make("01", 20L), getter1.get("B"));
             assertEquals(ValueAndTimestamp.make("01", 15L), getter1.get("C"));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", "02", 30L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "02", 5L));
+            inputTopic1.pipeInput("A", "02", 30L);
+            inputTopic1.pipeInput("B", "02", 5L);
 
             assertEquals(ValueAndTimestamp.make("02", 30L), getter1.get("A"));
             assertEquals(ValueAndTimestamp.make("02", 5L), getter1.get("B"));
             assertEquals(ValueAndTimestamp.make("01", 15L), getter1.get("C"));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", "03", 29L));
+            inputTopic1.pipeInput("A", "03", 29L);
 
             assertEquals(ValueAndTimestamp.make("03", 29L), getter1.get("A"));
             assertEquals(ValueAndTimestamp.make("02", 5L), getter1.get("B"));
             assertEquals(ValueAndTimestamp.make("01", 15L), getter1.get("C"));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", (String) null, 50L));
-            driver.pipeInput(recordFactory.create(topic1, "B", (String) null, 3L));
+            inputTopic1.pipeInput("A", (String) null, 50L);
+            inputTopic1.pipeInput("B", (String) null, 3L);
 
             assertNull(getter1.get("A"));
             assertNull(getter1.get("B"));
@@ -161,25 +165,27 @@ public void testNotSendingOldValue() {
         final Topology topology = builder.build().addProcessor("proc1", supplier, table1.name);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             final MockProcessor proc1 = supplier.theCapturedProcessor();
 
-            driver.pipeInput(recordFactory.create(topic1, "A", "01", 10L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "01", 20L));
-            driver.pipeInput(recordFactory.create(topic1, "C", "01", 15L));
+            inputTopic1.pipeInput("A", "01", 10L);
+            inputTopic1.pipeInput("B", "01", 20L);
+            inputTopic1.pipeInput("C", "01", 15L);
             proc1.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>("01", null), 10),
                 new KeyValueTimestamp<>("B", new Change<>("01", null), 20),
                 new KeyValueTimestamp<>("C", new Change<>("01", null), 15));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", "02", 8L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "02", 22L));
+            inputTopic1.pipeInput("A", "02", 8L);
+            inputTopic1.pipeInput("B", "02", 22L);
             proc1.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>("02", null), 8),
                 new KeyValueTimestamp<>("B", new Change<>("02", null), 22));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", "03", 12L));
+            inputTopic1.pipeInput("A", "03", 12L);
             proc1.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>("03", null), 12));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", (String) null, 15L));
-            driver.pipeInput(recordFactory.create(topic1, "B", (String) null, 20L));
+            inputTopic1.pipeInput("A", (String) null, 15L);
+            inputTopic1.pipeInput("B", (String) null, 20L);
             proc1.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(null, null), 15),
                 new KeyValueTimestamp<>("B", new Change<>(null, null), 20));
         }
@@ -199,25 +205,27 @@ public void testSendingOldValue() {
         final Topology topology = builder.build().addProcessor("proc1", supplier, table1.name);
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) {
+            final TestInputTopic inputTopic1 =
+                    driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
             final MockProcessor proc1 = supplier.theCapturedProcessor();
 
-            driver.pipeInput(recordFactory.create(topic1, "A", "01", 10L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "01", 20L));
-            driver.pipeInput(recordFactory.create(topic1, "C", "01", 15L));
+            inputTopic1.pipeInput("A", "01", 10L);
+            inputTopic1.pipeInput("B", "01", 20L);
+            inputTopic1.pipeInput("C", "01", 15L);
             proc1.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>("01", null), 10),
                 new KeyValueTimestamp<>("B", new Change<>("01", null), 20),
                 new KeyValueTimestamp<>("C", new Change<>("01", null), 15));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", "02", 8L));
-            driver.pipeInput(recordFactory.create(topic1, "B", "02", 22L));
+            inputTopic1.pipeInput("A", "02", 8L);
+            inputTopic1.pipeInput("B", "02", 22L);
             proc1.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>("02", "01"), 8),
                 new KeyValueTimestamp<>("B", new Change<>("02", "01"), 22));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", "03", 12L));
+            inputTopic1.pipeInput("A", "03", 12L);
             proc1.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>("03", "02"), 12));
 
-            driver.pipeInput(recordFactory.create(topic1, "A", (String) null, 15L));
-            driver.pipeInput(recordFactory.create(topic1, "B", (String) null, 20L));
+            inputTopic1.pipeInput("A", (String) null, 15L);
+            inputTopic1.pipeInput("B", (String) null, 20L);
             proc1.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(null, "03"), 15),
                 new KeyValueTimestamp<>("B", new Change<>(null, "02"), 20));
         }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableTransformValuesTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableTransformValuesTest.java
index ab091390c0327..80cdae4776b57 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableTransformValuesTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableTransformValuesTest.java
@@ -40,7 +40,7 @@
 import org.apache.kafka.streams.state.Stores;
 import org.apache.kafka.streams.state.TimestampedKeyValueStore;
 import org.apache.kafka.streams.state.ValueAndTimestamp;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockReducer;
 import org.apache.kafka.test.SingletonNoOpValueTransformer;
@@ -78,9 +78,6 @@ public class KTableTransformValuesTest {
 
     private static final Consumed CONSUMED = Consumed.with(Serdes.String(), Serdes.String());
 
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer(), 0L);
-
     private TopologyTestDriver driver;
     private MockProcessorSupplier capture;
     private StreamsBuilder builder;
@@ -327,10 +324,12 @@ public void shouldTransformValuesWithKey() {
             .process(capture);
 
         driver = new TopologyTestDriver(builder.build(), props());
+        final TestInputTopic inputTopic =
+                driver.createInputTopic(INPUT_TOPIC, new StringSerializer(), new StringSerializer());
 
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "a", 5L));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC, "B", "b", 10L));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC, "D", (String) null, 15L));
+        inputTopic.pipeInput("A", "a", 5L);
+        inputTopic.pipeInput("B", "b", 10L);
+        inputTopic.pipeInput("D", (String) null, 15L);
 
 
         assertThat(output(), hasItems(new KeyValueTimestamp<>("A", "A->a!", 5),
@@ -355,10 +354,11 @@ public void shouldTransformValuesWithKeyAndMaterialize() {
             .process(capture);
 
         driver = new TopologyTestDriver(builder.build(), props());
-
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "a", 5L));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC, "B", "b", 10L));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC, "C", (String) null, 15L));
+        final TestInputTopic inputTopic =
+                driver.createInputTopic(INPUT_TOPIC, new StringSerializer(), new StringSerializer());
+        inputTopic.pipeInput("A", "a", 5L);
+        inputTopic.pipeInput("B", "b", 10L);
+        inputTopic.pipeInput("C", (String) null, 15L);
 
         assertThat(output(), hasItems(new KeyValueTimestamp<>("A", "A->a!", 5),
                 new KeyValueTimestamp<>("B", "B->b!", 10),
@@ -394,10 +394,12 @@ public void shouldCalculateCorrectOldValuesIfMaterializedEvenIfStateful() {
             .process(capture);
 
         driver = new TopologyTestDriver(builder.build(), props());
+        final TestInputTopic inputTopic =
+                driver.createInputTopic(INPUT_TOPIC, new StringSerializer(), new StringSerializer());
 
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "ignored", 5L));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "ignored", 15L));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "ignored", 10L));
+        inputTopic.pipeInput("A", "ignored", 5L);
+        inputTopic.pipeInput("A", "ignored", 15L);
+        inputTopic.pipeInput("A", "ignored", 10L);
 
         assertThat(output(), hasItems(new KeyValueTimestamp<>("A", "1", 5),
                 new KeyValueTimestamp<>("A", "0", 15),
@@ -421,10 +423,12 @@ public void shouldCalculateCorrectOldValuesIfNotStatefulEvenIfNotMaterialized()
             .process(capture);
 
         driver = new TopologyTestDriver(builder.build(), props());
+        final TestInputTopic inputTopic =
+                driver.createInputTopic(INPUT_TOPIC, new StringSerializer(), new StringSerializer());
 
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "a", 5L));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "aa", 15L));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC, "A", "aaa", 10));
+        inputTopic.pipeInput("A", "a", 5L);
+        inputTopic.pipeInput("A", "aa", 15L);
+        inputTopic.pipeInput("A", "aaa", 10);
 
         assertThat(output(), hasItems(new KeyValueTimestamp<>("A", "1", 5),
                  new KeyValueTimestamp<>("A", "0", 15),
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImplTest.java
index b693dc7640fb5..e47619327dd17 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImplTest.java
@@ -35,7 +35,7 @@
 import org.apache.kafka.streams.kstream.Windowed;
 import org.apache.kafka.streams.state.SessionStore;
 import org.apache.kafka.streams.state.ValueAndTimestamp;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockAggregator;
 import org.apache.kafka.test.MockInitializer;
 import org.apache.kafka.test.MockProcessorSupplier;
@@ -56,8 +56,6 @@
 public class SessionWindowedKStreamImplTest {
     private static final String TOPIC = "input";
     private final StreamsBuilder builder = new StreamsBuilder();
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String());
     private final Merger sessionMerger = (aggKey, aggOne, aggTwo) -> aggOne + "+" + aggTwo;
     private SessionWindowedKStream stream;
@@ -295,10 +293,12 @@ public void shouldThrowNullPointerOnCountIfMaterializedIsNull() {
     }
 
     private void processData(final TopologyTestDriver driver) {
-        driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 10));
-        driver.pipeInput(recordFactory.create(TOPIC, "1", "2", 15));
-        driver.pipeInput(recordFactory.create(TOPIC, "1", "3", 600));
-        driver.pipeInput(recordFactory.create(TOPIC, "2", "1", 600));
-        driver.pipeInput(recordFactory.create(TOPIC, "2", "2", 599));
+        final TestInputTopic inputTopic =
+                driver.createInputTopic(TOPIC, new StringSerializer(), new StringSerializer());
+        inputTopic.pipeInput("1", "1", 10);
+        inputTopic.pipeInput("1", "2", 15);
+        inputTopic.pipeInput("1", "3", 600);
+        inputTopic.pipeInput("2", "1", 600);
+        inputTopic.pipeInput("2", "2", 599);
     }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SuppressScenarioTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SuppressScenarioTest.java
index 3c7fd1e2c2e64..55f8670a87bb8 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SuppressScenarioTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SuppressScenarioTest.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
-import org.apache.kafka.clients.producer.ProducerRecord;
 import org.apache.kafka.common.serialization.Deserializer;
 import org.apache.kafka.common.serialization.LongDeserializer;
 import org.apache.kafka.common.serialization.Serde;
@@ -42,14 +41,12 @@
 import org.apache.kafka.streams.state.KeyValueStore;
 import org.apache.kafka.streams.state.SessionStore;
 import org.apache.kafka.streams.state.WindowStore;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
-import org.apache.kafka.streams.test.OutputVerifier;
+import org.apache.kafka.streams.TestInputTopic;
+import org.apache.kafka.streams.test.TestRecord;
 import org.apache.kafka.test.TestUtils;
 import org.junit.Test;
 
-import java.util.ArrayList;
 import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Properties;
@@ -64,6 +61,8 @@
 import static org.apache.kafka.streams.kstream.Suppressed.BufferConfig.unbounded;
 import static org.apache.kafka.streams.kstream.Suppressed.untilTimeLimit;
 import static org.apache.kafka.streams.kstream.Suppressed.untilWindowCloses;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.MatcherAssert.assertThat;
 
 public class SuppressScenarioTest {
     private static final StringDeserializer STRING_DESERIALIZER = new StringDeserializer();
@@ -102,14 +101,12 @@ public void shouldImmediatelyEmitEventsWithZeroEmitAfter() {
 
         final Topology topology = builder.build();
 
-
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(STRING_SERIALIZER, STRING_SERIALIZER);
-
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, config)) {
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 0L));
-            driver.pipeInput(recordFactory.create("input", "k1", "v2", 1L));
-            driver.pipeInput(recordFactory.create("input", "k2", "v1", 2L));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic("input", STRING_SERIALIZER, STRING_SERIALIZER);
+            inputTopic.pipeInput("k1", "v1", 0L);
+            inputTopic.pipeInput("k1", "v2", 1L);
+            inputTopic.pipeInput("k2", "v1", 2L);
             verify(
                 drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER),
                 asList(
@@ -128,7 +125,7 @@ public void shouldImmediatelyEmitEventsWithZeroEmitAfter() {
                     new KeyValueTimestamp<>("v1", 1L, 2L)
                 )
             );
-            driver.pipeInput(recordFactory.create("input", "x", "x", 3L));
+            inputTopic.pipeInput("x", "x", 3L);
             verify(
                 drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER),
                 singletonList(
@@ -141,7 +138,7 @@ public void shouldImmediatelyEmitEventsWithZeroEmitAfter() {
                     new KeyValueTimestamp<>("x", 1L, 3L)
                 )
             );
-            driver.pipeInput(recordFactory.create("input", "x", "x", 4L));
+            inputTopic.pipeInput("x", "x", 4L);
             verify(
                 drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER),
                 asList(
@@ -180,12 +177,12 @@ public void shouldSuppressIntermediateEventsWithTimeLimit() {
             .toStream()
             .to("output-raw", Produced.with(STRING_SERDE, Serdes.Long()));
         final Topology topology = builder.build();
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(STRING_SERIALIZER, STRING_SERIALIZER);
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, config)) {
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 0L));
-            driver.pipeInput(recordFactory.create("input", "k1", "v2", 1L));
-            driver.pipeInput(recordFactory.create("input", "k2", "v1", 2L));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic("input", STRING_SERIALIZER, STRING_SERIALIZER);
+            inputTopic.pipeInput("k1", "v1", 0L);
+            inputTopic.pipeInput("k1", "v2", 1L);
+            inputTopic.pipeInput("k2", "v1", 2L);
             verify(
                 drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER),
                 asList(
@@ -200,7 +197,7 @@ public void shouldSuppressIntermediateEventsWithTimeLimit() {
                 singletonList(new KeyValueTimestamp<>("v1", 1L, 2L))
             );
             // inserting a dummy "tick" record just to advance stream time
-            driver.pipeInput(recordFactory.create("input", "tick", "tick", 3L));
+            inputTopic.pipeInput("tick", "tick", 3L);
             verify(
                 drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER),
                 singletonList(new KeyValueTimestamp<>("tick", 1L, 3L))
@@ -212,7 +209,7 @@ public void shouldSuppressIntermediateEventsWithTimeLimit() {
             );
 
 
-            driver.pipeInput(recordFactory.create("input", "tick", "tick", 4L));
+            inputTopic.pipeInput("tick", "tick", 4L);
             verify(
                 drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER),
                 asList(
@@ -250,12 +247,12 @@ public void shouldSuppressIntermediateEventsWithRecordLimit() {
             .to("output-raw", Produced.with(STRING_SERDE, Serdes.Long()));
         final Topology topology = builder.build();
         System.out.println(topology.describe());
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(STRING_SERIALIZER, STRING_SERIALIZER);
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, config)) {
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 0L));
-            driver.pipeInput(recordFactory.create("input", "k1", "v2", 1L));
-            driver.pipeInput(recordFactory.create("input", "k2", "v1", 2L));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic("input", STRING_SERIALIZER, STRING_SERIALIZER);
+            inputTopic.pipeInput("k1", "v1", 0L);
+            inputTopic.pipeInput("k1", "v2", 1L);
+            inputTopic.pipeInput("k2", "v1", 2L);
             verify(
                 drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER),
                 asList(
@@ -274,7 +271,7 @@ public void shouldSuppressIntermediateEventsWithRecordLimit() {
                     // the last update won't be evicted until another key comes along.
                 )
             );
-            driver.pipeInput(recordFactory.create("input", "x", "x", 3L));
+            inputTopic.pipeInput("x", "x", 3L);
             verify(
                 drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER),
                 singletonList(
@@ -314,12 +311,12 @@ public void shouldSuppressIntermediateEventsWithBytesLimit() {
             .to("output-raw", Produced.with(STRING_SERDE, Serdes.Long()));
         final Topology topology = builder.build();
         System.out.println(topology.describe());
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(STRING_SERIALIZER, STRING_SERIALIZER);
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, config)) {
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 0L));
-            driver.pipeInput(recordFactory.create("input", "k1", "v2", 1L));
-            driver.pipeInput(recordFactory.create("input", "k2", "v1", 2L));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic("input", STRING_SERIALIZER, STRING_SERIALIZER);
+            inputTopic.pipeInput("k1", "v1", 0L);
+            inputTopic.pipeInput("k1", "v2", 1L);
+            inputTopic.pipeInput("k2", "v1", 2L);
             verify(
                 drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER),
                 asList(
@@ -338,7 +335,7 @@ public void shouldSuppressIntermediateEventsWithBytesLimit() {
                     // the last update won't be evicted until another key comes along.
                 )
             );
-            driver.pipeInput(recordFactory.create("input", "x", "x", 3L));
+            inputTopic.pipeInput("x", "x", 3L);
             verify(
                 drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER),
                 singletonList(
@@ -374,17 +371,17 @@ public void shouldSupportFinalResultsForTimeWindows() {
             .to("output-raw", Produced.with(STRING_SERDE, Serdes.Long()));
         final Topology topology = builder.build();
         System.out.println(topology.describe());
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(STRING_SERIALIZER, STRING_SERIALIZER);
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, config)) {
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 0L));
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 1L));
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 2L));
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 1L));
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 0L));
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 5L));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic("input", STRING_SERIALIZER, STRING_SERIALIZER);
+            inputTopic.pipeInput("k1", "v1", 0L);
+            inputTopic.pipeInput("k1", "v1", 1L);
+            inputTopic.pipeInput("k1", "v1", 2L);
+            inputTopic.pipeInput("k1", "v1", 1L);
+            inputTopic.pipeInput("k1", "v1", 0L);
+            inputTopic.pipeInput("k1", "v1", 5L);
             // note this last record gets dropped because it is out of the grace period
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 0L));
+            inputTopic.pipeInput("k1", "v1", 0L);
             verify(
                 drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER),
                 asList(
@@ -425,19 +422,19 @@ public void shouldSupportFinalResultsForTimeWindowsWithLargeJump() {
             .to("output-raw", Produced.with(STRING_SERDE, Serdes.Long()));
         final Topology topology = builder.build();
         System.out.println(topology.describe());
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(STRING_SERIALIZER, STRING_SERIALIZER);
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, config)) {
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 0L));
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 1L));
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 2L));
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 0L));
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 3L));
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 0L));
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 4L));
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic("input", STRING_SERIALIZER, STRING_SERIALIZER);
+            inputTopic.pipeInput("k1", "v1", 0L);
+            inputTopic.pipeInput("k1", "v1", 1L);
+            inputTopic.pipeInput("k1", "v1", 2L);
+            inputTopic.pipeInput("k1", "v1", 0L);
+            inputTopic.pipeInput("k1", "v1", 3L);
+            inputTopic.pipeInput("k1", "v1", 0L);
+            inputTopic.pipeInput("k1", "v1", 4L);
             // this update should get dropped, since the previous event advanced the stream time and closed the window.
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 0L));
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 30L));
+            inputTopic.pipeInput("k1", "v1", 0L);
+            inputTopic.pipeInput("k1", "v1", 30L);
             verify(
                 drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER),
                 asList(
@@ -481,20 +478,20 @@ public void shouldSupportFinalResultsForSessionWindows() {
             .to("output-raw", Produced.with(STRING_SERDE, Serdes.Long()));
         final Topology topology = builder.build();
         System.out.println(topology.describe());
-        final ConsumerRecordFactory recordFactory =
-            new ConsumerRecordFactory<>(STRING_SERIALIZER, STRING_SERIALIZER);
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, config)) {
+            final TestInputTopic inputTopic =
+                    driver.createInputTopic("input", STRING_SERIALIZER, STRING_SERIALIZER);
             // first window
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 0L));
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 5L));
+            inputTopic.pipeInput("k1", "v1", 0L);
+            inputTopic.pipeInput("k1", "v1", 5L);
             // arbitrarily disordered records are admitted, because the *window* is not closed until stream-time > window-end + grace
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 1L));
+            inputTopic.pipeInput("k1", "v1", 1L);
             // any record in the same partition advances stream time (note the key is different)
-            driver.pipeInput(recordFactory.create("input", "k2", "v1", 6L));
+            inputTopic.pipeInput("k2", "v1", 6L);
             // late event for first window - this should get dropped from all streams, since the first window is now closed.
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 5L));
+            inputTopic.pipeInput("k1", "v1", 5L);
             // just pushing stream time forward to flush the other events through.
-            driver.pipeInput(recordFactory.create("input", "k1", "v1", 30L));
+            inputTopic.pipeInput("k1", "v1", 30L);
             verify(
                 drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER),
                 asList(
@@ -530,11 +527,11 @@ public void shouldWorkBeforeGroupBy() {
             .to("output", Produced.with(Serdes.String(), Serdes.Long()));
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), config)) {
-            final ConsumerRecordFactory recordFactory =
-                new ConsumerRecordFactory<>(STRING_SERIALIZER, STRING_SERIALIZER);
+            final TestInputTopic inputTopic =
+                driver.createInputTopic("topic", STRING_SERIALIZER, STRING_SERIALIZER);
 
-            driver.pipeInput(recordFactory.create("topic", "A", "a", 0L));
-            driver.pipeInput(recordFactory.create("topic", "tick", "tick", 10L));
+            inputTopic.pipeInput("A", "a", 0L);
+            inputTopic.pipeInput("tick", "tick", 10L);
 
             verify(
                 drainProducerRecords(driver, "output", STRING_DESERIALIZER, LONG_DESERIALIZER),
@@ -560,11 +557,13 @@ public void shouldWorkBeforeJoinRight() {
             .to("output", Produced.with(Serdes.String(), Serdes.String()));
 
         try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), config)) {
-            final ConsumerRecordFactory recordFactory =
-                new ConsumerRecordFactory<>(STRING_SERIALIZER, STRING_SERIALIZER);
+            final TestInputTopic inputTopicRight =
+                driver.createInputTopic("right", STRING_SERIALIZER, STRING_SERIALIZER);
+            final TestInputTopic inputTopicLeft =
+                    driver.createInputTopic("left", STRING_SERIALIZER, STRING_SERIALIZER);
 
-            driver.pipeInput(recordFactory.create("right", "B", "1", 0L));
-            driver.pipeInput(recordFactory.create("right", "A", "1", 0L));
+            inputTopicRight.pipeInput("B", "1", 0L);
+            inputTopicRight.pipeInput("A", "1", 0L);
             // buffered, no output
             verify(
                 drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER),
@@ -572,7 +571,7 @@ public void shouldWorkBeforeJoinRight() {
             );
 
 
-            driver.pipeInput(recordFactory.create("right", "tick", "tick", 10L));
+            inputTopicRight.pipeInput("tick", "tick", 10L);
             // flush buffer
             verify(
                 drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER),
@@ -583,7 +582,7 @@ public void shouldWorkBeforeJoinRight() {
             );
 
 
-            driver.pipeInput(recordFactory.create("right", "A", "2", 11L));
+            inputTopicRight.pipeInput("A", "2", 11L);
             // buffered, no output
             verify(
                 drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER),
@@ -591,7 +590,7 @@ public void shouldWorkBeforeJoinRight() {
             );
 
 
-            driver.pipeInput(recordFactory.create("left", "A", "a", 12L));
+            inputTopicLeft.pipeInput("A", "a", 12L);
             // should join with previously emitted right side
             verify(
                 drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER),
@@ -599,7 +598,7 @@ public void shouldWorkBeforeJoinRight() {
             );
 
 
-            driver.pipeInput(recordFactory.create("left", "B", "b", 12L));
+            inputTopicLeft.pipeInput("B", "b", 12L);
             // should view through to the parent KTable, since B is no longer buffered
             verify(
                 drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER),
@@ -607,7 +606,7 @@ public void shouldWorkBeforeJoinRight() {
             );
 
 
-            driver.pipeInput(recordFactory.create("left", "A", "b", 13L));
+            inputTopicLeft.pipeInput("A", "b", 13L);
             // should join with previously emitted right side
             verify(
                 drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER),
@@ -615,7 +614,7 @@ public void shouldWorkBeforeJoinRight() {
             );
 
 
-            driver.pipeInput(recordFactory.create("right", "tick", "tick", 21L));
+            inputTopicRight.pipeInput("tick", "tick", 21L);
             verify(
                 drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER),
                 asList(
@@ -647,11 +646,13 @@ public void shouldWorkBeforeJoinLeft() {
         final Topology topology = builder.build();
         System.out.println(topology.describe());
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, config)) {
-            final ConsumerRecordFactory recordFactory =
-                new ConsumerRecordFactory<>(STRING_SERIALIZER, STRING_SERIALIZER);
+            final TestInputTopic inputTopicRight =
+                driver.createInputTopic("right", STRING_SERIALIZER, STRING_SERIALIZER);
+            final TestInputTopic inputTopicLeft =
+                    driver.createInputTopic("left", STRING_SERIALIZER, STRING_SERIALIZER);
 
-            driver.pipeInput(recordFactory.create("left", "B", "1", 0L));
-            driver.pipeInput(recordFactory.create("left", "A", "1", 0L));
+            inputTopicLeft.pipeInput("B", "1", 0L);
+            inputTopicLeft.pipeInput("A", "1", 0L);
             // buffered, no output
             verify(
                 drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER),
@@ -659,7 +660,7 @@ public void shouldWorkBeforeJoinLeft() {
             );
 
 
-            driver.pipeInput(recordFactory.create("left", "tick", "tick", 10L));
+            inputTopicLeft.pipeInput("tick", "tick", 10L);
             // flush buffer
             verify(
                 drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER),
@@ -670,7 +671,7 @@ public void shouldWorkBeforeJoinLeft() {
             );
 
 
-            driver.pipeInput(recordFactory.create("left", "A", "2", 11L));
+            inputTopicLeft.pipeInput("A", "2", 11L);
             // buffered, no output
             verify(
                 drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER),
@@ -678,7 +679,7 @@ public void shouldWorkBeforeJoinLeft() {
             );
 
 
-            driver.pipeInput(recordFactory.create("right", "A", "a", 12L));
+            inputTopicRight.pipeInput("A", "a", 12L);
             // should join with previously emitted left side
             verify(
                 drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER),
@@ -686,7 +687,7 @@ public void shouldWorkBeforeJoinLeft() {
             );
 
 
-            driver.pipeInput(recordFactory.create("right", "B", "b", 12L));
+            inputTopicRight.pipeInput("B", "b", 12L);
             // should view through to the parent KTable, since B is no longer buffered
             verify(
                 drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER),
@@ -694,7 +695,7 @@ public void shouldWorkBeforeJoinLeft() {
             );
 
 
-            driver.pipeInput(recordFactory.create("right", "A", "b", 13L));
+            inputTopicRight.pipeInput("A", "b", 13L);
             // should join with previously emitted left side
             verify(
                 drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER),
@@ -702,7 +703,7 @@ public void shouldWorkBeforeJoinLeft() {
             );
 
 
-            driver.pipeInput(recordFactory.create("left", "tick", "tick", 21L));
+            inputTopicLeft.pipeInput("tick", "tick", 21L);
             verify(
                 drainProducerRecords(driver, "output", STRING_DESERIALIZER, STRING_DESERIALIZER),
                 asList(
@@ -715,39 +716,33 @@ public void shouldWorkBeforeJoinLeft() {
     }
 
 
-    private static  void verify(final List> results,
+    private static  void verify(final List> results,
                                       final List> expectedResults) {
         if (results.size() != expectedResults.size()) {
             throw new AssertionError(printRecords(results) + " != " + expectedResults);
         }
         final Iterator> expectedIterator = expectedResults.iterator();
-        for (final ProducerRecord result : results) {
+        for (final TestRecord result : results) {
             final KeyValueTimestamp expected = expectedIterator.next();
             try {
-                OutputVerifier.compareKeyValueTimestamp(result, expected.key(), expected.value(), expected.timestamp());
+                assertThat(result, equalTo(new TestRecord(expected.key(), expected.value(), null, expected.timestamp())));
             } catch (final AssertionError e) {
                 throw new AssertionError(printRecords(results) + " != " + expectedResults, e);
             }
         }
     }
 
-    private static  List> drainProducerRecords(final TopologyTestDriver driver,
-                                                                          final String topic,
-                                                                          final Deserializer keyDeserializer,
-                                                                          final Deserializer valueDeserializer) {
-        final List> result = new LinkedList<>();
-        for (ProducerRecord next = driver.readOutput(topic, keyDeserializer, valueDeserializer);
-             next != null;
-             next = driver.readOutput(topic, keyDeserializer, valueDeserializer)) {
-            result.add(next);
-        }
-        return new ArrayList<>(result);
+    private static  List> drainProducerRecords(final TopologyTestDriver driver,
+                                                                      final String topic,
+                                                                      final Deserializer keyDeserializer,
+                                                                      final Deserializer valueDeserializer) {
+        return driver.createOutputTopic(topic, keyDeserializer, valueDeserializer).readRecordsToList();
     }
 
-    private static  String printRecords(final List> result) {
+    private static  String printRecords(final List> result) {
         final StringBuilder resultStr = new StringBuilder();
         resultStr.append("[\n");
-        for (final ProducerRecord record : result) {
+        for (final TestRecord record : result) {
             resultStr.append("  ").append(record).append("\n");
         }
         resultStr.append("]");
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImplTest.java
index 0327ebd20cadb..63d263a7f439d 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImplTest.java
@@ -33,7 +33,7 @@
 import org.apache.kafka.streams.kstream.Windowed;
 import org.apache.kafka.streams.state.ValueAndTimestamp;
 import org.apache.kafka.streams.state.WindowStore;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.MockAggregator;
 import org.apache.kafka.test.MockInitializer;
 import org.apache.kafka.test.MockProcessorSupplier;
@@ -54,8 +54,6 @@
 public class TimeWindowedKStreamImplTest {
     private static final String TOPIC = "input";
     private final StreamsBuilder builder = new StreamsBuilder();
-    private final ConsumerRecordFactory recordFactory =
-        new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
     private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.String(), Serdes.String());
     private TimeWindowedKStream windowedStream;
 
@@ -314,11 +312,13 @@ public void shouldThrowNullPointerOnCountIfMaterializedIsNull() {
     }
 
     private void processData(final TopologyTestDriver driver) {
-        driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 10L));
-        driver.pipeInput(recordFactory.create(TOPIC, "1", "2", 15L));
-        driver.pipeInput(recordFactory.create(TOPIC, "1", "3", 500L));
-        driver.pipeInput(recordFactory.create(TOPIC, "2", "10", 550L));
-        driver.pipeInput(recordFactory.create(TOPIC, "2", "20", 500L));
+        final TestInputTopic inputTopic =
+                driver.createInputTopic(TOPIC, new StringSerializer(), new StringSerializer());
+        inputTopic.pipeInput("1", "1", 10L);
+        inputTopic.pipeInput("1", "2", 15L);
+        inputTopic.pipeInput("1", "3", 500L);
+        inputTopic.pipeInput("2", "10", 550L);
+        inputTopic.pipeInput("2", "20", 500L);
     }
 
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java
index 2669039ed61d7..5e3f9b527d56e 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java
@@ -17,7 +17,6 @@
 package org.apache.kafka.streams.processor.internals;
 
 import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.clients.producer.ProducerRecord;
 import org.apache.kafka.common.header.Header;
 import org.apache.kafka.common.header.Headers;
 import org.apache.kafka.common.header.internals.RecordHeader;
@@ -28,6 +27,8 @@
 import org.apache.kafka.common.serialization.StringDeserializer;
 import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TestInputTopic;
+import org.apache.kafka.streams.TestOutputTopic;
 import org.apache.kafka.streams.Topology;
 import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.TopologyWrapper;
@@ -42,7 +43,7 @@
 import org.apache.kafka.streams.state.KeyValueStore;
 import org.apache.kafka.streams.state.StoreBuilder;
 import org.apache.kafka.streams.state.Stores;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.test.TestRecord;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.TestUtils;
 import org.junit.After;
@@ -50,6 +51,8 @@
 import org.junit.Test;
 
 import java.io.File;
+import java.time.Duration;
+import java.time.Instant;
 import java.util.Properties;
 
 import static org.hamcrest.CoreMatchers.containsString;
@@ -77,7 +80,6 @@ public class ProcessorTopologyTest {
 
     private final TopologyWrapper topology = new TopologyWrapper();
     private final MockProcessorSupplier mockProcessorSupplier = new MockProcessorSupplier();
-    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(STRING_SERIALIZER, STRING_SERIALIZER, 0L);
 
     private TopologyTestDriver driver;
     private final Properties props = new Properties();
@@ -134,77 +136,95 @@ public void testTopologyMetadata() {
     public void testDrivingSimpleTopology() {
         final int partition = 10;
         driver = new TopologyTestDriver(createSimpleTopology(partition), props);
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1"));
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1", partition);
-        assertNoOutputRecord(OUTPUT_TOPIC_2);
+        final TestInputTopic inputTopic = driver.createInputTopic(INPUT_TOPIC_1, STRING_SERIALIZER, STRING_SERIALIZER, Instant.ofEpochMilli(0L), Duration.ZERO);
+        final TestOutputTopic outputTopic1 =
+                driver.createOutputTopic(OUTPUT_TOPIC_1, Serdes.String().deserializer(), Serdes.String().deserializer());
 
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2"));
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key2", "value2", partition);
-        assertNoOutputRecord(OUTPUT_TOPIC_2);
+        inputTopic.pipeInput("key1", "value1");
+        assertNextOutputRecord(outputTopic1.readRecord(), "key1", "value1");
+        assertTrue(outputTopic1.isEmpty());
 
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key3", "value3"));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key4", "value4"));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key5", "value5"));
-        assertNoOutputRecord(OUTPUT_TOPIC_2);
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key3", "value3", partition);
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key4", "value4", partition);
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key5", "value5", partition);
+        inputTopic.pipeInput("key2", "value2");
+        assertNextOutputRecord(outputTopic1.readRecord(), "key2", "value2");
+        assertTrue(outputTopic1.isEmpty());
+
+        inputTopic.pipeInput("key3", "value3");
+        inputTopic.pipeInput("key4", "value4");
+        inputTopic.pipeInput("key5", "value5");
+        assertNextOutputRecord(outputTopic1.readRecord(), "key3", "value3");
+        assertNextOutputRecord(outputTopic1.readRecord(), "key4", "value4");
+        assertNextOutputRecord(outputTopic1.readRecord(), "key5", "value5");
+        assertTrue(outputTopic1.isEmpty());
     }
 
 
     @Test
     public void testDrivingMultiplexingTopology() {
         driver = new TopologyTestDriver(createMultiplexingTopology(), props);
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1"));
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1(1)");
-        assertNextOutputRecord(OUTPUT_TOPIC_2, "key1", "value1(2)");
-
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2"));
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key2", "value2(1)");
-        assertNextOutputRecord(OUTPUT_TOPIC_2, "key2", "value2(2)");
-
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key3", "value3"));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key4", "value4"));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key5", "value5"));
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key3", "value3(1)");
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key4", "value4(1)");
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key5", "value5(1)");
-        assertNextOutputRecord(OUTPUT_TOPIC_2, "key3", "value3(2)");
-        assertNextOutputRecord(OUTPUT_TOPIC_2, "key4", "value4(2)");
-        assertNextOutputRecord(OUTPUT_TOPIC_2, "key5", "value5(2)");
+        final TestInputTopic inputTopic = driver.createInputTopic(INPUT_TOPIC_1, STRING_SERIALIZER, STRING_SERIALIZER, Instant.ofEpochMilli(0L), Duration.ZERO);
+        final TestOutputTopic outputTopic1 =
+                driver.createOutputTopic(OUTPUT_TOPIC_1, Serdes.String().deserializer(), Serdes.String().deserializer());
+        final TestOutputTopic outputTopic2 =
+                driver.createOutputTopic(OUTPUT_TOPIC_2, Serdes.String().deserializer(), Serdes.String().deserializer());
+        inputTopic.pipeInput("key1", "value1");
+        assertNextOutputRecord(outputTopic1.readRecord(), "key1", "value1(1)");
+        assertNextOutputRecord(outputTopic2.readRecord(), "key1", "value1(2)");
+
+        inputTopic.pipeInput("key2", "value2");
+        assertNextOutputRecord(outputTopic1.readRecord(), "key2", "value2(1)");
+        assertNextOutputRecord(outputTopic2.readRecord(), "key2", "value2(2)");
+
+        inputTopic.pipeInput("key3", "value3");
+        inputTopic.pipeInput("key4", "value4");
+        inputTopic.pipeInput("key5", "value5");
+        assertNextOutputRecord(outputTopic1.readRecord(), "key3", "value3(1)");
+        assertNextOutputRecord(outputTopic1.readRecord(), "key4", "value4(1)");
+        assertNextOutputRecord(outputTopic1.readRecord(), "key5", "value5(1)");
+        assertNextOutputRecord(outputTopic2.readRecord(), "key3", "value3(2)");
+        assertNextOutputRecord(outputTopic2.readRecord(), "key4", "value4(2)");
+        assertNextOutputRecord(outputTopic2.readRecord(), "key5", "value5(2)");
     }
 
     @Test
     public void testDrivingMultiplexByNameTopology() {
         driver = new TopologyTestDriver(createMultiplexByNameTopology(), props);
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1"));
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1(1)");
-        assertNextOutputRecord(OUTPUT_TOPIC_2, "key1", "value1(2)");
-
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2"));
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key2", "value2(1)");
-        assertNextOutputRecord(OUTPUT_TOPIC_2, "key2", "value2(2)");
-
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key3", "value3"));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key4", "value4"));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key5", "value5"));
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key3", "value3(1)");
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key4", "value4(1)");
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key5", "value5(1)");
-        assertNextOutputRecord(OUTPUT_TOPIC_2, "key3", "value3(2)");
-        assertNextOutputRecord(OUTPUT_TOPIC_2, "key4", "value4(2)");
-        assertNextOutputRecord(OUTPUT_TOPIC_2, "key5", "value5(2)");
+        final TestInputTopic inputTopic = driver.createInputTopic(INPUT_TOPIC_1, STRING_SERIALIZER, STRING_SERIALIZER, Instant.ofEpochMilli(0L), Duration.ZERO);
+        inputTopic.pipeInput("key1", "value1");
+        final TestOutputTopic outputTopic1 =
+                driver.createOutputTopic(OUTPUT_TOPIC_1, Serdes.String().deserializer(), Serdes.String().deserializer());
+        final TestOutputTopic outputTopic2 =
+                driver.createOutputTopic(OUTPUT_TOPIC_2, Serdes.String().deserializer(), Serdes.String().deserializer());
+        assertNextOutputRecord(outputTopic1.readRecord(), "key1", "value1(1)");
+        assertNextOutputRecord(outputTopic2.readRecord(), "key1", "value1(2)");
+
+        inputTopic.pipeInput("key2", "value2");
+        assertNextOutputRecord(outputTopic1.readRecord(), "key2", "value2(1)");
+        assertNextOutputRecord(outputTopic2.readRecord(), "key2", "value2(2)");
+
+        inputTopic.pipeInput("key3", "value3");
+        inputTopic.pipeInput("key4", "value4");
+        inputTopic.pipeInput("key5", "value5");
+        assertNextOutputRecord(outputTopic1.readRecord(), "key3", "value3(1)");
+        assertNextOutputRecord(outputTopic1.readRecord(), "key4", "value4(1)");
+        assertNextOutputRecord(outputTopic1.readRecord(), "key5", "value5(1)");
+        assertNextOutputRecord(outputTopic2.readRecord(), "key3", "value3(2)");
+        assertNextOutputRecord(outputTopic2.readRecord(), "key4", "value4(2)");
+        assertNextOutputRecord(outputTopic2.readRecord(), "key5", "value5(2)");
     }
 
     @Test
     public void testDrivingStatefulTopology() {
         final String storeName = "entries";
         driver = new TopologyTestDriver(createStatefulTopology(storeName), props);
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1"));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2"));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key3", "value3"));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value4"));
-        assertNoOutputRecord(OUTPUT_TOPIC_1);
+        final TestInputTopic inputTopic = driver.createInputTopic(INPUT_TOPIC_1, STRING_SERIALIZER, STRING_SERIALIZER);
+        final TestOutputTopic outputTopic1 =
+                driver.createOutputTopic(OUTPUT_TOPIC_1, Serdes.Integer().deserializer(), Serdes.String().deserializer());
+
+        inputTopic.pipeInput("key1", "value1");
+        inputTopic.pipeInput("key2", "value2");
+        inputTopic.pipeInput("key3", "value3");
+        inputTopic.pipeInput("key1", "value4");
+        assertTrue(outputTopic1.isEmpty());
 
         final KeyValueStore store = driver.getKeyValueStore(storeName);
         assertEquals("value4", store.get("key1"));
@@ -223,9 +243,10 @@ public void shouldDriveGlobalStore() {
                 global, STRING_DESERIALIZER, STRING_DESERIALIZER, topic, "processor", define(new StatefulProcessor(storeName)));
 
         driver = new TopologyTestDriver(topology, props);
+        final TestInputTopic inputTopic = driver.createInputTopic(topic, STRING_SERIALIZER, STRING_SERIALIZER);
         final KeyValueStore globalStore = driver.getKeyValueStore(storeName);
-        driver.pipeInput(recordFactory.create(topic, "key1", "value1"));
-        driver.pipeInput(recordFactory.create(topic, "key2", "value2"));
+        inputTopic.pipeInput("key1", "value1");
+        inputTopic.pipeInput("key2", "value2");
         assertEquals("value1", globalStore.get("key1"));
         assertEquals("value2", globalStore.get("key2"));
     }
@@ -234,50 +255,63 @@ public void shouldDriveGlobalStore() {
     public void testDrivingSimpleMultiSourceTopology() {
         final int partition = 10;
         driver = new TopologyTestDriver(createSimpleMultiSourceTopology(partition), props);
+        final TestInputTopic inputTopic = driver.createInputTopic(INPUT_TOPIC_1, STRING_SERIALIZER, STRING_SERIALIZER, Instant.ofEpochMilli(0L), Duration.ZERO);
+        final TestOutputTopic outputTopic1 =
+                driver.createOutputTopic(OUTPUT_TOPIC_1, Serdes.String().deserializer(), Serdes.String().deserializer());
+        final TestOutputTopic outputTopic2 =
+                driver.createOutputTopic(OUTPUT_TOPIC_2, Serdes.String().deserializer(), Serdes.String().deserializer());
 
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1"));
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1", partition);
-        assertNoOutputRecord(OUTPUT_TOPIC_2);
+        inputTopic.pipeInput("key1", "value1");
+        assertNextOutputRecord(outputTopic1.readRecord(), "key1", "value1");
+        assertTrue(outputTopic2.isEmpty());
 
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_2, "key2", "value2"));
-        assertNextOutputRecord(OUTPUT_TOPIC_2, "key2", "value2", partition);
-        assertNoOutputRecord(OUTPUT_TOPIC_1);
+        final TestInputTopic inputTopic2 = driver.createInputTopic(INPUT_TOPIC_2, STRING_SERIALIZER, STRING_SERIALIZER, Instant.ofEpochMilli(0L), Duration.ZERO);
+        inputTopic2.pipeInput("key2", "value2");
+        assertNextOutputRecord(outputTopic2.readRecord(), "key2", "value2");
+        assertTrue(outputTopic2.isEmpty());
     }
 
     @Test
     public void testDrivingForwardToSourceTopology() {
         driver = new TopologyTestDriver(createForwardToSourceTopology(), props);
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1"));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2"));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key3", "value3"));
-        assertNextOutputRecord(OUTPUT_TOPIC_2, "key1", "value1");
-        assertNextOutputRecord(OUTPUT_TOPIC_2, "key2", "value2");
-        assertNextOutputRecord(OUTPUT_TOPIC_2, "key3", "value3");
+        final TestInputTopic inputTopic = driver.createInputTopic(INPUT_TOPIC_1, STRING_SERIALIZER, STRING_SERIALIZER, Instant.ofEpochMilli(0L), Duration.ZERO);
+        inputTopic.pipeInput("key1", "value1");
+        inputTopic.pipeInput("key2", "value2");
+        inputTopic.pipeInput("key3", "value3");
+        final TestOutputTopic outputTopic2 =
+                driver.createOutputTopic(OUTPUT_TOPIC_2, Serdes.String().deserializer(), Serdes.String().deserializer());
+        assertNextOutputRecord(outputTopic2.readRecord(), "key1", "value1");
+        assertNextOutputRecord(outputTopic2.readRecord(), "key2", "value2");
+        assertNextOutputRecord(outputTopic2.readRecord(), "key3", "value3");
     }
 
     @Test
     public void testDrivingInternalRepartitioningTopology() {
         driver = new TopologyTestDriver(createInternalRepartitioningTopology(), props);
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1"));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2"));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key3", "value3"));
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1");
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key2", "value2");
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key3", "value3");
+        final TestInputTopic inputTopic = driver.createInputTopic(INPUT_TOPIC_1, STRING_SERIALIZER, STRING_SERIALIZER, Instant.ofEpochMilli(0L), Duration.ZERO);
+        inputTopic.pipeInput("key1", "value1");
+        inputTopic.pipeInput("key2", "value2");
+        inputTopic.pipeInput("key3", "value3");
+        final TestOutputTopic outputTopic1 = driver.createOutputTopic(OUTPUT_TOPIC_1, STRING_DESERIALIZER, STRING_DESERIALIZER);
+        assertNextOutputRecord(outputTopic1.readRecord(), "key1", "value1");
+        assertNextOutputRecord(outputTopic1.readRecord(), "key2", "value2");
+        assertNextOutputRecord(outputTopic1.readRecord(), "key3", "value3");
     }
 
     @Test
     public void testDrivingInternalRepartitioningForwardingTimestampTopology() {
         driver = new TopologyTestDriver(createInternalRepartitioningWithValueTimestampTopology(), props);
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1@1000"));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2@2000"));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key3", "value3@3000"));
-        assertThat(driver.readOutput(OUTPUT_TOPIC_1, STRING_DESERIALIZER, STRING_DESERIALIZER),
-                equalTo(new ProducerRecord<>(OUTPUT_TOPIC_1, null, 1000L, "key1", "value1")));
-        assertThat(driver.readOutput(OUTPUT_TOPIC_1, STRING_DESERIALIZER, STRING_DESERIALIZER),
-                equalTo(new ProducerRecord<>(OUTPUT_TOPIC_1, null, 2000L, "key2", "value2")));
-        assertThat(driver.readOutput(OUTPUT_TOPIC_1, STRING_DESERIALIZER, STRING_DESERIALIZER),
-                equalTo(new ProducerRecord<>(OUTPUT_TOPIC_1, null, 3000L, "key3", "value3")));
+        final TestInputTopic inputTopic = driver.createInputTopic(INPUT_TOPIC_1, STRING_SERIALIZER, STRING_SERIALIZER);
+        inputTopic.pipeInput("key1", "value1@1000");
+        inputTopic.pipeInput("key2", "value2@2000");
+        inputTopic.pipeInput("key3", "value3@3000");
+        final TestOutputTopic outputTopic = driver.createOutputTopic(OUTPUT_TOPIC_1, STRING_DESERIALIZER, STRING_DESERIALIZER);
+        assertThat(outputTopic.readRecord(),
+                equalTo(new TestRecord<>("key1", "value1", null, 1000L)));
+        assertThat(outputTopic.readRecord(),
+                equalTo(new TestRecord<>("key2", "value2", null, 2000L)));
+        assertThat(outputTopic.readRecord(),
+                equalTo(new TestRecord<>("key3", "value3", null, 3000L)));
     }
 
     @Test
@@ -328,73 +362,88 @@ public void shouldRecursivelyPrintChildren() {
     public void shouldConsiderTimeStamps() {
         final int partition = 10;
         driver = new TopologyTestDriver(createSimpleTopology(partition), props);
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1", 10L));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2", 20L));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key3", "value3", 30L));
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1", partition, 10L);
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key2", "value2", partition, 20L);
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key3", "value3", partition, 30L);
+        final TestInputTopic inputTopic = driver.createInputTopic(INPUT_TOPIC_1, STRING_SERIALIZER, STRING_SERIALIZER);
+        inputTopic.pipeInput("key1", "value1", 10L);
+        inputTopic.pipeInput("key2", "value2", 20L);
+        inputTopic.pipeInput("key3", "value3", 30L);
+        final TestOutputTopic outputTopic1 =
+                driver.createOutputTopic(OUTPUT_TOPIC_1, Serdes.String().deserializer(), Serdes.String().deserializer());
+        assertNextOutputRecord(outputTopic1.readRecord(), "key1", "value1", 10L);
+        assertNextOutputRecord(outputTopic1.readRecord(), "key2", "value2", 20L);
+        assertNextOutputRecord(outputTopic1.readRecord(), "key3", "value3", 30L);
     }
 
     @Test
     public void shouldConsiderModifiedTimeStamps() {
         final int partition = 10;
         driver = new TopologyTestDriver(createTimestampTopology(partition), props);
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1", 10L));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2", 20L));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key3", "value3", 30L));
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1", partition, 20L);
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key2", "value2", partition, 30L);
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key3", "value3", partition, 40L);
+        final TestInputTopic inputTopic = driver.createInputTopic(INPUT_TOPIC_1, STRING_SERIALIZER, STRING_SERIALIZER);
+        inputTopic.pipeInput("key1", "value1", 10L);
+        inputTopic.pipeInput("key2", "value2", 20L);
+        inputTopic.pipeInput("key3", "value3", 30L);
+        final TestOutputTopic outputTopic1 =
+                driver.createOutputTopic(OUTPUT_TOPIC_1, Serdes.String().deserializer(), Serdes.String().deserializer());
+        assertNextOutputRecord(outputTopic1.readRecord(), "key1", "value1", 20L);
+        assertNextOutputRecord(outputTopic1.readRecord(), "key2", "value2", 30L);
+        assertNextOutputRecord(outputTopic1.readRecord(), "key3", "value3", 40L);
     }
 
     @Test
     public void shouldConsiderModifiedTimeStampsForMultipleProcessors() {
         final int partition = 10;
         driver = new TopologyTestDriver(createMultiProcessorTimestampTopology(partition), props);
-
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1", 10L));
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1", partition, 10L);
-        assertNextOutputRecord(OUTPUT_TOPIC_2, "key1", "value1", partition, 20L);
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1", partition, 15L);
-        assertNextOutputRecord(OUTPUT_TOPIC_2, "key1", "value1", partition, 20L);
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1", partition, 12L);
-        assertNextOutputRecord(OUTPUT_TOPIC_2, "key1", "value1", partition, 22L);
-        assertNoOutputRecord(OUTPUT_TOPIC_1);
-        assertNoOutputRecord(OUTPUT_TOPIC_2);
-
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2", 20L));
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key2", "value2", partition, 20L);
-        assertNextOutputRecord(OUTPUT_TOPIC_2, "key2", "value2", partition, 30L);
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key2", "value2", partition, 25L);
-        assertNextOutputRecord(OUTPUT_TOPIC_2, "key2", "value2", partition, 30L);
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key2", "value2", partition, 22L);
-        assertNextOutputRecord(OUTPUT_TOPIC_2, "key2", "value2", partition, 32L);
-        assertNoOutputRecord(OUTPUT_TOPIC_1);
-        assertNoOutputRecord(OUTPUT_TOPIC_2);
+        final TestInputTopic inputTopic = driver.createInputTopic(INPUT_TOPIC_1, STRING_SERIALIZER, STRING_SERIALIZER);
+        final TestOutputTopic outputTopic1 =
+                driver.createOutputTopic(OUTPUT_TOPIC_1, Serdes.String().deserializer(), Serdes.String().deserializer());
+        final TestOutputTopic outputTopic2 =
+                driver.createOutputTopic(OUTPUT_TOPIC_2, Serdes.String().deserializer(), Serdes.String().deserializer());
+
+        inputTopic.pipeInput("key1", "value1", 10L);
+        assertNextOutputRecord(outputTopic1.readRecord(), "key1", "value1", 10L);
+        assertNextOutputRecord(outputTopic2.readRecord(), "key1", "value1", 20L);
+        assertNextOutputRecord(outputTopic1.readRecord(), "key1", "value1", 15L);
+        assertNextOutputRecord(outputTopic2.readRecord(), "key1", "value1", 20L);
+        assertNextOutputRecord(outputTopic1.readRecord(), "key1", "value1", 12L);
+        assertNextOutputRecord(outputTopic2.readRecord(), "key1", "value1", 22L);
+        assertTrue(outputTopic1.isEmpty());
+        assertTrue(outputTopic2.isEmpty());
+
+        inputTopic.pipeInput("key2", "value2", 20L);
+        assertNextOutputRecord(outputTopic1.readRecord(), "key2", "value2", 20L);
+        assertNextOutputRecord(outputTopic2.readRecord(), "key2", "value2", 30L);
+        assertNextOutputRecord(outputTopic1.readRecord(), "key2", "value2", 25L);
+        assertNextOutputRecord(outputTopic2.readRecord(), "key2", "value2", 30L);
+        assertNextOutputRecord(outputTopic1.readRecord(), "key2", "value2", 22L);
+        assertNextOutputRecord(outputTopic2.readRecord(), "key2", "value2", 32L);
+        assertTrue(outputTopic1.isEmpty());
+        assertTrue(outputTopic2.isEmpty());
     }
 
     @Test
     public void shouldConsiderHeaders() {
         final int partition = 10;
         driver = new TopologyTestDriver(createSimpleTopology(partition), props);
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1", HEADERS, 10L));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2", HEADERS, 20L));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key3", "value3", HEADERS, 30L));
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1", HEADERS, partition, 10L);
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key2", "value2", HEADERS, partition, 20L);
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key3", "value3", HEADERS, partition, 30L);
+        final TestInputTopic inputTopic = driver.createInputTopic(INPUT_TOPIC_1, STRING_SERIALIZER, STRING_SERIALIZER);
+        inputTopic.pipeInput(new TestRecord("key1", "value1", HEADERS, 10L));
+        inputTopic.pipeInput(new TestRecord("key2", "value2", HEADERS, 20L));
+        inputTopic.pipeInput(new TestRecord("key3", "value3", HEADERS, 30L));
+        final TestOutputTopic outputTopic1 = driver.createOutputTopic(OUTPUT_TOPIC_1, STRING_DESERIALIZER, STRING_DESERIALIZER);
+        assertNextOutputRecord(outputTopic1.readRecord(), "key1", "value1", HEADERS, 10L);
+        assertNextOutputRecord(outputTopic1.readRecord(), "key2", "value2", HEADERS, 20L);
+        assertNextOutputRecord(outputTopic1.readRecord(), "key3", "value3", HEADERS, 30L);
     }
 
     @Test
     public void shouldAddHeaders() {
         driver = new TopologyTestDriver(createAddHeaderTopology(), props);
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1", 10L));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2", 20L));
-        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key3", "value3", 30L));
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1", HEADERS, 10L);
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key2", "value2", HEADERS, 20L);
-        assertNextOutputRecord(OUTPUT_TOPIC_1, "key3", "value3", HEADERS, 30L);
+        final TestInputTopic inputTopic = driver.createInputTopic(INPUT_TOPIC_1, STRING_SERIALIZER, STRING_SERIALIZER);
+        inputTopic.pipeInput("key1", "value1", 10L);
+        inputTopic.pipeInput("key2", "value2", 20L);
+        inputTopic.pipeInput("key3", "value3", 30L);
+        final TestOutputTopic outputTopic1 = driver.createOutputTopic(OUTPUT_TOPIC_1, STRING_DESERIALIZER, STRING_DESERIALIZER);
+        assertNextOutputRecord(outputTopic1.readRecord(), "key1", "value1", HEADERS, 10L);
+        assertNextOutputRecord(outputTopic1.readRecord(), "key2", "value2", HEADERS, 20L);
+        assertNextOutputRecord(outputTopic1.readRecord(), "key3", "value3", HEADERS, 30L);
     }
 
     @Test
@@ -449,54 +498,30 @@ private ProcessorTopology createGlobalStoreTopology(final KeyValueBytesStoreSupp
         return topology.getInternalBuilder("anyAppId").build();
     }
 
-    private void assertNextOutputRecord(final String topic,
+    private void assertNextOutputRecord(final TestRecord record,
                                         final String key,
                                         final String value) {
-        assertNextOutputRecord(topic, key, value, (Integer) null, 0L);
+        assertNextOutputRecord(record, key, value, 0L);
     }
 
-    private void assertNextOutputRecord(final String topic,
+    private void assertNextOutputRecord(final TestRecord record,
                                         final String key,
                                         final String value,
-                                        final Integer partition) {
-        assertNextOutputRecord(topic, key, value, partition, 0L);
-    }
-
-    private void assertNextOutputRecord(final String topic,
-                                        final String key,
-                                        final String value,
-                                        final Headers headers,
                                         final Long timestamp) {
-        assertNextOutputRecord(topic, key, value, headers, null, timestamp);
+        assertNextOutputRecord(record, key, value, new RecordHeaders(), timestamp);
     }
 
-    private void assertNextOutputRecord(final String topic,
-                                        final String key,
-                                        final String value,
-                                        final Integer partition,
-                                        final Long timestamp) {
-        assertNextOutputRecord(topic, key, value, new RecordHeaders(), partition, timestamp);
-    }
-
-    private void assertNextOutputRecord(final String topic,
+    private void assertNextOutputRecord(final TestRecord record,
                                         final String key,
                                         final String value,
                                         final Headers headers,
-                                        final Integer partition,
                                         final Long timestamp) {
-        final ProducerRecord record = driver.readOutput(topic, STRING_DESERIALIZER, STRING_DESERIALIZER);
-        assertEquals(topic, record.topic());
         assertEquals(key, record.key());
         assertEquals(value, record.value());
-        assertEquals(partition, record.partition());
         assertEquals(timestamp, record.timestamp());
         assertEquals(headers, record.headers());
     }
 
-    private void assertNoOutputRecord(final String topic) {
-        assertNull(driver.readOutput(topic));
-    }
-
     private StreamPartitioner constantPartitioner(final Integer partition) {
         return (topic, key, value, numPartitions) -> partition;
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java
index c8d9cc101f5ba..b9db53f8e4775 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java
@@ -41,7 +41,7 @@
 import org.apache.kafka.streams.state.Stores;
 import org.apache.kafka.streams.state.WindowStore;
 import org.apache.kafka.streams.state.WindowStoreIterator;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.test.InternalMockProcessorContext;
 import org.apache.kafka.test.TestUtils;
 import org.junit.After;
@@ -49,6 +49,8 @@
 import org.junit.Test;
 
 import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.time.Instant;
 import java.util.List;
 import java.util.Properties;
 import java.util.UUID;
@@ -174,31 +176,32 @@ public void close() {}
         streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath());
         streamsConfiguration.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 10 * 1000);
 
-        final long initialWallClockTime = 0L;
+        final Instant initialWallClockTime = Instant.ofEpochMilli(0L);
         final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), streamsConfiguration, initialWallClockTime);
 
-        final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(
+        final TestInputTopic inputTopic = driver.createInputTopic(topic,
             Serdes.String().serializer(),
             Serdes.String().serializer(),
-            initialWallClockTime);
+            initialWallClockTime,
+            Duration.ZERO);
 
         for (int i = 0; i < 5; i++) {
-            driver.pipeInput(recordFactory.create(topic, UUID.randomUUID().toString(), UUID.randomUUID().toString()));
+            inputTopic.pipeInput(UUID.randomUUID().toString(), UUID.randomUUID().toString());
         }
-        driver.advanceWallClockTime(10 * 1000L);
-        recordFactory.advanceTimeMs(10 * 1000L);
+        driver.advanceWallClockTime(Duration.ofSeconds(10));
+        inputTopic.advanceTime(Duration.ofSeconds(10));
         for (int i = 0; i < 5; i++) {
-            driver.pipeInput(recordFactory.create(topic, UUID.randomUUID().toString(), UUID.randomUUID().toString()));
+            inputTopic.pipeInput(UUID.randomUUID().toString(), UUID.randomUUID().toString());
         }
-        driver.advanceWallClockTime(10 * 1000L);
-        recordFactory.advanceTimeMs(10 * 1000L);
+        driver.advanceWallClockTime(Duration.ofSeconds(10));
+        inputTopic.advanceTime(Duration.ofSeconds(10));
         for (int i = 0; i < 5; i++) {
-            driver.pipeInput(recordFactory.create(topic, UUID.randomUUID().toString(), UUID.randomUUID().toString()));
+            inputTopic.pipeInput(UUID.randomUUID().toString(), UUID.randomUUID().toString());
         }
-        driver.advanceWallClockTime(10 * 1000L);
-        recordFactory.advanceTimeMs(10 * 1000L);
+        driver.advanceWallClockTime(Duration.ofSeconds(10));
+        inputTopic.advanceTime(Duration.ofSeconds(10));
         for (int i = 0; i < 5; i++) {
-            driver.pipeInput(recordFactory.create(topic, UUID.randomUUID().toString(), UUID.randomUUID().toString()));
+            inputTopic.pipeInput(UUID.randomUUID().toString(), UUID.randomUUID().toString());
         }
     }
 
diff --git a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/kstream/KStreamTest.scala b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/kstream/KStreamTest.scala
index c317de03813b0..b3bcfe952438d 100644
--- a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/kstream/KStreamTest.scala
+++ b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/kstream/KStreamTest.scala
@@ -19,6 +19,7 @@
 package org.apache.kafka.streams.scala.kstream
 
 import java.time.Duration.ofSeconds
+import java.time.Instant
 
 import org.apache.kafka.streams.kstream.JoinWindows
 import org.apache.kafka.streams.scala.ImplicitConversions._
@@ -40,17 +41,19 @@ class KStreamTest extends FlatSpec with Matchers with TestDriver {
     builder.stream[String, String](sourceTopic).filter((_, value) => value != "value2").to(sinkTopic)
 
     val testDriver = createTestDriver(builder)
+    val testInput = testDriver.createInput[String, String](sourceTopic)
+    val testOutput = testDriver.createOutput[String, String](sinkTopic)
 
-    testDriver.pipeRecord(sourceTopic, ("1", "value1"))
-    testDriver.readRecord[String, String](sinkTopic).value shouldBe "value1"
+    testInput.pipeInput("1", "value1")
+    testOutput.readValue shouldBe "value1"
 
-    testDriver.pipeRecord(sourceTopic, ("2", "value2"))
-    testDriver.readRecord[String, String](sinkTopic) shouldBe null
+    testInput.pipeInput("2", "value2")
+    testOutput.isEmpty shouldBe true
 
-    testDriver.pipeRecord(sourceTopic, ("3", "value3"))
-    testDriver.readRecord[String, String](sinkTopic).value shouldBe "value3"
+    testInput.pipeInput("3", "value3")
+    testOutput.readValue shouldBe "value3"
 
-    testDriver.readRecord[String, String](sinkTopic) shouldBe null
+    testOutput.isEmpty shouldBe true
 
     testDriver.close()
   }
@@ -63,17 +66,19 @@ class KStreamTest extends FlatSpec with Matchers with TestDriver {
     builder.stream[String, String](sourceTopic).filterNot((_, value) => value == "value2").to(sinkTopic)
 
     val testDriver = createTestDriver(builder)
+    val testInput = testDriver.createInput[String, String](sourceTopic)
+    val testOutput = testDriver.createOutput[String, String](sinkTopic)
 
-    testDriver.pipeRecord(sourceTopic, ("1", "value1"))
-    testDriver.readRecord[String, String](sinkTopic).value shouldBe "value1"
+    testInput.pipeInput("1", "value1")
+    testOutput.readValue shouldBe "value1"
 
-    testDriver.pipeRecord(sourceTopic, ("2", "value2"))
-    testDriver.readRecord[String, String](sinkTopic) shouldBe null
+    testInput.pipeInput("2", "value2")
+    testOutput.isEmpty shouldBe true
 
-    testDriver.pipeRecord(sourceTopic, ("3", "value3"))
-    testDriver.readRecord[String, String](sinkTopic).value shouldBe "value3"
+    testInput.pipeInput("3", "value3")
+    testOutput.readValue shouldBe "value3"
 
-    testDriver.readRecord[String, String](sinkTopic) shouldBe null
+    testOutput.isEmpty shouldBe true
 
     testDriver.close()
   }
@@ -86,11 +91,12 @@ class KStreamTest extends FlatSpec with Matchers with TestDriver {
     builder.stream[String, String](sourceTopic).foreach((_, value) => acc += value)
 
     val testDriver = createTestDriver(builder)
+    val testInput = testDriver.createInput[String, String](sourceTopic)
 
-    testDriver.pipeRecord(sourceTopic, ("1", "value1"))
+    testInput.pipeInput("1", "value1")
     acc shouldBe "value1"
 
-    testDriver.pipeRecord(sourceTopic, ("2", "value2"))
+    testInput.pipeInput("2", "value2")
     acc shouldBe "value1value2"
 
     testDriver.close()
@@ -105,14 +111,16 @@ class KStreamTest extends FlatSpec with Matchers with TestDriver {
     builder.stream[String, String](sourceTopic).peek((_, v) => acc += v).to(sinkTopic)
 
     val testDriver = createTestDriver(builder)
+    val testInput = testDriver.createInput[String, String](sourceTopic)
+    val testOutput = testDriver.createOutput[String, String](sinkTopic)
 
-    testDriver.pipeRecord(sourceTopic, ("1", "value1"))
+    testInput.pipeInput("1", "value1")
     acc shouldBe "value1"
-    testDriver.readRecord[String, String](sinkTopic).value shouldBe "value1"
+    testOutput.readValue shouldBe "value1"
 
-    testDriver.pipeRecord(sourceTopic, ("2", "value2"))
+    testInput.pipeInput("2", "value2")
     acc shouldBe "value1value2"
-    testDriver.readRecord[String, String](sinkTopic).value shouldBe "value2"
+    testOutput.readValue shouldBe "value2"
 
     testDriver.close()
   }
@@ -125,14 +133,16 @@ class KStreamTest extends FlatSpec with Matchers with TestDriver {
     builder.stream[String, String](sourceTopic).selectKey((_, value) => value).to(sinkTopic)
 
     val testDriver = createTestDriver(builder)
+    val testInput = testDriver.createInput[String, String](sourceTopic)
+    val testOutput = testDriver.createOutput[String, String](sinkTopic)
 
-    testDriver.pipeRecord(sourceTopic, ("1", "value1"))
-    testDriver.readRecord[String, String](sinkTopic).key shouldBe "value1"
+    testInput.pipeInput("1", "value1")
+    testOutput.readKeyValue.key shouldBe "value1"
 
-    testDriver.pipeRecord(sourceTopic, ("1", "value2"))
-    testDriver.readRecord[String, String](sinkTopic).key shouldBe "value2"
+    testInput.pipeInput("1", "value2")
+    testOutput.readKeyValue.key shouldBe "value2"
 
-    testDriver.readRecord[String, String](sinkTopic) shouldBe null
+    testOutput.isEmpty shouldBe true
 
     testDriver.close()
   }
@@ -147,16 +157,19 @@ class KStreamTest extends FlatSpec with Matchers with TestDriver {
     val stream2 = builder.stream[String, String](sourceTopic2)
     stream1.join(stream2)((a, b) => s"$a-$b", JoinWindows.of(ofSeconds(1))).to(sinkTopic)
 
-    val now = System.currentTimeMillis()
+    val now = Instant.now()
 
     val testDriver = createTestDriver(builder, now)
+    val testInput1 = testDriver.createInput[String, String](sourceTopic1)
+    val testInput2 = testDriver.createInput[String, String](sourceTopic2)
+    val testOutput = testDriver.createOutput[String, String](sinkTopic)
 
-    testDriver.pipeRecord(sourceTopic1, ("1", "topic1value1"), now)
-    testDriver.pipeRecord(sourceTopic2, ("1", "topic2value1"), now)
+    testInput1.pipeInput("1", "topic1value1", now)
+    testInput2.pipeInput("1", "topic2value1", now)
 
-    testDriver.readRecord[String, String](sinkTopic).value shouldBe "topic1value1-topic2value1"
+    testOutput.readValue shouldBe "topic1value1-topic2value1"
 
-    testDriver.readRecord[String, String](sinkTopic) shouldBe null
+    testOutput.isEmpty shouldBe true
 
     testDriver.close()
   }
diff --git a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/kstream/KTableTest.scala b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/kstream/KTableTest.scala
index c0110a10d1eca..349408ce2b068 100644
--- a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/kstream/KTableTest.scala
+++ b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/kstream/KTableTest.scala
@@ -42,26 +42,28 @@ class KTableTest extends FlatSpec with Matchers with TestDriver {
     table.filter((_, value) => value > 1).toStream.to(sinkTopic)
 
     val testDriver = createTestDriver(builder)
+    val testInput = testDriver.createInput[String, String](sourceTopic)
+    val testOutput = testDriver.createOutput[String, Long](sinkTopic)
 
     {
-      testDriver.pipeRecord(sourceTopic, ("1", "value1"))
-      val record = testDriver.readRecord[String, Long](sinkTopic)
+      testInput.pipeInput("1", "value1")
+      val record = testOutput.readKeyValue
       record.key shouldBe "1"
       record.value shouldBe (null: java.lang.Long)
     }
     {
-      testDriver.pipeRecord(sourceTopic, ("1", "value2"))
-      val record = testDriver.readRecord[String, Long](sinkTopic)
+      testInput.pipeInput("1", "value2")
+      val record = testOutput.readKeyValue
       record.key shouldBe "1"
       record.value shouldBe 2
     }
     {
-      testDriver.pipeRecord(sourceTopic, ("2", "value1"))
-      val record = testDriver.readRecord[String, Long](sinkTopic)
+      testInput.pipeInput("2", "value1")
+      val record = testOutput.readKeyValue
       record.key shouldBe "2"
       record.value shouldBe (null: java.lang.Long)
     }
-    testDriver.readRecord[String, Long](sinkTopic) shouldBe null
+    testOutput.isEmpty shouldBe true
 
     testDriver.close()
   }
@@ -75,26 +77,28 @@ class KTableTest extends FlatSpec with Matchers with TestDriver {
     table.filterNot((_, value) => value > 1).toStream.to(sinkTopic)
 
     val testDriver = createTestDriver(builder)
+    val testInput = testDriver.createInput[String, String](sourceTopic)
+    val testOutput = testDriver.createOutput[String, Long](sinkTopic)
 
     {
-      testDriver.pipeRecord(sourceTopic, ("1", "value1"))
-      val record = testDriver.readRecord[String, Long](sinkTopic)
+      testInput.pipeInput("1", "value1")
+      val record = testOutput.readKeyValue
       record.key shouldBe "1"
       record.value shouldBe 1
     }
     {
-      testDriver.pipeRecord(sourceTopic, ("1", "value2"))
-      val record = testDriver.readRecord[String, Long](sinkTopic)
+      testInput.pipeInput("1", "value2")
+      val record = testOutput.readKeyValue
       record.key shouldBe "1"
       record.value shouldBe (null: java.lang.Long)
     }
     {
-      testDriver.pipeRecord(sourceTopic, ("2", "value1"))
-      val record = testDriver.readRecord[String, Long](sinkTopic)
+      testInput.pipeInput("2", "value1")
+      val record = testOutput.readKeyValue
       record.key shouldBe "2"
       record.value shouldBe 1
     }
-    testDriver.readRecord[String, Long](sinkTopic) shouldBe null
+    testOutput.isEmpty shouldBe true
 
     testDriver.close()
   }
@@ -110,12 +114,15 @@ class KTableTest extends FlatSpec with Matchers with TestDriver {
     table1.join(table2)((a, b) => a + b).toStream.to(sinkTopic)
 
     val testDriver = createTestDriver(builder)
+    val testInput1 = testDriver.createInput[String, String](sourceTopic1)
+    val testInput2 = testDriver.createInput[String, String](sourceTopic2)
+    val testOutput = testDriver.createOutput[String, Long](sinkTopic)
 
-    testDriver.pipeRecord(sourceTopic1, ("1", "topic1value1"))
-    testDriver.pipeRecord(sourceTopic2, ("1", "topic2value1"))
-    testDriver.readRecord[String, Long](sinkTopic).value shouldBe 2
+    testInput1.pipeInput("1", "topic1value1")
+    testInput2.pipeInput("1", "topic2value1")
+    testOutput.readValue shouldBe 2
 
-    testDriver.readRecord[String, Long](sinkTopic) shouldBe null
+    testOutput.isEmpty shouldBe true
 
     testDriver.close()
   }
@@ -133,13 +140,16 @@ class KTableTest extends FlatSpec with Matchers with TestDriver {
     table1.join(table2, materialized)((a, b) => a + b).toStream.to(sinkTopic)
 
     val testDriver = createTestDriver(builder)
+    val testInput1 = testDriver.createInput[String, String](sourceTopic1)
+    val testInput2 = testDriver.createInput[String, String](sourceTopic2)
+    val testOutput = testDriver.createOutput[String, Long](sinkTopic)
 
-    testDriver.pipeRecord(sourceTopic1, ("1", "topic1value1"))
-    testDriver.pipeRecord(sourceTopic2, ("1", "topic2value1"))
-    testDriver.readRecord[String, Long](sinkTopic).value shouldBe 2
+    testInput1.pipeInput("1", "topic1value1")
+    testInput2.pipeInput("1", "topic2value1")
+    testOutput.readValue shouldBe 2
     testDriver.getKeyValueStore[String, Long](stateStore).get("1") shouldBe 2
 
-    testDriver.readRecord[String, Long](sinkTopic) shouldBe null
+    testOutput.isEmpty shouldBe true
 
     testDriver.close()
   }
@@ -161,40 +171,42 @@ class KTableTest extends FlatSpec with Matchers with TestDriver {
     table.toStream((k, _) => s"${k.window().start()}:${k.window().end()}:${k.key()}").to(sinkTopic)
 
     val testDriver = createTestDriver(builder)
+    val testInput = testDriver.createInput[String, String](sourceTopic)
+    val testOutput = testDriver.createOutput[String, Long](sinkTopic)
 
     {
       // publish key=1 @ time 0 => count==1
-      testDriver.pipeRecord(sourceTopic, ("1", "value1"), 0L)
-      Option(testDriver.readRecord[String, Long](sinkTopic)) shouldBe None
+      testInput.pipeInput("1", "value1", 0L)
+      testOutput.isEmpty shouldBe true
     }
     {
       // publish key=1 @ time 1 => count==2
-      testDriver.pipeRecord(sourceTopic, ("1", "value2"), 1L)
-      Option(testDriver.readRecord[String, Long](sinkTopic)) shouldBe None
+      testInput.pipeInput("1", "value2", 1L)
+      testOutput.isEmpty shouldBe true
     }
     {
       // move event time past the first window, but before the suppression window
-      testDriver.pipeRecord(sourceTopic, ("2", "value1"), 1001L)
-      Option(testDriver.readRecord[String, Long](sinkTopic)) shouldBe None
+      testInput.pipeInput("2", "value1", 1001L)
+      testOutput.isEmpty shouldBe true
     }
     {
       // move event time riiiight before suppression window ends
-      testDriver.pipeRecord(sourceTopic, ("2", "value2"), 1999L)
-      Option(testDriver.readRecord[String, Long](sinkTopic)) shouldBe None
+      testInput.pipeInput("2", "value2", 1999L)
+      testOutput.isEmpty shouldBe true
     }
     {
       // publish a late event before suppression window terminates => count==3
-      testDriver.pipeRecord(sourceTopic, ("1", "value3"), 999L)
-      Option(testDriver.readRecord[String, Long](sinkTopic)) shouldBe None
+      testInput.pipeInput("1", "value3", 999L)
+      testOutput.isEmpty shouldBe true
     }
     {
       // move event time right past the suppression window of the first window.
-      testDriver.pipeRecord(sourceTopic, ("2", "value3"), 2001L)
-      val record = testDriver.readRecord[String, Long](sinkTopic)
+      testInput.pipeInput("2", "value3", 2001L)
+      val record = testOutput.readKeyValue
       record.key shouldBe "0:1000:1"
       record.value shouldBe 3L
     }
-    testDriver.readRecord[String, Long](sinkTopic) shouldBe null
+    testOutput.isEmpty shouldBe true
 
     testDriver.close()
   }
@@ -216,40 +228,42 @@ class KTableTest extends FlatSpec with Matchers with TestDriver {
     table.toStream((k, _) => s"${k.window().start()}:${k.window().end()}:${k.key()}").to(sinkTopic)
 
     val testDriver = createTestDriver(builder)
+    val testInput = testDriver.createInput[String, String](sourceTopic)
+    val testOutput = testDriver.createOutput[String, Long](sinkTopic)
 
     {
       // publish key=1 @ time 0 => count==1
-      testDriver.pipeRecord(sourceTopic, ("1", "value1"), 0L)
-      Option(testDriver.readRecord[String, Long](sinkTopic)) shouldBe None
+      testInput.pipeInput("1", "value1", 0L)
+      testOutput.isEmpty shouldBe true
     }
     {
       // publish key=1 @ time 1 => count==2
-      testDriver.pipeRecord(sourceTopic, ("1", "value2"), 1L)
-      Option(testDriver.readRecord[String, Long](sinkTopic)) shouldBe None
+      testInput.pipeInput("1", "value2", 1L)
+      testOutput.isEmpty shouldBe true
     }
     {
       // move event time past the window, but before the grace period
-      testDriver.pipeRecord(sourceTopic, ("2", "value1"), 1001L)
-      Option(testDriver.readRecord[String, Long](sinkTopic)) shouldBe None
+      testInput.pipeInput("2", "value1", 1001L)
+      testOutput.isEmpty shouldBe true
     }
     {
       // move event time riiiight before grace period ends
-      testDriver.pipeRecord(sourceTopic, ("2", "value2"), 1999L)
-      Option(testDriver.readRecord[String, Long](sinkTopic)) shouldBe None
+      testInput.pipeInput("2", "value2", 1999L)
+      testOutput.isEmpty shouldBe true
     }
     {
       // publish a late event before grace period terminates => count==3
-      testDriver.pipeRecord(sourceTopic, ("1", "value3"), 999L)
-      Option(testDriver.readRecord[String, Long](sinkTopic)) shouldBe None
+      testInput.pipeInput("1", "value3", 999L)
+      testOutput.isEmpty shouldBe true
     }
     {
       // move event time right past the grace period of the first window.
-      testDriver.pipeRecord(sourceTopic, ("2", "value3"), 2001L)
-      val record = testDriver.readRecord[String, Long](sinkTopic)
+      testInput.pipeInput("2", "value3", 2001L)
+      val record = testOutput.readKeyValue
       record.key shouldBe "0:1000:1"
       record.value shouldBe 3L
     }
-    testDriver.readRecord[String, Long](sinkTopic) shouldBe null
+    testOutput.isEmpty shouldBe true
 
     testDriver.close()
   }
@@ -272,53 +286,55 @@ class KTableTest extends FlatSpec with Matchers with TestDriver {
     table.toStream((k, _) => s"${k.window().start()}:${k.window().end()}:${k.key()}").to(sinkTopic)
 
     val testDriver = createTestDriver(builder)
+    val testInput = testDriver.createInput[String, String](sourceTopic)
+    val testOutput = testDriver.createOutput[String, Long](sinkTopic)
 
     {
       // first window
-      testDriver.pipeRecord(sourceTopic, ("k1", "v1"), 0L)
-      Option(testDriver.readRecord[String, Long](sinkTopic)) shouldBe None
+      testInput.pipeInput("k1", "v1", 0L)
+      testOutput.isEmpty shouldBe true
     }
     {
       // first window
-      testDriver.pipeRecord(sourceTopic, ("k1", "v1"), 1L)
-      Option(testDriver.readRecord[String, Long](sinkTopic)) shouldBe None
+      testInput.pipeInput("k1", "v1", 1L)
+      testOutput.isEmpty shouldBe true
     }
     {
       // new window, but grace period hasn't ended for first window
-      testDriver.pipeRecord(sourceTopic, ("k1", "v1"), 8L)
-      Option(testDriver.readRecord[String, Long](sinkTopic)) shouldBe None
+      testInput.pipeInput("k1", "v1", 8L)
+      testOutput.isEmpty shouldBe true
     }
     {
       // out-of-order event for first window, included since grade period hasn't passed
-      testDriver.pipeRecord(sourceTopic, ("k1", "v1"), 2L)
-      Option(testDriver.readRecord[String, Long](sinkTopic)) shouldBe None
+      testInput.pipeInput("k1", "v1", 2L)
+      testOutput.isEmpty shouldBe true
     }
     {
       // add to second window
-      testDriver.pipeRecord(sourceTopic, ("k1", "v1"), 13L)
-      Option(testDriver.readRecord[String, Long](sinkTopic)) shouldBe None
+      testInput.pipeInput("k1", "v1", 13L)
+      testOutput.isEmpty shouldBe true
     }
     {
       // add out-of-order to second window
-      testDriver.pipeRecord(sourceTopic, ("k1", "v1"), 10L)
-      Option(testDriver.readRecord[String, Long](sinkTopic)) shouldBe None
+      testInput.pipeInput("k1", "v1", 10L)
+      testOutput.isEmpty shouldBe true
     }
     {
       // push stream time forward to flush other events through
-      testDriver.pipeRecord(sourceTopic, ("k1", "v1"), 30L)
+      testInput.pipeInput("k1", "v1", 30L)
       // late event should get dropped from the stream
-      testDriver.pipeRecord(sourceTopic, ("k1", "v1"), 3L)
+      testInput.pipeInput("k1", "v1", 3L)
       // should now have to results
-      val r1 = testDriver.readRecord[String, Long](sinkTopic)
+      val r1 = testOutput.readRecord
       r1.key shouldBe "0:2:k1"
       r1.value shouldBe 3L
       r1.timestamp shouldBe 2L
-      val r2 = testDriver.readRecord[String, Long](sinkTopic)
+      val r2 = testOutput.readRecord
       r2.key shouldBe "8:13:k1"
       r2.value shouldBe 3L
       r2.timestamp shouldBe 13L
     }
-    testDriver.readRecord[String, Long](sinkTopic) shouldBe null
+    testOutput.isEmpty shouldBe true
 
     testDriver.close()
   }
@@ -338,40 +354,42 @@ class KTableTest extends FlatSpec with Matchers with TestDriver {
     table.toStream.to(sinkTopic)
 
     val testDriver = createTestDriver(builder)
+    val testInput = testDriver.createInput[String, String](sourceTopic)
+    val testOutput = testDriver.createOutput[String, Long](sinkTopic)
 
     {
       // publish key=1 @ time 0 => count==1
-      testDriver.pipeRecord(sourceTopic, ("1", "value1"), 0L)
-      Option(testDriver.readRecord[String, Long](sinkTopic)) shouldBe None
+      testInput.pipeInput("1", "value1", 0L)
+      testOutput.isEmpty shouldBe true
     }
     {
       // publish key=1 @ time 1 => count==2
-      testDriver.pipeRecord(sourceTopic, ("1", "value2"), 1L)
-      Option(testDriver.readRecord[String, Long](sinkTopic)) shouldBe None
+      testInput.pipeInput("1", "value2", 1L)
+      testOutput.isEmpty shouldBe true
     }
     {
       // move event time past the window, but before the grace period
-      testDriver.pipeRecord(sourceTopic, ("2", "value1"), 1001L)
-      Option(testDriver.readRecord[String, Long](sinkTopic)) shouldBe None
+      testInput.pipeInput("2", "value1", 1001L)
+      testOutput.isEmpty shouldBe true
     }
     {
-      // move event time riiiight before grace period ends
-      testDriver.pipeRecord(sourceTopic, ("2", "value2"), 1999L)
-      Option(testDriver.readRecord[String, Long](sinkTopic)) shouldBe None
+      // move event time right before grace period ends
+      testInput.pipeInput("2", "value2", 1999L)
+      testOutput.isEmpty shouldBe true
     }
     {
       // publish a late event before grace period terminates => count==3
-      testDriver.pipeRecord(sourceTopic, ("1", "value3"), 999L)
-      Option(testDriver.readRecord[String, Long](sinkTopic)) shouldBe None
+      testInput.pipeInput("1", "value3", 999L)
+      testOutput.isEmpty shouldBe true
     }
     {
       // move event time right past the grace period of the first window.
-      testDriver.pipeRecord(sourceTopic, ("2", "value3"), 2001L)
-      val record = testDriver.readRecord[String, Long](sinkTopic)
+      testInput.pipeInput("2", "value3", 2001L)
+      val record = testOutput.readKeyValue
       record.key shouldBe "1"
       record.value shouldBe 3L
     }
-    testDriver.readRecord[String, Long](sinkTopic) shouldBe null
+    testOutput.isEmpty shouldBe true
 
     testDriver.close()
   }
diff --git a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/utils/TestDriver.scala b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/utils/TestDriver.scala
index 17b25709f5de1..fde1af19bdc05 100644
--- a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/utils/TestDriver.scala
+++ b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/utils/TestDriver.scala
@@ -18,36 +18,29 @@
  */
 package org.apache.kafka.streams.scala.utils
 
+import java.time.Instant
 import java.util.{Properties, UUID}
 
-import org.apache.kafka.clients.producer.ProducerRecord
 import org.apache.kafka.common.serialization.Serde
 import org.apache.kafka.streams.scala.StreamsBuilder
-import org.apache.kafka.streams.test.ConsumerRecordFactory
-import org.apache.kafka.streams.{StreamsConfig, TopologyTestDriver}
+import org.apache.kafka.streams.{StreamsConfig, TestInputTopic, TestOutputTopic, TopologyTestDriver}
 import org.scalatest.Suite
 
 trait TestDriver { this: Suite =>
 
-  def createTestDriver(builder: StreamsBuilder,
-                       initialWallClockTimeMs: Long = System.currentTimeMillis()): TopologyTestDriver = {
+  def createTestDriver(builder: StreamsBuilder, initialWallClockTime: Instant = Instant.now()): TopologyTestDriver = {
     val config = new Properties()
     config.put(StreamsConfig.APPLICATION_ID_CONFIG, "test")
     config.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy:1234")
     config.put(StreamsConfig.STATE_DIR_CONFIG, s"out/state-store-${UUID.randomUUID()}")
-    new TopologyTestDriver(builder.build(), config, initialWallClockTimeMs)
+    new TopologyTestDriver(builder.build(), config, initialWallClockTime)
   }
 
   implicit class TopologyTestDriverOps(inner: TopologyTestDriver) {
-    def pipeRecord[K, V](topic: String, record: (K, V), timestampMs: Long = System.currentTimeMillis())(
-      implicit serdeKey: Serde[K],
-      serdeValue: Serde[V]
-    ): Unit = {
-      val recordFactory = new ConsumerRecordFactory[K, V](serdeKey.serializer, serdeValue.serializer)
-      inner.pipeInput(recordFactory.create(topic, record._1, record._2, timestampMs))
-    }
+    def createInput[K, V](topic: String)(implicit serdeKey: Serde[K], serdeValue: Serde[V]): TestInputTopic[K, V] =
+      inner.createInputTopic(topic, serdeKey.serializer, serdeValue.serializer)
 
-    def readRecord[K, V](topic: String)(implicit serdeKey: Serde[K], serdeValue: Serde[V]): ProducerRecord[K, V] =
-      inner.readOutput(topic, serdeKey.deserializer, serdeValue.deserializer)
+    def createOutput[K, V](topic: String)(implicit serdeKey: Serde[K], serdeValue: Serde[V]): TestOutputTopic[K, V] =
+      inner.createOutputTopic(topic, serdeKey.deserializer, serdeValue.deserializer)
   }
 }
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TestInputTopic.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TestInputTopic.java
new file mode 100644
index 0000000000000..162ce4cc9f28e
--- /dev/null
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TestInputTopic.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams;
+
+import org.apache.kafka.common.serialization.Serializer;
+import org.apache.kafka.streams.test.TestRecord;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Objects;
+import java.util.StringJoiner;
+
+/**
+ * {@code TestInputTopic} is used to pipe records to topic in {@link TopologyTestDriver}.
+ * To use {@code TestInputTopic} create a new instance via {@link TopologyTestDriver#createInputTopic(String, Serializer, Serializer)}
+ * In actual test code, you can pipe new record values, keys and values or list of {@link KeyValue} pairs
+ * If you have multiple source topics, you need to create a {@code TestInputTopic} for each.
+ *
+ *
+ * 
                Processing messages
                
+ * 
                {@code
+ *     private TestInputTopic inputTopic;
+ *     ...
+ *     inputTopic = testDriver.createInputTopic(INPUT_TOPIC, longSerializer, stringSerializer);
+ *     ...
+ *     inputTopic.pipeInput("Hello");
+ * }
                
+ *
+ * @param  the type of the record key
+ * @param  the type of the record value
+ * @see TopologyTestDriver
+ */
+
+public class TestInputTopic {
+    private final TopologyTestDriver driver;
+    private final String topic;
+    private final Serializer keySerializer;
+    private final Serializer valueSerializer;
+
+    //Timing
+    private Instant currentTime;
+    private final Duration advanceDuration;
+
+    TestInputTopic(final TopologyTestDriver driver,
+                   final String topicName,
+                   final Serializer keySerializer,
+                   final Serializer valueSerializer,
+                   final Instant startTimestamp,
+                   final Duration autoAdvance) {
+        Objects.requireNonNull(driver, "TopologyTestDriver cannot be null");
+        Objects.requireNonNull(topicName, "topicName cannot be null");
+        Objects.requireNonNull(keySerializer, "keySerializer cannot be null");
+        Objects.requireNonNull(valueSerializer, "valueSerializer cannot be null");
+        Objects.requireNonNull(startTimestamp, "startTimestamp cannot be null");
+        Objects.requireNonNull(autoAdvance, "autoAdvance cannot be null");
+        this.driver = driver;
+        this.topic = topicName;
+        this.keySerializer = keySerializer;
+        this.valueSerializer = valueSerializer;
+        this.currentTime = startTimestamp;
+        if (autoAdvance.isNegative()) {
+            throw new IllegalArgumentException("autoAdvance must be positive");
+        }
+        this.advanceDuration = autoAdvance;
+
+    }
+
+    /**
+     * Advances the internally tracked event time of this input topic. Each time a record without explicitly defined timestamp is piped, the current topic event time is used as record timestamp.
+     * 
                
+     * Note: advancing the event time on the input topic, does not advance the tracked stream time in {@link TopologyTestDriver} as long as no new input records are piped. Furthermore, it does not advance the wall-clock time of {@link TopologyTestDriver}.
+     *
+     * @param advance the duration of time to advance
+     */
+    public void advanceTime(final Duration advance) {
+        if (advance.isNegative()) {
+            throw new IllegalArgumentException("advance must be positive");
+        }
+        currentTime = currentTime.plus(advance);
+    }
+
+    private Instant getTimestampAndAdvanced() {
+        final Instant timestamp = currentTime;
+        currentTime = currentTime.plus(advanceDuration);
+        return timestamp;
+    }
+
+    /**
+     * Send an input record with the given record on the topic and then commit the records.
+     * May auto advance topic time
+     *
+     * @param record the record to sent
+     */
+    public void pipeInput(final TestRecord record) {
+        //if record timestamp not set get timestamp and advance
+        final Instant timestamp = (record.getRecordTime() == null) ? getTimestampAndAdvanced() : record.getRecordTime();
+        driver.pipeRecord(topic, record, keySerializer, valueSerializer, timestamp);
+    }
+
+    /**
+     * Send an input record with the given value on the topic and then commit the records.
+     * May auto advance topic time
+     *
+     * @param value the record value
+     */
+    public void pipeInput(final V value) {
+        pipeInput(new TestRecord<>(value));
+    }
+
+    /**
+     * Send an input record with the given key and value on the topic and then commit the records.
+     * May auto advance topic time
+     *
+     * @param key   the record key
+     * @param value the record value
+     */
+    public void pipeInput(final K key, final V value) {
+        pipeInput(new TestRecord<>(key, value));
+    }
+
+    /**
+     * Send an input record with the given value and timestamp on the topic and then commit the records.
+     * Does not auto advance internally tracked time.
+     *
+     * @param value       the record value
+     * @param timestamp the record timestamp
+     */
+    public void pipeInput(final V value,
+                          final Instant timestamp) {
+        pipeInput(new TestRecord(null, value, timestamp));
+    }
+
+    /**
+     * Send an input record with the given key, value and timestamp on the topic and then commit the records.
+     * Does not auto advance internally tracked time.
+     *
+     * @param key         the record key
+     * @param value       the record value
+     * @param timestampMs the record timestamp
+     */
+    public void pipeInput(final K key,
+                          final V value,
+                          final long timestampMs) {
+        pipeInput(new TestRecord(key, value, null, timestampMs));
+    }
+
+    /**
+     * Send an input record with the given key, value and timestamp on the topic and then commit the records.
+     * Does not auto advance internally tracked time.
+     *
+     * @param key         the record key
+     * @param value       the record value
+     * @param timestamp the record timestamp
+     */
+    public void pipeInput(final K key,
+                          final V value,
+                          final Instant timestamp) {
+        pipeInput(new TestRecord(key, value, timestamp));
+    }
+
+    /**
+     * Send input records with the given KeyValue  list on the topic  then commit each record individually.
+     * The timestamp will be generated based on the constructor provided start time and time will auto advance.
+     *
+     * @param records the list of TestRecord records
+     */
+    public void pipeRecordList(final List> records) {
+        for (final TestRecord record : records) {
+            pipeInput(record);
+        }
+    }
+
+    /**
+     * Send input records with the given KeyValue list on the topic then commit each record individually.
+     * The timestamp will be generated based on the constructor provided start time and time will auto advance based on autoAdvance setting.
+     *
+     * @param keyValues the {@link List} of {@link KeyValue} records
+     */
+    public void pipeKeyValueList(final List> keyValues) {
+        for (final KeyValue keyValue : keyValues) {
+            pipeInput(keyValue.key, keyValue.value);
+        }
+    }
+
+    /**
+     * Send input records with the given value list on the topic then commit each record individually.
+     * The timestamp will be generated based on the constructor provided start time and time will auto advance based on autoAdvance setting.
+     *
+     * @param values the {@link List} of {@link KeyValue} records
+     */
+    public void pipeValueList(final List values) {
+        for (final V value : values) {
+            pipeInput(value);
+        }
+    }
+
+    /**
+     * Send input records with the given {@link KeyValue} list on the topic then commit each record individually.
+     * Does not auto advance internally tracked time.
+     *
+     * @param keyValues      the {@link List} of {@link KeyValue} records
+     * @param startTimestamp the timestamp for the first generated record
+     * @param advance        the time difference between two consecutive generated records
+     */
+    public void pipeKeyValueList(final List> keyValues,
+                                 final Instant startTimestamp,
+                                 final Duration advance) {
+        Instant recordTime = startTimestamp;
+        for (final KeyValue keyValue : keyValues) {
+            pipeInput(keyValue.key, keyValue.value, recordTime);
+            recordTime = recordTime.plus(advance);
+        }
+    }
+
+    /**
+     * Send input records with the given value list on the topic then commit each record individually.
+     * The timestamp will be generated based on the constructor provided start time and time will auto advance based on autoAdvance setting.
+     *
+     * @param values         the {@link List} of values
+     * @param startTimestamp the timestamp for the first generated record
+     * @param advance        the time difference between two consecutive generated records
+     */
+    public void pipeValueList(final List values,
+                              final Instant startTimestamp,
+                              final Duration advance) {
+        Instant recordTime = startTimestamp;
+        for (final V value : values) {
+            pipeInput(value, recordTime);
+            recordTime = recordTime.plus(advance);
+        }
+    }
+
+    @Override
+    public String toString() {
+        return new StringJoiner(", ", TestInputTopic.class.getSimpleName() + "[", "]")
+                .add("topic='" + topic + "'")
+                .add("keySerializer=" + keySerializer.getClass().getSimpleName())
+                .add("valueSerializer=" + valueSerializer.getClass().getSimpleName())
+                .toString();
+    }
+}
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TestOutputTopic.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TestOutputTopic.java
new file mode 100644
index 0000000000000..c3f690a18005e
--- /dev/null
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TestOutputTopic.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams;
+
+import org.apache.kafka.common.serialization.Deserializer;
+import org.apache.kafka.streams.test.TestRecord;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.StringJoiner;
+
+/**
+ * {@code TestOutputTopic} is used to read records from a topic in {@link TopologyTestDriver}.
+ * To use {@code TestOutputTopic} create a new instance via {@link TopologyTestDriver#createOutputTopic(String, Deserializer, Deserializer)}
+ * In actual test code, you can read record values, keys, {@link KeyValue} or {@link TestRecord}
+ * If you have multiple source topics, you need to create a {@code TestOutputTopic} for each.
+ * 
                
+ * If you need to test key, value and headers, use {@link #readRecord()} methods.
+ * Using {@link #readKeyValue()} you get a {@link KeyValue} pair, and thus, don't get access to the record's timestamp or headers.
+ * Similarly using {@link #readValue()} you only get the value of a record.
+ *
+ * 
                Processing records
                
+ * 
                {@code
+ *     private TestOutputTopic outputTopic;
+ *      ...
+ *     outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, stringDeserializer, longDeserializer);
+ *     ...
+ *     assertThat(outputTopic.readValue()).isEqual(1);
+ * }
                
+ *
+ * @param  the type of the record key
+ * @param  the type of the record value
+ * @see TopologyTestDriver
+ */
+public class TestOutputTopic {
+    private final TopologyTestDriver driver;
+    private final String topic;
+    private final Deserializer keyDeserializer;
+    private final Deserializer valueDeserializer;
+
+    TestOutputTopic(final TopologyTestDriver driver,
+                    final String topicName,
+                    final Deserializer keyDeserializer,
+                    final Deserializer valueDeserializer) {
+        Objects.requireNonNull(driver, "TopologyTestDriver cannot be null");
+        Objects.requireNonNull(topicName, "topicName cannot be null");
+        Objects.requireNonNull(keyDeserializer, "keyDeserializer cannot be null");
+        Objects.requireNonNull(valueDeserializer, "valueDeserializer cannot be null");
+        this.driver = driver;
+        this.topic = topicName;
+        this.keyDeserializer = keyDeserializer;
+        this.valueDeserializer = valueDeserializer;
+    }
+
+
+    /**
+     * Read one record from the output topic and return record's value.
+     *
+     * @return Next value for output topic
+     */
+    public V readValue() {
+        final TestRecord record = readRecord();
+        return record.value();
+    }
+
+    /**
+     * Read one record from the output topic and return its key and value as pair.
+     *
+     * @return Next output as {@link KeyValue}
+     */
+    public KeyValue readKeyValue() {
+        final TestRecord record = readRecord();
+        return new KeyValue<>(record.key(), record.value());
+    }
+
+    /**
+     * Read one Record from output topic.
+     *
+     * @return Next output as {@link TestRecord}
+     */
+    public TestRecord readRecord() {
+        return driver.readRecord(topic, keyDeserializer, valueDeserializer);
+    }
+
+    /**
+     * Read output to List.
+     * This method can be used if the result is considered a stream. If the result is considered a table, the list will
+     * contain all updated, ie, a key might be contained multiple times. If you are only interested in the last table update (ie, the final table state), you can use {@link #readKeyValuesToMap()} instead.
+     *
+     * @return List of output
+     */
+    public List> readRecordsToList() {
+        final List> output = new LinkedList<>();
+        while (!isEmpty()) {
+            output.add(readRecord());
+        }
+        return output;
+    }
+
+
+    /**
+     * Read output to map.
+     * This method can be used if the result is considered a table, when
+     * you are only interested in the last table update (ie, the final table state).
+     * If the result is considered a stream, you can use {@link #readRecordsToList()} instead.
+     * The list will contain all updated, ie, a key might be contained multiple times.
+     * If the last update to a key is a delete/tombstone, the key will still be in the map (with null-value).
+     *
+     * @return Map of output by key
+     */
+    public Map readKeyValuesToMap() {
+        final Map output = new HashMap<>();
+        TestRecord outputRow;
+        while (!isEmpty()) {
+            outputRow = readRecord();
+            if (outputRow.key() == null) {
+                throw new NullPointerException("Null keys not allowed");
+            }
+            output.put(outputRow.key(), outputRow.value());
+        }
+        return output;
+    }
+
+    /**
+     * Read all KeyValues from topic to List.
+     *
+     * @return List of output KeyValues
+     */
+    public List> readKeyValuesToList() {
+        final List> output = new LinkedList<>();
+        KeyValue outputRow;
+        while (!isEmpty()) {
+            outputRow = readKeyValue();
+            output.add(outputRow);
+        }
+        return output;
+    }
+
+    /**
+     * Read all values from topic to List.
+     *
+     * @return List of output values
+     */
+    public List readValuesToList() {
+        final List output = new LinkedList<>();
+        V outputValue;
+        while (!isEmpty()) {
+            outputValue = readValue();
+            output.add(outputValue);
+        }
+        return output;
+    }
+
+    /**
+     * Get size of unread record in the topic queue.
+     *
+     * @return size of topic queue
+     */
+    public final long getQueueSize() {
+        return driver.getQueueSize(topic);
+    }
+
+    /**
+     * Verify if the topic queue is empty.
+     *
+     * @return true if no more record in the topic queue
+     */
+    public final boolean isEmpty() {
+        return driver.isEmpty(topic);
+    }
+
+    @Override
+    public String toString() {
+        return new StringJoiner(", ", TestOutputTopic.class.getSimpleName() + "[", "]")
+                .add("topic='" + topic + "'")
+                .add("keyDeserializer=" + keyDeserializer.getClass().getSimpleName())
+                .add("valueDeserializer=" + valueDeserializer.getClass().getSimpleName())
+                .add("size=" + getQueueSize())
+                .toString();
+    }
+}
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
index 9b3d0fdd7faf2..5abe89cdb8041 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
@@ -27,6 +27,7 @@
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.PartitionInfo;
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.header.Headers;
 import org.apache.kafka.common.header.internals.RecordHeaders;
 import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
@@ -75,14 +76,14 @@
 import org.apache.kafka.streams.state.WindowStore;
 import org.apache.kafka.streams.state.internals.ThreadCache;
 import org.apache.kafka.streams.state.internals.metrics.RocksDBMetricsRecordingTrigger;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
-import org.apache.kafka.streams.test.OutputVerifier;
+import org.apache.kafka.streams.test.TestRecord;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.Closeable;
 import java.io.IOException;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -91,6 +92,8 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Objects;
 import java.util.Properties;
 import java.util.Queue;
 import java.util.Set;
@@ -107,13 +110,15 @@
  * Best of all, the class works without a real Kafka broker, so the tests execute very quickly with very little overhead.
  * 
                
  * Using the {@code TopologyTestDriver} in tests is easy: simply instantiate the driver and provide a {@link Topology}
- * (cf. {@link StreamsBuilder#build()}) and {@link Properties configs}, use the driver to supply an
- * input message to the topology, and then use the driver to read and verify any messages output by the topology.
+ * (cf. {@link StreamsBuilder#build()}) and {@link Properties configs}, {@link #createInputTopic(String, Serializer, Serializer) create}
+ * and use a {@link TestInputTopic} to supply an input records to the topology,
+ * and then {@link #createOutputTopic(String, Deserializer, Deserializer) create} and use a {@link TestOutputTopic} to read and
+ * verify any output records by the topology.
  * 
                
  * Although the driver doesn't use a real Kafka broker, it does simulate Kafka {@link Consumer consumers} and
  * {@link Producer producers} that read and write raw {@code byte[]} messages.
- * You can either deal with messages that have {@code byte[]} keys and values or you use {@link ConsumerRecordFactory}
- * and {@link OutputVerifier} that work with regular Java objects instead of raw bytes.
+ * You can let {@link TestInputTopic} and {@link TestOutputTopic} to handle conversion
+ * form regular Java objects to raw bytes.
  *
  * 
                Driver setup
                
  * In order to create a {@code TopologyTestDriver} instance, you need a {@link Topology} and a {@link Properties config}.
@@ -138,32 +143,31 @@
  * This test driver simulates single-partitioned input topics.
  * Here's an example of an input message on the topic named {@code input-topic}:
  *
- * 
                - * ConsumerRecordFactory factory = new ConsumerRecordFactory(strSerializer, strSerializer);
- * driver.pipeInput(factory.create("input-topic","key1", "value1"));
- * 
                
+ * 
                {@code
+ * TestInputTopic inputTopic = driver.createInputTopic("input-topic", stringSerdeSerializer, stringSerializer);
+ * inputTopic.pipeInput("key1", "value1");
+ * }
                
  *
- * When {@code #pipeInput()} is called, the driver passes the input message through to the appropriate source that
+ * When {@link TestInputTopic#pipeInput(Object, Object)} is called, the driver passes the input message through to the appropriate source that
  * consumes the named topic, and will invoke the processor(s) downstream of the source.
  * If your topology's processors forward messages to sinks, your test can then consume these output messages to verify
  * they match the expected outcome.
  * For example, if our topology should have generated 2 messages on {@code output-topic-1} and 1 message on
  * {@code output-topic-2}, then our test can obtain these messages using the
- * {@link #readOutput(String, Deserializer, Deserializer)} method:
+ * {@link TestOutputTopic#readKeyValue()}  method:
  *
  * 
                {@code
- * ProducerRecord record1 = driver.readOutput("output-topic-1", strDeserializer, strDeserializer);
- * ProducerRecord record2 = driver.readOutput("output-topic-1", strDeserializer, strDeserializer);
- * ProducerRecord record3 = driver.readOutput("output-topic-2", strDeserializer, strDeserializer);
+ * TestOutputTopic outputTopic1 = driver.createOutputTopic("output-topic-1", stringDeserializer, stringDeserializer);
+ * TestOutputTopic outputTopic2 = driver.createOutputTopic("output-topic-2", stringDeserializer, stringDeserializer);
+ *
+ * KeyValue record1 = outputTopic1.readKeyValue();
+ * KeyValue record2 = outputTopic2.readKeyValue();
+ * KeyValue record3 = outputTopic1.readKeyValue();
  * }
                
  *
  * Again, our example topology generates messages with string keys and values, so we supply our string deserializer
  * instance for use on both the keys and values. Your test logic can then verify whether these output records are
  * correct.
- * Note, that calling {@link ProducerRecord#equals(Object)} compares all attributes including key, value, timestamp,
- * topic, partition, and headers.
- * If you only want to compare key and value (and maybe timestamp), using {@link OutputVerifier} instead of
- * {@link ProducerRecord#equals(Object)} can simplify your code as you can ignore attributes you are not interested in.
  * 
                
  * Note, that calling {@code pipeInput()} will also trigger {@link PunctuationType#STREAM_TIME event-time} base
  * {@link ProcessorContext#schedule(Duration, PunctuationType, Punctuator) punctuation} callbacks.
@@ -183,8 +187,8 @@
  * Or, our test might have pre-populated some state before submitting the input message, and verified afterward
  * that the processor(s) correctly updated the state.
  *
- * @see ConsumerRecordFactory
- * @see OutputVerifier
+ * @see TestInputTopic
+ * @see TestOutputTopic
  */
 public class TopologyTestDriver implements Closeable {
 
@@ -224,23 +228,41 @@ public class TopologyTestDriver implements Closeable {
     @SuppressWarnings("WeakerAccess")
     public TopologyTestDriver(final Topology topology,
                               final Properties config) {
-        this(topology, config, System.currentTimeMillis());
+        this(topology, config, (Instant) null);
     }
 
     /**
      * Create a new test diver instance.
      *
+     * @deprecated Since 2.4 use {@link #TopologyTestDriver(Topology, Properties, Instant)}
+     *
      * @param topology               the topology to be tested
      * @param config                 the configuration for the topology
      * @param initialWallClockTimeMs the initial value of internally mocked wall-clock time
      */
     @SuppressWarnings("WeakerAccess")
+    @Deprecated
     public TopologyTestDriver(final Topology topology,
                               final Properties config,
                               final long initialWallClockTimeMs) {
         this(topology.internalTopologyBuilder, config, initialWallClockTimeMs);
     }
 
+    /**
+     * Create a new test diver instance.
+     *
+     * @param topology               the topology to be tested
+     * @param config                 the configuration for the topology
+     * @param initialWallClockTime   the initial value of internally mocked wall-clock time
+     */
+    @SuppressWarnings("WeakerAccess")
+    public TopologyTestDriver(final Topology topology,
+                              final Properties config,
+                              final Instant initialWallClockTime) {
+        this(topology.internalTopologyBuilder, config, initialWallClockTime == null ? System.currentTimeMillis() : initialWallClockTime.toEpochMilli());
+    }
+
+
     /**
      * Create a new test diver instance.
      *
@@ -408,30 +430,41 @@ public void onRestoreEnd(final TopicPartition topicPartition, final String store
      * Send an input message with the given key, value, and timestamp on the specified topic to the topology and then
      * commit the messages.
      *
+     * @deprecated Since 2.4 use methods of {@link TestInputTopic} instead
+     *
      * @param consumerRecord the record to be processed
      */
     @SuppressWarnings("WeakerAccess")
+    @Deprecated
     public void pipeInput(final ConsumerRecord consumerRecord) {
-        final String topicName = consumerRecord.topic();
+        pipeRecord(consumerRecord.topic(), consumerRecord.timestamp(), consumerRecord.key(), consumerRecord.value(), consumerRecord.headers());
+    }
+
+    private void pipeRecord(final ProducerRecord record) {
+        pipeRecord(record.topic(), record.timestamp(), record.key(), record.value(), record.headers());
+    }
+
+    private void pipeRecord(final String topic, final Long timestamp, final byte[] key, final byte[] value, final Headers headers) {
+        final String topicName = topic;
 
         if (!internalTopologyBuilder.getSourceTopicNames().isEmpty()) {
-            validateSourceTopicNameRegexPattern(consumerRecord.topic());
+            validateSourceTopicNameRegexPattern(topic);
         }
         final TopicPartition topicPartition = getTopicPartition(topicName);
         if (topicPartition != null) {
             final long offset = offsetsByTopicPartition.get(topicPartition).incrementAndGet() - 1;
             task.addRecords(topicPartition, Collections.singleton(new ConsumerRecord<>(
-                topicName,
-                topicPartition.partition(),
-                offset,
-                consumerRecord.timestamp(),
-                consumerRecord.timestampType(),
-                (long) ConsumerRecord.NULL_CHECKSUM,
-                consumerRecord.serializedKeySize(),
-                consumerRecord.serializedValueSize(),
-                consumerRecord.key(),
-                consumerRecord.value(),
-                consumerRecord.headers())));
+                    topicName,
+                    topicPartition.partition(),
+                    offset,
+                    timestamp,
+                    TimestampType.CREATE_TIME,
+                    (long) ConsumerRecord.NULL_CHECKSUM,
+                    key == null ? ConsumerRecord.NULL_SIZE : key.length,
+                    value == null ? ConsumerRecord.NULL_SIZE : value.length,
+                    key,
+                    value,
+                    headers)));
 
             // Process the record ...
             task.process();
@@ -445,21 +478,22 @@ public void pipeInput(final ConsumerRecord consumerRecord) {
             }
             final long offset = offsetsByTopicPartition.get(globalTopicPartition).incrementAndGet() - 1;
             globalStateTask.update(new ConsumerRecord<>(
-                globalTopicPartition.topic(),
-                globalTopicPartition.partition(),
-                offset,
-                consumerRecord.timestamp(),
-                consumerRecord.timestampType(),
-                (long) ConsumerRecord.NULL_CHECKSUM,
-                consumerRecord.serializedKeySize(),
-                consumerRecord.serializedValueSize(),
-                consumerRecord.key(),
-                consumerRecord.value(),
-                consumerRecord.headers()));
+                    globalTopicPartition.topic(),
+                    globalTopicPartition.partition(),
+                    offset,
+                    timestamp,
+                    TimestampType.CREATE_TIME,
+                    (long) ConsumerRecord.NULL_CHECKSUM,
+                    key == null ? ConsumerRecord.NULL_SIZE : key.length,
+                    value == null ? ConsumerRecord.NULL_SIZE : value.length,
+                    key,
+                    value,
+                    headers));
             globalStateTask.flushState();
         }
     }
 
+
     private void validateSourceTopicNameRegexPattern(final String inputRecordTopic) {
         for (final String sourceTopicName : internalTopologyBuilder.getSourceTopicNames()) {
             if (!sourceTopicName.equals(inputRecordTopic) && Pattern.compile(sourceTopicName).matcher(inputRecordTopic).matches()) {
@@ -497,22 +531,7 @@ private void captureOutputRecords() {
             final String outputTopicName = record.topic();
             if (internalTopics.contains(outputTopicName) || processorTopology.sourceTopics().contains(outputTopicName)
                 || globalPartitionsByTopic.containsKey(outputTopicName)) {
-
-                final byte[] serializedKey = record.key();
-                final byte[] serializedValue = record.value();
-
-                pipeInput(new ConsumerRecord<>(
-                    outputTopicName,
-                    -1,
-                    -1L,
-                    record.timestamp(),
-                    TimestampType.CREATE_TIME,
-                    0L,
-                    serializedKey == null ? 0 : serializedKey.length,
-                    serializedValue == null ? 0 : serializedValue.length,
-                    serializedKey,
-                    serializedValue,
-                    record.headers()));
+                pipeRecord(record);
             }
         }
     }
@@ -520,9 +539,12 @@ private void captureOutputRecords() {
     /**
      * Send input messages to the topology and then commit each message individually.
      *
+     * @deprecated Since 2.4 use methods of {@link TestInputTopic} instead
+     *
      * @param records a list of records to be processed
      */
     @SuppressWarnings("WeakerAccess")
+    @Deprecated
     public void pipeInput(final List> records) {
         for (final ConsumerRecord record : records) {
             pipeInput(record);
@@ -534,11 +556,27 @@ public void pipeInput(final List> records) {
      * This might trigger a {@link PunctuationType#WALL_CLOCK_TIME wall-clock} type
      * {@link ProcessorContext#schedule(Duration, PunctuationType, Punctuator) punctuations}.
      *
+     * @deprecated Since 2.4 use {@link #advanceWallClockTime(Duration)} instead
+     *
      * @param advanceMs the amount of time to advance wall-clock time in milliseconds
      */
     @SuppressWarnings("WeakerAccess")
+    @Deprecated
     public void advanceWallClockTime(final long advanceMs) {
-        mockWallClockTime.sleep(advanceMs);
+        advanceWallClockTime(Duration.ofMillis(advanceMs));
+    }
+
+    /**
+     * Advances the internally mocked wall-clock time.
+     * This might trigger a {@link PunctuationType#WALL_CLOCK_TIME wall-clock} type
+     * {@link ProcessorContext#schedule(Duration, PunctuationType, Punctuator) punctuations}.
+     *
+     * @param advance the amount of time to advance wall-clock time
+     */
+    @SuppressWarnings("WeakerAccess")
+    public void advanceWallClockTime(final Duration advance) {
+        Objects.requireNonNull(advance, "advance cannot be null");
+        mockWallClockTime.sleep(advance.toMillis());
         if (task != null) {
             task.maybePunctuateSystemTime();
             task.commit();
@@ -550,10 +588,13 @@ public void advanceWallClockTime(final long advanceMs) {
      * Read the next record from the given topic.
      * These records were output by the topology during the previous calls to {@link #pipeInput(ConsumerRecord)}.
      *
+     * @deprecated Since 2.4 use methods of {@link TestOutputTopic} instead
+     *
      * @param topic the name of the topic
      * @return the next record on that topic, or {@code null} if there is no record available
      */
     @SuppressWarnings("WeakerAccess")
+    @Deprecated
     public ProducerRecord readOutput(final String topic) {
         final Queue> outputRecords = outputRecordsByTopic.get(topic);
         if (outputRecords == null) {
@@ -566,12 +607,15 @@ public ProducerRecord readOutput(final String topic) {
      * Read the next record from the given topic.
      * These records were output by the topology during the previous calls to {@link #pipeInput(ConsumerRecord)}.
      *
+     * @deprecated Since 2.4 use methods of {@link TestOutputTopic} instead
+     *
      * @param topic             the name of the topic
      * @param keyDeserializer   the deserializer for the key type
      * @param valueDeserializer the deserializer for the value type
      * @return the next record on that topic, or {@code null} if there is no record available
      */
     @SuppressWarnings("WeakerAccess")
+    @Deprecated
     public  ProducerRecord readOutput(final String topic,
                                                   final Deserializer keyDeserializer,
                                                   final Deserializer valueDeserializer) {
@@ -584,6 +628,142 @@ public  ProducerRecord readOutput(final String topic,
         return new ProducerRecord<>(record.topic(), record.partition(), record.timestamp(), key, value, record.headers());
     }
 
+
+    private final Queue> getRecordsQueue(final String topicName) {
+        final Queue> outputRecords = outputRecordsByTopic.get(topicName);
+        if (outputRecords == null) {
+            if (!processorTopology.sinkTopics().contains(topicName)) {
+                throw new IllegalArgumentException("Unknown topic: " + topicName);
+            }
+        }
+        return outputRecords;
+    }
+
+    /**
+     * Create {@link TestInputTopic} to be used for piping records to topic
+     * Uses current system time as start timestamp for records.
+     * Auto-advance is disabled.
+     *
+     * @param topicName             the name of the topic
+     * @param keySerializer   the Serializer for the key type
+     * @param valueSerializer the Serializer for the value type
+     * @param  the key type
+     * @param  the value type
+     * @return {@link TestInputTopic} object
+     */
+    public final  TestInputTopic createInputTopic(final String topicName,
+                                                              final Serializer keySerializer,
+                                                              final Serializer valueSerializer) {
+        return new TestInputTopic(this, topicName, keySerializer, valueSerializer, Instant.now(), Duration.ZERO);
+    }
+
+    /**
+     * Create {@link TestInputTopic} to be used for piping records to topic
+     * Uses provided start timestamp and autoAdvance parameter for records
+     *
+     * @param topicName             the name of the topic
+     * @param keySerializer   the Serializer for the key type
+     * @param valueSerializer the Serializer for the value type
+     * @param startTimestamp Start timestamp for auto-generated record time
+     * @param autoAdvance autoAdvance duration for auto-generated record time
+     * @param  the key type
+     * @param  the value type
+     * @return {@link TestInputTopic} object
+     */
+    public final  TestInputTopic createInputTopic(final String topicName,
+                                                              final Serializer keySerializer,
+                                                              final Serializer valueSerializer,
+                                                              final Instant startTimestamp,
+                                                              final Duration autoAdvance) {
+        return new TestInputTopic(this, topicName, keySerializer, valueSerializer, startTimestamp, autoAdvance);
+    }
+
+    /**
+     * Create {@link TestOutputTopic} to be used for reading records from topic
+     *
+     * @param topicName             the name of the topic
+     * @param keyDeserializer   the Deserializer for the key type
+     * @param valueDeserializer the Deserializer for the value type
+     * @param  the key type
+     * @param  the value type
+     * @return {@link TestOutputTopic} object
+     */
+    public final  TestOutputTopic createOutputTopic(final String topicName,
+                                                                final Deserializer keyDeserializer,
+                                                                final Deserializer valueDeserializer) {
+        return new TestOutputTopic(this, topicName, keyDeserializer, valueDeserializer);
+    }
+
+    @SuppressWarnings("WeakerAccess")
+    ProducerRecord readRecord(final String topic) {
+        final Queue> outputRecords = getRecordsQueue(topic);
+        if (outputRecords == null) {
+            return null;
+        }
+        return outputRecords.poll();
+    }
+
+    @SuppressWarnings("WeakerAccess")
+     TestRecord readRecord(final String topic,
+                                         final Deserializer keyDeserializer,
+                                         final Deserializer valueDeserializer) {
+        final Queue> outputRecords = getRecordsQueue(topic);
+        if (outputRecords == null) {
+            throw new NoSuchElementException("Uninitialized topic: " + topic);
+        }
+        final ProducerRecord record = outputRecords.poll();
+        if (record == null) {
+            throw new NoSuchElementException("Empty topic: " + topic);
+        }
+        final K key = keyDeserializer.deserialize(record.topic(), record.key());
+        final V value = valueDeserializer.deserialize(record.topic(), record.value());
+        return new TestRecord<>(key, value, record.headers(), record.timestamp());
+    }
+
+    @SuppressWarnings("WeakerAccess")
+     void pipeRecord(final String topic,
+                           final TestRecord record,
+                           final Serializer keySerializer,
+                           final Serializer valueSerializer,
+                           final Instant time) {
+        final byte[] serializedKey = keySerializer.serialize(topic, record.headers(), record.key());
+        final byte[] serializedValue = valueSerializer.serialize(topic, record.headers(), record.value());
+        long timestamp = 0;
+        if (time != null) {
+            timestamp = time.toEpochMilli();
+        } else if (record.timestamp() != null) {
+            timestamp = record.timestamp();
+        }
+
+        pipeRecord(topic, timestamp, serializedKey, serializedValue, record.headers());
+    }
+
+    /**
+     * Get size of unread record in the topic queue.
+     *
+     * @param topic
+     * @return size of topic queue
+     */
+    final long getQueueSize(final String topic) {
+        final Queue> queue = getRecordsQueue(topic);
+        if (queue == null) {
+            //Return 0 if not initialized, getRecordsQueue throw exception if non existing topic
+            return 0;
+        }
+        return queue.size();
+    }
+
+    /**
+     * Verify if the topic queue is empty.
+     *
+     * @param topic
+     * @return true if no more record in the topic queue
+     */
+    final boolean isEmpty(final String topic) {
+        return getQueueSize(topic) == 0;
+    }
+
+
     /**
      * Get all {@link StateStore StateStores} from the topology.
      * The stores can be a "regular" or global stores.
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/test/ConsumerRecordFactory.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/test/ConsumerRecordFactory.java
index e0fcefb983de7..ae71de85bb7b3 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/test/ConsumerRecordFactory.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/test/ConsumerRecordFactory.java
@@ -22,6 +22,7 @@
 import org.apache.kafka.common.record.TimestampType;
 import org.apache.kafka.common.serialization.Serializer;
 import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.streams.TopologyTestDriver;
 
 import java.util.ArrayList;
@@ -32,11 +33,14 @@
  * Factory to create {@link ConsumerRecord consumer records} for a single single-partitioned topic with given key and
  * value {@link Serializer serializers}.
  *
+ * @deprecated Since 2.4 use methods of {@link TestInputTopic} instead
+ *
  * @param  the type of the key
  * @param  the type of the value
  *
  * @see TopologyTestDriver
  */
+@Deprecated
 public class ConsumerRecordFactory {
     private final String topicName;
     private final Serializer keySerializer;
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/test/OutputVerifier.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/test/OutputVerifier.java
index f78e926e4316c..af0f51cb4dd57 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/test/OutputVerifier.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/test/OutputVerifier.java
@@ -18,6 +18,7 @@
 
 import org.apache.kafka.clients.producer.ProducerRecord;
 import org.apache.kafka.common.header.Headers;
+import org.apache.kafka.streams.TestOutputTopic;
 import org.apache.kafka.streams.TopologyTestDriver;
 
 import java.util.Objects;
@@ -25,8 +26,11 @@
 /**
  * Helper class to verify topology result records.
  *
+ * @deprecated Since 2.4 use methods of {@link TestOutputTopic} and standard assertion libraries instead
+ *
  * @see TopologyTestDriver
  */
+@Deprecated
 public class OutputVerifier {
 
     /**
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/test/TestRecord.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/test/TestRecord.java
new file mode 100644
index 0000000000000..a1992ae9d2ace
--- /dev/null
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/test/TestRecord.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.test;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.header.Headers;
+import org.apache.kafka.common.header.internals.RecordHeaders;
+import org.apache.kafka.streams.TestInputTopic;
+import org.apache.kafka.streams.TopologyTestDriver;
+
+import java.time.Instant;
+import java.util.Objects;
+import java.util.StringJoiner;
+
+/**
+ * A key/value pair, including timestamp and record headers, to be sent to or received from {@link TopologyTestDriver}.
+ * If [a] record does not contain a timestamp,
+ * {@link TestInputTopic} will auto advance it's time when the record is piped.
+ */
+public class TestRecord {
+    private final Headers headers;
+    private final K key;
+    private final V value;
+    private final Instant recordTime;
+
+
+    /**
+     * Creates a record.
+     *
+     * @param key The key that will be included in the record
+     * @param value The value of the record
+     * @param headers the record headers that will be included in the record
+     * @param recordTime The timestamp of the record.
+     */
+    public TestRecord(final K key, final V value, final Headers headers, final Instant recordTime) {
+        this.key = key;
+        this.value = value;
+        this.recordTime = recordTime;
+        this.headers = new RecordHeaders(headers);
+    }
+
+
+    /**
+     * Creates a record.
+     * 
+     * @param key The key that will be included in the record
+     * @param value The value of the record
+     * @param headers the record headers that will be included in the record
+     * @param timestampMs The timestamp of the record, in milliseconds since the beginning of the epoch.
+     */
+    public TestRecord(final K key, final V value, final Headers headers, final Long timestampMs) {
+        if (timestampMs != null) {
+            if (timestampMs < 0) {
+                throw new IllegalArgumentException(
+                        String.format("Invalid timestamp: %d. Timestamp should always be non-negative or null.", timestampMs));
+            }
+            this.recordTime = Instant.ofEpochMilli(timestampMs);
+        } else {
+            this.recordTime = null;
+        }
+        this.key = key;
+        this.value = value;
+        this.headers = new RecordHeaders(headers);
+    }
+
+    /**
+     * Creates a record.
+     *
+     * @param key The key of the record
+     * @param value The value of the record
+     * @param recordTime The timestamp of the record as Instant.
+     */
+    public TestRecord(final K key, final V value, final Instant recordTime) {
+        this(key, value, null, recordTime);
+    }
+
+    /**
+     * Creates a record.
+     *
+     * @param key The key of the record
+     * @param value The value of the record
+     * @param headers The record headers that will be included in the record
+     */
+    public TestRecord(final K key, final V value, final Headers headers) {
+        this.key = key;
+        this.value = value;
+        this.headers = new RecordHeaders(headers);
+        this.recordTime = null;
+    }
+    
+    /**
+     * Creates a record.
+     *
+     * @param key The key of the record
+     * @param value The value of the record
+     */
+    public TestRecord(final K key, final V value) {
+        this.key = key;
+        this.value = value;
+        this.headers = new RecordHeaders();
+        this.recordTime = null;
+    }
+
+    /**
+     * Create a record with {@code null} key.
+     *
+     * @param value The value of the record
+     */
+    public TestRecord(final V value) {
+        this(null, value);
+    }
+
+    /**
+     * Create a {@code TestRecord} from a {@link ConsumerRecord}.
+     *
+     * @param record The v
+     */
+    public TestRecord(final ConsumerRecord record) {
+        Objects.requireNonNull(record);
+        this.key = record.key();
+        this.value = record.value();
+        this.headers = record.headers();
+        this.recordTime = Instant.ofEpochMilli(record.timestamp());
+    }
+
+    /**
+     * Create a {@code TestRecord} from a {@link ProducerRecord}.
+     *
+     * @param record The record contents
+     */
+    public TestRecord(final ProducerRecord record) {
+        Objects.requireNonNull(record);
+        this.key = record.key();
+        this.value = record.value();
+        this.headers = record.headers();
+        this.recordTime = Instant.ofEpochMilli(record.timestamp());
+    }
+
+    /**
+     * @return The headers
+     */
+    public Headers headers() {
+        return headers;
+    }
+
+    /**
+     * @return The key (or null if no key is specified)
+     */
+    public K key() {
+        return key;
+    }
+
+
+    /**
+     * @return The value
+     */
+    public V value() {
+        return value;
+    }
+
+    /**
+     * @return The timestamp, which is in milliseconds since epoch.
+     */
+    public Long timestamp() {
+        return this.recordTime == null ? null : this.recordTime.toEpochMilli();
+    }
+
+    /**
+     * @return The headers
+     */
+    public Headers getHeaders() {
+        return headers;
+    }
+
+    /**
+     * @return The key (or null if no key is specified)
+     */
+    public K getKey() {
+        return key;
+    }
+
+    /**
+     * @return The value
+     */
+    public V getValue() {
+        return value;
+    }
+
+    /**
+     * @return The recordTime as Instant
+     */
+    public Instant getRecordTime() {
+        return recordTime;
+    }
+
+    @Override
+    public String toString() {
+        return new StringJoiner(", ", TestRecord.class.getSimpleName() + "[", "]")
+                .add("key=" + key)
+                .add("value=" + value)
+                .add("headers=" + headers)
+                .add("recordTime=" + recordTime)
+                .toString();
+    }
+
+    @Override
+    public boolean equals(final Object o) {
+        if (this == o)
+            return true;
+        else if (!(o instanceof TestRecord))
+            return false;
+
+        final TestRecord that = (TestRecord) o;
+
+        if (key != null ? !key.equals(that.key) : that.key != null) 
+            return false;
+        else if (headers != null ? !headers.equals(that.headers) : that.headers != null)
+            return false;
+        else if (value != null ? !value.equals(that.value) : that.value != null) 
+            return false;
+        else if (recordTime != null ? !recordTime.equals(that.recordTime) : that.recordTime != null)
+            return false;
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = headers != null ? headers.hashCode() : 0;
+        result = 31 * result + (key != null ? key.hashCode() : 0);
+        result = 31 * result + (value != null ? value.hashCode() : 0);
+        result = 31 * result + (recordTime != null ? recordTime.hashCode() : 0);
+        return result;
+    }
+}
diff --git a/streams/test-utils/src/test/java/org/apache/kafka/streams/TestTopicsTest.java b/streams/test-utils/src/test/java/org/apache/kafka/streams/TestTopicsTest.java
new file mode 100644
index 0000000000000..d8c78476415f4
--- /dev/null
+++ b/streams/test-utils/src/test/java/org/apache/kafka/streams/TestTopicsTest.java
@@ -0,0 +1,412 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams;
+
+import org.apache.kafka.common.errors.SerializationException;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.header.Headers;
+import org.apache.kafka.common.header.internals.RecordHeader;
+import org.apache.kafka.common.header.internals.RecordHeaders;
+import org.apache.kafka.common.serialization.Serde;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.streams.errors.StreamsException;
+import org.apache.kafka.streams.kstream.Consumed;
+import org.apache.kafka.streams.kstream.KStream;
+import org.apache.kafka.streams.kstream.Produced;
+import org.apache.kafka.streams.test.TestRecord;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Properties;
+
+import static org.apache.kafka.common.utils.Utils.mkEntry;
+import static org.apache.kafka.common.utils.Utils.mkMap;
+import static org.apache.kafka.common.utils.Utils.mkProperties;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.hasItems;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.hasProperty;
+import static org.junit.Assert.assertThrows;
+
+public class TestTopicsTest {
+    private static final Logger log = LoggerFactory.getLogger(TestTopicsTest.class);
+
+    private final static String INPUT_TOPIC = "input";
+    private final static String OUTPUT_TOPIC = "output1";
+    private final static String INPUT_TOPIC_MAP = OUTPUT_TOPIC;
+    private final static String OUTPUT_TOPIC_MAP = "output2";
+
+    private TopologyTestDriver testDriver;
+    private final Serde stringSerde = new Serdes.StringSerde();
+    private final Serde longSerde = new Serdes.LongSerde();
+
+    private final Properties config = mkProperties(mkMap(
+            mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, "TestTopicsTest"),
+            mkEntry(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy:1234")
+    ));
+    private final Instant testBaseTime = Instant.parse("2019-06-01T10:00:00Z");
+
+    @Before
+    public void setup() {
+        final StreamsBuilder builder = new StreamsBuilder();
+        //Create Actual Stream Processing pipeline
+        builder.stream(INPUT_TOPIC).to(OUTPUT_TOPIC);
+        final KStream source = builder.stream(INPUT_TOPIC_MAP, Consumed.with(longSerde, stringSerde));
+        final KStream mapped = source.map((key, value) -> new KeyValue<>(value, key));
+        mapped.to(OUTPUT_TOPIC_MAP, Produced.with(stringSerde, longSerde));
+        testDriver = new TopologyTestDriver(builder.build(), config);
+    }
+
+    @After
+    public void tearDown() {
+        try {
+            testDriver.close();
+        } catch (final RuntimeException e) {
+            // https://issues.apache.org/jira/browse/KAFKA-6647 causes exception when executed in Windows, ignoring it
+            // Logged stacktrace cannot be avoided
+            log.warn("Ignoring exception, test failing in Windows due this exception: {}", e.getLocalizedMessage());
+        }
+    }
+
+    @Test
+    public void testValue() {
+        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC, stringSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, stringSerde.deserializer(), stringSerde.deserializer());
+        //Feed word "Hello" to inputTopic and no kafka key, timestamp is irrelevant in this case
+        inputTopic.pipeInput("Hello");
+        assertThat(outputTopic.readValue(), equalTo("Hello"));
+        //No more output in topic
+        assertThat(outputTopic.isEmpty(), is(true));
+    }
+
+    @Test
+    public void testValueList() {
+        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC, stringSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, stringSerde.deserializer(), stringSerde.deserializer());
+        final List inputList = Arrays.asList("This", "is", "an", "example");
+        //Feed list of words to inputTopic and no kafka key, timestamp is irrelevant in this case
+        inputTopic.pipeValueList(inputList);
+        final List output = outputTopic.readValuesToList();
+        assertThat(output, hasItems("This", "is", "an", "example"));
+        assertThat(output, is(equalTo(inputList)));
+    }
+
+    @Test
+    public void testKeyValue() {
+        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
+        inputTopic.pipeInput(1L, "Hello");
+        assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>(1L, "Hello")));
+        assertThat(outputTopic.isEmpty(), is(true));
+    }
+
+    @Test
+    public void testKeyValueList() {
+        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC_MAP, stringSerde.deserializer(), longSerde.deserializer());
+        final List inputList = Arrays.asList("This", "is", "an", "example");
+        final List> input = new LinkedList<>();
+        final List> expected = new LinkedList<>();
+        long i = 0;
+        for (final String s : inputList) {
+            input.add(new KeyValue<>(i, s));
+            expected.add(new KeyValue<>(s, i));
+            i++;
+        }
+        inputTopic.pipeKeyValueList(input);
+        final List> output = outputTopic.readKeyValuesToList();
+        assertThat(output, is(equalTo(expected)));
+    }
+
+    @Test
+    public void testKeyValuesToMap() {
+        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC_MAP, stringSerde.deserializer(), longSerde.deserializer());
+        final List inputList = Arrays.asList("This", "is", "an", "example");
+        final List> input = new LinkedList<>();
+        final Map expected = new HashMap<>();
+        long i = 0;
+        for (final String s : inputList) {
+            input.add(new KeyValue<>(i, s));
+            expected.put(s, i);
+            i++;
+        }
+        inputTopic.pipeKeyValueList(input);
+        final Map output = outputTopic.readKeyValuesToMap();
+        assertThat(output, is(equalTo(expected)));
+    }
+
+    @Test
+    public void testKeyValueListDuration() {
+        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC_MAP, stringSerde.deserializer(), longSerde.deserializer());
+        final List inputList = Arrays.asList("This", "is", "an", "example");
+        final List> input = new LinkedList<>();
+        final List> expected = new LinkedList<>();
+        long i = 0;
+        final Duration advance = Duration.ofSeconds(15);
+        Instant recordInstant = testBaseTime;
+        for (final String s : inputList) {
+            input.add(new KeyValue<>(i, s));
+            expected.add(new TestRecord<>(s, i, recordInstant));
+            i++;
+            recordInstant = recordInstant.plus(advance);
+        }
+        inputTopic.pipeKeyValueList(input, testBaseTime, advance);
+        final List> output = outputTopic.readRecordsToList();
+        assertThat(output, is(equalTo(expected)));
+    }
+
+    @Test
+    public void testRecordList() {
+        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC_MAP, stringSerde.deserializer(), longSerde.deserializer());
+        final List inputList = Arrays.asList("This", "is", "an", "example");
+        final List> input = new LinkedList<>();
+        final List> expected = new LinkedList<>();
+        final Duration advance = Duration.ofSeconds(15);
+        Instant recordInstant = testBaseTime;
+        Long i = 0L;
+        for (final String s : inputList) {
+            input.add(new TestRecord(i, s, recordInstant));
+            expected.add(new TestRecord(s, i, recordInstant));
+            i++;
+            recordInstant = recordInstant.plus(advance);
+        }
+        inputTopic.pipeRecordList(input);
+        final List> output = outputTopic.readRecordsToList();
+        assertThat(output, is(equalTo(expected)));
+    }
+
+    @Test
+    public void testTimestamp() {
+        long baseTime = 3;
+        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
+        inputTopic.pipeInput(null, "Hello", baseTime);
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(null, "Hello", null, baseTime))));
+
+        inputTopic.pipeInput(2L, "Kafka", ++baseTime);
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(2L, "Kafka", null, baseTime))));
+
+        inputTopic.pipeInput(2L, "Kafka", testBaseTime);
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(2L, "Kafka", testBaseTime))));
+
+        final List inputList = Arrays.asList("Advancing", "time");
+        //Feed list of words to inputTopic and no kafka key, timestamp advancing from testInstant
+        final Duration advance = Duration.ofSeconds(15);
+        final Instant recordInstant = testBaseTime.plus(Duration.ofDays(1));
+        inputTopic.pipeValueList(inputList, recordInstant, advance);
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(null, "Advancing", recordInstant))));
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(null, "time", null, recordInstant.plus(advance)))));
+    }
+
+    @Test
+    public void testWithHeaders() {
+        long baseTime = 3;
+        final Headers headers = new RecordHeaders(
+                new Header[]{
+                    new RecordHeader("foo", "value".getBytes()),
+                    new RecordHeader("bar", (byte[]) null),
+                    new RecordHeader("\"A\\u00ea\\u00f1\\u00fcC\"", "value".getBytes())
+                });
+        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
+        inputTopic.pipeInput(new TestRecord(1L, "Hello", headers));
+        assertThat(outputTopic.readRecord(), allOf(
+                hasProperty("key", equalTo(1L)),
+                hasProperty("value", equalTo("Hello")),
+                hasProperty("headers", equalTo(headers))));
+        inputTopic.pipeInput(new TestRecord(2L, "Kafka", headers, ++baseTime));
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(2L, "Kafka", headers, baseTime))));
+    }
+
+    @Test
+    public void testStartTimestamp() {
+        final long baseTime = testBaseTime.toEpochMilli();
+        final Duration advance = Duration.ofSeconds(2);
+        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC, longSerde.serializer(), stringSerde.serializer(), testBaseTime, Duration.ZERO);
+        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
+        inputTopic.pipeInput(1L, "Hello");
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(1L, "Hello", testBaseTime))));
+        inputTopic.pipeInput(2L, "World");
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(2L, "World", null, testBaseTime.toEpochMilli()))));
+        inputTopic.advanceTime(advance);
+        inputTopic.pipeInput(3L, "Kafka");
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(3L, "Kafka", testBaseTime.plus(advance)))));
+    }
+
+    @Test
+    public void testTimestampAutoAdvance() {
+        final Duration advance = Duration.ofSeconds(2);
+        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC, longSerde.serializer(), stringSerde.serializer(), testBaseTime, advance);
+        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
+        inputTopic.pipeInput("Hello");
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(null, "Hello", testBaseTime))));
+        inputTopic.pipeInput(2L, "Kafka");
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(2L, "Kafka", testBaseTime.plus(advance)))));
+    }
+
+
+    @Test
+    public void testMultipleTopics() {
+        final TestInputTopic inputTopic1 = testDriver.createInputTopic(INPUT_TOPIC, longSerde.serializer(), stringSerde.serializer());
+        final TestInputTopic inputTopic2 = testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic1 = testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
+        final TestOutputTopic outputTopic2 = testDriver.createOutputTopic(OUTPUT_TOPIC_MAP, stringSerde.deserializer(), longSerde.deserializer());
+        inputTopic1.pipeInput(1L, "Hello");
+        assertThat(outputTopic1.readKeyValue(), equalTo(new KeyValue<>(1L, "Hello")));
+        assertThat(outputTopic2.readKeyValue(), equalTo(new KeyValue<>("Hello", 1L)));
+        assertThat(outputTopic1.isEmpty(), is(true));
+        assertThat(outputTopic2.isEmpty(), is(true));
+        inputTopic2.pipeInput(1L, "Hello");
+        //This is not visible in outputTopic1 even it is the same topic
+        assertThat(outputTopic2.readKeyValue(), equalTo(new KeyValue<>("Hello", 1L)));
+        assertThat(outputTopic1.isEmpty(), is(true));
+        assertThat(outputTopic2.isEmpty(), is(true));
+    }
+
+    @Test
+    public void testNonExistingOutputTopic() {
+        final TestOutputTopic outputTopic = testDriver.createOutputTopic("no-exist", longSerde.deserializer(), stringSerde.deserializer());
+        assertThrows("Unknown topic", IllegalArgumentException.class, () -> outputTopic.readRecord());
+    }
+
+    @Test
+    public void testNonUsedOutputTopic() {
+        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
+        assertThrows("Uninitialized topic", NoSuchElementException.class, () -> outputTopic.readRecord());
+    }
+
+    @Test
+    public void testEmptyTopic() {
+        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC, stringSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, stringSerde.deserializer(), stringSerde.deserializer());
+        //Feed word "Hello" to inputTopic and no kafka key, timestamp is irrelevant in this case
+        inputTopic.pipeInput("Hello");
+        assertThat(outputTopic.readValue(), equalTo("Hello"));
+        //No more output in topic
+        assertThrows("Empty topic", NoSuchElementException.class, () -> outputTopic.readRecord());
+    }
+
+    @Test
+    public void testNonExistingInputTopic() {
+        final TestInputTopic inputTopic = testDriver.createInputTopic("no-exist", longSerde.serializer(), stringSerde.serializer());
+        assertThrows("Unknown topic", IllegalArgumentException.class, () -> inputTopic.pipeInput(1L, "Hello"));
+    }
+
+    @Test(expected = NullPointerException.class)
+    public void shouldNotAllowToCreateTopicWithNullTopicName() {
+        final TestInputTopic inputTopic = testDriver.createInputTopic(null, stringSerde.serializer(), stringSerde.serializer());
+    }
+
+    @Test(expected = NullPointerException.class)
+    public void shouldNotAllowToCreateWithNullDriver() {
+        final TestInputTopic inputTopic = new TestInputTopic<>(null, INPUT_TOPIC, stringSerde.serializer(), stringSerde.serializer(), Instant.now(), Duration.ZERO);
+    }
+
+
+    @Test(expected = StreamsException.class)
+    public void testWrongSerde() {
+        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC_MAP, stringSerde.serializer(), stringSerde.serializer());
+        inputTopic.pipeInput("1L", "Hello");
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testDuration() {
+        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC_MAP, stringSerde.serializer(), stringSerde.serializer(), testBaseTime, Duration.ofDays(-1));
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testNegativeAdvance() {
+        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC_MAP, stringSerde.serializer(), stringSerde.serializer());
+        inputTopic.advanceTime(Duration.ofDays(-1));
+    }
+
+    @Test
+    public void testInputToString() {
+        final TestInputTopic inputTopic = testDriver.createInputTopic("topicName", stringSerde.serializer(), stringSerde.serializer());
+        assertThat(inputTopic.toString(), allOf(
+                containsString("TestInputTopic"),
+                containsString("topic='topicName'"),
+                containsString("StringSerializer")));
+    }
+
+    @Test(expected = NullPointerException.class)
+    public void shouldNotAllowToCreateOutputTopicWithNullTopicName() {
+        final TestOutputTopic outputTopic = testDriver.createOutputTopic(null, stringSerde.deserializer(), stringSerde.deserializer());
+    }
+
+    @Test(expected = NullPointerException.class)
+    public void shouldNotAllowToCreateOutputWithNullDriver() {
+        final TestOutputTopic outputTopic = new TestOutputTopic<>(null, OUTPUT_TOPIC, stringSerde.deserializer(), stringSerde.deserializer());
+    }
+
+    @Test(expected = SerializationException.class)
+    public void testOutputWrongSerde() {
+        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC_MAP, longSerde.deserializer(), stringSerde.deserializer());
+        inputTopic.pipeInput(1L, "Hello");
+        assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>(1L, "Hello")));
+    }
+
+    @Test
+    public void testOutputToString() {
+        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, stringSerde.deserializer(), stringSerde.deserializer());
+        assertThat(outputTopic.toString(), allOf(
+                containsString("TestOutputTopic"),
+                containsString("topic='output1'"),
+                containsString("size=0"),
+                containsString("StringDeserializer")));
+    }
+
+    @Test
+    public void testRecordsToList() {
+        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC_MAP, stringSerde.deserializer(), longSerde.deserializer());
+        final List inputList = Arrays.asList("This", "is", "an", "example");
+        final List> input = new LinkedList<>();
+        final List> expected = new LinkedList<>();
+        long i = 0;
+        final Duration advance = Duration.ofSeconds(15);
+        Instant recordInstant = Instant.parse("2019-06-01T10:00:00Z");
+        for (final String s : inputList) {
+            input.add(new KeyValue<>(i, s));
+            expected.add(new TestRecord<>(s, i, recordInstant));
+            i++;
+            recordInstant = recordInstant.plus(advance);
+        }
+        inputTopic.pipeKeyValueList(input, Instant.parse("2019-06-01T10:00:00Z"), advance);
+        final List> output = outputTopic.readRecordsToList();
+        assertThat(output, is(equalTo(expected)));
+    }
+
+}
diff --git a/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java b/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java
index 84fae99712dc7..14cfbc602bf47 100644
--- a/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java
+++ b/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java
@@ -45,8 +45,7 @@
 import org.apache.kafka.streams.state.KeyValueStore;
 import org.apache.kafka.streams.state.Stores;
 import org.apache.kafka.streams.state.internals.KeyValueStoreBuilder;
-import org.apache.kafka.streams.test.ConsumerRecordFactory;
-import org.apache.kafka.streams.test.OutputVerifier;
+import org.apache.kafka.streams.test.TestRecord;
 import org.apache.kafka.test.TestUtils;
 import org.junit.After;
 import org.junit.Assert;
@@ -56,6 +55,7 @@
 
 import java.io.File;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -89,20 +89,24 @@ public class TopologyTestDriverTest {
     private final static String SINK_TOPIC_1 = "sink-topic-1";
     private final static String SINK_TOPIC_2 = "sink-topic-2";
 
-    private final ConsumerRecordFactory consumerRecordFactory = new ConsumerRecordFactory<>(
-        new ByteArraySerializer(),
-        new ByteArraySerializer());
-
     private final Headers headers = new RecordHeaders(new Header[]{new RecordHeader("key", "value".getBytes())});
 
     private final byte[] key1 = new byte[0];
     private final byte[] value1 = new byte[0];
     private final long timestamp1 = 42L;
-    private final ConsumerRecord consumerRecord1 = consumerRecordFactory.create(SOURCE_TOPIC_1, key1, value1, headers, timestamp1);
+    private final TestRecord testRecord1 = new TestRecord<>(key1, value1, headers, timestamp1);
 
     private final byte[] key2 = new byte[0];
     private final byte[] value2 = new byte[0];
     private final long timestamp2 = 43L;
+    private final TestRecord testRecord2 = new TestRecord<>(key2, value2, null, timestamp2);
+
+    //Factory and records for testing already deprecated methods
+    @SuppressWarnings("deprecation")
+    private final org.apache.kafka.streams.test.ConsumerRecordFactory consumerRecordFactory = new org.apache.kafka.streams.test.ConsumerRecordFactory<>(
+            new ByteArraySerializer(),
+            new ByteArraySerializer());
+    private final ConsumerRecord consumerRecord1 = consumerRecordFactory.create(SOURCE_TOPIC_1, key1, value1, headers, timestamp1);
     private final ConsumerRecord consumerRecord2 = consumerRecordFactory.create(SOURCE_TOPIC_2, key2, value2, timestamp2);
 
     private TopologyTestDriver testDriver;
@@ -115,9 +119,6 @@ public class TopologyTestDriverTest {
 
     private final StringDeserializer stringDeserializer = new StringDeserializer();
     private final LongDeserializer longDeserializer = new LongDeserializer();
-    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(
-        new StringSerializer(),
-        new LongSerializer());
 
     @Parameterized.Parameters(name = "Eos enabled = {0}")
     public static Collection data() {
@@ -152,6 +153,17 @@ private final static class Record {
             headers = consumerRecord.headers();
         }
 
+        Record(final String newTopic,
+               final TestRecord consumerRecord,
+               final long newOffset) {
+            key = consumerRecord.key();
+            value = consumerRecord.value();
+            timestamp = consumerRecord.timestamp();
+            offset = newOffset;
+            topic = newTopic;
+            headers = consumerRecord.headers();
+        }
+
         Record(final Object key,
                final Object value,
                final Headers headers,
@@ -385,10 +397,22 @@ public void shouldCloseProcessor() {
     @Test
     public void shouldThrowForUnknownTopic() {
         final String unknownTopic = "unknownTopic";
-        final ConsumerRecordFactory consumerRecordFactory = new ConsumerRecordFactory<>(
-            "unknownTopic",
-            new ByteArraySerializer(),
-            new ByteArraySerializer());
+        final byte[] nullValue = null;
+
+        testDriver = new TopologyTestDriver(new Topology(), config);
+        assertThrows(IllegalArgumentException.class, () -> {
+            pipeRecord(unknownTopic, new TestRecord(nullValue));
+        });
+    }
+
+    @Deprecated
+    @Test
+    public void shouldThrowForUnknownTopicDeprecated() {
+        final String unknownTopic = "unknownTopic";
+        final org.apache.kafka.streams.test.ConsumerRecordFactory consumerRecordFactory = new org.apache.kafka.streams.test.ConsumerRecordFactory<>(
+                "unknownTopic",
+                new ByteArraySerializer(),
+                new ByteArraySerializer());
 
         testDriver = new TopologyTestDriver(new Topology(), config);
         try {
@@ -403,8 +427,8 @@ public void shouldThrowForUnknownTopic() {
     public void shouldProcessRecordForTopic() {
         testDriver = new TopologyTestDriver(setupSourceSinkTopology(), config);
 
-        testDriver.pipeInput(consumerRecord1);
-        final ProducerRecord outputRecord = testDriver.readOutput(SINK_TOPIC_1);
+        pipeRecord(SOURCE_TOPIC_1, testRecord1);
+        final ProducerRecord outputRecord = testDriver.readRecord(SINK_TOPIC_1);
 
         assertEquals(key1, outputRecord.key());
         assertEquals(value1, outputRecord.value());
@@ -415,19 +439,26 @@ public void shouldProcessRecordForTopic() {
     public void shouldSetRecordMetadata() {
         testDriver = new TopologyTestDriver(setupSingleProcessorTopology(), config);
 
-        testDriver.pipeInput(consumerRecord1);
+        pipeRecord(SOURCE_TOPIC_1, testRecord1);
 
         final List processedRecords = mockProcessors.get(0).processedRecords;
         assertEquals(1, processedRecords.size());
 
         final Record record = processedRecords.get(0);
-        final Record expectedResult = new Record(consumerRecord1, 0L);
+        final Record expectedResult = new Record(SOURCE_TOPIC_1, testRecord1, 0L);
 
         assertThat(record, equalTo(expectedResult));
     }
 
+    private void pipeRecord(final String topic, final TestRecord record) {
+        testDriver.pipeRecord(topic, record, new ByteArraySerializer(), new ByteArraySerializer(), null);
+    }
+
+
+    @Deprecated
+    //Test not migrated to non-deprecated methods, topic handling not based on record any more
     @Test
-    public void shouldSendRecordViaCorrectSourceTopic() {
+    public void shouldSendRecordViaCorrectSourceTopicDeprecated() {
         testDriver = new TopologyTestDriver(setupMultipleSourceTopology(SOURCE_TOPIC_1, SOURCE_TOPIC_2), config);
 
         final List processedRecords1 = mockProcessors.get(0).processedRecords;
@@ -452,8 +483,9 @@ record = processedRecords2.get(0);
         assertThat(record, equalTo(expectedResult));
     }
 
+    @Deprecated
     @Test
-    public void shouldUseSourceSpecificDeserializers() {
+    public void shouldUseSourceSpecificDeserializersDeprecated() {
         final Topology topology = new Topology();
 
         final String sourceName1 = "source-1";
@@ -482,11 +514,11 @@ public void shouldUseSourceSpecificDeserializers() {
 
         testDriver = new TopologyTestDriver(topology, config);
 
-        final ConsumerRecordFactory source1Factory = new ConsumerRecordFactory<>(
+        final org.apache.kafka.streams.test.ConsumerRecordFactory source1Factory = new org.apache.kafka.streams.test.ConsumerRecordFactory<>(
             SOURCE_TOPIC_1,
             Serdes.Long().serializer(),
             Serdes.String().serializer());
-        final ConsumerRecordFactory source2Factory = new ConsumerRecordFactory<>(
+        final org.apache.kafka.streams.test.ConsumerRecordFactory source2Factory = new org.apache.kafka.streams.test.ConsumerRecordFactory<>(
             SOURCE_TOPIC_2,
             Serdes.Integer().serializer(),
             Serdes.Double().serializer());
@@ -500,16 +532,73 @@ public void shouldUseSourceSpecificDeserializers() {
         final ConsumerRecord consumerRecord2 = source2Factory.create(source2Key, source2Value);
 
         testDriver.pipeInput(consumerRecord1);
-        OutputVerifier.compareKeyValue(
-            testDriver.readOutput(SINK_TOPIC_1, Serdes.Long().deserializer(), Serdes.String().deserializer()),
-            source1Key,
-            source1Value);
+        final ProducerRecord record1 = testDriver.readOutput(SINK_TOPIC_1, Serdes.Long().deserializer(), Serdes.String().deserializer());
+        assertThat(record1.key(), equalTo(source1Key));
+        assertThat(record1.value(), equalTo(source1Value));
 
         testDriver.pipeInput(consumerRecord2);
-        OutputVerifier.compareKeyValue(
-            testDriver.readOutput(SINK_TOPIC_1, Serdes.Integer().deserializer(), Serdes.Double().deserializer()),
-            source2Key,
-            source2Value);
+        final ProducerRecord record2 = testDriver.readOutput(SINK_TOPIC_1, Serdes.Integer().deserializer(), Serdes.Double().deserializer());
+        assertThat(record2.key(), equalTo(source2Key));
+        assertThat(record2.value(), equalTo(source2Value));
+    }
+
+    @Test
+    public void shouldUseSourceSpecificDeserializers() {
+        final Topology topology = new Topology();
+
+        final String sourceName1 = "source-1";
+        final String sourceName2 = "source-2";
+        final String processor = "processor";
+
+        topology.addSource(sourceName1, Serdes.Long().deserializer(), Serdes.String().deserializer(), SOURCE_TOPIC_1);
+        topology.addSource(sourceName2, Serdes.Integer().deserializer(), Serdes.Double().deserializer(), SOURCE_TOPIC_2);
+        topology.addProcessor(processor, new MockProcessorSupplier(), sourceName1, sourceName2);
+        topology.addSink(
+            "sink",
+            SINK_TOPIC_1,
+            (topic, data) -> {
+                if (data instanceof Long) {
+                    return Serdes.Long().serializer().serialize(topic, (Long) data);
+                }
+                return Serdes.Integer().serializer().serialize(topic, (Integer) data);
+            },
+            (topic, data) -> {
+                if (data instanceof String) {
+                    return Serdes.String().serializer().serialize(topic, (String) data);
+                }
+                return Serdes.Double().serializer().serialize(topic, (Double) data);
+            },
+            processor);
+
+        testDriver = new TopologyTestDriver(topology, config);
+
+        final Long source1Key = 42L;
+        final String source1Value = "anyString";
+        final Integer source2Key = 73;
+        final Double source2Value = 3.14;
+
+        final TestRecord consumerRecord1 = new TestRecord<>(source1Key, source1Value);
+        final TestRecord consumerRecord2 = new TestRecord<>(source2Key, source2Value);
+
+        testDriver.pipeRecord(SOURCE_TOPIC_1,
+                consumerRecord1,
+                Serdes.Long().serializer(),
+                Serdes.String().serializer(),
+                null);
+        final TestRecord result1 =
+            testDriver.readRecord(SINK_TOPIC_1, Serdes.Long().deserializer(), Serdes.String().deserializer());
+        assertThat(result1.getKey(), equalTo(source1Key));
+        assertThat(result1.getValue(), equalTo(source1Value));
+
+        testDriver.pipeRecord(SOURCE_TOPIC_2,
+                consumerRecord2,
+                Serdes.Integer().serializer(),
+                Serdes.Double().serializer(),
+                null);
+        final TestRecord result2 =
+            testDriver.readRecord(SINK_TOPIC_1, Serdes.Integer().deserializer(), Serdes.Double().deserializer());
+        assertThat(result2.getKey(), equalTo(source2Key));
+        assertThat(result2.getValue(), equalTo(source2Value));
     }
 
     @Test
@@ -526,36 +615,37 @@ public void shouldUseSinkSpecificSerializers() {
 
         testDriver = new TopologyTestDriver(topology, config);
 
-        final ConsumerRecordFactory source1Factory = new ConsumerRecordFactory<>(
-            SOURCE_TOPIC_1,
-            Serdes.Long().serializer(),
-            Serdes.String().serializer());
-        final ConsumerRecordFactory source2Factory = new ConsumerRecordFactory<>(
-            SOURCE_TOPIC_2,
-            Serdes.Integer().serializer(),
-            Serdes.Double().serializer());
-
         final Long source1Key = 42L;
         final String source1Value = "anyString";
         final Integer source2Key = 73;
         final Double source2Value = 3.14;
 
-        final ConsumerRecord consumerRecord1 = source1Factory.create(source1Key, source1Value);
-        final ConsumerRecord consumerRecord2 = source2Factory.create(source2Key, source2Value);
-
-        testDriver.pipeInput(consumerRecord1);
-        OutputVerifier.compareKeyValue(
-            testDriver.readOutput(SINK_TOPIC_1, Serdes.Long().deserializer(), Serdes.String().deserializer()),
-            source1Key,
-            source1Value);
-
-        testDriver.pipeInput(consumerRecord2);
-        OutputVerifier.compareKeyValue(
-            testDriver.readOutput(SINK_TOPIC_2, Serdes.Integer().deserializer(), Serdes.Double().deserializer()),
-            source2Key,
-            source2Value);
+        final TestRecord consumerRecord1 = new TestRecord<>(source1Key, source1Value);
+        final TestRecord consumerRecord2 = new TestRecord<>(source2Key, source2Value);
+
+        testDriver.pipeRecord(SOURCE_TOPIC_1,
+                consumerRecord1,
+                Serdes.Long().serializer(),
+                Serdes.String().serializer(),
+                null);
+        final TestRecord result1 =
+                testDriver.readRecord(SINK_TOPIC_1, Serdes.Long().deserializer(), Serdes.String().deserializer());
+        assertThat(result1.getKey(), equalTo(source1Key));
+        assertThat(result1.getValue(), equalTo(source1Value));
+
+        testDriver.pipeRecord(SOURCE_TOPIC_2,
+                consumerRecord2,
+                Serdes.Integer().serializer(),
+                Serdes.Double().serializer(),
+                null);
+        final TestRecord result2 =
+                testDriver.readRecord(SINK_TOPIC_2, Serdes.Integer().deserializer(), Serdes.Double().deserializer());
+        assertThat(result2.getKey(), equalTo(source2Key));
+        assertThat(result2.getValue(), equalTo(source2Value));
     }
 
+    @Deprecated
+    //Test not migrated to non-deprecated methods, List processing now in TestInputTopic
     @Test
     public void shouldProcessConsumerRecordList() {
         testDriver = new TopologyTestDriver(setupMultipleSourceTopology(SOURCE_TOPIC_1, SOURCE_TOPIC_2), config);
@@ -585,14 +675,14 @@ record = processedRecords2.get(0);
     public void shouldForwardRecordsFromSubtopologyToSubtopology() {
         testDriver = new TopologyTestDriver(setupTopologyWithTwoSubtopologies(), config);
 
-        testDriver.pipeInput(consumerRecord1);
+        pipeRecord(SOURCE_TOPIC_1, testRecord1);
 
-        ProducerRecord outputRecord = testDriver.readOutput(SINK_TOPIC_1);
+        ProducerRecord outputRecord = testDriver.readRecord(SINK_TOPIC_1);
         assertEquals(key1, outputRecord.key());
         assertEquals(value1, outputRecord.value());
         assertEquals(SINK_TOPIC_1, outputRecord.topic());
 
-        outputRecord = testDriver.readOutput(SINK_TOPIC_2);
+        outputRecord = testDriver.readRecord(SINK_TOPIC_2);
         assertEquals(key1, outputRecord.key());
         assertEquals(value1, outputRecord.value());
         assertEquals(SINK_TOPIC_2, outputRecord.topic());
@@ -606,13 +696,13 @@ public void shouldPopulateGlobalStore() {
         Assert.assertNotNull(globalStore);
         Assert.assertNotNull(testDriver.getAllStateStores().get(SOURCE_TOPIC_1 + "-globalStore"));
 
-        testDriver.pipeInput(consumerRecord1);
+        pipeRecord(SOURCE_TOPIC_1, testRecord1);
 
         final List processedRecords = mockProcessors.get(0).processedRecords;
         assertEquals(1, processedRecords.size());
 
         final Record record = processedRecords.get(0);
-        final Record expectedResult = new Record(consumerRecord1, 0L);
+        final Record expectedResult = new Record(SOURCE_TOPIC_1, testRecord1, 0L);
         assertThat(record, equalTo(expectedResult));
     }
 
@@ -626,47 +716,49 @@ public void shouldPunctuateOnStreamsTime() {
         final List expectedPunctuations = new LinkedList<>();
 
         expectedPunctuations.add(42L);
-        testDriver.pipeInput(consumerRecordFactory.create(SOURCE_TOPIC_1, key1, value1, 42L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 42L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
 
-        testDriver.pipeInput(consumerRecordFactory.create(SOURCE_TOPIC_1, key1, value1, 42L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 42L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
 
         expectedPunctuations.add(51L);
-        testDriver.pipeInput(consumerRecordFactory.create(SOURCE_TOPIC_1, key1, value1, 51L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 51L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
 
-        testDriver.pipeInput(consumerRecordFactory.create(SOURCE_TOPIC_1, key1, value1, 52L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 52L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
 
         expectedPunctuations.add(61L);
-        testDriver.pipeInput(consumerRecordFactory.create(SOURCE_TOPIC_1, key1, value1, 61L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 61L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
 
-        testDriver.pipeInput(consumerRecordFactory.create(SOURCE_TOPIC_1, key1, value1, 65L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 65L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
 
         expectedPunctuations.add(71L);
-        testDriver.pipeInput(consumerRecordFactory.create(SOURCE_TOPIC_1, key1, value1, 71L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 71L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
 
-        testDriver.pipeInput(consumerRecordFactory.create(SOURCE_TOPIC_1, key1, value1, 72L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 72L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
 
         expectedPunctuations.add(95L);
-        testDriver.pipeInput(consumerRecordFactory.create(SOURCE_TOPIC_1, key1, value1, 95L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 95L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
 
         expectedPunctuations.add(101L);
-        testDriver.pipeInput(consumerRecordFactory.create(SOURCE_TOPIC_1, key1, value1, 101L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 101L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
 
-        testDriver.pipeInput(consumerRecordFactory.create(SOURCE_TOPIC_1, key1, value1, 102L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 102L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
     }
 
+    @SuppressWarnings("deprecation")
+    //Testing already deprecatd methods until methods removed
     @Test
-    public void shouldPunctuateOnWallClockTime() {
+    public void shouldPunctuateOnWallClockTimeDeprecated() {
         final MockPunctuator mockPunctuator = new MockPunctuator();
         testDriver = new TopologyTestDriver(
             setupSingleProcessorTopology(10L, PunctuationType.WALL_CLOCK_TIME, mockPunctuator),
@@ -694,6 +786,34 @@ public void shouldPunctuateOnWallClockTime() {
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
     }
 
+    @Test
+    public void shouldPunctuateOnWallClockTime() {
+        final MockPunctuator mockPunctuator = new MockPunctuator();
+        testDriver = new TopologyTestDriver(
+            setupSingleProcessorTopology(10L, PunctuationType.WALL_CLOCK_TIME, mockPunctuator),
+            config, Instant.ofEpochMilli(0L));
+
+        final List expectedPunctuations = new LinkedList<>();
+
+        testDriver.advanceWallClockTime(Duration.ofMillis(5L));
+        assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
+
+        expectedPunctuations.add(14L);
+        testDriver.advanceWallClockTime(Duration.ofMillis(9L));
+        assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
+
+        testDriver.advanceWallClockTime(Duration.ofMillis(1L));
+        assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
+
+        expectedPunctuations.add(35L);
+        testDriver.advanceWallClockTime(Duration.ofMillis(20L));
+        assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
+
+        expectedPunctuations.add(40L);
+        testDriver.advanceWallClockTime(Duration.ofMillis(5L));
+        assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
+    }
+
     @Test
     public void shouldReturnAllStores() {
         final Topology topology = setupSourceSinkTopology();
@@ -1057,62 +1177,72 @@ private void setup(final KeyValueBytesStoreSupplier storeSupplier) {
         store.put("a", 21L);
     }
 
+    private void pipeInput(final String topic, final String key, final Long value, final Long time) {
+        testDriver.pipeRecord(topic, new TestRecord(key, value, null, time),
+                new StringSerializer(), new LongSerializer(), null);
+    }
+    
+    private void compareKeyValue(final TestRecord record, final String key, final Long value) {
+        assertThat(record.getKey(), equalTo(key));
+        assertThat(record.getValue(), equalTo(value));
+    }
+
     @Test
     public void shouldFlushStoreForFirstInput() {
         setup();
-        testDriver.pipeInput(recordFactory.create("input-topic", "a", 1L, 9999L));
-        OutputVerifier.compareKeyValue(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer), "a", 21L);
-        assertNull(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer));
+        pipeInput("input-topic", "a", 1L, 9999L);
+        compareKeyValue(testDriver.readRecord("result-topic", stringDeserializer, longDeserializer), "a", 21L);
+        assertTrue(testDriver.isEmpty("result-topic"));
     }
 
     @Test
     public void shouldNotUpdateStoreForSmallerValue() {
         setup();
-        testDriver.pipeInput(recordFactory.create("input-topic", "a", 1L, 9999L));
+        pipeInput("input-topic", "a", 1L, 9999L);
         assertThat(store.get("a"), equalTo(21L));
-        OutputVerifier.compareKeyValue(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer), "a", 21L);
-        assertNull(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer));
+        compareKeyValue(testDriver.readRecord("result-topic", stringDeserializer, longDeserializer), "a", 21L);
+        assertTrue(testDriver.isEmpty("result-topic"));
     }
 
     @Test
     public void shouldNotUpdateStoreForLargerValue() {
         setup();
-        testDriver.pipeInput(recordFactory.create("input-topic", "a", 42L, 9999L));
+        pipeInput("input-topic", "a", 42L, 9999L);
         assertThat(store.get("a"), equalTo(42L));
-        OutputVerifier.compareKeyValue(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer), "a", 42L);
-        assertNull(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer));
+        compareKeyValue(testDriver.readRecord("result-topic", stringDeserializer, longDeserializer), "a", 42L);
+        assertTrue(testDriver.isEmpty("result-topic"));
     }
 
     @Test
     public void shouldUpdateStoreForNewKey() {
         setup();
-        testDriver.pipeInput(recordFactory.create("input-topic", "b", 21L, 9999L));
+        pipeInput("input-topic", "b", 21L, 9999L);
         assertThat(store.get("b"), equalTo(21L));
-        OutputVerifier.compareKeyValue(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer), "a", 21L);
-        OutputVerifier.compareKeyValue(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer), "b", 21L);
-        assertNull(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer));
+        compareKeyValue(testDriver.readRecord("result-topic", stringDeserializer, longDeserializer), "a", 21L);
+        compareKeyValue(testDriver.readRecord("result-topic", stringDeserializer, longDeserializer), "b", 21L);
+        assertTrue(testDriver.isEmpty("result-topic"));
     }
 
     @Test
     public void shouldPunctuateIfEvenTimeAdvances() {
         setup();
-        testDriver.pipeInput(recordFactory.create("input-topic", "a", 1L, 9999L));
-        OutputVerifier.compareKeyValue(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer), "a", 21L);
+        pipeInput("input-topic", "a", 1L, 9999L);
+        compareKeyValue(testDriver.readRecord("result-topic", stringDeserializer, longDeserializer), "a", 21L);
 
-        testDriver.pipeInput(recordFactory.create("input-topic", "a", 1L, 9999L));
-        assertNull(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer));
+        pipeInput("input-topic", "a", 1L, 9999L);
+        assertTrue(testDriver.isEmpty("result-topic"));
 
-        testDriver.pipeInput(recordFactory.create("input-topic", "a", 1L, 10000L));
-        OutputVerifier.compareKeyValue(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer), "a", 21L);
-        assertNull(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer));
+        pipeInput("input-topic", "a", 1L, 10000L);
+        compareKeyValue(testDriver.readRecord("result-topic", stringDeserializer, longDeserializer), "a", 21L);
+        assertTrue(testDriver.isEmpty("result-topic"));
     }
 
     @Test
     public void shouldPunctuateIfWallClockTimeAdvances() {
         setup();
-        testDriver.advanceWallClockTime(60000);
-        OutputVerifier.compareKeyValue(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer), "a", 21L);
-        assertNull(testDriver.readOutput("result-topic", stringDeserializer, longDeserializer));
+        testDriver.advanceWallClockTime(Duration.ofMillis(60000));
+        compareKeyValue(testDriver.readRecord("result-topic", stringDeserializer, longDeserializer), "a", 21L);
+        assertTrue(testDriver.isEmpty("result-topic"));
     }
 
     private class CustomMaxAggregatorSupplier implements ProcessorSupplier {
@@ -1214,7 +1344,8 @@ public void close() {}
 
         try (final TopologyTestDriver testDriver = new TopologyTestDriver(topology, config)) {
             assertNull(testDriver.getKeyValueStore("storeProcessorStore").get("a"));
-            testDriver.pipeInput(recordFactory.create("input-topic", "a", 1L));
+            testDriver.pipeRecord("input-topic", new TestRecord("a", 1L),
+                    new StringSerializer(), new LongSerializer(), null);
             Assert.assertEquals(1L, testDriver.getKeyValueStore("storeProcessorStore").get("a"));
         }
 
@@ -1238,8 +1369,7 @@ public void shouldFeedStoreFromGlobalKTable() {
             final KeyValueStore globalStore = testDriver.getKeyValueStore("globalStore");
             Assert.assertNotNull(globalStore);
             Assert.assertNotNull(testDriver.getAllStateStores().get("globalStore"));
-            final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
-            testDriver.pipeInput(recordFactory.create("topic", "k1", "value1"));
+            testDriver.pipeRecord("topic", new TestRecord("k1", "value1"), new StringSerializer(), new StringSerializer(), null);
             // we expect to have both in the global store, the one from pipeInput and the one from the producer
             Assert.assertEquals("value1", globalStore.get("k1"));
         }
@@ -1268,29 +1398,29 @@ public void shouldProcessFromSourcesThatMatchMultiplePattern() {
         final  Pattern pattern2Source2 = Pattern.compile("source-topic-[A-Z]");
         final  String consumerTopic2 = "source-topic-Z";
 
-        final ConsumerRecord consumerRecord2 = consumerRecordFactory.create(consumerTopic2, key2, value2, timestamp2);
+        final TestRecord consumerRecord2 = new TestRecord(key2, value2, null, timestamp2);
 
         testDriver = new TopologyTestDriver(setupMultipleSourcesPatternTopology(pattern2Source1, pattern2Source2), config);
 
         final List processedRecords1 = mockProcessors.get(0).processedRecords;
         final List processedRecords2 = mockProcessors.get(1).processedRecords;
 
-        testDriver.pipeInput(consumerRecord1);
+        pipeRecord(SOURCE_TOPIC_1, testRecord1);
 
         assertEquals(1, processedRecords1.size());
         assertEquals(0, processedRecords2.size());
 
         final Record record1 = processedRecords1.get(0);
-        final Record expectedResult1 = new Record(consumerRecord1, 0L);
+        final Record expectedResult1 = new Record(SOURCE_TOPIC_1, testRecord1, 0L);
         assertThat(record1, equalTo(expectedResult1));
 
-        testDriver.pipeInput(consumerRecord2);
+        pipeRecord(consumerTopic2, consumerRecord2);
 
         assertEquals(1, processedRecords1.size());
         assertEquals(1, processedRecords2.size());
 
         final Record record2 = processedRecords2.get(0);
-        final Record expectedResult2 = new Record(consumerRecord2, 0L);
+        final Record expectedResult2 = new Record(consumerTopic2, consumerRecord2, 0L);
         assertThat(record2, equalTo(expectedResult2));
     }
 
@@ -1305,9 +1435,9 @@ public void shouldProcessFromSourceThatMatchPattern() {
         topology.addSink("sink", SINK_TOPIC_1, sourceName);
 
         testDriver = new TopologyTestDriver(topology, config);
-        testDriver.pipeInput(consumerRecord1);
+        pipeRecord(SOURCE_TOPIC_1, testRecord1);
 
-        final ProducerRecord outputRecord = testDriver.readOutput(SINK_TOPIC_1);
+        final ProducerRecord outputRecord = testDriver.readRecord(SINK_TOPIC_1);
         assertEquals(key1, outputRecord.key());
         assertEquals(value1, outputRecord.value());
         assertEquals(SINK_TOPIC_1, outputRecord.topic());
@@ -1325,7 +1455,7 @@ public void shouldThrowPatternNotValidForTopicNameException() {
 
         testDriver = new TopologyTestDriver(topology, config);
         try {
-            testDriver.pipeInput(consumerRecord1);
+            pipeRecord(SOURCE_TOPIC_1, testRecord1);
         } catch (final TopologyException exception) {
             final String str =
                     String.format(
diff --git a/streams/test-utils/src/test/java/org/apache/kafka/streams/test/ConsumerRecordFactoryTest.java b/streams/test-utils/src/test/java/org/apache/kafka/streams/test/ConsumerRecordFactoryTest.java
index 855aa9f086556..4ad16b3c3e535 100644
--- a/streams/test-utils/src/test/java/org/apache/kafka/streams/test/ConsumerRecordFactoryTest.java
+++ b/streams/test-utils/src/test/java/org/apache/kafka/streams/test/ConsumerRecordFactoryTest.java
@@ -30,6 +30,7 @@
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 
+@Deprecated
 public class ConsumerRecordFactoryTest {
     private final StringSerializer stringSerializer = new StringSerializer();
     private final IntegerSerializer integerSerializer = new IntegerSerializer();
diff --git a/streams/test-utils/src/test/java/org/apache/kafka/streams/test/OutputVerifierTest.java b/streams/test-utils/src/test/java/org/apache/kafka/streams/test/OutputVerifierTest.java
index 82201886a7c6f..173f5e351084b 100644
--- a/streams/test-utils/src/test/java/org/apache/kafka/streams/test/OutputVerifierTest.java
+++ b/streams/test-utils/src/test/java/org/apache/kafka/streams/test/OutputVerifierTest.java
@@ -19,6 +19,7 @@
 import org.apache.kafka.clients.producer.ProducerRecord;
 import org.junit.Test;
 
+@Deprecated
 public class OutputVerifierTest {
     private final byte[] key = new byte[0];
     private  final byte[] value = new byte[0];
diff --git a/streams/test-utils/src/test/java/org/apache/kafka/streams/test/TestRecordTest.java b/streams/test-utils/src/test/java/org/apache/kafka/streams/test/TestRecordTest.java
new file mode 100644
index 0000000000000..9977fc54af273
--- /dev/null
+++ b/streams/test-utils/src/test/java/org/apache/kafka/streams/test/TestRecordTest.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.test;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.header.Headers;
+import org.apache.kafka.common.header.internals.RecordHeader;
+import org.apache.kafka.common.header.internals.RecordHeaders;
+import org.apache.kafka.common.record.TimestampType;
+import org.junit.Test;
+
+import java.time.Instant;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.hasProperty;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+
+public class TestRecordTest {
+    private final String key = "testKey";
+    private final int value = 1;
+    private final Headers headers = new RecordHeaders(
+            new Header[]{
+                new RecordHeader("foo", "value".getBytes()),
+                new RecordHeader("bar", (byte[]) null),
+                new RecordHeader("\"A\\u00ea\\u00f1\\u00fcC\"", "value".getBytes())
+            });
+    private final Instant recordTime = Instant.parse("2019-06-01T10:00:00Z");
+    private final long recordMs = recordTime.toEpochMilli();
+
+    @Test
+    public void testFields() {
+        final TestRecord testRecord = new TestRecord<>(key, value, headers, recordTime);
+        assertThat(testRecord.key(), equalTo(key));
+        assertThat(testRecord.value(), equalTo(value));
+        assertThat(testRecord.headers(), equalTo(headers));
+        assertThat(testRecord.timestamp(), equalTo(recordMs));
+
+        assertThat(testRecord.getKey(), equalTo(key));
+        assertThat(testRecord.getValue(), equalTo(value));
+        assertThat(testRecord.getHeaders(), equalTo(headers));
+        assertThat(testRecord.getRecordTime(), equalTo(recordTime));
+    }
+
+    @Test
+    public void testMultiFieldMatcher() {
+        final TestRecord testRecord = new TestRecord<>(key, value, headers, recordTime);
+
+        assertThat(testRecord, allOf(
+                hasProperty("key", equalTo(key)),
+                hasProperty("value", equalTo(value)),
+                hasProperty("headers", equalTo(headers))));
+
+        assertThat(testRecord, allOf(
+                hasProperty("key", equalTo(key)),
+                hasProperty("value", equalTo(value)),
+                hasProperty("headers", equalTo(headers)),
+                hasProperty("recordTime", equalTo(recordTime))));
+
+        assertThat(testRecord, allOf(
+                hasProperty("key", equalTo(key)),
+                hasProperty("value", equalTo(value))));
+    }
+
+
+    @Test
+    public void testEqualsAndHashCode() {
+        final TestRecord testRecord = new TestRecord<>(key, value, headers, recordTime);
+        assertEquals(testRecord, testRecord);
+        assertEquals(testRecord.hashCode(), testRecord.hashCode());
+
+        final TestRecord equalRecord = new TestRecord<>(key, value, headers, recordTime);
+        assertEquals(testRecord, equalRecord);
+        assertEquals(testRecord.hashCode(), equalRecord.hashCode());
+
+        final TestRecord equalRecordMs = new TestRecord<>(key, value, headers, recordMs);
+        assertEquals(testRecord, equalRecordMs);
+        assertEquals(testRecord.hashCode(), equalRecordMs.hashCode());
+
+        final Headers headers2 = new RecordHeaders(
+                new Header[]{
+                    new RecordHeader("foo", "value".getBytes()),
+                    new RecordHeader("bar", (byte[]) null),
+                });
+        final TestRecord headerMismatch = new TestRecord<>(key, value, headers2, recordTime);
+        assertFalse(testRecord.equals(headerMismatch));
+
+        final TestRecord keyMisMatch = new TestRecord<>("test-mismatch", value, headers, recordTime);
+        assertFalse(testRecord.equals(keyMisMatch));
+
+        final TestRecord valueMisMatch = new TestRecord<>(key, 2, headers, recordTime);
+        assertFalse(testRecord.equals(valueMisMatch));
+
+        final TestRecord timeMisMatch = new TestRecord<>(key, value, headers, recordTime.plusMillis(1));
+        assertFalse(testRecord.equals(timeMisMatch));
+
+        final TestRecord nullFieldsRecord = new TestRecord<>(null, null, null, (Instant) null);
+        assertEquals(nullFieldsRecord, nullFieldsRecord);
+        assertEquals(nullFieldsRecord.hashCode(), nullFieldsRecord.hashCode());
+    }
+
+    @Test
+    public void testPartialConstructorEquals() {
+        final TestRecord record1 = new TestRecord<>(value);
+        assertThat(record1, equalTo(new TestRecord(null, value, null, (Instant) null)));
+
+        final TestRecord record2 = new TestRecord<>(key, value);
+        assertThat(record2, equalTo(new TestRecord(key, value, null, (Instant) null)));
+
+        final TestRecord record3 = new TestRecord<>(key, value, headers);
+        assertThat(record3, equalTo(new TestRecord(key, value, headers, (Long) null)));
+
+        final TestRecord record4 = new TestRecord<>(key, value, recordTime);
+        assertThat(record4, equalTo(new TestRecord(key, value, null, recordMs)));
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testInvalidRecords() {
+        new TestRecord(key, value, headers,  -1L);
+    }
+
+    @Test
+    public void testToString() {
+        final TestRecord testRecord = new TestRecord<>(key, value, headers, recordTime);
+        assertThat(testRecord.toString(), equalTo("TestRecord[key=testKey, value=1, "
+                + "headers=RecordHeaders(headers = [RecordHeader(key = foo, value = [118, 97, 108, 117, 101]), "
+                + "RecordHeader(key = bar, value = null), RecordHeader(key = \"A\\u00ea\\u00f1\\u00fcC\", value = [118, 97, 108, 117, 101])], isReadOnly = false), "
+                + "recordTime=2019-06-01T10:00:00Z]"));
+    }
+
+    @Test
+    public void testConsumerRecord() {
+        final String topicName = "topic";
+        final ConsumerRecord consumerRecord = new ConsumerRecord<>(topicName, 1, 0, recordMs, TimestampType.CREATE_TIME, 0L, 0, 0, key, value, headers);
+        final TestRecord testRecord = new TestRecord<>(consumerRecord);
+        final TestRecord expectedRecord = new TestRecord<>(key, value, headers, recordTime);
+        assertEquals(expectedRecord, testRecord);
+    }
+
+    @Test
+    public void testProducerRecord() {
+        final String topicName = "topic";
+        final ProducerRecord producerRecord = new ProducerRecord<>(topicName, 1, recordMs, key, value, headers);
+        final TestRecord testRecord = new TestRecord<>(producerRecord);
+        final TestRecord expectedRecord = new TestRecord<>(key, value, headers, recordTime);
+        assertEquals(expectedRecord, testRecord);
+        assertNotEquals(expectedRecord, producerRecord);
+    }
+}

From debf225ff4c2700d982c0054068859f5afbdd816 Mon Sep 17 00:00:00 2001
From: Manikumar Reddy 
Date: Mon, 7 Oct 2019 13:36:20 +0530
Subject: [PATCH 0694/1071] MINOR: Update documentation.html to refer to 2.4
 (#7454)

---
 docs/documentation.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/documentation.html b/docs/documentation.html
index 8d790c1660ba2..8decc9bea5cf3 100644
--- a/docs/documentation.html
+++ b/docs/documentation.html
@@ -26,8 +26,8 @@
 	
                
 		
     
                Documentation
                
-    
                Kafka 2.3 Documentation
                
-    Prior releases:                 0.7.x, 0.8.0, 0.8.1.X, 0.8.2.X, 0.9.0.X, 0.10.0.X, 0.10.1.X, 0.10.2.X, 0.11.0.X, 1.0.X, 1.1.X, 2.0.X, 2.1.X, 2.2.X.
+    
                Kafka 2.4 Documentation
                
+    Prior releases: 0.7.x, 0.8.0, 0.8.1.X, 0.8.2.X, 0.9.0.X, 0.10.0.X, 0.10.1.X, 0.10.2.X, 0.11.0.X, 1.0.X, 1.1.X, 2.0.X, 2.1.X, 2.2.X, 2.3.X.
 
     
 

From 4ac892ca783acab8e574b9b24d17e767eedb3d5f Mon Sep 17 00:00:00 2001
From: Ryanne Dolan 
Date: Mon, 7 Oct 2019 13:57:54 +0530
Subject: [PATCH 0695/1071] KAFKA-7500: MirrorMaker 2.0 (KIP-382)

Implementation of [KIP-382 "MirrorMaker 2.0"](https://cwiki.apache.org/confluence/display/KAFKA/KIP-382%3A+MirrorMaker+2.0)

Author: Ryanne Dolan 
Author: Arun Mathew 
Author: In Park 
Author: Andre Price 
Author: christian.hagel@rio.cloud 

Reviewers: Eno Thereska , William Hammond , Viktor Somogyi , Jakub Korzeniowski, Tim Carey-Smith, Kamal Chandraprakash , Arun Mathew, Jeremy-l-ford, vpernin, Oleg Kasian , Mickael Maison , Qihong Chen, Sriharsha Chintalapani , Jun Rao , Randall Hauch , Manikumar Reddy , Ismael Juma 

Closes #6295 from ryannedolan/KIP-382
---
 bin/connect-mirror-maker.sh                   |  45 ++
 bin/kafka-run-class.sh                        |   2 +-
 build.gradle                                  |  86 ++-
 checkstyle/import-control.xml                 |  15 +
 config/connect-mirror-maker.properties        |  37 ++
 .../kafka/connect/source/SourceTask.java      |  38 +-
 .../kafka/connect/mirror/Checkpoint.java      | 184 ++++++
 .../mirror/DefaultReplicationPolicy.java      |  73 +++
 .../kafka/connect/mirror/Heartbeat.java       | 145 +++++
 .../kafka/connect/mirror/MirrorClient.java    | 243 +++++++
 .../connect/mirror/MirrorClientConfig.java    | 135 ++++
 .../connect/mirror/RemoteClusterUtils.java    |  97 +++
 .../connect/mirror/ReplicationPolicy.java     |  60 ++
 .../kafka/connect/mirror/SourceAndTarget.java |  52 ++
 .../connect/mirror/MirrorClientTest.java      | 163 +++++
 connect/mirror/README.md                      | 222 +++++++
 .../connect/mirror/ConfigPropertyFilter.java  |  37 ++
 .../mirror/DefaultConfigPropertyFilter.java   |  77 +++
 .../connect/mirror/DefaultGroupFilter.java    |  91 +++
 .../connect/mirror/DefaultTopicFilter.java    |  91 +++
 .../kafka/connect/mirror/GroupFilter.java     |  37 ++
 .../mirror/MirrorCheckpointConnector.java     | 156 +++++
 .../connect/mirror/MirrorCheckpointTask.java  | 193 ++++++
 .../connect/mirror/MirrorConnectorConfig.java | 601 ++++++++++++++++++
 .../mirror/MirrorHeartbeatConnector.java      |  71 +++
 .../connect/mirror/MirrorHeartbeatTask.java   |  84 +++
 .../kafka/connect/mirror/MirrorMaker.java     | 309 +++++++++
 .../connect/mirror/MirrorMakerConfig.java     | 255 ++++++++
 .../kafka/connect/mirror/MirrorMetrics.java   | 208 ++++++
 .../connect/mirror/MirrorSourceConnector.java | 390 ++++++++++++
 .../connect/mirror/MirrorSourceTask.java      | 293 +++++++++
 .../connect/mirror/MirrorTaskConfig.java      |  75 +++
 .../kafka/connect/mirror/MirrorUtils.java     | 116 ++++
 .../kafka/connect/mirror/OffsetSync.java      | 120 ++++
 .../kafka/connect/mirror/OffsetSyncStore.java |  84 +++
 .../kafka/connect/mirror/Scheduler.java       | 115 ++++
 .../kafka/connect/mirror/TopicFilter.java     |  37 ++
 .../kafka/connect/mirror/CheckpointTest.java  |  40 ++
 .../kafka/connect/mirror/HeartbeatTest.java   |  38 ++
 .../mirror/MirrorCheckpointTaskTest.java      |  67 ++
 .../mirror/MirrorConnectorConfigTest.java     | 109 ++++
 .../MirrorConnectorsIntegrationTest.java      | 302 +++++++++
 .../connect/mirror/MirrorMakerConfigTest.java | 234 +++++++
 .../mirror/MirrorSourceConnectorTest.java     | 115 ++++
 .../connect/mirror/MirrorSourceTaskTest.java  |  99 +++
 .../connect/mirror/OffsetSyncStoreTest.java   |  67 ++
 .../kafka/connect/mirror/OffsetSyncTest.java  |  39 ++
 .../src/test/resources/log4j.properties       |  34 +
 .../connect/runtime/WorkerSourceTask.java     |   8 +-
 .../distributed/DistributedHerder.java        |   2 +-
 .../runtime/isolation/PluginUtils.java        |   2 +
 .../MonitorableSourceConnector.java           |   3 +-
 .../connect/runtime/WorkerSourceTaskTest.java |   6 +-
 .../runtime/isolation/PluginUtilsTest.java    |   6 +
 settings.gradle                               |   2 +
 55 files changed, 6197 insertions(+), 13 deletions(-)
 create mode 100755 bin/connect-mirror-maker.sh
 create mode 100644 config/connect-mirror-maker.properties
 create mode 100644 connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/Checkpoint.java
 create mode 100644 connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/DefaultReplicationPolicy.java
 create mode 100644 connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/Heartbeat.java
 create mode 100644 connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/MirrorClient.java
 create mode 100644 connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/MirrorClientConfig.java
 create mode 100644 connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/RemoteClusterUtils.java
 create mode 100644 connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/ReplicationPolicy.java
 create mode 100644 connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/SourceAndTarget.java
 create mode 100644 connect/mirror-client/src/test/java/org/apache/kafka/connect/mirror/MirrorClientTest.java
 create mode 100644 connect/mirror/README.md
 create mode 100644 connect/mirror/src/main/java/org/apache/kafka/connect/mirror/ConfigPropertyFilter.java
 create mode 100644 connect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultConfigPropertyFilter.java
 create mode 100644 connect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultGroupFilter.java
 create mode 100644 connect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultTopicFilter.java
 create mode 100644 connect/mirror/src/main/java/org/apache/kafka/connect/mirror/GroupFilter.java
 create mode 100644 connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorCheckpointConnector.java
 create mode 100644 connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorCheckpointTask.java
 create mode 100644 connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorConnectorConfig.java
 create mode 100644 connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorHeartbeatConnector.java
 create mode 100644 connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorHeartbeatTask.java
 create mode 100644 connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorMaker.java
 create mode 100644 connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorMakerConfig.java
 create mode 100644 connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorMetrics.java
 create mode 100644 connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceConnector.java
 create mode 100644 connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceTask.java
 create mode 100644 connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorTaskConfig.java
 create mode 100644 connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorUtils.java
 create mode 100644 connect/mirror/src/main/java/org/apache/kafka/connect/mirror/OffsetSync.java
 create mode 100644 connect/mirror/src/main/java/org/apache/kafka/connect/mirror/OffsetSyncStore.java
 create mode 100644 connect/mirror/src/main/java/org/apache/kafka/connect/mirror/Scheduler.java
 create mode 100644 connect/mirror/src/main/java/org/apache/kafka/connect/mirror/TopicFilter.java
 create mode 100644 connect/mirror/src/test/java/org/apache/kafka/connect/mirror/CheckpointTest.java
 create mode 100644 connect/mirror/src/test/java/org/apache/kafka/connect/mirror/HeartbeatTest.java
 create mode 100644 connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorCheckpointTaskTest.java
 create mode 100644 connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorConfigTest.java
 create mode 100644 connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java
 create mode 100644 connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorMakerConfigTest.java
 create mode 100644 connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConnectorTest.java
 create mode 100644 connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceTaskTest.java
 create mode 100644 connect/mirror/src/test/java/org/apache/kafka/connect/mirror/OffsetSyncStoreTest.java
 create mode 100644 connect/mirror/src/test/java/org/apache/kafka/connect/mirror/OffsetSyncTest.java
 create mode 100644 connect/mirror/src/test/resources/log4j.properties

diff --git a/bin/connect-mirror-maker.sh b/bin/connect-mirror-maker.sh
new file mode 100755
index 0000000000000..a2c040dad297e
--- /dev/null
+++ b/bin/connect-mirror-maker.sh
@@ -0,0 +1,45 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ];
+then
+        echo "USAGE: $0 [-daemon] mm2.properties"
+        exit 1
+fi
+
+base_dir=$(dirname $0)
+
+if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then
+    export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/connect-log4j.properties"
+fi
+
+if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
+  export KAFKA_HEAP_OPTS="-Xms256M -Xmx2G"
+fi
+
+EXTRA_ARGS=${EXTRA_ARGS-'-name mirrorMaker'}
+
+COMMAND=$1
+case $COMMAND in
+  -daemon)
+    EXTRA_ARGS="-daemon "$EXTRA_ARGS
+    shift
+    ;;
+  *)
+    ;;
+esac
+
+exec $(dirname $0)/kafka-run-class.sh $EXTRA_ARGS org.apache.kafka.connect.mirror.MirrorMaker "$@"
diff --git a/bin/kafka-run-class.sh b/bin/kafka-run-class.sh
index 1221860cb1116..018e52f8388fa 100755
--- a/bin/kafka-run-class.sh
+++ b/bin/kafka-run-class.sh
@@ -139,7 +139,7 @@ do
   CLASSPATH="$CLASSPATH:$dir/*"
 done
 
-for cc_pkg in "api" "transforms" "runtime" "file" "json" "tools" "basic-auth-extension"
+for cc_pkg in "api" "transforms" "runtime" "file" "mirror" "mirror-client" "json" "tools" "basic-auth-extension"
 do
   for file in "$base_dir"/connect/${cc_pkg}/build/libs/connect-${cc_pkg}*.jar;
   do
diff --git a/build.gradle b/build.gradle
index 1c567e65e88dc..65e6a69dbca5f 100644
--- a/build.gradle
+++ b/build.gradle
@@ -617,7 +617,9 @@ def connectPkgs = [
     'connect:file',
     'connect:json',
     'connect:runtime',
-    'connect:transforms'
+    'connect:transforms',
+    'connect:mirror',
+    'connect:mirror-client'
 ]
 
 def pkgs = [
@@ -860,6 +862,10 @@ project(':core') {
     from(project(':connect:file').configurations.runtime) { into("libs/") }
     from(project(':connect:basic-auth-extension').jar) { into("libs/") }
     from(project(':connect:basic-auth-extension').configurations.runtime) { into("libs/") }
+    from(project(':connect:mirror').jar) { into("libs/") }
+    from(project(':connect:mirror').configurations.runtime) { into("libs/") }
+    from(project(':connect:mirror-client').jar) { into("libs/") }
+    from(project(':connect:mirror-client').configurations.runtime) { into("libs/") }
     from(project(':streams').jar) { into("libs/") }
     from(project(':streams').configurations.runtime) { into("libs/") }
     from(project(':streams:streams-scala').jar) { into("libs/") }
@@ -1817,6 +1823,84 @@ project(':connect:basic-auth-extension') {
   }
 }
 
+project(':connect:mirror') {
+  archivesBaseName = "connect-mirror"
+
+  dependencies {
+    compile project(':connect:api')
+    compile project(':connect:runtime')
+    compile project(':connect:mirror-client')
+    compile project(':clients')
+    compile libs.argparse4j
+    compile libs.slf4jApi
+
+    testCompile libs.junit
+    testCompile project(':clients').sourceSets.test.output
+    testCompile project(':connect:runtime').sourceSets.test.output
+    testCompile project(':core')
+    testCompile project(':core').sourceSets.test.output
+
+    testRuntime project(':connect:runtime')
+    testRuntime libs.slf4jlog4j
+  }
+
+  javadoc {
+    enabled = false
+  }
+
+  tasks.create(name: "copyDependantLibs", type: Copy) {
+    from (configurations.testRuntime) {
+      include('slf4j-log4j12*')
+      include('log4j*jar')
+    }
+    from (configurations.runtime) {
+      exclude('kafka-clients*')
+      exclude('connect-*')
+    }
+    into "$buildDir/dependant-libs"
+    duplicatesStrategy 'exclude'
+  }
+
+  jar {
+    dependsOn copyDependantLibs
+  }
+}
+
+project(':connect:mirror-client') {
+  archivesBaseName = "connect-mirror-client"
+
+  dependencies {
+    compile project(':clients')
+    compile libs.slf4jApi
+
+    testCompile libs.junit
+    testCompile project(':clients').sourceSets.test.output
+
+    testRuntime libs.slf4jlog4j
+  }
+
+  javadoc {
+    enabled = true
+  }
+
+  tasks.create(name: "copyDependantLibs", type: Copy) {
+    from (configurations.testRuntime) {
+      include('slf4j-log4j12*')
+      include('log4j*jar')
+    }
+    from (configurations.runtime) {
+      exclude('kafka-clients*')
+      exclude('connect-*')
+    }
+    into "$buildDir/dependant-libs"
+    duplicatesStrategy 'exclude'
+  }
+
+  jar {
+    dependsOn copyDependantLibs
+  }
+}
+
 task aggregatedJavadoc(type: Javadoc) {
   def projectsWithJavadoc = subprojects.findAll { it.javadoc.enabled }
   source = projectsWithJavadoc.collect { it.sourceSets.main.allJava }
diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml
index 5f27dde64f091..17e6f57bcd634 100644
--- a/checkstyle/import-control.xml
+++ b/checkstyle/import-control.xml
@@ -342,6 +342,21 @@
       
     
 
+    
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+    
+
     
       
       
diff --git a/config/connect-mirror-maker.properties b/config/connect-mirror-maker.properties
new file mode 100644
index 0000000000000..16c1b791d3d5f
--- /dev/null
+++ b/config/connect-mirror-maker.properties
@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under A or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# 
+#    http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# see org.apache.kafka.clients.consumer.ConsumerConfig for more details
+
+# Sample MirrorMaker 2.0 top-level configuration file
+# Run with ./bin/connect-mirror-maker.sh connect-mirror-maker.properties 
+
+# specify any number of cluster aliases
+clusters = A, B, C
+
+# connection information for each cluster
+A.bootstrap.servers = A_host1:9092, A_host2:9092, A_host3:9092
+B.bootstrap.servers = B_host1:9092, B_host2:9092, B_host3:9092
+C.bootstrap.servers = C_host1:9092, C_host2:9092, C_host3:9092
+
+# enable and configure individual replication flows
+A->B.enabled = true
+A->B.topics = foo-.*
+B->C.enabled = true
+B->C.topics = bar-.*
+
+# customize as needed
+# replication.policy.separator = _
+# sync.topic.acls.enabled = false
+# emit.heartbeats.interval.seconds = 5
diff --git a/connect/api/src/main/java/org/apache/kafka/connect/source/SourceTask.java b/connect/api/src/main/java/org/apache/kafka/connect/source/SourceTask.java
index 8767a62045e4e..4dea6ce80bdbc 100644
--- a/connect/api/src/main/java/org/apache/kafka/connect/source/SourceTask.java
+++ b/connect/api/src/main/java/org/apache/kafka/connect/source/SourceTask.java
@@ -17,6 +17,7 @@
 package org.apache.kafka.connect.source;
 
 import org.apache.kafka.connect.connector.Task;
+import org.apache.kafka.clients.producer.RecordMetadata;
 
 import java.util.List;
 import java.util.Map;
@@ -88,7 +89,13 @@ public void commit() throws InterruptedException {
 
     /**
      * 
                
-     * Commit an individual {@link SourceRecord} when the callback from the producer client is received, or if a record is filtered by a transformation.
+     * Commit an individual {@link SourceRecord} when the callback from the producer client is received. This method is
+     * also called when a record is filtered by a transformation, and thus will never be ACK'd by a broker.
+     * 
                
+     * 
                
+     * This is an alias for {@link commitRecord(SourceRecord, RecordMetadata)} for backwards compatibility. The default
+     * implementation of {@link commitRecord(SourceRecord, RecordMetadata)} just calls this method. It is not necessary
+     * to override both methods.
      * 
                
      * 
                
      * SourceTasks are not required to implement this functionality; Kafka Connect will record offsets
@@ -96,10 +103,37 @@ public void commit() throws InterruptedException {
      * in their own system.
      * 
                
      *
-     * @param record {@link SourceRecord} that was successfully sent via the producer.
+     * @param record {@link SourceRecord} that was successfully sent via the producer or filtered by a transformation
      * @throws InterruptedException
+     * @see commitRecord(SourceRecord, RecordMetadata)
      */
     public void commitRecord(SourceRecord record) throws InterruptedException {
         // This space intentionally left blank.
     }
+
+    /**
+     * 
                
+     * Commit an individual {@link SourceRecord} when the callback from the producer client is received. This method is
+     * also called when a record is filtered by a transformation, and thus will never be ACK'd by a broker. In this case
+     * {@code metadata} will be null.
+     * 
                
+     * 
                
+     * SourceTasks are not required to implement this functionality; Kafka Connect will record offsets
+     * automatically. This hook is provided for systems that also need to store offsets internally
+     * in their own system.
+     * 
                
+     * 
                
+     * The default implementation just calls @{link commitRecord(SourceRecord)}, which is a nop by default. It is
+     * not necessary to implement both methods.
+     * 
                
+     *
+     * @param record {@link SourceRecord} that was successfully sent via the producer or filtered by a transformation
+     * @param metadata {@link RecordMetadata} record metadata returned from the broker, or null if the record was filtered
+     * @throws InterruptedException
+     */
+    public void commitRecord(SourceRecord record, RecordMetadata metadata)
+            throws InterruptedException {
+        // by default, just call other method for backwards compatability
+        commitRecord(record);
+    }
 }
diff --git a/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/Checkpoint.java b/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/Checkpoint.java
new file mode 100644
index 0000000000000..74db7461ef0b6
--- /dev/null
+++ b/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/Checkpoint.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.protocol.types.Field;
+import org.apache.kafka.common.protocol.types.Schema;
+import org.apache.kafka.common.protocol.types.Struct;
+import org.apache.kafka.common.protocol.types.Type;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+
+import java.util.Map;
+import java.util.HashMap;
+import java.nio.ByteBuffer;
+
+/** Checkpoint records emitted from MirrorCheckpointConnector. Encodes remote consumer group state. */
+public class Checkpoint {
+    public static final String TOPIC_KEY = "topic";
+    public static final String PARTITION_KEY = "partition";
+    public static final String CONSUMER_GROUP_ID_KEY = "group";
+    public static final String UPSTREAM_OFFSET_KEY = "upstreamOffset";
+    public static final String DOWNSTREAM_OFFSET_KEY = "offset";
+    public static final String METADATA_KEY = "metadata";
+    public static final String VERSION_KEY = "version";
+    public static final short VERSION = 0;
+
+    public static final Schema VALUE_SCHEMA_V0 = new Schema(
+            new Field(UPSTREAM_OFFSET_KEY, Type.INT64),
+            new Field(DOWNSTREAM_OFFSET_KEY, Type.INT64),
+            new Field(METADATA_KEY, Type.STRING));
+
+    public static final Schema KEY_SCHEMA = new Schema(
+            new Field(CONSUMER_GROUP_ID_KEY, Type.STRING),
+            new Field(TOPIC_KEY, Type.STRING),
+            new Field(PARTITION_KEY, Type.INT32));
+
+    public static final Schema HEADER_SCHEMA = new Schema(
+            new Field(VERSION_KEY, Type.INT16));
+
+    private String consumerGroupId;
+    private TopicPartition topicPartition;
+    private long upstreamOffset;
+    private long downstreamOffset;
+    private String metadata;
+
+    public Checkpoint(String consumerGroupId, TopicPartition topicPartition, long upstreamOffset,
+            long downstreamOffset, String metadata) {
+        this.consumerGroupId = consumerGroupId;
+        this.topicPartition = topicPartition;
+        this.upstreamOffset = upstreamOffset;
+        this.downstreamOffset = downstreamOffset;
+        this.metadata = metadata;
+    }
+
+    public String consumerGroupId() {
+        return consumerGroupId;
+    }
+
+    public TopicPartition topicPartition() {
+        return topicPartition;
+    }
+
+    public long upstreamOffset() {
+        return upstreamOffset;
+    }
+
+    public long downstreamOffset() {
+        return downstreamOffset;
+    }
+
+    public String metadata() {
+        return metadata;
+    }
+
+    public OffsetAndMetadata offsetAndMetadata() {
+        return new OffsetAndMetadata(downstreamOffset, metadata);
+    }
+
+    @Override
+    public String toString() {
+        return String.format("Checkpoint{consumerGroupId=%s, topicPartition=%s, "
+            + "upstreamOffset=%d, downstreamOffset=%d, metatadata=%s}",
+            consumerGroupId, topicPartition, upstreamOffset, downstreamOffset, metadata);
+    }
+
+    ByteBuffer serializeValue(short version) {
+        Struct header = headerStruct(version);
+        Schema valueSchema = valueSchema(version);
+        Struct valueStruct = valueStruct(valueSchema);
+        ByteBuffer buffer = ByteBuffer.allocate(HEADER_SCHEMA.sizeOf(header) + valueSchema.sizeOf(valueStruct));
+        HEADER_SCHEMA.write(buffer, header);
+        valueSchema.write(buffer, valueStruct);
+        buffer.flip();
+        return buffer;
+    }
+
+    ByteBuffer serializeKey() {
+        Struct struct = keyStruct();
+        ByteBuffer buffer = ByteBuffer.allocate(KEY_SCHEMA.sizeOf(struct));
+        KEY_SCHEMA.write(buffer, struct);
+        buffer.flip();
+        return buffer;
+    }
+
+    public static Checkpoint deserializeRecord(ConsumerRecord record) {
+        ByteBuffer value = ByteBuffer.wrap(record.value());
+        Struct header = HEADER_SCHEMA.read(value);
+        short version = header.getShort(VERSION_KEY);
+        Schema valueSchema = valueSchema(version);
+        Struct valueStruct = valueSchema.read(value);
+        long upstreamOffset = valueStruct.getLong(UPSTREAM_OFFSET_KEY);
+        long downstreamOffset = valueStruct.getLong(DOWNSTREAM_OFFSET_KEY);
+        String metadata = valueStruct.getString(METADATA_KEY);
+        Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(record.key()));
+        String group = keyStruct.getString(CONSUMER_GROUP_ID_KEY);
+        String topic = keyStruct.getString(TOPIC_KEY);
+        int partition = keyStruct.getInt(PARTITION_KEY);
+        return new Checkpoint(group, new TopicPartition(topic, partition), upstreamOffset,
+            downstreamOffset, metadata);
+    }
+
+    private static Schema valueSchema(short version) {
+        assert version == 0;
+        return VALUE_SCHEMA_V0;
+    }
+
+    private Struct valueStruct(Schema schema) {
+        Struct struct = new Struct(schema);
+        struct.set(UPSTREAM_OFFSET_KEY, upstreamOffset);
+        struct.set(DOWNSTREAM_OFFSET_KEY, downstreamOffset);
+        struct.set(METADATA_KEY, metadata);
+        return struct;
+    }
+
+    private Struct keyStruct() {
+        Struct struct = new Struct(KEY_SCHEMA);
+        struct.set(CONSUMER_GROUP_ID_KEY, consumerGroupId);
+        struct.set(TOPIC_KEY, topicPartition.topic());
+        struct.set(PARTITION_KEY, topicPartition.partition());
+        return struct;
+    }
+
+    private Struct headerStruct(short version) {
+        Struct struct = new Struct(HEADER_SCHEMA);
+        struct.set(VERSION_KEY, version);
+        return struct;
+    }
+
+    Map connectPartition() {
+        Map partition = new HashMap<>();
+        partition.put(CONSUMER_GROUP_ID_KEY, consumerGroupId);
+        partition.put(TOPIC_KEY, topicPartition.topic());
+        partition.put(PARTITION_KEY, topicPartition.partition());
+        return partition;
+    }
+
+    static String unwrapGroup(Map connectPartition) {
+        return connectPartition.get(CONSUMER_GROUP_ID_KEY).toString();
+    }
+
+    byte[] recordKey() {
+        return serializeKey().array();
+    }
+
+    byte[] recordValue() {
+        return serializeValue(VERSION).array();
+    }
+};
+
diff --git a/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/DefaultReplicationPolicy.java b/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/DefaultReplicationPolicy.java
new file mode 100644
index 0000000000000..30d75348a8a4e
--- /dev/null
+++ b/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/DefaultReplicationPolicy.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.Configurable;
+
+import java.util.Map;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Defines remote topics like "us-west.topic1". The separator is customizable and defaults to a period. */
+public class DefaultReplicationPolicy implements ReplicationPolicy, Configurable {
+    
+    private static final Logger log = LoggerFactory.getLogger(DefaultReplicationPolicy.class);
+
+    // In order to work with various metrics stores, we allow custom separators.
+    public static final String SEPARATOR_CONFIG = MirrorClientConfig.REPLICATION_POLICY_SEPARATOR;
+    public static final String SEPARATOR_DEFAULT = ".";
+
+    private String separator = SEPARATOR_DEFAULT;
+    private Pattern separatorPattern = Pattern.compile(Pattern.quote(SEPARATOR_DEFAULT));
+
+    @Override
+    public void configure(Map props) {
+        if (props.containsKey(SEPARATOR_CONFIG)) {
+            separator = (String) props.get(SEPARATOR_CONFIG);
+            log.info("Using custom remote topic separator: '{}'", separator);
+            separatorPattern = Pattern.compile(Pattern.quote(separator));
+        }
+    }
+
+    @Override
+    public String formatRemoteTopic(String sourceClusterAlias, String topic) {
+        return sourceClusterAlias + separator + topic;
+    }
+
+    @Override
+    public String topicSource(String topic) {
+        String[] parts = separatorPattern.split(topic);
+        if (parts.length < 2) {
+            // this is not a remote topic
+            return null;
+        } else {
+            return parts[0];
+        }
+    }
+
+    @Override
+    public String upstreamTopic(String topic) {
+        String source = topicSource(topic);
+        if (source == null) {
+            return null;
+        } else {
+            return topic.substring(source.length() + separator.length());
+        }
+    }
+}
diff --git a/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/Heartbeat.java b/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/Heartbeat.java
new file mode 100644
index 0000000000000..a34ce9efb326e
--- /dev/null
+++ b/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/Heartbeat.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.protocol.types.Field;
+import org.apache.kafka.common.protocol.types.Schema;
+import org.apache.kafka.common.protocol.types.Struct;
+import org.apache.kafka.common.protocol.types.Type;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+
+import java.util.Map;
+import java.util.HashMap;
+import java.nio.ByteBuffer;
+
+/** Heartbeat message sent from MirrorHeartbeatTask to target cluster. Heartbeats are always replicated. */
+public class Heartbeat {
+    public static final String SOURCE_CLUSTER_ALIAS_KEY = "sourceClusterAlias";
+    public static final String TARGET_CLUSTER_ALIAS_KEY = "targetClusterAlias";
+    public static final String TIMESTAMP_KEY = "timestamp";
+    public static final String VERSION_KEY = "version";
+    public static final short VERSION = 0;
+
+    public static final Schema VALUE_SCHEMA_V0 = new Schema(
+            new Field(TIMESTAMP_KEY, Type.INT64));
+
+    public static final Schema KEY_SCHEMA = new Schema(
+            new Field(SOURCE_CLUSTER_ALIAS_KEY, Type.STRING),
+            new Field(TARGET_CLUSTER_ALIAS_KEY, Type.STRING));
+
+    public static final Schema HEADER_SCHEMA = new Schema(
+            new Field(VERSION_KEY, Type.INT16));
+
+    private String sourceClusterAlias;
+    private String targetClusterAlias;
+    private long timestamp;
+
+    public Heartbeat(String sourceClusterAlias, String targetClusterAlias, long timestamp) {
+        this.sourceClusterAlias = sourceClusterAlias;
+        this.targetClusterAlias = targetClusterAlias;
+        this.timestamp = timestamp;
+    }
+
+    public String sourceClusterAlias() {
+        return sourceClusterAlias;
+    }
+
+    public String targetClusterAlias() {
+        return targetClusterAlias;
+    }
+
+    public long timestamp() {
+        return timestamp;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("Heartbeat{sourceClusterAlias=%s, targetClusterAlias=%s, timestamp=%d}",
+            sourceClusterAlias, targetClusterAlias, timestamp);
+    }
+
+    ByteBuffer serializeValue(short version) {
+        Schema valueSchema = valueSchema(version);
+        Struct header = headerStruct(version);
+        Struct value = valueStruct(valueSchema);
+        ByteBuffer buffer = ByteBuffer.allocate(HEADER_SCHEMA.sizeOf(header) + valueSchema.sizeOf(value));
+        HEADER_SCHEMA.write(buffer, header);
+        valueSchema.write(buffer, value);
+        buffer.flip();
+        return buffer;
+    }
+
+    ByteBuffer serializeKey() {
+        Struct struct = keyStruct();
+        ByteBuffer buffer = ByteBuffer.allocate(KEY_SCHEMA.sizeOf(struct));
+        KEY_SCHEMA.write(buffer, struct);
+        buffer.flip();
+        return buffer;
+    }
+
+    public static Heartbeat deserializeRecord(ConsumerRecord record) {
+        ByteBuffer value = ByteBuffer.wrap(record.value());
+        Struct headerStruct = HEADER_SCHEMA.read(value);
+        short version = headerStruct.getShort(VERSION_KEY);
+        Struct valueStruct = valueSchema(version).read(value);
+        long timestamp = valueStruct.getLong(TIMESTAMP_KEY);
+        Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(record.key()));
+        String sourceClusterAlias = keyStruct.getString(SOURCE_CLUSTER_ALIAS_KEY);
+        String targetClusterAlias = keyStruct.getString(TARGET_CLUSTER_ALIAS_KEY); 
+        return new Heartbeat(sourceClusterAlias, targetClusterAlias, timestamp);    
+    } 
+
+    private Struct headerStruct(short version) {
+        Struct struct = new Struct(HEADER_SCHEMA);
+        struct.set(VERSION_KEY, version);
+        return struct;
+    }
+
+    private Struct valueStruct(Schema schema) {
+        Struct struct = new Struct(schema);
+        struct.set(TIMESTAMP_KEY, timestamp);
+        return struct;
+    }
+
+    private Struct keyStruct() {
+        Struct struct = new Struct(KEY_SCHEMA);
+        struct.set(SOURCE_CLUSTER_ALIAS_KEY, sourceClusterAlias);
+        struct.set(TARGET_CLUSTER_ALIAS_KEY, targetClusterAlias);
+        return struct;
+    }
+
+    Map connectPartition() {
+        Map partition = new HashMap<>();
+        partition.put(SOURCE_CLUSTER_ALIAS_KEY, sourceClusterAlias);
+        partition.put(TARGET_CLUSTER_ALIAS_KEY, targetClusterAlias);
+        return partition;
+    }
+
+    byte[] recordKey() {
+        return serializeKey().array();
+    }
+
+    byte[] recordValue() {
+        return serializeValue(VERSION).array();
+    }
+
+    private static Schema valueSchema(short version) {
+        assert version == 0;
+        return VALUE_SCHEMA_V0;
+    }
+};
+
diff --git a/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/MirrorClient.java b/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/MirrorClient.java
new file mode 100644
index 0000000000000..17d18ecb58a27
--- /dev/null
+++ b/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/MirrorClient.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.common.protocol.types.SchemaException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+ 
+import java.time.Duration;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Collections;
+import java.util.Collection;
+import java.util.stream.Collectors;
+import java.util.concurrent.ExecutionException;
+
+/** Interprets MM2's internal topics (checkpoints, heartbeats) on a given cluster.
+ *  
                 
+ *  Given a top-level "mm2.properties" configuration file, MirrorClients can be constructed
+ *  for individual clusters as follows:
+ *  
                 
+ *  
                + *    MirrorMakerConfig mmConfig = new MirrorMakerConfig(props);
+ *    MirrorClientConfig mmClientConfig = mmConfig.clientConfig("some-cluster");
+ *    MirrorClient mmClient = new Mirrorclient(mmClientConfig);
+ *  
                
+ */
+public class MirrorClient implements AutoCloseable {
+    private static final Logger log = LoggerFactory.getLogger(MirrorClient.class);
+
+    private AdminClient adminClient;
+    private ReplicationPolicy replicationPolicy;
+    private Map consumerConfig;
+
+    public MirrorClient(Map props) {
+        this(new MirrorClientConfig(props));
+    }
+
+    public MirrorClient(MirrorClientConfig config) {
+        adminClient = AdminClient.create(config.adminConfig());
+        consumerConfig = config.consumerConfig();
+        replicationPolicy = config.replicationPolicy();
+    }
+
+    // for testing
+    MirrorClient(AdminClient adminClient, ReplicationPolicy replicationPolicy,
+            Map consumerConfig) {
+        this.adminClient = adminClient;
+        this.replicationPolicy = replicationPolicy;
+        this.consumerConfig = consumerConfig;
+    }
+
+    /** Close internal clients. */
+    public void close() {
+        adminClient.close();
+    }
+
+    /** Get the ReplicationPolicy instance used to interpret remote topics. This instance is constructed based on
+     *  relevant configuration properties, including {@code replication.policy.class}. */
+    public ReplicationPolicy replicationPolicy() {
+        return replicationPolicy;
+    }
+
+    /** Compute shortest number of hops from an upstream source cluster.
+     *  For example, given replication flow A->B->C, there are two hops from A to C.
+     *  Returns -1 if upstream cluster is unreachable.
+     */
+    public int replicationHops(String upstreamClusterAlias) throws InterruptedException {
+        return heartbeatTopics().stream()
+            .map(x -> countHopsForTopic(x, upstreamClusterAlias))
+            .filter(x -> x != -1)
+            .mapToInt(x -> x)
+            .min()
+            .orElse(-1);
+    }
+
+    /** Find all heartbeat topics on this cluster. Heartbeat topics are replicated from other clusters. */
+    public Set heartbeatTopics() throws InterruptedException {
+        return listTopics().stream()
+            .filter(this::isHeartbeatTopic)
+            .collect(Collectors.toSet());
+    }
+
+    /** Find all checkpoint topics on this cluster. */
+    public Set checkpointTopics() throws InterruptedException {
+        return listTopics().stream()
+            .filter(this::isCheckpointTopic)
+            .collect(Collectors.toSet());
+    }
+
+    /** Find upstream clusters, which may be multiple hops away, based on incoming heartbeats. */
+    public Set upstreamClusters() throws InterruptedException {
+        return listTopics().stream()
+            .filter(this::isHeartbeatTopic)
+            .flatMap(x -> allSources(x).stream())
+            .distinct()
+            .collect(Collectors.toSet());
+    }
+
+    /** Find all remote topics on this cluster. This does not include internal topics (heartbeats, checkpoints). */
+    public Set remoteTopics() throws InterruptedException {
+        return listTopics().stream()
+            .filter(this::isRemoteTopic)
+            .collect(Collectors.toSet());
+    }
+
+    /** Find all remote topics that have been replicated directly from the given source cluster. */
+    public Set remoteTopics(String source) throws InterruptedException {
+        return listTopics().stream()
+            .filter(this::isRemoteTopic)
+            .filter(x -> source.equals(replicationPolicy.topicSource(x)))
+            .distinct()
+            .collect(Collectors.toSet());
+    }
+
+    /** Translate a remote consumer group's offsets into corresponding local offsets. Topics are automatically
+     *  renamed according to the ReplicationPolicy.
+     *  @param consumerGroupId group ID of remote consumer group
+     *  @param remoteClusterAlias alias of remote cluster
+     *  @param timeout timeout
+     */
+    public Map remoteConsumerOffsets(String consumerGroupId,
+            String remoteClusterAlias, Duration timeout) {
+        long deadline = System.currentTimeMillis() + timeout.toMillis();
+        Map offsets = new HashMap<>();
+        KafkaConsumer consumer = new KafkaConsumer<>(consumerConfig,
+            new ByteArrayDeserializer(), new ByteArrayDeserializer());
+        try {
+            // checkpoint topics are not "remote topics", as they are not replicated. So we don't need
+            // to use ReplicationPolicy to create the checkpoint topic here.
+            String checkpointTopic = remoteClusterAlias + MirrorClientConfig.CHECKPOINTS_TOPIC_SUFFIX;
+            List checkpointAssignment =
+                Collections.singletonList(new TopicPartition(checkpointTopic, 0));
+            consumer.assign(checkpointAssignment);
+            consumer.seekToBeginning(checkpointAssignment);
+            while (System.currentTimeMillis() < deadline && !endOfStream(consumer, checkpointAssignment)) {
+                ConsumerRecords records = consumer.poll(timeout);
+                for (ConsumerRecord record : records) {
+                    try {
+                        Checkpoint checkpoint = Checkpoint.deserializeRecord(record);
+                        if (checkpoint.consumerGroupId().equals(consumerGroupId)) {
+                            offsets.put(checkpoint.topicPartition(), checkpoint.offsetAndMetadata());
+                        }
+                    } catch (SchemaException e) {
+                        log.info("Could not deserialize record. Skipping.", e);
+                    }
+                }
+            }
+            log.info("Consumed {} checkpoint records for {} from {}.", offsets.size(),
+                consumerGroupId, checkpointTopic);
+        } finally {
+            consumer.close();
+        }
+        return offsets;
+    }
+
+    Set listTopics() throws InterruptedException {
+        try {
+            return adminClient.listTopics().names().get();
+        } catch (ExecutionException e) {
+            throw new KafkaException(e.getCause());
+        }
+    }
+
+    int countHopsForTopic(String topic, String sourceClusterAlias) {
+        int hops = 0;
+        while (true) {
+            hops++;
+            String source = replicationPolicy.topicSource(topic);
+            if (source == null) {
+                return -1;
+            }
+            if (source.equals(sourceClusterAlias)) {
+                return hops;
+            }
+            topic = replicationPolicy.upstreamTopic(topic);
+        } 
+    }
+
+    boolean isHeartbeatTopic(String topic) {
+        // heartbeats are replicated, so we must use ReplicationPolicy here
+        return MirrorClientConfig.HEARTBEATS_TOPIC.equals(replicationPolicy.originalTopic(topic));
+    }
+
+    boolean isCheckpointTopic(String topic) {
+        // checkpoints are not replicated, so we don't need to use ReplicationPolicy here
+        return topic.endsWith(MirrorClientConfig.CHECKPOINTS_TOPIC_SUFFIX);
+    }
+
+    boolean isRemoteTopic(String topic) {
+        return !replicationPolicy.isInternalTopic(topic)
+            && replicationPolicy.topicSource(topic) != null;
+    }
+
+    Set allSources(String topic) {
+        Set sources = new HashSet<>();
+        String source = replicationPolicy.topicSource(topic);
+        while (source != null) {
+            sources.add(source);
+            topic = replicationPolicy.upstreamTopic(topic);
+            source = replicationPolicy.topicSource(topic);
+        }
+        return sources;
+    }
+
+    static private boolean endOfStream(Consumer consumer, Collection assignments) {
+        Map endOffsets = consumer.endOffsets(assignments);
+        for (TopicPartition topicPartition : assignments) {
+            if (consumer.position(topicPartition) < endOffsets.get(topicPartition)) {
+                return false;
+            }
+        }
+        return true;
+    }
+}
diff --git a/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/MirrorClientConfig.java b/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/MirrorClientConfig.java
new file mode 100644
index 0000000000000..0c163d87fb6ec
--- /dev/null
+++ b/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/MirrorClientConfig.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.config.AbstractConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.config.ConfigDef.Type;
+import org.apache.kafka.common.config.ConfigDef.Importance;
+import org.apache.kafka.clients.CommonClientConfigs;
+
+import java.util.Map;
+import java.util.HashMap;
+
+/** Configuration required for MirrorClient to talk to a given target cluster.
+ *  
                
+ *  Generally, these properties come from an mm2.properties configuration file
+ *  (@see MirrorMakerConfig.clientConfig):
+ *  
                
+ *  
                + *    MirrorMakerConfig mmConfig = new MirrorMakerConfig(props);
+ *    MirrorClientConfig mmClientConfig = mmConfig.clientConfig("some-cluster");
+ *  
                
+ *  
                
+ *  In addition to the properties defined here, sub-configs are supported for Admin, Consumer, and Producer clients.
+ *  For example:
+ *  
                
+ *  
                + *      bootstrap.servers = host1:9092
+ *      consumer.client.id = mm2-client
+ *      replication.policy.separator = __
+ *  
                
+ */
+public class MirrorClientConfig extends AbstractConfig {
+    public static final String REPLICATION_POLICY_CLASS = "replication.policy.class";
+    private static final String REPLICATION_POLICY_CLASS_DOC = "Class which defines the remote topic naming convention.";
+    public static final Class REPLICATION_POLICY_CLASS_DEFAULT = DefaultReplicationPolicy.class;
+    public static final String REPLICATION_POLICY_SEPARATOR = "replication.policy.separator";
+    private static final String REPLICATION_POLICY_SEPARATOR_DOC = "Separator used in remote topic naming convention.";
+    public static final String REPLICATION_POLICY_SEPARATOR_DEFAULT =
+        DefaultReplicationPolicy.SEPARATOR_DEFAULT;
+    
+    public static final String ADMIN_CLIENT_PREFIX = "admin.";
+    public static final String CONSUMER_CLIENT_PREFIX = "consumer.";
+    public static final String PRODUCER_CLIENT_PREFIX = "producer.";
+
+    static final String CHECKPOINTS_TOPIC_SUFFIX = ".checkpoints.internal"; // internal so not replicated
+    static final String HEARTBEATS_TOPIC = "heartbeats";
+ 
+    MirrorClientConfig(Map props) {
+        super(CONFIG_DEF, props, true);
+    }
+
+    public ReplicationPolicy replicationPolicy() {
+        return getConfiguredInstance(REPLICATION_POLICY_CLASS, ReplicationPolicy.class);
+    }
+
+    /** Sub-config for Admin clients. */
+    public Map adminConfig() {
+        return clientConfig(ADMIN_CLIENT_PREFIX);
+    }
+
+    /** Sub-config for Consumer clients. */
+    public Map consumerConfig() {
+        return clientConfig(CONSUMER_CLIENT_PREFIX);
+    }
+
+    /** Sub-config for Producer clients. */
+    public Map producerConfig() {
+        return clientConfig(PRODUCER_CLIENT_PREFIX);
+    }
+    
+    private Map clientConfig(String prefix) {
+        Map props = new HashMap<>();
+        props.putAll(valuesWithPrefixOverride(prefix));
+        props.keySet().retainAll(CLIENT_CONFIG_DEF.names());
+        props.entrySet().removeIf(x -> x.getValue() == null);
+        return props;
+    }
+
+    // Properties passed to internal Kafka clients
+    static final ConfigDef CLIENT_CONFIG_DEF = new ConfigDef()
+        .define(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG,
+            Type.LIST,
+            null,
+            Importance.HIGH,
+            CommonClientConfigs.BOOTSTRAP_SERVERS_DOC)
+        // security support
+        .define(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG,
+            Type.STRING,
+            CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL,
+            Importance.MEDIUM,
+            CommonClientConfigs.SECURITY_PROTOCOL_DOC)
+        .withClientSslSupport()
+        .withClientSaslSupport();
+ 
+    static final ConfigDef CONFIG_DEF = new ConfigDef()
+        .define(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG,
+            Type.STRING,
+            null,
+            Importance.HIGH,
+            CommonClientConfigs.BOOTSTRAP_SERVERS_DOC) 
+        .define(
+            REPLICATION_POLICY_CLASS,
+            ConfigDef.Type.CLASS,
+            REPLICATION_POLICY_CLASS_DEFAULT,
+            ConfigDef.Importance.LOW,
+            REPLICATION_POLICY_CLASS_DOC)
+        .define(
+            REPLICATION_POLICY_SEPARATOR,
+            ConfigDef.Type.STRING,
+            REPLICATION_POLICY_SEPARATOR_DEFAULT,
+            ConfigDef.Importance.LOW,
+            REPLICATION_POLICY_SEPARATOR_DOC)
+        .define(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG,
+                Type.STRING,
+                CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL,
+                Importance.MEDIUM,
+                CommonClientConfigs.SECURITY_PROTOCOL_DOC)
+        .withClientSslSupport()
+        .withClientSaslSupport();
+}
diff --git a/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/RemoteClusterUtils.java b/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/RemoteClusterUtils.java
new file mode 100644
index 0000000000000..f93431985455f
--- /dev/null
+++ b/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/RemoteClusterUtils.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeoutException;
+import java.time.Duration;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+ 
+/** Convenience methods for multi-cluster environments. Wraps MirrorClient (@see MirrorClient).
+ *  
                
+ *  Properties passed to these methods are used to construct internal Admin and Consumer clients.
+ *  Sub-configs like "admin.xyz" are also supported. For example:
+ *  
                
+ *  
                + *      bootstrap.servers = host1:9092
+ *      consumer.client.id = mm2-client
+ *  
                
+ *  
                
+ *  @see MirrorClientConfig for additional properties used by the internal MirrorClient.
+ *  
                
+ */
+public final class RemoteClusterUtils {
+    private static final Logger log = LoggerFactory.getLogger(RemoteClusterUtils.class);
+
+    // utility class
+    private RemoteClusterUtils() {}
+
+    /** Find shortest number of hops from an upstream cluster.
+     *  Returns -1 if the cluster is unreachable */ 
+    public static int replicationHops(Map properties, String upstreamClusterAlias)
+            throws InterruptedException, TimeoutException {
+        try (MirrorClient client = new MirrorClient(properties)) {
+            return client.replicationHops(upstreamClusterAlias);
+        }
+    }
+
+    /** Find all heartbeat topics */
+    public static Set heartbeatTopics(Map properties)
+            throws InterruptedException, TimeoutException {
+        try (MirrorClient client = new MirrorClient(properties)) {
+            return client.heartbeatTopics();
+        }
+    }
+
+    /** Find all checkpoint topics */
+    public static Set checkpointTopics(Map properties)
+            throws InterruptedException, TimeoutException {
+        try (MirrorClient client = new MirrorClient(properties)) {
+            return client.checkpointTopics();
+        }
+    }
+
+    /** Find all upstream clusters */
+    public static Set upstreamClusters(Map properties)
+            throws InterruptedException, TimeoutException {
+        try (MirrorClient client = new MirrorClient(properties)) {
+            return client.upstreamClusters();
+        }
+    }
+
+    /** Translate a remote consumer group's offsets into corresponding local offsets. Topics are automatically
+     *  renamed according to the ReplicationPolicy.
+     *  @param properties @see MirrorClientConfig
+     *  @param consumerGroupId group ID of remote consumer group
+     *  @param remoteClusterAlias alias of remote cluster
+     *  @param timeout timeout
+     */
+    public static Map translateOffsets(Map properties,
+            String remoteClusterAlias, String consumerGroupId, Duration timeout)
+            throws InterruptedException, TimeoutException {
+        try (MirrorClient client = new MirrorClient(properties)) {
+            return client.remoteConsumerOffsets(consumerGroupId, remoteClusterAlias, timeout);
+        }
+    }
+}
diff --git a/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/ReplicationPolicy.java b/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/ReplicationPolicy.java
new file mode 100644
index 0000000000000..11f73f50ceac5
--- /dev/null
+++ b/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/ReplicationPolicy.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.annotation.InterfaceStability;
+
+/** Defines which topics are "remote topics". e.g. "us-west.topic1". */
+@InterfaceStability.Evolving
+public interface ReplicationPolicy {
+
+    /** How to rename remote topics; generally should be like us-west.topic1. */
+    String formatRemoteTopic(String sourceClusterAlias, String topic);
+
+    /** Source cluster alias of given remote topic, e.g. "us-west" for "us-west.topic1".
+     *  Returns null if not a remote topic.
+     */
+    String topicSource(String topic);
+
+    /** Name of topic on the source cluster, e.g. "topic1" for "us-west.topic1".
+     *
+     *  Topics may be replicated multiple hops, so the immediately upstream topic
+     *  may itself be a remote topic.
+     *
+     *  Returns null if not a remote topic.
+     */
+    String upstreamTopic(String topic); 
+
+    /** The name of the original source-topic, which may have been replicated multiple hops.
+     *  Returns the topic if it is not a remote topic.
+     */
+    default String originalTopic(String topic) {
+        String upstream = upstreamTopic(topic);
+        if (upstream == null) {
+            return topic;
+        } else {
+            return originalTopic(upstream);
+        }
+    }
+
+    /** Internal topics are never replicated. */
+    default boolean isInternalTopic(String topic) {
+        return topic.endsWith(".internal") || topic.endsWith("-internal") || topic.startsWith("__")
+            || topic.startsWith(".");
+    }
+}
diff --git a/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/SourceAndTarget.java b/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/SourceAndTarget.java
new file mode 100644
index 0000000000000..f853dc40c16ac
--- /dev/null
+++ b/connect/mirror-client/src/main/java/org/apache/kafka/connect/mirror/SourceAndTarget.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+/** Directional pair of clustes, where source is replicated to target. */
+public class SourceAndTarget {
+    private String source;
+    private String target;
+
+    public SourceAndTarget(String source, String target) {
+        this.source = source;
+        this.target = target;
+    }
+
+    public String source() {
+        return source;
+    }
+
+    public String target() {
+        return target;
+    }
+
+    @Override
+    public String toString() {
+        return source + "->" + target;
+    }
+
+    @Override
+    public int hashCode() {
+        return toString().hashCode();
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        return other != null && toString().equals(other.toString());
+    }
+}
+
diff --git a/connect/mirror-client/src/test/java/org/apache/kafka/connect/mirror/MirrorClientTest.java b/connect/mirror-client/src/test/java/org/apache/kafka/connect/mirror/MirrorClientTest.java
new file mode 100644
index 0000000000000..c2536d5db8508
--- /dev/null
+++ b/connect/mirror-client/src/test/java/org/apache/kafka/connect/mirror/MirrorClientTest.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.Configurable;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.Arrays;
+import java.util.concurrent.TimeoutException;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+public class MirrorClientTest {
+
+    private static class FakeMirrorClient extends MirrorClient {
+
+        List topics;
+
+        FakeMirrorClient(List topics) {
+            super(null, new DefaultReplicationPolicy(), null);
+            this.topics = topics;
+        }
+
+        FakeMirrorClient() {
+            this(Collections.emptyList());
+        } 
+
+        @Override
+        protected Set listTopics() {
+            return new HashSet<>(topics);
+        }
+    }
+
+    @Test
+    public void testIsHeartbeatTopic() throws InterruptedException, TimeoutException {
+        MirrorClient client = new FakeMirrorClient();
+        assertTrue(client.isHeartbeatTopic("heartbeats"));
+        assertTrue(client.isHeartbeatTopic("source1.heartbeats"));
+        assertTrue(client.isHeartbeatTopic("source2.source1.heartbeats"));
+        assertFalse(client.isHeartbeatTopic("heartbeats!"));
+        assertFalse(client.isHeartbeatTopic("!heartbeats"));
+        assertFalse(client.isHeartbeatTopic("source1heartbeats"));
+        assertFalse(client.isHeartbeatTopic("source1-heartbeats"));
+    }
+
+    @Test
+    public void testIsCheckpointTopic() throws InterruptedException, TimeoutException {
+        MirrorClient client = new FakeMirrorClient();
+        assertTrue(client.isCheckpointTopic("source1.checkpoints.internal"));
+        assertFalse(client.isCheckpointTopic("checkpoints.internal"));
+        assertFalse(client.isCheckpointTopic("checkpoints-internal"));
+        assertFalse(client.isCheckpointTopic("checkpoints.internal!"));
+        assertFalse(client.isCheckpointTopic("!checkpoints.internal"));
+        assertFalse(client.isCheckpointTopic("source1checkpointsinternal"));
+    }
+
+    @Test
+    public void countHopsForTopicTest() throws InterruptedException, TimeoutException {
+        MirrorClient client = new FakeMirrorClient();
+        assertEquals(-1, client.countHopsForTopic("topic", "source"));
+        assertEquals(-1, client.countHopsForTopic("source", "source"));
+        assertEquals(-1, client.countHopsForTopic("sourcetopic", "source"));
+        assertEquals(-1, client.countHopsForTopic("source1.topic", "source2"));
+        assertEquals(1, client.countHopsForTopic("source1.topic", "source1"));
+        assertEquals(1, client.countHopsForTopic("source2.source1.topic", "source2"));
+        assertEquals(2, client.countHopsForTopic("source2.source1.topic", "source1"));
+        assertEquals(3, client.countHopsForTopic("source3.source2.source1.topic", "source1"));
+        assertEquals(-1, client.countHopsForTopic("source3.source2.source1.topic", "source4"));
+    }
+
+    @Test
+    public void heartbeatTopicsTest() throws InterruptedException, TimeoutException {
+        MirrorClient client = new FakeMirrorClient(Arrays.asList("topic1", "topic2", "heartbeats",
+            "source1.heartbeats", "source2.source1.heartbeats", "source3.heartbeats"));
+        Set heartbeatTopics = client.heartbeatTopics();
+        assertEquals(heartbeatTopics, new HashSet<>(Arrays.asList("heartbeats", "source1.heartbeats",
+            "source2.source1.heartbeats", "source3.heartbeats")));
+    }
+
+    @Test
+    public void checkpointsTopicsTest() throws InterruptedException, TimeoutException {
+        MirrorClient client = new FakeMirrorClient(Arrays.asList("topic1", "topic2", "checkpoints.internal",
+            "source1.checkpoints.internal", "source2.source1.checkpoints.internal", "source3.checkpoints.internal"));
+        Set checkpointTopics = client.checkpointTopics();
+        assertEquals(new HashSet<>(Arrays.asList("source1.checkpoints.internal",
+            "source2.source1.checkpoints.internal", "source3.checkpoints.internal")), checkpointTopics);
+    }
+
+    @Test
+    public void replicationHopsTest() throws InterruptedException, TimeoutException {
+        MirrorClient client = new FakeMirrorClient(Arrays.asList("topic1", "topic2", "heartbeats",
+            "source1.heartbeats", "source1.source2.heartbeats", "source3.heartbeats"));
+        assertEquals(1, client.replicationHops("source1"));
+        assertEquals(2, client.replicationHops("source2")); 
+        assertEquals(1, client.replicationHops("source3"));
+        assertEquals(-1, client.replicationHops("source4"));
+    }
+
+    @Test
+    public void upstreamClustersTest() throws InterruptedException {
+        MirrorClient client = new FakeMirrorClient(Arrays.asList("topic1", "topic2", "heartbeats",
+            "source1.heartbeats", "source1.source2.heartbeats", "source3.source4.source5.heartbeats"));
+        Set sources = client.upstreamClusters();
+        assertTrue(sources.contains("source1"));
+        assertTrue(sources.contains("source2"));
+        assertTrue(sources.contains("source3"));
+        assertTrue(sources.contains("source4"));
+        assertTrue(sources.contains("source5"));
+        assertFalse(sources.contains("sourceX"));
+        assertFalse(sources.contains(""));
+        assertFalse(sources.contains(null));
+    }
+
+    @Test
+    public void remoteTopicsTest() throws InterruptedException {
+        MirrorClient client = new FakeMirrorClient(Arrays.asList("topic1", "topic2", "topic3",
+            "source1.topic4", "source1.source2.topic5", "source3.source4.source5.topic6"));
+        Set remoteTopics = client.remoteTopics();
+        assertFalse(remoteTopics.contains("topic1"));
+        assertFalse(remoteTopics.contains("topic2"));
+        assertFalse(remoteTopics.contains("topic3"));
+        assertTrue(remoteTopics.contains("source1.topic4"));
+        assertTrue(remoteTopics.contains("source1.source2.topic5"));
+        assertTrue(remoteTopics.contains("source3.source4.source5.topic6"));
+    }
+
+    @Test
+    public void remoteTopicsSeparatorTest() throws InterruptedException {
+        MirrorClient client = new FakeMirrorClient(Arrays.asList("topic1", "topic2", "topic3",
+            "source1__topic4", "source1__source2__topic5", "source3__source4__source5__topic6"));
+        ((Configurable) client.replicationPolicy()).configure(
+            Collections.singletonMap("replication.policy.separator", "__"));
+        Set remoteTopics = client.remoteTopics();
+        assertFalse(remoteTopics.contains("topic1"));
+        assertFalse(remoteTopics.contains("topic2"));
+        assertFalse(remoteTopics.contains("topic3"));
+        assertTrue(remoteTopics.contains("source1__topic4"));
+        assertTrue(remoteTopics.contains("source1__source2__topic5"));
+        assertTrue(remoteTopics.contains("source3__source4__source5__topic6"));
+    }
+
+}
diff --git a/connect/mirror/README.md b/connect/mirror/README.md
new file mode 100644
index 0000000000000..68e3536d94c3d
--- /dev/null
+++ b/connect/mirror/README.md
@@ -0,0 +1,222 @@
+
+# MirrorMaker 2.0
+
+MM2 leverages the Connect framework to replicate topics between Kafka
+clusters. MM2 includes several new features, including:
+
+ - both topics and consumer groups are replicated
+ - topic configuration and ACLs are replicated
+ - cross-cluster offsets are synchronized
+ - partitioning is preserved
+
+## Replication flows
+
+MM2 replicates topics and consumer groups from upstream source clusters
+to downstream target clusters. These directional flows are notated
+`A->B`.
+
+It's possible to create complex replication topologies based on these
+`source->target` flows, including:
+
+ - *fan-out*, e.g. `K->A, K->B, K->C`
+ - *aggregation*, e.g. `A->K, B->K, C->K`
+ - *active/active*, e.g. `A->B, B->A`
+
+Each replication flow can be configured independently, e.g. to replicate
+specific topics or groups:
+
+    A->B.topics = topic-1, topic-2
+    A->B.groups = group-1, group-2
+
+By default, all topics and consumer groups are replicated (except
+blacklisted ones), across all enabled replication flows. Each
+replication flow must be explicitly enabled to begin replication:
+
+    A->B.enabled = true
+    B->A.enabled = true
+
+## Starting an MM2 process
+
+You can run any number of MM2 processes as needed. Any MM2 processes
+which are configured to replicate the same Kafka clusters will find each
+other, share configuration, load balance, etc.
+
+To start an MM2 process, first specify Kafka cluster information in a
+configuration file as follows:
+
+    # mm2.properties
+    clusters = us-west, us-east
+    us-west.bootstrap.servers = host1:9092
+    us-east.bootstrap.servers = host2:9092
+
+You can list any number of clusters this way.
+
+Optionally, you can override default MirrorMaker properties:
+
+    topics = .*
+    groups = group1, group2
+    emit.checkpoints.interval.seconds = 10
+
+These will apply to all replication flows. You can also override default
+properties for specific clusters or replication flows:
+
+    # configure a specific cluster
+    us-west.offset.storage.topic = mm2-offsets
+
+    # configure a specific source->target replication flow
+    us-west->us-east.emit.heartbeats = false
+
+Next, enable individual replication flows as follows:
+
+    us-west->us-east.enabled = true     # disabled by default
+
+Finally, launch one or more MirrorMaker processes with the `connect-mirror-maker.sh`
+script:
+
+    $ ./bin/connect-mirror-maker.sh mm2.properties
+
+## Multicluster environments
+
+MM2 supports replication between multiple Kafka clusters, whether in the
+same data center or across multiple data centers. A single MM2 cluster
+can span multiple data centers, but it is recommended to keep MM2's producers
+as close as possible to their target clusters. To do so, specify a subset
+of clusters for each MM2 node as follows:
+
+    # in west DC:
+    $ ./bin/connect-mirror-maker.sh mm2.properties --clusters west-1 west-2
+
+This signals to the node that the given clusters are nearby, and prevents the
+node from sending records or configuration to clusters in other data centers.
+
+### Example
+
+Say there are three data centers (west, east, north) with two Kafka
+clusters in each data center (west-1, west-2 etc). We can configure MM2
+for active/active replication within each data center, as well as cross data
+center replication (XDCR) as follows:
+
+    # mm2.properties
+    clusters: west-1, west-2, east-1, east-2, north-1, north-2
+
+    west-1.bootstrap.servers = ...
+    ---%<---
+
+    # active/active in west
+    west-1->west-2.enabled = true
+    west-2->west-1.enabled = true
+
+    # active/active in east
+    east-1->east-2.enabled = true
+    east-2->east-1.enabled = true
+
+    # active/active in north
+    north-1->north-2.enabled = true
+    north-2->north-1.enabled = true
+
+    # XDCR via west-1, east-1, north-1
+    west-1->east-1.enabled = true
+    west-1->north-1.enabled = true
+    east-1->west-1.enabled = true
+    east-1->north-1.enabled = true
+    north-1->west-1.enabled = true
+    north-1->east-1.enabled = true
+
+Then, launch MM2 in each data center as follows:
+
+    # in west:
+    $ ./bin/connect-mirror-maker.sh mm2.properties --clusters west-1 west-2
+
+    # in east:
+    $ ./bin/connect-mirror-maker.sh mm2.properties --clusters east-1 east-2
+
+    # in north:
+    $ ./bin/connect-mirror-maker.sh mm2.properties --clusters north-1 north-2
+    
+With this configuration, records produced to any cluster will be replicated
+within the data center, as well as across to other data centers. By providing
+the `--clusters` parameter, we ensure that each node only produces records to
+nearby clusters.
+
+N.B. that the `--clusters` parameter is not technically required here. MM2 will work fine without it; however, throughput may suffer from "producer lag" between
+data centers, and you may incur unnecessary data transfer costs.
+
+## Shared configuration
+
+MM2 processes share configuration via their target Kafka clusters. 
+For example, the following two processes would be racy:
+
+    # process1:
+    A->B.enabled = true
+    A->B.topics = foo
+
+    # process2:
+    A->B.enabled = true
+    A->B.topics = bar
+
+In this case, the two processes will share configuration via cluster `B`.
+Depending on which processes is elected "leader", the result will be
+that either `foo` or `bar` is replicated -- but not both. For this reason,
+it is important to keep configuration consistent across flows to the same
+target cluster. In most cases, your entire organization should use a single
+MM2 configuration file.
+
+## Remote topics
+
+MM2 employs a naming convention to ensure that records from different
+clusters are not written to the same partition. By default, replicated
+topics are renamed based on "source cluster aliases":
+
+    topic-1 --> source.topic-1
+
+This can be customized by overriding the `replication.policy.separator`
+property (default is a period). If you need more control over how
+remote topics are defined, you can implement a custom `ReplicationPolicy`
+and override `replication.policy.class` (default is
+`DefaultReplicationPolicy`).
+
+## Monitoring an MM2 process
+
+MM2 is built on the Connect framework and inherits all of Connect's metrics, e.g.
+`source-record-poll-rate`. In addition, MM2 produces its own metrics under the
+`kafka.connect.mirror` metric group. Metrics are tagged with the following properties:
+
+    - *target*: alias of target cluster
+    - *source*: alias of source cluster
+    - *topic*:  remote topic on target cluster 
+    - *partition*: partition being replicated
+
+Metrics are tracked for each *remote* topic. The source cluster can be inferred
+from the topic name. For example, replicating `topic1` from `A->B` will yield metrics
+like:
+
+    - `target=B`
+    - `topic=A.topic1`
+    - `partition=1`
+
+The following metrics are emitted:
+
+    # MBean: kafka.connect.mirror:type=MirrorSourceConnector,target=([-.w]+),topic=([-.w]+),partition=([0-9]+)
+
+    record-count            # number of records replicated source -> target
+    record-age-ms           # age of records when they are replicated
+    record-age-ms-min
+    record-age-ms-max
+    record-age-ms-avg
+    replication-latecny-ms  # time it takes records to propagate source->target
+    replication-latency-ms-min
+    replication-latency-ms-max
+    replication-latency-ms-avg
+    byte-rate               # average number of bytes/sec in replicated records
+
+
+    # MBean: kafka.connect.mirror:type=MirrorCheckpointConnector,source=([-.w]+),target=([-.w]+)
+
+    checkpoint-latency-ms   # time it takes to replicate consumer offsets
+    checkpoint-latency-ms-min
+    checkpoint-latency-ms-max
+    checkpoint-latency-ms-avg
+
+These metrics do not discern between created-at and log-append timestamps.
+
+
diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/ConfigPropertyFilter.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/ConfigPropertyFilter.java
new file mode 100644
index 0000000000000..ec6b3b910710b
--- /dev/null
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/ConfigPropertyFilter.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.Configurable;
+import org.apache.kafka.common.annotation.InterfaceStability;
+import java.util.Map;
+
+/** Defines which topic configuration properties should be replicated. */
+@InterfaceStability.Evolving
+public interface ConfigPropertyFilter extends Configurable, AutoCloseable {
+
+    boolean shouldReplicateConfigProperty(String prop);
+
+    default void close() {
+        //nop
+    }
+
+    default void configure(Map props) {
+        //nop
+    }
+}
diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultConfigPropertyFilter.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultConfigPropertyFilter.java
new file mode 100644
index 0000000000000..f51db1cbd7dbf
--- /dev/null
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultConfigPropertyFilter.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.config.AbstractConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.config.ConfigDef.Type;
+import org.apache.kafka.common.config.ConfigDef.Importance;
+
+import java.util.Map;
+import java.util.regex.Pattern;
+
+/** Uses a blacklist of property names or regexes. */
+public class DefaultConfigPropertyFilter implements ConfigPropertyFilter {
+    
+    public static final String CONFIG_PROPERTIES_BLACKLIST_CONFIG = "config.properties.blacklist";
+    private static final String CONFIG_PROPERTIES_BLACKLIST_DOC = "List of topic configuration properties and/or regexes "
+        + "that should not be replicated.";
+    public static final String CONFIG_PROPERTIES_BLACKLIST_DEFAULT = "follower\\.replication\\.throttled\\.replicas, "
+        + "leader\\.replication\\.throttled\\.replicas, "
+        + "message\\.timestamp\\.difference\\.max\\.ms, "
+        + "message\\.timestamp\\.type, "
+        + "unclean\\.leader\\.election\\.enable, "
+        + "min\\.insync\\.replicas";
+    private Pattern blacklistPattern = MirrorUtils.compilePatternList(CONFIG_PROPERTIES_BLACKLIST_DEFAULT);
+
+    @Override
+    public void configure(Map props) {
+        ConfigPropertyFilterConfig config = new ConfigPropertyFilterConfig(props);
+        blacklistPattern = config.blacklistPattern();
+    }
+
+    @Override
+    public void close() {
+    }
+
+    private boolean blacklisted(String prop) {
+        return blacklistPattern != null && blacklistPattern.matcher(prop).matches();
+    }
+
+    @Override
+    public boolean shouldReplicateConfigProperty(String prop) {
+        return !blacklisted(prop);
+    }
+
+    static class ConfigPropertyFilterConfig extends AbstractConfig {
+
+        static final ConfigDef DEF = new ConfigDef()
+            .define(CONFIG_PROPERTIES_BLACKLIST_CONFIG,
+                Type.LIST,
+                CONFIG_PROPERTIES_BLACKLIST_DEFAULT,
+                Importance.HIGH,
+                CONFIG_PROPERTIES_BLACKLIST_DOC);
+
+        ConfigPropertyFilterConfig(Map props) {
+            super(DEF, props, false);
+        }
+
+        Pattern blacklistPattern() {
+            return MirrorUtils.compilePatternList(getList(CONFIG_PROPERTIES_BLACKLIST_CONFIG));
+        }
+    }
+}
diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultGroupFilter.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultGroupFilter.java
new file mode 100644
index 0000000000000..acf5115236dd7
--- /dev/null
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultGroupFilter.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.config.AbstractConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.config.ConfigDef.Type;
+import org.apache.kafka.common.config.ConfigDef.Importance;
+
+import java.util.Map;
+import java.util.regex.Pattern;
+
+/** Uses a whitelist and blacklist. */
+public class DefaultGroupFilter implements GroupFilter {
+
+    public static final String GROUPS_WHITELIST_CONFIG = "groups";
+    private static final String GROUPS_WHITELIST_DOC = "List of consumer group names and/or regexes to replicate.";
+    public static final String GROUPS_WHITELIST_DEFAULT = ".*";
+
+    public static final String GROUPS_BLACKLIST_CONFIG = "groups.blacklist";
+    private static final String GROUPS_BLACKLIST_DOC = "List of consumer group names and/or regexes that should not be replicated.";
+    public static final String GROUPS_BLACKLIST_DEFAULT = "console-consumer-.*, connect-.*, __.*";
+
+    private Pattern whitelistPattern;
+    private Pattern blacklistPattern;
+
+    @Override
+    public void configure(Map props) {
+        GroupFilterConfig config = new GroupFilterConfig(props);
+        whitelistPattern = config.whitelistPattern();
+        blacklistPattern = config.blacklistPattern();
+    }
+
+    @Override
+    public void close() {
+    }
+
+    private boolean whitelisted(String group) {
+        return whitelistPattern != null && whitelistPattern.matcher(group).matches();
+    }
+
+    private boolean blacklisted(String group) {
+        return blacklistPattern != null && blacklistPattern.matcher(group).matches();
+    }
+
+    @Override
+    public boolean shouldReplicateGroup(String group) {
+        return whitelisted(group) && !blacklisted(group);
+    }
+
+    static class GroupFilterConfig extends AbstractConfig {
+
+        static final ConfigDef DEF = new ConfigDef()
+            .define(GROUPS_WHITELIST_CONFIG,
+                Type.LIST,
+                GROUPS_WHITELIST_DEFAULT,
+                Importance.HIGH,
+                GROUPS_WHITELIST_DOC) 
+            .define(GROUPS_BLACKLIST_CONFIG,
+                Type.LIST,
+                GROUPS_BLACKLIST_DEFAULT,
+                Importance.HIGH,
+                GROUPS_BLACKLIST_DOC);
+
+        GroupFilterConfig(Map props) {
+            super(DEF, props, false);
+        }
+
+        Pattern whitelistPattern() {
+            return MirrorUtils.compilePatternList(getList(GROUPS_WHITELIST_CONFIG));
+        }
+
+        Pattern blacklistPattern() {
+            return MirrorUtils.compilePatternList(getList(GROUPS_BLACKLIST_CONFIG));
+        }
+    }
+}
diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultTopicFilter.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultTopicFilter.java
new file mode 100644
index 0000000000000..308bdbfd8a8c4
--- /dev/null
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultTopicFilter.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.config.AbstractConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.config.ConfigDef.Type;
+import org.apache.kafka.common.config.ConfigDef.Importance;
+
+import java.util.Map;
+import java.util.regex.Pattern;
+
+/** Uses a whitelist and blacklist. */
+public class DefaultTopicFilter implements TopicFilter {
+    
+    public static final String TOPICS_WHITELIST_CONFIG = "topics";
+    private static final String TOPICS_WHITELIST_DOC = "List of topics and/or regexes to replicate.";
+    public static final String TOPICS_WHITELIST_DEFAULT = ".*";
+
+    public static final String TOPICS_BLACKLIST_CONFIG = "topics.blacklist";
+    private static final String TOPICS_BLACKLIST_DOC = "List of topics and/or regexes that should not be replicated.";
+    public static final String TOPICS_BLACKLIST_DEFAULT = ".*[\\-\\.]internal, .*\\.replica, __.*";
+
+    private Pattern whitelistPattern;
+    private Pattern blacklistPattern;
+
+    @Override
+    public void configure(Map props) {
+        TopicFilterConfig config = new TopicFilterConfig(props);
+        whitelistPattern = config.whitelistPattern();
+        blacklistPattern = config.blacklistPattern();
+    }
+
+    @Override
+    public void close() {
+    }
+
+    private boolean whitelisted(String topic) {
+        return whitelistPattern != null && whitelistPattern.matcher(topic).matches();
+    }
+
+    private boolean blacklisted(String topic) {
+        return blacklistPattern != null && blacklistPattern.matcher(topic).matches();
+    }
+
+    @Override
+    public boolean shouldReplicateTopic(String topic) {
+        return whitelisted(topic) && !blacklisted(topic);
+    }
+
+    static class TopicFilterConfig extends AbstractConfig {
+
+        static final ConfigDef DEF = new ConfigDef()
+            .define(TOPICS_WHITELIST_CONFIG,
+                Type.LIST,
+                TOPICS_WHITELIST_DEFAULT,
+                Importance.HIGH,
+                TOPICS_WHITELIST_DOC) 
+            .define(TOPICS_BLACKLIST_CONFIG,
+                Type.LIST,
+                TOPICS_BLACKLIST_DEFAULT,
+                Importance.HIGH,
+                TOPICS_BLACKLIST_DOC);
+
+        TopicFilterConfig(Map props) {
+            super(DEF, props, false);
+        }
+
+        Pattern whitelistPattern() {
+            return MirrorUtils.compilePatternList(getList(TOPICS_WHITELIST_CONFIG));
+        }
+
+        Pattern blacklistPattern() {
+            return MirrorUtils.compilePatternList(getList(TOPICS_BLACKLIST_CONFIG));
+        }
+    }
+}
diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/GroupFilter.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/GroupFilter.java
new file mode 100644
index 0000000000000..0202dd5d2b358
--- /dev/null
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/GroupFilter.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.Configurable;
+import org.apache.kafka.common.annotation.InterfaceStability;
+import java.util.Map;
+
+/** Defines which consumer groups should be replicated. */
+@InterfaceStability.Evolving
+public interface GroupFilter extends Configurable, AutoCloseable {
+
+    boolean shouldReplicateGroup(String group);
+
+    default void close() {
+        //nop
+    }
+
+    default void configure(Map props) {
+        //nop
+    }
+}
diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorCheckpointConnector.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorCheckpointConnector.java
new file mode 100644
index 0000000000000..a358584dff009
--- /dev/null
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorCheckpointConnector.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.ConsumerGroupListing;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.connect.connector.Task;
+import org.apache.kafka.connect.source.SourceConnector;
+import org.apache.kafka.connect.util.ConnectorUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.stream.Collectors;
+
+/** Replicate consumer group state between clusters. Emits checkpoint records.
+ *
+ *  @see MirrorConnectorConfig for supported config properties.
+ */
+public class MirrorCheckpointConnector extends SourceConnector {
+
+    private static final Logger log = LoggerFactory.getLogger(MirrorCheckpointConnector.class);
+
+    private Scheduler scheduler;
+    private MirrorConnectorConfig config;
+    private GroupFilter groupFilter;
+    private AdminClient sourceAdminClient;
+    private SourceAndTarget sourceAndTarget;
+    private String connectorName;
+    private List knownConsumerGroups = Collections.emptyList();
+
+    @Override
+    public void start(Map props) {
+        config = new MirrorConnectorConfig(props);
+        if (!config.enabled()) {
+            return;
+        }
+        connectorName = config.connectorName();
+        sourceAndTarget = new SourceAndTarget(config.sourceClusterAlias(), config.targetClusterAlias());
+        groupFilter = config.groupFilter();
+        sourceAdminClient = AdminClient.create(config.sourceAdminConfig());
+        scheduler = new Scheduler(MirrorCheckpointConnector.class, config.adminTimeout());
+        scheduler.execute(this::createInternalTopics, "creating internal topics");
+        scheduler.execute(this::loadInitialConsumerGroups, "loading initial consumer groups");
+        scheduler.scheduleRepeatingDelayed(this::refreshConsumerGroups, config.refreshGroupsInterval(),
+                "refreshing consumer groups");
+        log.info("Started {} with {} consumer groups.", connectorName, knownConsumerGroups.size());
+        log.debug("Started {} with consumer groups: {}", connectorName, knownConsumerGroups);
+    }
+
+    @Override
+    public void stop() {
+        if (!config.enabled()) {
+            return;
+        }
+        Utils.closeQuietly(scheduler, "scheduler");
+        Utils.closeQuietly(groupFilter, "group filter");
+        Utils.closeQuietly(sourceAdminClient, "source admin client");
+    }
+
+    @Override
+    public Class taskClass() {
+        return MirrorCheckpointTask.class;
+    }
+
+    // divide consumer groups among tasks
+    @Override
+    public List> taskConfigs(int maxTasks) {
+        if (!config.enabled() || knownConsumerGroups.isEmpty()) {
+            return Collections.emptyList();
+        }
+        int numTasks = Math.min(maxTasks, knownConsumerGroups.size());
+        return ConnectorUtils.groupPartitions(knownConsumerGroups, numTasks).stream()
+                .map(config::taskConfigForConsumerGroups)
+                .collect(Collectors.toList());
+    }
+
+    @Override
+    public ConfigDef config() {
+        return MirrorConnectorConfig.CONNECTOR_CONFIG_DEF;
+    }
+
+    @Override
+    public String version() {
+        return "1";
+    }
+
+    private void refreshConsumerGroups()
+            throws InterruptedException, ExecutionException {
+        List consumerGroups = findConsumerGroups();
+        Set newConsumerGroups = new HashSet<>();
+        newConsumerGroups.addAll(consumerGroups);
+        newConsumerGroups.removeAll(knownConsumerGroups);
+        Set deadConsumerGroups = new HashSet<>();
+        deadConsumerGroups.addAll(knownConsumerGroups);
+        deadConsumerGroups.removeAll(consumerGroups);
+        if (!newConsumerGroups.isEmpty() || !deadConsumerGroups.isEmpty()) {
+            log.info("Found {} consumer groups for {}. {} are new. {} were removed. Previously had {}.",
+                    consumerGroups.size(), sourceAndTarget, newConsumerGroups.size(), deadConsumerGroups.size(),
+                    knownConsumerGroups.size());
+            log.debug("Found new consumer groups: {}", newConsumerGroups);
+            knownConsumerGroups = consumerGroups;
+            context.requestTaskReconfiguration();
+        }
+    }
+
+    private void loadInitialConsumerGroups()
+            throws InterruptedException, ExecutionException {
+        knownConsumerGroups = findConsumerGroups();
+    }
+
+    private List findConsumerGroups()
+            throws InterruptedException, ExecutionException {
+        return listConsumerGroups().stream()
+                .filter(x -> !x.isSimpleConsumerGroup())
+                .map(x -> x.groupId())
+                .filter(this::shouldReplicate)
+                .collect(Collectors.toList());
+    }
+
+    private Collection listConsumerGroups()
+            throws InterruptedException, ExecutionException {
+        return sourceAdminClient.listConsumerGroups().valid().get();
+    }
+
+    private void createInternalTopics() {
+        MirrorUtils.createSinglePartitionCompactedTopic(config.checkpointsTopic(),
+            config.checkpointsTopicReplicationFactor(), config.targetAdminConfig());
+    } 
+
+    boolean shouldReplicate(String group) {
+        return groupFilter.shouldReplicateGroup(group);
+    }
+}
diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorCheckpointTask.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorCheckpointTask.java
new file mode 100644
index 0000000000000..47a05693c3313
--- /dev/null
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorCheckpointTask.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.connect.source.SourceTask;
+import org.apache.kafka.connect.source.SourceRecord;
+import org.apache.kafka.connect.data.Schema;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Set;
+import java.util.Collections;
+import java.util.stream.Collectors;
+import java.util.concurrent.ExecutionException;
+import java.time.Duration;
+
+/** Emits checkpoints for upstream consumer groups. */
+public class MirrorCheckpointTask extends SourceTask {
+
+    private static final Logger log = LoggerFactory.getLogger(MirrorCheckpointTask.class);
+
+    private AdminClient sourceAdminClient;
+    private String sourceClusterAlias;
+    private String targetClusterAlias;
+    private String checkpointsTopic;
+    private Duration interval;
+    private Duration pollTimeout;
+    private Duration adminTimeout;
+    private TopicFilter topicFilter;
+    private Set consumerGroups;
+    private ReplicationPolicy replicationPolicy;
+    private OffsetSyncStore offsetSyncStore;
+    private boolean stopping;
+    private MirrorMetrics metrics;
+
+    public MirrorCheckpointTask() {}
+
+    // for testing
+    MirrorCheckpointTask(String sourceClusterAlias, String targetClusterAlias,
+            ReplicationPolicy replicationPolicy, OffsetSyncStore offsetSyncStore) {
+        this.sourceClusterAlias = sourceClusterAlias;
+        this.targetClusterAlias = targetClusterAlias;
+        this.replicationPolicy = replicationPolicy;
+        this.offsetSyncStore = offsetSyncStore;
+    }
+
+    @Override
+    public void start(Map props) {
+        MirrorTaskConfig config = new MirrorTaskConfig(props);
+        stopping = false;
+        sourceClusterAlias = config.sourceClusterAlias();
+        targetClusterAlias = config.targetClusterAlias();
+        consumerGroups = config.taskConsumerGroups();
+        checkpointsTopic = config.checkpointsTopic();
+        topicFilter = config.topicFilter();
+        replicationPolicy = config.replicationPolicy();
+        interval = config.emitCheckpointsInterval();
+        pollTimeout = config.consumerPollTimeout();
+        adminTimeout = config.adminTimeout();
+        offsetSyncStore = new OffsetSyncStore(config);
+        sourceAdminClient = AdminClient.create(config.sourceAdminConfig());
+        metrics = config.metrics();
+    }
+
+    @Override
+    public void commit() throws InterruptedException {
+        // nop
+    }
+
+    @Override
+    public void stop() {
+        long start = System.currentTimeMillis();
+        stopping = true;
+        Utils.closeQuietly(offsetSyncStore, "offset sync store");
+        Utils.closeQuietly(sourceAdminClient, "source admin client");
+        Utils.closeQuietly(metrics, "metrics");
+        log.info("Stopping {} took {} ms.", Thread.currentThread().getName(), System.currentTimeMillis() - start);
+    }
+
+    @Override
+    public String version() {
+        return "1";
+    }
+
+    @Override
+    public List poll() throws InterruptedException {
+        try { 
+            long deadline = System.currentTimeMillis() + interval.toMillis();
+            while (!stopping && System.currentTimeMillis() < deadline) {
+                offsetSyncStore.update(pollTimeout);
+            }
+            List records = new ArrayList<>();
+            for (String group : consumerGroups) {
+                records.addAll(checkpointsForGroup(group));
+            }
+            if (records.isEmpty()) {
+                // WorkerSourceTask expects non-zero batches or null
+                return null;
+            } else {
+                return records;
+            }
+        } catch (Throwable e) {
+            log.warn("Failure polling consumer state for checkpoints.", e);
+            return null;
+        }
+    }
+
+    private List checkpointsForGroup(String group) throws InterruptedException {
+        try {
+            long timestamp = System.currentTimeMillis();
+            return listConsumerGroupOffsets(group).entrySet().stream()
+                .filter(x -> shouldCheckpointTopic(x.getKey().topic()))
+                .map(x -> checkpoint(group, x.getKey(), x.getValue()))
+                .filter(x -> x.downstreamOffset() > 0)  // ignore offsets we cannot translate accurately
+                .map(x -> checkpointRecord(x, timestamp))
+                .collect(Collectors.toList());
+        } catch (ExecutionException e) {
+            log.error("Error querying offsets for consumer group {} on cluster {}.",  group, sourceClusterAlias, e);
+            return Collections.emptyList();
+        }
+    }
+
+    private Map listConsumerGroupOffsets(String group)
+            throws InterruptedException, ExecutionException {
+        if (stopping) {
+            // short circuit if stopping
+            return Collections.emptyMap();
+        }
+        return sourceAdminClient.listConsumerGroupOffsets(group).partitionsToOffsetAndMetadata().get();
+    }
+
+    Checkpoint checkpoint(String group, TopicPartition topicPartition,
+            OffsetAndMetadata offsetAndMetadata) {
+        long upstreamOffset = offsetAndMetadata.offset();
+        long downstreamOffset = offsetSyncStore.translateDownstream(topicPartition, upstreamOffset);
+        return new Checkpoint(group, renameTopicPartition(topicPartition),
+            upstreamOffset, downstreamOffset, offsetAndMetadata.metadata());
+    }
+
+    SourceRecord checkpointRecord(Checkpoint checkpoint, long timestamp) {
+        return new SourceRecord(
+            checkpoint.connectPartition(), MirrorUtils.wrapOffset(0),
+            checkpointsTopic, 0,
+            Schema.BYTES_SCHEMA, checkpoint.recordKey(),
+            Schema.BYTES_SCHEMA, checkpoint.recordValue(),
+            timestamp);
+    }
+
+    TopicPartition renameTopicPartition(TopicPartition upstreamTopicPartition) {
+        if (targetClusterAlias.equals(replicationPolicy.topicSource(upstreamTopicPartition.topic()))) {
+            // this topic came from the target cluster, so we rename like us-west.topic1 -> topic1
+            return new TopicPartition(replicationPolicy.originalTopic(upstreamTopicPartition.topic()),
+                upstreamTopicPartition.partition());
+        } else {
+            // rename like topic1 -> us-west.topic1
+            return new TopicPartition(replicationPolicy.formatRemoteTopic(sourceClusterAlias,
+                upstreamTopicPartition.topic()), upstreamTopicPartition.partition());
+        }
+    }
+
+    boolean shouldCheckpointTopic(String topic) {
+        return topicFilter.shouldReplicateTopic(topic);
+    }
+
+    @Override
+    public void commitRecord(SourceRecord record) {
+        metrics.checkpointLatency(MirrorUtils.unwrapPartition(record.sourcePartition()),
+            Checkpoint.unwrapGroup(record.sourcePartition()),
+            System.currentTimeMillis() - record.timestamp());
+    }
+}
diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorConnectorConfig.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorConnectorConfig.java
new file mode 100644
index 0000000000000..d922eade5f372
--- /dev/null
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorConnectorConfig.java
@@ -0,0 +1,601 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.config.AbstractConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.metrics.MetricsReporter;
+import org.apache.kafka.common.metrics.JmxReporter;
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.connect.runtime.ConnectorConfig;
+
+import java.util.Map;
+import java.util.HashMap;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.time.Duration;
+
+/** Shared config properties used by MirrorSourceConnector, MirrorCheckpointConnector, and MirrorHeartbeatConnector.
+ *  
                
+ *  Generally, these properties are filled-in automatically by MirrorMaker based on a top-level mm2.properties file.
+ *  However, when running MM2 connectors as plugins on a Connect-as-a-Service cluster, these properties must be configured manually,
+ *  e.g. via the Connect REST API.
+ *  
                
+ *  
                
+ *  An example configuration when running on Connect (not via MirrorMaker driver):
+ *  
                
+ *  
                + *      {
+ *        "name": "MirrorSourceConnector",
+ *        "connector.class": "org.apache.kafka.connect.mirror.MirrorSourceConnector",
+ *        "replication.factor": "1",
+ *        "source.cluster.alias": "backup",
+ *        "target.cluster.alias": "primary",
+ *        "source.cluster.bootstrap.servers": "vip1:9092",
+ *        "target.cluster.bootstrap.servers": "vip2:9092",
+ *        "topics": ".*test-topic-.*",
+ *        "groups": "consumer-group-.*",
+ *        "emit.checkpoints.interval.seconds": "1",
+ *        "emit.heartbeats.interval.seconds": "1",
+ *        "sync.topic.acls.enabled": "false"
+ *      }
+ *  
                
+ */
+public class MirrorConnectorConfig extends AbstractConfig {
+
+    protected static final String ENABLED_SUFFIX = ".enabled";
+    protected static final String INTERVAL_SECONDS_SUFFIX = ".interval.seconds";
+
+    protected static final String REFRESH_TOPICS = "refresh.topics";
+    protected static final String REFRESH_GROUPS = "refresh.groups";
+    protected static final String SYNC_TOPIC_CONFIGS = "sync.topic.configs";
+    protected static final String SYNC_TOPIC_ACLS = "sync.topic.acls";
+    protected static final String EMIT_HEARTBEATS = "emit.heartbeats";
+    protected static final String EMIT_CHECKPOINTS = "emit.checkpoints";
+
+    public static final String ENABLED = "enabled";
+    private static final String ENABLED_DOC = "Whether to replicate source->target.";
+    public static final String SOURCE_CLUSTER_ALIAS = "source.cluster.alias";
+    private static final String SOURCE_CLUSTER_ALIAS_DOC = "Alias of source cluster";
+    public static final String TARGET_CLUSTER_ALIAS = "target.cluster.alias";
+    public static final String TARGET_CLUSTER_ALIAS_DEFAULT = "target";
+    private static final String TARGET_CLUSTER_ALIAS_DOC = "Alias of target cluster. Used in metrics reporting.";
+    public static final String REPLICATION_POLICY_CLASS = MirrorClientConfig.REPLICATION_POLICY_CLASS;
+    public static final Class REPLICATION_POLICY_CLASS_DEFAULT = MirrorClientConfig.REPLICATION_POLICY_CLASS_DEFAULT;
+    private static final String REPLICATION_POLICY_CLASS_DOC = "Class which defines the remote topic naming convention.";
+    public static final String REPLICATION_POLICY_SEPARATOR = MirrorClientConfig.REPLICATION_POLICY_SEPARATOR;
+    private static final String REPLICATION_POLICY_SEPARATOR_DOC = "Separator used in remote topic naming convention.";
+    public static final String REPLICATION_POLICY_SEPARATOR_DEFAULT =
+            MirrorClientConfig.REPLICATION_POLICY_SEPARATOR_DEFAULT;
+    public static final String REPLICATION_FACTOR = "replication.factor";
+    private static final String REPLICATION_FACTOR_DOC = "Replication factor for newly created remote topics.";
+    public static final int REPLICATION_FACTOR_DEFAULT = 2;
+    public static final String TOPICS = DefaultTopicFilter.TOPICS_WHITELIST_CONFIG;
+    public static final String TOPICS_DEFAULT = DefaultTopicFilter.TOPICS_WHITELIST_DEFAULT;
+    private static final String TOPICS_DOC = "Topics to replicate. Supports comma-separated topic names and regexes.";
+    public static final String TOPICS_BLACKLIST = DefaultTopicFilter.TOPICS_BLACKLIST_CONFIG;
+    public static final String TOPICS_BLACKLIST_DEFAULT = DefaultTopicFilter.TOPICS_BLACKLIST_DEFAULT;
+    private static final String TOPICS_BLACKLIST_DOC = "Blacklisted topics. Supports comma-separated topic names and regexes."
+            + " Blacklists take precedence over whitelists.";
+    public static final String GROUPS = DefaultGroupFilter.GROUPS_WHITELIST_CONFIG;
+    public static final String GROUPS_DEFAULT = DefaultGroupFilter.GROUPS_WHITELIST_DEFAULT;
+    private static final String GROUPS_DOC = "Consumer groups to replicate. Supports comma-separated group IDs and regexes.";
+    public static final String GROUPS_BLACKLIST = DefaultGroupFilter.GROUPS_BLACKLIST_CONFIG;
+    public static final String GROUPS_BLACKLIST_DEFAULT = DefaultGroupFilter.GROUPS_BLACKLIST_DEFAULT;
+    private static final String GROUPS_BLACKLIST_DOC = "Blacklisted groups. Supports comma-separated group IDs and regexes."
+            + " Blacklists take precedence over whitelists.";
+    public static final String CONFIG_PROPERTIES_BLACKLIST = DefaultConfigPropertyFilter.CONFIG_PROPERTIES_BLACKLIST_CONFIG;
+    public static final String CONFIG_PROPERTIES_BLACKLIST_DEFAULT = DefaultConfigPropertyFilter.CONFIG_PROPERTIES_BLACKLIST_DEFAULT;
+    private static final String CONFIG_PROPERTIES_BLACKLIST_DOC = "Topic config properties that should not be replicated. Supports "
+            + "comma-separated property names and regexes.";
+
+    public static final String HEARTBEATS_TOPIC_REPLICATION_FACTOR = "heartbeats.topic.replication.factor";
+    public static final String HEARTBEATS_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor for heartbeats topic.";
+    public static final short HEARTBEATS_TOPIC_REPLICATION_FACTOR_DEFAULT = 3;
+
+    public static final String CHECKPOINTS_TOPIC_REPLICATION_FACTOR = "checkpoints.topic.replication.factor";
+    public static final String CHECKPOINTS_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor for checkpoints topic.";
+    public static final short CHECKPOINTS_TOPIC_REPLICATION_FACTOR_DEFAULT = 3;
+
+    public static final String OFFSET_SYNCS_TOPIC_REPLICATION_FACTOR = "offset-syncs.topic.replication.factor";
+    public static final String OFFSET_SYNCS_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor for offset-syncs topic.";
+    public static final short OFFSET_SYNCS_TOPIC_REPLICATION_FACTOR_DEFAULT = 3;
+
+    protected static final String TASK_TOPIC_PARTITIONS = "task.assigned.partitions";
+    protected static final String TASK_CONSUMER_GROUPS = "task.assigned.groups";
+
+    public static final String CONSUMER_POLL_TIMEOUT_MILLIS = "consumer.poll.timeout.ms";
+    private static final String CONSUMER_POLL_TIMEOUT_MILLIS_DOC = "Timeout when polling source cluster.";
+    public static final long CONSUMER_POLL_TIMEOUT_MILLIS_DEFAULT = 1000L;
+
+    public static final String ADMIN_TASK_TIMEOUT_MILLIS = "admin.timeout.ms";
+    private static final String ADMIN_TASK_TIMEOUT_MILLIS_DOC = "Timeout for administrative tasks, e.g. detecting new topics.";
+    public static final long ADMIN_TASK_TIMEOUT_MILLIS_DEFAULT = 60000L;
+
+    public static final String REFRESH_TOPICS_ENABLED = REFRESH_TOPICS + ENABLED_SUFFIX;
+    private static final String REFRESH_TOPICS_ENABLED_DOC = "Whether to periodically check for new topics and partitions.";
+    public static final boolean REFRESH_TOPICS_ENABLED_DEFAULT = true;
+    public static final String REFRESH_TOPICS_INTERVAL_SECONDS = REFRESH_TOPICS + INTERVAL_SECONDS_SUFFIX;
+    private static final String REFRESH_TOPICS_INTERVAL_SECONDS_DOC = "Frequency of topic refresh.";
+    public static final long REFRESH_TOPICS_INTERVAL_SECONDS_DEFAULT = 10 * 60;
+
+    public static final String REFRESH_GROUPS_ENABLED = REFRESH_GROUPS + ENABLED_SUFFIX;
+    private static final String REFRESH_GROUPS_ENABLED_DOC = "Whether to periodically check for new consumer groups.";
+    public static final boolean REFRESH_GROUPS_ENABLED_DEFAULT = true;
+    public static final String REFRESH_GROUPS_INTERVAL_SECONDS = REFRESH_GROUPS + INTERVAL_SECONDS_SUFFIX;
+    private static final String REFRESH_GROUPS_INTERVAL_SECONDS_DOC = "Frequency of group refresh.";
+    public static final long REFRESH_GROUPS_INTERVAL_SECONDS_DEFAULT = 10 * 60;
+
+    public static final String SYNC_TOPIC_CONFIGS_ENABLED = SYNC_TOPIC_CONFIGS + ENABLED_SUFFIX;
+    private static final String SYNC_TOPIC_CONFIGS_ENABLED_DOC = "Whether to periodically configure remote topics to match their corresponding upstream topics.";
+    public static final boolean SYNC_TOPIC_CONFIGS_ENABLED_DEFAULT = true;
+    public static final String SYNC_TOPIC_CONFIGS_INTERVAL_SECONDS = SYNC_TOPIC_CONFIGS + INTERVAL_SECONDS_SUFFIX;
+    private static final String SYNC_TOPIC_CONFIGS_INTERVAL_SECONDS_DOC = "Frequency of topic config sync.";
+    public static final long SYNC_TOPIC_CONFIGS_INTERVAL_SECONDS_DEFAULT = 10 * 60;
+
+    public static final String SYNC_TOPIC_ACLS_ENABLED = SYNC_TOPIC_ACLS + ENABLED_SUFFIX;
+    private static final String SYNC_TOPIC_ACLS_ENABLED_DOC = "Whether to periodically configure remote topic ACLs to match their corresponding upstream topics.";
+    public static final boolean SYNC_TOPIC_ACLS_ENABLED_DEFAULT = true;
+    public static final String SYNC_TOPIC_ACLS_INTERVAL_SECONDS = SYNC_TOPIC_ACLS + INTERVAL_SECONDS_SUFFIX;
+    private static final String SYNC_TOPIC_ACLS_INTERVAL_SECONDS_DOC = "Frequency of topic ACL sync.";
+    public static final long SYNC_TOPIC_ACLS_INTERVAL_SECONDS_DEFAULT = 10 * 60;
+
+    public static final String EMIT_HEARTBEATS_ENABLED = EMIT_HEARTBEATS + ENABLED_SUFFIX;
+    private static final String EMIT_HEARTBEATS_ENABLED_DOC = "Whether to emit heartbeats to target cluster.";
+    public static final boolean EMIT_HEARTBEATS_ENABLED_DEFAULT = true;
+    public static final String EMIT_HEARTBEATS_INTERVAL_SECONDS = EMIT_HEARTBEATS + INTERVAL_SECONDS_SUFFIX;
+    private static final String EMIT_HEARTBEATS_INTERVAL_SECONDS_DOC = "Frequency of heartbeats.";
+    public static final long EMIT_HEARTBEATS_INTERVAL_SECONDS_DEFAULT = 1;
+
+    public static final String EMIT_CHECKPOINTS_ENABLED = EMIT_CHECKPOINTS + ENABLED_SUFFIX;
+    private static final String EMIT_CHECKPOINTS_ENABLED_DOC = "Whether to replicate consumer offsets to target cluster.";
+    public static final boolean EMIT_CHECKPOINTS_ENABLED_DEFAULT = true;
+    public static final String EMIT_CHECKPOINTS_INTERVAL_SECONDS = EMIT_CHECKPOINTS + INTERVAL_SECONDS_SUFFIX;
+    private static final String EMIT_CHECKPOINTS_INTERVAL_SECONDS_DOC = "Frequency of checkpoints.";
+    public static final long EMIT_CHECKPOINTS_INTERVAL_SECONDS_DEFAULT = 60;
+
+    public static final String TOPIC_FILTER_CLASS = "topic.filter.class";
+    private static final String TOPIC_FILTER_CLASS_DOC = "TopicFilter to use. Selects topics to replicate.";
+    public static final Class TOPIC_FILTER_CLASS_DEFAULT = DefaultTopicFilter.class;
+    public static final String GROUP_FILTER_CLASS = "group.filter.class";
+    private static final String GROUP_FILTER_CLASS_DOC = "GroupFilter to use. Selects consumer groups to replicate.";
+    public static final Class GROUP_FILTER_CLASS_DEFAULT = DefaultGroupFilter.class;
+    public static final String CONFIG_PROPERTY_FILTER_CLASS = "config.property.filter.class";
+    private static final String CONFIG_PROPERTY_FILTER_CLASS_DOC = "ConfigPropertyFilter to use. Selects topic config "
+            + " properties to replicate.";
+    public static final Class CONFIG_PROPERTY_FILTER_CLASS_DEFAULT = DefaultConfigPropertyFilter.class;
+
+    public static final String OFFSET_LAG_MAX = "offset.lag.max";
+    private static final String OFFSET_LAG_MAX_DOC = "How out-of-sync a remote partition can be before it is resynced.";
+    public static final long OFFSET_LAG_MAX_DEFAULT = 100L;
+
+    protected static final String SOURCE_CLUSTER_PREFIX = MirrorMakerConfig.SOURCE_CLUSTER_PREFIX;
+    protected static final String TARGET_CLUSTER_PREFIX = MirrorMakerConfig.TARGET_CLUSTER_PREFIX;
+    protected static final String PRODUCER_CLIENT_PREFIX = "producer.";
+    protected static final String CONSUMER_CLIENT_PREFIX = "consumer.";
+    protected static final String ADMIN_CLIENT_PREFIX = "admin.";
+    protected static final String SOURCE_ADMIN_CLIENT_PREFIX = "source.admin.";
+    protected static final String TARGET_ADMIN_CLIENT_PREFIX = "target.admin.";
+
+    public MirrorConnectorConfig(Map props) {
+        this(CONNECTOR_CONFIG_DEF, props);
+    }
+
+    protected MirrorConnectorConfig(ConfigDef configDef, Map props) {
+        super(configDef, props, true);
+    }
+
+    String connectorName() {
+        return getString(ConnectorConfig.NAME_CONFIG);
+    }
+
+    boolean enabled() {
+        return getBoolean(ENABLED);
+    }
+
+    Duration consumerPollTimeout() {
+        return Duration.ofMillis(getLong(CONSUMER_POLL_TIMEOUT_MILLIS));
+    }
+
+    Duration adminTimeout() {
+        return Duration.ofMillis(getLong(ADMIN_TASK_TIMEOUT_MILLIS));
+    }
+
+    Map sourceProducerConfig() {
+        Map props = new HashMap<>();
+        props.putAll(originalsWithPrefix(SOURCE_CLUSTER_PREFIX));
+        props.keySet().retainAll(MirrorClientConfig.CLIENT_CONFIG_DEF.names());
+        props.putAll(originalsWithPrefix(PRODUCER_CLIENT_PREFIX));
+        return props;
+    }
+
+    Map sourceConsumerConfig() {
+        Map props = new HashMap<>();
+        props.putAll(originalsWithPrefix(SOURCE_CLUSTER_PREFIX));
+        props.keySet().retainAll(MirrorClientConfig.CLIENT_CONFIG_DEF.names());
+        props.putAll(originalsWithPrefix(CONSUMER_CLIENT_PREFIX));
+        props.put("enable.auto.commit", "false");
+        props.put("auto.offset.reset", "earliest");
+        return props;
+    }
+
+    Map taskConfigForTopicPartitions(List topicPartitions) {
+        Map props = originalsStrings();
+        String topicPartitionsString = topicPartitions.stream()
+                .map(MirrorUtils::encodeTopicPartition)
+                .collect(Collectors.joining(","));
+        props.put(TASK_TOPIC_PARTITIONS, topicPartitionsString);
+        return props;
+    }
+
+    Map taskConfigForConsumerGroups(List groups) {
+        Map props = originalsStrings();
+        props.put(TASK_CONSUMER_GROUPS, String.join(",", groups));
+        return props;
+    }
+
+    Map targetAdminConfig() {
+        Map props = new HashMap<>();
+        props.putAll(originalsWithPrefix(TARGET_CLUSTER_PREFIX));
+        props.keySet().retainAll(MirrorClientConfig.CLIENT_CONFIG_DEF.names());
+        props.putAll(originalsWithPrefix(ADMIN_CLIENT_PREFIX));
+        props.putAll(originalsWithPrefix(TARGET_ADMIN_CLIENT_PREFIX));
+        return props;
+    }
+
+    Map sourceAdminConfig() {
+        Map props = new HashMap<>();
+        props.putAll(originalsWithPrefix(SOURCE_CLUSTER_PREFIX));
+        props.keySet().retainAll(MirrorClientConfig.CLIENT_CONFIG_DEF.names());
+        props.putAll(originalsWithPrefix(ADMIN_CLIENT_PREFIX));
+        props.putAll(originalsWithPrefix(SOURCE_ADMIN_CLIENT_PREFIX));
+        return props;
+    }
+
+    List metricsReporters() {
+        List reporters = getConfiguredInstances(
+                CommonClientConfigs.METRIC_REPORTER_CLASSES_CONFIG, MetricsReporter.class);
+        reporters.add(new JmxReporter("kafka.connect.mirror"));
+        return reporters;
+    }
+
+    String sourceClusterAlias() {
+        return getString(SOURCE_CLUSTER_ALIAS);
+    }
+
+    String targetClusterAlias() {
+        return getString(TARGET_CLUSTER_ALIAS);
+    }
+
+    String offsetSyncsTopic() {
+        // ".internal" suffix ensures this doesn't get replicated
+        return "mm2-offset-syncs." + targetClusterAlias() + ".internal";
+    }
+
+    String heartbeatsTopic() {
+        return MirrorClientConfig.HEARTBEATS_TOPIC;
+    }
+
+    // e.g. source1.heartbeats
+    String targetHeartbeatsTopic() {
+        return replicationPolicy().formatRemoteTopic(sourceClusterAlias(), heartbeatsTopic());
+    }
+
+    String checkpointsTopic() {
+        // Checkpoint topics are not "remote topics", as they are not replicated, so we don't
+        // need to use ReplicationPolicy here.
+        return sourceClusterAlias() + MirrorClientConfig.CHECKPOINTS_TOPIC_SUFFIX;
+    }
+
+    long maxOffsetLag() {
+        return getLong(OFFSET_LAG_MAX);
+    }
+
+    Duration emitHeartbeatsInterval() {
+        if (getBoolean(EMIT_HEARTBEATS_ENABLED)) {
+            return Duration.ofSeconds(getLong(EMIT_HEARTBEATS_INTERVAL_SECONDS));
+        } else {
+            // negative interval to disable
+            return Duration.ofMillis(-1);
+        }
+    }
+
+    Duration emitCheckpointsInterval() {
+        if (getBoolean(EMIT_CHECKPOINTS_ENABLED)) {
+            return Duration.ofSeconds(getLong(EMIT_CHECKPOINTS_INTERVAL_SECONDS));
+        } else {
+            // negative interval to disable
+            return Duration.ofMillis(-1);
+        }
+    }
+
+    Duration refreshTopicsInterval() {
+        if (getBoolean(REFRESH_TOPICS_ENABLED)) {
+            return Duration.ofSeconds(getLong(REFRESH_TOPICS_INTERVAL_SECONDS));
+        } else {
+            // negative interval to disable
+            return Duration.ofMillis(-1);
+        }
+    }
+
+    Duration refreshGroupsInterval() {
+        if (getBoolean(REFRESH_GROUPS_ENABLED)) {
+            return Duration.ofSeconds(getLong(REFRESH_GROUPS_INTERVAL_SECONDS));
+        } else {
+            // negative interval to disable
+            return Duration.ofMillis(-1);
+        }
+    }
+
+    Duration syncTopicConfigsInterval() {
+        if (getBoolean(SYNC_TOPIC_CONFIGS_ENABLED)) {
+            return Duration.ofSeconds(getLong(SYNC_TOPIC_CONFIGS_INTERVAL_SECONDS));
+        } else {
+            // negative interval to disable
+            return Duration.ofMillis(-1);
+        }
+    }
+
+    Duration syncTopicAclsInterval() {
+        if (getBoolean(SYNC_TOPIC_ACLS_ENABLED)) {
+            return Duration.ofSeconds(getLong(SYNC_TOPIC_ACLS_INTERVAL_SECONDS));
+        } else {
+            // negative interval to disable
+            return Duration.ofMillis(-1);
+        }
+    }
+
+    ReplicationPolicy replicationPolicy() {
+        return getConfiguredInstance(REPLICATION_POLICY_CLASS, ReplicationPolicy.class);
+    }
+
+    int replicationFactor() {
+        return getInt(REPLICATION_FACTOR);
+    }
+
+    short heartbeatsTopicReplicationFactor() {
+        return getShort(HEARTBEATS_TOPIC_REPLICATION_FACTOR);
+    }
+
+    short checkpointsTopicReplicationFactor() {
+        return getShort(CHECKPOINTS_TOPIC_REPLICATION_FACTOR);
+    }
+
+    short offsetSyncsTopicReplicationFactor() {
+        return getShort(OFFSET_SYNCS_TOPIC_REPLICATION_FACTOR);
+    }
+
+    TopicFilter topicFilter() {
+        return getConfiguredInstance(TOPIC_FILTER_CLASS, TopicFilter.class);
+    }
+
+    GroupFilter groupFilter() {
+        return getConfiguredInstance(GROUP_FILTER_CLASS, GroupFilter.class);
+    }
+
+    ConfigPropertyFilter configPropertyFilter() {
+        return getConfiguredInstance(CONFIG_PROPERTY_FILTER_CLASS, ConfigPropertyFilter.class);
+    }
+
+    protected static final ConfigDef CONNECTOR_CONFIG_DEF = ConnectorConfig.configDef()
+            .define(
+                    ENABLED,
+                    ConfigDef.Type.BOOLEAN,
+                    true,
+                    ConfigDef.Importance.LOW,
+                    ENABLED_DOC)
+            .define(
+                    TOPICS,
+                    ConfigDef.Type.LIST,
+                    TOPICS_DEFAULT,
+                    ConfigDef.Importance.HIGH,
+                    TOPICS_DOC) 
+            .define(
+                    TOPICS_BLACKLIST,
+                    ConfigDef.Type.LIST,
+                    TOPICS_BLACKLIST_DEFAULT,
+                    ConfigDef.Importance.HIGH,
+                    TOPICS_BLACKLIST_DOC)
+            .define(
+                    GROUPS,
+                    ConfigDef.Type.LIST,
+                    GROUPS_DEFAULT,
+                    ConfigDef.Importance.HIGH,
+                    GROUPS_DOC) 
+            .define(
+                    GROUPS_BLACKLIST,
+                    ConfigDef.Type.LIST,
+                    GROUPS_BLACKLIST_DEFAULT,
+                    ConfigDef.Importance.HIGH,
+                    GROUPS_BLACKLIST_DOC)
+            .define(
+                    CONFIG_PROPERTIES_BLACKLIST,
+                    ConfigDef.Type.LIST,
+                    CONFIG_PROPERTIES_BLACKLIST_DEFAULT,
+                    ConfigDef.Importance.HIGH,
+                    CONFIG_PROPERTIES_BLACKLIST_DOC)
+            .define(
+                    TOPIC_FILTER_CLASS,
+                    ConfigDef.Type.CLASS,
+                    TOPIC_FILTER_CLASS_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    TOPIC_FILTER_CLASS_DOC)
+            .define(
+                    GROUP_FILTER_CLASS,
+                    ConfigDef.Type.CLASS,
+                    GROUP_FILTER_CLASS_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    GROUP_FILTER_CLASS_DOC)
+            .define(
+                    CONFIG_PROPERTY_FILTER_CLASS,
+                    ConfigDef.Type.CLASS,
+                    CONFIG_PROPERTY_FILTER_CLASS_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    CONFIG_PROPERTY_FILTER_CLASS_DOC)
+            .define(
+                    SOURCE_CLUSTER_ALIAS,
+                    ConfigDef.Type.STRING,
+                    ConfigDef.Importance.HIGH,
+                    SOURCE_CLUSTER_ALIAS_DOC)
+            .define(
+                    TARGET_CLUSTER_ALIAS,
+                    ConfigDef.Type.STRING,
+                    TARGET_CLUSTER_ALIAS_DEFAULT,
+                    ConfigDef.Importance.HIGH,
+                    TARGET_CLUSTER_ALIAS_DOC)
+            .define(
+                    CONSUMER_POLL_TIMEOUT_MILLIS,
+                    ConfigDef.Type.LONG,
+                    CONSUMER_POLL_TIMEOUT_MILLIS_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    CONSUMER_POLL_TIMEOUT_MILLIS_DOC)
+            .define(
+                    ADMIN_TASK_TIMEOUT_MILLIS,
+                    ConfigDef.Type.LONG,
+                    ADMIN_TASK_TIMEOUT_MILLIS_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    ADMIN_TASK_TIMEOUT_MILLIS_DOC)
+            .define(
+                    REFRESH_TOPICS_ENABLED,
+                    ConfigDef.Type.BOOLEAN,
+                    REFRESH_TOPICS_ENABLED_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    REFRESH_TOPICS_ENABLED_DOC)
+            .define(
+                    REFRESH_TOPICS_INTERVAL_SECONDS,
+                    ConfigDef.Type.LONG,
+                    REFRESH_TOPICS_INTERVAL_SECONDS_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    REFRESH_TOPICS_INTERVAL_SECONDS_DOC)
+            .define(
+                    REFRESH_GROUPS_ENABLED,
+                    ConfigDef.Type.BOOLEAN,
+                    REFRESH_GROUPS_ENABLED_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    REFRESH_GROUPS_ENABLED_DOC)
+            .define(
+                    REFRESH_GROUPS_INTERVAL_SECONDS,
+                    ConfigDef.Type.LONG,
+                    REFRESH_GROUPS_INTERVAL_SECONDS_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    REFRESH_GROUPS_INTERVAL_SECONDS_DOC)
+            .define(
+                    SYNC_TOPIC_CONFIGS_ENABLED,
+                    ConfigDef.Type.BOOLEAN,
+                    SYNC_TOPIC_CONFIGS_ENABLED_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    SYNC_TOPIC_CONFIGS_ENABLED_DOC)
+            .define(
+                    SYNC_TOPIC_CONFIGS_INTERVAL_SECONDS,
+                    ConfigDef.Type.LONG,
+                    SYNC_TOPIC_CONFIGS_INTERVAL_SECONDS_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    SYNC_TOPIC_CONFIGS_INTERVAL_SECONDS_DOC)
+            .define(
+                    SYNC_TOPIC_ACLS_ENABLED,
+                    ConfigDef.Type.BOOLEAN,
+                    SYNC_TOPIC_ACLS_ENABLED_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    SYNC_TOPIC_ACLS_ENABLED_DOC)
+            .define(
+                    SYNC_TOPIC_ACLS_INTERVAL_SECONDS,
+                    ConfigDef.Type.LONG,
+                    SYNC_TOPIC_ACLS_INTERVAL_SECONDS_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    SYNC_TOPIC_ACLS_INTERVAL_SECONDS_DOC)
+            .define(
+                    EMIT_HEARTBEATS_ENABLED,
+                    ConfigDef.Type.BOOLEAN,
+                    EMIT_HEARTBEATS_ENABLED_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    EMIT_HEARTBEATS_ENABLED_DOC)
+            .define(
+                    EMIT_HEARTBEATS_INTERVAL_SECONDS,
+                    ConfigDef.Type.LONG,
+                    EMIT_HEARTBEATS_INTERVAL_SECONDS_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    EMIT_HEARTBEATS_INTERVAL_SECONDS_DOC)
+            .define(
+                    EMIT_CHECKPOINTS_ENABLED,
+                    ConfigDef.Type.BOOLEAN,
+                    EMIT_CHECKPOINTS_ENABLED_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    EMIT_CHECKPOINTS_ENABLED_DOC)
+            .define(
+                    EMIT_CHECKPOINTS_INTERVAL_SECONDS,
+                    ConfigDef.Type.LONG,
+                    EMIT_CHECKPOINTS_INTERVAL_SECONDS_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    EMIT_CHECKPOINTS_INTERVAL_SECONDS_DOC)
+            .define(
+                    REPLICATION_POLICY_CLASS,
+                    ConfigDef.Type.CLASS,
+                    REPLICATION_POLICY_CLASS_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    REPLICATION_POLICY_CLASS_DOC)
+            .define(
+                    REPLICATION_POLICY_SEPARATOR,
+                    ConfigDef.Type.STRING,
+                    REPLICATION_POLICY_SEPARATOR_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    REPLICATION_POLICY_SEPARATOR_DOC)
+            .define(
+                    REPLICATION_FACTOR,
+                    ConfigDef.Type.INT,
+                    REPLICATION_FACTOR_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    REPLICATION_FACTOR_DOC)
+            .define(
+                    HEARTBEATS_TOPIC_REPLICATION_FACTOR,
+                    ConfigDef.Type.SHORT,
+                    HEARTBEATS_TOPIC_REPLICATION_FACTOR_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    HEARTBEATS_TOPIC_REPLICATION_FACTOR_DOC)
+            .define(
+                    CHECKPOINTS_TOPIC_REPLICATION_FACTOR,
+                    ConfigDef.Type.SHORT,
+                    CHECKPOINTS_TOPIC_REPLICATION_FACTOR_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    CHECKPOINTS_TOPIC_REPLICATION_FACTOR_DOC)
+            .define(
+                    OFFSET_SYNCS_TOPIC_REPLICATION_FACTOR,
+                    ConfigDef.Type.SHORT,
+                    OFFSET_SYNCS_TOPIC_REPLICATION_FACTOR_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    OFFSET_SYNCS_TOPIC_REPLICATION_FACTOR_DOC)
+            .define(
+                    OFFSET_LAG_MAX,
+                    ConfigDef.Type.LONG,
+                    OFFSET_LAG_MAX_DEFAULT,
+                    ConfigDef.Importance.LOW,
+                    OFFSET_LAG_MAX_DOC)
+            .define(
+                    CommonClientConfigs.METRIC_REPORTER_CLASSES_CONFIG,
+                    ConfigDef.Type.LIST,
+                    null,
+                    ConfigDef.Importance.LOW,
+                    CommonClientConfigs.METRIC_REPORTER_CLASSES_DOC)
+            .define(
+                    CommonClientConfigs.SECURITY_PROTOCOL_CONFIG,
+                    ConfigDef.Type.STRING,
+                    CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL,
+                    ConfigDef.Importance.MEDIUM,
+                    CommonClientConfigs.SECURITY_PROTOCOL_DOC)
+            .withClientSslSupport()
+            .withClientSaslSupport();
+}
diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorHeartbeatConnector.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorHeartbeatConnector.java
new file mode 100644
index 0000000000000..3942c8454ab72
--- /dev/null
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorHeartbeatConnector.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.connect.connector.Task;
+import org.apache.kafka.connect.source.SourceConnector;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.utils.Utils;
+
+import java.util.Map;
+import java.util.List;
+import java.util.Collections;
+
+/** Emits heartbeats to Kafka.
+ */
+public class MirrorHeartbeatConnector extends SourceConnector {
+    private MirrorConnectorConfig config;
+    private Scheduler scheduler;
+
+    @Override
+    public void start(Map props) {
+        config = new MirrorConnectorConfig(props);
+        scheduler = new Scheduler(MirrorHeartbeatConnector.class, config.adminTimeout());
+        scheduler.execute(this::createInternalTopics, "creating internal topics");
+    }
+
+    @Override
+    public void stop() {
+        Utils.closeQuietly(scheduler, "scheduler");
+    }
+
+    @Override
+    public Class taskClass() {
+        return MirrorHeartbeatTask.class;
+    }
+
+    @Override
+    public List> taskConfigs(int maxTasks) {
+        // just need a single task
+        return Collections.singletonList(config.originalsStrings());
+    }
+
+    @Override
+    public ConfigDef config() {
+        return MirrorConnectorConfig.CONNECTOR_CONFIG_DEF;
+    }
+
+    @Override
+    public String version() {
+        return "1";
+    }
+
+    private void createInternalTopics() {
+        MirrorUtils.createSinglePartitionCompactedTopic(config.heartbeatsTopic(),
+            config.heartbeatsTopicReplicationFactor(), config.targetAdminConfig());
+    }
+}
diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorHeartbeatTask.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorHeartbeatTask.java
new file mode 100644
index 0000000000000..6bfe441512450
--- /dev/null
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorHeartbeatTask.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.connect.source.SourceTask;
+import org.apache.kafka.connect.source.SourceRecord;
+import org.apache.kafka.connect.data.Schema;
+
+import java.util.Map;
+import java.util.List;
+import java.util.Collections;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.time.Duration;
+
+/** Emits heartbeats. */
+public class MirrorHeartbeatTask extends SourceTask {
+    private String sourceClusterAlias;
+    private String targetClusterAlias;
+    private String heartbeatsTopic;
+    private Duration interval;
+    private CountDownLatch stopped;
+
+    @Override
+    public void start(Map props) {
+        stopped = new CountDownLatch(1);
+        MirrorTaskConfig config = new MirrorTaskConfig(props);
+        sourceClusterAlias = config.sourceClusterAlias();
+        targetClusterAlias = config.targetClusterAlias();
+        heartbeatsTopic = config.heartbeatsTopic();
+        interval = config.emitHeartbeatsInterval();
+    }
+
+    @Override
+    public void commit() throws InterruptedException {
+        // nop
+    }
+
+    @Override
+    public void stop() {
+        stopped.countDown();
+    }
+
+    @Override
+    public String version() {
+        return "1";
+    }
+
+    @Override
+    public List poll() throws InterruptedException {
+        // pause to throttle, unless we've stopped
+        if (stopped.await(interval.toMillis(), TimeUnit.MILLISECONDS)) {
+            // SourceWorkerTask expects non-zero batches or null
+            return null;
+        }
+        long timestamp = System.currentTimeMillis();
+        Heartbeat heartbeat = new Heartbeat(sourceClusterAlias, targetClusterAlias, timestamp);
+        SourceRecord record = new SourceRecord(
+            heartbeat.connectPartition(), MirrorUtils.wrapOffset(0),
+            heartbeatsTopic, 0,
+            Schema.BYTES_SCHEMA, heartbeat.recordKey(),
+            Schema.BYTES_SCHEMA, heartbeat.recordValue(),
+            timestamp);
+        return Collections.singletonList(record);
+    }
+
+    @Override
+    public void commitRecord(SourceRecord record) {
+    }
+}
diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorMaker.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorMaker.java
new file mode 100644
index 0000000000000..d635b1c8c3e62
--- /dev/null
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorMaker.java
@@ -0,0 +1,309 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.common.utils.Exit;
+import org.apache.kafka.connect.runtime.Herder;
+import org.apache.kafka.connect.runtime.isolation.Plugins;
+import org.apache.kafka.connect.runtime.Worker;
+import org.apache.kafka.connect.runtime.WorkerConfigTransformer;
+import org.apache.kafka.connect.runtime.distributed.DistributedConfig;
+import org.apache.kafka.connect.runtime.distributed.DistributedHerder;
+import org.apache.kafka.connect.runtime.distributed.NotLeaderException;
+import org.apache.kafka.connect.storage.KafkaOffsetBackingStore;
+import org.apache.kafka.connect.storage.StatusBackingStore;
+import org.apache.kafka.connect.storage.KafkaStatusBackingStore;
+import org.apache.kafka.connect.storage.ConfigBackingStore;
+import org.apache.kafka.connect.storage.KafkaConfigBackingStore;
+import org.apache.kafka.connect.storage.Converter;
+import org.apache.kafka.connect.util.ConnectUtils;
+import org.apache.kafka.connect.connector.policy.AllConnectorClientConfigOverridePolicy;
+import org.apache.kafka.connect.connector.policy.ConnectorClientConfigOverridePolicy;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import net.sourceforge.argparse4j.impl.Arguments;
+import net.sourceforge.argparse4j.inf.Namespace;
+import net.sourceforge.argparse4j.inf.ArgumentParser;
+import net.sourceforge.argparse4j.inf.ArgumentParserException;
+import net.sourceforge.argparse4j.ArgumentParsers;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Arrays;
+import java.util.Properties;
+import java.util.stream.Collectors;
+import java.io.File;
+
+/**
+ *  Entry point for "MirrorMaker 2.0".
+ *  
                
+ *  MirrorMaker runs a set of Connectors between multiple clusters, in order to replicate data, configuration,
+ *  ACL rules, and consumer group state.
+ *  
                
+ *  
                
+ *  Configuration is via a top-level "mm2.properties" file, which supports per-cluster and per-replication
+ *  sub-configs. Each source->target replication must be explicitly enabled. For example:
+ *  
                
+ *  
                + *    clusters = primary, backup
+ *    primary.bootstrap.servers = vip1:9092
+ *    backup.bootstrap.servers = vip2:9092
+ *    primary->backup.enabled = true
+ *    backup->primary.enabled = true
+ *  
                
+ *  
                
+ *  Run as follows:
+ *  
                
+ *  
                + *    ./bin/connect-mirror-maker.sh mm2.properties
+ *  
                
+ *  
                
+ *  Additional information and example configurations are provided in ./connect/mirror/README.md
+ *  
                
+ */
+public class MirrorMaker {
+    private static final Logger log = LoggerFactory.getLogger(MirrorMaker.class);
+
+    private static final long SHUTDOWN_TIMEOUT_SECONDS = 60L;
+    private static final ConnectorClientConfigOverridePolicy CLIENT_CONFIG_OVERRIDE_POLICY =
+            new AllConnectorClientConfigOverridePolicy();
+
+    private static final List CONNECTOR_CLASSES = Arrays.asList(
+        MirrorSourceConnector.class,
+        MirrorHeartbeatConnector.class,
+        MirrorCheckpointConnector.class);
+ 
+    private final Map herders = new HashMap<>();
+    private CountDownLatch startLatch;
+    private CountDownLatch stopLatch;
+    private final AtomicBoolean shutdown = new AtomicBoolean(false);
+    private final ShutdownHook shutdownHook;
+    private final String advertisedBaseUrl;
+    private final Time time;
+    private final MirrorMakerConfig config;
+    private final Set clusters;
+    private final Set herderPairs;
+
+    /**
+     * @param config    MM2 configuration from mm2.properties file
+     * @param clusters  target clusters for this node. These must match cluster
+     *                  aliases as defined in the config. If null or empty list,
+     *                  uses all clusters in the config.
+     * @param time      time source
+     */
+    public MirrorMaker(MirrorMakerConfig config, List clusters, Time time) {
+        log.debug("Kafka MirrorMaker instance created");
+        this.time = time;
+        this.advertisedBaseUrl = "NOTUSED";
+        this.config = config;
+        if (clusters != null && !clusters.isEmpty()) {
+            this.clusters = new HashSet<>(clusters);
+        } else {
+            // default to all clusters
+            this.clusters = config.clusters();
+        }
+        log.info("Targeting clusters {}", this.clusters);
+        this.herderPairs = config.clusterPairs().stream()
+            .filter(x -> this.clusters.contains(x.target()))
+            .collect(Collectors.toSet());
+        if (herderPairs.isEmpty()) {
+            throw new IllegalArgumentException("No source->target replication flows.");
+        }
+        this.herderPairs.forEach(x -> addHerder(x));
+        shutdownHook = new ShutdownHook();
+    }
+
+    /**
+     * @param config    MM2 configuration from mm2.properties file
+     * @param clusters  target clusters for this node. These must match cluster
+     *                  aliases as defined in the config. If null or empty list,
+     *                  uses all clusters in the config.
+     * @param time      time source
+     */
+    public MirrorMaker(Map config, List clusters, Time time) {
+        this(new MirrorMakerConfig(config), clusters, time);
+    }
+
+    public MirrorMaker(Map props, List clusters) {
+        this(props, clusters, Time.SYSTEM);
+    }
+
+    public MirrorMaker(Map props) {
+        this(props, null);
+    }
+
+
+    public void start() {
+        log.info("Kafka MirrorMaker starting with {} herders.", herders.size());
+        if (startLatch != null) {
+            throw new IllegalStateException("MirrorMaker instance already started");
+        }
+        startLatch = new CountDownLatch(herders.size());
+        stopLatch = new CountDownLatch(herders.size());
+        Runtime.getRuntime().addShutdownHook(shutdownHook);
+        for (Herder herder : herders.values()) {
+            try {
+                herder.start();
+            } finally {
+                startLatch.countDown();
+            }
+        }
+        log.info("Configuring connectors...");
+        herderPairs.forEach(x -> configureConnectors(x));
+        log.info("Kafka MirrorMaker started");
+    }
+
+    public void stop() {
+        boolean wasShuttingDown = shutdown.getAndSet(true);
+        if (!wasShuttingDown) {
+            log.info("Kafka MirrorMaker stopping");
+            for (Herder herder : herders.values()) {
+                try {
+                    herder.stop();
+                } finally {
+                    stopLatch.countDown();
+                }
+            }
+            log.info("Kafka MirrorMaker stopped.");
+        }
+    }
+
+    public void awaitStop() {
+        try {
+            stopLatch.await();
+        } catch (InterruptedException e) {
+            log.error("Interrupted waiting for MirrorMaker to shutdown");
+        }
+    }
+
+    private void configureConnector(SourceAndTarget sourceAndTarget, Class connectorClass) {
+        checkHerder(sourceAndTarget);
+        Map connectorProps = config.connectorBaseConfig(sourceAndTarget, connectorClass);
+        herders.get(sourceAndTarget)
+                .putConnectorConfig(connectorClass.getSimpleName(), connectorProps, true, (e, x) -> {
+                    if (e instanceof NotLeaderException) {
+                        // No way to determine if the connector is a leader or not beforehand.
+                        log.info("Connector {} is a follower. Using existing configuration.", sourceAndTarget);
+                    } else {
+                        log.info("Connector {} configured.", sourceAndTarget, e);
+                    }
+                });
+    }
+
+    private void checkHerder(SourceAndTarget sourceAndTarget) {
+        if (!herders.containsKey(sourceAndTarget)) {
+            throw new IllegalArgumentException("No herder for " + sourceAndTarget.toString());
+        }
+    }
+
+    private void configureConnectors(SourceAndTarget sourceAndTarget) {
+        CONNECTOR_CLASSES.forEach(x -> configureConnector(sourceAndTarget, x));
+    }
+
+    private void addHerder(SourceAndTarget sourceAndTarget) {
+        log.info("creating herder for " + sourceAndTarget.toString());
+        Map workerProps = config.workerConfig(sourceAndTarget);
+        String advertisedUrl = advertisedBaseUrl + "/" + sourceAndTarget.source();
+        String workerId = sourceAndTarget.toString();
+        Plugins plugins = new Plugins(workerProps);
+        plugins.compareAndSwapWithDelegatingLoader();
+        DistributedConfig distributedConfig = new DistributedConfig(workerProps);
+        String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(distributedConfig);
+        KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore();
+        offsetBackingStore.configure(distributedConfig);
+        Worker worker = new Worker(workerId, time, plugins, distributedConfig, offsetBackingStore, CLIENT_CONFIG_OVERRIDE_POLICY);
+        WorkerConfigTransformer configTransformer = worker.configTransformer();
+        Converter internalValueConverter = worker.getInternalValueConverter();
+        StatusBackingStore statusBackingStore = new KafkaStatusBackingStore(time, internalValueConverter);
+        statusBackingStore.configure(distributedConfig);
+        ConfigBackingStore configBackingStore = new KafkaConfigBackingStore(
+                internalValueConverter,
+                distributedConfig,
+                configTransformer);
+        Herder herder = new DistributedHerder(distributedConfig, time, worker,
+                kafkaClusterId, statusBackingStore, configBackingStore,
+                advertisedUrl, CLIENT_CONFIG_OVERRIDE_POLICY);
+        herders.put(sourceAndTarget, herder);
+    }
+
+    private class ShutdownHook extends Thread {
+        @Override
+        public void run() {
+            try {
+                if (!startLatch.await(SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
+                    log.error("Timed out in shutdown hook waiting for MirrorMaker startup to finish. Unable to shutdown cleanly.");
+                }
+            } catch (InterruptedException e) {
+                log.error("Interrupted in shutdown hook while waiting for MirrorMaker startup to finish. Unable to shutdown cleanly.");
+            } finally {
+                MirrorMaker.this.stop();
+            }
+        }
+    }
+
+    public static void main(String[] args) {
+        ArgumentParser parser = ArgumentParsers.newArgumentParser("connect-mirror-maker");
+        parser.description("MirrorMaker 2.0 driver");
+        parser.addArgument("config").type(Arguments.fileType().verifyCanRead())
+            .metavar("mm2.properties").required(true)
+            .help("MM2 configuration file.");
+        parser.addArgument("--clusters").nargs("+").metavar("CLUSTER").required(false)
+            .help("Target cluster to use for this node.");
+        Namespace ns;
+        try {
+            ns = parser.parseArgs(args);
+        } catch (ArgumentParserException e) {
+            parser.handleError(e);
+            System.exit(-1);
+            return;
+        }
+        File configFile = (File) ns.get("config");
+        List clusters = ns.getList("clusters");
+        try {
+            log.info("Kafka MirrorMaker initializing ...");
+
+            Properties props = Utils.loadProps(configFile.getPath());
+            Map config = Utils.propsToStringMap(props);
+            MirrorMaker mirrorMaker = new MirrorMaker(config, clusters, Time.SYSTEM);
+            
+            try {
+                mirrorMaker.start();
+            } catch (Exception e) {
+                log.error("Failed to start MirrorMaker", e);
+                mirrorMaker.stop();
+                Exit.exit(3);
+            }
+
+            mirrorMaker.awaitStop();
+
+        } catch (Throwable t) {
+            log.error("Stopping due to error", t);
+            Exit.exit(2);
+        }
+    }
+
+}
diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorMakerConfig.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorMakerConfig.java
new file mode 100644
index 0000000000000..df5d38f7c53b2
--- /dev/null
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorMakerConfig.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.common.config.AbstractConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.config.ConfigDef.Type;
+import org.apache.kafka.common.config.ConfigDef.Importance;
+import org.apache.kafka.common.config.provider.ConfigProvider;
+import org.apache.kafka.common.config.ConfigTransformer;
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.connect.runtime.WorkerConfig;
+import org.apache.kafka.connect.runtime.distributed.DistributedConfig;
+import org.apache.kafka.connect.runtime.isolation.Plugins;
+
+import java.util.Map;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.stream.Collectors;
+
+/** Top-level config describing replication flows between multiple Kafka clusters.
+ *
+ *  Supports cluster-level properties of the form cluster.x.y.z, and replication-level
+ *  properties of the form source->target.x.y.z.
+ *  e.g.
+ *
+ *      clusters = A, B, C
+ *      A.bootstrap.servers = aaa:9092
+ *      A.security.protocol = SSL
+ *      --->%---
+ *      A->B.enabled = true
+ *      A->B.producer.client.id = "A-B-producer"
+ *      --->%---
+ *
+ */
+public class MirrorMakerConfig extends AbstractConfig {
+
+    public static final String CLUSTERS_CONFIG = "clusters";
+    private static final String CLUSTERS_DOC = "List of cluster aliases.";
+    public static final String CONFIG_PROVIDERS_CONFIG = WorkerConfig.CONFIG_PROVIDERS_CONFIG;
+    private static final String CONFIG_PROVIDERS_DOC = "Names of ConfigProviders to use.";
+
+    private static final String NAME = "name";
+    private static final String CONNECTOR_CLASS = "connector.class";
+    private static final String SOURCE_CLUSTER_ALIAS = "source.cluster.alias";
+    private static final String TARGET_CLUSTER_ALIAS = "target.cluster.alias";
+    private static final String GROUP_ID_CONFIG = "group.id";
+    private static final String KEY_CONVERTER_CLASS_CONFIG = "key.converter";
+    private static final String VALUE_CONVERTER_CLASS_CONFIG = "value.converter";
+    private static final String HEADER_CONVERTER_CLASS_CONFIG = "header.converter";
+    private static final String BYTE_ARRAY_CONVERTER_CLASS =
+        "org.apache.kafka.connect.converters.ByteArrayConverter";
+    private static final String REPLICATION_FACTOR = "replication.factor";
+
+    static final String SOURCE_CLUSTER_PREFIX = "source.cluster.";
+    static final String TARGET_CLUSTER_PREFIX = "target.cluster.";
+
+    private final Plugins plugins;
+   
+    public MirrorMakerConfig(Map props) {
+        super(CONFIG_DEF, props, true);
+        plugins = new Plugins(originalsStrings());
+    }
+
+    public Set clusters() {
+        return new HashSet<>(getList(CLUSTERS_CONFIG));
+    }
+
+    public List clusterPairs() {
+        List pairs = new ArrayList<>();
+        Set clusters = clusters();
+        for (String source : clusters) {
+            for (String target : clusters) {
+                SourceAndTarget sourceAndTarget = new SourceAndTarget(source, target);
+                if (!source.equals(target)) {
+                    pairs.add(sourceAndTarget);
+                }
+            }
+        }
+        return pairs;
+    }
+
+    /** Construct a MirrorClientConfig from properties of the form cluster.x.y.z.
+      * Use to connect to a cluster based on the MirrorMaker top-level config file.
+      */
+    public MirrorClientConfig clientConfig(String cluster) {
+        Map props = new HashMap<>();
+        props.putAll(originalsStrings());
+        props.putAll(clusterProps(cluster));
+        return new MirrorClientConfig(transform(props));
+    }
+
+    // loads properties of the form cluster.x.y.z
+    Map clusterProps(String cluster) {
+        Map props = new HashMap<>();
+        Map strings = originalsStrings();
+
+        props.putAll(stringsWithPrefixStripped(cluster + "."));
+
+        for (String k : MirrorClientConfig.CLIENT_CONFIG_DEF.names()) {
+            String v = props.get(k);
+            if (v != null) {
+                props.putIfAbsent("producer." + k, v);
+                props.putIfAbsent("consumer." + k, v);
+                props.putIfAbsent("admin." + k, v);
+            }
+        }
+
+        for (String k : MirrorClientConfig.CLIENT_CONFIG_DEF.names()) {
+            String v = strings.get(k);
+            if (v != null) {
+                props.putIfAbsent("producer." + k, v);
+                props.putIfAbsent("consumer." + k, v);
+                props.putIfAbsent("admin." + k, v);
+                props.putIfAbsent(k, v);
+            }
+        }
+ 
+        return props;
+    }
+
+    // loads worker configs based on properties of the form x.y.z and cluster.x.y.z 
+    Map workerConfig(SourceAndTarget sourceAndTarget) {
+        Map props = new HashMap<>();
+        props.putAll(clusterProps(sourceAndTarget.target()));
+      
+        // Accept common top-level configs that are otherwise ignored by MM2.
+        // N.B. all other worker properties should be configured for specific herders,
+        // e.g. primary->backup.client.id
+        props.putAll(stringsWithPrefix("offset.storage"));
+        props.putAll(stringsWithPrefix("config.storage"));
+        props.putAll(stringsWithPrefix("status.storage"));
+        props.putAll(stringsWithPrefix("key.converter")); 
+        props.putAll(stringsWithPrefix("value.converter")); 
+        props.putAll(stringsWithPrefix("header.converter"));
+        props.putAll(stringsWithPrefix("task"));
+        props.putAll(stringsWithPrefix("worker"));
+ 
+        // transform any expression like ${provider:path:key}, since the worker doesn't do so
+        props = transform(props);
+        props.putAll(stringsWithPrefix(CONFIG_PROVIDERS_CONFIG));
+
+        // fill in reasonable defaults
+        props.putIfAbsent(GROUP_ID_CONFIG, sourceAndTarget.source() + "-mm2");
+        props.putIfAbsent(DistributedConfig.OFFSET_STORAGE_TOPIC_CONFIG, "mm2-offsets."
+                + sourceAndTarget.source() + ".internal");
+        props.putIfAbsent(DistributedConfig.STATUS_STORAGE_TOPIC_CONFIG, "mm2-status."
+                + sourceAndTarget.source() + ".internal");
+        props.putIfAbsent(DistributedConfig.CONFIG_TOPIC_CONFIG, "mm2-configs."
+                + sourceAndTarget.source() + ".internal");
+        props.putIfAbsent(KEY_CONVERTER_CLASS_CONFIG, BYTE_ARRAY_CONVERTER_CLASS); 
+        props.putIfAbsent(VALUE_CONVERTER_CLASS_CONFIG, BYTE_ARRAY_CONVERTER_CLASS); 
+        props.putIfAbsent(HEADER_CONVERTER_CLASS_CONFIG, BYTE_ARRAY_CONVERTER_CLASS);
+
+        return props;
+    }
+
+    // loads properties of the form cluster.x.y.z and source->target.x.y.z
+    Map connectorBaseConfig(SourceAndTarget sourceAndTarget, Class connectorClass) {
+        Map props = new HashMap<>();
+
+        props.putAll(originalsStrings());
+        props.keySet().retainAll(MirrorConnectorConfig.CONNECTOR_CONFIG_DEF.names());
+        
+        props.putAll(stringsWithPrefix(CONFIG_PROVIDERS_CONFIG));
+        
+        props.putAll(withPrefix(SOURCE_CLUSTER_PREFIX, clusterProps(sourceAndTarget.source())));
+        props.putAll(withPrefix(TARGET_CLUSTER_PREFIX, clusterProps(sourceAndTarget.target())));
+
+        props.putIfAbsent(NAME, connectorClass.getSimpleName());
+        props.putIfAbsent(CONNECTOR_CLASS, connectorClass.getName());
+        props.putIfAbsent(SOURCE_CLUSTER_ALIAS, sourceAndTarget.source());
+        props.putIfAbsent(TARGET_CLUSTER_ALIAS, sourceAndTarget.target());
+
+        // override with connector-level properties
+        props.putAll(stringsWithPrefixStripped(sourceAndTarget.source() + "->"
+            + sourceAndTarget.target() + "."));
+
+        // disabled by default
+        props.putIfAbsent(MirrorConnectorConfig.ENABLED, "false");
+
+        // don't transform -- the worker will handle transformation of Connector and Task configs
+        return props;
+    }
+
+    List configProviders() {
+        return getList(CONFIG_PROVIDERS_CONFIG);
+    } 
+
+    Map transform(Map props) {
+        // transform worker config according to config.providers
+        List providerNames = configProviders();
+        Map providers = new HashMap<>();
+        for (String name : providerNames) {
+            ConfigProvider configProvider = plugins.newConfigProvider(
+                    this,
+                    CONFIG_PROVIDERS_CONFIG + "." + name,
+                    Plugins.ClassLoaderUsage.PLUGINS
+            );
+            providers.put(name, configProvider);
+        }
+        ConfigTransformer transformer = new ConfigTransformer(providers);
+        Map transformed = transformer.transform(props).data();
+        providers.values().forEach(x -> Utils.closeQuietly(x, "config provider"));
+        return transformed;
+    }
+ 
+    protected static final ConfigDef CONFIG_DEF = new ConfigDef()
+            .define(CLUSTERS_CONFIG, Type.LIST, Importance.HIGH, CLUSTERS_DOC)
+            .define(CONFIG_PROVIDERS_CONFIG, Type.LIST, Collections.emptyList(), Importance.LOW, CONFIG_PROVIDERS_DOC)
+            // security support
+            .define(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG,
+                Type.STRING,
+                CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL,
+                Importance.MEDIUM,
+                CommonClientConfigs.SECURITY_PROTOCOL_DOC)
+            .withClientSslSupport()
+            .withClientSaslSupport();
+
+    private Map stringsWithPrefixStripped(String prefix) {
+        return originalsStrings().entrySet().stream()
+            .filter(x -> x.getKey().startsWith(prefix))
+            .collect(Collectors.toMap(x -> x.getKey().substring(prefix.length()), x -> x.getValue()));
+    }
+
+    private Map stringsWithPrefix(String prefix) {
+        Map strings = originalsStrings();
+        strings.keySet().removeIf(x -> !x.startsWith(prefix));
+        return strings;
+    } 
+
+    static Map withPrefix(String prefix, Map props) {
+        return props.entrySet().stream()
+            .collect(Collectors.toMap(x -> prefix + x.getKey(), x -> x.getValue()));
+    }
+}
diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorMetrics.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorMetrics.java
new file mode 100644
index 0000000000000..51ddafc86a102
--- /dev/null
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorMetrics.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.MetricNameTemplate;
+import org.apache.kafka.common.metrics.Metrics;
+import org.apache.kafka.common.metrics.MetricsReporter;
+import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.metrics.stats.WindowedCount;
+import org.apache.kafka.common.metrics.stats.Value;
+import org.apache.kafka.common.metrics.stats.Rate;
+import org.apache.kafka.common.metrics.stats.Min;
+import org.apache.kafka.common.metrics.stats.Max;
+import org.apache.kafka.common.metrics.stats.Avg;
+import org.apache.kafka.common.TopicPartition;
+
+import java.util.Arrays;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.stream.Collectors;
+
+/** Metrics for replicated topic-partitions */
+class MirrorMetrics implements AutoCloseable {
+
+    private static final String SOURCE_CONNECTOR_GROUP = MirrorSourceConnector.class.getSimpleName();
+    private static final String CHECKPOINT_CONNECTOR_GROUP = MirrorCheckpointConnector.class.getSimpleName();
+
+    private static final Set PARTITION_TAGS = new HashSet<>(Arrays.asList("target", "topic", "partition"));
+    private static final Set GROUP_TAGS = new HashSet<>(Arrays.asList("source", "target", "group", "topic", "partition"));
+    
+    private static final MetricNameTemplate RECORD_COUNT = new MetricNameTemplate(
+            "record-count", SOURCE_CONNECTOR_GROUP,
+            "Number of source records replicated to the target cluster.", PARTITION_TAGS);
+    private static final MetricNameTemplate RECORD_AGE = new MetricNameTemplate(
+            "record-age-ms", SOURCE_CONNECTOR_GROUP,
+            "The age of incoming source records when replicated to the target cluster.", PARTITION_TAGS);
+    private static final MetricNameTemplate RECORD_AGE_MAX = new MetricNameTemplate(
+            "record-age-ms-max", SOURCE_CONNECTOR_GROUP,
+            "The max age of incoming source records when replicated to the target cluster.", PARTITION_TAGS);
+    private static final MetricNameTemplate RECORD_AGE_MIN = new MetricNameTemplate(
+            "record-age-ms-min", SOURCE_CONNECTOR_GROUP,
+            "The min age of incoming source records when replicated to the target cluster.", PARTITION_TAGS);
+    private static final MetricNameTemplate RECORD_AGE_AVG = new MetricNameTemplate(
+            "record-age-ms-avg", SOURCE_CONNECTOR_GROUP,
+            "The average age of incoming source records when replicated to the target cluster.", PARTITION_TAGS);
+    private static final MetricNameTemplate BYTE_RATE = new MetricNameTemplate(
+            "byte-rate", SOURCE_CONNECTOR_GROUP,
+            "Average number of bytes replicated per second.", PARTITION_TAGS);
+    private static final MetricNameTemplate REPLICATION_LATENCY = new MetricNameTemplate(
+            "replication-latency-ms", SOURCE_CONNECTOR_GROUP,
+            "Time it takes records to replicate from source to target cluster.", PARTITION_TAGS);
+    private static final MetricNameTemplate REPLICATION_LATENCY_MAX = new MetricNameTemplate(
+            "replication-latency-ms-max", SOURCE_CONNECTOR_GROUP,
+            "Max time it takes records to replicate from source to target cluster.", PARTITION_TAGS);
+    private static final MetricNameTemplate REPLICATION_LATENCY_MIN = new MetricNameTemplate(
+            "replication-latency-ms-min", SOURCE_CONNECTOR_GROUP,
+            "Min time it takes records to replicate from source to target cluster.", PARTITION_TAGS);
+    private static final MetricNameTemplate REPLICATION_LATENCY_AVG = new MetricNameTemplate(
+            "replication-latency-ms-avg", SOURCE_CONNECTOR_GROUP,
+            "Average time it takes records to replicate from source to target cluster.", PARTITION_TAGS);
+
+    private static final MetricNameTemplate CHECKPOINT_LATENCY = new MetricNameTemplate(
+            "checkpoint-latency-ms", CHECKPOINT_CONNECTOR_GROUP,
+            "Time it takes consumer group offsets to replicate from source to target cluster.", GROUP_TAGS);
+    private static final MetricNameTemplate CHECKPOINT_LATENCY_MAX = new MetricNameTemplate(
+            "checkpoint-latency-ms-max", CHECKPOINT_CONNECTOR_GROUP,
+            "Max time it takes consumer group offsets to replicate from source to target cluster.", GROUP_TAGS);
+    private static final MetricNameTemplate CHECKPOINT_LATENCY_MIN = new MetricNameTemplate(
+            "checkpoint-latency-ms-min", CHECKPOINT_CONNECTOR_GROUP,
+            "Min time it takes consumer group offsets to replicate from source to target cluster.", GROUP_TAGS);
+    private static final MetricNameTemplate CHECKPOINT_LATENCY_AVG = new MetricNameTemplate(
+            "checkpoint-latency-ms-avg", CHECKPOINT_CONNECTOR_GROUP,
+            "Average time it takes consumer group offsets to replicate from source to target cluster.", GROUP_TAGS);
+
+
+    private final Metrics metrics; 
+    private final Map partitionMetrics; 
+    private final Map groupMetrics = new HashMap<>();
+    private final String source;
+    private final String target;
+    private final Set groups;
+
+    MirrorMetrics(MirrorTaskConfig taskConfig) {
+        this.target = taskConfig.targetClusterAlias();
+        this.source = taskConfig.sourceClusterAlias();
+        this.groups = taskConfig.taskConsumerGroups();
+        this.metrics = new Metrics();
+
+        // for side-effect
+        metrics.sensor("record-count");
+        metrics.sensor("byte-rate");
+        metrics.sensor("record-age");
+        metrics.sensor("replication-latency");
+
+        ReplicationPolicy replicationPolicy = taskConfig.replicationPolicy();
+        partitionMetrics = taskConfig.taskTopicPartitions().stream()
+            .map(x -> new TopicPartition(replicationPolicy.formatRemoteTopic(source, x.topic()), x.partition()))
+            .collect(Collectors.toMap(x -> x, x -> new PartitionMetrics(x)));
+
+    }
+
+    @Override
+    public void close() {
+        metrics.close();
+    }
+
+    void countRecord(TopicPartition topicPartition) {
+        partitionMetrics.get(topicPartition).recordSensor.record();
+    }
+
+    void recordAge(TopicPartition topicPartition, long ageMillis) {
+        partitionMetrics.get(topicPartition).recordAgeSensor.record((double) ageMillis);
+    }
+
+    void replicationLatency(TopicPartition topicPartition, long millis) {
+        partitionMetrics.get(topicPartition).replicationLatencySensor.record((double) millis);
+    }
+
+    void recordBytes(TopicPartition topicPartition, long bytes) {
+        partitionMetrics.get(topicPartition).byteRateSensor.record((double) bytes);
+    }
+
+    void checkpointLatency(TopicPartition topicPartition, String group, long millis) {
+        group(topicPartition, group).checkpointLatencySensor.record((double) millis);
+    }
+
+    GroupMetrics group(TopicPartition topicPartition, String group) {
+        return groupMetrics.computeIfAbsent(String.join("-", topicPartition.toString(), group),
+            x -> new GroupMetrics(topicPartition, group));
+    }
+
+    void addReporter(MetricsReporter reporter) {
+        metrics.addReporter(reporter);
+    }
+
+    private class PartitionMetrics {
+        private final Sensor recordSensor;
+        private final Sensor byteRateSensor;
+        private final Sensor recordAgeSensor;
+        private final Sensor replicationLatencySensor;
+        private final TopicPartition topicPartition;
+     
+        PartitionMetrics(TopicPartition topicPartition) {
+            this.topicPartition = topicPartition;
+
+            Map tags = new LinkedHashMap<>();
+            tags.put("target", target); 
+            tags.put("topic", topicPartition.topic());
+            tags.put("partition", Integer.toString(topicPartition.partition()));
+
+            recordSensor = metrics.sensor("record-count");
+            recordSensor.add(metrics.metricInstance(RECORD_COUNT, tags), new WindowedCount());
+
+            byteRateSensor = metrics.sensor("byte-rate");
+            byteRateSensor.add(metrics.metricInstance(BYTE_RATE, tags), new Rate());
+
+            recordAgeSensor = metrics.sensor("record-age");
+            recordAgeSensor.add(metrics.metricInstance(RECORD_AGE, tags), new Value());
+            recordAgeSensor.add(metrics.metricInstance(RECORD_AGE_MAX, tags), new Max());
+            recordAgeSensor.add(metrics.metricInstance(RECORD_AGE_MIN, tags), new Min());
+            recordAgeSensor.add(metrics.metricInstance(RECORD_AGE_AVG, tags), new Avg());
+
+            replicationLatencySensor = metrics.sensor("replication-latency");
+            replicationLatencySensor.add(metrics.metricInstance(REPLICATION_LATENCY, tags), new Value());
+            replicationLatencySensor.add(metrics.metricInstance(REPLICATION_LATENCY_MAX, tags), new Max());
+            replicationLatencySensor.add(metrics.metricInstance(REPLICATION_LATENCY_MIN, tags), new Min());
+            replicationLatencySensor.add(metrics.metricInstance(REPLICATION_LATENCY_AVG, tags), new Avg());
+        }
+
+        
+    }
+
+    private class GroupMetrics {
+        private final Sensor checkpointLatencySensor;
+
+        GroupMetrics(TopicPartition topicPartition, String group) {
+            Map tags = new LinkedHashMap<>();
+            tags.put("source", source); 
+            tags.put("target", target); 
+            tags.put("group", group);
+            tags.put("topic", topicPartition.topic());
+            tags.put("partition", Integer.toString(topicPartition.partition()));
+ 
+            checkpointLatencySensor = metrics.sensor("checkpoint-latency");
+            checkpointLatencySensor.add(metrics.metricInstance(CHECKPOINT_LATENCY, tags), new Value());
+            checkpointLatencySensor.add(metrics.metricInstance(CHECKPOINT_LATENCY_MAX, tags), new Max());
+            checkpointLatencySensor.add(metrics.metricInstance(CHECKPOINT_LATENCY_MIN, tags), new Min());
+            checkpointLatencySensor.add(metrics.metricInstance(CHECKPOINT_LATENCY_AVG, tags), new Avg());
+        }
+    }
+}
diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceConnector.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceConnector.java
new file mode 100644
index 0000000000000..081bedc78a0f4
--- /dev/null
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceConnector.java
@@ -0,0 +1,390 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.connect.connector.Task;
+import org.apache.kafka.connect.source.SourceConnector;
+import org.apache.kafka.connect.util.ConnectorUtils;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.config.ConfigResource;
+import org.apache.kafka.common.acl.AclBinding;
+import org.apache.kafka.common.acl.AclBindingFilter;
+import org.apache.kafka.common.acl.AccessControlEntry;
+import org.apache.kafka.common.acl.AccessControlEntryFilter;
+import org.apache.kafka.common.acl.AclPermissionType;
+import org.apache.kafka.common.acl.AclOperation;
+import org.apache.kafka.common.resource.ResourceType;
+import org.apache.kafka.common.resource.ResourcePattern;
+import org.apache.kafka.common.resource.ResourcePatternFilter;
+import org.apache.kafka.common.resource.PatternType;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.InvalidPartitionsException;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.clients.admin.Config;
+import org.apache.kafka.clients.admin.ConfigEntry;
+import org.apache.kafka.clients.admin.NewPartitions;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.CreateTopicsOptions;
+
+import java.util.Map;
+import java.util.List;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.concurrent.ExecutionException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Replicate data, configuration, and ACLs between clusters.
+ *
+ *  @see MirrorConnectorConfig for supported config properties.
+ */
+public class MirrorSourceConnector extends SourceConnector {
+
+    private static final Logger log = LoggerFactory.getLogger(MirrorSourceConnector.class);
+    private static final ResourcePatternFilter ANY_TOPIC = new ResourcePatternFilter(ResourceType.TOPIC,
+        null, PatternType.ANY);
+    private static final AclBindingFilter ANY_TOPIC_ACL = new AclBindingFilter(ANY_TOPIC, AccessControlEntryFilter.ANY);
+
+    private Scheduler scheduler;
+    private MirrorConnectorConfig config;
+    private SourceAndTarget sourceAndTarget;
+    private String connectorName;
+    private TopicFilter topicFilter;
+    private ConfigPropertyFilter configPropertyFilter;
+    private List knownTopicPartitions = Collections.emptyList();
+    private Set knownTargetTopics = Collections.emptySet();
+    private ReplicationPolicy replicationPolicy;
+    private int replicationFactor;
+    private AdminClient sourceAdminClient;
+    private AdminClient targetAdminClient;
+
+    public MirrorSourceConnector() {
+        // nop
+    }
+
+    // visible for testing
+    MirrorSourceConnector(SourceAndTarget sourceAndTarget, ReplicationPolicy replicationPolicy,
+            TopicFilter topicFilter, ConfigPropertyFilter configPropertyFilter) {
+        this.sourceAndTarget = sourceAndTarget;
+        this.replicationPolicy = replicationPolicy;
+        this.topicFilter = topicFilter;
+        this.configPropertyFilter = configPropertyFilter;
+    } 
+
+    @Override
+    public void start(Map props) {
+        long start = System.currentTimeMillis();
+        config = new MirrorConnectorConfig(props);
+        if (!config.enabled()) {
+            return;
+        }
+        connectorName = config.connectorName();
+        sourceAndTarget = new SourceAndTarget(config.sourceClusterAlias(), config.targetClusterAlias());
+        topicFilter = config.topicFilter();
+        configPropertyFilter = config.configPropertyFilter();
+        replicationPolicy = config.replicationPolicy();
+        replicationFactor = config.replicationFactor();
+        sourceAdminClient = AdminClient.create(config.sourceAdminConfig());
+        targetAdminClient = AdminClient.create(config.targetAdminConfig());
+        scheduler = new Scheduler(MirrorSourceConnector.class, config.adminTimeout());
+        scheduler.execute(this::createOffsetSyncsTopic, "creating upstream offset-syncs topic");
+        scheduler.execute(this::loadTopicPartitions, "loading initial set of topic-partitions");
+        scheduler.execute(this::createTopicPartitions, "creating downstream topic-partitions");
+        scheduler.execute(this::refreshKnownTargetTopics, "refreshing known target topics");
+        scheduler.scheduleRepeating(this::syncTopicAcls, config.syncTopicAclsInterval(), "syncing topic ACLs");
+        scheduler.scheduleRepeating(this::syncTopicConfigs, config.syncTopicConfigsInterval(),
+            "syncing topic configs");
+        scheduler.scheduleRepeatingDelayed(this::refreshTopicPartitions, config.refreshTopicsInterval(),
+            "refreshing topics");
+        log.info("Started {} with {} topic-partitions.", connectorName, knownTopicPartitions.size());
+        log.info("Starting {} took {} ms.", connectorName, System.currentTimeMillis() - start);
+    }
+
+    @Override
+    public void stop() {
+        long start = System.currentTimeMillis();
+        if (!config.enabled()) {
+            return;
+        }
+        Utils.closeQuietly(scheduler, "scheduler");
+        Utils.closeQuietly(topicFilter, "topic filter");
+        Utils.closeQuietly(configPropertyFilter, "config property filter");
+        Utils.closeQuietly(sourceAdminClient, "source admin client");
+        Utils.closeQuietly(targetAdminClient, "target admin client");
+        log.info("Stopping {} took {} ms.", connectorName, System.currentTimeMillis() - start);
+    }
+
+    @Override
+    public Class taskClass() {
+        return MirrorSourceTask.class;
+    }
+
+    // divide topic-partitions among tasks
+    @Override
+    public List> taskConfigs(int maxTasks) {
+        if (!config.enabled() || knownTopicPartitions.isEmpty()) {
+            return Collections.emptyList();
+        }
+        int numTasks = Math.min(maxTasks, knownTopicPartitions.size());
+        return ConnectorUtils.groupPartitions(knownTopicPartitions, numTasks).stream()
+            .map(config::taskConfigForTopicPartitions)
+            .collect(Collectors.toList());
+    }
+
+    @Override
+    public ConfigDef config() {
+        return MirrorConnectorConfig.CONNECTOR_CONFIG_DEF;
+    }
+
+    @Override
+    public String version() {
+        return "1";
+    }
+
+    private List findTopicPartitions()
+            throws InterruptedException, ExecutionException {
+        Set topics = listTopics(sourceAdminClient).stream()
+            .filter(this::shouldReplicateTopic)
+            .collect(Collectors.toSet());
+        return describeTopics(topics).stream()
+            .flatMap(MirrorSourceConnector::expandTopicDescription)
+            .collect(Collectors.toList());
+    }
+
+    private void refreshTopicPartitions()
+            throws InterruptedException, ExecutionException {
+        List topicPartitions = findTopicPartitions();
+        Set newTopicPartitions = new HashSet<>();
+        newTopicPartitions.addAll(topicPartitions);
+        newTopicPartitions.removeAll(knownTopicPartitions);
+        Set deadTopicPartitions = new HashSet<>();
+        deadTopicPartitions.addAll(knownTopicPartitions);
+        deadTopicPartitions.removeAll(topicPartitions);
+        if (!newTopicPartitions.isEmpty() || !deadTopicPartitions.isEmpty()) {
+            log.info("Found {} topic-partitions on {}. {} are new. {} were removed. Previously had {}.",
+                    topicPartitions.size(), sourceAndTarget.source(), newTopicPartitions.size(), 
+                    deadTopicPartitions.size(), knownTopicPartitions.size());
+            log.trace("Found new topic-partitions: {}", newTopicPartitions);
+            knownTopicPartitions = topicPartitions;
+            knownTargetTopics = findExistingTargetTopics(); 
+            createTopicPartitions();
+            context.requestTaskReconfiguration();
+        }
+    }
+
+    private void loadTopicPartitions()
+            throws InterruptedException, ExecutionException {
+        knownTopicPartitions = findTopicPartitions();
+        knownTargetTopics = findExistingTargetTopics(); 
+    }
+
+    private void refreshKnownTargetTopics()
+            throws InterruptedException, ExecutionException {
+        knownTargetTopics = findExistingTargetTopics();
+    }
+
+    private Set findExistingTargetTopics()
+            throws InterruptedException, ExecutionException {
+        return listTopics(targetAdminClient).stream()
+            .filter(x -> sourceAndTarget.source().equals(replicationPolicy.topicSource(x)))
+            .collect(Collectors.toSet());
+    }
+
+    private Set topicsBeingReplicated() {
+        return knownTopicPartitions.stream()
+            .map(x -> x.topic())
+            .distinct()
+            .filter(x -> knownTargetTopics.contains(formatRemoteTopic(x)))
+            .collect(Collectors.toSet());
+    }
+
+    private void syncTopicAcls()
+            throws InterruptedException, ExecutionException {
+        List bindings = listTopicAclBindings().stream()
+            .filter(x -> x.pattern().resourceType() == ResourceType.TOPIC)
+            .filter(x -> x.pattern().patternType() == PatternType.LITERAL)
+            .filter(this::shouldReplicateAcl)
+            .filter(x -> shouldReplicateTopic(x.pattern().name()))
+            .map(this::targetAclBinding)
+            .collect(Collectors.toList());
+        updateTopicAcls(bindings);
+    }
+
+    private void syncTopicConfigs()
+            throws InterruptedException, ExecutionException {
+        Map sourceConfigs = describeTopicConfigs(topicsBeingReplicated());
+        Map targetConfigs = sourceConfigs.entrySet().stream()
+            .collect(Collectors.toMap(x -> formatRemoteTopic(x.getKey()), x -> targetConfig(x.getValue())));
+        updateTopicConfigs(targetConfigs);
+    }
+
+    private void createOffsetSyncsTopic() {
+        MirrorUtils.createSinglePartitionCompactedTopic(config.offsetSyncsTopic(), config.offsetSyncsTopicReplicationFactor(), config.sourceAdminConfig());
+    }
+
+    private void createTopicPartitions()
+            throws InterruptedException, ExecutionException {
+        Map partitionCounts = knownTopicPartitions.stream()
+            .collect(Collectors.groupingBy(x -> x.topic(), Collectors.counting())).entrySet().stream()
+            .collect(Collectors.toMap(x -> formatRemoteTopic(x.getKey()), x -> x.getValue()));
+        List newTopics = partitionCounts.entrySet().stream()
+            .filter(x -> !knownTargetTopics.contains(x.getKey()))
+            .map(x -> new NewTopic(x.getKey(), x.getValue().intValue(), (short) replicationFactor))
+            .collect(Collectors.toList());
+        Map newPartitions = partitionCounts.entrySet().stream()
+            .filter(x -> knownTargetTopics.contains(x.getKey()))
+            .collect(Collectors.toMap(x -> x.getKey(), x -> NewPartitions.increaseTo(x.getValue().intValue())));
+        targetAdminClient.createTopics(newTopics, new CreateTopicsOptions()).values().forEach((k, v) -> v.whenComplete((x, e) -> {
+            if (e != null) {
+                log.warn("Could not create topic {}.", k, e);
+            } else {
+                log.info("Created remote topic {} with {} partitions.", k, partitionCounts.get(k));
+            }
+        }));
+        targetAdminClient.createPartitions(newPartitions).values().forEach((k, v) -> v.whenComplete((x, e) -> {
+            if (e instanceof InvalidPartitionsException) {
+                // swallow, this is normal
+            } else if (e != null) {
+                log.warn("Could not create topic-partitions for {}.", k, e);
+            } else {
+                log.info("Increased size of {} to {} partitions.", k, partitionCounts.get(k));
+            }
+        }));
+    }
+
+    private Set listTopics(AdminClient adminClient)
+            throws InterruptedException, ExecutionException {
+        return adminClient.listTopics().names().get();
+    }
+
+    private Collection listTopicAclBindings()
+            throws InterruptedException, ExecutionException {
+        return sourceAdminClient.describeAcls(ANY_TOPIC_ACL).values().get();
+    }
+
+    private Collection describeTopics(Collection topics)
+            throws InterruptedException, ExecutionException {
+        return sourceAdminClient.describeTopics(topics).all().get().values();
+    }
+
+    @SuppressWarnings("deprecation")
+    // use deprecated alterConfigs API for broker compatibility back to 0.11.0
+    private void updateTopicConfigs(Map topicConfigs)
+            throws InterruptedException, ExecutionException {
+        Map configs = topicConfigs.entrySet().stream()
+            .collect(Collectors.toMap(x ->
+                new ConfigResource(ConfigResource.Type.TOPIC, x.getKey()), x -> x.getValue()));
+        log.trace("Syncing configs for {} topics.", configs.size());
+        targetAdminClient.alterConfigs(configs).values().forEach((k, v) -> v.whenComplete((x, e) -> {
+            if (e != null) {
+                log.warn("Could not alter configuration of topic {}.", k.name(), e);
+            }
+        }));
+    }
+
+    private void updateTopicAcls(List bindings)
+            throws InterruptedException, ExecutionException {
+        log.trace("Syncing {} topic ACL bindings.", bindings.size());
+        targetAdminClient.createAcls(bindings).values().forEach((k, v) -> v.whenComplete((x, e) -> {
+            if (e != null) {
+                log.warn("Could not sync ACL of topic {}.", k.pattern().name(), e);
+            }
+        }));
+    }
+
+    private static Stream expandTopicDescription(TopicDescription description) {
+        String topic = description.name();
+        return description.partitions().stream()
+            .map(x -> new TopicPartition(topic, x.partition()));
+    }
+
+    private Map describeTopicConfigs(Set topics)
+            throws InterruptedException, ExecutionException {
+        Set resources = topics.stream()
+            .map(x -> new ConfigResource(ConfigResource.Type.TOPIC, x))
+            .collect(Collectors.toSet());
+        return sourceAdminClient.describeConfigs(resources).all().get().entrySet().stream()
+            .collect(Collectors.toMap(x -> x.getKey().name(), x -> x.getValue()));
+    }
+
+    Config targetConfig(Config sourceConfig) {
+        List entries = sourceConfig.entries().stream()
+            .filter(x -> !x.isDefault() && !x.isReadOnly() && !x.isSensitive())
+            .filter(x -> x.source() != ConfigEntry.ConfigSource.STATIC_BROKER_CONFIG)
+            .filter(x -> shouldReplicateTopicConfigurationProperty(x.name()))
+            .collect(Collectors.toList());
+        return new Config(entries);
+    }
+
+    private static AccessControlEntry downgradeAllowAllACL(AccessControlEntry entry) {
+        return new AccessControlEntry(entry.principal(), entry.host(), AclOperation.READ, entry.permissionType());
+    }
+
+    AclBinding targetAclBinding(AclBinding sourceAclBinding) {
+        String targetTopic = formatRemoteTopic(sourceAclBinding.pattern().name());
+        final AccessControlEntry entry;
+        if (sourceAclBinding.entry().permissionType() == AclPermissionType.ALLOW
+                && sourceAclBinding.entry().operation() == AclOperation.ALL) {
+            entry = downgradeAllowAllACL(sourceAclBinding.entry());
+        } else {
+            entry = sourceAclBinding.entry();
+        }
+        return new AclBinding(new ResourcePattern(ResourceType.TOPIC, targetTopic, PatternType.LITERAL), entry);
+    }
+
+    boolean shouldReplicateTopic(String topic) {
+        return (topicFilter.shouldReplicateTopic(topic) || isHeartbeatTopic(topic))
+            && !replicationPolicy.isInternalTopic(topic) && !isCycle(topic);
+    }
+
+    boolean shouldReplicateAcl(AclBinding aclBinding) {
+        return !(aclBinding.entry().permissionType() == AclPermissionType.ALLOW
+            && aclBinding.entry().operation() == AclOperation.WRITE);
+    }
+
+    boolean shouldReplicateTopicConfigurationProperty(String property) {
+        return configPropertyFilter.shouldReplicateConfigProperty(property);
+    }
+
+    // Recurse upstream to detect cycles, i.e. whether this topic is already on the target cluster
+    boolean isCycle(String topic) {
+        String source = replicationPolicy.topicSource(topic);
+        if (source == null) {
+            return false;
+        } else if (source.equals(sourceAndTarget.target())) {
+            return true;
+        } else {
+            return isCycle(replicationPolicy.upstreamTopic(topic));
+        }
+    }
+
+    // e.g. heartbeats, us-west.heartbeats
+    boolean isHeartbeatTopic(String topic) {
+        return MirrorClientConfig.HEARTBEATS_TOPIC.equals(replicationPolicy.originalTopic(topic));
+    }
+
+    String formatRemoteTopic(String topic) {
+        return replicationPolicy.formatRemoteTopic(sourceAndTarget.source(), topic);
+    }
+}
diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceTask.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceTask.java
new file mode 100644
index 0000000000000..0b864768c9f7d
--- /dev/null
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceTask.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.connect.data.Schema;
+import org.apache.kafka.connect.source.SourceTask;
+import org.apache.kafka.connect.source.SourceRecord;
+import org.apache.kafka.connect.header.Headers;
+import org.apache.kafka.connect.header.ConnectHeaders;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.errors.WakeupException;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.utils.Utils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Set;
+import java.util.ArrayList;
+import java.util.stream.Collectors;
+import java.util.concurrent.Semaphore;
+import java.time.Duration;
+
+/** Replicates a set of topic-partitions. */
+public class MirrorSourceTask extends SourceTask {
+
+    private static final Logger log = LoggerFactory.getLogger(MirrorSourceTask.class);
+
+    private static final int MAX_OUTSTANDING_OFFSET_SYNCS = 10;
+
+    private KafkaConsumer consumer;
+    private KafkaProducer offsetProducer;
+    private String sourceClusterAlias;
+    private String offsetSyncsTopic;
+    private Duration pollTimeout;
+    private long maxOffsetLag;
+    private Map partitionStates;
+    private ReplicationPolicy replicationPolicy;
+    private MirrorMetrics metrics;
+    private boolean stopping = false;
+    private Semaphore outstandingOffsetSyncs;
+    private Semaphore consumerAccess;
+
+    public MirrorSourceTask() {}
+
+    // for testing
+    MirrorSourceTask(String sourceClusterAlias, ReplicationPolicy replicationPolicy, long maxOffsetLag) {
+        this.sourceClusterAlias = sourceClusterAlias;
+        this.replicationPolicy = replicationPolicy;
+        this.maxOffsetLag = maxOffsetLag;
+    }
+
+    @Override
+    public void start(Map props) {
+        MirrorTaskConfig config = new MirrorTaskConfig(props);
+        outstandingOffsetSyncs = new Semaphore(MAX_OUTSTANDING_OFFSET_SYNCS);
+        consumerAccess = new Semaphore(1);  // let one thread at a time access the consumer
+        sourceClusterAlias = config.sourceClusterAlias();
+        metrics = config.metrics();
+        pollTimeout = config.consumerPollTimeout();
+        maxOffsetLag = config.maxOffsetLag();
+        replicationPolicy = config.replicationPolicy();
+        partitionStates = new HashMap<>();
+        offsetSyncsTopic = config.offsetSyncsTopic();
+        consumer = MirrorUtils.newConsumer(config.sourceConsumerConfig());
+        offsetProducer = MirrorUtils.newProducer(config.sourceProducerConfig());
+        Set taskTopicPartitions = config.taskTopicPartitions();
+        Map topicPartitionOffsets = loadOffsets(taskTopicPartitions);
+        consumer.assign(topicPartitionOffsets.keySet());
+        log.info("Starting with {} previously uncommitted partitions.", topicPartitionOffsets.entrySet().stream()
+            .filter(x -> x.getValue() == 0L).count());
+        log.trace("Seeking offsets: {}", topicPartitionOffsets);
+        topicPartitionOffsets.forEach(consumer::seek);
+        log.info("{} replicating {} topic-partitions {}->{}: {}.", Thread.currentThread().getName(),
+            taskTopicPartitions.size(), sourceClusterAlias, config.targetClusterAlias(), taskTopicPartitions);
+    }
+
+    @Override
+    public void commit() {
+        // nop
+    }
+
+    @Override
+    public void stop() {
+        long start = System.currentTimeMillis();
+        stopping = true;
+        consumer.wakeup();
+        try {
+            consumerAccess.acquire();
+        } catch (InterruptedException e) {
+            log.warn("Interrupted waiting for access to consumer. Will try closing anyway."); 
+        }
+        Utils.closeQuietly(consumer, "source consumer");
+        Utils.closeQuietly(offsetProducer, "offset producer");
+        Utils.closeQuietly(metrics, "metrics");
+        log.info("Stopping {} took {} ms.", Thread.currentThread().getName(), System.currentTimeMillis() - start);
+    }
+   
+    @Override
+    public String version() {
+        return "1";
+    }
+
+    @Override
+    public List poll() {
+        if (!consumerAccess.tryAcquire()) {
+            return null;
+        }
+        if (stopping) {
+            return null;
+        }
+        try {
+            ConsumerRecords records = consumer.poll(pollTimeout);
+            List sourceRecords = new ArrayList<>(records.count());
+            for (ConsumerRecord record : records) {
+                SourceRecord converted = convertRecord(record);
+                sourceRecords.add(converted);
+                TopicPartition topicPartition = new TopicPartition(converted.topic(), converted.kafkaPartition());
+                metrics.recordAge(topicPartition, System.currentTimeMillis() - record.timestamp());
+                metrics.recordBytes(topicPartition, byteSize(record.value()));
+            }
+            if (sourceRecords.isEmpty()) {
+                // WorkerSourceTasks expects non-zero batch size
+                return null;
+            } else {
+                log.trace("Polled {} records from {}.", sourceRecords.size(), records.partitions());
+                return sourceRecords;
+            }
+        } catch (WakeupException e) {
+            return null;
+        } catch (KafkaException e) {
+            log.warn("Failure during poll.", e);
+            return null;
+        } catch (Throwable e)  {
+            log.error("Failure during poll.", e);
+            // allow Connect to deal with the exception
+            throw e;
+        } finally {
+            consumerAccess.release();
+        }
+    }
+ 
+    @Override
+    public void commitRecord(SourceRecord record, RecordMetadata metadata) {
+        try {
+            if (stopping) {
+                return;
+            }
+            if (!metadata.hasOffset()) {
+                log.error("RecordMetadata has no offset -- can't sync offsets for {}.", record.topic());
+                return;
+            }
+            TopicPartition topicPartition = new TopicPartition(record.topic(), record.kafkaPartition());
+            long latency = System.currentTimeMillis() - record.timestamp();
+            metrics.countRecord(topicPartition);
+            metrics.replicationLatency(topicPartition, latency);
+            TopicPartition sourceTopicPartition = MirrorUtils.unwrapPartition(record.sourcePartition());
+            long upstreamOffset = MirrorUtils.unwrapOffset(record.sourceOffset());
+            long downstreamOffset = metadata.offset();
+            maybeSyncOffsets(sourceTopicPartition, upstreamOffset, downstreamOffset);
+        } catch (Throwable e) {
+            log.warn("Failure committing record.", e);
+        }
+    }
+
+    // updates partition state and sends OffsetSync if necessary
+    private void maybeSyncOffsets(TopicPartition topicPartition, long upstreamOffset,
+            long downstreamOffset) {
+        PartitionState partitionState =
+            partitionStates.computeIfAbsent(topicPartition, x -> new PartitionState(maxOffsetLag));
+        if (partitionState.update(upstreamOffset, downstreamOffset)) {
+            sendOffsetSync(topicPartition, upstreamOffset, downstreamOffset);
+        }
+    }
+
+    // sends OffsetSync record upstream to internal offsets topic
+    private void sendOffsetSync(TopicPartition topicPartition, long upstreamOffset,
+            long downstreamOffset) {
+        if (!outstandingOffsetSyncs.tryAcquire()) {
+            // Too many outstanding offset syncs.
+            return;
+        }
+        OffsetSync offsetSync = new OffsetSync(topicPartition, upstreamOffset, downstreamOffset);
+        ProducerRecord record = new ProducerRecord<>(offsetSyncsTopic, 0,
+                offsetSync.recordKey(), offsetSync.recordValue());
+        offsetProducer.send(record, (x, e) -> {
+            if (e != null) {
+                log.error("Failure sending offset sync.", e);
+            } else {
+                log.trace("Sync'd offsets for {}: {}=={}", topicPartition,
+                    upstreamOffset, downstreamOffset);
+            }
+            outstandingOffsetSyncs.release();
+        });
+    }
+ 
+    private Map loadOffsets(Set topicPartitions) {
+        return topicPartitions.stream().collect(Collectors.toMap(x -> x, x -> loadOffset(x)));
+    }
+
+    private Long loadOffset(TopicPartition topicPartition) {
+        Map wrappedPartition = MirrorUtils.wrapPartition(topicPartition, sourceClusterAlias);
+        Map wrappedOffset = context.offsetStorageReader().offset(wrappedPartition);
+        return MirrorUtils.unwrapOffset(wrappedOffset) + 1;
+    }
+
+    // visible for testing 
+    SourceRecord convertRecord(ConsumerRecord record) {
+        String targetTopic = formatRemoteTopic(record.topic());
+        Headers headers = convertHeaders(record);
+        return new SourceRecord(
+                MirrorUtils.wrapPartition(new TopicPartition(record.topic(), record.partition()), sourceClusterAlias),
+                MirrorUtils.wrapOffset(record.offset()),
+                targetTopic, record.partition(),
+                Schema.OPTIONAL_BYTES_SCHEMA, record.key(),
+                Schema.BYTES_SCHEMA, record.value(),
+                record.timestamp(), headers);
+    }
+
+    private Headers convertHeaders(ConsumerRecord record) {
+        ConnectHeaders headers = new ConnectHeaders();
+        for (Header header : record.headers()) {
+            headers.addBytes(header.key(), header.value());
+        }
+        return headers;
+    }
+
+    private String formatRemoteTopic(String topic) {
+        return replicationPolicy.formatRemoteTopic(sourceClusterAlias, topic);
+    }
+
+    private static int byteSize(byte[] bytes) {
+        if (bytes == null) {
+            return 0;
+        } else {
+            return bytes.length;
+        }
+    }
+
+    static class PartitionState {
+        long previousUpstreamOffset = -1L;
+        long previousDownstreamOffset = -1L;
+        long lastSyncUpstreamOffset = -1L;
+        long lastSyncDownstreamOffset = -1L;
+        long maxOffsetLag;
+
+        PartitionState(long maxOffsetLag) {
+            this.maxOffsetLag = maxOffsetLag;
+        }
+
+        // true if we should emit an offset sync
+        boolean update(long upstreamOffset, long downstreamOffset) {
+            boolean shouldSyncOffsets = false;
+            long upstreamStep = upstreamOffset - lastSyncUpstreamOffset;
+            long downstreamTargetOffset = lastSyncDownstreamOffset + upstreamStep;
+            if (lastSyncDownstreamOffset == -1L
+                    || downstreamOffset - downstreamTargetOffset >= maxOffsetLag
+                    || upstreamOffset - previousUpstreamOffset != 1L
+                    || downstreamOffset < previousDownstreamOffset) {
+                lastSyncUpstreamOffset = upstreamOffset;
+                lastSyncDownstreamOffset = downstreamOffset;
+                shouldSyncOffsets = true;
+            }
+            previousUpstreamOffset = upstreamOffset;
+            previousDownstreamOffset = downstreamOffset;
+            return shouldSyncOffsets;
+        }
+    }
+}
diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorTaskConfig.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorTaskConfig.java
new file mode 100644
index 0000000000000..839d41eb04040
--- /dev/null
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorTaskConfig.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.TopicPartition;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.List;
+import java.util.HashSet;
+import java.util.Collections;
+import java.util.stream.Collectors;
+
+public class MirrorTaskConfig extends MirrorConnectorConfig {
+
+    private static final String TASK_TOPIC_PARTITIONS_DOC = "Topic-partitions assigned to this task to replicate.";
+    private static final String TASK_CONSUMER_GROUPS_DOC = "Consumer groups assigned to this task to replicate.";
+
+    public MirrorTaskConfig(Map props) {
+        super(TASK_CONFIG_DEF, props);
+    }
+
+    Set taskTopicPartitions() {
+        List fields = getList(TASK_TOPIC_PARTITIONS);
+        if (fields == null || fields.isEmpty()) {
+            return Collections.emptySet();
+        }
+        return fields.stream()
+            .map(MirrorUtils::decodeTopicPartition)
+            .collect(Collectors.toSet());
+    }
+
+    Set taskConsumerGroups() {
+        List fields = getList(TASK_CONSUMER_GROUPS);
+        if (fields == null || fields.isEmpty()) {
+            return Collections.emptySet();
+        }
+        return new HashSet<>(fields);
+    } 
+
+    MirrorMetrics metrics() {
+        MirrorMetrics metrics = new MirrorMetrics(this);
+        metricsReporters().forEach(metrics::addReporter);
+        return metrics;
+    }
+ 
+    protected static final ConfigDef TASK_CONFIG_DEF = CONNECTOR_CONFIG_DEF
+        .define(
+            TASK_TOPIC_PARTITIONS,
+            ConfigDef.Type.LIST,
+            null,
+            ConfigDef.Importance.LOW,
+            TASK_TOPIC_PARTITIONS_DOC)
+        .define(
+            TASK_CONSUMER_GROUPS,
+            ConfigDef.Type.LIST,
+            null,
+            ConfigDef.Importance.LOW,
+            TASK_CONSUMER_GROUPS_DOC);
+}
diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorUtils.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorUtils.java
new file mode 100644
index 0000000000000..f15dda854c7a0
--- /dev/null
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorUtils.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.connect.util.TopicAdmin;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.List;
+import java.util.HashMap;
+import java.util.Collections;
+import java.util.regex.Pattern;
+
+/** Internal utility methods. */
+final class MirrorUtils {
+
+    // utility class
+    private MirrorUtils() {}
+
+    static KafkaProducer newProducer(Map props) {
+        return new KafkaProducer<>(props, new ByteArraySerializer(), new ByteArraySerializer());
+    }
+
+    static KafkaConsumer newConsumer(Map props) {
+        return new KafkaConsumer<>(props, new ByteArrayDeserializer(), new ByteArrayDeserializer());
+    }
+
+    static String encodeTopicPartition(TopicPartition topicPartition) {
+        return topicPartition.toString();
+    }
+
+    static Map wrapPartition(TopicPartition topicPartition, String sourceClusterAlias) {
+        Map wrapped = new HashMap<>();
+        wrapped.put("topic", topicPartition.topic());
+        wrapped.put("partition", topicPartition.partition());
+        wrapped.put("cluster", sourceClusterAlias);
+        return wrapped;
+    }
+
+    static Map wrapOffset(long offset) {
+        return Collections.singletonMap("offset", offset);
+    }
+
+    static TopicPartition unwrapPartition(Map wrapped) {
+        String topic = (String) wrapped.get("topic");
+        int partition = (Integer) wrapped.get("partition");
+        return new TopicPartition(topic, partition);
+    }
+
+    static Long unwrapOffset(Map wrapped) {
+        if (wrapped == null || wrapped.get("offset") == null) {
+            return -1L;
+        }
+        return (Long) wrapped.get("offset");
+    }
+
+    static TopicPartition decodeTopicPartition(String topicPartitionString) {
+        int sep = topicPartitionString.lastIndexOf('-');
+        String topic = topicPartitionString.substring(0, sep);
+        String partitionString = topicPartitionString.substring(sep + 1);
+        int partition = Integer.parseInt(partitionString);
+        return new TopicPartition(topic, partition);
+    }
+
+    // returns null if given empty list
+    static Pattern compilePatternList(List fields) {
+        if (fields.isEmpty()) {
+            // The empty pattern matches _everything_, but a blank
+            // config property should match _nothing_.
+            return null;
+        } else {
+            String joined = String.join("|", fields);
+            return Pattern.compile(joined);
+        }
+    }
+
+    static Pattern compilePatternList(String fields) {
+        return compilePatternList(Arrays.asList(fields.split("\\W*,\\W*")));
+    }
+
+    static void createCompactedTopic(String topicName, short partitions, short replicationFactor, Map adminProps) {
+        NewTopic topicDescription = TopicAdmin.defineTopic(topicName).
+                compacted().
+                partitions(partitions).
+                replicationFactor(replicationFactor).
+                build();
+
+        try (TopicAdmin admin = new TopicAdmin(adminProps)) {
+            admin.createTopics(topicDescription);
+        }
+    }
+
+    static void createSinglePartitionCompactedTopic(String topicName, short replicationFactor, Map adminProps) {
+        createCompactedTopic(topicName, (short) 1, replicationFactor, adminProps);
+    }
+}
diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/OffsetSync.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/OffsetSync.java
new file mode 100644
index 0000000000000..abdc64cc3c754
--- /dev/null
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/OffsetSync.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.protocol.types.Field;
+import org.apache.kafka.common.protocol.types.Schema;
+import org.apache.kafka.common.protocol.types.Struct;
+import org.apache.kafka.common.protocol.types.Type;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+
+import java.nio.ByteBuffer;
+
+public class OffsetSync {
+    public static final String TOPIC_KEY = "topic";
+    public static final String PARTITION_KEY = "partition";
+    public static final String UPSTREAM_OFFSET_KEY = "upstreamOffset";
+    public static final String DOWNSTREAM_OFFSET_KEY = "offset";
+
+    public static final Schema VALUE_SCHEMA = new Schema(
+            new Field(UPSTREAM_OFFSET_KEY, Type.INT64),
+            new Field(DOWNSTREAM_OFFSET_KEY, Type.INT64));
+
+    public static final Schema KEY_SCHEMA = new Schema(
+            new Field(TOPIC_KEY, Type.STRING),
+            new Field(PARTITION_KEY, Type.INT32));
+
+    private TopicPartition topicPartition;
+    private long upstreamOffset;
+    private long downstreamOffset;
+
+    public OffsetSync(TopicPartition topicPartition, long upstreamOffset, long downstreamOffset) {
+        this.topicPartition = topicPartition;
+        this.upstreamOffset = upstreamOffset;
+        this.downstreamOffset = downstreamOffset;
+    }
+
+    public TopicPartition topicPartition() {
+        return topicPartition;
+    }
+
+    public long upstreamOffset() {
+        return upstreamOffset;
+    }
+
+    public long downstreamOffset() {
+        return downstreamOffset;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("OffsetSync{topicPartition=%s, upstreamOffset=%d, downstreamOffset=%d}",
+            topicPartition, upstreamOffset, downstreamOffset);
+    }
+
+    ByteBuffer serializeValue() {
+        Struct struct = valueStruct();
+        ByteBuffer buffer = ByteBuffer.allocate(VALUE_SCHEMA.sizeOf(struct));
+        VALUE_SCHEMA.write(buffer, struct);
+        buffer.flip();
+        return buffer;
+    }
+
+    ByteBuffer serializeKey() {
+        Struct struct = keyStruct();
+        ByteBuffer buffer = ByteBuffer.allocate(KEY_SCHEMA.sizeOf(struct));
+        KEY_SCHEMA.write(buffer, struct);
+        buffer.flip();
+        return buffer;
+    }
+
+    static OffsetSync deserializeRecord(ConsumerRecord record) {
+        Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(record.key()));
+        String topic = keyStruct.getString(TOPIC_KEY);
+        int partition = keyStruct.getInt(PARTITION_KEY);
+        
+        Struct valueStruct = VALUE_SCHEMA.read(ByteBuffer.wrap(record.value()));
+        long upstreamOffset = valueStruct.getLong(UPSTREAM_OFFSET_KEY);
+        long downstreamOffset = valueStruct.getLong(DOWNSTREAM_OFFSET_KEY);
+        
+        return new OffsetSync(new TopicPartition(topic, partition), upstreamOffset, downstreamOffset);
+    } 
+
+    private Struct valueStruct() {
+        Struct struct = new Struct(VALUE_SCHEMA);
+        struct.set(UPSTREAM_OFFSET_KEY, upstreamOffset);
+        struct.set(DOWNSTREAM_OFFSET_KEY, downstreamOffset);
+        return struct;
+    }
+
+    private Struct keyStruct() {
+        Struct struct = new Struct(KEY_SCHEMA);
+        struct.set(TOPIC_KEY, topicPartition.topic());
+        struct.set(PARTITION_KEY, topicPartition.partition());
+        return struct;
+    }
+
+    byte[] recordKey() {
+        return serializeKey().array();
+    }
+
+    byte[] recordValue() {
+        return serializeValue().array();
+    }
+};
+
diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/OffsetSyncStore.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/OffsetSyncStore.java
new file mode 100644
index 0000000000000..fff1abd1cf080
--- /dev/null
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/OffsetSyncStore.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.WakeupException;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.utils.Utils;
+
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Collections;
+import java.time.Duration;
+
+/** Used internally by MirrorMaker. Stores offset syncs and performs offset translation. */
+class OffsetSyncStore implements AutoCloseable {
+    private KafkaConsumer consumer;
+    private Map offsetSyncs = new HashMap<>();
+    private TopicPartition offsetSyncTopicPartition;
+
+    OffsetSyncStore(MirrorConnectorConfig config) {
+        consumer = new KafkaConsumer<>(config.sourceConsumerConfig(),
+            new ByteArrayDeserializer(), new ByteArrayDeserializer());
+        offsetSyncTopicPartition = new TopicPartition(config.offsetSyncsTopic(), 0);
+        consumer.assign(Collections.singleton(offsetSyncTopicPartition));
+    }
+
+    // for testing
+    OffsetSyncStore(KafkaConsumer consumer, TopicPartition offsetSyncTopicPartition) {
+        this.consumer = consumer;
+        this.offsetSyncTopicPartition = offsetSyncTopicPartition;
+    }
+
+    long translateDownstream(TopicPartition sourceTopicPartition, long upstreamOffset) {
+        OffsetSync offsetSync = latestOffsetSync(sourceTopicPartition);
+        if (offsetSync.upstreamOffset() > upstreamOffset) {
+            // Offset is too far in the past to translate accurately
+            return -1;
+        }
+        long upstreamStep = upstreamOffset - offsetSync.upstreamOffset();
+        return offsetSync.downstreamOffset() + upstreamStep;
+    }
+
+    // poll and handle records
+    synchronized void update(Duration pollTimeout) {
+        try {
+            consumer.poll(pollTimeout).forEach(this::handleRecord);
+        } catch (WakeupException e) {
+            // swallow
+        }
+    }
+
+    public synchronized void close() {
+        consumer.wakeup();
+        Utils.closeQuietly(consumer, "offset sync store consumer");
+    }
+
+    protected void handleRecord(ConsumerRecord record) {
+        OffsetSync offsetSync = OffsetSync.deserializeRecord(record);
+        TopicPartition sourceTopicPartition = offsetSync.topicPartition();
+        offsetSyncs.put(sourceTopicPartition, offsetSync);
+    }
+
+    private OffsetSync latestOffsetSync(TopicPartition topicPartition) {
+        return offsetSyncs.computeIfAbsent(topicPartition, x -> new OffsetSync(topicPartition,
+            -1, -1));
+    }
+}
diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/Scheduler.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/Scheduler.java
new file mode 100644
index 0000000000000..cac9a80c0fd89
--- /dev/null
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/Scheduler.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
+import java.time.Duration;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class Scheduler implements AutoCloseable {
+    private static Logger log = LoggerFactory.getLogger(Scheduler.class);
+
+    private final String name;
+    private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
+    private final Duration timeout;
+    private boolean closed = false;
+
+    Scheduler(String name, Duration timeout) {
+        this.name = name;
+        this.timeout = timeout;
+    }
+
+    Scheduler(Class clazz, Duration timeout) {
+        this("Scheduler for " + clazz.getSimpleName(), timeout);
+    }
+
+    void scheduleRepeating(Task task, Duration interval, String description) {
+        if (interval.toMillis() < 0L) {
+            return;
+        }
+        executor.scheduleAtFixedRate(() -> executeThread(task, description), 0, interval.toMillis(), TimeUnit.MILLISECONDS);
+    }
+ 
+    void scheduleRepeatingDelayed(Task task, Duration interval, String description) {
+        if (interval.toMillis() < 0L) {
+            return;
+        }
+        executor.scheduleAtFixedRate(() -> executeThread(task, description), interval.toMillis(),
+            interval.toMillis(), TimeUnit.MILLISECONDS);
+    }
+
+    void execute(Task task, String description) {
+        try {
+            executor.submit(() -> executeThread(task, description)).get(timeout.toMillis(), TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            log.warn("{} was interrupted running task: {}", name, description);
+        } catch (TimeoutException e) {
+            log.error("{} timed out running task: {}", name, description);
+        } catch (Throwable e) {
+            log.error("{} caught exception in task: {}", name, description, e);
+        }
+    } 
+
+    public void close() {
+        closed = true;
+        executor.shutdown();
+        try {
+            boolean terminated = executor.awaitTermination(timeout.toMillis(), TimeUnit.MILLISECONDS);
+            if (!terminated) {
+                log.error("{} timed out during shutdown of internal scheduler.", name);
+            }
+        } catch (InterruptedException e) {
+            log.warn("{} was interrupted during shutdown of internal scheduler.", name);
+        }
+    }
+
+    interface Task {
+        void run() throws InterruptedException, ExecutionException;
+    }
+
+    private void run(Task task, String description) {
+        try {
+            long start = System.currentTimeMillis();
+            task.run();
+            long elapsed = System.currentTimeMillis() - start;
+            log.info("{} took {} ms", description, elapsed);
+            if (elapsed > timeout.toMillis()) {
+                log.warn("{} took too long ({} ms) running task: {}", name, elapsed, description);
+            }
+        } catch (InterruptedException e) {
+            log.warn("{} was interrupted running task: {}", name, description);
+        } catch (Throwable e) {
+            log.error("{} caught exception in scheduled task: {}", name, description, e);
+        }
+    }
+
+    private void executeThread(Task task, String description) {
+        Thread.currentThread().setName(description);
+        if (closed) {
+            log.info("{} skipping task due to shutdown: {}", name, description);
+            return;
+        }
+        run(task, description);
+    }
+}
+
diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/TopicFilter.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/TopicFilter.java
new file mode 100644
index 0000000000000..f13453f116850
--- /dev/null
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/TopicFilter.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.Configurable;
+import org.apache.kafka.common.annotation.InterfaceStability;
+import java.util.Map;
+
+/** Defines which topics should be replicated. */
+@InterfaceStability.Evolving
+public interface TopicFilter extends Configurable, AutoCloseable {
+
+    boolean shouldReplicateTopic(String topic);
+
+    default void close() {
+        //nop
+    }
+
+    default void configure(Map props) {
+        //nop
+    }
+}
diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/CheckpointTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/CheckpointTest.java
new file mode 100644
index 0000000000000..1a3f21011817a
--- /dev/null
+++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/CheckpointTest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.common.TopicPartition;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class CheckpointTest {
+
+    @Test
+    public void testSerde() {
+        Checkpoint checkpoint = new Checkpoint("group-1", new TopicPartition("topic-2", 3), 4, 5, "metadata-6");
+        byte[] key = checkpoint.recordKey();
+        byte[] value = checkpoint.recordValue();
+        ConsumerRecord record = new ConsumerRecord<>("any-topic", 7, 8, key, value);
+        Checkpoint deserialized = Checkpoint.deserializeRecord(record);
+        assertEquals(checkpoint.consumerGroupId(), deserialized.consumerGroupId());
+        assertEquals(checkpoint.topicPartition(), deserialized.topicPartition());
+        assertEquals(checkpoint.upstreamOffset(), deserialized.upstreamOffset());
+        assertEquals(checkpoint.downstreamOffset(), deserialized.downstreamOffset());
+    }
+}
diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/HeartbeatTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/HeartbeatTest.java
new file mode 100644
index 0000000000000..adc657862a9e4
--- /dev/null
+++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/HeartbeatTest.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class HeartbeatTest {
+
+    @Test
+    public void testSerde() {
+        Heartbeat heartbeat = new Heartbeat("source-1", "target-2", 1234567890L);
+        byte[] key = heartbeat.recordKey();
+        byte[] value = heartbeat.recordValue();
+        ConsumerRecord record = new ConsumerRecord<>("any-topic", 6, 7, key, value);
+        Heartbeat deserialized = Heartbeat.deserializeRecord(record);
+        assertEquals(heartbeat.sourceClusterAlias(), deserialized.sourceClusterAlias());
+        assertEquals(heartbeat.targetClusterAlias(), deserialized.targetClusterAlias());
+        assertEquals(heartbeat.timestamp(), deserialized.timestamp());
+    }
+}
diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorCheckpointTaskTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorCheckpointTaskTest.java
new file mode 100644
index 0000000000000..a66574444d65f
--- /dev/null
+++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorCheckpointTaskTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.connect.source.SourceRecord;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class MirrorCheckpointTaskTest {
+
+    @Test
+    public void testDownstreamTopicRenaming() {
+        MirrorCheckpointTask mirrorCheckpointTask = new MirrorCheckpointTask("source1", "target2",
+            new DefaultReplicationPolicy(), null);
+        assertEquals(new TopicPartition("source1.topic3", 4),
+            mirrorCheckpointTask.renameTopicPartition(new TopicPartition("topic3", 4)));
+        assertEquals(new TopicPartition("topic3", 5),
+            mirrorCheckpointTask.renameTopicPartition(new TopicPartition("target2.topic3", 5)));
+        assertEquals(new TopicPartition("source1.source6.topic7", 8),
+            mirrorCheckpointTask.renameTopicPartition(new TopicPartition("source6.topic7", 8)));
+    }
+
+    @Test
+    public void testCheckpoint() {
+        OffsetSyncStoreTest.FakeOffsetSyncStore offsetSyncStore = new OffsetSyncStoreTest.FakeOffsetSyncStore();
+        MirrorCheckpointTask mirrorCheckpointTask = new MirrorCheckpointTask("source1", "target2",
+            new DefaultReplicationPolicy(), offsetSyncStore);
+        offsetSyncStore.sync(new TopicPartition("topic1", 2), 3L, 4L);
+        offsetSyncStore.sync(new TopicPartition("target2.topic5", 6), 7L, 8L);
+        Checkpoint checkpoint1 = mirrorCheckpointTask.checkpoint("group9", new TopicPartition("topic1", 2),
+            new OffsetAndMetadata(10, null));
+        SourceRecord sourceRecord1 = mirrorCheckpointTask.checkpointRecord(checkpoint1, 123L);
+        assertEquals(new TopicPartition("source1.topic1", 2), checkpoint1.topicPartition());
+        assertEquals("group9", checkpoint1.consumerGroupId());
+        assertEquals("group9", Checkpoint.unwrapGroup(sourceRecord1.sourcePartition()));
+        assertEquals(10, checkpoint1.upstreamOffset());
+        assertEquals(11, checkpoint1.downstreamOffset());
+        assertEquals(123L, sourceRecord1.timestamp().longValue());
+        Checkpoint checkpoint2 = mirrorCheckpointTask.checkpoint("group11", new TopicPartition("target2.topic5", 6),
+            new OffsetAndMetadata(12, null));
+        SourceRecord sourceRecord2 = mirrorCheckpointTask.checkpointRecord(checkpoint2, 234L);
+        assertEquals(new TopicPartition("topic5", 6), checkpoint2.topicPartition());
+        assertEquals("group11", checkpoint2.consumerGroupId());
+        assertEquals("group11", Checkpoint.unwrapGroup(sourceRecord2.sourcePartition()));
+        assertEquals(12, checkpoint2.upstreamOffset());
+        assertEquals(13, checkpoint2.downstreamOffset());
+        assertEquals(234L, sourceRecord2.timestamp().longValue());
+    }
+}
diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorConfigTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorConfigTest.java
new file mode 100644
index 0000000000000..6003202bfc3b3
--- /dev/null
+++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorConfigTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.TopicPartition;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.HashSet;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertEquals;
+
+public class MirrorConnectorConfigTest {
+
+    private Map makeProps(String... keyValues) {
+        Map props = new HashMap<>();
+        props.put("name", "ConnectorName");
+        props.put("connector.class", "ConnectorClass");
+        props.put("source.cluster.alias", "source1");
+        props.put("target.cluster.alias", "target2");
+        for (int i = 0; i < keyValues.length; i += 2) {
+            props.put(keyValues[i], keyValues[i + 1]);
+        }
+        return props;
+    }
+
+    @Test
+    public void testTaskConfigTopicPartitions() {
+        List topicPartitions = Arrays.asList(new TopicPartition("topic-1", 2),
+            new TopicPartition("topic-3", 4), new TopicPartition("topic-5", 6));
+        MirrorConnectorConfig config = new MirrorConnectorConfig(makeProps());
+        Map props = config.taskConfigForTopicPartitions(topicPartitions);
+        MirrorTaskConfig taskConfig = new MirrorTaskConfig(props);
+        assertEquals(taskConfig.taskTopicPartitions(), new HashSet<>(topicPartitions));
+    }
+
+    @Test
+    public void testTaskConfigConsumerGroups() {
+        List groups = Arrays.asList("consumer-1", "consumer-2", "consumer-3");
+        MirrorConnectorConfig config = new MirrorConnectorConfig(makeProps());
+        Map props = config.taskConfigForConsumerGroups(groups);
+        MirrorTaskConfig taskConfig = new MirrorTaskConfig(props);
+        assertEquals(taskConfig.taskConsumerGroups(), new HashSet<>(groups));
+    }
+
+    @Test
+    public void testTopicMatching() {
+        MirrorConnectorConfig config = new MirrorConnectorConfig(makeProps("topics", "topic1"));
+        assertTrue(config.topicFilter().shouldReplicateTopic("topic1"));
+        assertFalse(config.topicFilter().shouldReplicateTopic("topic2"));
+    }
+
+    @Test
+    public void testGroupMatching() {
+        MirrorConnectorConfig config = new MirrorConnectorConfig(makeProps("groups", "group1"));
+        assertTrue(config.groupFilter().shouldReplicateGroup("group1"));
+        assertFalse(config.groupFilter().shouldReplicateGroup("group2"));
+    }
+
+    @Test
+    public void testConfigPropertyMatching() {
+        MirrorConnectorConfig config = new MirrorConnectorConfig(
+            makeProps("config.properties.blacklist", "prop2"));
+        assertTrue(config.configPropertyFilter().shouldReplicateConfigProperty("prop1"));
+        assertFalse(config.configPropertyFilter().shouldReplicateConfigProperty("prop2"));
+    }
+
+    @Test
+    public void testNoTopics() {
+        MirrorConnectorConfig config = new MirrorConnectorConfig(makeProps("topics", ""));
+        assertFalse(config.topicFilter().shouldReplicateTopic("topic1"));
+        assertFalse(config.topicFilter().shouldReplicateTopic("topic2"));
+        assertFalse(config.topicFilter().shouldReplicateTopic(""));
+    }
+
+    @Test
+    public void testAllTopics() {
+        MirrorConnectorConfig config = new MirrorConnectorConfig(makeProps("topics", ".*"));
+        assertTrue(config.topicFilter().shouldReplicateTopic("topic1"));
+        assertTrue(config.topicFilter().shouldReplicateTopic("topic2"));
+    }
+
+    @Test
+    public void testListOfTopics() {
+        MirrorConnectorConfig config = new MirrorConnectorConfig(makeProps("topics", "topic1, topic2"));
+        assertTrue(config.topicFilter().shouldReplicateTopic("topic1"));
+        assertTrue(config.topicFilter().shouldReplicateTopic("topic2"));
+        assertFalse(config.topicFilter().shouldReplicateTopic("topic3"));
+    }
+}
diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java
new file mode 100644
index 0000000000000..11abc142170f7
--- /dev/null
+++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java
@@ -0,0 +1,302 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.connect.util.clusters.EmbeddedConnectCluster;
+import org.apache.kafka.connect.util.clusters.EmbeddedKafkaCluster;
+import org.apache.kafka.test.IntegrationTest;
+import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.clients.admin.Admin;
+import org.apache.kafka.common.TopicPartition;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Collections;
+import java.util.Properties;
+import java.util.concurrent.TimeoutException;
+import java.time.Duration;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.apache.kafka.test.TestUtils.waitForCondition;
+
+/**
+ * Tests MM2 replication and failover/failback logic.
+ *
+ * MM2 is configured with active/active replication between two Kafka clusters. Tests validate that
+ * records sent to either cluster arrive at the other cluster. Then, a consumer group is migrated from
+ * one cluster to the other and back. Tests validate that consumer offsets are translated and replicated
+ * between clusters during this failover and failback.
+ */
+@Category(IntegrationTest.class)
+public class MirrorConnectorsIntegrationTest {
+
+    private static final Logger log = LoggerFactory.getLogger(MirrorConnectorsIntegrationTest.class);
+
+    private static final int NUM_RECORDS_PRODUCED = 100;  // to save trees
+    private static final int NUM_PARTITIONS = 10;
+    private static final int RECORD_TRANSFER_DURATION_MS = 10_000;
+    private static final int CHECKPOINT_DURATION_MS = 20_000;
+
+    private MirrorMakerConfig mm2Config; 
+    private EmbeddedConnectCluster primary;
+    private EmbeddedConnectCluster backup;
+
+    @Before
+    public void setup() throws IOException {
+        Properties brokerProps = new Properties();
+        brokerProps.put("auto.create.topics.enable", "false");
+
+        Map mm2Props = new HashMap<>();
+        mm2Props.put("clusters", "primary, backup");
+        mm2Props.put("max.tasks", "10");
+        mm2Props.put("topics", "test-topic-.*, primary.test-topic-.*, backup.test-topic-.*");
+        mm2Props.put("groups", "consumer-group-.*");
+        mm2Props.put("primary->backup.enabled", "true");
+        mm2Props.put("backup->primary.enabled", "true");
+        mm2Props.put("sync.topic.acls.enabled", "false");
+        mm2Props.put("emit.checkpoints.interval.seconds", "1");
+        mm2Props.put("emit.heartbeats.interval.seconds", "1");
+        mm2Props.put("refresh.topics.interval.seconds", "1");
+        mm2Props.put("refresh.groups.interval.seconds", "1");
+        mm2Props.put("checkpoints.topic.replication.factor", "1");
+        mm2Props.put("heartbeats.topic.replication.factor", "1");
+        mm2Props.put("offset-syncs.topic.replication.factor", "1");
+        mm2Props.put("config.storage.topic.replication.factor", "1");
+        mm2Props.put("offset.stoage.topic.replication.factor", "1");
+        mm2Props.put("status.stoage.topic.replication.factor", "1");
+        mm2Props.put("replication.factor", "1");
+        
+        mm2Config = new MirrorMakerConfig(mm2Props); 
+        Map primaryWorkerProps = mm2Config.workerConfig(new SourceAndTarget("backup", "primary"));
+        Map backupWorkerProps = mm2Config.workerConfig(new SourceAndTarget("primary", "backup"));
+
+        primary = new EmbeddedConnectCluster.Builder()
+                .name("primary-connect-cluster")
+                .numWorkers(3)
+                .numBrokers(1)
+                .brokerProps(brokerProps)
+                .workerProps(primaryWorkerProps)
+                .build();
+
+        backup = new EmbeddedConnectCluster.Builder()
+                .name("backup-connect-cluster")
+                .numWorkers(3)
+                .numBrokers(1)
+                .brokerProps(brokerProps)
+                .workerProps(backupWorkerProps)
+                .build();
+
+        primary.start();
+        backup.start();
+
+        // create these topics before starting the connectors so we don't need to wait for discovery
+        primary.kafka().createTopic("test-topic-1", NUM_PARTITIONS);
+        primary.kafka().createTopic("backup.test-topic-1", 1);
+        primary.kafka().createTopic("heartbeats", 1);
+        backup.kafka().createTopic("test-topic-1", NUM_PARTITIONS);
+        backup.kafka().createTopic("primary.test-topic-1", 1);
+        backup.kafka().createTopic("heartbeats", 1);
+
+        for (int i = 0; i < NUM_RECORDS_PRODUCED; i++) {
+            primary.kafka().produce("test-topic-1", i % NUM_PARTITIONS, "key", "message-1-" + i);
+            backup.kafka().produce("test-topic-1", i % NUM_PARTITIONS, "key", "message-2-" + i);
+        }
+
+        // create consumers before starting the connectors so we don't need to wait for discovery
+        Consumer consumer1 = primary.kafka().createConsumerAndSubscribeTo(Collections.singletonMap(
+            "group.id", "consumer-group-1"), "test-topic-1", "backup.test-topic-1");
+        consumer1.poll(Duration.ofMillis(500));
+        consumer1.commitSync();
+        consumer1.close();
+
+        Consumer consumer2 = backup.kafka().createConsumerAndSubscribeTo(Collections.singletonMap(
+            "group.id", "consumer-group-1"), "test-topic-1", "primary.test-topic-1");
+        consumer2.poll(Duration.ofMillis(500));
+        consumer2.commitSync();
+        consumer2.close();
+
+        log.info("primary REST service: {}", primary.endpointForResource("connectors"));
+        log.info("backup REST service: {}", backup.endpointForResource("connectors"));
+ 
+        log.info("primary brokers: {}", primary.kafka().bootstrapServers());
+        log.info("backup brokers: {}", backup.kafka().bootstrapServers());
+        
+        // now that the brokers are running, we can finish setting up the Connectors
+        mm2Props.put("primary.bootstrap.servers", primary.kafka().bootstrapServers());
+        mm2Props.put("backup.bootstrap.servers", backup.kafka().bootstrapServers());
+        mm2Config = new MirrorMakerConfig(mm2Props);
+
+        backup.configureConnector("MirrorSourceConnector", mm2Config.connectorBaseConfig(new SourceAndTarget("primary", "backup"),
+            MirrorSourceConnector.class));
+
+        backup.configureConnector("MirrorCheckpointConnector", mm2Config.connectorBaseConfig(new SourceAndTarget("primary", "backup"),
+            MirrorCheckpointConnector.class));
+
+        backup.configureConnector("MirrorHeartbeatConnector", mm2Config.connectorBaseConfig(new SourceAndTarget("primary", "backup"),
+            MirrorHeartbeatConnector.class));
+
+        primary.configureConnector("MirrorSourceConnector", mm2Config.connectorBaseConfig(new SourceAndTarget("backup", "primary"),
+            MirrorSourceConnector.class));
+
+        primary.configureConnector("MirrorCheckpointConnector", mm2Config.connectorBaseConfig(new SourceAndTarget("backup", "primary"),
+            MirrorCheckpointConnector.class));
+
+        primary.configureConnector("MirrorHeartbeatConnector", mm2Config.connectorBaseConfig(new SourceAndTarget("backup", "primary"),
+            MirrorHeartbeatConnector.class));
+    }
+
+    @After
+    public void close() throws IOException {
+        for (String x : primary.connectors()) {
+            primary.deleteConnector(x);
+        }
+        for (String x : backup.connectors()) {
+            backup.deleteConnector(x);
+        }
+        deleteAllTopics(primary.kafka());
+        deleteAllTopics(backup.kafka());
+        primary.stop();
+        backup.stop();
+    }
+
+    @Test
+    public void testReplication() throws InterruptedException, TimeoutException {
+        MirrorClient primaryClient = new MirrorClient(mm2Config.clientConfig("primary"));
+        MirrorClient backupClient = new MirrorClient(mm2Config.clientConfig("backup"));
+
+        assertEquals("Records were not produced to primary cluster.", NUM_RECORDS_PRODUCED,
+            primary.kafka().consume(NUM_RECORDS_PRODUCED, RECORD_TRANSFER_DURATION_MS, "test-topic-1").count());
+        assertEquals("Records were not replicated to backup cluster.", NUM_RECORDS_PRODUCED,
+            backup.kafka().consume(NUM_RECORDS_PRODUCED, RECORD_TRANSFER_DURATION_MS, "primary.test-topic-1").count());
+        assertEquals("Records were not produced to backup cluster.", NUM_RECORDS_PRODUCED,
+            backup.kafka().consume(NUM_RECORDS_PRODUCED, RECORD_TRANSFER_DURATION_MS, "test-topic-1").count());
+        assertEquals("Records were not replicated to primary cluster.", NUM_RECORDS_PRODUCED,
+            primary.kafka().consume(NUM_RECORDS_PRODUCED, RECORD_TRANSFER_DURATION_MS, "backup.test-topic-1").count());
+        assertEquals("Primary cluster doesn't have all records from both clusters.", NUM_RECORDS_PRODUCED * 2,
+            primary.kafka().consume(NUM_RECORDS_PRODUCED * 2, RECORD_TRANSFER_DURATION_MS, "backup.test-topic-1", "test-topic-1").count());
+        assertEquals("Backup cluster doesn't have all records from both clusters.", NUM_RECORDS_PRODUCED * 2,
+            backup.kafka().consume(NUM_RECORDS_PRODUCED * 2, RECORD_TRANSFER_DURATION_MS, "primary.test-topic-1", "test-topic-1").count());
+        assertTrue("Heartbeats were not emitted to primary cluster.", primary.kafka().consume(1,
+            RECORD_TRANSFER_DURATION_MS, "heartbeats").count() > 0);
+        assertTrue("Heartbeats were not emitted to backup cluster.", backup.kafka().consume(1,
+            RECORD_TRANSFER_DURATION_MS, "heartbeats").count() > 0);
+        assertTrue("Heartbeats were not replicated downstream to backup cluster.", backup.kafka().consume(1,
+            RECORD_TRANSFER_DURATION_MS, "primary.heartbeats").count() > 0);
+        assertTrue("Heartbeats were not replicated downstream to primary cluster.", primary.kafka().consume(1,
+            RECORD_TRANSFER_DURATION_MS, "backup.heartbeats").count() > 0);
+        assertTrue("Did not find upstream primary cluster.", backupClient.upstreamClusters().contains("primary"));
+        assertEquals("Did not calculate replication hops correctly.", 1, backupClient.replicationHops("primary"));
+        assertTrue("Did not find upstream backup cluster.", primaryClient.upstreamClusters().contains("backup"));
+        assertEquals("Did not calculate replication hops correctly.", 1, primaryClient.replicationHops("backup"));
+        assertTrue("Checkpoints were not emitted downstream to backup cluster.", backup.kafka().consume(1,
+            CHECKPOINT_DURATION_MS, "primary.checkpoints.internal").count() > 0);
+
+        Map backupOffsets = backupClient.remoteConsumerOffsets("consumer-group-1", "primary",
+            Duration.ofMillis(CHECKPOINT_DURATION_MS));
+
+        assertTrue("Offsets not translated downstream to backup cluster. Found: " + backupOffsets, backupOffsets.containsKey(
+            new TopicPartition("primary.test-topic-1", 0)));
+
+        // Failover consumer group to backup cluster.
+        Consumer consumer1 = backup.kafka().createConsumer(Collections.singletonMap("group.id", "consumer-group-1"));
+        consumer1.assign(backupOffsets.keySet());
+        backupOffsets.forEach(consumer1::seek);
+        consumer1.poll(Duration.ofMillis(500));
+        consumer1.commitSync();
+
+        assertTrue("Consumer failedover to zero offset.", consumer1.position(new TopicPartition("primary.test-topic-1", 0)) > 0);
+        assertTrue("Consumer failedover beyond expected offset.", consumer1.position(
+            new TopicPartition("primary.test-topic-1", 0)) <= NUM_RECORDS_PRODUCED);
+        assertTrue("Checkpoints were not emitted upstream to primary cluster.", primary.kafka().consume(1,
+            CHECKPOINT_DURATION_MS, "backup.checkpoints.internal").count() > 0);
+
+        consumer1.close();
+
+        waitForCondition(() -> {
+            try {
+                return primaryClient.remoteConsumerOffsets("consumer-group-1", "backup",
+                    Duration.ofMillis(CHECKPOINT_DURATION_MS)).containsKey(new TopicPartition("backup.test-topic-1", 0));
+            } catch (Throwable e) {
+                return false;
+            }
+        }, CHECKPOINT_DURATION_MS, "Offsets not translated downstream to primary cluster.");
+
+        waitForCondition(() -> {
+            try {
+                return primaryClient.remoteConsumerOffsets("consumer-group-1", "backup",
+                    Duration.ofMillis(CHECKPOINT_DURATION_MS)).containsKey(new TopicPartition("test-topic-1", 0));
+            } catch (Throwable e) {
+                return false;
+            }
+        }, CHECKPOINT_DURATION_MS, "Offsets not translated upstream to primary cluster.");
+
+        Map primaryOffsets = primaryClient.remoteConsumerOffsets("consumer-group-1", "backup",
+                Duration.ofMillis(CHECKPOINT_DURATION_MS));
+ 
+        // Failback consumer group to primary cluster
+        Consumer consumer2 = primary.kafka().createConsumer(Collections.singletonMap("group.id", "consumer-group-1"));
+        consumer2.assign(primaryOffsets.keySet());
+        primaryOffsets.forEach(consumer2::seek);
+        consumer2.poll(Duration.ofMillis(500));
+
+        assertTrue("Consumer failedback to zero upstream offset.", consumer2.position(new TopicPartition("test-topic-1", 0)) > 0);
+        assertTrue("Consumer failedback to zero downstream offset.", consumer2.position(new TopicPartition("backup.test-topic-1", 0)) > 0);
+        assertTrue("Consumer failedback beyond expected upstream offset.", consumer2.position(
+            new TopicPartition("test-topic-1", 0)) <= NUM_RECORDS_PRODUCED);
+        assertTrue("Consumer failedback beyond expected downstream offset.", consumer2.position(
+            new TopicPartition("backup.test-topic-1", 0)) <= NUM_RECORDS_PRODUCED);
+        
+        consumer2.close();
+      
+        // create more matching topics
+        primary.kafka().createTopic("test-topic-2", NUM_PARTITIONS);
+        backup.kafka().createTopic("test-topic-3", NUM_PARTITIONS);
+
+        for (int i = 0; i < NUM_RECORDS_PRODUCED; i++) {
+            primary.kafka().produce("test-topic-2", 0, "key", "message-2-" + i);
+            backup.kafka().produce("test-topic-3", 0, "key", "message-3-" + i);
+        }
+
+        assertEquals("Records were not produced to primary cluster.", NUM_RECORDS_PRODUCED,
+            primary.kafka().consume(NUM_RECORDS_PRODUCED, RECORD_TRANSFER_DURATION_MS, "test-topic-2").count());
+        assertEquals("Records were not produced to backup cluster.", NUM_RECORDS_PRODUCED,
+            backup.kafka().consume(NUM_RECORDS_PRODUCED, RECORD_TRANSFER_DURATION_MS, "test-topic-3").count());
+ 
+        assertEquals("New topic was not replicated to primary cluster.", NUM_RECORDS_PRODUCED,
+            primary.kafka().consume(NUM_RECORDS_PRODUCED, 2 * RECORD_TRANSFER_DURATION_MS, "backup.test-topic-3").count());
+        assertEquals("New topic was not replicated to backup cluster.", NUM_RECORDS_PRODUCED,
+            backup.kafka().consume(NUM_RECORDS_PRODUCED, 2 * RECORD_TRANSFER_DURATION_MS, "primary.test-topic-2").count());
+    }
+
+    private void deleteAllTopics(EmbeddedKafkaCluster cluster) {
+        Admin client = cluster.createAdminClient();
+        try {
+            client.deleteTopics(client.listTopics().names().get());
+        } catch (Throwable e) {
+        }
+    }
+}
diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorMakerConfigTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorMakerConfigTest.java
new file mode 100644
index 0000000000000..b618e37c3b3ba
--- /dev/null
+++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorMakerConfigTest.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.config.types.Password;
+import org.apache.kafka.common.config.provider.ConfigProvider;
+import org.apache.kafka.common.config.ConfigData;
+import org.apache.kafka.common.metrics.FakeMetricsReporter;
+
+import org.junit.Test;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Arrays;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+public class MirrorMakerConfigTest {
+
+    private Map makeProps(String... keyValues) {
+        Map props = new HashMap<>();
+        for (int i = 0; i < keyValues.length; i += 2) {
+            props.put(keyValues[i], keyValues[i + 1]);
+        }
+        return props;
+    }
+
+    @Test
+    public void testClusterConfigProperties() {
+        MirrorMakerConfig mirrorConfig = new MirrorMakerConfig(makeProps(
+            "clusters", "a, b",
+            "a.bootstrap.servers", "servers-one",
+            "b.bootstrap.servers", "servers-two",
+            "security.protocol", "SASL",
+            "replication.factor", "4"));
+        Map connectorProps = mirrorConfig.connectorBaseConfig(new SourceAndTarget("a", "b"),
+            MirrorSourceConnector.class);
+        assertEquals("source.cluster.bootstrap.servers is set",
+            "servers-one", connectorProps.get("source.cluster.bootstrap.servers"));
+        assertEquals("target.cluster.bootstrap.servers is set",
+            "servers-two", connectorProps.get("target.cluster.bootstrap.servers"));
+        assertEquals("top-level security.protocol is passed through to connector config",
+            "SASL", connectorProps.get("security.protocol"));
+    }
+
+    @Test
+    public void testReplicationConfigProperties() {
+        MirrorMakerConfig mirrorConfig = new MirrorMakerConfig(makeProps(
+            "clusters", "a, b",
+            "a->b.tasks.max", "123"));
+        Map connectorProps = mirrorConfig.connectorBaseConfig(new SourceAndTarget("a", "b"),
+            MirrorSourceConnector.class);
+        assertEquals("connector props should include tasks.max",
+            "123", connectorProps.get("tasks.max"));
+    }
+
+    @Test
+    public void testClientConfigProperties() {
+        MirrorMakerConfig mirrorConfig = new MirrorMakerConfig(makeProps(
+            "clusters", "a, b",
+            "config.providers", "fake",
+            "config.providers.fake.class", FakeConfigProvider.class.getName(),
+            "replication.policy.separator", "__",
+            "ssl.truststore.password", "secret1",
+            "ssl.key.password", "${fake:secret:password}",  // resolves to "secret2"
+            "security.protocol", "SSL", 
+            "a.security.protocol", "PLAINTEXT", 
+            "a.producer.security.protocol", "SASL", 
+            "a.bootstrap.servers", "one:9092, two:9092",
+            "metrics.reporter", FakeMetricsReporter.class.getName(),
+            "a.metrics.reporter", FakeMetricsReporter.class.getName(),
+            "b->a.metrics.reporter", FakeMetricsReporter.class.getName(),
+            "a.xxx", "yyy",
+            "xxx", "zzz"));
+        MirrorClientConfig aClientConfig = mirrorConfig.clientConfig("a");
+        MirrorClientConfig bClientConfig = mirrorConfig.clientConfig("b");
+        assertEquals("replication.policy.separator is picked up in MirrorClientConfig",
+            "__", aClientConfig.getString("replication.policy.separator"));
+        assertEquals("replication.policy.separator is honored",
+            "b__topic1", aClientConfig.replicationPolicy().formatRemoteTopic("b", "topic1"));
+        assertEquals("client configs include boostrap.servers",
+            "one:9092, two:9092", aClientConfig.adminConfig().get("bootstrap.servers"));
+        assertEquals("client configs include security.protocol",
+            "PLAINTEXT", aClientConfig.adminConfig().get("security.protocol"));
+        assertEquals("producer configs include security.protocol",
+            "SASL", aClientConfig.producerConfig().get("security.protocol"));
+        assertFalse("unknown properties aren't included in client configs",
+            aClientConfig.adminConfig().containsKey("xxx"));
+        assertFalse("top-leve metrics reporters aren't included in client configs",
+            aClientConfig.adminConfig().containsKey("metric.reporters"));
+        assertEquals("security properties are picked up in MirrorClientConfig",
+            "secret1", aClientConfig.getPassword("ssl.truststore.password").value());
+        assertEquals("client configs include top-level security properties",
+            "secret1", ((Password) aClientConfig.adminConfig().get("ssl.truststore.password")).value());
+        assertEquals("security properties are translated from external sources",
+            "secret2", aClientConfig.getPassword("ssl.key.password").value());
+        assertEquals("client configs are translated from external sources",
+            "secret2", ((Password) aClientConfig.adminConfig().get("ssl.key.password")).value());
+        assertFalse("client configs should not include metrics reporter",
+            aClientConfig.producerConfig().containsKey("metrics.reporter"));
+        assertFalse("client configs should not include metrics reporter",
+            bClientConfig.adminConfig().containsKey("metrics.reporter"));
+    }
+
+    @Test
+    public void testIncludesConnectorConfigProperties() {
+        MirrorMakerConfig mirrorConfig = new MirrorMakerConfig(makeProps(
+            "clusters", "a, b",
+            "tasks.max", "100",
+            "topics", "topic-1",
+            "groups", "group-2",
+            "replication.policy.separator", "__",
+            "config.properties.blacklist", "property-3",
+            "metric.reporters", "FakeMetricsReporter",
+            "topic.filter.class", DefaultTopicFilter.class.getName(),
+            "xxx", "yyy"));
+        SourceAndTarget sourceAndTarget = new SourceAndTarget("source", "target");
+        Map connectorProps = mirrorConfig.connectorBaseConfig(sourceAndTarget,
+            MirrorSourceConnector.class);
+        MirrorConnectorConfig connectorConfig = new MirrorConnectorConfig(connectorProps);
+        assertEquals("Connector properties like tasks.max should be passed through to underlying Connectors.",
+            100, (int) connectorConfig.getInt("tasks.max"));
+        assertEquals("Topics whitelist should be passed through to underlying Connectors.",
+            Arrays.asList("topic-1"), connectorConfig.getList("topics"));
+        assertEquals("Groups whitelist should be passed through to underlying Connectors.",
+            Arrays.asList("group-2"), connectorConfig.getList("groups"));
+        assertEquals("Config properties blacklist should be passed through to underlying Connectors.",
+            Arrays.asList("property-3"), connectorConfig.getList("config.properties.blacklist"));
+        assertEquals("Metrics reporters should be passed through to underlying Connectors.",
+            Arrays.asList("FakeMetricsReporter"), connectorConfig.getList("metric.reporters"));
+        assertEquals("Filters should be passed through to underlying Connectors.",
+            "DefaultTopicFilter", connectorConfig.getClass("topic.filter.class").getSimpleName());
+        assertEquals("replication policy separator should be passed through to underlying Connectors.",
+            "__", connectorConfig.getString("replication.policy.separator"));
+        assertFalse("Unknown properties should not be passed through to Connectors.",
+            connectorConfig.originals().containsKey("xxx"));
+    }
+
+    @Test
+    public void testIncludesTopicFilterProperties() {
+        MirrorMakerConfig mirrorConfig = new MirrorMakerConfig(makeProps(
+            "clusters", "a, b",
+            "source->target.topics", "topic1, topic2",
+            "source->target.topics.blacklist", "topic3"));
+        SourceAndTarget sourceAndTarget = new SourceAndTarget("source", "target");
+        Map connectorProps = mirrorConfig.connectorBaseConfig(sourceAndTarget,
+            MirrorSourceConnector.class);
+        DefaultTopicFilter.TopicFilterConfig filterConfig = 
+            new DefaultTopicFilter.TopicFilterConfig(connectorProps);
+        assertEquals("source->target.topics should be passed through to TopicFilters.",
+            Arrays.asList("topic1", "topic2"), filterConfig.getList("topics"));
+        assertEquals("source->target.topics.blacklist should be passed through to TopicFilters.",
+            Arrays.asList("topic3"), filterConfig.getList("topics.blacklist"));
+    }
+
+    @Test
+    public void testWorkerConfigs() {
+        MirrorMakerConfig mirrorConfig = new MirrorMakerConfig(makeProps(
+            "clusters", "a, b",
+            "config.providers", "fake",
+            "config.providers.fake.class", FakeConfigProvider.class.getName(),
+            "replication.policy.separator", "__",
+            "offset.storage.replication.factor", "123",
+            "b.status.storage.replication.factor", "456",
+            "b.producer.client.id", "client-one",
+            "b.security.protocol", "PLAINTEXT",
+            "b.producer.security.protocol", "SASL",
+            "ssl.truststore.password", "secret1",
+            "ssl.key.password", "${fake:secret:password}",  // resolves to "secret2"
+            "b.xxx", "yyy"));
+        SourceAndTarget a = new SourceAndTarget("b", "a");
+        SourceAndTarget b = new SourceAndTarget("a", "b");
+        Map aProps = mirrorConfig.workerConfig(a);
+        assertEquals("123", aProps.get("offset.storage.replication.factor"));
+        Map bProps = mirrorConfig.workerConfig(b);
+        assertEquals("456", bProps.get("status.storage.replication.factor"));
+        assertEquals("producer props should be passed through to worker producer config: " + bProps,
+            "client-one", bProps.get("producer.client.id"));
+        assertEquals("replication-level security props should be passed through to worker producer config",
+            "SASL", bProps.get("producer.security.protocol"));
+        assertEquals("replication-level security props should be passed through to worker producer config",
+            "SASL", bProps.get("producer.security.protocol"));
+        assertEquals("replication-level security props should be passed through to worker consumer config",
+            "PLAINTEXT", bProps.get("consumer.security.protocol"));
+        assertEquals("security properties should be passed through to worker config: " + bProps,
+            "secret1", bProps.get("ssl.truststore.password"));
+        assertEquals("security properties should be passed through to worker producer config: " + bProps,
+            "secret1", bProps.get("producer.ssl.truststore.password"));
+        assertEquals("security properties should be transformed in worker config",
+            "secret2", bProps.get("ssl.key.password"));
+        assertEquals("security properties should be transformed in worker producer config",
+            "secret2", bProps.get("producer.ssl.key.password"));
+    }
+
+    public static class FakeConfigProvider implements ConfigProvider {
+
+        Map secrets = Collections.singletonMap("password", "secret2");
+
+        @Override
+        public void configure(Map props) {
+        }
+
+        @Override
+        public void close() {
+        }
+
+        @Override
+        public ConfigData get(String path) {
+            return new ConfigData(secrets);
+        }
+
+        @Override
+        public ConfigData get(String path, Set keys) {
+            return get(path);
+        }
+    }
+}
diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConnectorTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConnectorTest.java
new file mode 100644
index 0000000000000..b1ccef8158df7
--- /dev/null
+++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConnectorTest.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.common.acl.AccessControlEntry;
+import org.apache.kafka.common.acl.AclBinding;
+import org.apache.kafka.common.acl.AclOperation;
+import org.apache.kafka.common.acl.AclPermissionType;
+import org.apache.kafka.common.resource.PatternType;
+import org.apache.kafka.common.resource.ResourcePattern;
+import org.apache.kafka.common.resource.ResourceType;
+import org.apache.kafka.clients.admin.Config;
+import org.apache.kafka.clients.admin.ConfigEntry;
+
+import org.junit.Test;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+import java.util.ArrayList;
+
+public class MirrorSourceConnectorTest {
+
+    @Test
+    public void testReplicatesHeartbeatsByDefault() {
+        MirrorSourceConnector connector = new MirrorSourceConnector(new SourceAndTarget("source", "target"), 
+            new DefaultReplicationPolicy(), new DefaultTopicFilter(), new DefaultConfigPropertyFilter());
+        assertTrue("should replicate heartbeats", connector.shouldReplicateTopic("heartbeats"));
+        assertTrue("should replicate upstream heartbeats", connector.shouldReplicateTopic("us-west.heartbeats"));
+    }
+
+    @Test
+    public void testReplicatesHeartbeatsDespiteFilter() {
+        MirrorSourceConnector connector = new MirrorSourceConnector(new SourceAndTarget("source", "target"),
+            new DefaultReplicationPolicy(), x -> false, new DefaultConfigPropertyFilter());
+        assertTrue("should replicate heartbeats", connector.shouldReplicateTopic("heartbeats"));
+        assertTrue("should replicate upstream heartbeats", connector.shouldReplicateTopic("us-west.heartbeats"));
+    }
+
+    @Test
+    public void testNoCycles() {
+        MirrorSourceConnector connector = new MirrorSourceConnector(new SourceAndTarget("source", "target"),
+            new DefaultReplicationPolicy(), x -> true, x -> true);
+        assertFalse("should not allow cycles", connector.shouldReplicateTopic("target.topic1"));
+        assertFalse("should not allow cycles", connector.shouldReplicateTopic("target.source.topic1"));
+        assertFalse("should not allow cycles", connector.shouldReplicateTopic("source.target.topic1"));
+        assertTrue("should allow anything else", connector.shouldReplicateTopic("topic1"));
+        assertTrue("should allow anything else", connector.shouldReplicateTopic("source.topic1"));
+    }
+
+    @Test
+    public void testAclFiltering() {
+        MirrorSourceConnector connector = new MirrorSourceConnector(new SourceAndTarget("source", "target"),
+            new DefaultReplicationPolicy(), x -> true, x -> true);
+        assertFalse("should not replicate ALLOW WRITE", connector.shouldReplicateAcl(
+            new AclBinding(new ResourcePattern(ResourceType.TOPIC, "test_topic", PatternType.LITERAL),
+            new AccessControlEntry("kafka", "", AclOperation.WRITE, AclPermissionType.ALLOW))));
+        assertTrue("should replicate ALLOW ALL", connector.shouldReplicateAcl(
+            new AclBinding(new ResourcePattern(ResourceType.TOPIC, "test_topic", PatternType.LITERAL),
+            new AccessControlEntry("kafka", "", AclOperation.ALL, AclPermissionType.ALLOW))));
+    }
+
+    @Test
+    public void testAclTransformation() {
+        MirrorSourceConnector connector = new MirrorSourceConnector(new SourceAndTarget("source", "target"),
+            new DefaultReplicationPolicy(), x -> true, x -> true);
+        AclBinding allowAllAclBinding = new AclBinding(
+            new ResourcePattern(ResourceType.TOPIC, "test_topic", PatternType.LITERAL),
+            new AccessControlEntry("kafka", "", AclOperation.ALL, AclPermissionType.ALLOW));
+        AclBinding processedAllowAllAclBinding = connector.targetAclBinding(allowAllAclBinding);
+        String expectedRemoteTopicName = "source" + DefaultReplicationPolicy.SEPARATOR_DEFAULT
+            + allowAllAclBinding.pattern().name();
+        assertTrue("should change topic name",
+            processedAllowAllAclBinding.pattern().name().equals(expectedRemoteTopicName));
+        assertTrue("should change ALL to READ", processedAllowAllAclBinding.entry().operation() == AclOperation.READ);
+        assertTrue("should not change ALLOW",
+            processedAllowAllAclBinding.entry().permissionType() == AclPermissionType.ALLOW);
+
+        AclBinding denyAllAclBinding = new AclBinding(
+            new ResourcePattern(ResourceType.TOPIC, "test_topic", PatternType.LITERAL),
+            new AccessControlEntry("kafka", "", AclOperation.ALL, AclPermissionType.DENY));
+        AclBinding processedDenyAllAclBinding = connector.targetAclBinding(denyAllAclBinding);
+        assertTrue("should not change ALL", processedDenyAllAclBinding.entry().operation() == AclOperation.ALL);
+        assertTrue("should not change DENY",
+            processedDenyAllAclBinding.entry().permissionType() == AclPermissionType.DENY);
+    }
+    
+    @Test
+    public void testConfigPropertyFiltering() {
+        MirrorSourceConnector connector = new MirrorSourceConnector(new SourceAndTarget("source", "target"),
+            new DefaultReplicationPolicy(), x -> true, new DefaultConfigPropertyFilter());
+        ArrayList entries = new ArrayList<>();
+        entries.add(new ConfigEntry("name-1", "value-1"));
+        entries.add(new ConfigEntry("min.insync.replicas", "2"));
+        Config config = new Config(entries);
+        Config targetConfig = connector.targetConfig(config);
+        assertTrue("should replicate properties", targetConfig.entries().stream()
+            .anyMatch(x -> x.name().equals("name-1")));
+        assertFalse("should not replicate blacklisted properties", targetConfig.entries().stream()
+            .anyMatch(x -> x.name().equals("min.insync.replicas")));
+    }
+}
diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceTaskTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceTaskTest.java
new file mode 100644
index 0000000000000..003511760fa26
--- /dev/null
+++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceTaskTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.common.record.TimestampType;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.header.Headers;
+import org.apache.kafka.common.header.internals.RecordHeaders;
+import org.apache.kafka.connect.source.SourceRecord;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+public class MirrorSourceTaskTest {
+
+    @Test
+    public void testSerde() {
+        byte[] key = new byte[]{'a', 'b', 'c', 'd', 'e'};
+        byte[] value = new byte[]{'f', 'g', 'h', 'i', 'j', 'k'};
+        Headers headers = new RecordHeaders();
+        headers.add("header1", new byte[]{'l', 'm', 'n', 'o'});
+        headers.add("header2", new byte[]{'p', 'q', 'r', 's', 't'});
+        ConsumerRecord consumerRecord = new ConsumerRecord<>("topic1", 2, 3L, 4L,
+            TimestampType.CREATE_TIME, 0L, 5, 6, key, value, headers);
+        MirrorSourceTask mirrorSourceTask = new MirrorSourceTask("cluster7",
+            new DefaultReplicationPolicy(), 50);
+        SourceRecord sourceRecord = mirrorSourceTask.convertRecord(consumerRecord);
+        assertEquals("cluster7.topic1", sourceRecord.topic());
+        assertEquals(2, sourceRecord.kafkaPartition().intValue());
+        assertEquals(new TopicPartition("topic1", 2), MirrorUtils.unwrapPartition(sourceRecord.sourcePartition()));
+        assertEquals(3L, MirrorUtils.unwrapOffset(sourceRecord.sourceOffset()).longValue());
+        assertEquals(4L, sourceRecord.timestamp().longValue());
+        assertEquals(key, sourceRecord.key());
+        assertEquals(value, sourceRecord.value());
+        assertEquals(headers.lastHeader("header1").value(), sourceRecord.headers().lastWithName("header1").value());
+        assertEquals(headers.lastHeader("header2").value(), sourceRecord.headers().lastWithName("header2").value());
+    }
+
+    @Test
+    public void testOffsetSync() {
+        MirrorSourceTask.PartitionState partitionState = new MirrorSourceTask.PartitionState(50);
+
+        assertTrue("always emit offset sync on first update",
+            partitionState.update(0, 100));
+        assertTrue("upstream offset skipped -> resync",
+            partitionState.update(2, 102));
+        assertFalse("no sync",
+            partitionState.update(3, 152));
+        assertFalse("no sync",
+            partitionState.update(4, 153));
+        assertFalse("no sync",
+            partitionState.update(5, 154));
+        assertTrue("one past target offset",
+            partitionState.update(6, 205));
+        assertTrue("upstream reset",
+            partitionState.update(2, 206));
+        assertFalse("no sync",
+            partitionState.update(3, 207));
+        assertTrue("downstream reset",
+                partitionState.update(4, 3));
+        assertFalse("no sync",
+            partitionState.update(5, 4));
+    }
+
+    @Test
+    public void testZeroOffsetSync() {
+        MirrorSourceTask.PartitionState partitionState = new MirrorSourceTask.PartitionState(0);
+
+        // if max offset lag is zero, should always emit offset syncs
+        assertTrue(partitionState.update(0, 100));
+        assertTrue(partitionState.update(2, 102));
+        assertTrue(partitionState.update(3, 153));
+        assertTrue(partitionState.update(4, 154));
+        assertTrue(partitionState.update(5, 155));
+        assertTrue(partitionState.update(6, 207));
+        assertTrue(partitionState.update(2, 208));
+        assertTrue(partitionState.update(3, 209));
+        assertTrue(partitionState.update(4, 3));
+        assertTrue(partitionState.update(5, 4));
+    }
+}
diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/OffsetSyncStoreTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/OffsetSyncStoreTest.java
new file mode 100644
index 0000000000000..19954cd8fcaf4
--- /dev/null
+++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/OffsetSyncStoreTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.common.TopicPartition;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class OffsetSyncStoreTest {
+
+    static TopicPartition tp = new TopicPartition("topic1", 2);
+
+    static class FakeOffsetSyncStore extends OffsetSyncStore {
+
+        FakeOffsetSyncStore() {
+            super(null, null);
+        }
+
+        void sync(TopicPartition topicPartition, long upstreamOffset, long downstreamOffset) {
+            OffsetSync offsetSync = new OffsetSync(topicPartition, upstreamOffset, downstreamOffset);
+            byte[] key = offsetSync.recordKey();
+            byte[] value = offsetSync.recordValue();
+            ConsumerRecord record = new ConsumerRecord<>("test.offsets.internal", 0, 3, key, value);
+            handleRecord(record);
+        }
+    }
+
+    @Test
+    public void testOffsetTranslation() {
+        FakeOffsetSyncStore store = new FakeOffsetSyncStore();
+
+        store.sync(tp, 100, 200);
+        assertEquals(store.translateDownstream(tp, 150), 250);
+
+        // Translate exact offsets
+        store.sync(tp, 150, 251);
+        assertEquals(store.translateDownstream(tp, 150), 251);
+
+        // Use old offset (5) prior to any sync -> can't translate
+        assertEquals(-1, store.translateDownstream(tp, 5));
+
+        // Downstream offsets reset
+        store.sync(tp, 200, 10);
+        assertEquals(store.translateDownstream(tp, 200), 10);
+
+        // Upstream offsets reset
+        store.sync(tp, 20, 20);
+        assertEquals(store.translateDownstream(tp, 20), 20);
+    }
+}
diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/OffsetSyncTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/OffsetSyncTest.java
new file mode 100644
index 0000000000000..5dc472964b345
--- /dev/null
+++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/OffsetSyncTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.mirror;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.common.TopicPartition;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class OffsetSyncTest {
+
+    @Test
+    public void testSerde() {
+        OffsetSync offsetSync = new OffsetSync(new TopicPartition("topic-1", 2), 3, 4);
+        byte[] key = offsetSync.recordKey();
+        byte[] value = offsetSync.recordValue();
+        ConsumerRecord record = new ConsumerRecord<>("any-topic", 6, 7, key, value);
+        OffsetSync deserialized = OffsetSync.deserializeRecord(record);
+        assertEquals(offsetSync.topicPartition(), deserialized.topicPartition());
+        assertEquals(offsetSync.upstreamOffset(), deserialized.upstreamOffset());
+        assertEquals(offsetSync.downstreamOffset(), deserialized.downstreamOffset());
+    }
+}
diff --git a/connect/mirror/src/test/resources/log4j.properties b/connect/mirror/src/test/resources/log4j.properties
new file mode 100644
index 0000000000000..a2ac021dfab98
--- /dev/null
+++ b/connect/mirror/src/test/resources/log4j.properties
@@ -0,0 +1,34 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+log4j.rootLogger=ERROR, stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+#
+# The `%X{connector.context}` parameter in the layout includes connector-specific and task-specific information
+# in the log message, where appropriate. This makes it easier to identify those log messages that apply to a
+# specific connector. Simply add this parameter to the log layout configuration below to include the contextual information.
+#
+log4j.appender.stdout.layout.ConversionPattern=[%d] %p %X{connector.context}%m (%c:%L)%n
+#
+# The following line includes no MDC context parameters:
+#log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n (%t)
+
+log4j.logger.org.reflections=OFF
+log4j.logger.kafka=OFF
+log4j.logger.state.change.logger=OFF
+log4j.logger.org.apache.kafka.connect.mirror=INFO
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
index 41b56c8900f6b..c66429d31b53f 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
@@ -309,7 +309,7 @@ private boolean sendRecords() {
             final ProducerRecord producerRecord = convertTransformedRecord(record);
             if (producerRecord == null || retryWithToleranceOperator.failed()) {
                 counter.skipRecord();
-                commitTaskRecord(preTransformRecord);
+                commitTaskRecord(preTransformRecord, null);
                 continue;
             }
 
@@ -347,7 +347,7 @@ public void onCompletion(RecordMetadata recordMetadata, Exception e) {
                                             WorkerSourceTask.this,
                                             recordMetadata.topic(), recordMetadata.partition(),
                                             recordMetadata.offset());
-                                    commitTaskRecord(preTransformRecord);
+                                    commitTaskRecord(preTransformRecord, recordMetadata);
                                 }
                             }
                         });
@@ -381,9 +381,9 @@ private RecordHeaders convertHeaderFor(SourceRecord record) {
         return result;
     }
 
-    private void commitTaskRecord(SourceRecord record) {
+    private void commitTaskRecord(SourceRecord record, RecordMetadata metadata) {
         try {
-            task.commitRecord(record);
+            task.commitRecord(record, metadata);
         } catch (Throwable t) {
             log.error("{} Exception thrown while calling task.commitRecord()", this, t);
         }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
index bce38d7a64a7b..eb618fb0053f3 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
@@ -427,7 +427,7 @@ private synchronized boolean checkForKeyRotation(long now) {
                         + "than required by current worker configuration. Distributing new key now.");
                     return true;
                 }
-            } else if (sessionKey == null) {
+            } else if (sessionKey == null && configState.sessionKey() != null) {
                 // This happens on startup for follower workers; the snapshot contains the session key,
                 // but no callback in the config update listener has been fired for it yet.
                 sessionKey = configState.sessionKey().key();
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/PluginUtils.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/PluginUtils.java
index b9d5470745ade..36feac50d1824 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/PluginUtils.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/PluginUtils.java
@@ -131,6 +131,8 @@ public class PluginUtils {
             + "transforms\\.(?!Transformation$).*"
             + "|json\\..*"
             + "|file\\..*"
+            + "|mirror\\..*"
+            + "|mirror-client\\..*"
             + "|converters\\..*"
             + "|storage\\.StringConverter"
             + "|storage\\.SimpleHeaderConverter"
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java
index e45578320945e..fbd763a928035 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java
@@ -17,6 +17,7 @@
 package org.apache.kafka.connect.integration;
 
 import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.clients.producer.RecordMetadata;
 import org.apache.kafka.connect.connector.Task;
 import org.apache.kafka.connect.data.Schema;
 import org.apache.kafka.connect.runtime.TestSourceConnector;
@@ -150,7 +151,7 @@ public void commit() {
         }
 
         @Override
-        public void commitRecord(SourceRecord record) {
+        public void commitRecord(SourceRecord record, RecordMetadata metadata) {
             log.trace("Committing record: {}", record);
             taskHandle.commit();
         }
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java
index 8751f1cc38727..dff267a368e7f 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java
@@ -883,7 +883,7 @@ public Future answer() throws Throwable {
 
         if (sendSuccess) {
             // 3. As a result of a successful producer send callback, we'll notify the source task of the record commit
-            expectTaskCommitRecord(anyTimes, commitSuccess);
+            expectTaskCommitRecordWithOffset(anyTimes, commitSuccess);
         }
 
         return sent;
@@ -932,8 +932,8 @@ public SourceRecord answer() {
             });
     }
 
-    private void expectTaskCommitRecord(boolean anyTimes, boolean succeed) throws InterruptedException {
-        sourceTask.commitRecord(EasyMock.anyObject(SourceRecord.class));
+    private void expectTaskCommitRecordWithOffset(boolean anyTimes, boolean succeed) throws InterruptedException {
+        sourceTask.commitRecord(EasyMock.anyObject(SourceRecord.class), EasyMock.anyObject(RecordMetadata.class));
         IExpectationSetters expect = EasyMock.expectLastCall();
         if (!succeed) {
             expect = expect.andThrow(new RuntimeException("Error committing record in source task"));
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/PluginUtilsTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/PluginUtilsTest.java
index bf441ff7f1bab..c406ead57e66f 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/PluginUtilsTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/PluginUtilsTest.java
@@ -142,6 +142,12 @@ public void testAllowedConnectFrameworkClasses() {
         assertTrue(PluginUtils.shouldLoadInIsolation(
                 "org.apache.kafka.connect.file.FileStreamSinkConnector")
         );
+        assertTrue(PluginUtils.shouldLoadInIsolation(
+                "org.apache.kafka.connect.mirror.MirrorSourceTask")
+        );
+        assertTrue(PluginUtils.shouldLoadInIsolation(
+                "org.apache.kafka.connect.mirror.MirrorSourceConnector")
+        );
         assertTrue(PluginUtils.shouldLoadInIsolation("org.apache.kafka.connect.converters."));
         assertTrue(PluginUtils.shouldLoadInIsolation(
                 "org.apache.kafka.connect.converters.ByteArrayConverter")
diff --git a/settings.gradle b/settings.gradle
index a31ea133c8721..813cf6443effd 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -18,6 +18,8 @@ include 'clients',
     'connect:basic-auth-extension',
     'connect:file',
     'connect:json',
+    'connect:mirror',
+    'connect:mirror-client',
     'connect:runtime',
     'connect:transforms',
     'core',

From 133c33fde1c4d3b79196b72522f83a75cb6b0e65 Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Mon, 7 Oct 2019 09:27:09 -0700
Subject: [PATCH 0696/1071] KAFKA-8179: Part 7, cooperative rebalancing in
 Streams (#7386)

Key improvements with this PR:

* tasks will remain available for IQ during a rebalance (but not during restore)
* continue restoring and processing standby tasks during a rebalance
* continue processing active tasks during rebalance until the RecordQueue is empty*
* only revoked tasks must suspended/closed
* StreamsPartitionAssignor tries to return tasks to their previous consumers within a client
* but do not try to commit, for now (pending KAFKA-7312)


Reviewers: John Roesler , Boyang Chen , Guozhang Wang 
---
 checkstyle/suppressions.xml                   |   3 +
 .../internals/ConsumerCoordinator.java        |  20 +-
 .../consumer/internals/SubscriptionState.java |  15 +-
 .../consumer/internals/FetcherTest.java       |  31 +-
 .../internals/AssignedStreamsTasks.java       |  15 +-
 .../processor/internals/StandbyTask.java      |  19 -
 .../internals/StoreChangelogReader.java       |  16 +-
 .../processor/internals/StreamTask.java       |   8 +-
 .../processor/internals/StreamThread.java     |  27 +-
 .../internals/StreamsPartitionAssignor.java   | 612 +++++++++++++-----
 .../streams/processor/internals/Task.java     |   8 +-
 .../processor/internals/TaskManager.java      |  34 +-
 .../assignment/AssignorConfiguration.java     |  12 +-
 .../internals/assignment/ClientState.java     |  56 +-
 .../assignment/StickyTaskAssignor.java        |   8 +-
 .../processor/internals/AbstractTaskTest.java |   6 -
 .../processor/internals/StandbyTaskTest.java  |  23 -
 .../StreamsPartitionAssignorTest.java         | 481 ++++++++++++--
 .../processor/internals/TaskManagerTest.java  |  28 +-
 .../internals/assignment/ClientStateTest.java |   8 +-
 .../assignment/StickyTaskAssignorTest.java    |  34 +-
 .../kafka/streams/tests/SmokeTestDriver.java  |   2 +-
 .../streams/tests/StreamsUpgradeTest.java     |  26 +-
 .../tests/streams/streams_eos_test.py         |   2 +-
 24 files changed, 1058 insertions(+), 436 deletions(-)

diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index 8927849784d03..2f213090da56d 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -197,6 +197,9 @@
     
 
+    
+
     
 
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index b5b5ce291292e..6b39acb6d7f2c 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -355,26 +355,29 @@ protected void onJoinComplete(int generation,
         Set addedPartitions = new HashSet<>(assignedPartitions);
         addedPartitions.removeAll(ownedPartitions);
 
-        // Invoke user's revocation callback before changing assignment or updating state
         if (protocol == RebalanceProtocol.COOPERATIVE) {
             Set revokedPartitions = new HashSet<>(ownedPartitions);
             revokedPartitions.removeAll(assignedPartitions);
 
-            log.info("Updating with newly assigned partitions: {}, compare with already owned partitions: {}, " +
-                    "newly added partitions: {}, revoking partitions: {}",
+            log.info("Updating assignment with\n" +
+                    "now assigned partitions: {}\n" +
+                    "compare with previously owned partitions: {}\n" +
+                    "newly added partitions: {}\n" +
+                    "revoked partitions: {}\n",
                 Utils.join(assignedPartitions, ", "),
                 Utils.join(ownedPartitions, ", "),
                 Utils.join(addedPartitions, ", "),
-                Utils.join(revokedPartitions, ", "));
-
+                Utils.join(revokedPartitions, ", ")
+            );
 
             if (!revokedPartitions.isEmpty()) {
-                // revoke partitions that was previously owned but no longer assigned;
-                // note that we should only change the assignment AFTER we've triggered
-                // the revoke callback
+                // revoke partitions that were previously owned but no longer assigned;
+                // note that we should only change the assignment (or update the assignor's state)
+                // AFTER we've triggered  the revoke callback
                 firstException.compareAndSet(null, invokePartitionsRevoked(revokedPartitions));
 
                 // if revoked any partitions, need to re-join the group afterwards
+                log.debug("Need to revoke partitions {} and re-join the group", revokedPartitions);
                 requestRejoin();
             }
         }
@@ -679,7 +682,6 @@ protected void onJoinPrepare(int generation, String memberId) {
             }
         }
 
-
         isLeader = false;
         subscriptions.resetGroupSubscription();
 
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
index 4641e5c4917e1..953505f0b3e35 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
@@ -270,8 +270,14 @@ public synchronized void assignFromSubscribed(Collection assignm
         if (!this.partitionsAutoAssigned())
             throw new IllegalArgumentException("Attempt to dynamically assign partitions while manual assignment in use");
 
+        Map assignedPartitionStates = new HashMap<>(assignments.size());
+        for (TopicPartition tp : assignments) {
+            TopicPartitionState state = this.assignment.stateValue(tp);
+            if (state == null)
+                state = new TopicPartitionState();
+            assignedPartitionStates.put(tp, state);
+        }
 
-        Map assignedPartitionStates = partitionToStateMap(assignments);
         assignmentId++;
         this.assignment.set(assignedPartitionStates);
     }
@@ -674,13 +680,6 @@ public synchronized ConsumerRebalanceListener rebalanceListener() {
         return rebalanceListener;
     }
 
-    private static Map partitionToStateMap(Collection assignments) {
-        Map map = new HashMap<>(assignments.size());
-        for (TopicPartition tp : assignments)
-            map.put(tp, new TopicPartitionState());
-        return map;
-    }
-
     private static class TopicPartitionState {
 
         private FetchState fetchState;
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
index ff0afe92f7106..9e281a7b6f822 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
@@ -848,7 +848,7 @@ public void testUnauthorizedTopic() {
     }
 
     @Test
-    public void testFetchDuringRebalance() {
+    public void testFetchDuringEagerRebalance() {
         buildFetcher();
 
         subscriptions.subscribe(singleton(topicName), listener);
@@ -859,7 +859,9 @@ public void testFetchDuringRebalance() {
 
         assertEquals(1, fetcher.sendFetches());
 
-        // Now the rebalance happens and fetch positions are cleared
+        // Now the eager rebalance happens and fetch positions are cleared
+        subscriptions.assignFromSubscribed(Collections.emptyList());
+
         subscriptions.assignFromSubscribed(singleton(tp0));
         client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0));
         consumerClient.poll(time.timer(0));
@@ -868,6 +870,31 @@ public void testFetchDuringRebalance() {
         assertTrue(fetcher.fetchedRecords().isEmpty());
     }
 
+    @Test
+    public void testFetchDuringCooperativeRebalance() {
+        buildFetcher();
+
+        subscriptions.subscribe(singleton(topicName), listener);
+        subscriptions.assignFromSubscribed(singleton(tp0));
+        subscriptions.seek(tp0, 0);
+
+        client.updateMetadata(initialUpdateResponse);
+
+        assertEquals(1, fetcher.sendFetches());
+
+        // Now the cooperative rebalance happens and fetch positions are NOT cleared for unrevoked partitions
+        subscriptions.assignFromSubscribed(singleton(tp0));
+
+        client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0));
+        consumerClient.poll(time.timer(0));
+
+        Map>> fetchedRecords = fetchedRecords();
+
+        // The active fetch should NOT be ignored since the position for tp0 is still valid
+        assertEquals(1, fetchedRecords.size());
+        assertEquals(3, fetchedRecords.get(tp0).size());
+    }
+
     @Test
     public void testInFlightFetchOnPausedPartition() {
         buildFetcher();
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasks.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasks.java
index da0fc20939106..65c4c95fadeff 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasks.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasks.java
@@ -74,11 +74,15 @@ boolean allTasksRunning() {
     @Override
     void closeTask(final StreamTask task, final boolean clean) {
         if (suspended.containsKey(task.id())) {
-            task.closeSuspended(clean, false, null);
+            task.closeSuspended(clean, null);
         } else {
             task.close(clean, false);
         }
     }
+
+    boolean hasRestoringTasks() {
+        return !restoring.isEmpty();
+    }
     
     Set suspendedTaskIds() {
         return suspended.keySet();
@@ -107,7 +111,7 @@ RuntimeException suspendOrCloseTasks(final Set revokedTasks,
             } else if (restoring.containsKey(task)) {
                 revokedRestoringTasks.add(task);
             } else if (!suspended.containsKey(task)) {
-                log.warn("Task {} was revoked but cannot be found in the assignment", task);
+                log.warn("Task {} was revoked but cannot be found in the assignment, may have been closed due to error", task);
             }
         }
 
@@ -131,7 +135,7 @@ private RuntimeException suspendRunningTasks(final Set runningTasksToSus
                 task.suspend();
                 suspended.put(id, task);
             } catch (final TaskMigratedException closeAsZombieAndSwallow) {
-                // as we suspend a task, we are either shutting down or rebalancing, thus, we swallow and move on
+                // swallow and move on since we are rebalancing
                 log.info("Failed to suspend {} {} since it got migrated to another thread already. " +
                     "Closing it as zombie and move on.", taskTypeName, id);
                 firstException.compareAndSet(null, closeZombieTask(task));
@@ -248,7 +252,7 @@ private RuntimeException closeSuspended(final boolean isZombie,
 
         try {
             final boolean clean = !isZombie;
-            task.closeSuspended(clean, isZombie, null);
+            task.closeSuspended(clean, null);
         } catch (final RuntimeException e) {
             log.error("Failed to close suspended {} {} due to the following error:", taskTypeName, task.id(), e);
             return e;
@@ -264,7 +268,6 @@ RuntimeException closeNotAssignedSuspendedTasks(final Set revokedTasks)
         for (final TaskId revokedTask : revokedTasks) {
             final StreamTask suspendedTask = suspended.get(revokedTask);
 
-            // task may not be in the suspended tasks if it was closed due to some error
             if (suspendedTask != null) {
                 firstException.compareAndSet(null, closeSuspended(false, suspendedTask));
             } else {
@@ -335,7 +338,7 @@ boolean maybeResumeSuspendedTask(final TaskId taskId,
                 return true;
             } else {
                 log.warn("Couldn't resume task {} assigned partitions {}, task partitions {}", taskId, partitions, task.partitions());
-                task.closeSuspended(true, false, null);
+                task.closeSuspended(true, null);
             }
         }
         return false;
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
index f10c25bc5a737..fbc116af61f0f 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
@@ -120,18 +120,6 @@ public void commit() {
         commitNeeded = false;
     }
 
-    /**
-     * 
                -     * - flush store
-     * - checkpoint store
-     * 
                
-     */
-    @Override
-    public void suspend() {
-        log.debug("Suspending");
-        flushAndCheckpointState();
-    }
-
     private void flushAndCheckpointState() {
         stateMgr.flush();
         stateMgr.checkpoint(Collections.emptyMap());
@@ -163,13 +151,6 @@ public void close(final boolean clean,
         taskClosed = true;
     }
 
-    @Override
-    public void closeSuspended(final boolean clean,
-                               final boolean isZombie,
-                               final RuntimeException e) {
-        close(clean, isZombie);
-    }
-
     /**
      * Updates a state store using records from one change log partition
      *
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java
index 6c6a1c4d12e75..55a33c0a92e58 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java
@@ -79,8 +79,7 @@ public Collection restore(final RestoringTasks active) {
             initialize(active);
         }
 
-        if (needsRestoring.isEmpty() || restoreConsumer.assignment().isEmpty()) {
-            restoreConsumer.unsubscribe();
+        if (checkForCompletedRestoration()) {
             return completedRestorers;
         }
 
@@ -116,9 +115,7 @@ public Collection restore(final RestoringTasks active) {
 
         needsRestoring.removeAll(completedRestorers);
 
-        if (needsRestoring.isEmpty()) {
-            restoreConsumer.unsubscribe();
-        }
+        checkForCompletedRestoration();
 
         return completedRestorers;
     }
@@ -337,7 +334,14 @@ private long processNext(final List> records,
         return nextPosition;
     }
 
-
+    private boolean checkForCompletedRestoration() {
+        if (needsRestoring.isEmpty()) {
+            log.info("Finished restoring all active tasks");
+            restoreConsumer.unsubscribe();
+            return true;
+        }
+        return false;
+    }
 
     private boolean hasPartition(final TopicPartition topicPartition) {
         final List partitions = partitionInfo.get(topicPartition.topic());
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
index 40466b388706d..780fe1ff05c3f 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
@@ -573,7 +573,6 @@ private void initTopology() {
      * @throws TaskMigratedException if committing offsets failed (non-EOS)
      *                               or if the task producer got fenced (EOS)
      */
-    @Override
     public void suspend() {
         log.debug("Suspending");
         suspend(true, false);
@@ -687,10 +686,7 @@ private void closeTopology() {
     }
 
     // helper to avoid calling suspend() twice if a suspended task is not reassigned and closed
-    @Override
-    public void closeSuspended(final boolean clean,
-                               final boolean isZombie,
-                               RuntimeException firstException) {
+    void closeSuspended(final boolean clean, RuntimeException firstException) {
         try {
             closeStateManager(clean);
         } catch (final RuntimeException e) {
@@ -742,7 +738,7 @@ public void close(boolean clean,
             log.error("Could not close task due to the following error:", e);
         }
 
-        closeSuspended(clean, isZombie, firstException);
+        closeSuspended(clean, firstException);
 
         taskClosed = true;
     }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index 29e1bc7050ee4..c71ff27b17db4 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -132,13 +132,13 @@ public class StreamThread extends Thread {
      */
     public enum State implements ThreadStateTransitionValidator {
 
-        CREATED(1, 5),                   // 0
-        STARTING(2, 3, 5),               // 1
-        PARTITIONS_REVOKED(3, 5),        // 2
-        PARTITIONS_ASSIGNED(2, 3, 4, 5), // 3
-        RUNNING(2, 3, 5),                // 4
-        PENDING_SHUTDOWN(6),             // 5
-        DEAD;                            // 6
+        CREATED(1, 5),                    // 0
+        STARTING(2, 3, 5),                // 1
+        PARTITIONS_REVOKED(2, 3, 5),      // 2
+        PARTITIONS_ASSIGNED(2, 3, 4, 5),  // 3
+        RUNNING(2, 3, 5),                 // 4
+        PENDING_SHUTDOWN(6),              // 5
+        DEAD;                             // 6
 
         private final Set validTransitions = new HashSet<>();
 
@@ -734,9 +734,9 @@ void runOnce() {
             // to unblock the restoration as soon as possible
             records = pollRequests(Duration.ZERO);
         } else if (state == State.PARTITIONS_REVOKED) {
-            // try to fetch some records with normal poll time
-            // in order to wait long enough to get the join response
-            records = pollRequests(pollTime);
+            // try to fetch som records with zero poll millis to unblock
+            // other useful work while waiting for the join response
+            records = pollRequests(Duration.ZERO);
         } else if (state == State.RUNNING || state == State.STARTING) {
             // try to fetch some records with normal poll time
             // in order to get long polling
@@ -970,7 +970,12 @@ boolean maybeCommit() {
                 }
             }
 
-            lastCommitMs = now;
+            if (committed == -1) {
+                log.trace("Unable to commit as we are in the middle of a rebalance, will try again when it completes.");
+            } else {
+                lastCommitMs = now;
+            }
+            
             processStandbyRecords = true;
         } else {
             committed = taskManager.maybeCommitActiveTasksPerUserRequested();
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
index 2e6c9c081d52f..8b2c95a8e989e 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
@@ -47,15 +47,18 @@
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.TreeMap;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
+import static java.util.UUID.randomUUID;
 import static org.apache.kafka.common.utils.Utils.getHost;
 import static org.apache.kafka.common.utils.Utils.getPort;
 import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.EARLIEST_PROBEABLE_VERSION;
@@ -63,20 +66,21 @@
 import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.UNKNOWN;
 import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.VERSION_FIVE;
 import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.VERSION_FOUR;
-import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.VERSION_ONE;
 import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.VERSION_THREE;
 import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.VERSION_TWO;
+import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.VERSION_ONE;
 
 public class StreamsPartitionAssignor implements ConsumerPartitionAssignor, Configurable {
+
     private Logger log;
     private String logPrefix;
 
     private static class AssignedPartition implements Comparable {
+
         private final TaskId taskId;
         private final TopicPartition partition;
 
-        AssignedPartition(final TaskId taskId,
-                          final TopicPartition partition) {
+        AssignedPartition(final TaskId taskId, final TopicPartition partition) {
             this.taskId = taskId;
             this.partition = partition;
         }
@@ -103,6 +107,7 @@ public int hashCode() {
     }
 
     private static class ClientMetadata {
+
         private final HostInfo hostInfo;
         private final Set consumers;
         private final ClientState state;
@@ -132,12 +137,15 @@ private static class ClientMetadata {
             state = new ClientState();
         }
 
-        void addConsumer(final String consumerMemberId,
-                         final SubscriptionInfo info) {
+        void addConsumer(final String consumerMemberId, final List ownedPartitions) {
             consumers.add(consumerMemberId);
-            state.addPreviousActiveTasks(consumerMemberId, info.prevTasks());
-            state.addPreviousStandbyTasks(consumerMemberId, info.standbyTasks());
             state.incrementCapacity();
+            state.addOwnedPartitions(ownedPartitions, consumerMemberId);
+        }
+
+        void addPreviousTasks(final SubscriptionInfo info) {
+            state.addPreviousActiveTasks(info.prevTasks());
+            state.addPreviousStandbyTasks(info.standbyTasks());
         }
 
         @Override
@@ -177,9 +185,9 @@ protected TaskManager taskManger() {
     }
 
     /**
-     * We need to have the PartitionAssignor and its StreamThread to be mutually accessible
-     * since the former needs later's cached metadata while sending subscriptions,
-     * and the latter needs former's returned assignment when adding tasks.
+     * We need to have the PartitionAssignor and its StreamThread to be mutually accessible since the former needs
+     * later's cached metadata while sending subscriptions, and the latter needs former's returned assignment when
+     * adding tasks.
      *
      * @throws KafkaException if the stream thread is not specified
      */
@@ -189,7 +197,8 @@ public void configure(final Map configs) {
 
         logPrefix = assignorConfiguration.logPrefix();
         log = new LogContext(logPrefix).logger(getClass());
-        usedSubscriptionMetadataVersion = assignorConfiguration.configuredMetadataVersion(usedSubscriptionMetadataVersion);
+        usedSubscriptionMetadataVersion = assignorConfiguration
+            .configuredMetadataVersion(usedSubscriptionMetadataVersion);
         taskManager = assignorConfiguration.getTaskManager();
         assignmentErrorCode = assignorConfiguration.getAssignmentErrorCode(configs);
         numStandbyReplicas = assignorConfiguration.getNumStandbyReplicas();
@@ -221,37 +230,64 @@ public ByteBuffer subscriptionUserData(final Set topics) {
         // 1. Client UUID (a unique id assigned to an instance of KafkaStreams)
         // 2. Task ids of previously running tasks
         // 3. Task ids of valid local states on the client's state directory.
-
-        final Set previousActiveTasks = taskManager.previousRunningTaskIds();
         final Set standbyTasks = taskManager.cachedTasksIds();
-        standbyTasks.removeAll(previousActiveTasks);
-        final SubscriptionInfo data = new SubscriptionInfo(
+        final Set activeTasks = prepareForSubscription(taskManager,
+            topics,
+            standbyTasks,
+            rebalanceProtocol);
+        return new SubscriptionInfo(
             usedSubscriptionMetadataVersion,
             taskManager.processId(),
-            previousActiveTasks,
+            activeTasks,
             standbyTasks,
-            userEndPoint);
+            userEndPoint)
+            .encode();
+    }
+
+    protected static Set prepareForSubscription(final TaskManager taskManager,
+        final Set topics,
+        final Set standbyTasks,
+        final RebalanceProtocol rebalanceProtocol) {
+        // Any tasks that are not yet running are counted as standby tasks for assignment purposes,
+        // along with any old tasks for which we still found state on disk
+        final Set activeTasks;
+
+        switch (rebalanceProtocol) {
+            case EAGER:
+                // In eager, onPartitionsRevoked is called first and we must get the previously saved running task ids
+                activeTasks = taskManager.previousRunningTaskIds();
+                standbyTasks.removeAll(activeTasks);
+                break;
+            case COOPERATIVE:
+                // In cooperative, we will use the encoded ownedPartitions to determine the running tasks
+                activeTasks = Collections.emptySet();
+                standbyTasks.removeAll(taskManager.activeTaskIds());
+                break;
+            default:
+                throw new IllegalStateException("Streams partition assignor's rebalance protocol is unknown");
+        }
 
         taskManager.updateSubscriptionsFromMetadata(topics);
+        taskManager.setRebalanceInProgress(true);
 
-        return data.encode();
+        return activeTasks;
     }
 
     private Map errorAssignment(final Map clientsMetadata,
                                                     final String topic,
                                                     final int errorCode) {
         log.error("{} is unknown yet during rebalance," +
-                      " please make sure they have been pre-created before starting the Streams application.", topic);
+            " please make sure they have been pre-created before starting the Streams application.", topic);
         final Map assignment = new HashMap<>();
         for (final ClientMetadata clientMetadata : clientsMetadata.values()) {
             for (final String consumerId : clientMetadata.consumers) {
                 assignment.put(consumerId, new Assignment(
                     Collections.emptyList(),
                     new AssignmentInfo(LATEST_SUPPORTED_VERSION,
-                                       Collections.emptyList(),
-                                       Collections.emptyMap(),
-                                       Collections.emptyMap(),
-                                       errorCode).encode()
+                        Collections.emptyList(),
+                        Collections.emptyMap(),
+                        Collections.emptyMap(),
+                        errorCode).encode()
                 ));
             }
         }
@@ -283,7 +319,12 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
         final Map subscriptions = groupSubscription.groupSubscription();
         // construct the client metadata from the decoded subscription info
         final Map clientMetadataMap = new HashMap<>();
-        final Set futureConsumers = new HashSet<>();
+        final Set allOwnedPartitions = new HashSet<>();
+
+        // keep track of any future consumers in a "dummy" Client since we can't decipher their subscription
+        final UUID futureId = randomUUID();
+        final ClientMetadata futureClient = new ClientMetadata(null);
+        clientMetadataMap.put(futureId, futureClient);
 
         int minReceivedMetadataVersion = LATEST_SUPPORTED_VERSION;
         int minSupportedMetadataVersion = LATEST_SUPPORTED_VERSION;
@@ -292,58 +333,59 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
         for (final Map.Entry entry : subscriptions.entrySet()) {
             final String consumerId = entry.getKey();
             final Subscription subscription = entry.getValue();
-
             final SubscriptionInfo info = SubscriptionInfo.decode(subscription.userData());
             final int usedVersion = info.version();
+
+            minReceivedMetadataVersion = updateMinReceivedVersion(usedVersion, minReceivedMetadataVersion);
+            minSupportedMetadataVersion = updateMinSupportedVersion(info.latestSupportedVersion(), minSupportedMetadataVersion);
+
+            final UUID processId;
             if (usedVersion > LATEST_SUPPORTED_VERSION) {
                 futureMetadataVersion = usedVersion;
-                futureConsumers.add(consumerId);
-                continue;
-            }
-            if (usedVersion < minReceivedMetadataVersion) {
-                minReceivedMetadataVersion = usedVersion;
+                processId = futureId;
+            } else {
+                processId = info.processId();
             }
 
-            final int latestSupportedVersion = info.latestSupportedVersion();
-            if (latestSupportedVersion < minSupportedMetadataVersion) {
-                minSupportedMetadataVersion = latestSupportedVersion;
-            }
+            ClientMetadata clientMetadata = clientMetadataMap.get(processId);
 
             // create the new client metadata if necessary
-            ClientMetadata clientMetadata = clientMetadataMap.get(info.processId());
-
             if (clientMetadata == null) {
                 clientMetadata = new ClientMetadata(info.userEndPoint());
                 clientMetadataMap.put(info.processId(), clientMetadata);
             }
 
-            // add the consumer to the client
-            clientMetadata.addConsumer(consumerId, info);
+            // add the consumer and any info its its subscription to the client
+            clientMetadata.addConsumer(consumerId, subscription.ownedPartitions());
+            allOwnedPartitions.addAll(subscription.ownedPartitions());
+            if (info.prevTasks() != null && info.standbyTasks() != null) {
+                clientMetadata.addPreviousTasks(info);
+            }
         }
 
         final boolean versionProbing;
         if (futureMetadataVersion == UNKNOWN) {
             versionProbing = false;
+            clientMetadataMap.remove(futureId);
+        } else if (minReceivedMetadataVersion >= EARLIEST_PROBEABLE_VERSION) {
+            versionProbing = true;
+            log.info("Received a future (version probing) subscription (version: {})."
+                    + " Sending assignment back (with supported version {}).",
+                futureMetadataVersion,
+                minSupportedMetadataVersion);
+
         } else {
-            if (minReceivedMetadataVersion >= EARLIEST_PROBEABLE_VERSION) {
-                log.info("Received a future (version probing) subscription (version: {})."
-                             + " Sending empty assignment back (with supported version {}).",
-                         futureMetadataVersion,
-                         LATEST_SUPPORTED_VERSION);
-                versionProbing = true;
-            } else {
-                throw new IllegalStateException(
-                    "Received a future (version probing) subscription (version: " + futureMetadataVersion
-                        + ") and an incompatible pre Kafka 2.0 subscription (version: " + minReceivedMetadataVersion
-                        + ") at the same time."
-                );
-            }
+            throw new IllegalStateException(
+                "Received a future (version probing) subscription (version: " + futureMetadataVersion
+                    + ") and an incompatible pre Kafka 2.0 subscription (version: " + minReceivedMetadataVersion
+                    + ") at the same time."
+            );
         }
 
         if (minReceivedMetadataVersion < LATEST_SUPPORTED_VERSION) {
             log.info("Downgrading metadata to version {}. Latest supported version is {}.",
-                     minReceivedMetadataVersion,
-                     LATEST_SUPPORTED_VERSION);
+                minReceivedMetadataVersion,
+                LATEST_SUPPORTED_VERSION);
         }
 
         log.debug("Constructed client metadata {} from the member subscriptions.", clientMetadataMap);
@@ -361,9 +403,10 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
                 if (!topicsInfo.repartitionSourceTopics.keySet().contains(topic) &&
                     !metadata.topics().contains(topic)) {
                     log.error("Missing source topic {} during assignment. Returning error {}.",
-                              topic, AssignorError.INCOMPLETE_SOURCE_TOPIC_METADATA.name());
+                        topic, AssignorError.INCOMPLETE_SOURCE_TOPIC_METADATA.name());
                     return new GroupAssignment(
-                        errorAssignment(clientMetadataMap, topic, AssignorError.INCOMPLETE_SOURCE_TOPIC_METADATA.code())
+                        errorAssignment(clientMetadataMap, topic,
+                            AssignorError.INCOMPLETE_SOURCE_TOPIC_METADATA.code())
                     );
                 }
             }
@@ -378,7 +421,8 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
 
             for (final InternalTopologyBuilder.TopicsInfo topicsInfo : topicGroups.values()) {
                 for (final String topicName : topicsInfo.repartitionSourceTopics.keySet()) {
-                    final Optional maybeNumPartitions = repartitionTopicMetadata.get(topicName).numberOfPartitions();
+                    final Optional maybeNumPartitions = repartitionTopicMetadata.get(topicName)
+                        .numberOfPartitions();
                     Integer numPartitions = null;
 
                     if (!maybeNumPartitions.isPresent()) {
@@ -395,7 +439,8 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
                                     // map().join().join(map())
                                     if (repartitionTopicMetadata.containsKey(sourceTopicName)) {
                                         if (repartitionTopicMetadata.get(sourceTopicName).numberOfPartitions().isPresent()) {
-                                            numPartitionsCandidate = repartitionTopicMetadata.get(sourceTopicName).numberOfPartitions().get();
+                                            numPartitionsCandidate =
+                                                repartitionTopicMetadata.get(sourceTopicName).numberOfPartitions().get();
                                         }
                                     } else {
                                         final Integer count = metadata.partitionCountForTopic(sourceTopicName);
@@ -427,7 +472,6 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
             }
         } while (numPartitionsNeeded);
 
-
         // ensure the co-partitioning topics within the group have the same number of partitions,
         // and enforce the number of partitions for those repartition topics to be the same if they
         // are co-partitioned as well.
@@ -470,19 +514,23 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
         final Map> partitionsForTask =
             partitionGrouper.partitionGroups(sourceTopicsByGroup, fullMetadata);
 
+        final Map taskForPartition = new HashMap<>();
+
         // check if all partitions are assigned, and there are no duplicates of partitions in multiple tasks
         final Set allAssignedPartitions = new HashSet<>();
         final Map> tasksByTopicGroup = new HashMap<>();
         for (final Map.Entry> entry : partitionsForTask.entrySet()) {
+            final TaskId id = entry.getKey();
             final Set partitions = entry.getValue();
+
             for (final TopicPartition partition : partitions) {
+                taskForPartition.put(partition, id);
                 if (allAssignedPartitions.contains(partition)) {
                     log.warn("Partition {} is assigned to more than one tasks: {}", partition, partitionsForTask);
                 }
             }
             allAssignedPartitions.addAll(partitions);
 
-            final TaskId id = entry.getKey();
             tasksByTopicGroup.computeIfAbsent(id.topicGroupId, k -> new HashSet<>()).add(id);
         }
         for (final String topic : allSourceTopics) {
@@ -491,13 +539,15 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
                 log.warn("No partitions found for topic {}", topic);
             } else {
                 for (final PartitionInfo partitionInfo : partitionInfoList) {
-                    final TopicPartition partition = new TopicPartition(partitionInfo.topic(), partitionInfo.partition());
+                    final TopicPartition partition = new TopicPartition(partitionInfo.topic(),
+                        partitionInfo.partition());
                     if (!allAssignedPartitions.contains(partition)) {
                         log.warn("Partition {} is not assigned to any tasks: {}"
-                                     + " Possible causes of a partition not getting assigned"
-                                     + " is that another topic defined in the topology has not been"
-                                     + " created when starting your streams application,"
-                                     + " resulting in no tasks created for this topology at all.", partition, partitionsForTask);
+                                + " Possible causes of a partition not getting assigned"
+                                + " is that another topic defined in the topology has not been"
+                                + " created when starting your streams application,"
+                                + " resulting in no tasks created for this topology at all.", partition,
+                            partitionsForTask);
                     }
                 }
             }
@@ -533,15 +583,32 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
 
         // ---------------- Step Two ---------------- //
 
-        // assign tasks to clients
         final Map states = new HashMap<>();
         for (final Map.Entry entry : clientMetadataMap.entrySet()) {
-            states.put(entry.getKey(), entry.getValue().state);
+            final ClientState state = entry.getValue().state;
+            states.put(entry.getKey(), state);
+
+            // Either the active tasks (eager) OR the owned partitions (cooperative) were encoded in the subscription
+            // according to the rebalancing protocol, so convert any partitions in a client to tasks where necessary
+            if (!state.ownedPartitions().isEmpty()) {
+                final Set previousActiveTasks = new HashSet<>();
+                for (final Map.Entry partitionEntry : state.ownedPartitions().entrySet()) {
+                    final TopicPartition tp = partitionEntry.getKey();
+                    final TaskId task = taskForPartition.get(tp);
+                    if (task != null) {
+                        previousActiveTasks.add(task);
+                    } else {
+                        log.error("No task found for topic partition {}", tp);
+                    }
+                }
+                state.addPreviousActiveTasks(previousActiveTasks);
+            }
         }
 
         log.debug("Assigning tasks {} to clients {} with number of replicas {}",
-                  partitionsForTask.keySet(), states, numStandbyReplicas);
+            partitionsForTask.keySet(), states, numStandbyReplicas);
 
+        // assign tasks to clients
         final StickyTaskAssignor taskAssignor = new StickyTaskAssignor<>(states, partitionsForTask.keySet());
         taskAssignor.assign(numStandbyReplicas);
 
@@ -577,7 +644,7 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
                 clientMetadataMap,
                 partitionsForTask,
                 partitionsByHostState,
-                futureConsumers,
+                allOwnedPartitions,
                 minReceivedMetadataVersion,
                 minSupportedMetadataVersion
             );
@@ -586,6 +653,7 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
                 clientMetadataMap,
                 partitionsForTask,
                 partitionsByHostState,
+                allOwnedPartitions,
                 minReceivedMetadataVersion,
                 minSupportedMetadataVersion
             );
@@ -594,132 +662,165 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
         return new GroupAssignment(assignment);
     }
 
-    private static Map computeNewAssignment(final Map clientsMetadata,
-                                                                final Map> partitionsForTask,
-                                                                final Map> partitionsByHostState,
-                                                                final int minUserMetadataVersion,
-                                                                final int minSupportedMetadataVersion) {
+    private Map computeNewAssignment(final Map clientsMetadata,
+                                                         final Map> partitionsForTask,
+                                                         final Map> partitionsByHostState,
+                                                         final Set allOwnedPartitions,
+                                                         final int minUserMetadataVersion,
+                                                         final int minSupportedMetadataVersion) {
+        // keep track of whether a 2nd rebalance is unavoidable so we can skip trying to get a completely sticky assignment
+        boolean rebalanceRequired = false;
         final Map assignment = new HashMap<>();
 
         // within the client, distribute tasks to its owned consumers
-        for (final Map.Entry entry : clientsMetadata.entrySet()) {
-            final Set consumers = entry.getValue().consumers;
-            final ClientState state = entry.getValue().state;
-
-            final List> interleavedActive =
-                interleaveTasksByGroupId(state.activeTasks(), consumers.size());
-            final List> interleavedStandby =
-                interleaveTasksByGroupId(state.standbyTasks(), consumers.size());
-
-            int consumerTaskIndex = 0;
-
-            for (final String consumer : consumers) {
-                final List activeTasks = interleavedActive.get(consumerTaskIndex);
-
-                // These will be filled in by buildAssignedActiveTaskAndPartitionsList below
-                final List activePartitionsList = new ArrayList<>();
-                final List assignedActiveList = new ArrayList<>();
-
-                buildAssignedActiveTaskAndPartitionsList(activeTasks, activePartitionsList, assignedActiveList, partitionsForTask);
+        for (final ClientMetadata clientMetadata : clientsMetadata.values()) {
+            final ClientState state = clientMetadata.state;
+            final Set consumers = clientMetadata.consumers;
+            Map> activeTaskAssignments;
+
+            // Try to avoid triggering another rebalance by giving active tasks back to their previous owners within a
+            // client, without violating load balance. If we already know another rebalance will be required, or the
+            // client had no owned partitions, try to balance the workload as evenly as possible by interleaving the
+            // tasks among consumers and hopefully spreading the heavier subtopologies evenly across threads.
+            if (rebalanceRequired || state.ownedPartitions().isEmpty()) {
+                activeTaskAssignments = interleaveConsumerTasksByGroupId(state.activeTasks(), consumers);
+            } else if ((activeTaskAssignments = tryStickyAndBalancedTaskAssignmentWithinClient(state, consumers, partitionsForTask, allOwnedPartitions))
+                        .equals(Collections.emptyMap())) {
+                rebalanceRequired = true;
+                activeTaskAssignments = interleaveConsumerTasksByGroupId(state.activeTasks(), consumers);
+            }
 
-                final Map> standby = new HashMap<>();
-                if (!state.standbyTasks().isEmpty()) {
-                    final List assignedStandbyList = interleavedStandby.get(consumerTaskIndex);
-                    for (final TaskId taskId : assignedStandbyList) {
-                        standby.computeIfAbsent(taskId, k -> new HashSet<>()).addAll(partitionsForTask.get(taskId));
-                    }
-                }
+            final Map> interleavedStandby =
+                interleaveConsumerTasksByGroupId(state.standbyTasks(), consumers);
 
-                consumerTaskIndex++;
-
-                // finally, encode the assignment before sending back to coordinator
-                assignment.put(
-                    consumer,
-                    new Assignment(
-                        activePartitionsList,
-                        new AssignmentInfo(
-                            minUserMetadataVersion,
-                            minSupportedMetadataVersion,
-                            assignedActiveList,
-                            standby,
-                            partitionsByHostState,
-                            0
-                        ).encode()
-                    )
-                );
-            }
+            addClientAssignments(
+                assignment,
+                clientMetadata,
+                partitionsForTask,
+                partitionsByHostState,
+                allOwnedPartitions,
+                activeTaskAssignments,
+                interleavedStandby,
+                minUserMetadataVersion,
+                minSupportedMetadataVersion);
         }
 
         return assignment;
     }
 
-    private static Map versionProbingAssignment(final Map clientsMetadata,
-                                                                    final Map> partitionsForTask,
-                                                                    final Map> partitionsByHostState,
-                                                                    final Set futureConsumers,
-                                                                    final int minUserMetadataVersion,
-                                                                    final int minSupportedMetadataVersion) {
+    private Map versionProbingAssignment(final Map clientsMetadata,
+                                                             final Map> partitionsForTask,
+                                                             final Map> partitionsByHostState,
+                                                             final Set allOwnedPartitions,
+                                                             final int minUserMetadataVersion,
+                                                             final int minSupportedMetadataVersion) {
         final Map assignment = new HashMap<>();
 
-        // assign previously assigned tasks to "old consumers"
+        // Since we know another rebalance will be triggered anyway, just try and generate a balanced assignment
+        // (without violating cooperative protocol) now so that on the second rebalance we can just give tasks
+        // back to their previous owners
+        // within the client, distribute tasks to its owned consumers
         for (final ClientMetadata clientMetadata : clientsMetadata.values()) {
-            for (final String consumerId : clientMetadata.consumers) {
+            final ClientState state = clientMetadata.state;
 
-                if (futureConsumers.contains(consumerId)) {
-                    continue;
-                }
+            final Map> interleavedActive =
+                interleaveConsumerTasksByGroupId(state.activeTasks(), clientMetadata.consumers);
+            final Map> interleavedStandby =
+                interleaveConsumerTasksByGroupId(state.standbyTasks(), clientMetadata.consumers);
 
-                // Return the same active tasks that were claimed in the subscription
-                final List activeTasks = new ArrayList<>(clientMetadata.state.prevActiveTasksForConsumer(consumerId));
-
-                // These will be filled in by buildAssignedActiveTaskAndPartitionsList below
-                final List activePartitionsList = new ArrayList<>();
-                final List assignedActiveList = new ArrayList<>();
-
-                buildAssignedActiveTaskAndPartitionsList(activeTasks, activePartitionsList, assignedActiveList, partitionsForTask);
+            addClientAssignments(
+                assignment,
+                clientMetadata,
+                partitionsForTask,
+                partitionsByHostState,
+                allOwnedPartitions,
+                interleavedActive,
+                interleavedStandby,
+                minUserMetadataVersion,
+                minSupportedMetadataVersion);
+        }
 
-                // Return the same standby tasks that were claimed in the subscription
-                final Map> standbyTasks = new HashMap<>();
-                for (final TaskId taskId : clientMetadata.state.prevStandbyTasksForConsumer(consumerId)) {
-                    standbyTasks.put(taskId, partitionsForTask.get(taskId));
-                }
+        return assignment;
+    }
 
-                assignment.put(consumerId, new Assignment(
+    private void addClientAssignments(final Map assignment,
+                                      final ClientMetadata clientMetadata,
+                                      final Map> partitionsForTask,
+                                      final Map> partitionsByHostState,
+                                      final Set allOwnedPartitions,
+                                      final Map> activeTaskAssignments,
+                                      final Map> standbyTaskAssignments,
+                                      final int minUserMetadataVersion,
+                                      final int minSupportedMetadataVersion) {
+
+        // Loop through the consumers and build their assignment
+        for (final String consumer : clientMetadata.consumers) {
+            final List activeTasksForConsumer = activeTaskAssignments.get(consumer);
+
+            // These will be filled in by buildAssignedActiveTaskAndPartitionsList below
+            final List activePartitionsList = new ArrayList<>();
+            final List assignedActiveList = new ArrayList<>();
+
+            buildAssignedActiveTaskAndPartitionsList(consumer,
+                                                     clientMetadata.state,
+                                                     activeTasksForConsumer,
+                                                     partitionsForTask,
+                                                     allOwnedPartitions,
+                                                     activePartitionsList,
+                                                     assignedActiveList);
+
+            final Map> standbyTaskMap =
+                buildStandbyTaskMap(standbyTaskAssignments.get(consumer), partitionsForTask);
+
+            // finally, encode the assignment and insert into map with all assignments
+            assignment.put(
+                consumer,
+                new Assignment(
                     activePartitionsList,
                     new AssignmentInfo(
                         minUserMetadataVersion,
                         minSupportedMetadataVersion,
                         assignedActiveList,
-                        standbyTasks,
+                        standbyTaskMap,
                         partitionsByHostState,
-                        0)
-                        .encode()
-                ));
-            }
-        }
-
-        // add empty assignment for "future version" clients (ie, empty version probing response)
-        for (final String consumerId : futureConsumers) {
-            assignment.put(consumerId, new Assignment(
-                Collections.emptyList(),
-                new AssignmentInfo(minUserMetadataVersion, minSupportedMetadataVersion).encode()
-            ));
+                        AssignorError.NONE.code()
+                    ).encode()
+                )
+            );
         }
-
-        return assignment;
     }
 
-    private static void buildAssignedActiveTaskAndPartitionsList(final List activeTasks,
-                                                                 final List activePartitionsList,
-                                                                 final List assignedActiveList,
-                                                                 final Map> partitionsForTask) {
+    private void buildAssignedActiveTaskAndPartitionsList(final String consumer,
+                                                          final ClientState clientState,
+                                                          final List activeTasksForConsumer,
+                                                          final Map> partitionsForTask,
+                                                          final Set allOwnedPartitions,
+                                                          final List activePartitionsList,
+                                                          final List assignedActiveList) {
         final List assignedPartitions = new ArrayList<>();
 
         // Build up list of all assigned partition-task pairs
-        for (final TaskId taskId : activeTasks) {
+        for (final TaskId taskId : activeTasksForConsumer) {
+            final List assignedPartitionsForTask = new ArrayList<>();
             for (final TopicPartition partition : partitionsForTask.get(taskId)) {
-                assignedPartitions.add(new AssignedPartition(taskId, partition));
+                final String oldOwner = clientState.ownedPartitions().get(partition);
+                final boolean newPartitionForConsumer = oldOwner == null || !oldOwner.equals(consumer);
+
+                // If the partition is new to this consumer but is still owned by another, remove from the assignment
+                // until it has been revoked and can safely be reassigned according the COOPERATIVE protocol
+                if (newPartitionForConsumer && allOwnedPartitions.contains(partition)) {
+                    log.debug("Removing task {} from assignment until it is safely revoked", taskId);
+                    clientState.removeFromAssignment(taskId);
+                    // Clear the assigned partitions list for this task if any partition can not safely be assigned,
+                    // so as not to encode a partial task
+                    assignedPartitionsForTask.clear();
+                    break;
+                } else {
+                    assignedPartitionsForTask.add(new AssignedPartition(taskId, partition));
+                }
             }
+            // assignedPartitionsForTask will either contain all partitions for the task or be empty, so just add all
+            assignedPartitions.addAll(assignedPartitionsForTask);
         }
 
         // Add one copy of a task for each corresponding partition, so the receiver can determine the task <-> tp mapping
@@ -730,17 +831,175 @@ private static void buildAssignedActiveTaskAndPartitionsList(final List
         }
     }
 
-    // visible for testing
-    static List> interleaveTasksByGroupId(final Collection taskIds, final int numberThreads) {
+    private static Map> buildStandbyTaskMap(final Collection standbys,
+                                                                        final Map> partitionsForTask) {
+        final Map> standbyTaskMap = new HashMap<>();
+        for (final TaskId task : standbys) {
+            standbyTaskMap.put(task, partitionsForTask.get(task));
+        }
+        return standbyTaskMap;
+    }
+
+    /**
+     * Generates an assignment that tries to satisfy two conditions: no active task previously owned by a consumer
+     * be assigned to another (ie nothing gets revoked), and the number of tasks is evenly distributed throughout
+     * the client.
+     * 
                
+     * If it is impossible to satisfy both constraints we abort early and return an empty map so we can use a
+     * different assignment strategy that tries to distribute tasks of a single subtopology across different threads.
+     *
+     * @param state state for this client
+     * @param consumers the consumers in this client
+     * @param partitionsForTask mapping from task to its associated partitions
+     * @param allOwnedPartitions set of all partitions claimed as owned by the group
+     * @return task assignment for the consumers of this client
+     *         empty map if it is not possible to generate a balanced assignment without moving a task to a new consumer
+     */
+    Map> tryStickyAndBalancedTaskAssignmentWithinClient(final ClientState state,
+                                                                             final Set consumers,
+                                                                             final Map> partitionsForTask,
+                                                                             final Set allOwnedPartitions) {
+        final Map> assignments = new HashMap<>();
+        final LinkedList newTasks = new LinkedList<>();
+        final Set unfilledConsumers = new HashSet<>(consumers);
+
+        final int maxTasksPerClient = (int) Math.ceil(((double) state.activeTaskCount()) / consumers.size());
+
+        // initialize task list for consumers
+        for (final String consumer : consumers) {
+            assignments.put(consumer, new ArrayList<>());
+        }
+
+        for (final TaskId task : state.activeTasks()) {
+            final Set previousConsumers = previousConsumersOfTaskPartitions(partitionsForTask.get(task), state.ownedPartitions(), allOwnedPartitions);
+
+            // If this task's partitions were owned by different consumers, we can't avoid revoking partitions
+            if (previousConsumers.size() > 1) {
+                log.warn("The partitions of task {} were claimed as owned by different StreamThreads. " +
+                    "This indicates the mapping from partitions to tasks has changed!", task);
+                return Collections.emptyMap();
+            }
+
+            // If this is a new task, or its old consumer no longer exists, it can be freely (re)assigned
+            if (previousConsumers.isEmpty()) {
+                log.debug("Task {} was not previously owned by any consumers still in the group. It's owner may " +
+                    "have died or it may be a new task", task);
+                newTasks.add(task);
+            } else {
+                final String consumer = previousConsumers.iterator().next();
+
+                // If the previous consumer was from another client, these partitions will have to be revoked
+                if (!consumers.contains(consumer)) {
+                    log.debug("This client was assigned a task {} whose partition(s) were previously owned by another " +
+                        "client, falling back to an interleaved assignment since a rebalance is inevitable.", task);
+                    return Collections.emptyMap();
+                }
+
+                // If this consumer previously owned more tasks than it has capacity for, some must be revoked
+                if (assignments.get(consumer).size() >= maxTasksPerClient) {
+                    log.debug("Cannot create a sticky and balanced assignment as this client's consumers owned more " +
+                        "previous tasks than it has capacity for during this assignment, falling back to interleaved " +
+                        "assignment since a realance is inevitable.");
+                    return Collections.emptyMap();
+                }
+
+                assignments.get(consumer).add(task);
+
+                // If we have now reached capacity, remove it from set of consumers who still need more tasks
+                if (assignments.get(consumer).size() == maxTasksPerClient) {
+                    unfilledConsumers.remove(consumer);
+                }
+            }
+        }
+
+        // Interleave any remaining tasks by groupId among the consumers with remaining capacity. For further
+        // explanation, see the javadocs for #interleaveConsumerTasksByGroupId
+        Collections.sort(newTasks);
+        while (!newTasks.isEmpty()) {
+            if (unfilledConsumers.isEmpty()) {
+                throw new IllegalStateException("Some tasks could not be distributed");
+            }
+
+            final Iterator consumerIt = unfilledConsumers.iterator();
+
+            // Loop through the unfilled consumers and distribute tasks until newTasks is empty
+            while (consumerIt.hasNext()) {
+                final String consumer = consumerIt.next();
+                final List consumerAssignment = assignments.get(consumer);
+                final TaskId task = newTasks.poll();
+                if (task == null) {
+                    break;
+                }
+
+                consumerAssignment.add(task);
+                if (consumerAssignment.size() == maxTasksPerClient) {
+                    consumerIt.remove();
+                }
+            }
+        }
+
+        return assignments;
+    }
+
+    /**
+     * Get the previous consumer for the partitions of a task
+     *
+     * @param taskPartitions the TopicPartitions for a single given task
+     * @param clientOwnedPartitions the partitions owned by all consumers in a client
+     * @param allOwnedPartitions all partitions claimed as owned by any consumer in any client
+     * @return set of consumer(s) that previously owned the partitions in this task
+     *         empty set signals that it is a new task, or its previous owner is no longer in the group
+     */
+    Set previousConsumersOfTaskPartitions(final Set taskPartitions,
+                                                  final Map clientOwnedPartitions,
+                                                  final Set allOwnedPartitions) {
+        // this "foreignConsumer" indicates a partition was owned by someone from another client -- we don't really care who
+        final String foreignConsumer = "";
+        final Set previousConsumers = new HashSet<>();
+
+        for (final TopicPartition tp : taskPartitions) {
+            final String currentPartitionConsumer = clientOwnedPartitions.get(tp);
+            if (currentPartitionConsumer != null) {
+                previousConsumers.add(currentPartitionConsumer);
+            } else if (allOwnedPartitions.contains(tp)) {
+                previousConsumers.add(foreignConsumer);
+            }
+        }
+
+        return previousConsumers;
+    }
+
+    /**
+     * Generate an assignment that attempts to maximize load balance without regard for stickiness, by spreading
+     * tasks of the same groupId (subtopology) over different consumers.
+     *
+     * @param taskIds the set of tasks to be distributed
+     * @param consumers the set of consumers to receive tasks
+     * @return a map of task assignments keyed by the consumer id
+     */
+    static Map> interleaveConsumerTasksByGroupId(final Collection taskIds,
+                                                                      final Set consumers) {
+        // First we make a sorted list of the tasks, grouping them by groupId
         final LinkedList sortedTasks = new LinkedList<>(taskIds);
         Collections.sort(sortedTasks);
-        final List> taskIdsForConsumerAssignment = new ArrayList<>(numberThreads);
-        for (int i = 0; i < numberThreads; i++) {
-            taskIdsForConsumerAssignment.add(new ArrayList<>());
+
+        // Initialize the assignment map and task list for each consumer. We use a TreeMap here for a consistent
+        // ordering of the consumers in the hope they will end up with the same set of tasks in subsequent assignments
+        final Map> taskIdsForConsumerAssignment = new TreeMap<>();
+        for (final String consumer : consumers) {
+            taskIdsForConsumerAssignment.put(consumer, new ArrayList<>());
         }
+
+        // We loop until the tasks have all been assigned, removing them from the list when they are given to a
+        // consumer. To interleave the tasks, we loop through the consumers and give each one task from the head
+        // of the list. When we finish going through the list of consumers we start over at the beginning of the
+        // consumers list, continuing until we run out of tasks.
         while (!sortedTasks.isEmpty()) {
-            for (final List taskIdList : taskIdsForConsumerAssignment) {
+            for (final Map.Entry> consumerTaskIds : taskIdsForConsumerAssignment.entrySet()) {
+                final List taskIdList = consumerTaskIds.getValue();
                 final TaskId taskId = sortedTasks.poll();
+
+                // Check for null here as we may run out of tasks before giving every consumer exactly the same number
                 if (taskId == null) {
                     break;
                 }
@@ -774,7 +1033,6 @@ private void validateMetadataVersions(final int receivedAssignmentMetadataVersio
     protected boolean maybeUpdateSubscriptionVersion(final int receivedAssignmentMetadataVersion,
                                                      final int latestCommonlySupportedVersion) {
         if (receivedAssignmentMetadataVersion >= EARLIEST_PROBEABLE_VERSION) {
-
             // If the latest commonly supported version is now greater than our used version, this indicates we have just
             // completed the rolling upgrade and can now update our subscription version for the final rebalance
             if (latestCommonlySupportedVersion > usedSubscriptionMetadataVersion) {
@@ -880,6 +1138,7 @@ public void onAssignment(final Assignment assignment, final ConsumerGroupMetadat
         taskManager.setPartitionsToTaskId(partitionsToTaskId);
         taskManager.setAssignmentMetadata(activeTasks, info.standbyTasks());
         taskManager.updateSubscriptionsFromAssignment(partitions);
+        taskManager.setRebalanceInProgress(false);
     }
 
     private static void processVersionOneAssignment(final String logPrefix,
@@ -973,12 +1232,23 @@ private void ensureCopartitioning(final Collection> copartitionGroup
         }
     }
 
+    private int updateMinReceivedVersion(final int usedVersion, final int minReceivedMetadataVersion) {
+        return usedVersion < minReceivedMetadataVersion ? usedVersion : minReceivedMetadataVersion;
+    }
+
+    private int updateMinSupportedVersion(final int supportedVersion, final int minSupportedMetadataVersion) {
+        return supportedVersion < minSupportedMetadataVersion ? supportedVersion : minSupportedMetadataVersion;
+    }
+
     protected void setAssignmentErrorCode(final Integer errorCode) {
         assignmentErrorCode.set(errorCode);
     }
 
-
     // following functions are for test only
+    void setRebalanceProtocol(final RebalanceProtocol rebalanceProtocol) {
+        this.rebalanceProtocol = rebalanceProtocol;
+    }
+
     void setInternalTopicManager(final InternalTopicManager internalTopicManager) {
         this.internalTopicManager = internalTopicManager;
     }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/Task.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/Task.java
index da9e656ad9e2b..af1b4bd6e8ea7 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/Task.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/Task.java
@@ -27,7 +27,7 @@
 
 public interface Task {
     /**
-     * Initialize the task and return {@code true} if the task is ready to run, i.e, it has not state stores
+     * Initialize the task and return {@code true} if the task is ready to run, i.e, it has no state stores
      * @return true if this task has no state stores that may need restoring.
      * @throws IllegalStateException If store gets registered after initialized is already finished
      * @throws StreamsException if the store's change log does not contain the partition
@@ -40,14 +40,8 @@ public interface Task {
 
     void commit();
 
-    void suspend();
-
     void resume();
 
-    void closeSuspended(final boolean clean,
-                        final boolean isZombie,
-                        final RuntimeException e);
-
     void close(final boolean clean,
                final boolean isZombie);
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
index cd90fad80b522..a2dac408b3618 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
@@ -60,6 +60,7 @@ public class TaskManager {
 
     private final Admin adminClient;
     private DeleteRecordsResult deleteRecordsResult;
+    private boolean rebalanceInProgress = false;  // if we are in the middle of a rebalance, it is not safe to commit
 
     // the restore consumer is only ever assigned changelogs from restoring tasks or standbys (but not both)
     private boolean restoreConsumerAssignedStandbys = false;
@@ -144,7 +145,7 @@ private void resumeSuspended(final Collection assignment) {
                     addedActiveTasks.put(taskId, partitions);
                 }
             } catch (final StreamsException e) {
-                log.error("Failed to resume an active task {} due to the following error:", taskId, e);
+                log.error("Failed to resume a suspended active task {} due to the following error:", taskId, e);
                 throw e;
             }
         }
@@ -303,7 +304,11 @@ void shutdown(final boolean clean) {
         }
     }
 
-    Set activeTaskIds() {
+    public Set previousRunningTaskIds() {
+        return active.previousRunningTaskIds();
+    }
+
+    public Set activeTaskIds() {
         return active.allAssignedTaskIds();
     }
 
@@ -319,10 +324,6 @@ Set revokedStandbyTaskIds() {
         return revokedStandbyTasks.keySet();
     }
 
-    public Set previousRunningTaskIds() {
-        return active.previousRunningTaskIds();
-    }
-
     Set previousActiveTaskIds() {
         final HashSet previousActiveTasks = new HashSet<>(assignedActiveTasks.keySet());
         previousActiveTasks.addAll(revokedActiveTasks.keySet());
@@ -378,9 +379,11 @@ boolean updateNewAndRestoringTasks() {
         active.initializeNewTasks();
         standby.initializeNewTasks();
 
-        final Collection restored = changelogReader.restore(active);
-        active.updateRestored(restored);
-        removeChangelogsFromRestoreConsumer(restored, false);
+        if (active.hasRestoringTasks()) {
+            final Collection restored = changelogReader.restore(active);
+            active.updateRestored(restored);
+            removeChangelogsFromRestoreConsumer(restored, false);
+        }
 
         if (active.allTasksRunning()) {
             final Set assignment = consumer.assignment();
@@ -420,6 +423,10 @@ private void assignStandbyPartitions() {
         }
     }
 
+    public void setRebalanceInProgress(final boolean rebalanceInProgress) {
+        this.rebalanceInProgress = rebalanceInProgress;
+    }
+
     public void setClusterMetadata(final Cluster cluster) {
         this.cluster = cluster;
     }
@@ -493,10 +500,10 @@ public void updateSubscriptionsFromMetadata(final Set topics) {
     /**
      * @throws TaskMigratedException if committing offsets failed (non-EOS)
      *                               or if the task producer got fenced (EOS)
+     * @return number of committed offsets, or -1 if we are in the middle of a rebalance and cannot commit
      */
     int commitAll() {
-        final int committed = active.commit();
-        return committed + standby.commit();
+        return rebalanceInProgress ? -1 : active.commit() + standby.commit();
     }
 
     /**
@@ -518,7 +525,7 @@ int punctuate() {
      *                               or if the task producer got fenced (EOS)
      */
     int maybeCommitActiveTasksPerUserRequested() {
-        return active.maybeCommitPerUserRequested();
+        return rebalanceInProgress ? -1 : active.maybeCommitPerUserRequested();
     }
 
     void maybePurgeCommitedRecords() {
@@ -528,7 +535,8 @@ void maybePurgeCommitedRecords() {
         if (deleteRecordsResult == null || deleteRecordsResult.all().isDone()) {
 
             if (deleteRecordsResult != null && deleteRecordsResult.all().isCompletedExceptionally()) {
-                log.debug("Previous delete-records request has failed: {}. Try sending the new request now", deleteRecordsResult.lowWatermarks());
+                log.debug("Previous delete-records request has failed: {}. Try sending the new request now",
+                    deleteRecordsResult.lowWatermarks());
             }
 
             final Map recordsToDelete = new HashMap<>();
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorConfiguration.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorConfiguration.java
index ac88f2f78e4ba..1e406e25b0ca5 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorConfiguration.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorConfiguration.java
@@ -153,7 +153,8 @@ public RebalanceProtocol rebalanceProtocol() {
                     throw new IllegalArgumentException("Unknown configuration value for parameter 'upgrade.from': " + upgradeFrom);
             }
         }
-        return RebalanceProtocol.EAGER;
+
+        return RebalanceProtocol.COOPERATIVE;
     }
 
     public String logPrefix() {
@@ -181,14 +182,19 @@ public int configuredMetadataVersion(final int priorVersion) {
                         upgradeFrom
                     );
                     return VERSION_TWO;
+                case StreamsConfig.UPGRADE_FROM_20:
+                case StreamsConfig.UPGRADE_FROM_21:
+                case StreamsConfig.UPGRADE_FROM_22:
+                case StreamsConfig.UPGRADE_FROM_23:
+                    // These configs are for cooperative rebalancing and should not affect the metadata version
+                    break;
                 default:
                     throw new IllegalArgumentException(
                         "Unknown configuration value for parameter 'upgrade.from': " + upgradeFrom
                     );
             }
-        } else {
-            return priorVersion;
         }
+        return priorVersion;
     }
 
     public int getNumStandbyReplicas() {
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/ClientState.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/ClientState.java
index ab213d520014a..df42b14e29e26 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/ClientState.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/ClientState.java
@@ -16,11 +16,13 @@
  */
 package org.apache.kafka.streams.processor.internals.assignment;
 
-import java.util.HashMap;
-import java.util.Map;
+import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.streams.processor.TaskId;
 
+import java.util.Collection;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 
 public class ClientState {
@@ -31,8 +33,7 @@ public class ClientState {
     private final Set prevStandbyTasks;
     private final Set prevAssignedTasks;
 
-    private final Map> prevActiveTasksByConsumer;
-    private final Map> prevStandbyTasksByConsumer;
+    private final Map ownedPartitions;
 
     private int capacity;
 
@@ -48,7 +49,6 @@ public ClientState() {
              new HashSet<>(),
              new HashSet<>(),
              new HashMap<>(),
-             new HashMap<>(),
              capacity);
     }
 
@@ -58,8 +58,7 @@ private ClientState(final Set activeTasks,
                         final Set prevActiveTasks,
                         final Set prevStandbyTasks,
                         final Set prevAssignedTasks,
-                        final Map> prevActiveTasksByConsumer,
-                        final Map> prevStandbyTasksByConsumer,
+                        final Map ownedPartitions,
                         final int capacity) {
         this.activeTasks = activeTasks;
         this.standbyTasks = standbyTasks;
@@ -67,8 +66,7 @@ private ClientState(final Set activeTasks,
         this.prevActiveTasks = prevActiveTasks;
         this.prevStandbyTasks = prevStandbyTasks;
         this.prevAssignedTasks = prevAssignedTasks;
-        this.prevActiveTasksByConsumer = prevActiveTasksByConsumer;
-        this.prevStandbyTasksByConsumer = prevStandbyTasksByConsumer;
+        this.ownedPartitions = ownedPartitions;
         this.capacity = capacity;
     }
 
@@ -80,8 +78,7 @@ public ClientState copy() {
             new HashSet<>(prevActiveTasks),
             new HashSet<>(prevStandbyTasks),
             new HashSet<>(prevAssignedTasks),
-            new HashMap<>(prevActiveTasksByConsumer),
-            new HashMap<>(prevStandbyTasksByConsumer),
+            new HashMap<>(ownedPartitions),
             capacity);
     }
 
@@ -111,6 +108,10 @@ public Set prevStandbyTasks() {
         return prevStandbyTasks;
     }
 
+    public Map ownedPartitions() {
+        return ownedPartitions;
+    }
+
     @SuppressWarnings("WeakerAccess")
     public int assignedTaskCount() {
         return assignedTasks.size();
@@ -125,24 +126,25 @@ public int activeTaskCount() {
         return activeTasks.size();
     }
 
-    public void addPreviousActiveTasks(final String consumer, final Set prevTasks) {
+    public void addPreviousActiveTasks(final Set prevTasks) {
         prevActiveTasks.addAll(prevTasks);
         prevAssignedTasks.addAll(prevTasks);
-        prevActiveTasksByConsumer.put(consumer, prevTasks);
     }
 
-    public void addPreviousStandbyTasks(final String consumer, final Set standbyTasks) {
+    public void addPreviousStandbyTasks(final Set standbyTasks) {
         prevStandbyTasks.addAll(standbyTasks);
         prevAssignedTasks.addAll(standbyTasks);
-        prevStandbyTasksByConsumer.put(consumer, standbyTasks);
     }
 
-    public Set prevActiveTasksForConsumer(final String consumer) {
-        return prevActiveTasksByConsumer.get(consumer);
+    public void addOwnedPartitions(final Collection ownedPartitions, final String consumer) {
+        for (final TopicPartition tp : ownedPartitions) {
+            this.ownedPartitions.put(tp, consumer);
+        }
     }
 
-    public Set prevStandbyTasksForConsumer(final String consumer) {
-        return prevStandbyTasksByConsumer.get(consumer);
+    public void removeFromAssignment(final TaskId task) {
+        activeTasks.remove(task);
+        assignedTasks.remove(task);
     }
 
     @Override
@@ -153,6 +155,7 @@ public String toString() {
                 ") prevActiveTasks: (" + prevActiveTasks +
                 ") prevStandbyTasks: (" + prevStandbyTasks +
                 ") prevAssignedTasks: (" + prevAssignedTasks +
+                ") prevOwnedPartitionsByConsumerId: (" + ownedPartitions.keySet() +
                 ") capacity: " + capacity +
                 "]";
     }
@@ -182,16 +185,6 @@ boolean hasMoreAvailableCapacityThan(final ClientState other) {
         }
     }
 
-    Set previousStandbyTasks() {
-        final Set standby = new HashSet<>(prevAssignedTasks);
-        standby.removeAll(prevActiveTasks);
-        return standby;
-    }
-
-    Set previousActiveTasks() {
-        return prevActiveTasks;
-    }
-
     boolean hasAssignedTask(final TaskId taskId) {
         return assignedTasks.contains(taskId);
     }
@@ -212,4 +205,9 @@ int capacity() {
     boolean hasUnfulfilledQuota(final int tasksPerThread) {
         return activeTasks.size() < capacity * tasksPerThread;
     }
+
+    // the following methods are used for testing only
+    public void assignActiveTasks(final Collection tasks) {
+        activeTasks.addAll(tasks);
+    }
 }
\ No newline at end of file
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/StickyTaskAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/StickyTaskAssignor.java
index 157497dc87efd..d1da8b8c732c5 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/StickyTaskAssignor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/StickyTaskAssignor.java
@@ -228,14 +228,12 @@ private ClientState findLeastLoaded(final TaskId taskId,
 
     private void mapPreviousTaskAssignment(final Map clients) {
         for (final Map.Entry clientState : clients.entrySet()) {
-            for (final TaskId activeTask : clientState.getValue().previousActiveTasks()) {
+            for (final TaskId activeTask : clientState.getValue().prevActiveTasks()) {
                 previousActiveTaskAssignment.put(activeTask, clientState.getKey());
             }
 
-            for (final TaskId prevAssignedTask : clientState.getValue().previousStandbyTasks()) {
-                if (!previousStandbyTaskAssignment.containsKey(prevAssignedTask)) {
-                    previousStandbyTaskAssignment.put(prevAssignedTask, new HashSet<>());
-                }
+            for (final TaskId prevAssignedTask : clientState.getValue().prevStandbyTasks()) {
+                previousStandbyTaskAssignment.computeIfAbsent(prevAssignedTask, t -> new HashSet<>());
                 previousStandbyTaskAssignment.get(prevAssignedTask).add(clientState.getKey());
             }
         }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractTaskTest.java
index a8526bf82067e..7ce97128b13bd 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractTaskTest.java
@@ -226,15 +226,9 @@ public void resume() {}
             @Override
             public void commit() {}
 
-            @Override
-            public void suspend() {}
-
             @Override
             public void close(final boolean clean, final boolean isZombie) {}
 
-            @Override
-            public void closeSuspended(final boolean clean, final boolean isZombie, final RuntimeException e) {}
-
             @Override
             public boolean initializeStateStores() {
                 return false;
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
index 88d4d6f46cf7c..c2b9cdb0b07b7 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
@@ -443,7 +443,6 @@ public void shouldWriteCheckpointFile() throws IOException {
             singletonList(makeWindowedConsumerRecord(changelogName, 10, 1, 0L, 60_000L))
         );
 
-        task.suspend();
         task.close(true, false);
 
         final File taskDir = stateDirectory.directoryForTask(taskId);
@@ -817,26 +816,4 @@ public void shouldRecordTaskClosedMetricOnClose() throws IOException {
         final double expectedCloseTaskMetric = 1.0;
         verifyCloseTaskMetric(expectedCloseTaskMetric, streamsMetrics, metricName);
     }
-
-    @Test
-    public void shouldRecordTaskClosedMetricOnCloseSuspended() throws IOException {
-        final MetricName metricName = setupCloseTaskMetric();
-        final StandbyTask task = new StandbyTask(
-            taskId,
-            ktablePartitions,
-            ktableTopology,
-            consumer,
-            changelogReader,
-            createConfig(baseDir),
-            streamsMetrics,
-            stateDirectory
-        );
-
-        final boolean clean = true;
-        final boolean isZombie = false;
-        task.closeSuspended(clean, isZombie, new RuntimeException());
-
-        final double expectedCloseTaskMetric = 1.0;
-        verifyCloseTaskMetric(expectedCloseTaskMetric, streamsMetrics, metricName);
-    }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
index e997f2e125c1d..9ff6e33b07d9e 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
@@ -16,8 +16,10 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
+import java.util.Arrays;
 import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor;
 import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.GroupSubscription;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.RebalanceProtocol;
 import org.apache.kafka.common.Cluster;
 import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.Node;
@@ -37,6 +39,7 @@
 import org.apache.kafka.streams.kstream.ValueJoiner;
 import org.apache.kafka.streams.processor.TaskId;
 import org.apache.kafka.streams.processor.internals.assignment.AssignmentInfo;
+import org.apache.kafka.streams.processor.internals.assignment.ClientState;
 import org.apache.kafka.streams.processor.internals.assignment.SubscriptionInfo;
 import org.apache.kafka.streams.state.HostInfo;
 import org.apache.kafka.test.MockClientSupplier;
@@ -66,11 +69,17 @@
 import static org.hamcrest.CoreMatchers.not;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 @SuppressWarnings("unchecked")
 public class StreamsPartitionAssignorTest {
+    private final String c1 = "consumer1";
+    private final String c2 = "consumer2";
+    private final String c3 = "consumer3";
+    private final String c4 = "consumer4";
 
     private final TopicPartition t1p0 = new TopicPartition("topic1", 0);
     private final TopicPartition t1p1 = new TopicPartition("topic1", 1);
@@ -84,6 +93,40 @@ public class StreamsPartitionAssignorTest {
     private final TopicPartition t3p1 = new TopicPartition("topic3", 1);
     private final TopicPartition t3p2 = new TopicPartition("topic3", 2);
     private final TopicPartition t3p3 = new TopicPartition("topic3", 3);
+    private final TopicPartition t4p0 = new TopicPartition("topic4", 0);
+    private final TopicPartition t4p1 = new TopicPartition("topic4", 1);
+    private final TopicPartition t4p2 = new TopicPartition("topic4", 2);
+    private final TopicPartition t4p3 = new TopicPartition("topic4", 3);
+
+    private final TaskId task0_0 = new TaskId(0, 0);
+    private final TaskId task0_1 = new TaskId(0, 1);
+    private final TaskId task0_2 = new TaskId(0, 2);
+    private final TaskId task0_3 = new TaskId(0, 3);
+    private final TaskId task1_0 = new TaskId(1, 0);
+    private final TaskId task1_1 = new TaskId(1, 1);
+    private final TaskId task1_2 = new TaskId(1, 2);
+    private final TaskId task1_3 = new TaskId(1, 3);
+    private final TaskId task2_0 = new TaskId(2, 0);
+    private final TaskId task2_1 = new TaskId(2, 1);
+    private final TaskId task2_2 = new TaskId(2, 2);
+    private final TaskId task2_3 = new TaskId(2, 3);
+
+    private final Map> partitionsForTask = new HashMap>() {{
+            put(task0_0, Utils.mkSet(t1p0, t2p0));
+            put(task0_1, Utils.mkSet(t1p1, t2p1));
+            put(task0_2, Utils.mkSet(t1p2, t2p2));
+            put(task0_3, Utils.mkSet(t1p3, t2p3));
+
+            put(task1_0, Utils.mkSet(t3p0));
+            put(task1_1, Utils.mkSet(t3p1));
+            put(task1_2, Utils.mkSet(t3p2));
+            put(task1_3, Utils.mkSet(t3p3));
+
+            put(task2_0, Utils.mkSet(t4p0));
+            put(task2_1, Utils.mkSet(t4p1));
+            put(task2_2, Utils.mkSet(t4p2));
+            put(task2_3, Utils.mkSet(t4p3));
+        }};
 
     private final Set allTopics = Utils.mkSet("topic1", "topic2");
 
@@ -109,10 +152,6 @@ public class StreamsPartitionAssignorTest {
         Collections.emptySet(),
         Collections.emptySet());
 
-    private final TaskId task0 = new TaskId(0, 0);
-    private final TaskId task1 = new TaskId(0, 1);
-    private final TaskId task2 = new TaskId(0, 2);
-    private final TaskId task3 = new TaskId(0, 3);
     private final StreamsPartitionAssignor partitionAssignor = new StreamsPartitionAssignor();
     private final MockClientSupplier mockClientSupplier = new MockClientSupplier();
     private final InternalTopologyBuilder builder = new InternalTopologyBuilder();
@@ -137,6 +176,11 @@ private void configurePartitionAssignor(final Map props) {
         partitionAssignor.configure(configurationMap);
     }
 
+    private void configureDefault() {
+        createMockTaskManager();
+        partitionAssignor.configure(configProps());
+    }
+
     private void createMockTaskManager() {
         final StreamsBuilder builder = new StreamsBuilder();
         final InternalTopologyBuilder internalTopologyBuilder = TopologyWrapper.getInternalTopologyBuilder(builder.build());
@@ -153,6 +197,7 @@ private void createMockTaskManager(final Set prevTasks,
         EasyMock.expect(taskManager.adminClient()).andReturn(null).anyTimes();
         EasyMock.expect(taskManager.builder()).andReturn(builder).anyTimes();
         EasyMock.expect(taskManager.previousRunningTaskIds()).andReturn(prevTasks).anyTimes();
+        EasyMock.expect(taskManager.activeTaskIds()).andReturn(prevTasks).anyTimes();
         EasyMock.expect(taskManager.cachedTasksIds()).andReturn(cachedTasks).anyTimes();
         EasyMock.expect(taskManager.processId()).andReturn(processId).anyTimes();
     }
@@ -168,6 +213,141 @@ public void setUp() {
         }
     }
 
+    @Test
+    public void shouldUseEagerRebalancingProtocol() {
+        createMockTaskManager();
+        final Map props = configProps();
+        props.put(StreamsConfig.UPGRADE_FROM_CONFIG, StreamsConfig.UPGRADE_FROM_23);
+        partitionAssignor.configure(props);
+
+        assertEquals(1, partitionAssignor.supportedProtocols().size());
+        assertTrue(partitionAssignor.supportedProtocols().contains(RebalanceProtocol.EAGER));
+        assertFalse(partitionAssignor.supportedProtocols().contains(RebalanceProtocol.COOPERATIVE));
+    }
+
+    @Test
+    public void shouldUseCooperativeRebalancingProtocol() {
+        createMockTaskManager();
+        final Map props = configProps();
+        partitionAssignor.configure(props);
+
+        assertEquals(2, partitionAssignor.supportedProtocols().size());
+        assertTrue(partitionAssignor.supportedProtocols().contains(RebalanceProtocol.COOPERATIVE));
+    }
+
+    @Test
+    public void shouldProduceStickyAndBalancedAssignmentWhenNothingChanges() {
+        configureDefault();
+        final ClientState state = new ClientState();
+        final List allTasks = Arrays.asList(task0_0, task0_1, task0_2, task0_3, task1_0, task1_1, task1_2, task1_3);
+
+        final Map> previousAssignment = new HashMap>() {{
+                put(c1, Arrays.asList(task0_0, task1_1, task1_3));
+                put(c2, Arrays.asList(task0_3, task1_0));
+                put(c3, Arrays.asList(task0_1, task0_2, task1_2));
+            }};
+
+        for (final Map.Entry> entry : previousAssignment.entrySet()) {
+            for (final TaskId task : entry.getValue()) {
+                state.addOwnedPartitions(partitionsForTask.get(task), entry.getKey());
+            }
+        }
+
+        final Set consumers = Utils.mkSet(c1, c2, c3);
+        state.assignActiveTasks(allTasks);
+
+        assertEquivalentAssignment(previousAssignment,
+            partitionAssignor.tryStickyAndBalancedTaskAssignmentWithinClient(state, consumers, partitionsForTask, Collections.emptySet()));
+    }
+
+    @Test
+    public void shouldProduceStickyAndBalancedAssignmentWhenNewTasksAreAdded() {
+        configureDefault();
+        final ClientState state = new ClientState();
+
+        final Set allTasks = Utils.mkSet(task0_0, task0_1, task0_2, task0_3, task1_0, task1_1, task1_2, task1_3);
+
+        final Map> previousAssignment = new HashMap>() {{
+                put(c1, new ArrayList<>(Arrays.asList(task0_0, task1_1, task1_3)));
+                put(c2, new ArrayList<>(Arrays.asList(task0_3, task1_0)));
+                put(c3, new ArrayList<>(Arrays.asList(task0_1, task0_2, task1_2)));
+            }};
+
+        for (final Map.Entry> entry : previousAssignment.entrySet()) {
+            for (final TaskId task : entry.getValue()) {
+                state.addOwnedPartitions(partitionsForTask.get(task), entry.getKey());
+            }
+        }
+
+        final Set consumers = Utils.mkSet(c1, c2, c3);
+
+        // We should be able to add a new task without sacrificing stickyness
+        final TaskId newTask = task2_0;
+        allTasks.add(newTask);
+        state.assignActiveTasks(allTasks);
+
+        final Map> newAssignment = partitionAssignor.tryStickyAndBalancedTaskAssignmentWithinClient(state, consumers, partitionsForTask, Collections.emptySet());
+
+        previousAssignment.get(c2).add(newTask);
+        assertEquivalentAssignment(previousAssignment, newAssignment);
+    }
+
+    @Test
+    public void shouldReturnEmptyMapWhenStickyAndBalancedAssignmentIsNotPossibleBecauseNewConsumerJoined() {
+        configureDefault();
+        final ClientState state = new ClientState();
+
+        final List allTasks = Arrays.asList(task0_0, task0_1, task0_2, task0_3, task1_0, task1_1, task1_2, task1_3);
+
+        final Map> previousAssignment = new HashMap>() {{
+                put(c1, Arrays.asList(task0_0, task1_1, task1_3));
+                put(c2, Arrays.asList(task0_3, task1_0));
+                put(c3, Arrays.asList(task0_1, task0_2, task1_2));
+            }};
+
+        for (final Map.Entry> entry : previousAssignment.entrySet()) {
+            for (final TaskId task : entry.getValue()) {
+                state.addOwnedPartitions(partitionsForTask.get(task), entry.getKey());
+            }
+        }
+
+        // If we add a new consumer here, we cannot produce an assignment that is both sticky and balanced
+        final Set consumers = Utils.mkSet(c1, c2, c3, c4);
+        state.assignActiveTasks(allTasks);
+
+        assertThat(partitionAssignor.tryStickyAndBalancedTaskAssignmentWithinClient(state, consumers, partitionsForTask, Collections.emptySet()),
+            equalTo(Collections.emptyMap()));
+    }
+
+    @Test
+    public void shouldReturnEmptyMapWhenStickyAndBalancedAssignmentIsNotPossibleBecauseOtherClientOwnedPartition() {
+        configureDefault();
+        final ClientState state = new ClientState();
+
+        final List allTasks = Arrays.asList(task0_0, task0_1, task0_2, task0_3, task1_0, task1_1, task1_2, task1_3);
+
+        final Map> previousAssignment = new HashMap>() {{
+                put(c1, new ArrayList<>(Arrays.asList(task1_1, task1_3)));
+                put(c2, new ArrayList<>(Arrays.asList(task0_3, task1_0)));
+                put(c3, new ArrayList<>(Arrays.asList(task0_1, task0_2, task1_2)));
+            }};
+
+        for (final Map.Entry> entry : previousAssignment.entrySet()) {
+            for (final TaskId task : entry.getValue()) {
+                state.addOwnedPartitions(partitionsForTask.get(task), entry.getKey());
+            }
+        }
+
+        // Add the partitions of task0_0 to allOwnedPartitions but not c1's ownedPartitions/previousAssignment
+        final Set allOwnedPartitions = new HashSet<>(partitionsForTask.get(task0_0));
+
+        final Set consumers = Utils.mkSet(c1, c2, c3);
+        state.assignActiveTasks(allTasks);
+
+        assertThat(partitionAssignor.tryStickyAndBalancedTaskAssignmentWithinClient(state, consumers, partitionsForTask, allOwnedPartitions),
+            equalTo(Collections.emptyMap()));
+    }
+
     @Test
     public void shouldInterleaveTasksByGroupId() {
         final TaskId taskIdA0 = new TaskId(0, 0);
@@ -182,21 +362,31 @@ public void shouldInterleaveTasksByGroupId() {
         final TaskId taskIdC0 = new TaskId(2, 0);
         final TaskId taskIdC1 = new TaskId(2, 1);
 
+        final String c1 = "c1";
+        final String c2 = "c2";
+        final String c3 = "c3";
+
+        final Set consumers = Utils.mkSet(c1, c2, c3);
+
         final List expectedSubList1 = asList(taskIdA0, taskIdA3, taskIdB2);
         final List expectedSubList2 = asList(taskIdA1, taskIdB0, taskIdC0);
         final List expectedSubList3 = asList(taskIdA2, taskIdB1, taskIdC1);
-        final List> embeddedList = asList(expectedSubList1, expectedSubList2, expectedSubList3);
+
+        final Map> assignment = new HashMap<>();
+        assignment.put(c1, expectedSubList1);
+        assignment.put(c2, expectedSubList2);
+        assignment.put(c3, expectedSubList3);
 
         final List tasks = asList(taskIdC0, taskIdC1, taskIdB0, taskIdB1, taskIdB2, taskIdA0, taskIdA1, taskIdA2, taskIdA3);
         Collections.shuffle(tasks);
 
-        final List> interleavedTaskIds = StreamsPartitionAssignor.interleaveTasksByGroupId(tasks, 3);
+        final Map> interleavedTaskIds = StreamsPartitionAssignor.interleaveConsumerTasksByGroupId(tasks, consumers);
 
-        assertThat(interleavedTaskIds, equalTo(embeddedList));
+        assertThat(interleavedTaskIds, equalTo(assignment));
     }
 
     @Test
-    public void testSubscription() {
+    public void testEagerSubscription() {
         builder.addSource(null, "source1", null, null, null, "topic1");
         builder.addSource(null, "source2", null, null, null, "topic2");
         builder.addProcessor("processor", new MockProcessorSupplier(), "source1", "source2");
@@ -212,6 +402,7 @@ public void testSubscription() {
         EasyMock.replay(taskManager);
 
         configurePartitionAssignor(Collections.emptyMap());
+        partitionAssignor.setRebalanceProtocol(RebalanceProtocol.EAGER);
 
         final Set topics = Utils.mkSet("topic1", "topic2");
         final ConsumerPartitionAssignor.Subscription subscription = new ConsumerPartitionAssignor.Subscription(new ArrayList<>(topics), partitionAssignor.subscriptionUserData(topics));
@@ -222,24 +413,60 @@ public void testSubscription() {
         final Set standbyTasks = new HashSet<>(cachedTasks);
         standbyTasks.removeAll(prevTasks);
 
+        // When following the eager protocol, we must encode the previous tasks ourselves since we must revoke
+        // everything and thus the "ownedPartitions" field in the subscription will be empty
         final SubscriptionInfo info = new SubscriptionInfo(processId, prevTasks, standbyTasks, null);
         assertEquals(info.encode(), subscription.userData());
     }
 
+    @Test
+    public void testCooperativeSubscription() {
+        builder.addSource(null, "source1", null, null, null, "topic1");
+        builder.addSource(null, "source2", null, null, null, "topic2");
+        builder.addProcessor("processor", new MockProcessorSupplier(), "source1", "source2");
+
+        final Set prevTasks = Utils.mkSet(
+            new TaskId(0, 1), new TaskId(1, 1), new TaskId(2, 1));
+        final Set cachedTasks = Utils.mkSet(
+            new TaskId(0, 1), new TaskId(1, 1), new TaskId(2, 1),
+            new TaskId(0, 2), new TaskId(1, 2), new TaskId(2, 2));
+
+        final UUID processId = UUID.randomUUID();
+        createMockTaskManager(prevTasks, cachedTasks, processId, builder);
+        EasyMock.replay(taskManager);
+
+        configurePartitionAssignor(Collections.emptyMap());
+
+        final Set topics = Utils.mkSet("topic1", "topic2");
+        final ConsumerPartitionAssignor.Subscription subscription = new ConsumerPartitionAssignor.Subscription(
+            new ArrayList<>(topics), partitionAssignor.subscriptionUserData(topics));
+
+        Collections.sort(subscription.topics());
+        assertEquals(asList("topic1", "topic2"), subscription.topics());
+
+        final Set standbyTasks = new HashSet<>(cachedTasks);
+        standbyTasks.removeAll(prevTasks);
+
+        // We don't encode the active tasks when following the cooperative protocol, as these are inferred from the
+        // ownedPartitions encoded in the subscription
+        final SubscriptionInfo info = new SubscriptionInfo(processId, Collections.emptySet(), standbyTasks, null);
+        assertEquals(info.encode(), subscription.userData());
+    }
+
     @Test
     public void testAssignBasic() {
         builder.addSource(null, "source1", null, null, null, "topic1");
         builder.addSource(null, "source2", null, null, null, "topic2");
         builder.addProcessor("processor", new MockProcessorSupplier(), "source1", "source2");
         final List topics = asList("topic1", "topic2");
-        final Set allTasks = Utils.mkSet(task0, task1, task2);
+        final Set allTasks = Utils.mkSet(task0_0, task0_1, task0_2);
 
-        final Set prevTasks10 = Utils.mkSet(task0);
-        final Set prevTasks11 = Utils.mkSet(task1);
-        final Set prevTasks20 = Utils.mkSet(task2);
-        final Set standbyTasks10 = Utils.mkSet(task1);
-        final Set standbyTasks11 = Utils.mkSet(task2);
-        final Set standbyTasks20 = Utils.mkSet(task0);
+        final Set prevTasks10 = Utils.mkSet(task0_0);
+        final Set prevTasks11 = Utils.mkSet(task0_1);
+        final Set prevTasks20 = Utils.mkSet(task0_2);
+        final Set standbyTasks10 = Utils.mkSet(task0_1);
+        final Set standbyTasks11 = Utils.mkSet(task0_2);
+        final Set standbyTasks20 = Utils.mkSet(task0_0);
 
         final UUID uuid1 = UUID.randomUUID();
         final UUID uuid2 = UUID.randomUUID();
@@ -251,14 +478,20 @@ public void testAssignBasic() {
         partitionAssignor.setInternalTopicManager(new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer));
 
         subscriptions.put("consumer10",
-                new ConsumerPartitionAssignor.Subscription(topics,
-                        new SubscriptionInfo(uuid1, prevTasks10, standbyTasks10, userEndPoint).encode()));
+                new ConsumerPartitionAssignor.Subscription(
+                    topics,
+                    new SubscriptionInfo(uuid1, prevTasks10, standbyTasks10, userEndPoint).encode(),
+                    Collections.singletonList(t1p0)));
         subscriptions.put("consumer11",
-                new ConsumerPartitionAssignor.Subscription(topics,
-                        new SubscriptionInfo(uuid1, prevTasks11, standbyTasks11, userEndPoint).encode()));
+                new ConsumerPartitionAssignor.Subscription(
+                    topics,
+                    new SubscriptionInfo(uuid1, prevTasks11, standbyTasks11, userEndPoint).encode(),
+                    Collections.singletonList(t1p1)));
         subscriptions.put("consumer20",
-                new ConsumerPartitionAssignor.Subscription(topics,
-                        new SubscriptionInfo(uuid2, prevTasks20, standbyTasks20, userEndPoint).encode()));
+                new ConsumerPartitionAssignor.Subscription(
+                    topics,
+                    new SubscriptionInfo(uuid2, prevTasks20, standbyTasks20, userEndPoint).encode(),
+                    Collections.singletonList(t1p2)));
 
         final Map assignments = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
 
@@ -277,7 +510,7 @@ public void testAssignBasic() {
         final AssignmentInfo info11 = checkAssignment(allTopics, assignments.get("consumer11"));
         allActiveTasks.addAll(info11.activeTasks());
 
-        assertEquals(Utils.mkSet(task0, task1), allActiveTasks);
+        assertEquals(Utils.mkSet(task0_0, task0_1), allActiveTasks);
 
         // the third consumer
         final AssignmentInfo info20 = checkAssignment(allTopics, assignments.get("consumer20"));
@@ -351,12 +584,12 @@ public void shouldAssignEvenlyAcrossConsumersOneClientMultipleThreads() {
         // the first consumer
         final AssignmentInfo info10 = AssignmentInfo.decode(assignments.get("consumer10").userData());
 
-        final List expectedInfo10TaskIds = asList(taskIdA1, taskIdA3, taskIdB1, taskIdB3);
+        final List expectedInfo10TaskIds = asList(taskIdA0, taskIdA2, taskIdB0, taskIdB2);
         assertEquals(expectedInfo10TaskIds, info10.activeTasks());
 
         // the second consumer
         final AssignmentInfo info11 = AssignmentInfo.decode(assignments.get("consumer11").userData());
-        final List expectedInfo11TaskIds = asList(taskIdA0, taskIdA2, taskIdB0, taskIdB2);
+        final List expectedInfo11TaskIds = asList(taskIdA1, taskIdA3, taskIdB1, taskIdB3);
 
         assertEquals(expectedInfo11TaskIds, info11.activeTasks());
     }
@@ -371,7 +604,7 @@ public void testAssignWithPartialTopology() {
         builder.addProcessor("processor2", new MockProcessorSupplier(), "source2");
         builder.addStateStore(new MockKeyValueStoreBuilder("store2", false), "processor2");
         final List topics = asList("topic1", "topic2");
-        final Set allTasks = Utils.mkSet(task0, task1, task2);
+        final Set allTasks = Utils.mkSet(task0_0, task0_1, task0_2);
 
         final UUID uuid1 = UUID.randomUUID();
 
@@ -403,10 +636,10 @@ public void testAssignEmptyMetadata() {
         builder.addSource(null, "source2", null, null, null, "topic2");
         builder.addProcessor("processor", new MockProcessorSupplier(), "source1", "source2");
         final List topics = asList("topic1", "topic2");
-        final Set allTasks = Utils.mkSet(task0, task1, task2);
+        final Set allTasks = Utils.mkSet(task0_0, task0_1, task0_2);
 
-        final Set prevTasks10 = Utils.mkSet(task0);
-        final Set standbyTasks10 = Utils.mkSet(task1);
+        final Set prevTasks10 = Utils.mkSet(task0_0);
+        final Set standbyTasks10 = Utils.mkSet(task0_1);
         final  Cluster emptyMetadata = new Cluster("cluster", Collections.singletonList(Node.noNode()),
             Collections.emptySet(),
             Collections.emptySet(),
@@ -459,12 +692,12 @@ public void testAssignWithNewTasks() {
         builder.addSource(null, "source3", null, null, null, "topic3");
         builder.addProcessor("processor", new MockProcessorSupplier(), "source1", "source2", "source3");
         final List topics = asList("topic1", "topic2", "topic3");
-        final Set allTasks = Utils.mkSet(task0, task1, task2, task3);
+        final Set allTasks = Utils.mkSet(task0_0, task0_1, task0_2, task0_3);
 
         // assuming that previous tasks do not have topic3
-        final Set prevTasks10 = Utils.mkSet(task0);
-        final Set prevTasks11 = Utils.mkSet(task1);
-        final Set prevTasks20 = Utils.mkSet(task2);
+        final Set prevTasks10 = Utils.mkSet(task0_0);
+        final Set prevTasks11 = Utils.mkSet(task0_1);
+        final Set prevTasks20 = Utils.mkSet(task0_2);
 
         final UUID uuid1 = UUID.randomUUID();
         final UUID uuid2 = UUID.randomUUID();
@@ -606,15 +839,15 @@ public void testAssignWithStandbyReplicas() {
         builder.addSource(null, "source2", null, null, null, "topic2");
         builder.addProcessor("processor", new MockProcessorSupplier(), "source1", "source2");
         final List topics = asList("topic1", "topic2");
-        final Set allTasks = Utils.mkSet(task0, task1, task2);
+        final Set allTasks = Utils.mkSet(task0_0, task0_1, task0_2);
 
 
-        final Set prevTasks00 = Utils.mkSet(task0);
-        final Set prevTasks01 = Utils.mkSet(task1);
-        final Set prevTasks02 = Utils.mkSet(task2);
-        final Set standbyTasks01 = Utils.mkSet(task1);
-        final Set standbyTasks02 = Utils.mkSet(task2);
-        final Set standbyTasks00 = Utils.mkSet(task0);
+        final Set prevTasks00 = Utils.mkSet(task0_0);
+        final Set prevTasks01 = Utils.mkSet(task0_1);
+        final Set prevTasks02 = Utils.mkSet(task0_2);
+        final Set standbyTasks01 = Utils.mkSet(task0_1);
+        final Set standbyTasks02 = Utils.mkSet(task0_2);
+        final Set standbyTasks00 = Utils.mkSet(task0_0);
 
         final UUID uuid1 = UUID.randomUUID();
         final UUID uuid2 = UUID.randomUUID();
@@ -651,8 +884,8 @@ public void testAssignWithStandbyReplicas() {
         assertNotEquals("same processId has same set of standby tasks", info11.standbyTasks().keySet(), info10.standbyTasks().keySet());
 
         // check active tasks assigned to the first client
-        assertEquals(Utils.mkSet(task0, task1), new HashSet<>(allActiveTasks));
-        assertEquals(Utils.mkSet(task2), new HashSet<>(allStandbyTasks));
+        assertEquals(Utils.mkSet(task0_0, task0_1), new HashSet<>(allActiveTasks));
+        assertEquals(Utils.mkSet(task0_2), new HashSet<>(allStandbyTasks));
 
         // the third consumer
         final AssignmentInfo info20 = checkAssignment(allTopics, assignments.get("consumer20"));
@@ -678,11 +911,11 @@ public void testOnAssignment() {
         EasyMock.expectLastCall();
 
         final Map> activeTasks = new HashMap<>();
-        activeTasks.put(task0, Utils.mkSet(t3p0));
-        activeTasks.put(task3, Utils.mkSet(t3p3));
+        activeTasks.put(task0_0, Utils.mkSet(t3p0));
+        activeTasks.put(task0_3, Utils.mkSet(t3p3));
         final Map> standbyTasks = new HashMap<>();
-        standbyTasks.put(task1, Utils.mkSet(t3p1));
-        standbyTasks.put(task2, Utils.mkSet(t3p2));
+        standbyTasks.put(task0_1, Utils.mkSet(t3p1));
+        standbyTasks.put(task0_2, Utils.mkSet(t3p2));
         taskManager.setAssignmentMetadata(activeTasks, standbyTasks);
         EasyMock.expectLastCall();
 
@@ -693,7 +926,7 @@ public void testOnAssignment() {
         EasyMock.replay(taskManager);
 
         configurePartitionAssignor(Collections.emptyMap());
-        final List activeTaskList = asList(task0, task3);
+        final List activeTaskList = asList(task0_0, task0_3);
         final AssignmentInfo info = new AssignmentInfo(activeTaskList, standbyTasks, hostState);
         final ConsumerPartitionAssignor.Assignment assignment = new ConsumerPartitionAssignor.Assignment(asList(t3p0, t3p3), info.encode());
 
@@ -715,7 +948,7 @@ public void testAssignWithInternalTopics() {
         builder.addSource(null, "source2", null, null, null, "topicX");
         builder.addProcessor("processor2", new MockProcessorSupplier(), "source2");
         final List topics = asList("topic1", applicationId + "-topicX");
-        final Set allTasks = Utils.mkSet(task0, task1, task2);
+        final Set allTasks = Utils.mkSet(task0_0, task0_1, task0_2);
 
         final UUID uuid1 = UUID.randomUUID();
         createMockTaskManager(emptyTasks, emptyTasks, uuid1, builder);
@@ -749,7 +982,7 @@ public void testAssignWithInternalTopicThatsSourceIsAnotherInternalTopic() {
         builder.addSink("sink2", "topicZ", null, null, null, "processor2");
         builder.addSource(null, "source3", null, null, null, "topicZ");
         final List topics = asList("topic1", "test-topicX", "test-topicZ");
-        final Set allTasks = Utils.mkSet(task0, task1, task2);
+        final Set allTasks = Utils.mkSet(task0_0, task0_1, task0_2);
 
         final UUID uuid1 = UUID.randomUUID();
         createMockTaskManager(emptyTasks, emptyTasks, uuid1, builder);
@@ -1197,36 +1430,95 @@ private void shouldDownGradeSubscriptionToVersion2(final Object upgradeFromValue
     }
 
     @Test
-    public void shouldReturnUnchangedAssignmentForOldInstancesAndEmptyAssignmentForFutureInstances() {
+    public void shouldReturnInterleavedAssignmentWithUnrevokedPartitionsRemovedWhenNewConsumerJoins() {
         builder.addSource(null, "source1", null, null, null, "topic1");
 
-        final Set allTasks = Utils.mkSet(task0, task1, task2);
+        final Set allTasks = Utils.mkSet(task0_0, task0_1, task0_2);
+
+        subscriptions.put(c1,
+            new ConsumerPartitionAssignor.Subscription(
+                Collections.singletonList("topic1"),
+                new SubscriptionInfo(UUID.randomUUID(), allTasks, Collections.emptySet(), null).encode(),
+                Arrays.asList(t1p0, t1p1, t1p2))
+        );
+        subscriptions.put(c2,
+            new ConsumerPartitionAssignor.Subscription(
+                Collections.singletonList("topic1"),
+                new SubscriptionInfo(UUID.randomUUID(), Collections.emptySet(), Collections.emptySet(), null).encode(),
+                Collections.emptyList())
+        );
 
-        final Set activeTasks = Utils.mkSet(task0, task1);
-        final Set standbyTasks = Utils.mkSet(task2);
+        createMockTaskManager(allTasks, allTasks, UUID.randomUUID(), builder);
+        EasyMock.replay(taskManager);
+        partitionAssignor.configure(configProps());
+
+        final Map assignment = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
+
+        assertThat(assignment.size(), equalTo(2));
+
+        assertThat(assignment.get(c1).partitions(), equalTo(asList(t1p0, t1p2)));
+        assertThat(
+            AssignmentInfo.decode(assignment.get(c1).userData()),
+            equalTo(new AssignmentInfo(
+                Arrays.asList(task0_0, task0_2),
+                Collections.emptyMap(),
+                Collections.emptyMap()
+            )));
+
+        // The new consumer's assignment should be empty until c1 has the chance to revoke its partitions/tasks
+        assertThat(assignment.get(c2).partitions(), equalTo(Collections.emptyList()));
+        assertThat(
+            AssignmentInfo.decode(assignment.get(c2).userData()),
+            equalTo(new AssignmentInfo(
+                Collections.emptyList(),
+                Collections.emptyMap(),
+                Collections.emptyMap()
+            )));
+    }
+
+    @Test
+    public void shouldReturnNormalAssignmentForOldAndFutureInstancesDuringVersionProbing() {
+        builder.addSource(null, "source1", null, null, null, "topic1");
+
+        final Set allTasks = Utils.mkSet(task0_0, task0_1, task0_2);
+
+        final Set activeTasks = Utils.mkSet(task0_0, task0_1);
+        final Set standbyTasks = Utils.mkSet(task0_2);
         final Map> standbyTaskMap = new HashMap>() {
             {
-                put(task2, Collections.singleton(t1p2));
+                put(task0_2, Collections.singleton(t1p2));
+            }
+        };
+        final Map> futureStandbyTaskMap = new HashMap>() {
+            {
+                put(task0_0, Collections.singleton(t1p0));
+                put(task0_1, Collections.singleton(t1p1));
             }
         };
 
         subscriptions.put("consumer1",
                 new ConsumerPartitionAssignor.Subscription(
                         Collections.singletonList("topic1"),
-                        new SubscriptionInfo(UUID.randomUUID(), activeTasks, standbyTasks, null).encode())
+                        new SubscriptionInfo(UUID.randomUUID(), activeTasks, standbyTasks, null).encode(),
+                        Arrays.asList(t1p0, t1p1))
         );
         subscriptions.put("future-consumer",
                 new ConsumerPartitionAssignor.Subscription(
                         Collections.singletonList("topic1"),
-                        encodeFutureSubscription())
+                        encodeFutureSubscription(),
+                        Collections.singletonList(t1p2))
         );
 
         createMockTaskManager(allTasks, allTasks, UUID.randomUUID(), builder);
         EasyMock.replay(taskManager);
-        partitionAssignor.configure(configProps());
+        final Map props = configProps();
+        props.put(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 1);
+        partitionAssignor.configure(props);
         final Map assignment = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
 
         assertThat(assignment.size(), equalTo(2));
+
+        assertThat(assignment.get("consumer1").partitions(), equalTo(asList(t1p0, t1p1)));
         assertThat(
             AssignmentInfo.decode(assignment.get("consumer1").userData()),
             equalTo(new AssignmentInfo(
@@ -1234,10 +1526,64 @@ public void shouldReturnUnchangedAssignmentForOldInstancesAndEmptyAssignmentForF
                 standbyTaskMap,
                 Collections.emptyMap()
             )));
-        assertThat(assignment.get("consumer1").partitions(), equalTo(asList(t1p0, t1p1)));
 
-        assertThat(AssignmentInfo.decode(assignment.get("future-consumer").userData()), equalTo(new AssignmentInfo(LATEST_SUPPORTED_VERSION, LATEST_SUPPORTED_VERSION)));
-        assertThat(assignment.get("future-consumer").partitions().size(), equalTo(0));
+
+        assertThat(assignment.get("future-consumer").partitions(), equalTo(Collections.singletonList(t1p2)));
+        assertThat(
+            AssignmentInfo.decode(assignment.get("future-consumer").userData()),
+            equalTo(new AssignmentInfo(
+                Collections.singletonList(task0_2),
+                futureStandbyTaskMap,
+                Collections.emptyMap()
+            )));
+    }
+
+    @Test
+    public void shouldReturnInterleavedAssignmentForOnlyFutureInstancesDuringVersionProbing() {
+        builder.addSource(null, "source1", null, null, null, "topic1");
+
+        final Set allTasks = Utils.mkSet(task0_0, task0_1, task0_2);
+
+        subscriptions.put(c1,
+            new ConsumerPartitionAssignor.Subscription(
+                Collections.singletonList("topic1"),
+                encodeFutureSubscription(),
+                Collections.emptyList())
+        );
+        subscriptions.put(c2,
+            new ConsumerPartitionAssignor.Subscription(
+                Collections.singletonList("topic1"),
+                encodeFutureSubscription(),
+                Collections.emptyList())
+        );
+
+        createMockTaskManager(allTasks, allTasks, UUID.randomUUID(), builder);
+        EasyMock.replay(taskManager);
+        final Map props = configProps();
+        props.put(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 1);
+        partitionAssignor.configure(props);
+        final Map assignment = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
+
+        assertThat(assignment.size(), equalTo(2));
+
+        assertThat(assignment.get(c1).partitions(), equalTo(asList(t1p0, t1p2)));
+        assertThat(
+            AssignmentInfo.decode(assignment.get(c1).userData()),
+            equalTo(new AssignmentInfo(
+                Arrays.asList(task0_0, task0_2),
+                Collections.emptyMap(),
+                Collections.emptyMap()
+            )));
+
+
+        assertThat(assignment.get(c2).partitions(), equalTo(Collections.singletonList(t1p1)));
+        assertThat(
+            AssignmentInfo.decode(assignment.get(c2).userData()),
+            equalTo(new AssignmentInfo(
+                Collections.singletonList(task0_1),
+                Collections.emptyMap(),
+                Collections.emptyMap()
+            )));
     }
 
     @Test
@@ -1334,4 +1680,21 @@ private AssignmentInfo checkAssignment(final Set expectedTopics,
 
         return info;
     }
+
+    private void assertEquivalentAssignment(final Map> thisAssignment,
+                                            final Map> otherAssignment) {
+        assertEquals(thisAssignment.size(), otherAssignment.size());
+        for (final Map.Entry> entry : thisAssignment.entrySet()) {
+            final String consumer = entry.getKey();
+            assertTrue(otherAssignment.containsKey(consumer));
+
+            final List thisTaskList = entry.getValue();
+            Collections.sort(thisTaskList);
+            final List otherTaskList = otherAssignment.get(consumer);
+            Collections.sort(otherTaskList);
+
+            assertThat(thisTaskList, equalTo(otherTaskList));
+        }
+    }
+
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java
index 7e1ca7f96a465..e46a4cc16fec3 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java
@@ -42,7 +42,6 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -245,7 +244,8 @@ public void shouldReturnCachedTaskIdsFromDirectory() throws IOException {
     @Test
     public void shouldCloseActiveUnAssignedSuspendedTasksWhenClosingRevokedTasks() {
         mockSingleActiveTask();
-        EasyMock.expect(active.closeNotAssignedSuspendedTasks(taskId0Assignment.keySet())).andReturn(null).once();
+
+        expect(active.closeNotAssignedSuspendedTasks(taskId0Assignment.keySet())).andReturn(null).once();
         expect(restoreConsumer.assignment()).andReturn(Collections.emptySet());
 
         replay();
@@ -281,6 +281,7 @@ public void shouldAddNonResumedSuspendedTasks() {
         // Need to call this twice so task manager doesn't consider all partitions "new"
         taskManager.setAssignmentMetadata(taskId0Assignment, Collections.>emptyMap());
         taskManager.setAssignmentMetadata(taskId0Assignment, Collections.>emptyMap());
+
         taskManager.setPartitionsToTaskId(taskId0PartitionToTaskId);
         taskManager.createTasks(taskId0Partitions);
 
@@ -404,9 +405,7 @@ public void shouldUnassignChangelogPartitionsOnShutdown() {
 
     @Test
     public void shouldInitializeNewActiveTasks() {
-        EasyMock.expect(restoreConsumer.assignment()).andReturn(Collections.emptySet()).once();
-        EasyMock.expect(changeLogReader.restore(active)).andReturn(taskId0Partitions).once();
-        active.updateRestored(EasyMock.>anyObject());
+        active.initializeNewTasks();
         expectLastCall();
         replay();
 
@@ -416,9 +415,7 @@ public void shouldInitializeNewActiveTasks() {
 
     @Test
     public void shouldInitializeNewStandbyTasks() {
-        EasyMock.expect(restoreConsumer.assignment()).andReturn(Collections.emptySet()).once();
-        EasyMock.expect(changeLogReader.restore(active)).andReturn(taskId0Partitions).once();
-        active.updateRestored(EasyMock.>anyObject());
+        standby.initializeNewTasks();
         expectLastCall();
         replay();
 
@@ -428,6 +425,7 @@ public void shouldInitializeNewStandbyTasks() {
 
     @Test
     public void shouldRestoreStateFromChangeLogReader() {
+        EasyMock.expect(active.hasRestoringTasks()).andReturn(true).once();
         EasyMock.expect(restoreConsumer.assignment()).andReturn(taskId0Partitions).once();
         expect(changeLogReader.restore(active)).andReturn(taskId0Partitions);
         active.updateRestored(taskId0Partitions);
@@ -440,11 +438,9 @@ public void shouldRestoreStateFromChangeLogReader() {
 
     @Test
     public void shouldResumeRestoredPartitions() {
-        EasyMock.expect(restoreConsumer.assignment()).andReturn(taskId0Partitions).once();
-        expect(changeLogReader.restore(active)).andReturn(taskId0Partitions);
-        expect(active.allTasksRunning()).andReturn(true);
+        expect(active.allTasksRunning()).andReturn(true).once();
         expect(consumer.assignment()).andReturn(taskId0Partitions);
-        expect(standby.running()).andReturn(Collections.emptySet());
+        expect(standby.running()).andReturn(Collections.emptySet());
 
         consumer.resume(taskId0Partitions);
         expectLastCall();
@@ -666,6 +662,7 @@ public void shouldUpdateTasksFromPartitionAssignment() {
     }
 
     private void mockAssignStandbyPartitions(final long offset) {
+        expect(active.hasRestoringTasks()).andReturn(true).once();
         final StandbyTask task = EasyMock.createNiceMock(StandbyTask.class);
         expect(active.allTasksRunning()).andReturn(true);
         expect(standby.running()).andReturn(Collections.singletonList(task));
@@ -679,13 +676,6 @@ private void mockAssignStandbyPartitions(final long offset) {
         EasyMock.expect(changeLogReader.restore(active)).andReturn(taskId0Partitions).once();
     }
 
-    private void mockStandbyTaskExpectations() {
-        expect(standbyTaskCreator.createTasks(EasyMock.>anyObject(),
-                                                   EasyMock.eq(taskId0Assignment)))
-                .andReturn(Collections.singletonList(standbyTask));
-
-    }
-
     private void mockSingleActiveTask() {
         expect(activeTaskCreator.createTasks(EasyMock.>anyObject(),
                                                   EasyMock.eq(taskId0Assignment)))
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/ClientStateTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/ClientStateTest.java
index dc54c865d2b54..1443edf13c0c3 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/ClientStateTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/ClientStateTest.java
@@ -70,8 +70,8 @@ public void shouldAddPreviousActiveTasksToPreviousAssignedAndPreviousActive() {
         final TaskId tid1 = new TaskId(0, 1);
         final TaskId tid2 = new TaskId(0, 2);
 
-        client.addPreviousActiveTasks("consumer", Utils.mkSet(tid1, tid2));
-        assertThat(client.previousActiveTasks(), equalTo(Utils.mkSet(tid1, tid2)));
+        client.addPreviousActiveTasks(Utils.mkSet(tid1, tid2));
+        assertThat(client.prevActiveTasks(), equalTo(Utils.mkSet(tid1, tid2)));
         assertThat(client.previousAssignedTasks(), equalTo(Utils.mkSet(tid1, tid2)));
     }
 
@@ -80,8 +80,8 @@ public void shouldAddPreviousStandbyTasksToPreviousAssigned() {
         final TaskId tid1 = new TaskId(0, 1);
         final TaskId tid2 = new TaskId(0, 2);
 
-        client.addPreviousStandbyTasks("consumer", Utils.mkSet(tid1, tid2));
-        assertThat(client.previousActiveTasks().size(), equalTo(0));
+        client.addPreviousStandbyTasks(Utils.mkSet(tid1, tid2));
+        assertThat(client.prevActiveTasks().size(), equalTo(0));
         assertThat(client.previousAssignedTasks(), equalTo(Utils.mkSet(tid1, tid2)));
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/StickyTaskAssignorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/StickyTaskAssignorTest.java
index 19d7730787681..17d403f13061a 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/StickyTaskAssignorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/StickyTaskAssignorTest.java
@@ -207,11 +207,11 @@ public void shouldKeepActiveTaskStickynessWhenMoreClientThanActiveTasks() {
     @Test
     public void shouldAssignTasksToClientWithPreviousStandbyTasks() {
         final ClientState client1 = createClient(p1, 1);
-        client1.addPreviousStandbyTasks("consumer", Utils.mkSet(task02));
+        client1.addPreviousStandbyTasks(Utils.mkSet(task02));
         final ClientState client2 = createClient(p2, 1);
-        client2.addPreviousStandbyTasks("consumer", Utils.mkSet(task01));
+        client2.addPreviousStandbyTasks(Utils.mkSet(task01));
         final ClientState client3 = createClient(p3, 1);
-        client3.addPreviousStandbyTasks("consumer", Utils.mkSet(task00));
+        client3.addPreviousStandbyTasks(Utils.mkSet(task00));
 
         final StickyTaskAssignor taskAssignor = createTaskAssignor(task00, task01, task02);
 
@@ -225,9 +225,9 @@ public void shouldAssignTasksToClientWithPreviousStandbyTasks() {
     @Test
     public void shouldAssignBasedOnCapacityWhenMultipleClientHaveStandbyTasks() {
         final ClientState c1 = createClientWithPreviousActiveTasks(p1, 1, task00);
-        c1.addPreviousStandbyTasks("consumer", Utils.mkSet(task01));
+        c1.addPreviousStandbyTasks(Utils.mkSet(task01));
         final ClientState c2 = createClientWithPreviousActiveTasks(p2, 2, task02);
-        c2.addPreviousStandbyTasks("consumer", Utils.mkSet(task01));
+        c2.addPreviousStandbyTasks(Utils.mkSet(task01));
 
         final StickyTaskAssignor taskAssignor = createTaskAssignor(task00, task01, task02);
 
@@ -455,9 +455,9 @@ public void shouldNotHaveSameAssignmentOnAnyTwoHostsWhenThereArePreviousActiveTa
     @Test
     public void shouldNotHaveSameAssignmentOnAnyTwoHostsWhenThereArePreviousStandbyTasks() {
         final ClientState c1 = createClientWithPreviousActiveTasks(p1, 1, task01, task02);
-        c1.addPreviousStandbyTasks("consumer", Utils.mkSet(task03, task00));
+        c1.addPreviousStandbyTasks(Utils.mkSet(task03, task00));
         final ClientState c2 = createClientWithPreviousActiveTasks(p2, 1, task03, task00);
-        c2.addPreviousStandbyTasks("consumer", Utils.mkSet(task01, task02));
+        c2.addPreviousStandbyTasks(Utils.mkSet(task01, task02));
 
         createClient(p3, 1);
         createClient(p4, 1);
@@ -577,14 +577,14 @@ public void shouldAssignTasksNotPreviouslyActiveToNewClient() {
         final TaskId task23 = new TaskId(2, 3);
 
         final ClientState c1 = createClientWithPreviousActiveTasks(p1, 1, task01, task12, task13);
-        c1.addPreviousStandbyTasks("consumer", Utils.mkSet(task00, task11, task20, task21, task23));
+        c1.addPreviousStandbyTasks(Utils.mkSet(task00, task11, task20, task21, task23));
         final ClientState c2 = createClientWithPreviousActiveTasks(p2, 1, task00, task11, task22);
-        c2.addPreviousStandbyTasks("consumer", Utils.mkSet(task01, task10, task02, task20, task03, task12, task21, task13, task23));
+        c2.addPreviousStandbyTasks(Utils.mkSet(task01, task10, task02, task20, task03, task12, task21, task13, task23));
         final ClientState c3 = createClientWithPreviousActiveTasks(p3, 1, task20, task21, task23);
-        c3.addPreviousStandbyTasks("consumer", Utils.mkSet(task02, task12));
+        c3.addPreviousStandbyTasks(Utils.mkSet(task02, task12));
 
         final ClientState newClient = createClient(p4, 1);
-        newClient.addPreviousStandbyTasks("consumer", Utils.mkSet(task00, task10, task01, task02, task11, task20, task03, task12, task21, task13, task22, task23));
+        newClient.addPreviousStandbyTasks(Utils.mkSet(task00, task10, task01, task02, task11, task20, task03, task12, task21, task13, task22, task23));
 
         final StickyTaskAssignor taskAssignor = createTaskAssignor(task00, task10, task01, task02, task11, task20, task03, task12, task21, task13, task22, task23);
         taskAssignor.assign(0);
@@ -607,15 +607,15 @@ public void shouldAssignTasksNotPreviouslyActiveToMultipleNewClients() {
         final TaskId task23 = new TaskId(2, 3);
 
         final ClientState c1 = createClientWithPreviousActiveTasks(p1, 1, task01, task12, task13);
-        c1.addPreviousStandbyTasks("c1onsumer", Utils.mkSet(task00, task11, task20, task21, task23));
+        c1.addPreviousStandbyTasks(Utils.mkSet(task00, task11, task20, task21, task23));
         final ClientState c2 = createClientWithPreviousActiveTasks(p2, 1, task00, task11, task22);
-        c2.addPreviousStandbyTasks("consumer", Utils.mkSet(task01, task10, task02, task20, task03, task12, task21, task13, task23));
+        c2.addPreviousStandbyTasks(Utils.mkSet(task01, task10, task02, task20, task03, task12, task21, task13, task23));
 
         final ClientState bounce1 = createClient(p3, 1);
-        bounce1.addPreviousStandbyTasks("consumer", Utils.mkSet(task20, task21, task23));
+        bounce1.addPreviousStandbyTasks(Utils.mkSet(task20, task21, task23));
 
         final ClientState bounce2 = createClient(p4, 1);
-        bounce2.addPreviousStandbyTasks("consumer", Utils.mkSet(task02, task03, task10));
+        bounce2.addPreviousStandbyTasks(Utils.mkSet(task02, task03, task10));
 
         final StickyTaskAssignor taskAssignor = createTaskAssignor(task00, task10, task01, task02, task11, task20, task03, task12, task21, task13, task22, task23);
         taskAssignor.assign(0);
@@ -658,7 +658,7 @@ public void shouldAssignTasksToNewClientWithoutFlippingAssignmentBetweenExisting
         final TaskId task06 = new TaskId(0, 6);
         final ClientState c1 = createClientWithPreviousActiveTasks(p1, 1, task00, task01, task02, task06);
         final ClientState c2 = createClient(p2, 1);
-        c2.addPreviousStandbyTasks("consumer", Utils.mkSet(task03, task04, task05));
+        c2.addPreviousStandbyTasks(Utils.mkSet(task03, task04, task05));
         final ClientState newClient = createClient(p3, 1);
 
         final StickyTaskAssignor taskAssignor = createTaskAssignor(task00, task01, task02, task03, task04, task05, task06);
@@ -705,7 +705,7 @@ private ClientState createClient(final Integer processId, final int capacity) {
 
     private ClientState createClientWithPreviousActiveTasks(final Integer processId, final int capacity, final TaskId... taskIds) {
         final ClientState clientState = new ClientState(capacity);
-        clientState.addPreviousActiveTasks("consumer", Utils.mkSet(taskIds));
+        clientState.addPreviousActiveTasks(Utils.mkSet(taskIds));
         clients.put(processId, clientState);
         return clientState;
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestDriver.java b/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestDriver.java
index 98e6e8fc97824..496f89ad144b8 100644
--- a/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestDriver.java
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestDriver.java
@@ -569,7 +569,7 @@ private static boolean verifyTAgg(final PrintStream resultStream,
                 }
 
                 if (entry.getValue().getLast().value().longValue() != expectedCount) {
-                    resultStream.println("fail: key=" + key + " tagg=" + entry.getValue() + " expected=" + expected.get(key));
+                    resultStream.println("fail: key=" + key + " tagg=" + entry.getValue() + " expected=" + expectedCount);
                     resultStream.println("\t outputEvents: " + entry.getValue());
                     return false;
                 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
index 0e07cac971970..185fa7c3bb14e 100644
--- a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java
@@ -20,6 +20,7 @@
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.clients.consumer.ConsumerGroupMetadata;
 import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.RebalanceProtocol;
 import org.apache.kafka.clients.consumer.KafkaConsumer;
 import org.apache.kafka.common.Cluster;
 import org.apache.kafka.common.PartitionInfo;
@@ -61,6 +62,8 @@
 
 public class StreamsUpgradeTest {
 
+    private static final RebalanceProtocol REBALANCE_PROTOCOL = RebalanceProtocol.COOPERATIVE;
+
     @SuppressWarnings("unchecked")
     public static void main(final String[] args) throws Exception {
         if (args.length < 1) {
@@ -123,26 +126,26 @@ public ByteBuffer subscriptionUserData(final Set topics) {
             // 1. Client UUID (a unique id assigned to an instance of KafkaStreams)
             // 2. Task ids of previously running tasks
             // 3. Task ids of valid local states on the client's state directory.
-
             final TaskManager taskManager = taskManger();
-            final Set previousActiveTasks = taskManager.previousRunningTaskIds();
+
             final Set standbyTasks = taskManager.cachedTasksIds();
-            standbyTasks.removeAll(previousActiveTasks);
-            final FutureSubscriptionInfo data = new FutureSubscriptionInfo(
+            final Set activeTasks = prepareForSubscription(taskManager,
+                                                                   topics,
+                                                                   standbyTasks,
+                                                                   REBALANCE_PROTOCOL);
+            return new FutureSubscriptionInfo(
                 usedSubscriptionMetadataVersion,
                 LATEST_SUPPORTED_VERSION + 1,
                 taskManager.processId(),
-                previousActiveTasks,
+                activeTasks,
                 standbyTasks,
-                userEndPoint());
-
-            taskManager.updateSubscriptionsFromMetadata(topics);
-
-            return data.encode();
+                userEndPoint())
+                .encode();
         }
 
         @Override
-        public void onAssignment(final ConsumerPartitionAssignor.Assignment assignment, final ConsumerGroupMetadata metadata) {
+        public void onAssignment(final ConsumerPartitionAssignor.Assignment assignment,
+                                 final ConsumerGroupMetadata metadata) {
             try {
                 super.onAssignment(assignment, metadata);
                 return;
@@ -193,6 +196,7 @@ public void onAssignment(final ConsumerPartitionAssignor.Assignment assignment,
             taskManager.setPartitionsToTaskId(partitionsToTaskId);
             taskManager.setAssignmentMetadata(activeTasks, info.standbyTasks());
             taskManager.updateSubscriptionsFromAssignment(partitions);
+            taskManager.setRebalanceInProgress(false);
         }
 
         @Override
diff --git a/tests/kafkatest/tests/streams/streams_eos_test.py b/tests/kafkatest/tests/streams/streams_eos_test.py
index 7e5cc26b3837c..428db9be1b809 100644
--- a/tests/kafkatest/tests/streams/streams_eos_test.py
+++ b/tests/kafkatest/tests/streams/streams_eos_test.py
@@ -159,7 +159,7 @@ def abort_streams(self, keep_alive_processor1, keep_alive_processor2, processor_
 
     def wait_for_startup(self, monitor, processor):
         self.wait_for(monitor, processor, "StateChange: REBALANCING -> RUNNING")
-        self.wait_for(monitor, processor, "processed 500 records from topic")
+        self.wait_for(monitor, processor, "processed [0-9]* records from topic")
 
     def wait_for(self, monitor, processor, output):
         monitor.wait_until(output,

From eac1b3140510d81d50d96ef118a2df7126019caf Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Mon, 7 Oct 2019 09:21:14 -0700
Subject: [PATCH 0697/1071] KAFKA-8985; Add flexible version support to
 inter-broker APIs (#7453)

This patch adds flexible version support for the following inter-broker APIs: ControlledShutdown, LeaderAndIsr, UpdateMetadata, and StopReplica. Version checks have been removed from `getErrorResponse` methods since they were redundant given the checks in `AbstractRequest` and the respective`*Data` types.

Reviewers: Ismael Juma 
---
 .../requests/ControlledShutdownRequest.java   |  8 +--
 .../common/requests/LeaderAndIsrRequest.java  | 13 +----
 .../common/requests/StopReplicaRequest.java   | 11 +---
 .../requests/UpdateMetadataRequest.java       |  9 ++--
 .../message/ControlledShutdownRequest.json    |  4 +-
 .../message/ControlledShutdownResponse.json   |  4 +-
 .../common/message/LeaderAndIsrRequest.json   |  4 +-
 .../common/message/LeaderAndIsrResponse.json  |  4 +-
 .../common/message/StopReplicaRequest.json    |  4 +-
 .../common/message/StopReplicaResponse.json   |  4 +-
 .../common/message/UpdateMetadataRequest.json |  4 +-
 .../message/UpdateMetadataResponse.json       |  4 +-
 .../ControlledShutdownRequestTest.java        | 51 +++++++++++++++++++
 .../requests/LeaderAndIsrRequestTest.java     | 24 +++++++++
 .../requests/StopReplicaRequestTest.java      | 27 ++++++++++
 .../requests/UpdateMetadataRequestTest.java   | 24 +++++++++
 .../src/main/scala/kafka/api/ApiVersion.scala | 11 +++-
 .../controller/ControllerChannelManager.scala |  9 ++--
 .../main/scala/kafka/server/KafkaServer.scala |  5 +-
 .../scala/unit/kafka/api/ApiVersionTest.scala |  3 +-
 .../ControllerChannelManagerTest.scala        | 12 +++--
 21 files changed, 180 insertions(+), 59 deletions(-)
 create mode 100644 clients/src/test/java/org/apache/kafka/common/requests/ControlledShutdownRequestTest.java

diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java
index 83e9444da0ad1..71238452cf782 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java
@@ -24,7 +24,6 @@
 
 import java.nio.ByteBuffer;
 
-
 public class ControlledShutdownRequest extends AbstractRequest {
 
     public static class Builder extends AbstractRequest.Builder {
@@ -63,9 +62,10 @@ public ControlledShutdownRequest(Struct struct, short version) {
     }
 
     @Override
-    public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
-        return new ControlledShutdownResponse(new ControlledShutdownResponseData().
-            setErrorCode(Errors.forException(e).code()));
+    public ControlledShutdownResponse getErrorResponse(int throttleTimeMs, Throwable e) {
+        ControlledShutdownResponseData data = new ControlledShutdownResponseData()
+                .setErrorCode(Errors.forException(e).code());
+        return new ControlledShutdownResponse(data);
     }
 
     public static ControlledShutdownRequest parse(ByteBuffer buffer, short version) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
index 63013fff2b783..270069aef3c58 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
@@ -146,18 +146,7 @@ public LeaderAndIsrResponse getErrorResponse(int throttleTimeMs, Throwable e) {
                 .setErrorCode(error.code()));
         }
         responseData.setPartitionErrors(partitions);
-
-        short versionId = version();
-        switch (versionId) {
-            case 0:
-            case 1:
-            case 2:
-            case 3:
-                return new LeaderAndIsrResponse(responseData);
-            default:
-                throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
-                        versionId, this.getClass().getSimpleName(), ApiKeys.LEADER_AND_ISR.latestVersion()));
-        }
+        return new LeaderAndIsrResponse(responseData);
     }
 
     @Override
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
index 425f66a161900..0f60e20f311ae 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
@@ -116,16 +116,7 @@ public StopReplicaResponse getErrorResponse(int throttleTimeMs, Throwable e) {
                 .setErrorCode(error.code()));
         }
         data.setPartitionErrors(partitions);
-
-        short versionId = version();
-        switch (versionId) {
-            case 0:
-            case 1:
-                return new StopReplicaResponse(data);
-            default:
-                throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
-                        versionId, this.getClass().getSimpleName(), ApiKeys.STOP_REPLICA.latestVersion()));
-        }
+        return new StopReplicaResponse(data);
     }
 
     public boolean deletePartitions() {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
index 90dbff3430256..49f962d28938e 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
@@ -187,12 +187,9 @@ public long brokerEpoch() {
 
     @Override
     public UpdateMetadataResponse getErrorResponse(int throttleTimeMs, Throwable e) {
-        short version = version();
-        if (version <= 5)
-            return new UpdateMetadataResponse(new UpdateMetadataResponseData().setErrorCode(Errors.forException(e).code()));
-        else
-            throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
-                version, this.getClass().getSimpleName(), ApiKeys.UPDATE_METADATA.latestVersion()));
+        UpdateMetadataResponseData data = new UpdateMetadataResponseData()
+                .setErrorCode(Errors.forException(e).code());
+        return new UpdateMetadataResponse(data);
     }
 
     public Iterable partitionStates() {
diff --git a/clients/src/main/resources/common/message/ControlledShutdownRequest.json b/clients/src/main/resources/common/message/ControlledShutdownRequest.json
index 6592e78d870d9..5756d1c16b5df 100644
--- a/clients/src/main/resources/common/message/ControlledShutdownRequest.json
+++ b/clients/src/main/resources/common/message/ControlledShutdownRequest.json
@@ -24,8 +24,8 @@
   // Version 1 is the same as version 0.
   //
   // Version 2 adds BrokerEpoch.
-  "validVersions": "0-2",
-  "flexibleVersions": "none",
+  "validVersions": "0-3",
+  "flexibleVersions": "3+",
   "fields": [
     { "name": "BrokerId", "type": "int32", "versions": "0+", "entityType": "brokerId",
       "about": "The id of the broker for which controlled shutdown has been requested." },
diff --git a/clients/src/main/resources/common/message/ControlledShutdownResponse.json b/clients/src/main/resources/common/message/ControlledShutdownResponse.json
index dc61d70e4d4c9..27feb1b69b087 100644
--- a/clients/src/main/resources/common/message/ControlledShutdownResponse.json
+++ b/clients/src/main/resources/common/message/ControlledShutdownResponse.json
@@ -18,8 +18,8 @@
   "type": "response",
   "name": "ControlledShutdownResponse",
   // Versions 1 and 2 are the same as version 0.
-  "validVersions": "0-2",
-  "flexibleVersions": "none",
+  "validVersions": "0-3",
+  "flexibleVersions": "3+",
   "fields": [
     { "name": "ErrorCode", "type": "int16", "versions": "0+",
       "about": "The top-level error code." },
diff --git a/clients/src/main/resources/common/message/LeaderAndIsrRequest.json b/clients/src/main/resources/common/message/LeaderAndIsrRequest.json
index 8af1df5f55c84..852968801cee3 100644
--- a/clients/src/main/resources/common/message/LeaderAndIsrRequest.json
+++ b/clients/src/main/resources/common/message/LeaderAndIsrRequest.json
@@ -22,8 +22,8 @@
   // Version 2 adds broker epoch and reorganizes the partitions by topic.
   //
   // Version 3 adds AddingReplicas and RemovingReplicas
-  "validVersions": "0-3",
-  "flexibleVersions": "none",
+  "validVersions": "0-4",
+  "flexibleVersions": "4+",
   "fields": [
     { "name": "ControllerId", "type": "int32", "versions": "0+", "entityType": "brokerId",
       "about": "The current controller ID." },
diff --git a/clients/src/main/resources/common/message/LeaderAndIsrResponse.json b/clients/src/main/resources/common/message/LeaderAndIsrResponse.json
index 3b67c4cc9e073..10c3cd95e5fb2 100644
--- a/clients/src/main/resources/common/message/LeaderAndIsrResponse.json
+++ b/clients/src/main/resources/common/message/LeaderAndIsrResponse.json
@@ -22,8 +22,8 @@
   // Version 2 is the same as version 1.
   //
   // Version 3 is the same as version 2.
-  "validVersions": "0-3",
-  "flexibleVersions": "none",
+  "validVersions": "0-4",
+  "flexibleVersions": "4+",
   "fields": [
     { "name": "ErrorCode", "type": "int16", "versions": "0+",
       "about": "The error code, or 0 if there was no error." },
diff --git a/clients/src/main/resources/common/message/StopReplicaRequest.json b/clients/src/main/resources/common/message/StopReplicaRequest.json
index 695c0c186d1de..5c13705318a54 100644
--- a/clients/src/main/resources/common/message/StopReplicaRequest.json
+++ b/clients/src/main/resources/common/message/StopReplicaRequest.json
@@ -19,8 +19,8 @@
   "name": "StopReplicaRequest",
   // Version 1 adds the broker epoch and reorganizes the partitions to be stored
   // per topic.
-  "validVersions": "0-1",
-  "flexibleVersions": "none",
+  "validVersions": "0-2",
+  "flexibleVersions": "2+",
   "fields": [
     { "name": "ControllerId", "type": "int32", "versions": "0+", "entityType": "brokerId",
       "about": "The controller id." },
diff --git a/clients/src/main/resources/common/message/StopReplicaResponse.json b/clients/src/main/resources/common/message/StopReplicaResponse.json
index d55a7b5e5a881..d864e91a41b0e 100644
--- a/clients/src/main/resources/common/message/StopReplicaResponse.json
+++ b/clients/src/main/resources/common/message/StopReplicaResponse.json
@@ -18,8 +18,8 @@
   "type": "response",
   "name": "StopReplicaResponse",
   // Version 1 is the same as version 0.
-  "validVersions": "0-1",
-  "flexibleVersions": "none",
+  "validVersions": "0-2",
+  "flexibleVersions": "2+",
   "fields": [
     { "name": "ErrorCode", "type": "int16", "versions": "0+",
       "about": "The top-level error code, or 0 if there was no top-level error." },
diff --git a/clients/src/main/resources/common/message/UpdateMetadataRequest.json b/clients/src/main/resources/common/message/UpdateMetadataRequest.json
index 203ee44cc6617..3c45f833a7cab 100644
--- a/clients/src/main/resources/common/message/UpdateMetadataRequest.json
+++ b/clients/src/main/resources/common/message/UpdateMetadataRequest.json
@@ -26,8 +26,8 @@
   // Version 4 adds the offline replica list.
   //
   // Version 5 adds the broker epoch field and normalizes partitions by topic.
-  "validVersions": "0-5",
-  "flexibleVersions": "none",
+  "validVersions": "0-6",
+  "flexibleVersions": "6+",
   "fields": [
     { "name": "ControllerId", "type": "int32", "versions": "0+", "entityType": "brokerId",
       "about": "The controller id." },
diff --git a/clients/src/main/resources/common/message/UpdateMetadataResponse.json b/clients/src/main/resources/common/message/UpdateMetadataResponse.json
index 5990bebf39d7a..aaebed04ab108 100644
--- a/clients/src/main/resources/common/message/UpdateMetadataResponse.json
+++ b/clients/src/main/resources/common/message/UpdateMetadataResponse.json
@@ -18,8 +18,8 @@
   "type": "response",
   "name": "UpdateMetadataResponse",
   // Versions 1, 2, 3, 4, and 5 are the same as version 0
-  "validVersions": "0-5",
-  "flexibleVersions": "none",
+  "validVersions": "0-6",
+  "flexibleVersions": "6+",
   "fields": [
       { "name": "ErrorCode", "type": "int16", "versions": "0+",
         "about": "The error code, or 0 if there was no error." }
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ControlledShutdownRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ControlledShutdownRequestTest.java
new file mode 100644
index 0000000000000..b7ec3da9d1b01
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/requests/ControlledShutdownRequestTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.requests;
+
+import org.apache.kafka.common.errors.ClusterAuthorizationException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ControlledShutdownRequestData;
+import org.apache.kafka.common.protocol.Errors;
+import org.junit.Test;
+
+import static org.apache.kafka.common.protocol.ApiKeys.CONTROLLED_SHUTDOWN;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+
+public class ControlledShutdownRequestTest {
+
+    @Test
+    public void testUnsupportedVersion() {
+        ControlledShutdownRequest.Builder builder = new ControlledShutdownRequest.Builder(
+                new ControlledShutdownRequestData().setBrokerId(1),
+                (short) (CONTROLLED_SHUTDOWN.latestVersion() + 1));
+        assertThrows(UnsupportedVersionException.class, builder::build);
+    }
+
+    @Test
+    public void testGetErrorResponse() {
+        for (short version = CONTROLLED_SHUTDOWN.oldestVersion(); version < CONTROLLED_SHUTDOWN.latestVersion(); version++) {
+            ControlledShutdownRequest.Builder builder = new ControlledShutdownRequest.Builder(
+                    new ControlledShutdownRequestData().setBrokerId(1), version);
+            ControlledShutdownRequest request = builder.build();
+            ControlledShutdownResponse response = request.getErrorResponse(0,
+                    new ClusterAuthorizationException("Not authorized"));
+            assertEquals(Errors.CLUSTER_AUTHORIZATION_FAILED, response.error());
+        }
+    }
+
+}
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java
index 6ad6197b6acae..2235a8f643cf8 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java
@@ -18,10 +18,13 @@
 
 import org.apache.kafka.common.Node;
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.ClusterAuthorizationException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
 import org.apache.kafka.common.message.LeaderAndIsrRequestData;
 import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrLiveLeader;
 import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState;
 import org.apache.kafka.common.protocol.ByteBufferAccessor;
+import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.protocol.MessageTestUtil;
 import org.apache.kafka.test.TestUtils;
 import org.junit.Test;
@@ -39,10 +42,31 @@
 import static java.util.Collections.emptyList;
 import static org.apache.kafka.common.protocol.ApiKeys.LEADER_AND_ISR;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 
 public class LeaderAndIsrRequestTest {
 
+    @Test
+    public void testUnsupportedVersion() {
+        LeaderAndIsrRequest.Builder builder = new LeaderAndIsrRequest.Builder(
+                (short) (LEADER_AND_ISR.latestVersion() + 1), 0, 0, 0,
+                Collections.emptyList(), Collections.emptySet());
+        assertThrows(UnsupportedVersionException.class, builder::build);
+    }
+
+    @Test
+    public void testGetErrorResponse() {
+        for (short version = LEADER_AND_ISR.oldestVersion(); version < LEADER_AND_ISR.latestVersion(); version++) {
+            LeaderAndIsrRequest.Builder builder = new LeaderAndIsrRequest.Builder(version, 0, 0, 0,
+                    Collections.emptyList(), Collections.emptySet());
+            LeaderAndIsrRequest request = builder.build();
+            LeaderAndIsrResponse response = request.getErrorResponse(0,
+                    new ClusterAuthorizationException("Not authorized"));
+            assertEquals(Errors.CLUSTER_AUTHORIZATION_FAILED, response.error());
+        }
+    }
+
     /**
      * Verifies the logic we have in LeaderAndIsrRequest to present a unified interface across the various versions
      * works correctly. For example, `LeaderAndIsrPartitionState.topicName` is not serialiazed/deserialized in
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java
index a143ff3301a12..5bb4998456af4 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java
@@ -17,16 +17,43 @@
 package org.apache.kafka.common.requests;
 
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.ClusterAuthorizationException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.protocol.MessageTestUtil;
 import org.apache.kafka.test.TestUtils;
 import org.junit.Test;
 
+import java.util.Collections;
 import java.util.Set;
 
+import static org.apache.kafka.common.protocol.ApiKeys.STOP_REPLICA;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 
 public class StopReplicaRequestTest {
 
+    @Test
+    public void testUnsupportedVersion() {
+        StopReplicaRequest.Builder builder = new StopReplicaRequest.Builder(
+                (short) (STOP_REPLICA.latestVersion() + 1),
+                0, 0, 0L, false, Collections.emptyList());
+        assertThrows(UnsupportedVersionException.class, builder::build);
+    }
+
+    @Test
+    public void testGetErrorResponse() {
+        for (short version = STOP_REPLICA.oldestVersion(); version < STOP_REPLICA.latestVersion(); version++) {
+            StopReplicaRequest.Builder builder = new StopReplicaRequest.Builder(version,
+                    0, 0, 0L, false, Collections.emptyList());
+            StopReplicaRequest request = builder.build();
+            StopReplicaResponse response = request.getErrorResponse(0,
+                    new ClusterAuthorizationException("Not authorized"));
+            assertEquals(Errors.CLUSTER_AUTHORIZATION_FAILED, response.error());
+        }
+    }
+
     @Test
     public void testStopReplicaRequestNormalization() {
         Set tps = TestUtils.generateRandomTopicPartitions(10, 10);
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java
index a5d0d99d3d61d..fe688ce75cc9a 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java
@@ -17,12 +17,15 @@
 package org.apache.kafka.common.requests;
 
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.ClusterAuthorizationException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
 import org.apache.kafka.common.message.UpdateMetadataRequestData;
 import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataBroker;
 import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataEndpoint;
 import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataPartitionState;
 import org.apache.kafka.common.network.ListenerName;
 import org.apache.kafka.common.protocol.ByteBufferAccessor;
+import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.protocol.MessageTestUtil;
 import org.apache.kafka.common.security.auth.SecurityProtocol;
 import org.apache.kafka.test.TestUtils;
@@ -41,10 +44,31 @@
 import static java.util.Collections.emptyList;
 import static org.apache.kafka.common.protocol.ApiKeys.UPDATE_METADATA;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 
 public class UpdateMetadataRequestTest {
 
+    @Test
+    public void testUnsupportedVersion() {
+        UpdateMetadataRequest.Builder builder = new UpdateMetadataRequest.Builder(
+                (short) (UPDATE_METADATA.latestVersion() + 1), 0, 0, 0,
+                Collections.emptyList(), Collections.emptyList());
+        assertThrows(UnsupportedVersionException.class, builder::build);
+    }
+
+    @Test
+    public void testGetErrorResponse() {
+        for (short version = UPDATE_METADATA.oldestVersion(); version < UPDATE_METADATA.latestVersion(); version++) {
+            UpdateMetadataRequest.Builder builder = new UpdateMetadataRequest.Builder(
+                    version, 0, 0, 0, Collections.emptyList(), Collections.emptyList());
+            UpdateMetadataRequest request = builder.build();
+            UpdateMetadataResponse response = request.getErrorResponse(0,
+                    new ClusterAuthorizationException("Not authorized"));
+            assertEquals(Errors.CLUSTER_AUTHORIZATION_FAILED, response.error());
+        }
+    }
+
     /**
      * Verifies the logic we have in UpdateMetadataRequest to present a unified interface across the various versions
      * works correctly. For example, `UpdateMetadataPartitionState.topicName` is not serialiazed/deserialized in
diff --git a/core/src/main/scala/kafka/api/ApiVersion.scala b/core/src/main/scala/kafka/api/ApiVersion.scala
index 1a3362508c100..90ffb10347d8a 100644
--- a/core/src/main/scala/kafka/api/ApiVersion.scala
+++ b/core/src/main/scala/kafka/api/ApiVersion.scala
@@ -92,7 +92,9 @@ object ApiVersion {
     // Add rack_id to FetchRequest, preferred_read_replica to FetchResponse, and replica_id to OffsetsForLeaderRequest
     KAFKA_2_3_IV1,
     // Add adding_replicas and removing_replicas fields to LeaderAndIsrRequest
-    KAFKA_2_4_IV0
+    KAFKA_2_4_IV0,
+    // Flexible version support in inter-broker APIs
+    KAFKA_2_4_IV1
   )
 
   // Map keys are the union of the short and full versions
@@ -325,6 +327,13 @@ case object KAFKA_2_4_IV0 extends DefaultApiVersion {
   val id: Int = 24
 }
 
+case object KAFKA_2_4_IV1 extends DefaultApiVersion {
+  val shortVersion: String = "2.4"
+  val subVersion = "IV1"
+  val recordVersion = RecordVersion.V2
+  val id: Int = 25
+}
+
 object ApiVersionValidator extends Validator {
 
   override def ensureValid(name: String, value: Any): Unit = {
diff --git a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
index 6c7450a0c7a99..c60130b1d3ad7 100755
--- a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
+++ b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
@@ -447,7 +447,8 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
 
   private def sendLeaderAndIsrRequest(controllerEpoch: Int, stateChangeLog: StateChangeLogger): Unit = {
     val leaderAndIsrRequestVersion: Short =
-      if (config.interBrokerProtocolVersion >= KAFKA_2_4_IV0) 3
+      if (config.interBrokerProtocolVersion >= KAFKA_2_4_IV1) 4
+      else if (config.interBrokerProtocolVersion >= KAFKA_2_4_IV0) 3
       else if (config.interBrokerProtocolVersion >= KAFKA_2_2_IV0) 2
       else if (config.interBrokerProtocolVersion >= KAFKA_1_0_IV0) 1
       else 0
@@ -483,7 +484,8 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
 
     val partitionStates = updateMetadataRequestPartitionInfoMap.values.toBuffer
     val updateMetadataRequestVersion: Short =
-      if (config.interBrokerProtocolVersion >= KAFKA_2_2_IV0) 5
+      if (config.interBrokerProtocolVersion >= KAFKA_2_4_IV1) 6
+      else if (config.interBrokerProtocolVersion >= KAFKA_2_2_IV0) 5
       else if (config.interBrokerProtocolVersion >= KAFKA_1_0_IV0) 4
       else if (config.interBrokerProtocolVersion >= KAFKA_0_10_2_IV0) 3
       else if (config.interBrokerProtocolVersion >= KAFKA_0_10_0_IV1) 2
@@ -528,7 +530,8 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
 
   private def sendStopReplicaRequests(controllerEpoch: Int): Unit = {
     val stopReplicaRequestVersion: Short =
-      if (config.interBrokerProtocolVersion >= KAFKA_2_2_IV0) 1
+      if (config.interBrokerProtocolVersion >= KAFKA_2_4_IV1) 2
+      else if (config.interBrokerProtocolVersion >= KAFKA_2_2_IV0) 1
       else 0
 
     def stopReplicaPartitionDeleteResponseCallback(brokerId: Int)(response: AbstractResponse): Unit = {
diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala
index 57cb0b6d20ed1..de0ee22155185 100755
--- a/core/src/main/scala/kafka/server/KafkaServer.scala
+++ b/core/src/main/scala/kafka/server/KafkaServer.scala
@@ -24,7 +24,7 @@ import java.util.concurrent._
 import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger}
 
 import com.yammer.metrics.core.Gauge
-import kafka.api.{KAFKA_0_9_0, KAFKA_2_2_IV0}
+import kafka.api.{KAFKA_0_9_0, KAFKA_2_2_IV0, KAFKA_2_4_IV1}
 import kafka.cluster.Broker
 import kafka.common.{GenerateBrokerIdException, InconsistentBrokerIdException, InconsistentBrokerMetadataException, InconsistentClusterIdException}
 import kafka.controller.KafkaController
@@ -528,7 +528,8 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
               val controlledShutdownApiVersion: Short =
                 if (config.interBrokerProtocolVersion < KAFKA_0_9_0) 0
                 else if (config.interBrokerProtocolVersion < KAFKA_2_2_IV0) 1
-                else 2
+                else if (config.interBrokerProtocolVersion < KAFKA_2_4_IV1) 2
+                else 3
 
               val controlledShutdownRequest = new ControlledShutdownRequest.Builder(
                   new ControlledShutdownRequestData()
diff --git a/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala b/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala
index dadef1d00d5d7..0519ea04c70e9 100644
--- a/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala
+++ b/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala
@@ -93,8 +93,9 @@ class ApiVersionTest {
     assertEquals(KAFKA_2_3_IV0, ApiVersion("2.3-IV0"))
     assertEquals(KAFKA_2_3_IV1, ApiVersion("2.3-IV1"))
 
-    assertEquals(KAFKA_2_4_IV0, ApiVersion("2.4"))
+    assertEquals(KAFKA_2_4_IV1, ApiVersion("2.4"))
     assertEquals(KAFKA_2_4_IV0, ApiVersion("2.4-IV0"))
+    assertEquals(KAFKA_2_4_IV1, ApiVersion("2.4-IV1"))
   }
 
   @Test
diff --git a/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala b/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala
index e603b25cfa3f4..88607f2a65900 100644
--- a/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala
@@ -18,7 +18,7 @@ package kafka.controller
 
 import java.util.Properties
 
-import kafka.api.{ApiVersion, KAFKA_0_10_0_IV1, KAFKA_0_10_2_IV0, KAFKA_0_9_0, KAFKA_1_0_IV0, KAFKA_2_2_IV0, KAFKA_2_4_IV0, LeaderAndIsr}
+import kafka.api.{ApiVersion, KAFKA_0_10_0_IV1, KAFKA_0_10_2_IV0, KAFKA_0_9_0, KAFKA_1_0_IV0, KAFKA_2_2_IV0, KAFKA_2_4_IV0, KAFKA_2_4_IV1, LeaderAndIsr}
 import kafka.cluster.{Broker, EndPoint}
 import kafka.server.KafkaConfig
 import kafka.utils.TestUtils
@@ -157,7 +157,8 @@ class ControllerChannelManagerTest {
 
     for (apiVersion <- ApiVersion.allVersions) {
       val leaderAndIsrRequestVersion: Short =
-        if (apiVersion >= KAFKA_2_4_IV0) 3
+        if (apiVersion >= KAFKA_2_4_IV1) 4
+        else if (apiVersion >= KAFKA_2_4_IV0) 3
         else if (apiVersion >= KAFKA_2_2_IV0) 2
         else if (apiVersion >= KAFKA_1_0_IV0) 1
         else 0
@@ -326,7 +327,8 @@ class ControllerChannelManagerTest {
 
     for (apiVersion <- ApiVersion.allVersions) {
       val updateMetadataRequestVersion: Short =
-        if (apiVersion >= KAFKA_2_2_IV0) 5
+        if (apiVersion >= KAFKA_2_4_IV1) 6
+        else if (apiVersion >= KAFKA_2_2_IV0) 5
         else if (apiVersion >= KAFKA_1_0_IV0) 4
         else if (apiVersion >= KAFKA_0_10_2_IV0) 3
         else if (apiVersion >= KAFKA_0_10_0_IV1) 2
@@ -597,8 +599,10 @@ class ControllerChannelManagerTest {
     for (apiVersion <- ApiVersion.allVersions) {
       if (apiVersion < KAFKA_2_2_IV0)
         testStopReplicaFollowsInterBrokerProtocolVersion(apiVersion, 0.toShort)
-      else
+      else if (apiVersion < KAFKA_2_4_IV1)
         testStopReplicaFollowsInterBrokerProtocolVersion(apiVersion, 1.toShort)
+      else
+        testStopReplicaFollowsInterBrokerProtocolVersion(apiVersion, 2.toShort)
     }
   }
 

From 6859820755332eb90f386a5b76a0c43e035637dd Mon Sep 17 00:00:00 2001
From: Bruno Cadonna 
Date: Mon, 7 Oct 2019 19:47:50 +0200
Subject: [PATCH 0698/1071] HOTFIX: Hide built-in metrics version (#7459)

Since currently not all refactorings on streams metrics proposed in KIP-444 has yet been implemented, this commit hides the built-in metrics version config from the user. Thus, the user cannot switch to the refactored streams metrics.

Reviewers: Guozhang Wang 
---
 .../apache/kafka/streams/KafkaStreams.java    |  3 +-
 .../apache/kafka/streams/StreamsConfig.java   | 23 -------------
 .../internals/metrics/StreamsMetricsImpl.java | 16 +++++++--
 .../kafka/streams/StreamsConfigTest.java      | 34 -------------------
 .../integration/MetricsIntegrationTest.java   | 21 +++++-------
 .../internals/GlobalStreamThreadTest.java     |  4 +--
 .../internals/MockStreamsMetrics.java         |  3 +-
 .../processor/internals/StandbyTaskTest.java  |  3 +-
 .../processor/internals/StreamThreadTest.java | 18 +---------
 .../metrics/StreamsMetricsImplTest.java       | 16 ++++-----
 .../GlobalStateStoreProviderTest.java         |  3 +-
 .../MeteredTimestampedWindowStoreTest.java    |  2 +-
 .../internals/MeteredWindowStoreTest.java     |  2 +-
 .../test/InternalMockProcessorContext.java    | 10 +++---
 .../kafka/streams/TopologyTestDriver.java     |  4 +--
 .../processor/MockProcessorContext.java       |  3 +-
 16 files changed, 47 insertions(+), 118 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
index 5a00fe39f6cad..146eefd966555 100644
--- a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
+++ b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
@@ -675,8 +675,7 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder,
                 Collections.singletonMap(StreamsConfig.CLIENT_ID_CONFIG, clientId));
         reporters.add(new JmxReporter(JMX_PREFIX));
         metrics = new Metrics(metricConfig, reporters, time);
-        streamsMetrics =
-            new StreamsMetricsImpl(metrics, clientId, config.getString(StreamsConfig.BUILT_IN_METRICS_VERSION_CONFIG));
+        streamsMetrics = new StreamsMetricsImpl(metrics, clientId, StreamsMetricsImpl.METRICS_0100_TO_23);
         streamsMetrics.setRocksDBMetricsRecordingTrigger(rocksDBMetricsRecordingTrigger);
         ClientMetrics.addVersionMetric(streamsMetrics);
         ClientMetrics.addCommitIdMetric(streamsMetrics);
diff --git a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
index eb6faffd45b03..0a4bfd1c09d8f 100644
--- a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
+++ b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
@@ -287,16 +287,6 @@ public class StreamsConfig extends AbstractConfig {
     @SuppressWarnings("WeakerAccess")
     public static final String EXACTLY_ONCE = "exactly_once";
 
-    /**
-     * Config value for parameter {@link #BUILT_IN_METRICS_VERSION_CONFIG "built.in.metrics.version"} for built-in metrics from version 0.10.0. to 2.3
-     */
-    public static final String METRICS_0100_TO_23 = "0.10.0-2.3";
-
-    /**
-     * Config value for parameter {@link #BUILT_IN_METRICS_VERSION_CONFIG "built.in.metrics.version"} for the latest built-in metrics version.
-     */
-    public static final String METRICS_LATEST = "latest";
-
     /** {@code application.id} */
     @SuppressWarnings("WeakerAccess")
     public static final String APPLICATION_ID_CONFIG = "application.id";
@@ -316,10 +306,6 @@ public class StreamsConfig extends AbstractConfig {
     public static final String BUFFERED_RECORDS_PER_PARTITION_CONFIG = "buffered.records.per.partition";
     private static final String BUFFERED_RECORDS_PER_PARTITION_DOC = "Maximum number of records to buffer per partition.";
 
-    /** {@code built.in.metrics.version} */
-    public static final String BUILT_IN_METRICS_VERSION_CONFIG = "built.in.metrics.version";
-    private static final String BUILT_IN_METRICS_VERSION_DOC = "Version of the built-in metrics to use.";
-
     /** {@code cache.max.bytes.buffering} */
     @SuppressWarnings("WeakerAccess")
     public static final String CACHE_MAX_BYTES_BUFFERING_CONFIG = "cache.max.bytes.buffering";
@@ -625,15 +611,6 @@ public class StreamsConfig extends AbstractConfig {
                     1000,
                     Importance.LOW,
                     BUFFERED_RECORDS_PER_PARTITION_DOC)
-            .define(BUILT_IN_METRICS_VERSION_CONFIG,
-                    Type.STRING,
-                    METRICS_LATEST,
-                    in(
-                        METRICS_0100_TO_23,
-                        METRICS_LATEST
-                    ),
-                    Importance.LOW,
-                    BUILT_IN_METRICS_VERSION_DOC)
             .define(COMMIT_INTERVAL_MS_CONFIG,
                     Type.LONG,
                     DEFAULT_COMMIT_INTERVAL_MS,
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
index aea1f03341517..80422234bef03 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
@@ -32,7 +32,6 @@
 import org.apache.kafka.common.metrics.stats.Value;
 import org.apache.kafka.common.metrics.stats.WindowedCount;
 import org.apache.kafka.common.metrics.stats.WindowedSum;
-import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.StreamsMetrics;
 import org.apache.kafka.streams.state.internals.metrics.RocksDBMetricsRecordingTrigger;
 
@@ -53,7 +52,20 @@ public enum Version {
         FROM_100_TO_23
     }
 
+    // Temporarily moved from StreamsConfig to here to hide the built-in metrics version config
+    /**
+     * Config value for parameter {@link #BUILT_IN_METRICS_VERSION_CONFIG "built.in.metrics.version"} for built-in metrics from version 0.10.0. to 2.3
+     */
+    public static final String METRICS_0100_TO_23 = "0.10.0-2.3";
+
+    /**
+     * Config value for parameter {@link #BUILT_IN_METRICS_VERSION_CONFIG "built.in.metrics.version"} for the latest built-in metrics version.
+     */
+    public static final String METRICS_LATEST = "latest";
+
+
     static class ImmutableMetricValue implements Gauge {
+
         private final T value;
 
         public ImmutableMetricValue(final T value) {
@@ -156,7 +168,7 @@ public StreamsMetricsImpl(final Metrics metrics, final String clientId, final St
     }
 
     private static Version parseBuiltInMetricsVersion(final String builtInMetricsVersion) {
-        if (builtInMetricsVersion.equals(StreamsConfig.METRICS_LATEST)) {
+        if (builtInMetricsVersion.equals(StreamsMetricsImpl.METRICS_LATEST)) {
             return Version.LATEST;
         } else {
             return Version.FROM_100_TO_23;
diff --git a/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java b/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
index ad607086f2c96..6ac866f0dc77b 100644
--- a/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
@@ -52,14 +52,11 @@
 import static org.apache.kafka.streams.StreamsConfig.consumerPrefix;
 import static org.apache.kafka.streams.StreamsConfig.producerPrefix;
 import static org.apache.kafka.test.StreamsTestUtils.getStreamsConfig;
-import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.hasItem;
-import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.core.IsEqual.equalTo;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -486,37 +483,6 @@ public void shouldThrowExceptionIfNotAtLeastOnceOrExactlyOnce() {
         new StreamsConfig(props);
     }
 
-    @Test
-    public void shouldAcceptBuiltInMetricsVersion0100To23() {
-        // don't use `StreamsConfig.METRICS_0100_TO_23` to actually do a useful test
-        props.put(StreamsConfig.BUILT_IN_METRICS_VERSION_CONFIG, "0.10.0-2.3");
-        new StreamsConfig(props);
-    }
-
-    @Test
-    public void shouldAcceptBuiltInMetricsLatestVersion() {
-        // don't use `StreamsConfig.METRICS_LATEST` to actually do a useful test
-        props.put(StreamsConfig.BUILT_IN_METRICS_VERSION_CONFIG, "latest");
-        new StreamsConfig(props);
-    }
-
-    @Test
-    public void shouldSetDefaultBuiltInMetricsVersionIfNoneIsSpecified() {
-        final StreamsConfig config = new StreamsConfig(props);
-        assertThat(config.getString(StreamsConfig.BUILT_IN_METRICS_VERSION_CONFIG), is(StreamsConfig.METRICS_LATEST));
-    }
-
-    @Test
-    public void shouldThrowIfBuiltInMetricsVersionInvalid() {
-        final String invalidVersion = "0.0.1";
-        props.put(StreamsConfig.BUILT_IN_METRICS_VERSION_CONFIG, invalidVersion);
-        final Exception exception = assertThrows(ConfigException.class, () -> new StreamsConfig(props));
-        assertThat(
-            exception.getMessage(),
-            containsString("Invalid value " + invalidVersion + " for configuration built.in.metrics.version")
-        );
-    }
-
     @Test
     public void shouldResetToDefaultIfConsumerIsolationLevelIsOverriddenIfEosEnabled() {
         props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE);
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
index 6030ac67f6778..010f277555846 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
@@ -37,6 +37,7 @@
 import org.apache.kafka.streams.kstream.Produced;
 import org.apache.kafka.streams.kstream.SessionWindows;
 import org.apache.kafka.streams.kstream.TimeWindows;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.apache.kafka.streams.state.SessionStore;
 import org.apache.kafka.streams.state.Stores;
 import org.apache.kafka.streams.state.WindowStore;
@@ -291,18 +292,12 @@ private void closeApplication() throws Exception {
             () -> "Kafka Streams application did not reach state NOT_RUNNING in " + timeout + " ms");
     }
 
-    @Test
-    public void shouldAddMetricsOnAllLevelsWithBuiltInMetricsLatestVersion() throws Exception {
-        shouldAddMetricsOnAllLevels(StreamsConfig.METRICS_LATEST);
-    }
-
     @Test
     public void shouldAddMetricsOnAllLevelsWithBuiltInMetricsVersion0100To23() throws Exception {
-        shouldAddMetricsOnAllLevels(StreamsConfig.METRICS_0100_TO_23);
+        shouldAddMetricsOnAllLevels(StreamsMetricsImpl.METRICS_0100_TO_23);
     }
 
     private void shouldAddMetricsOnAllLevels(final String builtInMetricsVersion) throws Exception {
-        streamsConfiguration.put(StreamsConfig.BUILT_IN_METRICS_VERSION_CONFIG, builtInMetricsVersion);
 
         builder.stream(STREAM_INPUT, Consumed.with(Serdes.Integer(), Serdes.String()))
             .to(STREAM_OUTPUT_1, Produced.with(Serdes.Integer(), Serdes.String()));
@@ -593,18 +588,18 @@ private void checkCacheMetrics(final String builtInMetricsVersion) {
             .collect(Collectors.toList());
         checkMetricByName(
             listMetricCache,
-            builtInMetricsVersion.equals(StreamsConfig.METRICS_LATEST) ? HIT_RATIO_AVG : HIT_RATIO_AVG_BEFORE_24,
-            builtInMetricsVersion.equals(StreamsConfig.METRICS_LATEST) ? 3 : 6 /* includes parent sensors */
+            builtInMetricsVersion.equals(StreamsMetricsImpl.METRICS_LATEST) ? HIT_RATIO_AVG : HIT_RATIO_AVG_BEFORE_24,
+            builtInMetricsVersion.equals(StreamsMetricsImpl.METRICS_LATEST) ? 3 : 6 /* includes parent sensors */
         );
         checkMetricByName(
             listMetricCache,
-            builtInMetricsVersion.equals(StreamsConfig.METRICS_LATEST) ? HIT_RATIO_MIN : HIT_RATIO_MIN_BEFORE_24,
-            builtInMetricsVersion.equals(StreamsConfig.METRICS_LATEST) ? 3 : 6 /* includes parent sensors */
+            builtInMetricsVersion.equals(StreamsMetricsImpl.METRICS_LATEST) ? HIT_RATIO_MIN : HIT_RATIO_MIN_BEFORE_24,
+            builtInMetricsVersion.equals(StreamsMetricsImpl.METRICS_LATEST) ? 3 : 6 /* includes parent sensors */
         );
         checkMetricByName(
             listMetricCache,
-            builtInMetricsVersion.equals(StreamsConfig.METRICS_LATEST) ? HIT_RATIO_MAX : HIT_RATIO_MAX_BEFORE_24,
-            builtInMetricsVersion.equals(StreamsConfig.METRICS_LATEST) ? 3 : 6 /* includes parent sensors */
+            builtInMetricsVersion.equals(StreamsMetricsImpl.METRICS_LATEST) ? HIT_RATIO_MAX : HIT_RATIO_MAX_BEFORE_24,
+            builtInMetricsVersion.equals(StreamsMetricsImpl.METRICS_LATEST) ? 3 : 6 /* includes parent sensors */
         );
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStreamThreadTest.java
index ddb20c951ca47..aa9950569a129 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStreamThreadTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStreamThreadTest.java
@@ -108,7 +108,7 @@ public String newStoreName(final String prefix) {
             mockConsumer,
             new StateDirectory(config, time, true),
             0,
-            new StreamsMetricsImpl(new Metrics(), "test-client", StreamsConfig.METRICS_LATEST),
+            new StreamsMetricsImpl(new Metrics(), "test-client", StreamsMetricsImpl.METRICS_LATEST),
             new MockTime(),
             "clientId",
             stateRestoreListener
@@ -143,7 +143,7 @@ public List partitionsFor(final String topic) {
             mockConsumer,
             new StateDirectory(config, time, true),
             0,
-            new StreamsMetricsImpl(new Metrics(), "test-client", StreamsConfig.METRICS_LATEST),
+            new StreamsMetricsImpl(new Metrics(), "test-client", StreamsMetricsImpl.METRICS_LATEST),
             new MockTime(),
             "clientId",
             stateRestoreListener
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/MockStreamsMetrics.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/MockStreamsMetrics.java
index 273f4b9e45ada..3398ba178408a 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/MockStreamsMetrics.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/MockStreamsMetrics.java
@@ -17,12 +17,11 @@
 package org.apache.kafka.streams.processor.internals;
 
 import org.apache.kafka.common.metrics.Metrics;
-import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 
 public class MockStreamsMetrics extends StreamsMetricsImpl {
 
     public MockStreamsMetrics(final Metrics metrics) {
-        super(metrics, "test", StreamsConfig.METRICS_LATEST);
+        super(metrics, "test", StreamsMetricsImpl.METRICS_LATEST);
     }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
index c2b9cdb0b07b7..6f4483ae2ec62 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
@@ -160,9 +160,8 @@ private StreamsConfig createConfig(final File baseDir) throws IOException {
     private final byte[] recordValue = intSerializer.serialize(null, 10);
     private final byte[] recordKey = intSerializer.serialize(null, 1);
 
-    private final String threadName = "threadName";
     private final StreamsMetricsImpl streamsMetrics =
-        new StreamsMetricsImpl(new Metrics(), threadName, StreamsConfig.METRICS_LATEST);
+        new StreamsMetricsImpl(new Metrics(), applicationId, StreamsMetricsImpl.METRICS_LATEST);
 
     @Before
     public void setup() throws Exception {
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
index 8d10e4cf2c3aa..73cf7684bd558 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
@@ -121,7 +121,7 @@ public class StreamThreadTest {
     private final MockTime mockTime = new MockTime();
     private final Metrics metrics = new Metrics();
     private final StreamsMetricsImpl streamsMetrics =
-        new StreamsMetricsImpl(metrics, APPLICATION_ID, StreamsConfig.METRICS_LATEST);
+        new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsMetricsImpl.METRICS_LATEST);
     private final MockClientSupplier clientSupplier = new MockClientSupplier();
     private final InternalStreamsBuilder internalStreamsBuilder = new InternalStreamsBuilder(new InternalTopologyBuilder());
     private final StreamsConfig config = new StreamsConfig(configProps(false));
@@ -345,7 +345,6 @@ public void shouldNotCommitBeforeTheCommitInterval() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
         final TaskManager taskManager = mockTaskManagerCommit(consumer, 1, 1);
 
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -473,7 +472,6 @@ public void shouldNotCauseExceptionIfNothingCommitted() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
         final TaskManager taskManager = mockTaskManagerCommit(consumer, 1, 0);
 
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -508,7 +506,6 @@ public void shouldCommitAfterTheCommitInterval() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
         final TaskManager taskManager = mockTaskManagerCommit(consumer, 2, 1);
 
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -662,8 +659,6 @@ public void shouldShutdownTaskManagerOnClose() {
         EasyMock.expectLastCall();
         EasyMock.replay(taskManager, consumer);
 
-        final StreamsMetricsImpl streamsMetrics =
-            new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -696,8 +691,6 @@ public void shouldShutdownTaskManagerOnCloseWithoutStart() {
         EasyMock.expectLastCall();
         EasyMock.replay(taskManager, consumer);
 
-        final StreamsMetricsImpl streamsMetrics =
-            new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -774,8 +767,6 @@ private void setStreamThread(final StreamThread streamThread) {
         taskManager.setAssignmentMetadata(Collections.emptyMap(), Collections.emptyMap());
         taskManager.setPartitionsToTaskId(Collections.emptyMap());
 
-        final StreamsMetricsImpl streamsMetrics =
-            new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -809,8 +800,6 @@ public void shouldOnlyShutdownOnce() {
         EasyMock.expectLastCall();
         EasyMock.replay(taskManager, consumer);
 
-        final StreamsMetricsImpl streamsMetrics =
-            new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -1231,8 +1220,6 @@ private void setupInternalTopologyWithoutState() {
     private StandbyTask createStandbyTask() {
         final LogContext logContext = new LogContext("test");
         final Logger log = logContext.logger(StreamThreadTest.class);
-        final StreamsMetricsImpl streamsMetrics =
-            new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST);
         final StreamThread.StandbyTaskCreator standbyTaskCreator = new StreamThread.StandbyTaskCreator(
             internalTopologyBuilder,
             config,
@@ -1677,8 +1664,6 @@ public void producerMetricsVerificationWithoutEOS() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
         final TaskManager taskManager = mockTaskManagerCommit(consumer, 1, 0);
 
-        final StreamsMetricsImpl streamsMetrics =
-            new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
@@ -1717,7 +1702,6 @@ public void adminClientMetricsVerification() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
         final TaskManager taskManager = EasyMock.createNiceMock(TaskManager.class);
 
-        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST);
         final StreamThread thread = new StreamThread(
             mockTime,
             config,
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java
index 69f7546d0e89e..d8cdfe4accc95 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImplTest.java
@@ -24,7 +24,6 @@
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.Sensor.RecordingLevel;
 import org.apache.kafka.common.utils.MockTime;
-import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.ImmutableMetricValue;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.Version;
 import org.easymock.EasyMock;
@@ -63,7 +62,7 @@ public class StreamsMetricsImplTest extends EasyMockSupport {
     private final static String SENSOR_PREFIX_DELIMITER = ".";
     private final static String SENSOR_NAME_DELIMITER = ".s.";
     private final static String INTERNAL_PREFIX = "internal";
-    private final static String VERSION = StreamsConfig.METRICS_LATEST;
+    private final static String VERSION = StreamsMetricsImpl.METRICS_LATEST;
     private final static String CLIENT_ID = "test-client";
     private final static String THREAD_ID = "test-thread";
     private final static String TASK_ID = "test-task";
@@ -439,17 +438,16 @@ public void shouldGetStoreLevelTagMap() {
 
     @Test
     public void shouldGetCacheLevelTagMapForBuiltInMetricsLatestVersion() {
-        shouldGetCacheLevelTagMap(StreamsConfig.METRICS_LATEST);
+        shouldGetCacheLevelTagMap(StreamsMetricsImpl.METRICS_LATEST);
     }
 
     @Test
     public void shouldGetCacheLevelTagMapForBuiltInMetricsVersion0100To23() {
-        shouldGetCacheLevelTagMap(StreamsConfig.METRICS_0100_TO_23);
+        shouldGetCacheLevelTagMap(StreamsMetricsImpl.METRICS_0100_TO_23);
     }
 
     private void shouldGetCacheLevelTagMap(final String builtInMetricsVersion) {
-        final StreamsMetricsImpl streamsMetrics =
-            new StreamsMetricsImpl(metrics, THREAD_ID, builtInMetricsVersion);
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, builtInMetricsVersion);
         final String taskName = "taskName";
         final String storeName = "storeName";
 
@@ -458,7 +456,7 @@ private void shouldGetCacheLevelTagMap(final String builtInMetricsVersion) {
         assertThat(tagMap.size(), equalTo(3));
         assertThat(
             tagMap.get(
-                builtInMetricsVersion.equals(StreamsConfig.METRICS_LATEST) ? StreamsMetricsImpl.THREAD_ID_TAG
+                builtInMetricsVersion.equals(StreamsMetricsImpl.METRICS_LATEST) ? StreamsMetricsImpl.THREAD_ID_TAG
                     : StreamsMetricsImpl.THREAD_ID_TAG_0100_TO_23),
             equalTo(THREAD_ID)
         );
@@ -551,7 +549,7 @@ public void shouldAddAvgAndMinAndMaxMetricsToSensor() {
     @Test
     public void shouldReturnMetricsVersionCurrent() {
         assertThat(
-            new StreamsMetricsImpl(metrics, THREAD_ID, StreamsConfig.METRICS_LATEST).version(),
+            new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsMetricsImpl.METRICS_LATEST).version(),
             equalTo(Version.LATEST)
         );
     }
@@ -559,7 +557,7 @@ public void shouldReturnMetricsVersionCurrent() {
     @Test
     public void shouldReturnMetricsVersionFrom100To23() {
         assertThat(
-            new StreamsMetricsImpl(metrics, THREAD_ID, StreamsConfig.METRICS_0100_TO_23).version(),
+            new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsMetricsImpl.METRICS_0100_TO_23).version(),
             equalTo(Version.FROM_100_TO_23)
         );
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/GlobalStateStoreProviderTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/GlobalStateStoreProviderTest.java
index 1ab8684e9052b..db9205ec505ee 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/GlobalStateStoreProviderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/GlobalStateStoreProviderTest.java
@@ -18,7 +18,6 @@
 
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.serialization.Serdes;
-import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.errors.InvalidStateStoreException;
 import org.apache.kafka.streams.processor.StateStore;
 import org.apache.kafka.streams.processor.TaskId;
@@ -92,7 +91,7 @@ public void before() {
         final ProcessorContextImpl mockContext = mock(ProcessorContextImpl.class);
         expect(mockContext.applicationId()).andReturn("appId").anyTimes();
         expect(mockContext.metrics())
-            .andReturn(new StreamsMetricsImpl(new Metrics(), "threadName", StreamsConfig.METRICS_LATEST))
+            .andReturn(new StreamsMetricsImpl(new Metrics(), "test-client", StreamsMetricsImpl.METRICS_LATEST))
             .anyTimes();
         expect(mockContext.taskId()).andReturn(new TaskId(0, 0)).anyTimes();
         expect(mockContext.recordCollector()).andReturn(null).anyTimes();
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedWindowStoreTest.java
index 53096d2bf13c7..67f13585a6558 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedWindowStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedWindowStoreTest.java
@@ -60,7 +60,7 @@ public class MeteredTimestampedWindowStoreTest {
     @Before
     public void setUp() {
         final StreamsMetricsImpl streamsMetrics =
-            new StreamsMetricsImpl(metrics, "test", StreamsConfig.METRICS_LATEST);
+            new StreamsMetricsImpl(metrics, "test", StreamsMetricsImpl.METRICS_LATEST);
 
         context = new InternalMockProcessorContext(
             TestUtils.tempDirectory(),
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java
index 588307ff37b23..6e65f0e70cfab 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java
@@ -76,7 +76,7 @@ public class MeteredWindowStoreTest {
     @Before
     public void setUp() {
         final StreamsMetricsImpl streamsMetrics =
-            new StreamsMetricsImpl(metrics, "test", StreamsConfig.METRICS_LATEST);
+            new StreamsMetricsImpl(metrics, "test", StreamsMetricsImpl.METRICS_LATEST);
 
         context = new InternalMockProcessorContext(
             TestUtils.tempDirectory(),
diff --git a/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java b/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java
index f5cb014550a99..526b855e58abd 100644
--- a/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java
+++ b/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java
@@ -70,7 +70,7 @@ public InternalMockProcessorContext() {
         this(null,
             null,
             null,
-            new StreamsMetricsImpl(new Metrics(), "mock", StreamsConfig.METRICS_LATEST),
+            new StreamsMetricsImpl(new Metrics(), "mock", StreamsMetricsImpl.METRICS_LATEST),
             new StreamsConfig(StreamsTestUtils.getStreamsConfig()),
             null,
             null
@@ -83,7 +83,7 @@ public InternalMockProcessorContext(final File stateDir,
             stateDir,
             null,
             null,
-            new StreamsMetricsImpl(new Metrics(), "mock", StreamsConfig.METRICS_LATEST),
+            new StreamsMetricsImpl(new Metrics(), "mock", StreamsMetricsImpl.METRICS_LATEST),
             config,
             null,
             null
@@ -98,7 +98,7 @@ public InternalMockProcessorContext(final File stateDir,
             stateDir,
             keySerde,
             valSerde,
-            new StreamsMetricsImpl(new Metrics(), "mock", StreamsConfig.METRICS_LATEST),
+            new StreamsMetricsImpl(new Metrics(), "mock", StreamsMetricsImpl.METRICS_LATEST),
             config,
             null,
             null
@@ -117,7 +117,7 @@ public InternalMockProcessorContext(final StateSerdes serdes,
             null,
             serdes.keySerde(),
             serdes.valueSerde(),
-            new StreamsMetricsImpl(metrics, "mock", StreamsConfig.METRICS_LATEST),
+            new StreamsMetricsImpl(metrics, "mock", StreamsMetricsImpl.METRICS_LATEST),
             new StreamsConfig(StreamsTestUtils.getStreamsConfig()),
             () -> collector,
             null
@@ -132,7 +132,7 @@ public InternalMockProcessorContext(final File stateDir,
         this(stateDir,
             keySerde,
             valSerde,
-            new StreamsMetricsImpl(new Metrics(), "mock", StreamsConfig.METRICS_LATEST),
+            new StreamsMetricsImpl(new Metrics(), "mock", StreamsMetricsImpl.METRICS_LATEST),
             new StreamsConfig(StreamsTestUtils.getStreamsConfig()),
             () -> collector,
             cache
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
index 5abe89cdb8041..9a6de246e020f 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
@@ -306,7 +306,7 @@ public List partitionsFor(final String topic) {
         final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(
             metrics,
             "test-client",
-            StreamsConfig.METRICS_LATEST
+            StreamsMetricsImpl.METRICS_LATEST
         );
         streamsMetrics.setRocksDBMetricsRecordingTrigger(new RocksDBMetricsRecordingTrigger());
         final Sensor skippedRecordsSensor =
@@ -1086,4 +1086,4 @@ public synchronized long position(final TopicPartition partition) {
         }
         return consumer;
     }
-}
\ No newline at end of file
+}
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java
index 09850e4ed5a42..947e65e9c0c5f 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java
@@ -214,7 +214,8 @@ public MockProcessorContext(final Properties config, final TaskId taskId, final
         final MetricConfig metricConfig = new MetricConfig();
         metricConfig.recordLevel(Sensor.RecordingLevel.DEBUG);
         final String threadId = Thread.currentThread().getName();
-        this.metrics = new StreamsMetricsImpl(new Metrics(metricConfig), threadId, StreamsConfig.METRICS_LATEST);
+        this.metrics =
+            new StreamsMetricsImpl(new Metrics(metricConfig), "test-client", StreamsMetricsImpl.METRICS_LATEST);
         ThreadMetrics.skipRecordSensor(threadId, metrics);
     }
 

From bd9a1772d5364bc0f07c3d8e018e38029070dea9 Mon Sep 17 00:00:00 2001
From: Omkar Mestry <30625612+omanges@users.noreply.github.com>
Date: Tue, 8 Oct 2019 03:20:46 +0530
Subject: [PATCH 0699/1071] KAFKA-7245: Deprecate WindowStore#put(key, value)
 (#7105)

Implements KIP-474.

Reviewers: A. Sophie Blee-Goldman , Matthias J. Sax 
---
 .../internals/ProcessorContextImpl.java          |  2 ++
 .../apache/kafka/streams/state/WindowStore.java  |  6 ++++++
 .../state/internals/CachingWindowStore.java      |  1 +
 .../internals/ChangeLoggingWindowBytesStore.java |  1 +
 .../state/internals/InMemoryWindowStore.java     |  1 +
 .../state/internals/MeteredWindowStore.java      |  1 +
 .../state/internals/RocksDBWindowStore.java      |  1 +
 .../internals/TimestampedWindowStoreBuilder.java |  1 +
 ...indowToTimestampedWindowByteStoreAdapter.java |  1 +
 .../kafka/streams/perf/SimpleBenchmark.java      |  2 +-
 .../internals/ProcessorContextImplTest.java      |  9 +++++++--
 .../state/internals/CachingWindowStoreTest.java  | 16 +++++++++++++++-
 ...geLoggingTimestampedWindowBytesStoreTest.java |  2 ++
 .../ChangeLoggingWindowBytesStoreTest.java       |  2 ++
 .../state/internals/InMemoryWindowStoreTest.java |  1 +
 .../MeteredTimestampedWindowStoreTest.java       |  2 ++
 .../state/internals/MeteredWindowStoreTest.java  |  1 +
 .../state/internals/RocksDBWindowStoreTest.java  |  4 ++++
 .../state/internals/WindowBytesStoreTest.java    |  8 ++++++++
 .../streams/internals/WindowStoreFacade.java     |  1 +
 .../streams/internals/WindowStoreFacadeTest.java |  1 +
 21 files changed, 60 insertions(+), 4 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextImpl.java
index cd2b17997f61c..dc8f85a5ed8de 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextImpl.java
@@ -321,6 +321,7 @@ private WindowStoreReadOnlyDecorator(final WindowStore inner) {
             super(inner);
         }
 
+        @Deprecated
         @Override
         public void put(final K key,
                         final V value) {
@@ -520,6 +521,7 @@ static class WindowStoreReadWriteDecorator
             super(inner);
         }
 
+        @Deprecated
         @Override
         public void put(final K key,
                         final V value) {
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/WindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/WindowStore.java
index 83a0ee1d2836e..f5e69bfd04b67 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/WindowStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/WindowStore.java
@@ -46,7 +46,13 @@ public interface WindowStore extends StateStore, ReadOnlyWindowStore
      * @param value The value to update, it can be null;
      *              if the serialized bytes are also null it is interpreted as deletes
      * @throws NullPointerException if the given key is {@code null}
+     *
+     * @deprecated as timestamp is not provided for the key-value pair, this causes inconsistency
+     * to identify the window frame to which the key belongs.
+     * Use {@link #put(Object, Object, long)} instead.
+     *
      */
+    @Deprecated
     void put(K key, V value);
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java
index b64c0eb859de3..4bb8116b80f26 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java
@@ -131,6 +131,7 @@ public boolean setFlushListener(final CacheFlushListener flushLi
         return true;
     }
 
+    @Deprecated
     @Override
     public synchronized void put(final Bytes key,
                                  final byte[] value) {
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStore.java
index c58e9f09e59f2..8a9b91a62ae60 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStore.java
@@ -94,6 +94,7 @@ public KeyValueIterator, byte[]> fetchAll(final long timeFrom,
         return wrapped().fetchAll(timeFrom, timeTo);
     }
 
+    @Deprecated
     @Override
     public void put(final Bytes key, final byte[] value) {
         // Note: It's incorrect to bypass the wrapped store here by delegating to another method,
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryWindowStore.java
index 49dc566f0281d..43877ae792266 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryWindowStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryWindowStore.java
@@ -115,6 +115,7 @@ public void init(final ProcessorContext context, final StateStore root) {
         open = true;
     }
 
+    @Deprecated
     @Override
     public void put(final Bytes key, final byte[] value) {
         put(key, value, context.timestamp());
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java
index 149f9f807819b..b84129608ee4b 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java
@@ -170,6 +170,7 @@ public boolean setFlushListener(final CacheFlushListener, V> listene
         return false;
     }
 
+    @Deprecated
     @Override
     public void put(final K key,
                     final V value) {
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStore.java
index 49eefa14db2bc..c45fe5363d3bd 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStore.java
@@ -48,6 +48,7 @@ public void init(final ProcessorContext context, final StateStore root) {
         super.init(context, root);
     }
 
+    @Deprecated
     @Override
     public void put(final Bytes key, final byte[] value) {
         put(key, value, context.timestamp());
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/TimestampedWindowStoreBuilder.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/TimestampedWindowStoreBuilder.java
index 808d31e6a47ed..d5459752ac326 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/TimestampedWindowStoreBuilder.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/TimestampedWindowStoreBuilder.java
@@ -105,6 +105,7 @@ public void init(final ProcessorContext context,
             wrapped.init(context, root);
         }
 
+        @Deprecated
         @Override
         public void put(final Bytes key,
                         final byte[] value) {
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowToTimestampedWindowByteStoreAdapter.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowToTimestampedWindowByteStoreAdapter.java
index 7bd8665bd351f..7bf8a0c5872d3 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowToTimestampedWindowByteStoreAdapter.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowToTimestampedWindowByteStoreAdapter.java
@@ -39,6 +39,7 @@ class WindowToTimestampedWindowByteStoreAdapter implements WindowStore>) store -> {
             verifyStoreCannotBeInitializedOrClosed(store);
@@ -194,7 +195,9 @@ public void globalWindowStoreShouldBeReadOnly() {
         });
     }
 
+
     @Test
+    @SuppressWarnings("deprecation")
     public void globalTimestampedWindowStoreShouldBeReadOnly() {
         doTest("GlobalTimestampedWindowStore", (Consumer>) store -> {
             verifyStoreCannotBeInitializedOrClosed(store);
@@ -282,6 +285,7 @@ public void localTimestampedKeyValueStoreShouldNotAllowInitOrClose() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void localWindowStoreShouldNotAllowInitOrClose() {
         doTest("LocalWindowStore", (Consumer>) store -> {
             verifyStoreCannotBeInitializedOrClosed(store);
@@ -301,6 +305,7 @@ public void localWindowStoreShouldNotAllowInitOrClose() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void localTimestampedWindowStoreShouldNotAllowInitOrClose() {
         doTest("LocalTimestampedWindowStore", (Consumer>) store -> {
             verifyStoreCannotBeInitializedOrClosed(store);
@@ -427,7 +432,7 @@ private TimestampedKeyValueStore timestampedKeyValueStoreMock() {
         return timestampedKeyValueStoreMock;
     }
 
-    @SuppressWarnings("unchecked")
+    @SuppressWarnings({"unchecked", "deprecation"})
     private WindowStore windowStoreMock() {
         final WindowStore windowStore = mock(WindowStore.class);
 
@@ -450,7 +455,7 @@ private WindowStore windowStoreMock() {
         return windowStore;
     }
 
-    @SuppressWarnings("unchecked")
+    @SuppressWarnings({"unchecked", "deprecation"})
     private TimestampedWindowStore timestampedWindowStoreMock() {
         final TimestampedWindowStore windowStore = mock(TimestampedWindowStore.class);
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java
index b9db53f8e4775..64ab25a6c654e 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java
@@ -128,10 +128,12 @@ public void shouldNotReturnDuplicatesInRanges() {
             .transform(() -> new Transformer>() {
                 private WindowStore store;
                 private int numRecordsProcessed;
+                private ProcessorContext context;
 
                 @SuppressWarnings("unchecked")
                 @Override
                 public void init(final ProcessorContext processorContext) {
+                    this.context = processorContext;
                     this.store = (WindowStore) processorContext.getStateStore("store-name");
                     int count = 0;
 
@@ -155,7 +157,7 @@ public KeyValue transform(final String key, final String value)
                     }
                     assertThat(count, equalTo(numRecordsProcessed));
 
-                    store.put(value, value);
+                    store.put(value, value, context.timestamp());
 
                     numRecordsProcessed++;
 
@@ -206,6 +208,7 @@ public void close() {}
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void shouldPutFetchFromCache() {
         cachingStore.put(bytesKey("a"), bytesValue("a"));
         cachingStore.put(bytesKey("b"), bytesValue("b"));
@@ -244,6 +247,7 @@ private String stringFrom(final byte[] from) {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void shouldPutFetchRangeFromCache() {
         cachingStore.put(bytesKey("a"), bytesValue("a"));
         cachingStore.put(bytesKey("b"), bytesValue("b"));
@@ -263,6 +267,7 @@ public void shouldPutFetchRangeFromCache() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void shouldGetAllFromCache() {
         cachingStore.put(bytesKey("a"), bytesValue("a"));
         cachingStore.put(bytesKey("b"), bytesValue("b"));
@@ -285,6 +290,7 @@ public void shouldGetAllFromCache() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void shouldFetchAllWithinTimestampRange() {
         final String[] array = {"a", "b", "c", "d", "e", "f", "g", "h"};
         for (int i = 0; i < array.length; i++) {
@@ -342,6 +348,7 @@ public void shouldFlushEvictedItemsIntoUnderlyingStore() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void shouldForwardDirtyItemsWhenFlushCalled() {
         final Windowed windowedKey =
             new Windowed<>("1", new TimeWindow(DEFAULT_TIMESTAMP, DEFAULT_TIMESTAMP + WINDOW_SIZE));
@@ -358,6 +365,7 @@ public void shouldSetFlushListener() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void shouldForwardOldValuesWhenEnabled() {
         cachingStore.setFlushListener(cacheListener, true);
         final Windowed windowedKey =
@@ -386,6 +394,7 @@ public void shouldForwardOldValuesWhenEnabled() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void shouldForwardOldValuesWhenDisabled() {
         final Windowed windowedKey =
             new Windowed<>("1", new TimeWindow(DEFAULT_TIMESTAMP, DEFAULT_TIMESTAMP + WINDOW_SIZE));
@@ -473,6 +482,7 @@ public void shouldIterateCacheAndStoreKeyRange() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void shouldClearNamespaceCacheOnClose() {
         cachingStore.put(bytesKey("a"), bytesValue("a"));
         assertEquals(1, cache.size());
@@ -493,6 +503,7 @@ public void shouldThrowIfTryingToFetchRangeFromClosedCachingStore() {
     }
 
     @Test(expected = InvalidStateStoreException.class)
+    @SuppressWarnings("deprecation")
     public void shouldThrowIfTryingToWriteToClosedCachingStore() {
         cachingStore.close();
         cachingStore.put(bytesKey("a"), bytesValue("a"));
@@ -569,11 +580,13 @@ public void shouldReturnSameResultsForSingleKeyFetchAndEqualKeyRangeFetch() {
     }
 
     @Test(expected = NullPointerException.class)
+    @SuppressWarnings("deprecation")
     public void shouldThrowNullPointerExceptionOnPutNullKey() {
         cachingStore.put(null, bytesValue("anyValue"));
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void shouldNotThrowNullPointerExceptionOnPutNullValue() {
         cachingStore.put(bytesKey("a"), null);
     }
@@ -616,6 +629,7 @@ private static KeyValue, byte[]> windowedPair(final String key,
             bytesValue(value));
     }
 
+    @SuppressWarnings("deprecation")
     private int addItemsToCache() {
         int cachedSize = 0;
         int i = 0;
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingTimestampedWindowBytesStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingTimestampedWindowBytesStoreTest.java
index edf210eaea6e1..957a616b75979 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingTimestampedWindowBytesStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingTimestampedWindowBytesStoreTest.java
@@ -86,6 +86,7 @@ private void init() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void shouldLogPuts() {
         inner.put(bytesKey, valueAndTimestamp, 0);
         EasyMock.expectLastCall();
@@ -128,6 +129,7 @@ public void shouldDelegateToUnderlyingStoreWhenFetchingRange() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void shouldRetainDuplicatesWhenSet() {
         store = new ChangeLoggingTimestampedWindowBytesStore(inner, true);
         inner.put(bytesKey, valueAndTimestamp, 0);
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStoreTest.java
index d7ad6d22d6971..4503ad12b3c4c 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStoreTest.java
@@ -83,6 +83,7 @@ private void init() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void shouldLogPuts() {
         inner.put(bytesKey, value, 0);
         EasyMock.expectLastCall();
@@ -122,6 +123,7 @@ public void shouldDelegateToUnderlyingStoreWhenFetchingRange() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void shouldRetainDuplicatesWhenSet() {
         store = new ChangeLoggingWindowBytesStore(inner, true);
         inner.put(bytesKey, value, 0);
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryWindowStoreTest.java
index 41ed07317e01e..9d5fc8c2999b2 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryWindowStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryWindowStoreTest.java
@@ -125,6 +125,7 @@ public void shouldNotExpireFromOpenIterator() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void testExpiration() {
 
         long currentTime = 0;
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedWindowStoreTest.java
index 67f13585a6558..2000709d0fab9 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedWindowStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedWindowStoreTest.java
@@ -94,6 +94,7 @@ public void shouldNotExceptionIfFetchReturnsNull() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void shouldNotThrowExceptionIfSerdesCorrectlySetFromProcessorContext() {
         EasyMock.expect(innerStoreMock.name()).andStubReturn("mocked-store");
         EasyMock.replay(innerStoreMock);
@@ -118,6 +119,7 @@ public void shouldNotThrowExceptionIfSerdesCorrectlySetFromProcessorContext() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void shouldNotThrowExceptionIfSerdesCorrectlySetFromConstructorParameters() {
         EasyMock.expect(innerStoreMock.name()).andStubReturn("mocked-store");
         EasyMock.replay(innerStoreMock);
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java
index 6e65f0e70cfab..1e6b7d2d5a44c 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java
@@ -114,6 +114,7 @@ public void shouldRecordRestoreLatencyOnInit() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void shouldRecordPutLatency() {
         final byte[] bytes = "a".getBytes();
         innerStoreMock.put(eq(Bytes.wrap(bytes)), anyObject(), eq(context.timestamp()));
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java
index 91f6aab726cd2..983bfa0a24c15 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java
@@ -77,6 +77,7 @@ void setClassLoggerToDebug() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void shouldOnlyIterateOpenSegments() {
         long currentTime = 0;
         setCurrentTime(currentTime);
@@ -104,6 +105,7 @@ public void shouldOnlyIterateOpenSegments() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void testRolling() {
 
         // to validate segments
@@ -379,6 +381,7 @@ public void testRolling() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void testSegmentMaintenance() {
 
         windowStore = buildWindowStore(RETENTION_PERIOD, WINDOW_SIZE, true, Serdes.Integer(),
@@ -505,6 +508,7 @@ public void testInitialLoading() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void testRestore() throws Exception {
         final long startTime = SEGMENT_INTERVAL * 2;
         final long increment = SEGMENT_INTERVAL / 2;
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/WindowBytesStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/WindowBytesStoreTest.java
index 489c8254b3c70..8faa68f74ef14 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/WindowBytesStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/WindowBytesStoreTest.java
@@ -663,6 +663,7 @@ public void testPutAndFetchAfter() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void testPutSameKeyTimestamp() {
         windowStore = buildWindowStore(RETENTION_PERIOD, WINDOW_SIZE, true, Serdes.Integer(), Serdes.String());
         windowStore.init(context, windowStore);
@@ -771,6 +772,7 @@ public void shouldFetchAndIterateOverExactKeys() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void testDeleteAndUpdate() {
 
         final long currentTime = 0;
@@ -792,6 +794,7 @@ public void shouldReturnNullOnWindowNotFound() {
     }
 
     @Test(expected = NullPointerException.class)
+    @SuppressWarnings("deprecation")
     public void shouldThrowNullPointerExceptionOnPutNullKey() {
         windowStore.put(null, "anyValue");
     }
@@ -933,6 +936,7 @@ public void shouldNotThrowExceptionWhenFetchRangeIsExpired() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void testWindowIteratorPeek() {
         final long currentTime = 0;
         setCurrentTime(currentTime);
@@ -963,6 +967,7 @@ public void testValueIteratorPeek() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void shouldNotThrowConcurrentModificationException() {
         long currentTime = 0;
         setCurrentTime(currentTime);
@@ -991,6 +996,7 @@ public void shouldNotThrowConcurrentModificationException() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void testFetchDuplicates() {
         windowStore = buildWindowStore(RETENTION_PERIOD, WINDOW_SIZE, true, Serdes.Integer(), Serdes.String());
         windowStore.init(context, windowStore);
@@ -1020,6 +1026,7 @@ public void testFetchDuplicates() {
     }
 
 
+    @SuppressWarnings("deprecation")
     private void putFirstBatch(final WindowStore store,
         @SuppressWarnings("SameParameterValue") final long startTime,
         final InternalMockProcessorContext context) {
@@ -1035,6 +1042,7 @@ private void putFirstBatch(final WindowStore store,
         store.put(5, "five");
     }
 
+    @SuppressWarnings("deprecation")
     private void putSecondBatch(final WindowStore store,
         @SuppressWarnings("SameParameterValue") final long startTime,
         final InternalMockProcessorContext context) {
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/internals/WindowStoreFacade.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/internals/WindowStoreFacade.java
index f6f8f336111a1..e49584f7026a9 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/internals/WindowStoreFacade.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/internals/WindowStoreFacade.java
@@ -36,6 +36,7 @@ public void init(final ProcessorContext context,
         inner.init(context, root);
     }
 
+    @Deprecated
     @Override
     public void put(final K key,
                     final V value) {
diff --git a/streams/test-utils/src/test/java/org/apache/kafka/streams/internals/WindowStoreFacadeTest.java b/streams/test-utils/src/test/java/org/apache/kafka/streams/internals/WindowStoreFacadeTest.java
index 972b308468809..3347ddd775d6b 100644
--- a/streams/test-utils/src/test/java/org/apache/kafka/streams/internals/WindowStoreFacadeTest.java
+++ b/streams/test-utils/src/test/java/org/apache/kafka/streams/internals/WindowStoreFacadeTest.java
@@ -60,6 +60,7 @@ public void shouldForwardInit() {
     }
 
     @Test
+    @SuppressWarnings("deprecation")
     public void shouldPutWithUnknownTimestamp() {
         mockedWindowTimestampStore.put("key", ValueAndTimestamp.make("value", ConsumerRecord.NO_TIMESTAMP));
         expectLastCall();

From 7f330725de8ad26f6e17d579f922a09aa003772a Mon Sep 17 00:00:00 2001
From: Jukka Karvanen <48978068+jukkakarvanen@users.noreply.github.com>
Date: Tue, 8 Oct 2019 05:32:40 +0300
Subject: [PATCH 0700/1071] MINOR: Modified Exception handling for KIP-470
 (#7461)

Reviewers: Bill Bejeck , Matthias J. Sax 
---
 .../apache/kafka/streams/TestOutputTopic.java |  2 +-
 .../kafka/streams/TopologyTestDriver.java     |  4 +++-
 .../apache/kafka/streams/TestTopicsTest.java  |  9 ++++++++
 .../kafka/streams/TopologyTestDriverTest.java | 22 +++++++++++++------
 4 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TestOutputTopic.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TestOutputTopic.java
index c3f690a18005e..da3629eaea740 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TestOutputTopic.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TestOutputTopic.java
@@ -131,7 +131,7 @@ public Map readKeyValuesToMap() {
         while (!isEmpty()) {
             outputRow = readRecord();
             if (outputRow.key() == null) {
-                throw new NullPointerException("Null keys not allowed");
+                throw new IllegalStateException("Null keys not allowed with readKeyValuesToMap method");
             }
             output.put(outputRow.key(), outputRow.value());
         }
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
index 9a6de246e020f..43324eec76cbb 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
@@ -728,11 +728,13 @@  void pipeRecord(final String topic,
                            final Instant time) {
         final byte[] serializedKey = keySerializer.serialize(topic, record.headers(), record.key());
         final byte[] serializedValue = valueSerializer.serialize(topic, record.headers(), record.value());
-        long timestamp = 0;
+        final long timestamp;
         if (time != null) {
             timestamp = time.toEpochMilli();
         } else if (record.timestamp() != null) {
             timestamp = record.timestamp();
+        } else {
+            throw new IllegalStateException("Provided `TestRecord` does not have a timestamp and no timestamp overwrite was provided via `time` parameter.");
         }
 
         pipeRecord(topic, timestamp, serializedKey, serializedValue, record.headers());
diff --git a/streams/test-utils/src/test/java/org/apache/kafka/streams/TestTopicsTest.java b/streams/test-utils/src/test/java/org/apache/kafka/streams/TestTopicsTest.java
index d8c78476415f4..381968851ba5f 100644
--- a/streams/test-utils/src/test/java/org/apache/kafka/streams/TestTopicsTest.java
+++ b/streams/test-utils/src/test/java/org/apache/kafka/streams/TestTopicsTest.java
@@ -164,6 +164,15 @@ public void testKeyValuesToMap() {
         assertThat(output, is(equalTo(expected)));
     }
 
+    @Test
+    public void testKeyValuesToMapWithNull() {
+        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
+        inputTopic.pipeInput("value");
+        assertThrows(IllegalStateException.class, () -> outputTopic.readKeyValuesToMap());
+    }
+
+
     @Test
     public void testKeyValueListDuration() {
         final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
diff --git a/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java b/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java
index 14cfbc602bf47..7e953920b0cb2 100644
--- a/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java
+++ b/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java
@@ -401,7 +401,15 @@ public void shouldThrowForUnknownTopic() {
 
         testDriver = new TopologyTestDriver(new Topology(), config);
         assertThrows(IllegalArgumentException.class, () -> {
-            pipeRecord(unknownTopic, new TestRecord(nullValue));
+            testDriver.pipeRecord(unknownTopic, new TestRecord(nullValue), new ByteArraySerializer(), new ByteArraySerializer(), Instant.now());
+        });
+    }
+
+    @Test
+    public void shouldThrowForMissingTime() {
+        testDriver = new TopologyTestDriver(new Topology(), config);
+        assertThrows(IllegalStateException.class, () -> {
+            testDriver.pipeRecord(SINK_TOPIC_1, new TestRecord("value"), new StringSerializer(), new StringSerializer(), null);
         });
     }
 
@@ -584,7 +592,7 @@ public void shouldUseSourceSpecificDeserializers() {
                 consumerRecord1,
                 Serdes.Long().serializer(),
                 Serdes.String().serializer(),
-                null);
+                Instant.now());
         final TestRecord result1 =
             testDriver.readRecord(SINK_TOPIC_1, Serdes.Long().deserializer(), Serdes.String().deserializer());
         assertThat(result1.getKey(), equalTo(source1Key));
@@ -594,7 +602,7 @@ public void shouldUseSourceSpecificDeserializers() {
                 consumerRecord2,
                 Serdes.Integer().serializer(),
                 Serdes.Double().serializer(),
-                null);
+                Instant.now());
         final TestRecord result2 =
             testDriver.readRecord(SINK_TOPIC_1, Serdes.Integer().deserializer(), Serdes.Double().deserializer());
         assertThat(result2.getKey(), equalTo(source2Key));
@@ -627,7 +635,7 @@ public void shouldUseSinkSpecificSerializers() {
                 consumerRecord1,
                 Serdes.Long().serializer(),
                 Serdes.String().serializer(),
-                null);
+                Instant.now());
         final TestRecord result1 =
                 testDriver.readRecord(SINK_TOPIC_1, Serdes.Long().deserializer(), Serdes.String().deserializer());
         assertThat(result1.getKey(), equalTo(source1Key));
@@ -637,7 +645,7 @@ public void shouldUseSinkSpecificSerializers() {
                 consumerRecord2,
                 Serdes.Integer().serializer(),
                 Serdes.Double().serializer(),
-                null);
+                Instant.now());
         final TestRecord result2 =
                 testDriver.readRecord(SINK_TOPIC_2, Serdes.Integer().deserializer(), Serdes.Double().deserializer());
         assertThat(result2.getKey(), equalTo(source2Key));
@@ -1345,7 +1353,7 @@ public void close() {}
         try (final TopologyTestDriver testDriver = new TopologyTestDriver(topology, config)) {
             assertNull(testDriver.getKeyValueStore("storeProcessorStore").get("a"));
             testDriver.pipeRecord("input-topic", new TestRecord("a", 1L),
-                    new StringSerializer(), new LongSerializer(), null);
+                    new StringSerializer(), new LongSerializer(), Instant.now());
             Assert.assertEquals(1L, testDriver.getKeyValueStore("storeProcessorStore").get("a"));
         }
 
@@ -1369,7 +1377,7 @@ public void shouldFeedStoreFromGlobalKTable() {
             final KeyValueStore globalStore = testDriver.getKeyValueStore("globalStore");
             Assert.assertNotNull(globalStore);
             Assert.assertNotNull(testDriver.getAllStateStores().get("globalStore"));
-            testDriver.pipeRecord("topic", new TestRecord("k1", "value1"), new StringSerializer(), new StringSerializer(), null);
+            testDriver.pipeRecord("topic", new TestRecord("k1", "value1"), new StringSerializer(), new StringSerializer(), Instant.now());
             // we expect to have both in the global store, the one from pipeInput and the one from the producer
             Assert.assertEquals("value1", globalStore.get("k1"));
         }

From c74b437d0081dbd9b3158ec10c9a54780d57e6a5 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Tue, 8 Oct 2019 08:19:07 -0700
Subject: [PATCH 0701/1071] KAFKA-8983; AdminClient deleteRecords should not
 fail all partitions unnecessarily (#7449)

The deleteRecords API in the AdminClient groups records to be sent by the partition leaders. If one of these requests fails, we currently fail all futures, including those tied to requests sent to other leaders. It would be better to fail only those partitions included in the failed request.

Reviewers: Ismael Juma 
---
 .../kafka/clients/admin/KafkaAdminClient.java | 73 +++++++++--------
 .../clients/admin/KafkaAdminClientTest.java   | 80 +++++++++++++++++--
 .../java/org/apache/kafka/test/TestUtils.java | 17 ++++
 3 files changed, 127 insertions(+), 43 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
index a7663da3789df..d4958f2738eaf 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
@@ -80,9 +80,9 @@
 import org.apache.kafka.common.message.ExpireDelegationTokenRequestData;
 import org.apache.kafka.common.message.FindCoordinatorRequestData;
 import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData;
-import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData.AlterableConfigCollection;
-import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData.AlterableConfig;
 import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData.AlterConfigsResource;
+import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData.AlterableConfig;
+import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData.AlterableConfigCollection;
 import org.apache.kafka.common.message.ListGroupsRequestData;
 import org.apache.kafka.common.message.ListGroupsResponseData;
 import org.apache.kafka.common.message.ListPartitionReassignmentsRequestData;
@@ -109,21 +109,21 @@
 import org.apache.kafka.common.requests.AlterReplicaLogDirsRequest;
 import org.apache.kafka.common.requests.AlterReplicaLogDirsResponse;
 import org.apache.kafka.common.requests.ApiError;
-import org.apache.kafka.common.requests.CreateAclsRequest.AclCreation;
 import org.apache.kafka.common.requests.CreateAclsRequest;
-import org.apache.kafka.common.requests.CreateAclsResponse.AclCreationResponse;
+import org.apache.kafka.common.requests.CreateAclsRequest.AclCreation;
 import org.apache.kafka.common.requests.CreateAclsResponse;
+import org.apache.kafka.common.requests.CreateAclsResponse.AclCreationResponse;
 import org.apache.kafka.common.requests.CreateDelegationTokenRequest;
 import org.apache.kafka.common.requests.CreateDelegationTokenResponse;
-import org.apache.kafka.common.requests.CreatePartitionsRequest.PartitionDetails;
 import org.apache.kafka.common.requests.CreatePartitionsRequest;
+import org.apache.kafka.common.requests.CreatePartitionsRequest.PartitionDetails;
 import org.apache.kafka.common.requests.CreatePartitionsResponse;
 import org.apache.kafka.common.requests.CreateTopicsRequest;
 import org.apache.kafka.common.requests.CreateTopicsResponse;
 import org.apache.kafka.common.requests.DeleteAclsRequest;
+import org.apache.kafka.common.requests.DeleteAclsResponse;
 import org.apache.kafka.common.requests.DeleteAclsResponse.AclDeletionResult;
 import org.apache.kafka.common.requests.DeleteAclsResponse.AclFilterResponse;
-import org.apache.kafka.common.requests.DeleteAclsResponse;
 import org.apache.kafka.common.requests.DeleteGroupsRequest;
 import org.apache.kafka.common.requests.DeleteGroupsResponse;
 import org.apache.kafka.common.requests.DeleteRecordsRequest;
@@ -144,8 +144,8 @@
 import org.apache.kafka.common.requests.ElectLeadersResponse;
 import org.apache.kafka.common.requests.ExpireDelegationTokenRequest;
 import org.apache.kafka.common.requests.ExpireDelegationTokenResponse;
-import org.apache.kafka.common.requests.FindCoordinatorRequest.CoordinatorType;
 import org.apache.kafka.common.requests.FindCoordinatorRequest;
+import org.apache.kafka.common.requests.FindCoordinatorRequest.CoordinatorType;
 import org.apache.kafka.common.requests.FindCoordinatorResponse;
 import org.apache.kafka.common.requests.IncrementalAlterConfigsRequest;
 import org.apache.kafka.common.requests.IncrementalAlterConfigsResponse;
@@ -158,7 +158,6 @@
 import org.apache.kafka.common.requests.MetadataRequest;
 import org.apache.kafka.common.requests.MetadataResponse;
 import org.apache.kafka.common.requests.OffsetDeleteRequest;
-import org.apache.kafka.common.requests.OffsetDeleteRequest.Builder;
 import org.apache.kafka.common.requests.OffsetDeleteResponse;
 import org.apache.kafka.common.requests.OffsetFetchRequest;
 import org.apache.kafka.common.requests.OffsetFetchResponse;
@@ -190,21 +189,22 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
-import java.util.TreeMap;
 import java.util.Set;
+import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData.ReassignablePartition;
-import static org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignableTopicResponse;
 import static org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignablePartitionResponse;
+import static org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignableTopicResponse;
 import static org.apache.kafka.common.message.ListPartitionReassignmentsRequestData.ListPartitionReassignmentsTopics;
-import static org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingTopicReassignment;
 import static org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingPartitionReassignment;
+import static org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingTopicReassignment;
 import static org.apache.kafka.common.requests.MetadataRequest.convertToMetadataRequestTopic;
 import static org.apache.kafka.common.utils.Utils.closeQuietly;
 
@@ -315,9 +315,18 @@ static  List getOrCreateListValue(Map> map, K key) {
      * @param        The KafkaFutureImpl result type.
      */
     private static  void completeAllExceptionally(Collection> futures, Throwable exc) {
-        for (KafkaFutureImpl future : futures) {
-            future.completeExceptionally(exc);
-        }
+        completeAllExceptionally(futures.stream(), exc);
+    }
+
+    /**
+     * Send an exception to all futures in the provided stream
+     *
+     * @param futures   The stream of KafkaFutureImpl objects.
+     * @param exc       The exception
+     * @param        The KafkaFutureImpl result type.
+     */
+    private static  void completeAllExceptionally(Stream> futures, Throwable exc) {
+        futures.forEach(future -> future.completeExceptionally(exc));
     }
 
     /**
@@ -2364,39 +2373,31 @@ void handleResponse(AbstractResponse abstractResponse) {
                 Map errors = response.errors();
                 Cluster cluster = response.cluster();
 
-                // completing futures for topics with errors
-                for (Map.Entry topicError: errors.entrySet()) {
-
-                    for (Map.Entry> future: futures.entrySet()) {
-                        if (future.getKey().topic().equals(topicError.getKey())) {
-                            future.getValue().completeExceptionally(topicError.getValue().exception());
-                        }
-                    }
-                }
-
-                // grouping topic partitions per leader
+                // Group topic partitions by leader
                 Map> leaders = new HashMap<>();
                 for (Map.Entry entry: recordsToDelete.entrySet()) {
+                    KafkaFutureImpl future = futures.get(entry.getKey());
 
-                    // avoiding to send deletion request for topics with errors
-                    if (!errors.containsKey(entry.getKey().topic())) {
-
+                    // Fail partitions with topic errors
+                    Errors topicError = errors.get(entry.getKey().topic());
+                    if (errors.containsKey(entry.getKey().topic())) {
+                        future.completeExceptionally(topicError.exception());
+                    } else {
                         Node node = cluster.leaderFor(entry.getKey());
                         if (node != null) {
                             if (!leaders.containsKey(node))
                                 leaders.put(node, new HashMap<>());
                             leaders.get(node).put(entry.getKey(), entry.getValue().beforeOffset());
                         } else {
-                            KafkaFutureImpl future = futures.get(entry.getKey());
                             future.completeExceptionally(Errors.LEADER_NOT_AVAILABLE.exception());
                         }
                     }
                 }
 
-                for (final Map.Entry> entry: leaders.entrySet()) {
-
-                    final long nowDelete = time.milliseconds();
+                final long deleteRecordsCallTimeMs = time.milliseconds();
 
+                for (final Map.Entry> entry : leaders.entrySet()) {
+                    final Map partitionDeleteOffsets = entry.getValue();
                     final int brokerId = entry.getKey().id();
 
                     runnable.call(new Call("deleteRecords", deadline,
@@ -2404,7 +2405,7 @@ void handleResponse(AbstractResponse abstractResponse) {
 
                         @Override
                         AbstractRequest.Builder createRequest(int timeoutMs) {
-                            return new DeleteRecordsRequest.Builder(timeoutMs, entry.getValue());
+                            return new DeleteRecordsRequest.Builder(timeoutMs, partitionDeleteOffsets);
                         }
 
                         @Override
@@ -2423,9 +2424,11 @@ void handleResponse(AbstractResponse abstractResponse) {
 
                         @Override
                         void handleFailure(Throwable throwable) {
-                            completeAllExceptionally(futures.values(), throwable);
+                            Stream> callFutures =
+                                    partitionDeleteOffsets.keySet().stream().map(futures::get);
+                            completeAllExceptionally(callFutures, throwable);
                         }
-                    }, nowDelete);
+                    }, deleteRecordsCallTimeMs);
                 }
             }
 
diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
index b6fece991d247..6d870fcd8b35d 100644
--- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
@@ -16,8 +16,6 @@
  */
 package org.apache.kafka.clients.admin;
 
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 import org.apache.kafka.clients.ClientDnsLookup;
 import org.apache.kafka.clients.ClientUtils;
 import org.apache.kafka.clients.MockClient;
@@ -52,13 +50,14 @@
 import org.apache.kafka.common.errors.SaslAuthenticationException;
 import org.apache.kafka.common.errors.SecurityDisabledException;
 import org.apache.kafka.common.errors.TimeoutException;
+import org.apache.kafka.common.errors.TopicAuthorizationException;
 import org.apache.kafka.common.errors.TopicDeletionDisabledException;
 import org.apache.kafka.common.errors.UnknownMemberIdException;
 import org.apache.kafka.common.errors.UnknownServerException;
 import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
 import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData;
-import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult;
 import org.apache.kafka.common.message.CreateTopicsResponseData;
+import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult;
 import org.apache.kafka.common.message.DeleteGroupsResponseData;
 import org.apache.kafka.common.message.DeleteGroupsResponseData.DeletableGroupResult;
 import org.apache.kafka.common.message.DeleteGroupsResponseData.DeletableGroupResultCollection;
@@ -68,8 +67,8 @@
 import org.apache.kafka.common.message.DescribeGroupsResponseData.DescribedGroupMember;
 import org.apache.kafka.common.message.ElectLeadersResponseData.PartitionResult;
 import org.apache.kafka.common.message.ElectLeadersResponseData.ReplicaElectionResult;
-import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData.AlterConfigsResourceResponse;
 import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData;
+import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData.AlterConfigsResourceResponse;
 import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity;
 import org.apache.kafka.common.message.LeaveGroupResponseData;
 import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse;
@@ -83,14 +82,14 @@
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.requests.AlterPartitionReassignmentsResponse;
 import org.apache.kafka.common.requests.ApiError;
-import org.apache.kafka.common.requests.CreateAclsResponse.AclCreationResponse;
 import org.apache.kafka.common.requests.CreateAclsResponse;
+import org.apache.kafka.common.requests.CreateAclsResponse.AclCreationResponse;
 import org.apache.kafka.common.requests.CreatePartitionsResponse;
 import org.apache.kafka.common.requests.CreateTopicsRequest;
 import org.apache.kafka.common.requests.CreateTopicsResponse;
+import org.apache.kafka.common.requests.DeleteAclsResponse;
 import org.apache.kafka.common.requests.DeleteAclsResponse.AclDeletionResult;
 import org.apache.kafka.common.requests.DeleteAclsResponse.AclFilterResponse;
-import org.apache.kafka.common.requests.DeleteAclsResponse;
 import org.apache.kafka.common.requests.DeleteGroupsResponse;
 import org.apache.kafka.common.requests.DeleteRecordsResponse;
 import org.apache.kafka.common.requests.DeleteTopicsRequest;
@@ -140,18 +139,20 @@
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static java.util.Arrays.asList;
 import static java.util.Collections.singletonList;
 import static org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignablePartitionResponse;
 import static org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignableTopicResponse;
-import static org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingTopicReassignment;
 import static org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingPartitionReassignment;
+import static org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingTopicReassignment;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.fail;
 
 /**
@@ -881,6 +882,69 @@ public void testCreatePartitions() throws Exception {
         }
     }
 
+    @Test
+    public void testDeleteRecordsTopicAuthorizationError() {
+        String topic = "foo";
+        TopicPartition partition = new TopicPartition(topic, 0);
+
+        try (AdminClientUnitTestEnv env = mockClientEnv()) {
+            List topics = new ArrayList<>();
+            topics.add(new MetadataResponse.TopicMetadata(Errors.TOPIC_AUTHORIZATION_FAILED, topic, false,
+                    Collections.emptyList()));
+
+            env.kafkaClient().prepareResponse(MetadataResponse.prepareResponse(env.cluster().nodes(),
+                    env.cluster().clusterResource().clusterId(), env.cluster().controller().id(), topics));
+
+            Map recordsToDelete = new HashMap<>();
+            recordsToDelete.put(partition, RecordsToDelete.beforeOffset(10L));
+            DeleteRecordsResult results = env.adminClient().deleteRecords(recordsToDelete);
+
+            TestUtils.assertFutureThrows(results.lowWatermarks().get(partition), TopicAuthorizationException.class);
+        }
+    }
+
+    @Test
+    public void testDeleteRecordsMultipleSends() throws Exception {
+        String topic = "foo";
+        TopicPartition tp0 = new TopicPartition(topic, 0);
+        TopicPartition tp1 = new TopicPartition(topic, 1);
+
+        Cluster cluster = mockCluster(0);
+        MockTime time = new MockTime();
+
+        try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(time, cluster)) {
+            List nodes = cluster.nodes();
+
+            List partitionMetadata = new ArrayList<>();
+            partitionMetadata.add(new MetadataResponse.PartitionMetadata(Errors.NONE, tp0.partition(), nodes.get(0),
+                    Optional.of(5), singletonList(nodes.get(0)), singletonList(nodes.get(0)),
+                    Collections.emptyList()));
+            partitionMetadata.add(new MetadataResponse.PartitionMetadata(Errors.NONE, tp1.partition(), nodes.get(1),
+                    Optional.of(5), singletonList(nodes.get(1)), singletonList(nodes.get(1)), Collections.emptyList()));
+
+            List topicMetadata = new ArrayList<>();
+            topicMetadata.add(new MetadataResponse.TopicMetadata(Errors.NONE, topic, false, partitionMetadata));
+
+            env.kafkaClient().prepareResponse(MetadataResponse.prepareResponse(cluster.nodes(),
+                    cluster.clusterResource().clusterId(), cluster.controller().id(), topicMetadata));
+
+            Map deletedPartitions = new HashMap<>();
+            deletedPartitions.put(tp0, new DeleteRecordsResponse.PartitionResponse(3, Errors.NONE));
+            env.kafkaClient().prepareResponseFrom(new DeleteRecordsResponse(0, deletedPartitions), nodes.get(0));
+
+            env.kafkaClient().disconnect(nodes.get(1).idString());
+            env.kafkaClient().createPendingAuthenticationError(nodes.get(1), 100);
+
+            Map recordsToDelete = new HashMap<>();
+            recordsToDelete.put(tp0, RecordsToDelete.beforeOffset(10L));
+            recordsToDelete.put(tp1, RecordsToDelete.beforeOffset(10L));
+            DeleteRecordsResult results = env.adminClient().deleteRecords(recordsToDelete);
+
+            assertEquals(3L, results.lowWatermarks().get(tp0).get().lowWatermark());
+            TestUtils.assertFutureThrows(results.lowWatermarks().get(tp1), AuthenticationException.class);
+        }
+    }
+
     @Test
     public void testDeleteRecords() throws Exception {
 
diff --git a/clients/src/test/java/org/apache/kafka/test/TestUtils.java b/clients/src/test/java/org/apache/kafka/test/TestUtils.java
index 5858a16e4a50c..2183e158af22c 100644
--- a/clients/src/test/java/org/apache/kafka/test/TestUtils.java
+++ b/clients/src/test/java/org/apache/kafka/test/TestUtils.java
@@ -64,6 +64,7 @@
 import static java.util.Arrays.asList;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -457,6 +458,22 @@ public static Set generateRandomTopicPartitions(int numTopic, in
         return tps;
     }
 
+    /**
+     * Assert that a future raises an expected exception cause type. Return the exception cause
+     * if the assertion succeeds; otherwise raise AssertionError.
+     *
+     * @param future The future to await
+     * @param exceptionCauseClass Class of the expected exception cause
+     * @param  Exception cause type parameter
+     * @return The caught exception cause
+     */
+    public static  T assertFutureThrows(Future future, Class exceptionCauseClass) {
+        ExecutionException exception = assertThrows(ExecutionException.class, future::get);
+        assertTrue("Unexpected exception cause " + exception.getCause(),
+                exceptionCauseClass.isInstance(exception.getCause()));
+        return exceptionCauseClass.cast(exception.getCause());
+    }
+
     public static void assertFutureError(Future future, Class exceptionClass)
         throws InterruptedException {
         try {

From 1a5971ebdc77f976627da8af30a9dfab596dfcb1 Mon Sep 17 00:00:00 2001
From: Bob Barrett 
Date: Tue, 8 Oct 2019 14:06:36 -0700
Subject: [PATCH 0702/1071] KAFKA-7190; Retain producer state until
 transactionalIdExpiration time passes (#7388)

As described in KIP-360, this patch changes producer state retention so that producer state remains cached even after it is removed from the log. Producer state will only be removed now when the transactional id expiration time has passed. This is intended to reduce the incidence of UNKNOWN_PRODUCER_ID errors for producers when records are deleted or when a topic has a short retention time. Tested with unit tests.

Reviewers: Jason Gustafson , Guozhang Wang 
---
 core/src/main/scala/kafka/log/Log.scala       |  1 -
 .../kafka/log/ProducerStateManager.scala      | 58 +++--------------
 .../test/scala/unit/kafka/log/LogTest.scala   | 13 ++--
 .../kafka/log/ProducerStateManagerTest.scala  | 63 ++++---------------
 4 files changed, 27 insertions(+), 108 deletions(-)

diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index 8068c9ba31a26..b0af105f5042b 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -1241,7 +1241,6 @@ class Log(@volatile var dir: File,
           info(s"Incrementing log start offset to $newLogStartOffset")
           logStartOffset = newLogStartOffset
           leaderEpochCache.foreach(_.truncateFromStart(logStartOffset))
-          producerStateManager.truncateHead(logStartOffset)
           maybeIncrementFirstUnstableOffset()
         }
       }
diff --git a/core/src/main/scala/kafka/log/ProducerStateManager.scala b/core/src/main/scala/kafka/log/ProducerStateManager.scala
index 995bf851eb035..ae5b77ab95fc6 100644
--- a/core/src/main/scala/kafka/log/ProducerStateManager.scala
+++ b/core/src/main/scala/kafka/log/ProducerStateManager.scala
@@ -113,7 +113,7 @@ private[log] class ProducerStateEntry(val producerId: Long,
 
   def lastDataOffset: Long = if (isEmpty) -1L else batchMetadata.last.lastOffset
 
-  def lastTimestamp = if (isEmpty) RecordBatch.NO_TIMESTAMP else batchMetadata.last.timestamp
+  def lastTimestamp: Long = if (isEmpty) RecordBatch.NO_TIMESTAMP else batchMetadata.last.timestamp
 
   def lastOffsetDelta : Int = if (isEmpty) 0 else batchMetadata.last.offsetDelta
 
@@ -148,8 +148,6 @@ private[log] class ProducerStateEntry(val producerId: Long,
     this.currentTxnFirstOffset = nextEntry.currentTxnFirstOffset
   }
 
-  def removeBatchesOlderThan(offset: Long): Unit = batchMetadata.dropWhile(_.lastOffset < offset)
-
   def findDuplicateBatch(batch: RecordBatch): Option[BatchMetadata] = {
     if (batch.producerEpoch != producerEpoch)
        None
@@ -542,7 +540,7 @@ class ProducerStateManager(val topicPartition: TopicPartition,
   /**
    * Returns the last offset of this map
    */
-  def mapEndOffset = lastMapOffset
+  def mapEndOffset: Long = lastMapOffset
 
   /**
    * Get a copy of the active producers
@@ -557,9 +555,7 @@ class ProducerStateManager(val topicPartition: TopicPartition,
         case Some(file) =>
           try {
             info(s"Loading producer state from snapshot file '$file'")
-            val loadedProducers = readSnapshot(file).filter { producerEntry =>
-              isProducerRetained(producerEntry, logStartOffset) && !isProducerExpired(currentTime, producerEntry)
-            }
+            val loadedProducers = readSnapshot(file).filter { producerEntry => !isProducerExpired(currentTime, producerEntry) }
             loadedProducers.foreach(loadProducerEntry)
             lastSnapOffset = offsetFromFile(file)
             lastMapOffset = lastSnapOffset
@@ -600,8 +596,10 @@ class ProducerStateManager(val topicPartition: TopicPartition,
 
   /**
    * Truncate the producer id mapping to the given offset range and reload the entries from the most recent
-   * snapshot in range (if there is one). Note that the log end offset is assumed to be less than
-   * or equal to the high watermark.
+   * snapshot in range (if there is one). We delete snapshot files prior to the logStartOffset but do not remove
+   * producer state from the map. This means that in-memory and on-disk state can diverge, and in the case of
+   * broker failover or unclean shutdown, any in-memory state not persisted in the snapshots will be lost.
+   * Note that the log end offset is assumed to be less than or equal to the high watermark.
    */
   def truncateAndReload(logStartOffset: Long, logEndOffset: Long, currentTimeMs: Long): Unit = {
     // remove all out of range snapshots
@@ -617,8 +615,6 @@ class ProducerStateManager(val topicPartition: TopicPartition,
       // safe to clear the unreplicated transactions
       unreplicatedTxns.clear()
       loadFromSnapshot(logStartOffset, currentTimeMs)
-    } else {
-      truncateHead(logStartOffset)
     }
   }
 
@@ -692,46 +688,6 @@ class ProducerStateManager(val topicPartition: TopicPartition,
    */
   def oldestSnapshotOffset: Option[Long] = oldestSnapshotFile.map(file => offsetFromFile(file))
 
-  private def isProducerRetained(producerStateEntry: ProducerStateEntry, logStartOffset: Long): Boolean = {
-    producerStateEntry.removeBatchesOlderThan(logStartOffset)
-    producerStateEntry.lastDataOffset >= logStartOffset
-  }
-
-  /**
-   * When we remove the head of the log due to retention, we need to clean up the id map. This method takes
-   * the new start offset and removes all producerIds which have a smaller last written offset. Additionally,
-   * we remove snapshots older than the new log start offset.
-   *
-   * Note that snapshots from offsets greater than the log start offset may have producers included which
-   * should no longer be retained: these producers will be removed if and when we need to load state from
-   * the snapshot.
-   */
-  def truncateHead(logStartOffset: Long): Unit = {
-    val evictedProducerEntries = producers.filter { case (_, producerState) =>
-      !isProducerRetained(producerState, logStartOffset)
-    }
-    val evictedProducerIds = evictedProducerEntries.keySet
-
-    producers --= evictedProducerIds
-    removeEvictedOngoingTransactions(evictedProducerIds)
-    removeUnreplicatedTransactions(logStartOffset)
-
-    if (lastMapOffset < logStartOffset)
-      lastMapOffset = logStartOffset
-
-    deleteSnapshotsBefore(logStartOffset)
-    lastSnapOffset = latestSnapshotOffset.getOrElse(logStartOffset)
-  }
-
-  private def removeEvictedOngoingTransactions(expiredProducerIds: collection.Set[Long]): Unit = {
-    val iterator = ongoingTxns.entrySet.iterator
-    while (iterator.hasNext) {
-      val txnEntry = iterator.next()
-      if (expiredProducerIds.contains(txnEntry.getValue.producerId))
-        iterator.remove()
-    }
-  }
-
   private def removeUnreplicatedTransactions(offset: Long): Unit = {
     val iterator = unreplicatedTxns.entrySet.iterator
     while (iterator.hasNext) {
diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala
index c446563570480..29b564e066049 100755
--- a/core/src/test/scala/unit/kafka/log/LogTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogTest.scala
@@ -1070,15 +1070,17 @@ class LogTest {
     log.updateHighWatermark(log.logEndOffset)
     log.maybeIncrementLogStartOffset(1L)
 
-    assertEquals(1, log.activeProducersWithLastSequence.size)
+    // Deleting records should not remove producer state
+    assertEquals(2, log.activeProducersWithLastSequence.size)
     val retainedLastSeqOpt = log.activeProducersWithLastSequence.get(pid2)
     assertTrue(retainedLastSeqOpt.isDefined)
     assertEquals(0, retainedLastSeqOpt.get)
 
     log.close()
 
+    // Because the log start offset did not advance, producer snapshots will still be present and the state will be rebuilt
     val reloadedLog = createLog(logDir, logConfig, logStartOffset = 1L)
-    assertEquals(1, reloadedLog.activeProducersWithLastSequence.size)
+    assertEquals(2, reloadedLog.activeProducersWithLastSequence.size)
     val reloadedLastSeqOpt = log.activeProducersWithLastSequence.get(pid2)
     assertEquals(retainedLastSeqOpt, reloadedLastSeqOpt)
   }
@@ -1104,14 +1106,16 @@ class LogTest {
     log.maybeIncrementLogStartOffset(1L)
     log.deleteOldSegments()
 
+    // Deleting records should not remove producer state
     assertEquals(1, log.logSegments.size)
-    assertEquals(1, log.activeProducersWithLastSequence.size)
+    assertEquals(2, log.activeProducersWithLastSequence.size)
     val retainedLastSeqOpt = log.activeProducersWithLastSequence.get(pid2)
     assertTrue(retainedLastSeqOpt.isDefined)
     assertEquals(0, retainedLastSeqOpt.get)
 
     log.close()
 
+    // After reloading log, producer state should not be regenerated
     val reloadedLog = createLog(logDir, logConfig, logStartOffset = 1L)
     assertEquals(1, reloadedLog.activeProducersWithLastSequence.size)
     val reloadedEntryOpt = log.activeProducersWithLastSequence.get(pid2)
@@ -1162,8 +1166,9 @@ class LogTest {
     log.updateHighWatermark(log.logEndOffset)
     log.deleteOldSegments()
 
+    // Producer state should not be removed when deleting log segment
     assertEquals(2, log.logSegments.size)
-    assertEquals(Set(pid2), log.activeProducersWithLastSequence.keySet)
+    assertEquals(Set(pid1, pid2), log.activeProducersWithLastSequence.keySet)
   }
 
   @Test
diff --git a/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala b/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala
index e98e59e2e271e..8500c94678ad7 100644
--- a/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala
@@ -561,52 +561,7 @@ class ProducerStateManagerTest {
   }
 
   @Test
-  def testFirstUnstableOffsetAfterEviction(): Unit = {
-    val epoch = 0.toShort
-    val sequence = 0
-    append(stateManager, producerId, epoch, sequence, offset = 99, isTransactional = true)
-    assertEquals(Some(99), stateManager.firstUnstableOffset.map(_.messageOffset))
-    append(stateManager, 2L, epoch, 0, offset = 106, isTransactional = true)
-    stateManager.truncateHead(100)
-    assertEquals(Some(106), stateManager.firstUnstableOffset.map(_.messageOffset))
-  }
-
-  @Test
-  def testTruncateHead(): Unit = {
-    val epoch = 0.toShort
-
-    append(stateManager, producerId, epoch, 0, 0L)
-    append(stateManager, producerId, epoch, 1, 1L)
-    stateManager.takeSnapshot()
-
-    val anotherPid = 2L
-    append(stateManager, anotherPid, epoch, 0, 2L)
-    append(stateManager, anotherPid, epoch, 1, 3L)
-    stateManager.takeSnapshot()
-    assertEquals(Set(2, 4), currentSnapshotOffsets)
-
-    stateManager.truncateHead(2)
-    assertEquals(Set(2, 4), currentSnapshotOffsets)
-    assertEquals(Set(anotherPid), stateManager.activeProducers.keySet)
-    assertEquals(None, stateManager.lastEntry(producerId))
-
-    val maybeEntry = stateManager.lastEntry(anotherPid)
-    assertTrue(maybeEntry.isDefined)
-    assertEquals(3L, maybeEntry.get.lastDataOffset)
-
-    stateManager.truncateHead(3)
-    assertEquals(Set(anotherPid), stateManager.activeProducers.keySet)
-    assertEquals(Set(4), currentSnapshotOffsets)
-    assertEquals(4, stateManager.mapEndOffset)
-
-    stateManager.truncateHead(5)
-    assertEquals(Set(), stateManager.activeProducers.keySet)
-    assertEquals(Set(), currentSnapshotOffsets)
-    assertEquals(5, stateManager.mapEndOffset)
-  }
-
-  @Test
-  def testLoadFromSnapshotRemovesNonRetainedProducers(): Unit = {
+  def testLoadFromSnapshotRetainsNonExpiredProducers(): Unit = {
     val epoch = 0.toShort
     val pid1 = 1L
     val pid2 = 2L
@@ -617,13 +572,17 @@ class ProducerStateManagerTest {
     assertEquals(2, stateManager.activeProducers.size)
 
     stateManager.truncateAndReload(1L, 2L, time.milliseconds())
-    assertEquals(1, stateManager.activeProducers.size)
-    assertEquals(None, stateManager.lastEntry(pid1))
+    assertEquals(2, stateManager.activeProducers.size)
+
+    val entry1 = stateManager.lastEntry(pid1)
+    assertTrue(entry1.isDefined)
+    assertEquals(0, entry1.get.lastSeq)
+    assertEquals(0L, entry1.get.lastDataOffset)
 
-    val entry = stateManager.lastEntry(pid2)
-    assertTrue(entry.isDefined)
-    assertEquals(0, entry.get.lastSeq)
-    assertEquals(1L, entry.get.lastDataOffset)
+    val entry2 = stateManager.lastEntry(pid2)
+    assertTrue(entry2.isDefined)
+    assertEquals(0, entry2.get.lastSeq)
+    assertEquals(1L, entry2.get.lastDataOffset)
   }
 
   @Test

From df46cbfdf0599de4b3eb035dbc1fa968c71b5af9 Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Wed, 9 Oct 2019 09:19:18 +0100
Subject: [PATCH 0703/1071] KAFKA-8932; Add tag for
 CreateTopicsResponse.TopicConfigErrorCode (KIP-525) (#7464)

Reviewers: Manikumar Reddy 
---
 .../common/message/CreateTopicsRequest.json         |  9 ++++-----
 .../common/message/CreateTopicsResponse.json        | 13 ++++++-------
 .../kafka/common/message/ApiMessageTypeTest.java    |  8 ++++----
 3 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/clients/src/main/resources/common/message/CreateTopicsRequest.json b/clients/src/main/resources/common/message/CreateTopicsRequest.json
index 6299023a9a9c9..2fc27717a4dee 100644
--- a/clients/src/main/resources/common/message/CreateTopicsRequest.json
+++ b/clients/src/main/resources/common/message/CreateTopicsRequest.json
@@ -21,11 +21,10 @@
   //
   // Version 4 makes partitions/replicationFactor optional even when assignments are not present (KIP-464)
   //
-  // Version 5 returns topic configs in the response (KIP-525).
-  //
-  // Version 6 is the first flexible version.
-  "validVersions": "0-6",
-  "flexibleVersions": "6+",
+  // Version 5 is the first flexible version.
+  // Version 5 also returns topic configs in the response (KIP-525).
+  "validVersions": "0-5",
+  "flexibleVersions": "5+",
   "fields": [
     { "name": "Topics", "type": "[]CreatableTopic", "versions": "0+",
       "about": "The topics to create.", "fields": [
diff --git a/clients/src/main/resources/common/message/CreateTopicsResponse.json b/clients/src/main/resources/common/message/CreateTopicsResponse.json
index 373258017961c..9d777ab6dd4f2 100644
--- a/clients/src/main/resources/common/message/CreateTopicsResponse.json
+++ b/clients/src/main/resources/common/message/CreateTopicsResponse.json
@@ -25,11 +25,10 @@
   //
   // Version 4 makes partitions/replicationFactor optional even when assignments are not present (KIP-464).
   //
-  // Version 5 returns topic configs in the response (KIP-525).
-  //
-  // Version 6 is the first flexible version.
-  "validVersions": "0-6",
-  "flexibleVersions": "6+",
+  // Version 5 is the first flexible version.
+  // Version 5 also returns topic configs in the response (KIP-525).
+  "validVersions": "0-5",
+  "flexibleVersions": "5+",
   "fields": [
     { "name": "ThrottleTimeMs", "type": "int32", "versions": "2+", "ignorable": true,
       "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
@@ -41,7 +40,7 @@
         "about": "The error code, or 0 if there was no error." },
       { "name": "ErrorMessage", "type": "string", "versions": "1+", "nullableVersions": "0+", "ignorable": true,
         "about": "The error message, or null if there was no error." },
-      { "name": "TopicConfigErrorCode", "type": "int16", "versions": "5+", "default": "0",
+      { "name": "TopicConfigErrorCode", "type": "int16", "versions": "5+", "tag": 0, "taggedVersions": "5+",
         "about": "Optional topic config error returned if configs are not returned in the response." },
       { "name": "NumPartitions", "type": "int32", "versions": "5+", "default": "-1",
         "about": "Number of partitions of the topic." },
@@ -51,7 +50,7 @@
         "about": "Configuration of the topic.", "fields": [
         { "name": "Name", "type": "string", "versions": "5+",
           "about": "The configuration name." },
-        { "name": "Value", "type": "string", "versions": "5+", "nullableVersions": "0+",
+        { "name": "Value", "type": "string", "versions": "5+", "nullableVersions": "5+",
           "about": "The configuration value." },
         { "name": "ReadOnly", "type": "bool", "versions": "5+",
           "about": "True if the configuration is read-only." },
diff --git a/clients/src/test/java/org/apache/kafka/common/message/ApiMessageTypeTest.java b/clients/src/test/java/org/apache/kafka/common/message/ApiMessageTypeTest.java
index 7acde182ddbea..8fdb6a090ad96 100644
--- a/clients/src/test/java/org/apache/kafka/common/message/ApiMessageTypeTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/message/ApiMessageTypeTest.java
@@ -88,10 +88,10 @@ public void testHeaderVersion() {
         assertEquals((short) 1, ApiMessageType.CONTROLLED_SHUTDOWN.requestHeaderVersion((short) 1));
         assertEquals((short) 0, ApiMessageType.CONTROLLED_SHUTDOWN.responseHeaderVersion((short) 1));
 
-        assertEquals((short) 1, ApiMessageType.CREATE_TOPICS.requestHeaderVersion((short) 5));
-        assertEquals((short) 0, ApiMessageType.CREATE_TOPICS.responseHeaderVersion((short) 5));
+        assertEquals((short) 1, ApiMessageType.CREATE_TOPICS.requestHeaderVersion((short) 4));
+        assertEquals((short) 0, ApiMessageType.CREATE_TOPICS.responseHeaderVersion((short) 4));
 
-        assertEquals((short) 2, ApiMessageType.CREATE_TOPICS.requestHeaderVersion((short) 6));
-        assertEquals((short) 1, ApiMessageType.CREATE_TOPICS.responseHeaderVersion((short) 6));
+        assertEquals((short) 2, ApiMessageType.CREATE_TOPICS.requestHeaderVersion((short) 5));
+        assertEquals((short) 1, ApiMessageType.CREATE_TOPICS.responseHeaderVersion((short) 5));
     }
 }

From 4596ca0a8036b46d27a8f449a2068bf89ad00033 Mon Sep 17 00:00:00 2001
From: David Jacot 
Date: Wed, 9 Oct 2019 17:36:49 +0200
Subject: [PATCH 0704/1071] KAFKA-8954; Topic existence check is wrongly
 implemented in the DeleteOffset API (KIP-496) (#7406)

This patch changes the way topic existence is checked in the DeleteOffset API. Previously, it was relying on the committed offsets. Now, it relies on the metadata cache which is better.

Reviewers: Jason Gustafson 
---
 .../coordinator/group/GroupCoordinator.scala  | 14 +---
 .../main/scala/kafka/server/KafkaApis.scala   | 32 +++++----
 .../group/GroupCoordinatorTest.scala          | 40 +++++-------
 .../unit/kafka/server/KafkaApisTest.scala     | 65 ++++++++++++++++++-
 4 files changed, 104 insertions(+), 47 deletions(-)

diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
index 64884ec5dbbd1..22f15f9811f5e 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
@@ -565,22 +565,14 @@ class GroupCoordinator(val brokerId: Int,
                     Errors.GROUP_ID_NOT_FOUND else Errors.NOT_COORDINATOR
 
                 case Empty =>
-                  val (knownPartitions, unknownPartitions) =
-                    partitions.partition(tp => group.offset(tp).nonEmpty)
-
-                  partitionEligibleForDeletion = knownPartitions
-                  partitionErrors = unknownPartitions.map(_ -> Errors.UNKNOWN_TOPIC_OR_PARTITION).toMap
+                  partitionEligibleForDeletion = partitions
 
                 case PreparingRebalance | CompletingRebalance | Stable if group.isConsumerGroup =>
-                  val (knownPartitions, unknownPartitions) =
-                    partitions.partition(tp => group.offset(tp).nonEmpty)
-
                   val (consumed, notConsumed) =
-                    knownPartitions.partition(tp => group.isSubscribedToTopic(tp.topic()))
+                    partitions.partition(tp => group.isSubscribedToTopic(tp.topic()))
 
                   partitionEligibleForDeletion = notConsumed
-                  partitionErrors = consumed.map(_ -> Errors.GROUP_SUBSCRIBED_TO_TOPIC).toMap ++
-                    unknownPartitions.map(_ -> Errors.UNKNOWN_TOPIC_OR_PARTITION).toMap
+                  partitionErrors = consumed.map(_ -> Errors.GROUP_SUBSCRIBED_TO_TOPIC).toMap
 
                 case _ =>
                   groupError = Errors.NON_EMPTY_GROUP
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index c7747b60dd494..9a0ebd7ac5fac 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -2688,27 +2688,35 @@ class KafkaApis(val requestChannel: RequestChannel,
     val groupId = offsetDeleteRequest.data.groupId
 
     if (authorize(request, DELETE, GROUP, groupId)) {
-      val topicPartitions = offsetDeleteRequest.data.topics.asScala.flatMap { topic =>
-        topic.partitions.asScala.map { partition =>
-          new TopicPartition(topic.name, partition.partitionIndex)
+      val authorizedTopics = filterAuthorized(request, READ, TOPIC,
+        offsetDeleteRequest.data.topics.asScala.map(_.name).toSeq)
+
+      val topicPartitionErrors = mutable.Map[TopicPartition, Errors]()
+      val topicPartitions = mutable.ArrayBuffer[TopicPartition]()
+
+      for (topic <- offsetDeleteRequest.data.topics.asScala) {
+        for (partition <- topic.partitions.asScala) {
+          val tp = new TopicPartition(topic.name, partition.partitionIndex)
+          if (!authorizedTopics.contains(topic.name))
+            topicPartitionErrors(tp) = Errors.TOPIC_AUTHORIZATION_FAILED
+          else if (!metadataCache.contains(tp))
+            topicPartitionErrors(tp) = Errors.UNKNOWN_TOPIC_OR_PARTITION
+          else
+            topicPartitions += tp
         }
-      }.toSeq
-
-      val authorizedTopics = filterAuthorized(request, READ, TOPIC, topicPartitions.map(_.topic))
-      val (authorizedTopicPartitions, unauthorizedTopicPartitions) = topicPartitions.partition { topicPartition =>
-        authorizedTopics.contains(topicPartition.topic)
       }
 
-      val unauthorizedTopicPartitionsErrors = unauthorizedTopicPartitions.map(_ -> Errors.TOPIC_AUTHORIZATION_FAILED)
-      val (groupError, authorizedTopicPartitionsErrors) = groupCoordinator.handleDeleteOffsets(groupId, authorizedTopicPartitions)
-      val topicPartitionsErrors = unauthorizedTopicPartitionsErrors ++ authorizedTopicPartitionsErrors
+      val (groupError, authorizedTopicPartitionsErrors) = groupCoordinator.handleDeleteOffsets(
+        groupId, topicPartitions)
+
+      topicPartitionErrors ++= authorizedTopicPartitionsErrors
 
       sendResponseMaybeThrottle(request, requestThrottleMs => {
         if (groupError != Errors.NONE)
           offsetDeleteRequest.getErrorResponse(requestThrottleMs, groupError)
         else {
           val topics = new OffsetDeleteResponseData.OffsetDeleteResponseTopicCollection
-          topicPartitionsErrors.groupBy(_._1.topic).map { case (topic, topicPartitions) =>
+          topicPartitionErrors.groupBy(_._1.topic).map { case (topic, topicPartitions) =>
             val partitions = new OffsetDeleteResponseData.OffsetDeleteResponsePartitionCollection
             topicPartitions.map { case (topicPartition, error) =>
               partitions.add(
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
index 91a963d7f14e1..cd85e0a86016d 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
@@ -2827,7 +2827,7 @@ class GroupCoordinatorTest {
     val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(tp))
 
     assertEquals(Errors.GROUP_ID_NOT_FOUND, groupError)
-    assert(topics.isEmpty)
+    assertTrue(topics.isEmpty)
   }
 
   @Test
@@ -2838,7 +2838,7 @@ class GroupCoordinatorTest {
     val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(tp))
 
     assertEquals(Errors.NON_EMPTY_GROUP, groupError)
-    assert(topics.isEmpty)
+    assertTrue(topics.isEmpty)
   }
 
   @Test
@@ -2868,7 +2868,7 @@ class GroupCoordinatorTest {
     val leaveGroupResults = singleLeaveGroup(groupId, joinGroupResult.memberId)
     verifyLeaveGroupResult(leaveGroupResults)
 
-    assert(groupCoordinator.groupManager.getGroup(groupId).exists(_.is(Empty)))
+    assertTrue(groupCoordinator.groupManager.getGroup(groupId).exists(_.is(Empty)))
 
     val groupTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupPartitionId)
     val partition: Partition = EasyMock.niceMock(classOf[Partition])
@@ -2882,7 +2882,7 @@ class GroupCoordinatorTest {
     val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(t1p0))
 
     assertEquals(Errors.NONE, groupError)
-    assert(topics.size == 1)
+    assertEquals(1, topics.size)
     assertEquals(Some(Errors.NONE), topics.get(t1p0))
 
     val cachedOffsets = groupCoordinator.groupManager.getOffsets(groupId, Some(Seq(t1p0, t2p0)))
@@ -2900,21 +2900,19 @@ class GroupCoordinatorTest {
     val syncGroupResult = syncGroupLeader(groupId, joinGroupResult.generationId, joinGroupResult.leaderId, Map.empty)
     assertEquals(Errors.NONE, syncGroupResult._2)
 
-    val t1p0 = new TopicPartition("foo", 0)
-    val t2p0 = new TopicPartition("bar", 0)
+    val tp = new TopicPartition("foo", 0)
     val offset = offsetAndMetadata(37)
 
     EasyMock.reset(replicaManager)
     val validOffsetCommitResult = commitOffsets(groupId, joinGroupResult.memberId, joinGroupResult.generationId,
-      Map(t1p0 -> offset))
-    assertEquals(Errors.NONE, validOffsetCommitResult(t1p0))
+      Map(tp -> offset))
+    assertEquals(Errors.NONE, validOffsetCommitResult(tp))
 
-    val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(t1p0, t2p0))
+    val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(tp))
 
     assertEquals(Errors.NONE, groupError)
-    assert(topics.size == 2)
-    assertEquals(Some(Errors.GROUP_SUBSCRIBED_TO_TOPIC), topics.get(t1p0))
-    assertEquals(Some(Errors.UNKNOWN_TOPIC_OR_PARTITION), topics.get(t2p0))
+    assertEquals(1, topics.size)
+    assertEquals(Some(Errors.GROUP_SUBSCRIBED_TO_TOPIC), topics.get(tp))
   }
 
   @Test
@@ -2927,7 +2925,7 @@ class GroupCoordinatorTest {
     val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(tp))
 
     assertEquals(Errors.GROUP_ID_NOT_FOUND, groupError)
-    assert(topics.size == 0)
+    assertTrue(topics.isEmpty)
   }
 
   @Test
@@ -2943,7 +2941,6 @@ class GroupCoordinatorTest {
 
     val t1p0 = new TopicPartition("foo", 0)
     val t2p0 = new TopicPartition("bar", 0)
-    val t3p0 = new TopicPartition("unknown", 0)
     val offset = offsetAndMetadata(37)
 
     EasyMock.reset(replicaManager)
@@ -2957,7 +2954,7 @@ class GroupCoordinatorTest {
     val leaveGroupResults = singleLeaveGroup(groupId, joinGroupResult.memberId)
     verifyLeaveGroupResult(leaveGroupResults)
 
-    assert(groupCoordinator.groupManager.getGroup(groupId).exists(_.is(Empty)))
+    assertTrue(groupCoordinator.groupManager.getGroup(groupId).exists(_.is(Empty)))
 
     val groupTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupPartitionId)
     val partition: Partition = EasyMock.niceMock(classOf[Partition])
@@ -2968,12 +2965,11 @@ class GroupCoordinatorTest {
     EasyMock.expect(replicaManager.nonOfflinePartition(groupTopicPartition)).andStubReturn(Some(partition))
     EasyMock.replay(replicaManager, partition)
 
-    val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(t1p0, t3p0))
+    val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(t1p0))
 
     assertEquals(Errors.NONE, groupError)
-    assert(topics.size == 2)
+    assertEquals(1, topics.size)
     assertEquals(Some(Errors.NONE), topics.get(t1p0))
-    assertEquals(Some(Errors.UNKNOWN_TOPIC_OR_PARTITION), topics.get(t3p0))
 
     val cachedOffsets = groupCoordinator.groupManager.getOffsets(groupId, Some(Seq(t1p0, t2p0)))
 
@@ -2997,7 +2993,6 @@ class GroupCoordinatorTest {
 
     val t1p0 = new TopicPartition("foo", 0)
     val t2p0 = new TopicPartition("bar", 0)
-    val t3p0 = new TopicPartition("unknown", 0)
     val offset = offsetAndMetadata(37)
 
     EasyMock.reset(replicaManager)
@@ -3006,7 +3001,7 @@ class GroupCoordinatorTest {
     assertEquals(Errors.NONE, validOffsetCommitResult(t1p0))
     assertEquals(Errors.NONE, validOffsetCommitResult(t2p0))
 
-    assert(groupCoordinator.groupManager.getGroup(groupId).exists(_.is(Stable)))
+    assertTrue(groupCoordinator.groupManager.getGroup(groupId).exists(_.is(Stable)))
 
     val groupTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupPartitionId)
     val partition: Partition = EasyMock.niceMock(classOf[Partition])
@@ -3017,13 +3012,12 @@ class GroupCoordinatorTest {
     EasyMock.expect(replicaManager.nonOfflinePartition(groupTopicPartition)).andStubReturn(Some(partition))
     EasyMock.replay(replicaManager, partition)
 
-    val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(t1p0, t2p0, t3p0))
+    val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(t1p0, t2p0))
 
     assertEquals(Errors.NONE, groupError)
-    assert(topics.size == 3)
+    assertEquals(2, topics.size)
     assertEquals(Some(Errors.NONE), topics.get(t1p0))
     assertEquals(Some(Errors.GROUP_SUBSCRIBED_TO_TOPIC), topics.get(t2p0))
-    assertEquals(Some(Errors.UNKNOWN_TOPIC_OR_PARTITION), topics.get(t3p0))
 
     val cachedOffsets = groupCoordinator.groupManager.getOffsets(groupId, Some(Seq(t1p0, t2p0)))
 
diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
index bf1a8c487b1cc..66bed52ac979d 100644
--- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
@@ -46,9 +46,10 @@ import org.apache.kafka.common.requests.{FetchMetadata => JFetchMetadata, _}
 import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
 import org.easymock.{Capture, EasyMock, IAnswer}
 import EasyMock._
-import org.apache.kafka.common.message.{HeartbeatRequestData, JoinGroupRequestData, OffsetCommitRequestData, OffsetCommitResponseData, SyncGroupRequestData, TxnOffsetCommitRequestData}
+import org.apache.kafka.common.message.{HeartbeatRequestData, JoinGroupRequestData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteRequestData, SyncGroupRequestData, TxnOffsetCommitRequestData}
 import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocol
 import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
+import org.apache.kafka.common.message.OffsetDeleteRequestData.{OffsetDeleteRequestPartition, OffsetDeleteRequestTopic, OffsetDeleteRequestTopicCollection}
 import org.apache.kafka.common.message.UpdateMetadataRequestData.{UpdateMetadataBroker, UpdateMetadataEndpoint, UpdateMetadataPartitionState}
 import org.apache.kafka.common.replica.ClientMetadata
 import org.apache.kafka.server.authorizer.Authorizer
@@ -393,6 +394,68 @@ class KafkaApisTest {
     testListOffsetFailedGetLeaderReplica(Errors.UNKNOWN_TOPIC_OR_PARTITION)
   }
 
+  @Test
+  def testOffsetDeleteWithInvalidPartition(): Unit = {
+    val group = "groupId"
+    val topic = "topic"
+    setupBasicMetadataCache(topic, numPartitions = 1)
+
+    def checkInvalidPartition(invalidPartitionId: Int): Unit = {
+      EasyMock.reset(groupCoordinator, replicaManager, clientRequestQuotaManager, requestChannel)
+
+      val topics = new OffsetDeleteRequestTopicCollection()
+      topics.add(new OffsetDeleteRequestTopic()
+        .setName(topic)
+        .setPartitions(Collections.singletonList(
+          new OffsetDeleteRequestPartition().setPartitionIndex(invalidPartitionId))))
+      val (offsetDeleteRequest, request) = buildRequest(new OffsetDeleteRequest.Builder(
+        new OffsetDeleteRequestData()
+          .setGroupId(group)
+          .setTopics(topics)
+      ))
+
+      val capturedResponse = expectNoThrottling()
+      EasyMock.expect(groupCoordinator.handleDeleteOffsets(EasyMock.eq(group), EasyMock.eq(Seq.empty)))
+        .andReturn((Errors.NONE, Map.empty))
+      EasyMock.replay(groupCoordinator, replicaManager, clientRequestQuotaManager, requestChannel)
+
+      createKafkaApis().handleOffsetDeleteRequest(request)
+
+      val response = readResponse(ApiKeys.OFFSET_DELETE, offsetDeleteRequest, capturedResponse)
+        .asInstanceOf[OffsetDeleteResponse]
+
+      assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION,
+        Errors.forCode(response.data.topics.find(topic).partitions.find(invalidPartitionId).errorCode()))
+    }
+
+    checkInvalidPartition(-1)
+    checkInvalidPartition(1) // topic has only one partition
+  }
+
+  @Test
+  def testOffsetDeleteWithInvalidGroup(): Unit = {
+    val group = "groupId"
+
+    EasyMock.reset(groupCoordinator, replicaManager, clientRequestQuotaManager, requestChannel)
+
+    val (offsetDeleteRequest, request) = buildRequest(new OffsetDeleteRequest.Builder(
+      new OffsetDeleteRequestData()
+        .setGroupId(group)
+    ))
+
+    val capturedResponse = expectNoThrottling()
+    EasyMock.expect(groupCoordinator.handleDeleteOffsets(EasyMock.eq(group), EasyMock.eq(Seq.empty)))
+      .andReturn((Errors.GROUP_ID_NOT_FOUND, Map.empty))
+    EasyMock.replay(groupCoordinator, replicaManager, clientRequestQuotaManager, requestChannel)
+
+    createKafkaApis().handleOffsetDeleteRequest(request)
+
+    val response = readResponse(ApiKeys.OFFSET_DELETE, offsetDeleteRequest, capturedResponse)
+      .asInstanceOf[OffsetDeleteResponse]
+
+    assertEquals(Errors.GROUP_ID_NOT_FOUND, Errors.forCode(response.data.errorCode()))
+  }
+
   private def testListOffsetFailedGetLeaderReplica(error: Errors): Unit = {
     val tp = new TopicPartition("foo", 0)
     val isolationLevel = IsolationLevel.READ_UNCOMMITTED

From c2e66e12293a7f313663ec317bceafcb380c7605 Mon Sep 17 00:00:00 2001
From: Stanislav Kozlovski 
Date: Wed, 9 Oct 2019 19:46:31 +0100
Subject: [PATCH 0705/1071] MINOR: Fix stale comment in partition reassignment
 javadoc (#7438)

Reviewers: Colin P. McCabe , Viktor Somogyi 
(cherry picked from commit c620b73883c675ef7c49cd87a5873b3f9da3a5e1)
---
 core/src/main/scala/kafka/controller/KafkaController.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala
index 642ebaa99345e..e4679d3ae978e 100644
--- a/core/src/main/scala/kafka/controller/KafkaController.scala
+++ b/core/src/main/scala/kafka/controller/KafkaController.scala
@@ -562,8 +562,8 @@ class KafkaController(val config: KafkaConfig,
    *   . OVRS denotes the overlapping replica set - replicas which are part of the AR of the ongoing reassignment and will be part of the overriding reassignment
    *       (it is essentially (RS - ORS) - URS)
    *
-   *   1 Set RS = ORS + OVRS, AR = OVRS, RS = [] in memory
-   *   2 Send LeaderAndIsr request with RS = ORS + OVRS, AR = [], RS = [] to all brokers in ORS + OVRS
+   *   1 Set RS = ORS + OVRS, AR = OVRS, RR = [] in memory
+   *   2 Send LeaderAndIsr request with RS = ORS + OVRS, AR = OVRS, RR = [] to all brokers in ORS + OVRS
    *     (because the ongoing reassignment is in phase A, we know we wouldn't have a leader in URS
    *      unless a preferred leader election was triggered while the reassignment was happening)
    *   3 Replicas in URS -> Offline (force those replicas out of ISR)

From 48b58e03481d4b7faec2d1906abb21729e01faf8 Mon Sep 17 00:00:00 2001
From: Stanislav Kozlovski 
Date: Wed, 9 Oct 2019 19:47:29 +0100
Subject: [PATCH 0706/1071] MINOR: Change topic zNode's new reassigning fields
 names to snake case in order to be consistent with other fields (#7458)

Reviewers: Colin P. McCabe , Ismael Juma 
(cherry picked from commit 4c01668f7725d4057d3ff09ad9898c15e2899892)
---
 core/src/main/scala/kafka/zk/ZkData.scala | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/core/src/main/scala/kafka/zk/ZkData.scala b/core/src/main/scala/kafka/zk/ZkData.scala
index 75d169d0e2f5d..4a81e83dd2480 100644
--- a/core/src/main/scala/kafka/zk/ZkData.scala
+++ b/core/src/main/scala/kafka/zk/ZkData.scala
@@ -256,8 +256,8 @@ object TopicZNode {
     Json.encodeAsBytes(Map(
       "version" -> 2,
       "partitions" -> replicaAssignmentJson.asJava,
-      "addingReplicas" -> addingReplicasAssignmentJson.asJava,
-      "removingReplicas" -> removingReplicasAssignmentJson.asJava
+      "adding_replicas" -> addingReplicasAssignmentJson.asJava,
+      "removing_replicas" -> removingReplicasAssignmentJson.asJava
     ).asJava)
   }
   def decode(topic: String, bytes: Array[Byte]): Map[TopicPartition, PartitionReplicaAssignment] = {
@@ -274,8 +274,8 @@ object TopicZNode {
     Json.parseBytes(bytes).flatMap { js =>
       val assignmentJson = js.asJsonObject
       val partitionsJsonOpt = assignmentJson.get("partitions").map(_.asJsonObject)
-      val addingReplicasJsonOpt = assignmentJson.get("addingReplicas").map(_.asJsonObject)
-      val removingReplicasJsonOpt = assignmentJson.get("removingReplicas").map(_.asJsonObject)
+      val addingReplicasJsonOpt = assignmentJson.get("adding_replicas").map(_.asJsonObject)
+      val removingReplicasJsonOpt = assignmentJson.get("removing_replicas").map(_.asJsonObject)
       partitionsJsonOpt.map { partitionsJson =>
         partitionsJson.iterator.map { case (partition, replicas) =>
           new TopicPartition(topic, partition.toInt) -> PartitionReplicaAssignment(

From ce4ecc296f0a70393ee3b2837de7f40fc40af32b Mon Sep 17 00:00:00 2001
From: Colin Patrick McCabe 
Date: Thu, 10 Oct 2019 13:52:07 -0700
Subject: [PATCH 0707/1071] MINOR: fix compatibility-breaking bug in
 RequestHeader (#7479)

Reviewers: David Arthur , Jason Gustafson , Ismael Juma 
(cherry picked from commit 5c4bbf9344d82ebcb98ff1444a5a6e5d8ff05215)
---
 .../src/main/resources/common/message/RequestHeader.json  | 8 +++++++-
 .../apache/kafka/common/requests/RequestHeaderTest.java   | 2 +-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/clients/src/main/resources/common/message/RequestHeader.json b/clients/src/main/resources/common/message/RequestHeader.json
index 560b0190d5b9a..fbf4e2ca9f454 100644
--- a/clients/src/main/resources/common/message/RequestHeader.json
+++ b/clients/src/main/resources/common/message/RequestHeader.json
@@ -30,7 +30,13 @@
       "about": "The API version of this request." },
     { "name": "CorrelationId", "type": "int32", "versions": "0+",
       "about": "The correlation ID of this request." },
+
+    // The ClientId string must be serialized with the old-style two-byte length prefix.
+    // The reason is that older brokers must be able to read the request header for any
+    // ApiVersionsRequest, even if it is from a newer version.
+    // Since the client is sending the ApiVersionsRequest in order to discover what
+    // versions are supported, the client does not know the best version to use.
     { "name": "ClientId", "type": "string", "versions": "1+", "nullableVersions": "1+", "ignorable": true,
-      "about": "The client ID string." }
+      "flexibleVersions": "none", "about": "The client ID string." }
   ]
 }
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java
index 1a7693919769a..2842ae8e019ce 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java
@@ -69,7 +69,7 @@ public void testRequestHeaderV2() {
         RequestHeader header = new RequestHeader(ApiKeys.CREATE_DELEGATION_TOKEN, (short) 2, "", 10);
         assertEquals(2, header.headerVersion());
         ByteBuffer buffer = toBuffer(header.toStruct());
-        assertEquals(10, buffer.remaining());
+        assertEquals(11, buffer.remaining());
         RequestHeader deserialized = RequestHeader.parse(buffer);
         assertEquals(header, deserialized);
     }

From 18cfd13a9b211cbe272563544904d463c0ed93bb Mon Sep 17 00:00:00 2001
From: "Tu V. Tran" 
Date: Thu, 10 Oct 2019 14:44:37 -0700
Subject: [PATCH 0708/1071] KAFKA-8729, pt 3: Add broker-side logic to handle
 the case when there are record_errors and error_message (#7167)

All the changes are in ReplicaManager.appendToLocalLog and ReplicaManager.appendRecords. Also, replaced LogAppendInfo.unknownLogAppendInfoWithLogStartOffset with LogAppendInfo.unknownLogAppendInfoWithAdditionalInfo to include those 2 new fields.

Reviewers: Guozhang Wang , Jason Gustafson 
---
 .../kafka/common/requests/ProduceRequest.java |  2 +-
 .../common/requests/ProduceResponse.java      | 99 ++++++++++++-------
 .../common/message/ProduceRequest.json        |  2 +-
 .../common/message/ProduceResponse.json       | 12 +--
 .../kafka/common/message/MessageTest.java     | 14 +--
 .../common/requests/RequestResponseTest.java  |  3 +-
 .../common/RecordValidationException.scala    | 25 +++++
 core/src/main/scala/kafka/log/Log.scala       | 18 +++-
 .../main/scala/kafka/log/LogValidator.scala   | 69 +++++++++----
 .../scala/kafka/server/ReplicaManager.scala   | 37 ++++---
 .../test/scala/unit/kafka/log/LogTest.scala   | 10 +-
 .../unit/kafka/log/LogValidatorTest.scala     | 58 +++++++++--
 .../kafka/server/ProduceRequestTest.scala     | 36 ++++++-
 13 files changed, 285 insertions(+), 100 deletions(-)
 create mode 100644 core/src/main/scala/kafka/common/RecordValidationException.scala

diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java
index 932473cb869ea..7b3ae1cf2afe0 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java
@@ -121,7 +121,7 @@ public class ProduceRequest extends AbstractRequest {
     private static final Schema PRODUCE_REQUEST_V7 = PRODUCE_REQUEST_V6;
 
     /**
-     * V8 bumped up to add two new fields error_records offset list and error_message to {@link org.apache.kafka.common.requests.ProduceResponse.PartitionResponse}
+     * V8 bumped up to add two new fields record_errors offset list and error_message to {@link org.apache.kafka.common.requests.ProduceResponse.PartitionResponse}
      * (See KIP-467)
      */
     private static final Schema PRODUCE_REQUEST_V8 = PRODUCE_REQUEST_V7;
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java
index a6df88002d582..8bfc6298604be 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java
@@ -74,14 +74,14 @@ public class ProduceResponse extends AbstractResponse {
     private static final String BASE_OFFSET_KEY_NAME = "base_offset";
     private static final String LOG_APPEND_TIME_KEY_NAME = "log_append_time";
     private static final String LOG_START_OFFSET_KEY_NAME = "log_start_offset";
-    private static final String ERROR_RECORDS_KEY_NAME = "error_records";
-    private static final String RELATIVE_OFFSET_KEY_NAME = "relative_offset";
-    private static final String RELATIVE_OFFSET_ERROR_MESSAGE_KEY_NAME = "relative_offset_error_message";
+    private static final String RECORD_ERRORS_KEY_NAME = "record_errors";
+    private static final String BATCH_INDEX_KEY_NAME = "batch_index";
+    private static final String BATCH_INDEX_ERROR_MESSAGE_KEY_NAME = "batch_index_error_message";
     private static final String ERROR_MESSAGE_KEY_NAME = "error_message";
 
     private static final Field.Int64 LOG_START_OFFSET_FIELD = new Field.Int64(LOG_START_OFFSET_KEY_NAME,
             "The start offset of the log at the time this produce response was created", INVALID_OFFSET);
-    private static final Field.NullableStr RELATIVE_OFFSET_ERROR_MESSAGE_FIELD = new Field.NullableStr(RELATIVE_OFFSET_ERROR_MESSAGE_KEY_NAME,
+    private static final Field.NullableStr BATCH_INDEX_ERROR_MESSAGE_FIELD = new Field.NullableStr(BATCH_INDEX_ERROR_MESSAGE_KEY_NAME,
             "The error message of the record that caused the batch to be dropped");
     private static final Field.NullableStr ERROR_MESSAGE_FIELD = new Field.NullableStr(ERROR_MESSAGE_KEY_NAME,
             "The global error message summarizing the common root cause of the records that caused the batch to be dropped");
@@ -160,7 +160,7 @@ public class ProduceResponse extends AbstractResponse {
     private static final Schema PRODUCE_RESPONSE_V7 = PRODUCE_RESPONSE_V6;
 
     /**
-     * V8 adds error_records and error_message. (see KIP-467)
+     * V8 adds record_errors and error_message. (see KIP-467)
      */
     public static final Schema PRODUCE_RESPONSE_V8 = new Schema(
             new Field(RESPONSES_KEY_NAME, new ArrayOf(new Schema(
@@ -174,11 +174,11 @@ public class ProduceResponse extends AbstractResponse {
                                     "If LogAppendTime is used for the topic, the timestamp will be the broker local " +
                                     "time when the messages are appended."),
                             LOG_START_OFFSET_FIELD,
-                            new Field(ERROR_RECORDS_KEY_NAME, new ArrayOf(new Schema(
-                                    new Field.Int32(RELATIVE_OFFSET_KEY_NAME, "The relative offset of the record " +
+                            new Field(RECORD_ERRORS_KEY_NAME, new ArrayOf(new Schema(
+                                    new Field.Int32(BATCH_INDEX_KEY_NAME, "The batch index of the record " +
                                             "that caused the batch to be dropped"),
-                                    RELATIVE_OFFSET_ERROR_MESSAGE_FIELD
-                            )), "The relative offsets of records that caused the batch to be dropped"),
+                                    BATCH_INDEX_ERROR_MESSAGE_FIELD
+                            )), "The batch indices of records that caused the batch to be dropped"),
                             ERROR_MESSAGE_FIELD)))))),
             THROTTLE_TIME_MS);
 
@@ -225,19 +225,24 @@ public ProduceResponse(Struct struct) {
                 long logAppendTime = partRespStruct.getLong(LOG_APPEND_TIME_KEY_NAME);
                 long logStartOffset = partRespStruct.getOrElse(LOG_START_OFFSET_FIELD, INVALID_OFFSET);
 
-                Map errorRecords = new HashMap<>();
-                if (partRespStruct.hasField(ERROR_RECORDS_KEY_NAME)) {
-                    for (Object recordOffsetAndMessage : partRespStruct.getArray(ERROR_RECORDS_KEY_NAME)) {
-                        Struct recordOffsetAndMessageStruct = (Struct) recordOffsetAndMessage;
-                        Integer relativeOffset = recordOffsetAndMessageStruct.getInt(RELATIVE_OFFSET_KEY_NAME);
-                        String relativeOffsetErrorMessage = recordOffsetAndMessageStruct.getOrElse(RELATIVE_OFFSET_ERROR_MESSAGE_FIELD, "");
-                        errorRecords.put(relativeOffset, relativeOffsetErrorMessage);
+                List recordErrors = Collections.emptyList();
+                if (partRespStruct.hasField(RECORD_ERRORS_KEY_NAME)) {
+                    Object[] recordErrorsArray = partRespStruct.getArray(RECORD_ERRORS_KEY_NAME);
+                    if (recordErrorsArray.length > 0) {
+                        recordErrors = new ArrayList<>(recordErrorsArray.length);
+                        for (Object indexAndMessage : recordErrorsArray) {
+                            Struct indexAndMessageStruct = (Struct) indexAndMessage;
+                            recordErrors.add(new RecordError(
+                                    indexAndMessageStruct.getInt(BATCH_INDEX_KEY_NAME),
+                                    indexAndMessageStruct.get(BATCH_INDEX_ERROR_MESSAGE_FIELD)
+                            ));
+                        }
                     }
                 }
 
-                String errorMessage = partRespStruct.getOrElse(ERROR_MESSAGE_FIELD, "");
+                String errorMessage = partRespStruct.getOrElse(ERROR_MESSAGE_FIELD, null);
                 TopicPartition tp = new TopicPartition(topic, partition);
-                responses.put(tp, new PartitionResponse(error, offset, logAppendTime, logStartOffset, errorRecords, errorMessage));
+                responses.put(tp, new PartitionResponse(error, offset, logAppendTime, logStartOffset, recordErrors, errorMessage));
             }
         }
         this.throttleTimeMs = struct.getOrElse(THROTTLE_TIME_MS, DEFAULT_THROTTLE_TIME);
@@ -266,19 +271,21 @@ protected Struct toStruct(short version) {
                         .set(PARTITION_ID, partitionEntry.getKey())
                         .set(ERROR_CODE, errorCode)
                         .set(BASE_OFFSET_KEY_NAME, part.baseOffset);
-                if (partStruct.hasField(LOG_APPEND_TIME_KEY_NAME))
-                    partStruct.set(LOG_APPEND_TIME_KEY_NAME, part.logAppendTime);
+                partStruct.setIfExists(LOG_APPEND_TIME_KEY_NAME, part.logAppendTime);
                 partStruct.setIfExists(LOG_START_OFFSET_FIELD, part.logStartOffset);
 
-                List errorRecords = new ArrayList<>();
-                for (Map.Entry recordOffsetAndMessage : part.errorRecords.entrySet()) {
-                    Struct recordOffsetAndMessageStruct = partStruct.instance(ERROR_RECORDS_KEY_NAME)
-                            .set(RELATIVE_OFFSET_KEY_NAME, recordOffsetAndMessage.getKey())
-                            .setIfExists(RELATIVE_OFFSET_ERROR_MESSAGE_FIELD, recordOffsetAndMessage.getValue());
-                    errorRecords.add(recordOffsetAndMessageStruct);
+                List recordErrors = Collections.emptyList();
+                if (!part.recordErrors.isEmpty()) {
+                    recordErrors = new ArrayList<>();
+                    for (RecordError indexAndMessage : part.recordErrors) {
+                        Struct indexAndMessageStruct = partStruct.instance(RECORD_ERRORS_KEY_NAME)
+                                .set(BATCH_INDEX_KEY_NAME, indexAndMessage.batchIndex)
+                                .set(BATCH_INDEX_ERROR_MESSAGE_FIELD, indexAndMessage.message);
+                        recordErrors.add(indexAndMessageStruct);
+                    }
                 }
 
-                partStruct.setIfExists(ERROR_RECORDS_KEY_NAME, errorRecords.toArray());
+                partStruct.setIfExists(RECORD_ERRORS_KEY_NAME, recordErrors.toArray());
 
                 partStruct.setIfExists(ERROR_MESSAGE_FIELD, part.errorMessage);
                 partitionArray.add(partStruct);
@@ -314,7 +321,7 @@ public static final class PartitionResponse {
         public long baseOffset;
         public long logAppendTime;
         public long logStartOffset;
-        public Map errorRecords;
+        public List recordErrors;
         public String errorMessage;
 
         public PartitionResponse(Errors error) {
@@ -322,19 +329,19 @@ public PartitionResponse(Errors error) {
         }
 
         public PartitionResponse(Errors error, long baseOffset, long logAppendTime, long logStartOffset) {
-            this(error, baseOffset, logAppendTime, logStartOffset, Collections.emptyMap(), null);
+            this(error, baseOffset, logAppendTime, logStartOffset, Collections.emptyList(), null);
         }
 
-        public PartitionResponse(Errors error, long baseOffset, long logAppendTime, long logStartOffset, Map errorRecords) {
-            this(error, baseOffset, logAppendTime, logStartOffset, errorRecords, "");
+        public PartitionResponse(Errors error, long baseOffset, long logAppendTime, long logStartOffset, List recordErrors) {
+            this(error, baseOffset, logAppendTime, logStartOffset, recordErrors, null);
         }
 
-        public PartitionResponse(Errors error, long baseOffset, long logAppendTime, long logStartOffset, Map errorRecords, String errorMessage) {
+        public PartitionResponse(Errors error, long baseOffset, long logAppendTime, long logStartOffset, List recordErrors, String errorMessage) {
             this.error = error;
             this.baseOffset = baseOffset;
             this.logAppendTime = logAppendTime;
             this.logStartOffset = logStartOffset;
-            this.errorRecords = errorRecords;
+            this.recordErrors = recordErrors;
             this.errorMessage = errorMessage;
         }
 
@@ -350,15 +357,35 @@ public String toString() {
             b.append(logAppendTime);
             b.append(", logStartOffset: ");
             b.append(logStartOffset);
-            b.append(", errorRecords: ");
-            b.append(errorRecords);
+            b.append(", recordErrors: ");
+            b.append(recordErrors);
             b.append(", errorMessage: ");
-            b.append(errorMessage);
+            if (errorMessage != null) {
+                b.append(errorMessage);
+            } else {
+                b.append("null");
+            }
             b.append('}');
             return b.toString();
         }
     }
 
+    public static final class RecordError {
+        public final int batchIndex;
+        public final String message;
+
+        public RecordError(int batchIndex, String message) {
+            this.batchIndex = batchIndex;
+            this.message = message;
+        }
+
+        public RecordError(int batchIndex) {
+            this.batchIndex = batchIndex;
+            this.message = null;
+        }
+
+    }
+
     public static ProduceResponse parse(ByteBuffer buffer, short version) {
         return new ProduceResponse(ApiKeys.PRODUCE.responseSchema(version).read(buffer));
     }
diff --git a/clients/src/main/resources/common/message/ProduceRequest.json b/clients/src/main/resources/common/message/ProduceRequest.json
index 4fcedf10da909..a872da75c7e9b 100644
--- a/clients/src/main/resources/common/message/ProduceRequest.json
+++ b/clients/src/main/resources/common/message/ProduceRequest.json
@@ -29,7 +29,7 @@
   //
   // Starting in version 7, records can be produced using ZStandard compression.  See KIP-110.
   //
-  // Version 8 is the same as version 7 (but see KIP-467 for the response changes).
+  // Starting in Version 8, response has RecordErrors and ErrorMEssage. See KIP-467.
   "validVersions": "0-8",
   "flexibleVersions": "none",
   "fields": [
diff --git a/clients/src/main/resources/common/message/ProduceResponse.json b/clients/src/main/resources/common/message/ProduceResponse.json
index c21b0aa93accd..77ab0655be57f 100644
--- a/clients/src/main/resources/common/message/ProduceResponse.json
+++ b/clients/src/main/resources/common/message/ProduceResponse.json
@@ -28,7 +28,7 @@
   // Version 5 added LogStartOffset to filter out spurious
   // OutOfOrderSequenceExceptions on the client.
   //
-  // Version 8 added ErrorRecords and ErrorMessage to include information about
+  // Version 8 added RecordErrors and ErrorMessage to include information about
   // records that cause the whole batch to be dropped.  See KIP-467 for details.
   "validVersions": "0-8",
   "flexibleVersions": "none",
@@ -49,11 +49,11 @@
           "about": "The timestamp returned by broker after appending the messages. If CreateTime is used for the topic, the timestamp will be -1.  If LogAppendTime is used for the topic, the timestamp will be the broker local time when the messages are appended." },
         { "name": "LogStartOffset", "type": "int64", "versions": "5+", "default": "-1", "ignorable": true,
           "about": "The log start offset." },
-        { "name": "ErrorRecords", "type": "[]RelativeOffsetAndErrorMessage", "versions": "8+", "ignorable": true,
-          "about": "The relative offsets of records that caused the batch to be dropped", "fields": [
-          { "name":  "RelativeOffset", "type": "int32", "versions":  "8+",
-            "about":  "The relative offset of the record that cause the batch to be dropped" },
-          { "name": "RelativeOffsetErrorMessage", "type": "string", "default": "null", "versions": "8+", "nullableVersions": "8+",
+        { "name": "RecordErrors", "type": "[]BatchIndexAndErrorMessage", "versions": "8+", "ignorable": true,
+          "about": "The batch indices of records that caused the batch to be dropped", "fields": [
+          { "name":  "BatchIndex", "type": "int32", "versions":  "8+",
+            "about":  "The batch index of the record that cause the batch to be dropped" },
+          { "name": "BatchIndexErrorMessage", "type": "string", "default": "null", "versions": "8+", "nullableVersions": "8+",
             "about":  "The error message of the record that caused the batch to be dropped"}
         ]},
         { "name":  "ErrorMessage", "type": "string", "default": "null", "versions": "8+", "nullableVersions": "8+", "ignorable":  true,
diff --git a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
index 73bb2afd97ca4..c7dedf94cdbf7 100644
--- a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
@@ -516,8 +516,8 @@ public void testProduceResponseVersions() throws Exception {
         int throttleTimeMs = 1234;
         long logAppendTimeMs = 1234L;
         long logStartOffset = 1234L;
-        int relativeOffset = 0;
-        String relativeOffsetErrorMessage = "error message";
+        int batchIndex = 0;
+        String batchIndexErrorMessage = "error message";
         String errorMessage = "global error message";
 
         testAllMessageRoundTrips(new ProduceResponseData()
@@ -542,10 +542,10 @@ public void testProduceResponseVersions() throws Exception {
                                          .setBaseOffset(baseOffset)
                                          .setLogAppendTimeMs(logAppendTimeMs)
                                          .setLogStartOffset(logStartOffset)
-                                         .setErrorRecords(singletonList(
-                                             new ProduceResponseData.RelativeOffsetAndErrorMessage()
-                                                 .setRelativeOffset(relativeOffset)
-                                                 .setRelativeOffsetErrorMessage(relativeOffsetErrorMessage)))
+                                         .setRecordErrors(singletonList(
+                                             new ProduceResponseData.BatchIndexAndErrorMessage()
+                                                 .setBatchIndex(batchIndex)
+                                                 .setBatchIndexErrorMessage(batchIndexErrorMessage)))
                                          .setErrorMessage(errorMessage)))))
                       .setThrottleTimeMs(throttleTimeMs);
 
@@ -553,7 +553,7 @@ public void testProduceResponseVersions() throws Exception {
             ProduceResponseData responseData = response.get();
 
             if (version < 8) {
-                responseData.responses().get(0).partitions().get(0).setErrorRecords(Collections.emptyList());
+                responseData.responses().get(0).partitions().get(0).setRecordErrors(Collections.emptyList());
                 responseData.responses().get(0).partitions().get(0).setErrorMessage(null);
             }
 
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index 32b0a00679eef..c8bd7518c67c9 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -1223,7 +1223,8 @@ private ProduceResponse createProduceResponse() {
     private ProduceResponse createProduceResponseWithErrorMessage() {
         Map responseData = new HashMap<>();
         responseData.put(new TopicPartition("test", 0), new ProduceResponse.PartitionResponse(Errors.NONE,
-                10000, RecordBatch.NO_TIMESTAMP, 100, Collections.singletonMap(0, "error message"), "global error message"));
+                10000, RecordBatch.NO_TIMESTAMP, 100, Collections.singletonList(new ProduceResponse.RecordError(0, "error message")),
+                "global error message"));
         return new ProduceResponse(responseData, 0);
     }
 
diff --git a/core/src/main/scala/kafka/common/RecordValidationException.scala b/core/src/main/scala/kafka/common/RecordValidationException.scala
new file mode 100644
index 0000000000000..2acdf84c00c06
--- /dev/null
+++ b/core/src/main/scala/kafka/common/RecordValidationException.scala
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.common
+
+import org.apache.kafka.common.errors.ApiException
+import org.apache.kafka.common.requests.ProduceResponse.RecordError
+
+class RecordValidationException(val invalidException: ApiException,
+                                val recordErrors: List[RecordError]) extends RuntimeException {
+}
diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index b0af105f5042b..94997a19a4a51 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -40,6 +40,7 @@ import org.apache.kafka.common.errors._
 import org.apache.kafka.common.record.FileRecords.TimestampAndOffset
 import org.apache.kafka.common.record._
 import org.apache.kafka.common.requests.FetchResponse.AbortedTransaction
+import org.apache.kafka.common.requests.ProduceResponse.RecordError
 import org.apache.kafka.common.requests.{EpochEndOffset, ListOffsetRequest}
 import org.apache.kafka.common.utils.{Time, Utils}
 import org.apache.kafka.common.{InvalidRecordException, KafkaException, TopicPartition}
@@ -54,7 +55,18 @@ object LogAppendInfo {
 
   def unknownLogAppendInfoWithLogStartOffset(logStartOffset: Long): LogAppendInfo =
     LogAppendInfo(None, -1, RecordBatch.NO_TIMESTAMP, -1L, RecordBatch.NO_TIMESTAMP, logStartOffset,
-      RecordConversionStats.EMPTY, NoCompressionCodec, NoCompressionCodec, -1, -1, offsetsMonotonic = false, -1L)
+      RecordConversionStats.EMPTY, NoCompressionCodec, NoCompressionCodec, -1, -1,
+      offsetsMonotonic = false, -1L)
+
+  /**
+   * In ProduceResponse V8+, we add two new fields record_errors and error_message (see KIP-467).
+   * For any record failures with InvalidTimestamp or InvalidRecordException, we construct a LogAppendInfo object like the one
+   * in unknownLogAppendInfoWithLogStartOffset, but with additiona fields recordErrors and errorMessage
+   */
+  def unknownLogAppendInfoWithAdditionalInfo(logStartOffset: Long, recordErrors: List[RecordError], errorMessage: String): LogAppendInfo =
+    LogAppendInfo(None, -1, RecordBatch.NO_TIMESTAMP, -1L, RecordBatch.NO_TIMESTAMP, logStartOffset,
+      RecordConversionStats.EMPTY, NoCompressionCodec, NoCompressionCodec, -1, -1,
+      offsetsMonotonic = false, -1L, recordErrors, errorMessage)
 }
 
 /**
@@ -87,7 +99,9 @@ case class LogAppendInfo(var firstOffset: Option[Long],
                          shallowCount: Int,
                          validBytes: Int,
                          offsetsMonotonic: Boolean,
-                         lastOffsetOfFirstBatch: Long) {
+                         lastOffsetOfFirstBatch: Long,
+                         recordErrors: List[RecordError] = List(),
+                         errorMessage: String = null) {
   /**
    * Get the first offset if it exists, else get the last offset of the first batch
    * For magic versions 2 and newer, this method will return first offset. For magic versions
diff --git a/core/src/main/scala/kafka/log/LogValidator.scala b/core/src/main/scala/kafka/log/LogValidator.scala
index 6bba8eb2d769e..70bf3bf00da15 100644
--- a/core/src/main/scala/kafka/log/LogValidator.scala
+++ b/core/src/main/scala/kafka/log/LogValidator.scala
@@ -19,7 +19,7 @@ package kafka.log
 import java.nio.ByteBuffer
 
 import kafka.api.{ApiVersion, KAFKA_2_1_IV0}
-import kafka.common.LongRef
+import kafka.common.{LongRef, RecordValidationException}
 import kafka.message.{CompressionCodec, NoCompressionCodec, ZStdCompressionCodec}
 import kafka.server.BrokerTopicStats
 import kafka.utils.Logging
@@ -27,6 +27,7 @@ import org.apache.kafka.common.errors.{CorruptRecordException, InvalidTimestampE
 import org.apache.kafka.common.record.{AbstractRecords, BufferSupplier, CompressionType, MemoryRecords, Record, RecordBatch, RecordConversionStats, TimestampType}
 import org.apache.kafka.common.InvalidRecordException
 import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.requests.ProduceResponse.RecordError
 import org.apache.kafka.common.utils.Time
 
 import scala.collection.{Seq, mutable}
@@ -146,11 +147,14 @@ private[kafka] object LogValidator extends Logging {
       throw new UnsupportedForMessageFormatException(s"Idempotent records cannot be used with magic version $toMagic")
   }
 
-  private def validateRecord(batch: RecordBatch, topicPartition: TopicPartition, record: Record, now: Long, timestampType: TimestampType,
-                             timestampDiffMaxMs: Long, compactedTopic: Boolean, brokerTopicStats: BrokerTopicStats): Unit = {
+  private def validateRecord(batch: RecordBatch, topicPartition: TopicPartition, record: Record, batchIndex: Int, now: Long,
+                             timestampType: TimestampType, timestampDiffMaxMs: Long, compactedTopic: Boolean,
+                             brokerTopicStats: BrokerTopicStats): Unit = {
     if (!record.hasMagic(batch.magic)) {
       brokerTopicStats.allTopicsStats.invalidMagicNumberRecordsPerSec.mark()
-      throw new InvalidRecordException(s"Log record $record's magic does not match outer magic ${batch.magic} in topic partition $topicPartition.")
+      throw new RecordValidationException(
+        new InvalidRecordException(s"Log record $record's magic does not match outer magic ${batch.magic} in topic partition $topicPartition."),
+        List(new RecordError(batchIndex)))
     }
 
     // verify the record-level CRC only if this is one of the deep entries of a compressed message
@@ -167,8 +171,8 @@ private[kafka] object LogValidator extends Logging {
       }
     }
 
-    validateKey(record, topicPartition, compactedTopic, brokerTopicStats)
-    validateTimestamp(batch, record, now, timestampType, timestampDiffMaxMs)
+    validateKey(record, batchIndex, topicPartition, compactedTopic, brokerTopicStats)
+    validateTimestamp(batch, record, batchIndex, now, timestampType, timestampDiffMaxMs)
   }
 
   private def convertAndAssignOffsetsNonCompressed(records: MemoryRecords,
@@ -201,8 +205,8 @@ private[kafka] object LogValidator extends Logging {
     for (batch <- records.batches.asScala) {
       validateBatch(topicPartition, firstBatch, batch, isFromClient, toMagicValue, brokerTopicStats)
 
-      for (record <- batch.asScala) {
-        validateRecord(batch, topicPartition, record, now, timestampType, timestampDiffMaxMs, compactedTopic, brokerTopicStats)
+      for ((record, batchIndex) <- batch.asScala.view.zipWithIndex) {
+        validateRecord(batch, topicPartition, record, batchIndex, now, timestampType, timestampDiffMaxMs, compactedTopic, brokerTopicStats)
         builder.appendWithOffset(offsetCounter.getAndIncrement(), record)
       }
     }
@@ -232,6 +236,7 @@ private[kafka] object LogValidator extends Logging {
                                          magic: Byte,
                                          brokerTopicStats: BrokerTopicStats): ValidationAndOffsetAssignResult = {
     var maxTimestamp = RecordBatch.NO_TIMESTAMP
+    val expectedInnerOffset = new LongRef(0)
     var offsetOfMaxTimestamp = -1L
     val initialOffset = offsetCounter.value
 
@@ -243,8 +248,19 @@ private[kafka] object LogValidator extends Logging {
       var maxBatchTimestamp = RecordBatch.NO_TIMESTAMP
       var offsetOfMaxBatchTimestamp = -1L
 
-      for (record <- batch.asScala) {
-        validateRecord(batch, topicPartition, record, now, timestampType, timestampDiffMaxMs, compactedTopic, brokerTopicStats)
+      for ((record, batchIndex) <- batch.asScala.view.zipWithIndex) {
+        validateRecord(batch, topicPartition, record, batchIndex, now, timestampType, timestampDiffMaxMs, compactedTopic, brokerTopicStats)
+
+        val expectedOffset = expectedInnerOffset.getAndIncrement()
+
+        // inner records offset should always be continuous
+        if (record.offset != expectedOffset) {
+          brokerTopicStats.allTopicsStats.invalidOffsetOrSequenceRecordsPerSec.mark()
+          throw new RecordValidationException(
+            new InvalidRecordException(s"Inner record $record inside the compressed record batch does not have incremental offsets, expected offset is $expectedOffset in topic partition $topicPartition."),
+            List(new RecordError(batchIndex)))
+        }
+
         val offset = offsetCounter.getAndIncrement()
         if (batch.magic > RecordBatch.MAGIC_VALUE_V0 && record.timestamp > maxBatchTimestamp) {
           maxBatchTimestamp = record.timestamp
@@ -349,11 +365,13 @@ private[kafka] object LogValidator extends Logging {
         batch.streamingIterator(BufferSupplier.NO_CACHING)
 
       try {
-        for (record <- batch.asScala) {
+        for ((record, batchIndex) <- batch.asScala.view.zipWithIndex) {
           if (sourceCodec != NoCompressionCodec && record.isCompressed)
-            throw new InvalidRecordException("Compressed outer record should not have an inner record with a " +
-              s"compression attribute set: $record")
-          validateRecord(batch, topicPartition, record, now, timestampType, timestampDiffMaxMs, compactedTopic, brokerTopicStats)
+            throw new RecordValidationException(
+              new InvalidRecordException(s"Compressed outer record should not have an inner record with a compression attribute set: $record"),
+              List(new RecordError(batchIndex)))
+
+          validateRecord(batch, topicPartition, record, batchIndex, now, timestampType, timestampDiffMaxMs, compactedTopic, brokerTopicStats)
 
           uncompressedSizeInBytes += record.sizeInBytes()
           if (batch.magic > RecordBatch.MAGIC_VALUE_V0 && toMagic > RecordBatch.MAGIC_VALUE_V0) {
@@ -361,7 +379,9 @@ private[kafka] object LogValidator extends Logging {
             val expectedOffset = expectedInnerOffset.getAndIncrement()
             if (record.offset != expectedOffset) {
               brokerTopicStats.allTopicsStats.invalidOffsetOrSequenceRecordsPerSec.mark()
-              throw new InvalidRecordException(s"Inner record $record inside the compressed record batch does not have incremental offsets, expected offset is $expectedOffset in topic partition $topicPartition.")
+              throw new RecordValidationException(
+                new InvalidRecordException(s"Inner record $record inside the compressed record batch does not have incremental offsets, expected offset is $expectedOffset in topic partition $topicPartition."),
+                List(new RecordError(batchIndex)))
             }
             if (record.timestamp > maxTimestamp)
               maxTimestamp = record.timestamp
@@ -456,10 +476,12 @@ private[kafka] object LogValidator extends Logging {
       recordConversionStats = recordConversionStats)
   }
 
-  private def validateKey(record: Record, topicPartition: TopicPartition, compactedTopic: Boolean, brokerTopicStats: BrokerTopicStats) {
+  private def validateKey(record: Record, batchIndex: Int, topicPartition: TopicPartition, compactedTopic: Boolean, brokerTopicStats: BrokerTopicStats) {
     if (compactedTopic && !record.hasKey) {
       brokerTopicStats.allTopicsStats.noKeyCompactedTopicRecordsPerSec.mark()
-      throw new InvalidRecordException(s"Compacted topic cannot accept message without key in topic partition $topicPartition.")
+      throw new RecordValidationException(
+        new InvalidRecordException(s"Compacted topic cannot accept message without key in topic partition $topicPartition."),
+        List(new RecordError(batchIndex)))
     }
   }
 
@@ -469,17 +491,22 @@ private[kafka] object LogValidator extends Logging {
    */
   private def validateTimestamp(batch: RecordBatch,
                                 record: Record,
+                                batchIndex: Int,
                                 now: Long,
                                 timestampType: TimestampType,
                                 timestampDiffMaxMs: Long): Unit = {
     if (timestampType == TimestampType.CREATE_TIME
       && record.timestamp != RecordBatch.NO_TIMESTAMP
       && math.abs(record.timestamp - now) > timestampDiffMaxMs)
-      throw new InvalidTimestampException(s"Timestamp ${record.timestamp} of message with offset ${record.offset} is " +
-        s"out of range. The timestamp should be within [${now - timestampDiffMaxMs}, ${now + timestampDiffMaxMs}]")
+      throw new RecordValidationException(
+        new InvalidTimestampException(s"Timestamp ${record.timestamp} of message with offset ${record.offset} is " +
+          s"out of range. The timestamp should be within [${now - timestampDiffMaxMs}, ${now + timestampDiffMaxMs}]"),
+        List(new RecordError(batchIndex)))
     if (batch.timestampType == TimestampType.LOG_APPEND_TIME)
-      throw new InvalidTimestampException(s"Invalid timestamp type in message $record. Producer should not set " +
-        s"timestamp type to LogAppendTime.")
+      throw new RecordValidationException(
+        new InvalidTimestampException(s"Invalid timestamp type in message $record. Producer should not set " +
+          s"timestamp type to LogAppendTime."),
+        List(new RecordError(batchIndex)))
   }
 
   case class ValidationAndOffsetAssignResult(validatedRecords: MemoryRecords,
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index 8a74aa7f8908e..d10c93645c36a 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -25,6 +25,7 @@ import java.util.concurrent.locks.Lock
 import com.yammer.metrics.core.{Gauge, Meter}
 import kafka.api._
 import kafka.cluster.{BrokerEndPoint, Partition}
+import kafka.common.RecordValidationException
 import kafka.controller.{KafkaController, StateChangeLogger}
 import kafka.log._
 import kafka.metrics.KafkaMetricsGroup
@@ -32,9 +33,7 @@ import kafka.server.QuotaFactory.QuotaManagers
 import kafka.server.checkpoints.{LazyOffsetCheckpoints, OffsetCheckpointFile, OffsetCheckpoints}
 import kafka.utils._
 import kafka.zk.KafkaZkClient
-import org.apache.kafka.common.ElectionType
-import org.apache.kafka.common.TopicPartition
-import org.apache.kafka.common.Node
+import org.apache.kafka.common.{ElectionType, Node, TopicPartition}
 import org.apache.kafka.common.errors._
 import org.apache.kafka.common.internals.Topic
 import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
@@ -499,7 +498,8 @@ class ReplicaManager(val config: KafkaConfig,
         topicPartition ->
                 ProducePartitionStatus(
                   result.info.lastOffset + 1, // required offset
-                  new PartitionResponse(result.error, result.info.firstOffset.getOrElse(-1), result.info.logAppendTime, result.info.logStartOffset)) // response status
+                  new PartitionResponse(result.error, result.info.firstOffset.getOrElse(-1), result.info.logAppendTime,
+                    result.info.logStartOffset, result.info.recordErrors.asJava, result.info.errorMessage)) // response status
       }
 
       recordConversionStatsCallback(localProduceResults.map { case (k, v) => k -> v.info.recordConversionStats })
@@ -753,6 +753,19 @@ class ReplicaManager(val config: KafkaConfig,
                                isFromClient: Boolean,
                                entriesPerPartition: Map[TopicPartition, MemoryRecords],
                                requiredAcks: Short): Map[TopicPartition, LogAppendResult] = {
+
+    def processFailedRecord(topicPartition: TopicPartition, t: Throwable) = {
+      val logStartOffset = getPartition(topicPartition) match {
+        case HostedPartition.Online(partition) => partition.logStartOffset
+        case HostedPartition.None | HostedPartition.Offline => -1L
+      }
+      brokerTopicStats.topicStats(topicPartition.topic).failedProduceRequestRate.mark()
+      brokerTopicStats.allTopicsStats.failedProduceRequestRate.mark()
+      error(s"Error processing append operation on partition $topicPartition", t)
+
+      logStartOffset
+    }
+
     trace(s"Append [$entriesPerPartition] to local log")
     entriesPerPartition.map { case (topicPartition, records) =>
       brokerTopicStats.topicStats(topicPartition.topic).totalProduceRequestRate.mark()
@@ -786,17 +799,15 @@ class ReplicaManager(val config: KafkaConfig,
                    _: RecordTooLargeException |
                    _: RecordBatchTooLargeException |
                    _: CorruptRecordException |
-                   _: KafkaStorageException |
-                   _: InvalidTimestampException) =>
+                   _: KafkaStorageException) =>
             (topicPartition, LogAppendResult(LogAppendInfo.UnknownLogAppendInfo, Some(e)))
+          case rve: RecordValidationException =>
+            val logStartOffset = processFailedRecord(topicPartition, rve.invalidException)
+            val recordErrors = rve.recordErrors
+            (topicPartition, LogAppendResult(LogAppendInfo.unknownLogAppendInfoWithAdditionalInfo(
+              logStartOffset, recordErrors, rve.invalidException.getMessage), Some(rve.invalidException)))
           case t: Throwable =>
-            val logStartOffset = getPartition(topicPartition) match {
-              case HostedPartition.Online(partition) => partition.logStartOffset
-              case HostedPartition.None | HostedPartition.Offline => -1L
-            }
-            brokerTopicStats.topicStats(topicPartition.topic).failedProduceRequestRate.mark()
-            brokerTopicStats.allTopicsStats.failedProduceRequestRate.mark()
-            error(s"Error processing append operation on partition $topicPartition", t)
+            val logStartOffset = processFailedRecord(topicPartition, t)
             (topicPartition, LogAppendResult(LogAppendInfo.unknownLogAppendInfoWithLogStartOffset(logStartOffset), Some(t)))
         }
       }
diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala
index 29b564e066049..c5de28e8e13c1 100755
--- a/core/src/test/scala/unit/kafka/log/LogTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogTest.scala
@@ -25,13 +25,13 @@ import java.util.{Collections, Optional, Properties}
 
 import com.yammer.metrics.Metrics
 import kafka.api.{ApiVersion, KAFKA_0_11_0_IV0}
-import kafka.common.{OffsetsOutOfOrderException, UnexpectedAppendOffsetException}
+import kafka.common.{OffsetsOutOfOrderException, RecordValidationException, UnexpectedAppendOffsetException}
 import kafka.log.Log.DeleteDirSuffix
 import kafka.server.checkpoints.LeaderEpochCheckpointFile
 import kafka.server.epoch.{EpochEntry, LeaderEpochFileCache}
 import kafka.server.{BrokerTopicStats, FetchDataInfo, FetchHighWatermark, FetchIsolation, FetchLogEnd, FetchTxnCommitted, KafkaConfig, LogDirFailureChannel, LogOffsetMetadata}
 import kafka.utils._
-import org.apache.kafka.common.{InvalidRecordException, KafkaException, TopicPartition}
+import org.apache.kafka.common.{KafkaException, TopicPartition}
 import org.apache.kafka.common.errors._
 import org.apache.kafka.common.record.FileRecords.TimestampAndOffset
 import org.apache.kafka.common.record.MemoryRecords.RecordFilter
@@ -1850,19 +1850,19 @@ class LogTest {
       log.appendAsLeader(messageSetWithUnkeyedMessage, leaderEpoch = 0)
       fail("Compacted topics cannot accept a message without a key.")
     } catch {
-      case _: InvalidRecordException => // this is good
+      case _: RecordValidationException => // this is good
     }
     try {
       log.appendAsLeader(messageSetWithOneUnkeyedMessage, leaderEpoch = 0)
       fail("Compacted topics cannot accept a message without a key.")
     } catch {
-      case _: InvalidRecordException => // this is good
+      case _: RecordValidationException => // this is good
     }
     try {
       log.appendAsLeader(messageSetWithCompressedUnkeyedMessage, leaderEpoch = 0)
       fail("Compacted topics cannot accept a message without a key.")
     } catch {
-      case _: InvalidRecordException => // this is good
+      case _: RecordValidationException => // this is good
     }
 
     // check if metric for NoKeyCompactedTopicRecordsPerSec is logged
diff --git a/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala b/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
index 7fd13210a8ccc..923ae9185211f 100644
--- a/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
@@ -21,7 +21,7 @@ import java.util.concurrent.TimeUnit
 
 import com.yammer.metrics.Metrics
 import kafka.api.{ApiVersion, KAFKA_2_0_IV1, KAFKA_2_3_IV1}
-import kafka.common.LongRef
+import kafka.common.{LongRef, RecordValidationException}
 import kafka.message._
 import kafka.server.BrokerTopicStats
 import kafka.utils.TestUtils.meterCount
@@ -81,7 +81,7 @@ class LogValidatorTest {
   }
 
   private def checkMismatchMagic(batchMagic: Byte, recordMagic: Byte, compressionType: CompressionType): Unit = {
-    assertThrows[InvalidRecordException] {
+    assertThrows[RecordValidationException] {
       validateMessages(recordsWithInvalidInnerMagic(batchMagic, recordMagic, compressionType), batchMagic, compressionType, compressionType)
     }
     assertEquals(metricsKeySet.count(_.getMBeanName.endsWith(s"${BrokerTopicStats.InvalidMagicNumberRecordsPerSec}")), 1)
@@ -574,7 +574,7 @@ class LogValidatorTest {
     checkCompressed(RecordBatch.MAGIC_VALUE_V2)
   }
 
-  @Test(expected = classOf[InvalidTimestampException])
+  @Test(expected = classOf[RecordValidationException])
   def testInvalidCreateTimeNonCompressedV1(): Unit = {
     val now = System.currentTimeMillis()
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V1, timestamp = now - 1001L,
@@ -597,7 +597,7 @@ class LogValidatorTest {
       brokerTopicStats = brokerTopicStats)
   }
 
-  @Test(expected = classOf[InvalidTimestampException])
+  @Test(expected = classOf[RecordValidationException])
   def testInvalidCreateTimeNonCompressedV2(): Unit = {
     val now = System.currentTimeMillis()
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V2, timestamp = now - 1001L,
@@ -620,7 +620,7 @@ class LogValidatorTest {
       brokerTopicStats = brokerTopicStats)
   }
 
-  @Test(expected = classOf[InvalidTimestampException])
+  @Test(expected = classOf[RecordValidationException])
   def testInvalidCreateTimeCompressedV1(): Unit = {
     val now = System.currentTimeMillis()
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V1, timestamp = now - 1001L,
@@ -643,7 +643,7 @@ class LogValidatorTest {
       brokerTopicStats = brokerTopicStats)
   }
 
-  @Test(expected = classOf[InvalidTimestampException])
+  @Test(expected = classOf[RecordValidationException])
   def testInvalidCreateTimeCompressedV2(): Unit = {
     val now = System.currentTimeMillis()
     val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V2, timestamp = now - 1001L,
@@ -1250,6 +1250,52 @@ class LogValidatorTest {
     testBatchWithoutRecordsNotAllowed(NoCompressionCodec, DefaultCompressionCodec)
   }
 
+  @Test
+  def testInvalidTimestampExceptionHasBatchIndex(): Unit = {
+    val now = System.currentTimeMillis()
+    val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V2, timestamp = now - 1001L,
+      codec = CompressionType.GZIP)
+    val e = intercept[RecordValidationException] {
+      LogValidator.validateMessagesAndAssignOffsets(
+        records,
+        topicPartition,
+        offsetCounter = new LongRef(0),
+        time = time,
+        now = System.currentTimeMillis(),
+        sourceCodec = DefaultCompressionCodec,
+        targetCodec = DefaultCompressionCodec,
+        magic = RecordBatch.MAGIC_VALUE_V1,
+        compactedTopic = false,
+        timestampType = TimestampType.CREATE_TIME,
+        timestampDiffMaxMs = 1000L,
+        partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
+        isFromClient = true,
+        interBrokerProtocolVersion = ApiVersion.latestVersion,
+        brokerTopicStats = brokerTopicStats)
+    }
+
+    assertTrue(e.invalidException.isInstanceOf[InvalidTimestampException])
+    assertTrue(e.recordErrors.nonEmpty)
+    assertEquals(e.recordErrors.size, 1)
+    assertEquals(e.recordErrors.head.batchIndex, 0)
+    assertNull(e.recordErrors.head.message)
+  }
+
+  @Test
+  def testInvalidRecordExceptionHasBatchIndex(): Unit = {
+    val e = intercept[RecordValidationException] {
+      validateMessages(recordsWithInvalidInnerMagic(
+        RecordBatch.MAGIC_VALUE_V0, RecordBatch.MAGIC_VALUE_V1, CompressionType.GZIP),
+        RecordBatch.MAGIC_VALUE_V0, CompressionType.GZIP, CompressionType.GZIP)
+    }
+
+    assertTrue(e.invalidException.isInstanceOf[InvalidRecordException])
+    assertTrue(e.recordErrors.nonEmpty)
+    assertEquals(e.recordErrors.size, 1)
+    assertEquals(e.recordErrors.head.batchIndex, 0)
+    assertNull(e.recordErrors.head.message)
+  }
+
   private def testBatchWithoutRecordsNotAllowed(sourceCodec: CompressionCodec, targetCodec: CompressionCodec): Unit = {
     val offset = 1234567
     val (producerId, producerEpoch, baseSequence, isTransactional, partitionLeaderEpoch) =
diff --git a/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala b/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala
index bedf6ff0f00e8..3bc8d0aacb8f3 100644
--- a/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala
@@ -17,6 +17,7 @@
 
 package kafka.server
 
+import java.nio.ByteBuffer
 import java.util.Properties
 
 import com.yammer.metrics.Metrics
@@ -56,7 +57,7 @@ class ProduceRequestTest extends BaseRequestTest {
       assertEquals(Errors.NONE, partitionResponse.error)
       assertEquals(expectedOffset, partitionResponse.baseOffset)
       assertEquals(-1, partitionResponse.logAppendTime)
-      assertTrue(partitionResponse.errorRecords.isEmpty)
+      assertTrue(partitionResponse.recordErrors.isEmpty)
       partitionResponse
     }
 
@@ -68,6 +69,39 @@ class ProduceRequestTest extends BaseRequestTest {
       new SimpleRecord(System.currentTimeMillis(), "key2".getBytes, "value2".getBytes)), 1)
   }
 
+  @Test
+  def testProduceWithInvalidTimestamp(): Unit = {
+    val topic = "topic"
+    val partition = 0
+    val topicConfig = new Properties
+    topicConfig.setProperty(LogConfig.MessageTimestampDifferenceMaxMsProp, "1000")
+    val partitionToLeader = TestUtils.createTopic(zkClient, topic, 1, 1, servers, topicConfig)
+    val leader = partitionToLeader(partition)
+
+    def createRecords(magicValue: Byte,
+                      timestamp: Long = RecordBatch.NO_TIMESTAMP,
+                      codec: CompressionType): MemoryRecords = {
+      val buf = ByteBuffer.allocate(512)
+      val builder = MemoryRecords.builder(buf, magicValue, codec, TimestampType.CREATE_TIME, 0L)
+      builder.appendWithOffset(0, timestamp, null, "hello".getBytes)
+      builder.appendWithOffset(1, timestamp, null, "there".getBytes)
+      builder.appendWithOffset(2, timestamp, null, "beautiful".getBytes)
+      builder.build()
+    }
+
+    val records = createRecords(RecordBatch.MAGIC_VALUE_V2, System.currentTimeMillis() - 1001L, CompressionType.GZIP)
+    val topicPartition = new TopicPartition("topic", partition)
+    val partitionRecords = Map(topicPartition -> records)
+    val produceResponse = sendProduceRequest(leader, ProduceRequest.Builder.forCurrentMagic(-1, 3000, partitionRecords.asJava).build())
+    val (tp, partitionResponse) = produceResponse.responses.asScala.head
+    assertEquals(topicPartition, tp)
+    assertEquals(Errors.INVALID_TIMESTAMP, partitionResponse.error)
+    assertEquals(1, partitionResponse.recordErrors.size())
+    assertEquals(0, partitionResponse.recordErrors.get(0).batchIndex)
+    assertNull(partitionResponse.recordErrors.get(0).message)
+    assertNotNull(partitionResponse.errorMessage)
+  }
+
   @Test
   def testProduceToNonReplica(): Unit = {
     val topic = "topic"

From 2f01a43575dc3f5a6250547382c5bb090a295d0f Mon Sep 17 00:00:00 2001
From: David Jacot 
Date: Fri, 11 Oct 2019 00:12:14 +0200
Subject: [PATCH 0709/1071] MINOR: ListPartitionReassignmentsResponse should
 not be entirely failed when a topic-partition does not exist  (#7486)

Reviewers: Colin P. McCabe 
(cherry picked from commit fa2a9f09e4042f821d7373e2d9e01b21aede775a)
---
 .../scala/kafka/controller/KafkaController.scala   |  5 +----
 .../kafka/api/AdminClientIntegrationTest.scala     | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala
index e4679d3ae978e..91c4534e143d9 100644
--- a/core/src/main/scala/kafka/controller/KafkaController.scala
+++ b/core/src/main/scala/kafka/controller/KafkaController.scala
@@ -1821,10 +1821,7 @@ class KafkaController(val config: KafkaConfig,
 
       partitionsToList.foreach { tp =>
         val assignment = controllerContext.partitionFullReplicaAssignment(tp)
-        if (assignment.replicas.isEmpty) {
-          callback(Right(new ApiError(Errors.UNKNOWN_TOPIC_OR_PARTITION)))
-          return
-        } else if (assignment.isBeingReassigned) {
+        if (assignment.isBeingReassigned) {
           results += tp -> assignment
         }
       }
diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index c4194363b3053..f2e7af6b26ddc 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -1819,6 +1819,20 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     assertEquals(0, allReassignmentsMap.size())
   }
 
+  @Test
+  def testListReassignmentsDoesNotShowDeletedPartitions(): Unit = {
+    client = AdminClient.create(createConfig())
+    
+    val topic = "list-reassignments-no-reassignments"
+    val tp = new TopicPartition(topic, 0)
+
+    val reassignmentsMap = client.listPartitionReassignments(Set(tp).asJava).reassignments().get()
+    assertEquals(0, reassignmentsMap.size())
+
+    val allReassignmentsMap = client.listPartitionReassignments().reassignments().get()
+    assertEquals(0, allReassignmentsMap.size())
+  }
+
   @Test
   def testValidIncrementalAlterConfigs(): Unit = {
     client = AdminClient.create(createConfig)

From de202a669777becd4a79fcdf6c0597e5a12d2370 Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Thu, 10 Oct 2019 16:23:18 -0700
Subject: [PATCH 0710/1071] KAFKA-8743: Flaky Test
 Repartition{WithMerge}OptimizingIntegrationTest (#7472)

All four flavors of the repartition/optimization tests have been reported as flaky and failed in one place or another:
* RepartitionOptimizingIntegrationTest.shouldSendCorrectRecords_OPTIMIZED
* RepartitionOptimizingIntegrationTest.shouldSendCorrectRecords_NO_OPTIMIZATION
* RepartitionWithMergeOptimizingIntegrationTest.shouldSendCorrectRecords_OPTIMIZED
* RepartitionWithMergeOptimizingIntegrationTest.shouldSendCorrectRecords_NO_OPTIMIZATION

They're pretty similar so it makes sense to knock them all out at once. This PR does three things:

* Switch to in-memory stores wherever possible
* Name all operators and update the Topology accordingly (not really a flaky test fix, but had to update the topology names anyway because of the IM stores so figured might as well)
* Port to TopologyTestDriver -- this is the "real" fix, should make a big difference as these repartition tests required multiple roundtrips with the Kafka cluster (while using only the default timeout)

Reviewers: Bill Bejeck , Guozhang Wang 
---
 .../internals/RepartitionOptimizingTest.java} | 548 +++++++++---------
 .../RepartitionWithMergeOptimizingTest.java}  | 328 ++++++-----
 .../StreamsBrokerDownResilienceTest.java      |  15 +-
 .../tests/streams/streams_upgrade_test.py     |   2 +-
 4 files changed, 465 insertions(+), 428 deletions(-)
 rename streams/src/test/java/org/apache/kafka/streams/{integration/RepartitionOptimizingIntegrationTest.java => processor/internals/RepartitionOptimizingTest.java} (54%)
 rename streams/src/test/java/org/apache/kafka/streams/{integration/RepartitionWithMergeOptimizingIntegrationTest.java => processor/internals/RepartitionWithMergeOptimizingTest.java} (54%)

diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/RepartitionOptimizingIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionOptimizingTest.java
similarity index 54%
rename from streams/src/test/java/org/apache/kafka/streams/integration/RepartitionOptimizingIntegrationTest.java
rename to streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionOptimizingTest.java
index bea32f2197f47..8425ad7001ae1 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/RepartitionOptimizingIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionOptimizingTest.java
@@ -15,40 +15,41 @@
  * limitations under the License.
  */
 
-package org.apache.kafka.streams.integration;
+package org.apache.kafka.streams.processor.internals;
 
-
-import kafka.utils.MockTime;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.kafka.common.serialization.Deserializer;
 import org.apache.kafka.common.serialization.IntegerDeserializer;
 import org.apache.kafka.common.serialization.LongDeserializer;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.Serializer;
 import org.apache.kafka.common.serialization.StringDeserializer;
 import org.apache.kafka.common.serialization.StringSerializer;
-import org.apache.kafka.streams.KafkaStreams;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TestInputTopic;
+import org.apache.kafka.streams.TestOutputTopic;
 import org.apache.kafka.streams.Topology;
-import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
-import org.apache.kafka.streams.integration.utils.IntegrationTestUtils;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.Aggregator;
 import org.apache.kafka.streams.kstream.Consumed;
+import org.apache.kafka.streams.kstream.Grouped;
 import org.apache.kafka.streams.kstream.Initializer;
 import org.apache.kafka.streams.kstream.JoinWindows;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.Materialized;
+import org.apache.kafka.streams.kstream.Named;
 import org.apache.kafka.streams.kstream.Produced;
 import org.apache.kafka.streams.kstream.Reducer;
 import org.apache.kafka.streams.kstream.StreamJoined;
 import org.apache.kafka.streams.processor.AbstractProcessor;
-import org.apache.kafka.test.IntegrationTest;
+import org.apache.kafka.streams.state.Stores;
 import org.apache.kafka.test.StreamsTestUtils;
-import org.apache.kafka.test.TestUtils;
 import org.junit.After;
 import org.junit.Before;
-import org.junit.ClassRule;
 import org.junit.Test;
-import org.junit.experimental.categories.Category;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -57,17 +58,19 @@
 import java.util.Properties;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import static java.time.Duration.ofDays;
 import static java.time.Duration.ofMillis;
-import static java.time.Duration.ofSeconds;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
 
-@Category({IntegrationTest.class})
-public class RepartitionOptimizingIntegrationTest {
+public class RepartitionOptimizingTest {
+
+    private final Logger log = LoggerFactory.getLogger(RepartitionOptimizingTest.class);
 
-    private static final int NUM_BROKERS = 1;
     private static final String INPUT_TOPIC = "input";
     private static final String COUNT_TOPIC = "outputTopic_0";
     private static final String AGGREGATION_TOPIC = "outputTopic_1";
@@ -77,143 +80,168 @@ public class RepartitionOptimizingIntegrationTest {
     private static final int ONE_REPARTITION_TOPIC = 1;
     private static final int FOUR_REPARTITION_TOPICS = 4;
 
+    private final Serializer stringSerializer = new StringSerializer();
+    private final Deserializer stringDeserializer = new StringDeserializer();
+
     private final Pattern repartitionTopicPattern = Pattern.compile("Sink: .*-repartition");
 
     private Properties streamsConfiguration;
+    private TopologyTestDriver topologyTestDriver;
 
+    private final Initializer initializer = () -> 0;
+    private final Aggregator aggregator = (k, v, agg) -> agg + v.length();
+    private final Reducer reducer = (v1, v2) -> v1 + ":" + v2;
 
-    @ClassRule
-    public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(NUM_BROKERS);
-    private final MockTime mockTime = CLUSTER.time;
+    private final List processorValueCollector = new ArrayList<>();
+
+    private final List> expectedCountKeyValues =
+        Arrays.asList(KeyValue.pair("A", 3L), KeyValue.pair("B", 3L), KeyValue.pair("C", 3L));
+    private final List> expectedAggKeyValues =
+        Arrays.asList(KeyValue.pair("A", 9), KeyValue.pair("B", 9), KeyValue.pair("C", 9));
+    private final List> expectedReduceKeyValues =
+        Arrays.asList(KeyValue.pair("A", "foo:bar:baz"), KeyValue.pair("B", "foo:bar:baz"), KeyValue.pair("C", "foo:bar:baz"));
+    private final List> expectedJoinKeyValues =
+        Arrays.asList(KeyValue.pair("A", "foo:3"), KeyValue.pair("A", "bar:3"), KeyValue.pair("A", "baz:3"));
+    private final List expectedCollectedProcessorValues =
+        Arrays.asList("FOO", "BAR", "BAZ");
 
     @Before
-    public void setUp() throws Exception {
+    public void setUp() {
         final Properties props = new Properties();
         props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 1024 * 10);
         props.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 5000);
 
         streamsConfiguration = StreamsTestUtils.getStreamsConfig(
             "maybe-optimized-test-app",
-            CLUSTER.bootstrapServers(),
+            "dummy-bootstrap-servers-config",
             Serdes.String().getClass().getName(),
             Serdes.String().getClass().getName(),
             props);
 
-        CLUSTER.createTopics(INPUT_TOPIC,
-                             COUNT_TOPIC,
-                             AGGREGATION_TOPIC,
-                             REDUCE_TOPIC,
-                             JOINED_TOPIC);
-
-        IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration);
+        processorValueCollector.clear();
     }
 
     @After
-    public void tearDown() throws Exception {
-        CLUSTER.deleteAllTopicsAndWait(30_000L);
+    public void tearDown() {
+        try {
+            topologyTestDriver.close();
+        } catch (final RuntimeException e) {
+            log.warn("The following exception was thrown while trying to close the TopologyTestDriver (note that " +
+                "KAFKA-6647 causes this when running on Windows):", e);
+        }
     }
 
     @Test
-    public void shouldSendCorrectRecords_OPTIMIZED() throws Exception {
-        runIntegrationTest(StreamsConfig.OPTIMIZE,
-                           ONE_REPARTITION_TOPIC);
+    public void shouldSendCorrectRecords_OPTIMIZED() {
+        runTest(StreamsConfig.OPTIMIZE, ONE_REPARTITION_TOPIC);
     }
 
     @Test
-    public void shouldSendCorrectResults_NO_OPTIMIZATION() throws Exception {
-        runIntegrationTest(StreamsConfig.NO_OPTIMIZATION,
-                           FOUR_REPARTITION_TOPICS);
+    public void shouldSendCorrectResults_NO_OPTIMIZATION() {
+        runTest(StreamsConfig.NO_OPTIMIZATION, FOUR_REPARTITION_TOPICS);
     }
 
 
-    private void runIntegrationTest(final String optimizationConfig,
-                                    final int expectedNumberRepartitionTopics) throws Exception {
-
-        final Initializer initializer = () -> 0;
-        final Aggregator aggregator = (k, v, agg) -> agg + v.length();
-
-        final Reducer reducer = (v1, v2) -> v1 + ":" + v2;
-
-        final List processorValueCollector = new ArrayList<>();
+    private void runTest(final String optimizationConfig, final int expectedNumberRepartitionTopics) {
 
         final StreamsBuilder builder = new StreamsBuilder();
 
-        final KStream sourceStream = builder.stream(INPUT_TOPIC, Consumed.with(Serdes.String(), Serdes.String()));
-
-        final KStream mappedStream = sourceStream.map((k, v) -> KeyValue.pair(k.toUpperCase(Locale.getDefault()), v));
-
-        mappedStream.filter((k, v) -> k.equals("B")).mapValues(v -> v.toUpperCase(Locale.getDefault()))
-            .process(() -> new SimpleProcessor(processorValueCollector));
-
-        final KStream countStream = mappedStream.groupByKey().count(Materialized.with(Serdes.String(), Serdes.Long())).toStream();
-
-        countStream.to(COUNT_TOPIC, Produced.with(Serdes.String(), Serdes.Long()));
-
-        mappedStream.groupByKey().aggregate(initializer,
-                                            aggregator,
-                                            Materialized.with(Serdes.String(), Serdes.Integer()))
-            .toStream().to(AGGREGATION_TOPIC, Produced.with(Serdes.String(), Serdes.Integer()));
+        final KStream sourceStream =
+            builder.stream(INPUT_TOPIC, Consumed.with(Serdes.String(), Serdes.String()).withName("sourceStream"));
+
+        final KStream mappedStream = sourceStream
+            .map((k, v) -> KeyValue.pair(k.toUpperCase(Locale.getDefault()), v), Named.as("source-map"));
+
+        mappedStream
+            .filter((k, v) -> k.equals("B"), Named.as("process-filter"))
+            .mapValues(v -> v.toUpperCase(Locale.getDefault()), Named.as("process-mapValues"))
+            .process(() -> new SimpleProcessor(processorValueCollector), Named.as("process"));
+
+        final KStream countStream = mappedStream
+            .groupByKey(Grouped.as("count-groupByKey"))
+            .count(Named.as("count"), Materialized.as(Stores.inMemoryKeyValueStore("count-store"))
+                                                                .withKeySerde(Serdes.String())
+                                                                .withValueSerde(Serdes.Long()))
+            .toStream(Named.as("count-toStream"));
+
+        countStream.to(COUNT_TOPIC, Produced.with(Serdes.String(), Serdes.Long()).withName("count-to"));
+
+        mappedStream
+            .groupByKey(Grouped.as("aggregate-groupByKey"))
+            .aggregate(initializer,
+                       aggregator,
+                       Named.as("aggregate"),
+                       Materialized.as(Stores.inMemoryKeyValueStore("aggregate-store"))
+                                                    .withKeySerde(Serdes.String())
+                                                    .withValueSerde(Serdes.Integer()))
+            .toStream(Named.as("aggregate-toStream"))
+            .to(AGGREGATION_TOPIC, Produced.with(Serdes.String(), Serdes.Integer()).withName("reduce-to"));
 
         // adding operators for case where the repartition node is further downstream
-        mappedStream.filter((k, v) -> true).peek((k, v) -> System.out.println(k + ":" + v)).groupByKey()
-            .reduce(reducer, Materialized.with(Serdes.String(), Serdes.String()))
-            .toStream().to(REDUCE_TOPIC, Produced.with(Serdes.String(), Serdes.String()));
-
-        mappedStream.filter((k, v) -> k.equals("A"))
+        mappedStream
+            .filter((k, v) -> true, Named.as("reduce-filter"))
+            .peek((k, v) -> System.out.println(k + ":" + v), Named.as("reduce-peek"))
+            .groupByKey(Grouped.as("reduce-groupByKey"))
+            .reduce(reducer,
+                    Named.as("reducer"),
+                    Materialized.as(Stores.inMemoryKeyValueStore("reduce-store")))
+            .toStream(Named.as("reduce-toStream"))
+            .to(REDUCE_TOPIC, Produced.with(Serdes.String(), Serdes.String()));
+
+        mappedStream
+            .filter((k, v) -> k.equals("A"), Named.as("join-filter"))
             .join(countStream, (v1, v2) -> v1 + ":" + v2.toString(),
                   JoinWindows.of(ofMillis(5000)),
-                  StreamJoined.with(Serdes.String(), Serdes.String(), Serdes.Long()))
-            .to(JOINED_TOPIC);
+                  StreamJoined.with(Stores.inMemoryWindowStore("join-store", ofDays(1), ofMillis(10000), true),
+                                                          Stores.inMemoryWindowStore("other-join-store",  ofDays(1), ofMillis(10000), true))
+                                                    .withName("join")
+                                                    .withKeySerde(Serdes.String())
+                                                    .withValueSerde(Serdes.String())
+                                                    .withOtherValueSerde(Serdes.Long()))
+            .to(JOINED_TOPIC, Produced.as("join-to"));
 
         streamsConfiguration.setProperty(StreamsConfig.TOPOLOGY_OPTIMIZATION, optimizationConfig);
+        final Topology topology = builder.build(streamsConfiguration);
 
-        final Properties producerConfig = TestUtils.producerConfig(CLUSTER.bootstrapServers(), StringSerializer.class, StringSerializer.class);
+        topologyTestDriver = new TopologyTestDriver(topology, streamsConfiguration);
 
-        IntegrationTestUtils.produceKeyValuesSynchronously(INPUT_TOPIC, getKeyValues(), producerConfig, mockTime);
+        final TestInputTopic inputTopicA = topologyTestDriver.createInputTopic(INPUT_TOPIC, stringSerializer, stringSerializer);
+        final TestOutputTopic countOutputTopic = topologyTestDriver.createOutputTopic(COUNT_TOPIC, stringDeserializer, new LongDeserializer());
+        final TestOutputTopic aggregationOutputTopic = topologyTestDriver.createOutputTopic(AGGREGATION_TOPIC, stringDeserializer, new IntegerDeserializer());
+        final TestOutputTopic reduceOutputTopic = topologyTestDriver.createOutputTopic(REDUCE_TOPIC, stringDeserializer, stringDeserializer);
+        final TestOutputTopic joinedOutputTopic = topologyTestDriver.createOutputTopic(JOINED_TOPIC, stringDeserializer, stringDeserializer);
 
-        final Properties consumerConfig1 = TestUtils.consumerConfig(CLUSTER.bootstrapServers(), StringDeserializer.class, LongDeserializer.class);
-        final Properties consumerConfig2 = TestUtils.consumerConfig(CLUSTER.bootstrapServers(), StringDeserializer.class, IntegerDeserializer.class);
-        final Properties consumerConfig3 = TestUtils.consumerConfig(CLUSTER.bootstrapServers(), StringDeserializer.class, StringDeserializer.class);
+        inputTopicA.pipeKeyValueList(getKeyValues());
 
-        final Topology topology = builder.build(streamsConfiguration);
+        // Verify the topology
         final String topologyString = topology.describe().toString();
-
         if (optimizationConfig.equals(StreamsConfig.OPTIMIZE)) {
             assertEquals(EXPECTED_OPTIMIZED_TOPOLOGY, topologyString);
         } else {
             assertEquals(EXPECTED_UNOPTIMIZED_TOPOLOGY, topologyString);
         }
 
-
-        /*
-           confirming number of expected repartition topics here
-         */
+        // Verify the number of repartition topics
         assertEquals(expectedNumberRepartitionTopics, getCountOfRepartitionTopicsFound(topologyString));
 
-        final KafkaStreams streams = new KafkaStreams(topology, streamsConfiguration);
-        streams.start();
-
-        final List> expectedCountKeyValues = Arrays.asList(KeyValue.pair("A", 3L), KeyValue.pair("B", 3L), KeyValue.pair("C", 3L));
-        IntegrationTestUtils.waitUntilFinalKeyValueRecordsReceived(consumerConfig1, COUNT_TOPIC, expectedCountKeyValues);
-
-        final List> expectedAggKeyValues = Arrays.asList(KeyValue.pair("A", 9), KeyValue.pair("B", 9), KeyValue.pair("C", 9));
-        IntegrationTestUtils.waitUntilFinalKeyValueRecordsReceived(consumerConfig2, AGGREGATION_TOPIC, expectedAggKeyValues);
-
-        final List> expectedReduceKeyValues = Arrays.asList(KeyValue.pair("A", "foo:bar:baz"), KeyValue.pair("B", "foo:bar:baz"), KeyValue.pair("C", "foo:bar:baz"));
-        IntegrationTestUtils.waitUntilFinalKeyValueRecordsReceived(consumerConfig3, REDUCE_TOPIC, expectedReduceKeyValues);
-
-        final List> expectedJoinKeyValues = Arrays.asList(KeyValue.pair("A", "foo:3"), KeyValue.pair("A", "bar:3"), KeyValue.pair("A", "baz:3"));
-        IntegrationTestUtils.waitUntilFinalKeyValueRecordsReceived(consumerConfig3, JOINED_TOPIC, expectedJoinKeyValues);
-
-
-        final List expectedCollectedProcessorValues = Arrays.asList("FOO", "BAR", "BAZ");
-
+        // Verify the values collected by the processor
         assertThat(3, equalTo(processorValueCollector.size()));
         assertThat(processorValueCollector, equalTo(expectedCollectedProcessorValues));
 
-        streams.close(ofSeconds(5));
+        // Verify the expected output
+        assertThat(countOutputTopic.readKeyValuesToMap(), equalTo(keyValueListToMap(expectedCountKeyValues)));
+        assertThat(aggregationOutputTopic.readKeyValuesToMap(), equalTo(keyValueListToMap(expectedAggKeyValues)));
+        assertThat(reduceOutputTopic.readKeyValuesToMap(), equalTo(keyValueListToMap(expectedReduceKeyValues)));
+        assertThat(joinedOutputTopic.readKeyValuesToMap(), equalTo(keyValueListToMap(expectedJoinKeyValues)));
     }
 
+    private  Map keyValueListToMap(final List> keyValuePairs) {
+        final Map map = new HashMap<>();
+        for (final KeyValue pair : keyValuePairs) {
+            map.put(pair.key, pair.value);
+        }
+        return map;
+    }
 
     private int getCountOfRepartitionTopicsFound(final String topologyString) {
         final Matcher matcher = repartitionTopicPattern.matcher(topologyString);
@@ -224,7 +252,6 @@ private int getCountOfRepartitionTopicsFound(final String topologyString) {
         return repartitionTopicsFound.size();
     }
 
-
     private List> getKeyValues() {
         final List> keyValueList = new ArrayList<>();
         final String[] keys = new String[]{"a", "b", "c"};
@@ -237,7 +264,6 @@ private List> getKeyValues() {
         return keyValueList;
     }
 
-
     private static class SimpleProcessor extends AbstractProcessor {
 
         final List valueList;
@@ -252,183 +278,183 @@ public void process(final String key, final String value) {
         }
     }
 
-
     private static final String EXPECTED_OPTIMIZED_TOPOLOGY = "Topologies:\n"
                                                               + "   Sub-topology: 0\n"
-                                                              + "    Source: KSTREAM-SOURCE-0000000000 (topics: [input])\n"
-                                                              + "      --> KSTREAM-MAP-0000000001\n"
-                                                              + "    Processor: KSTREAM-MAP-0000000001 (stores: [])\n"
-                                                              + "      --> KSTREAM-FILTER-0000000002, KSTREAM-FILTER-0000000040\n"
-                                                              + "      <-- KSTREAM-SOURCE-0000000000\n"
-                                                              + "    Processor: KSTREAM-FILTER-0000000002 (stores: [])\n"
-                                                              + "      --> KSTREAM-MAPVALUES-0000000003\n"
-                                                              + "      <-- KSTREAM-MAP-0000000001\n"
-                                                              + "    Processor: KSTREAM-FILTER-0000000040 (stores: [])\n"
-                                                              + "      --> KSTREAM-SINK-0000000039\n"
-                                                              + "      <-- KSTREAM-MAP-0000000001\n"
-                                                              + "    Processor: KSTREAM-MAPVALUES-0000000003 (stores: [])\n"
-                                                              + "      --> KSTREAM-PROCESSOR-0000000004\n"
-                                                              + "      <-- KSTREAM-FILTER-0000000002\n"
-                                                              + "    Processor: KSTREAM-PROCESSOR-0000000004 (stores: [])\n"
-                                                              + "      --> none\n"
-                                                              + "      <-- KSTREAM-MAPVALUES-0000000003\n"
-                                                              + "    Sink: KSTREAM-SINK-0000000039 (topic: KSTREAM-AGGREGATE-STATE-STORE-0000000006-repartition)\n"
-                                                              + "      <-- KSTREAM-FILTER-0000000040\n"
+                                                              + "    Source: KSTREAM-SOURCE-0000000036 (topics: [count-groupByKey-repartition])\n"
+                                                              + "      --> aggregate, count, join-filter, reduce-filter\n"
+                                                              + "    Processor: count (stores: [count-store])\n"
+                                                              + "      --> count-toStream\n"
+                                                              + "      <-- KSTREAM-SOURCE-0000000036\n"
+                                                              + "    Processor: count-toStream (stores: [])\n"
+                                                              + "      --> join-other-windowed, count-to\n"
+                                                              + "      <-- count\n"
+                                                              + "    Processor: join-filter (stores: [])\n"
+                                                              + "      --> join-this-windowed\n"
+                                                              + "      <-- KSTREAM-SOURCE-0000000036\n"
+                                                              + "    Processor: reduce-filter (stores: [])\n"
+                                                              + "      --> reduce-peek\n"
+                                                              + "      <-- KSTREAM-SOURCE-0000000036\n"
+                                                              + "    Processor: join-other-windowed (stores: [other-join-store])\n"
+                                                              + "      --> join-other-join\n"
+                                                              + "      <-- count-toStream\n"
+                                                              + "    Processor: join-this-windowed (stores: [join-store])\n"
+                                                              + "      --> join-this-join\n"
+                                                              + "      <-- join-filter\n"
+                                                              + "    Processor: reduce-peek (stores: [])\n"
+                                                              + "      --> reducer\n"
+                                                              + "      <-- reduce-filter\n"
+                                                              + "    Processor: aggregate (stores: [aggregate-store])\n"
+                                                              + "      --> aggregate-toStream\n"
+                                                              + "      <-- KSTREAM-SOURCE-0000000036\n"
+                                                              + "    Processor: join-other-join (stores: [join-store])\n"
+                                                              + "      --> join-merge\n"
+                                                              + "      <-- join-other-windowed\n"
+                                                              + "    Processor: join-this-join (stores: [other-join-store])\n"
+                                                              + "      --> join-merge\n"
+                                                              + "      <-- join-this-windowed\n"
+                                                              + "    Processor: reducer (stores: [reduce-store])\n"
+                                                              + "      --> reduce-toStream\n"
+                                                              + "      <-- reduce-peek\n"
+                                                              + "    Processor: aggregate-toStream (stores: [])\n"
+                                                              + "      --> reduce-to\n"
+                                                              + "      <-- aggregate\n"
+                                                              + "    Processor: join-merge (stores: [])\n"
+                                                              + "      --> join-to\n"
+                                                              + "      <-- join-this-join, join-other-join\n"
+                                                              + "    Processor: reduce-toStream (stores: [])\n"
+                                                              + "      --> KSTREAM-SINK-0000000023\n"
+                                                              + "      <-- reducer\n"
+                                                              + "    Sink: KSTREAM-SINK-0000000023 (topic: outputTopic_2)\n"
+                                                              + "      <-- reduce-toStream\n"
+                                                              + "    Sink: count-to (topic: outputTopic_0)\n"
+                                                              + "      <-- count-toStream\n"
+                                                              + "    Sink: join-to (topic: joinedOutputTopic)\n"
+                                                              + "      <-- join-merge\n"
+                                                              + "    Sink: reduce-to (topic: outputTopic_1)\n"
+                                                              + "      <-- aggregate-toStream\n"
                                                               + "\n"
                                                               + "  Sub-topology: 1\n"
-                                                              + "    Source: KSTREAM-SOURCE-0000000041 (topics: [KSTREAM-AGGREGATE-STATE-STORE-0000000006-repartition])\n"
-                                                              + "      --> KSTREAM-FILTER-0000000020, KSTREAM-AGGREGATE-0000000007, KSTREAM-AGGREGATE-0000000014, KSTREAM-FILTER-0000000029\n"
-                                                              + "    Processor: KSTREAM-AGGREGATE-0000000007 (stores: [KSTREAM-AGGREGATE-STATE-STORE-0000000006])\n"
-                                                              + "      --> KTABLE-TOSTREAM-0000000011\n"
-                                                              + "      <-- KSTREAM-SOURCE-0000000041\n"
-                                                              + "    Processor: KTABLE-TOSTREAM-0000000011 (stores: [])\n"
-                                                              + "      --> KSTREAM-SINK-0000000012, KSTREAM-WINDOWED-0000000034\n"
-                                                              + "      <-- KSTREAM-AGGREGATE-0000000007\n"
-                                                              + "    Processor: KSTREAM-FILTER-0000000020 (stores: [])\n"
-                                                              + "      --> KSTREAM-PEEK-0000000021\n"
-                                                              + "      <-- KSTREAM-SOURCE-0000000041\n"
-                                                              + "    Processor: KSTREAM-FILTER-0000000029 (stores: [])\n"
-                                                              + "      --> KSTREAM-WINDOWED-0000000033\n"
-                                                              + "      <-- KSTREAM-SOURCE-0000000041\n"
-                                                              + "    Processor: KSTREAM-PEEK-0000000021 (stores: [])\n"
-                                                              + "      --> KSTREAM-REDUCE-0000000023\n"
-                                                              + "      <-- KSTREAM-FILTER-0000000020\n"
-                                                              + "    Processor: KSTREAM-WINDOWED-0000000033 (stores: [KSTREAM-JOINTHIS-0000000035-store])\n"
-                                                              + "      --> KSTREAM-JOINTHIS-0000000035\n"
-                                                              + "      <-- KSTREAM-FILTER-0000000029\n"
-                                                              + "    Processor: KSTREAM-WINDOWED-0000000034 (stores: [KSTREAM-JOINOTHER-0000000036-store])\n"
-                                                              + "      --> KSTREAM-JOINOTHER-0000000036\n"
-                                                              + "      <-- KTABLE-TOSTREAM-0000000011\n"
-                                                              + "    Processor: KSTREAM-AGGREGATE-0000000014 (stores: [KSTREAM-AGGREGATE-STATE-STORE-0000000013])\n"
-                                                              + "      --> KTABLE-TOSTREAM-0000000018\n"
-                                                              + "      <-- KSTREAM-SOURCE-0000000041\n"
-                                                              + "    Processor: KSTREAM-JOINOTHER-0000000036 (stores: [KSTREAM-JOINTHIS-0000000035-store])\n"
-                                                              + "      --> KSTREAM-MERGE-0000000037\n"
-                                                              + "      <-- KSTREAM-WINDOWED-0000000034\n"
-                                                              + "    Processor: KSTREAM-JOINTHIS-0000000035 (stores: [KSTREAM-JOINOTHER-0000000036-store])\n"
-                                                              + "      --> KSTREAM-MERGE-0000000037\n"
-                                                              + "      <-- KSTREAM-WINDOWED-0000000033\n"
-                                                              + "    Processor: KSTREAM-REDUCE-0000000023 (stores: [KSTREAM-REDUCE-STATE-STORE-0000000022])\n"
-                                                              + "      --> KTABLE-TOSTREAM-0000000027\n"
-                                                              + "      <-- KSTREAM-PEEK-0000000021\n"
-                                                              + "    Processor: KSTREAM-MERGE-0000000037 (stores: [])\n"
-                                                              + "      --> KSTREAM-SINK-0000000038\n"
-                                                              + "      <-- KSTREAM-JOINTHIS-0000000035, KSTREAM-JOINOTHER-0000000036\n"
-                                                              + "    Processor: KTABLE-TOSTREAM-0000000018 (stores: [])\n"
-                                                              + "      --> KSTREAM-SINK-0000000019\n"
-                                                              + "      <-- KSTREAM-AGGREGATE-0000000014\n"
-                                                              + "    Processor: KTABLE-TOSTREAM-0000000027 (stores: [])\n"
-                                                              + "      --> KSTREAM-SINK-0000000028\n"
-                                                              + "      <-- KSTREAM-REDUCE-0000000023\n"
-                                                              + "    Sink: KSTREAM-SINK-0000000012 (topic: outputTopic_0)\n"
-                                                              + "      <-- KTABLE-TOSTREAM-0000000011\n"
-                                                              + "    Sink: KSTREAM-SINK-0000000019 (topic: outputTopic_1)\n"
-                                                              + "      <-- KTABLE-TOSTREAM-0000000018\n"
-                                                              + "    Sink: KSTREAM-SINK-0000000028 (topic: outputTopic_2)\n"
-                                                              + "      <-- KTABLE-TOSTREAM-0000000027\n"
-                                                              + "    Sink: KSTREAM-SINK-0000000038 (topic: joinedOutputTopic)\n"
-                                                              + "      <-- KSTREAM-MERGE-0000000037\n\n";
+                                                              + "    Source: sourceStream (topics: [input])\n"
+                                                              + "      --> source-map\n"
+                                                              + "    Processor: source-map (stores: [])\n"
+                                                              + "      --> process-filter, KSTREAM-FILTER-0000000035\n"
+                                                              + "      <-- sourceStream\n"
+                                                              + "    Processor: process-filter (stores: [])\n"
+                                                              + "      --> process-mapValues\n"
+                                                              + "      <-- source-map\n"
+                                                              + "    Processor: KSTREAM-FILTER-0000000035 (stores: [])\n"
+                                                              + "      --> KSTREAM-SINK-0000000034\n"
+                                                              + "      <-- source-map\n"
+                                                              + "    Processor: process-mapValues (stores: [])\n"
+                                                              + "      --> process\n"
+                                                              + "      <-- process-filter\n"
+                                                              + "    Sink: KSTREAM-SINK-0000000034 (topic: count-groupByKey-repartition)\n"
+                                                              + "      <-- KSTREAM-FILTER-0000000035\n"
+                                                              + "    Processor: process (stores: [])\n"
+                                                              + "      --> none\n"
+                                                              + "      <-- process-mapValues\n\n";
+
 
 
     private static final String EXPECTED_UNOPTIMIZED_TOPOLOGY = "Topologies:\n"
                                                                 + "   Sub-topology: 0\n"
-                                                                + "    Source: KSTREAM-SOURCE-0000000000 (topics: [input])\n"
-                                                                + "      --> KSTREAM-MAP-0000000001\n"
-                                                                + "    Processor: KSTREAM-MAP-0000000001 (stores: [])\n"
-                                                                + "      --> KSTREAM-FILTER-0000000020, KSTREAM-FILTER-0000000002, KSTREAM-FILTER-0000000009, KSTREAM-FILTER-0000000016, KSTREAM-FILTER-0000000029\n"
-                                                                + "      <-- KSTREAM-SOURCE-0000000000\n"
-                                                                + "    Processor: KSTREAM-FILTER-0000000020 (stores: [])\n"
-                                                                + "      --> KSTREAM-PEEK-0000000021\n"
-                                                                + "      <-- KSTREAM-MAP-0000000001\n"
-                                                                + "    Processor: KSTREAM-FILTER-0000000002 (stores: [])\n"
-                                                                + "      --> KSTREAM-MAPVALUES-0000000003\n"
-                                                                + "      <-- KSTREAM-MAP-0000000001\n"
-                                                                + "    Processor: KSTREAM-FILTER-0000000029 (stores: [])\n"
-                                                                + "      --> KSTREAM-FILTER-0000000031\n"
-                                                                + "      <-- KSTREAM-MAP-0000000001\n"
-                                                                + "    Processor: KSTREAM-PEEK-0000000021 (stores: [])\n"
-                                                                + "      --> KSTREAM-FILTER-0000000025\n"
-                                                                + "      <-- KSTREAM-FILTER-0000000020\n"
-                                                                + "    Processor: KSTREAM-FILTER-0000000009 (stores: [])\n"
-                                                                + "      --> KSTREAM-SINK-0000000008\n"
-                                                                + "      <-- KSTREAM-MAP-0000000001\n"
-                                                                + "    Processor: KSTREAM-FILTER-0000000016 (stores: [])\n"
-                                                                + "      --> KSTREAM-SINK-0000000015\n"
-                                                                + "      <-- KSTREAM-MAP-0000000001\n"
-                                                                + "    Processor: KSTREAM-FILTER-0000000025 (stores: [])\n"
-                                                                + "      --> KSTREAM-SINK-0000000024\n"
-                                                                + "      <-- KSTREAM-PEEK-0000000021\n"
-                                                                + "    Processor: KSTREAM-FILTER-0000000031 (stores: [])\n"
-                                                                + "      --> KSTREAM-SINK-0000000030\n"
-                                                                + "      <-- KSTREAM-FILTER-0000000029\n"
-                                                                + "    Processor: KSTREAM-MAPVALUES-0000000003 (stores: [])\n"
-                                                                + "      --> KSTREAM-PROCESSOR-0000000004\n"
-                                                                + "      <-- KSTREAM-FILTER-0000000002\n"
-                                                                + "    Processor: KSTREAM-PROCESSOR-0000000004 (stores: [])\n"
-                                                                + "      --> none\n"
-                                                                + "      <-- KSTREAM-MAPVALUES-0000000003\n"
-                                                                + "    Sink: KSTREAM-SINK-0000000008 (topic: KSTREAM-AGGREGATE-STATE-STORE-0000000006-repartition)\n"
-                                                                + "      <-- KSTREAM-FILTER-0000000009\n"
-                                                                + "    Sink: KSTREAM-SINK-0000000015 (topic: KSTREAM-AGGREGATE-STATE-STORE-0000000013-repartition)\n"
-                                                                + "      <-- KSTREAM-FILTER-0000000016\n"
-                                                                + "    Sink: KSTREAM-SINK-0000000024 (topic: KSTREAM-REDUCE-STATE-STORE-0000000022-repartition)\n"
-                                                                + "      <-- KSTREAM-FILTER-0000000025\n"
-                                                                + "    Sink: KSTREAM-SINK-0000000030 (topic: KSTREAM-FILTER-0000000029-repartition)\n"
-                                                                + "      <-- KSTREAM-FILTER-0000000031\n"
+                                                                + "    Source: KSTREAM-SOURCE-0000000007 (topics: [count-groupByKey-repartition])\n"
+                                                                + "      --> count\n"
+                                                                + "    Processor: count (stores: [count-store])\n"
+                                                                + "      --> count-toStream\n"
+                                                                + "      <-- KSTREAM-SOURCE-0000000007\n"
+                                                                + "    Processor: count-toStream (stores: [])\n"
+                                                                + "      --> join-other-windowed, count-to\n"
+                                                                + "      <-- count\n"
+                                                                + "    Source: KSTREAM-SOURCE-0000000027 (topics: [join-left-repartition])\n"
+                                                                + "      --> join-this-windowed\n"
+                                                                + "    Processor: join-other-windowed (stores: [other-join-store])\n"
+                                                                + "      --> join-other-join\n"
+                                                                + "      <-- count-toStream\n"
+                                                                + "    Processor: join-this-windowed (stores: [join-store])\n"
+                                                                + "      --> join-this-join\n"
+                                                                + "      <-- KSTREAM-SOURCE-0000000027\n"
+                                                                + "    Processor: join-other-join (stores: [join-store])\n"
+                                                                + "      --> join-merge\n"
+                                                                + "      <-- join-other-windowed\n"
+                                                                + "    Processor: join-this-join (stores: [other-join-store])\n"
+                                                                + "      --> join-merge\n"
+                                                                + "      <-- join-this-windowed\n"
+                                                                + "    Processor: join-merge (stores: [])\n"
+                                                                + "      --> join-to\n"
+                                                                + "      <-- join-this-join, join-other-join\n"
+                                                                + "    Sink: count-to (topic: outputTopic_0)\n"
+                                                                + "      <-- count-toStream\n"
+                                                                + "    Sink: join-to (topic: joinedOutputTopic)\n"
+                                                                + "      <-- join-merge\n"
                                                                 + "\n"
                                                                 + "  Sub-topology: 1\n"
-                                                                + "    Source: KSTREAM-SOURCE-0000000010 (topics: [KSTREAM-AGGREGATE-STATE-STORE-0000000006-repartition])\n"
-                                                                + "      --> KSTREAM-AGGREGATE-0000000007\n"
-                                                                + "    Processor: KSTREAM-AGGREGATE-0000000007 (stores: [KSTREAM-AGGREGATE-STATE-STORE-0000000006])\n"
-                                                                + "      --> KTABLE-TOSTREAM-0000000011\n"
-                                                                + "      <-- KSTREAM-SOURCE-0000000010\n"
-                                                                + "    Processor: KTABLE-TOSTREAM-0000000011 (stores: [])\n"
-                                                                + "      --> KSTREAM-SINK-0000000012, KSTREAM-WINDOWED-0000000034\n"
-                                                                + "      <-- KSTREAM-AGGREGATE-0000000007\n"
-                                                                + "    Source: KSTREAM-SOURCE-0000000032 (topics: [KSTREAM-FILTER-0000000029-repartition])\n"
-                                                                + "      --> KSTREAM-WINDOWED-0000000033\n"
-                                                                + "    Processor: KSTREAM-WINDOWED-0000000033 (stores: [KSTREAM-JOINTHIS-0000000035-store])\n"
-                                                                + "      --> KSTREAM-JOINTHIS-0000000035\n"
-                                                                + "      <-- KSTREAM-SOURCE-0000000032\n"
-                                                                + "    Processor: KSTREAM-WINDOWED-0000000034 (stores: [KSTREAM-JOINOTHER-0000000036-store])\n"
-                                                                + "      --> KSTREAM-JOINOTHER-0000000036\n"
-                                                                + "      <-- KTABLE-TOSTREAM-0000000011\n"
-                                                                + "    Processor: KSTREAM-JOINOTHER-0000000036 (stores: [KSTREAM-JOINTHIS-0000000035-store])\n"
-                                                                + "      --> KSTREAM-MERGE-0000000037\n"
-                                                                + "      <-- KSTREAM-WINDOWED-0000000034\n"
-                                                                + "    Processor: KSTREAM-JOINTHIS-0000000035 (stores: [KSTREAM-JOINOTHER-0000000036-store])\n"
-                                                                + "      --> KSTREAM-MERGE-0000000037\n"
-                                                                + "      <-- KSTREAM-WINDOWED-0000000033\n"
-                                                                + "    Processor: KSTREAM-MERGE-0000000037 (stores: [])\n"
-                                                                + "      --> KSTREAM-SINK-0000000038\n"
-                                                                + "      <-- KSTREAM-JOINTHIS-0000000035, KSTREAM-JOINOTHER-0000000036\n"
-                                                                + "    Sink: KSTREAM-SINK-0000000012 (topic: outputTopic_0)\n"
-                                                                + "      <-- KTABLE-TOSTREAM-0000000011\n"
-                                                                + "    Sink: KSTREAM-SINK-0000000038 (topic: joinedOutputTopic)\n"
-                                                                + "      <-- KSTREAM-MERGE-0000000037\n"
+                                                                + "    Source: KSTREAM-SOURCE-0000000013 (topics: [aggregate-groupByKey-repartition])\n"
+                                                                + "      --> aggregate\n"
+                                                                + "    Processor: aggregate (stores: [aggregate-store])\n"
+                                                                + "      --> aggregate-toStream\n"
+                                                                + "      <-- KSTREAM-SOURCE-0000000013\n"
+                                                                + "    Processor: aggregate-toStream (stores: [])\n"
+                                                                + "      --> reduce-to\n"
+                                                                + "      <-- aggregate\n"
+                                                                + "    Sink: reduce-to (topic: outputTopic_1)\n"
+                                                                + "      <-- aggregate-toStream\n"
                                                                 + "\n"
                                                                 + "  Sub-topology: 2\n"
-                                                                + "    Source: KSTREAM-SOURCE-0000000017 (topics: [KSTREAM-AGGREGATE-STATE-STORE-0000000013-repartition])\n"
-                                                                + "      --> KSTREAM-AGGREGATE-0000000014\n"
-                                                                + "    Processor: KSTREAM-AGGREGATE-0000000014 (stores: [KSTREAM-AGGREGATE-STATE-STORE-0000000013])\n"
-                                                                + "      --> KTABLE-TOSTREAM-0000000018\n"
-                                                                + "      <-- KSTREAM-SOURCE-0000000017\n"
-                                                                + "    Processor: KTABLE-TOSTREAM-0000000018 (stores: [])\n"
-                                                                + "      --> KSTREAM-SINK-0000000019\n"
-                                                                + "      <-- KSTREAM-AGGREGATE-0000000014\n"
-                                                                + "    Sink: KSTREAM-SINK-0000000019 (topic: outputTopic_1)\n"
-                                                                + "      <-- KTABLE-TOSTREAM-0000000018\n"
+                                                                + "    Source: KSTREAM-SOURCE-0000000021 (topics: [reduce-groupByKey-repartition])\n"
+                                                                + "      --> reducer\n"
+                                                                + "    Processor: reducer (stores: [reduce-store])\n"
+                                                                + "      --> reduce-toStream\n"
+                                                                + "      <-- KSTREAM-SOURCE-0000000021\n"
+                                                                + "    Processor: reduce-toStream (stores: [])\n"
+                                                                + "      --> KSTREAM-SINK-0000000023\n"
+                                                                + "      <-- reducer\n"
+                                                                + "    Sink: KSTREAM-SINK-0000000023 (topic: outputTopic_2)\n"
+                                                                + "      <-- reduce-toStream\n"
                                                                 + "\n"
                                                                 + "  Sub-topology: 3\n"
-                                                                + "    Source: KSTREAM-SOURCE-0000000026 (topics: [KSTREAM-REDUCE-STATE-STORE-0000000022-repartition])\n"
-                                                                + "      --> KSTREAM-REDUCE-0000000023\n"
-                                                                + "    Processor: KSTREAM-REDUCE-0000000023 (stores: [KSTREAM-REDUCE-STATE-STORE-0000000022])\n"
-                                                                + "      --> KTABLE-TOSTREAM-0000000027\n"
-                                                                + "      <-- KSTREAM-SOURCE-0000000026\n"
-                                                                + "    Processor: KTABLE-TOSTREAM-0000000027 (stores: [])\n"
-                                                                + "      --> KSTREAM-SINK-0000000028\n"
-                                                                + "      <-- KSTREAM-REDUCE-0000000023\n"
-                                                                + "    Sink: KSTREAM-SINK-0000000028 (topic: outputTopic_2)\n"
-                                                                + "      <-- KTABLE-TOSTREAM-0000000027\n\n";
+                                                                + "    Source: sourceStream (topics: [input])\n"
+                                                                + "      --> source-map\n"
+                                                                + "    Processor: source-map (stores: [])\n"
+                                                                + "      --> reduce-filter, process-filter, KSTREAM-FILTER-0000000006, join-filter, KSTREAM-FILTER-0000000012\n"
+                                                                + "      <-- sourceStream\n"
+                                                                + "    Processor: reduce-filter (stores: [])\n"
+                                                                + "      --> reduce-peek\n"
+                                                                + "      <-- source-map\n"
+                                                                + "    Processor: join-filter (stores: [])\n"
+                                                                + "      --> KSTREAM-FILTER-0000000026\n"
+                                                                + "      <-- source-map\n"
+                                                                + "    Processor: process-filter (stores: [])\n"
+                                                                + "      --> process-mapValues\n"
+                                                                + "      <-- source-map\n"
+                                                                + "    Processor: reduce-peek (stores: [])\n"
+                                                                + "      --> KSTREAM-FILTER-0000000020\n"
+                                                                + "      <-- reduce-filter\n"
+                                                                + "    Processor: KSTREAM-FILTER-0000000006 (stores: [])\n"
+                                                                + "      --> KSTREAM-SINK-0000000005\n"
+                                                                + "      <-- source-map\n"
+                                                                + "    Processor: KSTREAM-FILTER-0000000012 (stores: [])\n"
+                                                                + "      --> KSTREAM-SINK-0000000011\n"
+                                                                + "      <-- source-map\n"
+                                                                + "    Processor: KSTREAM-FILTER-0000000020 (stores: [])\n"
+                                                                + "      --> KSTREAM-SINK-0000000019\n"
+                                                                + "      <-- reduce-peek\n"
+                                                                + "    Processor: KSTREAM-FILTER-0000000026 (stores: [])\n"
+                                                                + "      --> KSTREAM-SINK-0000000025\n"
+                                                                + "      <-- join-filter\n"
+                                                                + "    Processor: process-mapValues (stores: [])\n"
+                                                                + "      --> process\n"
+                                                                + "      <-- process-filter\n"
+                                                                + "    Sink: KSTREAM-SINK-0000000005 (topic: count-groupByKey-repartition)\n"
+                                                                + "      <-- KSTREAM-FILTER-0000000006\n"
+                                                                + "    Sink: KSTREAM-SINK-0000000011 (topic: aggregate-groupByKey-repartition)\n"
+                                                                + "      <-- KSTREAM-FILTER-0000000012\n"
+                                                                + "    Sink: KSTREAM-SINK-0000000019 (topic: reduce-groupByKey-repartition)\n"
+                                                                + "      <-- KSTREAM-FILTER-0000000020\n"
+                                                                + "    Sink: KSTREAM-SINK-0000000025 (topic: join-left-repartition)\n"
+                                                                + "      <-- KSTREAM-FILTER-0000000026\n"
+                                                                + "    Processor: process (stores: [])\n"
+                                                                + "      --> none\n"
+                                                                + "      <-- process-mapValues\n\n";
 
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/RepartitionWithMergeOptimizingIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionWithMergeOptimizingTest.java
similarity index 54%
rename from streams/src/test/java/org/apache/kafka/streams/integration/RepartitionWithMergeOptimizingIntegrationTest.java
rename to streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionWithMergeOptimizingTest.java
index 473a626dd350c..0d081f71c5632 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/RepartitionWithMergeOptimizingIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionWithMergeOptimizingTest.java
@@ -15,33 +15,35 @@
  * limitations under the License.
  */
 
-package org.apache.kafka.streams.integration;
+package org.apache.kafka.streams.processor.internals;
 
-
-import java.time.Duration;
-import kafka.utils.MockTime;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.kafka.common.serialization.Deserializer;
 import org.apache.kafka.common.serialization.LongDeserializer;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.Serializer;
 import org.apache.kafka.common.serialization.StringDeserializer;
 import org.apache.kafka.common.serialization.StringSerializer;
-import org.apache.kafka.streams.KafkaStreams;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TestInputTopic;
+import org.apache.kafka.streams.TestOutputTopic;
 import org.apache.kafka.streams.Topology;
-import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
-import org.apache.kafka.streams.integration.utils.IntegrationTestUtils;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.Consumed;
+import org.apache.kafka.streams.kstream.Grouped;
 import org.apache.kafka.streams.kstream.KStream;
+import org.apache.kafka.streams.kstream.Materialized;
+import org.apache.kafka.streams.kstream.Named;
 import org.apache.kafka.streams.kstream.Produced;
-import org.apache.kafka.test.IntegrationTest;
+import org.apache.kafka.streams.state.Stores;
 import org.apache.kafka.test.StreamsTestUtils;
-import org.apache.kafka.test.TestUtils;
+
 import org.junit.After;
 import org.junit.Before;
-import org.junit.ClassRule;
 import org.junit.Test;
-import org.junit.experimental.categories.Category;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -49,126 +51,142 @@
 import java.util.Properties;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
 
-@Category({IntegrationTest.class})
-public class RepartitionWithMergeOptimizingIntegrationTest {
+public class RepartitionWithMergeOptimizingTest {
+
+    private final Logger log = LoggerFactory.getLogger(RepartitionWithMergeOptimizingTest.class);
 
-    private static final int NUM_BROKERS = 1;
     private static final String INPUT_A_TOPIC = "inputA";
     private static final String INPUT_B_TOPIC = "inputB";
     private static final String COUNT_TOPIC = "outputTopic_0";
-    private static final String COUNT_STRING_TOPIC = "outputTopic_1";
-
+    private static final String STRING_COUNT_TOPIC = "outputTopic_1";
 
     private static final int ONE_REPARTITION_TOPIC = 1;
     private static final int TWO_REPARTITION_TOPICS = 2;
 
+    private final Serializer stringSerializer = new StringSerializer();
+    private final Deserializer stringDeserializer = new StringDeserializer();
+
     private final Pattern repartitionTopicPattern = Pattern.compile("Sink: .*-repartition");
 
     private Properties streamsConfiguration;
+    private TopologyTestDriver topologyTestDriver;
 
-
-    @ClassRule
-    public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(NUM_BROKERS);
-    private final MockTime mockTime = CLUSTER.time;
+    private final List> expectedCountKeyValues =
+        Arrays.asList(KeyValue.pair("A", 6L), KeyValue.pair("B", 6L), KeyValue.pair("C", 6L));
+    private final List> expectedStringCountKeyValues =
+        Arrays.asList(KeyValue.pair("A", "6"), KeyValue.pair("B", "6"), KeyValue.pair("C", "6"));
 
     @Before
-    public void setUp() throws Exception {
+    public void setUp() {
         final Properties props = new Properties();
         props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 1024 * 10);
         props.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 5000);
 
         streamsConfiguration = StreamsTestUtils.getStreamsConfig(
             "maybe-optimized-with-merge-test-app",
-            CLUSTER.bootstrapServers(),
+            "dummy-bootstrap-servers-config",
             Serdes.String().getClass().getName(),
             Serdes.String().getClass().getName(),
             props);
-
-        CLUSTER.createTopics(COUNT_TOPIC,
-                             COUNT_STRING_TOPIC,
-                             INPUT_A_TOPIC,
-                             INPUT_B_TOPIC);
-
-        IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration);
     }
 
     @After
-    public void tearDown() throws Exception {
-        CLUSTER.deleteAllTopicsAndWait(30_000L);
+    public void tearDown() {
+        try {
+            topologyTestDriver.close();
+        } catch (final RuntimeException e) {
+            log.warn("The following exception was thrown while trying to close the TopologyTestDriver (note that " +
+                "KAFKA-6647 causes this when running on Windows):", e);
+        }
     }
 
     @Test
-    public void shouldSendCorrectRecords_OPTIMIZED() throws Exception {
-        runIntegrationTest(StreamsConfig.OPTIMIZE,
-                           ONE_REPARTITION_TOPIC);
+    public void shouldSendCorrectRecords_OPTIMIZED() {
+        runTest(StreamsConfig.OPTIMIZE, ONE_REPARTITION_TOPIC);
     }
 
     @Test
-    public void shouldSendCorrectResults_NO_OPTIMIZATION() throws Exception {
-        runIntegrationTest(StreamsConfig.NO_OPTIMIZATION,
-                           TWO_REPARTITION_TOPICS);
+    public void shouldSendCorrectResults_NO_OPTIMIZATION() {
+        runTest(StreamsConfig.NO_OPTIMIZATION, TWO_REPARTITION_TOPICS);
     }
 
 
-    private void runIntegrationTest(final String optimizationConfig,
-                                    final int expectedNumberRepartitionTopics) throws Exception {
+    private void runTest(final String optimizationConfig, final int expectedNumberRepartitionTopics) {
 
+        streamsConfiguration.setProperty(StreamsConfig.TOPOLOGY_OPTIMIZATION, optimizationConfig);
 
         final StreamsBuilder builder = new StreamsBuilder();
 
-        final KStream sourceAStream = builder.stream(INPUT_A_TOPIC, Consumed.with(Serdes.String(), Serdes.String()));
+        final KStream sourceAStream =
+            builder.stream(INPUT_A_TOPIC, Consumed.with(Serdes.String(), Serdes.String()).withName("sourceAStream"));
 
-        final KStream sourceBStream = builder.stream(INPUT_B_TOPIC, Consumed.with(Serdes.String(), Serdes.String()));
+        final KStream sourceBStream =
+            builder.stream(INPUT_B_TOPIC, Consumed.with(Serdes.String(), Serdes.String()).withName("sourceBStream"));
 
-        final KStream mappedAStream = sourceAStream.map((k, v) -> KeyValue.pair(v.split(":")[0], v));
-        final KStream mappedBStream = sourceBStream.map((k, v) -> KeyValue.pair(v.split(":")[0], v));
+        final KStream mappedAStream =
+            sourceAStream.map((k, v) -> KeyValue.pair(v.split(":")[0], v), Named.as("mappedAStream"));
+        final KStream mappedBStream =
+            sourceBStream.map((k, v) -> KeyValue.pair(v.split(":")[0], v), Named.as("mappedBStream"));
 
-        final KStream mergedStream = mappedAStream.merge(mappedBStream);
+        final KStream mergedStream = mappedAStream.merge(mappedBStream, Named.as("mergedStream"));
 
-        mergedStream.groupByKey().count().toStream().to(COUNT_TOPIC, Produced.with(Serdes.String(), Serdes.Long()));
-        mergedStream.groupByKey().count().toStream().mapValues(v -> v.toString()).to(COUNT_STRING_TOPIC, Produced.with(Serdes.String(), Serdes.String()));
+        mergedStream
+            .groupByKey(Grouped.as("long-groupByKey"))
+            .count(Named.as("long-count"), Materialized.as(Stores.inMemoryKeyValueStore("long-store")))
+            .toStream(Named.as("long-toStream"))
+            .to(COUNT_TOPIC, Produced.with(Serdes.String(), Serdes.Long()).withName("long-to"));
 
-        streamsConfiguration.setProperty(StreamsConfig.TOPOLOGY_OPTIMIZATION, optimizationConfig);
+        mergedStream
+            .groupByKey(Grouped.as("string-groupByKey"))
+            .count(Named.as("string-count"), Materialized.as(Stores.inMemoryKeyValueStore("string-store")))
+            .toStream(Named.as("string-toStream"))
+            .mapValues(v -> v.toString(), Named.as("string-mapValues"))
+            .to(STRING_COUNT_TOPIC, Produced.with(Serdes.String(), Serdes.String()).withName("string-to"));
 
-        final Properties producerConfig = TestUtils.producerConfig(CLUSTER.bootstrapServers(), StringSerializer.class, StringSerializer.class);
+        final Topology topology = builder.build(streamsConfiguration);
 
-        IntegrationTestUtils.produceKeyValuesSynchronously(INPUT_A_TOPIC, getKeyValues(), producerConfig, mockTime);
-        IntegrationTestUtils.produceKeyValuesSynchronously(INPUT_B_TOPIC, getKeyValues(), producerConfig, mockTime);
+        topologyTestDriver = new TopologyTestDriver(topology, streamsConfiguration);
 
-        final Properties consumerConfig1 = TestUtils.consumerConfig(CLUSTER.bootstrapServers(), StringDeserializer.class, LongDeserializer.class);
-        final Properties consumerConfig2 = TestUtils.consumerConfig(CLUSTER.bootstrapServers(), StringDeserializer.class, StringDeserializer.class);
+        final TestInputTopic inputTopicA = topologyTestDriver.createInputTopic(INPUT_A_TOPIC, stringSerializer, stringSerializer);
+        final TestInputTopic inputTopicB = topologyTestDriver.createInputTopic(INPUT_B_TOPIC, stringSerializer, stringSerializer);
+
+        final TestOutputTopic countOutputTopic = topologyTestDriver.createOutputTopic(COUNT_TOPIC, stringDeserializer, new LongDeserializer());
+        final TestOutputTopic stringCountOutputTopic = topologyTestDriver.createOutputTopic(STRING_COUNT_TOPIC, stringDeserializer, stringDeserializer);
+
+        inputTopicA.pipeKeyValueList(getKeyValues());
+        inputTopicB.pipeKeyValueList(getKeyValues());
 
-        final Topology topology = builder.build(streamsConfiguration);
         final String topologyString = topology.describe().toString();
-        System.out.println(topologyString);
 
+        // Verify the topology
         if (optimizationConfig.equals(StreamsConfig.OPTIMIZE)) {
             assertEquals(EXPECTED_OPTIMIZED_TOPOLOGY, topologyString);
         } else {
             assertEquals(EXPECTED_UNOPTIMIZED_TOPOLOGY, topologyString);
         }
 
-
-        /*
-           confirming number of expected repartition topics here
-         */
+        // Verify the number of repartition topics
         assertEquals(expectedNumberRepartitionTopics, getCountOfRepartitionTopicsFound(topologyString));
 
-        final KafkaStreams streams = new KafkaStreams(topology, streamsConfiguration);
-        streams.start();
-
-        final List> expectedCountKeyValues = Arrays.asList(KeyValue.pair("A", 6L), KeyValue.pair("B", 6L), KeyValue.pair("C", 6L));
-        IntegrationTestUtils.waitUntilFinalKeyValueRecordsReceived(consumerConfig1, COUNT_TOPIC, expectedCountKeyValues);
-
-        final List> expectedStringCountKeyValues = Arrays.asList(KeyValue.pair("A", "6"), KeyValue.pair("B", "6"), KeyValue.pair("C", "6"));
-        IntegrationTestUtils.waitUntilFinalKeyValueRecordsReceived(consumerConfig2, COUNT_STRING_TOPIC, expectedStringCountKeyValues);
-
-        streams.close(Duration.ofSeconds(5));
+        // Verify the expected output
+        assertThat(countOutputTopic.readKeyValuesToMap(), equalTo(keyValueListToMap(expectedCountKeyValues)));
+        assertThat(stringCountOutputTopic.readKeyValuesToMap(), equalTo(keyValueListToMap(expectedStringCountKeyValues)));
     }
 
+    private  Map keyValueListToMap(final List> keyValuePairs) {
+        final Map map = new HashMap<>();
+        for (final KeyValue pair : keyValuePairs) {
+            map.put(pair.key, pair.value);
+        }
+        return map;
+    }
 
     private int getCountOfRepartitionTopicsFound(final String topologyString) {
         final Matcher matcher = repartitionTopicPattern.matcher(topologyString);
@@ -179,7 +197,6 @@ private int getCountOfRepartitionTopicsFound(final String topologyString) {
         return repartitionTopicsFound.size();
     }
 
-
     private List> getKeyValues() {
         final List> keyValueList = new ArrayList<>();
         final String[] keys = new String[]{"X", "Y", "Z"};
@@ -192,104 +209,103 @@ private List> getKeyValues() {
         return keyValueList;
     }
 
-
-
     private static final String EXPECTED_OPTIMIZED_TOPOLOGY = "Topologies:\n"
                                                               + "   Sub-topology: 0\n"
-                                                              + "    Source: KSTREAM-SOURCE-0000000000 (topics: [inputA])\n"
-                                                              + "      --> KSTREAM-MAP-0000000002\n"
-                                                              + "    Source: KSTREAM-SOURCE-0000000001 (topics: [inputB])\n"
-                                                              + "      --> KSTREAM-MAP-0000000003\n"
-                                                              + "    Processor: KSTREAM-MAP-0000000002 (stores: [])\n"
-                                                              + "      --> KSTREAM-MERGE-0000000004\n"
-                                                              + "      <-- KSTREAM-SOURCE-0000000000\n"
-                                                              + "    Processor: KSTREAM-MAP-0000000003 (stores: [])\n"
-                                                              + "      --> KSTREAM-MERGE-0000000004\n"
-                                                              + "      <-- KSTREAM-SOURCE-0000000001\n"
-                                                              + "    Processor: KSTREAM-MERGE-0000000004 (stores: [])\n"
-                                                              + "      --> KSTREAM-FILTER-0000000021\n"
-                                                              + "      <-- KSTREAM-MAP-0000000002, KSTREAM-MAP-0000000003\n"
-                                                              + "    Processor: KSTREAM-FILTER-0000000021 (stores: [])\n"
-                                                              + "      --> KSTREAM-SINK-0000000020\n"
-                                                              + "      <-- KSTREAM-MERGE-0000000004\n"
-                                                              + "    Sink: KSTREAM-SINK-0000000020 (topic: KSTREAM-AGGREGATE-STATE-STORE-0000000005-repartition)\n"
-                                                              + "      <-- KSTREAM-FILTER-0000000021\n"
+                                                              + "    Source: KSTREAM-SOURCE-0000000020 (topics: [long-groupByKey-repartition])\n"
+                                                              + "      --> long-count, string-count\n"
+                                                              + "    Processor: string-count (stores: [string-store])\n"
+                                                              + "      --> string-toStream\n"
+                                                              + "      <-- KSTREAM-SOURCE-0000000020\n"
+                                                              + "    Processor: long-count (stores: [long-store])\n"
+                                                              + "      --> long-toStream\n"
+                                                              + "      <-- KSTREAM-SOURCE-0000000020\n"
+                                                              + "    Processor: string-toStream (stores: [])\n"
+                                                              + "      --> string-mapValues\n"
+                                                              + "      <-- string-count\n"
+                                                              + "    Processor: long-toStream (stores: [])\n"
+                                                              + "      --> long-to\n"
+                                                              + "      <-- long-count\n"
+                                                              + "    Processor: string-mapValues (stores: [])\n"
+                                                              + "      --> string-to\n"
+                                                              + "      <-- string-toStream\n"
+                                                              + "    Sink: long-to (topic: outputTopic_0)\n"
+                                                              + "      <-- long-toStream\n"
+                                                              + "    Sink: string-to (topic: outputTopic_1)\n"
+                                                              + "      <-- string-mapValues\n"
                                                               + "\n"
                                                               + "  Sub-topology: 1\n"
-                                                              + "    Source: KSTREAM-SOURCE-0000000022 (topics: [KSTREAM-AGGREGATE-STATE-STORE-0000000005-repartition])\n"
-                                                              + "      --> KSTREAM-AGGREGATE-0000000006, KSTREAM-AGGREGATE-0000000013\n"
-                                                              + "    Processor: KSTREAM-AGGREGATE-0000000013 (stores: [KSTREAM-AGGREGATE-STATE-STORE-0000000012])\n"
-                                                              + "      --> KTABLE-TOSTREAM-0000000017\n"
-                                                              + "      <-- KSTREAM-SOURCE-0000000022\n"
-                                                              + "    Processor: KSTREAM-AGGREGATE-0000000006 (stores: [KSTREAM-AGGREGATE-STATE-STORE-0000000005])\n"
-                                                              + "      --> KTABLE-TOSTREAM-0000000010\n"
-                                                              + "      <-- KSTREAM-SOURCE-0000000022\n"
-                                                              + "    Processor: KTABLE-TOSTREAM-0000000017 (stores: [])\n"
-                                                              + "      --> KSTREAM-MAPVALUES-0000000018\n"
-                                                              + "      <-- KSTREAM-AGGREGATE-0000000013\n"
-                                                              + "    Processor: KSTREAM-MAPVALUES-0000000018 (stores: [])\n"
-                                                              + "      --> KSTREAM-SINK-0000000019\n"
-                                                              + "      <-- KTABLE-TOSTREAM-0000000017\n"
-                                                              + "    Processor: KTABLE-TOSTREAM-0000000010 (stores: [])\n"
-                                                              + "      --> KSTREAM-SINK-0000000011\n"
-                                                              + "      <-- KSTREAM-AGGREGATE-0000000006\n"
-                                                              + "    Sink: KSTREAM-SINK-0000000011 (topic: outputTopic_0)\n"
-                                                              + "      <-- KTABLE-TOSTREAM-0000000010\n"
-                                                              + "    Sink: KSTREAM-SINK-0000000019 (topic: outputTopic_1)\n"
-                                                              + "      <-- KSTREAM-MAPVALUES-0000000018\n\n";
+                                                              + "    Source: sourceAStream (topics: [inputA])\n"
+                                                              + "      --> mappedAStream\n"
+                                                              + "    Source: sourceBStream (topics: [inputB])\n"
+                                                              + "      --> mappedBStream\n"
+                                                              + "    Processor: mappedAStream (stores: [])\n"
+                                                              + "      --> mergedStream\n"
+                                                              + "      <-- sourceAStream\n"
+                                                              + "    Processor: mappedBStream (stores: [])\n"
+                                                              + "      --> mergedStream\n"
+                                                              + "      <-- sourceBStream\n"
+                                                              + "    Processor: mergedStream (stores: [])\n"
+                                                              + "      --> KSTREAM-FILTER-0000000019\n"
+                                                              + "      <-- mappedAStream, mappedBStream\n"
+                                                              + "    Processor: KSTREAM-FILTER-0000000019 (stores: [])\n"
+                                                              + "      --> KSTREAM-SINK-0000000018\n"
+                                                              + "      <-- mergedStream\n"
+                                                              + "    Sink: KSTREAM-SINK-0000000018 (topic: long-groupByKey-repartition)\n"
+                                                              + "      <-- KSTREAM-FILTER-0000000019\n\n";
 
 
     private static final String EXPECTED_UNOPTIMIZED_TOPOLOGY = "Topologies:\n"
                                                                 + "   Sub-topology: 0\n"
-                                                                + "    Source: KSTREAM-SOURCE-0000000000 (topics: [inputA])\n"
-                                                                + "      --> KSTREAM-MAP-0000000002\n"
-                                                                + "    Source: KSTREAM-SOURCE-0000000001 (topics: [inputB])\n"
-                                                                + "      --> KSTREAM-MAP-0000000003\n"
-                                                                + "    Processor: KSTREAM-MAP-0000000002 (stores: [])\n"
-                                                                + "      --> KSTREAM-MERGE-0000000004\n"
-                                                                + "      <-- KSTREAM-SOURCE-0000000000\n"
-                                                                + "    Processor: KSTREAM-MAP-0000000003 (stores: [])\n"
-                                                                + "      --> KSTREAM-MERGE-0000000004\n"
-                                                                + "      <-- KSTREAM-SOURCE-0000000001\n"
-                                                                + "    Processor: KSTREAM-MERGE-0000000004 (stores: [])\n"
-                                                                + "      --> KSTREAM-FILTER-0000000008, KSTREAM-FILTER-0000000015\n"
-                                                                + "      <-- KSTREAM-MAP-0000000002, KSTREAM-MAP-0000000003\n"
-                                                                + "    Processor: KSTREAM-FILTER-0000000008 (stores: [])\n"
-                                                                + "      --> KSTREAM-SINK-0000000007\n"
-                                                                + "      <-- KSTREAM-MERGE-0000000004\n"
-                                                                + "    Processor: KSTREAM-FILTER-0000000015 (stores: [])\n"
-                                                                + "      --> KSTREAM-SINK-0000000014\n"
-                                                                + "      <-- KSTREAM-MERGE-0000000004\n"
-                                                                + "    Sink: KSTREAM-SINK-0000000007 (topic: KSTREAM-AGGREGATE-STATE-STORE-0000000005-repartition)\n"
-                                                                + "      <-- KSTREAM-FILTER-0000000008\n"
-                                                                + "    Sink: KSTREAM-SINK-0000000014 (topic: KSTREAM-AGGREGATE-STATE-STORE-0000000012-repartition)\n"
-                                                                + "      <-- KSTREAM-FILTER-0000000015\n"
+                                                                + "    Source: KSTREAM-SOURCE-0000000008 (topics: [long-groupByKey-repartition])\n"
+                                                                + "      --> long-count\n"
+                                                                + "    Processor: long-count (stores: [long-store])\n"
+                                                                + "      --> long-toStream\n"
+                                                                + "      <-- KSTREAM-SOURCE-0000000008\n"
+                                                                + "    Processor: long-toStream (stores: [])\n"
+                                                                + "      --> long-to\n"
+                                                                + "      <-- long-count\n"
+                                                                + "    Sink: long-to (topic: outputTopic_0)\n"
+                                                                + "      <-- long-toStream\n"
                                                                 + "\n"
                                                                 + "  Sub-topology: 1\n"
-                                                                + "    Source: KSTREAM-SOURCE-0000000009 (topics: [KSTREAM-AGGREGATE-STATE-STORE-0000000005-repartition])\n"
-                                                                + "      --> KSTREAM-AGGREGATE-0000000006\n"
-                                                                + "    Processor: KSTREAM-AGGREGATE-0000000006 (stores: [KSTREAM-AGGREGATE-STATE-STORE-0000000005])\n"
-                                                                + "      --> KTABLE-TOSTREAM-0000000010\n"
-                                                                + "      <-- KSTREAM-SOURCE-0000000009\n"
-                                                                + "    Processor: KTABLE-TOSTREAM-0000000010 (stores: [])\n"
-                                                                + "      --> KSTREAM-SINK-0000000011\n"
-                                                                + "      <-- KSTREAM-AGGREGATE-0000000006\n"
-                                                                + "    Sink: KSTREAM-SINK-0000000011 (topic: outputTopic_0)\n"
-                                                                + "      <-- KTABLE-TOSTREAM-0000000010\n"
+                                                                + "    Source: KSTREAM-SOURCE-0000000014 (topics: [string-groupByKey-repartition])\n"
+                                                                + "      --> string-count\n"
+                                                                + "    Processor: string-count (stores: [string-store])\n"
+                                                                + "      --> string-toStream\n"
+                                                                + "      <-- KSTREAM-SOURCE-0000000014\n"
+                                                                + "    Processor: string-toStream (stores: [])\n"
+                                                                + "      --> string-mapValues\n"
+                                                                + "      <-- string-count\n"
+                                                                + "    Processor: string-mapValues (stores: [])\n"
+                                                                + "      --> string-to\n"
+                                                                + "      <-- string-toStream\n"
+                                                                + "    Sink: string-to (topic: outputTopic_1)\n"
+                                                                + "      <-- string-mapValues\n"
                                                                 + "\n"
                                                                 + "  Sub-topology: 2\n"
-                                                                + "    Source: KSTREAM-SOURCE-0000000016 (topics: [KSTREAM-AGGREGATE-STATE-STORE-0000000012-repartition])\n"
-                                                                + "      --> KSTREAM-AGGREGATE-0000000013\n"
-                                                                + "    Processor: KSTREAM-AGGREGATE-0000000013 (stores: [KSTREAM-AGGREGATE-STATE-STORE-0000000012])\n"
-                                                                + "      --> KTABLE-TOSTREAM-0000000017\n"
-                                                                + "      <-- KSTREAM-SOURCE-0000000016\n"
-                                                                + "    Processor: KTABLE-TOSTREAM-0000000017 (stores: [])\n"
-                                                                + "      --> KSTREAM-MAPVALUES-0000000018\n"
-                                                                + "      <-- KSTREAM-AGGREGATE-0000000013\n"
-                                                                + "    Processor: KSTREAM-MAPVALUES-0000000018 (stores: [])\n"
-                                                                + "      --> KSTREAM-SINK-0000000019\n"
-                                                                + "      <-- KTABLE-TOSTREAM-0000000017\n"
-                                                                + "    Sink: KSTREAM-SINK-0000000019 (topic: outputTopic_1)\n"
-                                                                + "      <-- KSTREAM-MAPVALUES-0000000018\n\n";
+                                                                + "    Source: sourceAStream (topics: [inputA])\n"
+                                                                + "      --> mappedAStream\n"
+                                                                + "    Source: sourceBStream (topics: [inputB])\n"
+                                                                + "      --> mappedBStream\n"
+                                                                + "    Processor: mappedAStream (stores: [])\n"
+                                                                + "      --> mergedStream\n"
+                                                                + "      <-- sourceAStream\n"
+                                                                + "    Processor: mappedBStream (stores: [])\n"
+                                                                + "      --> mergedStream\n"
+                                                                + "      <-- sourceBStream\n"
+                                                                + "    Processor: mergedStream (stores: [])\n"
+                                                                + "      --> KSTREAM-FILTER-0000000007, KSTREAM-FILTER-0000000013\n"
+                                                                + "      <-- mappedAStream, mappedBStream\n"
+                                                                + "    Processor: KSTREAM-FILTER-0000000007 (stores: [])\n"
+                                                                + "      --> KSTREAM-SINK-0000000006\n"
+                                                                + "      <-- mergedStream\n"
+                                                                + "    Processor: KSTREAM-FILTER-0000000013 (stores: [])\n"
+                                                                + "      --> KSTREAM-SINK-0000000012\n"
+                                                                + "      <-- mergedStream\n"
+                                                                + "    Sink: KSTREAM-SINK-0000000006 (topic: long-groupByKey-repartition)\n"
+                                                                + "      <-- KSTREAM-FILTER-0000000007\n"
+                                                                + "    Sink: KSTREAM-SINK-0000000012 (topic: string-groupByKey-repartition)\n"
+                                                                + "      <-- KSTREAM-FILTER-0000000013\n\n";
+
 
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsBrokerDownResilienceTest.java b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsBrokerDownResilienceTest.java
index 25c642e5268db..4dd4954b5c498 100644
--- a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsBrokerDownResilienceTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsBrokerDownResilienceTest.java
@@ -104,27 +104,22 @@ public void apply(final String key, final String value) {
 
         final KafkaStreams streams = new KafkaStreams(builder.build(), streamsProperties);
 
-        streams.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
-            @Override
-            public void uncaughtException(final Thread t, final Throwable e) {
+        streams.setUncaughtExceptionHandler( (t,e) -> {
                 System.err.println("FATAL: An unexpected exception " + e);
                 System.err.flush();
                 streams.close(Duration.ofSeconds(30));
             }
-        });
+        );
+
         System.out.println("Start Kafka Streams");
         streams.start();
 
-        Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
-            @Override
-            public void run() {
+        Runtime.getRuntime().addShutdownHook(new Thread( () -> {
                 streams.close(Duration.ofSeconds(30));
                 System.out.println("Complete shutdown of streams resilience test app now");
                 System.out.flush();
             }
-        }));
-
-
+        ));
     }
 
     private static boolean confirmCorrectConfigs(final Properties properties) {
diff --git a/tests/kafkatest/tests/streams/streams_upgrade_test.py b/tests/kafkatest/tests/streams/streams_upgrade_test.py
index ab5709021b2ca..00dbe355f8086 100644
--- a/tests/kafkatest/tests/streams/streams_upgrade_test.py
+++ b/tests/kafkatest/tests/streams/streams_upgrade_test.py
@@ -441,7 +441,7 @@ def do_stop_start_bounce(self, processor, upgrade_from, new_version, counter):
         if upgrade_from is None:  # upgrade disabled -- second round of rolling bounces
             roll_counter = ".1-"  # second round of rolling bounces
         else:
-            roll_counter = ".0-"  # first  round of rolling boundes
+            roll_counter = ".0-"  # first  round of rolling bounces
 
         node.account.ssh("mv " + processor.STDOUT_FILE + " " + processor.STDOUT_FILE + roll_counter + str(counter), allow_fail=False)
         node.account.ssh("mv " + processor.STDERR_FILE + " " + processor.STDERR_FILE + roll_counter + str(counter), allow_fail=False)

From 8e048bac4576b0da00d0542a9e848af95c1dea03 Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Thu, 10 Oct 2019 19:38:14 -0700
Subject: [PATCH 0711/1071] HOTFIX: fix checkstyle in Streams system test
 (#7494)

Reviewers: Guozhang Wang 
---
 .../tests/StreamsBrokerDownResilienceTest.java       | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsBrokerDownResilienceTest.java b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsBrokerDownResilienceTest.java
index 4dd4954b5c498..b605f46ebb100 100644
--- a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsBrokerDownResilienceTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsBrokerDownResilienceTest.java
@@ -104,7 +104,7 @@ public void apply(final String key, final String value) {
 
         final KafkaStreams streams = new KafkaStreams(builder.build(), streamsProperties);
 
-        streams.setUncaughtExceptionHandler( (t,e) -> {
+        streams.setUncaughtExceptionHandler((t, e) -> {
                 System.err.println("FATAL: An unexpected exception " + e);
                 System.err.flush();
                 streams.close(Duration.ofSeconds(30));
@@ -114,11 +114,11 @@ public void apply(final String key, final String value) {
         System.out.println("Start Kafka Streams");
         streams.start();
 
-        Runtime.getRuntime().addShutdownHook(new Thread( () -> {
-                streams.close(Duration.ofSeconds(30));
-                System.out.println("Complete shutdown of streams resilience test app now");
-                System.out.flush();
-            }
+        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            streams.close(Duration.ofSeconds(30));
+            System.out.println("Complete shutdown of streams resilience test app now");
+            System.out.flush();
+        }
         ));
     }
 

From f965e79d49b85591a3d5bd7b6d3562b9278da05e Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Sat, 12 Oct 2019 11:07:12 -0700
Subject: [PATCH 0712/1071] KAFKA-9029: Flaky Test
 CooperativeStickyAssignorTest.testReassignmentWithRand: bump to 4 (#7503)

One of the sticky assignor tests involves a random change in subscriptions that the current assignor algorithm struggles to react to and in cooperative mode ends up requiring more than one followup rebalance.

Apparently, in rare cases it can also require more than 2. Bumping the "allowed subsequent rebalances" to 4 (increase of 2) to allow some breathing room and reduce flakiness (technically any number is "correct", but if it turns out to ever require more than 4 we should revisit and improve the algorithm because that would be excessive (see KAFKA-8767)

Reviewers: Guozhang Wang 
---
 .../kafka/clients/consumer/CooperativeStickyAssignorTest.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/CooperativeStickyAssignorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/CooperativeStickyAssignorTest.java
index 72a6d89877f66..aed8c09537068 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/CooperativeStickyAssignorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/CooperativeStickyAssignorTest.java
@@ -68,7 +68,7 @@ public void verifyValidityAndBalance(Map subscriptions,
             assignments.putAll(assignor.assign(partitionsPerTopic, subscriptions));
             ++rebalances;
 
-            assertTrue(rebalances <= 2);
+            assertTrue(rebalances <= 4);
         }
 
         // Check the validity and balance of the final assignment

From 68d6d99f62541ea77855e153258b5158afdf2dee Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Mon, 14 Oct 2019 13:15:03 -0700
Subject: [PATCH 0713/1071] KAFKA-9020: Streams sub-topologies should be sorted
 by sink -> source relationship (#7495)

Subtopologies are currently ordered alphabetically by source node, which prior to KIP-307 happened to always result in the "correct" (ie topological) order. Now that users may name their nodes anything they want, we must explicitly order them so that upstream node groups/subtopologies come first and the downstream ones come after.

Reviewers: Guozhang Wang , Bill Bejeck 
---
 .../internals/InternalTopologyBuilder.java    |  22 +-
 .../kafka/streams/StreamsBuilderTest.java     |   8 +-
 .../kstream/RepartitionTopicNamingTest.java   |   2 +-
 .../internals/RepartitionOptimizingTest.java  | 347 +++++++++---------
 .../RepartitionWithMergeOptimizingTest.java   | 145 ++++----
 .../kafka/streams/TopologyTestDriver.java     |   4 +-
 6 files changed, 259 insertions(+), 269 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java
index d34ff00c68629..5754b4a9f1ec2 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java
@@ -18,7 +18,6 @@
 
 import org.apache.kafka.common.serialization.Deserializer;
 import org.apache.kafka.common.serialization.Serializer;
-import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.Topology;
 import org.apache.kafka.streams.errors.TopologyException;
@@ -85,7 +84,7 @@ public class InternalTopologyBuilder {
     // map from source processor names to regex subscription patterns
     private final Map nodeToSourcePatterns = new LinkedHashMap<>();
 
-    // map from sink processor names to subscribed topic (without application-id prefix for internal topics)
+    // map from sink processor names to sink topic (without application-id prefix for internal topics)
     private final Map nodeToSinkTopic = new HashMap<>();
 
     // map from topics to their matched regex patterns, this is to ensure one topic is passed through on source node
@@ -749,27 +748,18 @@ public synchronized Map> nodeGroups() {
         return nodeGroups;
     }
 
+    // Order node groups by their position in the actual topology, ie upstream subtopologies come before downstream
     private Map> makeNodeGroups() {
         final Map> nodeGroups = new LinkedHashMap<>();
         final Map> rootToNodeGroup = new HashMap<>();
 
         int nodeGroupId = 0;
 
-        // Go through source nodes first. This makes the group id assignment easy to predict in tests
-        final Set allSourceNodes = new HashSet<>(nodeToSourceTopics.keySet());
-        allSourceNodes.addAll(nodeToSourcePatterns.keySet());
-
-        for (final String nodeName : Utils.sorted(allSourceNodes)) {
+        // Traverse in topological order
+        for (final String nodeName : nodeFactories.keySet()) {
             nodeGroupId = putNodeGroupName(nodeName, nodeGroupId, nodeGroups, rootToNodeGroup);
         }
 
-        // Go through non-source nodes
-        for (final String nodeName : Utils.sorted(nodeFactories.keySet())) {
-            if (!nodeToSourceTopics.containsKey(nodeName)) {
-                nodeGroupId = putNodeGroupName(nodeName, nodeGroupId, nodeGroups, rootToNodeGroup);
-            }
-        }
-
         return nodeGroups;
     }
 
@@ -1905,11 +1895,11 @@ void updateSubscribedTopics(final Set topics,
 
     // following functions are for test only
 
-    public synchronized Set getSourceTopicNames() {
+    public synchronized Set sourceTopicNames() {
         return sourceTopicNames;
     }
 
-    public synchronized Map getStateStores() {
+    public synchronized Map stateStores() {
         return stateFactories;
     }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java
index 411829771ef77..0cce24f1f7bac 100644
--- a/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java
@@ -385,10 +385,10 @@ public void shouldReuseSourceTopicAsChangelogsWithOptimization20() {
             internalTopologyBuilder.build().storeToChangelogTopic(),
             equalTo(Collections.singletonMap("store", "topic")));
         assertThat(
-            internalTopologyBuilder.getStateStores().keySet(),
+            internalTopologyBuilder.stateStores().keySet(),
             equalTo(Collections.singleton("store")));
         assertThat(
-            internalTopologyBuilder.getStateStores().get("store").loggingEnabled(),
+            internalTopologyBuilder.stateStores().get("store").loggingEnabled(),
             equalTo(false));
         assertThat(
             internalTopologyBuilder.topicGroups().get(0).stateChangelogTopics.isEmpty(),
@@ -407,10 +407,10 @@ public void shouldNotReuseSourceTopicAsChangelogsByDefault() {
             internalTopologyBuilder.build().storeToChangelogTopic(),
             equalTo(Collections.singletonMap("store", "appId-store-changelog")));
         assertThat(
-            internalTopologyBuilder.getStateStores().keySet(),
+            internalTopologyBuilder.stateStores().keySet(),
             equalTo(Collections.singleton("store")));
         assertThat(
-            internalTopologyBuilder.getStateStores().get("store").loggingEnabled(),
+            internalTopologyBuilder.stateStores().get("store").loggingEnabled(),
             equalTo(true));
         assertThat(
             internalTopologyBuilder.topicGroups().get(0).stateChangelogTopics.keySet(),
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/RepartitionTopicNamingTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/RepartitionTopicNamingTest.java
index 0f3e33d86f32a..cf8d6b9bce6a6 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/RepartitionTopicNamingTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/RepartitionTopicNamingTest.java
@@ -603,7 +603,7 @@ public void process(final String key, final String value) {
             "    Source: KSTREAM-SOURCE-0000000000 (topics: [input])\n" +
             "      --> KSTREAM-MAP-0000000001\n" +
             "    Processor: KSTREAM-MAP-0000000001 (stores: [])\n" +
-            "      --> KSTREAM-FILTER-0000000020, KSTREAM-FILTER-0000000002, KSTREAM-FILTER-0000000009, KSTREAM-FILTER-0000000016, KSTREAM-FILTER-0000000029\n" +
+            "      --> KSTREAM-FILTER-0000000002, KSTREAM-FILTER-0000000009, KSTREAM-FILTER-0000000020, KSTREAM-FILTER-0000000016, KSTREAM-FILTER-0000000029\n" +
             "      <-- KSTREAM-SOURCE-0000000000\n" +
             "    Processor: KSTREAM-FILTER-0000000020 (stores: [])\n" +
             "      --> KSTREAM-PEEK-0000000021\n" +
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionOptimizingTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionOptimizingTest.java
index 8425ad7001ae1..cf46af65f2642 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionOptimizingTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionOptimizingTest.java
@@ -279,182 +279,183 @@ public void process(final String key, final String value) {
     }
 
     private static final String EXPECTED_OPTIMIZED_TOPOLOGY = "Topologies:\n"
-                                                              + "   Sub-topology: 0\n"
-                                                              + "    Source: KSTREAM-SOURCE-0000000036 (topics: [count-groupByKey-repartition])\n"
-                                                              + "      --> aggregate, count, join-filter, reduce-filter\n"
-                                                              + "    Processor: count (stores: [count-store])\n"
-                                                              + "      --> count-toStream\n"
-                                                              + "      <-- KSTREAM-SOURCE-0000000036\n"
-                                                              + "    Processor: count-toStream (stores: [])\n"
-                                                              + "      --> join-other-windowed, count-to\n"
-                                                              + "      <-- count\n"
-                                                              + "    Processor: join-filter (stores: [])\n"
-                                                              + "      --> join-this-windowed\n"
-                                                              + "      <-- KSTREAM-SOURCE-0000000036\n"
-                                                              + "    Processor: reduce-filter (stores: [])\n"
-                                                              + "      --> reduce-peek\n"
-                                                              + "      <-- KSTREAM-SOURCE-0000000036\n"
-                                                              + "    Processor: join-other-windowed (stores: [other-join-store])\n"
-                                                              + "      --> join-other-join\n"
-                                                              + "      <-- count-toStream\n"
-                                                              + "    Processor: join-this-windowed (stores: [join-store])\n"
-                                                              + "      --> join-this-join\n"
-                                                              + "      <-- join-filter\n"
-                                                              + "    Processor: reduce-peek (stores: [])\n"
-                                                              + "      --> reducer\n"
-                                                              + "      <-- reduce-filter\n"
-                                                              + "    Processor: aggregate (stores: [aggregate-store])\n"
-                                                              + "      --> aggregate-toStream\n"
-                                                              + "      <-- KSTREAM-SOURCE-0000000036\n"
-                                                              + "    Processor: join-other-join (stores: [join-store])\n"
-                                                              + "      --> join-merge\n"
-                                                              + "      <-- join-other-windowed\n"
-                                                              + "    Processor: join-this-join (stores: [other-join-store])\n"
-                                                              + "      --> join-merge\n"
-                                                              + "      <-- join-this-windowed\n"
-                                                              + "    Processor: reducer (stores: [reduce-store])\n"
-                                                              + "      --> reduce-toStream\n"
-                                                              + "      <-- reduce-peek\n"
-                                                              + "    Processor: aggregate-toStream (stores: [])\n"
-                                                              + "      --> reduce-to\n"
-                                                              + "      <-- aggregate\n"
-                                                              + "    Processor: join-merge (stores: [])\n"
-                                                              + "      --> join-to\n"
-                                                              + "      <-- join-this-join, join-other-join\n"
-                                                              + "    Processor: reduce-toStream (stores: [])\n"
-                                                              + "      --> KSTREAM-SINK-0000000023\n"
-                                                              + "      <-- reducer\n"
-                                                              + "    Sink: KSTREAM-SINK-0000000023 (topic: outputTopic_2)\n"
-                                                              + "      <-- reduce-toStream\n"
-                                                              + "    Sink: count-to (topic: outputTopic_0)\n"
-                                                              + "      <-- count-toStream\n"
-                                                              + "    Sink: join-to (topic: joinedOutputTopic)\n"
-                                                              + "      <-- join-merge\n"
-                                                              + "    Sink: reduce-to (topic: outputTopic_1)\n"
-                                                              + "      <-- aggregate-toStream\n"
-                                                              + "\n"
-                                                              + "  Sub-topology: 1\n"
-                                                              + "    Source: sourceStream (topics: [input])\n"
-                                                              + "      --> source-map\n"
-                                                              + "    Processor: source-map (stores: [])\n"
-                                                              + "      --> process-filter, KSTREAM-FILTER-0000000035\n"
-                                                              + "      <-- sourceStream\n"
-                                                              + "    Processor: process-filter (stores: [])\n"
-                                                              + "      --> process-mapValues\n"
-                                                              + "      <-- source-map\n"
-                                                              + "    Processor: KSTREAM-FILTER-0000000035 (stores: [])\n"
-                                                              + "      --> KSTREAM-SINK-0000000034\n"
-                                                              + "      <-- source-map\n"
-                                                              + "    Processor: process-mapValues (stores: [])\n"
-                                                              + "      --> process\n"
-                                                              + "      <-- process-filter\n"
-                                                              + "    Sink: KSTREAM-SINK-0000000034 (topic: count-groupByKey-repartition)\n"
-                                                              + "      <-- KSTREAM-FILTER-0000000035\n"
-                                                              + "    Processor: process (stores: [])\n"
-                                                              + "      --> none\n"
-                                                              + "      <-- process-mapValues\n\n";
+                                                                  + "   Sub-topology: 0\n"
+                                                                  + "    Source: sourceStream (topics: [input])\n"
+                                                                  + "      --> source-map\n"
+                                                                  + "    Processor: source-map (stores: [])\n"
+                                                                  + "      --> process-filter, KSTREAM-FILTER-0000000035\n"
+                                                                  + "      <-- sourceStream\n"
+                                                                  + "    Processor: process-filter (stores: [])\n"
+                                                                  + "      --> process-mapValues\n"
+                                                                  + "      <-- source-map\n"
+                                                                  + "    Processor: KSTREAM-FILTER-0000000035 (stores: [])\n"
+                                                                  + "      --> KSTREAM-SINK-0000000034\n"
+                                                                  + "      <-- source-map\n"
+                                                                  + "    Processor: process-mapValues (stores: [])\n"
+                                                                  + "      --> process\n"
+                                                                  + "      <-- process-filter\n"
+                                                                  + "    Sink: KSTREAM-SINK-0000000034 (topic: count-groupByKey-repartition)\n"
+                                                                  + "      <-- KSTREAM-FILTER-0000000035\n"
+                                                                  + "    Processor: process (stores: [])\n"
+                                                                  + "      --> none\n"
+                                                                  + "      <-- process-mapValues\n"
+                                                                  + "\n"
+                                                                  + "  Sub-topology: 1\n"
+                                                                  + "    Source: KSTREAM-SOURCE-0000000036 (topics: [count-groupByKey-repartition])\n"
+                                                                  + "      --> aggregate, count, join-filter, reduce-filter\n"
+                                                                  + "    Processor: count (stores: [count-store])\n"
+                                                                  + "      --> count-toStream\n"
+                                                                  + "      <-- KSTREAM-SOURCE-0000000036\n"
+                                                                  + "    Processor: count-toStream (stores: [])\n"
+                                                                  + "      --> join-other-windowed, count-to\n"
+                                                                  + "      <-- count\n"
+                                                                  + "    Processor: join-filter (stores: [])\n"
+                                                                  + "      --> join-this-windowed\n"
+                                                                  + "      <-- KSTREAM-SOURCE-0000000036\n"
+                                                                  + "    Processor: reduce-filter (stores: [])\n"
+                                                                  + "      --> reduce-peek\n"
+                                                                  + "      <-- KSTREAM-SOURCE-0000000036\n"
+                                                                  + "    Processor: join-other-windowed (stores: [other-join-store])\n"
+                                                                  + "      --> join-other-join\n"
+                                                                  + "      <-- count-toStream\n"
+                                                                  + "    Processor: join-this-windowed (stores: [join-store])\n"
+                                                                  + "      --> join-this-join\n"
+                                                                  + "      <-- join-filter\n"
+                                                                  + "    Processor: reduce-peek (stores: [])\n"
+                                                                  + "      --> reducer\n"
+                                                                  + "      <-- reduce-filter\n"
+                                                                  + "    Processor: aggregate (stores: [aggregate-store])\n"
+                                                                  + "      --> aggregate-toStream\n"
+                                                                  + "      <-- KSTREAM-SOURCE-0000000036\n"
+                                                                  + "    Processor: join-other-join (stores: [join-store])\n"
+                                                                  + "      --> join-merge\n"
+                                                                  + "      <-- join-other-windowed\n"
+                                                                  + "    Processor: join-this-join (stores: [other-join-store])\n"
+                                                                  + "      --> join-merge\n"
+                                                                  + "      <-- join-this-windowed\n"
+                                                                  + "    Processor: reducer (stores: [reduce-store])\n"
+                                                                  + "      --> reduce-toStream\n"
+                                                                  + "      <-- reduce-peek\n"
+                                                                  + "    Processor: aggregate-toStream (stores: [])\n"
+                                                                  + "      --> reduce-to\n"
+                                                                  + "      <-- aggregate\n"
+                                                                  + "    Processor: join-merge (stores: [])\n"
+                                                                  + "      --> join-to\n"
+                                                                  + "      <-- join-this-join, join-other-join\n"
+                                                                  + "    Processor: reduce-toStream (stores: [])\n"
+                                                                  + "      --> KSTREAM-SINK-0000000023\n"
+                                                                  + "      <-- reducer\n"
+                                                                  + "    Sink: KSTREAM-SINK-0000000023 (topic: outputTopic_2)\n"
+                                                                  + "      <-- reduce-toStream\n"
+                                                                  + "    Sink: count-to (topic: outputTopic_0)\n"
+                                                                  + "      <-- count-toStream\n"
+                                                                  + "    Sink: join-to (topic: joinedOutputTopic)\n"
+                                                                  + "      <-- join-merge\n"
+                                                                  + "    Sink: reduce-to (topic: outputTopic_1)\n"
+                                                                  + "      <-- aggregate-toStream\n\n";
+
 
 
 
     private static final String EXPECTED_UNOPTIMIZED_TOPOLOGY = "Topologies:\n"
-                                                                + "   Sub-topology: 0\n"
-                                                                + "    Source: KSTREAM-SOURCE-0000000007 (topics: [count-groupByKey-repartition])\n"
-                                                                + "      --> count\n"
-                                                                + "    Processor: count (stores: [count-store])\n"
-                                                                + "      --> count-toStream\n"
-                                                                + "      <-- KSTREAM-SOURCE-0000000007\n"
-                                                                + "    Processor: count-toStream (stores: [])\n"
-                                                                + "      --> join-other-windowed, count-to\n"
-                                                                + "      <-- count\n"
-                                                                + "    Source: KSTREAM-SOURCE-0000000027 (topics: [join-left-repartition])\n"
-                                                                + "      --> join-this-windowed\n"
-                                                                + "    Processor: join-other-windowed (stores: [other-join-store])\n"
-                                                                + "      --> join-other-join\n"
-                                                                + "      <-- count-toStream\n"
-                                                                + "    Processor: join-this-windowed (stores: [join-store])\n"
-                                                                + "      --> join-this-join\n"
-                                                                + "      <-- KSTREAM-SOURCE-0000000027\n"
-                                                                + "    Processor: join-other-join (stores: [join-store])\n"
-                                                                + "      --> join-merge\n"
-                                                                + "      <-- join-other-windowed\n"
-                                                                + "    Processor: join-this-join (stores: [other-join-store])\n"
-                                                                + "      --> join-merge\n"
-                                                                + "      <-- join-this-windowed\n"
-                                                                + "    Processor: join-merge (stores: [])\n"
-                                                                + "      --> join-to\n"
-                                                                + "      <-- join-this-join, join-other-join\n"
-                                                                + "    Sink: count-to (topic: outputTopic_0)\n"
-                                                                + "      <-- count-toStream\n"
-                                                                + "    Sink: join-to (topic: joinedOutputTopic)\n"
-                                                                + "      <-- join-merge\n"
-                                                                + "\n"
-                                                                + "  Sub-topology: 1\n"
-                                                                + "    Source: KSTREAM-SOURCE-0000000013 (topics: [aggregate-groupByKey-repartition])\n"
-                                                                + "      --> aggregate\n"
-                                                                + "    Processor: aggregate (stores: [aggregate-store])\n"
-                                                                + "      --> aggregate-toStream\n"
-                                                                + "      <-- KSTREAM-SOURCE-0000000013\n"
-                                                                + "    Processor: aggregate-toStream (stores: [])\n"
-                                                                + "      --> reduce-to\n"
-                                                                + "      <-- aggregate\n"
-                                                                + "    Sink: reduce-to (topic: outputTopic_1)\n"
-                                                                + "      <-- aggregate-toStream\n"
-                                                                + "\n"
-                                                                + "  Sub-topology: 2\n"
-                                                                + "    Source: KSTREAM-SOURCE-0000000021 (topics: [reduce-groupByKey-repartition])\n"
-                                                                + "      --> reducer\n"
-                                                                + "    Processor: reducer (stores: [reduce-store])\n"
-                                                                + "      --> reduce-toStream\n"
-                                                                + "      <-- KSTREAM-SOURCE-0000000021\n"
-                                                                + "    Processor: reduce-toStream (stores: [])\n"
-                                                                + "      --> KSTREAM-SINK-0000000023\n"
-                                                                + "      <-- reducer\n"
-                                                                + "    Sink: KSTREAM-SINK-0000000023 (topic: outputTopic_2)\n"
-                                                                + "      <-- reduce-toStream\n"
-                                                                + "\n"
-                                                                + "  Sub-topology: 3\n"
-                                                                + "    Source: sourceStream (topics: [input])\n"
-                                                                + "      --> source-map\n"
-                                                                + "    Processor: source-map (stores: [])\n"
-                                                                + "      --> reduce-filter, process-filter, KSTREAM-FILTER-0000000006, join-filter, KSTREAM-FILTER-0000000012\n"
-                                                                + "      <-- sourceStream\n"
-                                                                + "    Processor: reduce-filter (stores: [])\n"
-                                                                + "      --> reduce-peek\n"
-                                                                + "      <-- source-map\n"
-                                                                + "    Processor: join-filter (stores: [])\n"
-                                                                + "      --> KSTREAM-FILTER-0000000026\n"
-                                                                + "      <-- source-map\n"
-                                                                + "    Processor: process-filter (stores: [])\n"
-                                                                + "      --> process-mapValues\n"
-                                                                + "      <-- source-map\n"
-                                                                + "    Processor: reduce-peek (stores: [])\n"
-                                                                + "      --> KSTREAM-FILTER-0000000020\n"
-                                                                + "      <-- reduce-filter\n"
-                                                                + "    Processor: KSTREAM-FILTER-0000000006 (stores: [])\n"
-                                                                + "      --> KSTREAM-SINK-0000000005\n"
-                                                                + "      <-- source-map\n"
-                                                                + "    Processor: KSTREAM-FILTER-0000000012 (stores: [])\n"
-                                                                + "      --> KSTREAM-SINK-0000000011\n"
-                                                                + "      <-- source-map\n"
-                                                                + "    Processor: KSTREAM-FILTER-0000000020 (stores: [])\n"
-                                                                + "      --> KSTREAM-SINK-0000000019\n"
-                                                                + "      <-- reduce-peek\n"
-                                                                + "    Processor: KSTREAM-FILTER-0000000026 (stores: [])\n"
-                                                                + "      --> KSTREAM-SINK-0000000025\n"
-                                                                + "      <-- join-filter\n"
-                                                                + "    Processor: process-mapValues (stores: [])\n"
-                                                                + "      --> process\n"
-                                                                + "      <-- process-filter\n"
-                                                                + "    Sink: KSTREAM-SINK-0000000005 (topic: count-groupByKey-repartition)\n"
-                                                                + "      <-- KSTREAM-FILTER-0000000006\n"
-                                                                + "    Sink: KSTREAM-SINK-0000000011 (topic: aggregate-groupByKey-repartition)\n"
-                                                                + "      <-- KSTREAM-FILTER-0000000012\n"
-                                                                + "    Sink: KSTREAM-SINK-0000000019 (topic: reduce-groupByKey-repartition)\n"
-                                                                + "      <-- KSTREAM-FILTER-0000000020\n"
-                                                                + "    Sink: KSTREAM-SINK-0000000025 (topic: join-left-repartition)\n"
-                                                                + "      <-- KSTREAM-FILTER-0000000026\n"
-                                                                + "    Processor: process (stores: [])\n"
-                                                                + "      --> none\n"
-                                                                + "      <-- process-mapValues\n\n";
+                                                                    + "   Sub-topology: 0\n"
+                                                                    + "    Source: sourceStream (topics: [input])\n"
+                                                                    + "      --> source-map\n"
+                                                                    + "    Processor: source-map (stores: [])\n"
+                                                                    + "      --> reduce-filter, process-filter, KSTREAM-FILTER-0000000006, join-filter, KSTREAM-FILTER-0000000012\n"
+                                                                    + "      <-- sourceStream\n"
+                                                                    + "    Processor: reduce-filter (stores: [])\n"
+                                                                    + "      --> reduce-peek\n"
+                                                                    + "      <-- source-map\n"
+                                                                    + "    Processor: join-filter (stores: [])\n"
+                                                                    + "      --> KSTREAM-FILTER-0000000026\n"
+                                                                    + "      <-- source-map\n"
+                                                                    + "    Processor: process-filter (stores: [])\n"
+                                                                    + "      --> process-mapValues\n"
+                                                                    + "      <-- source-map\n"
+                                                                    + "    Processor: reduce-peek (stores: [])\n"
+                                                                    + "      --> KSTREAM-FILTER-0000000020\n"
+                                                                    + "      <-- reduce-filter\n"
+                                                                    + "    Processor: KSTREAM-FILTER-0000000006 (stores: [])\n"
+                                                                    + "      --> KSTREAM-SINK-0000000005\n"
+                                                                    + "      <-- source-map\n"
+                                                                    + "    Processor: KSTREAM-FILTER-0000000012 (stores: [])\n"
+                                                                    + "      --> KSTREAM-SINK-0000000011\n"
+                                                                    + "      <-- source-map\n"
+                                                                    + "    Processor: KSTREAM-FILTER-0000000020 (stores: [])\n"
+                                                                    + "      --> KSTREAM-SINK-0000000019\n"
+                                                                    + "      <-- reduce-peek\n"
+                                                                    + "    Processor: KSTREAM-FILTER-0000000026 (stores: [])\n"
+                                                                    + "      --> KSTREAM-SINK-0000000025\n"
+                                                                    + "      <-- join-filter\n"
+                                                                    + "    Processor: process-mapValues (stores: [])\n"
+                                                                    + "      --> process\n"
+                                                                    + "      <-- process-filter\n"
+                                                                    + "    Sink: KSTREAM-SINK-0000000005 (topic: count-groupByKey-repartition)\n"
+                                                                    + "      <-- KSTREAM-FILTER-0000000006\n"
+                                                                    + "    Sink: KSTREAM-SINK-0000000011 (topic: aggregate-groupByKey-repartition)\n"
+                                                                    + "      <-- KSTREAM-FILTER-0000000012\n"
+                                                                    + "    Sink: KSTREAM-SINK-0000000019 (topic: reduce-groupByKey-repartition)\n"
+                                                                    + "      <-- KSTREAM-FILTER-0000000020\n"
+                                                                    + "    Sink: KSTREAM-SINK-0000000025 (topic: join-left-repartition)\n"
+                                                                    + "      <-- KSTREAM-FILTER-0000000026\n"
+                                                                    + "    Processor: process (stores: [])\n"
+                                                                    + "      --> none\n"
+                                                                    + "      <-- process-mapValues\n"
+                                                                    + "\n"
+                                                                    + "  Sub-topology: 1\n"
+                                                                    + "    Source: KSTREAM-SOURCE-0000000007 (topics: [count-groupByKey-repartition])\n"
+                                                                    + "      --> count\n"
+                                                                    + "    Processor: count (stores: [count-store])\n"
+                                                                    + "      --> count-toStream\n"
+                                                                    + "      <-- KSTREAM-SOURCE-0000000007\n"
+                                                                    + "    Processor: count-toStream (stores: [])\n"
+                                                                    + "      --> join-other-windowed, count-to\n"
+                                                                    + "      <-- count\n"
+                                                                    + "    Source: KSTREAM-SOURCE-0000000027 (topics: [join-left-repartition])\n"
+                                                                    + "      --> join-this-windowed\n"
+                                                                    + "    Processor: join-other-windowed (stores: [other-join-store])\n"
+                                                                    + "      --> join-other-join\n"
+                                                                    + "      <-- count-toStream\n"
+                                                                    + "    Processor: join-this-windowed (stores: [join-store])\n"
+                                                                    + "      --> join-this-join\n"
+                                                                    + "      <-- KSTREAM-SOURCE-0000000027\n"
+                                                                    + "    Processor: join-other-join (stores: [join-store])\n"
+                                                                    + "      --> join-merge\n"
+                                                                    + "      <-- join-other-windowed\n"
+                                                                    + "    Processor: join-this-join (stores: [other-join-store])\n"
+                                                                    + "      --> join-merge\n"
+                                                                    + "      <-- join-this-windowed\n"
+                                                                    + "    Processor: join-merge (stores: [])\n"
+                                                                    + "      --> join-to\n"
+                                                                    + "      <-- join-this-join, join-other-join\n"
+                                                                    + "    Sink: count-to (topic: outputTopic_0)\n"
+                                                                    + "      <-- count-toStream\n"
+                                                                    + "    Sink: join-to (topic: joinedOutputTopic)\n"
+                                                                    + "      <-- join-merge\n"
+                                                                    + "\n"
+                                                                    + "  Sub-topology: 2\n"
+                                                                    + "    Source: KSTREAM-SOURCE-0000000013 (topics: [aggregate-groupByKey-repartition])\n"
+                                                                    + "      --> aggregate\n"
+                                                                    + "    Processor: aggregate (stores: [aggregate-store])\n"
+                                                                    + "      --> aggregate-toStream\n"
+                                                                    + "      <-- KSTREAM-SOURCE-0000000013\n"
+                                                                    + "    Processor: aggregate-toStream (stores: [])\n"
+                                                                    + "      --> reduce-to\n"
+                                                                    + "      <-- aggregate\n"
+                                                                    + "    Sink: reduce-to (topic: outputTopic_1)\n"
+                                                                    + "      <-- aggregate-toStream\n"
+                                                                    + "\n"
+                                                                    + "  Sub-topology: 3\n"
+                                                                    + "    Source: KSTREAM-SOURCE-0000000021 (topics: [reduce-groupByKey-repartition])\n"
+                                                                    + "      --> reducer\n"
+                                                                    + "    Processor: reducer (stores: [reduce-store])\n"
+                                                                    + "      --> reduce-toStream\n"
+                                                                    + "      <-- KSTREAM-SOURCE-0000000021\n"
+                                                                    + "    Processor: reduce-toStream (stores: [])\n"
+                                                                    + "      --> KSTREAM-SINK-0000000023\n"
+                                                                    + "      <-- reducer\n"
+                                                                    + "    Sink: KSTREAM-SINK-0000000023 (topic: outputTopic_2)\n"
+                                                                    + "      <-- reduce-toStream\n\n";
 
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionWithMergeOptimizingTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionWithMergeOptimizingTest.java
index 0d081f71c5632..7e12abed63a04 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionWithMergeOptimizingTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionWithMergeOptimizingTest.java
@@ -211,6 +211,26 @@ private List> getKeyValues() {
 
     private static final String EXPECTED_OPTIMIZED_TOPOLOGY = "Topologies:\n"
                                                               + "   Sub-topology: 0\n"
+                                                              + "    Source: sourceAStream (topics: [inputA])\n"
+                                                              + "      --> mappedAStream\n"
+                                                              + "    Source: sourceBStream (topics: [inputB])\n"
+                                                              + "      --> mappedBStream\n"
+                                                              + "    Processor: mappedAStream (stores: [])\n"
+                                                              + "      --> mergedStream\n"
+                                                              + "      <-- sourceAStream\n"
+                                                              + "    Processor: mappedBStream (stores: [])\n"
+                                                              + "      --> mergedStream\n"
+                                                              + "      <-- sourceBStream\n"
+                                                              + "    Processor: mergedStream (stores: [])\n"
+                                                              + "      --> KSTREAM-FILTER-0000000019\n"
+                                                              + "      <-- mappedAStream, mappedBStream\n"
+                                                              + "    Processor: KSTREAM-FILTER-0000000019 (stores: [])\n"
+                                                              + "      --> KSTREAM-SINK-0000000018\n"
+                                                              + "      <-- mergedStream\n"
+                                                              + "    Sink: KSTREAM-SINK-0000000018 (topic: long-groupByKey-repartition)\n"
+                                                              + "      <-- KSTREAM-FILTER-0000000019\n"
+                                                              + "\n"
+                                                              + "  Sub-topology: 1\n"
                                                               + "    Source: KSTREAM-SOURCE-0000000020 (topics: [long-groupByKey-repartition])\n"
                                                               + "      --> long-count, string-count\n"
                                                               + "    Processor: string-count (stores: [string-store])\n"
@@ -231,81 +251,60 @@ private List> getKeyValues() {
                                                               + "    Sink: long-to (topic: outputTopic_0)\n"
                                                               + "      <-- long-toStream\n"
                                                               + "    Sink: string-to (topic: outputTopic_1)\n"
-                                                              + "      <-- string-mapValues\n"
-                                                              + "\n"
-                                                              + "  Sub-topology: 1\n"
-                                                              + "    Source: sourceAStream (topics: [inputA])\n"
-                                                              + "      --> mappedAStream\n"
-                                                              + "    Source: sourceBStream (topics: [inputB])\n"
-                                                              + "      --> mappedBStream\n"
-                                                              + "    Processor: mappedAStream (stores: [])\n"
-                                                              + "      --> mergedStream\n"
-                                                              + "      <-- sourceAStream\n"
-                                                              + "    Processor: mappedBStream (stores: [])\n"
-                                                              + "      --> mergedStream\n"
-                                                              + "      <-- sourceBStream\n"
-                                                              + "    Processor: mergedStream (stores: [])\n"
-                                                              + "      --> KSTREAM-FILTER-0000000019\n"
-                                                              + "      <-- mappedAStream, mappedBStream\n"
-                                                              + "    Processor: KSTREAM-FILTER-0000000019 (stores: [])\n"
-                                                              + "      --> KSTREAM-SINK-0000000018\n"
-                                                              + "      <-- mergedStream\n"
-                                                              + "    Sink: KSTREAM-SINK-0000000018 (topic: long-groupByKey-repartition)\n"
-                                                              + "      <-- KSTREAM-FILTER-0000000019\n\n";
+                                                              + "      <-- string-mapValues\n\n";
 
 
     private static final String EXPECTED_UNOPTIMIZED_TOPOLOGY = "Topologies:\n"
-                                                                + "   Sub-topology: 0\n"
-                                                                + "    Source: KSTREAM-SOURCE-0000000008 (topics: [long-groupByKey-repartition])\n"
-                                                                + "      --> long-count\n"
-                                                                + "    Processor: long-count (stores: [long-store])\n"
-                                                                + "      --> long-toStream\n"
-                                                                + "      <-- KSTREAM-SOURCE-0000000008\n"
-                                                                + "    Processor: long-toStream (stores: [])\n"
-                                                                + "      --> long-to\n"
-                                                                + "      <-- long-count\n"
-                                                                + "    Sink: long-to (topic: outputTopic_0)\n"
-                                                                + "      <-- long-toStream\n"
-                                                                + "\n"
-                                                                + "  Sub-topology: 1\n"
-                                                                + "    Source: KSTREAM-SOURCE-0000000014 (topics: [string-groupByKey-repartition])\n"
-                                                                + "      --> string-count\n"
-                                                                + "    Processor: string-count (stores: [string-store])\n"
-                                                                + "      --> string-toStream\n"
-                                                                + "      <-- KSTREAM-SOURCE-0000000014\n"
-                                                                + "    Processor: string-toStream (stores: [])\n"
-                                                                + "      --> string-mapValues\n"
-                                                                + "      <-- string-count\n"
-                                                                + "    Processor: string-mapValues (stores: [])\n"
-                                                                + "      --> string-to\n"
-                                                                + "      <-- string-toStream\n"
-                                                                + "    Sink: string-to (topic: outputTopic_1)\n"
-                                                                + "      <-- string-mapValues\n"
-                                                                + "\n"
-                                                                + "  Sub-topology: 2\n"
-                                                                + "    Source: sourceAStream (topics: [inputA])\n"
-                                                                + "      --> mappedAStream\n"
-                                                                + "    Source: sourceBStream (topics: [inputB])\n"
-                                                                + "      --> mappedBStream\n"
-                                                                + "    Processor: mappedAStream (stores: [])\n"
-                                                                + "      --> mergedStream\n"
-                                                                + "      <-- sourceAStream\n"
-                                                                + "    Processor: mappedBStream (stores: [])\n"
-                                                                + "      --> mergedStream\n"
-                                                                + "      <-- sourceBStream\n"
-                                                                + "    Processor: mergedStream (stores: [])\n"
-                                                                + "      --> KSTREAM-FILTER-0000000007, KSTREAM-FILTER-0000000013\n"
-                                                                + "      <-- mappedAStream, mappedBStream\n"
-                                                                + "    Processor: KSTREAM-FILTER-0000000007 (stores: [])\n"
-                                                                + "      --> KSTREAM-SINK-0000000006\n"
-                                                                + "      <-- mergedStream\n"
-                                                                + "    Processor: KSTREAM-FILTER-0000000013 (stores: [])\n"
-                                                                + "      --> KSTREAM-SINK-0000000012\n"
-                                                                + "      <-- mergedStream\n"
-                                                                + "    Sink: KSTREAM-SINK-0000000006 (topic: long-groupByKey-repartition)\n"
-                                                                + "      <-- KSTREAM-FILTER-0000000007\n"
-                                                                + "    Sink: KSTREAM-SINK-0000000012 (topic: string-groupByKey-repartition)\n"
-                                                                + "      <-- KSTREAM-FILTER-0000000013\n\n";
-
+                                                                    + "   Sub-topology: 0\n"
+                                                                    + "    Source: sourceAStream (topics: [inputA])\n"
+                                                                    + "      --> mappedAStream\n"
+                                                                    + "    Source: sourceBStream (topics: [inputB])\n"
+                                                                    + "      --> mappedBStream\n"
+                                                                    + "    Processor: mappedAStream (stores: [])\n"
+                                                                    + "      --> mergedStream\n"
+                                                                    + "      <-- sourceAStream\n"
+                                                                    + "    Processor: mappedBStream (stores: [])\n"
+                                                                    + "      --> mergedStream\n"
+                                                                    + "      <-- sourceBStream\n"
+                                                                    + "    Processor: mergedStream (stores: [])\n"
+                                                                    + "      --> KSTREAM-FILTER-0000000007, KSTREAM-FILTER-0000000013\n"
+                                                                    + "      <-- mappedAStream, mappedBStream\n"
+                                                                    + "    Processor: KSTREAM-FILTER-0000000007 (stores: [])\n"
+                                                                    + "      --> KSTREAM-SINK-0000000006\n"
+                                                                    + "      <-- mergedStream\n"
+                                                                    + "    Processor: KSTREAM-FILTER-0000000013 (stores: [])\n"
+                                                                    + "      --> KSTREAM-SINK-0000000012\n"
+                                                                    + "      <-- mergedStream\n"
+                                                                    + "    Sink: KSTREAM-SINK-0000000006 (topic: long-groupByKey-repartition)\n"
+                                                                    + "      <-- KSTREAM-FILTER-0000000007\n"
+                                                                    + "    Sink: KSTREAM-SINK-0000000012 (topic: string-groupByKey-repartition)\n"
+                                                                    + "      <-- KSTREAM-FILTER-0000000013\n"
+                                                                    + "\n"
+                                                                    + "  Sub-topology: 1\n"
+                                                                    + "    Source: KSTREAM-SOURCE-0000000008 (topics: [long-groupByKey-repartition])\n"
+                                                                    + "      --> long-count\n"
+                                                                    + "    Processor: long-count (stores: [long-store])\n"
+                                                                    + "      --> long-toStream\n"
+                                                                    + "      <-- KSTREAM-SOURCE-0000000008\n"
+                                                                    + "    Processor: long-toStream (stores: [])\n"
+                                                                    + "      --> long-to\n"
+                                                                    + "      <-- long-count\n"
+                                                                    + "    Sink: long-to (topic: outputTopic_0)\n"
+                                                                    + "      <-- long-toStream\n"
+                                                                    + "\n"
+                                                                    + "  Sub-topology: 2\n"
+                                                                    + "    Source: KSTREAM-SOURCE-0000000014 (topics: [string-groupByKey-repartition])\n"
+                                                                    + "      --> string-count\n"
+                                                                    + "    Processor: string-count (stores: [string-store])\n"
+                                                                    + "      --> string-toStream\n"
+                                                                    + "      <-- KSTREAM-SOURCE-0000000014\n"
+                                                                    + "    Processor: string-toStream (stores: [])\n"
+                                                                    + "      --> string-mapValues\n"
+                                                                    + "      <-- string-count\n"
+                                                                    + "    Processor: string-mapValues (stores: [])\n"
+                                                                    + "      --> string-to\n"
+                                                                    + "      <-- string-toStream\n"
+                                                                    + "    Sink: string-to (topic: outputTopic_1)\n"
+                                                                    + "      <-- string-mapValues\n\n";
 
 }
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
index 43324eec76cbb..da1ba77f13459 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
@@ -447,7 +447,7 @@ private void pipeRecord(final ProducerRecord record) {
     private void pipeRecord(final String topic, final Long timestamp, final byte[] key, final byte[] value, final Headers headers) {
         final String topicName = topic;
 
-        if (!internalTopologyBuilder.getSourceTopicNames().isEmpty()) {
+        if (!internalTopologyBuilder.sourceTopicNames().isEmpty()) {
             validateSourceTopicNameRegexPattern(topic);
         }
         final TopicPartition topicPartition = getTopicPartition(topicName);
@@ -495,7 +495,7 @@ private void pipeRecord(final String topic, final Long timestamp, final byte[] k
 
 
     private void validateSourceTopicNameRegexPattern(final String inputRecordTopic) {
-        for (final String sourceTopicName : internalTopologyBuilder.getSourceTopicNames()) {
+        for (final String sourceTopicName : internalTopologyBuilder.sourceTopicNames()) {
             if (!sourceTopicName.equals(inputRecordTopic) && Pattern.compile(sourceTopicName).matcher(inputRecordTopic).matches()) {
                 throw new TopologyException("Topology add source of type String for topic: " + sourceTopicName +
                         " cannot contain regex pattern for input record topic: " + inputRecordTopic +

From 1f0c25a86227a6cefae9f27e948b32a37ab6ff9d Mon Sep 17 00:00:00 2001
From: Chris Stromberger 
Date: Mon, 14 Oct 2019 16:41:57 -0500
Subject: [PATCH 0714/1071] MINOR: clarify wording around fault-tolerant state
 stores (#7510)

Reviewers: Matthias J. Sax , Guozhang Wang 
---
 docs/streams/developer-guide/processor-api.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/streams/developer-guide/processor-api.html b/docs/streams/developer-guide/processor-api.html
index 89d3447cdd78a..ae33ebb78f23a 100644
--- a/docs/streams/developer-guide/processor-api.html
+++ b/docs/streams/developer-guide/processor-api.html
@@ -289,11 +289,11 @@ 
                
                     you experience machine failure, the state store and the application’s state can be fully restored from its changelog. You can
                     enable or disable this backup feature for a
                     state store.
                
-                
                By default, persistent key-value stores are fault-tolerant.  They are backed by a
+                
                Fault-tolerant state stores are backed by a
                     compacted changelog topic.  The purpose of compacting this
                     topic is to prevent the topic from growing indefinitely, to reduce the storage consumed in the associated Kafka cluster,
                     and to minimize recovery time if a state store needs to be restored from its changelog topic.
                
-                
                Similarly, persistent window stores are fault-tolerant.  They are backed by a topic that uses both compaction and
+                
                Fault-tolerant windowed state stores are backed by a topic that uses both compaction and
                     deletion. Because of the structure of the message keys that are being sent to the changelog topics, this combination of
                     deletion and compaction is required for the changelog topics of window stores. For window stores, the message keys are
                     composite keys that include the “normal” key and window timestamps.  For these types of composite keys it would not

From 89548dd7719d12de29e2e0d4a026948f757ee499 Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Mon, 14 Oct 2019 21:22:55 -0700
Subject: [PATCH 0715/1071] MINOR: unify calls to get committed offsets and
 metadata (#7463)

Reviewers: Chris Pettitt , Bruno Cadonna , Guozhang Wang 
---
 .../processor/internals/AbstractTask.java     |  27 +---
 .../internals/AssignedStreamsTasks.java       |   2 +-
 .../processor/internals/AssignedTasks.java    |  14 +-
 .../processor/internals/StandbyTask.java      |  55 +++++---
 .../processor/internals/StreamTask.java       | 127 +++++++++--------
 .../processor/internals/StreamThread.java     |   4 +-
 .../streams/processor/internals/Task.java     |   5 +-
 .../processor/internals/AbstractTaskTest.java |  35 ++---
 .../internals/AssignedStreamsTasksTest.java   |  47 ++++---
 .../processor/internals/StandbyTaskTest.java  |   6 +-
 .../processor/internals/StreamTaskTest.java   | 128 ++++++++++--------
 .../StreamThreadStateStoreProviderTest.java   |   2 +-
 .../kafka/streams/TopologyTestDriver.java     |   2 +-
 13 files changed, 243 insertions(+), 211 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java
index 03a002127b7f3..a118898995643 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java
@@ -17,10 +17,7 @@
 package org.apache.kafka.streams.processor.internals;
 
 import org.apache.kafka.clients.consumer.Consumer;
-import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.errors.AuthorizationException;
-import org.apache.kafka.common.errors.WakeupException;
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.errors.LockException;
@@ -34,9 +31,7 @@
 import java.io.IOException;
 import java.util.Collection;
 import java.util.HashSet;
-import java.util.Map;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 public abstract class AbstractTask implements Task {
 
@@ -62,7 +57,7 @@ public abstract class AbstractTask implements Task {
      * @throws ProcessorStateException if the state manager cannot be created
      */
     AbstractTask(final TaskId id,
-                 final Collection partitions,
+                 final Set partitions,
                  final ProcessorTopology topology,
                  final Consumer consumer,
                  final ChangelogReader changelogReader,
@@ -251,24 +246,4 @@ public Collection changelogPartitions() {
         return stateMgr.changelogPartitions();
     }
 
-    Map committedOffsetForPartitions(final Set partitions) {
-        try {
-            final Map results = consumer.committed(partitions)
-                .entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().offset()));
-
-            // those do not have a committed offset would default to 0
-            for (final TopicPartition tp : partitions) {
-                results.putIfAbsent(tp, 0L);
-            }
-
-            return results;
-        } catch (final AuthorizationException e) {
-            throw new ProcessorStateException(String.format("task [%s] AuthorizationException when initializing offsets for %s", id, partitions), e);
-        } catch (final WakeupException e) {
-            throw e;
-        } catch (final KafkaException e) {
-            throw new ProcessorStateException(String.format("task [%s] Failed to initialize offsets for %s", id, partitions), e);
-        }
-    }
-
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasks.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasks.java
index 65c4c95fadeff..cba17d0cb3021 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasks.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasks.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
-import java.util.concurrent.atomic.AtomicReference;
 import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.utils.LogContext;
@@ -31,6 +30,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
 
 class AssignedStreamsTasks extends AssignedTasks implements RestoringTasks {
     private final Map suspended = new HashMap<>();
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
index 21f700bb6f5d8..8c4c68f305b38 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
@@ -67,11 +67,13 @@ void initializeNewTasks() {
         for (final Iterator> it = created.entrySet().iterator(); it.hasNext(); ) {
             final Map.Entry entry = it.next();
             try {
-                if (!entry.getValue().initializeStateStores()) {
+                final T task = entry.getValue();
+                task.initializeMetadata();
+                if (!task.initializeStateStores()) {
                     log.debug("Transitioning {} {} to restoring", taskTypeName, entry.getKey());
-                    ((AssignedStreamsTasks) this).addToRestoring((StreamTask) entry.getValue());
+                    ((AssignedStreamsTasks) this).addToRestoring((StreamTask) task);
                 } else {
-                    transitionToRunning(entry.getValue());
+                    transitionToRunning(task);
                 }
                 it.remove();
             } catch (final LockException e) {
@@ -110,7 +112,6 @@ boolean hasRunningTasks() {
     void transitionToRunning(final T task) {
         log.debug("Transitioning {} {} to running", taskTypeName, task.id());
         running.put(task.id(), task);
-        task.initializeTaskTime();
         task.initializeTopology();
         for (final TopicPartition topicPartition : task.partitions()) {
             runningByPartition.put(topicPartition, task);
@@ -251,7 +252,7 @@ void closeTask(final T task, final boolean clean) {
         task.close(clean, false);
     }
 
-    private boolean closeUnclean(final T task) {
+    private void closeUnclean(final T task) {
         log.info("Try to close {} {} unclean.", task.getClass().getSimpleName(), task.id());
         try {
             task.close(false, false);
@@ -260,10 +261,7 @@ private boolean closeUnclean(final T task) {
                 task.getClass().getSimpleName(),
                 task.id(),
                 fatalException);
-            return false;
         }
-
-        return true;
     }
 
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
index fbc116af61f0f..fc1ff4f5ad8c1 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
@@ -16,23 +16,28 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.AuthorizationException;
+import org.apache.kafka.common.errors.WakeupException;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.StreamsMetrics;
+import org.apache.kafka.streams.errors.ProcessorStateException;
 import org.apache.kafka.streams.processor.TaskId;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
 /**
  * A StandbyTask
  */
@@ -54,7 +59,7 @@ public class StandbyTask extends AbstractTask {
      * @param stateDirectory the {@link StateDirectory} created by the thread
      */
     StandbyTask(final TaskId id,
-                final Collection partitions,
+                final Set partitions,
                 final ProcessorTopology topology,
                 final Consumer consumer,
                 final ChangelogReader changelogReader,
@@ -74,6 +79,9 @@ public class StandbyTask extends AbstractTask {
         updateOffsetLimits = true;
     }
 
+    @Override
+    public void initializeMetadata() {}
+
     @Override
     public boolean initializeStateStores() {
         log.trace("Initializing state stores");
@@ -85,14 +93,7 @@ public boolean initializeStateStores() {
     }
 
     @Override
-    public void initializeTopology() {
-        //no-op
-    }
-
-    @Override
-    public void initializeTaskTime() {
-        //no-op
-    }
+    public void initializeTopology() {}
 
     /**
      * 
                @@ -219,6 +220,26 @@ private long updateOffsetLimits(final TopicPartition partition) {
         return offsetLimits.get(partition);
     }
 
+    private Map committedOffsetForPartitions(final Set partitions) {
+        try {
+            final Map results = consumer.committed(partitions)
+                .entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().offset()));
+
+            // those do not have a committed offset would default to 0
+            for (final TopicPartition tp : partitions) {
+                results.putIfAbsent(tp, 0L);
+            }
+
+            return results;
+        } catch (final AuthorizationException e) {
+            throw new ProcessorStateException(String.format("task [%s] AuthorizationException when initializing offsets for %s", id, partitions), e);
+        } catch (final WakeupException e) {
+            throw e;
+        } catch (final KafkaException e) {
+            throw new ProcessorStateException(String.format("task [%s] Failed to initialize offsets for %s", id, partitions), e);
+        }
+    }
+
     void allowUpdateOfOffsetLimit() {
         updateOffsetLimits = true;
     }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
index 780fe1ff05c3f..e859df810ece9 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
@@ -16,23 +16,6 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
-import static java.lang.String.format;
-import static java.util.Collections.singleton;
-import static org.apache.kafka.streams.kstream.internals.metrics.Sensors.recordLatenessSensor;
-
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.nio.ByteBuffer;
-import java.util.Base64;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
 import org.apache.kafka.clients.consumer.CommitFailedException;
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
@@ -41,8 +24,10 @@
 import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.AuthorizationException;
 import org.apache.kafka.common.errors.ProducerFencedException;
 import org.apache.kafka.common.errors.TimeoutException;
+import org.apache.kafka.common.errors.WakeupException;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
 import org.apache.kafka.common.metrics.stats.CumulativeCount;
@@ -65,6 +50,22 @@
 import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.apache.kafka.streams.state.internals.ThreadCache;
 
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.nio.ByteBuffer;
+import java.util.Base64;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static java.lang.String.format;
+import static java.util.Collections.singleton;
+import static org.apache.kafka.streams.kstream.internals.metrics.Sensors.recordLatenessSensor;
+
 /**
  * A StreamTask is associated with a {@link PartitionGroup}, and is assigned to a StreamThread for processing.
  */
@@ -161,7 +162,7 @@ public interface ProducerSupplier {
     }
 
     public StreamTask(final TaskId id,
-                      final Collection partitions,
+                      final Set partitions,
                       final ProcessorTopology topology,
                       final Consumer consumer,
                       final ChangelogReader changelogReader,
@@ -175,7 +176,7 @@ public StreamTask(final TaskId id,
     }
 
     public StreamTask(final TaskId id,
-                      final Collection partitions,
+                      final Set partitions,
                       final ProcessorTopology topology,
                       final Consumer consumer,
                       final ChangelogReader changelogReader,
@@ -254,9 +255,21 @@ public StreamTask(final TaskId id,
     }
 
     @Override
-    public boolean initializeStateStores() {
-        log.debug("Initializing state stores");
+    public void initializeMetadata() {
+        try {
+            final Map offsetsAndMetadata = consumer.committed(partitions);
+            initializeCommittedOffsets(offsetsAndMetadata);
+            initializeTaskTime(offsetsAndMetadata);
+        } catch (final AuthorizationException e) {
+            throw new ProcessorStateException(String.format("task [%s] AuthorizationException when initializing offsets for %s", id, partitions), e);
+        } catch (final WakeupException e) {
+            throw e;
+        } catch (final KafkaException e) {
+            throw new ProcessorStateException(String.format("task [%s] Failed to initialize offsets for %s", id, partitions), e);
+        }
+    }
 
+    private void initializeCommittedOffsets(final Map offsetsAndMetadata) {
         // Currently there is no easy way to tell the ProcessorStateManager to only restore up to
         // a specific offset. In most cases this is fine. However, in optimized topologies we can
         // have a source topic that also serves as a changelog, and in this case we want our active
@@ -264,13 +277,47 @@ public boolean initializeStateStores() {
         // partitions of topics that are both sources and changelogs and set the consumer committed
         // offset via stateMgr as there is not a more direct route.
         final Set changelogTopicNames = new HashSet<>(topology.storeToChangelogTopic().values());
-        final Set sourcePartitionsAsChangelog = new HashSet<>(partitions)
-            .stream().filter(tp -> changelogTopicNames.contains(tp.topic())).collect(Collectors.toSet());
-        final Map committedOffsets = committedOffsetForPartitions(sourcePartitionsAsChangelog);
-        stateMgr.putOffsetLimits(committedOffsets);
+        final Map committedOffsetsForChangelogs = offsetsAndMetadata
+            .entrySet()
+            .stream()
+            .filter(e -> changelogTopicNames.contains(e.getKey().topic()))
+            .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().offset()));
 
-        registerStateStores();
+        // those do not have a committed offset would default to 0
+        for (final TopicPartition tp : partitions) {
+            committedOffsetsForChangelogs.putIfAbsent(tp, 0L);
+        }
+
+        stateMgr.putOffsetLimits(committedOffsetsForChangelogs);
+    }
+
+    private void initializeTaskTime(final Map offsetsAndMetadata) {
+        for (final Map.Entry entry : offsetsAndMetadata.entrySet()) {
+            final TopicPartition partition = entry.getKey();
+            final OffsetAndMetadata metadata = entry.getValue();
+
+            if (metadata != null) {
+                final long committedTimestamp = decodeTimestamp(metadata.metadata());
+                partitionGroup.setPartitionTime(partition, committedTimestamp);
+                log.debug("A committed timestamp was detected: setting the partition time of partition {}"
+                    + " to {} in stream task {}", partition, committedTimestamp, this);
+            } else {
+                log.debug("No committed timestamp was found in metadata for partition {}", partition);
+            }
+        }
+
+        final Set nonCommitted = new HashSet<>(partitions);
+        nonCommitted.removeAll(offsetsAndMetadata.keySet());
+        for (final TopicPartition partition : nonCommitted) {
+            log.debug("No committed offset for partition {}, therefore no timestamp can be found for this partition", partition);
+        }
+    }
 
+
+    @Override
+    public boolean initializeStateStores() {
+        log.debug("Initializing state stores");
+        registerStateStores();
         return changelogPartitions().isEmpty();
     }
 
@@ -325,6 +372,7 @@ public void resume() {
                 throw new ProcessorStateException(format("%sError while deleting the checkpoint file", logPrefix), e);
             }
         }
+        initializeMetadata();
     }
 
     /**
@@ -743,33 +791,6 @@ public void close(boolean clean,
         taskClosed = true;
     }
 
-    /**
-     * Retrieves formerly committed timestamps and updates the local queue's partition time.
-     */
-    public void initializeTaskTime() {
-        final Map committed = consumer.committed(partitionGroup.partitions());
-
-        for (final Map.Entry entry : committed.entrySet()) {
-            final TopicPartition partition = entry.getKey();
-            final OffsetAndMetadata metadata = entry.getValue();
-
-            if (metadata != null) {
-                final long committedTimestamp = decodeTimestamp(metadata.metadata());
-                partitionGroup.setPartitionTime(partition, committedTimestamp);
-                log.debug("A committed timestamp was detected: setting the partition time of partition {}"
-                    + " to {} in stream task {}", partition, committedTimestamp, this);
-            } else {
-                log.debug("No committed timestamp was found in metadata for partition {}", partition);
-            }
-        }
-
-        final Set nonCommitted = new HashSet<>(partitionGroup.partitions());
-        nonCommitted.removeAll(committed.keySet());
-        for (final TopicPartition partition : nonCommitted) {
-            log.debug("No committed offset for partition {}, therefore no timestamp can be found for this partition", partition);
-        }
-    }
-
     /**
      * Adds records to queues. If a record has an invalid (i.e., negative) timestamp, the record is skipped
      * and not added to the queue for processing
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index c71ff27b17db4..222bf3ecdf01b 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -1243,9 +1243,7 @@ public Map consumerMetrics() {
 
     public Map adminClientMetrics() {
         final Map adminClientMetrics = taskManager.adminClient().metrics();
-        final LinkedHashMap result = new LinkedHashMap<>();
-        result.putAll(adminClientMetrics);
-        return result;
+        return new LinkedHashMap<>(adminClientMetrics);
     }
 
     // the following are for testing only
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/Task.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/Task.java
index af1b4bd6e8ea7..13e21b90f6fdb 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/Task.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/Task.java
@@ -26,6 +26,9 @@
 import java.util.Set;
 
 public interface Task {
+
+    void initializeMetadata();
+
     /**
      * Initialize the task and return {@code true} if the task is ready to run, i.e, it has no state stores
      * @return true if this task has no state stores that may need restoring.
@@ -65,6 +68,4 @@ void close(final boolean clean,
     boolean hasStateStores();
 
     String toString(final String indent);
-
-    void initializeTaskTime();
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractTaskTest.java
index 7ce97128b13bd..59573490358a5 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractTaskTest.java
@@ -16,21 +16,6 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
-import static org.apache.kafka.streams.processor.internals.ProcessorTopologyFactories.withLocalStores;
-import static org.easymock.EasyMock.expect;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.IOException;
-import java.time.Duration;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.utils.LogContext;
@@ -48,6 +33,22 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import java.io.File;
+import java.io.IOException;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+import static org.apache.kafka.streams.processor.internals.ProcessorTopologyFactories.withLocalStores;
+import static org.easymock.EasyMock.expect;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
 public class AbstractTaskTest {
 
     private final TaskId id = new TaskId(0, 0);
@@ -56,7 +57,7 @@ public class AbstractTaskTest {
     private final TopicPartition storeTopicPartition2 = new TopicPartition("t2", 0);
     private final TopicPartition storeTopicPartition3 = new TopicPartition("t3", 0);
     private final TopicPartition storeTopicPartition4 = new TopicPartition("t4", 0);
-    private final Collection storeTopicPartitions =
+    private final Set storeTopicPartitions =
         Utils.mkSet(storeTopicPartition1, storeTopicPartition2, storeTopicPartition3, storeTopicPartition4);
 
     @Before
@@ -218,7 +219,7 @@ private AbstractTask createTask(final Consumer consumer,
                                 config) {
 
             @Override
-            public void initializeTaskTime() {}
+            public void initializeMetadata() {}
 
             @Override
             public void resume() {}
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasksTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasksTest.java
index 94808b8043a75..fd9f0cc90c73c 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasksTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasksTest.java
@@ -17,20 +17,6 @@
 
 package org.apache.kafka.streams.processor.internals;
 
-import static org.hamcrest.CoreMatchers.not;
-import static org.hamcrest.CoreMatchers.nullValue;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.core.IsEqual.equalTo;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
 import org.apache.kafka.clients.consumer.MockConsumer;
 import org.apache.kafka.clients.consumer.OffsetResetStrategy;
 import org.apache.kafka.clients.producer.MockProducer;
@@ -52,6 +38,21 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.IsEqual.equalTo;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
 public class AssignedStreamsTasksTest {
 
     private final StreamTask t1 = EasyMock.createMock(StreamTask.class);
@@ -77,6 +78,7 @@ public void before() {
 
     @Test
     public void shouldInitializeNewTasks() {
+        t1.initializeMetadata();
         EasyMock.expect(t1.initializeStateStores()).andReturn(false);
         EasyMock.expect(t1.partitions()).andReturn(Collections.singleton(tp1));
         EasyMock.expect(t1.changelogPartitions()).andReturn(Collections.emptySet());
@@ -89,21 +91,19 @@ public void shouldInitializeNewTasks() {
 
     @Test
     public void shouldMoveInitializedTasksNeedingRestoreToRestoring() {
+        t1.initializeMetadata();
         EasyMock.expect(t1.initializeStateStores()).andReturn(false);
-        t1.initializeTaskTime();
         t1.initializeTopology();
         EasyMock.expectLastCall().once();
         EasyMock.expect(t1.partitions()).andReturn(Collections.singleton(tp1));
         EasyMock.expect(t1.changelogPartitions()).andReturn(Collections.emptySet());
+        t2.initializeMetadata();
         EasyMock.expect(t2.initializeStateStores()).andReturn(true);
-        t1.initializeTaskTime();
         t2.initializeTopology();
         EasyMock.expectLastCall().once();
         final Set t2partitions = Collections.singleton(tp2);
         EasyMock.expect(t2.partitions()).andReturn(t2partitions);
         EasyMock.expect(t2.changelogPartitions()).andReturn(Collections.emptyList());
-        t1.initializeTaskTime();
-        t2.initializeTaskTime();
 
         EasyMock.replay(t1, t2);
 
@@ -119,8 +119,8 @@ public void shouldMoveInitializedTasksNeedingRestoreToRestoring() {
 
     @Test
     public void shouldMoveInitializedTasksThatDontNeedRestoringToRunning() {
+        t2.initializeMetadata();
         EasyMock.expect(t2.initializeStateStores()).andReturn(true);
-        t2.initializeTaskTime();
         t2.initializeTopology();
         EasyMock.expectLastCall().once();
         EasyMock.expect(t2.partitions()).andReturn(Collections.singleton(tp2));
@@ -137,11 +137,11 @@ public void shouldMoveInitializedTasksThatDontNeedRestoringToRunning() {
     @Test
     public void shouldTransitionFullyRestoredTasksToRunning() {
         final Set task1Partitions = Utils.mkSet(tp1);
+        t1.initializeMetadata();
         EasyMock.expect(t1.initializeStateStores()).andReturn(false);
         EasyMock.expect(t1.partitions()).andReturn(task1Partitions).anyTimes();
         EasyMock.expect(t1.changelogPartitions()).andReturn(Utils.mkSet(changeLog1, changeLog2)).anyTimes();
         EasyMock.expect(t1.hasStateStores()).andReturn(true).anyTimes();
-        t1.initializeTaskTime();
         t1.initializeTopology();
         EasyMock.expectLastCall().once();
         EasyMock.replay(t1);
@@ -167,6 +167,7 @@ public void shouldSuspendRunningTasks() {
 
     @Test
     public void shouldCloseRestoringTasks() {
+        t1.initializeMetadata();
         EasyMock.expect(t1.initializeStateStores()).andReturn(false);
         EasyMock.expect(t1.partitions()).andReturn(Collections.singleton(tp1)).times(2);
         EasyMock.expect(t1.changelogPartitions()).andReturn(Collections.emptySet()).times(2);
@@ -246,7 +247,6 @@ public void shouldResumeMatchingSuspendedTasks() {
         mockRunningTaskSuspension();
         t1.resume();
         EasyMock.expectLastCall();
-        t1.initializeTaskTime();
         t1.initializeTopology();
         EasyMock.expectLastCall().once();
         EasyMock.replay(t1);
@@ -262,7 +262,6 @@ public void shouldResumeMatchingSuspendedTasks() {
     public void shouldCloseTaskOnResumeSuspendedIfTaskMigratedException() {
         mockRunningTaskSuspension();
         t1.resume();
-        t1.initializeTaskTime();
         t1.initializeTopology();
         EasyMock.expectLastCall().andThrow(new TaskMigratedException());
         t1.close(false, true);
@@ -281,8 +280,8 @@ public void shouldCloseTaskOnResumeSuspendedIfTaskMigratedException() {
     }
 
     private void mockTaskInitialization() {
+        t1.initializeMetadata();
         EasyMock.expect(t1.initializeStateStores()).andReturn(true);
-        t1.initializeTaskTime();
         t1.initializeTopology();
         EasyMock.expectLastCall().once();
         EasyMock.expect(t1.partitions()).andReturn(Collections.singleton(tp1));
@@ -561,8 +560,8 @@ private RuntimeException suspendTask() {
     }
 
     private void mockRunningTaskSuspension() {
+        t1.initializeMetadata();
         EasyMock.expect(t1.initializeStateStores()).andReturn(true);
-        t1.initializeTaskTime();
         t1.initializeTopology();
         EasyMock.expectLastCall().once();
         EasyMock.expect(t1.hasStateStores()).andReturn(false).anyTimes();
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
index 6f4483ae2ec62..0957ae6426ee4 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
-import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.clients.consumer.MockConsumer;
@@ -79,6 +78,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static java.time.Duration.ofMillis;
 import static java.util.Arrays.asList;
@@ -308,7 +308,7 @@ public void shouldRestoreToWindowedStores() throws IOException {
 
         final TopicPartition topicPartition = new TopicPartition(changelogName, 1);
 
-        final List partitions = Collections.singletonList(topicPartition);
+        final Set partitions = Collections.singleton(topicPartition);
 
         consumer.assign(partitions);
 
@@ -409,7 +409,7 @@ public void shouldWriteCheckpointFile() throws IOException {
         final String changelogName = applicationId + "-" + storeName + "-changelog";
 
         final TopicPartition topicPartition = new TopicPartition(changelogName, 1);
-        final List partitions = Collections.singletonList(topicPartition);
+        final Set partitions = Collections.singleton(topicPartition);
 
         final InternalTopologyBuilder internalTopologyBuilder = new InternalTopologyBuilder().setApplicationId(applicationId);
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
index bda86c4423eaf..cf33fc4e896a0 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
@@ -164,9 +164,9 @@ public void punctuate(final long timestamp) {
         }
     };
 
-    static ProcessorTopology withRepartitionTopics(final List processorNodes,
-                                                   final Map sourcesByTopic,
-                                                   final Set repartitionTopics) {
+    private static ProcessorTopology withRepartitionTopics(final List processorNodes,
+                                                           final Map sourcesByTopic,
+                                                           final Set repartitionTopics) {
         return new ProcessorTopology(processorNodes,
                                      sourcesByTopic,
                                      Collections.emptyMap(),
@@ -176,8 +176,8 @@ static ProcessorTopology withRepartitionTopics(final List process
                                      repartitionTopics);
     }
 
-    static ProcessorTopology withSources(final List processorNodes,
-                                         final Map sourcesByTopic) {
+    private static ProcessorTopology withSources(final List processorNodes,
+                                                 final Map sourcesByTopic) {
         return new ProcessorTopology(processorNodes,
                                      sourcesByTopic,
                                      Collections.emptyMap(),
@@ -233,7 +233,7 @@ public void shouldHandleInitTransactionsTimeoutExceptionOnCreation() {
 
         final ProcessorTopology topology = withSources(
             asList(source1, source2, processorStreamTime, processorSystemTime),
-            mkMap(mkEntry(topic1, (SourceNode) source1), mkEntry(topic2, (SourceNode) source2))
+            mkMap(mkEntry(topic1, source1), mkEntry(topic2, source2))
         );
 
         source1.addChild(processorStreamTime);
@@ -282,7 +282,7 @@ public void shouldHandleInitTransactionsTimeoutExceptionOnResume() {
 
         final ProcessorTopology topology = withSources(
             asList(source1, source2, processorStreamTime, processorSystemTime),
-            mkMap(mkEntry(topic1, (SourceNode) source1), mkEntry(topic2, (SourceNode) source2))
+            mkMap(mkEntry(topic1, source1), mkEntry(topic2, source2))
         );
 
         source1.addChild(processorStreamTime);
@@ -353,7 +353,6 @@ private void assertTimeoutErrorLog(final LogCaptureAppender appender) {
         assertThat(expectedError, is(singletonList("ERROR")));
     }
 
-    @SuppressWarnings("unchecked")
     @Test
     public void testProcessOrder() {
         task = createStatelessTask(createConfig(false));
@@ -437,7 +436,6 @@ private KafkaMetric getMetric(final String nameFormat, final String descriptionF
         ));
     }
 
-    @SuppressWarnings("unchecked")
     @Test
     public void testPauseResume() {
         task = createStatelessTask(createConfig(false));
@@ -492,7 +490,6 @@ public void testPauseResume() {
         assertEquals(0, consumer.paused().size());
     }
 
-    @SuppressWarnings("unchecked")
     @Test
     public void shouldPunctuateOnceStreamTimeAfterGap() {
         task = createStatelessTask(createConfig(false));
@@ -669,7 +666,7 @@ public void shouldRestorePartitionTimeAfterRestartWithEosDisabled() {
         // reset times here by creating a new task
         task = createStatelessTask(createConfig(false));
 
-        task.initializeTaskTime();
+        task.initializeMetadata();
         assertEquals(DEFAULT_TIMESTAMP, task.partitionTime(partition1));
         assertEquals(DEFAULT_TIMESTAMP, task.streamTime());
     }
@@ -678,24 +675,12 @@ public void shouldRestorePartitionTimeAfterRestartWithEosDisabled() {
     public void shouldRestorePartitionTimeAfterRestartWithEosEnabled() {
         createTaskWithProcessAndCommit(true);
 
-        // extract the committed metadata from MockProducer
-        final List>> metadataList = 
-            producer.consumerGroupOffsetsHistory();
-        final String storedMetadata = metadataList.get(0).get(APPLICATION_ID).get(partition1).metadata();
-        final long partitionTime = task.decodeTimestamp(storedMetadata);
-        assertEquals(DEFAULT_TIMESTAMP, partitionTime);
-
-        // since producer and consumer is mocked, we need to "connect" producer and consumer
-        // so we should manually commit offsets here to simulate this "connection"
-        final Map offsetMap = new HashMap<>();
-        final String encryptedMetadata = task.encodeTimestamp(partitionTime);
-        offsetMap.put(partition1, new OffsetAndMetadata(partitionTime, encryptedMetadata));
-        consumer.commitSync(offsetMap);
+        moveCommittedOffsetsFromProducerToConsumer(DEFAULT_TIMESTAMP);
 
         // reset times here by creating a new task
         task = createStatelessTask(createConfig(true));
 
-        task.initializeTaskTime();
+        task.initializeMetadata();
         assertEquals(DEFAULT_TIMESTAMP, task.partitionTime(partition1));
         assertEquals(DEFAULT_TIMESTAMP, task.streamTime());
     }
@@ -890,11 +875,8 @@ public void shouldWrapKafkaExceptionsWithStreamsExceptionAndAddContextWhenPunctu
         task.initializeTopology();
 
         try {
-            task.punctuate(processorStreamTime, 1, PunctuationType.STREAM_TIME, new Punctuator() {
-                @Override
-                public void punctuate(final long timestamp) {
-                    throw new KafkaException("KABOOM!");
-                }
+            task.punctuate(processorStreamTime, 1, PunctuationType.STREAM_TIME, timestamp -> {
+                throw new KafkaException("KABOOM!");
             });
             fail("Should've thrown StreamsException");
         } catch (final StreamsException e) {
@@ -911,11 +893,8 @@ public void shouldWrapKafkaExceptionsWithStreamsExceptionAndAddContextWhenPunctu
         task.initializeTopology();
 
         try {
-            task.punctuate(processorSystemTime, 1, PunctuationType.WALL_CLOCK_TIME, new Punctuator() {
-                @Override
-                public void punctuate(final long timestamp) {
-                    throw new KafkaException("KABOOM!");
-                }
+            task.punctuate(processorSystemTime, 1, PunctuationType.WALL_CLOCK_TIME, timestamp -> {
+                throw new KafkaException("KABOOM!");
             });
             fail("Should've thrown StreamsException");
         } catch (final StreamsException e) {
@@ -1015,24 +994,14 @@ public void shouldSetProcessorNodeOnContextBackToNullAfterSuccessfulPunctuate()
     @Test(expected = IllegalStateException.class)
     public void shouldThrowIllegalStateExceptionOnScheduleIfCurrentNodeIsNull() {
         task = createStatelessTask(createConfig(false));
-        task.schedule(1, PunctuationType.STREAM_TIME, new Punctuator() {
-            @Override
-            public void punctuate(final long timestamp) {
-                // no-op
-            }
-        });
+        task.schedule(1, PunctuationType.STREAM_TIME, timestamp -> { });
     }
 
     @Test
     public void shouldNotThrowExceptionOnScheduleIfCurrentNodeIsNotNull() {
         task = createStatelessTask(createConfig(false));
         task.processorContext.setCurrentNode(processorStreamTime);
-        task.schedule(1, PunctuationType.STREAM_TIME, new Punctuator() {
-            @Override
-            public void punctuate(final long timestamp) {
-                // no-op
-            }
-        });
+        task.schedule(1, PunctuationType.STREAM_TIME, timestamp -> { });
     }
 
     @Test
@@ -1333,6 +1302,41 @@ public void shouldWrapProducerFencedExceptionWithTaskMigragedExceptionInSuspendW
         assertTrue(producer.transactionCommitted());
     }
 
+    @Test
+    public void shouldInitTaskTimeOnResumeWithEosDisabled() {
+        shouldInitTaskTimeOnResume(false);
+    }
+
+    @Test
+    public void shouldInitTaskTimeOnResumeWithEosEnabled() {
+        shouldInitTaskTimeOnResume(true);
+    }
+
+    private void shouldInitTaskTimeOnResume(final boolean eosEnabled) {
+        task = createStatelessTask(createConfig(eosEnabled));
+        task.initializeTopology();
+
+        assertThat(task.partitionTime(partition1), is(RecordQueue.UNKNOWN));
+        assertThat(task.streamTime(), is(RecordQueue.UNKNOWN));
+
+        task.addRecords(partition1, singletonList(getConsumerRecord(partition1, 0)));
+        task.process();
+        assertThat(task.partitionTime(partition1), is(0L));
+        assertThat(task.streamTime(), is(0L));
+
+        task.suspend();
+        assertThat(task.partitionTime(partition1), is(RecordQueue.UNKNOWN));
+        assertThat(task.streamTime(), is(RecordQueue.UNKNOWN));
+
+        if (eosEnabled) {
+            moveCommittedOffsetsFromProducerToConsumer(0L);
+        }
+
+        task.resume();
+        assertThat(task.partitionTime(partition1), is(0L));
+        assertThat(task.streamTime(), is(0L));
+    }
+
     @Test
     public void shouldStartNewTransactionOnResumeIfEosEnabled() {
         task = createStatelessTask(createConfig(true));
@@ -1562,13 +1566,8 @@ public void shouldAlwaysCommitIfEosEnabled() {
 
         task.initializeStateStores();
         task.initializeTopology();
-        task.punctuate(processorSystemTime, 5, PunctuationType.WALL_CLOCK_TIME, new Punctuator() {
-            @Override
-            public void punctuate(final long timestamp) {
-                recordCollector.send("result-topic1", 3, 5, null, 0, time.milliseconds(),
-                        new IntegerSerializer(),  new IntegerSerializer());
-            }
-        });
+        task.punctuate(processorSystemTime, 5, PunctuationType.WALL_CLOCK_TIME, timestamp -> recordCollector.send("result-topic1", 3, 5, null, 0, time.milliseconds(),
+                new IntegerSerializer(),  new IntegerSerializer()));
         task.commit();
         assertEquals(1, producer.history().size());
     }
@@ -1577,6 +1576,7 @@ public void punctuate(final long timestamp) {
     public void shouldThrowProcessorStateExceptionOnInitializeOffsetsWhenAuthorizationException() {
         final Consumer consumer = mockConsumerWithCommittedException(new AuthorizationException("message"));
         final StreamTask task = createOptimizedStatefulTask(createConfig(false), consumer);
+        task.initializeMetadata();
         task.initializeStateStores();
     }
 
@@ -1584,6 +1584,7 @@ public void shouldThrowProcessorStateExceptionOnInitializeOffsetsWhenAuthorizati
     public void shouldThrowProcessorStateExceptionOnInitializeOffsetsWhenKafkaException() {
         final Consumer consumer = mockConsumerWithCommittedException(new KafkaException("message"));
         final AbstractTask task = createOptimizedStatefulTask(createConfig(false), consumer);
+        task.initializeMetadata();
         task.initializeStateStores();
     }
 
@@ -1591,6 +1592,7 @@ public void shouldThrowProcessorStateExceptionOnInitializeOffsetsWhenKafkaExcept
     public void shouldThrowWakeupExceptionOnInitializeOffsetsWhenWakeupException() {
         final Consumer consumer = mockConsumerWithCommittedException(new WakeupException());
         final AbstractTask task = createOptimizedStatefulTask(createConfig(false), consumer);
+        task.initializeMetadata();
         task.initializeStateStores();
     }
 
@@ -1603,11 +1605,27 @@ public Map committed(final Set>> metadataList =
+            producer.consumerGroupOffsetsHistory();
+        final String storedMetadata = metadataList.get(0).get(APPLICATION_ID).get(partition1).metadata();
+        final long partitionTime = task.decodeTimestamp(storedMetadata);
+        assertThat(partitionTime, is(expectedPartitionTime));
+
+        // since producer and consumer is mocked, we need to "connect" producer and consumer
+        // so we should manually commit offsets here to simulate this "connection"
+        final Map offsetMap = new HashMap<>();
+        final String encryptedMetadata = task.encodeTimestamp(partitionTime);
+        offsetMap.put(partition1, new OffsetAndMetadata(partitionTime, encryptedMetadata));
+        consumer.commitSync(offsetMap);
+    }
+
     private StreamTask createOptimizedStatefulTask(final StreamsConfig config, final Consumer consumer) {
         final StateStore stateStore = new MockKeyValueStore(storeName, true);
 
         final ProcessorTopology topology = ProcessorTopologyFactories.with(
-            asList(source1),
+            singletonList(source1),
             mkMap(mkEntry(topic1, source1)),
             singletonList(stateStore),
             Collections.singletonMap(storeName, topic1));
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/StreamThreadStateStoreProviderTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/StreamThreadStateStoreProviderTest.java
index a2e4557c27ea3..ce816195fb9db 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/StreamThreadStateStoreProviderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/StreamThreadStateStoreProviderTest.java
@@ -304,7 +304,7 @@ private StreamTask createStreamsTask(final StreamsConfig streamsConfig,
         final Metrics metrics = new Metrics();
         return new StreamTask(
             taskId,
-            Collections.singletonList(new TopicPartition(topicName, taskId.partition)),
+            Collections.singleton(new TopicPartition(topicName, taskId.partition)),
             topology,
             clientSupplier.consumer,
             new StoreChangelogReader(
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
index da1ba77f13459..65b3f08b74862 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
@@ -388,7 +388,7 @@ public void onRestoreEnd(final TopicPartition topicPartition, final String store
         if (!partitionsByTopic.isEmpty()) {
             task = new StreamTask(
                 TASK_ID,
-                partitionsByTopic.values(),
+                new HashSet<>(partitionsByTopic.values()),
                 processorTopology,
                 consumer,
                 new StoreChangelogReader(

From 0c7dccea55b13376d78cfcb0d080d68560115758 Mon Sep 17 00:00:00 2001
From: Alex Leung 
Date: Mon, 14 Oct 2019 21:47:50 -0700
Subject: [PATCH 0716/1071] KAFKA-8671: NullPointerException occurs if topic
 associated with GlobalKTable changes (#7437)

Reviewers: Matthias J. Sax , Boyang Chen 
---
 .../internals/GlobalStateManagerImpl.java     | 20 ++++++++++++++++
 .../internals/GlobalStateManagerImplTest.java | 24 +++++++++++++++++++
 2 files changed, 44 insertions(+)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStateManagerImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStateManagerImpl.java
index 6a3959fefa6ab..18574c7e52996 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStateManagerImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStateManagerImpl.java
@@ -131,10 +131,30 @@ public Set initialize() {
         }
 
         final List stateStores = topology.globalStateStores();
+        final Map storeNameToChangelog = topology.storeToChangelogTopic();
+        final Set changelogTopics = new HashSet<>();
         for (final StateStore stateStore : stateStores) {
             globalStoreNames.add(stateStore.name());
+            final String sourceTopic = storeNameToChangelog.get(stateStore.name());
+            changelogTopics.add(sourceTopic);
             stateStore.init(globalProcessorContext, stateStore);
         }
+
+        // make sure each topic-partition from checkpointFileCache is associated with a global state store
+        checkpointFileCache.keySet().forEach(tp -> {
+            if (!changelogTopics.contains(tp.topic())) {
+                log.error("Encountered a topic-partition in the global checkpoint file not associated with any global" +
+                    " state store, topic-partition: {}, checkpoint file: {}. If this topic-partition is no longer valid," +
+                    " an application reset and state store directory cleanup will be required.",
+                    tp.topic(), checkpointFile.toString());
+                try {
+                    stateDirectory.unlockGlobalState();
+                } catch (final IOException e) {
+                    log.error("Failed to unlock the global state directory", e);
+                }
+                throw new StreamsException("Encountered a topic-partition not associated with any global state store");
+            }
+        });
         return Collections.unmodifiableSet(globalStoreNames);
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStateManagerImplTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStateManagerImplTest.java
index 425c074615753..f242a4e803f47 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStateManagerImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStateManagerImplTest.java
@@ -66,6 +66,7 @@
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -171,6 +172,29 @@ public void shouldReadCheckpointOffsets() throws IOException {
         assertEquals(expected, offsets);
     }
 
+    @Test
+    public void shouldThrowStreamsExceptionForOldTopicPartitions() throws IOException {
+        final HashMap expectedOffsets = new HashMap<>();
+        expectedOffsets.put(t1, 1L);
+        expectedOffsets.put(t2, 1L);
+        expectedOffsets.put(t3, 1L);
+        expectedOffsets.put(t4, 1L);
+
+        // add an old topic (a topic not associated with any global state store)
+        final HashMap startOffsets = new HashMap<>(expectedOffsets);
+        final TopicPartition tOld = new TopicPartition("oldTopic", 1);
+        startOffsets.put(tOld, 1L);
+
+        // start with a checkpoint file will all topic-partitions: expected and old (not
+        // associated with any global state store).
+        final OffsetCheckpoint checkpoint = new OffsetCheckpoint(checkpointFile);
+        checkpoint.write(startOffsets);
+
+        // initialize will throw exception
+        final StreamsException e = assertThrows(StreamsException.class, () -> stateManager.initialize());
+        assertThat(e.getMessage(), equalTo("Encountered a topic-partition not associated with any global state store"));
+    }
+
     @Test
     public void shouldNotDeleteCheckpointFileAfterLoaded() throws IOException {
         writeCheckpoint();

From 26677731f87116154f74da421d4f440b80599da1 Mon Sep 17 00:00:00 2001
From: Bruno Cadonna 
Date: Tue, 15 Oct 2019 11:10:42 -0700
Subject: [PATCH 0717/1071] KAFKA-8942: Document RocksDB metrics

Author: Bruno Cadonna 

Reviewers: Guozhang Wang 

Closes #7490 from cadonna/AK8942-docs-rocksdb_metrics

Minor comments

(cherry picked from commit 9c80a06466dbfade96308ef26c20c6555612191e)
Signed-off-by: Guozhang Wang 
---
 docs/ops.html | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 102 insertions(+)

diff --git a/docs/ops.html b/docs/ops.html
index 05d7de117251b..d87d16a73a888 100644
--- a/docs/ops.html
+++ b/docs/ops.html
@@ -1979,6 +1979,108 @@ 
                RocksDB Metrics
                
+  All the following metrics have a recording level of debug.
+  The metrics are collected every minute from the RocksDB state stores.
+  If a state store consists of multiple RocksDB instances as it is the case for aggregations over time and session windows,
+  each metric reports an aggregation over the RocksDB instances of the state store.
+  Note that the store-scope for built-in RocksDB state stores are currently the following:
+  
                  
+    
                • rocksdb-state (for RocksDB backed key-value store)
                  • 
+    
                • rocksdb-window-state (for RocksDB backed window store)
                  • 
+    
                • rocksdb-session-state (for RocksDB backed session store)
                  • 
+  
                
+
+  
                
+    
+    
+      
+      
+      
+    
+    
+      
+      
+      
+    
+    
+      
+      
+      
+    
+    
+      
+      
+      
+    
+    
+      
+      
+      
+    
+    
+      
+      
+      
+    
+    
+      
+      
+      
+    
+    
+      
+      
+      
+    
+    
+      
+      
+      
+    
+    
+      
+      
+      
+    
+    
+      
+      
+      
+    
+    
+      
+      
+      
+    
+    
+      
+      
+      
+    
+    
+      
+      
+      
+    
+    
+      
+      
+      
+    
+    
+      
+      
+      
+    
+    
+      
+      
+      
+    
+    
+  
                Metric/Attribute nameDescriptionMbean name
                bytes-written-rateThe average number of bytes written per second to the RocksDB state store.kafka.streams:type=stream-state-metrics,client-id=([-.\w]+),task-id=([-.\w]+),[store-scope]-id=([-.\w]+)
                bytes-written-totalThe total number of bytes written to the RocksDB state store.kafka.streams:type=stream-state-metrics,client-id=([-.\w]+),task-id=([-.\w]+),[store-scope]-id=([-.\w]+)
                bytes-read-rateThe average number of bytes read per second from the RocksDB state store.kafka.streams:type=stream-state-metrics,client-id=([-.\w]+),task-id=([-.\w]+),[store-scope]-id=([-.\w]+)
                bytes-read-totalThe total number of bytes read from the RocksDB state store.kafka.streams:type=stream-state-metrics,client-id=([-.\w]+),task-id=([-.\w]+),[store-scope]-id=([-.\w]+)
                memtable-bytes-flushed-rateThe average number of bytes flushed per second from the memtable to disk.kafka.streams:type=stream-state-metrics,client-id=([-.\w]+),task-id=([-.\w]+),[store-scope]-id=([-.\w]+)
                memtable-bytes-flushed-totalThe total number of bytes flushed from the memtable to disk.kafka.streams:type=stream-state-metrics,client-id=([-.\w]+),task-id=([-.\w]+),[store-scope]-id=([-.\w]+)
                memtable-hit-ratioThe ratio of memtable hits relative to all lookups to the memtable.kafka.streams:type=stream-state-metrics,client-id=([-.\w]+),task-id=([-.\w]+),[store-scope]-id=([-.\w]+)
                block-cache-data-hit-ratioThe ratio of block cache hits for data blocks relative to all lookups for data blocks to the block cache.kafka.streams:type=stream-state-metrics,client-id=([-.\w]+),task-id=([-.\w]+),[store-scope]-id=([-.\w]+)
                block-cache-index-hit-ratioThe ratio of block cache hits for index blocks relative to all lookups for index blocks to the block cache.kafka.streams:type=stream-state-metrics,client-id=([-.\w]+),task-id=([-.\w]+),[store-scope]-id=([-.\w]+)
                block-cache-filter-hit-ratioThe ratio of block cache hits for filter blocks relative to all lookups for filter blocks to the block cache.kafka.streams:type=stream-state-metrics,client-id=([-.\w]+),task-id=([-.\w]+),[store-scope]-id=([-.\w]+)
                write-stall-duration-avgThe average duration of write stalls in ms.kafka.streams:type=stream-state-metrics,client-id=([-.\w]+),task-id=([-.\w]+),[store-scope]-id=([-.\w]+)
                write-stall-duration-totalThe total duration of write stalls in ms.kafka.streams:type=stream-state-metrics,client-id=([-.\w]+),task-id=([-.\w]+),[store-scope]-id=([-.\w]+)
                bytes-read-compaction-rateThe average number of bytes read per second during compaction.kafka.streams:type=stream-state-metrics,client-id=([-.\w]+),task-id=([-.\w]+),[store-scope]-id=([-.\w]+)
                bytes-written-compaction-rateThe average number of bytes written per second during compaction.kafka.streams:type=stream-state-metrics,client-id=([-.\w]+),task-id=([-.\w]+),[store-scope]-id=([-.\w]+)
                number-open-filesThe number of current open files.kafka.streams:type=stream-state-metrics,client-id=([-.\w]+),task-id=([-.\w]+),[store-scope]-id=([-.\w]+)
                number-file-errors-totalThe total number of file errors occurred.kafka.streams:type=stream-state-metrics,client-id=([-.\w]+),task-id=([-.\w]+),[store-scope]-id=([-.\w]+)
                
+
   
                Record Cache Metrics
                
   All the following metrics have a recording level of debug:
 

From 3cf2b7715a5dd1a617234b1bd11b92cc0f46006f Mon Sep 17 00:00:00 2001
From: Stanislav Kozlovski 
Date: Fri, 11 Oct 2019 23:10:29 +0100
Subject: [PATCH 0718/1071] KAFKA-8725; Improve
 LogCleanerManager#grabFilthiestLog error handling (#7475)

KAFKA-7215 improved the log cleaner error handling to mitigate thread death but missed one case. Exceptions in grabFilthiestCompactedLog still cause the thread to die.

This patch improves handling to ensure that errors in that function still mark a partition as uncleanable and do not crash the thread.

Reviewers: Jason Gustafson 
---
 .../src/main/scala/kafka/log/LogCleaner.scala | 76 ++++++++++---------
 .../scala/kafka/log/LogCleanerManager.scala   | 21 +++--
 .../kafka/log/LogCleanerManagerTest.scala     | 54 +++++++++++--
 3 files changed, 103 insertions(+), 48 deletions(-)

diff --git a/core/src/main/scala/kafka/log/LogCleaner.scala b/core/src/main/scala/kafka/log/LogCleaner.scala
index e37eacf8c06bf..bcb0e72007f45 100644
--- a/core/src/main/scala/kafka/log/LogCleaner.scala
+++ b/core/src/main/scala/kafka/log/LogCleaner.scala
@@ -304,53 +304,59 @@ class LogCleaner(initialConfig: CleanerConfig,
      * Clean a log if there is a dirty log available, otherwise sleep for a bit
      */
     override def doWork(): Unit = {
-      val cleaned = cleanFilthiestLog()
+      val cleaned = tryCleanFilthiestLog()
       if (!cleaned)
         pause(config.backOffMs, TimeUnit.MILLISECONDS)
     }
 
     /**
-      * Cleans a log if there is a dirty log available
-      * @return whether a log was cleaned
-      */
-    private def cleanFilthiestLog(): Boolean = {
-      var currentLog: Option[Log] = None
-
+     * Cleans a log if there is a dirty log available
+     * @return whether a log was cleaned
+     */
+    private def tryCleanFilthiestLog(): Boolean = {
       try {
-        val preCleanStats = new PreCleanStats()
-        val cleaned = cleanerManager.grabFilthiestCompactedLog(time, preCleanStats) match {
-          case None =>
-            false
-          case Some(cleanable) =>
-            this.lastPreCleanStats = preCleanStats
-            // there's a log, clean it
-            currentLog = Some(cleanable.log)
+        cleanFilthiestLog()
+      } catch {
+        case e: LogCleaningException =>
+          warn(s"Unexpected exception thrown when cleaning log ${e.log}. Marking its partition (${e.log.topicPartition}) as uncleanable", e)
+          cleanerManager.markPartitionUncleanable(e.log.dir.getParent, e.log.topicPartition)
+
+          false
+      }
+    }
+
+    @throws(classOf[LogCleaningException])
+    private def cleanFilthiestLog(): Boolean = {
+      val preCleanStats = new PreCleanStats()
+      val cleaned = cleanerManager.grabFilthiestCompactedLog(time, preCleanStats) match {
+        case None =>
+          false
+        case Some(cleanable) =>
+          // there's a log, clean it
+          this.lastPreCleanStats = preCleanStats
+          try {
             cleanLog(cleanable)
             true
-        }
-        val deletable: Iterable[(TopicPartition, Log)] = cleanerManager.deletableLogs()
-        try {
-          deletable.foreach { case (_, log) =>
-            currentLog = Some(log)
+          } catch {
+            case e @ (_: ThreadShutdownException | _: ControlThrowable) => throw e
+            case e: Exception => throw new LogCleaningException(cleanable.log, e.getMessage, e)
+          }
+      }
+      val deletable: Iterable[(TopicPartition, Log)] = cleanerManager.deletableLogs()
+      try {
+        deletable.foreach { case (_, log) =>
+          try {
             log.deleteOldSegments()
+          } catch {
+            case e @ (_: ThreadShutdownException | _: ControlThrowable) => throw e
+            case e: Exception => throw new LogCleaningException(log, e.getMessage, e)
           }
-        } finally  {
-          cleanerManager.doneDeleting(deletable.map(_._1))
         }
-
-        cleaned
-      } catch {
-        case e @ (_: ThreadShutdownException | _: ControlThrowable) => throw e
-        case e: Exception =>
-          if (currentLog.isEmpty) {
-            throw new IllegalStateException("currentLog cannot be empty on an unexpected exception", e)
-          }
-          val erroneousLog = currentLog.get
-          warn(s"Unexpected exception thrown when cleaning log $erroneousLog. Marking its partition (${erroneousLog.topicPartition}) as uncleanable", e)
-          cleanerManager.markPartitionUncleanable(erroneousLog.dir.getParent, erroneousLog.topicPartition)
-
-          false
+      } finally  {
+        cleanerManager.doneDeleting(deletable.map(_._1))
       }
+
+      cleaned
     }
 
     private def cleanLog(cleanable: LogToClean): Unit = {
diff --git a/core/src/main/scala/kafka/log/LogCleanerManager.scala b/core/src/main/scala/kafka/log/LogCleanerManager.scala
index bde30b011418a..a5cfed5c0943e 100755
--- a/core/src/main/scala/kafka/log/LogCleanerManager.scala
+++ b/core/src/main/scala/kafka/log/LogCleanerManager.scala
@@ -22,7 +22,7 @@ import java.util.concurrent.TimeUnit
 import java.util.concurrent.locks.ReentrantLock
 
 import com.yammer.metrics.core.Gauge
-import kafka.common.LogCleaningAbortedException
+import kafka.common.{KafkaException, LogCleaningAbortedException}
 import kafka.metrics.KafkaMetricsGroup
 import kafka.server.LogDirFailureChannel
 import kafka.server.checkpoints.OffsetCheckpointFile
@@ -39,6 +39,10 @@ private[log] case object LogCleaningInProgress extends LogCleaningState
 private[log] case object LogCleaningAborted extends LogCleaningState
 private[log] case class LogCleaningPaused(pausedCount: Int) extends LogCleaningState
 
+private[log] class LogCleaningException(val log: Log,
+                                        private val message: String,
+                                        private val cause: Throwable) extends KafkaException(message, cause)
+
 /**
   * This class manages the state of each partition being cleaned.
   * LogCleaningState defines the cleaning states that a TopicPartition can be in.
@@ -180,11 +184,16 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
           inProgress.contains(topicPartition) || isUncleanablePartition(log, topicPartition)
       }.map {
         case (topicPartition, log) => // create a LogToClean instance for each
-          val (firstDirtyOffset, firstUncleanableDirtyOffset) = cleanableOffsets(log, topicPartition, lastClean, now)
-          val compactionDelayMs = maxCompactionDelay(log, firstDirtyOffset, now)
-          preCleanStats.updateMaxCompactionDelay(compactionDelayMs)
-
-          LogToClean(topicPartition, log, firstDirtyOffset, firstUncleanableDirtyOffset, compactionDelayMs > 0)
+          try {
+            val (firstDirtyOffset, firstUncleanableDirtyOffset) = cleanableOffsets(log, topicPartition, lastClean, now)
+            val compactionDelayMs = maxCompactionDelay(log, firstDirtyOffset, now)
+            preCleanStats.updateMaxCompactionDelay(compactionDelayMs)
+
+            LogToClean(topicPartition, log, firstDirtyOffset, firstUncleanableDirtyOffset, compactionDelayMs > 0)
+          } catch {
+            case e: Throwable => throw new LogCleaningException(log,
+              s"Failed to calculate log cleaning stats for partition $topicPartition", e)
+          }
       }.filter(ltc => ltc.totalBytes > 0) // skip any empty logs
 
       this.dirtiestLogCleanableRatio = if (dirtyLogs.nonEmpty) dirtyLogs.max.cleanableRatio else 0
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
index 6a74874be8d94..db86c75c6adc5 100644
--- a/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
@@ -77,6 +77,42 @@ class LogCleanerManagerTest extends Logging {
     logs
   }
 
+  @Test
+  def testGrabFilthiestCompactedLogThrowsException(): Unit = {
+    val tp = new TopicPartition("A", 1)
+    val logSegmentSize = TestUtils.singletonRecords("test".getBytes).sizeInBytes * 10
+    val logSegmentsCount = 2
+    val tpDir = new File(logDir, "A-1")
+
+    // the exception should be catched and the partition that caused it marked as uncleanable
+    class LogMock(dir: File, config: LogConfig) extends Log(dir, config, 0L, 0L,
+      time.scheduler, new BrokerTopicStats, time, 60 * 60 * 1000, LogManager.ProducerIdExpirationCheckIntervalMs,
+      topicPartition, new ProducerStateManager(tp, tpDir, 60 * 60 * 1000), new LogDirFailureChannel(10)) {
+
+      // Throw an error in getFirstBatchTimestampForSegments since it is called in grabFilthiestLog()
+      override def getFirstBatchTimestampForSegments(segments: Iterable[LogSegment]): Iterable[Long] =
+        throw new IllegalStateException("Error!")
+    }
+
+    val log: Log = new LogMock(tpDir, createLowRetentionLogConfig(logSegmentSize, LogConfig.Compact))
+    writeRecords(log = log,
+      numBatches = logSegmentsCount * 2,
+      recordsPerBatch = 10,
+      batchesPerSegment = 2
+    )
+
+    val logsPool = new Pool[TopicPartition, Log]()
+    logsPool.put(tp, log)
+    val cleanerManager = createCleanerManagerMock(logsPool)
+    cleanerCheckpoints.put(tp, 1)
+
+    val thrownException = intercept[LogCleaningException] {
+      cleanerManager.grabFilthiestCompactedLog(time).get
+    }
+    assertEquals(log, thrownException.log)
+    assertTrue(thrownException.getCause.isInstanceOf[IllegalStateException])
+  }
+
   @Test
   def testGrabFilthiestCompactedLogReturnsLogWithDirtiestRatio(): Unit = {
     val tp0 = new TopicPartition("wishing-well", 0)
@@ -505,13 +541,7 @@ class LogCleanerManagerTest extends Logging {
   private def createLog(segmentSize: Int,
                         cleanupPolicy: String,
                         topicPartition: TopicPartition = new TopicPartition("log", 0)): Log = {
-    val logProps = new Properties()
-    logProps.put(LogConfig.SegmentBytesProp, segmentSize: Integer)
-    logProps.put(LogConfig.RetentionMsProp, 1: Integer)
-    logProps.put(LogConfig.CleanupPolicyProp, cleanupPolicy)
-    logProps.put(LogConfig.MinCleanableDirtyRatioProp, 0.05: java.lang.Double) // small for easier and clearer tests
-
-    val config = LogConfig(logProps)
+    val config = createLowRetentionLogConfig(segmentSize, cleanupPolicy)
     val partitionDir = new File(logDir, Log.logDirName(topicPartition))
 
     Log(partitionDir,
@@ -526,6 +556,16 @@ class LogCleanerManagerTest extends Logging {
       logDirFailureChannel = new LogDirFailureChannel(10))
   }
 
+  private def createLowRetentionLogConfig(segmentSize: Int, cleanupPolicy: String): LogConfig = {
+    val logProps = new Properties()
+    logProps.put(LogConfig.SegmentBytesProp, segmentSize: Integer)
+    logProps.put(LogConfig.RetentionMsProp, 1: Integer)
+    logProps.put(LogConfig.CleanupPolicyProp, cleanupPolicy)
+    logProps.put(LogConfig.MinCleanableDirtyRatioProp, 0.05: java.lang.Double) // small for easier and clearer tests
+
+    LogConfig(logProps)
+  }
+
   private def writeRecords(log: Log,
                            numBatches: Int,
                            recordsPerBatch: Int,

From 50d9b0c9111f13c91ae9fd6b525977e90db158fa Mon Sep 17 00:00:00 2001
From: Vikas Singh 
Date: Tue, 15 Oct 2019 11:22:26 -0700
Subject: [PATCH 0719/1071] KAFKA-8813: Refresh log config if it's updated
 before initialization (#7305)

A partition log in initialized in following steps:

1. Fetch log config from ZK
2. Call LogManager.getOrCreateLog which creates the Log object, then
3. Registers the Log object

Step #3 enables Configuration update thread to deliver configuration
updates to the log. But if any update arrives between step #1 and #3
then that update is missed. It breaks following use case:

1. Create a topic with default configuration, and immediately after that
2. Update the configuration of topic

There is a race condition here and in random cases update made in
second step will get dropped.

This change fixes it by tracking updates arriving between step #1 and #3
Once a Partition is done initializing log, it checks if it has missed any
update. If yes, then the configuration is read from ZK again.

Added unit tests to make sure a dirty configuration is refreshed. Tested
on local cluster to make sure that topic configuration and updates are
handled correctly.

Reviewers: Jason Gustafson 
---
 .../main/scala/kafka/cluster/Partition.scala  |  31 ++++--
 core/src/main/scala/kafka/log/Log.scala       |  13 +--
 .../src/main/scala/kafka/log/LogManager.scala |  55 ++++++++-
 .../scala/kafka/server/ConfigHandler.scala    |  19 +++-
 .../kafka/server/DynamicBrokerConfig.scala    |  20 ++--
 .../unit/kafka/cluster/PartitionTest.scala    | 105 +++++++++++++++++-
 .../scala/unit/kafka/log/LogManagerTest.scala |  98 ++++++++++++++++
 .../test/scala/unit/kafka/log/LogTest.scala   |   4 +-
 .../kafka/server/ReplicaManagerTest.scala     |  13 ++-
 9 files changed, 318 insertions(+), 40 deletions(-)

diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala
index c3282b778f1ee..53722585609c0 100755
--- a/core/src/main/scala/kafka/cluster/Partition.scala
+++ b/core/src/main/scala/kafka/cluster/Partition.scala
@@ -313,17 +313,28 @@ class Partition(val topicPartition: TopicPartition,
     }
   }
 
-  private def createLog(replicaId: Int, isNew: Boolean, isFutureReplica: Boolean, offsetCheckpoints: OffsetCheckpoints): Log = {
-    val props = stateStore.fetchTopicConfig()
-    val config = LogConfig.fromProps(logManager.currentDefaultConfig.originals, props)
-    val log = logManager.getOrCreateLog(topicPartition, config, isNew, isFutureReplica)
-    val checkpointHighWatermark = offsetCheckpoints.fetch(log.dir.getParent, topicPartition).getOrElse {
-      info(s"No checkpointed highwatermark is found for partition $topicPartition")
-      0L
+  // Visible for testing
+  private[cluster] def createLog(replicaId: Int, isNew: Boolean, isFutureReplica: Boolean, offsetCheckpoints: OffsetCheckpoints): Log = {
+    val fetchLogConfig = () => {
+      val props = stateStore.fetchTopicConfig()
+      LogConfig.fromProps(logManager.currentDefaultConfig.originals, props)
+    }
+
+    logManager.initializingLog(topicPartition)
+    var maybeLog: Option[Log] = None
+    try {
+      val log = logManager.getOrCreateLog(topicPartition, fetchLogConfig(), isNew, isFutureReplica)
+      val checkpointHighWatermark = offsetCheckpoints.fetch(log.dir.getParent, topicPartition).getOrElse {
+        info(s"No checkpointed highwatermark is found for partition $topicPartition")
+        0L
+      }
+      val initialHighWatermark = log.updateHighWatermark(checkpointHighWatermark)
+      info(s"Log loaded for partition $topicPartition with initial high watermark $initialHighWatermark")
+      maybeLog = Some(log)
+      log
+    } finally {
+      logManager.finishedInitializingLog(topicPartition, maybeLog, fetchLogConfig)
     }
-    val initialHighWatermark = log.updateHighWatermark(checkpointHighWatermark)
-    info(s"Log loaded for partition $topicPartition with initial high watermark $initialHighWatermark")
-    log
   }
 
   def getReplica(replicaId: Int): Option[Replica] = Option(remoteReplicasMap.get(replicaId))
diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index 94997a19a4a51..5e04c3cb55c97 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -243,16 +243,15 @@ class Log(@volatile var dir: File,
       0
   }
 
-  def updateConfig(updatedKeys: Set[String], newConfig: LogConfig): Unit = {
+  def updateConfig(newConfig: LogConfig): Unit = {
     val oldConfig = this.config
     this.config = newConfig
-    if (updatedKeys.contains(LogConfig.MessageFormatVersionProp)) {
-      val oldRecordVersion = oldConfig.messageFormatVersion.recordVersion
-      val newRecordVersion = newConfig.messageFormatVersion.recordVersion
-      if (newRecordVersion.precedes(oldRecordVersion))
-        warn(s"Record format version has been downgraded from $oldRecordVersion to $newRecordVersion.")
+    val oldRecordVersion = oldConfig.messageFormatVersion.recordVersion
+    val newRecordVersion = newConfig.messageFormatVersion.recordVersion
+    if (newRecordVersion.precedes(oldRecordVersion))
+      warn(s"Record format version has been downgraded from $oldRecordVersion to $newRecordVersion.")
+    if (newRecordVersion.value != oldRecordVersion.value)
       initializeLeaderEpochCache()
-    }
   }
 
   private def checkIfMemoryMappedBufferClosed(): Unit = {
diff --git a/core/src/main/scala/kafka/log/LogManager.scala b/core/src/main/scala/kafka/log/LogManager.scala
index c90ea7aabe4fc..075ba747ea532 100755
--- a/core/src/main/scala/kafka/log/LogManager.scala
+++ b/core/src/main/scala/kafka/log/LogManager.scala
@@ -82,6 +82,13 @@ class LogManager(logDirs: Seq[File],
   @volatile private var _currentDefaultConfig = initialDefaultConfig
   @volatile private var numRecoveryThreadsPerDataDir = recoveryThreadsPerDataDir
 
+  // This map contains all partitions whose logs are getting loaded and initialized. If log configuration
+  // of these partitions get updated at the same time, the corresponding entry in this map is set to "true",
+  // which triggers a config reload after initialization is finished (to get the latest config value).
+  // See KAFKA-8813 for more detail on the race condition
+  // Visible for testing
+  private[log] val partitionsInitializing = new ConcurrentHashMap[TopicPartition, Boolean]().asScala
+
   def reconfigureDefaultLogConfig(logConfig: LogConfig): Unit = {
     this._currentDefaultConfig = logConfig
   }
@@ -659,6 +666,48 @@ class LogManager(logDirs: Seq[File],
       Option(currentLogs.get(topicPartition))
   }
 
+  /**
+   * Method to indicate that logs are getting initialized for the partition passed in as argument.
+   * This method should always be followed by [[kafka.log.LogManager#finishedInitializingLog]] to indicate that log
+   * initialization is done.
+   */
+  def initializingLog(topicPartition: TopicPartition): Unit = {
+    partitionsInitializing(topicPartition) = false
+  }
+
+  /**
+   * Mark the partition configuration for all partitions that are getting initialized for topic
+   * as dirty. That will result in reloading of configuration once initialization is done.
+   */
+  def topicConfigUpdated(topic: String): Unit = {
+    partitionsInitializing.keys.filter(_.topic() == topic).foreach {
+      topicPartition => partitionsInitializing.replace(topicPartition, false, true)
+    }
+  }
+
+  /**
+   * Mark all in progress partitions having dirty configuration if broker configuration is updated.
+   */
+  def brokerConfigUpdated(): Unit = {
+    partitionsInitializing.keys.foreach {
+      topicPartition => partitionsInitializing.replace(topicPartition, false, true)
+    }
+  }
+
+  /**
+   * Method to indicate that the log initialization for the partition passed in as argument is
+   * finished. This method should follow a call to [[kafka.log.LogManager#initializingLog]]
+   */
+  def finishedInitializingLog(topicPartition: TopicPartition,
+                              maybeLog: Option[Log],
+                              fetchLogConfig: () => LogConfig): Unit = {
+    if (partitionsInitializing(topicPartition)) {
+      maybeLog.foreach(_.updateConfig(fetchLogConfig()))
+    }
+
+    partitionsInitializing -= topicPartition
+  }
+
   /**
    * If the log already exists, just return a copy of the existing log
    * Otherwise if isNew=true or if there is no offline log directory, create a log for the given topic and the given partition
@@ -955,9 +1004,9 @@ class LogManager(logDirs: Seq[File],
   def allLogs: Iterable[Log] = currentLogs.values ++ futureLogs.values
 
   def logsByTopic(topic: String): Seq[Log] = {
-    (currentLogs.toList ++ futureLogs.toList).filter { case (topicPartition, _) =>
-      topicPartition.topic() == topic
-    }.map { case (_, log) => log }
+    (currentLogs.toList ++ futureLogs.toList).collect {
+      case (topicPartition, log) if topicPartition.topic == topic => log
+    }
   }
 
   /**
diff --git a/core/src/main/scala/kafka/server/ConfigHandler.scala b/core/src/main/scala/kafka/server/ConfigHandler.scala
index a80d0f51334a7..2395dcde79db8 100644
--- a/core/src/main/scala/kafka/server/ConfigHandler.scala
+++ b/core/src/main/scala/kafka/server/ConfigHandler.scala
@@ -50,11 +50,11 @@ trait ConfigHandler {
   */
 class TopicConfigHandler(private val logManager: LogManager, kafkaConfig: KafkaConfig, val quotas: QuotaManagers, kafkaController: KafkaController) extends ConfigHandler with Logging  {
 
-  def processConfigChanges(topic: String, topicConfig: Properties): Unit = {
-    // Validate the configurations.
-    val configNamesToExclude = excludedConfigs(topic, topicConfig)
-
-    val logs = logManager.logsByTopic(topic).toBuffer
+  private def updateLogConfig(topic: String,
+                              topicConfig: Properties,
+                              configNamesToExclude: Set[String]): Unit = {
+    logManager.topicConfigUpdated(topic)
+    val logs = logManager.logsByTopic(topic)
     if (logs.nonEmpty) {
       /* combine the default properties with the overrides in zk to create the new LogConfig */
       val props = new Properties()
@@ -62,8 +62,15 @@ class TopicConfigHandler(private val logManager: LogManager, kafkaConfig: KafkaC
         if (!configNamesToExclude.contains(key)) props.put(key, value)
       }
       val logConfig = LogConfig.fromProps(logManager.currentDefaultConfig.originals, props)
-      logs.foreach(_.updateConfig(topicConfig.asScala.keySet, logConfig))
+      logs.foreach(_.updateConfig(logConfig))
     }
+  }
+
+  def processConfigChanges(topic: String, topicConfig: Properties): Unit = {
+    // Validate the configurations.
+    val configNamesToExclude = excludedConfigs(topic, topicConfig)
+
+    updateLogConfig(topic, topicConfig, configNamesToExclude)
 
     def updateThrottledList(prop: String, quotaManager: ReplicationQuotaManager) = {
       if (topicConfig.containsKey(prop) && topicConfig.getProperty(prop).length > 0) {
diff --git a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
index b517875e8e0b6..bf3d988244689 100755
--- a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
+++ b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
@@ -612,6 +612,18 @@ class DynamicLogConfig(logManager: LogManager, server: KafkaServer) extends Brok
     // validation, no additional validation is performed.
   }
 
+  private def updateLogsConfig(newBrokerDefaults: Map[String, Object]): Unit = {
+    logManager.brokerConfigUpdated()
+    logManager.allLogs.foreach { log =>
+      val props = mutable.Map.empty[Any, Any]
+      props ++= newBrokerDefaults
+      props ++= log.config.originals.asScala.filterKeys(log.config.overriddenConfigs.contains)
+
+      val logConfig = LogConfig(props.asJava)
+      log.updateConfig(logConfig)
+    }
+  }
+
   override def reconfigure(oldConfig: KafkaConfig, newConfig: KafkaConfig): Unit = {
     val currentLogConfig = logManager.currentDefaultConfig
     val origUncleanLeaderElectionEnable = logManager.currentDefaultConfig.uncleanLeaderElectionEnable
@@ -626,14 +638,8 @@ class DynamicLogConfig(logManager: LogManager, server: KafkaServer) extends Brok
 
     logManager.reconfigureDefaultLogConfig(LogConfig(newBrokerDefaults))
 
-    logManager.allLogs.foreach { log =>
-      val props = mutable.Map.empty[Any, Any]
-      props ++= newBrokerDefaults.asScala
-      props ++= log.config.originals.asScala.filterKeys(log.config.overriddenConfigs.contains)
+    updateLogsConfig(newBrokerDefaults.asScala)
 
-      val logConfig = LogConfig(props.asJava)
-      log.updateConfig(newBrokerDefaults.asScala.keySet, logConfig)
-    }
     if (logManager.currentDefaultConfig.uncleanLeaderElectionEnable && !origUncleanLeaderElectionEnable) {
       server.kafkaController.enableDefaultUncleanLeaderElection()
     }
diff --git a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
index 8e812ebd00318..6204174ede4f4 100644
--- a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
+++ b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
@@ -40,7 +40,7 @@ import org.apache.kafka.common.record._
 import org.apache.kafka.common.requests.{EpochEndOffset, IsolationLevel, ListOffsetRequest}
 import org.junit.{After, Before, Test}
 import org.junit.Assert._
-import org.mockito.Mockito.{doNothing, mock, when}
+import org.mockito.Mockito.{doAnswer, doNothing, mock, spy, times, verify, when}
 import org.scalatest.Assertions.assertThrows
 import org.mockito.ArgumentMatchers
 import org.mockito.invocation.InvocationOnMock
@@ -1545,6 +1545,108 @@ class PartitionTest {
     assertEquals(Set(), Metrics.defaultRegistry().allMetrics().asScala.keySet.filter(_.getType == "Partition"))
   }
 
+  /**
+   * Test when log is getting initialized, its config remains untouched after initialization is done.
+   */
+  @Test
+  def testLogConfigNotDirty(): Unit = {
+    val spyLogManager = spy(logManager)
+    val partition = new Partition(topicPartition,
+      replicaLagTimeMaxMs = Defaults.ReplicaLagTimeMaxMs,
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      localBrokerId = brokerId,
+      time,
+      stateStore,
+      delayedOperations,
+      metadataCache,
+      spyLogManager)
+
+    partition.createLog(brokerId, isNew = true, isFutureReplica = false, offsetCheckpoints)
+
+    // Validate that initializingLog and finishedInitializingLog was called
+    verify(spyLogManager).initializingLog(ArgumentMatchers.eq(topicPartition))
+    verify(spyLogManager).finishedInitializingLog(ArgumentMatchers.eq(topicPartition),
+      ArgumentMatchers.any(),
+      ArgumentMatchers.any()) // This doesn't get evaluated, but needed to satisfy compilation
+
+    // We should get config from ZK only once
+    verify(stateStore).fetchTopicConfig()
+  }
+
+  /**
+   * Test when log is getting initialized, its config remains gets reloaded if Topic config gets changed
+   * before initialization is done.
+   */
+  @Test
+  def testLogConfigDirtyAsTopicUpdated(): Unit = {
+    val spyLogManager = spy(logManager)
+    doAnswer(new Answer[Unit] {
+      def answer(invocation: InvocationOnMock): Unit = {
+        logManager.initializingLog(topicPartition)
+        logManager.topicConfigUpdated(topicPartition.topic())
+      }
+    }).when(spyLogManager).initializingLog(ArgumentMatchers.eq(topicPartition))
+
+    val partition = new Partition(topicPartition,
+      replicaLagTimeMaxMs = Defaults.ReplicaLagTimeMaxMs,
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      localBrokerId = brokerId,
+      time,
+      stateStore,
+      delayedOperations,
+      metadataCache,
+      spyLogManager)
+
+    partition.createLog(brokerId, isNew = true, isFutureReplica = false, offsetCheckpoints)
+
+    // Validate that initializingLog and finishedInitializingLog was called
+    verify(spyLogManager).initializingLog(ArgumentMatchers.eq(topicPartition))
+    verify(spyLogManager).finishedInitializingLog(ArgumentMatchers.eq(topicPartition),
+      ArgumentMatchers.any(),
+      ArgumentMatchers.any()) // This doesn't get evaluated, but needed to satisfy compilation
+
+    // We should get config from ZK twice, once before log is created, and second time once
+    // we find log config is dirty and refresh it.
+    verify(stateStore, times(2)).fetchTopicConfig()
+  }
+
+  /**
+   * Test when log is getting initialized, its config remains gets reloaded if Broker config gets changed
+   * before initialization is done.
+   */
+  @Test
+  def testLogConfigDirtyAsBrokerUpdated(): Unit = {
+    val spyLogManager = spy(logManager)
+    doAnswer(new Answer[Unit] {
+      def answer(invocation: InvocationOnMock): Unit = {
+        logManager.initializingLog(topicPartition)
+        logManager.brokerConfigUpdated()
+      }
+    }).when(spyLogManager).initializingLog(ArgumentMatchers.eq(topicPartition))
+
+    val partition = new Partition(topicPartition,
+      replicaLagTimeMaxMs = Defaults.ReplicaLagTimeMaxMs,
+      interBrokerProtocolVersion = ApiVersion.latestVersion,
+      localBrokerId = brokerId,
+      time,
+      stateStore,
+      delayedOperations,
+      metadataCache,
+      spyLogManager)
+
+    partition.createLog(brokerId, isNew = true, isFutureReplica = false, offsetCheckpoints)
+
+    // Validate that initializingLog and finishedInitializingLog was called
+    verify(spyLogManager).initializingLog(ArgumentMatchers.eq(topicPartition))
+    verify(spyLogManager).finishedInitializingLog(ArgumentMatchers.eq(topicPartition),
+      ArgumentMatchers.any(),
+      ArgumentMatchers.any()) // This doesn't get evaluated, but needed to satisfy compilation
+
+    // We should get config from ZK twice, once before log is created, and second time once
+    // we find log config is dirty and refresh it.
+    verify(stateStore, times(2)).fetchTopicConfig()
+  }
+
   private def seedLogData(log: Log, numRecords: Int, leaderEpoch: Int): Unit = {
     for (i <- 0 until numRecords) {
       val records = MemoryRecords.withRecords(0L, CompressionType.NONE, leaderEpoch,
@@ -1552,5 +1654,4 @@ class PartitionTest {
       log.appendAsLeader(records, leaderEpoch)
     }
   }
-
 }
diff --git a/core/src/test/scala/unit/kafka/log/LogManagerTest.scala b/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
index 91f2a405251a1..a7c17f440fc2e 100755
--- a/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
@@ -26,6 +26,7 @@ import kafka.utils._
 import org.apache.kafka.common.errors.OffsetOutOfRangeException
 import org.apache.kafka.common.utils.Utils
 import org.apache.kafka.common.{KafkaException, TopicPartition}
+import org.easymock.EasyMock
 import org.junit.Assert._
 import org.junit.{After, Before, Test}
 import org.mockito.ArgumentMatchers.any
@@ -468,4 +469,101 @@ class LogManagerTest {
     log.read(offset, maxLength, isolation = FetchLogEnd, minOneMessage = true)
   }
 
+  /**
+   * Test when a configuration of a topic is updated while its log is getting initialized,
+   * the config is refreshed when log initialization is finished.
+   */
+  @Test
+  def testTopicConfigChangeUpdatesLogConfig(): Unit = {
+    val testTopicOne = "test-topic-one"
+    val testTopicTwo = "test-topic-two"
+    val testTopicOnePartition: TopicPartition = new TopicPartition(testTopicOne, 1)
+    val testTopicTwoPartition: TopicPartition = new TopicPartition(testTopicTwo, 1)
+    val mockLog: Log = EasyMock.mock(classOf[Log])
+
+    logManager.initializingLog(testTopicOnePartition)
+    logManager.initializingLog(testTopicTwoPartition)
+
+    logManager.topicConfigUpdated(testTopicOne)
+
+    val logConfig: LogConfig = null
+    var configUpdated = false
+    logManager.finishedInitializingLog(testTopicOnePartition, Some(mockLog), () => {
+      configUpdated = true
+      logConfig
+    })
+    assertTrue(configUpdated)
+
+    var configNotUpdated = true
+    logManager.finishedInitializingLog(testTopicTwoPartition, Some(mockLog), () => {
+      configNotUpdated = false
+      logConfig
+    })
+    assertTrue(configNotUpdated)
+  }
+
+  /**
+   * Test if an error occurs when creating log, log manager removes corresponding
+   * topic partition from the list of initializing partitions.
+   */
+  @Test
+  def testConfigChangeGetsCleanedUp(): Unit = {
+    val testTopicPartition: TopicPartition = new TopicPartition("test-topic", 1)
+
+    logManager.initializingLog(testTopicPartition)
+
+    val logConfig: LogConfig = null
+    var configUpdateNotCalled = true
+    logManager.finishedInitializingLog(testTopicPartition, None, () => {
+      configUpdateNotCalled = false
+      logConfig
+    })
+
+    assertTrue(logManager.partitionsInitializing.isEmpty)
+    assertTrue(configUpdateNotCalled)
+  }
+
+  /**
+   * Test when a broker configuration change happens all logs in process of initialization
+   * pick up latest config when finished with initialization.
+   */
+  @Test
+  def testBrokerConfigChangeDeliveredToAllLogs(): Unit = {
+    val testTopicOne = "test-topic-one"
+    val testTopicTwo = "test-topic-two"
+    val testTopicOnePartition: TopicPartition = new TopicPartition(testTopicOne, 1)
+    val testTopicTwoPartition: TopicPartition = new TopicPartition(testTopicTwo, 1)
+    val mockLog: Log = EasyMock.mock(classOf[Log])
+
+    logManager.initializingLog(testTopicOnePartition)
+    logManager.initializingLog(testTopicTwoPartition)
+
+    logManager.brokerConfigUpdated()
+
+    val logConfig: LogConfig = null
+    var totalChanges = 0
+    logManager.finishedInitializingLog(testTopicOnePartition, Some(mockLog), () => {
+      totalChanges += 1
+      logConfig
+    })
+    logManager.finishedInitializingLog(testTopicTwoPartition, Some(mockLog), () => {
+      totalChanges += 1
+      logConfig
+    })
+
+    assertEquals(2, totalChanges)
+  }
+
+  /**
+   * Test even if no log is getting initialized, if config change events are delivered
+   * things continue to work correctly. This test should not throw.
+   *
+   * This makes sure that events can be delivered even when no log is getting initialized.
+   */
+  @Test
+  def testConfigChangesWithNoLogGettingInitialized(): Unit = {
+    logManager.brokerConfigUpdated()
+    logManager.topicConfigUpdated("test-topic")
+    assertTrue(logManager.partitionsInitializing.isEmpty)
+  }
 }
diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala
index c5de28e8e13c1..fdf40c64c0f78 100755
--- a/core/src/test/scala/unit/kafka/log/LogTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogTest.scala
@@ -2523,7 +2523,7 @@ class LogTest {
 
     val downgradedLogConfig = LogTest.createLogConfig(segmentBytes = 1000, indexIntervalBytes = 1,
       maxMessageBytes = 64 * 1024, messageFormatVersion = kafka.api.KAFKA_0_10_2_IV0.shortVersion)
-    log.updateConfig(Set(LogConfig.MessageFormatVersionProp), downgradedLogConfig)
+    log.updateConfig(downgradedLogConfig)
     assertLeaderEpochCacheEmpty(log)
 
     log.appendAsLeader(TestUtils.records(List(new SimpleRecord("bar".getBytes())),
@@ -2542,7 +2542,7 @@ class LogTest {
 
     val upgradedLogConfig = LogTest.createLogConfig(segmentBytes = 1000, indexIntervalBytes = 1,
       maxMessageBytes = 64 * 1024, messageFormatVersion = kafka.api.KAFKA_0_11_0_IV0.shortVersion)
-    log.updateConfig(Set(LogConfig.MessageFormatVersionProp), upgradedLogConfig)
+    log.updateConfig(upgradedLogConfig)
     log.appendAsLeader(TestUtils.records(List(new SimpleRecord("foo".getBytes()))), leaderEpoch = 5)
     assertEquals(Some(5), log.latestEpoch)
   }
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index 438dedf5b2836..484ee2d962a1e 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -53,6 +53,7 @@ import org.apache.kafka.common.{Node, TopicPartition}
 import org.easymock.EasyMock
 import org.junit.Assert._
 import org.junit.{After, Before, Test}
+import org.mockito.ArgumentMatchers
 
 import scala.collection.JavaConverters._
 import scala.collection.{Map, Seq}
@@ -972,15 +973,21 @@ class ReplicaManagerTest {
     }
 
     // Expect to call LogManager.truncateTo exactly once
+    val topicPartitionObj = new TopicPartition(topic, topicPartition)
     val mockLogMgr: LogManager = EasyMock.createMock(classOf[LogManager])
     EasyMock.expect(mockLogMgr.liveLogDirs).andReturn(config.logDirs.map(new File(_).getAbsoluteFile)).anyTimes
     EasyMock.expect(mockLogMgr.currentDefaultConfig).andReturn(LogConfig())
-    EasyMock.expect(mockLogMgr.getOrCreateLog(new TopicPartition(topic, topicPartition),
+    EasyMock.expect(mockLogMgr.getOrCreateLog(topicPartitionObj,
       LogConfig(), isNew = false, isFuture = false)).andReturn(mockLog).anyTimes
     if (expectTruncation) {
-      EasyMock.expect(mockLogMgr.truncateTo(Map(new TopicPartition(topic, topicPartition) -> offsetFromLeader),
+      EasyMock.expect(mockLogMgr.truncateTo(Map(topicPartitionObj -> offsetFromLeader),
         isFuture = false)).once
     }
+    EasyMock.expect(mockLogMgr.initializingLog(topicPartitionObj)).anyTimes
+
+    EasyMock.expect(mockLogMgr.finishedInitializingLog(
+      EasyMock.eq(topicPartitionObj), EasyMock.anyObject(), EasyMock.anyObject())).anyTimes
+
     EasyMock.replay(mockLogMgr)
 
     val aliveBrokerIds = Seq[Integer](followerBrokerId, leaderBrokerId)
@@ -1015,7 +1022,7 @@ class ReplicaManagerTest {
 
     // Mock network client to show leader offset of 5
     val quota = QuotaFactory.instantiate(config, metrics, time, "")
-    val blockingSend = new ReplicaFetcherMockBlockingSend(Map(new TopicPartition(topic, topicPartition) ->
+    val blockingSend = new ReplicaFetcherMockBlockingSend(Map(topicPartitionObj ->
       new EpochEndOffset(leaderEpochFromLeader, offsetFromLeader)).asJava, BrokerEndPoint(1, "host1" ,1), time)
     val replicaManager = new ReplicaManager(config, metrics, time, kafkaZkClient, mockScheduler, mockLogMgr,
       new AtomicBoolean(false), quota, mockBrokerTopicStats,

From bb758977e91c10b0e56da1c51dcaee4d6a094d73 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Tue, 15 Oct 2019 11:34:48 -0700
Subject: [PATCH 0720/1071] KAFKA-4422 / KAFKA-8700 / KAFKA-5566: Wait for
 state to transit to RUNNING upon start (#7519)

I looked into the logs of the above tickets, and I think for a couple fo them it is due to the fact that the threads takes time to restore, or just stabilize the rebalance since there are multi-threads. Adding the hook to wait for state to transit to RUNNING upon starting.

Reviewers: Chris Pettitt , Matthias J. Sax 
---
 .../QueryableStateIntegrationTest.java        | 33 ++++++++++++++-----
 1 file changed, 24 insertions(+), 9 deletions(-)

diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/QueryableStateIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/QueryableStateIntegrationTest.java
index c5dbabe4b050f..de2ae276d8917 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/QueryableStateIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/QueryableStateIntegrationTest.java
@@ -54,6 +54,7 @@
 import org.apache.kafka.streams.state.WindowStoreIterator;
 import org.apache.kafka.test.IntegrationTest;
 import org.apache.kafka.test.MockMapper;
+import org.apache.kafka.test.StreamsTestUtils;
 import org.apache.kafka.test.TestCondition;
 import org.apache.kafka.test.TestUtils;
 import org.junit.After;
@@ -92,6 +93,7 @@
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.apache.kafka.test.StreamsTestUtils.startKafkaStreamsAndWaitForRunningState;
 
 @Category({IntegrationTest.class})
 public class QueryableStateIntegrationTest {
@@ -261,6 +263,16 @@ private class StreamRunnable implements Runnable {
         @Override
         public void run() {
             myStream.start();
+
+            try {
+                TestUtils.waitForCondition(
+                    () -> stateListener.mapStates.containsKey(KafkaStreams.State.RUNNING),
+                    "Did not start successfully after " + TestUtils.DEFAULT_MAX_WAIT_MS + " ms"
+                );
+            } catch (final InterruptedException e) {
+                if (!stateListener.mapStates.containsKey(KafkaStreams.State.RUNNING))
+                    fail("Did not start successfully");
+            }
         }
 
         public void close() {
@@ -446,7 +458,8 @@ public void concurrentAccesses() throws Exception {
             windowStoreName,
             streamsConfiguration);
 
-        kafkaStreams.start();
+        startKafkaStreamsAndWaitForRunningState(kafkaStreams);
+
         producerThread.start();
 
         try {
@@ -515,7 +528,7 @@ public void shouldBeAbleToQueryFilterState() throws Exception {
         t2.toStream().to(outputTopic);
 
         kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration);
-        kafkaStreams.start();
+        startKafkaStreamsAndWaitForRunningState(kafkaStreams);
 
         waitUntilAtLeastNumRecordProcessed(outputTopic, 1);
 
@@ -581,7 +594,7 @@ public void shouldBeAbleToQueryMapValuesState() throws Exception {
             .to(outputTopic, Produced.with(Serdes.String(), Serdes.Long()));
 
         kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration);
-        kafkaStreams.start();
+        startKafkaStreamsAndWaitForRunningState(kafkaStreams);
 
         waitUntilAtLeastNumRecordProcessed(outputTopic, 5);
 
@@ -629,7 +642,7 @@ public void shouldBeAbleToQueryMapValuesAfterFilterState() throws Exception {
         t3.toStream().to(outputTopic, Produced.with(Serdes.String(), Serdes.Long()));
 
         kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration);
-        kafkaStreams.start();
+        startKafkaStreamsAndWaitForRunningState(kafkaStreams);
 
         waitUntilAtLeastNumRecordProcessed(outputTopic, 1);
 
@@ -690,7 +703,7 @@ private void verifyCanQueryState(final int cacheSizeBytes) throws Exception {
             .windowedBy(TimeWindows.of(ofMillis(WINDOW_SIZE)))
             .count(Materialized.as(windowStoreName));
         kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration);
-        kafkaStreams.start();
+        startKafkaStreamsAndWaitForRunningState(kafkaStreams);
 
         waitUntilAtLeastNumRecordProcessed(outputTopic, 1);
 
@@ -716,7 +729,7 @@ public void shouldNotMakeStoreAvailableUntilAllStoresAvailable() throws Exceptio
         final String storeName = "count-by-key";
         stream.groupByKey().count(Materialized.as(storeName));
         kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration);
-        kafkaStreams.start();
+        startKafkaStreamsAndWaitForRunningState(kafkaStreams);
 
         final KeyValue hello = KeyValue.pair("hello", "hello");
         IntegrationTestUtils.produceKeyValuesSynchronously(
@@ -747,9 +760,9 @@ public void shouldNotMakeStoreAvailableUntilAllStoresAvailable() throws Exceptio
         // close stream
         kafkaStreams.close();
 
-        // start again
+        // start again, and since it may take time to restore we wait for it to transit to RUNNING a bit longer
         kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration);
-        kafkaStreams.start();
+        startKafkaStreamsAndWaitForRunningState(kafkaStreams, maxWaitMs);
 
         // make sure we never get any value other than 8 for hello
         TestUtils.waitForCondition(
@@ -810,7 +823,9 @@ public void shouldAllowToQueryAfterThreadDied() throws Exception {
         streamsConfiguration.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, 2);
         kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration);
         kafkaStreams.setUncaughtExceptionHandler((t, e) -> failed.set(true));
-        kafkaStreams.start();
+
+        // since we start with two threads, wait for a bit longer for both of them to transit to running
+        startKafkaStreamsAndWaitForRunningState(kafkaStreams, 30000);
 
         IntegrationTestUtils.produceKeyValuesSynchronously(
             streamOne,

From 475508405eff45b149b5725e174c0f85690b9c08 Mon Sep 17 00:00:00 2001
From: Lucas Bradstreet 
Date: Tue, 15 Oct 2019 12:18:20 -0700
Subject: [PATCH 0721/1071] MINOR: remove unused import in
 QueryableStateIntegrationTest (#7521)

Reviewers: Guozhang Wang 
---
 .../kafka/streams/integration/QueryableStateIntegrationTest.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/QueryableStateIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/QueryableStateIntegrationTest.java
index de2ae276d8917..a0d38dc0732ae 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/QueryableStateIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/QueryableStateIntegrationTest.java
@@ -54,7 +54,6 @@
 import org.apache.kafka.streams.state.WindowStoreIterator;
 import org.apache.kafka.test.IntegrationTest;
 import org.apache.kafka.test.MockMapper;
-import org.apache.kafka.test.StreamsTestUtils;
 import org.apache.kafka.test.TestCondition;
 import org.apache.kafka.test.TestUtils;
 import org.junit.After;

From 641dfa47f3fc82917b23a676141eff2b87cc63c8 Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Fri, 11 Oct 2019 23:59:01 -0700
Subject: [PATCH 0722/1071] KAFKA-8122: Fix Kafka Streams EOS integration test
 (#7470)

Reviewers: Guozhang Wang , Chris Pettitt , Bill Bejeck 
---
 .../AbstractJoinIntegrationTest.java          | 44 ++++++++-----------
 .../integration/EosIntegrationTest.java       | 28 ++++++------
 .../apache/kafka/test/StreamsTestUtils.java   | 23 ++++++++++
 3 files changed, 57 insertions(+), 38 deletions(-)

diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/AbstractJoinIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/AbstractJoinIntegrationTest.java
index daea1d732ae0b..6660a205e3cad 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/AbstractJoinIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/AbstractJoinIntegrationTest.java
@@ -58,6 +58,7 @@
 import java.util.Properties;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.apache.kafka.test.StreamsTestUtils.startKafkaStreamsAndWaitForRunningState;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.core.Is.is;
 
@@ -102,21 +103,21 @@ public static Collection data() {
     AtomicBoolean finalResultReached = new AtomicBoolean(false);
 
     private final List> input = Arrays.asList(
-            new Input<>(INPUT_TOPIC_LEFT, null),
-            new Input<>(INPUT_TOPIC_RIGHT, null),
-            new Input<>(INPUT_TOPIC_LEFT, "A"),
-            new Input<>(INPUT_TOPIC_RIGHT, "a"),
-            new Input<>(INPUT_TOPIC_LEFT, "B"),
-            new Input<>(INPUT_TOPIC_RIGHT, "b"),
-            new Input<>(INPUT_TOPIC_LEFT, null),
-            new Input<>(INPUT_TOPIC_RIGHT, null),
-            new Input<>(INPUT_TOPIC_LEFT, "C"),
-            new Input<>(INPUT_TOPIC_RIGHT, "c"),
-            new Input<>(INPUT_TOPIC_RIGHT, null),
-            new Input<>(INPUT_TOPIC_LEFT, null),
-            new Input<>(INPUT_TOPIC_RIGHT, null),
-            new Input<>(INPUT_TOPIC_RIGHT, "d"),
-            new Input<>(INPUT_TOPIC_LEFT, "D")
+        new Input<>(INPUT_TOPIC_LEFT, null),
+        new Input<>(INPUT_TOPIC_RIGHT, null),
+        new Input<>(INPUT_TOPIC_LEFT, "A"),
+        new Input<>(INPUT_TOPIC_RIGHT, "a"),
+        new Input<>(INPUT_TOPIC_LEFT, "B"),
+        new Input<>(INPUT_TOPIC_RIGHT, "b"),
+        new Input<>(INPUT_TOPIC_LEFT, null),
+        new Input<>(INPUT_TOPIC_RIGHT, null),
+        new Input<>(INPUT_TOPIC_LEFT, "C"),
+        new Input<>(INPUT_TOPIC_RIGHT, "c"),
+        new Input<>(INPUT_TOPIC_RIGHT, null),
+        new Input<>(INPUT_TOPIC_LEFT, null),
+        new Input<>(INPUT_TOPIC_RIGHT, null),
+        new Input<>(INPUT_TOPIC_RIGHT, "d"),
+        new Input<>(INPUT_TOPIC_LEFT, "D")
     );
 
     final ValueJoiner valueJoiner = (value1, value2) -> value1 + "-" + value2;
@@ -198,7 +199,7 @@ void runTest(final List>> expectedResult, f
         KeyValueTimestamp expectedFinalResult = null;
 
         try {
-            streams.start();
+            startKafkaStreamsAndWaitForRunningState(streams);
 
             final long firstTimestamp = System.currentTimeMillis();
             long ts = firstTimestamp;
@@ -228,13 +229,6 @@ void runTest(final List>> expectedResult, f
         }
     }
 
-    /*
-     * Runs the actual test. Checks the final result only after expected number of records have been consumed.
-     */
-    void runTest(final KeyValueTimestamp expectedFinalResult) throws Exception {
-        runTest(expectedFinalResult, null);
-    }
-
     /*
      * Runs the actual test. Checks the final result only after expected number of records have been consumed.
      */
@@ -243,7 +237,7 @@ void runTest(final KeyValueTimestamp expectedFinalResult, final St
         streams = new KafkaStreams(builder.build(), STREAMS_CONFIG);
 
         try {
-            streams.start();
+            startKafkaStreamsAndWaitForRunningState(streams);
 
             final long firstTimestamp = System.currentTimeMillis();
             long ts = firstTimestamp;
@@ -288,7 +282,7 @@ private void checkQueryableStore(final String queryableName, final KeyValueTimes
         }
     }
 
-    private final class Input {
+    private static final class Input {
         String topic;
         KeyValue record;
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/EosIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/EosIntegrationTest.java
index 7523a62837fda..15d6789798dee 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/EosIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/EosIntegrationTest.java
@@ -60,6 +60,8 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import static org.apache.kafka.test.StreamsTestUtils.startKafkaStreamsAndWaitForRunningState;
+import static org.apache.kafka.test.TestUtils.waitForCondition;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertNotNull;
@@ -170,7 +172,7 @@ private void runSimpleCopyTest(final int numberOfRestarts,
                 properties);
 
             try (final KafkaStreams streams = new KafkaStreams(builder.build(), config)) {
-                streams.start();
+                startKafkaStreamsAndWaitForRunningState(streams, MAX_WAIT_TIME_MS);
 
                 final List> inputData = prepareData(i * 100, i * 100 + 10L, 0L, 1L);
 
@@ -253,7 +255,7 @@ public void shouldBeAbleToPerformMultipleTransactions() throws Exception {
             properties);
 
         try (final KafkaStreams streams = new KafkaStreams(builder.build(), config)) {
-            streams.start();
+            startKafkaStreamsAndWaitForRunningState(streams, MAX_WAIT_TIME_MS);
 
             final List> firstBurstOfData = prepareData(0L, 5L, 0L);
             final List> secondBurstOfData = prepareData(5L, 8L, 0L);
@@ -319,7 +321,7 @@ public void shouldNotViolateEosIfOneTaskFails() throws Exception {
         // after fail over, we should read 40 committed records (even if 50 record got written)
 
         try (final KafkaStreams streams = getKafkaStreams(false, "appDir", 2)) {
-            streams.start();
+            startKafkaStreamsAndWaitForRunningState(streams, MAX_WAIT_TIME_MS);
 
             final List> committedDataBeforeFailure = prepareData(0L, 10L, 0L, 1L);
             final List> uncommittedDataBeforeFailure = prepareData(10L, 15L, 0L, 1L);
@@ -332,7 +334,7 @@ public void shouldNotViolateEosIfOneTaskFails() throws Exception {
 
             writeInputData(committedDataBeforeFailure);
 
-            TestUtils.waitForCondition(
+            waitForCondition(
                 () -> commitRequested.get() == 2, MAX_WAIT_TIME_MS,
                 "StreamsTasks did not request commit.");
 
@@ -347,7 +349,7 @@ public void shouldNotViolateEosIfOneTaskFails() throws Exception {
             errorInjected.set(true);
             writeInputData(dataAfterFailure);
 
-            TestUtils.waitForCondition(
+            waitForCondition(
                 () -> uncaughtException != null, MAX_WAIT_TIME_MS,
                 "Should receive uncaught exception from one StreamThread.");
 
@@ -387,7 +389,7 @@ public void shouldNotViolateEosIfOneTaskFailsWithState() throws Exception {
         // per key (even if some records got processed twice)
 
         try (final KafkaStreams streams = getKafkaStreams(true, "appDir", 2)) {
-            streams.start();
+            startKafkaStreamsAndWaitForRunningState(streams, MAX_WAIT_TIME_MS);
 
             final List> committedDataBeforeFailure = prepareData(0L, 10L, 0L, 1L);
             final List> uncommittedDataBeforeFailure = prepareData(10L, 15L, 0L, 1L, 2L, 3L);
@@ -400,7 +402,7 @@ public void shouldNotViolateEosIfOneTaskFailsWithState() throws Exception {
 
             writeInputData(committedDataBeforeFailure);
 
-            TestUtils.waitForCondition(
+            waitForCondition(
                 () -> commitRequested.get() == 2, MAX_WAIT_TIME_MS,
                 "SteamsTasks did not request commit.");
 
@@ -417,7 +419,7 @@ public void shouldNotViolateEosIfOneTaskFailsWithState() throws Exception {
             errorInjected.set(true);
             writeInputData(dataAfterFailure);
 
-            TestUtils.waitForCondition(
+            waitForCondition(
                 () -> uncaughtException != null, MAX_WAIT_TIME_MS,
                 "Should receive uncaught exception from one StreamThread.");
 
@@ -462,8 +464,8 @@ public void shouldNotViolateEosIfOneTaskGetsFencedUsingIsolatedAppInstances() th
             final KafkaStreams streams1 = getKafkaStreams(false, "appDir1", 1);
             final KafkaStreams streams2 = getKafkaStreams(false, "appDir2", 1)
         ) {
-            streams1.start();
-            streams2.start();
+            startKafkaStreamsAndWaitForRunningState(streams1, MAX_WAIT_TIME_MS);
+            startKafkaStreamsAndWaitForRunningState(streams2, MAX_WAIT_TIME_MS);
 
             final List> committedDataBeforeGC = prepareData(0L, 10L, 0L, 1L);
             final List> uncommittedDataBeforeGC = prepareData(10L, 15L, 0L, 1L);
@@ -478,7 +480,7 @@ public void shouldNotViolateEosIfOneTaskGetsFencedUsingIsolatedAppInstances() th
 
             writeInputData(committedDataBeforeGC);
 
-            TestUtils.waitForCondition(
+            waitForCondition(
                 () -> commitRequested.get() == 2, MAX_WAIT_TIME_MS,
                 "SteamsTasks did not request commit.");
 
@@ -493,7 +495,7 @@ public void shouldNotViolateEosIfOneTaskGetsFencedUsingIsolatedAppInstances() th
             gcInjected.set(true);
             writeInputData(dataToTriggerFirstRebalance);
 
-            TestUtils.waitForCondition(
+            waitForCondition(
                 () -> streams1.allMetadata().size() == 1
                     && streams2.allMetadata().size() == 1
                     && (streams1.allMetadata().iterator().next().topicPartitions().size() == 2
@@ -511,7 +513,7 @@ public void shouldNotViolateEosIfOneTaskGetsFencedUsingIsolatedAppInstances() th
             checkResultPerKey(committedRecordsAfterRebalance, expectedCommittedRecordsAfterRebalance);
 
             doGC = false;
-            TestUtils.waitForCondition(
+            waitForCondition(
                 () -> streams1.allMetadata().size() == 1
                     && streams2.allMetadata().size() == 1
                     && streams1.allMetadata().iterator().next().topicPartitions().size() == 1
diff --git a/streams/src/test/java/org/apache/kafka/test/StreamsTestUtils.java b/streams/src/test/java/org/apache/kafka/test/StreamsTestUtils.java
index 87a693de177cc..f25326ee36436 100644
--- a/streams/src/test/java/org/apache/kafka/test/StreamsTestUtils.java
+++ b/streams/src/test/java/org/apache/kafka/test/StreamsTestUtils.java
@@ -21,6 +21,7 @@
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
 import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.KafkaStreams;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.kstream.Windowed;
@@ -31,8 +32,11 @@
 import java.util.Map;
 import java.util.Properties;
 import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 
 import static org.apache.kafka.common.metrics.Sensor.RecordingLevel.DEBUG;
+import static org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.MatcherAssert.assertThat;
 
@@ -79,6 +83,25 @@ public static Properties getStreamsConfig() {
         return getStreamsConfig(UUID.randomUUID().toString());
     }
 
+    public static void startKafkaStreamsAndWaitForRunningState(final KafkaStreams kafkaStreams) throws InterruptedException {
+        startKafkaStreamsAndWaitForRunningState(kafkaStreams, DEFAULT_MAX_WAIT_MS);
+    }
+
+    public static void startKafkaStreamsAndWaitForRunningState(final KafkaStreams kafkaStreams,
+                                                               final long timeoutMs) throws InterruptedException {
+        final CountDownLatch countDownLatch = new CountDownLatch(1);
+        kafkaStreams.setStateListener((newState, oldState) -> {
+            if (newState == KafkaStreams.State.RUNNING) {
+                countDownLatch.countDown();
+            }
+        });
+
+        kafkaStreams.start();
+        assertThat(
+            "KafkaStreams did not transit to RUNNING state within " + timeoutMs + " milli seconds.",
+            countDownLatch.await(timeoutMs, TimeUnit.MILLISECONDS), equalTo(true));
+    }
+
     public static  List> toList(final Iterator> iterator) {
         final List> results = new ArrayList<>();
 

From baff35d456d0f9f59c55cffc966c2592ff748d41 Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Tue, 15 Oct 2019 11:49:45 -0700
Subject: [PATCH 0723/1071] MINOR: code and JavaDoc cleanup (#7462)

Reviewers: Jukka Karvanen , Bill Bejeck 
---
 .../apache/kafka/streams/TestInputTopic.java  |  37 ++--
 .../apache/kafka/streams/TestOutputTopic.java |  35 ++--
 .../kafka/streams/TopologyTestDriver.java     |  81 ++++-----
 .../apache/kafka/streams/test/TestRecord.java |  45 ++---
 .../apache/kafka/streams/TestTopicsTest.java  | 172 +++++++++++-------
 .../kafka/streams/TopologyTestDriverTest.java | 117 +++++++-----
 .../kafka/streams/test/TestRecordTest.java    |  25 +--
 7 files changed, 282 insertions(+), 230 deletions(-)

diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TestInputTopic.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TestInputTopic.java
index 162ce4cc9f28e..50310aa87d0fc 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TestInputTopic.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TestInputTopic.java
@@ -27,11 +27,11 @@
 
 /**
  * {@code TestInputTopic} is used to pipe records to topic in {@link TopologyTestDriver}.
- * To use {@code TestInputTopic} create a new instance via {@link TopologyTestDriver#createInputTopic(String, Serializer, Serializer)}
- * In actual test code, you can pipe new record values, keys and values or list of {@link KeyValue} pairs
+ * To use {@code TestInputTopic} create a new instance via
+ * {@link TopologyTestDriver#createInputTopic(String, Serializer, Serializer)}.
+ * In actual test code, you can pipe new record values, keys and values or list of {@link KeyValue} pairs.
  * If you have multiple source topics, you need to create a {@code TestInputTopic} for each.
  *
- *
  * 
                Processing messages
                
  * 
                {@code
  *     private TestInputTopic inputTopic;
@@ -77,13 +77,16 @@ public class TestInputTopic {
             throw new IllegalArgumentException("autoAdvance must be positive");
         }
         this.advanceDuration = autoAdvance;
-
     }
 
     /**
-     * Advances the internally tracked event time of this input topic. Each time a record without explicitly defined timestamp is piped, the current topic event time is used as record timestamp.
+     * Advances the internally tracked event time of this input topic.
+     * Each time a record without explicitly defined timestamp is piped,
+     * the current topic event time is used as record timestamp.
      * 
                
-     * Note: advancing the event time on the input topic, does not advance the tracked stream time in {@link TopologyTestDriver} as long as no new input records are piped. Furthermore, it does not advance the wall-clock time of {@link TopologyTestDriver}.
+     * Note: advancing the event time on the input topic, does not advance the tracked stream time in
+     * {@link TopologyTestDriver} as long as no new input records are piped.
+     * Furthermore, it does not advance the wall-clock time of {@link TopologyTestDriver}.
      *
      * @param advance the duration of time to advance
      */
@@ -102,7 +105,7 @@ private Instant getTimestampAndAdvanced() {
 
     /**
      * Send an input record with the given record on the topic and then commit the records.
-     * May auto advance topic time
+     * May auto advance topic time.
      *
      * @param record the record to sent
      */
@@ -114,7 +117,7 @@ public void pipeInput(final TestRecord record) {
 
     /**
      * Send an input record with the given value on the topic and then commit the records.
-     * May auto advance topic time
+     * May auto advance topic time.
      *
      * @param value the record value
      */
@@ -129,7 +132,8 @@ public void pipeInput(final V value) {
      * @param key   the record key
      * @param value the record value
      */
-    public void pipeInput(final K key, final V value) {
+    public void pipeInput(final K key,
+                          final V value) {
         pipeInput(new TestRecord<>(key, value));
     }
 
@@ -142,7 +146,7 @@ public void pipeInput(final K key, final V value) {
      */
     public void pipeInput(final V value,
                           final Instant timestamp) {
-        pipeInput(new TestRecord(null, value, timestamp));
+        pipeInput(new TestRecord<>(null, value, timestamp));
     }
 
     /**
@@ -156,7 +160,7 @@ public void pipeInput(final V value,
     public void pipeInput(final K key,
                           final V value,
                           final long timestampMs) {
-        pipeInput(new TestRecord(key, value, null, timestampMs));
+        pipeInput(new TestRecord<>(key, value, null, timestampMs));
     }
 
     /**
@@ -170,7 +174,7 @@ public void pipeInput(final K key,
     public void pipeInput(final K key,
                           final V value,
                           final Instant timestamp) {
-        pipeInput(new TestRecord(key, value, timestamp));
+        pipeInput(new TestRecord<>(key, value, timestamp));
     }
 
     /**
@@ -187,7 +191,8 @@ public void pipeRecordList(final List> records) {
 
     /**
      * Send input records with the given KeyValue list on the topic then commit each record individually.
-     * The timestamp will be generated based on the constructor provided start time and time will auto advance based on autoAdvance setting.
+     * The timestamp will be generated based on the constructor provided start time and time will auto advance based on
+     * {@link #TestInputTopic(TopologyTestDriver, String, Serializer, Serializer, Instant, Duration) autoAdvance} setting.
      *
      * @param keyValues the {@link List} of {@link KeyValue} records
      */
@@ -199,7 +204,8 @@ public void pipeKeyValueList(final List> keyValues) {
 
     /**
      * Send input records with the given value list on the topic then commit each record individually.
-     * The timestamp will be generated based on the constructor provided start time and time will auto advance based on autoAdvance setting.
+     * The timestamp will be generated based on the constructor provided start time and time will auto advance based on
+     * {@link #TestInputTopic(TopologyTestDriver, String, Serializer, Serializer, Instant, Duration) autoAdvance} setting.
      *
      * @param values the {@link List} of {@link KeyValue} records
      */
@@ -229,7 +235,8 @@ public void pipeKeyValueList(final List> keyValues,
 
     /**
      * Send input records with the given value list on the topic then commit each record individually.
-     * The timestamp will be generated based on the constructor provided start time and time will auto advance based on autoAdvance setting.
+     * The timestamp will be generated based on the constructor provided start time and time will auto advance based on
+     * {@link #TestInputTopic(TopologyTestDriver, String, Serializer, Serializer, Instant, Duration) autoAdvance} setting.
      *
      * @param values         the {@link List} of values
      * @param startTimestamp the timestamp for the first generated record
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TestOutputTopic.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TestOutputTopic.java
index da3629eaea740..4296ce88dae6d 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TestOutputTopic.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TestOutputTopic.java
@@ -28,12 +28,14 @@
 
 /**
  * {@code TestOutputTopic} is used to read records from a topic in {@link TopologyTestDriver}.
- * To use {@code TestOutputTopic} create a new instance via {@link TopologyTestDriver#createOutputTopic(String, Deserializer, Deserializer)}
+ * To use {@code TestOutputTopic} create a new instance via
+ * {@link TopologyTestDriver#createOutputTopic(String, Deserializer, Deserializer)}.
  * In actual test code, you can read record values, keys, {@link KeyValue} or {@link TestRecord}
  * If you have multiple source topics, you need to create a {@code TestOutputTopic} for each.
  * 
                
  * If you need to test key, value and headers, use {@link #readRecord()} methods.
- * Using {@link #readKeyValue()} you get a {@link KeyValue} pair, and thus, don't get access to the record's timestamp or headers.
+ * Using {@link #readKeyValue()} you get a {@link KeyValue} pair, and thus, don't get access to the record's
+ * timestamp or headers.
  * Similarly using {@link #readValue()} you only get the value of a record.
  *
  * 
                Processing records
                
@@ -69,11 +71,10 @@ public class TestOutputTopic {
         this.valueDeserializer = valueDeserializer;
     }
 
-
     /**
      * Read one record from the output topic and return record's value.
      *
-     * @return Next value for output topic
+     * @return Next value for output topic.
      */
     public V readValue() {
         final TestRecord record = readRecord();
@@ -83,7 +84,7 @@ public V readValue() {
     /**
      * Read one record from the output topic and return its key and value as pair.
      *
-     * @return Next output as {@link KeyValue}
+     * @return Next output as {@link KeyValue}.
      */
     public KeyValue readKeyValue() {
         final TestRecord record = readRecord();
@@ -93,7 +94,7 @@ public KeyValue readKeyValue() {
     /**
      * Read one Record from output topic.
      *
-     * @return Next output as {@link TestRecord}
+     * @return Next output as {@link TestRecord}.
      */
     public TestRecord readRecord() {
         return driver.readRecord(topic, keyDeserializer, valueDeserializer);
@@ -101,10 +102,12 @@ public TestRecord readRecord() {
 
     /**
      * Read output to List.
-     * This method can be used if the result is considered a stream. If the result is considered a table, the list will
-     * contain all updated, ie, a key might be contained multiple times. If you are only interested in the last table update (ie, the final table state), you can use {@link #readKeyValuesToMap()} instead.
+     * This method can be used if the result is considered a stream.
+     * If the result is considered a table, the list will contain all updated, ie, a key might be contained multiple times.
+     * If you are only interested in the last table update (ie, the final table state),
+     * you can use {@link #readKeyValuesToMap()} instead.
      *
-     * @return List of output
+     * @return List of output.
      */
     public List> readRecordsToList() {
         final List> output = new LinkedList<>();
@@ -117,13 +120,13 @@ public List> readRecordsToList() {
 
     /**
      * Read output to map.
-     * This method can be used if the result is considered a table, when
-     * you are only interested in the last table update (ie, the final table state).
+     * This method can be used if the result is considered a table,
+     * when you are only interested in the last table update (ie, the final table state).
      * If the result is considered a stream, you can use {@link #readRecordsToList()} instead.
      * The list will contain all updated, ie, a key might be contained multiple times.
      * If the last update to a key is a delete/tombstone, the key will still be in the map (with null-value).
      *
-     * @return Map of output by key
+     * @return Map of output by key.
      */
     public Map readKeyValuesToMap() {
         final Map output = new HashMap<>();
@@ -141,7 +144,7 @@ public Map readKeyValuesToMap() {
     /**
      * Read all KeyValues from topic to List.
      *
-     * @return List of output KeyValues
+     * @return List of output KeyValues.
      */
     public List> readKeyValuesToList() {
         final List> output = new LinkedList<>();
@@ -156,7 +159,7 @@ public List> readKeyValuesToList() {
     /**
      * Read all values from topic to List.
      *
-     * @return List of output values
+     * @return List of output values.
      */
     public List readValuesToList() {
         final List output = new LinkedList<>();
@@ -171,7 +174,7 @@ public List readValuesToList() {
     /**
      * Get size of unread record in the topic queue.
      *
-     * @return size of topic queue
+     * @return size of topic queue.
      */
     public final long getQueueSize() {
         return driver.getQueueSize(topic);
@@ -180,7 +183,7 @@ public final long getQueueSize() {
     /**
      * Verify if the topic queue is empty.
      *
-     * @return true if no more record in the topic queue
+     * @return {@code true} if no more record in the topic queue.
      */
     public final boolean isEmpty() {
         return driver.isEmpty(topic);
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
index 65b3f08b74862..39d40a8abc875 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
@@ -225,10 +225,9 @@ public class TopologyTestDriver implements Closeable {
      * @param topology the topology to be tested
      * @param config   the configuration for the topology
      */
-    @SuppressWarnings("WeakerAccess")
     public TopologyTestDriver(final Topology topology,
                               final Properties config) {
-        this(topology, config, (Instant) null);
+        this(topology, config, null);
     }
 
     /**
@@ -240,7 +239,6 @@ public TopologyTestDriver(final Topology topology,
      * @param config                 the configuration for the topology
      * @param initialWallClockTimeMs the initial value of internally mocked wall-clock time
      */
-    @SuppressWarnings("WeakerAccess")
     @Deprecated
     public TopologyTestDriver(final Topology topology,
                               final Properties config,
@@ -255,11 +253,13 @@ public TopologyTestDriver(final Topology topology,
      * @param config                 the configuration for the topology
      * @param initialWallClockTime   the initial value of internally mocked wall-clock time
      */
-    @SuppressWarnings("WeakerAccess")
     public TopologyTestDriver(final Topology topology,
                               final Properties config,
                               final Instant initialWallClockTime) {
-        this(topology.internalTopologyBuilder, config, initialWallClockTime == null ? System.currentTimeMillis() : initialWallClockTime.toEpochMilli());
+        this(
+            topology.internalTopologyBuilder,
+            config,
+            initialWallClockTime == null ? System.currentTimeMillis() : initialWallClockTime.toEpochMilli());
     }
 
 
@@ -379,7 +379,12 @@ public void onRestoreEnd(final TopicPartition topicPartition, final String store
                 new LogContext()
             );
             globalStateTask.initialize();
-            globalProcessorContext.setRecordContext(new ProcessorRecordContext(0L, -1L, -1, ProcessorContextImpl.NONEXIST_TOPIC, new RecordHeaders()));
+            globalProcessorContext.setRecordContext(new ProcessorRecordContext(
+                0L,
+                -1L,
+                -1,
+                ProcessorContextImpl.NONEXIST_TOPIC,
+                new RecordHeaders()));
         } else {
             globalStateManager = null;
             globalStateTask = null;
@@ -421,7 +426,6 @@ public void onRestoreEnd(final TopicPartition topicPartition, final String store
      *
      * @return Map of all metrics.
      */
-    @SuppressWarnings("WeakerAccess")
     public Map metrics() {
         return Collections.unmodifiableMap(metrics.metrics());
     }
@@ -434,21 +438,28 @@ public void onRestoreEnd(final TopicPartition topicPartition, final String store
      *
      * @param consumerRecord the record to be processed
      */
-    @SuppressWarnings("WeakerAccess")
     @Deprecated
     public void pipeInput(final ConsumerRecord consumerRecord) {
-        pipeRecord(consumerRecord.topic(), consumerRecord.timestamp(), consumerRecord.key(), consumerRecord.value(), consumerRecord.headers());
+        pipeRecord(
+            consumerRecord.topic(),
+            consumerRecord.timestamp(),
+            consumerRecord.key(),
+            consumerRecord.value(),
+            consumerRecord.headers());
     }
 
     private void pipeRecord(final ProducerRecord record) {
         pipeRecord(record.topic(), record.timestamp(), record.key(), record.value(), record.headers());
     }
 
-    private void pipeRecord(final String topic, final Long timestamp, final byte[] key, final byte[] value, final Headers headers) {
-        final String topicName = topic;
+    private void pipeRecord(final String topicName,
+                            final Long timestamp,
+                            final byte[] key,
+                            final byte[] value,
+                            final Headers headers) {
 
         if (!internalTopologyBuilder.sourceTopicNames().isEmpty()) {
-            validateSourceTopicNameRegexPattern(topic);
+            validateSourceTopicNameRegexPattern(topicName);
         }
         final TopicPartition topicPartition = getTopicPartition(topicName);
         if (topicPartition != null) {
@@ -543,7 +554,6 @@ private void captureOutputRecords() {
      *
      * @param records a list of records to be processed
      */
-    @SuppressWarnings("WeakerAccess")
     @Deprecated
     public void pipeInput(final List> records) {
         for (final ConsumerRecord record : records) {
@@ -560,7 +570,6 @@ public void pipeInput(final List> records) {
      *
      * @param advanceMs the amount of time to advance wall-clock time in milliseconds
      */
-    @SuppressWarnings("WeakerAccess")
     @Deprecated
     public void advanceWallClockTime(final long advanceMs) {
         advanceWallClockTime(Duration.ofMillis(advanceMs));
@@ -573,7 +582,6 @@ public void advanceWallClockTime(final long advanceMs) {
      *
      * @param advance the amount of time to advance wall-clock time
      */
-    @SuppressWarnings("WeakerAccess")
     public void advanceWallClockTime(final Duration advance) {
         Objects.requireNonNull(advance, "advance cannot be null");
         mockWallClockTime.sleep(advance.toMillis());
@@ -593,7 +601,6 @@ public void advanceWallClockTime(final Duration advance) {
      * @param topic the name of the topic
      * @return the next record on that topic, or {@code null} if there is no record available
      */
-    @SuppressWarnings("WeakerAccess")
     @Deprecated
     public ProducerRecord readOutput(final String topic) {
         final Queue> outputRecords = outputRecordsByTopic.get(topic);
@@ -614,7 +621,6 @@ public ProducerRecord readOutput(final String topic) {
      * @param valueDeserializer the deserializer for the value type
      * @return the next record on that topic, or {@code null} if there is no record available
      */
-    @SuppressWarnings("WeakerAccess")
     @Deprecated
     public  ProducerRecord readOutput(final String topic,
                                                   final Deserializer keyDeserializer,
@@ -628,8 +634,7 @@ public  ProducerRecord readOutput(final String topic,
         return new ProducerRecord<>(record.topic(), record.partition(), record.timestamp(), key, value, record.headers());
     }
 
-
-    private final Queue> getRecordsQueue(final String topicName) {
+    private Queue> getRecordsQueue(final String topicName) {
         final Queue> outputRecords = outputRecordsByTopic.get(topicName);
         if (outputRecords == null) {
             if (!processorTopology.sinkTopics().contains(topicName)) {
@@ -654,7 +659,7 @@ private final Queue> getRecordsQueue(final String
     public final  TestInputTopic createInputTopic(final String topicName,
                                                               final Serializer keySerializer,
                                                               final Serializer valueSerializer) {
-        return new TestInputTopic(this, topicName, keySerializer, valueSerializer, Instant.now(), Duration.ZERO);
+        return new TestInputTopic<>(this, topicName, keySerializer, valueSerializer, Instant.now(), Duration.ZERO);
     }
 
     /**
@@ -675,7 +680,7 @@ public final  TestInputTopic createInputTopic(final String topicName
                                                               final Serializer valueSerializer,
                                                               final Instant startTimestamp,
                                                               final Duration autoAdvance) {
-        return new TestInputTopic(this, topicName, keySerializer, valueSerializer, startTimestamp, autoAdvance);
+        return new TestInputTopic<>(this, topicName, keySerializer, valueSerializer, startTimestamp, autoAdvance);
     }
 
     /**
@@ -691,10 +696,9 @@ public final  TestInputTopic createInputTopic(final String topicName
     public final  TestOutputTopic createOutputTopic(final String topicName,
                                                                 final Deserializer keyDeserializer,
                                                                 final Deserializer valueDeserializer) {
-        return new TestOutputTopic(this, topicName, keyDeserializer, valueDeserializer);
+        return new TestOutputTopic<>(this, topicName, keyDeserializer, valueDeserializer);
     }
 
-    @SuppressWarnings("WeakerAccess")
     ProducerRecord readRecord(final String topic) {
         final Queue> outputRecords = getRecordsQueue(topic);
         if (outputRecords == null) {
@@ -703,10 +707,9 @@ ProducerRecord readRecord(final String topic) {
         return outputRecords.poll();
     }
 
-    @SuppressWarnings("WeakerAccess")
      TestRecord readRecord(final String topic,
-                                         final Deserializer keyDeserializer,
-                                         final Deserializer valueDeserializer) {
+                                       final Deserializer keyDeserializer,
+                                       final Deserializer valueDeserializer) {
         final Queue> outputRecords = getRecordsQueue(topic);
         if (outputRecords == null) {
             throw new NoSuchElementException("Uninitialized topic: " + topic);
@@ -720,7 +723,6 @@  TestRecord readRecord(final String topic,
         return new TestRecord<>(key, value, record.headers(), record.timestamp());
     }
 
-    @SuppressWarnings("WeakerAccess")
      void pipeRecord(final String topic,
                            final TestRecord record,
                            final Serializer keySerializer,
@@ -740,12 +742,6 @@  void pipeRecord(final String topic,
         pipeRecord(topic, timestamp, serializedKey, serializedValue, record.headers());
     }
 
-    /**
-     * Get size of unread record in the topic queue.
-     *
-     * @param topic
-     * @return size of topic queue
-     */
     final long getQueueSize(final String topic) {
         final Queue> queue = getRecordsQueue(topic);
         if (queue == null) {
@@ -755,17 +751,10 @@ final long getQueueSize(final String topic) {
         return queue.size();
     }
 
-    /**
-     * Verify if the topic queue is empty.
-     *
-     * @param topic
-     * @return true if no more record in the topic queue
-     */
     final boolean isEmpty(final String topic) {
         return getQueueSize(topic) == 0;
     }
 
-
     /**
      * Get all {@link StateStore StateStores} from the topology.
      * The stores can be a "regular" or global stores.
@@ -817,7 +806,6 @@ public Map getAllStateStores() {
      * @see #getTimestampedWindowStore(String)
      * @see #getSessionStore(String)
      */
-    @SuppressWarnings("WeakerAccess")
     public StateStore getStateStore(final String name) throws IllegalArgumentException {
         return getStateStore(name, true);
     }
@@ -892,7 +880,7 @@ private void throwIfBuiltInStore(final StateStore stateStore) {
      * @see #getTimestampedWindowStore(String)
      * @see #getSessionStore(String)
      */
-    @SuppressWarnings({"unchecked", "WeakerAccess"})
+    @SuppressWarnings("unchecked")
     public  KeyValueStore getKeyValueStore(final String name) {
         final StateStore store = getStateStore(name, false);
         if (store instanceof TimestampedKeyValueStore) {
@@ -918,7 +906,7 @@ public  KeyValueStore getKeyValueStore(final String name) {
      * @see #getTimestampedWindowStore(String)
      * @see #getSessionStore(String)
      */
-    @SuppressWarnings({"unchecked", "WeakerAccess"})
+    @SuppressWarnings("unchecked")
     public  KeyValueStore> getTimestampedKeyValueStore(final String name) {
         final StateStore store = getStateStore(name, false);
         return store instanceof TimestampedKeyValueStore ? (TimestampedKeyValueStore) store : null;
@@ -945,7 +933,7 @@ public  KeyValueStore> getTimestampedKeyValueStore
      * @see #getTimestampedWindowStore(String)
      * @see #getSessionStore(String)
      */
-    @SuppressWarnings({"unchecked", "WeakerAccess"})
+    @SuppressWarnings("unchecked")
     public  WindowStore getWindowStore(final String name) {
         final StateStore store = getStateStore(name, false);
         if (store instanceof TimestampedWindowStore) {
@@ -971,7 +959,7 @@ public  WindowStore getWindowStore(final String name) {
      * @see #getWindowStore(String)
      * @see #getSessionStore(String)
      */
-    @SuppressWarnings({"unchecked", "WeakerAccess"})
+    @SuppressWarnings("unchecked")
     public  WindowStore> getTimestampedWindowStore(final String name) {
         final StateStore store = getStateStore(name, false);
         return store instanceof TimestampedWindowStore ? (TimestampedWindowStore) store : null;
@@ -993,7 +981,7 @@ public  WindowStore> getTimestampedWindowStore(fin
      * @see #getWindowStore(String)
      * @see #getTimestampedWindowStore(String)
      */
-    @SuppressWarnings({"unchecked", "WeakerAccess"})
+    @SuppressWarnings("unchecked")
     public  SessionStore getSessionStore(final String name) {
         final StateStore store = getStateStore(name, false);
         return store instanceof SessionStore ? (SessionStore) store : null;
@@ -1002,7 +990,6 @@ public  SessionStore getSessionStore(final String name) {
     /**
      * Close the driver, its topology, and all processors.
      */
-    @SuppressWarnings("WeakerAccess")
     public void close() {
         if (task != null) {
             task.close(true, false);
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/test/TestRecord.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/test/TestRecord.java
index a1992ae9d2ace..63f692106ee01 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/test/TestRecord.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/test/TestRecord.java
@@ -38,7 +38,6 @@ public class TestRecord {
     private final V value;
     private final Instant recordTime;
 
-
     /**
      * Creates a record.
      *
@@ -54,7 +53,6 @@ public TestRecord(final K key, final V value, final Headers headers, final Insta
         this.headers = new RecordHeaders(headers);
     }
 
-
     /**
      * Creates a record.
      * 
@@ -67,7 +65,7 @@ public TestRecord(final K key, final V value, final Headers headers, final Long
         if (timestampMs != null) {
             if (timestampMs < 0) {
                 throw new IllegalArgumentException(
-                        String.format("Invalid timestamp: %d. Timestamp should always be non-negative or null.", timestampMs));
+                    String.format("Invalid timestamp: %d. Timestamp should always be non-negative or null.", timestampMs));
             }
             this.recordTime = Instant.ofEpochMilli(timestampMs);
         } else {
@@ -152,22 +150,21 @@ public TestRecord(final ProducerRecord record) {
     }
 
     /**
-     * @return The headers
+     * @return The headers.
      */
     public Headers headers() {
         return headers;
     }
 
     /**
-     * @return The key (or null if no key is specified)
+     * @return The key (or {@code null} if no key is specified).
      */
     public K key() {
         return key;
     }
 
-
     /**
-     * @return The value
+     * @return The value.
      */
     public V value() {
         return value;
@@ -181,7 +178,7 @@ public Long timestamp() {
     }
 
     /**
-     * @return The headers
+     * @return The headers.
      */
     public Headers getHeaders() {
         return headers;
@@ -195,14 +192,14 @@ public K getKey() {
     }
 
     /**
-     * @return The value
+     * @return The value.
      */
     public V getValue() {
         return value;
     }
 
     /**
-     * @return The recordTime as Instant
+     * @return The timestamp.
      */
     public Instant getRecordTime() {
         return recordTime;
@@ -220,31 +217,21 @@ public String toString() {
 
     @Override
     public boolean equals(final Object o) {
-        if (this == o)
+        if (this == o) {
             return true;
-        else if (!(o instanceof TestRecord))
+        }
+        if (o == null || getClass() != o.getClass()) {
             return false;
-
+        }
         final TestRecord that = (TestRecord) o;
-
-        if (key != null ? !key.equals(that.key) : that.key != null) 
-            return false;
-        else if (headers != null ? !headers.equals(that.headers) : that.headers != null)
-            return false;
-        else if (value != null ? !value.equals(that.value) : that.value != null) 
-            return false;
-        else if (recordTime != null ? !recordTime.equals(that.recordTime) : that.recordTime != null)
-            return false;
-
-        return true;
+        return Objects.equals(headers, that.headers) &&
+            Objects.equals(key, that.key) &&
+            Objects.equals(value, that.value) &&
+            Objects.equals(recordTime, that.recordTime);
     }
 
     @Override
     public int hashCode() {
-        int result = headers != null ? headers.hashCode() : 0;
-        result = 31 * result + (key != null ? key.hashCode() : 0);
-        result = 31 * result + (value != null ? value.hashCode() : 0);
-        result = 31 * result + (recordTime != null ? recordTime.hashCode() : 0);
-        return result;
+        return Objects.hash(headers, key, value, recordTime);
     }
 }
diff --git a/streams/test-utils/src/test/java/org/apache/kafka/streams/TestTopicsTest.java b/streams/test-utils/src/test/java/org/apache/kafka/streams/TestTopicsTest.java
index 381968851ba5f..0e22ec95911e0 100644
--- a/streams/test-utils/src/test/java/org/apache/kafka/streams/TestTopicsTest.java
+++ b/streams/test-utils/src/test/java/org/apache/kafka/streams/TestTopicsTest.java
@@ -98,8 +98,10 @@ public void tearDown() {
 
     @Test
     public void testValue() {
-        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC, stringSerde.serializer(), stringSerde.serializer());
-        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, stringSerde.deserializer(), stringSerde.deserializer());
+        final TestInputTopic inputTopic =
+            testDriver.createInputTopic(INPUT_TOPIC, stringSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic =
+            testDriver.createOutputTopic(OUTPUT_TOPIC, stringSerde.deserializer(), stringSerde.deserializer());
         //Feed word "Hello" to inputTopic and no kafka key, timestamp is irrelevant in this case
         inputTopic.pipeInput("Hello");
         assertThat(outputTopic.readValue(), equalTo("Hello"));
@@ -109,8 +111,10 @@ public void testValue() {
 
     @Test
     public void testValueList() {
-        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC, stringSerde.serializer(), stringSerde.serializer());
-        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, stringSerde.deserializer(), stringSerde.deserializer());
+        final TestInputTopic inputTopic =
+            testDriver.createInputTopic(INPUT_TOPIC, stringSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic =
+            testDriver.createOutputTopic(OUTPUT_TOPIC, stringSerde.deserializer(), stringSerde.deserializer());
         final List inputList = Arrays.asList("This", "is", "an", "example");
         //Feed list of words to inputTopic and no kafka key, timestamp is irrelevant in this case
         inputTopic.pipeValueList(inputList);
@@ -121,8 +125,10 @@ public void testValueList() {
 
     @Test
     public void testKeyValue() {
-        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC, longSerde.serializer(), stringSerde.serializer());
-        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
+        final TestInputTopic inputTopic =
+            testDriver.createInputTopic(INPUT_TOPIC, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic =
+            testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
         inputTopic.pipeInput(1L, "Hello");
         assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>(1L, "Hello")));
         assertThat(outputTopic.isEmpty(), is(true));
@@ -130,8 +136,10 @@ public void testKeyValue() {
 
     @Test
     public void testKeyValueList() {
-        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
-        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC_MAP, stringSerde.deserializer(), longSerde.deserializer());
+        final TestInputTopic inputTopic =
+            testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic =
+            testDriver.createOutputTopic(OUTPUT_TOPIC_MAP, stringSerde.deserializer(), longSerde.deserializer());
         final List inputList = Arrays.asList("This", "is", "an", "example");
         final List> input = new LinkedList<>();
         final List> expected = new LinkedList<>();
@@ -148,8 +156,10 @@ public void testKeyValueList() {
 
     @Test
     public void testKeyValuesToMap() {
-        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
-        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC_MAP, stringSerde.deserializer(), longSerde.deserializer());
+        final TestInputTopic inputTopic =
+            testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic =
+            testDriver.createOutputTopic(OUTPUT_TOPIC_MAP, stringSerde.deserializer(), longSerde.deserializer());
         final List inputList = Arrays.asList("This", "is", "an", "example");
         final List> input = new LinkedList<>();
         final Map expected = new HashMap<>();
@@ -166,17 +176,21 @@ public void testKeyValuesToMap() {
 
     @Test
     public void testKeyValuesToMapWithNull() {
-        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC, longSerde.serializer(), stringSerde.serializer());
-        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
+        final TestInputTopic inputTopic =
+            testDriver.createInputTopic(INPUT_TOPIC, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic =
+            testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
         inputTopic.pipeInput("value");
-        assertThrows(IllegalStateException.class, () -> outputTopic.readKeyValuesToMap());
+        assertThrows(IllegalStateException.class, outputTopic::readKeyValuesToMap);
     }
 
 
     @Test
     public void testKeyValueListDuration() {
-        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
-        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC_MAP, stringSerde.deserializer(), longSerde.deserializer());
+        final TestInputTopic inputTopic =
+            testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic =
+            testDriver.createOutputTopic(OUTPUT_TOPIC_MAP, stringSerde.deserializer(), longSerde.deserializer());
         final List inputList = Arrays.asList("This", "is", "an", "example");
         final List> input = new LinkedList<>();
         final List> expected = new LinkedList<>();
@@ -196,8 +210,10 @@ public void testKeyValueListDuration() {
 
     @Test
     public void testRecordList() {
-        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
-        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC_MAP, stringSerde.deserializer(), longSerde.deserializer());
+        final TestInputTopic inputTopic =
+            testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic =
+            testDriver.createOutputTopic(OUTPUT_TOPIC_MAP, stringSerde.deserializer(), longSerde.deserializer());
         final List inputList = Arrays.asList("This", "is", "an", "example");
         final List> input = new LinkedList<>();
         final List> expected = new LinkedList<>();
@@ -205,8 +221,8 @@ public void testRecordList() {
         Instant recordInstant = testBaseTime;
         Long i = 0L;
         for (final String s : inputList) {
-            input.add(new TestRecord(i, s, recordInstant));
-            expected.add(new TestRecord(s, i, recordInstant));
+            input.add(new TestRecord<>(i, s, recordInstant));
+            expected.add(new TestRecord<>(s, i, recordInstant));
             i++;
             recordInstant = recordInstant.plus(advance);
         }
@@ -218,24 +234,26 @@ public void testRecordList() {
     @Test
     public void testTimestamp() {
         long baseTime = 3;
-        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC, longSerde.serializer(), stringSerde.serializer());
-        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
+        final TestInputTopic inputTopic =
+            testDriver.createInputTopic(INPUT_TOPIC, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic =
+            testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
         inputTopic.pipeInput(null, "Hello", baseTime);
-        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(null, "Hello", null, baseTime))));
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord<>(null, "Hello", null, baseTime))));
 
         inputTopic.pipeInput(2L, "Kafka", ++baseTime);
-        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(2L, "Kafka", null, baseTime))));
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord<>(2L, "Kafka", null, baseTime))));
 
         inputTopic.pipeInput(2L, "Kafka", testBaseTime);
-        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(2L, "Kafka", testBaseTime))));
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord<>(2L, "Kafka", testBaseTime))));
 
         final List inputList = Arrays.asList("Advancing", "time");
         //Feed list of words to inputTopic and no kafka key, timestamp advancing from testInstant
         final Duration advance = Duration.ofSeconds(15);
         final Instant recordInstant = testBaseTime.plus(Duration.ofDays(1));
         inputTopic.pipeValueList(inputList, recordInstant, advance);
-        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(null, "Advancing", recordInstant))));
-        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(null, "time", null, recordInstant.plus(advance)))));
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord<>(null, "Advancing", recordInstant))));
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord<>(null, "time", null, recordInstant.plus(advance)))));
     }
 
     @Test
@@ -247,50 +265,59 @@ public void testWithHeaders() {
                     new RecordHeader("bar", (byte[]) null),
                     new RecordHeader("\"A\\u00ea\\u00f1\\u00fcC\"", "value".getBytes())
                 });
-        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC, longSerde.serializer(), stringSerde.serializer());
-        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
-        inputTopic.pipeInput(new TestRecord(1L, "Hello", headers));
+        final TestInputTopic inputTopic =
+            testDriver.createInputTopic(INPUT_TOPIC, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic =
+            testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
+        inputTopic.pipeInput(new TestRecord<>(1L, "Hello", headers));
         assertThat(outputTopic.readRecord(), allOf(
                 hasProperty("key", equalTo(1L)),
                 hasProperty("value", equalTo("Hello")),
                 hasProperty("headers", equalTo(headers))));
-        inputTopic.pipeInput(new TestRecord(2L, "Kafka", headers, ++baseTime));
-        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(2L, "Kafka", headers, baseTime))));
+        inputTopic.pipeInput(new TestRecord<>(2L, "Kafka", headers, ++baseTime));
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord<>(2L, "Kafka", headers, baseTime))));
     }
 
     @Test
     public void testStartTimestamp() {
-        final long baseTime = testBaseTime.toEpochMilli();
         final Duration advance = Duration.ofSeconds(2);
-        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC, longSerde.serializer(), stringSerde.serializer(), testBaseTime, Duration.ZERO);
-        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
+        final TestInputTopic inputTopic =
+            testDriver.createInputTopic(INPUT_TOPIC, longSerde.serializer(), stringSerde.serializer(), testBaseTime, Duration.ZERO);
+        final TestOutputTopic outputTopic =
+            testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
         inputTopic.pipeInput(1L, "Hello");
-        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(1L, "Hello", testBaseTime))));
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord<>(1L, "Hello", testBaseTime))));
         inputTopic.pipeInput(2L, "World");
-        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(2L, "World", null, testBaseTime.toEpochMilli()))));
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord<>(2L, "World", null, testBaseTime.toEpochMilli()))));
         inputTopic.advanceTime(advance);
         inputTopic.pipeInput(3L, "Kafka");
-        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(3L, "Kafka", testBaseTime.plus(advance)))));
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord<>(3L, "Kafka", testBaseTime.plus(advance)))));
     }
 
     @Test
     public void testTimestampAutoAdvance() {
         final Duration advance = Duration.ofSeconds(2);
-        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC, longSerde.serializer(), stringSerde.serializer(), testBaseTime, advance);
-        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
+        final TestInputTopic inputTopic =
+            testDriver.createInputTopic(INPUT_TOPIC, longSerde.serializer(), stringSerde.serializer(), testBaseTime, advance);
+        final TestOutputTopic outputTopic =
+            testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
         inputTopic.pipeInput("Hello");
-        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(null, "Hello", testBaseTime))));
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord<>(null, "Hello", testBaseTime))));
         inputTopic.pipeInput(2L, "Kafka");
-        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord(2L, "Kafka", testBaseTime.plus(advance)))));
+        assertThat(outputTopic.readRecord(), is(equalTo(new TestRecord<>(2L, "Kafka", testBaseTime.plus(advance)))));
     }
 
 
     @Test
     public void testMultipleTopics() {
-        final TestInputTopic inputTopic1 = testDriver.createInputTopic(INPUT_TOPIC, longSerde.serializer(), stringSerde.serializer());
-        final TestInputTopic inputTopic2 = testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
-        final TestOutputTopic outputTopic1 = testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
-        final TestOutputTopic outputTopic2 = testDriver.createOutputTopic(OUTPUT_TOPIC_MAP, stringSerde.deserializer(), longSerde.deserializer());
+        final TestInputTopic inputTopic1 =
+            testDriver.createInputTopic(INPUT_TOPIC, longSerde.serializer(), stringSerde.serializer());
+        final TestInputTopic inputTopic2 =
+            testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic1 =
+            testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
+        final TestOutputTopic outputTopic2 =
+            testDriver.createOutputTopic(OUTPUT_TOPIC_MAP, stringSerde.deserializer(), longSerde.deserializer());
         inputTopic1.pipeInput(1L, "Hello");
         assertThat(outputTopic1.readKeyValue(), equalTo(new KeyValue<>(1L, "Hello")));
         assertThat(outputTopic2.readKeyValue(), equalTo(new KeyValue<>("Hello", 1L)));
@@ -305,64 +332,72 @@ public void testMultipleTopics() {
 
     @Test
     public void testNonExistingOutputTopic() {
-        final TestOutputTopic outputTopic = testDriver.createOutputTopic("no-exist", longSerde.deserializer(), stringSerde.deserializer());
-        assertThrows("Unknown topic", IllegalArgumentException.class, () -> outputTopic.readRecord());
+        final TestOutputTopic outputTopic =
+            testDriver.createOutputTopic("no-exist", longSerde.deserializer(), stringSerde.deserializer());
+        assertThrows("Unknown topic", IllegalArgumentException.class, outputTopic::readRecord);
     }
 
     @Test
     public void testNonUsedOutputTopic() {
-        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
-        assertThrows("Uninitialized topic", NoSuchElementException.class, () -> outputTopic.readRecord());
+        final TestOutputTopic outputTopic =
+            testDriver.createOutputTopic(OUTPUT_TOPIC, longSerde.deserializer(), stringSerde.deserializer());
+        assertThrows("Uninitialized topic", NoSuchElementException.class, outputTopic::readRecord);
     }
 
     @Test
     public void testEmptyTopic() {
-        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC, stringSerde.serializer(), stringSerde.serializer());
-        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, stringSerde.deserializer(), stringSerde.deserializer());
+        final TestInputTopic inputTopic =
+            testDriver.createInputTopic(INPUT_TOPIC, stringSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic =
+            testDriver.createOutputTopic(OUTPUT_TOPIC, stringSerde.deserializer(), stringSerde.deserializer());
         //Feed word "Hello" to inputTopic and no kafka key, timestamp is irrelevant in this case
         inputTopic.pipeInput("Hello");
         assertThat(outputTopic.readValue(), equalTo("Hello"));
         //No more output in topic
-        assertThrows("Empty topic", NoSuchElementException.class, () -> outputTopic.readRecord());
+        assertThrows("Empty topic", NoSuchElementException.class, outputTopic::readRecord);
     }
 
     @Test
     public void testNonExistingInputTopic() {
-        final TestInputTopic inputTopic = testDriver.createInputTopic("no-exist", longSerde.serializer(), stringSerde.serializer());
+        final TestInputTopic inputTopic =
+            testDriver.createInputTopic("no-exist", longSerde.serializer(), stringSerde.serializer());
         assertThrows("Unknown topic", IllegalArgumentException.class, () -> inputTopic.pipeInput(1L, "Hello"));
     }
 
     @Test(expected = NullPointerException.class)
     public void shouldNotAllowToCreateTopicWithNullTopicName() {
-        final TestInputTopic inputTopic = testDriver.createInputTopic(null, stringSerde.serializer(), stringSerde.serializer());
+        testDriver.createInputTopic(null, stringSerde.serializer(), stringSerde.serializer());
     }
 
     @Test(expected = NullPointerException.class)
     public void shouldNotAllowToCreateWithNullDriver() {
-        final TestInputTopic inputTopic = new TestInputTopic<>(null, INPUT_TOPIC, stringSerde.serializer(), stringSerde.serializer(), Instant.now(), Duration.ZERO);
+        new TestInputTopic<>(null, INPUT_TOPIC, stringSerde.serializer(), stringSerde.serializer(), Instant.now(), Duration.ZERO);
     }
 
 
     @Test(expected = StreamsException.class)
     public void testWrongSerde() {
-        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC_MAP, stringSerde.serializer(), stringSerde.serializer());
+        final TestInputTopic inputTopic =
+            testDriver.createInputTopic(INPUT_TOPIC_MAP, stringSerde.serializer(), stringSerde.serializer());
         inputTopic.pipeInput("1L", "Hello");
     }
 
     @Test(expected = IllegalArgumentException.class)
     public void testDuration() {
-        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC_MAP, stringSerde.serializer(), stringSerde.serializer(), testBaseTime, Duration.ofDays(-1));
+        testDriver.createInputTopic(INPUT_TOPIC_MAP, stringSerde.serializer(), stringSerde.serializer(), testBaseTime, Duration.ofDays(-1));
     }
 
     @Test(expected = IllegalArgumentException.class)
     public void testNegativeAdvance() {
-        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC_MAP, stringSerde.serializer(), stringSerde.serializer());
+        final TestInputTopic inputTopic =
+            testDriver.createInputTopic(INPUT_TOPIC_MAP, stringSerde.serializer(), stringSerde.serializer());
         inputTopic.advanceTime(Duration.ofDays(-1));
     }
 
     @Test
     public void testInputToString() {
-        final TestInputTopic inputTopic = testDriver.createInputTopic("topicName", stringSerde.serializer(), stringSerde.serializer());
+        final TestInputTopic inputTopic =
+            testDriver.createInputTopic("topicName", stringSerde.serializer(), stringSerde.serializer());
         assertThat(inputTopic.toString(), allOf(
                 containsString("TestInputTopic"),
                 containsString("topic='topicName'"),
@@ -371,25 +406,28 @@ public void testInputToString() {
 
     @Test(expected = NullPointerException.class)
     public void shouldNotAllowToCreateOutputTopicWithNullTopicName() {
-        final TestOutputTopic outputTopic = testDriver.createOutputTopic(null, stringSerde.deserializer(), stringSerde.deserializer());
+        testDriver.createOutputTopic(null, stringSerde.deserializer(), stringSerde.deserializer());
     }
 
     @Test(expected = NullPointerException.class)
     public void shouldNotAllowToCreateOutputWithNullDriver() {
-        final TestOutputTopic outputTopic = new TestOutputTopic<>(null, OUTPUT_TOPIC, stringSerde.deserializer(), stringSerde.deserializer());
+        new TestOutputTopic<>(null, OUTPUT_TOPIC, stringSerde.deserializer(), stringSerde.deserializer());
     }
 
     @Test(expected = SerializationException.class)
     public void testOutputWrongSerde() {
-        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
-        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC_MAP, longSerde.deserializer(), stringSerde.deserializer());
+        final TestInputTopic inputTopic =
+            testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic =
+            testDriver.createOutputTopic(OUTPUT_TOPIC_MAP, longSerde.deserializer(), stringSerde.deserializer());
         inputTopic.pipeInput(1L, "Hello");
         assertThat(outputTopic.readKeyValue(), equalTo(new KeyValue<>(1L, "Hello")));
     }
 
     @Test
     public void testOutputToString() {
-        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC, stringSerde.deserializer(), stringSerde.deserializer());
+        final TestOutputTopic outputTopic =
+            testDriver.createOutputTopic(OUTPUT_TOPIC, stringSerde.deserializer(), stringSerde.deserializer());
         assertThat(outputTopic.toString(), allOf(
                 containsString("TestOutputTopic"),
                 containsString("topic='output1'"),
@@ -399,8 +437,10 @@ public void testOutputToString() {
 
     @Test
     public void testRecordsToList() {
-        final TestInputTopic inputTopic = testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
-        final TestOutputTopic outputTopic = testDriver.createOutputTopic(OUTPUT_TOPIC_MAP, stringSerde.deserializer(), longSerde.deserializer());
+        final TestInputTopic inputTopic =
+            testDriver.createInputTopic(INPUT_TOPIC_MAP, longSerde.serializer(), stringSerde.serializer());
+        final TestOutputTopic outputTopic =
+            testDriver.createOutputTopic(OUTPUT_TOPIC_MAP, stringSerde.deserializer(), longSerde.deserializer());
         final List inputList = Arrays.asList("This", "is", "an", "example");
         final List> input = new LinkedList<>();
         final List> expected = new LinkedList<>();
diff --git a/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java b/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java
index 7e953920b0cb2..d7ac6b46557ea 100644
--- a/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java
+++ b/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java
@@ -99,15 +99,17 @@ public class TopologyTestDriverTest {
     private final byte[] key2 = new byte[0];
     private final byte[] value2 = new byte[0];
     private final long timestamp2 = 43L;
-    private final TestRecord testRecord2 = new TestRecord<>(key2, value2, null, timestamp2);
 
     //Factory and records for testing already deprecated methods
     @SuppressWarnings("deprecation")
-    private final org.apache.kafka.streams.test.ConsumerRecordFactory consumerRecordFactory = new org.apache.kafka.streams.test.ConsumerRecordFactory<>(
+    private final org.apache.kafka.streams.test.ConsumerRecordFactory consumerRecordFactory =
+        new org.apache.kafka.streams.test.ConsumerRecordFactory<>(
             new ByteArraySerializer(),
             new ByteArraySerializer());
-    private final ConsumerRecord consumerRecord1 = consumerRecordFactory.create(SOURCE_TOPIC_1, key1, value1, headers, timestamp1);
-    private final ConsumerRecord consumerRecord2 = consumerRecordFactory.create(SOURCE_TOPIC_2, key2, value2, timestamp2);
+    private final ConsumerRecord consumerRecord1 =
+        consumerRecordFactory.create(SOURCE_TOPIC_1, key1, value1, headers, timestamp1);
+    private final ConsumerRecord consumerRecord2 =
+        consumerRecordFactory.create(SOURCE_TOPIC_2, key2, value2, timestamp2);
 
     private TopologyTestDriver testDriver;
     private final Properties config = mkProperties(mkMap(
@@ -220,7 +222,7 @@ private final static class Punctuation {
         }
     }
 
-    private final class MockPunctuator implements Punctuator {
+    private final static class MockPunctuator implements Punctuator {
         private final List punctuatedAt = new LinkedList<>();
 
         @Override
@@ -229,7 +231,7 @@ public void punctuate(final long timestamp) {
         }
     }
 
-    private final class MockProcessor implements Processor {
+    private final static class MockProcessor implements Processor {
         private final Collection punctuations;
         private ProcessorContext context;
 
@@ -365,7 +367,12 @@ private Topology setupGlobalStoreTopology(final String... sourceTopicNames) {
 
         for (final String sourceTopicName : sourceTopicNames) {
             topology.addGlobalStore(
-                Stores.keyValueStoreBuilder(Stores.inMemoryKeyValueStore(sourceTopicName + "-globalStore"), null, null).withLoggingDisabled(),
+                Stores.keyValueStoreBuilder(
+                    Stores.inMemoryKeyValueStore(
+                        sourceTopicName + "-globalStore"),
+                    null,
+                    null)
+                    .withLoggingDisabled(),
                 sourceTopicName,
                 null,
                 null,
@@ -396,28 +403,37 @@ public void shouldCloseProcessor() {
 
     @Test
     public void shouldThrowForUnknownTopic() {
-        final String unknownTopic = "unknownTopic";
-        final byte[] nullValue = null;
-
         testDriver = new TopologyTestDriver(new Topology(), config);
-        assertThrows(IllegalArgumentException.class, () -> {
-            testDriver.pipeRecord(unknownTopic, new TestRecord(nullValue), new ByteArraySerializer(), new ByteArraySerializer(), Instant.now());
-        });
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> testDriver.pipeRecord(
+                "unknownTopic",
+                new TestRecord<>((byte[]) null),
+                new ByteArraySerializer(),
+                new ByteArraySerializer(),
+                Instant.now())
+        );
     }
 
     @Test
     public void shouldThrowForMissingTime() {
         testDriver = new TopologyTestDriver(new Topology(), config);
-        assertThrows(IllegalStateException.class, () -> {
-            testDriver.pipeRecord(SINK_TOPIC_1, new TestRecord("value"), new StringSerializer(), new StringSerializer(), null);
-        });
+        assertThrows(
+            IllegalStateException.class,
+            () -> testDriver.pipeRecord(
+                SINK_TOPIC_1,
+                new TestRecord<>("value"),
+                new StringSerializer(),
+                new StringSerializer(),
+                null));
     }
 
     @Deprecated
     @Test
     public void shouldThrowForUnknownTopicDeprecated() {
         final String unknownTopic = "unknownTopic";
-        final org.apache.kafka.streams.test.ConsumerRecordFactory consumerRecordFactory = new org.apache.kafka.streams.test.ConsumerRecordFactory<>(
+        final org.apache.kafka.streams.test.ConsumerRecordFactory consumerRecordFactory =
+            new org.apache.kafka.streams.test.ConsumerRecordFactory<>(
                 "unknownTopic",
                 new ByteArraySerializer(),
                 new ByteArraySerializer());
@@ -522,14 +538,16 @@ public void shouldUseSourceSpecificDeserializersDeprecated() {
 
         testDriver = new TopologyTestDriver(topology, config);
 
-        final org.apache.kafka.streams.test.ConsumerRecordFactory source1Factory = new org.apache.kafka.streams.test.ConsumerRecordFactory<>(
-            SOURCE_TOPIC_1,
-            Serdes.Long().serializer(),
-            Serdes.String().serializer());
-        final org.apache.kafka.streams.test.ConsumerRecordFactory source2Factory = new org.apache.kafka.streams.test.ConsumerRecordFactory<>(
-            SOURCE_TOPIC_2,
-            Serdes.Integer().serializer(),
-            Serdes.Double().serializer());
+        final org.apache.kafka.streams.test.ConsumerRecordFactory source1Factory =
+            new org.apache.kafka.streams.test.ConsumerRecordFactory<>(
+                SOURCE_TOPIC_1,
+                Serdes.Long().serializer(),
+                Serdes.String().serializer());
+        final org.apache.kafka.streams.test.ConsumerRecordFactory source2Factory =
+            new org.apache.kafka.streams.test.ConsumerRecordFactory<>(
+                SOURCE_TOPIC_2,
+                Serdes.Integer().serializer(),
+                Serdes.Double().serializer());
 
         final Long source1Key = 42L;
         final String source1Value = "anyString";
@@ -540,12 +558,14 @@ public void shouldUseSourceSpecificDeserializersDeprecated() {
         final ConsumerRecord consumerRecord2 = source2Factory.create(source2Key, source2Value);
 
         testDriver.pipeInput(consumerRecord1);
-        final ProducerRecord record1 = testDriver.readOutput(SINK_TOPIC_1, Serdes.Long().deserializer(), Serdes.String().deserializer());
+        final ProducerRecord record1 =
+            testDriver.readOutput(SINK_TOPIC_1, Serdes.Long().deserializer(), Serdes.String().deserializer());
         assertThat(record1.key(), equalTo(source1Key));
         assertThat(record1.value(), equalTo(source1Value));
 
         testDriver.pipeInput(consumerRecord2);
-        final ProducerRecord record2 = testDriver.readOutput(SINK_TOPIC_1, Serdes.Integer().deserializer(), Serdes.Double().deserializer());
+        final ProducerRecord record2 =
+            testDriver.readOutput(SINK_TOPIC_1, Serdes.Integer().deserializer(), Serdes.Double().deserializer());
         assertThat(record2.key(), equalTo(source2Key));
         assertThat(record2.value(), equalTo(source2Value));
     }
@@ -724,42 +744,42 @@ public void shouldPunctuateOnStreamsTime() {
         final List expectedPunctuations = new LinkedList<>();
 
         expectedPunctuations.add(42L);
-        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 42L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord<>(key1, value1, null, 42L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
 
-        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 42L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord<>(key1, value1, null, 42L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
 
         expectedPunctuations.add(51L);
-        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 51L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord<>(key1, value1, null, 51L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
 
-        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 52L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord<>(key1, value1, null, 52L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
 
         expectedPunctuations.add(61L);
-        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 61L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord<>(key1, value1, null, 61L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
 
-        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 65L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord<>(key1, value1, null, 65L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
 
         expectedPunctuations.add(71L);
-        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 71L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord<>(key1, value1, null, 71L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
 
-        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 72L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord<>(key1, value1, null, 72L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
 
         expectedPunctuations.add(95L);
-        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 95L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord<>(key1, value1, null, 95L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
 
         expectedPunctuations.add(101L);
-        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 101L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord<>(key1, value1, null, 101L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
 
-        pipeRecord(SOURCE_TOPIC_1, new TestRecord(key1, value1, null, 102L));
+        pipeRecord(SOURCE_TOPIC_1, new TestRecord<>(key1, value1, null, 102L));
         assertThat(mockPunctuator.punctuatedAt, equalTo(expectedPunctuations));
     }
 
@@ -1186,7 +1206,7 @@ private void setup(final KeyValueBytesStoreSupplier storeSupplier) {
     }
 
     private void pipeInput(final String topic, final String key, final Long value, final Long time) {
-        testDriver.pipeRecord(topic, new TestRecord(key, value, null, time),
+        testDriver.pipeRecord(topic, new TestRecord<>(key, value, null, time),
                 new StringSerializer(), new LongSerializer(), null);
     }
     
@@ -1253,14 +1273,14 @@ public void shouldPunctuateIfWallClockTimeAdvances() {
         assertTrue(testDriver.isEmpty("result-topic"));
     }
 
-    private class CustomMaxAggregatorSupplier implements ProcessorSupplier {
+    private static class CustomMaxAggregatorSupplier implements ProcessorSupplier {
         @Override
         public Processor get() {
             return new CustomMaxAggregator();
         }
     }
 
-    private class CustomMaxAggregator implements Processor {
+    private static class CustomMaxAggregator implements Processor {
         ProcessorContext context;
         private KeyValueStore store;
 
@@ -1341,7 +1361,9 @@ public void close() {}
             },
             "sourceProcessor"
         );
-        topology.addStateStore(Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore("storeProcessorStore"), Serdes.String(), Serdes.Long()), "storeProcessor");
+        topology.addStateStore(Stores.keyValueStoreBuilder(
+            Stores.persistentKeyValueStore("storeProcessorStore"), Serdes.String(), Serdes.Long()),
+            "storeProcessor");
 
         final Properties config = new Properties();
         config.put(StreamsConfig.APPLICATION_ID_CONFIG, "test-TopologyTestDriver-cleanup");
@@ -1352,7 +1374,7 @@ public void close() {}
 
         try (final TopologyTestDriver testDriver = new TopologyTestDriver(topology, config)) {
             assertNull(testDriver.getKeyValueStore("storeProcessorStore").get("a"));
-            testDriver.pipeRecord("input-topic", new TestRecord("a", 1L),
+            testDriver.pipeRecord("input-topic", new TestRecord<>("a", 1L),
                     new StringSerializer(), new LongSerializer(), Instant.now());
             Assert.assertEquals(1L, testDriver.getKeyValueStore("storeProcessorStore").get("a"));
         }
@@ -1377,7 +1399,12 @@ public void shouldFeedStoreFromGlobalKTable() {
             final KeyValueStore globalStore = testDriver.getKeyValueStore("globalStore");
             Assert.assertNotNull(globalStore);
             Assert.assertNotNull(testDriver.getAllStateStores().get("globalStore"));
-            testDriver.pipeRecord("topic", new TestRecord("k1", "value1"), new StringSerializer(), new StringSerializer(), Instant.now());
+            testDriver.pipeRecord(
+                "topic",
+                new TestRecord<>("k1", "value1"),
+                new StringSerializer(),
+                new StringSerializer(),
+                Instant.now());
             // we expect to have both in the global store, the one from pipeInput and the one from the producer
             Assert.assertEquals("value1", globalStore.get("k1"));
         }
@@ -1406,7 +1433,7 @@ public void shouldProcessFromSourcesThatMatchMultiplePattern() {
         final  Pattern pattern2Source2 = Pattern.compile("source-topic-[A-Z]");
         final  String consumerTopic2 = "source-topic-Z";
 
-        final TestRecord consumerRecord2 = new TestRecord(key2, value2, null, timestamp2);
+        final TestRecord consumerRecord2 = new TestRecord<>(key2, value2, null, timestamp2);
 
         testDriver = new TopologyTestDriver(setupMultipleSourcesPatternTopology(pattern2Source1, pattern2Source2), config);
 
diff --git a/streams/test-utils/src/test/java/org/apache/kafka/streams/test/TestRecordTest.java b/streams/test-utils/src/test/java/org/apache/kafka/streams/test/TestRecordTest.java
index 9977fc54af273..5739693c57a29 100644
--- a/streams/test-utils/src/test/java/org/apache/kafka/streams/test/TestRecordTest.java
+++ b/streams/test-utils/src/test/java/org/apache/kafka/streams/test/TestRecordTest.java
@@ -32,7 +32,6 @@
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.hasProperty;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
 
 public class TestRecordTest {
@@ -102,16 +101,16 @@ public void testEqualsAndHashCode() {
                     new RecordHeader("bar", (byte[]) null),
                 });
         final TestRecord headerMismatch = new TestRecord<>(key, value, headers2, recordTime);
-        assertFalse(testRecord.equals(headerMismatch));
+        assertNotEquals(testRecord, headerMismatch);
 
         final TestRecord keyMisMatch = new TestRecord<>("test-mismatch", value, headers, recordTime);
-        assertFalse(testRecord.equals(keyMisMatch));
+        assertNotEquals(testRecord, keyMisMatch);
 
         final TestRecord valueMisMatch = new TestRecord<>(key, 2, headers, recordTime);
-        assertFalse(testRecord.equals(valueMisMatch));
+        assertNotEquals(testRecord, valueMisMatch);
 
         final TestRecord timeMisMatch = new TestRecord<>(key, value, headers, recordTime.plusMillis(1));
-        assertFalse(testRecord.equals(timeMisMatch));
+        assertNotEquals(testRecord, timeMisMatch);
 
         final TestRecord nullFieldsRecord = new TestRecord<>(null, null, null, (Instant) null);
         assertEquals(nullFieldsRecord, nullFieldsRecord);
@@ -121,21 +120,21 @@ public void testEqualsAndHashCode() {
     @Test
     public void testPartialConstructorEquals() {
         final TestRecord record1 = new TestRecord<>(value);
-        assertThat(record1, equalTo(new TestRecord(null, value, null, (Instant) null)));
+        assertThat(record1, equalTo(new TestRecord<>(null, value, null, (Instant) null)));
 
         final TestRecord record2 = new TestRecord<>(key, value);
-        assertThat(record2, equalTo(new TestRecord(key, value, null, (Instant) null)));
+        assertThat(record2, equalTo(new TestRecord<>(key, value, null, (Instant) null)));
 
         final TestRecord record3 = new TestRecord<>(key, value, headers);
-        assertThat(record3, equalTo(new TestRecord(key, value, headers, (Long) null)));
+        assertThat(record3, equalTo(new TestRecord<>(key, value, headers, (Long) null)));
 
         final TestRecord record4 = new TestRecord<>(key, value, recordTime);
-        assertThat(record4, equalTo(new TestRecord(key, value, null, recordMs)));
+        assertThat(record4, equalTo(new TestRecord<>(key, value, null, recordMs)));
     }
 
     @Test(expected = IllegalArgumentException.class)
     public void testInvalidRecords() {
-        new TestRecord(key, value, headers,  -1L);
+        new TestRecord<>(key, value, headers,  -1L);
     }
 
     @Test
@@ -150,7 +149,8 @@ public void testToString() {
     @Test
     public void testConsumerRecord() {
         final String topicName = "topic";
-        final ConsumerRecord consumerRecord = new ConsumerRecord<>(topicName, 1, 0, recordMs, TimestampType.CREATE_TIME, 0L, 0, 0, key, value, headers);
+        final ConsumerRecord consumerRecord =
+            new ConsumerRecord<>(topicName, 1, 0, recordMs, TimestampType.CREATE_TIME, 0L, 0, 0, key, value, headers);
         final TestRecord testRecord = new TestRecord<>(consumerRecord);
         final TestRecord expectedRecord = new TestRecord<>(key, value, headers, recordTime);
         assertEquals(expectedRecord, testRecord);
@@ -159,7 +159,8 @@ public void testConsumerRecord() {
     @Test
     public void testProducerRecord() {
         final String topicName = "topic";
-        final ProducerRecord producerRecord = new ProducerRecord<>(topicName, 1, recordMs, key, value, headers);
+        final ProducerRecord producerRecord =
+            new ProducerRecord<>(topicName, 1, recordMs, key, value, headers);
         final TestRecord testRecord = new TestRecord<>(producerRecord);
         final TestRecord expectedRecord = new TestRecord<>(key, value, headers, recordTime);
         assertEquals(expectedRecord, testRecord);

From 9cf2fa5215b9ba53893a2aada3d698156727cd7a Mon Sep 17 00:00:00 2001
From: Bruno Cadonna 
Date: Tue, 15 Oct 2019 23:30:16 +0200
Subject: [PATCH 0724/1071] KAFKA-9030: Document client-level (a.k.a.
 instance-level) metrics (#7501)

Reviewers: A. Sophie Blee-Goldman , Matthias J. Sax 
---
 docs/ops.html | 40 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 39 insertions(+), 1 deletion(-)

diff --git a/docs/ops.html b/docs/ops.html
index d87d16a73a888..18d9bb5ba0629 100644
--- a/docs/ops.html
+++ b/docs/ops.html
@@ -1485,7 +1485,8 @@ 
                Streams Mo
   the info level records only the thread-level metrics.
 
   
                
-    Note that the metrics have a 3-layer hierarchy. At the top level there are per-thread metrics. Each thread has tasks, with their
+    Note that the metrics have a 4-layer hierarchy. At the top level there are client-level metrics for each started
+    Kafka Streams client. Each client has stream threads, with their own metrics. Each stream thread has tasks, with their
     own metrics. Each task has a number of processor nodes, with their own metrics. Each task also has a number of state stores
     and record caches, all with their own metrics.
   
                
@@ -1495,6 +1496,43 @@ 
                Streams Mo
 
 
                metrics.recording.level="info"
                
 
+
                Client Metrics
                
+All the following metrics have a recording level of info:
+
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+
                Metric/Attribute nameDescriptionMbean name
                versionThe version of the Kafka Streams client.kafka.streams:type=stream-metrics,client-id=([-.\w]+)
                commit-idThe version control commit ID of the Kafka Streams client.kafka.streams:type=stream-metrics,client-id=([-.\w]+)
                application-idThe application ID of the Kafka Streams client.kafka.streams:type=stream-metrics,client-id=([-.\w]+)
                topology-descriptionThe description of the topology executed in the Kafka Streams client.kafka.streams:type=stream-metrics,client-id=([-.\w]+)
                stateThe state of the Kafka Streams client.kafka.streams:type=stream-metrics,client-id=([-.\w]+)
                
+
 
                Thread Metrics
                
 All the following metrics have a recording level of info:
 

From 8f8160b3cdadaf91ed5a9c7a6780fb59ef46cd66 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Tue, 15 Oct 2019 14:52:26 -0700
Subject: [PATCH 0725/1071] KAFKA-9033; Use consumer/producer identity in
 generated clientId (#7514)

By default, if the user does not configure a `client.id`, then we use a very generic identifier, such as `consumer-15`. It is more useful to include identifying information when available such as `group.id` for the consumer and `transactional.id` for the producer.

Reviewers: Guozhang Wang 
---
 .../kafka/clients/consumer/KafkaConsumer.java | 25 +++++++++++++------
 .../kafka/clients/producer/KafkaProducer.java | 17 ++++++++++---
 2 files changed, 31 insertions(+), 11 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index 7e023415dc3dd..cacf8d56a7f78 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -21,6 +21,7 @@
 import org.apache.kafka.clients.ApiVersions;
 import org.apache.kafka.clients.ClientDnsLookup;
 import org.apache.kafka.clients.ClientUtils;
+import org.apache.kafka.clients.CommonClientConfigs;
 import org.apache.kafka.clients.GroupRebalanceConfig;
 import org.apache.kafka.clients.Metadata;
 import org.apache.kafka.clients.NetworkClient;
@@ -667,15 +668,14 @@ public KafkaConsumer(Properties properties,
     @SuppressWarnings("unchecked")
     private KafkaConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) {
         try {
-            String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG);
-            if (clientId.isEmpty())
-                clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement();
-            this.clientId = clientId;
-            this.groupId = config.getString(ConsumerConfig.GROUP_ID_CONFIG);
-
             GroupRebalanceConfig groupRebalanceConfig = new GroupRebalanceConfig(config,
-                                                                                 GroupRebalanceConfig.ProtocolType.CONSUMER);
+                    GroupRebalanceConfig.ProtocolType.CONSUMER);
+
+            this.groupId = groupRebalanceConfig.groupId;
+            this.clientId = buildClientId(config.getString(CommonClientConfigs.CLIENT_ID_CONFIG), groupRebalanceConfig);
+
             LogContext logContext;
+
             // If group.instance.id is set, we will append it to the log context.
             if (groupRebalanceConfig.groupInstanceId.isPresent()) {
                 logContext = new LogContext("[Consumer instanceId=" + groupRebalanceConfig.groupInstanceId.get() +
@@ -854,6 +854,17 @@ else if (enableAutoCommit)
         this.groupId = groupId;
     }
 
+    private static String buildClientId(String configuredClientId, GroupRebalanceConfig rebalanceConfig) {
+        if (!configuredClientId.isEmpty())
+            return configuredClientId;
+
+        if (rebalanceConfig.groupId != null && !rebalanceConfig.groupId.isEmpty())
+            return "consumer-" + rebalanceConfig.groupId + "-" + rebalanceConfig.groupInstanceId.orElseGet(() ->
+                    CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement() + "");
+
+        return "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement();
+    }
+
     private static Metrics buildMetrics(ConsumerConfig config, Time time, String clientId) {
         Map metricsTags = Collections.singletonMap(CLIENT_ID_METRIC_TAG, clientId);
         MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG))
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
index 3b3589fa5b95c..4b7f3d2bd983d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
@@ -329,13 +329,12 @@ public KafkaProducer(Properties properties, Serializer keySerializer, Seriali
             Map userProvidedConfigs = config.originals();
             this.producerConfig = config;
             this.time = time;
-            String clientId = config.getString(ProducerConfig.CLIENT_ID_CONFIG);
-            if (clientId.length() <= 0)
-                clientId = "producer-" + PRODUCER_CLIENT_ID_SEQUENCE.getAndIncrement();
-            this.clientId = clientId;
 
             String transactionalId = userProvidedConfigs.containsKey(ProducerConfig.TRANSACTIONAL_ID_CONFIG) ?
                     (String) userProvidedConfigs.get(ProducerConfig.TRANSACTIONAL_ID_CONFIG) : null;
+
+            this.clientId = buildClientId(config.getString(ProducerConfig.CLIENT_ID_CONFIG), transactionalId);
+
             LogContext logContext;
             if (transactionalId == null)
                 logContext = new LogContext(String.format("[Producer clientId=%s] ", clientId));
@@ -434,6 +433,16 @@ public KafkaProducer(Properties properties, Serializer keySerializer, Seriali
         }
     }
 
+    private static String buildClientId(String configuredClientId, String transactionalId) {
+        if (!configuredClientId.isEmpty())
+            return configuredClientId;
+
+        if (transactionalId != null)
+            return "producer-" + transactionalId;
+
+        return "producer-" + PRODUCER_CLIENT_ID_SEQUENCE.getAndIncrement();
+    }
+
     // visible for testing
     Sender newSender(LogContext logContext, KafkaClient kafkaClient, ProducerMetadata metadata) {
         int maxInflightRequests = configureInflightRequests(producerConfig, transactionManager != null);

From 5a8ad0a8024bcc7e9111fbd74dd35e02bea0491b Mon Sep 17 00:00:00 2001
From: Konstantine Karantasis 
Date: Tue, 15 Oct 2019 14:08:31 -0700
Subject: [PATCH 0726/1071] KAFKA-9014: Fix AssertionError when SourceTask.poll
 returns an empty list (#7491)

Author: Konstantine Karantasis 
Reviewer: Randall Hauch 
---
 .../connect/runtime/WorkerSourceTask.java     |  3 +-
 .../connect/runtime/WorkerSourceTaskTest.java | 64 +++++++++++++++++++
 2 files changed, 66 insertions(+), 1 deletion(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
index c66429d31b53f..5e24d4145aef2 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
@@ -300,7 +300,8 @@ private ProducerRecord convertTransformedRecord(SourceRecord rec
     private boolean sendRecords() {
         int processed = 0;
         recordBatch(toSend.size());
-        final SourceRecordWriteCounter counter = new SourceRecordWriteCounter(toSend.size(), sourceTaskMetricsGroup);
+        final SourceRecordWriteCounter counter =
+                toSend.size() > 0 ? new SourceRecordWriteCounter(toSend.size(), sourceTaskMetricsGroup) : null;
         for (final SourceRecord preTransformRecord : toSend) {
             maybeThrowProducerSendException();
 
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java
index dff267a368e7f..e8039310b20da 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java
@@ -84,6 +84,7 @@
 import static org.junit.Assert.assertTrue;
 
 @PowerMockIgnore({"javax.management.*",
+                  "org.apache.log4j.*",
                   "org.apache.kafka.connect.runtime.isolation.*"})
 @RunWith(PowerMockRunner.class)
 public class WorkerSourceTaskTest extends ThreadedTest {
@@ -350,6 +351,51 @@ public List answer() throws Throwable {
         PowerMock.verifyAll();
     }
 
+    @Test
+    public void testPollReturnsNoRecords() throws Exception {
+        // Test that the task handles an empty list of records
+        createWorkerTask();
+
+        sourceTask.initialize(EasyMock.anyObject(SourceTaskContext.class));
+        EasyMock.expectLastCall();
+        sourceTask.start(TASK_PROPS);
+        EasyMock.expectLastCall();
+        statusListener.onStartup(taskId);
+        EasyMock.expectLastCall();
+
+        // We'll wait for some data, then trigger a flush
+        final CountDownLatch pollLatch = expectEmptyPolls(1, new AtomicInteger());
+        expectOffsetFlush(true);
+
+        sourceTask.stop();
+        EasyMock.expectLastCall();
+        expectOffsetFlush(true);
+
+        statusListener.onShutdown(taskId);
+        EasyMock.expectLastCall();
+
+        producer.close(EasyMock.anyObject(Duration.class));
+        EasyMock.expectLastCall();
+
+        transformationChain.close();
+        EasyMock.expectLastCall();
+
+        PowerMock.replayAll();
+
+        workerTask.initialize(TASK_CONFIG);
+        Future taskFuture = executor.submit(workerTask);
+
+        assertTrue(awaitLatch(pollLatch));
+        assertTrue(workerTask.commitOffsets());
+        workerTask.stop();
+        assertTrue(workerTask.awaitStop(1000));
+
+        taskFuture.get();
+        assertPollMetrics(0);
+
+        PowerMock.verifyAll();
+    }
+
     @Test
     public void testCommit() throws Exception {
         // Test that the task commits properly when prompted
@@ -767,6 +813,24 @@ public void testHeadersWithCustomConverter() throws Exception {
         PowerMock.verifyAll();
     }
 
+    private CountDownLatch expectEmptyPolls(int minimum, final AtomicInteger count) throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(minimum);
+        // Note that we stub these to allow any number of calls because the thread will continue to
+        // run. The count passed in + latch returned just makes sure we get *at least* that number of
+        // calls
+        EasyMock.expect(sourceTask.poll())
+                .andStubAnswer(new IAnswer>() {
+                    @Override
+                    public List answer() throws Throwable {
+                        count.incrementAndGet();
+                        latch.countDown();
+                        Thread.sleep(10);
+                        return Collections.emptyList();
+                    }
+                });
+        return latch;
+    }
+
     private CountDownLatch expectPolls(int minimum, final AtomicInteger count) throws InterruptedException {
         final CountDownLatch latch = new CountDownLatch(minimum);
         // Note that we stub these to allow any number of calls because the thread will continue to

From 3b78db367533b98d45e7528326857b1d2b0562bc Mon Sep 17 00:00:00 2001
From: Chris Egerton 
Date: Tue, 15 Oct 2019 15:27:26 -0700
Subject: [PATCH 0727/1071] KAFKA-8945/KAFKA-8947: Fix bugs in Connect REST
 extension API (#7392)

Fix bug in Connect REST extension API caused by invalid constructor parameter validation, and update integration test to play nicely with Jenkins

Fix instantiation of TaskState objects by Connect framework.

Author: Chris Egerton 
Reviewers: Magesh Nandakumar , Randall Hauch 
---
 .../kafka/connect/health/AbstractState.java   |  23 +++-
 .../kafka/connect/health/ConnectorHealth.java |  29 ++++-
 .../kafka/connect/health/ConnectorState.java  |   9 ++
 .../kafka/connect/health/TaskState.java       |  22 ++--
 .../health/ConnectClusterStateImpl.java       |   2 +-
 .../RestExtensionIntegrationTest.java         | 108 ++++++++++++++++--
 6 files changed, 173 insertions(+), 20 deletions(-)

diff --git a/connect/api/src/main/java/org/apache/kafka/connect/health/AbstractState.java b/connect/api/src/main/java/org/apache/kafka/connect/health/AbstractState.java
index a18c46334cf29..f707b3c3026ed 100644
--- a/connect/api/src/main/java/org/apache/kafka/connect/health/AbstractState.java
+++ b/connect/api/src/main/java/org/apache/kafka/connect/health/AbstractState.java
@@ -17,6 +17,8 @@
 
 package org.apache.kafka.connect.health;
 
+import java.util.Objects;
+
 /**
  * Provides the current status along with identifier for Connect worker and tasks.
  */
@@ -34,10 +36,10 @@ public abstract class AbstractState {
      * @param traceMessage  any error trace message associated with the connector or the task; may be null or empty
      */
     public AbstractState(String state, String workerId, String traceMessage) {
-        if (state != null && !state.trim().isEmpty()) {
+        if (state == null || state.trim().isEmpty()) {
             throw new IllegalArgumentException("State must not be null or empty");
         }
-        if (workerId != null && !workerId.trim().isEmpty()) {
+        if (workerId == null || workerId.trim().isEmpty()) {
             throw new IllegalArgumentException("Worker ID must not be null or empty");
         }
         this.state = state;
@@ -71,4 +73,21 @@ public String workerId() {
     public String traceMessage() {
         return traceMessage;
     }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        AbstractState that = (AbstractState) o;
+        return state.equals(that.state)
+            && Objects.equals(traceMessage, that.traceMessage)
+            && workerId.equals(that.workerId);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(state, traceMessage, workerId);
+    }
 }
diff --git a/connect/api/src/main/java/org/apache/kafka/connect/health/ConnectorHealth.java b/connect/api/src/main/java/org/apache/kafka/connect/health/ConnectorHealth.java
index 3a9efd15372ed..12fa6b76aff1e 100644
--- a/connect/api/src/main/java/org/apache/kafka/connect/health/ConnectorHealth.java
+++ b/connect/api/src/main/java/org/apache/kafka/connect/health/ConnectorHealth.java
@@ -35,7 +35,7 @@ public ConnectorHealth(String name,
                            ConnectorState connectorState,
                            Map tasks,
                            ConnectorType type) {
-        if (name != null && !name.trim().isEmpty()) {
+        if (name == null || name.trim().isEmpty()) {
             throw new IllegalArgumentException("Connector name is required");
         }
         Objects.requireNonNull(connectorState, "connectorState can't be null");
@@ -83,4 +83,31 @@ public ConnectorType type() {
         return type;
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        ConnectorHealth that = (ConnectorHealth) o;
+        return name.equals(that.name)
+            && connectorState.equals(that.connectorState)
+            && tasks.equals(that.tasks)
+            && type == that.type;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, connectorState, tasks, type);
+    }
+
+    @Override
+    public String toString() {
+        return "ConnectorHealth{"
+            + "name='" + name + '\''
+            + ", connectorState=" + connectorState
+            + ", tasks=" + tasks
+            + ", type=" + type
+            + '}';
+    }
 }
diff --git a/connect/api/src/main/java/org/apache/kafka/connect/health/ConnectorState.java b/connect/api/src/main/java/org/apache/kafka/connect/health/ConnectorState.java
index d5571bc4ff5d0..63044265bb999 100644
--- a/connect/api/src/main/java/org/apache/kafka/connect/health/ConnectorState.java
+++ b/connect/api/src/main/java/org/apache/kafka/connect/health/ConnectorState.java
@@ -32,4 +32,13 @@ public class ConnectorState extends AbstractState {
     public ConnectorState(String state, String workerId, String traceMessage) {
         super(state, workerId, traceMessage);
     }
+
+    @Override
+    public String toString() {
+        return "ConnectorState{"
+            + "state='" + state() + '\''
+            + ", traceMessage='" + traceMessage() + '\''
+            + ", workerId='" + workerId() + '\''
+            + '}';
+    }
 }
diff --git a/connect/api/src/main/java/org/apache/kafka/connect/health/TaskState.java b/connect/api/src/main/java/org/apache/kafka/connect/health/TaskState.java
index 1c1be159970d9..ae78a5f3af990 100644
--- a/connect/api/src/main/java/org/apache/kafka/connect/health/TaskState.java
+++ b/connect/api/src/main/java/org/apache/kafka/connect/health/TaskState.java
@@ -50,20 +50,28 @@ public int taskId() {
 
     @Override
     public boolean equals(Object o) {
-        if (this == o) {
+        if (this == o)
             return true;
-        }
-        if (o == null || getClass() != o.getClass()) {
+        if (o == null || getClass() != o.getClass())
+            return false;
+        if (!super.equals(o))
             return false;
-        }
-
         TaskState taskState = (TaskState) o;
-
         return taskId == taskState.taskId;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(taskId);
+        return Objects.hash(super.hashCode(), taskId);
+    }
+
+    @Override
+    public String toString() {
+        return "TaskState{"
+            + "taskId='" + taskId + '\''
+            + "state='" + state() + '\''
+            + ", traceMessage='" + traceMessage() + '\''
+            + ", workerId='" + workerId() + '\''
+            + '}';
     }
 }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/health/ConnectClusterStateImpl.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/health/ConnectClusterStateImpl.java
index c3e950eb45910..38362b3fc2780 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/health/ConnectClusterStateImpl.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/health/ConnectClusterStateImpl.java
@@ -107,7 +107,7 @@ private Map taskStates(List st
         for (ConnectorStateInfo.TaskState state : states) {
             taskStates.put(
                 state.id(),
-                new TaskState(state.id(), state.workerId(), state.state(), state.trace())
+                new TaskState(state.id(), state.state(), state.workerId(), state.trace())
             );
         }
         return taskStates;
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RestExtensionIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RestExtensionIntegrationTest.java
index d4cac3976f8be..087246b917767 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RestExtensionIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RestExtensionIntegrationTest.java
@@ -16,10 +16,16 @@
  */
 package org.apache.kafka.connect.integration;
 
+import org.apache.kafka.connect.health.ConnectClusterState;
+import org.apache.kafka.connect.health.ConnectorHealth;
+import org.apache.kafka.connect.health.ConnectorState;
+import org.apache.kafka.connect.health.ConnectorType;
+import org.apache.kafka.connect.health.TaskState;
 import org.apache.kafka.connect.rest.ConnectRestExtension;
 import org.apache.kafka.connect.rest.ConnectRestExtensionContext;
 import org.apache.kafka.connect.runtime.rest.errors.ConnectRestException;
 import org.apache.kafka.connect.util.clusters.EmbeddedConnectCluster;
+import org.apache.kafka.connect.util.clusters.WorkerHandle;
 import org.apache.kafka.test.IntegrationTest;
 import org.junit.After;
 import org.junit.Test;
@@ -28,12 +34,18 @@
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
+import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLASS_CONFIG;
+import static org.apache.kafka.connect.runtime.ConnectorConfig.NAME_CONFIG;
+import static org.apache.kafka.connect.runtime.ConnectorConfig.TASKS_MAX_CONFIG;
+import static org.apache.kafka.connect.runtime.SinkConnectorConfig.TOPICS_CONFIG;
 import static org.apache.kafka.connect.runtime.WorkerConfig.REST_EXTENSION_CLASSES_CONFIG;
 import static org.apache.kafka.test.TestUtils.waitForCondition;
+import static org.junit.Assert.assertEquals;
 
 /**
  * A simple integration test to ensure that REST extensions are registered correctly.
@@ -41,39 +53,86 @@
 @Category(IntegrationTest.class)
 public class RestExtensionIntegrationTest {
 
-    private static final int NUM_WORKERS = 3;
     private static final long REST_EXTENSION_REGISTRATION_TIMEOUT_MS = TimeUnit.MINUTES.toMillis(1);
+    private static final long CONNECTOR_HEALTH_AND_CONFIG_TIMEOUT_MS = TimeUnit.MINUTES.toMillis(1);
 
     private EmbeddedConnectCluster connect;
 
     @Test
-    public void testImmediateRequestForListOfConnectors() throws IOException, InterruptedException {
+    public void testRestExtensionApi() throws IOException, InterruptedException {
         // setup Connect worker properties
         Map workerProps = new HashMap<>();
         workerProps.put(REST_EXTENSION_CLASSES_CONFIG, IntegrationTestRestExtension.class.getName());
-  
+
         // build a Connect cluster backed by Kafka and Zk
         connect = new EmbeddedConnectCluster.Builder()
-                .name("connect-cluster")
-                .numWorkers(NUM_WORKERS)
-                .numBrokers(1)
-                .workerProps(workerProps)
-                .build();
-  
+            .name("connect-cluster")
+            .numWorkers(1)
+            .numBrokers(1)
+            .workerProps(workerProps)
+            .build();
+
         // start the clusters
         connect.start();
 
+        WorkerHandle worker = connect.workers().stream()
+            .findFirst()
+            .orElseThrow(() -> new AssertionError("At least one worker handle should be available"));
+
         waitForCondition(
             this::extensionIsRegistered,
             REST_EXTENSION_REGISTRATION_TIMEOUT_MS,
             "REST extension was never registered"
         );
+
+        ConnectorHandle connectorHandle = RuntimeHandles.get().connectorHandle("test-conn");
+        try {
+            // setup up props for the connector
+            Map connectorProps = new HashMap<>();
+            connectorProps.put(CONNECTOR_CLASS_CONFIG, MonitorableSinkConnector.class.getSimpleName());
+            connectorProps.put(TASKS_MAX_CONFIG, String.valueOf(1));
+            connectorProps.put(TOPICS_CONFIG, "test-topic");
+
+            // start a connector
+            connectorHandle.taskHandle(connectorHandle.name() + "-0");
+            StartAndStopLatch connectorStartLatch = connectorHandle.expectedStarts(1);
+            connect.configureConnector(connectorHandle.name(), connectorProps);
+            connectorStartLatch.await(CONNECTOR_HEALTH_AND_CONFIG_TIMEOUT_MS, TimeUnit.MILLISECONDS);
+
+            String workerId = String.format("%s:%d", worker.url().getHost(), worker.url().getPort());
+            ConnectorHealth expectedHealth = new ConnectorHealth(
+                connectorHandle.name(),
+                new ConnectorState(
+                    "RUNNING",
+                    workerId,
+                    null
+                ),
+                Collections.singletonMap(
+                    0,
+                    new TaskState(0, "RUNNING", workerId, null)
+                ),
+                ConnectorType.SINK
+            );
+
+            connectorProps.put(NAME_CONFIG, connectorHandle.name());
+
+            // Test the REST extension API; specifically, that the connector's health and configuration
+            // are available to the REST extension we registered and that they contain expected values
+            waitForCondition(
+                () -> verifyConnectorHealthAndConfig(connectorHandle.name(), expectedHealth, connectorProps),
+                CONNECTOR_HEALTH_AND_CONFIG_TIMEOUT_MS,
+                "Connector health and/or config was never accessible by the REST extension"
+            );
+        } finally {
+            RuntimeHandles.get().deleteConnector(connectorHandle.name());
+        }
     }
 
     @After
     public void close() {
         // stop all Connect, Kafka and Zk threads.
         connect.stop();
+        IntegrationTestRestExtension.instance = null;
     }
 
     private boolean extensionIsRegistered() {
@@ -85,11 +144,42 @@ private boolean extensionIsRegistered() {
         }
     }
 
+    private boolean verifyConnectorHealthAndConfig(
+        String connectorName,
+        ConnectorHealth expectedHealth,
+        Map expectedConfig
+    ) {
+        ConnectClusterState clusterState =
+            IntegrationTestRestExtension.instance.restPluginContext.clusterState();
+        
+        ConnectorHealth actualHealth = clusterState.connectorHealth(connectorName);
+        if (actualHealth.tasksState().isEmpty()) {
+            // Happens if the task has been started but its status has not yet been picked up from
+            // the status topic by the worker.
+            return false;
+        }
+        Map actualConfig = clusterState.connectorConfig(connectorName);
+
+        assertEquals(expectedConfig, actualConfig);
+        assertEquals(expectedHealth, actualHealth);
+
+        return true;
+    }
+
     public static class IntegrationTestRestExtension implements ConnectRestExtension {
+        private static IntegrationTestRestExtension instance;
+
+        public ConnectRestExtensionContext restPluginContext;
 
         @Override
         public void register(ConnectRestExtensionContext restPluginContext) {
+            instance = this;
+            this.restPluginContext = restPluginContext;
+            // Immediately request a list of connectors to confirm that the context and its fields
+            // has been fully initialized and there is no risk of deadlock
             restPluginContext.clusterState().connectors();
+            // Install a new REST resource that can be used to confirm that the extension has been
+            // successfully registered
             restPluginContext.configurable().register(new IntegrationTestRestExtensionResource());
         }
     

From 707c84695b4d1f1f7b890c9cc6a844222e4f9930 Mon Sep 17 00:00:00 2001
From: Chris Pettitt <53191309+cpettitt-confluent@users.noreply.github.com>
Date: Tue, 15 Oct 2019 18:18:58 -0600
Subject: [PATCH 0728/1071] MINOR: Provide better messages when waiting for a
 condition in test (#7488)

Reviewers: Boyang Chen , Matthias J. Sax , Bill Bejeck 
---
 build.gradle                                  |  1 +
 .../java/org/apache/kafka/test/TestUtils.java | 77 +++++++++++++++----
 .../apache/kafka/test/ValuelessCallable.java  | 25 ++++++
 .../OptimizedKTableIntegrationTest.java       | 21 +----
 .../utils/IntegrationTestUtils.java           | 63 ++++++++-------
 5 files changed, 128 insertions(+), 59 deletions(-)
 create mode 100644 clients/src/test/java/org/apache/kafka/test/ValuelessCallable.java

diff --git a/build.gradle b/build.gradle
index 65e6a69dbca5f..fbf58241ea0ac 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1257,6 +1257,7 @@ project(':streams:streams-scala') {
     testCompile libs.junit
     testCompile libs.scalatest
     testCompile libs.easymock
+    testCompile libs.hamcrest
 
     testRuntime libs.slf4jlog4j
   }
diff --git a/clients/src/test/java/org/apache/kafka/test/TestUtils.java b/clients/src/test/java/org/apache/kafka/test/TestUtils.java
index 2183e158af22c..57db41335cfd6 100644
--- a/clients/src/test/java/org/apache/kafka/test/TestUtils.java
+++ b/clients/src/test/java/org/apache/kafka/test/TestUtils.java
@@ -62,6 +62,7 @@
 import java.util.regex.Pattern;
 
 import static java.util.Arrays.asList;
+import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertThrows;
@@ -83,6 +84,7 @@ public class TestUtils {
     /* A consistent random number generator to make tests repeatable */
     public static final Random SEEDED_RANDOM = new Random(192348092834L);
     public static final Random RANDOM = new Random();
+    public static final long DEFAULT_POLL_INTERVAL_MS = 100;
     public static final long DEFAULT_MAX_WAIT_MS = 15000;
 
     public static Cluster singletonCluster() {
@@ -353,7 +355,7 @@ public static void waitForCondition(final TestCondition testCondition, final Sup
     public static void waitForCondition(final TestCondition testCondition, final long maxWaitMs, String conditionDetails) throws InterruptedException {
         waitForCondition(testCondition, maxWaitMs, () -> conditionDetails);
     }
-    
+
     /**
      * Wait for condition to be met for at most {@code maxWaitMs} and throw assertion failure otherwise.
      * This should be used instead of {@code Thread.sleep} whenever possible as it allows a longer timeout to be used
@@ -361,20 +363,69 @@ public static void waitForCondition(final TestCondition testCondition, final lon
      * avoid transient failures due to slow or overloaded machines.
      */
     public static void waitForCondition(final TestCondition testCondition, final long maxWaitMs, Supplier conditionDetailsSupplier) throws InterruptedException {
-        final long startTime = System.currentTimeMillis();
+        String conditionDetailsSupplied = conditionDetailsSupplier != null ? conditionDetailsSupplier.get() : null;
+        String conditionDetails = conditionDetailsSupplied != null ? conditionDetailsSupplied : "";
+        retryOnExceptionWithTimeout(maxWaitMs, () -> {
+            assertThat("Condition not met within timeout " + maxWaitMs + ". " + conditionDetails,
+                testCondition.conditionMet());
+        });
+    }
 
-        boolean testConditionMet;
-        while (!(testConditionMet = testCondition.conditionMet()) && ((System.currentTimeMillis() - startTime) < maxWaitMs)) {
-            Thread.sleep(Math.min(maxWaitMs, 100L));
-        }
+    /**
+     * Wait for the given runnable to complete successfully, i.e. throw now {@link Exception}s or
+     * {@link AssertionError}s, or for the given timeout to expire. If the timeout expires then the
+     * last exception or assertion failure will be thrown thus providing context for the failure.
+     *
+     * @param timeoutMs the total time in milliseconds to wait for {@code runnable} to complete successfully.
+     * @param runnable the code to attempt to execute successfully.
+     * @throws InterruptedException if the current thread is interrupted while waiting for {@code runnable} to complete successfully.
+     */
+    public static void retryOnExceptionWithTimeout(final long timeoutMs,
+                                                   final ValuelessCallable runnable) throws InterruptedException {
+        retryOnExceptionWithTimeout(DEFAULT_POLL_INTERVAL_MS, timeoutMs, runnable);
+    }
 
-        // don't re-evaluate testCondition.conditionMet() because this might slow down some tests significantly (this
-        // could be avoided by making the implementations more robust, but we have a large number of such implementations
-        // and it's easier to simply avoid the issue altogether)
-        if (!testConditionMet) {
-            String conditionDetailsSupplied = conditionDetailsSupplier != null ? conditionDetailsSupplier.get() : null;
-            String conditionDetails = conditionDetailsSupplied != null ? conditionDetailsSupplied : "";
-            throw new AssertionError("Condition not met within timeout " + maxWaitMs + ". " + conditionDetails);
+    /**
+     * Wait for the given runnable to complete successfully, i.e. throw now {@link Exception}s or
+     * {@link AssertionError}s, or for the default timeout to expire. If the timeout expires then the
+     * last exception or assertion failure will be thrown thus providing context for the failure.
+     *
+     * @param runnable the code to attempt to execute successfully.
+     * @throws InterruptedException if the current thread is interrupted while waiting for {@code runnable} to complete successfully.
+     */
+    public static void retryOnExceptionWithTimeout(final ValuelessCallable runnable) throws InterruptedException {
+        retryOnExceptionWithTimeout(DEFAULT_POLL_INTERVAL_MS, DEFAULT_MAX_WAIT_MS, runnable);
+    }
+
+    /**
+     * Wait for the given runnable to complete successfully, i.e. throw now {@link Exception}s or
+     * {@link AssertionError}s, or for the given timeout to expire. If the timeout expires then the
+     * last exception or assertion failure will be thrown thus providing context for the failure.
+     *
+     * @param pollIntervalMs the interval in milliseconds to wait between invoking {@code runnable}.
+     * @param timeoutMs the total time in milliseconds to wait for {@code runnable} to complete successfully.
+     * @param runnable the code to attempt to execute successfully.
+     * @throws InterruptedException if the current thread is interrupted while waiting for {@code runnable} to complete successfully.
+     */
+    public static void retryOnExceptionWithTimeout(final long pollIntervalMs,
+                                                   final long timeoutMs,
+                                                   final ValuelessCallable runnable) throws InterruptedException {
+        final long expectedEnd = System.currentTimeMillis() + timeoutMs;
+
+        while (true) {
+            try {
+                runnable.call();
+                return;
+            } catch (final AssertionError t) {
+                if (expectedEnd <= System.currentTimeMillis()) {
+                    throw t;
+                }
+            } catch (final Exception e) {
+                if (expectedEnd <= System.currentTimeMillis()) {
+                    throw new AssertionError(e);
+                }
+            }
+            Thread.sleep(Math.min(pollIntervalMs, timeoutMs));
         }
     }
 
diff --git a/clients/src/test/java/org/apache/kafka/test/ValuelessCallable.java b/clients/src/test/java/org/apache/kafka/test/ValuelessCallable.java
new file mode 100644
index 0000000000000..62863b3f65ffe
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/test/ValuelessCallable.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.test;
+
+/**
+ * Like a {@link Runnable} that allows exceptions to be thrown or a {@link java.util.concurrent.Callable}
+ * that does not return a value.
+ */
+public interface ValuelessCallable {
+    void call() throws Exception;
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/OptimizedKTableIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/OptimizedKTableIntegrationTest.java
index 22083c117ab43..767a5a9e0ff02 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/OptimizedKTableIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/OptimizedKTableIntegrationTest.java
@@ -189,7 +189,7 @@ public void shouldApplyUpdatesToStandbyStore() throws Exception {
 
         final ReadOnlyKeyValueStore newActiveStore =
             kafkaStreams1WasFirstActive ? store2 : store1;
-        retryOnExceptionWithTimeout(100, 60 * 1000, TimeUnit.MILLISECONDS, () -> {
+        TestUtils.retryOnExceptionWithTimeout(100, 60 * 1000, () -> {
             // Assert that after failover we have recovered to the last store write
             assertThat(newActiveStore.get(key), is(equalTo(batch1NumMessages - 1)));
         });
@@ -226,25 +226,6 @@ private void produceValueRange(final int key, final int start, final int endExcl
             mockTime);
     }
 
-    private void retryOnExceptionWithTimeout(final long pollInterval,
-                                             final long timeout,
-                                             final TimeUnit timeUnit,
-                                             final Runnable runnable) throws InterruptedException {
-        final long expectedEnd = System.currentTimeMillis() + timeUnit.toMillis(timeout);
-
-        while (true) {
-            try {
-                runnable.run();
-                return;
-            } catch (final Throwable t) {
-                if (expectedEnd <= System.currentTimeMillis()) {
-                    throw new AssertionError(t);
-                }
-                Thread.sleep(timeUnit.toMillis(pollInterval));
-            }
-        }
-    }
-
     private void waitForKafkaStreamssToEnterRunningState(final Collection kafkaStreamss,
                                                          final long time,
                                                          final TimeUnit timeUnit) throws InterruptedException {
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java b/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java
index fad89b7d70133..6c67a9df13133 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java
@@ -65,8 +65,11 @@
 import java.util.concurrent.Future;
 import java.util.stream.Collectors;
 
+import static org.apache.kafka.test.TestUtils.retryOnExceptionWithTimeout;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.is;
 
 /**
  * Utility functions to make integration testing more convenient.
@@ -449,15 +452,14 @@ public static  List> waitUntilMinRecordsReceived(fina
                                                                                 final int expectedNumRecords,
                                                                                 final long waitTime) throws InterruptedException {
         final List> accumData = new ArrayList<>();
+        final String reason = String.format("Did not receive all %d records from topic %s within %d ms", expectedNumRecords, topic, waitTime);
         try (final Consumer consumer = createConsumer(consumerConfig)) {
-            final TestCondition valuesRead = () -> {
+            retryOnExceptionWithTimeout(waitTime, () -> {
                 final List> readData =
                     readRecords(topic, consumer, waitTime, expectedNumRecords);
                 accumData.addAll(readData);
-                return accumData.size() >= expectedNumRecords;
-            };
-            final String conditionDetails = "Did not receive all " + expectedNumRecords + " records from topic " + topic;
-            TestUtils.waitForCondition(valuesRead, waitTime, conditionDetails);
+                assertThat(reason, accumData.size(), is(greaterThanOrEqualTo(expectedNumRecords)));
+            });
         }
         return accumData;
     }
@@ -495,15 +497,14 @@ public static  List> waitUntilMinKeyValueRecordsReceived(fi
                                                                                   final int expectedNumRecords,
                                                                                   final long waitTime) throws InterruptedException {
         final List> accumData = new ArrayList<>();
+        final String reason = String.format("Did not receive all %d records from topic %s within %d ms", expectedNumRecords, topic, waitTime);
         try (final Consumer consumer = createConsumer(consumerConfig)) {
-            final TestCondition valuesRead = () -> {
+            retryOnExceptionWithTimeout(waitTime, () -> {
                 final List> readData =
                     readKeyValues(topic, consumer, waitTime, expectedNumRecords);
                 accumData.addAll(readData);
-                return accumData.size() >= expectedNumRecords;
-            };
-            final String conditionDetails = "Did not receive all " + expectedNumRecords + " records from topic " + topic;
-            TestUtils.waitForCondition(valuesRead, waitTime, conditionDetails);
+                assertThat(reason, accumData.size(), is(greaterThanOrEqualTo(expectedNumRecords)));
+            });
         }
         return accumData;
     }
@@ -524,15 +525,14 @@ public static  List> waitUntilMinKeyValueWithTimes
                                                                                                                final int expectedNumRecords,
                                                                                                                final long waitTime) throws InterruptedException {
         final List> accumData = new ArrayList<>();
+        final String reason = String.format("Did not receive all %d records from topic %s within %d ms", expectedNumRecords, topic, waitTime);
         try (final Consumer consumer = createConsumer(consumerConfig)) {
-            final TestCondition valuesRead = () -> {
+            retryOnExceptionWithTimeout(waitTime, () -> {
                 final List> readData =
                     readKeyValuesWithTimestamp(topic, consumer, waitTime, expectedNumRecords);
                 accumData.addAll(readData);
-                return accumData.size() >= expectedNumRecords;
-            };
-            final String conditionDetails = "Did not receive all " + expectedNumRecords + " records from topic " + topic;
-            TestUtils.waitForCondition(valuesRead, waitTime, conditionDetails);
+                assertThat(reason, accumData.size(), is(greaterThanOrEqualTo(expectedNumRecords)));
+            });
         }
         return accumData;
     }
@@ -671,15 +671,14 @@ public static  List waitUntilMinValuesRecordsReceived(final Properties con
                                                                 final int expectedNumRecords,
                                                                 final long waitTime) throws InterruptedException {
         final List accumData = new ArrayList<>();
+        final String reason = String.format("Did not receive all %d records from topic %s within %d ms", expectedNumRecords, topic, waitTime);
         try (final Consumer consumer = createConsumer(consumerConfig)) {
-            final TestCondition valuesRead = () -> {
+            retryOnExceptionWithTimeout(waitTime, () -> {
                 final List readData =
                     readValues(topic, consumer, waitTime, expectedNumRecords);
                 accumData.addAll(readData);
-                return accumData.size() >= expectedNumRecords;
-            };
-            final String conditionDetails = "Did not receive all " + expectedNumRecords + " records from topic " + topic;
-            TestUtils.waitForCondition(valuesRead, waitTime, conditionDetails);
+                assertThat(reason, accumData.size(), is(greaterThanOrEqualTo(expectedNumRecords)));
+            });
         }
         return accumData;
     }
@@ -702,22 +701,34 @@ public static void waitUntilMetadataIsPropagated(final List servers
                                                      final String topic,
                                                      final int partition,
                                                      final long timeout) throws InterruptedException {
-        TestUtils.waitForCondition(() -> {
+        final String baseReason = String.format("Metadata for topic=%s partition=%d was not propagated to all brokers within %d ms. ",
+            topic, partition, timeout);
+
+        retryOnExceptionWithTimeout(timeout, () -> {
+            final List emptyPartitionInfos = new ArrayList<>();
+            final List invalidBrokerIds = new ArrayList<>();
+
             for (final KafkaServer server : servers) {
                 final MetadataCache metadataCache = server.dataPlaneRequestProcessor().metadataCache();
                 final Option partitionInfo =
-                        metadataCache.getPartitionInfo(topic, partition);
+                    metadataCache.getPartitionInfo(topic, partition);
+
                 if (partitionInfo.isEmpty()) {
-                    return false;
+                    emptyPartitionInfos.add(server);
+                    continue;
                 }
+
                 final UpdateMetadataPartitionState metadataPartitionState = partitionInfo.get();
                 if (!Request.isValidBrokerId(metadataPartitionState.leader())) {
-                    return false;
+                    invalidBrokerIds.add(server);
+                    continue;
                 }
             }
-            return true;
-        }, timeout, "metadata for topic=" + topic + " partition=" + partition + " not propagated to all brokers");
 
+            final String reason = baseReason + ". Brokers without partition info: " + emptyPartitionInfos +
+                ". Brokers with invalid broker id for partition leader: " + invalidBrokerIds;
+            assertThat(reason, emptyPartitionInfos.isEmpty() && invalidBrokerIds.isEmpty());
+        });
     }
 
     public static  void verifyKeyValueTimestamps(final Properties consumerConfig,

From c732b7e863b4a77051bedf5188d97977f1ae2862 Mon Sep 17 00:00:00 2001
From: Bruno Cadonna 
Date: Wed, 16 Oct 2019 07:39:28 +0200
Subject: [PATCH 0729/1071] KAFKA-8897: Warn about no guaranteed backwards
 compatibility in RocksDBConfigSetter (#7483)

Reviewer: A. Sophie Blee-Goldman , John Roesler , Matthias J. Sax 
---
 .../apache/kafka/streams/KafkaStreams.java    |  9 +++++
 ...ToDbOptionsColumnFamilyOptionsAdapter.java | 15 ++++++--
 .../kafka/streams/KafkaStreamsTest.java       | 30 +++++++++++++++-
 ...OptionsColumnFamilyOptionsAdapterTest.java | 35 +++++++++++++++++++
 4 files changed, 85 insertions(+), 4 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
index 146eefd966555..621bb91593920 100644
--- a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
+++ b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
@@ -60,6 +60,7 @@
 import org.apache.kafka.streams.state.StreamsMetadata;
 import org.apache.kafka.streams.state.internals.GlobalStateStoreProvider;
 import org.apache.kafka.streams.state.internals.QueryableStoreProvider;
+import org.apache.kafka.streams.state.internals.RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter;
 import org.apache.kafka.streams.state.internals.StateStoreProvider;
 import org.apache.kafka.streams.state.internals.StreamThreadStateStoreProvider;
 import org.apache.kafka.streams.state.internals.metrics.RocksDBMetricsRecordingTrigger;
@@ -773,6 +774,7 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder,
             return thread;
         });
 
+        maybeWarnAboutCodeInRocksDBConfigSetter(log, config);
         rocksDBMetricsRecordingService = maybeCreateRocksDBMetricsRecordingService(clientId, config);
     }
 
@@ -788,6 +790,13 @@ private static ScheduledExecutorService maybeCreateRocksDBMetricsRecordingServic
         return null;
     }
 
+    private static void maybeWarnAboutCodeInRocksDBConfigSetter(final Logger log,
+                                                                final StreamsConfig config) {
+        if (config.getClass(StreamsConfig.ROCKSDB_CONFIG_SETTER_CLASS_CONFIG) != null) {
+            RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.logWarning(log);
+        }
+    }
+
     private static HostInfo parseHostInfo(final String endPoint) {
         if (endPoint == null || endPoint.trim().isEmpty()) {
             return StreamsMetadataState.UNKNOWN_HOST;
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java
index 5b892a27bf523..3d9c1d3c0f0e5 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java
@@ -43,11 +43,11 @@
 import org.rocksdb.Statistics;
 import org.rocksdb.TableFormatConfig;
 import org.rocksdb.WALRecoveryMode;
+import org.rocksdb.WriteBufferManager;
+import org.slf4j.LoggerFactory;
 
 import java.util.Collection;
 import java.util.List;
-import org.rocksdb.WriteBufferManager;
-import org.slf4j.LoggerFactory;
 
 /**
  * The generic {@link Options} class allows users to set all configs on one object if only default column family
@@ -56,7 +56,7 @@
  *
  * This class do the translation between generic {@link Options} into {@link DBOptions} and {@link ColumnFamilyOptions}.
  */
-class RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter extends Options {
+public class RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter extends Options {
     private final DBOptions dbOptions;
     private final ColumnFamilyOptions columnFamilyOptions;
 
@@ -1341,15 +1341,24 @@ public CompactionOptionsUniversal compactionOptionsUniversal() {
 
     @Override
     public Options setCompactionOptionsFIFO(final CompactionOptionsFIFO compactionOptionsFIFO) {
+        logWarning(LOG);
         columnFamilyOptions.setCompactionOptionsFIFO(compactionOptionsFIFO);
         return this;
     }
 
     @Override
     public CompactionOptionsFIFO compactionOptionsFIFO() {
+        logWarning(LOG);
         return columnFamilyOptions.compactionOptionsFIFO();
     }
 
+    public static void logWarning(final org.slf4j.Logger log) {
+        log.warn("RocksDB's version will be bumped to version 6+ via KAFKA-8897 in a future release. "
+            + "If you use `org.rocksdb.CompactionOptionsFIFO#setTtl(long)` or `#ttl()` you will need to rewrite "
+            + "your code after KAFKA-8897 is resolved and set TTL via `org.rocksdb.Options` "
+            + "(or `org.rocksdb.ColumnFamilyOptions`).");
+    }
+
     @Override
     public Options setForceConsistencyChecks(final boolean forceConsistencyChecks) {
         columnFamilyOptions.setForceConsistencyChecks(forceConsistencyChecks);
diff --git a/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java b/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java
index d4850634a9129..cd24685165a07 100644
--- a/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java
@@ -39,7 +39,9 @@
 import org.apache.kafka.streams.processor.internals.StreamThread;
 import org.apache.kafka.streams.processor.internals.StreamsMetadataState;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender;
 import org.apache.kafka.streams.state.KeyValueStore;
+import org.apache.kafka.streams.state.RocksDBConfigSetter;
 import org.apache.kafka.streams.state.StoreBuilder;
 import org.apache.kafka.streams.state.Stores;
 import org.apache.kafka.streams.state.internals.metrics.RocksDBMetricsRecordingTrigger;
@@ -79,6 +81,8 @@
 import static org.easymock.EasyMock.anyLong;
 import static org.easymock.EasyMock.anyObject;
 import static org.easymock.EasyMock.anyString;
+import static org.hamcrest.CoreMatchers.hasItem;
+import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
@@ -91,6 +95,7 @@ public class KafkaStreamsTest {
 
     private static final int NUM_THREADS = 2;
     private final static String APPLICATION_ID = "appId";
+    private final static String CLIENT_ID = "test-client";
 
     @Rule
     public TestName testName = new TestName();
@@ -143,7 +148,7 @@ public void before() throws Exception {
 
         props = new Properties();
         props.put(StreamsConfig.APPLICATION_ID_CONFIG, APPLICATION_ID);
-        props.put(StreamsConfig.CLIENT_ID_CONFIG, "clientId");
+        props.put(StreamsConfig.CLIENT_ID_CONFIG, CLIENT_ID);
         props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:2018");
         props.put(StreamsConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName());
         props.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath());
@@ -745,6 +750,29 @@ public void shouldNotTriggerRecordingOfRocksDBMetricsIfRecordingLevelIsInfo() {
         PowerMock.verify(Executors.class, rocksDBMetricsRecordingTriggerThread);
     }
 
+    public static class TestRocksDbConfigSetter implements RocksDBConfigSetter {
+        @Override
+        public void setConfig(final String storeName,
+                              final org.rocksdb.Options options,
+                              final Map configs) {
+        }
+    }
+
+    @Test
+    public void shouldWarnAboutRocksDBConfigSetterIsNotGuaranteedToBeBackwardsCompatible() {
+        props.setProperty(StreamsConfig.ROCKSDB_CONFIG_SETTER_CLASS_CONFIG, TestRocksDbConfigSetter.class.getName());
+
+        final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
+        new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        LogCaptureAppender.unregister(appender);
+
+        assertThat(appender.getMessages(), hasItem("stream-client [" + CLIENT_ID + "] "
+            + "RocksDB's version will be bumped to version 6+ via KAFKA-8897 in a future release. "
+            + "If you use `org.rocksdb.CompactionOptionsFIFO#setTtl(long)` or `#ttl()` you will need to rewrite "
+            + "your code after KAFKA-8897 is resolved and set TTL via `org.rocksdb.Options` "
+            + "(or `org.rocksdb.ColumnFamilyOptions`)."));
+    }
+
     @Test
     public void shouldCleanupOldStateDirs() throws Exception {
         PowerMock.mockStatic(Executors.class);
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapterTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapterTest.java
index 62513a52269dc..d07aee0807039 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapterTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapterTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.streams.state.internals;
 
+import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender;
 import org.easymock.EasyMockRunner;
 import org.easymock.Mock;
 import org.junit.Test;
@@ -26,6 +27,7 @@
 import org.rocksdb.AccessHint;
 import org.rocksdb.BuiltinComparator;
 import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.CompactionOptionsFIFO;
 import org.rocksdb.CompactionPriority;
 import org.rocksdb.CompactionStyle;
 import org.rocksdb.ComparatorOptions;
@@ -56,6 +58,7 @@
 import static org.easymock.EasyMock.replay;
 import static org.easymock.EasyMock.reset;
 import static org.easymock.EasyMock.verify;
+import static org.hamcrest.CoreMatchers.hasItem;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.matchesPattern;
@@ -109,6 +112,38 @@ public void shouldForwardAllDbOptionsCalls() throws Exception {
         }
     }
 
+    @Test
+    public void shouldWarnThanMethodCompactionOptionsFIFOSetTtlWillBeRemoved() {
+        final RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter optionsFacadeDbOptions
+            = new RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter(dbOptions, new ColumnFamilyOptions());
+
+        final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
+        optionsFacadeDbOptions.setCompactionOptionsFIFO(new CompactionOptionsFIFO());
+        LogCaptureAppender.unregister(appender);
+
+        assertThat(appender.getMessages(), hasItem(""
+            + "RocksDB's version will be bumped to version 6+ via KAFKA-8897 in a future release. "
+            + "If you use `org.rocksdb.CompactionOptionsFIFO#setTtl(long)` or `#ttl()` you will need to rewrite "
+            + "your code after KAFKA-8897 is resolved and set TTL via `org.rocksdb.Options` "
+            + "(or `org.rocksdb.ColumnFamilyOptions`)."));
+    }
+
+    @Test
+    public void shouldWarnThanMethodCompactionOptionsFIFOTtlWillBeRemoved() {
+        final RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter optionsFacadeDbOptions
+            = new RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter(dbOptions, new ColumnFamilyOptions());
+
+        final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
+        optionsFacadeDbOptions.compactionOptionsFIFO();
+        LogCaptureAppender.unregister(appender);
+
+        assertThat(appender.getMessages(), hasItem(""
+            + "RocksDB's version will be bumped to version 6+ via KAFKA-8897 in a future release. "
+            + "If you use `org.rocksdb.CompactionOptionsFIFO#setTtl(long)` or `#ttl()` you will need to rewrite "
+            + "your code after KAFKA-8897 is resolved and set TTL via `org.rocksdb.Options` "
+            + "(or `org.rocksdb.ColumnFamilyOptions`)."));
+    }
+
     private void verifyDBOptionsMethodCall(final Method method) throws Exception {
         final RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter optionsFacadeDbOptions
             = new RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter(dbOptions, new ColumnFamilyOptions());

From f2495e6fc81a4bfcc9ce0908643fb1aa63d3dcd3 Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Wed, 16 Oct 2019 11:34:52 -0500
Subject: [PATCH 0730/1071] KAFKA-9032: Bypass serdes for tombstones (#7518)

In a KTable context, null record values have a special "tombstone" significance. We should always bypass the serdes for such tombstones, since otherwise the serde could violate Streams' table semantics.

Added test coverage for this case and fixed the code accordingly.

Reviewers: A. Sophie Blee-Goldman , Matthias J. Sax , Guozhang Wang , Bill Bejeck 
---
 .../SubscriptionResponseWrapperSerde.java     | 12 ++--
 .../SubscriptionResponseWrapperSerdeTest.java | 63 ++++++++++++++++---
 2 files changed, 60 insertions(+), 15 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapperSerde.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapperSerde.java
index 6524b4fc7647a..4277f9a8fd74a 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapperSerde.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapperSerde.java
@@ -58,7 +58,7 @@ public byte[] serialize(final String topic, final SubscriptionResponseWrapper
                 throw new UnsupportedVersionException("SubscriptionResponseWrapper version is larger than maximum supported 0x7F");
             }
 
-            final byte[] serializedData = serializer.serialize(topic, data.getForeignValue());
+            final byte[] serializedData = data.getForeignValue() == null ? null : serializer.serialize(topic, data.getForeignValue());
             final int serializedDataLength = serializedData == null ? 0 : serializedData.length;
             final long[] originalHash = data.getOriginalValueHash();
             final int hashLength = originalHash == null ? 0 : 2 * Long.BYTES;
@@ -108,14 +108,16 @@ public SubscriptionResponseWrapper deserialize(final String topic, final byte
                 lengthSum += 2 * Long.BYTES;
             }
 
-            final byte[] serializedValue;
+            final V value;
             if (data.length - lengthSum > 0) {
+                final byte[] serializedValue;
                 serializedValue = new byte[data.length - lengthSum];
                 buf.get(serializedValue, 0, serializedValue.length);
-            } else
-                serializedValue = null;
+                value = deserializer.deserialize(topic, serializedValue);
+            } else {
+                value = null;
+            }
 
-            final V value = deserializer.deserialize(topic, serializedValue);
             return new SubscriptionResponseWrapper<>(hash, value, version);
         }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapperSerdeTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapperSerdeTest.java
index fde9bddfac37d..40948e33a2f2b 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapperSerdeTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapperSerdeTest.java
@@ -17,15 +17,58 @@
 package org.apache.kafka.streams.kstream.internals.foreignkeyjoin;
 
 import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.serialization.Deserializer;
+import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.Serializer;
 import org.apache.kafka.streams.state.internals.Murmur3;
 import org.junit.Test;
 
-import static org.junit.Assert.assertEquals;
+import java.util.Map;
+
+import static java.util.Objects.requireNonNull;
 import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 
 public class SubscriptionResponseWrapperSerdeTest {
+    private static final class NonNullableSerde implements Serde, Serializer, Deserializer {
+        private final Serde delegate;
+
+        NonNullableSerde(final Serde delegate) {
+            this.delegate = delegate;
+        }
+
+        @Override
+        public void configure(final Map configs, final boolean isKey) {
+
+        }
+
+        @Override
+        public void close() {
+
+        }
+
+        @Override
+        public Serializer serializer() {
+            return this;
+        }
+
+        @Override
+        public Deserializer deserializer() {
+            return this;
+        }
+
+        @Override
+        public byte[] serialize(final String topic, final T data) {
+            return delegate.serializer().serialize(topic, requireNonNull(data));
+        }
+
+        @Override
+        public T deserialize(final String topic, final byte[] data) {
+            return delegate.deserializer().deserialize(topic, requireNonNull(data));
+        }
+    }
 
     @Test
     @SuppressWarnings("unchecked")
@@ -33,9 +76,9 @@ public void ShouldSerdeWithNonNullsTest() {
         final long[] hashedValue = Murmur3.hash128(new byte[] {(byte) 0x01, (byte) 0x9A, (byte) 0xFF, (byte) 0x00});
         final String foreignValue = "foreignValue";
         final SubscriptionResponseWrapper srw = new SubscriptionResponseWrapper<>(hashedValue, foreignValue);
-        final SubscriptionResponseWrapperSerde srwSerde = new SubscriptionResponseWrapperSerde(Serdes.String());
+        final SubscriptionResponseWrapperSerde srwSerde = new SubscriptionResponseWrapperSerde(new NonNullableSerde(Serdes.String()));
         final byte[] serResponse = srwSerde.serializer().serialize(null, srw);
-        final SubscriptionResponseWrapper result = (SubscriptionResponseWrapper) srwSerde.deserializer().deserialize(null, serResponse);
+        final SubscriptionResponseWrapper result = srwSerde.deserializer().deserialize(null, serResponse);
 
         assertArrayEquals(hashedValue, result.getOriginalValueHash());
         assertEquals(foreignValue, result.getForeignValue());
@@ -46,9 +89,9 @@ public void ShouldSerdeWithNonNullsTest() {
     public void shouldSerdeWithNullForeignValueTest() {
         final long[] hashedValue = Murmur3.hash128(new byte[] {(byte) 0x01, (byte) 0x9A, (byte) 0xFF, (byte) 0x00});
         final SubscriptionResponseWrapper srw = new SubscriptionResponseWrapper<>(hashedValue, null);
-        final SubscriptionResponseWrapperSerde srwSerde = new SubscriptionResponseWrapperSerde(Serdes.String());
+        final SubscriptionResponseWrapperSerde srwSerde = new SubscriptionResponseWrapperSerde(new NonNullableSerde(Serdes.String()));
         final byte[] serResponse = srwSerde.serializer().serialize(null, srw);
-        final SubscriptionResponseWrapper result = (SubscriptionResponseWrapper) srwSerde.deserializer().deserialize(null, serResponse);
+        final SubscriptionResponseWrapper result = srwSerde.deserializer().deserialize(null, serResponse);
 
         assertArrayEquals(hashedValue, result.getOriginalValueHash());
         assertNull(result.getForeignValue());
@@ -60,9 +103,9 @@ public void shouldSerdeWithNullHashTest() {
         final long[] hashedValue = null;
         final String foreignValue = "foreignValue";
         final SubscriptionResponseWrapper srw = new SubscriptionResponseWrapper<>(hashedValue, foreignValue);
-        final SubscriptionResponseWrapperSerde srwSerde = new SubscriptionResponseWrapperSerde(Serdes.String());
+        final SubscriptionResponseWrapperSerde srwSerde = new SubscriptionResponseWrapperSerde(new NonNullableSerde(Serdes.String()));
         final byte[] serResponse = srwSerde.serializer().serialize(null, srw);
-        final SubscriptionResponseWrapper result = (SubscriptionResponseWrapper) srwSerde.deserializer().deserialize(null, serResponse);
+        final SubscriptionResponseWrapper result = srwSerde.deserializer().deserialize(null, serResponse);
 
         assertArrayEquals(hashedValue, result.getOriginalValueHash());
         assertEquals(foreignValue, result.getForeignValue());
@@ -74,15 +117,15 @@ public void shouldSerdeWithNullsTest() {
         final long[] hashedValue = null;
         final String foreignValue = null;
         final SubscriptionResponseWrapper srw = new SubscriptionResponseWrapper<>(hashedValue, foreignValue);
-        final SubscriptionResponseWrapperSerde srwSerde = new SubscriptionResponseWrapperSerde(Serdes.String());
+        final SubscriptionResponseWrapperSerde srwSerde = new SubscriptionResponseWrapperSerde(new NonNullableSerde(Serdes.String()));
         final byte[] serResponse = srwSerde.serializer().serialize(null, srw);
-        final SubscriptionResponseWrapper result = (SubscriptionResponseWrapper) srwSerde.deserializer().deserialize(null, serResponse);
+        final SubscriptionResponseWrapper result = srwSerde.deserializer().deserialize(null, serResponse);
 
         assertArrayEquals(hashedValue, result.getOriginalValueHash());
         assertEquals(foreignValue, result.getForeignValue());
     }
 
-    @Test (expected = UnsupportedVersionException.class)
+    @Test(expected = UnsupportedVersionException.class)
     @SuppressWarnings("unchecked")
     public void shouldThrowExceptionWithBadVersionTest() {
         final long[] hashedValue = null;

From b29c9f2468a9e1e9dddc1baf2864d4a67f9e55d1 Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Wed, 16 Oct 2019 11:53:36 -0700
Subject: [PATCH 0731/1071] MINOR: Update Kafka Streams upgrade docs for
 KIP-444, KIP-470, KIP-471, KIP-474, KIP-528 (#7515)

Reviewers: Jukka Karvanen , Guozhang Wang , A. Sophie Blee-Goldman 
---
 docs/streams/upgrade-guide.html | 53 +++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/docs/streams/upgrade-guide.html b/docs/streams/upgrade-guide.html
index bad37c70541f6..1cc5b505955dc 100644
--- a/docs/streams/upgrade-guide.html
+++ b/docs/streams/upgrade-guide.html
@@ -72,6 +72,59 @@ 
                Upgrade Guide and API Changes
                
         More details about the new config StreamsConfig#TOPOLOGY_OPTIMIZATION can be found in KIP-295.
     
                
 
+    
                Streams API changes in 2.4.0
                
+
+    
+
+    
+    
+
+    
+
+    
                
+        The 2.4.0 release contains newly added and reworked metrics.
+        KIP-444
+        adds new client level (i.e., KafkaStreams instance level) metrics to the existing
+        thread-level, task-level, and processor-/state-store-level metrics.
+        For a full list of available client level metrics, see the
+        KafkaStreams monitoring
+        section in the operations guide.
+        
                
+        Furthermore, RocksDB metrics are exposed via
+        KIP-471.
+        For a full list of available RocksDB metrics, see the
+        RocksDB monitoring
+        section in the operations guide.
+    
                
+
+    
                
+        Kafka Streams test-utils got improved via
+        KIP-470
+        to simplify the process of using TopologyTestDriver to test your application code.
+        We deprecated ConsumerRecordFactory, TopologyTestDriver#pipeInput(),
+        OutputVerifier, as well as TopologyTestDriver#readOutput() and replace them with
+        TestInputTopic and TestOutputTopic, respectively.
+        We also introduced a new class TestRecord that simplifies assertion code.
+        For full details see the
+        Testing section in the developer guide.
+    
                
+
+    
                
+        In 2.4.0, we deprecated WindowStore#put(K key, V value) that should never be used.
+        Instead the existing WindowStore#put(K key, V value, long windowStartTimestamp) should be used
+        (KIP-474).
+    
                
+
+    
                
+        Furthermore, the PartitionGrouper interface and its corresponding configuration parameter
+        partition.grouper were deprecated
+        (KIP-528)
+        and will be removed in the next major release (KAFKA-7785.
+        Hence, this feature won't be supported in the future any longer and you need to updated your code accordingly.
+        If you use a custom PartitionGrouper and stop to use it, the created tasks might change.
+        Hence, you will need to reset your application to upgrade it.
+    
                
+
     
                Streams API changes in 2.3.0
                
 
     
                Version 2.3.0 adds the Suppress operator to the kafka-streams-scala Ktable API.
                

From ee4f65dfa5bd2a14e17805734be3d9e1fdfc25a2 Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Wed, 16 Oct 2019 16:09:09 -0700
Subject: [PATCH 0732/1071] Explain the separate  upgrade paths for consumer
 groups and Streams (#7516)

Document the upgrade path for the consumer and for Streams (note that they differ significantly).

Needs to be cherry-picked to 2.4

Reviewers: Guozhang Wang 
---
 docs/streams/upgrade-guide.html | 17 +++++++++++------
 docs/upgrade.html               | 25 +++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/docs/streams/upgrade-guide.html b/docs/streams/upgrade-guide.html
index 1cc5b505955dc..2de79172a40a7 100644
--- a/docs/streams/upgrade-guide.html
+++ b/docs/streams/upgrade-guide.html
@@ -34,10 +34,10 @@ 
                Upgrade Guide and API Changes
                
     
 
     
                
-        Upgrading from any older version to {{fullDotVersion}} is possible: (1) if you are upgrading from 2.0.x to {{fullDotVersion}} then a single rolling bounce is needed to swap in the new jar,
-        (2) if you are upgrading from older versions than 2.0.x in the online mode, you would need two rolling bounces where
-        the first rolling bounce phase you need to set config upgrade.from="older version" (possible values are "0.10.0", "0.10.1", "0.10.2", "0.11.0", "1.0", and "1.1")
-        (cf. KIP-268):
+        Upgrading from any older version to {{fullDotVersion}} is possible: you will need to do two rolling bounces, where during the first rolling bounce phase you set the config upgrade.from="older version"
+        (possible values are "0.10.0" - "2.3") and during the second you remove it. This is required to safely upgrade to the new cooperative rebalancing protocol of the embedded consumer. Note that you will remain using the old eager
+        rebalancing protocol if you skip or delay the second rolling bounce, but you can safely switch over to cooperative at any time once the entire group is on 2.4+ by removing the config value and bouncing. For more details please refer to
+        KIP-429:
     
                
     
                  
         
                •  prepare your application instances for a rolling bounce and make sure that config upgrade.from is set to the version from which it is being upgrade.
                  • 
@@ -75,11 +75,16 @@ 
                  Upgrade Guide and API Changes
                  
     
                  Streams API changes in 2.4.0
                  
 
     
-
     
     
 
-    
+    
                  
+        With the introduction of incremental cooperative rebalancing, Streams no longer requires all tasks be revoked at the beginning of a rebalance. Instead, at the completion of the rebalance only those tasks which are to be migrated to another consumer
+        for overall load balance will need to be closed and revoked. This changes the semantics of the StateListener a bit, as it will not necessarily transition to REBALANCING at the beginning of a rebalance anymore. Note that
+        this means IQ will now be available at all times except during state restoration, including while a rebalance is in progress. If restoration is occurring when a rebalance begins, we will continue to actively restore the state stores and/or process
+        standby tasks during a cooperative rebalance. Note that with this new rebalancing protocol, you may sometimes see a rebalance be followed by a second short rebalance that ensures all tasks are safely distributed. For details on please see
+        KIP-429.
+    
                  
 
     
                  
         The 2.4.0 release contains newly added and reworked metrics.
diff --git a/docs/upgrade.html b/docs/upgrade.html
index 0c69336127ca5..6d9f7020b5d9f 100644
--- a/docs/upgrade.html
+++ b/docs/upgrade.html
@@ -45,6 +45,31 @@ 
                  Notable changes in 2
         The old overloaded functions are deprecated and we would recommend users to make their code changes to leverage the new methods (details
         can be found in KIP-520).
     
+    
                • We are introducing incremental cooperative rebalancing to the clients' group protocol, which allows consumers to keep all of their assigned partitions during a rebalance
+        and at the end revoke only those which must be migrated to another consumer for overall cluster balance. The ConsumerCoordinator will choose the latest RebalanceProtocol
+        that is commonly supported by all of the consumer's supported assignors. You can use the new built-in CooperativeStickyAssignor or plug in your own custom cooperative assignor. To do
+        so you must implement the ConsumerPartitionAssignor interface and include RebalanceProtocol.COOPERATIVE in the list returned by ConsumerPartitionAssignor#supportedProtocols.
+        Your custom assignor can then leverage the ownedPartitions field in each consumer's Subscription to give partitions back to their previous owners whenever possible. Note that when
+        a partition is to be reassigned to another consumer, it must be removed from the new assignment until it has been revoked from its original owner. Any consumer that has to revoke a partition will trigger
+        a followup rebalance to allow the revoked partition to safely be assigned to its new owner. See the
+        ConsumerPartitionAssignor RebalanceProtocol javadocs for more information.
+        
                  
+        To upgrade from the old (eager) protocol, which always revokes all partitions before rebalancing, to cooperative rebalancing, you must follow a specific upgrade path to get all clients on the same ConsumerPartitionAssignor
+        that supports the cooperative protocol. This can be done with two rolling bounces, using the CooperativeStickyAssignor for the example: during the first one, add "cooperative-sticky" to the list of supported assignors
+        for each member (without removing the previous assignor -- note that if previously using the default, you must include that explicitly as well). You then bounce and/or upgrade it.
+        Once the entire group is on 2.4+ and all members have the "cooperative-sticky" among their supported assignors, remove the other assignor(s) and perform a second rolling bounce so that by the end all members support only the
+        cooperative protocol. For further details on the cooperative rebalancing protocol and upgrade path, see KIP-429.
+    
                  • 
+    
                • There are some behavioral changes to the ConsumerRebalanceListener, as well as a new API. Exceptions thrown during any of the listener's three callbacks will no longer be swallowed, and will instead be re-thrown
+        all the way up to the Consumer.poll() call. The onPartitionsLost method has been added to allow users to react to abnormal circumstances where a consumer may have lost ownership of its partitions
+        (such as a missed rebalance) and cannot commit offsets. By default, this will simply call the existing onPartitionsRevoked API to align with previous behavior. Note however that onPartitionsLost will not
+        be called when the set of lost partitions is empty. This means that no callback will be invoked at the beginning of the first rebalance of a new consumer joining the group.
+        
                  
+        The semantics of the ConsumerRebalanceListener's callbacks are further changed when following the cooperative rebalancing protocol described above. In addition to onPartitionsLost, onPartitionsRevoked
+        will also never be called when the set of revoked partitions is empty. The callback will generally be invoked only at the end of a rebalance, and only on the set of partitions that are being moved to another consumer. The
+        onPartitionsAssigned callback will however always be called, even with an empty set of partitions, as a way to notify users of a rebalance event (this is true for both cooperative and eager). For details on
+        the new callback semantics, see the ConsumerRebalanceListener javadocs.
+    
                  • 
 
                
 
 
                Upgrading from 0.8.x, 0.9.x, 0.10.0.x, 0.10.1.x, 0.10.2.x, 0.11.0.x, 1.0.x, 1.1.x, 2.0.x or 2.1.x or 2.2.x to 2.3.0
                

From 0299993c09ac4da2f43e58f28a60ec1fd1937a16 Mon Sep 17 00:00:00 2001
From: Viktor Somogyi 
Date: Fri, 11 Oct 2019 19:53:32 +0200
Subject: [PATCH 0733/1071] KAFKA-7981; Add fetcher and log cleaner thread
 count metrics (#6514)

This patch adds metrics for failed threads as documented in KIP-434: https://cwiki.apache.org/confluence/display/KAFKA/KIP-434%3A+Add+Replica+Fetcher+and+Log+Cleaner+Count+Metrics.

Reviewers: Stanislav Kozlovski , Jason Gustafson 
---
 .../src/main/scala/kafka/log/LogCleaner.scala | 12 +++-
 .../kafka/server/AbstractFetcherManager.scala | 10 +++
 .../kafka/utils/ShutdownableThread.scala      | 19 ++++--
 .../kafka/log/LogCleanerIntegrationTest.scala | 64 ++++++++++++++-----
 .../server/AbstractFetcherManagerTest.scala   | 34 ++++++++++
 5 files changed, 113 insertions(+), 26 deletions(-)

diff --git a/core/src/main/scala/kafka/log/LogCleaner.scala b/core/src/main/scala/kafka/log/LogCleaner.scala
index bcb0e72007f45..219f49716f065 100644
--- a/core/src/main/scala/kafka/log/LogCleaner.scala
+++ b/core/src/main/scala/kafka/log/LogCleaner.scala
@@ -110,8 +110,7 @@ class LogCleaner(initialConfig: CleanerConfig,
                                         "bytes",
                                         time = time)
 
-  /* the threads */
-  private val cleaners = mutable.ArrayBuffer[CleanerThread]()
+  private[log] val cleaners = mutable.ArrayBuffer[CleanerThread]()
 
   /* a metric to track the maximum utilization of any thread's buffer in the last cleaning */
   newGauge("max-buffer-utilization-percent",
@@ -139,6 +138,13 @@ class LogCleaner(initialConfig: CleanerConfig,
           def value: Int = Math.max(0, (cleaners.map(_.lastPreCleanStats).map(_.maxCompactionDelayMs).max / 1000).toInt)
           })
 
+  newGauge("DeadThreadCount",
+    new Gauge[Int] {
+      def value: Int = deadThreadCount
+    })
+
+  private[log] def deadThreadCount: Int = cleaners.count(_.isThreadFailed)
+
   /**
    * Start the background cleaning
    */
@@ -272,7 +278,7 @@ class LogCleaner(initialConfig: CleanerConfig,
    * The cleaner threads do the actual log cleaning. Each thread processes does its cleaning repeatedly by
    * choosing the dirtiest log, cleaning it, and then swapping in the cleaned segments.
    */
-  private class CleanerThread(threadId: Int)
+  private[log] class CleanerThread(threadId: Int)
     extends ShutdownableThread(name = s"kafka-log-cleaner-thread-$threadId", isInterruptible = false) {
 
     protected override def loggerName = classOf[LogCleaner].getName
diff --git a/core/src/main/scala/kafka/server/AbstractFetcherManager.scala b/core/src/main/scala/kafka/server/AbstractFetcherManager.scala
index d76fbfc0acb8b..49451def01025 100755
--- a/core/src/main/scala/kafka/server/AbstractFetcherManager.scala
+++ b/core/src/main/scala/kafka/server/AbstractFetcherManager.scala
@@ -76,6 +76,16 @@ abstract class AbstractFetcherManager[T <: AbstractFetcherThread](val name: Stri
     Map("clientId" -> clientId)
   )
 
+  newGauge("DeadThreadCount", {
+    new Gauge[Int] {
+      def value: Int = {
+        deadThreadCount
+      }
+    }
+  }, Map("clientId" -> clientId))
+
+  private[server] def deadThreadCount: Int = lock synchronized { fetcherThreadMap.values.count(_.isThreadFailed) }
+
   def resizeThreadPool(newSize: Int): Unit = {
     def migratePartitions(newSize: Int): Unit = {
       fetcherThreadMap.foreach { case (id, thread) =>
diff --git a/core/src/main/scala/kafka/utils/ShutdownableThread.scala b/core/src/main/scala/kafka/utils/ShutdownableThread.scala
index 02d09dab96d82..0ca21c4ab166d 100644
--- a/core/src/main/scala/kafka/utils/ShutdownableThread.scala
+++ b/core/src/main/scala/kafka/utils/ShutdownableThread.scala
@@ -34,9 +34,16 @@ abstract class ShutdownableThread(val name: String, val isInterruptible: Boolean
     awaitShutdown()
   }
 
-  def isShutdownComplete: Boolean = {
-    shutdownComplete.getCount == 0
-  }
+  def isShutdownInitiated: Boolean = shutdownInitiated.getCount == 0
+
+  def isShutdownComplete: Boolean = shutdownComplete.getCount == 0
+
+  /**
+    * @return true if there has been an unexpected error and the thread shut down
+    */
+  // mind that run() might set both when we're shutting down the broker
+  // but the return value of this function at that point wouldn't matter
+  def isThreadFailed: Boolean = isShutdownComplete && !isShutdownInitiated
 
   def initiateShutdown(): Boolean = {
     this.synchronized {
@@ -55,7 +62,7 @@ abstract class ShutdownableThread(val name: String, val isInterruptible: Boolean
    * After calling initiateShutdown(), use this API to wait until the shutdown is complete
    */
   def awaitShutdown(): Unit = {
-    if (shutdownInitiated.getCount != 0)
+    if (!isShutdownInitiated)
       throw new IllegalStateException("initiateShutdown() was not called before awaitShutdown()")
     else {
       if (isStarted)
@@ -102,7 +109,5 @@ abstract class ShutdownableThread(val name: String, val isInterruptible: Boolean
     info("Stopped")
   }
 
-  def isRunning: Boolean = {
-    shutdownInitiated.getCount() != 0
-  }
+  def isRunning: Boolean = !isShutdownInitiated
 }
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerIntegrationTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerIntegrationTest.scala
index df6df0bd4ed11..56975075e0d30 100644
--- a/core/src/test/scala/unit/kafka/log/LogCleanerIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerIntegrationTest.scala
@@ -20,21 +20,22 @@ package kafka.log
 import java.io.PrintWriter
 
 import com.yammer.metrics.Metrics
-import com.yammer.metrics.core.Gauge
+import com.yammer.metrics.core.{Gauge, MetricName}
+import kafka.metrics.KafkaMetricsGroup
 import kafka.utils.{MockTime, TestUtils}
 import org.apache.kafka.common.TopicPartition
-import org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS
 import org.apache.kafka.common.record.{CompressionType, RecordBatch}
+import org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS
 import org.junit.Assert._
 import org.junit.Test
 
-import scala.collection.{Iterable, JavaConverters, Seq}
 import scala.collection.JavaConverters.mapAsScalaMapConverter
+import scala.collection.{Iterable, JavaConverters, Seq}
 
 /**
   * This is an integration test that tests the fully integrated log cleaner
   */
-class LogCleanerIntegrationTest extends AbstractLogCleanerIntegrationTest {
+class LogCleanerIntegrationTest extends AbstractLogCleanerIntegrationTest with KafkaMetricsGroup {
 
   val codec: CompressionType = CompressionType.LZ4
 
@@ -60,18 +61,6 @@ class LogCleanerIntegrationTest extends AbstractLogCleanerIntegrationTest {
       writeDups(numKeys = 20, numDups = 3, log = log, codec = codec)
     }
 
-    def getGauge[T](metricName: String, metricScope: String): Gauge[T] = {
-      Metrics.defaultRegistry.allMetrics.asScala
-        .filterKeys(k => {
-          k.getName.endsWith(metricName) && k.getScope.endsWith(metricScope)
-        })
-        .headOption
-        .getOrElse { fail(s"Unable to find metric $metricName") }
-        .asInstanceOf[(Any, Gauge[Any])]
-        ._2
-        .asInstanceOf[Gauge[T]]
-    }
-
     breakPartitionLog(topicPartitions(0))
     breakPartitionLog(topicPartitions(1))
 
@@ -95,6 +84,24 @@ class LogCleanerIntegrationTest extends AbstractLogCleanerIntegrationTest {
     assertFalse(uncleanablePartitions.contains(topicPartitions(2)))
   }
 
+  private def getGauge[T](filter: MetricName => Boolean): Gauge[T] = {
+    Metrics.defaultRegistry.allMetrics.asScala
+      .filterKeys(filter(_))
+      .headOption
+      .getOrElse { fail(s"Unable to find metric") }
+      .asInstanceOf[(Any, Gauge[Any])]
+      ._2
+      .asInstanceOf[Gauge[T]]
+  }
+
+  private def getGauge[T](metricName: String): Gauge[T] = {
+    getGauge(_.getName.endsWith(metricName))
+  }
+
+  private def getGauge[T](metricName: String, metricScope: String): Gauge[T] = {
+    getGauge(k => k.getName.endsWith(metricName) && k.getScope.endsWith(metricScope))
+  }
+
   @Test
   def testMaxLogCompactionLag(): Unit = {
     val msPerHour = 60 * 60 * 1000
@@ -185,4 +192,29 @@ class LogCleanerIntegrationTest extends AbstractLogCleanerIntegrationTest {
       (key, curValue)
     }
   }
+
+  @Test
+  def testIsThreadFailed(): Unit = {
+    val metricName = "DeadThreadCount"
+    removeMetric(metricName) // remove the existing metric so it will be attached to this object below on creation
+    cleaner = makeCleaner(partitions = topicPartitions, maxMessageSize = 100000, backOffMs = 100)
+    cleaner.startup()
+    assertEquals(0, cleaner.deadThreadCount)
+    // we simulate the unexpected error with an interrupt
+    cleaner.cleaners.foreach(_.interrupt())
+    // wait until interruption is propagated to all the threads
+    TestUtils.waitUntilTrue(
+      () => cleaner.cleaners.foldLeft(true)((result, thread) => {
+        thread.isThreadFailed && result
+      }), "Threads didn't terminate unexpectedly"
+    )
+    assertEquals(cleaner.cleaners.size, getGauge[Int](metricName).value())
+    assertEquals(cleaner.cleaners.size, cleaner.deadThreadCount)
+  }
+
+  private def removeMetric(name: String): Unit = {
+    val metricName = Metrics.defaultRegistry().allMetrics()
+      .asScala.find(p => p._1.getName.endsWith(name)).get._1
+    Metrics.defaultRegistry().removeMetric(metricName)
+  }
 }
diff --git a/core/src/test/scala/unit/kafka/server/AbstractFetcherManagerTest.scala b/core/src/test/scala/unit/kafka/server/AbstractFetcherManagerTest.scala
index 15ce97133809d..d197845d4da55 100644
--- a/core/src/test/scala/unit/kafka/server/AbstractFetcherManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/AbstractFetcherManagerTest.scala
@@ -97,4 +97,38 @@ class AbstractFetcherManagerTest {
     fetcherManager.removeFetcherForPartitions(Set(tp))
     assertEquals(0, getMetricValue(metricName))
   }
+  @Test
+  def testDeadThreadCountMetric(): Unit = {
+    val fetcher: AbstractFetcherThread = EasyMock.mock(classOf[AbstractFetcherThread])
+    val fetcherManager = new AbstractFetcherManager[AbstractFetcherThread]("fetcher-manager", "fetcher-manager", 2) {
+      override def createFetcherThread(fetcherId: Int, sourceBroker: BrokerEndPoint): AbstractFetcherThread = {
+        fetcher
+      }
+    }
+
+    val fetchOffset = 10L
+    val leaderEpoch = 15
+    val tp = new TopicPartition("topic", 0)
+    val initialFetchState = InitialFetchState(
+      leader = new BrokerEndPoint(0, "localhost", 9092),
+      currentLeaderEpoch = leaderEpoch,
+      initOffset = fetchOffset)
+
+    EasyMock.expect(fetcher.start())
+    EasyMock.expect(fetcher.addPartitions(Map(tp -> OffsetAndEpoch(fetchOffset, leaderEpoch))))
+    EasyMock.expect(fetcher.isThreadFailed).andReturn(true)
+    EasyMock.replay(fetcher)
+
+    fetcherManager.addFetcherForPartitions(Map(tp -> initialFetchState))
+
+    assertEquals(1, fetcherManager.deadThreadCount)
+    EasyMock.verify(fetcher)
+
+    EasyMock.reset(fetcher)
+    EasyMock.expect(fetcher.isThreadFailed).andReturn(false)
+    EasyMock.replay(fetcher)
+
+    assertEquals(0, fetcherManager.deadThreadCount)
+    EasyMock.verify(fetcher)
+  }
 }

From 90f2020c940dd25356357accfc84b23e3db77989 Mon Sep 17 00:00:00 2001
From: Greg Harris 
Date: Wed, 16 Oct 2019 18:43:01 -0700
Subject: [PATCH 0734/1071] KAFKA-8340, KAFKA-8819: Use PluginClassLoader while
 statically initializing plugins (#7315)

Added plugin isolation unit tests for various scenarios, with a `TestPlugins` class that compiles and builds multiple test plugins without them being on the classpath and verifies that the Plugins and DelegatingClassLoader behave properly. These initially failed for several cases, but now pass since the issues have been fixed.

KAFKA-8340 and KAFKA-8819 are closely related, and this fix corrects the problems reported in both issues.

Author: Greg Harris 
Reviewers: Chris Egerton , Magesh Nandakumar , Konstantine Karantasis , Randall Hauch 
---
 checkstyle/import-control.xml                 |   1 +
 .../apache/kafka/connect/runtime/Worker.java  |   4 +-
 .../isolation/DelegatingClassLoader.java      |  51 ++--
 .../connect/runtime/isolation/Plugins.java    | 131 +++++----
 .../isolation/DelegatingClassLoaderTest.java  |  23 ++
 .../runtime/isolation/PluginsTest.java        | 205 +++++++++++++-
 .../runtime/isolation/SamplingTestPlugin.java | 122 ++++++++
 .../runtime/isolation/TestPlugins.java        | 267 ++++++++++++++++++
 .../test/plugins/AliasedStaticField.java      |  75 +++++
 .../test/plugins/AlwaysThrowException.java    |  53 ++++
 ...afka.common.config.provider.ConfigProvider |  16 ++
 .../test/plugins/SamplingConfigProvider.java  | 101 +++++++
 .../test/plugins/SamplingConfigurable.java    |  79 ++++++
 .../test/plugins/SamplingConverter.java       |  76 +++++
 .../test/plugins/SamplingHeaderConverter.java |  89 ++++++
 .../services/test.plugins.ServiceLoadedClass  |  16 ++
 .../test/plugins/ServiceLoadedClass.java      |  48 ++++
 .../test/plugins/ServiceLoadedSubclass.java   |  46 +++
 .../test/plugins/ServiceLoaderPlugin.java     |  85 ++++++
 19 files changed, 1402 insertions(+), 86 deletions(-)
 create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/SamplingTestPlugin.java
 create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/TestPlugins.java
 create mode 100644 connect/runtime/src/test/resources/test-plugins/aliased-static-field/test/plugins/AliasedStaticField.java
 create mode 100644 connect/runtime/src/test/resources/test-plugins/always-throw-exception/test/plugins/AlwaysThrowException.java
 create mode 100644 connect/runtime/src/test/resources/test-plugins/sampling-config-provider/META-INF/services/org.apache.kafka.common.config.provider.ConfigProvider
 create mode 100644 connect/runtime/src/test/resources/test-plugins/sampling-config-provider/test/plugins/SamplingConfigProvider.java
 create mode 100644 connect/runtime/src/test/resources/test-plugins/sampling-configurable/test/plugins/SamplingConfigurable.java
 create mode 100644 connect/runtime/src/test/resources/test-plugins/sampling-converter/test/plugins/SamplingConverter.java
 create mode 100644 connect/runtime/src/test/resources/test-plugins/sampling-header-converter/test/plugins/SamplingHeaderConverter.java
 create mode 100644 connect/runtime/src/test/resources/test-plugins/service-loader/META-INF/services/test.plugins.ServiceLoadedClass
 create mode 100644 connect/runtime/src/test/resources/test-plugins/service-loader/test/plugins/ServiceLoadedClass.java
 create mode 100644 connect/runtime/src/test/resources/test-plugins/service-loader/test/plugins/ServiceLoadedSubclass.java
 create mode 100644 connect/runtime/src/test/resources/test-plugins/service-loader/test/plugins/ServiceLoaderPlugin.java

diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml
index 17e6f57bcd634..6e52bf056bc19 100644
--- a/checkstyle/import-control.xml
+++ b/checkstyle/import-control.xml
@@ -378,6 +378,7 @@
       
         
         
+        
       
 
       
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
index 27955477fc573..fbece7941b0a5 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
@@ -423,10 +423,10 @@ public boolean startTask(
             connectorStatusMetricsGroup.recordTaskAdded(id);
             ClassLoader savedLoader = plugins.currentThreadLoader();
             try {
-                final ConnectorConfig connConfig = new ConnectorConfig(plugins, connProps);
-                String connType = connConfig.getString(ConnectorConfig.CONNECTOR_CLASS_CONFIG);
+                String connType = connProps.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG);
                 ClassLoader connectorLoader = plugins.delegatingLoader().connectorLoader(connType);
                 savedLoader = Plugins.compareAndSwapLoaders(connectorLoader);
+                final ConnectorConfig connConfig = new ConnectorConfig(plugins, connProps);
                 final TaskConfig taskConfig = new TaskConfig(taskProps);
                 final Class taskClass = taskConfig.getClass(TaskConfig.TASK_CLASS_CONFIG).asSubclass(Task.class);
                 final Task task = plugins.newTask(taskClass);
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/DelegatingClassLoader.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/DelegatingClassLoader.java
index d8c4ccaca6b78..8d964b4c94a6d 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/DelegatingClassLoader.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/DelegatingClassLoader.java
@@ -132,6 +132,25 @@ public Set> connectorClientConfi
         return connectorClientConfigPolicies;
     }
 
+    /**
+     * Retrieve the PluginClassLoader associated with a plugin class
+     * @param name The fully qualified class name of the plugin
+     * @return the PluginClassLoader that should be used to load this, or null if the plugin is not isolated.
+     */
+    public PluginClassLoader pluginClassLoader(String name) {
+        if (!PluginUtils.shouldLoadInIsolation(name)) {
+            return null;
+        }
+        SortedMap, ClassLoader> inner = pluginLoaders.get(name);
+        if (inner == null) {
+            return null;
+        }
+        ClassLoader pluginLoader = inner.get(inner.lastKey());
+        return pluginLoader instanceof PluginClassLoader
+               ? (PluginClassLoader) pluginLoader
+               : null;
+    }
+
     public ClassLoader connectorLoader(Connector connector) {
         return connectorLoader(connector.getClass().getName());
     }
@@ -141,8 +160,8 @@ public ClassLoader connectorLoader(String connectorClassOrAlias) {
         String fullName = aliases.containsKey(connectorClassOrAlias)
                           ? aliases.get(connectorClassOrAlias)
                           : connectorClassOrAlias;
-        SortedMap, ClassLoader> inner = pluginLoaders.get(fullName);
-        if (inner == null) {
+        PluginClassLoader classLoader = pluginClassLoader(fullName);
+        if (classLoader == null) {
             log.error(
                     "Plugin class loader for connector: '{}' was not found. Returning: {}",
                     connectorClassOrAlias,
@@ -150,7 +169,7 @@ public ClassLoader connectorLoader(String connectorClassOrAlias) {
             );
             return this;
         }
-        return inner.get(inner.lastKey());
+        return classLoader;
     }
 
     private static PluginClassLoader newPluginClassLoader(
@@ -338,10 +357,16 @@ private  Collection> getPluginDesc(
 
     @SuppressWarnings("unchecked")
     private  Collection> getServiceLoaderPluginDesc(Class klass, ClassLoader loader) {
-        ServiceLoader serviceLoader = ServiceLoader.load(klass, loader);
+        ClassLoader savedLoader = Plugins.compareAndSwapLoaders(loader);
         Collection> result = new ArrayList<>();
-        for (T pluginImpl : serviceLoader) {
-            result.add(new PluginDesc<>((Class) pluginImpl.getClass(), versionFor(pluginImpl), loader));
+        try {
+            ServiceLoader serviceLoader = ServiceLoader.load(klass, loader);
+            for (T pluginImpl : serviceLoader) {
+                result.add(new PluginDesc<>((Class) pluginImpl.getClass(),
+                    versionFor(pluginImpl), loader));
+            }
+        } finally {
+            Plugins.compareAndSwapLoaders(savedLoader);
         }
         return result;
     }
@@ -357,19 +382,11 @@ private static  String versionFor(Class pluginKlass) throws Ille
 
     @Override
     protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException {
-        if (!PluginUtils.shouldLoadInIsolation(name)) {
-            // There are no paths in this classloader, will attempt to load with the parent.
-            return super.loadClass(name, resolve);
-        }
-
         String fullName = aliases.containsKey(name) ? aliases.get(name) : name;
-        SortedMap, ClassLoader> inner = pluginLoaders.get(fullName);
-        if (inner != null) {
-            ClassLoader pluginLoader = inner.get(inner.lastKey());
+        PluginClassLoader pluginLoader = pluginClassLoader(fullName);
+        if (pluginLoader != null) {
             log.trace("Retrieving loaded class '{}' from '{}'", fullName, pluginLoader);
-            return pluginLoader instanceof PluginClassLoader
-                   ? ((PluginClassLoader) pluginLoader).loadClass(fullName, resolve)
-                   : super.loadClass(fullName, resolve);
+            return pluginLoader.loadClass(fullName, resolve);
         }
 
         return super.loadClass(fullName, resolve);
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/Plugins.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/Plugins.java
index e438db8147950..d068a03ceecde 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/Plugins.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/Plugins.java
@@ -17,7 +17,6 @@
 package org.apache.kafka.connect.runtime.isolation;
 
 import org.apache.kafka.common.Configurable;
-import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.config.AbstractConfig;
 import org.apache.kafka.common.config.provider.ConfigProvider;
 import org.apache.kafka.common.utils.Utils;
@@ -72,13 +71,37 @@ private static  String pluginNames(Collection> plugins) {
     }
 
     protected static  T newPlugin(Class klass) {
+        // KAFKA-8340: The thread classloader is used during static initialization and must be
+        // set to the plugin's classloader during instantiation
+        ClassLoader savedLoader = compareAndSwapLoaders(klass.getClassLoader());
         try {
             return Utils.newInstance(klass);
         } catch (Throwable t) {
             throw new ConnectException("Instantiation error", t);
+        } finally {
+            compareAndSwapLoaders(savedLoader);
         }
     }
 
+    @SuppressWarnings("unchecked")
+    protected  Class pluginClassFromConfig(
+            AbstractConfig config,
+            String propertyName,
+            Class pluginClass,
+            Collection> plugins
+    ) {
+        Class klass = config.getClass(propertyName);
+        if (pluginClass.isAssignableFrom(klass)) {
+            return (Class) klass;
+        }
+        throw new ConnectException(
+            "Failed to find any class that implements " + pluginClass.getSimpleName()
+                + " for the config "
+                + propertyName + ", available classes are: "
+                + pluginNames(plugins)
+        );
+    }
+
     @SuppressWarnings("unchecked")
     protected static  Class pluginClass(
             DelegatingClassLoader loader,
@@ -215,18 +238,17 @@ public Converter newConverter(AbstractConfig config, String classPropertyName, C
             // it does not represent an internal converter (which has a default available)
             return null;
         }
-        Converter plugin = null;
+        Class klass = null;
         switch (classLoaderUsage) {
             case CURRENT_CLASSLOADER:
                 // Attempt to load first with the current classloader, and plugins as a fallback.
                 // Note: we can't use config.getConfiguredInstance because Converter doesn't implement Configurable, and even if it did
                 // we have to remove the property prefixes before calling config(...) and we still always want to call Converter.config.
-                plugin = getInstance(config, classPropertyName, Converter.class);
+                klass = pluginClassFromConfig(config, classPropertyName, Converter.class, delegatingLoader.converters());
                 break;
             case PLUGINS:
                 // Attempt to load with the plugin class loader, which uses the current classloader as a fallback
                 String converterClassOrAlias = config.getClass(classPropertyName).getName();
-                Class klass;
                 try {
                     klass = pluginClass(delegatingLoader, converterClassOrAlias, Converter.class);
                 } catch (ClassNotFoundException e) {
@@ -236,11 +258,10 @@ public Converter newConverter(AbstractConfig config, String classPropertyName, C
                             + pluginNames(delegatingLoader.converters())
                     );
                 }
-                plugin = newPlugin(klass);
                 break;
         }
-        if (plugin == null) {
-            throw new ConnectException("Unable to instantiate the Converter specified in '" + classPropertyName + "'");
+        if (klass == null) {
+            throw new ConnectException("Unable to initialize the Converter specified in '" + classPropertyName + "'");
         }
 
         // Determine whether this is a key or value converter based upon the supplied property name ...
@@ -257,7 +278,7 @@ public Converter newConverter(AbstractConfig config, String classPropertyName, C
         // Have to override schemas.enable from true to false for internal JSON converters
         // Don't have to warn the user about anything since all deprecation warnings take place in the
         // WorkerConfig class
-        if (plugin instanceof JsonConverter && isInternalConverter(classPropertyName)) {
+        if (JsonConverter.class.isAssignableFrom(klass) && isInternalConverter(classPropertyName)) {
             // If they haven't explicitly specified values for internal.key.converter.schemas.enable
             // or internal.value.converter.schemas.enable, we can safely default them to false
             if (!converterConfig.containsKey(JsonConverterConfig.SCHEMAS_ENABLE_CONFIG)) {
@@ -265,7 +286,14 @@ public Converter newConverter(AbstractConfig config, String classPropertyName, C
             }
         }
 
-        plugin.configure(converterConfig, isKeyConverter);
+        Converter plugin;
+        ClassLoader savedLoader = compareAndSwapLoaders(klass.getClassLoader());
+        try {
+            plugin = newPlugin(klass);
+            plugin.configure(converterConfig, isKeyConverter);
+        } finally {
+            compareAndSwapLoaders(savedLoader);
+        }
         return plugin;
     }
 
@@ -280,7 +308,7 @@ public Converter newConverter(AbstractConfig config, String classPropertyName, C
      * @throws ConnectException if the {@link HeaderConverter} implementation class could not be found
      */
     public HeaderConverter newHeaderConverter(AbstractConfig config, String classPropertyName, ClassLoaderUsage classLoaderUsage) {
-        HeaderConverter plugin = null;
+        Class klass = null;
         switch (classLoaderUsage) {
             case CURRENT_CLASSLOADER:
                 if (!config.originals().containsKey(classPropertyName)) {
@@ -290,13 +318,12 @@ public HeaderConverter newHeaderConverter(AbstractConfig config, String classPro
                 // Attempt to load first with the current classloader, and plugins as a fallback.
                 // Note: we can't use config.getConfiguredInstance because we have to remove the property prefixes
                 // before calling config(...)
-                plugin = getInstance(config, classPropertyName, HeaderConverter.class);
+                klass = pluginClassFromConfig(config, classPropertyName, HeaderConverter.class, delegatingLoader.headerConverters());
                 break;
             case PLUGINS:
                 // Attempt to load with the plugin class loader, which uses the current classloader as a fallback.
                 // Note that there will always be at least a default header converter for the worker
                 String converterClassOrAlias = config.getClass(classPropertyName).getName();
-                Class klass;
                 try {
                     klass = pluginClass(
                             delegatingLoader,
@@ -311,17 +338,24 @@ public HeaderConverter newHeaderConverter(AbstractConfig config, String classPro
                                     + pluginNames(delegatingLoader.headerConverters())
                     );
                 }
-                plugin = newPlugin(klass);
         }
-        if (plugin == null) {
-            throw new ConnectException("Unable to instantiate the Converter specified in '" + classPropertyName + "'");
+        if (klass == null) {
+            throw new ConnectException("Unable to initialize the HeaderConverter specified in '" + classPropertyName + "'");
         }
 
         String configPrefix = classPropertyName + ".";
         Map converterConfig = config.originalsWithPrefix(configPrefix);
         converterConfig.put(ConverterConfig.TYPE_CONFIG, ConverterType.HEADER.getName());
         log.debug("Configuring the header converter with configuration keys:{}{}", System.lineSeparator(), converterConfig.keySet());
-        plugin.configure(converterConfig);
+
+        HeaderConverter plugin;
+        ClassLoader savedLoader = compareAndSwapLoaders(klass.getClassLoader());
+        try {
+            plugin = newPlugin(klass);
+            plugin.configure(converterConfig);
+        } finally {
+            compareAndSwapLoaders(savedLoader);
+        }
         return plugin;
     }
 
@@ -332,16 +366,15 @@ public ConfigProvider newConfigProvider(AbstractConfig config, String providerPr
             // This configuration does not define the config provider via the specified property name
             return null;
         }
-        ConfigProvider plugin = null;
+        Class klass = null;
         switch (classLoaderUsage) {
             case CURRENT_CLASSLOADER:
                 // Attempt to load first with the current classloader, and plugins as a fallback.
-                plugin = getInstance(config, classPropertyName, ConfigProvider.class);
+                klass = pluginClassFromConfig(config, classPropertyName, ConfigProvider.class, delegatingLoader.configProviders());
                 break;
             case PLUGINS:
                 // Attempt to load with the plugin class loader, which uses the current classloader as a fallback
                 String configProviderClassOrAlias = originalConfig.get(classPropertyName);
-                Class klass;
                 try {
                     klass = pluginClass(delegatingLoader, configProviderClassOrAlias, ConfigProvider.class);
                 } catch (ClassNotFoundException e) {
@@ -351,17 +384,24 @@ public ConfigProvider newConfigProvider(AbstractConfig config, String providerPr
                                     + pluginNames(delegatingLoader.configProviders())
                     );
                 }
-                plugin = newPlugin(klass);
                 break;
         }
-        if (plugin == null) {
-            throw new ConnectException("Unable to instantiate the ConfigProvider specified in '" + classPropertyName + "'");
+        if (klass == null) {
+            throw new ConnectException("Unable to initialize the ConfigProvider specified in '" + classPropertyName + "'");
         }
 
         // Configure the ConfigProvider
         String configPrefix = providerPrefix + ".param.";
         Map configProviderConfig = config.originalsWithPrefix(configPrefix);
-        plugin.configure(configProviderConfig);
+
+        ConfigProvider plugin;
+        ClassLoader savedLoader = compareAndSwapLoaders(klass.getClassLoader());
+        try {
+            plugin = newPlugin(klass);
+            plugin.configure(configProviderConfig);
+        } finally {
+            compareAndSwapLoaders(savedLoader);
+        }
         return plugin;
     }
 
@@ -395,42 +435,25 @@ public  T newPlugin(String klassName, AbstractConfig config, Class pluginK
                                        + "name matches %s", pluginKlass, klassName);
             throw new ConnectException(msg);
         }
-        plugin = newPlugin(klass);
-        if (plugin == null) {
-            throw new ConnectException("Unable to instantiate '" + klassName + "'");
-        }
-        if (plugin instanceof Versioned) {
-            Versioned versionedPlugin = (Versioned) plugin;
-            if (versionedPlugin.version() == null || versionedPlugin.version().trim().isEmpty()) {
-                throw new ConnectException("Version not defined for '" + klassName + "'");
+        ClassLoader savedLoader = compareAndSwapLoaders(klass.getClassLoader());
+        try {
+            plugin = newPlugin(klass);
+            if (plugin instanceof Versioned) {
+                Versioned versionedPlugin = (Versioned) plugin;
+                if (versionedPlugin.version() == null || versionedPlugin.version().trim()
+                    .isEmpty()) {
+                    throw new ConnectException("Version not defined for '" + klassName + "'");
+                }
             }
-        }
-        if (plugin instanceof Configurable) {
-            ((Configurable) plugin).configure(config.originals());
+            if (plugin instanceof Configurable) {
+                ((Configurable) plugin).configure(config.originals());
+            }
+        } finally {
+            compareAndSwapLoaders(savedLoader);
         }
         return plugin;
     }
 
-    /**
-     * Get an instance of the give class specified by the given configuration key.
-     *
-     * @param key The configuration key for the class
-     * @param t The interface the class should implement
-     * @return A instance of the class
-     */
-    private  T getInstance(AbstractConfig config, String key, Class t) {
-        Class c = config.getClass(key);
-        if (c == null) {
-            return null;
-        }
-        // Instantiate the class, but we don't know if the class extends the supplied type
-        Object o = Utils.newInstance(c);
-        if (!t.isInstance(o)) {
-            throw new KafkaException(c.getName() + " is not an instance of " + t.getName());
-        }
-        return t.cast(o);
-    }
-
     public > Transformation newTranformations(
             String transformationClassOrAlias
     ) {
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/DelegatingClassLoaderTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/DelegatingClassLoaderTest.java
index 5c06eaa7ba11f..3e346bb824623 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/DelegatingClassLoaderTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/DelegatingClassLoaderTest.java
@@ -17,8 +17,10 @@
 
 package org.apache.kafka.connect.runtime.isolation;
 
+import java.util.Collections;
 import org.junit.Test;
 
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
@@ -38,4 +40,25 @@ public void testOtherResources() {
             DelegatingClassLoader.serviceLoaderManifestForPlugin("META-INF/services/org.apache.kafka.connect.transforms.Transformation"));
         assertFalse(DelegatingClassLoader.serviceLoaderManifestForPlugin("resource/version.properties"));
     }
+
+    @Test(expected = ClassNotFoundException.class)
+    public void testLoadingUnloadedPluginClass() throws ClassNotFoundException {
+        TestPlugins.assertAvailable();
+        DelegatingClassLoader classLoader = new DelegatingClassLoader(Collections.emptyList());
+        classLoader.initLoaders();
+        for (String pluginClassName : TestPlugins.pluginClasses()) {
+            classLoader.loadClass(pluginClassName);
+        }
+    }
+
+    @Test
+    public void testLoadingPluginClass() throws ClassNotFoundException {
+        TestPlugins.assertAvailable();
+        DelegatingClassLoader classLoader = new DelegatingClassLoader(TestPlugins.pluginPath());
+        classLoader.initLoaders();
+        for (String pluginClassName : TestPlugins.pluginClasses()) {
+            assertNotNull(classLoader.loadClass(pluginClassName));
+            assertNotNull(classLoader.pluginClassLoader(pluginClassName));
+        }
+    }
 }
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/PluginsTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/PluginsTest.java
index 1100910df5f43..4d304c710d838 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/PluginsTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/PluginsTest.java
@@ -17,11 +17,16 @@
 
 package org.apache.kafka.connect.runtime.isolation;
 
+import java.util.Collections;
+import java.util.Map.Entry;
 import org.apache.kafka.common.Configurable;
 import org.apache.kafka.common.config.AbstractConfig;
 import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.config.ConfigException;
+import org.apache.kafka.common.config.provider.ConfigProvider;
 import org.apache.kafka.connect.data.Schema;
 import org.apache.kafka.connect.data.SchemaAndValue;
+import org.apache.kafka.connect.errors.ConnectException;
 import org.apache.kafka.connect.json.JsonConverter;
 import org.apache.kafka.connect.json.JsonConverterConfig;
 import org.apache.kafka.connect.rest.ConnectRestExtension;
@@ -34,7 +39,6 @@
 import org.apache.kafka.connect.storage.HeaderConverter;
 import org.apache.kafka.connect.storage.SimpleHeaderConverter;
 import org.junit.Before;
-import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.io.IOException;
@@ -45,31 +49,26 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 
 public class PluginsTest {
 
-    private static Map pluginProps;
-    private static Plugins plugins;
+    private Plugins plugins;
     private Map props;
     private AbstractConfig config;
     private TestConverter converter;
     private TestHeaderConverter headerConverter;
     private TestInternalConverter internalConverter;
 
-    @BeforeClass
-    public static void beforeAll() {
-        pluginProps = new HashMap<>();
-
-        // Set up the plugins to have no additional plugin directories.
-        // This won't allow us to test classpath isolation, but it will allow us to test some of the utility methods.
-        pluginProps.put(WorkerConfig.PLUGIN_PATH_CONFIG, "");
-        plugins = new Plugins(pluginProps);
-    }
-
     @SuppressWarnings("deprecation")
     @Before
     public void setup() {
+        Map pluginProps = new HashMap<>();
+
+        // Set up the plugins with some test plugins to test isolation
+        pluginProps.put(WorkerConfig.PLUGIN_PATH_CONFIG, String.join(",", TestPlugins.pluginPath()));
+        plugins = new Plugins(pluginProps);
         props = new HashMap<>(pluginProps);
         props.put(WorkerConfig.KEY_CONVERTER_CLASS_CONFIG, TestConverter.class.getName());
         props.put(WorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, TestConverter.class.getName());
@@ -185,6 +184,186 @@ public void shouldInstantiateAndConfigureDefaultHeaderConverter() {
         assertTrue(headerConverter instanceof SimpleHeaderConverter);
     }
 
+    @Test(expected = ConnectException.class)
+    public void shouldThrowIfPluginThrows() {
+        TestPlugins.assertAvailable();
+
+        plugins.newPlugin(
+            TestPlugins.ALWAYS_THROW_EXCEPTION,
+            new AbstractConfig(new ConfigDef(), Collections.emptyMap()),
+            Converter.class
+        );
+    }
+
+    @Test
+    public void shouldShareStaticValuesBetweenSamePlugin() {
+        // Plugins are not isolated from other instances of their own class.
+        TestPlugins.assertAvailable();
+        Converter firstPlugin = plugins.newPlugin(
+            TestPlugins.ALIASED_STATIC_FIELD,
+            new AbstractConfig(new ConfigDef(), Collections.emptyMap()),
+            Converter.class
+        );
+
+        assertInstanceOf(SamplingTestPlugin.class, firstPlugin, "Cannot collect samples");
+
+        Converter secondPlugin = plugins.newPlugin(
+            TestPlugins.ALIASED_STATIC_FIELD,
+            new AbstractConfig(new ConfigDef(), Collections.emptyMap()),
+            Converter.class
+        );
+
+        assertInstanceOf(SamplingTestPlugin.class, secondPlugin, "Cannot collect samples");
+        assertSame(
+            ((SamplingTestPlugin) firstPlugin).otherSamples(),
+            ((SamplingTestPlugin) secondPlugin).otherSamples()
+        );
+    }
+
+    @Test
+    public void newPluginShouldServiceLoadWithPluginClassLoader() {
+        TestPlugins.assertAvailable();
+        Converter plugin = plugins.newPlugin(
+            TestPlugins.SERVICE_LOADER,
+            new AbstractConfig(new ConfigDef(), Collections.emptyMap()),
+            Converter.class
+        );
+
+        assertInstanceOf(SamplingTestPlugin.class, plugin, "Cannot collect samples");
+        Map samples = ((SamplingTestPlugin) plugin).flatten();
+        // Assert that the service loaded subclass is found in both environments
+        assertTrue(samples.containsKey("ServiceLoadedSubclass.static"));
+        assertTrue(samples.containsKey("ServiceLoadedSubclass.dynamic"));
+        assertPluginClassLoaderAlwaysActive(samples);
+    }
+
+    @Test
+    public void newPluginShouldInstantiateWithPluginClassLoader() {
+        TestPlugins.assertAvailable();
+        Converter plugin = plugins.newPlugin(
+            TestPlugins.ALIASED_STATIC_FIELD,
+            new AbstractConfig(new ConfigDef(), Collections.emptyMap()),
+            Converter.class
+        );
+
+        assertInstanceOf(SamplingTestPlugin.class, plugin, "Cannot collect samples");
+        Map samples = ((SamplingTestPlugin) plugin).flatten();
+        assertPluginClassLoaderAlwaysActive(samples);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void shouldFailToFindConverterInCurrentClassloader() {
+        TestPlugins.assertAvailable();
+        props.put(WorkerConfig.KEY_CONVERTER_CLASS_CONFIG, TestPlugins.SAMPLING_CONVERTER);
+        createConfig();
+    }
+
+    @Test
+    public void newConverterShouldConfigureWithPluginClassLoader() {
+        TestPlugins.assertAvailable();
+        props.put(WorkerConfig.KEY_CONVERTER_CLASS_CONFIG, TestPlugins.SAMPLING_CONVERTER);
+        ClassLoader classLoader = plugins.delegatingLoader().pluginClassLoader(TestPlugins.SAMPLING_CONVERTER);
+        ClassLoader savedLoader = Plugins.compareAndSwapLoaders(classLoader);
+        createConfig();
+        Plugins.compareAndSwapLoaders(savedLoader);
+
+        Converter plugin = plugins.newConverter(
+            config,
+            WorkerConfig.KEY_CONVERTER_CLASS_CONFIG,
+            ClassLoaderUsage.PLUGINS
+        );
+
+        assertInstanceOf(SamplingTestPlugin.class, plugin, "Cannot collect samples");
+        Map samples = ((SamplingTestPlugin) plugin).flatten();
+        assertTrue(samples.containsKey("configure"));
+        assertPluginClassLoaderAlwaysActive(samples);
+    }
+
+    @Test
+    public void newConfigProviderShouldConfigureWithPluginClassLoader() {
+        TestPlugins.assertAvailable();
+        String providerPrefix = "some.provider";
+        props.put(providerPrefix + ".class", TestPlugins.SAMPLING_CONFIG_PROVIDER);
+
+        PluginClassLoader classLoader = plugins.delegatingLoader().pluginClassLoader(TestPlugins.SAMPLING_CONFIG_PROVIDER);
+        assertNotNull(classLoader);
+        ClassLoader savedLoader = Plugins.compareAndSwapLoaders(classLoader);
+        createConfig();
+        Plugins.compareAndSwapLoaders(savedLoader);
+
+        ConfigProvider plugin = plugins.newConfigProvider(
+            config,
+            providerPrefix,
+            ClassLoaderUsage.PLUGINS
+        );
+
+        assertInstanceOf(SamplingTestPlugin.class, plugin, "Cannot collect samples");
+        Map samples = ((SamplingTestPlugin) plugin).flatten();
+        assertTrue(samples.containsKey("configure"));
+        assertPluginClassLoaderAlwaysActive(samples);
+    }
+
+    @Test
+    public void newHeaderConverterShouldConfigureWithPluginClassLoader() {
+        TestPlugins.assertAvailable();
+        props.put(WorkerConfig.HEADER_CONVERTER_CLASS_CONFIG, TestPlugins.SAMPLING_HEADER_CONVERTER);
+        ClassLoader classLoader = plugins.delegatingLoader().pluginClassLoader(TestPlugins.SAMPLING_HEADER_CONVERTER);
+        ClassLoader savedLoader = Plugins.compareAndSwapLoaders(classLoader);
+        createConfig();
+        Plugins.compareAndSwapLoaders(savedLoader);
+
+        HeaderConverter plugin = plugins.newHeaderConverter(
+            config,
+            WorkerConfig.HEADER_CONVERTER_CLASS_CONFIG,
+            ClassLoaderUsage.PLUGINS
+        );
+
+        assertInstanceOf(SamplingTestPlugin.class, plugin, "Cannot collect samples");
+        Map samples = ((SamplingTestPlugin) plugin).flatten();
+        assertTrue(samples.containsKey("configure")); // HeaderConverter::configure was called
+        assertPluginClassLoaderAlwaysActive(samples);
+    }
+
+    @Test
+    public void newPluginsShouldConfigureWithPluginClassLoader() {
+        TestPlugins.assertAvailable();
+        List configurables = plugins.newPlugins(
+            Collections.singletonList(TestPlugins.SAMPLING_CONFIGURABLE),
+            config,
+            Configurable.class
+        );
+        assertEquals(1, configurables.size());
+        Configurable plugin = configurables.get(0);
+
+        assertInstanceOf(SamplingTestPlugin.class, plugin, "Cannot collect samples");
+        Map samples = ((SamplingTestPlugin) plugin).flatten();
+        assertTrue(samples.containsKey("configure")); // Configurable::configure was called
+        assertPluginClassLoaderAlwaysActive(samples);
+    }
+
+    public static void assertPluginClassLoaderAlwaysActive(Map samples) {
+        for (Entry e : samples.entrySet()) {
+            String sampleName = "\"" + e.getKey() + "\" (" + e.getValue() + ")";
+            assertInstanceOf(
+                PluginClassLoader.class,
+                e.getValue().staticClassloader(),
+                sampleName + " has incorrect static classloader"
+            );
+            assertInstanceOf(
+                PluginClassLoader.class,
+                e.getValue().classloader(),
+                sampleName + " has incorrect dynamic classloader"
+            );
+        }
+    }
+
+    public static void assertInstanceOf(Class expected, Object actual, String message) {
+        assertTrue(
+            "Expected an instance of " + expected.getSimpleName() + ", found " + actual + " instead: " + message,
+            expected.isInstance(actual)
+        );
+    }
+
     protected void instantiateAndConfigureConverter(String configPropName, ClassLoaderUsage classLoaderUsage) {
         converter = (TestConverter) plugins.newConverter(config, configPropName, classLoaderUsage);
         assertNotNull(converter);
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/SamplingTestPlugin.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/SamplingTestPlugin.java
new file mode 100644
index 0000000000000..bcf8881898e8e
--- /dev/null
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/SamplingTestPlugin.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.connect.runtime.isolation;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * Base class for plugins so we can sample information about their initialization
+ */
+public abstract class SamplingTestPlugin {
+
+    /**
+     * @return the ClassLoader used to statically initialize this plugin class
+     */
+    public abstract ClassLoader staticClassloader();
+
+    /**
+     * @return the ClassLoader used to initialize this plugin instance
+     */
+    public abstract ClassLoader classloader();
+
+    /**
+     * @return a group of other SamplingTestPlugin instances known by this plugin
+     * This should only return direct children, and not reference this instance directly
+     */
+    public Map otherSamples() {
+        return Collections.emptyMap();
+    }
+
+    /**
+     * @return a flattened list of child samples including this entry keyed as "this"
+     */
+    public Map flatten() {
+        Map out = new HashMap<>();
+        Map otherSamples = otherSamples();
+        if (otherSamples != null) {
+            for (Entry child : otherSamples.entrySet()) {
+                for (Entry flattened : child.getValue().flatten().entrySet()) {
+                    String key = child.getKey();
+                    if (flattened.getKey().length() > 0) {
+                        key += "." + flattened.getKey();
+                    }
+                    out.put(key, flattened.getValue());
+                }
+            }
+        }
+        out.put("", this);
+        return out;
+    }
+
+    /**
+     * Log the parent method call as a child sample.
+     * Stores only the last invocation of each method if there are multiple invocations.
+     * @param samples The collection of samples to which this method call should be added
+     */
+    public void logMethodCall(Map samples) {
+        StackTraceElement[] stackTraces = Thread.currentThread().getStackTrace();
+        if (stackTraces.length < 2) {
+            return;
+        }
+        // 0 is inside getStackTrace
+        // 1 is this method
+        // 2 is our caller method
+        StackTraceElement caller = stackTraces[2];
+
+        samples.put(caller.getMethodName(), new MethodCallSample(
+            caller,
+            Thread.currentThread().getContextClassLoader(),
+            getClass().getClassLoader()
+        ));
+    }
+
+    public static class MethodCallSample extends SamplingTestPlugin {
+
+        private final StackTraceElement caller;
+        private final ClassLoader staticClassLoader;
+        private final ClassLoader dynamicClassLoader;
+
+        public MethodCallSample(
+            StackTraceElement caller,
+            ClassLoader staticClassLoader,
+            ClassLoader dynamicClassLoader
+        ) {
+            this.caller = caller;
+            this.staticClassLoader = staticClassLoader;
+            this.dynamicClassLoader = dynamicClassLoader;
+        }
+
+        @Override
+        public ClassLoader staticClassloader() {
+            return staticClassLoader;
+        }
+
+        @Override
+        public ClassLoader classloader() {
+            return dynamicClassLoader;
+        }
+
+        @Override
+        public String toString() {
+            return caller.toString();
+        }
+    }
+}
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/TestPlugins.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/TestPlugins.java
new file mode 100644
index 0000000000000..9561ffb0f5b14
--- /dev/null
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/TestPlugins.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.connect.runtime.isolation;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.jar.Attributes;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+import java.util.stream.Collectors;
+import javax.tools.JavaCompiler;
+import javax.tools.StandardJavaFileManager;
+import javax.tools.ToolProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility class for constructing test plugins for Connect.
+ *
+ * 
                Plugins are built from their source under resources/test-plugins/ and placed into temporary
+ * jar files that are deleted when the process exits.
+ *
+ * 
                To add a plugin, create the source files in the resource tree, and edit this class to build
+ * that plugin during initialization. For example, the plugin class {@literal package.Class} should
+ * be placed in {@literal resources/test-plugins/something/package/Class.java} and loaded using
+ * {@code createPluginJar("something")}. The class name, contents, and plugin directory can take
+ * any value you need for testing.
+ *
+ * 
                To use this class in your tests, make sure to first call
+ * {@link TestPlugins#assertAvailable()} to verify that the plugins initialized correctly.
+ * Otherwise, exceptions during the plugin build are not propagated, and may invalidate your test.
+ * You can access the list of plugin jars for assembling a {@literal plugin.path}, and reference
+ * the names of the different plugins directly via the exposed constants.
+ */
+public class TestPlugins {
+
+    /**
+     * Class name of a plugin which will always throw an exception during loading
+     */
+    public static final String ALWAYS_THROW_EXCEPTION = "test.plugins.AlwaysThrowException";
+    /**
+     * Class name of a plugin which samples information about its initialization.
+     */
+    public static final String ALIASED_STATIC_FIELD = "test.plugins.AliasedStaticField";
+    /**
+     * Class name of a {@link org.apache.kafka.connect.storage.Converter}
+     * which samples information about its method calls.
+     */
+    public static final String SAMPLING_CONVERTER = "test.plugins.SamplingConverter";
+    /**
+     * Class name of a {@link org.apache.kafka.common.Configurable}
+     * which samples information about its method calls.
+     */
+    public static final String SAMPLING_CONFIGURABLE = "test.plugins.SamplingConfigurable";
+    /**
+     * Class name of a {@link org.apache.kafka.connect.storage.HeaderConverter}
+     * which samples information about its method calls.
+     */
+    public static final String SAMPLING_HEADER_CONVERTER = "test.plugins.SamplingHeaderConverter";
+    /**
+     * Class name of a {@link org.apache.kafka.common.config.provider.ConfigProvider}
+     * which samples information about its method calls.
+     */
+    public static final String SAMPLING_CONFIG_PROVIDER = "test.plugins.SamplingConfigProvider";
+    /**
+     * Class name of a plugin which uses a {@link java.util.ServiceLoader}
+     * to load internal classes, and samples information about their initialization.
+     */
+    public static final String SERVICE_LOADER = "test.plugins.ServiceLoaderPlugin";
+
+    private static final Logger log = LoggerFactory.getLogger(TestPlugins.class);
+    private static final Map PLUGIN_JARS;
+    private static final Throwable INITIALIZATION_EXCEPTION;
+
+    static {
+        Throwable err = null;
+        HashMap pluginJars = new HashMap<>();
+        try {
+            pluginJars.put(ALWAYS_THROW_EXCEPTION, createPluginJar("always-throw-exception"));
+            pluginJars.put(ALIASED_STATIC_FIELD, createPluginJar("aliased-static-field"));
+            pluginJars.put(SAMPLING_CONVERTER, createPluginJar("sampling-converter"));
+            pluginJars.put(SAMPLING_CONFIGURABLE, createPluginJar("sampling-configurable"));
+            pluginJars.put(SAMPLING_HEADER_CONVERTER, createPluginJar("sampling-header-converter"));
+            pluginJars.put(SAMPLING_CONFIG_PROVIDER, createPluginJar("sampling-config-provider"));
+            pluginJars.put(SERVICE_LOADER, createPluginJar("service-loader"));
+        } catch (Throwable e) {
+            log.error("Could not set up plugin test jars", e);
+            err = e;
+        }
+        PLUGIN_JARS = Collections.unmodifiableMap(pluginJars);
+        INITIALIZATION_EXCEPTION = err;
+    }
+
+    /**
+     * Ensure that the test plugin JARs were assembled without error before continuing.
+     * @throws AssertionError if any plugin failed to load, or no plugins were loaded.
+     */
+    public static void assertAvailable() throws AssertionError {
+        if (INITIALIZATION_EXCEPTION != null) {
+            throw new AssertionError("TestPlugins did not initialize completely",
+                INITIALIZATION_EXCEPTION);
+        }
+        if (PLUGIN_JARS.isEmpty()) {
+            throw new AssertionError("No test plugins loaded");
+        }
+    }
+
+    /**
+     * A list of jar files containing test plugins
+     * @return A list of plugin jar filenames
+     */
+    public static List pluginPath() {
+        return PLUGIN_JARS.values()
+            .stream()
+            .map(File::getPath)
+            .collect(Collectors.toList());
+    }
+
+    /**
+     * Get all of the classes that were successfully built by this class
+     * @return A list of plugin class names
+     */
+    public static List pluginClasses() {
+        return new ArrayList<>(PLUGIN_JARS.keySet());
+    }
+
+    private static File createPluginJar(String resourceDir) throws IOException {
+        Path inputDir = resourceDirectoryPath("test-plugins/" + resourceDir);
+        Path binDir = Files.createTempDirectory(resourceDir + ".bin.");
+        compileJavaSources(inputDir, binDir);
+        File jarFile = Files.createTempFile(resourceDir + ".", ".jar").toFile();
+        try (JarOutputStream jar = openJarFile(jarFile)) {
+            writeJar(jar, inputDir);
+            writeJar(jar, binDir);
+        }
+        removeDirectory(binDir);
+        jarFile.deleteOnExit();
+        return jarFile;
+    }
+
+    private static Path resourceDirectoryPath(String resourceDir) throws IOException {
+        URL resource = Thread.currentThread()
+            .getContextClassLoader()
+            .getResource(resourceDir);
+        if (resource == null) {
+            throw new IOException("Could not find test plugin resource: " + resourceDir);
+        }
+        File file = new File(resource.getFile());
+        if (!file.isDirectory()) {
+            throw new IOException("Resource is not a directory: " + resourceDir);
+        }
+        if (!file.canRead()) {
+            throw new IOException("Resource directory is not readable: " + resourceDir);
+        }
+        return file.toPath();
+    }
+
+    private static JarOutputStream openJarFile(File jarFile) throws IOException {
+        Manifest manifest = new Manifest();
+        manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
+        return new JarOutputStream(new FileOutputStream(jarFile), manifest);
+    }
+
+    private static void removeDirectory(Path binDir) throws IOException {
+        List classFiles = Files.walk(binDir)
+            .sorted(Comparator.reverseOrder())
+            .map(Path::toFile)
+            .collect(Collectors.toList());
+        for (File classFile : classFiles) {
+            if (!classFile.delete()) {
+                throw new IOException("Could not delete: " + classFile);
+            }
+        }
+    }
+
+    /**
+     * Compile a directory of .java source files into .class files
+     * .class files are placed into the same directory as their sources.
+     *
+     * 
                Dependencies between source files in this directory are resolved against one another
+     * and the classes present in the test environment.
+     * See https://stackoverflow.com/questions/1563909/ for more information.
+     * Additional dependencies in your plugins should be added as test scope to :connect:runtime.
+     * @param sourceDir Directory containing java source files
+     * @throws IOException if the files cannot be compiled
+     */
+    private static void compileJavaSources(Path sourceDir, Path binDir) throws IOException {
+        JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
+        List sourceFiles = Files.walk(sourceDir)
+            .filter(Files::isRegularFile)
+            .filter(path -> path.toFile().getName().endsWith(".java"))
+            .map(Path::toFile)
+            .collect(Collectors.toList());
+        StringWriter writer = new StringWriter();
+        List options = Arrays.asList(
+            "-d", binDir.toString() // Write class output to a different directory.
+        );
+
+        try (StandardJavaFileManager fileManager = compiler.getStandardFileManager(null, null, null)) {
+            boolean success = compiler.getTask(
+                writer,
+                fileManager,
+                null,
+                options,
+                null,
+                fileManager.getJavaFileObjectsFromFiles(sourceFiles)
+            ).call();
+            if (!success) {
+                throw new RuntimeException("Failed to compile test plugin:\n" + writer);
+            }
+        }
+    }
+
+    private static void writeJar(JarOutputStream jar, Path inputDir) throws IOException {
+        List paths = Files.walk(inputDir)
+            .filter(Files::isRegularFile)
+            .filter(path -> !path.toFile().getName().endsWith(".java"))
+            .collect(Collectors.toList());
+        for (Path path : paths) {
+            try (InputStream in = new BufferedInputStream(new FileInputStream(path.toFile()))) {
+                jar.putNextEntry(new JarEntry(
+                    inputDir.relativize(path)
+                        .toFile()
+                        .getPath()
+                        .replace(File.separator, "/")
+                ));
+                byte[] buffer = new byte[1024];
+                for (int count; (count = in.read(buffer)) != -1; ) {
+                    jar.write(buffer, 0, count);
+                }
+                jar.closeEntry();
+            }
+        }
+    }
+
+}
diff --git a/connect/runtime/src/test/resources/test-plugins/aliased-static-field/test/plugins/AliasedStaticField.java b/connect/runtime/src/test/resources/test-plugins/aliased-static-field/test/plugins/AliasedStaticField.java
new file mode 100644
index 0000000000000..d865f4e91ba86
--- /dev/null
+++ b/connect/runtime/src/test/resources/test-plugins/aliased-static-field/test/plugins/AliasedStaticField.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.plugins;
+
+import java.util.Map;
+import java.util.HashMap;
+import org.apache.kafka.connect.data.Schema;
+import org.apache.kafka.connect.data.SchemaAndValue;
+import org.apache.kafka.connect.storage.Converter;
+import org.apache.kafka.connect.runtime.isolation.SamplingTestPlugin;
+
+/**
+ * Samples data about its initialization environment for later analysis
+ * Samples are shared between instances of the same class in a static variable
+ */
+public class AliasedStaticField extends SamplingTestPlugin implements Converter {
+
+  private static final Map SAMPLES;
+  private static final ClassLoader STATIC_CLASS_LOADER;
+  private final ClassLoader classloader;
+
+  static {
+    SAMPLES = new HashMap<>();
+    STATIC_CLASS_LOADER = Thread.currentThread().getContextClassLoader();
+  }
+
+  {
+    classloader = Thread.currentThread().getContextClassLoader();
+  }
+
+  @Override
+  public void configure(final Map configs, final boolean isKey) {
+
+  }
+
+  @Override
+  public byte[] fromConnectData(final String topic, final Schema schema, final Object value) {
+    return new byte[0];
+  }
+
+  @Override
+  public SchemaAndValue toConnectData(final String topic, final byte[] value) {
+    return null;
+  }
+
+  @Override
+  public ClassLoader staticClassloader() {
+    return STATIC_CLASS_LOADER;
+  }
+
+  @Override
+  public ClassLoader classloader() {
+    return classloader;
+  }
+
+  @Override
+  public Map otherSamples() {
+      return SAMPLES;
+  }
+}
diff --git a/connect/runtime/src/test/resources/test-plugins/always-throw-exception/test/plugins/AlwaysThrowException.java b/connect/runtime/src/test/resources/test-plugins/always-throw-exception/test/plugins/AlwaysThrowException.java
new file mode 100644
index 0000000000000..858f3ed5eacbe
--- /dev/null
+++ b/connect/runtime/src/test/resources/test-plugins/always-throw-exception/test/plugins/AlwaysThrowException.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.plugins;
+
+import java.util.Map;
+import org.apache.kafka.connect.data.Schema;
+import org.apache.kafka.connect.data.SchemaAndValue;
+import org.apache.kafka.connect.runtime.isolation.SamplingTestPlugin;
+import org.apache.kafka.connect.storage.Converter;
+
+/**
+ * Unconditionally throw an exception during static initialization.
+ */
+public class AlwaysThrowException implements Converter {
+
+    static {
+        setup();
+    }
+
+    public static void setup() {
+        throw new RuntimeException("I always throw an exception");
+    }
+
+    @Override
+    public void configure(final Map configs, final boolean isKey) {
+
+    }
+
+    @Override
+    public byte[] fromConnectData(final String topic, final Schema schema, final Object value) {
+        return new byte[0];
+    }
+
+    @Override
+    public SchemaAndValue toConnectData(final String topic, final byte[] value) {
+        return null;
+    }
+}
diff --git a/connect/runtime/src/test/resources/test-plugins/sampling-config-provider/META-INF/services/org.apache.kafka.common.config.provider.ConfigProvider b/connect/runtime/src/test/resources/test-plugins/sampling-config-provider/META-INF/services/org.apache.kafka.common.config.provider.ConfigProvider
new file mode 100644
index 0000000000000..62d8df254bbc3
--- /dev/null
+++ b/connect/runtime/src/test/resources/test-plugins/sampling-config-provider/META-INF/services/org.apache.kafka.common.config.provider.ConfigProvider
@@ -0,0 +1,16 @@
+ # Licensed to the Apache Software Foundation (ASF) under one or more
+ # contributor license agreements. See the NOTICE file distributed with
+ # this work for additional information regarding copyright ownership.
+ # The ASF licenses this file to You under the Apache License, Version 2.0
+ # (the "License"); you may not use this file except in compliance with
+ # the License. You may obtain a copy of the License at
+ #
+ #    http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+
+test.plugins.SamplingConfigProvider
diff --git a/connect/runtime/src/test/resources/test-plugins/sampling-config-provider/test/plugins/SamplingConfigProvider.java b/connect/runtime/src/test/resources/test-plugins/sampling-config-provider/test/plugins/SamplingConfigProvider.java
new file mode 100644
index 0000000000000..df8285eba9a1a
--- /dev/null
+++ b/connect/runtime/src/test/resources/test-plugins/sampling-config-provider/test/plugins/SamplingConfigProvider.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.plugins;
+
+import java.util.Set;
+import java.util.Map;
+import java.util.HashMap;
+import org.apache.kafka.common.config.provider.ConfigProvider;
+import org.apache.kafka.common.config.ConfigData;
+import org.apache.kafka.common.config.ConfigChangeCallback;
+import org.apache.kafka.connect.data.Schema;
+import org.apache.kafka.connect.data.SchemaAndValue;
+import org.apache.kafka.connect.storage.Converter;
+import org.apache.kafka.connect.runtime.isolation.SamplingTestPlugin;
+import org.apache.kafka.connect.storage.HeaderConverter;
+
+/**
+ * Samples data about its initialization environment for later analysis
+ */
+public class SamplingConfigProvider extends SamplingTestPlugin implements ConfigProvider {
+
+  private static final ClassLoader STATIC_CLASS_LOADER;
+  private final ClassLoader classloader;
+  private Map samples;
+
+  static {
+    STATIC_CLASS_LOADER = Thread.currentThread().getContextClassLoader();
+  }
+
+  {
+    samples = new HashMap<>();
+    classloader = Thread.currentThread().getContextClassLoader();
+  }
+
+  @Override
+  public ConfigData get(String path) {
+    logMethodCall(samples);
+    return null;
+  }
+
+  @Override
+  public ConfigData get(String path, Set keys) {
+    logMethodCall(samples);
+    return null;
+  }
+
+  @Override
+  public void subscribe(String path, Set keys, ConfigChangeCallback callback) {
+    logMethodCall(samples);
+  }
+
+  @Override
+  public void unsubscribe(String path, Set keys, ConfigChangeCallback callback) {
+    logMethodCall(samples);
+  }
+
+  @Override
+  public void unsubscribeAll() {
+    logMethodCall(samples);
+  }
+
+  @Override
+  public void configure(final Map configs) {
+    logMethodCall(samples);
+  }
+
+  @Override
+  public void close() {
+    logMethodCall(samples);
+  }
+
+  @Override
+  public ClassLoader staticClassloader() {
+    return STATIC_CLASS_LOADER;
+  }
+
+  @Override
+  public ClassLoader classloader() {
+    return classloader;
+  }
+
+  @Override
+  public Map otherSamples() {
+    return samples;
+  }
+}
diff --git a/connect/runtime/src/test/resources/test-plugins/sampling-configurable/test/plugins/SamplingConfigurable.java b/connect/runtime/src/test/resources/test-plugins/sampling-configurable/test/plugins/SamplingConfigurable.java
new file mode 100644
index 0000000000000..a917f2f2ca1a8
--- /dev/null
+++ b/connect/runtime/src/test/resources/test-plugins/sampling-configurable/test/plugins/SamplingConfigurable.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.plugins;
+
+import java.util.Map;
+import java.util.HashMap;
+import org.apache.kafka.common.Configurable;
+import org.apache.kafka.connect.data.Schema;
+import org.apache.kafka.connect.data.SchemaAndValue;
+import org.apache.kafka.connect.storage.Converter;
+import org.apache.kafka.connect.runtime.isolation.SamplingTestPlugin;
+
+/**
+ * Samples data about its initialization environment for later analysis
+ */
+public class SamplingConfigurable extends SamplingTestPlugin implements Converter, Configurable {
+
+  private static final ClassLoader STATIC_CLASS_LOADER;
+  private final ClassLoader classloader;
+  private Map samples;
+
+  static {
+    STATIC_CLASS_LOADER = Thread.currentThread().getContextClassLoader();
+  }
+
+  {
+    samples = new HashMap<>();
+    classloader = Thread.currentThread().getContextClassLoader();
+  }
+
+  @Override
+  public void configure(final Map configs) {
+    logMethodCall(samples);
+  }
+
+  @Override
+  public void configure(final Map configs, final boolean isKey) {
+  }
+
+  @Override
+  public byte[] fromConnectData(final String topic, final Schema schema, final Object value) {
+    return new byte[0];
+  }
+
+  @Override
+  public SchemaAndValue toConnectData(final String topic, final byte[] value) {
+    return null;
+  }
+
+  @Override
+  public ClassLoader staticClassloader() {
+    return STATIC_CLASS_LOADER;
+  }
+
+  @Override
+  public ClassLoader classloader() {
+    return classloader;
+  }
+
+  @Override
+  public Map otherSamples() {
+    return samples;
+  }
+}
diff --git a/connect/runtime/src/test/resources/test-plugins/sampling-converter/test/plugins/SamplingConverter.java b/connect/runtime/src/test/resources/test-plugins/sampling-converter/test/plugins/SamplingConverter.java
new file mode 100644
index 0000000000000..39109a1d4e573
--- /dev/null
+++ b/connect/runtime/src/test/resources/test-plugins/sampling-converter/test/plugins/SamplingConverter.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.plugins;
+
+import java.util.Map;
+import java.util.HashMap;
+import org.apache.kafka.connect.data.Schema;
+import org.apache.kafka.connect.data.SchemaAndValue;
+import org.apache.kafka.connect.storage.Converter;
+import org.apache.kafka.connect.runtime.isolation.SamplingTestPlugin;
+
+/**
+ * Samples data about its initialization environment for later analysis
+ */
+public class SamplingConverter extends SamplingTestPlugin implements Converter {
+
+  private static final ClassLoader STATIC_CLASS_LOADER;
+  private final ClassLoader classloader;
+  private Map samples;
+
+  static {
+    STATIC_CLASS_LOADER = Thread.currentThread().getContextClassLoader();
+  }
+
+  {
+    samples = new HashMap<>();
+    classloader = Thread.currentThread().getContextClassLoader();
+  }
+
+  @Override
+  public void configure(final Map configs, final boolean isKey) {
+    logMethodCall(samples);
+  }
+
+  @Override
+  public byte[] fromConnectData(final String topic, final Schema schema, final Object value) {
+    logMethodCall(samples);
+    return new byte[0];
+  }
+
+  @Override
+  public SchemaAndValue toConnectData(final String topic, final byte[] value) {
+    logMethodCall(samples);
+    return null;
+  }
+
+  @Override
+  public ClassLoader staticClassloader() {
+    return STATIC_CLASS_LOADER;
+  }
+
+  @Override
+  public ClassLoader classloader() {
+    return classloader;
+  }
+
+  @Override
+  public Map otherSamples() {
+    return samples;
+  }
+}
diff --git a/connect/runtime/src/test/resources/test-plugins/sampling-header-converter/test/plugins/SamplingHeaderConverter.java b/connect/runtime/src/test/resources/test-plugins/sampling-header-converter/test/plugins/SamplingHeaderConverter.java
new file mode 100644
index 0000000000000..11a1e28e7278c
--- /dev/null
+++ b/connect/runtime/src/test/resources/test-plugins/sampling-header-converter/test/plugins/SamplingHeaderConverter.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.plugins;
+
+import java.util.Map;
+import java.util.HashMap;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.connect.data.Schema;
+import org.apache.kafka.connect.data.SchemaAndValue;
+import org.apache.kafka.connect.storage.Converter;
+import org.apache.kafka.connect.runtime.isolation.SamplingTestPlugin;
+import org.apache.kafka.connect.storage.HeaderConverter;
+
+/**
+ * Samples data about its initialization environment for later analysis
+ */
+public class SamplingHeaderConverter extends SamplingTestPlugin implements HeaderConverter {
+
+  private static final ClassLoader STATIC_CLASS_LOADER;
+  private final ClassLoader classloader;
+  private Map samples;
+
+  static {
+    STATIC_CLASS_LOADER = Thread.currentThread().getContextClassLoader();
+  }
+
+  {
+    samples = new HashMap<>();
+    classloader = Thread.currentThread().getContextClassLoader();
+  }
+
+  @Override
+  public SchemaAndValue toConnectHeader(String topic, String headerKey, byte[] value) {
+    logMethodCall(samples);
+    return null;
+  }
+
+  @Override
+  public byte[] fromConnectHeader(String topic, String headerKey, Schema schema, Object value) {
+    logMethodCall(samples);
+    return new byte[0];
+  }
+
+  @Override
+  public ConfigDef config() {
+    logMethodCall(samples);
+    return null;
+  }
+
+  @Override
+  public void configure(final Map configs) {
+    logMethodCall(samples);
+  }
+
+  @Override
+  public void close() {
+    logMethodCall(samples);
+  }
+
+  @Override
+  public ClassLoader staticClassloader() {
+    return STATIC_CLASS_LOADER;
+  }
+
+  @Override
+  public ClassLoader classloader() {
+    return classloader;
+  }
+
+  @Override
+  public Map otherSamples() {
+    return samples;
+  }
+}
diff --git a/connect/runtime/src/test/resources/test-plugins/service-loader/META-INF/services/test.plugins.ServiceLoadedClass b/connect/runtime/src/test/resources/test-plugins/service-loader/META-INF/services/test.plugins.ServiceLoadedClass
new file mode 100644
index 0000000000000..b8db8656487d2
--- /dev/null
+++ b/connect/runtime/src/test/resources/test-plugins/service-loader/META-INF/services/test.plugins.ServiceLoadedClass
@@ -0,0 +1,16 @@
+ # Licensed to the Apache Software Foundation (ASF) under one or more
+ # contributor license agreements. See the NOTICE file distributed with
+ # this work for additional information regarding copyright ownership.
+ # The ASF licenses this file to You under the Apache License, Version 2.0
+ # (the "License"); you may not use this file except in compliance with
+ # the License. You may obtain a copy of the License at
+ #
+ #    http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+
+test.plugins.ServiceLoadedSubclass
\ No newline at end of file
diff --git a/connect/runtime/src/test/resources/test-plugins/service-loader/test/plugins/ServiceLoadedClass.java b/connect/runtime/src/test/resources/test-plugins/service-loader/test/plugins/ServiceLoadedClass.java
new file mode 100644
index 0000000000000..98677ed43d65d
--- /dev/null
+++ b/connect/runtime/src/test/resources/test-plugins/service-loader/test/plugins/ServiceLoadedClass.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.plugins;
+
+import org.apache.kafka.connect.runtime.isolation.SamplingTestPlugin;
+
+/**
+ * Superclass for service loaded classes
+ */
+public class ServiceLoadedClass extends SamplingTestPlugin {
+
+  private static final ClassLoader STATIC_CLASS_LOADER;
+  private final ClassLoader classloader;
+
+  static {
+    STATIC_CLASS_LOADER = Thread.currentThread().getContextClassLoader();
+  }
+
+  {
+    classloader = Thread.currentThread().getContextClassLoader();
+  }
+
+  @Override
+  public ClassLoader staticClassloader() {
+    return STATIC_CLASS_LOADER;
+  }
+
+  @Override
+  public ClassLoader classloader() {
+    return classloader;
+  }
+
+}
diff --git a/connect/runtime/src/test/resources/test-plugins/service-loader/test/plugins/ServiceLoadedSubclass.java b/connect/runtime/src/test/resources/test-plugins/service-loader/test/plugins/ServiceLoadedSubclass.java
new file mode 100644
index 0000000000000..cfc6b6f9cfc94
--- /dev/null
+++ b/connect/runtime/src/test/resources/test-plugins/service-loader/test/plugins/ServiceLoadedSubclass.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.plugins;
+
+/**
+ * Instance of a service loaded class
+ */
+public class ServiceLoadedSubclass extends ServiceLoadedClass {
+
+  private static final ClassLoader STATIC_CLASS_LOADER;
+  private final ClassLoader classloader;
+
+  static {
+    STATIC_CLASS_LOADER = Thread.currentThread().getContextClassLoader();
+  }
+
+  {
+    classloader = Thread.currentThread().getContextClassLoader();
+  }
+
+  @Override
+  public ClassLoader staticClassloader() {
+    return STATIC_CLASS_LOADER;
+  }
+
+  @Override
+  public ClassLoader classloader() {
+    return classloader;
+  }
+
+}
diff --git a/connect/runtime/src/test/resources/test-plugins/service-loader/test/plugins/ServiceLoaderPlugin.java b/connect/runtime/src/test/resources/test-plugins/service-loader/test/plugins/ServiceLoaderPlugin.java
new file mode 100644
index 0000000000000..e6371baf56aad
--- /dev/null
+++ b/connect/runtime/src/test/resources/test-plugins/service-loader/test/plugins/ServiceLoaderPlugin.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.plugins;
+
+import java.util.Map;
+import java.util.HashMap;
+import java.util.ServiceLoader;
+import java.util.Iterator;
+import org.apache.kafka.connect.data.Schema;
+import org.apache.kafka.connect.data.SchemaAndValue;
+import org.apache.kafka.connect.storage.Converter;
+import org.apache.kafka.connect.runtime.isolation.SamplingTestPlugin;
+
+/**
+ * Samples data about its initialization environment for later analysis
+ */
+public class ServiceLoaderPlugin extends SamplingTestPlugin implements Converter {
+
+  private static final ClassLoader STATIC_CLASS_LOADER;
+  private static final Map SAMPLES;
+  private final ClassLoader classloader;
+
+  static {
+    STATIC_CLASS_LOADER = Thread.currentThread().getContextClassLoader();
+    SAMPLES = new HashMap<>();
+    Iterator it = ServiceLoader.load(ServiceLoadedClass.class).iterator();
+    while (it.hasNext()) {
+      ServiceLoadedClass loaded = it.next();
+      SAMPLES.put(loaded.getClass().getSimpleName() + ".static", loaded);
+    }
+  }
+
+  {
+    classloader = Thread.currentThread().getContextClassLoader();
+    Iterator it = ServiceLoader.load(ServiceLoadedClass.class).iterator();
+    while (it.hasNext()) {
+      ServiceLoadedClass loaded = it.next();
+      SAMPLES.put(loaded.getClass().getSimpleName() + ".dynamic", loaded);
+    }
+  }
+
+  @Override
+  public void configure(final Map configs, final boolean isKey) {
+  }
+
+  @Override
+  public byte[] fromConnectData(final String topic, final Schema schema, final Object value) {
+    return new byte[0];
+  }
+
+  @Override
+  public SchemaAndValue toConnectData(final String topic, final byte[] value) {
+    return null;
+  }
+
+  @Override
+  public ClassLoader staticClassloader() {
+    return STATIC_CLASS_LOADER;
+  }
+
+  @Override
+  public ClassLoader classloader() {
+    return classloader;
+  }
+
+  @Override
+  public Map otherSamples() {
+    return SAMPLES;
+  }
+}

From 40b843310de1b18943387fa4c548c498f2def0c5 Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Wed, 16 Oct 2019 22:55:20 -0500
Subject: [PATCH 0735/1071] MINOR: Improve FK Join docs and optimize null-fk
 case (#7536)

Fix the formatting and wording of the foreign-key join javadoc
Optimize handling of null extracted foreign keys

Reviewers: Sophie Blee-Goldman , Bill Bejeck 
---
 .../apache/kafka/streams/kstream/KTable.java  | 96 ++++++++++---------
 ...JoinSubscriptionSendProcessorSupplier.java | 49 ++++++++--
 2 files changed, 91 insertions(+), 54 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java
index 21e42df688ee0..5092d35f9e1d0 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java
@@ -2120,19 +2120,20 @@  KTable outerJoin(final KTable other,
                                      final Materialized> materialized);
 
     /**
+     * Join records of this {@code KTable} with another {@code KTable} using non-windowed inner join.
+     * 
                
+     * This is a foreign key join, where the joining key is determined by the {@code foreignKeyExtractor}.
      *
-     * Join records of this [[KTable]] with another [[KTable]]'s records using non-windowed inner join. Records from this
-     * table are joined according to the result of keyExtractor on the other KTable.
-     *
-     * @param other  the other {@code KTable} to be joined with this {@code KTable}. Keyed by KO.
-     * @param foreignKeyExtractor a {@link Function} that extracts the key (KO) from this table's value (V)
-     * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records
-     * @param named     a {@link Named} config used to name the processor in the topology
-     * @param materialized  a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
-     *                      should be materialized. Cannot be {@code null}
-     * @param  the value type of the result {@code KTable}
-     * @param  the key type of the other {@code KTable}
-     * @param  the value type of the other {@code KTable}
+     * @param other               the other {@code KTable} to be joined with this {@code KTable}. Keyed by KO.
+     * @param foreignKeyExtractor a {@link Function} that extracts the key (KO) from this table's value (V). If the
+     *                            result is null, the update is ignored as invalid.
+     * @param joiner              a {@link ValueJoiner} that computes the join result for a pair of matching records
+     * @param named               a {@link Named} config used to name the processor in the topology
+     * @param materialized        a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
+     *                            should be materialized. Cannot be {@code null}
+     * @param                 the value type of the result {@code KTable}
+     * @param                 the key type of the other {@code KTable}
+     * @param                 the value type of the other {@code KTable}
      * @return
      */
      KTable join(final KTable other,
@@ -2142,18 +2143,19 @@  KTable join(final KTable other,
                                     final Materialized> materialized);
 
     /**
+     * Join records of this {@code KTable} with another {@code KTable} using non-windowed inner join.
+     * 
                
+     * This is a foreign key join, where the joining key is determined by the {@code foreignKeyExtractor}.
      *
-     * Join records of this [[KTable]] with another [[KTable]]'s records using non-windowed inner join. Records from this
-     * table are joined according to the result of keyExtractor on the other KTable.
-     *
-     * @param other  the other {@code KTable} to be joined with this {@code KTable}. Keyed by KO.
-     * @param foreignKeyExtractor a {@link Function} that extracts the key (KO) from this table's value (V)
-     * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records
-     * @param materialized  a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
-     *                      should be materialized. Cannot be {@code null}
-     * @param  the value type of the result {@code KTable}
-     * @param  the key type of the other {@code KTable}
-     * @param  the value type of the other {@code KTable}
+     * @param other               the other {@code KTable} to be joined with this {@code KTable}. Keyed by KO.
+     * @param foreignKeyExtractor a {@link Function} that extracts the key (KO) from this table's value (V). If the
+     *                            result is null, the update is ignored as invalid.
+     * @param joiner              a {@link ValueJoiner} that computes the join result for a pair of matching records
+     * @param materialized        a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
+     *                            should be materialized. Cannot be {@code null}
+     * @param                 the value type of the result {@code KTable}
+     * @param                 the key type of the other {@code KTable}
+     * @param                 the value type of the other {@code KTable}
      * @return
      */
      KTable join(final KTable other,
@@ -2162,20 +2164,20 @@  KTable join(final KTable other,
                                     final Materialized> materialized);
 
     /**
+     * Join records of this {@code KTable} with another {@code KTable} using non-windowed left join.
+     * 
                
+     * This is a foreign key join, where the joining key is determined by the {@code foreignKeyExtractor}.
      *
-     * Join records of this [[KTable]] with another [[KTable]]'s records using non-windowed left join. Records from this
-     * table are joined according to the result of keyExtractor on the other KTable.
-     *
-     * @param other  the other {@code KTable} to be joined with this {@code KTable}. Keyed by KO.
-     * @param foreignKeyExtractor a {@link Function} that extracts the key (KO) from this table's value (V). If the
-     *      *                            resultant foreignKey is null, the record will not propagate to the output.
-     * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records
-     * @param named     a {@link Named} config used to name the processor in the topology
-     * @param materialized  a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
-     *                      should be materialized. Cannot be {@code null}
-     * @param  the value type of the result {@code KTable}
-     * @param  the key type of the other {@code KTable}
-     * @param  the value type of the other {@code KTable}
+     * @param other               the other {@code KTable} to be joined with this {@code KTable}. Keyed by KO.
+     * @param foreignKeyExtractor a {@link Function} that extracts the key (KO) from this table's value (V) If the
+     *                            result is null, the update is ignored as invalid.
+     * @param joiner              a {@link ValueJoiner} that computes the join result for a pair of matching records
+     * @param named               a {@link Named} config used to name the processor in the topology
+     * @param materialized        a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
+     *                            should be materialized. Cannot be {@code null}
+     * @param                 the value type of the result {@code KTable}
+     * @param                 the key type of the other {@code KTable}
+     * @param                 the value type of the other {@code KTable}
      * @return a {@code KTable} that contains only those records that satisfy the given predicate
      */
      KTable leftJoin(final KTable other,
@@ -2185,19 +2187,19 @@  KTable leftJoin(final KTable other,
                                         final Materialized> materialized);
 
     /**
+     * Join records of this {@code KTable} with another {@code KTable} using non-windowed left join.
+     * 
                
+     * This is a foreign key join, where the joining key is determined by the {@code foreignKeyExtractor}.
      *
-     * Join records of this [[KTable]] with another [[KTable]]'s records using non-windowed left join. Records from this
-     * table are joined according to the result of keyExtractor on the other KTable.
-     *
-     * @param other  the other {@code KTable} to be joined with this {@code KTable}. Keyed by KO.
+     * @param other               the other {@code KTable} to be joined with this {@code KTable}. Keyed by KO.
      * @param foreignKeyExtractor a {@link Function} that extracts the key (KO) from this table's value (V). If the
-     *                            resultant foreignKey is null, the record will not propagate to the output.
-     * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records
-     * @param materialized  a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
-     *                      should be materialized. Cannot be {@code null}
-     * @param  the value type of the result {@code KTable}
-     * @param  the key type of the other {@code KTable}
-     * @param  the value type of the other {@code KTable}
+     *                            result is null, the update is ignored as invalid.
+     * @param joiner              a {@link ValueJoiner} that computes the join result for a pair of matching records
+     * @param materialized        a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
+     *                            should be materialized. Cannot be {@code null}
+     * @param                 the value type of the result {@code KTable}
+     * @param                 the key type of the other {@code KTable}
+     * @param                 the value type of the other {@code KTable}
      * @return a {@code KTable} that contains only those records that satisfy the given predicate
      */
      KTable leftJoin(final KTable other,
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionSendProcessorSupplier.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionSendProcessorSupplier.java
index f122258d93049..806cf5c98fd15 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionSendProcessorSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionSendProcessorSupplier.java
@@ -17,6 +17,7 @@
 
 package org.apache.kafka.streams.kstream.internals.foreignkeyjoin;
 
+import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serializer;
 import org.apache.kafka.streams.kstream.internals.Change;
@@ -24,17 +25,22 @@
 import org.apache.kafka.streams.processor.Processor;
 import org.apache.kafka.streams.processor.ProcessorContext;
 import org.apache.kafka.streams.processor.ProcessorSupplier;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.processor.internals.metrics.ThreadMetrics;
 import org.apache.kafka.streams.state.internals.Murmur3;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-import java.util.function.Function;
 import java.util.Arrays;
+import java.util.function.Function;
 
-import static org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionWrapper.Instruction.PROPAGATE_ONLY_IF_FK_VAL_AVAILABLE;
-import static org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionWrapper.Instruction.PROPAGATE_NULL_IF_NO_FK_VAL_AVAILABLE;
 import static org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionWrapper.Instruction.DELETE_KEY_AND_PROPAGATE;
 import static org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionWrapper.Instruction.DELETE_KEY_NO_PROPAGATE;
+import static org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionWrapper.Instruction.PROPAGATE_NULL_IF_NO_FK_VAL_AVAILABLE;
+import static org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionWrapper.Instruction.PROPAGATE_ONLY_IF_FK_VAL_AVAILABLE;
 
 public class ForeignJoinSubscriptionSendProcessorSupplier implements ProcessorSupplier> {
+    private static final Logger LOG = LoggerFactory.getLogger(ForeignJoinSubscriptionSendProcessorSupplier.class);
 
     private final Function foreignKeyExtractor;
     private final String repartitionTopicName;
@@ -60,7 +66,9 @@ public Processor> get() {
     }
 
     private class UnbindChangeProcessor extends AbstractProcessor> {
-        
+
+        private Sensor skippedRecordsSensor;
+
         @SuppressWarnings("unchecked")
         @Override
         public void init(final ProcessorContext context) {
@@ -69,18 +77,36 @@ public void init(final ProcessorContext context) {
             if (foreignKeySerializer == null) {
                 foreignKeySerializer = (Serializer) context.keySerde().serializer();
             }
+            skippedRecordsSensor = ThreadMetrics.skipRecordSensor(Thread.currentThread().getName(),
+                                                                  (StreamsMetricsImpl) context.metrics());
         }
 
         @Override
         public void process(final K key, final Change change) {
             final long[] currentHash = change.newValue == null ?
-                    null :
-                    Murmur3.hash128(valueSerializer.serialize(repartitionTopicName, change.newValue));
+                null :
+                Murmur3.hash128(valueSerializer.serialize(repartitionTopicName, change.newValue));
 
             if (change.oldValue != null) {
                 final KO oldForeignKey = foreignKeyExtractor.apply(change.oldValue);
+                if (oldForeignKey == null) {
+                    LOG.warn(
+                        "Skipping record due to null foreign key. value=[{}] topic=[{}] partition=[{}] offset=[{}]",
+                        change.oldValue, context().topic(), context().partition(), context().offset()
+                    );
+                    skippedRecordsSensor.record();
+                    return;
+                }
                 if (change.newValue != null) {
                     final KO newForeignKey = foreignKeyExtractor.apply(change.newValue);
+                    if (newForeignKey == null) {
+                        LOG.warn(
+                            "Skipping record due to null foreign key. value=[{}] topic=[{}] partition=[{}] offset=[{}]",
+                            change.newValue, context().topic(), context().partition(), context().offset()
+                        );
+                        skippedRecordsSensor.record();
+                        return;
+                    }
 
                     final byte[] serialOldForeignKey = foreignKeySerializer.serialize(repartitionTopicName, oldForeignKey);
                     final byte[] serialNewForeignKey = foreignKeySerializer.serialize(repartitionTopicName, newForeignKey);
@@ -109,7 +135,16 @@ public void process(final K key, final Change change) {
                 } else {
                     instruction = PROPAGATE_ONLY_IF_FK_VAL_AVAILABLE;
                 }
-                context().forward(foreignKeyExtractor.apply(change.newValue), new SubscriptionWrapper<>(currentHash, instruction, key));
+                final KO newForeignKey = foreignKeyExtractor.apply(change.newValue);
+                if (newForeignKey == null) {
+                    LOG.warn(
+                        "Skipping record due to null foreign key. value=[{}] topic=[{}] partition=[{}] offset=[{}]",
+                        change.newValue, context().topic(), context().partition(), context().offset()
+                    );
+                    skippedRecordsSensor.record();
+                } else {
+                    context().forward(newForeignKey, new SubscriptionWrapper<>(currentHash, instruction, key));
+                }
             }
         }
     }

From 8c1d33fba1b2ab22818e0a91d9758dadecb20118 Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Wed, 16 Oct 2019 22:23:15 -0700
Subject: [PATCH 0736/1071] KAFKA-9053: AssignmentInfo#encode hardcodes the
 LATEST_SUPPORTED_VERSION (#7537)

Also put in some additional logging that makes sense to add, and proved helpful in debugging this particular issue.

Unit tests verifying the encoded supported version were added.

This should get cherry-picked back to 2.1

Reviewers: Bill Bejeck , Guozhang Wang 
---
 .../internals/StreamsPartitionAssignor.java   | 22 +++++++++++++++----
 .../internals/assignment/AssignmentInfo.java  | 22 +++++++++----------
 .../assignment/SubscriptionInfo.java          |  4 ++--
 .../assignment/AssignmentInfoTest.java        |  9 ++++++++
 .../assignment/SubscriptionInfoTest.java      | 17 ++++++++++++++
 5 files changed, 57 insertions(+), 17 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
index 8b2c95a8e989e..78ee40c422344 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
@@ -383,10 +383,15 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr
         }
 
         if (minReceivedMetadataVersion < LATEST_SUPPORTED_VERSION) {
-            log.info("Downgrading metadata to version {}. Latest supported version is {}.",
+            log.info("Downgrade metadata to version {}. Latest supported version is {}.",
                 minReceivedMetadataVersion,
                 LATEST_SUPPORTED_VERSION);
         }
+        if (minSupportedMetadataVersion < LATEST_SUPPORTED_VERSION) {
+            log.info("Downgrade latest supported metadata to version {}. Latest supported version is {}.",
+                minSupportedMetadataVersion,
+                LATEST_SUPPORTED_VERSION);
+        }
 
         log.debug("Constructed client metadata {} from the member subscriptions.", clientMetadataMap);
 
@@ -1055,9 +1060,10 @@ protected boolean maybeUpdateSubscriptionVersion(final int receivedAssignmentMet
                 log.info(
                     "Sent a version {} subscription and got version {} assignment back (successful version probing). "
                         +
-                        "Downgrade subscription metadata to commonly supported version and trigger new rebalance.",
+                        "Downgrade subscription metadata to commonly supported version {} and trigger new rebalance.",
                     usedSubscriptionMetadataVersion,
-                    receivedAssignmentMetadataVersion
+                    receivedAssignmentMetadataVersion,
+                    latestCommonlySupportedVersion
                 );
                 usedSubscriptionMetadataVersion = latestCommonlySupportedVersion;
                 return true;
@@ -1237,7 +1243,15 @@ private int updateMinReceivedVersion(final int usedVersion, final int minReceive
     }
 
     private int updateMinSupportedVersion(final int supportedVersion, final int minSupportedMetadataVersion) {
-        return supportedVersion < minSupportedMetadataVersion ? supportedVersion : minSupportedMetadataVersion;
+        if (supportedVersion < minSupportedMetadataVersion) {
+            log.debug("Downgrade the current minimum supported version {} to the smaller seen supported version {}",
+                minSupportedMetadataVersion, supportedVersion);
+            return supportedVersion;
+        } else {
+            log.debug("Current minimum supported version remains at {}, last seen supported version was {}",
+                minSupportedMetadataVersion, supportedVersion);
+            return minSupportedMetadataVersion;
+        }
     }
 
     protected void setAssignmentErrorCode(final Integer errorCode) {
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfo.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfo.java
index dcaaa9a7fec39..a36391e4f6809 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfo.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfo.java
@@ -145,7 +145,7 @@ public ByteBuffer encode() {
                     break;
                 default:
                     throw new IllegalStateException("Unknown metadata version: " + usedVersion
-                        + "; latest supported version: " + LATEST_SUPPORTED_VERSION);
+                        + "; latest commonly supported version: " + commonlySupportedVersion);
             }
 
             out.flush();
@@ -245,14 +245,14 @@ private void writeTopicPartitions(final DataOutputStream out,
 
     private void encodeVersionThree(final DataOutputStream out) throws IOException {
         out.writeInt(3);
-        out.writeInt(LATEST_SUPPORTED_VERSION);
+        out.writeInt(commonlySupportedVersion);
         encodeActiveAndStandbyTaskAssignment(out);
         encodePartitionsByHost(out);
     }
 
     private void encodeVersionFour(final DataOutputStream out) throws IOException {
         out.writeInt(4);
-        out.writeInt(LATEST_SUPPORTED_VERSION);
+        out.writeInt(commonlySupportedVersion);
         encodeActiveAndStandbyTaskAssignment(out);
         encodePartitionsByHost(out);
         out.writeInt(errCode);
@@ -260,7 +260,7 @@ private void encodeVersionFour(final DataOutputStream out) throws IOException {
 
     private void encodeVersionFive(final DataOutputStream out) throws IOException {
         out.writeInt(5);
-        out.writeInt(LATEST_SUPPORTED_VERSION);
+        out.writeInt(commonlySupportedVersion);
         encodeActiveAndStandbyTaskAssignment(out);
         encodePartitionsByHostAsDictionary(out);
         out.writeInt(errCode);
@@ -277,7 +277,7 @@ public static AssignmentInfo decode(final ByteBuffer data) {
             final AssignmentInfo assignmentInfo;
 
             final int usedVersion = in.readInt();
-            final int latestSupportedVersion;
+            final int commonlySupportedVersion;
             switch (usedVersion) {
                 case 1:
                     assignmentInfo = new AssignmentInfo(usedVersion, UNKNOWN);
@@ -288,18 +288,18 @@ public static AssignmentInfo decode(final ByteBuffer data) {
                     decodeVersionTwoData(assignmentInfo, in);
                     break;
                 case 3:
-                    latestSupportedVersion = in.readInt();
-                    assignmentInfo = new AssignmentInfo(usedVersion, latestSupportedVersion);
+                    commonlySupportedVersion = in.readInt();
+                    assignmentInfo = new AssignmentInfo(usedVersion, commonlySupportedVersion);
                     decodeVersionThreeData(assignmentInfo, in);
                     break;
                 case 4:
-                    latestSupportedVersion = in.readInt();
-                    assignmentInfo = new AssignmentInfo(usedVersion, latestSupportedVersion);
+                    commonlySupportedVersion = in.readInt();
+                    assignmentInfo = new AssignmentInfo(usedVersion, commonlySupportedVersion);
                     decodeVersionFourData(assignmentInfo, in);
                     break;
                 case 5:
-                    latestSupportedVersion = in.readInt();
-                    assignmentInfo = new AssignmentInfo(usedVersion, latestSupportedVersion);
+                    commonlySupportedVersion = in.readInt();
+                    assignmentInfo = new AssignmentInfo(usedVersion, commonlySupportedVersion);
                     decodeVersionFiveData(assignmentInfo, in);
                     break;
                 default:
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfo.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfo.java
index 81430ff605311..20f2ce001b4b1 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfo.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfo.java
@@ -212,7 +212,7 @@ private ByteBuffer encodeVersionThreeFourAndFive(final int usedVersion) {
         final ByteBuffer buf = ByteBuffer.allocate(getVersionThreeFourAndFiveByteLength(endPointBytes));
 
         buf.putInt(usedVersion); // used version
-        buf.putInt(LATEST_SUPPORTED_VERSION); // supported version
+        buf.putInt(latestSupportedVersion); // supported version
         encodeClientUUID(buf);
         encodeTasks(buf, prevTasks);
         encodeTasks(buf, standbyTasks);
@@ -272,7 +272,7 @@ public static SubscriptionInfo decode(final ByteBuffer data) {
             default:
                 latestSupportedVersion = data.getInt();
                 subscriptionInfo = new SubscriptionInfo(usedVersion, latestSupportedVersion);
-                log.info("Unable to decode subscription data: used version: {}; latest supported version: {}", usedVersion, LATEST_SUPPORTED_VERSION);
+                log.info("Unable to decode subscription data: used version: {}; latest supported version: {}", usedVersion, latestSupportedVersion);
         }
 
         return subscriptionInfo;
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfoTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfoTest.java
index b65994ef3c184..e9f02370dd8ed 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfoTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfoTest.java
@@ -103,4 +103,13 @@ public void shouldEncodeAndDecodeVersion5() {
         final AssignmentInfo expectedInfo = new AssignmentInfo(5, LATEST_SUPPORTED_VERSION, activeTasks, standbyTasks, globalAssignment, 2);
         assertEquals(expectedInfo, AssignmentInfo.decode(info.encode()));
     }
+
+    @Test
+    public void shouldEncodeAndDecodeSmallerCommonlySupportedVersion() {
+        final int usedVersion = LATEST_SUPPORTED_VERSION - 1;
+        final int commonlySupportedVersion = LATEST_SUPPORTED_VERSION - 1;
+        final AssignmentInfo info = new AssignmentInfo(usedVersion, commonlySupportedVersion, activeTasks, standbyTasks, globalAssignment, 2);
+        final AssignmentInfo expectedInfo = new AssignmentInfo(usedVersion, commonlySupportedVersion, activeTasks, standbyTasks, globalAssignment, 2);
+        assertEquals(expectedInfo, AssignmentInfo.decode(info.encode()));
+    }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfoTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfoTest.java
index a6150c054c0bd..a0806a6e8caee 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfoTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfoTest.java
@@ -85,6 +85,13 @@ public void shouldEncodeAndDecodeVersion4() {
         assertEquals(expectedInfo, SubscriptionInfo.decode(info.encode()));
     }
 
+    @Test
+    public void shouldEncodeAndDecodeVersion5() {
+        final SubscriptionInfo info = new SubscriptionInfo(5, processId, activeTasks, standbyTasks, "localhost:80");
+        final SubscriptionInfo expectedInfo = new SubscriptionInfo(5, LATEST_SUPPORTED_VERSION, processId, activeTasks, standbyTasks, "localhost:80");
+        assertEquals(expectedInfo, SubscriptionInfo.decode(info.encode()));
+    }
+
     @Test
     public void shouldAllowToDecodeFutureSupportedVersion() {
         final SubscriptionInfo info = SubscriptionInfo.decode(encodeFutureVersion());
@@ -92,6 +99,16 @@ public void shouldAllowToDecodeFutureSupportedVersion() {
         assertEquals(LATEST_SUPPORTED_VERSION + 1, info.latestSupportedVersion());
     }
 
+    @Test
+    public void shouldEncodeAndDecodeSmallerLatestSupportedVersion() {
+        final int usedVersion = LATEST_SUPPORTED_VERSION - 1;
+        final int latestSupportedVersion = LATEST_SUPPORTED_VERSION - 1;
+
+        final SubscriptionInfo info = new SubscriptionInfo(usedVersion, latestSupportedVersion, processId, activeTasks, standbyTasks, "localhost:80");
+        final SubscriptionInfo expectedInfo = new SubscriptionInfo(usedVersion, latestSupportedVersion, processId, activeTasks, standbyTasks, "localhost:80");
+        assertEquals(expectedInfo, SubscriptionInfo.decode(info.encode()));
+    }
+
     private ByteBuffer encodeFutureVersion() {
         final ByteBuffer buf = ByteBuffer.allocate(4 /* used version */
                                                    + 4 /* supported version */);

From f38d47bf074ec5c23576b0124068d3b8206c903e Mon Sep 17 00:00:00 2001
From: Bill Bejeck 
Date: Thu, 17 Oct 2019 01:29:33 -0400
Subject: [PATCH 0737/1071] KAFKA-8496: System test for KIP-429 upgrades and
 compatibility (#7529)

Reviewers: A. Sophie Blee-Goldman , Guozhang Wang 
---
 .../assignment/AssignorConfiguration.java     |   4 +-
 ...eamsUpgradeToCooperativeRebalanceTest.java | 136 ++++++++++++
 ...eamsUpgradeToCooperativeRebalanceTest.java |  90 ++++++++
 ...eamsUpgradeToCooperativeRebalanceTest.java |  89 ++++++++
 ...eamsUpgradeToCooperativeRebalanceTest.java |  85 +++++++
 ...eamsUpgradeToCooperativeRebalanceTest.java |  87 ++++++++
 ...eamsUpgradeToCooperativeRebalanceTest.java | 136 ++++++++++++
 ...eamsUpgradeToCooperativeRebalanceTest.java | 136 ++++++++++++
 ...eamsUpgradeToCooperativeRebalanceTest.java | 136 ++++++++++++
 ...eamsUpgradeToCooperativeRebalanceTest.java | 136 ++++++++++++
 ...eamsUpgradeToCooperativeRebalanceTest.java | 136 ++++++++++++
 ...eamsUpgradeToCooperativeRebalanceTest.java | 125 +++++++++++
 tests/kafkatest/services/streams.py           |  85 +++++++
 ...eams_cooperative_rebalance_upgrade_test.py | 209 ++++++++++++++++++
 tests/kafkatest/version.py                    |   1 +
 15 files changed, 1589 insertions(+), 2 deletions(-)
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
 create mode 100644 streams/upgrade-system-tests-0100/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
 create mode 100644 streams/upgrade-system-tests-0101/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
 create mode 100644 streams/upgrade-system-tests-0102/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
 create mode 100644 streams/upgrade-system-tests-0110/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
 create mode 100644 streams/upgrade-system-tests-10/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
 create mode 100644 streams/upgrade-system-tests-11/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
 create mode 100644 streams/upgrade-system-tests-20/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
 create mode 100644 streams/upgrade-system-tests-21/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
 create mode 100644 streams/upgrade-system-tests-22/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
 create mode 100644 streams/upgrade-system-tests-23/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
 create mode 100644 tests/kafkatest/tests/streams/streams_cooperative_rebalance_upgrade_test.py

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorConfiguration.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorConfiguration.java
index 1e406e25b0ca5..be5670778fde4 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorConfiguration.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignorConfiguration.java
@@ -147,13 +147,13 @@ public RebalanceProtocol rebalanceProtocol() {
                 case StreamsConfig.UPGRADE_FROM_21:
                 case StreamsConfig.UPGRADE_FROM_22:
                 case StreamsConfig.UPGRADE_FROM_23:
-                    log.info("Turning off cooperative rebalancing for upgrade from {}.x", upgradeFrom);
+                    log.info("Eager rebalancing enabled now for upgrade from {}.x", upgradeFrom);
                     return RebalanceProtocol.EAGER;
                 default:
                     throw new IllegalArgumentException("Unknown configuration value for parameter 'upgrade.from': " + upgradeFrom);
             }
         }
-
+        log.info("Cooperative rebalancing enabled now");
         return RebalanceProtocol.COOPERATIVE;
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
new file mode 100644
index 0000000000000..df7359388dbcf
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.tests;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.KafkaStreams.State;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.kstream.ForeachAction;
+import org.apache.kafka.streams.processor.TaskMetadata;
+import org.apache.kafka.streams.processor.ThreadMetadata;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+
+public class StreamsUpgradeToCooperativeRebalanceTest {
+
+
+    @SuppressWarnings("unchecked")
+    public static void main(final String[] args) throws Exception {
+        if (args.length < 1) {
+            System.err.println("StreamsUpgradeToCooperativeRebalanceTest requires one argument (properties-file) but no args provided");
+        }
+        System.out.println("Args are " + Arrays.toString(args));
+        final String propFileName = args[0];
+        final Properties streamsProperties = Utils.loadProps(propFileName);
+
+        final Properties config = new Properties();
+        System.out.println("StreamsTest instance started (StreamsUpgradeToCooperativeRebalanceTest)");
+        System.out.println("props=" + streamsProperties);
+
+        config.put(StreamsConfig.APPLICATION_ID_CONFIG, "cooperative-rebalance-upgrade");
+        config.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000);
+        config.putAll(streamsProperties);
+
+        final String sourceTopic = streamsProperties.getProperty("source.topic", "source");
+        final String sinkTopic = streamsProperties.getProperty("sink.topic", "sink");
+        final String taskDelimiter = "#";
+        final int reportInterval = Integer.parseInt(streamsProperties.getProperty("report.interval", "100"));
+        final String upgradePhase = streamsProperties.getProperty("upgrade.phase", "");
+
+        final StreamsBuilder builder = new StreamsBuilder();
+
+        builder.stream(sourceTopic)
+            .peek(new ForeachAction() {
+                int recordCounter = 0;
+
+                @Override
+                public void apply(final String key, final String value) {
+                    if (recordCounter++ % reportInterval == 0) {
+                        System.out.println(String.format("%sProcessed %d records so far", upgradePhase, recordCounter));
+                        System.out.flush();
+                    }
+                }
+            }
+            ).to(sinkTopic);
+
+        final KafkaStreams streams = new KafkaStreams(builder.build(), config);
+
+        streams.setStateListener((newState, oldState) -> {
+            if (newState == State.RUNNING && oldState == State.REBALANCING) {
+                System.out.println(String.format("%sSTREAMS in a RUNNING State", upgradePhase));
+                final Set allThreadMetadata = streams.localThreadsMetadata();
+                final StringBuilder taskReportBuilder = new StringBuilder();
+                final List activeTasks = new ArrayList<>();
+                final List standbyTasks = new ArrayList<>();
+                for (final ThreadMetadata threadMetadata : allThreadMetadata) {
+                    getTasks(threadMetadata.activeTasks(), activeTasks);
+                    if (!threadMetadata.standbyTasks().isEmpty()) {
+                        getTasks(threadMetadata.standbyTasks(), standbyTasks);
+                    }
+                }
+                addTasksToBuilder(activeTasks, taskReportBuilder);
+                taskReportBuilder.append(taskDelimiter);
+                if (!standbyTasks.isEmpty()) {
+                    addTasksToBuilder(standbyTasks, taskReportBuilder);
+                }
+                System.out.println("TASK-ASSIGNMENTS:" + taskReportBuilder);
+            }
+
+            if (newState == State.REBALANCING) {
+                System.out.println(String.format("%sStarting a REBALANCE", upgradePhase));
+            }
+        });
+
+
+        streams.start();
+
+        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            streams.close();
+            System.out.println(String.format("%sCOOPERATIVE-REBALANCE-TEST-CLIENT-CLOSED", upgradePhase));
+            System.out.flush();
+        }));
+    }
+
+    private static void addTasksToBuilder(final List tasks, final StringBuilder builder) {
+        if (!tasks.isEmpty()) {
+            for (final String task : tasks) {
+                builder.append(task).append(",");
+            }
+            builder.setLength(builder.length() - 1);
+        }
+    }
+
+    private static void getTasks(final Set taskMetadata,
+                                 final List taskList) {
+        for (final TaskMetadata task : taskMetadata) {
+            final Set topicPartitions = task.topicPartitions();
+            for (final TopicPartition topicPartition : topicPartitions) {
+                taskList.add(topicPartition.toString());
+            }
+        }
+    }
+}
diff --git a/streams/upgrade-system-tests-0100/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java b/streams/upgrade-system-tests-0100/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
new file mode 100644
index 0000000000000..da63d9a61ad2d
--- /dev/null
+++ b/streams/upgrade-system-tests-0100/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.tests;
+
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.kstream.ForeachAction;
+import org.apache.kafka.streams.kstream.KStream;
+import org.apache.kafka.streams.kstream.KStreamBuilder;
+
+import java.util.Properties;
+
+public class StreamsUpgradeToCooperativeRebalanceTest {
+
+
+    @SuppressWarnings("unchecked")
+    public static void main(final String[] args) throws Exception {
+        if (args.length < 3) {
+            System.err.println("StreamsUpgradeTest requires three argument (kafka-url, zookeeper-url, properties-file) but only " + args.length + " provided: "
+                + (args.length > 0 ? args[0] + " " : "")
+                + (args.length > 1 ? args[1] : ""));
+        }
+
+        final String zookeeper = args[1];
+        final String propFileName = args.length > 2 ? args[2] : null;
+
+        final Properties streamsProperties = Utils.loadProps(propFileName);
+        final Properties config = new Properties();
+
+        System.out.println("StreamsTest instance started (StreamsUpgradeToCooperativeRebalanceTest v0.10.0)");
+        System.out.println("zookeeper=" + zookeeper);
+        System.out.println("props=" + config);
+
+        config.put(StreamsConfig.APPLICATION_ID_CONFIG, "cooperative-rebalance-upgrade");
+        config.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.setProperty(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG, zookeeper);
+        config.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000);
+        config.putAll(streamsProperties);
+
+        final String sourceTopic = config.getProperty("source.topic", "source");
+        final String sinkTopic = config.getProperty("sink.topic", "sink");
+        final int reportInterval = Integer.parseInt(config.getProperty("report.interval", "100"));
+        final String upgradePhase = config.getProperty("upgrade.phase",  "");
+
+        final KStreamBuilder builder = new KStreamBuilder();
+
+        final KStream upgradeStream = builder.stream(sourceTopic);
+        upgradeStream.foreach(new ForeachAction() {
+            int recordCounter = 0;
+
+            @Override
+            public void apply(final String key, final String value) {
+                if (recordCounter++ % reportInterval == 0) {
+                    System.out.println(String.format("%sProcessed %d records so far", upgradePhase, recordCounter));
+                    System.out.flush();
+                }
+            }
+        }
+        );
+        upgradeStream.to(sinkTopic);
+
+        final KafkaStreams streams = new KafkaStreams(builder, config);
+
+
+        streams.start();
+
+        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            streams.close();
+            System.out.println(String.format("%sCOOPERATIVE-REBALANCE-TEST-CLIENT-CLOSED", upgradePhase));
+            System.out.flush();
+        }));
+    }
+}
diff --git a/streams/upgrade-system-tests-0101/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java b/streams/upgrade-system-tests-0101/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
new file mode 100644
index 0000000000000..70ad666362979
--- /dev/null
+++ b/streams/upgrade-system-tests-0101/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.tests;
+
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.kstream.ForeachAction;
+import org.apache.kafka.streams.kstream.KStream;
+import org.apache.kafka.streams.kstream.KStreamBuilder;
+
+import java.util.Properties;
+
+public class StreamsUpgradeToCooperativeRebalanceTest {
+
+
+    @SuppressWarnings("unchecked")
+    public static void main(final String[] args) throws Exception {
+        if (args.length < 3) {
+            System.err.println("StreamsUpgradeTest requires three argument (kafka-url, zookeeper-url, properties-file) but only " + args.length + " provided: "
+                + (args.length > 0 ? args[0] + " " : "")
+                + (args.length > 1 ? args[1] : ""));
+        }
+        final String zookeeper = args[1];
+        final String propFileName = args.length > 2 ? args[2] : null;
+
+        final Properties streamsProperties = Utils.loadProps(propFileName);
+        final Properties config = new Properties();
+
+        System.out.println("StreamsTest instance started (StreamsUpgradeToCooperativeRebalanceTest v0.10.1)");
+        System.out.println("zookeeper=" + zookeeper);
+        System.out.println("props=" + config);
+
+        config.put(StreamsConfig.APPLICATION_ID_CONFIG, "cooperative-rebalance-upgrade");
+        config.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.setProperty(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG, zookeeper);
+        config.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000);
+        config.putAll(streamsProperties);
+
+        final String sourceTopic = config.getProperty("source.topic", "source");
+        final String sinkTopic = config.getProperty("sink.topic", "sink");
+        final int reportInterval = Integer.parseInt(config.getProperty("report.interval", "100"));
+        final String upgradePhase = config.getProperty("upgrade.phase",  "");
+
+        final KStreamBuilder builder = new KStreamBuilder();
+
+        final KStream upgradeStream = builder.stream(sourceTopic);
+        upgradeStream.foreach(new ForeachAction() {
+            int recordCounter = 0;
+
+            @Override
+            public void apply(final String key, final String value) {
+                if (recordCounter++ % reportInterval == 0) {
+                    System.out.println(String.format("%sProcessed %d records so far", upgradePhase, recordCounter));
+                    System.out.flush();
+                }
+            }
+        }
+        );
+        upgradeStream.to(sinkTopic);
+
+        final KafkaStreams streams = new KafkaStreams(builder, config);
+
+
+        streams.start();
+
+        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            streams.close();
+            System.out.println(String.format("%sCOOPERATIVE-REBALANCE-TEST-CLIENT-CLOSED", upgradePhase));
+            System.out.flush();
+        }));
+    }
+}
diff --git a/streams/upgrade-system-tests-0102/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java b/streams/upgrade-system-tests-0102/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
new file mode 100644
index 0000000000000..75256358353f1
--- /dev/null
+++ b/streams/upgrade-system-tests-0102/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.tests;
+
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.kstream.ForeachAction;
+import org.apache.kafka.streams.kstream.KStream;
+import org.apache.kafka.streams.kstream.KStreamBuilder;
+
+import java.util.Properties;
+
+public class StreamsUpgradeToCooperativeRebalanceTest {
+
+
+    @SuppressWarnings("unchecked")
+    public static void main(final String[] args) throws Exception {
+        if (args.length < 2) {
+            System.err.println("StreamsUpgradeTest requires three argument (kafka-url, properties-file) but only " + args.length + " provided: "
+                + (args.length > 0 ? args[0] + " " : ""));
+        }
+        final String propFileName = args[1];
+
+        final Properties streamsProperties = Utils.loadProps(propFileName);
+        final Properties config = new Properties();
+
+        System.out.println("StreamsTest instance started (StreamsUpgradeToCooperativeRebalanceTest v0.10.2)");
+        System.out.println("props=" + config);
+
+        config.put(StreamsConfig.APPLICATION_ID_CONFIG, "cooperative-rebalance-upgrade");
+        config.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000);
+        config.putAll(streamsProperties);
+
+        final String sourceTopic = config.getProperty("source.topic", "source");
+        final String sinkTopic = config.getProperty("sink.topic", "sink");
+        final int reportInterval = Integer.parseInt(config.getProperty("report.interval", "100"));
+        final String upgradePhase = config.getProperty("upgrade.phase",  "");
+
+        final KStreamBuilder builder = new KStreamBuilder();
+
+        final KStream upgradeStream = builder.stream(sourceTopic);
+        upgradeStream.foreach(new ForeachAction() {
+            int recordCounter = 0;
+
+            @Override
+            public void apply(final String key, final String value) {
+                if (recordCounter++ % reportInterval == 0) {
+                    System.out.println(String.format("%sProcessed %d records so far", upgradePhase, recordCounter));
+                    System.out.flush();
+                }
+            }
+        }
+        );
+        upgradeStream.to(sinkTopic);
+
+        final KafkaStreams streams = new KafkaStreams(builder, config);
+
+
+        streams.start();
+
+        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            streams.close();
+            System.out.println(String.format("%sCOOPERATIVE-REBALANCE-TEST-CLIENT-CLOSED", upgradePhase));
+            System.out.flush();
+        }));
+    }
+}
diff --git a/streams/upgrade-system-tests-0110/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java b/streams/upgrade-system-tests-0110/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
new file mode 100644
index 0000000000000..a46596306fd44
--- /dev/null
+++ b/streams/upgrade-system-tests-0110/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.tests;
+
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.kstream.ForeachAction;
+import org.apache.kafka.streams.kstream.KStream;
+import org.apache.kafka.streams.kstream.KStreamBuilder;
+
+import java.util.Properties;
+
+public class StreamsUpgradeToCooperativeRebalanceTest {
+
+
+    @SuppressWarnings("unchecked")
+    public static void main(final String[] args) throws Exception {
+        if (args.length < 2) {
+            System.err.println("StreamsUpgradeTest requires three argument (kafka-url, properties-file) but only " + args.length + " provided: "
+                + (args.length > 0 ? args[0] + " " : ""));
+        }
+        final String kafka = args[0];
+        final String propFileName = args[1];
+
+        final Properties streamsProperties = Utils.loadProps(propFileName);
+        final Properties config = new Properties();
+
+        System.out.println("StreamsTest instance started (StreamsUpgradeToCooperativeRebalanceTest v0.11.0)");
+        System.out.println("kafka=" + kafka);
+        System.out.println("props=" + config);
+
+        config.put(StreamsConfig.APPLICATION_ID_CONFIG, "cooperative-rebalance-upgrade");
+        config.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000);
+        config.putAll(streamsProperties);
+
+        final String sourceTopic = config.getProperty("source.topic", "source");
+        final String sinkTopic = config.getProperty("sink.topic", "sink");
+        final int reportInterval = Integer.parseInt(config.getProperty("report.interval", "100"));
+        final String upgradePhase = config.getProperty("upgrade.phase",  "");
+
+        final KStreamBuilder builder = new KStreamBuilder();
+
+        final KStream upgradeStream = builder.stream(sourceTopic);
+        upgradeStream.foreach(new ForeachAction() {
+            int recordCounter = 0;
+
+            @Override
+            public void apply(final String key, final String value) {
+                if (recordCounter++ % reportInterval == 0) {
+                    System.out.println(String.format("%sProcessed %d records so far", upgradePhase, recordCounter));
+                    System.out.flush();
+                }
+            }
+        }
+        );
+        upgradeStream.to(sinkTopic);
+
+        final KafkaStreams streams = new KafkaStreams(builder, config);
+
+
+        streams.start();
+
+        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            streams.close();
+            System.out.println(String.format("%sCOOPERATIVE-REBALANCE-TEST-CLIENT-CLOSED", upgradePhase));
+            System.out.flush();
+        }));
+    }
+}
diff --git a/streams/upgrade-system-tests-10/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java b/streams/upgrade-system-tests-10/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
new file mode 100644
index 0000000000000..09e8458e0f4ee
--- /dev/null
+++ b/streams/upgrade-system-tests-10/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.tests;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.KafkaStreams.State;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.kstream.ForeachAction;
+import org.apache.kafka.streams.processor.TaskMetadata;
+import org.apache.kafka.streams.processor.ThreadMetadata;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+
+public class StreamsUpgradeToCooperativeRebalanceTest {
+
+
+    @SuppressWarnings("unchecked")
+    public static void main(final String[] args) throws Exception {
+        if (args.length < 2) {
+            System.err.println("StreamsUpgradeToCooperativeRebalanceTest requires two argument (kafka-url, properties-file) but only " + args.length + " provided: "
+                + (args.length > 0 ? args[0] : ""));
+        }
+        System.out.println("Args are " + Arrays.toString(args));
+        final String propFileName = args[1];
+        final Properties streamsProperties = Utils.loadProps(propFileName);
+
+        final Properties config = new Properties();
+        System.out.println("StreamsTest instance started (StreamsUpgradeToCooperativeRebalanceTest v1.0)");
+        System.out.println("props=" + streamsProperties);
+
+        config.put(StreamsConfig.APPLICATION_ID_CONFIG, "cooperative-rebalance-upgrade");
+        config.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000);
+        config.putAll(streamsProperties);
+
+        final String sourceTopic = streamsProperties.getProperty("source.topic", "source");
+        final String sinkTopic = streamsProperties.getProperty("sink.topic", "sink");
+        final String taskDelimiter = streamsProperties.getProperty("task.delimiter", "#");
+        final int reportInterval = Integer.parseInt(streamsProperties.getProperty("report.interval", "100"));
+        final String upgradePhase = streamsProperties.getProperty("upgrade.phase",  "");
+
+        final StreamsBuilder builder = new StreamsBuilder();
+
+        builder.stream(sourceTopic)
+            .peek(new ForeachAction() {
+                int recordCounter = 0;
+
+                @Override
+                public void apply(final String key, final String value) {
+                    if (recordCounter++ % reportInterval == 0) {
+                        System.out.println(String.format("%sProcessed %d records so far", upgradePhase, recordCounter));
+                        System.out.flush();
+                    }
+                }
+            }
+            ).to(sinkTopic);
+
+        final KafkaStreams streams = new KafkaStreams(builder.build(), config);
+
+        streams.setStateListener((newState, oldState) -> {
+            if (newState == State.RUNNING && oldState == State.REBALANCING) {
+                System.out.println(String.format("%sSTREAMS in a RUNNING State", upgradePhase));
+                final Set allThreadMetadata = streams.localThreadsMetadata();
+                final StringBuilder taskReportBuilder = new StringBuilder();
+                final List activeTasks = new ArrayList<>();
+                final List standbyTasks = new ArrayList<>();
+                for (final ThreadMetadata threadMetadata : allThreadMetadata) {
+                    getTasks(threadMetadata.activeTasks(), activeTasks);
+                    if (!threadMetadata.standbyTasks().isEmpty()) {
+                        getTasks(threadMetadata.standbyTasks(), standbyTasks);
+                    }
+                }
+                addTasksToBuilder(activeTasks, taskReportBuilder);
+                taskReportBuilder.append(taskDelimiter);
+                if (!standbyTasks.isEmpty()) {
+                    addTasksToBuilder(standbyTasks, taskReportBuilder);
+                }
+                System.out.println("TASK-ASSIGNMENTS:" + taskReportBuilder);
+            }
+
+            if (newState == State.REBALANCING) {
+                System.out.println(String.format("%sStarting a REBALANCE", upgradePhase));
+            }
+        });
+
+
+        streams.start();
+
+        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            streams.close();
+            System.out.println(String.format("%sCOOPERATIVE-REBALANCE-TEST-CLIENT-CLOSED", upgradePhase));
+            System.out.flush();
+        }));
+    }
+
+    private static void addTasksToBuilder(final List tasks, final StringBuilder builder) {
+        if (!tasks.isEmpty()) {
+            for (final String task : tasks) {
+                builder.append(task).append(",");
+            }
+            builder.setLength(builder.length() - 1);
+        }
+    }
+    private static void getTasks(final Set taskMetadata,
+                                 final List taskList) {
+        for (final TaskMetadata task : taskMetadata) {
+            final Set topicPartitions = task.topicPartitions();
+            for (final TopicPartition topicPartition : topicPartitions) {
+                taskList.add(topicPartition.toString());
+            }
+        }
+    }
+}
diff --git a/streams/upgrade-system-tests-11/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java b/streams/upgrade-system-tests-11/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
new file mode 100644
index 0000000000000..afade7569a7b4
--- /dev/null
+++ b/streams/upgrade-system-tests-11/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.tests;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.KafkaStreams.State;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.kstream.ForeachAction;
+import org.apache.kafka.streams.processor.TaskMetadata;
+import org.apache.kafka.streams.processor.ThreadMetadata;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+
+public class StreamsUpgradeToCooperativeRebalanceTest {
+
+
+    @SuppressWarnings("unchecked")
+    public static void main(final String[] args) throws Exception {
+        if (args.length < 2) {
+            System.err.println("StreamsUpgradeToCooperativeRebalanceTest requires two argument (kafka-url, properties-file) but only " + args.length + " provided: "
+                + (args.length > 0 ? args[0] : ""));
+        }
+        System.out.println("Args are " + Arrays.toString(args));
+        final String propFileName = args[1];
+        final Properties streamsProperties = Utils.loadProps(propFileName);
+
+        final Properties config = new Properties();
+        System.out.println("StreamsTest instance started (StreamsUpgradeToCooperativeRebalanceTest v1.1)");
+        System.out.println("props=" + streamsProperties);
+
+        config.put(StreamsConfig.APPLICATION_ID_CONFIG, "cooperative-rebalance-upgrade");
+        config.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000);
+        config.putAll(streamsProperties);
+
+        final String sourceTopic = streamsProperties.getProperty("source.topic", "source");
+        final String sinkTopic = streamsProperties.getProperty("sink.topic", "sink");
+        final String taskDelimiter = streamsProperties.getProperty("task.delimiter", "#");
+        final int reportInterval = Integer.parseInt(streamsProperties.getProperty("report.interval", "100"));
+        final String upgradePhase = streamsProperties.getProperty("upgrade.phase",  "");
+
+        final StreamsBuilder builder = new StreamsBuilder();
+
+        builder.stream(sourceTopic)
+            .peek(new ForeachAction() {
+                int recordCounter = 0;
+
+                @Override
+                public void apply(final String key, final String value) {
+                    if (recordCounter++ % reportInterval == 0) {
+                        System.out.println(String.format("%sProcessed %d records so far", upgradePhase, recordCounter));
+                        System.out.flush();
+                    }
+                }
+            }
+            ).to(sinkTopic);
+
+        final KafkaStreams streams = new KafkaStreams(builder.build(), config);
+
+        streams.setStateListener((newState, oldState) -> {
+            if (newState == State.RUNNING && oldState == State.REBALANCING) {
+                System.out.println(String.format("%sSTREAMS in a RUNNING State", upgradePhase));
+                final Set allThreadMetadata = streams.localThreadsMetadata();
+                final StringBuilder taskReportBuilder = new StringBuilder();
+                final List activeTasks = new ArrayList<>();
+                final List standbyTasks = new ArrayList<>();
+                for (final ThreadMetadata threadMetadata : allThreadMetadata) {
+                    getTasks(threadMetadata.activeTasks(), activeTasks);
+                    if (!threadMetadata.standbyTasks().isEmpty()) {
+                        getTasks(threadMetadata.standbyTasks(), standbyTasks);
+                    }
+                }
+                addTasksToBuilder(activeTasks, taskReportBuilder);
+                taskReportBuilder.append(taskDelimiter);
+                if (!standbyTasks.isEmpty()) {
+                    addTasksToBuilder(standbyTasks, taskReportBuilder);
+                }
+                System.out.println("TASK-ASSIGNMENTS:" + taskReportBuilder);
+            }
+
+            if (newState == State.REBALANCING) {
+                System.out.println(String.format("%sStarting a REBALANCE", upgradePhase));
+            }
+        });
+
+
+        streams.start();
+
+        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            streams.close();
+            System.out.println(String.format("%sCOOPERATIVE-REBALANCE-TEST-CLIENT-CLOSED", upgradePhase));
+            System.out.flush();
+        }));
+    }
+
+    private static void addTasksToBuilder(final List tasks, final StringBuilder builder) {
+        if (!tasks.isEmpty()) {
+            for (final String task : tasks) {
+                builder.append(task).append(",");
+            }
+            builder.setLength(builder.length() - 1);
+        }
+    }
+    private static void getTasks(final Set taskMetadata,
+                                 final List taskList) {
+        for (final TaskMetadata task : taskMetadata) {
+            final Set topicPartitions = task.topicPartitions();
+            for (final TopicPartition topicPartition : topicPartitions) {
+                taskList.add(topicPartition.toString());
+            }
+        }
+    }
+}
diff --git a/streams/upgrade-system-tests-20/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java b/streams/upgrade-system-tests-20/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
new file mode 100644
index 0000000000000..5b4069aa5e937
--- /dev/null
+++ b/streams/upgrade-system-tests-20/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.tests;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.KafkaStreams.State;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.kstream.ForeachAction;
+import org.apache.kafka.streams.processor.TaskMetadata;
+import org.apache.kafka.streams.processor.ThreadMetadata;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+
+public class StreamsUpgradeToCooperativeRebalanceTest {
+
+
+    @SuppressWarnings("unchecked")
+    public static void main(final String[] args) throws Exception {
+        if (args.length < 2) {
+            System.err.println("StreamsUpgradeToCooperativeRebalanceTest requires two argument (kafka-url, properties-file) but only " + args.length + " provided: "
+                + (args.length > 0 ? args[0] : ""));
+        }
+        System.out.println("Args are " + Arrays.toString(args));
+        final String propFileName = args[1];
+        final Properties streamsProperties = Utils.loadProps(propFileName);
+
+        final Properties config = new Properties();
+        System.out.println("StreamsTest instance started (StreamsUpgradeToCooperativeRebalanceTest v2.0)");
+        System.out.println("props=" + streamsProperties);
+
+        config.put(StreamsConfig.APPLICATION_ID_CONFIG, "cooperative-rebalance-upgrade");
+        config.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000);
+        config.putAll(streamsProperties);
+
+        final String sourceTopic = streamsProperties.getProperty("source.topic", "source");
+        final String sinkTopic = streamsProperties.getProperty("sink.topic", "sink");
+        final String taskDelimiter = streamsProperties.getProperty("task.delimiter", "#");
+        final int reportInterval = Integer.parseInt(streamsProperties.getProperty("report.interval", "100"));
+        final String upgradePhase = streamsProperties.getProperty("upgrade.phase",  "");
+
+        final StreamsBuilder builder = new StreamsBuilder();
+
+        builder.stream(sourceTopic)
+            .peek(new ForeachAction() {
+                int recordCounter = 0;
+
+                @Override
+                public void apply(final String key, final String value) {
+                    if (recordCounter++ % reportInterval == 0) {
+                        System.out.println(String.format("%sProcessed %d records so far", upgradePhase, recordCounter));
+                        System.out.flush();
+                    }
+                }
+            }
+            ).to(sinkTopic);
+
+        final KafkaStreams streams = new KafkaStreams(builder.build(), config);
+
+        streams.setStateListener((newState, oldState) -> {
+            if (newState == State.RUNNING && oldState == State.REBALANCING) {
+                System.out.println(String.format("%sSTREAMS in a RUNNING State", upgradePhase));
+                final Set allThreadMetadata = streams.localThreadsMetadata();
+                final StringBuilder taskReportBuilder = new StringBuilder();
+                final List activeTasks = new ArrayList<>();
+                final List standbyTasks = new ArrayList<>();
+                for (final ThreadMetadata threadMetadata : allThreadMetadata) {
+                    getTasks(threadMetadata.activeTasks(), activeTasks);
+                    if (!threadMetadata.standbyTasks().isEmpty()) {
+                        getTasks(threadMetadata.standbyTasks(), standbyTasks);
+                    }
+                }
+                addTasksToBuilder(activeTasks, taskReportBuilder);
+                taskReportBuilder.append(taskDelimiter);
+                if (!standbyTasks.isEmpty()) {
+                    addTasksToBuilder(standbyTasks, taskReportBuilder);
+                }
+                System.out.println("TASK-ASSIGNMENTS:" + taskReportBuilder);
+            }
+
+            if (newState == State.REBALANCING) {
+                System.out.println(String.format("%sStarting a REBALANCE", upgradePhase));
+            }
+        });
+
+
+        streams.start();
+
+        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            streams.close();
+            System.out.println(String.format("%sCOOPERATIVE-REBALANCE-TEST-CLIENT-CLOSED", upgradePhase));
+            System.out.flush();
+        }));
+    }
+
+    private static void addTasksToBuilder(final List tasks, final StringBuilder builder) {
+        if (!tasks.isEmpty()) {
+            for (final String task : tasks) {
+                builder.append(task).append(",");
+            }
+            builder.setLength(builder.length() - 1);
+        }
+    }
+    private static void getTasks(final Set taskMetadata,
+                                 final List taskList) {
+        for (final TaskMetadata task : taskMetadata) {
+            final Set topicPartitions = task.topicPartitions();
+            for (final TopicPartition topicPartition : topicPartitions) {
+                taskList.add(topicPartition.toString());
+            }
+        }
+    }
+}
diff --git a/streams/upgrade-system-tests-21/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java b/streams/upgrade-system-tests-21/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
new file mode 100644
index 0000000000000..174c34f58ede6
--- /dev/null
+++ b/streams/upgrade-system-tests-21/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.tests;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.KafkaStreams.State;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.kstream.ForeachAction;
+import org.apache.kafka.streams.processor.TaskMetadata;
+import org.apache.kafka.streams.processor.ThreadMetadata;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+
+public class StreamsUpgradeToCooperativeRebalanceTest {
+
+
+    @SuppressWarnings("unchecked")
+    public static void main(final String[] args) throws Exception {
+        if (args.length < 2) {
+            System.err.println("StreamsUpgradeToCooperativeRebalanceTest requires two argument (kafka-url, properties-file) but only " + args.length + " provided: "
+                + (args.length > 0 ? args[0] : ""));
+        }
+        System.out.println("Args are " + Arrays.toString(args));
+        final String propFileName = args[1];
+        final Properties streamsProperties = Utils.loadProps(propFileName);
+
+        final Properties config = new Properties();
+        System.out.println("StreamsTest instance started (StreamsUpgradeToCooperativeRebalanceTest v2.2)");
+        System.out.println("props=" + streamsProperties);
+
+        config.put(StreamsConfig.APPLICATION_ID_CONFIG, "cooperative-rebalance-upgrade");
+        config.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000);
+        config.putAll(streamsProperties);
+
+        final String sourceTopic = streamsProperties.getProperty("source.topic", "source");
+        final String sinkTopic = streamsProperties.getProperty("sink.topic", "sink");
+        final String taskDelimiter = streamsProperties.getProperty("task.delimiter", "#");
+        final int reportInterval = Integer.parseInt(streamsProperties.getProperty("report.interval", "100"));
+        final String upgradePhase = streamsProperties.getProperty("upgrade.phase",  "");
+
+        final StreamsBuilder builder = new StreamsBuilder();
+
+        builder.stream(sourceTopic)
+            .peek(new ForeachAction() {
+                int recordCounter = 0;
+
+                @Override
+                public void apply(final String key, final String value) {
+                    if (recordCounter++ % reportInterval == 0) {
+                        System.out.println(String.format("%sProcessed %d records so far", upgradePhase, recordCounter));
+                        System.out.flush();
+                    }
+                }
+            }
+            ).to(sinkTopic);
+
+        final KafkaStreams streams = new KafkaStreams(builder.build(), config);
+
+        streams.setStateListener((newState, oldState) -> {
+            if (newState == State.RUNNING && oldState == State.REBALANCING) {
+                System.out.println(String.format("%sSTREAMS in a RUNNING State", upgradePhase));
+                final Set allThreadMetadata = streams.localThreadsMetadata();
+                final StringBuilder taskReportBuilder = new StringBuilder();
+                final List activeTasks = new ArrayList<>();
+                final List standbyTasks = new ArrayList<>();
+                for (final ThreadMetadata threadMetadata : allThreadMetadata) {
+                    getTasks(threadMetadata.activeTasks(), activeTasks);
+                    if (!threadMetadata.standbyTasks().isEmpty()) {
+                        getTasks(threadMetadata.standbyTasks(), standbyTasks);
+                    }
+                }
+                addTasksToBuilder(activeTasks, taskReportBuilder);
+                taskReportBuilder.append(taskDelimiter);
+                if (!standbyTasks.isEmpty()) {
+                    addTasksToBuilder(standbyTasks, taskReportBuilder);
+                }
+                System.out.println("TASK-ASSIGNMENTS:" + taskReportBuilder);
+            }
+
+            if (newState == State.REBALANCING) {
+                System.out.println(String.format("%sStarting a REBALANCE", upgradePhase));
+            }
+        });
+
+
+        streams.start();
+
+        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            streams.close();
+            System.out.println(String.format("%sCOOPERATIVE-REBALANCE-TEST-CLIENT-CLOSED", upgradePhase));
+            System.out.flush();
+        }));
+    }
+
+    private static void addTasksToBuilder(final List tasks, final StringBuilder builder) {
+        if (!tasks.isEmpty()) {
+            for (final String task : tasks) {
+                builder.append(task).append(",");
+            }
+            builder.setLength(builder.length() - 1);
+        }
+    }
+    private static void getTasks(final Set taskMetadata,
+                                 final List taskList) {
+        for (final TaskMetadata task : taskMetadata) {
+            final Set topicPartitions = task.topicPartitions();
+            for (final TopicPartition topicPartition : topicPartitions) {
+                taskList.add(topicPartition.toString());
+            }
+        }
+    }
+}
diff --git a/streams/upgrade-system-tests-22/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java b/streams/upgrade-system-tests-22/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
new file mode 100644
index 0000000000000..174c34f58ede6
--- /dev/null
+++ b/streams/upgrade-system-tests-22/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.tests;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.KafkaStreams.State;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.kstream.ForeachAction;
+import org.apache.kafka.streams.processor.TaskMetadata;
+import org.apache.kafka.streams.processor.ThreadMetadata;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+
+public class StreamsUpgradeToCooperativeRebalanceTest {
+
+
+    @SuppressWarnings("unchecked")
+    public static void main(final String[] args) throws Exception {
+        if (args.length < 2) {
+            System.err.println("StreamsUpgradeToCooperativeRebalanceTest requires two argument (kafka-url, properties-file) but only " + args.length + " provided: "
+                + (args.length > 0 ? args[0] : ""));
+        }
+        System.out.println("Args are " + Arrays.toString(args));
+        final String propFileName = args[1];
+        final Properties streamsProperties = Utils.loadProps(propFileName);
+
+        final Properties config = new Properties();
+        System.out.println("StreamsTest instance started (StreamsUpgradeToCooperativeRebalanceTest v2.2)");
+        System.out.println("props=" + streamsProperties);
+
+        config.put(StreamsConfig.APPLICATION_ID_CONFIG, "cooperative-rebalance-upgrade");
+        config.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000);
+        config.putAll(streamsProperties);
+
+        final String sourceTopic = streamsProperties.getProperty("source.topic", "source");
+        final String sinkTopic = streamsProperties.getProperty("sink.topic", "sink");
+        final String taskDelimiter = streamsProperties.getProperty("task.delimiter", "#");
+        final int reportInterval = Integer.parseInt(streamsProperties.getProperty("report.interval", "100"));
+        final String upgradePhase = streamsProperties.getProperty("upgrade.phase",  "");
+
+        final StreamsBuilder builder = new StreamsBuilder();
+
+        builder.stream(sourceTopic)
+            .peek(new ForeachAction() {
+                int recordCounter = 0;
+
+                @Override
+                public void apply(final String key, final String value) {
+                    if (recordCounter++ % reportInterval == 0) {
+                        System.out.println(String.format("%sProcessed %d records so far", upgradePhase, recordCounter));
+                        System.out.flush();
+                    }
+                }
+            }
+            ).to(sinkTopic);
+
+        final KafkaStreams streams = new KafkaStreams(builder.build(), config);
+
+        streams.setStateListener((newState, oldState) -> {
+            if (newState == State.RUNNING && oldState == State.REBALANCING) {
+                System.out.println(String.format("%sSTREAMS in a RUNNING State", upgradePhase));
+                final Set allThreadMetadata = streams.localThreadsMetadata();
+                final StringBuilder taskReportBuilder = new StringBuilder();
+                final List activeTasks = new ArrayList<>();
+                final List standbyTasks = new ArrayList<>();
+                for (final ThreadMetadata threadMetadata : allThreadMetadata) {
+                    getTasks(threadMetadata.activeTasks(), activeTasks);
+                    if (!threadMetadata.standbyTasks().isEmpty()) {
+                        getTasks(threadMetadata.standbyTasks(), standbyTasks);
+                    }
+                }
+                addTasksToBuilder(activeTasks, taskReportBuilder);
+                taskReportBuilder.append(taskDelimiter);
+                if (!standbyTasks.isEmpty()) {
+                    addTasksToBuilder(standbyTasks, taskReportBuilder);
+                }
+                System.out.println("TASK-ASSIGNMENTS:" + taskReportBuilder);
+            }
+
+            if (newState == State.REBALANCING) {
+                System.out.println(String.format("%sStarting a REBALANCE", upgradePhase));
+            }
+        });
+
+
+        streams.start();
+
+        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            streams.close();
+            System.out.println(String.format("%sCOOPERATIVE-REBALANCE-TEST-CLIENT-CLOSED", upgradePhase));
+            System.out.flush();
+        }));
+    }
+
+    private static void addTasksToBuilder(final List tasks, final StringBuilder builder) {
+        if (!tasks.isEmpty()) {
+            for (final String task : tasks) {
+                builder.append(task).append(",");
+            }
+            builder.setLength(builder.length() - 1);
+        }
+    }
+    private static void getTasks(final Set taskMetadata,
+                                 final List taskList) {
+        for (final TaskMetadata task : taskMetadata) {
+            final Set topicPartitions = task.topicPartitions();
+            for (final TopicPartition topicPartition : topicPartitions) {
+                taskList.add(topicPartition.toString());
+            }
+        }
+    }
+}
diff --git a/streams/upgrade-system-tests-23/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java b/streams/upgrade-system-tests-23/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
new file mode 100644
index 0000000000000..0aff8e50a640a
--- /dev/null
+++ b/streams/upgrade-system-tests-23/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeToCooperativeRebalanceTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.tests;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.KafkaStreams.State;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.kstream.ForeachAction;
+import org.apache.kafka.streams.processor.TaskMetadata;
+import org.apache.kafka.streams.processor.ThreadMetadata;
+
+import java.util.Arrays;
+import java.util.Properties;
+import java.util.Set;
+
+public class StreamsUpgradeToCooperativeRebalanceTest {
+
+
+    @SuppressWarnings("unchecked")
+    public static void main(final String[] args) throws Exception {
+        if (args.length < 2) {
+            System.err.println("StreamsUpgradeToCooperativeRebalanceTest requires two argument (kafka-url, properties-file) but only " + args.length + " provided: "
+                + (args.length > 0 ? args[0] : ""));
+        }
+        System.out.println("Args are " + Arrays.toString(args));
+        final String propFileName = args[1];
+        final Properties streamsProperties = Utils.loadProps(propFileName);
+
+        final Properties config = new Properties();
+        System.out.println("StreamsTest instance started (StreamsUpgradeToCooperativeRebalanceTest v2.3)");
+        System.out.println("props=" + streamsProperties);
+
+        config.put(StreamsConfig.APPLICATION_ID_CONFIG, "cooperative-rebalance-upgrade");
+        config.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        config.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000);
+        config.putAll(streamsProperties);
+
+        final String sourceTopic = streamsProperties.getProperty("source.topic", "source");
+        final String sinkTopic = streamsProperties.getProperty("sink.topic", "sink");
+        final String threadDelimiter = streamsProperties.getProperty("thread.delimiter", "&");
+        final String taskDelimiter = streamsProperties.getProperty("task.delimiter", "#");
+        final int reportInterval = Integer.parseInt(streamsProperties.getProperty("report.interval", "100"));
+
+        final StreamsBuilder builder = new StreamsBuilder();
+
+        builder.stream(sourceTopic)
+            .peek(new ForeachAction() {
+                int recordCounter = 0;
+
+                @Override
+                public void apply(final String key, final String value) {
+                    if (recordCounter++ % reportInterval == 0) {
+                        System.out.println(String.format("Processed %d records so far", recordCounter));
+                        System.out.flush();
+                    }
+                }
+            }
+            ).to(sinkTopic);
+
+        final KafkaStreams streams = new KafkaStreams(builder.build(), config);
+
+        streams.setStateListener((newState, oldState) -> {
+            if (newState == State.RUNNING && oldState == State.REBALANCING) {
+                System.out.println("STREAMS in a RUNNING State");
+                final Set allThreadMetadata = streams.localThreadsMetadata();
+                final StringBuilder taskReportBuilder = new StringBuilder();
+                for (final ThreadMetadata threadMetadata : allThreadMetadata) {
+                    buildTaskAssignmentReport(taskReportBuilder, threadMetadata.activeTasks(), "ACTIVE-TASKS:");
+                    if (!threadMetadata.standbyTasks().isEmpty()) {
+                        taskReportBuilder.append(taskDelimiter);
+                        buildTaskAssignmentReport(taskReportBuilder, threadMetadata.standbyTasks(), "STANDBY-TASKS:");
+                    }
+                    taskReportBuilder.append(threadDelimiter);
+                }
+                taskReportBuilder.setLength(taskReportBuilder.length() - 1);
+                System.out.println("TASK-ASSIGNMENTS:" + taskReportBuilder);
+            }
+
+            if (newState == State.REBALANCING) {
+                System.out.println("Starting a REBALANCE");
+            }
+        });
+
+
+        streams.start();
+
+        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            streams.close();
+            System.out.println("COOPERATIVE-REBALANCE-TEST-CLIENT-CLOSED");
+            System.out.flush();
+        }));
+    }
+
+    private static void buildTaskAssignmentReport(final StringBuilder taskReportBuilder,
+                                                  final Set taskMetadata,
+                                                  final String taskType) {
+        taskReportBuilder.append(taskType);
+        for (final TaskMetadata task : taskMetadata) {
+            final Set topicPartitions = task.topicPartitions();
+            for (final TopicPartition topicPartition : topicPartitions) {
+                taskReportBuilder.append(topicPartition.toString()).append(",");
+            }
+        }
+        taskReportBuilder.setLength(taskReportBuilder.length() - 1);
+    }
+}
diff --git a/tests/kafkatest/services/streams.py b/tests/kafkatest/services/streams.py
index 48bac706e7489..52afe4ee052a3 100644
--- a/tests/kafkatest/services/streams.py
+++ b/tests/kafkatest/services/streams.py
@@ -536,6 +536,7 @@ def prop_file(self):
         cfg = KafkaConfig(**properties)
         return cfg.render()
 
+
 class StaticMemberTestService(StreamsTestBaseService):
     def __init__(self, test_context, kafka, group_instance_id, num_threads):
         super(StaticMemberTestService, self).__init__(test_context,
@@ -556,3 +557,87 @@ def prop_file(self):
 
         cfg = KafkaConfig(**properties)
         return cfg.render()
+
+
+class CooperativeRebalanceUpgradeService(StreamsTestBaseService):
+    def __init__(self, test_context, kafka):
+        super(CooperativeRebalanceUpgradeService, self).__init__(test_context,
+                                                                 kafka,
+                                                                 "org.apache.kafka.streams.tests.StreamsUpgradeToCooperativeRebalanceTest",
+                                                                 "")
+        self.UPGRADE_FROM = None
+        # these properties will be overridden in test
+        self.SOURCE_TOPIC = None
+        self.SINK_TOPIC = None
+        self.TASK_DELIMITER = "#"
+        self.REPORT_INTERVAL = None
+
+        self.standby_tasks = None
+        self.active_tasks = None
+        self.upgrade_phase = None
+
+    def set_tasks(self, task_string):
+        label = "TASK-ASSIGNMENTS:"
+        task_string_substr = task_string[len(label):]
+        all_tasks = task_string_substr.split(self.TASK_DELIMITER)
+        self.active_tasks = set(all_tasks[0].split(","))
+        if len(all_tasks) > 1:
+            self.standby_tasks = set(all_tasks[1].split(","))
+
+    def set_version(self, kafka_streams_version):
+        self.KAFKA_STREAMS_VERSION = kafka_streams_version
+
+    def set_upgrade_phase(self, upgrade_phase):
+        self.upgrade_phase = upgrade_phase
+
+    def start_cmd(self, node):
+        args = self.args.copy()
+        if self.KAFKA_STREAMS_VERSION in [str(LATEST_0_10_0), str(LATEST_0_10_1), str(LATEST_0_10_2),
+                                          str(LATEST_0_11_0), str(LATEST_1_0), str(LATEST_1_1),
+                                          str(LATEST_2_0), str(LATEST_2_1), str(LATEST_2_2), str(LATEST_2_3)]:
+            args['kafka'] = self.kafka.bootstrap_servers()
+        else:
+            args['kafka'] = ""
+        if self.KAFKA_STREAMS_VERSION == str(LATEST_0_10_0) or self.KAFKA_STREAMS_VERSION == str(LATEST_0_10_1):
+            args['zk'] = self.kafka.zk.connect_setting()
+        else:
+            args['zk'] = ""
+        args['config_file'] = self.CONFIG_FILE
+        args['stdout'] = self.STDOUT_FILE
+        args['stderr'] = self.STDERR_FILE
+        args['pidfile'] = self.PID_FILE
+        args['log4j'] = self.LOG4J_CONFIG_FILE
+        args['version'] = self.KAFKA_STREAMS_VERSION
+        args['kafka_run_class'] = self.path.script("kafka-run-class.sh", node)
+
+        cmd = "( export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%(log4j)s\"; " \
+              "INCLUDE_TEST_JARS=true UPGRADE_KAFKA_STREAMS_TEST_VERSION=%(version)s " \
+              " %(kafka_run_class)s %(streams_class_name)s %(kafka)s %(zk)s %(config_file)s " \
+              " & echo $! >&3 ) 1>> %(stdout)s 2>> %(stderr)s 3> %(pidfile)s" % args
+
+        self.logger.info("Executing: " + cmd)
+
+        return cmd
+
+    def prop_file(self):
+        properties = {streams_property.STATE_DIR: self.PERSISTENT_ROOT,
+                      streams_property.KAFKA_SERVERS: self.kafka.bootstrap_servers()}
+
+        if self.UPGRADE_FROM is not None:
+            properties['upgrade.from'] = self.UPGRADE_FROM
+        else:
+            try:
+                del properties['upgrade.from']
+            except KeyError:
+                self.logger.info("Key 'upgrade.from' not there, better safe than sorry")
+
+        if self.upgrade_phase is not None:
+            properties['upgrade.phase'] = self.upgrade_phase
+
+        properties['source.topic'] = self.SOURCE_TOPIC
+        properties['sink.topic'] = self.SINK_TOPIC
+        properties['task.delimiter'] = self.TASK_DELIMITER
+        properties['report.interval'] = self.REPORT_INTERVAL
+
+        cfg = KafkaConfig(**properties)
+        return cfg.render()
diff --git a/tests/kafkatest/tests/streams/streams_cooperative_rebalance_upgrade_test.py b/tests/kafkatest/tests/streams/streams_cooperative_rebalance_upgrade_test.py
new file mode 100644
index 0000000000000..3128d21bc4793
--- /dev/null
+++ b/tests/kafkatest/tests/streams/streams_cooperative_rebalance_upgrade_test.py
@@ -0,0 +1,209 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+from ducktape.mark import matrix
+from ducktape.tests.test import Test
+from kafkatest.services.kafka import KafkaService
+from kafkatest.services.verifiable_producer import VerifiableProducer
+from kafkatest.services.zookeeper import ZookeeperService
+from kafkatest.version import LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, \
+    LATEST_2_0, LATEST_2_1, LATEST_2_2, LATEST_2_3, DEV_BRANCH, DEV_VERSION, KafkaVersion
+from kafkatest.services.streams import CooperativeRebalanceUpgradeService
+from kafkatest.tests.streams.utils import verify_stopped, stop_processors, verify_running
+
+
+class StreamsCooperativeRebalanceUpgradeTest(Test):
+    """
+    Test of a rolling upgrade from eager rebalance to
+    cooperative rebalance
+    """
+
+    source_topic = "source"
+    sink_topic = "sink"
+    task_delimiter = "#"
+    report_interval = "1000"
+    processing_message = "Processed [0-9]* records so far"
+    stopped_message = "COOPERATIVE-REBALANCE-TEST-CLIENT-CLOSED"
+    running_state_msg = "STREAMS in a RUNNING State"
+    cooperative_turned_off_msg = "Eager rebalancing enabled now for upgrade from %s"
+    cooperative_enabled_msg = "Cooperative rebalancing enabled now"
+    first_bounce_phase = "first_bounce_phase-"
+    second_bounce_phase = "second_bounce_phase-"
+
+    streams_eager_rebalance_upgrade_versions = [str(LATEST_0_10_0), str(LATEST_0_10_1), str(LATEST_0_10_2), str(LATEST_0_11_0),
+                                                str(LATEST_1_0), str(LATEST_1_1), str(LATEST_2_0), str(LATEST_2_1), str(LATEST_2_2),
+                                                str(LATEST_2_3)]
+
+    def __init__(self, test_context):
+        super(StreamsCooperativeRebalanceUpgradeTest, self).__init__(test_context)
+        self.topics = {
+            self.source_topic: {'partitions': 9},
+            self.sink_topic: {'partitions': 9}
+        }
+
+        self.zookeeper = ZookeeperService(self.test_context, num_nodes=1)
+        self.kafka = KafkaService(self.test_context, num_nodes=3,
+                                  zk=self.zookeeper, topics=self.topics)
+
+        self.producer = VerifiableProducer(self.test_context,
+                                           1,
+                                           self.kafka,
+                                           self.source_topic,
+                                           throughput=1000,
+                                           acks=1)
+
+    @matrix(upgrade_from_version=streams_eager_rebalance_upgrade_versions)
+    def test_upgrade_to_cooperative_rebalance(self, upgrade_from_version):
+        self.zookeeper.start()
+        self.kafka.start()
+
+        processor1 = CooperativeRebalanceUpgradeService(self.test_context, self.kafka)
+        processor2 = CooperativeRebalanceUpgradeService(self.test_context, self.kafka)
+        processor3 = CooperativeRebalanceUpgradeService(self.test_context, self.kafka)
+
+        processors = [processor1, processor2, processor3]
+
+        # produce records continually during the test
+        self.producer.start()
+
+        # start all processors without upgrade_from config; normal operations mode
+        self.logger.info("Starting all streams clients in normal running mode")
+        for processor in processors:
+            processor.set_version(upgrade_from_version)
+            self.set_props(processor)
+            processor.CLEAN_NODE_ENABLED = False
+            # can't use state as older version don't have state listener
+            # so just verify up and running
+            verify_running(processor, self.processing_message)
+
+        # all running rebalancing has ceased
+        for processor in processors:
+            self.verify_processing(processor, self.processing_message)
+
+        # first rolling bounce with "upgrade.from" config set
+        previous_phase = ""
+        self.maybe_upgrade_rolling_bounce_and_verify(processors,
+                                                     previous_phase,
+                                                     self.first_bounce_phase,
+                                                     upgrade_from_version)
+
+        # All nodes processing, rebalancing has ceased
+        for processor in processors:
+            self.verify_processing(processor, self.first_bounce_phase + self.processing_message)
+
+        # second rolling bounce without "upgrade.from" config
+        self.maybe_upgrade_rolling_bounce_and_verify(processors,
+                                                     self.first_bounce_phase,
+                                                     self.second_bounce_phase)
+
+        # All nodes processing, rebalancing has ceased
+        for processor in processors:
+            self.verify_processing(processor, self.second_bounce_phase + self.processing_message)
+
+        # now verify tasks are unique
+        for processor in processors:
+            self.get_tasks_for_processor(processor)
+            self.logger.info("Active tasks %s" % processor.active_tasks)
+
+        overlapping_tasks = processor1.active_tasks.intersection(processor2.active_tasks)
+        assert len(overlapping_tasks) == int(0), \
+            "Final task assignments are not unique %s %s" % (processor1.active_tasks, processor2.active_tasks)
+
+        overlapping_tasks = processor1.active_tasks.intersection(processor3.active_tasks)
+        assert len(overlapping_tasks) == int(0), \
+            "Final task assignments are not unique %s %s" % (processor1.active_tasks, processor3.active_tasks)
+
+        overlapping_tasks = processor2.active_tasks.intersection(processor3.active_tasks)
+        assert len(overlapping_tasks) == int(0), \
+            "Final task assignments are not unique %s %s" % (processor2.active_tasks, processor3.active_tasks)
+
+        # test done close all down
+        stop_processors(processors, self.second_bounce_phase + self.stopped_message)
+
+        self.producer.stop()
+        self.kafka.stop()
+        self.zookeeper.stop()
+
+    def maybe_upgrade_rolling_bounce_and_verify(self,
+                                                processors,
+                                                previous_phase,
+                                                current_phase,
+                                                upgrade_from_version=None):
+        for processor in processors:
+            # stop the processor in prep for setting "update.from" or removing "update.from"
+            verify_stopped(processor, previous_phase + self.stopped_message)
+            # upgrade to version with cooperative rebalance
+            processor.set_version("")
+            processor.set_upgrade_phase(current_phase)
+
+            if upgrade_from_version is not None:
+                # need to remove minor version numbers for check of valid upgrade from numbers
+                upgrade_version = upgrade_from_version[:upgrade_from_version.rfind('.')]
+                rebalance_mode_msg = self.cooperative_turned_off_msg % upgrade_version
+            else:
+                upgrade_version = None
+                rebalance_mode_msg = self.cooperative_enabled_msg
+
+            self.set_props(processor, upgrade_version)
+            node = processor.node
+            with node.account.monitor_log(processor.STDOUT_FILE) as stdout_monitor:
+                with node.account.monitor_log(processor.LOG_FILE) as log_monitor:
+                    processor.start()
+                    # verify correct rebalance mode either turned off for upgrade or enabled after upgrade
+                    log_monitor.wait_until(rebalance_mode_msg,
+                                           timeout_sec=60,
+                                           err_msg="Never saw '%s' message " % rebalance_mode_msg + str(processor.node.account))
+
+                # verify rebalanced into a running state
+                rebalance_msg = current_phase + self.running_state_msg
+                stdout_monitor.wait_until(rebalance_msg,
+                                          timeout_sec=60,
+                                          err_msg="Never saw '%s' message " % rebalance_msg + str(
+                                              processor.node.account))
+
+                # verify processing
+                verify_processing_msg = current_phase + self.processing_message
+                stdout_monitor.wait_until(verify_processing_msg,
+                                          timeout_sec=60,
+                                          err_msg="Never saw '%s' message " % verify_processing_msg + str(
+                                              processor.node.account))
+
+    def verify_processing(self, processor, pattern):
+        self.logger.info("Verifying %s processing pattern in STDOUT_FILE" % pattern)
+        with processor.node.account.monitor_log(processor.STDOUT_FILE) as monitor:
+            monitor.wait_until(pattern,
+                               timeout_sec=60,
+                               err_msg="Never saw processing of %s " % pattern + str(processor.node.account))
+
+    def get_tasks_for_processor(self, processor):
+        retries = 0
+        while retries < 5:
+            found_tasks = list(processor.node.account.ssh_capture("grep TASK-ASSIGNMENTS %s | tail -n 1" % processor.STDOUT_FILE, allow_fail=True))
+            self.logger.info("Returned %s from assigned task check" % found_tasks)
+            if len(found_tasks) > 0:
+                task_string = str(found_tasks[0]).strip()
+                self.logger.info("Converted %s from assigned task check" % task_string)
+                processor.set_tasks(task_string)
+                return
+            retries += 1
+            time.sleep(1)
+        return
+
+    def set_props(self, processor, upgrade_from=None):
+        processor.SOURCE_TOPIC = self.source_topic
+        processor.SINK_TOPIC = self.sink_topic
+        processor.REPORT_INTERVAL = self.report_interval
+        processor.UPGRADE_FROM = upgrade_from
diff --git a/tests/kafkatest/version.py b/tests/kafkatest/version.py
index 573e2ed4de853..8deceac2f4fd3 100644
--- a/tests/kafkatest/version.py
+++ b/tests/kafkatest/version.py
@@ -129,4 +129,5 @@ def get_version(node=None):
 
 # 2.3.x versions
 V_2_3_0 = KafkaVersion("2.3.0")
+V_2_3_1 = KafkaVersion("2.3.1")
 LATEST_2_3 = V_2_3_0

From 2a4b27c02ad487954eb971e8e28bade475d6d816 Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Thu, 17 Oct 2019 00:40:57 -0500
Subject: [PATCH 0738/1071] KAFKA-9000: fix flaky FK join test by using TTD
 (#7517)

Migrate this integration test to use TopologyTestDriver instead of running 3 Streams instances.

Dropped one test that was attempting to produce specific interleavings. If anything, these should be verified deterministically by unit testing.

Reviewers: Matthias J. Sax , Guozhang Wang 
---
 checkstyle/suppressions.xml                   |    4 +-
 .../integration/ForeignKeyJoinSuite.java      |    4 +-
 ...leKTableForeignKeyJoinIntegrationTest.java | 1024 +++++++----------
 ...tionResolverJoinProcessorSupplierTest.java |  223 ++++
 .../processor/MockProcessorContext.java       |    9 +
 5 files changed, 637 insertions(+), 627 deletions(-)
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplierTest.java

diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index 2f213090da56d..c0aeb95b4a2ff 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -217,9 +217,7 @@
               files="SmokeTestDriver.java"/>
 
     
-    
+              files="EosTestDriver|KStreamKStreamJoinTest.java|SmokeTestDriver.java|KStreamKStreamLeftJoinTest.java|KTableKTableForeignKeyJoinIntegrationTest.java"/>
 
     
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/ForeignKeyJoinSuite.java b/streams/src/test/java/org/apache/kafka/streams/integration/ForeignKeyJoinSuite.java
index 6245b11af6004..ac6adb876aa15 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/ForeignKeyJoinSuite.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/ForeignKeyJoinSuite.java
@@ -18,6 +18,7 @@
 
 import org.apache.kafka.common.utils.BytesTest;
 import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.CombinedKeySchemaTest;
+import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionResolverJoinProcessorSupplierTest;
 import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionResponseWrapperSerdeTest;
 import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionWrapperSerdeTest;
 import org.junit.runner.RunWith;
@@ -39,7 +40,8 @@
     KTableKTableForeignKeyJoinIntegrationTest.class,
     CombinedKeySchemaTest.class,
     SubscriptionWrapperSerdeTest.class,
-    SubscriptionResponseWrapperSerdeTest.class
+    SubscriptionResponseWrapperSerdeTest.class,
+    SubscriptionResolverJoinProcessorSupplierTest.class
 })
 public class ForeignKeyJoinSuite {
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinIntegrationTest.java
index ae0f3c200fa19..14a39b52ea307 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinIntegrationTest.java
@@ -16,684 +16,462 @@
  */
 package org.apache.kafka.streams.integration;
 
-import kafka.utils.MockTime;
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.apache.kafka.clients.producer.ProducerConfig;
-import org.apache.kafka.common.serialization.FloatSerializer;
-import org.apache.kafka.common.serialization.IntegerDeserializer;
-import org.apache.kafka.common.serialization.IntegerSerializer;
-import org.apache.kafka.common.serialization.LongSerializer;
 import org.apache.kafka.common.serialization.Serdes;
 import org.apache.kafka.common.serialization.StringDeserializer;
 import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.common.utils.Bytes;
-import org.apache.kafka.streams.KafkaStreams;
-import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TestInputTopic;
+import org.apache.kafka.streams.TestOutputTopic;
 import org.apache.kafka.streams.Topology;
-import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
-import org.apache.kafka.streams.integration.utils.IntegrationTestUtils;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.KTable;
 import org.apache.kafka.streams.kstream.Materialized;
-import org.apache.kafka.streams.kstream.Named;
 import org.apache.kafka.streams.kstream.Produced;
 import org.apache.kafka.streams.kstream.ValueJoiner;
-import org.apache.kafka.streams.state.KeyValueIterator;
 import org.apache.kafka.streams.state.KeyValueStore;
-import org.apache.kafka.streams.state.QueryableStoreTypes;
-import org.apache.kafka.streams.state.ReadOnlyKeyValueStore;
-import org.apache.kafka.test.IntegrationTest;
+import org.apache.kafka.streams.state.Stores;
 import org.apache.kafka.test.TestUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
 import org.junit.Test;
-import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
-import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
 import java.util.Map;
 import java.util.Properties;
-import java.util.Set;
 import java.util.function.Function;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertArrayEquals;
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.common.utils.Utils.mkEntry;
+import static org.apache.kafka.common.utils.Utils.mkMap;
+import static org.apache.kafka.common.utils.Utils.mkProperties;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
 
 
-@Category({IntegrationTest.class})
+@RunWith(value = Parameterized.class)
 public class KTableKTableForeignKeyJoinIntegrationTest {
-    private final static int NUM_BROKERS = 1;
-
-    @ClassRule
-    public final static EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(NUM_BROKERS);
-    private final static MockTime MOCK_TIME = CLUSTER.time;
-    private final static String LEFT_TABLE = "left_table";
-    private final static String RIGHT_TABLE = "right_table";
-    private final static String OUTPUT = "output-topic";
-    private static Properties streamsConfig;
-    private KafkaStreams streams;
-    private KafkaStreams streamsTwo;
-    private KafkaStreams streamsThree;
-    private static final  Properties CONSUMER_CONFIG = new Properties();
-    private static final Properties LEFT_PROD_CONF = new Properties();
-    private static final Properties RIGHT_PROD_CONF = new Properties();
-
-    @BeforeClass
-    public static void beforeTest() {
-        //Use multiple partitions to ensure distribution of keys.
-        LEFT_PROD_CONF.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
-        LEFT_PROD_CONF.put(ProducerConfig.ACKS_CONFIG, "all");
-        LEFT_PROD_CONF.put(ProducerConfig.RETRIES_CONFIG, 0);
-        LEFT_PROD_CONF.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class);
-        LEFT_PROD_CONF.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, FloatSerializer.class);
-
-        RIGHT_PROD_CONF.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
-        RIGHT_PROD_CONF.put(ProducerConfig.ACKS_CONFIG, "all");
-        RIGHT_PROD_CONF.put(ProducerConfig.RETRIES_CONFIG, 0);
-        RIGHT_PROD_CONF.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
-        RIGHT_PROD_CONF.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, LongSerializer.class);
-
-        streamsConfig = new Properties();
-        streamsConfig.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
-        streamsConfig.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
-        streamsConfig.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath());
-        streamsConfig.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
-        streamsConfig.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 100);
-        streamsConfig.put(StreamsConfig.TOPOLOGY_OPTIMIZATION, StreamsConfig.OPTIMIZE);
-
-        CONSUMER_CONFIG.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
-        CONSUMER_CONFIG.put(ConsumerConfig.GROUP_ID_CONFIG, "ktable-ktable-consumer");
-        CONSUMER_CONFIG.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, IntegerDeserializer.class);
-        CONSUMER_CONFIG.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
-    }
-
-    @Before
-    public void before() throws IOException, InterruptedException {
-        CLUSTER.deleteTopicsAndWait(LEFT_TABLE);
-        CLUSTER.deleteTopicsAndWait(RIGHT_TABLE);
-        CLUSTER.deleteTopicsAndWait(OUTPUT);
-
-        CLUSTER.createTopic(LEFT_TABLE, 3, 1);
-        CLUSTER.createTopic(RIGHT_TABLE, 3, 1);
-        CLUSTER.createTopic(OUTPUT, 3, 1);
-
-        IntegrationTestUtils.purgeLocalStreamsState(streamsConfig);
-    }
 
-    @After
-    public void after() throws IOException {
-        if (streams != null) {
-            streams.close();
-            streams = null;
-        }
-        if (streamsTwo != null) {
-            streamsTwo.close();
-            streamsTwo = null;
-        }
-        if (streamsThree != null) {
-            streamsThree.close();
-            streamsThree = null;
-        }
-        IntegrationTestUtils.purgeLocalStreamsState(streamsConfig);
+    private static final String LEFT_TABLE = "left_table";
+    private static final String RIGHT_TABLE = "right_table";
+    private static final String OUTPUT = "output-topic";
+    private final Properties streamsConfig;
+    private final boolean leftJoin;
+
+    public KTableKTableForeignKeyJoinIntegrationTest(final boolean leftJoin, final String optimization) {
+        this.leftJoin = leftJoin;
+        streamsConfig = mkProperties(mkMap(
+            mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, "ktable-ktable-joinOnForeignKey"),
+            mkEntry(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "asdf:0000"),
+            mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath()),
+            mkEntry(StreamsConfig.TOPOLOGY_OPTIMIZATION, optimization)
+        ));
     }
 
-    @Test
-    public void doInnerJoinFromLeftThenDeleteLeftEntity() throws Exception {
-        final List> rightTableEvents = Arrays.asList(new KeyValue<>("1", 10L), new KeyValue<>("2", 20L)); //partition 0
-        IntegrationTestUtils.produceKeyValuesSynchronously(RIGHT_TABLE, rightTableEvents, RIGHT_PROD_CONF, MOCK_TIME);
-
-        final String currentMethodName = new Object() { }
-                .getClass()
-                .getEnclosingMethod()
-                .getName();
-        createAndStartStreamsApplication(currentMethodName, false);
-
-        final List> leftTableEvents = Arrays.asList(new KeyValue<>(1, 1.33f), new KeyValue<>(2, 2.77f));
-        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
-
-        final Set> expected = new HashSet<>();
-        expected.add(new KeyValue<>(1, "value1=1.33,value2=10"));
-        expected.add(new KeyValue<>(2, "value1=2.77,value2=20"));
-
-        final Set> result = new HashSet<>(IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
-                CONSUMER_CONFIG,
-                OUTPUT,
-                expected.size()));
-        assertEquals(expected, result);
-
-        //Now delete one LHS entity such that one delete is propagated down to the output.
-        final Set> expectedDeleted = new HashSet<>();
-        expectedDeleted.add(new KeyValue<>(1, null));
-
-        final List> rightTableDeleteEvents = Arrays.asList(new KeyValue<>(1, null));
-        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, rightTableDeleteEvents, LEFT_PROD_CONF, MOCK_TIME);
-        final Set> resultDeleted = new HashSet<>(IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
-                CONSUMER_CONFIG,
-                OUTPUT,
-                expectedDeleted.size()));
-        assertEquals(expectedDeleted, resultDeleted);
-
-        //Ensure the state stores have the correct values within:
-        final Set> expMatResults = new HashSet<>();
-        expMatResults.add(new KeyValue<>(2, "value1=2.77,value2=20"));
-        validateQueryableStoresContainExpectedKeyValues(expMatResults, currentMethodName);
-    }
-
-    @Test
-    public void doLeftJoinFromLeftThenDeleteLeftEntity() throws Exception {
-        final List> rightTableEvents = Arrays.asList(new KeyValue<>("1", 10L), new KeyValue<>("2", 20L)); //partition 0
-        IntegrationTestUtils.produceKeyValuesSynchronously(RIGHT_TABLE, rightTableEvents, RIGHT_PROD_CONF, MOCK_TIME);
-
-        final String currentMethodName = new Object() { }
-                .getClass()
-                .getEnclosingMethod()
-                .getName();
-        createAndStartStreamsApplication(currentMethodName, true);
-
-        final List> leftTableEvents = Arrays.asList(new KeyValue<>(1, 1.33f), new KeyValue<>(2, 2.77f));
-        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
-
-        final Set> expected = new HashSet<>();
-        expected.add(new KeyValue<>(1, "value1=1.33,value2=10"));
-        expected.add(new KeyValue<>(2, "value1=2.77,value2=20"));
-
-        final Set> result = new HashSet<>(IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
-                CONSUMER_CONFIG,
-                OUTPUT,
-                expected.size()));
-        assertEquals(expected, result);
-
-        //Now delete one LHS entity such that one delete is propagated down to the output.
-        final Set> expectedDeleted = new HashSet<>();
-        expectedDeleted.add(new KeyValue<>(1, null));
-
-        final List> rightTableDeleteEvents = Arrays.asList(new KeyValue<>(1, null));
-        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, rightTableDeleteEvents, LEFT_PROD_CONF, MOCK_TIME);
-        final Set> resultDeleted = new HashSet<>(IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
-                CONSUMER_CONFIG,
-                OUTPUT,
-                expectedDeleted.size()));
-        assertEquals(expectedDeleted, resultDeleted);
-
-        //Ensure the state stores have the correct values within:
-        final Set> expMatResults = new HashSet<>();
-        expMatResults.add(new KeyValue<>(2, "value1=2.77,value2=20"));
-        validateQueryableStoresContainExpectedKeyValues(expMatResults, currentMethodName);
-    }
-
-    @Test
-    public void doInnerJoinFromRightThenDeleteRightEntity() throws Exception {
-        final List> leftTableEvents = Arrays.asList(
-                new KeyValue<>(1, 1.33f),
-                new KeyValue<>(2, 1.77f),
-                new KeyValue<>(3, 3.77f));
-        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
-        final String currentMethodName = new Object() { }
-                .getClass()
-                .getEnclosingMethod()
-                .getName();
-        createAndStartStreamsApplication(currentMethodName, false);
-
-        final List> rightTableEvents = Arrays.asList(
-                new KeyValue<>("1", 10L),  //partition 0
-                new KeyValue<>("2", 20L),  //partition 2
-                new KeyValue<>("3", 30L)); //partition 2
-        IntegrationTestUtils.produceKeyValuesSynchronously(RIGHT_TABLE, rightTableEvents, RIGHT_PROD_CONF, MOCK_TIME);
-
-        //Ensure that the joined values exist in the output
-        final Set> expected = new HashSet<>();
-        expected.add(new KeyValue<>(1, "value1=1.33,value2=10"));   //Will be deleted.
-        expected.add(new KeyValue<>(2, "value1=1.77,value2=10"));   //Will be deleted.
-        expected.add(new KeyValue<>(3, "value1=3.77,value2=30"));   //Will not be deleted.
-
-        final Set> result = new HashSet<>(IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
-                CONSUMER_CONFIG,
-                OUTPUT,
-                expected.size()));
-        assertEquals(expected, result);
-
-        //Now delete the RHS entity such that all matching keys have deletes propagated.
-        final Set> expectedDeleted = new HashSet<>();
-        expectedDeleted.add(new KeyValue<>(1, null));
-        expectedDeleted.add(new KeyValue<>(2, null));
-
-        final List> rightTableDeleteEvents = Arrays.asList(new KeyValue<>("1", null));
-        IntegrationTestUtils.produceKeyValuesSynchronously(RIGHT_TABLE, rightTableDeleteEvents, RIGHT_PROD_CONF, MOCK_TIME);
-        final Set> resultDeleted = new HashSet<>(IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
-                CONSUMER_CONFIG,
-                OUTPUT,
-                expectedDeleted.size()));
-        assertEquals(expectedDeleted, resultDeleted);
-
-        //Ensure the state stores have the correct values within:
-        final Set> expMatResults = new HashSet<>();
-        expMatResults.add(new KeyValue<>(3, "value1=3.77,value2=30"));
-        validateQueryableStoresContainExpectedKeyValues(expMatResults, currentMethodName);
+    @Parameterized.Parameters(name = "leftJoin={0}, optimization={1}")
+    public static Collection data() {
+        return Arrays.asList(
+            new Object[] {false, StreamsConfig.OPTIMIZE},
+            new Object[] {false, StreamsConfig.NO_OPTIMIZATION},
+            new Object[] {true, StreamsConfig.OPTIMIZE},
+            new Object[] {true, StreamsConfig.NO_OPTIMIZATION}
+        );
     }
 
     @Test
-    public void doLeftJoinFromRightThenDeleteRightEntity() throws Exception {
-        final List> leftTableEvents = Arrays.asList(
-                new KeyValue<>(1, 1.33f),
-                new KeyValue<>(2, 1.77f),
-                new KeyValue<>(3, 3.77f));
-        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
-        final String currentMethodName = new Object() { }
-                .getClass()
-                .getEnclosingMethod()
-                .getName();
-        createAndStartStreamsApplication(currentMethodName, true);
-
-        final List> rightTableEvents = Arrays.asList(
-                new KeyValue<>("1", 10L),  //partition 0
-                new KeyValue<>("2", 20L),  //partition 2
-                new KeyValue<>("3", 30L)); //partition 2
-        IntegrationTestUtils.produceKeyValuesSynchronously(RIGHT_TABLE, rightTableEvents, RIGHT_PROD_CONF, MOCK_TIME);
-
-        //Ensure that the joined values exist in the output
-        final Set> expected = new HashSet<>();
-        expected.add(new KeyValue<>(1, "value1=1.33,value2=10"));   //Will be deleted.
-        expected.add(new KeyValue<>(2, "value1=1.77,value2=10"));   //Will be deleted.
-        expected.add(new KeyValue<>(3, "value1=3.77,value2=30"));   //Will not be deleted.
-        //final HashSet> expected = new HashSet<>(buildExpectedResults(leftTableEvents, rightTableEvents, false));
-
-        final Set> result = new HashSet<>(IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
-                CONSUMER_CONFIG,
-                OUTPUT,
-                expected.size()));
-        assertEquals(expected, result);
-
-        //Now delete the RHS entity such that all matching keys have deletes propagated.
-        //This will exercise the joiner with the RHS value == null.
-        final Set> expectedDeleted = new HashSet<>();
-        expectedDeleted.add(new KeyValue<>(1, "value1=1.33,value2=null"));
-        expectedDeleted.add(new KeyValue<>(2, "value1=1.77,value2=null"));
-
-        final List> rightTableDeleteEvents = Arrays.asList(new KeyValue<>("1", null));
-        IntegrationTestUtils.produceKeyValuesSynchronously(RIGHT_TABLE, rightTableDeleteEvents, RIGHT_PROD_CONF, MOCK_TIME);
-        final Set> resultDeleted = new HashSet<>(IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
-                CONSUMER_CONFIG,
-                OUTPUT,
-                expectedDeleted.size()));
-        assertEquals(expectedDeleted, resultDeleted);
-
-        //Ensure the state stores have the correct values within:
-        final Set> expMatResults = new HashSet<>();
-        expMatResults.add(new KeyValue<>(1, "value1=1.33,value2=null"));
-        expMatResults.add(new KeyValue<>(2, "value1=1.77,value2=null"));
-        expMatResults.add(new KeyValue<>(3, "value1=3.77,value2=30"));
-        validateQueryableStoresContainExpectedKeyValues(expMatResults, currentMethodName);
+    public void doJoinFromLeftThenDeleteLeftEntity() {
+        final Topology topology = getTopology(streamsConfig, "store", leftJoin);
+        try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
+            final TestInputTopic right = driver.createInputTopic(RIGHT_TABLE, new StringSerializer(), new StringSerializer());
+            final TestInputTopic left = driver.createInputTopic(LEFT_TABLE, new StringSerializer(), new StringSerializer());
+            final TestOutputTopic outputTopic = driver.createOutputTopic(OUTPUT, new StringDeserializer(), new StringDeserializer());
+            final KeyValueStore store = driver.getKeyValueStore("store");
+
+            // Pre-populate the RHS records. This test is all about what happens when we add/remove LHS records
+            right.pipeInput("rhs1", "rhsValue1");
+            right.pipeInput("rhs2", "rhsValue2");
+            right.pipeInput("rhs3", "rhsValue3"); // this unreferenced FK won't show up in any results
+
+            assertThat(
+                outputTopic.readKeyValuesToMap(),
+                is(emptyMap())
+            );
+            assertThat(
+                asMap(store),
+                is(emptyMap())
+            );
+
+            left.pipeInput("lhs1", "lhsValue1|rhs1");
+            left.pipeInput("lhs2", "lhsValue2|rhs2");
+
+            {
+                final Map expected = mkMap(
+                    mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)"),
+                    mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)")
+                );
+                assertThat(
+                    outputTopic.readKeyValuesToMap(),
+                    is(expected)
+                );
+                assertThat(
+                    asMap(store),
+                    is(expected)
+                );
+            }
+
+            // Add another reference to an existing FK
+            left.pipeInput("lhs3", "lhsValue3|rhs1");
+            {
+                assertThat(
+                    outputTopic.readKeyValuesToMap(),
+                    is(mkMap(
+                        mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)")
+                    ))
+                );
+                assertThat(
+                    asMap(store),
+                    is(mkMap(
+                        mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)"),
+                        mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)"),
+                        mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)")
+                    ))
+                );
+            }
+            // Now delete one LHS entity such that one delete is propagated down to the output.
+
+            left.pipeInput("lhs1", (String) null);
+            assertThat(
+                outputTopic.readKeyValuesToMap(),
+                is(mkMap(
+                    mkEntry("lhs1", null)
+                ))
+            );
+            assertThat(
+                asMap(store),
+                is(mkMap(
+                    mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)"),
+                    mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)")
+                ))
+            );
+        }
     }
 
     @Test
-    public void doInnerJoinProduceNullsWhenValueHasNonMatchingForeignKey() throws Exception {
-        //There is no matching extracted foreign-key of 8 anywhere. Should not produce any output for INNER JOIN, only
-        //because the state is transitioning from oldValue=null -> newValue=8.33.
-        List> leftTableEvents = Arrays.asList(new KeyValue<>(1, 8.33f));
-        final List> rightTableEvents = Arrays.asList(new KeyValue<>("1", 10L)); //partition 0
-        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
-        IntegrationTestUtils.produceKeyValuesSynchronously(RIGHT_TABLE, rightTableEvents, RIGHT_PROD_CONF, MOCK_TIME);
-
-        final String currentMethodName = new Object() { }
-                .getClass()
-                .getEnclosingMethod()
-                .getName();
-        createAndStartStreamsApplication(currentMethodName, false);
-
-        //There is also no matching extracted foreign-key for 18 anywhere. This WILL produce a null output for INNER JOIN,
-        //since we cannot remember (maintain state) that the FK=8 also produced a null result.
-        leftTableEvents = Arrays.asList(new KeyValue<>(1, 18.00f));
-        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
-
-        final List> expected = new LinkedList<>();
-        expected.add(new KeyValue<>(1, null));
-
-        final List> result = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
-                CONSUMER_CONFIG,
-                OUTPUT,
-                expected.size());
-        assertEquals(result, expected);
-
-        //Another change to FK that has no match on the RHS will result in another null
-        leftTableEvents = Arrays.asList(new KeyValue<>(1, 100.00f));
-        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
-        //Consume the next event - note that we are using the same consumerGroupId, so this will consume a new event.
-        final List> result2 = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
-                CONSUMER_CONFIG,
-                OUTPUT,
-                expected.size());
-        assertEquals(result2, expected);
-
-        //Now set the LHS event FK to match the rightTableEvents key-value.
-        leftTableEvents = Arrays.asList(new KeyValue<>(1, 1.11f));
-
-        final List> expected3 = new LinkedList<>();
-        expected3.add(new KeyValue<>(1, "value1=1.11,value2=10"));
-
-        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
-        final List> result3 = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
-                CONSUMER_CONFIG,
-                OUTPUT,
-                expected3.size());
-        assertEquals(result3, expected3);
-
-        //Ensure the state stores have the correct values within:
-        final Set> expMatResults = new HashSet<>();
-        expMatResults.add(new KeyValue<>(1, "value1=1.11,value2=10"));
-        validateQueryableStoresContainExpectedKeyValues(expMatResults, currentMethodName);
+    public void doJoinFromRightThenDeleteRightEntity() {
+        final Topology topology = getTopology(streamsConfig, "store", leftJoin);
+        try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
+            final TestInputTopic right = driver.createInputTopic(RIGHT_TABLE, new StringSerializer(), new StringSerializer());
+            final TestInputTopic left = driver.createInputTopic(LEFT_TABLE, new StringSerializer(), new StringSerializer());
+            final TestOutputTopic outputTopic = driver.createOutputTopic(OUTPUT, new StringDeserializer(), new StringDeserializer());
+            final KeyValueStore store = driver.getKeyValueStore("store");
+
+            // Pre-populate the LHS records. This test is all about what happens when we add/remove RHS records
+            left.pipeInput("lhs1", "lhsValue1|rhs1");
+            left.pipeInput("lhs2", "lhsValue2|rhs2");
+            left.pipeInput("lhs3", "lhsValue3|rhs1");
+
+            assertThat(
+                outputTopic.readKeyValuesToMap(),
+                is(leftJoin
+                       ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,null)"),
+                               mkEntry("lhs2", "(lhsValue2|rhs2,null)"),
+                               mkEntry("lhs3", "(lhsValue3|rhs1,null)"))
+                       : emptyMap()
+                )
+            );
+            assertThat(
+                asMap(store),
+                is(leftJoin
+                       ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,null)"),
+                               mkEntry("lhs2", "(lhsValue2|rhs2,null)"),
+                               mkEntry("lhs3", "(lhsValue3|rhs1,null)"))
+                       : emptyMap()
+                )
+            );
+
+            right.pipeInput("rhs1", "rhsValue1");
+
+            assertThat(
+                outputTopic.readKeyValuesToMap(),
+                is(mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)"),
+                         mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)"))
+                )
+            );
+            assertThat(
+                asMap(store),
+                is(leftJoin
+                       ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)"),
+                               mkEntry("lhs2", "(lhsValue2|rhs2,null)"),
+                               mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)"))
+
+                       : mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)"),
+                               mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)"))
+                )
+            );
+
+            right.pipeInput("rhs2", "rhsValue2");
+
+            assertThat(
+                outputTopic.readKeyValuesToMap(),
+                is(mkMap(mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)")))
+            );
+            assertThat(
+                asMap(store),
+                is(mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)"),
+                         mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)"),
+                         mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)"))
+                )
+            );
+
+            right.pipeInput("rhs3", "rhsValue3"); // this unreferenced FK won't show up in any results
+
+            assertThat(
+                outputTopic.readKeyValuesToMap(),
+                is(emptyMap())
+            );
+            assertThat(
+                asMap(store),
+                is(mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)"),
+                         mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)"),
+                         mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)"))
+                )
+            );
+
+            // Now delete the RHS entity such that all matching keys have deletes propagated.
+            right.pipeInput("rhs1", (String) null);
+
+            assertThat(
+                outputTopic.readKeyValuesToMap(),
+                is(mkMap(mkEntry("lhs1", leftJoin ? "(lhsValue1|rhs1,null)" : null),
+                         mkEntry("lhs3", leftJoin ? "(lhsValue3|rhs1,null)" : null))
+                )
+            );
+            assertThat(
+                asMap(store),
+                is(leftJoin
+                       ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,null)"),
+                               mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)"),
+                               mkEntry("lhs3", "(lhsValue3|rhs1,null)"))
+
+                       : mkMap(mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)"))
+                )
+            );
+        }
     }
 
     @Test
-    public void doLeftJoinProduceJoinedResultsWhenValueHasNonMatchingForeignKey() throws Exception {
-        //There is no matching extracted foreign-key of 8 anywhere.
-        //However, it will still run the join function since this is LEFT join.
-        List> leftTableEvents = Arrays.asList(new KeyValue<>(1, 8.33f));
-        final List> rightTableEvents = Arrays.asList(new KeyValue<>("1", 10L)); //partition 0
-        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
-        IntegrationTestUtils.produceKeyValuesSynchronously(RIGHT_TABLE, rightTableEvents, RIGHT_PROD_CONF, MOCK_TIME);
-
-        final String currentMethodName = new Object() { }
-                .getClass()
-                .getEnclosingMethod()
-                .getName();
-        createAndStartStreamsApplication(currentMethodName, true);
-
-        final List> expected = new LinkedList<>();
-        expected.add(new KeyValue<>(1, "value1=8.33,value2=null"));
-        final List> result = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
-                CONSUMER_CONFIG,
-                OUTPUT,
-                expected.size());
-        assertEquals(expected, result);
-
-        //There is also no matching extracted foreign-key for 18 anywhere.
-        //However, it will still run the join function since this if LEFT join.
-        leftTableEvents = Arrays.asList(new KeyValue<>(1, 18.0f));
-        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
-
-        final List> expected2 = new LinkedList<>();
-        expected2.add(new KeyValue<>(1, "value1=18.0,value2=null"));
-        final List> result2 = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
-                CONSUMER_CONFIG,
-                OUTPUT,
-                expected2.size());
-        assertEquals(expected2, result2);
-
-
-        leftTableEvents = Arrays.asList(new KeyValue<>(1, 1.11f));
-        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
-
-        final List> expected3 = new LinkedList<>();
-        expected3.add(new KeyValue<>(1, "value1=1.11,value2=10"));
-        final List> result3 = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
-                CONSUMER_CONFIG,
-                OUTPUT,
-                expected3.size());
-        assertEquals(expected3, result3);
-
-        //Ensure the state stores have the correct values within:
-        final Set> expMatResults = new HashSet<>();
-        expMatResults.add(new KeyValue<>(1, "value1=1.11,value2=10"));
-        validateQueryableStoresContainExpectedKeyValues(expMatResults, currentMethodName);
+    public void shouldEmitTombstonedWhenDeletingNonJoiningRecords() {
+        final Topology topology = getTopology(streamsConfig, "store", leftJoin);
+        try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
+            final TestInputTopic right = driver.createInputTopic(RIGHT_TABLE, new StringSerializer(), new StringSerializer());
+            final TestInputTopic left = driver.createInputTopic(LEFT_TABLE, new StringSerializer(), new StringSerializer());
+            final TestOutputTopic outputTopic = driver.createOutputTopic(OUTPUT, new StringDeserializer(), new StringDeserializer());
+            final KeyValueStore store = driver.getKeyValueStore("store");
+
+            left.pipeInput("lhs1", "lhsValue1|rhs1");
+
+            {
+                final Map expected =
+                    leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,null)")) : emptyMap();
+                assertThat(
+                    outputTopic.readKeyValuesToMap(),
+                    is(expected)
+                );
+                assertThat(
+                    asMap(store),
+                    is(expected)
+                );
+            }
+
+            // Deleting a non-joining record produces an unnecessary tombstone for inner joins, because
+            // it's not possible to know whether a result was previously emitted.
+            // For the left join, the tombstone is necessary.
+            left.pipeInput("lhs1", (String) null);
+            {
+                assertThat(
+                    outputTopic.readKeyValuesToMap(),
+                    is(Utils.mkMap(mkEntry("lhs1", null)))
+                );
+                assertThat(
+                    asMap(store),
+                    is(emptyMap())
+                );
+            }
+
+            // Deleting a non-existing record is idempotent
+            left.pipeInput("lhs1", (String) null);
+            {
+                assertThat(
+                    outputTopic.readKeyValuesToMap(),
+                    is(emptyMap())
+                );
+                assertThat(
+                    asMap(store),
+                    is(emptyMap())
+                );
+            }
+        }
     }
 
     @Test
-    public void doInnerJoinFilterOutRapidlyChangingForeignKeyValues() throws Exception {
-        final List> leftTableEvents = Arrays.asList(
-                new KeyValue<>(1, 1.33f),
-                new KeyValue<>(2, 2.22f),
-                new KeyValue<>(3, -1.22f), //Won't be joined in
-                new KeyValue<>(4, -2.22f), //Won't be joined in
-                new KeyValue<>(5, 2.22f)
-        );
-
-        //Partitions pre-computed using the default Murmur2 hash, just to ensure that all 3 partitions will be exercised.
-        final List> rightTableEvents = Arrays.asList(
-                new KeyValue<>("0", 0L),  //partition 2
-                new KeyValue<>("1", 10L), //partition 0
-                new KeyValue<>("2", 20L), //partition 2
-                new KeyValue<>("3", 30L), //partition 2
-                new KeyValue<>("4", 40L), //partition 1
-                new KeyValue<>("5", 50L), //partition 0
-                new KeyValue<>("6", 60L), //partition 1
-                new KeyValue<>("7", 70L), //partition 0
-                new KeyValue<>("8", 80L), //partition 0
-                new KeyValue<>("9", 90L)  //partition 2
-        );
-
-        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
-        IntegrationTestUtils.produceKeyValuesSynchronously(RIGHT_TABLE, rightTableEvents, RIGHT_PROD_CONF, MOCK_TIME);
-
-        final Set> expected = new HashSet<>();
-        expected.add(new KeyValue<>(1, "value1=1.33,value2=10"));
-        expected.add(new KeyValue<>(2, "value1=2.22,value2=20"));
-        expected.add(new KeyValue<>(5, "value1=2.22,value2=20"));
-
-        final String currentMethodName = new Object() { }
-                .getClass()
-                .getEnclosingMethod()
-                .getName();
-        createAndStartStreamsApplication(currentMethodName, false);
-
-        final Set> result = new HashSet<>(IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
-                CONSUMER_CONFIG,
-                OUTPUT,
-                expected.size()));
-
-        assertEquals(result, expected);
-
-        //Rapidly change the foreign key, to validate that the hashing prevents incorrect results from being output,
-        //and that eventually the correct value is output.
-        final List> table1ForeignKeyChange = Arrays.asList(
-                new KeyValue<>(3, 2.22f), //Partition 2
-                new KeyValue<>(3, 3.33f), //Partition 2
-                new KeyValue<>(3, 4.44f), //Partition 1
-                new KeyValue<>(3, 5.55f), //Partition 0
-                new KeyValue<>(3, 9.99f), //Partition 2
-                new KeyValue<>(3, 8.88f), //Partition 0
-                new KeyValue<>(3, 0.23f), //Partition 2
-                new KeyValue<>(3, 7.77f), //Partition 0
-                new KeyValue<>(3, 6.66f), //Partition 1
-                new KeyValue<>(3, 1.11f)  //Partition 0 - This will be the final result.
-        );
-
-        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, table1ForeignKeyChange, LEFT_PROD_CONF, MOCK_TIME);
-        final List> resultTwo = IntegrationTestUtils.readKeyValues(OUTPUT, CONSUMER_CONFIG, 15 * 1000L, Integer.MAX_VALUE);
-
-        final List> expectedTwo = new LinkedList<>();
-        expectedTwo.add(new KeyValue<>(3, "value1=1.11,value2=10"));
-        assertArrayEquals(resultTwo.toArray(), expectedTwo.toArray());
-
-        //Ensure the state stores have the correct values within:
-        final Set> expMatResults = new HashSet<>();
-        expMatResults.addAll(expected);
-        expMatResults.addAll(expectedTwo);
-        validateQueryableStoresContainExpectedKeyValues(expMatResults, currentMethodName);
+    public void shouldNotEmitTombstonesWhenDeletingNonExistingRecords() {
+        final Topology topology = getTopology(streamsConfig, "store", leftJoin);
+        try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
+            final TestInputTopic right = driver.createInputTopic(RIGHT_TABLE, new StringSerializer(), new StringSerializer());
+            final TestInputTopic left = driver.createInputTopic(LEFT_TABLE, new StringSerializer(), new StringSerializer());
+            final TestOutputTopic outputTopic = driver.createOutputTopic(OUTPUT, new StringDeserializer(), new StringDeserializer());
+            final KeyValueStore store = driver.getKeyValueStore("store");
+
+            // Deleting a record that never existed doesn't need to emit tombstones.
+            left.pipeInput("lhs1", (String) null);
+            {
+                assertThat(
+                    outputTopic.readKeyValuesToMap(),
+                    is(emptyMap())
+                );
+                assertThat(
+                    asMap(store),
+                    is(emptyMap())
+                );
+            }
+        }
     }
 
     @Test
-    public void doLeftJoinFilterOutRapidlyChangingForeignKeyValues() throws Exception {
-        final List> leftTableEvents = Arrays.asList(
-                new KeyValue<>(1, 1.33f),
-                new KeyValue<>(2, 2.22f)
-        );
-
-        //Partitions pre-computed using the default Murmur2 hash, just to ensure that all 3 partitions will be exercised.
-        final List> rightTableEvents = Arrays.asList(
-                new KeyValue<>("0", 0L),  //partition 2
-                new KeyValue<>("1", 10L), //partition 0
-                new KeyValue<>("2", 20L), //partition 2
-                new KeyValue<>("3", 30L), //partition 2
-                new KeyValue<>("4", 40L), //partition 1
-                new KeyValue<>("5", 50L), //partition 0
-                new KeyValue<>("6", 60L), //partition 1
-                new KeyValue<>("7", 70L), //partition 0
-                new KeyValue<>("8", 80L), //partition 0
-                new KeyValue<>("9", 90L)  //partition 2
-        );
-
-        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, leftTableEvents, LEFT_PROD_CONF, MOCK_TIME);
-        IntegrationTestUtils.produceKeyValuesSynchronously(RIGHT_TABLE, rightTableEvents, RIGHT_PROD_CONF, MOCK_TIME);
-
-        final Set> expected = new HashSet<>();
-        expected.add(new KeyValue<>(1, "value1=1.33,value2=10"));
-        expected.add(new KeyValue<>(2, "value1=2.22,value2=20"));
-
-        final String currentMethodName = new Object() { }
-                .getClass()
-                .getEnclosingMethod()
-                .getName();
-        createAndStartStreamsApplication(currentMethodName, false);
-
-        final Set> result = new HashSet<>(IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
-                CONSUMER_CONFIG,
-                OUTPUT,
-                expected.size()));
-
-        assertEquals(result, expected);
-
-        //Rapidly change the foreign key, to validate that the hashing prevents incorrect results from being output,
-        //and that eventually the correct value is output.
-        final List> table1ForeignKeyChange = Arrays.asList(
-                new KeyValue<>(3, 2.22f), //Partition 2
-                new KeyValue<>(3, 3.33f), //Partition 2
-                new KeyValue<>(3, 4.44f), //Partition 1
-                new KeyValue<>(3, 5.55f), //Partition 0
-                new KeyValue<>(3, 9.99f), //Partition 2
-                new KeyValue<>(3, 8.88f), //Partition 0
-                new KeyValue<>(3, 0.23f), //Partition 2
-                new KeyValue<>(3, 7.77f), //Partition 0
-                new KeyValue<>(3, 6.66f), //Partition 1
-                new KeyValue<>(3, 1.11f)  //Partition 0 - This will be the final result.
-        );
-
-        IntegrationTestUtils.produceKeyValuesSynchronously(LEFT_TABLE, table1ForeignKeyChange, LEFT_PROD_CONF, MOCK_TIME);
-        final List> resultTwo = IntegrationTestUtils.readKeyValues(OUTPUT, CONSUMER_CONFIG, 15 * 1000L, Integer.MAX_VALUE);
-
-        final List> expectedTwo = new LinkedList<>();
-        expectedTwo.add(new KeyValue<>(3, "value1=1.11,value2=10"));
-
-        assertArrayEquals(resultTwo.toArray(), expectedTwo.toArray());
-
-        //Ensure the state stores have the correct values within:
-        final Set> expMatResults = new HashSet<>();
-        expMatResults.addAll(expected);
-        expMatResults.addAll(expectedTwo);
-        validateQueryableStoresContainExpectedKeyValues(expMatResults, currentMethodName);
+    public void joinShouldProduceNullsWhenValueHasNonMatchingForeignKey() {
+        final Topology topology = getTopology(streamsConfig, "store", leftJoin);
+        try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
+            final TestInputTopic right = driver.createInputTopic(RIGHT_TABLE, new StringSerializer(), new StringSerializer());
+            final TestInputTopic left = driver.createInputTopic(LEFT_TABLE, new StringSerializer(), new StringSerializer());
+            final TestOutputTopic outputTopic = driver.createOutputTopic(OUTPUT, new StringDeserializer(), new StringDeserializer());
+            final KeyValueStore store = driver.getKeyValueStore("store");
+
+            left.pipeInput("lhs1", "lhsValue1|rhs1");
+            // no output for a new inner join on a non-existent FK
+            // the left join of course emits the half-joined output
+            assertThat(
+                outputTopic.readKeyValuesToMap(),
+                is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,null)")) : emptyMap())
+            );
+            assertThat(
+                asMap(store),
+                is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,null)")) : emptyMap())
+            );
+            // "moving" our subscription to another non-existent FK results in an unnecessary tombstone for inner join,
+            // since it impossible to know whether the prior FK existed or not (and thus whether any results have
+            // previously been emitted)
+            // The left join emits a _necessary_ update (since the lhs record has actually changed)
+            left.pipeInput("lhs1", "lhsValue1|rhs2");
+            assertThat(
+                outputTopic.readKeyValuesToMap(),
+                is(mkMap(
+                    mkEntry("lhs1", leftJoin ? "(lhsValue1|rhs2,null)" : null)
+                ))
+            );
+            assertThat(
+                asMap(store),
+                is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs2,null)")) : emptyMap())
+            );
+            // of course, moving it again to yet another non-existent FK has the same effect
+            left.pipeInput("lhs1", "lhsValue1|rhs3");
+            assertThat(
+                outputTopic.readKeyValuesToMap(),
+                is(mkMap(
+                    mkEntry("lhs1", leftJoin ? "(lhsValue1|rhs3,null)" : null)
+                ))
+            );
+            assertThat(
+                asMap(store),
+                is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs3,null)")) : emptyMap())
+            );
+
+            // Adding an RHS record now, so that we can demonstrate "moving" from a non-existent FK to an existent one
+            // This RHS key was previously referenced, but it's not referenced now, so adding this record should
+            // result in no changes whatsoever.
+            right.pipeInput("rhs1", "rhsValue1");
+            assertThat(
+                outputTopic.readKeyValuesToMap(),
+                is(emptyMap())
+            );
+            assertThat(
+                asMap(store),
+                is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs3,null)")) : emptyMap())
+            );
+
+            // now, we change to a FK that exists, and see the join completes
+            left.pipeInput("lhs1", "lhsValue1|rhs1");
+            assertThat(
+                outputTopic.readKeyValuesToMap(),
+                is(mkMap(
+                    mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)")
+                ))
+            );
+            assertThat(
+                asMap(store),
+                is(mkMap(
+                    mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)")
+                ))
+            );
+
+            // but if we update it again to a non-existent one, we'll get a tombstone for the inner join, and the
+            // left join updates appropriately.
+            left.pipeInput("lhs1", "lhsValue1|rhs2");
+            assertThat(
+                outputTopic.readKeyValuesToMap(),
+                is(mkMap(
+                    mkEntry("lhs1", leftJoin ? "(lhsValue1|rhs2,null)" : null)
+                ))
+            );
+            assertThat(
+                asMap(store),
+                is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs2,null)")) : emptyMap())
+            );
+        }
     }
 
-    private void createAndStartStreamsApplication(final String queryableStoreName, final boolean leftJoin) {
-        streamsConfig.put(StreamsConfig.APPLICATION_ID_CONFIG, "ktable-ktable-joinOnForeignKey-" + queryableStoreName);
-        streams = prepareTopology(queryableStoreName, leftJoin);
-        streamsTwo = prepareTopology(queryableStoreName, leftJoin);
-        streamsThree = prepareTopology(queryableStoreName, leftJoin);
-        streams.start();
-        streamsTwo.start();
-        streamsThree.start();
+    private static Map asMap(final KeyValueStore store) {
+        final HashMap result = new HashMap<>();
+        store.all().forEachRemaining(kv -> result.put(kv.key, kv.value));
+        return result;
     }
 
-    // These are hardwired into the test logic for readability sake.
-    // Do not change unless you want to change all the test results as well.
-    private ValueJoiner joiner = (value1, value2) -> "value1=" + value1 + ",value2=" + value2;
-    //Do not change. See above comment.
-    private Function tableOneKeyExtractor = value -> Integer.toString((int) value.floatValue());
-
-    private void validateQueryableStoresContainExpectedKeyValues(final Set> expectedResult,
-                                                                 final String queryableStoreName) {
-        final ReadOnlyKeyValueStore myJoinStoreOne = streams.store(queryableStoreName,
-                QueryableStoreTypes.keyValueStore());
-
-        final ReadOnlyKeyValueStore myJoinStoreTwo = streamsTwo.store(queryableStoreName,
-                QueryableStoreTypes.keyValueStore());
-
-        final ReadOnlyKeyValueStore myJoinStoreThree = streamsThree.store(queryableStoreName,
-                QueryableStoreTypes.keyValueStore());
-
-        // store only keeps last set of values, not entire stream of value changes
-        final Map expectedInStore = new HashMap<>();
-        for (final KeyValue expected : expectedResult) {
-            expectedInStore.put(expected.key, expected.value);
-        }
-
-        // depending on partition assignment, the values will be in one of the three stream clients.
-        for (final Map.Entry expected : expectedInStore.entrySet()) {
-            final String one = myJoinStoreOne.get(expected.getKey());
-            final String two = myJoinStoreTwo.get(expected.getKey());
-            final String three = myJoinStoreThree.get(expected.getKey());
-
-            String result;
-            if (one != null)
-                result = one;
-            else if (two != null)
-                result = two;
-            else if (three != null)
-                result = three;
-            else
-                throw new RuntimeException("Cannot find key " + expected.getKey() + " in any of the state stores");
-            assertEquals(expected.getValue(), result);
-        }
-
-        //Merge all the iterators together to ensure that their sum equals the total set of expected elements.
-        final KeyValueIterator allOne = myJoinStoreOne.all();
-        final KeyValueIterator allTwo = myJoinStoreTwo.all();
-        final KeyValueIterator allThree = myJoinStoreThree.all();
-
-        final List> all = new LinkedList<>();
-
-        while (allOne.hasNext()) {
-            all.add(allOne.next());
-        }
-        while (allTwo.hasNext()) {
-            all.add(allTwo.next());
-        }
-        while (allThree.hasNext()) {
-            all.add(allThree.next());
-        }
-        allOne.close();
-        allTwo.close();
-        allThree.close();
+    private static Topology getTopology(final Properties streamsConfig,
+                                        final String queryableStoreName,
+                                        final boolean leftJoin) {
+        final StreamsBuilder builder = new StreamsBuilder();
 
-        for (final KeyValue elem : all) {
-            assertTrue(expectedResult.contains(elem));
-        }
-    }
+        final KTable left = builder.table(LEFT_TABLE, Consumed.with(Serdes.String(), Serdes.String()));
+        final KTable right = builder.table(RIGHT_TABLE, Consumed.with(Serdes.String(), Serdes.String()));
 
-    private KafkaStreams prepareTopology(final String queryableStoreName, final boolean leftJoin) {
-        final StreamsBuilder builder = new StreamsBuilder();
+        final Materialized> materialized =
+            Materialized.as(Stores.inMemoryKeyValueStore(queryableStoreName))
+                .withKeySerde(Serdes.String())
+                .withValueSerde(Serdes.String())
+                .withCachingDisabled();
 
-        final KTable left = builder.table(LEFT_TABLE, Consumed.with(Serdes.Integer(), Serdes.Float()));
-        final KTable right = builder.table(RIGHT_TABLE, Consumed.with(Serdes.String(), Serdes.Long()));
-
-        final Materialized> materialized;
-        if (queryableStoreName != null) {
-            materialized = Materialized.>as(queryableStoreName)
-                    .withKeySerde(Serdes.Integer())
-                    .withValueSerde(Serdes.String())
-                    .withCachingDisabled();
-        } else {
-            throw new RuntimeException("Current implementation of join on foreign key requires a materialized store");
-        }
+        final Function extractor = value -> value.split("\\|")[1];
+        final ValueJoiner joiner = (value1, value2) -> "(" + value1 + "," + value2 + ")";
 
         if (leftJoin)
-            left.leftJoin(right, tableOneKeyExtractor, joiner, Named.as("customName"), materialized)
+            left.leftJoin(right,
+                          extractor,
+                          joiner,
+                          materialized)
                 .toStream()
-                .to(OUTPUT, Produced.with(Serdes.Integer(), Serdes.String()));
+                .to(OUTPUT, Produced.with(Serdes.String(), Serdes.String()));
         else
-            left.join(right, tableOneKeyExtractor, joiner, materialized)
+            left.join(right,
+                      extractor,
+                      joiner,
+                      materialized)
                 .toStream()
-                .to(OUTPUT, Produced.with(Serdes.Integer(), Serdes.String()));
-
-        final Topology topology = builder.build(streamsConfig);
+                .to(OUTPUT, Produced.with(Serdes.String(), Serdes.String()));
 
-        return new KafkaStreams(topology, streamsConfig);
+        return builder.build(streamsConfig);
     }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplierTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplierTest.java
new file mode 100644
index 0000000000000..3ec19de384254
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplierTest.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.kstream.internals.foreignkeyjoin;
+
+import org.apache.kafka.common.header.internals.RecordHeaders;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.kstream.ValueJoiner;
+import org.apache.kafka.streams.kstream.internals.KTableValueGetter;
+import org.apache.kafka.streams.kstream.internals.KTableValueGetterSupplier;
+import org.apache.kafka.streams.processor.MockProcessorContext;
+import org.apache.kafka.streams.processor.Processor;
+import org.apache.kafka.streams.processor.ProcessorContext;
+import org.apache.kafka.streams.state.ValueAndTimestamp;
+import org.apache.kafka.streams.state.internals.Murmur3;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.collection.IsEmptyCollection.empty;
+
+public class SubscriptionResolverJoinProcessorSupplierTest {
+    private static final StringSerializer STRING_SERIALIZER = new StringSerializer();
+    private static final ValueJoiner JOINER =
+        (value1, value2) -> "(" + value1 + "," + value2 + ")";
+
+    private static class TestKTableValueGetterSupplier implements KTableValueGetterSupplier {
+        private final Map map = new HashMap<>();
+
+        @Override
+        public KTableValueGetter get() {
+            return new KTableValueGetter() {
+                @Override
+                public void init(final ProcessorContext context) {
+                }
+
+                @Override
+                public ValueAndTimestamp get(final K key) {
+                    return ValueAndTimestamp.make(map.get(key), -1);
+                }
+
+                @Override
+                public void close() {}
+            };
+        }
+
+        @Override
+        public String[] storeNames() {
+            return new String[0];
+        }
+
+        void put(final K key, final V value) {
+            map.put(key, value);
+        }
+    }
+
+    @Test
+    public void shouldNotForwardWhenHashDoesNotMatch() {
+        final TestKTableValueGetterSupplier valueGetterSupplier =
+            new TestKTableValueGetterSupplier<>();
+        final boolean leftJoin = false;
+        final SubscriptionResolverJoinProcessorSupplier processorSupplier =
+            new SubscriptionResolverJoinProcessorSupplier<>(
+                valueGetterSupplier,
+                STRING_SERIALIZER,
+                JOINER,
+                leftJoin
+            );
+        final Processor> processor = processorSupplier.get();
+        final MockProcessorContext context = new MockProcessorContext();
+        processor.init(context);
+        context.setRecordMetadata("topic", 0, 0, new RecordHeaders(), 0);
+
+        valueGetterSupplier.put("lhs1", "lhsValue");
+        final long[] oldHash = Murmur3.hash128(STRING_SERIALIZER.serialize("topic-join-resolver", "oldLhsValue"));
+        processor.process("lhs1", new SubscriptionResponseWrapper<>(oldHash, "rhsValue"));
+        final List forwarded = context.forwarded();
+        assertThat(forwarded, empty());
+    }
+
+    @Test
+    public void shouldIgnoreUpdateWhenLeftHasBecomeNull() {
+        final TestKTableValueGetterSupplier valueGetterSupplier =
+            new TestKTableValueGetterSupplier<>();
+        final boolean leftJoin = false;
+        final SubscriptionResolverJoinProcessorSupplier processorSupplier =
+            new SubscriptionResolverJoinProcessorSupplier<>(
+                valueGetterSupplier,
+                STRING_SERIALIZER,
+                JOINER,
+                leftJoin
+            );
+        final Processor> processor = processorSupplier.get();
+        final MockProcessorContext context = new MockProcessorContext();
+        processor.init(context);
+        context.setRecordMetadata("topic", 0, 0, new RecordHeaders(), 0);
+
+        valueGetterSupplier.put("lhs1", null);
+        final long[] hash = Murmur3.hash128(STRING_SERIALIZER.serialize("topic-join-resolver", "lhsValue"));
+        processor.process("lhs1", new SubscriptionResponseWrapper<>(hash, "rhsValue"));
+        final List forwarded = context.forwarded();
+        assertThat(forwarded, empty());
+    }
+
+    @Test
+    public void shouldForwardWhenHashMatches() {
+        final TestKTableValueGetterSupplier valueGetterSupplier =
+            new TestKTableValueGetterSupplier<>();
+        final boolean leftJoin = false;
+        final SubscriptionResolverJoinProcessorSupplier processorSupplier =
+            new SubscriptionResolverJoinProcessorSupplier<>(
+                valueGetterSupplier,
+                STRING_SERIALIZER,
+                JOINER,
+                leftJoin
+            );
+        final Processor> processor = processorSupplier.get();
+        final MockProcessorContext context = new MockProcessorContext();
+        processor.init(context);
+        context.setRecordMetadata("topic", 0, 0, new RecordHeaders(), 0);
+
+        valueGetterSupplier.put("lhs1", "lhsValue");
+        final long[] hash = Murmur3.hash128(STRING_SERIALIZER.serialize("topic-join-resolver", "lhsValue"));
+        processor.process("lhs1", new SubscriptionResponseWrapper<>(hash, "rhsValue"));
+        final List forwarded = context.forwarded();
+        assertThat(forwarded.size(), is(1));
+        assertThat(forwarded.get(0).keyValue(), is(new KeyValue<>("lhs1", "(lhsValue,rhsValue)")));
+    }
+
+    @Test
+    public void shouldEmitTombstoneForInnerJoinWhenRightIsNull() {
+        final TestKTableValueGetterSupplier valueGetterSupplier =
+            new TestKTableValueGetterSupplier<>();
+        final boolean leftJoin = false;
+        final SubscriptionResolverJoinProcessorSupplier processorSupplier =
+            new SubscriptionResolverJoinProcessorSupplier<>(
+                valueGetterSupplier,
+                STRING_SERIALIZER,
+                JOINER,
+                leftJoin
+            );
+        final Processor> processor = processorSupplier.get();
+        final MockProcessorContext context = new MockProcessorContext();
+        processor.init(context);
+        context.setRecordMetadata("topic", 0, 0, new RecordHeaders(), 0);
+
+        valueGetterSupplier.put("lhs1", "lhsValue");
+        final long[] hash = Murmur3.hash128(STRING_SERIALIZER.serialize("topic-join-resolver", "lhsValue"));
+        processor.process("lhs1", new SubscriptionResponseWrapper<>(hash, null));
+        final List forwarded = context.forwarded();
+        assertThat(forwarded.size(), is(1));
+        assertThat(forwarded.get(0).keyValue(), is(new KeyValue<>("lhs1", null)));
+    }
+
+    @Test
+    public void shouldEmitResultForLeftJoinWhenRightIsNull() {
+        final TestKTableValueGetterSupplier valueGetterSupplier =
+            new TestKTableValueGetterSupplier<>();
+        final boolean leftJoin = true;
+        final SubscriptionResolverJoinProcessorSupplier processorSupplier =
+            new SubscriptionResolverJoinProcessorSupplier<>(
+                valueGetterSupplier,
+                STRING_SERIALIZER,
+                JOINER,
+                leftJoin
+            );
+        final Processor> processor = processorSupplier.get();
+        final MockProcessorContext context = new MockProcessorContext();
+        processor.init(context);
+        context.setRecordMetadata("topic", 0, 0, new RecordHeaders(), 0);
+
+        valueGetterSupplier.put("lhs1", "lhsValue");
+        final long[] hash = Murmur3.hash128(STRING_SERIALIZER.serialize("topic-join-resolver", "lhsValue"));
+        processor.process("lhs1", new SubscriptionResponseWrapper<>(hash, null));
+        final List forwarded = context.forwarded();
+        assertThat(forwarded.size(), is(1));
+        assertThat(forwarded.get(0).keyValue(), is(new KeyValue<>("lhs1", "(lhsValue,null)")));
+    }
+
+    @Test
+    public void shouldEmitTombstoneForLeftJoinWhenRightIsNullAndLeftIsNull() {
+        final TestKTableValueGetterSupplier valueGetterSupplier =
+            new TestKTableValueGetterSupplier<>();
+        final boolean leftJoin = true;
+        final SubscriptionResolverJoinProcessorSupplier processorSupplier =
+            new SubscriptionResolverJoinProcessorSupplier<>(
+                valueGetterSupplier,
+                STRING_SERIALIZER,
+                JOINER,
+                leftJoin
+            );
+        final Processor> processor = processorSupplier.get();
+        final MockProcessorContext context = new MockProcessorContext();
+        processor.init(context);
+        context.setRecordMetadata("topic", 0, 0, new RecordHeaders(), 0);
+
+        valueGetterSupplier.put("lhs1", null);
+        final long[] hash = null;
+        processor.process("lhs1", new SubscriptionResponseWrapper<>(hash, null));
+        final List forwarded = context.forwarded();
+        assertThat(forwarded.size(), is(1));
+        assertThat(forwarded.get(0).keyValue(), is(new KeyValue<>("lhs1", null)));
+    }
+}
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java
index 947e65e9c0c5f..7fb8b68cbe40d 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/processor/MockProcessorContext.java
@@ -161,6 +161,15 @@ public long timestamp() {
         public KeyValue keyValue() {
             return keyValue;
         }
+
+        @Override
+        public String toString() {
+            return "CapturedForward{" +
+                "childName='" + childName + '\'' +
+                ", timestamp=" + timestamp +
+                ", keyValue=" + keyValue +
+                '}';
+        }
     }
 
     // constructors ================================================

From 21a53b023a1c91e5970bf82374759db05da1fc74 Mon Sep 17 00:00:00 2001
From: Nikolay 
Date: Thu, 17 Oct 2019 08:53:14 +0300
Subject: [PATCH 0739/1071] KAFKA-8104: Consumer cannot rejoin to the group
 after rebalancing (#7460)

This PR contains the fix of race condition bug between "consumer thread" and "consumer coordinator heartbeat thread". It reproduces in many production environments.

Condition for reproducing:

1. Consumer thread initiates rejoin to the group because of commit timeout. Call of AbstractCoordinator#joinGroupIfNeeded which leads to sendJoinGroupRequest.
2. JoinGroupResponseHandler writes to the AbstractCoordinator.this.generation new generation data and leaves the synchronized section.
3. Heartbeat thread executes mabeLeaveGroup and clears generation data via resetGenerationOnLeaveGroup.
4. Consumer thread executes onJoinComplete(generation.generationId, generation.memberId, generation.protocol, memberAssignment); with the cleared generation data. This leads to the corresponding exception.

Reviewers: Guozhang Wang 
---
 .../internals/AbstractCoordinator.java        | 76 +++++++++++++------
 .../internals/ConsumerCoordinatorTest.java    | 47 ++++++++++++
 2 files changed, 100 insertions(+), 23 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index a44e7db4367d3..057cfb04115b9 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -404,14 +404,32 @@ boolean joinGroupIfNeeded(final Timer timer) {
             }
 
             if (future.succeeded()) {
-                // Duplicate the buffer in case `onJoinComplete` does not complete and needs to be retried.
-                ByteBuffer memberAssignment = future.value().duplicate();
-                onJoinComplete(generation.generationId, generation.memberId, generation.protocol, memberAssignment);
+                Generation generationSnapshot;
+
+                // Generation data maybe concurrently cleared by Heartbeat thread.
+                // Can't use synchronized for {@code onJoinComplete}, because it can be long enough
+                // and  shouldn't block hearbeat thread.
+                // See {@link PlaintextConsumerTest#testMaxPollIntervalMsDelayInAssignment
+                synchronized (this) {
+                    generationSnapshot = this.generation;
+                }
 
-                // We reset the join group future only after the completion callback returns. This ensures
-                // that if the callback is woken up, we will retry it on the next joinGroupIfNeeded.
-                resetJoinGroupFuture();
-                needsJoinPrepare = true;
+                if (generationSnapshot != Generation.NO_GENERATION) {
+                    // Duplicate the buffer in case `onJoinComplete` does not complete and needs to be retried.
+                    ByteBuffer memberAssignment = future.value().duplicate();
+
+                    onJoinComplete(generationSnapshot.generationId, generationSnapshot.memberId, generationSnapshot.protocol, memberAssignment);
+
+                    // We reset the join group future only after the completion callback returns. This ensures
+                    // that if the callback is woken up, we will retry it on the next joinGroupIfNeeded.
+                    resetJoinGroupFuture();
+                    needsJoinPrepare = true;
+                } else {
+                    log.info("Generation data was cleared by heartbeat thread. Initiating rejoin.");
+                    resetStateAndRejoin();
+
+                    return false;
+                }
             } else {
                 resetJoinGroupFuture();
                 final RuntimeException exception = future.exception();
@@ -433,6 +451,11 @@ private synchronized void resetJoinGroupFuture() {
         this.joinFuture = null;
     }
 
+    private void resetStateAndRejoin() {
+        rejoinNeeded = true;
+        state = MemberState.UNJOINED;
+    }
+
     private synchronized RequestFuture initiateJoinGroup() {
         // we store the join future in case we are woken up by the user after beginning the
         // rebalance in the call to poll below. This ensures that we do not mistakenly attempt
@@ -455,16 +478,21 @@ public void onSuccess(ByteBuffer value) {
                     // handle join completion in the callback so that the callback will be invoked
                     // even if the consumer is woken up before finishing the rebalance
                     synchronized (AbstractCoordinator.this) {
-                        log.info("Successfully joined group with generation {}", generation.generationId);
-                        state = MemberState.STABLE;
-                        rejoinNeeded = false;
-                        // record rebalance latency
-                        lastRebalanceEndMs = time.milliseconds();
-                        sensors.successfulRebalanceSensor.record(lastRebalanceEndMs - lastRebalanceStartMs);
-                        lastRebalanceStartMs = -1L;
-
-                        if (heartbeatThread != null)
-                            heartbeatThread.enable();
+                        if (generation != Generation.NO_GENERATION) {
+                            log.info("Successfully joined group with generation {}", generation.generationId);
+                            state = MemberState.STABLE;
+                            rejoinNeeded = false;
+                            // record rebalance latency
+                            lastRebalanceEndMs = time.milliseconds();
+                            sensors.successfulRebalanceSensor.record(lastRebalanceEndMs - lastRebalanceStartMs);
+                            lastRebalanceStartMs = -1L;
+
+                            if (heartbeatThread != null)
+                                heartbeatThread.enable();
+                        } else {
+                            log.info("Generation data was cleared by heartbeat thread. Rejoin failed.");
+                            recordRebalanceFailure();
+                        }
                     }
                 }
 
@@ -473,10 +501,14 @@ public void onFailure(RuntimeException e) {
                     // we handle failures below after the request finishes. if the join completes
                     // after having been woken up, the exception is ignored and we will rejoin
                     synchronized (AbstractCoordinator.this) {
-                        state = MemberState.UNJOINED;
-                        sensors.failedRebalanceSensor.record();
+                        recordRebalanceFailure();
                     }
                 }
+
+                private void recordRebalanceFailure() {
+                    state = MemberState.UNJOINED;
+                    sensors.failedRebalanceSensor.record();
+                }
             });
         }
         return joinFuture;
@@ -584,8 +616,7 @@ public void handle(JoinGroupResponse joinResponse, RequestFuture fut
                 synchronized (AbstractCoordinator.this) {
                     AbstractCoordinator.this.generation = new Generation(OffsetCommitRequest.DEFAULT_GENERATION_ID,
                             joinResponse.data().memberId(), null);
-                    AbstractCoordinator.this.rejoinNeeded = true;
-                    AbstractCoordinator.this.state = MemberState.UNJOINED;
+                    AbstractCoordinator.this.resetStateAndRejoin();
                 }
                 future.raise(error);
             } else {
@@ -822,8 +853,7 @@ final synchronized boolean hasValidMemberId() {
 
     private synchronized void resetGeneration() {
         this.generation = Generation.NO_GENERATION;
-        this.state = MemberState.UNJOINED;
-        this.rejoinNeeded = true;
+        resetStateAndRejoin();
     }
 
     synchronized void resetGenerationOnResponseError(ApiKeys api, Errors error) {
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index 068f7bd8809e3..930eb54cc2b0c 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -2279,6 +2279,53 @@ public void testCommitOffsetRequestAsyncAlwaysReceiveFencedException() {
                 coordinator.commitOffsetsSync(singletonMap(t1p, new OffsetAndMetadata(100L)), time.timer(Long.MAX_VALUE)));
     }
 
+    @Test
+    public void testConsumerRejoinAfterRebalance() throws Exception {
+        try (ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, false, Optional.of("group-id"))) {
+            coordinator.ensureActiveGroup();
+
+            prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.REBALANCE_IN_PROGRESS);
+
+            assertThrows(CommitFailedException.class, () -> coordinator.commitOffsetsSync(
+                singletonMap(t1p, new OffsetAndMetadata(100L)),
+                time.timer(Long.MAX_VALUE)));
+
+            assertFalse(client.hasPendingResponses());
+            assertFalse(client.hasInFlightRequests());
+
+            int generationId = 42;
+            String memberId = "consumer-42";
+
+            client.prepareResponse(joinGroupFollowerResponse(generationId, memberId, "leader", Errors.NONE));
+
+            MockTime time = new MockTime(1);
+
+            //onJoinPrepare will be executed and onJoinComplete will not.
+            boolean res = coordinator.joinGroupIfNeeded(time.timer(2));
+
+            assertFalse(res);
+            assertFalse(client.hasPendingResponses());
+            //SynGroupRequest not responded.
+            assertEquals(1, client.inFlightRequestCount());
+            assertEquals(generationId, coordinator.generation().generationId);
+            assertEquals(memberId, coordinator.generation().memberId);
+
+            // Imitating heartbeat thread that clears generation data.
+            coordinator.maybeLeaveGroup("Clear generation data.");
+
+            assertEquals(AbstractCoordinator.Generation.NO_GENERATION, coordinator.generation());
+
+            client.respond(syncGroupResponse(singletonList(t1p), Errors.NONE));
+
+            //Join future should succeed but generation already cleared so result of join is false.
+            res = coordinator.joinGroupIfNeeded(time.timer(1));
+
+            assertFalse(res);
+            assertFalse(client.hasPendingResponses());
+            assertFalse(client.hasInFlightRequests());
+        }
+    }
+
     private void receiveFencedInstanceIdException() {
         subscriptions.assignFromUser(singleton(t1p));
 

From de5254db7284bbcb44661518c34ff2a38f8256c1 Mon Sep 17 00:00:00 2001
From: Antony Stubbs 
Date: Thu, 17 Oct 2019 08:00:39 +0100
Subject: [PATCH 0740/1071] KAFKA-8884: class cast exception improvement
 (#7309)

Reviewers: John Roesler , Matthias J. Sax 
---
 .../processor/internals/ProcessorNode.java    | 18 ++++++-
 .../streams/processor/internals/SinkNode.java |  4 +-
 .../internals/ProcessorNodeTest.java          | 52 ++++++++++++++++++-
 3 files changed, 70 insertions(+), 4 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorNode.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorNode.java
index 6e10e83830056..3a5a7db965154 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorNode.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorNode.java
@@ -114,7 +114,23 @@ public void close() {
 
     public void process(final K key, final V value) {
         final long startNs = time.nanoseconds();
-        processor.process(key, value);
+        try {
+            processor.process(key, value);
+        } catch (final ClassCastException e) {
+            final String keyClass = key == null ? "unknown because key is null" : key.getClass().getName();
+            final String valueClass = value == null ? "unknown because value is null" : value.getClass().getName();
+            throw new StreamsException(String.format("ClassCastException invoking Processor. Do the Processor's "
+                    + "input types match the deserialized types? Check the Serde setup and change the default Serdes in "
+                    + "StreamConfig or provide correct Serdes via method parameters. Make sure the Processor can accept "
+                    + "the deserialized input of type key: %s, and value: %s.%n"
+                    + "Note that although incorrect Serdes are a common cause of error, the cast exception might have "
+                    + "another cause (in user code, for example). For example, if a processor wires in a store, but casts "
+                    + "the generics incorrectly, a class cast exception could be raised during processing, but the "
+                    + "cause would not be wrong Serdes.",
+                    keyClass,
+                    valueClass),
+                e);
+        }
         nodeMetrics.nodeProcessTimeSensor.record(time.nanoseconds() - startNs);
     }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/SinkNode.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/SinkNode.java
index 73bffc80ed37f..b94291b3503fa 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/SinkNode.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/SinkNode.java
@@ -91,9 +91,9 @@ public void process(final K key, final V value) {
             final String keyClass = key == null ? "unknown because key is null" : key.getClass().getName();
             final String valueClass = value == null ? "unknown because value is null" : value.getClass().getName();
             throw new StreamsException(
-                    String.format("A serializer (key: %s / value: %s) is not compatible to the actual key or value type " +
+                    String.format("ClassCastException while producing data to a sink topic. A serializer (key: %s / value: %s) is not compatible to the actual key or value type " +
                                     "(key type: %s / value type: %s). Change the default Serdes in StreamConfig or " +
-                                    "provide correct Serdes via method parameters.",
+                                    "provide correct Serdes via method parameters (for example if using the DSL, `#to(String topic, Produced produced)` with `Produced.keySerde(WindowedSerdes.timeWindowedSerdeFrom(String.class))`).",
                                     keySerializer.getClass().getName(),
                                     valSerializer.getClass().getName(),
                                     keyClass,
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorNodeTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorNodeTest.java
index 42ad94e23cb0b..3ae38524b3961 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorNodeTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorNodeTest.java
@@ -16,10 +16,18 @@
  */
 package org.apache.kafka.streams.processor.internals;
 
+import java.util.Arrays;
+import java.util.Properties;
 import org.apache.kafka.common.metrics.JmxReporter;
 import org.apache.kafka.common.metrics.Metrics;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TestInputTopic;
+import org.apache.kafka.streams.Topology;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.errors.DefaultProductionExceptionHandler;
 import org.apache.kafka.streams.errors.StreamsException;
 import org.apache.kafka.streams.processor.Processor;
@@ -33,7 +41,11 @@
 import java.util.LinkedHashMap;
 import java.util.Map;
 
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 
 public class ProcessorNodeTest {
@@ -52,7 +64,7 @@ public void shouldThrowStreamsExceptionIfExceptionCaughtDuringClose() {
         node.close();
     }
 
-    private static class ExceptionalProcessor implements Processor {
+    private static class ExceptionalProcessor implements Processor {
         @Override
         public void init(final ProcessorContext context) {
             throw new RuntimeException();
@@ -142,4 +154,42 @@ public void testMetrics() {
                 groupName, threadId, context.taskId().toString(), node.name())));
     }
 
+    @Test
+    public void testTopologyLevelClassCastException() {
+        final Properties props = new Properties();
+        props.put(StreamsConfig.APPLICATION_ID_CONFIG, "test");
+        props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy:1234");
+        // Serdes configuration is missing (default will be used which don't match the DSL below), which will trigger the new exception
+        final StreamsBuilder builder = new StreamsBuilder();
+
+        builder.stream("streams-plaintext-input")
+            .flatMapValues(value -> {
+                return Arrays.asList("");
+            });
+        final Topology topology = builder.build();
+
+        final TopologyTestDriver testDriver = new TopologyTestDriver(topology, props);
+        final TestInputTopic topic = testDriver.createInputTopic("streams-plaintext-input", new StringSerializer(), new StringSerializer());
+
+        final StreamsException se = assertThrows(StreamsException.class, () -> topic.pipeInput("a-key", "a value"));
+        final String msg = se.getMessage();
+        assertTrue("Error about class cast with serdes", msg.contains("ClassCastException"));
+        assertTrue("Error about class cast with serdes", msg.contains("Serdes"));
+    }
+
+    private static class ClassCastProcessor extends ExceptionalProcessor {
+        @Override
+        public void process(final Object key, final Object value) {
+            throw new ClassCastException("Incompatible types simulation exception.");
+        }
+    }
+
+    @Test
+    public void testTopologyLevelClassCastExceptionDirect() {
+        final ProcessorNode node = new ProcessorNode("name", new ClassCastProcessor(), Collections.emptySet());
+        final StreamsException se = assertThrows(StreamsException.class, () -> node.process("aKey", "aValue"));
+        assertThat(se.getCause(), instanceOf(ClassCastException.class));
+        assertThat(se.getMessage(), containsString("default Serdes"));
+        assertThat(se.getMessage(), containsString("input types"));
+    }
 }

From 5894875df7e68cd83756a18f2b9a8949280a7f7f Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Thu, 17 Oct 2019 02:26:36 -0500
Subject: [PATCH 0741/1071] MINOR: log reason for fatal error in locking state
 dir (#7534)

Reviewers: Bill Bejeck , Matthias J. Sax 
---
 .../kafka/streams/processor/internals/AbstractTask.java      | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java
index a118898995643..d8494fa8ce17e 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java
@@ -190,8 +190,9 @@ void registerStateStores() {
             }
         } catch (final IOException e) {
             throw new StreamsException(
-                String.format("%sFatal error while trying to lock the state directory for task %s",
-                logPrefix, id));
+                String.format("%sFatal error while trying to lock the state directory for task %s", logPrefix, id),
+                e
+            );
         }
         log.trace("Initializing state stores");
 

From 5193ecb8d3f41868c2933e1d9b87e63c43c08a5a Mon Sep 17 00:00:00 2001
From: Kevin Lu 
Date: Thu, 17 Oct 2019 14:46:42 +0530
Subject: [PATCH 0742/1071] KAFKA-8874: Add consumer metrics to observe user
 poll behavior (KIP-517)

https://cwiki.apache.org/confluence/display/KAFKA/KIP-517%3A+Add+consumer+metrics+to+observe+user+poll+behavior

Author: Kevin Lu 

Reviewers: Sriharsha Chintalapani , Jason Gustafson 

Closes #7395 from KevinLiLu/KIP517-KAFKA8874

(cherry picked from commit ed078bd702e30b300050aefd7a65561fbd75049f)
Signed-off-by: Manikumar Reddy 
---
 .../kafka/clients/consumer/KafkaConsumer.java |  8 ++
 .../internals/KafkaConsumerMetrics.java       | 81 ++++++++++++++++
 .../clients/consumer/KafkaConsumerTest.java   | 97 ++++++++++++++++++-
 docs/ops.html                                 | 30 ++++++
 4 files changed, 215 insertions(+), 1 deletion(-)
 create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetrics.java

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index cacf8d56a7f78..f12beaf8d5391 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -31,6 +31,7 @@
 import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient;
 import org.apache.kafka.clients.consumer.internals.Fetcher;
 import org.apache.kafka.clients.consumer.internals.FetcherMetricsRegistry;
+import org.apache.kafka.clients.consumer.internals.KafkaConsumerMetrics;
 import org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener;
 import org.apache.kafka.clients.consumer.internals.SubscriptionState;
 import org.apache.kafka.common.Cluster;
@@ -565,6 +566,7 @@ public class KafkaConsumer implements Consumer {
 
     // Visible for testing
     final Metrics metrics;
+    final KafkaConsumerMetrics kafkaConsumerMetrics;
 
     private final Logger log;
     private final String clientId;
@@ -806,6 +808,8 @@ else if (enableAutoCommit)
                     isolationLevel,
                     apiVersions);
 
+            this.kafkaConsumerMetrics = new KafkaConsumerMetrics(metrics, metricGrpPrefix);
+
             config.logUnused();
             AppInfoParser.registerAppInfo(JMX_PREFIX, clientId, metrics, time.milliseconds());
             log.debug("Kafka consumer initialized");
@@ -852,6 +856,7 @@ else if (enableAutoCommit)
         this.defaultApiTimeoutMs = defaultApiTimeoutMs;
         this.assignors = assignors;
         this.groupId = groupId;
+        this.kafkaConsumerMetrics = new KafkaConsumerMetrics(metrics, "consumer");
     }
 
     private static String buildClientId(String configuredClientId, GroupRebalanceConfig rebalanceConfig) {
@@ -1212,6 +1217,8 @@ public ConsumerRecords poll(final Duration timeout) {
     private ConsumerRecords poll(final Timer timer, final boolean includeMetadataInTimeout) {
         acquireAndEnsureOpen();
         try {
+            this.kafkaConsumerMetrics.recordPollStart(timer.currentTimeMs());
+
             if (this.subscriptions.hasNoSubscriptionOrUserAssignment()) {
                 throw new IllegalStateException("Consumer is not subscribed to any topics or assigned any partitions");
             }
@@ -1249,6 +1256,7 @@ private ConsumerRecords poll(final Timer timer, final boolean includeMetad
             return ConsumerRecords.empty();
         } finally {
             release();
+            this.kafkaConsumerMetrics.recordPollEnd(timer.currentTimeMs());
         }
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetrics.java
new file mode 100644
index 0000000000000..ae61ff1b76cba
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetrics.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.consumer.internals;
+
+import org.apache.kafka.common.metrics.Measurable;
+import org.apache.kafka.common.metrics.Metrics;
+import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.metrics.stats.Avg;
+import org.apache.kafka.common.metrics.stats.Max;
+
+import java.util.concurrent.TimeUnit;
+
+public class KafkaConsumerMetrics {
+    private final Metrics metrics;
+
+    private Sensor timeBetweenPollSensor;
+    private Sensor pollIdleSensor;
+    private long lastPollMs;
+    private long pollStartMs;
+    private long timeSinceLastPollMs;
+
+    public KafkaConsumerMetrics(Metrics metrics, String metricGrpPrefix) {
+        this.metrics = metrics;
+
+        String metricGroupName = metricGrpPrefix + "-metrics";
+        Measurable lastPoll = (mConfig, now) -> {
+            if (lastPollMs == 0L)
+                // if no poll is ever triggered, just return -1.
+                return -1d;
+            else
+                return TimeUnit.SECONDS.convert(now - lastPollMs, TimeUnit.MILLISECONDS);
+        };
+        metrics.addMetric(metrics.metricName("last-poll-seconds-ago",
+                metricGroupName,
+                "The number of seconds since the last poll() invocation."),
+                lastPoll);
+
+        this.timeBetweenPollSensor = metrics.sensor("time-between-poll");
+        this.timeBetweenPollSensor.add(metrics.metricName("time-between-poll-avg",
+                metricGroupName,
+                "The average delay between invocations of poll()."),
+                new Avg());
+        this.timeBetweenPollSensor.add(metrics.metricName("time-between-poll-max",
+                metricGroupName,
+                "The max delay between invocations of poll()."),
+                new Max());
+
+        this.pollIdleSensor = metrics.sensor("poll-idle-ratio-avg");
+        this.pollIdleSensor.add(metrics.metricName("poll-idle-ratio-avg",
+                metricGroupName,
+                "The average fraction of time the consumer's poll() is idle as opposed to waiting for the user code to process records."),
+                new Avg());
+    }
+
+    public void recordPollStart(long pollStartMs) {
+        this.pollStartMs = pollStartMs;
+        this.timeSinceLastPollMs = lastPollMs != 0L ? pollStartMs - lastPollMs : 0;
+        this.timeBetweenPollSensor.record(timeSinceLastPollMs);
+        this.lastPollMs = pollStartMs;
+    }
+
+    public void recordPollEnd(long pollEndMs) {
+        long pollTimeMs = pollEndMs - pollStartMs;
+        double pollIdleRatio = pollTimeMs * 1.0 / (pollTimeMs + timeSinceLastPollMs);
+        this.pollIdleSensor.record(pollIdleRatio);
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
index a9c4d259aa4ea..68fac2b7e6764 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
@@ -31,6 +31,7 @@
 import org.apache.kafka.clients.consumer.internals.SubscriptionState;
 import org.apache.kafka.common.Cluster;
 import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.Node;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.errors.AuthenticationException;
@@ -1947,7 +1948,7 @@ private KafkaConsumer newConsumer(Time time,
         List assignors = singletonList(assignor);
         ConsumerInterceptors interceptors = new ConsumerInterceptors<>(Collections.emptyList());
 
-        Metrics metrics = new Metrics();
+        Metrics metrics = new Metrics(time);
         ConsumerMetrics metricsRegistry = new ConsumerMetrics(metricGroupPrefix);
 
         LogContext loggerFactory = new LogContext();
@@ -2062,4 +2063,98 @@ public void testSubscriptionOnInvalidTopic() {
 
         consumer.poll(Duration.ZERO);
     }
+
+    @Test
+    public void testPollTimeMetrics() {
+        Time time = new MockTime();
+        SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.EARLIEST);
+        ConsumerMetadata metadata = createMetadata(subscription);
+        MockClient client = new MockClient(time, metadata);
+        initMetadata(client, Collections.singletonMap(topic, 1));
+
+        ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
+
+        KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
+        consumer.subscribe(singletonList(topic));
+        // MetricName objects to check
+        Metrics metrics = consumer.metrics;
+        MetricName lastPollSecondsAgoName = metrics.metricName("last-poll-seconds-ago", "consumer-metrics");
+        MetricName timeBetweenPollAvgName = metrics.metricName("time-between-poll-avg", "consumer-metrics");
+        MetricName timeBetweenPollMaxName = metrics.metricName("time-between-poll-max", "consumer-metrics");
+        // Test default values
+        assertEquals(-1.0d, consumer.metrics().get(lastPollSecondsAgoName).metricValue());
+        assertEquals(Double.NaN, consumer.metrics().get(timeBetweenPollAvgName).metricValue());
+        assertEquals(Double.NaN, consumer.metrics().get(timeBetweenPollMaxName).metricValue());
+        // Call first poll
+        consumer.poll(Duration.ZERO);
+        assertEquals(0.0d, consumer.metrics().get(lastPollSecondsAgoName).metricValue());
+        assertEquals(0.0d, consumer.metrics().get(timeBetweenPollAvgName).metricValue());
+        assertEquals(0.0d, consumer.metrics().get(timeBetweenPollMaxName).metricValue());
+        // Advance time by 5,000 (total time = 5,000)
+        time.sleep(5 * 1000L);
+        assertEquals(5.0d, consumer.metrics().get(lastPollSecondsAgoName).metricValue());
+        // Call second poll
+        consumer.poll(Duration.ZERO);
+        assertEquals(2.5 * 1000d, consumer.metrics().get(timeBetweenPollAvgName).metricValue());
+        assertEquals(5 * 1000d, consumer.metrics().get(timeBetweenPollMaxName).metricValue());
+        // Advance time by 10,000 (total time = 15,000)
+        time.sleep(10 * 1000L);
+        assertEquals(10.0d, consumer.metrics().get(lastPollSecondsAgoName).metricValue());
+        // Call third poll
+        consumer.poll(Duration.ZERO);
+        assertEquals(5 * 1000d, consumer.metrics().get(timeBetweenPollAvgName).metricValue());
+        assertEquals(10 * 1000d, consumer.metrics().get(timeBetweenPollMaxName).metricValue());
+        // Advance time by 5,000 (total time = 20,000)
+        time.sleep(5 * 1000L);
+        assertEquals(5.0d, consumer.metrics().get(lastPollSecondsAgoName).metricValue());
+        // Call fourth poll
+        consumer.poll(Duration.ZERO);
+        assertEquals(5 * 1000d, consumer.metrics().get(timeBetweenPollAvgName).metricValue());
+        assertEquals(10 * 1000d, consumer.metrics().get(timeBetweenPollMaxName).metricValue());
+    }
+
+    @Test
+    public void testPollIdleRatio() {
+        Time time = new MockTime();
+        SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.EARLIEST);
+        ConsumerMetadata metadata = createMetadata(subscription);
+        MockClient client = new MockClient(time, metadata);
+        initMetadata(client, Collections.singletonMap(topic, 1));
+
+        ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
+
+        KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
+        // MetricName object to check
+        Metrics metrics = consumer.metrics;
+        MetricName pollIdleRatio = metrics.metricName("poll-idle-ratio-avg", "consumer-metrics");
+        // Test default value
+        assertEquals(Double.NaN, consumer.metrics().get(pollIdleRatio).metricValue());
+
+        // 1st poll
+        // Spend 50ms in poll so value = 1.0
+        consumer.kafkaConsumerMetrics.recordPollStart(time.milliseconds());
+        time.sleep(50);
+        consumer.kafkaConsumerMetrics.recordPollEnd(time.milliseconds());
+
+        assertEquals(1.0d, consumer.metrics().get(pollIdleRatio).metricValue());
+
+        // 2nd poll
+        // Spend 50m outside poll and 0ms in poll so value = 0.0
+        time.sleep(50);
+        consumer.kafkaConsumerMetrics.recordPollStart(time.milliseconds());
+        consumer.kafkaConsumerMetrics.recordPollEnd(time.milliseconds());
+
+        // Avg of first two data points
+        assertEquals((1.0d + 0.0d) / 2, consumer.metrics().get(pollIdleRatio).metricValue());
+
+        // 3rd poll
+        // Spend 25ms outside poll and 25ms in poll so value = 0.5
+        time.sleep(25);
+        consumer.kafkaConsumerMetrics.recordPollStart(time.milliseconds());
+        time.sleep(25);
+        consumer.kafkaConsumerMetrics.recordPollEnd(time.milliseconds());
+
+        // Avg of three data points
+        assertEquals((1.0d + 0.0d + 0.5d) / 3, consumer.metrics().get(pollIdleRatio).metricValue());
+    }
 }
diff --git a/docs/ops.html b/docs/ops.html
index 18d9bb5ba0629..08967585efab2 100644
--- a/docs/ops.html
+++ b/docs/ops.html
@@ -1371,6 +1371,36 @@ 
                consumer monitoring<
 
   The following metrics are available on consumer instances.
 
+  
                
+    
+      
+        
+        
+        
+      
+      
+        
+        
+        
+      
+      
+        
+        
+        
+      
+      
+        
+        
+        
+      
+      
+        
+        
+        
+      
+    
+  
                Metric/Attribute nameDescriptionMbean name
                time-between-poll-avgThe average delay between invocations of poll().kafka.consumer:type=consumer-metrics,client-id=([-.\w]+)
                time-between-poll-maxThe max delay between invocations of poll().kafka.consumer:type=consumer-metrics,client-id=([-.\w]+)
                last-poll-seconds-agoThe number of seconds since the last poll() invocation.kafka.consumer:type=consumer-metrics,client-id=([-.\w]+)
                poll-idle-ratio-avgThe average fraction of time the consumer's poll() is idle as opposed to waiting for the user code to process records.kafka.consumer:type=consumer-metrics,client-id=([-.\w]+)
                
+
   
                Consumer Group Metrics
                
   

From d904e8032d006b95f512f65442c4e3174b2e3a61 Mon Sep 17 00:00:00 2001
From: Will James 
Date: Thu, 17 Oct 2019 05:56:41 -0500
Subject: [PATCH 0743/1071] KAKFA-8950: Fix KafkaConsumer Fetcher breaking on
 concurrent disconnect (#7511)

The KafkaConsumer Fetcher can sometimes get into an invalid state where it believes that there are ongoing fetch requests, but in fact there are none. This may be caused by the heartbeat thread concurrently handling a disconnection event just after the fetcher thread submits a request which would cause the Fetcher to enter an invalid state where it believes it has ongoing requests to the disconnected node but in fact it does not. This is due to a thread safety issue in the Fetcher where it was possible for the ordering of the modifications to the nodesWithPendingFetchRequests to be incorrect - the Fetcher was adding it after the listener had already been invoked, which would mean that pending node never gets removed again.

This PR addresses that thread safety issue by ensuring that the pending node is added to the nodesWithPendingFetchRequests before the listener is added to the future, ensuring the finally block is called after the node is added.

Reviewers: Tom Lee, Jason Gustafson , Rajini Sivaram 
---
 .../clients/consumer/internals/Fetcher.java   | 136 +++++++++---------
 .../org/apache/kafka/clients/MockClient.java  |   8 ++
 .../consumer/internals/FetcherTest.java       |  28 ++++
 3 files changed, 106 insertions(+), 66 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
index 6dd76048c72b3..384da315e70f4 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
@@ -258,82 +258,86 @@ public synchronized int sendFetches() {
             if (log.isDebugEnabled()) {
                 log.debug("Sending {} {} to broker {}", isolationLevel, data.toString(), fetchTarget);
             }
-            client.send(fetchTarget, request)
-                    .addListener(new RequestFutureListener() {
-                        @Override
-                        public void onSuccess(ClientResponse resp) {
-                            synchronized (Fetcher.this) {
-                                try {
-                                    @SuppressWarnings("unchecked")
-                                    FetchResponse response = (FetchResponse) resp.responseBody();
-                                    FetchSessionHandler handler = sessionHandler(fetchTarget.id());
-                                    if (handler == null) {
-                                        log.error("Unable to find FetchSessionHandler for node {}. Ignoring fetch response.",
-                                                fetchTarget.id());
-                                        return;
-                                    }
-                                    if (!handler.handleResponse(response)) {
-                                        return;
-                                    }
+            RequestFuture future = client.send(fetchTarget, request);
+            // We add the node to the set of nodes with pending fetch requests before adding the
+            // listener because the future may have been fulfilled on another thread (e.g. during a
+            // disconnection being handled by the heartbeat thread) which will mean the listener
+            // will be invoked synchronously.
+            this.nodesWithPendingFetchRequests.add(entry.getKey().id());
+            future.addListener(new RequestFutureListener() {
+                @Override
+                public void onSuccess(ClientResponse resp) {
+                    synchronized (Fetcher.this) {
+                        try {
+                            @SuppressWarnings("unchecked")
+                            FetchResponse response = (FetchResponse) resp.responseBody();
+                            FetchSessionHandler handler = sessionHandler(fetchTarget.id());
+                            if (handler == null) {
+                                log.error("Unable to find FetchSessionHandler for node {}. Ignoring fetch response.",
+                                        fetchTarget.id());
+                                return;
+                            }
+                            if (!handler.handleResponse(response)) {
+                                return;
+                            }
 
-                                    Set partitions = new HashSet<>(response.responseData().keySet());
-                                    FetchResponseMetricAggregator metricAggregator = new FetchResponseMetricAggregator(sensors, partitions);
-
-                                    for (Map.Entry> entry : response.responseData().entrySet()) {
-                                        TopicPartition partition = entry.getKey();
-                                        FetchRequest.PartitionData requestData = data.sessionPartitions().get(partition);
-                                        if (requestData == null) {
-                                            String message;
-                                            if (data.metadata().isFull()) {
-                                                message = MessageFormatter.arrayFormat(
-                                                        "Response for missing full request partition: partition={}; metadata={}",
-                                                        new Object[]{partition, data.metadata()}).getMessage();
-                                            } else {
-                                                message = MessageFormatter.arrayFormat(
-                                                        "Response for missing session request partition: partition={}; metadata={}; toSend={}; toForget={}",
-                                                        new Object[]{partition, data.metadata(), data.toSend(), data.toForget()}).getMessage();
-                                            }
-
-                                            // Received fetch response for missing session partition
-                                            throw new IllegalStateException(message);
-                                        } else {
-                                            long fetchOffset = requestData.fetchOffset;
-                                            FetchResponse.PartitionData partitionData = entry.getValue();
-
-                                            log.debug("Fetch {} at offset {} for partition {} returned fetch data {}",
-                                                    isolationLevel, fetchOffset, partition, partitionData);
-
-                                            Iterator batches = partitionData.records.batches().iterator();
-                                            short responseVersion = resp.requestHeader().apiVersion();
-
-                                            completedFetches.add(new CompletedFetch(partition, partitionData,
-                                                    metricAggregator, batches, fetchOffset, responseVersion));
-                                        }
+                            Set partitions = new HashSet<>(response.responseData().keySet());
+                            FetchResponseMetricAggregator metricAggregator = new FetchResponseMetricAggregator(sensors, partitions);
+
+                            for (Map.Entry> entry : response.responseData().entrySet()) {
+                                TopicPartition partition = entry.getKey();
+                                FetchRequest.PartitionData requestData = data.sessionPartitions().get(partition);
+                                if (requestData == null) {
+                                    String message;
+                                    if (data.metadata().isFull()) {
+                                        message = MessageFormatter.arrayFormat(
+                                                "Response for missing full request partition: partition={}; metadata={}",
+                                                new Object[]{partition, data.metadata()}).getMessage();
+                                    } else {
+                                        message = MessageFormatter.arrayFormat(
+                                                "Response for missing session request partition: partition={}; metadata={}; toSend={}; toForget={}",
+                                                new Object[]{partition, data.metadata(), data.toSend(), data.toForget()}).getMessage();
                                     }
 
-                                    sensors.fetchLatency.record(resp.requestLatencyMs());
-                                } finally {
-                                    nodesWithPendingFetchRequests.remove(fetchTarget.id());
+                                    // Received fetch response for missing session partition
+                                    throw new IllegalStateException(message);
+                                } else {
+                                    long fetchOffset = requestData.fetchOffset;
+                                    FetchResponse.PartitionData partitionData = entry.getValue();
+
+                                    log.debug("Fetch {} at offset {} for partition {} returned fetch data {}",
+                                            isolationLevel, fetchOffset, partition, partitionData);
+
+                                    Iterator batches = partitionData.records.batches().iterator();
+                                    short responseVersion = resp.requestHeader().apiVersion();
+
+                                    completedFetches.add(new CompletedFetch(partition, partitionData,
+                                            metricAggregator, batches, fetchOffset, responseVersion));
                                 }
                             }
+
+                            sensors.fetchLatency.record(resp.requestLatencyMs());
+                        } finally {
+                            nodesWithPendingFetchRequests.remove(fetchTarget.id());
                         }
+                    }
+                }
 
-                        @Override
-                        public void onFailure(RuntimeException e) {
-                            synchronized (Fetcher.this) {
-                                try {
-                                    FetchSessionHandler handler = sessionHandler(fetchTarget.id());
-                                    if (handler != null) {
-                                        handler.handleError(e);
-                                    }
-                                } finally {
-                                    nodesWithPendingFetchRequests.remove(fetchTarget.id());
-                                }
+                @Override
+                public void onFailure(RuntimeException e) {
+                    synchronized (Fetcher.this) {
+                        try {
+                            FetchSessionHandler handler = sessionHandler(fetchTarget.id());
+                            if (handler != null) {
+                                handler.handleError(e);
                             }
+                        } finally {
+                            nodesWithPendingFetchRequests.remove(fetchTarget.id());
                         }
-                    });
+                    }
+                }
+            });
 
-            this.nodesWithPendingFetchRequests.add(entry.getKey().id());
         }
         return fetchRequestMap.size();
     }
diff --git a/clients/src/test/java/org/apache/kafka/clients/MockClient.java b/clients/src/test/java/org/apache/kafka/clients/MockClient.java
index 650f670e545c5..f9fb96ce0c0a8 100644
--- a/clients/src/test/java/org/apache/kafka/clients/MockClient.java
+++ b/clients/src/test/java/org/apache/kafka/clients/MockClient.java
@@ -73,6 +73,7 @@ public FutureResponse(Node node,
     }
 
     private int correlation;
+    private Runnable wakeupHook;
     private final Time time;
     private final MockMetadataUpdater metadataUpdater;
     private final Map connections = new HashMap<>();
@@ -254,6 +255,9 @@ public synchronized void wakeup() {
             numBlockingWakeups--;
             notify();
         }
+        if (wakeupHook != null) {
+            wakeupHook.run();
+        }
     }
 
     private synchronized void maybeAwaitWakeup() {
@@ -545,6 +549,10 @@ public Node leastLoadedNode(long now) {
         return null;
     }
 
+    public void setWakeupHook(Runnable wakeupHook) {
+        this.wakeupHook = wakeupHook;
+    }
+
     /**
      * The RequestMatcher provides a way to match a particular request to a response prepared
      * through {@link #prepareResponse(RequestMatcher, AbstractResponse)}. Basically this allows testers
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
index 9e281a7b6f822..d3fcbc334c77b 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
@@ -114,6 +114,7 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -3772,6 +3773,33 @@ public void testPreferredReadReplicaOffsetError() {
         assertEquals(selected.id(), -1);
     }
 
+    @Test
+    public void testFetchCompletedBeforeHandlerAdded() {
+        buildFetcher();
+        assignFromUser(singleton(tp0));
+        subscriptions.seek(tp0, 0);
+        fetcher.sendFetches();
+        client.prepareResponse(fullFetchResponse(tp0, buildRecords(1L, 1, 1), Errors.NONE, 100L, 0));
+        consumerClient.poll(time.timer(0));
+        fetchedRecords();
+        Node node = fetcher.selectReadReplica(tp0, subscriptions.position(tp0).currentLeader.leader, time.milliseconds());
+
+        AtomicBoolean wokenUp = new AtomicBoolean(false);
+        client.setWakeupHook(() -> {
+            if (!wokenUp.getAndSet(true)) {
+                consumerClient.disconnectAsync(node);
+                consumerClient.poll(time.timer(0));
+            }
+        });
+
+        assertEquals(1, fetcher.sendFetches());
+
+        consumerClient.disconnectAsync(node);
+        consumerClient.poll(time.timer(0));
+
+        assertEquals(1, fetcher.sendFetches());
+    }
+
     private MockClient.RequestMatcher listOffsetRequestMatcher(final long timestamp) {
         // matches any list offset request with the provided timestamp
         return new MockClient.RequestMatcher() {

From f666cf37a5de88cac0842cd3e15613bb4940db88 Mon Sep 17 00:00:00 2001
From: Colin Patrick McCabe 
Date: Thu, 17 Oct 2019 09:21:34 -0700
Subject: [PATCH 0744/1071] MINOR: AbstractRequestResponse should be an
 interface (#7513)

AbstractRequestResponse should be an interface, since it has no concrete elements or implementation.  Move AbstractRequestResponse#serialize to RequestUtils#serialize and make it package-private, since it doesn't need to be public.

Reviewers: Ismael Juma 
(cherry picked from commit 3cb8ccf63a3f88b4de5a2133b39fb4bf175f3532)
---
 .../kafka/common/requests/AbstractRequest.java   |  4 ++--
 .../common/requests/AbstractRequestResponse.java | 16 +---------------
 .../kafka/common/requests/AbstractResponse.java  | 10 +++++-----
 .../kafka/common/requests/RequestHeader.java     |  2 +-
 .../kafka/common/requests/RequestUtils.java      | 11 ++++++++++-
 .../kafka/common/requests/ResponseHeader.java    |  2 +-
 .../clients/consumer/internals/FetcherTest.java  |  6 ++++--
 .../clients/producer/internals/SenderTest.java   |  5 +++--
 .../unit/kafka/server/BaseRequestTest.scala      |  4 ++--
 9 files changed, 29 insertions(+), 31 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
index b737b49f4a5c1..97bc72852e39f 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
@@ -26,7 +26,7 @@
 import java.nio.ByteBuffer;
 import java.util.Map;
 
-public abstract class AbstractRequest extends AbstractRequestResponse {
+public abstract class AbstractRequest implements AbstractRequestResponse {
 
     public static abstract class Builder {
         private final ApiKeys apiKey;
@@ -100,7 +100,7 @@ public Send toSend(String destination, RequestHeader header) {
      * Use with care, typically {@link #toSend(String, RequestHeader)} should be used instead.
      */
     public ByteBuffer serialize(RequestHeader header) {
-        return serialize(header.toStruct(), toStruct());
+        return RequestUtils.serialize(header.toStruct(), toStruct());
     }
 
     protected abstract Struct toStruct();
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequestResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequestResponse.java
index 0ba373d6fea7a..b02659d4f354c 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequestResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequestResponse.java
@@ -16,19 +16,5 @@
  */
 package org.apache.kafka.common.requests;
 
-import org.apache.kafka.common.protocol.types.Struct;
-
-import java.nio.ByteBuffer;
-
-public abstract class AbstractRequestResponse {
-    /**
-     * Visible for testing.
-     */
-    public static ByteBuffer serialize(Struct headerStruct, Struct bodyStruct) {
-        ByteBuffer buffer = ByteBuffer.allocate(headerStruct.sizeOf() + bodyStruct.sizeOf());
-        headerStruct.writeTo(buffer);
-        bodyStruct.writeTo(buffer);
-        buffer.rewind();
-        return buffer;
-    }
+public interface AbstractRequestResponse {
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
index 8a6edf14cbac0..64701529403de 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
@@ -28,18 +28,18 @@
 import java.util.HashMap;
 import java.util.Map;
 
-public abstract class AbstractResponse extends AbstractRequestResponse {
+public abstract class AbstractResponse implements AbstractRequestResponse {
     public static final int DEFAULT_THROTTLE_TIME = 0;
 
     protected Send toSend(String destination, ResponseHeader header, short apiVersion) {
-        return new NetworkSend(destination, serialize(header.toStruct(), toStruct(apiVersion)));
+        return new NetworkSend(destination, RequestUtils.serialize(header.toStruct(), toStruct(apiVersion)));
     }
 
     /**
      * Visible for testing, typically {@link #toSend(String, ResponseHeader, short)} should be used instead.
      */
-    public ByteBuffer serialize(ApiKeys apiKey, int correlationId) {
-        return serialize(apiKey, apiKey.latestVersion(), correlationId);
+    public ByteBuffer serialize(short version, ResponseHeader responseHeader) {
+        return RequestUtils.serialize(responseHeader.toStruct(), toStruct(version));
     }
 
     /**
@@ -48,7 +48,7 @@ public ByteBuffer serialize(ApiKeys apiKey, int correlationId) {
     public ByteBuffer serialize(ApiKeys apiKey, short version, int correlationId) {
         ResponseHeader header =
             new ResponseHeader(correlationId, apiKey.responseHeaderVersion(version));
-        return serialize(header.toStruct(), toStruct(version));
+        return RequestUtils.serialize(header.toStruct(), toStruct(version));
     }
 
     public abstract Map errorCounts();
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java
index 8da1eb5fde9ed..3d80c4e7563d5 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java
@@ -28,7 +28,7 @@
 /**
  * The header for a request in the Kafka protocol
  */
-public class RequestHeader extends AbstractRequestResponse {
+public class RequestHeader implements AbstractRequestResponse {
     private final RequestHeaderData data;
     private final short headerVersion;
 
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java
index b4a2420c46a8a..c3dfaa13b996f 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java
@@ -28,6 +28,7 @@
 import org.apache.kafka.common.protocol.types.Struct;
 import org.apache.kafka.common.resource.ResourceType;
 
+import java.nio.ByteBuffer;
 import java.util.Optional;
 
 import static org.apache.kafka.common.protocol.CommonFields.HOST;
@@ -42,7 +43,7 @@
 import static org.apache.kafka.common.protocol.CommonFields.RESOURCE_PATTERN_TYPE_FILTER;
 import static org.apache.kafka.common.protocol.CommonFields.RESOURCE_TYPE;
 
-final class RequestUtils {
+public final class RequestUtils {
 
     private RequestUtils() {}
 
@@ -122,4 +123,12 @@ static Optional getLeaderEpoch(int leaderEpoch) {
             Optional.empty() : Optional.of(leaderEpoch);
         return leaderEpochOpt;
     }
+
+    public static ByteBuffer serialize(Struct headerStruct, Struct bodyStruct) {
+        ByteBuffer buffer = ByteBuffer.allocate(headerStruct.sizeOf() + bodyStruct.sizeOf());
+        headerStruct.writeTo(buffer);
+        bodyStruct.writeTo(buffer);
+        buffer.rewind();
+        return buffer;
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java b/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java
index 249b5d021081f..118e5d3506d72 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java
@@ -25,7 +25,7 @@
 /**
  * A response header in the kafka protocol.
  */
-public class ResponseHeader extends AbstractRequestResponse {
+public class ResponseHeader implements AbstractRequestResponse {
     private final ResponseHeaderData data;
     private final short headerVersion;
 
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
index d3fcbc334c77b..fd1be374b7eea 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
@@ -1999,7 +1999,7 @@ public void testQuotaMetrics() {
 
         ByteBuffer buffer = ApiVersionsResponse.
             createApiVersionsResponse(400, RecordBatch.CURRENT_MAGIC_VALUE).
-                serialize(ApiKeys.API_VERSIONS, 0);
+                serialize(ApiKeys.API_VERSIONS, ApiKeys.API_VERSIONS.latestVersion(), 0);
         selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer)));
         while (!client.ready(node, time.milliseconds())) {
             client.poll(1, time.milliseconds());
@@ -2016,7 +2016,9 @@ public void testQuotaMetrics() {
             client.send(request, time.milliseconds());
             client.poll(1, time.milliseconds());
             FetchResponse response = fullFetchResponse(tp0, nextRecords, Errors.NONE, i, throttleTimeMs);
-            buffer = response.serialize(ApiKeys.FETCH, request.correlationId());
+            buffer = response.serialize(ApiKeys.FETCH,
+                    ApiKeys.FETCH.latestVersion(),
+                    request.correlationId());
             selector.completeReceive(new NetworkReceive(node.idString(), buffer));
             client.poll(1, time.milliseconds());
             // If a throttled response is received, advance the time to ensure progress.
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java
index 71180bdd7f7a7..1b35e0b43116e 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java
@@ -275,7 +275,7 @@ public void testQuotaMetrics() throws Exception {
                 time, true, new ApiVersions(), throttleTimeSensor, logContext);
 
         ByteBuffer buffer = ApiVersionsResponse.createApiVersionsResponse(400, RecordBatch.CURRENT_MAGIC_VALUE).
-            serialize(ApiKeys.API_VERSIONS, 0);
+            serialize(ApiKeys.API_VERSIONS, ApiKeys.API_VERSIONS.latestVersion(), 0);
         selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer)));
         while (!client.ready(node, time.milliseconds())) {
             client.poll(1, time.milliseconds());
@@ -292,7 +292,8 @@ public void testQuotaMetrics() throws Exception {
             client.send(request, time.milliseconds());
             client.poll(1, time.milliseconds());
             ProduceResponse response = produceResponse(tp0, i, Errors.NONE, throttleTimeMs);
-            buffer = response.serialize(ApiKeys.PRODUCE, request.correlationId());
+            buffer = response.
+                serialize(ApiKeys.PRODUCE, ApiKeys.PRODUCE.latestVersion(), request.correlationId());
             selector.completeReceive(new NetworkReceive(node.idString(), buffer));
             client.poll(1, time.milliseconds());
             // If a throttled response is received, advance the time to ensure progress.
diff --git a/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala b/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala
index 5d00d540b149a..2cc355395d4a6 100644
--- a/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala
@@ -29,7 +29,7 @@ import kafka.network.SocketServer
 import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.protocol.types.Struct
 import org.apache.kafka.common.protocol.ApiKeys
-import org.apache.kafka.common.requests.{AbstractRequest, AbstractRequestResponse, RequestHeader, ResponseHeader}
+import org.apache.kafka.common.requests.{AbstractRequest, RequestHeader, RequestUtils, ResponseHeader}
 import org.apache.kafka.common.security.auth.SecurityProtocol
 
 abstract class BaseRequestTest extends IntegrationTestHarness {
@@ -170,7 +170,7 @@ abstract class BaseRequestTest extends IntegrationTestHarness {
     */
   def sendStructAndReceive(requestStruct: Struct, apiKey: ApiKeys, socket: Socket, apiVersion: Short): ByteBuffer = {
     val header = nextRequestHeader(apiKey, apiVersion)
-    val serializedBytes = AbstractRequestResponse.serialize(header.toStruct, requestStruct).array
+    val serializedBytes = RequestUtils.serialize(header.toStruct, requestStruct).array
     val response = requestAndReceive(socket, serializedBytes)
     skipResponseHeader(response, apiKey.responseHeaderVersion(apiVersion))
   }

From 45fd4768a38e6499ea7b9680918cae6ed34bcd3d Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Thu, 17 Oct 2019 09:59:02 -0700
Subject: [PATCH 0745/1071] MINOR: Upgrade zk to 3.5.6 (#7544)

It includes an important fix for people running on k8s:

* ZOOKEEPER-3320: Leader election port stop listen when
hostname unresolvable for some time

Reviewers: Manikumar Reddy 
---
 gradle/dependencies.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle
index 993273b9ff74d..c08474fdee423 100644
--- a/gradle/dependencies.gradle
+++ b/gradle/dependencies.gradle
@@ -114,7 +114,7 @@ versions += [
   spotbugs: "3.1.12",
   spotbugsPlugin: "1.6.9",
   spotlessPlugin: "3.23.1",
-  zookeeper: "3.5.5",
+  zookeeper: "3.5.6",
   zstd: "1.4.3-1"
 ]
 

From 0fbf012abda9e9b446c84dd66f605c7f5a0b8a8d Mon Sep 17 00:00:00 2001
From: David Arthur 
Date: Thu, 17 Oct 2019 12:44:14 -0400
Subject: [PATCH 0746/1071] KAFKA-9004; Prevent older clients from fetching
 from a follower (#7531)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

With KIP-392, we allow consumers to fetch from followers. This capability is enabled when a replica selector has been provided in the configuration. When not in use, the intent is to preserve current behavior of fetching only from leader. The leader epoch is the mechanism that keeps us honest. When there is a leader change, the epoch gets bumped, consumer fetches fail due to the fenced epoch, and we find the new leader.

However, for old consumers, there is no similar protection. The leader epoch was not available to clients until recently. If there is a preferred leader election (for example), the old consumer will happily continue fetching from the demoted leader until a periodic metadata fetch causes us to discover the new leader. This does not create any problems from a correctness perspective–fetches are still bound by the high watermark–but it is unexpected and may cause unexpected performance characteristics.

This patch fixes this problem by enforcing leader-only fetching for older versions of the fetch request.

Reviewers: Jason Gustafson 
---
 .../scala/kafka/server/ReplicaManager.scala   | 12 ++-
 .../kafka/server/ReplicaManagerTest.scala     | 94 ++++++++++++++++++-
 2 files changed, 98 insertions(+), 8 deletions(-)

diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index d10c93645c36a..ff393390e27d2 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -848,18 +848,20 @@ class ReplicaManager(val config: KafkaConfig,
                     isolationLevel: IsolationLevel,
                     clientMetadata: Option[ClientMetadata]): Unit = {
     val isFromFollower = Request.isValidBrokerId(replicaId)
-
-    val fetchIsolation = if (isFromFollower || replicaId == Request.FutureLocalReplicaId)
+    val isFromConsumer = !(isFromFollower || replicaId == Request.FutureLocalReplicaId)
+    val fetchIsolation = if (!isFromConsumer)
       FetchLogEnd
     else if (isolationLevel == IsolationLevel.READ_COMMITTED)
       FetchTxnCommitted
     else
       FetchHighWatermark
 
+    // Restrict fetching to leader if request is from follower or from a client with older version (no ClientMetadata)
+    val fetchOnlyFromLeader = isFromFollower || (isFromConsumer && clientMetadata.isEmpty)
     def readFromLog(): Seq[(TopicPartition, LogReadResult)] = {
       val result = readFromLocalLog(
         replicaId = replicaId,
-        fetchOnlyFromLeader = isFromFollower,
+        fetchOnlyFromLeader = fetchOnlyFromLeader,
         fetchIsolation = fetchIsolation,
         fetchMaxBytes = fetchMaxBytes,
         hardMaxBytesLimit = hardMaxBytesLimit,
@@ -917,7 +919,7 @@ class ReplicaManager(val config: KafkaConfig,
           fetchPartitionStatus += (topicPartition -> FetchPartitionStatus(logOffsetMetadata, partitionData))
         })
       }
-      val fetchMetadata = FetchMetadata(fetchMinBytes, fetchMaxBytes, hardMaxBytesLimit, isFromFollower,
+      val fetchMetadata = FetchMetadata(fetchMinBytes, fetchMaxBytes, hardMaxBytesLimit, fetchOnlyFromLeader,
         fetchIsolation, isFromFollower, replicaId, fetchPartitionStatus)
       val delayedFetch = new DelayedFetch(timeout, fetchMetadata, this, quota, clientMetadata,
         maybeUpdateHwAndSendResponse)
@@ -971,7 +973,7 @@ class ReplicaManager(val config: KafkaConfig,
               s"${preferredReadReplica.get} for $clientMetadata")
           }
           // If a preferred read-replica is set, skip the read
-          val offsetSnapshot = partition.fetchOffsetSnapshot(fetchInfo.currentLeaderEpoch, false)
+          val offsetSnapshot = partition.fetchOffsetSnapshot(fetchInfo.currentLeaderEpoch, fetchOnlyFromLeader = false)
           LogReadResult(info = FetchDataInfo(LogOffsetMetadata.UnknownOffsetMetadata, MemoryRecords.EMPTY),
             highWatermark = offsetSnapshot.highWatermark.messageOffset,
             leaderLogStartOffset = offsetSnapshot.logStartOffset,
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index 484ee2d962a1e..ef05fa1499c5f 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -24,10 +24,8 @@ import java.util.concurrent.{CountDownLatch, TimeUnit}
 import java.util.{Optional, Properties}
 
 import kafka.api.Request
-import kafka.cluster.BrokerEndPoint
 import kafka.log.{Log, LogConfig, LogManager, ProducerStateManager}
-import kafka.utils.{MockScheduler, MockTime, TestUtils}
-import TestUtils.createBroker
+import kafka.utils.TestUtils
 import kafka.cluster.BrokerEndPoint
 import kafka.server.QuotaFactory.UnboundedQuota
 import kafka.server.checkpoints.LazyOffsetCheckpoints
@@ -921,6 +919,96 @@ class ReplicaManagerTest {
     assertFalse(replicaManager.replicaSelectorOpt.isDefined)
   }
 
+  @Test
+  def testOlderClientFetchFromLeaderOnly(): Unit = {
+    val replicaManager = setupReplicaManagerWithMockedPurgatories(new MockTimer, aliveBrokerIds = Seq(0, 1))
+
+    val tp0 = new TopicPartition(topic, 0)
+    val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
+    replicaManager.createPartition(tp0).createLogIfNotExists(0, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    val partition0Replicas = Seq[Integer](0, 1).asJava
+    val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+      Seq(
+        new LeaderAndIsrPartitionState()
+          .setTopicName(tp0.topic)
+          .setPartitionIndex(tp0.partition)
+          .setControllerEpoch(0)
+          .setLeader(1)
+          .setLeaderEpoch(0)
+          .setIsr(partition0Replicas)
+          .setZkVersion(0)
+          .setReplicas(partition0Replicas)
+          .setIsNew(true)).asJava,
+      Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
+    replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest, (_, _) => ())
+
+    def doFetch(replicaId: Int, partitionData: FetchRequest.PartitionData, clientMetadataOpt: Option[ClientMetadata]):
+        Option[FetchPartitionData] = {
+      var fetchResult: Option[FetchPartitionData] = None
+      def callback(response: Seq[(TopicPartition, FetchPartitionData)]): Unit = {
+        fetchResult = response.headOption.filter(_._1 == tp0).map(_._2)
+      }
+      replicaManager.fetchMessages(
+        timeout = 0L,
+        replicaId = replicaId,
+        fetchMinBytes = 1,
+        fetchMaxBytes = 100,
+        hardMaxBytesLimit = false,
+        fetchInfos = Seq(tp0 -> partitionData),
+        quota = UnboundedQuota,
+        isolationLevel = IsolationLevel.READ_UNCOMMITTED,
+        responseCallback = callback,
+        clientMetadata = clientMetadataOpt
+      )
+      fetchResult
+    }
+
+    // Fetch from follower, with non-empty ClientMetadata (FetchRequest v11+)
+    val clientMetadata = new DefaultClientMetadata("", "", null, KafkaPrincipal.ANONYMOUS, "")
+    var partitionData = new FetchRequest.PartitionData(0L, 0L, 100,
+      Optional.of(0))
+    var fetchResult = doFetch(Request.OrdinaryConsumerId, partitionData, Some(clientMetadata))
+    assertTrue(fetchResult.isDefined)
+    assertEquals(fetchResult.get.error, Errors.NONE)
+
+    // Fetch from follower, with empty ClientMetadata
+    fetchResult = None
+    partitionData = new FetchRequest.PartitionData(0L, 0L, 100,
+      Optional.of(0))
+    fetchResult = doFetch(Request.OrdinaryConsumerId, partitionData, None)
+    assertTrue(fetchResult.isDefined)
+    assertEquals(fetchResult.get.error, Errors.NOT_LEADER_FOR_PARTITION)
+
+    // Change to a leader, both cases are allowed
+    val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+      Seq(
+        new LeaderAndIsrPartitionState()
+          .setTopicName(tp0.topic)
+          .setPartitionIndex(tp0.partition)
+          .setControllerEpoch(0)
+          .setLeader(0)
+          .setLeaderEpoch(1)
+          .setIsr(partition0Replicas)
+          .setZkVersion(0)
+          .setReplicas(partition0Replicas)
+          .setIsNew(true)).asJava,
+      Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
+    replicaManager.becomeLeaderOrFollower(1, leaderAndIsrRequest2, (_, _) => ())
+
+    partitionData = new FetchRequest.PartitionData(0L, 0L, 100,
+      Optional.of(1))
+    fetchResult = doFetch(Request.OrdinaryConsumerId, partitionData, Some(clientMetadata))
+    assertTrue(fetchResult.isDefined)
+    assertEquals(fetchResult.get.error, Errors.NONE)
+
+    fetchResult = None
+    partitionData = new FetchRequest.PartitionData(0L, 0L, 100,
+      Optional.empty())
+    fetchResult = doFetch(Request.OrdinaryConsumerId, partitionData, None)
+    assertTrue(fetchResult.isDefined)
+    assertEquals(fetchResult.get.error, Errors.NONE)
+  }
+
   /**
    * This method assumes that the test using created ReplicaManager calls
    * ReplicaManager.becomeLeaderOrFollower() once with LeaderAndIsrRequest containing

From f357fc0339c11d9412f433df1920b509c575c31b Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Wed, 9 Oct 2019 10:34:19 -0700
Subject: [PATCH 0747/1071] MINOR: Augment log4j to add generation number in
 performAssign (#7451)

Since generation is private in AbstractCoordinator, I need to modify the generation() to let it return the object directly.

Reviewers: A. Sophie Blee-Goldman , Bill Bejeck 
---
 .../internals/AbstractCoordinator.java        | 66 ++++++++++++-------
 .../internals/ConsumerCoordinator.java        | 11 ++--
 .../internals/AbstractCoordinatorTest.java    |  2 +-
 .../internals/ConsumerCoordinatorTest.java    |  6 +-
 .../distributed/WorkerCoordinator.java        |  2 +-
 5 files changed, 52 insertions(+), 35 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index 057cfb04115b9..3a7627640d9f5 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -638,7 +638,7 @@ private RequestFuture onJoinFollower() {
                                 .setGenerationId(generation.generationId)
                                 .setAssignments(Collections.emptyList())
                 );
-        log.debug("Sending follower SyncGroup to coordinator {}: {}", this.coordinator, requestBuilder);
+        log.debug("Sending follower SyncGroup to coordinator {} at generation {}: {}", this.coordinator, this.generation, requestBuilder);
         return sendSyncGroupRequest(requestBuilder);
     }
 
@@ -665,7 +665,7 @@ private RequestFuture onJoinLeader(JoinGroupResponse joinResponse) {
                                     .setGenerationId(generation.generationId)
                                     .setAssignments(groupAssignmentList)
                     );
-            log.debug("Sending leader SyncGroup to coordinator {}: {}", this.coordinator, requestBuilder);
+            log.debug("Sending leader SyncGroup to coordinator {} at generation {}: {}", this.coordinator, this.generation, requestBuilder);
             return sendSyncGroupRequest(requestBuilder);
         } catch (RuntimeException e) {
             return RequestFuture.failure(e);
@@ -819,36 +819,29 @@ protected synchronized void markCoordinatorUnknown(boolean isDisconnected) {
     }
 
     /**
-     * Get the current generation state if the group is stable.
-     * @return the current generation or null if the group is unjoined/rebalancing
+     * Get the current generation state, regardless of whether it is currently stable.
+     * Note that the generation information can be updated while we are still in the middle
+     * of a rebalance, after the join-group response is received.
+     *
+     * @return the current generation
      */
     protected synchronized Generation generation() {
-        if (this.state != MemberState.STABLE)
-            return null;
         return generation;
     }
 
-    protected synchronized String memberId() {
-        return generation == null ? JoinGroupRequest.UNKNOWN_MEMBER_ID :
-                generation.memberId;
-    }
-
     /**
-     * Check whether given generation id is matching the record within current generation.
-     * Only using in unit tests.
-     * @param generationId generation id
-     * @return true if the two ids are matching.
+     * Get the current generation state if the group is stable, otherwise return null
+     *
+     * @return the current generation or null
      */
-    final synchronized boolean hasMatchingGenerationId(int generationId) {
-        return generation != null && generation.generationId == generationId;
+    protected synchronized Generation generationIfStable() {
+        if (this.state != MemberState.STABLE)
+            return null;
+        return generation;
     }
 
-    /**
-     * @return true if the current generation's member ID is valid, false otherwise
-     */
-    // Visible for testing
-    final synchronized boolean hasValidMemberId() {
-        return generation != null && generation.hasMemberId();
+    protected synchronized String memberId() {
+        return generation.memberId;
     }
 
     private synchronized void resetGeneration() {
@@ -1348,12 +1341,35 @@ private static class UnjoinedGroupException extends RetriableException {
 
     }
 
-    // For testing only
+    // For testing only below
     public Heartbeat heartbeat() {
         return heartbeat;
     }
 
-    public void setLastRebalanceTime(final long timestamp) {
+    final void setLastRebalanceTime(final long timestamp) {
         lastRebalanceEndMs = timestamp;
     }
+
+    /**
+     * Check whether given generation id is matching the record within current generation.
+     *
+     * @param generationId generation id
+     * @return true if the two ids are matching.
+     */
+    final boolean hasMatchingGenerationId(int generationId) {
+        return generation != Generation.NO_GENERATION && generation.generationId == generationId;
+    }
+
+    final boolean hasUnknownGeneration() {
+        return generation == Generation.NO_GENERATION;
+    }
+
+    /**
+     * @return true if the current generation's member ID is valid, false otherwise
+     */
+    final boolean hasValidMemberId() {
+        return generation != Generation.NO_GENERATION && generation.hasMemberId();
+    }
+
+
 }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 6b39acb6d7f2c..bceb9b8ba0973 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -582,7 +582,7 @@ protected Map performAssignment(String leaderId,
 
         assignmentSnapshot = metadataSnapshot;
 
-        log.debug("Finished assignment for group: {}", assignments);
+        log.info("Finished assignment for group at generation {}: {}", generation().generationId, assignments);
 
         Map groupAssignment = new HashMap<>();
         for (Map.Entry assignmentEntry : assignments.entrySet()) {
@@ -764,8 +764,9 @@ public Map fetchCommittedOffsets(final Set fetchCommittedOffsets(final Set sendOffsetCommitRequest(final Map
Date: Thu, 17 Oct 2019 11:33:08 -0600
Subject: [PATCH 0748/1071] MINOR: Add ability to wait for all instances in an
 application to be RUNNING (#7500)

Reviewers: Matthias J. Sax , A. Sophie Blee-Goldman , Guozhang Wang 
---
 .../OptimizedKTableIntegrationTest.java       | 68 +++-----------
 .../utils/CompositeStateListener.java         | 50 +++++++++++
 .../utils/IntegrationTestUtils.java           | 89 +++++++++++++++++++
 3 files changed, 149 insertions(+), 58 deletions(-)
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/integration/utils/CompositeStateListener.java

diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/OptimizedKTableIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/OptimizedKTableIntegrationTest.java
index 767a5a9e0ff02..de0ea3938873a 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/OptimizedKTableIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/OptimizedKTableIntegrationTest.java
@@ -16,26 +16,22 @@
  */
 package org.apache.kafka.streams.integration;
 
+import static org.apache.kafka.streams.integration.utils.IntegrationTestUtils.startApplicationAndWaitUntilRunning;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
-import static org.junit.Assert.fail;
 
+import java.time.Duration;
+import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Properties;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
@@ -46,7 +42,6 @@
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.common.utils.MockTime;
 import org.apache.kafka.streams.KafkaStreams;
-import org.apache.kafka.streams.KafkaStreams.State;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsConfig;
@@ -76,9 +71,7 @@ public class OptimizedKTableIntegrationTest {
     @Rule
     public final EmbeddedKafkaCluster cluster = new EmbeddedKafkaCluster(NUM_BROKERS);
 
-    private final Map kafkaStreamsStates = new HashMap<>();
-    private final Lock kafkaStreamsStatesLock = new ReentrantLock();
-    private final Condition kafkaStreamsStateUpdate = kafkaStreamsStatesLock.newCondition();
+    private final List streamsToCleanup = new ArrayList<>();
     private final MockTime mockTime = cluster.time;
 
     @Before
@@ -88,7 +81,7 @@ public void before() throws InterruptedException {
 
     @After
     public void after() {
-        for (final KafkaStreams kafkaStreams : kafkaStreamsStates.keySet()) {
+        for (final KafkaStreams kafkaStreams : streamsToCleanup) {
             kafkaStreams.close();
         }
     }
@@ -116,9 +109,8 @@ public void standbyShouldNotPerformRestoreAtStartup() throws Exception {
         final AtomicLong restoreStartOffset = new AtomicLong(-1);
         kafkaStreamsList.forEach(kafkaStreams -> {
             kafkaStreams.setGlobalStateRestoreListener(createTrackingRestoreListener(restoreStartOffset, new AtomicLong()));
-            kafkaStreams.start();
         });
-        waitForKafkaStreamssToEnterRunningState(kafkaStreamsList, 60, TimeUnit.SECONDS);
+        startApplicationAndWaitUntilRunning(kafkaStreamsList, Duration.ofSeconds(60));
 
         // Assert that all messages in the first batch were processed in a timely manner
         assertThat(semaphore.tryAcquire(numMessages, 60, TimeUnit.SECONDS), is(equalTo(true)));
@@ -150,9 +142,8 @@ public void shouldApplyUpdatesToStandbyStore() throws Exception {
         final AtomicLong restoreEndOffset = new AtomicLong(-1L);
         kafkaStreamsList.forEach(kafkaStreams -> {
             kafkaStreams.setGlobalStateRestoreListener(createTrackingRestoreListener(restoreStartOffset, restoreEndOffset));
-            kafkaStreams.start();
         });
-        waitForKafkaStreamssToEnterRunningState(kafkaStreamsList, 60, TimeUnit.SECONDS);
+        startApplicationAndWaitUntilRunning(kafkaStreamsList, Duration.ofSeconds(60));
 
         produceValueRange(key, 0, batch1NumMessages);
 
@@ -226,49 +217,10 @@ private void produceValueRange(final int key, final int start, final int endExcl
             mockTime);
     }
 
-    private void waitForKafkaStreamssToEnterRunningState(final Collection kafkaStreamss,
-                                                         final long time,
-                                                         final TimeUnit timeUnit) throws InterruptedException {
-
-        final long expectedEnd = System.currentTimeMillis() + timeUnit.toMillis(time);
-
-        kafkaStreamsStatesLock.lock();
-        try {
-            while (!kafkaStreamss.stream().allMatch(kafkaStreams -> kafkaStreamsStates.get(kafkaStreams) == State.RUNNING)) {
-                if (expectedEnd <= System.currentTimeMillis()) {
-                    fail("one or more kafkaStreamss did not enter RUNNING in a timely manner");
-                }
-                final long millisRemaining = Math.max(1, expectedEnd - System.currentTimeMillis());
-                kafkaStreamsStateUpdate.await(millisRemaining, TimeUnit.MILLISECONDS);
-            }
-        } finally {
-            kafkaStreamsStatesLock.unlock();
-        }
-    }
-
     private KafkaStreams createKafkaStreams(final StreamsBuilder builder, final Properties config) {
-        final KafkaStreams kafkaStreams = new KafkaStreams(builder.build(config), config);
-        kafkaStreamsStatesLock.lock();
-        try {
-            kafkaStreamsStates.put(kafkaStreams, kafkaStreams.state());
-        } finally {
-            kafkaStreamsStatesLock.unlock();
-        }
-
-        kafkaStreams.setStateListener((newState, oldState) -> {
-            kafkaStreamsStatesLock.lock();
-            try {
-                kafkaStreamsStates.put(kafkaStreams, newState);
-                if (newState == State.RUNNING) {
-                    if (kafkaStreamsStates.values().stream().allMatch(state -> state == State.RUNNING)) {
-                        kafkaStreamsStateUpdate.signalAll();
-                    }
-                }
-            } finally {
-                kafkaStreamsStatesLock.unlock();
-            }
-        });
-        return kafkaStreams;
+        final KafkaStreams streams = new KafkaStreams(builder.build(config), config);
+        streamsToCleanup.add(streams);
+        return streams;
     }
 
     private StateRestoreListener createTrackingRestoreListener(final AtomicLong restoreStartOffset,
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/utils/CompositeStateListener.java b/streams/src/test/java/org/apache/kafka/streams/integration/utils/CompositeStateListener.java
new file mode 100644
index 0000000000000..825894280e636
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/utils/CompositeStateListener.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.integration.utils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import org.apache.kafka.streams.KafkaStreams.State;
+import org.apache.kafka.streams.KafkaStreams.StateListener;
+
+/**
+ * A {@link StateListener} that holds zero or more listeners internally and invokes all of them
+ * when a state transition occurs (i.e. {@link #onChange(State, State)} is called). If any listener
+ * throws {@link RuntimeException} or {@link Error} this immediately stops execution of listeners
+ * and causes the thrown exception to be raised.
+ */
+public class CompositeStateListener implements StateListener {
+    private final List listeners;
+
+    public CompositeStateListener(final StateListener... listeners) {
+        this(Arrays.asList(listeners));
+    }
+
+    public CompositeStateListener(final Collection stateListeners) {
+        this.listeners = Collections.unmodifiableList(new ArrayList<>(stateListeners));
+    }
+
+    @Override
+    public void onChange(final State newState, final State oldState) {
+        for (final StateListener listener : listeners) {
+            listener.onChange(newState, oldState);
+        }
+    }
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java b/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java
index 6c67a9df13133..4921c4f39dabf 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java
@@ -16,6 +16,12 @@
  */
 package org.apache.kafka.streams.integration.utils;
 
+import java.lang.reflect.Field;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 import kafka.api.Request;
 import kafka.server.KafkaServer;
 import kafka.server.MetadataCache;
@@ -35,6 +41,8 @@
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.KafkaStreams.State;
+import org.apache.kafka.streams.KafkaStreams.StateListener;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.KeyValueTimestamp;
 import org.apache.kafka.streams.StreamsBuilder;
@@ -70,6 +78,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.fail;
 
 /**
  * Utility functions to make integration testing more convenient.
@@ -731,6 +740,86 @@ public static void waitUntilMetadataIsPropagated(final List servers
         });
     }
 
+    /**
+     * Starts the given {@link KafkaStreams} instances and waits for all of them to reach the
+     * {@link State#RUNNING} state at the same time. Note that states may change between the time
+     * that this method returns and the calling function executes its next statement.
+     *
+     * @param streamsList the list of streams instances to run.
+     * @param timeout the time to wait for the streams to all be in @{link State#RUNNING} state.
+     */
+    public static void startApplicationAndWaitUntilRunning(final List streamsList,
+                                                           final Duration timeout) throws InterruptedException {
+        final Lock stateLock = new ReentrantLock();
+        final Condition stateUpdate = stateLock.newCondition();
+        final Map stateMap = new HashMap<>();
+        for (final KafkaStreams streams : streamsList) {
+            stateMap.put(streams, streams.state());
+            final StateListener prevStateListener = getStateListener(streams);
+            final StateListener newStateListener = (newState, oldState) -> {
+                stateLock.lock();
+                try {
+                    stateMap.put(streams, newState);
+                    if (newState == State.RUNNING) {
+                        if (stateMap.values().stream().allMatch(state -> state == State.RUNNING)) {
+                            stateUpdate.signalAll();
+                        }
+                    }
+                } finally {
+                    stateLock.unlock();
+                }
+            };
+
+            streams.setStateListener(prevStateListener != null
+                ? new CompositeStateListener(prevStateListener, newStateListener)
+                : newStateListener);
+        }
+
+        for (final KafkaStreams streams : streamsList) {
+            streams.start();
+        }
+
+        final long expectedEnd = System.currentTimeMillis() + timeout.toMillis();
+        stateLock.lock();
+        try {
+            // We use while true here because we want to run this test at least once, even if the
+            // timeout has expired
+            while (true) {
+                final Map nonRunningStreams = new HashMap<>();
+                for (final Entry entry : stateMap.entrySet()) {
+                    if (entry.getValue() != State.RUNNING) {
+                        nonRunningStreams.put(entry.getKey(), entry.getValue());
+                    }
+                }
+
+                if (nonRunningStreams.isEmpty()) {
+                    return;
+                }
+
+                final long millisRemaining = expectedEnd - System.currentTimeMillis();
+                if (millisRemaining <= 0) {
+                    fail("Application did not reach a RUNNING state for all streams instances. Non-running instances: " +
+                        nonRunningStreams);
+                }
+
+                stateUpdate.await(millisRemaining, TimeUnit.MILLISECONDS);
+            }
+        } finally {
+            stateLock.unlock();
+        }
+    }
+
+    private static StateListener getStateListener(final KafkaStreams streams) {
+        try {
+            final Field field = streams.getClass().getDeclaredField("stateListener");
+            field.setAccessible(true);
+            return (StateListener) field.get(streams);
+        } catch (final IllegalAccessException | NoSuchFieldException e) {
+            throw new RuntimeException("Failed to get StateListener through reflection", e);
+        }
+    }
+
+
     public static  void verifyKeyValueTimestamps(final Properties consumerConfig,
                                                        final String topic,
                                                        final List> expected) {

From 09569668ae2ae3e86f2e6de7819f9496fc12a1f9 Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Thu, 17 Oct 2019 13:42:10 -0500
Subject: [PATCH 0749/1071] KAFKA-9058: Lift queriable and materialized
 restrictions on FK Join (#7541)

Reviewers: Bill Bejeck , Matthias J. Sax 
---
 .../apache/kafka/streams/kstream/KTable.java  |  90 +++++++-
 .../streams/kstream/internals/KTableImpl.java | 123 ++++++++---
 .../integration/ForeignKeyJoinSuite.java      |   1 +
 ...leKTableForeignKeyJoinIntegrationTest.java |  58 +++---
 ...KeyJoinMaterializationIntegrationTest.java | 192 ++++++++++++++++++
 5 files changed, 403 insertions(+), 61 deletions(-)
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinMaterializationIntegrationTest.java

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java
index 5092d35f9e1d0..f9b38bb435344 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java
@@ -2119,6 +2119,24 @@  KTable outerJoin(final KTable other,
                                      final Named named,
                                      final Materialized> materialized);
 
+    /**
+     * Join records of this {@code KTable} with another {@code KTable} using non-windowed inner join.
+     * 
                
+     * This is a foreign key join, where the joining key is determined by the {@code foreignKeyExtractor}.
+     *
+     * @param other               the other {@code KTable} to be joined with this {@code KTable}. Keyed by KO.
+     * @param foreignKeyExtractor a {@link Function} that extracts the key (KO) from this table's value (V). If the
+     *                            result is null, the update is ignored as invalid.
+     * @param joiner              a {@link ValueJoiner} that computes the join result for a pair of matching records
+     * @param                 the value type of the result {@code KTable}
+     * @param                 the key type of the other {@code KTable}
+     * @param                 the value type of the other {@code KTable}
+     * @return a {@code KTable} that contains the result of joining this table with {@code other}
+     */
+     KTable join(final KTable other,
+                                    final Function foreignKeyExtractor,
+                                    final ValueJoiner joiner);
+
     /**
      * Join records of this {@code KTable} with another {@code KTable} using non-windowed inner join.
      * 
                
@@ -2129,17 +2147,35 @@  KTable outerJoin(final KTable other,
      *                            result is null, the update is ignored as invalid.
      * @param joiner              a {@link ValueJoiner} that computes the join result for a pair of matching records
      * @param named               a {@link Named} config used to name the processor in the topology
+     * @param                 the value type of the result {@code KTable}
+     * @param                 the key type of the other {@code KTable}
+     * @param                 the value type of the other {@code KTable}
+     * @return a {@code KTable} that contains the result of joining this table with {@code other}
+     */
+     KTable join(final KTable other,
+                                    final Function foreignKeyExtractor,
+                                    final ValueJoiner joiner,
+                                    final Named named);
+
+    /**
+     * Join records of this {@code KTable} with another {@code KTable} using non-windowed inner join.
+     * 
                
+     * This is a foreign key join, where the joining key is determined by the {@code foreignKeyExtractor}.
+     *
+     * @param other               the other {@code KTable} to be joined with this {@code KTable}. Keyed by KO.
+     * @param foreignKeyExtractor a {@link Function} that extracts the key (KO) from this table's value (V). If the
+     *                            result is null, the update is ignored as invalid.
+     * @param joiner              a {@link ValueJoiner} that computes the join result for a pair of matching records
      * @param materialized        a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
      *                            should be materialized. Cannot be {@code null}
      * @param                 the value type of the result {@code KTable}
      * @param                 the key type of the other {@code KTable}
      * @param                 the value type of the other {@code KTable}
-     * @return
+     * @return a {@code KTable} that contains the result of joining this table with {@code other}
      */
      KTable join(final KTable other,
                                     final Function foreignKeyExtractor,
                                     final ValueJoiner joiner,
-                                    final Named named,
                                     final Materialized> materialized);
 
     /**
@@ -2151,18 +2187,38 @@  KTable join(final KTable other,
      * @param foreignKeyExtractor a {@link Function} that extracts the key (KO) from this table's value (V). If the
      *                            result is null, the update is ignored as invalid.
      * @param joiner              a {@link ValueJoiner} that computes the join result for a pair of matching records
+     * @param named               a {@link Named} config used to name the processor in the topology
      * @param materialized        a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
      *                            should be materialized. Cannot be {@code null}
      * @param                 the value type of the result {@code KTable}
      * @param                 the key type of the other {@code KTable}
      * @param                 the value type of the other {@code KTable}
-     * @return
+     * @return a {@code KTable} that contains the result of joining this table with {@code other}
      */
      KTable join(final KTable other,
                                     final Function foreignKeyExtractor,
                                     final ValueJoiner joiner,
+                                    final Named named,
                                     final Materialized> materialized);
 
+    /**
+     * Join records of this {@code KTable} with another {@code KTable} using non-windowed left join.
+     * 
                
+     * This is a foreign key join, where the joining key is determined by the {@code foreignKeyExtractor}.
+     *
+     * @param other               the other {@code KTable} to be joined with this {@code KTable}. Keyed by KO.
+     * @param foreignKeyExtractor a {@link Function} that extracts the key (KO) from this table's value (V). If the
+     *                            result is null, the update is ignored as invalid.
+     * @param joiner              a {@link ValueJoiner} that computes the join result for a pair of matching records
+     * @param                 the value type of the result {@code KTable}
+     * @param                 the key type of the other {@code KTable}
+     * @param                 the value type of the other {@code KTable}
+     * @return a {@code KTable} that contains only those records that satisfy the given predicate
+     */
+     KTable leftJoin(final KTable other,
+                                        final Function foreignKeyExtractor,
+                                        final ValueJoiner joiner);
+
     /**
      * Join records of this {@code KTable} with another {@code KTable} using non-windowed left join.
      * 
                
@@ -2173,17 +2229,35 @@  KTable join(final KTable other,
      *                            result is null, the update is ignored as invalid.
      * @param joiner              a {@link ValueJoiner} that computes the join result for a pair of matching records
      * @param named               a {@link Named} config used to name the processor in the topology
+     * @param                 the value type of the result {@code KTable}
+     * @param                 the key type of the other {@code KTable}
+     * @param                 the value type of the other {@code KTable}
+     * @return a {@code KTable} that contains the result of joining this table with {@code other}
+     */
+     KTable leftJoin(final KTable other,
+                                        final Function foreignKeyExtractor,
+                                        final ValueJoiner joiner,
+                                        final Named named);
+
+    /**
+     * Join records of this {@code KTable} with another {@code KTable} using non-windowed left join.
+     * 
                
+     * This is a foreign key join, where the joining key is determined by the {@code foreignKeyExtractor}.
+     *
+     * @param other               the other {@code KTable} to be joined with this {@code KTable}. Keyed by KO.
+     * @param foreignKeyExtractor a {@link Function} that extracts the key (KO) from this table's value (V). If the
+     *                            result is null, the update is ignored as invalid.
+     * @param joiner              a {@link ValueJoiner} that computes the join result for a pair of matching records
      * @param materialized        a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
      *                            should be materialized. Cannot be {@code null}
      * @param                 the value type of the result {@code KTable}
      * @param                 the key type of the other {@code KTable}
      * @param                 the value type of the other {@code KTable}
-     * @return a {@code KTable} that contains only those records that satisfy the given predicate
+     * @return a {@code KTable} that contains the result of joining this table with {@code other}
      */
      KTable leftJoin(final KTable other,
                                         final Function foreignKeyExtractor,
                                         final ValueJoiner joiner,
-                                        final Named named,
                                         final Materialized> materialized);
 
     /**
@@ -2192,19 +2266,21 @@  KTable leftJoin(final KTable other,
      * This is a foreign key join, where the joining key is determined by the {@code foreignKeyExtractor}.
      *
      * @param other               the other {@code KTable} to be joined with this {@code KTable}. Keyed by KO.
-     * @param foreignKeyExtractor a {@link Function} that extracts the key (KO) from this table's value (V). If the
+     * @param foreignKeyExtractor a {@link Function} that extracts the key (KO) from this table's value (V) If the
      *                            result is null, the update is ignored as invalid.
      * @param joiner              a {@link ValueJoiner} that computes the join result for a pair of matching records
+     * @param named               a {@link Named} config used to name the processor in the topology
      * @param materialized        a {@link Materialized} that describes how the {@link StateStore} for the resulting {@code KTable}
      *                            should be materialized. Cannot be {@code null}
      * @param                 the value type of the result {@code KTable}
      * @param                 the key type of the other {@code KTable}
      * @param                 the value type of the other {@code KTable}
-     * @return a {@code KTable} that contains only those records that satisfy the given predicate
+     * @return a {@code KTable} that contains the result of joining this table with {@code other}
      */
      KTable leftJoin(final KTable other,
                                         final Function foreignKeyExtractor,
                                         final ValueJoiner joiner,
+                                        final Named named,
                                         final Materialized> materialized);
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
index 05e04e8c2912d..301710ddb4400 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
@@ -110,12 +110,14 @@ public class KTableImpl extends AbstractStream implements KTable<
 
     private static final String TRANSFORMVALUES_NAME = "KTABLE-TRANSFORMVALUES-";
 
-    private static final String FK_JOIN_STATE_STORE_NAME = "KTABLE-INTERNAL-SUBSCRIPTION-STATE-STORE-";
-    private static final String SUBSCRIPTION_REGISTRATION = "KTABLE-SUBSCRIPTION-REGISTRATION-";
-    private static final String SUBSCRIPTION_RESPONSE = "KTABLE-SUBSCRIPTION-RESPONSE-";
-    private static final String SUBSCRIPTION_PROCESSOR = "KTABLE-SUBSCRIPTION-PROCESSOR-";
-    private static final String SUBSCRIPTION_RESPONSE_RESOLVER_PROCESSOR = "KTABLE-SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-";
-    private static final String FK_JOIN_OUTPUT_PROCESSOR = "KTABLE-OUTPUT-PROCESSOR-";
+    private static final String FK_JOIN = "KTABLE-FK-JOIN-";
+    private static final String FK_JOIN_STATE_STORE_NAME = FK_JOIN + "SUBSCRIPTION-STATE-STORE-";
+    private static final String SUBSCRIPTION_REGISTRATION = FK_JOIN + "SUBSCRIPTION-REGISTRATION-";
+    private static final String SUBSCRIPTION_RESPONSE = FK_JOIN + "SUBSCRIPTION-RESPONSE-";
+    private static final String SUBSCRIPTION_PROCESSOR = FK_JOIN + "SUBSCRIPTION-PROCESSOR-";
+    private static final String SUBSCRIPTION_RESPONSE_RESOLVER_PROCESSOR = FK_JOIN + "SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-";
+    private static final String FK_JOIN_OUTPUT_NAME = FK_JOIN + "OUTPUT-";
+
     private static final String TOPIC_SUFFIX = "-topic";
     private static final String SINK_NAME = "KTABLE-SINK-";
 
@@ -833,23 +835,79 @@ private  ProcessorParameters unsafeCastProcessorParametersToCompletel
         return (ProcessorParameters) kObjectProcessorParameters;
     }
 
+    @Override
+    public  KTable join(final KTable other,
+                                           final Function foreignKeyExtractor,
+                                           final ValueJoiner joiner) {
+        return doJoinOnForeignKey(
+            other,
+            foreignKeyExtractor,
+            joiner,
+            NamedInternal.empty(),
+            Materialized.with(null, null),
+            false
+        );
+    }
+
     @Override
     public  KTable join(final KTable other,
                                            final Function foreignKeyExtractor,
                                            final ValueJoiner joiner,
-                                           final Named named,
-                                           final Materialized> materialized) {
+                                           final Named named) {
+        return doJoinOnForeignKey(
+            other,
+            foreignKeyExtractor,
+            joiner,
+            named,
+            Materialized.with(null, null),
+            false
+        );
+    }
 
-        return doJoinOnForeignKey(other, foreignKeyExtractor, joiner, named, new MaterializedInternal<>(materialized), false);
+    @Override
+    public  KTable join(final KTable other,
+                                           final Function foreignKeyExtractor,
+                                           final ValueJoiner joiner,
+                                           final Materialized> materialized) {
+        return doJoinOnForeignKey(other, foreignKeyExtractor, joiner, NamedInternal.empty(), materialized, false);
     }
 
     @Override
     public  KTable join(final KTable other,
                                            final Function foreignKeyExtractor,
                                            final ValueJoiner joiner,
+                                           final Named named,
                                            final Materialized> materialized) {
+        return doJoinOnForeignKey(other, foreignKeyExtractor, joiner, named, materialized, false);
+    }
+
+    @Override
+    public  KTable leftJoin(final KTable other,
+                                               final Function foreignKeyExtractor,
+                                               final ValueJoiner joiner) {
+        return doJoinOnForeignKey(
+            other,
+            foreignKeyExtractor,
+            joiner,
+            NamedInternal.empty(),
+            Materialized.with(null, null),
+            true
+        );
+    }
 
-        return doJoinOnForeignKey(other, foreignKeyExtractor, joiner, NamedInternal.empty(), new MaterializedInternal<>(materialized), false);
+    @Override
+    public  KTable leftJoin(final KTable other,
+                                               final Function foreignKeyExtractor,
+                                               final ValueJoiner joiner,
+                                               final Named named) {
+        return doJoinOnForeignKey(
+            other,
+            foreignKeyExtractor,
+            joiner,
+            named,
+            Materialized.with(null, null),
+            true
+        );
     }
 
     @Override
@@ -858,7 +916,7 @@ public  KTable leftJoin(final KTable other,
                                                final ValueJoiner joiner,
                                                final Named named,
                                                final Materialized> materialized) {
-        return doJoinOnForeignKey(other, foreignKeyExtractor, joiner, named, new MaterializedInternal<>(materialized), true);
+        return doJoinOnForeignKey(other, foreignKeyExtractor, joiner, named, materialized, true);
     }
 
     @Override
@@ -866,23 +924,21 @@ public  KTable leftJoin(final KTable other,
                                                final Function foreignKeyExtractor,
                                                final ValueJoiner joiner,
                                                final Materialized> materialized) {
-
-        return doJoinOnForeignKey(other, foreignKeyExtractor, joiner, NamedInternal.empty(), new MaterializedInternal<>(materialized), true);
+        return doJoinOnForeignKey(other, foreignKeyExtractor, joiner, NamedInternal.empty(), materialized, true);
     }
 
-
     @SuppressWarnings("unchecked")
     private  KTable doJoinOnForeignKey(final KTable foreignKeyTable,
                                                           final Function foreignKeyExtractor,
                                                           final ValueJoiner joiner,
                                                           final Named joinName,
-                                                          final MaterializedInternal> materializedInternal,
+                                                          final Materialized> materialized,
                                                           final boolean leftJoin) {
         Objects.requireNonNull(foreignKeyTable, "foreignKeyTable can't be null");
         Objects.requireNonNull(foreignKeyExtractor, "foreignKeyExtractor can't be null");
         Objects.requireNonNull(joiner, "joiner can't be null");
         Objects.requireNonNull(joinName, "joinName can't be null");
-        Objects.requireNonNull(materializedInternal, "materialized can't be null");
+        Objects.requireNonNull(materialized, "materialized can't be null");
 
         //Old values are a useful optimization. The old values from the foreignKeyTable table are compared to the new values,
         //such that identical values do not cause a prefixScan. PrefixScan and propagation can be expensive and should
@@ -1011,14 +1067,15 @@ private  KTable doJoinOnForeignKey(final KTable forei
         builder.internalTopologyBuilder.copartitionSources(resultSourceNodes);
 
         final KTableValueGetterSupplier primaryKeyValueGetter = valueGetterSupplier();
+        final SubscriptionResolverJoinProcessorSupplier resolverProcessorSupplier = new SubscriptionResolverJoinProcessorSupplier<>(
+            primaryKeyValueGetter,
+            valueSerde().serializer(),
+            joiner,
+            leftJoin
+        );
         final StatefulProcessorNode> resolverNode = new StatefulProcessorNode<>(
             new ProcessorParameters<>(
-                new SubscriptionResolverJoinProcessorSupplier<>(
-                    primaryKeyValueGetter,
-                    valueSerde().serializer(),
-                    joiner,
-                    leftJoin
-                ),
+                resolverProcessorSupplier,
                 renamed.suffixWithOrElseGet("-subscription-response-resolver", builder, SUBSCRIPTION_RESPONSE_RESOLVER_PROCESSOR)
             ),
             Collections.emptySet(),
@@ -1026,8 +1083,26 @@ private  KTable doJoinOnForeignKey(final KTable forei
         );
         builder.addGraphNode(foreignResponseSource, resolverNode);
 
-        final String resultProcessorName = renamed.suffixWithOrElseGet("-result", builder, FK_JOIN_OUTPUT_PROCESSOR);
-        final KTableSource resultProcessorSupplier = new KTableSource<>(materializedInternal.storeName(), materializedInternal.queryableStoreName());
+        final String resultProcessorName = renamed.suffixWithOrElseGet("-result", builder, FK_JOIN_OUTPUT_NAME);
+
+        final MaterializedInternal> materializedInternal =
+            new MaterializedInternal<>(
+                materialized,
+                builder,
+                FK_JOIN_OUTPUT_NAME
+            );
+
+        // If we have a key serde, it's still valid, but we don't know the value serde, since it's the result
+        // of the joiner (VR).
+        if (materializedInternal.keySerde() == null) {
+            materializedInternal.withKeySerde(keySerde);
+        }
+
+        final KTableSource resultProcessorSupplier = new KTableSource<>(
+            materializedInternal.storeName(),
+            materializedInternal.queryableStoreName()
+        );
+
         final StoreBuilder> resultStore =
             materializedInternal.queryableStoreName() == null
                 ? null
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/ForeignKeyJoinSuite.java b/streams/src/test/java/org/apache/kafka/streams/integration/ForeignKeyJoinSuite.java
index ac6adb876aa15..47e2d95da63bc 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/ForeignKeyJoinSuite.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/ForeignKeyJoinSuite.java
@@ -38,6 +38,7 @@
     BytesTest.class,
     KTableKTableForeignKeyInnerJoinMultiIntegrationTest.class,
     KTableKTableForeignKeyJoinIntegrationTest.class,
+    KTableKTableForeignKeyJoinMaterializationIntegrationTest.class,
     CombinedKeySchemaTest.class,
     SubscriptionWrapperSerdeTest.class,
     SubscriptionResponseWrapperSerdeTest.class,
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinIntegrationTest.java
index 14a39b52ea307..80c0f524e4f83 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinIntegrationTest.java
@@ -20,7 +20,6 @@
 import org.apache.kafka.common.serialization.StringDeserializer;
 import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.common.utils.Bytes;
-import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.TestInputTopic;
@@ -30,7 +29,6 @@
 import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.KTable;
 import org.apache.kafka.streams.kstream.Materialized;
-import org.apache.kafka.streams.kstream.Produced;
 import org.apache.kafka.streams.kstream.ValueJoiner;
 import org.apache.kafka.streams.state.KeyValueStore;
 import org.apache.kafka.streams.state.Stores;
@@ -54,7 +52,7 @@
 import static org.hamcrest.MatcherAssert.assertThat;
 
 
-@RunWith(value = Parameterized.class)
+@RunWith(Parameterized.class)
 public class KTableKTableForeignKeyJoinIntegrationTest {
 
     private static final String LEFT_TABLE = "left_table";
@@ -265,10 +263,9 @@ public void doJoinFromRightThenDeleteRightEntity() {
     }
 
     @Test
-    public void shouldEmitTombstonedWhenDeletingNonJoiningRecords() {
+    public void shouldEmitTombstoneWhenDeletingNonJoiningRecords() {
         final Topology topology = getTopology(streamsConfig, "store", leftJoin);
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
-            final TestInputTopic right = driver.createInputTopic(RIGHT_TABLE, new StringSerializer(), new StringSerializer());
             final TestInputTopic left = driver.createInputTopic(LEFT_TABLE, new StringSerializer(), new StringSerializer());
             final TestOutputTopic outputTopic = driver.createOutputTopic(OUTPUT, new StringDeserializer(), new StringDeserializer());
             final KeyValueStore store = driver.getKeyValueStore("store");
@@ -295,7 +292,7 @@ public void shouldEmitTombstonedWhenDeletingNonJoiningRecords() {
             {
                 assertThat(
                     outputTopic.readKeyValuesToMap(),
-                    is(Utils.mkMap(mkEntry("lhs1", null)))
+                    is(mkMap(mkEntry("lhs1", null)))
                 );
                 assertThat(
                     asMap(store),
@@ -322,7 +319,6 @@ public void shouldEmitTombstonedWhenDeletingNonJoiningRecords() {
     public void shouldNotEmitTombstonesWhenDeletingNonExistingRecords() {
         final Topology topology = getTopology(streamsConfig, "store", leftJoin);
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
-            final TestInputTopic right = driver.createInputTopic(RIGHT_TABLE, new StringSerializer(), new StringSerializer());
             final TestInputTopic left = driver.createInputTopic(LEFT_TABLE, new StringSerializer(), new StringSerializer());
             final TestOutputTopic outputTopic = driver.createOutputTopic(OUTPUT, new StringDeserializer(), new StringDeserializer());
             final KeyValueStore store = driver.getKeyValueStore("store");
@@ -369,9 +365,7 @@ public void joinShouldProduceNullsWhenValueHasNonMatchingForeignKey() {
             left.pipeInput("lhs1", "lhsValue1|rhs2");
             assertThat(
                 outputTopic.readKeyValuesToMap(),
-                is(mkMap(
-                    mkEntry("lhs1", leftJoin ? "(lhsValue1|rhs2,null)" : null)
-                ))
+                is(mkMap(mkEntry("lhs1", leftJoin ? "(lhsValue1|rhs2,null)" : null)))
             );
             assertThat(
                 asMap(store),
@@ -381,9 +375,7 @@ public void joinShouldProduceNullsWhenValueHasNonMatchingForeignKey() {
             left.pipeInput("lhs1", "lhsValue1|rhs3");
             assertThat(
                 outputTopic.readKeyValuesToMap(),
-                is(mkMap(
-                    mkEntry("lhs1", leftJoin ? "(lhsValue1|rhs3,null)" : null)
-                ))
+                is(mkMap(mkEntry("lhs1", leftJoin ? "(lhsValue1|rhs3,null)" : null)))
             );
             assertThat(
                 asMap(store),
@@ -448,29 +440,35 @@ private static Topology getTopology(final Properties streamsConfig,
         final KTable left = builder.table(LEFT_TABLE, Consumed.with(Serdes.String(), Serdes.String()));
         final KTable right = builder.table(RIGHT_TABLE, Consumed.with(Serdes.String(), Serdes.String()));
 
+        final Function extractor = value -> value.split("\\|")[1];
+        final ValueJoiner joiner = (value1, value2) -> "(" + value1 + "," + value2 + ")";
+
         final Materialized> materialized =
             Materialized.as(Stores.inMemoryKeyValueStore(queryableStoreName))
-                .withKeySerde(Serdes.String())
                 .withValueSerde(Serdes.String())
+                // the cache suppresses some of the unnecessary tombstones we want to make assertions about
                 .withCachingDisabled();
 
-        final Function extractor = value -> value.split("\\|")[1];
-        final ValueJoiner joiner = (value1, value2) -> "(" + value1 + "," + value2 + ")";
+        final KTable joinResult;
+        if (leftJoin) {
+            joinResult = left.leftJoin(
+                right,
+                extractor,
+                joiner,
+                materialized
+            );
+        } else {
+            joinResult = left.join(
+                right,
+                extractor,
+                joiner,
+                materialized
+            );
+        }
 
-        if (leftJoin)
-            left.leftJoin(right,
-                          extractor,
-                          joiner,
-                          materialized)
-                .toStream()
-                .to(OUTPUT, Produced.with(Serdes.String(), Serdes.String()));
-        else
-            left.join(right,
-                      extractor,
-                      joiner,
-                      materialized)
-                .toStream()
-                .to(OUTPUT, Produced.with(Serdes.String(), Serdes.String()));
+        joinResult
+            .toStream()
+            .to(OUTPUT);
 
         return builder.build(streamsConfig);
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinMaterializationIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinMaterializationIntegrationTest.java
new file mode 100644
index 0000000000000..6abedb2f88487
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinMaterializationIntegrationTest.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.integration;
+
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TestInputTopic;
+import org.apache.kafka.streams.TestOutputTopic;
+import org.apache.kafka.streams.Topology;
+import org.apache.kafka.streams.TopologyTestDriver;
+import org.apache.kafka.streams.kstream.Consumed;
+import org.apache.kafka.streams.kstream.KTable;
+import org.apache.kafka.streams.kstream.Materialized;
+import org.apache.kafka.streams.kstream.ValueJoiner;
+import org.apache.kafka.streams.state.KeyValueStore;
+import org.apache.kafka.test.TestUtils;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.function.Function;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.common.utils.Utils.mkEntry;
+import static org.apache.kafka.common.utils.Utils.mkMap;
+import static org.apache.kafka.common.utils.Utils.mkProperties;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+
+@RunWith(Parameterized.class)
+public class KTableKTableForeignKeyJoinMaterializationIntegrationTest {
+
+    private static final String LEFT_TABLE = "left_table";
+    private static final String RIGHT_TABLE = "right_table";
+    private static final String OUTPUT = "output-topic";
+    private final Properties streamsConfig;
+    private final boolean materialized;
+    private final boolean queriable;
+
+    public KTableKTableForeignKeyJoinMaterializationIntegrationTest(final boolean materialized, final boolean queriable) {
+        this.materialized = materialized;
+        this.queriable = queriable;
+        streamsConfig = mkProperties(mkMap(
+            mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, "ktable-ktable-joinOnForeignKey"),
+            mkEntry(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "asdf:0000"),
+            mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath())
+        ));
+    }
+
+    @Parameterized.Parameters(name = "materialized={0}, queriable={1}")
+    public static Collection data() {
+        return Arrays.asList(
+            new Object[] {false, false},
+            new Object[] {true, false},
+            new Object[] {true, true}
+        );
+    }
+
+    @Test
+    public void shouldEmitTombstoneWhenDeletingNonJoiningRecords() {
+        final Topology topology = getTopology(streamsConfig, "store");
+        try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
+            final TestInputTopic left = driver.createInputTopic(LEFT_TABLE, new StringSerializer(), new StringSerializer());
+            final TestOutputTopic outputTopic = driver.createOutputTopic(OUTPUT, new StringDeserializer(), new StringDeserializer());
+            final KeyValueStore store = driver.getKeyValueStore("store");
+
+            left.pipeInput("lhs1", "lhsValue1|rhs1");
+
+            assertThat(
+                outputTopic.readKeyValuesToMap(),
+                is(emptyMap())
+            );
+            if (materialized && queriable) {
+                assertThat(
+                    asMap(store),
+                    is(emptyMap())
+                );
+            }
+
+            // Deleting a non-joining record produces an unnecessary tombstone for inner joins, because
+            // it's not possible to know whether a result was previously emitted.
+            left.pipeInput("lhs1", (String) null);
+            {
+                if (materialized && queriable) {
+                    // in only this specific case, the record cache will actually be activated and
+                    // suppress the unnecessary tombstone. This is because the cache is able to determine
+                    // for sure that there has never been a previous result. (Because the "old" and "new" values
+                    // are both null, and the underlying store is also missing the record in question).
+                    assertThat(
+                        outputTopic.readKeyValuesToMap(),
+                        is(emptyMap())
+                    );
+
+                    assertThat(
+                        asMap(store),
+                        is(emptyMap())
+                    );
+                } else {
+                    assertThat(
+                        outputTopic.readKeyValuesToMap(),
+                        is(mkMap(mkEntry("lhs1", null)))
+                    );
+                }
+            }
+
+            // Deleting a non-existing record is idempotent
+            left.pipeInput("lhs1", (String) null);
+            {
+                assertThat(
+                    outputTopic.readKeyValuesToMap(),
+                    is(emptyMap())
+                );
+                if (materialized && queriable) {
+                    assertThat(
+                        asMap(store),
+                        is(emptyMap())
+                    );
+                }
+            }
+        }
+    }
+
+    private static Map asMap(final KeyValueStore store) {
+        final HashMap result = new HashMap<>();
+        store.all().forEachRemaining(kv -> result.put(kv.key, kv.value));
+        return result;
+    }
+
+    private Topology getTopology(final Properties streamsConfig,
+                                 final String queryableStoreName) {
+        final StreamsBuilder builder = new StreamsBuilder();
+
+        final KTable left = builder.table(LEFT_TABLE, Consumed.with(Serdes.String(), Serdes.String()));
+        final KTable right = builder.table(RIGHT_TABLE, Consumed.with(Serdes.String(), Serdes.String()));
+
+        final Function extractor = value -> value.split("\\|")[1];
+        final ValueJoiner joiner = (value1, value2) -> "(" + value1 + "," + value2 + ")";
+
+        final Materialized> materialized;
+        if (queriable) {
+            materialized = Materialized.>as(queryableStoreName).withValueSerde(Serdes.String());
+        } else {
+            materialized = Materialized.with(null, Serdes.String());
+        }
+
+        final KTable joinResult;
+        if (this.materialized) {
+            joinResult = left.join(
+                right,
+                extractor,
+                joiner,
+                materialized
+            );
+        } else {
+            joinResult = left.join(
+                right,
+                extractor,
+                joiner
+            );
+        }
+
+        joinResult
+            .toStream()
+            .to(OUTPUT);
+
+        return builder.build(streamsConfig);
+    }
+}

From 1be8fb9e3781533dc2c0d269979ef6a18aa56a6b Mon Sep 17 00:00:00 2001
From: Dhruvil Shah 
Date: Fri, 18 Oct 2019 02:08:35 -0400
Subject: [PATCH 0750/1071] KAFKA-8962; Use least loaded node for
 AdminClient#describeTopics (#7421)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Allow routing of `AdminClient#describeTopics` to any broker in the cluster than just the controller, so that we don't create a hotspot for this API call. `AdminClient#describeTopics` uses the broker's metadata cache which is asynchronously maintained, so routing to brokers other than the controller is not expected to have a significant difference in terms of metadata consistency; all metadata requests are eventually consistent.

This patch also fixes a few flaky test failures.

Reviewers: Ismael Juma , José Armando García Sancio , Jason Gustafson 
---
 .../kafka/clients/admin/KafkaAdminClient.java |   2 +-
 .../kafka/controller/KafkaController.scala    |   2 +-
 .../api/AdminClientIntegrationTest.scala      | 153 ++++++++++--------
 .../SaslSslAdminClientIntegrationTest.scala   |  25 ++-
 .../admin/LeaderElectionCommandTest.scala     |  26 +--
 .../scala/unit/kafka/utils/TestUtils.scala    |  36 +++--
 6 files changed, 142 insertions(+), 102 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
index d4958f2738eaf..0850cedd71037 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
@@ -1537,7 +1537,7 @@ public DescribeTopicsResult describeTopics(final Collection topicNames,
         }
         final long now = time.milliseconds();
         Call call = new Call("describeTopics", calcDeadlineMs(now, options.timeoutMs()),
-            new ControllerNodeProvider()) {
+            new LeastLoadedNodeProvider()) {
 
             private boolean supportsDisablingTopicCreation = true;
 
diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala
index 91c4534e143d9..0dd2ec793d621 100644
--- a/core/src/main/scala/kafka/controller/KafkaController.scala
+++ b/core/src/main/scala/kafka/controller/KafkaController.scala
@@ -790,7 +790,7 @@ class KafkaController(val config: KafkaConfig,
     electionType: ElectionType,
     electionTrigger: ElectionTrigger
   ): Map[TopicPartition, Either[Throwable, LeaderAndIsr]] = {
-    info(s"Starting replica leader election ($electionType) for partitions ${partitions.mkString(",")} triggerd by $electionTrigger")
+    info(s"Starting replica leader election ($electionType) for partitions ${partitions.mkString(",")} triggered by $electionTrigger")
     try {
       val strategy = electionType match {
         case ElectionType.PREFERRED => PreferredReplicaPartitionLeaderElectionStrategy
diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index f2e7af6b26ddc..6ff4f0e677e91 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -37,10 +37,7 @@ import org.apache.kafka.clients.consumer.ConsumerRecords
 import org.apache.kafka.clients.consumer.{ConsumerConfig, KafkaConsumer}
 import org.apache.kafka.clients.producer.KafkaProducer
 import org.apache.kafka.clients.producer.ProducerRecord
-import org.apache.kafka.common.ConsumerGroupState
-import org.apache.kafka.common.ElectionType
-import org.apache.kafka.common.TopicPartition
-import org.apache.kafka.common.TopicPartitionReplica
+import org.apache.kafka.common.{ConsumerGroupState, ElectionType, TopicPartition, TopicPartitionInfo, TopicPartitionReplica}
 import org.apache.kafka.common.acl._
 import org.apache.kafka.common.config.{ConfigResource, LogLevelConfig}
 import org.apache.kafka.common.errors._
@@ -296,15 +293,14 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     waitForTopics(client, expectedPresent = Seq(topic), expectedMissing = List())
 
     // without includeAuthorizedOperations flag
-    var topicResult = client.describeTopics(Seq(topic).asJava).values
-    assertEquals(Set().asJava, topicResult.get(topic).get().authorizedOperations())
+    var topicResult = getTopicMetadata(client, topic)
+    assertEquals(Set().asJava, topicResult.authorizedOperations)
 
     //with includeAuthorizedOperations flag
-    topicResult = client.describeTopics(Seq(topic).asJava,
-      new DescribeTopicsOptions().includeAuthorizedOperations(true)).values
+    topicResult = getTopicMetadata(client, topic, new DescribeTopicsOptions().includeAuthorizedOperations(true))
     expectedOperations = Topic.supportedOperations
       .map(operation => operation.toJava).asJava
-    assertEquals(expectedOperations, topicResult.get(topic).get().authorizedOperations())
+    assertEquals(expectedOperations, topicResult.authorizedOperations)
   }
 
   def configuredClusterPermissions() : Set[AclOperation] = {
@@ -559,17 +555,19 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     createTopic(topic2, numPartitions = 1, replicationFactor = 2)
 
     // assert that both the topics have 1 partition
-    assertEquals(1, client.describeTopics(Set(topic1).asJava).values.get(topic1).get.partitions.size)
-    assertEquals(1, client.describeTopics(Set(topic2).asJava).values.get(topic2).get.partitions.size)
+    val topic1_metadata = getTopicMetadata(client, topic1)
+    val topic2_metadata = getTopicMetadata(client, topic2)
+    assertEquals(1, topic1_metadata.partitions.size)
+    assertEquals(1, topic2_metadata.partitions.size)
 
     val validateOnly = new CreatePartitionsOptions().validateOnly(true)
     val actuallyDoIt = new CreatePartitionsOptions().validateOnly(false)
 
-    def partitions(topic: String) =
-      client.describeTopics(Set(topic).asJava).values.get(topic).get.partitions
+    def partitions(topic: String, expectedNumPartitionsOpt: Option[Int] = None): util.List[TopicPartitionInfo] = {
+      getTopicMetadata(client, topic, expectedNumPartitionsOpt = expectedNumPartitionsOpt).partitions
+    }
 
-    def numPartitions(topic: String) =
-      partitions(topic).size
+    def numPartitions(topic: String): Int = partitions(topic).size
 
     // validateOnly: try creating a new partition (no assignments), to bring the total to 3 partitions
     var alterResult = client.createPartitions(Map(topic1 ->
@@ -581,7 +579,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     alterResult = client.createPartitions(Map(topic1 ->
       NewPartitions.increaseTo(3)).asJava, actuallyDoIt)
     altered = alterResult.values.get(topic1).get
-    assertEquals(3, numPartitions(topic1))
+    TestUtils.waitUntilTrue(() => numPartitions(topic1) == 3, "Timed out waiting for new partitions to appear")
 
     // validateOnly: now try creating a new partition (with assignments), to bring the total to 3 partitions
     val newPartition2Assignments = asList[util.List[Integer]](asList(0, 1), asList(1, 2))
@@ -594,7 +592,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     alterResult = client.createPartitions(Map(topic2 ->
       NewPartitions.increaseTo(3, newPartition2Assignments)).asJava, actuallyDoIt)
     altered = alterResult.values.get(topic2).get
-    val actualPartitions2 = partitions(topic2)
+    val actualPartitions2 = partitions(topic2, expectedNumPartitionsOpt = Some(3))
     assertEquals(3, actualPartitions2.size)
     assertEquals(Seq(0, 1), actualPartitions2.get(1).replicas.asScala.map(_.id).toList)
     assertEquals(Seq(1, 2), actualPartitions2.get(2).replicas.asScala.map(_.id).toList)
@@ -782,7 +780,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
       topic2 -> NewPartitions.increaseTo(2)).asJava, actuallyDoIt)
     // assert that the topic1 now has 4 partitions
     altered = alterResult.values.get(topic1).get
-    assertEquals(4, numPartitions(topic1))
+    TestUtils.waitUntilTrue(() => numPartitions(topic1) == 4, "Timed out waiting for new partitions to appear")
     try {
       altered = alterResult.values.get(topic2).get
     } catch {
@@ -1452,15 +1450,17 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     val partition2 = new TopicPartition("elect-preferred-leaders-topic-2", 0)
     TestUtils.createTopic(zkClient, partition2.topic, Map[Int, Seq[Int]](partition2.partition -> prefer0), servers)
 
-    def preferredLeader(topicPartition: TopicPartition) =
-      client.describeTopics(asList(topicPartition.topic)).values.get(topicPartition.topic).
-        get.partitions.get(topicPartition.partition).replicas.get(0).id
+    def preferredLeader(topicPartition: TopicPartition): Int = {
+      val partitionMetadata = getTopicMetadata(client, topicPartition.topic).partitions.get(topicPartition.partition)
+      val preferredLeaderMetadata = partitionMetadata.replicas.get(0)
+      preferredLeaderMetadata.id
+    }
 
     /** Changes the preferred leader without changing the current leader. */
     def changePreferredLeader(newAssignment: Seq[Int]) = {
       val preferred = newAssignment.head
-      val prior1 = TestUtils.currentLeader(client, partition1).get
-      val prior2 = TestUtils.currentLeader(client, partition2).get
+      val prior1 = zkClient.getLeaderForPartition(partition1).get
+      val prior2 = zkClient.getLeaderForPartition(partition2).get
 
       var m = Map.empty[TopicPartition, Seq[Int]]
 
@@ -1475,26 +1475,26 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
         s"Expected preferred leader to become $preferred, but is ${preferredLeader(partition1)} and ${preferredLeader(partition2)}",
         10000)
       // Check the leader hasn't moved
-      assertEquals(Some(prior1), TestUtils.currentLeader(client, partition1))
-      assertEquals(Some(prior2), TestUtils.currentLeader(client, partition2))
+      TestUtils.assertLeader(client, partition1, prior1)
+      TestUtils.assertLeader(client, partition2, prior2)
     }
 
     // Check current leaders are 0
-    assertEquals(Some(0), TestUtils.currentLeader(client, partition1))
-    assertEquals(Some(0), TestUtils.currentLeader(client, partition2))
+    TestUtils.assertLeader(client, partition1, 0)
+    TestUtils.assertLeader(client, partition2, 0)
 
     // Noop election
     var electResult = client.electLeaders(ElectionType.PREFERRED, Set(partition1).asJava)
     var exception = electResult.partitions.get.get(partition1).get
     assertEquals(classOf[ElectionNotNeededException], exception.getClass)
     assertEquals("Leader election not needed for topic partition", exception.getMessage)
-    assertEquals(Some(0), TestUtils.currentLeader(client, partition1))
+    TestUtils.assertLeader(client, partition1, 0)
 
     // Noop election with null partitions
     electResult = client.electLeaders(ElectionType.PREFERRED, null)
     assertTrue(electResult.partitions.get.isEmpty)
-    assertEquals(Some(0), TestUtils.currentLeader(client, partition1))
-    assertEquals(Some(0), TestUtils.currentLeader(client, partition2))
+    TestUtils.assertLeader(client, partition1, 0)
+    TestUtils.assertLeader(client, partition2, 0)
 
     // Now change the preferred leader to 1
     changePreferredLeader(prefer1)
@@ -1503,17 +1503,17 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     electResult = client.electLeaders(ElectionType.PREFERRED, Set(partition1).asJava)
     assertEquals(Set(partition1).asJava, electResult.partitions.get.keySet)
     assertFalse(electResult.partitions.get.get(partition1).isPresent)
-    TestUtils.waitForLeaderToBecome(client, partition1, Some(1))
+    TestUtils.assertLeader(client, partition1, 1)
 
     // topic 2 unchanged
     assertFalse(electResult.partitions.get.containsKey(partition2))
-    assertEquals(Some(0), TestUtils.currentLeader(client, partition2))
+    TestUtils.assertLeader(client, partition2, 0)
 
     // meaningful election with null partitions
     electResult = client.electLeaders(ElectionType.PREFERRED, null)
     assertEquals(Set(partition2), electResult.partitions.get.keySet.asScala)
     assertFalse(electResult.partitions.get.get(partition2).isPresent)
-    TestUtils.waitForLeaderToBecome(client, partition2, Some(1))
+    TestUtils.assertLeader(client, partition2, 1)
 
     // unknown topic
     val unknownPartition = new TopicPartition("topic-does-not-exist", 0)
@@ -1522,8 +1522,8 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     exception = electResult.partitions.get.get(unknownPartition).get
     assertEquals(classOf[UnknownTopicOrPartitionException], exception.getClass)
     assertEquals("The partition does not exist.", exception.getMessage)
-    assertEquals(Some(1), TestUtils.currentLeader(client, partition1))
-    assertEquals(Some(1), TestUtils.currentLeader(client, partition2))
+    TestUtils.assertLeader(client, partition1, 1)
+    TestUtils.assertLeader(client, partition2, 1)
 
     // Now change the preferred leader to 2
     changePreferredLeader(prefer2)
@@ -1531,8 +1531,8 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     // mixed results
     electResult = client.electLeaders(ElectionType.PREFERRED, Set(unknownPartition, partition1).asJava)
     assertEquals(Set(unknownPartition, partition1).asJava, electResult.partitions.get.keySet)
-    TestUtils.waitForLeaderToBecome(client, partition1, Some(2))
-    assertEquals(Some(1), TestUtils.currentLeader(client, partition2))
+    TestUtils.assertLeader(client, partition1, 2)
+    TestUtils.assertLeader(client, partition2, 1)
     exception = electResult.partitions.get.get(unknownPartition).get
     assertEquals(classOf[UnknownTopicOrPartitionException], exception.getClass)
     assertEquals("The partition does not exist.", exception.getMessage)
@@ -1541,7 +1541,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     electResult = client.electLeaders(ElectionType.PREFERRED, Set(partition2).asJava)
     assertEquals(Set(partition2).asJava, electResult.partitions.get.keySet)
     assertFalse(electResult.partitions.get.get(partition2).isPresent)
-    TestUtils.waitForLeaderToBecome(client, partition2, Some(2))
+    TestUtils.assertLeader(client, partition2, 2)
 
     // Now change the preferred leader to 1
     changePreferredLeader(prefer1)
@@ -1557,7 +1557,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     assertEquals(classOf[PreferredLeaderNotAvailableException], exception.getClass)
     assertTrue(s"Wrong message ${exception.getMessage}", exception.getMessage.contains(
       "Failed to elect leader for partition elect-preferred-leaders-topic-1-0 under strategy PreferredReplicaPartitionLeaderElectionStrategy"))
-    assertEquals(Some(2), TestUtils.currentLeader(client, partition1))
+    TestUtils.assertLeader(client, partition1, 2)
 
     // preferred leader unavailable with null argument
     electResult = client.electLeaders(ElectionType.PREFERRED, null, shortTimeout)
@@ -1572,8 +1572,8 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     assertTrue(s"Wrong message ${exception.getMessage}", exception.getMessage.contains(
       "Failed to elect leader for partition elect-preferred-leaders-topic-2-0 under strategy PreferredReplicaPartitionLeaderElectionStrategy"))
 
-    assertEquals(Some(2), TestUtils.currentLeader(client, partition1))
-    assertEquals(Some(2), TestUtils.currentLeader(client, partition2))
+    TestUtils.assertLeader(client, partition1, 2)
+    TestUtils.assertLeader(client, partition2, 2)
   }
 
   @Test
@@ -1588,17 +1588,17 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     val partition1 = new TopicPartition("unclean-test-topic-1", 0)
     TestUtils.createTopic(zkClient, partition1.topic, Map[Int, Seq[Int]](partition1.partition -> assignment1), servers)
 
-    TestUtils.waitForLeaderToBecome(client, partition1, Option(broker1))
+    TestUtils.assertLeader(client, partition1, broker1)
 
     servers(broker2).shutdown()
     TestUtils.waitForBrokersOutOfIsr(client, Set(partition1), Set(broker2))
     servers(broker1).shutdown()
-    TestUtils.waitForLeaderToBecome(client, partition1, None)
+    TestUtils.assertNoLeader(client, partition1)
     servers(broker2).startup()
 
     val electResult = client.electLeaders(ElectionType.UNCLEAN, Set(partition1).asJava)
     assertFalse(electResult.partitions.get.get(partition1).isPresent)
-    assertEquals(Option(broker2), TestUtils.currentLeader(client, partition1))
+    TestUtils.assertLeader(client, partition1, broker2)
   }
 
   @Test
@@ -1622,21 +1622,21 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
       servers
     )
 
-    TestUtils.waitForLeaderToBecome(client, partition1, Option(broker1))
-    TestUtils.waitForLeaderToBecome(client, partition2, Option(broker1))
+    TestUtils.assertLeader(client, partition1, broker1)
+    TestUtils.assertLeader(client, partition2, broker1)
 
     servers(broker2).shutdown()
     TestUtils.waitForBrokersOutOfIsr(client, Set(partition1, partition2), Set(broker2))
     servers(broker1).shutdown()
-    TestUtils.waitForLeaderToBecome(client, partition1, None)
-    TestUtils.waitForLeaderToBecome(client, partition2, None)
+    TestUtils.assertNoLeader(client, partition1)
+    TestUtils.assertNoLeader(client, partition2)
     servers(broker2).startup()
 
     val electResult = client.electLeaders(ElectionType.UNCLEAN, Set(partition1, partition2).asJava)
     assertFalse(electResult.partitions.get.get(partition1).isPresent)
     assertFalse(electResult.partitions.get.get(partition2).isPresent)
-    assertEquals(Option(broker2), TestUtils.currentLeader(client, partition1))
-    assertEquals(Option(broker2), TestUtils.currentLeader(client, partition2))
+    TestUtils.assertLeader(client, partition1, broker2)
+    TestUtils.assertLeader(client, partition2, broker2)
   }
 
   @Test
@@ -1661,21 +1661,21 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
       servers
     )
 
-    TestUtils.waitForLeaderToBecome(client, partition1, Option(broker1))
-    TestUtils.waitForLeaderToBecome(client, partition2, Option(broker1))
+    TestUtils.assertLeader(client, partition1, broker1)
+    TestUtils.assertLeader(client, partition2, broker1)
 
     servers(broker2).shutdown()
     TestUtils.waitForBrokersOutOfIsr(client, Set(partition1), Set(broker2))
     servers(broker1).shutdown()
-    TestUtils.waitForLeaderToBecome(client, partition1, None)
-    TestUtils.waitForLeaderToBecome(client, partition2, Some(broker3))
+    TestUtils.assertNoLeader(client, partition1)
+    TestUtils.assertLeader(client, partition2, broker3)
     servers(broker2).startup()
 
     val electResult = client.electLeaders(ElectionType.UNCLEAN, null)
     assertFalse(electResult.partitions.get.get(partition1).isPresent)
     assertFalse(electResult.partitions.get.containsKey(partition2))
-    assertEquals(Option(broker2), TestUtils.currentLeader(client, partition1))
-    assertEquals(Option(broker3), TestUtils.currentLeader(client, partition2))
+    TestUtils.assertLeader(client, partition1, broker2)
+    TestUtils.assertLeader(client, partition2, broker3)
   }
 
   @Test
@@ -1698,7 +1698,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
       servers
     )
 
-    TestUtils.waitForLeaderToBecome(client, new TopicPartition(topic, 0), Option(broker1))
+    TestUtils.assertLeader(client, new TopicPartition(topic, 0), broker1)
 
     val electResult = client.electLeaders(ElectionType.UNCLEAN, Set(unknownPartition, unknownTopic).asJava)
     assertTrue(electResult.partitions.get.get(unknownPartition).get.isInstanceOf[UnknownTopicOrPartitionException])
@@ -1724,12 +1724,12 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
       servers
     )
 
-    TestUtils.waitForLeaderToBecome(client, partition1, Option(broker1))
+    TestUtils.assertLeader(client, partition1, broker1)
 
     servers(broker2).shutdown()
     TestUtils.waitForBrokersOutOfIsr(client, Set(partition1), Set(broker2))
     servers(broker1).shutdown()
-    TestUtils.waitForLeaderToBecome(client, partition1, None)
+    TestUtils.assertNoLeader(client, partition1)
 
     val electResult = client.electLeaders(ElectionType.UNCLEAN, Set(partition1).asJava)
     assertTrue(electResult.partitions.get.get(partition1).get.isInstanceOf[EligibleLeadersNotAvailableException])
@@ -1754,10 +1754,10 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
       servers
     )
 
-    TestUtils.waitForLeaderToBecome(client, partition1, Option(broker1))
+    TestUtils.assertLeader(client, partition1, broker1)
 
     servers(broker1).shutdown()
-    TestUtils.waitForLeaderToBecome(client, partition1, Some(broker2))
+    TestUtils.assertLeader(client, partition1, broker2)
     servers(broker1).startup()
 
     val electResult = client.electLeaders(ElectionType.UNCLEAN, Set(partition1).asJava)
@@ -1786,21 +1786,21 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
       servers
     )
 
-    TestUtils.waitForLeaderToBecome(client, partition1, Option(broker1))
-    TestUtils.waitForLeaderToBecome(client, partition2, Option(broker1))
+    TestUtils.assertLeader(client, partition1, broker1)
+    TestUtils.assertLeader(client, partition2, broker1)
 
     servers(broker2).shutdown()
     TestUtils.waitForBrokersOutOfIsr(client, Set(partition1), Set(broker2))
     servers(broker1).shutdown()
-    TestUtils.waitForLeaderToBecome(client, partition1, None)
-    TestUtils.waitForLeaderToBecome(client, partition2, Some(broker3))
+    TestUtils.assertNoLeader(client, partition1)
+    TestUtils.assertLeader(client, partition2, broker3)
     servers(broker2).startup()
 
     val electResult = client.electLeaders(ElectionType.UNCLEAN, Set(partition1, partition2).asJava)
     assertFalse(electResult.partitions.get.get(partition1).isPresent)
     assertTrue(electResult.partitions.get.get(partition2).get.isInstanceOf[ElectionNotNeededException])
-    assertEquals(Option(broker2), TestUtils.currentLeader(client, partition1))
-    assertEquals(Option(broker3), TestUtils.currentLeader(client, partition2))
+    TestUtils.assertLeader(client, partition1, broker2)
+    TestUtils.assertLeader(client, partition2, broker3)
   }
 
   @Test
@@ -2428,4 +2428,23 @@ object AdminClientIntegrationTest {
 
     assertEquals(Defaults.CompressionType.toString, configs.get(brokerResource).get(KafkaConfig.CompressionTypeProp).value)
   }
+
+  private def getTopicMetadata(client: Admin,
+                               topic: String,
+                               describeOptions: DescribeTopicsOptions = new DescribeTopicsOptions,
+                               expectedNumPartitionsOpt: Option[Int] = None): TopicDescription = {
+    var result: TopicDescription = null
+
+    TestUtils.waitUntilTrue(() => {
+      val topicResult = client.describeTopics(Set(topic).asJava, describeOptions).values.get(topic)
+      try {
+        result = topicResult.get
+        expectedNumPartitionsOpt.map(_ == result.partitions.size).getOrElse(true)
+      } catch {
+        case e: ExecutionException if e.getCause.isInstanceOf[UnknownTopicOrPartitionException] => false  // metadata may not have propagated yet, so retry
+      }
+    }, s"Timed out waiting for metadata for $topic")
+
+    result
+  }
 }
diff --git a/core/src/test/scala/integration/kafka/api/SaslSslAdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/SaslSslAdminClientIntegrationTest.scala
index 2a8131e560f2b..fd4b29f3ff484 100644
--- a/core/src/test/scala/integration/kafka/api/SaslSslAdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SaslSslAdminClientIntegrationTest.scala
@@ -24,7 +24,7 @@ import kafka.utils.TestUtils._
 import org.apache.kafka.clients.admin._
 import org.apache.kafka.common.acl._
 import org.apache.kafka.common.config.ConfigResource
-import org.apache.kafka.common.errors.{ClusterAuthorizationException, InvalidRequestException, TopicAuthorizationException}
+import org.apache.kafka.common.errors.{ClusterAuthorizationException, InvalidRequestException, TopicAuthorizationException, UnknownTopicOrPartitionException}
 import org.apache.kafka.common.resource.{PatternType, ResourcePattern, ResourcePatternFilter, ResourceType}
 import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
 import org.junit.Assert.{assertEquals, assertTrue}
@@ -33,6 +33,7 @@ import org.junit.{After, Assert, Before, Test}
 import scala.collection.JavaConverters._
 import scala.collection.Seq
 import scala.compat.java8.OptionConverters._
+import scala.concurrent.ExecutionException
 import scala.util.{Failure, Success, Try}
 
 class SaslSslAdminClientIntegrationTest extends AdminClientIntegrationTest with SaslSetup {
@@ -448,9 +449,8 @@ class SaslSslAdminClientIntegrationTest extends AdminClientIntegrationTest with
     validateMetadataAndConfigs(createResult)
     val createResponseConfig = createResult.config(topic1).get().entries.asScala
 
-    val topicResource = new ConfigResource(ConfigResource.Type.TOPIC, topic1)
-    val describeResponseConfig = client.describeConfigs(List(topicResource).asJava).values.get(topicResource).get().entries.asScala
-    assertEquals(describeResponseConfig.size, createResponseConfig.size)
+    val describeResponseConfig = describeConfigs(topic1)
+    assertEquals(describeResponseConfig.map(_.name).toSet, createResponseConfig.map(_.name).toSet)
     describeResponseConfig.foreach { describeEntry =>
       val name = describeEntry.name
       val createEntry = createResponseConfig.find(_.name == name).get
@@ -461,6 +461,23 @@ class SaslSslAdminClientIntegrationTest extends AdminClientIntegrationTest with
     }
   }
 
+  private def describeConfigs(topic: String): Iterable[ConfigEntry] = {
+    val topicResource = new ConfigResource(ConfigResource.Type.TOPIC, topic)
+    var configEntries: Iterable[ConfigEntry] = null
+
+    TestUtils.waitUntilTrue(() => {
+      try {
+        val topicResponse = client.describeConfigs(List(topicResource).asJava).all.get.get(topicResource)
+        configEntries = topicResponse.entries.asScala
+        true
+      } catch {
+        case e: ExecutionException if e.getCause.isInstanceOf[UnknownTopicOrPartitionException] => false
+      }
+    }, "Timed out waiting for describeConfigs")
+
+    configEntries
+  }
+
   private def waitForDescribeAcls(client: Admin, filter: AclBindingFilter, acls: Set[AclBinding]): Unit = {
     var lastResults: util.Collection[AclBinding] = null
     TestUtils.waitUntilTrue(() => {
diff --git a/core/src/test/scala/unit/kafka/admin/LeaderElectionCommandTest.scala b/core/src/test/scala/unit/kafka/admin/LeaderElectionCommandTest.scala
index c30011fbb5cd2..45fb7b9ebc3e8 100644
--- a/core/src/test/scala/unit/kafka/admin/LeaderElectionCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/LeaderElectionCommandTest.scala
@@ -80,12 +80,12 @@ final class LeaderElectionCommandTest extends ZooKeeperTestHarness {
 
       val topicPartition = new TopicPartition(topic, partition)
 
-      TestUtils.waitForLeaderToBecome(client, topicPartition, Option(broker2))
+      TestUtils.assertLeader(client, topicPartition, broker2)
 
       servers(broker3).shutdown()
       TestUtils.waitForBrokersOutOfIsr(client, Set(topicPartition), Set(broker3))
       servers(broker2).shutdown()
-      TestUtils.waitForLeaderToBecome(client, topicPartition, None)
+      TestUtils.assertNoLeader(client, topicPartition)
       servers(broker3).startup()
 
       LeaderElectionCommand.main(
@@ -96,7 +96,7 @@ final class LeaderElectionCommandTest extends ZooKeeperTestHarness {
         )
       )
 
-      assertEquals(Option(broker3), TestUtils.currentLeader(client, topicPartition))
+      TestUtils.assertLeader(client, topicPartition, broker3)
     }
   }
 
@@ -111,12 +111,12 @@ final class LeaderElectionCommandTest extends ZooKeeperTestHarness {
 
       val topicPartition = new TopicPartition(topic, partition)
 
-      TestUtils.waitForLeaderToBecome(client, topicPartition, Option(broker2))
+      TestUtils.assertLeader(client, topicPartition, broker2)
 
       servers(broker3).shutdown()
       TestUtils.waitForBrokersOutOfIsr(client, Set(topicPartition), Set(broker3))
       servers(broker2).shutdown()
-      TestUtils.waitForLeaderToBecome(client, topicPartition, None)
+      TestUtils.assertNoLeader(client, topicPartition)
       servers(broker3).startup()
 
       LeaderElectionCommand.main(
@@ -128,7 +128,7 @@ final class LeaderElectionCommandTest extends ZooKeeperTestHarness {
         )
       )
 
-      assertEquals(Option(broker3), TestUtils.currentLeader(client, topicPartition))
+      TestUtils.assertLeader(client, topicPartition, broker3)
     }
   }
 
@@ -143,12 +143,12 @@ final class LeaderElectionCommandTest extends ZooKeeperTestHarness {
 
       val topicPartition = new TopicPartition(topic, partition)
 
-      TestUtils.waitForLeaderToBecome(client, topicPartition, Option(broker2))
+      TestUtils.assertLeader(client, topicPartition, broker2)
 
       servers(broker3).shutdown()
       TestUtils.waitForBrokersOutOfIsr(client, Set(topicPartition), Set(broker3))
       servers(broker2).shutdown()
-      TestUtils.waitForLeaderToBecome(client, topicPartition, None)
+      TestUtils.assertNoLeader(client, topicPartition)
       servers(broker3).startup()
 
       val topicPartitionPath = tempTopicPartitionFile(Set(topicPartition))
@@ -161,7 +161,7 @@ final class LeaderElectionCommandTest extends ZooKeeperTestHarness {
         )
       )
 
-      assertEquals(Option(broker3), TestUtils.currentLeader(client, topicPartition))
+      TestUtils.assertLeader(client, topicPartition, broker3)
     }
   }
 
@@ -176,10 +176,10 @@ final class LeaderElectionCommandTest extends ZooKeeperTestHarness {
 
       val topicPartition = new TopicPartition(topic, partition)
 
-      TestUtils.waitForLeaderToBecome(client, topicPartition, Option(broker2))
+      TestUtils.assertLeader(client, topicPartition, broker2)
 
       servers(broker2).shutdown()
-      TestUtils.waitForLeaderToBecome(client, topicPartition, Some(broker3))
+      TestUtils.assertLeader(client, topicPartition, broker3)
       servers(broker2).startup()
       TestUtils.waitForBrokersInIsr(client, topicPartition, Set(broker2))
 
@@ -191,7 +191,7 @@ final class LeaderElectionCommandTest extends ZooKeeperTestHarness {
         )
       )
 
-      assertEquals(Option(broker2), TestUtils.currentLeader(client, topicPartition))
+      TestUtils.assertLeader(client, topicPartition, broker2)
     }
   }
 
@@ -273,7 +273,7 @@ final class LeaderElectionCommandTest extends ZooKeeperTestHarness {
       LeaderElectionCommand.main(
         Array(
           "--bootstrap-server", bootstrapServers(servers),
-          "--election-type", "preferrred"
+          "--election-type", "preferred"
         )
       )
       fail()
diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index 6df853d8660b0..2e8afe317f0a9 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -28,8 +28,8 @@ import java.util.Arrays
 import java.util.Collections
 import java.util.Properties
 import java.util.concurrent.{Callable, ExecutionException, Executors, TimeUnit}
-import javax.net.ssl.X509TrustManager
 
+import javax.net.ssl.X509TrustManager
 import kafka.api._
 import kafka.cluster.{Broker, EndPoint}
 import kafka.log._
@@ -49,6 +49,7 @@ import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig, Produce
 import org.apache.kafka.common.acl.{AccessControlEntry, AccessControlEntryFilter, AclBindingFilter}
 import org.apache.kafka.common.{KafkaFuture, TopicPartition}
 import org.apache.kafka.common.config.ConfigResource
+import org.apache.kafka.common.errors.UnknownTopicOrPartitionException
 import org.apache.kafka.common.header.Header
 import org.apache.kafka.common.internals.Topic
 import org.apache.kafka.common.network.{ListenerName, Mode}
@@ -1488,24 +1489,27 @@ object TestUtils extends Logging {
     adminClient.alterConfigs(configs)
   }
 
-  def currentLeader(client: Admin, topicPartition: TopicPartition): Option[Int] = {
-    Option(
-      client
-        .describeTopics(Arrays.asList(topicPartition.topic))
-        .all
-        .get
-        .get(topicPartition.topic)
-        .partitions
-        .get(topicPartition.partition)
-        .leader
-    ).map(_.id)
+  def assertLeader(client: Admin, topicPartition: TopicPartition, expectedLeader: Int): Unit = {
+    waitForLeaderToBecome(client, topicPartition, Some(expectedLeader))
+  }
+
+  def assertNoLeader(client: Admin, topicPartition: TopicPartition): Unit = {
+    waitForLeaderToBecome(client, topicPartition, None)
   }
 
   def waitForLeaderToBecome(client: Admin, topicPartition: TopicPartition, leader: Option[Int]): Unit = {
-    TestUtils.waitUntilTrue(
-      () => currentLeader(client, topicPartition) == leader,
-      s"Expected leader to become $leader", 10000
-    )
+    val topic = topicPartition.topic
+    val partition = topicPartition.partition
+
+    TestUtils.waitUntilTrue(() => {
+      try {
+        val topicResult = client.describeTopics(Arrays.asList(topic)).all.get.get(topic)
+        val partitionResult = topicResult.partitions.get(partition)
+        Option(partitionResult.leader).map(_.id) == leader
+      } catch {
+        case e: ExecutionException if e.getCause.isInstanceOf[UnknownTopicOrPartitionException] => false
+      }
+    }, "Timed out waiting for leader metadata")
   }
 
   def waitForBrokersOutOfIsr(client: Admin, partition: Set[TopicPartition], brokerIds: Set[Int]): Unit = {

From 9c3082e6e2867fa56a9613faf1f22fcc9bd41234 Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Fri, 18 Oct 2019 16:23:33 -0700
Subject: [PATCH 0751/1071] MINOR: fix typo in
 TestInputTopic.getTimestampAndAdvance (#7553)

Reviewers: Bill Bejeck , John Roesler , Matthias J. Sax 
---
 .../main/java/org/apache/kafka/streams/TestInputTopic.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TestInputTopic.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TestInputTopic.java
index 50310aa87d0fc..c5966a7e0860f 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TestInputTopic.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TestInputTopic.java
@@ -97,7 +97,7 @@ public void advanceTime(final Duration advance) {
         currentTime = currentTime.plus(advance);
     }
 
-    private Instant getTimestampAndAdvanced() {
+    private Instant getTimestampAndAdvance() {
         final Instant timestamp = currentTime;
         currentTime = currentTime.plus(advanceDuration);
         return timestamp;
@@ -111,7 +111,7 @@ private Instant getTimestampAndAdvanced() {
      */
     public void pipeInput(final TestRecord record) {
         //if record timestamp not set get timestamp and advance
-        final Instant timestamp = (record.getRecordTime() == null) ? getTimestampAndAdvanced() : record.getRecordTime();
+        final Instant timestamp = (record.getRecordTime() == null) ? getTimestampAndAdvance() : record.getRecordTime();
         driver.pipeRecord(topic, record, keySerializer, valueSerializer, timestamp);
     }
 

From b987289bc7223662ede1b132367889b2d378a044 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Fri, 18 Oct 2019 17:32:03 -0700
Subject: [PATCH 0752/1071] MINOR: Add metrics in operations doc for KIP-429 /
 KIP-467 (#7527)

Reviewers: Tu V. Tran , A. Sophie Blee-Goldman , Matthias J. Sax 
---
 docs/ops.html | 90 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 90 insertions(+)

diff --git a/docs/ops.html b/docs/ops.html
index 08967585efab2..5cb1f65b6ed42 100644
--- a/docs/ops.html
+++ b/docs/ops.html
@@ -860,6 +860,26 @@ 
                Security Considerations for Remote Mon
         
                
+      
+        
+        
+        
+      
+      
+        
+        
+        
+      
+      
+        
+        
+        
+      
+      
+        
+        
+        
+      
@@ -1489,11 +1509,81 @@ 
                Consumer
         
                
+      
+        
+        
+        
+      
+      
+        
+        
+        
+      
+      
+        
+        
+        
+      
+      
+        
+        
+        
+      
+      
+        
+        
+        
+      
+      
+        
+        
+        
+      
+      
+        
+        
+        
+      
+      
+        
+        
+        
+      
+      
+        
+        
+        
+      
+      
+        
+        
+        
+      
+      
+        
+        
+        
+      
+      
+        
+        
+        
+      
+      
+        
+        
+        
+      
+      
+        
+        
+        
+      
                
     
                kafka.server:type=BrokerTopicMetrics,name=ReplicationBytesOutPerSec
         
       
                Message validation failure rate due to no key specified for compacted topickafka.server:type=BrokerTopicMetrics,name=NoKeyCompactedTopicRecordsPerSec
                Message validation failure rate due to invalid magic numberkafka.server:type=BrokerTopicMetrics,name=InvalidMagicNumberRecordsPerSec
                Message validation failure rate due to incorrect crc checksumkafka.server:type=BrokerTopicMetrics,name=InvalidMessageCrcRecordsPerSec
                Message validation failure rate due to non-continuous offset or sequence number in batchkafka.server:type=BrokerTopicMetrics,name=InvalidOffsetOrSequenceRecordsPerSec
                
       
                
         Log flush rate and time
         kafka.log:type=LogFlushStats,name=LogFlushRateAndTimeMsThe total number of group syncs
         kafka.consumer:type=consumer-coordinator-metrics,client-id=([-.\w]+)
       
                rebalance-latency-avgThe average time taken for a group rebalancekafka.consumer:type=consumer-coordinator-metrics,client-id=([-.\w]+)
                rebalance-latency-maxThe max time taken for a group rebalancekafka.consumer:type=consumer-coordinator-metrics,client-id=([-.\w]+)
                rebalance-latency-totalThe total time taken for group rebalances so farkafka.consumer:type=consumer-coordinator-metrics,client-id=([-.\w]+)
                rebalance-totalThe total number of group rebalances participatedkafka.consumer:type=consumer-coordinator-metrics,client-id=([-.\w]+)
                rebalance-rate-per-hourThe number of group rebalance participated per hourkafka.consumer:type=consumer-coordinator-metrics,client-id=([-.\w]+)
                failed-rebalance-totalThe total number of failed group rebalanceskafka.consumer:type=consumer-coordinator-metrics,client-id=([-.\w]+)
                failed-rebalance-rate-per-hourThe number of failed group rebalance event per hourkafka.consumer:type=consumer-coordinator-metrics,client-id=([-.\w]+)
                last-rebalance-seconds-agoThe number of seconds since the last rebalance eventkafka.consumer:type=consumer-coordinator-metrics,client-id=([-.\w]+)
                
       
                
         last-heartbeat-seconds-ago
         The number of seconds since the last controller heartbeat
         kafka.consumer:type=consumer-coordinator-metrics,client-id=([-.\w]+)
       
                partitions-revoked-latency-avgThe average time taken by the on-partitions-revoked rebalance listener callbackkafka.consumer:type=consumer-coordinator-metrics,client-id=([-.\w]+)
                partitions-revoked-latency-maxThe max time taken by the on-partitions-revoked rebalance listener callbackkafka.consumer:type=consumer-coordinator-metrics,client-id=([-.\w]+)
                partitions-assigned-latency-avgThe average time taken by the on-partitions-assigned rebalance listener callbackkafka.consumer:type=consumer-coordinator-metrics,client-id=([-.\w]+)
                partitions-assigned-latency-maxThe max time taken by the on-partitions-assigned rebalance listener callbackkafka.consumer:type=consumer-coordinator-metrics,client-id=([-.\w]+)
                partitions-lost-latency-avgThe average time taken by the on-partitions-lost rebalance listener callbackkafka.consumer:type=consumer-coordinator-metrics,client-id=([-.\w]+)
                partitions-lost-latency-maxThe max time taken by the on-partitions-lost rebalance listener callbackkafka.consumer:type=consumer-coordinator-metrics,client-id=([-.\w]+)
                
                     
   
                
 

From dd8f8230a80897d4d5807325c2f7816de9c4a2d3 Mon Sep 17 00:00:00 2001
From: Viktor Somogyi 
Date: Sat, 19 Oct 2019 19:13:19 +0200
Subject: [PATCH 0753/1071] KAFKA-9041; Flaky Test
 LogCleanerIntegrationTest#testIsThreadFailed (#7542)

Aims to fix the flaky LogCleanerIntegrationTest#testIsThreadFailed by changing how metrics are cleaned.

Reviewers: Jason Gustafson 
---
 .../kafka/log/LogCleanerIntegrationTest.scala    | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerIntegrationTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerIntegrationTest.scala
index 56975075e0d30..d148c3f89598b 100644
--- a/core/src/test/scala/unit/kafka/log/LogCleanerIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerIntegrationTest.scala
@@ -27,7 +27,7 @@ import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.record.{CompressionType, RecordBatch}
 import org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS
 import org.junit.Assert._
-import org.junit.Test
+import org.junit.{After, Test}
 
 import scala.collection.JavaConverters.mapAsScalaMapConverter
 import scala.collection.{Iterable, JavaConverters, Seq}
@@ -42,6 +42,11 @@ class LogCleanerIntegrationTest extends AbstractLogCleanerIntegrationTest with K
   val time = new MockTime()
   val topicPartitions = Array(new TopicPartition("log", 0), new TopicPartition("log", 1), new TopicPartition("log", 2))
 
+  @After
+  def cleanup(): Unit = {
+    TestUtils.clearYammerMetrics()
+  }
+
   @Test(timeout = DEFAULT_MAX_WAIT_MS)
   def testMarksPartitionsAsOfflineAndPopulatesUncleanableMetrics(): Unit = {
     val largeMessageKey = 20
@@ -95,7 +100,7 @@ class LogCleanerIntegrationTest extends AbstractLogCleanerIntegrationTest with K
   }
 
   private def getGauge[T](metricName: String): Gauge[T] = {
-    getGauge(_.getName.endsWith(metricName))
+    getGauge(mName => mName.getName.endsWith(metricName) && mName.getScope == null)
   }
 
   private def getGauge[T](metricName: String, metricScope: String): Gauge[T] = {
@@ -196,7 +201,6 @@ class LogCleanerIntegrationTest extends AbstractLogCleanerIntegrationTest with K
   @Test
   def testIsThreadFailed(): Unit = {
     val metricName = "DeadThreadCount"
-    removeMetric(metricName) // remove the existing metric so it will be attached to this object below on creation
     cleaner = makeCleaner(partitions = topicPartitions, maxMessageSize = 100000, backOffMs = 100)
     cleaner.startup()
     assertEquals(0, cleaner.deadThreadCount)
@@ -211,10 +215,4 @@ class LogCleanerIntegrationTest extends AbstractLogCleanerIntegrationTest with K
     assertEquals(cleaner.cleaners.size, getGauge[Int](metricName).value())
     assertEquals(cleaner.cleaners.size, cleaner.deadThreadCount)
   }
-
-  private def removeMetric(name: String): Unit = {
-    val metricName = Metrics.defaultRegistry().allMetrics()
-      .asScala.find(p => p._1.getName.endsWith(name)).get._1
-    Metrics.defaultRegistry().removeMetric(metricName)
-  }
 }

From e9fcfe4fb7b9ae2f537ce355fe2ab51a58034c64 Mon Sep 17 00:00:00 2001
From: Manikumar Reddy 
Date: Mon, 21 Oct 2019 13:18:33 +0530
Subject: [PATCH 0754/1071] KAFKA-8943: Move SecurityProviderCreator to
 org.apache.kafka.common.security.auth package (#7564)

Reviewers: Mickael Maison , Rajini Sivaram 
---
 .../java/org/apache/kafka/common/config/SecurityConfig.java  | 5 +++--
 .../common/security/{ => auth}/SecurityProviderCreator.java  | 4 +++-
 .../java/org/apache/kafka/common/utils/SecurityUtils.java    | 2 +-
 .../ssl/mock/TestPlainSaslServerProviderCreator.java         | 2 +-
 .../kafka/common/security/ssl/mock/TestProviderCreator.java  | 2 +-
 .../ssl/mock/TestScramSaslServerProviderCreator.java         | 2 +-
 .../org/apache/kafka/common/utils/SecurityUtilsTest.java     | 2 +-
 7 files changed, 11 insertions(+), 8 deletions(-)
 rename clients/src/main/java/org/apache/kafka/common/security/{ => auth}/SecurityProviderCreator.java (90%)

diff --git a/clients/src/main/java/org/apache/kafka/common/config/SecurityConfig.java b/clients/src/main/java/org/apache/kafka/common/config/SecurityConfig.java
index 0889aa5f44dea..b4dc26c7ea5cb 100644
--- a/clients/src/main/java/org/apache/kafka/common/config/SecurityConfig.java
+++ b/clients/src/main/java/org/apache/kafka/common/config/SecurityConfig.java
@@ -22,7 +22,8 @@
 public class SecurityConfig {
 
     public static final String SECURITY_PROVIDERS_CONFIG = "security.providers";
-    public static final String SECURITY_PROVIDERS_DOC = "A list of configurable creators each returning a provider " +
-            "implementing security algorithms";
+    public static final String SECURITY_PROVIDERS_DOC = "A list of configurable creator classes each returning a provider" +
+        " implementing security algorithms. These classes should implement the" +
+        " org.apache.kafka.common.security.auth.SecurityProviderCreator interface.";
 
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/security/SecurityProviderCreator.java b/clients/src/main/java/org/apache/kafka/common/security/auth/SecurityProviderCreator.java
similarity index 90%
rename from clients/src/main/java/org/apache/kafka/common/security/SecurityProviderCreator.java
rename to clients/src/main/java/org/apache/kafka/common/security/auth/SecurityProviderCreator.java
index e5f3a7c30ad0b..ae56f9a3520a3 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/SecurityProviderCreator.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/auth/SecurityProviderCreator.java
@@ -14,9 +14,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.kafka.common.security;
+package org.apache.kafka.common.security.auth;
 
 import org.apache.kafka.common.Configurable;
+import org.apache.kafka.common.annotation.InterfaceStability;
 
 import java.security.Provider;
 import java.util.Map;
@@ -24,6 +25,7 @@
 /**
  * An interface for generating security providers.
  */
+@InterfaceStability.Evolving
 public interface SecurityProviderCreator extends Configurable {
 
     /**
diff --git a/clients/src/main/java/org/apache/kafka/common/utils/SecurityUtils.java b/clients/src/main/java/org/apache/kafka/common/utils/SecurityUtils.java
index d6595ef9e8417..09c97be15a31c 100644
--- a/clients/src/main/java/org/apache/kafka/common/utils/SecurityUtils.java
+++ b/clients/src/main/java/org/apache/kafka/common/utils/SecurityUtils.java
@@ -19,7 +19,7 @@
 import org.apache.kafka.common.acl.AclOperation;
 import org.apache.kafka.common.config.SecurityConfig;
 import org.apache.kafka.common.resource.ResourceType;
-import org.apache.kafka.common.security.SecurityProviderCreator;
+import org.apache.kafka.common.security.auth.SecurityProviderCreator;
 import org.apache.kafka.common.security.auth.KafkaPrincipal;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/clients/src/test/java/org/apache/kafka/common/security/ssl/mock/TestPlainSaslServerProviderCreator.java b/clients/src/test/java/org/apache/kafka/common/security/ssl/mock/TestPlainSaslServerProviderCreator.java
index fa476a5f12df2..0ab927eb392ce 100644
--- a/clients/src/test/java/org/apache/kafka/common/security/ssl/mock/TestPlainSaslServerProviderCreator.java
+++ b/clients/src/test/java/org/apache/kafka/common/security/ssl/mock/TestPlainSaslServerProviderCreator.java
@@ -16,7 +16,7 @@
  */
 package org.apache.kafka.common.security.ssl.mock;
 
-import org.apache.kafka.common.security.SecurityProviderCreator;
+import org.apache.kafka.common.security.auth.SecurityProviderCreator;
 
 import java.security.Provider;
 
diff --git a/clients/src/test/java/org/apache/kafka/common/security/ssl/mock/TestProviderCreator.java b/clients/src/test/java/org/apache/kafka/common/security/ssl/mock/TestProviderCreator.java
index 5f4929f058036..57c455a3e3c69 100644
--- a/clients/src/test/java/org/apache/kafka/common/security/ssl/mock/TestProviderCreator.java
+++ b/clients/src/test/java/org/apache/kafka/common/security/ssl/mock/TestProviderCreator.java
@@ -16,7 +16,7 @@
  */
 package org.apache.kafka.common.security.ssl.mock;
 
-import org.apache.kafka.common.security.SecurityProviderCreator;
+import org.apache.kafka.common.security.auth.SecurityProviderCreator;
 
 import java.security.Provider;
 
diff --git a/clients/src/test/java/org/apache/kafka/common/security/ssl/mock/TestScramSaslServerProviderCreator.java b/clients/src/test/java/org/apache/kafka/common/security/ssl/mock/TestScramSaslServerProviderCreator.java
index f03f8c96a275a..72eb8800d6089 100644
--- a/clients/src/test/java/org/apache/kafka/common/security/ssl/mock/TestScramSaslServerProviderCreator.java
+++ b/clients/src/test/java/org/apache/kafka/common/security/ssl/mock/TestScramSaslServerProviderCreator.java
@@ -16,7 +16,7 @@
  */
 package org.apache.kafka.common.security.ssl.mock;
 
-import org.apache.kafka.common.security.SecurityProviderCreator;
+import org.apache.kafka.common.security.auth.SecurityProviderCreator;
 
 import java.security.Provider;
 
diff --git a/clients/src/test/java/org/apache/kafka/common/utils/SecurityUtilsTest.java b/clients/src/test/java/org/apache/kafka/common/utils/SecurityUtilsTest.java
index 903ac5c9dd79f..d182a00c21c2c 100644
--- a/clients/src/test/java/org/apache/kafka/common/utils/SecurityUtilsTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/utils/SecurityUtilsTest.java
@@ -17,7 +17,7 @@
 package org.apache.kafka.common.utils;
 
 import org.apache.kafka.common.config.SecurityConfig;
-import org.apache.kafka.common.security.SecurityProviderCreator;
+import org.apache.kafka.common.security.auth.SecurityProviderCreator;
 import org.apache.kafka.common.security.auth.KafkaPrincipal;
 import org.apache.kafka.common.security.ssl.mock.TestPlainSaslServerProviderCreator;
 import org.apache.kafka.common.security.ssl.mock.TestScramSaslServerProviderCreator;

From 24ed6c65916d0a32661400d4d07656ad115b74c4 Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Mon, 21 Oct 2019 16:25:41 -0500
Subject: [PATCH 0755/1071] MINOR: don't require key serde in join materialized
 (#7557)

Reviewers: Bill Bejeck , Guozhang Wang , Matthias J. Sax 
---
 .../apache/kafka/streams/kstream/internals/KTableImpl.java   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
index 301710ddb4400..f6f2ada6ce5e4 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
@@ -712,7 +712,10 @@ private  KTable doJoin(final KTable other,
         final StoreBuilder> storeBuilder;
 
         if (materializedInternal != null) {
-            keySerde = materializedInternal.keySerde() != null ? materializedInternal.keySerde() : this.keySerde;
+            if (materializedInternal.keySerde() == null) {
+                materializedInternal.withKeySerde(this.keySerde);
+            }
+            keySerde = materializedInternal.keySerde();
             valueSerde = materializedInternal.valueSerde();
             queryableStoreName = materializedInternal.storeName();
             storeBuilder = new TimestampedKeyValueStoreMaterializer<>(materializedInternal).materialize();

From 860982e1d674e9957f1078f6e5ddb63067d3b971 Mon Sep 17 00:00:00 2001
From: Bill Bejeck 
Date: Mon, 21 Oct 2019 18:42:08 -0400
Subject: [PATCH 0756/1071] MINOR: KIP-307 upgrade guide docs (#7547)

Reviewers: John Roesler , Matthias J. Sax 
---
 docs/streams/upgrade-guide.html | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/docs/streams/upgrade-guide.html b/docs/streams/upgrade-guide.html
index 2de79172a40a7..8f4948e196fdd 100644
--- a/docs/streams/upgrade-guide.html
+++ b/docs/streams/upgrade-guide.html
@@ -75,7 +75,20 @@ 
                Upgrade Guide and API Changes
                
     
                Streams API changes in 2.4.0
                
 
     
-    
+    
                
+        In the 2.4 release, you now can name all operators in a Kafka Streams DSL topology via
+        KIP-307
                . 
+        Giving your operators meaningful names makes it easier to understand the topology 
+        description (Topology#describe()#toString()) and 
+        understand the full context of what your Kafka Streams application is doing.
+        
                
+        There are new overloads on most KStream and KTable methods 
+        that accept a Named object.  Typically you'll provide a name for the DSL operation by 
+        using Named.as("my operator name").  Naming of repartition topics for aggregation 
+        operations will still use Grouped and join operations will use 
+        either Joined or the new StreamJoined object.
+
+    
                
     
 
     
                

From 65dc4f3df5bde8452ac8e5f7521c09f5732b71b1 Mon Sep 17 00:00:00 2001
From: Bill Bejeck 
Date: Mon, 21 Oct 2019 20:27:23 -0400
Subject: [PATCH 0757/1071] MINOR:Upgrade guide updates for KIP-479 (#7550)

Reviewers: A. Sophie Blee-Goldmann , John Roesler , Matthias J. Sax 
---
 docs/streams/upgrade-guide.html | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/docs/streams/upgrade-guide.html b/docs/streams/upgrade-guide.html
index 8f4948e196fdd..04ec73444bd6f 100644
--- a/docs/streams/upgrade-guide.html
+++ b/docs/streams/upgrade-guide.html
@@ -77,7 +77,7 @@ 
                Streams API
     
     
                
         In the 2.4 release, you now can name all operators in a Kafka Streams DSL topology via
-        KIP-307
                . 
+        KIP-307.
         Giving your operators meaningful names makes it easier to understand the topology 
         description (Topology#describe()#toString()) and 
         understand the full context of what your Kafka Streams application is doing.
@@ -89,7 +89,24 @@ 
                Streams API
         either Joined or the new StreamJoined object.
 
     
                
-    
+    
                
+        Before the 2.4.0 version of Kafa Streams, users of the DSL could not name the state stores involved in a stream-stream join.
+        If users changed their topology and added a operator before the
+        join, the internal names of the state stores would shift, requiring an application reset when redeploying.
+        In the 2.4.0 release, Kafka Streams adds the StreamJoined
+        class, which gives users the ability to name the join processor, repartition topic(s) (if a repartition is required),
+        and the state stores involved in the join.  Also, by naming the state stores, the changelog topics
+        backing the state stores are named as well.  It's important to note that naming the stores
+        will not make them queryable via Interactive Queries.
+        
                
+        Another feature delivered by StreamJoined is that you can now configure the type of state store used in the join.
+        You can elect to use in-memory stores or custom state stores for a stream-stream join.  Note that the provided stores
+        will not be available for querying via Interactive Queries.  With the addition
+        of StreamJoined, stream-stream join operations
+        using Joined have been deprecated. Please switch over to stream-stream join methods using the
+        new overloaded methods.  You can get more details from
+                        KIP-479.
+    
                
 
     
                
         With the introduction of incremental cooperative rebalancing, Streams no longer requires all tasks be revoked at the beginning of a rebalance. Instead, at the completion of the rebalance only those tasks which are to be migrated to another consumer

From cecb1e2fa583d40fb42231a730df6fce1e7d19c3 Mon Sep 17 00:00:00 2001
From: Manikumar Reddy 
Date: Tue, 22 Oct 2019 20:51:10 +0530
Subject: [PATCH 0758/1071] MINOR: Add upgrade docs for 2.4.0 release

Author: Manikumar Reddy 

Reviewers: Reviewers: Manikumar Reddy 

Closes #7530 from omkreddy/2.4-upgrade-docs

(cherry picked from commit f46eb292f5be892fbfcd6551d57c5a425c514673)
Signed-off-by: Manikumar Reddy 
---
 docs/upgrade.html | 42 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/docs/upgrade.html b/docs/upgrade.html
index 6d9f7020b5d9f..65c7ba96a1c06 100644
--- a/docs/upgrade.html
+++ b/docs/upgrade.html
@@ -19,6 +19,48 @@
 
 
+
+
+
+
+
+
                
+	
+	
                
+		
+		
+		
                
+	
                
+
                
+
+
+
+
+
+

From 446f787dfc2073d1490df3d76c19db719e3db7fc Mon Sep 17 00:00:00 2001
From: Lucas Bradstreet 
Date: Thu, 14 Nov 2019 13:47:10 +0530
Subject: [PATCH 0798/1071] MINOR: fix flaky ConsumerBounceTest.testClose

Fixes `java.util.concurrent.ExecutionException: java.lang.AssertionError: Close finished too quickly 5999`.

The close test sets a close duration in milliseconds, but measures the time taken in nanoseconds. This leads to small error due to the resolution in each, where the close is deemed to have taken too little time.

When I measured the start and end with nanoTime, I found the time taken to close was `5999641566 ns (5999.6ms)` which seems close enough to be a resolution error. I've run the test 50 times and have not hit the "Close finished too quickly" issue again, whereas previously I hit a failure pretty quickly.

Author: Lucas Bradstreet 

Reviewers: Ismael Juma 

Closes #7683 from lbradstreet/flaky-consumer-bounce-test

(cherry picked from commit 2bd6b6ff0f86d588fc09f96973a469989b9693df)
Signed-off-by: Manikumar Reddy 
---
 .../test/scala/integration/kafka/api/ConsumerBounceTest.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
index e355400fccc58..6c3201b5dc0a4 100644
--- a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
+++ b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
@@ -457,10 +457,10 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
       closeTimeoutMs: Long, minCloseTimeMs: Option[Long], maxCloseTimeMs: Option[Long]): Future[Any] = {
     executor.submit(CoreUtils.runnable {
       val closeGraceTimeMs = 2000
-      val startNanos = System.nanoTime
+      val startMs = System.currentTimeMillis()
       info("Closing consumer with timeout " + closeTimeoutMs + " ms.")
       consumer.close(time.Duration.ofMillis(closeTimeoutMs))
-      val timeTakenMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime - startNanos)
+      val timeTakenMs = System.currentTimeMillis() - startMs
       maxCloseTimeMs.foreach { ms =>
         assertTrue("Close took too long " + timeTakenMs, timeTakenMs < ms + closeGraceTimeMs)
       }

From 970c7ee9db941a9804763f61d4e5a6c76c873de2 Mon Sep 17 00:00:00 2001
From: Chris Egerton 
Date: Thu, 14 Nov 2019 14:09:04 +0530
Subject: [PATCH 0799/1071] KAFKA-9046: Use top-level worker configs for
 connector admin clients

[Jira](https://issues.apache.org/jira/browse/KAFKA-9046)

The changes here are meant to find a healthy compromise between the pre- and post-KIP-458 functionality of Connect workers when configuring admin clients for use with DLQs. Before KIP-458, admin clients were configured using the top-level worker configs; after KIP-458, they are configured using worker configs with a prefix of `admin.` and then optionally overridden by connector configs with a prefix of `admin.override.`. The behavior proposed here is to use, in ascending order of precedence, the top-level worker configs, worker configs prefixed with `admin.`, and connector configs prefixed with `admin.override.`; essentially, use the pre-KIP-458 behavior by default but allow it to be overridden by the post-KIP-458 behavior.

Author: Chris Egerton 

Reviewers: Konstantine Karantasis , Randall Hauch , Nigel Liang 

Closes #7525 from C0urante/kafka-9046

(cherry picked from commit 38d243b022336ecaf5cb400ae015c485f56ff978)
Signed-off-by: Manikumar Reddy 
---
 .../apache/kafka/connect/runtime/Worker.java   | 18 ++++++++++++++++--
 .../kafka/connect/runtime/WorkerTest.java      |  7 ++++---
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
index fbece7941b0a5..814c750b35c96 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.connect.runtime;
 
+import org.apache.kafka.clients.admin.AdminClientConfig;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.clients.consumer.KafkaConsumer;
 import org.apache.kafka.clients.producer.KafkaProducer;
@@ -607,8 +608,21 @@ static Map adminConfigs(ConnectorTaskId id,
                                             Class connectorClass,
                                             ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy) {
         Map adminProps = new HashMap<>();
-        adminProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, Utils.join(config.getList(WorkerConfig.BOOTSTRAP_SERVERS_CONFIG), ","));
-        // User-specified overrides
+        // Use the top-level worker configs to retain backwards compatibility with older releases which
+        // did not require a prefix for connector admin client configs in the worker configuration file
+        // Ignore configs that begin with "admin." since those will be added next (with the prefix stripped)
+        // and those that begin with "producer." and "consumer.", since we know they aren't intended for
+        // the admin client
+        Map nonPrefixedWorkerConfigs = config.originals().entrySet().stream()
+            .filter(e -> !e.getKey().startsWith("admin.")
+                && !e.getKey().startsWith("producer.")
+                && !e.getKey().startsWith("consumer."))
+            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+        adminProps.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG,
+            Utils.join(config.getList(WorkerConfig.BOOTSTRAP_SERVERS_CONFIG), ","));
+        adminProps.putAll(nonPrefixedWorkerConfigs);
+
+        // Admin client-specific overrides in the worker config
         adminProps.putAll(config.originalsWithPrefix("admin."));
 
         // Connector-specified overrides
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java
index 36a9b66d7cf04..7021503adbed6 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java
@@ -1138,12 +1138,15 @@ public void testAdminConfigsClientOverridesWithAllPolicy() {
         Map props = new HashMap<>(workerProps);
         props.put("admin.client.id", "testid");
         props.put("admin.metadata.max.age.ms", "5000");
+        props.put("producer.bootstrap.servers", "cbeauho.com");
+        props.put("consumer.bootstrap.servers", "localhost:4761");
         WorkerConfig configWithOverrides = new StandaloneConfig(props);
 
         Map connConfig = new HashMap();
         connConfig.put("metadata.max.age.ms", "10000");
 
-        Map expectedConfigs = new HashMap<>();
+        Map expectedConfigs = new HashMap<>(workerProps);
+
         expectedConfigs.put("bootstrap.servers", "localhost:9092");
         expectedConfigs.put("client.id", "testid");
         expectedConfigs.put("metadata.max.age.ms", "10000");
@@ -1153,7 +1156,6 @@ public void testAdminConfigsClientOverridesWithAllPolicy() {
         PowerMock.replayAll();
         assertEquals(expectedConfigs, Worker.adminConfigs(new ConnectorTaskId("test", 1), configWithOverrides, connectorConfig,
                                                              null, allConnectorClientConfigOverridePolicy));
-
     }
 
     @Test(expected = ConnectException.class)
@@ -1171,7 +1173,6 @@ public void testAdminConfigsClientOverridesWithNonePolicy() {
         PowerMock.replayAll();
         Worker.adminConfigs(new ConnectorTaskId("test", 1), configWithOverrides, connectorConfig,
                                                           null, noneConnectorClientConfigOverridePolicy);
-
     }
 
     private void assertStatusMetrics(long expected, String metricName) {

From c09c2175f67fb15bdadf8eb2fd1b5ac860c0e545 Mon Sep 17 00:00:00 2001
From: Arvind Thirunarayanan 
Date: Sat, 16 Nov 2019 14:38:35 +0530
Subject: [PATCH 0800/1071] MINOR: add dependency analysis debugging tasks

Author: Arvind Thirunarayanan 

Reviewers: Colin Patrick McCabe 

Closes #7694 from arvindth/ath-debugdeps-task

(cherry picked from commit 255fcdc2369dccb03c91b14b99a177bce6e966ea)
Signed-off-by: Manikumar Reddy 
---
 README.md    | 12 ++++++++++++
 build.gradle |  8 ++++++++
 2 files changed, 20 insertions(+)

diff --git a/README.md b/README.md
index 308d9df8c98c7..76fbc4e403210 100644
--- a/README.md
+++ b/README.md
@@ -200,6 +200,18 @@ The following options should be set with a `-P` switch, for example `./gradlew -
 * `testLoggingEvents`: unit test events to be logged, separated by comma. For example `./gradlew -PtestLoggingEvents=started,passed,skipped,failed test`.
 * `xmlSpotBugsReport`: enable XML reports for spotBugs. This also disables HTML reports as only one can be enabled at a time.
 
+### Dependency Analysis ###
+
+The gradle [dependency debugging documentation](https://docs.gradle.org/current/userguide/viewing_debugging_dependencies.html) mentions using the `dependencies` or `dependencyInsight` tasks to debug dependencies for the root project or individual subprojects.
+
+Alternatively, use the `allDeps` or `allDepInsight` tasks for recursively iterating through all subprojects:
+
+    ./gradlew allDeps
+
+    ./gradlew allDepInsight --configuration runtime --dependency com.fasterxml.jackson.core:jackson-databind
+
+These take the same arguments as the builtin variants.
+
 ### Running system tests ###
 
 See [tests/README.md](tests/README.md).
diff --git a/build.gradle b/build.gradle
index fbf58241ea0ac..c55a2af4c8d55 100644
--- a/build.gradle
+++ b/build.gradle
@@ -143,6 +143,14 @@ if (file('.git').exists()) {
 
 
 subprojects {
+
+  // enable running :dependencies task recursively on all subprojects
+  // eg: ./gradlew allDeps
+  task allDeps(type: DependencyReportTask) {}
+  // enable running :dependencyInsight task recursively on all subprojects
+  // eg: ./gradlew allDepInsight --configuration runtime --dependency com.fasterxml.jackson.core:jackson-databind
+  task allDepInsight(type: DependencyInsightReportTask) doLast {}
+
   apply plugin: 'java'
   // apply the eclipse plugin only to subprojects that hold code. 'connect' is just a folder.
   if (!project.name.equals('connect')) {

From 221d66774ced8f3d9360bcd138ff3a8553cf1919 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Sun, 17 Nov 2019 14:42:00 -0800
Subject: [PATCH 0801/1071] KAFKA-9196; Update high watermark metadata after
 segment roll (#7695)

When we roll a new segment, the log offset metadata tied to the high watermark may
need to be updated. This is needed when the high watermark is equal to the log end
offset at the time of the roll. Otherwise, we risk exposing uncommitted data early.

Reviewers: Dhruvil Shah , Ismael Juma 
---
 core/src/main/scala/kafka/log/Log.scala       |  8 +++--
 .../test/scala/unit/kafka/log/LogTest.scala   | 36 +++++++++++++++++++
 2 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index bf0486ad8b0d5..6514aa2038c23 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -744,9 +744,9 @@ class Log(@volatile var dir: File,
   private def updateLogEndOffset(messageOffset: Long): Unit = {
     nextOffsetMetadata = LogOffsetMetadata(messageOffset, activeSegment.baseOffset, activeSegment.size)
 
-    // Update the high watermark in case it has gotten ahead of the log end offset
-    // following a truncation.
-    if (highWatermark > messageOffset) {
+    // Update the high watermark in case it has gotten ahead of the log end offset following a truncation
+    // or if a new segment has been rolled and the offset metadata needs to be updated.
+    if (highWatermark >= messageOffset) {
       updateHighWatermarkMetadata(nextOffsetMetadata)
     }
   }
@@ -1912,9 +1912,11 @@ class Log(@volatile var dir: File,
           initFileSize = initFileSize,
           preallocate = config.preallocate)
         addSegment(segment)
+
         // We need to update the segment base offset and append position data of the metadata when log rolls.
         // The next offset should not change.
         updateLogEndOffset(nextOffsetMetadata.messageOffset)
+
         // schedule an asynchronous flush of the old segment
         scheduler.schedule("flush-log", () => flush(newOffset), delay = 0L)
 
diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala
index 9a69e00a527e2..a948cf331b164 100755
--- a/core/src/test/scala/unit/kafka/log/LogTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogTest.scala
@@ -77,6 +77,42 @@ class LogTest {
     }
   }
 
+  @Test
+  def testHighWatermarkMetadataUpdatedAfterSegmentRoll(): Unit = {
+    val logConfig = LogTest.createLogConfig(segmentBytes = 1024 * 1024)
+    val log = createLog(logDir, logConfig)
+
+    def assertFetchSizeAndOffsets(fetchOffset: Long,
+                                  expectedSize: Int,
+                                  expectedOffsets: Seq[Long]): Unit = {
+      val readInfo = log.read(
+        startOffset = fetchOffset,
+        maxLength = 2048,
+        isolation = FetchHighWatermark,
+        minOneMessage = false)
+      assertEquals(expectedSize, readInfo.records.sizeInBytes)
+      assertEquals(expectedOffsets, readInfo.records.records.asScala.map(_.offset))
+    }
+
+    val records = TestUtils.records(List(
+      new SimpleRecord(mockTime.milliseconds, "a".getBytes, "value".getBytes),
+      new SimpleRecord(mockTime.milliseconds, "b".getBytes, "value".getBytes),
+      new SimpleRecord(mockTime.milliseconds, "c".getBytes, "value".getBytes)
+    ))
+
+    log.appendAsLeader(records, leaderEpoch = 0)
+    assertFetchSizeAndOffsets(fetchOffset = 0L, 0, Seq())
+
+    log.maybeIncrementHighWatermark(log.logEndOffsetMetadata)
+    assertFetchSizeAndOffsets(fetchOffset = 0L, records.sizeInBytes, Seq(0, 1, 2))
+
+    log.roll()
+    assertFetchSizeAndOffsets(fetchOffset = 0L, records.sizeInBytes, Seq(0, 1, 2))
+
+    log.appendAsLeader(records, leaderEpoch = 0)
+    assertFetchSizeAndOffsets(fetchOffset = 3L, 0, Seq())
+  }
+
   @Test
   def testHighWatermarkMaintenance(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 1024 * 1024)

From 28af3e30c1d88be567537a2d400adca29c8e21dc Mon Sep 17 00:00:00 2001
From: Lucas Bradstreet 
Date: Sun, 17 Nov 2019 19:18:20 -0800
Subject: [PATCH 0802/1071] KAFKA-9200: ListOffsetRequest missing error
 response for v5 (#7704)

ListOffsetResponse getErrorResponse is missing a a case for version 5, introduced
by 152292994e4 and released in 2.3.0.

```
java.lang.IllegalArgumentException: Version 5 is not valid. Valid versions for ListOffsetRequest are 0 to 5
        at org.apache.kafka.common.requests.ListOffsetRequest.getErrorResponse(ListOffsetRequest.java:282)
        at kafka.server.KafkaApis.sendErrorOrCloseConnection(KafkaApis.scala:3062)
        at kafka.server.KafkaApis.sendErrorResponseMaybeThrottle(KafkaApis.scala:3045)
        at kafka.server.KafkaApis.handleError(KafkaApis.scala:3027)
        at kafka.server.KafkaApis.handle(KafkaApis.scala:209)
        at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:78)
        at java.lang.Thread.run(Thread.java:748)
```

Reviewers: Ismael Juma 
---
 .../kafka/common/requests/ListOffsetRequest.java  |  3 ++-
 .../common/requests/RequestResponseTest.java      | 15 +++++++--------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java
index e9fe942e8581a..47c47d2968dc0 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java
@@ -269,12 +269,13 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
             responseData.put(partition, partitionError);
         }
 
-        switch (version()) {
+        switch (versionId) {
             case 0:
             case 1:
             case 2:
             case 3:
             case 4:
+            case 5:
                 return new ListOffsetResponse(throttleTimeMs, responseData);
             default:
                 throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index c8bd7518c67c9..5934dc5ec7584 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -209,12 +209,11 @@ public void testSerialization() throws Exception {
         checkRequest(createDeleteGroupsRequest(), true);
         checkErrorResponse(createDeleteGroupsRequest(), new UnknownServerException(), true);
         checkResponse(createDeleteGroupsResponse(), 0, true);
-        checkRequest(createListOffsetRequest(1), true);
-        checkErrorResponse(createListOffsetRequest(1), new UnknownServerException(), true);
-        checkResponse(createListOffsetResponse(1), 1, true);
-        checkRequest(createListOffsetRequest(2), true);
-        checkErrorResponse(createListOffsetRequest(2), new UnknownServerException(), true);
-        checkResponse(createListOffsetResponse(2), 2, true);
+        for (int i = 0; i < ApiKeys.LIST_OFFSETS.latestVersion(); i++) {
+            checkRequest(createListOffsetRequest(i), true);
+            checkErrorResponse(createListOffsetRequest(i), new UnknownServerException(), true);
+            checkResponse(createListOffsetResponse(i), i, true);
+        }
         checkRequest(MetadataRequest.Builder.allTopics().build((short) 2), true);
         checkRequest(createMetadataRequest(1, Collections.singletonList("topic1")), true);
         checkErrorResponse(createMetadataRequest(1, Collections.singletonList("topic1")), new UnknownServerException(), true);
@@ -1096,7 +1095,7 @@ private ListOffsetRequest createListOffsetRequest(int version) {
                     .forConsumer(true, IsolationLevel.READ_UNCOMMITTED)
                     .setTargetTimes(offsetData)
                     .build((short) version);
-        } else if (version == 2) {
+        } else if (version >= 2 && version <= 5) {
             Map offsetData = Collections.singletonMap(
                     new TopicPartition("test", 0),
                     new ListOffsetRequest.PartitionData(1000000L, Optional.of(5)));
@@ -1116,7 +1115,7 @@ private ListOffsetResponse createListOffsetResponse(int version) {
             responseData.put(new TopicPartition("test", 0),
                     new ListOffsetResponse.PartitionData(Errors.NONE, asList(100L)));
             return new ListOffsetResponse(responseData);
-        } else if (version == 1 || version == 2) {
+        } else if (version >= 1 && version <= 5) {
             Map responseData = new HashMap<>();
             responseData.put(new TopicPartition("test", 0),
                     new ListOffsetResponse.PartitionData(Errors.NONE, 10000L, 100L, Optional.of(27)));

From 4c277b61613f7329e5425dd07ac198f9b528a4a5 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Wed, 6 Nov 2019 09:22:47 -0800
Subject: [PATCH 0803/1071] MINOR: Fetch only from leader should be respected
 in purgatory (#7650)

In #7361, we inadvertently reverted a change to enforce leader only fetching for old versions of the protocol. This patch fixes the problem and adds a new test case to cover fetches which hit purgatory.

Reviewers: Viktor Somogyi , David Arthur 
---
 .../kafka/server/ReplicaManagerTest.scala     | 242 +++++++++++++-----
 1 file changed, 173 insertions(+), 69 deletions(-)

diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index ef05fa1499c5f..ab31cfb9810f4 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -19,13 +19,12 @@ package kafka.server
 
 import java.io.File
 import java.net.InetAddress
-import java.util.concurrent.atomic.AtomicBoolean
+import java.util.concurrent.atomic.{AtomicBoolean, AtomicReference}
 import java.util.concurrent.{CountDownLatch, TimeUnit}
 import java.util.{Optional, Properties}
 
 import kafka.api.Request
 import kafka.log.{Log, LogConfig, LogManager, ProducerStateManager}
-import kafka.utils.TestUtils
 import kafka.cluster.BrokerEndPoint
 import kafka.server.QuotaFactory.UnboundedQuota
 import kafka.server.checkpoints.LazyOffsetCheckpoints
@@ -51,7 +50,6 @@ import org.apache.kafka.common.{Node, TopicPartition}
 import org.easymock.EasyMock
 import org.junit.Assert._
 import org.junit.{After, Before, Test}
-import org.mockito.ArgumentMatchers
 
 import scala.collection.JavaConverters._
 import scala.collection.{Map, Seq}
@@ -763,7 +761,7 @@ class ReplicaManagerTest {
     val countDownLatch = new CountDownLatch(1)
 
     // Prepare the mocked components for the test
-    val (replicaManager, mockLogMgr) = prepareReplicaManagerAndLogManager(
+    val (replicaManager, _) = prepareReplicaManagerAndLogManager(
       topicPartition, leaderEpoch + leaderEpochIncrement, followerBrokerId,
       leaderBrokerId, countDownLatch, expectTruncation = true)
 
@@ -800,7 +798,7 @@ class ReplicaManagerTest {
     val countDownLatch = new CountDownLatch(1)
 
     // Prepare the mocked components for the test
-    val (replicaManager, mockLogMgr) = prepareReplicaManagerAndLogManager(
+    val (replicaManager, _) = prepareReplicaManagerAndLogManager(
       topicPartition, leaderEpoch + leaderEpochIncrement, followerBrokerId,
       leaderBrokerId, countDownLatch, expectTruncation = true)
 
@@ -849,7 +847,7 @@ class ReplicaManagerTest {
     val countDownLatch = new CountDownLatch(1)
 
     // Prepare the mocked components for the test
-    val (replicaManager, mockLogMgr) = prepareReplicaManagerAndLogManager(
+    val (replicaManager, _) = prepareReplicaManagerAndLogManager(
       topicPartition, leaderEpoch + leaderEpochIncrement, followerBrokerId,
       leaderBrokerId, countDownLatch, expectTruncation = true)
 
@@ -899,7 +897,7 @@ class ReplicaManagerTest {
 
     val props = new Properties()
     props.put(KafkaConfig.ReplicaSelectorClassProp, "non-a-class")
-    val (replicaManager, mockLogMgr) = prepareReplicaManagerAndLogManager(
+    prepareReplicaManagerAndLogManager(
       topicPartition, leaderEpoch + leaderEpochIncrement, followerBrokerId,
       leaderBrokerId, countDownLatch, expectTruncation = true, extraProps = props)
   }
@@ -913,102 +911,208 @@ class ReplicaManagerTest {
     val leaderEpochIncrement = 2
     val countDownLatch = new CountDownLatch(1)
 
-    val (replicaManager, mockLogMgr) = prepareReplicaManagerAndLogManager(
+    val (replicaManager, _) = prepareReplicaManagerAndLogManager(
       topicPartition, leaderEpoch + leaderEpochIncrement, followerBrokerId,
       leaderBrokerId, countDownLatch, expectTruncation = true)
     assertFalse(replicaManager.replicaSelectorOpt.isDefined)
   }
 
   @Test
-  def testOlderClientFetchFromLeaderOnly(): Unit = {
+  def testFetchFollowerNotAllowedForOlderClients(): Unit = {
     val replicaManager = setupReplicaManagerWithMockedPurgatories(new MockTimer, aliveBrokerIds = Seq(0, 1))
 
     val tp0 = new TopicPartition(topic, 0)
     val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
     replicaManager.createPartition(tp0).createLogIfNotExists(0, isNew = false, isFutureReplica = false, offsetCheckpoints)
     val partition0Replicas = Seq[Integer](0, 1).asJava
-    val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
-      Seq(
-        new LeaderAndIsrPartitionState()
-          .setTopicName(tp0.topic)
-          .setPartitionIndex(tp0.partition)
-          .setControllerEpoch(0)
-          .setLeader(1)
-          .setLeaderEpoch(0)
-          .setIsr(partition0Replicas)
-          .setZkVersion(0)
-          .setReplicas(partition0Replicas)
-          .setIsNew(true)).asJava,
+    val becomeFollowerRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+      Seq(new LeaderAndIsrPartitionState()
+        .setTopicName(tp0.topic)
+        .setPartitionIndex(tp0.partition)
+        .setControllerEpoch(0)
+        .setLeader(1)
+        .setLeaderEpoch(0)
+        .setIsr(partition0Replicas)
+        .setZkVersion(0)
+        .setReplicas(partition0Replicas)
+        .setIsNew(true)).asJava,
       Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
-    replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest, (_, _) => ())
-
-    def doFetch(replicaId: Int, partitionData: FetchRequest.PartitionData, clientMetadataOpt: Option[ClientMetadata]):
-        Option[FetchPartitionData] = {
-      var fetchResult: Option[FetchPartitionData] = None
-      def callback(response: Seq[(TopicPartition, FetchPartitionData)]): Unit = {
-        fetchResult = response.headOption.filter(_._1 == tp0).map(_._2)
-      }
-      replicaManager.fetchMessages(
-        timeout = 0L,
-        replicaId = replicaId,
-        fetchMinBytes = 1,
-        fetchMaxBytes = 100,
-        hardMaxBytesLimit = false,
-        fetchInfos = Seq(tp0 -> partitionData),
-        quota = UnboundedQuota,
-        isolationLevel = IsolationLevel.READ_UNCOMMITTED,
-        responseCallback = callback,
-        clientMetadata = clientMetadataOpt
-      )
-      fetchResult
-    }
+    replicaManager.becomeLeaderOrFollower(0, becomeFollowerRequest, (_, _) => ())
 
     // Fetch from follower, with non-empty ClientMetadata (FetchRequest v11+)
     val clientMetadata = new DefaultClientMetadata("", "", null, KafkaPrincipal.ANONYMOUS, "")
     var partitionData = new FetchRequest.PartitionData(0L, 0L, 100,
       Optional.of(0))
-    var fetchResult = doFetch(Request.OrdinaryConsumerId, partitionData, Some(clientMetadata))
-    assertTrue(fetchResult.isDefined)
+    var fetchResult = sendConsumerFetch(replicaManager, tp0, partitionData, Some(clientMetadata))
+    assertNotNull(fetchResult.get)
     assertEquals(fetchResult.get.error, Errors.NONE)
 
-    // Fetch from follower, with empty ClientMetadata
-    fetchResult = None
+    // Fetch from follower, with empty ClientMetadata (which implies an older version)
     partitionData = new FetchRequest.PartitionData(0L, 0L, 100,
       Optional.of(0))
-    fetchResult = doFetch(Request.OrdinaryConsumerId, partitionData, None)
-    assertTrue(fetchResult.isDefined)
+    fetchResult = sendConsumerFetch(replicaManager, tp0, partitionData, None)
+    assertNotNull(fetchResult.get)
     assertEquals(fetchResult.get.error, Errors.NOT_LEADER_FOR_PARTITION)
+  }
 
-    // Change to a leader, both cases are allowed
-    val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
-      Seq(
-        new LeaderAndIsrPartitionState()
-          .setTopicName(tp0.topic)
-          .setPartitionIndex(tp0.partition)
-          .setControllerEpoch(0)
-          .setLeader(0)
-          .setLeaderEpoch(1)
-          .setIsr(partition0Replicas)
-          .setZkVersion(0)
-          .setReplicas(partition0Replicas)
-          .setIsNew(true)).asJava,
+  @Test
+  def testBecomeFollowerWhileOldClientFetchInPurgatory(): Unit = {
+    val mockTimer = new MockTimer
+    val replicaManager = setupReplicaManagerWithMockedPurgatories(mockTimer, aliveBrokerIds = Seq(0, 1))
+
+    val tp0 = new TopicPartition(topic, 0)
+    val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
+    replicaManager.createPartition(tp0).createLogIfNotExists(0, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    val partition0Replicas = Seq[Integer](0, 1).asJava
+
+    val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+      Seq(new LeaderAndIsrPartitionState()
+        .setTopicName(tp0.topic)
+        .setPartitionIndex(tp0.partition)
+        .setControllerEpoch(0)
+        .setLeader(0)
+        .setLeaderEpoch(1)
+        .setIsr(partition0Replicas)
+        .setZkVersion(0)
+        .setReplicas(partition0Replicas)
+        .setIsNew(true)).asJava,
       Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
-    replicaManager.becomeLeaderOrFollower(1, leaderAndIsrRequest2, (_, _) => ())
+    replicaManager.becomeLeaderOrFollower(1, becomeLeaderRequest, (_, _) => ())
 
-    partitionData = new FetchRequest.PartitionData(0L, 0L, 100,
+    val partitionData = new FetchRequest.PartitionData(0L, 0L, 100,
+      Optional.empty())
+    val fetchResult = sendConsumerFetch(replicaManager, tp0, partitionData, None, timeout = 10)
+    assertNull(fetchResult.get)
+
+    // Become a follower and ensure that the delayed fetch returns immediately
+    val becomeFollowerRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+      Seq(new LeaderAndIsrPartitionState()
+        .setTopicName(tp0.topic)
+        .setPartitionIndex(tp0.partition)
+        .setControllerEpoch(0)
+        .setLeader(1)
+        .setLeaderEpoch(2)
+        .setIsr(partition0Replicas)
+        .setZkVersion(0)
+        .setReplicas(partition0Replicas)
+        .setIsNew(true)).asJava,
+      Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
+    replicaManager.becomeLeaderOrFollower(0, becomeFollowerRequest, (_, _) => ())
+
+    assertNotNull(fetchResult.get)
+    assertEquals(fetchResult.get.error, Errors.NOT_LEADER_FOR_PARTITION)
+  }
+
+  @Test
+  def testBecomeFollowerWhileNewClientFetchInPurgatory(): Unit = {
+    val mockTimer = new MockTimer
+    val replicaManager = setupReplicaManagerWithMockedPurgatories(mockTimer, aliveBrokerIds = Seq(0, 1))
+
+    val tp0 = new TopicPartition(topic, 0)
+    val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
+    replicaManager.createPartition(tp0).createLogIfNotExists(0, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    val partition0Replicas = Seq[Integer](0, 1).asJava
+
+    val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+      Seq(new LeaderAndIsrPartitionState()
+        .setTopicName(tp0.topic)
+        .setPartitionIndex(tp0.partition)
+        .setControllerEpoch(0)
+        .setLeader(0)
+        .setLeaderEpoch(1)
+        .setIsr(partition0Replicas)
+        .setZkVersion(0)
+        .setReplicas(partition0Replicas)
+        .setIsNew(true)).asJava,
+      Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
+    replicaManager.becomeLeaderOrFollower(1, becomeLeaderRequest, (_, _) => ())
+
+    val clientMetadata = new DefaultClientMetadata("", "", null, KafkaPrincipal.ANONYMOUS, "")
+    val partitionData = new FetchRequest.PartitionData(0L, 0L, 100,
       Optional.of(1))
-    fetchResult = doFetch(Request.OrdinaryConsumerId, partitionData, Some(clientMetadata))
-    assertTrue(fetchResult.isDefined)
+    val fetchResult = sendConsumerFetch(replicaManager, tp0, partitionData, Some(clientMetadata), timeout = 10)
+    assertNull(fetchResult.get)
+
+    // Become a follower and ensure that the delayed fetch returns immediately
+    val becomeFollowerRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+      Seq(new LeaderAndIsrPartitionState()
+        .setTopicName(tp0.topic)
+        .setPartitionIndex(tp0.partition)
+        .setControllerEpoch(0)
+        .setLeader(1)
+        .setLeaderEpoch(2)
+        .setIsr(partition0Replicas)
+        .setZkVersion(0)
+        .setReplicas(partition0Replicas)
+        .setIsNew(true)).asJava,
+      Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
+    replicaManager.becomeLeaderOrFollower(0, becomeFollowerRequest, (_, _) => ())
+
+    assertNotNull(fetchResult.get)
+    assertEquals(fetchResult.get.error, Errors.FENCED_LEADER_EPOCH)
+  }
+
+  @Test
+  def testFetchFromLeaderAlwaysAllowed(): Unit = {
+    val replicaManager = setupReplicaManagerWithMockedPurgatories(new MockTimer, aliveBrokerIds = Seq(0, 1))
+
+    val tp0 = new TopicPartition(topic, 0)
+    val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
+    replicaManager.createPartition(tp0).createLogIfNotExists(0, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    val partition0Replicas = Seq[Integer](0, 1).asJava
+
+    val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+      Seq(new LeaderAndIsrPartitionState()
+        .setTopicName(tp0.topic)
+        .setPartitionIndex(tp0.partition)
+        .setControllerEpoch(0)
+        .setLeader(0)
+        .setLeaderEpoch(1)
+        .setIsr(partition0Replicas)
+        .setZkVersion(0)
+        .setReplicas(partition0Replicas)
+        .setIsNew(true)).asJava,
+      Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
+    replicaManager.becomeLeaderOrFollower(1, becomeLeaderRequest, (_, _) => ())
+
+    val clientMetadata = new DefaultClientMetadata("", "", null, KafkaPrincipal.ANONYMOUS, "")
+    var partitionData = new FetchRequest.PartitionData(0L, 0L, 100,
+      Optional.of(1))
+    var fetchResult = sendConsumerFetch(replicaManager, tp0, partitionData, Some(clientMetadata))
+    assertNotNull(fetchResult.get)
     assertEquals(fetchResult.get.error, Errors.NONE)
 
-    fetchResult = None
     partitionData = new FetchRequest.PartitionData(0L, 0L, 100,
       Optional.empty())
-    fetchResult = doFetch(Request.OrdinaryConsumerId, partitionData, None)
-    assertTrue(fetchResult.isDefined)
+    fetchResult = sendConsumerFetch(replicaManager, tp0, partitionData, Some(clientMetadata))
+    assertNotNull(fetchResult.get)
     assertEquals(fetchResult.get.error, Errors.NONE)
   }
 
+  private def sendConsumerFetch(replicaManager: ReplicaManager,
+                                topicPartition: TopicPartition,
+                                partitionData: FetchRequest.PartitionData,
+                                clientMetadataOpt: Option[ClientMetadata],
+                                timeout: Long = 0L): AtomicReference[FetchPartitionData] = {
+    val fetchResult = new AtomicReference[FetchPartitionData]()
+    def callback(response: Seq[(TopicPartition, FetchPartitionData)]): Unit = {
+      fetchResult.set(response.toMap.apply(topicPartition))
+    }
+    replicaManager.fetchMessages(
+      timeout = timeout,
+      replicaId = Request.OrdinaryConsumerId,
+      fetchMinBytes = 1,
+      fetchMaxBytes = 100,
+      hardMaxBytesLimit = false,
+      fetchInfos = Seq(topicPartition -> partitionData),
+      quota = UnboundedQuota,
+      isolationLevel = IsolationLevel.READ_UNCOMMITTED,
+      responseCallback = callback,
+      clientMetadata = clientMetadataOpt
+    )
+    fetchResult
+  }
+
   /**
    * This method assumes that the test using created ReplicaManager calls
    * ReplicaManager.becomeLeaderOrFollower() once with LeaderAndIsrRequest containing

From 876eada04ed52a7eb360a613e285b1f999ba0a44 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Mon, 18 Nov 2019 15:11:42 -0800
Subject: [PATCH 0804/1071] KAFKA-9198; Complete purgatory operations on
 receiving StopReplica (#7701)

Force completion of delayed operations when receiving a StopReplica request. In the case of a partition reassignment, we cannot rely on receiving a LeaderAndIsr request in order to complete these operations because the leader may no longer be a replica. Previously when this happened, the delayed operations were left to eventually timeout.

Reviewers: Stanislav Kozlovski , Ismael Juma 

Co-Authored-By: Kun Du 
---
 .../scala/kafka/server/DelayedFetch.scala     |   7 +-
 .../scala/kafka/server/DelayedProduce.scala   |  17 +--
 .../scala/kafka/server/ReplicaManager.scala   |  37 +++--
 .../kafka/server/ReplicaManagerTest.scala     | 126 ++++++++++++++++--
 4 files changed, 155 insertions(+), 32 deletions(-)

diff --git a/core/src/main/scala/kafka/server/DelayedFetch.scala b/core/src/main/scala/kafka/server/DelayedFetch.scala
index 7e0da30b8ef07..3cf55d7b8f242 100644
--- a/core/src/main/scala/kafka/server/DelayedFetch.scala
+++ b/core/src/main/scala/kafka/server/DelayedFetch.scala
@@ -29,8 +29,11 @@ import scala.collection._
 
 case class FetchPartitionStatus(startOffsetMetadata: LogOffsetMetadata, fetchInfo: PartitionData) {
 
-  override def toString = "[startOffsetMetadata: " + startOffsetMetadata + ", " +
-                          "fetchInfo: " + fetchInfo + "]"
+  override def toString: String = {
+    "[startOffsetMetadata: " + startOffsetMetadata +
+      ", fetchInfo: " + fetchInfo +
+      "]"
+  }
 }
 
 /**
diff --git a/core/src/main/scala/kafka/server/DelayedProduce.scala b/core/src/main/scala/kafka/server/DelayedProduce.scala
index 6ec20c1d518ff..83e6142008205 100644
--- a/core/src/main/scala/kafka/server/DelayedProduce.scala
+++ b/core/src/main/scala/kafka/server/DelayedProduce.scala
@@ -17,7 +17,6 @@
 
 package kafka.server
 
-
 import java.util.concurrent.TimeUnit
 import java.util.concurrent.locks.Lock
 
@@ -86,17 +85,15 @@ class DelayedProduce(delayMs: Long,
       trace(s"Checking produce satisfaction for $topicPartition, current status $status")
       // skip those partitions that have already been satisfied
       if (status.acksPending) {
-        val (hasEnough, error) = replicaManager.getPartition(topicPartition) match {
-          case HostedPartition.Online(partition) =>
-            partition.checkEnoughReplicasReachOffset(status.requiredOffset)
-
-          case HostedPartition.Offline =>
-            (false, Errors.KAFKA_STORAGE_ERROR)
-
-          case HostedPartition.None =>
+        val (hasEnough, error) = replicaManager.getPartitionOrError(topicPartition, expectLeader = true) match {
+          case Left(err) =>
             // Case A
-            (false, Errors.UNKNOWN_TOPIC_OR_PARTITION)
+            (false, err)
+
+          case Right(partition) =>
+            partition.checkEnoughReplicasReachOffset(status.requiredOffset)
         }
+
         // Case B.1 || B.2
         if (error != Errors.NONE || hasEnough) {
           status.acksPending = false
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index a59c1b5643c7a..42302bb7e527f 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -372,9 +372,20 @@ class ReplicaManager(val config: KafkaConfig,
       if (logManager.getLog(topicPartition, isFuture = true).isDefined)
         logManager.asyncDelete(topicPartition, isFuture = true)
     }
+
+    // If we were the leader, we may have some operations still waiting for completion.
+    // We force completion to prevent them from timing out.
+    completeDelayedFetchOrProduceRequests(topicPartition)
+
     stateChangeLogger.trace(s"Finished handling stop replica (delete=$deletePartition) for partition $topicPartition")
   }
 
+  private def completeDelayedFetchOrProduceRequests(topicPartition: TopicPartition): Unit = {
+    val topicPartitionOperationKey = TopicPartitionOperationKey(topicPartition)
+    delayedProducePurgatory.checkAndComplete(topicPartitionOperationKey)
+    delayedFetchPurgatory.checkAndComplete(topicPartitionOperationKey)
+  }
+
   def stopReplicas(stopReplicaRequest: StopReplicaRequest): (mutable.Map[TopicPartition, Errors], Errors) = {
     replicaStateChangeLock synchronized {
       val responseMap = new collection.mutable.HashMap[TopicPartition, Errors]
@@ -436,12 +447,24 @@ class ReplicaManager(val config: KafkaConfig,
   }
 
   def getPartitionOrException(topicPartition: TopicPartition, expectLeader: Boolean): Partition = {
+    getPartitionOrError(topicPartition, expectLeader) match {
+      case Left(Errors.KAFKA_STORAGE_ERROR) =>
+        throw new KafkaStorageException(s"Partition $topicPartition is in an offline log directory")
+
+      case Left(error) =>
+        throw error.exception(s"Error while fetching partition state for $topicPartition")
+
+      case Right(partition) => partition
+    }
+  }
+
+  def getPartitionOrError(topicPartition: TopicPartition, expectLeader: Boolean): Either[Errors, Partition] = {
     getPartition(topicPartition) match {
       case HostedPartition.Online(partition) =>
-        partition
+        Right(partition)
 
       case HostedPartition.Offline =>
-        throw new KafkaStorageException(s"Partition $topicPartition is in an offline log directory")
+        Left(Errors.KAFKA_STORAGE_ERROR)
 
       case HostedPartition.None if metadataCache.contains(topicPartition) =>
         if (expectLeader) {
@@ -449,13 +472,13 @@ class ReplicaManager(val config: KafkaConfig,
           // forces clients to refresh metadata to find the new location. This can happen, for example,
           // during a partition reassignment if a produce request from the client is sent to a broker after
           // the local replica has been deleted.
-          throw new NotLeaderForPartitionException(s"Broker $localBrokerId is not a replica of $topicPartition")
+          Left(Errors.NOT_LEADER_FOR_PARTITION)
         } else {
-          throw new ReplicaNotAvailableException(s"Partition $topicPartition is not available")
+          Left(Errors.REPLICA_NOT_AVAILABLE)
         }
 
       case HostedPartition.None =>
-        throw new UnknownTopicOrPartitionException(s"Partition $topicPartition doesn't exist")
+        Left(Errors.UNKNOWN_TOPIC_OR_PARTITION)
     }
   }
 
@@ -1466,9 +1489,7 @@ class ReplicaManager(val config: KafkaConfig,
       }
 
       partitionsToMakeFollower.foreach { partition =>
-        val topicPartitionOperationKey = TopicPartitionOperationKey(partition.topicPartition)
-        delayedProducePurgatory.checkAndComplete(topicPartitionOperationKey)
-        delayedFetchPurgatory.checkAndComplete(topicPartitionOperationKey)
+        completeDelayedFetchOrProduceRequests(partition.topicPartition)
       }
 
       partitionsToMakeFollower.foreach { partition =>
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index ab31cfb9810f4..73971a7fc25b1 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -50,6 +50,7 @@ import org.apache.kafka.common.{Node, TopicPartition}
 import org.easymock.EasyMock
 import org.junit.Assert._
 import org.junit.{After, Before, Test}
+import org.mockito.Mockito
 
 import scala.collection.JavaConverters._
 import scala.collection.{Map, Seq}
@@ -945,14 +946,14 @@ class ReplicaManagerTest {
       Optional.of(0))
     var fetchResult = sendConsumerFetch(replicaManager, tp0, partitionData, Some(clientMetadata))
     assertNotNull(fetchResult.get)
-    assertEquals(fetchResult.get.error, Errors.NONE)
+    assertEquals(Errors.NONE, fetchResult.get.error)
 
     // Fetch from follower, with empty ClientMetadata (which implies an older version)
     partitionData = new FetchRequest.PartitionData(0L, 0L, 100,
       Optional.of(0))
     fetchResult = sendConsumerFetch(replicaManager, tp0, partitionData, None)
     assertNotNull(fetchResult.get)
-    assertEquals(fetchResult.get.error, Errors.NOT_LEADER_FOR_PARTITION)
+    assertEquals(Errors.NOT_LEADER_FOR_PARTITION, fetchResult.get.error)
   }
 
   @Test
@@ -1000,7 +1001,7 @@ class ReplicaManagerTest {
     replicaManager.becomeLeaderOrFollower(0, becomeFollowerRequest, (_, _) => ())
 
     assertNotNull(fetchResult.get)
-    assertEquals(fetchResult.get.error, Errors.NOT_LEADER_FOR_PARTITION)
+    assertEquals(Errors.NOT_LEADER_FOR_PARTITION, fetchResult.get.error)
   }
 
   @Test
@@ -1049,7 +1050,7 @@ class ReplicaManagerTest {
     replicaManager.becomeLeaderOrFollower(0, becomeFollowerRequest, (_, _) => ())
 
     assertNotNull(fetchResult.get)
-    assertEquals(fetchResult.get.error, Errors.FENCED_LEADER_EPOCH)
+    assertEquals(Errors.FENCED_LEADER_EPOCH, fetchResult.get.error)
   }
 
   @Test
@@ -1080,13 +1081,114 @@ class ReplicaManagerTest {
       Optional.of(1))
     var fetchResult = sendConsumerFetch(replicaManager, tp0, partitionData, Some(clientMetadata))
     assertNotNull(fetchResult.get)
-    assertEquals(fetchResult.get.error, Errors.NONE)
+    assertEquals(Errors.NONE, fetchResult.get.error)
 
     partitionData = new FetchRequest.PartitionData(0L, 0L, 100,
       Optional.empty())
     fetchResult = sendConsumerFetch(replicaManager, tp0, partitionData, Some(clientMetadata))
     assertNotNull(fetchResult.get)
-    assertEquals(fetchResult.get.error, Errors.NONE)
+    assertEquals(Errors.NONE, fetchResult.get.error)
+  }
+
+  @Test
+  def testClearFetchPurgatoryOnStopReplica(): Unit = {
+    // As part of a reassignment, we may send StopReplica to the old leader.
+    // In this case, we should ensure that pending purgatory operations are cancelled
+    // immediately rather than sitting around to timeout.
+
+    val mockTimer = new MockTimer
+    val replicaManager = setupReplicaManagerWithMockedPurgatories(mockTimer, aliveBrokerIds = Seq(0, 1))
+
+    val tp0 = new TopicPartition(topic, 0)
+    val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
+    replicaManager.createPartition(tp0).createLogIfNotExists(0, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    val partition0Replicas = Seq[Integer](0, 1).asJava
+
+    val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+      Seq(new LeaderAndIsrPartitionState()
+        .setTopicName(tp0.topic)
+        .setPartitionIndex(tp0.partition)
+        .setControllerEpoch(0)
+        .setLeader(0)
+        .setLeaderEpoch(1)
+        .setIsr(partition0Replicas)
+        .setZkVersion(0)
+        .setReplicas(partition0Replicas)
+        .setIsNew(true)).asJava,
+      Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
+    replicaManager.becomeLeaderOrFollower(1, becomeLeaderRequest, (_, _) => ())
+
+    val partitionData = new FetchRequest.PartitionData(0L, 0L, 100,
+      Optional.of(1))
+    val fetchResult = sendConsumerFetch(replicaManager, tp0, partitionData, None, timeout = 10)
+    assertNull(fetchResult.get)
+
+    Mockito.when(replicaManager.metadataCache.contains(tp0)).thenReturn(true)
+
+    // We have a fetch in purgatory, now receive a stop replica request and
+    // assert that the fetch returns with a NOT_LEADER error
+
+    replicaManager.stopReplica(tp0, deletePartition = true)
+    assertNotNull(fetchResult.get)
+    assertEquals(Errors.NOT_LEADER_FOR_PARTITION, fetchResult.get.error)
+  }
+
+  @Test
+  def testClearProducePurgatoryOnStopReplica(): Unit = {
+    val mockTimer = new MockTimer
+    val replicaManager = setupReplicaManagerWithMockedPurgatories(mockTimer, aliveBrokerIds = Seq(0, 1))
+
+    val tp0 = new TopicPartition(topic, 0)
+    val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
+    replicaManager.createPartition(tp0).createLogIfNotExists(0, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    val partition0Replicas = Seq[Integer](0, 1).asJava
+
+    val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+      Seq(new LeaderAndIsrPartitionState()
+        .setTopicName(tp0.topic)
+        .setPartitionIndex(tp0.partition)
+        .setControllerEpoch(0)
+        .setLeader(0)
+        .setLeaderEpoch(1)
+        .setIsr(partition0Replicas)
+        .setZkVersion(0)
+        .setReplicas(partition0Replicas)
+        .setIsNew(true)).asJava,
+      Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
+    replicaManager.becomeLeaderOrFollower(1, becomeLeaderRequest, (_, _) => ())
+
+    val produceResult = sendProducerAppend(replicaManager, tp0)
+    assertNull(produceResult.get)
+
+    Mockito.when(replicaManager.metadataCache.contains(tp0)).thenReturn(true)
+
+    replicaManager.stopReplica(tp0, deletePartition = true)
+    assertNotNull(produceResult.get)
+    assertEquals(Errors.NOT_LEADER_FOR_PARTITION, produceResult.get.error)
+  }
+
+  private def sendProducerAppend(replicaManager: ReplicaManager,
+                                 topicPartition: TopicPartition): AtomicReference[PartitionResponse] = {
+    val produceResult = new AtomicReference[PartitionResponse]()
+    def callback(response: Map[TopicPartition, PartitionResponse]): Unit = {
+      produceResult.set(response(topicPartition))
+    }
+
+    val records = MemoryRecords.withRecords(CompressionType.NONE,
+      new SimpleRecord("a".getBytes()),
+      new SimpleRecord("b".getBytes()),
+      new SimpleRecord("c".getBytes())
+    )
+
+    replicaManager.appendRecords(
+      timeout = 10,
+      requiredAcks = -1,
+      internalTopicsAllowed = false,
+      isFromClient = true,
+      entriesPerPartition = Map(topicPartition -> records),
+      responseCallback = callback
+    )
+    produceResult
   }
 
   private def sendConsumerFetch(replicaManager: ReplicaManager,
@@ -1371,14 +1473,14 @@ class ReplicaManagerTest {
     val logProps = new Properties()
     val mockLogMgr = TestUtils.createLogManager(config.logDirs.map(new File(_)), LogConfig(logProps))
     val aliveBrokers = aliveBrokerIds.map(brokerId => createBroker(brokerId, s"host$brokerId", brokerId))
-    val metadataCache: MetadataCache = EasyMock.createMock(classOf[MetadataCache])
-    EasyMock.expect(metadataCache.getAliveBrokers).andReturn(aliveBrokers).anyTimes()
+
+    val metadataCache: MetadataCache = Mockito.mock(classOf[MetadataCache])
+    Mockito.when(metadataCache.getAliveBrokers).thenReturn(aliveBrokers)
+
     aliveBrokerIds.foreach { brokerId =>
-      EasyMock.expect(metadataCache.getAliveBroker(EasyMock.eq(brokerId)))
-        .andReturn(Option(createBroker(brokerId, s"host$brokerId", brokerId)))
-        .anyTimes()
+      Mockito.when(metadataCache.getAliveBroker(brokerId))
+        .thenReturn(Option(createBroker(brokerId, s"host$brokerId", brokerId)))
     }
-    EasyMock.replay(metadataCache)
 
     val mockProducePurgatory = new DelayedOperationPurgatory[DelayedProduce](
       purgatoryName = "Produce", timer, reaperEnabled = false)

From 70402c1fa7131d527974c01cbaa41ce41c2d1fc2 Mon Sep 17 00:00:00 2001
From: Bruno Cadonna 
Date: Tue, 19 Nov 2019 01:32:21 +0100
Subject: [PATCH 0805/1071] HOTFIX: Fix unit tests that failed when executed
 from IDE (#7707)

Reviewers: John Roesler , Matthias J. Sax 
---
 .../internals/metrics/ClientMetricsTest.java  | 67 ++++++-------------
 .../kafka/kafka-streams-version.properties    | 16 +++++
 2 files changed, 36 insertions(+), 47 deletions(-)
 create mode 100644 streams/src/test/resources/kafka/kafka-streams-version.properties

diff --git a/streams/src/test/java/org/apache/kafka/streams/internals/metrics/ClientMetricsTest.java b/streams/src/test/java/org/apache/kafka/streams/internals/metrics/ClientMetricsTest.java
index 20f5ac85e4d8b..9ba9cc4177880 100644
--- a/streams/src/test/java/org/apache/kafka/streams/internals/metrics/ClientMetricsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/internals/metrics/ClientMetricsTest.java
@@ -16,71 +16,60 @@
  */
 package org.apache.kafka.streams.internals.metrics;
 
-
 import org.apache.kafka.common.metrics.Gauge;
 import org.apache.kafka.common.metrics.Sensor.RecordingLevel;
 import org.apache.kafka.streams.KafkaStreams.State;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.junit.Test;
 
-import static org.easymock.EasyMock.and;
 import static org.easymock.EasyMock.eq;
 import static org.easymock.EasyMock.mock;
-import static org.easymock.EasyMock.not;
-import static org.easymock.EasyMock.notNull;
 import static org.powermock.api.easymock.PowerMock.replay;
 import static org.powermock.api.easymock.PowerMock.verify;
 
 public class ClientMetricsTest {
+    private static final String COMMIT_ID = "test-commit-ID";
+    private static final String VERSION = "test-version";
 
     private final StreamsMetricsImpl streamsMetrics = mock(StreamsMetricsImpl.class);
 
-    private interface OneParamMetricAdder {
-        void addMetric(final StreamsMetricsImpl streamsMetrics);
-    }
-
-    private interface TwoParamMetricAdder {
-        void addMetric(final StreamsMetricsImpl streamsMetrics, final String value);
-    }
-
-    /*
-     * This test may fail when executed from a IDE since it expects the /kafka/kafka-streams-version.properties on the
-     * class path.
-     */
     @Test
     public void shouldAddVersionMetric() {
         final String name = "version";
         final String description = "The version of the Kafka Streams client";
-        setUpAndVerifyMetricOneParam(name, description, ClientMetrics::addVersionMetric);
+        setUpAndVerifyMetric(name, description, VERSION, () -> ClientMetrics.addVersionMetric(streamsMetrics));
     }
 
-    /*
-     * This test may fail when executed from a IDE since it expects the /kafka/kafka-streams-version.properties on the
-     * class path.
-     */
     @Test
     public void shouldAddCommitIdMetric() {
         final String name = "commit-id";
         final String description = "The version control commit ID of the Kafka Streams client";
-        setUpAndVerifyMetricOneParam(name, description, ClientMetrics::addCommitIdMetric);
+        setUpAndVerifyMetric(name, description, COMMIT_ID, () -> ClientMetrics.addCommitIdMetric(streamsMetrics));
     }
 
     @Test
     public void shouldAddApplicationIdMetric() {
         final String name = "application-id";
         final String description = "The application ID of the Kafka Streams client";
-        setUpAndVerifyMetricTwoParam(name, description, "thisIsAnID", ClientMetrics::addApplicationIdMetric);
+        final String applicationId = "thisIsAnID";
+        setUpAndVerifyMetric(
+            name,
+            description,
+            applicationId,
+            () -> ClientMetrics.addApplicationIdMetric(streamsMetrics, applicationId)
+        );
     }
 
     @Test
     public void shouldAddTopologyDescriptionMetric() {
         final String name = "topology-description";
         final String description = "The description of the topology executed in the Kafka Streams client";
-        setUpAndVerifyMetricTwoParam(
+        final String topologyDescription = "thisIsATopologyDescription";
+        setUpAndVerifyMetric(
             name,
             description,
-            "thisIsATopologyDescription",
-            ClientMetrics::addTopologyDescriptionMetric
+            topologyDescription,
+            () -> ClientMetrics.addTopologyDescriptionMetric(streamsMetrics, topologyDescription)
         );
     }
 
@@ -102,26 +91,10 @@ public void shouldAddStateMetric() {
         verify(streamsMetrics);
     }
 
-    private void setUpAndVerifyMetricOneParam(final String name,
-                                              final String description,
-                                              final OneParamMetricAdder metricAdder) {
-        streamsMetrics.addClientLevelImmutableMetric(
-            eq(name),
-            eq(description),
-            eq(RecordingLevel.INFO),
-            and(not(eq("unknown")), notNull())
-        );
-        replay(streamsMetrics);
-
-        metricAdder.addMetric(streamsMetrics);
-
-        verify(streamsMetrics);
-    }
-
-    private void setUpAndVerifyMetricTwoParam(final String name,
-                                              final String description,
-                                              final String value,
-                                              final TwoParamMetricAdder metricAdder) {
+    private void setUpAndVerifyMetric(final String name,
+                                      final String description,
+                                      final String value,
+                                      final Runnable metricAdder) {
         streamsMetrics.addClientLevelImmutableMetric(
             eq(name),
             eq(description),
@@ -130,7 +103,7 @@ private void setUpAndVerifyMetricTwoParam(final String name,
         );
         replay(streamsMetrics);
 
-        metricAdder.addMetric(streamsMetrics, value);
+        metricAdder.run();
 
         verify(streamsMetrics);
     }
diff --git a/streams/src/test/resources/kafka/kafka-streams-version.properties b/streams/src/test/resources/kafka/kafka-streams-version.properties
new file mode 100644
index 0000000000000..333ba7183b4f9
--- /dev/null
+++ b/streams/src/test/resources/kafka/kafka-streams-version.properties
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+commitId=test-commit-ID
+version=test-version
\ No newline at end of file

From cbc9f57c0e6a2f9cd5252c11aeb19abefba4263e Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Tue, 19 Nov 2019 13:34:38 -0800
Subject: [PATCH 0806/1071] HOTFIX: safely clear all active state in
 onPartitionsLost (#7691)

After a number of last minute bugs were found stemming from the incremental closing of lost tasks in StreamsRebalanceListener#onPartitionsLost, a safer approach to this edge case seems warranted. We initially wanted to be as "future-proof" as possible, and avoid baking further protocol assumptions into the code that may be broken as the protocol evolves. This meant that rather than simply closing all active tasks and clearing all associated state in #onPartitionsLost(lostPartitions) we would loop through the lostPartitions/lost tasks and remove them one by one from the various data structures/assignments, then verify that everything was empty in the end. This verification in particular has caused us significant trouble, as it turns out to be nontrivial to determine what should in fact be empty, and if so whether it is also being correctly updated.

Therefore, before worrying about it being "future-proof" it seems we should make sure it is "present-day-proof" and implement this callback in the safest possible way, by blindly clearing and closing all active task state. We log all the relevant state (at debug level) before clearing it, so we can at least tell from the logs whether/which emptiness checks were being violated.

Reviewers: Guozhang Wang , Bill Bejeck , Andrew Choi 
---
 .../consumer/ConsumerRebalanceListener.java   |   3 +-
 .../internals/AssignedStandbyTasks.java       |   3 +-
 .../internals/AssignedStreamsTasks.java       | 128 ++++++++++--------
 .../processor/internals/AssignedTasks.java    |  64 ++++++---
 .../internals/StoreChangelogReader.java       |   2 +-
 .../internals/StreamsRebalanceListener.java   |   6 +-
 .../processor/internals/TaskManager.java      |  58 +++-----
 .../internals/AssignedStreamsTasksTest.java   |  62 +++------
 8 files changed, 156 insertions(+), 170 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
index ec2ce39a2d9be..2f43b603fc8ff 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
@@ -188,7 +188,8 @@ public interface ConsumerRebalanceListener {
      * necessary to catch these exceptions and re-attempt to wakeup or interrupt the consumer thread.
      *
      * @param partitions The list of partitions that were assigned to the consumer and now have been reassigned
-     *                   to other consumers (may not include all currently assigned partitions, i.e. there may still
+     *                   to other consumers. With the current protocol this will always include all of the consumer's
+     *                   previously assigned partitions, but this may change in future protocols (ie there would still
      *                   be some partitions left)
      * @throws org.apache.kafka.common.errors.WakeupException If raised from a nested call to {@link KafkaConsumer}
      * @throws org.apache.kafka.common.errors.InterruptException If raised from a nested call to {@link KafkaConsumer}
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStandbyTasks.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStandbyTasks.java
index 0f8896eea7975..299390bfd1666 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStandbyTasks.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStandbyTasks.java
@@ -17,6 +17,7 @@
 package org.apache.kafka.streams.processor.internals;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -79,7 +80,7 @@ List closeRevokedStandbyTasks(final Map suspendedTaskIds() {
         return suspended.keySet();
@@ -152,7 +162,7 @@ private RuntimeException suspendRunningTasks(final Set runningTasksToSus
                         id, f);
                 }
             } finally {
-                removeTaskFromRunning(task);
+                removeTaskFromAllStateMaps(task, suspended);
                 taskChangelogs.addAll(task.changelogPartitions());
             }
         }
@@ -189,10 +199,8 @@ RuntimeException closeRestoringTasks(final Set restoringTasksToClose,
     }
 
     private RuntimeException closeRunning(final boolean isZombie,
-                                          final StreamTask task,
-                                          final List closedTaskChangelogs) {
-        removeTaskFromRunning(task);
-        closedTaskChangelogs.addAll(task.changelogPartitions());
+                                          final StreamTask task) {
+        removeTaskFromAllStateMaps(task, Collections.emptyMap());
 
         try {
             final boolean clean = !isZombie;
@@ -208,7 +216,7 @@ private RuntimeException closeRunning(final boolean isZombie,
     private RuntimeException closeNonRunning(final boolean isZombie,
                                              final StreamTask task,
                                              final List closedTaskChangelogs) {
-        created.remove(task.id());
+        removeTaskFromAllStateMaps(task, Collections.emptyMap());
         closedTaskChangelogs.addAll(task.changelogPartitions());
 
         try {
@@ -221,10 +229,11 @@ private RuntimeException closeNonRunning(final boolean isZombie,
         return null;
     }
 
+    // Since a restoring task has not had its topology initialized yet, we need only close the state manager
     private RuntimeException closeRestoring(final boolean isZombie,
                                             final StreamTask task,
                                             final List closedTaskChangelogs) {
-        removeTaskFromRestoring(task);
+        removeTaskFromAllStateMaps(task, Collections.emptyMap());
         closedTaskChangelogs.addAll(task.changelogPartitions());
 
         try {
@@ -240,7 +249,7 @@ private RuntimeException closeRestoring(final boolean isZombie,
 
     private RuntimeException closeSuspended(final boolean isZombie,
                                             final StreamTask task) {
-        suspended.remove(task.id());
+        removeTaskFromAllStateMaps(task, Collections.emptyMap());
 
         try {
             final boolean clean = !isZombie;
@@ -269,37 +278,30 @@ RuntimeException closeNotAssignedSuspendedTasks(final Set revokedTasks)
         return firstException.get();
     }
 
-    RuntimeException closeZombieTasks(final Set lostTasks, final List lostTaskChangelogs) {
+    RuntimeException closeAllTasksAsZombies() {
+        log.debug("Closing all active tasks as zombies, current state of active tasks: {}", toString());
+
         final AtomicReference firstException = new AtomicReference<>(null);
+        final List changelogs = new ArrayList<>(); // not used, as we clear/unsubscribe all changelogs
 
-        for (final TaskId id : lostTasks) {
-            if (suspended.containsKey(id)) {
-                log.debug("Closing the zombie suspended stream task {}.", id);
-                firstException.compareAndSet(null, closeSuspended(true, suspended.get(id)));
+        for (final TaskId id : allAssignedTaskIds()) {
+            if (running.containsKey(id)) {
+                log.debug("Closing the zombie running stream task {}.", id);
+                firstException.compareAndSet(null, closeRunning(true, running.get(id)));
             } else if (created.containsKey(id)) {
                 log.debug("Closing the zombie created stream task {}.", id);
-                firstException.compareAndSet(null, closeNonRunning(true, created.get(id), lostTaskChangelogs));
+                firstException.compareAndSet(null, closeNonRunning(true, created.get(id), changelogs));
             } else if (restoring.containsKey(id)) {
                 log.debug("Closing the zombie restoring stream task {}.", id);
-                firstException.compareAndSet(null, closeRestoring(true, restoring.get(id), lostTaskChangelogs));
-            } else if (running.containsKey(id)) {
-                log.debug("Closing the zombie running stream task {}.", id);
-                firstException.compareAndSet(null, closeRunning(true, running.get(id), lostTaskChangelogs));
-            } else {
-                log.warn("Skipping closing the zombie stream task {} as it was already removed.", id);
+                firstException.compareAndSet(null, closeRestoring(true, restoring.get(id), changelogs));
+            } else if (suspended.containsKey(id)) {
+                log.debug("Closing the zombie suspended stream task {}.", id);
+                firstException.compareAndSet(null, closeSuspended(true, suspended.get(id)));
             }
         }
 
-        // We always clear the prevActiveTasks and replace with current set of running tasks to encode in subscription
-        // We should exclude any tasks that were lost however, they will be counted as standbys for assignment purposes
-        prevActiveTasks.clear();
-        prevActiveTasks.addAll(running.keySet());
+        clear();
 
-        // With the current rebalance protocol, there should not be any running tasks left as they were all lost
-        if (!prevActiveTasks.isEmpty()) {
-            log.error("Found the still running stream tasks {} after closing all tasks lost as zombies", prevActiveTasks);
-            firstException.compareAndSet(null, new IllegalStateException("Not all lost tasks were closed as zombies"));
-        }
         return firstException.get();
     }
 
@@ -311,7 +313,7 @@ boolean maybeResumeSuspendedTask(final TaskId taskId,
         if (suspended.containsKey(taskId)) {
             final StreamTask task = suspended.get(taskId);
             log.trace("Found suspended stream task {}", taskId);
-            suspended.remove(taskId);
+            removeTaskFromAllStateMaps(task, Collections.emptyMap());
 
             if (task.partitions().equals(partitions)) {
                 task.resume();
@@ -346,8 +348,12 @@ void updateRestored(final Collection restored) {
             if (restoredPartitions.containsAll(task.changelogPartitions())) {
                 transitionToRunning(task);
                 it.remove();
-                restoringByPartition.keySet().removeAll(task.partitions());
-                restoringByPartition.keySet().removeAll(task.changelogPartitions());
+                // Note that because we add back all restored partitions at the top of this loop, clearing them from
+                // restoredPartitions here doesn't really matter. We do it anyway as it is the correct thing to do,
+                // and may matter with future changes.
+                removeFromRestoredPartitions(task);
+                removeFromRestoringByPartition(task);
+
                 log.debug("Stream task {} completed restoration as all its changelog partitions {} have been applied to restore state",
                     task.id(),
                     task.changelogPartitions());
@@ -372,6 +378,24 @@ void updateRestored(final Collection restored) {
         }
     }
 
+    @Override
+     void removeTaskFromAllStateMaps(final StreamTask task, final Map currentStateMap) {
+        super.removeTaskFromAllStateMaps(task, currentStateMap);
+
+        final TaskId id = task.id();
+        final Set taskPartitions = new HashSet<>(task.partitions());
+        taskPartitions.addAll(task.changelogPartitions());
+
+        if (currentStateMap != restoring) {
+            restoring.remove(id);
+            restoringByPartition.keySet().removeAll(taskPartitions);
+            restoredPartitions.removeAll(taskPartitions);
+        }
+        if (currentStateMap != suspended) {
+            suspended.remove(id);
+        }
+    }
+
     void addTaskToRestoring(final StreamTask task) {
         restoring.put(task.id(), task);
         for (final TopicPartition topicPartition : task.partitions()) {
@@ -382,16 +406,14 @@ void addTaskToRestoring(final StreamTask task) {
         }
     }
 
-    private void removeTaskFromRestoring(final StreamTask task) {
-        restoring.remove(task.id());
-        for (final TopicPartition topicPartition : task.partitions()) {
-            restoredPartitions.remove(topicPartition);
-            restoringByPartition.remove(topicPartition);
-        }
-        for (final TopicPartition topicPartition : task.changelogPartitions()) {
-            restoredPartitions.remove(topicPartition);
-            restoringByPartition.remove(topicPartition);
-        }
+    private void removeFromRestoringByPartition(final StreamTask task) {
+        restoringByPartition.keySet().removeAll(task.partitions());
+        restoringByPartition.keySet().removeAll(task.changelogPartitions());
+    }
+
+    private void removeFromRestoredPartitions(final StreamTask task) {
+        restoredPartitions.removeAll(task.partitions());
+        restoredPartitions.removeAll(task.changelogPartitions());
     }
 
     /**
@@ -497,6 +519,7 @@ void clear() {
         restoringByPartition.clear();
         restoredPartitions.clear();
         suspended.clear();
+        prevActiveTasks.clear();
     }
 
     @Override
@@ -511,26 +534,13 @@ public void shutdown(final boolean clean) {
         super.shutdown(clean);
     }
 
-    @Override
-    public boolean isEmpty() throws IllegalStateException {
-        if (restoring.isEmpty() && !restoringByPartition.isEmpty()) {
-            log.error("Assigned stream tasks in an inconsistent state: the set of restoring tasks is empty but the " +
-                      "restoring by partitions map contained {}", restoringByPartition);
-            throw new IllegalStateException("Found inconsistent state: no tasks restoring but nonempty restoringByPartition");
-        } else {
-            return super.isEmpty()
-                       && restoring.isEmpty()
-                       && restoringByPartition.isEmpty()
-                       && restoredPartitions.isEmpty()
-                       && suspended.isEmpty();
-        }
-    }
-
     public String toString(final String indent) {
         final StringBuilder builder = new StringBuilder();
         builder.append(super.toString(indent));
-        describe(builder, restoring.values(), indent, "Restoring:");
-        describe(builder, suspended.values(), indent, "Suspended:");
+        describeTasks(builder, restoring.values(), indent, "Restoring:");
+        describePartitions(builder, restoringByPartition.keySet(), indent, "Restoring Partitions:");
+        describePartitions(builder, restoredPartitions, indent, "Restored Partitions:");
+        describeTasks(builder, suspended.values(), indent, "Suspended:");
         return builder.toString();
     }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
index cc4c0f9dfc6b7..45ac26b1f65ec 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
@@ -69,12 +69,17 @@ void initializeNewTasks() {
             try {
                 final T task = entry.getValue();
                 task.initializeMetadata();
+
+                // don't remove from created until the task has been successfully initialized
+                removeTaskFromAllStateMaps(task, created);
+
                 if (!task.initializeStateStores()) {
                     log.debug("Transitioning {} {} to restoring", taskTypeName, entry.getKey());
                     ((AssignedStreamsTasks) this).addTaskToRestoring((StreamTask) task);
                 } else {
                     transitionToRunning(task);
                 }
+
                 it.remove();
             } catch (final LockException e) {
                 // If this is a permanent error, then we could spam the log since this is in the run loop. But, other related
@@ -121,10 +126,25 @@ void transitionToRunning(final T task) {
         }
     }
 
-    void removeTaskFromRunning(final T task) {
-        running.remove(task.id());
-        runningByPartition.keySet().removeAll(task.partitions());
-        runningByPartition.keySet().removeAll(task.changelogPartitions());
+    /**
+     * Removes the passed in task (and its corresponding partitions) from all state maps and sets,
+     * except for the one it currently resides in.
+     *
+     * @param task the task to be removed
+     * @param currentStateMap the current state map, which the task should not be removed from
+     */
+    void removeTaskFromAllStateMaps(final T task, final Map currentStateMap) {
+        final TaskId id = task.id();
+        final Set taskPartitions = new HashSet<>(task.partitions());
+        taskPartitions.addAll(task.changelogPartitions());
+
+        if (currentStateMap != running) {
+            running.remove(id);
+            runningByPartition.keySet().removeAll(taskPartitions);
+        }
+        if (currentStateMap != created) {
+            created.remove(id);
+        }
     }
 
     T runningTaskFor(final TopicPartition partition) {
@@ -146,15 +166,16 @@ public String toString() {
 
     public String toString(final String indent) {
         final StringBuilder builder = new StringBuilder();
-        describe(builder, running.values(), indent, "Running:");
-        describe(builder, created.values(), indent, "New:");
+        describeTasks(builder, running.values(), indent, "Running:");
+        describePartitions(builder, runningByPartition.keySet(), indent, "Running Partitions:");
+        describeTasks(builder, created.values(), indent, "New:");
         return builder.toString();
     }
 
-    void describe(final StringBuilder builder,
-                  final Collection tasks,
-                  final String indent,
-                  final String name) {
+    void describeTasks(final StringBuilder builder,
+                       final Collection tasks,
+                       final String indent,
+                       final String name) {
         builder.append(indent).append(name);
         for (final T t : tasks) {
             builder.append(indent).append(t.toString(indent + "\t\t"));
@@ -162,6 +183,17 @@ void describe(final StringBuilder builder,
         builder.append("\n");
     }
 
+    void describePartitions(final StringBuilder builder,
+                            final Collection partitions,
+                            final String indent,
+                            final String name) {
+        builder.append(indent).append(name);
+        for (final TopicPartition tp : partitions) {
+            builder.append(indent).append(tp.toString());
+        }
+        builder.append("\n");
+    }
+
     List allTasks() {
         final List tasks = new ArrayList<>();
         tasks.addAll(running.values());
@@ -182,18 +214,6 @@ void clear() {
         created.clear();
     }
 
-    boolean isEmpty() throws IllegalStateException {
-        if (running.isEmpty() && !runningByPartition.isEmpty()) {
-            log.error("Assigned stream tasks in an inconsistent state: the set of running tasks is empty but the " +
-                          "running by partitions map contained {}", runningByPartition);
-            throw new IllegalStateException("Found inconsistent state: no tasks running but nonempty runningByPartition");
-        } else {
-            return runningByPartition.isEmpty()
-                       && running.isEmpty()
-                       && created.isEmpty();
-        }
-    }
-
     /**
      * @throws TaskMigratedException if committing offsets failed (non-EOS)
      *                               or if the task producer got fenced (EOS)
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java
index c14f1cca3a0b7..669eabf8bb5fc 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java
@@ -207,7 +207,7 @@ private void startRestoration(final Set initialized,
                         restoreToOffsets.get(partition));
                 restorer.setStartingOffset(restoreConsumer.position(partition));
 
-                log.debug("Calling restorer for partition {} of task {}", partition, active.restoringTaskFor(partition));
+                log.debug("Calling restorer for partition {}", partition);
                 restorer.restoreStarted();
             } else {
                 log.trace("Did not find checkpoint from changelog {} for store {}, rewinding to beginning.", partition, restorer.storeName());
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsRebalanceListener.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsRebalanceListener.java
index a4f1f6a4a900c..9544a5b645183 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsRebalanceListener.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsRebalanceListener.java
@@ -143,8 +143,8 @@ public void onPartitionsLost(final Collection lostPartitions) {
         Set lostTasks = new HashSet<>();
         final long start = time.milliseconds();
         try {
-            // close lost active tasks but don't try to commit offsets as we no longer own them
-            lostTasks = taskManager.closeLostTasks(lostPartitions);
+            // close all active tasks as lost but don't try to commit offsets as we no longer own them
+            lostTasks = taskManager.closeLostTasks();
         } catch (final Throwable t) {
             log.error(
                 "Error caught during partitions lost, " +
@@ -154,7 +154,7 @@ public void onPartitionsLost(final Collection lostPartitions) {
             streamThread.setRebalanceException(t);
         } finally {
             log.info("partitions lost took {} ms.\n" +
-                    "\tsuspended lost active tasks: {}\n",
+                    "\tclosed lost active tasks: {}\n",
                 time.milliseconds() - start,
                 lostTasks);
         }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
index 9c3539e3cd2c2..21126d55249e5 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
@@ -257,56 +257,32 @@ Set suspendActiveTasksAndState(final Collection revokedP
 
     /**
      * Closes active tasks as zombies, as these partitions have been lost and are no longer owned.
+     * NOTE this method assumes that when it is called, EVERY task/partition has been lost and must
+     * be closed as a zombie.
      * @return list of lost tasks
      */
-    Set closeLostTasks(final Collection lostPartitions) {
-        final Set zombieTasks = partitionsToTaskSet(lostPartitions);
-        log.debug("Closing lost tasks as zombies: {}", zombieTasks);
+    Set closeLostTasks() {
+        final Set lostTasks = new HashSet<>(assignedActiveTasks.keySet());
+        log.debug("Closing lost active tasks as zombies: {}", lostTasks);
 
-        final List lostTaskChangelogs = new ArrayList<>();
+        final RuntimeException exception = active.closeAllTasksAsZombies();
 
-        final RuntimeException exception = active.closeZombieTasks(zombieTasks, lostTaskChangelogs);
+        log.debug("Clearing assigned active tasks: {}", assignedActiveTasks);
+        assignedActiveTasks.clear();
 
-        assignedActiveTasks.keySet().removeAll(zombieTasks);
-        changelogReader.remove(lostTaskChangelogs);
-        removeChangelogsFromRestoreConsumer(lostTaskChangelogs, false);
+        log.debug("Clearing the store changelog reader: {}", changelogReader);
+        changelogReader.clear();
 
-        if (exception != null) {
-            throw exception;
-        }
-
-        verifyActiveTaskStateIsEmpty();
-
-        return zombieTasks;
-    }
-
-    private void verifyActiveTaskStateIsEmpty() throws RuntimeException {
-        final AtomicReference firstException = new AtomicReference<>(null);
-
-        // Verify active has no remaining state, and catch if active.isEmpty throws so we can log any non-empty state
-        try {
-            if (!(active.isEmpty())) {
-                log.error("The set of active tasks was non-empty: {}", active);
-                firstException.compareAndSet(null, new IllegalStateException("TaskManager found leftover active task state after closing all zombies"));
-            }
-        } catch (final IllegalStateException e) {
-            firstException.compareAndSet(null, e);
-        }
-
-        if (!(assignedActiveTasks.isEmpty())) {
-            log.error("The set assignedActiveTasks was non-empty: {}", assignedActiveTasks);
-            firstException.compareAndSet(null, new IllegalStateException("TaskManager found leftover assignedActiveTasks after closing all zombies"));
+        if (!restoreConsumerAssignedStandbys) {
+            log.debug("Clearing the restore consumer's assignment: {}", restoreConsumer.assignment());
+            restoreConsumer.unsubscribe();
         }
 
-        if (!(changelogReader.isEmpty())) {
-            log.error("The changelog-reader's internal state was non-empty: {}", changelogReader);
-            firstException.compareAndSet(null, new IllegalStateException("TaskManager found leftover changelog reader state after closing all zombies"));
+        if (exception != null) {
+            throw exception;
         }
 
-        final RuntimeException fatalException = firstException.get();
-        if (fatalException != null) {
-            throw fatalException;
-        }
+        return lostTasks;
     }
 
     void shutdown(final boolean clean) {
@@ -413,6 +389,8 @@ boolean updateNewAndRestoringTasks() {
             final Collection restored = changelogReader.restore(active);
             active.updateRestored(restored);
             removeChangelogsFromRestoreConsumer(restored, false);
+        } else {
+            active.clearRestoringPartitions();
         }
 
         if (active.allTasksRunning()) {
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasksTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasksTest.java
index 42dc58ba628e2..6d98ada278ca3 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasksTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasksTest.java
@@ -84,8 +84,8 @@ public void before() {
     public void shouldInitializeNewTasks() {
         t1.initializeMetadata();
         EasyMock.expect(t1.initializeStateStores()).andReturn(false);
-        EasyMock.expect(t1.partitions()).andReturn(Collections.singleton(tp1));
-        EasyMock.expect(t1.changelogPartitions()).andReturn(Collections.emptySet());
+        EasyMock.expect(t1.partitions()).andReturn(Collections.singleton(tp1)).anyTimes();
+        EasyMock.expect(t1.changelogPartitions()).andReturn(Collections.emptySet()).anyTimes();
         EasyMock.replay(t1);
 
         addAndInitTask();
@@ -99,15 +99,15 @@ public void shouldMoveInitializedTasksNeedingRestoreToRestoring() {
         EasyMock.expect(t1.initializeStateStores()).andReturn(false);
         t1.initializeTopology();
         EasyMock.expectLastCall().once();
-        EasyMock.expect(t1.partitions()).andReturn(Collections.singleton(tp1));
-        EasyMock.expect(t1.changelogPartitions()).andReturn(Collections.emptySet());
+        EasyMock.expect(t1.partitions()).andReturn(Collections.singleton(tp1)).anyTimes();
+        EasyMock.expect(t1.changelogPartitions()).andReturn(Collections.emptySet()).anyTimes();
         t2.initializeMetadata();
         EasyMock.expect(t2.initializeStateStores()).andReturn(true);
         t2.initializeTopology();
         EasyMock.expectLastCall().once();
         final Set t2partitions = Collections.singleton(tp2);
-        EasyMock.expect(t2.partitions()).andReturn(t2partitions);
-        EasyMock.expect(t2.changelogPartitions()).andReturn(Collections.emptyList());
+        EasyMock.expect(t2.partitions()).andReturn(t2partitions).anyTimes();
+        EasyMock.expect(t2.changelogPartitions()).andReturn(Collections.emptyList()).anyTimes();
 
         EasyMock.replay(t1, t2);
 
@@ -127,8 +127,8 @@ public void shouldMoveInitializedTasksThatDontNeedRestoringToRunning() {
         EasyMock.expect(t2.initializeStateStores()).andReturn(true);
         t2.initializeTopology();
         EasyMock.expectLastCall().once();
-        EasyMock.expect(t2.partitions()).andReturn(Collections.singleton(tp2));
-        EasyMock.expect(t2.changelogPartitions()).andReturn(Collections.emptyList());
+        EasyMock.expect(t2.partitions()).andReturn(Collections.singleton(tp2)).anyTimes();
+        EasyMock.expect(t2.changelogPartitions()).andReturn(Collections.emptyList()).anyTimes();
 
         EasyMock.replay(t2);
 
@@ -173,8 +173,8 @@ public void shouldSuspendRunningTasks() {
     public void shouldCloseRestoringTasks() {
         t1.initializeMetadata();
         EasyMock.expect(t1.initializeStateStores()).andReturn(false);
-        EasyMock.expect(t1.partitions()).andReturn(Collections.singleton(tp1)).times(2);
-        EasyMock.expect(t1.changelogPartitions()).andReturn(Collections.emptySet()).times(3);
+        EasyMock.expect(t1.partitions()).andReturn(Collections.singleton(tp1)).anyTimes();
+        EasyMock.expect(t1.changelogPartitions()).andReturn(Collections.emptySet()).anyTimes();
         t1.closeStateManager(true);
         EasyMock.expectLastCall();
         EasyMock.replay(t1);
@@ -186,8 +186,9 @@ public void shouldCloseRestoringTasks() {
     }
 
     @Test
-    public void shouldClosedUnInitializedTasksOnSuspend() {
-        EasyMock.expect(t1.changelogPartitions()).andAnswer(Collections::emptyList);
+    public void shouldCloseUnInitializedTasksOnSuspend() {
+        EasyMock.expect(t1.partitions()).andAnswer(Collections::emptySet).anyTimes();
+        EasyMock.expect(t1.changelogPartitions()).andAnswer(Collections::emptyList).anyTimes();
 
         t1.close(false, false);
         EasyMock.expectLastCall();
@@ -213,8 +214,6 @@ public void shouldNotSuspendSuspendedTasks() {
     @Test
     public void shouldCloseTaskOnSuspendWhenRuntimeException() {
         mockTaskInitialization();
-        EasyMock.expect(t1.partitions()).andReturn(Collections.emptySet()).anyTimes();
-        EasyMock.expect(t1.changelogPartitions()).andReturn(Collections.emptySet()).anyTimes();
 
         t1.suspend();
         EasyMock.expectLastCall().andThrow(new RuntimeException("KABOOM!"));
@@ -232,8 +231,6 @@ public void shouldCloseTaskOnSuspendWhenRuntimeException() {
     @Test
     public void shouldCloseTaskOnSuspendIfTaskMigratedException() {
         mockTaskInitialization();
-        EasyMock.expect(t1.partitions()).andReturn(Collections.emptySet()).anyTimes();
-        EasyMock.expect(t1.changelogPartitions()).andReturn(Collections.emptySet()).anyTimes();
 
         t1.suspend();
         EasyMock.expectLastCall().andThrow(new TaskMigratedException());
@@ -281,8 +278,8 @@ private void mockTaskInitialization() {
         EasyMock.expect(t1.initializeStateStores()).andReturn(true);
         t1.initializeTopology();
         EasyMock.expectLastCall().once();
-        EasyMock.expect(t1.partitions()).andReturn(Collections.singleton(tp1));
-        EasyMock.expect(t1.changelogPartitions()).andReturn(Collections.emptyList());
+        EasyMock.expect(t1.partitions()).andReturn(Collections.singleton(tp1)).anyTimes();
+        EasyMock.expect(t1.changelogPartitions()).andReturn(Collections.emptyList()).anyTimes();
     }
 
     @Test
@@ -537,10 +534,6 @@ public Set taskIds() {
                 return assignedTasks.created.keySet();
             }
 
-            @Override
-            public List expectedLostChangelogs() {
-                return clearingPartitions;
-            }
         }.createTaskAndClear();
     }
 
@@ -549,7 +542,6 @@ public void shouldClearZombieRestoringTasks() {
         new TaskTestSuite() {
             @Override
             public void additionalSetup(final StreamTask task) {
-                EasyMock.expect(task.partitions()).andReturn(Collections.emptySet()).anyTimes();
                 task.closeStateManager(false);
             }
 
@@ -563,10 +555,6 @@ public Set taskIds() {
                 return assignedTasks.restoringTaskIds();
             }
 
-            @Override
-            public List expectedLostChangelogs() {
-                return clearingPartitions;
-            }
         }.createTaskAndClear();
     }
 
@@ -576,7 +564,6 @@ public void shouldClearZombieRunningTasks() {
             @Override
             public void additionalSetup(final StreamTask task) {
                 task.initializeTopology();
-                EasyMock.expect(task.partitions()).andReturn(Collections.emptySet()).anyTimes();
                 task.close(false, true);
             }
 
@@ -590,10 +577,6 @@ public Set taskIds() {
                 return assignedTasks.runningTaskIds();
             }
 
-            @Override
-            public List expectedLostChangelogs() {
-                return clearingPartitions;
-            }
         }.createTaskAndClear();
     }
 
@@ -603,7 +586,6 @@ public void shouldClearZombieSuspendedTasks() {
             @Override
             public void additionalSetup(final StreamTask task) {
                 task.initializeTopology();
-                EasyMock.expect(task.partitions()).andReturn(Collections.emptySet()).anyTimes();
                 task.suspend();
                 task.closeSuspended(false, null);
             }
@@ -621,11 +603,7 @@ public void action(final StreamTask task) {
             public Set taskIds() {
                 return assignedTasks.suspendedTaskIds();
             }
-
-            @Override
-            public List expectedLostChangelogs() {
-                return Collections.emptyList();
-            }
+            
         }.createTaskAndClear();
     }
 
@@ -640,23 +618,21 @@ abstract class TaskTestSuite {
 
         abstract Set taskIds();
 
-        abstract List expectedLostChangelogs();
-
         void createTaskAndClear() {
             final StreamTask task = EasyMock.createMock(StreamTask.class);
             EasyMock.expect(task.id()).andReturn(clearingTaskId).anyTimes();
+            EasyMock.expect(task.partitions()).andReturn(Collections.emptySet()).anyTimes();
             EasyMock.expect(task.changelogPartitions()).andReturn(clearingPartitions).anyTimes();
+            EasyMock.expect(task.toString(EasyMock.anyString())).andReturn("task").anyTimes();
             additionalSetup(task);
             EasyMock.replay(task);
 
             action(task);
-            final List changelogs = new ArrayList<>();
             final Set ids = new HashSet<>(Collections.singleton(task.id()));
             assertEquals(ids, taskIds());
 
-            assignedTasks.closeZombieTasks(ids, changelogs);
+            assignedTasks.closeAllTasksAsZombies();
             assertEquals(Collections.emptySet(), taskIds());
-            assertEquals(expectedLostChangelogs(), changelogs);
         }
     }
 

From d03f52c91558d8bff44522ca5598d673d6c3935c Mon Sep 17 00:00:00 2001
From: Grant Henke 
Date: Thu, 21 Nov 2019 10:06:06 -0600
Subject: [PATCH 0807/1071] KAFKA-1714: Fix gradle wrapper bootstrapping
 (#6031)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Given we need to follow the Apache rule of not checking
any binaries into the source code, Kafka has always had
a bit of a tricky Gradle bootstrap.
Using ./gradlew as users expect doesn’t work and a
local and compatible version of Gradle was required to
generate the wrapper first.

This patch changes the behavior of the wrapper task to
instead generate a gradlew script that can bootstrap the
jar itself. Additionally it adds a license, removes the bat
script, and handles retries.

The documentation in the readme was also updated.

Going forward patches that upgrade gradle should run
`gradle wrapper` before checking in the change.

With this change users using ./gradlew can be sure they
are always building with the correct version of Gradle.

Reviewers: Viktor Somogyi , Ismael Juma  \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m"'
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn () {
+    echo "$*"
+}
+
+die () {
+    echo
+    echo "$*"
+    echo
+    exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "`uname`" in
+  CYGWIN* )
+    cygwin=true
+    ;;
+  Darwin* )
+    darwin=true
+    ;;
+  MINGW* )
+    msys=true
+    ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
+esac
+
+
+# Loop in case we encounter an error.
+for attempt in 1 2 3; do
+  if [ ! -e $APP_HOME/gradle/wrapper/gradle-wrapper.jar ]; then
+    if ! curl -s -S --retry 3 -L -o "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" "https://raw.githubusercontent.com/gradle/gradle/v5.0.0/gradle/wrapper/gradle-wrapper.jar"; then
+      rm -f "$APP_HOME/gradle/wrapper/gradle-wrapper.jar"
+      # Pause for a bit before looping in case the server throttled us.
+      sleep 5
+      continue
+    fi
+  fi
+done
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+        JAVACMD="$JAVA_HOME/bin/java"
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD="java"
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
+    MAX_FD_LIMIT=`ulimit -H -n`
+    if [ $? -eq 0 ] ; then
+        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+            MAX_FD="$MAX_FD_LIMIT"
+        fi
+        ulimit -n $MAX_FD
+        if [ $? -ne 0 ] ; then
+            warn "Could not set maximum file descriptor limit: $MAX_FD"
+        fi
+    else
+        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+    fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin ; then
+    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    JAVACMD=`cygpath --unix "$JAVACMD"`
+
+    # We build the pattern for arguments to be converted via cygpath
+    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+    SEP=""
+    for dir in $ROOTDIRSRAW ; do
+        ROOTDIRS="$ROOTDIRS$SEP$dir"
+        SEP="|"
+    done
+    OURCYGPATTERN="(^($ROOTDIRS))"
+    # Add a user-defined pattern to the cygpath arguments
+    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+    fi
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    i=0
+    for arg in "$@" ; do
+        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
+
+        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
+            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+        else
+            eval `echo args$i`="\"$arg\""
+        fi
+        i=$((i+1))
+    done
+    case $i in
+        (0) set -- ;;
+        (1) set -- "$args0" ;;
+        (2) set -- "$args0" "$args1" ;;
+        (3) set -- "$args0" "$args1" "$args2" ;;
+        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+    esac
+fi
+
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
+}
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+  cd "$(dirname "$0")"
+fi
+
+exec "$JAVACMD" "$@"
diff --git a/wrapper.gradle b/wrapper.gradle
index bc6350632e2a6..3ce451ab4f13d 100644
--- a/wrapper.gradle
+++ b/wrapper.gradle
@@ -17,9 +17,108 @@
  * under the License.
  */
 
-defaultTasks 'downloadWrapper'
+// This file contains tasks for the gradle wrapper generation.
 
-task downloadWrapper(type: Wrapper) {
-    description = "Download the gradle wrapper and requisite files. Overwrites existing wrapper files."
+// Ensure the wrapper script is generated based on the version defined in the project
+// and not the version installed on the machine running the task.
+// Read more about the wrapper here: https://docs.gradle.org/current/userguide/gradle_wrapper.html
+wrapper {
     gradleVersion = project.gradleVersion
-}
\ No newline at end of file
+    distributionType = Wrapper.DistributionType.ALL
+}
+
+def licenseString = """#
+# Copyright 2017 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License."""
+
+// Custom task to inject support for downloading the gradle wrapper jar if it doesn't exist.
+// This allows us to avoid checking in the jar to our repository.
+// Additionally adds a license header to the wrapper while editing the file contents.
+task bootstrapWrapper() {
+    // In the doLast block so this runs when the task is called and not during project configuration.
+    doLast {
+        def wrapperBasePath = "\$APP_HOME/gradle/wrapper"
+        def wrapperJarPath = wrapperBasePath + "/gradle-wrapper.jar"
+
+        // Add a trailing zero to the version if needed.
+        def fullVersion = project.gradleVersion.count(".") == 1 ? "${project.gradleVersion}.0" : versions.gradle
+        // Leverages the wrapper jar checked into the gradle project on github because the jar isn't
+        // available elsewhere. Using raw.githubusercontent.com instead of github.com because
+        // github.com servers deprecated TLSv1/TLSv1.1 support some time ago, so older versions
+        // of curl (built against OpenSSL library that doesn't support TLSv1.2) would fail to
+        // fetch the jar.
+        def wrapperBaseUrl = "https://raw.githubusercontent.com/gradle/gradle/v$fullVersion/gradle/wrapper"
+        def wrapperJarUrl = wrapperBaseUrl + "/gradle-wrapper.jar"
+
+        def bootstrapString = """
+      # Loop in case we encounter an error.
+      for attempt in 1 2 3; do
+        if [ ! -e $wrapperJarPath ]; then
+          if ! curl -s -S --retry 3 -L -o "$wrapperJarPath" "$wrapperJarUrl"; then
+            rm -f "$wrapperJarPath"
+            # Pause for a bit before looping in case the server throttled us.
+            sleep 5
+            continue
+          fi
+        fi
+      done
+      """.stripIndent()
+
+        def wrapperScript = wrapper.scriptFile
+        def wrapperLines = wrapperScript.readLines()
+        wrapperScript.withPrintWriter { out ->
+            def licenseWritten = false
+            def bootstrapWritten = false
+            wrapperLines.each { line ->
+                // Print the wrapper bootstrap before the first usage of the wrapper jar.
+                if (!bootstrapWritten && line.contains("gradle-wrapper.jar")) {
+                    out.println(bootstrapString)
+                    bootstrapWritten = true
+                }
+                out.print(line)
+                // Print the licence after the shebang.
+                if(!licenseWritten && line.contains("#!/usr/bin/env sh")) {
+                    out.println()
+                    out.print(licenseString)
+                    licenseWritten = true
+                }
+                out.println() // New Line
+            }
+        }
+    }
+}
+wrapper.finalizedBy bootstrapWrapper
+
+// Custom task to add a license header to the gradle-wrapper.properties file.
+task bootstrapWrapperProperties() {
+    // In the doLast block so this runs when the task is called and not during project configuration.
+    doLast {
+        def wrapperProperties = wrapper.propertiesFile
+        def wrapperLines = wrapperProperties.readLines()
+        wrapperProperties.withPrintWriter { out ->
+            // Print the license
+            out.println(licenseString)
+            wrapperLines.each { line ->
+                out.println(line)
+            }
+        }
+    }
+}
+wrapper.finalizedBy bootstrapWrapperProperties
+
+// Remove the generated batch file since we don't test building in the Windows environment.
+task removeWindowsScript(type: Delete) {
+    delete "$rootDir/gradlew.bat"
+}
+wrapper.finalizedBy removeWindowsScript

From c19266accbe46ba3fde9d40ee17313e631037463 Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Thu, 21 Nov 2019 08:29:09 -0800
Subject: [PATCH 0808/1071] MINOR: Update gradle wrapper

---
 gradle/wrapper/gradle-wrapper.properties |  2 +-
 gradlew                                  | 24 ++++++++++++++++++++----
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index 3a8f80162ea8a..a3f32a4b1a347 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -14,6 +14,6 @@
 # limitations under the License.
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-5.0-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.2-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
diff --git a/gradlew b/gradlew
index f3276ba494422..db5089da448a3 100755
--- a/gradlew
+++ b/gradlew
@@ -14,6 +14,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+#
+# Copyright 2015 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 ##############################################################################
 ##
 ##  Gradle start up script for UN*X
@@ -42,7 +58,7 @@ APP_NAME="Gradle"
 APP_BASE_NAME=`basename "$0"`
 
 # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS='"-Xmx64m"'
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
@@ -82,7 +98,7 @@ esac
 # Loop in case we encounter an error.
 for attempt in 1 2 3; do
   if [ ! -e $APP_HOME/gradle/wrapper/gradle-wrapper.jar ]; then
-    if ! curl -s -S --retry 3 -L -o "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" "https://raw.githubusercontent.com/gradle/gradle/v5.0.0/gradle/wrapper/gradle-wrapper.jar"; then
+    if ! curl -s -S --retry 3 -L -o "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" "https://raw.githubusercontent.com/gradle/gradle/v5.6.2/gradle/wrapper/gradle-wrapper.jar"; then
       rm -f "$APP_HOME/gradle/wrapper/gradle-wrapper.jar"
       # Pause for a bit before looping in case the server throttled us.
       sleep 5
@@ -136,8 +152,8 @@ if $darwin; then
     GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
 fi
 
-# For Cygwin, switch paths to Windows format before running java
-if $cygwin ; then
+# For Cygwin or MSYS, switch paths to Windows format before running java
+if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
     JAVACMD=`cygpath --unix "$JAVACMD"`

From 0bc40c63e1b5f8c6385e1202407ef026644a97a8 Mon Sep 17 00:00:00 2001
From: Bruno Cadonna 
Date: Thu, 21 Nov 2019 16:32:31 +0100
Subject: [PATCH 0809/1071] MINOR: Fix Streams EOS system tests by adding
 clean-up of state dir (#7693)

Recently, system tests test_rebalance_[simple|complex] failed
repeatedly with a verfication error. The cause was most probably
the missing clean-up of a state directory of one of the processors.

A node is cleaned up when a service on that node is started and when
a test is torn down.

If the clean-up flag clean_node_enabled of a EOS Streams service is
unset, the clean-up of the node is skipped.

The clean-up flag of processor1 in the EOS tests should stay set before
its first start, so that the node is cleaned before the service is started.
Afterwards for the multiple restarts of processor1 the cleans-up flag should
be unset to re-use the local state.

After the multiple restarts are done, the clean-up flag of processor1 should
again be set to trigger node clean-up during the test teardown.

A dirty node can lead to test failures when tests from Streams EOS tests are
scheduled on the same node, because the state store would not start empty
since it reads the local state that was not cleaned up.

Reviewers: Matthias J. Sax , Andrew Choi , Bill Bejeck 
---
 tests/kafkatest/tests/streams/streams_eos_test.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/kafkatest/tests/streams/streams_eos_test.py b/tests/kafkatest/tests/streams/streams_eos_test.py
index 428db9be1b809..117b4484c4dff 100644
--- a/tests/kafkatest/tests/streams/streams_eos_test.py
+++ b/tests/kafkatest/tests/streams/streams_eos_test.py
@@ -60,9 +60,8 @@ def run_rebalance(self, processor1, processor2, processor3, verifier):
 
         self.driver.start()
 
-        processor1.clean_node_enabled = False
-
         self.add_streams(processor1)
+        processor1.clean_node_enabled = False
         self.add_streams2(processor1, processor2)
         self.add_streams3(processor1, processor2, processor3)
         self.stop_streams3(processor2, processor3, processor1)
@@ -70,6 +69,7 @@ def run_rebalance(self, processor1, processor2, processor3, verifier):
         self.stop_streams3(processor1, processor3, processor2)
         self.stop_streams2(processor1, processor3)
         self.stop_streams(processor1)
+        processor1.clean_node_enabled = True
 
         self.driver.stop()
 
@@ -100,9 +100,8 @@ def run_failure_and_recovery(self, processor1, processor2, processor3, verifier)
 
         self.driver.start()
 
-        processor1.clean_node_enabled = False
-
         self.add_streams(processor1)
+        processor1.clean_node_enabled = False
         self.add_streams2(processor1, processor2)
         self.add_streams3(processor1, processor2, processor3)
         self.abort_streams(processor2, processor3, processor1)
@@ -112,6 +111,7 @@ def run_failure_and_recovery(self, processor1, processor2, processor3, verifier)
         self.abort_streams(processor1, processor3, processor2)
         self.stop_streams2(processor1, processor3)
         self.stop_streams(processor1)
+        processor1.clean_node_enabled = True
 
         self.driver.stop()
 

From 0f9381976a34e845df38005cd5402494d63aa5bf Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Thu, 21 Nov 2019 10:51:55 -0800
Subject: [PATCH 0810/1071] MINOR: Rat should ignore generated directories
 (#7729)

For some reason, PR builds are failing due to the `rat` license
check even though it should ignore files included in `.gitignore`.

Reviewers: Jason Gustafson 
---
 build.gradle | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index c55a2af4c8d55..121d1eb2f9d99 100644
--- a/build.gradle
+++ b/build.gradle
@@ -136,7 +136,8 @@ if (file('.git').exists()) {
         '**/id_rsa.pub',
         'checkstyle/suppressions.xml',
         'streams/quickstart/java/src/test/resources/projects/basic/goal.txt',
-        'streams/streams-scala/logs/*'
+        'streams/streams-scala/logs/*',
+        '**/generated/**'
     ])
   }
 }

From a7485bac5f6d030a7c8d08c63a05b49b1c7bafe7 Mon Sep 17 00:00:00 2001
From: Chris Egerton 
Date: Tue, 19 Nov 2019 21:18:21 -0800
Subject: [PATCH 0811/1071] KAFKA-9051: Prematurely complete source offset read
 requests for stopped tasks (#7532)

Prematurely complete source offset read requests for stopped tasks, and added unit tests.

Author: Chris Egerton 
Reviewers: Arjun Satish , Nigel Liang , Jinxin Liu , Randall Hauch 
---
 .../apache/kafka/connect/runtime/Worker.java  |   4 +-
 .../connect/runtime/WorkerSourceTask.java     |  12 +-
 .../storage/CloseableOffsetStorageReader.java |  33 +++
 .../storage/KafkaOffsetBackingStore.java      |   7 +-
 .../storage/MemoryOffsetBackingStore.java     |   6 +-
 .../connect/storage/OffsetBackingStore.java   |   8 +-
 .../storage/OffsetStorageReaderImpl.java      |  51 +++-
 .../util/ConvertingFutureCallback.java        |  58 ++++-
 .../runtime/ErrorHandlingTaskTest.java        |   4 +-
 .../connect/runtime/WorkerSourceTaskTest.java |  18 +-
 .../storage/FileOffsetBackingStoreTest.java   |  14 +-
 .../storage/KafkaOffsetBackingStoreTest.java  |  54 +---
 .../util/ConvertingFutureCallbackTest.java    | 242 ++++++++++++++++++
 13 files changed, 420 insertions(+), 91 deletions(-)
 create mode 100644 connect/runtime/src/main/java/org/apache/kafka/connect/storage/CloseableOffsetStorageReader.java
 create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/util/ConvertingFutureCallbackTest.java

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
index 814c750b35c96..ff5eee3d5fd6a 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
@@ -50,10 +50,10 @@
 import org.apache.kafka.connect.sink.SinkTask;
 import org.apache.kafka.connect.source.SourceRecord;
 import org.apache.kafka.connect.source.SourceTask;
+import org.apache.kafka.connect.storage.CloseableOffsetStorageReader;
 import org.apache.kafka.connect.storage.Converter;
 import org.apache.kafka.connect.storage.HeaderConverter;
 import org.apache.kafka.connect.storage.OffsetBackingStore;
-import org.apache.kafka.connect.storage.OffsetStorageReader;
 import org.apache.kafka.connect.storage.OffsetStorageReaderImpl;
 import org.apache.kafka.connect.storage.OffsetStorageWriter;
 import org.apache.kafka.connect.util.ConnectorTaskId;
@@ -512,7 +512,7 @@ private WorkerTask buildWorkerTask(ClusterConfigState configState,
             retryWithToleranceOperator.reporters(sourceTaskReporters(id, connConfig, errorHandlingMetrics));
             TransformationChain transformationChain = new TransformationChain<>(connConfig.transformations(), retryWithToleranceOperator);
             log.info("Initializing: {}", transformationChain);
-            OffsetStorageReader offsetReader = new OffsetStorageReaderImpl(offsetBackingStore, id.connector(),
+            CloseableOffsetStorageReader offsetReader = new OffsetStorageReaderImpl(offsetBackingStore, id.connector(),
                     internalKeyConverter, internalValueConverter);
             OffsetStorageWriter offsetWriter = new OffsetStorageWriter(offsetBackingStore, id.connector(),
                     internalKeyConverter, internalValueConverter);
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
index 5e24d4145aef2..cea99ee79744a 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
@@ -39,9 +39,9 @@
 import org.apache.kafka.connect.runtime.errors.Stage;
 import org.apache.kafka.connect.source.SourceRecord;
 import org.apache.kafka.connect.source.SourceTask;
+import org.apache.kafka.connect.storage.CloseableOffsetStorageReader;
 import org.apache.kafka.connect.storage.Converter;
 import org.apache.kafka.connect.storage.HeaderConverter;
-import org.apache.kafka.connect.storage.OffsetStorageReader;
 import org.apache.kafka.connect.storage.OffsetStorageWriter;
 import org.apache.kafka.connect.util.ConnectUtils;
 import org.apache.kafka.connect.util.ConnectorTaskId;
@@ -75,7 +75,7 @@ class WorkerSourceTask extends WorkerTask {
     private final HeaderConverter headerConverter;
     private final TransformationChain transformationChain;
     private KafkaProducer producer;
-    private final OffsetStorageReader offsetReader;
+    private final CloseableOffsetStorageReader offsetReader;
     private final OffsetStorageWriter offsetWriter;
     private final Time time;
     private final SourceTaskMetricsGroup sourceTaskMetricsGroup;
@@ -105,7 +105,7 @@ public WorkerSourceTask(ConnectorTaskId id,
                             HeaderConverter headerConverter,
                             TransformationChain transformationChain,
                             KafkaProducer producer,
-                            OffsetStorageReader offsetReader,
+                            CloseableOffsetStorageReader offsetReader,
                             OffsetStorageWriter offsetWriter,
                             WorkerConfig workerConfig,
                             ClusterConfigState configState,
@@ -172,6 +172,12 @@ protected void releaseResources() {
         sourceTaskMetricsGroup.close();
     }
 
+    @Override
+    public void cancel() {
+        super.cancel();
+        offsetReader.close();
+    }
+
     @Override
     public void stop() {
         super.stop();
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/CloseableOffsetStorageReader.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/CloseableOffsetStorageReader.java
new file mode 100644
index 0000000000000..b90273936bfb0
--- /dev/null
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/CloseableOffsetStorageReader.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.storage;
+
+import java.io.Closeable;
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.Future;
+
+public interface CloseableOffsetStorageReader extends Closeable, OffsetStorageReader {
+
+    /**
+     * {@link Future#cancel(boolean) Cancel} all outstanding offset read requests, and throw an
+     * exception in all current and future calls to {@link #offsets(Collection)} and
+     * {@link #offset(Map)}. This is useful for unblocking task threads which need to shut down but
+     * are blocked on offset reads.
+     */
+    void close();
+}
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStore.java
index 195c498edb76f..22bcb7a5ae2e6 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStore.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStore.java
@@ -118,9 +118,8 @@ public void stop() {
     }
 
     @Override
-    public Future> get(final Collection keys,
-                                                   final Callback> callback) {
-        ConvertingFutureCallback> future = new ConvertingFutureCallback>(callback) {
+    public Future> get(final Collection keys) {
+        ConvertingFutureCallback> future = new ConvertingFutureCallback>() {
             @Override
             public Map convert(Void result) {
                 Map values = new HashMap<>();
@@ -230,6 +229,4 @@ public synchronized Void get(long timeout, TimeUnit unit) throws InterruptedExce
             return null;
         }
     }
-
-
 }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/MemoryOffsetBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/MemoryOffsetBackingStore.java
index ab8130b6ef776..72439e7d687b3 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/MemoryOffsetBackingStore.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/MemoryOffsetBackingStore.java
@@ -75,9 +75,7 @@ public void stop() {
     }
 
     @Override
-    public Future> get(
-            final Collection keys,
-            final Callback> callback) {
+    public Future> get(final Collection keys) {
         return executor.submit(new Callable>() {
             @Override
             public Map call() throws Exception {
@@ -85,8 +83,6 @@ public Map call() throws Exception {
                 for (ByteBuffer key : keys) {
                     result.put(key, data.get(key));
                 }
-                if (callback != null)
-                    callback.onCompletion(null, result);
                 return result;
             }
         });
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/OffsetBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/OffsetBackingStore.java
index 9998164ddf5bf..1e4375b7d8eff 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/OffsetBackingStore.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/OffsetBackingStore.java
@@ -53,12 +53,9 @@ public interface OffsetBackingStore {
     /**
      * Get the values for the specified keys
      * @param keys list of keys to look up
-     * @param callback callback to invoke on completion
      * @return future for the resulting map from key to value
      */
-    Future> get(
-            Collection keys,
-            Callback> callback);
+    Future> get(Collection keys);
 
     /**
      * Set the specified keys and values.
@@ -66,8 +63,7 @@ Future> get(
      * @param callback callback to invoke on completion
      * @return void future for the operation
      */
-    Future set(Map values,
-                            Callback callback);
+    Future set(Map values, Callback callback);
 
     /**
      * Configure class with the given key-value pairs
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/OffsetStorageReaderImpl.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/OffsetStorageReaderImpl.java
index 9f926dc5040aa..a1eea43103a39 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/OffsetStorageReaderImpl.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/OffsetStorageReaderImpl.java
@@ -26,20 +26,27 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * Implementation of OffsetStorageReader. Unlike OffsetStorageWriter which is implemented
  * directly, the interface is only separate from this implementation because it needs to be
  * included in the public API package.
  */
-public class OffsetStorageReaderImpl implements OffsetStorageReader {
+public class OffsetStorageReaderImpl implements CloseableOffsetStorageReader {
     private static final Logger log = LoggerFactory.getLogger(OffsetStorageReaderImpl.class);
 
     private final OffsetBackingStore backingStore;
     private final String namespace;
     private final Converter keyConverter;
     private final Converter valueConverter;
+    private final AtomicBoolean closed;
+    private final Set>> offsetReadFutures;
 
     public OffsetStorageReaderImpl(OffsetBackingStore backingStore, String namespace,
                                    Converter keyConverter, Converter valueConverter) {
@@ -47,6 +54,8 @@ public OffsetStorageReaderImpl(OffsetBackingStore backingStore, String namespace
         this.namespace = namespace;
         this.keyConverter = keyConverter;
         this.valueConverter = valueConverter;
+        this.closed = new AtomicBoolean(false);
+        this.offsetReadFutures = new HashSet<>();
     }
 
     @Override
@@ -76,7 +85,30 @@ public  Map, Map> offsets(Collection serialized value from backing store
         Map raw;
         try {
-            raw = backingStore.get(serializedToOriginal.keySet(), null).get();
+            Future> offsetReadFuture;
+            synchronized (offsetReadFutures) {
+                if (closed.get()) {
+                    throw new ConnectException(
+                        "Offset reader is closed. This is likely because the task has already been "
+                            + "scheduled to stop but has taken longer than the graceful shutdown "
+                            + "period to do so.");
+                }
+                offsetReadFuture = backingStore.get(serializedToOriginal.keySet());
+                offsetReadFutures.add(offsetReadFuture);
+            }
+
+            try {
+                raw = offsetReadFuture.get();
+            } catch (CancellationException e) {
+                throw new ConnectException(
+                    "Offset reader closed while attempting to read offsets. This is likely because "
+                        + "the task was been scheduled to stop but has taken longer than the "
+                        + "graceful shutdown period to do so.");
+            } finally {
+                synchronized (offsetReadFutures) {
+                    offsetReadFutures.remove(offsetReadFuture);
+                }
+            }
         } catch (Exception e) {
             log.error("Failed to fetch offsets from namespace {}: ", namespace, e);
             throw new ConnectException("Failed to fetch offsets.", e);
@@ -108,4 +140,19 @@ public  Map, Map> offsets(Collection> offsetReadFuture : offsetReadFutures) {
+                    try {
+                        offsetReadFuture.cancel(true);
+                    } catch (Throwable t) {
+                        log.error("Failed to cancel offset read future", t);
+                    }
+                }
+                offsetReadFutures.clear();
+            }
+        }
+    }
 }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/util/ConvertingFutureCallback.java b/connect/runtime/src/main/java/org/apache/kafka/connect/util/ConvertingFutureCallback.java
index d5abed9385cc0..e15c38ea4c4ae 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/util/ConvertingFutureCallback.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/util/ConvertingFutureCallback.java
@@ -16,6 +16,9 @@
  */
 package org.apache.kafka.connect.util;
 
+import org.apache.kafka.connect.errors.ConnectException;
+
+import java.util.concurrent.CancellationException;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
@@ -24,10 +27,15 @@
 
 public abstract class ConvertingFutureCallback implements Callback, Future {
 
-    private Callback underlying;
-    private CountDownLatch finishedLatch;
-    private T result = null;
-    private Throwable exception = null;
+    private final Callback underlying;
+    private final CountDownLatch finishedLatch;
+    private volatile T result = null;
+    private volatile Throwable exception = null;
+    private volatile boolean cancelled = false;
+
+    public ConvertingFutureCallback() {
+        this(null);
+    }
 
     public ConvertingFutureCallback(Callback underlying) {
         this.underlying = underlying;
@@ -38,21 +46,46 @@ public ConvertingFutureCallback(Callback underlying) {
 
     @Override
     public void onCompletion(Throwable error, U result) {
-        this.exception = error;
-        this.result = convert(result);
-        if (underlying != null)
-            underlying.onCompletion(error, this.result);
-        finishedLatch.countDown();
+        synchronized (this) {
+            if (isDone()) {
+                return;
+            }
+            
+            if (error != null) {
+                this.exception = error;
+            } else {
+                this.result = convert(result);
+            }
+
+            if (underlying != null)
+                underlying.onCompletion(error, this.result);
+            finishedLatch.countDown();
+        }
     }
 
     @Override
-    public boolean cancel(boolean b) {
+    public boolean cancel(boolean mayInterruptIfRunning) {
+        synchronized (this) {
+            if (isDone()) {
+                return false;
+            }
+            if (mayInterruptIfRunning) {
+                this.cancelled = true;
+                finishedLatch.countDown();
+                return true;
+            }
+        }
+        try {
+            finishedLatch.await();
+        } catch (InterruptedException e) {
+            throw new ConnectException("Interrupted while waiting for task to complete", e);
+        }
         return false;
     }
 
     @Override
     public boolean isCancelled() {
-        return false;
+        return cancelled;
     }
 
     @Override
@@ -75,6 +108,9 @@ public T get(long l, TimeUnit timeUnit)
     }
 
     private T result() throws ExecutionException {
+        if (cancelled) {
+            throw new CancellationException();
+        }
         if (exception != null) {
             throw new ExecutionException(exception);
         }
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java
index 5d223f4713ec0..428b3e4f022f5 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java
@@ -46,7 +46,7 @@
 import org.apache.kafka.connect.source.SourceTask;
 import org.apache.kafka.connect.storage.Converter;
 import org.apache.kafka.connect.storage.HeaderConverter;
-import org.apache.kafka.connect.storage.OffsetStorageReader;
+import org.apache.kafka.connect.storage.OffsetStorageReaderImpl;
 import org.apache.kafka.connect.storage.OffsetStorageWriter;
 import org.apache.kafka.connect.transforms.Transformation;
 import org.apache.kafka.connect.transforms.util.SimpleConfig;
@@ -127,7 +127,7 @@ public class ErrorHandlingTaskTest {
     private KafkaProducer producer;
 
     @Mock
-    OffsetStorageReader offsetReader;
+    OffsetStorageReaderImpl offsetReader;
     @Mock
     OffsetStorageWriter offsetWriter;
 
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java
index e8039310b20da..ea3716d207111 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java
@@ -41,9 +41,9 @@
 import org.apache.kafka.connect.source.SourceRecord;
 import org.apache.kafka.connect.source.SourceTask;
 import org.apache.kafka.connect.source.SourceTaskContext;
+import org.apache.kafka.connect.storage.CloseableOffsetStorageReader;
 import org.apache.kafka.connect.storage.Converter;
 import org.apache.kafka.connect.storage.HeaderConverter;
-import org.apache.kafka.connect.storage.OffsetStorageReader;
 import org.apache.kafka.connect.storage.OffsetStorageWriter;
 import org.apache.kafka.connect.storage.StringConverter;
 import org.apache.kafka.connect.util.Callback;
@@ -114,7 +114,7 @@ public class WorkerSourceTaskTest extends ThreadedTest {
     @Mock private HeaderConverter headerConverter;
     @Mock private TransformationChain transformationChain;
     @Mock private KafkaProducer producer;
-    @Mock private OffsetStorageReader offsetReader;
+    @Mock private CloseableOffsetStorageReader offsetReader;
     @Mock private OffsetStorageWriter offsetWriter;
     @Mock private ClusterConfigState clusterConfigState;
     private WorkerSourceTask workerTask;
@@ -693,6 +693,20 @@ public Object answer() throws Throwable {
         PowerMock.verifyAll();
     }
 
+    @Test
+    public void testCancel() {
+        createWorkerTask();
+
+        offsetReader.close();
+        PowerMock.expectLastCall();
+
+        PowerMock.replayAll();
+
+        workerTask.cancel();
+
+        PowerMock.verifyAll();
+    }
+
     @Test
     public void testMetricsGroup() {
         SourceTaskMetricsGroup group = new SourceTaskMetricsGroup(taskId, metrics);
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/FileOffsetBackingStoreTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/FileOffsetBackingStoreTest.java
index df955f8813d53..1fdb91a2dcb2d 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/FileOffsetBackingStoreTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/FileOffsetBackingStoreTest.java
@@ -71,12 +71,11 @@ public void teardown() {
     @Test
     public void testGetSet() throws Exception {
         Callback setCallback = expectSuccessfulSetCallback();
-        Callback> getCallback = expectSuccessfulGetCallback();
         PowerMock.replayAll();
 
         store.set(firstSet, setCallback).get();
 
-        Map values = store.get(Arrays.asList(buffer("key"), buffer("bad")), getCallback).get();
+        Map values = store.get(Arrays.asList(buffer("key"), buffer("bad"))).get();
         assertEquals(buffer("value"), values.get(buffer("key")));
         assertEquals(null, values.get(buffer("bad")));
 
@@ -86,7 +85,6 @@ public void testGetSet() throws Exception {
     @Test
     public void testSaveRestore() throws Exception {
         Callback setCallback = expectSuccessfulSetCallback();
-        Callback> getCallback = expectSuccessfulGetCallback();
         PowerMock.replayAll();
 
         store.set(firstSet, setCallback).get();
@@ -96,7 +94,7 @@ public void testSaveRestore() throws Exception {
         FileOffsetBackingStore restore = new FileOffsetBackingStore();
         restore.configure(config);
         restore.start();
-        Map values = restore.get(Arrays.asList(buffer("key")), getCallback).get();
+        Map values = restore.get(Arrays.asList(buffer("key"))).get();
         assertEquals(buffer("value"), values.get(buffer("key")));
 
         PowerMock.verifyAll();
@@ -113,12 +111,4 @@ private Callback expectSuccessfulSetCallback() {
         PowerMock.expectLastCall();
         return setCallback;
     }
-
-    @SuppressWarnings("unchecked")
-    private Callback> expectSuccessfulGetCallback() {
-        Callback> getCallback = PowerMock.createMock(Callback.class);
-        getCallback.onCompletion(EasyMock.isNull(Throwable.class), EasyMock.anyObject(Map.class));
-        PowerMock.expectLastCall();
-        return getCallback;
-    }
 }
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStoreTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStoreTest.java
index 1df63f35e8a76..c04363b974605 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStoreTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStoreTest.java
@@ -217,17 +217,10 @@ public Object answer() throws Throwable {
         store.start();
 
         // Getting from empty store should return nulls
-        final AtomicBoolean getInvokedAndPassed = new AtomicBoolean(false);
-        store.get(Arrays.asList(TP0_KEY, TP1_KEY), new Callback>() {
-            @Override
-            public void onCompletion(Throwable error, Map result) {
-                // Since we didn't read them yet, these will be null
-                assertEquals(null, result.get(TP0_KEY));
-                assertEquals(null, result.get(TP1_KEY));
-                getInvokedAndPassed.set(true);
-            }
-        }).get(10000, TimeUnit.MILLISECONDS);
-        assertTrue(getInvokedAndPassed.get());
+        Map offsets = store.get(Arrays.asList(TP0_KEY, TP1_KEY)).get(10000, TimeUnit.MILLISECONDS);
+        // Since we didn't read them yet, these will be null
+        assertNull(offsets.get(TP0_KEY));
+        assertNull(offsets.get(TP1_KEY));
 
         // Set some offsets
         Map toSet = new HashMap<>();
@@ -250,28 +243,14 @@ public void onCompletion(Throwable error, Void result) {
         assertTrue(invoked.get());
 
         // Getting data should read to end of our published data and return it
-        final AtomicBoolean secondGetInvokedAndPassed = new AtomicBoolean(false);
-        store.get(Arrays.asList(TP0_KEY, TP1_KEY), new Callback>() {
-            @Override
-            public void onCompletion(Throwable error, Map result) {
-                assertEquals(TP0_VALUE, result.get(TP0_KEY));
-                assertEquals(TP1_VALUE, result.get(TP1_KEY));
-                secondGetInvokedAndPassed.set(true);
-            }
-        }).get(10000, TimeUnit.MILLISECONDS);
-        assertTrue(secondGetInvokedAndPassed.get());
+        offsets = store.get(Arrays.asList(TP0_KEY, TP1_KEY)).get(10000, TimeUnit.MILLISECONDS);
+        assertEquals(TP0_VALUE, offsets.get(TP0_KEY));
+        assertEquals(TP1_VALUE, offsets.get(TP1_KEY));
 
         // Getting data should read to end of our published data and return it
-        final AtomicBoolean thirdGetInvokedAndPassed = new AtomicBoolean(false);
-        store.get(Arrays.asList(TP0_KEY, TP1_KEY), new Callback>() {
-            @Override
-            public void onCompletion(Throwable error, Map result) {
-                assertEquals(TP0_VALUE_NEW, result.get(TP0_KEY));
-                assertEquals(TP1_VALUE_NEW, result.get(TP1_KEY));
-                thirdGetInvokedAndPassed.set(true);
-            }
-        }).get(10000, TimeUnit.MILLISECONDS);
-        assertTrue(thirdGetInvokedAndPassed.get());
+        offsets = store.get(Arrays.asList(TP0_KEY, TP1_KEY)).get(10000, TimeUnit.MILLISECONDS);
+        assertEquals(TP0_VALUE_NEW, offsets.get(TP0_KEY));
+        assertEquals(TP1_VALUE_NEW, offsets.get(TP1_KEY));
 
         store.stop();
 
@@ -329,16 +308,9 @@ public void onCompletion(Throwable error, Void result) {
         assertTrue(invoked.get());
 
         // Getting data should read to end of our published data and return it
-        final AtomicBoolean secondGetInvokedAndPassed = new AtomicBoolean(false);
-        store.get(Arrays.asList(null, TP1_KEY), new Callback>() {
-            @Override
-            public void onCompletion(Throwable error, Map result) {
-                assertEquals(TP0_VALUE, result.get(null));
-                assertNull(result.get(TP1_KEY));
-                secondGetInvokedAndPassed.set(true);
-            }
-        }).get(10000, TimeUnit.MILLISECONDS);
-        assertTrue(secondGetInvokedAndPassed.get());
+        Map offsets = store.get(Arrays.asList(null, TP1_KEY)).get(10000, TimeUnit.MILLISECONDS);
+        assertEquals(TP0_VALUE, offsets.get(null));
+        assertNull(offsets.get(TP1_KEY));
 
         store.stop();
 
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/ConvertingFutureCallbackTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/ConvertingFutureCallbackTest.java
new file mode 100644
index 0000000000000..9535003c863c3
--- /dev/null
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/ConvertingFutureCallbackTest.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.util;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class ConvertingFutureCallbackTest {
+
+    private ExecutorService executor;
+
+    @Before
+    public void setup() {
+        executor = Executors.newSingleThreadExecutor();
+    }
+  
+    @Test
+    public void shouldConvertBeforeGetOnSuccessfulCompletion() throws Exception {
+        final Object expectedConversion = new Object();
+        TestConvertingFutureCallback testCallback = new TestConvertingFutureCallback();
+        testCallback.onCompletion(null, expectedConversion);
+        assertEquals(1, testCallback.numberOfConversions());
+        assertEquals(expectedConversion, testCallback.get());
+    }
+
+    @Test
+    public void shouldConvertOnlyOnceBeforeGetOnSuccessfulCompletion() throws Exception {
+        final Object expectedConversion = new Object();
+        TestConvertingFutureCallback testCallback = new TestConvertingFutureCallback();
+        testCallback.onCompletion(null, expectedConversion);
+        testCallback.onCompletion(null, 69);
+        testCallback.cancel(true);
+        testCallback.onCompletion(new RuntimeException(), null);
+        assertEquals(1, testCallback.numberOfConversions());
+        assertEquals(expectedConversion, testCallback.get());
+    }
+
+    @Test
+    public void shouldNotConvertBeforeGetOnFailedCompletion() throws Exception {
+        final Throwable expectedError = new Throwable();
+        TestConvertingFutureCallback testCallback = new TestConvertingFutureCallback();
+        testCallback.onCompletion(expectedError, null);
+        assertEquals(0, testCallback.numberOfConversions());
+        try {
+            testCallback.get();
+            fail("Expected ExecutionException");
+        } catch (ExecutionException e) {
+            assertEquals(expectedError, e.getCause());
+        }
+    }
+
+    @Test
+    public void shouldRecordOnlyFirstErrorBeforeGetOnFailedCompletion() throws Exception {
+        final Throwable expectedError = new Throwable();
+        TestConvertingFutureCallback testCallback = new TestConvertingFutureCallback();
+        testCallback.onCompletion(expectedError, null);
+        testCallback.onCompletion(new RuntimeException(), null);
+        testCallback.cancel(true);
+        testCallback.onCompletion(null, "420");
+        assertEquals(0, testCallback.numberOfConversions());
+        try {
+            testCallback.get();
+            fail("Expected ExecutionException");
+        } catch (ExecutionException e) {
+            assertEquals(expectedError, e.getCause());
+        }
+    }
+  
+    @Test(expected = CancellationException.class)
+    public void shouldCancelBeforeGetIfMayCancelWhileRunning() throws Exception {
+        TestConvertingFutureCallback testCallback = new TestConvertingFutureCallback();
+        assertTrue(testCallback.cancel(true));
+        testCallback.get();
+    }
+
+    @Test
+    public void shouldBlockUntilSuccessfulCompletion() throws Exception {
+        AtomicReference testThreadException = new AtomicReference<>();
+        TestConvertingFutureCallback testCallback = new TestConvertingFutureCallback();
+        final Object expectedConversion = new Object();
+        executor.submit(() -> {
+            try {
+                testCallback.waitForGet();
+                testCallback.onCompletion(null, expectedConversion);
+            } catch (Exception e) {
+                testThreadException.compareAndSet(null, e);
+            }
+        });
+        assertFalse(testCallback.isDone());
+        assertEquals(expectedConversion, testCallback.get());
+        assertEquals(1, testCallback.numberOfConversions());
+        assertTrue(testCallback.isDone());
+        if (testThreadException.get() != null) {
+            throw testThreadException.get();
+        }
+    }
+
+    @Test
+    public void shouldBlockUntilFailedCompletion() throws Exception {
+        AtomicReference testThreadException = new AtomicReference<>();
+        TestConvertingFutureCallback testCallback = new TestConvertingFutureCallback();
+        final Throwable expectedError = new Throwable();
+        executor.submit(() -> {
+            try {
+                testCallback.waitForGet();
+                testCallback.onCompletion(expectedError, null);
+            } catch (Exception e) {
+                testThreadException.compareAndSet(null, e);
+            }
+        });
+        assertFalse(testCallback.isDone());
+        try {
+            testCallback.get();
+            fail("Expected ExecutionException");
+        } catch (ExecutionException e) {
+            assertEquals(expectedError, e.getCause());
+        }
+        assertEquals(0, testCallback.numberOfConversions());
+        assertTrue(testCallback.isDone());
+        if (testThreadException.get() != null) {
+            throw testThreadException.get();
+        }
+    }
+
+    @Test(expected = CancellationException.class)
+    public void shouldBlockUntilCancellation() throws Exception {
+        AtomicReference testThreadException = new AtomicReference<>();
+        TestConvertingFutureCallback testCallback = new TestConvertingFutureCallback();
+        executor.submit(() -> {
+            try {
+                testCallback.waitForGet();
+                testCallback.cancel(true);
+            } catch (Exception e) {
+                testThreadException.compareAndSet(null, e);
+            }
+        });
+        assertFalse(testCallback.isDone());
+        testCallback.get();
+        if (testThreadException.get() != null) {
+            throw testThreadException.get();
+        }
+    }
+
+    @Test
+    public void shouldNotCancelIfMayNotCancelWhileRunning() throws Exception {
+        AtomicReference testThreadException = new AtomicReference<>();
+        TestConvertingFutureCallback testCallback = new TestConvertingFutureCallback();
+        final Object expectedConversion = new Object();
+        executor.submit(() -> {
+            try {
+                testCallback.waitForCancel();
+                testCallback.onCompletion(null, expectedConversion);
+            } catch (Exception e) {
+                testThreadException.compareAndSet(null, e);
+            }
+        });
+        assertFalse(testCallback.isCancelled());
+        assertFalse(testCallback.isDone());
+        testCallback.cancel(false);
+        assertFalse(testCallback.isCancelled());
+        assertTrue(testCallback.isDone());
+        assertEquals(expectedConversion, testCallback.get());
+        assertEquals(1, testCallback.numberOfConversions());
+        if (testThreadException.get() != null) {
+            throw testThreadException.get();
+        }
+    }
+  
+    protected static class TestConvertingFutureCallback extends ConvertingFutureCallback {
+        private AtomicInteger numberOfConversions = new AtomicInteger();
+        private CountDownLatch getInvoked = new CountDownLatch(1);
+        private CountDownLatch cancelInvoked = new CountDownLatch(1);
+    
+        public int numberOfConversions() {
+            return numberOfConversions.get();
+        }
+
+        public void waitForGet() throws InterruptedException {
+            getInvoked.await();
+        }
+
+        public void waitForCancel() throws InterruptedException {
+            cancelInvoked.await();
+        }
+    
+        @Override
+        public Object convert(Object result) {
+            numberOfConversions.incrementAndGet();
+            return result;
+        }
+
+        @Override
+        public Object get() throws InterruptedException, ExecutionException {
+            getInvoked.countDown();
+            return super.get();
+        }
+
+        @Override
+        public Object get(
+            long duration,
+            TimeUnit unit
+        ) throws InterruptedException, ExecutionException, TimeoutException {
+            getInvoked.countDown();
+            return super.get(duration, unit);
+        }
+
+        @Override
+        public boolean cancel(boolean mayInterruptIfRunning) {
+            cancelInvoked.countDown();
+            return super.cancel(mayInterruptIfRunning);
+        }
+    }
+}

From ff12de576c64596b29320fe4998e1444e4b8d48b Mon Sep 17 00:00:00 2001
From: Bill Bejeck 
Date: Fri, 22 Nov 2019 16:25:57 -0500
Subject: [PATCH 0812/1071] MINOR: Updated StreamTableJoinIntegrationTest to
 use TTD (#7722)

Convert StreamTableJoinIntegrationTest to use the ToplogyTestDriver to eliminate flakiness and speed up the build.

Reviewers: John Roesler 
---
 .../AbstractJoinIntegrationTest.java          | 45 +++++++++++++++++++
 .../StreamTableJoinIntegrationTest.java       | 30 +++++++------
 2 files changed, 61 insertions(+), 14 deletions(-)

diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/AbstractJoinIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/AbstractJoinIntegrationTest.java
index 6660a205e3cad..e7b87a22c0966 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/AbstractJoinIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/AbstractJoinIntegrationTest.java
@@ -30,6 +30,9 @@
 import org.apache.kafka.streams.KeyValueTimestamp;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TestInputTopic;
+import org.apache.kafka.streams.TestOutputTopic;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
 import org.apache.kafka.streams.integration.utils.IntegrationTestUtils;
 import org.apache.kafka.streams.kstream.ValueJoiner;
@@ -37,6 +40,7 @@
 import org.apache.kafka.streams.state.QueryableStoreTypes;
 import org.apache.kafka.streams.state.ReadOnlyKeyValueStore;
 import org.apache.kafka.streams.state.ValueAndTimestamp;
+import org.apache.kafka.streams.test.TestRecord;
 import org.apache.kafka.test.IntegrationTest;
 import org.apache.kafka.test.TestUtils;
 import org.junit.After;
@@ -52,15 +56,18 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.apache.kafka.test.StreamsTestUtils.startKafkaStreamsAndWaitForRunningState;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertEquals;
 
 /**
  * Tests all available joins of Kafka Streams DSL.
@@ -186,6 +193,44 @@ void runTest(final List>> expectedResult) t
     }
 
 
+    void runTestWithDriver(final List>> expectedResult) {
+        runTestWithDriver(expectedResult, null);
+    }
+
+    void runTestWithDriver(final List>> expectedResult, final String storeName) {
+        try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(STREAMS_CONFIG), STREAMS_CONFIG)) {
+            final TestInputTopic right = driver.createInputTopic(INPUT_TOPIC_RIGHT, new LongSerializer(), new StringSerializer());
+            final TestInputTopic left = driver.createInputTopic(INPUT_TOPIC_LEFT, new LongSerializer(), new StringSerializer());
+            final TestOutputTopic outputTopic = driver.createOutputTopic(OUTPUT_TOPIC, new LongDeserializer(), new StringDeserializer());
+            final Map> testInputTopicMap = new HashMap<>();
+
+            testInputTopicMap.put(INPUT_TOPIC_RIGHT, right);
+            testInputTopicMap.put(INPUT_TOPIC_LEFT, left);
+
+            TestRecord expectedFinalResult = null;
+
+            final long firstTimestamp = System.currentTimeMillis();
+            long ts = firstTimestamp;
+            final Iterator>> resultIterator = expectedResult.iterator();
+            for (final Input singleInputRecord : input) {
+                testInputTopicMap.get(singleInputRecord.topic).pipeInput(singleInputRecord.record.key, singleInputRecord.record.value, ++ts);
+
+                final List> expected = resultIterator.next();
+                if (expected != null) {
+                    final List> updatedExpected = new LinkedList<>();
+                    for (final TestRecord record : expected) {
+                        updatedExpected.add(new TestRecord<>(record.key(), record.value(), null, firstTimestamp + record.timestamp()));
+                    }
+
+                    final List> output = outputTopic.readRecordsToList();
+                    assertEquals(output, updatedExpected);
+                    expectedFinalResult = updatedExpected.get(expected.size() - 1);
+                }
+            }
+        }
+    }
+
+
     /*
      * Runs the actual test. Checks the result after each input record to ensure fixed processing order.
      * If an input tuple does not trigger any result, "expectedResult" should contain a "null" entry
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/StreamTableJoinIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/StreamTableJoinIntegrationTest.java
index 4ddab57099d33..66f0a04b91b55 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/StreamTableJoinIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/StreamTableJoinIntegrationTest.java
@@ -17,12 +17,12 @@
 package org.apache.kafka.streams.integration;
 
 import org.apache.kafka.streams.KafkaStreamsWrapper;
-import org.apache.kafka.streams.KeyValueTimestamp;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.integration.utils.IntegrationTestUtils;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KTable;
+import org.apache.kafka.streams.test.TestRecord;
 import org.apache.kafka.test.IntegrationTest;
 import org.apache.kafka.test.TestUtils;
 import org.junit.Before;
@@ -64,6 +64,7 @@ public void prepareTopology() throws InterruptedException {
     @Test
     public void testShouldAutoShutdownOnIncompleteMetadata() throws InterruptedException {
         STREAMS_CONFIG.put(StreamsConfig.APPLICATION_ID_CONFIG, appID + "-incomplete");
+        STREAMS_CONFIG.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
 
         final KStream notExistStream = builder.stream(INPUT_TOPIC_LEFT + "-not-existed");
 
@@ -86,15 +87,16 @@ public void testShouldAutoShutdownOnIncompleteMetadata() throws InterruptedExcep
     }
 
     @Test
-    public void testInner() throws Exception {
+    public void testInner() {
         STREAMS_CONFIG.put(StreamsConfig.APPLICATION_ID_CONFIG, appID + "-inner");
+        STREAMS_CONFIG.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "topology_driver:0000");
 
-        final List>> expectedResult = Arrays.asList(
+        final List>> expectedResult = Arrays.asList(
             null,
             null,
             null,
             null,
-            Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a", 5L)),
+            Collections.singletonList(new TestRecord<>(ANY_UNIQUE_KEY, "B-a", null, 5L)),
             null,
             null,
             null,
@@ -104,38 +106,38 @@ public void testInner() throws Exception {
             null,
             null,
             null,
-            Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d", 15L))
+            Collections.singletonList(new TestRecord<>(ANY_UNIQUE_KEY, "D-d", null,  15L))
         );
 
         leftStream.join(rightTable, valueJoiner).to(OUTPUT_TOPIC);
-
-        runTest(expectedResult);
+        runTestWithDriver(expectedResult);
     }
 
     @Test
-    public void testLeft() throws Exception {
+    public void testLeft() {
         STREAMS_CONFIG.put(StreamsConfig.APPLICATION_ID_CONFIG, appID + "-left");
+        STREAMS_CONFIG.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "topology_driver:0000");
 
-        final List>> expectedResult = Arrays.asList(
+        final List>> expectedResult = Arrays.asList(
             null,
             null,
-            Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "A-null", 3L)),
+            Collections.singletonList(new TestRecord<>(ANY_UNIQUE_KEY, "A-null", null, 3L)),
             null,
-            Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "B-a", 5L)),
+            Collections.singletonList(new TestRecord<>(ANY_UNIQUE_KEY, "B-a", null, 5L)),
             null,
             null,
             null,
-            Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "C-null", 9L)),
+            Collections.singletonList(new TestRecord<>(ANY_UNIQUE_KEY, "C-null", null, 9L)),
             null,
             null,
             null,
             null,
             null,
-            Collections.singletonList(new KeyValueTimestamp<>(ANY_UNIQUE_KEY, "D-d", 15L))
+            Collections.singletonList(new TestRecord<>(ANY_UNIQUE_KEY, "D-d", null, 15L))
         );
 
         leftStream.leftJoin(rightTable, valueJoiner).to(OUTPUT_TOPIC);
 
-        runTest(expectedResult);
+        runTestWithDriver(expectedResult);
     }
 }

From b8536cc7c67b0b4ac7895c606bb4a522ce17b62d Mon Sep 17 00:00:00 2001
From: David Arthur 
Date: Mon, 18 Nov 2019 20:36:00 -0500
Subject: [PATCH 0813/1071] KAFKA-8981 Add rate limiting to NetworkDegradeSpec
 (#7446)

---
 tests/docker/Dockerfile                       |   2 +-
 .../trogdor/degraded_network_fault_spec.py    |  19 ++-
 tests/kafkatest/services/trogdor/trogdor.py   |  36 ++++-
 .../tests/core/network_degrade_test.py        | 138 ++++++++++++++++++
 .../tests/core/round_trip_fault_test.py       |   5 +-
 tests/kafkatest/utils/remote_account.py       |   2 +-
 .../fault/DegradedNetworkFaultSpec.java       |  21 ++-
 .../fault/DegradedNetworkFaultWorker.java     |  81 ++++++++--
 vagrant/aws/aws-init.sh                       |   4 +-
 vagrant/base.sh                               |   3 +
 10 files changed, 282 insertions(+), 29 deletions(-)
 create mode 100644 tests/kafkatest/tests/core/network_degrade_test.py

diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile
index 66d5c4a6779b9..ec0656adb6ba3 100644
--- a/tests/docker/Dockerfile
+++ b/tests/docker/Dockerfile
@@ -32,7 +32,7 @@ ARG ducker_creator=default
 LABEL ducker.creator=$ducker_creator
 
 # Update Linux and install necessary utilities.
-RUN apt update && apt install -y sudo netcat iptables rsync unzip wget curl jq coreutils openssh-server net-tools vim python-pip python-dev libffi-dev libssl-dev cmake pkg-config libfuse-dev  && apt-get -y clean
+RUN apt update && apt install -y sudo netcat iptables rsync unzip wget curl jq coreutils openssh-server net-tools vim python-pip python-dev libffi-dev libssl-dev cmake pkg-config libfuse-dev iperf traceroute && apt-get -y clean
 RUN python -m pip install -U pip==9.0.3;
 RUN pip install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 && pip install --upgrade ducktape==0.7.6
 
diff --git a/tests/kafkatest/services/trogdor/degraded_network_fault_spec.py b/tests/kafkatest/services/trogdor/degraded_network_fault_spec.py
index 450a7870cf3e6..2a3b142f3ecfd 100644
--- a/tests/kafkatest/services/trogdor/degraded_network_fault_spec.py
+++ b/tests/kafkatest/services/trogdor/degraded_network_fault_spec.py
@@ -23,15 +23,26 @@ class DegradedNetworkFaultSpec(TaskSpec):
     Degrades the network so that traffic on a subset of nodes has higher latency
     """
 
-    def __init__(self, start_ms, duration_ms, node_specs):
+    def __init__(self, start_ms, duration_ms):
         """
         Create a new NetworkDegradeFaultSpec.
 
         :param start_ms:        The start time, as described in task_spec.py
         :param duration_ms:     The duration in milliseconds.
-        :param node_latencies:  A dict of node name to desired latency
-        :param network_device:  The name of the network device
         """
         super(DegradedNetworkFaultSpec, self).__init__(start_ms, duration_ms)
         self.message["class"] = "org.apache.kafka.trogdor.fault.DegradedNetworkFaultSpec"
-        self.message["nodeSpecs"] = node_specs
+        self.message["nodeSpecs"] = {}
+
+    def add_node_spec(self, node, networkDevice, latencyMs=0, rateLimitKbit=0):
+        """
+        Add a node spec to this fault spec
+        :param node:            The node name which is to be degraded
+        :param networkDevice:   The network device name (e.g., eth0) to apply the degradation to
+        :param latencyMs:       Optional. How much latency to add to each packet
+        :param rateLimitKbit:   Optional. Maximum throughput in kilobits per second to allow
+        :return:
+        """
+        self.message["nodeSpecs"][node] = {
+            "rateLimitKbit": rateLimitKbit, "latencyMs": latencyMs, "networkDevice": networkDevice
+        }
diff --git a/tests/kafkatest/services/trogdor/trogdor.py b/tests/kafkatest/services/trogdor/trogdor.py
index 4d514f2a39428..6a6e888dc1e10 100644
--- a/tests/kafkatest/services/trogdor/trogdor.py
+++ b/tests/kafkatest/services/trogdor/trogdor.py
@@ -300,6 +300,16 @@ def __init__(self, id, trogdor):
         self.id = id
         self.trogdor = trogdor
 
+    def task_state_or_error(self):
+        task_state = self.trogdor.tasks()["tasks"][self.id]
+        if task_state is None:
+            raise RuntimeError("Coordinator did not know about %s." % self.id)
+        error = task_state.get("error")
+        if error is None or error == "":
+            return task_state["state"], None
+        else:
+            return None, error
+
     def done(self):
         """
         Check if this task is done.
@@ -308,13 +318,25 @@ def done(self):
         :returns:                   True if the task is in DONE_STATE;
                                     False if it is in a different state.
         """
-        task_state = self.trogdor.tasks()["tasks"][self.id]
-        if task_state is None:
-            raise RuntimeError("Coordinator did not know about %s." % self.id)
-        error = task_state.get("error")
-        if error is None or error == "":
-            return task_state["state"] == TrogdorTask.DONE_STATE
-        raise RuntimeError("Failed to gracefully stop %s: got task error: %s" % (self.id, error))
+        (task_state, error) = self.task_state_or_error()
+        if task_state is not None:
+            return task_state == TrogdorTask.DONE_STATE
+        else:
+            raise RuntimeError("Failed to gracefully stop %s: got task error: %s" % (self.id, error))
+
+    def running(self):
+        """
+        Check if this task is running.
+
+        :raises RuntimeError:       If the task encountered an error.
+        :returns:                   True if the task is in RUNNING_STATE;
+                                    False if it is in a different state.
+        """
+        (task_state, error) = self.task_state_or_error()
+        if task_state is not None:
+            return task_state == TrogdorTask.RUNNING_STATE
+        else:
+            raise RuntimeError("Failed to start %s: got task error: %s" % (self.id, error))
 
     def stop(self):
         """
diff --git a/tests/kafkatest/tests/core/network_degrade_test.py b/tests/kafkatest/tests/core/network_degrade_test.py
new file mode 100644
index 0000000000000..5b77d99c62da3
--- /dev/null
+++ b/tests/kafkatest/tests/core/network_degrade_test.py
@@ -0,0 +1,138 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+from ducktape.mark import parametrize
+from ducktape.mark.resource import cluster
+from ducktape.tests.test import Test
+from ducktape.utils.util import wait_until
+
+from kafkatest.services.trogdor.degraded_network_fault_spec import DegradedNetworkFaultSpec
+from kafkatest.services.trogdor.trogdor import TrogdorService
+from kafkatest.services.zookeeper import ZookeeperService
+
+
+class NetworkDegradeTest(Test):
+    """
+    These tests ensure that the network degrade Trogdor specs (which use "tc") are working as expected in whatever
+    environment the system tests may be running in. The linux tools "ping" and "iperf" are used for validation
+    and need to be available along with "tc" in the test environment.
+    """
+
+    def __init__(self, test_context):
+        super(NetworkDegradeTest, self).__init__(test_context)
+        self.zk = ZookeeperService(test_context, num_nodes=3)
+        self.trogdor = TrogdorService(context=self.test_context, client_services=[self.zk])
+
+    def setUp(self):
+        self.zk.start()
+        self.trogdor.start()
+
+    def teardown(self):
+        self.trogdor.stop()
+        self.zk.stop()
+
+    @cluster(num_nodes=5)
+    @parametrize(task_name="latency-100", device_name="eth0", latency_ms=50, rate_limit_kbit=0)
+    @parametrize(task_name="latency-100-rate-1000", device_name="eth0", latency_ms=50, rate_limit_kbit=1000)
+    def test_latency(self, task_name, device_name, latency_ms, rate_limit_kbit):
+        spec = DegradedNetworkFaultSpec(0, 10000)
+        for node in self.zk.nodes:
+            spec.add_node_spec(node.name, device_name, latency_ms, rate_limit_kbit)
+
+        latency = self.trogdor.create_task(task_name, spec)
+
+        zk0 = self.zk.nodes[0]
+        zk1 = self.zk.nodes[1]
+
+        # Capture the ping times from the ping stdout
+        # 64 bytes from ducker01 (172.24.0.2): icmp_seq=1 ttl=64 time=0.325 ms
+        r = re.compile(r".*time=(?P
          
    */
-  private def analyzeAndValidateRecords(records: MemoryRecords, isFromClient: Boolean): LogAppendInfo = {
+  private def analyzeAndValidateRecords(records: MemoryRecords, origin: AppendOrigin): LogAppendInfo = {
     var shallowMessageCount = 0
     var validBytesCount = 0
     var firstOffset: Option[Long] = None
@@ -1333,7 +1339,7 @@ class Log(@volatile var dir: File,
 
     for (batch <- records.batches.asScala) {
       // we only validate V2 and higher to avoid potential compatibility issues with older clients
-      if (batch.magic >= RecordBatch.MAGIC_VALUE_V2 && isFromClient && batch.baseOffset != 0)
+      if (batch.magic >= RecordBatch.MAGIC_VALUE_V2 && origin == AppendOrigin.Client && batch.baseOffset != 0)
         throw new InvalidRecordException(s"The baseOffset of the record batch in the append to $topicPartition should " +
           s"be 0, but it is ${batch.baseOffset}")
 
@@ -1394,9 +1400,9 @@ class Log(@volatile var dir: File,
   private def updateProducers(batch: RecordBatch,
                               producers: mutable.Map[Long, ProducerAppendInfo],
                               firstOffsetMetadata: Option[LogOffsetMetadata],
-                              isFromClient: Boolean): Option[CompletedTxn] = {
+                              origin: AppendOrigin): Option[CompletedTxn] = {
     val producerId = batch.producerId
-    val appendInfo = producers.getOrElseUpdate(producerId, producerStateManager.prepareUpdate(producerId, isFromClient))
+    val appendInfo = producers.getOrElseUpdate(producerId, producerStateManager.prepareUpdate(producerId, origin))
     appendInfo.append(batch, firstOffsetMetadata)
   }
 
diff --git a/core/src/main/scala/kafka/log/LogSegment.scala b/core/src/main/scala/kafka/log/LogSegment.scala
index a54c2c2eacc74..64e96e1cc73eb 100755
--- a/core/src/main/scala/kafka/log/LogSegment.scala
+++ b/core/src/main/scala/kafka/log/LogSegment.scala
@@ -245,7 +245,7 @@ class LogSegment private[log] (val log: FileRecords,
   private def updateProducerState(producerStateManager: ProducerStateManager, batch: RecordBatch): Unit = {
     if (batch.hasProducerId) {
       val producerId = batch.producerId
-      val appendInfo = producerStateManager.prepareUpdate(producerId, isFromClient = false)
+      val appendInfo = producerStateManager.prepareUpdate(producerId, origin = AppendOrigin.Replication)
       val maybeCompletedTxn = appendInfo.append(batch, firstOffsetMetadataOpt = None)
       producerStateManager.update(appendInfo)
       maybeCompletedTxn.foreach { completedTxn =>
diff --git a/core/src/main/scala/kafka/log/LogValidator.scala b/core/src/main/scala/kafka/log/LogValidator.scala
index c4dda087af1e4..b72ab1fc41918 100644
--- a/core/src/main/scala/kafka/log/LogValidator.scala
+++ b/core/src/main/scala/kafka/log/LogValidator.scala
@@ -33,7 +33,32 @@ import org.apache.kafka.common.utils.Time
 import scala.collection.{Seq, mutable}
 import scala.collection.JavaConverters._
 
-private[kafka] object LogValidator extends Logging {
+/**
+ * The source of an append to the log. This is used when determining required validations.
+ */
+private[kafka] sealed trait AppendOrigin
+private[kafka] object AppendOrigin {
+
+  /**
+   * The log append came through replication from the leader. This typically implies minimal validation.
+   * Particularly, we do not decompress record batches in order to validate records individually.
+   */
+  case object Replication extends AppendOrigin
+
+  /**
+   * The log append came from either the group coordinator or the transaction coordinator. We validate
+   * producer epochs for normal log entries (specifically offset commits from the group coordinator) and
+   * we validate coordinate end transaction markers from the transaction coordinator.
+   */
+  case object Coordinator extends AppendOrigin
+
+  /**
+   * The log append came from the client, which implies full validation.
+   */
+  case object Client extends AppendOrigin
+}
+
+private[log] object LogValidator extends Logging {
 
   /**
    * Update the offsets for this message set and do further validation on messages including:
@@ -50,37 +75,37 @@ private[kafka] object LogValidator extends Logging {
    * Returns a ValidationAndOffsetAssignResult containing the validated message set, maximum timestamp, the offset
    * of the shallow message with the max timestamp and a boolean indicating whether the message sizes may have changed.
    */
-  private[kafka] def validateMessagesAndAssignOffsets(records: MemoryRecords,
-                                                      topicPartition: TopicPartition,
-                                                      offsetCounter: LongRef,
-                                                      time: Time,
-                                                      now: Long,
-                                                      sourceCodec: CompressionCodec,
-                                                      targetCodec: CompressionCodec,
-                                                      compactedTopic: Boolean,
-                                                      magic: Byte,
-                                                      timestampType: TimestampType,
-                                                      timestampDiffMaxMs: Long,
-                                                      partitionLeaderEpoch: Int,
-                                                      isFromClient: Boolean,
-                                                      interBrokerProtocolVersion: ApiVersion,
-                                                      brokerTopicStats: BrokerTopicStats): ValidationAndOffsetAssignResult = {
+  private[log] def validateMessagesAndAssignOffsets(records: MemoryRecords,
+                                                    topicPartition: TopicPartition,
+                                                    offsetCounter: LongRef,
+                                                    time: Time,
+                                                    now: Long,
+                                                    sourceCodec: CompressionCodec,
+                                                    targetCodec: CompressionCodec,
+                                                    compactedTopic: Boolean,
+                                                    magic: Byte,
+                                                    timestampType: TimestampType,
+                                                    timestampDiffMaxMs: Long,
+                                                    partitionLeaderEpoch: Int,
+                                                    origin: AppendOrigin,
+                                                    interBrokerProtocolVersion: ApiVersion,
+                                                    brokerTopicStats: BrokerTopicStats): ValidationAndOffsetAssignResult = {
     if (sourceCodec == NoCompressionCodec && targetCodec == NoCompressionCodec) {
       // check the magic value
       if (!records.hasMatchingMagic(magic))
         convertAndAssignOffsetsNonCompressed(records, topicPartition, offsetCounter, compactedTopic, time, now, timestampType,
-          timestampDiffMaxMs, magic, partitionLeaderEpoch, isFromClient, brokerTopicStats)
+          timestampDiffMaxMs, magic, partitionLeaderEpoch, origin, brokerTopicStats)
       else
         // Do in-place validation, offset assignment and maybe set timestamp
         assignOffsetsNonCompressed(records, topicPartition, offsetCounter, now, compactedTopic, timestampType, timestampDiffMaxMs,
-          partitionLeaderEpoch, isFromClient, magic, brokerTopicStats)
+          partitionLeaderEpoch, origin, magic, brokerTopicStats)
     } else {
       validateMessagesAndAssignOffsetsCompressed(records, topicPartition, offsetCounter, time, now, sourceCodec, targetCodec, compactedTopic,
-        magic, timestampType, timestampDiffMaxMs, partitionLeaderEpoch, isFromClient, interBrokerProtocolVersion, brokerTopicStats)
+        magic, timestampType, timestampDiffMaxMs, partitionLeaderEpoch, origin, interBrokerProtocolVersion, brokerTopicStats)
     }
   }
 
-  private[kafka] def getFirstBatchAndMaybeValidateNoMoreBatches(records: MemoryRecords, sourceCodec: CompressionCodec): RecordBatch = {
+  private def getFirstBatchAndMaybeValidateNoMoreBatches(records: MemoryRecords, sourceCodec: CompressionCodec): RecordBatch = {
     val batchIterator = records.batches.iterator
 
     if (!batchIterator.hasNext) {
@@ -99,14 +124,19 @@ private[kafka] object LogValidator extends Logging {
     batch
   }
 
-  private def validateBatch(topicPartition: TopicPartition, firstBatch: RecordBatch, batch: RecordBatch, isFromClient: Boolean, toMagic: Byte, brokerTopicStats: BrokerTopicStats): Unit = {
+  private def validateBatch(topicPartition: TopicPartition,
+                            firstBatch: RecordBatch,
+                            batch: RecordBatch,
+                            origin: AppendOrigin,
+                            toMagic: Byte,
+                            brokerTopicStats: BrokerTopicStats): Unit = {
     // batch magic byte should have the same magic as the first batch
     if (firstBatch.magic() != batch.magic()) {
       brokerTopicStats.allTopicsStats.invalidMagicNumberRecordsPerSec.mark()
       throw new InvalidRecordException(s"Batch magic ${batch.magic()} is not the same as the first batch'es magic byte ${firstBatch.magic()} in topic partition $topicPartition.")
     }
 
-    if (isFromClient) {
+    if (origin == AppendOrigin.Client) {
       if (batch.magic >= RecordBatch.MAGIC_VALUE_V2) {
         val countFromOffsets = batch.lastOffset - batch.baseOffset + 1
         if (countFromOffsets <= 0) {
@@ -128,15 +158,15 @@ private[kafka] object LogValidator extends Logging {
         }
       }
 
-      if (batch.hasProducerId && batch.baseSequence < 0) {
+      if (batch.isControlBatch) {
         brokerTopicStats.allTopicsStats.invalidOffsetOrSequenceRecordsPerSec.mark()
-        throw new InvalidRecordException(s"Invalid sequence number ${batch.baseSequence} in record batch " +
-          s"with producerId ${batch.producerId} in topic partition $topicPartition.")
+        throw new InvalidRecordException(s"Clients are not allowed to write control records in topic partition $topicPartition.")
       }
 
-      if (batch.isControlBatch) {
+      if (batch.hasProducerId && batch.baseSequence < 0) {
         brokerTopicStats.allTopicsStats.invalidOffsetOrSequenceRecordsPerSec.mark()
-        throw new InvalidRecordException(s"Clients are not allowed to write control records in topic partition $topicPartition.")
+        throw new InvalidRecordException(s"Invalid sequence number ${batch.baseSequence} in record batch " +
+          s"with producerId ${batch.producerId} in topic partition $topicPartition.")
       }
     }
 
@@ -185,7 +215,7 @@ private[kafka] object LogValidator extends Logging {
                                                    timestampDiffMaxMs: Long,
                                                    toMagicValue: Byte,
                                                    partitionLeaderEpoch: Int,
-                                                   isFromClient: Boolean,
+                                                   origin: AppendOrigin,
                                                    brokerTopicStats: BrokerTopicStats): ValidationAndOffsetAssignResult = {
     val startNanos = time.nanoseconds
     val sizeInBytesAfterConversion = AbstractRecords.estimateSizeInBytes(toMagicValue, offsetCounter.value,
@@ -203,7 +233,7 @@ private[kafka] object LogValidator extends Logging {
     val firstBatch = getFirstBatchAndMaybeValidateNoMoreBatches(records, NoCompressionCodec)
 
     for (batch <- records.batches.asScala) {
-      validateBatch(topicPartition, firstBatch, batch, isFromClient, toMagicValue, brokerTopicStats)
+      validateBatch(topicPartition, firstBatch, batch, origin, toMagicValue, brokerTopicStats)
 
       for ((record, batchIndex) <- batch.asScala.view.zipWithIndex) {
         validateRecord(batch, topicPartition, record, batchIndex, now, timestampType, timestampDiffMaxMs, compactedTopic, brokerTopicStats)
@@ -232,7 +262,7 @@ private[kafka] object LogValidator extends Logging {
                                          timestampType: TimestampType,
                                          timestampDiffMaxMs: Long,
                                          partitionLeaderEpoch: Int,
-                                         isFromClient: Boolean,
+                                         origin: AppendOrigin,
                                          magic: Byte,
                                          brokerTopicStats: BrokerTopicStats): ValidationAndOffsetAssignResult = {
     var maxTimestamp = RecordBatch.NO_TIMESTAMP
@@ -242,7 +272,7 @@ private[kafka] object LogValidator extends Logging {
     val firstBatch = getFirstBatchAndMaybeValidateNoMoreBatches(records, NoCompressionCodec)
 
     for (batch <- records.batches.asScala) {
-      validateBatch(topicPartition, firstBatch, batch, isFromClient, magic, brokerTopicStats)
+      validateBatch(topicPartition, firstBatch, batch, origin, magic, brokerTopicStats)
 
       var maxBatchTimestamp = RecordBatch.NO_TIMESTAMP
       var offsetOfMaxBatchTimestamp = -1L
@@ -309,7 +339,7 @@ private[kafka] object LogValidator extends Logging {
                                                  timestampType: TimestampType,
                                                  timestampDiffMaxMs: Long,
                                                  partitionLeaderEpoch: Int,
-                                                 isFromClient: Boolean,
+                                                 origin: AppendOrigin,
                                                  interBrokerProtocolVersion: ApiVersion,
                                                  brokerTopicStats: BrokerTopicStats): ValidationAndOffsetAssignResult = {
 
@@ -343,7 +373,7 @@ private[kafka] object LogValidator extends Logging {
 
     val batches = records.batches.asScala
     for (batch <- batches) {
-      validateBatch(topicPartition, firstBatch, batch, isFromClient, toMagic, brokerTopicStats)
+      validateBatch(topicPartition, firstBatch, batch, origin, toMagic, brokerTopicStats)
       uncompressedSizeInBytes += AbstractRecords.recordBatchHeaderSizeInBytes(toMagic, batch.compressionType())
 
       // if we are on version 2 and beyond, and we know we are going for in place assignment,
@@ -391,8 +421,8 @@ private[kafka] object LogValidator extends Logging {
         val first = records.batches.asScala.head
         (first.producerId, first.producerEpoch, first.baseSequence, first.isTransactional)
       }
-      buildRecordsAndAssignOffsets(toMagic, offsetCounter, time, timestampType, CompressionType.forId(targetCodec.codec), now,
-        validatedRecords, producerId, producerEpoch, sequence, isTransactional, partitionLeaderEpoch, isFromClient,
+      buildRecordsAndAssignOffsets(toMagic, offsetCounter, time, timestampType, CompressionType.forId(targetCodec.codec),
+        now, validatedRecords, producerId, producerEpoch, sequence, isTransactional, partitionLeaderEpoch,
         uncompressedSizeInBytes)
     } else {
       // we can update the batch only and write the compressed payload as is;
@@ -432,7 +462,6 @@ private[kafka] object LogValidator extends Logging {
                                            baseSequence: Int,
                                            isTransactional: Boolean,
                                            partitionLeaderEpoch: Int,
-                                           isFromClient: Boolean,
                                            uncompressedSizeInBytes: Int): ValidationAndOffsetAssignResult = {
     val startNanos = time.nanoseconds
     val estimatedSize = AbstractRecords.estimateSizeInBytes(magic, offsetCounter.value, compressionType,
diff --git a/core/src/main/scala/kafka/log/ProducerStateManager.scala b/core/src/main/scala/kafka/log/ProducerStateManager.scala
index 04dafd8a04a61..f195906f6fcc9 100644
--- a/core/src/main/scala/kafka/log/ProducerStateManager.scala
+++ b/core/src/main/scala/kafka/log/ProducerStateManager.scala
@@ -26,7 +26,6 @@ import kafka.server.LogOffsetMetadata
 import kafka.utils.{Logging, nonthreadsafe, threadsafe}
 import org.apache.kafka.common.{KafkaException, TopicPartition}
 import org.apache.kafka.common.errors._
-import org.apache.kafka.common.internals.Topic
 import org.apache.kafka.common.protocol.types._
 import org.apache.kafka.common.record.{ControlRecordType, DefaultRecordBatch, EndTransactionMarker, RecordBatch}
 import org.apache.kafka.common.utils.{ByteUtils, Crc32C}
@@ -44,28 +43,6 @@ class CorruptSnapshotException(msg: String) extends KafkaException(msg)
 case class LastRecord(lastDataOffset: Option[Long], producerEpoch: Short)
 
 
-// ValidationType and its subtypes define the extent of the validation to perform on a given ProducerAppendInfo instance
-private[log] sealed trait ValidationType
-private[log] object ValidationType {
-
-  /**
-    * This indicates no validation should be performed on the incoming append. This is the case for all appends on
-    * a replica, as well as appends when the producer state is being built from the log.
-    */
-  case object None extends ValidationType
-
-  /**
-    * We only validate the epoch (and not the sequence numbers) for offset commit requests coming from the transactional
-    * producer. These appends will not have sequence numbers, so we can't validate them.
-    */
-  case object EpochOnly extends ValidationType
-
-  /**
-    * Perform the full validation. This should be used fo regular produce requests coming to the leader.
-    */
-  case object Full extends ValidationType
-}
-
 private[log] case class TxnMetadata(producerId: Long, var firstOffset: LogOffsetMetadata, var lastOffset: Option[Long] = None) {
   def this(producerId: Long, firstOffset: Long) = this(producerId, LogOffsetMetadata(firstOffset))
 
@@ -79,12 +56,18 @@ private[log] case class TxnMetadata(producerId: Long, var firstOffset: LogOffset
 
 private[log] object ProducerStateEntry {
   private[log] val NumBatchesToRetain = 5
-  def empty(producerId: Long) = new ProducerStateEntry(producerId, mutable.Queue[BatchMetadata](), RecordBatch.NO_PRODUCER_EPOCH, -1, None)
+
+  def empty(producerId: Long) = new ProducerStateEntry(producerId,
+    batchMetadata = mutable.Queue[BatchMetadata](),
+    producerEpoch = RecordBatch.NO_PRODUCER_EPOCH,
+    coordinatorEpoch = -1,
+    lastTimestamp = RecordBatch.NO_TIMESTAMP,
+    currentTxnFirstOffset = None)
 }
 
 private[log] case class BatchMetadata(lastSeq: Int, lastOffset: Long, offsetDelta: Int, timestamp: Long) {
-  def firstSeq =  DefaultRecordBatch.decrementSequence(lastSeq, offsetDelta)
-  def firstOffset = lastOffset - offsetDelta
+  def firstSeq: Int =  DefaultRecordBatch.decrementSequence(lastSeq, offsetDelta)
+  def firstOffset: Long = lastOffset - offsetDelta
 
   override def toString: String = {
     "BatchMetadata(" +
@@ -103,28 +86,28 @@ private[log] class ProducerStateEntry(val producerId: Long,
                                       val batchMetadata: mutable.Queue[BatchMetadata],
                                       var producerEpoch: Short,
                                       var coordinatorEpoch: Int,
+                                      var lastTimestamp: Long,
                                       var currentTxnFirstOffset: Option[Long]) {
 
   def firstSeq: Int = if (isEmpty) RecordBatch.NO_SEQUENCE else batchMetadata.front.firstSeq
 
-  def firstOffset: Long = if (isEmpty) -1L else batchMetadata.front.firstOffset
+  def firstDataOffset: Long = if (isEmpty) -1L else batchMetadata.front.firstOffset
 
   def lastSeq: Int = if (isEmpty) RecordBatch.NO_SEQUENCE else batchMetadata.last.lastSeq
 
   def lastDataOffset: Long = if (isEmpty) -1L else batchMetadata.last.lastOffset
 
-  def lastTimestamp: Long = if (isEmpty) RecordBatch.NO_TIMESTAMP else batchMetadata.last.timestamp
-
   def lastOffsetDelta : Int = if (isEmpty) 0 else batchMetadata.last.offsetDelta
 
   def isEmpty: Boolean = batchMetadata.isEmpty
 
   def addBatch(producerEpoch: Short, lastSeq: Int, lastOffset: Long, offsetDelta: Int, timestamp: Long): Unit = {
-    maybeUpdateEpoch(producerEpoch)
+    maybeUpdateProducerEpoch(producerEpoch)
     addBatchMetadata(BatchMetadata(lastSeq, lastOffset, offsetDelta, timestamp))
+    this.lastTimestamp = timestamp
   }
 
-  def maybeUpdateEpoch(producerEpoch: Short): Boolean = {
+  def maybeUpdateProducerEpoch(producerEpoch: Short): Boolean = {
     if (this.producerEpoch != producerEpoch) {
       batchMetadata.clear()
       this.producerEpoch = producerEpoch
@@ -141,11 +124,12 @@ private[log] class ProducerStateEntry(val producerId: Long,
   }
 
   def update(nextEntry: ProducerStateEntry): Unit = {
-    maybeUpdateEpoch(nextEntry.producerEpoch)
+    maybeUpdateProducerEpoch(nextEntry.producerEpoch)
     while (nextEntry.batchMetadata.nonEmpty)
       addBatchMetadata(nextEntry.batchMetadata.dequeue())
     this.coordinatorEpoch = nextEntry.coordinatorEpoch
     this.currentTxnFirstOffset = nextEntry.currentTxnFirstOffset
+    this.lastTimestamp = nextEntry.lastTimestamp
   }
 
   def findDuplicateBatch(batch: RecordBatch): Option[BatchMetadata] = {
@@ -169,6 +153,7 @@ private[log] class ProducerStateEntry(val producerId: Long,
       s"producerEpoch=$producerEpoch, " +
       s"currentTxnFirstOffset=$currentTxnFirstOffset, " +
       s"coordinatorEpoch=$coordinatorEpoch, " +
+      s"lastTimestamp=$lastTimestamp, " +
       s"batchMetadata=$batchMetadata"
   }
 }
@@ -184,39 +169,41 @@ private[log] class ProducerStateEntry(val producerId: Long,
  *                      the most recent appends made by the producer. Validation of the first incoming append will
  *                      be made against the latest append in the current entry. New appends will replace older appends
  *                      in the current entry so that the space overhead is constant.
- * @param validationType Indicates the extent of validation to perform on the appends on this instance. Offset commits
- *                       coming from the producer should have ValidationType.EpochOnly. Appends which aren't from a client
- *                       should have ValidationType.None. Appends coming from a client for produce requests should have
- *                       ValidationType.Full.
+ * @param origin Indicates the origin of the append which implies the extent of validation. For example, offset
+ *               commits, which originate from the group coordinator, do not have sequence numbers and therefore
+ *               only producer epoch validation is done. Appends which come through replciation are not validated
+ *               (we assume the validation has already been done) and appends from clients require full validation.
  */
 private[log] class ProducerAppendInfo(val topicPartition: TopicPartition,
                                       val producerId: Long,
                                       val currentEntry: ProducerStateEntry,
-                                      val validationType: ValidationType) {
+                                      val origin: AppendOrigin) extends Logging {
+
   private val transactions = ListBuffer.empty[TxnMetadata]
   private val updatedEntry = ProducerStateEntry.empty(producerId)
 
   updatedEntry.producerEpoch = currentEntry.producerEpoch
   updatedEntry.coordinatorEpoch = currentEntry.coordinatorEpoch
+  updatedEntry.lastTimestamp = currentEntry.lastTimestamp
   updatedEntry.currentTxnFirstOffset = currentEntry.currentTxnFirstOffset
 
-  private def maybeValidateAppend(producerEpoch: Short, firstSeq: Int, offset: Long): Unit = {
-    validationType match {
-      case ValidationType.None =>
-
-      case ValidationType.EpochOnly =>
-        checkProducerEpoch(producerEpoch, offset)
-
-      case ValidationType.Full =>
-        checkProducerEpoch(producerEpoch, offset)
-        checkSequence(producerEpoch, firstSeq, offset)
+  private def maybeValidateDataBatch(producerEpoch: Short, firstSeq: Int, offset: Long): Unit = {
+    checkProducerEpoch(producerEpoch, offset)
+    if (origin == AppendOrigin.Client) {
+      checkSequence(producerEpoch, firstSeq, offset)
     }
   }
 
   private def checkProducerEpoch(producerEpoch: Short, offset: Long): Unit = {
     if (producerEpoch < updatedEntry.producerEpoch) {
-      throw new ProducerFencedException(s"Producer's epoch at offset $offset is no longer valid in " +
-        s"partition $topicPartition: $producerEpoch (request epoch), ${updatedEntry.producerEpoch} (current epoch)")
+      val message = s"Producer's epoch at offset $offset in $topicPartition is $producerEpoch, which is " +
+        s"smaller than the last seen epoch ${updatedEntry.producerEpoch}"
+
+      if (origin == AppendOrigin.Replication) {
+        warn(message)
+      } else {
+        throw new ProducerFencedException(message)
+      }
     }
   }
 
@@ -261,8 +248,7 @@ private[log] class ProducerAppendInfo(val topicPartition: TopicPartition,
     nextSeq == lastSeq + 1L || (nextSeq == 0 && lastSeq == Int.MaxValue)
   }
 
-  def append(batch: RecordBatch,
-             firstOffsetMetadataOpt: Option[LogOffsetMetadata]): Option[CompletedTxn] = {
+  def append(batch: RecordBatch, firstOffsetMetadataOpt: Option[LogOffsetMetadata]): Option[CompletedTxn] = {
     if (batch.isControlBatch) {
       val recordIterator = batch.iterator
       if (recordIterator.hasNext) {
@@ -276,21 +262,21 @@ private[log] class ProducerAppendInfo(val topicPartition: TopicPartition,
       }
     } else {
       val firstOffsetMetadata = firstOffsetMetadataOpt.getOrElse(LogOffsetMetadata(batch.baseOffset))
-      append(batch.producerEpoch, batch.baseSequence, batch.lastSequence, batch.maxTimestamp,
+      appendDataBatch(batch.producerEpoch, batch.baseSequence, batch.lastSequence, batch.maxTimestamp,
         firstOffsetMetadata, batch.lastOffset, batch.isTransactional)
       None
     }
   }
 
-  def append(epoch: Short,
-             firstSeq: Int,
-             lastSeq: Int,
-             lastTimestamp: Long,
-             firstOffsetMetadata: LogOffsetMetadata,
-             lastOffset: Long,
-             isTransactional: Boolean): Unit = {
+  def appendDataBatch(epoch: Short,
+                      firstSeq: Int,
+                      lastSeq: Int,
+                      lastTimestamp: Long,
+                      firstOffsetMetadata: LogOffsetMetadata,
+                      lastOffset: Long,
+                      isTransactional: Boolean): Unit = {
     val firstOffset = firstOffsetMetadata.messageOffset
-    maybeValidateAppend(epoch, firstSeq, firstOffset)
+    maybeValidateDataBatch(epoch, firstSeq, firstOffset)
     updatedEntry.addBatch(epoch, lastSeq, lastOffset, (lastOffset - firstOffset).toInt, lastTimestamp)
 
     updatedEntry.currentTxnFirstOffset match {
@@ -308,18 +294,26 @@ private[log] class ProducerAppendInfo(val topicPartition: TopicPartition,
     }
   }
 
+  private def checkCoordinatorEpoch(endTxnMarker: EndTransactionMarker, offset: Long): Unit = {
+    if (updatedEntry.coordinatorEpoch > endTxnMarker.coordinatorEpoch) {
+      if (origin == AppendOrigin.Replication) {
+        info(s"Detected invalid coordinator epoch for producerId $producerId at " +
+          s"offset $offset in partition $topicPartition: ${endTxnMarker.coordinatorEpoch} " +
+          s"is older than previously known coordinator epoch ${updatedEntry.coordinatorEpoch}")
+      } else {
+        throw new TransactionCoordinatorFencedException(s"Invalid coordinator epoch for producerId $producerId at " +
+          s"offset $offset in partition $topicPartition: ${endTxnMarker.coordinatorEpoch} " +
+          s"(zombie), ${updatedEntry.coordinatorEpoch} (current)")
+      }
+    }
+  }
+
   def appendEndTxnMarker(endTxnMarker: EndTransactionMarker,
                          producerEpoch: Short,
                          offset: Long,
                          timestamp: Long): CompletedTxn = {
     checkProducerEpoch(producerEpoch, offset)
-
-    if (updatedEntry.coordinatorEpoch > endTxnMarker.coordinatorEpoch)
-      throw new TransactionCoordinatorFencedException(s"Invalid coordinator epoch for producerId $producerId at " +
-        s"offset $offset in partition $topicPartition: ${endTxnMarker.coordinatorEpoch} " +
-        s"(zombie), ${updatedEntry.coordinatorEpoch} (current)")
-
-    updatedEntry.maybeUpdateEpoch(producerEpoch)
+    checkCoordinatorEpoch(endTxnMarker, offset)
 
     val firstOffset = updatedEntry.currentTxnFirstOffset match {
       case Some(txnFirstOffset) => txnFirstOffset
@@ -328,8 +322,11 @@ private[log] class ProducerAppendInfo(val topicPartition: TopicPartition,
         offset
     }
 
+    updatedEntry.maybeUpdateProducerEpoch(producerEpoch)
     updatedEntry.currentTxnFirstOffset = None
     updatedEntry.coordinatorEpoch = endTxnMarker.coordinatorEpoch
+    updatedEntry.lastTimestamp = timestamp
+
     CompletedTxn(producerId, firstOffset, offset, endTxnMarker.controlType == ControlRecordType.ABORT)
   }
 
@@ -345,6 +342,7 @@ private[log] class ProducerAppendInfo(val topicPartition: TopicPartition,
       s"lastSequence=${updatedEntry.lastSeq}, " +
       s"currentTxnFirstOffset=${updatedEntry.currentTxnFirstOffset}, " +
       s"coordinatorEpoch=${updatedEntry.coordinatorEpoch}, " +
+      s"lastTimestamp=${updatedEntry.lastTimestamp}, " +
       s"startedTransactions=$transactions)"
   }
 }
@@ -398,7 +396,7 @@ object ProducerStateManager {
 
       struct.getArray(ProducerEntriesField).map { producerEntryObj =>
         val producerEntryStruct = producerEntryObj.asInstanceOf[Struct]
-        val producerId: Long = producerEntryStruct.getLong(ProducerIdField)
+        val producerId = producerEntryStruct.getLong(ProducerIdField)
         val producerEpoch = producerEntryStruct.getShort(ProducerEpochField)
         val seq = producerEntryStruct.getInt(LastSequenceField)
         val offset = producerEntryStruct.getLong(LastOffsetField)
@@ -406,8 +404,12 @@ object ProducerStateManager {
         val offsetDelta = producerEntryStruct.getInt(OffsetDeltaField)
         val coordinatorEpoch = producerEntryStruct.getInt(CoordinatorEpochField)
         val currentTxnFirstOffset = producerEntryStruct.getLong(CurrentTxnFirstOffsetField)
-        val newEntry = new ProducerStateEntry(producerId, mutable.Queue[BatchMetadata](BatchMetadata(seq, offset, offsetDelta, timestamp)), producerEpoch,
-          coordinatorEpoch, if (currentTxnFirstOffset >= 0) Some(currentTxnFirstOffset) else None)
+        val lastAppendedDataBatches = mutable.Queue.empty[BatchMetadata]
+        if (offset >= 0)
+          lastAppendedDataBatches += BatchMetadata(seq, offset, offsetDelta, timestamp)
+
+        val newEntry = new ProducerStateEntry(producerId, lastAppendedDataBatches, producerEpoch,
+          coordinatorEpoch, timestamp, if (currentTxnFirstOffset >= 0) Some(currentTxnFirstOffset) else None)
         newEntry
       }
     } catch {
@@ -621,17 +623,9 @@ class ProducerStateManager(val topicPartition: TopicPartition,
     }
   }
 
-  def prepareUpdate(producerId: Long, isFromClient: Boolean): ProducerAppendInfo = {
-    val validationToPerform =
-      if (!isFromClient)
-        ValidationType.None
-      else if (topicPartition.topic == Topic.GROUP_METADATA_TOPIC_NAME)
-        ValidationType.EpochOnly
-      else
-        ValidationType.Full
-
+  def prepareUpdate(producerId: Long, origin: AppendOrigin): ProducerAppendInfo = {
     val currentEntry = lastEntry(producerId).getOrElse(ProducerStateEntry.empty(producerId))
-    new ProducerAppendInfo(topicPartition, producerId, currentEntry, validationToPerform)
+    new ProducerAppendInfo(topicPartition, producerId, currentEntry, origin)
   }
 
   /**
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index abfd03a2322ae..e3d53a1089c07 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -33,6 +33,7 @@ import kafka.common.OffsetAndMetadata
 import kafka.controller.{KafkaController, ReplicaAssignment}
 import kafka.coordinator.group.{GroupCoordinator, JoinGroupResult, LeaveGroupResult, SyncGroupResult}
 import kafka.coordinator.transaction.{InitProducerIdResult, TransactionCoordinator}
+import kafka.log.AppendOrigin
 import kafka.message.ZStdCompressionCodec
 import kafka.network.RequestChannel
 import kafka.security.authorizer.AuthorizerUtils
@@ -571,7 +572,7 @@ class KafkaApis(val requestChannel: RequestChannel,
         timeout = produceRequest.timeout.toLong,
         requiredAcks = produceRequest.acks,
         internalTopicsAllowed = internalTopicsAllowed,
-        isFromClient = true,
+        origin = AppendOrigin.Client,
         entriesPerPartition = authorizedRequestInfo,
         responseCallback = sendResponseCallback,
         recordConversionStatsCallback = processingStatsCallback)
@@ -1996,7 +1997,7 @@ class KafkaApis(val requestChannel: RequestChannel,
           timeout = config.requestTimeoutMs.toLong,
           requiredAcks = -1,
           internalTopicsAllowed = true,
-          isFromClient = false,
+          origin = AppendOrigin.Coordinator,
           entriesPerPartition = controlRecords,
           responseCallback = maybeSendResponseCallback(producerId, marker.transactionResult))
       }
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index 42302bb7e527f..180b0e7d171da 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -506,7 +506,7 @@ class ReplicaManager(val config: KafkaConfig,
   def appendRecords(timeout: Long,
                     requiredAcks: Short,
                     internalTopicsAllowed: Boolean,
-                    isFromClient: Boolean,
+                    origin: AppendOrigin,
                     entriesPerPartition: Map[TopicPartition, MemoryRecords],
                     responseCallback: Map[TopicPartition, PartitionResponse] => Unit,
                     delayedProduceLock: Option[Lock] = None,
@@ -514,7 +514,7 @@ class ReplicaManager(val config: KafkaConfig,
     if (isValidRequiredAcks(requiredAcks)) {
       val sTime = time.milliseconds
       val localProduceResults = appendToLocalLog(internalTopicsAllowed = internalTopicsAllowed,
-        isFromClient = isFromClient, entriesPerPartition, requiredAcks)
+        origin, entriesPerPartition, requiredAcks)
       debug("Produce to local log in %d ms".format(time.milliseconds - sTime))
 
       val produceStatus = localProduceResults.map { case (topicPartition, result) =>
@@ -773,7 +773,7 @@ class ReplicaManager(val config: KafkaConfig,
    * Append the messages to the local replica logs
    */
   private def appendToLocalLog(internalTopicsAllowed: Boolean,
-                               isFromClient: Boolean,
+                               origin: AppendOrigin,
                                entriesPerPartition: Map[TopicPartition, MemoryRecords],
                                requiredAcks: Short): Map[TopicPartition, LogAppendResult] = {
 
@@ -802,7 +802,7 @@ class ReplicaManager(val config: KafkaConfig,
       } else {
         try {
           val partition = getPartitionOrException(topicPartition, expectLeader = true)
-          val info = partition.appendRecordsToLeader(records, isFromClient, requiredAcks)
+          val info = partition.appendRecordsToLeader(records, origin, requiredAcks)
           val numAppendedMessages = info.numMessages
 
           // update stats for successfully appended bytes and messages as bytesInRate and messageInRate
diff --git a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
index 133ce1db286c9..7f50ce1fd8b64 100644
--- a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
+++ b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
@@ -499,8 +499,8 @@ class PartitionTest {
 
     // after makeLeader(() call, partition should know about all the replicas
     // append records with initial leader epoch
-    partition.appendRecordsToLeader(batch1, isFromClient = true)
-    partition.appendRecordsToLeader(batch2, isFromClient = true)
+    partition.appendRecordsToLeader(batch1, origin = AppendOrigin.Client, requiredAcks = 0)
+    partition.appendRecordsToLeader(batch2, origin = AppendOrigin.Client, requiredAcks = 0)
     assertEquals("Expected leader's HW not move", partition.localLogOrException.logStartOffset,
       partition.localLogOrException.highWatermark)
 
@@ -756,7 +756,7 @@ class PartitionTest {
       new SimpleRecord("k2".getBytes, "v2".getBytes),
       new SimpleRecord("k3".getBytes, "v3".getBytes)),
       baseOffset = 0L)
-    partition.appendRecordsToLeader(records, isFromClient = true)
+    partition.appendRecordsToLeader(records, origin = AppendOrigin.Client, requiredAcks = 0)
 
     def fetchLatestOffset(isolationLevel: Option[IsolationLevel]): TimestampAndOffset = {
       val res = partition.fetchOffsetForTimestamp(ListOffsetRequest.LATEST_TIMESTAMP,
@@ -875,8 +875,9 @@ class PartitionTest {
 
     // after makeLeader(() call, partition should know about all the replicas
     // append records with initial leader epoch
-    val lastOffsetOfFirstBatch = partition.appendRecordsToLeader(batch1, isFromClient = true).lastOffset
-    partition.appendRecordsToLeader(batch2, isFromClient = true)
+    val lastOffsetOfFirstBatch = partition.appendRecordsToLeader(batch1, origin = AppendOrigin.Client,
+      requiredAcks = 0).lastOffset
+    partition.appendRecordsToLeader(batch2, origin = AppendOrigin.Client, requiredAcks = 0)
     assertEquals("Expected leader's HW not move", partition.localLogOrException.logStartOffset,
       partition.log.get.highWatermark)
 
@@ -919,7 +920,7 @@ class PartitionTest {
     val currentLeaderEpochStartOffset = partition.localLogOrException.logEndOffset
 
     // append records with the latest leader epoch
-    partition.appendRecordsToLeader(batch3, isFromClient = true)
+    partition.appendRecordsToLeader(batch3, origin = AppendOrigin.Client, requiredAcks = 0)
 
     // fetch from follower not in ISR from log start offset should not add this follower to ISR
     updateFollowerFetchState(follower1, LogOffsetMetadata(0))
@@ -1014,7 +1015,11 @@ class PartitionTest {
       // Append records to partitions, one partition-per-thread
       val futures = partitions.map { partition =>
         executor.submit(CoreUtils.runnable {
-          (1 to 10000).foreach { _ => partition.appendRecordsToLeader(createRecords(baseOffset = 0), isFromClient = true) }
+          (1 to 10000).foreach { _ =>
+            partition.appendRecordsToLeader(createRecords(baseOffset = 0),
+              origin = AppendOrigin.Client,
+              requiredAcks = 0)
+          }
         })
       }
       futures.foreach(_.get(15, TimeUnit.SECONDS))
diff --git a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala
index 709d5a04b47a6..c39a23f2b9686 100644
--- a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala
@@ -23,7 +23,7 @@ import java.util.concurrent.atomic.AtomicInteger
 import java.util.concurrent.locks.Lock
 
 import kafka.coordinator.AbstractCoordinatorConcurrencyTest._
-import kafka.log.Log
+import kafka.log.{AppendOrigin, Log}
 import kafka.server._
 import kafka.utils._
 import kafka.utils.timer.MockTimer
@@ -173,7 +173,7 @@ object AbstractCoordinatorConcurrencyTest {
     override def appendRecords(timeout: Long,
                                requiredAcks: Short,
                                internalTopicsAllowed: Boolean,
-                               isFromClient: Boolean,
+                               origin: AppendOrigin,
                                entriesPerPartition: Map[TopicPartition, MemoryRecords],
                                responseCallback: Map[TopicPartition, PartitionResponse] => Unit,
                                delayedProduceLock: Option[Lock] = None,
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
index cb67c7060b13b..54116a376ab88 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
@@ -33,6 +33,7 @@ import java.util.concurrent.TimeUnit
 import java.util.concurrent.locks.ReentrantLock
 
 import kafka.cluster.Partition
+import kafka.log.AppendOrigin
 import kafka.zk.KafkaZkClient
 import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription
 import org.apache.kafka.clients.consumer.internals.ConsumerProtocol
@@ -3221,7 +3222,7 @@ class GroupCoordinatorTest {
     EasyMock.expect(replicaManager.appendRecords(EasyMock.anyLong(),
       EasyMock.anyShort(),
       internalTopicsAllowed = EasyMock.eq(true),
-      isFromClient = EasyMock.eq(false),
+      origin = EasyMock.eq(AppendOrigin.Coordinator),
       EasyMock.anyObject().asInstanceOf[Map[TopicPartition, MemoryRecords]],
       EasyMock.capture(capturedArgument),
       EasyMock.anyObject().asInstanceOf[Option[ReentrantLock]],
@@ -3336,7 +3337,7 @@ class GroupCoordinatorTest {
     EasyMock.expect(replicaManager.appendRecords(EasyMock.anyLong(),
       EasyMock.anyShort(),
       internalTopicsAllowed = EasyMock.eq(true),
-      isFromClient = EasyMock.eq(false),
+      origin = EasyMock.eq(AppendOrigin.Coordinator),
       EasyMock.anyObject().asInstanceOf[Map[TopicPartition, MemoryRecords]],
       EasyMock.capture(capturedArgument),
       EasyMock.anyObject().asInstanceOf[Option[ReentrantLock]],
@@ -3366,7 +3367,7 @@ class GroupCoordinatorTest {
     EasyMock.expect(replicaManager.appendRecords(EasyMock.anyLong(),
       EasyMock.anyShort(),
       internalTopicsAllowed = EasyMock.eq(true),
-      isFromClient = EasyMock.eq(false),
+      origin = EasyMock.eq(AppendOrigin.Coordinator),
       EasyMock.anyObject().asInstanceOf[Map[TopicPartition, MemoryRecords]],
       EasyMock.capture(capturedArgument),
       EasyMock.anyObject().asInstanceOf[Option[ReentrantLock]],
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
index b0f81c8fe49be..b3a7f661471ad 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
@@ -28,7 +28,7 @@ import javax.management.ObjectName
 import kafka.api._
 import kafka.cluster.Partition
 import kafka.common.OffsetAndMetadata
-import kafka.log.{Log, LogAppendInfo}
+import kafka.log.{AppendOrigin, Log, LogAppendInfo}
 import kafka.server.{FetchDataInfo, FetchLogEnd, HostedPartition, KafkaConfig, LogOffsetMetadata, ReplicaManager}
 import kafka.utils.{KafkaScheduler, MockTime, TestUtils}
 import kafka.zk.KafkaZkClient
@@ -1345,7 +1345,7 @@ class GroupMetadataManagerTest {
 
     EasyMock.reset(partition)
     EasyMock.expect(partition.appendRecordsToLeader(EasyMock.anyObject(classOf[MemoryRecords]),
-      isFromClient = EasyMock.eq(false), requiredAcks = EasyMock.anyInt()))
+      origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt()))
       .andReturn(LogAppendInfo.UnknownLogAppendInfo)
     EasyMock.replay(partition)
 
@@ -1380,7 +1380,7 @@ class GroupMetadataManagerTest {
     EasyMock.expect(replicaManager.getMagic(EasyMock.anyObject())).andStubReturn(Some(RecordBatch.CURRENT_MAGIC_VALUE))
     mockGetPartition()
     EasyMock.expect(partition.appendRecordsToLeader(EasyMock.capture(recordsCapture),
-      isFromClient = EasyMock.eq(false), requiredAcks = EasyMock.anyInt()))
+      origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt()))
       .andReturn(LogAppendInfo.UnknownLogAppendInfo)
     EasyMock.replay(replicaManager, partition)
 
@@ -1428,7 +1428,7 @@ class GroupMetadataManagerTest {
     EasyMock.expect(replicaManager.getMagic(EasyMock.anyObject())).andStubReturn(Some(RecordBatch.CURRENT_MAGIC_VALUE))
     mockGetPartition()
     EasyMock.expect(partition.appendRecordsToLeader(EasyMock.capture(recordsCapture),
-      isFromClient = EasyMock.eq(false), requiredAcks = EasyMock.anyInt()))
+      origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt()))
       .andReturn(LogAppendInfo.UnknownLogAppendInfo)
     EasyMock.replay(replicaManager, partition)
 
@@ -1503,7 +1503,7 @@ class GroupMetadataManagerTest {
     val recordsCapture: Capture[MemoryRecords] = EasyMock.newCapture()
 
     EasyMock.expect(partition.appendRecordsToLeader(EasyMock.capture(recordsCapture),
-      isFromClient = EasyMock.eq(false), requiredAcks = EasyMock.anyInt()))
+      origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt()))
       .andReturn(LogAppendInfo.UnknownLogAppendInfo)
     EasyMock.replay(partition)
 
@@ -1604,7 +1604,7 @@ class GroupMetadataManagerTest {
     // expect the offset tombstone
     EasyMock.reset(partition)
     EasyMock.expect(partition.appendRecordsToLeader(EasyMock.anyObject(classOf[MemoryRecords]),
-      isFromClient = EasyMock.eq(false), requiredAcks = EasyMock.anyInt()))
+      origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt()))
       .andReturn(LogAppendInfo.UnknownLogAppendInfo)
     EasyMock.replay(partition)
 
@@ -1628,7 +1628,7 @@ class GroupMetadataManagerTest {
     // expect the offset tombstone
     EasyMock.reset(partition)
     EasyMock.expect(partition.appendRecordsToLeader(EasyMock.anyObject(classOf[MemoryRecords]),
-      isFromClient = EasyMock.eq(false), requiredAcks = EasyMock.anyInt()))
+      origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt()))
       .andReturn(LogAppendInfo.UnknownLogAppendInfo)
     EasyMock.replay(partition)
 
@@ -1671,7 +1671,7 @@ class GroupMetadataManagerTest {
     // expect the offset tombstone
     EasyMock.reset(partition)
     EasyMock.expect(partition.appendRecordsToLeader(EasyMock.anyObject(classOf[MemoryRecords]),
-      isFromClient = EasyMock.eq(false), requiredAcks = EasyMock.anyInt()))
+      origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt()))
       .andReturn(LogAppendInfo.UnknownLogAppendInfo)
     EasyMock.replay(partition)
 
@@ -1749,7 +1749,7 @@ class GroupMetadataManagerTest {
     // expect the offset tombstone
     EasyMock.reset(partition)
     EasyMock.expect(partition.appendRecordsToLeader(EasyMock.anyObject(classOf[MemoryRecords]),
-      isFromClient = EasyMock.eq(false), requiredAcks = EasyMock.anyInt()))
+      origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt()))
       .andReturn(LogAppendInfo.UnknownLogAppendInfo)
     EasyMock.replay(partition)
 
@@ -1877,7 +1877,7 @@ class GroupMetadataManagerTest {
 
     // expect the offset tombstone
     EasyMock.expect(partition.appendRecordsToLeader(EasyMock.anyObject(classOf[MemoryRecords]),
-      isFromClient = EasyMock.eq(false), requiredAcks = EasyMock.anyInt()))
+      origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt()))
       .andReturn(LogAppendInfo.UnknownLogAppendInfo)
     EasyMock.expectLastCall().times(1)
 
@@ -2196,7 +2196,7 @@ class GroupMetadataManagerTest {
     EasyMock.expect(replicaManager.appendRecords(EasyMock.anyLong(),
       EasyMock.anyShort(),
       internalTopicsAllowed = EasyMock.eq(true),
-      isFromClient = EasyMock.eq(false),
+      origin = EasyMock.eq(AppendOrigin.Coordinator),
       EasyMock.anyObject().asInstanceOf[Map[TopicPartition, MemoryRecords]],
       EasyMock.capture(capturedArgument),
       EasyMock.anyObject().asInstanceOf[Option[ReentrantLock]],
@@ -2212,7 +2212,7 @@ class GroupMetadataManagerTest {
     EasyMock.expect(replicaManager.appendRecords(EasyMock.anyLong(),
       EasyMock.anyShort(),
       internalTopicsAllowed = EasyMock.eq(true),
-      isFromClient = EasyMock.eq(false),
+      origin = EasyMock.eq(AppendOrigin.Coordinator),
       EasyMock.capture(capturedRecords),
       EasyMock.capture(capturedCallback),
       EasyMock.anyObject().asInstanceOf[Option[ReentrantLock]],
diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala
index e462c1458838d..5ab3f82431f97 100644
--- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala
@@ -19,12 +19,11 @@ package kafka.coordinator.transaction
 import java.lang.management.ManagementFactory
 import java.nio.ByteBuffer
 import java.util.concurrent.locks.ReentrantLock
-import javax.management.ObjectName
 
-import kafka.log.Log
+import javax.management.ObjectName
+import kafka.log.{AppendOrigin, Log}
 import kafka.server.{FetchDataInfo, FetchLogEnd, LogOffsetMetadata, ReplicaManager}
 import kafka.utils.{MockScheduler, Pool}
-import org.scalatest.Assertions.fail
 import kafka.zk.KafkaZkClient
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.internals.Topic.TRANSACTION_STATE_TOPIC_NAME
@@ -34,13 +33,13 @@ import org.apache.kafka.common.record._
 import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse
 import org.apache.kafka.common.requests.TransactionResult
 import org.apache.kafka.common.utils.MockTime
+import org.easymock.{Capture, EasyMock, IAnswer}
 import org.junit.Assert.{assertEquals, assertFalse, assertTrue}
 import org.junit.{After, Before, Test}
-import org.easymock.{Capture, EasyMock, IAnswer}
+import org.scalatest.Assertions.fail
 
-import scala.collection.Map
-import scala.collection.mutable
 import scala.collection.JavaConverters._
+import scala.collection.{Map, mutable}
 
 class TransactionStateManagerTest {
 
@@ -558,7 +557,7 @@ class TransactionStateManagerTest {
         EasyMock.expect(replicaManager.appendRecords(EasyMock.anyLong(),
           EasyMock.eq((-1).toShort),
           EasyMock.eq(true),
-          EasyMock.eq(false),
+          EasyMock.eq(AppendOrigin.Coordinator),
           EasyMock.eq(recordsByPartition),
           EasyMock.capture(capturedArgument),
           EasyMock.anyObject().asInstanceOf[Option[ReentrantLock]],
@@ -670,7 +669,7 @@ class TransactionStateManagerTest {
     EasyMock.expect(replicaManager.appendRecords(EasyMock.anyLong(),
       EasyMock.anyShort(),
       internalTopicsAllowed = EasyMock.eq(true),
-      isFromClient = EasyMock.eq(false),
+      origin = EasyMock.eq(AppendOrigin.Coordinator),
       EasyMock.anyObject().asInstanceOf[Map[TopicPartition, MemoryRecords]],
       EasyMock.capture(capturedArgument),
       EasyMock.anyObject().asInstanceOf[Option[ReentrantLock]],
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
index fcb82e5ba845d..bdf5cb74c66e1 100644
--- a/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
@@ -509,7 +509,8 @@ class LogCleanerManagerTest extends Logging {
     assertEquals(0L, cleanableOffsets._2)
 
     log.appendAsLeader(MemoryRecords.withEndTransactionMarker(time.milliseconds(), producerId, producerEpoch,
-      new EndTransactionMarker(ControlRecordType.ABORT, 15)), leaderEpoch = 0, isFromClient = false)
+      new EndTransactionMarker(ControlRecordType.ABORT, 15)), leaderEpoch = 0,
+      origin = AppendOrigin.Coordinator)
     log.roll()
     log.updateHighWatermark(4L)
 
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala
index bb30287bb12bd..18d86b68c1fa9 100755
--- a/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala
@@ -265,10 +265,10 @@ class LogCleanerTest {
     appendProducer1(Seq(1, 2))
     appendProducer2(Seq(2, 3))
     appendProducer1(Seq(3, 4))
-    log.appendAsLeader(abortMarker(pid1, producerEpoch), leaderEpoch = 0, isFromClient = false)
-    log.appendAsLeader(commitMarker(pid2, producerEpoch), leaderEpoch = 0, isFromClient = false)
+    log.appendAsLeader(abortMarker(pid1, producerEpoch), leaderEpoch = 0, origin = AppendOrigin.Coordinator)
+    log.appendAsLeader(commitMarker(pid2, producerEpoch), leaderEpoch = 0, origin = AppendOrigin.Coordinator)
     appendProducer1(Seq(2))
-    log.appendAsLeader(commitMarker(pid1, producerEpoch), leaderEpoch = 0, isFromClient = false)
+    log.appendAsLeader(commitMarker(pid1, producerEpoch), leaderEpoch = 0, origin = AppendOrigin.Coordinator)
 
     val abortedTransactions = log.collectAbortedTransactions(log.logStartOffset, log.logEndOffset)
 
@@ -306,11 +306,11 @@ class LogCleanerTest {
     appendProducer2(Seq(5, 6))
     appendProducer3(Seq(6, 7))
     appendProducer1(Seq(7, 8))
-    log.appendAsLeader(abortMarker(pid2, producerEpoch), leaderEpoch = 0, isFromClient = false)
+    log.appendAsLeader(abortMarker(pid2, producerEpoch), leaderEpoch = 0, origin = AppendOrigin.Coordinator)
     appendProducer3(Seq(8, 9))
-    log.appendAsLeader(commitMarker(pid3, producerEpoch), leaderEpoch = 0, isFromClient = false)
+    log.appendAsLeader(commitMarker(pid3, producerEpoch), leaderEpoch = 0, origin = AppendOrigin.Coordinator)
     appendProducer1(Seq(9, 10))
-    log.appendAsLeader(abortMarker(pid1, producerEpoch), leaderEpoch = 0, isFromClient = false)
+    log.appendAsLeader(abortMarker(pid1, producerEpoch), leaderEpoch = 0, origin = AppendOrigin.Coordinator)
 
     // we have only cleaned the records in the first segment
     val dirtyOffset = cleaner.clean(LogToClean(new TopicPartition("test", 0), log, 0L, log.activeSegment.baseOffset))._1
@@ -341,9 +341,9 @@ class LogCleanerTest {
 
     appendProducer(Seq(1))
     appendProducer(Seq(2, 3))
-    log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, isFromClient = false)
+    log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, origin = AppendOrigin.Coordinator)
     appendProducer(Seq(2))
-    log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, isFromClient = false)
+    log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, origin = AppendOrigin.Coordinator)
     log.roll()
 
     // cannot remove the marker in this pass because there are still valid records
@@ -352,7 +352,7 @@ class LogCleanerTest {
     assertEquals(List(0, 2, 3, 4, 5), offsetsInLog(log))
 
     appendProducer(Seq(1, 3))
-    log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, isFromClient = false)
+    log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, origin = AppendOrigin.Coordinator)
     log.roll()
 
     // the first cleaning preserves the commit marker (at offset 3) since there were still records for the transaction
@@ -389,10 +389,10 @@ class LogCleanerTest {
     val appendProducer = appendTransactionalAsLeader(log, producerId, producerEpoch)
 
     appendProducer(Seq(1))
-    log.appendAsLeader(abortMarker(producerId, producerEpoch), leaderEpoch = 0, isFromClient = false)
+    log.appendAsLeader(abortMarker(producerId, producerEpoch), leaderEpoch = 0, origin = AppendOrigin.Coordinator)
     appendProducer(Seq(2))
     appendProducer(Seq(2))
-    log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, isFromClient = false)
+    log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, origin = AppendOrigin.Coordinator)
     log.roll()
 
     cleaner.doClean(LogToClean(tp, log, 0L, log.activeSegment.baseOffset), deleteHorizonMs = Long.MaxValue)
@@ -422,14 +422,16 @@ class LogCleanerTest {
 
     // [{Producer1: 2, 3}], [{Producer2: 2, 3}, {Producer2: Commit}]
     producer2(Seq(2, 3)) // offsets 2, 3
-    log.appendAsLeader(commitMarker(2L, producerEpoch), leaderEpoch = 0, isFromClient = false) // offset 4
+    log.appendAsLeader(commitMarker(2L, producerEpoch), leaderEpoch = 0,
+      origin = AppendOrigin.Coordinator) // offset 4
     log.roll()
 
     // [{Producer1: 2, 3}], [{Producer2: 2, 3}, {Producer2: Commit}], [{2}, {3}, {Producer1: Commit}]
     //  {0, 1},              {2, 3},            {4},                   {5}, {6}, {7} ==> Offsets
     log.appendAsLeader(record(2, 2), leaderEpoch = 0) // offset 5
     log.appendAsLeader(record(3, 3), leaderEpoch = 0) // offset 6
-    log.appendAsLeader(commitMarker(1L, producerEpoch), leaderEpoch = 0, isFromClient = false) // offset 7
+    log.appendAsLeader(commitMarker(1L, producerEpoch), leaderEpoch = 0,
+      origin = AppendOrigin.Coordinator) // offset 7
     log.roll()
 
     // first time through the records are removed
@@ -450,7 +452,8 @@ class LogCleanerTest {
     // [{Producer1: EmptyBatch}, {Producer2: EmptyBatch}, {Producer2: Commit}, {2}, {3}, {Producer1: Commit}, {Producer2: 1}, {Producer2: Commit}]
     //  {1},                     {3},                     {4},                 {5}, {6}, {7},                 {8},            {9} ==> Offsets
     producer2(Seq(1)) // offset 8
-    log.appendAsLeader(commitMarker(2L, producerEpoch), leaderEpoch = 0, isFromClient = false) // offset 9
+    log.appendAsLeader(commitMarker(2L, producerEpoch), leaderEpoch = 0,
+      origin = AppendOrigin.Coordinator) // offset 9
     log.roll()
 
     // Expected State: [{Producer1: EmptyBatch}, {Producer2: Commit}, {2}, {3}, {Producer1: Commit}, {Producer2: 1}, {Producer2: Commit}]
@@ -477,7 +480,8 @@ class LogCleanerTest {
     val producerEpoch = 0.toShort
 
     // [{Producer1: Commit}, {2}, {3}]
-    log.appendAsLeader(commitMarker(1L, producerEpoch), leaderEpoch = 0, isFromClient = false) // offset 7
+    log.appendAsLeader(commitMarker(1L, producerEpoch), leaderEpoch = 0,
+      origin = AppendOrigin.Coordinator) // offset 1
     log.appendAsLeader(record(2, 2), leaderEpoch = 0) // offset 2
     log.appendAsLeader(record(3, 3), leaderEpoch = 0) // offset 3
     log.roll()
@@ -511,7 +515,7 @@ class LogCleanerTest {
     appendTransaction(Seq(1))
     log.roll()
 
-    log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, isFromClient = false)
+    log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, origin = AppendOrigin.Coordinator)
     log.roll()
 
     // Both the record and the marker should remain after cleaning
@@ -534,7 +538,7 @@ class LogCleanerTest {
     appendTransaction(Seq(1))
     log.roll()
 
-    log.appendAsLeader(abortMarker(producerId, producerEpoch), leaderEpoch = 0, isFromClient = false)
+    log.appendAsLeader(abortMarker(producerId, producerEpoch), leaderEpoch = 0, origin = AppendOrigin.Coordinator)
     log.roll()
 
     // Both the batch and the marker should remain after cleaning. The batch is retained
@@ -564,9 +568,9 @@ class LogCleanerTest {
 
     appendProducer(Seq(1))
     appendProducer(Seq(2, 3))
-    log.appendAsLeader(abortMarker(producerId, producerEpoch), leaderEpoch = 0, isFromClient = false)
+    log.appendAsLeader(abortMarker(producerId, producerEpoch), leaderEpoch = 0, origin = AppendOrigin.Coordinator)
     appendProducer(Seq(3))
-    log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, isFromClient = false)
+    log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, origin = AppendOrigin.Coordinator)
     log.roll()
 
     // delete horizon set to 0 to verify marker is not removed early
@@ -593,13 +597,15 @@ class LogCleanerTest {
     logProps.put(LogConfig.SegmentBytesProp, 2048: java.lang.Integer)
     val log = makeLog(config = LogConfig.fromProps(logConfig.originals, logProps))
 
-    val appendFirstTransaction = appendTransactionalAsLeader(log, producerId, producerEpoch, isFromClient = false)
+    val appendFirstTransaction = appendTransactionalAsLeader(log, producerId, producerEpoch,
+      origin = AppendOrigin.Replication)
     appendFirstTransaction(Seq(1))
-    log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, isFromClient = false)
+    log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, origin = AppendOrigin.Coordinator)
 
-    val appendSecondTransaction = appendTransactionalAsLeader(log, producerId, producerEpoch, isFromClient = false)
+    val appendSecondTransaction = appendTransactionalAsLeader(log, producerId, producerEpoch,
+      origin = AppendOrigin.Replication)
     appendSecondTransaction(Seq(2))
-    log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, isFromClient = false)
+    log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, origin = AppendOrigin.Coordinator)
 
     log.appendAsLeader(record(1, 1), leaderEpoch = 0)
     log.appendAsLeader(record(2, 1), leaderEpoch = 0)
@@ -632,7 +638,7 @@ class LogCleanerTest {
     val appendProducer = appendTransactionalAsLeader(log, producerId, producerEpoch)
 
     appendProducer(Seq(2, 3)) // batch last offset is 1
-    log.appendAsLeader(abortMarker(producerId, producerEpoch), leaderEpoch = 0, isFromClient = false)
+    log.appendAsLeader(abortMarker(producerId, producerEpoch), leaderEpoch = 0, origin = AppendOrigin.Coordinator)
     log.roll()
 
     def assertAbortedTransactionIndexed(): Unit = {
@@ -872,7 +878,7 @@ class LogCleanerTest {
 
     appendProducer(Seq(1))
     appendProducer(Seq(2, 3))
-    log.appendAsLeader(abortMarker(producerId, producerEpoch), leaderEpoch = 0, isFromClient = false)
+    log.appendAsLeader(abortMarker(producerId, producerEpoch), leaderEpoch = 0, origin = AppendOrigin.Coordinator)
     log.roll()
 
     cleaner.clean(LogToClean(new TopicPartition("test", 0), log, 0L, log.activeSegment.baseOffset))
@@ -1625,8 +1631,8 @@ class LogCleanerTest {
                                           producerId: Long,
                                           producerEpoch: Short,
                                           leaderEpoch: Int = 0,
-                                          isFromClient: Boolean = true): Seq[Int] => LogAppendInfo = {
-    appendIdempotentAsLeader(log, producerId, producerEpoch, isTransactional = true, isFromClient = isFromClient)
+                                          origin: AppendOrigin = AppendOrigin.Client): Seq[Int] => LogAppendInfo = {
+    appendIdempotentAsLeader(log, producerId, producerEpoch, isTransactional = true, origin = origin)
   }
 
   private def appendIdempotentAsLeader(log: Log,
@@ -1634,7 +1640,7 @@ class LogCleanerTest {
                                        producerEpoch: Short,
                                        isTransactional: Boolean = false,
                                        leaderEpoch: Int = 0,
-                                       isFromClient: Boolean = true): Seq[Int] => LogAppendInfo = {
+                                       origin: AppendOrigin = AppendOrigin.Client): Seq[Int] => LogAppendInfo = {
     var sequence = 0
     keys: Seq[Int] => {
       val simpleRecords = keys.map { key =>
@@ -1646,7 +1652,7 @@ class LogCleanerTest {
       else
         MemoryRecords.withIdempotentRecords(CompressionType.NONE, producerId, producerEpoch, sequence, simpleRecords.toArray: _*)
       sequence += simpleRecords.size
-      log.appendAsLeader(records, leaderEpoch, isFromClient)
+      log.appendAsLeader(records, leaderEpoch, origin)
     }
   }
 
diff --git a/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala b/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala
index 5e0a1a83c3af0..6cc6354ef2cf8 100644
--- a/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala
@@ -336,7 +336,8 @@ class LogSegmentTest {
     // recover again, but this time assuming the transaction from pid2 began on a previous segment
     stateManager = new ProducerStateManager(topicPartition, logDir)
     stateManager.loadProducerEntry(new ProducerStateEntry(pid2,
-      mutable.Queue[BatchMetadata](BatchMetadata(10, 10L, 5, RecordBatch.NO_TIMESTAMP)), producerEpoch, 0, Some(75L)))
+      mutable.Queue[BatchMetadata](BatchMetadata(10, 10L, 5, RecordBatch.NO_TIMESTAMP)), producerEpoch,
+      0, RecordBatch.NO_TIMESTAMP, Some(75L)))
     segment.recover(stateManager)
     assertEquals(108L, stateManager.mapEndOffset)
 
diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala
index 4dba8bae07843..8eaf47c3555a3 100755
--- a/core/src/test/scala/unit/kafka/log/LogTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogTest.scala
@@ -299,7 +299,6 @@ class LogTest {
     assertFalse(Log.FutureDirPattern.matcher(name1).matches())
     val name2 = Log.logDeleteDirName(
       new TopicPartition("n" + String.join("", Collections.nCopies(248, "o")), 5))
-    System.out.println("name2 = " + name2)
     assertEquals(255, name2.length)
     assertTrue(Pattern.compile("n[o]{212}-5\\.[0-9a-z]{32}-delete").matcher(name2).matches())
     assertTrue(Log.DeleteDirPattern.matcher(name2).matches())
@@ -500,6 +499,39 @@ class LogTest {
     log.close()
   }
 
+
+  @Test
+  def testRecoverAfterNonMonotonicCoordinatorEpochWrite(): Unit = {
+    // Due to KAFKA-9144, we may encounter a coordinator epoch which goes backwards.
+    // This test case verifies that recovery logic relaxes validation in this case and
+    // just takes the latest write.
+
+    val producerId = 1L
+    val coordinatorEpoch = 5
+    val logConfig = LogTest.createLogConfig(segmentBytes = 1024 * 1024 * 5)
+    var log = createLog(logDir, logConfig)
+    val epoch = 0.toShort
+
+    val firstAppendTimestamp = mockTime.milliseconds()
+    appendEndTxnMarkerAsLeader(log, producerId, epoch, ControlRecordType.ABORT,
+      timestamp = firstAppendTimestamp, coordinatorEpoch = coordinatorEpoch)
+    assertEquals(firstAppendTimestamp, log.producerStateManager.lastEntry(producerId).get.lastTimestamp)
+
+    mockTime.sleep(log.maxProducerIdExpirationMs)
+    assertEquals(None, log.producerStateManager.lastEntry(producerId))
+
+    val secondAppendTimestamp = mockTime.milliseconds()
+    appendEndTxnMarkerAsLeader(log, producerId, epoch, ControlRecordType.ABORT,
+      timestamp = secondAppendTimestamp, coordinatorEpoch = coordinatorEpoch - 1)
+
+    log.close()
+
+    // Force recovery by setting the recoveryPoint to the log start
+    log = createLog(logDir, logConfig, recoveryPoint = 0L)
+    assertEquals(secondAppendTimestamp, log.producerStateManager.lastEntry(producerId).get.lastTimestamp)
+    log.close()
+  }
+
   @Test
   def testProducerSnapshotsRecoveryAfterUncleanShutdownV1(): Unit = {
     testProducerSnapshotsRecoveryAfterUncleanShutdown(ApiVersion.minSupportedFor(RecordVersion.V1).version)
@@ -959,10 +991,10 @@ class LogTest {
 
     // create a batch with a couple gaps to simulate compaction
     val records = TestUtils.records(producerId = pid, producerEpoch = epoch, sequence = seq, baseOffset = baseOffset, records = List(
-      new SimpleRecord(System.currentTimeMillis(), "a".getBytes),
-      new SimpleRecord(System.currentTimeMillis(), "key".getBytes, "b".getBytes),
-      new SimpleRecord(System.currentTimeMillis(), "c".getBytes),
-      new SimpleRecord(System.currentTimeMillis(), "key".getBytes, "d".getBytes)))
+      new SimpleRecord(mockTime.milliseconds(), "a".getBytes),
+      new SimpleRecord(mockTime.milliseconds(), "key".getBytes, "b".getBytes),
+      new SimpleRecord(mockTime.milliseconds(), "c".getBytes),
+      new SimpleRecord(mockTime.milliseconds(), "key".getBytes, "d".getBytes)))
     records.batches.asScala.foreach(_.setPartitionLeaderEpoch(0))
 
     val filtered = ByteBuffer.allocate(2048)
@@ -977,8 +1009,8 @@ class LogTest {
 
     // append some more data and then truncate to force rebuilding of the PID map
     val moreRecords = TestUtils.records(baseOffset = baseOffset + 4, records = List(
-      new SimpleRecord(System.currentTimeMillis(), "e".getBytes),
-      new SimpleRecord(System.currentTimeMillis(), "f".getBytes)))
+      new SimpleRecord(mockTime.milliseconds(), "e".getBytes),
+      new SimpleRecord(mockTime.milliseconds(), "f".getBytes)))
     moreRecords.batches.asScala.foreach(_.setPartitionLeaderEpoch(0))
     log.appendAsFollower(moreRecords)
 
@@ -1002,8 +1034,8 @@ class LogTest {
 
     // create an empty batch
     val records = TestUtils.records(producerId = pid, producerEpoch = epoch, sequence = seq, baseOffset = baseOffset, records = List(
-      new SimpleRecord(System.currentTimeMillis(), "key".getBytes, "a".getBytes),
-      new SimpleRecord(System.currentTimeMillis(), "key".getBytes, "b".getBytes)))
+      new SimpleRecord(mockTime.milliseconds(), "key".getBytes, "a".getBytes),
+      new SimpleRecord(mockTime.milliseconds(), "key".getBytes, "b".getBytes)))
     records.batches.asScala.foreach(_.setPartitionLeaderEpoch(0))
 
     val filtered = ByteBuffer.allocate(2048)
@@ -1018,8 +1050,8 @@ class LogTest {
 
     // append some more data and then truncate to force rebuilding of the PID map
     val moreRecords = TestUtils.records(baseOffset = baseOffset + 2, records = List(
-      new SimpleRecord(System.currentTimeMillis(), "e".getBytes),
-      new SimpleRecord(System.currentTimeMillis(), "f".getBytes)))
+      new SimpleRecord(mockTime.milliseconds(), "e".getBytes),
+      new SimpleRecord(mockTime.milliseconds(), "f".getBytes)))
     moreRecords.batches.asScala.foreach(_.setPartitionLeaderEpoch(0))
     log.appendAsFollower(moreRecords)
 
@@ -1043,10 +1075,10 @@ class LogTest {
 
     // create a batch with a couple gaps to simulate compaction
     val records = TestUtils.records(producerId = pid, producerEpoch = epoch, sequence = seq, baseOffset = baseOffset, records = List(
-      new SimpleRecord(System.currentTimeMillis(), "a".getBytes),
-      new SimpleRecord(System.currentTimeMillis(), "key".getBytes, "b".getBytes),
-      new SimpleRecord(System.currentTimeMillis(), "c".getBytes),
-      new SimpleRecord(System.currentTimeMillis(), "key".getBytes, "d".getBytes)))
+      new SimpleRecord(mockTime.milliseconds(), "a".getBytes),
+      new SimpleRecord(mockTime.milliseconds(), "key".getBytes, "b".getBytes),
+      new SimpleRecord(mockTime.milliseconds(), "c".getBytes),
+      new SimpleRecord(mockTime.milliseconds(), "key".getBytes, "d".getBytes)))
     records.batches.asScala.foreach(_.setPartitionLeaderEpoch(0))
 
     val filtered = ByteBuffer.allocate(2048)
@@ -1293,7 +1325,7 @@ class LogTest {
 
     val lastEntry = log.producerStateManager.lastEntry(producerId)
     assertTrue(lastEntry.isDefined)
-    assertEquals(0L, lastEntry.get.firstOffset)
+    assertEquals(0L, lastEntry.get.firstDataOffset)
     assertEquals(0L, lastEntry.get.lastDataOffset)
   }
 
@@ -1312,9 +1344,8 @@ class LogTest {
       new SimpleRecord("bar".getBytes),
       new SimpleRecord("baz".getBytes))
     log.appendAsLeader(records, leaderEpoch = 0)
-    val commitAppendInfo = log.appendAsLeader(endTxnRecords(ControlRecordType.ABORT, pid, epoch),
-      isFromClient = false, leaderEpoch = 0)
-    log.updateHighWatermark(commitAppendInfo.lastOffset + 1)
+    val abortAppendInfo = appendEndTxnMarkerAsLeader(log, pid, epoch, ControlRecordType.ABORT)
+    log.updateHighWatermark(abortAppendInfo.lastOffset + 1)
 
     // now there should be no first unstable offset
     assertEquals(None, log.firstUnstableOffset)
@@ -1322,7 +1353,7 @@ class LogTest {
     log.close()
 
     val reopenedLog = createLog(logDir, logConfig)
-    reopenedLog.updateHighWatermark(commitAppendInfo.lastOffset + 1)
+    reopenedLog.updateHighWatermark(abortAppendInfo.lastOffset + 1)
     assertEquals(None, reopenedLog.firstUnstableOffset)
   }
 
@@ -1331,9 +1362,10 @@ class LogTest {
                             epoch: Short,
                             offset: Long = 0L,
                             coordinatorEpoch: Int = 0,
-                            partitionLeaderEpoch: Int = 0): MemoryRecords = {
+                            partitionLeaderEpoch: Int = 0,
+                            timestamp: Long = mockTime.milliseconds()): MemoryRecords = {
     val marker = new EndTransactionMarker(controlRecordType, coordinatorEpoch)
-    MemoryRecords.withEndTransactionMarker(offset, mockTime.milliseconds(), partitionLeaderEpoch, producerId, epoch, marker)
+    MemoryRecords.withEndTransactionMarker(offset, timestamp, partitionLeaderEpoch, producerId, epoch, marker)
   }
 
   @Test
@@ -3542,7 +3574,7 @@ class LogTest {
 
     val buf = ByteBuffer.allocate(DefaultRecordBatch.sizeInBytes(records.asJava))
     val builder = MemoryRecords.builder(buf, magicValue, codec, TimestampType.CREATE_TIME, offset,
-      System.currentTimeMillis, leaderEpoch)
+      mockTime.milliseconds, leaderEpoch)
     records.foreach(builder.append)
     builder.build()
   }
@@ -3588,8 +3620,7 @@ class LogTest {
     assertEquals(firstAppendInfo.firstOffset, log.firstUnstableOffset)
 
     // now transaction is committed
-    val commitAppendInfo = log.appendAsLeader(endTxnRecords(ControlRecordType.COMMIT, pid, epoch),
-      isFromClient = false, leaderEpoch = 0)
+    val commitAppendInfo = appendEndTxnMarkerAsLeader(log, pid, epoch, ControlRecordType.COMMIT)
 
     // first unstable offset is not updated until the high watermark is advanced
     assertEquals(firstAppendInfo.firstOffset, log.firstUnstableOffset)
@@ -3941,6 +3972,19 @@ class LogTest {
     }
   }
 
+  @Test
+  def testEndTxnWithFencedProducerEpoch(): Unit = {
+    val producerId = 1L
+    val epoch = 5.toShort
+    val logConfig = LogTest.createLogConfig(segmentBytes = 1024 * 1024 * 5)
+    val log = createLog(logDir, logConfig)
+    appendEndTxnMarkerAsLeader(log, producerId, epoch, ControlRecordType.ABORT, coordinatorEpoch = 1)
+
+    assertThrows[ProducerFencedException] {
+      appendEndTxnMarkerAsLeader(log, producerId, (epoch - 1).toShort, ControlRecordType.ABORT, coordinatorEpoch = 1)
+    }
+  }
+
   @Test
   def testLastStableOffsetDoesNotExceedLogStartOffsetMidSegment(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 1024 * 1024 * 5)
@@ -4093,16 +4137,14 @@ class LogTest {
     assertEquals(firstAppendInfo.firstOffset, log.firstUnstableOffset)
 
     // now first producer's transaction is aborted
-    val abortAppendInfo = log.appendAsLeader(endTxnRecords(ControlRecordType.ABORT, pid1, epoch),
-      isFromClient = false, leaderEpoch = 0)
+    val abortAppendInfo = appendEndTxnMarkerAsLeader(log, pid1, epoch, ControlRecordType.ABORT)
     log.updateHighWatermark(abortAppendInfo.lastOffset + 1)
 
     // LSO should now point to one less than the first offset of the second transaction
     assertEquals(secondAppendInfo.firstOffset, log.firstUnstableOffset)
 
     // commit the second transaction
-    val commitAppendInfo = log.appendAsLeader(endTxnRecords(ControlRecordType.COMMIT, pid2, epoch),
-      isFromClient = false, leaderEpoch = 0)
+    val commitAppendInfo = appendEndTxnMarkerAsLeader(log, pid2, epoch, ControlRecordType.COMMIT)
     log.updateHighWatermark(commitAppendInfo.lastOffset + 1)
 
     // now there should be no first unstable offset
@@ -4136,9 +4178,8 @@ class LogTest {
     assertEquals(3L, log.logEndOffsetMetadata.segmentBaseOffset)
 
     // now abort the transaction
-    val appendInfo = log.appendAsLeader(endTxnRecords(ControlRecordType.ABORT, pid, epoch),
-      isFromClient = false, leaderEpoch = 0)
-    log.updateHighWatermark(appendInfo.lastOffset + 1)
+    val abortAppendInfo = appendEndTxnMarkerAsLeader(log, pid, epoch, ControlRecordType.ABORT)
+    log.updateHighWatermark(abortAppendInfo.lastOffset + 1)
     assertEquals(None, log.firstUnstableOffset)
 
     // now check that a fetch includes the aborted transaction
@@ -4168,7 +4209,7 @@ class LogTest {
     var sequence = 0
     numRecords: Int => {
       val simpleRecords = (sequence until sequence + numRecords).map { seq =>
-        new SimpleRecord(s"$seq".getBytes)
+        new SimpleRecord(mockTime.milliseconds(), s"$seq".getBytes)
       }
       val records = MemoryRecords.withTransactionalRecords(CompressionType.NONE, producerId,
         producerEpoch, sequence, simpleRecords: _*)
@@ -4182,9 +4223,11 @@ class LogTest {
                                          producerEpoch: Short,
                                          controlType: ControlRecordType,
                                          coordinatorEpoch: Int = 0,
-                                         leaderEpoch: Int = 0): Unit = {
-    val records = endTxnRecords(controlType, producerId, producerEpoch, coordinatorEpoch = coordinatorEpoch)
-    log.appendAsLeader(records, isFromClient = false, leaderEpoch = leaderEpoch)
+                                         leaderEpoch: Int = 0,
+                                         timestamp: Long = mockTime.milliseconds()): LogAppendInfo = {
+    val records = endTxnRecords(controlType, producerId, producerEpoch,
+      coordinatorEpoch = coordinatorEpoch, timestamp = timestamp)
+    log.appendAsLeader(records, origin = AppendOrigin.Coordinator, leaderEpoch = leaderEpoch)
   }
 
   private def appendNonTransactionalAsLeader(log: Log, numRecords: Int): Unit = {
@@ -4202,7 +4245,7 @@ class LogTest {
     var sequence = 0
     (offset: Long, numRecords: Int) => {
       val builder = MemoryRecords.builder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE, TimestampType.CREATE_TIME,
-        offset, System.currentTimeMillis(), producerId, producerEpoch, sequence, true, leaderEpoch)
+        offset, mockTime.milliseconds(), producerId, producerEpoch, sequence, true, leaderEpoch)
       for (seq <- sequence until sequence + numRecords) {
         val record = new SimpleRecord(s"$seq".getBytes)
         builder.append(record)
diff --git a/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala b/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
index 923ae9185211f..0515e79b4b2af 100644
--- a/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
@@ -101,7 +101,7 @@ class LogValidatorTest {
       TimestampType.CREATE_TIME,
       1000L,
       RecordBatch.NO_PRODUCER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       KAFKA_2_3_IV1,
       brokerTopicStats
     )
@@ -128,7 +128,7 @@ class LogValidatorTest {
       timestampType = TimestampType.LOG_APPEND_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats)
     val validatedRecords = validatedResults.validatedRecords
@@ -168,7 +168,7 @@ class LogValidatorTest {
       timestampType = TimestampType.LOG_APPEND_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats)
     val validatedRecords = validatedResults.validatedRecords
@@ -212,7 +212,7 @@ class LogValidatorTest {
       timestampType = TimestampType.LOG_APPEND_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats)
     val validatedRecords = validatedResults.validatedRecords
@@ -272,7 +272,7 @@ class LogValidatorTest {
       timestampType = TimestampType.LOG_APPEND_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats)
   }
@@ -316,7 +316,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = partitionLeaderEpoch,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats)
     val validatedRecords = validatingResults.validatedRecords
@@ -385,7 +385,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = partitionLeaderEpoch,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats)
     val validatedRecords = validatingResults.validatedRecords
@@ -438,7 +438,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats)
     val validatedRecords = validatedResults.validatedRecords
@@ -482,7 +482,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats)
     val validatedRecords = validatedResults.validatedRecords
@@ -539,7 +539,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = partitionLeaderEpoch,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats)
     val validatedRecords = validatedResults.validatedRecords
@@ -592,7 +592,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats)
   }
@@ -615,7 +615,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats)
   }
@@ -638,7 +638,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats)
   }
@@ -661,7 +661,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats)
   }
@@ -683,7 +683,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
@@ -705,7 +705,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
@@ -728,7 +728,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats).validatedRecords
     checkOffsets(messageWithOffset, offset)
@@ -752,7 +752,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats).validatedRecords
     checkOffsets(messageWithOffset, offset)
@@ -777,7 +777,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats).validatedRecords
     checkOffsets(compressedMessagesWithOffset, offset)
@@ -802,7 +802,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats).validatedRecords
     checkOffsets(compressedMessagesWithOffset, offset)
@@ -825,7 +825,7 @@ class LogValidatorTest {
       timestampType = TimestampType.LOG_APPEND_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats)
     checkOffsets(validatedResults.validatedRecords, offset)
@@ -850,7 +850,7 @@ class LogValidatorTest {
       timestampType = TimestampType.LOG_APPEND_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats)
     checkOffsets(validatedResults.validatedRecords, offset)
@@ -875,7 +875,7 @@ class LogValidatorTest {
       timestampType = TimestampType.LOG_APPEND_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats)
     checkOffsets(validatedResults.validatedRecords, offset)
@@ -900,7 +900,7 @@ class LogValidatorTest {
       timestampType = TimestampType.LOG_APPEND_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats)
     checkOffsets(validatedResults.validatedRecords, offset)
@@ -925,7 +925,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats)
   }
@@ -947,7 +947,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = false,
+      origin = AppendOrigin.Coordinator,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats)
     val batches = TestUtils.toList(result.validatedRecords.batches)
@@ -974,7 +974,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
@@ -997,7 +997,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
@@ -1019,7 +1019,7 @@ class LogValidatorTest {
       timestampType = TimestampType.LOG_APPEND_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
@@ -1041,7 +1041,7 @@ class LogValidatorTest {
       timestampType = TimestampType.LOG_APPEND_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
@@ -1064,7 +1064,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
@@ -1087,7 +1087,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
@@ -1112,7 +1112,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
@@ -1137,7 +1137,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
@@ -1160,7 +1160,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
@@ -1183,7 +1183,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats).validatedRecords, offset)
   }
@@ -1205,7 +1205,7 @@ class LogValidatorTest {
         timestampType = TimestampType.CREATE_TIME,
         timestampDiffMaxMs = 5000L,
         partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-        isFromClient = true,
+        origin = AppendOrigin.Client,
         interBrokerProtocolVersion = ApiVersion.latestVersion,
         brokerTopicStats = brokerTopicStats)
     }
@@ -1235,7 +1235,7 @@ class LogValidatorTest {
       timestampType = TimestampType.LOG_APPEND_TIME,
       timestampDiffMaxMs = 1000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = KAFKA_2_0_IV1,
       brokerTopicStats = brokerTopicStats)
   }
@@ -1269,7 +1269,7 @@ class LogValidatorTest {
         timestampType = TimestampType.CREATE_TIME,
         timestampDiffMaxMs = 1000L,
         partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-        isFromClient = true,
+        origin = AppendOrigin.Client,
         interBrokerProtocolVersion = ApiVersion.latestVersion,
         brokerTopicStats = brokerTopicStats)
     }
@@ -1318,7 +1318,7 @@ class LogValidatorTest {
       timestampType = TimestampType.CREATE_TIME,
       timestampDiffMaxMs = 5000L,
       partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       interBrokerProtocolVersion = ApiVersion.latestVersion,
       brokerTopicStats = brokerTopicStats)
   }
diff --git a/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala b/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala
index 8500c94678ad7..94f34283d89fe 100644
--- a/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala
@@ -117,7 +117,7 @@ class ProducerStateManagerTest {
     val epoch = 15.toShort
     val sequence = Int.MaxValue
     val offset = 735L
-    append(stateManager, producerId, epoch, sequence, offset, isFromClient = false)
+    append(stateManager, producerId, epoch, sequence, offset, origin = AppendOrigin.Replication)
 
     append(stateManager, producerId, epoch, 0, offset + 500)
 
@@ -135,9 +135,9 @@ class ProducerStateManagerTest {
   def testProducerSequenceWithWrapAroundBatchRecord(): Unit = {
     val epoch = 15.toShort
 
-    val appendInfo = stateManager.prepareUpdate(producerId, isFromClient = false)
+    val appendInfo = stateManager.prepareUpdate(producerId, origin = AppendOrigin.Replication)
     // Sequence number wrap around
-    appendInfo.append(epoch, Int.MaxValue - 10, 9, time.milliseconds(),
+    appendInfo.appendDataBatch(epoch, Int.MaxValue - 10, 9, time.milliseconds(),
       LogOffsetMetadata(2000L), 2020L, isTransactional = false)
     assertEquals(None, stateManager.lastEntry(producerId))
     stateManager.update(appendInfo)
@@ -146,7 +146,7 @@ class ProducerStateManagerTest {
     val lastEntry = stateManager.lastEntry(producerId).get
     assertEquals(Int.MaxValue-10, lastEntry.firstSeq)
     assertEquals(9, lastEntry.lastSeq)
-    assertEquals(2000L, lastEntry.firstOffset)
+    assertEquals(2000L, lastEntry.firstDataOffset)
     assertEquals(2020L, lastEntry.lastDataOffset)
   }
 
@@ -155,7 +155,7 @@ class ProducerStateManagerTest {
     val epoch = 15.toShort
     val sequence = Int.MaxValue
     val offset = 735L
-    append(stateManager, producerId, epoch, sequence, offset, isFromClient = false)
+    append(stateManager, producerId, epoch, sequence, offset, origin = AppendOrigin.Replication)
     append(stateManager, producerId, epoch, 1, offset + 500)
   }
 
@@ -164,7 +164,7 @@ class ProducerStateManagerTest {
     val epoch = 5.toShort
     val sequence = 16
     val offset = 735L
-    append(stateManager, producerId, epoch, sequence, offset, isFromClient = false)
+    append(stateManager, producerId, epoch, sequence, offset, origin = AppendOrigin.Replication)
 
     val maybeLastEntry = stateManager.lastEntry(producerId)
     assertTrue(maybeLastEntry.isDefined)
@@ -174,7 +174,7 @@ class ProducerStateManagerTest {
     assertEquals(sequence, lastEntry.firstSeq)
     assertEquals(sequence, lastEntry.lastSeq)
     assertEquals(offset, lastEntry.lastDataOffset)
-    assertEquals(offset, lastEntry.firstOffset)
+    assertEquals(offset, lastEntry.firstDataOffset)
   }
 
   @Test
@@ -209,11 +209,11 @@ class ProducerStateManagerTest {
     val producerEpoch = 0.toShort
     val offset = 992342L
     val seq = 0
-    val producerAppendInfo = new ProducerAppendInfo(partition, producerId, ProducerStateEntry.empty(producerId), ValidationType.Full)
+    val producerAppendInfo = new ProducerAppendInfo(partition, producerId, ProducerStateEntry.empty(producerId), AppendOrigin.Client)
 
     val firstOffsetMetadata = LogOffsetMetadata(messageOffset = offset, segmentBaseOffset = 990000L,
       relativePositionInSegment = 234224)
-    producerAppendInfo.append(producerEpoch, seq, seq, time.milliseconds(),
+    producerAppendInfo.appendDataBatch(producerEpoch, seq, seq, time.milliseconds(),
       firstOffsetMetadata, offset, isTransactional = true)
     stateManager.update(producerAppendInfo)
 
@@ -231,11 +231,11 @@ class ProducerStateManagerTest {
         partition,
         producerId,
         ProducerStateEntry.empty(producerId),
-        ValidationType.Full
+        AppendOrigin.Client
       )
       val firstOffsetMetadata = LogOffsetMetadata(messageOffset = startOffset, segmentBaseOffset = segmentBaseOffset,
         relativePositionInSegment = 50 * relativeOffset)
-      producerAppendInfo.append(producerEpoch, 0, 0, time.milliseconds(),
+      producerAppendInfo.appendDataBatch(producerEpoch, 0, 0, time.milliseconds(),
         firstOffsetMetadata, startOffset, isTransactional = true)
       stateManager.update(producerAppendInfo)
     }
@@ -278,15 +278,15 @@ class ProducerStateManagerTest {
   def testPrepareUpdateDoesNotMutate(): Unit = {
     val producerEpoch = 0.toShort
 
-    val appendInfo = stateManager.prepareUpdate(producerId, isFromClient = true)
-    appendInfo.append(producerEpoch, 0, 5, time.milliseconds(),
+    val appendInfo = stateManager.prepareUpdate(producerId, origin = AppendOrigin.Client)
+    appendInfo.appendDataBatch(producerEpoch, 0, 5, time.milliseconds(),
       LogOffsetMetadata(15L), 20L, isTransactional = false)
     assertEquals(None, stateManager.lastEntry(producerId))
     stateManager.update(appendInfo)
     assertTrue(stateManager.lastEntry(producerId).isDefined)
 
-    val nextAppendInfo = stateManager.prepareUpdate(producerId, isFromClient = true)
-    nextAppendInfo.append(producerEpoch, 6, 10, time.milliseconds(),
+    val nextAppendInfo = stateManager.prepareUpdate(producerId, origin = AppendOrigin.Client)
+    nextAppendInfo.appendDataBatch(producerEpoch, 6, 10, time.milliseconds(),
       LogOffsetMetadata(26L), 30L, isTransactional = false)
     assertTrue(stateManager.lastEntry(producerId).isDefined)
 
@@ -309,25 +309,25 @@ class ProducerStateManagerTest {
     val offset = 9L
     append(stateManager, producerId, producerEpoch, 0, offset)
 
-    val appendInfo = stateManager.prepareUpdate(producerId, isFromClient = true)
-    appendInfo.append(producerEpoch, 1, 5, time.milliseconds(),
+    val appendInfo = stateManager.prepareUpdate(producerId, origin = AppendOrigin.Client)
+    appendInfo.appendDataBatch(producerEpoch, 1, 5, time.milliseconds(),
       LogOffsetMetadata(16L), 20L, isTransactional = true)
     var lastEntry = appendInfo.toEntry
     assertEquals(producerEpoch, lastEntry.producerEpoch)
     assertEquals(1, lastEntry.firstSeq)
     assertEquals(5, lastEntry.lastSeq)
-    assertEquals(16L, lastEntry.firstOffset)
+    assertEquals(16L, lastEntry.firstDataOffset)
     assertEquals(20L, lastEntry.lastDataOffset)
     assertEquals(Some(16L), lastEntry.currentTxnFirstOffset)
     assertEquals(List(new TxnMetadata(producerId, 16L)), appendInfo.startedTransactions)
 
-    appendInfo.append(producerEpoch, 6, 10, time.milliseconds(),
+    appendInfo.appendDataBatch(producerEpoch, 6, 10, time.milliseconds(),
       LogOffsetMetadata(26L), 30L, isTransactional = true)
     lastEntry = appendInfo.toEntry
     assertEquals(producerEpoch, lastEntry.producerEpoch)
     assertEquals(1, lastEntry.firstSeq)
     assertEquals(10, lastEntry.lastSeq)
-    assertEquals(16L, lastEntry.firstOffset)
+    assertEquals(16L, lastEntry.firstDataOffset)
     assertEquals(30L, lastEntry.lastDataOffset)
     assertEquals(Some(16L), lastEntry.currentTxnFirstOffset)
     assertEquals(List(new TxnMetadata(producerId, 16L)), appendInfo.startedTransactions)
@@ -344,7 +344,7 @@ class ProducerStateManagerTest {
     // verify that appending the transaction marker doesn't affect the metadata of the cached record batches.
     assertEquals(1, lastEntry.firstSeq)
     assertEquals(10, lastEntry.lastSeq)
-    assertEquals(16L, lastEntry.firstOffset)
+    assertEquals(16L, lastEntry.firstDataOffset)
     assertEquals(30L, lastEntry.lastDataOffset)
     assertEquals(coordinatorEpoch, lastEntry.coordinatorEpoch)
     assertEquals(None, lastEntry.currentTxnFirstOffset)
@@ -417,17 +417,78 @@ class ProducerStateManagerTest {
   }
 
   @Test
-  def testRecoverFromSnapshot(): Unit = {
+  def testRecoverFromSnapshotUnfinishedTransaction(): Unit = {
     val epoch = 0.toShort
-    append(stateManager, producerId, epoch, 0, 0L)
-    append(stateManager, producerId, epoch, 1, 1L)
+    append(stateManager, producerId, epoch, 0, 0L, isTransactional = true)
+    append(stateManager, producerId, epoch, 1, 1L, isTransactional = true)
 
     stateManager.takeSnapshot()
     val recoveredMapping = new ProducerStateManager(partition, logDir, maxPidExpirationMs)
     recoveredMapping.truncateAndReload(0L, 3L, time.milliseconds)
 
+    // The snapshot only persists the last appended batch metadata
+    val loadedEntry = recoveredMapping.lastEntry(producerId)
+    assertEquals(1, loadedEntry.get.firstDataOffset)
+    assertEquals(1, loadedEntry.get.firstSeq)
+    assertEquals(1, loadedEntry.get.lastDataOffset)
+    assertEquals(1, loadedEntry.get.lastSeq)
+    assertEquals(Some(0), loadedEntry.get.currentTxnFirstOffset)
+
     // entry added after recovery
-    append(recoveredMapping, producerId, epoch, 2, 2L)
+    append(recoveredMapping, producerId, epoch, 2, 2L, isTransactional = true)
+  }
+
+  @Test
+  def testRecoverFromSnapshotFinishedTransaction(): Unit = {
+    val epoch = 0.toShort
+    append(stateManager, producerId, epoch, 0, 0L, isTransactional = true)
+    append(stateManager, producerId, epoch, 1, 1L, isTransactional = true)
+    appendEndTxnMarker(stateManager, producerId, epoch, ControlRecordType.ABORT, offset = 2L)
+
+    stateManager.takeSnapshot()
+    val recoveredMapping = new ProducerStateManager(partition, logDir, maxPidExpirationMs)
+    recoveredMapping.truncateAndReload(0L, 3L, time.milliseconds)
+
+    // The snapshot only persists the last appended batch metadata
+    val loadedEntry = recoveredMapping.lastEntry(producerId)
+    assertEquals(1, loadedEntry.get.firstDataOffset)
+    assertEquals(1, loadedEntry.get.firstSeq)
+    assertEquals(1, loadedEntry.get.lastDataOffset)
+    assertEquals(1, loadedEntry.get.lastSeq)
+    assertEquals(None, loadedEntry.get.currentTxnFirstOffset)
+  }
+
+  @Test
+  def testRecoverFromSnapshotEmptyTransaction(): Unit = {
+    val epoch = 0.toShort
+    val appendTimestamp = time.milliseconds()
+    appendEndTxnMarker(stateManager, producerId, epoch, ControlRecordType.ABORT,
+      offset = 0L, timestamp = appendTimestamp)
+    stateManager.takeSnapshot()
+
+    val recoveredMapping = new ProducerStateManager(partition, logDir, maxPidExpirationMs)
+    recoveredMapping.truncateAndReload(logStartOffset = 0L, logEndOffset = 1L, time.milliseconds)
+
+    val lastEntry = recoveredMapping.lastEntry(producerId)
+    assertTrue(lastEntry.isDefined)
+    assertEquals(appendTimestamp, lastEntry.get.lastTimestamp)
+    assertEquals(None, lastEntry.get.currentTxnFirstOffset)
+  }
+
+  @Test
+  def testProducerStateAfterFencingAbortMarker(): Unit = {
+    val epoch = 0.toShort
+    append(stateManager, producerId, epoch, 0, 0L, isTransactional = true)
+    appendEndTxnMarker(stateManager, producerId, (epoch + 1).toShort, ControlRecordType.ABORT, offset = 1L)
+
+    val lastEntry = stateManager.lastEntry(producerId).get
+    assertEquals(None, lastEntry.currentTxnFirstOffset)
+    assertEquals(-1, lastEntry.lastDataOffset)
+    assertEquals(-1, lastEntry.firstDataOffset)
+
+    // The producer should not be expired because we want to preserve fencing epochs
+    stateManager.removeExpiredProducers(time.milliseconds())
+    assertTrue(stateManager.lastEntry(producerId).isDefined)
   }
 
   @Test(expected = classOf[UnknownProducerIdException])
@@ -459,7 +520,7 @@ class ProducerStateManagerTest {
     // entry added after recovery. The pid should be expired now, and would not exist in the pid mapping. Nonetheless
     // the append on a replica should be accepted with the local producer state updated to the appended value.
     assertFalse(recoveredMapping.activeProducers.contains(producerId))
-    append(recoveredMapping, producerId, epoch, sequence, 2L, 70001, isFromClient = false)
+    append(recoveredMapping, producerId, epoch, sequence, 2L, 70001, origin = AppendOrigin.Replication)
     assertTrue(recoveredMapping.activeProducers.contains(producerId))
     val producerStateEntry = recoveredMapping.activeProducers.get(producerId).head
     assertEquals(epoch, producerStateEntry.producerEpoch)
@@ -475,7 +536,7 @@ class ProducerStateManagerTest {
 
     // First we ensure that we raise an OutOfOrderSequenceException is raised when the append comes from a client.
     try {
-      append(stateManager, producerId, epoch, outOfOrderSequence, 1L, 1, isFromClient = true)
+      append(stateManager, producerId, epoch, outOfOrderSequence, 1L, 1, origin = AppendOrigin.Client)
       fail("Expected an OutOfOrderSequenceException to be raised.")
     } catch {
       case _ : OutOfOrderSequenceException =>
@@ -485,7 +546,7 @@ class ProducerStateManagerTest {
     }
 
     assertEquals(0L, stateManager.activeProducers(producerId).lastSeq)
-    append(stateManager, producerId, epoch, outOfOrderSequence, 1L, 1, isFromClient = false)
+    append(stateManager, producerId, epoch, outOfOrderSequence, 1L, 1, origin = AppendOrigin.Replication)
     assertEquals(outOfOrderSequence, stateManager.activeProducers(producerId).lastSeq)
   }
 
@@ -685,9 +746,10 @@ class ProducerStateManagerTest {
     val stateManager = new ProducerStateManager(partition, logDir, maxPidExpirationMs)
 
     val epoch = 0.toShort
-    append(stateManager, producerId, epoch, RecordBatch.NO_SEQUENCE, offset = 99, isTransactional = true)
-    append(stateManager, producerId, epoch, RecordBatch.NO_SEQUENCE, offset = 100, isTransactional = true)
-
+    append(stateManager, producerId, epoch, RecordBatch.NO_SEQUENCE, offset = 99,
+      isTransactional = true, origin = AppendOrigin.Coordinator)
+    append(stateManager, producerId, epoch, RecordBatch.NO_SEQUENCE, offset = 100,
+      isTransactional = true, origin = AppendOrigin.Coordinator)
   }
 
   @Test(expected = classOf[ProducerFencedException])
@@ -778,7 +840,7 @@ class ProducerStateManagerTest {
     EasyMock.replay(batch)
 
     // Appending the empty control batch should not throw and a new transaction shouldn't be started
-    append(stateManager, producerId, producerEpoch, baseOffset, batch, isFromClient = true)
+    append(stateManager, producerId, producerEpoch, baseOffset, batch, origin = AppendOrigin.Client)
     assertEquals(None, stateManager.lastEntry(producerId).get.currentTxnFirstOffset)
   }
 
@@ -819,7 +881,7 @@ class ProducerStateManagerTest {
                                  offset: Long,
                                  coordinatorEpoch: Int = 0,
                                  timestamp: Long = time.milliseconds()): (CompletedTxn, Long) = {
-    val producerAppendInfo = stateManager.prepareUpdate(producerId, isFromClient = true)
+    val producerAppendInfo = stateManager.prepareUpdate(producerId, origin = AppendOrigin.Coordinator)
     val endTxnMarker = new EndTransactionMarker(controlType, coordinatorEpoch)
     val completedTxn = producerAppendInfo.appendEndTxnMarker(endTxnMarker, producerEpoch, offset, timestamp)
     mapping.update(producerAppendInfo)
@@ -836,9 +898,9 @@ class ProducerStateManagerTest {
                      offset: Long,
                      timestamp: Long = time.milliseconds(),
                      isTransactional: Boolean = false,
-                     isFromClient : Boolean = true): Unit = {
-    val producerAppendInfo = stateManager.prepareUpdate(producerId, isFromClient)
-    producerAppendInfo.append(producerEpoch, seq, seq, timestamp,
+                     origin : AppendOrigin = AppendOrigin.Client): Unit = {
+    val producerAppendInfo = stateManager.prepareUpdate(producerId, origin)
+    producerAppendInfo.appendDataBatch(producerEpoch, seq, seq, timestamp,
       LogOffsetMetadata(offset), offset, isTransactional)
     stateManager.update(producerAppendInfo)
     stateManager.updateMapEndOffset(offset + 1)
@@ -849,14 +911,14 @@ class ProducerStateManagerTest {
                      producerEpoch: Short,
                      offset: Long,
                      batch: RecordBatch,
-                     isFromClient : Boolean): Unit = {
-    val producerAppendInfo = stateManager.prepareUpdate(producerId, isFromClient)
+                     origin: AppendOrigin): Unit = {
+    val producerAppendInfo = stateManager.prepareUpdate(producerId, origin)
     producerAppendInfo.append(batch, firstOffsetMetadataOpt = None)
     stateManager.update(producerAppendInfo)
     stateManager.updateMapEndOffset(offset + 1)
   }
 
-  private def currentSnapshotOffsets =
+  private def currentSnapshotOffsets: Set[Long] =
     logDir.listFiles.map(Log.offsetFromFile).toSet
 
 }
diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
index ddc6b2f53d8e3..773ea29e432a0 100644
--- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
@@ -28,6 +28,7 @@ import kafka.api.{ApiVersion, KAFKA_0_10_2_IV0, KAFKA_2_2_IV1}
 import kafka.controller.KafkaController
 import kafka.coordinator.group.{GroupCoordinator, GroupSummary, MemberSummary}
 import kafka.coordinator.transaction.TransactionCoordinator
+import kafka.log.AppendOrigin
 import kafka.network.RequestChannel
 import kafka.network.RequestChannel.SendResponse
 import kafka.server.QuotaFactory.QuotaManagers
@@ -293,7 +294,7 @@ class KafkaApisTest {
     EasyMock.expect(replicaManager.appendRecords(EasyMock.anyLong(),
       EasyMock.anyShort(),
       EasyMock.eq(true),
-      EasyMock.eq(false),
+      EasyMock.eq(AppendOrigin.Coordinator),
       EasyMock.anyObject(),
       EasyMock.capture(responseCallback),
       EasyMock.anyObject(),
@@ -332,7 +333,7 @@ class KafkaApisTest {
     EasyMock.expect(replicaManager.appendRecords(EasyMock.anyLong(),
       EasyMock.anyShort(),
       EasyMock.eq(true),
-      EasyMock.eq(false),
+      EasyMock.eq(AppendOrigin.Coordinator),
       EasyMock.anyObject(),
       EasyMock.capture(responseCallback),
       EasyMock.anyObject(),
@@ -363,7 +364,7 @@ class KafkaApisTest {
     EasyMock.expect(replicaManager.appendRecords(EasyMock.anyLong(),
       EasyMock.anyShort(),
       EasyMock.eq(true),
-      EasyMock.eq(false),
+      EasyMock.eq(AppendOrigin.Coordinator),
       EasyMock.anyObject(),
       EasyMock.anyObject(),
       EasyMock.anyObject(),
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index 73971a7fc25b1..b05ff8b235232 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -24,7 +24,7 @@ import java.util.concurrent.{CountDownLatch, TimeUnit}
 import java.util.{Optional, Properties}
 
 import kafka.api.Request
-import kafka.log.{Log, LogConfig, LogManager, ProducerStateManager}
+import kafka.log.{AppendOrigin, Log, LogConfig, LogManager, ProducerStateManager}
 import kafka.cluster.BrokerEndPoint
 import kafka.server.QuotaFactory.UnboundedQuota
 import kafka.server.checkpoints.LazyOffsetCheckpoints
@@ -135,7 +135,7 @@ class ReplicaManagerTest {
         timeout = 0,
         requiredAcks = 3,
         internalTopicsAllowed = false,
-        isFromClient = true,
+        origin = AppendOrigin.Client,
         entriesPerPartition = Map(new TopicPartition("test1", 0) -> MemoryRecords.withRecords(CompressionType.NONE,
           new SimpleRecord("first message".getBytes))),
         responseCallback = callback)
@@ -343,7 +343,8 @@ class ReplicaManagerTest {
       // now commit the transaction
       val endTxnMarker = new EndTransactionMarker(ControlRecordType.COMMIT, 0)
       val commitRecordBatch = MemoryRecords.withEndTransactionMarker(producerId, epoch, endTxnMarker)
-      appendRecords(replicaManager, new TopicPartition(topic, 0), commitRecordBatch, isFromClient = false)
+      appendRecords(replicaManager, new TopicPartition(topic, 0), commitRecordBatch,
+        origin = AppendOrigin.Coordinator)
         .onFire { response => assertEquals(Errors.NONE, response.error) }
 
       // the LSO has advanced, but the appended commit marker has not been replicated, so
@@ -420,7 +421,8 @@ class ReplicaManagerTest {
       // now abort the transaction
       val endTxnMarker = new EndTransactionMarker(ControlRecordType.ABORT, 0)
       val abortRecordBatch = MemoryRecords.withEndTransactionMarker(producerId, epoch, endTxnMarker)
-      appendRecords(replicaManager, new TopicPartition(topic, 0), abortRecordBatch, isFromClient = false)
+      appendRecords(replicaManager, new TopicPartition(topic, 0), abortRecordBatch,
+        origin = AppendOrigin.Coordinator)
         .onFire { response => assertEquals(Errors.NONE, response.error) }
 
       // fetch as follower to advance the high watermark
@@ -1184,7 +1186,7 @@ class ReplicaManagerTest {
       timeout = 10,
       requiredAcks = -1,
       internalTopicsAllowed = false,
-      isFromClient = true,
+      origin = AppendOrigin.Client,
       entriesPerPartition = Map(topicPartition -> records),
       responseCallback = callback
     )
@@ -1397,7 +1399,7 @@ class ReplicaManagerTest {
   private def appendRecords(replicaManager: ReplicaManager,
                             partition: TopicPartition,
                             records: MemoryRecords,
-                            isFromClient: Boolean = true,
+                            origin: AppendOrigin = AppendOrigin.Client,
                             requiredAcks: Short = -1): CallbackResult[PartitionResponse] = {
     val result = new CallbackResult[PartitionResponse]()
     def appendCallback(responses: Map[TopicPartition, PartitionResponse]): Unit = {
@@ -1410,7 +1412,7 @@ class ReplicaManagerTest {
       timeout = 1000,
       requiredAcks = requiredAcks,
       internalTopicsAllowed = false,
-      isFromClient = isFromClient,
+      origin = origin,
       entriesPerPartition = Map(partition -> records),
       responseCallback = appendCallback)
 

From 179d0d73d65ab2c3eb8bc79c70b9893f07038447 Mon Sep 17 00:00:00 2001
From: Brian Bushree 
Date: Thu, 9 Jan 2020 16:53:36 -0800
Subject: [PATCH 0859/1071] MINOR: Disable JmxTool in kafkatest
 console-consumer by default (#7785)

Do not initialize `JmxTool` by default when running console consumer. In order to support this, we remove `has_partitions_assigned` and its only usage in an assertion inside `ProduceConsumeValidateTest`, which did not seem to contribute much to the validation.

Reviewers: David Arthur , Jason Gustafson 
---
 tests/kafkatest/services/console_consumer.py  | 26 -------------------
 .../tests/produce_consume_validate.py         | 12 ---------
 2 files changed, 38 deletions(-)

diff --git a/tests/kafkatest/services/console_consumer.py b/tests/kafkatest/services/console_consumer.py
index 3aeed906b1d84..5fd471217ec8c 100644
--- a/tests/kafkatest/services/console_consumer.py
+++ b/tests/kafkatest/services/console_consumer.py
@@ -249,7 +249,6 @@ def _worker(self, idx, node):
         consumer_output = node.account.ssh_capture(cmd, allow_fail=False)
 
         with self.lock:
-            self._init_jmx_attributes()
             self.logger.debug("collecting following jmx objects: %s", self.jmx_object_names)
             self.start_jmx_tool(idx, node)
 
@@ -292,28 +291,3 @@ def clean_node(self, node):
 
     def java_class_name(self):
         return "ConsoleConsumer"
-
-    def has_partitions_assigned(self, node):
-        if self.new_consumer is False:
-            return False
-        idx = self.idx(node)
-        with self.lock:
-            self._init_jmx_attributes()
-            self.start_jmx_tool(idx, node)
-            self.read_jmx_output(idx, node)
-        if not self.assigned_partitions_jmx_attr in self.maximum_jmx_value:
-            return False
-        self.logger.debug("Number of partitions assigned %f" % self.maximum_jmx_value[self.assigned_partitions_jmx_attr])
-        return self.maximum_jmx_value[self.assigned_partitions_jmx_attr] > 0.0
-
-    def _init_jmx_attributes(self):
-        # Must hold lock
-        if self.new_consumer:
-            # We use a flag to track whether we're using this automatically generated ID because the service could be
-            # restarted multiple times and the client ID may be changed.
-            if getattr(self, '_automatic_metrics', False) or not self.jmx_object_names:
-                self._automatic_metrics = True
-                self.jmx_object_names = ["kafka.consumer:type=consumer-coordinator-metrics,client-id=%s" % self.client_id]
-                self.jmx_attributes = ["assigned-partitions"]
-                self.assigned_partitions_jmx_attr = "kafka.consumer:type=consumer-coordinator-metrics,client-id=%s:assigned-partitions" % self.client_id
-
diff --git a/tests/kafkatest/tests/produce_consume_validate.py b/tests/kafkatest/tests/produce_consume_validate.py
index d915524e69576..22aa096db0320 100644
--- a/tests/kafkatest/tests/produce_consume_validate.py
+++ b/tests/kafkatest/tests/produce_consume_validate.py
@@ -18,8 +18,6 @@
 
 from kafkatest.utils import validate_delivery
 
-import time
-
 class ProduceConsumeValidateTest(Test):
     """This class provides a shared template for tests which follow the common pattern of:
 
@@ -56,20 +54,10 @@ def start_producer_and_consumer(self):
         if (self.consumer_init_timeout_sec > 0):
             self.logger.debug("Waiting %ds for the consumer to initialize.",
                               self.consumer_init_timeout_sec)
-            start = int(time.time())
             wait_until(lambda: self.consumer.alive(self.consumer.nodes[0]) is True,
                        timeout_sec=self.consumer_init_timeout_sec,
                        err_msg="Consumer process took more than %d s to fork" %\
                        self.consumer_init_timeout_sec)
-            end = int(time.time())
-            remaining_time = self.consumer_init_timeout_sec - (end - start)
-            if remaining_time < 0 :
-                remaining_time = 0
-            if self.consumer.new_consumer:
-                wait_until(lambda: self.consumer.has_partitions_assigned(self.consumer.nodes[0]) is True,
-                           timeout_sec=remaining_time,
-                           err_msg="Consumer process took more than %d s to have partitions assigned" %\
-                           remaining_time)
 
         self.producer.start()
         wait_until(lambda: self.producer.num_acked > 5,

From 65f27964ad63e124296844cc35c7b2bea0dd52bd Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Thu, 16 Jan 2020 13:50:24 -0800
Subject: [PATCH 0860/1071] KAFKA-9235; Ensure transaction coordinator is
 stopped after replica deletion (#7963)

During a reassignment, it can happen that the current leader of a partition is demoted and removed from the replica set at the same time. In this case, we rely on the StopReplica request in order to stop replica fetchers and to clear the group coordinator cache. This patch adds similar logic to ensure that the transaction coordinator state cache also gets cleared.

Reviewers: Rajini Sivaram 
---
 .../coordinator/group/GroupCoordinator.scala  |  14 +-
 .../transaction/TransactionCoordinator.scala  |  89 +++++++-----
 .../transaction/TransactionStateManager.scala |  61 ++++----
 .../main/scala/kafka/server/KafkaApis.scala   |  25 ++--
 ...ransactionCoordinatorConcurrencyTest.scala |   4 +-
 .../TransactionCoordinatorTest.scala          |   8 +-
 .../TransactionStateManagerTest.scala         | 137 +++++++++++++++---
 .../unit/kafka/server/KafkaApisTest.scala     |  37 ++++-
 8 files changed, 264 insertions(+), 111 deletions(-)

diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
index 24a17807d9d31..2e2358db1083b 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
@@ -860,11 +860,21 @@ class GroupCoordinator(val brokerId: Int,
     }
   }
 
-  def handleGroupImmigration(offsetTopicPartitionId: Int): Unit = {
+  /**
+   * Load cached state from the given partition and begin handling requests for groups which map to it.
+   *
+   * @param offsetTopicPartitionId The partition we are now leading
+   */
+  def onElection(offsetTopicPartitionId: Int): Unit = {
     groupManager.scheduleLoadGroupAndOffsets(offsetTopicPartitionId, onGroupLoaded)
   }
 
-  def handleGroupEmigration(offsetTopicPartitionId: Int): Unit = {
+  /**
+   * Unload cached state for the given partition and stop handling requests for groups which map to it.
+   *
+   * @param offsetTopicPartitionId The partition we are no longer leading
+   */
+  def onResignation(offsetTopicPartitionId: Int): Unit = {
     groupManager.removeGroupsForPartition(offsetTopicPartitionId, onGroupUnloaded)
   }
 
diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala
index d646757bede71..c7601ef47f32c 100644
--- a/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala
+++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala
@@ -129,7 +129,7 @@ class TransactionCoordinator(brokerId: Int,
             state = Empty,
             topicPartitions = collection.mutable.Set.empty[TopicPartition],
             txnLastUpdateTimestamp = time.milliseconds())
-          txnManager.putTransactionStateIfNotExists(transactionalId, createdMetadata)
+          txnManager.putTransactionStateIfNotExists(createdMetadata)
 
         case Some(epochAndTxnMetadata) => Right(epochAndTxnMetadata)
       }
@@ -274,7 +274,13 @@ class TransactionCoordinator(brokerId: Int,
     }
   }
 
-  def handleTxnImmigration(txnTopicPartitionId: Int, coordinatorEpoch: Int): Unit = {
+  /**
+   * Load state from the given partition and begin handling requests for groups which map to this partition.
+   *
+   * @param txnTopicPartitionId The partition that we are now leading
+   * @param coordinatorEpoch The partition coordinator (or leader) epoch from the received LeaderAndIsr request
+   */
+  def onElection(txnTopicPartitionId: Int, coordinatorEpoch: Int): Unit = {
     // The operations performed during immigration must be resilient to any previous errors we saw or partial state we
     // left off during the unloading phase. Ensure we remove all associated state for this partition before we continue
     // loading it.
@@ -284,8 +290,20 @@ class TransactionCoordinator(brokerId: Int,
     txnManager.loadTransactionsForTxnTopicPartition(txnTopicPartitionId, coordinatorEpoch, txnMarkerChannelManager.addTxnMarkersToSend)
   }
 
-  def handleTxnEmigration(txnTopicPartitionId: Int, coordinatorEpoch: Int): Unit = {
-    txnManager.removeTransactionsForTxnTopicPartition(txnTopicPartitionId, coordinatorEpoch)
+  /**
+   * Clear coordinator caches for the given partition after giving up leadership.
+   *
+   * @param txnTopicPartitionId The partition that we are no longer leading
+   * @param coordinatorEpoch The partition coordinator (or leader) epoch, which may be absent if we
+   *                         are resigning after receiving a StopReplica request from the controller
+   */
+  def onResignation(txnTopicPartitionId: Int, coordinatorEpoch: Option[Int]): Unit = {
+    coordinatorEpoch match {
+      case Some(epoch) =>
+        txnManager.removeTransactionsForTxnTopicPartition(txnTopicPartitionId, epoch)
+      case None =>
+        txnManager.removeTransactionsForTxnTopicPartition(txnTopicPartitionId)
+    }
     txnMarkerChannelManager.removeMarkersForTxnTopicPartition(txnTopicPartitionId)
   }
 
@@ -450,47 +468,52 @@ class TransactionCoordinator(brokerId: Int,
   def partitionFor(transactionalId: String): Int = txnManager.partitionFor(transactionalId)
 
   private def abortTimedOutTransactions(): Unit = {
+    def onComplete(txnIdAndPidEpoch: TransactionalIdAndProducerIdEpoch)(error: Errors): Unit = {
+      error match {
+        case Errors.NONE =>
+          info("Completed rollback of ongoing transaction for transactionalId " +
+            s"${txnIdAndPidEpoch.transactionalId} due to timeout")
+
+        case error@(Errors.INVALID_PRODUCER_ID_MAPPING |
+                    Errors.INVALID_PRODUCER_EPOCH |
+                    Errors.CONCURRENT_TRANSACTIONS) =>
+          debug(s"Rollback of ongoing transaction for transactionalId ${txnIdAndPidEpoch.transactionalId} " +
+            s"has been cancelled due to error $error")
+
+        case error =>
+          warn(s"Rollback of ongoing transaction for transactionalId ${txnIdAndPidEpoch.transactionalId} " +
+            s"failed due to error $error")
+      }
+    }
+
     txnManager.timedOutTransactions().foreach { txnIdAndPidEpoch =>
-      txnManager.getTransactionState(txnIdAndPidEpoch.transactionalId).right.flatMap {
+      txnManager.getTransactionState(txnIdAndPidEpoch.transactionalId).right.foreach {
         case None =>
-          error(s"Could not find transaction metadata when trying to timeout transaction with transactionalId " +
-            s"${txnIdAndPidEpoch.transactionalId}. ProducerId: ${txnIdAndPidEpoch.producerId}. ProducerEpoch: " +
-            s"${txnIdAndPidEpoch.producerEpoch}")
-          Left(Errors.INVALID_TXN_STATE)
+          error(s"Could not find transaction metadata when trying to timeout transaction for $txnIdAndPidEpoch")
 
         case Some(epochAndTxnMetadata) =>
           val txnMetadata = epochAndTxnMetadata.transactionMetadata
-          val transitMetadata = txnMetadata.inLock {
+          val transitMetadataOpt = txnMetadata.inLock {
             if (txnMetadata.producerId != txnIdAndPidEpoch.producerId) {
               error(s"Found incorrect producerId when expiring transactionalId: ${txnIdAndPidEpoch.transactionalId}. " +
                 s"Expected producerId: ${txnIdAndPidEpoch.producerId}. Found producerId: " +
                 s"${txnMetadata.producerId}")
-              Left(Errors.INVALID_PRODUCER_ID_MAPPING)
+              None
             } else if (txnMetadata.pendingTransitionInProgress) {
-              Left(Errors.CONCURRENT_TRANSACTIONS)
+              debug(s"Skipping abort of timed out transaction $txnIdAndPidEpoch since there is a " +
+                "pending state transition")
+              None
             } else {
-              Right(txnMetadata.prepareFenceProducerEpoch())
+              Some(txnMetadata.prepareFenceProducerEpoch())
             }
           }
-          transitMetadata match {
-            case Right(txnTransitMetadata) =>
-              handleEndTransaction(txnMetadata.transactionalId,
-                txnTransitMetadata.producerId,
-                txnTransitMetadata.producerEpoch,
-                TransactionResult.ABORT,
-                {
-                  case Errors.NONE =>
-                    info(s"Completed rollback ongoing transaction of transactionalId: ${txnIdAndPidEpoch.transactionalId} due to timeout")
-                  case e @ (Errors.INVALID_PRODUCER_ID_MAPPING |
-                            Errors.INVALID_PRODUCER_EPOCH |
-                            Errors.CONCURRENT_TRANSACTIONS) =>
-                    debug(s"Rolling back ongoing transaction of transactionalId: ${txnIdAndPidEpoch.transactionalId} has aborted due to ${e.exceptionName}")
-                  case e =>
-                    warn(s"Rolling back ongoing transaction of transactionalId: ${txnIdAndPidEpoch.transactionalId} failed due to ${e.exceptionName}")
-                })
-              Right(txnTransitMetadata)
-            case (error) =>
-              Left(error)
+
+          transitMetadataOpt.foreach { txnTransitMetadata =>
+            handleEndTransaction(txnMetadata.transactionalId,
+              txnTransitMetadata.producerId,
+              txnTransitMetadata.producerEpoch,
+              TransactionResult.ABORT,
+              onComplete(txnIdAndPidEpoch))
           }
       }
     }
@@ -503,7 +526,7 @@ class TransactionCoordinator(brokerId: Int,
     info("Starting up.")
     scheduler.startup()
     scheduler.schedule("transaction-abort",
-      () => abortTimedOutTransactions,
+      abortTimedOutTransactions,
       txnConfig.abortTimedOutTransactionsIntervalMs,
       txnConfig.abortTimedOutTransactionsIntervalMs
     )
diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala
index 7d731a11fd831..36bc9659427ea 100644
--- a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala
+++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala
@@ -88,9 +88,6 @@ class TransactionStateManager(brokerId: Int,
   /** partitions of transaction topic that are being loaded, state lock should be called BEFORE accessing this set */
   private[transaction] val loadingPartitions: mutable.Set[TransactionPartitionAndLeaderEpoch] = mutable.Set()
 
-  /** partitions of transaction topic that are being removed, state lock should be called BEFORE accessing this set */
-  private[transaction] val leavingPartitions: mutable.Set[TransactionPartitionAndLeaderEpoch] = mutable.Set()
-
   /** transaction metadata cache indexed by assigned transaction topic partition ids */
   private[transaction] val transactionMetadataCache: mutable.Map[Int, TxnMetadataCacheEntry] = mutable.Map()
 
@@ -110,9 +107,7 @@ class TransactionStateManager(brokerId: Int,
   // visible for testing only
   private[transaction] def addLoadingPartition(partitionId: Int, coordinatorEpoch: Int): Unit = {
     val partitionAndLeaderEpoch = TransactionPartitionAndLeaderEpoch(partitionId, coordinatorEpoch)
-
     inWriteLock(stateLock) {
-      leavingPartitions.remove(partitionAndLeaderEpoch)
       loadingPartitions.add(partitionAndLeaderEpoch)
     }
   }
@@ -125,9 +120,7 @@ class TransactionStateManager(brokerId: Int,
   def timedOutTransactions(): Iterable[TransactionalIdAndProducerIdEpoch] = {
     val now = time.milliseconds()
     inReadLock(stateLock) {
-      transactionMetadataCache.filter { case (txnPartitionId, _) =>
-        !leavingPartitions.exists(_.txnPartitionId == txnPartitionId)
-      }.flatMap { case (_, entry) =>
+      transactionMetadataCache.flatMap { case (_, entry) =>
         entry.metadataPerTransactionalId.filter { case (_, txnMetadata) =>
           if (txnMetadata.pendingTransitionInProgress) {
             false
@@ -224,10 +217,10 @@ class TransactionStateManager(brokerId: Int,
   def getTransactionState(transactionalId: String): Either[Errors, Option[CoordinatorEpochAndTxnMetadata]] =
     getAndMaybeAddTransactionState(transactionalId, None)
 
-  def putTransactionStateIfNotExists(transactionalId: String,
-                                     txnMetadata: TransactionMetadata): Either[Errors, CoordinatorEpochAndTxnMetadata] =
-    getAndMaybeAddTransactionState(transactionalId, Some(txnMetadata))
+  def putTransactionStateIfNotExists(txnMetadata: TransactionMetadata): Either[Errors, CoordinatorEpochAndTxnMetadata] = {
+    getAndMaybeAddTransactionState(txnMetadata.transactionalId, Some(txnMetadata))
       .right.map(_.getOrElse(throw new IllegalStateException(s"Unexpected empty transaction metadata returned while putting $txnMetadata")))
+  }
 
   /**
    * Get the transaction metadata associated with the given transactional id, or an error if
@@ -242,8 +235,6 @@ class TransactionStateManager(brokerId: Int,
       val partitionId = partitionFor(transactionalId)
       if (loadingPartitions.exists(_.txnPartitionId == partitionId))
         Left(Errors.COORDINATOR_LOAD_IN_PROGRESS)
-      else if (leavingPartitions.exists(_.txnPartitionId == partitionId))
-        Left(Errors.NOT_COORDINATOR)
       else {
         transactionMetadataCache.get(partitionId) match {
           case Some(cacheEntry) =>
@@ -396,7 +387,6 @@ class TransactionStateManager(brokerId: Int,
     val partitionAndLeaderEpoch = TransactionPartitionAndLeaderEpoch(partitionId, coordinatorEpoch)
 
     inWriteLock(stateLock) {
-      leavingPartitions.remove(partitionAndLeaderEpoch)
       loadingPartitions.add(partitionAndLeaderEpoch)
     }
 
@@ -423,7 +413,7 @@ class TransactionStateManager(brokerId: Int,
                     transactionsPendingForCompletion +=
                       TransactionalIdCoordinatorEpochAndTransitMetadata(transactionalId, coordinatorEpoch, TransactionResult.COMMIT, txnMetadata, txnMetadata.prepareComplete(time.milliseconds()))
                   case _ =>
-                    // nothing need to be done
+                    // nothing needs to be done
                 }
               }
           }
@@ -442,7 +432,18 @@ class TransactionStateManager(brokerId: Int,
       info(s"Completed loading transaction metadata from $topicPartition for coordinator epoch $coordinatorEpoch")
     }
 
-    scheduler.schedule(s"load-txns-for-partition-$topicPartition", () => loadTransactions)
+    scheduler.schedule(s"load-txns-for-partition-$topicPartition", loadTransactions)
+  }
+
+  def removeTransactionsForTxnTopicPartition(partitionId: Int): Unit = {
+    val topicPartition = new TopicPartition(Topic.TRANSACTION_STATE_TOPIC_NAME, partitionId)
+    inWriteLock(stateLock) {
+      loadingPartitions.retain(_.txnPartitionId != partitionId)
+      transactionMetadataCache.remove(partitionId).foreach { txnMetadataCacheEntry =>
+        info(s"Unloaded transaction metadata $txnMetadataCacheEntry for $topicPartition following " +
+          s"local partition deletion")
+      }
+    }
   }
 
   /**
@@ -455,26 +456,14 @@ class TransactionStateManager(brokerId: Int,
 
     inWriteLock(stateLock) {
       loadingPartitions.remove(partitionAndLeaderEpoch)
-      leavingPartitions.add(partitionAndLeaderEpoch)
-    }
+      transactionMetadataCache.remove(partitionId) match {
+        case Some(txnMetadataCacheEntry) =>
+          info(s"Unloaded transaction metadata $txnMetadataCacheEntry for $topicPartition on become-follower transition")
 
-    def removeTransactions(): Unit = {
-      inWriteLock(stateLock) {
-        if (leavingPartitions.contains(partitionAndLeaderEpoch)) {
-          transactionMetadataCache.remove(partitionId) match {
-            case Some(txnMetadataCacheEntry) =>
-              info(s"Unloaded transaction metadata $txnMetadataCacheEntry for $topicPartition on become-follower transition")
-
-            case None =>
-              info(s"No cached transaction metadata found for $topicPartition during become-follower transition")
-          }
-
-          leavingPartitions.remove(partitionAndLeaderEpoch)
-        }
+        case None =>
+          info(s"No cached transaction metadata found for $topicPartition during become-follower transition")
       }
     }
-
-    scheduler.schedule(s"remove-txns-for-partition-$topicPartition", () => removeTransactions)
   }
 
   private def validateTransactionTopicPartitionCountIsStable(): Unit = {
@@ -679,7 +668,11 @@ private[transaction] case class TransactionConfig(transactionalIdExpirationMs: I
                                                   removeExpiredTransactionalIdsIntervalMs: Int = TransactionStateManager.DefaultRemoveExpiredTransactionalIdsIntervalMs,
                                                   requestTimeoutMs: Int = Defaults.RequestTimeoutMs)
 
-case class TransactionalIdAndProducerIdEpoch(transactionalId: String, producerId: Long, producerEpoch: Short)
+case class TransactionalIdAndProducerIdEpoch(transactionalId: String, producerId: Long, producerEpoch: Short) {
+  override def toString: String = {
+    s"(transactionalId=$transactionalId, producerId=$producerId, producerEpoch=$producerEpoch)"
+  }
+}
 
 case class TransactionPartitionAndLeaderEpoch(txnPartitionId: Int, coordinatorEpoch: Int)
 
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index e3d53a1089c07..124b2e3564f8a 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -194,16 +194,16 @@ class KafkaApis(val requestChannel: RequestChannel,
       // leadership changes
       updatedLeaders.foreach { partition =>
         if (partition.topic == GROUP_METADATA_TOPIC_NAME)
-          groupCoordinator.handleGroupImmigration(partition.partitionId)
+          groupCoordinator.onElection(partition.partitionId)
         else if (partition.topic == TRANSACTION_STATE_TOPIC_NAME)
-          txnCoordinator.handleTxnImmigration(partition.partitionId, partition.getLeaderEpoch)
+          txnCoordinator.onElection(partition.partitionId, partition.getLeaderEpoch)
       }
 
       updatedFollowers.foreach { partition =>
         if (partition.topic == GROUP_METADATA_TOPIC_NAME)
-          groupCoordinator.handleGroupEmigration(partition.partitionId)
+          groupCoordinator.onResignation(partition.partitionId)
         else if (partition.topic == TRANSACTION_STATE_TOPIC_NAME)
-          txnCoordinator.handleTxnEmigration(partition.partitionId, partition.getLeaderEpoch)
+          txnCoordinator.onResignation(partition.partitionId, Some(partition.getLeaderEpoch))
       }
     }
 
@@ -234,15 +234,16 @@ class KafkaApis(val requestChannel: RequestChannel,
       sendResponseExemptThrottle(request, new StopReplicaResponse(new StopReplicaResponseData().setErrorCode(Errors.STALE_BROKER_EPOCH.code)))
     } else {
       val (result, error) = replicaManager.stopReplicas(stopReplicaRequest)
-      // Clearing out the cache for groups that belong to an offsets topic partition for which this broker was the leader,
-      // since this broker is no longer a replica for that offsets topic partition.
-      // This is required to handle the following scenario :
-      // Consider old replicas : {[1,2,3], Leader = 1} is reassigned to new replicas : {[2,3,4], Leader = 2}, broker 1 does not receive a LeaderAndIsr
-      // request to become a follower due to which cache for groups that belong to an offsets topic partition for which broker 1 was the leader,
-      // is not cleared.
+      // Clear the coordinator caches in case we were the leader. In the case of a reassignment, we
+      // cannot rely on the LeaderAndIsr API for this since it is only sent to active replicas.
       result.foreach { case (topicPartition, error) =>
-        if (error == Errors.NONE && stopReplicaRequest.deletePartitions && topicPartition.topic == GROUP_METADATA_TOPIC_NAME) {
-          groupCoordinator.handleGroupEmigration(topicPartition.partition)
+        if (error == Errors.NONE && stopReplicaRequest.deletePartitions) {
+          if (topicPartition.topic == GROUP_METADATA_TOPIC_NAME) {
+            groupCoordinator.onResignation(topicPartition.partition)
+          } else if (topicPartition.topic == TRANSACTION_STATE_TOPIC_NAME) {
+            // The StopReplica API does not pass through the leader epoch
+            txnCoordinator.onResignation(topicPartition.partition, coordinatorEpoch = None)
+          }
         }
       }
 
diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala
index 3031671f4b43d..79611cbf3a908 100644
--- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala
@@ -344,7 +344,7 @@ class TransactionCoordinatorConcurrencyTest extends AbstractCoordinatorConcurren
 
   class LoadTxnPartitionAction(txnTopicPartitionId: Int) extends Action {
     override def run(): Unit = {
-      transactionCoordinator.handleTxnImmigration(txnTopicPartitionId, coordinatorEpoch)
+      transactionCoordinator.onElection(txnTopicPartitionId, coordinatorEpoch)
     }
     override def await(): Unit = {
       allTransactions.foreach { txn =>
@@ -358,7 +358,7 @@ class TransactionCoordinatorConcurrencyTest extends AbstractCoordinatorConcurren
   class UnloadTxnPartitionAction(txnTopicPartitionId: Int) extends Action {
     val txnRecords: mutable.ArrayBuffer[SimpleRecord] = mutable.ArrayBuffer[SimpleRecord]()
     override def run(): Unit = {
-      transactionCoordinator.handleTxnEmigration(txnTopicPartitionId, coordinatorEpoch)
+      transactionCoordinator.onResignation(txnTopicPartitionId, Some(coordinatorEpoch))
     }
     override def await(): Unit = {
       allTransactions.foreach { txn =>
diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorTest.scala
index 75f06d5d58457..a637d16551e37 100644
--- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorTest.scala
@@ -107,7 +107,7 @@ class TransactionCoordinatorTest {
       .andReturn(Right(None))
       .once()
 
-    EasyMock.expect(transactionManager.putTransactionStateIfNotExists(EasyMock.eq(transactionalId), EasyMock.capture(capturedTxn)))
+    EasyMock.expect(transactionManager.putTransactionStateIfNotExists(EasyMock.capture(capturedTxn)))
       .andAnswer(new IAnswer[Either[Errors, CoordinatorEpochAndTxnMetadata]] {
         override def answer(): Either[Errors, CoordinatorEpochAndTxnMetadata] = {
           assertTrue(capturedTxn.hasCaptured)
@@ -512,7 +512,7 @@ class TransactionCoordinatorTest {
     EasyMock.expect(transactionManager.validateTransactionTimeoutMs(EasyMock.anyInt()))
       .andReturn(true)
 
-    EasyMock.expect(transactionManager.putTransactionStateIfNotExists(EasyMock.eq(transactionalId), EasyMock.anyObject[TransactionMetadata]()))
+    EasyMock.expect(transactionManager.putTransactionStateIfNotExists(EasyMock.anyObject[TransactionMetadata]()))
       .andReturn(Right(CoordinatorEpochAndTxnMetadata(coordinatorEpoch, txnMetadata)))
       .anyTimes()
 
@@ -551,7 +551,7 @@ class TransactionCoordinatorTest {
     EasyMock.expect(transactionManager.validateTransactionTimeoutMs(EasyMock.anyInt()))
       .andReturn(true)
 
-    EasyMock.expect(transactionManager.putTransactionStateIfNotExists(EasyMock.eq(transactionalId), EasyMock.anyObject[TransactionMetadata]()))
+    EasyMock.expect(transactionManager.putTransactionStateIfNotExists(EasyMock.anyObject[TransactionMetadata]()))
       .andReturn(Right(CoordinatorEpochAndTxnMetadata(coordinatorEpoch, txnMetadata)))
       .anyTimes()
 
@@ -593,7 +593,7 @@ class TransactionCoordinatorTest {
     EasyMock.expect(transactionMarkerChannelManager.removeMarkersForTxnTopicPartition(0))
     EasyMock.replay(transactionManager, transactionMarkerChannelManager)
 
-    coordinator.handleTxnEmigration(0, coordinatorEpoch)
+    coordinator.onResignation(0, Some(coordinatorEpoch))
 
     EasyMock.verify(transactionManager, transactionMarkerChannelManager)
   }
diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala
index 5ab3f82431f97..7f6ccfb8c870e 100644
--- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala
@@ -18,12 +18,13 @@ package kafka.coordinator.transaction
 
 import java.lang.management.ManagementFactory
 import java.nio.ByteBuffer
+import java.util.concurrent.CountDownLatch
 import java.util.concurrent.locks.ReentrantLock
 
 import javax.management.ObjectName
 import kafka.log.{AppendOrigin, Log}
 import kafka.server.{FetchDataInfo, FetchLogEnd, LogOffsetMetadata, ReplicaManager}
-import kafka.utils.{MockScheduler, Pool}
+import kafka.utils.{MockScheduler, Pool, TestUtils}
 import kafka.zk.KafkaZkClient
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.internals.Topic.TRANSACTION_STATE_TOPIC_NAME
@@ -104,11 +105,103 @@ class TransactionStateManagerTest {
 
     assertEquals(Right(None), transactionManager.getTransactionState(transactionalId1))
     assertEquals(Right(CoordinatorEpochAndTxnMetadata(coordinatorEpoch, txnMetadata1)),
-      transactionManager.putTransactionStateIfNotExists(transactionalId1, txnMetadata1))
+      transactionManager.putTransactionStateIfNotExists(txnMetadata1))
     assertEquals(Right(Some(CoordinatorEpochAndTxnMetadata(coordinatorEpoch, txnMetadata1))),
       transactionManager.getTransactionState(transactionalId1))
-    assertEquals(Right(CoordinatorEpochAndTxnMetadata(coordinatorEpoch, txnMetadata1)),
-      transactionManager.putTransactionStateIfNotExists(transactionalId1, txnMetadata2))
+    assertEquals(Right(CoordinatorEpochAndTxnMetadata(coordinatorEpoch, txnMetadata2)),
+      transactionManager.putTransactionStateIfNotExists(txnMetadata2))
+  }
+
+  @Test
+  def testDeletePartition(): Unit = {
+    val metadata1 = transactionMetadata("b", 5L)
+    val metadata2 = transactionMetadata("a", 10L)
+
+    assertEquals(0, transactionManager.partitionFor(metadata1.transactionalId))
+    assertEquals(1, transactionManager.partitionFor(metadata2.transactionalId))
+
+    transactionManager.addLoadedTransactionsToCache(0, coordinatorEpoch, new Pool[String, TransactionMetadata]())
+    transactionManager.putTransactionStateIfNotExists(metadata1)
+
+    transactionManager.addLoadedTransactionsToCache(1, coordinatorEpoch, new Pool[String, TransactionMetadata]())
+    transactionManager.putTransactionStateIfNotExists(metadata2)
+
+    def cachedProducerEpoch(transactionalId: String): Option[Short] = {
+      transactionManager.getTransactionState(transactionalId).toOption.flatten
+        .map(_.transactionMetadata.producerEpoch)
+    }
+
+    assertEquals(Some(metadata1.producerEpoch), cachedProducerEpoch(metadata1.transactionalId))
+    assertEquals(Some(metadata2.producerEpoch), cachedProducerEpoch(metadata2.transactionalId))
+
+    transactionManager.removeTransactionsForTxnTopicPartition(0)
+
+    assertEquals(None, cachedProducerEpoch(metadata1.transactionalId))
+    assertEquals(Some(metadata2.producerEpoch), cachedProducerEpoch(metadata2.transactionalId))
+  }
+
+  @Test
+  def testDeleteLoadingPartition(): Unit = {
+    // Verify the handling of a call to delete state for a partition while it is in the
+    // process of being loaded. Basically should be treated as a no-op.
+
+    val startOffset = 0L
+    val endOffset = 1L
+
+    val fileRecordsMock = EasyMock.mock[FileRecords](classOf[FileRecords])
+    val logMock = EasyMock.mock[Log](classOf[Log])
+    EasyMock.expect(replicaManager.getLog(topicPartition)).andStubReturn(Some(logMock))
+    EasyMock.expect(logMock.logStartOffset).andStubReturn(startOffset)
+    EasyMock.expect(logMock.read(EasyMock.eq(startOffset),
+      maxLength = EasyMock.anyInt(),
+      isolation = EasyMock.eq(FetchLogEnd),
+      minOneMessage = EasyMock.eq(true))
+    ).andReturn(FetchDataInfo(LogOffsetMetadata(startOffset), fileRecordsMock))
+    EasyMock.expect(replicaManager.getLogEndOffset(topicPartition)).andStubReturn(Some(endOffset))
+
+    txnMetadata1.state = PrepareCommit
+    txnMetadata1.addPartitions(Set[TopicPartition](
+      new TopicPartition("topic1", 0),
+      new TopicPartition("topic1", 1)))
+    val records = MemoryRecords.withRecords(startOffset, CompressionType.NONE,
+      new SimpleRecord(txnMessageKeyBytes1, TransactionLog.valueToBytes(txnMetadata1.prepareNoTransit())))
+
+    // We create a latch which is awaited while the log is loading. This ensures that the deletion
+    // is triggered before the loading returns
+    val latch = new CountDownLatch(1)
+
+    EasyMock.expect(fileRecordsMock.sizeInBytes()).andStubReturn(records.sizeInBytes)
+    val bufferCapture = EasyMock.newCapture[ByteBuffer]
+    fileRecordsMock.readInto(EasyMock.capture(bufferCapture), EasyMock.anyInt())
+    EasyMock.expectLastCall().andAnswer(new IAnswer[Unit] {
+      override def answer: Unit = {
+        latch.await()
+        val buffer = bufferCapture.getValue
+        buffer.put(records.buffer.duplicate)
+        buffer.flip()
+      }
+    })
+
+    EasyMock.replay(logMock, fileRecordsMock, replicaManager)
+
+    val coordinatorEpoch = 0
+    val partitionAndLeaderEpoch = TransactionPartitionAndLeaderEpoch(partitionId, coordinatorEpoch)
+
+    val loadingThread = new Thread(() => {
+      transactionManager.loadTransactionsForTxnTopicPartition(partitionId, coordinatorEpoch, (_, _, _, _, _) => ())
+    })
+    loadingThread.start()
+    TestUtils.waitUntilTrue(() => transactionManager.loadingPartitions.contains(partitionAndLeaderEpoch),
+      "Timed out waiting for loading partition", pause = 10)
+
+    transactionManager.removeTransactionsForTxnTopicPartition(partitionId)
+    assertFalse(transactionManager.loadingPartitions.contains(partitionAndLeaderEpoch))
+
+    latch.countDown()
+    loadingThread.join()
+
+    // Verify that transaction state was not loaded
+    assertEquals(Left(Errors.NOT_COORDINATOR), transactionManager.getTransactionState(txnMetadata1.transactionalId))
   }
 
   @Test
@@ -216,7 +309,7 @@ class TransactionStateManagerTest {
     transactionManager.addLoadedTransactionsToCache(partitionId, coordinatorEpoch, new Pool[String, TransactionMetadata]())
 
     // first insert the initial transaction metadata
-    transactionManager.putTransactionStateIfNotExists(transactionalId1, txnMetadata1)
+    transactionManager.putTransactionStateIfNotExists(txnMetadata1)
 
     prepareForTxnMessageAppend(Errors.NONE)
     expectedError = Errors.NONE
@@ -235,7 +328,7 @@ class TransactionStateManagerTest {
   @Test
   def testAppendFailToCoordinatorNotAvailableError(): Unit = {
     transactionManager.addLoadedTransactionsToCache(partitionId, coordinatorEpoch, new Pool[String, TransactionMetadata]())
-    transactionManager.putTransactionStateIfNotExists(transactionalId1, txnMetadata1)
+    transactionManager.putTransactionStateIfNotExists(txnMetadata1)
 
     expectedError = Errors.COORDINATOR_NOT_AVAILABLE
     var failedMetadata = txnMetadata1.prepareAddPartitions(Set[TopicPartition](new TopicPartition("topic2", 0)), time.milliseconds())
@@ -267,7 +360,7 @@ class TransactionStateManagerTest {
   @Test
   def testAppendFailToNotCoordinatorError(): Unit = {
     transactionManager.addLoadedTransactionsToCache(partitionId, coordinatorEpoch, new Pool[String, TransactionMetadata]())
-    transactionManager.putTransactionStateIfNotExists(transactionalId1, txnMetadata1)
+    transactionManager.putTransactionStateIfNotExists(txnMetadata1)
 
     expectedError = Errors.NOT_COORDINATOR
     var failedMetadata = txnMetadata1.prepareAddPartitions(Set[TopicPartition](new TopicPartition("topic2", 0)), time.milliseconds())
@@ -285,7 +378,7 @@ class TransactionStateManagerTest {
     prepareForTxnMessageAppend(Errors.NONE)
     transactionManager.removeTransactionsForTxnTopicPartition(partitionId, coordinatorEpoch)
     transactionManager.addLoadedTransactionsToCache(partitionId, coordinatorEpoch + 1, new Pool[String, TransactionMetadata]())
-    transactionManager.putTransactionStateIfNotExists(transactionalId1, txnMetadata1)
+    transactionManager.putTransactionStateIfNotExists(txnMetadata1)
     transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback)
 
     prepareForTxnMessageAppend(Errors.NONE)
@@ -297,7 +390,7 @@ class TransactionStateManagerTest {
   @Test
   def testAppendFailToCoordinatorLoadingError(): Unit = {
     transactionManager.addLoadedTransactionsToCache(partitionId, coordinatorEpoch, new Pool[String, TransactionMetadata]())
-    transactionManager.putTransactionStateIfNotExists(transactionalId1, txnMetadata1)
+    transactionManager.putTransactionStateIfNotExists(txnMetadata1)
 
     expectedError = Errors.COORDINATOR_LOAD_IN_PROGRESS
     val failedMetadata = txnMetadata1.prepareAddPartitions(Set[TopicPartition](new TopicPartition("topic2", 0)), time.milliseconds())
@@ -311,7 +404,7 @@ class TransactionStateManagerTest {
   @Test
   def testAppendFailToUnknownError(): Unit = {
     transactionManager.addLoadedTransactionsToCache(partitionId, coordinatorEpoch, new Pool[String, TransactionMetadata]())
-    transactionManager.putTransactionStateIfNotExists(transactionalId1, txnMetadata1)
+    transactionManager.putTransactionStateIfNotExists(txnMetadata1)
 
     expectedError = Errors.UNKNOWN_SERVER_ERROR
     var failedMetadata = txnMetadata1.prepareAddPartitions(Set[TopicPartition](new TopicPartition("topic2", 0)), time.milliseconds())
@@ -331,7 +424,7 @@ class TransactionStateManagerTest {
   @Test
   def testPendingStateNotResetOnRetryAppend(): Unit = {
     transactionManager.addLoadedTransactionsToCache(partitionId, coordinatorEpoch, new Pool[String, TransactionMetadata]())
-    transactionManager.putTransactionStateIfNotExists(transactionalId1, txnMetadata1)
+    transactionManager.putTransactionStateIfNotExists(txnMetadata1)
 
     expectedError = Errors.COORDINATOR_NOT_AVAILABLE
     val failedMetadata = txnMetadata1.prepareAddPartitions(Set[TopicPartition](new TopicPartition("topic2", 0)), time.milliseconds())
@@ -347,7 +440,7 @@ class TransactionStateManagerTest {
     transactionManager.addLoadedTransactionsToCache(partitionId, 0, new Pool[String, TransactionMetadata]())
 
     // first insert the initial transaction metadata
-    transactionManager.putTransactionStateIfNotExists(transactionalId1, txnMetadata1)
+    transactionManager.putTransactionStateIfNotExists(txnMetadata1)
 
     prepareForTxnMessageAppend(Errors.NONE)
     expectedError = Errors.NOT_COORDINATOR
@@ -366,7 +459,7 @@ class TransactionStateManagerTest {
   def testAppendTransactionToLogWhilePendingStateChanged() = {
     // first insert the initial transaction metadata
     transactionManager.addLoadedTransactionsToCache(partitionId, coordinatorEpoch, new Pool[String, TransactionMetadata]())
-    transactionManager.putTransactionStateIfNotExists(transactionalId1, txnMetadata1)
+    transactionManager.putTransactionStateIfNotExists(txnMetadata1)
 
     prepareForTxnMessageAppend(Errors.NONE)
     expectedError = Errors.INVALID_PRODUCER_EPOCH
@@ -395,12 +488,12 @@ class TransactionStateManagerTest {
       transactionManager.addLoadedTransactionsToCache(partitionId, 0, new Pool[String, TransactionMetadata]())
     }
 
-    transactionManager.putTransactionStateIfNotExists("ongoing", transactionMetadata("ongoing", producerId = 0, state = Ongoing))
-    transactionManager.putTransactionStateIfNotExists("not-expiring", transactionMetadata("not-expiring", producerId = 1, state = Ongoing, txnTimeout = 10000))
-    transactionManager.putTransactionStateIfNotExists("prepare-commit", transactionMetadata("prepare-commit", producerId = 2, state = PrepareCommit))
-    transactionManager.putTransactionStateIfNotExists("prepare-abort", transactionMetadata("prepare-abort", producerId = 3, state = PrepareAbort))
-    transactionManager.putTransactionStateIfNotExists("complete-commit", transactionMetadata("complete-commit", producerId = 4, state = CompleteCommit))
-    transactionManager.putTransactionStateIfNotExists("complete-abort", transactionMetadata("complete-abort", producerId = 5, state = CompleteAbort))
+    transactionManager.putTransactionStateIfNotExists(transactionMetadata("ongoing", producerId = 0, state = Ongoing))
+    transactionManager.putTransactionStateIfNotExists(transactionMetadata("not-expiring", producerId = 1, state = Ongoing, txnTimeout = 10000))
+    transactionManager.putTransactionStateIfNotExists(transactionMetadata("prepare-commit", producerId = 2, state = PrepareCommit))
+    transactionManager.putTransactionStateIfNotExists(transactionMetadata("prepare-abort", producerId = 3, state = PrepareAbort))
+    transactionManager.putTransactionStateIfNotExists(transactionMetadata("complete-commit", producerId = 4, state = CompleteCommit))
+    transactionManager.putTransactionStateIfNotExists(transactionMetadata("complete-abort", producerId = 5, state = CompleteAbort))
 
     time.sleep(2000)
     val expiring = transactionManager.timedOutTransactions()
@@ -481,13 +574,11 @@ class TransactionStateManagerTest {
     // immigrate partition at epoch 0
     transactionManager.loadTransactionsForTxnTopicPartition(partitionId, coordinatorEpoch = 0, (_, _, _, _, _) => ())
     assertEquals(0, transactionManager.loadingPartitions.size)
-    assertEquals(0, transactionManager.leavingPartitions.size)
 
     // Re-immigrate partition at epoch 1. This should be successful even though we didn't get to emigrate the partition.
     prepareTxnLog(topicPartition, 0, records)
     transactionManager.loadTransactionsForTxnTopicPartition(partitionId, coordinatorEpoch = 1, (_, _, _, _, _) => ())
     assertEquals(0, transactionManager.loadingPartitions.size)
-    assertEquals(0, transactionManager.leavingPartitions.size)
     assertTrue(transactionManager.transactionMetadataCache.get(partitionId).isDefined)
     assertEquals(1, transactionManager.transactionMetadataCache.get(partitionId).get.coordinatorEpoch)
   }
@@ -578,10 +669,10 @@ class TransactionStateManagerTest {
 
     txnMetadata1.txnLastUpdateTimestamp = time.milliseconds() - txnConfig.transactionalIdExpirationMs
     txnMetadata1.state = txnState
-    transactionManager.putTransactionStateIfNotExists(transactionalId1, txnMetadata1)
+    transactionManager.putTransactionStateIfNotExists(txnMetadata1)
 
     txnMetadata2.txnLastUpdateTimestamp = time.milliseconds()
-    transactionManager.putTransactionStateIfNotExists(transactionalId2, txnMetadata2)
+    transactionManager.putTransactionStateIfNotExists(txnMetadata2)
 
     transactionManager.enableTransactionalIdExpiration()
     time.sleep(txnConfig.removeExpiredTransactionalIdsIntervalMs)
diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
index 773ea29e432a0..a3f58e9e6cad3 100644
--- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
@@ -36,6 +36,7 @@ import kafka.utils.{MockTime, TestUtils}
 import kafka.zk.KafkaZkClient
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.UnsupportedVersionException
+import org.apache.kafka.common.internals.Topic
 import org.apache.kafka.common.memory.MemoryPool
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.network.ListenerName
@@ -59,7 +60,7 @@ import org.junit.Assert.{assertArrayEquals, assertEquals, assertNull, assertTrue
 import org.junit.{After, Test}
 
 import scala.collection.JavaConverters._
-import scala.collection.{Map, Seq}
+import scala.collection.{Map, Seq, mutable}
 
 class KafkaApisTest {
 
@@ -315,6 +316,40 @@ class KafkaApisTest {
     EasyMock.verify(replicaManager)
   }
 
+  @Test
+  def shouldResignCoordinatorsIfStopReplicaReceivedWithDeleteFlag(): Unit = {
+    val controllerId = 0
+    val controllerEpoch = 5
+    val brokerEpoch = 230498320L
+
+    val groupMetadataPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, 0)
+    val txnStatePartition = new TopicPartition(Topic.TRANSACTION_STATE_TOPIC_NAME, 0)
+
+    val (_, request) = buildRequest(new StopReplicaRequest.Builder(
+      ApiKeys.STOP_REPLICA.latestVersion,
+      controllerId,
+      controllerEpoch,
+      brokerEpoch,
+      true,
+      Set(groupMetadataPartition, txnStatePartition).asJava))
+
+    EasyMock.expect(replicaManager.stopReplicas(anyObject())).andReturn(
+      (mutable.Map(groupMetadataPartition -> Errors.NONE, txnStatePartition -> Errors.NONE), Errors.NONE))
+    EasyMock.expect(controller.brokerEpoch).andStubReturn(brokerEpoch)
+
+    txnCoordinator.onResignation(txnStatePartition.partition, None)
+    EasyMock.expectLastCall()
+
+    groupCoordinator.onResignation(groupMetadataPartition.partition)
+    EasyMock.expectLastCall()
+
+    EasyMock.replay(controller, replicaManager, txnCoordinator, groupCoordinator)
+
+    createKafkaApis().handleStopReplicaRequest(request)
+
+    EasyMock.verify(txnCoordinator, groupCoordinator)
+  }
+
   @Test
   def shouldRespondWithUnknownTopicOrPartitionForBadPartitionAndNoErrorsForGoodPartition(): Unit = {
     val tp1 = new TopicPartition("t", 0)

From c8d8dc9d1e298bd0f06dee7f4c8a78c00a247d6e Mon Sep 17 00:00:00 2001
From: Brian Byrne 
Date: Fri, 17 Jan 2020 14:27:31 -0800
Subject: [PATCH 0861/1071] KAFKA-9449; Adds support for closing the producer's
 BufferPool. (#7967)

The producer's BufferPool may block allocations if its memory limit has hit capacity. If the producer is closed, it's possible for the allocation waiters to wait for max.block.ms if progress cannot be made, even when force-closed (immediate), which can cause indefinite blocking if max.block.ms is particularly high.

This patch fixes the problem by adding a `close()` method to `BufferPool`, which wakes up any waiters that have pending allocations and throws an exception.

Reviewers: Jason Gustafson 
---
 .../producer/internals/BufferPool.java        | 27 ++++++++++
 .../producer/internals/RecordAccumulator.java |  1 +
 .../producer/internals/BufferPoolTest.java    | 51 ++++++++++++++++++-
 3 files changed, 78 insertions(+), 1 deletion(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/BufferPool.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/BufferPool.java
index 22d747265d618..b49a7e2215f42 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/BufferPool.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/BufferPool.java
@@ -23,6 +23,7 @@
 import java.util.concurrent.locks.Condition;
 import java.util.concurrent.locks.ReentrantLock;
 
+import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.errors.TimeoutException;
 import org.apache.kafka.common.metrics.Metrics;
@@ -55,6 +56,7 @@ public class BufferPool {
     private final Metrics metrics;
     private final Time time;
     private final Sensor waitTime;
+    private boolean closed;
 
     /**
      * Create a new buffer pool
@@ -82,6 +84,7 @@ public BufferPool(long memory, int poolableSize, Metrics metrics, Time time, Str
                                                    metricGrpName,
                                                    "The total time an appender waits for space allocation.");
         this.waitTime.add(new Meter(TimeUnit.NANOSECONDS, rateMetricName, totalMetricName));
+        this.closed = false;
     }
 
     /**
@@ -104,6 +107,12 @@ public ByteBuffer allocate(int size, long maxTimeToBlockMs) throws InterruptedEx
 
         ByteBuffer buffer = null;
         this.lock.lock();
+
+        if (this.closed) {
+            this.lock.unlock();
+            throw new KafkaException("Producer closed while allocating memory");
+        }
+
         try {
             // check if we have a free buffer of the right size pooled
             if (size == poolableSize && !this.free.isEmpty())
@@ -138,6 +147,9 @@ public ByteBuffer allocate(int size, long maxTimeToBlockMs) throws InterruptedEx
                             recordWaitTime(timeNs);
                         }
 
+                        if (this.closed)
+                            throw new KafkaException("Producer closed while allocating memory");
+
                         if (waitingTimeElapsed) {
                             throw new TimeoutException("Failed to allocate memory within the configured max blocking time " + maxTimeToBlockMs + " ms.");
                         }
@@ -316,4 +328,19 @@ public long totalMemory() {
     Deque waiters() {
         return this.waiters;
     }
+
+    /**
+     * Closes the buffer pool. Memory will be prevented from being allocated, but may be deallocated. All allocations
+     * awaiting available memory will be notified to abort.
+     */
+    public void close() {
+        this.lock.lock();
+        this.closed = true;
+        try {
+            for (Condition waiter : this.waiters)
+                waiter.signal();
+        } finally {
+            this.lock.unlock();
+        }
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
index 745382d88a720..d002fc4061b63 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
@@ -796,6 +796,7 @@ public void unmutePartition(TopicPartition tp, long throttleUntilTimeMs) {
      */
     public void close() {
         this.closed = true;
+        this.free.close();
     }
 
     /*
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/BufferPoolTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/BufferPoolTest.java
index 8e44fa18146d4..724d9d4df10f4 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/BufferPoolTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/BufferPoolTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.clients.producer.internals;
 
+import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.errors.TimeoutException;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.utils.MockTime;
@@ -28,7 +29,10 @@
 import java.util.ArrayList;
 import java.util.Deque;
 import java.util.List;
+import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -36,6 +40,7 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.mockito.ArgumentMatchers.anyLong;
@@ -232,7 +237,7 @@ public void testCleanupMemoryAvailabilityWaiterOnInterruption() throws Exception
         // both the allocate() called by threads t1 and t2 should have been interrupted and the waiters queue should be empty
         assertEquals(pool.queued(), 0);
     }
-    
+
     @Test
     public void testCleanupMemoryAvailabilityOnMetricsException() throws Exception {
         BufferPool bufferPool = spy(new BufferPool(2, 1, new Metrics(), time, metricGroup));
@@ -377,4 +382,48 @@ public void run() {
         }
     }
 
+    @Test
+    public void testCloseAllocations() throws Exception {
+        BufferPool pool = new BufferPool(10, 1, metrics, Time.SYSTEM, metricGroup);
+        ByteBuffer buffer = pool.allocate(1, maxBlockTimeMs);
+
+        // Close the buffer pool. This should prevent any further allocations.
+        pool.close();
+
+        assertThrows(KafkaException.class, () -> pool.allocate(1, maxBlockTimeMs));
+
+        // Ensure deallocation still works.
+        pool.deallocate(buffer);
+    }
+
+    @Test
+    public void testCloseNotifyWaiters() throws Exception {
+        final int numWorkers = 2;
+
+        BufferPool pool = new BufferPool(1, 1, metrics, Time.SYSTEM, metricGroup);
+        ByteBuffer buffer = pool.allocate(1, Long.MAX_VALUE);
+
+        CountDownLatch completed = new CountDownLatch(numWorkers);
+        ExecutorService executor = Executors.newFixedThreadPool(numWorkers);
+        Callable work = new Callable() {
+                public Void call() throws Exception {
+                    assertThrows(KafkaException.class, () -> pool.allocate(1, Long.MAX_VALUE));
+                    completed.countDown();
+                    return null;
+                }
+            };
+        for (int i = 0; i < numWorkers; ++i) {
+            executor.submit(work);
+        }
+
+        assertEquals("Allocation shouldn't have happened yet, waiting on memory", numWorkers, completed.getCount());
+
+        // Close the buffer pool. This should notify all waiters.
+        pool.close();
+
+        completed.await(15, TimeUnit.SECONDS);
+
+        pool.deallocate(buffer);
+    }
+
 }

From 8761b006c18446bb64075218e934421350044ad1 Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Mon, 14 Oct 2019 12:38:33 -0700
Subject: [PATCH 0862/1071] MINOR: move "Added/Removed sensor" log messages to
 TRACE (#7502)

Reviewers: Matthias J. Sax , Guozhang Wang , Bill Bejeck 
---
 .../main/java/org/apache/kafka/common/metrics/Metrics.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/metrics/Metrics.java b/clients/src/main/java/org/apache/kafka/common/metrics/Metrics.java
index d9b4b034f0f7d..3fc73e131e284 100644
--- a/clients/src/main/java/org/apache/kafka/common/metrics/Metrics.java
+++ b/clients/src/main/java/org/apache/kafka/common/metrics/Metrics.java
@@ -413,7 +413,7 @@ public synchronized Sensor sensor(String name, MetricConfig config, long inactiv
                     children.add(s);
                 }
             }
-            log.debug("Added sensor with name {}", name);
+            log.trace("Added sensor with name {}", name);
         }
         return s;
     }
@@ -446,7 +446,7 @@ public void removeSensor(String name) {
                     if (sensors.remove(name, sensor)) {
                         for (KafkaMetric metric : sensor.metrics())
                             removeMetric(metric.metricName());
-                        log.debug("Removed sensor with name {}", name);
+                        log.trace("Removed sensor with name {}", name);
                         childSensors = childrenSensors.remove(sensor);
                         for (final Sensor parent : sensor.parents()) {
                             childrenSensors.getOrDefault(parent, emptyList()).remove(sensor);

From 194c988f1abfb4ff9a3ee2ab806b76496ced6bd9 Mon Sep 17 00:00:00 2001
From: Chris Egerton 
Date: Tue, 21 Jan 2020 13:33:50 -0600
Subject: [PATCH 0863/1071] KAFKA-9083: Various fixes/improvements for
 Connect's Values class (#7593)

Author: Chris Egerton 
Reviewers: Greg Harris , Randall Hauch 
---
 .../kafka/connect/data/SchemaBuilder.java     |  20 +++
 .../org/apache/kafka/connect/data/Values.java | 108 ++++++++---
 .../apache/kafka/connect/data/ValuesTest.java | 167 ++++++++++++++++--
 .../storage/SimpleHeaderConverterTest.java    |  30 +++-
 4 files changed, 278 insertions(+), 47 deletions(-)

diff --git a/connect/api/src/main/java/org/apache/kafka/connect/data/SchemaBuilder.java b/connect/api/src/main/java/org/apache/kafka/connect/data/SchemaBuilder.java
index fdcf05a7ac055..722f5fc89ed5a 100644
--- a/connect/api/src/main/java/org/apache/kafka/connect/data/SchemaBuilder.java
+++ b/connect/api/src/main/java/org/apache/kafka/connect/data/SchemaBuilder.java
@@ -382,6 +382,26 @@ public static SchemaBuilder map(Schema keySchema, Schema valueSchema) {
         return builder;
     }
 
+    static SchemaBuilder arrayOfNull() {
+        return new SchemaBuilder(Type.ARRAY);
+    }
+
+    static SchemaBuilder mapOfNull() {
+        return new SchemaBuilder(Type.MAP);
+    }
+
+    static SchemaBuilder mapWithNullKeys(Schema valueSchema) {
+        SchemaBuilder result = new SchemaBuilder(Type.MAP);
+        result.valueSchema = valueSchema;
+        return result;
+    }
+
+    static SchemaBuilder mapWithNullValues(Schema keySchema) {
+        SchemaBuilder result = new SchemaBuilder(Type.MAP);
+        result.keySchema = keySchema;
+        return result;
+    }
+
     @Override
     public Schema keySchema() {
         return keySchema;
diff --git a/connect/api/src/main/java/org/apache/kafka/connect/data/Values.java b/connect/api/src/main/java/org/apache/kafka/connect/data/Values.java
index c1bebdfe14e35..93c320a236760 100644
--- a/connect/api/src/main/java/org/apache/kafka/connect/data/Values.java
+++ b/connect/api/src/main/java/org/apache/kafka/connect/data/Values.java
@@ -723,14 +723,30 @@ public static DateFormat dateFormatFor(java.util.Date value) {
         return new SimpleDateFormat(ISO_8601_TIMESTAMP_FORMAT_PATTERN);
     }
 
+    protected static boolean canParseSingleTokenLiteral(Parser parser, boolean embedded, String tokenLiteral) {
+        int startPosition = parser.mark();
+        // If the next token is what we expect, then either...
+        if (parser.canConsume(tokenLiteral)) {
+            //   ...we're reading an embedded value, in which case the next token will be handled appropriately
+            //      by the caller if it's something like an end delimiter for a map or array, or a comma to
+            //      separate multiple embedded values...
+            //   ...or it's being parsed as part of a top-level string, in which case, any other tokens should
+            //      cause use to stop parsing this single-token literal as such and instead just treat it like
+            //      a string. For example, the top-level string "true}" will be tokenized as the tokens "true" and
+            //      "}", but should ultimately be parsed as just the string "true}" instead of the boolean true.
+            if (embedded || !parser.hasNext()) {
+                return true;
+            }
+        }
+        parser.rewindTo(startPosition);
+        return false;
+    }
+
     protected static SchemaAndValue parse(Parser parser, boolean embedded) throws NoSuchElementException {
         if (!parser.hasNext()) {
             return null;
         }
         if (embedded) {
-            if (parser.canConsume(NULL_VALUE)) {
-                return null;
-            }
             if (parser.canConsume(QUOTE_DELIMITER)) {
                 StringBuilder sb = new StringBuilder();
                 while (parser.hasNext()) {
@@ -742,34 +758,51 @@ protected static SchemaAndValue parse(Parser parser, boolean embedded) throws No
                 return new SchemaAndValue(Schema.STRING_SCHEMA, sb.toString());
             }
         }
-        if (parser.canConsume(TRUE_LITERAL)) {
+
+        if (canParseSingleTokenLiteral(parser, embedded, NULL_VALUE)) {
+            return null;
+        }
+        if (canParseSingleTokenLiteral(parser, embedded, TRUE_LITERAL)) {
             return TRUE_SCHEMA_AND_VALUE;
         }
-        if (parser.canConsume(FALSE_LITERAL)) {
+        if (canParseSingleTokenLiteral(parser, embedded, FALSE_LITERAL)) {
             return FALSE_SCHEMA_AND_VALUE;
         }
+
         int startPosition = parser.mark();
+
         try {
             if (parser.canConsume(ARRAY_BEGIN_DELIMITER)) {
                 List result = new ArrayList<>();
                 Schema elementSchema = null;
                 while (parser.hasNext()) {
                     if (parser.canConsume(ARRAY_END_DELIMITER)) {
-                        Schema listSchema = null;
+                        Schema listSchema;
                         if (elementSchema != null) {
                             listSchema = SchemaBuilder.array(elementSchema).schema();
+                            result = alignListEntriesWithSchema(listSchema, result);
+                        } else {
+                            // Every value is null
+                            listSchema = SchemaBuilder.arrayOfNull().build();
                         }
-                        result = alignListEntriesWithSchema(listSchema, result);
                         return new SchemaAndValue(listSchema, result);
                     }
+
                     if (parser.canConsume(COMMA_DELIMITER)) {
                         throw new DataException("Unable to parse an empty array element: " + parser.original());
                     }
                     SchemaAndValue element = parse(parser, true);
                     elementSchema = commonSchemaFor(elementSchema, element);
-                    result.add(element.value());
-                    parser.canConsume(COMMA_DELIMITER);
+                    result.add(element != null ? element.value() : null);
+
+                    int currentPosition = parser.mark();
+                    if (parser.canConsume(ARRAY_END_DELIMITER)) {
+                        parser.rewindTo(currentPosition);
+                    } else if (!parser.canConsume(COMMA_DELIMITER)) {
+                        throw new DataException("Array elements missing '" + COMMA_DELIMITER + "' delimiter");
+                    }
                 }
+
                 // Missing either a comma or an end delimiter
                 if (COMMA_DELIMITER.equals(parser.previous())) {
                     throw new DataException("Array is missing element after ',': " + parser.original());
@@ -783,26 +816,34 @@ protected static SchemaAndValue parse(Parser parser, boolean embedded) throws No
                 Schema valueSchema = null;
                 while (parser.hasNext()) {
                     if (parser.canConsume(MAP_END_DELIMITER)) {
-                        Schema mapSchema = null;
+                        Schema mapSchema;
                         if (keySchema != null && valueSchema != null) {
-                            mapSchema = SchemaBuilder.map(keySchema, valueSchema).schema();
+                            mapSchema = SchemaBuilder.map(keySchema, valueSchema).build();
+                            result = alignMapKeysAndValuesWithSchema(mapSchema, result);
+                        } else if (keySchema != null) {
+                            mapSchema = SchemaBuilder.mapWithNullValues(keySchema);
+                            result = alignMapKeysWithSchema(mapSchema, result);
+                        } else {
+                            mapSchema = SchemaBuilder.mapOfNull().build();
                         }
-                        result = alignMapKeysAndValuesWithSchema(mapSchema, result);
                         return new SchemaAndValue(mapSchema, result);
                     }
+
                     if (parser.canConsume(COMMA_DELIMITER)) {
-                        throw new DataException("Unable to parse a map entry has no key or value: " + parser.original());
+                        throw new DataException("Unable to parse a map entry with no key or value: " + parser.original());
                     }
                     SchemaAndValue key = parse(parser, true);
                     if (key == null || key.value() == null) {
                         throw new DataException("Map entry may not have a null key: " + parser.original());
                     }
+
                     if (!parser.canConsume(ENTRY_DELIMITER)) {
                         throw new DataException("Map entry is missing '=': " + parser.original());
                     }
                     SchemaAndValue value = parse(parser, true);
                     Object entryValue = value != null ? value.value() : null;
                     result.put(key.value(), entryValue);
+
                     parser.canConsume(COMMA_DELIMITER);
                     keySchema = commonSchemaFor(keySchema, key);
                     valueSchema = commonSchemaFor(valueSchema, value);
@@ -811,14 +852,19 @@ protected static SchemaAndValue parse(Parser parser, boolean embedded) throws No
                 if (COMMA_DELIMITER.equals(parser.previous())) {
                     throw new DataException("Map is missing element after ',': " + parser.original());
                 }
-                throw new DataException("Map is missing terminating ']': " + parser.original());
+                throw new DataException("Map is missing terminating '}': " + parser.original());
             }
         } catch (DataException e) {
-            LOG.debug("Unable to parse the value as a map; reverting to string", e);
+            LOG.debug("Unable to parse the value as a map or an array; reverting to string", e);
             parser.rewindTo(startPosition);
         }
-        String token = parser.next().trim();
-        assert !token.isEmpty(); // original can be empty string but is handled right away; no way for token to be empty here
+
+        String token = parser.next();
+        if (token.trim().isEmpty()) {
+            return new SchemaAndValue(Schema.STRING_SCHEMA, token);
+        }
+        token = token.trim();
+
         char firstChar = token.charAt(0);
         boolean firstCharIsDigit = Character.isDigit(firstChar);
         if (firstCharIsDigit || firstChar == '+' || firstChar == '-') {
@@ -878,6 +924,9 @@ protected static SchemaAndValue parse(Parser parser, boolean embedded) throws No
                 }
             }
         }
+        if (embedded) {
+            throw new DataException("Failed to parse embedded value");
+        }
         // At this point, the only thing this can be is a string. Embedded strings were processed above,
         // so this is not embedded and we can use the original string...
         return new SchemaAndValue(Schema.STRING_SCHEMA, parser.original());
@@ -953,9 +1002,6 @@ protected static Schema commonSchemaFor(Schema previous, SchemaAndValue latest)
     }
 
     protected static List alignListEntriesWithSchema(Schema schema, List input) {
-        if (schema == null) {
-            return input;
-        }
         Schema valueSchema = schema.valueSchema();
         List result = new ArrayList<>();
         for (Object value : input) {
@@ -966,9 +1012,6 @@ protected static List alignListEntriesWithSchema(Schema schema, List alignMapKeysAndValuesWithSchema(Schema mapSchema, Map input) {
-        if (mapSchema == null) {
-            return input;
-        }
         Schema keySchema = mapSchema.keySchema();
         Schema valueSchema = mapSchema.valueSchema();
         Map result = new LinkedHashMap<>();
@@ -980,6 +1023,16 @@ protected static Map alignMapKeysAndValuesWithSchema(Schema mapS
         return result;
     }
 
+    protected static Map alignMapKeysWithSchema(Schema mapSchema, Map input) {
+        Schema keySchema = mapSchema.keySchema();
+        Map result = new LinkedHashMap<>();
+        for (Map.Entry entry : input.entrySet()) {
+            Object newKey = convertTo(keySchema, null, entry.getKey());
+            result.put(newKey, entry.getValue());
+        }
+        return result;
+    }
+
     protected static class SchemaDetector {
         private Type knownType = null;
         private boolean optional = false;
@@ -1123,12 +1176,13 @@ protected boolean isNext(String expected, boolean ignoreLeadingAndTrailingWhites
                 nextToken = consumeNextToken();
             }
             if (ignoreLeadingAndTrailingWhitespace) {
-                nextToken = nextToken.trim();
-                while (nextToken.isEmpty() && canConsumeNextToken()) {
-                    nextToken = consumeNextToken().trim();
+                while (nextToken.trim().isEmpty() && canConsumeNextToken()) {
+                    nextToken = consumeNextToken();
                 }
             }
-            return nextToken.equals(expected);
+            return ignoreLeadingAndTrailingWhitespace
+                ? nextToken.trim().equals(expected)
+                : nextToken.equals(expected);
         }
     }
 }
diff --git a/connect/api/src/test/java/org/apache/kafka/connect/data/ValuesTest.java b/connect/api/src/test/java/org/apache/kafka/connect/data/ValuesTest.java
index 162222db155c8..a5909f3ece9bf 100644
--- a/connect/api/src/test/java/org/apache/kafka/connect/data/ValuesTest.java
+++ b/connect/api/src/test/java/org/apache/kafka/connect/data/ValuesTest.java
@@ -22,6 +22,9 @@
 import org.junit.Test;
 
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -35,6 +38,8 @@
 
 public class ValuesTest {
 
+    private static final String WHITESPACE = "\n \t \t\n";
+
     private static final long MILLIS_PER_DAY = 24 * 60 * 60 * 1000;
 
     private static final Map STRING_MAP = new LinkedHashMap<>();
@@ -67,6 +72,149 @@ public class ValuesTest {
         INT_LIST.add(-987654321);
     }
 
+    @Test
+    public void shouldNotParseUnquotedEmbeddedMapKeysAsStrings() {
+        SchemaAndValue schemaAndValue = Values.parseString("{foo: 3}");
+        assertEquals(Type.STRING, schemaAndValue.schema().type());
+        assertEquals("{foo: 3}", schemaAndValue.value());
+    }
+
+    @Test
+    public void shouldNotParseUnquotedEmbeddedMapValuesAsStrings() {
+        SchemaAndValue schemaAndValue = Values.parseString("{3: foo}");
+        assertEquals(Type.STRING, schemaAndValue.schema().type());
+        assertEquals("{3: foo}", schemaAndValue.value());
+    }
+
+    @Test
+    public void shouldNotParseUnquotedArrayElementsAsStrings() {
+        SchemaAndValue schemaAndValue = Values.parseString("[foo]");
+        assertEquals(Type.STRING, schemaAndValue.schema().type());
+        assertEquals("[foo]", schemaAndValue.value());
+    }
+
+    @Test
+    public void shouldNotParseStringsBeginningWithNullAsStrings() {
+        SchemaAndValue schemaAndValue = Values.parseString("null=");
+        assertEquals(Type.STRING, schemaAndValue.schema().type());
+        assertEquals("null=", schemaAndValue.value());
+    }
+
+    @Test
+    public void shouldParseStringsBeginningWithTrueAsStrings() {
+        SchemaAndValue schemaAndValue = Values.parseString("true}");
+        assertEquals(Type.STRING, schemaAndValue.schema().type());
+        assertEquals("true}", schemaAndValue.value());
+    }
+
+    @Test
+    public void shouldParseStringsBeginningWithFalseAsStrings() {
+        SchemaAndValue schemaAndValue = Values.parseString("false]");
+        assertEquals(Type.STRING, schemaAndValue.schema().type());
+        assertEquals("false]", schemaAndValue.value());
+    }
+
+    @Test
+    public void shouldParseTrueAsBooleanIfSurroundedByWhitespace() {
+        SchemaAndValue schemaAndValue = Values.parseString(WHITESPACE + "true" + WHITESPACE);
+        assertEquals(Type.BOOLEAN, schemaAndValue.schema().type());
+        assertEquals(true, schemaAndValue.value());
+    }
+
+    @Test
+    public void shouldParseFalseAsBooleanIfSurroundedByWhitespace() {
+        SchemaAndValue schemaAndValue = Values.parseString(WHITESPACE + "false" + WHITESPACE);
+        assertEquals(Type.BOOLEAN, schemaAndValue.schema().type());
+        assertEquals(false, schemaAndValue.value());
+    }
+
+    @Test
+    public void shouldParseNullAsNullIfSurroundedByWhitespace() {
+        SchemaAndValue schemaAndValue = Values.parseString(WHITESPACE + "null" + WHITESPACE);
+        assertNull(schemaAndValue);
+    }
+
+    @Test
+    public void shouldParseBooleanLiteralsEmbeddedInArray() {
+        SchemaAndValue schemaAndValue = Values.parseString("[true, false]");
+        assertEquals(Type.ARRAY, schemaAndValue.schema().type());
+        assertEquals(Type.BOOLEAN, schemaAndValue.schema().valueSchema().type());
+        assertEquals(Arrays.asList(true, false), schemaAndValue.value());
+    }
+
+    @Test
+    public void shouldParseBooleanLiteralsEmbeddedInMap() {
+        SchemaAndValue schemaAndValue = Values.parseString("{true: false, false: true}");
+        assertEquals(Type.MAP, schemaAndValue.schema().type());
+        assertEquals(Type.BOOLEAN, schemaAndValue.schema().keySchema().type());
+        assertEquals(Type.BOOLEAN, schemaAndValue.schema().valueSchema().type());
+        Map expectedValue = new HashMap<>();
+        expectedValue.put(true, false);
+        expectedValue.put(false, true);
+        assertEquals(expectedValue, schemaAndValue.value());
+    }
+
+    @Test
+    public void shouldNotParseAsMapWithoutCommas() {
+        SchemaAndValue schemaAndValue = Values.parseString("{6:9 4:20}");
+        assertEquals(Type.STRING, schemaAndValue.schema().type());
+        assertEquals("{6:9 4:20}", schemaAndValue.value());
+    }
+
+    @Test
+    public void shouldNotParseAsArrayWithoutCommas() {
+        SchemaAndValue schemaAndValue = Values.parseString("[0 1 2]");
+        assertEquals(Type.STRING, schemaAndValue.schema().type());
+        assertEquals("[0 1 2]", schemaAndValue.value());
+    }
+
+    @Test
+    public void shouldParseEmptyMap() {
+        SchemaAndValue schemaAndValue = Values.parseString("{}");
+        assertEquals(Type.MAP, schemaAndValue.schema().type());
+        assertEquals(Collections.emptyMap(), schemaAndValue.value());
+    }
+
+    @Test
+    public void shouldParseEmptyArray() {
+        SchemaAndValue schemaAndValue = Values.parseString("[]");
+        assertEquals(Type.ARRAY, schemaAndValue.schema().type());
+        assertEquals(Collections.emptyList(), schemaAndValue.value());
+    }
+
+    @Test
+    public void shouldNotParseAsMapWithNullKeys() {
+        SchemaAndValue schemaAndValue = Values.parseString("{null: 3}");
+        assertEquals(Type.STRING, schemaAndValue.schema().type());
+        assertEquals("{null: 3}", schemaAndValue.value());
+    }
+
+    @Test
+    public void shouldParseNull() {
+        SchemaAndValue schemaAndValue = Values.parseString("null");
+        assertNull(schemaAndValue);
+    }
+
+    @Test
+    public void shouldConvertStringOfNull() {
+        assertRoundTrip(Schema.STRING_SCHEMA, "null");
+    }
+
+    @Test
+    public void shouldParseNullMapValues() {
+        SchemaAndValue schemaAndValue = Values.parseString("{3: null}");
+        assertEquals(Type.MAP, schemaAndValue.schema().type());
+        assertEquals(Type.INT8, schemaAndValue.schema().keySchema().type());
+        assertEquals(Collections.singletonMap((byte) 3, null), schemaAndValue.value());
+    }
+
+    @Test
+    public void shouldParseNullArrayElements() {
+        SchemaAndValue schemaAndValue = Values.parseString("[null]");
+        assertEquals(Type.ARRAY, schemaAndValue.schema().type());
+        assertEquals(Collections.singletonList(null), schemaAndValue.value());
+    }
+
     @Test
     public void shouldEscapeStringsWithEmbeddedQuotesAndBackslashes() {
         String original = "three\"blind\\\"mice";
@@ -214,7 +362,8 @@ public void shouldConvertStringOfListWithMixedElementTypesIntoListWithDifferentE
     public void shouldParseStringListWithMultipleElementTypesAndReturnListWithNoSchema() {
         String str = "[1, 2, 3, \"four\"]";
         SchemaAndValue result = Values.parseString(str);
-        assertNull(result.schema());
+        assertEquals(Type.ARRAY, result.schema().type());
+        assertNull(result.schema().valueSchema());
         List list = (List) result.value();
         assertEquals(4, list.size());
         assertEquals(1, ((Number) list.get(0)).intValue());
@@ -256,7 +405,7 @@ public void shouldFailToConvertToListFromStringWithNonCommonElementTypeAndBlankE
      */
     @Test(expected = DataException.class)
     public void shouldFailToParseStringOfMapWithIntValuesWithBlankEntry() {
-        Values.convertToList(Schema.STRING_SCHEMA, " { \"foo\" :  1234567890 ,, \"bar\" : 0,  \"baz\" : -987654321 }  ");
+        Values.convertToMap(Schema.STRING_SCHEMA, " { \"foo\" :  1234567890 ,, \"bar\" : 0,  \"baz\" : -987654321 }  ");
     }
 
     /**
@@ -264,7 +413,7 @@ public void shouldFailToParseStringOfMapWithIntValuesWithBlankEntry() {
      */
     @Test(expected = DataException.class)
     public void shouldFailToParseStringOfMalformedMap() {
-        Values.convertToList(Schema.STRING_SCHEMA, " { \"foo\" :  1234567890 , \"a\", \"bar\" : 0,  \"baz\" : -987654321 }  ");
+        Values.convertToMap(Schema.STRING_SCHEMA, " { \"foo\" :  1234567890 , \"a\", \"bar\" : 0,  \"baz\" : -987654321 }  ");
     }
 
     /**
@@ -272,7 +421,7 @@ public void shouldFailToParseStringOfMalformedMap() {
      */
     @Test(expected = DataException.class)
     public void shouldFailToParseStringOfMapWithIntValuesWithOnlyBlankEntries() {
-        Values.convertToList(Schema.STRING_SCHEMA, " { ,,  , , }  ");
+        Values.convertToMap(Schema.STRING_SCHEMA, " { ,,  , , }  ");
     }
 
     /**
@@ -280,15 +429,7 @@ public void shouldFailToParseStringOfMapWithIntValuesWithOnlyBlankEntries() {
      */
     @Test(expected = DataException.class)
     public void shouldFailToParseStringOfMapWithIntValuesWithBlankEntries() {
-        Values.convertToList(Schema.STRING_SCHEMA, " { \"foo\" :  \"1234567890\" ,, \"bar\" : \"0\",  \"baz\" : \"boz\" }  ");
-    }
-
-    /**
-     * Schema for Map requires a schema for key and value, but we have no key or value and Connect has no "any" type
-     */
-    @Test(expected = DataException.class)
-    public void shouldFailToParseStringOfEmptyMap() {
-        Values.convertToList(Schema.STRING_SCHEMA, " { }  ");
+        Values.convertToMap(Schema.STRING_SCHEMA, " { \"foo\" :  \"1234567890\" ,, \"bar\" : \"0\",  \"baz\" : \"boz\" }  ");
     }
 
     @Test
diff --git a/connect/api/src/test/java/org/apache/kafka/connect/storage/SimpleHeaderConverterTest.java b/connect/api/src/test/java/org/apache/kafka/connect/storage/SimpleHeaderConverterTest.java
index fdfdd32f139f3..58a17f57ba192 100644
--- a/connect/api/src/test/java/org/apache/kafka/connect/storage/SimpleHeaderConverterTest.java
+++ b/connect/api/src/test/java/org/apache/kafka/connect/storage/SimpleHeaderConverterTest.java
@@ -164,11 +164,15 @@ public void shouldConvertListWithIntegerValues() {
     }
 
     @Test
-    public void shouldConvertMapWithStringKeysAndMixedValuesToMapWithoutSchema() {
+    public void shouldConvertMapWithStringKeysAndMixedValuesToMap() {
         Map map = new LinkedHashMap<>();
         map.put("foo", "bar");
         map.put("baz", (short) 3456);
-        assertRoundTrip(null, map);
+        SchemaAndValue result = roundTrip(null, map);
+        assertEquals(Schema.Type.MAP, result.schema().type());
+        assertEquals(Schema.Type.STRING, result.schema().keySchema().type());
+        assertNull(result.schema().valueSchema());
+        assertEquals(map, result.value());
     }
 
     @Test
@@ -176,17 +180,29 @@ public void shouldConvertListWithMixedValuesToListWithoutSchema() {
         List list = new ArrayList<>();
         list.add("foo");
         list.add((short) 13344);
-        assertRoundTrip(null, list);
+        SchemaAndValue result = roundTrip(null, list);
+        assertEquals(Schema.Type.ARRAY, result.schema().type());
+        assertNull(result.schema().valueSchema());
+        assertEquals(list, result.value());
     }
 
     @Test
-    public void shouldConvertEmptyMapToMapWithoutSchema() {
-        assertRoundTrip(null, new LinkedHashMap<>());
+    public void shouldConvertEmptyMapToMap() {
+        Map map = new LinkedHashMap<>();
+        SchemaAndValue result = roundTrip(null, map);
+        assertEquals(Schema.Type.MAP, result.schema().type());
+        assertNull(result.schema().keySchema());
+        assertNull(result.schema().valueSchema());
+        assertEquals(map, result.value());
     }
 
     @Test
-    public void shouldConvertEmptyListToListWithoutSchema() {
-        assertRoundTrip(null, new ArrayList<>());
+    public void shouldConvertEmptyListToList() {
+        List list = new ArrayList<>();
+        SchemaAndValue result = roundTrip(null, list);
+        assertEquals(Schema.Type.ARRAY, result.schema().type());
+        assertNull(result.schema().valueSchema());
+        assertEquals(list, result.value());
     }
 
     protected SchemaAndValue roundTrip(Schema schema, Object input) {

From 2f52d6330aeb47921b0dc1a582e36597f88b71aa Mon Sep 17 00:00:00 2001
From: Nigel Liang 
Date: Tue, 21 Jan 2020 12:25:35 -0800
Subject: [PATCH 0864/1071] KAFKA-9024: Better error message when field
 specified does not exist (#7819)

Author: Nigel Liang 
Reviewer: Randall Hauch 
---
 .../kafka/connect/transforms/ValueToKey.java   |  9 +++++++--
 .../connect/transforms/ValueToKeyTest.java     | 18 ++++++++++++++++++
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/ValueToKey.java b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/ValueToKey.java
index a9d960103a0b8..b2226d38fca87 100644
--- a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/ValueToKey.java
+++ b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/ValueToKey.java
@@ -21,9 +21,11 @@
 import org.apache.kafka.common.cache.SynchronizedCache;
 import org.apache.kafka.common.config.ConfigDef;
 import org.apache.kafka.connect.connector.ConnectRecord;
+import org.apache.kafka.connect.data.Field;
 import org.apache.kafka.connect.data.Schema;
 import org.apache.kafka.connect.data.SchemaBuilder;
 import org.apache.kafka.connect.data.Struct;
+import org.apache.kafka.connect.errors.DataException;
 import org.apache.kafka.connect.transforms.util.NonEmptyListValidator;
 import org.apache.kafka.connect.transforms.util.SimpleConfig;
 
@@ -82,8 +84,11 @@ private R applyWithSchema(R record) {
         if (keySchema == null) {
             final SchemaBuilder keySchemaBuilder = SchemaBuilder.struct();
             for (String field : fields) {
-                final Schema fieldSchema = value.schema().field(field).schema();
-                keySchemaBuilder.field(field, fieldSchema);
+                final Field fieldFromValue = value.schema().field(field);
+                if (fieldFromValue == null) {
+                    throw new DataException("Field does not exist: " + field);
+                }
+                keySchemaBuilder.field(field, fieldFromValue.schema());
             }
             keySchema = keySchemaBuilder.build();
             valueToKeySchemaCache.put(value.schema(), keySchema);
diff --git a/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/ValueToKeyTest.java b/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/ValueToKeyTest.java
index e2dfa1773b96f..5854658aa29b3 100644
--- a/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/ValueToKeyTest.java
+++ b/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/ValueToKeyTest.java
@@ -19,6 +19,7 @@
 import org.apache.kafka.connect.data.Schema;
 import org.apache.kafka.connect.data.SchemaBuilder;
 import org.apache.kafka.connect.data.Struct;
+import org.apache.kafka.connect.errors.DataException;
 import org.apache.kafka.connect.sink.SinkRecord;
 import org.junit.After;
 import org.junit.Test;
@@ -28,6 +29,7 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThrows;
 
 public class ValueToKeyTest {
     private final ValueToKey xform = new ValueToKey<>();
@@ -88,4 +90,20 @@ public void withSchema() {
         assertEquals(expectedKey, transformedRecord.key());
     }
 
+    @Test
+    public void nonExistingField() {
+        xform.configure(Collections.singletonMap("fields", "not_exist"));
+
+        final Schema valueSchema = SchemaBuilder.struct()
+            .field("a", Schema.INT32_SCHEMA)
+            .build();
+
+        final Struct value = new Struct(valueSchema);
+        value.put("a", 1);
+
+        final SinkRecord record = new SinkRecord("", 0, null, null, valueSchema, value, 0);
+
+        DataException actual = assertThrows(DataException.class, () -> xform.apply(record));
+        assertEquals("Field does not exist: not_exist", actual.getMessage());
+    }
 }

From 7ce89f849b334b3f186aca450edf1715d0cdd334 Mon Sep 17 00:00:00 2001
From: Ivan Yurchenko 
Date: Tue, 21 Jan 2020 22:52:40 +0200
Subject: [PATCH 0865/1071] KAFKA-9143: Log task reconfiguration error only
 when it happened (#7648)

This commit makes `DistributedHerder` log that some error has happened during task reconfiguration only when it actually has happened.

Author: Ivan Yurchenko 
Reviewer: Randall Hauch 
---
 .../connect/runtime/distributed/DistributedHerder.java      | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
index f3861ddd59bf9..1dcc9e65e5797 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
@@ -1245,8 +1245,10 @@ public Void call() throws Exception {
                             }, new Callback() {
                                 @Override
                                 public void onCompletion(Throwable error, Void result) {
-                                    log.error("Unexpected error during connector task reconfiguration: ", error);
-                                    log.error("Task reconfiguration for {} failed unexpectedly, this connector will not be properly reconfigured unless manually triggered.", connName);
+                                    if (error != null) {
+                                        log.error("Unexpected error during connector task reconfiguration: ", error);
+                                        log.error("Task reconfiguration for {} failed unexpectedly, this connector will not be properly reconfigured unless manually triggered.", connName);
+                                    }
                                 }
                             }
                     );

From ca4eaa9255f2ced888fc7f6213e64dafd57827f7 Mon Sep 17 00:00:00 2001
From: huxi 
Date: Sat, 25 Jan 2020 06:47:22 +0800
Subject: [PATCH 0866/1071] KAFKA-9254; Overridden topic configs are reset
 after dynamic default change (#7870)

Currently, when a dynamic change is made to the broker-level default log configuration, existing log configs will be recreated with an empty overridden configs. In such case, when updating dynamic broker configs a second round, the topic-level configs are lost. This can cause unexpected data loss, for example, if the cleanup policy changes from "compact" to "delete."

Reviewers: Rajini Sivaram , Jason Gustafson 
---
 .../kafka/server/DynamicBrokerConfig.scala    |  2 +-
 .../DynamicBrokerReconfigurationTest.scala    | 36 ++++++++++++++++++-
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
index bf3d988244689..75081847e3e98 100755
--- a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
+++ b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
@@ -619,7 +619,7 @@ class DynamicLogConfig(logManager: LogManager, server: KafkaServer) extends Brok
       props ++= newBrokerDefaults
       props ++= log.config.originals.asScala.filterKeys(log.config.overriddenConfigs.contains)
 
-      val logConfig = LogConfig(props.asJava)
+      val logConfig = LogConfig(props.asJava, log.config.overriddenConfigs)
       log.updateConfig(logConfig)
     }
   }
diff --git a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
index acab29bd161b9..715ff20c9134a 100644
--- a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
+++ b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
@@ -504,6 +504,40 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
     stopAndVerifyProduceConsume(producerThread, consumerThread)
   }
 
+  @Test
+  def testConsecutiveConfigChange(): Unit = {
+    val topic2 = "testtopic2"
+    val topicProps = new Properties
+    topicProps.put(KafkaConfig.MinInSyncReplicasProp, "2")
+    TestUtils.createTopic(zkClient, topic2, 1, replicationFactor = numServers, servers, topicProps)
+    var log = servers.head.logManager.getLog(new TopicPartition(topic2, 0)).getOrElse(throw new IllegalStateException("Log not found"))
+    assertTrue(log.config.overriddenConfigs.contains(KafkaConfig.MinInSyncReplicasProp))
+    assertEquals("2", log.config.originals().get(KafkaConfig.MinInSyncReplicasProp).toString)
+
+    val props = new Properties
+    props.put(KafkaConfig.MinInSyncReplicasProp, "3")
+    // Make a broker-default config
+    reconfigureServers(props, perBrokerConfig = false, (KafkaConfig.MinInSyncReplicasProp, "3"))
+    // Verify that all broker defaults have been updated again
+    servers.foreach { server =>
+      props.asScala.foreach { case (k, v) =>
+        assertEquals(s"Not reconfigured $k", v, server.config.originals.get(k).toString)
+      }
+    }
+
+    log = servers.head.logManager.getLog(new TopicPartition(topic2, 0)).getOrElse(throw new IllegalStateException("Log not found"))
+    assertTrue(log.config.overriddenConfigs.contains(KafkaConfig.MinInSyncReplicasProp))
+    assertEquals("2", log.config.originals().get(KafkaConfig.MinInSyncReplicasProp).toString) // Verify topic-level config survives
+
+    // Make a second broker-default change
+    props.clear()
+    props.put(KafkaConfig.LogRetentionTimeMillisProp, "604800000")
+    reconfigureServers(props, perBrokerConfig = false, (KafkaConfig.LogRetentionTimeMillisProp, "604800000"))
+    log = servers.head.logManager.getLog(new TopicPartition(topic2, 0)).getOrElse(throw new IllegalStateException("Log not found"))
+    assertTrue(log.config.overriddenConfigs.contains(KafkaConfig.MinInSyncReplicasProp))
+    assertEquals("2", log.config.originals().get(KafkaConfig.MinInSyncReplicasProp).toString) // Verify topic-level config still survives
+  }
+
   @Test
   def testDefaultTopicConfig(): Unit = {
     val (producerThread, consumerThread) = startProduceConsume(retries = 0)
@@ -774,7 +808,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
     val partitions = (0 until numPartitions).map(i => new TopicPartition(topic, i)).filter { tp =>
       zkClient.getLeaderForPartition(tp).contains(leaderId)
     }
-    assertTrue(s"Partitons not found with leader $leaderId", partitions.nonEmpty)
+    assertTrue(s"Partitions not found with leader $leaderId", partitions.nonEmpty)
     partitions.foreach { tp =>
       (1 to 2).foreach { i =>
         val replicaFetcherManager = servers(i).replicaManager.replicaFetcherManager

From f78878e6c1efbd98e9ebeff61979738871a54ffb Mon Sep 17 00:00:00 2001
From: Tomislav 
Date: Tue, 14 Jan 2020 23:03:44 +0100
Subject: [PATCH 0867/1071] KAFKA-8764: LogCleanerManager endless loop while
 compacting/cleaning (#7932)

This fix makes the LogCleaner tolerant of gaps in the offset sequence. Previously, this could lead to endless loops of cleaning which required manual intervention.

Reviewers: Jun Rao , David Arthur 
---
 .../src/main/scala/kafka/log/LogCleaner.scala | 15 ++++++-
 .../scala/unit/kafka/log/LogCleanerTest.scala | 43 +++++++++++++++++++
 2 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/kafka/log/LogCleaner.scala b/core/src/main/scala/kafka/log/LogCleaner.scala
index bcb358631c8c5..f7babf7ab9b33 100644
--- a/core/src/main/scala/kafka/log/LogCleaner.scala
+++ b/core/src/main/scala/kafka/log/LogCleaner.scala
@@ -36,6 +36,7 @@ import org.apache.kafka.common.record._
 import org.apache.kafka.common.utils.Time
 
 import scala.collection.JavaConverters._
+import scala.collection.mutable.ListBuffer
 import scala.collection.{Iterable, Seq, Set, mutable}
 import scala.util.control.ControlThrowable
 
@@ -878,6 +879,11 @@ private[log] class Cleaner(val id: Int,
                                   stats: CleanerStats): Unit = {
     map.clear()
     val dirty = log.logSegments(start, end).toBuffer
+    val nextSegmentStartOffsets = new ListBuffer[Long]
+    if (dirty.nonEmpty) {
+      for (nextSegment <- dirty.tail) nextSegmentStartOffsets.append(nextSegment.baseOffset)
+      nextSegmentStartOffsets.append(end)
+    }
     info("Building offset map for log %s for %d segments in offset range [%d, %d).".format(log.name, dirty.size, start, end))
 
     val transactionMetadata = new CleanedTransactionMetadata
@@ -887,10 +893,10 @@ private[log] class Cleaner(val id: Int,
     // Add all the cleanable dirty segments. We must take at least map.slots * load_factor,
     // but we may be able to fit more (if there is lots of duplication in the dirty section of the log)
     var full = false
-    for (segment <- dirty if !full) {
+    for ( (segment, nextSegmentStartOffset) <- dirty.zip(nextSegmentStartOffsets) if !full) {
       checkDone(log.topicPartition)
 
-      full = buildOffsetMapForSegment(log.topicPartition, segment, map, start, log.config.maxMessageSize,
+      full = buildOffsetMapForSegment(log.topicPartition, segment, map, start, nextSegmentStartOffset, log.config.maxMessageSize,
         transactionMetadata, stats)
       if (full)
         debug("Offset map is full, %d segments fully mapped, segment with base offset %d is partially mapped".format(dirty.indexOf(segment), segment.baseOffset))
@@ -911,6 +917,7 @@ private[log] class Cleaner(val id: Int,
                                        segment: LogSegment,
                                        map: OffsetMap,
                                        startOffset: Long,
+                                       nextSegmentStartOffset: Long,
                                        maxLogMessageSize: Int,
                                        transactionMetadata: CleanedTransactionMetadata,
                                        stats: CleanerStats): Boolean = {
@@ -964,6 +971,10 @@ private[log] class Cleaner(val id: Int,
       if(position == startPosition)
         growBuffersOrFail(segment.log, position, maxLogMessageSize, records)
     }
+
+    // In the case of offsets gap, fast forward to latest expected offset in this segment.
+    map.updateLatestOffset(nextSegmentStartOffset - 1L)
+
     restoreBuffers()
     false
   }
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala
index 18d86b68c1fa9..644822cc4f45e 100755
--- a/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala
@@ -1561,6 +1561,49 @@ class LogCleanerTest {
     assertEquals("The tombstone should be retained.", 1, log.logSegments.head.log.batches.iterator.next().lastOffset)
   }
 
+  /**
+   * Verify that the clean is able to move beyond missing offsets records in dirty log
+   */
+  @Test
+  def testCleaningBeyondMissingOffsets(): Unit = {
+    val logProps = new Properties()
+    logProps.put(LogConfig.SegmentBytesProp, 1024*1024: java.lang.Integer)
+    logProps.put(LogConfig.CleanupPolicyProp, LogConfig.Compact)
+    val logConfig = LogConfig(logProps)
+    val cleaner = makeCleaner(Int.MaxValue)
+
+    {
+      val log = makeLog(dir = TestUtils.randomPartitionLogDir(tmpdir), config = logConfig)
+      writeToLog(log, (0 to 9) zip (0 to 9), (0L to 9L))
+      // roll new segment with baseOffset 11, leaving previous with holes in offset range [9,10]
+      log.roll(Some(11L))
+
+      // active segment record
+      log.appendAsFollower(messageWithOffset(1015, 1015, 11L))
+
+      val (nextDirtyOffset, _) = cleaner.clean(LogToClean(log.topicPartition, log, 0L, log.activeSegment.baseOffset, needCompactionNow = true))
+      assertEquals("Cleaning point should pass offset gap", log.activeSegment.baseOffset, nextDirtyOffset)
+    }
+
+
+    {
+      val log = makeLog(dir = TestUtils.randomPartitionLogDir(tmpdir), config = logConfig)
+      writeToLog(log, (0 to 9) zip (0 to 9), (0L to 9L))
+      // roll new segment with baseOffset 15, leaving previous with holes in offset rage [10, 14]
+      log.roll(Some(15L))
+
+      writeToLog(log, (15 to 24) zip (15 to 24), (15L to 24L))
+      // roll new segment with baseOffset 30, leaving previous with holes in offset range [25, 29]
+      log.roll(Some(30L))
+
+      // active segment record
+      log.appendAsFollower(messageWithOffset(1015, 1015, 30L))
+
+      val (nextDirtyOffset, _) = cleaner.clean(LogToClean(log.topicPartition, log, 0L, log.activeSegment.baseOffset, needCompactionNow = true))
+      assertEquals("Cleaning point should pass offset gap in multiple segments", log.activeSegment.baseOffset, nextDirtyOffset)
+    }
+  }
+
   private def writeToLog(log: Log, keysAndValues: Iterable[(Int, Int)], offsetSeq: Iterable[Long]): Iterable[Long] = {
     for(((key, value), offset) <- keysAndValues.zip(offsetSeq))
       yield log.appendAsFollower(messageWithOffset(key, value, offset)).lastOffset

From 044013ce3a15e0a8d19b4dbae981777e54fdb4ab Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Tue, 4 Feb 2020 17:01:08 +0000
Subject: [PATCH 0868/1071] KAFKA-9492; Ignore record errors in ProduceResponse
 for older versions (#8030)

Fixes NPE in brokers when processing record errors in produce response for older versions.

Reviewers: Chia-Ping Tsai , Ismael Juma , Jason Gustafson , Guozhang Wang 
---
 .../common/requests/ProduceResponse.java      |  24 ++--
 .../common/requests/ProduceResponseTest.java  | 115 ++++++++++++++++++
 .../common/requests/RequestResponseTest.java  |  51 --------
 3 files changed, 128 insertions(+), 62 deletions(-)
 create mode 100644 clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java

diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java
index 8bfc6298604be..7bbab0879bf42 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java
@@ -222,7 +222,8 @@ public ProduceResponse(Struct struct) {
                 int partition = partRespStruct.get(PARTITION_ID);
                 Errors error = Errors.forCode(partRespStruct.get(ERROR_CODE));
                 long offset = partRespStruct.getLong(BASE_OFFSET_KEY_NAME);
-                long logAppendTime = partRespStruct.getLong(LOG_APPEND_TIME_KEY_NAME);
+                long logAppendTime = partRespStruct.hasField(LOG_APPEND_TIME_KEY_NAME) ?
+                        partRespStruct.getLong(LOG_APPEND_TIME_KEY_NAME) : RecordBatch.NO_TIMESTAMP;
                 long logStartOffset = partRespStruct.getOrElse(LOG_START_OFFSET_FIELD, INVALID_OFFSET);
 
                 List recordErrors = Collections.emptyList();
@@ -274,19 +275,20 @@ protected Struct toStruct(short version) {
                 partStruct.setIfExists(LOG_APPEND_TIME_KEY_NAME, part.logAppendTime);
                 partStruct.setIfExists(LOG_START_OFFSET_FIELD, part.logStartOffset);
 
-                List recordErrors = Collections.emptyList();
-                if (!part.recordErrors.isEmpty()) {
-                    recordErrors = new ArrayList<>();
-                    for (RecordError indexAndMessage : part.recordErrors) {
-                        Struct indexAndMessageStruct = partStruct.instance(RECORD_ERRORS_KEY_NAME)
-                                .set(BATCH_INDEX_KEY_NAME, indexAndMessage.batchIndex)
-                                .set(BATCH_INDEX_ERROR_MESSAGE_FIELD, indexAndMessage.message);
-                        recordErrors.add(indexAndMessageStruct);
+                if (partStruct.hasField(RECORD_ERRORS_KEY_NAME)) {
+                    List recordErrors = Collections.emptyList();
+                    if (!part.recordErrors.isEmpty()) {
+                        recordErrors = new ArrayList<>();
+                        for (RecordError indexAndMessage : part.recordErrors) {
+                            Struct indexAndMessageStruct = partStruct.instance(RECORD_ERRORS_KEY_NAME)
+                                    .set(BATCH_INDEX_KEY_NAME, indexAndMessage.batchIndex)
+                                    .set(BATCH_INDEX_ERROR_MESSAGE_FIELD, indexAndMessage.message);
+                            recordErrors.add(indexAndMessageStruct);
+                        }
                     }
+                    partStruct.set(RECORD_ERRORS_KEY_NAME, recordErrors.toArray());
                 }
 
-                partStruct.setIfExists(RECORD_ERRORS_KEY_NAME, recordErrors.toArray());
-
                 partStruct.setIfExists(ERROR_MESSAGE_FIELD, part.errorMessage);
                 partitionArray.add(partStruct);
             }
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java
new file mode 100644
index 0000000000000..ea6e99882d200
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.common.requests;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.protocol.types.Struct;
+import org.apache.kafka.common.record.RecordBatch;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.kafka.common.protocol.ApiKeys.PRODUCE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class ProduceResponseTest {
+
+    @Test
+    public void produceResponseV5Test() {
+        Map responseData = new HashMap<>();
+        TopicPartition tp0 = new TopicPartition("test", 0);
+        responseData.put(tp0, new ProduceResponse.PartitionResponse(Errors.NONE,
+                10000, RecordBatch.NO_TIMESTAMP, 100));
+
+        ProduceResponse v5Response = new ProduceResponse(responseData, 10);
+        short version = 5;
+
+        ByteBuffer buffer = v5Response.serialize(ApiKeys.PRODUCE, version, 0);
+        buffer.rewind();
+
+        ResponseHeader.parse(buffer, ApiKeys.PRODUCE.responseHeaderVersion(version)); // throw away.
+
+        Struct deserializedStruct = ApiKeys.PRODUCE.parseResponse(version, buffer);
+
+        ProduceResponse v5FromBytes = (ProduceResponse) AbstractResponse.parseResponse(ApiKeys.PRODUCE,
+                deserializedStruct, version);
+
+        assertEquals(1, v5FromBytes.responses().size());
+        assertTrue(v5FromBytes.responses().containsKey(tp0));
+        ProduceResponse.PartitionResponse partitionResponse = v5FromBytes.responses().get(tp0);
+        assertEquals(100, partitionResponse.logStartOffset);
+        assertEquals(10000, partitionResponse.baseOffset);
+        assertEquals(10, v5FromBytes.throttleTimeMs());
+        assertEquals(responseData, v5Response.responses());
+    }
+
+    @Test
+    public void produceResponseVersionTest() {
+        Map responseData = new HashMap<>();
+        responseData.put(new TopicPartition("test", 0), new ProduceResponse.PartitionResponse(Errors.NONE,
+                10000, RecordBatch.NO_TIMESTAMP, 100));
+        ProduceResponse v0Response = new ProduceResponse(responseData);
+        ProduceResponse v1Response = new ProduceResponse(responseData, 10);
+        ProduceResponse v2Response = new ProduceResponse(responseData, 10);
+        assertEquals("Throttle time must be zero", 0, v0Response.throttleTimeMs());
+        assertEquals("Throttle time must be 10", 10, v1Response.throttleTimeMs());
+        assertEquals("Throttle time must be 10", 10, v2Response.throttleTimeMs());
+        assertEquals("Should use schema version 0", ApiKeys.PRODUCE.responseSchema((short) 0),
+                v0Response.toStruct((short) 0).schema());
+        assertEquals("Should use schema version 1", ApiKeys.PRODUCE.responseSchema((short) 1),
+                v1Response.toStruct((short) 1).schema());
+        assertEquals("Should use schema version 2", ApiKeys.PRODUCE.responseSchema((short) 2),
+                v2Response.toStruct((short) 2).schema());
+        assertEquals("Response data does not match", responseData, v0Response.responses());
+        assertEquals("Response data does not match", responseData, v1Response.responses());
+        assertEquals("Response data does not match", responseData, v2Response.responses());
+    }
+
+    @Test
+    public void produceResponseRecordErrorsTest() {
+        Map responseData = new HashMap<>();
+        TopicPartition tp = new TopicPartition("test", 0);
+        ProduceResponse.PartitionResponse partResponse = new ProduceResponse.PartitionResponse(Errors.NONE,
+                10000, RecordBatch.NO_TIMESTAMP, 100,
+                Collections.singletonList(new ProduceResponse.RecordError(3, "Record error")),
+                "Produce failed");
+        responseData.put(tp, partResponse);
+
+        for (short ver = 0; ver <= PRODUCE.latestVersion(); ver++) {
+            ProduceResponse response = new ProduceResponse(responseData);
+            Struct struct = response.toStruct(ver);
+            assertEquals("Should use schema version " + ver, ApiKeys.PRODUCE.responseSchema(ver), struct.schema());
+            ProduceResponse.PartitionResponse deserialized = new ProduceResponse(struct).responses().get(tp);
+            if (ver >= 8) {
+                assertEquals(1, deserialized.recordErrors.size());
+                assertEquals(3, deserialized.recordErrors.get(0).batchIndex);
+                assertEquals("Record error", deserialized.recordErrors.get(0).message);
+                assertEquals("Produce failed", deserialized.errorMessage);
+            } else {
+                assertEquals(0, deserialized.recordErrors.size());
+                assertEquals(null, deserialized.errorMessage);
+            }
+        }
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index 5934dc5ec7584..7e8410a7a2b08 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -563,57 +563,6 @@ public void produceRequestGetErrorResponseTest() {
         assertEquals(RecordBatch.NO_TIMESTAMP, partitionResponse.logAppendTime);
     }
 
-    @Test
-    public void produceResponseV5Test() {
-        Map responseData = new HashMap<>();
-        TopicPartition tp0 = new TopicPartition("test", 0);
-        responseData.put(tp0, new ProduceResponse.PartitionResponse(Errors.NONE,
-                10000, RecordBatch.NO_TIMESTAMP, 100));
-
-        ProduceResponse v5Response = new ProduceResponse(responseData, 10);
-        short version = 5;
-
-        ByteBuffer buffer = v5Response.serialize(ApiKeys.PRODUCE, version, 0);
-        buffer.rewind();
-
-        ResponseHeader.parse(buffer, ApiKeys.PRODUCE.responseHeaderVersion(version)); // throw away.
-
-        Struct deserializedStruct = ApiKeys.PRODUCE.parseResponse(version, buffer);
-
-        ProduceResponse v5FromBytes = (ProduceResponse) AbstractResponse.parseResponse(ApiKeys.PRODUCE,
-                deserializedStruct, version);
-
-        assertEquals(1, v5FromBytes.responses().size());
-        assertTrue(v5FromBytes.responses().containsKey(tp0));
-        ProduceResponse.PartitionResponse partitionResponse = v5FromBytes.responses().get(tp0);
-        assertEquals(100, partitionResponse.logStartOffset);
-        assertEquals(10000, partitionResponse.baseOffset);
-        assertEquals(10, v5FromBytes.throttleTimeMs());
-        assertEquals(responseData, v5Response.responses());
-    }
-
-    @Test
-    public void produceResponseVersionTest() {
-        Map responseData = new HashMap<>();
-        responseData.put(new TopicPartition("test", 0), new ProduceResponse.PartitionResponse(Errors.NONE,
-                10000, RecordBatch.NO_TIMESTAMP, 100));
-        ProduceResponse v0Response = new ProduceResponse(responseData);
-        ProduceResponse v1Response = new ProduceResponse(responseData, 10);
-        ProduceResponse v2Response = new ProduceResponse(responseData, 10);
-        assertEquals("Throttle time must be zero", 0, v0Response.throttleTimeMs());
-        assertEquals("Throttle time must be 10", 10, v1Response.throttleTimeMs());
-        assertEquals("Throttle time must be 10", 10, v2Response.throttleTimeMs());
-        assertEquals("Should use schema version 0", ApiKeys.PRODUCE.responseSchema((short) 0),
-                v0Response.toStruct((short) 0).schema());
-        assertEquals("Should use schema version 1", ApiKeys.PRODUCE.responseSchema((short) 1),
-                v1Response.toStruct((short) 1).schema());
-        assertEquals("Should use schema version 2", ApiKeys.PRODUCE.responseSchema((short) 2),
-                v2Response.toStruct((short) 2).schema());
-        assertEquals("Response data does not match", responseData, v0Response.responses());
-        assertEquals("Response data does not match", responseData, v1Response.responses());
-        assertEquals("Response data does not match", responseData, v2Response.responses());
-    }
-
     @Test
     public void fetchResponseVersionTest() {
         LinkedHashMap> responseData = new LinkedHashMap<>();

From 4491a0dbb8fb183c21343b3d7145574b9c728485 Mon Sep 17 00:00:00 2001
From: Randall Hauch 
Date: Tue, 4 Feb 2020 11:15:04 -0600
Subject: [PATCH 0869/1071] =?UTF-8?q?KAFKA-9074:=20Correct=20Connect?=
 =?UTF-8?q?=E2=80=99s=20`Values.parseString`=20to=20properly=20parse=20a?=
 =?UTF-8?q?=20time=20and=20timestamp=20literal=20(#7568)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* KAFKA-9074: Correct Connect’s `Values.parseString` to properly parse a time and timestamp literal

Time and timestamp literal strings contain a `:` character, but the internal parser used in the `Values.parseString(String)` method tokenizes on the colon character to tokenize and parse map entries. The colon could be escaped, but then the backslash character used to escape the colon is not removed and the parser fails to match the literal as a time or timestamp value.

This fix corrects the parsing logic to properly parse timestamp and time literal strings whose colon characters are either escaped or unescaped. Additional unit tests were added to first verify the incorrect behavior and then to validate the correction.

Author: Randall Hauch 
Reviewers: Chris Egerton , Nigel Liang , Jason Gustafson 
---
 .../org/apache/kafka/connect/data/Values.java | 133 +++++++++++----
 .../apache/kafka/connect/data/ValuesTest.java | 155 ++++++++++++++++++
 2 files changed, 257 insertions(+), 31 deletions(-)

diff --git a/connect/api/src/main/java/org/apache/kafka/connect/data/Values.java b/connect/api/src/main/java/org/apache/kafka/connect/data/Values.java
index 93c320a236760..d99fbcabf86df 100644
--- a/connect/api/src/main/java/org/apache/kafka/connect/data/Values.java
+++ b/connect/api/src/main/java/org/apache/kafka/connect/data/Values.java
@@ -30,13 +30,17 @@
 import java.text.SimpleDateFormat;
 import java.text.StringCharacterIterator;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Base64;
 import java.util.Calendar;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
+import java.util.Set;
 import java.util.TimeZone;
 import java.util.regex.Pattern;
 
@@ -70,9 +74,18 @@ public class Values {
     private static final String FALSE_LITERAL = Boolean.FALSE.toString();
     private static final long MILLIS_PER_DAY = 24 * 60 * 60 * 1000;
     private static final String NULL_VALUE = "null";
-    private static final String ISO_8601_DATE_FORMAT_PATTERN = "yyyy-MM-dd";
-    private static final String ISO_8601_TIME_FORMAT_PATTERN = "HH:mm:ss.SSS'Z'";
-    private static final String ISO_8601_TIMESTAMP_FORMAT_PATTERN = ISO_8601_DATE_FORMAT_PATTERN + "'T'" + ISO_8601_TIME_FORMAT_PATTERN;
+    static final String ISO_8601_DATE_FORMAT_PATTERN = "yyyy-MM-dd";
+    static final String ISO_8601_TIME_FORMAT_PATTERN = "HH:mm:ss.SSS'Z'";
+    static final String ISO_8601_TIMESTAMP_FORMAT_PATTERN = ISO_8601_DATE_FORMAT_PATTERN + "'T'" + ISO_8601_TIME_FORMAT_PATTERN;
+    private static final Set TEMPORAL_LOGICAL_TYPE_NAMES =
+            Collections.unmodifiableSet(
+                    new HashSet<>(
+                            Arrays.asList(Time.LOGICAL_NAME,
+                            Timestamp.LOGICAL_NAME,
+                            Date.LOGICAL_NAME
+                            )
+                    )
+            );
 
     private static final String QUOTE_DELIMITER = "\"";
     private static final String COMMA_DELIMITER = ",";
@@ -467,6 +480,9 @@ protected static Object convertTo(Schema toSchema, Schema fromSchema, Object val
                                 int days = (int) (millis / MILLIS_PER_DAY); // truncates
                                 return Date.toLogical(toSchema, days);
                             }
+                        } else {
+                            // There is no fromSchema, so no conversion is needed
+                            return value;
                         }
                     }
                     long numeric = asLong(value, fromSchema, null);
@@ -492,6 +508,9 @@ protected static Object convertTo(Schema toSchema, Schema fromSchema, Object val
                                 calendar.set(Calendar.DAY_OF_MONTH, 1);
                                 return Time.toLogical(toSchema, (int) calendar.getTimeInMillis());
                             }
+                        } else {
+                            // There is no fromSchema, so no conversion is needed
+                            return value;
                         }
                     }
                     long numeric = asLong(value, fromSchema, null);
@@ -523,6 +542,9 @@ protected static Object convertTo(Schema toSchema, Schema fromSchema, Object val
                             if (Timestamp.LOGICAL_NAME.equals(fromSchemaName)) {
                                 return value;
                             }
+                        } else {
+                            // There is no fromSchema, so no conversion is needed
+                            return value;
                         }
                     }
                     long numeric = asLong(value, fromSchema, null);
@@ -755,7 +777,14 @@ protected static SchemaAndValue parse(Parser parser, boolean embedded) throws No
                     }
                     sb.append(parser.next());
                 }
-                return new SchemaAndValue(Schema.STRING_SCHEMA, sb.toString());
+                String content = sb.toString();
+                // We can parse string literals as temporal logical types, but all others
+                // are treated as strings
+                SchemaAndValue parsed = parseString(content);
+                if (parsed != null && TEMPORAL_LOGICAL_TYPE_NAMES.contains(parsed.schema().name())) {
+                    return parsed;
+                }
+                return new SchemaAndValue(Schema.STRING_SCHEMA, content);
             }
         }
 
@@ -838,7 +867,9 @@ protected static SchemaAndValue parse(Parser parser, boolean embedded) throws No
                     }
 
                     if (!parser.canConsume(ENTRY_DELIMITER)) {
-                        throw new DataException("Map entry is missing '=': " + parser.original());
+                        throw new DataException("Map entry is missing '" + ENTRY_DELIMITER
+                                                + "' at " + parser.position()
+                                                + " in " + parser.original());
                     }
                     SchemaAndValue value = parse(parser, true);
                     Object entryValue = value != null ? value.value() : null;
@@ -855,7 +886,7 @@ protected static SchemaAndValue parse(Parser parser, boolean embedded) throws No
                 throw new DataException("Map is missing terminating '}': " + parser.original());
             }
         } catch (DataException e) {
-            LOG.debug("Unable to parse the value as a map or an array; reverting to string", e);
+            LOG.trace("Unable to parse the value as a map or an array; reverting to string", e);
             parser.rewindTo(startPosition);
         }
 
@@ -867,6 +898,27 @@ protected static SchemaAndValue parse(Parser parser, boolean embedded) throws No
 
         char firstChar = token.charAt(0);
         boolean firstCharIsDigit = Character.isDigit(firstChar);
+
+        // Temporal types are more restrictive, so try them first
+        if (firstCharIsDigit) {
+            // The time and timestamp literals may be split into 5 tokens since an unescaped colon
+            // is a delimiter. Check these first since the first of these tokens is a simple numeric
+            int position = parser.mark();
+            String remainder = parser.next(4);
+            if (remainder != null) {
+                String timeOrTimestampStr = token + remainder;
+                SchemaAndValue temporal = parseAsTemporal(timeOrTimestampStr);
+                if (temporal != null) {
+                    return temporal;
+                }
+            }
+            // No match was found using the 5 tokens, so rewind and see if the current token has a date, time, or timestamp
+            parser.rewindTo(position);
+            SchemaAndValue temporal = parseAsTemporal(token);
+            if (temporal != null) {
+                return temporal;
+            }
+        }
         if (firstCharIsDigit || firstChar == '+' || firstChar == '-') {
             try {
                 // Try to parse as a number ...
@@ -901,37 +953,42 @@ protected static SchemaAndValue parse(Parser parser, boolean embedded) throws No
                 // can't parse as a number
             }
         }
-        if (firstCharIsDigit) {
-            // Check for a date, time, or timestamp ...
-            int tokenLength = token.length();
-            if (tokenLength == ISO_8601_DATE_LENGTH) {
-                try {
-                    return new SchemaAndValue(Date.SCHEMA, new SimpleDateFormat(ISO_8601_DATE_FORMAT_PATTERN).parse(token));
-                } catch (ParseException e) {
-                    // not a valid date
-                }
-            } else if (tokenLength == ISO_8601_TIME_LENGTH) {
-                try {
-                    return new SchemaAndValue(Time.SCHEMA, new SimpleDateFormat(ISO_8601_TIME_FORMAT_PATTERN).parse(token));
-                } catch (ParseException e) {
-                    // not a valid date
-                }
-            } else if (tokenLength == ISO_8601_TIMESTAMP_LENGTH) {
-                try {
-                    return new SchemaAndValue(Timestamp.SCHEMA, new SimpleDateFormat(ISO_8601_TIMESTAMP_FORMAT_PATTERN).parse(token));
-                } catch (ParseException e) {
-                    // not a valid date
-                }
-            }
-        }
         if (embedded) {
             throw new DataException("Failed to parse embedded value");
         }
-        // At this point, the only thing this can be is a string. Embedded strings were processed above,
-        // so this is not embedded and we can use the original string...
+        // At this point, the only thing this non-embedded value can be is a string.
         return new SchemaAndValue(Schema.STRING_SCHEMA, parser.original());
     }
 
+    private static SchemaAndValue parseAsTemporal(String token) {
+        if (token == null) {
+            return null;
+        }
+        // If the colons were escaped, we'll see the escape chars and need to remove them
+        token = token.replace("\\:", ":");
+        int tokenLength = token.length();
+        if (tokenLength == ISO_8601_TIME_LENGTH) {
+            try {
+                return new SchemaAndValue(Time.SCHEMA, new SimpleDateFormat(ISO_8601_TIME_FORMAT_PATTERN).parse(token));
+            } catch (ParseException e) {
+              // not a valid date
+            }
+        } else if (tokenLength == ISO_8601_TIMESTAMP_LENGTH) {
+            try {
+                return new SchemaAndValue(Timestamp.SCHEMA, new SimpleDateFormat(ISO_8601_TIMESTAMP_FORMAT_PATTERN).parse(token));
+            } catch (ParseException e) {
+              // not a valid date
+            }
+        } else if (tokenLength == ISO_8601_DATE_LENGTH) {
+            try {
+                return new SchemaAndValue(Date.SCHEMA, new SimpleDateFormat(ISO_8601_DATE_FORMAT_PATTERN).parse(token));
+            } catch (ParseException e) {
+                // not a valid date
+            }
+        }
+        return null;
+    }
+
     protected static Schema commonSchemaFor(Schema previous, SchemaAndValue latest) {
         if (latest == null) {
             return previous;
@@ -1088,6 +1145,7 @@ public int mark() {
         public void rewindTo(int position) {
             iter.setIndex(position);
             nextToken = null;
+            previousToken = null;
         }
 
         public String original() {
@@ -1112,6 +1170,19 @@ public String next() {
             return previousToken;
         }
 
+        public String next(int n) {
+            int current = mark();
+            int start = mark();
+            for (int i = 0; i != n; ++i) {
+                if (!hasNext()) {
+                    rewindTo(start);
+                    return null;
+                }
+                next();
+            }
+            return original.substring(current, position());
+        }
+
         private String consumeNextToken() throws NoSuchElementException {
             boolean escaped = false;
             int start = iter.getIndex();
diff --git a/connect/api/src/test/java/org/apache/kafka/connect/data/ValuesTest.java b/connect/api/src/test/java/org/apache/kafka/connect/data/ValuesTest.java
index a5909f3ece9bf..c437e46c25956 100644
--- a/connect/api/src/test/java/org/apache/kafka/connect/data/ValuesTest.java
+++ b/connect/api/src/test/java/org/apache/kafka/connect/data/ValuesTest.java
@@ -21,6 +21,7 @@
 import org.apache.kafka.connect.errors.DataException;
 import org.junit.Test;
 
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -383,6 +384,146 @@ public void shouldParseStringListWithExtraDelimitersAndReturnString() {
         assertEquals(str, result.value());
     }
 
+    @Test
+    public void shouldParseTimestampStringAsTimestamp() throws Exception {
+        String str = "2019-08-23T14:34:54.346Z";
+        SchemaAndValue result = Values.parseString(str);
+        assertEquals(Type.INT64, result.schema().type());
+        assertEquals(Timestamp.LOGICAL_NAME, result.schema().name());
+        java.util.Date expected = new SimpleDateFormat(Values.ISO_8601_TIMESTAMP_FORMAT_PATTERN).parse(str);
+        assertEquals(expected, result.value());
+    }
+
+    @Test
+    public void shouldParseDateStringAsDate() throws Exception {
+        String str = "2019-08-23";
+        SchemaAndValue result = Values.parseString(str);
+        assertEquals(Type.INT32, result.schema().type());
+        assertEquals(Date.LOGICAL_NAME, result.schema().name());
+        java.util.Date expected = new SimpleDateFormat(Values.ISO_8601_DATE_FORMAT_PATTERN).parse(str);
+        assertEquals(expected, result.value());
+    }
+
+    @Test
+    public void shouldParseTimeStringAsDate() throws Exception {
+        String str = "14:34:54.346Z";
+        SchemaAndValue result = Values.parseString(str);
+        assertEquals(Type.INT32, result.schema().type());
+        assertEquals(Time.LOGICAL_NAME, result.schema().name());
+        java.util.Date expected = new SimpleDateFormat(Values.ISO_8601_TIME_FORMAT_PATTERN).parse(str);
+        assertEquals(expected, result.value());
+    }
+
+    @Test
+    public void shouldParseTimestampStringWithEscapedColonsAsTimestamp() throws Exception {
+        String str = "2019-08-23T14\\:34\\:54.346Z";
+        SchemaAndValue result = Values.parseString(str);
+        assertEquals(Type.INT64, result.schema().type());
+        assertEquals(Timestamp.LOGICAL_NAME, result.schema().name());
+        String expectedStr = "2019-08-23T14:34:54.346Z";
+        java.util.Date expected = new SimpleDateFormat(Values.ISO_8601_TIMESTAMP_FORMAT_PATTERN).parse(expectedStr);
+        assertEquals(expected, result.value());
+    }
+
+    @Test
+    public void shouldParseTimeStringWithEscapedColonsAsDate() throws Exception {
+        String str = "14\\:34\\:54.346Z";
+        SchemaAndValue result = Values.parseString(str);
+        assertEquals(Type.INT32, result.schema().type());
+        assertEquals(Time.LOGICAL_NAME, result.schema().name());
+        String expectedStr = "14:34:54.346Z";
+        java.util.Date expected = new SimpleDateFormat(Values.ISO_8601_TIME_FORMAT_PATTERN).parse(expectedStr);
+        assertEquals(expected, result.value());
+    }
+
+    @Test
+    public void shouldParseDateStringAsDateInArray() throws Exception {
+        String dateStr = "2019-08-23";
+        String arrayStr = "[" + dateStr + "]";
+        SchemaAndValue result = Values.parseString(arrayStr);
+        assertEquals(Type.ARRAY, result.schema().type());
+        Schema elementSchema = result.schema().valueSchema();
+        assertEquals(Type.INT32, elementSchema.type());
+        assertEquals(Date.LOGICAL_NAME, elementSchema.name());
+        java.util.Date expected = new SimpleDateFormat(Values.ISO_8601_DATE_FORMAT_PATTERN).parse(dateStr);
+        assertEquals(Collections.singletonList(expected), result.value());
+    }
+
+    @Test
+    public void shouldParseTimeStringAsTimeInArray() throws Exception {
+        String timeStr = "14:34:54.346Z";
+        String arrayStr = "[" + timeStr + "]";
+        SchemaAndValue result = Values.parseString(arrayStr);
+        assertEquals(Type.ARRAY, result.schema().type());
+        Schema elementSchema = result.schema().valueSchema();
+        assertEquals(Type.INT32, elementSchema.type());
+        assertEquals(Time.LOGICAL_NAME, elementSchema.name());
+        java.util.Date expected = new SimpleDateFormat(Values.ISO_8601_TIME_FORMAT_PATTERN).parse(timeStr);
+        assertEquals(Collections.singletonList(expected), result.value());
+    }
+
+    @Test
+    public void shouldParseTimestampStringAsTimestampInArray() throws Exception {
+        String tsStr = "2019-08-23T14:34:54.346Z";
+        String arrayStr = "[" + tsStr + "]";
+        SchemaAndValue result = Values.parseString(arrayStr);
+        assertEquals(Type.ARRAY, result.schema().type());
+        Schema elementSchema = result.schema().valueSchema();
+        assertEquals(Type.INT64, elementSchema.type());
+        assertEquals(Timestamp.LOGICAL_NAME, elementSchema.name());
+        java.util.Date expected = new SimpleDateFormat(Values.ISO_8601_TIMESTAMP_FORMAT_PATTERN).parse(tsStr);
+        assertEquals(Collections.singletonList(expected), result.value());
+    }
+
+    @Test
+    public void shouldParseMultipleTimestampStringAsTimestampInArray() throws Exception {
+        String tsStr1 = "2019-08-23T14:34:54.346Z";
+        String tsStr2 = "2019-01-23T15:12:34.567Z";
+        String tsStr3 = "2019-04-23T19:12:34.567Z";
+        String arrayStr = "[" + tsStr1 + "," + tsStr2 + ",   " + tsStr3 + "]";
+        SchemaAndValue result = Values.parseString(arrayStr);
+        assertEquals(Type.ARRAY, result.schema().type());
+        Schema elementSchema = result.schema().valueSchema();
+        assertEquals(Type.INT64, elementSchema.type());
+        assertEquals(Timestamp.LOGICAL_NAME, elementSchema.name());
+        java.util.Date expected1 = new SimpleDateFormat(Values.ISO_8601_TIMESTAMP_FORMAT_PATTERN).parse(tsStr1);
+        java.util.Date expected2 = new SimpleDateFormat(Values.ISO_8601_TIMESTAMP_FORMAT_PATTERN).parse(tsStr2);
+        java.util.Date expected3 = new SimpleDateFormat(Values.ISO_8601_TIMESTAMP_FORMAT_PATTERN).parse(tsStr3);
+        assertEquals(Arrays.asList(expected1, expected2, expected3), result.value());
+    }
+
+    @Test
+    public void shouldParseQuotedTimeStringAsTimeInMap() throws Exception {
+        String keyStr = "k1";
+        String timeStr = "14:34:54.346Z";
+        String mapStr = "{\"" + keyStr + "\":\"" + timeStr + "\"}";
+        SchemaAndValue result = Values.parseString(mapStr);
+        assertEquals(Type.MAP, result.schema().type());
+        Schema keySchema = result.schema().keySchema();
+        Schema valueSchema = result.schema().valueSchema();
+        assertEquals(Type.STRING, keySchema.type());
+        assertEquals(Type.INT32, valueSchema.type());
+        assertEquals(Time.LOGICAL_NAME, valueSchema.name());
+        java.util.Date expected = new SimpleDateFormat(Values.ISO_8601_TIME_FORMAT_PATTERN).parse(timeStr);
+        assertEquals(Collections.singletonMap(keyStr, expected), result.value());
+    }
+
+    @Test
+    public void shouldParseTimeStringAsTimeInMap() throws Exception {
+        String keyStr = "k1";
+        String timeStr = "14:34:54.346Z";
+        String mapStr = "{\"" + keyStr + "\":" + timeStr + "}";
+        SchemaAndValue result = Values.parseString(mapStr);
+        assertEquals(Type.MAP, result.schema().type());
+        Schema keySchema = result.schema().keySchema();
+        Schema valueSchema = result.schema().valueSchema();
+        assertEquals(Type.STRING, keySchema.type());
+        assertEquals(Type.INT32, valueSchema.type());
+        assertEquals(Time.LOGICAL_NAME, valueSchema.name());
+        java.util.Date expected = new SimpleDateFormat(Values.ISO_8601_TIME_FORMAT_PATTERN).parse(timeStr);
+        assertEquals(Collections.singletonMap(keyStr, expected), result.value());
+    }
+
     /**
      * This is technically invalid JSON, and we don't want to simply ignore the blank elements.
      */
@@ -432,6 +573,20 @@ public void shouldFailToParseStringOfMapWithIntValuesWithBlankEntries() {
         Values.convertToMap(Schema.STRING_SCHEMA, " { \"foo\" :  \"1234567890\" ,, \"bar\" : \"0\",  \"baz\" : \"boz\" }  ");
     }
 
+    @Test
+    public void shouldConsumeMultipleTokens() {
+        String value = "a:b:c:d:e:f:g:h";
+        Parser parser = new Parser(value);
+        String firstFive = parser.next(5);
+        assertEquals("a:b:c", firstFive);
+        assertEquals(":", parser.next());
+        assertEquals("d", parser.next());
+        assertEquals(":", parser.next());
+        String lastEight = parser.next(8); // only 7 remain
+        assertNull(lastEight);
+        assertEquals("e", parser.next());
+    }
+
     @Test
     public void shouldParseStringsWithoutDelimiters() {
         //assertParsed("");

From a692a18e6b830846ec9b151488842cde0a50e45e Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Tue, 4 Feb 2020 11:22:33 -0800
Subject: [PATCH 0870/1071] KAFKA-9491; Increment high watermark after full log
 truncation (#8037)

When a follower's fetch offset is behind the leader's log start offset, the
follower will do a full log truncation. When it does so, it must update both
its log start offset and high watermark. The previous code did the former,
but not the latter. Failure to update the high watermark in this case can lead
to out of range errors if the follower becomes leader before getting the latest
high watermark from the previous leader. The out of range errors occur when
we attempt to resolve the log position of the high watermark in DelayedFetch
in order to determine if a fetch is satisfied.

Reviewers: Rajini Sivaram , Chia-Ping Tsai , Ismael Juma 
---
 .../main/scala/kafka/cluster/Partition.scala  |  8 ++---
 core/src/main/scala/kafka/log/Log.scala       | 33 +++++++++++++------
 .../test/scala/unit/kafka/log/LogTest.scala   | 20 ++++++++---
 3 files changed, 43 insertions(+), 18 deletions(-)

diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala
index a8796eb8080a9..7ca7965647a1b 100755
--- a/core/src/main/scala/kafka/cluster/Partition.scala
+++ b/core/src/main/scala/kafka/cluster/Partition.scala
@@ -506,8 +506,9 @@ class Partition(val topicPartition: TopicPartition,
 
       val leaderLog = localLogOrException
       val leaderEpochStartOffset = leaderLog.logEndOffset
-      info(s"$topicPartition starts at Leader Epoch ${partitionState.leaderEpoch} from " +
-        s"offset $leaderEpochStartOffset. Previous Leader Epoch was: $leaderEpoch")
+      info(s"$topicPartition starts at leader epoch ${partitionState.leaderEpoch} from " +
+        s"offset $leaderEpochStartOffset with high watermark ${leaderLog.highWatermark}. " +
+        s"Previous leader epoch was $leaderEpoch.")
 
       //We cache the leader epoch here, persisting it only if it's local (hence having a log dir)
       leaderEpoch = partitionState.leaderEpoch
@@ -522,12 +523,11 @@ class Partition(val topicPartition: TopicPartition,
       leaderLog.maybeAssignEpochStartOffset(leaderEpoch, leaderEpochStartOffset)
 
       val isNewLeader = !isLeader
-      val curLeaderLogEndOffset = leaderLog.logEndOffset
       val curTimeMs = time.milliseconds
       // initialize lastCaughtUpTime of replicas as well as their lastFetchTimeMs and lastFetchLeaderLogEndOffset.
       remoteReplicas.foreach { replica =>
         val lastCaughtUpTimeMs = if (inSyncReplicaIds.contains(replica.brokerId)) curTimeMs else 0L
-        replica.resetLastCaughtUpTime(curLeaderLogEndOffset, curTimeMs, lastCaughtUpTimeMs)
+        replica.resetLastCaughtUpTime(leaderEpochStartOffset, curTimeMs, lastCaughtUpTimeMs)
       }
 
       if (isNewLeader) {
diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index c2cd2e2ba6535..634f2163213da 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -302,7 +302,7 @@ class Log(@volatile var dir: File,
 
     leaderEpochCache.foreach(_.truncateFromEnd(nextOffsetMetadata.messageOffset))
 
-    logStartOffset = math.max(logStartOffset, segments.firstEntry.getValue.baseOffset)
+    updateLogStartOffset(math.max(logStartOffset, segments.firstEntry.getValue.baseOffset))
 
     // The earliest leader epoch may not be flushed during a hard failure. Recover it here.
     leaderEpochCache.foreach(_.truncateFromStart(logStartOffset))
@@ -741,14 +741,30 @@ class Log(@volatile var dir: File,
     }
   }
 
-  private def updateLogEndOffset(messageOffset: Long): Unit = {
-    nextOffsetMetadata = LogOffsetMetadata(messageOffset, activeSegment.baseOffset, activeSegment.size)
+  private def updateLogEndOffset(offset: Long): Unit = {
+    nextOffsetMetadata = LogOffsetMetadata(offset, activeSegment.baseOffset, activeSegment.size)
 
     // Update the high watermark in case it has gotten ahead of the log end offset following a truncation
     // or if a new segment has been rolled and the offset metadata needs to be updated.
-    if (highWatermark >= messageOffset) {
+    if (highWatermark >= offset) {
       updateHighWatermarkMetadata(nextOffsetMetadata)
     }
+
+    if (this.recoveryPoint > offset) {
+      this.recoveryPoint = offset
+    }
+  }
+
+  private def updateLogStartOffset(offset: Long): Unit = {
+    logStartOffset = offset
+
+    if (highWatermark < offset) {
+      updateHighWatermark(offset)
+    }
+
+    if (this.recoveryPoint < offset) {
+      this.recoveryPoint = offset
+    }
   }
 
   /**
@@ -1262,7 +1278,7 @@ class Log(@volatile var dir: File,
         checkIfMemoryMappedBufferClosed()
         if (newLogStartOffset > logStartOffset) {
           info(s"Incrementing log start offset to $newLogStartOffset")
-          logStartOffset = newLogStartOffset
+          updateLogStartOffset(newLogStartOffset)
           leaderEpochCache.foreach(_.truncateFromStart(logStartOffset))
           producerStateManager.truncateHead(newLogStartOffset)
           maybeIncrementFirstUnstableOffset()
@@ -2066,8 +2082,7 @@ class Log(@volatile var dir: File,
             removeAndDeleteSegments(deletable, asyncDelete = true)
             activeSegment.truncateTo(targetOffset)
             updateLogEndOffset(targetOffset)
-            this.recoveryPoint = math.min(targetOffset, this.recoveryPoint)
-            this.logStartOffset = math.min(targetOffset, this.logStartOffset)
+            updateLogStartOffset(math.min(targetOffset, this.logStartOffset))
             leaderEpochCache.foreach(_.truncateFromEnd(targetOffset))
             loadProducerState(targetOffset, reloadFromCleanShutdown = false)
           }
@@ -2101,9 +2116,7 @@ class Log(@volatile var dir: File,
         producerStateManager.truncate()
         producerStateManager.updateMapEndOffset(newOffset)
         maybeIncrementFirstUnstableOffset()
-
-        this.recoveryPoint = math.min(newOffset, this.recoveryPoint)
-        this.logStartOffset = newOffset
+        updateLogStartOffset(newOffset)
       }
     }
   }
diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala
index 8eaf47c3555a3..1e2e047ee6358 100755
--- a/core/src/test/scala/unit/kafka/log/LogTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogTest.scala
@@ -117,12 +117,13 @@ class LogTest {
   def testHighWatermarkMaintenance(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 1024 * 1024)
     val log = createLog(logDir, logConfig)
+    val leaderEpoch = 0
 
-    val records = TestUtils.records(List(
+    def records(offset: Long): MemoryRecords = TestUtils.records(List(
       new SimpleRecord(mockTime.milliseconds, "a".getBytes, "value".getBytes),
       new SimpleRecord(mockTime.milliseconds, "b".getBytes, "value".getBytes),
       new SimpleRecord(mockTime.milliseconds, "c".getBytes, "value".getBytes)
-    ))
+    ), baseOffset = offset, partitionLeaderEpoch= leaderEpoch)
 
     def assertHighWatermark(offset: Long): Unit = {
       assertEquals(offset, log.highWatermark)
@@ -133,7 +134,7 @@ class LogTest {
     assertHighWatermark(0L)
 
     // High watermark not changed by append
-    log.appendAsLeader(records, leaderEpoch = 0)
+    log.appendAsLeader(records(0), leaderEpoch)
     assertHighWatermark(0L)
 
     // Update high watermark as leader
@@ -145,13 +146,24 @@ class LogTest {
     assertHighWatermark(3L)
 
     // Update high watermark as follower
-    log.appendAsLeader(records, leaderEpoch = 0)
+    log.appendAsFollower(records(3L))
     log.updateHighWatermark(6L)
     assertHighWatermark(6L)
 
     // High watermark should be adjusted by truncation
     log.truncateTo(3L)
     assertHighWatermark(3L)
+
+    log.appendAsLeader(records(0L), leaderEpoch = 0)
+    assertHighWatermark(3L)
+    assertEquals(6L, log.logEndOffset)
+    assertEquals(0L, log.logStartOffset)
+
+    // Full truncation should also reset high watermark
+    log.truncateFullyAndStartAt(4L)
+    assertEquals(4L, log.logEndOffset)
+    assertEquals(4L, log.logStartOffset)
+    assertHighWatermark(4L)
   }
 
   private def assertNonEmptyFetch(log: Log, offset: Long, isolation: FetchIsolation): Unit = {

From 2e00ecfe9fa743ddf98b80d9362970de7a275ec4 Mon Sep 17 00:00:00 2001
From: David Mao <47232755+splett2@users.noreply.github.com>
Date: Fri, 7 Feb 2020 16:43:52 -0800
Subject: [PATCH 0871/1071] KAFKA-9507; AdminClient should check for missing
 committed offsets (#8057)

Addresses exception being thrown by `AdminClient` when `listConsumerGroupOffsets` returns a negative offset. A negative offset indicates the absence of a committed offset for a requested partition, and should result in a null in the returned offset map.

Reviewers: Anna Povzner , Jason Gustafson 
---
 .../org/apache/kafka/clients/admin/KafkaAdminClient.java   | 7 ++++++-
 .../clients/admin/ListConsumerGroupOffsetsResult.java      | 1 +
 .../apache/kafka/clients/admin/KafkaAdminClientTest.java   | 7 ++++++-
 3 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
index 344cef8bde4d3..c201fe65c224b 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
@@ -3019,7 +3019,12 @@ void handleResponse(AbstractResponse abstractResponse) {
                         final Long offset = partitionData.offset;
                         final String metadata = partitionData.metadata;
                         final Optional leaderEpoch = partitionData.leaderEpoch;
-                        groupOffsetsListing.put(topicPartition, new OffsetAndMetadata(offset, leaderEpoch, metadata));
+                        // Negative offset indicates that the group has no committed offset for this partition
+                        if (offset < 0) {
+                            groupOffsetsListing.put(topicPartition, null);
+                        } else {
+                            groupOffsetsListing.put(topicPartition, new OffsetAndMetadata(offset, leaderEpoch, metadata));
+                        }
                     } else {
                         log.warn("Skipping return offset for {} due to error {}.", topicPartition, error);
                     }
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupOffsetsResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupOffsetsResult.java
index ea5193402bd91..48f4531418110 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupOffsetsResult.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupOffsetsResult.java
@@ -40,6 +40,7 @@ public class ListConsumerGroupOffsetsResult {
 
     /**
      * Return a future which yields a map of topic partitions to OffsetAndMetadata objects.
+     * If the group does not have a committed offset for this partition, the corresponding value in the returned map will be null.
      */
     public KafkaFuture> partitionsToOffsetAndMetadata() {
         return future;
diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
index a274d87d0551c..4105f802c50a0 100644
--- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
@@ -1547,6 +1547,7 @@ public void testDescribeConsumerGroupOffsets() throws Exception {
             TopicPartition myTopicPartition0 = new TopicPartition("my_topic", 0);
             TopicPartition myTopicPartition1 = new TopicPartition("my_topic", 1);
             TopicPartition myTopicPartition2 = new TopicPartition("my_topic", 2);
+            TopicPartition myTopicPartition3 = new TopicPartition("my_topic", 3);
 
             final Map responseData = new HashMap<>();
             responseData.put(myTopicPartition0, new OffsetFetchResponse.PartitionData(10,
@@ -1555,15 +1556,19 @@ public void testDescribeConsumerGroupOffsets() throws Exception {
                     Optional.empty(), "", Errors.NONE));
             responseData.put(myTopicPartition2, new OffsetFetchResponse.PartitionData(20,
                     Optional.empty(), "", Errors.NONE));
+            responseData.put(myTopicPartition3, new OffsetFetchResponse.PartitionData(OffsetFetchResponse.INVALID_OFFSET,
+                    Optional.empty(), "", Errors.NONE));
             env.kafkaClient().prepareResponse(new OffsetFetchResponse(Errors.NONE, responseData));
 
             final ListConsumerGroupOffsetsResult result = env.adminClient().listConsumerGroupOffsets("group-0");
             final Map partitionToOffsetAndMetadata = result.partitionsToOffsetAndMetadata().get();
 
-            assertEquals(3, partitionToOffsetAndMetadata.size());
+            assertEquals(4, partitionToOffsetAndMetadata.size());
             assertEquals(10, partitionToOffsetAndMetadata.get(myTopicPartition0).offset());
             assertEquals(0, partitionToOffsetAndMetadata.get(myTopicPartition1).offset());
             assertEquals(20, partitionToOffsetAndMetadata.get(myTopicPartition2).offset());
+            assertTrue(partitionToOffsetAndMetadata.containsKey(myTopicPartition3));
+            assertNull(partitionToOffsetAndMetadata.get(myTopicPartition3));
         }
     }
 

From 624f5396c72f2478717181eaa54860809e3b82b8 Mon Sep 17 00:00:00 2001
From: Vikas Singh 
Date: Fri, 7 Feb 2020 19:08:00 -0800
Subject: [PATCH 0872/1071] MINOR: Fix scala 2.11 compilation errors (#8064)

Fixes some compilation errors due to the backport of a fix from trunk which is not compatible with Scala 2.11. Specifically:

1. Either doesn't have toOption method, so used match directly
2. SAM can't be used, fixed that to mention interface

Reviewers: Jason Gustafson 
---
 .../transaction/TransactionStateManagerTest.scala      | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala
index 7f6ccfb8c870e..b628534058f2b 100644
--- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala
@@ -127,8 +127,10 @@ class TransactionStateManagerTest {
     transactionManager.putTransactionStateIfNotExists(metadata2)
 
     def cachedProducerEpoch(transactionalId: String): Option[Short] = {
-      transactionManager.getTransactionState(transactionalId).toOption.flatten
-        .map(_.transactionMetadata.producerEpoch)
+      transactionManager.getTransactionState(transactionalId) match {
+        case Right(Some(epochAndTxnMetadata)) => Some(epochAndTxnMetadata.transactionMetadata.producerEpoch)
+        case _ => None
+      }
     }
 
     assertEquals(Some(metadata1.producerEpoch), cachedProducerEpoch(metadata1.transactionalId))
@@ -187,8 +189,8 @@ class TransactionStateManagerTest {
     val coordinatorEpoch = 0
     val partitionAndLeaderEpoch = TransactionPartitionAndLeaderEpoch(partitionId, coordinatorEpoch)
 
-    val loadingThread = new Thread(() => {
-      transactionManager.loadTransactionsForTxnTopicPartition(partitionId, coordinatorEpoch, (_, _, _, _, _) => ())
+    val loadingThread = new Thread(new Runnable {
+      override def run(): Unit = transactionManager.loadTransactionsForTxnTopicPartition(partitionId, coordinatorEpoch, (_, _, _, _, _) => ())
     })
     loadingThread.start()
     TestUtils.waitUntilTrue(() => transactionManager.loadingPartitions.contains(partitionAndLeaderEpoch),

From 16f074bfcb1128d79706e3aa27c8a6607545c72c Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Mon, 10 Feb 2020 15:33:23 -0600
Subject: [PATCH 0873/1071] KAFKA-9517: Fix default serdes with FK join (#8061)

During the KIP-213 implementation and verification, we neglected to test the
code path for falling back to default serdes if none are given in the topology.

Cherry-pick of 520a76155c66486cbcd0c519ef2980359964fd7c from trunk
Cherry-pick of 661a660cf37be25f8795665e0de74be13bdf44a4 from 2.5

Reviewer: Bill Bejeck 
---
 .../internals/ChangedDeserializer.java        |  10 +-
 .../kstream/internals/ChangedSerializer.java  |  10 +-
 .../streams/kstream/internals/KTableImpl.java |   4 +-
 .../WrappingNullableDeserializer.java         |  23 +++
 .../internals/WrappingNullableSerializer.java |  23 +++
 .../foreignkeyjoin/CombinedKeySchema.java     |   8 +-
 ...JoinSubscriptionSendProcessorSupplier.java |   5 +-
 ...criptionResolverJoinProcessorSupplier.java |  11 +-
 .../SubscriptionResponseWrapperSerde.java     |  33 +++-
 ...criptionStoreReceiveProcessorSupplier.java |   2 +
 .../SubscriptionWrapperSerde.java             |  35 +++-
 .../streams/processor/internals/SinkNode.java |  10 +-
 .../processor/internals/SourceNode.java       |  10 +-
 .../integration/ForeignKeyJoinSuite.java      |   1 +
 ...eKTableForeignKeyJoinDefaultSerdeTest.java | 178 ++++++++++++++++++
 .../StreamsPartitionAssignorTest.java         |   5 +-
 16 files changed, 327 insertions(+), 41 deletions(-)
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/WrappingNullableDeserializer.java
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/WrappingNullableSerializer.java
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinDefaultSerdeTest.java

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/ChangedDeserializer.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/ChangedDeserializer.java
index 36f77b81b238c..90d5882887f3f 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/ChangedDeserializer.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/ChangedDeserializer.java
@@ -20,8 +20,9 @@
 import org.apache.kafka.common.serialization.Deserializer;
 
 import java.nio.ByteBuffer;
+import java.util.Objects;
 
-public class ChangedDeserializer implements Deserializer> {
+public class ChangedDeserializer implements Deserializer>, WrappingNullableDeserializer, T> {
 
     private static final int NEWFLAG_SIZE = 1;
 
@@ -35,8 +36,11 @@ public Deserializer inner() {
         return inner;
     }
 
-    public void setInner(final Deserializer inner) {
-        this.inner = inner;
+    @Override
+    public void setIfUnset(final Deserializer defaultDeserializer) {
+        if (inner == null) {
+            inner = Objects.requireNonNull(defaultDeserializer, "defaultDeserializer cannot be null");
+        }
     }
 
     @Override
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/ChangedSerializer.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/ChangedSerializer.java
index bfd0afa1b1051..551d948392266 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/ChangedSerializer.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/ChangedSerializer.java
@@ -21,8 +21,9 @@
 import org.apache.kafka.streams.errors.StreamsException;
 
 import java.nio.ByteBuffer;
+import java.util.Objects;
 
-public class ChangedSerializer implements Serializer> {
+public class ChangedSerializer implements Serializer>, WrappingNullableSerializer, T> {
 
     private static final int NEWFLAG_SIZE = 1;
 
@@ -36,8 +37,11 @@ public Serializer inner() {
         return inner;
     }
 
-    public void setInner(final Serializer inner) {
-        this.inner = inner;
+    @Override
+    public void setIfUnset(final Serializer defaultSerializer) {
+        if (inner == null) {
+            inner = Objects.requireNonNull(defaultSerializer, "defaultSerializer cannot be null");
+        }
     }
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
index f6f2ada6ce5e4..ef4a323df6472 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
@@ -969,7 +969,7 @@ private  KTable doJoinOnForeignKey(final KTable forei
                     foreignKeyExtractor,
                     foreignKeySerde,
                     subscriptionTopicName,
-                    valSerde.serializer(),
+                    valSerde == null ? null : valSerde.serializer(),
                     leftJoin
                 ),
                 renamed.suffixWithOrElseGet("-subscription-registration-processor", builder, SUBSCRIPTION_REGISTRATION)
@@ -1072,7 +1072,7 @@ private  KTable doJoinOnForeignKey(final KTable forei
         final KTableValueGetterSupplier primaryKeyValueGetter = valueGetterSupplier();
         final SubscriptionResolverJoinProcessorSupplier resolverProcessorSupplier = new SubscriptionResolverJoinProcessorSupplier<>(
             primaryKeyValueGetter,
-            valueSerde().serializer(),
+            valSerde == null ? null : valSerde.serializer(),
             joiner,
             leftJoin
         );
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/WrappingNullableDeserializer.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/WrappingNullableDeserializer.java
new file mode 100644
index 0000000000000..a57e9a15315c4
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/WrappingNullableDeserializer.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.kstream.internals;
+
+import org.apache.kafka.common.serialization.Deserializer;
+
+public interface WrappingNullableDeserializer extends Deserializer {
+    void setIfUnset(final Deserializer defaultDeserializer);
+}
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/WrappingNullableSerializer.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/WrappingNullableSerializer.java
new file mode 100644
index 0000000000000..2d28e52db2bbd
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/WrappingNullableSerializer.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.kstream.internals;
+
+import org.apache.kafka.common.serialization.Serializer;
+
+public interface WrappingNullableSerializer extends Serializer {
+    void setIfUnset(final Serializer defaultSerializer);
+}
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchema.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchema.java
index 8abe583b0c002..7cda404e01d9e 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchema.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchema.java
@@ -36,10 +36,10 @@ public class CombinedKeySchema {
 
     public CombinedKeySchema(final String serdeTopic, final Serde foreignKeySerde, final Serde primaryKeySerde) {
         this.serdeTopic = serdeTopic;
-        primaryKeySerializer = primaryKeySerde.serializer();
-        primaryKeyDeserializer = primaryKeySerde.deserializer();
-        foreignKeyDeserializer = foreignKeySerde.deserializer();
-        foreignKeySerializer = foreignKeySerde.serializer();
+        primaryKeySerializer = primaryKeySerde == null ? null : primaryKeySerde.serializer();
+        primaryKeyDeserializer = primaryKeySerde == null ? null : primaryKeySerde.deserializer();
+        foreignKeyDeserializer = foreignKeySerde == null ? null : foreignKeySerde.deserializer();
+        foreignKeySerializer = foreignKeySerde == null ? null : foreignKeySerde.serializer();
     }
 
     @SuppressWarnings("unchecked")
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionSendProcessorSupplier.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionSendProcessorSupplier.java
index 806cf5c98fd15..dcf5b2a98c873 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionSendProcessorSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionSendProcessorSupplier.java
@@ -44,9 +44,9 @@ public class ForeignJoinSubscriptionSendProcessorSupplier implements P
 
     private final Function foreignKeyExtractor;
     private final String repartitionTopicName;
-    private final Serializer valueSerializer;
     private final boolean leftJoin;
     private Serializer foreignKeySerializer;
+    private Serializer valueSerializer;
 
     public ForeignJoinSubscriptionSendProcessorSupplier(final Function foreignKeyExtractor,
                                                         final Serde foreignKeySerde,
@@ -77,6 +77,9 @@ public void init(final ProcessorContext context) {
             if (foreignKeySerializer == null) {
                 foreignKeySerializer = (Serializer) context.keySerde().serializer();
             }
+            if (valueSerializer == null) {
+                valueSerializer = (Serializer) context.valueSerde().serializer();
+            }
             skippedRecordsSensor = ThreadMetrics.skipRecordSensor(Thread.currentThread().getName(),
                                                                   (StreamsMetricsImpl) context.metrics());
         }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplier.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplier.java
index a188f15062c66..8fa77aa16715b 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplier.java
@@ -41,7 +41,7 @@
  */
 public class SubscriptionResolverJoinProcessorSupplier implements ProcessorSupplier> {
     private final KTableValueGetterSupplier valueGetterSupplier;
-    private final Serializer valueSerializer;
+    private final Serializer constructionTimeValueSerializer;
     private final ValueJoiner joiner;
     private final boolean leftJoin;
 
@@ -50,7 +50,7 @@ public SubscriptionResolverJoinProcessorSupplier(final KTableValueGetterSupplier
                                                      final ValueJoiner joiner,
                                                      final boolean leftJoin) {
         this.valueGetterSupplier = valueGetterSupplier;
-        this.valueSerializer = valueSerializer;
+        constructionTimeValueSerializer = valueSerializer;
         this.joiner = joiner;
         this.leftJoin = leftJoin;
     }
@@ -58,14 +58,19 @@ public SubscriptionResolverJoinProcessorSupplier(final KTableValueGetterSupplier
     @Override
     public Processor> get() {
         return new AbstractProcessor>() {
+            private Serializer runtimeValueSerializer = constructionTimeValueSerializer;
 
             private KTableValueGetter valueGetter;
 
+            @SuppressWarnings("unchecked")
             @Override
             public void init(final ProcessorContext context) {
                 super.init(context);
                 valueGetter = valueGetterSupplier.get();
                 valueGetter.init(context);
+                if (runtimeValueSerializer == null) {
+                    runtimeValueSerializer = (Serializer) context.valueSerde().serializer();
+                }
             }
 
             @Override
@@ -86,7 +91,7 @@ public void process(final K key, final SubscriptionResponseWrapper value) {
                 final String dummySerializationTopic = context().topic() + "-join-resolver";
                 final long[] currentHash = currentValueWithTimestamp == null ?
                     null :
-                    Murmur3.hash128(valueSerializer.serialize(dummySerializationTopic, currentValueWithTimestamp.value()));
+                    Murmur3.hash128(runtimeValueSerializer.serialize(dummySerializationTopic, currentValueWithTimestamp.value()));
 
                 final long[] messageHash = value.getOriginalValueHash();
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapperSerde.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapperSerde.java
index 4277f9a8fd74a..31317c500df6c 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapperSerde.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResponseWrapperSerde.java
@@ -20,16 +20,19 @@
 import org.apache.kafka.common.serialization.Deserializer;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serializer;
+import org.apache.kafka.streams.kstream.internals.WrappingNullableDeserializer;
+import org.apache.kafka.streams.kstream.internals.WrappingNullableSerializer;
 
 import java.nio.ByteBuffer;
+import java.util.Objects;
 
 public class SubscriptionResponseWrapperSerde implements Serde> {
     private final SubscriptionResponseWrapperSerializer serializer;
     private final SubscriptionResponseWrapperDeserializer deserializer;
 
     public SubscriptionResponseWrapperSerde(final Serde foreignValueSerde) {
-        serializer = new SubscriptionResponseWrapperSerializer<>(foreignValueSerde.serializer());
-        deserializer = new SubscriptionResponseWrapperDeserializer<>(foreignValueSerde.deserializer());
+        serializer = new SubscriptionResponseWrapperSerializer<>(foreignValueSerde == null ? null : foreignValueSerde.serializer());
+        deserializer = new SubscriptionResponseWrapperDeserializer<>(foreignValueSerde == null ? null : foreignValueSerde.deserializer());
     }
 
     @Override
@@ -42,13 +45,22 @@ public Deserializer> deserializer() {
         return deserializer;
     }
 
-    private static final class SubscriptionResponseWrapperSerializer implements Serializer> {
-        private final Serializer serializer;
+    private static final class SubscriptionResponseWrapperSerializer
+        implements Serializer>, WrappingNullableSerializer, V> {
+
+        private Serializer serializer;
 
         private SubscriptionResponseWrapperSerializer(final Serializer serializer) {
             this.serializer = serializer;
         }
 
+        @Override
+        public void setIfUnset(final Serializer defaultSerializer) {
+            if (serializer == null) {
+                serializer = Objects.requireNonNull(defaultSerializer, "defaultSerializer cannot be null");
+            }
+        }
+
         @Override
         public byte[] serialize(final String topic, final SubscriptionResponseWrapper data) {
             //{1-bit-isHashNull}{7-bits-version}{Optional-16-byte-Hash}{n-bytes serialized data}
@@ -81,13 +93,22 @@ public byte[] serialize(final String topic, final SubscriptionResponseWrapper
 
     }
 
-    private static final class SubscriptionResponseWrapperDeserializer implements Deserializer> {
-        private final Deserializer deserializer;
+    private static final class SubscriptionResponseWrapperDeserializer
+        implements Deserializer>, WrappingNullableDeserializer, V> {
+
+        private Deserializer deserializer;
 
         private SubscriptionResponseWrapperDeserializer(final Deserializer deserializer) {
             this.deserializer = deserializer;
         }
 
+        @Override
+        public void setIfUnset(final Deserializer defaultDeserializer) {
+            if (deserializer == null) {
+                deserializer = Objects.requireNonNull(defaultDeserializer, "defaultDeserializer cannot be null");
+            }
+        }
+
         @Override
         public SubscriptionResponseWrapper deserialize(final String topic, final byte[] data) {
             //{1-bit-isHashNull}{7-bits-version}{Optional-16-byte-Hash}{n-bytes serialized data}
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionStoreReceiveProcessorSupplier.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionStoreReceiveProcessorSupplier.java
index 09744639b8aad..24d7ddf9d14bb 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionStoreReceiveProcessorSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionStoreReceiveProcessorSupplier.java
@@ -67,6 +67,8 @@ public void init(final ProcessorContext context) {
                 metrics = internalProcessorContext.metrics();
                 skippedRecordsSensor = ThreadMetrics.skipRecordSensor(Thread.currentThread().getName(), metrics);
                 store = internalProcessorContext.getStateStore(storeBuilder);
+
+                keySchema.init(context);
             }
 
             @Override
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerde.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerde.java
index ae53ba8b3498a..1cb32935fbee1 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerde.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerde.java
@@ -20,16 +20,19 @@
 import org.apache.kafka.common.serialization.Deserializer;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serializer;
+import org.apache.kafka.streams.kstream.internals.WrappingNullableDeserializer;
+import org.apache.kafka.streams.kstream.internals.WrappingNullableSerializer;
 
 import java.nio.ByteBuffer;
+import java.util.Objects;
 
 public class SubscriptionWrapperSerde implements Serde> {
     private final SubscriptionWrapperSerializer serializer;
     private final SubscriptionWrapperDeserializer deserializer;
 
     public SubscriptionWrapperSerde(final Serde primaryKeySerde) {
-        serializer = new SubscriptionWrapperSerializer<>(primaryKeySerde.serializer());
-        deserializer = new SubscriptionWrapperDeserializer<>(primaryKeySerde.deserializer());
+        serializer = new SubscriptionWrapperSerializer<>(primaryKeySerde == null ? null : primaryKeySerde.serializer());
+        deserializer = new SubscriptionWrapperDeserializer<>(primaryKeySerde == null ? null : primaryKeySerde.deserializer());
     }
 
     @Override
@@ -42,12 +45,22 @@ public Deserializer> deserializer() {
         return deserializer;
     }
 
-    private static class SubscriptionWrapperSerializer implements Serializer> {
-        private final Serializer primaryKeySerializer;
+    public static class SubscriptionWrapperSerializer
+        implements Serializer>, WrappingNullableSerializer, K> {
+
+        private Serializer primaryKeySerializer;
+
         SubscriptionWrapperSerializer(final Serializer primaryKeySerializer) {
             this.primaryKeySerializer = primaryKeySerializer;
         }
 
+        @Override
+        public void setIfUnset(final Serializer defaultSerializer) {
+            if (primaryKeySerializer == null) {
+                primaryKeySerializer = Objects.requireNonNull(defaultSerializer, "defaultSerializer cannot be null");
+            }
+        }
+
         @Override
         public byte[] serialize(final String topic, final SubscriptionWrapper data) {
             //{1-bit-isHashNull}{7-bits-version}{1-byte-instruction}{Optional-16-byte-Hash}{PK-serialized}
@@ -81,12 +94,22 @@ public byte[] serialize(final String topic, final SubscriptionWrapper data) {
 
     }
 
-    private static class SubscriptionWrapperDeserializer implements Deserializer> {
-        private final Deserializer primaryKeyDeserializer;
+    public static class SubscriptionWrapperDeserializer
+        implements Deserializer>, WrappingNullableDeserializer, K> {
+
+        private Deserializer primaryKeyDeserializer;
+
         SubscriptionWrapperDeserializer(final Deserializer primaryKeyDeserializer) {
             this.primaryKeyDeserializer = primaryKeyDeserializer;
         }
 
+        @Override
+        public void setIfUnset(final Deserializer defaultDeserializer) {
+            if (primaryKeyDeserializer == null) {
+                primaryKeyDeserializer = Objects.requireNonNull(defaultDeserializer, "defaultDeserializer cannot be null");
+            }
+        }
+
         @Override
         public SubscriptionWrapper deserialize(final String topic, final byte[] data) {
             //{7-bits-version}{1-bit-isHashNull}{1-byte-instruction}{Optional-16-byte-Hash}{PK-serialized}
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/SinkNode.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/SinkNode.java
index b94291b3503fa..e3333be03e06e 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/SinkNode.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/SinkNode.java
@@ -18,7 +18,7 @@
 
 import org.apache.kafka.common.serialization.Serializer;
 import org.apache.kafka.streams.errors.StreamsException;
-import org.apache.kafka.streams.kstream.internals.ChangedSerializer;
+import org.apache.kafka.streams.kstream.internals.WrappingNullableSerializer;
 import org.apache.kafka.streams.processor.StreamPartitioner;
 import org.apache.kafka.streams.processor.TopicNameExtractor;
 
@@ -66,10 +66,10 @@ public void init(final InternalProcessorContext context) {
             valSerializer = (Serializer) context.valueSerde().serializer();
         }
 
-        // if value serializers are for {@code Change} values, set the inner serializer when necessary
-        if (valSerializer instanceof ChangedSerializer &&
-                ((ChangedSerializer) valSerializer).inner() == null) {
-            ((ChangedSerializer) valSerializer).setInner(context.valueSerde().serializer());
+        // if value serializers are internal wrapping serializers that may need to be given the default serializer
+        // then pass it the default one from the context
+        if (valSerializer instanceof WrappingNullableSerializer) {
+            ((WrappingNullableSerializer) valSerializer).setIfUnset(context.valueSerde().serializer());
         }
     }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/SourceNode.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/SourceNode.java
index 87505cab22a8e..d70d6132c429f 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/SourceNode.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/SourceNode.java
@@ -18,7 +18,7 @@
 
 import org.apache.kafka.common.header.Headers;
 import org.apache.kafka.common.serialization.Deserializer;
-import org.apache.kafka.streams.kstream.internals.ChangedDeserializer;
+import org.apache.kafka.streams.kstream.internals.WrappingNullableDeserializer;
 import org.apache.kafka.streams.processor.ProcessorContext;
 import org.apache.kafka.streams.processor.TimestampExtractor;
 
@@ -74,10 +74,10 @@ public void init(final InternalProcessorContext context) {
             this.valDeserializer = (Deserializer) context.valueSerde().deserializer();
         }
 
-        // if value deserializers are for {@code Change} values, set the inner deserializer when necessary
-        if (this.valDeserializer instanceof ChangedDeserializer &&
-                ((ChangedDeserializer) this.valDeserializer).inner() == null) {
-            ((ChangedDeserializer) this.valDeserializer).setInner(context.valueSerde().deserializer());
+        // if value deserializers are internal wrapping deserializers that may need to be given the default
+        // then pass it the default one from the context
+        if (valDeserializer instanceof WrappingNullableDeserializer) {
+            ((WrappingNullableDeserializer) valDeserializer).setIfUnset(context.valueSerde().deserializer());
         }
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/ForeignKeyJoinSuite.java b/streams/src/test/java/org/apache/kafka/streams/integration/ForeignKeyJoinSuite.java
index 47e2d95da63bc..dbe5bc2981cfa 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/ForeignKeyJoinSuite.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/ForeignKeyJoinSuite.java
@@ -39,6 +39,7 @@
     KTableKTableForeignKeyInnerJoinMultiIntegrationTest.class,
     KTableKTableForeignKeyJoinIntegrationTest.class,
     KTableKTableForeignKeyJoinMaterializationIntegrationTest.class,
+    KTableKTableForeignKeyJoinDefaultSerdeTest.class,
     CombinedKeySchemaTest.class,
     SubscriptionWrapperSerdeTest.class,
     SubscriptionResponseWrapperSerdeTest.class,
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinDefaultSerdeTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinDefaultSerdeTest.java
new file mode 100644
index 0000000000000..df6349a8510e5
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinDefaultSerdeTest.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.integration;
+
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TestInputTopic;
+import org.apache.kafka.streams.TestOutputTopic;
+import org.apache.kafka.streams.TopologyTestDriver;
+import org.apache.kafka.streams.kstream.Consumed;
+import org.apache.kafka.streams.kstream.KTable;
+import org.apache.kafka.streams.kstream.Materialized;
+import org.apache.kafka.streams.kstream.Produced;
+import org.apache.kafka.streams.state.KeyValueStore;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Properties;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
+public class KTableKTableForeignKeyJoinDefaultSerdeTest {
+    @Test
+    public void shouldWorkWithDefaultSerdes() {
+        final StreamsBuilder builder = new StreamsBuilder();
+        final KTable aTable = builder.table("A");
+        final KTable bTable = builder.table("B");
+
+        final KTable fkJoinResult = aTable.join(
+            bTable,
+            value -> value.split("-")[0],
+            (aVal, bVal) -> "(" + aVal + "," + bVal + ")",
+            Materialized.as("asdf")
+        );
+
+        final KTable finalJoinResult = aTable.join(
+            fkJoinResult,
+            (aVal, fkJoinVal) -> "(" + aVal + "," + fkJoinVal + ")"
+        );
+
+        finalJoinResult.toStream().to("output");
+
+        validateTopologyCanProcessData(builder);
+    }
+
+    @Test
+    public void shouldWorkWithDefaultAndConsumedSerdes() {
+        final StreamsBuilder builder = new StreamsBuilder();
+        final KTable aTable = builder.table("A", Consumed.with(Serdes.String(), Serdes.String()));
+        final KTable bTable = builder.table("B");
+
+        final KTable fkJoinResult = aTable.join(
+            bTable,
+            value -> value.split("-")[0],
+            (aVal, bVal) -> "(" + aVal + "," + bVal + ")",
+            Materialized.as("asdf")
+        );
+
+        final KTable finalJoinResult = aTable.join(
+            fkJoinResult,
+            (aVal, fkJoinVal) -> "(" + aVal + "," + fkJoinVal + ")"
+        );
+
+        finalJoinResult.toStream().to("output");
+
+        validateTopologyCanProcessData(builder);
+    }
+
+    @Test
+    public void shouldWorkWithDefaultAndJoinResultSerdes() {
+        final StreamsBuilder builder = new StreamsBuilder();
+        final KTable aTable = builder.table("A");
+        final KTable bTable = builder.table("B");
+
+        final KTable fkJoinResult = aTable.join(
+            bTable,
+            value -> value.split("-")[0],
+            (aVal, bVal) -> "(" + aVal + "," + bVal + ")",
+            Materialized
+                .>as("asdf")
+                .withKeySerde(Serdes.String())
+                .withValueSerde(Serdes.String())
+        );
+
+        final KTable finalJoinResult = aTable.join(
+            fkJoinResult,
+            (aVal, fkJoinVal) -> "(" + aVal + "," + fkJoinVal + ")"
+        );
+
+        finalJoinResult.toStream().to("output");
+
+        validateTopologyCanProcessData(builder);
+    }
+
+    @Test
+    public void shouldWorkWithDefaultAndEquiJoinResultSerdes() {
+        final StreamsBuilder builder = new StreamsBuilder();
+        final KTable aTable = builder.table("A");
+        final KTable bTable = builder.table("B");
+
+        final KTable fkJoinResult = aTable.join(
+            bTable,
+            value -> value.split("-")[0],
+            (aVal, bVal) -> "(" + aVal + "," + bVal + ")",
+            Materialized.as("asdf")
+        );
+
+        final KTable finalJoinResult = aTable.join(
+            fkJoinResult,
+            (aVal, fkJoinVal) -> "(" + aVal + "," + fkJoinVal + ")",
+            Materialized.with(Serdes.String(), Serdes.String())
+        );
+
+        finalJoinResult.toStream().to("output");
+
+        validateTopologyCanProcessData(builder);
+    }
+
+    @Test
+    public void shouldWorkWithDefaultAndProducedSerdes() {
+        final StreamsBuilder builder = new StreamsBuilder();
+        final KTable aTable = builder.table("A");
+        final KTable bTable = builder.table("B");
+
+        final KTable fkJoinResult = aTable.join(
+            bTable,
+            value -> value.split("-")[0],
+            (aVal, bVal) -> "(" + aVal + "," + bVal + ")",
+            Materialized.as("asdf")
+        );
+
+        final KTable finalJoinResult = aTable.join(
+            fkJoinResult,
+            (aVal, fkJoinVal) -> "(" + aVal + "," + fkJoinVal + ")"
+        );
+
+        finalJoinResult.toStream().to("output", Produced.with(Serdes.String(), Serdes.String()));
+
+        validateTopologyCanProcessData(builder);
+    }
+
+    private static void validateTopologyCanProcessData(final StreamsBuilder builder) {
+        final Properties config = new Properties();
+        config.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "dummy");
+        config.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy");
+        config.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.StringSerde.class.getName());
+        config.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.StringSerde.class.getName());
+        try (final TopologyTestDriver topologyTestDriver = new TopologyTestDriver(builder.build(), config)) {
+            final TestInputTopic aTopic = topologyTestDriver.createInputTopic("A", new StringSerializer(), new StringSerializer());
+            final TestInputTopic bTopic = topologyTestDriver.createInputTopic("B", new StringSerializer(), new StringSerializer());
+            final TestOutputTopic output = topologyTestDriver.createOutputTopic("output", new StringDeserializer(), new StringDeserializer());
+            aTopic.pipeInput("a1", "b1-alpha");
+            bTopic.pipeInput("b1", "beta");
+            final Map x = output.readKeyValuesToMap();
+            assertThat(x, is(Collections.singletonMap("a1", "(b1-alpha,(b1-alpha,beta))")));
+        }
+    }
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
index 9ff6e33b07d9e..0a296cd297ba5 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
@@ -1596,9 +1596,8 @@ public void shouldThrowIfV2SubscriptionAndFutureSubscriptionIsMixed() {
         shouldThrowIfPreVersionProbingSubscriptionAndFutureSubscriptionIsMixed(2);
     }
 
-    private ByteBuffer encodeFutureSubscription() {
-        final ByteBuffer buf = ByteBuffer.allocate(4 /* used version */
-                                                   + 4 /* supported version */);
+    private static ByteBuffer encodeFutureSubscription() {
+        final ByteBuffer buf = ByteBuffer.allocate(4 /* used version */ + 4 /* supported version */);
         buf.putInt(LATEST_SUPPORTED_VERSION + 1);
         buf.putInt(LATEST_SUPPORTED_VERSION + 1);
         return buf;

From f39290c785e1862a24c50680108234cb5801daec Mon Sep 17 00:00:00 2001
From: Gunnar Morling 
Date: Tue, 11 Feb 2020 20:48:22 +0100
Subject: [PATCH 0874/1071] KAFKA-7052 Avoiding NPE in ExtractField SMT in case
 of non-existent fields (#8059)

Author: Gunnar Morling 
Reviewer: Randall Hauch 
---
 .../connect/transforms/ExtractField.java      |  9 ++++++-
 .../connect/transforms/ExtractFieldTest.java  | 27 +++++++++++++++++++
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/ExtractField.java b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/ExtractField.java
index eb8c357f3ff16..bd3cbd9288c26 100644
--- a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/ExtractField.java
+++ b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/ExtractField.java
@@ -18,6 +18,7 @@
 
 import org.apache.kafka.common.config.ConfigDef;
 import org.apache.kafka.connect.connector.ConnectRecord;
+import org.apache.kafka.connect.data.Field;
 import org.apache.kafka.connect.data.Schema;
 import org.apache.kafka.connect.data.Struct;
 import org.apache.kafka.connect.transforms.util.SimpleConfig;
@@ -58,7 +59,13 @@ public R apply(R record) {
             return newRecord(record, null, value == null ? null : value.get(fieldName));
         } else {
             final Struct value = requireStructOrNull(operatingValue(record), PURPOSE);
-            return newRecord(record, schema.field(fieldName).schema(), value == null ? null : value.get(fieldName));
+            Field field = schema.field(fieldName);
+
+            if (field == null) {
+                throw new IllegalArgumentException("Unknown field: " + fieldName);
+            }
+
+            return newRecord(record, field.schema(), value == null ? null : value.get(fieldName));
         }
     }
 
diff --git a/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/ExtractFieldTest.java b/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/ExtractFieldTest.java
index acb0beb0049ec..a78c6d5804b20 100644
--- a/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/ExtractFieldTest.java
+++ b/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/ExtractFieldTest.java
@@ -28,6 +28,7 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
 
 public class ExtractFieldTest {
     private final ExtractField xform = new ExtractField.Key<>();
@@ -86,4 +87,30 @@ public void testNullWithSchema() {
         assertNull(transformedRecord.key());
     }
 
+    @Test
+    public void nonExistentFieldSchemalessShouldReturnNull() {
+        xform.configure(Collections.singletonMap("field", "nonexistent"));
+
+        final SinkRecord record = new SinkRecord("test", 0, null, Collections.singletonMap("magic", 42), null, null, 0);
+        final SinkRecord transformedRecord = xform.apply(record);
+
+        assertNull(transformedRecord.keySchema());
+        assertNull(transformedRecord.key());
+    }
+
+    @Test
+    public void nonExistentFieldWithSchemaShouldFail() {
+        xform.configure(Collections.singletonMap("field", "nonexistent"));
+
+        final Schema keySchema = SchemaBuilder.struct().field("magic", Schema.INT32_SCHEMA).build();
+        final Struct key = new Struct(keySchema).put("magic", 42);
+        final SinkRecord record = new SinkRecord("test", 0, keySchema, key, null, null, 0);
+
+        try {
+            xform.apply(record);
+            fail("Expected exception wasn't raised");
+        } catch (IllegalArgumentException iae) {
+            assertEquals("Unknown field: nonexistent", iae.getMessage());
+        }
+    }
 }

From 2b3a5379fdb1217e9156a93896bbf07540b3f117 Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Tue, 11 Feb 2020 14:59:36 -0800
Subject: [PATCH 0875/1071] KAFKA-9540: Move "Could not find the standby task
 while closing it" log to debug level (#8092)

Reviewers: Guozhang Wang 
---
 .../streams/processor/internals/AssignedStandbyTasks.java      | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStandbyTasks.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStandbyTasks.java
index 299390bfd1666..ba41fd84461c4 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStandbyTasks.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStandbyTasks.java
@@ -71,7 +71,8 @@ List closeRevokedStandbyTasks(final Map
Date: Tue, 11 Feb 2020 12:49:26 -0800
Subject: [PATCH 0876/1071] MINOR: Start using Response and replace IOException
 in EmbeddedConnectCluster for failures (#8055)

Changed `EmbeddedConnectCluster` to add utility methods that return `Response`, throw `ConnectException` instead of `IOException` for failures, and deprecate the old methods that returned primitive types rather than `Response`.

Also introduce common assertions for embedded clusters under `EmbeddedConnectClusterAssertions`.

Author: Konstantine Karantasis 
Reviewer: Randall Hauch , Chia-Ping Tsai 
---
 .../ConnectWorkerIntegrationTest.java         | 124 ++-----
 ...alanceSourceConnectorsIntegrationTest.java | 145 +++-----
 .../RestExtensionIntegrationTest.java         |   9 +-
 .../SessionedProtocolIntegrationTest.java     |   8 +-
 .../util/clusters/EmbeddedConnectCluster.java | 317 ++++++++++++------
 .../EmbeddedConnectClusterAssertions.java     | 246 ++++++++++++++
 .../util/clusters/EmbeddedKafkaCluster.java   |  25 +-
 7 files changed, 561 insertions(+), 313 deletions(-)
 create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectClusterAssertions.java

diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java
index 4a704664b69db..6cc43a471f3ba 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java
@@ -16,9 +16,7 @@
  */
 package org.apache.kafka.connect.integration;
 
-import org.apache.kafka.connect.runtime.AbstractStatus;
 import org.apache.kafka.connect.runtime.distributed.DistributedConfig;
-import org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo;
 import org.apache.kafka.connect.storage.StringConverter;
 import org.apache.kafka.connect.util.clusters.EmbeddedConnectCluster;
 import org.apache.kafka.connect.util.clusters.WorkerHandle;
@@ -35,7 +33,6 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
@@ -48,7 +45,6 @@
 import static org.apache.kafka.connect.runtime.ConnectorConfig.VALUE_CONVERTER_CLASS_CONFIG;
 import static org.apache.kafka.connect.runtime.WorkerConfig.CONNECTOR_CLIENT_POLICY_CLASS_CONFIG;
 import static org.apache.kafka.connect.runtime.WorkerConfig.OFFSET_COMMIT_INTERVAL_MS_CONFIG;
-import static org.apache.kafka.test.TestUtils.waitForCondition;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
@@ -60,8 +56,6 @@ public class ConnectWorkerIntegrationTest {
     private static final Logger log = LoggerFactory.getLogger(ConnectWorkerIntegrationTest.class);
 
     private static final int NUM_TOPIC_PARTITIONS = 3;
-    private static final long CONNECTOR_SETUP_DURATION_MS = TimeUnit.SECONDS.toMillis(30);
-    private static final long WORKER_SETUP_DURATION_MS = TimeUnit.SECONDS.toMillis(60);
     private static final long OFFSET_COMMIT_INTERVAL_MS = TimeUnit.SECONDS.toMillis(30);
     private static final int NUM_WORKERS = 3;
     private static final String CONNECTOR_NAME = "simple-source";
@@ -118,30 +112,30 @@ public void testAddAndRemoveWorker() throws Exception {
         props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
         props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
 
-        waitForCondition(() -> assertWorkersUp(NUM_WORKERS).orElse(false),
-                WORKER_SETUP_DURATION_MS, "Initial group of workers did not start in time.");
+        connect.assertions().assertAtLeastNumWorkersAreUp(NUM_WORKERS,
+                "Initial group of workers did not start in time.");
 
         // start a source connector
         connect.configureConnector(CONNECTOR_NAME, props);
 
-        waitForCondition(() -> assertConnectorAndTasksRunning(CONNECTOR_NAME, numTasks).orElse(false),
-                CONNECTOR_SETUP_DURATION_MS, "Connector tasks did not start in time.");
+        connect.assertions().assertConnectorAndAtLeastNumTasksAreRunning(CONNECTOR_NAME, numTasks,
+                "Connector tasks did not start in time.");
 
         WorkerHandle extraWorker = connect.addWorker();
 
-        waitForCondition(() -> assertWorkersUp(NUM_WORKERS + 1).orElse(false),
-                WORKER_SETUP_DURATION_MS, "Expanded group of workers did not start in time.");
+        connect.assertions().assertAtLeastNumWorkersAreUp(NUM_WORKERS + 1,
+                "Expanded group of workers did not start in time.");
 
-        waitForCondition(() -> assertConnectorAndTasksRunning(CONNECTOR_NAME, numTasks).orElse(false),
-                CONNECTOR_SETUP_DURATION_MS, "Connector tasks are not all in running state.");
+        connect.assertions().assertConnectorAndAtLeastNumTasksAreRunning(CONNECTOR_NAME, numTasks,
+                "Connector tasks are not all in running state.");
 
         Set workers = connect.activeWorkers();
         assertTrue(workers.contains(extraWorker));
 
         connect.removeWorker(extraWorker);
 
-        waitForCondition(() -> assertWorkersUp(NUM_WORKERS).orElse(false) && !assertWorkersUp(NUM_WORKERS + 1).orElse(false),
-                WORKER_SETUP_DURATION_MS, "Group of workers did not shrink in time.");
+        connect.assertions().assertExactlyNumWorkersAreUp(NUM_WORKERS,
+                "Group of workers did not shrink in time.");
 
         workers = connect.activeWorkers();
         assertFalse(workers.contains(extraWorker));
@@ -164,14 +158,14 @@ public void testRestartFailedTask() throws Exception {
         connectorProps.put(TASKS_MAX_CONFIG, Objects.toString(numTasks));
         connectorProps.put(CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX + BOOTSTRAP_SERVERS_CONFIG, "nobrokerrunningatthisaddress");
 
-        waitForCondition(() -> assertWorkersUp(NUM_WORKERS).orElse(false),
-                WORKER_SETUP_DURATION_MS, "Initial group of workers did not start in time.");
+        connect.assertions().assertExactlyNumWorkersAreUp(NUM_WORKERS,
+                "Initial group of workers did not start in time.");
 
         // Try to start the connector and its single task.
         connect.configureConnector(CONNECTOR_NAME, connectorProps);
 
-        waitForCondition(() -> assertConnectorTasksFailed(CONNECTOR_NAME, numTasks).orElse(false),
-                CONNECTOR_SETUP_DURATION_MS, "Connector tasks did not fail in time");
+        connect.assertions().assertConnectorIsRunningAndTasksHaveFailed(CONNECTOR_NAME, numTasks,
+                "Connector tasks did not fail in time");
 
         // Reconfigure the connector without the bad broker address.
         connectorProps.remove(CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX + BOOTSTRAP_SERVERS_CONFIG);
@@ -180,11 +174,11 @@ public void testRestartFailedTask() throws Exception {
         // Restart the failed task
         String taskRestartEndpoint = connect.endpointForResource(
             String.format("connectors/%s/tasks/0/restart", CONNECTOR_NAME));
-        connect.executePost(taskRestartEndpoint, "", Collections.emptyMap());
+        connect.requestPost(taskRestartEndpoint, "", Collections.emptyMap());
 
         // Ensure the task started successfully this time
-        waitForCondition(() -> assertConnectorAndTasksRunning(CONNECTOR_NAME, numTasks).orElse(false),
-            CONNECTOR_SETUP_DURATION_MS, "Connector tasks are not all in running state.");
+        connect.assertions().assertConnectorAndAtLeastNumTasksAreRunning(CONNECTOR_NAME, numTasks,
+            "Connector tasks are not all in running state.");
     }
 
     /**
@@ -210,19 +204,19 @@ public void testBrokerCoordinator() throws Exception {
         props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
         props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
 
-        waitForCondition(() -> assertWorkersUp(NUM_WORKERS).orElse(false),
-                WORKER_SETUP_DURATION_MS, "Initial group of workers did not start in time.");
+        connect.assertions().assertAtLeastNumWorkersAreUp(NUM_WORKERS,
+                "Initial group of workers did not start in time.");
 
         // start a source connector
         connect.configureConnector(CONNECTOR_NAME, props);
 
-        waitForCondition(() -> assertConnectorAndTasksRunning(CONNECTOR_NAME, numTasks).orElse(false),
-                CONNECTOR_SETUP_DURATION_MS, "Connector tasks did not start in time.");
+        connect.assertions().assertConnectorAndAtLeastNumTasksAreRunning(CONNECTOR_NAME, numTasks,
+                "Connector tasks did not start in time.");
 
         connect.kafka().stopOnlyKafka();
 
-        waitForCondition(() -> assertWorkersUp(NUM_WORKERS).orElse(false),
-                WORKER_SETUP_DURATION_MS, "Group of workers did not remain the same after broker shutdown");
+        connect.assertions().assertExactlyNumWorkersAreUp(NUM_WORKERS,
+                "Group of workers did not remain the same after broker shutdown");
 
         // Allow for the workers to discover that the coordinator is unavailable, wait is
         // heartbeat timeout * 2 + 4sec
@@ -233,78 +227,14 @@ public void testBrokerCoordinator() throws Exception {
         // Allow for the kafka brokers to come back online
         Thread.sleep(TimeUnit.SECONDS.toMillis(10));
 
-        waitForCondition(() -> assertWorkersUp(NUM_WORKERS).orElse(false),
-                WORKER_SETUP_DURATION_MS, "Group of workers did not remain the same within the "
-                        + "designated time.");
+        connect.assertions().assertExactlyNumWorkersAreUp(NUM_WORKERS,
+                "Group of workers did not remain the same within the designated time.");
 
         // Allow for the workers to rebalance and reach a steady state
         Thread.sleep(TimeUnit.SECONDS.toMillis(10));
 
-        waitForCondition(() -> assertConnectorAndTasksRunning(CONNECTOR_NAME, numTasks).orElse(false),
-                CONNECTOR_SETUP_DURATION_MS, "Connector tasks did not start in time.");
+        connect.assertions().assertConnectorAndAtLeastNumTasksAreRunning(CONNECTOR_NAME, numTasks,
+                "Connector tasks did not start in time.");
     }
 
-    /**
-     * Confirm that the requested number of workers is up and running.
-     *
-     * @param numWorkers the number of online workers
-     * @return true if at least {@code numWorkers} are up; false otherwise
-     */
-    private Optional assertWorkersUp(int numWorkers) {
-        try {
-            int numUp = connect.activeWorkers().size();
-            return Optional.of(numUp >= numWorkers);
-        } catch (Exception e) {
-            log.error("Could not check active workers.", e);
-            return Optional.empty();
-        }
-    }
-
-    /**
-     * Confirm that a connector with an exact number of tasks is running.
-     *
-     * @param connectorName the connector
-     * @param numTasks the expected number of tasks
-     * @return true if the connector and tasks are in RUNNING state; false otherwise
-     */
-    private Optional assertConnectorAndTasksRunning(String connectorName, int numTasks) {
-        return assertConnectorState(
-            connectorName,
-            AbstractStatus.State.RUNNING,
-            numTasks,
-            AbstractStatus.State.RUNNING);
-    }
-
-    /**
-     * Confirm that a connector is running, that it has a specific number of tasks, and that all of
-     * its tasks are in the FAILED state.
-     * @param connectorName the connector
-     * @param numTasks the expected number of tasks
-     * @return true if the connector is in RUNNING state and its tasks are in FAILED state; false otherwise
-     */
-    private Optional assertConnectorTasksFailed(String connectorName, int numTasks) {
-        return assertConnectorState(
-            connectorName,
-            AbstractStatus.State.RUNNING,
-            numTasks,
-            AbstractStatus.State.FAILED);
-    }
-
-    private Optional assertConnectorState(
-        String connectorName,
-        AbstractStatus.State connectorState,
-        int numTasks,
-        AbstractStatus.State tasksState) {
-        try {
-            ConnectorStateInfo info = connect.connectorStatus(connectorName);
-            boolean result = info != null
-                    && info.connector().state().equals(connectorState.toString())
-                    && info.tasks().size() == numTasks
-                    && info.tasks().stream().allMatch(s -> s.state().equals(tasksState.toString()));
-            return Optional.of(result);
-        } catch (Exception e) {
-            log.error("Could not check connector state info.", e);
-            return Optional.empty();
-        }
-    }
 }
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java
index d9ff22326cc15..073b3072cf478 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java
@@ -16,8 +16,6 @@
  */
 package org.apache.kafka.connect.integration;
 
-import org.apache.kafka.connect.errors.ConnectException;
-import org.apache.kafka.connect.runtime.AbstractStatus;
 import org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo;
 import org.apache.kafka.connect.storage.StringConverter;
 import org.apache.kafka.connect.util.clusters.EmbeddedConnectCluster;
@@ -35,7 +33,6 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
@@ -65,6 +62,7 @@ public class RebalanceSourceConnectorsIntegrationTest {
     private static final int NUM_TOPIC_PARTITIONS = 3;
     private static final long CONNECTOR_SETUP_DURATION_MS = TimeUnit.SECONDS.toMillis(30);
     private static final long WORKER_SETUP_DURATION_MS = TimeUnit.SECONDS.toMillis(60);
+    private static final int NUM_WORKERS = 3;
     private static final int NUM_TASKS = 4;
     private static final String CONNECTOR_NAME = "seq-source1";
     private static final String TOPIC_NAME = "sequential-topic";
@@ -85,7 +83,7 @@ public void setup() throws IOException {
         // build a Connect cluster backed by Kafka and Zk
         connect = new EmbeddedConnectCluster.Builder()
                 .name("connect-cluster")
-                .numWorkers(3)
+                .numWorkers(NUM_WORKERS)
                 .numBrokers(1)
                 .workerProps(workerProps)
                 .brokerProps(brokerProps)
@@ -117,20 +115,23 @@ public void testStartTwoConnectors() throws Exception {
         props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
         props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
 
+        connect.assertions().assertAtLeastNumWorkersAreUp(NUM_WORKERS,
+                "Connect workers did not start in time.");
+
         // start a source connector
         connect.configureConnector(CONNECTOR_NAME, props);
 
-        waitForCondition(() -> this.assertConnectorAndTasksRunning(CONNECTOR_NAME, NUM_TASKS).orElse(false),
-                CONNECTOR_SETUP_DURATION_MS, "Connector tasks did not start in time.");
+        connect.assertions().assertConnectorAndAtLeastNumTasksAreRunning(CONNECTOR_NAME, NUM_TASKS,
+                "Connector tasks did not start in time.");
 
         // start a source connector
         connect.configureConnector("another-source", props);
 
-        waitForCondition(() -> this.assertConnectorAndTasksRunning(CONNECTOR_NAME, NUM_TASKS).orElse(false),
-                CONNECTOR_SETUP_DURATION_MS, "Connector tasks did not start in time.");
+        connect.assertions().assertConnectorAndAtLeastNumTasksAreRunning(CONNECTOR_NAME, NUM_TASKS,
+                "Connector tasks did not start in time.");
 
-        waitForCondition(() -> this.assertConnectorAndTasksRunning("another-source", 4).orElse(false),
-                CONNECTOR_SETUP_DURATION_MS, "Connector tasks did not start in time.");
+        connect.assertions().assertConnectorAndAtLeastNumTasksAreRunning("another-source", 4,
+                "Connector tasks did not start in time.");
     }
 
     @Ignore("Flaky and disruptive. See KAFKA-8391, KAFKA-8661 for details.")
@@ -153,11 +154,14 @@ public void testReconfigConnector() throws Exception {
         props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
         props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
 
+        connect.assertions().assertAtLeastNumWorkersAreUp(NUM_WORKERS,
+                "Connect workers did not start in time.");
+
         // start a source connector
         connect.configureConnector(CONNECTOR_NAME, props);
 
-        waitForCondition(() -> this.assertConnectorAndTasksRunning(CONNECTOR_NAME, NUM_TASKS).orElse(false),
-                CONNECTOR_SETUP_DURATION_MS, "Connector tasks did not start in time.");
+        connect.assertions().assertConnectorAndAtLeastNumTasksAreRunning(CONNECTOR_NAME, NUM_TASKS,
+                "Connector tasks did not start in time.");
 
         int numRecordsProduced = 100;
         long recordTransferDurationMs = TimeUnit.SECONDS.toMillis(30);
@@ -180,8 +184,8 @@ public void testReconfigConnector() throws Exception {
                 restartLatch.await(CONNECTOR_SETUP_DURATION_MS, TimeUnit.MILLISECONDS));
 
         // And wait for the Connect to show the connectors and tasks are running
-        waitForCondition(() -> this.assertConnectorAndTasksRunning(CONNECTOR_NAME, NUM_TASKS).orElse(false),
-                CONNECTOR_SETUP_DURATION_MS, "Connector tasks did not start in time.");
+        connect.assertions().assertConnectorAndAtLeastNumTasksAreRunning(CONNECTOR_NAME, NUM_TASKS,
+                "Connector tasks did not start in time.");
 
         // consume all records from the source topic or fail, to ensure that they were correctly produced
         recordNum = connect.kafka().consume(numRecordsProduced, recordTransferDurationMs, anotherTopic).count();
@@ -205,27 +209,20 @@ public void testDeleteConnector() throws Exception {
         props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
         props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
 
-        waitForCondition(() -> this.assertWorkersUp(3),
-                WORKER_SETUP_DURATION_MS, "Connect workers did not start in time.");
+        connect.assertions().assertAtLeastNumWorkersAreUp(NUM_WORKERS,
+                "Connect workers did not start in time.");
 
-        // start a source connector
-        IntStream.range(0, 4).forEachOrdered(
-            i -> {
-                try {
-                    connect.configureConnector(CONNECTOR_NAME + i, props);
-                } catch (IOException e) {
-                    throw new ConnectException(e);
-                }
-            });
-
-        waitForCondition(() -> this.assertConnectorAndTasksRunning(CONNECTOR_NAME + 3, NUM_TASKS).orElse(true),
-                CONNECTOR_SETUP_DURATION_MS, "Connector tasks did not start in time.");
+        // start several source connectors
+        IntStream.range(0, 4).forEachOrdered(i -> connect.configureConnector(CONNECTOR_NAME + i, props));
+
+        connect.assertions().assertConnectorAndAtLeastNumTasksAreRunning(CONNECTOR_NAME + 3, NUM_TASKS,
+                "Connector tasks did not start in time.");
 
         // delete connector
         connect.deleteConnector(CONNECTOR_NAME + 3);
 
-        waitForCondition(() -> !this.assertConnectorAndTasksRunning(CONNECTOR_NAME + 3, NUM_TASKS).orElse(true),
-                CONNECTOR_SETUP_DURATION_MS, "Connector tasks did not stop in time.");
+        connect.assertions().assertConnectorAndTasksAreStopped(CONNECTOR_NAME + 3,
+                "Connector tasks did not stop in time.");
 
         waitForCondition(this::assertConnectorAndTasksAreUnique,
                 WORKER_SETUP_DURATION_MS, "Connect and tasks are imbalanced between the workers.");
@@ -246,29 +243,22 @@ public void testAddingWorker() throws Exception {
         props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
         props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
 
-        waitForCondition(() -> this.assertWorkersUp(3),
-                WORKER_SETUP_DURATION_MS, "Connect workers did not start in time.");
+        connect.assertions().assertAtLeastNumWorkersAreUp(NUM_WORKERS,
+                "Connect workers did not start in time.");
 
         // start a source connector
-        IntStream.range(0, 4).forEachOrdered(
-            i -> {
-                try {
-                    connect.configureConnector(CONNECTOR_NAME + i, props);
-                } catch (IOException e) {
-                    throw new ConnectException(e);
-                }
-            });
-
-        waitForCondition(() -> this.assertConnectorAndTasksRunning(CONNECTOR_NAME + 3, NUM_TASKS).orElse(false),
-                CONNECTOR_SETUP_DURATION_MS, "Connector tasks did not start in time.");
+        IntStream.range(0, 4).forEachOrdered(i -> connect.configureConnector(CONNECTOR_NAME + i, props));
+
+        connect.assertions().assertConnectorAndAtLeastNumTasksAreRunning(CONNECTOR_NAME + 3, NUM_TASKS,
+                "Connector tasks did not start in time.");
 
         connect.addWorker();
 
-        waitForCondition(() -> this.assertWorkersUp(4),
-                WORKER_SETUP_DURATION_MS, "Connect workers did not start in time.");
+        connect.assertions().assertAtLeastNumWorkersAreUp(NUM_WORKERS + 1,
+                "Connect workers did not start in time.");
 
-        waitForCondition(() -> this.assertConnectorAndTasksRunning(CONNECTOR_NAME + 3, NUM_TASKS).orElse(false),
-                CONNECTOR_SETUP_DURATION_MS, "Connector tasks did not start in time.");
+        connect.assertions().assertConnectorAndAtLeastNumTasksAreRunning(CONNECTOR_NAME + 3, NUM_TASKS,
+                "Connector tasks did not start in time.");
 
         waitForCondition(this::assertConnectorAndTasksAreUnique,
                 WORKER_SETUP_DURATION_MS, "Connect and tasks are imbalanced between the workers.");
@@ -289,69 +279,24 @@ public void testRemovingWorker() throws Exception {
         props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
         props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
 
-        waitForCondition(() -> this.assertWorkersUp(3),
-                WORKER_SETUP_DURATION_MS, "Connect workers did not start in time.");
+        connect.assertions().assertExactlyNumWorkersAreUp(NUM_WORKERS,
+                "Connect workers did not start in time.");
 
         // start a source connector
-        IntStream.range(0, 4).forEachOrdered(
-            i -> {
-                try {
-                    connect.configureConnector(CONNECTOR_NAME + i, props);
-                } catch (IOException e) {
-                    throw new ConnectException(e);
-                }
-            });
-
-        waitForCondition(() -> this.assertConnectorAndTasksRunning(CONNECTOR_NAME + 3, NUM_TASKS).orElse(false),
-                CONNECTOR_SETUP_DURATION_MS, "Connector tasks did not start in time.");
+        IntStream.range(0, 4).forEachOrdered(i -> connect.configureConnector(CONNECTOR_NAME + i, props));
+
+        connect.assertions().assertConnectorAndAtLeastNumTasksAreRunning(CONNECTOR_NAME + 3, NUM_TASKS,
+                "Connector tasks did not start in time.");
 
         connect.removeWorker();
 
-        waitForCondition(() -> this.assertWorkersUp(2),
-                WORKER_SETUP_DURATION_MS, "Connect workers did not start in time.");
+        connect.assertions().assertExactlyNumWorkersAreUp(NUM_WORKERS - 1,
+                "Connect workers did not start in time.");
 
         waitForCondition(this::assertConnectorAndTasksAreUnique,
                 WORKER_SETUP_DURATION_MS, "Connect and tasks are imbalanced between the workers.");
     }
 
-    /**
-     * Confirm that a connector with an exact number of tasks is running.
-     *
-     * @param connectorName the connector
-     * @param numTasks the expected number of tasks
-     * @return true if the connector and tasks are in RUNNING state; false otherwise
-     */
-    private Optional assertConnectorAndTasksRunning(String connectorName, int numTasks) {
-        try {
-            ConnectorStateInfo info = connect.connectorStatus(connectorName);
-            boolean result = info != null
-                    && info.tasks().size() == numTasks
-                    && info.connector().state().equals(AbstractStatus.State.RUNNING.toString())
-                    && info.tasks().stream().allMatch(s -> s.state().equals(AbstractStatus.State.RUNNING.toString()));
-            log.debug("Found connector and tasks running: {}", result);
-            return Optional.of(result);
-        } catch (Exception e) {
-            log.error("Could not check connector state info.", e);
-            return Optional.empty();
-        }
-    }
-
-    /**
-     * Verifies whether the supplied number of workers matches the number of workers
-     * currently running.
-     * @param numWorkers the expected number of active workers
-     * @return true if exactly numWorkers are active; false if more or fewer workers are running
-     */
-    private boolean assertWorkersUp(int numWorkers) {
-        try {
-            int numUp = connect.activeWorkers().size();
-            return numUp == numWorkers;
-        } catch (Exception e) {
-            log.error("Could not check active workers.", e);
-            return false;
-        }
-    }
-
     private boolean assertConnectorAndTasksAreUnique() {
         try {
             Map> connectors = new HashMap<>();
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RestExtensionIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RestExtensionIntegrationTest.java
index 087246b917767..d5328a6ace967 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RestExtensionIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RestExtensionIntegrationTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.connect.integration;
 
+import org.apache.kafka.connect.errors.ConnectException;
 import org.apache.kafka.connect.health.ConnectClusterState;
 import org.apache.kafka.connect.health.ConnectorHealth;
 import org.apache.kafka.connect.health.ConnectorState;
@@ -23,7 +24,6 @@
 import org.apache.kafka.connect.health.TaskState;
 import org.apache.kafka.connect.rest.ConnectRestExtension;
 import org.apache.kafka.connect.rest.ConnectRestExtensionContext;
-import org.apache.kafka.connect.runtime.rest.errors.ConnectRestException;
 import org.apache.kafka.connect.util.clusters.EmbeddedConnectCluster;
 import org.apache.kafka.connect.util.clusters.WorkerHandle;
 import org.apache.kafka.test.IntegrationTest;
@@ -33,12 +33,14 @@
 
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
+import javax.ws.rs.core.Response;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
+import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
 import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLASS_CONFIG;
 import static org.apache.kafka.connect.runtime.ConnectorConfig.NAME_CONFIG;
 import static org.apache.kafka.connect.runtime.ConnectorConfig.TASKS_MAX_CONFIG;
@@ -138,8 +140,9 @@ public void close() {
     private boolean extensionIsRegistered() {
         try {
             String extensionUrl = connect.endpointForResource("integration-test-rest-extension/registered");
-            return "true".equals(connect.executeGet(extensionUrl));
-        } catch (ConnectRestException | IOException e) {
+            Response response = connect.requestGet(extensionUrl);
+            return response.getStatus() < BAD_REQUEST.getStatusCode();
+        } catch (ConnectException e) {
             return false;
         }
     }
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/SessionedProtocolIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/SessionedProtocolIntegrationTest.java
index d01ab3b7c4a8d..1f13c17d32ee5 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/SessionedProtocolIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/SessionedProtocolIntegrationTest.java
@@ -110,7 +110,7 @@ public void ensureInternalEndpointIsSecured() throws Throwable {
         );
         assertEquals(
             BAD_REQUEST.getStatusCode(),
-            connect.executePost(connectorTasksEndpoint, "[]", emptyHeaders)
+            connect.requestPost(connectorTasksEndpoint, "[]", emptyHeaders).getStatus()
         );
 
         // Try again, but with an invalid signature
@@ -121,7 +121,7 @@ public void ensureInternalEndpointIsSecured() throws Throwable {
         );
         assertEquals(
             FORBIDDEN.getStatusCode(),
-            connect.executePost(connectorTasksEndpoint, "[]", invalidSignatureHeaders)
+            connect.requestPost(connectorTasksEndpoint, "[]", invalidSignatureHeaders).getStatus()
         );
 
         // Create the connector now
@@ -151,7 +151,7 @@ public void ensureInternalEndpointIsSecured() throws Throwable {
         );
         assertEquals(
             BAD_REQUEST.getStatusCode(),
-            connect.executePost(connectorTasksEndpoint, "[]", emptyHeaders)
+            connect.requestPost(connectorTasksEndpoint, "[]", emptyHeaders).getStatus()
         );
 
         // Try again, but with an invalid signature
@@ -162,7 +162,7 @@ public void ensureInternalEndpointIsSecured() throws Throwable {
         );
         assertEquals(
             FORBIDDEN.getStatusCode(),
-            connect.executePost(connectorTasksEndpoint, "[]", invalidSignatureHeaders)
+            connect.requestPost(connectorTasksEndpoint, "[]", invalidSignatureHeaders).getStatus()
         );
     }
 }
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java
index 94cc880234663..2276340f8d40c 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java
@@ -25,7 +25,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.servlet.http.HttpServletResponse;
 import javax.ws.rs.core.Response;
 import java.io.IOException;
 import java.io.InputStream;
@@ -33,6 +32,7 @@
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashSet;
@@ -82,6 +82,7 @@ public class EmbeddedConnectCluster {
     private final boolean maskExitProcedures;
     private final String workerNamePrefix;
     private final AtomicInteger nextWorkerId = new AtomicInteger(0);
+    private final EmbeddedConnectClusterAssertions assertions;
 
     private EmbeddedConnectCluster(String name, Map workerProps, int numWorkers,
                                    int numBrokers, Properties brokerProps,
@@ -95,6 +96,7 @@ private EmbeddedConnectCluster(String name, Map workerProps, int
         this.maskExitProcedures = maskExitProcedures;
         // leaving non-configurable for now
         this.workerNamePrefix = DEFAULT_WORKER_NAME_PREFIX;
+        this.assertions = new EmbeddedConnectClusterAssertions(this);
     }
 
     /**
@@ -124,7 +126,7 @@ private EmbeddedConnectCluster(String name, Map workerProps, int
     /**
      * Start the connect cluster and the embedded Kafka and Zookeeper cluster.
      */
-    public void start() throws IOException {
+    public void start() {
         if (maskExitProcedures) {
             Exit.setExitProcedure(exitProcedure);
             Exit.setHaltProcedure(haltProcedure);
@@ -136,6 +138,8 @@ public void start() throws IOException {
     /**
      * Stop the connect cluster and the embedded Kafka and Zookeeper cluster.
      * Clean up any temp directories created locally.
+     *
+     * @throws RuntimeException if Kafka brokers fail to stop
      */
     public void stop() {
         connectCluster.forEach(this::stopWorker);
@@ -182,6 +186,7 @@ public void removeWorker() {
      * Decommission a specific worker from this Connect cluster.
      *
      * @param worker the handle of the worker to remove from the cluster
+     * @throws IllegalStateException if the Connect cluster has no workers
      */
     public void removeWorker(WorkerHandle worker) {
         if (connectCluster.isEmpty()) {
@@ -238,9 +243,9 @@ public Set activeWorkers() {
                 .filter(w -> {
                     try {
                         mapper.readerFor(ServerInfo.class)
-                                .readValue(executeGet(w.url().toString()));
+                                .readValue(responseToString(requestGet(w.url().toString())));
                         return true;
-                    } catch (IOException e) {
+                    } catch (ConnectException | IOException e) {
                         // Worker failed to respond. Consider it's offline
                         return false;
                     }
@@ -263,36 +268,37 @@ public Set workers() {
      *
      * @param connName   the name of the connector
      * @param connConfig the intended configuration
-     * @throws IOException          if call to the REST api fails.
-     * @throws ConnectRestException if REST api returns error status
+     * @throws ConnectRestException if the REST api returns error status
+     * @throws ConnectException if the configuration fails to be serialized or if the request could not be sent
      */
-    public void configureConnector(String connName, Map connConfig) throws IOException {
+    public String configureConnector(String connName, Map connConfig) {
         String url = endpointForResource(String.format("connectors/%s/config", connName));
         ObjectMapper mapper = new ObjectMapper();
-        int status;
+        String content;
         try {
-            String content = mapper.writeValueAsString(connConfig);
-            status = executePut(url, content);
+            content = mapper.writeValueAsString(connConfig);
         } catch (IOException e) {
-            log.error("Could not execute PUT request to " + url, e);
-            throw e;
+            throw new ConnectException("Could not serialize connector configuration and execute PUT request");
         }
-        if (status >= HttpServletResponse.SC_BAD_REQUEST) {
-            throw new ConnectRestException(status, "Could not execute PUT request");
+        Response response = requestPut(url, content);
+        if (response.getStatus() < Response.Status.BAD_REQUEST.getStatusCode()) {
+            return responseToString(response);
         }
+        throw new ConnectRestException(response.getStatus(), "Could not execute PUT request");
     }
 
     /**
      * Delete an existing connector.
      *
      * @param connName name of the connector to be deleted
-     * @throws IOException if call to the REST api fails.
+     * @throws ConnectRestException if the REST api returns error status
+     * @throws ConnectException for any other error.
      */
-    public void deleteConnector(String connName) throws IOException {
+    public void deleteConnector(String connName) {
         String url = endpointForResource(String.format("connectors/%s", connName));
-        int status = executeDelete(url);
-        if (status >= HttpServletResponse.SC_BAD_REQUEST) {
-            throw new ConnectRestException(status, "Could not execute DELETE request.");
+        Response response = requestDelete(url);
+        if (response.getStatus() >= Response.Status.BAD_REQUEST.getStatusCode()) {
+            throw new ConnectRestException(response.getStatus(), "Could not execute DELETE request.");
         }
     }
 
@@ -305,50 +311,78 @@ public void deleteConnector(String connName) throws IOException {
      */
     public Collection connectors() {
         ObjectMapper mapper = new ObjectMapper();
-        try {
-            String url = endpointForResource("connectors");
-            return mapper.readerFor(Collection.class).readValue(executeGet(url));
-        } catch (IOException e) {
-            log.error("Could not read connector list", e);
-            throw new ConnectException("Could not read connector list", e);
+        String url = endpointForResource("connectors");
+        Response response = requestGet(url);
+        if (response.getStatus() < Response.Status.BAD_REQUEST.getStatusCode()) {
+            try {
+                return mapper.readerFor(Collection.class).readValue(responseToString(response));
+            } catch (IOException e) {
+                log.error("Could not parse connector list from response: {}",
+                        responseToString(response), e
+                );
+                throw new ConnectException("Could not not parse connector list", e);
+            }
         }
+        throw new ConnectRestException(response.getStatus(),
+                "Could not read connector list. Error response: " + responseToString(response));
     }
 
     /**
      * Get the status for a connector running in this cluster.
      *
      * @param connectorName name of the connector
-     * @return an instance of {@link ConnectorStateInfo} populated with state informaton of the connector and it's tasks.
+     * @return an instance of {@link ConnectorStateInfo} populated with state information of the connector and its tasks.
      * @throws ConnectRestException if the HTTP request to the REST API failed with a valid status code.
      * @throws ConnectException for any other error.
      */
     public ConnectorStateInfo connectorStatus(String connectorName) {
         ObjectMapper mapper = new ObjectMapper();
+        String url = endpointForResource(String.format("connectors/%s/status", connectorName));
+        Response response = requestGet(url);
         try {
-            String url = endpointForResource(String.format("connectors/%s/status", connectorName));
-            return mapper.readerFor(ConnectorStateInfo.class).readValue(executeGet(url));
+            if (response.getStatus() < Response.Status.BAD_REQUEST.getStatusCode()) {
+                return mapper.readerFor(ConnectorStateInfo.class)
+                        .readValue(responseToString(response));
+            }
         } catch (IOException e) {
-            log.error("Could not read connector state", e);
-            throw new ConnectException("Could not read connector state", e);
+            log.error("Could not read connector state from response: {}",
+                    responseToString(response), e);
+            throw new ConnectException("Could not not parse connector state", e);
         }
+        throw new ConnectRestException(response.getStatus(),
+                "Could not read connector state. Error response: " + responseToString(response));
     }
 
-    public String adminEndpoint(String resource) throws IOException {
+    /**
+     * Get the full URL of the admin endpoint that corresponds to the given REST resource
+     *
+     * @param resource the resource under the worker's admin endpoint
+     * @return the admin endpoint URL
+     * @throws ConnectRestException if no admin REST endpoint is available
+     */
+    public String adminEndpoint(String resource) {
         String url = connectCluster.stream()
                 .map(WorkerHandle::adminUrl)
                 .filter(Objects::nonNull)
                 .findFirst()
-                .orElseThrow(() -> new IOException("Admin endpoint is disabled."))
+                .orElseThrow(() -> new ConnectException("Admin endpoint is disabled."))
                 .toString();
         return url + resource;
     }
 
-    public String endpointForResource(String resource) throws IOException {
+    /**
+     * Get the full URL of the endpoint that corresponds to the given REST resource
+     *
+     * @param resource the resource under the worker's admin endpoint
+     * @return the admin endpoint URL
+     * @throws ConnectRestException if no REST endpoint is available
+     */
+    public String endpointForResource(String resource) {
         String url = connectCluster.stream()
                 .map(WorkerHandle::url)
                 .filter(Objects::nonNull)
                 .findFirst()
-                .orElseThrow(() -> new IOException("Connect workers have not been provisioned"))
+                .orElseThrow(() -> new ConnectException("Connect workers have not been provisioned"))
                 .toString();
         return url + resource;
     }
@@ -359,91 +393,161 @@ private static void putIfAbsent(Map props, String propertyKey, S
         }
     }
 
+    /**
+     * Return the handle to the Kafka cluster this Connect cluster connects to.
+     *
+     * @return the Kafka cluster handle
+     */
     public EmbeddedKafkaCluster kafka() {
         return kafkaCluster;
     }
 
-    public int executePut(String url, String body) throws IOException {
-        log.debug("Executing PUT request to URL={}. Payload={}", url, body);
-        HttpURLConnection httpCon = (HttpURLConnection) new URL(url).openConnection();
-        httpCon.setDoOutput(true);
-        httpCon.setRequestProperty("Content-Type", "application/json");
-        httpCon.setRequestMethod("PUT");
-        try (OutputStreamWriter out = new OutputStreamWriter(httpCon.getOutputStream())) {
-            out.write(body);
-        }
-        if (httpCon.getResponseCode() < HttpURLConnection.HTTP_BAD_REQUEST) {
-            try (InputStream is = httpCon.getInputStream()) {
-                log.info("PUT response for URL={} is {}", url, responseToString(is));
-            }
-        } else {
-            try (InputStream is = httpCon.getErrorStream()) {
-                log.info("PUT error response for URL={} is {}", url, responseToString(is));
-            }
-        }
-        return httpCon.getResponseCode();
+    /**
+     * Execute a GET request on the given URL.
+     *
+     * @param url the HTTP endpoint
+     * @return the response to the GET request
+     * @throws ConnectException if execution of the GET request fails
+     * @deprecated Use {@link #requestGet(String)} instead.
+     */
+    @Deprecated
+    public String executeGet(String url) {
+        return responseToString(requestGet(url));
     }
 
-    public int executePost(String url, String body, Map headers) throws IOException {
-        log.debug("Executing POST request to URL={}. Payload={}", url, body);
-        HttpURLConnection httpCon = (HttpURLConnection) new URL(url).openConnection();
-        httpCon.setDoOutput(true);
-        httpCon.setRequestProperty("Content-Type", "application/json");
-        headers.forEach(httpCon::setRequestProperty);
-        httpCon.setRequestMethod("POST");
-        try (OutputStreamWriter out = new OutputStreamWriter(httpCon.getOutputStream())) {
-            out.write(body);
-        }
-        if (httpCon.getResponseCode() < HttpURLConnection.HTTP_BAD_REQUEST) {
-            try (InputStream is = httpCon.getInputStream()) {
-                log.info("POST response for URL={} is {}", url, responseToString(is));
-            }
-        } else {
-            try (InputStream is = httpCon.getErrorStream()) {
-                log.info("POST error response for URL={} is {}", url, responseToString(is));
-            }
-        }
-        return httpCon.getResponseCode();
+    /**
+     * Execute a GET request on the given URL.
+     *
+     * @param url the HTTP endpoint
+     * @return the response to the GET request
+     * @throws ConnectException if execution of the GET request fails
+     */
+    public Response requestGet(String url) {
+        return requestHttpMethod(url, null, Collections.emptyMap(), "GET");
     }
 
     /**
-     * Execute a GET request on the given URL.
+     * Execute a PUT request on the given URL.
+     *
+     * @param url the HTTP endpoint
+     * @param body the payload of the PUT request
+     * @return the response to the PUT request
+     * @throws ConnectException if execution of the PUT request fails
+     * @deprecated Use {@link #requestPut(String, String)} instead.
+     */
+    @Deprecated
+    public int executePut(String url, String body) {
+        return requestPut(url, body).getStatus();
+    }
+
+    /**
+     * Execute a PUT request on the given URL.
+     *
+     * @param url the HTTP endpoint
+     * @param body the payload of the PUT request
+     * @return the response to the PUT request
+     * @throws ConnectException if execution of the PUT request fails
+     */
+    public Response requestPut(String url, String body) {
+        return requestHttpMethod(url, body, Collections.emptyMap(), "PUT");
+    }
+
+    /**
+     * Execute a POST request on the given URL.
+     *
+     * @param url the HTTP endpoint
+     * @param body the payload of the POST request
+     * @param headers a map that stores the POST request headers
+     * @return the response to the POST request
+     * @throws ConnectException if execution of the POST request fails
+     * @deprecated Use {@link #requestPost(String, String, java.util.Map)} instead.
+     */
+    @Deprecated
+    public int executePost(String url, String body, Map headers) {
+        return requestPost(url, body, headers).getStatus();
+    }
+
+    /**
+     * Execute a POST request on the given URL.
+     *
+     * @param url the HTTP endpoint
+     * @param body the payload of the POST request
+     * @param headers a map that stores the POST request headers
+     * @return the response to the POST request
+     * @throws ConnectException if execution of the POST request fails
+     */
+    public Response requestPost(String url, String body, Map headers) {
+        return requestHttpMethod(url, body, headers, "POST");
+    }
+
+    /**
+     * Execute a DELETE request on the given URL.
      *
      * @param url the HTTP endpoint
-     * @return response body encoded as a String
-     * @throws ConnectRestException if the HTTP request fails with a valid status code
-     * @throws IOException for any other I/O error.
+     * @return the response to the DELETE request
+     * @throws ConnectException if execution of the DELETE request fails
+     * @deprecated Use {@link #requestDelete(String)} instead.
      */
-    public String executeGet(String url) throws IOException {
-        log.debug("Executing GET request to URL={}.", url);
-        HttpURLConnection httpCon = (HttpURLConnection) new URL(url).openConnection();
-        httpCon.setDoOutput(true);
-        httpCon.setRequestMethod("GET");
-        try (InputStream is = httpCon.getInputStream()) {
-            int c;
-            StringBuilder response = new StringBuilder();
-            while ((c = is.read()) != -1) {
-                response.append((char) c);
+    @Deprecated
+    public int executeDelete(String url) {
+        return requestDelete(url).getStatus();
+    }
+
+    /**
+     * Execute a DELETE request on the given URL.
+     *
+     * @param url the HTTP endpoint
+     * @return the response to the DELETE request
+     * @throws ConnectException if execution of the DELETE request fails
+     */
+    public Response requestDelete(String url) {
+        return requestHttpMethod(url, null, Collections.emptyMap(), "DELETE");
+    }
+
+    /**
+     * A general method that executes an HTTP request on a given URL.
+     *
+     * @param url the HTTP endpoint
+     * @param body the payload of the request; null if there isn't one
+     * @param headers a map that stores the request headers; empty if there are no headers
+     * @param httpMethod the name of the HTTP method to execute
+     * @return the response to the HTTP request
+     * @throws ConnectException if execution of the HTTP method fails
+     */
+    protected Response requestHttpMethod(String url, String body, Map headers,
+                                      String httpMethod) {
+        log.debug("Executing {} request to URL={}." + (body != null ? " Payload={}" : ""),
+                httpMethod, url, body);
+        try {
+            HttpURLConnection httpCon = (HttpURLConnection) new URL(url).openConnection();
+            httpCon.setDoOutput(true);
+            httpCon.setRequestMethod(httpMethod);
+            if (body != null) {
+                httpCon.setRequestProperty("Content-Type", "application/json");
+                headers.forEach(httpCon::setRequestProperty);
+                try (OutputStreamWriter out = new OutputStreamWriter(httpCon.getOutputStream())) {
+                    out.write(body);
+                }
             }
-            log.debug("GET response for URL={} is {}", url, response);
-            return response.toString();
-        } catch (IOException e) {
-            Response.Status status = Response.Status.fromStatusCode(httpCon.getResponseCode());
-            if (status != null) {
-                throw new ConnectRestException(status, "Invalid endpoint: " + url, e);
+            try (InputStream is = httpCon.getResponseCode() < HttpURLConnection.HTTP_BAD_REQUEST
+                                  ? httpCon.getInputStream()
+                                  : httpCon.getErrorStream()
+            ) {
+                String responseEntity = responseToString(is);
+                log.info("{} response for URL={} is {}",
+                        httpMethod, url, responseEntity.isEmpty() ? "empty" : responseEntity);
+                return Response.status(Response.Status.fromStatusCode(httpCon.getResponseCode()))
+                        .entity(responseEntity)
+                        .build();
             }
-            // invalid response code, re-throw the IOException.
-            throw e;
+        } catch (IOException e) {
+            log.error("Could not execute " + httpMethod + " request to " + url, e);
+            throw new ConnectException(e);
         }
     }
 
-    public int executeDelete(String url) throws IOException {
-        log.debug("Executing DELETE request to URL={}", url);
-        HttpURLConnection httpCon = (HttpURLConnection) new URL(url).openConnection();
-        httpCon.setDoOutput(true);
-        httpCon.setRequestMethod("DELETE");
-        httpCon.connect();
-        return httpCon.getResponseCode();
+    private String responseToString(Response response) {
+        return response == null ? "empty" : (String) response.getEntity();
     }
 
     private String responseToString(InputStream stream) throws IOException {
@@ -511,4 +615,13 @@ public EmbeddedConnectCluster build() {
         }
     }
 
+    /**
+     * Return the available assertions for this Connect cluster
+     *
+     * @return the assertions object
+     */
+    public EmbeddedConnectClusterAssertions assertions() {
+        return assertions;
+    }
+
 }
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectClusterAssertions.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectClusterAssertions.java
new file mode 100644
index 0000000000000..4b13d89b315cc
--- /dev/null
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectClusterAssertions.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.util.clusters;
+
+import org.apache.kafka.connect.runtime.AbstractStatus;
+import org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo;
+import org.apache.kafka.connect.runtime.rest.errors.ConnectRestException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.Response;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiFunction;
+
+import static org.apache.kafka.test.TestUtils.waitForCondition;
+
+/**
+ * A set of common assertions that can be applied to a Connect cluster during integration testing
+ */
+public class EmbeddedConnectClusterAssertions {
+
+    private static final Logger log = LoggerFactory.getLogger(EmbeddedConnectClusterAssertions.class);
+    public static final long WORKER_SETUP_DURATION_MS = TimeUnit.SECONDS.toMillis(60);
+    private static final long CONNECTOR_SETUP_DURATION_MS = TimeUnit.SECONDS.toMillis(30);
+
+    private final EmbeddedConnectCluster connect;
+
+    EmbeddedConnectClusterAssertions(EmbeddedConnectCluster connect) {
+        this.connect = connect;
+    }
+
+    /**
+     * Assert that at least the requested number of workers are up and running.
+     *
+     * @param numWorkers the number of online workers
+     */
+    public void assertAtLeastNumWorkersAreUp(int numWorkers, String detailMessage) throws InterruptedException {
+        try {
+            waitForCondition(
+                () -> checkWorkersUp(numWorkers, (actual, expected) -> actual >= expected).orElse(false),
+                WORKER_SETUP_DURATION_MS,
+                "Didn't meet the minimum requested number of online workers: " + numWorkers);
+        } catch (AssertionError e) {
+            throw new AssertionError(detailMessage, e);
+        }
+    }
+
+    /**
+     * Assert that at least the requested number of workers are up and running.
+     *
+     * @param numWorkers the number of online workers
+     */
+    public void assertExactlyNumWorkersAreUp(int numWorkers, String detailMessage) throws InterruptedException {
+        try {
+            waitForCondition(
+                () -> checkWorkersUp(numWorkers, (actual, expected) -> actual == expected).orElse(false),
+                WORKER_SETUP_DURATION_MS,
+                "Didn't meet the exact requested number of online workers: " + numWorkers);
+        } catch (AssertionError e) {
+            throw new AssertionError(detailMessage, e);
+        }
+    }
+
+    /**
+     * Confirm that the requested number of workers are up and running.
+     *
+     * @param numWorkers the number of online workers
+     * @return true if at least {@code numWorkers} are up; false otherwise
+     */
+    protected Optional checkWorkersUp(int numWorkers, BiFunction comp) {
+        try {
+            int numUp = connect.activeWorkers().size();
+            return Optional.of(comp.apply(numUp, numWorkers));
+        } catch (Exception e) {
+            log.error("Could not check active workers.", e);
+            return Optional.empty();
+        }
+    }
+
+    /**
+     * Assert that a connector is running with at least the given number of tasks all in running state
+     *
+     * @param connectorName the connector name
+     * @param numTasks the number of tasks
+     * @param detailMessage
+     * @throws InterruptedException
+     */
+    public void assertConnectorAndAtLeastNumTasksAreRunning(String connectorName, int numTasks, String detailMessage)
+            throws InterruptedException {
+        try {
+            waitForCondition(
+                () -> checkConnectorState(
+                    connectorName,
+                    AbstractStatus.State.RUNNING,
+                    numTasks,
+                    AbstractStatus.State.RUNNING,
+                    (actual, expected) -> actual >= expected
+                ).orElse(false),
+                CONNECTOR_SETUP_DURATION_MS,
+                "The connector or at least " + numTasks + " of tasks are not running.");
+        } catch (AssertionError e) {
+            throw new AssertionError(detailMessage, e);
+        }
+    }
+
+    /**
+     * Assert that a connector is running with at least the given number of tasks all in running state
+     *
+     * @param connectorName the connector name
+     * @param numTasks the number of tasks
+     * @param detailMessage the assertion message
+     * @throws InterruptedException
+     */
+    public void assertConnectorAndExactlyNumTasksAreRunning(String connectorName, int numTasks, String detailMessage)
+            throws InterruptedException {
+        try {
+            waitForCondition(
+                () -> checkConnectorState(
+                    connectorName,
+                    AbstractStatus.State.RUNNING,
+                    numTasks,
+                    AbstractStatus.State.RUNNING,
+                    (actual, expected) -> actual == expected
+                ).orElse(false),
+                CONNECTOR_SETUP_DURATION_MS,
+                "The connector or exactly " + numTasks + " tasks are not running.");
+        } catch (AssertionError e) {
+            throw new AssertionError(detailMessage, e);
+        }
+    }
+
+    /**
+     * Assert that a connector is running, that it has a specific number of tasks, and that all of
+     * its tasks are in the FAILED state.
+     *
+     * @param connectorName the connector name
+     * @param numTasks the number of tasks
+     * @param detailMessage the assertion message
+     * @throws InterruptedException
+     */
+    public void assertConnectorIsRunningAndTasksHaveFailed(String connectorName, int numTasks, String detailMessage)
+            throws InterruptedException {
+        try {
+            waitForCondition(
+                () -> checkConnectorState(
+                    connectorName,
+                    AbstractStatus.State.RUNNING,
+                    numTasks,
+                    AbstractStatus.State.FAILED,
+                    (actual, expected) -> actual >= expected
+                ).orElse(false),
+                CONNECTOR_SETUP_DURATION_MS,
+                "Either the connector is not running or not all the " + numTasks + " tasks have failed.");
+        } catch (AssertionError e) {
+            throw new AssertionError(detailMessage, e);
+        }
+    }
+
+    /**
+     * Assert that a connector and its tasks are not running.
+     *
+     * @param connectorName the connector name
+     * @param detailMessage the assertion message
+     * @throws InterruptedException
+     */
+    public void assertConnectorAndTasksAreStopped(String connectorName, String detailMessage)
+            throws InterruptedException {
+        try {
+            waitForCondition(
+                () -> checkConnectorAndTasksAreStopped(connectorName),
+                CONNECTOR_SETUP_DURATION_MS,
+                "At least the connector or one of its tasks is still");
+        } catch (AssertionError e) {
+            throw new AssertionError(detailMessage, e);
+        }
+    }
+
+    /**
+     * Check whether the connector or any of its tasks are still in RUNNING state
+     *
+     * @param connectorName the connector
+     * @return true if the connector and all the tasks are not in RUNNING state; false otherwise
+     */
+    protected boolean checkConnectorAndTasksAreStopped(String connectorName) {
+        ConnectorStateInfo info;
+        try {
+            info = connect.connectorStatus(connectorName);
+        } catch (ConnectRestException e) {
+            return e.statusCode() == Response.Status.NOT_FOUND.getStatusCode();
+        } catch (Exception e) {
+            log.error("Could not check connector state info.", e);
+            return false;
+        }
+        if (info == null) {
+            return true;
+        }
+        return !info.connector().state().equals(AbstractStatus.State.RUNNING.toString())
+                && info.tasks().stream().noneMatch(s -> s.state().equals(AbstractStatus.State.RUNNING.toString()));
+    }
+
+    /**
+     * Check whether the given connector state matches the current state of the connector and
+     * whether it has at least the given number of tasks, with all the tasks matching the given
+     * task state.
+     * @param connectorName the connector
+     * @param connectorState
+     * @param numTasks the expected number of tasks
+     * @param tasksState
+     * @return true if the connector and tasks are in RUNNING state; false otherwise
+     */
+    protected Optional checkConnectorState(
+            String connectorName,
+            AbstractStatus.State connectorState,
+            int numTasks,
+            AbstractStatus.State tasksState,
+            BiFunction comp
+    ) {
+        try {
+            ConnectorStateInfo info = connect.connectorStatus(connectorName);
+            boolean result = info != null
+                    && comp.apply(info.tasks().size(), numTasks)
+                    && info.connector().state().equals(connectorState.toString())
+                    && info.tasks().stream().allMatch(s -> s.state().equals(tasksState.toString()));
+            return Optional.of(result);
+        } catch (Exception e) {
+            log.error("Could not check connector state info.", e);
+            return Optional.empty();
+        }
+    }
+
+}
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedKafkaCluster.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedKafkaCluster.java
index d36ae5b078e18..b365e651ee473 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedKafkaCluster.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedKafkaCluster.java
@@ -97,7 +97,7 @@ public EmbeddedKafkaCluster(final int numBrokers,
     }
 
     @Override
-    protected void before() throws IOException {
+    protected void before() {
         start();
     }
 
@@ -106,11 +106,17 @@ protected void after() {
         stop();
     }
 
-    public void startOnlyKafkaOnSamePorts() throws IOException {
+    /**
+     * Starts the Kafka cluster alone using the ports that were assigned during initialization of
+     * the harness.
+     *
+     * @throws ConnectException if a directory to store the data cannot be created
+     */
+    public void startOnlyKafkaOnSamePorts() {
         start(currentBrokerPorts, currentBrokerLogDirs);
     }
 
-    private void start() throws IOException {
+    private void start() {
         // pick a random port
         zookeeper = new EmbeddedZookeeper();
         Arrays.fill(currentBrokerPorts, 0);
@@ -118,7 +124,7 @@ private void start() throws IOException {
         start(currentBrokerPorts, currentBrokerLogDirs);
     }
 
-    private void start(int[] brokerPorts, String[] logDirs) throws IOException {
+    private void start(int[] brokerPorts, String[] logDirs) {
         brokerConfig.put(KafkaConfig$.MODULE$.ZkConnectProp(), zKConnectString());
 
         putIfAbsent(brokerConfig, KafkaConfig$.MODULE$.HostNameProp(), "localhost");
@@ -205,10 +211,15 @@ private static void putIfAbsent(final Properties props, final String propertyKey
         }
     }
 
-    private String createLogDir() throws IOException {
+    private String createLogDir() {
         TemporaryFolder tmpFolder = new TemporaryFolder();
-        tmpFolder.create();
-        return tmpFolder.newFolder().getAbsolutePath();
+        try {
+            tmpFolder.create();
+            return tmpFolder.newFolder().getAbsolutePath();
+        } catch (IOException e) {
+            log.error("Unable to create temporary log directory", e);
+            throw new ConnectException("Unable to create temporary log directory", e);
+        }
     }
 
     public String bootstrapServers() {

From 4a0859abba5d3d43f55eb72bd1fe421145b48274 Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Tue, 11 Feb 2020 17:34:00 -0600
Subject: [PATCH 0877/1071] KAFKA-9390: Make serde pseudo-topics unique (#8054)

During the discussion for KIP-213, we decided to pass "pseudo-topics"
to the internal serdes we use to construct the wrapper serdes for
CombinedKey and hashing the left-hand-side value. However, during
the implementation, this strategy wasn't fully implemented, and we wound
up using the same topic name for a few different data types.

Cherry-pick of e16859dc48c679b3c7d9735438df046479b8ec4a from trunk
Cherry-pick of 557ad5517dd290cc76b26cfa585e126e629534c9 from 2.5

Reviewers: Guozhang Wang 
---
 .../streams/kstream/internals/KTableImpl.java |  23 ++-
 .../foreignkeyjoin/CombinedKeySchema.java     |  23 ++-
 ...JoinSubscriptionSendProcessorSupplier.java |  17 +-
 .../SubscriptionWrapperSerde.java             |  35 +++--
 ...reignKeyInnerJoinMultiIntegrationTest.java |  32 +++-
 ...eKTableForeignKeyJoinDefaultSerdeTest.java |   5 +-
 ...leKTableForeignKeyJoinIntegrationTest.java |  16 +-
 .../foreignkeyjoin/CombinedKeySchemaTest.java |  18 ++-
 .../SubscriptionWrapperSerdeTest.java         |   8 +-
 .../streams/utils/UniqueTopicSerdeScope.java  | 148 ++++++++++++++++++
 10 files changed, 272 insertions(+), 53 deletions(-)
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/utils/UniqueTopicSerdeScope.java

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
index ef4a323df6472..6eb8823f4e403 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
@@ -952,23 +952,34 @@ private  KTable doJoinOnForeignKey(final KTable forei
         //This occurs whenever the extracted foreignKey changes values.
         enableSendingOldValues();
 
-        final Serde foreignKeySerde = ((KTableImpl) foreignKeyTable).keySerde;
-        final Serde> subscriptionWrapperSerde = new SubscriptionWrapperSerde<>(keySerde);
-        final SubscriptionResponseWrapperSerde responseWrapperSerde =
-            new SubscriptionResponseWrapperSerde<>(((KTableImpl) foreignKeyTable).valSerde);
 
 
         final NamedInternal renamed = new NamedInternal(joinName);
         final String subscriptionTopicName = renamed.suffixWithOrElseGet("-subscription-registration", builder, SUBSCRIPTION_REGISTRATION) + TOPIC_SUFFIX;
+        final String subscriptionPrimaryKeySerdePseudoTopic = subscriptionTopicName + "-pk";
+        final String subscriptionForeignKeySerdePseudoTopic = subscriptionTopicName + "-fk";
+        final String valueHashSerdePseudoTopic = subscriptionTopicName + "-vh";
         builder.internalTopologyBuilder.addInternalTopic(subscriptionTopicName);
-        final CombinedKeySchema combinedKeySchema = new CombinedKeySchema<>(subscriptionTopicName, foreignKeySerde, keySerde);
+
+        final Serde foreignKeySerde = ((KTableImpl) foreignKeyTable).keySerde;
+        final Serde> subscriptionWrapperSerde = new SubscriptionWrapperSerde<>(subscriptionPrimaryKeySerdePseudoTopic, keySerde);
+        final SubscriptionResponseWrapperSerde responseWrapperSerde =
+            new SubscriptionResponseWrapperSerde<>(((KTableImpl) foreignKeyTable).valSerde);
+
+        final CombinedKeySchema combinedKeySchema = new CombinedKeySchema<>(
+            subscriptionForeignKeySerdePseudoTopic,
+            foreignKeySerde,
+            subscriptionPrimaryKeySerdePseudoTopic,
+            keySerde
+        );
 
         final ProcessorGraphNode> subscriptionNode = new ProcessorGraphNode<>(
             new ProcessorParameters<>(
                 new ForeignJoinSubscriptionSendProcessorSupplier<>(
                     foreignKeyExtractor,
+                    subscriptionForeignKeySerdePseudoTopic,
+                    valueHashSerdePseudoTopic,
                     foreignKeySerde,
-                    subscriptionTopicName,
                     valSerde == null ? null : valSerde.serializer(),
                     leftJoin
                 ),
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchema.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchema.java
index 7cda404e01d9e..92fb72c7ae6b9 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchema.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchema.java
@@ -28,14 +28,19 @@
  * Factory for creating CombinedKey serializers / deserializers.
  */
 public class CombinedKeySchema {
-    private final String serdeTopic;
+    private final String primaryKeySerdeTopic;
+    private final String foreignKeySerdeTopic;
     private Serializer primaryKeySerializer;
     private Deserializer primaryKeyDeserializer;
     private Serializer foreignKeySerializer;
     private Deserializer foreignKeyDeserializer;
 
-    public CombinedKeySchema(final String serdeTopic, final Serde foreignKeySerde, final Serde primaryKeySerde) {
-        this.serdeTopic = serdeTopic;
+    public CombinedKeySchema(final String foreignKeySerdeTopic,
+                             final Serde foreignKeySerde,
+                             final String primaryKeySerdeTopic,
+                             final Serde primaryKeySerde) {
+        this.primaryKeySerdeTopic = primaryKeySerdeTopic;
+        this.foreignKeySerdeTopic = foreignKeySerdeTopic;
         primaryKeySerializer = primaryKeySerde == null ? null : primaryKeySerde.serializer();
         primaryKeyDeserializer = primaryKeySerde == null ? null : primaryKeySerde.deserializer();
         foreignKeyDeserializer = foreignKeySerde == null ? null : foreignKeySerde.deserializer();
@@ -54,10 +59,12 @@ Bytes toBytes(final KO foreignKey, final K primaryKey) {
         //The serialization format - note that primaryKeySerialized may be null, such as when a prefixScan
         //key is being created.
         //{Integer.BYTES foreignKeyLength}{foreignKeySerialized}{Optional-primaryKeySerialized}
-        final byte[] foreignKeySerializedData = foreignKeySerializer.serialize(serdeTopic, foreignKey);
+        final byte[] foreignKeySerializedData = foreignKeySerializer.serialize(foreignKeySerdeTopic,
+                                                                               foreignKey);
 
         //? bytes
-        final byte[] primaryKeySerializedData = primaryKeySerializer.serialize(serdeTopic, primaryKey);
+        final byte[] primaryKeySerializedData = primaryKeySerializer.serialize(primaryKeySerdeTopic,
+                                                                               primaryKey);
 
         final ByteBuffer buf = ByteBuffer.allocate(Integer.BYTES + foreignKeySerializedData.length + primaryKeySerializedData.length);
         buf.putInt(foreignKeySerializedData.length);
@@ -74,11 +81,11 @@ public CombinedKey fromBytes(final Bytes data) {
         final int foreignKeyLength = dataBuffer.getInt();
         final byte[] foreignKeyRaw = new byte[foreignKeyLength];
         dataBuffer.get(foreignKeyRaw, 0, foreignKeyLength);
-        final KO foreignKey = foreignKeyDeserializer.deserialize(serdeTopic, foreignKeyRaw);
+        final KO foreignKey = foreignKeyDeserializer.deserialize(foreignKeySerdeTopic, foreignKeyRaw);
 
         final byte[] primaryKeyRaw = new byte[dataArray.length - foreignKeyLength - Integer.BYTES];
         dataBuffer.get(primaryKeyRaw, 0, primaryKeyRaw.length);
-        final K primaryKey = primaryKeyDeserializer.deserialize(serdeTopic, primaryKeyRaw);
+        final K primaryKey = primaryKeyDeserializer.deserialize(primaryKeySerdeTopic, primaryKeyRaw);
         return new CombinedKey<>(foreignKey, primaryKey);
     }
 
@@ -86,7 +93,7 @@ Bytes prefixBytes(final KO key) {
         //The serialization format. Note that primaryKeySerialized is not required/used in this function.
         //{Integer.BYTES foreignKeyLength}{foreignKeySerialized}{Optional-primaryKeySerialized}
 
-        final byte[] foreignKeySerializedData = foreignKeySerializer.serialize(serdeTopic, key);
+        final byte[] foreignKeySerializedData = foreignKeySerializer.serialize(foreignKeySerdeTopic, key);
 
         final ByteBuffer buf = ByteBuffer.allocate(Integer.BYTES + foreignKeySerializedData.length);
         buf.putInt(foreignKeySerializedData.length);
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionSendProcessorSupplier.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionSendProcessorSupplier.java
index dcf5b2a98c873..0bd0d88cb09e1 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionSendProcessorSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionSendProcessorSupplier.java
@@ -43,20 +43,23 @@ public class ForeignJoinSubscriptionSendProcessorSupplier implements P
     private static final Logger LOG = LoggerFactory.getLogger(ForeignJoinSubscriptionSendProcessorSupplier.class);
 
     private final Function foreignKeyExtractor;
-    private final String repartitionTopicName;
+    private final String foreignKeySerdeTopic;
+    private final String valueSerdeTopic;
     private final boolean leftJoin;
     private Serializer foreignKeySerializer;
     private Serializer valueSerializer;
 
     public ForeignJoinSubscriptionSendProcessorSupplier(final Function foreignKeyExtractor,
+                                                        final String foreignKeySerdeTopic,
+                                                        final String valueSerdeTopic,
                                                         final Serde foreignKeySerde,
-                                                        final String repartitionTopicName,
                                                         final Serializer valueSerializer,
                                                         final boolean leftJoin) {
         this.foreignKeyExtractor = foreignKeyExtractor;
+        this.foreignKeySerdeTopic = foreignKeySerdeTopic;
+        this.valueSerdeTopic = valueSerdeTopic;
         this.valueSerializer = valueSerializer;
         this.leftJoin = leftJoin;
-        this.repartitionTopicName = repartitionTopicName;
         foreignKeySerializer = foreignKeySerde == null ? null : foreignKeySerde.serializer();
     }
 
@@ -88,7 +91,7 @@ public void init(final ProcessorContext context) {
         public void process(final K key, final Change change) {
             final long[] currentHash = change.newValue == null ?
                 null :
-                Murmur3.hash128(valueSerializer.serialize(repartitionTopicName, change.newValue));
+                Murmur3.hash128(valueSerializer.serialize(valueSerdeTopic, change.newValue));
 
             if (change.oldValue != null) {
                 final KO oldForeignKey = foreignKeyExtractor.apply(change.oldValue);
@@ -111,8 +114,10 @@ change.newValue, context().topic(), context().partition(), context().offset()
                         return;
                     }
 
-                    final byte[] serialOldForeignKey = foreignKeySerializer.serialize(repartitionTopicName, oldForeignKey);
-                    final byte[] serialNewForeignKey = foreignKeySerializer.serialize(repartitionTopicName, newForeignKey);
+                    final byte[] serialOldForeignKey =
+                        foreignKeySerializer.serialize(foreignKeySerdeTopic, oldForeignKey);
+                    final byte[] serialNewForeignKey =
+                        foreignKeySerializer.serialize(foreignKeySerdeTopic, newForeignKey);
                     if (!Arrays.equals(serialNewForeignKey, serialOldForeignKey)) {
                         //Different Foreign Key - delete the old key value and propagate the new one.
                         //Delete it from the oldKey's state store
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerde.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerde.java
index 1cb32935fbee1..42aed940ef4bc 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerde.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerde.java
@@ -30,9 +30,12 @@ public class SubscriptionWrapperSerde implements Serde
     private final SubscriptionWrapperSerializer serializer;
     private final SubscriptionWrapperDeserializer deserializer;
 
-    public SubscriptionWrapperSerde(final Serde primaryKeySerde) {
-        serializer = new SubscriptionWrapperSerializer<>(primaryKeySerde == null ? null : primaryKeySerde.serializer());
-        deserializer = new SubscriptionWrapperDeserializer<>(primaryKeySerde == null ? null : primaryKeySerde.deserializer());
+    public SubscriptionWrapperSerde(final String primaryKeySerializationPseudoTopic,
+                                    final Serde primaryKeySerde) {
+        serializer = new SubscriptionWrapperSerializer<>(primaryKeySerializationPseudoTopic,
+                                                         primaryKeySerde == null ? null : primaryKeySerde.serializer());
+        deserializer = new SubscriptionWrapperDeserializer<>(primaryKeySerializationPseudoTopic,
+                                                             primaryKeySerde == null ? null : primaryKeySerde.deserializer());
     }
 
     @Override
@@ -45,12 +48,15 @@ public Deserializer> deserializer() {
         return deserializer;
     }
 
-    public static class SubscriptionWrapperSerializer
+    private static class SubscriptionWrapperSerializer
         implements Serializer>, WrappingNullableSerializer, K> {
 
+        private final String primaryKeySerializationPseudoTopic;
         private Serializer primaryKeySerializer;
 
-        SubscriptionWrapperSerializer(final Serializer primaryKeySerializer) {
+        SubscriptionWrapperSerializer(final String primaryKeySerializationPseudoTopic,
+                                      final Serializer primaryKeySerializer) {
+            this.primaryKeySerializationPseudoTopic = primaryKeySerializationPseudoTopic;
             this.primaryKeySerializer = primaryKeySerializer;
         }
 
@@ -62,7 +68,7 @@ public void setIfUnset(final Serializer defaultSerializer) {
         }
 
         @Override
-        public byte[] serialize(final String topic, final SubscriptionWrapper data) {
+        public byte[] serialize(final String ignored, final SubscriptionWrapper data) {
             //{1-bit-isHashNull}{7-bits-version}{1-byte-instruction}{Optional-16-byte-Hash}{PK-serialized}
 
             //7-bit (0x7F) maximum for data version.
@@ -70,7 +76,10 @@ public byte[] serialize(final String topic, final SubscriptionWrapper data) {
                 throw new UnsupportedVersionException("SubscriptionWrapper version is larger than maximum supported 0x7F");
             }
 
-            final byte[] primaryKeySerializedData = primaryKeySerializer.serialize(topic, data.getPrimaryKey());
+            final byte[] primaryKeySerializedData = primaryKeySerializer.serialize(
+                primaryKeySerializationPseudoTopic,
+                data.getPrimaryKey()
+            );
 
             final ByteBuffer buf;
             if (data.getHash() != null) {
@@ -94,12 +103,15 @@ public byte[] serialize(final String topic, final SubscriptionWrapper data) {
 
     }
 
-    public static class SubscriptionWrapperDeserializer
+    private static class SubscriptionWrapperDeserializer
         implements Deserializer>, WrappingNullableDeserializer, K> {
 
+        private final String primaryKeySerializationPseudoTopic;
         private Deserializer primaryKeyDeserializer;
 
-        SubscriptionWrapperDeserializer(final Deserializer primaryKeyDeserializer) {
+        SubscriptionWrapperDeserializer(final String primaryKeySerializationPseudoTopic,
+                                        final Deserializer primaryKeyDeserializer) {
+            this.primaryKeySerializationPseudoTopic = primaryKeySerializationPseudoTopic;
             this.primaryKeyDeserializer = primaryKeyDeserializer;
         }
 
@@ -111,7 +123,7 @@ public void setIfUnset(final Deserializer defaultDeserializer) {
         }
 
         @Override
-        public SubscriptionWrapper deserialize(final String topic, final byte[] data) {
+        public SubscriptionWrapper deserialize(final String ignored, final byte[] data) {
             //{7-bits-version}{1-bit-isHashNull}{1-byte-instruction}{Optional-16-byte-Hash}{PK-serialized}
             final ByteBuffer buf = ByteBuffer.wrap(data);
             final byte versionAndIsHashNull = buf.get();
@@ -132,7 +144,8 @@ public SubscriptionWrapper deserialize(final String topic, final byte[] data)
 
             final byte[] primaryKeyRaw = new byte[data.length - lengthSum]; //The remaining data is the serialized pk
             buf.get(primaryKeyRaw, 0, primaryKeyRaw.length);
-            final K primaryKey = primaryKeyDeserializer.deserialize(topic, primaryKeyRaw);
+            final K primaryKey = primaryKeyDeserializer.deserialize(primaryKeySerializationPseudoTopic,
+                                                                    primaryKeyRaw);
 
             return new SubscriptionWrapper<>(hash, inst, primaryKey, version);
         }
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyInnerJoinMultiIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyInnerJoinMultiIntegrationTest.java
index ad746d8f611c6..c7cb7120e0704 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyInnerJoinMultiIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyInnerJoinMultiIntegrationTest.java
@@ -39,6 +39,7 @@
 import org.apache.kafka.streams.kstream.Produced;
 import org.apache.kafka.streams.kstream.ValueJoiner;
 import org.apache.kafka.streams.state.KeyValueStore;
+import org.apache.kafka.streams.utils.UniqueTopicSerdeScope;
 import org.apache.kafka.test.IntegrationTest;
 import org.apache.kafka.test.TestUtils;
 import org.junit.After;
@@ -206,17 +207,30 @@ private void verifyKTableKTableJoin(final JoinType joinType,
     }
 
     private KafkaStreams prepareTopology(final String queryableName, final String queryableNameTwo) {
+        final UniqueTopicSerdeScope serdeScope = new UniqueTopicSerdeScope();
         final StreamsBuilder builder = new StreamsBuilder();
 
-        final KTable table1 = builder.table(TABLE_1, Consumed.with(Serdes.Integer(), Serdes.Float()));
-        final KTable table2 = builder.table(TABLE_2, Consumed.with(Serdes.String(), Serdes.Long()));
-        final KTable table3 = builder.table(TABLE_3, Consumed.with(Serdes.Integer(), Serdes.String()));
+        final KTable table1 = builder.table(
+            TABLE_1,
+            Consumed.with(serdeScope.decorateSerde(Serdes.Integer(), streamsConfig, true),
+                          serdeScope.decorateSerde(Serdes.Float(), streamsConfig, false))
+        );
+        final KTable table2 = builder.table(
+            TABLE_2,
+            Consumed.with(serdeScope.decorateSerde(Serdes.String(), streamsConfig, true),
+                          serdeScope.decorateSerde(Serdes.Long(), streamsConfig, false))
+        );
+        final KTable table3 = builder.table(
+            TABLE_3,
+            Consumed.with(serdeScope.decorateSerde(Serdes.Integer(), streamsConfig, true),
+                          serdeScope.decorateSerde(Serdes.String(), streamsConfig, false))
+        );
 
         final Materialized> materialized;
         if (queryableName != null) {
             materialized = Materialized.>as(queryableName)
-                    .withKeySerde(Serdes.Integer())
-                    .withValueSerde(Serdes.String())
+                    .withKeySerde(serdeScope.decorateSerde(Serdes.Integer(), streamsConfig, true))
+                    .withValueSerde(serdeScope.decorateSerde(Serdes.String(), streamsConfig, false))
                     .withCachingDisabled();
         } else {
             throw new RuntimeException("Current implementation of joinOnForeignKey requires a materialized store");
@@ -225,8 +239,8 @@ private KafkaStreams prepareTopology(final String queryableName, final String qu
         final Materialized> materializedTwo;
         if (queryableNameTwo != null) {
             materializedTwo = Materialized.>as(queryableNameTwo)
-                    .withKeySerde(Serdes.Integer())
-                    .withValueSerde(Serdes.String())
+                    .withKeySerde(serdeScope.decorateSerde(Serdes.Integer(), streamsConfig, true))
+                    .withValueSerde(serdeScope.decorateSerde(Serdes.String(), streamsConfig, false))
                     .withCachingDisabled();
         } else {
             throw new RuntimeException("Current implementation of joinOnForeignKey requires a materialized store");
@@ -247,7 +261,9 @@ private KafkaStreams prepareTopology(final String queryableName, final String qu
         table1.join(table2, tableOneKeyExtractor, joiner, materialized)
               .join(table3, joinedTableKeyExtractor, joinerTwo, materializedTwo)
             .toStream()
-            .to(OUTPUT, Produced.with(Serdes.Integer(), Serdes.String()));
+            .to(OUTPUT,
+                Produced.with(serdeScope.decorateSerde(Serdes.Integer(), streamsConfig, true),
+                              serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)));
 
         return new KafkaStreams(builder.build(streamsConfig), streamsConfig);
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinDefaultSerdeTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinDefaultSerdeTest.java
index df6349a8510e5..5af0530ca53a0 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinDefaultSerdeTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinDefaultSerdeTest.java
@@ -30,11 +30,13 @@
 import org.apache.kafka.streams.kstream.Materialized;
 import org.apache.kafka.streams.kstream.Produced;
 import org.apache.kafka.streams.state.KeyValueStore;
+import org.apache.kafka.test.TestUtils;
 import org.junit.Test;
 
 import java.util.Collections;
 import java.util.Map;
 import java.util.Properties;
+import java.util.UUID;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.is;
@@ -161,10 +163,11 @@ public void shouldWorkWithDefaultAndProducedSerdes() {
 
     private static void validateTopologyCanProcessData(final StreamsBuilder builder) {
         final Properties config = new Properties();
-        config.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "dummy");
+        config.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "dummy-" + UUID.randomUUID());
         config.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy");
         config.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.StringSerde.class.getName());
         config.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.StringSerde.class.getName());
+        config.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
         try (final TopologyTestDriver topologyTestDriver = new TopologyTestDriver(builder.build(), config)) {
             final TestInputTopic aTopic = topologyTestDriver.createInputTopic("A", new StringSerializer(), new StringSerializer());
             final TestInputTopic bTopic = topologyTestDriver.createInputTopic("B", new StringSerializer(), new StringSerializer());
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinIntegrationTest.java
index 746d6b3195b5c..db0e2de138412 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinIntegrationTest.java
@@ -32,6 +32,7 @@
 import org.apache.kafka.streams.kstream.ValueJoiner;
 import org.apache.kafka.streams.state.KeyValueStore;
 import org.apache.kafka.streams.state.Stores;
+import org.apache.kafka.streams.utils.UniqueTopicSerdeScope;
 import org.apache.kafka.test.TestUtils;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -506,17 +507,26 @@ private static Map asMap(final KeyValueStore sto
     private static Topology getTopology(final Properties streamsConfig,
                                         final String queryableStoreName,
                                         final boolean leftJoin) {
+        final UniqueTopicSerdeScope serdeScope = new UniqueTopicSerdeScope();
         final StreamsBuilder builder = new StreamsBuilder();
 
-        final KTable left = builder.table(LEFT_TABLE, Consumed.with(Serdes.String(), Serdes.String()));
-        final KTable right = builder.table(RIGHT_TABLE, Consumed.with(Serdes.String(), Serdes.String()));
+        final KTable left = builder.table(
+            LEFT_TABLE,
+            Consumed.with(serdeScope.decorateSerde(Serdes.String(), streamsConfig, true),
+                          serdeScope.decorateSerde(Serdes.String(), streamsConfig, false))
+        );
+        final KTable right = builder.table(
+            RIGHT_TABLE,
+            Consumed.with(serdeScope.decorateSerde(Serdes.String(), streamsConfig, true),
+                          serdeScope.decorateSerde(Serdes.String(), streamsConfig, false))
+        );
 
         final Function extractor = value -> value.split("\\|")[1];
         final ValueJoiner joiner = (value1, value2) -> "(" + value1 + "," + value2 + ")";
 
         final Materialized> materialized =
             Materialized.as(Stores.inMemoryKeyValueStore(queryableStoreName))
-                .withValueSerde(Serdes.String())
+                .withValueSerde(serdeScope.decorateSerde(Serdes.String(), streamsConfig, false))
                 // the cache suppresses some of the unnecessary tombstones we want to make assertions about
                 .withCachingDisabled();
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchemaTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchemaTest.java
index 47348038521ca..cb1ef596db3f0 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchemaTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchemaTest.java
@@ -28,7 +28,8 @@ public class CombinedKeySchemaTest {
 
     @Test
     public void nonNullPrimaryKeySerdeTest() {
-        final CombinedKeySchema cks = new CombinedKeySchema<>("someTopic", Serdes.String(), Serdes.Integer());
+        final CombinedKeySchema cks = new CombinedKeySchema<>("fkTopic", Serdes.String(),
+                                                                               "pkTopic", Serdes.Integer());
         final Integer primary = -999;
         final Bytes result = cks.toBytes("foreignKey", primary);
 
@@ -39,21 +40,25 @@ public void nonNullPrimaryKeySerdeTest() {
 
     @Test(expected = NullPointerException.class)
     public void nullPrimaryKeySerdeTest() {
-        final CombinedKeySchema cks = new CombinedKeySchema<>("someTopic", Serdes.String(), Serdes.Integer());
+        final CombinedKeySchema cks = new CombinedKeySchema<>("fkTopic", Serdes.String(),
+                                                                               "pkTopic", Serdes.Integer());
         cks.toBytes("foreignKey", null);
     }
 
     @Test(expected = NullPointerException.class)
     public void nullForeignKeySerdeTest() {
-        final CombinedKeySchema cks = new CombinedKeySchema<>("someTopic", Serdes.String(), Serdes.Integer());
+        final CombinedKeySchema cks = new CombinedKeySchema<>("fkTopic", Serdes.String(),
+                                                                               "pkTopic", Serdes.Integer());
         cks.toBytes(null, 10);
     }
 
     @Test
     public void prefixKeySerdeTest() {
-        final CombinedKeySchema cks = new CombinedKeySchema<>("someTopic", Serdes.String(), Serdes.Integer());
+        final CombinedKeySchema cks = new CombinedKeySchema<>("fkTopic", Serdes.String(),
+                                                                               "pkTopic", Serdes.Integer());
         final String foreignKey = "someForeignKey";
-        final byte[] foreignKeySerializedData = Serdes.String().serializer().serialize("someTopic", foreignKey);
+        final byte[] foreignKeySerializedData =
+            Serdes.String().serializer().serialize("fkTopic", foreignKey);
         final Bytes prefix = cks.prefixBytes(foreignKey);
 
         final ByteBuffer buf = ByteBuffer.allocate(Integer.BYTES + foreignKeySerializedData.length);
@@ -66,7 +71,8 @@ public void prefixKeySerdeTest() {
 
     @Test(expected = NullPointerException.class)
     public void nullPrefixKeySerdeTest() {
-        final CombinedKeySchema cks = new CombinedKeySchema<>("someTopic", Serdes.String(), Serdes.Integer());
+        final CombinedKeySchema cks = new CombinedKeySchema<>("fkTopic", Serdes.String(),
+                                                                               "pkTopic", Serdes.Integer());
         final String foreignKey = null;
         cks.prefixBytes(foreignKey);
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerdeTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerdeTest.java
index d948d1f2a35fb..5c6551c3e7db5 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerdeTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerdeTest.java
@@ -31,7 +31,7 @@ public class SubscriptionWrapperSerdeTest {
     @SuppressWarnings("unchecked")
     public void shouldSerdeTest() {
         final String originalKey = "originalKey";
-        final SubscriptionWrapperSerde swSerde = new SubscriptionWrapperSerde<>(Serdes.String());
+        final SubscriptionWrapperSerde swSerde = new SubscriptionWrapperSerde<>("pkTopic", Serdes.String());
         final long[] hashedValue = Murmur3.hash128(new byte[] {(byte) 0xFF, (byte) 0xAA, (byte) 0x00, (byte) 0x19});
         final SubscriptionWrapper wrapper = new SubscriptionWrapper<>(hashedValue, SubscriptionWrapper.Instruction.DELETE_KEY_AND_PROPAGATE, originalKey);
         final byte[] serialized = swSerde.serializer().serialize(null, wrapper);
@@ -46,7 +46,7 @@ public void shouldSerdeTest() {
     @SuppressWarnings("unchecked")
     public void shouldSerdeNullHashTest() {
         final String originalKey = "originalKey";
-        final SubscriptionWrapperSerde swSerde = new SubscriptionWrapperSerde<>(Serdes.String());
+        final SubscriptionWrapperSerde swSerde = new SubscriptionWrapperSerde<>("pkTopic", Serdes.String());
         final long[] hashedValue = null;
         final SubscriptionWrapper wrapper = new SubscriptionWrapper<>(hashedValue, SubscriptionWrapper.Instruction.PROPAGATE_ONLY_IF_FK_VAL_AVAILABLE, originalKey);
         final byte[] serialized = swSerde.serializer().serialize(null, wrapper);
@@ -61,7 +61,7 @@ public void shouldSerdeNullHashTest() {
     @SuppressWarnings("unchecked")
     public void shouldThrowExceptionOnNullKeyTest() {
         final String originalKey = null;
-        final SubscriptionWrapperSerde swSerde = new SubscriptionWrapperSerde<>(Serdes.String());
+        final SubscriptionWrapperSerde swSerde = new SubscriptionWrapperSerde<>("pkTopic", Serdes.String());
         final long[] hashedValue = Murmur3.hash128(new byte[] {(byte) 0xFF, (byte) 0xAA, (byte) 0x00, (byte) 0x19});
         final SubscriptionWrapper wrapper = new SubscriptionWrapper<>(hashedValue, SubscriptionWrapper.Instruction.PROPAGATE_ONLY_IF_FK_VAL_AVAILABLE, originalKey);
         swSerde.serializer().serialize(null, wrapper);
@@ -71,7 +71,7 @@ public void shouldThrowExceptionOnNullKeyTest() {
     @SuppressWarnings("unchecked")
     public void shouldThrowExceptionOnNullInstructionTest() {
         final String originalKey = "originalKey";
-        final SubscriptionWrapperSerde swSerde = new SubscriptionWrapperSerde<>(Serdes.String());
+        final SubscriptionWrapperSerde swSerde = new SubscriptionWrapperSerde<>("pkTopic", Serdes.String());
         final long[] hashedValue = Murmur3.hash128(new byte[] {(byte) 0xFF, (byte) 0xAA, (byte) 0x00, (byte) 0x19});
         final SubscriptionWrapper wrapper = new SubscriptionWrapper<>(hashedValue, null, originalKey);
         swSerde.serializer().serialize(null, wrapper);
diff --git a/streams/src/test/java/org/apache/kafka/streams/utils/UniqueTopicSerdeScope.java b/streams/src/test/java/org/apache/kafka/streams/utils/UniqueTopicSerdeScope.java
new file mode 100644
index 0000000000000..04b1a7b2a786b
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/utils/UniqueTopicSerdeScope.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.utils;
+
+import org.apache.kafka.common.header.Headers;
+import org.apache.kafka.common.serialization.Deserializer;
+import org.apache.kafka.common.serialization.Serde;
+import org.apache.kafka.common.serialization.Serializer;
+
+import java.util.Map;
+import java.util.Properties;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+
+public class UniqueTopicSerdeScope {
+    private final Map> topicTypeRegistry = new TreeMap<>();
+
+    public  UniqueTopicSerdeDecorator decorateSerde(final Serde delegate,
+                                                          final Properties config,
+                                                          final boolean isKey) {
+        final UniqueTopicSerdeDecorator decorator = new UniqueTopicSerdeDecorator<>(delegate);
+        decorator.configure(config.entrySet().stream().collect(Collectors.toMap(e -> e.getKey().toString(), Map.Entry::getValue)), isKey);
+        return decorator;
+    }
+
+    public class UniqueTopicSerdeDecorator implements Serde {
+        private final AtomicBoolean isKey = new AtomicBoolean(false);
+        private final Serde delegate;
+
+        public UniqueTopicSerdeDecorator(final Serde delegate) {
+            this.delegate = delegate;
+        }
+
+        @Override
+        public void configure(final Map configs, final boolean isKey) {
+            delegate.configure(configs, isKey);
+            this.isKey.set(isKey);
+        }
+
+        @Override
+        public void close() {
+            delegate.close();
+        }
+
+        @Override
+        public Serializer serializer() {
+            return new UniqueTopicSerializerDecorator<>(isKey, delegate.serializer());
+        }
+
+        @Override
+        public Deserializer deserializer() {
+            return new UniqueTopicDeserializerDecorator<>(isKey, delegate.deserializer());
+        }
+    }
+
+    public class UniqueTopicSerializerDecorator implements Serializer {
+        private final AtomicBoolean isKey;
+        private final Serializer delegate;
+
+        public UniqueTopicSerializerDecorator(final AtomicBoolean isKey, final Serializer delegate) {
+            this.isKey = isKey;
+            this.delegate = delegate;
+        }
+
+        @Override
+        public void configure(final Map configs, final boolean isKey) {
+            delegate.configure(configs, isKey);
+            this.isKey.set(isKey);
+        }
+
+        @Override
+        public byte[] serialize(final String topic, final T data) {
+            verifyTopic(topic, data);
+            return delegate.serialize(topic, data);
+        }
+
+        @Override
+        public byte[] serialize(final String topic, final Headers headers, final T data) {
+            verifyTopic(topic, data);
+            return delegate.serialize(topic, headers, data);
+        }
+
+        private void verifyTopic(final String topic, final T data) {
+            if (data != null) {
+                final String key = topic + (isKey.get() ? "--key" : "--value");
+                if (topicTypeRegistry.containsKey(key)) {
+                    assertThat(String.format("key[%s] data[%s][%s]", key, data, data.getClass()), topicTypeRegistry.get(key), equalTo(data.getClass()));
+                } else {
+                    topicTypeRegistry.put(key, data.getClass());
+                }
+            }
+        }
+
+        @Override
+        public void close() {
+            delegate.close();
+        }
+    }
+
+    public class UniqueTopicDeserializerDecorator implements Deserializer {
+        private final AtomicBoolean isKey;
+        private final Deserializer delegate;
+
+        public UniqueTopicDeserializerDecorator(final AtomicBoolean isKey, final Deserializer delegate) {
+            this.isKey = isKey;
+            this.delegate = delegate;
+        }
+
+        @Override
+        public void configure(final Map configs, final boolean isKey) {
+            delegate.configure(configs, isKey);
+            this.isKey.set(isKey);
+        }
+
+        @Override
+        public T deserialize(final String topic, final byte[] data) {
+            return delegate.deserialize(topic, data);
+        }
+
+        @Override
+        public T deserialize(final String topic, final Headers headers, final byte[] data) {
+            return delegate.deserialize(topic, headers, data);
+        }
+
+        @Override
+        public void close() {
+            delegate.close();
+        }
+    }
+}

From 42062a89f2a7333263c8aff3e67b0eba47b29377 Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Tue, 11 Feb 2020 21:00:17 -0600
Subject: [PATCH 0878/1071] KAKFA-9503: Fix TopologyTestDriver output order
 (#8065)

Migrates TopologyTestDriver processing to be closer to the same processing/ordering
semantics as KafkaStreams. This corrects the output order for recursive topologies
as reported in KAFKA-9503, and also works similarly in the case of task idling.

Cherry-pick of 998f1520f9af2dddfec9a9ac072f8dcf9d9004fd from trunk
Cherry-pick of 7b71cb92b539a547a99dc6dd094f475fd55a8572 from 2.5

Reviewers: Matthias J. Sax 
---
 .../processor/internals/StreamTask.java       |   6 +-
 .../kafka/streams/TopologyTestDriver.java     | 219 ++++++++++++------
 .../kafka/streams/TopologyTestDriverTest.java | 178 +++++++++++++-
 .../src/test/resources/log4j.properties       |  21 ++
 4 files changed, 340 insertions(+), 84 deletions(-)
 create mode 100644 streams/test-utils/src/test/resources/log4j.properties

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
index cf161ac1fd8c6..fe0d7e3f31d28 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
@@ -382,7 +382,7 @@ public void resume() {
      * An active task is processable if its buffer contains data for all of its input
      * source topic partitions, or if it is enforced to be processable
      */
-    boolean isProcessable(final long now) {
+    public boolean isProcessable(final long now) {
         if (partitionGroup.allPartitionsBuffered()) {
             idleStartTime = RecordQueue.UNKNOWN;
             return true;
@@ -1004,4 +1004,8 @@ private Map extractPartitionTimes() {
         }
         return partitionTimes;
     }
+
+    public boolean hasRecordsQueued() {
+        return numBuffered() > 0;
+    }
 }
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
index 39d40a8abc875..b9e6d4b4dd722 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
@@ -211,9 +211,9 @@ public class TopologyTestDriver implements Closeable {
     private final MockProducer producer;
 
     private final Set internalTopics = new HashSet<>();
-    private final Map partitionsByTopic = new HashMap<>();
-    private final Map globalPartitionsByTopic = new HashMap<>();
-    private final Map offsetsByTopicPartition = new HashMap<>();
+    private final Map partitionsByInputTopic = new HashMap<>();
+    private final Map globalPartitionsByInputTopic = new HashMap<>();
+    private final Map offsetsByTopicOrPatternPartition = new HashMap<>();
 
     private final Map>> outputRecordsByTopic = new HashMap<>();
     private final boolean eosEnabled;
@@ -274,6 +274,7 @@ private TopologyTestDriver(final InternalTopologyBuilder builder,
                                final Properties config,
                                final long initialWallClockTimeMs) {
         final StreamsConfig streamsConfig = new QuietStreamsConfig(config);
+        logIfTaskIdleEnabled(streamsConfig);
         mockWallClockTime = new MockTime(initialWallClockTimeMs);
 
         internalTopologyBuilder = builder;
@@ -343,16 +344,16 @@ public void onRestoreEnd(final TopicPartition topicPartition, final String store
 
         for (final String topic : processorTopology.sourceTopics()) {
             final TopicPartition tp = new TopicPartition(topic, PARTITION_ID);
-            partitionsByTopic.put(topic, tp);
-            offsetsByTopicPartition.put(tp, new AtomicLong());
+            partitionsByInputTopic.put(topic, tp);
+            offsetsByTopicOrPatternPartition.put(tp, new AtomicLong());
         }
-        consumer.assign(partitionsByTopic.values());
+        consumer.assign(partitionsByInputTopic.values());
 
         if (globalTopology != null) {
             for (final String topicName : globalTopology.sourceTopics()) {
                 final TopicPartition partition = new TopicPartition(topicName, 0);
-                globalPartitionsByTopic.put(topicName, partition);
-                offsetsByTopicPartition.put(partition, new AtomicLong());
+                globalPartitionsByInputTopic.put(topicName, partition);
+                offsetsByTopicOrPatternPartition.put(partition, new AtomicLong());
                 consumer.updatePartitions(topicName, Collections.singletonList(
                     new PartitionInfo(topicName, 0, null, null, null)));
                 consumer.updateBeginningOffsets(Collections.singletonMap(partition, 0L));
@@ -390,10 +391,10 @@ public void onRestoreEnd(final TopicPartition topicPartition, final String store
             globalStateTask = null;
         }
 
-        if (!partitionsByTopic.isEmpty()) {
+        if (!partitionsByInputTopic.isEmpty()) {
             task = new StreamTask(
                 TASK_ID,
-                new HashSet<>(partitionsByTopic.values()),
+                new HashSet<>(partitionsByInputTopic.values()),
                 processorTopology,
                 consumer,
                 new StoreChangelogReader(
@@ -421,6 +422,20 @@ public void onRestoreEnd(final TopicPartition topicPartition, final String store
         eosEnabled = streamsConfig.getString(StreamsConfig.PROCESSING_GUARANTEE_CONFIG).equals(StreamsConfig.EXACTLY_ONCE);
     }
 
+    private static void logIfTaskIdleEnabled(final StreamsConfig streamsConfig) {
+        final Long taskIdleTime = streamsConfig.getLong(StreamsConfig.MAX_TASK_IDLE_MS_CONFIG);
+        if (taskIdleTime > 0) {
+            log.info("Detected {} config in use with TopologyTestDriver (set to {}ms)." +
+                         " This means you might need to use TopologyTestDriver#advanceWallClockTime()" +
+                         " or enqueue records on all partitions to allow Steams to make progress." +
+                         " TopologyTestDriver will log a message each time it cannot process enqueued" +
+                         " records due to {}.",
+                     StreamsConfig.MAX_TASK_IDLE_MS_CONFIG,
+                     taskIdleTime,
+                     StreamsConfig.MAX_TASK_IDLE_MS_CONFIG);
+        }
+    }
+
     /**
      * Get read-only handle on global metrics registry.
      *
@@ -448,77 +463,114 @@ public void pipeInput(final ConsumerRecord consumerRecord) {
             consumerRecord.headers());
     }
 
-    private void pipeRecord(final ProducerRecord record) {
-        pipeRecord(record.topic(), record.timestamp(), record.key(), record.value(), record.headers());
-    }
-
     private void pipeRecord(final String topicName,
-                            final Long timestamp,
+                            final long timestamp,
                             final byte[] key,
                             final byte[] value,
                             final Headers headers) {
+        final TopicPartition inputTopicOrPatternPartition = getInputTopicOrPatternPartition(topicName);
+        final TopicPartition globalInputTopicPartition = globalPartitionsByInputTopic.get(topicName);
 
-        if (!internalTopologyBuilder.sourceTopicNames().isEmpty()) {
-            validateSourceTopicNameRegexPattern(topicName);
+        if (inputTopicOrPatternPartition == null && globalInputTopicPartition == null) {
+            throw new IllegalArgumentException("Unknown topic: " + topicName);
         }
-        final TopicPartition topicPartition = getTopicPartition(topicName);
-        if (topicPartition != null) {
-            final long offset = offsetsByTopicPartition.get(topicPartition).incrementAndGet() - 1;
-            task.addRecords(topicPartition, Collections.singleton(new ConsumerRecord<>(
-                    topicName,
-                    topicPartition.partition(),
-                    offset,
-                    timestamp,
-                    TimestampType.CREATE_TIME,
-                    (long) ConsumerRecord.NULL_CHECKSUM,
-                    key == null ? ConsumerRecord.NULL_SIZE : key.length,
-                    value == null ? ConsumerRecord.NULL_SIZE : value.length,
-                    key,
-                    value,
-                    headers)));
-
-            // Process the record ...
-            task.process();
-            task.maybePunctuateStreamTime();
-            task.commit();
-            captureOutputRecords();
-        } else {
-            final TopicPartition globalTopicPartition = globalPartitionsByTopic.get(topicName);
-            if (globalTopicPartition == null) {
-                throw new IllegalArgumentException("Unknown topic: " + topicName);
+
+        if (inputTopicOrPatternPartition != null) {
+            enqueueTaskRecord(topicName, inputTopicOrPatternPartition, timestamp, key, value, headers);
+            completeAllProcessableWork();
+        }
+
+        if (globalInputTopicPartition != null) {
+            processGlobalRecord(globalInputTopicPartition, timestamp, key, value, headers);
+        }
+    }
+
+    private void enqueueTaskRecord(final String inputTopic,
+                                   final TopicPartition topicOrPatternPartition,
+                                   final long timestamp,
+                                   final byte[] key,
+                                   final byte[] value,
+                                   final Headers headers) {
+        task.addRecords(topicOrPatternPartition, Collections.singleton(new ConsumerRecord<>(
+            inputTopic,
+            topicOrPatternPartition.partition(),
+            offsetsByTopicOrPatternPartition.get(topicOrPatternPartition).incrementAndGet() - 1,
+            timestamp,
+            TimestampType.CREATE_TIME,
+            (long) ConsumerRecord.NULL_CHECKSUM,
+            key == null ? ConsumerRecord.NULL_SIZE : key.length,
+            value == null ? ConsumerRecord.NULL_SIZE : value.length,
+            key,
+            value,
+            headers)));
+    }
+
+    private void completeAllProcessableWork() {
+        // for internally triggered processing (like wall-clock punctuations),
+        // we might have buffered some records to internal topics that need to
+        // be piped back in to kick-start the processing loop. This is idempotent
+        // and therefore harmless in the case where all we've done is enqueued an
+        // input record from the user.
+        captureOutputsAndReEnqueueInternalResults();
+
+        // If the topology only has global tasks, then `task` would be null.
+        // For this method, it just means there's nothing to do.
+        if (task != null) {
+            while (task.hasRecordsQueued()) {
+                // Process the record ...
+                task.process();
+                task.maybePunctuateStreamTime();
+                task.commit();
+                captureOutputsAndReEnqueueInternalResults();
+            }
+            if (task.hasRecordsQueued()) {
+                log.info("Due to the {} configuration, there are currently some records" +
+                             " that cannot be processed. Advancing wall-clock time or" +
+                             " enqueuing records on the empty topics will allow" +
+                             " Streams to process more.",
+                         StreamsConfig.MAX_TASK_IDLE_MS_CONFIG);
             }
-            final long offset = offsetsByTopicPartition.get(globalTopicPartition).incrementAndGet() - 1;
-            globalStateTask.update(new ConsumerRecord<>(
-                    globalTopicPartition.topic(),
-                    globalTopicPartition.partition(),
-                    offset,
-                    timestamp,
-                    TimestampType.CREATE_TIME,
-                    (long) ConsumerRecord.NULL_CHECKSUM,
-                    key == null ? ConsumerRecord.NULL_SIZE : key.length,
-                    value == null ? ConsumerRecord.NULL_SIZE : value.length,
-                    key,
-                    value,
-                    headers));
-            globalStateTask.flushState();
         }
     }
 
+    private void processGlobalRecord(final TopicPartition globalInputTopicPartition,
+                                     final long timestamp,
+                                     final byte[] key,
+                                     final byte[] value,
+                                     final Headers headers) {
+        globalStateTask.update(new ConsumerRecord<>(
+            globalInputTopicPartition.topic(),
+            globalInputTopicPartition.partition(),
+            offsetsByTopicOrPatternPartition.get(globalInputTopicPartition).incrementAndGet() - 1,
+            timestamp,
+            TimestampType.CREATE_TIME,
+            (long) ConsumerRecord.NULL_CHECKSUM,
+            key == null ? ConsumerRecord.NULL_SIZE : key.length,
+            value == null ? ConsumerRecord.NULL_SIZE : value.length,
+            key,
+            value,
+            headers));
+        globalStateTask.flushState();
+    }
 
     private void validateSourceTopicNameRegexPattern(final String inputRecordTopic) {
         for (final String sourceTopicName : internalTopologyBuilder.sourceTopicNames()) {
             if (!sourceTopicName.equals(inputRecordTopic) && Pattern.compile(sourceTopicName).matcher(inputRecordTopic).matches()) {
                 throw new TopologyException("Topology add source of type String for topic: " + sourceTopicName +
-                        " cannot contain regex pattern for input record topic: " + inputRecordTopic +
-                        " and hence cannot process the message.");
+                                                " cannot contain regex pattern for input record topic: " + inputRecordTopic +
+                                                " and hence cannot process the message.");
             }
         }
     }
 
-    private TopicPartition getTopicPartition(final String topicName) {
-        final TopicPartition topicPartition = partitionsByTopic.get(topicName);
+    private TopicPartition getInputTopicOrPatternPartition(final String topicName) {
+        if (!internalTopologyBuilder.sourceTopicNames().isEmpty()) {
+            validateSourceTopicNameRegexPattern(topicName);
+        }
+
+        final TopicPartition topicPartition = partitionsByInputTopic.get(topicName);
         if (topicPartition == null) {
-            for (final Map.Entry entry : partitionsByTopic.entrySet()) {
+            for (final Map.Entry entry : partitionsByInputTopic.entrySet()) {
                 if (Pattern.compile(entry.getKey()).matcher(topicName).matches()) {
                     return entry.getValue();
                 }
@@ -527,7 +579,7 @@ private TopicPartition getTopicPartition(final String topicName) {
         return topicPartition;
     }
 
-    private void captureOutputRecords() {
+    private void captureOutputsAndReEnqueueInternalResults() {
         // Capture all the records sent to the producer ...
         final List> output = producer.history();
         producer.clear();
@@ -540,9 +592,27 @@ private void captureOutputRecords() {
 
             // Forward back into the topology if the produced record is to an internal or a source topic ...
             final String outputTopicName = record.topic();
-            if (internalTopics.contains(outputTopicName) || processorTopology.sourceTopics().contains(outputTopicName)
-                || globalPartitionsByTopic.containsKey(outputTopicName)) {
-                pipeRecord(record);
+
+            final TopicPartition inputTopicOrPatternPartition = getInputTopicOrPatternPartition(outputTopicName);
+            final TopicPartition globalInputTopicPartition = globalPartitionsByInputTopic.get(outputTopicName);
+
+            if (inputTopicOrPatternPartition != null) {
+                enqueueTaskRecord(
+                    outputTopicName,
+                    inputTopicOrPatternPartition,
+                    record.timestamp(),
+                    record.key(),
+                    record.value(),
+                    record.headers());
+            }
+
+            if (globalInputTopicPartition != null) {
+                processGlobalRecord(
+                    globalInputTopicPartition,
+                    record.timestamp(),
+                    record.key(),
+                    record.value(),
+                    record.headers());
             }
         }
     }
@@ -589,7 +659,7 @@ public void advanceWallClockTime(final Duration advance) {
             task.maybePunctuateSystemTime();
             task.commit();
         }
-        captureOutputRecords();
+        completeAllProcessableWork();
     }
 
     /**
@@ -839,23 +909,23 @@ private StateStore getStateStore(final String name,
     private void throwIfBuiltInStore(final StateStore stateStore) {
         if (stateStore instanceof TimestampedKeyValueStore) {
             throw new IllegalArgumentException("Store " + stateStore.name()
-                + " is a timestamped key-value store and should be accessed via `getTimestampedKeyValueStore()`");
+                                                   + " is a timestamped key-value store and should be accessed via `getTimestampedKeyValueStore()`");
         }
         if (stateStore instanceof ReadOnlyKeyValueStore) {
             throw new IllegalArgumentException("Store " + stateStore.name()
-                + " is a key-value store and should be accessed via `getKeyValueStore()`");
+                                                   + " is a key-value store and should be accessed via `getKeyValueStore()`");
         }
         if (stateStore instanceof TimestampedWindowStore) {
             throw new IllegalArgumentException("Store " + stateStore.name()
-                + " is a timestamped window store and should be accessed via `getTimestampedWindowStore()`");
+                                                   + " is a timestamped window store and should be accessed via `getTimestampedWindowStore()`");
         }
         if (stateStore instanceof ReadOnlyWindowStore) {
             throw new IllegalArgumentException("Store " + stateStore.name()
-                + " is a window store and should be accessed via `getWindowStore()`");
+                                                   + " is a window store and should be accessed via `getWindowStore()`");
         }
         if (stateStore instanceof ReadOnlySessionStore) {
             throw new IllegalArgumentException("Store " + stateStore.name()
-                + " is a session store and should be accessed via `getSessionStore()`");
+                                                   + " is a session store and should be accessed via `getSessionStore()`");
         }
     }
 
@@ -1001,7 +1071,12 @@ public void close() {
                 // ignore
             }
         }
-        captureOutputRecords();
+        completeAllProcessableWork();
+        if (task != null && task.hasRecordsQueued()) {
+            log.warn("Found some records that cannot be processed due to the" +
+                         " {} configuration during TopologyTestDriver#close().",
+                     StreamsConfig.MAX_TASK_IDLE_MS_CONFIG);
+        }
         if (!eosEnabled) {
             producer.close();
         }
diff --git a/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java b/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java
index d7ac6b46557ea..160c3fb7ffe48 100644
--- a/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java
+++ b/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java
@@ -33,6 +33,7 @@
 import org.apache.kafka.streams.errors.TopologyException;
 import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.Materialized;
+import org.apache.kafka.streams.processor.AbstractProcessor;
 import org.apache.kafka.streams.processor.Processor;
 import org.apache.kafka.streams.processor.ProcessorContext;
 import org.apache.kafka.streams.processor.ProcessorSupplier;
@@ -40,6 +41,7 @@
 import org.apache.kafka.streams.processor.Punctuator;
 import org.apache.kafka.streams.processor.StateStore;
 import org.apache.kafka.streams.processor.TaskId;
+import org.apache.kafka.streams.processor.To;
 import org.apache.kafka.streams.state.KeyValueBytesStoreSupplier;
 import org.apache.kafka.streams.state.KeyValueIterator;
 import org.apache.kafka.streams.state.KeyValueStore;
@@ -73,6 +75,8 @@
 import static org.apache.kafka.common.utils.Utils.mkMap;
 import static org.apache.kafka.common.utils.Utils.mkProperties;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -145,7 +149,7 @@ private final static class Record {
         private final String topic;
         private final Headers headers;
 
-        Record(final ConsumerRecord consumerRecord,
+        Record(final ConsumerRecord consumerRecord,
                final long newOffset) {
             key = consumerRecord.key();
             value = consumerRecord.value();
@@ -156,7 +160,7 @@ private final static class Record {
         }
 
         Record(final String newTopic,
-               final TestRecord consumerRecord,
+               final TestRecord consumerRecord,
                final long newOffset) {
             key = consumerRecord.key();
             value = consumerRecord.value();
@@ -231,7 +235,7 @@ public void punctuate(final long timestamp) {
         }
     }
 
-    private final static class MockProcessor implements Processor {
+    private final static class MockProcessor implements Processor {
         private final Collection punctuations;
         private ProcessorContext context;
 
@@ -266,7 +270,7 @@ public void close() {
 
     private final List mockProcessors = new ArrayList<>();
 
-    private final class MockProcessorSupplier implements ProcessorSupplier {
+    private final class MockProcessorSupplier implements ProcessorSupplier {
         private final Collection punctuations;
 
         private MockProcessorSupplier() {
@@ -278,7 +282,7 @@ private MockProcessorSupplier(final Collection punctuations) {
         }
 
         @Override
-        public Processor get() {
+        public Processor get() {
             final MockProcessor mockProcessor = new MockProcessor(punctuations);
             mockProcessors.add(mockProcessor);
             return mockProcessor;
@@ -452,7 +456,7 @@ public void shouldProcessRecordForTopic() {
         testDriver = new TopologyTestDriver(setupSourceSinkTopology(), config);
 
         pipeRecord(SOURCE_TOPIC_1, testRecord1);
-        final ProducerRecord outputRecord = testDriver.readRecord(SINK_TOPIC_1);
+        final ProducerRecord outputRecord = testDriver.readRecord(SINK_TOPIC_1);
 
         assertEquals(key1, outputRecord.key());
         assertEquals(value1, outputRecord.value());
@@ -705,7 +709,7 @@ public void shouldForwardRecordsFromSubtopologyToSubtopology() {
 
         pipeRecord(SOURCE_TOPIC_1, testRecord1);
 
-        ProducerRecord outputRecord = testDriver.readRecord(SINK_TOPIC_1);
+        ProducerRecord outputRecord = testDriver.readRecord(SINK_TOPIC_1);
         assertEquals(key1, outputRecord.key());
         assertEquals(value1, outputRecord.value());
         assertEquals(SINK_TOPIC_1, outputRecord.topic());
@@ -1209,7 +1213,7 @@ private void pipeInput(final String topic, final String key, final Long value, f
         testDriver.pipeRecord(topic, new TestRecord<>(key, value, null, time),
                 new StringSerializer(), new LongSerializer(), null);
     }
-    
+
     private void compareKeyValue(final TestRecord record, final String key, final Long value) {
         assertThat(record.getKey(), equalTo(key));
         assertThat(record.getValue(), equalTo(value));
@@ -1337,9 +1341,9 @@ public void shouldCleanUpPersistentStateStoresOnClose() {
         topology.addSource("sourceProcessor", "input-topic");
         topology.addProcessor(
             "storeProcessor",
-            new ProcessorSupplier() {
+            new ProcessorSupplier() {
                 @Override
-                public Processor get() {
+                public Processor get() {
                     return new Processor() {
                         private KeyValueStore store;
 
@@ -1472,7 +1476,7 @@ public void shouldProcessFromSourceThatMatchPattern() {
         testDriver = new TopologyTestDriver(topology, config);
         pipeRecord(SOURCE_TOPIC_1, testRecord1);
 
-        final ProducerRecord outputRecord = testDriver.readRecord(SINK_TOPIC_1);
+        final ProducerRecord outputRecord = testDriver.readRecord(SINK_TOPIC_1);
         assertEquals(key1, outputRecord.key());
         assertEquals(value1, outputRecord.value());
         assertEquals(SINK_TOPIC_1, outputRecord.topic());
@@ -1522,4 +1526,156 @@ public void shouldCreateStateDirectoryForStatefulTopology() {
         final TaskId taskId = new TaskId(0, 0);
         assertTrue(new File(appDir, taskId.toString()).exists());
     }
+
+    @Test
+    public void shouldEnqueueLaterOutputsAfterEarlierOnes() {
+        final Properties properties = new Properties();
+        properties.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "dummy");
+        properties.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy");
+
+        final Topology topology = new Topology();
+        topology.addSource("source", new StringDeserializer(), new StringDeserializer(), "input");
+        topology.addProcessor(
+            "recursiveProcessor",
+            () -> new AbstractProcessor() {
+                @Override
+                public void process(final String key, final String value) {
+                    if (!value.startsWith("recurse-")) {
+                        context().forward(key, "recurse-" + value, To.child("recursiveSink"));
+                    }
+                    context().forward(key, value, To.child("sink"));
+                }
+            },
+            "source"
+        );
+        topology.addSink("recursiveSink", "input", new StringSerializer(), new StringSerializer(), "recursiveProcessor");
+        topology.addSink("sink", "output", new StringSerializer(), new StringSerializer(), "recursiveProcessor");
+
+        try (final TopologyTestDriver topologyTestDriver = new TopologyTestDriver(topology, properties)) {
+            final TestInputTopic in = topologyTestDriver.createInputTopic("input", new StringSerializer(), new StringSerializer());
+            final TestOutputTopic out = topologyTestDriver.createOutputTopic("output", new StringDeserializer(), new StringDeserializer());
+
+            // given the topology above, we expect to see the output _first_ echo the input
+            // and _then_ print it with "recurse-" prepended.
+
+            in.pipeInput("B", "beta");
+            final List> events = out.readKeyValuesToList();
+            assertThat(
+                events,
+                is(Arrays.asList(
+                    new KeyValue<>("B", "beta"),
+                    new KeyValue<>("B", "recurse-beta")
+                ))
+            );
+
+        }
+    }
+
+    @Test
+    public void shouldApplyGlobalUpdatesCorrectlyInRecursiveTopologies() {
+        final Properties properties = new Properties();
+        properties.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "dummy");
+        properties.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy");
+
+        final Topology topology = new Topology();
+        topology.addSource("source", new StringDeserializer(), new StringDeserializer(), "input");
+        topology.addGlobalStore(
+            Stores.keyValueStoreBuilder(Stores.inMemoryKeyValueStore("globule-store"), Serdes.String(), Serdes.String()).withLoggingDisabled(),
+            "globuleSource",
+            new StringDeserializer(),
+            new StringDeserializer(),
+            "globule-topic",
+            "globuleProcessor",
+            () -> new Processor() {
+                private KeyValueStore stateStore;
+
+                @SuppressWarnings("unchecked")
+                @Override
+                public void init(final ProcessorContext context) {
+                    stateStore = (KeyValueStore) context.getStateStore("globule-store");
+                }
+
+                @Override
+                public void process(final String key, final String value) {
+                    stateStore.put(key, value);
+                }
+
+                @Override
+                public void close() {
+
+                }
+            }
+        );
+        topology.addProcessor(
+            "recursiveProcessor",
+            () -> new AbstractProcessor() {
+                @Override
+                public void process(final String key, final String value) {
+                    if (!value.startsWith("recurse-")) {
+                        context().forward(key, "recurse-" + value, To.child("recursiveSink"));
+                    }
+                    context().forward(key, value, To.child("sink"));
+                    context().forward(key, value, To.child("globuleSink"));
+                }
+            },
+            "source"
+        );
+        topology.addSink("recursiveSink", "input", new StringSerializer(), new StringSerializer(), "recursiveProcessor");
+        topology.addSink("sink", "output", new StringSerializer(), new StringSerializer(), "recursiveProcessor");
+        topology.addSink("globuleSink", "globule-topic", new StringSerializer(), new StringSerializer(), "recursiveProcessor");
+
+        try (final TopologyTestDriver topologyTestDriver = new TopologyTestDriver(topology, properties)) {
+            final TestInputTopic in = topologyTestDriver.createInputTopic("input", new StringSerializer(), new StringSerializer());
+            final TestOutputTopic globalTopic = topologyTestDriver.createOutputTopic("globule-topic", new StringDeserializer(), new StringDeserializer());
+
+            in.pipeInput("A", "alpha");
+
+            // expect the global store to correctly reflect the last update
+            final KeyValueStore keyValueStore = topologyTestDriver.getKeyValueStore("globule-store");
+            assertThat(keyValueStore, notNullValue());
+            assertThat(keyValueStore.get("A"), is("recurse-alpha"));
+
+            // and also just make sure the test really sent both events to the topic.
+            final List> events = globalTopic.readKeyValuesToList();
+            assertThat(
+                events,
+                is(Arrays.asList(
+                    new KeyValue<>("A", "alpha"),
+                    new KeyValue<>("A", "recurse-alpha")
+                ))
+            );
+        }
+    }
+
+    @Test
+    public void shouldTerminateWhenUsingTaskIdling() {
+        final Properties properties = new Properties();
+        properties.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "dummy");
+        properties.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy");
+
+        // This is the key to this test. Wall-clock time doesn't advance automatically in TopologyTestDriver,
+        // so with an idle time specified, TTD can't just expect all enqueued records to be processable.
+        properties.setProperty(StreamsConfig.MAX_TASK_IDLE_MS_CONFIG, "1000");
+
+        final Topology topology = new Topology();
+        topology.addSource("source1", new StringDeserializer(), new StringDeserializer(), "input1");
+        topology.addSource("source2", new StringDeserializer(), new StringDeserializer(), "input2");
+        topology.addSink("sink", "output", new StringSerializer(), new StringSerializer(), "source1", "source2");
+
+        try (final TopologyTestDriver topologyTestDriver = new TopologyTestDriver(topology, properties)) {
+            final TestInputTopic in1 = topologyTestDriver.createInputTopic("input1", new StringSerializer(), new StringSerializer());
+            final TestOutputTopic out = topologyTestDriver.createOutputTopic("output", new StringDeserializer(), new StringDeserializer());
+
+            in1.pipeInput("A", "alpha");
+
+            // Since the task has two inputs, and only one is buffered, task idling would normally prevent us from
+            // processing A, but we ignore that and process it anyway.
+            assertThat(
+                out.readKeyValuesToList(),
+                is(Collections.singletonList(
+                    new KeyValue<>("A", "alpha")
+                ))
+            );
+        }
+    }
 }
diff --git a/streams/test-utils/src/test/resources/log4j.properties b/streams/test-utils/src/test/resources/log4j.properties
new file mode 100644
index 0000000000000..be36f90299a77
--- /dev/null
+++ b/streams/test-utils/src/test/resources/log4j.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+log4j.rootLogger=INFO, stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n
+
+log4j.logger.org.apache.kafka=INFO

From 62233f10797f1284c1c4f7c4dab603af3e8cfee5 Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Tue, 11 Feb 2020 22:38:05 -0600
Subject: [PATCH 0879/1071] KAFKA-9500: Fix FK Join Topology (#8015)

Corrects a flaw leading to an exception while building topologies that include both:

* A foreign-key join with the result not explicitly materialized
* An operation after the join that requires source materialization

Also corrects a flaw in TopologyTestDriver leading to output records being enqueued in the wrong order under some (presumably rare) circumstances.

Cherry-pick of 1681c78f60a3a75e69e2222be0649cd61f02f042 from trunk
Cherry-pick of 1679839cdef342585bc19572a89672757bbb6d05 from 2.5

Reviewers: Matthias J. Sax , Guozhang Wang 
---
 checkstyle/suppressions.xml                   |   2 +-
 .../streams/kstream/internals/KTableImpl.java |   5 +-
 .../kstream/internals/KTableSource.java       |   4 +
 .../internals/graph/TableProcessorNode.java   |  11 +-
 .../internals/graph/TableSourceNode.java      |   2 +-
 ...leKTableForeignKeyJoinIntegrationTest.java | 461 +++++++++++-------
 6 files changed, 315 insertions(+), 170 deletions(-)

diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index c0aeb95b4a2ff..c4d8453a8e545 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -207,7 +207,7 @@
               files="SmokeTestDriver.java"/>
 
     
+              files="KStreamKStreamJoinTest.java|KTableKTableForeignKeyJoinIntegrationTest.java"/>
     
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
index 6eb8823f4e403..941a86607cc0b 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
@@ -1118,9 +1118,8 @@ private  KTable doJoinOnForeignKey(final KTable forei
         );
 
         final StoreBuilder> resultStore =
-            materializedInternal.queryableStoreName() == null
-                ? null
-                : new TimestampedKeyValueStoreMaterializer<>(materializedInternal).materialize();
+            new TimestampedKeyValueStoreMaterializer<>(materializedInternal).materialize();
+
         final TableProcessorNode resultNode = new TableProcessorNode<>(
             resultProcessorName,
             new ProcessorParameters<>(
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableSource.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableSource.java
index 3bbf0c4b380a4..8e002d3bfeede 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableSource.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableSource.java
@@ -67,6 +67,10 @@ public void materialize() {
         this.queryableName = storeName;
     }
 
+    public boolean materialized() {
+        return queryableName != null;
+    }
+
     private class KTableSourceProcessor extends AbstractProcessor {
 
         private TimestampedKeyValueStore store;
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/TableProcessorNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/TableProcessorNode.java
index 6fc5e25862b93..34a0e17c53fd0 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/TableProcessorNode.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/TableProcessorNode.java
@@ -17,11 +17,13 @@
 
 package org.apache.kafka.streams.kstream.internals.graph;
 
+import org.apache.kafka.streams.kstream.internals.KTableSource;
 import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder;
 import org.apache.kafka.streams.state.StoreBuilder;
 import org.apache.kafka.streams.state.TimestampedKeyValueStore;
 
 import java.util.Arrays;
+import java.util.Objects;
 
 public class TableProcessorNode extends StreamsGraphNode {
 
@@ -37,6 +39,7 @@ public TableProcessorNode(final String nodeName,
 
     public TableProcessorNode(final String nodeName,
                               final ProcessorParameters processorParameters,
+                              // TODO KIP-300: we are enforcing this as a keyvalue store, but it should go beyond any type of stores
                               final StoreBuilder> storeBuilder,
                               final String[] storeNames) {
         super(nodeName);
@@ -64,8 +67,12 @@ public void writeToTopology(final InternalTopologyBuilder topologyBuilder) {
             topologyBuilder.connectProcessorAndStateStores(processorName, storeNames);
         }
 
-        // TODO: we are enforcing this as a keyvalue store, but it should go beyond any type of stores
-        if (storeBuilder != null) {
+        if (processorParameters.processorSupplier() instanceof KTableSource) {
+            if (((KTableSource) processorParameters.processorSupplier()).materialized()) {
+                topologyBuilder.addStateStore(Objects.requireNonNull(storeBuilder, "storeBuilder was null"),
+                                              processorName);
+            }
+        } else if (storeBuilder != null) {
             topologyBuilder.addStateStore(storeBuilder, processorName);
         }
     }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/TableSourceNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/TableSourceNode.java
index b1df6ecbea741..6217ff089628c 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/TableSourceNode.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/TableSourceNode.java
@@ -109,7 +109,7 @@ public void writeToTopology(final InternalTopologyBuilder topologyBuilder) {
 
             // only add state store if the source KTable should be materialized
             final KTableSource ktableSource = (KTableSource) processorParameters.processorSupplier();
-            if (ktableSource.queryableName() != null) {
+            if (ktableSource.materialized()) {
                 topologyBuilder.addStateStore(storeBuilder, nodeName());
 
                 if (shouldReuseSourceTopicForChangelog) {
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinIntegrationTest.java
index db0e2de138412..3900d7063439a 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinIntegrationTest.java
@@ -41,6 +41,8 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.function.Function;
@@ -59,10 +61,16 @@ public class KTableKTableForeignKeyJoinIntegrationTest {
     private static final String LEFT_TABLE = "left_table";
     private static final String RIGHT_TABLE = "right_table";
     private static final String OUTPUT = "output-topic";
+    private static final String REJOIN_OUTPUT = "rejoin-output-topic";
     private final Properties streamsConfig;
     private final boolean leftJoin;
+    private final boolean materialized;
+    private final boolean rejoin;
 
-    public KTableKTableForeignKeyJoinIntegrationTest(final boolean leftJoin, final String optimization) {
+    public KTableKTableForeignKeyJoinIntegrationTest(final boolean leftJoin,
+                                                     final String optimization,
+                                                     final boolean materialized,
+                                                     final boolean rejoin) {
         this.leftJoin = leftJoin;
         streamsConfig = mkProperties(mkMap(
             mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, "ktable-ktable-joinOnForeignKey"),
@@ -70,25 +78,49 @@ public KTableKTableForeignKeyJoinIntegrationTest(final boolean leftJoin, final S
             mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath()),
             mkEntry(StreamsConfig.TOPOLOGY_OPTIMIZATION, optimization)
         ));
+        this.materialized = materialized;
+        this.rejoin = rejoin;
     }
 
-    @Parameterized.Parameters(name = "leftJoin={0}, optimization={1}")
+    @Parameterized.Parameters(name = "leftJoin={0}, optimization={1}, materialized={2}, rejoin={3}")
     public static Collection data() {
-        return Arrays.asList(
-            new Object[] {false, StreamsConfig.OPTIMIZE},
-            new Object[] {false, StreamsConfig.NO_OPTIMIZATION},
-            new Object[] {true, StreamsConfig.OPTIMIZE},
-            new Object[] {true, StreamsConfig.NO_OPTIMIZATION}
-        );
+        final List booleans = Arrays.asList(true, false);
+        final List optimizations = Arrays.asList(StreamsConfig.OPTIMIZE, StreamsConfig.NO_OPTIMIZATION);
+        return buildParameters(booleans, optimizations, booleans, booleans);
+    }
+
+    private static Collection buildParameters(final List... argOptions) {
+        List result = new LinkedList<>();
+        result.add(new Object[0]);
+
+        for (final List argOption : argOptions) {
+            result = times(result, argOption);
+        }
+
+        return result;
+    }
+
+    private static List times(final List left, final List right) {
+        final List result = new LinkedList<>();
+        for (final Object[] args : left) {
+            for (final Object rightElem : right) {
+                final Object[] resArgs = new Object[args.length + 1];
+                System.arraycopy(args, 0, resArgs, 0, args.length);
+                resArgs[args.length] = rightElem;
+                result.add(resArgs);
+            }
+        }
+        return result;
     }
 
     @Test
     public void doJoinFromLeftThenDeleteLeftEntity() {
-        final Topology topology = getTopology(streamsConfig, "store", leftJoin);
+        final Topology topology = getTopology(streamsConfig, materialized ? "store" : null, leftJoin, rejoin);
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
             final TestInputTopic right = driver.createInputTopic(RIGHT_TABLE, new StringSerializer(), new StringSerializer());
             final TestInputTopic left = driver.createInputTopic(LEFT_TABLE, new StringSerializer(), new StringSerializer());
             final TestOutputTopic outputTopic = driver.createOutputTopic(OUTPUT, new StringDeserializer(), new StringDeserializer());
+            final TestOutputTopic rejoinOutputTopic = rejoin ? driver.createOutputTopic(REJOIN_OUTPUT, new StringDeserializer(), new StringDeserializer()) : null;
             final KeyValueStore store = driver.getKeyValueStore("store");
 
             // Pre-populate the RHS records. This test is all about what happens when we add/remove LHS records
@@ -100,10 +132,18 @@ public void doJoinFromLeftThenDeleteLeftEntity() {
                 outputTopic.readKeyValuesToMap(),
                 is(emptyMap())
             );
-            assertThat(
-                asMap(store),
-                is(emptyMap())
-            );
+            if (rejoin) {
+                assertThat(
+                    rejoinOutputTopic.readKeyValuesToMap(),
+                    is(emptyMap())
+                );
+            }
+            if (materialized) {
+                assertThat(
+                    asMap(store),
+                    is(emptyMap())
+                );
+            }
 
             left.pipeInput("lhs1", "lhsValue1|rhs1");
             left.pipeInput("lhs2", "lhsValue2|rhs2");
@@ -117,10 +157,21 @@ public void doJoinFromLeftThenDeleteLeftEntity() {
                     outputTopic.readKeyValuesToMap(),
                     is(expected)
                 );
-                assertThat(
-                    asMap(store),
-                    is(expected)
-                );
+                if (rejoin) {
+                    assertThat(
+                        rejoinOutputTopic.readKeyValuesToMap(),
+                        is(mkMap(
+                            mkEntry("lhs1", "rejoin((lhsValue1|rhs1,rhsValue1),lhsValue1|rhs1)"),
+                            mkEntry("lhs2", "rejoin((lhsValue2|rhs2,rhsValue2),lhsValue2|rhs2)")
+                        ))
+                    );
+                }
+                if (materialized) {
+                    assertThat(
+                        asMap(store),
+                        is(expected)
+                    );
+                }
             }
 
             // Add another reference to an existing FK
@@ -132,14 +183,24 @@ public void doJoinFromLeftThenDeleteLeftEntity() {
                         mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)")
                     ))
                 );
-                assertThat(
-                    asMap(store),
-                    is(mkMap(
-                        mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)"),
-                        mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)"),
-                        mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)")
-                    ))
-                );
+                if (rejoin) {
+                    assertThat(
+                        rejoinOutputTopic.readKeyValuesToMap(),
+                        is(mkMap(
+                            mkEntry("lhs3", "rejoin((lhsValue3|rhs1,rhsValue1),lhsValue3|rhs1)")
+                        ))
+                    );
+                }
+                if (materialized) {
+                    assertThat(
+                        asMap(store),
+                        is(mkMap(
+                            mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)"),
+                            mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)"),
+                            mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)")
+                        ))
+                    );
+                }
             }
             // Now delete one LHS entity such that one delete is propagated down to the output.
 
@@ -150,19 +211,29 @@ public void doJoinFromLeftThenDeleteLeftEntity() {
                     mkEntry("lhs1", null)
                 ))
             );
-            assertThat(
-                asMap(store),
-                is(mkMap(
-                    mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)"),
-                    mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)")
-                ))
-            );
+            if (rejoin) {
+                assertThat(
+                    rejoinOutputTopic.readKeyValuesToMap(),
+                    is(mkMap(
+                        mkEntry("lhs1", null)
+                    ))
+                );
+            }
+            if (materialized) {
+                assertThat(
+                    asMap(store),
+                    is(mkMap(
+                        mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)"),
+                        mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)")
+                    ))
+                );
+            }
         }
     }
 
     @Test
     public void doJoinFromRightThenDeleteRightEntity() {
-        final Topology topology = getTopology(streamsConfig, "store", leftJoin);
+        final Topology topology = getTopology(streamsConfig, materialized ? "store" : null, leftJoin, rejoin);
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
             final TestInputTopic right = driver.createInputTopic(RIGHT_TABLE, new StringSerializer(), new StringSerializer());
             final TestInputTopic left = driver.createInputTopic(LEFT_TABLE, new StringSerializer(), new StringSerializer());
@@ -183,15 +254,17 @@ public void doJoinFromRightThenDeleteRightEntity() {
                        : emptyMap()
                 )
             );
-            assertThat(
-                asMap(store),
-                is(leftJoin
-                       ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,null)"),
-                               mkEntry("lhs2", "(lhsValue2|rhs2,null)"),
-                               mkEntry("lhs3", "(lhsValue3|rhs1,null)"))
-                       : emptyMap()
-                )
-            );
+            if (materialized) {
+                assertThat(
+                    asMap(store),
+                    is(leftJoin
+                           ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,null)"),
+                                   mkEntry("lhs2", "(lhsValue2|rhs2,null)"),
+                                   mkEntry("lhs3", "(lhsValue3|rhs1,null)"))
+                           : emptyMap()
+                    )
+                );
+            }
 
             right.pipeInput("rhs1", "rhsValue1");
 
@@ -201,17 +274,19 @@ public void doJoinFromRightThenDeleteRightEntity() {
                          mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)"))
                 )
             );
-            assertThat(
-                asMap(store),
-                is(leftJoin
-                       ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)"),
-                               mkEntry("lhs2", "(lhsValue2|rhs2,null)"),
-                               mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)"))
-
-                       : mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)"),
-                               mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)"))
-                )
-            );
+            if (materialized) {
+                assertThat(
+                    asMap(store),
+                    is(leftJoin
+                           ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)"),
+                                   mkEntry("lhs2", "(lhsValue2|rhs2,null)"),
+                                   mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)"))
+
+                           : mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)"),
+                                   mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)"))
+                    )
+                );
+            }
 
             right.pipeInput("rhs2", "rhsValue2");
 
@@ -219,13 +294,15 @@ public void doJoinFromRightThenDeleteRightEntity() {
                 outputTopic.readKeyValuesToMap(),
                 is(mkMap(mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)")))
             );
-            assertThat(
-                asMap(store),
-                is(mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)"),
-                         mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)"),
-                         mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)"))
-                )
-            );
+            if (materialized) {
+                assertThat(
+                    asMap(store),
+                    is(mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)"),
+                             mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)"),
+                             mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)"))
+                    )
+                );
+            }
 
             right.pipeInput("rhs3", "rhsValue3"); // this unreferenced FK won't show up in any results
 
@@ -233,13 +310,15 @@ public void doJoinFromRightThenDeleteRightEntity() {
                 outputTopic.readKeyValuesToMap(),
                 is(emptyMap())
             );
-            assertThat(
-                asMap(store),
-                is(mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)"),
-                         mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)"),
-                         mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)"))
-                )
-            );
+            if (materialized) {
+                assertThat(
+                    asMap(store),
+                    is(mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)"),
+                             mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)"),
+                             mkEntry("lhs3", "(lhsValue3|rhs1,rhsValue1)"))
+                    )
+                );
+            }
 
             // Now delete the RHS entity such that all matching keys have deletes propagated.
             right.pipeInput("rhs1", (String) null);
@@ -250,22 +329,24 @@ public void doJoinFromRightThenDeleteRightEntity() {
                          mkEntry("lhs3", leftJoin ? "(lhsValue3|rhs1,null)" : null))
                 )
             );
-            assertThat(
-                asMap(store),
-                is(leftJoin
-                       ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,null)"),
-                               mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)"),
-                               mkEntry("lhs3", "(lhsValue3|rhs1,null)"))
+            if (materialized) {
+                assertThat(
+                    asMap(store),
+                    is(leftJoin
+                           ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,null)"),
+                                   mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)"),
+                                   mkEntry("lhs3", "(lhsValue3|rhs1,null)"))
 
-                       : mkMap(mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)"))
-                )
-            );
+                           : mkMap(mkEntry("lhs2", "(lhsValue2|rhs2,rhsValue2)"))
+                    )
+                );
+            }
         }
     }
 
     @Test
     public void shouldEmitTombstoneWhenDeletingNonJoiningRecords() {
-        final Topology topology = getTopology(streamsConfig, "store", leftJoin);
+        final Topology topology = getTopology(streamsConfig, materialized ? "store" : null, leftJoin, rejoin);
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
             final TestInputTopic left = driver.createInputTopic(LEFT_TABLE, new StringSerializer(), new StringSerializer());
             final TestOutputTopic outputTopic = driver.createOutputTopic(OUTPUT, new StringDeserializer(), new StringDeserializer());
@@ -280,10 +361,12 @@ public void shouldEmitTombstoneWhenDeletingNonJoiningRecords() {
                     outputTopic.readKeyValuesToMap(),
                     is(expected)
                 );
-                assertThat(
-                    asMap(store),
-                    is(expected)
-                );
+                if (materialized) {
+                    assertThat(
+                        asMap(store),
+                        is(expected)
+                    );
+                }
             }
 
             // Deleting a non-joining record produces an unnecessary tombstone for inner joins, because
@@ -295,10 +378,12 @@ public void shouldEmitTombstoneWhenDeletingNonJoiningRecords() {
                     outputTopic.readKeyValuesToMap(),
                     is(mkMap(mkEntry("lhs1", null)))
                 );
-                assertThat(
-                    asMap(store),
-                    is(emptyMap())
-                );
+                if (materialized) {
+                    assertThat(
+                        asMap(store),
+                        is(emptyMap())
+                    );
+                }
             }
 
             // Deleting a non-existing record is idempotent
@@ -308,17 +393,19 @@ public void shouldEmitTombstoneWhenDeletingNonJoiningRecords() {
                     outputTopic.readKeyValuesToMap(),
                     is(emptyMap())
                 );
-                assertThat(
-                    asMap(store),
-                    is(emptyMap())
-                );
+                if (materialized) {
+                    assertThat(
+                        asMap(store),
+                        is(emptyMap())
+                    );
+                }
             }
         }
     }
 
     @Test
     public void shouldNotEmitTombstonesWhenDeletingNonExistingRecords() {
-        final Topology topology = getTopology(streamsConfig, "store", leftJoin);
+        final Topology topology = getTopology(streamsConfig, materialized ? "store" : null, leftJoin, rejoin);
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
             final TestInputTopic left = driver.createInputTopic(LEFT_TABLE, new StringSerializer(), new StringSerializer());
             final TestOutputTopic outputTopic = driver.createOutputTopic(OUTPUT, new StringDeserializer(), new StringDeserializer());
@@ -331,17 +418,19 @@ public void shouldNotEmitTombstonesWhenDeletingNonExistingRecords() {
                     outputTopic.readKeyValuesToMap(),
                     is(emptyMap())
                 );
-                assertThat(
-                    asMap(store),
-                    is(emptyMap())
-                );
+                if (materialized) {
+                    assertThat(
+                        asMap(store),
+                        is(emptyMap())
+                    );
+                }
             }
         }
     }
 
     @Test
     public void joinShouldProduceNullsWhenValueHasNonMatchingForeignKey() {
-        final Topology topology = getTopology(streamsConfig, "store", leftJoin);
+        final Topology topology = getTopology(streamsConfig, materialized ? "store" : null, leftJoin, rejoin);
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
             final TestInputTopic right = driver.createInputTopic(RIGHT_TABLE, new StringSerializer(), new StringSerializer());
             final TestInputTopic left = driver.createInputTopic(LEFT_TABLE, new StringSerializer(), new StringSerializer());
@@ -355,10 +444,12 @@ public void joinShouldProduceNullsWhenValueHasNonMatchingForeignKey() {
                 outputTopic.readKeyValuesToMap(),
                 is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,null)")) : emptyMap())
             );
-            assertThat(
-                asMap(store),
-                is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,null)")) : emptyMap())
-            );
+            if (materialized) {
+                assertThat(
+                    asMap(store),
+                    is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs1,null)")) : emptyMap())
+                );
+            }
             // "moving" our subscription to another non-existent FK results in an unnecessary tombstone for inner join,
             // since it impossible to know whether the prior FK existed or not (and thus whether any results have
             // previously been emitted)
@@ -368,20 +459,24 @@ public void joinShouldProduceNullsWhenValueHasNonMatchingForeignKey() {
                 outputTopic.readKeyValuesToMap(),
                 is(mkMap(mkEntry("lhs1", leftJoin ? "(lhsValue1|rhs2,null)" : null)))
             );
-            assertThat(
-                asMap(store),
-                is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs2,null)")) : emptyMap())
-            );
+            if (materialized) {
+                assertThat(
+                    asMap(store),
+                    is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs2,null)")) : emptyMap())
+                );
+            }
             // of course, moving it again to yet another non-existent FK has the same effect
             left.pipeInput("lhs1", "lhsValue1|rhs3");
             assertThat(
                 outputTopic.readKeyValuesToMap(),
                 is(mkMap(mkEntry("lhs1", leftJoin ? "(lhsValue1|rhs3,null)" : null)))
             );
-            assertThat(
-                asMap(store),
-                is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs3,null)")) : emptyMap())
-            );
+            if (materialized) {
+                assertThat(
+                    asMap(store),
+                    is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs3,null)")) : emptyMap())
+                );
+            }
 
             // Adding an RHS record now, so that we can demonstrate "moving" from a non-existent FK to an existent one
             // This RHS key was previously referenced, but it's not referenced now, so adding this record should
@@ -391,10 +486,12 @@ public void joinShouldProduceNullsWhenValueHasNonMatchingForeignKey() {
                 outputTopic.readKeyValuesToMap(),
                 is(emptyMap())
             );
-            assertThat(
-                asMap(store),
-                is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs3,null)")) : emptyMap())
-            );
+            if (materialized) {
+                assertThat(
+                    asMap(store),
+                    is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs3,null)")) : emptyMap())
+                );
+            }
 
             // now, we change to a FK that exists, and see the join completes
             left.pipeInput("lhs1", "lhsValue1|rhs1");
@@ -404,12 +501,14 @@ public void joinShouldProduceNullsWhenValueHasNonMatchingForeignKey() {
                     mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)")
                 ))
             );
-            assertThat(
-                asMap(store),
-                is(mkMap(
-                    mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)")
-                ))
-            );
+            if (materialized) {
+                assertThat(
+                    asMap(store),
+                    is(mkMap(
+                        mkEntry("lhs1", "(lhsValue1|rhs1,rhsValue1)")
+                    ))
+                );
+            }
 
             // but if we update it again to a non-existent one, we'll get a tombstone for the inner join, and the
             // left join updates appropriately.
@@ -420,16 +519,18 @@ public void joinShouldProduceNullsWhenValueHasNonMatchingForeignKey() {
                     mkEntry("lhs1", leftJoin ? "(lhsValue1|rhs2,null)" : null)
                 ))
             );
-            assertThat(
-                asMap(store),
-                is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs2,null)")) : emptyMap())
-            );
+            if (materialized) {
+                assertThat(
+                    asMap(store),
+                    is(leftJoin ? mkMap(mkEntry("lhs1", "(lhsValue1|rhs2,null)")) : emptyMap())
+                );
+            }
         }
     }
 
     @Test
     public void shouldUnsubscribeOldForeignKeyIfLeftSideIsUpdated() {
-        final Topology topology = getTopology(streamsConfig, "store", leftJoin);
+        final Topology topology = getTopology(streamsConfig, materialized ? "store" : null, leftJoin, rejoin);
         try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
             final TestInputTopic right = driver.createInputTopic(RIGHT_TABLE, new StringSerializer(), new StringSerializer());
             final TestInputTopic left = driver.createInputTopic(LEFT_TABLE, new StringSerializer(), new StringSerializer());
@@ -445,10 +546,12 @@ public void shouldUnsubscribeOldForeignKeyIfLeftSideIsUpdated() {
                 outputTopic.readKeyValuesToMap(),
                 is(emptyMap())
             );
-            assertThat(
-                asMap(store),
-                is(emptyMap())
-            );
+            if (materialized) {
+                assertThat(
+                    asMap(store),
+                    is(emptyMap())
+                );
+            }
 
             left.pipeInput("lhs1", "lhsValue1|rhs1");
             {
@@ -459,10 +562,12 @@ public void shouldUnsubscribeOldForeignKeyIfLeftSideIsUpdated() {
                     outputTopic.readKeyValuesToMap(),
                     is(expected)
                 );
-                assertThat(
-                    asMap(store),
-                    is(expected)
-                );
+                if (materialized) {
+                    assertThat(
+                        asMap(store),
+                        is(expected)
+                    );
+                }
             }
 
             // Change LHS foreign key reference
@@ -475,10 +580,12 @@ public void shouldUnsubscribeOldForeignKeyIfLeftSideIsUpdated() {
                     outputTopic.readKeyValuesToMap(),
                     is(expected)
                 );
-                assertThat(
-                    asMap(store),
-                    is(expected)
-                );
+                if (materialized) {
+                    assertThat(
+                        asMap(store),
+                        is(expected)
+                    );
+                }
             }
 
             // Populate RHS update on old LHS foreign key ref
@@ -488,12 +595,14 @@ public void shouldUnsubscribeOldForeignKeyIfLeftSideIsUpdated() {
                     outputTopic.readKeyValuesToMap(),
                     is(emptyMap())
                 );
-                assertThat(
-                    asMap(store),
-                    is(mkMap(
-                        mkEntry("lhs1", "(lhsValue1|rhs2,rhsValue2)")
-                    ))
-                );
+                if (materialized) {
+                    assertThat(
+                        asMap(store),
+                        is(mkMap(
+                            mkEntry("lhs1", "(lhsValue1|rhs2,rhsValue2)")
+                        ))
+                    );
+                }
             }
         }
     }
@@ -506,7 +615,8 @@ private static Map asMap(final KeyValueStore sto
 
     private static Topology getTopology(final Properties streamsConfig,
                                         final String queryableStoreName,
-                                        final boolean leftJoin) {
+                                        final boolean leftJoin,
+                                        final boolean rejoin) {
         final UniqueTopicSerdeScope serdeScope = new UniqueTopicSerdeScope();
         final StreamsBuilder builder = new StreamsBuilder();
 
@@ -523,33 +633,58 @@ private static Topology getTopology(final Properties streamsConfig,
 
         final Function extractor = value -> value.split("\\|")[1];
         final ValueJoiner joiner = (value1, value2) -> "(" + value1 + "," + value2 + ")";
+        final ValueJoiner rejoiner = rejoin ? (value1, value2) -> "rejoin(" + value1 + "," + value2 + ")" : null;
+
+        // the cache suppresses some of the unnecessary tombstones we want to make assertions about
+        final Materialized> mainMaterialized =
+            queryableStoreName == null ?
+                Materialized.>with(
+                    null,
+                    serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)
+                ).withCachingDisabled() :
+                Materialized.as(Stores.inMemoryKeyValueStore(queryableStoreName))
+                    .withValueSerde(serdeScope.decorateSerde(Serdes.String(), streamsConfig, false))
+                    .withCachingDisabled();
+
+        final Materialized> rejoinMaterialized =
+            !rejoin ? null :
+                queryableStoreName == null ?
+                    Materialized.with(null, serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)) :
+                    // not actually going to query this store, but we need to force materialization here
+                    // to really test this confuguration
+                    Materialized.as(Stores.inMemoryKeyValueStore(queryableStoreName + "-rejoin"))
+                        .withValueSerde(serdeScope.decorateSerde(Serdes.String(), streamsConfig, false))
+                        // the cache suppresses some of the unnecessary tombstones we want to make assertions about
+                        .withCachingDisabled();
 
-        final Materialized> materialized =
-            Materialized.as(Stores.inMemoryKeyValueStore(queryableStoreName))
-                .withValueSerde(serdeScope.decorateSerde(Serdes.String(), streamsConfig, false))
-                // the cache suppresses some of the unnecessary tombstones we want to make assertions about
-                .withCachingDisabled();
-
-        final KTable joinResult;
         if (leftJoin) {
-            joinResult = left.leftJoin(
-                right,
-                extractor,
-                joiner,
-                materialized
-            );
+            final KTable fkJoin =
+                left.leftJoin(right, extractor, joiner, mainMaterialized);
+
+            fkJoin.toStream()
+                  .to(OUTPUT);
+
+            // also make sure the FK join is set up right for downstream operations that require materialization
+            if (rejoin) {
+                fkJoin.leftJoin(left, rejoiner, rejoinMaterialized)
+                      .toStream()
+                      .to(REJOIN_OUTPUT);
+            }
         } else {
-            joinResult = left.join(
-                right,
-                extractor,
-                joiner,
-                materialized
-            );
+            final KTable fkJoin = left.join(right, extractor, joiner, mainMaterialized);
+
+            fkJoin
+                .toStream()
+                .to(OUTPUT);
+
+            // also make sure the FK join is set up right for downstream operations that require materialization
+            if (rejoin) {
+                fkJoin.join(left, rejoiner, rejoinMaterialized)
+                      .toStream()
+                      .to(REJOIN_OUTPUT);
+            }
         }
 
-        joinResult
-            .toStream()
-            .to(OUTPUT);
 
         return builder.build(streamsConfig);
     }

From f670136fd3bd2b41323bcf9480b48146c680c159 Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Fri, 24 Jan 2020 10:38:21 +0000
Subject: [PATCH 0880/1071] KAFKA-9181; Maintain clean separation between local
 and group subscriptions in consumer's SubscriptionState (#7941)

Reviewers: Jason Gustafson , Guozhang Wang 
---
 checkstyle/suppressions.xml                   |  2 +-
 .../internals/ConsumerCoordinator.java        |  2 +-
 .../consumer/internals/ConsumerMetadata.java  |  4 +-
 .../consumer/internals/SubscriptionState.java | 22 ++++-----
 .../internals/ConsumerCoordinatorTest.java    | 47 +++++++++++++++++--
 .../internals/ConsumerMetadataTest.java       |  5 +-
 6 files changed, 59 insertions(+), 23 deletions(-)

diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index c4d8453a8e545..941dd1b1b9234 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -60,7 +60,7 @@
               files="(ConsumerCoordinator|Fetcher|Sender|KafkaProducer|BufferPool|ConfigDef|RecordAccumulator|KerberosLogin|AbstractRequest|AbstractResponse|Selector|SslFactory|SslTransportLayer|SaslClientAuthenticator|SaslClientCallbackHandler|SaslServerAuthenticator|AbstractCoordinator).java"/>
 
     
+              files="(AbstractRequest|KerberosLogin|WorkerSinkTaskTest|TransactionManagerTest|SenderTest|KafkaAdminClient|ConsumerCoordinatorTest).java"/>
 
     
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 3f8d81d0e3fc6..db8b25507096d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -1307,7 +1307,7 @@ private static class MetadataSnapshot {
 
         private MetadataSnapshot(SubscriptionState subscription, Cluster cluster, int version) {
             Map partitionsPerTopic = new HashMap<>();
-            for (String topic : subscription.groupSubscription()) {
+            for (String topic : subscription.metadataTopics()) {
                 Integer numPartitions = cluster.partitionCountForTopic(topic);
                 if (numPartitions != null)
                     partitionsPerTopic.put(topic, numPartitions);
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerMetadata.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerMetadata.java
index fbdf1c6c8595c..ef7d92417471b 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerMetadata.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerMetadata.java
@@ -55,7 +55,7 @@ public synchronized MetadataRequest.Builder newMetadataRequestBuilder() {
         if (subscription.hasPatternSubscription())
             return MetadataRequest.Builder.allTopics();
         List topics = new ArrayList<>();
-        topics.addAll(subscription.groupSubscription());
+        topics.addAll(subscription.metadataTopics());
         topics.addAll(transientTopics);
         return new MetadataRequest.Builder(topics, allowAutoTopicCreation);
     }
@@ -72,7 +72,7 @@ synchronized void clearTransientTopics() {
 
     @Override
     protected synchronized boolean retainTopic(String topic, boolean isInternal, long nowMs) {
-        if (transientTopics.contains(topic) || subscription.isGroupSubscribed(topic))
+        if (transientTopics.contains(topic) || subscription.needsMetadata(topic))
             return true;
 
         if (isInternal && !includeInternalTopics)
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
index 953505f0b3e35..4a3d15b45718c 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
@@ -183,12 +183,6 @@ private boolean changeSubscription(Set topicsToSubscribe) {
             return false;
 
         subscription = topicsToSubscribe;
-        if (subscriptionType != SubscriptionType.USER_ASSIGNED) {
-            groupSubscription = new HashSet<>(groupSubscription);
-            groupSubscription.addAll(topicsToSubscribe);
-        } else {
-            groupSubscription = new HashSet<>(topicsToSubscribe);
-        }
         return true;
     }
 
@@ -208,7 +202,7 @@ synchronized boolean groupSubscribe(Collection topics) {
      * Reset the group's subscription to only contain topics subscribed by this consumer.
      */
     synchronized void resetGroupSubscription() {
-        groupSubscription = subscription;
+        groupSubscription = Collections.emptySet();
     }
 
     /**
@@ -332,9 +326,9 @@ public synchronized Set pausedPartitions() {
     }
 
     /**
-     * Get the subscription for the group. For the leader, this will include the union of the
-     * subscriptions of all group members. For followers, it is just that member's subscription.
-     * This is used when querying topic metadata to detect the metadata changes which would
+     * Get the subcription topics for which metadata is required . For the leader, this will include
+     * the union of the subscriptions of all group members. For followers, it is just that member's
+     * subscription. This is used when querying topic metadata to detect the metadata changes which would
      * require rebalancing. The leader fetches metadata for all topics in the group so that it
      * can do the partition assignment (which requires at least partition counts for all topics
      * to be assigned).
@@ -342,12 +336,12 @@ public synchronized Set pausedPartitions() {
      * @return The union of all subscribed topics in the group if this member is the leader
      *   of the current generation; otherwise it returns the same set as {@link #subscription()}
      */
-    synchronized Set groupSubscription() {
-        return this.groupSubscription;
+    synchronized Set metadataTopics() {
+        return groupSubscription.isEmpty() ? subscription : groupSubscription;
     }
 
-    synchronized boolean isGroupSubscribed(String topic) {
-        return groupSubscription.contains(topic);
+    synchronized boolean needsMetadata(String topic) {
+        return subscription.contains(topic) || groupSubscription.contains(topic);
     }
 
     private TopicPartitionState assignedState(TopicPartition tp) {
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index 6617fa2fd9ad5..9e6128badd4a2 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -578,7 +578,7 @@ public void testNormalJoinGroupLeader() {
 
         assertFalse(coordinator.rejoinNeededOrPending());
         assertEquals(toSet(assigned), subscriptions.assignedPartitions());
-        assertEquals(subscription, subscriptions.groupSubscription());
+        assertEquals(subscription, subscriptions.metadataTopics());
         assertEquals(0, rebalanceListener.revokedCount);
         assertNull(rebalanceListener.revoked);
         assertEquals(1, rebalanceListener.assignedCount);
@@ -637,7 +637,7 @@ public void testOutdatedCoordinatorAssignment() {
 
         assertFalse(coordinator.rejoinNeededOrPending());
         assertEquals(toSet(newAssignment), subscriptions.assignedPartitions());
-        assertEquals(toSet(newSubscription), subscriptions.groupSubscription());
+        assertEquals(toSet(newSubscription), subscriptions.metadataTopics());
         assertEquals(protocol == EAGER ? 1 : 0, rebalanceListener.revokedCount);
         assertEquals(1, rebalanceListener.assignedCount);
         assertEquals(assigned, rebalanceListener.assigned);
@@ -676,7 +676,7 @@ public void testPatternJoinGroupLeader() {
 
         assertFalse(coordinator.rejoinNeededOrPending());
         assertEquals(2, subscriptions.numAssignedPartitions());
-        assertEquals(2, subscriptions.groupSubscription().size());
+        assertEquals(2, subscriptions.metadataTopics().size());
         assertEquals(2, subscriptions.subscription().size());
         // callback not triggered at all since there's nothing to be revoked
         assertEquals(0, rebalanceListener.revokedCount);
@@ -907,7 +907,7 @@ public void testNormalJoinGroupFollower() {
 
         assertFalse(coordinator.rejoinNeededOrPending());
         assertEquals(toSet(assigned), subscriptions.assignedPartitions());
-        assertEquals(subscription, subscriptions.groupSubscription());
+        assertEquals(subscription, subscriptions.metadataTopics());
         assertEquals(0, rebalanceListener.revokedCount);
         assertNull(rebalanceListener.revoked);
         assertEquals(1, rebalanceListener.assignedCount);
@@ -1229,6 +1229,45 @@ public void testUpdateMetadataDuringRebalance() {
         assertEquals(new HashSet<>(Arrays.asList(tp1, tp2)), subscriptions.assignedPartitions());
     }
 
+    /**
+     * Verifies that subscription change updates SubscriptionState correctly even after JoinGroup failures
+     * that don't re-invoke onJoinPrepare.
+     */
+    @Test
+    public void testSubscriptionChangeWithAuthorizationFailure() {
+        final String consumerId = "consumer";
+
+        // Subscribe to two topics of which only one is authorized and verify that metadata failure is propagated.
+        subscriptions.subscribe(Utils.mkSet(topic1, topic2), rebalanceListener);
+        client.prepareMetadataUpdate(TestUtils.metadataUpdateWith("kafka-cluster", 1,
+                Collections.singletonMap(topic2, Errors.TOPIC_AUTHORIZATION_FAILED), singletonMap(topic1, 1)));
+        assertThrows(TopicAuthorizationException.class, () -> coordinator.poll(time.timer(Long.MAX_VALUE)));
+
+        client.respond(groupCoordinatorResponse(node, Errors.NONE));
+        coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
+
+        // Fail the first JoinGroup request
+        client.prepareResponse(joinGroupLeaderResponse(0, consumerId, Collections.emptyMap(),
+                Errors.GROUP_AUTHORIZATION_FAILED));
+        assertThrows(GroupAuthorizationException.class, () -> coordinator.poll(time.timer(Long.MAX_VALUE)));
+
+        // Change subscription to include only the authorized topic. Complete rebalance and check that
+        // references to topic2 have been removed from SubscriptionState.
+        subscriptions.subscribe(Utils.mkSet(topic1), rebalanceListener);
+        assertEquals(Collections.singleton(topic1), subscriptions.metadataTopics());
+        client.prepareMetadataUpdate(TestUtils.metadataUpdateWith("kafka-cluster", 1,
+                Collections.emptyMap(), singletonMap(topic1, 1)));
+
+        Map> memberSubscriptions = singletonMap(consumerId, singletonList(topic1));
+        partitionAssignor.prepare(singletonMap(consumerId, Arrays.asList(t1p)));
+        client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE));
+        client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE));
+        coordinator.poll(time.timer(Long.MAX_VALUE));
+
+        assertEquals(singleton(topic1), subscriptions.subscription());
+        assertEquals(singleton(topic1), subscriptions.metadataTopics());
+    }
+
     @Test
     public void testWakeupFromAssignmentCallback() {
         final String topic = "topic1";
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetadataTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetadataTest.java
index 33d102d3b748f..b373192ca7066 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetadataTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetadataTest.java
@@ -102,8 +102,11 @@ public void testUserAssignment() {
     @Test
     public void testNormalSubscription() {
         subscription.subscribe(Utils.mkSet("foo", "bar", "__consumer_offsets"), new NoOpConsumerRebalanceListener());
-        subscription.groupSubscribe(Utils.mkSet("baz"));
+        subscription.groupSubscribe(Utils.mkSet("baz", "foo", "bar", "__consumer_offsets"));
         testBasicSubscription(Utils.mkSet("foo", "bar", "baz"), Utils.mkSet("__consumer_offsets"));
+
+        subscription.resetGroupSubscription();
+        testBasicSubscription(Utils.mkSet("foo", "bar"), Utils.mkSet("__consumer_offsets"));
     }
 
     @Test

From 1b0c10d2bf0d9da95dfea230d8d839e5f3769006 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Wed, 12 Feb 2020 11:45:06 -0800
Subject: [PATCH 0881/1071] MINOR: Fix unnecessary metadata fetch before group
 assignment (#8095)

The recent increase in the flakiness of one of the offset reset tests (KAFKA-9538) traces back to https://github.com/apache/kafka/pull/7941. After investigation, we found that following this patch, the consumer was sending an additional metadata request prior to performing the group assignment. This slight timing difference was enough to trigger the test failures. The problem turned out to be due to a bug in `SubscriptionState.groupSubscribe`, which no longer counted the local subscription when determining if there were new topics to fetch metadata for. Hence the extra metadata update. This patch restores the old logic.

Without the fix, we saw 30-50% test failures locally. With it, I could no longer reproduce the failure. However, #6561 is probably still needed to improve the resilience of this test.

Reviewers: Rajini Sivaram 
---
 .../consumer/ConsumerPartitionAssignor.java   |  8 +++++
 .../internals/ConsumerCoordinator.java        |  3 --
 .../consumer/internals/SubscriptionState.java | 12 ++++---
 .../internals/SubscriptionStateTest.java      | 20 +++++++++--
 .../admin/ConsumerGroupCommandTest.scala      | 14 +++++---
 .../admin/ResetConsumerGroupOffsetTest.scala  | 33 ++++++++++++++-----
 6 files changed, 67 insertions(+), 23 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java
index f9a42171a0447..8708ea4f7e343 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java
@@ -154,6 +154,14 @@ public List partitions() {
         public ByteBuffer userData() {
             return userData;
         }
+
+        @Override
+        public String toString() {
+            return "Assignment(" +
+                    "partitions=" + partitions +
+                    (userData == null ? "" : ", userDataSize=" + userData.remaining()) +
+                    ')';
+        }
     }
 
     final class GroupSubscription {
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index db8b25507096d..b4d9fa5bf283f 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -1320,9 +1320,6 @@ boolean matches(MetadataSnapshot other) {
             return version == other.version || partitionsPerTopic.equals(other.partitionsPerTopic);
         }
 
-        Map partitionsPerTopic() {
-            return partitionsPerTopic;
-        }
     }
 
     private static class OffsetCommitCompletion {
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
index 4a3d15b45718c..cdf358ecf3a03 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
@@ -187,15 +187,17 @@ private boolean changeSubscription(Set topicsToSubscribe) {
     }
 
     /**
-     * Add topics to the current group subscription. This is used by the group leader to ensure
+     * Set the current group subscription. This is used by the group leader to ensure
      * that it receives metadata updates for all topics that the group is interested in.
-     * @param topics The topics to add to the group subscription
+     *
+     * @param topics All topics from the group subscription
+     * @return true if the group subscription contains topics which are not part of the local subscription
      */
     synchronized boolean groupSubscribe(Collection topics) {
         if (!partitionsAutoAssigned())
             throw new IllegalStateException(SUBSCRIPTION_EXCEPTION_MESSAGE);
-        groupSubscription = new HashSet<>(groupSubscription);
-        return groupSubscription.addAll(topics);
+        groupSubscription = new HashSet<>(topics);
+        return !subscription.containsAll(groupSubscription);
     }
 
     /**
@@ -326,7 +328,7 @@ public synchronized Set pausedPartitions() {
     }
 
     /**
-     * Get the subcription topics for which metadata is required . For the leader, this will include
+     * Get the subscription topics for which metadata is required. For the leader, this will include
      * the union of the subscriptions of all group members. For followers, it is just that member's
      * subscription. This is used when querying topic metadata to detect the metadata changes which would
      * require rebalancing. The leader fetches metadata for all topics in the group so that it
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java
index c3ce02a79a155..47d654e04ac45 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java
@@ -105,6 +105,22 @@ public void partitionAssignmentChangeOnTopicSubscription() {
         assertEquals(0, state.numAssignedPartitions());
     }
 
+    @Test
+    public void testGroupSubscribe() {
+        state.subscribe(singleton(topic1), rebalanceListener);
+        assertEquals(singleton(topic1), state.metadataTopics());
+
+        assertFalse(state.groupSubscribe(singleton(topic1)));
+        assertEquals(singleton(topic1), state.metadataTopics());
+
+        assertTrue(state.groupSubscribe(Utils.mkSet(topic, topic1)));
+        assertEquals(Utils.mkSet(topic, topic1), state.metadataTopics());
+
+        // `groupSubscribe` does not accumulate
+        assertFalse(state.groupSubscribe(singleton(topic1)));
+        assertEquals(singleton(topic1), state.metadataTopics());
+    }
+
     @Test
     public void partitionAssignmentChangeOnPatternSubscription() {
         state.subscribe(Pattern.compile(".*"), rebalanceListener);
@@ -112,7 +128,7 @@ public void partitionAssignmentChangeOnPatternSubscription() {
         assertTrue(state.assignedPartitions().isEmpty());
         assertEquals(0, state.numAssignedPartitions());
 
-        state.subscribeFromPattern(new HashSet<>(Collections.singletonList(topic)));
+        state.subscribeFromPattern(Collections.singleton(topic));
         // assigned partitions should remain unchanged
         assertTrue(state.assignedPartitions().isEmpty());
         assertEquals(0, state.numAssignedPartitions());
@@ -244,7 +260,7 @@ public void cantAssignPartitionForUnsubscribedTopics() {
     @Test
     public void cantAssignPartitionForUnmatchedPattern() {
         state.subscribe(Pattern.compile(".*t"), rebalanceListener);
-        state.subscribeFromPattern(new HashSet<>(Collections.singletonList(topic)));
+        state.subscribeFromPattern(Collections.singleton(topic));
         assertFalse(state.checkAssignmentMatchedSubscription(Collections.singletonList(t1p0)));
     }
 
diff --git a/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala b/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
index 97b638f8e6ffa..853b2caa7c751 100644
--- a/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
@@ -65,20 +65,24 @@ class ConsumerGroupCommandTest extends KafkaServerTestHarness {
   }
 
   def committedOffsets(topic: String = topic, group: String = group): Map[TopicPartition, Long] = {
-    val props = new Properties
-    props.put("bootstrap.servers", brokerList)
-    props.put("group.id", group)
-    val consumer = new KafkaConsumer(props, new StringDeserializer, new StringDeserializer)
+    val consumer = createNoAutoCommitConsumer(group)
     try {
       val partitions: Set[TopicPartition] = consumer.partitionsFor(topic)
         .asScala.toSet.map {partitionInfo : PartitionInfo => new TopicPartition(partitionInfo.topic, partitionInfo.partition)}
-
       consumer.committed(partitions.asJava).asScala.filter(_._2 != null).mapValues(_.offset()).toMap
     } finally {
       consumer.close()
     }
   }
 
+  def createNoAutoCommitConsumer(group: String): KafkaConsumer[String, String] = {
+    val props = new Properties
+    props.put("bootstrap.servers", brokerList)
+    props.put("group.id", group)
+    props.put("enable.auto.commit", "false")
+    new KafkaConsumer(props, new StringDeserializer, new StringDeserializer)
+  }
+
   def getConsumerGroupService(args: Array[String]): ConsumerGroupService = {
     val opts = new ConsumerGroupCommandOptions(args)
     val service = new ConsumerGroupService(opts, Map(AdminClientConfig.RETRIES_CONFIG -> Int.MaxValue.toString))
diff --git a/core/src/test/scala/unit/kafka/admin/ResetConsumerGroupOffsetTest.scala b/core/src/test/scala/unit/kafka/admin/ResetConsumerGroupOffsetTest.scala
index 838444ce748f1..f9322b36e76f1 100644
--- a/core/src/test/scala/unit/kafka/admin/ResetConsumerGroupOffsetTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ResetConsumerGroupOffsetTest.scala
@@ -16,8 +16,6 @@ import java.io.{BufferedWriter, File, FileWriter}
 import java.text.{ParseException, SimpleDateFormat}
 import java.util.{Calendar, Date, Properties}
 
-import scala.collection.Seq
-
 import joptsimple.OptionException
 import kafka.admin.ConsumerGroupCommand.ConsumerGroupService
 import kafka.server.KafkaConfig
@@ -28,6 +26,9 @@ import org.apache.kafka.test
 import org.junit.Assert._
 import org.junit.Test
 
+import scala.collection.JavaConverters._
+import scala.collection.Seq
+
 class TimeConversionTests {
 
   @Test
@@ -462,12 +463,28 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
     executor.shutdown()
   }
 
-  private def awaitConsumerProgress(topic: String = topic, group: String = group, count: Long): Unit = {
-    TestUtils.waitUntilTrue(() => {
-      val offsets = committedOffsets(topic = topic, group = group).values
-      count == offsets.sum
-    }, "Expected that consumer group has consumed all messages from topic/partition. " +
-      s"Expected offset: $count. Actual offset: ${committedOffsets(topic, group).values.sum}")
+  private def awaitConsumerProgress(topic: String = topic,
+                                    group: String = group,
+                                    count: Long): Unit = {
+    val consumer = createNoAutoCommitConsumer(group)
+    try {
+      val partitions = consumer.partitionsFor(topic).asScala.map { partitionInfo =>
+        new TopicPartition(partitionInfo.topic, partitionInfo.partition)
+      }.toSet
+
+      TestUtils.waitUntilTrue(() => {
+        val committed = consumer.committed(partitions.asJava).values.asScala
+        val total = committed.foldLeft(0L) { case (currentSum, offsetAndMetadata) =>
+          currentSum + Option(offsetAndMetadata).map(_.offset).getOrElse(0L)
+        }
+        total == count
+      }, "Expected that consumer group has consumed all messages from topic/partition. " +
+        s"Expected offset: $count. Actual offset: ${committedOffsets(topic, group).values.sum}")
+
+    } finally {
+      consumer.close()
+    }
+
   }
 
   private def resetAndAssertOffsets(args: Array[String],

From 452d93c3c2bb46a2094aa7cbec05386c6209a636 Mon Sep 17 00:00:00 2001
From: Lev Zemlyanov 
Date: Wed, 12 Feb 2020 13:44:23 -0800
Subject: [PATCH 0882/1071] KAFKA-9192: fix NPE when for converting optional
 json schema in structs (#7733)

Author: Lev Zemlyanov 
Reviewers: Greg Harris , Randall Hauch 
---
 .../org/apache/kafka/connect/json/JsonConverter.java     | 2 +-
 .../org/apache/kafka/connect/json/JsonConverterTest.java | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/connect/json/src/main/java/org/apache/kafka/connect/json/JsonConverter.java b/connect/json/src/main/java/org/apache/kafka/connect/json/JsonConverter.java
index fe83ddf7e9ee6..3ef6aae816a43 100644
--- a/connect/json/src/main/java/org/apache/kafka/connect/json/JsonConverter.java
+++ b/connect/json/src/main/java/org/apache/kafka/connect/json/JsonConverter.java
@@ -708,7 +708,7 @@ private static Object convertToConnect(Schema schema, JsonNode jsonValue) {
         final Schema.Type schemaType;
         if (schema != null) {
             schemaType = schema.type();
-            if (jsonValue.isNull()) {
+            if (jsonValue == null || jsonValue.isNull()) {
                 if (schema.defaultValue() != null)
                     return schema.defaultValue(); // any logical type conversions should already have been applied
                 if (schema.isOptional())
diff --git a/connect/json/src/test/java/org/apache/kafka/connect/json/JsonConverterTest.java b/connect/json/src/test/java/org/apache/kafka/connect/json/JsonConverterTest.java
index fb9d8cb8ebb8a..2a5695031e49d 100644
--- a/connect/json/src/test/java/org/apache/kafka/connect/json/JsonConverterTest.java
+++ b/connect/json/src/test/java/org/apache/kafka/connect/json/JsonConverterTest.java
@@ -173,6 +173,15 @@ public void structToConnect() {
         assertEquals(new SchemaAndValue(expectedSchema, expected), converted);
     }
 
+    @Test
+    public void structWithOptionalFieldToConnect() {
+        byte[] structJson = "{ \"schema\": { \"type\": \"struct\", \"fields\": [{ \"field\":\"optional\", \"type\": \"string\", \"optional\": true }, {  \"field\": \"required\", \"type\": \"string\" }] }, \"payload\": { \"required\": \"required\" } }".getBytes();
+        Schema expectedSchema = SchemaBuilder.struct().field("optional", Schema.OPTIONAL_STRING_SCHEMA).field("required", Schema.STRING_SCHEMA).build();
+        Struct expected = new Struct(expectedSchema).put("required", "required");
+        SchemaAndValue converted = converter.toConnectData(TOPIC, structJson);
+        assertEquals(new SchemaAndValue(expectedSchema, expected), converted);
+    }
+
     @Test
     public void nullToConnect() {
         // When schemas are enabled, trying to decode a tombstone should be an empty envelope

From b2e4fa3320cf460b55e345c5ba1ee7c563411ce5 Mon Sep 17 00:00:00 2001
From: Lev Zemlyanov 
Date: Wed, 12 Feb 2020 14:36:06 -0800
Subject: [PATCH 0883/1071] allow ReplaceField SMT to handle tombstone records
 (#7731)

Signed-off-by: Lev Zemlyanov 
---
 .../connect/transforms/ReplaceField.java      |  4 +-
 .../connect/transforms/ReplaceFieldTest.java  | 38 +++++++++++++++++++
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/ReplaceField.java b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/ReplaceField.java
index f071bdad3156a..3d9abc25bc731 100644
--- a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/ReplaceField.java
+++ b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/ReplaceField.java
@@ -123,7 +123,9 @@ String reverseRenamed(String fieldName) {
 
     @Override
     public R apply(R record) {
-        if (operatingSchema(record) == null) {
+        if (operatingValue(record) == null) {
+            return record;
+        } else if (operatingSchema(record) == null) {
             return applySchemaless(record);
         } else {
             return applyWithSchema(record);
diff --git a/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/ReplaceFieldTest.java b/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/ReplaceFieldTest.java
index 6a1a13ada9dde..7ab00ed8da8c3 100644
--- a/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/ReplaceFieldTest.java
+++ b/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/ReplaceFieldTest.java
@@ -27,6 +27,7 @@
 import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 
 public class ReplaceFieldTest {
     private ReplaceField xform = new ReplaceField.Value<>();
@@ -36,6 +37,43 @@ public void teardown() {
         xform.close();
     }
 
+    @Test
+    public void tombstoneSchemaless() {
+        final Map props = new HashMap<>();
+        props.put("whitelist", "abc,foo");
+        props.put("renames", "abc:xyz,foo:bar");
+
+        xform.configure(props);
+
+        final SinkRecord record = new SinkRecord("test", 0, null, null, null, null, 0);
+        final SinkRecord transformedRecord = xform.apply(record);
+
+        assertNull(transformedRecord.value());
+        assertNull(transformedRecord.valueSchema());
+    }
+
+    @Test
+    public void tombstoneWithSchema() {
+        final Map props = new HashMap<>();
+        props.put("whitelist", "abc,foo");
+        props.put("renames", "abc:xyz,foo:bar");
+
+        xform.configure(props);
+
+        final Schema schema = SchemaBuilder.struct()
+            .field("dont", Schema.STRING_SCHEMA)
+            .field("abc", Schema.INT32_SCHEMA)
+            .field("foo", Schema.BOOLEAN_SCHEMA)
+            .field("etc", Schema.STRING_SCHEMA)
+            .build();
+
+        final SinkRecord record = new SinkRecord("test", 0, null, null, schema, null, 0);
+        final SinkRecord transformedRecord = xform.apply(record);
+
+        assertNull(transformedRecord.value());
+        assertEquals(schema, transformedRecord.valueSchema());
+    }
+
     @Test
     public void schemaless() {
         final Map props = new HashMap<>();

From 32ea859f04f3714c31af1d8c653c8e06c454d6da Mon Sep 17 00:00:00 2001
From: Konstantine Karantasis 
Date: Wed, 12 Feb 2020 15:40:37 -0800
Subject: [PATCH 0884/1071] MINOR: Small Connect integration test fixes (#8100)

Author: Konstantine Karantasis 
Reviewer: Randall Hauch 
---
 .../MirrorConnectorsIntegrationTest.java      | 39 +++++++++++++------
 .../ConnectWorkerIntegrationTest.java         |  3 +-
 .../ConnectorCientPolicyIntegrationTest.java  | 15 ++++---
 .../ErrorHandlingIntegrationTest.java         | 11 +++++-
 .../ExampleConnectIntegrationTest.java        |  3 +-
 ...alanceSourceConnectorsIntegrationTest.java |  3 +-
 .../RestExtensionIntegrationTest.java         | 11 ++++--
 .../SessionedProtocolIntegrationTest.java     |  3 +-
 .../util/clusters/EmbeddedConnectCluster.java | 10 +++--
 9 files changed, 63 insertions(+), 35 deletions(-)

diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java
index 11abc142170f7..8ae8df3f0bd59 100644
--- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java
+++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java
@@ -16,14 +16,13 @@
  */
 package org.apache.kafka.connect.mirror;
 
-import org.apache.kafka.connect.util.clusters.EmbeddedConnectCluster;
-import org.apache.kafka.connect.util.clusters.EmbeddedKafkaCluster;
-import org.apache.kafka.test.IntegrationTest;
+import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.OffsetAndMetadata;
-import org.apache.kafka.clients.admin.Admin;
 import org.apache.kafka.common.TopicPartition;
-
+import org.apache.kafka.connect.util.clusters.EmbeddedConnectCluster;
+import org.apache.kafka.connect.util.clusters.EmbeddedKafkaCluster;
+import org.apache.kafka.test.IntegrationTest;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -31,17 +30,18 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
-import java.util.Collections;
 import java.util.Properties;
-import java.util.concurrent.TimeoutException;
-import java.time.Duration;
+import java.util.Set;
 
+import static org.apache.kafka.test.TestUtils.waitForCondition;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
-import static org.apache.kafka.test.TestUtils.waitForCondition;
 
 /**
  * Tests MM2 replication and failover/failback logic.
@@ -66,7 +66,7 @@ public class MirrorConnectorsIntegrationTest {
     private EmbeddedConnectCluster backup;
 
     @Before
-    public void setup() throws IOException {
+    public void setup() throws InterruptedException {
         Properties brokerProps = new Properties();
         brokerProps.put("auto.create.topics.enable", "false");
 
@@ -111,7 +111,11 @@ public void setup() throws IOException {
                 .build();
 
         primary.start();
+        primary.assertions().assertAtLeastNumWorkersAreUp(3,
+                "Workers of primary-connect-cluster did not start in time.");
         backup.start();
+        primary.assertions().assertAtLeastNumWorkersAreUp(3,
+                "Workers of backup-connect-cluster did not start in time.");
 
         // create these topics before starting the connectors so we don't need to wait for discovery
         primary.kafka().createTopic("test-topic-1", NUM_PARTITIONS);
@@ -167,10 +171,21 @@ public void setup() throws IOException {
 
         primary.configureConnector("MirrorHeartbeatConnector", mm2Config.connectorBaseConfig(new SourceAndTarget("backup", "primary"),
             MirrorHeartbeatConnector.class));
+
+        waitUntilMirrorMakerIsRunning(primary, connectorNames);
+    }
+
+
+    private void waitUntilMirrorMakerIsRunning(EmbeddedConnectCluster connectCluster,
+        Set connNames) throws InterruptedException {
+        for (String connector : connNames) {
+            connectCluster.assertions().assertConnectorAndAtLeastNumTasksAreRunning(connector, 1,
+                    "Connector " + connector + " tasks did not start in time on cluster: " + connectCluster);
+        }
     }
 
     @After
-    public void close() throws IOException {
+    public void close() {
         for (String x : primary.connectors()) {
             primary.deleteConnector(x);
         }
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java
index 6cc43a471f3ba..b8e04970f610d 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java
@@ -28,7 +28,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -66,7 +65,7 @@ public class ConnectWorkerIntegrationTest {
     Properties brokerProps = new Properties();
 
     @Before
-    public void setup() throws IOException {
+    public void setup() {
         // setup Connect worker properties
         workerProps.put(OFFSET_COMMIT_INTERVAL_MS_CONFIG, String.valueOf(OFFSET_COMMIT_INTERVAL_MS));
         workerProps.put(CONNECTOR_CLIENT_POLICY_CLASS_CONFIG, "All");
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorCientPolicyIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorCientPolicyIntegrationTest.java
index 499916bf69299..00f541beee6a7 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorCientPolicyIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorCientPolicyIntegrationTest.java
@@ -29,7 +29,6 @@
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
-import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
@@ -50,7 +49,6 @@ public class ConnectorCientPolicyIntegrationTest {
     private static final int NUM_WORKERS = 1;
     private static final String CONNECTOR_NAME = "simple-conn";
 
-
     @After
     public void close() {
     }
@@ -73,7 +71,7 @@ public void testCreateWithNotAllowedOverridesForPrincipalPolicy() throws Excepti
     @Test
     public void testCreateWithAllowedOverridesForPrincipalPolicy() throws Exception {
         Map props = basicConnectorConfig();
-        props.put(ConnectorConfig.CONNECTOR_CLIENT_CONSUMER_OVERRIDES_PREFIX + CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "PLAIN");
+        props.put(ConnectorConfig.CONNECTOR_CLIENT_CONSUMER_OVERRIDES_PREFIX + CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "PLAINTEXT");
         assertPassCreateConnector("Principal", props);
     }
 
@@ -85,7 +83,7 @@ public void testCreateWithAllowedOverridesForAllPolicy() throws Exception {
         assertPassCreateConnector("All", props);
     }
 
-    private EmbeddedConnectCluster connectClusterWithPolicy(String policy) throws IOException {
+    private EmbeddedConnectCluster connectClusterWithPolicy(String policy) throws InterruptedException {
         // setup Connect worker properties
         Map workerProps = new HashMap<>();
         workerProps.put(OFFSET_COMMIT_INTERVAL_MS_CONFIG, String.valueOf(5_000));
@@ -106,10 +104,13 @@ private EmbeddedConnectCluster connectClusterWithPolicy(String policy) throws IO
 
         // start the clusters
         connect.start();
+        connect.assertions().assertAtLeastNumWorkersAreUp(NUM_WORKERS,
+                "Initial group of workers did not start in time.");
+
         return connect;
     }
 
-    private void assertFailCreateConnector(String policy, Map props) throws IOException {
+    private void assertFailCreateConnector(String policy, Map props) throws InterruptedException {
         EmbeddedConnectCluster connect = connectClusterWithPolicy(policy);
         try {
             connect.configureConnector(CONNECTOR_NAME, props);
@@ -121,10 +122,12 @@ private void assertFailCreateConnector(String policy, Map props)
         }
     }
 
-    private void assertPassCreateConnector(String policy, Map props) throws IOException {
+    private void assertPassCreateConnector(String policy, Map props) throws InterruptedException {
         EmbeddedConnectCluster connect = connectClusterWithPolicy(policy);
         try {
             connect.configureConnector(CONNECTOR_NAME, props);
+            connect.assertions().assertConnectorAndAtLeastNumTasksAreRunning(CONNECTOR_NAME, NUM_TASKS,
+                    "Connector tasks did not start in time.");
         } catch (ConnectRestException e) {
             fail("Should be able to create connector");
         } finally {
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ErrorHandlingIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ErrorHandlingIntegrationTest.java
index 33e6cf58234b2..8963b8cb7ca3a 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ErrorHandlingIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ErrorHandlingIntegrationTest.java
@@ -34,7 +34,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
@@ -69,6 +68,7 @@ public class ErrorHandlingIntegrationTest {
 
     private static final Logger log = LoggerFactory.getLogger(ErrorHandlingIntegrationTest.class);
 
+    private static final int NUM_WORKERS = 1;
     private static final String DLQ_TOPIC = "my-connector-errors";
     private static final String CONNECTOR_NAME = "error-conn";
     private static final String TASK_ID = "error-conn-0";
@@ -83,12 +83,14 @@ public class ErrorHandlingIntegrationTest {
     private ConnectorHandle connectorHandle;
 
     @Before
-    public void setup() throws IOException {
+    public void setup() throws InterruptedException {
         // setup Connect cluster with defaults
         connect = new EmbeddedConnectCluster.Builder().build();
 
         // start Connect cluster
         connect.start();
+        connect.assertions().assertAtLeastNumWorkersAreUp(NUM_WORKERS,
+                "Initial group of workers did not start in time.");
 
         // get connector handles before starting test.
         connectorHandle = RuntimeHandles.get().connectorHandle(CONNECTOR_NAME);
@@ -134,6 +136,8 @@ public void testSkipRetryAndDLQWithHeaders() throws Exception {
         connectorHandle.taskHandle(TASK_ID).expectedRecords(EXPECTED_CORRECT_RECORDS);
 
         connect.configureConnector(CONNECTOR_NAME, props);
+        connect.assertions().assertConnectorAndAtLeastNumTasksAreRunning(CONNECTOR_NAME, NUM_TASKS,
+                "Connector tasks did not start in time.");
 
         waitForCondition(this::checkForPartitionAssignment,
                 CONNECTOR_SETUP_DURATION_MS,
@@ -172,6 +176,9 @@ public void testSkipRetryAndDLQWithHeaders() throws Exception {
         }
 
         connect.deleteConnector(CONNECTOR_NAME);
+        connect.assertions().assertConnectorAndTasksAreStopped(CONNECTOR_NAME,
+                "Connector tasks did not stop in time.");
+
     }
 
     /**
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExampleConnectIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExampleConnectIntegrationTest.java
index 4da89d7406648..f4cba8793bd0b 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExampleConnectIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExampleConnectIntegrationTest.java
@@ -27,7 +27,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
@@ -66,7 +65,7 @@ public class ExampleConnectIntegrationTest {
     private ConnectorHandle connectorHandle;
 
     @Before
-    public void setup() throws IOException {
+    public void setup() {
         // setup Connect worker properties
         Map exampleWorkerProps = new HashMap<>();
         exampleWorkerProps.put(OFFSET_COMMIT_INTERVAL_MS_CONFIG, String.valueOf(5_000));
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java
index 073b3072cf478..19e786317089b 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java
@@ -28,7 +28,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -70,7 +69,7 @@ public class RebalanceSourceConnectorsIntegrationTest {
     private EmbeddedConnectCluster connect;
 
     @Before
-    public void setup() throws IOException {
+    public void setup() {
         // setup Connect worker properties
         Map workerProps = new HashMap<>();
         workerProps.put(CONNECT_PROTOCOL_CONFIG, COMPATIBLE.toString());
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RestExtensionIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RestExtensionIntegrationTest.java
index d5328a6ace967..6ec86bd9342b1 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RestExtensionIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RestExtensionIntegrationTest.java
@@ -34,7 +34,6 @@
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
 import javax.ws.rs.core.Response;
-import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -57,11 +56,12 @@ public class RestExtensionIntegrationTest {
 
     private static final long REST_EXTENSION_REGISTRATION_TIMEOUT_MS = TimeUnit.MINUTES.toMillis(1);
     private static final long CONNECTOR_HEALTH_AND_CONFIG_TIMEOUT_MS = TimeUnit.MINUTES.toMillis(1);
+    private static final int NUM_WORKERS = 1;
 
     private EmbeddedConnectCluster connect;
 
     @Test
-    public void testRestExtensionApi() throws IOException, InterruptedException {
+    public void testRestExtensionApi() throws InterruptedException {
         // setup Connect worker properties
         Map workerProps = new HashMap<>();
         workerProps.put(REST_EXTENSION_CLASSES_CONFIG, IntegrationTestRestExtension.class.getName());
@@ -69,7 +69,7 @@ public void testRestExtensionApi() throws IOException, InterruptedException {
         // build a Connect cluster backed by Kafka and Zk
         connect = new EmbeddedConnectCluster.Builder()
             .name("connect-cluster")
-            .numWorkers(1)
+            .numWorkers(NUM_WORKERS)
             .numBrokers(1)
             .workerProps(workerProps)
             .build();
@@ -77,6 +77,9 @@ public void testRestExtensionApi() throws IOException, InterruptedException {
         // start the clusters
         connect.start();
 
+        connect.assertions().assertAtLeastNumWorkersAreUp(NUM_WORKERS,
+                "Initial group of workers did not start in time.");
+
         WorkerHandle worker = connect.workers().stream()
             .findFirst()
             .orElseThrow(() -> new AssertionError("At least one worker handle should be available"));
@@ -99,6 +102,8 @@ public void testRestExtensionApi() throws IOException, InterruptedException {
             connectorHandle.taskHandle(connectorHandle.name() + "-0");
             StartAndStopLatch connectorStartLatch = connectorHandle.expectedStarts(1);
             connect.configureConnector(connectorHandle.name(), connectorProps);
+            connect.assertions().assertConnectorAndAtLeastNumTasksAreRunning(connectorHandle.name(), 1,
+                    "Connector tasks did not start in time.");
             connectorStartLatch.await(CONNECTOR_HEALTH_AND_CONFIG_TIMEOUT_MS, TimeUnit.MILLISECONDS);
 
             String workerId = String.format("%s:%d", worker.url().getHost(), worker.url().getPort());
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/SessionedProtocolIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/SessionedProtocolIntegrationTest.java
index 1f13c17d32ee5..8956a86e7c73c 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/SessionedProtocolIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/SessionedProtocolIntegrationTest.java
@@ -28,7 +28,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
@@ -61,7 +60,7 @@ public class SessionedProtocolIntegrationTest {
     private ConnectorHandle connectorHandle;
 
     @Before
-    public void setup() throws IOException {
+    public void setup() {
         // setup Connect worker properties
         Map workerProps = new HashMap<>();
         workerProps.put(CONNECT_PROTOCOL_CONFIG, ConnectProtocolCompatibility.SESSIONED.protocol());
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java
index 2276340f8d40c..961b1d840c63d 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java
@@ -284,7 +284,8 @@ public String configureConnector(String connName, Map connConfig
         if (response.getStatus() < Response.Status.BAD_REQUEST.getStatusCode()) {
             return responseToString(response);
         }
-        throw new ConnectRestException(response.getStatus(), "Could not execute PUT request");
+        throw new ConnectRestException(response.getStatus(),
+                "Could not execute PUT request. Error response: " + responseToString(response));
     }
 
     /**
@@ -298,7 +299,8 @@ public void deleteConnector(String connName) {
         String url = endpointForResource(String.format("connectors/%s", connName));
         Response response = requestDelete(url);
         if (response.getStatus() >= Response.Status.BAD_REQUEST.getStatusCode()) {
-            throw new ConnectRestException(response.getStatus(), "Could not execute DELETE request.");
+            throw new ConnectRestException(response.getStatus(),
+                    "Could not execute DELETE request. Error response: " + responseToString(response));
         }
     }
 
@@ -358,7 +360,7 @@ public ConnectorStateInfo connectorStatus(String connectorName) {
      *
      * @param resource the resource under the worker's admin endpoint
      * @return the admin endpoint URL
-     * @throws ConnectRestException if no admin REST endpoint is available
+     * @throws ConnectException if no admin REST endpoint is available
      */
     public String adminEndpoint(String resource) {
         String url = connectCluster.stream()
@@ -375,7 +377,7 @@ public String adminEndpoint(String resource) {
      *
      * @param resource the resource under the worker's admin endpoint
      * @return the admin endpoint URL
-     * @throws ConnectRestException if no REST endpoint is available
+     * @throws ConnectException if no REST endpoint is available
      */
     public String endpointForResource(String resource) {
         String url = connectCluster.stream()

From 161ffe155ebfd9b3b6612601d8868332cc143104 Mon Sep 17 00:00:00 2001
From: Randall Hauch 
Date: Wed, 12 Feb 2020 18:50:01 -0600
Subject: [PATCH 0885/1071] MINOR: Fix poor backport that prevented compiling
 the MirrorMaker integration test (#8102)

Author: Randall Hauch 
Reviewer: Jason Gustafson 
---
 .../mirror/MirrorConnectorsIntegrationTest.java       | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java
index 8ae8df3f0bd59..8b1bac9e2e74d 100644
--- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java
+++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java
@@ -154,6 +154,13 @@ public void setup() throws InterruptedException {
         mm2Props.put("backup.bootstrap.servers", backup.kafka().bootstrapServers());
         mm2Config = new MirrorMakerConfig(mm2Props);
 
+        // we wait for the connector and tasks to come up for each connector, so that when we do the
+        // actual testing, we are certain that the tasks are up and running; this will prevent
+        // flaky tests where the connector and tasks didn't start up in time for the tests to be
+        // run
+        Set connectorNames = new HashSet<>(Arrays.asList("MirrorSourceConnector",
+                "MirrorCheckpointConnector", "MirrorHeartbeatConnector"));
+
         backup.configureConnector("MirrorSourceConnector", mm2Config.connectorBaseConfig(new SourceAndTarget("primary", "backup"),
             MirrorSourceConnector.class));
 
@@ -163,6 +170,8 @@ public void setup() throws InterruptedException {
         backup.configureConnector("MirrorHeartbeatConnector", mm2Config.connectorBaseConfig(new SourceAndTarget("primary", "backup"),
             MirrorHeartbeatConnector.class));
 
+        waitUntilMirrorMakerIsRunning(backup, connectorNames);
+
         primary.configureConnector("MirrorSourceConnector", mm2Config.connectorBaseConfig(new SourceAndTarget("backup", "primary"),
             MirrorSourceConnector.class));
 
@@ -199,7 +208,7 @@ public void close() {
     }
 
     @Test
-    public void testReplication() throws InterruptedException, TimeoutException {
+    public void testReplication() throws InterruptedException {
         MirrorClient primaryClient = new MirrorClient(mm2Config.clientConfig("primary"));
         MirrorClient backupClient = new MirrorClient(mm2Config.clientConfig("backup"));
 

From a19a15ffa07a26066b5ad560a29965c708de909c Mon Sep 17 00:00:00 2001
From: Bruno Cadonna 
Date: Wed, 12 Feb 2020 02:31:13 +0100
Subject: [PATCH 0886/1071] KAFKA-9355: Fix bug that removed RocksDB metrics
 after failure in EOS (#7996)

* Added init() method to RocksDBMetricsRecorder
* Added call to init() of RocksDBMetricsRecorder to init() of RocksDB store
* Added call to init() of RocksDBMetricsRecorder to openExisting() of segmented state stores
* Adapted unit tests
* Added integration test that reproduces the situation in which the bug occurred

Reviewers: Guozhang Wang 
---
 .../state/internals/KeyValueSegments.java     |   8 +-
 .../streams/state/internals/RocksDBStore.java |   3 +-
 .../state/internals/TimestampedSegments.java  |   6 +
 .../metrics/RocksDBMetricsRecorder.java       |  31 +-
 .../integration/MetricsIntegrationTest.java   |  24 +-
 .../RocksDBMetricsIntegrationTest.java        | 295 ++++++++++++++++++
 .../state/internals/KeyValueSegmentsTest.java |   2 +
 .../state/internals/RocksDBStoreTest.java     |   8 +-
 .../state/internals/SegmentIteratorTest.java  |   4 +-
 .../internals/TimestampedSegmentsTest.java    |   2 +
 .../metrics/RocksDBMetricsRecorderTest.java   | 145 +++++----
 11 files changed, 427 insertions(+), 101 deletions(-)
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/integration/RocksDBMetricsIntegrationTest.java

diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/KeyValueSegments.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/KeyValueSegments.java
index fc49c129b998b..9dbbae4667beb 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/KeyValueSegments.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/KeyValueSegments.java
@@ -51,4 +51,10 @@ public KeyValueSegment getOrCreateSegment(final long segmentId,
             return newSegment;
         }
     }
-}
+
+    @Override
+    public void openExisting(final InternalProcessorContext context, final long streamTime) {
+        metricsRecorder.init(context.metrics(), context.taskId());
+        super.openExisting(context, streamTime);
+    }
+}
\ No newline at end of file
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
index 96ffe3beb157e..2b9b3f612a172 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
@@ -203,7 +203,7 @@ private void maybeSetUpMetricsRecorder(final ProcessorContext context, final Map
             // metrics recorder will clean up statistics object
             final Statistics statistics = new Statistics();
             userSpecifiedOptions.setStatistics(statistics);
-            metricsRecorder.addStatistics(name, statistics, (StreamsMetricsImpl) context.metrics(), context.taskId());
+            metricsRecorder.addStatistics(name, statistics);
         }
     }
 
@@ -225,6 +225,7 @@ public void init(final ProcessorContext context,
                      final StateStore root) {
         // open the DB dir
         internalProcessorContext = context;
+        metricsRecorder.init((StreamsMetricsImpl) context.metrics(), context.taskId());
         openDB(context);
         batchingStateRestoreCallback = new RocksDBBatchingRestoreCallback(this);
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/TimestampedSegments.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/TimestampedSegments.java
index 400511f8ff4e0..e7c2edbf14b7c 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/TimestampedSegments.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/TimestampedSegments.java
@@ -51,4 +51,10 @@ public TimestampedSegment getOrCreateSegment(final long segmentId,
             return newSegment;
         }
     }
+
+    @Override
+    public void openExisting(final InternalProcessorContext context, final long streamTime) {
+        metricsRecorder.init(context.metrics(), context.taskId());
+        super.openExisting(context, streamTime);
+    }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorder.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorder.java
index 59ade54132ab1..b5d603c903d8e 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorder.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorder.java
@@ -51,7 +51,6 @@ public class RocksDBMetricsRecorder {
     private final String threadId;
     private TaskId taskId;
     private StreamsMetricsImpl streamsMetrics;
-    private boolean isInitialized = false;
 
     public RocksDBMetricsRecorder(final String metricsScope,
                                   final String threadId,
@@ -71,20 +70,26 @@ public TaskId taskId() {
         return taskId;
     }
 
-    public void addStatistics(final String segmentName,
-                              final Statistics statistics,
-                              final StreamsMetricsImpl streamsMetrics,
-                              final TaskId taskId) {
-        if (!isInitialized) {
-            initSensors(streamsMetrics, taskId);
-            this.taskId = taskId;
-            this.streamsMetrics = streamsMetrics;
-            isInitialized = true;
+    /**
+     * The initialisation of the metrics recorder is idempotent.
+     */
+    public void init(final StreamsMetricsImpl streamsMetrics,
+                     final TaskId taskId) {
+        if (this.taskId != null && !this.taskId.equals(taskId)) {
+            throw new IllegalStateException("Metrics recorder is re-initialised with different task: previous task is " +
+                this.taskId + " whereas current task is " + taskId + ". This is a bug in Kafka Streams.");
         }
-        if (this.taskId != taskId) {
-            throw new IllegalStateException("Statistics of store \"" + segmentName + "\" for task " + taskId
-                + " cannot be added to metrics recorder for task " + this.taskId + ". This is a bug in Kafka Streams.");
+        if (this.streamsMetrics != null && this.streamsMetrics != streamsMetrics) {
+            throw new IllegalStateException("Metrics recorder is re-initialised with different Streams metrics. "
+                + "This is a bug in Kafka Streams.");
         }
+        initSensors(streamsMetrics, taskId);
+        this.taskId = taskId;
+        this.streamsMetrics = streamsMetrics;
+    }
+
+    public void addStatistics(final String segmentName,
+                              final Statistics statistics) {
         if (statisticsToRecord.isEmpty()) {
             logger.debug(
                 "Adding metrics recorder of task {} to metrics recording trigger",
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
index 010f277555846..74bfb97ead815 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/MetricsIntegrationTest.java
@@ -59,7 +59,7 @@
 
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.MatcherAssert.assertThat;
-import static org.junit.Assert.assertTrue;
+
 
 @SuppressWarnings("unchecked")
 @Category({IntegrationTest.class})
@@ -393,28 +393,6 @@ public void shouldAddMetricsForSessionStore() throws Exception {
         checkMetricsDeregistration();
     }
 
-    @Test
-    public void shouldNotAddRocksDBMetricsIfRecordingLevelIsInfo() throws Exception {
-        builder.table(
-            STREAM_INPUT,
-            Materialized.as(Stores.persistentKeyValueStore(MY_STORE_PERSISTENT_KEY_VALUE)).withCachingEnabled()
-        ).toStream().to(STREAM_OUTPUT_1);
-        streamsConfiguration.put(StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG, Sensor.RecordingLevel.INFO.name);
-        kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration);
-        kafkaStreams.start();
-        TestUtils.waitForCondition(
-            () -> kafkaStreams.state() == State.RUNNING,
-            timeout,
-            () -> "Kafka Streams application did not reach state RUNNING in " + timeout + " ms");
-
-        final List listMetricStore = new ArrayList(kafkaStreams.metrics().values()).stream()
-            .filter(m -> m.metricName().group().equals("stream-state-metrics") && m.metricName().tags().containsKey("rocksdb-state-id"))
-            .collect(Collectors.toList());
-        assertTrue(listMetricStore.isEmpty());
-
-        closeApplication();
-    }
-
     private void verifyStateMetric(final State state) {
         final List metricsList = new ArrayList(kafkaStreams.metrics().values()).stream()
             .filter(m -> m.metricName().name().equals(STATE) &&
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/RocksDBMetricsIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/RocksDBMetricsIntegrationTest.java
new file mode 100644
index 0000000000000..b0ac1faa7238c
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/RocksDBMetricsIntegrationTest.java
@@ -0,0 +1,295 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.integration;
+
+import org.apache.kafka.common.Metric;
+import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.serialization.IntegerDeserializer;
+import org.apache.kafka.common.serialization.IntegerSerializer;
+import org.apache.kafka.common.serialization.LongDeserializer;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
+import org.apache.kafka.streams.integration.utils.IntegrationTestUtils;
+import org.apache.kafka.streams.kstream.Consumed;
+import org.apache.kafka.streams.kstream.Materialized;
+import org.apache.kafka.streams.kstream.Produced;
+import org.apache.kafka.streams.kstream.TimeWindows;
+import org.apache.kafka.streams.state.Stores;
+import org.apache.kafka.streams.state.WindowStore;
+import org.apache.kafka.test.IntegrationTest;
+import org.apache.kafka.test.StreamsTestUtils;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+import java.util.stream.Collectors;
+
+@Category({IntegrationTest.class})
+@RunWith(Parameterized.class)
+public class RocksDBMetricsIntegrationTest {
+
+    private static final int NUM_BROKERS = 3;
+
+    @ClassRule
+    public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(NUM_BROKERS);
+
+    private static final String STREAM_INPUT = "STREAM_INPUT";
+    private static final String STREAM_OUTPUT = "STREAM_OUTPUT";
+    private static final String MY_STORE_PERSISTENT_KEY_VALUE = "myStorePersistentKeyValue";
+    private static final Duration WINDOW_SIZE = Duration.ofMillis(50);
+
+    // RocksDB metrics
+    private static final String BYTES_WRITTEN_RATE = "bytes-written-rate";
+    private static final String BYTES_WRITTEN_TOTAL = "bytes-written-total";
+    private static final String BYTES_READ_RATE = "bytes-read-rate";
+    private static final String BYTES_READ_TOTAL = "bytes-read-total";
+    private static final String MEMTABLE_BYTES_FLUSHED_RATE = "memtable-bytes-flushed-rate";
+    private static final String MEMTABLE_BYTES_FLUSHED_TOTAL = "memtable-bytes-flushed-total";
+    private static final String MEMTABLE_HIT_RATIO = "memtable-hit-ratio";
+    private static final String WRITE_STALL_DURATION_AVG = "write-stall-duration-avg";
+    private static final String WRITE_STALL_DURATION_TOTAL = "write-stall-duration-total";
+    private static final String BLOCK_CACHE_DATA_HIT_RATIO = "block-cache-data-hit-ratio";
+    private static final String BLOCK_CACHE_INDEX_HIT_RATIO = "block-cache-index-hit-ratio";
+    private static final String BLOCK_CACHE_FILTER_HIT_RATIO = "block-cache-filter-hit-ratio";
+    private static final String BYTES_READ_DURING_COMPACTION_RATE = "bytes-read-compaction-rate";
+    private static final String BYTES_WRITTEN_DURING_COMPACTION_RATE = "bytes-written-compaction-rate";
+    private static final String NUMBER_OF_OPEN_FILES = "number-open-files";
+    private static final String NUMBER_OF_FILE_ERRORS = "number-file-errors-total";
+
+    @Parameters(name = "{0}")
+    public static Collection data() {
+        return Arrays.asList(new Object[][] {
+            {StreamsConfig.EXACTLY_ONCE},
+            {StreamsConfig.AT_LEAST_ONCE}
+        });
+    }
+
+    @Parameter
+    public String processingGuarantee;
+
+    @Before
+    public void before() throws Exception {
+        CLUSTER.createTopic(STREAM_INPUT, 1, 3);
+    }
+
+    @After
+    public void after() throws Exception {
+        CLUSTER.deleteTopicsAndWait(STREAM_INPUT, STREAM_OUTPUT);
+    }
+
+    @Test
+    public void shouldExposeRocksDBMetricsForNonSegmentedStateStoreBeforeAndAfterFailureWithEmptyStateDir() throws Exception {
+        final Properties streamsConfiguration = streamsConfig();
+        IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration);
+        final StreamsBuilder builder = builderForNonSegmentedStateStore();
+
+        cleanUpStateRunAndVerify(
+            builder,
+            streamsConfiguration,
+            IntegerDeserializer.class,
+            StringDeserializer.class,
+            "rocksdb-state-id"
+        );
+
+        cleanUpStateRunAndVerify(
+            builder,
+            streamsConfiguration,
+            IntegerDeserializer.class,
+            StringDeserializer.class,
+            "rocksdb-state-id"
+        );
+    }
+
+    @Test
+    public void shouldExposeRocksDBMetricsForSegmentedStateStoreBeforeAndAfterFailureWithEmptyStateDir() throws Exception {
+        final Properties streamsConfiguration = streamsConfig();
+        IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration);
+        final StreamsBuilder builder = builderForSegmentedStateStore();
+
+        cleanUpStateRunAndVerify(
+            builder,
+            streamsConfiguration,
+            LongDeserializer.class,
+            LongDeserializer.class,
+            "rocksdb-window-state-id"
+        );
+
+        cleanUpStateRunAndVerify(
+            builder,
+            streamsConfiguration,
+            LongDeserializer.class,
+            LongDeserializer.class,
+            "rocksdb-window-state-id"
+        );
+    }
+
+    private Properties streamsConfig() {
+        final Properties streamsConfiguration = new Properties();
+        streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, "test-application");
+        streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
+        streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass());
+        streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        streamsConfiguration.put(StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG, Sensor.RecordingLevel.DEBUG.name);
+        streamsConfiguration.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 10 * 1024 * 1024L);
+        streamsConfiguration.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, processingGuarantee);
+        streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath());
+        return streamsConfiguration;
+    }
+
+    private StreamsBuilder builderForNonSegmentedStateStore() {
+        final StreamsBuilder builder = new StreamsBuilder();
+        builder.table(
+            STREAM_INPUT,
+            Materialized.as(Stores.persistentKeyValueStore(MY_STORE_PERSISTENT_KEY_VALUE)).withCachingEnabled()
+        ).toStream().to(STREAM_OUTPUT);
+        return builder;
+    }
+
+    private StreamsBuilder builderForSegmentedStateStore() {
+        final StreamsBuilder builder = new StreamsBuilder();
+        builder.stream(STREAM_INPUT, Consumed.with(Serdes.Integer(), Serdes.String()))
+            .groupByKey()
+            .windowedBy(TimeWindows.of(WINDOW_SIZE).grace(Duration.ZERO))
+            .aggregate(() -> 0L,
+                (aggKey, newValue, aggValue) -> aggValue,
+                Materialized.>as("time-windowed-aggregated-stream-store")
+                    .withValueSerde(Serdes.Long())
+                    .withRetention(WINDOW_SIZE))
+            .toStream()
+            .map((key, value) -> KeyValue.pair(value, value))
+            .to(STREAM_OUTPUT, Produced.with(Serdes.Long(), Serdes.Long()));
+        return builder;
+    }
+
+    private void cleanUpStateRunAndVerify(final StreamsBuilder builder,
+                                          final Properties streamsConfiguration,
+                                          final Class outputKeyDeserializer,
+                                          final Class outputValueDeserializer,
+                                          final String metricsScope) throws Exception {
+        final KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration);
+        kafkaStreams.cleanUp();
+        produceRecords();
+
+        StreamsTestUtils.startKafkaStreamsAndWaitForRunningState(kafkaStreams, 60000);
+
+        IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
+            TestUtils.consumerConfig(
+                CLUSTER.bootstrapServers(),
+                "consumerApp",
+                outputKeyDeserializer,
+                outputValueDeserializer,
+                new Properties()
+            ),
+            STREAM_OUTPUT,
+            1
+        );
+        verifyRocksDBMetrics(kafkaStreams, metricsScope);
+        kafkaStreams.close();
+    }
+
+    private void produceRecords() throws Exception {
+        final MockTime mockTime = new MockTime(WINDOW_SIZE.toMillis());
+        IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(
+            STREAM_INPUT,
+            Collections.singletonList(new KeyValue<>(1, "A")),
+            TestUtils.producerConfig(
+                CLUSTER.bootstrapServers(),
+                IntegerSerializer.class,
+                StringSerializer.class,
+                new Properties()
+            ),
+            mockTime.milliseconds()
+        );
+        IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(
+            STREAM_INPUT,
+            Collections.singletonList(new KeyValue<>(1, "B")),
+            TestUtils.producerConfig(
+                CLUSTER.bootstrapServers(),
+                IntegerSerializer.class,
+                StringSerializer.class,
+                new Properties()
+            ),
+            mockTime.milliseconds()
+        );
+        IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(
+            STREAM_INPUT,
+            Collections.singletonList(new KeyValue<>(1, "C")),
+            TestUtils.producerConfig(
+                CLUSTER.bootstrapServers(),
+                IntegerSerializer.class,
+                StringSerializer.class,
+                new Properties()
+            ),
+            mockTime.milliseconds()
+        );
+    }
+
+    private void verifyRocksDBMetrics(final KafkaStreams kafkaStreams, final String metricsScope) {
+        final List listMetricStore = new ArrayList(kafkaStreams.metrics().values()).stream()
+            .filter(m -> m.metricName().group().equals("stream-state-metrics") && m.metricName().tags().containsKey(metricsScope))
+            .collect(Collectors.toList());
+        checkMetricByName(listMetricStore, BYTES_WRITTEN_RATE, 1);
+        checkMetricByName(listMetricStore, BYTES_WRITTEN_TOTAL, 1);
+        checkMetricByName(listMetricStore, BYTES_READ_RATE, 1);
+        checkMetricByName(listMetricStore, BYTES_READ_TOTAL, 1);
+        checkMetricByName(listMetricStore, MEMTABLE_BYTES_FLUSHED_RATE, 1);
+        checkMetricByName(listMetricStore, MEMTABLE_BYTES_FLUSHED_TOTAL, 1);
+        checkMetricByName(listMetricStore, MEMTABLE_HIT_RATIO, 1);
+        checkMetricByName(listMetricStore, WRITE_STALL_DURATION_AVG, 1);
+        checkMetricByName(listMetricStore, WRITE_STALL_DURATION_TOTAL, 1);
+        checkMetricByName(listMetricStore, BLOCK_CACHE_DATA_HIT_RATIO, 1);
+        checkMetricByName(listMetricStore, BLOCK_CACHE_INDEX_HIT_RATIO, 1);
+        checkMetricByName(listMetricStore, BLOCK_CACHE_FILTER_HIT_RATIO, 1);
+        checkMetricByName(listMetricStore, BYTES_READ_DURING_COMPACTION_RATE, 1);
+        checkMetricByName(listMetricStore, BYTES_WRITTEN_DURING_COMPACTION_RATE, 1);
+        checkMetricByName(listMetricStore, NUMBER_OF_OPEN_FILES, 1);
+        checkMetricByName(listMetricStore, NUMBER_OF_FILE_ERRORS, 1);
+    }
+
+    private void checkMetricByName(final List listMetric, final String metricName, final int numMetric) {
+        final List metrics = listMetric.stream()
+            .filter(m -> m.metricName().name().equals(metricName))
+            .collect(Collectors.toList());
+        Assert.assertEquals("Size of metrics of type:'" + metricName + "' must be equal to " + numMetric + " but it's equal to " + metrics.size(), numMetric, metrics.size());
+        for (final Metric m : metrics) {
+            Assert.assertNotNull("Metric:'" + m.metricName() + "' must be not null", m.metricValue());
+        }
+    }
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/KeyValueSegmentsTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/KeyValueSegmentsTest.java
index ddf617596c417..b052ecb58a13d 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/KeyValueSegmentsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/KeyValueSegmentsTest.java
@@ -63,6 +63,7 @@ public void createContext() {
             new ThreadCache(new LogContext("testCache "), 0, new MockStreamsMetrics(new Metrics()))
         );
         segments = new KeyValueSegments(storeName, METRICS_SCOPE, RETENTION_PERIOD, SEGMENT_INTERVAL);
+        segments.openExisting(context, -1L);
     }
 
     @After
@@ -154,6 +155,7 @@ public void shouldCloseAllOpenSegments() {
     @Test
     public void shouldOpenExistingSegments() {
         segments = new KeyValueSegments("test",  METRICS_SCOPE, 4, 1);
+        segments.openExisting(context, -1L);
         segments.getOrCreateSegmentIfLive(0, context, -1L);
         segments.getOrCreateSegmentIfLive(1, context, -1L);
         segments.getOrCreateSegmentIfLive(2, context, -1L);
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java
index 0a28a773e9a90..36d53eacfb97b 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java
@@ -98,12 +98,12 @@ public class RocksDBStoreTest {
     public void setUp() {
         final Properties props = StreamsTestUtils.getStreamsConfig();
         props.put(StreamsConfig.ROCKSDB_CONFIG_SETTER_CLASS_CONFIG, MockRocksDbConfigSetter.class);
-        rocksDBStore = getRocksDBStore();
         dir = TestUtils.tempDirectory();
         context = new InternalMockProcessorContext(dir,
             Serdes.String(),
             Serdes.String(),
             new StreamsConfig(props));
+        rocksDBStore = getRocksDBStore();
         context.metrics().setRocksDBMetricsRecordingTrigger(new RocksDBMetricsRecordingTrigger());
     }
 
@@ -150,9 +150,7 @@ public void shouldAddStatisticsToInjectedMetricsRecorderWhenRecordingLevelIsDebu
         reset(metricsRecorder);
         metricsRecorder.addStatistics(
             eq(DB_NAME),
-            anyObject(Statistics.class),
-            eq(mockContext.metrics()),
-            eq(mockContext.taskId())
+            anyObject(Statistics.class)
         );
         replay(metricsRecorder);
 
@@ -286,7 +284,7 @@ public void shouldNotThrowExceptionOnRestoreWhenThereIsPreExistingRocksDbFiles()
     public void shouldCallRocksDbConfigSetter() {
         MockRocksDbConfigSetter.called = false;
 
-        rocksDBStore.openDB(context);
+        rocksDBStore.init(context, rocksDBStore);
 
         assertTrue(MockRocksDbConfigSetter.called);
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentIteratorTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentIteratorTest.java
index 7944e5f0f5188..437c556ed171d 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentIteratorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentIteratorTest.java
@@ -62,8 +62,8 @@ public void before() {
                     new LogContext("testCache "),
                     0,
                     new MockStreamsMetrics(new Metrics())));
-        segmentOne.openDB(context);
-        segmentTwo.openDB(context);
+        segmentOne.init(context, segmentOne);
+        segmentTwo.init(context, segmentTwo);
         segmentOne.put(Bytes.wrap("a".getBytes()), "1".getBytes());
         segmentOne.put(Bytes.wrap("b".getBytes()), "2".getBytes());
         segmentTwo.put(Bytes.wrap("c".getBytes()), "3".getBytes());
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/TimestampedSegmentsTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/TimestampedSegmentsTest.java
index 8d294ce33ee86..ce4652d5b475c 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/TimestampedSegmentsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/TimestampedSegmentsTest.java
@@ -63,6 +63,7 @@ public void createContext() {
             new ThreadCache(new LogContext("testCache "), 0, new MockStreamsMetrics(new Metrics()))
         );
         segments = new TimestampedSegments(storeName, METRICS_SCOPE, RETENTION_PERIOD, SEGMENT_INTERVAL);
+        segments.openExisting(context, -1L);
     }
 
     @After
@@ -155,6 +156,7 @@ public void shouldCloseAllOpenSegments() {
     @Test
     public void shouldOpenExistingSegments() {
         segments = new TimestampedSegments("test", METRICS_SCOPE, 4, 1);
+        segments.openExisting(context, -1L);
         segments.getOrCreateSegmentIfLive(0, context, -1L);
         segments.getOrCreateSegmentIfLive(1, context, -1L);
         segments.getOrCreateSegmentIfLive(2, context, -1L);
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorderTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorderTest.java
index f22c02ed4d4c2..fc60c2784b35a 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorderTest.java
@@ -16,7 +16,9 @@
  */
 package org.apache.kafka.streams.state.internals.metrics;
 
+import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.processor.TaskId;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.apache.kafka.streams.state.internals.metrics.RocksDBMetrics.RocksDBMetricContext;
@@ -35,11 +37,12 @@
 import static org.easymock.EasyMock.mock;
 import static org.easymock.EasyMock.niceMock;
 import static org.easymock.EasyMock.resetToNice;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertThrows;
 import static org.powermock.api.easymock.PowerMock.reset;
 import static org.powermock.api.easymock.PowerMock.createMock;
 import static org.powermock.api.easymock.PowerMock.mockStatic;
-import static org.powermock.api.easymock.PowerMock.mockStaticNice;
 import static org.powermock.api.easymock.PowerMock.replay;
 import static org.powermock.api.easymock.PowerMock.verify;
 
@@ -71,106 +74,110 @@ public class RocksDBMetricsRecorderTest {
     private final StreamsMetricsImpl streamsMetrics = niceMock(StreamsMetricsImpl.class);
     private final RocksDBMetricsRecordingTrigger recordingTrigger = mock(RocksDBMetricsRecordingTrigger.class);
     private final TaskId taskId1 = new TaskId(0, 0);
-    private final TaskId taskId2 = new TaskId(0, 2);
+    private final TaskId taskId2 = new TaskId(0, 1);
 
     private final RocksDBMetricsRecorder recorder = new RocksDBMetricsRecorder(METRICS_SCOPE, THREAD_ID, STORE_NAME);
 
     @Before
     public void setUp() {
+        setUpMetricsStubMock();
         expect(streamsMetrics.rocksDBMetricsRecordingTrigger()).andStubReturn(recordingTrigger);
         replay(streamsMetrics);
+        recorder.init(streamsMetrics, taskId1);
     }
 
     @Test
-    public void shouldSetStatsLevelToExceptDetailedTimersWhenStatisticsIsAdded() {
-        mockStaticNice(RocksDBMetrics.class);
-        replay(RocksDBMetrics.class);
-        statisticsToAdd1.setStatsLevel(StatsLevel.EXCEPT_DETAILED_TIMERS);
-        replay(statisticsToAdd1);
+    public void shouldInitMetricsRecorder() {
+        setUpMetricsMock();
 
-        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
+        recorder.init(streamsMetrics, taskId1);
 
-        verify(statisticsToAdd1);
+        verify(RocksDBMetrics.class);
+        assertThat(recorder.taskId(), is(taskId1));
     }
 
     @Test
-    public void shouldThrowIfTaskIdOfStatisticsToAddDiffersFromInitialisedOne() {
-        mockStaticNice(RocksDBMetrics.class);
-        replay(RocksDBMetrics.class);
-        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
+    public void shouldThrowIfMetricRecorderIsReInitialisedWithDifferentTask() {
+        setUpMetricsStubMock();
+        recorder.init(streamsMetrics, taskId1);
+
         assertThrows(
             IllegalStateException.class,
-            () -> recorder.addStatistics(SEGMENT_STORE_NAME_2, statisticsToAdd2, streamsMetrics, taskId2)
+            () -> recorder.init(streamsMetrics, taskId2)
         );
     }
 
     @Test
-    public void shouldThrowIfStatisticsToAddHasBeenAlreadyAdded() {
-        mockStaticNice(RocksDBMetrics.class);
-        replay(RocksDBMetrics.class);
-        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
+    public void shouldThrowIfMetricRecorderIsReInitialisedWithDifferentStreamsMetrics() {
+        setUpMetricsStubMock();
+        recorder.init(streamsMetrics, taskId1);
 
         assertThrows(
             IllegalStateException.class,
-            () -> recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1)
+            () -> recorder.init(
+                new StreamsMetricsImpl(new Metrics(), "test-client", StreamsConfig.METRICS_LATEST),
+                taskId1
+            )
         );
     }
 
     @Test
-    public void shouldInitMetricsAndAddItselfToRecordingTriggerOnlyWhenFirstStatisticsIsAdded() {
-        setUpMetricsMock();
-        recordingTrigger.addMetricsRecorder(recorder);
-        replay(recordingTrigger);
+    public void shouldSetStatsLevelToExceptDetailedTimersWhenStatisticsIsAdded() {
+        statisticsToAdd1.setStatsLevel(StatsLevel.EXCEPT_DETAILED_TIMERS);
+        replay(statisticsToAdd1);
 
-        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
+        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1);
 
-        verify(recordingTrigger);
-        verify(RocksDBMetrics.class);
+        verify(statisticsToAdd1);
+    }
 
-        mockStatic(RocksDBMetrics.class);
-        replay(RocksDBMetrics.class);
-        reset(recordingTrigger);
+    @Test
+    public void shouldThrowIfStatisticsToAddHasBeenAlreadyAdded() {
+        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1);
+
+        assertThrows(
+            IllegalStateException.class,
+            () -> recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1)
+        );
+    }
+
+    @Test
+    public void shouldAddItselfToRecordingTriggerWhenFirstStatisticsIsAddedToNewlyCreatedRecorder() {
+        recordingTrigger.addMetricsRecorder(recorder);
         replay(recordingTrigger);
 
-        recorder.addStatistics(SEGMENT_STORE_NAME_2, statisticsToAdd2, streamsMetrics, taskId1);
+        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1);
 
         verify(recordingTrigger);
-        verify(RocksDBMetrics.class);
     }
 
     @Test
-    public void shouldAddItselfToRecordingTriggerWhenEmptyButInitialised() {
-        mockStaticNice(RocksDBMetrics.class);
-        replay(RocksDBMetrics.class);
-        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
+    public void shouldAddItselfToRecordingTriggerWhenFirstStatisticsIsAddedAfterLastStatisticsWasRemoved() {
+        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1);
         recorder.removeStatistics(SEGMENT_STORE_NAME_1);
         reset(recordingTrigger);
         recordingTrigger.addMetricsRecorder(recorder);
         replay(recordingTrigger);
 
-        recorder.addStatistics(SEGMENT_STORE_NAME_2, statisticsToAdd2, streamsMetrics, taskId1);
+        recorder.addStatistics(SEGMENT_STORE_NAME_2, statisticsToAdd2);
 
         verify(recordingTrigger);
     }
 
     @Test
     public void shouldNotAddItselfToRecordingTriggerWhenNotEmpty() {
-        mockStaticNice(RocksDBMetrics.class);
-        replay(RocksDBMetrics.class);
-        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
+        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1);
         reset(recordingTrigger);
         replay(recordingTrigger);
 
-        recorder.addStatistics(SEGMENT_STORE_NAME_2, statisticsToAdd2, streamsMetrics, taskId1);
+        recorder.addStatistics(SEGMENT_STORE_NAME_2, statisticsToAdd2);
 
         verify(recordingTrigger);
     }
 
     @Test
     public void shouldCloseStatisticsWhenStatisticsIsRemoved() {
-        mockStaticNice(RocksDBMetrics.class);
-        replay(RocksDBMetrics.class);
-        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
+        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1);
         reset(statisticsToAdd1);
         statisticsToAdd1.close();
         replay(statisticsToAdd1);
@@ -182,10 +189,8 @@ public void shouldCloseStatisticsWhenStatisticsIsRemoved() {
 
     @Test
     public void shouldRemoveItselfFromRecordingTriggerWhenLastStatisticsIsRemoved() {
-        mockStaticNice(RocksDBMetrics.class);
-        replay(RocksDBMetrics.class);
-        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
-        recorder.addStatistics(SEGMENT_STORE_NAME_2, statisticsToAdd2, streamsMetrics, taskId1);
+        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1);
+        recorder.addStatistics(SEGMENT_STORE_NAME_2, statisticsToAdd2);
         reset(recordingTrigger);
         replay(recordingTrigger);
 
@@ -204,9 +209,8 @@ public void shouldRemoveItselfFromRecordingTriggerWhenLastStatisticsIsRemoved()
 
     @Test
     public void shouldThrowIfStatisticsToRemoveNotFound() {
-        mockStaticNice(RocksDBMetrics.class);
-        replay(RocksDBMetrics.class);
-        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
+        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1);
+
         assertThrows(
             IllegalStateException.class,
             () -> recorder.removeStatistics(SEGMENT_STORE_NAME_2)
@@ -215,9 +219,8 @@ public void shouldThrowIfStatisticsToRemoveNotFound() {
 
     @Test
     public void shouldRecordMetrics() {
-        setUpMetricsMock();
-        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
-        recorder.addStatistics(SEGMENT_STORE_NAME_2, statisticsToAdd2, streamsMetrics, taskId1);
+        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1);
+        recorder.addStatistics(SEGMENT_STORE_NAME_2, statisticsToAdd2);
         reset(statisticsToAdd1);
         reset(statisticsToAdd2);
 
@@ -303,10 +306,9 @@ public void shouldRecordMetrics() {
 
     @Test
     public void shouldCorrectlyHandleHitRatioRecordingsWithZeroHitsAndMisses() {
-        setUpMetricsMock();
-        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1, streamsMetrics, taskId1);
         resetToNice(statisticsToAdd1);
-        expect(statisticsToAdd1.getTickerCount(anyObject())).andReturn(0L).anyTimes();
+        recorder.addStatistics(SEGMENT_STORE_NAME_1, statisticsToAdd1);
+        expect(statisticsToAdd1.getTickerCount(anyObject())).andStubReturn(0L);
         replay(statisticsToAdd1);
         memtableHitRatioSensor.record(0);
         blockCacheDataHitRatioSensor.record(0);
@@ -355,4 +357,35 @@ private void setUpMetricsMock() {
             .andReturn(numberOfFileErrorsSensor);
         replay(RocksDBMetrics.class);
     }
+
+    private void setUpMetricsStubMock() {
+        mockStatic(RocksDBMetrics.class);
+        final RocksDBMetricContext metricsContext =
+            new RocksDBMetricContext(THREAD_ID, taskId1.toString(), METRICS_SCOPE, STORE_NAME);
+        expect(RocksDBMetrics.bytesWrittenToDatabaseSensor(streamsMetrics, metricsContext))
+            .andStubReturn(bytesWrittenToDatabaseSensor);
+        expect(RocksDBMetrics.bytesReadFromDatabaseSensor(streamsMetrics, metricsContext))
+            .andStubReturn(bytesReadFromDatabaseSensor);
+        expect(RocksDBMetrics.memtableBytesFlushedSensor(streamsMetrics, metricsContext))
+            .andStubReturn(memtableBytesFlushedSensor);
+        expect(RocksDBMetrics.memtableHitRatioSensor(streamsMetrics, metricsContext))
+            .andStubReturn(memtableHitRatioSensor);
+        expect(RocksDBMetrics.writeStallDurationSensor(streamsMetrics, metricsContext))
+            .andStubReturn(writeStallDurationSensor);
+        expect(RocksDBMetrics.blockCacheDataHitRatioSensor(streamsMetrics, metricsContext))
+            .andStubReturn(blockCacheDataHitRatioSensor);
+        expect(RocksDBMetrics.blockCacheIndexHitRatioSensor(streamsMetrics, metricsContext))
+            .andStubReturn(blockCacheIndexHitRatioSensor);
+        expect(RocksDBMetrics.blockCacheFilterHitRatioSensor(streamsMetrics, metricsContext))
+            .andStubReturn(blockCacheFilterHitRatioSensor);
+        expect(RocksDBMetrics.bytesWrittenDuringCompactionSensor(streamsMetrics, metricsContext))
+            .andStubReturn(bytesWrittenDuringCompactionSensor);
+        expect(RocksDBMetrics.bytesReadDuringCompactionSensor(streamsMetrics, metricsContext))
+            .andStubReturn(bytesReadDuringCompactionSensor);
+        expect(RocksDBMetrics.numberOfOpenFilesSensor(streamsMetrics, metricsContext))
+            .andStubReturn(numberOfOpenFilesSensor);
+        expect(RocksDBMetrics.numberOfFileErrorsSensor(streamsMetrics, metricsContext))
+            .andStubReturn(numberOfFileErrorsSensor);
+        replay(RocksDBMetrics.class);
+    }
 }
\ No newline at end of file

From e5a8961c58a89f542e799096ed5435a022154600 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Thu, 13 Feb 2020 15:00:36 -0800
Subject: [PATCH 0887/1071] HOTFIX: cherry-pick conflicts

---
 .../state/internals/metrics/RocksDBMetricsRecorderTest.java    | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorderTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorderTest.java
index fc60c2784b35a..9f518c82416da 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorderTest.java
@@ -18,7 +18,6 @@
 
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
-import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.processor.TaskId;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.apache.kafka.streams.state.internals.metrics.RocksDBMetrics.RocksDBMetricContext;
@@ -115,7 +114,7 @@ public void shouldThrowIfMetricRecorderIsReInitialisedWithDifferentStreamsMetric
         assertThrows(
             IllegalStateException.class,
             () -> recorder.init(
-                new StreamsMetricsImpl(new Metrics(), "test-client", StreamsConfig.METRICS_LATEST),
+                new StreamsMetricsImpl(new Metrics(), "test-client", StreamsMetricsImpl.METRICS_LATEST),
                 taskId1
             )
         );

From 667ec2ddf9be741ab48cc011039e70e985099ed9 Mon Sep 17 00:00:00 2001
From: Boyang Chen 
Date: Sun, 16 Feb 2020 12:06:33 -0800
Subject: [PATCH 0888/1071] KAFKA-9535; Update metadata before retrying
 partitions when fetching offsets (#8088)

Today if we attempt to list offsets with a fenced leader epoch, consumer will retry without updating the metadata until the timeout is reached. This affects synchronous APIs such as `offsetsForTimes`, `beginningOffsets`, and `endOffsets`. The fix in this patch is to trigger the metadata update call whenever we see a retriable error before additional attempts.

Reviewers: Jason Gustafson 
---
 .../clients/consumer/internals/Fetcher.java   | 128 +++++++--------
 .../common/requests/ListOffsetRequest.java    |  14 ++
 .../consumer/internals/FetcherTest.java       | 147 +++++++++++++-----
 3 files changed, 186 insertions(+), 103 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
index 384da315e70f4..a8cb4ebfd8a10 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
@@ -70,6 +70,7 @@
 import org.apache.kafka.common.requests.MetadataRequest;
 import org.apache.kafka.common.requests.MetadataResponse;
 import org.apache.kafka.common.requests.OffsetsForLeaderEpochRequest;
+import org.apache.kafka.common.requests.RequestUtils;
 import org.apache.kafka.common.serialization.Deserializer;
 import org.apache.kafka.common.utils.CloseableIterator;
 import org.apache.kafka.common.utils.LogContext;
@@ -536,24 +537,21 @@ private ListOffsetResult fetchOffsetsByTimes(Map timestamp
             RequestFuture future = sendListOffsetsRequests(remainingToSearch, requireTimestamps);
             client.poll(future, timer);
 
-            if (!future.isDone())
+            if (!future.isDone()) {
                 break;
-
-            if (future.succeeded()) {
+            } else if (future.succeeded()) {
                 ListOffsetResult value = future.value();
                 result.fetchedOffsets.putAll(value.fetchedOffsets);
-                if (value.partitionsToRetry.isEmpty())
-                    return result;
-
                 remainingToSearch.keySet().retainAll(value.partitionsToRetry);
             } else if (!future.isRetriable()) {
                 throw future.exception();
             }
 
-            if (metadata.updateRequested())
+            if (remainingToSearch.isEmpty()) {
+                return result;
+            } else {
                 client.awaitMetadataUpdate(timer);
-            else
-                timer.sleep(retryBackoffMs);
+            }
         } while (timer.notExpired());
 
         throw new TimeoutException("Failed to get offsets by times in " + timer.elapsedMs() + "ms");
@@ -921,8 +919,9 @@ private Map> groupLis
                         currentInfo.get().partitionInfo().leader(), tp);
                 partitionsToRetry.add(tp);
             } else {
-                partitionDataMap.put(tp,
-                        new ListOffsetRequest.PartitionData(offset, Optional.of(currentInfo.get().epoch())));
+                int currentEpoch = currentInfo.get().epoch();
+                Optional currentEpochOpt = currentEpoch < 0 ? Optional.empty() : Optional.of(currentEpoch);
+                partitionDataMap.put(tp, new ListOffsetRequest.PartitionData(offset, currentEpochOpt));
             }
         }
         return regroupPartitionMapByNode(partitionDataMap);
@@ -978,62 +977,66 @@ private void handleListOffsetResponse(Map 1) {
-                        future.raise(new IllegalStateException("Unexpected partitionData response of length " +
-                                partitionData.offsets.size()));
-                        return;
-                    } else if (partitionData.offsets.isEmpty()) {
-                        offset = ListOffsetResponse.UNKNOWN_OFFSET;
-                    } else {
-                        offset = partitionData.offsets.get(0);
-                    }
-                    log.debug("Handling v0 ListOffsetResponse response for {}. Fetched offset {}",
+            switch (error) {
+                case NONE:
+                    if (partitionData.offsets != null) {
+                        // Handle v0 response
+                        long offset;
+                        if (partitionData.offsets.size() > 1) {
+                            future.raise(new IllegalStateException("Unexpected partitionData response of length " +
+                                                                       partitionData.offsets.size()));
+                            return;
+                        } else if (partitionData.offsets.isEmpty()) {
+                            offset = ListOffsetResponse.UNKNOWN_OFFSET;
+                        } else {
+                            offset = partitionData.offsets.get(0);
+                        }
+                        log.debug("Handling v0 ListOffsetResponse response for {}. Fetched offset {}",
                             topicPartition, offset);
-                    if (offset != ListOffsetResponse.UNKNOWN_OFFSET) {
-                        ListOffsetData offsetData = new ListOffsetData(offset, null, Optional.empty());
-                        fetchedOffsets.put(topicPartition, offsetData);
-                    }
-                } else {
-                    // Handle v1 and later response
-                    log.debug("Handling ListOffsetResponse response for {}. Fetched offset {}, timestamp {}",
+                        if (offset != ListOffsetResponse.UNKNOWN_OFFSET) {
+                            ListOffsetData offsetData = new ListOffsetData(offset, null, Optional.empty());
+                            fetchedOffsets.put(topicPartition, offsetData);
+                        }
+                    } else {
+                        // Handle v1 and later response
+                        log.debug("Handling ListOffsetResponse response for {}. Fetched offset {}, timestamp {}",
                             topicPartition, partitionData.offset, partitionData.timestamp);
-                    if (partitionData.offset != ListOffsetResponse.UNKNOWN_OFFSET) {
-                        ListOffsetData offsetData = new ListOffsetData(partitionData.offset, partitionData.timestamp,
+                        if (partitionData.offset != ListOffsetResponse.UNKNOWN_OFFSET) {
+                            ListOffsetData offsetData = new ListOffsetData(partitionData.offset, partitionData.timestamp,
                                 partitionData.leaderEpoch);
-                        fetchedOffsets.put(topicPartition, offsetData);
+                            fetchedOffsets.put(topicPartition, offsetData);
+                        }
                     }
-                }
-            } else if (error == Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT) {
-                // The message format on the broker side is before 0.10.0, which means it does not
-                // support timestamps. We treat this case the same as if we weren't able to find an
-                // offset corresponding to the requested timestamp and leave it out of the result.
-                log.debug("Cannot search by timestamp for partition {} because the message format version " +
-                        "is before 0.10.0", topicPartition);
-            } else if (error == Errors.NOT_LEADER_FOR_PARTITION ||
-                       error == Errors.REPLICA_NOT_AVAILABLE ||
-                       error == Errors.KAFKA_STORAGE_ERROR ||
-                       error == Errors.OFFSET_NOT_AVAILABLE ||
-                       error == Errors.LEADER_NOT_AVAILABLE) {
-                log.debug("Attempt to fetch offsets for partition {} failed due to {}, retrying.",
+                    break;
+                case UNSUPPORTED_FOR_MESSAGE_FORMAT:
+                    // The message format on the broker side is before 0.10.0, which means it does not
+                    // support timestamps. We treat this case the same as if we weren't able to find an
+                    // offset corresponding to the requested timestamp and leave it out of the result.
+                    log.debug("Cannot search by timestamp for partition {} because the message format version " +
+                                  "is before 0.10.0", topicPartition);
+                    break;
+                case NOT_LEADER_FOR_PARTITION:
+                case REPLICA_NOT_AVAILABLE:
+                case KAFKA_STORAGE_ERROR:
+                case OFFSET_NOT_AVAILABLE:
+                case LEADER_NOT_AVAILABLE:
+                case FENCED_LEADER_EPOCH:
+                case UNKNOWN_LEADER_EPOCH:
+                    log.debug("Attempt to fetch offsets for partition {} failed due to {}, retrying.",
                         topicPartition, error);
-                partitionsToRetry.add(topicPartition);
-            } else if (error == Errors.FENCED_LEADER_EPOCH ||
-                       error == Errors.UNKNOWN_LEADER_EPOCH) {
-                log.debug("Attempt to fetch offsets for partition {} failed due to {}, retrying.",
-                        topicPartition, error);
-                partitionsToRetry.add(topicPartition);
-            } else if (error == Errors.UNKNOWN_TOPIC_OR_PARTITION) {
-                log.warn("Received unknown topic or partition error in ListOffset request for partition {}", topicPartition);
-                partitionsToRetry.add(topicPartition);
-            } else if (error == Errors.TOPIC_AUTHORIZATION_FAILED) {
-                unauthorizedTopics.add(topicPartition.topic());
-            } else {
-                log.warn("Attempt to fetch offsets for partition {} failed due to: {}, retrying.", topicPartition, error.message());
-                partitionsToRetry.add(topicPartition);
+                    partitionsToRetry.add(topicPartition);
+                    break;
+                case UNKNOWN_TOPIC_OR_PARTITION:
+                    log.warn("Received unknown topic or partition error in ListOffset request for partition {}", topicPartition);
+                    partitionsToRetry.add(topicPartition);
+                    break;
+                case TOPIC_AUTHORIZATION_FAILED:
+                    unauthorizedTopics.add(topicPartition.topic());
+                    break;
+                default:
+                    log.warn("Attempt to fetch offsets for partition {} failed due to unexpected exception: {}, retrying.",
+                        topicPartition, error.message());
+                    partitionsToRetry.add(topicPartition);
             }
         }
 
@@ -1233,7 +1236,6 @@ private CompletedFetch initializeCompletedFetch(CompletedFetch nextCompletedFetc
                     });
                 }
 
-
                 nextCompletedFetch.initialized = true;
             } else if (error == Errors.NOT_LEADER_FOR_PARTITION ||
                        error == Errors.REPLICA_NOT_AVAILABLE ||
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java
index 47c47d2968dc0..01418c86e5832 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java
@@ -31,6 +31,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 
@@ -203,6 +204,19 @@ public PartitionData(long timestamp, Optional currentLeaderEpoch) {
             this(timestamp, 1, currentLeaderEpoch);
         }
 
+        @Override
+        public boolean equals(Object obj) {
+            if (!(obj instanceof PartitionData)) return false;
+            PartitionData other = (PartitionData) obj;
+            return this.timestamp == other.timestamp &&
+                this.currentLeaderEpoch.equals(other.currentLeaderEpoch);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(timestamp, currentLeaderEpoch);
+        }
+
         @Override
         public String toString() {
             StringBuilder bld = new StringBuilder();
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
index 1aa7814b484b0..436c40b76a3c3 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
@@ -65,7 +65,6 @@
 import org.apache.kafka.common.record.Records;
 import org.apache.kafka.common.record.SimpleRecord;
 import org.apache.kafka.common.record.TimestampType;
-import org.apache.kafka.common.requests.AbstractRequest;
 import org.apache.kafka.common.requests.ApiVersionsResponse;
 import org.apache.kafka.common.requests.EpochEndOffset;
 import org.apache.kafka.common.requests.FetchRequest;
@@ -406,13 +405,10 @@ public void testFetchError() {
     }
 
     private MockClient.RequestMatcher matchesOffset(final TopicPartition tp, final long offset) {
-        return new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                FetchRequest fetch = (FetchRequest) body;
-                return fetch.fetchData().containsKey(tp) &&
-                        fetch.fetchData().get(tp).fetchOffset == offset;
-            }
+        return body -> {
+            FetchRequest fetch = (FetchRequest) body;
+            return fetch.fetchData().containsKey(tp) &&
+                    fetch.fetchData().get(tp).fetchOffset == offset;
         };
     }
 
@@ -2371,13 +2367,9 @@ private Map>> fetchRecords(
 
     @Test
     public void testGetOffsetsForTimesTimeout() {
-        try {
-            buildFetcher();
-            fetcher.offsetsForTimes(Collections.singletonMap(new TopicPartition(topicName, 2), 1000L), time.timer(100L));
-            fail("Should throw timeout exception.");
-        } catch (TimeoutException e) {
-            // let it go.
-        }
+        buildFetcher();
+        assertThrows(TimeoutException.class, () -> fetcher.offsetsForTimes(
+            Collections.singletonMap(new TopicPartition(topicName, 2), 1000L), time.timer(100L)));
     }
 
     @Test
@@ -2385,7 +2377,7 @@ public void testGetOffsetsForTimes() {
         buildFetcher();
 
         // Empty map
-        assertTrue(fetcher.offsetsForTimes(new HashMap(), time.timer(100L)).isEmpty());
+        assertTrue(fetcher.offsetsForTimes(new HashMap<>(), time.timer(100L)).isEmpty());
         // Unknown Offset
         testGetOffsetsForTimesWithUnknownOffset();
         // Error code none with unknown offset
@@ -2421,6 +2413,89 @@ public void testGetOffsetsFencedLeaderEpoch() {
         assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds()));
     }
 
+    @Test
+    public void testGetOffsetByTimeWithPartitionsRetryCouldTriggerMetadataUpdate() {
+        List retriableErrors = Arrays.asList(Errors.NOT_LEADER_FOR_PARTITION,
+            Errors.REPLICA_NOT_AVAILABLE, Errors.KAFKA_STORAGE_ERROR, Errors.OFFSET_NOT_AVAILABLE,
+            Errors.LEADER_NOT_AVAILABLE, Errors.FENCED_LEADER_EPOCH, Errors.UNKNOWN_LEADER_EPOCH);
+
+        final int newLeaderEpoch = 3;
+        MetadataResponse updatedMetadata = TestUtils.metadataUpdateWith("dummy", 3,
+            singletonMap(topicName, Errors.NONE), singletonMap(topicName, 4), tp -> newLeaderEpoch);
+
+        Node originalLeader = initialUpdateResponse.cluster().leaderFor(tp1);
+        Node newLeader = updatedMetadata.cluster().leaderFor(tp1);
+        assertNotEquals(originalLeader, newLeader);
+
+        for (Errors retriableError : retriableErrors) {
+            buildFetcher();
+
+            subscriptions.assignFromUser(Utils.mkSet(tp0, tp1));
+            client.updateMetadata(initialUpdateResponse);
+
+            final long fetchTimestamp = 10L;
+            Map allPartitionData = new HashMap<>();
+            allPartitionData.put(tp0, new ListOffsetResponse.PartitionData(
+                Errors.NONE, fetchTimestamp, 4L, Optional.empty()));
+            allPartitionData.put(tp1, new ListOffsetResponse.PartitionData(
+                retriableError, ListOffsetRequest.LATEST_TIMESTAMP, -1L, Optional.empty()));
+
+            client.prepareResponseFrom(body -> {
+                boolean isListOffsetRequest = body instanceof ListOffsetRequest;
+                if (isListOffsetRequest) {
+                    ListOffsetRequest request = (ListOffsetRequest) body;
+                    Map expectedTopicPartitions = new HashMap<>();
+                    expectedTopicPartitions.put(tp0, new ListOffsetRequest.PartitionData(
+                        fetchTimestamp, Optional.empty()));
+                    expectedTopicPartitions.put(tp1, new ListOffsetRequest.PartitionData(
+                        fetchTimestamp, Optional.empty()));
+
+                    return request.partitionTimestamps().equals(expectedTopicPartitions);
+                } else {
+                    return false;
+                }
+            }, new ListOffsetResponse(allPartitionData), originalLeader);
+
+            client.prepareMetadataUpdate(updatedMetadata);
+
+            // If the metadata wasn't updated before retrying, the fetcher would consult the original leader and hit a NOT_LEADER exception.
+            // We will count the answered future response in the end to verify if this is the case.
+            Map paritionDataWithFatalError = new HashMap<>(allPartitionData);
+            paritionDataWithFatalError.put(tp1, new ListOffsetResponse.PartitionData(
+                Errors.NOT_LEADER_FOR_PARTITION, ListOffsetRequest.LATEST_TIMESTAMP, -1L, Optional.empty()));
+            client.prepareResponseFrom(new ListOffsetResponse(paritionDataWithFatalError), originalLeader);
+
+            // The request to new leader must only contain one partition tp1 with error.
+            client.prepareResponseFrom(body -> {
+                boolean isListOffsetRequest = body instanceof ListOffsetRequest;
+                if (isListOffsetRequest) {
+                    ListOffsetRequest request = (ListOffsetRequest) body;
+
+                    return request.partitionTimestamps().equals(
+                        Collections.singletonMap(tp1, new ListOffsetRequest.PartitionData(
+                            fetchTimestamp, Optional.of(newLeaderEpoch))));
+                } else {
+                    return false;
+                }
+            }, listOffsetResponse(tp1, Errors.NONE, fetchTimestamp, 5L), newLeader);
+
+            Map offsetAndTimestampMap =
+                fetcher.offsetsForTimes(
+                    Utils.mkMap(Utils.mkEntry(tp0, fetchTimestamp),
+                    Utils.mkEntry(tp1, fetchTimestamp)), time.timer(Integer.MAX_VALUE));
+
+            assertEquals(Utils.mkMap(
+                Utils.mkEntry(tp0, new OffsetAndTimestamp(4L, fetchTimestamp)),
+                Utils.mkEntry(tp1, new OffsetAndTimestamp(5L, fetchTimestamp))), offsetAndTimestampMap);
+
+            // The NOT_LEADER exception future should not be cleared as we already refreshed the metadata before
+            // first retry, thus never hitting.
+            assertEquals(1, client.numAwaitingResponses());
+
+            fetcher.close();
+        }
+    }
+
     @Test
     public void testGetOffsetsUnknownLeaderEpoch() {
         buildFetcher();
@@ -2475,6 +2550,7 @@ public void testGetOffsetsIncludesLeaderEpoch() {
     public void testGetOffsetsForTimesWhenSomeTopicPartitionLeadersNotKnownInitially() {
         buildFetcher();
 
+        subscriptions.assignFromUser(Utils.mkSet(tp0, tp1));
         final String anotherTopic = "another-topic";
         final TopicPartition t2p0 = new TopicPartition(anotherTopic, 0);
 
@@ -2583,7 +2659,7 @@ public void testReturnCommittedTransactions() {
                 new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes()),
                 new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes()));
 
-        currentOffset += commitTransaction(buffer, 1L, currentOffset);
+        commitTransaction(buffer, 1L, currentOffset);
         buffer.flip();
 
         List abortedTransactions = new ArrayList<>();
@@ -2595,13 +2671,10 @@ public void testReturnCommittedTransactions() {
         // normal fetch
         assertEquals(1, fetcher.sendFetches());
         assertFalse(fetcher.hasCompletedFetches());
-        client.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                FetchRequest request = (FetchRequest) body;
-                assertEquals(IsolationLevel.READ_COMMITTED, request.isolationLevel());
-                return true;
-            }
+        client.prepareResponse(body -> {
+            FetchRequest request = (FetchRequest) body;
+            assertEquals(IsolationLevel.READ_COMMITTED, request.isolationLevel());
+            return true;
         }, fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0));
 
         consumerClient.poll(time.timer(0));
@@ -2733,7 +2806,7 @@ public void testMultipleAbortMarkers() {
         for (ConsumerRecord consumerRecord : fetchedConsumerRecords) {
             actuallyCommittedKeys.add(new String(consumerRecord.key(), StandardCharsets.UTF_8));
         }
-        assertTrue(actuallyCommittedKeys.equals(committedKeys));
+        assertEquals(actuallyCommittedKeys, committedKeys);
     }
 
     @Test
@@ -3271,13 +3344,10 @@ public void testEmptyControlBatch() {
         // normal fetch
         assertEquals(1, fetcher.sendFetches());
         assertFalse(fetcher.hasCompletedFetches());
-        client.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                FetchRequest request = (FetchRequest) body;
-                assertEquals(IsolationLevel.READ_COMMITTED, request.isolationLevel());
-                return true;
-            }
+        client.prepareResponse(body -> {
+            FetchRequest request = (FetchRequest) body;
+            assertEquals(IsolationLevel.READ_COMMITTED, request.isolationLevel());
+            return true;
         }, fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0));
 
         consumerClient.poll(time.timer(0));
@@ -3311,12 +3381,11 @@ private int appendTransactionalRecords(ByteBuffer buffer, long pid, long baseOff
         return appendTransactionalRecords(buffer, pid, baseOffset, (int) baseOffset, records);
     }
 
-    private int commitTransaction(ByteBuffer buffer, long producerId, long baseOffset) {
+    private void commitTransaction(ByteBuffer buffer, long producerId, long baseOffset) {
         short producerEpoch = 0;
         int partitionLeaderEpoch = 0;
         MemoryRecords.writeEndTransactionalMarker(buffer, baseOffset, time.milliseconds(), partitionLeaderEpoch, producerId, producerEpoch,
                 new EndTransactionMarker(ControlRecordType.COMMIT, 0));
-        return 1;
     }
 
     private int abortTransaction(ByteBuffer buffer, long producerId, long baseOffset) {
@@ -3804,12 +3873,10 @@ public void testFetchCompletedBeforeHandlerAdded() {
 
     private MockClient.RequestMatcher listOffsetRequestMatcher(final long timestamp) {
         // matches any list offset request with the provided timestamp
-        return new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                ListOffsetRequest req = (ListOffsetRequest) body;
-                return timestamp == req.partitionTimestamps().get(tp0).timestamp;
-            }
+        return body -> {
+            ListOffsetRequest req = (ListOffsetRequest) body;
+            return req.partitionTimestamps().equals(Collections.singletonMap(
+                tp0, new ListOffsetRequest.PartitionData(timestamp, Optional.empty())));
         };
     }
 

From d0ea2b1065a2934e9444b47e27aa03c664b073c9 Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Mon, 17 Feb 2020 07:51:26 -0800
Subject: [PATCH 0889/1071] KAFKA-9515: Upgrade ZooKeeper to 3.5.7 (#8125)

A couple of critical fixes:

ZOOKEEPER-3644: Data loss after upgrading standalone ZK server 3.4.14 to 3.5.6 with snapshot.trust.empty=true
ZOOKEEPER-3701: Split brain on log disk full (3.5)

Full release notes:
https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12310801&version=12346098

Also fixed an existing unused import issue that was causing checkstyle to fail in the
cherry-pick to the 2.4 branch.

Reviewers: Bill Bejeck 
---
 .../org/apache/kafka/clients/consumer/internals/Fetcher.java    | 1 -
 gradle/dependencies.gradle                                      | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
index a8cb4ebfd8a10..00b4319c53e13 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
@@ -70,7 +70,6 @@
 import org.apache.kafka.common.requests.MetadataRequest;
 import org.apache.kafka.common.requests.MetadataResponse;
 import org.apache.kafka.common.requests.OffsetsForLeaderEpochRequest;
-import org.apache.kafka.common.requests.RequestUtils;
 import org.apache.kafka.common.serialization.Deserializer;
 import org.apache.kafka.common.utils.CloseableIterator;
 import org.apache.kafka.common.utils.LogContext;
diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle
index 903cbd415455e..7597b9c52da06 100644
--- a/gradle/dependencies.gradle
+++ b/gradle/dependencies.gradle
@@ -114,7 +114,7 @@ versions += [
   spotbugs: "3.1.12",
   spotbugsPlugin: "1.6.9",
   spotlessPlugin: "3.23.1",
-  zookeeper: "3.5.6",
+  zookeeper: "3.5.7",
   zstd: "1.4.3-1"
 ]
 

From 350dce865ae5420a25496bc502c55de4c15bf71e Mon Sep 17 00:00:00 2001
From: Chia-Ping Tsai 
Date: Tue, 18 Feb 2020 01:58:02 +0800
Subject: [PATCH 0890/1071] KAFKA-8245: Fix Flaky Test
 DeleteConsumerGroupsTest#testDeleteCmdAllGroups (#8032)

Change unit tests to make sure the consumer group is in Stable state (i.e. consumers have completed joining the group)
---
 .../unit/kafka/admin/DeleteConsumerGroupsTest.scala   | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/core/src/test/scala/unit/kafka/admin/DeleteConsumerGroupsTest.scala b/core/src/test/scala/unit/kafka/admin/DeleteConsumerGroupsTest.scala
index 35828f0a21d9f..23de7904d5eb6 100644
--- a/core/src/test/scala/unit/kafka/admin/DeleteConsumerGroupsTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/DeleteConsumerGroupsTest.scala
@@ -107,7 +107,7 @@ class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
     val service = getConsumerGroupService(cgcArgs)
 
     TestUtils.waitUntilTrue(() => {
-      service.listGroups().contains(group)
+      service.listGroups().contains(group) && service.collectGroupState(group).state == "Stable"
     }, "The group did not initialize as expected.")
 
     executor.shutdown()
@@ -137,7 +137,8 @@ class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
     val service = getConsumerGroupService(cgcArgs)
 
     TestUtils.waitUntilTrue(() => {
-      service.listGroups().forall(groupId => groups.keySet.contains(groupId))
+      service.listGroups().toSet == groups.keySet &&
+        groups.keySet.forall(groupId => service.collectGroupState(groupId).state == "Stable")
     }, "The group did not initialize as expected.")
 
     // Shutdown consumers to empty out groups
@@ -168,7 +169,7 @@ class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
     val service = getConsumerGroupService(cgcArgs)
 
     TestUtils.waitUntilTrue(() => {
-      service.listGroups().contains(group)
+      service.listGroups().contains(group) && service.collectGroupState(group).state == "Stable"
     }, "The group did not initialize as expected.")
 
     executor.shutdown()
@@ -193,7 +194,7 @@ class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
     val service = getConsumerGroupService(cgcArgs)
 
     TestUtils.waitUntilTrue(() => {
-      service.listGroups().contains(group)
+      service.listGroups().contains(group) && service.collectGroupState(group).state == "Stable"
     }, "The group did not initialize as expected.")
 
     executor.shutdown()
@@ -220,7 +221,7 @@ class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest {
     val service = getConsumerGroupService(cgcArgs)
 
     TestUtils.waitUntilTrue(() => {
-      service.listGroups().contains(group)
+      service.listGroups().contains(group) && service.collectGroupState(group).state == "Stable"
     }, "The group did not initialize as expected.")
 
     executor.shutdown()

From 9ecd160f6a1f6b20c3f5b847290dc5752795ce9f Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Tue, 18 Feb 2020 11:29:14 -0800
Subject: [PATCH 0891/1071] KAFKA-8025: Fix flaky RocksDB test (#8126)

Reviewers: Bill Bejeck 
---
 ...OptionsColumnFamilyOptionsAdapterTest.java | 87 +++++++++----------
 1 file changed, 43 insertions(+), 44 deletions(-)

diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapterTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapterTest.java
index d07aee0807039..31e9a95d56c65 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapterTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapterTest.java
@@ -18,7 +18,6 @@
 
 import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender;
 import org.easymock.EasyMockRunner;
-import org.easymock.Mock;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.rocksdb.AbstractCompactionFilter;
@@ -55,6 +54,7 @@
 import java.util.LinkedList;
 import java.util.List;
 
+import static org.easymock.EasyMock.mock;
 import static org.easymock.EasyMock.replay;
 import static org.easymock.EasyMock.reset;
 import static org.easymock.EasyMock.verify;
@@ -85,11 +85,6 @@ public class RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapterTest {
         }
     };
 
-    @Mock
-    private DBOptions dbOptions;
-    @Mock
-    private ColumnFamilyOptions columnFamilyOptions;
-
     @Test
     public void shouldOverwriteAllOptionsMethods() throws Exception {
         for (final Method method : Options.class.getMethods()) {
@@ -112,47 +107,16 @@ public void shouldForwardAllDbOptionsCalls() throws Exception {
         }
     }
 
-    @Test
-    public void shouldWarnThanMethodCompactionOptionsFIFOSetTtlWillBeRemoved() {
-        final RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter optionsFacadeDbOptions
-            = new RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter(dbOptions, new ColumnFamilyOptions());
-
-        final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
-        optionsFacadeDbOptions.setCompactionOptionsFIFO(new CompactionOptionsFIFO());
-        LogCaptureAppender.unregister(appender);
-
-        assertThat(appender.getMessages(), hasItem(""
-            + "RocksDB's version will be bumped to version 6+ via KAFKA-8897 in a future release. "
-            + "If you use `org.rocksdb.CompactionOptionsFIFO#setTtl(long)` or `#ttl()` you will need to rewrite "
-            + "your code after KAFKA-8897 is resolved and set TTL via `org.rocksdb.Options` "
-            + "(or `org.rocksdb.ColumnFamilyOptions`)."));
-    }
-
-    @Test
-    public void shouldWarnThanMethodCompactionOptionsFIFOTtlWillBeRemoved() {
-        final RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter optionsFacadeDbOptions
-            = new RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter(dbOptions, new ColumnFamilyOptions());
-
-        final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
-        optionsFacadeDbOptions.compactionOptionsFIFO();
-        LogCaptureAppender.unregister(appender);
-
-        assertThat(appender.getMessages(), hasItem(""
-            + "RocksDB's version will be bumped to version 6+ via KAFKA-8897 in a future release. "
-            + "If you use `org.rocksdb.CompactionOptionsFIFO#setTtl(long)` or `#ttl()` you will need to rewrite "
-            + "your code after KAFKA-8897 is resolved and set TTL via `org.rocksdb.Options` "
-            + "(or `org.rocksdb.ColumnFamilyOptions`)."));
-    }
-
     private void verifyDBOptionsMethodCall(final Method method) throws Exception {
+        final DBOptions mockedDbOptions = mock(DBOptions.class);
         final RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter optionsFacadeDbOptions
-            = new RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter(dbOptions, new ColumnFamilyOptions());
+            = new RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter(mockedDbOptions, new ColumnFamilyOptions());
 
         final Object[] parameters = getDBOptionsParameters(method.getParameterTypes());
 
         try {
-            reset(dbOptions);
-            replay(dbOptions);
+            reset(mockedDbOptions);
+            replay(mockedDbOptions);
             method.invoke(optionsFacadeDbOptions, parameters);
             verify();
             fail("Should have called DBOptions." + method.getName() + "()");
@@ -231,14 +195,15 @@ public void shouldForwardAllColumnFamilyCalls() throws Exception {
     }
 
     private void verifyColumnFamilyOptionsMethodCall(final Method method) throws Exception {
+        final ColumnFamilyOptions mockedColumnFamilyOptions = mock(ColumnFamilyOptions.class);
         final RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter optionsFacadeColumnFamilyOptions
-            = new RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter(new DBOptions(), columnFamilyOptions);
+            = new RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter(new DBOptions(), mockedColumnFamilyOptions);
 
         final Object[] parameters = getColumnFamilyOptionsParameters(method.getParameterTypes());
 
         try {
-            reset(columnFamilyOptions);
-            replay(columnFamilyOptions);
+            reset(mockedColumnFamilyOptions);
+            replay(mockedColumnFamilyOptions);
             method.invoke(optionsFacadeColumnFamilyOptions, parameters);
             verify();
             fail("Should have called ColumnFamilyOptions." + method.getName() + "()");
@@ -320,4 +285,38 @@ public String name() {
 
         return parameters;
     }
+
+    @Test
+    public void shouldWarnThanMethodCompactionOptionsFIFOSetTtlWillBeRemoved() {
+        final DBOptions mockedDbOptions = mock(DBOptions.class);
+        final RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter optionsFacadeDbOptions
+            = new RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter(mockedDbOptions, new ColumnFamilyOptions());
+
+        final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
+        optionsFacadeDbOptions.setCompactionOptionsFIFO(new CompactionOptionsFIFO());
+        LogCaptureAppender.unregister(appender);
+
+        assertThat(appender.getMessages(), hasItem(""
+            + "RocksDB's version will be bumped to version 6+ via KAFKA-8897 in a future release. "
+            + "If you use `org.rocksdb.CompactionOptionsFIFO#setTtl(long)` or `#ttl()` you will need to rewrite "
+            + "your code after KAFKA-8897 is resolved and set TTL via `org.rocksdb.Options` "
+            + "(or `org.rocksdb.ColumnFamilyOptions`)."));
+    }
+
+    @Test
+    public void shouldWarnThanMethodCompactionOptionsFIFOTtlWillBeRemoved() {
+        final DBOptions mockedDbOptions = mock(DBOptions.class);
+        final RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter optionsFacadeDbOptions
+            = new RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter(mockedDbOptions, new ColumnFamilyOptions());
+
+        final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
+        optionsFacadeDbOptions.compactionOptionsFIFO();
+        LogCaptureAppender.unregister(appender);
+
+        assertThat(appender.getMessages(), hasItem(""
+            + "RocksDB's version will be bumped to version 6+ via KAFKA-8897 in a future release. "
+            + "If you use `org.rocksdb.CompactionOptionsFIFO#setTtl(long)` or `#ttl()` you will need to rewrite "
+            + "your code after KAFKA-8897 is resolved and set TTL via `org.rocksdb.Options` "
+            + "(or `org.rocksdb.ColumnFamilyOptions`)."));
+    }
 }

From 5e0d6e8db0b91974c5d5b9268613ff60959e27cc Mon Sep 17 00:00:00 2001
From: Michael Viamari 
Date: Wed, 19 Feb 2020 13:20:35 -0800
Subject: [PATCH 0892/1071] KAFKA-9533: ValueTransform forwards `null` values
 (#8108)

Fixes a bug where KStream#transformValues would forward null values from the provided ValueTransform#transform operation.

A test was added for verification KStreamTransformValuesTest#shouldEmitNoRecordIfTransformReturnsNull. A parallel test for non-key ValueTransformer was not added, as they share the same code path.

Reviewers: Bruno Cadonna , Bill Bejeck 
---
 .../internals/KStreamTransformValues.java     |  5 ++++-
 .../internals/KStreamTransformValuesTest.java | 20 +++++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValues.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValues.java
index 843606b4d79f9..06216fc9d8c71 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValues.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValues.java
@@ -53,7 +53,10 @@ public void init(final ProcessorContext context) {
 
         @Override
         public void process(final K key, final V value) {
-            context.forward(key, valueTransformer.transform(key, value));
+            final R transformedValue = valueTransformer.transform(key, value);
+            if (transformedValue != null) {
+                context.forward(key, transformedValue);
+            }
         }
 
         @Override
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValuesTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValuesTest.java
index 196f71c9e0a95..4dc68e0d63115 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValuesTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValuesTest.java
@@ -34,6 +34,7 @@
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.SingletonNoOpValueTransformer;
 import org.apache.kafka.test.StreamsTestUtils;
+import org.easymock.EasyMock;
 import org.easymock.EasyMockRunner;
 import org.easymock.Mock;
 import org.easymock.MockType;
@@ -42,6 +43,7 @@
 
 import java.util.Properties;
 
+import static org.easymock.EasyMock.mock;
 import static org.hamcrest.CoreMatchers.isA;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertArrayEquals;
@@ -138,6 +140,24 @@ public void close() { }
         assertArrayEquals(expected, supplier.theCapturedProcessor().processed.toArray());
     }
 
+    @Test
+    public void shouldEmitNoRecordIfTransformReturnsNull() {
+        final ProcessorContext context = mock(ProcessorContext.class);
+        final ValueTransformerWithKey valueTransformer = mock(ValueTransformerWithKey.class);
+        final KStreamTransformValues.KStreamTransformValuesProcessor processor =
+            new KStreamTransformValues.KStreamTransformValuesProcessor<>(valueTransformer);
+        processor.init(context);
+
+        final Integer inputKey = 1;
+        final Integer inputValue = 10;
+        EasyMock.expect(valueTransformer.transform(inputKey, inputValue)).andStubReturn(null);
+        EasyMock.replay(context);
+
+        processor.process(inputKey, inputValue);
+
+        EasyMock.verify(context);
+    }
+
     @SuppressWarnings("unchecked")
     @Test
     public void shouldInitializeTransformerWithForwardDisabledProcessorContext() {

From ce4aa9c0943436f77393609b8db986d5ffb7a8b6 Mon Sep 17 00:00:00 2001
From: David Mao <47232755+splett2@users.noreply.github.com>
Date: Thu, 20 Feb 2020 18:14:26 -0800
Subject: [PATCH 0893/1071] =?UTF-8?q?KAFKA-6266:=20Repeated=20occurrence?=
 =?UTF-8?q?=20of=20WARN=20Resetting=20first=20dirty=20offset=20=E2=80=A6?=
 =?UTF-8?q?=20(#8136)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Previously, checkpointed offsets for a log were only updated if the log was chosen for cleaning once the cleaning job completes. This caused issues in cases where logs with invalid checkpointed offsets would repeatedly emit warnings if the log with an invalid cleaning checkpoint wasn't chosen for cleaning.

Proposed fix is to update the checkpointed offset for logs with invalid checkpoints regardless of whether it gets chosen for cleaning.

Reviewers: Anna Povzner , Jun Rao 
---
 .../scala/kafka/log/LogCleanerManager.scala   | 83 +++++++++++-------
 .../kafka/log/LogCleanerManagerTest.scala     | 87 ++++++++++++++++---
 2 files changed, 125 insertions(+), 45 deletions(-)

diff --git a/core/src/main/scala/kafka/log/LogCleanerManager.scala b/core/src/main/scala/kafka/log/LogCleanerManager.scala
index 32ad708810121..19f7e4d2c87e9 100755
--- a/core/src/main/scala/kafka/log/LogCleanerManager.scala
+++ b/core/src/main/scala/kafka/log/LogCleanerManager.scala
@@ -97,32 +97,32 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
   }
 
   /* gauges for tracking the number of uncleanable bytes from uncleanable partitions for each log directory */
-    for (dir <- logDirs) {
-      newGauge(
-        "uncleanable-bytes",
-        new Gauge[Long] {
-          def value = {
-            inLock(lock) {
-              uncleanablePartitions.get(dir.getAbsolutePath) match {
-                case Some(partitions) => {
-                  val lastClean = allCleanerCheckpoints
-                  val now = Time.SYSTEM.milliseconds
-                  partitions.map { tp =>
-                    val log = logs.get(tp)
-                    val lastCleanOffset = lastClean.get(tp)
-                    val (firstDirtyOffset, firstUncleanableDirtyOffset) = cleanableOffsets(log, lastCleanOffset, now)
-                    val (_, uncleanableBytes) = calculateCleanableBytes(log, firstDirtyOffset, firstUncleanableDirtyOffset)
-                    uncleanableBytes
-                  }.sum
-                }
-                case _ => 0
+  for (dir <- logDirs) {
+    newGauge(
+      "uncleanable-bytes",
+      new Gauge[Long] {
+        def value = {
+          inLock(lock) {
+            uncleanablePartitions.get(dir.getAbsolutePath) match {
+              case Some(partitions) => {
+                val lastClean = allCleanerCheckpoints
+                val now = Time.SYSTEM.milliseconds
+                partitions.map { tp =>
+                  val log = logs.get(tp)
+                  val lastCleanOffset = lastClean.get(tp)
+                  val offsetsToClean = cleanableOffsets(log, lastCleanOffset, now)
+                  val (_, uncleanableBytes) = calculateCleanableBytes(log, offsetsToClean.firstDirtyOffset, offsetsToClean.firstUncleanableDirtyOffset)
+                  uncleanableBytes
+                }.sum
               }
+              case _ => 0
             }
           }
-        },
-        Map("logDirectory" -> dir.getAbsolutePath)
-      )
-    }
+        }
+      },
+      Map("logDirectory" -> dir.getAbsolutePath)
+    )
+  }
 
   /* a gauge for tracking the cleanable ratio of the dirtiest log */
   @volatile private var dirtiestLogCleanableRatio = 0.0
@@ -187,11 +187,14 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
         case (topicPartition, log) => // create a LogToClean instance for each
           try {
             val lastCleanOffset = lastClean.get(topicPartition)
-            val (firstDirtyOffset, firstUncleanableDirtyOffset) = cleanableOffsets(log, lastCleanOffset, now)
-            val compactionDelayMs = maxCompactionDelay(log, firstDirtyOffset, now)
+            val offsetsToClean = cleanableOffsets(log, lastCleanOffset, now)
+            // update checkpoint for logs with invalid checkpointed offsets
+            if (offsetsToClean.forceUpdateCheckpoint)
+              updateCheckpoints(log.dir.getParentFile(), Option(topicPartition, offsetsToClean.firstDirtyOffset))
+            val compactionDelayMs = maxCompactionDelay(log, offsetsToClean.firstDirtyOffset, now)
             preCleanStats.updateMaxCompactionDelay(compactionDelayMs)
 
-            LogToClean(topicPartition, log, firstDirtyOffset, firstUncleanableDirtyOffset, compactionDelayMs > 0)
+            LogToClean(topicPartition, log, offsetsToClean.firstDirtyOffset, offsetsToClean.firstUncleanableDirtyOffset, compactionDelayMs > 0)
           } catch {
             case e: Throwable => throw new LogCleaningException(log,
               s"Failed to calculate log cleaning stats for partition $topicPartition", e)
@@ -488,6 +491,20 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
   }
 }
 
+/**
+ * Helper class for the range of cleanable dirty offsets of a log and whether to update the checkpoint associated with
+ * the log
+ *
+ * @param firstDirtyOffset the lower (inclusive) offset to begin cleaning from
+ * @param firstUncleanableDirtyOffset the upper(exclusive) offset to clean to
+ * @param forceUpdateCheckpoint whether to update the checkpoint associated with this log. if true, checkpoint should be
+ *                             reset to firstDirtyOffset
+ */
+private case class OffsetsToClean(firstDirtyOffset: Long,
+                                  firstUncleanableDirtyOffset: Long,
+                                  forceUpdateCheckpoint: Boolean = false) {
+}
+
 private[log] object LogCleanerManager extends Logging {
 
   def isCompactAndDelete(log: Log): Boolean = {
@@ -523,12 +540,12 @@ private[log] object LogCleanerManager extends Logging {
     * @param log the log
     * @param lastCleanOffset the last checkpointed offset
     * @param now the current time in milliseconds of the cleaning operation
-    * @return the lower (inclusive) and upper (exclusive) offsets
+    * @return OffsetsToClean containing offsets for cleanable portion of log and whether the log checkpoint needs updating
     */
-  def cleanableOffsets(log: Log, lastCleanOffset: Option[Long], now: Long): (Long, Long) = {
+  def cleanableOffsets(log: Log, lastCleanOffset: Option[Long], now: Long): OffsetsToClean = {
     // If the log segments are abnormally truncated and hence the checkpointed offset is no longer valid;
     // reset to the log starting offset and log the error
-    val firstDirtyOffset = {
+    val (firstDirtyOffset, forceUpdateCheckpoint) = {
       val logStartOffset = log.logStartOffset
       val checkpointDirtyOffset = lastCleanOffset.getOrElse(logStartOffset)
 
@@ -537,15 +554,15 @@ private[log] object LogCleanerManager extends Logging {
         if (!isCompactAndDelete(log))
           warn(s"Resetting first dirty offset of ${log.name} to log start offset $logStartOffset " +
             s"since the checkpointed offset $checkpointDirtyOffset is invalid.")
-        logStartOffset
+        (logStartOffset, true)
       } else if (checkpointDirtyOffset > log.logEndOffset) {
         // The dirty offset has gotten ahead of the log end offset. This could happen if there was data
         // corruption at the end of the log. We conservatively assume that the full log needs cleaning.
         warn(s"The last checkpoint dirty offset for partition ${log.name} is $checkpointDirtyOffset, " +
           s"which is larger than the log end offset ${log.logEndOffset}. Resetting to the log start offset $logStartOffset.")
-        logStartOffset
+        (logStartOffset, true)
       } else {
-        checkpointDirtyOffset
+        (checkpointDirtyOffset, false)
       }
     }
 
@@ -580,7 +597,7 @@ private[log] object LogCleanerManager extends Logging {
       s"now=$now => firstDirtyOffset=$firstDirtyOffset firstUncleanableOffset=$firstUncleanableDirtyOffset " +
       s"activeSegment.baseOffset=${log.activeSegment.baseOffset}")
 
-    (firstDirtyOffset, math.max(firstDirtyOffset, firstUncleanableDirtyOffset))
+    OffsetsToClean(firstDirtyOffset, math.max(firstDirtyOffset, firstUncleanableDirtyOffset), forceUpdateCheckpoint)
   }
 
   /**
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
index bdf5cb74c66e1..5e2620ca0b7cf 100644
--- a/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
@@ -54,6 +54,11 @@ class LogCleanerManagerTest extends Logging {
     override def allCleanerCheckpoints: Map[TopicPartition, Long] = {
       cleanerCheckpoints.toMap
     }
+
+    override def updateCheckpoints(dataDir: File, update: Option[(TopicPartition,Long)]): Unit = {
+      val (tp, offset) = update.getOrElse(throw new IllegalArgumentException("update=None argument not yet handled"))
+      cleanerCheckpoints.put(tp, offset)
+    }
   }
 
   @After
@@ -423,8 +428,8 @@ class LogCleanerManagerTest extends Logging {
 
     val lastCleanOffset = Some(0L)
     val cleanableOffsets = LogCleanerManager.cleanableOffsets(log, lastCleanOffset, time.milliseconds)
-    assertEquals("The first cleanable offset starts at the beginning of the log.", 0L, cleanableOffsets._1)
-    assertEquals("The first uncleanable offset begins with the active segment.", log.activeSegment.baseOffset, cleanableOffsets._2)
+    assertEquals("The first cleanable offset starts at the beginning of the log.", 0L, cleanableOffsets.firstDirtyOffset)
+    assertEquals("The first uncleanable offset begins with the active segment.", log.activeSegment.baseOffset, cleanableOffsets.firstUncleanableDirtyOffset)
   }
 
   /**
@@ -453,8 +458,8 @@ class LogCleanerManagerTest extends Logging {
 
     val lastCleanOffset = Some(0L)
     val cleanableOffsets = LogCleanerManager.cleanableOffsets(log, lastCleanOffset, time.milliseconds)
-    assertEquals("The first cleanable offset starts at the beginning of the log.", 0L, cleanableOffsets._1)
-    assertEquals("The first uncleanable offset begins with the second block of log entries.", activeSegAtT0.baseOffset, cleanableOffsets._2)
+    assertEquals("The first cleanable offset starts at the beginning of the log.", 0L, cleanableOffsets.firstDirtyOffset)
+    assertEquals("The first uncleanable offset begins with the second block of log entries.", activeSegAtT0.baseOffset, cleanableOffsets.firstUncleanableDirtyOffset)
   }
 
   /**
@@ -478,8 +483,27 @@ class LogCleanerManagerTest extends Logging {
 
     val lastCleanOffset = Some(0L)
     val cleanableOffsets = LogCleanerManager.cleanableOffsets(log, lastCleanOffset, time.milliseconds)
-    assertEquals("The first cleanable offset starts at the beginning of the log.", 0L, cleanableOffsets._1)
-    assertEquals("The first uncleanable offset begins with active segment.", log.activeSegment.baseOffset, cleanableOffsets._2)
+    assertEquals("The first cleanable offset starts at the beginning of the log.", 0L, cleanableOffsets.firstDirtyOffset)
+    assertEquals("The first uncleanable offset begins with active segment.", log.activeSegment.baseOffset, cleanableOffsets.firstUncleanableDirtyOffset)
+  }
+
+  @Test
+  def testCleanableOffsetsNeedsCheckpointReset(): Unit = {
+    val tp = new TopicPartition("foo", 0)
+    val logs = setupIncreasinglyFilthyLogs(Seq(tp), startNumBatches = 20, batchIncrement = 5)
+    logs.get(tp).maybeIncrementLogStartOffset(10L)
+
+    var lastCleanOffset = Some(15L)
+    var cleanableOffsets = LogCleanerManager.cleanableOffsets(logs.get(tp), lastCleanOffset, time.milliseconds)
+    assertFalse("Checkpoint offset should not be reset if valid", cleanableOffsets.forceUpdateCheckpoint)
+
+    logs.get(tp).maybeIncrementLogStartOffset(20L)
+    cleanableOffsets = LogCleanerManager.cleanableOffsets(logs.get(tp), lastCleanOffset, time.milliseconds)
+    assertTrue("Checkpoint offset needs to be reset if less than log start offset", cleanableOffsets.forceUpdateCheckpoint)
+
+    lastCleanOffset = Some(25L)
+    cleanableOffsets = LogCleanerManager.cleanableOffsets(logs.get(tp), lastCleanOffset, time.milliseconds)
+    assertTrue("Checkpoint offset needs to be reset if greater than log end offset", cleanableOffsets.forceUpdateCheckpoint)
   }
 
   @Test
@@ -505,8 +529,8 @@ class LogCleanerManagerTest extends Logging {
     time.sleep(compactionLag + 1)
     // although the compaction lag has been exceeded, the undecided data should not be cleaned
     var cleanableOffsets = LogCleanerManager.cleanableOffsets(log, Some(0L), time.milliseconds())
-    assertEquals(0L, cleanableOffsets._1)
-    assertEquals(0L, cleanableOffsets._2)
+    assertEquals(0L, cleanableOffsets.firstDirtyOffset)
+    assertEquals(0L, cleanableOffsets.firstUncleanableDirtyOffset)
 
     log.appendAsLeader(MemoryRecords.withEndTransactionMarker(time.milliseconds(), producerId, producerEpoch,
       new EndTransactionMarker(ControlRecordType.ABORT, 15)), leaderEpoch = 0,
@@ -516,15 +540,15 @@ class LogCleanerManagerTest extends Logging {
 
     // the first segment should now become cleanable immediately
     cleanableOffsets = LogCleanerManager.cleanableOffsets(log, Some(0L), time.milliseconds())
-    assertEquals(0L, cleanableOffsets._1)
-    assertEquals(3L, cleanableOffsets._2)
+    assertEquals(0L, cleanableOffsets.firstDirtyOffset)
+    assertEquals(3L, cleanableOffsets.firstUncleanableDirtyOffset)
 
     time.sleep(compactionLag + 1)
 
     // the second segment becomes cleanable after the compaction lag
     cleanableOffsets = LogCleanerManager.cleanableOffsets(log, Some(0L), time.milliseconds())
-    assertEquals(0L, cleanableOffsets._1)
-    assertEquals(4L, cleanableOffsets._2)
+    assertEquals(0L, cleanableOffsets.firstDirtyOffset)
+    assertEquals(4L, cleanableOffsets.firstUncleanableDirtyOffset)
   }
 
   @Test
@@ -574,6 +598,45 @@ class LogCleanerManagerTest extends Logging {
     assertEquals(LogCleaningPaused(1), cleanerManager.cleaningState(tp).get)
   }
 
+  /**
+   * Logs with invalid checkpoint offsets should update their checkpoint offset even if the log doesn't need cleaning
+   */
+  @Test
+  def testCheckpointUpdatedForInvalidOffsetNoCleaning(): Unit = {
+    val tp = new TopicPartition("foo", 0)
+    val logs = setupIncreasinglyFilthyLogs(Seq(tp), startNumBatches = 20, batchIncrement = 5)
+
+    logs.get(tp).maybeIncrementLogStartOffset(20L)
+    val cleanerManager = createCleanerManagerMock(logs)
+    cleanerCheckpoints.put(tp, 15L)
+
+    val filthiestLog = cleanerManager.grabFilthiestCompactedLog(time)
+    assertEquals("Log should not be selected for cleaning", None, filthiestLog)
+    assertEquals("Unselected log should have checkpoint offset updated", 20L, cleanerCheckpoints.get(tp).get)
+  }
+
+  /**
+   * Logs with invalid checkpoint offsets should update their checkpoint offset even if they aren't selected
+   * for immediate cleaning
+   */
+  @Test
+  def testCheckpointUpdatedForInvalidOffsetNotSelected(): Unit = {
+    val tp0 = new TopicPartition("foo", 0)
+    val tp1 = new TopicPartition("foo", 1)
+    val partitions = Seq(tp0, tp1)
+
+    // create two logs, one with an invalid offset, and one that is dirtier than the log with an invalid offset
+    val logs = setupIncreasinglyFilthyLogs(partitions, startNumBatches = 20, batchIncrement = 5)
+    logs.get(tp0).maybeIncrementLogStartOffset(15L)
+    val cleanerManager = createCleanerManagerMock(logs)
+    cleanerCheckpoints.put(tp0, 10L)
+    cleanerCheckpoints.put(tp1, 5L)
+
+    val filthiestLog = cleanerManager.grabFilthiestCompactedLog(time).get
+    assertEquals("Dirtier log should be selected", tp1, filthiestLog.topicPartition)
+    assertEquals("Unselected log should have checkpoint offset updated", 15L, cleanerCheckpoints.get(tp0).get)
+  }
+
   private def createCleanerManager(log: Log): LogCleanerManager = {
     val logs = new Pool[TopicPartition, Log]()
     logs.put(topicPartition, log)

From 44fbd96de890a33cb51bb9d051ff396d070c6e8c Mon Sep 17 00:00:00 2001
From: Chia-Ping Tsai 
Date: Tue, 25 Feb 2020 03:32:15 +0800
Subject: [PATCH 0894/1071] KAFKA-9599 create unique sensor to record group
 rebalance (#8159)

The "offset deletion" and "group rebalance" should not be recorded by the same sensor since they are totally different.

The code is introduced by #7276.

Reviewers: David Jacot , Guozhang Wang 
---
 .../main/scala/kafka/coordinator/group/GroupCoordinator.scala   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
index 2e2358db1083b..7b69632c99b64 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
@@ -75,7 +75,7 @@ class GroupCoordinator(val brokerId: Int,
       "group-coordinator-metrics",
       "The total number of administrative deleted offsets")))
 
-  val groupCompletedRebalanceSensor = metrics.sensor("OffsetDeletions")
+  val groupCompletedRebalanceSensor = metrics.sensor("CompletedRebalances")
 
   groupCompletedRebalanceSensor.add(new Meter(
     metrics.metricName("group-completed-rebalance-rate",

From b1a132146f029810930f54f0dbb4a3a5d099b87d Mon Sep 17 00:00:00 2001
From: bill 
Date: Tue, 25 Feb 2020 12:41:24 -0500
Subject: [PATCH 0895/1071] Revert "KAFKA-9533: ValueTransform forwards `null`
 values (#8108)"

This reverts commit a41d3d86c13a55a75e67b1635bc74361ebe6d7af.
---
 .../internals/KStreamTransformValues.java     |  5 +----
 .../internals/KStreamTransformValuesTest.java | 20 -------------------
 2 files changed, 1 insertion(+), 24 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValues.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValues.java
index 06216fc9d8c71..843606b4d79f9 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValues.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValues.java
@@ -53,10 +53,7 @@ public void init(final ProcessorContext context) {
 
         @Override
         public void process(final K key, final V value) {
-            final R transformedValue = valueTransformer.transform(key, value);
-            if (transformedValue != null) {
-                context.forward(key, transformedValue);
-            }
+            context.forward(key, valueTransformer.transform(key, value));
         }
 
         @Override
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValuesTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValuesTest.java
index 4dc68e0d63115..196f71c9e0a95 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValuesTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValuesTest.java
@@ -34,7 +34,6 @@
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.SingletonNoOpValueTransformer;
 import org.apache.kafka.test.StreamsTestUtils;
-import org.easymock.EasyMock;
 import org.easymock.EasyMockRunner;
 import org.easymock.Mock;
 import org.easymock.MockType;
@@ -43,7 +42,6 @@
 
 import java.util.Properties;
 
-import static org.easymock.EasyMock.mock;
 import static org.hamcrest.CoreMatchers.isA;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertArrayEquals;
@@ -140,24 +138,6 @@ public void close() { }
         assertArrayEquals(expected, supplier.theCapturedProcessor().processed.toArray());
     }
 
-    @Test
-    public void shouldEmitNoRecordIfTransformReturnsNull() {
-        final ProcessorContext context = mock(ProcessorContext.class);
-        final ValueTransformerWithKey valueTransformer = mock(ValueTransformerWithKey.class);
-        final KStreamTransformValues.KStreamTransformValuesProcessor processor =
-            new KStreamTransformValues.KStreamTransformValuesProcessor<>(valueTransformer);
-        processor.init(context);
-
-        final Integer inputKey = 1;
-        final Integer inputValue = 10;
-        EasyMock.expect(valueTransformer.transform(inputKey, inputValue)).andStubReturn(null);
-        EasyMock.replay(context);
-
-        processor.process(inputKey, inputValue);
-
-        EasyMock.verify(context);
-    }
-
     @SuppressWarnings("unchecked")
     @Test
     public void shouldInitializeTransformerWithForwardDisabledProcessorContext() {

From 61e3e65c063e1024db3f0da68b324f059838a63b Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Wed, 26 Feb 2020 11:10:00 -0600
Subject: [PATCH 0896/1071] MINOR: Fix gradle error writing test stdout (#8133)

Adds a couple of extra checks to the test-output-capturing logic in our gradle build.
Previously, we were seeing a lot of error logs while attempting to write output for a
test whose output file hadn't been initialized.

Reviewers: Ewen Cheslack-Postava 
---
 build.gradle | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index 121d1eb2f9d99..6edbd02fe3ba9 100644
--- a/build.gradle
+++ b/build.gradle
@@ -246,7 +246,13 @@ subprojects {
       // code in the test (best guess is that it is tail output from last test). We won't have
       // an output file for these, so simply ignore them. If they become critical for debugging,
       // they can be seen with showStandardStreams.
-      if (td.name == td.className) {
+      if (td.name == td.className || td.className == null) {
+        // silently ignore output unrelated to specific test methods
+        return
+      } else if (logStreams.get(tid) == null) {
+        println "WARNING: unexpectedly got output for a test [${tid}]" +
+                " that we didn't previously see in the beforeTest hook." +
+                " Message for debugging: [" + toe.message + "]."
         return
       }
       try {

From 0eb45896bf7e563a68a91623a7bd6c15dfe0773d Mon Sep 17 00:00:00 2001
From: Chris Egerton 
Date: Wed, 26 Feb 2020 13:52:16 -0800
Subject: [PATCH 0897/1071] KAFKA-9601: Stop logging raw connector config
 values (#8165)

Author: Chris Egerton 
Reviewer: Randall Hauch 
---
 .../java/org/apache/kafka/connect/runtime/WorkerConnector.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java
index 8ec93cfe370be..7923943d121f7 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java
@@ -75,7 +75,7 @@ public WorkerConnector(String connName,
     public void initialize(ConnectorConfig connectorConfig) {
         try {
             this.config = connectorConfig.originalsStrings();
-            log.debug("{} Initializing connector {} with config {}", this, connName, config);
+            log.debug("{} Initializing connector {}", this, connName);
             if (isSinkConnector()) {
                 SinkConnectorConfig.validate(config);
             }

From e60eb69afa6ebc4ff1f9c51ab364f8774f2c4ca0 Mon Sep 17 00:00:00 2001
From: Matthew Wong 
Date: Thu, 27 Feb 2020 14:46:55 -0800
Subject: [PATCH 0898/1071] throttle consumer timeout increase (#8188)

The test_throttled_reassignment test fails because the consumer that is used to validate reassignment does not start on time to consume all messages. This does not seem like an issue with the throttling of the reassignment, since increasing the timeout allowed the test to pass multiple consecutive runs locally.

This test seemed to rely on the default JmxTool for the console consumer that was removed in this commit: 179d0d7
The console consumer would check to see if it had partitions assigned to it before beginning to consume. Although the test occasionally failed with the JmxTool, it began to fail much more after the removal.

Error messages of failures followed the below format with varying numbers of missed messages. They are the first messages by the producer.

535 acked message did not make it to the Consumer. They are: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19...plus 515 more. Total Acked: 192792, Total Consumed: 192259. We validated that the first 535 of these missing messages correctly made it into Kafka's data files. This suggests they were lost on their way to the consumer.
In the scope of the test, this error suggests that the test is falling into the race condition described in produce_consume_validate.py, which has the timeout to prevent the consumer from missing initial messages.

This can serve as a temporary fix until the logic of consumer startup is addressed further.

Reviewers: Jason Gustafson , Bill Bejeck 
---
 tests/kafkatest/tests/core/throttling_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/kafkatest/tests/core/throttling_test.py b/tests/kafkatest/tests/core/throttling_test.py
index 586bac9c634c3..4a8327ebb0a12 100644
--- a/tests/kafkatest/tests/core/throttling_test.py
+++ b/tests/kafkatest/tests/core/throttling_test.py
@@ -53,7 +53,7 @@ def __init__(self, test_context):
         # ensure that the consumer is fully started before the producer starts
         # so that we don't miss any messages. This timeout ensures the sufficient
         # condition.
-        self.consumer_init_timeout_sec =  20
+        self.consumer_init_timeout_sec =  60
         self.num_brokers = 6
         self.num_partitions = 3
         self.kafka = KafkaService(test_context,

From a696296fe3201291dfb34e5af9d96f2d6a104366 Mon Sep 17 00:00:00 2001
From: Bill Bejeck 
Date: Thu, 27 Feb 2020 18:01:44 -0500
Subject: [PATCH 0899/1071] MINOR: Update upgrade guide for ZK (#8182)

Adding the ZK upgrade information to the Notable Changes section for 2.4.1

Reviewers: Ron Dagostino , Jim Galasyn 
---
 docs/upgrade.html | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/docs/upgrade.html b/docs/upgrade.html
index 43947843b4ce9..6b3775cd3d5d0 100644
--- a/docs/upgrade.html
+++ b/docs/upgrade.html
@@ -82,6 +82,17 @@ 
          Upgrading from 0.8.x, 0.9.x, 0.1
     
 
 
+
          Notable changes in 2.4.1
          
+
            
+    
          • 
+        ZooKeeper has been upgraded to 3.5.7, and a ZooKeeper upgrade from 3.4.X to 3.5.7 can fail if there are no snapshot files in the 3.4 data directory.
+        This usually happens in test upgrades where ZooKeeper 3.5.7 is trying to load an existing 3.4 data dir in which no snapshot file has been created.
+        For more information, see ZOOKEEPER-3056.
+        A fix is given in ZOOKEEPER-3056, which is to set the snapshot.trust.empty=true
+        config in zookeeper.properties before the upgrade.
+    
            • 
+
          
+
 
          Notable changes in 2.4.0
          
 
            
     
          • A new Admin API has been added for partition reassignments. Due to changing the way Kafka propagates reassignment information,

From bd4feef3dd66e4908d901be772344cb64c333087 Mon Sep 17 00:00:00 2001
From: Brian Bushree 
Date: Sat, 29 Feb 2020 16:43:51 -0800
Subject: [PATCH 0900/1071] MINOR: add wait_for_assigned_partitions to
 console-consumer (#8192)

what/why
the throttling_test was broken by this PR (#7785) since it depends on the consumer having partitions-assigned before starting the producer

this PR provides the ability to wait for partitions to be assigned in the console consumer before considering it started.

caveat
this does not support starting up the JmxTool inside the console-consumer for custom metrics while using this wait_until_partitions_assigned flag since the code assumes one JmxTool running per node.

I think a proper fix for this would be to make JmxTool its own standalone single-node service

alternatives
we could use the EndToEnd test suite which uses the verifiable producer/consumer under the hood but I found that there were more changes necessary to get this working unfortunately (specifically doesn't seem like this test suite plays nicely with the ProducerPerformanceService)

Reviewers: Mathew Wong , Bill Bejeck 
---
 tests/kafkatest/services/console_consumer.py  | 28 +++++++++++++++++--
 tests/kafkatest/services/monitor/jmx.py       | 19 +++++++++++--
 .../tests/core/fetch_from_follower_test.py    | 16 +----------
 tests/kafkatest/tests/core/throttling_test.py |  3 +-
 4 files changed, 46 insertions(+), 20 deletions(-)

diff --git a/tests/kafkatest/services/console_consumer.py b/tests/kafkatest/services/console_consumer.py
index 5fd471217ec8c..e3d191059fc56 100644
--- a/tests/kafkatest/services/console_consumer.py
+++ b/tests/kafkatest/services/console_consumer.py
@@ -17,9 +17,10 @@
 import os
 
 from ducktape.services.background_thread import BackgroundThreadService
+from ducktape.utils.util import wait_until
 
 from kafkatest.directory_layout.kafka_path import KafkaPathResolverMixin
-from kafkatest.services.monitor.jmx import JmxMixin
+from kafkatest.services.monitor.jmx import JmxMixin, JmxTool
 from kafkatest.version import DEV_BRANCH, LATEST_0_8_2, LATEST_0_9, LATEST_0_10_0, V_0_9_0_0, V_0_10_0_0, V_0_11_0_0, V_2_0_0
 
 """
@@ -62,7 +63,8 @@ def __init__(self, context, num_nodes, kafka, topic, group_id="test-consumer-gro
                  client_id="console-consumer", print_key=False, jmx_object_names=None, jmx_attributes=None,
                  enable_systest_events=False, stop_timeout_sec=35, print_timestamp=False,
                  isolation_level="read_uncommitted", jaas_override_variables=None,
-                 kafka_opts_override="", client_prop_file_override="", consumer_properties={}):
+                 kafka_opts_override="", client_prop_file_override="", consumer_properties={},
+                 wait_until_partitions_assigned=False):
         """
         Args:
             context:                    standard context
@@ -122,6 +124,7 @@ def __init__(self, context, num_nodes, kafka, topic, group_id="test-consumer-gro
         self.kafka_opts_override = kafka_opts_override
         self.client_prop_file_override = client_prop_file_override
         self.consumer_properties = consumer_properties
+        self.wait_until_partitions_assigned = wait_until_partitions_assigned
 
 
     def prop_file(self, node):
@@ -268,8 +271,29 @@ def _worker(self, idx, node):
         with self.lock:
             self.read_jmx_output(idx, node)
 
+    def _wait_until_partitions_assigned(self, node, timeout_sec=60):
+        if self.jmx_object_names is not None:
+            raise Exception("'wait_until_partitions_assigned' is not supported while using 'jmx_object_names'/'jmx_attributes'")
+        jmx_tool = JmxTool(self.context, jmx_poll_ms=100)
+        jmx_tool.jmx_object_names = ["kafka.consumer:type=consumer-coordinator-metrics,client-id=%s" % self.client_id]
+        jmx_tool.jmx_attributes = ["assigned-partitions"]
+        jmx_tool.assigned_partitions_jmx_attr = "kafka.consumer:type=consumer-coordinator-metrics,client-id=%s:assigned-partitions" % self.client_id
+        jmx_tool.start_jmx_tool(self.idx(node), node)
+        assigned_partitions_jmx_attr = "kafka.consumer:type=consumer-coordinator-metrics,client-id=%s:assigned-partitions" % self.client_id
+
+        def read_and_check():
+            jmx_tool.read_jmx_output(self.idx(node), node)
+            return assigned_partitions_jmx_attr in jmx_tool.maximum_jmx_value
+
+        wait_until(lambda: read_and_check(),
+                timeout_sec=timeout_sec,
+                backoff_sec=.5,
+                err_msg="consumer was not assigned partitions within %d seconds" % timeout_sec)
+
     def start_node(self, node):
         BackgroundThreadService.start_node(self, node)
+        if self.wait_until_partitions_assigned:
+            self._wait_until_partitions_assigned(node)
 
     def stop_node(self, node):
         self.logger.info("%s Stopping node %s" % (self.__class__.__name__, str(node.account)))
diff --git a/tests/kafkatest/services/monitor/jmx.py b/tests/kafkatest/services/monitor/jmx.py
index c5b747da08244..2dcd369512dac 100644
--- a/tests/kafkatest/services/monitor/jmx.py
+++ b/tests/kafkatest/services/monitor/jmx.py
@@ -17,6 +17,8 @@
 
 from ducktape.cluster.remoteaccount import RemoteCommandError
 from ducktape.utils.util import wait_until
+
+from kafkatest.directory_layout.kafka_path import KafkaPathResolverMixin
 from kafkatest.version import get_version, V_0_11_0_0, DEV_BRANCH
 
 class JmxMixin(object):
@@ -41,10 +43,11 @@ def __init__(self, num_nodes, jmx_object_names=None, jmx_attributes=None, jmx_po
         self.jmx_tool_log = os.path.join(root, "jmx_tool.log")
         self.jmx_tool_err_log = os.path.join(root, "jmx_tool.err.log")
 
-    def clean_node(self, node):
+    def clean_node(self, node, idx=None):
         node.account.kill_java_processes(self.jmx_class_name(), clean_shutdown=False,
                                          allow_fail=True)
-        idx = self.idx(node)
+        if idx is None:
+            idx = self.idx(node)
         self.started[idx-1] = False
         node.account.ssh("rm -f -- %s %s" % (self.jmx_tool_log, self.jmx_tool_err_log), allow_fail=False)
 
@@ -139,3 +142,15 @@ def read_jmx_output_all_nodes(self):
 
     def jmx_class_name(self):
         return "kafka.tools.JmxTool"
+
+class JmxTool(JmxMixin, KafkaPathResolverMixin):
+    """
+    Simple helper class for using the JmxTool directly instead of as a mix-in
+    """
+    def __init__(self, text_context, *args, **kwargs):
+        JmxMixin.__init__(self, num_nodes=1, *args, **kwargs)
+        self.context = text_context
+
+    @property
+    def logger(self):
+        return self.context.logger
diff --git a/tests/kafkatest/tests/core/fetch_from_follower_test.py b/tests/kafkatest/tests/core/fetch_from_follower_test.py
index fde1bafe07c41..ef3772880c82a 100644
--- a/tests/kafkatest/tests/core/fetch_from_follower_test.py
+++ b/tests/kafkatest/tests/core/fetch_from_follower_test.py
@@ -18,29 +18,15 @@
 
 from ducktape.mark.resource import cluster
 
-from kafkatest.directory_layout.kafka_path import KafkaPathResolverMixin
 from kafkatest.services.console_consumer import ConsoleConsumer
 from kafkatest.services.kafka import KafkaService
-from kafkatest.services.monitor.jmx import JmxMixin
+from kafkatest.services.monitor.jmx import JmxTool
 from kafkatest.services.verifiable_producer import VerifiableProducer
 from kafkatest.services.zookeeper import ZookeeperService
 from kafkatest.tests.produce_consume_validate import ProduceConsumeValidateTest
 from kafkatest.utils import is_int
 
 
-class JmxTool(JmxMixin, KafkaPathResolverMixin):
-    """
-    Simple helper class for using the JmxTool directly instead of as a mix-in
-    """
-    def __init__(self, text_context, *args, **kwargs):
-        JmxMixin.__init__(self, num_nodes=1, *args, **kwargs)
-        self.context = text_context
-
-    @property
-    def logger(self):
-        return self.context.logger
-
-
 class FetchFromFollowerTest(ProduceConsumeValidateTest):
 
     RACK_AWARE_REPLICA_SELECTOR = "org.apache.kafka.common.replica.RackAwareReplicaSelector"
diff --git a/tests/kafkatest/tests/core/throttling_test.py b/tests/kafkatest/tests/core/throttling_test.py
index 4a8327ebb0a12..f29ec2b63cb42 100644
--- a/tests/kafkatest/tests/core/throttling_test.py
+++ b/tests/kafkatest/tests/core/throttling_test.py
@@ -165,7 +165,8 @@ def test_throttled_reassignment(self, bounce_brokers):
                                         self.topic,
                                         consumer_timeout_ms=60000,
                                         message_validator=is_int,
-                                        from_beginning=False)
+                                        from_beginning=False,
+                                        wait_until_partitions_assigned=True)
 
         self.kafka.start()
         bulk_producer.run()

From dfdb6bf0248fa76f5365ad0ffd04b9919c45cf8e Mon Sep 17 00:00:00 2001
From: Bill Bejeck 
Date: Mon, 2 Mar 2020 15:53:23 -0500
Subject: [PATCH 0901/1071] MINOR: Update year in NOTICE (#8207)

Reviewers: Ismael Juma 
---
 NOTICE | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/NOTICE b/NOTICE
index bd7b0dcf8a6e8..f5505f39ad215 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,5 +1,5 @@
 Apache Kafka
-Copyright 2019 The Apache Software Foundation.
+Copyright 2020 The Apache Software Foundation.
 
 This product includes software developed at
 The Apache Software Foundation (https://www.apache.org/).

From c57222ae8cd7866b7191722656f3065afaee5b7a Mon Sep 17 00:00:00 2001
From: bill 
Date: Mon, 2 Mar 2020 19:32:42 -0500
Subject: [PATCH 0902/1071] Bump version to 2.4.1

---
 gradle.properties                                               | 2 +-
 streams/quickstart/java/pom.xml                                 | 2 +-
 .../java/src/main/resources/archetype-resources/pom.xml         | 2 +-
 streams/quickstart/pom.xml                                      | 2 +-
 tests/kafkatest/__init__.py                                     | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/gradle.properties b/gradle.properties
index afc7e5628bf13..bc2ddfecbc9c8 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -20,7 +20,7 @@ group=org.apache.kafka
 #  - tests/kafkatest/__init__.py
 #  - tests/kafkatest/version.py (variable DEV_VERSION)
 #  - kafka-merge-pr.py
-version=2.4.1-SNAPSHOT
+version=2.4.1
 scalaVersion=2.12.10
 task=build
 org.gradle.jvmargs=-Xmx1024m -Xss2m
diff --git a/streams/quickstart/java/pom.xml b/streams/quickstart/java/pom.xml
index 9fd10303e0f2b..cc974f9fb79e4 100644
--- a/streams/quickstart/java/pom.xml
+++ b/streams/quickstart/java/pom.xml
@@ -26,7 +26,7 @@
     
         org.apache.kafka
         streams-quickstart
-        2.4.1-SNAPSHOT
+        2.4.1
         ..
     
 
diff --git a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
index 017c30a1fabcb..8d75a280b1397 100644
--- a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
+++ b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
@@ -29,7 +29,7 @@
 
     
         UTF-8
-        2.4.1-SNAPSHOT
+        2.4.1
         1.7.7
         1.2.17
     
diff --git a/streams/quickstart/pom.xml b/streams/quickstart/pom.xml
index acf435f10cde5..581eea5dbeda4 100644
--- a/streams/quickstart/pom.xml
+++ b/streams/quickstart/pom.xml
@@ -22,7 +22,7 @@
     org.apache.kafka
     streams-quickstart
     pom
-    2.4.1-SNAPSHOT
+    2.4.1
 
     Kafka Streams :: Quickstart
 
diff --git a/tests/kafkatest/__init__.py b/tests/kafkatest/__init__.py
index 16acde5bd9750..c2453f0df4ab7 100644
--- a/tests/kafkatest/__init__.py
+++ b/tests/kafkatest/__init__.py
@@ -22,4 +22,4 @@
 # Instead, in
 #
 # For example, when Kafka is at version 1.0.0-SNAPSHOT, this should be something like "1.0.0
-__version__ = '2.4.1.dev0'
+__version__ = '2.4.1'

From 7f740a2ade362c236ab9359ac28241c7c4b4cd38 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Mon, 2 Mar 2020 17:24:19 -0800
Subject: [PATCH 0903/1071] KAFKA-8995: delete all topics before recreating
 (#8208)

I think the root cause of KAFKA-8893, KAFKA-8894, KAFKA-8895 and KSTREAMS-3779 are the same: some intermediate topics are not deleted in the setup logic before recreating the user topics, which could cause the waitForDeletion (that check exact match of all existing topics) to fail, and also could cause more records to be returned because of the intermediate topics that are not deleted from the previous test case.

Also inspired by https://github.com/apache/kafka/pull/5418/files I used a longer timeout (120 secs) for deleting all topics.

Reviewers: John Roesler 
---
 .../streams/integration/AbstractResetIntegrationTest.java      | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/AbstractResetIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/AbstractResetIntegrationTest.java
index 675286bd259c8..dcf03046a2fd9 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/AbstractResetIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/AbstractResetIntegrationTest.java
@@ -190,7 +190,8 @@ void prepareTest() throws Exception {
         TestUtils.waitForCondition(new ConsumerGroupInactiveCondition(), TIMEOUT_MULTIPLIER * CLEANUP_CONSUMER_TIMEOUT,
                 "Test consumer group " + appID + " still active even after waiting " + (TIMEOUT_MULTIPLIER * CLEANUP_CONSUMER_TIMEOUT) + " ms.");
 
-        cluster.deleteAndRecreateTopics(INPUT_TOPIC, OUTPUT_TOPIC, OUTPUT_TOPIC_2, OUTPUT_TOPIC_2_RERUN);
+        cluster.deleteAllTopicsAndWait(120000);
+        cluster.createTopics(INPUT_TOPIC, OUTPUT_TOPIC, OUTPUT_TOPIC_2, OUTPUT_TOPIC_2_RERUN);
 
         add10InputElements();
     }

From c2d41888d69989dca12b27ccf8cab25ade5844eb Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Mon, 6 Jan 2020 09:06:11 -0800
Subject: [PATCH 0904/1071] MINOR: Fix failing test case in TransactionLogTest
 (#7895)

This patch fixes a brittle expectation on the `toString` implementation coming from `Set`. This was failing on jenkins with the following error:
```
java.lang.AssertionError: expected: but was:
```
Instead we convert the collection to a string directly.

Reviewers: Boyang Chen , Rajini Sivaram 
---
 .../scala/kafka/coordinator/transaction/TransactionLog.scala    | 2 +-
 .../unit/kafka/coordinator/transaction/TransactionLogTest.scala | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionLog.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionLog.scala
index 7c7d055db6ce6..2109095f08f35 100644
--- a/core/src/main/scala/kafka/coordinator/transaction/TransactionLog.scala
+++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionLog.scala
@@ -281,7 +281,7 @@ object TransactionLog {
       case Some(txnMetadata) => s"producerId:${txnMetadata.producerId}," +
         s"producerEpoch:${txnMetadata.producerEpoch}," +
         s"state=${txnMetadata.state}," +
-        s"partitions=${txnMetadata.topicPartitions}," +
+        s"partitions=${txnMetadata.topicPartitions.mkString("[", ",", "]")}," +
         s"txnLastUpdateTimestamp=${txnMetadata.txnLastUpdateTimestamp}," +
         s"txnTimeoutMs=${txnMetadata.txnTimeoutMs}"
     }
diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionLogTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionLogTest.scala
index e9bf6d51454ec..e9bdf3110a890 100644
--- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionLogTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionLogTest.scala
@@ -123,7 +123,7 @@ class TransactionLogTest {
     val (keyStringOpt, valueStringOpt) = TransactionLog.formatRecordKeyAndValue(transactionMetadataRecord)
     assertEquals(Some(s"transaction_metadata::transactionalId=$transactionalId"), keyStringOpt)
     assertEquals(Some(s"producerId:$producerId,producerEpoch:$producerEpoch,state=Ongoing," +
-      s"partitions=Set($topicPartition),txnLastUpdateTimestamp=0,txnTimeoutMs=$transactionTimeoutMs"), valueStringOpt)
+      s"partitions=[$topicPartition],txnLastUpdateTimestamp=0,txnTimeoutMs=$transactionTimeoutMs"), valueStringOpt)
   }
 
   @Test

From 8df0f52963da4ad3f478ec4133809e9a5458a62c Mon Sep 17 00:00:00 2001
From: Anna Povzner 
Date: Tue, 10 Mar 2020 12:30:58 -0700
Subject: [PATCH 0905/1071] KAFKA-9658; Fix user quota removal (#8232)

Adding (add-config) default user, user, or  quota and then removing it via delete-config does not update quota bound in ClientQuotaManager.Metrics for existing users or . This causes brokers to continue to throttle with the previously set quotas until brokers restart (or  stops sending traffic for sometime and sensor expires). This happens only when removing the user or user,client-id where there are no more quotas  to fall back to. Common example where the issue happens: Initial no quota state --> add default user quota --> remove default user quota.

The cause of the issue was `DefaultQuotaCallback.quotaLimit` was returning `null` when no default user quota set, which caused `ClientQuotaManager.updateQuotaMetricConfigs` to skip updating the appropriate sensor, which left it unchanged with the previous quota. Since `null` is an acceptable return value for `ClientQuotaCallback.quotaLimit`, which is already treated as unlimited quota in other parts of the code, this PR ensures that `ClientQuotaManager.updateQuotaMetricConfigs` updates the quotas for which  `ClientQuotaCallback.quotaLimit` returns `null` to unlimited quota.

Reviewers: Jason Gustafson , Rajini Sivaram 
---
 .../kafka/server/ClientQuotaManager.scala     |   5 +-
 .../kafka/server/ClientQuotaManagerTest.scala | 128 +++++++++++++-----
 2 files changed, 96 insertions(+), 37 deletions(-)

diff --git a/core/src/main/scala/kafka/server/ClientQuotaManager.scala b/core/src/main/scala/kafka/server/ClientQuotaManager.scala
index 730cc618122f0..45265287ce908 100644
--- a/core/src/main/scala/kafka/server/ClientQuotaManager.scala
+++ b/core/src/main/scala/kafka/server/ClientQuotaManager.scala
@@ -483,7 +483,7 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig,
       // Change the underlying metric config if the sensor has been created
       val metric = allMetrics.get(quotaMetricName)
       if (metric != null) {
-        Option(quotaCallback.quotaLimit(clientQuotaType, metricTags.asJava)).foreach { newQuota =>
+        Option(quotaLimit(metricTags.asJava)).foreach { newQuota =>
           info(s"Sensor for $quotaEntity already exists. Changing quota to $newQuota in MetricConfig")
           metric.config(getQuotaMetricConfig(newQuota))
         }
@@ -493,8 +493,7 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig,
       allMetrics.asScala.filterKeys(n => n.name == quotaMetricName.name && n.group == quotaMetricName.group).foreach {
         case (metricName, metric) =>
           val metricTags = metricName.tags
-          Option(quotaCallback.quotaLimit(clientQuotaType, metricTags)).foreach { quota =>
-            val newQuota = quota.asInstanceOf[Double]
+          Option(quotaLimit(metricTags)).foreach { newQuota =>
             if (newQuota != metric.config.quota.bound) {
               info(s"Sensor for quota-id $metricTags already exists. Setting quota to $newQuota in MetricConfig")
               metric.config(getQuotaMetricConfig(newQuota))
diff --git a/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala
index 828fb4841a271..accbdb76cc929 100644
--- a/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala
@@ -191,19 +191,79 @@ class ClientQuotaManagerTest {
     testQuotaParsing(config, client1, client2, randomClient, defaultConfigClient)
   }
 
+  private def checkQuota(quotaManager: ClientQuotaManager, user: String, clientId: String, expectedBound: Long, value: Int, expectThrottle: Boolean): Unit = {
+    assertEquals(expectedBound, quotaManager.quota(user, clientId).bound, 0.0)
+    val throttleTimeMs = maybeRecord(quotaManager, user, clientId, value * config.numQuotaSamples)
+    if (expectThrottle)
+      assertTrue(s"throttleTimeMs should be > 0. was $throttleTimeMs", throttleTimeMs > 0)
+    else
+      assertEquals(s"throttleTimeMs should be 0. was $throttleTimeMs", 0, throttleTimeMs)
+  }
+
   @Test
-  def testQuotaConfigPrecedence(): Unit = {
-    val quotaManager = new ClientQuotaManager(ClientQuotaManagerConfig(quotaBytesPerSecondDefault=Long.MaxValue),
+  def testSetAndRemoveDefaultUserQuota(): Unit = {
+    // quotaTypesEnabled will be QuotaTypes.NoQuotas initially
+    val quotaManager = new ClientQuotaManager(ClientQuotaManagerConfig(quotaBytesPerSecondDefault = Long.MaxValue),
       metrics, Produce, time, "")
 
-    def checkQuota(user: String, clientId: String, expectedBound: Int, value: Int, expectThrottle: Boolean): Unit = {
-      assertEquals(expectedBound, quotaManager.quota(user, clientId).bound, 0.0)
-      val throttleTimeMs = maybeRecord(quotaManager, user, clientId, value * config.numQuotaSamples)
-      if (expectThrottle)
-        assertTrue(s"throttleTimeMs should be > 0. was $throttleTimeMs", throttleTimeMs > 0)
-      else
-        assertEquals(s"throttleTimeMs should be 0. was $throttleTimeMs", 0, throttleTimeMs)
+    try {
+      // no quota set yet, should not throttle
+      checkQuota(quotaManager, "userA", "client1", Long.MaxValue, 1000, false)
+
+      // Set default  quota config
+      quotaManager.updateQuota(Some(ConfigEntityName.Default), None, None, Some(new Quota(10, true)))
+      checkQuota(quotaManager, "userA", "client1", 10, 1000, true)
+
+      // Remove default  quota config, back to no quotas
+      quotaManager.updateQuota(Some(ConfigEntityName.Default), None, None, None)
+      checkQuota(quotaManager, "userA", "client1", Long.MaxValue, 1000, false)
+    } finally {
+      quotaManager.shutdown()
     }
+  }
+
+  @Test
+  def testSetAndRemoveUserQuota(): Unit = {
+    // quotaTypesEnabled will be QuotaTypes.NoQuotas initially
+    val quotaManager = new ClientQuotaManager(ClientQuotaManagerConfig(quotaBytesPerSecondDefault = Long.MaxValue),
+      metrics, Produce, time, "")
+
+    try {
+      // Set  quota config
+      quotaManager.updateQuota(Some("userA"), None, None, Some(new Quota(10, true)))
+      checkQuota(quotaManager, "userA", "client1", 10, 1000, true)
+
+      // Remove  quota config, back to no quotas
+      quotaManager.updateQuota(Some("userA"), None, None, None)
+      checkQuota(quotaManager, "userA", "client1", Long.MaxValue, 1000, false)
+    } finally {
+      quotaManager.shutdown()
+    }
+  }
+
+  @Test
+  def testSetAndRemoveUserClientQuota(): Unit = {
+    // quotaTypesEnabled will be QuotaTypes.NoQuotas initially
+    val quotaManager = new ClientQuotaManager(ClientQuotaManagerConfig(quotaBytesPerSecondDefault = Long.MaxValue),
+      metrics, Produce, time, "")
+
+    try {
+      // Set  quota config
+      quotaManager.updateQuota(Some("userA"), Some("client1"), Some("client1"), Some(new Quota(10, true)))
+      checkQuota(quotaManager, "userA", "client1", 10, 1000, true)
+
+      // Remove  quota config, back to no quotas
+      quotaManager.updateQuota(Some("userA"), Some("client1"), Some("client1"), None)
+      checkQuota(quotaManager, "userA", "client1", Long.MaxValue, 1000, false)
+    } finally {
+      quotaManager.shutdown()
+    }
+  }
+
+  @Test
+  def testQuotaConfigPrecedence(): Unit = {
+    val quotaManager = new ClientQuotaManager(ClientQuotaManagerConfig(quotaBytesPerSecondDefault=Long.MaxValue),
+      metrics, Produce, time, "")
 
     try {
       quotaManager.updateQuota(Some(ConfigEntityName.Default), None, None, Some(new Quota(1000, true)))
@@ -217,47 +277,47 @@ class ClientQuotaManagerTest {
       quotaManager.updateQuota(Some("userC"), None, None, Some(new Quota(10000, true)))
       quotaManager.updateQuota(None, Some("client1"), Some("client1"), Some(new Quota(9000, true)))
 
-      checkQuota("userA", "client1", 5000, 4500, false) //  quota takes precedence over 
-      checkQuota("userA", "client2", 4000, 4500, true)  //  quota takes precedence over  and defaults
-      checkQuota("userA", "client3", 4000, 0, true)     //  quota is shared across clients of user
-      checkQuota("userA", "client1", 5000, 0, false)    //  is exclusive use, unaffected by other clients
+      checkQuota(quotaManager, "userA", "client1", 5000, 4500, false) //  quota takes precedence over 
+      checkQuota(quotaManager, "userA", "client2", 4000, 4500, true)  //  quota takes precedence over  and defaults
+      checkQuota(quotaManager, "userA", "client3", 4000, 0, true)     //  quota is shared across clients of user
+      checkQuota(quotaManager, "userA", "client1", 5000, 0, false)    //  is exclusive use, unaffected by other clients
 
-      checkQuota("userB", "client1", 7000, 8000, true)
-      checkQuota("userB", "client2", 8000, 7000, false) // Default per-client quota for exclusive use of 
-      checkQuota("userB", "client3", 8000, 7000, false)
+      checkQuota(quotaManager, "userB", "client1", 7000, 8000, true)
+      checkQuota(quotaManager, "userB", "client2", 8000, 7000, false) // Default per-client quota for exclusive use of 
+      checkQuota(quotaManager, "userB", "client3", 8000, 7000, false)
 
-      checkQuota("userD", "client1", 3000, 3500, true)  // Default  quota
-      checkQuota("userD", "client2", 3000, 2500, false)
-      checkQuota("userE", "client1", 3000, 2500, false)
+      checkQuota(quotaManager, "userD", "client1", 3000, 3500, true)  // Default  quota
+      checkQuota(quotaManager, "userD", "client2", 3000, 2500, false)
+      checkQuota(quotaManager, "userE", "client1", 3000, 2500, false)
 
       // Remove default  quota config, revert to  default
       quotaManager.updateQuota(Some(ConfigEntityName.Default), Some(ConfigEntityName.Default), Some(ConfigEntityName.Default), None)
-      checkQuota("userD", "client1", 1000, 0, false)    // Metrics tags changed, restart counter
-      checkQuota("userE", "client4", 1000, 1500, true)
-      checkQuota("userF", "client4", 1000, 800, false)  // Default  quota shared across clients of user
-      checkQuota("userF", "client5", 1000, 800, true)
+      checkQuota(quotaManager, "userD", "client1", 1000, 0, false)    // Metrics tags changed, restart counter
+      checkQuota(quotaManager, "userE", "client4", 1000, 1500, true)
+      checkQuota(quotaManager, "userF", "client4", 1000, 800, false)  // Default  quota shared across clients of user
+      checkQuota(quotaManager, "userF", "client5", 1000, 800, true)
 
       // Remove default  quota config, revert to  default
       quotaManager.updateQuota(Some(ConfigEntityName.Default), None, None, None)
-      checkQuota("userF", "client4", 2000, 0, false)  // Default  quota shared across client-id of all users
-      checkQuota("userF", "client5", 2000, 0, false)
-      checkQuota("userF", "client5", 2000, 2500, true)
-      checkQuota("userG", "client5", 2000, 0, true)
+      checkQuota(quotaManager, "userF", "client4", 2000, 0, false)  // Default  quota shared across client-id of all users
+      checkQuota(quotaManager, "userF", "client5", 2000, 0, false)
+      checkQuota(quotaManager, "userF", "client5", 2000, 2500, true)
+      checkQuota(quotaManager, "userG", "client5", 2000, 0, true)
 
       // Update quotas
       quotaManager.updateQuota(Some("userA"), None, None, Some(new Quota(8000, true)))
       quotaManager.updateQuota(Some("userA"), Some("client1"), Some("client1"), Some(new Quota(10000, true)))
-      checkQuota("userA", "client2", 8000, 0, false)
-      checkQuota("userA", "client2", 8000, 4500, true) // Throttled due to sum of new and earlier values
-      checkQuota("userA", "client1", 10000, 0, false)
-      checkQuota("userA", "client1", 10000, 6000, true)
+      checkQuota(quotaManager, "userA", "client2", 8000, 0, false)
+      checkQuota(quotaManager, "userA", "client2", 8000, 4500, true) // Throttled due to sum of new and earlier values
+      checkQuota(quotaManager, "userA", "client1", 10000, 0, false)
+      checkQuota(quotaManager, "userA", "client1", 10000, 6000, true)
       quotaManager.updateQuota(Some("userA"), Some("client1"), Some("client1"), None)
-      checkQuota("userA", "client6", 8000, 0, true)    // Throttled due to shared user quota
+      checkQuota(quotaManager, "userA", "client6", 8000, 0, true)    // Throttled due to shared user quota
       quotaManager.updateQuota(Some("userA"), Some("client6"), Some("client6"), Some(new Quota(11000, true)))
-      checkQuota("userA", "client6", 11000, 8500, false)
+      checkQuota(quotaManager, "userA", "client6", 11000, 8500, false)
       quotaManager.updateQuota(Some("userA"), Some(ConfigEntityName.Default), Some(ConfigEntityName.Default), Some(new Quota(12000, true)))
       quotaManager.updateQuota(Some("userA"), Some("client6"), Some("client6"), None)
-      checkQuota("userA", "client6", 12000, 4000, true) // Throttled due to sum of new and earlier values
+      checkQuota(quotaManager, "userA", "client6", 12000, 4000, true) // Throttled due to sum of new and earlier values
 
     } finally {
       quotaManager.shutdown()

From 91446ef8a5cfc69eb3655e6d23dd861507cb3e25 Mon Sep 17 00:00:00 2001
From: bill 
Date: Tue, 10 Mar 2020 22:20:16 -0400
Subject: [PATCH 0906/1071] Bump version to 2.4.2-SNAPSHOT

---
 gradle.properties | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gradle.properties b/gradle.properties
index bc2ddfecbc9c8..8b42e427935dd 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -20,7 +20,7 @@ group=org.apache.kafka
 #  - tests/kafkatest/__init__.py
 #  - tests/kafkatest/version.py (variable DEV_VERSION)
 #  - kafka-merge-pr.py
-version=2.4.1
+version=2.4.2-SNAPSHOT
 scalaVersion=2.12.10
 task=build
 org.gradle.jvmargs=-Xmx1024m -Xss2m

From 054f3345799b90c415f134ed24e470333ec691b2 Mon Sep 17 00:00:00 2001
From: bill 
Date: Wed, 11 Mar 2020 13:18:35 -0400
Subject: [PATCH 0907/1071] Bump version to 2.4.2-SNAPSHOT

---
 docs/js/templateData.js                                         | 2 +-
 streams/quickstart/java/pom.xml                                 | 2 +-
 .../java/src/main/resources/archetype-resources/pom.xml         | 2 +-
 streams/quickstart/pom.xml                                      | 2 +-
 tests/kafkatest/__init__.py                                     | 2 +-
 tests/kafkatest/version.py                                      | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/js/templateData.js b/docs/js/templateData.js
index b51afd1093f75..66123b93d42be 100644
--- a/docs/js/templateData.js
+++ b/docs/js/templateData.js
@@ -19,6 +19,6 @@ limitations under the License.
 var context={
     "version": "24",
     "dotVersion": "2.4",
-    "fullDotVersion": "2.4.1",
+    "fullDotVersion": "2.4.2-SNAPSHOT",
     "scalaVersion": "2.12"
 };
diff --git a/streams/quickstart/java/pom.xml b/streams/quickstart/java/pom.xml
index cc974f9fb79e4..fd12838e481fe 100644
--- a/streams/quickstart/java/pom.xml
+++ b/streams/quickstart/java/pom.xml
@@ -26,7 +26,7 @@
     
         org.apache.kafka
         streams-quickstart
-        2.4.1
+        2.4.2-SNAPSHOT
         ..
     
 
diff --git a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
index 8d75a280b1397..1e79eed3006fa 100644
--- a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
+++ b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
@@ -29,7 +29,7 @@
 
     
         UTF-8
-        2.4.1
+        2.4.2-SNAPSHOT
         1.7.7
         1.2.17
     
diff --git a/streams/quickstart/pom.xml b/streams/quickstart/pom.xml
index 581eea5dbeda4..07c3c050e5a18 100644
--- a/streams/quickstart/pom.xml
+++ b/streams/quickstart/pom.xml
@@ -22,7 +22,7 @@
     org.apache.kafka
     streams-quickstart
     pom
-    2.4.1
+    2.4.2-SNAPSHOT
 
     Kafka Streams :: Quickstart
 
diff --git a/tests/kafkatest/__init__.py b/tests/kafkatest/__init__.py
index c2453f0df4ab7..61456b968c2d9 100644
--- a/tests/kafkatest/__init__.py
+++ b/tests/kafkatest/__init__.py
@@ -22,4 +22,4 @@
 # Instead, in
 #
 # For example, when Kafka is at version 1.0.0-SNAPSHOT, this should be something like "1.0.0
-__version__ = '2.4.1'
+__version__ = '2.4.2-SNAPSHOT'
diff --git a/tests/kafkatest/version.py b/tests/kafkatest/version.py
index 050483bba82b2..e647cf6a426dd 100644
--- a/tests/kafkatest/version.py
+++ b/tests/kafkatest/version.py
@@ -63,7 +63,7 @@ def get_version(node=None):
         return DEV_BRANCH
 
 DEV_BRANCH = KafkaVersion("dev")
-DEV_VERSION = KafkaVersion("2.4.1-SNAPSHOT")
+DEV_VERSION = KafkaVersion("2.4.2-SNAPSHOT")
 
 # 0.8.2.x versions
 V_0_8_2_1 = KafkaVersion("0.8.2.1")

From 372e81ea62d4747fa1acfeb80e472cb2e9b9aaf0 Mon Sep 17 00:00:00 2001
From: Bruno Cadonna 
Date: Thu, 12 Mar 2020 19:51:33 +0100
Subject: [PATCH 0908/1071] KAFKA-9675: Fix bug that prevents RocksDB metrics
 to be updated (#8256)

Reviewers: John Roesler 
---
 .../streams/state/internals/RocksDBStore.java |   7 +-
 .../RocksDBMetricsIntegrationTest.java        | 130 ++++++++++++++----
 2 files changed, 108 insertions(+), 29 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
index 2b9b3f612a172..bf6a0332406c7 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
@@ -188,11 +188,12 @@ void openDB(final ProcessorContext context) {
             throw new ProcessorStateException(fatal);
         }
 
+        // Setup metrics before the database is opened, otherwise the metrics are not updated
+        // with the measurements from Rocks DB
+        maybeSetUpMetricsRecorder(context, configs);
+
         openRocksDB(dbOptions, columnFamilyOptions);
         open = true;
-
-        // Do this last because the prior operations could throw exceptions.
-        maybeSetUpMetricsRecorder(context, configs);
     }
 
     private void maybeSetUpMetricsRecorder(final ProcessorContext context, final Map configs) {
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/RocksDBMetricsIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/RocksDBMetricsIntegrationTest.java
index b0ac1faa7238c..34cc428bb958a 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/RocksDBMetricsIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/RocksDBMetricsIntegrationTest.java
@@ -42,7 +42,6 @@
 import org.apache.kafka.test.StreamsTestUtils;
 import org.apache.kafka.test.TestUtils;
 import org.junit.After;
-import org.junit.Assert;
 import org.junit.Before;
 import org.junit.ClassRule;
 import org.junit.Test;
@@ -61,6 +60,10 @@
 import java.util.Properties;
 import java.util.stream.Collectors;
 
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+
 @Category({IntegrationTest.class})
 @RunWith(Parameterized.class)
 public class RocksDBMetricsIntegrationTest {
@@ -74,8 +77,10 @@ public class RocksDBMetricsIntegrationTest {
     private static final String STREAM_OUTPUT = "STREAM_OUTPUT";
     private static final String MY_STORE_PERSISTENT_KEY_VALUE = "myStorePersistentKeyValue";
     private static final Duration WINDOW_SIZE = Duration.ofMillis(50);
+    private static final long TIMEOUT = 60000;
 
     // RocksDB metrics
+    private static final String METRICS_GROUP = "stream-state-metrics";
     private static final String BYTES_WRITTEN_RATE = "bytes-written-rate";
     private static final String BYTES_WRITTEN_TOTAL = "bytes-written-total";
     private static final String BYTES_READ_RATE = "bytes-read-rate";
@@ -114,26 +119,36 @@ public void after() throws Exception {
         CLUSTER.deleteTopicsAndWait(STREAM_INPUT, STREAM_OUTPUT);
     }
 
+    @FunctionalInterface
+    private interface MetricsVerifier {
+        void verify(final KafkaStreams kafkaStreams, final String metricScope) throws Exception;
+    }
+
     @Test
     public void shouldExposeRocksDBMetricsForNonSegmentedStateStoreBeforeAndAfterFailureWithEmptyStateDir() throws Exception {
         final Properties streamsConfiguration = streamsConfig();
         IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration);
         final StreamsBuilder builder = builderForNonSegmentedStateStore();
+        final String metricsScope = "rocksdb-state-id";
 
-        cleanUpStateRunAndVerify(
+        cleanUpStateRunVerifyAndClose(
             builder,
             streamsConfiguration,
             IntegerDeserializer.class,
             StringDeserializer.class,
-            "rocksdb-state-id"
+            this::verifyThatRocksDBMetricsAreExposed,
+            metricsScope
         );
 
-        cleanUpStateRunAndVerify(
+        // simulated failure
+
+        cleanUpStateRunVerifyAndClose(
             builder,
             streamsConfiguration,
             IntegerDeserializer.class,
             StringDeserializer.class,
-            "rocksdb-state-id"
+            this::verifyThatRocksDBMetricsAreExposed,
+            metricsScope
         );
     }
 
@@ -142,21 +157,60 @@ public void shouldExposeRocksDBMetricsForSegmentedStateStoreBeforeAndAfterFailur
         final Properties streamsConfiguration = streamsConfig();
         IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration);
         final StreamsBuilder builder = builderForSegmentedStateStore();
+        final String metricsScope = "rocksdb-window-state-id";
 
-        cleanUpStateRunAndVerify(
+        cleanUpStateRunVerifyAndClose(
             builder,
             streamsConfiguration,
             LongDeserializer.class,
             LongDeserializer.class,
-            "rocksdb-window-state-id"
+            this::verifyThatRocksDBMetricsAreExposed,
+            metricsScope
+        );
+
+        // simulated failure
+
+        cleanUpStateRunVerifyAndClose(
+            builder,
+            streamsConfiguration,
+            LongDeserializer.class,
+            LongDeserializer.class,
+            this::verifyThatRocksDBMetricsAreExposed,
+            metricsScope
+        );
+    }
+
+    @Test
+    public void shouldVerifyThatMetricsGetMeasurementsFromRocksDBForNonSegmentedStateStore() throws Exception {
+        final Properties streamsConfiguration = streamsConfig();
+        IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration);
+        final StreamsBuilder builder = builderForNonSegmentedStateStore();
+        final String metricsScope = "rocksdb-state-id";
+
+        cleanUpStateRunVerifyAndClose(
+            builder,
+            streamsConfiguration,
+            IntegerDeserializer.class,
+            StringDeserializer.class,
+            this::verifyThatBytesWrittenTotalIncreases,
+            metricsScope
         );
+    }
+
+    @Test
+    public void shouldVerifyThatMetricsGetMeasurementsFromRocksDBForSegmentedStateStore() throws Exception {
+        final Properties streamsConfiguration = streamsConfig();
+        IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration);
+        final StreamsBuilder builder = builderForSegmentedStateStore();
+        final String metricsScope = "rocksdb-window-state-id";
 
-        cleanUpStateRunAndVerify(
+        cleanUpStateRunVerifyAndClose(
             builder,
             streamsConfiguration,
             LongDeserializer.class,
             LongDeserializer.class,
-            "rocksdb-window-state-id"
+            this::verifyThatBytesWrittenTotalIncreases,
+            metricsScope
         );
     }
 
@@ -167,7 +221,6 @@ private Properties streamsConfig() {
         streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass());
         streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
         streamsConfiguration.put(StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG, Sensor.RecordingLevel.DEBUG.name);
-        streamsConfiguration.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 10 * 1024 * 1024L);
         streamsConfiguration.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, processingGuarantee);
         streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath());
         return streamsConfiguration;
@@ -198,16 +251,17 @@ private StreamsBuilder builderForSegmentedStateStore() {
         return builder;
     }
 
-    private void cleanUpStateRunAndVerify(final StreamsBuilder builder,
-                                          final Properties streamsConfiguration,
-                                          final Class outputKeyDeserializer,
-                                          final Class outputValueDeserializer,
-                                          final String metricsScope) throws Exception {
+    private void cleanUpStateRunVerifyAndClose(final StreamsBuilder builder,
+                                               final Properties streamsConfiguration,
+                                               final Class outputKeyDeserializer,
+                                               final Class outputValueDeserializer,
+                                               final MetricsVerifier metricsVerifier,
+                                               final String metricsScope) throws Exception {
         final KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration);
         kafkaStreams.cleanUp();
         produceRecords();
 
-        StreamsTestUtils.startKafkaStreamsAndWaitForRunningState(kafkaStreams, 60000);
+        StreamsTestUtils.startKafkaStreamsAndWaitForRunningState(kafkaStreams, TIMEOUT);
 
         IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
             TestUtils.consumerConfig(
@@ -220,7 +274,7 @@ private void cleanUpStateRunAndVerify(final StreamsBuilder builder,
             STREAM_OUTPUT,
             1
         );
-        verifyRocksDBMetrics(kafkaStreams, metricsScope);
+        metricsVerifier.verify(kafkaStreams, metricsScope);
         kafkaStreams.close();
     }
 
@@ -261,10 +315,9 @@ private void produceRecords() throws Exception {
         );
     }
 
-    private void verifyRocksDBMetrics(final KafkaStreams kafkaStreams, final String metricsScope) {
-        final List listMetricStore = new ArrayList(kafkaStreams.metrics().values()).stream()
-            .filter(m -> m.metricName().group().equals("stream-state-metrics") && m.metricName().tags().containsKey(metricsScope))
-            .collect(Collectors.toList());
+    private void verifyThatRocksDBMetricsAreExposed(final KafkaStreams kafkaStreams,
+                                                    final String metricsScope) {
+        final List listMetricStore = getRocksDBMetrics(kafkaStreams, metricsScope);
         checkMetricByName(listMetricStore, BYTES_WRITTEN_RATE, 1);
         checkMetricByName(listMetricStore, BYTES_WRITTEN_TOTAL, 1);
         checkMetricByName(listMetricStore, BYTES_READ_RATE, 1);
@@ -283,13 +336,38 @@ private void verifyRocksDBMetrics(final KafkaStreams kafkaStreams, final String
         checkMetricByName(listMetricStore, NUMBER_OF_FILE_ERRORS, 1);
     }
 
-    private void checkMetricByName(final List listMetric, final String metricName, final int numMetric) {
+    private void checkMetricByName(final List listMetric,
+                                   final String metricName,
+                                   final int numMetric) {
         final List metrics = listMetric.stream()
             .filter(m -> m.metricName().name().equals(metricName))
             .collect(Collectors.toList());
-        Assert.assertEquals("Size of metrics of type:'" + metricName + "' must be equal to " + numMetric + " but it's equal to " + metrics.size(), numMetric, metrics.size());
-        for (final Metric m : metrics) {
-            Assert.assertNotNull("Metric:'" + m.metricName() + "' must be not null", m.metricValue());
+        assertThat(
+            "Size of metrics of type:'" + metricName + "' must be equal to " + numMetric + " but it's equal to " + metrics.size(),
+            metrics.size(),
+            is(numMetric)
+        );
+        for (final Metric metric : metrics) {
+            assertThat("Metric:'" + metric.metricName() + "' must be not null", metric.metricValue(), is(notNullValue()));
         }
     }
-}
+
+    private void verifyThatBytesWrittenTotalIncreases(final KafkaStreams kafkaStreams,
+                                                      final String metricsScope) throws InterruptedException {
+        final List metric = getRocksDBMetrics(kafkaStreams, metricsScope).stream()
+            .filter(m -> BYTES_WRITTEN_TOTAL.equals(m.metricName().name()))
+            .collect(Collectors.toList());
+        TestUtils.waitForCondition(
+            () -> (double) metric.get(0).metricValue() > 0,
+            TIMEOUT,
+            () -> "RocksDB metric bytes.written.total did not increase in " + TIMEOUT + " ms"
+        );
+    }
+
+    private List getRocksDBMetrics(final KafkaStreams kafkaStreams,
+                                           final String metricsScope) {
+        return new ArrayList(kafkaStreams.metrics().values()).stream()
+            .filter(m -> m.metricName().group().equals(METRICS_GROUP) && m.metricName().tags().containsKey(metricsScope))
+            .collect(Collectors.toList());
+    }
+}
\ No newline at end of file

From 2cd966a6b9e37260f8216083ea088454117790d5 Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Sat, 14 Mar 2020 11:34:43 -0700
Subject: [PATCH 0909/1071] KAFKA-9533: Fix JavaDocs of KStream.transformValues
 (#8298)

Reviewers: Bill Bejeck 
---
 .../java/org/apache/kafka/streams/kstream/KStream.java    | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java
index b4e02253bf256..5631835a20092 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java
@@ -1269,8 +1269,6 @@  KStream flatTransform(final TransformerSupplier KStream transformValues(final ValueTransformerSupplier KStream transformValues(final ValueTransformerSupplier KStream transformValues(final ValueTransformerWithKeySupplier
Date: Fri, 20 Mar 2020 07:49:35 +0800
Subject: [PATCH 0910/1071] KAFKA-9654; Update epoch in
 `ReplicaAlterLogDirsThread` after new LeaderAndIsr  (#8223)

Currently when there is a leader change with a log dir reassignment in progress, we do not update the leader epoch in the partition state maintained by `ReplicaAlterLogDirsThread`. This can lead to a FENCED_LEADER_EPOCH error, which results in the partition being marked as failed, which is a permanent failure until the broker is restarted. This patch fixes the problem by updating the epoch in `ReplicaAlterLogDirsThread` after receiving a new LeaderAndIsr request from the controller.

Reviewers: Jun Rao , Jason Gustafson 
---
 .../kafka/server/AbstractFetcherThread.scala  | 57 +++++++++++++------
 .../scala/kafka/server/ReplicaManager.scala   | 10 ++--
 .../admin/ReassignPartitionsClusterTest.scala | 22 +++++--
 .../kafka/server/ReplicaManagerTest.scala     | 55 +++++++++++++++++-
 4 files changed, 116 insertions(+), 28 deletions(-)

diff --git a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
index 6e2e5da4c9a5b..6061f2603a860 100755
--- a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
+++ b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
@@ -219,7 +219,7 @@ abstract class AbstractFetcherThread(name: String,
         curPartitionState != null && leaderEpochInRequest == curPartitionState.currentLeaderEpoch
       }
 
-      val ResultWithPartitions(fetchOffsets, partitionsWithError) = maybeTruncateToEpochEndOffsets(epochEndOffsets)
+      val ResultWithPartitions(fetchOffsets, partitionsWithError) = maybeTruncateToEpochEndOffsets(epochEndOffsets, latestEpochsForPartitions)
       handlePartitionsWithErrors(partitionsWithError, "truncateToEpochEndOffsets")
       updateFetchOffsetAndMaybeMarkTruncationComplete(fetchOffsets)
     }
@@ -244,7 +244,8 @@ abstract class AbstractFetcherThread(name: String,
     updateFetchOffsetAndMaybeMarkTruncationComplete(fetchOffsets)
   }
 
-  private def maybeTruncateToEpochEndOffsets(fetchedEpochs: Map[TopicPartition, EpochEndOffset]): ResultWithPartitions[Map[TopicPartition, OffsetTruncationState]] = {
+  private def maybeTruncateToEpochEndOffsets(fetchedEpochs: Map[TopicPartition, EpochEndOffset],
+                                             latestEpochsForPartitions: Map[TopicPartition, EpochData]): ResultWithPartitions[Map[TopicPartition, OffsetTruncationState]] = {
     val fetchOffsets = mutable.HashMap.empty[TopicPartition, OffsetTruncationState]
     val partitionsWithError = mutable.HashSet.empty[TopicPartition]
 
@@ -256,7 +257,11 @@ abstract class AbstractFetcherThread(name: String,
             fetchOffsets.put(tp, offsetTruncationState)
 
         case Errors.FENCED_LEADER_EPOCH =>
-          onPartitionFenced(tp)
+          if (onPartitionFenced(tp, latestEpochsForPartitions.get(tp).flatMap {
+            p =>
+              if (p.currentLeaderEpoch.isPresent) Some(p.currentLeaderEpoch.get())
+              else None
+          })) partitionsWithError += tp
 
         case error =>
           info(s"Retrying leaderEpoch request for partition $tp as the leader reported an error: $error")
@@ -267,12 +272,22 @@ abstract class AbstractFetcherThread(name: String,
     ResultWithPartitions(fetchOffsets, partitionsWithError)
   }
 
-  private def onPartitionFenced(tp: TopicPartition): Unit = inLock(partitionMapLock) {
-    Option(partitionStates.stateValue(tp)).foreach { currentFetchState =>
+  /**
+   * remove the partition if the partition state is NOT updated. Otherwise, keep the partition active.
+   * @return true if the epoch in this thread is updated. otherwise, false
+   */
+  private def onPartitionFenced(tp: TopicPartition, requestEpoch: Option[Int]): Boolean = inLock(partitionMapLock) {
+    Option(partitionStates.stateValue(tp)).exists { currentFetchState =>
       val currentLeaderEpoch = currentFetchState.currentLeaderEpoch
-      info(s"Partition $tp has an older epoch ($currentLeaderEpoch) than the current leader. Will await " +
-        s"the new LeaderAndIsr state before resuming fetching.")
-      markPartitionFailed(tp)
+      if (requestEpoch.contains(currentLeaderEpoch)) {
+        info(s"Partition $tp has an older epoch ($currentLeaderEpoch) than the current leader. Will await " +
+          s"the new LeaderAndIsr state before resuming fetching.")
+        markPartitionFailed(tp)
+        false
+      } else {
+        info(s"Partition $tp has an new epoch ($currentLeaderEpoch) than the current leader. retry the partition later")
+        true
+      }
     }
   }
 
@@ -309,6 +324,10 @@ abstract class AbstractFetcherThread(name: String,
             // the current offset is the same as the offset requested.
             val fetchState = fetchStates(topicPartition)
             if (fetchState.fetchOffset == currentFetchState.fetchOffset && currentFetchState.isReadyForFetch) {
+              val requestEpoch = if (fetchState.currentLeaderEpoch >= 0)
+                Some(fetchState.currentLeaderEpoch)
+              else
+                None
               partitionData.error match {
                 case Errors.NONE =>
                   try {
@@ -351,7 +370,7 @@ abstract class AbstractFetcherThread(name: String,
                       markPartitionFailed(topicPartition)
                   }
                 case Errors.OFFSET_OUT_OF_RANGE =>
-                  if (!handleOutOfRangeError(topicPartition, currentFetchState))
+                  if (handleOutOfRangeError(topicPartition, currentFetchState, requestEpoch))
                     partitionsWithError += topicPartition
 
                 case Errors.UNKNOWN_LEADER_EPOCH =>
@@ -360,7 +379,7 @@ abstract class AbstractFetcherThread(name: String,
                   partitionsWithError += topicPartition
 
                 case Errors.FENCED_LEADER_EPOCH =>
-                  onPartitionFenced(topicPartition)
+                  if (onPartitionFenced(topicPartition, requestEpoch)) partitionsWithError += topicPartition
 
                 case Errors.NOT_LEADER_FOR_PARTITION =>
                   debug(s"Remote broker is not the leader for partition $topicPartition, which could indicate " +
@@ -522,32 +541,34 @@ abstract class AbstractFetcherThread(name: String,
   }
 
   /**
-   * Handle the out of range error. Return true if the request succeeded or was fenced, which means we need
-   * not backoff and retry. False if there was a retriable error.
+   * Handle the out of range error. Return false if
+   * 1) the request succeeded or
+   * 2) was fenced and this thread haven't received new epoch,
+   * which means we need not backoff and retry. True if there was a retriable error.
    */
   private def handleOutOfRangeError(topicPartition: TopicPartition,
-                                    fetchState: PartitionFetchState): Boolean = {
+                                    fetchState: PartitionFetchState,
+                                    requestEpoch: Option[Int]): Boolean = {
     try {
       val newOffset = fetchOffsetAndTruncate(topicPartition, fetchState.currentLeaderEpoch)
       val newFetchState = PartitionFetchState(newOffset, fetchState.currentLeaderEpoch, state = Fetching)
       partitionStates.updateAndMoveToEnd(topicPartition, newFetchState)
       info(s"Current offset ${fetchState.fetchOffset} for partition $topicPartition is " +
         s"out of range, which typically implies a leader change. Reset fetch offset to $newOffset")
-      true
+      false
     } catch {
       case _: FencedLeaderEpochException =>
-        onPartitionFenced(topicPartition)
-        true
+        onPartitionFenced(topicPartition, requestEpoch)
 
       case e @ (_ : UnknownTopicOrPartitionException |
                 _ : UnknownLeaderEpochException |
                 _ : NotLeaderForPartitionException) =>
         info(s"Could not fetch offset for $topicPartition due to error: ${e.getMessage}")
-        false
+        true
 
       case e: Throwable =>
         error(s"Error getting offset for partition $topicPartition", e)
-        false
+        true
     }
   }
 
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index 180b0e7d171da..ca90c30b9a7a9 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -1193,7 +1193,7 @@ class ReplicaManager(val config: KafkaConfig,
 
         // First check partition's leader epoch
         val partitionStates = new mutable.HashMap[Partition, LeaderAndIsrPartitionState]()
-        val newPartitions = new mutable.HashSet[Partition]
+        val updatedPartitions = new mutable.HashSet[Partition]
 
         leaderAndIsrRequest.partitionStates.asScala.foreach { partitionState =>
           val topicPartition = new TopicPartition(partitionState.topicName, partitionState.partitionIndex)
@@ -1206,12 +1206,14 @@ class ReplicaManager(val config: KafkaConfig,
               responseMap.put(topicPartition, Errors.KAFKA_STORAGE_ERROR)
               None
 
-            case HostedPartition.Online(partition) => Some(partition)
+            case HostedPartition.Online(partition) =>
+              updatedPartitions.add(partition)
+              Some(partition)
 
             case HostedPartition.None =>
               val partition = Partition(topicPartition, time, this)
               allPartitions.putIfNotExists(topicPartition, HostedPartition.Online(partition))
-              newPartitions.add(partition)
+              updatedPartitions.add(partition)
               Some(partition)
           }
 
@@ -1293,7 +1295,7 @@ class ReplicaManager(val config: KafkaConfig,
         startHighWatermarkCheckPointThread()
 
         val futureReplicasAndInitialOffset = new mutable.HashMap[TopicPartition, InitialFetchState]
-        for (partition <- newPartitions) {
+        for (partition <- updatedPartitions) {
           val topicPartition = partition.topicPartition
           if (logManager.getLog(topicPartition, isFuture = true).isDefined) {
             partition.log.foreach { log =>
diff --git a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
index c794570536e7c..0b0788ffd7d43 100644
--- a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
@@ -71,9 +71,9 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     JAdminClient.create(props)
   }
 
-  def getRandomLogDirAssignment(brokerId: Int): String = {
+  def getRandomLogDirAssignment(brokerId: Int, excluded: Option[String] = None): String = {
     val server = servers.find(_.config.brokerId == brokerId).get
-    val logDirs = server.config.logDirs
+    val logDirs = server.config.logDirs.filterNot(excluded.contains)
     new File(logDirs(Random.nextInt(logDirs.size))).getAbsolutePath
   }
 
@@ -161,19 +161,31 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
   }
 
   @Test
-  def shouldMoveSinglePartitionWithinBroker(): Unit = {
+  def shouldMoveSinglePartitionToSameFolderWithinBroker(): Unit = shouldMoveSinglePartitionWithinBroker(true)
+
+  @Test
+  def shouldMoveSinglePartitionToDifferentFolderWithinBroker(): Unit = shouldMoveSinglePartitionWithinBroker(false)
+
+  private[this] def shouldMoveSinglePartitionWithinBroker(moveToSameFolder: Boolean): Unit = {
     // Given a single replica on server 100
     startBrokers(Seq(100, 101))
     adminClient = createAdminClient(servers)
-    val expectedLogDir = getRandomLogDirAssignment(100)
     createTopic(zkClient, topicName, Map(tp0.partition() -> Seq(100)), servers = servers)
 
+    val replica = new TopicPartitionReplica(topicName, 0, 100)
+    val currentLogDir = adminClient.describeReplicaLogDirs(java.util.Collections.singleton(replica))
+      .all()
+      .get()
+      .get(replica)
+      .getCurrentReplicaLogDir
+
+    val expectedLogDir = if (moveToSameFolder) currentLogDir else getRandomLogDirAssignment(100, excluded = Some(currentLogDir))
+
     // When we execute an assignment that moves an existing replica to another log directory on the same broker
     val topicJson = executeAssignmentJson(Seq(
       PartitionAssignmentJson(tp0, replicas = Seq(100), logDirectories = Some(Seq(expectedLogDir)))
     ))
     ReassignPartitionsCommand.executeAssignment(zkClient, Some(adminClient), topicJson, NoThrottle)
-    val replica = new TopicPartitionReplica(topicName, 0, 100)
     waitUntilTrue(() => {
       expectedLogDir == adminClient.describeReplicaLogDirs(Collections.singleton(replica)).all().get.get(replica).getCurrentReplicaLogDir
     }, "Partition should have been moved to the expected log directory", 1000)
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index b05ff8b235232..17075f17db1c6 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -210,6 +210,58 @@ class ReplicaManagerTest {
     }
   }
 
+  @Test
+  def testFencedErrorCausedByBecomeLeader(): Unit = {
+    val replicaManager = setupReplicaManagerWithMockedPurgatories(new MockTimer)
+    try {
+      val brokerList = Seq[Integer](0, 1).asJava
+      val topicPartition = new TopicPartition(topic, 0)
+      replicaManager.createPartition(topicPartition)
+        .createLogIfNotExists(0, isNew = false, isFutureReplica = false,
+        new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
+
+      def leaderAndIsrRequest(epoch: Int): LeaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+        Seq(new LeaderAndIsrPartitionState()
+          .setTopicName(topic)
+          .setPartitionIndex(0)
+          .setControllerEpoch(0)
+          .setLeader(0)
+          .setLeaderEpoch(epoch)
+          .setIsr(brokerList)
+          .setZkVersion(0)
+          .setReplicas(brokerList)
+          .setIsNew(true)).asJava,
+        Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
+
+      replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(0), (_, _) => ())
+      val partition = replicaManager.getPartitionOrException(new TopicPartition(topic, 0), expectLeader = true)
+        .localLogOrException
+      assertEquals(1, replicaManager.logManager.liveLogDirs.filterNot(_ == partition.dir.getParentFile).size)
+
+      // find the live and different folder
+      val newReplicaFolder = replicaManager.logManager.liveLogDirs.filterNot(_ == partition.dir.getParentFile).head
+      assertEquals(0, replicaManager.replicaAlterLogDirsManager.fetcherThreadMap.size)
+      replicaManager.alterReplicaLogDirs(Map(topicPartition -> newReplicaFolder.getAbsolutePath))
+      replicaManager.futureLocalLogOrException(topicPartition)
+      assertEquals(1, replicaManager.replicaAlterLogDirsManager.fetcherThreadMap.size)
+      // change the epoch from 0 to 1 in order to make fenced error
+      replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(1), (_, _) => ())
+      TestUtils.waitUntilTrue(() => replicaManager.replicaAlterLogDirsManager.fetcherThreadMap.values.forall(_.partitionCount() == 0),
+        s"the partition=$topicPartition should be removed from pending state")
+      // the partition is added to failedPartitions if fenced error happens
+      // if the thread is done before ReplicaManager#becomeLeaderOrFollower updates epoch,the fenced error does
+      // not happen and failedPartitions is empty.
+      if (replicaManager.replicaAlterLogDirsManager.failedPartitions.size != 0) {
+        replicaManager.replicaAlterLogDirsManager.shutdownIdleFetcherThreads()
+        assertEquals(0, replicaManager.replicaAlterLogDirsManager.fetcherThreadMap.size)
+        // send request again
+        replicaManager.alterReplicaLogDirs(Map(topicPartition -> newReplicaFolder.getAbsolutePath))
+        // the future folder exists so it fails to invoke thread
+        assertEquals(1, replicaManager.replicaAlterLogDirsManager.fetcherThreadMap.size)
+      }
+    } finally replicaManager.shutdown(checkpointHW = false)
+  }
+
   @Test
   def testReceiveOutOfOrderSequenceExceptionWithLogStartOffset(): Unit = {
     val timer = new MockTimer
@@ -1280,6 +1332,7 @@ class ReplicaManagerTest {
         isFuture = false)).once
     }
     EasyMock.expect(mockLogMgr.initializingLog(topicPartitionObj)).anyTimes
+    EasyMock.expect(mockLogMgr.getLog(topicPartitionObj, isFuture = true)).andReturn(None)
 
     EasyMock.expect(mockLogMgr.finishedInitializingLog(
       EasyMock.eq(topicPartitionObj), EasyMock.anyObject(), EasyMock.anyObject())).anyTimes
@@ -1470,7 +1523,7 @@ class ReplicaManagerTest {
 
   private def setupReplicaManagerWithMockedPurgatories(timer: MockTimer, aliveBrokerIds: Seq[Int] = Seq(0, 1)): ReplicaManager = {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect)
-    props.put("log.dir", TestUtils.tempRelativeDir("data").getAbsolutePath)
+    props.put("log.dirs", TestUtils.tempRelativeDir("data").getAbsolutePath + "," + TestUtils.tempRelativeDir("data2").getAbsolutePath)
     val config = KafkaConfig.fromProps(props)
     val logProps = new Properties()
     val mockLogMgr = TestUtils.createLogManager(config.logDirs.map(new File(_)), LogConfig(logProps))

From e241e7c5dac4b427ab343b61602973e306394f62 Mon Sep 17 00:00:00 2001
From: Konstantine Karantasis 
Date: Fri, 20 Mar 2020 16:39:17 -0700
Subject: [PATCH 0911/1071] MINOR: Use Exit.exit instead of System.exit in MM2
 (#8321)

Exit.exit needs to be used in code instead of System.exit.

Particularly in integration tests using System.exit is disrupting because it exits the jvm process and does not just fail the test correctly. Integration tests override procedures in Exit to protect against such cases.

Reviewers: Ryanne Dolan , Ismael Juma , Randall Hauch 
---
 .../main/java/org/apache/kafka/connect/mirror/MirrorMaker.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorMaker.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorMaker.java
index d635b1c8c3e62..5ba689062072f 100644
--- a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorMaker.java
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorMaker.java
@@ -278,7 +278,7 @@ public static void main(String[] args) {
             ns = parser.parseArgs(args);
         } catch (ArgumentParserException e) {
             parser.handleError(e);
-            System.exit(-1);
+            Exit.exit(-1);
             return;
         }
         File configFile = (File) ns.get("config");

From 7c2edd746a819313228ba75722790224c2e188e1 Mon Sep 17 00:00:00 2001
From: Alaa Zbair <54908410+azbair@users.noreply.github.com>
Date: Sun, 22 Mar 2020 06:00:33 +0100
Subject: [PATCH 0912/1071] KAFKA-8842: : Reading/Writing confused in Connect
 QuickStart Guide

In step 7 of the QuickStart guide, "Writing data from the console and writing it back to the console" should be "Reading data from the console and writing it back to the console".

Co-authored-by: Alaa Zbair 

Reviewer: Konstantine Karantasis 
---
 docs/quickstart.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/quickstart.html b/docs/quickstart.html
index b9e0c7ccb6b3a..168c480ee48ea 100644
--- a/docs/quickstart.html
+++ b/docs/quickstart.html
@@ -211,7 +211,7 @@ 
            Step 6: Settin
 
 
            Step 7: Use Kafka Connect to import/export data
            
 
-
            Writing data from the console and writing it back to the console is a convenient place to start, but you'll probably want
+
            Reading data from the console and writing it back to the console is a convenient place to start, but you'll probably want
 to use data from other sources or export data from Kafka to other systems. For many systems, instead of writing custom
 integration code you can use Kafka Connect to import or export data.
            
 

From 07bb15e1ef6f1c8d9b301a0df41cd4d0d17a2fec Mon Sep 17 00:00:00 2001
From: nicolasguyomar 
Date: Sun, 22 Mar 2020 20:42:45 +0100
Subject: [PATCH 0913/1071] MINOR: Update Connect error message to point to the
 correct config validation REST endpoint (#7991)

When incorrect connector configuration is detected, the returned exception message suggests to check the connector's configuration against the `{connectorType}/config/validate` endpoint.

Changing the error message to refer to the exact REST endpoint which is `/connector-plugins/{connectorType}/config/validate`

This aligns the exception message with the documentation at: https://kafka.apache.org/documentation/#connect_rest

Reviewers: Konstantine Karantasis 
---
 .../java/org/apache/kafka/connect/runtime/AbstractHerder.java   | 2 +-
 .../kafka/connect/runtime/standalone/StandaloneHerderTest.java  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
index 83d81bb347da0..9a6553d0b9b96 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
@@ -556,7 +556,7 @@ protected final boolean maybeAddConfigErrors(
             callback.onCompletion(
                 new BadRequestException(
                     messages.append(
-                        "\nYou can also find the above list of errors at the endpoint `/{connectorType}/config/validate`"
+                        "\nYou can also find the above list of errors at the endpoint `/connector-plugins/{connectorType}/config/validate`"
                     ).toString()
                 ), null
             );
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java
index 3018677b82a9c..89d4e45fa91e9 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java
@@ -617,7 +617,7 @@ public void testCorruptConfig() {
             capture.getValue().getMessage(),
             "Connector configuration is invalid and contains the following 1 error(s):\n" +
                 error + "\n" +
-                "You can also find the above list of errors at the endpoint `/{connectorType}/config/validate`"
+                "You can also find the above list of errors at the endpoint `/connector-plugins/{connectorType}/config/validate`"
         );
 
         PowerMock.verifyAll();

From 14364b040c5493b29d51dae5b24a6313aaac0484 Mon Sep 17 00:00:00 2001
From: Tom Bentley 
Date: Sun, 22 Mar 2020 19:58:21 +0000
Subject: [PATCH 0914/1071] KAFKA-9634: Add note about thread safety in the
 ConfigProvider interface (#8205)

In Kafka Connect, a ConfigProvider instance can be used concurrently (e.g. via a PUT request to the `/connector-plugins/{connectorType}/config/validate` REST endpoint), but there is no mention of concurrent usage in the Javadocs of the ConfigProvider interface.

It's worth calling out that implementations need to be thread safe.

Reviewers: Konstantine Karantasis 
---
 .../org/apache/kafka/common/config/provider/ConfigProvider.java  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/clients/src/main/java/org/apache/kafka/common/config/provider/ConfigProvider.java b/clients/src/main/java/org/apache/kafka/common/config/provider/ConfigProvider.java
index 8561511e64e30..fe65ddfa30656 100644
--- a/clients/src/main/java/org/apache/kafka/common/config/provider/ConfigProvider.java
+++ b/clients/src/main/java/org/apache/kafka/common/config/provider/ConfigProvider.java
@@ -25,6 +25,7 @@
 
 /**
  * A provider of configuration data, which may optionally support subscriptions to configuration changes.
+ * Implementations are required to safely support concurrent calls to any of the methods in this interface.
  */
 public interface ConfigProvider extends Configurable, Closeable {
 

From 23ce049572953acd41f567c395c21f635f0da8ac Mon Sep 17 00:00:00 2001
From: Hossein Torabi 
Date: Mon, 23 Mar 2020 07:37:21 +0430
Subject: [PATCH 0915/1071] KAFKA-9563: Fix Kafka Connect documentation around
 consumer and producer overrides (#8124)

Kafka Connect main doc required a fix to distinguish between worker level producer and consumer overrides and per-connector level producer and consumer overrides, after the latter were introduced with KIP-458.

* [KAFKA-9563] Fix Kafka connect consumer and producer override documentation

Co-authored-by: Konstantine Karantasis 

Reviewers: Konstantine Karantasis 
---
 docs/connect.html | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/connect.html b/docs/connect.html
index a92bb04849fa1..8101da9792b09 100644
--- a/docs/connect.html
+++ b/docs/connect.html
@@ -56,7 +56,9 @@ 
            Running Kafka Connectoffset.storage.file.filename - File to store offset data in
            • 
     
          
 
-    
          The parameters that are configured here are intended for producers and consumers used by Kafka Connect to access the configuration, offset and status topics. For configuration of Kafka source and Kafka sink tasks, the same parameters can be used but need to be prefixed with consumer. and producer. respectively. The only parameter that is inherited from the worker configuration is bootstrap.servers, which in most cases will be sufficient, since the same cluster is often used for all purposes. A notable exception is a secured cluster, which requires extra parameters to allow connections. These parameters will need to be set up to three times in the worker configuration, once for management access, once for Kafka sinks and once for Kafka sources.
          
+    
          The parameters that are configured here are intended for producers and consumers used by Kafka Connect to access the configuration, offset and status topics. For configuration of the producers used by Kafka source tasks and the consumers used by Kafka sink tasks, the same parameters can be used but need to be prefixed with producer. and consumer. respectively. The only Kafka client parameter that is inherited without a prefix from the worker configuration is bootstrap.servers, which in most cases will be sufficient, since the same cluster is often used for all purposes. A notable exception is a secured cluster, which requires extra parameters to allow connections. These parameters will need to be set up to three times in the worker configuration, once for management access, once for Kafka sources and once for Kafka sinks.
           
+    
+    
          Starting with 2.3.0, client configuration overrides can be configured individually per connector by using the prefixes producer.override. and consumer.override. for Kafka sources or Kafka sinks respectively. These overrides are included with the rest of the connector's configuration properties.
          
 
     
          The remaining parameters are connector configuration files. You may include as many as you want, but all will execute within the same process (on different threads).
          
 

From e463bce0401bd8d73ce1f3f012052e679c285abd Mon Sep 17 00:00:00 2001
From: Bob Barrett 
Date: Tue, 24 Mar 2020 14:27:53 -0700
Subject: [PATCH 0916/1071] KAFKA-9749; Transaction coordinator should treat
 KAFKA_STORAGE_ERROR as retriable (#8336)

When handling a WriteTxnResponse, the TransactionMarkerRequestCompletionHandler throws an IllegalStateException when the remote broker responds with a KAFKA_STORAGE_ERROR and does not retry the request. This leaves the transaction state stuck in PendingAbort or PendingCommit, with no way to change that state other than restarting the broker, because both EndTxnRequest and InitProducerIdRequest return CONCURRENT_TRANSACTIONS if the state is PendingAbort or PendingCommit. This patch changes the error handling behavior in TransactionMarkerRequestCompletionHandler to retry KAFKA_STORAGE_ERRORs. This matches the existing client behavior, and makes sense because KAFKA_STORAGE_ERROR causes the host broker to shut down, meaning that the partition being written to will move its leadership to a new, healthy broker.

Reviewers: Boyang Chen , Jason Gustafson 
---
 .../TransactionMarkerRequestCompletionHandler.scala          | 3 ++-
 .../TransactionMarkerRequestCompletionHandlerTest.scala      | 5 +++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala
index fefe767eb40b8..f655770ce6b0c 100644
--- a/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala
+++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala
@@ -146,7 +146,8 @@ class TransactionMarkerRequestCompletionHandler(brokerId: Int,
                          Errors.NOT_LEADER_FOR_PARTITION |
                          Errors.NOT_ENOUGH_REPLICAS |
                          Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND |
-                         Errors.REQUEST_TIMED_OUT => // these are retriable errors
+                         Errors.REQUEST_TIMED_OUT |
+                         Errors.KAFKA_STORAGE_ERROR => // these are retriable errors
 
                       info(s"Sending $transactionalId's transaction marker for partition $topicPartition has failed with error ${error.exceptionName}, retrying " +
                         s"with current coordinator epoch ${epochAndMetadata.coordinatorEpoch}")
diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandlerTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandlerTest.scala
index 84f3dffbfbd58..1db652acd7adc 100644
--- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandlerTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandlerTest.scala
@@ -189,6 +189,11 @@ class TransactionMarkerRequestCompletionHandlerTest {
     verifyRetriesPartitionOnError(Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND)
   }
 
+  @Test
+  def shouldRetryPartitionWhenKafkaStorageError(): Unit = {
+    verifyRetriesPartitionOnError(Errors.KAFKA_STORAGE_ERROR)
+  }
+
   @Test
   def shouldRemoveTopicPartitionFromWaitingSetOnUnsupportedForMessageFormat(): Unit = {
     mockCache()

From bb2ea951ba20f253bf46c61176b1d091d6c2fd33 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Tue, 24 Mar 2020 22:16:49 -0700
Subject: [PATCH 0917/1071] KAFKA-9752; New member timeout can leave group
 rebalance stuck (#8339)

Older versions of the JoinGroup rely on a new member timeout to keep the group from growing indefinitely in the case of client disconnects and retrying. The logic for resetting the heartbeat expiration task following completion of the rebalance failed to account for an implicit expectation that shouldKeepAlive would return false the first time it is invoked when a heartbeat expiration is scheduled. This patch fixes the issue by making heartbeat satisfaction logic explicit.

Reviewers:  Chia-Ping Tsai , Guozhang Wang , Rajini Sivaram 
---
 .../coordinator/group/DelayedHeartbeat.scala  |   5 +-
 .../coordinator/group/GroupCoordinator.scala  |  37 +++---
 .../coordinator/group/MemberMetadata.scala    |  20 ++--
 .../group/GroupCoordinatorTest.scala          | 107 ++++++++++++++++--
 4 files changed, 133 insertions(+), 36 deletions(-)

diff --git a/core/src/main/scala/kafka/coordinator/group/DelayedHeartbeat.scala b/core/src/main/scala/kafka/coordinator/group/DelayedHeartbeat.scala
index 09c5eea58ea3e..3f402d9b5f747 100644
--- a/core/src/main/scala/kafka/coordinator/group/DelayedHeartbeat.scala
+++ b/core/src/main/scala/kafka/coordinator/group/DelayedHeartbeat.scala
@@ -27,11 +27,10 @@ private[group] class DelayedHeartbeat(coordinator: GroupCoordinator,
                                       group: GroupMetadata,
                                       memberId: String,
                                       isPending: Boolean,
-                                      deadline: Long,
                                       timeoutMs: Long)
   extends DelayedOperation(timeoutMs, Some(group.lock)) {
 
-  override def tryComplete(): Boolean = coordinator.tryCompleteHeartbeat(group, memberId, isPending, deadline, forceComplete _)
-  override def onExpiration() = coordinator.onExpireHeartbeat(group, memberId, isPending, deadline)
+  override def tryComplete(): Boolean = coordinator.tryCompleteHeartbeat(group, memberId, isPending, forceComplete _)
+  override def onExpiration() = coordinator.onExpireHeartbeat(group, memberId, isPending)
   override def onComplete() = coordinator.onCompleteHeartbeat()
 }
diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
index 7b69632c99b64..5f812b10dc685 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
@@ -910,14 +910,15 @@ class GroupCoordinator(val brokerId: Int,
   }
 
   private def completeAndScheduleNextExpiration(group: GroupMetadata, member: MemberMetadata, timeoutMs: Long): Unit = {
-    // complete current heartbeat expectation
-    member.latestHeartbeat = time.milliseconds()
     val memberKey = MemberKey(member.groupId, member.memberId)
+
+    // complete current heartbeat expectation
+    member.heartbeatSatisfied = true
     heartbeatPurgatory.checkAndComplete(memberKey)
 
     // reschedule the next heartbeat expiration deadline
-    val deadline = member.latestHeartbeat + timeoutMs
-    val delayedHeartbeat = new DelayedHeartbeat(this, group, member.memberId, isPending = false, deadline, timeoutMs)
+    member.heartbeatSatisfied = false
+    val delayedHeartbeat = new DelayedHeartbeat(this, group, member.memberId, isPending = false, timeoutMs)
     heartbeatPurgatory.tryCompleteElseWatch(delayedHeartbeat, Seq(memberKey))
   }
 
@@ -926,8 +927,7 @@ class GroupCoordinator(val brokerId: Int,
     */
   private def addPendingMemberExpiration(group: GroupMetadata, pendingMemberId: String, timeoutMs: Long): Unit = {
     val pendingMemberKey = MemberKey(group.groupId, pendingMemberId)
-    val deadline = time.milliseconds() + timeoutMs
-    val delayedHeartbeat = new DelayedHeartbeat(this, group, pendingMemberId, isPending = true, deadline, timeoutMs)
+    val delayedHeartbeat = new DelayedHeartbeat(this, group, pendingMemberId, isPending = true, timeoutMs)
     heartbeatPurgatory.tryCompleteElseWatch(delayedHeartbeat, Seq(pendingMemberKey))
   }
 
@@ -1110,7 +1110,10 @@ class GroupCoordinator(val brokerId: Int,
     }
   }
 
-  def tryCompleteHeartbeat(group: GroupMetadata, memberId: String, isPending: Boolean, heartbeatDeadline: Long, forceComplete: () => Boolean) = {
+  def tryCompleteHeartbeat(group: GroupMetadata,
+                           memberId: String,
+                           isPending: Boolean,
+                           forceComplete: () => Boolean): Boolean = {
     group.inLock {
       // The group has been unloaded and invalid, we should complete the heartbeat.
       if (group.is(Dead)) {
@@ -1120,25 +1123,23 @@ class GroupCoordinator(val brokerId: Int,
         if (group.has(memberId)) {
           forceComplete()
         } else false
-      } else {
-        if (shouldCompleteNonPendingHeartbeat(group, memberId, heartbeatDeadline)) {
-          forceComplete()
-        } else false
-      }
+      } else if (shouldCompleteNonPendingHeartbeat(group, memberId)) {
+        forceComplete()
+      } else false
     }
   }
 
-  def shouldCompleteNonPendingHeartbeat(group: GroupMetadata, memberId: String, heartbeatDeadline: Long): Boolean = {
+  def shouldCompleteNonPendingHeartbeat(group: GroupMetadata, memberId: String): Boolean = {
     if (group.has(memberId)) {
       val member = group.get(memberId)
-      member.shouldKeepAlive(heartbeatDeadline) || member.isLeaving
+      member.hasSatisfiedHeartbeat || member.isLeaving
     } else {
-      info(s"Member id $memberId was not found in ${group.groupId} during heartbeat expiration.")
-      false
+      info(s"Member id $memberId was not found in ${group.groupId} during heartbeat completion check")
+      true
     }
   }
 
-  def onExpireHeartbeat(group: GroupMetadata, memberId: String, isPending: Boolean, heartbeatDeadline: Long): Unit = {
+  def onExpireHeartbeat(group: GroupMetadata, memberId: String, isPending: Boolean): Unit = {
     group.inLock {
       if (group.is(Dead)) {
         info(s"Received notification of heartbeat expiration for member $memberId after group ${group.groupId} had already been unloaded or deleted.")
@@ -1149,7 +1150,7 @@ class GroupCoordinator(val brokerId: Int,
         debug(s"Member $memberId has already been removed from the group.")
       } else {
         val member = group.get(memberId)
-        if (!member.shouldKeepAlive(heartbeatDeadline)) {
+        if (!member.hasSatisfiedHeartbeat) {
           info(s"Member ${member.memberId} in group ${group.groupId} has failed, removing it from the group")
           removeMemberAndUpdateGroup(group, member, s"removing member ${member.memberId} on heartbeat expiration")
         }
diff --git a/core/src/main/scala/kafka/coordinator/group/MemberMetadata.scala b/core/src/main/scala/kafka/coordinator/group/MemberMetadata.scala
index f9a45d4e9d672..e9ec23e519234 100644
--- a/core/src/main/scala/kafka/coordinator/group/MemberMetadata.scala
+++ b/core/src/main/scala/kafka/coordinator/group/MemberMetadata.scala
@@ -66,11 +66,17 @@ private[group] class MemberMetadata(var memberId: String,
   var assignment: Array[Byte] = Array.empty[Byte]
   var awaitingJoinCallback: JoinGroupResult => Unit = null
   var awaitingSyncCallback: SyncGroupResult => Unit = null
-  var latestHeartbeat: Long = -1
   var isLeaving: Boolean = false
   var isNew: Boolean = false
   val isStaticMember: Boolean = groupInstanceId.isDefined
 
+  // This variable is used to track heartbeat completion through the delayed
+  // heartbeat purgatory. When scheduling a new heartbeat expiration, we set
+  // this value to `false`. Upon receiving the heartbeat (or any other event
+  // indicating the liveness of the client), we set it to `true` so that the
+  // delayed heartbeat can be completed.
+  var heartbeatSatisfied: Boolean = false
+
   def isAwaitingJoin = awaitingJoinCallback != null
   def isAwaitingSync = awaitingSyncCallback != null
 
@@ -85,16 +91,16 @@ private[group] class MemberMetadata(var memberId: String,
     }
   }
 
-  def shouldKeepAlive(deadlineMs: Long): Boolean = {
+  def hasSatisfiedHeartbeat: Boolean = {
     if (isNew) {
-      // New members are expired after the static join timeout
-      latestHeartbeat + GroupCoordinator.NewMemberJoinTimeoutMs > deadlineMs
+      // New members can be expired while awaiting join, so we have to check this first
+      heartbeatSatisfied
     } else if (isAwaitingJoin || isAwaitingSync) {
-      // Don't remove members as long as they have a request in purgatory
+      // Members that are awaiting a rebalance automatically satisfy expected heartbeats
       true
     } else {
-      // Otherwise check for session expiration
-      latestHeartbeat + sessionTimeoutMs > deadlineMs
+      // Otherwise we require the next heartbeat
+      heartbeatSatisfied
     }
   }
 
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
index 54116a376ab88..4d6d694f6d7f3 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala
@@ -385,6 +385,95 @@ class GroupCoordinatorTest {
     assertEquals(firstMemberId, group.allMembers.head)
   }
 
+  @Test
+  def testNewMemberFailureAfterJoinGroupCompletion(): Unit = {
+    // For old versions of the JoinGroup protocol, new members were subject
+    // to expiration if the rebalance took long enough. This test case ensures
+    // that following completion of the JoinGroup phase, new members follow
+    // normal heartbeat expiration logic.
+
+    val firstJoinResult = dynamicJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols)
+    val firstMemberId = firstJoinResult.memberId
+    val firstGenerationId = firstJoinResult.generationId
+    assertEquals(firstMemberId, firstJoinResult.leaderId)
+    assertEquals(Errors.NONE, firstJoinResult.error)
+
+    EasyMock.reset(replicaManager)
+    val firstSyncResult = syncGroupLeader(groupId, firstGenerationId, firstMemberId,
+      Map(firstMemberId -> Array[Byte]()))
+    assertEquals(Errors.NONE, firstSyncResult._2)
+
+    EasyMock.reset(replicaManager)
+    val otherJoinFuture = sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols)
+
+    EasyMock.reset(replicaManager)
+    val joinFuture = sendJoinGroup(groupId, firstMemberId, protocolType, protocols,
+      requireKnownMemberId = false)
+
+    val joinResult = await(joinFuture, DefaultSessionTimeout+100)
+    val otherJoinResult = await(otherJoinFuture, DefaultSessionTimeout+100)
+    assertEquals(Errors.NONE, joinResult.error)
+    assertEquals(Errors.NONE, otherJoinResult.error)
+
+    verifySessionExpiration(groupId)
+  }
+
+  @Test
+  def testNewMemberFailureAfterSyncGroupCompletion(): Unit = {
+    // For old versions of the JoinGroup protocol, new members were subject
+    // to expiration if the rebalance took long enough. This test case ensures
+    // that following completion of the SyncGroup phase, new members follow
+    // normal heartbeat expiration logic.
+
+    val firstJoinResult = dynamicJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols)
+    val firstMemberId = firstJoinResult.memberId
+    val firstGenerationId = firstJoinResult.generationId
+    assertEquals(firstMemberId, firstJoinResult.leaderId)
+    assertEquals(Errors.NONE, firstJoinResult.error)
+
+    EasyMock.reset(replicaManager)
+    val firstSyncResult = syncGroupLeader(groupId, firstGenerationId, firstMemberId,
+      Map(firstMemberId -> Array[Byte]()))
+    assertEquals(Errors.NONE, firstSyncResult._2)
+
+    EasyMock.reset(replicaManager)
+    val otherJoinFuture = sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols)
+
+    EasyMock.reset(replicaManager)
+    val joinFuture = sendJoinGroup(groupId, firstMemberId, protocolType, protocols,
+      requireKnownMemberId = false)
+
+    val joinResult = await(joinFuture, DefaultSessionTimeout+100)
+    val otherJoinResult = await(otherJoinFuture, DefaultSessionTimeout+100)
+    assertEquals(Errors.NONE, joinResult.error)
+    assertEquals(Errors.NONE, otherJoinResult.error)
+    val secondGenerationId = joinResult.generationId
+    val secondMemberId = otherJoinResult.memberId
+
+    EasyMock.reset(replicaManager)
+    sendSyncGroupFollower(groupId, secondGenerationId, secondMemberId)
+
+    EasyMock.reset(replicaManager)
+    val syncGroupResult = syncGroupLeader(groupId, secondGenerationId, firstMemberId,
+      Map(firstMemberId -> Array.emptyByteArray, secondMemberId -> Array.emptyByteArray))
+    assertEquals(Errors.NONE, syncGroupResult._2)
+
+    verifySessionExpiration(groupId)
+  }
+
+  private def verifySessionExpiration(groupId: String): Unit = {
+    EasyMock.reset(replicaManager)
+    EasyMock.expect(replicaManager.getMagic(EasyMock.anyObject()))
+      .andReturn(Some(RecordBatch.CURRENT_MAGIC_VALUE)).anyTimes()
+    EasyMock.replay(replicaManager)
+
+    timer.advanceClock(DefaultSessionTimeout + 1)
+
+    val groupMetadata = group(groupId)
+    assertEquals(Empty, groupMetadata.currentState)
+    assertEquals(0, groupMetadata.allMembers.size)
+  }
+
   @Test
   def testJoinGroupInconsistentGroupProtocol(): Unit = {
     val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID
@@ -675,11 +764,13 @@ class GroupCoordinatorTest {
 
   @Test
   def staticMemberRejoinWithLeaderIdAndKnownMemberId(): Unit = {
-    val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId, sessionTimeout = DefaultRebalanceTimeout / 2)
+    val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId,
+      sessionTimeout = DefaultRebalanceTimeout / 2)
 
     // A static leader with known id rejoin will trigger rebalance.
     EasyMock.reset(replicaManager)
-    val joinGroupResult = staticJoinGroup(groupId, rebalanceResult.leaderId, leaderInstanceId, protocolType, protocolSuperset, clockAdvance = DefaultRebalanceTimeout + 1)
+    val joinGroupResult = staticJoinGroup(groupId, rebalanceResult.leaderId, leaderInstanceId,
+      protocolType, protocolSuperset, clockAdvance = DefaultRebalanceTimeout + 1)
     // Timeout follower in the meantime.
     assertFalse(getGroup(groupId).hasStaticMember(followerInstanceId))
     checkJoinGroupResult(joinGroupResult,
@@ -3134,8 +3225,8 @@ class GroupCoordinatorTest {
     val group = getGroup(groupId)
     group.transitionTo(Dead)
     val leaderMemberId = rebalanceResult.leaderId
-    assertTrue(groupCoordinator.tryCompleteHeartbeat(group, leaderMemberId, false, DefaultSessionTimeout, () => true))
-    groupCoordinator.onExpireHeartbeat(group, leaderMemberId, false, DefaultSessionTimeout)
+    assertTrue(groupCoordinator.tryCompleteHeartbeat(group, leaderMemberId, false, () => true))
+    groupCoordinator.onExpireHeartbeat(group, leaderMemberId, false)
     assertTrue(group.has(leaderMemberId))
   }
 
@@ -3147,8 +3238,7 @@ class GroupCoordinatorTest {
     val group = getGroup(groupId)
     val leaderMemberId = rebalanceResult.leaderId
     group.remove(leaderMemberId)
-    assertFalse(groupCoordinator.tryCompleteHeartbeat(group, leaderMemberId, false, DefaultSessionTimeout, () => true))
-    groupCoordinator.onExpireHeartbeat(group, leaderMemberId, false, DefaultSessionTimeout)
+    assertTrue(groupCoordinator.tryCompleteHeartbeat(group, leaderMemberId, false, () => true))
   }
 
   private def getGroup(groupId: String): GroupMetadata = {
@@ -3242,12 +3332,13 @@ class GroupCoordinatorTest {
   private def sendSyncGroupFollower(groupId: String,
                                     generation: Int,
                                     memberId: String,
-                                    groupInstanceId: Option[String]): Future[SyncGroupCallbackParams] = {
+                                    groupInstanceId: Option[String] = None): Future[SyncGroupCallbackParams] = {
     val (responseFuture, responseCallback) = setupSyncGroupCallback
 
     EasyMock.replay(replicaManager)
 
-    groupCoordinator.handleSyncGroup(groupId, generation, memberId, groupInstanceId, Map.empty[String, Array[Byte]], responseCallback)
+    groupCoordinator.handleSyncGroup(groupId, generation, memberId, groupInstanceId,
+      Map.empty[String, Array[Byte]], responseCallback)
     responseFuture
   }
 

From 587e43d18e8ae7c9fa401d57e06652d5c2eea35c Mon Sep 17 00:00:00 2001
From: Anna Povzner 
Date: Sat, 14 Mar 2020 13:45:50 -0700
Subject: [PATCH 0918/1071] KAFKA-9677: Fix consumer fetch with small consume
 bandwidth quotas (#8290)

When we changed quota communication with KIP-219, fetch requests get throttled by returning empty response with the delay in throttle_time_ms and Kafka consumer retries again after the delay. With default configs, the maximum fetch size could be as big as 50MB (or 10MB per partition). The default broker config (1-second window, 10 full windows of tracked bandwidth/thread utilization usage) means that < 5MB/s consumer quota (per broker) may block consumers from being able to fetch any data.

This PR ensures that consumers cannot get blocked by quota by capping fetchMaxBytes in KafkaApis.handleFetchRequest() to quota window * consume bandwidth quota. In the example of default configs (10-second quota window) and 1MB/s consumer bandwidth quota, fetchMaxBytes would be capped to 10MB.

Reviewers: Rajini Sivaram 
---
 .../kafka/server/ClientQuotaManager.scala     | 16 +++++++++++
 .../main/scala/kafka/server/KafkaApis.scala   | 14 ++++++++--
 .../integration/kafka/api/BaseQuotaTest.scala | 28 +++++++++++++++++--
 .../kafka/api/CustomQuotaCallbackTest.scala   |  6 ++--
 .../kafka/server/ClientQuotaManagerTest.scala | 24 ++++++++++++++++
 5 files changed, 81 insertions(+), 7 deletions(-)

diff --git a/core/src/main/scala/kafka/server/ClientQuotaManager.scala b/core/src/main/scala/kafka/server/ClientQuotaManager.scala
index 45265287ce908..8316d0c637b1c 100644
--- a/core/src/main/scala/kafka/server/ClientQuotaManager.scala
+++ b/core/src/main/scala/kafka/server/ClientQuotaManager.scala
@@ -233,6 +233,22 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig,
     }
   }
 
+  /**
+   * Returns maximum value (produced/consume bytes or request processing time) that could be recorded without guaranteed throttling.
+   * Recording any larger value will always be throttled, even if no other values were recorded in the quota window.
+   * This is used for deciding the maximum bytes that can be fetched at once
+   */
+  def getMaxValueInQuotaWindow(session: Session, clientId: String): Double = {
+    if (quotasEnabled) {
+      val clientSensors = getOrCreateQuotaSensors(session, clientId)
+      Option(quotaCallback.quotaLimit(clientQuotaType, clientSensors.metricTags.asJava))
+        .map(_.toDouble * (config.numQuotaSamples - 1) * config.quotaWindowSizeSeconds)
+        .getOrElse(Double.MaxValue)
+    } else {
+      Double.MaxValue
+    }
+  }
+
   def recordAndGetThrottleTimeMs(session: Session, clientId: String, value: Double, timeMs: Long): Int = {
     var throttleTimeMs = 0
     val clientSensors = getOrCreateQuotaSensors(session, clientId)
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 124b2e3564f8a..da402ec4f8764 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -794,6 +794,16 @@ class KafkaApis(val requestChannel: RequestChannel,
       }
     }
 
+    // for fetch from consumer, cap fetchMaxBytes to the maximum bytes that could be fetched without being throttled given
+    // no bytes were recorded in the recent quota window
+    // trying to fetch more bytes would result in a guaranteed throttling potentially blocking consumer progress
+    val maxQuotaWindowBytes = if (fetchRequest.isFromFollower)
+      Int.MaxValue
+    else
+      quotas.fetch.getMaxValueInQuotaWindow(request.session, clientId).toInt
+
+    val fetchMaxBytes = Math.min(fetchRequest.maxBytes, maxQuotaWindowBytes)
+    val fetchMinBytes = Math.min(fetchRequest.minBytes, fetchMaxBytes)
     if (interesting.isEmpty)
       processResponseCallback(Seq.empty)
     else {
@@ -801,8 +811,8 @@ class KafkaApis(val requestChannel: RequestChannel,
       replicaManager.fetchMessages(
         fetchRequest.maxWait.toLong,
         fetchRequest.replicaId,
-        fetchRequest.minBytes,
-        fetchRequest.maxBytes,
+        fetchMinBytes,
+        fetchMaxBytes,
         versionId <= 2,
         interesting,
         replicationQuota(fetchRequest),
diff --git a/core/src/test/scala/integration/kafka/api/BaseQuotaTest.scala b/core/src/test/scala/integration/kafka/api/BaseQuotaTest.scala
index 651eef6188706..4d9d23e70f90c 100644
--- a/core/src/test/scala/integration/kafka/api/BaseQuotaTest.scala
+++ b/core/src/test/scala/integration/kafka/api/BaseQuotaTest.scala
@@ -15,6 +15,7 @@
 package kafka.api
 
 import java.time.Duration
+import java.util.concurrent.TimeUnit
 import java.util.{Collections, HashMap, Properties}
 
 import kafka.api.QuotaTestClients._
@@ -83,7 +84,7 @@ abstract class BaseQuotaTest extends IntegrationTestHarness {
     quotaTestClients.verifyProduceThrottle(expectThrottle = true)
 
     // Consumer should read in a bursty manner and get throttled immediately
-    quotaTestClients.consumeUntilThrottled(produced)
+    assertTrue("Should have consumed at least one record", quotaTestClients.consumeUntilThrottled(produced) > 0)
     quotaTestClients.verifyConsumeThrottle(expectThrottle = true)
   }
 
@@ -106,6 +107,23 @@ abstract class BaseQuotaTest extends IntegrationTestHarness {
     quotaTestClients.verifyConsumeThrottle(expectThrottle = false)
   }
 
+  @Test
+  def testProducerConsumerOverrideLowerQuota(): Unit = {
+    // consumer quota is set such that consumer quota * default quota window (10 seconds) is less than
+    // MAX_PARTITION_FETCH_BYTES_CONFIG, so that we can test consumer ability to fetch in this case
+    // In this case, 250 * 10 < 4096
+    quotaTestClients.overrideQuotas(2000, 250, Int.MaxValue)
+    quotaTestClients.waitForQuotaUpdate(2000, 250, Int.MaxValue)
+
+    val numRecords = 1000
+    val produced = quotaTestClients.produceUntilThrottled(numRecords)
+    quotaTestClients.verifyProduceThrottle(expectThrottle = true)
+
+    // Consumer should be able to consume at least one record, even when throttled
+    assertTrue("Should have consumed at least one record", quotaTestClients.consumeUntilThrottled(produced) > 0)
+    quotaTestClients.verifyConsumeThrottle(expectThrottle = true)
+  }
+
   @Test
   def testQuotaOverrideDelete(): Unit = {
     // Override producer and consumer quotas to unlimited
@@ -194,19 +212,23 @@ abstract class QuotaTestClients(topic: String,
   }
 
   def consumeUntilThrottled(maxRecords: Int, waitForRequestCompletion: Boolean = true): Int = {
+    val timeoutMs = TimeUnit.MINUTES.toMillis(1)
+
     consumer.subscribe(Collections.singleton(topic))
     var numConsumed = 0
     var throttled = false
+    val startMs = System.currentTimeMillis
     do {
       numConsumed += consumer.poll(Duration.ofMillis(100L)).count
       val metric = throttleMetric(QuotaType.Fetch, consumerClientId)
       throttled = metric != null && metricValue(metric) > 0
-    }  while (numConsumed < maxRecords && !throttled)
+    }  while (numConsumed < maxRecords && !throttled && System.currentTimeMillis < startMs + timeoutMs)
 
     // If throttled, wait for the records from the last fetch to be received
     if (throttled && numConsumed < maxRecords && waitForRequestCompletion) {
       val minRecords = numConsumed + 1
-      while (numConsumed < minRecords)
+      val startMs = System.currentTimeMillis
+      while (numConsumed < minRecords && System.currentTimeMillis < startMs + timeoutMs)
         numConsumed += consumer.poll(Duration.ofMillis(100L)).count
     }
     numConsumed
diff --git a/core/src/test/scala/integration/kafka/api/CustomQuotaCallbackTest.scala b/core/src/test/scala/integration/kafka/api/CustomQuotaCallbackTest.scala
index 15cb6b98166a3..09b163a2c3973 100644
--- a/core/src/test/scala/integration/kafka/api/CustomQuotaCallbackTest.scala
+++ b/core/src/test/scala/integration/kafka/api/CustomQuotaCallbackTest.scala
@@ -100,9 +100,11 @@ class CustomQuotaCallbackTest extends IntegrationTestHarness with SaslSetup {
     quotaLimitCalls.values.foreach(_.set(0))
     user.produceConsume(expectProduceThrottle = false, expectConsumeThrottle = false)
 
-    // ClientQuotaCallback#quotaLimit is invoked by each quota manager once for each new client
+    // ClientQuotaCallback#quotaLimit is invoked by each quota manager once per throttled produce request for each client
     assertEquals(1, quotaLimitCalls(ClientQuotaType.PRODUCE).get)
-    assertEquals(1, quotaLimitCalls(ClientQuotaType.FETCH).get)
+    // ClientQuotaCallback#quotaLimit is invoked once per each unthrottled and two for each throttled request
+    // since we don't know the total number of requests, we verify it was called at least twice (at least one throttled request)
+    assertTrue("quotaLimit must be called at least twice", quotaLimitCalls(ClientQuotaType.FETCH).get > 2)
     assertTrue(s"Too many quotaLimit calls $quotaLimitCalls", quotaLimitCalls(ClientQuotaType.REQUEST).get <= 10) // sanity check
     // Large quota updated to small quota, should throttle
     user.configureAndWaitForQuota(9000, 3000)
diff --git a/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala
index accbdb76cc929..f277e5f323d8c 100644
--- a/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala
@@ -193,6 +193,11 @@ class ClientQuotaManagerTest {
 
   private def checkQuota(quotaManager: ClientQuotaManager, user: String, clientId: String, expectedBound: Long, value: Int, expectThrottle: Boolean): Unit = {
     assertEquals(expectedBound, quotaManager.quota(user, clientId).bound, 0.0)
+    val session = Session(new KafkaPrincipal(KafkaPrincipal.USER_TYPE, user), InetAddress.getLocalHost)
+    val expectedMaxValueInQuotaWindow =
+      if (expectedBound < Long.MaxValue) config.quotaWindowSizeSeconds * (config.numQuotaSamples - 1) * expectedBound else Double.MaxValue
+    assertEquals(expectedMaxValueInQuotaWindow, quotaManager.getMaxValueInQuotaWindow(session, clientId), 0.01)
+
     val throttleTimeMs = maybeRecord(quotaManager, user, clientId, value * config.numQuotaSamples)
     if (expectThrottle)
       assertTrue(s"throttleTimeMs should be > 0. was $throttleTimeMs", throttleTimeMs > 0)
@@ -200,6 +205,25 @@ class ClientQuotaManagerTest {
       assertEquals(s"throttleTimeMs should be 0. was $throttleTimeMs", 0, throttleTimeMs)
   }
 
+  @Test
+  def testGetMaxValueInQuotaWindowWithNonDefaultQuotaWindow(): Unit = {
+    val numFullQuotaWindows = 3   // 3 seconds window (vs. 10 seconds default)
+    val nonDefaultConfig = ClientQuotaManagerConfig(quotaBytesPerSecondDefault = Long.MaxValue, numQuotaSamples = numFullQuotaWindows + 1)
+    val quotaManager = new ClientQuotaManager(nonDefaultConfig, metrics, Fetch, time, "")
+    val userSession = Session(new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "userA"), InetAddress.getLocalHost)
+
+    try {
+      // no quota set
+      assertEquals(Double.MaxValue, quotaManager.getMaxValueInQuotaWindow(userSession, "client1"), 0.01)
+
+      // Set default  quota config
+      quotaManager.updateQuota(Some(ConfigEntityName.Default), None, None, Some(new Quota(10, true)))
+      assertEquals(10 * numFullQuotaWindows, quotaManager.getMaxValueInQuotaWindow(userSession, "client1"), 0.01)
+    } finally {
+      quotaManager.shutdown()
+    }
+  }
+
   @Test
   def testSetAndRemoveDefaultUserQuota(): Unit = {
     // quotaTypesEnabled will be QuotaTypes.NoQuotas initially

From d9ecbc229b6dff4cf31a26fdcb53dee1441f9939 Mon Sep 17 00:00:00 2001
From: Greg Harris 
Date: Thu, 26 Mar 2020 10:21:11 -0700
Subject: [PATCH 0919/1071] KAFKA-9707: Fix InsertField.Key should apply to
 keys of tombstone records (#8280)

* KAFKA-9707: Fix InsertField.Key not applying to tombstone events

* Fix typo that hardcoded .value() instead of abstract operatingValue
* Add test for Key transform that was previously not tested

Signed-off-by: Greg Harris 

* Add null value assertion to tombstone test

* Remove mis-named function and add test for passing-through a null-keyed record.

Signed-off-by: Greg Harris 

* Simplify unchanged record assertion

Signed-off-by: Greg Harris 

* Replace assertEquals with assertSame

Signed-off-by: Greg Harris 

* Fix checkstyleTest indent issue

Signed-off-by: Greg Harris 
---
 .../kafka/connect/transforms/InsertField.java |  6 +-
 .../connect/transforms/InsertFieldTest.java   | 70 +++++++++++++++----
 2 files changed, 58 insertions(+), 18 deletions(-)

diff --git a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/InsertField.java b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/InsertField.java
index 93ba79c0e3545..8f0c5aa1f63fb 100644
--- a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/InsertField.java
+++ b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/InsertField.java
@@ -127,7 +127,7 @@ public void configure(Map props) {
 
     @Override
     public R apply(R record) {
-        if (isTombstoneRecord(record)) {
+        if (operatingValue(record) == null) {
             return record;
         } else if (operatingSchema(record) == null) {
             return applySchemaless(record);
@@ -136,10 +136,6 @@ public R apply(R record) {
         }
     }
 
-    private boolean isTombstoneRecord(R record) {
-        return record.value() == null;
-    }
-
     private R applySchemaless(R record) {
         final Map value = requireMap(operatingValue(record), PURPOSE);
 
diff --git a/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/InsertFieldTest.java b/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/InsertFieldTest.java
index b22872cacb236..dc6611c21c096 100644
--- a/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/InsertFieldTest.java
+++ b/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/InsertFieldTest.java
@@ -33,17 +33,18 @@
 import static org.junit.Assert.assertSame;
 
 public class InsertFieldTest {
-    private InsertField xform = new InsertField.Value<>();
+    private InsertField xformKey = new InsertField.Key<>();
+    private InsertField xformValue = new InsertField.Value<>();
 
     @After
     public void teardown() {
-        xform.close();
+        xformValue.close();
     }
 
     @Test(expected = DataException.class)
     public void topLevelStructRequired() {
-        xform.configure(Collections.singletonMap("topic.field", "topic_field"));
-        xform.apply(new SourceRecord(null, null, "", 0, Schema.INT32_SCHEMA, 42));
+        xformValue.configure(Collections.singletonMap("topic.field", "topic_field"));
+        xformValue.apply(new SourceRecord(null, null, "", 0, Schema.INT32_SCHEMA, 42));
     }
 
     @Test
@@ -55,13 +56,13 @@ public void copySchemaAndInsertConfiguredFields() {
         props.put("static.field", "instance_id");
         props.put("static.value", "my-instance-id");
 
-        xform.configure(props);
+        xformValue.configure(props);
 
         final Schema simpleStructSchema = SchemaBuilder.struct().name("name").version(1).doc("doc").field("magic", Schema.OPTIONAL_INT64_SCHEMA).build();
         final Struct simpleStruct = new Struct(simpleStructSchema).put("magic", 42L);
 
         final SourceRecord record = new SourceRecord(null, null, "test", 0, simpleStructSchema, simpleStruct);
-        final SourceRecord transformedRecord = xform.apply(record);
+        final SourceRecord transformedRecord = xformValue.apply(record);
 
         assertEquals(simpleStructSchema.name(), transformedRecord.valueSchema().name());
         assertEquals(simpleStructSchema.version(), transformedRecord.valueSchema().version());
@@ -83,7 +84,7 @@ public void copySchemaAndInsertConfiguredFields() {
         assertEquals("my-instance-id", ((Struct) transformedRecord.value()).getString("instance_id"));
 
         // Exercise caching
-        final SourceRecord transformedRecord2 = xform.apply(
+        final SourceRecord transformedRecord2 = xformValue.apply(
                 new SourceRecord(null, null, "test", 1, simpleStructSchema, new Struct(simpleStructSchema)));
         assertSame(transformedRecord.valueSchema(), transformedRecord2.valueSchema());
     }
@@ -97,12 +98,12 @@ public void schemalessInsertConfiguredFields() {
         props.put("static.field", "instance_id");
         props.put("static.value", "my-instance-id");
 
-        xform.configure(props);
+        xformValue.configure(props);
 
         final SourceRecord record = new SourceRecord(null, null, "test", 0,
                 null, Collections.singletonMap("magic", 42L));
 
-        final SourceRecord transformedRecord = xform.apply(record);
+        final SourceRecord transformedRecord = xformValue.apply(record);
 
         assertEquals(42L, ((Map) transformedRecord.value()).get("magic"));
         assertEquals("test", ((Map) transformedRecord.value()).get("topic_field"));
@@ -121,12 +122,12 @@ public void insertConfiguredFieldsIntoTombstoneEventWithoutSchemaLeavesValueUnch
         props.put("static.field", "instance_id");
         props.put("static.value", "my-instance-id");
 
-        xform.configure(props);
+        xformValue.configure(props);
 
         final SourceRecord record = new SourceRecord(null, null, "test", 0,
                 null, null);
 
-        final SourceRecord transformedRecord = xform.apply(record);
+        final SourceRecord transformedRecord = xformValue.apply(record);
 
         assertEquals(null, transformedRecord.value());
         assertEquals(null, transformedRecord.valueSchema());
@@ -141,16 +142,59 @@ public void insertConfiguredFieldsIntoTombstoneEventWithSchemaLeavesValueUnchang
         props.put("static.field", "instance_id");
         props.put("static.value", "my-instance-id");
 
-        xform.configure(props);
+        xformValue.configure(props);
 
         final Schema simpleStructSchema = SchemaBuilder.struct().name("name").version(1).doc("doc").field("magic", Schema.OPTIONAL_INT64_SCHEMA).build();
 
         final SourceRecord record = new SourceRecord(null, null, "test", 0,
                 simpleStructSchema, null);
 
-        final SourceRecord transformedRecord = xform.apply(record);
+        final SourceRecord transformedRecord = xformValue.apply(record);
 
         assertEquals(null, transformedRecord.value());
         assertEquals(simpleStructSchema, transformedRecord.valueSchema());
     }
+
+    @Test
+    public void insertKeyFieldsIntoTombstoneEvent() {
+        final Map props = new HashMap<>();
+        props.put("topic.field", "topic_field!");
+        props.put("partition.field", "partition_field");
+        props.put("timestamp.field", "timestamp_field?");
+        props.put("static.field", "instance_id");
+        props.put("static.value", "my-instance-id");
+
+        xformKey.configure(props);
+
+        final SourceRecord record = new SourceRecord(null, null, "test", 0,
+            null, Collections.singletonMap("magic", 42L), null, null);
+
+        final SourceRecord transformedRecord = xformKey.apply(record);
+
+        assertEquals(42L, ((Map) transformedRecord.key()).get("magic"));
+        assertEquals("test", ((Map) transformedRecord.key()).get("topic_field"));
+        assertEquals(0, ((Map) transformedRecord.key()).get("partition_field"));
+        assertEquals(null, ((Map) transformedRecord.key()).get("timestamp_field"));
+        assertEquals("my-instance-id", ((Map) transformedRecord.key()).get("instance_id"));
+        assertEquals(null, transformedRecord.value());
+    }
+
+    @Test
+    public void insertIntoNullKeyLeavesRecordUnchanged() {
+        final Map props = new HashMap<>();
+        props.put("topic.field", "topic_field!");
+        props.put("partition.field", "partition_field");
+        props.put("timestamp.field", "timestamp_field?");
+        props.put("static.field", "instance_id");
+        props.put("static.value", "my-instance-id");
+
+        xformKey.configure(props);
+
+        final SourceRecord record = new SourceRecord(null, null, "test", 0,
+            null, null, null, Collections.singletonMap("magic", 42L));
+
+        final SourceRecord transformedRecord = xformKey.apply(record);
+
+        assertSame(record, transformedRecord);
+    }
 }

From 22956b351fde0219b9a0efebdc3a6d1a30b9cce5 Mon Sep 17 00:00:00 2001
From: Scott 
Date: Sat, 28 Mar 2020 21:34:07 -0400
Subject: [PATCH 0920/1071] MINOR: Fix code example reference to SchemaBuilder
 call in Connect's documentation (#3029)

Simple doc fix in a code snippet in connect.html

Co-authored-by: Scott Ferguson 

Reviewers: Ewen Cheslack-Postava , Konstantine Karantasis 
---
 docs/connect.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/connect.html b/docs/connect.html
index 8101da9792b09..30750b38bd957 100644
--- a/docs/connect.html
+++ b/docs/connect.html
@@ -489,7 +489,7 @@ 
          Working with Schemas
          
Date: Sun, 29 Mar 2020 06:28:06 +0200
Subject: [PATCH 0921/1071] MINOR: Fix error message in exception when records
 have schemas in Connect's Flatten transformation (#3982)

In case of an error while flattening a record with schema, the Flatten transformation was reporting an error about a record without schema, as follows:

```
org.apache.kafka.connect.errors.DataException: Flatten transformation does not support ARRAY for record without schemas (for field ...)
```

The expected behaviour would be an error message specifying "with schemas".

This looks like a simple copy/paste typo from the schemaless equivalent methods, in the same file

Reviewers: Ewen Cheslack-Postava , Konstantine Karantasis 
---
 .../java/org/apache/kafka/connect/transforms/Flatten.java     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Flatten.java b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Flatten.java
index d7d21445d73d8..42528e2e0121b 100644
--- a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Flatten.java
+++ b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Flatten.java
@@ -194,7 +194,7 @@ private void buildUpdatedSchema(Schema schema, String fieldNamePrefix, SchemaBui
                     break;
                 default:
                     throw new DataException("Flatten transformation does not support " + field.schema().type()
-                            + " for record without schemas (for field " + fieldName + ").");
+                            + " for record with schemas (for field " + fieldName + ").");
             }
         }
     }
@@ -242,7 +242,7 @@ private void buildWithSchema(Struct record, String fieldNamePrefix, Struct newRe
                     break;
                 default:
                     throw new DataException("Flatten transformation does not support " + field.schema().type()
-                            + " for record without schemas (for field " + fieldName + ").");
+                            + " for record with schemas (for field " + fieldName + ").");
             }
         }
     }

From 791c67b74037e40d0126efd768744812449585fa Mon Sep 17 00:00:00 2001
From: Greg Harris 
Date: Mon, 30 Mar 2020 15:09:27 -0700
Subject: [PATCH 0922/1071] KAFKA-9706: Handle null in keys or values when
 Flatten transformation is used (#8279)

* Fixed DataException thrown when handling tombstone events with null value
* Passes through original record when finding a null key when it's configured for keys or a null value when it's configured for values.
* Added unit tests for schema and schemaless data
---
 .../kafka/connect/transforms/Flatten.java     |  4 ++-
 .../kafka/connect/transforms/FlattenTest.java | 25 +++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Flatten.java b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Flatten.java
index 42528e2e0121b..dbd8a6830390a 100644
--- a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Flatten.java
+++ b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Flatten.java
@@ -69,7 +69,9 @@ public void configure(Map props) {
 
     @Override
     public R apply(R record) {
-        if (operatingSchema(record) == null) {
+        if (operatingValue(record) == null) {
+            return record;
+        } else if (operatingSchema(record) == null) {
             return applySchemaless(record);
         } else {
             return applyWithSchema(record);
diff --git a/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/FlattenTest.java b/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/FlattenTest.java
index 2e7be9562cb26..d044338f5e25d 100644
--- a/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/FlattenTest.java
+++ b/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/FlattenTest.java
@@ -297,4 +297,29 @@ public void testOptionalAndDefaultValuesNested() {
         Schema transformedOptFieldSchema = SchemaBuilder.string().optional().defaultValue("child_default").build();
         assertEquals(transformedOptFieldSchema, transformedSchema.field("opt_field").schema());
     }
+
+    @Test
+    public void tombstoneEventWithoutSchemaShouldPassThrough() {
+        xformValue.configure(Collections.emptyMap());
+
+        final SourceRecord record = new SourceRecord(null, null, "test", 0,
+                null, null);
+        final SourceRecord transformedRecord = xformValue.apply(record);
+
+        assertEquals(null, transformedRecord.value());
+        assertEquals(null, transformedRecord.valueSchema());
+    }
+
+    @Test
+    public void tombstoneEventWithSchemaShouldPassThrough() {
+        xformValue.configure(Collections.emptyMap());
+
+        final Schema simpleStructSchema = SchemaBuilder.struct().name("name").version(1).doc("doc").field("magic", Schema.OPTIONAL_INT64_SCHEMA).build();
+        final SourceRecord record = new SourceRecord(null, null, "test", 0,
+                simpleStructSchema, null);
+        final SourceRecord transformedRecord = xformValue.apply(record);
+
+        assertEquals(null, transformedRecord.value());
+        assertEquals(simpleStructSchema, transformedRecord.valueSchema());
+    }
 }

From 9c91e052f8531fe65df989ad0761ab10623ac116 Mon Sep 17 00:00:00 2001
From: Bill Bejeck 
Date: Fri, 3 Apr 2020 15:14:23 -0400
Subject: [PATCH 0923/1071] KAFKA-9739:  Fixes null key changing child node
 (#8416)

2.4 port of #8400 since cherry-picking not possible

Reviewers: John Roesler 
---
 .../internals/InternalStreamsBuilder.java     |  17 +-
 .../internals/graph/StreamsGraphTest.java     | 186 ++++++++++++++++++
 2 files changed, 198 insertions(+), 5 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java
index 9509431e3eeaf..cbb4a6574a4cd 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java
@@ -396,11 +396,13 @@ private void maybeUpdateKeyChangingRepartitionNodeMap() {
         final Set mergeNodeKeyChangingParentsToRemove = new HashSet<>();
         for (final StreamsGraphNode mergeNode : mergeNodes) {
             mergeNodesToKeyChangers.put(mergeNode, new LinkedHashSet<>());
-            final Collection keys = keyChangingOperationsToOptimizableRepartitionNodes.keySet();
-            for (final StreamsGraphNode key : keys) {
-                final StreamsGraphNode maybeParentKey = findParentNodeMatching(mergeNode, node -> node.parentNodes().contains(key));
-                if (maybeParentKey != null) {
-                    mergeNodesToKeyChangers.get(mergeNode).add(key);
+            final Set>> entrySet = keyChangingOperationsToOptimizableRepartitionNodes.entrySet();
+            for (final Map.Entry> entry : entrySet) {
+                if (mergeNodeHasRepartitionChildren(mergeNode, entry.getValue())) {
+                    final StreamsGraphNode maybeParentKey = findParentNodeMatching(mergeNode, node -> node.parentNodes().contains(entry.getKey()));
+                    if (maybeParentKey != null) {
+                        mergeNodesToKeyChangers.get(mergeNode).add(entry.getKey());
+                    }
                 }
             }
         }
@@ -421,6 +423,11 @@ private void maybeUpdateKeyChangingRepartitionNodeMap() {
         }
     }
 
+    private boolean mergeNodeHasRepartitionChildren(final StreamsGraphNode mergeNode,
+                                                    final LinkedHashSet repartitionNodes) {
+        return repartitionNodes.stream().allMatch(n -> findParentNodeMatching(n, gn -> gn.parentNodes().contains(mergeNode)) != null);
+    }
+
     @SuppressWarnings("unchecked")
     private OptimizableRepartitionNode createRepartitionNode(final String repartitionTopicName,
                                                              final Serde keySerde,
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/graph/StreamsGraphTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/graph/StreamsGraphTest.java
index e2006e607e8c8..6a0a0b69ea3f4 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/graph/StreamsGraphTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/graph/StreamsGraphTest.java
@@ -22,14 +22,22 @@
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.Topology;
+import org.apache.kafka.streams.kstream.Aggregator;
 import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.Grouped;
+import org.apache.kafka.streams.kstream.Initializer;
 import org.apache.kafka.streams.kstream.JoinWindows;
+import org.apache.kafka.streams.kstream.Joined;
 import org.apache.kafka.streams.kstream.KStream;
+import org.apache.kafka.streams.kstream.KTable;
 import org.apache.kafka.streams.kstream.Materialized;
 import org.apache.kafka.streams.kstream.Produced;
+import org.apache.kafka.streams.kstream.Suppressed;
 import org.apache.kafka.streams.kstream.TimeWindows;
+import org.apache.kafka.streams.kstream.Transformer;
+import org.apache.kafka.streams.kstream.TransformerSupplier;
 import org.apache.kafka.streams.kstream.ValueJoiner;
+import org.apache.kafka.streams.processor.ProcessorContext;
 import org.junit.Test;
 
 import java.time.Duration;
@@ -47,6 +55,8 @@
 public class StreamsGraphTest {
 
     private final Pattern repartitionTopicPattern = Pattern.compile("Sink: .*-repartition");
+    private Initializer initializer;
+    private Aggregator aggregator;
 
     // Test builds topology in succesive manner but only graph node not yet processed written to topology
 
@@ -101,6 +111,76 @@ public void shouldBeAbleToProcessNestedMultipleKeyChangingNodes() {
         builder.build(properties);
     }
 
+    @Test
+    @SuppressWarnings("unchecked")
+    public void shouldNotThrowNPEWithMergeNodes() {
+        final Properties properties = new Properties();
+        properties.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "test-application");
+        properties.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
+        properties.setProperty(StreamsConfig.TOPOLOGY_OPTIMIZATION, StreamsConfig.OPTIMIZE);
+
+        final StreamsBuilder builder = new StreamsBuilder();
+        initializer = () -> "";
+        aggregator = (aggKey, value, aggregate) -> aggregate + value.length();
+        final TransformerSupplier> transformSupplier = () -> new Transformer>() {
+            @Override
+            public void init(final ProcessorContext context) {
+
+            }
+
+            @Override
+            public KeyValue transform(final String key, final String value) {
+                return KeyValue.pair(key, value);
+            }
+
+            @Override
+            public void close() {
+
+            }
+        };
+
+        final KStream retryStream = builder.stream("retryTopic", Consumed.with(Serdes.String(), Serdes.String()))
+                .transform(transformSupplier)
+                .groupByKey(Grouped.with(Serdes.String(), Serdes.String()))
+                .aggregate(initializer,
+                        aggregator,
+                        Materialized.with(Serdes.String(), Serdes.String()))
+                .suppress(Suppressed.untilTimeLimit(Duration.ofSeconds(500), Suppressed.BufferConfig.maxBytes(64_000_000)))
+                .toStream()
+                .flatMap((k, v) -> new ArrayList<>());
+
+        final KTable idTable = builder.stream("id-table-topic", Consumed.with(Serdes.String(), Serdes.String()))
+                .flatMap((k, v) -> new ArrayList>())
+                .peek((subscriptionId, recipientId) -> System.out.println("data " + subscriptionId + " " + recipientId))
+                .groupByKey(Grouped.with(Serdes.String(), Serdes.String()))
+                .aggregate(initializer,
+                        aggregator,
+                        Materialized.with(Serdes.String(), Serdes.String()));
+
+        final KStream joinStream = builder.stream("internal-topic-command", Consumed.with(Serdes.String(), Serdes.String()))
+                .peek((subscriptionId, command) -> System.out.println("stdoutput"))
+                .mapValues((k, v) -> v)
+                .merge(retryStream)
+                .leftJoin(idTable, (v1, v2) -> v1 + v2,
+                        Joined.with(Serdes.String(), Serdes.String(), Serdes.String()));
+
+        final KStream[] branches = joinStream.branch((k, v) -> v.equals("some-value"), (k, v) -> true);
+
+        branches[0].map(KeyValue::pair)
+                .peek((recipientId, command) -> System.out.println("printing out"))
+                .to("external-command", Produced.with(Serdes.String(), Serdes.String()));
+
+        branches[1].filter((k, v) -> v != null)
+                .peek((subscriptionId, wrapper) -> System.out.println("Printing output"))
+                .mapValues((k, v) -> v)
+                .to("dlq-topic", Produced.with(Serdes.String(), Serdes.String()));
+
+        branches[1].map(KeyValue::pair).to("retryTopic", Produced.with(Serdes.String(), Serdes.String()));
+
+        final Topology topology = builder.build(properties);
+        assertEquals(expectedComplexMergeOptimizeTopology, topology.describe().toString());
+    }
+
     @Test
     public void shouldNotOptimizeWithValueOrKeyChangingOperatorsAfterInitialKeyChange() {
 
@@ -291,4 +371,110 @@ private int getCountOfRepartitionTopicsFound(final String topologyString) {
         "    Sink: KSTREAM-SINK-0000000007 (topic: output_topic)\n" +
         "      <-- KSTREAM-MERGE-0000000006\n\n";
 
+
+    private final String expectedComplexMergeOptimizeTopology = "Topologies:\n" +
+            "   Sub-topology: 0\n" +
+            "    Source: KSTREAM-SOURCE-0000000000 (topics: [retryTopic])\n" +
+            "      --> KSTREAM-TRANSFORM-0000000001\n" +
+            "    Processor: KSTREAM-TRANSFORM-0000000001 (stores: [])\n" +
+            "      --> KSTREAM-FILTER-0000000040\n" +
+            "      <-- KSTREAM-SOURCE-0000000000\n" +
+            "    Processor: KSTREAM-FILTER-0000000040 (stores: [])\n" +
+            "      --> KSTREAM-SINK-0000000039\n" +
+            "      <-- KSTREAM-TRANSFORM-0000000001\n" +
+            "    Sink: KSTREAM-SINK-0000000039 (topic: KSTREAM-AGGREGATE-STATE-STORE-0000000002-repartition)\n" +
+            "      <-- KSTREAM-FILTER-0000000040\n" +
+            "\n" +
+            "  Sub-topology: 1\n" +
+            "    Source: KSTREAM-SOURCE-0000000041 (topics: [KSTREAM-AGGREGATE-STATE-STORE-0000000002-repartition])\n" +
+            "      --> KSTREAM-AGGREGATE-0000000003\n" +
+            "    Processor: KSTREAM-AGGREGATE-0000000003 (stores: [KSTREAM-AGGREGATE-STATE-STORE-0000000002])\n" +
+            "      --> KTABLE-SUPPRESS-0000000007\n" +
+            "      <-- KSTREAM-SOURCE-0000000041\n" +
+            "    Source: KSTREAM-SOURCE-0000000019 (topics: [internal-topic-command])\n" +
+            "      --> KSTREAM-PEEK-0000000020\n" +
+            "    Processor: KTABLE-SUPPRESS-0000000007 (stores: [KTABLE-SUPPRESS-STATE-STORE-0000000008])\n" +
+            "      --> KTABLE-TOSTREAM-0000000009\n" +
+            "      <-- KSTREAM-AGGREGATE-0000000003\n" +
+            "    Processor: KSTREAM-PEEK-0000000020 (stores: [])\n" +
+            "      --> KSTREAM-MAPVALUES-0000000021\n" +
+            "      <-- KSTREAM-SOURCE-0000000019\n" +
+            "    Processor: KTABLE-TOSTREAM-0000000009 (stores: [])\n" +
+            "      --> KSTREAM-FLATMAP-0000000010\n" +
+            "      <-- KTABLE-SUPPRESS-0000000007\n" +
+            "    Processor: KSTREAM-FLATMAP-0000000010 (stores: [])\n" +
+            "      --> KSTREAM-MERGE-0000000022\n" +
+            "      <-- KTABLE-TOSTREAM-0000000009\n" +
+            "    Processor: KSTREAM-MAPVALUES-0000000021 (stores: [])\n" +
+            "      --> KSTREAM-MERGE-0000000022\n" +
+            "      <-- KSTREAM-PEEK-0000000020\n" +
+            "    Processor: KSTREAM-MERGE-0000000022 (stores: [])\n" +
+            "      --> KSTREAM-FILTER-0000000024\n" +
+            "      <-- KSTREAM-MAPVALUES-0000000021, KSTREAM-FLATMAP-0000000010\n" +
+            "    Processor: KSTREAM-FILTER-0000000024 (stores: [])\n" +
+            "      --> KSTREAM-SINK-0000000023\n" +
+            "      <-- KSTREAM-MERGE-0000000022\n" +
+            "    Sink: KSTREAM-SINK-0000000023 (topic: KSTREAM-MERGE-0000000022-repartition)\n" +
+            "      <-- KSTREAM-FILTER-0000000024\n" +
+            "\n" +
+            "  Sub-topology: 2\n" +
+            "    Source: KSTREAM-SOURCE-0000000011 (topics: [id-table-topic])\n" +
+            "      --> KSTREAM-FLATMAP-0000000012\n" +
+            "    Processor: KSTREAM-FLATMAP-0000000012 (stores: [])\n" +
+            "      --> KSTREAM-FILTER-0000000043\n" +
+            "      <-- KSTREAM-SOURCE-0000000011\n" +
+            "    Processor: KSTREAM-FILTER-0000000043 (stores: [])\n" +
+            "      --> KSTREAM-SINK-0000000042\n" +
+            "      <-- KSTREAM-FLATMAP-0000000012\n" +
+            "    Sink: KSTREAM-SINK-0000000042 (topic: KSTREAM-AGGREGATE-STATE-STORE-0000000014-repartition)\n" +
+            "      <-- KSTREAM-FILTER-0000000043\n" +
+            "\n" +
+            "  Sub-topology: 3\n" +
+            "    Source: KSTREAM-SOURCE-0000000025 (topics: [KSTREAM-MERGE-0000000022-repartition])\n" +
+            "      --> KSTREAM-LEFTJOIN-0000000026\n" +
+            "    Processor: KSTREAM-LEFTJOIN-0000000026 (stores: [KSTREAM-AGGREGATE-STATE-STORE-0000000014])\n" +
+            "      --> KSTREAM-BRANCH-0000000027\n" +
+            "      <-- KSTREAM-SOURCE-0000000025\n" +
+            "    Processor: KSTREAM-BRANCH-0000000027 (stores: [])\n" +
+            "      --> KSTREAM-BRANCHCHILD-0000000029, KSTREAM-BRANCHCHILD-0000000028\n" +
+            "      <-- KSTREAM-LEFTJOIN-0000000026\n" +
+            "    Processor: KSTREAM-BRANCHCHILD-0000000029 (stores: [])\n" +
+            "      --> KSTREAM-FILTER-0000000033, KSTREAM-MAP-0000000037\n" +
+            "      <-- KSTREAM-BRANCH-0000000027\n" +
+            "    Processor: KSTREAM-BRANCHCHILD-0000000028 (stores: [])\n" +
+            "      --> KSTREAM-MAP-0000000030\n" +
+            "      <-- KSTREAM-BRANCH-0000000027\n" +
+            "    Processor: KSTREAM-FILTER-0000000033 (stores: [])\n" +
+            "      --> KSTREAM-PEEK-0000000034\n" +
+            "      <-- KSTREAM-BRANCHCHILD-0000000029\n" +
+            "    Processor: KSTREAM-MAP-0000000030 (stores: [])\n" +
+            "      --> KSTREAM-PEEK-0000000031\n" +
+            "      <-- KSTREAM-BRANCHCHILD-0000000028\n" +
+            "    Processor: KSTREAM-PEEK-0000000034 (stores: [])\n" +
+            "      --> KSTREAM-MAPVALUES-0000000035\n" +
+            "      <-- KSTREAM-FILTER-0000000033\n" +
+            "    Source: KSTREAM-SOURCE-0000000044 (topics: [KSTREAM-AGGREGATE-STATE-STORE-0000000014-repartition])\n" +
+            "      --> KSTREAM-PEEK-0000000013\n" +
+            "    Processor: KSTREAM-MAP-0000000037 (stores: [])\n" +
+            "      --> KSTREAM-SINK-0000000038\n" +
+            "      <-- KSTREAM-BRANCHCHILD-0000000029\n" +
+            "    Processor: KSTREAM-MAPVALUES-0000000035 (stores: [])\n" +
+            "      --> KSTREAM-SINK-0000000036\n" +
+            "      <-- KSTREAM-PEEK-0000000034\n" +
+            "    Processor: KSTREAM-PEEK-0000000013 (stores: [])\n" +
+            "      --> KSTREAM-AGGREGATE-0000000015\n" +
+            "      <-- KSTREAM-SOURCE-0000000044\n" +
+            "    Processor: KSTREAM-PEEK-0000000031 (stores: [])\n" +
+            "      --> KSTREAM-SINK-0000000032\n" +
+            "      <-- KSTREAM-MAP-0000000030\n" +
+            "    Processor: KSTREAM-AGGREGATE-0000000015 (stores: [KSTREAM-AGGREGATE-STATE-STORE-0000000014])\n" +
+            "      --> none\n" +
+            "      <-- KSTREAM-PEEK-0000000013\n" +
+            "    Sink: KSTREAM-SINK-0000000032 (topic: external-command)\n" +
+            "      <-- KSTREAM-PEEK-0000000031\n" +
+            "    Sink: KSTREAM-SINK-0000000036 (topic: dlq-topic)\n" +
+            "      <-- KSTREAM-MAPVALUES-0000000035\n" +
+            "    Sink: KSTREAM-SINK-0000000038 (topic: retryTopic)\n" +
+            "      <-- KSTREAM-MAP-0000000037\n\n";
+
 }

From 0fffb920469ac91a6e72a7320d98cf33ee7fc407 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Fri, 3 Apr 2020 11:51:04 -0700
Subject: [PATCH 0924/1071] KAFKA-9750; Fix race condition with log dir
 reassign completion (#8412)

There is a race on receiving a LeaderAndIsr request for a replica with an active log dir reassignment. If the reassignment completes just before the LeaderAndIsr handler updates epoch information, it can lead to an illegal state error since no future log dir exists. This patch fixes the problem by ensuring that the future log dir exists when the fetcher is started. Removal cannot happen concurrently because it requires access the same partition state lock.

Reviewers: Chia-Ping Tsai , David Arthur 

Co-authored-by: Chia-Ping Tsai 
---
 .../kafka/server/AbstractFetcherManager.scala |  14 +-
 .../kafka/server/AbstractFetcherThread.scala  |  10 +-
 .../server/ReplicaAlterLogDirsManager.scala   |  16 ++
 .../server/ReplicaAlterLogDirsThread.scala    |  22 +-
 .../scala/kafka/server/ReplicaManager.scala   |   4 +
 .../server/AbstractFetcherManagerTest.scala   |   2 +
 .../ReplicaAlterLogDirsThreadTest.scala       | 234 +++++++++++++++++-
 .../kafka/server/ReplicaManagerTest.scala     |  46 ++--
 8 files changed, 307 insertions(+), 41 deletions(-)

diff --git a/core/src/main/scala/kafka/server/AbstractFetcherManager.scala b/core/src/main/scala/kafka/server/AbstractFetcherManager.scala
index 49451def01025..e326c4b27e690 100755
--- a/core/src/main/scala/kafka/server/AbstractFetcherManager.scala
+++ b/core/src/main/scala/kafka/server/AbstractFetcherManager.scala
@@ -146,7 +146,8 @@ abstract class AbstractFetcherManager[T <: AbstractFetcherThread](val name: Stri
         BrokerAndFetcherId(brokerAndInitialFetchOffset.leader, getFetcherId(topicPartition))
       }
 
-      def addAndStartFetcherThread(brokerAndFetcherId: BrokerAndFetcherId, brokerIdAndFetcherId: BrokerIdAndFetcherId): AbstractFetcherThread = {
+      def addAndStartFetcherThread(brokerAndFetcherId: BrokerAndFetcherId,
+                                   brokerIdAndFetcherId: BrokerIdAndFetcherId): T = {
         val fetcherThread = createFetcherThread(brokerAndFetcherId.fetcherId, brokerAndFetcherId.broker)
         fetcherThreadMap.put(brokerIdAndFetcherId, fetcherThread)
         fetcherThread.start()
@@ -170,14 +171,17 @@ abstract class AbstractFetcherManager[T <: AbstractFetcherThread](val name: Stri
           tp -> OffsetAndEpoch(brokerAndInitOffset.initOffset, brokerAndInitOffset.currentLeaderEpoch)
         }
 
-        fetcherThread.addPartitions(initialOffsetAndEpochs)
-        info(s"Added fetcher to broker ${brokerAndFetcherId.broker} for partitions $initialOffsetAndEpochs")
-
-        failedPartitions.removeAll(partitionAndOffsets.keySet)
+        addPartitionsToFetcherThread(fetcherThread, initialOffsetAndEpochs)
       }
     }
   }
 
+  protected def addPartitionsToFetcherThread(fetcherThread: T,
+                                             initialOffsetAndEpochs: collection.Map[TopicPartition, OffsetAndEpoch]): Unit = {
+    fetcherThread.addPartitions(initialOffsetAndEpochs)
+    info(s"Added fetcher to broker ${fetcherThread.sourceBroker.id} for partitions $initialOffsetAndEpochs")
+  }
+
   def removeFetcherForPartitions(partitions: Set[TopicPartition]): Unit = {
     lock synchronized {
       for (fetcher <- fetcherThreadMap.values)
diff --git a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
index 6061f2603a860..a58ad99d8a642 100755
--- a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
+++ b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
@@ -61,7 +61,7 @@ abstract class AbstractFetcherThread(name: String,
   type EpochData = OffsetsForLeaderEpochRequest.PartitionData
 
   private val partitionStates = new PartitionStates[PartitionFetchState]
-  private val partitionMapLock = new ReentrantLock
+  protected val partitionMapLock = new ReentrantLock
   private val partitionMapCond = partitionMapLock.newCondition()
 
   private val metricId = ClientIdAndBroker(clientId, sourceBroker.host, sourceBroker.port)
@@ -200,7 +200,7 @@ abstract class AbstractFetcherThread(name: String,
     * - Send OffsetsForLeaderEpochRequest, retrieving the latest offset for each partition's
     *   leader epoch. This is the offset the follower should truncate to ensure
     *   accurate log replication.
-    * - Finally truncate the logs for partitions in the truncating phase and mark them
+    * - Finally truncate the logs for partitions in the truncating phase and mark the
     *   truncation complete. Do this within a lock to ensure no leadership changes can
     *   occur during truncation.
     */
@@ -423,10 +423,11 @@ abstract class AbstractFetcherThread(name: String,
     warn(s"Partition $topicPartition marked as failed")
   }
 
-
-  def addPartitions(initialFetchStates: Map[TopicPartition, OffsetAndEpoch]): Unit = {
+  def addPartitions(initialFetchStates: Map[TopicPartition, OffsetAndEpoch]): Set[TopicPartition] = {
     partitionMapLock.lockInterruptibly()
     try {
+      failedPartitions.removeAll(initialFetchStates.keySet)
+
       initialFetchStates.foreach { case (tp, initialFetchState) =>
         // We can skip the truncation step iff the leader epoch matches the existing epoch
         val currentState = partitionStates.stateValue(tp)
@@ -443,6 +444,7 @@ abstract class AbstractFetcherThread(name: String,
       }
 
       partitionMapCond.signalAll()
+      initialFetchStates.keySet
     } finally partitionMapLock.unlock()
   }
 
diff --git a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsManager.scala b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsManager.scala
index 5b6c5b18da20f..85eaeaa89caf3 100644
--- a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsManager.scala
@@ -18,6 +18,7 @@
 package kafka.server
 
 import kafka.cluster.BrokerEndPoint
+import org.apache.kafka.common.TopicPartition
 
 class ReplicaAlterLogDirsManager(brokerConfig: KafkaConfig,
                                  replicaManager: ReplicaManager,
@@ -34,6 +35,21 @@ class ReplicaAlterLogDirsManager(brokerConfig: KafkaConfig,
       quotaManager, brokerTopicStats)
   }
 
+  override protected def addPartitionsToFetcherThread(fetcherThread: ReplicaAlterLogDirsThread,
+                                                      initialOffsetAndEpochs: collection.Map[TopicPartition, OffsetAndEpoch]): Unit = {
+    val addedPartitions = fetcherThread.addPartitions(initialOffsetAndEpochs)
+    val (addedInitialOffsets, notAddedInitialOffsets) = initialOffsetAndEpochs.partition { case (tp, _) =>
+      addedPartitions.contains(tp)
+    }
+
+    if (addedInitialOffsets.nonEmpty)
+      info(s"Added log dir fetcher for partitions with initial offsets $addedInitialOffsets")
+
+    if (notAddedInitialOffsets.nonEmpty)
+      info(s"Failed to add log dir fetch for partitions ${notAddedInitialOffsets.keySet} " +
+        s"since the log dir reassignment has already completed")
+  }
+
   def shutdown(): Unit = {
     info("shutting down")
     closeAllFetchers()
diff --git a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
index fdb2bfd6c0365..d0dc9efb969bd 100644
--- a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
+++ b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
@@ -85,7 +85,7 @@ class ReplicaAlterLogDirsThread(name: String,
       Request.FutureLocalReplicaId,
       request.minBytes,
       request.maxBytes,
-      request.version <= 2,
+      false,
       request.fetchData.asScala.toSeq,
       UnboundedQuota,
       processResponseCallback,
@@ -110,7 +110,11 @@ class ReplicaAlterLogDirsThread(name: String,
       throw new IllegalStateException("Offset mismatch for the future replica %s: fetched offset = %d, log end offset = %d.".format(
         topicPartition, fetchOffset, futureLog.logEndOffset))
 
-    val logAppendInfo = partition.appendRecordsToFollowerOrFutureReplica(records, isFuture = true)
+    val logAppendInfo = if (records.sizeInBytes() > 0)
+      partition.appendRecordsToFollowerOrFutureReplica(records, isFuture = true)
+    else
+      None
+
     futureLog.updateHighWatermark(partitionData.highWatermark)
     futureLog.maybeIncrementLogStartOffset(partitionData.logStartOffset)
 
@@ -121,6 +125,20 @@ class ReplicaAlterLogDirsThread(name: String,
     logAppendInfo
   }
 
+  override def addPartitions(initialFetchStates: Map[TopicPartition, OffsetAndEpoch]): Set[TopicPartition] = {
+    partitionMapLock.lockInterruptibly()
+    try {
+      // It is possible that the log dir fetcher completed just before this call, so we
+      // filter only the partitions which still have a future log dir.
+      val filteredFetchStates = initialFetchStates.filter { case (tp, _) =>
+        replicaMgr.futureLogExists(tp)
+      }
+      super.addPartitions(filteredFetchStates)
+    } finally {
+      partitionMapLock.unlock()
+    }
+  }
+
   override protected def fetchEarliestOffsetFromLeader(topicPartition: TopicPartition, leaderEpoch: Int): Long = {
     val partition = replicaMgr.getPartitionOrException(topicPartition, expectLeader = false)
     partition.localLogOrException.logStartOffset
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index ca90c30b9a7a9..c8380d937954e 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -490,6 +490,10 @@ class ReplicaManager(val config: KafkaConfig,
     getPartitionOrException(topicPartition, expectLeader = false).futureLocalLogOrException
   }
 
+  def futureLogExists(topicPartition: TopicPartition): Boolean = {
+    getPartitionOrException(topicPartition, expectLeader = false).futureLog.isDefined
+  }
+
   def localLog(topicPartition: TopicPartition): Option[Log] = {
     nonOfflinePartition(topicPartition).flatMap(_.log)
   }
diff --git a/core/src/test/scala/unit/kafka/server/AbstractFetcherManagerTest.scala b/core/src/test/scala/unit/kafka/server/AbstractFetcherManagerTest.scala
index d197845d4da55..f940b836ba9b7 100644
--- a/core/src/test/scala/unit/kafka/server/AbstractFetcherManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/AbstractFetcherManagerTest.scala
@@ -58,6 +58,7 @@ class AbstractFetcherManagerTest {
 
     EasyMock.expect(fetcher.start())
     EasyMock.expect(fetcher.addPartitions(Map(tp -> OffsetAndEpoch(fetchOffset, leaderEpoch))))
+        .andReturn(Set(tp))
     EasyMock.expect(fetcher.fetchState(tp))
       .andReturn(Some(PartitionFetchState(fetchOffset, leaderEpoch, Truncating)))
     EasyMock.expect(fetcher.removePartitions(Set(tp)))
@@ -116,6 +117,7 @@ class AbstractFetcherManagerTest {
 
     EasyMock.expect(fetcher.start())
     EasyMock.expect(fetcher.addPartitions(Map(tp -> OffsetAndEpoch(fetchOffset, leaderEpoch))))
+        .andReturn(Set(tp))
     EasyMock.expect(fetcher.isThreadFailed).andReturn(true)
     EasyMock.replay(fetcher)
 
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala
index 79457c0ee825c..1faaefee3b100 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala
@@ -18,22 +18,27 @@ package kafka.server
 
 import java.util.Optional
 
+import kafka.api.Request
 import kafka.cluster.{BrokerEndPoint, Partition}
 import kafka.log.{Log, LogManager}
 import kafka.server.AbstractFetcherThread.ResultWithPartitions
+import kafka.server.QuotaFactory.UnboundedQuota
 import kafka.utils.{DelayedItem, TestUtils}
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.KafkaStorageException
 import org.apache.kafka.common.protocol.Errors
-import org.apache.kafka.common.requests.{EpochEndOffset, OffsetsForLeaderEpochRequest}
+import org.apache.kafka.common.record.MemoryRecords
 import org.apache.kafka.common.requests.EpochEndOffset.{UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET}
+import org.apache.kafka.common.requests.{EpochEndOffset, FetchRequest, IsolationLevel, OffsetsForLeaderEpochRequest}
 import org.easymock.EasyMock._
 import org.easymock.{Capture, CaptureType, EasyMock, IAnswer, IExpectationSetters}
 import org.junit.Assert._
 import org.junit.Test
+import org.mockito.Mockito.{doNothing, when}
+import org.mockito.{ArgumentCaptor, ArgumentMatchers, Mockito}
 
-import scala.collection.JavaConverters._
 import scala.collection.{Map, Seq}
+import scala.jdk.CollectionConverters._
 
 class ReplicaAlterLogDirsThreadTest {
 
@@ -45,6 +50,204 @@ class ReplicaAlterLogDirsThreadTest {
     OffsetAndEpoch(offset = fetchOffset, leaderEpoch = leaderEpoch)
   }
 
+  @Test
+  def shouldNotAddPartitionIfFutureLogIsNotDefined(): Unit = {
+    val brokerId = 1
+    val config = KafkaConfig.fromProps(TestUtils.createBrokerConfig(brokerId, "localhost:1234"))
+
+    val replicaManager = Mockito.mock(classOf[ReplicaManager])
+    val quotaManager = Mockito.mock(classOf[ReplicationQuotaManager])
+
+    when(replicaManager.futureLogExists(t1p0)).thenReturn(false)
+
+    val endPoint = new BrokerEndPoint(0, "localhost", 1000)
+    val thread = new ReplicaAlterLogDirsThread(
+      "alter-logs-dirs-thread",
+      sourceBroker = endPoint,
+      brokerConfig = config,
+      failedPartitions = failedPartitions,
+      replicaMgr = replicaManager,
+      quota = quotaManager,
+      brokerTopicStats = new BrokerTopicStats)
+
+    val addedPartitions = thread.addPartitions(Map(t1p0 -> offsetAndEpoch(0L)))
+    assertEquals(Set.empty, addedPartitions)
+    assertEquals(0, thread.partitionCount())
+    assertEquals(None, thread.fetchState(t1p0))
+  }
+
+  @Test
+  def shouldUpdateLeaderEpochAfterFencedEpochError(): Unit = {
+    val brokerId = 1
+    val config = KafkaConfig.fromProps(TestUtils.createBrokerConfig(brokerId, "localhost:1234"))
+
+    val partition = Mockito.mock(classOf[Partition])
+    val replicaManager = Mockito.mock(classOf[ReplicaManager])
+    val quotaManager = Mockito.mock(classOf[ReplicationQuotaManager])
+    val futureLog = Mockito.mock(classOf[Log])
+
+    val leaderEpoch = 5
+    val logEndOffset = 0
+
+    when(replicaManager.futureLocalLogOrException(t1p0)).thenReturn(futureLog)
+    when(replicaManager.futureLogExists(t1p0)).thenReturn(true)
+    when(replicaManager.nonOfflinePartition(t1p0)).thenReturn(Some(partition))
+    when(replicaManager.getPartitionOrException(t1p0, expectLeader = false)).thenReturn(partition)
+
+    when(quotaManager.isQuotaExceeded).thenReturn(false)
+
+    when(partition.lastOffsetForLeaderEpoch(Optional.empty(), leaderEpoch, fetchOnlyFromLeader = false))
+      .thenReturn(new EpochEndOffset(leaderEpoch, logEndOffset))
+    when(partition.futureLocalLogOrException).thenReturn(futureLog)
+    doNothing().when(partition).truncateTo(offset = 0, isFuture = true)
+    when(partition.maybeReplaceCurrentWithFutureReplica()).thenReturn(true)
+
+    when(futureLog.logStartOffset).thenReturn(0L)
+    when(futureLog.logEndOffset).thenReturn(0L)
+    when(futureLog.latestEpoch).thenReturn(None)
+
+    val fencedRequestData = new FetchRequest.PartitionData(0L, 0L,
+      config.replicaFetchMaxBytes, Optional.of(leaderEpoch - 1))
+    val fencedResponseData = FetchPartitionData(
+      error = Errors.FENCED_LEADER_EPOCH,
+      highWatermark = -1,
+      logStartOffset = -1,
+      records = MemoryRecords.EMPTY,
+      lastStableOffset = None,
+      abortedTransactions = None,
+      preferredReadReplica = None)
+    mockFetchFromCurrentLog(t1p0, fencedRequestData, config, replicaManager, fencedResponseData)
+
+    val endPoint = new BrokerEndPoint(0, "localhost", 1000)
+    val thread = new ReplicaAlterLogDirsThread(
+      "alter-logs-dirs-thread",
+      sourceBroker = endPoint,
+      brokerConfig = config,
+      failedPartitions = failedPartitions,
+      replicaMgr = replicaManager,
+      quota = quotaManager,
+      brokerTopicStats = new BrokerTopicStats)
+
+    // Initially we add the partition with an older epoch which results in an error
+    thread.addPartitions(Map(t1p0 -> offsetAndEpoch(fetchOffset = 0L, leaderEpoch - 1)))
+    assertTrue(thread.fetchState(t1p0).isDefined)
+    assertEquals(1, thread.partitionCount())
+
+    thread.doWork()
+
+    assertTrue(failedPartitions.contains(t1p0))
+    assertEquals(None, thread.fetchState(t1p0))
+    assertEquals(0, thread.partitionCount())
+
+    // Next we update the epoch and assert that we can continue
+    thread.addPartitions(Map(t1p0 -> offsetAndEpoch(fetchOffset = 0L, leaderEpoch)))
+    assertEquals(Some(leaderEpoch), thread.fetchState(t1p0).map(_.currentLeaderEpoch))
+    assertEquals(1, thread.partitionCount())
+
+    val requestData = new FetchRequest.PartitionData(0L, 0L,
+      config.replicaFetchMaxBytes, Optional.of(leaderEpoch))
+    val responseData = FetchPartitionData(
+      error = Errors.NONE,
+      highWatermark = 0L,
+      logStartOffset = 0L,
+      records = MemoryRecords.EMPTY,
+      lastStableOffset = None,
+      abortedTransactions = None,
+      preferredReadReplica = None)
+    mockFetchFromCurrentLog(t1p0, requestData, config, replicaManager, responseData)
+
+    thread.doWork()
+
+    assertFalse(failedPartitions.contains(t1p0))
+    assertEquals(None, thread.fetchState(t1p0))
+    assertEquals(0, thread.partitionCount())
+  }
+
+  @Test
+  def shouldReplaceCurrentLogDirWhenCaughtUp(): Unit = {
+    val brokerId = 1
+    val config = KafkaConfig.fromProps(TestUtils.createBrokerConfig(brokerId, "localhost:1234"))
+
+    val partition = Mockito.mock(classOf[Partition])
+    val replicaManager = Mockito.mock(classOf[ReplicaManager])
+    val quotaManager = Mockito.mock(classOf[ReplicationQuotaManager])
+    val futureLog = Mockito.mock(classOf[Log])
+
+    val leaderEpoch = 5
+    val logEndOffset = 0
+
+    when(replicaManager.futureLocalLogOrException(t1p0)).thenReturn(futureLog)
+    when(replicaManager.futureLogExists(t1p0)).thenReturn(true)
+    when(replicaManager.nonOfflinePartition(t1p0)).thenReturn(Some(partition))
+    when(replicaManager.getPartitionOrException(t1p0, expectLeader = false)).thenReturn(partition)
+
+    when(quotaManager.isQuotaExceeded).thenReturn(false)
+
+    when(partition.lastOffsetForLeaderEpoch(Optional.empty(), leaderEpoch, fetchOnlyFromLeader = false))
+      .thenReturn(new EpochEndOffset(leaderEpoch, logEndOffset))
+    when(partition.futureLocalLogOrException).thenReturn(futureLog)
+    doNothing().when(partition).truncateTo(offset = 0, isFuture = true)
+    when(partition.maybeReplaceCurrentWithFutureReplica()).thenReturn(true)
+
+    when(futureLog.logStartOffset).thenReturn(0L)
+    when(futureLog.logEndOffset).thenReturn(0L)
+    when(futureLog.latestEpoch).thenReturn(None)
+
+    val requestData = new FetchRequest.PartitionData(0L, 0L,
+      config.replicaFetchMaxBytes, Optional.of(leaderEpoch))
+    val responseData = FetchPartitionData(
+      error = Errors.NONE,
+      highWatermark = 0L,
+      logStartOffset = 0L,
+      records = MemoryRecords.EMPTY,
+      lastStableOffset = None,
+      abortedTransactions = None,
+      preferredReadReplica = None)
+    mockFetchFromCurrentLog(t1p0, requestData, config, replicaManager, responseData)
+
+    val endPoint = new BrokerEndPoint(0, "localhost", 1000)
+    val thread = new ReplicaAlterLogDirsThread(
+      "alter-logs-dirs-thread",
+      sourceBroker = endPoint,
+      brokerConfig = config,
+      failedPartitions = failedPartitions,
+      replicaMgr = replicaManager,
+      quota = quotaManager,
+      brokerTopicStats = new BrokerTopicStats)
+
+    thread.addPartitions(Map(t1p0 -> offsetAndEpoch(fetchOffset = 0L, leaderEpoch)))
+    assertTrue(thread.fetchState(t1p0).isDefined)
+    assertEquals(1, thread.partitionCount())
+
+    thread.doWork()
+
+    assertEquals(None, thread.fetchState(t1p0))
+    assertEquals(0, thread.partitionCount())
+  }
+
+  private def mockFetchFromCurrentLog(topicPartition: TopicPartition,
+                                      requestData: FetchRequest.PartitionData,
+                                      config: KafkaConfig,
+                                      replicaManager: ReplicaManager,
+                                      responseData: FetchPartitionData): Unit = {
+    val callbackCaptor: ArgumentCaptor[Seq[(TopicPartition, FetchPartitionData)] => Unit] =
+      ArgumentCaptor.forClass(classOf[Seq[(TopicPartition, FetchPartitionData)] => Unit])
+    when(replicaManager.fetchMessages(
+      timeout = ArgumentMatchers.eq(0L),
+      replicaId = ArgumentMatchers.eq(Request.FutureLocalReplicaId),
+      fetchMinBytes = ArgumentMatchers.eq(0),
+      fetchMaxBytes = ArgumentMatchers.eq(config.replicaFetchResponseMaxBytes),
+      hardMaxBytesLimit = ArgumentMatchers.eq(false),
+      fetchInfos = ArgumentMatchers.eq(Seq(topicPartition -> requestData)),
+      quota = ArgumentMatchers.eq(UnboundedQuota),
+      responseCallback = callbackCaptor.capture(),
+      isolationLevel = ArgumentMatchers.eq(IsolationLevel.READ_UNCOMMITTED),
+      clientMetadata = ArgumentMatchers.eq(None)
+    )).thenAnswer(_ => {
+      callbackCaptor.getValue.apply(Seq((topicPartition, responseData)))
+    })
+  }
+
   @Test
   def issuesEpochRequestFromLocalReplica(): Unit = {
     val config = KafkaConfig.fromProps(TestUtils.createBrokerConfig(1, "localhost:1234"))
@@ -79,7 +282,7 @@ class ReplicaAlterLogDirsThreadTest {
       "alter-logs-dirs-thread-test1",
       sourceBroker = endPoint,
       brokerConfig = config,
-      failedPartitions : FailedPartitions,
+      failedPartitions = failedPartitions,
       replicaMgr = replicaManager,
       quota = null,
       brokerTopicStats = null)
@@ -125,7 +328,7 @@ class ReplicaAlterLogDirsThreadTest {
       "alter-logs-dirs-thread-test1",
       sourceBroker = endPoint,
       brokerConfig = config,
-      failedPartitions: FailedPartitions,
+      failedPartitions = failedPartitions,
       replicaMgr = replicaManager,
       quota = null,
       brokerTopicStats = null)
@@ -174,7 +377,9 @@ class ReplicaAlterLogDirsThreadTest {
     expect(replicaManager.getPartitionOrException(t1p1, expectLeader = false))
       .andStubReturn(partitionT1p1)
     expect(replicaManager.futureLocalLogOrException(t1p0)).andStubReturn(futureLogT1p0)
+    expect(replicaManager.futureLogExists(t1p0)).andStubReturn(true)
     expect(replicaManager.futureLocalLogOrException(t1p1)).andStubReturn(futureLogT1p1)
+    expect(replicaManager.futureLogExists(t1p1)).andStubReturn(true)
     expect(partitionT1p0.truncateTo(capture(truncateCaptureT1p0), anyBoolean())).anyTimes()
     expect(partitionT1p1.truncateTo(capture(truncateCaptureT1p1), anyBoolean())).anyTimes()
 
@@ -206,7 +411,7 @@ class ReplicaAlterLogDirsThreadTest {
       "alter-logs-dirs-thread-test1",
       sourceBroker = endPoint,
       brokerConfig = config,
-      failedPartitions: FailedPartitions,
+      failedPartitions = failedPartitions,
       replicaMgr = replicaManager,
       quota = quotaManager,
       brokerTopicStats = null)
@@ -247,6 +452,7 @@ class ReplicaAlterLogDirsThreadTest {
     expect(replicaManager.getPartitionOrException(t1p0, expectLeader = false))
       .andStubReturn(partition)
     expect(replicaManager.futureLocalLogOrException(t1p0)).andStubReturn(futureLog)
+    expect(replicaManager.futureLogExists(t1p0)).andStubReturn(true)
 
     expect(partition.truncateTo(capture(truncateToCapture), EasyMock.eq(true))).anyTimes()
     expect(futureLog.logEndOffset).andReturn(futureReplicaLEO).anyTimes()
@@ -278,7 +484,7 @@ class ReplicaAlterLogDirsThreadTest {
       "alter-logs-dirs-thread-test1",
       sourceBroker = endPoint,
       brokerConfig = config,
-      failedPartitions : FailedPartitions,
+      failedPartitions = failedPartitions,
       replicaMgr = replicaManager,
       quota = quotaManager,
       brokerTopicStats = null)
@@ -317,6 +523,7 @@ class ReplicaAlterLogDirsThreadTest {
       .andStubReturn(partition)
     expect(partition.truncateTo(capture(truncated), isFuture = EasyMock.eq(true))).anyTimes()
     expect(replicaManager.futureLocalLogOrException(t1p0)).andStubReturn(futureLog)
+    expect(replicaManager.futureLogExists(t1p0)).andStubReturn(true)
 
     expect(replicaManager.logManager).andReturn(logManager).anyTimes()
 
@@ -332,7 +539,7 @@ class ReplicaAlterLogDirsThreadTest {
       "alter-logs-dirs-thread-test1",
       sourceBroker = endPoint,
       brokerConfig = config,
-      failedPartitions: FailedPartitions,
+      failedPartitions = failedPartitions,
       replicaMgr = replicaManager,
       quota = quotaManager,
       brokerTopicStats = null)
@@ -372,12 +579,12 @@ class ReplicaAlterLogDirsThreadTest {
     expect(partition.truncateTo(capture(truncated), isFuture = EasyMock.eq(true))).once()
 
     expect(replicaManager.futureLocalLogOrException(t1p0)).andStubReturn(futureLog)
+    expect(replicaManager.futureLogExists(t1p0)).andStubReturn(true)
     expect(futureLog.logEndOffset).andReturn(futureReplicaLEO).anyTimes()
     expect(futureLog.latestEpoch).andStubReturn(Some(futureReplicaLeaderEpoch))
     expect(futureLog.endOffsetForEpoch(futureReplicaLeaderEpoch)).andReturn(
       Some(OffsetAndEpoch(futureReplicaLEO, futureReplicaLeaderEpoch)))
     expect(replicaManager.localLog(t1p0)).andReturn(Some(log)).anyTimes()
-    expect(replicaManager.futureLocalLogOrException(t1p0)).andReturn(futureLog).anyTimes()
 
     // this will cause fetchEpochsFromLeader return an error with undefined offset
     expect(partition.lastOffsetForLeaderEpoch(Optional.of(1), futureReplicaLeaderEpoch, fetchOnlyFromLeader = false))
@@ -411,7 +618,7 @@ class ReplicaAlterLogDirsThreadTest {
       "alter-logs-dirs-thread-test1",
       sourceBroker = endPoint,
       brokerConfig = config,
-      failedPartitions: FailedPartitions,
+      failedPartitions = failedPartitions,
       replicaMgr = replicaManager,
       quota = quotaManager,
       brokerTopicStats = null)
@@ -456,6 +663,7 @@ class ReplicaAlterLogDirsThreadTest {
     expect(partition.truncateTo(futureReplicaLEO, isFuture = true)).once()
 
     expect(replicaManager.futureLocalLogOrException(t1p0)).andStubReturn(futureLog)
+    expect(replicaManager.futureLogExists(t1p0)).andStubReturn(true)
     expect(futureLog.latestEpoch).andStubReturn(Some(leaderEpoch))
     expect(futureLog.logEndOffset).andStubReturn(futureReplicaLEO)
     expect(futureLog.endOffsetForEpoch(leaderEpoch)).andReturn(
@@ -471,7 +679,7 @@ class ReplicaAlterLogDirsThreadTest {
       "alter-logs-dirs-thread-test1",
       sourceBroker = endPoint,
       brokerConfig = config,
-      failedPartitions: FailedPartitions,
+      failedPartitions = failedPartitions,
       replicaMgr = replicaManager,
       quota = quotaManager,
       brokerTopicStats = null)
@@ -511,7 +719,7 @@ class ReplicaAlterLogDirsThreadTest {
       "alter-logs-dirs-thread-test1",
       sourceBroker = endPoint,
       brokerConfig = config,
-      failedPartitions: FailedPartitions,
+      failedPartitions = failedPartitions,
       replicaMgr = replicaManager,
       quota = quotaManager,
       brokerTopicStats = null)
@@ -562,7 +770,7 @@ class ReplicaAlterLogDirsThreadTest {
       "alter-logs-dirs-thread-test1",
       sourceBroker = endPoint,
       brokerConfig = config,
-      failedPartitions: FailedPartitions,
+      failedPartitions = failedPartitions,
       replicaMgr = replicaManager,
       quota = quotaManager,
       brokerTopicStats = null)
@@ -611,10 +819,12 @@ class ReplicaAlterLogDirsThreadTest {
     expect(replicaManager.localLog(t1p0)).andReturn(Some(logT1p0)).anyTimes()
     expect(replicaManager.localLogOrException(t1p0)).andReturn(logT1p0).anyTimes()
     expect(replicaManager.futureLocalLogOrException(t1p0)).andReturn(futureLog).anyTimes()
+    expect(replicaManager.futureLogExists(t1p0)).andStubReturn(true)
     expect(replicaManager.nonOfflinePartition(t1p0)).andReturn(Some(partition)).anyTimes()
     expect(replicaManager.localLog(t1p1)).andReturn(Some(logT1p1)).anyTimes()
     expect(replicaManager.localLogOrException(t1p1)).andReturn(logT1p1).anyTimes()
     expect(replicaManager.futureLocalLogOrException(t1p1)).andReturn(futureLog).anyTimes()
+    expect(replicaManager.futureLogExists(t1p1)).andStubReturn(true)
     expect(replicaManager.nonOfflinePartition(t1p1)).andReturn(Some(partition)).anyTimes()
   }
 
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index 17075f17db1c6..ffd3d3034b192 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -212,13 +212,19 @@ class ReplicaManagerTest {
 
   @Test
   def testFencedErrorCausedByBecomeLeader(): Unit = {
+    testFencedErrorCausedByBecomeLeader(0)
+    testFencedErrorCausedByBecomeLeader(1)
+    testFencedErrorCausedByBecomeLeader(10)
+  }
+
+  private[this] def testFencedErrorCausedByBecomeLeader(loopEpochChange: Int): Unit = {
     val replicaManager = setupReplicaManagerWithMockedPurgatories(new MockTimer)
     try {
       val brokerList = Seq[Integer](0, 1).asJava
       val topicPartition = new TopicPartition(topic, 0)
       replicaManager.createPartition(topicPartition)
         .createLogIfNotExists(0, isNew = false, isFutureReplica = false,
-        new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
+          new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
 
       def leaderAndIsrRequest(epoch: Int): LeaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
         Seq(new LeaderAndIsrPartitionState()
@@ -235,30 +241,34 @@ class ReplicaManagerTest {
 
       replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(0), (_, _) => ())
       val partition = replicaManager.getPartitionOrException(new TopicPartition(topic, 0), expectLeader = true)
-        .localLogOrException
-      assertEquals(1, replicaManager.logManager.liveLogDirs.filterNot(_ == partition.dir.getParentFile).size)
+      assertEquals(1, replicaManager.logManager.liveLogDirs.filterNot(_ == partition.log.get.dir.getParentFile).size)
 
+      val previousReplicaFolder = partition.log.get.dir.getParentFile
       // find the live and different folder
-      val newReplicaFolder = replicaManager.logManager.liveLogDirs.filterNot(_ == partition.dir.getParentFile).head
+      val newReplicaFolder = replicaManager.logManager.liveLogDirs.filterNot(_ == partition.log.get.dir.getParentFile).head
       assertEquals(0, replicaManager.replicaAlterLogDirsManager.fetcherThreadMap.size)
       replicaManager.alterReplicaLogDirs(Map(topicPartition -> newReplicaFolder.getAbsolutePath))
+      // make sure the future log is created
       replicaManager.futureLocalLogOrException(topicPartition)
       assertEquals(1, replicaManager.replicaAlterLogDirsManager.fetcherThreadMap.size)
-      // change the epoch from 0 to 1 in order to make fenced error
-      replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(1), (_, _) => ())
-      TestUtils.waitUntilTrue(() => replicaManager.replicaAlterLogDirsManager.fetcherThreadMap.values.forall(_.partitionCount() == 0),
-        s"the partition=$topicPartition should be removed from pending state")
-      // the partition is added to failedPartitions if fenced error happens
-      // if the thread is done before ReplicaManager#becomeLeaderOrFollower updates epoch,the fenced error does
-      // not happen and failedPartitions is empty.
-      if (replicaManager.replicaAlterLogDirsManager.failedPartitions.size != 0) {
+      (1 to loopEpochChange).foreach(epoch => replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(epoch), (_, _) => ()))
+      // wait for the ReplicaAlterLogDirsThread to complete
+      TestUtils.waitUntilTrue(() => {
         replicaManager.replicaAlterLogDirsManager.shutdownIdleFetcherThreads()
-        assertEquals(0, replicaManager.replicaAlterLogDirsManager.fetcherThreadMap.size)
-        // send request again
-        replicaManager.alterReplicaLogDirs(Map(topicPartition -> newReplicaFolder.getAbsolutePath))
-        // the future folder exists so it fails to invoke thread
-        assertEquals(1, replicaManager.replicaAlterLogDirsManager.fetcherThreadMap.size)
-      }
+        replicaManager.replicaAlterLogDirsManager.fetcherThreadMap.isEmpty
+      }, s"ReplicaAlterLogDirsThread should be gone")
+
+      // the fenced error should be recoverable
+      assertEquals(0, replicaManager.replicaAlterLogDirsManager.failedPartitions.size)
+      // the replica change is completed after retrying
+      assertTrue(partition.futureLog.isEmpty)
+      assertEquals(newReplicaFolder.getAbsolutePath, partition.log.get.dir.getParent)
+      // change the replica folder again
+      val response = replicaManager.alterReplicaLogDirs(Map(topicPartition -> previousReplicaFolder.getAbsolutePath))
+      assertNotEquals(0, response.size)
+      response.values.foreach(assertEquals(Errors.NONE, _))
+      // should succeed to invoke ReplicaAlterLogDirsThread again
+      assertEquals(1, replicaManager.replicaAlterLogDirsManager.fetcherThreadMap.size)
     } finally replicaManager.shutdown(checkpointHW = false)
   }
 

From e66b6aa895b5a9a9e53bd5c3cd8d794d32cf3786 Mon Sep 17 00:00:00 2001
From: Konstantine Karantasis 
Date: Fri, 3 Apr 2020 13:27:27 -0700
Subject: [PATCH 0925/1071] KAFKA-9810: Document Connect Root REST API on /
 (#8408)

Document the supported endpoint at the top-level (root) REST API resource and the information that it returns when a request is made to a Connect worker.

Fixes an omission in documentation after KAFKA-2369 and KAFKA-6311 (KIP-238)

Reviewers: Toby Drake , Soenke Liebau 
---
 docs/connect.html | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/connect.html b/docs/connect.html
index 30750b38bd957..f1a9d05f2e4d5 100644
--- a/docs/connect.html
+++ b/docs/connect.html
@@ -250,6 +250,12 @@ 
          REST API
          
         
        4. PUT /connector-plugins/{connector-type}/config/validate - validate the provided configuration values against the configuration definition. This API performs per config validation, returns suggested values and error messages during validation.
          5. 
     
 
+    
          The following is a supported REST request at the top-level (root) endpoint:
          
+
+    
            
+        
          • GET /- return basic information about the Kafka Connect cluster such as the version of the Connect worker that serves the REST request (including git commit ID of the source code) and the Kafka cluster ID that is connected to.
+    
          
+
     
          8.3 Connector Development Guide
          
 
     
          This guide describes how developers can write new connectors for Kafka Connect to move data between Kafka and other systems. It briefly reviews a few key concepts and then describes how to create a simple connector.
          

From 627d8e093cddf70b5e092dee44c250433abc3a96 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Fri, 3 Apr 2020 13:56:42 -0700
Subject: [PATCH 0926/1071] KAFKA-9807; Protect LSO reads from concurrent
 high-watermark updates (#8418)

If the high-watermark is updated in the middle of a read with the `read_committed` isolation level, it is possible to return data above the LSO. In the worst case, this can lead to the read of an aborted transaction. The root cause is that the logic depends on reading the high-watermark twice. We fix the problem by reading it once and caching the value.

Reviewers: David Arthur , Guozhang Wang , Ismael Juma 
---
 core/src/main/scala/kafka/log/Log.scala       |  7 ++-
 .../test/scala/unit/kafka/log/LogTest.scala   | 54 +++++++++++++++++++
 2 files changed, 59 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index 634f2163213da..98790d293955b 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -411,8 +411,11 @@ class Log(@volatile var dir: File,
   private def fetchLastStableOffsetMetadata: LogOffsetMetadata = {
     checkIfMemoryMappedBufferClosed()
 
+    // cache the current high watermark to avoid a concurrent update invalidating the range check
+    val highWatermarkMetadata = fetchHighWatermarkMetadata
+
     firstUnstableOffsetMetadata match {
-      case Some(offsetMetadata) if offsetMetadata.messageOffset < highWatermark =>
+      case Some(offsetMetadata) if offsetMetadata.messageOffset < highWatermarkMetadata.messageOffset =>
         if (offsetMetadata.messageOffsetOnly) {
           lock synchronized {
             val fullOffset = convertToOffsetMetadataOrThrow(offsetMetadata.messageOffset)
@@ -423,7 +426,7 @@ class Log(@volatile var dir: File,
         } else {
           offsetMetadata
         }
-      case _ => fetchHighWatermarkMetadata
+      case _ => highWatermarkMetadata
     }
   }
 
diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala
index 1e2e047ee6358..0831a85d17486 100755
--- a/core/src/test/scala/unit/kafka/log/LogTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogTest.scala
@@ -20,6 +20,7 @@ package kafka.log
 import java.io._
 import java.nio.ByteBuffer
 import java.nio.file.{Files, Paths}
+import java.util.concurrent.{Callable, Executors}
 import java.util.regex.Pattern
 import java.util.{Collections, Optional, Properties}
 
@@ -3642,6 +3643,59 @@ class LogTest {
     assertEquals(None, log.firstUnstableOffset)
   }
 
+  @Test
+  def testReadCommittedWithConcurrentHighWatermarkUpdates(): Unit = {
+    val logConfig = LogTest.createLogConfig(segmentBytes = 1024 * 1024 * 5)
+    val log = createLog(logDir, logConfig)
+    val lastOffset = 50L
+
+    val producerEpoch = 0.toShort
+    val producerId = 15L
+    val appendProducer = appendTransactionalAsLeader(log, producerId, producerEpoch)
+
+    // Thread 1 writes single-record transactions and attempts to read them
+    // before they have been aborted, and then aborts them
+    val txnWriteAndReadLoop: Callable[Int] = () => {
+      var nonEmptyReads = 0
+      while (log.logEndOffset < lastOffset) {
+        val currentLogEndOffset = log.logEndOffset
+
+        appendProducer(1)
+
+        val readInfo = log.read(
+          startOffset = currentLogEndOffset,
+          maxLength = Int.MaxValue,
+          isolation = FetchTxnCommitted,
+          minOneMessage = false)
+
+        if (readInfo.records.sizeInBytes() > 0)
+          nonEmptyReads += 1
+
+        appendEndTxnMarkerAsLeader(log, producerId, producerEpoch, ControlRecordType.ABORT)
+      }
+      nonEmptyReads
+    }
+
+    // Thread 2 watches the log and updates the high watermark
+    val hwUpdateLoop: Runnable = () => {
+      while (log.logEndOffset < lastOffset) {
+        log.updateHighWatermark(log.logEndOffset)
+      }
+    }
+
+    val executor = Executors.newFixedThreadPool(2)
+    try {
+      executor.submit(hwUpdateLoop)
+
+      val future = executor.submit(txnWriteAndReadLoop)
+      val nonEmptyReads = future.get()
+
+      assertEquals(0, nonEmptyReads)
+    } finally {
+      executor.shutdownNow()
+    }
+  }
+
   @Test
   def testTransactionIndexUpdated(): Unit = {
     val logConfig = LogTest.createLogConfig(segmentBytes = 1024 * 1024 * 5)

From d783a9f9f697c4df74abdc2a8ee18c533aa0716e Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Tue, 7 Apr 2020 01:00:11 +0100
Subject: [PATCH 0927/1071] KAFKA-9815; Ensure consumer always re-joins if
 JoinGroup fails (#8420)

On metadata change for assigned topics, we trigger rebalance, revoke partitions and send JoinGroup. If metadata reverts to the original value and JoinGroup fails, we don't resend JoinGroup because we don't set `rejoinNeeded`. This PR sets `rejoinNeeded=true` when rebalance is triggered due to metadata change to ensure that we retry on failure.

Reviewers: Boyang Chen , Chia-Ping Tsai , Jason Gustafson 
---
 .../internals/ConsumerCoordinator.java        |  5 +-
 .../internals/ConsumerCoordinatorTest.java    | 58 +++++++++++++++++++
 2 files changed, 62 insertions(+), 1 deletion(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index b4d9fa5bf283f..6b905c57774f5 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -727,11 +727,14 @@ public boolean rejoinNeededOrPending() {
 
         // we need to rejoin if we performed the assignment and metadata has changed;
         // also for those owned-but-no-longer-existed partitions we should drop them as lost
-        if (assignmentSnapshot != null && !assignmentSnapshot.matches(metadataSnapshot))
+        if (assignmentSnapshot != null && !assignmentSnapshot.matches(metadataSnapshot)) {
+            requestRejoin();
             return true;
+        }
 
         // we need to join if our subscription has changed since the last join
         if (joinedSubscription != null && !joinedSubscription.equals(subscriptions.subscription())) {
+            requestRejoin();
             return true;
         }
 
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index 9e6128badd4a2..afef951e3cc54 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -843,6 +843,64 @@ public void testForceMetadataRefreshForPatternSubscriptionDuringRebalance() {
         assertFalse(coordinator.rejoinNeededOrPending());
     }
 
+    /**
+     * Verifies that the consumer re-joins after a metadata change. If JoinGroup fails
+     * and metadata reverts to its original value, the consumer should still retry JoinGroup.
+     */
+    @Test
+    public void testRebalanceWithMetadataChange() {
+        final String consumerId = "leader";
+        final List topics = Arrays.asList(topic1, topic2);
+        final List partitions = Arrays.asList(t1p, t2p);
+        subscriptions.subscribe(toSet(topics), rebalanceListener);
+        client.updateMetadata(TestUtils.metadataUpdateWith(1,
+                Utils.mkMap(Utils.mkEntry(topic1, 1), Utils.mkEntry(topic2, 1))));
+        coordinator.maybeUpdateSubscriptionMetadata();
+
+        client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
+        coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
+
+        Map> initialSubscription = singletonMap(consumerId, topics);
+        partitionAssignor.prepare(singletonMap(consumerId, partitions));
+
+        client.prepareResponse(joinGroupLeaderResponse(1, consumerId, initialSubscription, Errors.NONE));
+        client.prepareResponse(syncGroupResponse(partitions, Errors.NONE));
+        coordinator.poll(time.timer(Long.MAX_VALUE));
+
+        // rejoin will only be set in the next poll call
+        assertFalse(coordinator.rejoinNeededOrPending());
+        assertEquals(toSet(topics), subscriptions.subscription());
+        assertEquals(toSet(partitions), subscriptions.assignedPartitions());
+        assertEquals(0, rebalanceListener.revokedCount);
+        assertNull(rebalanceListener.revoked);
+        assertEquals(1, rebalanceListener.assignedCount);
+
+        // Change metadata to trigger rebalance.
+        client.updateMetadata(TestUtils.metadataUpdateWith(1, singletonMap(topic1, 1)));
+        coordinator.poll(time.timer(0));
+
+        // Revert metadata to original value. Fail pending JoinGroup. Another
+        // JoinGroup should be sent, which will be completed successfully.
+        client.updateMetadata(TestUtils.metadataUpdateWith(1,
+                Utils.mkMap(Utils.mkEntry(topic1, 1), Utils.mkEntry(topic2, 1))));
+        client.respond(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NOT_COORDINATOR));
+        client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
+        coordinator.poll(time.timer(0));
+        assertTrue(coordinator.rejoinNeededOrPending());
+
+        client.respond(joinGroupLeaderResponse(2, consumerId, initialSubscription, Errors.NONE));
+        client.prepareResponse(syncGroupResponse(partitions, Errors.NONE));
+        coordinator.poll(time.timer(Long.MAX_VALUE));
+
+        assertFalse(coordinator.rejoinNeededOrPending());
+        Collection revoked = getRevoked(partitions, partitions);
+        assertEquals(revoked.isEmpty() ? 0 : 1, rebalanceListener.revokedCount);
+        assertEquals(revoked.isEmpty() ? null : revoked, rebalanceListener.revoked);
+        assertEquals(2, rebalanceListener.assignedCount);
+        assertEquals(getAdded(partitions, partitions), rebalanceListener.assigned);
+        assertEquals(toSet(partitions), subscriptions.assignedPartitions());
+    }
+
     @Test
     public void testWakeupDuringJoin() {
         final String consumerId = "leader";

From 5b97467f4562e7229cbdbb76aa3d9b9ed0ba1ba6 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Wed, 8 Apr 2020 11:31:27 -0700
Subject: [PATCH 0928/1071] KAFKA-9835; Protect `FileRecords.slice` from
 concurrent write (#8451)

A read from the end of the log interleaved with a concurrent write can result in reading data above the expected read limit. In particular, this would allow a read above the high watermark. The root of the problem is consecutive calls to `sizeInBytes` in `FileRecords.slice` which do not account for an increase in size due to a concurrent write. This patch fixes the problem by using a single call to `sizeInBytes` and caching the result.

Reviewers: Ismael Juma 
---
 .../kafka/common/record/FileRecords.java      |  9 +++--
 .../kafka/common/record/FileRecordsTest.java  | 33 +++++++++++++++++++
 2 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java b/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java
index 9b312d9e7f73b..4d770cbf845ab 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java
@@ -135,17 +135,20 @@ public void readInto(ByteBuffer buffer, int position) throws IOException {
      * @return A sliced wrapper on this message set limited based on the given position and size
      */
     public FileRecords slice(int position, int size) throws IOException {
+        // Cache current size in case concurrent write changes it
+        int currentSizeInBytes = sizeInBytes();
+
         if (position < 0)
             throw new IllegalArgumentException("Invalid position: " + position + " in read from " + this);
-        if (position > sizeInBytes() - start)
+        if (position > currentSizeInBytes - start)
             throw new IllegalArgumentException("Slice from position " + position + " exceeds end position of " + this);
         if (size < 0)
             throw new IllegalArgumentException("Invalid size: " + size + " in read from " + this);
 
         int end = this.start + position + size;
         // handle integer overflow or if end is beyond the end of the file
-        if (end < 0 || end >= start + sizeInBytes())
-            end = start + sizeInBytes();
+        if (end < 0 || end > start + currentSizeInBytes)
+            end = start + currentSizeInBytes;
         return new FileRecords(file, channel, this.start + position, end, true);
     }
 
diff --git a/clients/src/test/java/org/apache/kafka/common/record/FileRecordsTest.java b/clients/src/test/java/org/apache/kafka/common/record/FileRecordsTest.java
index bf799879e61a9..08dbc57d7f29d 100644
--- a/clients/src/test/java/org/apache/kafka/common/record/FileRecordsTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/record/FileRecordsTest.java
@@ -36,6 +36,9 @@
 import java.util.List;
 import java.util.Optional;
 import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 
 import static java.util.Arrays.asList;
 import static org.apache.kafka.common.utils.Utils.utf8;
@@ -118,6 +121,36 @@ public void testIterationOverPartialAndTruncation() throws IOException {
         testPartialWrite(6, fileRecords);
     }
 
+    @Test
+    public void testSliceSizeLimitWithConcurrentWrite() throws Exception {
+        FileRecords log = FileRecords.open(tempFile());
+        ExecutorService executor = Executors.newFixedThreadPool(2);
+        int maxSizeInBytes = 16384;
+
+        try {
+            Future readerCompletion = executor.submit(() -> {
+                while (log.sizeInBytes() < maxSizeInBytes) {
+                    int currentSize = log.sizeInBytes();
+                    FileRecords slice = log.slice(0, currentSize);
+                    assertEquals(currentSize, slice.sizeInBytes());
+                }
+                return null;
+            });
+
+            Future writerCompletion = executor.submit(() -> {
+                while (log.sizeInBytes() < maxSizeInBytes) {
+                    append(log, values);
+                }
+                return null;
+            });
+
+            writerCompletion.get();
+            readerCompletion.get();
+        } finally {
+            executor.shutdownNow();
+        }
+    }
+
     private void testPartialWrite(int size, FileRecords fileRecords) throws IOException {
         ByteBuffer buffer = ByteBuffer.allocate(size);
         for (int i = 0; i < size; i++)

From efc131ab14609b280771545de0d6fcd182aed1d4 Mon Sep 17 00:00:00 2001
From: Brian Bushree 
Date: Mon, 18 Nov 2019 10:50:36 -0800
Subject: [PATCH 0929/1071] [MINOR] allow additional JVM args in KafkaService
 (#7297)

Reviewers: Colin P. McCabe , Vikas Singh 
---
 tests/kafkatest/services/kafka/kafka.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/kafkatest/services/kafka/kafka.py b/tests/kafkatest/services/kafka/kafka.py
index e71ed7c678336..9afb62259620b 100644
--- a/tests/kafkatest/services/kafka/kafka.py
+++ b/tests/kafkatest/services/kafka/kafka.py
@@ -96,7 +96,7 @@ def __init__(self, context, num_nodes, zk, security_protocol=SecurityConfig.PLAI
                  client_sasl_mechanism=SecurityConfig.SASL_MECHANISM_GSSAPI, interbroker_sasl_mechanism=SecurityConfig.SASL_MECHANISM_GSSAPI,
                  authorizer_class_name=None, topics=None, version=DEV_BRANCH, jmx_object_names=None,
                  jmx_attributes=None, zk_connect_timeout=5000, zk_session_timeout=6000, server_prop_overides=None, zk_chroot=None,
-                 listener_security_config=ListenerSecurityConfig(), per_node_server_prop_overrides={}):
+                 listener_security_config=ListenerSecurityConfig(), per_node_server_prop_overrides={}, extra_kafka_opts=""):
         """
         :param context: test context
         :param ZookeeperService zk:
@@ -114,6 +114,8 @@ def __init__(self, context, num_nodes, zk, security_protocol=SecurityConfig.PLAI
         :param dict server_prop_overides: overrides for kafka.properties file
         :param zk_chroot:
         :param ListenerSecurityConfig listener_security_config: listener config to use
+        :param dict per_node_server_prop_overrides:
+        :param str extra_kafka_opts: jvm args to add to KAFKA_OPTS variable
         """
         Service.__init__(self, context, num_nodes)
         JmxMixin.__init__(self, num_nodes=num_nodes, jmx_object_names=jmx_object_names, jmx_attributes=(jmx_attributes or []),
@@ -138,6 +140,7 @@ def __init__(self, context, num_nodes, zk, security_protocol=SecurityConfig.PLAI
         self.log_level = "DEBUG"
         self.zk_chroot = zk_chroot
         self.listener_security_config = listener_security_config
+        self.extra_kafka_opts = extra_kafka_opts
 
         #
         # In a heavily loaded and not very fast machine, it is
@@ -325,8 +328,8 @@ def start_cmd(self, node):
         cmd += "export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % self.LOG4J_CONFIG
         heap_kafka_opts = "-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=%s" % \
                           self.logs["kafka_heap_dump_file"]["path"]
-        other_kafka_opts = self.security_config.kafka_opts.strip('\"')
-        cmd += "export KAFKA_OPTS=\"%s %s\"; " % (heap_kafka_opts, other_kafka_opts)
+        security_kafka_opts = self.security_config.kafka_opts.strip('\"')
+        cmd += "export KAFKA_OPTS=\"%s %s %s\"; " % (heap_kafka_opts, security_kafka_opts, extra_kafka_opts)
         cmd += "%s %s 1>> %s 2>> %s &" % \
                (self.path.script("kafka-server-start.sh", node),
                 KafkaService.CONFIG_FILE,

From 51b8f26e2c4629b6a737cbc84c2203f19647bf0b Mon Sep 17 00:00:00 2001
From: David Arthur 
Date: Tue, 19 Nov 2019 15:19:48 -0500
Subject: [PATCH 0930/1071] Fix missing reference in kafka.py (#7715)

Also fix a default value for a dictionary arg
---
 tests/kafkatest/services/kafka/kafka.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/kafkatest/services/kafka/kafka.py b/tests/kafkatest/services/kafka/kafka.py
index 9afb62259620b..b26afcfde189e 100644
--- a/tests/kafkatest/services/kafka/kafka.py
+++ b/tests/kafkatest/services/kafka/kafka.py
@@ -96,7 +96,7 @@ def __init__(self, context, num_nodes, zk, security_protocol=SecurityConfig.PLAI
                  client_sasl_mechanism=SecurityConfig.SASL_MECHANISM_GSSAPI, interbroker_sasl_mechanism=SecurityConfig.SASL_MECHANISM_GSSAPI,
                  authorizer_class_name=None, topics=None, version=DEV_BRANCH, jmx_object_names=None,
                  jmx_attributes=None, zk_connect_timeout=5000, zk_session_timeout=6000, server_prop_overides=None, zk_chroot=None,
-                 listener_security_config=ListenerSecurityConfig(), per_node_server_prop_overrides={}, extra_kafka_opts=""):
+                 listener_security_config=ListenerSecurityConfig(), per_node_server_prop_overrides=None, extra_kafka_opts=""):
         """
         :param context: test context
         :param ZookeeperService zk:
@@ -329,7 +329,7 @@ def start_cmd(self, node):
         heap_kafka_opts = "-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=%s" % \
                           self.logs["kafka_heap_dump_file"]["path"]
         security_kafka_opts = self.security_config.kafka_opts.strip('\"')
-        cmd += "export KAFKA_OPTS=\"%s %s %s\"; " % (heap_kafka_opts, security_kafka_opts, extra_kafka_opts)
+        cmd += "export KAFKA_OPTS=\"%s %s %s\"; " % (heap_kafka_opts, security_kafka_opts, self.extra_kafka_opts)
         cmd += "%s %s 1>> %s 2>> %s &" % \
                (self.path.script("kafka-server-start.sh", node),
                 KafkaService.CONFIG_FILE,

From ed628805710454b4bf38f8d70cfa4b090b7cfdc2 Mon Sep 17 00:00:00 2001
From: Ewen Cheslack-Postava 
Date: Tue, 14 Apr 2020 16:36:52 -0700
Subject: [PATCH 0931/1071] MINOR: Upgrade ducktape to 0.7.7 (#8487)

This fixes a version pinning issue where a transitive dependency had a
major version upgrade that a dependency did not account for, breaking
the build.

Reviewers: Andrew Egelhofer , Matthias J. Sax 
---
 tests/docker/Dockerfile | 2 +-
 tests/setup.py          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile
index f990663246714..63bac30f55935 100644
--- a/tests/docker/Dockerfile
+++ b/tests/docker/Dockerfile
@@ -34,7 +34,7 @@ LABEL ducker.creator=$ducker_creator
 # Update Linux and install necessary utilities.
 RUN apt update && apt install -y sudo netcat iptables rsync unzip wget curl jq coreutils openssh-server net-tools vim python-pip python-dev libffi-dev libssl-dev cmake pkg-config libfuse-dev iperf traceroute && apt-get -y clean
 RUN python -m pip install -U pip==9.0.3;
-RUN pip install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 && pip install --upgrade ducktape==0.7.6
+RUN pip install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 && pip install --upgrade ducktape==0.7.7
 
 # Set up ssh
 COPY ./ssh-config /root/.ssh/config
diff --git a/tests/setup.py b/tests/setup.py
index c28fdfddff3fc..fb6393498342b 100644
--- a/tests/setup.py
+++ b/tests/setup.py
@@ -51,7 +51,7 @@ def run_tests(self):
       license="apache2.0",
       packages=find_packages(),
       include_package_data=True,
-      install_requires=["ducktape==0.7.6", "requests==2.20.0"],
+      install_requires=["ducktape==0.7.7", "requests==2.20.0"],
       tests_require=["pytest", "mock"],
       cmdclass={'test': PyTest},
       )

From e1f18df7f6615109b6cc77b66d9be37b09256a0a Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Wed, 15 Apr 2020 17:18:30 -0700
Subject: [PATCH 0932/1071] KAFKA-9838; Add log concurrency test and fix minor
 race condition (#8476)

The patch adds a new test case for validating concurrent read/write behavior in the `Log` implementation. In the process of verifying this, we found a race condition in `read`. The previous logic checks whether the start offset is equal to the end offset before collecting the high watermark. It is possible that the log is truncated in between these two conditions which could cause the high watermark to be equal to the log end offset. When this happens, `LogSegment.read` fails because it is unable to find the starting position to read from.

Reviewers: Guozhang Wang 
---
 core/src/main/scala/kafka/log/Log.scala       |  11 +-
 .../unit/kafka/log/LogConcurrencyTest.scala   | 183 ++++++++++++++++++
 2 files changed, 188 insertions(+), 6 deletions(-)
 create mode 100644 core/src/test/scala/unit/kafka/log/LogConcurrencyTest.scala

diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index 98790d293955b..91bb1ebff6009 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -1480,10 +1480,7 @@ class Log(@volatile var dir: File,
       // Because we don't use the lock for reading, the synchronization is a little bit tricky.
       // We create the local variables to avoid race conditions with updates to the log.
       val endOffsetMetadata = nextOffsetMetadata
-      val endOffset = nextOffsetMetadata.messageOffset
-      if (startOffset == endOffset)
-        return emptyFetchDataInfo(endOffsetMetadata, includeAbortedTxns)
-
+      val endOffset = endOffsetMetadata.messageOffset
       var segmentEntry = segments.floorEntry(startOffset)
 
       // return error on attempt to read beyond the log end offset or read below log start offset
@@ -1492,12 +1489,14 @@ class Log(@volatile var dir: File,
           s"but we only have log segments in the range $logStartOffset to $endOffset.")
 
       val maxOffsetMetadata = isolation match {
-        case FetchLogEnd => nextOffsetMetadata
+        case FetchLogEnd => endOffsetMetadata
         case FetchHighWatermark => fetchHighWatermarkMetadata
         case FetchTxnCommitted => fetchLastStableOffsetMetadata
       }
 
-      if (startOffset > maxOffsetMetadata.messageOffset) {
+      if (startOffset == maxOffsetMetadata.messageOffset) {
+        return emptyFetchDataInfo(maxOffsetMetadata, includeAbortedTxns)
+      } else if (startOffset > maxOffsetMetadata.messageOffset) {
         val startOffsetMetadata = convertToOffsetMetadataOrThrow(startOffset)
         return emptyFetchDataInfo(startOffsetMetadata, includeAbortedTxns)
       }
diff --git a/core/src/test/scala/unit/kafka/log/LogConcurrencyTest.scala b/core/src/test/scala/unit/kafka/log/LogConcurrencyTest.scala
new file mode 100644
index 0000000000000..0a9e71f6febae
--- /dev/null
+++ b/core/src/test/scala/unit/kafka/log/LogConcurrencyTest.scala
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.log
+
+import java.util.Properties
+import java.util.concurrent.{Callable, Executors}
+
+import kafka.server.{BrokerTopicStats, FetchHighWatermark, LogDirFailureChannel}
+import kafka.utils.{KafkaScheduler, TestUtils}
+import org.apache.kafka.common.record.SimpleRecord
+import org.apache.kafka.common.utils.{Time, Utils}
+import org.junit.Assert._
+import org.junit.{After, Before, Test}
+
+import scala.collection.mutable.ListBuffer
+import scala.util.Random
+
+class LogConcurrencyTest {
+  private val brokerTopicStats = new BrokerTopicStats
+  private val random = new Random()
+  private val scheduler = new KafkaScheduler(1)
+  private val tmpDir = TestUtils.tempDir()
+  private val logDir = TestUtils.randomPartitionLogDir(tmpDir)
+
+  @Before
+  def setup(): Unit = {
+    scheduler.startup()
+  }
+
+  @After
+  def shutdown(): Unit = {
+    scheduler.shutdown()
+    Utils.delete(tmpDir)
+  }
+
+  @Test
+  def testUncommittedDataNotConsumed(): Unit = {
+    testUncommittedDataNotConsumed(createLog())
+  }
+
+  @Test
+  def testUncommittedDataNotConsumedFrequentSegmentRolls(): Unit = {
+    val logProps = new Properties()
+    logProps.put(LogConfig.SegmentBytesProp, 237: Integer)
+    val logConfig = LogConfig(logProps)
+    testUncommittedDataNotConsumed(createLog(logConfig))
+  }
+
+  def testUncommittedDataNotConsumed(log: Log): Unit = {
+    val executor = Executors.newFixedThreadPool(2)
+    try {
+      val maxOffset = 5000
+      val consumer = new ConsumerTask(log, maxOffset)
+      val appendTask = new LogAppendTask(log, maxOffset)
+
+      val consumerFuture = executor.submit(consumer)
+      val fetcherTaskFuture = executor.submit(appendTask)
+
+      fetcherTaskFuture.get()
+      consumerFuture.get()
+
+      validateConsumedData(log, consumer.consumedBatches)
+    } finally executor.shutdownNow()
+  }
+
+  /**
+   * Simple consumption task which reads the log in ascending order and collects
+   * consumed batches for validation
+   */
+  private class ConsumerTask(log: Log, lastOffset: Int) extends Callable[Unit] {
+    val consumedBatches = ListBuffer.empty[FetchedBatch]
+
+    override def call(): Unit = {
+      var fetchOffset = 0L
+      while (log.highWatermark < lastOffset) {
+        val readInfo = log.read(
+          startOffset = fetchOffset,
+          maxLength = 1,
+          isolation = FetchHighWatermark,
+          minOneMessage = true
+        )
+        readInfo.records.batches().forEach { batch =>
+          consumedBatches += FetchedBatch(batch.baseOffset, batch.partitionLeaderEpoch)
+          fetchOffset = batch.lastOffset + 1
+        }
+      }
+    }
+  }
+
+  /**
+   * This class simulates basic leader/follower behavior.
+   */
+  private class LogAppendTask(log: Log, lastOffset: Long) extends Callable[Unit] {
+    override def call(): Unit = {
+      var leaderEpoch = 1
+      var isLeader = true
+
+      while (log.highWatermark < lastOffset) {
+        random.nextInt(2) match {
+          case 0 =>
+            val logEndOffsetMetadata = log.logEndOffsetMetadata
+            val logEndOffset = logEndOffsetMetadata.messageOffset
+            val batchSize = random.nextInt(9) + 1
+            val records = (0 to batchSize).map(i => new SimpleRecord(s"$i".getBytes))
+
+            if (isLeader) {
+              log.appendAsLeader(TestUtils.records(records), leaderEpoch)
+              log.maybeIncrementHighWatermark(logEndOffsetMetadata)
+            } else {
+              log.appendAsFollower(TestUtils.records(records,
+                baseOffset = logEndOffset,
+                partitionLeaderEpoch = leaderEpoch))
+              log.updateHighWatermark(logEndOffset)
+            }
+
+          case 1 =>
+            isLeader = !isLeader
+            leaderEpoch += 1
+
+            if (!isLeader) {
+              log.truncateTo(log.highWatermark)
+            }
+        }
+      }
+    }
+  }
+
+  private def createLog(config: LogConfig = LogConfig(new Properties())): Log = {
+    Log(dir = logDir,
+      config = config,
+      logStartOffset = 0L,
+      recoveryPoint = 0L,
+      scheduler = scheduler,
+      brokerTopicStats = brokerTopicStats,
+      time = Time.SYSTEM,
+      maxProducerIdExpirationMs = 60 * 60 * 1000,
+      producerIdExpirationCheckIntervalMs = LogManager.ProducerIdExpirationCheckIntervalMs,
+      logDirFailureChannel = new LogDirFailureChannel(10))
+  }
+
+  private def validateConsumedData(log: Log, consumedBatches: Iterable[FetchedBatch]): Unit = {
+    val iter = consumedBatches.iterator
+    log.logSegments.foreach { segment =>
+      segment.log.batches.forEach { batch =>
+        if (iter.hasNext) {
+          val consumedBatch = iter.next()
+          try {
+            assertEquals("Consumed batch with unexpected leader epoch",
+              batch.partitionLeaderEpoch, consumedBatch.epoch)
+            assertEquals("Consumed batch with unexpected base offset",
+              batch.baseOffset, consumedBatch.baseOffset)
+          } catch {
+            case t: Throwable =>
+              throw new AssertionError(s"Consumed batch $consumedBatch " +
+                s"does not match next expected batch in log $batch", t)
+          }
+        }
+      }
+    }
+  }
+
+  private case class FetchedBatch(baseOffset: Long, epoch: Int) {
+    override def toString: String = {
+      s"FetchedBatch(baseOffset=$baseOffset, epoch=$epoch)"
+    }
+  }
+
+}

From c94e4e722f2de5ef7755e715cb371db404718e27 Mon Sep 17 00:00:00 2001
From: Chia-Ping Tsai 
Date: Thu, 16 Apr 2020 20:26:30 +0800
Subject: [PATCH 0933/1071] KAFKA-9854 Re-authenticating causes mismatched
 parse of response (#8471)

Reviewers: Rajini Sivaram , Ron Dagostino 
---
 .../apache/kafka/clients/NetworkClient.java   | 21 +++++++-
 .../SaslClientAuthenticator.java              | 41 ++++++++++++++-
 .../kafka/clients/NetworkClientTest.java      | 14 +++++
 .../authenticator/SaslAuthenticatorTest.java  | 51 +++++++++++++++++++
 4 files changed, 123 insertions(+), 4 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
index 3431b83c9dacd..9e52b4e2d2f43 100644
--- a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
@@ -32,6 +32,7 @@
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.CommonFields;
 import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.protocol.types.SchemaException;
 import org.apache.kafka.common.protocol.types.Struct;
 import org.apache.kafka.common.requests.AbstractRequest;
 import org.apache.kafka.common.requests.AbstractResponse;
@@ -41,6 +42,7 @@
 import org.apache.kafka.common.requests.MetadataResponse;
 import org.apache.kafka.common.requests.RequestHeader;
 import org.apache.kafka.common.requests.ResponseHeader;
+import org.apache.kafka.common.security.authenticator.SaslClientAuthenticator;
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.common.utils.Utils;
@@ -932,9 +934,15 @@ private void handleInitiateApiVersionRequests(long now) {
      * Validate that the response corresponds to the request we expect or else explode
      */
     private static void correlate(RequestHeader requestHeader, ResponseHeader responseHeader) {
-        if (requestHeader.correlationId() != responseHeader.correlationId())
+        if (requestHeader.correlationId() != responseHeader.correlationId()) {
+            if (SaslClientAuthenticator.isReserved(requestHeader.correlationId())
+                    && !SaslClientAuthenticator.isReserved(responseHeader.correlationId()))
+                throw new SchemaException("the response is unrelated to Sasl request since its correlation id is " + responseHeader.correlationId()
+                    + " and the reserved range for Sasl request is [ "
+                    + SaslClientAuthenticator.MIN_RESERVED_CORRELATION_ID + "," + SaslClientAuthenticator.MAX_RESERVED_CORRELATION_ID + "]");
             throw new IllegalStateException("Correlation id for response (" + responseHeader.correlationId()
                     + ") does not match request (" + requestHeader.correlationId() + "), request header: " + requestHeader);
+        }
     }
 
     /**
@@ -1139,6 +1147,15 @@ public ClientRequest newClientRequest(String nodeId,
         return newClientRequest(nodeId, requestBuilder, createdTimeMs, expectResponse, defaultRequestTimeoutMs, null);
     }
 
+    // visible for testing
+    int nextCorrelationId() {
+        if (SaslClientAuthenticator.isReserved(correlation)) {
+            // the numeric overflow is fine as negative values is acceptable
+            correlation = SaslClientAuthenticator.MAX_RESERVED_CORRELATION_ID + 1;
+        }
+        return correlation++;
+    }
+
     @Override
     public ClientRequest newClientRequest(String nodeId,
                                           AbstractRequest.Builder requestBuilder,
@@ -1146,7 +1163,7 @@ public ClientRequest newClientRequest(String nodeId,
                                           boolean expectResponse,
                                           int requestTimeoutMs,
                                           RequestCompletionHandler callback) {
-        return new ClientRequest(nodeId, requestBuilder, correlation++, clientId, createdTimeMs, expectResponse,
+        return new ClientRequest(nodeId, requestBuilder, nextCorrelationId(), clientId, createdTimeMs, expectResponse,
                 requestTimeoutMs, callback);
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
index 6aa68cd57dabd..c75bb82265b03 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
@@ -106,6 +106,35 @@ public enum SaslState {
     private static final short DISABLE_KAFKA_SASL_AUTHENTICATE_HEADER = -1;
     private static final Random RNG = new Random();
 
+    /**
+     * the reserved range of correlation id for Sasl requests.
+     *
+     * Noted: there is a story about reserved range. The response of LIST_OFFSET is compatible to response of SASL_HANDSHAKE.
+     * Hence, we could miss the schema error when using schema of SASL_HANDSHAKE to parse response of LIST_OFFSET.
+     * For example: the IllegalStateException caused by mismatched correlation id is thrown if following steps happens.
+     * 1) sent LIST_OFFSET
+     * 2) sent SASL_HANDSHAKE
+     * 3) receive response of LIST_OFFSET
+     * 4) succeed to use schema of SASL_HANDSHAKE to parse response of LIST_OFFSET
+     * 5) throw IllegalStateException due to mismatched correlation id
+     * As a simple approach, we force Sasl requests to use a reserved correlation id which is separated from those
+     * used in NetworkClient for Kafka requests. Hence, we can guarantee that every SASL request will throw
+     * SchemaException due to correlation id mismatch during reauthentication
+     */
+    public static final int MAX_RESERVED_CORRELATION_ID = Integer.MAX_VALUE;
+
+    /**
+     * We only expect one request in-flight a time during authentication so the small range is fine.
+     */
+    public static final int MIN_RESERVED_CORRELATION_ID = MAX_RESERVED_CORRELATION_ID - 7;
+
+    /**
+     * @return true if the correlation id is reserved for SASL request. otherwise, false
+     */
+    public static boolean isReserved(int correlationId) {
+        return correlationId >= MIN_RESERVED_CORRELATION_ID;
+    }
+
     private final Subject subject;
     private final String servicePrincipal;
     private final String host;
@@ -174,7 +203,8 @@ public SaslClientAuthenticator(Map configs,
         }
     }
 
-    private SaslClient createSaslClient() {
+    // visible for testing
+    SaslClient createSaslClient() {
         try {
             return Subject.doAs(subject, (PrivilegedExceptionAction) () -> {
                 String[] mechs = {mechanism};
@@ -323,6 +353,13 @@ public Long reauthenticationLatencyMs() {
         return reauthInfo.reauthenticationLatencyMs();
     }
 
+    // visible for testing
+    int nextCorrelationId() {
+        if (!isReserved(correlationId))
+            correlationId = MIN_RESERVED_CORRELATION_ID;
+        return correlationId++;
+    }
+
     private RequestHeader nextRequestHeader(ApiKeys apiKey, short version) {
         String clientId = (String) configs.get(CommonClientConfigs.CLIENT_ID_CONFIG);
         short requestApiKey = apiKey.id;
@@ -331,7 +368,7 @@ private RequestHeader nextRequestHeader(ApiKeys apiKey, short version) {
                 setRequestApiKey(requestApiKey).
                 setRequestApiVersion(version).
                 setClientId(clientId).
-                setCorrelationId(correlationId++),
+                setCorrelationId(nextCorrelationId()),
             apiKey.requestHeaderVersion(version));
         return currentRequestHeader;
     }
diff --git a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
index a4145d1bc0c15..e445c55413675 100644
--- a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
@@ -38,6 +38,7 @@
 import org.apache.kafka.common.requests.ProduceRequest;
 import org.apache.kafka.common.requests.RequestHeader;
 import org.apache.kafka.common.requests.ResponseHeader;
+import org.apache.kafka.common.security.authenticator.SaslClientAuthenticator;
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.common.utils.MockTime;
 import org.apache.kafka.test.DelayedReceive;
@@ -53,6 +54,9 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 import static org.apache.kafka.common.protocol.ApiKeys.PRODUCE;
 import static org.junit.Assert.assertEquals;
@@ -862,6 +866,16 @@ public void testCallDisconnect() throws Exception {
         assertTrue(client.canConnect(node, time.milliseconds()));
     }
 
+    @Test
+    public void testCorrelationId() {
+        int count = 100;
+        Set ids = IntStream.range(0, count)
+            .mapToObj(i -> client.nextCorrelationId())
+            .collect(Collectors.toSet());
+        assertEquals(count, ids.size());
+        ids.forEach(id -> assertTrue(id < SaslClientAuthenticator.MIN_RESERVED_CORRELATION_ID));
+    }
+
     private RequestHeader parseHeader(ByteBuffer buffer) {
         buffer.getInt(); // skip size
         return RequestHeader.parse(buffer.slice());
diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
index 6d95d2e91d857..dd3c61dc99bee 100644
--- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
@@ -29,10 +29,14 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Random;
 import java.util.Base64.Encoder;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 import javax.security.auth.Subject;
 import javax.security.auth.callback.Callback;
@@ -44,10 +48,12 @@
 import javax.security.auth.login.AppConfigurationEntry;
 import javax.security.auth.login.LoginContext;
 import javax.security.auth.login.LoginException;
+import javax.security.sasl.SaslClient;
 import javax.security.sasl.SaslException;
 
 import org.apache.kafka.clients.NetworkClient;
 import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.config.SaslConfigs;
 import org.apache.kafka.common.config.internals.BrokerSecurityConfigs;
 import org.apache.kafka.common.config.types.Password;
@@ -74,6 +80,8 @@
 import org.apache.kafka.common.network.TransportLayer;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.protocol.types.SchemaException;
+import org.apache.kafka.common.requests.ListOffsetResponse;
 import org.apache.kafka.common.security.auth.Login;
 import org.apache.kafka.common.security.auth.SecurityProtocol;
 import org.apache.kafka.common.requests.AbstractRequest;
@@ -113,9 +121,11 @@
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.common.utils.Utils;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
+import static org.apache.kafka.common.protocol.ApiKeys.LIST_OFFSETS;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
@@ -1506,6 +1516,47 @@ public void testCannotReauthenticateWithDifferentPrincipal() throws Exception {
             server.verifyReauthenticationMetrics(0, 1);
         }
     }
+
+    @Test
+    public void testCorrelationId() {
+        SaslClientAuthenticator authenticator = new SaslClientAuthenticator(
+              Collections.emptyMap(),
+              null,
+              "node",
+              null,
+              null,
+              null,
+              "plain",
+              false,
+              null,
+              null
+        ) {
+            @Override
+            SaslClient createSaslClient() {
+                return null;
+            }
+        };
+        int count = (SaslClientAuthenticator.MAX_RESERVED_CORRELATION_ID - SaslClientAuthenticator.MIN_RESERVED_CORRELATION_ID) * 2;
+        Set ids = IntStream.range(0, count)
+            .mapToObj(i -> authenticator.nextCorrelationId())
+            .collect(Collectors.toSet());
+        assertEquals(SaslClientAuthenticator.MAX_RESERVED_CORRELATION_ID - SaslClientAuthenticator.MIN_RESERVED_CORRELATION_ID + 1, ids.size());
+        ids.forEach(id -> {
+            assertTrue(id >= SaslClientAuthenticator.MIN_RESERVED_CORRELATION_ID);
+            assertTrue(SaslClientAuthenticator.isReserved(id));
+        });
+    }
+
+    @Test
+    public void testConvertListOffsetResponseToSaslHandshakeResponse() {
+        ListOffsetResponse response = new ListOffsetResponse(0, Collections.singletonMap(new TopicPartition("topic", 0),
+            new ListOffsetResponse.PartitionData(Errors.NONE, 0, 0, Optional.empty())));
+        ByteBuffer buffer = response.serialize(ApiKeys.LIST_OFFSETS, LIST_OFFSETS.latestVersion(), 0);
+        final RequestHeader header0 = new RequestHeader(LIST_OFFSETS, LIST_OFFSETS.latestVersion(), "id", SaslClientAuthenticator.MIN_RESERVED_CORRELATION_ID);
+        Assert.assertThrows(SchemaException.class, () -> NetworkClient.parseResponse(buffer.duplicate(), header0));
+        final RequestHeader header1 = new RequestHeader(LIST_OFFSETS, LIST_OFFSETS.latestVersion(), "id", 1);
+        Assert.assertThrows(IllegalStateException.class, () -> NetworkClient.parseResponse(buffer.duplicate(), header1));
+    }
     
     /**
      * Re-authentication must fail if mechanism changes

From c6954b79cadaafe3d439ed006c352ec386c623b4 Mon Sep 17 00:00:00 2001
From: Randall Hauch 
Date: Thu, 23 Apr 2020 13:37:45 -0500
Subject: [PATCH 0934/1071] KAFKA-9883: Add better error message when REST API
 forwards a request and leader is not known (#8536)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When the Connect worker forwards a REST API request to the leader, it might get back a `RequestTargetException` that suggests the worker should forward the request to a different worker. This can happen when the leader changes, and the worker that receives the original request forwards the request to the worker that it thinks is the current leader, but that worker is not the current leader. In this case. In most cases, the worker that received the forwarded request includes the URL of the current leader, but it is possible (albeit rare) that the worker doesn’t know the current leader and will include a null leader URL in the resulting `RequestTargetException`.

When this rare case happens, the user gets a null pointer exception in their response and the NPE is logged. Instead, the worker should catch this condition and provide a more useful error message that is similar to other existing error messages that might occur.

Added a unit test that verifies this corner case is caught and this particular NPE does not occur.

Author: Randall Hauch 
Reviewer: Konstantine Karantasis 
---
 .../rest/resources/ConnectorsResource.java    |  9 ++++++++-
 .../resources/ConnectorsResourceTest.java     | 19 +++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResource.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResource.java
index 3b41fb7c143a2..8728e1cd1748e 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResource.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResource.java
@@ -306,7 +306,14 @@ private  T completeOrForwardRequest(FutureCallback cb,
                     // this gives two total hops to resolve the request before giving up.
                     boolean recursiveForward = forward == null;
                     RequestTargetException targetException = (RequestTargetException) cause;
-                    String forwardUrl = UriBuilder.fromUri(targetException.forwardUrl())
+                    String forwardedUrl = targetException.forwardUrl();
+                    if (forwardedUrl == null) {
+                        // the target didn't know of the leader at this moment.
+                        throw new ConnectRestException(Response.Status.CONFLICT.getStatusCode(),
+                                "Cannot complete request momentarily due to no known leader URL, "
+                                + "likely because a rebalance was underway.");
+                    }
+                    String forwardUrl = UriBuilder.fromUri(forwardedUrl)
                             .path(path)
                             .queryParam("forward", recursiveForward)
                             .build()
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java
index 967b8fca07b19..f604c80ceec69 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java
@@ -36,6 +36,7 @@
 import org.apache.kafka.connect.runtime.rest.entities.ConnectorType;
 import org.apache.kafka.connect.runtime.rest.entities.CreateConnectorRequest;
 import org.apache.kafka.connect.runtime.rest.entities.TaskInfo;
+import org.apache.kafka.connect.runtime.rest.errors.ConnectRestException;
 import org.apache.kafka.connect.util.Callback;
 import org.apache.kafka.connect.util.ConnectorTaskId;
 import org.easymock.Capture;
@@ -69,6 +70,8 @@
 import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
 
 @RunWith(PowerMockRunner.class)
 @PrepareForTest(RestClient.class)
@@ -803,6 +806,22 @@ public void testRestartTaskOwnerRedirect() throws Throwable {
         PowerMock.verifyAll();
     }
 
+    @Test
+    public void testCompleteOrForwardWithErrorAndNoForwardUrl() throws Throwable {
+        final Capture>> cb = Capture.newInstance();
+        herder.deleteConnectorConfig(EasyMock.eq(CONNECTOR_NAME), EasyMock.capture(cb));
+        String leaderUrl = null;
+        expectAndCallbackException(cb, new NotLeaderException("not leader", leaderUrl));
+
+        PowerMock.replayAll();
+
+        ConnectRestException e = assertThrows(ConnectRestException.class, () -> {
+            connectorsResource.destroyConnector(CONNECTOR_NAME, NULL_HEADERS, FORWARD);
+        });
+        assertTrue(e.getMessage().contains("no known leader URL"));
+        PowerMock.verifyAll();
+    }
+
     private  byte[] serializeAsBytes(final T value) throws IOException {
         return new ObjectMapper().writeValueAsBytes(value);
     }

From ecbea1455ee9ef143b9f9ac316c76b243d82199e Mon Sep 17 00:00:00 2001
From: Jeff Widman 
Date: Mon, 27 Apr 2020 23:10:08 -0700
Subject: [PATCH 0935/1071] MINOR: Fix typos in config properties in MM2 test
 (#8561)

Fixed typos in two MM2 configs that define the replication factor for internal Connect topics.
Only a single test was affected.

Reviewers: Ryanne Dolan , Konstantine Karantasis 
---
 .../kafka/connect/mirror/MirrorConnectorsIntegrationTest.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java
index 8b1bac9e2e74d..aacf42ba63f4c 100644
--- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java
+++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java
@@ -86,8 +86,8 @@ public void setup() throws InterruptedException {
         mm2Props.put("heartbeats.topic.replication.factor", "1");
         mm2Props.put("offset-syncs.topic.replication.factor", "1");
         mm2Props.put("config.storage.topic.replication.factor", "1");
-        mm2Props.put("offset.stoage.topic.replication.factor", "1");
-        mm2Props.put("status.stoage.topic.replication.factor", "1");
+        mm2Props.put("offset.storage.topic.replication.factor", "1");
+        mm2Props.put("status.storage.topic.replication.factor", "1");
         mm2Props.put("replication.factor", "1");
         
         mm2Config = new MirrorMakerConfig(mm2Props); 

From 32e43921bee696b535f10bec6cb9bf5d6a3330e1 Mon Sep 17 00:00:00 2001
From: Steve Rodrigues <42848865+steverod@users.noreply.github.com>
Date: Tue, 28 Apr 2020 14:49:33 -0700
Subject: [PATCH 0936/1071] MINOR: Test compilation fixes for Scala 2.11
 (#8562)

Earlier log test changes went into trunk (for KAFKA-9807, KAFKA-9838) and were cherry-picked to 2.4, but they broke the Scala 2.11 build. This change cherry-picks the fixes needed to make those tests build for 2.11.

Reviewers: Jun Rao , Jason Gustafson 
---
 .../unit/kafka/log/LogConcurrencyTest.scala   |  5 ++-
 .../test/scala/unit/kafka/log/LogTest.scala   | 38 ++++++++++---------
 .../ReplicaAlterLogDirsThreadTest.scala       |  7 +++-
 3 files changed, 29 insertions(+), 21 deletions(-)

diff --git a/core/src/test/scala/unit/kafka/log/LogConcurrencyTest.scala b/core/src/test/scala/unit/kafka/log/LogConcurrencyTest.scala
index 0a9e71f6febae..0670a4b1aec80 100644
--- a/core/src/test/scala/unit/kafka/log/LogConcurrencyTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogConcurrencyTest.scala
@@ -27,6 +27,7 @@ import org.apache.kafka.common.utils.{Time, Utils}
 import org.junit.Assert._
 import org.junit.{After, Before, Test}
 
+import scala.collection.JavaConverters._
 import scala.collection.mutable.ListBuffer
 import scala.util.Random
 
@@ -94,7 +95,7 @@ class LogConcurrencyTest {
           isolation = FetchHighWatermark,
           minOneMessage = true
         )
-        readInfo.records.batches().forEach { batch =>
+        readInfo.records.batches().asScala.foreach { batch =>
           consumedBatches += FetchedBatch(batch.baseOffset, batch.partitionLeaderEpoch)
           fetchOffset = batch.lastOffset + 1
         }
@@ -156,7 +157,7 @@ class LogConcurrencyTest {
   private def validateConsumedData(log: Log, consumedBatches: Iterable[FetchedBatch]): Unit = {
     val iter = consumedBatches.iterator
     log.logSegments.foreach { segment =>
-      segment.log.batches.forEach { batch =>
+      segment.log.batches.asScala.foreach { batch =>
         if (iter.hasNext) {
           val consumedBatch = iter.next()
           try {
diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala
index 0831a85d17486..d0eb5a31a5398 100755
--- a/core/src/test/scala/unit/kafka/log/LogTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogTest.scala
@@ -3655,31 +3655,35 @@ class LogTest {
 
     // Thread 1 writes single-record transactions and attempts to read them
     // before they have been aborted, and then aborts them
-    val txnWriteAndReadLoop: Callable[Int] = () => {
-      var nonEmptyReads = 0
-      while (log.logEndOffset < lastOffset) {
-        val currentLogEndOffset = log.logEndOffset
+    val txnWriteAndReadLoop: Callable[Int] = new Callable[Int]() {
+      override def call(): Int = {
+        var nonEmptyReads = 0
+        while (log.logEndOffset < lastOffset) {
+          val currentLogEndOffset = log.logEndOffset
 
-        appendProducer(1)
+          appendProducer(1)
 
-        val readInfo = log.read(
-          startOffset = currentLogEndOffset,
-          maxLength = Int.MaxValue,
-          isolation = FetchTxnCommitted,
-          minOneMessage = false)
+          val readInfo = log.read(
+            startOffset = currentLogEndOffset,
+            maxLength = Int.MaxValue,
+            isolation = FetchTxnCommitted,
+            minOneMessage = false)
 
-        if (readInfo.records.sizeInBytes() > 0)
-          nonEmptyReads += 1
+          if (readInfo.records.sizeInBytes() > 0)
+            nonEmptyReads += 1
 
-        appendEndTxnMarkerAsLeader(log, producerId, producerEpoch, ControlRecordType.ABORT)
+          appendEndTxnMarkerAsLeader(log, producerId, producerEpoch, ControlRecordType.ABORT)
+        }
+        nonEmptyReads
       }
-      nonEmptyReads
     }
 
     // Thread 2 watches the log and updates the high watermark
-    val hwUpdateLoop: Runnable = () => {
-      while (log.logEndOffset < lastOffset) {
-        log.updateHighWatermark(log.logEndOffset)
+    val hwUpdateLoop: Runnable = new Runnable() {
+      override def run(): Unit = {
+        while (log.logEndOffset < lastOffset) {
+          log.updateHighWatermark(log.logEndOffset)
+        }
       }
     }
 
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala
index 1faaefee3b100..05015087304e7 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala
@@ -35,6 +35,8 @@ import org.easymock.{Capture, CaptureType, EasyMock, IAnswer, IExpectationSetter
 import org.junit.Assert._
 import org.junit.Test
 import org.mockito.Mockito.{doNothing, when}
+import org.mockito.invocation.InvocationOnMock
+import org.mockito.stubbing.Answer
 import org.mockito.{ArgumentCaptor, ArgumentMatchers, Mockito}
 
 import scala.collection.{Map, Seq}
@@ -243,8 +245,9 @@ class ReplicaAlterLogDirsThreadTest {
       responseCallback = callbackCaptor.capture(),
       isolationLevel = ArgumentMatchers.eq(IsolationLevel.READ_UNCOMMITTED),
       clientMetadata = ArgumentMatchers.eq(None)
-    )).thenAnswer(_ => {
-      callbackCaptor.getValue.apply(Seq((topicPartition, responseData)))
+    ))
+    .thenAnswer(new Answer[Unit] {
+      override def answer(invocation: InvocationOnMock): Unit = callbackCaptor.getValue.apply(Seq((topicPartition, responseData)))
     })
   }
 

From 9d447be1ef8399bb162944bd0ded02384162f41f Mon Sep 17 00:00:00 2001
From: Steve Rodrigues <42848865+steverod@users.noreply.github.com>
Date: Wed, 29 Apr 2020 11:10:12 -0700
Subject: [PATCH 0937/1071] [KAFKA-9826] Handle an unaligned first dirty offset
 during log cleaning. (#8469) (#8543)

In KAFKA-9826, a log whose first dirty offset was past the start of the active segment and past the last cleaned point resulted in an endless cycle of picking the segment to clean and discarding it. Though this didn't interfere with cleaning other log segments, it kept the log cleaner thread continuously busy (potentially wasting CPU and impacting other running threads) and filled the logs with lots of extraneous messages.

This was determined to be because the active segment was getting mistakenly picked for cleaning, and because the logSegments code handles (start == end) cases only for (start, end) on a segment boundary: the intent is to return a null list, but if they're not on a segment boundary, the routine returns that segment.

This fix has two parts:

It changes logSegments to handle start==end by returning an empty List always.

It changes the definition of calculateCleanableBytes to not consider anything past the UncleanableOffset; previously, it would potentially shift the UncleanableOffset to match the firstDirtyOffset even if the firstDirtyOffset was past the firstUncleanableOffset. This has no real effect now in the context of the fix for (1) but it makes the code read more like the model that the code is attempting to follow.

These changes require modifications to a few test cases that handled this particular test case; they were introduced in the context of KAFKA-8764. Those situations are now handled elsewhere in code, but the tests themselves allowed a DirtyOffset in the active segment, and expected an active segment to be selected for cleaning.

Reviewer: Jun Rao 
---
 core/src/main/scala/kafka/log/Log.scala       | 23 +++++++-----
 .../scala/kafka/log/LogCleanerManager.scala   |  2 +-
 .../kafka/log/LogCleanerManagerTest.scala     | 10 +++---
 .../test/scala/unit/kafka/log/LogTest.scala   | 35 +++++++++++++++++++
 4 files changed, 56 insertions(+), 14 deletions(-)

diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index 91bb1ebff6009..401cb2c92ac48 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -2140,17 +2140,22 @@ class Log(@volatile var dir: File,
 
   /**
    * Get all segments beginning with the segment that includes "from" and ending with the segment
-   * that includes up to "to-1" or the end of the log (if to > logEndOffset)
+   * that includes up to "to-1" or the end of the log (if to > logEndOffset).
    */
   def logSegments(from: Long, to: Long): Iterable[LogSegment] = {
-    lock synchronized {
-      val view = Option(segments.floorKey(from)).map { floor =>
-        if (to < floor)
-          throw new IllegalArgumentException(s"Invalid log segment range: requested segments in $topicPartition " +
-            s"from offset $from mapping to segment with base offset $floor, which is greater than limit offset $to")
-        segments.subMap(floor, to)
-      }.getOrElse(segments.headMap(to))
-      view.values.asScala
+    if (from == to) {
+      // Handle non-segment-aligned empty sets
+      List.empty[LogSegment]
+    } else if (to < from) {
+      throw new IllegalArgumentException(s"Invalid log segment range: requested segments in $topicPartition " +
+        s"from offset $from which is greater than limit offset $to")
+    } else {
+      lock synchronized {
+        val view = Option(segments.floorKey(from)).map { floor =>
+          segments.subMap(floor, to)
+        }.getOrElse(segments.headMap(to))
+        view.values.asScala
+      }
     }
   }
 
diff --git a/core/src/main/scala/kafka/log/LogCleanerManager.scala b/core/src/main/scala/kafka/log/LogCleanerManager.scala
index 19f7e4d2c87e9..04a97fdd38a5a 100755
--- a/core/src/main/scala/kafka/log/LogCleanerManager.scala
+++ b/core/src/main/scala/kafka/log/LogCleanerManager.scala
@@ -607,7 +607,7 @@ private[log] object LogCleanerManager extends Logging {
   def calculateCleanableBytes(log: Log, firstDirtyOffset: Long, uncleanableOffset: Long): (Long, Long) = {
     val firstUncleanableSegment = log.nonActiveLogSegmentsFrom(uncleanableOffset).headOption.getOrElse(log.activeSegment)
     val firstUncleanableOffset = firstUncleanableSegment.baseOffset
-    val cleanableBytes = log.logSegments(firstDirtyOffset, math.max(firstDirtyOffset, firstUncleanableOffset)).map(_.size.toLong).sum
+    val cleanableBytes = log.logSegments(math.min(firstDirtyOffset, firstUncleanableOffset), firstUncleanableOffset).map(_.size.toLong).sum
 
     (firstUncleanableOffset, cleanableBytes)
   }
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
index 5e2620ca0b7cf..ed5b1e8a91423 100644
--- a/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
@@ -237,8 +237,9 @@ class LogCleanerManagerTest extends Logging {
     val cleanerManager = createCleanerManagerMock(logs)
     cleanerCheckpoints.put(tp, 0L)
 
-    val filthiestLog = cleanerManager.grabFilthiestCompactedLog(time).get
-    assertEquals(2L, filthiestLog.firstDirtyOffset)
+    // The active segment is uncleanable and hence not filthy from the POV of the CleanerManager.
+    val filthiestLog = cleanerManager.grabFilthiestCompactedLog(time)
+    assertEquals(None, filthiestLog)
   }
 
   @Test
@@ -262,8 +263,9 @@ class LogCleanerManagerTest extends Logging {
     val cleanerManager = createCleanerManagerMock(logs)
     cleanerCheckpoints.put(tp, 3L)
 
-    val filthiestLog = cleanerManager.grabFilthiestCompactedLog(time).get
-    assertEquals(3L, filthiestLog.firstDirtyOffset)
+    // These segments are uncleanable and hence not filthy
+    val filthiestLog = cleanerManager.grabFilthiestCompactedLog(time)
+    assertEquals(None, filthiestLog)
   }
 
   /**
diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala
index d0eb5a31a5398..19067ab7e8591 100755
--- a/core/src/test/scala/unit/kafka/log/LogTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogTest.scala
@@ -469,6 +469,41 @@ class LogTest {
     assertEquals(101L, log.logEndOffset)
   }
 
+  /**
+   * Test the values returned by the logSegments call
+   */
+  @Test
+  def testLogSegmentsCallCorrect(): Unit = {
+    // Create 3 segments and make sure we get the right values from various logSegments calls.
+    def createRecords = TestUtils.singletonRecords(value = "test".getBytes, timestamp = mockTime.milliseconds)
+    def getSegmentOffsets(log :Log, from: Long, to: Long) = log.logSegments(from, to).map { _.baseOffset }
+    val setSize = createRecords.sizeInBytes
+    val msgPerSeg = 10
+    val segmentSize = msgPerSeg * setSize  // each segment will be 10 messages
+    // create a log
+    val logConfig = LogTest.createLogConfig(segmentBytes = segmentSize)
+    val log = createLog(logDir, logConfig)
+    assertEquals("There should be exactly 1 segment.", 1, log.numberOfSegments)
+
+    // segments expire in size
+    for (_ <- 1 to (2 * msgPerSeg + 2))
+      log.appendAsLeader(createRecords, leaderEpoch = 0)
+    assertEquals("There should be exactly 3 segments.", 3, log.numberOfSegments)
+
+    // from == to should always be null
+    assertEquals(List.empty[LogSegment], getSegmentOffsets(log, 10, 10))
+    assertEquals(List.empty[LogSegment], getSegmentOffsets(log, 15, 15))
+
+    assertEquals(List[Long](0, 10, 20), getSegmentOffsets(log, 0, 21))
+
+    assertEquals(List[Long](0), getSegmentOffsets(log, 1, 5))
+    assertEquals(List[Long](10, 20), getSegmentOffsets(log, 13, 21))
+    assertEquals(List[Long](10), getSegmentOffsets(log, 13, 17))
+
+    // from < to is bad
+    assertThrows[IllegalArgumentException]({ log.logSegments(10, 0) })
+  }
+
   @Test
   def testInitializationOfProducerSnapshotsUpgradePath(): Unit = {
     // simulate the upgrade path by creating a new log with several segments, deleting the

From 6314ca6e971ee850932b663d08f7fd976c81d441 Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Fri, 13 Mar 2020 23:04:14 -0500
Subject: [PATCH 0938/1071] MINOR: reuse pseudo-topic in FKJoin (#8296)

Reuse the same pseudo-topic for serializing the LHS value in the foreign-key join resolver as
we originally used to serialize it before sending the subscription request.

Reviewers: Boyang Chen 
---
 .../streams/kstream/internals/KTableImpl.java |   1 +
 ...criptionResolverJoinProcessorSupplier.java |  11 +-
 ...leKTableForeignKeyJoinPseudoTopicTest.java | 138 ++++++++++++++++++
 ...tionResolverJoinProcessorSupplierTest.java |   6 +
 .../streams/utils/UniqueTopicSerdeScope.java  |   6 +
 5 files changed, 155 insertions(+), 7 deletions(-)
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinPseudoTopicTest.java

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
index 941a86607cc0b..c91c475887e6d 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
@@ -1084,6 +1084,7 @@ private  KTable doJoinOnForeignKey(final KTable forei
         final SubscriptionResolverJoinProcessorSupplier resolverProcessorSupplier = new SubscriptionResolverJoinProcessorSupplier<>(
             primaryKeyValueGetter,
             valSerde == null ? null : valSerde.serializer(),
+            valueHashSerdePseudoTopic,
             joiner,
             leftJoin
         );
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplier.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplier.java
index 8fa77aa16715b..31de0687d6ef7 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplier.java
@@ -42,15 +42,18 @@
 public class SubscriptionResolverJoinProcessorSupplier implements ProcessorSupplier> {
     private final KTableValueGetterSupplier valueGetterSupplier;
     private final Serializer constructionTimeValueSerializer;
+    private final String valueHashSerdePseudoTopic;
     private final ValueJoiner joiner;
     private final boolean leftJoin;
 
     public SubscriptionResolverJoinProcessorSupplier(final KTableValueGetterSupplier valueGetterSupplier,
                                                      final Serializer valueSerializer,
+                                                     final String valueHashSerdePseudoTopic,
                                                      final ValueJoiner joiner,
                                                      final boolean leftJoin) {
         this.valueGetterSupplier = valueGetterSupplier;
         constructionTimeValueSerializer = valueSerializer;
+        this.valueHashSerdePseudoTopic = valueHashSerdePseudoTopic;
         this.joiner = joiner;
         this.leftJoin = leftJoin;
     }
@@ -83,15 +86,9 @@ public void process(final K key, final SubscriptionResponseWrapper value) {
                 }
                 final ValueAndTimestamp currentValueWithTimestamp = valueGetter.get(key);
 
-                //We are unable to access the actual source topic name for the valueSerializer at runtime, without
-                //tightly coupling to KTableRepartitionProcessorSupplier.
-                //While we can use the source topic from where the events came from, we shouldn't serialize against it
-                //as it causes problems with the confluent schema registry, which requires each topic have only a single
-                //registered schema.
-                final String dummySerializationTopic = context().topic() + "-join-resolver";
                 final long[] currentHash = currentValueWithTimestamp == null ?
                     null :
-                    Murmur3.hash128(runtimeValueSerializer.serialize(dummySerializationTopic, currentValueWithTimestamp.value()));
+                    Murmur3.hash128(runtimeValueSerializer.serialize(valueHashSerdePseudoTopic, currentValueWithTimestamp.value()));
 
                 final long[] messageHash = value.getOriginalValueHash();
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinPseudoTopicTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinPseudoTopicTest.java
new file mode 100644
index 0000000000000..b08c29380b2c7
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinPseudoTopicTest.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.integration;
+
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TestInputTopic;
+import org.apache.kafka.streams.Topology;
+import org.apache.kafka.streams.TopologyTestDriver;
+import org.apache.kafka.streams.kstream.Consumed;
+import org.apache.kafka.streams.kstream.KTable;
+import org.apache.kafka.streams.kstream.Materialized;
+import org.apache.kafka.streams.utils.UniqueTopicSerdeScope;
+import org.apache.kafka.test.TestUtils;
+import org.junit.Test;
+
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Properties;
+
+import static org.apache.kafka.common.utils.Utils.mkEntry;
+import static org.apache.kafka.common.utils.Utils.mkMap;
+import static org.apache.kafka.common.utils.Utils.mkProperties;
+import static org.apache.kafka.common.utils.Utils.mkSet;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+
+public class KTableKTableForeignKeyJoinPseudoTopicTest {
+
+    private static final String LEFT_TABLE = "left_table";
+    private static final String RIGHT_TABLE = "right_table";
+    private static final String OUTPUT = "output-topic";
+    private static final String REJOIN_OUTPUT = "rejoin-output-topic";
+    private final Properties streamsConfig = mkProperties(mkMap(
+        mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, "ktable-ktable-joinOnForeignKey"),
+        mkEntry(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "asdf:0000"),
+        mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath())
+    ));
+
+
+    private static Collection buildParameters(final List... argOptions) {
+        List result = new LinkedList<>();
+        result.add(new Object[0]);
+
+        for (final List argOption : argOptions) {
+            result = times(result, argOption);
+        }
+
+        return result;
+    }
+
+    private static List times(final List left, final List right) {
+        final List result = new LinkedList<>();
+        for (final Object[] args : left) {
+            for (final Object rightElem : right) {
+                final Object[] resArgs = new Object[args.length + 1];
+                System.arraycopy(args, 0, resArgs, 0, args.length);
+                resArgs[args.length] = rightElem;
+                result.add(resArgs);
+            }
+        }
+        return result;
+    }
+
+
+    @Test
+    public void shouldUseExpectedTopicsWithSerde() {
+        final UniqueTopicSerdeScope serdeScope = new UniqueTopicSerdeScope();
+        final StreamsBuilder builder = new StreamsBuilder();
+
+        final KTable left = builder.table(
+            LEFT_TABLE,
+            Consumed.with(serdeScope.decorateSerde(Serdes.String(), streamsConfig, true),
+                          serdeScope.decorateSerde(Serdes.String(), streamsConfig, false))
+        );
+        final KTable right = builder.table(
+            RIGHT_TABLE,
+            Consumed.with(serdeScope.decorateSerde(Serdes.String(), streamsConfig, true),
+                          serdeScope.decorateSerde(Serdes.String(), streamsConfig, false))
+        );
+
+        left.join(
+            right,
+            value -> value.split("\\|")[1],
+            (value1, value2) -> "(" + value1 + "," + value2 + ")",
+            Materialized.with(null, serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)
+            ))
+            .toStream()
+            .to(OUTPUT);
+
+
+        final Topology topology = builder.build(streamsConfig);
+        try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
+            final TestInputTopic leftInput = driver.createInputTopic(LEFT_TABLE, new StringSerializer(), new StringSerializer());
+            final TestInputTopic rightInput = driver.createInputTopic(RIGHT_TABLE, new StringSerializer(), new StringSerializer());
+            leftInput.pipeInput("lhs1", "lhsValue1|rhs1");
+            rightInput.pipeInput("rhs1", "rhsValue1");
+        }
+        // verifying primarily that no extra pseudo-topics were used, but it's nice to also verify the rest of the
+        // topics our serdes serialize data for
+        assertThat(serdeScope.registeredTopics(), is(mkSet(
+            // expected pseudo-topics
+            "KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic-fk--key",
+            "KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic-pk--key",
+            "KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic-vh--value",
+            // internal topics
+            "ktable-ktable-joinOnForeignKey-KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic--key",
+            "ktable-ktable-joinOnForeignKey-KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000014-topic--key",
+            "ktable-ktable-joinOnForeignKey-KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000014-topic--value",
+            "ktable-ktable-joinOnForeignKey-left_table-STATE-STORE-0000000000-changelog--key",
+            "ktable-ktable-joinOnForeignKey-left_table-STATE-STORE-0000000000-changelog--value",
+            "ktable-ktable-joinOnForeignKey-right_table-STATE-STORE-0000000003-changelog--key",
+            "ktable-ktable-joinOnForeignKey-right_table-STATE-STORE-0000000003-changelog--value",
+            // output topics
+            "output-topic--key",
+            "output-topic--value"
+        )));
+    }
+
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplierTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplierTest.java
index 3ec19de384254..aae99ec6687d7 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplierTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplierTest.java
@@ -82,6 +82,7 @@ public void shouldNotForwardWhenHashDoesNotMatch() {
             new SubscriptionResolverJoinProcessorSupplier<>(
                 valueGetterSupplier,
                 STRING_SERIALIZER,
+                "value-hash-dummy-topic",
                 JOINER,
                 leftJoin
             );
@@ -106,6 +107,7 @@ public void shouldIgnoreUpdateWhenLeftHasBecomeNull() {
             new SubscriptionResolverJoinProcessorSupplier<>(
                 valueGetterSupplier,
                 STRING_SERIALIZER,
+                "value-hash-dummy-topic",
                 JOINER,
                 leftJoin
             );
@@ -130,6 +132,7 @@ public void shouldForwardWhenHashMatches() {
             new SubscriptionResolverJoinProcessorSupplier<>(
                 valueGetterSupplier,
                 STRING_SERIALIZER,
+                "value-hash-dummy-topic",
                 JOINER,
                 leftJoin
             );
@@ -155,6 +158,7 @@ public void shouldEmitTombstoneForInnerJoinWhenRightIsNull() {
             new SubscriptionResolverJoinProcessorSupplier<>(
                 valueGetterSupplier,
                 STRING_SERIALIZER,
+                "value-hash-dummy-topic",
                 JOINER,
                 leftJoin
             );
@@ -180,6 +184,7 @@ public void shouldEmitResultForLeftJoinWhenRightIsNull() {
             new SubscriptionResolverJoinProcessorSupplier<>(
                 valueGetterSupplier,
                 STRING_SERIALIZER,
+                "value-hash-dummy-topic",
                 JOINER,
                 leftJoin
             );
@@ -205,6 +210,7 @@ public void shouldEmitTombstoneForLeftJoinWhenRightIsNullAndLeftIsNull() {
             new SubscriptionResolverJoinProcessorSupplier<>(
                 valueGetterSupplier,
                 STRING_SERIALIZER,
+                "value-hash-dummy-topic",
                 JOINER,
                 leftJoin
             );
diff --git a/streams/src/test/java/org/apache/kafka/streams/utils/UniqueTopicSerdeScope.java b/streams/src/test/java/org/apache/kafka/streams/utils/UniqueTopicSerdeScope.java
index 04b1a7b2a786b..c385187a5b081 100644
--- a/streams/src/test/java/org/apache/kafka/streams/utils/UniqueTopicSerdeScope.java
+++ b/streams/src/test/java/org/apache/kafka/streams/utils/UniqueTopicSerdeScope.java
@@ -21,8 +21,10 @@
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serializer;
 
+import java.util.Collections;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
@@ -41,6 +43,10 @@ public  UniqueTopicSerdeDecorator decorateSerde(final Serde delegate,
         return decorator;
     }
 
+    public Set registeredTopics() {
+        return Collections.unmodifiableSet(topicTypeRegistry.keySet());
+    }
+
     public class UniqueTopicSerdeDecorator implements Serde {
         private final AtomicBoolean isKey = new AtomicBoolean(false);
         private final Serde delegate;

From 306c008d76cb2ae0fe4cf26a1e3fdf2f92a7fb80 Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Wed, 29 Apr 2020 16:17:34 -0500
Subject: [PATCH 0939/1071] KAFKA-9925: decorate pseudo-topics with app id
 (#8574)

Reviewers: Boyang Chen , Kin Siu
---
 .../streams/kstream/internals/KTableImpl.java |  24 ++-
 .../foreignkeyjoin/CombinedKeySchema.java     |  17 ++-
 ...JoinSubscriptionSendProcessorSupplier.java |  17 ++-
 ...criptionResolverJoinProcessorSupplier.java |  10 +-
 .../SubscriptionWrapperSerde.java             |  30 ++--
 .../internals/InternalTopologyBuilder.java    |   4 +
 ...eKTableForeignKeyJoinDefaultSerdeTest.java |  74 +++++++++-
 ...leKTableForeignKeyJoinPseudoTopicTest.java | 138 ------------------
 .../foreignkeyjoin/CombinedKeySchemaTest.java |  30 ++--
 ...tionResolverJoinProcessorSupplierTest.java |  12 +-
 .../SubscriptionWrapperSerdeTest.java         |   8 +-
 11 files changed, 176 insertions(+), 188 deletions(-)
 delete mode 100644 streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinPseudoTopicTest.java

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
index c91c475887e6d..062d3178e2984 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
@@ -75,6 +75,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 import static org.apache.kafka.streams.kstream.internals.graph.GraphGraceSearchUtil.findAndVerifyWindowGrace;
 
@@ -952,13 +953,26 @@ private  KTable doJoinOnForeignKey(final KTable forei
         //This occurs whenever the extracted foreignKey changes values.
         enableSendingOldValues();
 
+        final NamedInternal renamed = new NamedInternal(joinName);
+
+        final String subscriptionTopicName = renamed.suffixWithOrElseGet(
+            "-subscription-registration",
+            builder,
+            SUBSCRIPTION_REGISTRATION
+        ) + TOPIC_SUFFIX;
 
+        // the decoration can't be performed until we have the configuration available when the app runs,
+        // so we pass Suppliers into the components, which they can call at run time
+
+        final Supplier subscriptionPrimaryKeySerdePseudoTopic =
+            () -> internalTopologyBuilder().decoratePseudoTopic(subscriptionTopicName + "-pk");
+
+        final Supplier subscriptionForeignKeySerdePseudoTopic =
+            () -> internalTopologyBuilder().decoratePseudoTopic(subscriptionTopicName + "-fk");
+
+        final Supplier valueHashSerdePseudoTopic =
+            () -> internalTopologyBuilder().decoratePseudoTopic(subscriptionTopicName + "-vh");
 
-        final NamedInternal renamed = new NamedInternal(joinName);
-        final String subscriptionTopicName = renamed.suffixWithOrElseGet("-subscription-registration", builder, SUBSCRIPTION_REGISTRATION) + TOPIC_SUFFIX;
-        final String subscriptionPrimaryKeySerdePseudoTopic = subscriptionTopicName + "-pk";
-        final String subscriptionForeignKeySerdePseudoTopic = subscriptionTopicName + "-fk";
-        final String valueHashSerdePseudoTopic = subscriptionTopicName + "-vh";
         builder.internalTopologyBuilder.addInternalTopic(subscriptionTopicName);
 
         final Serde foreignKeySerde = ((KTableImpl) foreignKeyTable).keySerde;
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchema.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchema.java
index 92fb72c7ae6b9..57bc646a13e11 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchema.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchema.java
@@ -23,24 +23,27 @@
 import org.apache.kafka.streams.processor.ProcessorContext;
 
 import java.nio.ByteBuffer;
+import java.util.function.Supplier;
 
 /**
  * Factory for creating CombinedKey serializers / deserializers.
  */
 public class CombinedKeySchema {
-    private final String primaryKeySerdeTopic;
-    private final String foreignKeySerdeTopic;
+    private final Supplier undecoratedPrimaryKeySerdeTopicSupplier;
+    private final Supplier undecoratedForeignKeySerdeTopicSupplier;
+    private String primaryKeySerdeTopic;
+    private String foreignKeySerdeTopic;
     private Serializer primaryKeySerializer;
     private Deserializer primaryKeyDeserializer;
     private Serializer foreignKeySerializer;
     private Deserializer foreignKeyDeserializer;
 
-    public CombinedKeySchema(final String foreignKeySerdeTopic,
+    public CombinedKeySchema(final Supplier foreignKeySerdeTopicSupplier,
                              final Serde foreignKeySerde,
-                             final String primaryKeySerdeTopic,
+                             final Supplier primaryKeySerdeTopicSupplier,
                              final Serde primaryKeySerde) {
-        this.primaryKeySerdeTopic = primaryKeySerdeTopic;
-        this.foreignKeySerdeTopic = foreignKeySerdeTopic;
+        undecoratedPrimaryKeySerdeTopicSupplier = primaryKeySerdeTopicSupplier;
+        undecoratedForeignKeySerdeTopicSupplier = foreignKeySerdeTopicSupplier;
         primaryKeySerializer = primaryKeySerde == null ? null : primaryKeySerde.serializer();
         primaryKeyDeserializer = primaryKeySerde == null ? null : primaryKeySerde.deserializer();
         foreignKeyDeserializer = foreignKeySerde == null ? null : foreignKeySerde.deserializer();
@@ -49,6 +52,8 @@ public CombinedKeySchema(final String foreignKeySerdeTopic,
 
     @SuppressWarnings("unchecked")
     public void init(final ProcessorContext context) {
+        primaryKeySerdeTopic = undecoratedPrimaryKeySerdeTopicSupplier.get();
+        foreignKeySerdeTopic = undecoratedForeignKeySerdeTopicSupplier.get();
         primaryKeySerializer = primaryKeySerializer == null ? (Serializer) context.keySerde().serializer() : primaryKeySerializer;
         primaryKeyDeserializer = primaryKeyDeserializer == null ? (Deserializer) context.keySerde().deserializer() : primaryKeyDeserializer;
         foreignKeySerializer = foreignKeySerializer == null ? (Serializer) context.keySerde().serializer() : foreignKeySerializer;
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionSendProcessorSupplier.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionSendProcessorSupplier.java
index 0bd0d88cb09e1..40c0ee42361bd 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionSendProcessorSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/ForeignJoinSubscriptionSendProcessorSupplier.java
@@ -33,6 +33,7 @@
 
 import java.util.Arrays;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 import static org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionWrapper.Instruction.DELETE_KEY_AND_PROPAGATE;
 import static org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionWrapper.Instruction.DELETE_KEY_NO_PROPAGATE;
@@ -43,21 +44,21 @@ public class ForeignJoinSubscriptionSendProcessorSupplier implements P
     private static final Logger LOG = LoggerFactory.getLogger(ForeignJoinSubscriptionSendProcessorSupplier.class);
 
     private final Function foreignKeyExtractor;
-    private final String foreignKeySerdeTopic;
-    private final String valueSerdeTopic;
+    private final Supplier foreignKeySerdeTopicSupplier;
+    private final Supplier valueSerdeTopicSupplier;
     private final boolean leftJoin;
     private Serializer foreignKeySerializer;
     private Serializer valueSerializer;
 
     public ForeignJoinSubscriptionSendProcessorSupplier(final Function foreignKeyExtractor,
-                                                        final String foreignKeySerdeTopic,
-                                                        final String valueSerdeTopic,
+                                                        final Supplier foreignKeySerdeTopicSupplier,
+                                                        final Supplier valueSerdeTopicSupplier,
                                                         final Serde foreignKeySerde,
                                                         final Serializer valueSerializer,
                                                         final boolean leftJoin) {
         this.foreignKeyExtractor = foreignKeyExtractor;
-        this.foreignKeySerdeTopic = foreignKeySerdeTopic;
-        this.valueSerdeTopic = valueSerdeTopic;
+        this.foreignKeySerdeTopicSupplier = foreignKeySerdeTopicSupplier;
+        this.valueSerdeTopicSupplier = valueSerdeTopicSupplier;
         this.valueSerializer = valueSerializer;
         this.leftJoin = leftJoin;
         foreignKeySerializer = foreignKeySerde == null ? null : foreignKeySerde.serializer();
@@ -71,11 +72,15 @@ public Processor> get() {
     private class UnbindChangeProcessor extends AbstractProcessor> {
 
         private Sensor skippedRecordsSensor;
+        private String foreignKeySerdeTopic;
+        private String valueSerdeTopic;
 
         @SuppressWarnings("unchecked")
         @Override
         public void init(final ProcessorContext context) {
             super.init(context);
+            foreignKeySerdeTopic = foreignKeySerdeTopicSupplier.get();
+            valueSerdeTopic = valueSerdeTopicSupplier.get();
             // get default key serde if it wasn't supplied directly at construction
             if (foreignKeySerializer == null) {
                 foreignKeySerializer = (Serializer) context.keySerde().serializer();
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplier.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplier.java
index 31de0687d6ef7..3cd06368a7131 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplier.java
@@ -29,6 +29,8 @@
 import org.apache.kafka.streams.state.ValueAndTimestamp;
 import org.apache.kafka.streams.state.internals.Murmur3;
 
+import java.util.function.Supplier;
+
 /**
  * Receives {@code SubscriptionResponseWrapper} events and filters out events which do not match the current hash
  * of the primary key. This eliminates race-condition results for rapidly-changing foreign-keys for a given primary key.
@@ -42,18 +44,18 @@
 public class SubscriptionResolverJoinProcessorSupplier implements ProcessorSupplier> {
     private final KTableValueGetterSupplier valueGetterSupplier;
     private final Serializer constructionTimeValueSerializer;
-    private final String valueHashSerdePseudoTopic;
+    private final Supplier valueHashSerdePseudoTopicSupplier;
     private final ValueJoiner joiner;
     private final boolean leftJoin;
 
     public SubscriptionResolverJoinProcessorSupplier(final KTableValueGetterSupplier valueGetterSupplier,
                                                      final Serializer valueSerializer,
-                                                     final String valueHashSerdePseudoTopic,
+                                                     final Supplier valueHashSerdePseudoTopicSupplier,
                                                      final ValueJoiner joiner,
                                                      final boolean leftJoin) {
         this.valueGetterSupplier = valueGetterSupplier;
         constructionTimeValueSerializer = valueSerializer;
-        this.valueHashSerdePseudoTopic = valueHashSerdePseudoTopic;
+        this.valueHashSerdePseudoTopicSupplier = valueHashSerdePseudoTopicSupplier;
         this.joiner = joiner;
         this.leftJoin = leftJoin;
     }
@@ -61,6 +63,7 @@ public SubscriptionResolverJoinProcessorSupplier(final KTableValueGetterSupplier
     @Override
     public Processor> get() {
         return new AbstractProcessor>() {
+            private String valueHashSerdePseudoTopic;
             private Serializer runtimeValueSerializer = constructionTimeValueSerializer;
 
             private KTableValueGetter valueGetter;
@@ -69,6 +72,7 @@ public Processor> get() {
             @Override
             public void init(final ProcessorContext context) {
                 super.init(context);
+                valueHashSerdePseudoTopic = valueHashSerdePseudoTopicSupplier.get();
                 valueGetter = valueGetterSupplier.get();
                 valueGetter.init(context);
                 if (runtimeValueSerializer == null) {
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerde.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerde.java
index 42aed940ef4bc..136128c53bbee 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerde.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerde.java
@@ -25,16 +25,17 @@
 
 import java.nio.ByteBuffer;
 import java.util.Objects;
+import java.util.function.Supplier;
 
 public class SubscriptionWrapperSerde implements Serde> {
     private final SubscriptionWrapperSerializer serializer;
     private final SubscriptionWrapperDeserializer deserializer;
 
-    public SubscriptionWrapperSerde(final String primaryKeySerializationPseudoTopic,
+    public SubscriptionWrapperSerde(final Supplier primaryKeySerializationPseudoTopicSupplier,
                                     final Serde primaryKeySerde) {
-        serializer = new SubscriptionWrapperSerializer<>(primaryKeySerializationPseudoTopic,
+        serializer = new SubscriptionWrapperSerializer<>(primaryKeySerializationPseudoTopicSupplier,
                                                          primaryKeySerde == null ? null : primaryKeySerde.serializer());
-        deserializer = new SubscriptionWrapperDeserializer<>(primaryKeySerializationPseudoTopic,
+        deserializer = new SubscriptionWrapperDeserializer<>(primaryKeySerializationPseudoTopicSupplier,
                                                              primaryKeySerde == null ? null : primaryKeySerde.deserializer());
     }
 
@@ -51,12 +52,13 @@ public Deserializer> deserializer() {
     private static class SubscriptionWrapperSerializer
         implements Serializer>, WrappingNullableSerializer, K> {
 
-        private final String primaryKeySerializationPseudoTopic;
+        private final Supplier primaryKeySerializationPseudoTopicSupplier;
+        private String primaryKeySerializationPseudoTopic = null;
         private Serializer primaryKeySerializer;
 
-        SubscriptionWrapperSerializer(final String primaryKeySerializationPseudoTopic,
+        SubscriptionWrapperSerializer(final Supplier primaryKeySerializationPseudoTopicSupplier,
                                       final Serializer primaryKeySerializer) {
-            this.primaryKeySerializationPseudoTopic = primaryKeySerializationPseudoTopic;
+            this.primaryKeySerializationPseudoTopicSupplier = primaryKeySerializationPseudoTopicSupplier;
             this.primaryKeySerializer = primaryKeySerializer;
         }
 
@@ -76,6 +78,10 @@ public byte[] serialize(final String ignored, final SubscriptionWrapper data)
                 throw new UnsupportedVersionException("SubscriptionWrapper version is larger than maximum supported 0x7F");
             }
 
+            if (primaryKeySerializationPseudoTopic == null) {
+                primaryKeySerializationPseudoTopic = primaryKeySerializationPseudoTopicSupplier.get();
+            }
+
             final byte[] primaryKeySerializedData = primaryKeySerializer.serialize(
                 primaryKeySerializationPseudoTopic,
                 data.getPrimaryKey()
@@ -106,12 +112,13 @@ public byte[] serialize(final String ignored, final SubscriptionWrapper data)
     private static class SubscriptionWrapperDeserializer
         implements Deserializer>, WrappingNullableDeserializer, K> {
 
-        private final String primaryKeySerializationPseudoTopic;
+        private final Supplier primaryKeySerializationPseudoTopicSupplier;
+        private String primaryKeySerializationPseudoTopic = null;
         private Deserializer primaryKeyDeserializer;
 
-        SubscriptionWrapperDeserializer(final String primaryKeySerializationPseudoTopic,
+        SubscriptionWrapperDeserializer(final Supplier primaryKeySerializationPseudoTopicSupplier,
                                         final Deserializer primaryKeyDeserializer) {
-            this.primaryKeySerializationPseudoTopic = primaryKeySerializationPseudoTopic;
+            this.primaryKeySerializationPseudoTopicSupplier = primaryKeySerializationPseudoTopicSupplier;
             this.primaryKeyDeserializer = primaryKeyDeserializer;
         }
 
@@ -144,6 +151,11 @@ public SubscriptionWrapper deserialize(final String ignored, final byte[] dat
 
             final byte[] primaryKeyRaw = new byte[data.length - lengthSum]; //The remaining data is the serialized pk
             buf.get(primaryKeyRaw, 0, primaryKeyRaw.length);
+
+            if (primaryKeySerializationPseudoTopic == null) {
+                primaryKeySerializationPseudoTopic = primaryKeySerializationPseudoTopicSupplier.get();
+            }
+
             final K primaryKey = primaryKeyDeserializer.deserialize(primaryKeySerializationPseudoTopic,
                                                                     primaryKeyRaw);
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java
index 5754b4a9f1ec2..d28aee8907b3d 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java
@@ -1190,6 +1190,10 @@ private List maybeDecorateInternalSourceTopics(final Collection
         return decoratedTopics;
     }
 
+    public String decoratePseudoTopic(final String topic) {
+        return decorateTopic(topic);
+    }
+
     private String decorateTopic(final String topic) {
         if (applicationId == null) {
             throw new TopologyException("there are internal topics and "
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinDefaultSerdeTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinDefaultSerdeTest.java
index 5af0530ca53a0..64c6b06b80409 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinDefaultSerdeTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinDefaultSerdeTest.java
@@ -24,12 +24,14 @@
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.TestInputTopic;
 import org.apache.kafka.streams.TestOutputTopic;
+import org.apache.kafka.streams.Topology;
 import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.KTable;
 import org.apache.kafka.streams.kstream.Materialized;
 import org.apache.kafka.streams.kstream.Produced;
 import org.apache.kafka.streams.state.KeyValueStore;
+import org.apache.kafka.streams.utils.UniqueTopicSerdeScope;
 import org.apache.kafka.test.TestUtils;
 import org.junit.Test;
 
@@ -38,10 +40,15 @@
 import java.util.Properties;
 import java.util.UUID;
 
+import static org.apache.kafka.common.utils.Utils.mkEntry;
+import static org.apache.kafka.common.utils.Utils.mkMap;
+import static org.apache.kafka.common.utils.Utils.mkProperties;
+import static org.apache.kafka.common.utils.Utils.mkSet;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.is;
 
 public class KTableKTableForeignKeyJoinDefaultSerdeTest {
+
     @Test
     public void shouldWorkWithDefaultSerdes() {
         final StreamsBuilder builder = new StreamsBuilder();
@@ -161,7 +168,72 @@ public void shouldWorkWithDefaultAndProducedSerdes() {
         validateTopologyCanProcessData(builder);
     }
 
-    private static void validateTopologyCanProcessData(final StreamsBuilder builder) {
+    @Test
+    public void shouldUseExpectedTopicsWithSerde() {
+        final String applicationId = "ktable-ktable-joinOnForeignKey";
+        final Properties streamsConfig = mkProperties(mkMap(
+            mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, applicationId),
+            mkEntry(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "asdf:0000"),
+            mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath())
+        ));
+
+        final UniqueTopicSerdeScope serdeScope = new UniqueTopicSerdeScope();
+        final StreamsBuilder builder = new StreamsBuilder();
+
+        final String leftTable = "left_table";
+        final String rightTable = "right_table";
+        final String output = "output-topic";
+
+        final KTable left = builder.table(
+            leftTable,
+            Consumed.with(serdeScope.decorateSerde(Serdes.String(), streamsConfig, true),
+                          serdeScope.decorateSerde(Serdes.String(), streamsConfig, false))
+        );
+        final KTable right = builder.table(
+            rightTable,
+            Consumed.with(serdeScope.decorateSerde(Serdes.String(), streamsConfig, true),
+                          serdeScope.decorateSerde(Serdes.String(), streamsConfig, false))
+        );
+
+        left.join(
+            right,
+            value -> value.split("\\|")[1],
+            (value1, value2) -> "(" + value1 + "," + value2 + ")",
+            Materialized.with(null, serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)
+            ))
+            .toStream()
+            .to(output);
+
+
+        final Topology topology = builder.build(streamsConfig);
+        try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
+            final TestInputTopic leftInput = driver.createInputTopic(leftTable, new StringSerializer(), new StringSerializer());
+            final TestInputTopic rightInput = driver.createInputTopic(rightTable, new StringSerializer(), new StringSerializer());
+            leftInput.pipeInput("lhs1", "lhsValue1|rhs1");
+            rightInput.pipeInput("rhs1", "rhsValue1");
+        }
+        // verifying primarily that no extra pseudo-topics were used, but it's nice to also verify the rest of the
+        // topics our serdes serialize data for
+        assertThat(serdeScope.registeredTopics(), is(mkSet(
+            // expected pseudo-topics
+            applicationId + "-KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic-fk--key",
+            applicationId + "-KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic-pk--key",
+            applicationId + "-KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic-vh--value",
+            // internal topics
+            applicationId + "-KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic--key",
+            applicationId + "-KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000014-topic--key",
+            applicationId + "-KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000014-topic--value",
+            applicationId + "-left_table-STATE-STORE-0000000000-changelog--key",
+            applicationId + "-left_table-STATE-STORE-0000000000-changelog--value",
+            applicationId + "-right_table-STATE-STORE-0000000003-changelog--key",
+            applicationId + "-right_table-STATE-STORE-0000000003-changelog--value",
+            // output topics
+            "output-topic--key",
+            "output-topic--value"
+        )));
+    }
+
+    private void validateTopologyCanProcessData(final StreamsBuilder builder) {
         final Properties config = new Properties();
         config.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "dummy-" + UUID.randomUUID());
         config.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy");
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinPseudoTopicTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinPseudoTopicTest.java
deleted file mode 100644
index b08c29380b2c7..0000000000000
--- a/streams/src/test/java/org/apache/kafka/streams/integration/KTableKTableForeignKeyJoinPseudoTopicTest.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.streams.integration;
-
-import org.apache.kafka.common.serialization.Serdes;
-import org.apache.kafka.common.serialization.StringSerializer;
-import org.apache.kafka.streams.StreamsBuilder;
-import org.apache.kafka.streams.StreamsConfig;
-import org.apache.kafka.streams.TestInputTopic;
-import org.apache.kafka.streams.Topology;
-import org.apache.kafka.streams.TopologyTestDriver;
-import org.apache.kafka.streams.kstream.Consumed;
-import org.apache.kafka.streams.kstream.KTable;
-import org.apache.kafka.streams.kstream.Materialized;
-import org.apache.kafka.streams.utils.UniqueTopicSerdeScope;
-import org.apache.kafka.test.TestUtils;
-import org.junit.Test;
-
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Properties;
-
-import static org.apache.kafka.common.utils.Utils.mkEntry;
-import static org.apache.kafka.common.utils.Utils.mkMap;
-import static org.apache.kafka.common.utils.Utils.mkProperties;
-import static org.apache.kafka.common.utils.Utils.mkSet;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.MatcherAssert.assertThat;
-
-
-public class KTableKTableForeignKeyJoinPseudoTopicTest {
-
-    private static final String LEFT_TABLE = "left_table";
-    private static final String RIGHT_TABLE = "right_table";
-    private static final String OUTPUT = "output-topic";
-    private static final String REJOIN_OUTPUT = "rejoin-output-topic";
-    private final Properties streamsConfig = mkProperties(mkMap(
-        mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, "ktable-ktable-joinOnForeignKey"),
-        mkEntry(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "asdf:0000"),
-        mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath())
-    ));
-
-
-    private static Collection buildParameters(final List... argOptions) {
-        List result = new LinkedList<>();
-        result.add(new Object[0]);
-
-        for (final List argOption : argOptions) {
-            result = times(result, argOption);
-        }
-
-        return result;
-    }
-
-    private static List times(final List left, final List right) {
-        final List result = new LinkedList<>();
-        for (final Object[] args : left) {
-            for (final Object rightElem : right) {
-                final Object[] resArgs = new Object[args.length + 1];
-                System.arraycopy(args, 0, resArgs, 0, args.length);
-                resArgs[args.length] = rightElem;
-                result.add(resArgs);
-            }
-        }
-        return result;
-    }
-
-
-    @Test
-    public void shouldUseExpectedTopicsWithSerde() {
-        final UniqueTopicSerdeScope serdeScope = new UniqueTopicSerdeScope();
-        final StreamsBuilder builder = new StreamsBuilder();
-
-        final KTable left = builder.table(
-            LEFT_TABLE,
-            Consumed.with(serdeScope.decorateSerde(Serdes.String(), streamsConfig, true),
-                          serdeScope.decorateSerde(Serdes.String(), streamsConfig, false))
-        );
-        final KTable right = builder.table(
-            RIGHT_TABLE,
-            Consumed.with(serdeScope.decorateSerde(Serdes.String(), streamsConfig, true),
-                          serdeScope.decorateSerde(Serdes.String(), streamsConfig, false))
-        );
-
-        left.join(
-            right,
-            value -> value.split("\\|")[1],
-            (value1, value2) -> "(" + value1 + "," + value2 + ")",
-            Materialized.with(null, serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)
-            ))
-            .toStream()
-            .to(OUTPUT);
-
-
-        final Topology topology = builder.build(streamsConfig);
-        try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
-            final TestInputTopic leftInput = driver.createInputTopic(LEFT_TABLE, new StringSerializer(), new StringSerializer());
-            final TestInputTopic rightInput = driver.createInputTopic(RIGHT_TABLE, new StringSerializer(), new StringSerializer());
-            leftInput.pipeInput("lhs1", "lhsValue1|rhs1");
-            rightInput.pipeInput("rhs1", "rhsValue1");
-        }
-        // verifying primarily that no extra pseudo-topics were used, but it's nice to also verify the rest of the
-        // topics our serdes serialize data for
-        assertThat(serdeScope.registeredTopics(), is(mkSet(
-            // expected pseudo-topics
-            "KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic-fk--key",
-            "KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic-pk--key",
-            "KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic-vh--value",
-            // internal topics
-            "ktable-ktable-joinOnForeignKey-KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic--key",
-            "ktable-ktable-joinOnForeignKey-KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000014-topic--key",
-            "ktable-ktable-joinOnForeignKey-KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000014-topic--value",
-            "ktable-ktable-joinOnForeignKey-left_table-STATE-STORE-0000000000-changelog--key",
-            "ktable-ktable-joinOnForeignKey-left_table-STATE-STORE-0000000000-changelog--value",
-            "ktable-ktable-joinOnForeignKey-right_table-STATE-STORE-0000000003-changelog--key",
-            "ktable-ktable-joinOnForeignKey-right_table-STATE-STORE-0000000003-changelog--value",
-            // output topics
-            "output-topic--key",
-            "output-topic--value"
-        )));
-    }
-
-}
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchemaTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchemaTest.java
index cb1ef596db3f0..17f0c7969b4a8 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchemaTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/CombinedKeySchemaTest.java
@@ -28,8 +28,10 @@ public class CombinedKeySchemaTest {
 
     @Test
     public void nonNullPrimaryKeySerdeTest() {
-        final CombinedKeySchema cks = new CombinedKeySchema<>("fkTopic", Serdes.String(),
-                                                                               "pkTopic", Serdes.Integer());
+        final CombinedKeySchema cks = new CombinedKeySchema<>(
+            () -> "fkTopic", Serdes.String(),
+            () -> "pkTopic", Serdes.Integer()
+        );
         final Integer primary = -999;
         final Bytes result = cks.toBytes("foreignKey", primary);
 
@@ -40,22 +42,28 @@ public void nonNullPrimaryKeySerdeTest() {
 
     @Test(expected = NullPointerException.class)
     public void nullPrimaryKeySerdeTest() {
-        final CombinedKeySchema cks = new CombinedKeySchema<>("fkTopic", Serdes.String(),
-                                                                               "pkTopic", Serdes.Integer());
+        final CombinedKeySchema cks = new CombinedKeySchema<>(
+            () -> "fkTopic", Serdes.String(),
+            () -> "pkTopic", Serdes.Integer()
+        );
         cks.toBytes("foreignKey", null);
     }
 
     @Test(expected = NullPointerException.class)
     public void nullForeignKeySerdeTest() {
-        final CombinedKeySchema cks = new CombinedKeySchema<>("fkTopic", Serdes.String(),
-                                                                               "pkTopic", Serdes.Integer());
+        final CombinedKeySchema cks = new CombinedKeySchema<>(
+            () -> "fkTopic", Serdes.String(),
+            () -> "pkTopic", Serdes.Integer()
+        );
         cks.toBytes(null, 10);
     }
 
     @Test
     public void prefixKeySerdeTest() {
-        final CombinedKeySchema cks = new CombinedKeySchema<>("fkTopic", Serdes.String(),
-                                                                               "pkTopic", Serdes.Integer());
+        final CombinedKeySchema cks = new CombinedKeySchema<>(
+            () -> "fkTopic", Serdes.String(),
+            () -> "pkTopic", Serdes.Integer()
+        );
         final String foreignKey = "someForeignKey";
         final byte[] foreignKeySerializedData =
             Serdes.String().serializer().serialize("fkTopic", foreignKey);
@@ -71,8 +79,10 @@ public void prefixKeySerdeTest() {
 
     @Test(expected = NullPointerException.class)
     public void nullPrefixKeySerdeTest() {
-        final CombinedKeySchema cks = new CombinedKeySchema<>("fkTopic", Serdes.String(),
-                                                                               "pkTopic", Serdes.Integer());
+        final CombinedKeySchema cks = new CombinedKeySchema<>(
+            () -> "fkTopic", Serdes.String(),
+            () -> "pkTopic", Serdes.Integer()
+        );
         final String foreignKey = null;
         cks.prefixBytes(foreignKey);
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplierTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplierTest.java
index aae99ec6687d7..b95569f7906c8 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplierTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionResolverJoinProcessorSupplierTest.java
@@ -82,7 +82,7 @@ public void shouldNotForwardWhenHashDoesNotMatch() {
             new SubscriptionResolverJoinProcessorSupplier<>(
                 valueGetterSupplier,
                 STRING_SERIALIZER,
-                "value-hash-dummy-topic",
+                () -> "value-hash-dummy-topic",
                 JOINER,
                 leftJoin
             );
@@ -107,7 +107,7 @@ public void shouldIgnoreUpdateWhenLeftHasBecomeNull() {
             new SubscriptionResolverJoinProcessorSupplier<>(
                 valueGetterSupplier,
                 STRING_SERIALIZER,
-                "value-hash-dummy-topic",
+                () -> "value-hash-dummy-topic",
                 JOINER,
                 leftJoin
             );
@@ -132,7 +132,7 @@ public void shouldForwardWhenHashMatches() {
             new SubscriptionResolverJoinProcessorSupplier<>(
                 valueGetterSupplier,
                 STRING_SERIALIZER,
-                "value-hash-dummy-topic",
+                () -> "value-hash-dummy-topic",
                 JOINER,
                 leftJoin
             );
@@ -158,7 +158,7 @@ public void shouldEmitTombstoneForInnerJoinWhenRightIsNull() {
             new SubscriptionResolverJoinProcessorSupplier<>(
                 valueGetterSupplier,
                 STRING_SERIALIZER,
-                "value-hash-dummy-topic",
+                () -> "value-hash-dummy-topic",
                 JOINER,
                 leftJoin
             );
@@ -184,7 +184,7 @@ public void shouldEmitResultForLeftJoinWhenRightIsNull() {
             new SubscriptionResolverJoinProcessorSupplier<>(
                 valueGetterSupplier,
                 STRING_SERIALIZER,
-                "value-hash-dummy-topic",
+                () -> "value-hash-dummy-topic",
                 JOINER,
                 leftJoin
             );
@@ -210,7 +210,7 @@ public void shouldEmitTombstoneForLeftJoinWhenRightIsNullAndLeftIsNull() {
             new SubscriptionResolverJoinProcessorSupplier<>(
                 valueGetterSupplier,
                 STRING_SERIALIZER,
-                "value-hash-dummy-topic",
+                () -> "value-hash-dummy-topic",
                 JOINER,
                 leftJoin
             );
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerdeTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerdeTest.java
index 5c6551c3e7db5..dd67b4bd55594 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerdeTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/foreignkeyjoin/SubscriptionWrapperSerdeTest.java
@@ -31,7 +31,7 @@ public class SubscriptionWrapperSerdeTest {
     @SuppressWarnings("unchecked")
     public void shouldSerdeTest() {
         final String originalKey = "originalKey";
-        final SubscriptionWrapperSerde swSerde = new SubscriptionWrapperSerde<>("pkTopic", Serdes.String());
+        final SubscriptionWrapperSerde swSerde = new SubscriptionWrapperSerde<>(() -> "pkTopic", Serdes.String());
         final long[] hashedValue = Murmur3.hash128(new byte[] {(byte) 0xFF, (byte) 0xAA, (byte) 0x00, (byte) 0x19});
         final SubscriptionWrapper wrapper = new SubscriptionWrapper<>(hashedValue, SubscriptionWrapper.Instruction.DELETE_KEY_AND_PROPAGATE, originalKey);
         final byte[] serialized = swSerde.serializer().serialize(null, wrapper);
@@ -46,7 +46,7 @@ public void shouldSerdeTest() {
     @SuppressWarnings("unchecked")
     public void shouldSerdeNullHashTest() {
         final String originalKey = "originalKey";
-        final SubscriptionWrapperSerde swSerde = new SubscriptionWrapperSerde<>("pkTopic", Serdes.String());
+        final SubscriptionWrapperSerde swSerde = new SubscriptionWrapperSerde<>(() -> "pkTopic", Serdes.String());
         final long[] hashedValue = null;
         final SubscriptionWrapper wrapper = new SubscriptionWrapper<>(hashedValue, SubscriptionWrapper.Instruction.PROPAGATE_ONLY_IF_FK_VAL_AVAILABLE, originalKey);
         final byte[] serialized = swSerde.serializer().serialize(null, wrapper);
@@ -61,7 +61,7 @@ public void shouldSerdeNullHashTest() {
     @SuppressWarnings("unchecked")
     public void shouldThrowExceptionOnNullKeyTest() {
         final String originalKey = null;
-        final SubscriptionWrapperSerde swSerde = new SubscriptionWrapperSerde<>("pkTopic", Serdes.String());
+        final SubscriptionWrapperSerde swSerde = new SubscriptionWrapperSerde<>(() -> "pkTopic", Serdes.String());
         final long[] hashedValue = Murmur3.hash128(new byte[] {(byte) 0xFF, (byte) 0xAA, (byte) 0x00, (byte) 0x19});
         final SubscriptionWrapper wrapper = new SubscriptionWrapper<>(hashedValue, SubscriptionWrapper.Instruction.PROPAGATE_ONLY_IF_FK_VAL_AVAILABLE, originalKey);
         swSerde.serializer().serialize(null, wrapper);
@@ -71,7 +71,7 @@ public void shouldThrowExceptionOnNullKeyTest() {
     @SuppressWarnings("unchecked")
     public void shouldThrowExceptionOnNullInstructionTest() {
         final String originalKey = "originalKey";
-        final SubscriptionWrapperSerde swSerde = new SubscriptionWrapperSerde<>("pkTopic", Serdes.String());
+        final SubscriptionWrapperSerde swSerde = new SubscriptionWrapperSerde<>(() -> "pkTopic", Serdes.String());
         final long[] hashedValue = Murmur3.hash128(new byte[] {(byte) 0xFF, (byte) 0xAA, (byte) 0x00, (byte) 0x19});
         final SubscriptionWrapper wrapper = new SubscriptionWrapper<>(hashedValue, null, originalKey);
         swSerde.serializer().serialize(null, wrapper);

From b27c363ce527895e16f8b9c653fcfc08fb12f490 Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Wed, 29 Apr 2020 17:11:24 -0500
Subject: [PATCH 0940/1071] MINOR: ignore streams generated directory from 2.5+

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index f640e8d09d63e..4e717b9549772 100644
--- a/.gitignore
+++ b/.gitignore
@@ -53,3 +53,4 @@ systest/
 *.swp
 clients/src/generated
 clients/src/generated-test
+streams/src/generated/

From 45063c03f1c01b2f04fbced9023d8c1079dc0457 Mon Sep 17 00:00:00 2001
From: showuon <43372967+showuon@users.noreply.github.com>
Date: Thu, 30 Apr 2020 11:25:33 +0800
Subject: [PATCH 0941/1071] MINOR: Fix broken JMX link in docs by adding
 missing starting double quote (#8587)

The href attribute missed the starting double quote, so the hyperlink is interpreted as https://docs.oracle.com/.../agent.html", with a redundant tailing double quote.

Add the missing starting double quote back to fix this issue.

Reviewers: Konstantine Karantasis 
---
 docs/ops.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/ops.html b/docs/ops.html
index 5cb1f65b6ed42..6f84186b299ae 100644
--- a/docs/ops.html
+++ b/docs/ops.html
@@ -794,7 +794,7 @@ 
          Security Considerations for Remote Mon
   control your broker or application as well as the platform on which these are running. Note that authentication is disabled for
   JMX by default in Kafka and security configs must be overridden for production deployments by setting the environment variable
   KAFKA_JMX_OPTS for processes started using the CLI or by setting appropriate Java system properties. See
-  Monitoring and Management Using JMX Technology
+  Monitoring and Management Using JMX Technology
   for details on securing JMX.
   
          
   We do graphing and alerting on the following metrics:

From d76883c066cbd43c10fd2b18709a8f374fdcf5b0 Mon Sep 17 00:00:00 2001
From: Chia-Ping Tsai 
Date: Thu, 30 Apr 2020 13:37:05 +0800
Subject: [PATCH 0942/1071] MINOR: Fix unused arguments used in formatted
 string and log messages (#8036)

Reviewers: Ron Dagostino , Ismael Juma , Konstantine Karantasis 
---
 .../org/apache/kafka/clients/admin/KafkaAdminClient.java    | 2 +-
 .../main/java/org/apache/kafka/connect/runtime/Worker.java  | 2 +-
 .../main/java/org/apache/kafka/connect/util/TopicAdmin.java | 6 +++---
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
index c201fe65c224b..1da0e66a07193 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
@@ -2809,7 +2809,7 @@ void handleResponse(AbstractResponse abstractResponse) {
                     context.getFuture().complete(consumerGroupDescription);
                 } else {
                     context.getFuture().completeExceptionally(new IllegalArgumentException(
-                        String.format("GroupId {} is not a consumer group ({}).",
+                        String.format("GroupId %s is not a consumer group (%s).",
                             context.getGroupId(), protocolType)));
                 }
             }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
index e496d416ef11f..3c4ece7755f6b 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
@@ -537,7 +537,7 @@ private WorkerTask buildWorkerTask(ClusterConfigState configState,
                                       valueConverter, headerConverter, transformationChain, consumer, loader, time,
                                       retryWithToleranceOperator);
         } else {
-            log.error("Tasks must be a subclass of either SourceTask or SinkTask", task);
+            log.error("Tasks must be a subclass of either SourceTask or SinkTask and current is {}", task);
             throw new ConnectException("Tasks must be a subclass of either SourceTask or SinkTask");
         }
     }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/util/TopicAdmin.java b/connect/runtime/src/main/java/org/apache/kafka/connect/util/TopicAdmin.java
index 1fa04de981f16..e644e805ea613 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/util/TopicAdmin.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/util/TopicAdmin.java
@@ -236,19 +236,19 @@ public Set createTopics(NewTopic... topics) {
                     continue;
                 }
                 if (cause instanceof UnsupportedVersionException) {
-                    log.debug("Unable to create topic(s) '{}' since the brokers at {} do not support the CreateTopics API.",
+                    log.debug("Unable to create topic(s) '{}' since the brokers at {} do not support the CreateTopics API." +
                             " Falling back to assume topic(s) exist or will be auto-created by the broker.",
                             topicNameList, bootstrapServers);
                     return Collections.emptySet();
                 }
                 if (cause instanceof ClusterAuthorizationException) {
-                    log.debug("Not authorized to create topic(s) '{}'." +
+                    log.debug("Not authorized to create topic(s) '{}' upon the brokers {}." +
                             " Falling back to assume topic(s) exist or will be auto-created by the broker.",
                             topicNameList, bootstrapServers);
                     return Collections.emptySet();
                 }
                 if (cause instanceof TopicAuthorizationException) {
-                    log.debug("Not authorized to create topic(s) '{}'." +
+                    log.debug("Not authorized to create topic(s) '{}' upon the brokers {}." +
                                     " Falling back to assume topic(s) exist or will be auto-created by the broker.",
                             topicNameList, bootstrapServers);
                     return Collections.emptySet();

From 4e32bc85c44eda0820d73acd0d13a8a04dcde99c Mon Sep 17 00:00:00 2001
From: Greg Harris 
Date: Wed, 29 Apr 2020 17:07:01 -0700
Subject: [PATCH 0943/1071] KAFKA-9830: Implement AutoCloseable in
 ErrorReporter and subclasses (#8442)

* The DeadLetterQueueReporter has a KafkaProducer that it must close to clean up resources
* Currently, the producer and its threads are leaked every time a task is stopped
* Responsibility for cleaning up ErrorReporters is transitively assigned to the
    ProcessingContext, RetryWithToleranceOperator, and WorkerSinkTask/WorkerSinkTask classes
* One new unit test in ErrorReporterTest asserts that the producer is closed by the dlq reporter

Reviewers: Arjun Satish , Chris Egerton , Chia-Ping Tsai , Konstantine Karantasis 
---
 .../kafka/connect/runtime/WorkerSinkTask.java |  2 +
 .../connect/runtime/WorkerSourceTask.java     |  2 +
 .../errors/DeadLetterQueueReporter.java       |  5 ++
 .../connect/runtime/errors/ErrorReporter.java |  4 +-
 .../runtime/errors/ProcessingContext.java     | 18 +++-
 .../errors/RetryWithToleranceOperator.java    |  7 +-
 .../runtime/ErrorHandlingTaskTest.java        | 89 +++++++++++++++++++
 .../runtime/errors/ErrorReporterTest.java     | 14 +++
 8 files changed, 138 insertions(+), 3 deletions(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
index 9a71a6658a3b3..f89918c220e85 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
@@ -32,6 +32,7 @@
 import org.apache.kafka.common.metrics.stats.Rate;
 import org.apache.kafka.common.metrics.stats.Value;
 import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.connect.data.SchemaAndValue;
 import org.apache.kafka.connect.errors.ConnectException;
 import org.apache.kafka.connect.errors.RetriableException;
@@ -171,6 +172,7 @@ protected void close() {
         } catch (Throwable t) {
             log.warn("Could not close transformation chain", t);
         }
+        Utils.closeQuietly(retryWithToleranceOperator, "retry operator");
     }
 
     @Override
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
index cea99ee79744a..c3739b5efd2e6 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
@@ -29,6 +29,7 @@
 import org.apache.kafka.common.metrics.stats.Rate;
 import org.apache.kafka.common.metrics.stats.Value;
 import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.connect.errors.ConnectException;
 import org.apache.kafka.connect.errors.RetriableException;
 import org.apache.kafka.connect.header.Header;
@@ -165,6 +166,7 @@ protected void close() {
         } catch (Throwable t) {
             log.warn("Could not close transformation chain", t);
         }
+        Utils.closeQuietly(retryWithToleranceOperator, "retry operator");
     }
 
     @Override
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/DeadLetterQueueReporter.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/DeadLetterQueueReporter.java
index fc6181d4904de..20ed2f2d577f2 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/DeadLetterQueueReporter.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/DeadLetterQueueReporter.java
@@ -204,4 +204,9 @@ private byte[] toBytes(String value) {
             return null;
         }
     }
+
+    @Override
+    public void close() {
+        kafkaProducer.close();
+    }
 }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/ErrorReporter.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/ErrorReporter.java
index 58336163fbf4a..5eaa42716b6b3 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/ErrorReporter.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/ErrorReporter.java
@@ -19,7 +19,7 @@
 /**
  * Report an error using the information contained in the {@link ProcessingContext}.
  */
-public interface ErrorReporter {
+public interface ErrorReporter extends AutoCloseable {
 
     /**
      * Report an error.
@@ -28,4 +28,6 @@ public interface ErrorReporter {
      */
     void report(ProcessingContext context);
 
+    @Override
+    default void close() { }
 }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/ProcessingContext.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/ProcessingContext.java
index f826d74fe70cb..e7fb03185f85e 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/ProcessingContext.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/ProcessingContext.java
@@ -18,6 +18,7 @@
 
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.common.record.TimestampType;
+import org.apache.kafka.connect.errors.ConnectException;
 import org.apache.kafka.connect.source.SourceRecord;
 
 import java.util.Collection;
@@ -28,7 +29,7 @@
  * Contains all the metadata related to the currently evaluating operation. Only one instance of this class is meant
  * to exist per task in a JVM.
  */
-class ProcessingContext {
+class ProcessingContext implements AutoCloseable {
 
     private Collection reporters = Collections.emptyList();
 
@@ -216,4 +217,19 @@ public void reporters(Collection reporters) {
         this.reporters = reporters;
     }
 
+    @Override
+    public void close() {
+        ConnectException e = null;
+        for (ErrorReporter reporter : reporters) {
+            try {
+                reporter.close();
+            } catch (Throwable t) {
+                e = e != null ? e : new ConnectException("Failed to close all reporters");
+                e.addSuppressed(t);
+            }
+        }
+        if (e != null) {
+            throw e;
+        }
+    }
 }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/RetryWithToleranceOperator.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/RetryWithToleranceOperator.java
index 2513514475582..4e627ef2e70c9 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/RetryWithToleranceOperator.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/RetryWithToleranceOperator.java
@@ -51,7 +51,7 @@
  * then it is wrapped into a ConnectException and rethrown to the caller.
  * 
          
  */
-public class RetryWithToleranceOperator {
+public class RetryWithToleranceOperator implements AutoCloseable {
 
     private static final Logger log = LoggerFactory.getLogger(RetryWithToleranceOperator.class);
 
@@ -270,4 +270,9 @@ public void consumerRecord(ConsumerRecord consumedMessage) {
     public boolean failed() {
         return this.context.failed();
     }
+
+    @Override
+    public void close() {
+        this.context.close();
+    }
 }
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java
index 428b3e4f022f5..bb42fc6433882 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.connect.runtime;
 
+import java.util.Arrays;
 import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.clients.consumer.ConsumerRecords;
@@ -33,6 +34,7 @@
 import org.apache.kafka.connect.json.JsonConverter;
 import org.apache.kafka.connect.runtime.distributed.ClusterConfigState;
 import org.apache.kafka.connect.runtime.errors.ErrorHandlingMetrics;
+import org.apache.kafka.connect.runtime.errors.ErrorReporter;
 import org.apache.kafka.connect.runtime.errors.LogReporter;
 import org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator;
 import org.apache.kafka.connect.runtime.errors.ToleranceType;
@@ -162,6 +164,93 @@ public void tearDown() {
         }
     }
 
+    @Test
+    public void testSinkTasksCloseErrorReporters() throws Exception {
+        ErrorReporter reporter = EasyMock.mock(ErrorReporter.class);
+
+        RetryWithToleranceOperator retryWithToleranceOperator = operator();
+        retryWithToleranceOperator.metrics(errorHandlingMetrics);
+        retryWithToleranceOperator.reporters(singletonList(reporter));
+
+        createSinkTask(initialState, retryWithToleranceOperator);
+
+        expectInitializeTask();
+        reporter.close();
+        EasyMock.expectLastCall();
+        sinkTask.stop();
+        EasyMock.expectLastCall();
+
+        consumer.close();
+        EasyMock.expectLastCall();
+
+        PowerMock.replayAll();
+
+        workerSinkTask.initialize(TASK_CONFIG);
+        workerSinkTask.initializeAndStart();
+        workerSinkTask.close();
+
+        PowerMock.verifyAll();
+    }
+
+    @Test
+    public void testSourceTasksCloseErrorReporters() {
+        ErrorReporter reporter = EasyMock.mock(ErrorReporter.class);
+
+        RetryWithToleranceOperator retryWithToleranceOperator = operator();
+        retryWithToleranceOperator.metrics(errorHandlingMetrics);
+        retryWithToleranceOperator.reporters(singletonList(reporter));
+
+        createSourceTask(initialState, retryWithToleranceOperator);
+
+        sourceTask.stop();
+        PowerMock.expectLastCall();
+
+        producer.close(EasyMock.anyObject());
+        PowerMock.expectLastCall();
+
+        reporter.close();
+        EasyMock.expectLastCall();
+
+        PowerMock.replayAll();
+
+        workerSourceTask.initialize(TASK_CONFIG);
+        workerSourceTask.close();
+
+        PowerMock.verifyAll();
+    }
+
+    @Test
+    public void testCloseErrorReportersExceptionPropagation() {
+        ErrorReporter reporterA = EasyMock.mock(ErrorReporter.class);
+        ErrorReporter reporterB = EasyMock.mock(ErrorReporter.class);
+
+        RetryWithToleranceOperator retryWithToleranceOperator = operator();
+        retryWithToleranceOperator.metrics(errorHandlingMetrics);
+        retryWithToleranceOperator.reporters(Arrays.asList(reporterA, reporterB));
+
+        createSourceTask(initialState, retryWithToleranceOperator);
+
+        sourceTask.stop();
+        PowerMock.expectLastCall();
+
+        producer.close(EasyMock.anyObject());
+        PowerMock.expectLastCall();
+
+        // Even though the reporters throw exceptions, they should both still be closed.
+        reporterA.close();
+        EasyMock.expectLastCall().andThrow(new RuntimeException());
+
+        reporterB.close();
+        EasyMock.expectLastCall().andThrow(new RuntimeException());
+
+        PowerMock.replayAll();
+
+        workerSourceTask.initialize(TASK_CONFIG);
+        workerSourceTask.close();
+
+        PowerMock.verifyAll();
+    }
+
     @Test
     public void testErrorHandlingInSinkTasks() throws Exception {
         Map reportProps = new HashMap<>();
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/errors/ErrorReporterTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/errors/ErrorReporterTest.java
index 00a922f76ad97..f01cd4995c052 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/errors/ErrorReporterTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/errors/ErrorReporterTest.java
@@ -146,6 +146,20 @@ public void testReportDLQTwice() {
         PowerMock.verifyAll();
     }
 
+    @Test
+    public void testCloseDLQ() {
+        DeadLetterQueueReporter deadLetterQueueReporter = new DeadLetterQueueReporter(
+            producer, config(singletonMap(SinkConnectorConfig.DLQ_TOPIC_NAME_CONFIG, DLQ_TOPIC)), TASK_ID, errorHandlingMetrics);
+
+        producer.close();
+        EasyMock.expectLastCall();
+        replay(producer);
+
+        deadLetterQueueReporter.close();
+
+        PowerMock.verifyAll();
+    }
+
     @Test
     public void testLogOnDisabledLogReporter() {
         LogReporter logReporter = new LogReporter(TASK_ID, config(emptyMap()), errorHandlingMetrics);

From 7a72a2a6da414b4974963de89365b710c6730699 Mon Sep 17 00:00:00 2001
From: Chris Egerton 
Date: Wed, 29 Apr 2020 20:42:13 -0700
Subject: [PATCH 0944/1071] KAFKA-9919: Add logging to
 KafkaBasedLog::readToLogEnd (#8554)

Simple logging additions at TRACE level that should help when the worker can't get caught up to the end of an internal topic.

Reviewers: Gwen Shapira , Aakash Shah , Konstantine Karantasis 
---
 .../org/apache/kafka/connect/util/KafkaBasedLog.java | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java b/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java
index e78276a264dc8..e3015812a9112 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java
@@ -281,9 +281,15 @@ private void readToLogEnd() {
             Iterator> it = endOffsets.entrySet().iterator();
             while (it.hasNext()) {
                 Map.Entry entry = it.next();
-                if (consumer.position(entry.getKey()) >= entry.getValue())
+                TopicPartition topicPartition = entry.getKey();
+                long endOffset = entry.getValue();
+                long lastConsumedOffset = consumer.position(topicPartition);
+                if (lastConsumedOffset >= endOffset) {
+                    log.trace("Read to end offset {} for {}", endOffset, topicPartition);
                     it.remove();
-                else {
+                } else {
+                    log.trace("Behind end offset {} for {}; last-read offset is {}",
+                            endOffset, topicPartition, lastConsumedOffset);
                     poll(Integer.MAX_VALUE);
                     break;
                 }
@@ -345,4 +351,4 @@ public void run() {
             }
         }
     }
-}
\ No newline at end of file
+}

From ad7030bd658b6037842f398c2787a4dedd75e321 Mon Sep 17 00:00:00 2001
From: Tom Bentley 
Date: Thu, 30 Apr 2020 18:33:10 +0100
Subject: [PATCH 0945/1071] KAFKA-9633: Ensure ConfigProviders are closed
 (#8204)

ConfigProvider extends Closeable, but were not closed in the following contexts:
* AbstractConfig
* WorkerConfigTransformer
* Worker

This commit ensures that ConfigProviders are close in the above contexts.

It also adds MockFileConfigProvider.assertClosed()
Gradle executes test classes concurrently, so MockFileConfigProvider
can't simply use a static field to hold its closure state.
Instead use a protocol whereby the MockFileConfigProvider is configured
with some unique ket identifying the test which also used when calling
assertClosed().

Reviewers: Konstantine Karantasis 
---
 .../kafka/common/config/AbstractConfig.java   |  1 +
 .../common/config/AbstractConfigTest.java     | 18 +++++++++-
 .../provider/MockFileConfigProvider.java      | 35 +++++++++++++++++++
 .../apache/kafka/connect/runtime/Worker.java  | 16 +++++----
 .../runtime/WorkerConfigTransformer.java      | 10 +++++-
 .../kafka/connect/runtime/WorkerTest.java     | 33 +++++++++++++++--
 6 files changed, 102 insertions(+), 11 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java b/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java
index 3992a4171e2b7..e91a9a5f836f9 100644
--- a/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java
+++ b/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java
@@ -483,6 +483,7 @@ private Map extractPotentialVariables(Map  configMap) {
                 resolvedOriginals.putAll(result.data());
             }
         }
+        providers.values().forEach(x -> Utils.closeQuietly(x, "config provider"));
 
         return new ResolvingMap<>(resolvedOriginals, originals);
     }
diff --git a/clients/src/test/java/org/apache/kafka/common/config/AbstractConfigTest.java b/clients/src/test/java/org/apache/kafka/common/config/AbstractConfigTest.java
index b5fff6f78d74d..834278b18d5f4 100644
--- a/clients/src/test/java/org/apache/kafka/common/config/AbstractConfigTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/config/AbstractConfigTest.java
@@ -34,6 +34,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.UUID;
 
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
@@ -340,6 +341,8 @@ public void testOriginalsWithConfigProvidersProps() {
         // Test Case: Valid Test Case for ConfigProviders as part of config.properties
         props.put("config.providers", "file");
         props.put("config.providers.file.class", MockFileConfigProvider.class.getName());
+        String id = UUID.randomUUID().toString();
+        props.put("config.providers.file.param.testId", id);
         props.put("prefix.ssl.truststore.location.number", 5);
         props.put("sasl.kerberos.service.name", "service name");
         props.put("sasl.kerberos.key", "${file:/usr/kerberos:key}");
@@ -349,6 +352,7 @@ public void testOriginalsWithConfigProvidersProps() {
         assertEquals(config.originals().get("sasl.kerberos.password"), "randomPassword");
         assertEquals(config.originals().get("prefix.ssl.truststore.location.number"), 5);
         assertEquals(config.originals().get("sasl.kerberos.service.name"), "service name");
+        MockFileConfigProvider.assertClosed(id);
     }
 
     @Test
@@ -357,12 +361,15 @@ public void testConfigProvidersPropsAsParam() {
         Properties providers = new Properties();
         providers.put("config.providers", "file");
         providers.put("config.providers.file.class", MockFileConfigProvider.class.getName());
+        String id = UUID.randomUUID().toString();
+        providers.put("config.providers.file.param.testId", id);
         Properties props = new Properties();
         props.put("sasl.kerberos.key", "${file:/usr/kerberos:key}");
         props.put("sasl.kerberos.password", "${file:/usr/kerberos:password}");
         TestIndirectConfigResolution config = new TestIndirectConfigResolution(props, convertPropertiesToMap(providers));
         assertEquals(config.originals().get("sasl.kerberos.key"), "testKey");
         assertEquals(config.originals().get("sasl.kerberos.password"), "randomPassword");
+        MockFileConfigProvider.assertClosed(id);
     }
 
     @Test
@@ -370,13 +377,16 @@ public void testImmutableOriginalsWithConfigProvidersProps() {
         // Test Case: Valid Test Case for ConfigProviders as a separate variable
         Properties providers = new Properties();
         providers.put("config.providers", "file");
-        providers.put("config.providers.file.class", "org.apache.kafka.common.config.provider.MockFileConfigProvider");
+        providers.put("config.providers.file.class", MockFileConfigProvider.class.getName());
+        String id = UUID.randomUUID().toString();
+        providers.put("config.providers.file.param.testId", id);
         Properties props = new Properties();
         props.put("sasl.kerberos.key", "${file:/usr/kerberos:key}");
         Map immutableMap = Collections.unmodifiableMap(props);
         Map provMap = convertPropertiesToMap(providers);
         TestIndirectConfigResolution config = new TestIndirectConfigResolution(immutableMap, provMap);
         assertEquals(config.originals().get("sasl.kerberos.key"), "testKey");
+        MockFileConfigProvider.assertClosed(id);
     }
 
     @Test
@@ -385,6 +395,8 @@ public void testAutoConfigResolutionWithMultipleConfigProviders() {
         Properties providers = new Properties();
         providers.put("config.providers", "file,vault");
         providers.put("config.providers.file.class", MockFileConfigProvider.class.getName());
+        String id = UUID.randomUUID().toString();
+        providers.put("config.providers.file.param.testId", id);
         providers.put("config.providers.vault.class", MockVaultConfigProvider.class.getName());
         Properties props = new Properties();
         props.put("sasl.kerberos.key", "${file:/usr/kerberos:key}");
@@ -396,6 +408,7 @@ public void testAutoConfigResolutionWithMultipleConfigProviders() {
         assertEquals(config.originals().get("sasl.kerberos.password"), "randomPassword");
         assertEquals(config.originals().get("sasl.truststore.key"), "testTruststoreKey");
         assertEquals(config.originals().get("sasl.truststore.password"), "randomtruststorePassword");
+        MockFileConfigProvider.assertClosed(id);
     }
 
     @Test
@@ -429,9 +442,12 @@ public void testAutoConfigResolutionWithMissingConfigKey() {
         Properties props = new Properties();
         props.put("config.providers", "test");
         props.put("config.providers.test.class", MockFileConfigProvider.class.getName());
+        String id = UUID.randomUUID().toString();
+        props.put("config.providers.test.param.testId", id);
         props.put("random", "${test:/foo/bar/testpath:random}");
         TestIndirectConfigResolution config = new TestIndirectConfigResolution(props);
         assertEquals(config.originals().get("random"), "${test:/foo/bar/testpath:random}");
+        MockFileConfigProvider.assertClosed(id);
     }
 
     @Test
diff --git a/clients/src/test/java/org/apache/kafka/common/config/provider/MockFileConfigProvider.java b/clients/src/test/java/org/apache/kafka/common/config/provider/MockFileConfigProvider.java
index e779cbe21bee7..3409096446895 100644
--- a/clients/src/test/java/org/apache/kafka/common/config/provider/MockFileConfigProvider.java
+++ b/clients/src/test/java/org/apache/kafka/common/config/provider/MockFileConfigProvider.java
@@ -19,11 +19,46 @@
 import java.io.IOException;
 import java.io.Reader;
 import java.io.StringReader;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 
 public class MockFileConfigProvider extends FileConfigProvider {
 
+    private static final Map INSTANCES = Collections.synchronizedMap(new HashMap<>());
+    private String id;
+    private boolean closed = false;
+
+    public void configure(Map configs) {
+        Object id = configs.get("testId");
+        if (id == null) {
+            throw new RuntimeException(getClass().getName() + " missing 'testId' config");
+        }
+        if (this.id != null) {
+            throw new RuntimeException(getClass().getName() + " instance was configured twice");
+        }
+        this.id = id.toString();
+        INSTANCES.put(id.toString(), this);
+    }
+
     @Override
     protected Reader reader(String path) throws IOException {
         return new StringReader("key=testKey\npassword=randomPassword");
     }
+
+    @Override
+    public synchronized void close() {
+        closed = true;
+    }
+
+    public static void assertClosed(String id) {
+        MockFileConfigProvider instance = INSTANCES.remove(id);
+        assertNotNull(instance);
+        synchronized (instance) {
+            assertTrue(instance.closed);
+        }
+    }
 }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
index 3c4ece7755f6b..3cf4d01ac1cd6 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
@@ -104,8 +104,8 @@ public class Worker {
     private final ConcurrentMap connectors = new ConcurrentHashMap<>();
     private final ConcurrentMap tasks = new ConcurrentHashMap<>();
     private SourceTaskOffsetCommitter sourceTaskOffsetCommitter;
-    private WorkerConfigTransformer workerConfigTransformer;
-    private ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy;
+    private final WorkerConfigTransformer workerConfigTransformer;
+    private final ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy;
 
     public Worker(
         String workerId,
@@ -220,6 +220,8 @@ public void stop() {
 
         workerMetricsGroup.close();
         connectorStatusMetricsGroup.close();
+
+        workerConfigTransformer.close();
     }
 
     /**
@@ -854,11 +856,11 @@ WorkerMetricsGroup workerMetricsGroup() {
     }
 
     static class ConnectorStatusMetricsGroup {
-        private ConnectMetrics connectMetrics;
-        private ConnectMetricsRegistry registry;
-        private ConcurrentMap connectorStatusMetrics = new ConcurrentHashMap<>();
-        private Herder herder;
-        private ConcurrentMap tasks;
+        private final ConnectMetrics connectMetrics;
+        private final ConnectMetricsRegistry registry;
+        private final ConcurrentMap connectorStatusMetrics = new ConcurrentHashMap<>();
+        private final Herder herder;
+        private final ConcurrentMap tasks;
 
 
         protected ConnectorStatusMetricsGroup(
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfigTransformer.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfigTransformer.java
index 1a799bb372976..318626bd5cade 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfigTransformer.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfigTransformer.java
@@ -20,6 +20,7 @@
 import org.apache.kafka.common.config.provider.ConfigProvider;
 import org.apache.kafka.common.config.ConfigTransformer;
 import org.apache.kafka.common.config.ConfigTransformerResult;
+import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.connect.runtime.Herder.ConfigReloadAction;
 import org.apache.kafka.connect.util.Callback;
 import org.slf4j.Logger;
@@ -34,15 +35,17 @@
  * A wrapper class to perform configuration transformations and schedule reloads for any
  * retrieved TTL values.
  */
-public class WorkerConfigTransformer {
+public class WorkerConfigTransformer implements AutoCloseable {
     private static final Logger log = LoggerFactory.getLogger(WorkerConfigTransformer.class);
 
     private final Worker worker;
     private final ConfigTransformer configTransformer;
     private final ConcurrentMap> requests = new ConcurrentHashMap<>();
+    private final Map configProviders;
 
     public WorkerConfigTransformer(Worker worker, Map configProviders) {
         this.worker = worker;
+        this.configProviders = configProviders;
         this.configTransformer = new ConfigTransformer(configProviders);
     }
 
@@ -98,4 +101,9 @@ public void onCompletion(Throwable error, Void result) {
         HerderRequest request = worker.herder().restartConnector(ttl, connectorName, cb);
         connectorRequests.put(path, request);
     }
+
+    @Override
+    public void close() {
+        configProviders.values().forEach(x -> Utils.closeQuietly(x, "config provider"));
+    }
 }
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java
index 7021503adbed6..e7ffd607f9431 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java
@@ -24,6 +24,7 @@
 import org.apache.kafka.common.config.AbstractConfig;
 import org.apache.kafka.common.config.ConfigDef;
 import org.apache.kafka.common.config.ConfigException;
+import org.apache.kafka.common.config.provider.MockFileConfigProvider;
 import org.apache.kafka.common.utils.MockTime;
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.connect.connector.Connector;
@@ -76,6 +77,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
@@ -132,6 +134,7 @@ public class WorkerTest extends ThreadedTest {
     @Mock private HeaderConverter taskHeaderConverter;
     @Mock private ExecutorService executorService;
     @MockNice private ConnectorConfig connectorConfig;
+    private String mockFileProviderTestId;
 
     @Before
     public void setup() {
@@ -144,6 +147,10 @@ public void setup() {
         workerProps.put("internal.value.converter.schemas.enable", "false");
         workerProps.put("offset.storage.file.filename", "/tmp/connect.offsets");
         workerProps.put(CommonClientConfigs.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName());
+        workerProps.put("config.providers", "file");
+        workerProps.put("config.providers.file.class", MockFileConfigProvider.class.getName());
+        mockFileProviderTestId = UUID.randomUUID().toString();
+        workerProps.put("config.providers.file.param.testId", mockFileProviderTestId);
         config = new StandaloneConfig(workerProps);
 
         defaultProducerConfigs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
@@ -187,9 +194,11 @@ public void testStartAndStopConnector() {
 
         EasyMock.expect(connector.version()).andReturn("1.0");
 
+        expectFileConfigProvider();
         EasyMock.expect(plugins.compareAndSwapLoaders(connector))
                 .andReturn(delegatingLoader)
                 .times(2);
+
         connector.initialize(anyObject(ConnectorContext.class));
         EasyMock.expectLastCall();
         connector.start(props);
@@ -235,12 +244,24 @@ public void testStartAndStopConnector() {
         assertStatistics(worker, 0, 0);
 
         PowerMock.verifyAll();
+        MockFileConfigProvider.assertClosed(mockFileProviderTestId);
+    }
+
+    private void expectFileConfigProvider() {
+        EasyMock.expect(plugins.newConfigProvider(EasyMock.anyObject(),
+                    EasyMock.eq("config.providers.file"), EasyMock.anyObject()))
+                .andAnswer(() -> {
+                    MockFileConfigProvider mockFileConfigProvider = new MockFileConfigProvider();
+                    mockFileConfigProvider.configure(Collections.singletonMap("testId", mockFileProviderTestId));
+                    return mockFileConfigProvider;
+                });
     }
 
     @Test
     public void testStartConnectorFailure() {
         expectConverters();
         expectStartStorage();
+        expectFileConfigProvider();
 
         Map props = new HashMap<>();
         props.put(SinkConnectorConfig.TOPICS_CONFIG, "foo,bar");
@@ -285,6 +306,7 @@ public void testStartConnectorFailure() {
     public void testAddConnectorByAlias() {
         expectConverters();
         expectStartStorage();
+        expectFileConfigProvider();
 
         EasyMock.expect(plugins.currentThreadLoader()).andReturn(delegatingLoader).times(2);
         EasyMock.expect(plugins.newConnector("WorkerTestConnector")).andReturn(connector);
@@ -349,6 +371,7 @@ public void testAddConnectorByAlias() {
     public void testAddConnectorByShortAlias() {
         expectConverters();
         expectStartStorage();
+        expectFileConfigProvider();
 
         EasyMock.expect(plugins.currentThreadLoader()).andReturn(delegatingLoader).times(2);
         EasyMock.expect(plugins.newConnector("WorkerTest")).andReturn(connector);
@@ -410,6 +433,7 @@ public void testAddConnectorByShortAlias() {
     public void testStopInvalidConnector() {
         expectConverters();
         expectStartStorage();
+        expectFileConfigProvider();
 
         PowerMock.replayAll();
 
@@ -425,6 +449,7 @@ public void testStopInvalidConnector() {
     public void testReconfigureConnectorTasks() {
         expectConverters();
         expectStartStorage();
+        expectFileConfigProvider();
 
         EasyMock.expect(plugins.currentThreadLoader()).andReturn(delegatingLoader).times(3);
         EasyMock.expect(plugins.newConnector(WorkerTestConnector.class.getName()))
@@ -513,6 +538,7 @@ public void testReconfigureConnectorTasks() {
     public void testAddRemoveTask() throws Exception {
         expectConverters();
         expectStartStorage();
+        expectFileConfigProvider();
 
         EasyMock.expect(workerTask.id()).andStubReturn(TASK_ID);
 
@@ -607,6 +633,7 @@ public void testAddRemoveTask() throws Exception {
     public void testTaskStatusMetricsStatuses() throws Exception {
         expectConverters();
         expectStartStorage();
+        expectFileConfigProvider();
 
         EasyMock.expect(workerTask.id()).andStubReturn(TASK_ID);
 
@@ -738,10 +765,9 @@ public void testConnectorStatusMetricsGroup_taskStatusCounter() {
         tasks.put(new ConnectorTaskId("c1", 1), workerTask);
         tasks.put(new ConnectorTaskId("c2", 0), workerTask);
 
-
         expectConverters();
-
         expectStartStorage();
+        expectFileConfigProvider();
 
         EasyMock.expect(Plugins.compareAndSwapLoaders(pluginLoader)).andReturn(delegatingLoader);
         EasyMock.expect(Plugins.compareAndSwapLoaders(pluginLoader)).andReturn(delegatingLoader);
@@ -772,6 +798,7 @@ public void testConnectorStatusMetricsGroup_taskStatusCounter() {
     public void testStartTaskFailure() {
         expectConverters();
         expectStartStorage();
+        expectFileConfigProvider();
 
         Map origProps = new HashMap<>();
         origProps.put(TaskConfig.TASK_CLASS_CONFIG, "missing.From.This.Workers.Classpath");
@@ -817,6 +844,7 @@ public void testStartTaskFailure() {
     public void testCleanupTasksOnStop() throws Exception {
         expectConverters();
         expectStartStorage();
+        expectFileConfigProvider();
 
         EasyMock.expect(workerTask.id()).andStubReturn(TASK_ID);
 
@@ -907,6 +935,7 @@ public void testCleanupTasksOnStop() throws Exception {
     public void testConverterOverrides() throws Exception {
         expectConverters();
         expectStartStorage();
+        expectFileConfigProvider();
 
         EasyMock.expect(workerTask.id()).andStubReturn(TASK_ID);
 

From 26e216e0cb361ed8584d2c8aec010b0f10eb2152 Mon Sep 17 00:00:00 2001
From: belugabehr <12578579+belugabehr@users.noreply.github.com>
Date: Wed, 6 May 2020 19:02:26 -0400
Subject: [PATCH 0946/1071] KAFKA-9419: Fix possible integer overflow in
 CircularIterator (#7950)

The CircularIterator class uses a wrapping index-based approach to iterate over a list. This can be a performance problem O(n^2) for a LinkedList. Also, the index counter itself is never reset, a modulo is applied to it for every list access. At some point, it may be possible that the index counter overflows to a negative value and therefore may cause a negative index read and an ArrayIndexOutOfBoundsException.

This fix changes the implementation to avoid these two scenarios. Uses the Collection Iterator classes to avoid using an index counter and it avoids having to seek to the correct index every time, this avoiding the LinkedList performance issue.

I have added unit tests to validate the new implementation.

* KAFKA-9419: Integer Overflow Possible with CircularIterator
* Added JavaDoc. Support null values in the underlying collection
* Always return true for hasNext(). Add more JavaDoc
* Use an advance method to load next value and always return true in hasNext()
* Simplify test suite
* Use assertThrows in tests and remove redundant 'this' identifier

Co-authored-by: David Mollitor 
Co-authored-by: Konstantine Karantasis 

Reviewers: Ron Dagostino , Konstantine Karantasis 
---
 .../kafka/common/utils/CircularIterator.java  | 71 ++++++++++++++++---
 .../common/utils/CircularIteratorTest.java    | 66 +++++++++++++++++
 2 files changed, 128 insertions(+), 9 deletions(-)
 create mode 100644 clients/src/test/java/org/apache/kafka/common/utils/CircularIteratorTest.java

diff --git a/clients/src/main/java/org/apache/kafka/common/utils/CircularIterator.java b/clients/src/main/java/org/apache/kafka/common/utils/CircularIterator.java
index 3e7d5eb29fbcc..925f4adf9e9f7 100644
--- a/clients/src/main/java/org/apache/kafka/common/utils/CircularIterator.java
+++ b/clients/src/main/java/org/apache/kafka/common/utils/CircularIterator.java
@@ -14,22 +14,54 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.kafka.common.utils;
 
+import java.util.Collection;
+import java.util.ConcurrentModificationException;
 import java.util.Iterator;
-import java.util.List;
+import java.util.Objects;
 
+/**
+ * An iterator that cycles through the {@code Iterator} of a {@code Collection}
+ * indefinitely. Useful for tasks such as round-robin load balancing. This class
+ * does not provide thread-safe access. This {@code Iterator} supports
+ * {@code null} elements in the underlying {@code Collection}. This
+ * {@code Iterator} does not support any modification to the underlying
+ * {@code Collection} after it has been wrapped by this class. Changing the
+ * underlying {@code Collection} may cause a
+ * {@link ConcurrentModificationException} or some other undefined behavior.
+ */
 public class CircularIterator implements Iterator {
-    int i = 0;
-    private List list;
 
-    public CircularIterator(List list) {
-        if (list.isEmpty()) {
+    private final Iterable iterable;
+    private Iterator iterator;
+    private T nextValue;
+
+    /**
+     * Create a new instance of a CircularIterator. The ordering of this
+     * Iterator will be dictated by the Iterator returned by Collection itself.
+     *
+     * @param col The collection to iterate indefinitely
+     *
+     * @throws NullPointerException if col is {@code null}
+     * @throws IllegalArgumentException if col is empty.
+     */
+    public CircularIterator(final Collection col) {
+        this.iterable = Objects.requireNonNull(col);
+        this.iterator = col.iterator();
+        if (col.isEmpty()) {
             throw new IllegalArgumentException("CircularIterator can only be used on non-empty lists");
         }
-        this.list = list;
+        this.nextValue = advance();
     }
 
+    /**
+     * Returns true since the iteration will forever cycle through the provided
+     * {@code Collection}.
+     *
+     * @return Always true
+     */
     @Override
     public boolean hasNext() {
         return true;
@@ -37,13 +69,34 @@ public boolean hasNext() {
 
     @Override
     public T next() {
-        T next = list.get(i);
-        i = (i + 1) % list.size();
+        final T next = nextValue;
+        nextValue = advance();
         return next;
     }
 
+    /**
+     * Return the next value in the {@code Iterator}, restarting the
+     * {@code Iterator} if necessary.
+     *
+     * @return The next value in the iterator
+     */
+    private T advance() {
+        if (!iterator.hasNext()) {
+            iterator = iterable.iterator();
+        }
+        return iterator.next();
+    }
+
+    /**
+     * Peek at the next value in the Iterator. Calling this method multiple
+     * times will return the same element without advancing this Iterator. The
+     * value returned by this method will be the next item returned by
+     * {@code next()}.
+     *
+     * @return The next value in this {@code Iterator}
+     */
     public T peek() {
-        return list.get(i);
+        return nextValue;
     }
 
     @Override
diff --git a/clients/src/test/java/org/apache/kafka/common/utils/CircularIteratorTest.java b/clients/src/test/java/org/apache/kafka/common/utils/CircularIteratorTest.java
new file mode 100644
index 0000000000000..a37c85e4beeae
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/utils/CircularIteratorTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.common.utils;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.Collections;
+
+import org.junit.Test;
+
+public class CircularIteratorTest {
+
+    @Test
+    public void testNullCollection() {
+        assertThrows(NullPointerException.class, () -> new CircularIterator<>(null));
+    }
+
+    @Test
+    public void testEmptyCollection() {
+        assertThrows(IllegalArgumentException.class, () -> new CircularIterator<>(Collections.emptyList()));
+    }
+
+    @Test()
+    public void testCycleCollection() {
+        final CircularIterator it = new CircularIterator<>(Arrays.asList("A", "B", null, "C"));
+
+        assertEquals("A", it.peek());
+        assertTrue(it.hasNext());
+        assertEquals("A", it.next());
+        assertEquals("B", it.peek());
+        assertTrue(it.hasNext());
+        assertEquals("B", it.next());
+        assertEquals(null, it.peek());
+        assertTrue(it.hasNext());
+        assertEquals(null, it.next());
+        assertEquals("C", it.peek());
+        assertTrue(it.hasNext());
+        assertEquals("C", it.next());
+        assertEquals("A", it.peek());
+        assertTrue(it.hasNext());
+        assertEquals("A", it.next());
+        assertEquals("B", it.peek());
+
+        // Check that peek does not have any side-effects
+        assertEquals("B", it.peek());
+    }
+
+}

From 1f225d2414f11af5b43df67b3b83e45eb2f5c55f Mon Sep 17 00:00:00 2001
From: Chris Egerton 
Date: Wed, 6 May 2020 18:09:47 -0700
Subject: [PATCH 0947/1071] KAFKA-9768: Fix handling of
 rest.advertised.listener config (#8360)

The rest.advertised.listener config is currently broken as setting it to http when listeners are configured for both https and http will cause the framework to choose whichever of the two listeners is listed first. The changes here attempt to fix this by checking not only that ServerConnector::getName begins with the specified protocol, but also that that protocol is immediately followed by an underscore, which the framework uses as a delimiter between the protocol and the remainder of the connector name.

An existing unit test for the RestServer::advertisedUrl method has been expanded to include a case that fails with the framework in its current state and passes with the changes in this commit.

* KAFKA-9768: Fix handling of rest.advertised.listener config
* KAFKA-9768: Add comments on server connector names
* KAFKA-9768: Update RestServerTest comment

Co-authored-by: Randall Hauch 

Reviewers: Randall Hauch , Konstantine Karantasis , Andrew Choi 
---
 .../kafka/connect/runtime/rest/RestServer.java      | 13 ++++++++++++-
 .../kafka/connect/runtime/rest/RestServerTest.java  |  8 ++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestServer.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestServer.java
index 9ad24468ee564..02b4677521fb1 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestServer.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestServer.java
@@ -416,10 +416,21 @@ else if (listeners.contains(String.format("%s://", PROTOCOL_HTTPS)))
         }
     }
 
+    /**
+     * Locate a Jetty connector for the standard (non-admin) REST API that uses the given protocol.
+     * @param protocol the protocol for the connector (e.g., "http" or "https").
+     * @return a {@link ServerConnector} for the server that uses the requested protocol, or
+     * {@code null} if none exist.
+     */
     ServerConnector findConnector(String protocol) {
         for (Connector connector : jettyServer.getConnectors()) {
             String connectorName = connector.getName();
-            if (connectorName.startsWith(protocol) && !ADMIN_SERVER_CONNECTOR_NAME.equals(connectorName))
+            // We set the names for these connectors when instantiating them, beginning with the
+            // protocol for the connector and then an underscore ("_"). We rely on that format here
+            // when trying to locate a connector with the requested protocol; if the naming format
+            // for the connectors we create is ever changed, we'll need to adjust the logic here
+            // accordingly.
+            if (connectorName.startsWith(protocol + "_") && !ADMIN_SERVER_CONNECTOR_NAME.equals(connectorName))
                 return (ServerConnector) connector;
         }
 
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/RestServerTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/RestServerTest.java
index ea6e98f01f7b9..575c4dae2a2cd 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/RestServerTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/RestServerTest.java
@@ -174,6 +174,14 @@ public void testAdvertisedUri() {
         config = new DistributedConfig(configMap);
         server = new RestServer(config);
         Assert.assertEquals("http://my-hostname:8080/", server.advertisedUrl().toString());
+
+        // correct listener is chosen when https listener is configured before http listener and advertised listener is http
+        configMap = new HashMap<>(baseWorkerProps());
+        configMap.put(WorkerConfig.LISTENERS_CONFIG, "https://encrypted-localhost:42069,http://plaintext-localhost:4761");
+        configMap.put(WorkerConfig.REST_ADVERTISED_LISTENER_CONFIG, "http");
+        config = new DistributedConfig(configMap);
+        server = new RestServer(config);
+        Assert.assertEquals("http://plaintext-localhost:4761/", server.advertisedUrl().toString());
     }
 
     @Test

From 2a80b7299366640b1ae266cc0d1b94d4a4726f28 Mon Sep 17 00:00:00 2001
From: Andy Coates <8012398+big-andy-coates@users.noreply.github.com>
Date: Thu, 7 May 2020 22:21:08 +0100
Subject: [PATCH 0948/1071] KAFKA-9667: Connect JSON serde strip trailing zeros
 (#8230)

This change turns on exact decimal processing in JSON Converter for deserializing decimals, meaning trailing zeros are maintained. Serialization was already using the decimal scale to output the right value, so this change means a value of `1.2300` can now be serialized to JSON and deserialized back to Connect without any loss of information.

Author: Andy Coates 
Reviewers: Randall Hauch , Almog Gavra 
---
 .../kafka/connect/json/JsonConverter.java     | 81 +++++++++++--------
 .../kafka/connect/json/JsonDeserializer.java  | 12 ++-
 .../kafka/connect/json/JsonSerializer.java    | 20 +++++
 .../kafka/connect/json/JsonConverterTest.java | 31 ++++++-
 4 files changed, 103 insertions(+), 41 deletions(-)

diff --git a/connect/json/src/main/java/org/apache/kafka/connect/json/JsonConverter.java b/connect/json/src/main/java/org/apache/kafka/connect/json/JsonConverter.java
index 3ef6aae816a43..7af202b7b9ff1 100644
--- a/connect/json/src/main/java/org/apache/kafka/connect/json/JsonConverter.java
+++ b/connect/json/src/main/java/org/apache/kafka/connect/json/JsonConverter.java
@@ -21,8 +21,6 @@
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import com.fasterxml.jackson.databind.node.ObjectNode;
-import java.util.HashSet;
-import java.util.Set;
 import org.apache.kafka.common.cache.Cache;
 import org.apache.kafka.common.cache.LRUCache;
 import org.apache.kafka.common.cache.SynchronizedCache;
@@ -54,6 +52,8 @@
 import java.util.Iterator;
 import java.util.Map;
 
+import static org.apache.kafka.common.utils.Utils.mkSet;
+
 /**
  * Implementation of Converter that uses JSON to store schemas and objects. By default this converter will serialize Connect keys, values,
  * and headers with schemas, although this can be disabled with {@link JsonConverterConfig#SCHEMAS_ENABLE_CONFIG schemas.enable}
@@ -191,6 +191,9 @@ public Object convert(Schema schema, JsonNode value) {
     // Convert values in Kafka Connect form into/from their logical types. These logical converters are discovered by logical type
     // names specified in the field
     private static final HashMap LOGICAL_CONVERTERS = new HashMap<>();
+
+    private static final JsonNodeFactory JSON_NODE_FACTORY = JsonNodeFactory.withExactBigDecimals(true);
+
     static {
         LOGICAL_CONVERTERS.put(Decimal.LOGICAL_NAME, new LogicalTypeConverter() {
             @Override
@@ -201,9 +204,9 @@ public JsonNode toJson(final Schema schema, final Object value, final JsonConver
                 final BigDecimal decimal = (BigDecimal) value;
                 switch (config.decimalFormat()) {
                     case NUMERIC:
-                        return JsonNodeFactory.instance.numberNode(decimal);
+                        return JSON_NODE_FACTORY.numberNode(decimal);
                     case BASE64:
-                        return JsonNodeFactory.instance.binaryNode(Decimal.fromLogical(schema, decimal));
+                        return JSON_NODE_FACTORY.binaryNode(Decimal.fromLogical(schema, decimal));
                     default:
                         throw new DataException("Unexpected " + JsonConverterConfig.DECIMAL_FORMAT_CONFIG + ": " + config.decimalFormat());
                 }
@@ -229,7 +232,7 @@ public Object toConnect(final Schema schema, final JsonNode value) {
             public JsonNode toJson(final Schema schema, final Object value, final JsonConverterConfig config) {
                 if (!(value instanceof java.util.Date))
                     throw new DataException("Invalid type for Date, expected Date but was " + value.getClass());
-                return JsonNodeFactory.instance.numberNode(Date.fromLogical(schema, (java.util.Date) value));
+                return JSON_NODE_FACTORY.numberNode(Date.fromLogical(schema, (java.util.Date) value));
             }
 
             @Override
@@ -245,7 +248,7 @@ public Object toConnect(final Schema schema, final JsonNode value) {
             public JsonNode toJson(final Schema schema, final Object value, final JsonConverterConfig config) {
                 if (!(value instanceof java.util.Date))
                     throw new DataException("Invalid type for Time, expected Date but was " + value.getClass());
-                return JsonNodeFactory.instance.numberNode(Time.fromLogical(schema, (java.util.Date) value));
+                return JSON_NODE_FACTORY.numberNode(Time.fromLogical(schema, (java.util.Date) value));
             }
 
             @Override
@@ -261,7 +264,7 @@ public Object toConnect(final Schema schema, final JsonNode value) {
             public JsonNode toJson(final Schema schema, final Object value, final JsonConverterConfig config) {
                 if (!(value instanceof java.util.Date))
                     throw new DataException("Invalid type for Timestamp, expected Date but was " + value.getClass());
-                return JsonNodeFactory.instance.numberNode(Timestamp.fromLogical(schema, (java.util.Date) value));
+                return JSON_NODE_FACTORY.numberNode(Timestamp.fromLogical(schema, (java.util.Date) value));
             }
 
             @Override
@@ -277,15 +280,23 @@ public Object toConnect(final Schema schema, final JsonNode value) {
     private Cache fromConnectSchemaCache;
     private Cache toConnectSchemaCache;
 
-    private final JsonSerializer serializer = new JsonSerializer();
+    private final JsonSerializer serializer;
     private final JsonDeserializer deserializer;
 
     public JsonConverter() {
-        // this ensures that the JsonDeserializer maintains full precision on
-        // floating point numbers that cannot fit into float64
-        final Set deserializationFeatures = new HashSet<>();
-        deserializationFeatures.add(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS);
-        deserializer = new JsonDeserializer(deserializationFeatures);
+        serializer = new JsonSerializer(
+            mkSet(),
+            JSON_NODE_FACTORY
+        );
+
+        deserializer = new JsonDeserializer(
+            mkSet(
+                // this ensures that the JsonDeserializer maintains full precision on
+                // floating point numbers that cannot fit into float64
+                DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS
+            ),
+            JSON_NODE_FACTORY
+        );
     }
 
     @Override
@@ -362,7 +373,7 @@ public SchemaAndValue toConnectData(String topic, byte[] value) {
         // The deserialized data should either be an envelope object containing the schema and the payload or the schema
         // was stripped during serialization and we need to fill in an all-encompassing schema.
         if (!config.schemasEnabled()) {
-            ObjectNode envelope = JsonNodeFactory.instance.objectNode();
+            ObjectNode envelope = JSON_NODE_FACTORY.objectNode();
             envelope.set(JsonSchema.ENVELOPE_SCHEMA_FIELD_NAME, null);
             envelope.set(JsonSchema.ENVELOPE_PAYLOAD_FIELD_NAME, jsonValue);
             jsonValue = envelope;
@@ -413,17 +424,17 @@ public ObjectNode asJsonSchema(Schema schema) {
                 jsonSchema = JsonSchema.STRING_SCHEMA.deepCopy();
                 break;
             case ARRAY:
-                jsonSchema = JsonNodeFactory.instance.objectNode().put(JsonSchema.SCHEMA_TYPE_FIELD_NAME, JsonSchema.ARRAY_TYPE_NAME);
+                jsonSchema = JSON_NODE_FACTORY.objectNode().put(JsonSchema.SCHEMA_TYPE_FIELD_NAME, JsonSchema.ARRAY_TYPE_NAME);
                 jsonSchema.set(JsonSchema.ARRAY_ITEMS_FIELD_NAME, asJsonSchema(schema.valueSchema()));
                 break;
             case MAP:
-                jsonSchema = JsonNodeFactory.instance.objectNode().put(JsonSchema.SCHEMA_TYPE_FIELD_NAME, JsonSchema.MAP_TYPE_NAME);
+                jsonSchema = JSON_NODE_FACTORY.objectNode().put(JsonSchema.SCHEMA_TYPE_FIELD_NAME, JsonSchema.MAP_TYPE_NAME);
                 jsonSchema.set(JsonSchema.MAP_KEY_FIELD_NAME, asJsonSchema(schema.keySchema()));
                 jsonSchema.set(JsonSchema.MAP_VALUE_FIELD_NAME, asJsonSchema(schema.valueSchema()));
                 break;
             case STRUCT:
-                jsonSchema = JsonNodeFactory.instance.objectNode().put(JsonSchema.SCHEMA_TYPE_FIELD_NAME, JsonSchema.STRUCT_TYPE_NAME);
-                ArrayNode fields = JsonNodeFactory.instance.arrayNode();
+                jsonSchema = JSON_NODE_FACTORY.objectNode().put(JsonSchema.SCHEMA_TYPE_FIELD_NAME, JsonSchema.STRUCT_TYPE_NAME);
+                ArrayNode fields = JSON_NODE_FACTORY.arrayNode();
                 for (Field field : schema.fields()) {
                     ObjectNode fieldJsonSchema = asJsonSchema(field.schema()).deepCopy();
                     fieldJsonSchema.put(JsonSchema.STRUCT_FIELD_NAME_FIELD_NAME, field.name());
@@ -443,7 +454,7 @@ public ObjectNode asJsonSchema(Schema schema) {
         if (schema.doc() != null)
             jsonSchema.put(JsonSchema.SCHEMA_DOC_FIELD_NAME, schema.doc());
         if (schema.parameters() != null) {
-            ObjectNode jsonSchemaParams = JsonNodeFactory.instance.objectNode();
+            ObjectNode jsonSchemaParams = JSON_NODE_FACTORY.objectNode();
             for (Map.Entry prop : schema.parameters().entrySet())
                 jsonSchemaParams.put(prop.getKey(), prop.getValue());
             jsonSchema.set(JsonSchema.SCHEMA_PARAMETERS_FIELD_NAME, jsonSchemaParams);
@@ -596,7 +607,7 @@ private JsonNode convertToJson(Schema schema, Object value) {
             if (schema.defaultValue() != null)
                 return convertToJson(schema, schema.defaultValue());
             if (schema.isOptional())
-                return JsonNodeFactory.instance.nullNode();
+                return JSON_NODE_FACTORY.nullNode();
             throw new DataException("Conversion error: null value for field that is required and has no default value");
         }
 
@@ -617,32 +628,32 @@ private JsonNode convertToJson(Schema schema, Object value) {
             }
             switch (schemaType) {
                 case INT8:
-                    return JsonNodeFactory.instance.numberNode((Byte) value);
+                    return JSON_NODE_FACTORY.numberNode((Byte) value);
                 case INT16:
-                    return JsonNodeFactory.instance.numberNode((Short) value);
+                    return JSON_NODE_FACTORY.numberNode((Short) value);
                 case INT32:
-                    return JsonNodeFactory.instance.numberNode((Integer) value);
+                    return JSON_NODE_FACTORY.numberNode((Integer) value);
                 case INT64:
-                    return JsonNodeFactory.instance.numberNode((Long) value);
+                    return JSON_NODE_FACTORY.numberNode((Long) value);
                 case FLOAT32:
-                    return JsonNodeFactory.instance.numberNode((Float) value);
+                    return JSON_NODE_FACTORY.numberNode((Float) value);
                 case FLOAT64:
-                    return JsonNodeFactory.instance.numberNode((Double) value);
+                    return JSON_NODE_FACTORY.numberNode((Double) value);
                 case BOOLEAN:
-                    return JsonNodeFactory.instance.booleanNode((Boolean) value);
+                    return JSON_NODE_FACTORY.booleanNode((Boolean) value);
                 case STRING:
                     CharSequence charSeq = (CharSequence) value;
-                    return JsonNodeFactory.instance.textNode(charSeq.toString());
+                    return JSON_NODE_FACTORY.textNode(charSeq.toString());
                 case BYTES:
                     if (value instanceof byte[])
-                        return JsonNodeFactory.instance.binaryNode((byte[]) value);
+                        return JSON_NODE_FACTORY.binaryNode((byte[]) value);
                     else if (value instanceof ByteBuffer)
-                        return JsonNodeFactory.instance.binaryNode(((ByteBuffer) value).array());
+                        return JSON_NODE_FACTORY.binaryNode(((ByteBuffer) value).array());
                     else
                         throw new DataException("Invalid type for bytes type: " + value.getClass());
                 case ARRAY: {
                     Collection collection = (Collection) value;
-                    ArrayNode list = JsonNodeFactory.instance.arrayNode();
+                    ArrayNode list = JSON_NODE_FACTORY.arrayNode();
                     for (Object elem : collection) {
                         Schema valueSchema = schema == null ? null : schema.valueSchema();
                         JsonNode fieldValue = convertToJson(valueSchema, elem);
@@ -668,9 +679,9 @@ else if (value instanceof ByteBuffer)
                     ObjectNode obj = null;
                     ArrayNode list = null;
                     if (objectMode)
-                        obj = JsonNodeFactory.instance.objectNode();
+                        obj = JSON_NODE_FACTORY.objectNode();
                     else
-                        list = JsonNodeFactory.instance.arrayNode();
+                        list = JSON_NODE_FACTORY.arrayNode();
                     for (Map.Entry entry : map.entrySet()) {
                         Schema keySchema = schema == null ? null : schema.keySchema();
                         Schema valueSchema = schema == null ? null : schema.valueSchema();
@@ -680,7 +691,7 @@ else if (value instanceof ByteBuffer)
                         if (objectMode)
                             obj.set(mapKey.asText(), mapValue);
                         else
-                            list.add(JsonNodeFactory.instance.arrayNode().add(mapKey).add(mapValue));
+                            list.add(JSON_NODE_FACTORY.arrayNode().add(mapKey).add(mapValue));
                     }
                     return objectMode ? obj : list;
                 }
@@ -688,7 +699,7 @@ else if (value instanceof ByteBuffer)
                     Struct struct = (Struct) value;
                     if (!struct.schema().equals(schema))
                         throw new DataException("Mismatching schema.");
-                    ObjectNode obj = JsonNodeFactory.instance.objectNode();
+                    ObjectNode obj = JSON_NODE_FACTORY.objectNode();
                     for (Field field : schema.fields()) {
                         obj.set(field.name(), convertToJson(field.schema(), struct.get(field)));
                     }
diff --git a/connect/json/src/main/java/org/apache/kafka/connect/json/JsonDeserializer.java b/connect/json/src/main/java/org/apache/kafka/connect/json/JsonDeserializer.java
index a656e53283fea..2e6e821b2d3bc 100644
--- a/connect/json/src/main/java/org/apache/kafka/connect/json/JsonDeserializer.java
+++ b/connect/json/src/main/java/org/apache/kafka/connect/json/JsonDeserializer.java
@@ -19,6 +19,7 @@
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import java.util.Collections;
 import java.util.Set;
 import org.apache.kafka.common.errors.SerializationException;
@@ -29,13 +30,13 @@
  * structured data without having associated Java classes. This deserializer also supports Connect schemas.
  */
 public class JsonDeserializer implements Deserializer {
-    private ObjectMapper objectMapper = new ObjectMapper();
+    private final ObjectMapper objectMapper = new ObjectMapper();
 
     /**
      * Default constructor needed by Kafka
      */
     public JsonDeserializer() {
-        this(Collections.emptySet());
+        this(Collections.emptySet(), JsonNodeFactory.withExactBigDecimals(true));
     }
 
     /**
@@ -43,9 +44,14 @@ public JsonDeserializer() {
      * for the deserializer
      *
      * @param deserializationFeatures the specified deserialization features
+     * @param jsonNodeFactory the json node factory to use.
      */
-    JsonDeserializer(final Set deserializationFeatures) {
+    JsonDeserializer(
+        final Set deserializationFeatures,
+        final JsonNodeFactory jsonNodeFactory
+    ) {
         deserializationFeatures.forEach(objectMapper::enable);
+        objectMapper.setNodeFactory(jsonNodeFactory);
     }
 
     @Override
diff --git a/connect/json/src/main/java/org/apache/kafka/connect/json/JsonSerializer.java b/connect/json/src/main/java/org/apache/kafka/connect/json/JsonSerializer.java
index 94ec0a83e8ce6..0f2b62bd0a40e 100644
--- a/connect/json/src/main/java/org/apache/kafka/connect/json/JsonSerializer.java
+++ b/connect/json/src/main/java/org/apache/kafka/connect/json/JsonSerializer.java
@@ -18,9 +18,14 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import org.apache.kafka.common.errors.SerializationException;
 import org.apache.kafka.common.serialization.Serializer;
 
+import java.util.Collections;
+import java.util.Set;
+
 /**
  * Serialize Jackson JsonNode tree model objects to UTF-8 JSON. Using the tree model allows handling arbitrarily
  * structured data without corresponding Java classes. This serializer also supports Connect schemas.
@@ -32,7 +37,22 @@ public class JsonSerializer implements Serializer {
      * Default constructor needed by Kafka
      */
     public JsonSerializer() {
+        this(Collections.emptySet(), JsonNodeFactory.withExactBigDecimals(true));
+    }
 
+    /**
+     * A constructor that additionally specifies some {@link SerializationFeature}
+     * for the serializer
+     *
+     * @param serializationFeatures the specified serialization features
+     * @param jsonNodeFactory the json node factory to use.
+     */
+    JsonSerializer(
+        final Set serializationFeatures,
+        final JsonNodeFactory jsonNodeFactory
+    ) {
+        serializationFeatures.forEach(objectMapper::enable);
+        objectMapper.setNodeFactory(jsonNodeFactory);
     }
 
     @Override
diff --git a/connect/json/src/test/java/org/apache/kafka/connect/json/JsonConverterTest.java b/connect/json/src/test/java/org/apache/kafka/connect/json/JsonConverterTest.java
index 2a5695031e49d..2e189e2d584ae 100644
--- a/connect/json/src/test/java/org/apache/kafka/connect/json/JsonConverterTest.java
+++ b/connect/json/src/test/java/org/apache/kafka/connect/json/JsonConverterTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.connect.json;
 
+import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ArrayNode;
@@ -65,8 +66,11 @@
 public class JsonConverterTest {
     private static final String TOPIC = "topic";
 
-    ObjectMapper objectMapper = new ObjectMapper();
-    JsonConverter converter = new JsonConverter();
+    private final ObjectMapper objectMapper = new ObjectMapper()
+        .enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS)
+        .setNodeFactory(JsonNodeFactory.withExactBigDecimals(true));
+
+    private final JsonConverter converter = new JsonConverter();
 
     @Before
     public void setUp() {
@@ -272,6 +276,16 @@ public void numericDecimalToConnect() {
         assertEquals(reference, schemaAndValue.value());
     }
 
+    @Test
+    public void numericDecimalWithTrailingZerosToConnect() {
+        BigDecimal reference = new BigDecimal(new BigInteger("15600"), 4);
+        Schema schema = Decimal.schema(4);
+        String msg = "{ \"schema\": { \"type\": \"bytes\", \"name\": \"org.apache.kafka.connect.data.Decimal\", \"version\": 1, \"parameters\": { \"scale\": \"4\" } }, \"payload\": 1.5600 }";
+        SchemaAndValue schemaAndValue = converter.toConnectData(TOPIC, msg.getBytes());
+        assertEquals(schema, schemaAndValue.schema());
+        assertEquals(reference, schemaAndValue.value());
+    }
+
     @Test
     public void highPrecisionNumericDecimalToConnect() {
         // this number is too big to be kept in a float64!
@@ -634,7 +648,18 @@ public void decimalToNumericJson() {
     }
 
     @Test
-    public void decimalToJsonWithoutSchema() throws IOException {
+    public void decimalWithTrailingZerosToNumericJson() {
+        converter.configure(Collections.singletonMap(JsonConverterConfig.DECIMAL_FORMAT_CONFIG, DecimalFormat.NUMERIC.name()), false);
+        JsonNode converted = parse(converter.fromConnectData(TOPIC, Decimal.schema(4), new BigDecimal(new BigInteger("15600"), 4)));
+        validateEnvelope(converted);
+        assertEquals(parse("{ \"type\": \"bytes\", \"optional\": false, \"name\": \"org.apache.kafka.connect.data.Decimal\", \"version\": 1, \"parameters\": { \"scale\": \"4\" } }"),
+            converted.get(JsonSchema.ENVELOPE_SCHEMA_FIELD_NAME));
+        assertTrue("expected node to be numeric", converted.get(JsonSchema.ENVELOPE_PAYLOAD_FIELD_NAME).isNumber());
+        assertEquals(new BigDecimal("1.5600"), converted.get(JsonSchema.ENVELOPE_PAYLOAD_FIELD_NAME).decimalValue());
+    }
+
+    @Test
+    public void decimalToJsonWithoutSchema() {
         assertThrows(
             "expected data exception when serializing BigDecimal without schema",
             DataException.class,

From 8d54432437378cad873ef15ca3b4718d7f1c1152 Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Fri, 8 May 2020 12:09:41 -0700
Subject: [PATCH 0949/1071] KAFKA-9127: don't create StreamThreads for
 global-only topology (2.4) (#8616)

Backports: https://github.com/apache/kafka/pull/8540

Reviewers: Matthias J. Sax , John Roesler 
---
 checkstyle/suppressions.xml                   |   2 +-
 .../apache/kafka/streams/KafkaStreams.java    |  40 +++++--
 .../internals/InternalTopologyBuilder.java    |   4 +
 .../kafka/streams/KafkaStreamsTest.java       | 101 +++++++++++++-----
 .../GlobalKTableIntegrationTest.java          |  18 ++++
 .../utils/IntegrationTestUtils.java           |  27 +++++
 6 files changed, 156 insertions(+), 36 deletions(-)

diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index 941dd1b1b9234..c8678accde7d0 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -175,7 +175,7 @@
               files="StreamsPartitionAssignor.java"/>
 
     
+              files="(ProcessorStateManager|InternalTopologyBuilder|KafkaStreams|StreamsPartitionAssignor|StreamThread).java"/>
 
     
diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
index 621bb91593920..3a22e14cf87a2 100644
--- a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
+++ b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
@@ -36,6 +36,7 @@
 import org.apache.kafka.streams.errors.InvalidStateStoreException;
 import org.apache.kafka.streams.errors.ProcessorStateException;
 import org.apache.kafka.streams.errors.StreamsException;
+import org.apache.kafka.streams.errors.TopologyException;
 import org.apache.kafka.streams.internals.ApiUtils;
 import org.apache.kafka.streams.internals.metrics.ClientMetrics;
 import org.apache.kafka.streams.kstream.KStream;
@@ -48,6 +49,7 @@
 import org.apache.kafka.streams.processor.ThreadMetadata;
 import org.apache.kafka.streams.processor.internals.DefaultKafkaClientSupplier;
 import org.apache.kafka.streams.processor.internals.GlobalStreamThread;
+import org.apache.kafka.streams.processor.internals.GlobalStreamThread.State;
 import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder;
 import org.apache.kafka.streams.processor.internals.ProcessorTopology;
 import org.apache.kafka.streams.processor.internals.StateDirectory;
@@ -480,8 +482,9 @@ public synchronized void onChange(final Thread thread,
                     final GlobalStreamThread.State newState = (GlobalStreamThread.State) abstractNewState;
                     globalThreadState = newState;
 
-                    // special case when global thread is dead
-                    if (newState == GlobalStreamThread.State.DEAD) {
+                    if (newState == GlobalStreamThread.State.RUNNING) {
+                        maybeSetRunning();
+                    } else if (newState == GlobalStreamThread.State.DEAD) {
                         if (setState(State.ERROR)) {
                             log.error("Global thread has died. The instance will be in error state and should be closed.");
                         }
@@ -696,28 +699,45 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder,
                 internalTopologyBuilder,
                 parseHostInfo(config.getString(StreamsConfig.APPLICATION_SERVER_CONFIG)));
 
+        final int numStreamThreads;
+        if (internalTopologyBuilder.hasNoNonGlobalTopology()) {
+            log.info("Overriding number of StreamThreads to zero for global-only topology");
+            numStreamThreads = 0;
+        } else {
+            numStreamThreads = config.getInt(StreamsConfig.NUM_STREAM_THREADS_CONFIG);
+        }
+
         // create the stream thread, global update thread, and cleanup thread
-        threads = new StreamThread[config.getInt(StreamsConfig.NUM_STREAM_THREADS_CONFIG)];
+        threads = new StreamThread[numStreamThreads];
+
+        final ProcessorTopology globalTaskTopology = internalTopologyBuilder.buildGlobalStateTopology();
+        final boolean hasGlobalTopology = globalTaskTopology != null;
+
+        if (numStreamThreads == 0 && !hasGlobalTopology) {
+            log.error("Topology with no input topics will create no stream threads and no global thread.");
+            throw new TopologyException("Topology has no stream threads and no global threads, " +
+                "must subscribe to at least one source topic or global table.");
+        }
 
         long totalCacheSize = config.getLong(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG);
         if (totalCacheSize < 0) {
             totalCacheSize = 0;
             log.warn("Negative cache size passed in. Reverting to cache size of 0 bytes.");
         }
-        final ProcessorTopology globalTaskTopology = internalTopologyBuilder.buildGlobalStateTopology();
-        final long cacheSizePerThread = totalCacheSize / (threads.length + (globalTaskTopology == null ? 0 : 1));
-        final boolean createStateDirectory = taskTopology.hasPersistentLocalStore() ||
-                (globalTaskTopology != null && globalTaskTopology.hasPersistentGlobalStore());
+
+        final long cacheSizePerThread = totalCacheSize / (threads.length + (hasGlobalTopology ? 1 : 0));
+        final boolean hasPersistentStores = taskTopology.hasPersistentLocalStore() ||
+                (hasGlobalTopology && globalTaskTopology.hasPersistentGlobalStore());
 
         try {
-            stateDirectory = new StateDirectory(config, time, createStateDirectory);
+            stateDirectory = new StateDirectory(config, time, hasPersistentStores);
         } catch (final ProcessorStateException fatal) {
             throw new StreamsException(fatal);
         }
 
         final StateRestoreListener delegatingStateRestoreListener = new DelegatingStateRestoreListener();
         GlobalStreamThread.State globalThreadState = null;
-        if (globalTaskTopology != null) {
+        if (hasGlobalTopology) {
             final String globalThreadId = clientId + "-GlobalStreamThread";
             globalStreamThread = new GlobalStreamThread(
                 globalTaskTopology,
@@ -758,7 +778,7 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder,
         }
 
         final StreamStateListener streamStateListener = new StreamStateListener(threadState, globalThreadState);
-        if (globalTaskTopology != null) {
+        if (hasGlobalTopology) {
             globalStreamThread.setStateListener(streamStateListener);
         }
         for (final StreamThread thread : threads) {
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java
index d28aee8907b3d..f11536a54bbcc 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java
@@ -1235,6 +1235,10 @@ synchronized void updateSubscriptions(final SubscriptionUpdates subscriptionUpda
         setRegexMatchedTopicToStateStore();
     }
 
+    public boolean hasNoNonGlobalTopology() {
+        return sourceTopicNames.isEmpty() && nodeToSourcePatterns.isEmpty();
+    }
+
     private boolean isGlobalSource(final String nodeName) {
         final NodeFactory nodeFactory = nodeFactories.get(nodeName);
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java b/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java
index cd24685165a07..8124e3efd3b29 100644
--- a/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java
@@ -28,6 +28,7 @@
 import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.common.utils.MockTime;
 import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.streams.errors.TopologyException;
 import org.apache.kafka.streams.internals.metrics.ClientMetrics;
 import org.apache.kafka.streams.kstream.Materialized;
 import org.apache.kafka.streams.processor.AbstractProcessor;
@@ -83,6 +84,7 @@
 import static org.easymock.EasyMock.anyString;
 import static org.hamcrest.CoreMatchers.hasItem;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
@@ -104,7 +106,7 @@ public class KafkaStreamsTest {
     private MockTime time;
 
     private Properties props;
-
+    
     @Mock
     private StateDirectory stateDirectory;
     @Mock
@@ -317,7 +319,7 @@ private void prepareStreamThread(final StreamThread thread, final boolean termin
 
     @Test
     public void testShouldTransitToNotRunningIfCloseRightAfterCreated() {
-        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time);
         streams.close();
 
         Assert.assertEquals(KafkaStreams.State.NOT_RUNNING, streams.state());
@@ -325,7 +327,7 @@ public void testShouldTransitToNotRunningIfCloseRightAfterCreated() {
 
     @Test
     public void stateShouldTransitToRunningIfNonDeadThreadsBackToRunning() throws InterruptedException {
-        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time);
         streams.setStateListener(streamsStateListener);
 
         Assert.assertEquals(0, streamsStateListener.numChanges);
@@ -393,7 +395,7 @@ public void stateShouldTransitToRunningIfNonDeadThreadsBackToRunning() throws In
 
     @Test
     public void stateShouldTransitToErrorIfAllThreadsDead() throws InterruptedException {
-        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time);
         streams.setStateListener(streamsStateListener);
 
         Assert.assertEquals(0, streamsStateListener.numChanges);
@@ -457,7 +459,7 @@ public void stateShouldTransitToErrorIfAllThreadsDead() throws InterruptedExcept
 
     @Test
     public void shouldCleanupResourcesOnCloseWithoutPreviousStart() throws Exception {
-        final StreamsBuilder builder = new StreamsBuilder();
+        final StreamsBuilder builder = getBuilderWithSource();
         builder.globalTable("anyTopic");
 
         final KafkaStreams streams = new KafkaStreams(builder.build(), props, supplier, time);
@@ -477,7 +479,7 @@ public void shouldCleanupResourcesOnCloseWithoutPreviousStart() throws Exception
     @Test
     public void testStateThreadClose() throws Exception {
         // make sure we have the global state thread running too
-        final StreamsBuilder builder = new StreamsBuilder();
+        final StreamsBuilder builder = getBuilderWithSource();
         builder.globalTable("anyTopic");
         final KafkaStreams streams = new KafkaStreams(builder.build(), props, supplier, time);
 
@@ -514,7 +516,7 @@ public void testStateThreadClose() throws Exception {
     @Test
     public void testStateGlobalThreadClose() throws Exception {
         // make sure we have the global state thread running too
-        final StreamsBuilder builder = new StreamsBuilder();
+        final StreamsBuilder builder = getBuilderWithSource();
         builder.globalTable("anyTopic");
         final KafkaStreams streams = new KafkaStreams(builder.build(), props, supplier, time);
 
@@ -542,7 +544,7 @@ public void testStateGlobalThreadClose() throws Exception {
     public void testInitializesAndDestroysMetricsReporters() {
         final int oldInitCount = MockMetricsReporter.INIT_COUNT.get();
 
-        try (final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time)) {
+        try (final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time)) {
             final int newInitCount = MockMetricsReporter.INIT_COUNT.get();
             final int initDiff = newInitCount - oldInitCount;
             assertTrue("some reporters should be initialized by calling on construction", initDiff > 0);
@@ -556,7 +558,7 @@ public void testInitializesAndDestroysMetricsReporters() {
 
     @Test
     public void testCloseIsIdempotent() {
-        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time);
         streams.close();
         final int closeCount = MockMetricsReporter.CLOSE_COUNT.get();
 
@@ -567,7 +569,7 @@ public void testCloseIsIdempotent() {
 
     @Test
     public void testCannotStartOnceClosed() {
-        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time);
         streams.start();
         streams.close();
         try {
@@ -582,7 +584,7 @@ public void testCannotStartOnceClosed() {
 
     @Test
     public void shouldNotSetGlobalRestoreListenerAfterStarting() {
-        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time);
         streams.start();
         try {
             streams.setGlobalStateRestoreListener(null);
@@ -596,7 +598,7 @@ public void shouldNotSetGlobalRestoreListenerAfterStarting() {
 
     @Test
     public void shouldThrowExceptionSettingUncaughtExceptionHandlerNotInCreateState() {
-        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time);
         streams.start();
         try {
             streams.setUncaughtExceptionHandler(null);
@@ -608,7 +610,7 @@ public void shouldThrowExceptionSettingUncaughtExceptionHandlerNotInCreateState(
 
     @Test
     public void shouldThrowExceptionSettingStateListenerNotInCreateState() {
-        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time);
         streams.start();
         try {
             streams.setStateListener(null);
@@ -620,7 +622,7 @@ public void shouldThrowExceptionSettingStateListenerNotInCreateState() {
 
     @Test
     public void shouldAllowCleanupBeforeStartAndAfterClose() {
-        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time);
         try {
             streams.cleanUp();
             streams.start();
@@ -632,7 +634,7 @@ public void shouldAllowCleanupBeforeStartAndAfterClose() {
 
     @Test
     public void shouldThrowOnCleanupWhileRunning() throws InterruptedException {
-        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time);
         streams.start();
         TestUtils.waitForCondition(
             () -> streams.state() == KafkaStreams.State.RUNNING,
@@ -648,46 +650,46 @@ public void shouldThrowOnCleanupWhileRunning() throws InterruptedException {
 
     @Test(expected = IllegalStateException.class)
     public void shouldNotGetAllTasksWhenNotRunning() {
-        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time);
         streams.allMetadata();
     }
 
     @Test(expected = IllegalStateException.class)
     public void shouldNotGetAllTasksWithStoreWhenNotRunning() {
-        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time);
         streams.allMetadataForStore("store");
     }
 
     @Test(expected = IllegalStateException.class)
     public void shouldNotGetTaskWithKeyAndSerializerWhenNotRunning() {
-        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time);
         streams.metadataForKey("store", "key", Serdes.String().serializer());
     }
 
     @Test(expected = IllegalStateException.class)
     public void shouldNotGetTaskWithKeyAndPartitionerWhenNotRunning() {
-        final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time);
         streams.metadataForKey("store", "key", (topic, key, value, numPartitions) -> 0);
     }
 
     @Test
     public void shouldReturnFalseOnCloseWhenThreadsHaventTerminated() {
         // do not use mock time so that it can really elapse
-        try (final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier)) {
+        try (final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier)) {
             assertFalse(streams.close(Duration.ofMillis(10L)));
         }
     }
 
     @Test(expected = IllegalArgumentException.class)
     public void shouldThrowOnNegativeTimeoutForClose() {
-        try (final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time)) {
+        try (final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time)) {
             streams.close(Duration.ofMillis(-1L));
         }
     }
 
     @Test
     public void shouldNotBlockInCloseForZeroDuration() {
-        try (final KafkaStreams streams = new KafkaStreams(new StreamsBuilder().build(), props, supplier, time)) {
+        try (final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time)) {
             // with mock time that does not elapse, close would not return if it ever waits on the state transition
             assertFalse(streams.close(Duration.ZERO));
         }
@@ -720,7 +722,7 @@ public void shouldTriggerRecordingOfRocksDBMetricsIfRecordingLevelIsDebug() {
         builder.table("topic", Materialized.as("store"));
         props.setProperty(StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG, RecordingLevel.DEBUG.name());
 
-        try (final KafkaStreams streams = new KafkaStreams(builder.build(), props, supplier, time)) {
+        try (final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time)) {
             streams.start();
         }
 
@@ -743,7 +745,7 @@ public void shouldNotTriggerRecordingOfRocksDBMetricsIfRecordingLevelIsInfo() {
         builder.table("topic", Materialized.as("store"));
         props.setProperty(StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG, RecordingLevel.INFO.name());
 
-        try (final KafkaStreams streams = new KafkaStreams(builder.build(), props, supplier, time)) {
+        try (final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time)) {
             streams.start();
         }
 
@@ -763,7 +765,7 @@ public void shouldWarnAboutRocksDBConfigSetterIsNotGuaranteedToBeBackwardsCompat
         props.setProperty(StreamsConfig.ROCKSDB_CONFIG_SETTER_CLASS_CONFIG, TestRocksDbConfigSetter.class.getName());
 
         final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
-        new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+        new KafkaStreams(getBuilderWithSource().build(), props, supplier, time);
         LogCaptureAppender.unregister(appender);
 
         assertThat(appender.getMessages(), hasItem("stream-client [" + CLIENT_ID + "] "
@@ -845,6 +847,49 @@ public void statefulTopologyShouldCreateStateDirectory() throws Exception {
         startStreamsAndCheckDirExists(topology, true);
     }
 
+    @Test
+    public void shouldThrowTopologyExceptionOnEmptyTopology() {
+        try {
+            new KafkaStreams(new StreamsBuilder().build(), props, supplier, time);
+            fail("Should have thrown TopologyException");
+        } catch (final TopologyException e) {
+            assertThat(
+                e.getMessage(),
+                equalTo("Invalid topology: Topology has no stream threads and no global threads, " +
+                            "must subscribe to at least one source topic or global table."));
+        }
+    }
+
+    @Test
+    public void shouldNotCreateStreamThreadsForGlobalOnlyTopology() {
+        final StreamsBuilder builder = new StreamsBuilder();
+        builder.globalTable("anyTopic");
+        final KafkaStreams streams = new KafkaStreams(builder.build(), props, supplier, time);
+
+        assertThat(streams.threads.length, equalTo(0));
+    }
+
+    @Test
+    public void shouldTransitToRunningWithGlobalOnlyTopology() throws InterruptedException {
+        final StreamsBuilder builder = new StreamsBuilder();
+        builder.globalTable("anyTopic");
+        final KafkaStreams streams = new KafkaStreams(builder.build(), props, supplier, time);
+
+        assertThat(streams.threads.length, equalTo(0));
+        assertEquals(streams.state(), KafkaStreams.State.CREATED);
+
+        streams.start();
+        TestUtils.waitForCondition(
+            () -> streams.state() == KafkaStreams.State.RUNNING,
+            "Streams never started, state is " + streams.state());
+
+        streams.close();
+
+        TestUtils.waitForCondition(
+            () -> streams.state() == KafkaStreams.State.NOT_RUNNING,
+            "Streams never stopped.");
+    }
+
     @SuppressWarnings("unchecked")
     private Topology getStatefulTopology(final String inputTopic,
                                          final String outputTopic,
@@ -886,6 +931,12 @@ public void process(final String key, final String value) {
             new MockProcessorSupplier());
         return topology;
     }
+    
+    private StreamsBuilder getBuilderWithSource() {
+        final StreamsBuilder builder = new StreamsBuilder();
+        builder.stream("source-topic");
+        return builder;
+    }
 
     private void startStreamsAndCheckDirExists(final Topology topology,
                                                final boolean shouldFilesExist) throws Exception {
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/GlobalKTableIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/GlobalKTableIntegrationTest.java
index 0a9148d61fb10..e33bdd1e906ef 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/GlobalKTableIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/GlobalKTableIntegrationTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.streams.integration;
 
+import java.time.Duration;
 import kafka.utils.MockTime;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.common.serialization.LongSerializer;
@@ -23,6 +24,7 @@
 import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.KafkaStreams.State;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsConfig;
@@ -54,6 +56,8 @@
 import java.util.Properties;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import static java.util.Collections.singletonList;
+import static org.apache.kafka.streams.integration.utils.IntegrationTestUtils.waitForApplicationState;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.core.IsEqual.equalTo;
 
@@ -267,6 +271,20 @@ public void shouldRestoreGlobalInMemoryKTableOnRestart() throws Exception {
         assertThat(timestampedStore.approximateNumEntries(), equalTo(4L));
     }
 
+    @Test
+    public void shouldGetToRunningWithOnlyGlobalTopology() throws Exception {
+        builder = new StreamsBuilder();
+        globalTable = builder.globalTable(
+            globalTableTopic,
+            Consumed.with(Serdes.Long(), Serdes.String()),
+            Materialized.as(Stores.inMemoryKeyValueStore(globalStore)));
+
+        startStreams();
+        waitForApplicationState(singletonList(kafkaStreams), State.RUNNING, Duration.ofSeconds(30));
+
+        kafkaStreams.close();
+    }
+
     private void createTopics() throws Exception {
         streamTopic = "stream-" + testNo;
         globalTableTopic = "globalTable-" + testNo;
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java b/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java
index 4921c4f39dabf..29e584c4af043 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java
@@ -809,6 +809,33 @@ public static void startApplicationAndWaitUntilRunning(final List
         }
     }
 
+    /**
+     * Waits for the given {@link KafkaStreams} instances to all be in a  {@link State#RUNNING}
+     * state. Prefer {@link #startApplicationAndWaitUntilRunning(List, Duration)} when possible
+     * because this method uses polling, which can be more error prone and slightly slower.
+     *
+     * @param streamsList the list of streams instances to run.
+     * @param timeout the time to wait for the streams to all be in {@link State#RUNNING} state.
+     */
+    public static void waitForApplicationState(final List streamsList,
+                                               final State state,
+                                               final Duration timeout) throws InterruptedException {
+        retryOnExceptionWithTimeout(timeout.toMillis(), () -> {
+            final Map streamsToStates = streamsList
+                .stream()
+                .collect(Collectors.toMap(stream -> stream, KafkaStreams::state));
+
+            final Map wrongStateMap = streamsToStates.entrySet()
+                .stream()
+                .filter(entry -> entry.getValue() != state)
+                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+            final String reason = String.format("Expected all streams instances in %s to be %s within %d ms, but the following were not: %s",
+                                                streamsList, state, timeout.toMillis(), wrongStateMap);
+            assertThat(reason, wrongStateMap.isEmpty());
+        });
+    }
+
     private static StateListener getStateListener(final KafkaStreams streams) {
         try {
             final Field field = streams.getClass().getDeclaredField("stateListener");

From cc26fccea330087441fe276f1d2047f067e49e46 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Tue, 12 May 2020 12:06:08 -0700
Subject: [PATCH 0950/1071] KAFKA-9669; Loosen validation of inner offsets for
 older message formats (#8647)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Prior to KAFKA-8106, we allowed the v0 and v1 message formats to contain non-consecutive inner offsets. Inside `LogValidator`, we would detect this case and rewrite the batch. After KAFKA-8106, we changed the logic to raise an error in the case of the v1 message format (v0 was still expected to be rewritten). This caused an incompatibility for older clients which were depending on the looser validation. This patch reverts the old logic of rewriting the batch to fix the invalid inner offsets.

Note that the v2 message format has always had stricter validation. This patch also adds a test case for this.

Reviewers: José Armando García Sancio , Ismael Juma 
---
 .../common/record/MemoryRecordsBuilder.java   | 30 ++++++++++++
 .../main/scala/kafka/log/LogValidator.scala   | 14 +++---
 .../unit/kafka/log/LogValidatorTest.scala     | 49 +++++++++++++++++--
 3 files changed, 83 insertions(+), 10 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java
index 054fb86199884..fd5a680cda835 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java
@@ -20,6 +20,7 @@
 import org.apache.kafka.common.header.Header;
 import org.apache.kafka.common.protocol.types.Struct;
 import org.apache.kafka.common.utils.ByteBufferOutputStream;
+import org.apache.kafka.common.utils.Utils;
 
 import java.io.DataOutputStream;
 import java.io.IOException;
@@ -587,6 +588,35 @@ public void appendUncheckedWithOffset(long offset, LegacyRecord record) {
         }
     }
 
+    /**
+     * Append a record without doing offset/magic validation (this should only be used in testing).
+     *
+     * @param offset The offset of the record
+     * @param record The record to add
+     */
+    public void appendUncheckedWithOffset(long offset, SimpleRecord record) throws IOException {
+        if (magic >= RecordBatch.MAGIC_VALUE_V2) {
+            int offsetDelta = (int) (offset - baseOffset);
+            long timestamp = record.timestamp();
+            if (firstTimestamp == null)
+                firstTimestamp = timestamp;
+
+            int sizeInBytes = DefaultRecord.writeTo(appendStream,
+                offsetDelta,
+                timestamp - firstTimestamp,
+                record.key(),
+                record.value(),
+                record.headers());
+            recordWritten(offset, timestamp, sizeInBytes);
+        } else {
+            LegacyRecord legacyRecord = LegacyRecord.create(magic,
+                record.timestamp(),
+                Utils.toNullableArray(record.key()),
+                Utils.toNullableArray(record.value()));
+            appendUncheckedWithOffset(offset, legacyRecord);
+        }
+    }
+
     /**
      * Append a record at the next sequential offset.
      * @param record the record to add
diff --git a/core/src/main/scala/kafka/log/LogValidator.scala b/core/src/main/scala/kafka/log/LogValidator.scala
index b72ab1fc41918..b31d45abfaa86 100644
--- a/core/src/main/scala/kafka/log/LogValidator.scala
+++ b/core/src/main/scala/kafka/log/LogValidator.scala
@@ -394,14 +394,14 @@ private[log] object LogValidator extends Logging {
 
           uncompressedSizeInBytes += record.sizeInBytes()
           if (batch.magic > RecordBatch.MAGIC_VALUE_V0 && toMagic > RecordBatch.MAGIC_VALUE_V0) {
-            // inner records offset should always be continuous
+            // Some older clients do not implement the V1 internal offsets correctly.
+            // Historically the broker handled this by rewriting the batches rather
+            // than rejecting the request. We must continue this handling here to avoid
+            // breaking these clients.
             val expectedOffset = expectedInnerOffset.getAndIncrement()
-            if (record.offset != expectedOffset) {
-              brokerTopicStats.allTopicsStats.invalidOffsetOrSequenceRecordsPerSec.mark()
-              throw new RecordValidationException(
-                new InvalidRecordException(s"Inner record $record inside the compressed record batch does not have incremental offsets, expected offset is $expectedOffset in topic partition $topicPartition."),
-                List(new RecordError(batchIndex)))
-            }
+            if (record.offset != expectedOffset)
+              inPlaceAssignment = false
+
             if (record.timestamp > maxTimestamp)
               maxTimestamp = record.timestamp
           }
diff --git a/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala b/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
index 0515e79b4b2af..ba56f1449faee 100644
--- a/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
@@ -22,14 +22,14 @@ import java.util.concurrent.TimeUnit
 import com.yammer.metrics.Metrics
 import kafka.api.{ApiVersion, KAFKA_2_0_IV1, KAFKA_2_3_IV1}
 import kafka.common.{LongRef, RecordValidationException}
+import kafka.log.LogValidator.ValidationAndOffsetAssignResult
 import kafka.message._
 import kafka.server.BrokerTopicStats
 import kafka.utils.TestUtils.meterCount
-import org.apache.kafka.common.InvalidRecordException
-import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.{InvalidTimestampException, UnsupportedCompressionTypeException, UnsupportedForMessageFormatException}
 import org.apache.kafka.common.record._
 import org.apache.kafka.common.utils.Time
+import org.apache.kafka.common.{InvalidRecordException, TopicPartition}
 import org.apache.kafka.test.TestUtils
 import org.junit.Assert._
 import org.junit.Test
@@ -64,6 +64,29 @@ class LogValidatorTest {
     checkAllowMultiBatch(RecordBatch.MAGIC_VALUE_V1, CompressionType.NONE, CompressionType.GZIP)
   }
 
+  @Test
+  def testValidationOfBatchesWithNonSequentialInnerOffsets(): Unit = {
+    def testMessageValidation(magicValue: Byte): Unit = {
+      val numRecords = 20
+      val invalidRecords = recordsWithNonSequentialInnerOffsets(magicValue, CompressionType.GZIP, numRecords)
+
+      // Validation for v2 and above is strict for this case. For older formats, we fix invalid
+      // internal offsets by rewriting the batch.
+      if (magicValue >= RecordBatch.MAGIC_VALUE_V2) {
+        assertThrows[InvalidRecordException] {
+          validateMessages(invalidRecords, magicValue, CompressionType.GZIP, CompressionType.GZIP)
+        }
+      } else {
+        val result = validateMessages(invalidRecords, magicValue, CompressionType.GZIP, CompressionType.GZIP)
+        assertEquals(0 until numRecords, result.validatedRecords.records.asScala.map(_.offset))
+      }
+    }
+
+    for (version <- RecordVersion.values) {
+      testMessageValidation(version.value)
+    }
+  }
+
   @Test
   def testMisMatchMagic(): Unit = {
     checkMismatchMagic(RecordBatch.MAGIC_VALUE_V0, RecordBatch.MAGIC_VALUE_V1, CompressionType.GZIP)
@@ -88,7 +111,10 @@ class LogValidatorTest {
     assertTrue(meterCount(s"${BrokerTopicStats.InvalidMagicNumberRecordsPerSec}") > 0)
   }
 
-  private def validateMessages(records: MemoryRecords, magic: Byte, sourceCompressionType: CompressionType, targetCompressionType: CompressionType): Unit = {
+  private def validateMessages(records: MemoryRecords,
+                               magic: Byte,
+                               sourceCompressionType: CompressionType,
+                               targetCompressionType: CompressionType): ValidationAndOffsetAssignResult = {
     LogValidator.validateMessagesAndAssignOffsets(records,
       topicPartition,
       new LongRef(0L),
@@ -1371,6 +1397,23 @@ class LogValidatorTest {
     }
   }
 
+  private def recordsWithNonSequentialInnerOffsets(magicValue: Byte,
+                                                   codec: CompressionType,
+                                                   numRecords: Int): MemoryRecords = {
+    val records = (0 until numRecords).map { id =>
+      new SimpleRecord(id.toString.getBytes)
+    }
+
+    val buffer = ByteBuffer.allocate(1024)
+    val builder = MemoryRecords.builder(buffer, magicValue, codec, TimestampType.CREATE_TIME, 0L)
+
+    records.foreach { record =>
+      builder.appendUncheckedWithOffset(0, record)
+    }
+
+    builder.build()
+  }
+
   private def recordsWithInvalidInnerMagic(batchMagicValue: Byte,
                                            recordMagicValue: Byte,
                                            codec: CompressionType): MemoryRecords = {

From c7633dcbe9ac18bec8857ca0b9661a03715a5fc4 Mon Sep 17 00:00:00 2001
From: Jeremy Custenborder 
Date: Fri, 15 May 2020 12:14:20 -0500
Subject: [PATCH 0951/1071] KAFKA-9537 - Cleanup error messages for abstract
 transformations (#8090)

Added check if the transformation is abstract. If so throw an error message with guidance for the user. Ensure that the child classes are also not abstract.

Author: Jeremy Custenborder 
Reviewer: Randall Hauch 
---
 .../connect/runtime/ConnectorConfig.java      | 19 ++++-
 .../connect/runtime/ConnectorConfigTest.java  | 73 +++++++++++++++++++
 2 files changed, 91 insertions(+), 1 deletion(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectorConfig.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectorConfig.java
index 632f5d3454e70..b7340ae1e8e03 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectorConfig.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectorConfig.java
@@ -29,6 +29,7 @@
 import org.apache.kafka.connect.runtime.isolation.Plugins;
 import org.apache.kafka.connect.transforms.Transformation;
 
+import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -37,6 +38,8 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.apache.kafka.common.config.ConfigDef.NonEmptyStringWithoutControlChars.nonEmptyStringWithoutControlChars;
 import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
@@ -329,11 +332,25 @@ static ConfigDef getConfigDefFromTransformation(String key, Class transformat
         if (transformationCls == null || !Transformation.class.isAssignableFrom(transformationCls)) {
             throw new ConfigException(key, String.valueOf(transformationCls), "Not a Transformation");
         }
+        if (Modifier.isAbstract(transformationCls.getModifiers())) {
+            String childClassNames = Stream.of(transformationCls.getClasses())
+                .filter(transformationCls::isAssignableFrom)
+                .filter(c -> !Modifier.isAbstract(c.getModifiers()))
+                .filter(c -> Modifier.isPublic(c.getModifiers()))
+                .map(Class::getName)
+                .collect(Collectors.joining(", "));
+            String message = childClassNames.trim().isEmpty() ?
+                "Transformation is abstract and cannot be created." :
+                "Transformation is abstract and cannot be created. Did you mean " + childClassNames + "?";
+            throw new ConfigException(key, String.valueOf(transformationCls), message);
+        }
         Transformation transformation;
         try {
             transformation = transformationCls.asSubclass(Transformation.class).newInstance();
         } catch (Exception e) {
-            throw new ConfigException(key, String.valueOf(transformationCls), "Error getting config definition from Transformation: " + e.getMessage());
+            ConfigException exception = new ConfigException(key, String.valueOf(transformationCls), "Error getting config definition from Transformation: " + e.getMessage());
+            exception.initCause(e);
+            throw exception;
         }
         ConfigDef configDef = transformation.config();
         if (null == configDef) {
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ConnectorConfigTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ConnectorConfigTest.java
index fe1bf26465468..f674a8e22f244 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ConnectorConfigTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ConnectorConfigTest.java
@@ -177,4 +177,77 @@ public void multipleTransforms() {
         assertEquals(84, ((SimpleTransformation) transformations.get(1)).magicNumber);
     }
 
+    @Test
+    public void abstractTransform() {
+        Map props = new HashMap<>();
+        props.put("name", "test");
+        props.put("connector.class", TestConnector.class.getName());
+        props.put("transforms", "a");
+        props.put("transforms.a.type", AbstractTransformation.class.getName());
+        try {
+            new ConnectorConfig(MOCK_PLUGINS, props);
+        } catch (ConfigException ex) {
+            assertTrue(
+                ex.getMessage().contains("Transformation is abstract and cannot be created.")
+            );
+        }
+    }
+    @Test
+    public void abstractKeyValueTransform() {
+        Map props = new HashMap<>();
+        props.put("name", "test");
+        props.put("connector.class", TestConnector.class.getName());
+        props.put("transforms", "a");
+        props.put("transforms.a.type", AbstractKeyValueTransformation.class.getName());
+        try {
+            new ConnectorConfig(MOCK_PLUGINS, props);
+        } catch (ConfigException ex) {
+            assertTrue(
+                ex.getMessage().contains("Transformation is abstract and cannot be created.")
+            );
+            assertTrue(
+                ex.getMessage().contains(AbstractKeyValueTransformation.Key.class.getName())
+            );
+            assertTrue(
+                ex.getMessage().contains(AbstractKeyValueTransformation.Value.class.getName())
+            );
+        }
+    }
+
+    public static abstract class AbstractTransformation> implements Transformation  {
+
+    }
+
+    public static abstract class AbstractKeyValueTransformation> implements Transformation  {
+        @Override
+        public R apply(R record) {
+            return null;
+        }
+
+        @Override
+        public ConfigDef config() {
+            return new ConfigDef();
+        }
+
+        @Override
+        public void close() {
+
+        }
+
+        @Override
+        public void configure(Map configs) {
+
+        }
+
+
+        public static class Key extends AbstractKeyValueTransformation {
+
+
+        }
+        public static class Value extends AbstractKeyValueTransformation {
+
+        }
+    }
+
+
 }

From bdc3126137088f74f991d21ef3beec5b33b5bc07 Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Fri, 15 May 2020 08:56:00 -0700
Subject: [PATCH 0952/1071] KAFKA-9996: Upgrade zookeeper to 3.5.8 (#8674)

It fixes 30 issues, including third party CVE fixes, several leader-election
related fixes and a compatibility issue with applications built against earlier
3.5 client libraries (by restoring a few non public APIs).

See ZooKeeper 3.5.8 Release Notes for details: https://zookeeper.apache.org/doc/r3.5.8/releasenotes.html

Reviewers: Manikumar Reddy 
---
 gradle/dependencies.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle
index 7597b9c52da06..da353006c7036 100644
--- a/gradle/dependencies.gradle
+++ b/gradle/dependencies.gradle
@@ -114,7 +114,7 @@ versions += [
   spotbugs: "3.1.12",
   spotbugsPlugin: "1.6.9",
   spotlessPlugin: "3.23.1",
-  zookeeper: "3.5.7",
+  zookeeper: "3.5.8",
   zstd: "1.4.3-1"
 ]
 

From 03671d3bb5ea6e25602de82bf9197be5481c1d9f Mon Sep 17 00:00:00 2001
From: Greg Harris 
Date: Fri, 15 May 2020 17:53:32 -0700
Subject: [PATCH 0953/1071] KAFKA-9955: Prevent SinkTask::close from shadowing
 other exceptions (#8618)

* If two exceptions are thrown the `closePartitions` exception is suppressed
* Add unit tests that throw exceptions in put and close to verify that
  the exceptions are propagated and suppressed appropriately out of WorkerSinkTask::execute

Reviewers: Chris Egerton , Nigel Liang , Konstantine Karantasis 
---
 .../org/apache/kafka/common/utils/Utils.java  | 12 +++
 .../kafka/connect/runtime/WorkerSinkTask.java |  9 +-
 .../connect/runtime/WorkerSinkTaskTest.java   | 84 +++++++++++++++++++
 3 files changed, 100 insertions(+), 5 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/utils/Utils.java b/clients/src/main/java/org/apache/kafka/common/utils/Utils.java
index c7b70af60a164..5a69db0c021a4 100755
--- a/clients/src/main/java/org/apache/kafka/common/utils/Utils.java
+++ b/clients/src/main/java/org/apache/kafka/common/utils/Utils.java
@@ -825,6 +825,18 @@ public static void closeAll(Closeable... closeables) throws IOException {
             throw exception;
     }
 
+    /**
+     * An {@link AutoCloseable} interface without a throws clause in the signature
+     *
+     * This is used with lambda expressions in try-with-resources clauses
+     * to avoid casting un-checked exceptions to checked exceptions unnecessarily.
+     */
+    @FunctionalInterface
+    public interface UncheckedCloseable extends AutoCloseable {
+        @Override
+        void close();
+    }
+
     /**
      * Closes {@code closeable} and if an exception is thrown, it is logged at the WARN level.
      */
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
index f89918c220e85..227f2f63a47d5 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
@@ -33,6 +33,7 @@
 import org.apache.kafka.common.metrics.stats.Value;
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.common.utils.Utils.UncheckedCloseable;
 import org.apache.kafka.connect.data.SchemaAndValue;
 import org.apache.kafka.connect.errors.ConnectException;
 import org.apache.kafka.connect.errors.RetriableException;
@@ -189,13 +190,11 @@ public void transitionTo(TargetState state) {
     @Override
     public void execute() {
         initializeAndStart();
-        try {
+        // Make sure any uncommitted data has been committed and the task has
+        // a chance to clean up its state
+        try (UncheckedCloseable suppressible = this::closePartitions) {
             while (!isStopping())
                 iteration();
-        } finally {
-            // Make sure any uncommitted data has been committed and the task has
-            // a chance to clean up its state
-            closePartitions();
         }
     }
 
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskTest.java
index 8c93528b76521..c6d25a4b82224 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskTest.java
@@ -35,6 +35,7 @@
 import org.apache.kafka.common.utils.MockTime;
 import org.apache.kafka.connect.data.Schema;
 import org.apache.kafka.connect.data.SchemaAndValue;
+import org.apache.kafka.connect.errors.ConnectException;
 import org.apache.kafka.connect.errors.RetriableException;
 import org.apache.kafka.connect.runtime.ConnectMetrics.MetricGroup;
 import org.apache.kafka.connect.runtime.distributed.ClusterConfigState;
@@ -83,6 +84,7 @@
 
 import static java.util.Arrays.asList;
 import static java.util.Collections.singleton;
+import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
@@ -843,6 +845,88 @@ public void run() {
         PowerMock.verifyAll();
     }
 
+    @Test
+    public void testSinkTasksHandleCloseErrors() throws Exception {
+        createTask(initialState);
+        expectInitializeTask();
+
+        // Put one message through the task to get some offsets to commit
+        expectConsumerPoll(1);
+        expectConversionAndTransformation(1);
+        sinkTask.put(EasyMock.anyObject());
+        PowerMock.expectLastCall().andVoid();
+
+        // Stop the task during the next put
+        expectConsumerPoll(1);
+        expectConversionAndTransformation(1);
+        sinkTask.put(EasyMock.anyObject());
+        PowerMock.expectLastCall().andAnswer(() -> {
+            workerTask.stop();
+            return null;
+        });
+
+        consumer.wakeup();
+        PowerMock.expectLastCall();
+
+        // Throw another exception while closing the task's assignment
+        EasyMock.expect(sinkTask.preCommit(EasyMock.anyObject()))
+            .andStubReturn(Collections.emptyMap());
+        Throwable closeException = new RuntimeException();
+        sinkTask.close(EasyMock.anyObject());
+        PowerMock.expectLastCall().andThrow(closeException);
+
+        PowerMock.replayAll();
+
+        workerTask.initialize(TASK_CONFIG);
+        try {
+            workerTask.execute();
+            fail("workerTask.execute should have thrown an exception");
+        } catch (RuntimeException e) {
+            PowerMock.verifyAll();
+            assertSame("Exception from close should propagate as-is", closeException, e);
+        }
+    }
+
+    @Test
+    public void testSuppressCloseErrors() throws Exception {
+        createTask(initialState);
+        expectInitializeTask();
+
+        // Put one message through the task to get some offsets to commit
+        expectConsumerPoll(1);
+        expectConversionAndTransformation(1);
+        sinkTask.put(EasyMock.anyObject());
+        PowerMock.expectLastCall().andVoid();
+
+        // Throw an exception on the next put to trigger shutdown behavior
+        // This exception is the true "cause" of the failure
+        expectConsumerPoll(1);
+        expectConversionAndTransformation(1);
+        Throwable putException = new RuntimeException();
+        sinkTask.put(EasyMock.anyObject());
+        PowerMock.expectLastCall().andThrow(putException);
+
+        // Throw another exception while closing the task's assignment
+        EasyMock.expect(sinkTask.preCommit(EasyMock.anyObject()))
+            .andStubReturn(Collections.emptyMap());
+        Throwable closeException = new RuntimeException();
+        sinkTask.close(EasyMock.anyObject());
+        PowerMock.expectLastCall().andThrow(closeException);
+
+        PowerMock.replayAll();
+
+        workerTask.initialize(TASK_CONFIG);
+        try {
+            workerTask.execute();
+            fail("workerTask.execute should have thrown an exception");
+        } catch (ConnectException e) {
+            PowerMock.verifyAll();
+            assertSame("Exception from put should be the cause", putException, e.getCause());
+            assertTrue("Exception from close should be suppressed", e.getSuppressed().length > 0);
+            assertSame(closeException, e.getSuppressed()[0]);
+        }
+    }
+
     // Verify that when commitAsync is called but the supplied callback is not called by the consumer before a
     // rebalance occurs, the async callback does not reset the last committed offset from the rebalance.
     // See KAFKA-5731 for more information.

From d1200f3f4098667afc729d162befa1a3dd407d5e Mon Sep 17 00:00:00 2001
From: Andras Katona <41361962+akatona84@users.noreply.github.com>
Date: Tue, 19 May 2020 20:38:35 +0200
Subject: [PATCH 0954/1071] KAFKA-9992: Eliminate JavaConverters in
 EmbeddedKafkaCluster (#8673)

Fixes EmbeddedKafkaCluster.deleteTopicAndWait for use with kafka_2.13

Reviewers: Boyang Chen , Chia-Ping Tsai , John Roesler 
---
 .../integration/utils/EmbeddedKafkaCluster.java | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/utils/EmbeddedKafkaCluster.java b/streams/src/test/java/org/apache/kafka/streams/integration/utils/EmbeddedKafkaCluster.java
index f30ecedc3aaa1..69c533a2d1f80 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/utils/EmbeddedKafkaCluster.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/utils/EmbeddedKafkaCluster.java
@@ -27,7 +27,6 @@
 import org.junit.rules.ExternalResource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import scala.collection.JavaConverters;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -273,7 +272,7 @@ public void deleteTopicsAndWait(final long timeoutMs, final String... topics) th
      * @param timeoutMs the max time to wait for the topics to be deleted (does not block if {@code <= 0})
      */
     public void deleteAllTopicsAndWait(final long timeoutMs) throws InterruptedException {
-        final Set topics = JavaConverters.setAsJavaSetConverter(brokers[0].kafkaServer().zkClient().getAllTopicsInCluster()).asJava();
+        final Set topics = getAllTopicsInCluster();
         for (final String topic : topics) {
             try {
                 brokers[0].deleteTopic(topic);
@@ -312,8 +311,7 @@ private TopicsDeletedCondition(final Collection topics) {
 
         @Override
         public boolean conditionMet() {
-            final Set allTopics = new HashSet<>(
-                    JavaConverters.setAsJavaSetConverter(brokers[0].kafkaServer().zkClient().getAllTopicsInCluster()).asJava());
+            final Set allTopics = getAllTopicsInCluster();
             return !allTopics.removeAll(deletedTopics);
         }
     }
@@ -327,7 +325,7 @@ private TopicsRemainingCondition(final String... topics) {
 
         @Override
         public boolean conditionMet() {
-            final Set allTopics = JavaConverters.setAsJavaSetConverter(brokers[0].kafkaServer().zkClient().getAllTopicsInCluster()).asJava();
+            final Set allTopics = getAllTopicsInCluster();
             return allTopics.equals(remainingTopics);
         }
     }
@@ -339,4 +337,13 @@ private List brokers() {
         }
         return servers;
     }
+
+    public Set getAllTopicsInCluster() {
+        final scala.collection.Iterator topicsIterator = brokers[0].kafkaServer().zkClient().getAllTopicsInCluster().iterator();
+        final Set topics = new HashSet<>();
+        while (topicsIterator.hasNext()) {
+            topics.add(topicsIterator.next());
+        }
+        return topics;
+    }
 }

From 5cea8210913459c2f2909cb43e4e570460abe6fd Mon Sep 17 00:00:00 2001
From: Chris Egerton 
Date: Wed, 20 May 2020 20:15:43 -0700
Subject: [PATCH 0955/1071] KAFKA-8869: Remove task configs for deleted
 connectors from config snapshot (#8444)

Currently, if a connector is deleted, its task configurations will remain in the config snapshot tracked by the KafkaConfigBackingStore. This causes issues with incremental cooperative rebalancing, which utilizes that config snapshot to determine which connectors and tasks need to be assigned across the cluster. Specifically, it first checks to see which connectors are present in the config snapshot, and then, for each of those connectors, queries the snapshot for that connector's task configs.

The lifecycle of a connector is for its configuration to be written to the config topic, that write to be picked up by the workers in the cluster and trigger a rebalance, the connector to be assigned to and started by a worker, task configs to be generated by the connector and then written to the config topic, that write to be picked up by the workers in the cluster and trigger a second rebalance, and finally, the tasks to be assigned to and started by workers across the cluster.

There is a brief period in between the first time the connector is started and when the second rebalance has completed during which those stale task configs from a previously-deleted version of the connector will be used by the framework to start tasks for that connector. This fix aims to eliminate that window by preemptively clearing the task configs from the config snapshot for a connector whenever it has been deleted.

An existing unit test is modified to verify this behavior, and should provide sufficient guarantees that the bug has been fixed.

Reviewers: Nigel Liang , Konstantine Karantasis 
---
 .../connect/storage/KafkaConfigBackingStore.java     |  2 ++
 .../connect/storage/KafkaConfigBackingStoreTest.java | 12 +++++++++++-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java
index 1227da55f9f43..39a8f35be3014 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java
@@ -548,6 +548,8 @@ public void onCompletion(Throwable error, ConsumerRecord record)
                         // Connector deletion will be written as a null value
                         log.info("Successfully processed removal of connector '{}'", connectorName);
                         connectorConfigs.remove(connectorName);
+                        connectorTaskCounts.remove(connectorName);
+                        taskConfigs.keySet().removeIf(taskId -> taskId.connector().equals(connectorName));
                         removed = true;
                     } else {
                         // Connector configs can be applied and callbacks invoked immediately
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java
index 03ce9eb95e394..5e98fce254411 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java
@@ -57,6 +57,7 @@
 import java.util.concurrent.TimeUnit;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
@@ -537,7 +538,7 @@ public void testBackgroundConnectorDeletion() throws Exception {
         LinkedHashMap deserialized = new LinkedHashMap<>();
         deserialized.put(CONFIGS_SERIALIZED.get(0), CONNECTOR_CONFIG_STRUCTS.get(0));
         deserialized.put(CONFIGS_SERIALIZED.get(1), TASK_CONFIG_STRUCTS.get(0));
-        deserialized.put(CONFIGS_SERIALIZED.get(2), TASK_CONFIG_STRUCTS.get(0));
+        deserialized.put(CONFIGS_SERIALIZED.get(2), TASK_CONFIG_STRUCTS.get(1));
         deserialized.put(CONFIGS_SERIALIZED.get(3), TASKS_COMMIT_STRUCT_TWO_TASK_CONNECTOR);
         logOffset = 5;
 
@@ -566,8 +567,17 @@ public void testBackgroundConnectorDeletion() throws Exception {
         // Should see a single connector with initial state paused
         ClusterConfigState configState = configStorage.snapshot();
         assertEquals(TargetState.STARTED, configState.targetState(CONNECTOR_IDS.get(0)));
+        assertEquals(SAMPLE_CONFIGS.get(0), configState.connectorConfig(CONNECTOR_IDS.get(0)));
+        assertEquals(SAMPLE_CONFIGS.subList(0, 2), configState.allTaskConfigs(CONNECTOR_IDS.get(0)));
+        assertEquals(2, configState.taskCount(CONNECTOR_IDS.get(0)));
 
         configStorage.refresh(0, TimeUnit.SECONDS);
+        configState = configStorage.snapshot();
+        // Connector should now be removed from the snapshot
+        assertFalse(configState.contains(CONNECTOR_IDS.get(0)));
+        // Task configs for the deleted connector should also be removed from the snapshot
+        assertEquals(Collections.emptyList(), configState.allTaskConfigs(CONNECTOR_IDS.get(0)));
+        assertEquals(0, configState.taskCount(CONNECTOR_IDS.get(0)));
 
         configStorage.stop();
 

From 298337281550b1cbb6c78202ea2c03f3a6549740 Mon Sep 17 00:00:00 2001
From: Chris Egerton 
Date: Wed, 20 May 2020 21:14:40 -0700
Subject: [PATCH 0956/1071] KAFKA-9950: Construct new ConfigDef for
 MirrorTaskConfig before defining new properties (#8608)

`MirrorTaskConfig` class mutates the `ConfigDef` by defining additional properties, which leads to a potential `ConcurrentModificationException` during worker configuration validation and unintended inclusion of those new properties in the `ConfigDef` for the connectors which in turn is then visible via the REST API's `/connectors/{name}/config/validate` endpoint.

The fix here is a one-liner that just creates a copy of the `ConfigDef` before defining new properties.

Reviewers: Ryanne Dolan , Konstantine Karantasis 
---
 .../connect/mirror/MirrorTaskConfig.java      |  2 +-
 .../mirror/MirrorConnectorConfigTest.java     | 25 +++++++++++++++++++
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorTaskConfig.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorTaskConfig.java
index 839d41eb04040..73024f5914f4e 100644
--- a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorTaskConfig.java
+++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorTaskConfig.java
@@ -59,7 +59,7 @@ MirrorMetrics metrics() {
         return metrics;
     }
  
-    protected static final ConfigDef TASK_CONFIG_DEF = CONNECTOR_CONFIG_DEF
+    protected static final ConfigDef TASK_CONFIG_DEF = new ConfigDef(CONNECTOR_CONFIG_DEF)
         .define(
             TASK_TOPIC_PARTITIONS,
             ConfigDef.Type.LIST,
diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorConfigTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorConfigTest.java
index 6003202bfc3b3..8e99779e8f7c0 100644
--- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorConfigTest.java
+++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorConfigTest.java
@@ -17,9 +17,11 @@
 package org.apache.kafka.connect.mirror;
 
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.config.ConfigDef;
 import org.junit.Test;
 
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.HashMap;
@@ -106,4 +108,27 @@ public void testListOfTopics() {
         assertTrue(config.topicFilter().shouldReplicateTopic("topic2"));
         assertFalse(config.topicFilter().shouldReplicateTopic("topic3"));
     }
+
+    @Test
+    public void testNonMutationOfConfigDef() {
+        Collection taskSpecificProperties = Arrays.asList(
+            MirrorConnectorConfig.TASK_TOPIC_PARTITIONS,
+            MirrorConnectorConfig.TASK_CONSUMER_GROUPS
+        );
+
+        // Sanity check to make sure that these properties are actually defined for the task config,
+        // and that the task config class has been loaded and statically initialized by the JVM
+        ConfigDef taskConfigDef = MirrorTaskConfig.TASK_CONFIG_DEF;
+        taskSpecificProperties.forEach(taskSpecificProperty -> assertTrue(
+            taskSpecificProperty + " should be defined for task ConfigDef",
+            taskConfigDef.names().contains(taskSpecificProperty)
+        ));
+
+        // Ensure that the task config class hasn't accidentally modified the connector config
+        ConfigDef connectorConfigDef = MirrorConnectorConfig.CONNECTOR_CONFIG_DEF;
+        taskSpecificProperties.forEach(taskSpecificProperty -> assertFalse(
+            taskSpecificProperty + " should not be defined for connector ConfigDef",
+            connectorConfigDef.names().contains(taskSpecificProperty)
+        ));
+    }
 }

From 76ccc1dac186830191d75ce767c8867bd247e109 Mon Sep 17 00:00:00 2001
From: Randall Hauch 
Date: Thu, 21 May 2020 10:56:04 -0500
Subject: [PATCH 0957/1071] MINOR: Correct MirrorMaker2 integration test
 configs for Connect internal topics (#8653)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The replication factor for the Connect workers’ internal topics were incorrectly named and not actually used.
---
 .../connect/mirror/MirrorConnectorsIntegrationTest.java     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java
index aacf42ba63f4c..30924ef84bff4 100644
--- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java
+++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java
@@ -85,9 +85,9 @@ public void setup() throws InterruptedException {
         mm2Props.put("checkpoints.topic.replication.factor", "1");
         mm2Props.put("heartbeats.topic.replication.factor", "1");
         mm2Props.put("offset-syncs.topic.replication.factor", "1");
-        mm2Props.put("config.storage.topic.replication.factor", "1");
-        mm2Props.put("offset.storage.topic.replication.factor", "1");
-        mm2Props.put("status.storage.topic.replication.factor", "1");
+        mm2Props.put("config.storage.replication.factor", "1");
+        mm2Props.put("offset.storage.replication.factor", "1");
+        mm2Props.put("status.storage.replication.factor", "1");
         mm2Props.put("replication.factor", "1");
         
         mm2Config = new MirrorMakerConfig(mm2Props); 

From 0ec8cf591faa9d4784982ed452711466e4243a9e Mon Sep 17 00:00:00 2001
From: Chris Egerton 
Date: Sat, 23 May 2020 15:35:43 -0700
Subject: [PATCH 0958/1071] KAFKA-9888: Copy connector configs before passing
 to REST extensions (#8511)

The changes made in KIP-454 involved adding a `connectorConfig` method to the ConnectClusterState interface that REST extensions could use to query the worker for the configuration of a given connector. The implementation for this method returns the Java `Map` that's stored in the worker's view of the config topic (when running in distributed mode). No copying is performed, which causes mutations of that `Map` to persist across invocations of `connectorConfig` and, even worse, propagate to the worker when, e.g., starting a connector.

In this commit the map is copied before it's returned to REST extensions.

An existing unit test is modified to ensure that REST extensions receive a copy of the connector config, not the original.

Reviewers: Nigel Liang , Konstantine Karantasis 
---
 .../connect/runtime/health/ConnectClusterStateImpl.java  | 2 +-
 .../kafka/connect/storage/KafkaConfigBackingStore.java   | 4 ++--
 .../runtime/health/ConnectClusterStateImplTest.java      | 9 ++++++++-
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/health/ConnectClusterStateImpl.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/health/ConnectClusterStateImpl.java
index 38362b3fc2780..6b7285df50b97 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/health/ConnectClusterStateImpl.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/health/ConnectClusterStateImpl.java
@@ -86,7 +86,7 @@ public Map connectorConfig(String connName) {
         FutureCallback> connectorConfigCallback = new FutureCallback<>();
         herder.connectorConfig(connName, connectorConfigCallback);
         try {
-            return connectorConfigCallback.get(herderRequestTimeoutMs, TimeUnit.MILLISECONDS);
+            return new HashMap<>(connectorConfigCallback.get(herderRequestTimeoutMs, TimeUnit.MILLISECONDS));
         } catch (InterruptedException | ExecutionException | TimeoutException e) {
             throw new ConnectException(
                 String.format("Failed to retrieve configuration for connector '%s'", connName),
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java
index 39a8f35be3014..658d6c3359352 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java
@@ -280,8 +280,8 @@ public void stop() {
     @Override
     public ClusterConfigState snapshot() {
         synchronized (lock) {
-            // Doing a shallow copy of the data is safe here because the complex nested data that is copied should all be
-            // immutable configs
+            // Only a shallow copy is performed here; in order to avoid accidentally corrupting the worker's view
+            // of the config topic, any nested structures should be copied before making modifications
             return new ClusterConfigState(
                     offset,
                     sessionKey,
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/health/ConnectClusterStateImplTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/health/ConnectClusterStateImplTest.java
index d8984f0aaf23b..d8a7e49ee8104 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/health/ConnectClusterStateImplTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/health/ConnectClusterStateImplTest.java
@@ -36,6 +36,7 @@
 import java.util.concurrent.TimeoutException;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
 import static org.junit.Assert.assertThrows;
 
 @RunWith(PowerMockRunner.class)
@@ -87,7 +88,13 @@ public Void answer() {
             }
         });
         EasyMock.replay(herder);
-        assertEquals(expectedConfig, connectClusterState.connectorConfig(connName));
+        Map actualConfig = connectClusterState.connectorConfig(connName);
+        assertEquals(expectedConfig, actualConfig);
+        assertNotSame(
+            "Config should be copied in order to avoid mutation by REST extensions",
+            expectedConfig,
+            actualConfig
+        );
     }
 
     @Test

From 4e523ae458c7097973f5bd96f7da1641877b94f4 Mon Sep 17 00:00:00 2001
From: Chris Egerton 
Date: Mon, 25 May 2020 09:10:03 -0700
Subject: [PATCH 0959/1071] KAFKA-9472: Remove deleted Connect tasks from
 status store (#8118)

Although the statuses for tasks are removed from the status store when their connector is deleted, their statuses are not removed when only the task is deleted, which happens in the case that the number of tasks for a connector is reduced.

This commit adds logic for deleting the statuses for those tasks from the status store whenever a rebalance has completed and the leader of a distributed cluster has detected that there are recently-deleted tasks. Standalone is also updated to accomplish this.

Unit tests for the `DistributedHerder` and `StandaloneHerder` classes are updated and an integration test has been added.

Reviewers: Nigel Liang , Konstantine Karantasis 
---
 .../kafka/connect/runtime/AbstractHerder.java |  7 +++-
 .../kafka/connect/runtime/TaskStatus.java     |  7 ++++
 .../kafka/connect/runtime/WorkerTask.java     |  6 +++
 .../distributed/DistributedHerder.java        | 23 +++++++++--
 .../runtime/standalone/StandaloneHerder.java  |  1 +
 .../ConnectWorkerIntegrationTest.java         | 39 ++++++++++++++++++-
 .../distributed/DistributedHerderTest.java    |  3 +-
 .../standalone/StandaloneHerderTest.java      |  3 ++
 8 files changed, 81 insertions(+), 8 deletions(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
index 9a6553d0b9b96..b5032a07ce3c4 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
@@ -187,10 +187,15 @@ public void onPause(ConnectorTaskId id) {
     @Override
     public void onDeletion(String connector) {
         for (TaskStatus status : statusBackingStore.getAll(connector))
-            statusBackingStore.put(new TaskStatus(status.id(), TaskStatus.State.DESTROYED, workerId, generation()));
+            onDeletion(status.id());
         statusBackingStore.put(new ConnectorStatus(connector, ConnectorStatus.State.DESTROYED, workerId, generation()));
     }
 
+    @Override
+    public void onDeletion(ConnectorTaskId id) {
+        statusBackingStore.put(new TaskStatus(id, TaskStatus.State.DESTROYED, workerId, generation()));
+    }
+
     @Override
     public void pauseConnector(String connector) {
         if (!configBackingStore.contains(connector))
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/TaskStatus.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/TaskStatus.java
index 5ee90415b9660..62bb1c717112b 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/TaskStatus.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/TaskStatus.java
@@ -61,5 +61,12 @@ public interface Listener {
          */
         void onShutdown(ConnectorTaskId id);
 
+        /**
+         * Invoked after the task has been deleted. Can be called if the
+         * connector tasks have been reduced, or if the connector itself has
+         * been deleted.
+         * @param id The id of the task
+         */
+        void onDeletion(ConnectorTaskId id);
     }
 }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerTask.java
index 28d2a8ff96c30..05d18ff0cff36 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerTask.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerTask.java
@@ -415,6 +415,12 @@ public void onShutdown(ConnectorTaskId id) {
             delegateListener.onShutdown(id);
         }
 
+        @Override
+        public void onDeletion(ConnectorTaskId id) {
+            taskStateTimer.changeState(State.DESTROYED, time.milliseconds());
+            delegateListener.onDeletion(id);
+        }
+
         public void recordState(TargetState state) {
             switch (state) {
                 case STARTED:
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
index 1dcc9e65e5797..079865fa5fbb9 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
@@ -46,6 +46,7 @@
 import org.apache.kafka.connect.runtime.SinkConnectorConfig;
 import org.apache.kafka.connect.runtime.SourceConnectorConfig;
 import org.apache.kafka.connect.runtime.TargetState;
+import org.apache.kafka.connect.runtime.TaskStatus;
 import org.apache.kafka.connect.runtime.Worker;
 import org.apache.kafka.connect.runtime.rest.InternalRequestSignature;
 import org.apache.kafka.connect.runtime.rest.RestClient;
@@ -1519,6 +1520,18 @@ private void updateDeletedConnectorStatus() {
         }
     }
 
+    private void updateDeletedTaskStatus() {
+        ClusterConfigState snapshot = configBackingStore.snapshot();
+        for (String connector : statusBackingStore.connectors()) {
+            Set remainingTasks = new HashSet<>(snapshot.tasks(connector));
+            
+            statusBackingStore.getAll(connector).stream()
+                .map(TaskStatus::id)
+                .filter(task -> !remainingTasks.contains(task))
+                .forEach(this::onDeletion);
+        }
+    }
+
     protected HerderMetrics herderMetrics() {
         return herderMetrics;
     }
@@ -1581,11 +1594,13 @@ public void onAssigned(ExtendedAssignment assignment, int generation) {
                 herderMetrics.rebalanceStarted(time.milliseconds());
             }
 
-            // Delete the statuses of all connectors removed prior to the start of this rebalance. This has to
-            // be done after the rebalance completes to avoid race conditions as the previous generation attempts
-            // to change the state to UNASSIGNED after tasks have been stopped.
-            if (isLeader())
+            // Delete the statuses of all connectors and tasks removed prior to the start of this rebalance. This
+            // has to be done after the rebalance completes to avoid race conditions as the previous generation
+            // attempts to change the state to UNASSIGNED after tasks have been stopped.
+            if (isLeader()) {
                 updateDeletedConnectorStatus();
+                updateDeletedTaskStatus();
+            }
 
             // We *must* interrupt any poll() call since this could occur when the poll starts, and we might then
             // sleep in the poll() for a long time. Forcing a wakeup ensures we'll get to process this event in the
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java
index a3e75b5829b5f..6c5398f8e4573 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java
@@ -319,6 +319,7 @@ private void removeConnectorTasks(String connName) {
         if (!tasks.isEmpty()) {
             worker.stopAndAwaitTasks(tasks);
             configBackingStore.removeTaskConfigs(connName);
+            tasks.forEach(this::onDeletion);
         }
     }
 
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java
index b8e04970f610d..b3f3020205528 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java
@@ -102,7 +102,7 @@ public void testAddAndRemoveWorker() throws Exception {
         // create test topic
         connect.kafka().createTopic("test-topic", NUM_TOPIC_PARTITIONS);
 
-        // setup up props for the sink connector
+        // set up props for the source connector
         Map props = new HashMap<>();
         props.put(CONNECTOR_CLASS_CONFIG, MonitorableSourceConnector.class.getSimpleName());
         props.put(TASKS_MAX_CONFIG, String.valueOf(numTasks));
@@ -193,7 +193,7 @@ public void testBrokerCoordinator() throws Exception {
         // create test topic
         connect.kafka().createTopic("test-topic", NUM_TOPIC_PARTITIONS);
 
-        // setup up props for the sink connector
+        // set up props for the source connector
         Map props = new HashMap<>();
         props.put(CONNECTOR_CLASS_CONFIG, MonitorableSourceConnector.class.getSimpleName());
         props.put(TASKS_MAX_CONFIG, String.valueOf(numTasks));
@@ -236,4 +236,39 @@ public void testBrokerCoordinator() throws Exception {
                 "Connector tasks did not start in time.");
     }
 
+    /**
+     * Verify that the number of tasks listed in the REST API is updated correctly after changes to
+     * the "tasks.max" connector configuration.
+     */
+    @Test
+    public void testTaskStatuses() throws Exception {
+        connect = connectBuilder.build();
+        // start the clusters
+        connect.start();
+
+        connect.assertions().assertAtLeastNumWorkersAreUp(NUM_WORKERS,
+            "Initial group of workers did not start in time.");
+
+        // base connector props
+        Map connectorProps = new HashMap<>();
+        connectorProps.put(CONNECTOR_CLASS_CONFIG, MonitorableSourceConnector.class.getSimpleName());
+
+        // start the connector with only one task
+        final int initialNumTasks = 1;
+        connectorProps.put(TASKS_MAX_CONFIG, String.valueOf(initialNumTasks));
+        connect.configureConnector(CONNECTOR_NAME, connectorProps);
+        connect.assertions().assertConnectorAndExactlyNumTasksAreRunning(CONNECTOR_NAME, initialNumTasks, "Connector tasks did not start in time");
+
+        // then reconfigure it to use more tasks
+        final int increasedNumTasks = 5;
+        connectorProps.put(TASKS_MAX_CONFIG, String.valueOf(increasedNumTasks));
+        connect.configureConnector(CONNECTOR_NAME, connectorProps);
+        connect.assertions().assertConnectorAndExactlyNumTasksAreRunning(CONNECTOR_NAME, increasedNumTasks, "Connector task statuses did not update in time.");
+
+        // then reconfigure it to use fewer tasks
+        final int decreasedNumTasks = 3;
+        connectorProps.put(TASKS_MAX_CONFIG, String.valueOf(decreasedNumTasks));
+        connect.configureConnector(CONNECTOR_NAME, connectorProps);
+        connect.assertions().assertConnectorAndExactlyNumTasksAreRunning(CONNECTOR_NAME, decreasedNumTasks, "Connector task statuses did not update in time.");
+    }
 }
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java
index ddbd560fa5c56..cfc08c4ad774c 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java
@@ -198,7 +198,7 @@ public void setUp() throws Exception {
         connectProtocolVersion = CONNECT_PROTOCOL_V0;
 
         herder = PowerMock.createPartialMock(DistributedHerder.class,
-                new String[]{"backoff", "connectorTypeForClass", "updateDeletedConnectorStatus"},
+                new String[]{"backoff", "connectorTypeForClass", "updateDeletedConnectorStatus", "updateDeletedTaskStatus"},
                 new DistributedConfig(HERDER_CONFIG), worker, WORKER_ID, KAFKA_CLUSTER_ID,
                 statusBackingStore, configBackingStore, member, MEMBER_URL, metrics, time, noneConnectorClientConfigOverridePolicy);
 
@@ -212,6 +212,7 @@ public void setUp() throws Exception {
         delegatingLoader = PowerMock.createMock(DelegatingClassLoader.class);
         PowerMock.mockStatic(Plugins.class);
         PowerMock.expectPrivate(herder, "updateDeletedConnectorStatus").andVoid().anyTimes();
+        PowerMock.expectPrivate(herder, "updateDeletedTaskStatus").andVoid().anyTimes();
     }
 
     @After
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java
index 89d4e45fa91e9..feb0f995b7f45 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java
@@ -266,6 +266,7 @@ public void testDestroyConnector() throws Exception {
 
         EasyMock.expect(statusBackingStore.getAll(CONNECTOR_NAME)).andReturn(Collections.emptyList());
         statusBackingStore.put(new ConnectorStatus(CONNECTOR_NAME, AbstractStatus.State.DESTROYED, WORKER_ID, 0));
+        statusBackingStore.put(new TaskStatus(new ConnectorTaskId(CONNECTOR_NAME, 0), TaskStatus.State.DESTROYED, WORKER_ID, 0));
 
         expectDestroy();
 
@@ -434,6 +435,8 @@ public void testCreateAndStop() throws Exception {
         // herder.stop() should stop any running connectors and tasks even if destroyConnector was not invoked
         expectStop();
 
+        statusBackingStore.put(new TaskStatus(new ConnectorTaskId(CONNECTOR_NAME, 0), AbstractStatus.State.DESTROYED, WORKER_ID, 0));
+
         statusBackingStore.stop();
         EasyMock.expectLastCall();
         worker.stop();

From 7dcfdcb70f3e90dac574ff4c879e7bd8592dd795 Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Fri, 29 May 2020 17:03:49 +0100
Subject: [PATCH 0960/1071] KAFKA-10056; Ensure consumer metadata contains new
 topics on subscription change (#8739)

Reviewers: Jason Gustafson 
---
 .../consumer/internals/SubscriptionState.java | 12 +++++-
 .../internals/ConsumerCoordinatorTest.java    | 42 +++++++++++++++++++
 .../internals/SubscriptionStateTest.java      |  6 +++
 3 files changed, 59 insertions(+), 1 deletion(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
index cdf358ecf3a03..1102acafe0d64 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
@@ -339,7 +339,17 @@ public synchronized Set pausedPartitions() {
      *   of the current generation; otherwise it returns the same set as {@link #subscription()}
      */
     synchronized Set metadataTopics() {
-        return groupSubscription.isEmpty() ? subscription : groupSubscription;
+        if (groupSubscription.isEmpty())
+            return subscription;
+        else if (groupSubscription.containsAll(subscription))
+            return groupSubscription;
+        else {
+            // When subscription changes `groupSubscription` may be outdated, ensure that
+            // new subscription topics are returned.
+            Set topics = new HashSet<>(groupSubscription);
+            topics.addAll(subscription);
+            return topics;
+        }
     }
 
     synchronized boolean needsMetadata(String topic) {
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index afef951e3cc54..6d818df83d9b1 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -643,6 +643,35 @@ public void testOutdatedCoordinatorAssignment() {
         assertEquals(assigned, rebalanceListener.assigned);
     }
 
+    @Test
+    public void testMetadataTopicsDuringSubscriptionChange() {
+        final String consumerId = "subscription_change";
+        final List oldSubscription = singletonList(topic1);
+        final List oldAssignment = Collections.singletonList(t1p);
+        final List newSubscription = singletonList(topic2);
+        final List newAssignment = Collections.singletonList(t2p);
+
+        subscriptions.subscribe(toSet(oldSubscription), rebalanceListener);
+        assertEquals(toSet(oldSubscription), subscriptions.metadataTopics());
+
+        client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
+        coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
+
+        prepareJoinAndSyncResponse(consumerId, 1, oldSubscription, oldAssignment);
+
+        coordinator.poll(time.timer(0));
+        assertEquals(toSet(oldSubscription), subscriptions.metadataTopics());
+
+        subscriptions.subscribe(toSet(newSubscription), rebalanceListener);
+        assertEquals(Utils.mkSet(topic1, topic2), subscriptions.metadataTopics());
+
+        prepareJoinAndSyncResponse(consumerId, 2, newSubscription, newAssignment);
+        coordinator.poll(time.timer(Long.MAX_VALUE));
+        assertFalse(coordinator.rejoinNeededOrPending());
+        assertEquals(toSet(newAssignment), subscriptions.assignedPartitions());
+        assertEquals(toSet(newSubscription), subscriptions.metadataTopics());
+    }
+
     @Test
     public void testPatternJoinGroupLeader() {
         final String consumerId = "leader";
@@ -2726,6 +2755,19 @@ private void prepareOffsetCommitRequest(final Map expected
         client.prepareResponse(offsetCommitRequestMatcher(expectedOffsets), offsetCommitResponse(errors), disconnected);
     }
 
+    private void prepareJoinAndSyncResponse(String consumerId, int generation, List subscription, List assignment) {
+        partitionAssignor.prepare(singletonMap(consumerId, assignment));
+        client.prepareResponse(
+                joinGroupLeaderResponse(
+                        generation, consumerId, singletonMap(consumerId, subscription), Errors.NONE));
+        client.prepareResponse(body -> {
+            SyncGroupRequest sync = (SyncGroupRequest) body;
+            return sync.data.memberId().equals(consumerId) &&
+                    sync.data.generationId() == generation &&
+                    sync.groupAssignments().containsKey(consumerId);
+        }, syncGroupResponse(assignment, Errors.NONE));
+    }
+
     private Map partitionErrors(Collection partitions, Errors error) {
         final Map errors = new HashMap<>();
         for (TopicPartition partition : partitions) {
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java
index 47d654e04ac45..58d8b888822ec 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java
@@ -119,6 +119,12 @@ public void testGroupSubscribe() {
         // `groupSubscribe` does not accumulate
         assertFalse(state.groupSubscribe(singleton(topic1)));
         assertEquals(singleton(topic1), state.metadataTopics());
+
+        state.subscribe(singleton("anotherTopic"), rebalanceListener);
+        assertEquals(Utils.mkSet(topic1, "anotherTopic"), state.metadataTopics());
+
+        assertFalse(state.groupSubscribe(singleton("anotherTopic")));
+        assertEquals(singleton("anotherTopic"), state.metadataTopics());
     }
 
     @Test

From 116368604bd28427429844582ddb5fe4623bbb5e Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Mon, 1 Jun 2020 15:57:15 -0700
Subject: [PATCH 0961/1071] KAFKA-9987: optimize sticky assignment algorithm
 for same-subscription case (#8668)

Motivation and pseudo code algorithm in the ticket.

Added a scale test with large number of topic partitions and consumers and 30s timeout.
With these changes, assignment with 2,000 consumers and 200 topics with 2,000 each completes within a few seconds.

Porting the same test to trunk, it took 2 minutes even with a 100x reduction in the number of topics (ie, 2 minutes for 2,000 consumers and 2 topics with 2,000 partitions)

Should be cherry-picked to 2.6, 2.5, and 2.4

Reviewers: Guozhang Wang 
---
 checkstyle/suppressions.xml                   |   4 +-
 .../consumer/CooperativeStickyAssignor.java   |  46 +--
 .../internals/AbstractStickyAssignor.java     | 319 ++++++++++++------
 .../clients/consumer/StickyAssignorTest.java  |   5 +-
 .../internals/AbstractStickyAssignorTest.java |  82 ++---
 5 files changed, 283 insertions(+), 173 deletions(-)

diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index c8678accde7d0..bee700b4ea44a 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -57,13 +57,13 @@
               files="(Utils|Topic|KafkaLZ4BlockOutputStream|AclData|JoinGroupRequest).java"/>
 
     
+              files="(ConsumerCoordinator|Fetcher|Sender|KafkaProducer|BufferPool|ConfigDef|RecordAccumulator|KerberosLogin|AbstractRequest|AbstractResponse|Selector|SslFactory|SslTransportLayer|SaslClientAuthenticator|SaslClientCallbackHandler|SaslServerAuthenticator|AbstractCoordinator|TransactionManager|AbstractStickyAssignor).java"/>
 
     
 
     
+              files="(ConsumerCoordinator|BufferPool|Fetcher|MetricName|Node|ConfigDef|RecordBatch|SslFactory|SslTransportLayer|MetadataResponse|KerberosLogin|Selector|Sender|Serdes|TokenInformation|Agent|Values|PluginUtils|MiniTrogdorCluster|TasksRequest|KafkaProducer|AbstractStickyAssignor).java"/>
 
     
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/CooperativeStickyAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/CooperativeStickyAssignor.java
index bef32bf4d185d..c7c0679575a9b 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/CooperativeStickyAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/CooperativeStickyAssignor.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.clients.consumer;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -62,16 +61,26 @@ protected MemberData memberData(Subscription subscription) {
     @Override
     public Map> assign(Map partitionsPerTopic,
                                                     Map subscriptions) {
+        Map> assignments = super.assign(partitionsPerTopic, subscriptions);
 
-        final Map> assignments = super.assign(partitionsPerTopic, subscriptions);
-        adjustAssignment(subscriptions, assignments);
+        Map partitionsTransferringOwnership = super.partitionsTransferringOwnership == null ?
+            computePartitionsTransferringOwnership(subscriptions, assignments) :
+            super.partitionsTransferringOwnership;
+
+        adjustAssignment(assignments, partitionsTransferringOwnership);
         return assignments;
     }
 
     // Following the cooperative rebalancing protocol requires removing partitions that must first be revoked from the assignment
-    private void adjustAssignment(final Map subscriptions,
-                                  final Map> assignments) {
+    private void adjustAssignment(Map> assignments,
+                                  Map partitionsTransferringOwnership) {
+        for (Map.Entry partitionEntry : partitionsTransferringOwnership.entrySet()) {
+            assignments.get(partitionEntry.getValue()).remove(partitionEntry.getKey());
+        }
+    }
 
+    private Map computePartitionsTransferringOwnership(Map subscriptions,
+                                                                               Map> assignments) {
         Map allAddedPartitions = new HashMap<>();
         Set allRevokedPartitions = new HashSet<>();
 
@@ -81,25 +90,20 @@ private void adjustAssignment(final Map subscriptions,
             List ownedPartitions = subscriptions.get(consumer).ownedPartitions();
             List assignedPartitions = entry.getValue();
 
-            List addedPartitions = new ArrayList<>(assignedPartitions);
-            addedPartitions.removeAll(ownedPartitions);
-            for (TopicPartition tp : addedPartitions) {
-                allAddedPartitions.put(tp, consumer);
+            Set ownedPartitionsSet = new HashSet<>(ownedPartitions);
+            for (TopicPartition tp : assignedPartitions) {
+                if (!ownedPartitionsSet.contains(tp))
+                    allAddedPartitions.put(tp, consumer);
             }
 
-            final Set revokedPartitions = new HashSet<>(ownedPartitions);
-            revokedPartitions.removeAll(assignedPartitions);
-            allRevokedPartitions.addAll(revokedPartitions);
-        }
-
-        // remove any partitions to be revoked from the current assignment
-        for (TopicPartition tp : allRevokedPartitions) {
-            // if partition is being migrated to another consumer, don't assign it there yet
-            if (allAddedPartitions.containsKey(tp)) {
-                String assignedConsumer = allAddedPartitions.get(tp);
-                assignments.get(assignedConsumer).remove(tp);
+            Set assignedPartitionsSet = new HashSet<>(assignedPartitions);
+            for (TopicPartition tp : ownedPartitions) {
+                if (!assignedPartitionsSet.contains(tp))
+                    allRevokedPartitions.add(tp);
             }
         }
-    }
 
+        allAddedPartitions.keySet().retainAll(allRevokedPartitions);
+        return allAddedPartitions;
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractStickyAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractStickyAssignor.java
index 12864dedbca41..d4e023cc098a8 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractStickyAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractStickyAssignor.java
@@ -18,19 +18,22 @@
 
 import java.io.Serializable;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Optional;
+import java.util.Queue;
 import java.util.Set;
+import java.util.SortedSet;
 import java.util.TreeMap;
 import java.util.TreeSet;
+import java.util.stream.Collectors;
 import org.apache.kafka.common.TopicPartition;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -40,7 +43,11 @@ public abstract class AbstractStickyAssignor extends AbstractPartitionAssignor {
 
     public static final int DEFAULT_GENERATION = -1;
 
-    private PartitionMovements partitionMovements;
+    private PartitionMovements partitionMovements = new PartitionMovements();
+
+    // Keep track of the partitions being migrated from one consumer to another during assignment
+    // so the cooperative assignor can adjust the assignment
+    protected Map partitionsTransferringOwnership = new HashMap<>();
 
     static final class ConsumerGenerationPair {
         final String consumer;
@@ -65,9 +72,206 @@ public MemberData(List partitions, Optional generation)
     @Override
     public Map> assign(Map partitionsPerTopic,
                                                     Map subscriptions) {
+        Map> consumerToOwnedPartitions = new HashMap<>();
+        if (allSubscriptionsEqual(partitionsPerTopic.keySet(), subscriptions, consumerToOwnedPartitions)) {
+            log.debug("Detected that all consumers were subscribed to same set of topics, invoking the "
+                          + "optimized assignment algorithm");
+            partitionsTransferringOwnership = new HashMap<>();
+            return constrainedAssign(partitionsPerTopic, consumerToOwnedPartitions);
+        } else {
+            log.debug("Detected that all not consumers were subscribed to same set of topics, falling back to the "
+                          + "general case assignment algorithm");
+            partitionsTransferringOwnership = null;
+            return generalAssign(partitionsPerTopic, subscriptions);
+        }
+    }
+
+    /**
+     * Returns true iff all consumers have an identical subscription. Also fills out the passed in
+     * {@code consumerToOwnedPartitions} with each consumer's previously owned and still-subscribed partitions
+     */
+    private boolean allSubscriptionsEqual(Set allTopics,
+                                          Map subscriptions,
+                                          Map> consumerToOwnedPartitions) {
+        Set membersWithOldGeneration = new HashSet<>();
+        Set membersOfCurrentHighestGeneration = new HashSet<>();
+        int maxGeneration = DEFAULT_GENERATION;
+
+        Set subscribedTopics = new HashSet<>();
+
+        for (Map.Entry subscriptionEntry : subscriptions.entrySet()) {
+            String consumer = subscriptionEntry.getKey();
+            Subscription subscription = subscriptionEntry.getValue();
+
+            // initialize the subscribed topics set if this is the first subscription
+            if (subscribedTopics.isEmpty()) {
+                subscribedTopics.addAll(subscription.topics());
+            } else if (!(subscription.topics().size() == subscribedTopics.size()
+                && subscribedTopics.containsAll(subscription.topics()))) {
+                return false;
+            }
+
+            MemberData memberData = memberData(subscription);
+
+            List ownedPartitions = new ArrayList<>();
+            consumerToOwnedPartitions.put(consumer, ownedPartitions);
+
+            // Only consider this consumer's owned partitions as valid if it is a member of the current highest
+            // generation, or it's generation is not present but we have not seen any known generation so far
+            if (memberData.generation.isPresent() && memberData.generation.get() >= maxGeneration
+                || !memberData.generation.isPresent() && maxGeneration == DEFAULT_GENERATION) {
+
+                membersOfCurrentHighestGeneration.add(consumer);
+                for (final TopicPartition tp : memberData.partitions) {
+                    // filter out any topics that no longer exist or aren't part of the current subscription
+                    if (allTopics.contains(tp.topic())) {
+                        ownedPartitions.add(tp);
+                    }
+                }
+
+                // If the current member's generation is higher, all the previous owned partitions are invalid
+                if (memberData.generation.isPresent() && memberData.generation.get() > maxGeneration) {
+                    membersWithOldGeneration.addAll(membersOfCurrentHighestGeneration);
+                    membersOfCurrentHighestGeneration.clear();
+                    maxGeneration = memberData.generation.get();
+                }
+            }
+        }
+
+        for (String consumer : membersWithOldGeneration) {
+            consumerToOwnedPartitions.get(consumer).clear();
+        }
+        return true;
+    }
+
+    private Map> constrainedAssign(Map partitionsPerTopic,
+                                                                Map> consumerToOwnedPartitions) {
+        SortedSet unassignedPartitions = getTopicPartitions(partitionsPerTopic);
+
+        Set allRevokedPartitions = new HashSet<>();
+
+        // Each consumer should end up in exactly one of the below
+        List unfilledMembers = new LinkedList<>();
+        Queue maxCapacityMembers = new LinkedList<>();
+        Queue minCapacityMembers = new LinkedList<>();
+
+        int numberOfConsumers = consumerToOwnedPartitions.size();
+        int minQuota = (int) Math.floor(((double) unassignedPartitions.size()) / numberOfConsumers);
+        int maxQuota = (int) Math.ceil(((double) unassignedPartitions.size()) / numberOfConsumers);
+
+        // initialize the assignment map with an empty array of size minQuota for all members
+        Map> assignment = new HashMap<>(
+            consumerToOwnedPartitions.keySet().stream().collect(Collectors.toMap(c -> c, c -> new ArrayList<>(minQuota))));
+
+        for (Map.Entry> consumerEntry : consumerToOwnedPartitions.entrySet()) {
+            String consumer = consumerEntry.getKey();
+            List ownedPartitions = consumerEntry.getValue();
+
+            List consumerAssignment = assignment.get(consumer);
+            int i = 0;
+            // assign the first N partitions up to the max quota, and mark the remaining as being revoked
+            for (TopicPartition tp : ownedPartitions) {
+                if (i < maxQuota) {
+                    consumerAssignment.add(tp);
+                    unassignedPartitions.remove(tp);
+                } else {
+                    allRevokedPartitions.add(tp);
+                }
+                ++i;
+            }
+
+            if (ownedPartitions.size() < minQuota) {
+                unfilledMembers.add(consumer);
+            } else {
+                // It's possible for a consumer to be at both min and max capacity if minQuota == maxQuota
+                if (consumerAssignment.size() == minQuota)
+                    minCapacityMembers.add(consumer);
+                if (consumerAssignment.size() == maxQuota)
+                    maxCapacityMembers.add(consumer);
+            }
+        }
+
+        Collections.sort(unfilledMembers);
+        Iterator unassignedPartitionsIter = unassignedPartitions.iterator();
+
+        while (!unfilledMembers.isEmpty() && !unassignedPartitions.isEmpty()) {
+            Iterator unfilledConsumerIter = unfilledMembers.iterator();
+
+            while (unfilledConsumerIter.hasNext()) {
+                String consumer = unfilledConsumerIter.next();
+                List consumerAssignment = assignment.get(consumer);
+
+                if (unassignedPartitionsIter.hasNext()) {
+                    TopicPartition tp = unassignedPartitionsIter.next();
+                    consumerAssignment.add(tp);
+                    unassignedPartitionsIter.remove();
+                    // We already assigned all possible ownedPartitions, so we know this must be newly to this consumer
+                    if (allRevokedPartitions.contains(tp))
+                        partitionsTransferringOwnership.put(tp, consumer);
+                } else {
+                    break;
+                }
+
+                if (consumerAssignment.size() == minQuota) {
+                    minCapacityMembers.add(consumer);
+                    unfilledConsumerIter.remove();
+                }
+            }
+        }
+
+        // If we ran out of unassigned partitions before filling all consumers, we need to start stealing partitions
+        // from the over-full consumers at max capacity
+        for (String consumer : unfilledMembers) {
+            List consumerAssignment = assignment.get(consumer);
+            int remainingCapacity = minQuota - consumerAssignment.size();
+            while (remainingCapacity > 0) {
+                String overloadedConsumer = maxCapacityMembers.poll();
+                if (overloadedConsumer == null) {
+                    throw new IllegalStateException("Some consumers are under capacity but all partitions have been assigned");
+                }
+                TopicPartition swappedPartition = assignment.get(overloadedConsumer).remove(0);
+                consumerAssignment.add(swappedPartition);
+                --remainingCapacity;
+                // This partition is by definition transferring ownership, the swapped partition must have come from
+                // the max capacity member's owned partitions since it can only reach max capacity with owned partitions
+                partitionsTransferringOwnership.put(swappedPartition, consumer);
+            }
+            minCapacityMembers.add(consumer);
+        }
+
+        // Otherwise we may have run out of unfilled consumers before assigning all partitions, in which case we
+        // should just distribute one partition each to all consumers at min capacity
+        for (TopicPartition unassignedPartition : unassignedPartitions) {
+            String underCapacityConsumer = minCapacityMembers.poll();
+            if (underCapacityConsumer == null) {
+                throw new IllegalStateException("Some partitions are unassigned but all consumers are at maximum capacity");
+            }
+            // We can skip the bookkeeping of unassignedPartitions and maxCapacityMembers here since we are at the end
+            assignment.get(underCapacityConsumer).add(unassignedPartition);
+
+            if (allRevokedPartitions.contains(unassignedPartition))
+                partitionsTransferringOwnership.put(unassignedPartition, underCapacityConsumer);
+        }
+
+        return assignment;
+    }
+
+    private SortedSet getTopicPartitions(Map partitionsPerTopic) {
+        SortedSet allPartitions =
+            new TreeSet<>(Comparator.comparing(TopicPartition::topic).thenComparing(TopicPartition::partition));
+        for (Entry entry: partitionsPerTopic.entrySet()) {
+            String topic = entry.getKey();
+            for (int i = 0; i < entry.getValue(); ++i) {
+                allPartitions.add(new TopicPartition(topic, i));
+            }
+        }
+        return allPartitions;
+    }
+
+    private Map> generalAssign(Map partitionsPerTopic,
+                                                            Map subscriptions) {
         Map> currentAssignment = new HashMap<>();
         Map prevAssignment = new HashMap<>();
-        partitionMovements = new PartitionMovements();
 
         prepopulateCurrentAssignments(subscriptions, currentAssignment, prevAssignment);
         boolean isFreshAssignment = currentAssignment.isEmpty();
@@ -105,8 +309,7 @@ public Map> assign(Map partitionsP
             for (TopicPartition topicPartition: entry.getValue())
                 currentPartitionConsumer.put(topicPartition, entry.getKey());
 
-        List sortedPartitions = sortPartitions(
-            currentAssignment, prevAssignment.keySet(), isFreshAssignment, partition2AllPotentialConsumers, consumer2AllPotentialPartitions);
+        List sortedPartitions = sortPartitions(partition2AllPotentialConsumers);
 
         // all partitions that need to be assigned (initially set to all partitions but adjusted in the following loop)
         List unassignedPartitions = new ArrayList<>(sortedPartitions);
@@ -287,95 +490,15 @@ private int getBalanceScore(Map> assignment) {
      * Sort valid partitions so they are processed in the potential reassignment phase in the proper order
      * that causes minimal partition movement among consumers (hence honoring maximal stickiness)
      *
-     * @param currentAssignment the calculated assignment so far
-     * @param partitionsWithADifferentPreviousAssignment partitions that had a different consumer before (for every
-     *                                                   such partition there should also be a mapping in
-     *                                                   @currentAssignment to a different consumer)
-     * @param isFreshAssignment whether this is a new assignment, or a reassignment of an existing one
      * @param partition2AllPotentialConsumers a mapping of partitions to their potential consumers
-     * @param consumer2AllPotentialPartitions a mapping of consumers to potential partitions they can consumer from
-     * @return sorted list of valid partitions
+     * @return  an ascending sorted list of topic partitions based on how many consumers can potentially use them
      */
-    private List sortPartitions(Map> currentAssignment,
-                                                Set partitionsWithADifferentPreviousAssignment,
-                                                boolean isFreshAssignment,
-                                                Map> partition2AllPotentialConsumers,
-                                                Map> consumer2AllPotentialPartitions) {
-        List sortedPartitions = new ArrayList<>();
-
-        if (!isFreshAssignment && areSubscriptionsIdentical(partition2AllPotentialConsumers, consumer2AllPotentialPartitions)) {
-            // if this is a reassignment and the subscriptions are identical (all consumers can consumer from all topics)
-            // then we just need to simply list partitions in a round robin fashion (from consumers with
-            // most assigned partitions to those with least)
-            Map> assignments = deepCopy(currentAssignment);
-            for (Entry> entry: assignments.entrySet()) {
-                List toRemove = new ArrayList<>();
-                for (TopicPartition partition: entry.getValue())
-                    if (!partition2AllPotentialConsumers.keySet().contains(partition))
-                        toRemove.add(partition);
-                for (TopicPartition partition: toRemove)
-                    entry.getValue().remove(partition);
-            }
-            TreeSet sortedConsumers = new TreeSet<>(new SubscriptionComparator(assignments));
-            sortedConsumers.addAll(assignments.keySet());
-            // at this point, sortedConsumers contains an ascending-sorted list of consumers based on
-            // how many valid partitions are currently assigned to them
-
-            while (!sortedConsumers.isEmpty()) {
-                // take the consumer with the most partitions
-                String consumer = sortedConsumers.pollLast();
-                // currently assigned partitions to this consumer
-                List remainingPartitions = assignments.get(consumer);
-                // partitions that were assigned to a different consumer last time
-                List prevPartitions = new ArrayList<>(partitionsWithADifferentPreviousAssignment);
-                // from partitions that had a different consumer before, keep only those that are
-                // assigned to this consumer now
-                prevPartitions.retainAll(remainingPartitions);
-                if (!prevPartitions.isEmpty()) {
-                    // if there is a partition of this consumer that was assigned to another consumer before
-                    // mark it as good options for reassignment
-                    TopicPartition partition = prevPartitions.remove(0);
-                    remainingPartitions.remove(partition);
-                    sortedPartitions.add(partition);
-                    sortedConsumers.add(consumer);
-                } else if (!remainingPartitions.isEmpty()) {
-                    // otherwise, mark any other one of the current partitions as a reassignment candidate
-                    sortedPartitions.add(remainingPartitions.remove(0));
-                    sortedConsumers.add(consumer);
-                }
-            }
-
-            for (TopicPartition partition: partition2AllPotentialConsumers.keySet()) {
-                if (!sortedPartitions.contains(partition))
-                    sortedPartitions.add(partition);
-            }
-
-        } else {
-            // an ascending sorted set of topic partitions based on how many consumers can potentially use them
-            TreeSet sortedAllPartitions = new TreeSet<>(new PartitionComparator(partition2AllPotentialConsumers));
-            sortedAllPartitions.addAll(partition2AllPotentialConsumers.keySet());
-
-            while (!sortedAllPartitions.isEmpty())
-                sortedPartitions.add(sortedAllPartitions.pollFirst());
-        }
-
+    private List sortPartitions(Map> partition2AllPotentialConsumers) {
+        List sortedPartitions = new ArrayList<>(partition2AllPotentialConsumers.keySet());
+        Collections.sort(sortedPartitions, new PartitionComparator(partition2AllPotentialConsumers));
         return sortedPartitions;
     }
 
-    /**
-     * @param partition2AllPotentialConsumers a mapping of partitions to their potential consumers
-     * @param consumer2AllPotentialPartitions a mapping of consumers to potential partitions they can consumer from
-     * @return true if potential consumers of partitions are the same, and potential partitions consumers can
-     * consumer from are the same too
-     */
-    private boolean areSubscriptionsIdentical(Map> partition2AllPotentialConsumers,
-        Map> consumer2AllPotentialPartitions) {
-        if (!hasIdenticalListElements(partition2AllPotentialConsumers.values()))
-            return false;
-
-        return hasIdenticalListElements(consumer2AllPotentialPartitions.values());
-    }
-
     /**
      * The assignment should improve the overall balance of the partition assignments to consumers.
      */
@@ -601,24 +724,6 @@ public boolean isSticky() {
         return partitionMovements.isSticky();
     }
 
-    /**
-     * @param col a collection of elements of type list
-     * @return true if all lists in the collection have the same members; false otherwise
-     */
-    private  boolean hasIdenticalListElements(Collection> col) {
-        Iterator> it = col.iterator();
-        if (!it.hasNext())
-            return true;
-        List cur = it.next();
-        while (it.hasNext()) {
-            List next = it.next();
-            if (!(cur.containsAll(next) && next.containsAll(cur)))
-                return false;
-            cur = next;
-        }
-        return true;
-    }
-
     private void deepCopy(Map> source, Map> dest) {
         dest.clear();
         for (Entry> entry: source.entrySet())
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/StickyAssignorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/StickyAssignorTest.java
index 01a8f3ef226a8..fb89944903739 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/StickyAssignorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/StickyAssignorTest.java
@@ -169,10 +169,10 @@ public void testAssignmentWithConflictingPreviousGenerations() {
         TopicPartition tp5 = new TopicPartition(topic, 5);
 
         List c1partitions0 = partitions(tp0, tp1, tp4);
-        List c2partitions0 = partitions(tp0, tp2, tp3);
+        List c2partitions0 = partitions(tp0, tp1, tp2);
         List c3partitions0 = partitions(tp3, tp4, tp5);
         subscriptions.put(consumer1, buildSubscriptionWithGeneration(topics(topic), c1partitions0, 1));
-        subscriptions.put(consumer2, buildSubscriptionWithGeneration(topics(topic), c2partitions0, 1));
+        subscriptions.put(consumer2, buildSubscriptionWithGeneration(topics(topic), c2partitions0, 2));
         subscriptions.put(consumer3, buildSubscriptionWithGeneration(topics(topic), c3partitions0, 2));
 
         Map> assignment = assignor.assign(partitionsPerTopic, subscriptions);
@@ -181,7 +181,6 @@ public void testAssignmentWithConflictingPreviousGenerations() {
         List c3partitions = assignment.get(consumer3);
 
         assertTrue(c1partitions.size() == 2 && c2partitions.size() == 2 && c3partitions.size() == 2);
-        assertTrue(c1partitions0.containsAll(c1partitions));
         assertTrue(c2partitions0.containsAll(c2partitions));
         assertTrue(c3partitions0.containsAll(c3partitions));
         verifyValidityAndBalance(subscriptions, assignment, partitionsPerTopic);
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractStickyAssignorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractStickyAssignorTest.java
index 52f6747215252..c7b45233ca1e9 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractStickyAssignorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractStickyAssignorTest.java
@@ -33,12 +33,13 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import static org.apache.kafka.common.utils.Utils.mkEntry;
+import static org.apache.kafka.common.utils.Utils.mkMap;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 public abstract class AbstractStickyAssignorTest {
-
     protected AbstractStickyAssignor assignor;
     protected String consumerId = "consumer";
     protected Map subscriptions;
@@ -105,12 +106,16 @@ public void testOnlyAssignsPartitionsFromSubscribedTopics() {
         String otherTopic = "other";
 
         Map partitionsPerTopic = new HashMap<>();
-        partitionsPerTopic.put(topic, 3);
-        partitionsPerTopic.put(otherTopic, 3);
-        subscriptions = Collections.singletonMap(consumerId, new Subscription(topics(topic)));
+        partitionsPerTopic.put(topic, 2);
+        subscriptions = mkMap(
+                mkEntry(consumerId, buildSubscription(
+                        topics(topic),
+                        Arrays.asList(tp(topic, 0), tp(topic, 1), tp(otherTopic, 0), tp(otherTopic, 1)))
+                )
+        );
 
         Map> assignment = assignor.assign(partitionsPerTopic, subscriptions);
-        assertEquals(partitions(tp(topic, 0), tp(topic, 1), tp(topic, 2)), assignment.get(consumerId));
+        assertEquals(partitions(tp(topic, 0), tp(topic, 1)), assignment.get(consumerId));
 
         verifyValidityAndBalance(subscriptions, assignment, partitionsPerTopic);
         assertTrue(isFullyBalanced(assignment));
@@ -145,8 +150,6 @@ public void testTwoConsumersOneTopicOnePartition() {
         subscriptions.put(consumer2, new Subscription(topics(topic)));
 
         Map> assignment = assignor.assign(partitionsPerTopic, subscriptions);
-        assertEquals(partitions(tp(topic, 0)), assignment.get(consumer1));
-        assertEquals(Collections.emptyList(), assignment.get(consumer2));
 
         verifyValidityAndBalance(subscriptions, assignment, partitionsPerTopic);
         assertTrue(isFullyBalanced(assignment));
@@ -238,8 +241,8 @@ public void testAddRemoveConsumerOneTopic() {
         assignment = assignor.assign(partitionsPerTopic, subscriptions);
 
         verifyValidityAndBalance(subscriptions, assignment, partitionsPerTopic);
-        assertEquals(partitions(tp(topic, 2), tp(topic, 1)), assignment.get(consumer1));
-        assertEquals(partitions(tp(topic, 0)), assignment.get(consumer2));
+        assertEquals(partitions(tp(topic, 0), tp(topic, 1)), assignment.get(consumer1));
+        assertEquals(partitions(tp(topic, 2)), assignment.get(consumer2));
         assertTrue(isFullyBalanced(assignment));
         assertTrue(assignor.isSticky());
 
@@ -425,8 +428,37 @@ public void testSameSubscriptions() {
         assertTrue(assignor.isSticky());
     }
 
+    @Test(timeout = 30 * 1000)
+    public void testLargeAssignmentAndGroupWithUniformSubscription() {
+        // 1 million partitions!
+        int topicCount = 500;
+        int partitionCount = 2_000;
+        int consumerCount = 2_000;
+
+        List topics = new ArrayList<>();
+        Map partitionsPerTopic = new HashMap<>();
+        for (int i = 0; i < topicCount; i++) {
+            String topicName = getTopicName(i, topicCount);
+            topics.add(topicName);
+            partitionsPerTopic.put(topicName, partitionCount);
+        }
+
+        for (int i = 0; i < consumerCount; i++) {
+            subscriptions.put(getConsumerName(i, consumerCount), new Subscription(topics));
+        }
+
+        Map> assignment = assignor.assign(partitionsPerTopic, subscriptions);
+
+        for (int i = 1; i < consumerCount; i++) {
+            String consumer = getConsumerName(i, consumerCount);
+            subscriptions.put(consumer, buildSubscription(topics, assignment.get(consumer)));
+        }
+
+        assignor.assign(partitionsPerTopic, subscriptions);
+    }
+
     @Test
-    public void testLargeAssignmentWithMultipleConsumersLeaving() {
+    public void testLargeAssignmentWithMultipleConsumersLeavingAndRandomSubscription() {
         Random rand = new Random();
         int topicCount = 40;
         int consumerCount = 200;
@@ -555,7 +587,6 @@ public void testStickiness() {
         }
     }
 
-
     @Test
     public void testAssignmentUpdatedForDeletedTopic() {
         Map partitionsPerTopic = new HashMap<>();
@@ -582,35 +613,6 @@ public void testNoExceptionThrownWhenOnlySubscribedTopicDeleted() {
         assertTrue(assignment.get(consumerId).isEmpty());
     }
 
-    @Test
-    public void testConflictingPreviousAssignments() {
-        String consumer1 = "consumer1";
-        String consumer2 = "consumer2";
-
-        Map partitionsPerTopic = new HashMap<>();
-        partitionsPerTopic.put(topic, 2);
-        subscriptions.put(consumer1, new Subscription(topics(topic)));
-        subscriptions.put(consumer2, new Subscription(topics(topic)));
-
-        TopicPartition tp0 = new TopicPartition(topic, 0);
-        TopicPartition tp1 = new TopicPartition(topic, 1);
-
-        // both c1 and c2 have partition 1 assigned to them in generation 1
-        List c1partitions0 = partitions(tp0, tp1);
-        List c2partitions0 = partitions(tp0, tp1);
-        subscriptions.put(consumer1, buildSubscription(topics(topic), c1partitions0));
-        subscriptions.put(consumer2, buildSubscription(topics(topic), c2partitions0));
-
-        Map> assignment = assignor.assign(partitionsPerTopic, subscriptions);
-        List c1partitions = assignment.get(consumer1);
-        List c2partitions = assignment.get(consumer2);
-
-        assertTrue(c1partitions.size() == 1 && c2partitions.size() == 1);
-        verifyValidityAndBalance(subscriptions, assignment, partitionsPerTopic);
-        assertTrue(isFullyBalanced(assignment));
-        assertTrue(assignor.isSticky());
-    }
-
     @Test
     public void testReassignmentWithRandomSubscriptionsAndChanges() {
         final int minNumConsumers = 20;

From 86c4dae1327e80b0c7917127a98f1e8ddbf1bbcb Mon Sep 17 00:00:00 2001
From: Nikolay 
Date: Tue, 2 Jun 2020 18:59:56 +0300
Subject: [PATCH 0962/1071] MINOR: Remove unused variable to fix spotBugs
 failure (#8779)

Fixed spotBugs error introduced by c6633a1:

>Dead store to isFreshAssignment in org.apache.kafka.clients.consumer.internals.AbstractStickyAssignor.generalAssign(Map, Map)

Reviewers: Ismael Juma 
---
 .../kafka/clients/consumer/internals/AbstractStickyAssignor.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractStickyAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractStickyAssignor.java
index d4e023cc098a8..353a225c8fcd0 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractStickyAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractStickyAssignor.java
@@ -274,7 +274,6 @@ private Map> generalAssign(Map par
         Map prevAssignment = new HashMap<>();
 
         prepopulateCurrentAssignments(subscriptions, currentAssignment, prevAssignment);
-        boolean isFreshAssignment = currentAssignment.isEmpty();
 
         // a mapping of all topic partitions to all consumers that can be assigned to them
         final Map> partition2AllPotentialConsumers = new HashMap<>();

From 8ecfe6dffa67e027b1fd3bdac5c7a3f80c12cff6 Mon Sep 17 00:00:00 2001
From: showuon <43372967+showuon@users.noreply.github.com>
Date: Wed, 3 Jun 2020 00:50:20 +0800
Subject: [PATCH 0963/1071] KAFKA-10082: Fix the failed
 testMultiConsumerStickyAssignment (#8777)

Fix the failed testMultiConsumerStickyAssignment by modifying the logic error in allSubscriptionsEqual method.

We will create the consumerToOwnedPartitions to keep the set of previously owned partitions encoded in the Subscription. It's our basis to do the reassignment. In the allSubscriptionsEqual, we'll get the member generation of the subscription, and remove all previously owned partitions as invalid if the current generation is higher. However, the logic before my fix, will remove the current highest member out of the consumerToOwnedPartitions, which should be kept because it's the current higher generation member. Fix this logic error.

Reviewers: A. Sophie Blee-Goldman , Guozhang Wang 
---
 .../consumer/internals/AbstractStickyAssignor.java | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractStickyAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractStickyAssignor.java
index 353a225c8fcd0..9743688dfa6d9 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractStickyAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractStickyAssignor.java
@@ -121,6 +121,13 @@ private boolean allSubscriptionsEqual(Set allTopics,
             if (memberData.generation.isPresent() && memberData.generation.get() >= maxGeneration
                 || !memberData.generation.isPresent() && maxGeneration == DEFAULT_GENERATION) {
 
+                // If the current member's generation is higher, all the previously owned partitions are invalid
+                if (memberData.generation.isPresent() && memberData.generation.get() > maxGeneration) {
+                    membersWithOldGeneration.addAll(membersOfCurrentHighestGeneration);
+                    membersOfCurrentHighestGeneration.clear();
+                    maxGeneration = memberData.generation.get();
+                }
+
                 membersOfCurrentHighestGeneration.add(consumer);
                 for (final TopicPartition tp : memberData.partitions) {
                     // filter out any topics that no longer exist or aren't part of the current subscription
@@ -128,13 +135,6 @@ private boolean allSubscriptionsEqual(Set allTopics,
                         ownedPartitions.add(tp);
                     }
                 }
-
-                // If the current member's generation is higher, all the previous owned partitions are invalid
-                if (memberData.generation.isPresent() && memberData.generation.get() > maxGeneration) {
-                    membersWithOldGeneration.addAll(membersOfCurrentHighestGeneration);
-                    membersOfCurrentHighestGeneration.clear();
-                    maxGeneration = memberData.generation.get();
-                }
             }
         }
 

From c71ea4ccf4fc99ca12f48b3df01e16ec5d8ed7d6 Mon Sep 17 00:00:00 2001
From: "A. Sophie Blee-Goldman" 
Date: Tue, 2 Jun 2020 20:39:02 -0700
Subject: [PATCH 0964/1071] KAFKA-10083: fix failed
 testReassignmentWithRandomSubscriptionsAndChanges tests (#8786)

Minimum fix needed to stop this test failing and unblock others

Co-authored-by: Luke Chen 

Reviewers: Guozhang Wang 
---
 .../clients/consumer/internals/AbstractStickyAssignor.java     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractStickyAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractStickyAssignor.java
index 9743688dfa6d9..a1af6d959a3f7 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractStickyAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractStickyAssignor.java
@@ -43,7 +43,7 @@ public abstract class AbstractStickyAssignor extends AbstractPartitionAssignor {
 
     public static final int DEFAULT_GENERATION = -1;
 
-    private PartitionMovements partitionMovements = new PartitionMovements();
+    private PartitionMovements partitionMovements;
 
     // Keep track of the partitions being migrated from one consumer to another during assignment
     // so the cooperative assignor can adjust the assignment
@@ -72,6 +72,7 @@ public MemberData(List partitions, Optional generation)
     @Override
     public Map> assign(Map partitionsPerTopic,
                                                     Map subscriptions) {
+        partitionMovements = new PartitionMovements();
         Map> consumerToOwnedPartitions = new HashMap<>();
         if (allSubscriptionsEqual(partitionsPerTopic.keySet(), subscriptions, consumerToOwnedPartitions)) {
             log.debug("Detected that all consumers were subscribed to same set of topics, invoking the "

From 6be91ee018e02c9e77fd96ec8c95eeb9508db1da Mon Sep 17 00:00:00 2001
From: Randall Hauch 
Date: Fri, 5 Jun 2020 15:02:11 -0500
Subject: [PATCH 0965/1071] MINOR: Change the order that Connect calls
 `config()` and `validate()` to avoid validating if the required ConfigDef is
 null (#8810)

Author: Randall Hauch 
Reviewer: Konstantine Karantasis 
---
 .../kafka/connect/runtime/AbstractHerder.java | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
index b5032a07ce3c4..c1881503ff6fd 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
@@ -315,22 +315,22 @@ public ConfigInfos validateConnectorConfig(Map connectorProps) {
             Set allGroups = new LinkedHashSet<>(enrichedConfigDef.groups());
 
             // do custom connector-specific validation
-            Config config = connector.validate(connectorProps);
-            if (null == config) {
+            ConfigDef configDef = connector.config();
+            if (null == configDef) {
                 throw new BadRequestException(
-                    String.format(
-                        "%s.validate() must return a Config that is not null.",
-                        connector.getClass().getName()
-                    )
+                        String.format(
+                                "%s.config() must return a ConfigDef that is not null.",
+                                connector.getClass().getName()
+                        )
                 );
             }
-            ConfigDef configDef = connector.config();
-            if (null == configDef) {
+            Config config = connector.validate(connectorProps);
+            if (null == config) {
                 throw new BadRequestException(
-                    String.format(
-                        "%s.config() must return a ConfigDef that is not null.",
-                        connector.getClass().getName()
-                    )
+                        String.format(
+                                "%s.validate() must return a Config that is not null.",
+                                connector.getClass().getName()
+                        )
                 );
             }
             configKeys.putAll(configDef.configKeys());

From 63fb0d546960604729dab5f31980c8e1f49d1139 Mon Sep 17 00:00:00 2001
From: Chris Egerton 
Date: Fri, 5 Jun 2020 14:02:17 -0700
Subject: [PATCH 0966/1071] KAFKA-9570: Define SSL configs in all worker config
 classes, not just distributed (#8135)

Define SSL configs in all worker config classes, not just distributed

Author: Chris Egerton 
Reviewers: Nigel Liang , Randall Hauch 
---
 .../kafka/connect/runtime/WorkerConfig.java   |  4 +-
 .../distributed/DistributedConfig.java        |  1 -
 .../standalone/StandaloneConfigTest.java      | 88 +++++++++++++++++++
 3 files changed, 91 insertions(+), 2 deletions(-)
 create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneConfigTest.java

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java
index 17d1d5fe0c89d..c0231103e5715 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java
@@ -310,7 +310,9 @@ protected static ConfigDef baseConfigDef() {
                 .define(ADMIN_LISTENERS_CONFIG, Type.LIST, null,
                         new AdminListenersValidator(), Importance.LOW, ADMIN_LISTENERS_DOC)
                 .define(CONNECTOR_CLIENT_POLICY_CLASS_CONFIG, Type.STRING, CONNECTOR_CLIENT_POLICY_CLASS_DEFAULT,
-                        Importance.MEDIUM, CONNECTOR_CLIENT_POLICY_CLASS_DOC);
+                        Importance.MEDIUM, CONNECTOR_CLIENT_POLICY_CLASS_DOC)
+                // security support
+                .withClientSslSupport();
     }
 
     private void logInternalConverterDeprecationWarnings(Map props) {
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java
index 68c7f61a39619..c38992516e2de 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java
@@ -257,7 +257,6 @@ public class DistributedConfig extends WorkerConfig {
                     CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL,
                     ConfigDef.Importance.MEDIUM,
                     CommonClientConfigs.SECURITY_PROTOCOL_DOC)
-            .withClientSslSupport()
             .withClientSaslSupport()
             .define(WORKER_SYNC_TIMEOUT_MS_CONFIG,
                     ConfigDef.Type.INT,
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneConfigTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneConfigTest.java
new file mode 100644
index 0000000000000..e2e886f7925f7
--- /dev/null
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneConfigTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.runtime.standalone;
+
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.config.SslConfigs;
+import org.apache.kafka.common.config.types.Password;
+import org.apache.kafka.connect.runtime.WorkerConfig;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+
+public class StandaloneConfigTest {
+
+    private static final String HTTPS_LISTENER_PREFIX = "listeners.https.";
+
+    private Map sslProps() {
+        return new HashMap() {
+            {
+                put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, new Password("ssl_key_password"));
+                put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, "ssl_keystore");
+                put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, new Password("ssl_keystore_password"));
+                put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, "ssl_truststore");
+                put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, new Password("ssl_truststore_password"));
+            }
+        };
+    }
+
+    private Map baseWorkerProps() {
+        return new HashMap() {
+            {
+                put(WorkerConfig.KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.json.JsonConverter");
+                put(WorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.json.JsonConverter");
+                put(StandaloneConfig.OFFSET_STORAGE_FILE_FILENAME_CONFIG, "/tmp/foo");
+            }
+        };
+    }
+
+    private static Map withStringValues(Map inputs, String prefix) {
+        return ConfigDef.convertToStringMapWithPasswordValues(inputs).entrySet().stream()
+            .collect(Collectors.toMap(
+                entry -> prefix + entry.getKey(),
+                Map.Entry::getValue
+            ));
+    }
+
+    @Test
+    public void testRestServerPrefixedSslConfigs() {
+        Map workerProps = baseWorkerProps();
+        Map expectedSslProps = sslProps();
+        workerProps.putAll(withStringValues(expectedSslProps, HTTPS_LISTENER_PREFIX));
+
+        StandaloneConfig config = new StandaloneConfig(workerProps);
+        assertEquals(expectedSslProps, config.valuesWithPrefixAllOrNothing(HTTPS_LISTENER_PREFIX));
+    }
+
+    @Test
+    public void testRestServerNonPrefixedSslConfigs() {
+        Map props = baseWorkerProps();
+        Map expectedSslProps = sslProps();
+        props.putAll(withStringValues(expectedSslProps, ""));
+
+        StandaloneConfig config = new StandaloneConfig(props);
+        Map actualProps = config.valuesWithPrefixAllOrNothing(HTTPS_LISTENER_PREFIX)
+            .entrySet().stream()
+            .filter(entry -> expectedSslProps.containsKey(entry.getKey()))
+            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+        assertEquals(expectedSslProps, actualProps);
+    }
+}

From 40b8a7f75b811844148cda9407fb6eedc4682841 Mon Sep 17 00:00:00 2001
From: Konstantine Karantasis 
Date: Fri, 5 Jun 2020 15:56:02 -0700
Subject: [PATCH 0967/1071] KAFKA-9851: Revoking Connect tasks due to
 connectivity issues should also clear the running assignment (#8804)

Until recently revocation of connectors and tasks was the result of a rebalance that contained a new assignment. Therefore the view of the running assignment was kept consistent outside the call to `RebalanceListener#onRevoke`. However, after KAFKA-9184 the need appeared for the worker to revoke tasks voluntarily and proactively without having received a new assignment.

This commit will allow the worker to restart tasks that have been stopped as a result of voluntary revocation after a rebalance reassigns these tasks to the work.

The fix is tested by extending an existing integration test.

Reviewers: Randall Hauch 
---
 .../distributed/DistributedHerder.java        | 52 +++++++++++++------
 .../ConnectWorkerIntegrationTest.java         | 16 ++++++
 .../EmbeddedConnectClusterAssertions.java     |  4 +-
 3 files changed, 54 insertions(+), 18 deletions(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
index 079865fa5fbb9..703386e996e99 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
@@ -1102,26 +1102,39 @@ private void startAndStop(Collection> callables) {
 
     private void startWork() {
         // Start assigned connectors and tasks
-        log.info("Starting connectors and tasks using config offset {}", assignment.offset());
-
         List> callables = new ArrayList<>();
-        for (String connectorName : assignmentDifference(assignment.connectors(), runningAssignment.connectors())) {
-            callables.add(getConnectorStartingCallable(connectorName));
-        }
 
-        // These tasks have been stopped by this worker due to task reconfiguration. In order to
-        // restart them, they are removed just before the overall task startup from the set of
-        // currently running tasks. Therefore, they'll be restarted only if they are included in
-        // the assignment that was just received after rebalancing.
-        runningAssignment.tasks().removeAll(tasksToRestart);
-        tasksToRestart.clear();
-        for (ConnectorTaskId taskId : assignmentDifference(assignment.tasks(), runningAssignment.tasks())) {
-            callables.add(getTaskStartingCallable(taskId));
+        // The sets in runningAssignment may change when onRevoked is called voluntarily by this
+        // herder (e.g. when a broker coordinator failure is detected). Otherwise the
+        // runningAssignment is always replaced by the assignment here.
+        synchronized (this) {
+            log.info("Starting connectors and tasks using config offset {}", assignment.offset());
+            log.debug("Received assignment: {}", assignment);
+            log.debug("Currently running assignment: {}", runningAssignment);
+
+            for (String connectorName : assignmentDifference(assignment.connectors(), runningAssignment.connectors())) {
+                callables.add(getConnectorStartingCallable(connectorName));
+            }
+
+            // These tasks have been stopped by this worker due to task reconfiguration. In order to
+            // restart them, they are removed just before the overall task startup from the set of
+            // currently running tasks. Therefore, they'll be restarted only if they are included in
+            // the assignment that was just received after rebalancing.
+            log.debug("Tasks to restart from currently running assignment: {}", tasksToRestart);
+            runningAssignment.tasks().removeAll(tasksToRestart);
+            tasksToRestart.clear();
+            for (ConnectorTaskId taskId : assignmentDifference(assignment.tasks(), runningAssignment.tasks())) {
+                callables.add(getTaskStartingCallable(taskId));
+            }
         }
+
         startAndStop(callables);
-        runningAssignment = member.currentProtocolVersion() == CONNECT_PROTOCOL_V0
-                            ? ExtendedAssignment.empty()
-                            : assignment;
+
+        synchronized (this) {
+            runningAssignment = member.currentProtocolVersion() == CONNECT_PROTOCOL_V0
+                                ? ExtendedAssignment.empty()
+                                : assignment;
+        }
 
         log.info("Finished starting connectors and tasks");
     }
@@ -1631,6 +1644,13 @@ public void onRevoked(String leader, Collection connectors, Collection checkConnectorAndTasksAreStopped(connectorName),
                 CONNECTOR_SETUP_DURATION_MS,
-                "At least the connector or one of its tasks is still");
+                "At least the connector or one of its tasks is still running");
         } catch (AssertionError e) {
             throw new AssertionError(detailMessage, e);
         }

From cdb9f1d5bc782b038414bd3ac39bcd426c9c18d1 Mon Sep 17 00:00:00 2001
From: Evelyn Bayes <30654260+Evelyn-Bayes@users.noreply.github.com>
Date: Mon, 8 Jun 2020 05:42:00 +1000
Subject: [PATCH 0968/1071] KAFKA-9216: Enforce internal config topic settings
 for Connect workers during startup (#8270)

Currently, Kafka Connect creates its config backing topic with a fire and forget approach.
This is fine unless someone has manually created that topic already with the wrong partition count.

In such a case Kafka Connect may run for some time. Especially if it's in standalone mode and once switched to distributed mode it will almost certainly fail.

This commits adds a check when the KafkaConfigBackingStore is starting.
This check will throw a ConfigException if there is more than one partition in the backing store.

This exception is then caught upstream and logged by either:
- DistributedHerder#run
- ConnectStandalone#main

A unit tests was added in KafkaConfigBackingStoreTest to verify the behaviour.

Author: Evelyn Bayes 
Co-authored-by: Randall Hauch 

Reviewer: Konstantine Karantasis 
---
 .../storage/KafkaConfigBackingStore.java      | 10 ++++++
 .../kafka/connect/util/KafkaBasedLog.java     |  5 +++
 .../storage/KafkaConfigBackingStoreTest.java  | 36 +++++++++++++++++++
 3 files changed, 51 insertions(+)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java
index 658d6c3359352..4c6429ce9304f 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java
@@ -263,6 +263,16 @@ public void start() {
         // Before startup, callbacks are *not* invoked. You can grab a snapshot after starting -- just take care that
         // updates can continue to occur in the background
         configLog.start();
+
+        int partitionCount = configLog.partitionCount();
+        if (partitionCount > 1) {
+            String msg = String.format("Topic '%s' supplied via the '%s' property is required "
+                    + "to have a single partition in order to guarantee consistency of "
+                    + "connector configurations, but found %d partitions.",
+                    topic, DistributedConfig.CONFIG_TOPIC_CONFIG, partitionCount);
+            throw new ConfigException(msg);
+        }
+
         started = true;
         log.info("Started KafkaConfigBackingStore");
     }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java b/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java
index e3015812a9112..69d2588fdea7a 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java
@@ -74,6 +74,7 @@ public class KafkaBasedLog {
 
     private Time time;
     private final String topic;
+    private int partitionCount;
     private final Map producerConfigs;
     private final Map consumerConfigs;
     private final Callback> consumedCallback;
@@ -145,6 +146,7 @@ public void start() {
 
         for (PartitionInfo partition : partitionInfos)
             partitions.add(new TopicPartition(partition.topic(), partition.partition()));
+        partitionCount = partitions.size();
         consumer.assign(partitions);
 
         // Always consume from the beginning of all partitions. Necessary to ensure that we don't use committed offsets
@@ -238,6 +240,9 @@ public void send(K key, V value, org.apache.kafka.clients.producer.Callback call
         producer.send(new ProducerRecord<>(topic, key, value), callback);
     }
 
+    public int partitionCount() {
+        return partitionCount;
+    }
 
     private Producer createProducer() {
         // Always require producer acks to all to ensure durable writes
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java
index 5e98fce254411..fc1b814fec1ac 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java
@@ -22,6 +22,7 @@
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.clients.producer.ProducerConfig;
 import org.apache.kafka.common.record.TimestampType;
+import org.apache.kafka.common.config.ConfigException;
 import org.apache.kafka.connect.data.Field;
 import org.apache.kafka.connect.data.Schema;
 import org.apache.kafka.connect.data.SchemaAndValue;
@@ -59,6 +60,7 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 
 @RunWith(PowerMockRunner.class)
@@ -156,6 +158,7 @@ public void setUp() {
     public void testStartStop() throws Exception {
         expectConfigure();
         expectStart(Collections.emptyList(), Collections.emptyMap());
+        expectPartitionCount(1);
         expectStop();
 
         PowerMock.replayAll();
@@ -199,6 +202,7 @@ public void testPutConnectorConfig() throws Exception {
         configUpdateListener.onConnectorConfigRemove(CONNECTOR_IDS.get(1));
         EasyMock.expectLastCall();
 
+        expectPartitionCount(1);
         expectStop();
 
         PowerMock.replayAll();
@@ -267,6 +271,7 @@ public void testPutTaskConfigs() throws Exception {
         serializedConfigs.put(COMMIT_TASKS_CONFIG_KEYS.get(0), CONFIGS_SERIALIZED.get(2));
         expectReadToEnd(serializedConfigs);
 
+        expectPartitionCount(1);
         expectStop();
 
         PowerMock.replayAll();
@@ -351,6 +356,7 @@ public void testPutTaskConfigsStartsOnlyReconfiguredTasks() throws Exception {
         serializedConfigs.put(COMMIT_TASKS_CONFIG_KEYS.get(1), CONFIGS_SERIALIZED.get(4));
         expectReadToEnd(serializedConfigs);
 
+        expectPartitionCount(1);
         expectStop();
 
         PowerMock.replayAll();
@@ -412,6 +418,7 @@ public void testPutTaskConfigsZeroTasks() throws Exception {
         serializedConfigs.put(COMMIT_TASKS_CONFIG_KEYS.get(0), CONFIGS_SERIALIZED.get(0));
         expectReadToEnd(serializedConfigs);
 
+        expectPartitionCount(1);
         expectStop();
 
         PowerMock.replayAll();
@@ -465,6 +472,7 @@ public void testRestoreTargetState() throws Exception {
 
         // Shouldn't see any callbacks since this is during startup
 
+        expectPartitionCount(1);
         expectStop();
 
         PowerMock.replayAll();
@@ -507,6 +515,7 @@ public void testBackgroundUpdateTargetState() throws Exception {
         configUpdateListener.onConnectorTargetStateChange(CONNECTOR_IDS.get(0));
         EasyMock.expectLastCall();
 
+        expectPartitionCount(1);
         expectStop();
 
         PowerMock.replayAll();
@@ -557,6 +566,7 @@ public void testBackgroundConnectorDeletion() throws Exception {
         configUpdateListener.onConnectorConfigRemove(CONNECTOR_IDS.get(0));
         EasyMock.expectLastCall();
 
+        expectPartitionCount(1);
         expectStop();
 
         PowerMock.replayAll();
@@ -602,6 +612,7 @@ public void testRestoreTargetStateUnexpectedDeletion() throws Exception {
         logOffset = 5;
 
         expectStart(existingRecords, deserialized);
+        expectPartitionCount(1);
 
         // Shouldn't see any callbacks since this is during startup
 
@@ -649,6 +660,7 @@ public void testRestore() throws Exception {
         deserialized.put(CONFIGS_SERIALIZED.get(6), TASK_CONFIG_STRUCTS.get(1));
         logOffset = 7;
         expectStart(existingRecords, deserialized);
+        expectPartitionCount(1);
 
         // Shouldn't see any callbacks since this is during startup
 
@@ -703,6 +715,7 @@ public void testRestoreConnectorDeletion() throws Exception {
 
         logOffset = 6;
         expectStart(existingRecords, deserialized);
+        expectPartitionCount(1);
 
         // Shouldn't see any callbacks since this is during startup
 
@@ -750,6 +763,7 @@ public void testRestoreZeroTasks() throws Exception {
         deserialized.put(CONFIGS_SERIALIZED.get(7), TASKS_COMMIT_STRUCT_ZERO_TASK_CONNECTOR);
         logOffset = 8;
         expectStart(existingRecords, deserialized);
+        expectPartitionCount(1);
 
         // Shouldn't see any callbacks since this is during startup
 
@@ -797,6 +811,7 @@ public void testPutTaskConfigsDoesNotResolveAllInconsistencies() throws Exceptio
         deserialized.put(CONFIGS_SERIALIZED.get(5), TASK_CONFIG_STRUCTS.get(1));
         logOffset = 6;
         expectStart(existingRecords, deserialized);
+        expectPartitionCount(1);
 
         // Successful attempt to write new task config
         expectReadToEnd(new LinkedHashMap());
@@ -851,6 +866,22 @@ public void testPutTaskConfigsDoesNotResolveAllInconsistencies() throws Exceptio
         PowerMock.verifyAll();
     }
 
+    @Test
+    public void testExceptionOnStartWhenConfigTopicHasMultiplePartitions() throws Exception {
+        expectConfigure();
+        expectStart(Collections.emptyList(), Collections.emptyMap());
+
+        expectPartitionCount(2);
+
+        PowerMock.replayAll();
+
+        configStorage.setupAndCreateKafkaBasedLog(TOPIC, DEFAULT_DISTRIBUTED_CONFIG);
+        ConfigException e = assertThrows(ConfigException.class, () -> configStorage.start());
+        assertTrue(e.getMessage().contains("required to have a single partition"));
+
+        PowerMock.verifyAll();
+    }
+
     private void expectConfigure() throws Exception {
         PowerMock.expectPrivate(configStorage, "createKafkaBasedLog",
                 EasyMock.capture(capturedTopic), EasyMock.capture(capturedProducerProps),
@@ -859,6 +890,11 @@ private void expectConfigure() throws Exception {
                 .andReturn(storeLog);
     }
 
+    private void expectPartitionCount(int partitionCount) {
+        EasyMock.expect(storeLog.partitionCount())
+                .andReturn(partitionCount);
+    }
+
     // If non-empty, deserializations should be a LinkedHashMap
     private void expectStart(final List> preexistingRecords,
                              final Map deserializations) {

From 69ff9dd75dc3ebbb54f1efae9a4b06795ca44c1b Mon Sep 17 00:00:00 2001
From: Konstantine Karantasis 
Date: Tue, 9 Jun 2020 09:41:11 -0700
Subject: [PATCH 0969/1071] KAFKA-9848: Avoid triggering scheduled rebalance
 delay when task assignment fails but Connect workers remain in the group
 (#8805)

In the first version of the incremental cooperative protocol, in the presence of a failed sync request by the leader, the assignor was designed to treat the unapplied assignments as lost and trigger a rebalance delay.

This commit applies optimizations in these cases to avoid the unnecessary activation of the rebalancing delay. First, if the worker that loses the sync group request or response is the leader, then it detects this failure by checking the what is the expected generation when it performs task assignments. If it's not the expected one, it resets its view of the previous assignment because it wasn't successfully applied and it doesn't represent a correct state. Furthermore, if the worker that has missed the assignment sync is an ordinary worker, then the leader is able to detect that there are lost assignments and instead of triggering a rebalance delay among the same members of the group, it treats the lost tasks as new tasks and reassigns them immediately. If the lost assignment included revocations that were not applied, the leader reapplies these revocations again.

Existing unit tests and integration tests are adapted to test the proposed optimizations.

Reviewers: Randall Hauch 
---
 checkstyle/suppressions.xml                   |   6 +-
 .../internals/AbstractCoordinator.java        |   4 +-
 .../distributed/DistributedConfig.java        |   2 +-
 .../IncrementalCooperativeAssignor.java       |  38 ++-
 .../distributed/WorkerCoordinator.java        |  13 +
 .../ConnectIntegrationTestUtils.java          |  42 ++++
 .../ConnectWorkerIntegrationTest.java         |   5 +
 .../ExampleConnectIntegrationTest.java        |   5 +
 ...alanceSourceConnectorsIntegrationTest.java |  13 +-
 .../IncrementalCooperativeAssignorTest.java   | 227 ++++++++++++++----
 10 files changed, 282 insertions(+), 73 deletions(-)
 create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectIntegrationTestUtils.java

diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index bee700b4ea44a..2e493efa5073a 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -101,12 +101,10 @@
     
     
+              files="(KafkaConfigBackingStore|IncrementalCooperativeAssignor|RequestResponseTest|WorkerSinkTaskTest).java"/>
 
     
-    
+              files="Worker(SinkTask|SourceTask|Coordinator).java"/>
     
 
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index 62f720ec74b7e..8c482dec0cbef 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -408,8 +408,8 @@ boolean joinGroupIfNeeded(final Timer timer) {
 
                 // Generation data maybe concurrently cleared by Heartbeat thread.
                 // Can't use synchronized for {@code onJoinComplete}, because it can be long enough
-                // and  shouldn't block hearbeat thread.
-                // See {@link PlaintextConsumerTest#testMaxPollIntervalMsDelayInAssignment
+                // and shouldn't block heartbeat thread.
+                // See {@link PlaintextConsumerTest#testMaxPollIntervalMsDelayInAssignment}
                 synchronized (AbstractCoordinator.this) {
                     generationSnapshot = this.generation;
                 }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java
index c38992516e2de..00f6663724b9d 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java
@@ -146,7 +146,7 @@ public class DistributedConfig extends WorkerConfig {
     public static final String CONNECT_PROTOCOL_DEFAULT = ConnectProtocolCompatibility.SESSIONED.toString();
 
     /**
-     * connect.protocol
+     * scheduled.rebalance.max.delay.ms
      */
     public static final String SCHEDULED_REBALANCE_MAX_DELAY_MS_CONFIG = "scheduled.rebalance.max.delay.ms";
     public static final String SCHEDULED_REBALANCE_MAX_DELAY_MS_DOC = "The maximum delay that is "
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java
index c6d7cc62e1145..d4e9d878c5515 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java
@@ -66,6 +66,8 @@ public class IncrementalCooperativeAssignor implements ConnectAssignor {
     protected final Set candidateWorkersForReassignment;
     protected long scheduledRebalance;
     protected int delay;
+    protected int previousGenerationId;
+    protected Set previousMembers;
 
     public IncrementalCooperativeAssignor(LogContext logContext, Time time, int maxDelay) {
         this.log = logContext.logger(IncrementalCooperativeAssignor.class);
@@ -77,6 +79,8 @@ public IncrementalCooperativeAssignor(LogContext logContext, Time time, int maxD
         this.scheduledRebalance = 0;
         this.candidateWorkersForReassignment = new LinkedHashSet<>();
         this.delay = 0;
+        this.previousGenerationId = -1;
+        this.previousMembers = Collections.emptySet();
     }
 
     @Override
@@ -159,6 +163,17 @@ protected Map performTaskAssignment(String leaderId, long ma
         // Base set: The previous assignment of connectors-and-tasks is a standalone snapshot that
         // can be used to calculate derived sets
         log.debug("Previous assignments: {}", previousAssignment);
+        int lastCompletedGenerationId = coordinator.lastCompletedGenerationId();
+        if (previousGenerationId != lastCompletedGenerationId) {
+            log.debug("Clearing the view of previous assignments due to generation mismatch between "
+                    + "previous generation ID {} and last completed generation ID {}. This can "
+                    + "happen if the leader fails to sync the assignment within a rebalancing round. "
+                    + "The following view of previous assignments might be outdated and will be "
+                    + "ignored by the leader in the current computation of new assignments. "
+                    + "Possibly outdated previous assignments: {}",
+                    previousGenerationId, lastCompletedGenerationId, previousAssignment);
+            this.previousAssignment = ConnectorsAndTasks.EMPTY;
+        }
 
         ClusterConfigState snapshot = coordinator.configSnapshot();
         Set configuredConnectors = new TreeSet<>(snapshot.connectors());
@@ -236,7 +251,7 @@ protected Map performTaskAssignment(String leaderId, long ma
         taskAssignments =
                 completeWorkerAssignment.stream().collect(Collectors.toMap(WorkerLoad::worker, WorkerLoad::tasks));
 
-        handleLostAssignments(lostAssignments, newSubmissions, completeWorkerAssignment);
+        handleLostAssignments(lostAssignments, newSubmissions, completeWorkerAssignment, memberConfigs);
 
         // Do not revoke resources for re-assignment while a delayed rebalance is active
         // Also we do not revoke in two consecutive rebalances by the same leader
@@ -267,19 +282,15 @@ protected Map performTaskAssignment(String leaderId, long ma
 
         assignConnectors(completeWorkerAssignment, newSubmissions.connectors());
         assignTasks(completeWorkerAssignment, newSubmissions.tasks());
-
         log.debug("Current complete assignments: {}", currentWorkerAssignment);
         log.debug("New complete assignments: {}", completeWorkerAssignment);
 
         Map> currentConnectorAssignments =
                 currentWorkerAssignment.stream().collect(Collectors.toMap(WorkerLoad::worker, WorkerLoad::connectors));
-
         Map> currentTaskAssignments =
                 currentWorkerAssignment.stream().collect(Collectors.toMap(WorkerLoad::worker, WorkerLoad::tasks));
-
         Map> incrementalConnectorAssignments =
                 diff(connectorAssignments, currentConnectorAssignments);
-
         Map> incrementalTaskAssignments =
                 diff(taskAssignments, currentTaskAssignments);
 
@@ -292,9 +303,9 @@ protected Map performTaskAssignment(String leaderId, long ma
                 fillAssignments(memberConfigs.keySet(), Assignment.NO_ERROR, leaderId,
                                 memberConfigs.get(leaderId).url(), maxOffset, incrementalConnectorAssignments,
                                 incrementalTaskAssignments, toRevoke, delay, protocolVersion);
-
         previousAssignment = computePreviousAssignment(toRevoke, connectorAssignments, taskAssignments, lostAssignments);
-
+        previousGenerationId = coordinator.generationId();
+        previousMembers = memberConfigs.keySet();
         log.debug("Actual assignments: {}", assignments);
         return serializeAssignments(assignments);
     }
@@ -351,7 +362,8 @@ private ConnectorsAndTasks computePreviousAssignment(Map completeWorkerAssignment) {
+                                         List completeWorkerAssignment,
+                                         Map memberConfigs) {
         if (lostAssignments.isEmpty()) {
             resetDelay();
             return;
@@ -361,6 +373,16 @@ protected void handleLostAssignments(ConnectorsAndTasks lostAssignments,
         log.debug("Found the following connectors and tasks missing from previous assignments: "
                 + lostAssignments);
 
+        if (scheduledRebalance <= 0 && memberConfigs.keySet().containsAll(previousMembers)) {
+            log.debug("No worker seems to have departed the group during the rebalance. The "
+                    + "missing assignments that the leader is detecting are probably due to some "
+                    + "workers failing to receive the new assignments in the previous rebalance. "
+                    + "Will reassign missing tasks as new tasks");
+            newSubmissions.connectors().addAll(lostAssignments.connectors());
+            newSubmissions.tasks().addAll(lostAssignments.tasks());
+            return;
+        }
+
         if (scheduledRebalance > 0 && now >= scheduledRebalance) {
             // delayed rebalance expired and it's time to assign resources
             log.debug("Delayed rebalance expired. Reassigning lost tasks");
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java
index 71f351df9ea63..17d6690c69d72 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java
@@ -64,6 +64,7 @@ public class WorkerCoordinator extends AbstractCoordinator implements Closeable
 
     private boolean rejoinRequested;
     private volatile ConnectProtocolCompatibility currentConnectProtocol;
+    private volatile int lastCompletedGenerationId;
     private final ConnectAssignor eagerAssignor;
     private final ConnectAssignor incrementalAssignor;
     private final int coordinatorDiscoveryTimeoutMs;
@@ -100,6 +101,7 @@ public WorkerCoordinator(GroupRebalanceConfig config,
         this.eagerAssignor = new EagerAssignor(logContext);
         this.currentConnectProtocol = protocolCompatibility;
         this.coordinatorDiscoveryTimeoutMs = config.heartbeatIntervalMs;
+        this.lastCompletedGenerationId = Generation.NO_GENERATION.generationId;
     }
 
     @Override
@@ -207,6 +209,7 @@ protected void onJoinComplete(int generation, String memberId, String protocol,
             log.debug("Augmented new assignment: {}", newAssignment);
         }
         assignmentSnapshot = newAssignment;
+        lastCompletedGenerationId = generation;
         listener.onAssigned(newAssignment, generation);
     }
 
@@ -256,6 +259,16 @@ public int generationId() {
         return super.generation().generationId;
     }
 
+    /**
+     * Return id that corresponds to the group generation that was active when the last join was successful
+     *
+     * @return the generation ID of the last group that was joined successfully by this member or -1 if no generation
+     * was stable at that point
+     */
+    public int lastCompletedGenerationId() {
+        return lastCompletedGenerationId;
+    }
+
     private boolean isLeader() {
         final ExtendedAssignment localAssignmentSnapshot = assignmentSnapshot;
         return localAssignmentSnapshot != null && memberId().equals(localAssignmentSnapshot.leader());
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectIntegrationTestUtils.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectIntegrationTestUtils.java
new file mode 100644
index 0000000000000..1aeaa0784cd25
--- /dev/null
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectIntegrationTestUtils.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.integration;
+
+import org.junit.rules.TestRule;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+import org.slf4j.Logger;
+
+/**
+ */
+public class ConnectIntegrationTestUtils {
+    public static TestRule newTestWatcher(Logger log) {
+        return new TestWatcher() {
+            @Override
+            protected void starting(Description description) {
+                super.starting(description);
+                log.info("Starting test {}", description.getMethodName());
+            }
+
+            @Override
+            protected void finished(Description description) {
+                super.finished(description);
+                log.info("Finished test {}", description.getMethodName());
+            }
+        };
+    }
+}
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java
index a0f672e148be4..99c8c4c9a7ed5 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java
@@ -23,8 +23,10 @@
 import org.apache.kafka.test.IntegrationTest;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.junit.rules.TestRule;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -65,6 +67,9 @@ public class ConnectWorkerIntegrationTest {
     Map workerProps = new HashMap<>();
     Properties brokerProps = new Properties();
 
+    @Rule
+    public TestRule watcher = ConnectIntegrationTestUtils.newTestWatcher(log);
+
     @Before
     public void setup() {
         // setup Connect worker properties
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExampleConnectIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExampleConnectIntegrationTest.java
index f4cba8793bd0b..5d36486f018f1 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExampleConnectIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExampleConnectIntegrationTest.java
@@ -22,8 +22,10 @@
 import org.apache.kafka.test.IntegrationTest;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.junit.rules.TestRule;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -64,6 +66,9 @@ public class ExampleConnectIntegrationTest {
     private EmbeddedConnectCluster connect;
     private ConnectorHandle connectorHandle;
 
+    @Rule
+    public TestRule watcher = ConnectIntegrationTestUtils.newTestWatcher(log);
+
     @Before
     public void setup() {
         // setup Connect worker properties
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java
index 19e786317089b..be8ce6136a351 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java
@@ -22,9 +22,10 @@
 import org.apache.kafka.test.IntegrationTest;
 import org.junit.After;
 import org.junit.Before;
-import org.junit.Ignore;
+import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.junit.rules.TestRule;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -45,6 +46,7 @@
 import static org.apache.kafka.connect.runtime.WorkerConfig.OFFSET_COMMIT_INTERVAL_MS_CONFIG;
 import static org.apache.kafka.connect.runtime.distributed.ConnectProtocolCompatibility.COMPATIBLE;
 import static org.apache.kafka.connect.runtime.distributed.DistributedConfig.CONNECT_PROTOCOL_CONFIG;
+import static org.apache.kafka.connect.runtime.distributed.DistributedConfig.SCHEDULED_REBALANCE_MAX_DELAY_MS_CONFIG;
 import static org.apache.kafka.test.TestUtils.waitForCondition;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
@@ -68,12 +70,16 @@ public class RebalanceSourceConnectorsIntegrationTest {
 
     private EmbeddedConnectCluster connect;
 
+    @Rule
+    public TestRule watcher = ConnectIntegrationTestUtils.newTestWatcher(log);
+
     @Before
     public void setup() {
         // setup Connect worker properties
         Map workerProps = new HashMap<>();
         workerProps.put(CONNECT_PROTOCOL_CONFIG, COMPATIBLE.toString());
-        workerProps.put(OFFSET_COMMIT_INTERVAL_MS_CONFIG, "30000");
+        workerProps.put(OFFSET_COMMIT_INTERVAL_MS_CONFIG, String.valueOf(TimeUnit.SECONDS.toMillis(30)));
+        workerProps.put(SCHEDULED_REBALANCE_MAX_DELAY_MS_CONFIG, String.valueOf(TimeUnit.SECONDS.toMillis(30)));
 
         // setup Kafka broker properties
         Properties brokerProps = new Properties();
@@ -98,7 +104,6 @@ public void close() {
         connect.stop();
     }
 
-    @Ignore("Flaky and disruptive. See KAFKA-8391, KAFKA-8661 for details.")
     @Test
     public void testStartTwoConnectors() throws Exception {
         // create test topic
@@ -133,7 +138,6 @@ public void testStartTwoConnectors() throws Exception {
                 "Connector tasks did not start in time.");
     }
 
-    @Ignore("Flaky and disruptive. See KAFKA-8391, KAFKA-8661 for details.")
     @Test
     public void testReconfigConnector() throws Exception {
         ConnectorHandle connectorHandle = RuntimeHandles.get().connectorHandle(CONNECTOR_NAME);
@@ -192,7 +196,6 @@ public void testReconfigConnector() throws Exception {
                 recordNum >= numRecordsProduced);
     }
 
-    @Ignore("Flaky and disruptive. See KAFKA-8391, KAFKA-8661 for details.")
     @Test
     public void testDeleteConnector() throws Exception {
         // create test topic
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java
index a5e4ef0ca1bea..3e72ce9913df5 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java
@@ -118,6 +118,7 @@ public void initAssignor() {
                 new LogContext(),
                 time,
                 rebalanceDelay));
+        assignor.previousGenerationId = 1000;
     }
 
     @Test
@@ -126,6 +127,7 @@ public void testTaskAssignmentWhenWorkerJoins() {
         doReturn(Collections.EMPTY_MAP).when(assignor).serializeAssignments(assignmentsCapture.capture());
 
         // First assignment with 1 worker and 2 connectors configured but not yet assigned
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -138,6 +140,7 @@ public void testTaskAssignmentWhenWorkerJoins() {
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
         memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, null));
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -149,6 +152,7 @@ public void testTaskAssignmentWhenWorkerJoins() {
         // Third assignment after revocations
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -160,6 +164,7 @@ public void testTaskAssignmentWhenWorkerJoins() {
         // A fourth rebalance should not change assignments
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -170,8 +175,9 @@ public void testTaskAssignmentWhenWorkerJoins() {
 
         verify(coordinator, times(rebalanceNum)).configSnapshot();
         verify(coordinator, times(rebalanceNum)).leaderState(any());
-        verify(coordinator, times(rebalanceNum)).generationId();
+        verify(coordinator, times(2 * rebalanceNum)).generationId();
         verify(coordinator, times(rebalanceNum)).memberId();
+        verify(coordinator, times(rebalanceNum)).lastCompletedGenerationId();
     }
 
     @Test
@@ -185,6 +191,7 @@ public void testTaskAssignmentWhenWorkerLeavesPermanently() {
 
         // First assignment with 2 workers and 2 connectors configured but not yet assigned
         memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, null));
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -199,6 +206,7 @@ public void testTaskAssignmentWhenWorkerLeavesPermanently() {
         applyAssignments(returnedAssignments);
         assignments.remove("worker2");
         memberConfigs = memberConfigs(leader, offset, assignments);
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -213,6 +221,7 @@ public void testTaskAssignmentWhenWorkerLeavesPermanently() {
         // been reached yet
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -226,6 +235,7 @@ public void testTaskAssignmentWhenWorkerLeavesPermanently() {
         // Fourth assignment after delay expired
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -236,8 +246,9 @@ public void testTaskAssignmentWhenWorkerLeavesPermanently() {
 
         verify(coordinator, times(rebalanceNum)).configSnapshot();
         verify(coordinator, times(rebalanceNum)).leaderState(any());
-        verify(coordinator, times(rebalanceNum)).generationId();
+        verify(coordinator, times(2 * rebalanceNum)).generationId();
         verify(coordinator, times(rebalanceNum)).memberId();
+        verify(coordinator, times(rebalanceNum)).lastCompletedGenerationId();
     }
 
     @Test
@@ -251,6 +262,7 @@ public void testTaskAssignmentWhenWorkerBounces() {
 
         // First assignment with 2 workers and 2 connectors configured but not yet assigned
         memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, null));
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -265,6 +277,7 @@ public void testTaskAssignmentWhenWorkerBounces() {
         applyAssignments(returnedAssignments);
         assignments.remove("worker2");
         memberConfigs = memberConfigs(leader, offset, assignments);
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -279,6 +292,7 @@ public void testTaskAssignmentWhenWorkerBounces() {
         // been reached yet
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -294,6 +308,7 @@ public void testTaskAssignmentWhenWorkerBounces() {
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
         memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, null));
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -308,6 +323,7 @@ public void testTaskAssignmentWhenWorkerBounces() {
         // assignments ought to be assigned to the worker that has appeared as returned.
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -318,8 +334,9 @@ public void testTaskAssignmentWhenWorkerBounces() {
 
         verify(coordinator, times(rebalanceNum)).configSnapshot();
         verify(coordinator, times(rebalanceNum)).leaderState(any());
-        verify(coordinator, times(rebalanceNum)).generationId();
+        verify(coordinator, times(2 * rebalanceNum)).generationId();
         verify(coordinator, times(rebalanceNum)).memberId();
+        verify(coordinator, times(rebalanceNum)).lastCompletedGenerationId();
     }
 
     @Test
@@ -334,6 +351,7 @@ public void testTaskAssignmentWhenLeaderLeavesPermanently() {
         // First assignment with 3 workers and 2 connectors configured but not yet assigned
         memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, null));
         memberConfigs.put("worker3", new ExtendedWorkerState(leaderUrl, offset, null));
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -355,6 +373,7 @@ public void testTaskAssignmentWhenLeaderLeavesPermanently() {
 
         // Capture needs to be reset to point to the new assignor
         doReturn(Collections.EMPTY_MAP).when(assignor).serializeAssignments(assignmentsCapture.capture());
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -366,6 +385,7 @@ public void testTaskAssignmentWhenLeaderLeavesPermanently() {
         // Third (incidental) assignment with still only one worker in the group.
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -375,8 +395,9 @@ public void testTaskAssignmentWhenLeaderLeavesPermanently() {
 
         verify(coordinator, times(rebalanceNum)).configSnapshot();
         verify(coordinator, times(rebalanceNum)).leaderState(any());
-        verify(coordinator, times(rebalanceNum)).generationId();
+        verify(coordinator, times(2 * rebalanceNum)).generationId();
         verify(coordinator, times(rebalanceNum)).memberId();
+        verify(coordinator, times(rebalanceNum)).lastCompletedGenerationId();
     }
 
     @Test
@@ -391,6 +412,7 @@ public void testTaskAssignmentWhenLeaderBounces() {
         // First assignment with 3 workers and 2 connectors configured but not yet assigned
         memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, null));
         memberConfigs.put("worker3", new ExtendedWorkerState(leaderUrl, offset, null));
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -412,6 +434,7 @@ public void testTaskAssignmentWhenLeaderBounces() {
 
         // Capture needs to be reset to point to the new assignor
         doReturn(Collections.EMPTY_MAP).when(assignor).serializeAssignments(assignmentsCapture.capture());
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -426,6 +449,7 @@ public void testTaskAssignmentWhenLeaderBounces() {
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
         memberConfigs.put("worker1", new ExtendedWorkerState(leaderUrl, offset, null));
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -436,6 +460,7 @@ public void testTaskAssignmentWhenLeaderBounces() {
         // Fourth assignment after revocations
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -446,8 +471,9 @@ public void testTaskAssignmentWhenLeaderBounces() {
 
         verify(coordinator, times(rebalanceNum)).configSnapshot();
         verify(coordinator, times(rebalanceNum)).leaderState(any());
-        verify(coordinator, times(rebalanceNum)).generationId();
+        verify(coordinator, times(2 * rebalanceNum)).generationId();
         verify(coordinator, times(rebalanceNum)).memberId();
+        verify(coordinator, times(rebalanceNum)).lastCompletedGenerationId();
     }
 
     @Test
@@ -463,6 +489,7 @@ public void testTaskAssignmentWhenFirstAssignmentAttemptFails() {
         // First assignment with 2 workers and 2 connectors configured but not yet assigned
         memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, null));
         try {
+            expectGeneration();
             assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         } catch (RuntimeException e) {
             RequestFuture.failure(e);
@@ -470,68 +497,106 @@ public void testTaskAssignmentWhenFirstAssignmentAttemptFails() {
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
-        // This was the assignment that should have been sent, but didn't make it after all the way
+        // This was the assignment that should have been sent, but didn't make it all the way
         assertDelay(0, returnedAssignments);
         assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(2, 8, 0, 0, "worker1", "worker2");
 
         // Second assignment happens with members returning the same assignments (memberConfigs)
-        // as the first time. The assignor can not tell whether it was the assignment that failed
-        // or the workers that were bounced. Therefore it goes into assignment freeze for
-        // the new assignments for a rebalance delay period
+        // as the first time. The assignor detects that the number of members did not change and
+        // avoids the rebalance delay, treating the lost assignments as new assignments.
         doReturn(Collections.EMPTY_MAP).when(assignor).serializeAssignments(assignmentsCapture.capture());
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
+        assertDelay(0, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
-        assertDelay(rebalanceDelay, returnedAssignments);
         assertNoReassignments(memberConfigs, expectedMemberConfigs);
-        assertAssignment(0, 0, 0, 0, "worker1", "worker2");
+        assertAssignment(2, 8, 0, 0, "worker1", "worker2");
 
-        time.sleep(rebalanceDelay / 2);
+        verify(coordinator, times(rebalanceNum)).configSnapshot();
+        verify(coordinator, times(rebalanceNum)).leaderState(any());
+        verify(coordinator, times(2 * rebalanceNum)).generationId();
+        verify(coordinator, times(rebalanceNum)).memberId();
+        verify(coordinator, times(rebalanceNum)).lastCompletedGenerationId();
+    }
 
-        // Third (incidental) assignment with still only one worker in the group. Max delay has not
-        // been reached yet
-        applyAssignments(returnedAssignments);
-        memberConfigs = memberConfigs(leader, offset, assignments);
+    @Test
+    public void testTaskAssignmentWhenSubsequentAssignmentAttemptFails() {
+        // Customize assignor for this test case
+        time = new MockTime();
+        initAssignor();
+
+        when(coordinator.configSnapshot()).thenReturn(configState);
+        doReturn(Collections.EMPTY_MAP).when(assignor).serializeAssignments(assignmentsCapture.capture());
+
+        // First assignment with 2 workers and 2 connectors configured but not yet assigned
+        memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, null));
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
-        assertDelay(rebalanceDelay / 2, returnedAssignments);
+        assertDelay(0, returnedAssignments);
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
         assertNoReassignments(memberConfigs, expectedMemberConfigs);
-        assertAssignment(0, 0, 0, 0, "worker1", "worker2");
+        assertAssignment(2, 8, 0, 0, "worker1", "worker2");
 
-        time.sleep(rebalanceDelay / 2 + 1);
+        when(coordinator.configSnapshot()).thenReturn(configState);
+        doThrow(new RuntimeException("Unable to send computed assignment with SyncGroupRequest"))
+                .when(assignor).serializeAssignments(assignmentsCapture.capture());
 
-        // Fourth assignment after delay expired. Finally all the new assignments are assigned
+        // Second assignment triggered by a third worker joining. The computed assignment should
+        // revoke tasks from the existing group. But the assignment won't be correctly delivered.
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
-        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
+        memberConfigs.put("worker3", new ExtendedWorkerState(leaderUrl, offset, null));
+        try {
+            expectGeneration();
+            assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
+        } catch (RuntimeException e) {
+            RequestFuture.failure(e);
+        }
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
+        expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        // This was the assignment that should have been sent, but didn't make it all the way
         assertDelay(0, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
+        assertAssignment(0, 0, 0, 2, "worker1", "worker2", "worker3");
+
+        // Third assignment happens with members returning the same assignments (memberConfigs)
+        // as the first time.
+        doReturn(Collections.EMPTY_MAP).when(assignor).serializeAssignments(assignmentsCapture.capture());
+        expectGeneration();
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
+        ++rebalanceNum;
+        returnedAssignments = assignmentsCapture.getValue();
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertDelay(0, returnedAssignments);
         assertNoReassignments(memberConfigs, expectedMemberConfigs);
-        assertAssignment(2, 8, 0, 0, "worker1", "worker2");
+        assertAssignment(0, 0, 0, 2, "worker1", "worker2", "worker3");
 
         verify(coordinator, times(rebalanceNum)).configSnapshot();
         verify(coordinator, times(rebalanceNum)).leaderState(any());
-        verify(coordinator, times(rebalanceNum)).generationId();
+        verify(coordinator, times(2 * rebalanceNum)).generationId();
         verify(coordinator, times(rebalanceNum)).memberId();
+        verify(coordinator, times(rebalanceNum)).lastCompletedGenerationId();
     }
 
     @Test
-    public void testTaskAssignmentWhenSubsequentAssignmentAttemptFails() {
+    public void testTaskAssignmentWhenSubsequentAssignmentAttemptFailsOutsideTheAssignor() {
         // Customize assignor for this test case
         time = new MockTime();
         initAssignor();
 
+        expectGeneration();
         when(coordinator.configSnapshot()).thenReturn(configState);
         doReturn(Collections.EMPTY_MAP).when(assignor).serializeAssignments(assignmentsCapture.capture());
 
         // First assignment with 2 workers and 2 connectors configured but not yet assigned
         memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, null));
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -540,31 +605,30 @@ public void testTaskAssignmentWhenSubsequentAssignmentAttemptFails() {
         assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(2, 8, 0, 0, "worker1", "worker2");
 
-        when(coordinator.configSnapshot()).thenReturn(configState);
-        doThrow(new RuntimeException("Unable to send computed assignment with SyncGroupRequest"))
-                .when(assignor).serializeAssignments(assignmentsCapture.capture());
-
         // Second assignment triggered by a third worker joining. The computed assignment should
-        // revoke tasks from the existing group. But the assignment won't be correctly delivered.
+        // revoke tasks from the existing group. But the assignment won't be correctly delivered
+        // and sync group with fail on the leader worker.
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
         memberConfigs.put("worker3", new ExtendedWorkerState(leaderUrl, offset, null));
-        try {
-            assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
-        } catch (RuntimeException e) {
-            RequestFuture.failure(e);
-        }
+        when(coordinator.generationId())
+                .thenReturn(assignor.previousGenerationId + 1)
+                .thenReturn(assignor.previousGenerationId + 1);
+        when(coordinator.lastCompletedGenerationId()).thenReturn(assignor.previousGenerationId - 1);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
         expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
-        // This was the assignment that should have been sent, but didn't make it after all the way
+        // This was the assignment that should have been sent, but didn't make it all the way
         assertDelay(0, returnedAssignments);
         assertNoReassignments(memberConfigs, expectedMemberConfigs);
         assertAssignment(0, 0, 0, 2, "worker1", "worker2", "worker3");
 
         // Third assignment happens with members returning the same assignments (memberConfigs)
         // as the first time.
+        when(coordinator.lastCompletedGenerationId()).thenReturn(assignor.previousGenerationId - 1);
         doReturn(Collections.EMPTY_MAP).when(assignor).serializeAssignments(assignmentsCapture.capture());
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -575,8 +639,9 @@ public void testTaskAssignmentWhenSubsequentAssignmentAttemptFails() {
 
         verify(coordinator, times(rebalanceNum)).configSnapshot();
         verify(coordinator, times(rebalanceNum)).leaderState(any());
-        verify(coordinator, times(rebalanceNum)).generationId();
+        verify(coordinator, times(2 * rebalanceNum)).generationId();
         verify(coordinator, times(rebalanceNum)).memberId();
+        verify(coordinator, times(rebalanceNum)).lastCompletedGenerationId();
     }
 
     @Test
@@ -587,6 +652,7 @@ public void testTaskAssignmentWhenConnectorsAreDeleted() {
 
         // First assignment with 1 worker and 2 connectors configured but not yet assigned
         memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, null));
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -600,6 +666,7 @@ public void testTaskAssignmentWhenConnectorsAreDeleted() {
         when(coordinator.configSnapshot()).thenReturn(configState);
         applyAssignments(returnedAssignments);
         memberConfigs = memberConfigs(leader, offset, assignments);
+        expectGeneration();
         assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
         ++rebalanceNum;
         returnedAssignments = assignmentsCapture.getValue();
@@ -610,8 +677,9 @@ public void testTaskAssignmentWhenConnectorsAreDeleted() {
 
         verify(coordinator, times(rebalanceNum)).configSnapshot();
         verify(coordinator, times(rebalanceNum)).leaderState(any());
-        verify(coordinator, times(rebalanceNum)).generationId();
+        verify(coordinator, times(2 * rebalanceNum)).generationId();
         verify(coordinator, times(rebalanceNum)).memberId();
+        verify(coordinator, times(rebalanceNum)).lastCompletedGenerationId();
     }
 
     @Test
@@ -689,14 +757,17 @@ public void testLostAssignmentHandlingWhenWorkerBounces() {
 
         Map configuredAssignment = new HashMap<>();
         configuredAssignment.put("worker0", workerLoad("worker0", 0, 2, 0, 4));
+        configuredAssignment.put("worker1", workerLoad("worker1", 2, 2, 4, 4));
         configuredAssignment.put("worker2", workerLoad("worker2", 4, 2, 8, 4));
+        memberConfigs = memberConfigs(leader, offset, 0, 2);
 
         ConnectorsAndTasks newSubmissions = new ConnectorsAndTasks.Builder().build();
 
         // No lost assignments
         assignor.handleLostAssignments(new ConnectorsAndTasks.Builder().build(),
                 newSubmissions,
-                new ArrayList<>(configuredAssignment.values()));
+                new ArrayList<>(configuredAssignment.values()),
+                memberConfigs);
 
         assertThat("Wrong set of workers for reassignments",
                 Collections.emptySet(),
@@ -704,14 +775,17 @@ public void testLostAssignmentHandlingWhenWorkerBounces() {
         assertEquals(0, assignor.scheduledRebalance);
         assertEquals(0, assignor.delay);
 
+        assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
         String flakyWorker = "worker1";
         WorkerLoad lostLoad = workerLoad(flakyWorker, 2, 2, 4, 4);
+        memberConfigs.remove(flakyWorker);
 
         ConnectorsAndTasks lostAssignments = new ConnectorsAndTasks.Builder()
                 .withCopies(lostLoad.connectors(), lostLoad.tasks()).build();
 
         // Lost assignments detected - No candidate worker has appeared yet (worker with no assignments)
-        assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()));
+        assignor.handleLostAssignments(lostAssignments, newSubmissions,
+                new ArrayList<>(configuredAssignment.values()), memberConfigs);
 
         assertThat("Wrong set of workers for reassignments",
                 Collections.emptySet(),
@@ -719,12 +793,15 @@ public void testLostAssignmentHandlingWhenWorkerBounces() {
         assertEquals(time.milliseconds() + rebalanceDelay, assignor.scheduledRebalance);
         assertEquals(rebalanceDelay, assignor.delay);
 
+        assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
         time.sleep(rebalanceDelay / 2);
         rebalanceDelay /= 2;
 
         // A new worker (probably returning worker) has joined
         configuredAssignment.put(flakyWorker, new WorkerLoad.Builder(flakyWorker).build());
-        assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()));
+        memberConfigs.put(flakyWorker, new ExtendedWorkerState(leaderUrl, offset, null));
+        assignor.handleLostAssignments(lostAssignments, newSubmissions,
+                new ArrayList<>(configuredAssignment.values()), memberConfigs);
 
         assertThat("Wrong set of workers for reassignments",
                 Collections.singleton(flakyWorker),
@@ -732,10 +809,12 @@ public void testLostAssignmentHandlingWhenWorkerBounces() {
         assertEquals(time.milliseconds() + rebalanceDelay, assignor.scheduledRebalance);
         assertEquals(rebalanceDelay, assignor.delay);
 
+        assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
         time.sleep(rebalanceDelay);
 
         // The new worker has still no assignments
-        assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()));
+        assignor.handleLostAssignments(lostAssignments, newSubmissions,
+                new ArrayList<>(configuredAssignment.values()), memberConfigs);
 
         assertTrue("Wrong assignment of lost connectors",
                 configuredAssignment.getOrDefault(flakyWorker, new WorkerLoad.Builder(flakyWorker).build())
@@ -764,14 +843,17 @@ public void testLostAssignmentHandlingWhenWorkerLeavesPermanently() {
 
         Map configuredAssignment = new HashMap<>();
         configuredAssignment.put("worker0", workerLoad("worker0", 0, 2, 0, 4));
+        configuredAssignment.put("worker1", workerLoad("worker1", 2, 2, 4, 4));
         configuredAssignment.put("worker2", workerLoad("worker2", 4, 2, 8, 4));
+        memberConfigs = memberConfigs(leader, offset, 0, 2);
 
         ConnectorsAndTasks newSubmissions = new ConnectorsAndTasks.Builder().build();
 
         // No lost assignments
         assignor.handleLostAssignments(new ConnectorsAndTasks.Builder().build(),
                 newSubmissions,
-                new ArrayList<>(configuredAssignment.values()));
+                new ArrayList<>(configuredAssignment.values()),
+                memberConfigs);
 
         assertThat("Wrong set of workers for reassignments",
                 Collections.emptySet(),
@@ -779,14 +861,17 @@ public void testLostAssignmentHandlingWhenWorkerLeavesPermanently() {
         assertEquals(0, assignor.scheduledRebalance);
         assertEquals(0, assignor.delay);
 
+        assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
         String removedWorker = "worker1";
         WorkerLoad lostLoad = workerLoad(removedWorker, 2, 2, 4, 4);
+        memberConfigs.remove(removedWorker);
 
         ConnectorsAndTasks lostAssignments = new ConnectorsAndTasks.Builder()
                 .withCopies(lostLoad.connectors(), lostLoad.tasks()).build();
 
         // Lost assignments detected - No candidate worker has appeared yet (worker with no assignments)
-        assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()));
+        assignor.handleLostAssignments(lostAssignments, newSubmissions,
+                new ArrayList<>(configuredAssignment.values()), memberConfigs);
 
         assertThat("Wrong set of workers for reassignments",
                 Collections.emptySet(),
@@ -794,11 +879,13 @@ public void testLostAssignmentHandlingWhenWorkerLeavesPermanently() {
         assertEquals(time.milliseconds() + rebalanceDelay, assignor.scheduledRebalance);
         assertEquals(rebalanceDelay, assignor.delay);
 
+        assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
         time.sleep(rebalanceDelay / 2);
         rebalanceDelay /= 2;
 
         // No new worker has joined
-        assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()));
+        assignor.handleLostAssignments(lostAssignments, newSubmissions,
+                new ArrayList<>(configuredAssignment.values()), memberConfigs);
 
         assertThat("Wrong set of workers for reassignments",
                 Collections.emptySet(),
@@ -808,7 +895,8 @@ public void testLostAssignmentHandlingWhenWorkerLeavesPermanently() {
 
         time.sleep(rebalanceDelay);
 
-        assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()));
+        assignor.handleLostAssignments(lostAssignments, newSubmissions,
+                new ArrayList<>(configuredAssignment.values()), memberConfigs);
 
         assertTrue("Wrong assignment of lost connectors",
                 newSubmissions.connectors().containsAll(lostAssignments.connectors()));
@@ -833,14 +921,17 @@ public void testLostAssignmentHandlingWithMoreThanOneCandidates() {
 
         Map configuredAssignment = new HashMap<>();
         configuredAssignment.put("worker0", workerLoad("worker0", 0, 2, 0, 4));
+        configuredAssignment.put("worker1", workerLoad("worker1", 2, 2, 4, 4));
         configuredAssignment.put("worker2", workerLoad("worker2", 4, 2, 8, 4));
+        memberConfigs = memberConfigs(leader, offset, 0, 2);
 
         ConnectorsAndTasks newSubmissions = new ConnectorsAndTasks.Builder().build();
 
         // No lost assignments
         assignor.handleLostAssignments(new ConnectorsAndTasks.Builder().build(),
                 newSubmissions,
-                new ArrayList<>(configuredAssignment.values()));
+                new ArrayList<>(configuredAssignment.values()),
+                memberConfigs);
 
         assertThat("Wrong set of workers for reassignments",
                 Collections.emptySet(),
@@ -848,8 +939,10 @@ public void testLostAssignmentHandlingWithMoreThanOneCandidates() {
         assertEquals(0, assignor.scheduledRebalance);
         assertEquals(0, assignor.delay);
 
+        assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
         String flakyWorker = "worker1";
         WorkerLoad lostLoad = workerLoad(flakyWorker, 2, 2, 4, 4);
+        memberConfigs.remove(flakyWorker);
         String newWorker = "worker3";
 
         ConnectorsAndTasks lostAssignments = new ConnectorsAndTasks.Builder()
@@ -857,7 +950,9 @@ public void testLostAssignmentHandlingWithMoreThanOneCandidates() {
 
         // Lost assignments detected - A new worker also has joined that is not the returning worker
         configuredAssignment.put(newWorker, new WorkerLoad.Builder(newWorker).build());
-        assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()));
+        memberConfigs.put(newWorker, new ExtendedWorkerState(leaderUrl, offset, null));
+        assignor.handleLostAssignments(lostAssignments, newSubmissions,
+                new ArrayList<>(configuredAssignment.values()), memberConfigs);
 
         assertThat("Wrong set of workers for reassignments",
                 Collections.singleton(newWorker),
@@ -865,12 +960,15 @@ public void testLostAssignmentHandlingWithMoreThanOneCandidates() {
         assertEquals(time.milliseconds() + rebalanceDelay, assignor.scheduledRebalance);
         assertEquals(rebalanceDelay, assignor.delay);
 
+        assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
         time.sleep(rebalanceDelay / 2);
         rebalanceDelay /= 2;
 
         // Now two new workers have joined
         configuredAssignment.put(flakyWorker, new WorkerLoad.Builder(flakyWorker).build());
-        assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()));
+        memberConfigs.put(flakyWorker, new ExtendedWorkerState(leaderUrl, offset, null));
+        assignor.handleLostAssignments(lostAssignments, newSubmissions,
+                new ArrayList<>(configuredAssignment.values()), memberConfigs);
 
         Set expectedWorkers = new HashSet<>();
         expectedWorkers.addAll(Arrays.asList(newWorker, flakyWorker));
@@ -880,12 +978,16 @@ public void testLostAssignmentHandlingWithMoreThanOneCandidates() {
         assertEquals(time.milliseconds() + rebalanceDelay, assignor.scheduledRebalance);
         assertEquals(rebalanceDelay, assignor.delay);
 
+        assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
         time.sleep(rebalanceDelay);
 
         // The new workers have new assignments, other than the lost ones
         configuredAssignment.put(flakyWorker, workerLoad(flakyWorker, 6, 2, 8, 4));
         configuredAssignment.put(newWorker, workerLoad(newWorker, 8, 2, 12, 4));
-        assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()));
+        // we don't reflect these new assignments in memberConfigs currently because they are not
+        // used in handleLostAssignments method
+        assignor.handleLostAssignments(lostAssignments, newSubmissions,
+                new ArrayList<>(configuredAssignment.values()), memberConfigs);
 
         // newWorker joined first, so should be picked up first as a candidate for reassignment
         assertTrue("Wrong assignment of lost connectors",
@@ -915,14 +1017,17 @@ public void testLostAssignmentHandlingWhenWorkerBouncesBackButFinallyLeaves() {
 
         Map configuredAssignment = new HashMap<>();
         configuredAssignment.put("worker0", workerLoad("worker0", 0, 2, 0, 4));
+        configuredAssignment.put("worker1", workerLoad("worker1", 2, 2, 4, 4));
         configuredAssignment.put("worker2", workerLoad("worker2", 4, 2, 8, 4));
+        memberConfigs = memberConfigs(leader, offset, 0, 2);
 
         ConnectorsAndTasks newSubmissions = new ConnectorsAndTasks.Builder().build();
 
         // No lost assignments
         assignor.handleLostAssignments(new ConnectorsAndTasks.Builder().build(),
                 newSubmissions,
-                new ArrayList<>(configuredAssignment.values()));
+                new ArrayList<>(configuredAssignment.values()),
+                memberConfigs);
 
         assertThat("Wrong set of workers for reassignments",
                 Collections.emptySet(),
@@ -930,14 +1035,17 @@ public void testLostAssignmentHandlingWhenWorkerBouncesBackButFinallyLeaves() {
         assertEquals(0, assignor.scheduledRebalance);
         assertEquals(0, assignor.delay);
 
+        assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
         String veryFlakyWorker = "worker1";
         WorkerLoad lostLoad = workerLoad(veryFlakyWorker, 2, 2, 4, 4);
+        memberConfigs.remove(veryFlakyWorker);
 
         ConnectorsAndTasks lostAssignments = new ConnectorsAndTasks.Builder()
                 .withCopies(lostLoad.connectors(), lostLoad.tasks()).build();
 
         // Lost assignments detected - No candidate worker has appeared yet (worker with no assignments)
-        assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()));
+        assignor.handleLostAssignments(lostAssignments, newSubmissions,
+                new ArrayList<>(configuredAssignment.values()), memberConfigs);
 
         assertThat("Wrong set of workers for reassignments",
                 Collections.emptySet(),
@@ -945,12 +1053,15 @@ public void testLostAssignmentHandlingWhenWorkerBouncesBackButFinallyLeaves() {
         assertEquals(time.milliseconds() + rebalanceDelay, assignor.scheduledRebalance);
         assertEquals(rebalanceDelay, assignor.delay);
 
+        assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
         time.sleep(rebalanceDelay / 2);
         rebalanceDelay /= 2;
 
         // A new worker (probably returning worker) has joined
         configuredAssignment.put(veryFlakyWorker, new WorkerLoad.Builder(veryFlakyWorker).build());
-        assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()));
+        memberConfigs.put(veryFlakyWorker, new ExtendedWorkerState(leaderUrl, offset, null));
+        assignor.handleLostAssignments(lostAssignments, newSubmissions,
+                new ArrayList<>(configuredAssignment.values()), memberConfigs);
 
         assertThat("Wrong set of workers for reassignments",
                 Collections.singleton(veryFlakyWorker),
@@ -958,11 +1069,14 @@ public void testLostAssignmentHandlingWhenWorkerBouncesBackButFinallyLeaves() {
         assertEquals(time.milliseconds() + rebalanceDelay, assignor.scheduledRebalance);
         assertEquals(rebalanceDelay, assignor.delay);
 
+        assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
         time.sleep(rebalanceDelay);
 
         // The returning worker leaves permanently after joining briefly during the delay
         configuredAssignment.remove(veryFlakyWorker);
-        assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()));
+        memberConfigs.remove(veryFlakyWorker);
+        assignor.handleLostAssignments(lostAssignments, newSubmissions,
+                new ArrayList<>(configuredAssignment.values()), memberConfigs);
 
         assertTrue("Wrong assignment of lost connectors",
                 newSubmissions.connectors().containsAll(lostAssignments.connectors()));
@@ -1185,4 +1299,11 @@ private void assertNoDuplicateInAssignment(Map exis
                 Collections.emptyList(),
                 is(existingTasks));
     }
+
+    private void expectGeneration() {
+        when(coordinator.generationId())
+                .thenReturn(assignor.previousGenerationId + 1)
+                .thenReturn(assignor.previousGenerationId + 1);
+        when(coordinator.lastCompletedGenerationId()).thenReturn(assignor.previousGenerationId);
+    }
 }

From c4626b8d72b1cea934063aac699cf845fa3e9cbc Mon Sep 17 00:00:00 2001
From: Konstantine Karantasis 
Date: Tue, 9 Jun 2020 13:02:35 -0700
Subject: [PATCH 0970/1071] KAFKA-9849: Fix issue with worker.unsync.backoff.ms
 creating zombie workers when incremental cooperative rebalancing is used
 (#8827)

When Incremental Cooperative Rebalancing is enabled and a worker fails to read to the end of the config topic, it needs to voluntarily revoke its locally running tasks on time, before these tasks get assigned to another worker, creating a situation where redundant tasks are running in the Connect cluster.

Additionally, instead of using the delay `worker.unsync.backoff.ms` that was defined for the eager rebalancing protocol and has a long default value (which coincidentally is equal to the default value of the rebalance delay of the incremental cooperative protocol), the worker should quickly attempt to re-read the config topic and backoff for a fraction of the rebalance delay. After this fix, the worker will retry for a maximum time of 5 times before it revokes its running assignment and for a cumulative delay less than the configured `scheduled.rebalance.max.delay.ms`.

Unit tests are added to cover the backoff logic with incremental cooperative rebalancing.

Reviewers: Randall Hauch 
---
 .../distributed/DistributedHerder.java        |  29 ++-
 .../distributed/ExtendedAssignment.java       |  15 ++
 .../distributed/WorkerCoordinator.java        |   4 +
 .../distributed/WorkerGroupMember.java        |   4 +
 .../ConnectIntegrationTestUtils.java          |   1 +
 .../distributed/DistributedHerderTest.java    | 207 +++++++++++++++++-
 6 files changed, 247 insertions(+), 13 deletions(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
index 703386e996e99..91139c2241e87 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
@@ -27,7 +27,6 @@
 import org.apache.kafka.common.utils.Exit;
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.common.utils.Time;
-import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.connect.connector.Connector;
 import org.apache.kafka.connect.connector.ConnectorContext;
 import org.apache.kafka.connect.connector.policy.ConnectorClientConfigOverridePolicy;
@@ -92,6 +91,7 @@
 import java.util.stream.Collectors;
 
 import static org.apache.kafka.connect.runtime.distributed.ConnectProtocol.CONNECT_PROTOCOL_V0;
+import static org.apache.kafka.connect.runtime.distributed.ConnectProtocolCompatibility.EAGER;
 import static org.apache.kafka.connect.runtime.distributed.IncrementalCooperativeConnectProtocol.CONNECT_PROTOCOL_V2;
 
 /**
@@ -131,6 +131,7 @@ public class DistributedHerder extends AbstractHerder implements Runnable {
     private static final long START_AND_STOP_SHUTDOWN_TIMEOUT_MS = TimeUnit.SECONDS.toMillis(1);
     private static final long RECONFIGURE_CONNECTOR_TASKS_BACKOFF_MS = 250;
     private static final int START_STOP_THREAD_POOL_SIZE = 8;
+    private static final short BACKOFF_RETRIES = 5;
 
     private final AtomicLong requestSeqNum = new AtomicLong();
 
@@ -177,6 +178,7 @@ public class DistributedHerder extends AbstractHerder implements Runnable {
     private SecretKey sessionKey;
     private volatile long keyExpiration;
     private short currentProtocolVersion;
+    private short backoffRetries;
 
     private final DistributedConfig config;
 
@@ -247,6 +249,7 @@ public Thread newThread(Runnable herder) {
         scheduledRebalance = Long.MAX_VALUE;
         keyExpiration = Long.MAX_VALUE;
         sessionKey = null;
+        backoffRetries = BACKOFF_RETRIES;
 
         currentProtocolVersion = ConnectProtocolCompatibility.compatibility(
             config.getString(DistributedConfig.CONNECT_PROTOCOL_CONFIG)
@@ -1077,6 +1080,7 @@ private boolean readConfigToEnd(long timeoutMs) {
             configBackingStore.refresh(timeoutMs, TimeUnit.MILLISECONDS);
             configState = configBackingStore.snapshot();
             log.info("Finished reading to end of log and updated config snapshot, new config log offset: {}", configState.offset());
+            backoffRetries = BACKOFF_RETRIES;
             return true;
         } catch (TimeoutException e) {
             // in case reading the log takes too long, leave the group to ensure a quick rebalance (although by default we should be out of the group already)
@@ -1089,7 +1093,28 @@ private boolean readConfigToEnd(long timeoutMs) {
     }
 
     private void backoff(long ms) {
-        Utils.sleep(ms);
+        if (ConnectProtocolCompatibility.fromProtocolVersion(currentProtocolVersion) == EAGER) {
+            time.sleep(ms);
+            return;
+        }
+
+        if (backoffRetries > 0) {
+            int rebalanceDelayFraction =
+                    config.getInt(DistributedConfig.SCHEDULED_REBALANCE_MAX_DELAY_MS_CONFIG) / 10 / backoffRetries;
+            time.sleep(rebalanceDelayFraction);
+            --backoffRetries;
+            return;
+        }
+
+        ExtendedAssignment runningAssignmentSnapshot;
+        synchronized (this) {
+            runningAssignmentSnapshot = ExtendedAssignment.duplicate(runningAssignment);
+        }
+        log.info("Revoking current running assignment {} because after {} retries the worker "
+                + "has not caught up with the latest Connect cluster updates",
+                runningAssignmentSnapshot, BACKOFF_RETRIES);
+        member.revokeAssignment(runningAssignmentSnapshot);
+        backoffRetries = BACKOFF_RETRIES;
     }
 
     private void startAndStop(Collection> callables) {
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/ExtendedAssignment.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/ExtendedAssignment.java
index 7518e06302a8e..e544407ca0912 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/ExtendedAssignment.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/ExtendedAssignment.java
@@ -23,6 +23,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -85,6 +86,20 @@ public ExtendedAssignment(short version, short error, String leader, String lead
         this.delay = delay;
     }
 
+    public static ExtendedAssignment duplicate(ExtendedAssignment assignment) {
+        return new ExtendedAssignment(
+                assignment.version(),
+                assignment.error(),
+                assignment.leader(),
+                assignment.leaderUrl(),
+                assignment.offset(),
+                new LinkedHashSet<>(assignment.connectors()),
+                new LinkedHashSet<>(assignment.tasks()),
+                new LinkedHashSet<>(assignment.revokedConnectors()),
+                new LinkedHashSet<>(assignment.revokedTasks()),
+                assignment.delay());
+    }
+
     /**
      * Return the version of the connect protocol that this assignment belongs to.
      *
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java
index 17d6690c69d72..800c1d361e3ad 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java
@@ -269,6 +269,10 @@ public int lastCompletedGenerationId() {
         return lastCompletedGenerationId;
     }
 
+    public void revokeAssignment(ExtendedAssignment assignment) {
+        listener.onRevoked(assignment.leader(), assignment.connectors(), assignment.tasks());
+    }
+
     private boolean isLeader() {
         final ExtendedAssignment localAssignmentSnapshot = assignmentSnapshot;
         return localAssignmentSnapshot != null && memberId().equals(localAssignmentSnapshot.leader());
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerGroupMember.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerGroupMember.java
index 13a52d44ff8a2..739575e38904c 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerGroupMember.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerGroupMember.java
@@ -208,6 +208,10 @@ public short currentProtocolVersion() {
         return coordinator.currentProtocolVersion();
     }
 
+    public void revokeAssignment(ExtendedAssignment assignment) {
+        coordinator.revokeAssignment(assignment);
+    }
+
     private void stop(boolean swallowException) {
         log.trace("Stopping the Connect group member.");
         AtomicReference firstException = new AtomicReference<>();
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectIntegrationTestUtils.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectIntegrationTestUtils.java
index 1aeaa0784cd25..058dbe206fe0d 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectIntegrationTestUtils.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectIntegrationTestUtils.java
@@ -22,6 +22,7 @@
 import org.slf4j.Logger;
 
 /**
+ * A utility class for Connect's integration tests
  */
 public class ConnectIntegrationTestUtils {
     public static TestRule newTestWatcher(Logger log) {
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java
index cfc08c4ad774c..81208d398b415 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java
@@ -84,6 +84,8 @@
 import static org.apache.kafka.connect.runtime.distributed.ConnectProtocol.CONNECT_PROTOCOL_V0;
 import static org.apache.kafka.connect.runtime.distributed.IncrementalCooperativeConnectProtocol.CONNECT_PROTOCOL_V1;
 import static org.apache.kafka.connect.runtime.distributed.IncrementalCooperativeConnectProtocol.CONNECT_PROTOCOL_V2;
+import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.newCapture;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -198,7 +200,7 @@ public void setUp() throws Exception {
         connectProtocolVersion = CONNECT_PROTOCOL_V0;
 
         herder = PowerMock.createPartialMock(DistributedHerder.class,
-                new String[]{"backoff", "connectorTypeForClass", "updateDeletedConnectorStatus", "updateDeletedTaskStatus"},
+                new String[]{"connectorTypeForClass", "updateDeletedConnectorStatus", "updateDeletedTaskStatus"},
                 new DistributedConfig(HERDER_CONFIG), worker, WORKER_ID, KAFKA_CLUSTER_ID,
                 statusBackingStore, configBackingStore, member, MEMBER_URL, metrics, time, noneConnectorClientConfigOverridePolicy);
 
@@ -526,7 +528,7 @@ public void testCreateConnector() throws Exception {
         // config validation
         Connector connectorMock = PowerMock.createMock(SourceConnector.class);
         EasyMock.expect(worker.configTransformer()).andReturn(transformer).times(2);
-        final Capture> configCapture = EasyMock.newCapture();
+        final Capture> configCapture = newCapture();
         EasyMock.expect(transformer.transform(EasyMock.capture(configCapture))).andAnswer(configCapture::getValue);
         EasyMock.expect(worker.getPlugins()).andReturn(plugins).times(3);
         EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader);
@@ -573,7 +575,7 @@ public void testCreateConnectorFailedBasicValidation() throws Exception {
         // config validation
         Connector connectorMock = PowerMock.createMock(SourceConnector.class);
         EasyMock.expect(worker.configTransformer()).andReturn(transformer).times(2);
-        final Capture> configCapture = EasyMock.newCapture();
+        final Capture> configCapture = newCapture();
         EasyMock.expect(transformer.transform(EasyMock.capture(configCapture))).andAnswer(configCapture::getValue);
         EasyMock.expect(worker.getPlugins()).andReturn(plugins).times(3);
         EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader);
@@ -587,7 +589,7 @@ public void testCreateConnectorFailedBasicValidation() throws Exception {
 
         // CONN2 creation should fail
 
-        Capture error = EasyMock.newCapture();
+        Capture error = newCapture();
         putConnectorCallback.onCompletion(EasyMock.capture(error), EasyMock.>isNull());
         PowerMock.expectLastCall();
 
@@ -622,7 +624,7 @@ public void testCreateConnectorFailedCustomValidation() throws Exception {
         // config validation
         Connector connectorMock = PowerMock.createMock(SourceConnector.class);
         EasyMock.expect(worker.configTransformer()).andReturn(transformer).times(2);
-        final Capture> configCapture = EasyMock.newCapture();
+        final Capture> configCapture = newCapture();
         EasyMock.expect(transformer.transform(EasyMock.capture(configCapture))).andAnswer(configCapture::getValue);
         EasyMock.expect(worker.getPlugins()).andReturn(plugins).times(3);
         EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader);
@@ -639,7 +641,7 @@ public void testCreateConnectorFailedCustomValidation() throws Exception {
 
         // CONN2 creation should fail
 
-        Capture error = EasyMock.newCapture();
+        Capture error = newCapture();
         putConnectorCallback.onCompletion(EasyMock.capture(error), EasyMock.>isNull());
         PowerMock.expectLastCall();
 
@@ -678,7 +680,7 @@ public void testConnectorNameConflictsWithWorkerGroupId() throws Exception {
         // config validation
         Connector connectorMock = PowerMock.createMock(SinkConnector.class);
         EasyMock.expect(worker.configTransformer()).andReturn(transformer).times(2);
-        final Capture> configCapture = EasyMock.newCapture();
+        final Capture> configCapture = newCapture();
         EasyMock.expect(transformer.transform(EasyMock.capture(configCapture))).andAnswer(configCapture::getValue);
         EasyMock.expect(worker.getPlugins()).andReturn(plugins).times(3);
         EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader);
@@ -690,7 +692,7 @@ public void testConnectorNameConflictsWithWorkerGroupId() throws Exception {
         // CONN2 creation should fail because the worker group id (connect-test-group) conflicts with
         // the consumer group id we would use for this sink
 
-        Capture error = EasyMock.newCapture();
+        Capture error = newCapture();
         putConnectorCallback.onCompletion(EasyMock.capture(error), EasyMock.isNull(Herder.Created.class));
         PowerMock.expectLastCall();
 
@@ -717,7 +719,7 @@ public void testCreateConnectorAlreadyExists() throws Exception {
         EasyMock.expect(member.memberId()).andStubReturn("leader");
         EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0);
         EasyMock.expect(worker.configTransformer()).andReturn(transformer).times(2);
-        final Capture> configCapture = EasyMock.newCapture();
+        final Capture> configCapture = newCapture();
         EasyMock.expect(transformer.transform(EasyMock.capture(configCapture))).andAnswer(configCapture::getValue);
         EasyMock.expect(worker.getPlugins()).andReturn(plugins);
         EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(null);
@@ -1450,7 +1452,6 @@ public void testJoinLeaderCatchUpFails() throws Exception {
         EasyMock.expectLastCall().andThrow(new TimeoutException());
         member.maybeLeaveGroup(EasyMock.eq("taking too long to read the log"));
         EasyMock.expectLastCall();
-        PowerMock.expectPrivate(herder, "backoff", DistributedConfig.WORKER_UNSYNC_BACKOFF_MS_DEFAULT);
         member.requestRejoin();
 
         // After backoff, restart the process and this time succeed
@@ -1472,14 +1473,198 @@ public void testJoinLeaderCatchUpFails() throws Exception {
 
         PowerMock.replayAll();
 
+        long before = time.milliseconds();
+        int workerUnsyncBackoffMs = DistributedConfig.WORKER_UNSYNC_BACKOFF_MS_DEFAULT;
+        int coordinatorDiscoveryTimeoutMs = 100;
         herder.tick();
+        assertEquals(before + coordinatorDiscoveryTimeoutMs + workerUnsyncBackoffMs, time.milliseconds());
+
         time.sleep(1000L);
         assertStatistics("leaderUrl", true, 3, 0, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY);
 
+        before = time.milliseconds();
         herder.tick();
+        assertEquals(before + coordinatorDiscoveryTimeoutMs, time.milliseconds());
         time.sleep(2000L);
         assertStatistics("leaderUrl", false, 3, 1, 100, 2000L);
 
+        PowerMock.verifyAll();
+    }
+
+    @Test
+    public void testJoinLeaderCatchUpRetriesForIncrementalCooperative() throws Exception {
+        connectProtocolVersion = CONNECT_PROTOCOL_V1;
+
+        // Join group and as leader fail to do assignment
+        EasyMock.expect(member.memberId()).andStubReturn("leader");
+        EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V1);
+        expectRebalance(1, Arrays.asList(CONN1), Arrays.asList(TASK1));
+        expectPostRebalanceCatchup(SNAPSHOT);
+
+        member.poll(EasyMock.anyInt());
+        PowerMock.expectLastCall();
+
+        // The leader got its assignment
+        expectRebalance(Collections.emptyList(), Collections.emptyList(),
+                ConnectProtocol.Assignment.NO_ERROR,
+                1, Arrays.asList(CONN1), Arrays.asList(TASK1), 0);
+
+        EasyMock.expect(worker.getPlugins()).andReturn(plugins);
+        // and the new assignment started
+        worker.startConnector(EasyMock.eq(CONN1), EasyMock.anyObject(), EasyMock.anyObject(),
+                EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED));
+        PowerMock.expectLastCall().andReturn(true);
+        EasyMock.expect(worker.isRunning(CONN1)).andReturn(true);
+        EasyMock.expect(worker.connectorTaskConfigs(CONN1, conn1SinkConfig)).andReturn(TASK_CONFIGS);
+
+        worker.startTask(EasyMock.eq(TASK1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(),
+                EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED));
+        PowerMock.expectLastCall().andReturn(true);
+        member.poll(EasyMock.anyInt());
+        PowerMock.expectLastCall();
+
+        // Another rebalance is triggered but this time it fails to read to the max offset and
+        // triggers a re-sync
+        expectRebalance(Collections.emptyList(), Collections.emptyList(),
+                ConnectProtocol.Assignment.CONFIG_MISMATCH, 1, Collections.emptyList(),
+                Collections.emptyList());
+
+        // The leader will retry a few times to read to the end of the config log
+        int retries = 2;
+        member.requestRejoin();
+        for (int i = retries; i >= 0; --i) {
+            // Reading to end of log times out
+            configBackingStore.refresh(EasyMock.anyLong(), EasyMock.anyObject(TimeUnit.class));
+            EasyMock.expectLastCall().andThrow(new TimeoutException());
+            member.maybeLeaveGroup(EasyMock.eq("taking too long to read the log"));
+            EasyMock.expectLastCall();
+        }
+
+        // After a few retries succeed to read the log to the end
+        expectRebalance(Collections.emptyList(), Collections.emptyList(),
+                ConnectProtocol.Assignment.NO_ERROR,
+                1, Arrays.asList(CONN1), Arrays.asList(TASK1), 0);
+        expectPostRebalanceCatchup(SNAPSHOT);
+        member.poll(EasyMock.anyInt());
+        PowerMock.expectLastCall();
+
+        PowerMock.replayAll();
+
+        assertStatistics(0, 0, 0, Double.POSITIVE_INFINITY);
+        herder.tick();
+
+        time.sleep(2000L);
+        assertStatistics(3, 1, 100, 2000);
+        herder.tick();
+
+        long before;
+        int coordinatorDiscoveryTimeoutMs = 100;
+        int maxRetries = 5;
+        for (int i = maxRetries; i >= maxRetries - retries; --i) {
+            before = time.milliseconds();
+            int workerUnsyncBackoffMs =
+                    DistributedConfig.SCHEDULED_REBALANCE_MAX_DELAY_MS_DEFAULT / 10 / i;
+            herder.tick();
+            assertEquals(before + coordinatorDiscoveryTimeoutMs + workerUnsyncBackoffMs, time.milliseconds());
+            coordinatorDiscoveryTimeoutMs = 0;
+        }
+
+        before = time.milliseconds();
+        coordinatorDiscoveryTimeoutMs = 100;
+        herder.tick();
+        assertEquals(before + coordinatorDiscoveryTimeoutMs, time.milliseconds());
+
+        PowerMock.verifyAll();
+    }
+
+    @Test
+    public void testJoinLeaderCatchUpFailsForIncrementalCooperative() throws Exception {
+        connectProtocolVersion = CONNECT_PROTOCOL_V1;
+
+        // Join group and as leader fail to do assignment
+        EasyMock.expect(member.memberId()).andStubReturn("leader");
+        EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V1);
+        expectRebalance(1, Arrays.asList(CONN1), Arrays.asList(TASK1));
+        expectPostRebalanceCatchup(SNAPSHOT);
+
+        member.poll(EasyMock.anyInt());
+        PowerMock.expectLastCall();
+
+        // The leader got its assignment
+        expectRebalance(Collections.emptyList(), Collections.emptyList(),
+                ConnectProtocol.Assignment.NO_ERROR,
+                1, Arrays.asList(CONN1), Arrays.asList(TASK1), 0);
+
+        EasyMock.expect(worker.getPlugins()).andReturn(plugins);
+        // and the new assignment started
+        worker.startConnector(EasyMock.eq(CONN1), EasyMock.anyObject(), EasyMock.anyObject(),
+                EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED));
+        PowerMock.expectLastCall().andReturn(true);
+        EasyMock.expect(worker.isRunning(CONN1)).andReturn(true);
+        EasyMock.expect(worker.connectorTaskConfigs(CONN1, conn1SinkConfig)).andReturn(TASK_CONFIGS);
+
+        worker.startTask(EasyMock.eq(TASK1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(),
+                EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED));
+        PowerMock.expectLastCall().andReturn(true);
+        member.poll(EasyMock.anyInt());
+        PowerMock.expectLastCall();
+
+        // Another rebalance is triggered but this time it fails to read to the max offset and
+        // triggers a re-sync
+        expectRebalance(Collections.emptyList(), Collections.emptyList(),
+                ConnectProtocol.Assignment.CONFIG_MISMATCH, 1, Collections.emptyList(),
+                Collections.emptyList());
+
+        // The leader will exhaust the retries while trying to read to the end of the config log
+        int maxRetries = 5;
+        member.requestRejoin();
+        for (int i = maxRetries; i >= 0; --i) {
+            // Reading to end of log times out
+            configBackingStore.refresh(EasyMock.anyLong(), EasyMock.anyObject(TimeUnit.class));
+            EasyMock.expectLastCall().andThrow(new TimeoutException());
+            member.maybeLeaveGroup(EasyMock.eq("taking too long to read the log"));
+            EasyMock.expectLastCall();
+        }
+
+        Capture assignmentCapture = newCapture();
+        member.revokeAssignment(capture(assignmentCapture));
+        PowerMock.expectLastCall();
+
+        // After a complete backoff and a revocation of running tasks rejoin and this time succeed
+        // The worker gets back the assignment that had given up
+        expectRebalance(Collections.emptyList(), Collections.emptyList(),
+                ConnectProtocol.Assignment.NO_ERROR,
+                1, Arrays.asList(CONN1), Arrays.asList(TASK1), 0);
+        expectPostRebalanceCatchup(SNAPSHOT);
+        member.poll(EasyMock.anyInt());
+        PowerMock.expectLastCall();
+
+        PowerMock.replayAll();
+
+        assertStatistics(0, 0, 0, Double.POSITIVE_INFINITY);
+        herder.tick();
+
+        time.sleep(2000L);
+        assertStatistics(3, 1, 100, 2000);
+        herder.tick();
+
+        long before;
+        int coordinatorDiscoveryTimeoutMs = 100;
+        for (int i = maxRetries; i > 0; --i) {
+            before = time.milliseconds();
+            int workerUnsyncBackoffMs =
+                    DistributedConfig.SCHEDULED_REBALANCE_MAX_DELAY_MS_DEFAULT / 10 / i;
+            herder.tick();
+            assertEquals(before + coordinatorDiscoveryTimeoutMs + workerUnsyncBackoffMs, time.milliseconds());
+            coordinatorDiscoveryTimeoutMs = 0;
+        }
+
+        before = time.milliseconds();
+        herder.tick();
+        assertEquals(before, time.milliseconds());
+        assertEquals(Collections.singleton(CONN1), assignmentCapture.getValue().connectors());
+        assertEquals(Collections.singleton(TASK1), assignmentCapture.getValue().tasks());
+        herder.tick();
 
         PowerMock.verifyAll();
     }
@@ -1566,7 +1751,7 @@ public void testPutConnectorConfig() throws Exception {
         // config validation
         Connector connectorMock = PowerMock.createMock(SourceConnector.class);
         EasyMock.expect(worker.configTransformer()).andReturn(transformer).times(2);
-        final Capture> configCapture = EasyMock.newCapture();
+        final Capture> configCapture = newCapture();
         EasyMock.expect(transformer.transform(EasyMock.capture(configCapture))).andAnswer(configCapture::getValue);
         EasyMock.expect(worker.getPlugins()).andReturn(plugins).anyTimes();
         EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader);

From 817ed01a59938e277ca57be0541417b9ee5b2ba7 Mon Sep 17 00:00:00 2001
From: Greg Harris 
Date: Wed, 10 Jun 2020 15:04:36 -0700
Subject: [PATCH 0971/1071] KAFKA-9969: Exclude ConnectorClientConfigRequest
 from class loading isolation (#8630)

This fix excludes `ConnectorClientConfigRequest` and its inner class from class loading isolation in a similar way that KAFKA-8415 excluded `ConnectorClientConfigOverridePolicy`.

Reviewer: Konstantine Karantasis 
---
 .../runtime/isolation/PluginUtils.java        |   2 +-
 .../runtime/isolation/PluginUtilsTest.java    | 349 +++++++++++++-----
 2 files changed, 253 insertions(+), 98 deletions(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/PluginUtils.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/PluginUtils.java
index 36feac50d1824..acb26fb9ebcc3 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/PluginUtils.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/PluginUtils.java
@@ -137,7 +137,7 @@ public class PluginUtils {
             + "|storage\\.StringConverter"
             + "|storage\\.SimpleHeaderConverter"
             + "|rest\\.basic\\.auth\\.extension\\.BasicAuthSecurityRestExtension"
-            + "|connector\\.policy\\.(?!ConnectorClientConfigOverridePolicy$).*"
+            + "|connector\\.policy\\.(?!ConnectorClientConfig(?:OverridePolicy|Request(?:\\$ClientType)?)$).*"
             + ")"
             + "|common\\.config\\.provider\\.(?!ConfigProvider$).*"
             + ")$");
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/PluginUtilsTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/PluginUtilsTest.java
index c406ead57e66f..7ac08cc13bf34 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/PluginUtilsTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/isolation/PluginUtilsTest.java
@@ -70,7 +70,7 @@ public void testThirdPartyClasses() {
     }
 
     @Test
-    public void testConnectFrameworkClasses() {
+    public void testKafkaDependencyClasses() {
         assertFalse(PluginUtils.shouldLoadInIsolation("org.apache.kafka.common."));
         assertFalse(PluginUtils.shouldLoadInIsolation(
                 "org.apache.kafka.common.config.AbstractConfig")
@@ -81,30 +81,6 @@ public void testConnectFrameworkClasses() {
         assertFalse(PluginUtils.shouldLoadInIsolation(
                 "org.apache.kafka.common.serialization.Deserializer")
         );
-        assertFalse(PluginUtils.shouldLoadInIsolation("org.apache.kafka.connect."));
-        assertFalse(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.connector.Connector")
-        );
-        assertFalse(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.source.SourceConnector")
-        );
-        assertFalse(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.sink.SinkConnector")
-        );
-        assertFalse(PluginUtils.shouldLoadInIsolation("org.apache.kafka.connect.connector.Task"));
-        assertFalse(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.source.SourceTask")
-        );
-        assertFalse(PluginUtils.shouldLoadInIsolation("org.apache.kafka.connect.sink.SinkTask"));
-        assertFalse(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.transforms.Transformation")
-        );
-        assertFalse(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.storage.Converter")
-        );
-        assertFalse(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.storage.OffsetBackingStore")
-        );
         assertFalse(PluginUtils.shouldLoadInIsolation(
                 "org.apache.kafka.clients.producer.ProducerConfig")
         );
@@ -114,68 +90,266 @@ public void testConnectFrameworkClasses() {
         assertFalse(PluginUtils.shouldLoadInIsolation(
                 "org.apache.kafka.clients.admin.KafkaAdminClient")
         );
-        assertFalse(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.rest.ConnectRestExtension")
-        );
     }
 
     @Test
-    public void testAllowedConnectFrameworkClasses() {
-        assertTrue(PluginUtils.shouldLoadInIsolation("org.apache.kafka.connect.transforms."));
-        assertTrue(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.transforms.ExtractField")
-        );
-        assertTrue(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.transforms.ExtractField$Key")
-        );
-        assertTrue(PluginUtils.shouldLoadInIsolation("org.apache.kafka.connect.json."));
-        assertTrue(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.json.JsonConverter")
-        );
-        assertTrue(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.json.JsonConverter$21")
-        );
-        assertTrue(PluginUtils.shouldLoadInIsolation("org.apache.kafka.connect.file."));
-        assertTrue(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.file.FileStreamSourceTask")
-        );
-        assertTrue(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.file.FileStreamSinkConnector")
-        );
+    public void testConnectApiClasses() {
+        List apiClasses = Arrays.asList(
+            // Enumerate all packages and classes
+            "org.apache.kafka.connect.",
+            "org.apache.kafka.connect.components.",
+            "org.apache.kafka.connect.components.Versioned",
+            //"org.apache.kafka.connect.connector.policy.", isolated by default
+            "org.apache.kafka.connect.connector.policy.ConnectorClientConfigOverridePolicy",
+            "org.apache.kafka.connect.connector.policy.ConnectorClientConfigRequest",
+            "org.apache.kafka.connect.connector.policy.ConnectorClientConfigRequest$ClientType",
+            "org.apache.kafka.connect.connector.",
+            "org.apache.kafka.connect.connector.Connector",
+            "org.apache.kafka.connect.connector.ConnectorContext",
+            "org.apache.kafka.connect.connector.ConnectRecord",
+            "org.apache.kafka.connect.connector.Task",
+            "org.apache.kafka.connect.data.",
+            "org.apache.kafka.connect.data.ConnectSchema",
+            "org.apache.kafka.connect.data.Date",
+            "org.apache.kafka.connect.data.Decimal",
+            "org.apache.kafka.connect.data.Field",
+            "org.apache.kafka.connect.data.Schema",
+            "org.apache.kafka.connect.data.SchemaAndValue",
+            "org.apache.kafka.connect.data.SchemaBuilder",
+            "org.apache.kafka.connect.data.SchemaProjector",
+            "org.apache.kafka.connect.data.Struct",
+            "org.apache.kafka.connect.data.Time",
+            "org.apache.kafka.connect.data.Timestamp",
+            "org.apache.kafka.connect.data.Values",
+            "org.apache.kafka.connect.errors.",
+            "org.apache.kafka.connect.errors.AlreadyExistsException",
+            "org.apache.kafka.connect.errors.ConnectException",
+            "org.apache.kafka.connect.errors.DataException",
+            "org.apache.kafka.connect.errors.IllegalWorkerStateException",
+            "org.apache.kafka.connect.errors.NotFoundException",
+            "org.apache.kafka.connect.errors.RetriableException",
+            "org.apache.kafka.connect.errors.SchemaBuilderException",
+            "org.apache.kafka.connect.errors.SchemaProjectorException",
+            "org.apache.kafka.connect.header.",
+            "org.apache.kafka.connect.header.ConnectHeader",
+            "org.apache.kafka.connect.header.ConnectHeaders",
+            "org.apache.kafka.connect.header.Header",
+            "org.apache.kafka.connect.header.Headers",
+            "org.apache.kafka.connect.health.",
+            "org.apache.kafka.connect.health.AbstractState",
+            "org.apache.kafka.connect.health.ConnectClusterDetails",
+            "org.apache.kafka.connect.health.ConnectClusterState",
+            "org.apache.kafka.connect.health.ConnectorHealth",
+            "org.apache.kafka.connect.health.ConnectorState",
+            "org.apache.kafka.connect.health.ConnectorType",
+            "org.apache.kafka.connect.health.TaskState",
+            "org.apache.kafka.connect.rest.",
+            "org.apache.kafka.connect.rest.ConnectRestExtension",
+            "org.apache.kafka.connect.rest.ConnectRestExtensionContext",
+            "org.apache.kafka.connect.sink.",
+            "org.apache.kafka.connect.sink.SinkConnector",
+            "org.apache.kafka.connect.sink.SinkRecord",
+            "org.apache.kafka.connect.sink.SinkTask",
+            "org.apache.kafka.connect.sink.SinkTaskContext",
+            "org.apache.kafka.connect.sink.ErrantRecordReporter",
+            "org.apache.kafka.connect.source.",
+            "org.apache.kafka.connect.source.SourceConnector",
+            "org.apache.kafka.connect.source.SourceRecord",
+            "org.apache.kafka.connect.source.SourceTask",
+            "org.apache.kafka.connect.source.SourceTaskContext",
+            "org.apache.kafka.connect.storage.",
+            "org.apache.kafka.connect.storage.Converter",
+            "org.apache.kafka.connect.storage.ConverterConfig",
+            "org.apache.kafka.connect.storage.ConverterType",
+            "org.apache.kafka.connect.storage.HeaderConverter",
+            "org.apache.kafka.connect.storage.OffsetStorageReader",
+            //"org.apache.kafka.connect.storage.SimpleHeaderConverter", explicitly isolated
+            //"org.apache.kafka.connect.storage.StringConverter", explicitly isolated
+            "org.apache.kafka.connect.storage.StringConverterConfig",
+            //"org.apache.kafka.connect.transforms.", isolated by default
+            "org.apache.kafka.connect.transforms.Transformation",
+            "org.apache.kafka.connect.util.",
+            "org.apache.kafka.connect.util.ConnectorUtils"
+        );
+        // Classes in the API should never be loaded in isolation.
+        for (String clazz : apiClasses) {
+            assertFalse(
+                clazz + " from 'api' is loaded in isolation but should not be",
+                PluginUtils.shouldLoadInIsolation(clazz)
+            );
+        }
+    }
+
+    @Test
+    public void testConnectRuntimeClasses() {
+        // Only list packages, because there are too many classes.
+        List runtimeClasses = Arrays.asList(
+            "org.apache.kafka.connect.cli.",
+            //"org.apache.kafka.connect.connector.policy.", isolated by default
+            //"org.apache.kafka.connect.converters.", isolated by default
+            "org.apache.kafka.connect.runtime.",
+            "org.apache.kafka.connect.runtime.distributed",
+            "org.apache.kafka.connect.runtime.errors",
+            "org.apache.kafka.connect.runtime.health",
+            "org.apache.kafka.connect.runtime.isolation",
+            "org.apache.kafka.connect.runtime.rest.",
+            "org.apache.kafka.connect.runtime.rest.entities.",
+            "org.apache.kafka.connect.runtime.rest.errors.",
+            "org.apache.kafka.connect.runtime.rest.resources.",
+            "org.apache.kafka.connect.runtime.rest.util.",
+            "org.apache.kafka.connect.runtime.standalone.",
+            "org.apache.kafka.connect.runtime.rest.",
+            "org.apache.kafka.connect.storage.",
+            "org.apache.kafka.connect.tools.",
+            "org.apache.kafka.connect.util."
+        );
+        for (String clazz : runtimeClasses) {
+            assertFalse(
+                clazz + " from 'runtime' is loaded in isolation but should not be",
+                PluginUtils.shouldLoadInIsolation(clazz)
+            );
+        }
+    }
+
+    @Test
+    public void testAllowedRuntimeClasses() {
+        List jsonConverterClasses = Arrays.asList(
+            "org.apache.kafka.connect.connector.policy.",
+            "org.apache.kafka.connect.connector.policy.AbstractConnectorClientConfigOverridePolicy",
+            "org.apache.kafka.connect.connector.policy.AllConnectorClientConfigOverridePolicy",
+            "org.apache.kafka.connect.connector.policy.NoneConnectorClientConfigOverridePolicy",
+            "org.apache.kafka.connect.connector.policy.PrincipalConnectorClientConfigOverridePolicy",
+            "org.apache.kafka.connect.converters.",
+            "org.apache.kafka.connect.converters.ByteArrayConverter",
+            "org.apache.kafka.connect.converters.DoubleConverter",
+            "org.apache.kafka.connect.converters.FloatConverter",
+            "org.apache.kafka.connect.converters.IntegerConverter",
+            "org.apache.kafka.connect.converters.LongConverter",
+            "org.apache.kafka.connect.converters.NumberConverter",
+            "org.apache.kafka.connect.converters.NumberConverterConfig",
+            "org.apache.kafka.connect.converters.ShortConverter",
+            //"org.apache.kafka.connect.storage.", not isolated by default
+            "org.apache.kafka.connect.storage.StringConverter",
+            "org.apache.kafka.connect.storage.SimpleHeaderConverter"
+        );
+        for (String clazz : jsonConverterClasses) {
+            assertTrue(
+                clazz + " from 'runtime' is not loaded in isolation but should be",
+                PluginUtils.shouldLoadInIsolation(clazz)
+            );
+        }
+    }
+
+    @Test
+    public void testTransformsClasses() {
+        List transformsClasses = Arrays.asList(
+            "org.apache.kafka.connect.transforms.",
+            "org.apache.kafka.connect.transforms.util.",
+            "org.apache.kafka.connect.transforms.util.NonEmptyListValidator",
+            "org.apache.kafka.connect.transforms.util.RegexValidator",
+            "org.apache.kafka.connect.transforms.util.Requirements",
+            "org.apache.kafka.connect.transforms.util.SchemaUtil",
+            "org.apache.kafka.connect.transforms.util.SimpleConfig",
+            "org.apache.kafka.connect.transforms.Cast",
+            "org.apache.kafka.connect.transforms.Cast$Key",
+            "org.apache.kafka.connect.transforms.Cast$Value",
+            "org.apache.kafka.connect.transforms.ExtractField",
+            "org.apache.kafka.connect.transforms.ExtractField$Key",
+            "org.apache.kafka.connect.transforms.ExtractField$Value",
+            "org.apache.kafka.connect.transforms.Flatten",
+            "org.apache.kafka.connect.transforms.Flatten$Key",
+            "org.apache.kafka.connect.transforms.Flatten$Value",
+            "org.apache.kafka.connect.transforms.HoistField",
+            "org.apache.kafka.connect.transforms.HoistField$Key",
+            "org.apache.kafka.connect.transforms.HoistField$Key",
+            "org.apache.kafka.connect.transforms.InsertField",
+            "org.apache.kafka.connect.transforms.InsertField$Key",
+            "org.apache.kafka.connect.transforms.InsertField$Value",
+            "org.apache.kafka.connect.transforms.MaskField",
+            "org.apache.kafka.connect.transforms.MaskField$Key",
+            "org.apache.kafka.connect.transforms.MaskField$Value",
+            "org.apache.kafka.connect.transforms.RegexRouter",
+            "org.apache.kafka.connect.transforms.ReplaceField",
+            "org.apache.kafka.connect.transforms.ReplaceField$Key",
+            "org.apache.kafka.connect.transforms.ReplaceField$Value",
+            "org.apache.kafka.connect.transforms.SetSchemaMetadata",
+            "org.apache.kafka.connect.transforms.SetSchemaMetadata$Key",
+            "org.apache.kafka.connect.transforms.SetSchemaMetadata$Value",
+            "org.apache.kafka.connect.transforms.TimestampConverter",
+            "org.apache.kafka.connect.transforms.TimestampConverter$Key",
+            "org.apache.kafka.connect.transforms.TimestampConverter$Value",
+            "org.apache.kafka.connect.transforms.TimestampRouter",
+            "org.apache.kafka.connect.transforms.TimestampRouter$Key",
+            "org.apache.kafka.connect.transforms.TimestampRouter$Value",
+            "org.apache.kafka.connect.transforms.ValueToKey"
+        );
+        for (String clazz : transformsClasses) {
+            assertTrue(
+                clazz + " from 'transforms' is not loaded in isolation but should be",
+                PluginUtils.shouldLoadInIsolation(clazz)
+            );
+        }
+    }
+
+    @Test
+    public void testAllowedJsonConverterClasses() {
+        List jsonConverterClasses = Arrays.asList(
+            "org.apache.kafka.connect.json.",
+            "org.apache.kafka.connect.json.DecimalFormat",
+            "org.apache.kafka.connect.json.JsonConverter",
+            "org.apache.kafka.connect.json.JsonConverterConfig",
+            "org.apache.kafka.connect.json.JsonDeserializer",
+            "org.apache.kafka.connect.json.JsonSchema",
+            "org.apache.kafka.connect.json.JsonSerializer"
+        );
+        for (String clazz : jsonConverterClasses) {
+            assertTrue(
+                clazz + " from 'json' is not loaded in isolation but should be",
+                PluginUtils.shouldLoadInIsolation(clazz)
+            );
+        }
+    }
+
+    @Test
+    public void testAllowedFileConnectors() {
+        List jsonConverterClasses = Arrays.asList(
+            "org.apache.kafka.connect.file.",
+            "org.apache.kafka.connect.file.FileStreamSinkConnector",
+            "org.apache.kafka.connect.file.FileStreamSinkTask",
+            "org.apache.kafka.connect.file.FileStreamSourceConnector",
+            "org.apache.kafka.connect.file.FileStreamSourceTask"
+        );
+        for (String clazz : jsonConverterClasses) {
+            assertTrue(
+                clazz + " from 'file' is not loaded in isolation but should be",
+                PluginUtils.shouldLoadInIsolation(clazz)
+            );
+        }
+    }
+
+    @Test
+    public void testAllowedBasicAuthExtensionClasses() {
+        List basicAuthExtensionClasses = Arrays.asList(
+            "org.apache.kafka.connect.rest.basic.auth.extension.BasicAuthSecurityRestExtension"
+            //"org.apache.kafka.connect.rest.basic.auth.extension.JaasBasicAuthFilter", TODO fix?
+            //"org.apache.kafka.connect.rest.basic.auth.extension.PropertyFileLoginModule" TODO fix?
+        );
+        for (String clazz : basicAuthExtensionClasses) {
+            assertTrue(
+                clazz + " from 'basic-auth-extension' is not loaded in isolation but should be",
+                PluginUtils.shouldLoadInIsolation(clazz)
+            );
+        }
+    }
+
+    @Test
+    public void testMirrorClasses() {
         assertTrue(PluginUtils.shouldLoadInIsolation(
                 "org.apache.kafka.connect.mirror.MirrorSourceTask")
         );
         assertTrue(PluginUtils.shouldLoadInIsolation(
                 "org.apache.kafka.connect.mirror.MirrorSourceConnector")
         );
-        assertTrue(PluginUtils.shouldLoadInIsolation("org.apache.kafka.connect.converters."));
-        assertTrue(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.converters.ByteArrayConverter")
-        );
-        assertTrue(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.converters.DoubleConverter")
-        );
-        assertTrue(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.converters.FloatConverter")
-        );
-        assertTrue(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.converters.IntegerConverter")
-        );
-        assertTrue(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.converters.LongConverter")
-        );
-        assertTrue(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.converters.ShortConverter")
-        );
-        assertTrue(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.storage.StringConverter")
-        );
-        assertTrue(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.storage.SimpleHeaderConverter")
-        );
-        assertTrue(PluginUtils.shouldLoadInIsolation(
-            "org.apache.kafka.connect.rest.basic.auth.extension.BasicAuthSecurityRestExtension"
-        ));
     }
 
     @Test
@@ -191,25 +365,6 @@ public void testClientConfigProvider() {
         );
     }
 
-    @Test
-    public void testConnectorClientConfigOverridePolicy() {
-        assertFalse(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.connector.policy.ConnectorClientConfigOverridePolicy")
-        );
-        assertTrue(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.connector.policy.AbstractConnectorClientConfigOverridePolicy")
-        );
-        assertTrue(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.connector.policy.AllConnectorClientConfigOverridePolicy")
-        );
-        assertTrue(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.connector.policy.NoneConnectorClientConfigOverridePolicy")
-        );
-        assertTrue(PluginUtils.shouldLoadInIsolation(
-                "org.apache.kafka.connect.connector.policy.PrincipalConnectorClientConfigOverridePolicy")
-        );
-    }
-
     @Test
     public void testEmptyPluginUrls() throws Exception {
         assertEquals(Collections.emptyList(), PluginUtils.pluginUrls(pluginPath));

From 5abed274efd1b6c0b04025243ca04e04a9740cf6 Mon Sep 17 00:00:00 2001
From: Lucent-Wong 
Date: Thu, 11 Jun 2020 08:59:32 +0800
Subject: [PATCH 0972/1071] KAFKA-9841: Revoke duplicate connectors and tasks
 when zombie workers return with an outdated assignment (#8453)

With Incremental Cooperative Rebalancing, if a worker returns after it's been out of the group for sometime (essentially as a zombie worker) and hasn't voluntarily revoked its own connectors and tasks in the meantime, there's the possibility that these assignments have been distributed to other workers and redundant connectors and tasks might be running now in the Connect cluster.

This PR complements previous fixes such as KAFKA-9184, KAFKA-9849 and KAFKA-9851 providing a last line of defense against zombie tasks: if at any rebalance round the leader worker detects that there are duplicate assignments in the group, it revokes them completely and resolves duplication with a correct assignment in the rebalancing round that will follow task revocation.

Author: Wang 

Reviewer: Konstantine Karantasis 
---
 .../IncrementalCooperativeAssignor.java       |  59 +++++++
 .../IncrementalCooperativeAssignorTest.java   | 162 +++++++++++++++++-
 2 files changed, 217 insertions(+), 4 deletions(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java
index d4e9d878c5515..744a099730097 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java
@@ -244,6 +244,10 @@ protected Map performTaskAssignment(String leaderId, long ma
         Map toRevoke = computeDeleted(deleted, connectorAssignments, taskAssignments);
         log.debug("Connector and task to delete assignments: {}", toRevoke);
 
+        // Revoking redundant connectors/tasks if the the workers have duplicate assignments
+        toRevoke.putAll(computeDuplicatedAssignments(memberConfigs, connectorAssignments, taskAssignments));
+        log.debug("Connector and task to revoke assignments (include duplicated assignments): {}", toRevoke);
+
         // Recompute the complete assignment excluding the deleted connectors-and-tasks
         completeWorkerAssignment = workerAssignment(memberConfigs, deleted);
         connectorAssignments =
@@ -359,6 +363,61 @@ private ConnectorsAndTasks computePreviousAssignment(Map memberConfigs) {
+        Set connectors = memberConfigs.entrySet().stream()
+                .flatMap(memberConfig -> memberConfig.getValue().assignment().connectors().stream())
+                .collect(Collectors.groupingBy(Function.identity(), Collectors.counting()))
+                .entrySet().stream()
+                .filter(entry -> entry.getValue() > 1L)
+                .map(entry -> entry.getKey())
+                .collect(Collectors.toSet());
+
+        Set tasks = memberConfigs.values().stream()
+                .flatMap(state -> state.assignment().tasks().stream())
+                .collect(Collectors.groupingBy(Function.identity(), Collectors.counting()))
+                .entrySet().stream()
+                .filter(entry -> entry.getValue() > 1L)
+                .map(entry -> entry.getKey())
+                .collect(Collectors.toSet());
+        return new ConnectorsAndTasks.Builder().with(connectors, tasks).build();
+    }
+
+    private Map computeDuplicatedAssignments(Map memberConfigs,
+                                             Map> connectorAssignments,
+                                             Map> taskAssignment) {
+        ConnectorsAndTasks duplicatedAssignments = duplicatedAssignments(memberConfigs);
+        log.debug("Duplicated assignments: {}", duplicatedAssignments);
+
+        Map toRevoke = new HashMap<>();
+        if (!duplicatedAssignments.connectors().isEmpty()) {
+            connectorAssignments.entrySet().stream()
+                    .forEach(entry -> {
+                        Set duplicatedConnectors = new HashSet<>(duplicatedAssignments.connectors());
+                        duplicatedConnectors.retainAll(entry.getValue());
+                        if (!duplicatedConnectors.isEmpty()) {
+                            toRevoke.computeIfAbsent(
+                                entry.getKey(),
+                                v -> new ConnectorsAndTasks.Builder().build()
+                            ).connectors().addAll(duplicatedConnectors);
+                        }
+                    });
+        }
+        if (!duplicatedAssignments.tasks().isEmpty()) {
+            taskAssignment.entrySet().stream()
+                    .forEach(entry -> {
+                        Set duplicatedTasks = new HashSet<>(duplicatedAssignments.tasks());
+                        duplicatedTasks.retainAll(entry.getValue());
+                        if (!duplicatedTasks.isEmpty()) {
+                            toRevoke.computeIfAbsent(
+                                entry.getKey(),
+                                v -> new ConnectorsAndTasks.Builder().build()
+                            ).tasks().addAll(duplicatedTasks);
+                        }
+                    });
+        }
+        return toRevoke;
+    }
+
     // visible for testing
     protected void handleLostAssignments(ConnectorsAndTasks lostAssignments,
                                          ConnectorsAndTasks newSubmissions,
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java
index 3e72ce9913df5..684da46bdadb1 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java
@@ -1089,6 +1089,148 @@ public void testLostAssignmentHandlingWhenWorkerBouncesBackButFinallyLeaves() {
         assertEquals(0, assignor.delay);
     }
 
+    @Test
+    public void testTaskAssignmentWhenTasksDuplicatedInWorkerAssignment() {
+        when(coordinator.configSnapshot()).thenReturn(configState);
+        doReturn(Collections.EMPTY_MAP).when(assignor).serializeAssignments(assignmentsCapture.capture());
+
+        // First assignment with 1 worker and 2 connectors configured but not yet assigned
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
+        ++rebalanceNum;
+        returnedAssignments = assignmentsCapture.getValue();
+        assertDelay(0, returnedAssignments);
+        expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
+        assertAssignment(2, 8, 0, 0, "worker1");
+
+        // Second assignment with a second worker with duplicate assignment joining and all connectors running on previous worker
+        applyAssignments(returnedAssignments);
+        memberConfigs = memberConfigs(leader, offset, assignments);
+        ExtendedAssignment duplicatedWorkerAssignment = newExpandableAssignment();
+        duplicatedWorkerAssignment.connectors().addAll(newConnectors(1, 2));
+        duplicatedWorkerAssignment.tasks().addAll(newTasks("connector1", 0, 4));
+        memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, duplicatedWorkerAssignment));
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
+        ++rebalanceNum;
+        returnedAssignments = assignmentsCapture.getValue();
+        assertDelay(0, returnedAssignments);
+        expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
+        assertAssignment(0, 0, 2, 8, "worker1", "worker2");
+
+        // Third assignment after revocations
+        applyAssignments(returnedAssignments);
+        memberConfigs = memberConfigs(leader, offset, assignments);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
+        ++rebalanceNum;
+        returnedAssignments = assignmentsCapture.getValue();
+        assertDelay(0, returnedAssignments);
+        expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
+        assertAssignment(1, 4, 0, 2, "worker1", "worker2");
+
+        // fourth rebalance after revocations
+        applyAssignments(returnedAssignments);
+        memberConfigs = memberConfigs(leader, offset, assignments);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
+        ++rebalanceNum;
+        returnedAssignments = assignmentsCapture.getValue();
+        assertDelay(0, returnedAssignments);
+        expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
+        assertAssignment(0, 2, 0, 0, "worker1", "worker2");
+
+        // Fifth rebalance should not change assignments
+        applyAssignments(returnedAssignments);
+        memberConfigs = memberConfigs(leader, offset, assignments);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
+        ++rebalanceNum;
+        returnedAssignments = assignmentsCapture.getValue();
+        assertDelay(0, returnedAssignments);
+        expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
+        assertAssignment(0, 0, 0, 0, "worker1", "worker2");
+
+        verify(coordinator, times(rebalanceNum)).configSnapshot();
+        verify(coordinator, times(rebalanceNum)).leaderState(any());
+        verify(coordinator, times(2 * rebalanceNum)).generationId();
+        verify(coordinator, times(rebalanceNum)).memberId();
+        verify(coordinator, times(rebalanceNum)).lastCompletedGenerationId();
+    }
+
+    @Test
+    public void testDuplicatedAssignmentHandleWhenTheDuplicatedAssignmentsDeleted() {
+        when(coordinator.configSnapshot()).thenReturn(configState);
+        doReturn(Collections.EMPTY_MAP).when(assignor).serializeAssignments(assignmentsCapture.capture());
+
+        // First assignment with 1 worker and 2 connectors configured but not yet assigned
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
+        ++rebalanceNum;
+        returnedAssignments = assignmentsCapture.getValue();
+        assertDelay(0, returnedAssignments);
+        expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
+        assertAssignment(2, 8, 0, 0, "worker1");
+
+        //delete connector1
+        configState = clusterConfigState(offset, 2, 1, 4);
+        when(coordinator.configSnapshot()).thenReturn(configState);
+
+        // Second assignment with a second worker with duplicate assignment joining and the duplicated assignment is deleted at the same time
+        applyAssignments(returnedAssignments);
+        memberConfigs = memberConfigs(leader, offset, assignments);
+        ExtendedAssignment duplicatedWorkerAssignment = newExpandableAssignment();
+        duplicatedWorkerAssignment.connectors().addAll(newConnectors(1, 2));
+        duplicatedWorkerAssignment.tasks().addAll(newTasks("connector1", 0, 4));
+        memberConfigs.put("worker2", new ExtendedWorkerState(leaderUrl, offset, duplicatedWorkerAssignment));
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
+        ++rebalanceNum;
+        returnedAssignments = assignmentsCapture.getValue();
+        assertDelay(0, returnedAssignments);
+        expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
+        assertAssignment(0, 0, 2, 8, "worker1", "worker2");
+
+        // Third assignment after revocations
+        applyAssignments(returnedAssignments);
+        memberConfigs = memberConfigs(leader, offset, assignments);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
+        ++rebalanceNum;
+        returnedAssignments = assignmentsCapture.getValue();
+        assertDelay(0, returnedAssignments);
+        expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
+        assertAssignment(0, 0, 0, 2, "worker1", "worker2");
+
+        // fourth rebalance after revocations
+        applyAssignments(returnedAssignments);
+        memberConfigs = memberConfigs(leader, offset, assignments);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
+        ++rebalanceNum;
+        returnedAssignments = assignmentsCapture.getValue();
+        assertDelay(0, returnedAssignments);
+        expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
+        assertAssignment(0, 2, 0, 0, "worker1", "worker2");
+
+        // Fifth rebalance should not change assignments
+        applyAssignments(returnedAssignments);
+        memberConfigs = memberConfigs(leader, offset, assignments);
+        assignor.performTaskAssignment(leader, offset, memberConfigs, coordinator, protocolVersion);
+        ++rebalanceNum;
+        returnedAssignments = assignmentsCapture.getValue();
+        assertDelay(0, returnedAssignments);
+        expectedMemberConfigs = memberConfigs(leader, offset, returnedAssignments);
+        assertNoReassignments(memberConfigs, expectedMemberConfigs);
+        assertAssignment(0, 0, 0, 0, "worker1", "worker2");
+
+        verify(coordinator, times(rebalanceNum)).configSnapshot();
+        verify(coordinator, times(rebalanceNum)).leaderState(any());
+        verify(coordinator, times(2 * rebalanceNum)).generationId();
+        verify(coordinator, times(rebalanceNum)).memberId();
+        verify(coordinator, times(rebalanceNum)).lastCompletedGenerationId();
+    }
+
     private WorkerLoad emptyWorkerLoad(String worker) {
         return new WorkerLoad.Builder(worker).build();
     }
@@ -1107,20 +1249,32 @@ private static List newConnectors(int start, int end) {
     }
 
     private static List newTasks(int start, int end) {
+        return newTasks("task", start, end);
+    }
+
+    private static List newTasks(String connectorName, int start, int end) {
         return IntStream.range(start, end)
-                .mapToObj(i -> new ConnectorTaskId("task", i))
+                .mapToObj(i -> new ConnectorTaskId(connectorName, i))
                 .collect(Collectors.toList());
     }
 
     private static ClusterConfigState clusterConfigState(long offset,
                                                          int connectorNum,
                                                          int taskNum) {
+        return clusterConfigState(offset, 1, connectorNum, taskNum);
+    }
+
+    private static ClusterConfigState clusterConfigState(long offset,
+                                                         int connectorStart,
+                                                         int connectorNum,
+                                                         int taskNum) {
+        int connectorNumEnd = connectorStart + connectorNum - 1;
         return new ClusterConfigState(
                 offset,
                 null,
-                connectorTaskCounts(1, connectorNum, taskNum),
-                connectorConfigs(1, connectorNum),
-                connectorTargetStates(1, connectorNum, TargetState.STARTED),
+                connectorTaskCounts(connectorStart, connectorNumEnd, taskNum),
+                connectorConfigs(connectorStart, connectorNumEnd),
+                connectorTargetStates(connectorStart, connectorNumEnd, TargetState.STARTED),
                 taskConfigs(0, connectorNum, connectorNum * taskNum),
                 Collections.emptySet());
     }

From 30264c2709a645f5725e662160c1d072cc9245a9 Mon Sep 17 00:00:00 2001
From: Mario Molina 
Date: Wed, 10 Jun 2020 19:58:00 -0500
Subject: [PATCH 0973/1071] KAFKA-9985: Sink connector may exhaust broker when
 writing in DLQ (#8663)

* Validate topic name against DQL topic in Sink connector config

* Adding parseTopicsList method

* Suppress warning

* KAFKA-9985: Minor changes to improve readability of exception messages

Co-authored-by: Randall Hauch 
---
 .../connect/runtime/SinkConnectorConfig.java  | 43 +++++++++++++++++++
 .../kafka/connect/runtime/WorkerSinkTask.java |  8 ++--
 .../connect/runtime/AbstractHerderTest.java   | 30 +++++++++++++
 3 files changed, 76 insertions(+), 5 deletions(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SinkConnectorConfig.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SinkConnectorConfig.java
index 0672f4e9c7f7f..0ed1b2b9608c8 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SinkConnectorConfig.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SinkConnectorConfig.java
@@ -18,12 +18,17 @@
 
 import org.apache.kafka.common.config.ConfigDef;
 import org.apache.kafka.common.config.ConfigDef.Importance;
+import org.apache.kafka.common.config.ConfigDef.Type;
 import org.apache.kafka.common.config.ConfigException;
 import org.apache.kafka.connect.runtime.isolation.Plugins;
 import org.apache.kafka.connect.sink.SinkTask;
 import org.apache.kafka.connect.transforms.util.RegexValidator;
 
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 /**
  * Configuration needed for all sink connectors
@@ -86,6 +91,7 @@ public SinkConnectorConfig(Plugins plugins, Map props) {
     public static void validate(Map props) {
         final boolean hasTopicsConfig = hasTopicsConfig(props);
         final boolean hasTopicsRegexConfig = hasTopicsRegexConfig(props);
+        final boolean hasDlqTopicConfig = hasDlqTopicConfig(props);
 
         if (hasTopicsConfig && hasTopicsRegexConfig) {
             throw new ConfigException(SinkTask.TOPICS_CONFIG + " and " + SinkTask.TOPICS_REGEX_CONFIG +
@@ -96,6 +102,25 @@ public static void validate(Map props) {
             throw new ConfigException("Must configure one of " +
                 SinkTask.TOPICS_CONFIG + " or " + SinkTask.TOPICS_REGEX_CONFIG);
         }
+
+        if (hasDlqTopicConfig) {
+            String dlqTopic = props.get(DLQ_TOPIC_NAME_CONFIG).trim();
+            if (hasTopicsConfig) {
+                List topics = parseTopicsList(props);
+                if (topics.contains(dlqTopic)) {
+                    throw new ConfigException(String.format("The DLQ topic '%s' may not be included in the list of "
+                            + "topics ('%s=%s') consumed by the connector", dlqTopic, SinkTask.TOPICS_REGEX_CONFIG, topics));
+                }
+            }
+            if (hasTopicsRegexConfig) {
+                String topicsRegexStr = props.get(SinkTask.TOPICS_REGEX_CONFIG);
+                Pattern pattern = Pattern.compile(topicsRegexStr);
+                if (pattern.matcher(dlqTopic).matches()) {
+                    throw new ConfigException(String.format("The DLQ topic '%s' may not be included in the regex matching the "
+                            + "topics ('%s=%s') consumed by the connector", dlqTopic, SinkTask.TOPICS_REGEX_CONFIG, topicsRegexStr));
+                }
+            }
+        }
     }
 
     public static boolean hasTopicsConfig(Map props) {
@@ -108,6 +133,24 @@ public static boolean hasTopicsRegexConfig(Map props) {
         return topicsRegexStr != null && !topicsRegexStr.trim().isEmpty();
     }
 
+    public static boolean hasDlqTopicConfig(Map props) {
+        String dqlTopicStr = props.get(DLQ_TOPIC_NAME_CONFIG);
+        return dqlTopicStr != null && !dqlTopicStr.trim().isEmpty();
+    }
+
+    @SuppressWarnings("unchecked")
+    public static List parseTopicsList(Map props) {
+        List topics = (List) ConfigDef.parseType(TOPICS_CONFIG, props.get(TOPICS_CONFIG), Type.LIST);
+        if (topics == null) {
+            return Collections.emptyList();
+        }
+        return topics
+                .stream()
+                .filter(topic -> !topic.isEmpty())
+                .distinct()
+                .collect(Collectors.toList());
+    }
+
     public String dlqTopicName() {
         return getString(DLQ_TOPIC_NAME_CONFIG);
     }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
index 227f2f63a47d5..0fa28e6383cff 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
@@ -54,7 +54,6 @@
 
 import java.time.Duration;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
@@ -287,10 +286,9 @@ protected void initializeAndStart() {
         SinkConnectorConfig.validate(taskConfig);
 
         if (SinkConnectorConfig.hasTopicsConfig(taskConfig)) {
-            String[] topics = taskConfig.get(SinkTask.TOPICS_CONFIG).split(",");
-            Arrays.setAll(topics, i -> topics[i].trim());
-            consumer.subscribe(Arrays.asList(topics), new HandleRebalance());
-            log.debug("{} Initializing and starting task for topics {}", this, topics);
+            List topics = SinkConnectorConfig.parseTopicsList(taskConfig);
+            consumer.subscribe(topics, new HandleRebalance());
+            log.debug("{} Initializing and starting task for topics {}", this, Utils.join(topics, ", "));
         } else {
             String topicsRegexStr = taskConfig.get(SinkTask.TOPICS_REGEX_CONFIG);
             Pattern pattern = Pattern.compile(topicsRegexStr);
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java
index 48a79b1ca361d..fe57d57972c39 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java
@@ -318,6 +318,36 @@ public void testConfigValidationInvalidTopics() {
         verifyAll();
     }
 
+    @Test(expected = ConfigException.class)
+    public void testConfigValidationTopicsWithDlq() {
+        AbstractHerder herder = createConfigValidationHerder(TestSinkConnector.class, noneConnectorClientConfigOverridePolicy);
+        replayAll();
+
+        Map config = new HashMap<>();
+        config.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, TestSinkConnector.class.getName());
+        config.put(SinkConnectorConfig.TOPICS_CONFIG, "topic1");
+        config.put(SinkConnectorConfig.DLQ_TOPIC_NAME_CONFIG, "topic1");
+
+        herder.validateConnectorConfig(config);
+
+        verifyAll();
+    }
+
+    @Test(expected = ConfigException.class)
+    public void testConfigValidationTopicsRegexWithDlq() {
+        AbstractHerder herder = createConfigValidationHerder(TestSinkConnector.class, noneConnectorClientConfigOverridePolicy);
+        replayAll();
+
+        Map config = new HashMap<>();
+        config.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, TestSinkConnector.class.getName());
+        config.put(SinkConnectorConfig.TOPICS_REGEX_CONFIG, "topic.*");
+        config.put(SinkConnectorConfig.DLQ_TOPIC_NAME_CONFIG, "topic1");
+
+        herder.validateConnectorConfig(config);
+
+        verifyAll();
+    }
+
     @Test()
     public void testConfigValidationTransformsExtendResults() {
         AbstractHerder herder = createConfigValidationHerder(TestSourceConnector.class, noneConnectorClientConfigOverridePolicy);

From e19fa3c35450df7ed518e28036d8b8979d0febf5 Mon Sep 17 00:00:00 2001
From: Chris Egerton 
Date: Wed, 10 Jun 2020 20:06:38 -0700
Subject: [PATCH 0974/1071] KAFKA-9845: Warn users about using config providers
 with plugin.path property (#8455)

* KAFKA-9845: Fix plugin.path when config provider is used

* Revert "KAFKA-9845: Fix plugin.path when config provider is used"

This reverts commit 96caaa9a4934bcef78d7b145d18aa1718cb10009.

* KAFKA-9845: Emit ERROR-level log message when config provider is used for plugin.path property

* KAFKA-9845: Demote log message level from ERROR to WARN

Co-Authored-By: Nigel Liang 

* KAFKA-94845: Fix failing unit tests

* KAFKA-9845: Add warning message to docstring for plugin.path config

* KAFKA-9845: Apply suggestions from code review

Co-authored-by: Randall Hauch 

Co-authored-by: Nigel Liang 
Co-authored-by: Randall Hauch 
---
 .../kafka/connect/runtime/WorkerConfig.java   | 24 ++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java
index c0231103e5715..837cfe5c19c1f 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java
@@ -37,6 +37,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.regex.Pattern;
 
 import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
@@ -205,7 +206,10 @@ public class WorkerConfig extends AbstractConfig {
             + "plugins and their dependencies\n"
             + "Note: symlinks will be followed to discover dependencies or plugins.\n"
             + "Examples: plugin.path=/usr/local/share/java,/usr/local/share/kafka/plugins,"
-            + "/opt/connectors";
+            + "/opt/connectors\n" 
+            + "Do not use config provider variables in this property, since the raw path is used "
+            + "by the worker's scanner before config providers are initialized and used to "
+            + "replace variables.";
 
     public static final String CONFIG_PROVIDERS_CONFIG = "config.providers";
     protected static final String CONFIG_PROVIDERS_DOC =
@@ -367,6 +371,23 @@ private void logDeprecatedProperty(String propName, String propValue, String def
         }
     }
 
+    private void logPluginPathConfigProviderWarning(Map rawOriginals) {
+        String rawPluginPath = rawOriginals.get(PLUGIN_PATH_CONFIG);
+        // Can't use AbstractConfig::originalsStrings here since some values may be null, which
+        // causes that method to fail
+        String transformedPluginPath = Objects.toString(originals().get(PLUGIN_PATH_CONFIG));
+        if (!Objects.equals(rawPluginPath, transformedPluginPath)) {
+            log.warn(
+                "Variables cannot be used in the 'plugin.path' property, since the property is "
+                + "used by plugin scanning before the config providers that replace the " 
+                + "variables are initialized. The raw value '{}' was used for plugin scanning, as " 
+                + "opposed to the transformed value '{}', and this may cause unexpected results.",
+                rawPluginPath,
+                transformedPluginPath
+            );
+        }
+    }
+
     public Integer getRebalanceTimeout() {
         return null;
     }
@@ -386,6 +407,7 @@ public static List pluginLocations(Map props) {
     public WorkerConfig(ConfigDef definition, Map props) {
         super(definition, props);
         logInternalConverterDeprecationWarnings(props);
+        logPluginPathConfigProviderWarning(props);
     }
 
     private static class AdminListenersValidator implements ConfigDef.Validator {

From 7f6ada825f452fb2635a032308af504b32bb7077 Mon Sep 17 00:00:00 2001
From: Aditya Auradkar 
Date: Wed, 6 Sep 2017 10:18:30 -0700
Subject: [PATCH 0975/1071] [LI-HOTFIX] LIKAFKA-4425 and LIKAFKA-4424 - add
 quota metrics log TICKET = LI_DESCRIPTION =

This patch has 2 changes:
- Added a background thread that dumps the quota metrics into the log every 30 seconds.
- Also changed the delayQueue sensor into an absolute counter

EXIT_CRITERIA = MANUAL ["when quota metrics log is no longer needed"]
---
 .../kafka/server/ClientQuotaManager.scala     | 21 ++++++++++-
 .../server/ClientRequestQuotaManager.scala    |  2 +-
 .../main/scala/kafka/server/KafkaServer.scala |  2 +-
 .../scala/kafka/server/QuotaFactory.scala     |  8 +++--
 .../kafka/server/ClientQuotaManagerTest.scala | 35 +++++++++++++------
 .../ThrottledChannelExpirationTest.scala      |  2 +-
 6 files changed, 53 insertions(+), 17 deletions(-)

diff --git a/core/src/main/scala/kafka/server/ClientQuotaManager.scala b/core/src/main/scala/kafka/server/ClientQuotaManager.scala
index 8316d0c637b1c..54c1d0da28ab4 100644
--- a/core/src/main/scala/kafka/server/ClientQuotaManager.scala
+++ b/core/src/main/scala/kafka/server/ClientQuotaManager.scala
@@ -23,7 +23,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock
 import kafka.network.RequestChannel
 import kafka.network.RequestChannel._
 import kafka.server.ClientQuotaManager._
-import kafka.utils.{Logging, ShutdownableThread}
+import kafka.utils.{KafkaScheduler, Logging, ShutdownableThread}
 import org.apache.kafka.common.{Cluster, MetricName}
 import org.apache.kafka.common.metrics._
 import org.apache.kafka.common.metrics.Metrics
@@ -160,6 +160,7 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig,
                          private val metrics: Metrics,
                          private val quotaType: QuotaType,
                          private val time: Time,
+                         private val schedulerOpt: Option[KafkaScheduler],
                          threadNamePrefix: String,
                          clientQuotaCallback: Option[ClientQuotaCallback] = None) extends Logging {
   private val staticConfigClientIdQuota = Quota.upperBound(config.quotaBytesPerSecondDefault)
@@ -183,6 +184,11 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig,
   start() // Use start method to keep spotbugs happy
   private def start(): Unit = {
     throttledChannelReaper.start()
+    schedulerOpt match {
+      case Some(scheduler) =>
+        scheduler.schedule("quota-metrics-logger-%s".format(quotaType), logQuotaMetrics, 60, 60, TimeUnit.SECONDS)
+      case _ =>
+    }
   }
 
   /**
@@ -203,6 +209,19 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig,
     }
   }
 
+  def logQuotaMetrics(): Unit = {
+    import scala.collection.JavaConverters._
+    val metricsMap = metrics.metrics().asScala;
+    metricsMap.foreach {
+      case (metricName: MetricName, kafkaMetric: KafkaMetric) =>
+        if (metricName.group().equals(quotaType.toString) &&
+          (metricName.name().equals("byte-rate") || metricName.name().equals("throttle-time")) &&
+          metricName.tags().containsKey("client-id")) {
+          info("Metric name (" + metricName + ") has value (" + kafkaMetric.value() + ")")
+        }
+    }
+  }
+
   /**
    * Returns true if any quotas are enabled for this quota manager. This is used
    * to determine if quota related metrics should be created.
diff --git a/core/src/main/scala/kafka/server/ClientRequestQuotaManager.scala b/core/src/main/scala/kafka/server/ClientRequestQuotaManager.scala
index eb921dca7d32f..1f8af13bb450c 100644
--- a/core/src/main/scala/kafka/server/ClientRequestQuotaManager.scala
+++ b/core/src/main/scala/kafka/server/ClientRequestQuotaManager.scala
@@ -32,7 +32,7 @@ class ClientRequestQuotaManager(private val config: ClientQuotaManagerConfig,
                                 private val time: Time,
                                 threadNamePrefix: String,
                                 quotaCallback: Option[ClientQuotaCallback])
-                                extends ClientQuotaManager(config, metrics, QuotaType.Request, time, threadNamePrefix, quotaCallback) {
+                                extends ClientQuotaManager(config, metrics, QuotaType.Request, time, None, threadNamePrefix, quotaCallback) {
   val maxThrottleTimeMs = TimeUnit.SECONDS.toMillis(this.config.quotaWindowSizeSeconds)
   def exemptSensor = getOrCreateSensor(exemptSensorName, exemptMetricName)
 
diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala
index de0ee22155185..41d3beb8b9ee6 100755
--- a/core/src/main/scala/kafka/server/KafkaServer.scala
+++ b/core/src/main/scala/kafka/server/KafkaServer.scala
@@ -241,7 +241,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
         /* register broker metrics */
         _brokerTopicStats = new BrokerTopicStats
 
-        quotaManagers = QuotaFactory.instantiate(config, metrics, time, threadNamePrefix.getOrElse(""))
+        quotaManagers = QuotaFactory.instantiate(config, metrics, time, Some(kafkaScheduler), threadNamePrefix.getOrElse(""))
         notifyClusterListeners(kafkaMetricsReporters ++ metrics.reporters.asScala)
 
         logDirFailureChannel = new LogDirFailureChannel(config.logDirs.size)
diff --git a/core/src/main/scala/kafka/server/QuotaFactory.scala b/core/src/main/scala/kafka/server/QuotaFactory.scala
index c941163d017c4..78a265ac7e6a7 100644
--- a/core/src/main/scala/kafka/server/QuotaFactory.scala
+++ b/core/src/main/scala/kafka/server/QuotaFactory.scala
@@ -19,6 +19,7 @@ package kafka.server
 import kafka.server.QuotaType._
 import kafka.utils.Logging
 import org.apache.kafka.common.TopicPartition
+import kafka.utils.KafkaScheduler
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.server.quota.ClientQuotaCallback
 import org.apache.kafka.common.utils.Time
@@ -57,12 +58,15 @@ object QuotaFactory extends Logging {
   }
 
   def instantiate(cfg: KafkaConfig, metrics: Metrics, time: Time, threadNamePrefix: String): QuotaManagers = {
+    instantiate(cfg, metrics, time, None, threadNamePrefix)
+  }
 
+  def instantiate(cfg: KafkaConfig, metrics: Metrics, time: Time, schedulerOpt: Option[KafkaScheduler], threadNamePrefix: String): QuotaManagers = {
     val clientQuotaCallback = Option(cfg.getConfiguredInstance(KafkaConfig.ClientQuotaCallbackClassProp,
       classOf[ClientQuotaCallback]))
     QuotaManagers(
-      new ClientQuotaManager(clientFetchConfig(cfg), metrics, Fetch, time, threadNamePrefix, clientQuotaCallback),
-      new ClientQuotaManager(clientProduceConfig(cfg), metrics, Produce, time, threadNamePrefix, clientQuotaCallback),
+      new ClientQuotaManager(clientFetchConfig(cfg), metrics, Fetch, time, schedulerOpt, threadNamePrefix, clientQuotaCallback),
+      new ClientQuotaManager(clientProduceConfig(cfg), metrics, Produce, time, schedulerOpt, threadNamePrefix, clientQuotaCallback),
       new ClientRequestQuotaManager(clientRequestConfig(cfg), metrics, time, threadNamePrefix, clientQuotaCallback),
       new ReplicationQuotaManager(replicationConfig(cfg), metrics, LeaderReplication, time),
       new ReplicationQuotaManager(replicationConfig(cfg), metrics, FollowerReplication, time),
diff --git a/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala
index f277e5f323d8c..344e0e760ec17 100644
--- a/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala
@@ -23,6 +23,7 @@ import java.util.Collections
 import kafka.network.RequestChannel
 import kafka.network.RequestChannel.{EndThrottlingResponse, Session, StartThrottlingResponse}
 import kafka.server.QuotaType._
+import kafka.utils.KafkaScheduler
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.memory.MemoryPool
 import org.apache.kafka.common.metrics.{MetricConfig, Metrics, Quota}
@@ -33,12 +34,13 @@ import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
 import org.apache.kafka.common.utils.{MockTime, Sanitizer}
 import org.easymock.EasyMock
 import org.junit.Assert.{assertEquals, assertTrue}
-import org.junit.{After, Test}
+import org.junit.{Before, After, Test}
 
 class ClientQuotaManagerTest {
   private val time = new MockTime
   private val metrics = new Metrics(new MetricConfig(), Collections.emptyList(), time)
   private val config = ClientQuotaManagerConfig(quotaBytesPerSecondDefault = 500)
+  private val scheduler = new KafkaScheduler(1)
 
   var numCallbacks: Int = 0
 
@@ -55,6 +57,17 @@ class ClientQuotaManagerTest {
     }
   }
 
+  @Before
+  def beforeMethod() {
+    numCallbacks = 0
+    scheduler.startup()
+  }
+
+  @After
+  def afterClass() {
+    scheduler.shutdown()
+  }
+
   private def buildRequest[T <: AbstractRequest](builder: AbstractRequest.Builder[T],
                                                  listenerName: ListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)): (T, RequestChannel.Request) = {
 
@@ -82,7 +95,7 @@ class ClientQuotaManagerTest {
   }
 
   private def testQuotaParsing(config: ClientQuotaManagerConfig, client1: UserClient, client2: UserClient, randomClient: UserClient, defaultConfigClient: UserClient): Unit = {
-    val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, "")
+    val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, Some(scheduler), "")
 
     try {
       // Case 1: Update the quota. Assert that the new quota value is returned
@@ -209,7 +222,7 @@ class ClientQuotaManagerTest {
   def testGetMaxValueInQuotaWindowWithNonDefaultQuotaWindow(): Unit = {
     val numFullQuotaWindows = 3   // 3 seconds window (vs. 10 seconds default)
     val nonDefaultConfig = ClientQuotaManagerConfig(quotaBytesPerSecondDefault = Long.MaxValue, numQuotaSamples = numFullQuotaWindows + 1)
-    val quotaManager = new ClientQuotaManager(nonDefaultConfig, metrics, Fetch, time, "")
+    val quotaManager = new ClientQuotaManager(nonDefaultConfig, metrics, Fetch, time, Some(scheduler), "")
     val userSession = Session(new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "userA"), InetAddress.getLocalHost)
 
     try {
@@ -228,7 +241,7 @@ class ClientQuotaManagerTest {
   def testSetAndRemoveDefaultUserQuota(): Unit = {
     // quotaTypesEnabled will be QuotaTypes.NoQuotas initially
     val quotaManager = new ClientQuotaManager(ClientQuotaManagerConfig(quotaBytesPerSecondDefault = Long.MaxValue),
-      metrics, Produce, time, "")
+      metrics, Produce, time, Some(scheduler), "")
 
     try {
       // no quota set yet, should not throttle
@@ -250,7 +263,7 @@ class ClientQuotaManagerTest {
   def testSetAndRemoveUserQuota(): Unit = {
     // quotaTypesEnabled will be QuotaTypes.NoQuotas initially
     val quotaManager = new ClientQuotaManager(ClientQuotaManagerConfig(quotaBytesPerSecondDefault = Long.MaxValue),
-      metrics, Produce, time, "")
+      metrics, Produce, time, Some(scheduler), "")
 
     try {
       // Set  quota config
@@ -269,7 +282,7 @@ class ClientQuotaManagerTest {
   def testSetAndRemoveUserClientQuota(): Unit = {
     // quotaTypesEnabled will be QuotaTypes.NoQuotas initially
     val quotaManager = new ClientQuotaManager(ClientQuotaManagerConfig(quotaBytesPerSecondDefault = Long.MaxValue),
-      metrics, Produce, time, "")
+      metrics, Produce, time, Some(scheduler), "")
 
     try {
       // Set  quota config
@@ -287,7 +300,7 @@ class ClientQuotaManagerTest {
   @Test
   def testQuotaConfigPrecedence(): Unit = {
     val quotaManager = new ClientQuotaManager(ClientQuotaManagerConfig(quotaBytesPerSecondDefault=Long.MaxValue),
-      metrics, Produce, time, "")
+      metrics, Produce, time, Some(scheduler), "")
 
     try {
       quotaManager.updateQuota(Some(ConfigEntityName.Default), None, None, Some(new Quota(1000, true)))
@@ -350,7 +363,7 @@ class ClientQuotaManagerTest {
 
   @Test
   def testQuotaViolation(): Unit = {
-    val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, "")
+    val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, Some(scheduler), "")
     val queueSizeMetric = metrics.metrics().get(metrics.metricName("queue-size", "Produce", ""))
     try {
       /* We have 10 second windows. Make sure that there is no quota violation
@@ -459,7 +472,7 @@ class ClientQuotaManagerTest {
 
   @Test
   def testExpireThrottleTimeSensor(): Unit = {
-    val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, "")
+    val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, Some(scheduler), "")
     try {
       maybeRecord(clientMetrics, "ANONYMOUS", "client1", 100)
       // remove the throttle time sensor
@@ -478,7 +491,7 @@ class ClientQuotaManagerTest {
 
   @Test
   def testExpireQuotaSensors(): Unit = {
-    val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, "")
+    val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, Some(scheduler), "")
     try {
       maybeRecord(clientMetrics, "ANONYMOUS", "client1", 100)
       // remove all the sensors
@@ -501,7 +514,7 @@ class ClientQuotaManagerTest {
 
   @Test
   def testClientIdNotSanitized(): Unit = {
-    val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, "")
+    val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, Some(scheduler), "")
     val clientId = "client@#$%"
     try {
       maybeRecord(clientMetrics, "ANONYMOUS", clientId, 100)
diff --git a/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala b/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala
index 15bbcc6e2b906..746ee93b392e2 100644
--- a/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala
@@ -75,7 +75,7 @@ class ThrottledChannelExpirationTest {
 
   @Test
   def testCallbackInvocationAfterExpiration(): Unit = {
-    val clientMetrics = new ClientQuotaManager(ClientQuotaManagerConfig(), metrics, QuotaType.Produce, time, "")
+    val clientMetrics = new ClientQuotaManager(ClientQuotaManagerConfig(), metrics, QuotaType.Produce, time, None, "")
 
     val delayQueue = new DelayQueue[ThrottledChannel]()
     val reaper = new clientMetrics.ThrottledChannelReaper(delayQueue, "")

From 7bd85471eef0d4dc8e7bac5ebd646de00b4d23dc Mon Sep 17 00:00:00 2001
From: Aditya Auradkar 
Date: Thu, 17 Aug 2017 12:05:49 -0700
Subject: [PATCH 0976/1071] [LI-HOTFIX] LIKAFKA-4424: Add a metric to track the
 total number of requests that have been throttled and rename the delay queue
 size metric TICKET = LI_DESCRIPTION =

add throttle-count metric and rename delay queue size sensor.

EXIT_CRITERIA = MANUAL ["when throttle-count metric is no longer needed"]
---
 .../main/scala/kafka/server/ClientQuotaManager.scala   | 10 ++++++++--
 .../unit/kafka/server/ClientQuotaManagerTest.scala     |  7 +++++++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/kafka/server/ClientQuotaManager.scala b/core/src/main/scala/kafka/server/ClientQuotaManager.scala
index 54c1d0da28ab4..4158298606023 100644
--- a/core/src/main/scala/kafka/server/ClientQuotaManager.scala
+++ b/core/src/main/scala/kafka/server/ClientQuotaManager.scala
@@ -177,7 +177,7 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig,
   private[server] val throttledChannelReaper = new ThrottledChannelReaper(delayQueue, threadNamePrefix)
   private val quotaCallback = clientQuotaCallback.getOrElse(new DefaultQuotaCallback)
 
-  private val delayQueueSensor = metrics.sensor(quotaType + "-delayQueue")
+  private val delayQueueSensor = metrics.sensor(quotaType + "-delayQueueSize")
   delayQueueSensor.add(metrics.metricName("queue-size",
     quotaType.toString,
     "Tracks the size of the delay queue"), new CumulativeSum())
@@ -191,6 +191,11 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig,
     }
   }
 
+  private val throttledRequestCountSensor = metrics.sensor(quotaType + "-throttledRequestCount")
+  throttledRequestCountSensor.add(metrics.metricName("throttle-count",
+                                                     quotaType.toString,
+                                                     "Tracks the number of requests that have been throttled"), new CumulativeSum())
+
   /**
    * Reaper thread that triggers channel unmute callbacks on all throttled channels
    * @param delayQueue DelayQueue to dequeue from
@@ -217,7 +222,7 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig,
         if (metricName.group().equals(quotaType.toString) &&
           (metricName.name().equals("byte-rate") || metricName.name().equals("throttle-time")) &&
           metricName.tags().containsKey("client-id")) {
-          info("Metric name (" + metricName + ") has value (" + kafkaMetric.value() + ")")
+          info("Metric name (" + metricName + ") has value (" + kafkaMetric.metricValue() + ")")
         }
     }
   }
@@ -311,6 +316,7 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig,
       val throttledChannel = new ThrottledChannel(request, time, throttleTimeMs, channelThrottlingCallback)
       delayQueue.add(throttledChannel)
       delayQueueSensor.record()
+      throttledRequestCountSensor.record()
       debug("Channel throttled for sensor (%s). Delay time: (%d)".format(clientSensors.quotaSensor.name(), throttleTimeMs))
     }
   }
diff --git a/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala
index 344e0e760ec17..9a86496fc178f 100644
--- a/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala
@@ -365,6 +365,7 @@ class ClientQuotaManagerTest {
   def testQuotaViolation(): Unit = {
     val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, Some(scheduler), "")
     val queueSizeMetric = metrics.metrics().get(metrics.metricName("queue-size", "Produce", ""))
+    val throttleCountMetric = metrics.metrics().get(metrics.metricName("throttle-count", "Produce", ""))
     try {
       /* We have 10 second windows. Make sure that there is no quota violation
        * if we produce under the quota
@@ -384,7 +385,10 @@ class ClientQuotaManagerTest {
 
       assertEquals("Should be throttled", 2100, sleepTime)
       throttle(clientMetrics, "ANONYMOYUS", "unknown", sleepTime, callback)
+
       assertEquals(1, queueSizeMetric.metricValue.asInstanceOf[Double].toInt)
+      assertEquals(1, throttleCountMetric.metricValue.asInstanceOf[Double].toInt)
+
       // After a request is delayed, the callback cannot be triggered immediately
       clientMetrics.throttledChannelReaper.doWork()
       assertEquals(0, numCallbacks)
@@ -392,7 +396,10 @@ class ClientQuotaManagerTest {
 
       // Callback can only be triggered after the delay time passes
       clientMetrics.throttledChannelReaper.doWork()
+
+      assertEquals(1, throttleCountMetric.metricValue.asInstanceOf[Double].toInt)
       assertEquals(0, queueSizeMetric.metricValue.asInstanceOf[Double].toInt)
+
       assertEquals(1, numCallbacks)
 
       // Could continue to see delays until the bursty sample disappears

From 92fd7d606f403ac8dad4ad67651c7e9a2f04a9e3 Mon Sep 17 00:00:00 2001
From: Onur Karaman 
Date: Wed, 6 Sep 2017 10:34:18 -0700
Subject: [PATCH 0977/1071] [LI-HOTFIX] LIKAFKA-8529: avoid duplicate app-info
 registrations TICKET = LI_DESCRIPTION =

avoid duplicate app-info registrations

RB=877780
BUG=LIKAFKA-8529
G=Kafka-Code-Reviews
A=jkoshy

EXIT_CRITERIA = MANUAL [""]
---
 .../java/org/apache/kafka/common/utils/AppInfoParser.java    | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/utils/AppInfoParser.java b/clients/src/main/java/org/apache/kafka/common/utils/AppInfoParser.java
index 3ceca99d74de4..997ba7610340b 100644
--- a/clients/src/main/java/org/apache/kafka/common/utils/AppInfoParser.java
+++ b/clients/src/main/java/org/apache/kafka/common/utils/AppInfoParser.java
@@ -61,8 +61,9 @@ public static synchronized void registerAppInfo(String prefix, String id, Metric
         try {
             ObjectName name = new ObjectName(prefix + ":type=app-info,id=" + Sanitizer.jmxSanitize(id));
             AppInfo mBean = new AppInfo(nowMs);
-            ManagementFactory.getPlatformMBeanServer().registerMBean(mBean, name);
-
+            if (!ManagementFactory.getPlatformMBeanServer().isRegistered(name)) {
+                ManagementFactory.getPlatformMBeanServer().registerMBean(mBean, name);
+            }
             registerMetrics(metrics, mBean); // prefix will be added later by JmxReporter
         } catch (JMException e) {
             log.warn("Error registering AppInfo mbean", e);

From a10fd70e9b75447ddb582fd67903bc618c295bdd Mon Sep 17 00:00:00 2001
From: Dong Lin 
Date: Sat, 19 Aug 2017 00:18:45 -0700
Subject: [PATCH 0978/1071] [LI-HOTFIX] add metadata.topic.expiry.ms config to
 KafkaProducer TICKET = LI_DESCRIPTION =

add metadata.topic.expiry.ms config to KafkaProducer
RB=1015577
G=Kafka-Code-Reviews
A=jkoshy

EXIT_CRITERIA = MANUAL ["NONE"]
---
 .../kafka/clients/CommonClientConfigs.java       |  3 +++
 .../kafka/clients/producer/KafkaProducer.java    |  3 ++-
 .../kafka/clients/producer/ProducerConfig.java   | 10 ++++++++++
 .../producer/internals/ProducerMetadata.java     | 16 +++++++++++++---
 4 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java b/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java
index af47e55d75a6a..758a41782de11 100644
--- a/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java
+++ b/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java
@@ -49,6 +49,9 @@ public class CommonClientConfigs {
     public static final String METADATA_MAX_AGE_CONFIG = "metadata.max.age.ms";
     public static final String METADATA_MAX_AGE_DOC = "The period of time in milliseconds after which we force a refresh of metadata even if we haven't seen any partition leadership changes to proactively discover any new brokers or partitions.";
 
+    public static final String METADATA_TOPIC_EXPIRY_MS_CONFIG = "metadata.topic.expiry.ms";
+    public static final String METADATA_TOPIC_EXPIRY_MS_DOC = "The period of time in milliseconds before we expire topic from metadata cache";
+
     public static final String SEND_BUFFER_CONFIG = "send.buffer.bytes";
     public static final String SEND_BUFFER_DOC = "The size of the TCP send buffer (SO_SNDBUF) to use when sending data. If the value is -1, the OS default will be used.";
     public static final int SEND_BUFFER_LOWER_BOUND = -1;
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
index 51e1a8bd09b89..78cdc8cc67057 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
@@ -414,7 +414,8 @@ public KafkaProducer(Properties properties, Serializer keySerializer, Seriali
                         config.getLong(ProducerConfig.METADATA_MAX_AGE_CONFIG),
                         logContext,
                         clusterResourceListeners,
-                        Time.SYSTEM);
+                        Time.SYSTEM,
+                        config.getLong(ProducerConfig.METADATA_TOPIC_EXPIRY_MS_CONFIG));
                 this.metadata.bootstrap(addresses);
             }
             this.errors = this.metrics.sensor("errors");
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
index f81480433df11..047244fc8964c 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
@@ -19,6 +19,7 @@
 import org.apache.kafka.clients.ClientDnsLookup;
 import org.apache.kafka.clients.CommonClientConfigs;
 import org.apache.kafka.clients.producer.internals.DefaultPartitioner;
+import org.apache.kafka.clients.producer.internals.ProducerMetadata;
 import org.apache.kafka.common.config.AbstractConfig;
 import org.apache.kafka.common.config.ConfigDef;
 import org.apache.kafka.common.config.ConfigDef.Importance;
@@ -53,9 +54,13 @@ public class ProducerConfig extends AbstractConfig {
     /** bootstrap.servers */
     public static final String BOOTSTRAP_SERVERS_CONFIG = CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
 
+
     /** client.dns.lookup */
     public static final String CLIENT_DNS_LOOKUP_CONFIG = CommonClientConfigs.CLIENT_DNS_LOOKUP_CONFIG;
 
+    public static final String METADATA_TOPIC_EXPIRY_MS_CONFIG = CommonClientConfigs.METADATA_TOPIC_EXPIRY_MS_CONFIG;
+    private static final String METADATA_TOPIC_EXPIRY_MS_DOC = CommonClientConfigs.METADATA_TOPIC_EXPIRY_MS_DOC;
+
     /** metadata.max.age.ms */
     public static final String METADATA_MAX_AGE_CONFIG = CommonClientConfigs.METADATA_MAX_AGE_CONFIG;
     private static final String METADATA_MAX_AGE_DOC = CommonClientConfigs.METADATA_MAX_AGE_DOC;
@@ -366,6 +371,11 @@ public class ProducerConfig extends AbstractConfig {
                                         60000,
                                         Importance.LOW,
                                         TRANSACTION_TIMEOUT_DOC)
+                                .define(METADATA_TOPIC_EXPIRY_MS_CONFIG,
+                                        Type.LONG,
+                                        ProducerMetadata.TOPIC_EXPIRY_MS,
+                                        Importance.LOW,
+                                        METADATA_TOPIC_EXPIRY_MS_DOC)
                                 .define(TRANSACTIONAL_ID_CONFIG,
                                         Type.STRING,
                                         null,
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerMetadata.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerMetadata.java
index da18189426d70..c21ae4d097867 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerMetadata.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerMetadata.java
@@ -33,21 +33,31 @@
 
 public class ProducerMetadata extends Metadata {
     private static final long TOPIC_EXPIRY_NEEDS_UPDATE = -1L;
-    static final long TOPIC_EXPIRY_MS = 5 * 60 * 1000;
+    public static final long TOPIC_EXPIRY_MS = 5 * 60 * 1000;
 
     /* Topics with expiry time */
     private final Map topics = new HashMap<>();
     private final Logger log;
     private final Time time;
+    private final long topicExpiryMs;
 
+    public ProducerMetadata(long refreshBackoffMs,
+        long metadataExpireMs,
+        LogContext logContext,
+        ClusterResourceListeners clusterResourceListeners,
+        Time time) {
+        this(refreshBackoffMs, metadataExpireMs, logContext, clusterResourceListeners, time, TOPIC_EXPIRY_MS);
+    }
     public ProducerMetadata(long refreshBackoffMs,
                             long metadataExpireMs,
                             LogContext logContext,
                             ClusterResourceListeners clusterResourceListeners,
-                            Time time) {
+                            Time time,
+                            long topicExpiryMs) {
         super(refreshBackoffMs, metadataExpireMs, logContext, clusterResourceListeners);
         this.log = logContext.logger(ProducerMetadata.class);
         this.time = time;
+        this.topicExpiryMs = topicExpiryMs;
     }
 
     @Override
@@ -77,7 +87,7 @@ public synchronized boolean retainTopic(String topic, boolean isInternal, long n
         if (expireMs == null) {
             return false;
         } else if (expireMs == TOPIC_EXPIRY_NEEDS_UPDATE) {
-            topics.put(topic, nowMs + TOPIC_EXPIRY_MS);
+            topics.put(topic, Long.MAX_VALUE - nowMs > topicExpiryMs ? nowMs + topicExpiryMs : Long.MAX_VALUE);
             return true;
         } else if (expireMs <= nowMs) {
             log.debug("Removing unused topic {} from the metadata list, expiryMs {} now {}", topic, expireMs, nowMs);

From 7fcaf767b74852a5188554cf6c51b93216f68e2c Mon Sep 17 00:00:00 2001
From: Onur Karaman 
Date: Mon, 8 Jan 2018 12:32:54 -0800
Subject: [PATCH 0979/1071] [LI-HOTFIX] LIKAFKA-12852: mirror maker with new
 consumer should make progress after committing to a deleted topic TICKET =
 LI_DESCRIPTION =

New consumer throws a KafkaException when trying to commit to a topic that has been deleted. MirrorMaker.commitOffsets doesn't attempt to catch the KafkaException and just kills the process. This hotfix just catches KafkaException in MirrorMaker.commitOffsets until we make a cleaner long-term fix.

RB=1120784
BUG=LIKAFKA-12852
G=Kafka-Code-Reviews
A=jqin

EXIT_CRITERIA = MANUAL ["needed until we make a long-term fix"]
---
 core/src/main/scala/kafka/tools/MirrorMaker.scala | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core/src/main/scala/kafka/tools/MirrorMaker.scala b/core/src/main/scala/kafka/tools/MirrorMaker.scala
index a37fa6e58e819..8ef0e08c1e7cb 100755
--- a/core/src/main/scala/kafka/tools/MirrorMaker.scala
+++ b/core/src/main/scala/kafka/tools/MirrorMaker.scala
@@ -155,7 +155,11 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
               "another instance. If you see this regularly, it could indicate that you need to either increase " +
               s"the consumer's ${ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG} or reduce the number of records " +
               s"handled on each iteration with ${ConsumerConfig.MAX_POLL_RECORDS_CONFIG}")
+          // HOTFIX LIKAFKA-12852
+          case e: KafkaException if e.getMessage != null && e.getMessage.contains("may not exist or user may not have Describe access to topic") =>
+            error("Failed to commit offsets due to an unrecoverable error such as committing to a deleted topic", e)
         }
+
       }
     } else {
       info("Exiting on send failure, skip committing offsets.")

From 4e3b991dfbfc7cdfe99820abf5626d4056ae2837 Mon Sep 17 00:00:00 2001
From: Jiangjie Qin 
Date: Mon, 8 Jan 2018 12:35:55 -0800
Subject: [PATCH 0980/1071] [LI-HOTFIX] LIKAFKA-12852, attempt #2. Clear the
 offset map after receiving offset commit exception. TICKET = LI_DESCRIPTION =

EXIT_CRITERIA = MANUAL ["needed until we make a long-term fix"]
---
 core/src/main/scala/kafka/tools/MirrorMaker.scala | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/kafka/tools/MirrorMaker.scala b/core/src/main/scala/kafka/tools/MirrorMaker.scala
index 8ef0e08c1e7cb..6975f8ea86dfd 100755
--- a/core/src/main/scala/kafka/tools/MirrorMaker.scala
+++ b/core/src/main/scala/kafka/tools/MirrorMaker.scala
@@ -157,9 +157,9 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
               s"handled on each iteration with ${ConsumerConfig.MAX_POLL_RECORDS_CONFIG}")
           // HOTFIX LIKAFKA-12852
           case e: KafkaException if e.getMessage != null && e.getMessage.contains("may not exist or user may not have Describe access to topic") =>
+            consumerWrapper.clearOffsetMap()
             error("Failed to commit offsets due to an unrecoverable error such as committing to a deleted topic", e)
         }
-
       }
     } else {
       info("Exiting on send failure, skip committing offsets.")
@@ -352,6 +352,12 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
       consumer.commitSync(offsets.map { case (tp, offset) => (tp, new OffsetAndMetadata(offset)) }.asJava)
       offsets.clear()
     }
+
+    // HOTFIX LIKAFKA-12852, need to clear the offsets map when a KafkaException has been thrown due to topic
+    // deletion during offset commet.
+    def clearOffsetMap(): Unit = {
+      offsets.clear()
+    }
   }
 
   private class InternalRebalanceListener(consumerWrapper: ConsumerWrapper,

From 130aa35d18972595aeced6b96357d9300abcc9b9 Mon Sep 17 00:00:00 2001
From: Navina Ramesh 
Date: Mon, 8 Jan 2018 13:44:30 -0800
Subject: [PATCH 0981/1071] [LI-HOTFIX] Adding metrics for log compaction
 threads alive TICKET = KAFKA-6588 LI_DESCRIPTION =  Introduces a new metric
 that is useful to monitor log compaction threads in the broker. Too much
 overhead in open-source to introduce this metric. Hence, the hotfix.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

EXIT_CRITERIA = MANUAL ["when this metric is no longer needed. KAFKA-6588 (Won’t Fix)"]
---
 core/src/main/scala/kafka/log/LogCleaner.scala | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/core/src/main/scala/kafka/log/LogCleaner.scala b/core/src/main/scala/kafka/log/LogCleaner.scala
index f7babf7ab9b33..cac1ac50cfe9c 100644
--- a/core/src/main/scala/kafka/log/LogCleaner.scala
+++ b/core/src/main/scala/kafka/log/LogCleaner.scala
@@ -145,6 +145,12 @@ class LogCleaner(initialConfig: CleanerConfig,
     })
 
   private[log] def deadThreadCount: Int = cleaners.count(_.isThreadFailed)
+  /* a metric to track the number of cleaner threads alive */
+  newGauge("live-cleaner-thread-count",
+          new Gauge[Int] {
+            def value: Int = cleaners.count(_.asInstanceOf[Thread].isAlive)
+          })
+
 
   /**
    * Start the background cleaning

From 5c2542cfbaca6aba0a3b2330d8b1c546e6802cfc Mon Sep 17 00:00:00 2001
From: Jiangjie Qin 
Date: Mon, 8 Jan 2018 14:14:30 -0800
Subject: [PATCH 0982/1071] [LI-HOTFIX] Clear the offset map even when the
 offset commit failed. We need to do this to avoid committing old offsets.
 TICKET = LI_DESCRIPTION =

EXIT_CRITERIA = MANUAL ["NONE"]
---
 core/src/main/scala/kafka/tools/MirrorMaker.scala | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/core/src/main/scala/kafka/tools/MirrorMaker.scala b/core/src/main/scala/kafka/tools/MirrorMaker.scala
index 6975f8ea86dfd..de4e1097b0367 100755
--- a/core/src/main/scala/kafka/tools/MirrorMaker.scala
+++ b/core/src/main/scala/kafka/tools/MirrorMaker.scala
@@ -133,6 +133,7 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
             // so if we catch it in commit we can safely retry
             // and re-throw to break the loop
             commitOffsets(consumerWrapper)
+            consumerWrapper.clearOffsetMap()
             throw e
 
           case _: TimeoutException =>
@@ -151,6 +152,7 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
 
           case _: CommitFailedException =>
             retryNeeded = false
+            consumerWrapper.clearOffsetMap()
             warn("Failed to commit offsets because the consumer group has rebalanced and assigned partitions to " +
               "another instance. If you see this regularly, it could indicate that you need to either increase " +
               s"the consumer's ${ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG} or reduce the number of records " +

From d85b38ebf80123d020361a159d4207d4dd3e698a Mon Sep 17 00:00:00 2001
From: Dong Lin 
Date: Wed, 25 Oct 2017 16:38:29 -0700
Subject: [PATCH 0983/1071] [LI-HOTFIX] Consumer should fetch metadata from the
 same broker until it is disconnected from that broker TICKET = LI_DESCRIPTION
 = - HOTFIX LIKAFKA-13869; Prevent MetadataRequest starvation by trying to
 send MetadataRequest before the end of networkClient.poll()

EXIT_CRITERIA = MANUAL ["when sticky metadata fetch feature is implemented upstream"]
---
 .../kafka/clients/CommonClientConfigs.java    |  3 +++
 .../apache/kafka/clients/NetworkClient.java   | 23 ++++++++++++++++---
 .../clients/consumer/ConsumerConfig.java      |  5 ++++
 .../kafka/clients/consumer/KafkaConsumer.java |  1 +
 .../clients/producer/ProducerConfig.java      |  5 ++++
 5 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java b/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java
index 758a41782de11..ecb3f78eff847 100644
--- a/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java
+++ b/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java
@@ -143,6 +143,9 @@ public class CommonClientConfigs {
                                                            + "consumer's session stays active and to facilitate rebalancing when new consumers join or leave the group. "
                                                            + "The value must be set lower than session.timeout.ms, but typically should be set no higher "
                                                            + "than 1/3 of that value. It can be adjusted even lower to control the expected time for normal rebalances.";
+    public static final String ENABLE_STICKY_METADATA_FETCH_CONFIG = "enable.sticky.metadata.fetch";
+    public static final String ENABLE_STICKY_METADATA_FETCH_DOC = "Fetch metadata from the least loaded broker if false. Otherwise fetch metadata "
+                                                         + "from the same broker until it is disconnected.";
 
     /**
      * Postprocess the configuration so that exponential backoff is disabled when reconnect backoff
diff --git a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
index 9e52b4e2d2f43..d8af1aee6441f 100644
--- a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
@@ -115,6 +115,8 @@ private enum State {
 
     private final Time time;
 
+    private boolean enableStickyMetadataFetch = true;
+
     /**
      * True if we should send an ApiVersionRequest when first connecting to a broker.
      */
@@ -273,6 +275,10 @@ private NetworkClient(MetadataUpdater metadataUpdater,
         this.state = new AtomicReference<>(State.ACTIVE);
     }
 
+    public void setEnableStickyMetadataFetch(boolean enableStickyMetadataFetch) {
+        this.enableStickyMetadataFetch = enableStickyMetadataFetch;
+    }
+
     /**
      * Begin connecting to the given node, return true if we are already connected and ready to send to that node.
      *
@@ -562,6 +568,12 @@ public List poll(long timeout, long now) {
         handleTimedOutRequests(responses, updatedNow);
         completeResponses(responses);
 
+        // We changed the metadataUpdater.maybeUpdate() such that it will keep sending MetadataRequest
+        // to the same broker instead choosing the least loaded node. If we don't try to send metadata here, it is possible that
+        // another request is sent to the broker before the next networkClient.poll(). This can cause starvation
+        // for the MetadataRequest and consumer's metadata may be stale for a long time.
+        metadataUpdater.maybeUpdate(updatedNow);
+
         return responses;
     }
 
@@ -973,6 +985,8 @@ class DefaultMetadataUpdater implements MetadataUpdater {
 
         /* the current cluster metadata */
         private final Metadata metadata;
+        // Consumer needs to keep fetching metadata from the same node until that node goes down
+        private Node nodeToFetchMetadata;
 
         // Defined if there is a request in progress, null otherwise
         private Integer inProgressRequestVersion;
@@ -980,6 +994,7 @@ class DefaultMetadataUpdater implements MetadataUpdater {
         DefaultMetadataUpdater(Metadata metadata) {
             this.metadata = metadata;
             this.inProgressRequestVersion = null;
+            this.nodeToFetchMetadata = null;
         }
 
         @Override
@@ -1009,13 +1024,15 @@ public long maybeUpdate(long now) {
 
             // Beware that the behavior of this method and the computation of timeouts for poll() are
             // highly dependent on the behavior of leastLoadedNode.
-            Node node = leastLoadedNode(now);
-            if (node == null) {
+            if (!enableStickyMetadataFetch || nodeToFetchMetadata == null || !connectionStates.isReady(nodeToFetchMetadata.idString(), now))
+                nodeToFetchMetadata = leastLoadedNode(now);
+
+            if (nodeToFetchMetadata == null) {
                 log.debug("Give up sending metadata request since no node is available");
                 return reconnectBackoffMs;
             }
 
-            return maybeUpdate(now, node);
+            return maybeUpdate(now, nodeToFetchMetadata);
         }
 
         @Override
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
index e0f7c4d160d5e..d7ff73b9016ec 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
@@ -503,6 +503,11 @@ public class ConsumerConfig extends AbstractConfig {
                                         CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL,
                                         Importance.MEDIUM,
                                         CommonClientConfigs.SECURITY_PROTOCOL_DOC)
+                                .define(CommonClientConfigs.ENABLE_STICKY_METADATA_FETCH_CONFIG,
+                                        Type.BOOLEAN,
+                                        true,
+                                        Importance.MEDIUM,
+                                        CommonClientConfigs.ENABLE_STICKY_METADATA_FETCH_DOC)
                                 .withClientSslSupport()
                                 .withClientSaslSupport();
     }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index a431ab3b554ff..eaf8cff9a8ddc 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -761,6 +761,7 @@ else if (enableAutoCommit)
                     apiVersions,
                     throttleTimeSensor,
                     logContext);
+            netClient.setEnableStickyMetadataFetch(config.getBoolean(CommonClientConfigs.ENABLE_STICKY_METADATA_FETCH_CONFIG));
             this.client = new ConsumerNetworkClient(
                     logContext,
                     netClient,
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
index 047244fc8964c..96a14bca61c39 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
@@ -359,6 +359,11 @@ public class ProducerConfig extends AbstractConfig {
                                         null,
                                         Importance.LOW,
                                         SECURITY_PROVIDERS_DOC)
+                                .define(CommonClientConfigs.ENABLE_STICKY_METADATA_FETCH_CONFIG,
+                                        Type.BOOLEAN,
+                                        false,
+                                        Importance.MEDIUM,
+                                        CommonClientConfigs.ENABLE_STICKY_METADATA_FETCH_DOC)
                                 .withClientSslSupport()
                                 .withClientSaslSupport()
                                 .define(ENABLE_IDEMPOTENCE_CONFIG,

From 026392b2f35c1c38fa668fe92a56a431785e5ac5 Mon Sep 17 00:00:00 2001
From: Radai Rosenblatt 
Date: Fri, 19 Jan 2018 17:43:17 -0800
Subject: [PATCH 0984/1071] [LI-HOTFIX] LIKAFKA-14632 - add more heartbeat and
 poll timing sensors to new consumer TICKET =  LIKAFKA-14632 LI_DESCRIPTION =
 adds "poll-interval", "last-poll-seconds-ago" and
 "last-heartbeat-received-seconds-ago" sensors to consumer

RB=1198496
G=Kafka-Code-Reviews
R=tpalino,kambroff
A=dolin

EXIT_CRITERIA = MANUAL ["none right now"]
---
 .../internals/AbstractCoordinator.java        | 16 ++++++++
 .../internals/ConsumerCoordinator.java        | 38 ++++++++++++++++++-
 .../clients/consumer/internals/Heartbeat.java | 10 ++++-
 3 files changed, 61 insertions(+), 3 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index 8c482dec0cbef..15d1c278572ef 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -39,6 +39,7 @@
 import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse;
 import org.apache.kafka.common.message.SyncGroupRequestData;
 import org.apache.kafka.common.metrics.Measurable;
+import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
@@ -1143,6 +1144,21 @@ public GroupCoordinatorMetrics(Metrics metrics, String metricGrpPrefix) {
                 this.metricGrpName,
                 "The number of seconds since the last coordinator heartbeat was sent"),
                 lastHeartbeat);
+
+            //HOTFIX - extra liveliness-related metrics
+
+            Measurable lastHeartbeatReceived =
+                new Measurable() {
+                    public double measure(MetricConfig config, long now) {
+                        return TimeUnit.SECONDS.convert(now - heartbeat.lastHeartbeatReceive(), TimeUnit.MILLISECONDS);
+                    }
+                };
+            metrics.addMetric(metrics.metricName("last-heartbeat-received-seconds-ago",
+                this.metricGrpName,
+                "The number of seconds since the last successful controller heartbeat was received"),
+                lastHeartbeatReceived);
+
+            //end HOTFIX
         }
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 6b905c57774f5..9784955e6e1ce 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -17,6 +17,7 @@
 package org.apache.kafka.clients.consumer.internals;
 
 import org.apache.kafka.clients.GroupRebalanceConfig;
+import java.util.concurrent.TimeUnit;
 import org.apache.kafka.clients.consumer.CommitFailedException;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor;
@@ -45,6 +46,7 @@
 import org.apache.kafka.common.message.OffsetCommitRequestData;
 import org.apache.kafka.common.message.OffsetCommitResponseData;
 import org.apache.kafka.common.metrics.Measurable;
+import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
@@ -105,6 +107,9 @@ public final class ConsumerCoordinator extends AbstractCoordinator {
     private Timer nextAutoCommitTimer;
     private AtomicBoolean asyncCommitFenced;
 
+    private volatile long prevPollTime = Long.MIN_VALUE; //volatile for metrics
+
+
     // hold onto request&future for committed offset requests to enable async calls.
     private PendingCommittedOffsetRequest pendingCommittedOffsetRequest = null;
 
@@ -429,10 +434,16 @@ void maybeUpdateSubscriptionMetadata() {
      * @return true iff the operation succeeded
      */
     public boolean poll(Timer timer) {
+
+        long currentTime = time.milliseconds();
+        if (prevPollTime > Long.MIN_VALUE) {
+            sensors.pollInterval.record(currentTime - prevPollTime);
+        }
+        prevPollTime = currentTime;
+
         maybeUpdateSubscriptionMetadata();
 
         invokeCompletedOffsetCommitCallbacks();
-
         if (subscriptions.partitionsAutoAssigned()) {
             if (protocol == null) {
                 throw new IllegalStateException("User configured " + ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG +
@@ -1260,6 +1271,7 @@ private class ConsumerCoordinatorMetrics {
         private final Sensor revokeCallbackSensor;
         private final Sensor assignCallbackSensor;
         private final Sensor loseCallbackSensor;
+        private final Sensor pollInterval;
 
         private ConsumerCoordinatorMetrics(Metrics metrics, String metricGrpPrefix) {
             this.metricGrpName = metricGrpPrefix + "-coordinator-metrics";
@@ -1301,6 +1313,30 @@ private ConsumerCoordinatorMetrics(Metrics metrics, String metricGrpPrefix) {
             metrics.addMetric(metrics.metricName("assigned-partitions",
                 this.metricGrpName,
                 "The number of partitions currently assigned to this consumer"), numParts);
+
+            //HOTFIX - extra liveliness-related metrics
+
+            this.pollInterval = metrics.sensor("poll-interval");
+            this.pollInterval.add(metrics.metricName("poll-interval-avg",
+                this.metricGrpName,
+                "The average time between subsequent poll calls"), new Avg());
+            this.pollInterval.add(metrics.metricName("poll-interval-max",
+                this.metricGrpName,
+                "The max time between subsequent poll calls"), new Max());
+            this.pollInterval.add(createMeter(metrics, metricGrpName, "poll", "poll calls"));
+
+            Measurable lastHeartbeat =
+                new Measurable() {
+                    public double measure(MetricConfig config, long now) {
+                        return TimeUnit.SECONDS.convert(now - prevPollTime, TimeUnit.MILLISECONDS);
+                    }
+                };
+            metrics.addMetric(metrics.metricName("last-poll-seconds-ago",
+                this.metricGrpName,
+                "The number of seconds since the last poll call"),
+                lastHeartbeat);
+
+            //end HOTFIX
         }
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Heartbeat.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Heartbeat.java
index 4d19ef4a0141e..0697d62cf6d14 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Heartbeat.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Heartbeat.java
@@ -32,6 +32,7 @@ public final class Heartbeat {
     private final Timer pollTimer;
 
     private volatile long lastHeartbeatSend = 0L;
+    private volatile long lastHeartbeatReceive;
 
     public Heartbeat(GroupRebalanceConfig config,
                      Time time) {
@@ -46,6 +47,7 @@ public Heartbeat(GroupRebalanceConfig config,
     }
 
     private void update(long now) {
+        lastHeartbeatReceive = now;
         heartbeatTimer.update(now);
         sessionTimer.update(now);
         pollTimer.update(now);
@@ -76,11 +78,15 @@ public boolean shouldHeartbeat(long now) {
         update(now);
         return heartbeatTimer.isExpired();
     }
-    
+
     public long lastHeartbeatSend() {
         return this.lastHeartbeatSend;
     }
 
+    public long lastHeartbeatReceive() {
+        return lastHeartbeatReceive;
+    }
+
     public long timeToNextHeartbeat(long now) {
         update(now);
         return heartbeatTimer.remainingMs();
@@ -112,4 +118,4 @@ public long lastPollTime() {
         return pollTimer.currentTimeMs();
     }
 
-}
\ No newline at end of file
+}

From e61bde5d7cf751e5ed4a4c4d4d0415919e4cbce0 Mon Sep 17 00:00:00 2001
From: Sean McCauliff 
Date: Fri, 19 Jan 2018 13:54:39 -0800
Subject: [PATCH 0985/1071] [LI-HOTFIX] Do not let callers of
 BufferPool.dellocate deallocate more memory than total memory. TICKET =
 LI_DESCRIPTION =

RB=1186874
G=Kafka-Code-Reviews
A=dolin,jqin

EXIT_CRITERIA = MANUAL ["Moving this upstream would probably involve dropping some of the features of this patch."]
---
 .../producer/internals/BufferPool.java        | 55 +++++++++++++++----
 .../producer/internals/BufferPoolTest.java    | 17 ++++++
 2 files changed, 60 insertions(+), 12 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/BufferPool.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/BufferPool.java
index b49a7e2215f42..f96080648acd2 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/BufferPool.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/BufferPool.java
@@ -19,6 +19,9 @@
 import java.nio.ByteBuffer;
 import java.util.ArrayDeque;
 import java.util.Deque;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.Condition;
 import java.util.concurrent.locks.ReentrantLock;
@@ -30,6 +33,8 @@
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Meter;
 import org.apache.kafka.common.utils.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 
 /**
@@ -44,12 +49,14 @@
  */
 public class BufferPool {
 
+    private static final Logger log = LoggerFactory.getLogger(BufferPool.class);
+
     static final String WAIT_TIME_SENSOR_NAME = "bufferpool-wait-time";
 
     private final long totalMemory;
     private final int poolableSize;
     private final ReentrantLock lock;
-    private final Deque free;
+    private final Set free;
     private final Deque waiters;
     /** Total available memory is the sum of nonPooledAvailableMemory and the number of byte buffers in free * poolableSize.  */
     private long nonPooledAvailableMemory;
@@ -57,6 +64,7 @@ public class BufferPool {
     private final Time time;
     private final Sensor waitTime;
     private boolean closed;
+    private long nextOvermemoryWarn;
 
     /**
      * Create a new buffer pool
@@ -70,7 +78,7 @@ public class BufferPool {
     public BufferPool(long memory, int poolableSize, Metrics metrics, Time time, String metricGrpName) {
         this.poolableSize = poolableSize;
         this.lock = new ReentrantLock();
-        this.free = new ArrayDeque<>();
+        this.free = new LinkedHashSet<>();
         this.waiters = new ArrayDeque<>();
         this.totalMemory = memory;
         this.nonPooledAvailableMemory = memory;
@@ -85,6 +93,7 @@ public BufferPool(long memory, int poolableSize, Metrics metrics, Time time, Str
                                                    "The total time an appender waits for space allocation.");
         this.waitTime.add(new Meter(TimeUnit.NANOSECONDS, rateMetricName, totalMetricName));
         this.closed = false;
+        this.nextOvermemoryWarn = 0;
     }
 
     /**
@@ -116,7 +125,7 @@ public ByteBuffer allocate(int size, long maxTimeToBlockMs) throws InterruptedEx
         try {
             // check if we have a free buffer of the right size pooled
             if (size == poolableSize && !this.free.isEmpty())
-                return this.free.pollFirst();
+                return popBuffer();
 
             // now check if the request is immediately satisfiable with the
             // memory on hand or if we need to block
@@ -160,7 +169,7 @@ public ByteBuffer allocate(int size, long maxTimeToBlockMs) throws InterruptedEx
                         // otherwise allocate memory
                         if (accumulated == 0 && size == this.poolableSize && !this.free.isEmpty()) {
                             // just grab a buffer from the free list
-                            buffer = this.free.pollFirst();
+                            buffer = popBuffer();
                             accumulated = size;
                         } else {
                             // we'll need to allocate memory, but we may only get
@@ -237,25 +246,36 @@ protected ByteBuffer allocateByteBuffer(int size) {
      */
     private void freeUp(int size) {
         while (!this.free.isEmpty() && this.nonPooledAvailableMemory < size)
-            this.nonPooledAvailableMemory += this.free.pollLast().capacity();
+            this.nonPooledAvailableMemory += popBuffer().capacity();
     }
 
     /**
      * Return buffers to the pool. If they are of the poolable size add them to the free list, otherwise just mark the
      * memory as free.
      *
-     * @param buffer The buffer to return
+     * @param buffer The buffer to return to the pool.
      * @param size The size of the buffer to mark as deallocated, note that this may be smaller than buffer.capacity
-     *             since the buffer may re-allocate itself during in-place compression
+     *          since the buffer may re-allocate itself during in-place compression
      */
     public void deallocate(ByteBuffer buffer, int size) {
         lock.lock();
         try {
-            if (size == this.poolableSize && size == buffer.capacity()) {
-                buffer.clear();
-                this.free.add(buffer);
+            long availableMemory = availableMemoryUnlocked();
+            if (availableMemory + size > this.totalMemory && nextOvermemoryWarn < this.time.milliseconds()) {
+                //Don't flood logs.
+                log.error("Detected an attempt to bring available memory " + availableMemory + " to " +
+                    (availableMemory + size) + " which is higher than totalMemory " + this.totalMemory + ".", new Exception());
+                this.nextOvermemoryWarn = this.time.milliseconds() + TimeUnit.HOURS.toMillis(1);
+            }
+            if (buffer.hasArray() && buffer.array().length == this.poolableSize &&
+                this.poolableSize + availableMemory <= this.totalMemory) {
+                if (!this.free.add(buffer.array()) && this.nextOvermemoryWarn < time.milliseconds()) {
+                    log.error("Detected an attempt to double deallocate the same buffer.", new Exception());
+                    this.nextOvermemoryWarn = this.time.milliseconds() + TimeUnit.HOURS.toMillis(1);
+                }
             } else {
-                this.nonPooledAvailableMemory += size;
+                long freeMem = Math.min(Math.max(buffer.capacity(), size), this.totalMemory - availableMemory);
+                this.nonPooledAvailableMemory += freeMem;
             }
             Condition moreMem = this.waiters.peekFirst();
             if (moreMem != null)
@@ -275,12 +295,16 @@ public void deallocate(ByteBuffer buffer) {
     public long availableMemory() {
         lock.lock();
         try {
-            return this.nonPooledAvailableMemory + freeSize() * (long) this.poolableSize;
+            return availableMemoryUnlocked();
         } finally {
             lock.unlock();
         }
     }
 
+    private long availableMemoryUnlocked() {
+        return this.nonPooledAvailableMemory + freeSize() * (long) this.poolableSize;
+    }
+
     // Protected for testing.
     protected int freeSize() {
         return this.free.size();
@@ -343,4 +367,11 @@ public void close() {
             this.lock.unlock();
         }
     }
+
+    private ByteBuffer popBuffer() {
+        Iterator it = free.iterator();
+        byte[] array = it.next();
+        it.remove();
+        return ByteBuffer.wrap(array);
+    }
 }
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/BufferPoolTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/BufferPoolTest.java
index 724d9d4df10f4..e6bb5d9264979 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/BufferPoolTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/BufferPoolTest.java
@@ -351,6 +351,23 @@ protected ByteBuffer allocateByteBuffer(int size) {
         assertEquals(bufferPool.availableMemory(), 1024);
     }
 
+    @Test
+    public void overDeallocate() {
+        BufferPool bufferPool = new BufferPool(1024, 512, metrics, time, metricGroup);
+        bufferPool.deallocate(ByteBuffer.allocate(512));
+        assertEquals(bufferPool.availableMemory(), bufferPool.totalMemory());
+    }
+
+    @Test
+    public void dedupeDeallocations() throws Exception {
+        BufferPool bufferPool = new BufferPool(1024, 512, metrics, time, metricGroup);
+        ByteBuffer bbuf = bufferPool.allocate(512, 1);
+        ByteBuffer shallowCopy = bbuf.slice();
+        bufferPool.deallocate(bbuf);
+        bufferPool.deallocate(shallowCopy);
+        assertEquals(bufferPool.totalMemory(), bufferPool.availableMemory());
+    }
+
     public static class StressTestThread extends Thread {
         private final int iterations;
         private final BufferPool pool;

From 8926f588329bd071b532c05d2875c80305a71036 Mon Sep 17 00:00:00 2001
From: Dong Lin 
Date: Wed, 8 Aug 2018 17:12:10 -0700
Subject: [PATCH 0986/1071] [LI-HOTFIX] LIKAFKA-8072: Kafka should shutdown if
 it couldn't make progress in processing request TICKET = LI_DESCRIPTION =
 This patch adds a new config called request.processing.max.time.ms. By
 default its value is Long.MaxValue to be backward compatible. Network threads
 will log error message and call Runtime.getRuntime.halt(1) if one of the
 following is true: - New request can not be inserted into requestChannel
 after request.processing.max.time.ms - requestChannel.lastEnqueueTimeMs -
 requestChannel.lastDequeueTimeMs > request.processing.max.time.ms

This guarantees that Kafka broker will shutdown if no request was dequeued from requestChannel for more than request.processing.max.time.ms.
RB=844988
G=Kafka-Code-Reviews
A=jkoshy

EXIT_CRITERIA = MANUAL ["NONE"]
---
 .../scala/kafka/network/RequestChannel.scala  | 19 +++++++++++++++---
 .../scala/kafka/network/SocketServer.scala    |  4 ++--
 .../main/scala/kafka/server/KafkaConfig.scala |  7 +++++++
 .../main/scala/kafka/server/KafkaServer.scala | 20 +++++++++++++++++++
 4 files changed, 45 insertions(+), 5 deletions(-)

diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala
index 4a8c290d945e8..e44edba4b0cb4 100644
--- a/core/src/main/scala/kafka/network/RequestChannel.scala
+++ b/core/src/main/scala/kafka/network/RequestChannel.scala
@@ -272,7 +272,7 @@ object RequestChannel extends Logging {
   }
 }
 
-class RequestChannel(val queueSize: Int, val metricNamePrefix : String) extends KafkaMetricsGroup {
+class RequestChannel(val queueSize: Int, val metricNamePrefix : String, val time: Time) extends KafkaMetricsGroup {
   import RequestChannel._
   val metrics = new RequestChannel.Metrics
   private val requestQueue = new ArrayBlockingQueue[BaseRequest](queueSize)
@@ -280,6 +280,11 @@ class RequestChannel(val queueSize: Int, val metricNamePrefix : String) extends
   val requestQueueSizeMetricName = metricNamePrefix.concat(RequestQueueSizeMetric)
   val responseQueueSizeMetricName = metricNamePrefix.concat(ResponseQueueSizeMetric)
 
+  @volatile var lastDequeueTimeMs = time.milliseconds
+  // This metric can help user select a suitable threshold for requestMaxLocalTimeMs so that broker can shutdown itself only when it
+  // is stuck or too slow. A suggested value of requestMaxLocalTimeMs could be twice the 999'th percentile of the RequestDequeuePollIntervalMs.
+  private val requestDequeuePollIntervalMs = newHistogram("RequestDequeuePollIntervalMs")
+
   newGauge(requestQueueSizeMetricName, new Gauge[Int] {
       def value = requestQueue.size
   })
@@ -340,12 +345,20 @@ class RequestChannel(val queueSize: Int, val metricNamePrefix : String) extends
   }
 
   /** Get the next request or block until specified time has elapsed */
-  def receiveRequest(timeout: Long): RequestChannel.BaseRequest =
+  def receiveRequest(timeout: Long): RequestChannel.BaseRequest = {
+    val curTime = time.milliseconds
+    requestDequeuePollIntervalMs.update(curTime - lastDequeueTimeMs)
+    lastDequeueTimeMs = curTime
     requestQueue.poll(timeout, TimeUnit.MILLISECONDS)
+  }
 
   /** Get the next request or block until there is one */
-  def receiveRequest(): RequestChannel.BaseRequest =
+  def receiveRequest(): RequestChannel.BaseRequest = {
+    val curTime = time.milliseconds
+    requestDequeuePollIntervalMs.update(curTime - lastDequeueTimeMs)
+    lastDequeueTimeMs = curTime
     requestQueue.take()
+  }
 
   def updateErrorMetrics(apiKey: ApiKeys, errors: collection.Map[Errors, Integer]): Unit = {
     errors.foreach { case (error, count) =>
diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala
index a147aaa3722f5..6094082b64ab6 100644
--- a/core/src/main/scala/kafka/network/SocketServer.scala
+++ b/core/src/main/scala/kafka/network/SocketServer.scala
@@ -91,11 +91,11 @@ class SocketServer(val config: KafkaConfig,
   // data-plane
   private val dataPlaneProcessors = new ConcurrentHashMap[Int, Processor]()
   private[network] val dataPlaneAcceptors = new ConcurrentHashMap[EndPoint, Acceptor]()
-  val dataPlaneRequestChannel = new RequestChannel(maxQueuedRequests, DataPlaneMetricPrefix)
+  val dataPlaneRequestChannel = new RequestChannel(maxQueuedRequests, DataPlaneMetricPrefix, time)
   // control-plane
   private var controlPlaneProcessorOpt : Option[Processor] = None
   private[network] var controlPlaneAcceptorOpt : Option[Acceptor] = None
-  val controlPlaneRequestChannelOpt: Option[RequestChannel] = config.controlPlaneListenerName.map(_ => new RequestChannel(20, ControlPlaneMetricPrefix))
+  val controlPlaneRequestChannelOpt: Option[RequestChannel] = config.controlPlaneListenerName.map(_ => new RequestChannel(20, ControlPlaneMetricPrefix, time))
 
   private var nextProcessorId = 0
   private var connectionQuotas: ConnectionQuotas = _
diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index 17ebd278f19a3..bdd419abe2c86 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -76,6 +76,7 @@ object Defaults {
   val SocketSendBufferBytes: Int = 100 * 1024
   val SocketReceiveBufferBytes: Int = 100 * 1024
   val SocketRequestMaxBytes: Int = 100 * 1024 * 1024
+  val RequestMaxLocalTimeMs = Long.MaxValue
   val MaxConnectionsPerIp: Int = Int.MaxValue
   val MaxConnectionsPerIpOverrides: String = ""
   val MaxConnections: Int = Int.MaxValue
@@ -293,6 +294,7 @@ object KafkaConfig {
   val ControlPlaneListenerNameProp = "control.plane.listener.name"
   val SocketSendBufferBytesProp = "socket.send.buffer.bytes"
   val SocketReceiveBufferBytesProp = "socket.receive.buffer.bytes"
+  val RequestMaxLocalTimeMsProp = "request.max.local.time.ms"
   val SocketRequestMaxBytesProp = "socket.request.max.bytes"
   val MaxConnectionsPerIpProp = "max.connections.per.ip"
   val MaxConnectionsPerIpOverridesProp = "max.connections.per.ip.overrides"
@@ -577,6 +579,9 @@ object KafkaConfig {
 
   val SocketSendBufferBytesDoc = "The SO_SNDBUF buffer of the socket server sockets. If the value is -1, the OS default will be used."
   val SocketReceiveBufferBytesDoc = "The SO_RCVBUF buffer of the socket server sockets. If the value is -1, the OS default will be used."
+  val RequestMaxLocalTimeMsDoc = "The maximum allowable request local processing time. If a request's local processing " +
+    "takes longer than this time the broker will kill itself as violating this timeout is a symptom of a more serious broker zombie state." +
+    " It is useful to observe the RequestDequeuePollIntervalMs metric to find a suitable setting for this configuration."
   val SocketRequestMaxBytesDoc = "The maximum number of bytes in a socket request"
   val MaxConnectionsPerIpDoc = "The maximum number of connections we allow from each ip address. This can be set to 0 if there are overrides " +
     s"configured using $MaxConnectionsPerIpOverridesProp property. New connections from the ip address are dropped if the limit is reached."
@@ -889,6 +894,7 @@ object KafkaConfig {
       .define(ListenerSecurityProtocolMapProp, STRING, Defaults.ListenerSecurityProtocolMap, LOW, ListenerSecurityProtocolMapDoc)
       .define(ControlPlaneListenerNameProp, STRING, null, HIGH, controlPlaneListenerNameDoc)
       .define(SocketSendBufferBytesProp, INT, Defaults.SocketSendBufferBytes, HIGH, SocketSendBufferBytesDoc)
+      .define(RequestMaxLocalTimeMsProp, LONG, Defaults.RequestMaxLocalTimeMs, atLeast(1), MEDIUM, RequestMaxLocalTimeMsDoc)
       .define(SocketReceiveBufferBytesProp, INT, Defaults.SocketReceiveBufferBytes, HIGH, SocketReceiveBufferBytesDoc)
       .define(SocketRequestMaxBytesProp, INT, Defaults.SocketRequestMaxBytes, atLeast(1), HIGH, SocketRequestMaxBytesDoc)
       .define(MaxConnectionsPerIpProp, INT, Defaults.MaxConnectionsPerIp, atLeast(0), MEDIUM, MaxConnectionsPerIpDoc)
@@ -1197,6 +1203,7 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO
   val socketSendBufferBytes = getInt(KafkaConfig.SocketSendBufferBytesProp)
   val socketReceiveBufferBytes = getInt(KafkaConfig.SocketReceiveBufferBytesProp)
   val socketRequestMaxBytes = getInt(KafkaConfig.SocketRequestMaxBytesProp)
+  val requestMaxLocalTimeMs = getLong(KafkaConfig.RequestMaxLocalTimeMsProp)
   val maxConnectionsPerIp = getInt(KafkaConfig.MaxConnectionsPerIpProp)
   val maxConnectionsPerIpOverrides: Map[String, Int] =
     getMap(KafkaConfig.MaxConnectionsPerIpOverridesProp, getString(KafkaConfig.MaxConnectionsPerIpOverridesProp)).map { case (k, v) => (k, v.toInt)}
diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala
index 41d3beb8b9ee6..729cafe3b720e 100755
--- a/core/src/main/scala/kafka/server/KafkaServer.scala
+++ b/core/src/main/scala/kafka/server/KafkaServer.scala
@@ -154,6 +154,16 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
   private var _clusterId: String = null
   private var _brokerTopicStats: BrokerTopicStats = null
 
+  private var healthCheckScheduler: KafkaScheduler = null
+
+  private def haltIfNotHealthy() {
+    // This relies on io-thread to receive request from RequestChannel with 300 ms timeout, so that lastDequeueTimeMs
+    // will keep increasing even if there is no incoming request
+    if (time.milliseconds - socketServer.dataPlaneRequestChannel.lastDequeueTimeMs > config.requestMaxLocalTimeMs) {
+      fatal(s"It has been more than ${config.requestMaxLocalTimeMs} ms since the last time any io-thread reads from RequestChannel. Shutdown broker now.")
+      Runtime.getRuntime.halt(1)
+    }
+  }
 
   def clusterId: String = _clusterId
 
@@ -269,6 +279,13 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
         val brokerInfo = createBrokerInfo
         val brokerEpoch = zkClient.registerBroker(brokerInfo)
 
+        healthCheckScheduler = new KafkaScheduler(threads = 1, threadNamePrefix = "kafka-healthcheck-scheduler-")
+        healthCheckScheduler.startup()
+        healthCheckScheduler.schedule(name = "halt-broker-if-not-healthy",
+                                      fun = haltIfNotHealthy,
+                                      period = 10000,
+                                      unit = TimeUnit.MILLISECONDS)
+
         // Now that the broker is successfully registered, checkpoint its metadata
         checkpointBrokerMetadata(BrokerMetadata(config.brokerId, Some(clusterId)))
 
@@ -603,6 +620,9 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
         CoreUtils.swallow(controlledShutdown(), this)
         brokerState.newState(BrokerShuttingDown)
 
+        if (healthCheckScheduler != null)
+          healthCheckScheduler.shutdown()
+
         if (dynamicConfigManager != null)
           CoreUtils.swallow(dynamicConfigManager.shutdown(), this)
 

From 634f4ffe8da5b993b3286bfefe56f2e8fe74a7cb Mon Sep 17 00:00:00 2001
From: rrosenbl 
Date: Sat, 19 Aug 2017 13:04:06 -0700
Subject: [PATCH 0987/1071] [LI-HOTFIX] LIKAFKA-11492: capture heap dump on
 healthcheck suicide TICKET = LIKAFKA-11492 LI_DESCRIPTION = captures a heap
 dump and dies if kafka broker fails to do any IO activity for a configurable
 amount of time

EXIT_CRITERIA = MANUAL ["likely never"]
---
 checkstyle/import-control.xml                 |   3 +-
 .../apache/kafka/common/utils/PoisonPill.java | 108 ++++++++++++++++++
 .../main/scala/kafka/server/KafkaConfig.scala |  12 ++
 .../main/scala/kafka/server/KafkaServer.scala |   4 +-
 .../unit/kafka/server/KafkaConfigTest.scala   |   2 +
 5 files changed, 126 insertions(+), 3 deletions(-)
 create mode 100644 clients/src/main/java/org/apache/kafka/common/utils/PoisonPill.java

diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml
index 6e52bf056bc19..1b13be58ecf89 100644
--- a/checkstyle/import-control.xml
+++ b/checkstyle/import-control.xml
@@ -161,6 +161,7 @@
 
     
       
+      
     
   
 
@@ -241,7 +242,7 @@
     
       
     
-    
+
     
       
       
diff --git a/clients/src/main/java/org/apache/kafka/common/utils/PoisonPill.java b/clients/src/main/java/org/apache/kafka/common/utils/PoisonPill.java
new file mode 100644
index 0000000000000..a5f334f4c6b10
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/utils/PoisonPill.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.utils;
+
+import com.sun.management.HotSpotDiagnosticMXBean;
+import java.io.File;
+import java.lang.management.ManagementFactory;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import javax.management.MBeanServer;
+
+
+public class PoisonPill {
+
+    public static void die() {
+        die(null, -1);
+    }
+
+    public static void die(File heapDumpFolder, final long maxWaitForDump) {
+        try {
+            if (maxWaitForDump > 0 && heapDumpFolder != null) {
+                grabHeapDump(heapDumpFolder, maxWaitForDump);
+            }
+        } catch (Exception e) {
+            System.err.println("unable to complete heap dump");
+            e.printStackTrace(System.err);
+            System.err.flush();
+        } finally {
+            Runtime.getRuntime().halt(1);
+        }
+    }
+
+    private static void grabHeapDump(File heapDumpFolder, final long maxWait) throws Exception {
+
+        //set up a watchdog background thread that will halt in ~maxWait regardless of whether or not
+        //we succeed in taking a heap dump (since we dont know when it'll ever complete)
+        final CountDownLatch latch = new CountDownLatch(1);
+        Thread watchdog = new Thread(new Runnable() {
+            @Override
+            public void run() {
+                try {
+                    latch.countDown();
+                    Thread.sleep(maxWait);
+                    //at this point ~maxWait has passed since the call to die().
+                    //if the heap dump process completed successfully die() would
+                    //have called halt() and we wouldnt be here (99.99%)
+                    System.err.println("heap dump (probably) did not complete within timeout. halting.");
+                    System.err.flush();
+                } catch (Exception e) {
+                    System.err.println("watchdog caught exception");
+                    e.printStackTrace(System.err);
+                    System.err.flush();
+                } finally {
+                    Runtime.getRuntime().halt(1);
+                }
+            }
+        });
+        watchdog.setDaemon(true);
+        watchdog.setName("clark the death watchdog");
+        watchdog.start();
+
+        //make sure the watchdog is up and running before we go off attempting to dump
+        if (!latch.await(maxWait, TimeUnit.MILLISECONDS)) {
+            System.err.println("unable to start watchdog within timeout. will not proceed with dump");
+            System.err.flush();
+            return;
+        }
+
+        System.err.println("dumping heap to " + heapDumpFolder.getCanonicalPath());
+        System.err.flush();
+
+        //we dump into dump.inprogress and atomically rename it to be dump.complete
+        //(overwriting any previous such file). this attempts to guarantee there are
+        //at most 2 (potentially large) dump files at any point in time.
+
+        File inProgress = new File(heapDumpFolder, "dump.inprogress");
+        File complete = new File(heapDumpFolder, "dump.complete");
+        if (inProgress.exists() && !inProgress.delete()) {
+            System.err.println("unable to delete existing dump file. will not proceed with dump");
+            System.err.flush();
+            return;
+        }
+
+        MBeanServer server = ManagementFactory.getPlatformMBeanServer();
+        HotSpotDiagnosticMXBean diagnosticMBean =
+            ManagementFactory.newPlatformMXBeanProxy(server, "com.sun.management:type=HotSpotDiagnostic",
+                HotSpotDiagnosticMXBean.class);
+        diagnosticMBean.dumpHeap(inProgress.getCanonicalPath(), false /* disable only live - dump all objects */);
+        Files.move(inProgress.toPath(), complete.toPath(), StandardCopyOption.ATOMIC_MOVE,
+            StandardCopyOption.REPLACE_EXISTING);
+    }
+}
diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index bdd419abe2c86..5465d3b25a785 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -17,6 +17,7 @@
 
 package kafka.server
 
+import java.io.File
 import java.util
 import java.util.{Collections, Locale, Properties}
 
@@ -84,6 +85,9 @@ object Defaults {
   val RequestTimeoutMs = 30000
   val FailedAuthenticationDelayMs = 100
 
+  val HeapDumpFolder = "."
+  val HeapDumpTimeout = 30000
+
   /** ********* Log Configuration ***********/
   val NumPartitions = 1
   val LogDir = "/tmp/kafka-logs"
@@ -281,6 +285,8 @@ object KafkaConfig {
   val QueuedMaxRequestsProp = "queued.max.requests"
   val QueuedMaxBytesProp = "queued.max.request.bytes"
   val RequestTimeoutMsProp = CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG
+  val HeapDumpFolderProp = "heap.dump.folder"
+  val HeapDumpTimeoutProp = "heap.dump.timeout"
   /************* Authorizer Configuration ***********/
   val AuthorizerClassNameProp = "authorizer.class.name"
   /** ********* Socket Server Configuration ***********/
@@ -518,6 +524,8 @@ object KafkaConfig {
   val QueuedMaxRequestsDoc = "The number of queued requests allowed for data-plane, before blocking the network threads"
   val QueuedMaxRequestBytesDoc = "The number of queued bytes allowed before no more requests are read"
   val RequestTimeoutMsDoc = CommonClientConfigs.REQUEST_TIMEOUT_MS_DOC
+  val HeapDumpFolderDoc = "The Folder under which heap dumps will be written by the watchdog"
+  val HeapDumpTimeoutDoc = "The max amount of time (in millis) to wait for heap dump to complete before halting regardless"
   /************* Authorizer Configuration ***********/
   val AuthorizerClassNameDoc = s"The fully qualified name of a class that implements s${classOf[Authorizer].getName}" +
   " interface, which is used by the broker for authorization. This config also supports authorizers that implement the deprecated" +
@@ -880,6 +888,8 @@ object KafkaConfig {
       .define(QueuedMaxRequestsProp, INT, Defaults.QueuedMaxRequests, atLeast(1), HIGH, QueuedMaxRequestsDoc)
       .define(QueuedMaxBytesProp, LONG, Defaults.QueuedMaxRequestBytes, MEDIUM, QueuedMaxRequestBytesDoc)
       .define(RequestTimeoutMsProp, INT, Defaults.RequestTimeoutMs, HIGH, RequestTimeoutMsDoc)
+      .define(HeapDumpFolderProp, STRING, Defaults.HeapDumpFolder, LOW, HeapDumpFolderDoc)
+      .define(HeapDumpTimeoutProp, LONG, Defaults.HeapDumpTimeout, LOW, HeapDumpTimeoutDoc)
 
       /************* Authorizer Configuration ***********/
       .define(AuthorizerClassNameProp, STRING, Defaults.AuthorizerClassName, LOW, AuthorizerClassNameDoc)
@@ -1173,6 +1183,8 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO
   def numIoThreads = getInt(KafkaConfig.NumIoThreadsProp)
   def messageMaxBytes = getInt(KafkaConfig.MessageMaxBytesProp)
   val requestTimeoutMs = getInt(KafkaConfig.RequestTimeoutMsProp)
+  val heapDumpFolder = new File(getString(KafkaConfig.HeapDumpFolderProp))
+  val heapDumpTimeout = getLong(KafkaConfig.HeapDumpTimeoutProp)
 
   def getNumReplicaAlterLogDirsThreads: Int = {
     val numThreads: Integer = Option(getInt(KafkaConfig.NumReplicaAlterLogDirsThreadsProp)).getOrElse(logDirs.size)
diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala
index 729cafe3b720e..8c47d20a7846f 100755
--- a/core/src/main/scala/kafka/server/KafkaServer.scala
+++ b/core/src/main/scala/kafka/server/KafkaServer.scala
@@ -46,7 +46,7 @@ import org.apache.kafka.common.requests.{ControlledShutdownRequest, ControlledSh
 import org.apache.kafka.common.security.scram.internals.ScramMechanism
 import org.apache.kafka.common.security.token.delegation.internals.DelegationTokenCache
 import org.apache.kafka.common.security.{JaasContext, JaasUtils}
-import org.apache.kafka.common.utils.{AppInfoParser, LogContext, Time}
+import org.apache.kafka.common.utils.{AppInfoParser, LogContext, PoisonPill, Time}
 import org.apache.kafka.common.{ClusterResource, Endpoint, Node}
 import org.apache.kafka.server.authorizer.Authorizer
 
@@ -161,7 +161,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
     // will keep increasing even if there is no incoming request
     if (time.milliseconds - socketServer.dataPlaneRequestChannel.lastDequeueTimeMs > config.requestMaxLocalTimeMs) {
       fatal(s"It has been more than ${config.requestMaxLocalTimeMs} ms since the last time any io-thread reads from RequestChannel. Shutdown broker now.")
-      Runtime.getRuntime.halt(1)
+      PoisonPill.die(config.heapDumpFolder, config.heapDumpTimeout)
     }
   }
 
diff --git a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
index 2709ee6047014..39e791f9dd8d9 100755
--- a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
@@ -599,6 +599,8 @@ class KafkaConfigTest {
         case KafkaConfig.NumReplicaAlterLogDirsThreadsProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_number")
         case KafkaConfig.QueuedMaxBytesProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_number")
         case KafkaConfig.RequestTimeoutMsProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_number")
+        case KafkaConfig.HeapDumpFolderProp => //ignore string
+        case KafkaConfig.HeapDumpTimeoutProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_number")
 
         case KafkaConfig.AuthorizerClassNameProp => //ignore string
         case KafkaConfig.CreateTopicPolicyClassNameProp => //ignore string

From aedab73912351b3f33ac8ca04fe5b6b1071e9ced Mon Sep 17 00:00:00 2001
From: Kyle Ambroff 
Date: Wed, 7 Feb 2018 09:52:08 -0800
Subject: [PATCH 0988/1071] [LI-HOTFIX] KAFKA-6469 Batch ISR change
 notifications

TICKET = KAFKA-6469
LI_DESCRIPTION = This patch is required for clusters with a really
large number of partitions. Without it ISR change notifications will
become larger than the ZooKeeper max message size and the cluster will
brick itself.

KAFKA-6469 Batch ISR change notifications

We've failures in one of our test clusters as the partition count
started to climb north of 60k per broker. We had brokers writing child
nodes under /isr_change_notification that were larger than the
jute.maxbuffer size in ZooKeeper (1MB), causing the ZooKeeper server
to drop the controller's session, effectively bricking the cluster.

This can be mitigated by chunking ISR notifications to increase
the maximum number of partitions a broker can host, which is the
purpose of this patch.

ReplicaManager#maybePropagateIsrChanges() now batches the set of
TopicPartitions into sets of at most 3000 entries. This ensures that
the JSON payload is never more than around 820k at the worst case,
which leaves almost 200k of room for metadata in the ZooKeeper
CreateRequest.

A unit test was added which propagates a set of 5000 TopicPartitions
in two batches.

EXIT_CRITERIA = TICKET [KAFKA-6469]
---
 .../scala/kafka/server/ReplicaManager.scala   | 13 ++++--
 .../kafka/server/ReplicaManagerTest.scala     | 43 ++++++++++++++++++-
 2 files changed, 51 insertions(+), 5 deletions(-)

diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index c8380d937954e..b3d4e2e9a5cc5 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -211,6 +211,7 @@ class ReplicaManager(val config: KafkaConfig,
   this.logIdent = s"[ReplicaManager broker=$localBrokerId] "
   private val stateChangeLogger = new StateChangeLogger(localBrokerId, inControllerContext = false, None)
 
+  private val isrChangeNotificationBatchSize = 3000
   private val isrChangeSet: mutable.Set[TopicPartition] = new mutable.HashSet[TopicPartition]()
   private val lastIsrChangeMs = new AtomicLong(System.currentTimeMillis())
   private val lastIsrPropagationMs = new AtomicLong(System.currentTimeMillis())
@@ -281,23 +282,29 @@ class ReplicaManager(val config: KafkaConfig,
   def recordIsrChange(topicPartition: TopicPartition): Unit = {
     isrChangeSet synchronized {
       isrChangeSet += topicPartition
-      lastIsrChangeMs.set(System.currentTimeMillis())
+      lastIsrChangeMs.set(time.milliseconds())
     }
   }
+
   /**
    * This function periodically runs to see if ISR needs to be propagated. It propagates ISR when:
    * 1. There is ISR change not propagated yet.
    * 2. There is no ISR Change in the last five seconds, or it has been more than 60 seconds since the last ISR propagation.
    * This allows an occasional ISR change to be propagated within a few seconds, and avoids overwhelming controller and
    * other brokers when large amount of ISR change occurs.
+   *
+   * ISR changes are batched into chunks of isrChangeNotificationBatchSize TopicPartitions to avoid writing ZNodes which
+   * exceed the default jute.maxbuffer size of 1MB in ZooKeeper. With this batch size, the worst case data size is about
+   * 820k, which gives about 203k of head room for the rest of the ZooKeeper CreateRequest. Since we have no idea what
+   * ACLs or other metadata could be included we need to leave lots of extra room. See KAFKA-6469.
    */
   def maybePropagateIsrChanges(): Unit = {
-    val now = System.currentTimeMillis()
+    val now = time.milliseconds()
     isrChangeSet synchronized {
       if (isrChangeSet.nonEmpty &&
         (lastIsrChangeMs.get() + ReplicaManager.IsrChangePropagationBlackOut < now ||
           lastIsrPropagationMs.get() + ReplicaManager.IsrChangePropagationInterval < now)) {
-        zkClient.propagateIsrChanges(isrChangeSet)
+        isrChangeSet.grouped(isrChangeNotificationBatchSize).foreach(zkClient.propagateIsrChanges)
         isrChangeSet.clear()
         lastIsrPropagationMs.set(now)
       }
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index ffd3d3034b192..d605eff6b7b29 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -32,7 +32,7 @@ import kafka.server.epoch.util.ReplicaFetcherMockBlockingSend
 import kafka.utils.TestUtils.createBroker
 import kafka.utils.timer.MockTimer
 import kafka.utils.{MockScheduler, MockTime, TestUtils}
-import kafka.zk.KafkaZkClient
+import kafka.zk.{IsrChangeNotificationSequenceZNode, KafkaZkClient}
 import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
@@ -46,8 +46,9 @@ import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse
 import org.apache.kafka.common.requests.{EpochEndOffset, IsolationLevel, LeaderAndIsrRequest}
 import org.apache.kafka.common.security.auth.KafkaPrincipal
 import org.apache.kafka.common.utils.Time
+import org.apache.kafka.common.utils.Utils
 import org.apache.kafka.common.{Node, TopicPartition}
-import org.easymock.EasyMock
+import org.easymock.{Capture, EasyMock}
 import org.junit.Assert._
 import org.junit.{After, Before, Test}
 import org.mockito.Mockito
@@ -80,6 +81,44 @@ class ReplicaManagerTest {
     metrics.close()
   }
 
+  @Test
+  def testBatchIsrChanges() : Unit = {
+    val time = new org.apache.kafka.common.utils.MockTime(ReplicaManager.IsrChangePropagationInterval)
+
+    kafkaZkClient = EasyMock.createMock(classOf[KafkaZkClient])
+    val capturedIsrChangeNotifications: Capture[collection.Set[TopicPartition]] = EasyMock.newCapture()
+    EasyMock.expect(kafkaZkClient.propagateIsrChanges(EasyMock.capture(capturedIsrChangeNotifications)))
+        .andAnswer(() => {
+          // Ensure that each serialized ISR change notification is well under the 1MiB limit in ZooKeeper.
+          val serializedSize = {
+            IsrChangeNotificationSequenceZNode.encode(capturedIsrChangeNotifications.getValue).length
+          }
+
+          // Just make sure the size never exceeds around 900KiB.
+          assertTrue(serializedSize < (900 * 1024));
+        }).times(2)
+    EasyMock.replay(kafkaZkClient)
+
+    val props = TestUtils.createBrokerConfig(1, TestUtils.MockZkConnect)
+    val config = KafkaConfig.fromProps(props)
+    val mockLogMgr = TestUtils.createLogManager(config.logDirs.map(new File(_)))
+    val rm = new ReplicaManager(config, metrics, time, kafkaZkClient, new MockScheduler(time), mockLogMgr,
+      new AtomicBoolean(false), QuotaFactory.instantiate(config, metrics, time, ""), new BrokerTopicStats,
+      new MetadataCache(config.brokerId), new LogDirFailureChannel(config.logDirs.size))
+
+    // Topic name which is the maximum length of 249.
+    val largeTopicName = "topic-".padTo(249, "x").mkString
+
+    // 5000 partitions. Large partition numbers chosen to make the serialized values as long as possible.
+    (10000 to 15000)
+      .map(n => new TopicPartition(largeTopicName, n))
+      .foreach(rm.recordIsrChange)
+
+    // Trigger sending of ISR notifications.
+    rm.maybePropagateIsrChanges()
+    EasyMock.verify(kafkaZkClient)
+  }
+
   @Test
   def testHighWaterMarkDirectoryMapping(): Unit = {
     val props = TestUtils.createBrokerConfig(1, TestUtils.MockZkConnect)

From b7fc450f4953e613f799b7874a5734d0698952ad Mon Sep 17 00:00:00 2001
From: Navina Ramesh 
Date: Thu, 14 Jun 2018 16:57:39 -0700
Subject: [PATCH 0989/1071] [LI-HOTFIX] Allow poison pill methods to take an
 exit status code to use on halt RB=1340022 TICKET = LI_DESCRIPTION = Allows
 failing process (in this case, KafkaServer.scala) to generate heap dump and
 exit status code, before exiting. I think this codepath is also used by KMM
 (venice).

EXIT_CRITERIA = MANUAL ["As long as we use KafkaServer.scala"]
---
 .../org/apache/kafka/common/utils/PoisonPill.java  | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/utils/PoisonPill.java b/clients/src/main/java/org/apache/kafka/common/utils/PoisonPill.java
index a5f334f4c6b10..b98853762ed60 100644
--- a/clients/src/main/java/org/apache/kafka/common/utils/PoisonPill.java
+++ b/clients/src/main/java/org/apache/kafka/common/utils/PoisonPill.java
@@ -29,24 +29,28 @@
 public class PoisonPill {
 
     public static void die() {
-        die(null, -1);
+        die(null, -1, 1);
     }
 
     public static void die(File heapDumpFolder, final long maxWaitForDump) {
+        die(heapDumpFolder, maxWaitForDump, 1);
+    }
+
+    public static void die(File heapDumpFolder, final long maxWaitForDump, int haltStatusCode) {
         try {
             if (maxWaitForDump > 0 && heapDumpFolder != null) {
-                grabHeapDump(heapDumpFolder, maxWaitForDump);
+                grabHeapDump(heapDumpFolder, maxWaitForDump, haltStatusCode);
             }
         } catch (Exception e) {
             System.err.println("unable to complete heap dump");
             e.printStackTrace(System.err);
             System.err.flush();
         } finally {
-            Runtime.getRuntime().halt(1);
+            Runtime.getRuntime().halt(haltStatusCode);
         }
     }
 
-    private static void grabHeapDump(File heapDumpFolder, final long maxWait) throws Exception {
+    private static void grabHeapDump(File heapDumpFolder, final long maxWait, final int haltStatusCode) throws Exception {
 
         //set up a watchdog background thread that will halt in ~maxWait regardless of whether or not
         //we succeed in taking a heap dump (since we dont know when it'll ever complete)
@@ -67,7 +71,7 @@ public void run() {
                     e.printStackTrace(System.err);
                     System.err.flush();
                 } finally {
-                    Runtime.getRuntime().halt(1);
+                    Runtime.getRuntime().halt(haltStatusCode);
                 }
             }
         });

From f14e0bf7d8b5cd1e5622b66ac9c08a55226707e7 Mon Sep 17 00:00:00 2001
From: Dong Lin 
Date: Tue, 9 Jan 2018 13:32:24 -0800
Subject: [PATCH 0990/1071] [LI-HOTFIX] LIKAFKA-9486 followup; Initialize
 lastDequeueTimeMs to be Long.MaxValue TICKET = LI_DESCRIPTION =

RB=1137541
G=Kafka-Code-Reviews
A=jkoshy

EXIT_CRITERIA = MANUAL ["NONE"]
---
 core/src/main/scala/kafka/network/RequestChannel.scala | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala
index e44edba4b0cb4..542b82e00d4b5 100644
--- a/core/src/main/scala/kafka/network/RequestChannel.scala
+++ b/core/src/main/scala/kafka/network/RequestChannel.scala
@@ -280,7 +280,9 @@ class RequestChannel(val queueSize: Int, val metricNamePrefix : String, val time
   val requestQueueSizeMetricName = metricNamePrefix.concat(RequestQueueSizeMetric)
   val responseQueueSizeMetricName = metricNamePrefix.concat(ResponseQueueSizeMetric)
 
-  @volatile var lastDequeueTimeMs = time.milliseconds
+  // Set this to Long.Maxvalue so that KafkaHealthCheck will not shutdown broker if it
+  // reads lastDequeueTimeMs before lastDequeueTimeMs is updated by any KafkaRequestHandler thread.
+  @volatile var lastDequeueTimeMs = Long.MaxValue
   // This metric can help user select a suitable threshold for requestMaxLocalTimeMs so that broker can shutdown itself only when it
   // is stuck or too slow. A suggested value of requestMaxLocalTimeMs could be twice the 999'th percentile of the RequestDequeuePollIntervalMs.
   private val requestDequeuePollIntervalMs = newHistogram("RequestDequeuePollIntervalMs")

From 638d13c2ae005d5f1d4b5dc6fa3d9aae0b1d4303 Mon Sep 17 00:00:00 2001
From: Kun Du 
Date: Fri, 25 May 2018 17:13:36 -0700
Subject: [PATCH 0991/1071] [LI-HOTFIX] Add bounded flush to KafkaProducer.
 TICKET = KAFKA-7711 LI_DESCRIPTION =  Add a bounded flush() API and timeout
 if producer is unable to flush all the batch records in a limited time.

EXIT_CRITERIA = TICKET [KAFKA-7711]
---
 .../kafka/clients/producer/KafkaProducer.java | 24 ++++++++++++---
 .../kafka/clients/producer/MockProducer.java  |  9 ++++--
 .../kafka/clients/producer/Producer.java      |  5 ++++
 .../producer/internals/RecordAccumulator.java | 19 +++++++++---
 .../internals/RecordAccumulatorTest.java      | 22 ++++++++++++--
 .../kafka/api/BaseProducerSendTest.scala      | 30 +++++++++++++++++++
 6 files changed, 97 insertions(+), 12 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
index 78cdc8cc67057..8d0c224ea7172 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
@@ -1103,11 +1103,27 @@ private void ensureValidRecordSize(int size) {
      */
     @Override
     public void flush() {
-        log.trace("Flushing accumulated records in producer.");
-        this.accumulator.beginFlush();
-        this.sender.wakeup();
+        flush(Integer.MAX_VALUE, TimeUnit.MILLISECONDS);
+    }
+
+    /**
+     * This method waits up to timeout for the producer to send out all the buffered records.
+     * @param timeout The maximum time to wait for producer to complete. The value should be non-negative.
+     * @param unit The time unit for the timeout
+     * @throws TimeoutException If producer fail to finish in time
+     * @throws InterruptException If the thread is interrupted while blocked
+     * @throws IllegalArgumentException If the timeout is negative.
+     */
+    @Override
+    public void flush(long timeout, TimeUnit unit) {
+        if (timeout < 0)
+            throw new IllegalArgumentException("The timeout cannot be negative.");
         try {
-            this.accumulator.awaitFlushCompletion();
+            this.accumulator.beginFlush();
+            this.sender.wakeup();
+            this.accumulator.awaitFlushCompletion(unit.toMillis(timeout));
+        } catch (TimeoutException e) {
+            throw e;
         } catch (InterruptedException e) {
             throw new InterruptException("Flush interrupted.", e);
         }
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/MockProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/MockProducer.java
index c2561cfc79b69..cddcc2a463f58 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/MockProducer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/MockProducer.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.clients.producer;
 
+import java.util.concurrent.TimeUnit;
 import org.apache.kafka.clients.consumer.OffsetAndMetadata;
 import org.apache.kafka.clients.producer.internals.DefaultPartitioner;
 import org.apache.kafka.clients.producer.internals.FutureRecordMetadata;
@@ -229,7 +230,7 @@ private void verifyNoTransactionInFlight() {
 
     /**
      * Adds the record to the list of sent records. The {@link RecordMetadata} returned will be immediately satisfied.
-     * 
+     *
      * @see #history()
      */
     @Override
@@ -289,12 +290,16 @@ private long nextOffset(TopicPartition tp) {
         }
     }
 
-    public synchronized void flush() {
+    public synchronized void flush(long timeout, TimeUnit unit) {
         verifyProducerState();
         while (!this.completions.isEmpty())
             completeNext();
     }
 
+    public synchronized void flush() {
+        flush(Long.MAX_VALUE, null);
+    }
+
     public List partitionsFor(String topic) {
         return this.cluster.partitionsForTopic(topic);
     }
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java b/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java
index 96d487a79235b..0e923f09d7b4e 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java
@@ -79,6 +79,11 @@ void sendOffsetsToTransaction(Map offsets,
     void flush();
 
     /**
+     * See {@link KafkaProducer#flush(long, TimeUnit)}
+     */
+    void flush(long timeout, TimeUnit unit);
+
+     /**
      * See {@link KafkaProducer#partitionsFor(String)}
      */
     List partitionsFor(String topic);
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
index d002fc4061b63..e5d8ec32f2c15 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
@@ -29,6 +29,7 @@
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.TimeUnit;
 import org.apache.kafka.clients.ApiVersions;
 import org.apache.kafka.clients.producer.Callback;
 import org.apache.kafka.common.Cluster;
@@ -37,6 +38,7 @@
 import org.apache.kafka.common.Node;
 import org.apache.kafka.common.PartitionInfo;
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.TimeoutException;
 import org.apache.kafka.common.errors.UnsupportedVersionException;
 import org.apache.kafka.common.header.Header;
 import org.apache.kafka.common.metrics.Measurable;
@@ -703,12 +705,21 @@ private boolean appendsInProgress() {
     }
 
     /**
-     * Mark all partitions as ready to send and block until the send is complete
+     * Mark all partitions as ready to send and block until the send is complete or time expires
      */
-    public void awaitFlushCompletion() throws InterruptedException {
+    public void awaitFlushCompletion(long timeoutMs) throws InterruptedException {
         try {
-            for (ProducerBatch batch : this.incomplete.copyAll())
-                batch.produceFuture.await();
+            Long expireMs = System.currentTimeMillis() + timeoutMs;
+            for (ProducerBatch batch : this.incomplete.copyAll()) {
+                Long currentMs = System.currentTimeMillis();
+                if (currentMs > expireMs) {
+                    throw new TimeoutException("Failed to flush accumulated records within" + timeoutMs + "milliseconds.");
+                }
+                boolean completed = batch.produceFuture.await(Math.max(expireMs - currentMs, 0), TimeUnit.MILLISECONDS);
+                if (!completed) {
+                    throw new TimeoutException("Failed to flush accumulated records within" + timeoutMs + "milliseconds.");
+                }
+            }
         } finally {
             this.flushesInProgress.decrementAndGet();
         }
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java
index b04e5ddcfe530..c1a63c92e5f23 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java
@@ -26,6 +26,7 @@
 import org.apache.kafka.common.Node;
 import org.apache.kafka.common.PartitionInfo;
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.TimeoutException;
 import org.apache.kafka.common.errors.UnsupportedVersionException;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.protocol.ApiKeys;
@@ -398,7 +399,7 @@ public void testFlush() throws Exception {
                 accum.deallocate(batch);
 
         // should be complete with no unsent records.
-        accum.awaitFlushCompletion();
+        accum.awaitFlushCompletion(Integer.MAX_VALUE);
         assertFalse(accum.hasUndrained());
         assertFalse(accum.hasIncomplete());
     }
@@ -424,13 +425,30 @@ public void testAwaitFlushComplete() throws Exception {
         assertTrue(accum.flushInProgress());
         delayedInterrupt(Thread.currentThread(), 1000L);
         try {
-            accum.awaitFlushCompletion();
+            accum.awaitFlushCompletion(Integer.MAX_VALUE);
             fail("awaitFlushCompletion should throw InterruptException");
         } catch (InterruptedException e) {
             assertFalse("flushInProgress count should be decremented even if thread is interrupted", accum.flushInProgress());
         }
     }
 
+    @Test
+    public void testAwaitFlushTimeout() throws Exception {
+        RecordAccumulator accum = createTestRecordAccumulator(
+             4 * 1024 + DefaultRecordBatch.RECORD_BATCH_OVERHEAD, 64 * 1024, CompressionType.NONE, Integer.MAX_VALUE);
+        accum.append(new TopicPartition(topic, 0), 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false);
+
+        accum.beginFlush();
+        assertTrue(accum.flushInProgress());
+        try {
+            accum.awaitFlushCompletion(100);
+            fail("testAwaitFlushTimeout should throw TimeoutException");
+        } catch (TimeoutException e) {
+            assertFalse("flushInProgress count should be decremented even if flush timeout expires", accum.flushInProgress());
+        }
+    }
+
+
     @Test
     public void testAbortIncompleteBatches() throws Exception {
         int lingerMs = Integer.MAX_VALUE;
diff --git a/core/src/test/scala/integration/kafka/api/BaseProducerSendTest.scala b/core/src/test/scala/integration/kafka/api/BaseProducerSendTest.scala
index 9ae095372aa40..c4979ec6a5d8b 100644
--- a/core/src/test/scala/integration/kafka/api/BaseProducerSendTest.scala
+++ b/core/src/test/scala/integration/kafka/api/BaseProducerSendTest.scala
@@ -433,6 +433,36 @@ abstract class BaseProducerSendTest extends KafkaServerTestHarness {
       producer.close()
     }
   }
+  /**
+  * Test that flush return with TimeoutException when producer is unable to finish sending buffered records in time.
+*/
+  @Test
+  def testBoundedFlush() {
+    val producer = createProducer(brokerList)
+    createTopic(topic, 2, 2)
+    try {
+      producer.send(new ProducerRecord(topic, null, "value1".getBytes())).get()
+      try {
+        killBroker(0)
+        } catch {
+        case _: Throwable =>
+        }
+      try {
+        killBroker(1)
+        } catch {
+        case _: Throwable =>
+        }
+      producer.send(new ProducerRecord(topic, null, "value2".getBytes()))
+      try {
+        producer.flush(1000, TimeUnit.MILLISECONDS)
+        fail("TimeoutException should have thrown")
+        } catch {
+        case _: TimeoutException =>
+        }
+      }finally {
+      producer.close(1000, TimeUnit.MILLISECONDS)
+      }
+    }
 
   /**
    * Test close with zero timeout from caller thread

From 5b404c71024f662384836487a7823e4d2a60772c Mon Sep 17 00:00:00 2001
From: Alex Wang 
Date: Thu, 31 May 2018 15:53:46 -0700
Subject: [PATCH 0992/1071] [LI-HOTFIX] Added controller sensors to measure
 queue and remote time for LeaderAndIsr, Update_metadata, Stop_replica
 requests TICKET = LI_DESCRIPTION = Add controller sensors for requests queue
 and remote times. Added kafka sensors to display LeaderAndIsr, Stop_replica,
 and Update_metadata request times (queue time, remote time) on InGraph.
 Displaying time for 999, 99, 75, 50, Avg, Max percentiles.

RB=1326056
G=kafka-reviewers
R=dolin,luwang
A=dolin,luwang

EXIT_CRITERIA = MANUAL [""]
---
 .../controller/ControllerChannelManager.scala | 49 +++++++++++++++++--
 1 file changed, 44 insertions(+), 5 deletions(-)

diff --git a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
index 375f0d34f7ab4..152743d197b0f 100755
--- a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
+++ b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
@@ -39,7 +39,7 @@ import org.apache.kafka.common.{KafkaException, Node, Reconfigurable, TopicParti
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable.{HashMap, ListBuffer}
-import scala.collection.{Seq, Set, mutable}
+import scala.collection.{Seq, Map, Set, mutable}
 
 object ControllerChannelManager {
   val QueueSizeMetricName = "QueueSize"
@@ -57,7 +57,7 @@ class ControllerChannelManager(controllerContext: ControllerContext,
   protected val brokerStateInfo = new HashMap[Int, ControllerBrokerStateInfo]
   private val brokerLock = new Object
   this.logIdent = "[Channel manager on controller " + config.brokerId + "]: "
-
+  val brokerResponseSensors: mutable.Map[ApiKeys, BrokerResponseTimeStats] = mutable.HashMap.empty
   newGauge(
     "TotalQueueSize",
     new Gauge[Int] {
@@ -73,12 +73,27 @@ class ControllerChannelManager(controllerContext: ControllerContext,
     brokerLock synchronized {
       brokerStateInfo.foreach(brokerState => startRequestSendThread(brokerState._1))
     }
+    initBrokerResponseSensors()
   }
 
   def shutdown() = {
     brokerLock synchronized {
       brokerStateInfo.values.toList.foreach(removeExistingBroker)
     }
+    removeBrokerResponseSensors()
+  }
+
+  def initBrokerResponseSensors(): Unit = {
+    Array(ApiKeys.STOP_REPLICA, ApiKeys.LEADER_AND_ISR, ApiKeys.UPDATE_METADATA).foreach { k: ApiKeys =>
+      brokerResponseSensors.put(k, new BrokerResponseTimeStats(k))
+    }
+  }
+
+  def removeBrokerResponseSensors(): Unit = {
+    brokerResponseSensors.keySet.foreach { k: ApiKeys =>
+      brokerResponseSensors(k).removeMetrics()
+      brokerResponseSensors.remove(k)
+    }
   }
 
   def sendRequest(brokerId: Int, request: AbstractControlRequest.Builder[_ <: AbstractControlRequest],
@@ -172,7 +187,7 @@ class ControllerChannelManager(controllerContext: ControllerContext,
     )
 
     val requestThread = new RequestSendThread(config.brokerId, controllerContext, messageQueue, networkClient,
-      brokerNode, config, time, requestRateAndQueueTimeMetrics, stateChangeLogger, threadName)
+      brokerNode, config, time, requestRateAndQueueTimeMetrics, stateChangeLogger, threadName, this)
     requestThread.setDaemon(false)
 
     val queueSizeGauge = newGauge(
@@ -226,7 +241,8 @@ class RequestSendThread(val controllerId: Int,
                         val time: Time,
                         val requestRateAndQueueTimeMetrics: Timer,
                         val stateChangeLogger: StateChangeLogger,
-                        name: String)
+                        name: String,
+                        val controllerChannelManager: ControllerChannelManager)
   extends ShutdownableThread(name = name) {
 
   logIdent = s"[RequestSendThread controllerId=$controllerId] "
@@ -238,7 +254,9 @@ class RequestSendThread(val controllerId: Int,
     def backoff(): Unit = pause(100, TimeUnit.MILLISECONDS)
 
     val QueueItem(apiKey, requestBuilder, callback, enqueueTimeMs) = queue.take()
-    requestRateAndQueueTimeMetrics.update(time.milliseconds() - enqueueTimeMs, TimeUnit.MILLISECONDS)
+    var queueTimeMs = time.milliseconds() - enqueueTimeMs
+    var remoteTimeMs: Long = 0
+    requestRateAndQueueTimeMetrics.update(queueTimeMs, TimeUnit.MILLISECONDS)
 
     var clientResponse: ClientResponse = null
     try {
@@ -256,6 +274,7 @@ class RequestSendThread(val controllerId: Int,
               time.milliseconds(), true)
             clientResponse = NetworkClientUtils.sendAndReceive(networkClient, clientRequest, time)
             isSendSuccessful = true
+            remoteTimeMs = time.milliseconds() - enqueueTimeMs - queueTimeMs
           }
         } catch {
           case e: Throwable => // if the send was not successful, reconnect to broker and resend the message
@@ -281,6 +300,7 @@ class RequestSendThread(val controllerId: Int,
         if (callback != null) {
           callback(response)
         }
+        controllerChannelManager.brokerResponseSensors(api).update(queueTimeMs, remoteTimeMs)
       }
     } catch {
       case e: Throwable =>
@@ -603,3 +623,22 @@ case class ControllerBrokerStateInfo(networkClient: NetworkClient,
                                      requestRateAndTimeMetrics: Timer,
                                      reconfigurableChannelBuilder: Option[Reconfigurable])
 
+
+class BrokerResponseTimeStats(val key: ApiKeys) extends KafkaMetricsGroup {
+  // Records time for request waits on local send thread queue
+  val brokerRequestQueueTime = newHistogram("brokerRequestQueueTimeMs", true, responseTimeTags)
+  // Records time for controller to send request and receive response
+  val brokerRequestRemoteTime = newHistogram("brokerRequestRemoteTimeMs", true, responseTimeTags)
+
+  def responseTimeTags = Map("request" -> key.toString)
+
+  def update(queueTime: Long, remoteTime: Long): Unit = {
+    brokerRequestQueueTime.update(queueTime)
+    brokerRequestRemoteTime.update(remoteTime)
+  }
+
+  def removeMetrics(): Unit = {
+    removeMetric("brokerRequestQueueTimeMs", responseTimeTags)
+    removeMetric("brokerRequestRemoteTimeMs", responseTimeTags)
+  }
+}

From 1e600f3a1f9d00c35d53a44327e0587f914a29a0 Mon Sep 17 00:00:00 2001
From: Ke Hu 
Date: Tue, 19 Jun 2018 10:04:05 -0700
Subject: [PATCH 0993/1071] [LI-HOTFIX] Add multiple config defaults for
 __consumer_offsets topic creation TICKET = LI_DESCRIPTION =

Add max.message.bytes, min.insync.replicas, min.compaction.lag.ms config defaults for __consumer_offsets topic during create time.

RB=1342860
G=Kafka-Code-Reviews
R=okaraman
A=okaraman

EXIT_CRITERIA = MANUAL ["None"]
---
 .../coordinator/group/GroupCoordinator.scala      |  9 ++++++++-
 .../kafka/coordinator/group/OffsetConfig.scala    | 11 ++++++++++-
 .../src/main/scala/kafka/server/KafkaConfig.scala | 15 +++++++++++++++
 .../scala/unit/kafka/server/KafkaConfigTest.scala |  3 +++
 4 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
index 5f812b10dc685..c877fb5982172 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
@@ -94,6 +94,9 @@ class GroupCoordinator(val brokerId: Int,
     props.put(LogConfig.CleanupPolicyProp, LogConfig.Compact)
     props.put(LogConfig.SegmentBytesProp, offsetConfig.offsetsTopicSegmentBytes.toString)
     props.put(LogConfig.CompressionTypeProp, ProducerCompressionCodec.name)
+    props.put(LogConfig.MaxMessageBytesProp, offsetConfig.offsetsTopicMaxMessageBytes.toString)
+    props.put(LogConfig.MinInSyncReplicasProp, offsetConfig.offsetsTopicMinInSyncReplicas.toString)
+    props.put(LogConfig.MinCompactionLagMsProp, offsetConfig.offsetsTopicMinCompactionLagMs.toString)
     props
   }
 
@@ -1206,7 +1209,11 @@ object GroupCoordinator {
     offsetsTopicReplicationFactor = config.offsetsTopicReplicationFactor,
     offsetsTopicCompressionCodec = config.offsetsTopicCompressionCodec,
     offsetCommitTimeoutMs = config.offsetCommitTimeoutMs,
-    offsetCommitRequiredAcks = config.offsetCommitRequiredAcks
+    offsetCommitRequiredAcks = config.offsetCommitRequiredAcks,
+    offsetsTopicMaxMessageBytes = config.offsetsTopicMaxMessageBytes,
+    offsetsTopicMinInSyncReplicas = config.offsetsTopicMinInSyncReplicas,
+    offsetsTopicMinCompactionLagMs = config.offsetsTopicMinCompactionLagMs
+
   )
 
   def apply(config: KafkaConfig,
diff --git a/core/src/main/scala/kafka/coordinator/group/OffsetConfig.scala b/core/src/main/scala/kafka/coordinator/group/OffsetConfig.scala
index 55ec590852cd0..f4a22d4da6ddf 100644
--- a/core/src/main/scala/kafka/coordinator/group/OffsetConfig.scala
+++ b/core/src/main/scala/kafka/coordinator/group/OffsetConfig.scala
@@ -36,6 +36,9 @@ import kafka.message.{CompressionCodec, NoCompressionCodec}
  *                              commit or this timeout is reached. (Similar to the producer request timeout.)
  * @param offsetCommitRequiredAcks The required acks before the commit can be accepted. In general, the default (-1)
  *                                 should not be overridden.
+ * @param offsetsTopicMaxMessageBytes The maximum record batch size for the offset commit topic
+ * @param offsetsTopicMinInSyncReplicas The minimum number of replicas that must acknowledged a write for the write to be considered successful
+ * @param offsetsTopicMinCompactionLagMs The minimum time a message will stay un-compacted in the log
  */
 case class OffsetConfig(maxMetadataSize: Int = OffsetConfig.DefaultMaxMetadataSize,
                         loadBufferSize: Int = OffsetConfig.DefaultLoadBufferSize,
@@ -46,7 +49,10 @@ case class OffsetConfig(maxMetadataSize: Int = OffsetConfig.DefaultMaxMetadataSi
                         offsetsTopicReplicationFactor: Short = OffsetConfig.DefaultOffsetsTopicReplicationFactor,
                         offsetsTopicCompressionCodec: CompressionCodec = OffsetConfig.DefaultOffsetsTopicCompressionCodec,
                         offsetCommitTimeoutMs: Int = OffsetConfig.DefaultOffsetCommitTimeoutMs,
-                        offsetCommitRequiredAcks: Short = OffsetConfig.DefaultOffsetCommitRequiredAcks)
+                        offsetCommitRequiredAcks: Short = OffsetConfig.DefaultOffsetCommitRequiredAcks,
+                        offsetsTopicMaxMessageBytes: Int = OffsetConfig.DefaultOffsetsTopicMaxMessageBytes,
+                        offsetsTopicMinInSyncReplicas: Int = OffsetConfig.DefaultOffsetsTopicMinInSyncReplicas,
+                        offsetsTopicMinCompactionLagMs: Long = OffsetConfig.DefaultOffsetsTopicMinCompactionLagMs)
 
 object OffsetConfig {
   val DefaultMaxMetadataSize = 4096
@@ -59,4 +65,7 @@ object OffsetConfig {
   val DefaultOffsetsTopicCompressionCodec = NoCompressionCodec
   val DefaultOffsetCommitTimeoutMs = 5000
   val DefaultOffsetCommitRequiredAcks = (-1).toShort
+  val DefaultOffsetsTopicMaxMessageBytes = 20 * 1024 * 1024
+  val DefaultOffsetsTopicMinInSyncReplicas = 1
+  val DefaultOffsetsTopicMinCompactionLagMs = 0L
 }
\ No newline at end of file
diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index 5465d3b25a785..bcdaa62d61ee1 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -175,6 +175,9 @@ object Defaults {
   val OffsetsRetentionCheckIntervalMs: Long = OffsetConfig.DefaultOffsetsRetentionCheckIntervalMs
   val OffsetCommitTimeoutMs = OffsetConfig.DefaultOffsetCommitTimeoutMs
   val OffsetCommitRequiredAcks = OffsetConfig.DefaultOffsetCommitRequiredAcks
+  val OffsetsTopicMaxMessageBytes = OffsetConfig.DefaultOffsetsTopicMaxMessageBytes
+  val OffsetsTopicMinInSyncReplicas = OffsetConfig.DefaultOffsetsTopicMinInSyncReplicas
+  val OffsetsTopicMinCompactionLagMs = OffsetConfig.DefaultOffsetsTopicMinCompactionLagMs
 
   /** ********* Transaction management configuration ***********/
   val TransactionalIdExpirationMs = TransactionStateManager.DefaultTransactionalIdExpirationMs
@@ -402,6 +405,9 @@ object KafkaConfig {
   val OffsetsRetentionCheckIntervalMsProp = "offsets.retention.check.interval.ms"
   val OffsetCommitTimeoutMsProp = "offsets.commit.timeout.ms"
   val OffsetCommitRequiredAcksProp = "offsets.commit.required.acks"
+  val OffsetsTopicMaxMessageBytesProp = "offsets.topic.max.message.bytes"
+  val OffsetsTopicMinInSyncReplicasProp = "offsets.topic.min.insync.replicas"
+  val OffsetsTopicMinCompactionLagMsProp = "offsets.topic.min.compaction.lag.ms"
   /** ********* Transaction management configuration ***********/
   val TransactionalIdExpirationMsProp = "transactional.id.expiration.ms"
   val TransactionsMaxTimeoutMsProp = "transaction.max.timeout.ms"
@@ -743,6 +749,9 @@ object KafkaConfig {
   val OffsetCommitTimeoutMsDoc = "Offset commit will be delayed until all replicas for the offsets topic receive the commit " +
   "or this timeout is reached. This is similar to the producer request timeout."
   val OffsetCommitRequiredAcksDoc = "The required acks before the commit can be accepted. In general, the default (-1) should not be overridden"
+  val OffsetsTopicMaxMessageBytesDoc = "Overriden " + MessageMaxBytesProp + " config for the consumer_offset topic."
+  val OffsetsTopicMinInSyncReplicasDoc = "Overridden " + MinInSyncReplicasProp + " config for the consumer_offset topic."
+  val OffsetsTopicMinCompactionLagMsDoc = "Overridden " + LogCleanerMinCompactionLagMsProp + " config for the consumer_offset topic."
   /** ********* Transaction management configuration ***********/
   val TransactionalIdExpirationMsDoc = "The time in ms that the transaction coordinator will wait without receiving any transaction status updates " +
     "for the current transaction before expiring its transactional id. This setting also influences producer id expiration - producer ids are expired " + 
@@ -1014,6 +1023,9 @@ object KafkaConfig {
       .define(OffsetCommitRequiredAcksProp, SHORT, Defaults.OffsetCommitRequiredAcks, HIGH, OffsetCommitRequiredAcksDoc)
       .define(DeleteTopicEnableProp, BOOLEAN, Defaults.DeleteTopicEnable, HIGH, DeleteTopicEnableDoc)
       .define(CompressionTypeProp, STRING, Defaults.CompressionType, HIGH, CompressionTypeDoc)
+      .define(OffsetsTopicMaxMessageBytesProp, INT, Defaults.OffsetsTopicMaxMessageBytes, atLeast(0), HIGH, OffsetsTopicMaxMessageBytesDoc)
+      .define(OffsetsTopicMinInSyncReplicasProp, INT, Defaults.OffsetsTopicMinInSyncReplicas, atLeast(1), HIGH, OffsetsTopicMinInSyncReplicasDoc)
+      .define(OffsetsTopicMinCompactionLagMsProp, LONG, Defaults.OffsetsTopicMinCompactionLagMs, atLeast(0), HIGH, OffsetsTopicMinCompactionLagMsDoc)
 
       /** ********* Transaction management configuration ***********/
       .define(TransactionalIdExpirationMsProp, INT, Defaults.TransactionalIdExpirationMs, atLeast(1), HIGH, TransactionalIdExpirationMsDoc)
@@ -1315,6 +1327,9 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO
   val offsetCommitRequiredAcks = getShort(KafkaConfig.OffsetCommitRequiredAcksProp)
   val offsetsTopicSegmentBytes = getInt(KafkaConfig.OffsetsTopicSegmentBytesProp)
   val offsetsTopicCompressionCodec = Option(getInt(KafkaConfig.OffsetsTopicCompressionCodecProp)).map(value => CompressionCodec.getCompressionCodec(value)).orNull
+  val offsetsTopicMaxMessageBytes = getInt(KafkaConfig.OffsetsTopicMaxMessageBytesProp)
+  val offsetsTopicMinInSyncReplicas = getInt(KafkaConfig.OffsetsTopicMinInSyncReplicasProp)
+  val offsetsTopicMinCompactionLagMs = getLong(KafkaConfig.OffsetsTopicMinCompactionLagMsProp)
 
   /** ********* Transaction management configuration ***********/
   val transactionalIdExpirationMs = getInt(KafkaConfig.TransactionalIdExpirationMsProp)
diff --git a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
index 39e791f9dd8d9..fee636094277e 100755
--- a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
@@ -684,6 +684,9 @@ class KafkaConfigTest {
         case KafkaConfig.OffsetsRetentionCheckIntervalMsProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_number", "0")
         case KafkaConfig.OffsetCommitTimeoutMsProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_number", "0")
         case KafkaConfig.OffsetCommitRequiredAcksProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_number", "-2")
+        case KafkaConfig.OffsetsTopicMaxMessageBytesProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_number", "-1")
+        case KafkaConfig.OffsetsTopicMinInSyncReplicasProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_number", "0")
+        case KafkaConfig.OffsetsTopicMinCompactionLagMsProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_number", "-1")
         case KafkaConfig.TransactionalIdExpirationMsProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_number", "0", "-2")
         case KafkaConfig.TransactionsMaxTimeoutMsProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_number", "0", "-2")
         case KafkaConfig.TransactionsTopicMinISRProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_number", "0", "-2")

From 4e1a1addc9fc7964c06d93685558d135228425bc Mon Sep 17 00:00:00 2001
From: Jon Lee 
Date: Tue, 17 Jul 2018 12:58:44 -0700
Subject: [PATCH 0994/1071] [LI-HOTFIX] Log request quota metrics to
 kafka-quota-metrics.log TICKET = LI_DESCRIPTION =

RB=1362648
BUG=LIKAFKA-18357
G=Kafka-Code-Reviews
R=dolin
A=dolin

EXIT_CRITERIA = MANUAL [""]
---
 core/src/main/scala/kafka/server/ClientQuotaManager.scala    | 5 +++--
 .../main/scala/kafka/server/ClientRequestQuotaManager.scala  | 4 +++-
 core/src/main/scala/kafka/server/QuotaFactory.scala          | 2 +-
 .../scala/unit/kafka/server/ClientQuotaManagerTest.scala     | 2 +-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/core/src/main/scala/kafka/server/ClientQuotaManager.scala b/core/src/main/scala/kafka/server/ClientQuotaManager.scala
index 4158298606023..dd461a0a40912 100644
--- a/core/src/main/scala/kafka/server/ClientQuotaManager.scala
+++ b/core/src/main/scala/kafka/server/ClientQuotaManager.scala
@@ -220,8 +220,9 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig,
     metricsMap.foreach {
       case (metricName: MetricName, kafkaMetric: KafkaMetric) =>
         if (metricName.group().equals(quotaType.toString) &&
-          (metricName.name().equals("byte-rate") || metricName.name().equals("throttle-time")) &&
-          metricName.tags().containsKey("client-id")) {
+          (metricName.name().equals("byte-rate") || metricName.name().equals("throttle-time") ||
+            metricName.name().equals("request-time")) &&
+          (metricName.tags().containsKey("client-id") || metricName.tags().containsKey("user"))) {
           info("Metric name (" + metricName + ") has value (" + kafkaMetric.metricValue() + ")")
         }
     }
diff --git a/core/src/main/scala/kafka/server/ClientRequestQuotaManager.scala b/core/src/main/scala/kafka/server/ClientRequestQuotaManager.scala
index 1f8af13bb450c..d2e655189535b 100644
--- a/core/src/main/scala/kafka/server/ClientRequestQuotaManager.scala
+++ b/core/src/main/scala/kafka/server/ClientRequestQuotaManager.scala
@@ -19,6 +19,7 @@ package kafka.server
 import java.util.concurrent.TimeUnit
 
 import kafka.network.RequestChannel
+import kafka.utils.KafkaScheduler
 import org.apache.kafka.common.MetricName
 import org.apache.kafka.common.metrics._
 import org.apache.kafka.common.utils.Time
@@ -30,9 +31,10 @@ import scala.collection.JavaConverters._
 class ClientRequestQuotaManager(private val config: ClientQuotaManagerConfig,
                                 private val metrics: Metrics,
                                 private val time: Time,
+                                private val schedulerOpt: Option[KafkaScheduler],
                                 threadNamePrefix: String,
                                 quotaCallback: Option[ClientQuotaCallback])
-                                extends ClientQuotaManager(config, metrics, QuotaType.Request, time, None, threadNamePrefix, quotaCallback) {
+                                extends ClientQuotaManager(config, metrics, QuotaType.Request, time, schedulerOpt, threadNamePrefix, quotaCallback) {
   val maxThrottleTimeMs = TimeUnit.SECONDS.toMillis(this.config.quotaWindowSizeSeconds)
   def exemptSensor = getOrCreateSensor(exemptSensorName, exemptMetricName)
 
diff --git a/core/src/main/scala/kafka/server/QuotaFactory.scala b/core/src/main/scala/kafka/server/QuotaFactory.scala
index 78a265ac7e6a7..00951c6fabcb8 100644
--- a/core/src/main/scala/kafka/server/QuotaFactory.scala
+++ b/core/src/main/scala/kafka/server/QuotaFactory.scala
@@ -67,7 +67,7 @@ object QuotaFactory extends Logging {
     QuotaManagers(
       new ClientQuotaManager(clientFetchConfig(cfg), metrics, Fetch, time, schedulerOpt, threadNamePrefix, clientQuotaCallback),
       new ClientQuotaManager(clientProduceConfig(cfg), metrics, Produce, time, schedulerOpt, threadNamePrefix, clientQuotaCallback),
-      new ClientRequestQuotaManager(clientRequestConfig(cfg), metrics, time, threadNamePrefix, clientQuotaCallback),
+      new ClientRequestQuotaManager(clientRequestConfig(cfg), metrics, time, schedulerOpt, threadNamePrefix, clientQuotaCallback),
       new ReplicationQuotaManager(replicationConfig(cfg), metrics, LeaderReplication, time),
       new ReplicationQuotaManager(replicationConfig(cfg), metrics, FollowerReplication, time),
       new ReplicationQuotaManager(alterLogDirsReplicationConfig(cfg), metrics, AlterLogDirsReplication, time),
diff --git a/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala
index 9a86496fc178f..9b38ec2afeae8 100644
--- a/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala
@@ -417,7 +417,7 @@ class ClientQuotaManagerTest {
 
   @Test
   def testRequestPercentageQuotaViolation(): Unit = {
-    val quotaManager = new ClientRequestQuotaManager(config, metrics, time, "", None)
+    val quotaManager = new ClientRequestQuotaManager(config, metrics, time, None, "", None)
     quotaManager.updateQuota(Some("ANONYMOUS"), Some("test-client"), Some("test-client"), Some(Quota.upperBound(1)))
     val queueSizeMetric = metrics.metrics().get(metrics.metricName("queue-size", "Request", ""))
     def millisToPercent(millis: Double) = millis * 1000 * 1000 * ClientQuotaManagerConfig.NanosToPercentagePerSecond

From 2f685e1d1cdf4c8d8722315880d23df9deae5929 Mon Sep 17 00:00:00 2001
From: Navina Ramesh 
Date: Mon, 9 Jul 2018 10:45:35 -0700
Subject: [PATCH 0995/1071] [LI-HOTFIX] LIKAFKA-18568 : Adding new offset reset
 strategy - LICLOSEST (LI only) TICKET = LI_DESCRIPTION = New reset policy for
 consumers of topics affected by ULEs. Provides a trade-off between message
 loss and time to bootstrap the partition (aka availability).

EXIT_CRITERIA = MANUAL ["As long as we have large consumers like audit, mirror-makers etc using this feature and we allow ULEs for topics in our kafka clusters"]
---
 .../java/org/apache/kafka/clients/consumer/ConsumerConfig.java  | 2 +-
 .../org/apache/kafka/clients/consumer/OffsetResetStrategy.java  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
index d7ff73b9016ec..86988f0e0337e 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
@@ -109,7 +109,7 @@ public class ConsumerConfig extends AbstractConfig {
      * auto.offset.reset
      */
     public static final String AUTO_OFFSET_RESET_CONFIG = "auto.offset.reset";
-    public static final String AUTO_OFFSET_RESET_DOC = "What to do when there is no initial offset in Kafka or if the current offset does not exist any more on the server (e.g. because that data has been deleted): 
          • earliest: automatically reset the offset to the earliest offset
          • latest: automatically reset the offset to the latest offset
          • none: throw exception to the consumer if no previous offset is found for the consumer's group
          • anything else: throw exception to the consumer.
          ";
+    public static final String AUTO_OFFSET_RESET_DOC = "What to do when there is no initial offset in Kafka or if the current offset does not exist any more on the server (e.g. because that data has been deleted): 
          • earliest: automatically reset the offset to the earliest offset
          • latest: automatically reset the offset to the latest offset
          • liclosest: uses 'earliest' reset strategy when there is no committed offset or when fetched offset is smaller than Log Start Offset (LSO) and uses 'latest' reset strategy when the fetched offset is greater than Log End Offset (LEO)
          • none: throw exception to the caller if no previous offset is found for the consumer's group
            • <
          • anything else: throw exception to the caller.
          ";
 
     /**
      * fetch.min.bytes
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/OffsetResetStrategy.java b/clients/src/main/java/org/apache/kafka/clients/consumer/OffsetResetStrategy.java
index 6d742b850a134..1c2ddafcb3d0c 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/OffsetResetStrategy.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/OffsetResetStrategy.java
@@ -17,5 +17,5 @@
 package org.apache.kafka.clients.consumer;
 
 public enum OffsetResetStrategy {
-    LATEST, EARLIEST, NONE
+    LATEST, EARLIEST, NONE, LICLOSEST
 }

From e00929d647d9d081922fdf20ace568572e8ddaf4 Mon Sep 17 00:00:00 2001
From: Zhanxiang Huang 
Date: Mon, 13 Aug 2018 16:28:37 -0700
Subject: [PATCH 0996/1071] [LI-HOTFIX] Add topic deletion hook in zookeeper to
 trigger in-memory topic deletion flag changes: TICKET = LI_DESCRIPTION =

- Add TopicDeletionFlagPath `/topic_deletion_flag` to trigger changes in topic deletion enable flag in TopicDeletionManager.
- Controller will listen on this path when active and un-listen when resignation.
- If the znode is deleted, the topic deletion flag will be set according to the config.

RB=1306354
BUG=LIKAFKA-17224
G=Kafka-Code-Reviews
R=luwang
A=luwang

EXIT_CRITERIA = MANUAL ["After upstream makes delete.topic.enable a dynamic broker config"]
---
 .../kafka/controller/ControllerState.scala    |  6 +-
 .../kafka/controller/KafkaController.scala    | 43 +++++++++++++-
 .../controller/TopicDeletionManager.scala     | 38 +++++++++++-
 .../main/scala/kafka/zk/KafkaZkClient.scala   | 31 ++++++++++
 core/src/main/scala/kafka/zk/ZkData.scala     |  6 ++
 .../unit/kafka/admin/DeleteTopicTest.scala    | 58 ++++++++++++++++++-
 6 files changed, 177 insertions(+), 5 deletions(-)

diff --git a/core/src/main/scala/kafka/controller/ControllerState.scala b/core/src/main/scala/kafka/controller/ControllerState.scala
index c89cb18d51618..cf2137ae7b741 100644
--- a/core/src/main/scala/kafka/controller/ControllerState.scala
+++ b/core/src/main/scala/kafka/controller/ControllerState.scala
@@ -104,7 +104,11 @@ object ControllerState {
     def value = 15
   }
 
+  case object TopicDeletionFlagChange extends ControllerState {
+    def value = 16
+  }
+
   val values: Seq[ControllerState] = Seq(Idle, ControllerChange, BrokerChange, TopicChange, TopicDeletion,
     AlterPartitionReassignment, AutoLeaderBalance, ManualLeaderBalance, ControlledShutdown, IsrChange, LeaderAndIsrResponseReceived,
-    LogDirChange, ControllerShutdown, UncleanLeaderElectionEnable, TopicUncleanLeaderElectionEnable, ListPartitionReassignment)
+    LogDirChange, ControllerShutdown, UncleanLeaderElectionEnable, TopicUncleanLeaderElectionEnable, ListPartitionReassignment, TopicDeletionFlagChange)
 }
diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala
index 54e9023af2108..bdcb46bcccf57 100644
--- a/core/src/main/scala/kafka/controller/KafkaController.scala
+++ b/core/src/main/scala/kafka/controller/KafkaController.scala
@@ -106,6 +106,7 @@ class KafkaController(val config: KafkaConfig,
   private val preferredReplicaElectionHandler = new PreferredReplicaElectionHandler(eventManager)
   private val isrChangeNotificationHandler = new IsrChangeNotificationHandler(eventManager)
   private val logDirEventNotificationHandler = new LogDirEventNotificationHandler(eventManager)
+  private val topicDeletionFlagHandler = new TopicDeletionFlagHandler(this, eventManager)
 
   @volatile private var activeControllerId = -1
   @volatile private var offlinePartitionCount = 0
@@ -281,7 +282,7 @@ class KafkaController(val config: KafkaConfig,
     val childChangeHandlers = Seq(brokerChangeHandler, topicChangeHandler, topicDeletionHandler, logDirEventNotificationHandler,
       isrChangeNotificationHandler)
     childChangeHandlers.foreach(zkClient.registerZNodeChildChangeHandler)
-    val nodeChangeHandlers = Seq(preferredReplicaElectionHandler, partitionReassignmentHandler)
+    val nodeChangeHandlers = Seq(preferredReplicaElectionHandler, partitionReassignmentHandler, topicDeletionFlagHandler)
     nodeChangeHandlers.foreach(zkClient.registerZNodeChangeHandlerAndCheckExistence)
 
     info("Deleting log dir event notifications")
@@ -367,6 +368,7 @@ class KafkaController(val config: KafkaConfig,
     zkClient.unregisterZNodeChildChangeHandler(topicChangeHandler.path)
     unregisterPartitionModificationsHandlers(partitionModificationsHandlers.keys.toSeq)
     zkClient.unregisterZNodeChildChangeHandler(topicDeletionHandler.path)
+    zkClient.unregisterZNodeChangeHandler(topicDeletionFlagHandler.path)
     // shutdown replica state machine
     replicaStateMachine.shutdown()
     zkClient.unregisterZNodeChildChangeHandler(brokerChangeHandler.path)
@@ -1556,7 +1558,7 @@ class KafkaController(val config: KafkaConfig,
       zkClient.deleteTopicDeletions(nonExistentTopics.toSeq, controllerContext.epochZkVersion)
     }
     topicsToBeDeleted --= nonExistentTopics
-    if (config.deleteTopicEnable) {
+    if (topicDeletionManager.isDeleteTopicEnabled) {
       if (topicsToBeDeleted.nonEmpty) {
         info(s"Starting topic deletion for topics ${topicsToBeDeleted.mkString(",")}")
         // mark topic ineligible for deletion if other state changes are in progress
@@ -1577,6 +1579,24 @@ class KafkaController(val config: KafkaConfig,
     }
   }
 
+  private def processTopicDeletionFlagChange(reset: Boolean = false): Unit = {
+    info("Process TopicDeletionFlagChange event")
+    if (!isActive) return
+    if (reset)
+      topicDeletionManager.resetDeleteTopicEnabled()
+    else {
+      val topicDeletionFlag = zkClient.getTopicDeletionFlag
+      if (!topicDeletionFlag.equalsIgnoreCase("true") && !topicDeletionFlag.equalsIgnoreCase("false")) {
+        info(s"Overwrite ${DeleteTopicFlagZNode.path} to ${topicDeletionManager.isDeleteTopicEnabled}")
+        zkClient.setTopicDeletionFlag(topicDeletionManager.isDeleteTopicEnabled.toString)
+      }
+      else {
+        info(s"Set isDeleteTopicEnabled flag to $topicDeletionFlag")
+        topicDeletionManager.isDeleteTopicEnabled = topicDeletionFlag.toBoolean
+      }
+    }
+  }
+
   private def processZkPartitionReassignment(): Set[TopicPartition] = {
     // We need to register the watcher if the path doesn't exist in order to detect future
     // reassignments and we get the `path exists` check for free
@@ -1900,6 +1920,8 @@ class KafkaController(val config: KafkaConfig,
           processZkPartitionReassignment()
         case ListPartitionReassignments(partitions, callback) =>
           processListPartitionReassignments(partitions, callback)
+        case TopicDeletionFlagChange(reset) =>
+          processTopicDeletionFlagChange(reset)
         case PartitionReassignmentIsrChange(partition) =>
           processPartitionReassignmentIsrChange(partition)
         case IsrChangeNotification =>
@@ -1972,6 +1994,19 @@ class TopicDeletionHandler(eventManager: ControllerEventManager) extends ZNodeCh
 
   override def handleChildChange(): Unit = eventManager.put(TopicDeletion)
 }
+/**
+  * Listener for /topic_deletion_flag znode.
+  *   If the data of the znode is set to true/false, it will trigger the in memory isDeleteTopicEnabled to be set accordingly.
+  *   If the znode data cannot be converted to boolean, it will overwrite znode with the previous valid value.
+  *   If the znode path is deleted, it will reset the in memory isDeleteTopicEnabled to the config value.
+  */
+class TopicDeletionFlagHandler(controller: KafkaController, eventManager: ControllerEventManager) extends ZNodeChangeHandler {
+  override val path: String = DeleteTopicFlagZNode.path
+
+  override def handleDataChange(): Unit = eventManager.put(TopicDeletionFlagChange())
+
+  override def handleDeletion(): Unit = eventManager.put(TopicDeletionFlagChange(true))
+}
 
 class PartitionReassignmentHandler(eventManager: ControllerEventManager) extends ZNodeChangeHandler {
   override val path: String = ReassignPartitionsZNode.path
@@ -2138,6 +2173,10 @@ case object IsrChangeNotification extends ControllerEvent {
   override def state: ControllerState = ControllerState.IsrChange
 }
 
+case class TopicDeletionFlagChange(reset: Boolean = false) extends ControllerEvent {
+  def state = ControllerState.TopicDeletionFlagChange
+}
+
 case class ReplicaLeaderElection(
   partitionsFromAdminClientOpt: Option[Set[TopicPartition]],
   electionType: ElectionType,
diff --git a/core/src/main/scala/kafka/controller/TopicDeletionManager.scala b/core/src/main/scala/kafka/controller/TopicDeletionManager.scala
index d032b3b499dbf..fa8cfdf547e5d 100755
--- a/core/src/main/scala/kafka/controller/TopicDeletionManager.scala
+++ b/core/src/main/scala/kafka/controller/TopicDeletionManager.scala
@@ -20,6 +20,7 @@ import kafka.server.KafkaConfig
 import kafka.utils.Logging
 import kafka.zk.KafkaZkClient
 import org.apache.kafka.common.TopicPartition
+import org.apache.zookeeper.KeeperException.{NoNodeException, NodeExistsException}
 
 import scala.collection.Set
 
@@ -28,6 +29,8 @@ trait DeletionClient {
   def deleteTopicDeletions(topics: Seq[String], epochZkVersion: Int): Unit
   def mutePartitionModifications(topic: String): Unit
   def sendMetadataUpdate(partitions: Set[TopicPartition]): Unit
+  def createDeleteTopicFlagPath(): Unit
+  def getTopicDeletionFlag(): String
 }
 
 class ControllerDeletionClient(controller: KafkaController, zkClient: KafkaZkClient) extends DeletionClient {
@@ -48,6 +51,14 @@ class ControllerDeletionClient(controller: KafkaController, zkClient: KafkaZkCli
   override def sendMetadataUpdate(partitions: Set[TopicPartition]): Unit = {
     controller.sendUpdateMetadataRequest(controller.controllerContext.liveOrShuttingDownBrokerIds.toSeq, partitions)
   }
+
+  override def createDeleteTopicFlagPath(): Unit = {
+    zkClient.createDeleteTopicFlagPath
+  }
+
+  override def getTopicDeletionFlag(): String = {
+    zkClient.getTopicDeletionFlag
+  }
 }
 
 /**
@@ -89,12 +100,21 @@ class TopicDeletionManager(config: KafkaConfig,
                            partitionStateMachine: PartitionStateMachine,
                            client: DeletionClient) extends Logging {
   this.logIdent = s"[Topic Deletion Manager ${config.brokerId}] "
-  val isDeleteTopicEnabled: Boolean = config.deleteTopicEnable
+  var isDeleteTopicEnabled: Boolean = config.deleteTopicEnable
+
+  // Try to create the znode for delete topic flag
+  try {
+    client.createDeleteTopicFlagPath()
+  } catch {
+    case _: NodeExistsException =>
+  }
 
   def init(initialTopicsToBeDeleted: Set[String], initialTopicsIneligibleForDeletion: Set[String]): Unit = {
     info(s"Initializing manager with initial deletions: $initialTopicsToBeDeleted, " +
       s"initial ineligible deletions: $initialTopicsIneligibleForDeletion")
 
+    isDeleteTopicEnabled = getDeleteTopicEnabled()
+
     if (isDeleteTopicEnabled) {
       controllerContext.queueTopicDeletion(initialTopicsToBeDeleted)
       controllerContext.topicsIneligibleForDeletion ++= initialTopicsIneligibleForDeletion & controllerContext.topicsToBeDeleted
@@ -355,4 +375,20 @@ class TopicDeletionManager(config: KafkaConfig,
       }
     }
   }
+
+  private def getDeleteTopicEnabled(): Boolean = {
+    try {
+      val deleteTopicFlag = client.getTopicDeletionFlag
+      if (deleteTopicFlag == null || (!deleteTopicFlag.equalsIgnoreCase("true") && !deleteTopicFlag.equalsIgnoreCase("false")))
+        isDeleteTopicEnabled
+      else deleteTopicFlag.toBoolean
+    } catch {
+      case _: NoNodeException => config.deleteTopicEnable
+    }
+  }
+
+  def resetDeleteTopicEnabled(): Unit = {
+    info("Reset isDeleteTopicEnabled flag to %s".format(config.deleteTopicEnable))
+    isDeleteTopicEnabled = config.deleteTopicEnable
+  }
 }
diff --git a/core/src/main/scala/kafka/zk/KafkaZkClient.scala b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
index 4b1686036d810..f34ae3f97c5a0 100644
--- a/core/src/main/scala/kafka/zk/KafkaZkClient.scala
+++ b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
@@ -793,6 +793,37 @@ class KafkaZkClient private[zk] (zooKeeperClient: ZooKeeperClient, isSecure: Boo
     }
   }
 
+  /**
+   * Creates the delete topic flag znode.
+   * @throws KeeperException if there is an error while setting or creating the znode
+   */
+  def createDeleteTopicFlagPath(): Unit = {
+    createRecursive(DeleteTopicFlagZNode.path)
+  }
+
+  /**
+   * Get topic deletion flag in zookeeper.
+   * @return topic deletion flag in zookeeper.
+   */
+  def getTopicDeletionFlag: String = {
+    val getDataResponse = retryRequestUntilConnected(GetDataRequest(DeleteTopicFlagZNode.path))
+    getDataResponse.resultCode match {
+      case Code.OK => DeleteTopicFlagZNode.decode(getDataResponse.data)
+      case _ => throw getDataResponse.resultException.get
+    }
+  }
+
+  /**
+   * Set topic deletion flag in zookeeper.
+   */
+  def setTopicDeletionFlag(flag: String): Unit = {
+    val setDataResponse = retryRequestUntilConnected(SetDataRequest(DeleteTopicFlagZNode.path, DeleteTopicFlagZNode.encode(flag), -1))
+    setDataResponse.resultCode match {
+      case Code.OK =>
+      case _ => throw setDataResponse.resultException.get
+    }
+  }
+
   /**
    * Remove the given topics from the topics marked for deletion.
    * @param topics the topics to remove.
diff --git a/core/src/main/scala/kafka/zk/ZkData.scala b/core/src/main/scala/kafka/zk/ZkData.scala
index 0d1d525005f80..81deca92da116 100644
--- a/core/src/main/scala/kafka/zk/ZkData.scala
+++ b/core/src/main/scala/kafka/zk/ZkData.scala
@@ -405,6 +405,12 @@ object DeleteTopicsTopicZNode {
   def path(topic: String) = s"${DeleteTopicsZNode.path}/$topic"
 }
 
+object DeleteTopicFlagZNode {
+  def path = "/topic_deletion_flag"
+  def encode(topicDeletionFlag: String): Array[Byte] = topicDeletionFlag.getBytes(UTF_8)
+  def decode(bytes: Array[Byte]): String = if (bytes != null) new String(bytes, UTF_8) else ""
+}
+
 /**
  * The znode for initiating a partition reassignment.
  * @deprecated Since 2.4, use the PartitionReassignment Kafka API instead.
diff --git a/core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala b/core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala
index eb61dd2b60ba5..d2374cccc2094 100644
--- a/core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/DeleteTopicTest.scala
@@ -21,7 +21,7 @@ import java.util.Properties
 import scala.collection.Seq
 
 import kafka.log.Log
-import kafka.zk.{TopicPartitionZNode, ZooKeeperTestHarness}
+import kafka.zk.{DeleteTopicFlagZNode, TopicPartitionZNode, TopicZNode, ZooKeeperTestHarness}
 import kafka.utils.TestUtils
 import kafka.server.{KafkaConfig, KafkaServer}
 import org.junit.Assert._
@@ -413,6 +413,62 @@ class DeleteTopicTest extends ZooKeeperTestHarness {
     assertTrue("Leader should exist for topic test", leaderIdOpt.isDefined)
   }
 
+  def testDeleteTopicAfterEnableZkDeleteTopicFlag() {
+    val topicPartition = new TopicPartition("test", 0)
+    val topic = topicPartition.topic
+    servers = createTestTopicAndCluster(topic, deleteTopicEnabled = false)
+    // mark the topic for deletion
+    adminZkClient.deleteTopic("test")
+    TestUtils.waitUntilTrue(() => !zkClient.isTopicMarkedForDeletion(topic),
+      "Admin path /admin/delete_topic/%s path not deleted even if deleteTopic is disabled".format(topic))
+    // verify that topic test is untouched
+    assertTrue(servers.forall(_.getLogManager().getLog(topicPartition).isDefined))
+    // test the topic path exists
+    assertTrue("Topic path disappeared even when topic deletion is disabled", zkClient.pathExists(TopicZNode.path(topic)))
+    // topic test should have a leader
+    val leaderIdOpt = zkClient.getLeaderForPartition(new TopicPartition(topic, 0))
+    assertTrue("Leader should exist for topic test", leaderIdOpt.isDefined)
+
+    // Set TopicDeletionFlag to true in zk and try delete topic again
+    zkClient.setTopicDeletionFlag("true")
+    TestUtils.waitUntilTrue(() =>
+      try {
+        zkClient.getTopicDeletionFlag.equalsIgnoreCase("true")
+      } catch {
+        case _: Throwable => false
+      },
+      "TopicDeletionFlag is not set")
+    TestUtils.waitUntilTrue( () => getController()._1.kafkaController.topicDeletionManager.isDeleteTopicEnabled,
+      "Delete topic is not enabled")
+    // mark the topic for deletion
+    adminZkClient.deleteTopic("test")
+    TestUtils.verifyTopicDeletion(zkClient, "test", 1, servers)
+
+    // Set TopicDeletionFlag to invalid value in zk
+    zkClient.setTopicDeletionFlag("flase")
+    TestUtils.waitUntilTrue(() =>
+      try {
+        zkClient.getTopicDeletionFlag.equalsIgnoreCase("true")
+      } catch {
+        case _: Throwable => false
+      },
+      "TopicDeletionFlag is not overwritten")
+
+    // delete TopicDeletionFlagPath in zk
+    zkClient.deletePath(DeleteTopicFlagZNode.path)
+    TestUtils.waitUntilTrue(() =>
+      try {
+        !zkClient.pathExists(DeleteTopicFlagZNode.path)
+      } catch {
+        case _: Throwable => false
+      },
+      "TopicDeletionFlagPath is not deleted")
+    TestUtils.waitUntilTrue(() =>
+      getController()._1.kafkaController.topicDeletionManager.isDeleteTopicEnabled == false,
+      "Topic deletion flag is not rest"
+    )
+  }
+
   @Test
   def testDeletingPartiallyDeletedTopic(): Unit = {
     /**

From 11c2060f92bf3fb40cbfcb0475432905d8bf400b Mon Sep 17 00:00:00 2001
From: Zhanxiang Huang 
Date: Fri, 17 Aug 2018 13:18:08 -0700
Subject: [PATCH 0997/1071] [LI-HOTFIX] Allow deleteRecordsBefore() to handle
 topics with only [compacted] in clean.up.policy TICKET = LI_DESCRIPTION =

This patch supports trimming a compact only topic using the delete records request by:
Not responding back with a PolicyVolationException error in the broker side when processing delete record request on compact only topic.
Allow the log cleaner thread to periodically cleanup data below the start offset of a log compacted topic partition
If the log start offset == end offset, active segment will be rolled over and data will be deleted in order to meet GDPR

RB=1366651
G=Kafka-Code-Reviews
R=dolin,sutambe,jkoshy
A=dolin

EXIT_CRITERIA = MANUAL ["This patch should exist as long as we want to allow trimming a topic with compact only cleanup policy"]
---
 .../main/scala/kafka/cluster/Partition.scala  |  3 ---
 core/src/main/scala/kafka/log/Log.scala       |  3 ++-
 .../src/main/scala/kafka/log/LogCleaner.scala |  4 ++-
 .../scala/kafka/log/LogCleanerManager.scala   | 26 ++++++++++++-------
 .../src/main/scala/kafka/log/LogManager.scala |  2 +-
 .../scala/kafka/server/ReplicaManager.scala   |  1 -
 .../AbstractLogCleanerIntegrationTest.scala   |  1 +
 .../kafka/log/LogCleanerManagerTest.scala     |  4 +--
 8 files changed, 26 insertions(+), 18 deletions(-)

diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala
index 7ca7965647a1b..fb8aa38466b6e 100755
--- a/core/src/main/scala/kafka/cluster/Partition.scala
+++ b/core/src/main/scala/kafka/cluster/Partition.scala
@@ -1085,9 +1085,6 @@ class Partition(val topicPartition: TopicPartition,
   def deleteRecordsOnLeader(offset: Long): LogDeleteRecordsResult = inReadLock(leaderIsrUpdateLock) {
     leaderLogIfLocal match {
       case Some(leaderLog) =>
-        if (!leaderLog.config.delete)
-          throw new PolicyViolationException(s"Records of partition $topicPartition can not be deleted due to the configured policy")
-
         val convertedOffset = if (offset == DeleteRecordsRequest.HIGH_WATERMARK)
           leaderLog.highWatermark
         else
diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index 401cb2c92ac48..4cf45a6976b22 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -1804,7 +1804,8 @@ class Log(@volatile var dir: File,
 
   private def deleteLogStartOffsetBreachedSegments(): Int = {
     def shouldDelete(segment: LogSegment, nextSegmentOpt: Option[LogSegment]) =
-      nextSegmentOpt.exists(_.baseOffset <= logStartOffset)
+      nextSegmentOpt.exists(_.baseOffset <= logStartOffset) ||
+        (nextSegmentOpt.isEmpty && logEndOffset == logStartOffset)
 
     deleteOldSegments(shouldDelete, reason = s"log start offset $logStartOffset breach")
   }
diff --git a/core/src/main/scala/kafka/log/LogCleaner.scala b/core/src/main/scala/kafka/log/LogCleaner.scala
index cac1ac50cfe9c..cf74520d3499e 100644
--- a/core/src/main/scala/kafka/log/LogCleaner.scala
+++ b/core/src/main/scala/kafka/log/LogCleaner.scala
@@ -88,12 +88,14 @@ import scala.util.control.ControlThrowable
  * @param initialConfig Initial configuration parameters for the cleaner. Actual config may be dynamically updated.
  * @param logDirs The directories where offset checkpoints reside
  * @param logs The pool of logs
+ * @param retentionCheckMs The frequency that the log cleaner threads checks whether any log is eligible for deletion
  * @param time A way to control the passage of time
  */
 class LogCleaner(initialConfig: CleanerConfig,
                  val logDirs: Seq[File],
                  val logs: Pool[TopicPartition, Log],
                  val logDirFailureChannel: LogDirFailureChannel,
+                 val retentionCheckMs: Long,
                  time: Time = Time.SYSTEM) extends Logging with KafkaMetricsGroup with BrokerReconfigurable
 {
 
@@ -101,7 +103,7 @@ class LogCleaner(initialConfig: CleanerConfig,
   @volatile private var config = initialConfig
 
   /* for managing the state of partitions being cleaned. package-private to allow access in tests */
-  private[log] val cleanerManager = new LogCleanerManager(logDirs, logs, logDirFailureChannel)
+  private[log] val cleanerManager = new LogCleanerManager(logDirs, logs, logDirFailureChannel, retentionCheckMs)
 
   /* a throttle used to limit the I/O of all the cleaner threads to a user-specified maximum rate */
   private val throttler = new Throttler(desiredRatePerSec = config.maxIoBytesPerSecond,
diff --git a/core/src/main/scala/kafka/log/LogCleanerManager.scala b/core/src/main/scala/kafka/log/LogCleanerManager.scala
index 04a97fdd38a5a..c929a9d80c153 100755
--- a/core/src/main/scala/kafka/log/LogCleanerManager.scala
+++ b/core/src/main/scala/kafka/log/LogCleanerManager.scala
@@ -61,7 +61,8 @@ private[log] class LogCleaningException(val log: Log,
   */
 private[log] class LogCleanerManager(val logDirs: Seq[File],
                                      val logs: Pool[TopicPartition, Log],
-                                     val logDirFailureChannel: LogDirFailureChannel) extends Logging with KafkaMetricsGroup {
+                                     val logDirFailureChannel: LogDirFailureChannel,
+                                     val retentionCheckMs: Long) extends Logging with KafkaMetricsGroup {
   import LogCleanerManager._
 
 
@@ -87,6 +88,9 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
   /* for coordinating the pausing and the cleaning of a partition */
   private val pausedCleaningCond = lock.newCondition()
 
+  /* last time the cleaner threads check logs for retention*/
+  private var lastRetentionCheckTime: Long = Time.SYSTEM.milliseconds()
+
   /* gauges for tracking the number of partitions marked as uncleanable for each log directory */
   for (dir <- logDirs) {
     newGauge(
@@ -246,12 +250,17 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
     */
   def deletableLogs(): Iterable[(TopicPartition, Log)] = {
     inLock(lock) {
-      val toClean = logs.filter { case (topicPartition, log) =>
-        !inProgress.contains(topicPartition) && log.config.compact &&
-          !isUncleanablePartition(log, topicPartition)
+      val now = Time.SYSTEM.milliseconds()
+      if (now - lastRetentionCheckTime >= retentionCheckMs) {
+        lastRetentionCheckTime = now
+        val toClean = logs.filter { case (topicPartition, log) =>
+          !inProgress.contains(topicPartition) && log.config.compact && !isUncleanablePartition(log, topicPartition)
+        }
+        toClean.foreach { case (tp, _) => inProgress.put(tp, LogCleaningInProgress) }
+        toClean
       }
-      toClean.foreach { case (tp, _) => inProgress.put(tp, LogCleaningInProgress) }
-      toClean
+      else
+        Iterable.empty
     }
 
   }
@@ -507,6 +516,7 @@ private case class OffsetsToClean(firstDirtyOffset: Long,
 
 private[log] object LogCleanerManager extends Logging {
 
+
   def isCompactAndDelete(log: Log): Boolean = {
     log.config.compact && log.config.delete
   }
@@ -550,9 +560,7 @@ private[log] object LogCleanerManager extends Logging {
       val checkpointDirtyOffset = lastCleanOffset.getOrElse(logStartOffset)
 
       if (checkpointDirtyOffset < logStartOffset) {
-        // Don't bother with the warning if compact and delete are enabled.
-        if (!isCompactAndDelete(log))
-          warn(s"Resetting first dirty offset of ${log.name} to log start offset $logStartOffset " +
+        debug(s"Resetting first dirty offset of ${log.name} to log start offset $logStartOffset " +
             s"since the checkpointed offset $checkpointDirtyOffset is invalid.")
         (logStartOffset, true)
       } else if (checkpointDirtyOffset > log.logEndOffset) {
diff --git a/core/src/main/scala/kafka/log/LogManager.scala b/core/src/main/scala/kafka/log/LogManager.scala
index 075ba747ea532..24e600319520a 100755
--- a/core/src/main/scala/kafka/log/LogManager.scala
+++ b/core/src/main/scala/kafka/log/LogManager.scala
@@ -121,7 +121,7 @@ class LogManager(logDirs: Seq[File],
   // public, so we can access this from kafka.admin.DeleteTopicTest
   val cleaner: LogCleaner =
     if(cleanerConfig.enableCleaner)
-      new LogCleaner(cleanerConfig, liveLogDirs, currentLogs, logDirFailureChannel, time = time)
+      new LogCleaner(cleanerConfig, liveLogDirs, currentLogs, logDirFailureChannel, retentionCheckMs, time = time)
     else
       null
 
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index b3d4e2e9a5cc5..0c1b97c72aaeb 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -586,7 +586,6 @@ class ReplicaManager(val config: KafkaConfig,
           case e@ (_: UnknownTopicOrPartitionException |
                    _: NotLeaderForPartitionException |
                    _: OffsetOutOfRangeException |
-                   _: PolicyViolationException |
                    _: KafkaStorageException) =>
             (topicPartition, LogDeleteRecordsResult(-1L, -1L, Some(e)))
           case t: Throwable =>
diff --git a/core/src/test/scala/unit/kafka/log/AbstractLogCleanerIntegrationTest.scala b/core/src/test/scala/unit/kafka/log/AbstractLogCleanerIntegrationTest.scala
index fe98ebfe64c57..f2991afbc3368 100644
--- a/core/src/test/scala/unit/kafka/log/AbstractLogCleanerIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/log/AbstractLogCleanerIntegrationTest.scala
@@ -127,6 +127,7 @@ abstract class AbstractLogCleanerIntegrationTest {
       logDirs = Array(logDir),
       logs = logMap,
       logDirFailureChannel = new LogDirFailureChannel(1),
+      retentionCheckMs = 0L,
       time = time)
   }
 
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
index ed5b1e8a91423..0630fcc1d2fb2 100644
--- a/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
@@ -50,7 +50,7 @@ class LogCleanerManagerTest extends Logging {
 
   class LogCleanerManagerMock(logDirs: Seq[File],
                               logs: Pool[TopicPartition, Log],
-                              logDirFailureChannel: LogDirFailureChannel) extends LogCleanerManager(logDirs, logs, logDirFailureChannel) {
+                              logDirFailureChannel: LogDirFailureChannel) extends LogCleanerManager(logDirs, logs, logDirFailureChannel, 0L) {
     override def allCleanerCheckpoints: Map[TopicPartition, Long] = {
       cleanerCheckpoints.toMap
     }
@@ -642,7 +642,7 @@ class LogCleanerManagerTest extends Logging {
   private def createCleanerManager(log: Log): LogCleanerManager = {
     val logs = new Pool[TopicPartition, Log]()
     logs.put(topicPartition, log)
-    new LogCleanerManager(Array(logDir), logs, null)
+    new LogCleanerManager(Array(logDir), logs, null, 0L)
   }
 
   private def createCleanerManagerMock(pool: Pool[TopicPartition, Log]): LogCleanerManagerMock = {

From 9701dbe6de30d3693821aafc433a367ddc56616b Mon Sep 17 00:00:00 2001
From: Jon Lee 
Date: Wed, 29 Aug 2018 16:52:46 -0700
Subject: [PATCH 0998/1071] [LI-HOTFIX] LIKAFKA-18966: do not expire quota
 sensors TICKET = LI_DESCRIPTION = The Kafka sensor for throttle time is
 recorded only when there is throttling and thus if a client is not throttled
 for a while, the sensor expires. And when it expires, it is not registered as
 an attribute of the corresponding mbean sensor. However, healthcheck expects
 this attribute to exist in the mbean sensor all the time and hence raising
 the attribute not found exception and polluting application logs.
 EXIT_CRITERIA = MANUAL [""]

---
 core/src/main/scala/kafka/server/ClientQuotaManager.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/kafka/server/ClientQuotaManager.scala b/core/src/main/scala/kafka/server/ClientQuotaManager.scala
index dd461a0a40912..b83469980762b 100644
--- a/core/src/main/scala/kafka/server/ClientQuotaManager.scala
+++ b/core/src/main/scala/kafka/server/ClientQuotaManager.scala
@@ -62,8 +62,8 @@ object ClientQuotaManagerConfig {
   // Always have 10 whole windows + 1 current window
   val DefaultNumQuotaSamples = 11
   val DefaultQuotaWindowSizeSeconds = 1
-  // Purge sensors after 1 hour of inactivity
-  val InactiveSensorExpirationTimeSeconds  = 3600
+  // Do not expire quota sensors
+  val InactiveSensorExpirationTimeSeconds  = Int.MaxValue
   val QuotaRequestPercentDefault = Int.MaxValue.toDouble
   val NanosToPercentagePerSecond = 100.0 / TimeUnit.SECONDS.toNanos(1)
 

From 3eda8705aaa271cb2f690ede6a5a088e3bf02600 Mon Sep 17 00:00:00 2001
From: Lucas Wang 
Date: Wed, 29 Aug 2018 09:54:42 -0700
Subject: [PATCH 0999/1071] [LI-HOTFIX] Showing the provided urls in the
 exception when no resolvable url can be found TICKET = LI_DESCRIPTION = Add
 url information to exception message to help debugging RB=1405576
 BUG=TOOLS-194012 G=Kafka-Code-Reviews R=dolin,mgharat,rrosenbl
 A=rrosenbl,mgharat

EXIT_CRITERIA = MANUAL ["NONE"]
---
 .../src/main/java/org/apache/kafka/clients/ClientUtils.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/ClientUtils.java b/clients/src/main/java/org/apache/kafka/clients/ClientUtils.java
index 1d71ea17c0d61..9b3df8284c4aa 100644
--- a/clients/src/main/java/org/apache/kafka/clients/ClientUtils.java
+++ b/clients/src/main/java/org/apache/kafka/clients/ClientUtils.java
@@ -85,7 +85,8 @@ public static List parseAndValidateAddresses(List url
             }
         }
         if (addresses.isEmpty())
-            throw new ConfigException("No resolvable bootstrap urls given in " + CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG);
+            throw new ConfigException("No resolvable bootstrap server in provided urls: " +
+                String.join(",", urls));
         return addresses;
     }
 

From 25469836082c6945b4806a111c4e0adbfb43f90d Mon Sep 17 00:00:00 2001
From: Jon Lee 
Date: Fri, 7 Sep 2018 11:14:10 -0700
Subject: [PATCH 1000/1071] [LI-HOTFIX] Added
 org.apache.kafka.common.protocol.SecurityProtocol, which is a duplicate of
 org.apache.kafka.common.security.auth.SecurityProtocol, to break circular
 dependency TICKET = LI_DESCRIPTION =

RB=1414299
R=jkoshy,mgharat
A=mgharat

EXIT_CRITERIA = MANUAL ["Can be removed when org.apache.kafka.common.protocol.SecurityProtocol is not imported anywhere"]
---
 .../common/protocol/SecurityProtocol.java     | 78 +++++++++++++++++++
 1 file changed, 78 insertions(+)
 create mode 100644 clients/src/main/java/org/apache/kafka/common/protocol/SecurityProtocol.java

diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/SecurityProtocol.java b/clients/src/main/java/org/apache/kafka/common/protocol/SecurityProtocol.java
new file mode 100644
index 0000000000000..9739ad50078b8
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/SecurityProtocol.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.common.protocol;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+// A duplicate of org.apache.kafka.common.security.auth.SecurityProtocol. Temporarily added to break circular dependency
+// among kafka-server, container, and linkedin-kafka-clients.
+public enum SecurityProtocol {
+    /** Un-authenticated, non-encrypted channel */
+    PLAINTEXT(0, "PLAINTEXT"),
+    /** SSL channel */
+    SSL(1, "SSL"),
+    /** SASL authenticated, non-encrypted channel */
+    SASL_PLAINTEXT(2, "SASL_PLAINTEXT"),
+    /** SASL authenticated, SSL channel */
+    SASL_SSL(3, "SASL_SSL");
+
+    private static final Map CODE_TO_SECURITY_PROTOCOL;
+    private static final List NAMES;
+
+    static {
+        SecurityProtocol[] protocols = SecurityProtocol.values();
+        List names = new ArrayList<>(protocols.length);
+        Map codeToSecurityProtocol = new HashMap<>(protocols.length);
+        for (SecurityProtocol proto : protocols) {
+            codeToSecurityProtocol.put(proto.id, proto);
+            names.add(proto.name);
+        }
+        CODE_TO_SECURITY_PROTOCOL = Collections.unmodifiableMap(codeToSecurityProtocol);
+        NAMES = Collections.unmodifiableList(names);
+    }
+
+    /** The permanent and immutable id of a security protocol -- this can't change, and must match kafka.cluster.SecurityProtocol  */
+    public final short id;
+
+    /** Name of the security protocol. This may be used by client configuration. */
+    public final String name;
+
+    SecurityProtocol(int id, String name) {
+        this.id = (short) id;
+        this.name = name;
+    }
+
+    public static List names() {
+        return NAMES;
+    }
+
+    public static SecurityProtocol forId(short id) {
+        return CODE_TO_SECURITY_PROTOCOL.get(id);
+    }
+
+    /** Case insensitive lookup by protocol name */
+    public static SecurityProtocol forName(String name) {
+        return SecurityProtocol.valueOf(name.toUpperCase(Locale.ROOT));
+    }
+
+}

From 153d37f9ec7a7bd225bf4f0b989d11904be726c6 Mon Sep 17 00:00:00 2001
From: Abhishek Mendhekar 
Date: Mon, 1 Oct 2018 13:33:58 -0700
Subject: [PATCH 1001/1071] [LI-HOTFIX] Log FindCoordinatorRequest send and
 failure to info in AbstractCoordinator TICKET = LI_DESCRIPTION = No logging
 for FindCooridnatorRequest send and failure makes it hard to investigate
 issues. Since there are no metrics for this having logging here makes
 investigation easier.

EXIT_CRITERIA = MANUAL ["Changed logging from debug to info hence limited to LinkedIn branch"]
---
 .../kafka/clients/consumer/internals/AbstractCoordinator.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index 15d1c278572ef..efb2765ec4152 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -725,7 +725,7 @@ public void handle(SyncGroupResponse syncResponse,
      */
     private RequestFuture sendFindCoordinatorRequest(Node node) {
         // initiate the group metadata request
-        log.debug("Sending FindCoordinator request to broker {}", node);
+        log.info("Sending FindCoordinator request to broker {}", node);
         FindCoordinatorRequest.Builder requestBuilder =
                 new FindCoordinatorRequest.Builder(
                         new FindCoordinatorRequestData()
@@ -762,7 +762,7 @@ public void onSuccess(ClientResponse resp, RequestFuture future) {
             } else if (error == Errors.GROUP_AUTHORIZATION_FAILED) {
                 future.raise(GroupAuthorizationException.forGroupId(rebalanceConfig.groupId));
             } else {
-                log.debug("Group coordinator lookup failed: {}", findCoordinatorResponse.data().errorMessage());
+                log.info("Group coordinator lookup failed: {}", findCoordinatorResponse.data().errorMessage());
                 future.raise(error);
             }
         }

From 155b4f8be83f2a65c9237079f0f5006f45bd9650 Mon Sep 17 00:00:00 2001
From: Zhanxiang Huang 
Date: Wed, 19 Sep 2018 17:43:42 -0700
Subject: [PATCH 1002/1071] [LI-HOTFIX] Reduce lock retention and improve
 broker shutdown time: TICKET = [KAFKA-8667, KAFKA-8668] LI_DESCRIPTION = -
 Avoid acquiring partitionMap lock in shutdownIdleFetcherThread - Avoid
 appending to the time index during shutdown if the time index has not yet be
 initialized

RB=1431408
BUG=LIKAFKA-19361
G=Kafka-Code-Reviews
R=jkoshy,jonlee
A=jkoshy,jonlee

EXIT_CRITERIA = TICKET [KAFKA-8667, KAFKA-8668]
---
 core/src/main/scala/kafka/log/LogSegment.scala                | 4 +++-
 core/src/main/scala/kafka/server/AbstractFetcherManager.scala | 2 +-
 core/src/main/scala/kafka/server/AbstractFetcherThread.scala  | 3 +++
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/kafka/log/LogSegment.scala b/core/src/main/scala/kafka/log/LogSegment.scala
index 64e96e1cc73eb..97fb45baabd0a 100755
--- a/core/src/main/scala/kafka/log/LogSegment.scala
+++ b/core/src/main/scala/kafka/log/LogSegment.scala
@@ -585,7 +585,9 @@ class LogSegment private[log] (val log: FileRecords,
    * Close this log segment
    */
   def close(): Unit = {
-    CoreUtils.swallow(timeIndex.maybeAppend(maxTimestampSoFar, offsetOfMaxTimestampSoFar, skipFullCheck = true), this)
+    if (_maxTimestampSoFar.nonEmpty || _offsetOfMaxTimestampSoFar.nonEmpty) {
+      CoreUtils.swallow(timeIndex.maybeAppend(maxTimestampSoFar, offsetOfMaxTimestampSoFar, skipFullCheck = true), this)
+    }
     CoreUtils.swallow(offsetIndex.close(), this)
     CoreUtils.swallow(timeIndex.close(), this)
     CoreUtils.swallow(log.close(), this)
diff --git a/core/src/main/scala/kafka/server/AbstractFetcherManager.scala b/core/src/main/scala/kafka/server/AbstractFetcherManager.scala
index e326c4b27e690..2089858a50a74 100755
--- a/core/src/main/scala/kafka/server/AbstractFetcherManager.scala
+++ b/core/src/main/scala/kafka/server/AbstractFetcherManager.scala
@@ -196,7 +196,7 @@ abstract class AbstractFetcherManager[T <: AbstractFetcherThread](val name: Stri
     lock synchronized {
       val keysToBeRemoved = new mutable.HashSet[BrokerIdAndFetcherId]
       for ((key, fetcher) <- fetcherThreadMap) {
-        if (fetcher.partitionCount <= 0) {
+        if (fetcher.idle) {
           fetcher.shutdown()
           keysToBeRemoved += key
         }
diff --git a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
index a58ad99d8a642..5b8be07fce42a 100755
--- a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
+++ b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
@@ -68,6 +68,8 @@ abstract class AbstractFetcherThread(name: String,
   val fetcherStats = new FetcherStats(metricId)
   val fetcherLagStats = new FetcherLagStats(metricId)
 
+  @volatile var idle = false
+
   /* callbacks to be defined in subclass */
 
   // process fetched data
@@ -651,6 +653,7 @@ abstract class AbstractFetcherThread(name: String,
         partitionStates.remove(topicPartition)
         fetcherLagStats.unregister(topicPartition)
       }
+      idle = partitionStates.size() <= 0
     } finally partitionMapLock.unlock()
   }
 

From c53fffd480bcd603295d9cc95d33afc65fe86ce2 Mon Sep 17 00:00:00 2001
From: Zhanxiang Huang 
Date: Wed, 3 Oct 2018 14:07:33 -0700
Subject: [PATCH 1003/1071] [LI-HOTFIX] Update fetcher thread idle flag in
 addPartitions TICKET = KAFKA-8667 LI_DESCRIPTION =
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This patch fixes in bug introduced by “[LI-HOTFIX] Reduce lock retention and improve broker shutdown time” HOTFIX where the fetcher thread idle flag is not set in addPartitions, which can cause idle fetcher thread not shutdown in time.


RB=1431408
BUG=LIKAFKA-19361
G=Kafka-Code-Reviews
R=jkoshy,jonlee
A=jkoshy,jonlee

EXIT_CRITERIA = TICKET [KAFKA-8667]
---
 .../main/scala/kafka/server/AbstractFetcherThread.scala  | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
index 5b8be07fce42a..3747a0c8ddb8e 100755
--- a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
+++ b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
@@ -445,6 +445,7 @@ abstract class AbstractFetcherThread(name: String,
         partitionStates.updateAndMoveToEnd(tp, updatedState)
       }
 
+      maybeUpdateIdleFlag()
       partitionMapCond.signalAll()
       initialFetchStates.keySet
     } finally partitionMapLock.unlock()
@@ -470,6 +471,7 @@ abstract class AbstractFetcherThread(name: String,
         (state.topicPartition, maybeTruncationComplete)
       }.toMap
     partitionStates.set(newStates.asJava)
+    maybeUpdateIdleFlag()
   }
 
   /**
@@ -653,7 +655,7 @@ abstract class AbstractFetcherThread(name: String,
         partitionStates.remove(topicPartition)
         fetcherLagStats.unregister(topicPartition)
       }
-      idle = partitionStates.size() <= 0
+      maybeUpdateIdleFlag()
     } finally partitionMapLock.unlock()
   }
 
@@ -687,6 +689,11 @@ abstract class AbstractFetcherThread(name: String,
     }
   }
 
+  // This method should only be called when holding the partitionMapLock
+  private def maybeUpdateIdleFlag(): Unit = {
+    idle = partitionStates.size() <= 0
+  }
+
 }
 
 object AbstractFetcherThread {

From 958d0adfa40b769c6b7c18d81a8d3330d86e78a0 Mon Sep 17 00:00:00 2001
From: Lucas Wang 
Date: Thu, 4 Oct 2018 16:35:07 -0700
Subject: [PATCH 1004/1071] [LI-HOTFIX] Removing the
 PreferredReplicaImbalanceCount metric to make the controller performant
 TICKET = LI_DESCRIPTION = Avoid expensive update of
 PreferredReplicaImbalanceCount metric

RB=1442797
G=Kafka-Code-Reviews
A=jonlee

EXIT_CRITERIA = MANUAL ["NONE"]
---
 .../kafka/controller/KafkaController.scala    | 32 -------------------
 ...tricsDuringTopicCreationDeletionTest.scala | 10 ------
 .../unit/kafka/metrics/MetricsTest.scala      |  1 -
 3 files changed, 43 deletions(-)

diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala
index bdcb46bcccf57..0723b16686d3b 100644
--- a/core/src/main/scala/kafka/controller/KafkaController.scala
+++ b/core/src/main/scala/kafka/controller/KafkaController.scala
@@ -110,7 +110,6 @@ class KafkaController(val config: KafkaConfig,
 
   @volatile private var activeControllerId = -1
   @volatile private var offlinePartitionCount = 0
-  @volatile private var preferredReplicaImbalanceCount = 0
   @volatile private var globalTopicCount = 0
   @volatile private var globalPartitionCount = 0
   @volatile private var topicsToDeleteCount = 0
@@ -135,13 +134,6 @@ class KafkaController(val config: KafkaConfig,
     }
   )
 
-  newGauge(
-    "PreferredReplicaImbalanceCount",
-    new Gauge[Int] {
-      def value: Int = preferredReplicaImbalanceCount
-    }
-  )
-
   newGauge(
     "ControllerState",
     new Gauge[Byte] {
@@ -349,7 +341,6 @@ class KafkaController(val config: KafkaConfig,
     // shutdown leader rebalance scheduler
     kafkaScheduler.shutdown()
     offlinePartitionCount = 0
-    preferredReplicaImbalanceCount = 0
     globalTopicCount = 0
     globalPartitionCount = 0
     topicsToDeleteCount = 0
@@ -1296,29 +1287,6 @@ class KafkaController(val config: KafkaConfig,
         controllerContext.offlinePartitionCount
       }
 
-    preferredReplicaImbalanceCount =
-      if (!isActive) {
-        0
-      } else {
-        controllerContext.allPartitions.count { topicPartition =>
-          val replicaAssignment: ReplicaAssignment = controllerContext.partitionFullReplicaAssignment(topicPartition)
-          val replicas = replicaAssignment.replicas
-          val preferredReplica = replicas.head
-
-          val isImbalanced = controllerContext.partitionLeadershipInfo.get(topicPartition) match {
-            case Some(leadershipInfo) =>
-              if (replicaAssignment.isBeingReassigned && replicaAssignment.addingReplicas.contains(preferredReplica))
-                // reassigning partitions are not counted as imbalanced until the new replica joins the ISR (completes reassignment)
-                leadershipInfo.leaderAndIsr.isr.contains(preferredReplica)
-              else
-                leadershipInfo.leaderAndIsr.leader != preferredReplica
-            case None => false
-          }
-
-          isImbalanced && !topicDeletionManager.isTopicQueuedUpForDeletion(topicPartition.topic)
-        }
-      }
-
     globalTopicCount = if (!isActive) 0 else controllerContext.allTopics.size
 
     globalPartitionCount = if (!isActive) 0 else controllerContext.partitionLeadershipInfo.size
diff --git a/core/src/test/scala/unit/kafka/integration/MetricsDuringTopicCreationDeletionTest.scala b/core/src/test/scala/unit/kafka/integration/MetricsDuringTopicCreationDeletionTest.scala
index 0acc184cd1a30..c42d8eecfe071 100644
--- a/core/src/test/scala/unit/kafka/integration/MetricsDuringTopicCreationDeletionTest.scala
+++ b/core/src/test/scala/unit/kafka/integration/MetricsDuringTopicCreationDeletionTest.scala
@@ -81,10 +81,6 @@ class MetricsDuringTopicCreationDeletionTest extends KafkaServerTestHarness with
     @volatile var offlinePartitionsCount = offlinePartitionsCountGauge.value
     assert(offlinePartitionsCount == 0)
 
-    val preferredReplicaImbalanceCountGauge = getGauge("PreferredReplicaImbalanceCount")
-    @volatile var preferredReplicaImbalanceCount = preferredReplicaImbalanceCountGauge.value
-    assert(preferredReplicaImbalanceCount == 0)
-
     // Thread checking the metric continuously
     running = true
     val thread = new Thread(new Runnable {
@@ -97,11 +93,6 @@ class MetricsDuringTopicCreationDeletionTest extends KafkaServerTestHarness with
             }
           }
 
-          preferredReplicaImbalanceCount = preferredReplicaImbalanceCountGauge.value
-          if (preferredReplicaImbalanceCount > 0) {
-             running = false
-          }
-
           offlinePartitionsCount = offlinePartitionsCountGauge.value
           if (offlinePartitionsCount > 0) {
              running = false
@@ -119,7 +110,6 @@ class MetricsDuringTopicCreationDeletionTest extends KafkaServerTestHarness with
     thread.join
 
     assert(offlinePartitionsCount==0, "OfflinePartitionCount not 0: "+ offlinePartitionsCount)
-    assert(preferredReplicaImbalanceCount==0, "PreferredReplicaImbalanceCount not 0: " + preferredReplicaImbalanceCount)
     assert(underReplicatedPartitionCount==0, "UnderReplicatedPartitionCount not 0: " + underReplicatedPartitionCount)
   }
 
diff --git a/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala b/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
index 62040d5e262f9..587bf5e71e3ac 100644
--- a/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
+++ b/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
@@ -152,7 +152,6 @@ class MetricsTest extends KafkaServerTestHarness with Logging {
 
     assertEquals(metrics.keySet.asScala.count(_.getMBeanName == "kafka.controller:type=KafkaController,name=ActiveControllerCount"), 1)
     assertEquals(metrics.keySet.asScala.count(_.getMBeanName == "kafka.controller:type=KafkaController,name=OfflinePartitionsCount"), 1)
-    assertEquals(metrics.keySet.asScala.count(_.getMBeanName == "kafka.controller:type=KafkaController,name=PreferredReplicaImbalanceCount"), 1)
     assertEquals(metrics.keySet.asScala.count(_.getMBeanName == "kafka.controller:type=KafkaController,name=GlobalTopicCount"), 1)
     assertEquals(metrics.keySet.asScala.count(_.getMBeanName == "kafka.controller:type=KafkaController,name=GlobalPartitionCount"), 1)
     assertEquals(metrics.keySet.asScala.count(_.getMBeanName == "kafka.controller:type=KafkaController,name=TopicsToDeleteCount"), 1)

From f5d4f3f34655b2f93d78148223bef1057fa721de Mon Sep 17 00:00:00 2001
From: Ke Hu 
Date: Fri, 5 Oct 2018 12:34:32 -0700
Subject: [PATCH 1005/1071] [LI-HOTFIX] Mirror maker passthrough mode,
 including shallow iterator over record batches and producer compression
 config and enable passthrough for mirror maker TICKET = LI_DESCRIPTION =

RB=1409907
R=okaraman,jkoshy,nramesh
A=jkoshy,nramesh

EXIT_CRITERIA = MANUAL ["When we decide to get rid of passthrough mode"]
---
 .../clients/consumer/ConsumerConfig.java      |  11 ++
 .../kafka/clients/consumer/KafkaConsumer.java |   1 +
 .../clients/consumer/internals/Fetcher.java   |  15 ++-
 .../record/AbstractLegacyRecordBatch.java     |   5 +
 .../kafka/common/record/CompressionType.java  |  16 +++
 .../kafka/common/record/DefaultRecord.java    |  10 ++
 .../common/record/DefaultRecordBatch.java     |  30 +++++
 .../common/record/FileLogInputStream.java     |   6 +
 .../common/record/MemoryRecordsBuilder.java   |  69 +++++++++-
 .../kafka/common/record/RecordBatch.java      |  10 ++
 .../clients/consumer/KafkaConsumerTest.java   |   2 +
 .../consumer/internals/FetcherTest.java       |  42 ++++++
 .../producer/internals/ProducerBatchTest.java |   6 +-
 .../common/record/FileLogInputStreamTest.java |   7 +-
 .../record/MemoryRecordsBuilderTest.java      |   7 +-
 .../common/record/MemoryRecordsTest.java      | 127 +++++++++++++++++-
 core/src/main/scala/kafka/log/Log.scala       |   5 -
 .../main/scala/kafka/tools/MirrorMaker.scala  |   7 +
 ...gCleanerParameterizedIntegrationTest.scala |   2 +-
 19 files changed, 356 insertions(+), 22 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
index 86988f0e0337e..d50dfb65ab193 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
@@ -206,6 +206,12 @@ public class ConsumerConfig extends AbstractConfig {
     public static final String CHECK_CRCS_CONFIG = "check.crcs";
     private static final String CHECK_CRCS_DOC = "Automatically check the CRC32 of the records consumed. This ensures no on-the-wire or on-disk corruption to the messages occurred. This check adds some overhead, so it may be disabled in cases seeking extreme performance.";
 
+    /**
+     * enable.shallow.iterator
+     */
+    public static final String ENABLE_SHALLOW_ITERATOR_CONFIG = "enable.shallow.iterator";
+    private static final String ENABLE_SHALLOW_ITERATOR_DOC = "Shallow iterate message batches in the consumer, returning message batches (potentially compressed) instead of individual records";
+
     /** key.deserializer */
     public static final String KEY_DESERIALIZER_CLASS_CONFIG = "key.deserializer";
     public static final String KEY_DESERIALIZER_CLASS_DOC = "Deserializer class for key that implements the org.apache.kafka.common.serialization.Deserializer interface.";
@@ -404,6 +410,11 @@ public class ConsumerConfig extends AbstractConfig {
                                         true,
                                         Importance.LOW,
                                         CHECK_CRCS_DOC)
+                                .define(ENABLE_SHALLOW_ITERATOR_CONFIG,
+                                        Type.BOOLEAN,
+                                        false,
+                                        Importance.LOW,
+                                        ENABLE_SHALLOW_ITERATOR_DOC)
                                 .define(METRICS_SAMPLE_WINDOW_MS_CONFIG,
                                         Type.LONG,
                                         30000,
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index eaf8cff9a8ddc..28ac47c23be82 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -797,6 +797,7 @@ else if (enableAutoCommit)
                     config.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG),
                     config.getBoolean(ConsumerConfig.CHECK_CRCS_CONFIG),
                     config.getString(ConsumerConfig.CLIENT_RACK_CONFIG),
+                    config.getBoolean(ConsumerConfig.ENABLE_SHALLOW_ITERATOR_CONFIG),
                     this.keyDeserializer,
                     this.valueDeserializer,
                     this.metadata,
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
index 00b4319c53e13..18ac4001ffdc3 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
@@ -56,6 +56,7 @@
 import org.apache.kafka.common.metrics.stats.WindowedCount;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.record.AbstractLegacyRecordBatch;
 import org.apache.kafka.common.record.BufferSupplier;
 import org.apache.kafka.common.record.ControlRecordType;
 import org.apache.kafka.common.record.Record;
@@ -103,6 +104,8 @@
 import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyList;
+import static org.apache.kafka.common.record.RecordBatch.MAGIC_VALUE_V2;
+
 
 /**
  * This class manages the fetching process with the brokers.
@@ -139,6 +142,7 @@ public class Fetcher implements Closeable {
     private final int maxPollRecords;
     private final boolean checkCrcs;
     private final String clientRackId;
+    private final boolean enableShallowIteration;
     private final ConsumerMetadata metadata;
     private final FetchManagerMetrics sensors;
     private final SubscriptionState subscriptions;
@@ -165,6 +169,7 @@ public Fetcher(LogContext logContext,
                    int maxPollRecords,
                    boolean checkCrcs,
                    String clientRackId,
+                   boolean enableShallowIteration,
                    Deserializer keyDeserializer,
                    Deserializer valueDeserializer,
                    ConsumerMetadata metadata,
@@ -191,6 +196,7 @@ public Fetcher(LogContext logContext,
         this.clientRackId = clientRackId;
         this.keyDeserializer = keyDeserializer;
         this.valueDeserializer = valueDeserializer;
+        this.enableShallowIteration = enableShallowIteration;
         this.completedFetches = new ConcurrentLinkedQueue<>();
         this.sensors = new FetchManagerMetrics(metrics, metricsRegistry);
         this.retryBackoffMs = retryBackoffMs;
@@ -1302,7 +1308,8 @@ private ConsumerRecord parseRecord(TopicPartition partition,
             ByteBuffer keyBytes = record.key();
             byte[] keyByteArray = keyBytes == null ? null : Utils.toArray(keyBytes);
             K key = keyBytes == null ? null : this.keyDeserializer.deserialize(partition.topic(), headers, keyByteArray);
-            ByteBuffer valueBytes = record.value();
+            ByteBuffer valueBytes = record.hasMagic(MAGIC_VALUE_V2) ? record.value() :
+                (enableShallowIteration ? ((AbstractLegacyRecordBatch) record).outerRecord().buffer() : record.value());
             byte[] valueByteArray = valueBytes == null ? null : Utils.toArray(valueBytes);
             V value = valueBytes == null ? null : this.valueDeserializer.deserialize(partition.topic(), headers, valueByteArray);
             return new ConsumerRecord<>(partition.topic(), partition.partition(), offset,
@@ -1493,7 +1500,11 @@ private Record nextFetchedRecord() {
                         }
                     }
 
-                    records = currentBatch.streamingIterator(decompressionBufferSupplier);
+                    if (enableShallowIteration) {
+                        records = currentBatch.shallowIterator();
+                    } else {
+                        records = currentBatch.streamingIterator(decompressionBufferSupplier);
+                    }
                 } else {
                     Record record = records.next();
                     // skip any records out of range
diff --git a/clients/src/main/java/org/apache/kafka/common/record/AbstractLegacyRecordBatch.java b/clients/src/main/java/org/apache/kafka/common/record/AbstractLegacyRecordBatch.java
index 83637640af49d..93901da15c74c 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/AbstractLegacyRecordBatch.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/AbstractLegacyRecordBatch.java
@@ -232,6 +232,11 @@ CloseableIterator iterator(BufferSupplier bufferSupplier) {
         if (isCompressed())
             return new DeepRecordsIterator(this, false, Integer.MAX_VALUE, bufferSupplier);
 
+        return shallowIterator();
+    }
+
+    @Override
+    public CloseableIterator shallowIterator() {
         return new CloseableIterator() {
             private boolean hasNext = true;
 
diff --git a/clients/src/main/java/org/apache/kafka/common/record/CompressionType.java b/clients/src/main/java/org/apache/kafka/common/record/CompressionType.java
index 352d12d834977..454dd17bf8240 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/CompressionType.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/CompressionType.java
@@ -133,6 +133,18 @@ public InputStream wrapForInput(ByteBuffer buffer, byte messageVersion, BufferSu
                 throw new KafkaException(e);
             }
         }
+    },
+
+    PASSTHROUGH(5, "passthrough", 1.0f) {
+        @Override
+        public OutputStream wrapForOutput(ByteBufferOutputStream buffer, byte messageVersion) {
+            return buffer;
+        }
+
+        @Override
+        public InputStream wrapForInput(ByteBuffer buffer, byte messageVersion, BufferSupplier decompressionBufferSupplier) {
+            return new ByteBufferInputStream(buffer);
+        }
     };
 
     public final int id;
@@ -178,6 +190,8 @@ public static CompressionType forId(int id) {
                 return LZ4;
             case 4:
                 return ZSTD;
+            case 5:
+                return PASSTHROUGH;
             default:
                 throw new IllegalArgumentException("Unknown compression type id: " + id);
         }
@@ -194,6 +208,8 @@ else if (LZ4.name.equals(name))
             return LZ4;
         else if (ZSTD.name.equals(name))
             return ZSTD;
+        else if (PASSTHROUGH.name.equals(name))
+            return PASSTHROUGH;
         else
             throw new IllegalArgumentException("Unknown compression name: " + name);
     }
diff --git a/clients/src/main/java/org/apache/kafka/common/record/DefaultRecord.java b/clients/src/main/java/org/apache/kafka/common/record/DefaultRecord.java
index 976b5567a3709..637b5a9a522e1 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/DefaultRecord.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/DefaultRecord.java
@@ -315,6 +315,16 @@ public static DefaultRecord readFrom(ByteBuffer buffer,
                 baseSequence, logAppendTime);
     }
 
+    public static DefaultRecord readAllRecords(int sizeInBytes,
+        byte attributes,
+        long offset,
+        long timestamp,
+        int sequence,
+        ByteBuffer buffer) {
+        return new DefaultRecord(sizeInBytes, attributes, offset,
+            timestamp, sequence, null, buffer, Record.EMPTY_HEADERS);
+    }
+
     private static DefaultRecord readFrom(ByteBuffer buffer,
                                           int sizeInBytes,
                                           int sizeOfBodyInBytes,
diff --git a/clients/src/main/java/org/apache/kafka/common/record/DefaultRecordBatch.java b/clients/src/main/java/org/apache/kafka/common/record/DefaultRecordBatch.java
index d4a9587ffa56e..09e5abd4d9d66 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/DefaultRecordBatch.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/DefaultRecordBatch.java
@@ -340,6 +340,36 @@ public CloseableIterator skipKeyValueIterator(BufferSupplier bufferSuppl
         return compressedIterator(bufferSupplier, true);
     }
 
+    @Override
+    public CloseableIterator shallowIterator() {
+        return new CloseableIterator() {
+            private boolean hasNext = true;
+            @Override
+            public void close() {
+
+            }
+
+            @Override
+            public boolean hasNext() {
+                return hasNext;
+            }
+
+            @Override
+            public Record next() {
+                if (!hasNext)
+                    throw new NoSuchElementException();
+                hasNext = false;
+                return DefaultRecord.readAllRecords(sizeInBytes(), attributes(), baseOffset(),
+                    firstTimestamp(), baseSequence(), buffer);
+            }
+
+            @Override
+            public void remove() {
+                throw new UnsupportedOperationException();
+            }
+        };
+    }
+
     @Override
     public CloseableIterator streamingIterator(BufferSupplier bufferSupplier) {
         if (isCompressed())
diff --git a/clients/src/main/java/org/apache/kafka/common/record/FileLogInputStream.java b/clients/src/main/java/org/apache/kafka/common/record/FileLogInputStream.java
index 15c09dea32c8d..eb04705af2e48 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/FileLogInputStream.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/FileLogInputStream.java
@@ -157,6 +157,12 @@ public CloseableIterator streamingIterator(BufferSupplier bufferSupplier
             return loadFullBatch().streamingIterator(bufferSupplier);
         }
 
+        @Override
+        public CloseableIterator shallowIterator() {
+            // Not implemented
+            throw new UnsupportedOperationException();
+        }
+
         @Override
         public boolean isValid() {
             return loadFullBatch().isValid();
diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java
index fd5a680cda835..95ee13ea1c53c 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java
@@ -69,6 +69,7 @@ public void write(int b) {
     // Used to append records, may compress data on the fly
     private DataOutputStream appendStream;
     private boolean isTransactional;
+    private boolean usePassthrough = false;
     private long producerId;
     private short producerEpoch;
     private int baseSequence;
@@ -109,7 +110,6 @@ public MemoryRecordsBuilder(ByteBufferOutputStream bufferStream,
 
         this.magic = magic;
         this.timestampType = timestampType;
-        this.compressionType = compressionType;
         this.baseOffset = baseOffset;
         this.logAppendTime = logAppendTime;
         this.numRecords = 0;
@@ -124,7 +124,15 @@ public MemoryRecordsBuilder(ByteBufferOutputStream bufferStream,
         this.partitionLeaderEpoch = partitionLeaderEpoch;
         this.writeLimit = writeLimit;
         this.initialPosition = bufferStream.position();
-        this.batchHeaderSizeInBytes = AbstractRecords.recordBatchHeaderSizeInBytes(magic, compressionType);
+
+        if (compressionType == CompressionType.PASSTHROUGH) {
+            usePassthrough = true;
+            this.compressionType = CompressionType.NONE;
+        } else {
+            this.compressionType = compressionType;
+        }
+
+        this.batchHeaderSizeInBytes = usePassthrough ? 0 : AbstractRecords.recordBatchHeaderSizeInBytes(magic, this.compressionType);
 
         bufferStream.position(initialPosition + batchHeaderSizeInBytes);
         this.bufferStream = bufferStream;
@@ -317,9 +325,11 @@ public void close() {
             buffer().position(initialPosition);
             builtRecords = MemoryRecords.EMPTY;
         } else {
-            if (magic > RecordBatch.MAGIC_VALUE_V1)
-                this.actualCompressionRatio = (float) writeDefaultBatchHeader() / this.uncompressedRecordsSizeInBytes;
-            else if (compressionType != CompressionType.NONE)
+            if (magic > RecordBatch.MAGIC_VALUE_V1) {
+                if (!usePassthrough) {
+                    this.actualCompressionRatio = (float) writeDefaultBatchHeader() / this.uncompressedRecordsSizeInBytes;
+                }
+            } else if (compressionType != CompressionType.NONE)
                 this.actualCompressionRatio = (float) writeLegacyCompressedWrapperHeader() / this.uncompressedRecordsSizeInBytes;
 
             ByteBuffer buffer = buffer().duplicate();
@@ -419,7 +429,11 @@ private Long appendWithOffset(long offset, boolean isControlRecord, long timesta
                 appendDefaultRecord(offset, timestamp, key, value, headers);
                 return null;
             } else {
-                return appendLegacyRecord(offset, timestamp, key, value, magic);
+                if (usePassthrough) {
+                    return appendPassthroughLegacyBatch(timestamp, value);
+                } else {
+                    return appendLegacyRecord(offset, timestamp, key, value, magic);
+                }
             }
         } catch (IOException e) {
             throw new KafkaException("I/O exception when writing to the append stream, closing", e);
@@ -617,6 +631,31 @@ public void appendUncheckedWithOffset(long offset, SimpleRecord record) throws I
         }
     }
 
+    /**
+     * Append an already assembled (and compressed) batch. The offset is always recorded as 0 as the value is
+     * a single complete batch. If multiple batches are appended, they will be handled with request aggregation on
+     * the broker
+     * @param timestamp
+     * @param value
+     * @return crc of the record
+     */
+    public long appendPassthroughLegacyBatch(long timestamp, ByteBuffer value) {
+        try {
+            ensureOpenForRecordAppend();
+            // For passthrough compression (from shallow iterator), the value is a wrapper message
+            LegacyRecord wrapperMessage = new LegacyRecord(value);
+
+            int size = LegacyRecord.recordSize(wrapperMessage.magic(), wrapperMessage.key(), wrapperMessage.value());
+            AbstractLegacyRecordBatch.writeHeader(appendStream, toInnerOffset(0L), size);
+
+            long crc = LegacyRecord.write(appendStream, wrapperMessage.magic(), timestamp, wrapperMessage.key(), wrapperMessage.value(), wrapperMessage.compressionType(), timestampType);
+            recordWritten(0L, timestamp, size + Records.LOG_OVERHEAD);
+            return crc;
+        } catch (IOException e) {
+            throw new KafkaException("I/O exception when writing to the legacy append stream, closing", e);
+        }
+    }
+
     /**
      * Append a record at the next sequential offset.
      * @param record the record to add
@@ -653,12 +692,23 @@ public void append(LegacyRecord record) {
         appendWithOffset(nextSequentialOffset(), record);
     }
 
+    /**
+     * Write the entire buffer to `out` as-it-is and return its size
+     */
+    public int writePassthrough(DataOutputStream out,
+        ByteBuffer buffer) throws IOException {
+        int bufferSize = buffer.remaining();
+        Utils.writeTo(out, buffer, bufferSize);
+        return bufferSize;
+    }
+
     private void appendDefaultRecord(long offset, long timestamp, ByteBuffer key, ByteBuffer value,
                                      Header[] headers) throws IOException {
         ensureOpenForRecordAppend();
         int offsetDelta = (int) (offset - baseOffset);
         long timestampDelta = timestamp - firstTimestamp;
-        int sizeInBytes = DefaultRecord.writeTo(appendStream, offsetDelta, timestampDelta, key, value, headers);
+        int sizeInBytes = usePassthrough ? writePassthrough(appendStream, value) :
+            DefaultRecord.writeTo(appendStream, offsetDelta, timestampDelta, key, value, headers);
         recordWritten(offset, timestamp, sizeInBytes);
     }
 
@@ -757,6 +807,11 @@ public boolean hasRoomFor(long timestamp, ByteBuffer key, ByteBuffer value, Head
         if (numRecords == 0)
             return true;
 
+        // For passthrough V2, ensure one producerBatch only has one DefaultRecordBatch, and since
+        // in this case, DefaultRecordBatch is the value part of a DefaultRecord, so we only allow one record
+        if (magic >= RecordBatch.MAGIC_VALUE_V2 && usePassthrough)
+            return false;
+
         final int recordSize;
         if (magic < RecordBatch.MAGIC_VALUE_V2) {
             recordSize = Records.LOG_OVERHEAD + LegacyRecord.recordSize(magic, key, value);
diff --git a/clients/src/main/java/org/apache/kafka/common/record/RecordBatch.java b/clients/src/main/java/org/apache/kafka/common/record/RecordBatch.java
index 65a6a95fbe41f..2a8b547f09df1 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/RecordBatch.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/RecordBatch.java
@@ -230,6 +230,16 @@ public interface RecordBatch extends Iterable {
      */
     CloseableIterator streamingIterator(BufferSupplier decompressionBufferSupplier);
 
+    /**
+     * Return a iterator which does shallow iteration over the record batch, i.e. no decompression.
+     * If the record batch is not compressed, it will also return the entire recordBatch.
+     * 1. For V0/V1, since the record and recordBatch shares the same schema, the behavior is not changed, i.e. it will return the entire
+     *    recordBatch and send it to the producer.
+     * 2. For V2, it will also return the entire recordBatch and later wrap the recordBatch as a single record and send it to producer.
+     * @return The closeable iterator
+     */
+    CloseableIterator shallowIterator();
+
     /**
      * Check whether this is a control batch (i.e. whether the control bit is set in the batch attributes).
      * For magic versions prior to 2, this is always false.
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
index a31e77a8c3ccd..10027194bed9c 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
@@ -2041,6 +2041,7 @@ private KafkaConsumer newConsumer(Time time,
         int fetchSize = 1024 * 1024;
         int maxPollRecords = Integer.MAX_VALUE;
         boolean checkCrcs = true;
+        boolean usePassthrough = false;
         int rebalanceTimeoutMs = 60000;
 
         Deserializer keyDeserializer = new StringDeserializer();
@@ -2085,6 +2086,7 @@ private KafkaConsumer newConsumer(Time time,
                 maxPollRecords,
                 checkCrcs,
                 "",
+                usePassthrough,
                 keyDeserializer,
                 valueDeserializer,
                 metadata,
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
index 436c40b76a3c3..feb9a9d40c874 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
@@ -118,6 +118,7 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static java.util.Arrays.asList;
 import static java.util.Collections.singleton;
 import static java.util.Collections.singletonMap;
 import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID;
@@ -164,6 +165,8 @@ public class FetcherTest {
     private ConsumerNetworkClient consumerClient;
     private Fetcher fetcher;
 
+    private boolean testPassthrough = false;
+
     private MemoryRecords records;
     private MemoryRecords nextRecords;
     private MemoryRecords emptyRecords;
@@ -615,6 +618,43 @@ public void testParseInvalidRecordBatch() {
         }
     }
 
+    @Test
+    public void testFetchEntireBatchWithShallowIteratorEnabled() {
+        for (byte magic : asList(RecordBatch.MAGIC_VALUE_V0, RecordBatch.MAGIC_VALUE_V1, RecordBatch.MAGIC_VALUE_V2)) {
+            ByteBuffer buffer = ByteBuffer.allocate(1024);
+            // create compressed batches
+            MemoryRecordsBuilder builder =
+                new MemoryRecordsBuilder(buffer, magic, CompressionType.GZIP, TimestampType.CREATE_TIME,
+                    0L, 10L, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, false, 0, 1024);
+
+            builder.append(10L, "key".getBytes(), "value".getBytes());
+            builder.append(11L, "key1".getBytes(), "value1".getBytes());
+            builder.append(12L, "key2".getBytes(), "value2".getBytes());
+            builder.append(13L, "key3".getBytes(), "value3".getBytes());
+
+            builder.close();
+            buffer.flip();
+
+            // create new fetcher with shallow iterator enabled and maxPollRecords set to 6
+            testPassthrough = true;
+            buildFetcher(6);
+
+            assignFromUser(singleton(tp0));
+            subscriptions.seek(tp0, 0);
+
+            // normal fetch
+            assertEquals(1, fetcher.sendFetches());
+            client.prepareResponse(fullFetchResponse(tp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0));
+            consumerClient.poll(time.timer(0));
+
+            Map>> partitionRecords = fetchedRecords();
+            List> records = partitionRecords.get(tp0);
+            assertEquals(1, records.size());
+        }
+    }
+
+
+
     @Test
     public void testHeaders() {
         buildFetcher();
@@ -3145,6 +3185,7 @@ public void testFetcherConcurrency() throws Exception {
                 2 * numPartitions,
                 true,
                 "",
+                testPassthrough,
                 new ByteArrayDeserializer(),
                 new ByteArrayDeserializer(),
                 metadata,
@@ -4023,6 +4064,7 @@ private  void buildFetcher(MetricConfig metricConfig,
                 maxPollRecords,
                 true, // check crc
                 "",
+                testPassthrough,
                 keyDeserializer,
                 valueDeserializer,
                 metadata,
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/ProducerBatchTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/ProducerBatchTest.java
index f9887f9033e81..fd25c844afd95 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/ProducerBatchTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/ProducerBatchTest.java
@@ -158,6 +158,9 @@ public void testAppendedChecksumMagicV0AndV1() {
     @Test
     public void testSplitPreservesHeaders() {
         for (CompressionType compressionType : CompressionType.values()) {
+            if (compressionType == CompressionType.PASSTHROUGH)
+                continue;
+
             MemoryRecordsBuilder builder = MemoryRecords.builder(
                     ByteBuffer.allocate(1024),
                     MAGIC_VALUE_V2,
@@ -194,7 +197,8 @@ public void testSplitPreservesHeaders() {
     public void testSplitPreservesMagicAndCompressionType() {
         for (byte magic : Arrays.asList(MAGIC_VALUE_V0, MAGIC_VALUE_V1, MAGIC_VALUE_V2)) {
             for (CompressionType compressionType : CompressionType.values()) {
-                if (compressionType == CompressionType.NONE && magic < MAGIC_VALUE_V2)
+                if ((compressionType == CompressionType.NONE && magic < MAGIC_VALUE_V2) ||
+                    compressionType == CompressionType.PASSTHROUGH)
                     continue;
 
                 if (compressionType == CompressionType.ZSTD && magic < MAGIC_VALUE_V2)
diff --git a/clients/src/test/java/org/apache/kafka/common/record/FileLogInputStreamTest.java b/clients/src/test/java/org/apache/kafka/common/record/FileLogInputStreamTest.java
index 783a5b531ef27..eba1a9cf48ea5 100644
--- a/clients/src/test/java/org/apache/kafka/common/record/FileLogInputStreamTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/record/FileLogInputStreamTest.java
@@ -283,8 +283,11 @@ private void assertGenericRecordBatchData(RecordBatch batch, long baseOffset, lo
     public static Collection data() {
         List values = new ArrayList<>();
         for (byte magic : asList(MAGIC_VALUE_V0, MAGIC_VALUE_V1, MAGIC_VALUE_V2))
-            for (CompressionType type: CompressionType.values())
-                values.add(new Object[] {magic, type});
+            for (CompressionType type: CompressionType.values()) {
+                if (type == CompressionType.PASSTHROUGH)
+                    continue;
+                values.add(new Object[]{magic, type});
+            }
         return values;
     }
 }
diff --git a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java
index 522915f064034..ccd62b3ba8bcb 100644
--- a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java
@@ -686,8 +686,11 @@ public void shouldThrowIllegalStateExceptionOnAppendWhenAborted() {
     public static Collection data() {
         List values = new ArrayList<>();
         for (int bufferOffset : Arrays.asList(0, 15))
-            for (CompressionType compressionType : CompressionType.values())
-                values.add(new Object[] {bufferOffset, compressionType});
+            for (CompressionType compressionType : CompressionType.values()) {
+                if (compressionType == CompressionType.PASSTHROUGH)
+                    continue;
+                values.add(new Object[]{bufferOffset, compressionType});
+            }
         return values;
     }
 
diff --git a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsTest.java b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsTest.java
index b8824d3a8276c..23f4dca9fd117 100644
--- a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsTest.java
@@ -20,6 +20,7 @@
 import org.apache.kafka.common.errors.CorruptRecordException;
 import org.apache.kafka.common.header.internals.RecordHeaders;
 import org.apache.kafka.common.record.MemoryRecords.RecordFilter.BatchRetention;
+import org.apache.kafka.common.utils.CloseableIterator;
 import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.test.TestUtils;
 import org.junit.Test;
@@ -154,6 +155,125 @@ public void testIterator() {
         }
     }
 
+    @Test
+    public void testShallowIteration() {
+        assumeAtLeastV2OrNotZstd();
+        // Create the record batch
+        ByteBuffer buffer = ByteBuffer.allocate(1024);
+
+        MemoryRecordsBuilder builder1 = new MemoryRecordsBuilder(buffer, magic, compression,
+            TimestampType.CREATE_TIME, firstOffset, logAppendTime, pid, epoch, firstSequence, false, false,
+            partitionLeaderEpoch, buffer.limit());
+
+        SimpleRecord[] records = new SimpleRecord[] {
+            new SimpleRecord(0L, "a".getBytes(), "1".getBytes()),
+            new SimpleRecord(0L, "b".getBytes(), "2".getBytes()),
+            new SimpleRecord(0L, "c".getBytes(), "3".getBytes()),
+            new SimpleRecord(0L, "d".getBytes(), "4".getBytes())
+        };
+
+        for (SimpleRecord record : records)
+            builder1.append(record);
+
+        // Build MemoryRecords with normal appender
+        MemoryRecords records1 = builder1.build();
+        List recordBatches = Utils.toList(records1.batches().iterator());
+
+        // Shallow iterate over the records, if compression is NONE, there should be 4 entries during shallow iteration;
+        // otherwise, there should be only 1 entry
+        int recordCount = 0;
+
+        for (int i = 0; i < recordBatches.size(); i++) {
+            CloseableIterator iter = recordBatches.get(i).shallowIterator();
+
+            while (iter.hasNext()) {
+                Record curRecord = iter.next();
+                curRecord.ensureValid();
+                recordCount++;
+            }
+        }
+
+        int expectedRecordCount = compression == CompressionType.NONE ?
+            (magic == RecordBatch.MAGIC_VALUE_V2 ? 1 : 4) : 1;
+        assertEquals(expectedRecordCount, recordCount);
+    }
+
+    @Test
+    public void testIteratorWithProducerCompression() {
+        assumeAtLeastV2OrNotZstd();
+        ByteBuffer buffer = ByteBuffer.allocate(1024);
+
+        MemoryRecordsBuilder builder1 = new MemoryRecordsBuilder(buffer, magic, compression,
+            TimestampType.CREATE_TIME, firstOffset, logAppendTime, pid, epoch, firstSequence, false, false,
+            partitionLeaderEpoch, buffer.limit());
+
+        SimpleRecord[] records = new SimpleRecord[] {
+            new SimpleRecord(0L, "a".getBytes(), "1".getBytes()),
+            new SimpleRecord(0L, "b".getBytes(), "2".getBytes()),
+            new SimpleRecord(0L, "c".getBytes(), "3".getBytes()),
+            new SimpleRecord(0L, "d".getBytes(), "4".getBytes())
+        };
+
+        for (SimpleRecord record : records)
+            builder1.append(record);
+
+        // Build MemoryRecords with normal appender
+        MemoryRecords records1 = builder1.build();
+
+        List recordBatches = Utils.toList(records1.batches().iterator());
+
+        ByteBuffer buffer2 = ByteBuffer.allocate(1024);
+
+        MemoryRecordsBuilder builder2 = new MemoryRecordsBuilder(buffer2, magic, CompressionType.PASSTHROUGH,
+            TimestampType.CREATE_TIME, firstOffset, logAppendTime, pid, epoch, firstSequence, false, false,
+            partitionLeaderEpoch, buffer2.limit());
+
+        List recordList = new ArrayList<>();
+        // Part 1: Shallow iterate over the original record batch and use passthrough to append to new buffer
+        for (RecordBatch currentBatch : recordBatches) {
+            CloseableIterator currentRecords = currentBatch.shallowIterator();
+            while (currentRecords.hasNext()) {
+                Record curRecord = currentRecords.next();
+                recordList.add(curRecord);
+
+                if (magic >= RecordBatch.MAGIC_VALUE_V2) {
+                    builder2.append(0, null, curRecord.value());
+                } else {
+                    builder2.append(0, null, ((AbstractLegacyRecordBatch) curRecord).outerRecord().buffer());
+                }
+            }
+        }
+
+        MemoryRecords records2 = builder2.build();
+
+        List list2 = Utils.toList(records2.batches().iterator());
+
+        // Part 2: the newly produced records should be the same as the original ones
+        for (int i = 0; i < list2.size(); i++) {
+            CloseableIterator origRecIter = recordBatches.get(i).shallowIterator();
+            CloseableIterator curRecIter = list2.get(i).shallowIterator();
+
+            while (curRecIter.hasNext()) {
+                Record curRecord = curRecIter.next(), origRecord = origRecIter.next();
+
+                assertEquals(magic == RecordBatch.MAGIC_VALUE_V2 ? firstOffset : 0, curRecord.offset());
+
+                if (magic == RecordBatch.MAGIC_VALUE_V2) {
+                    assertEquals(curRecord, origRecord);
+                } else {
+                    assertEquals(((AbstractLegacyRecordBatch) curRecord).outerRecord().buffer(),
+                                 ((AbstractLegacyRecordBatch) origRecord).outerRecord().buffer());
+                }
+
+                curRecord.ensureValid();
+            }
+
+            assertFalse(origRecIter.hasNext());
+        }
+    }
+
+
+
     @Test
     public void testHasRoomForMethod() {
         assumeAtLeastV2OrNotZstd();
@@ -903,8 +1023,11 @@ public static Collection data() {
         List values = new ArrayList<>();
         for (long firstOffset : asList(0L, 57L))
             for (byte magic : asList(RecordBatch.MAGIC_VALUE_V0, RecordBatch.MAGIC_VALUE_V1, RecordBatch.MAGIC_VALUE_V2))
-                for (CompressionType type: CompressionType.values())
-                    values.add(new Object[] {magic, firstOffset, type});
+                for (CompressionType type: CompressionType.values()) {
+                    if (type == CompressionType.PASSTHROUGH)
+                        continue;
+                    values.add(new Object[]{magic, firstOffset, type});
+                }
         return values;
     }
 
diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index 4cf45a6976b22..94c97f9e85e4b 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -1357,11 +1357,6 @@ class Log(@volatile var dir: File,
     var lastOffsetOfFirstBatch = -1L
 
     for (batch <- records.batches.asScala) {
-      // we only validate V2 and higher to avoid potential compatibility issues with older clients
-      if (batch.magic >= RecordBatch.MAGIC_VALUE_V2 && origin == AppendOrigin.Client && batch.baseOffset != 0)
-        throw new InvalidRecordException(s"The baseOffset of the record batch in the append to $topicPartition should " +
-          s"be 0, but it is ${batch.baseOffset}")
-
       // update the first offset if on the first message. For magic versions older than 2, we use the last offset
       // to avoid the need to decompress the data (the last offset can be obtained directly from the wrapper message).
       // For magic version 2, we can get the first offset directly from the batch header.
diff --git a/core/src/main/scala/kafka/tools/MirrorMaker.scala b/core/src/main/scala/kafka/tools/MirrorMaker.scala
index de4e1097b0367..5d6ea9ab0d288 100755
--- a/core/src/main/scala/kafka/tools/MirrorMaker.scala
+++ b/core/src/main/scala/kafka/tools/MirrorMaker.scala
@@ -510,6 +510,9 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
       .ofType(classOf[String])
       .defaultsTo("true")
 
+    val passthroughCompressionOpt = parser.accepts("enable.passthrough",
+      "When enabled, it avoids decompressing consumed record batches and doesn't re-compress in the producer")
+
     options = parser.parse(args: _*)
 
     def checkArgs() = {
@@ -546,6 +549,10 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
       maybeSetDefaultProperty(producerProps, ProducerConfig.MAX_BLOCK_MS_CONFIG, Long.MaxValue.toString)
       maybeSetDefaultProperty(producerProps, ProducerConfig.ACKS_CONFIG, "all")
       maybeSetDefaultProperty(producerProps, ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "1")
+      if (options.has(passthroughCompressionOpt)) {
+        consumerProps.setProperty("enable.shallow.iterator", "true")
+        producerProps.setProperty(ProducerConfig.COMPRESSION_TYPE_CONFIG, "passthrough")
+      }
       // Always set producer key and value serializer to ByteArraySerializer.
       producerProps.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, classOf[ByteArraySerializer].getName)
       producerProps.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, classOf[ByteArraySerializer].getName)
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerParameterizedIntegrationTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerParameterizedIntegrationTest.scala
index 815405319df8c..0d1e311659571 100755
--- a/core/src/test/scala/unit/kafka/log/LogCleanerParameterizedIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerParameterizedIntegrationTest.scala
@@ -328,7 +328,7 @@ object LogCleanerParameterizedIntegrationTest {
   @Parameters
   def parameters: java.util.Collection[Array[String]] = {
     val list = new java.util.ArrayList[Array[String]]()
-    for (codec <- CompressionType.values)
+    for (codec <- CompressionType.values.filter(_!=CompressionType.PASSTHROUGH))
       list.add(Array(codec.name))
     list
   }

From 5fb8ee38ff4d64f2ec06526433fe104ba52c7ac9 Mon Sep 17 00:00:00 2001
From: Ke Hu 
Date: Wed, 17 Oct 2018 18:27:32 -0700
Subject: [PATCH 1006/1071] [LI-HOTFIX] Fix integ tests with added passthrough
 compression codec TICKET = LI_DESCRIPTION =

Fix integ tests with added passthrough compression codec

RB=1456181
R=nramesh
A=nramesh

EXIT_CRITERIA = MANUAL ["When we decide to get rid of passthrough feature"]
---
 core/src/test/scala/unit/kafka/log/BrokerCompressionTest.scala  | 1 +
 .../scala/unit/kafka/log/LogCleanerLagIntegrationTest.scala     | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/test/scala/unit/kafka/log/BrokerCompressionTest.scala b/core/src/test/scala/unit/kafka/log/BrokerCompressionTest.scala
index 8e4093ea83b39..717697a7c69d3 100755
--- a/core/src/test/scala/unit/kafka/log/BrokerCompressionTest.scala
+++ b/core/src/test/scala/unit/kafka/log/BrokerCompressionTest.scala
@@ -86,6 +86,7 @@ object BrokerCompressionTest {
   def parameters: Collection[Array[String]] = {
     (for (brokerCompression <- BrokerCompressionCodec.brokerCompressionOptions;
          messageCompression <- CompressionType.values
+      if messageCompression != CompressionType.PASSTHROUGH
     ) yield Array(messageCompression.name, brokerCompression)).asJava
   }
 }
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerLagIntegrationTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerLagIntegrationTest.scala
index 0232e5772d648..ffdf1ad0b63d3 100644
--- a/core/src/test/scala/unit/kafka/log/LogCleanerLagIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerLagIntegrationTest.scala
@@ -128,7 +128,7 @@ object LogCleanerLagIntegrationTest {
   @Parameters
   def parameters: java.util.Collection[Array[String]] = {
     val list = new java.util.ArrayList[Array[String]]()
-    for (codec <- CompressionType.values)
+    for (codec <- CompressionType.values.filter(_!=CompressionType.PASSTHROUGH))
       list.add(Array(codec.name))
     list
   }

From 9007e8febb1954d8a8d5932cdee8c0b638fc6864 Mon Sep 17 00:00:00 2001
From: Xiongqi Wesley Wu 
Date: Tue, 16 Oct 2018 16:36:11 -0700
Subject: [PATCH 1007/1071] [LI-HOTFIX] LIKAFKA-18913: fix bytebuffer size
 miscalculation when deallocating expanded buffer TICKET = LI_DESCRIPTION =
 When returning allocated memory buffer to available memory pool, do not
 return more than allocated size.

RB=1454548
G=Kafka-Code-Reviews
R=rrosenbl,smccauli
A=smccauli,mgharat

EXIT_CRITERIA = MANUAL ["NONE"]
---
 .../producer/internals/BufferPool.java        |  2 +-
 .../producer/internals/BufferPoolTest.java    | 21 +++++++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/BufferPool.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/BufferPool.java
index f96080648acd2..70fc3a62605b2 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/BufferPool.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/BufferPool.java
@@ -274,7 +274,7 @@ public void deallocate(ByteBuffer buffer, int size) {
                     this.nextOvermemoryWarn = this.time.milliseconds() + TimeUnit.HOURS.toMillis(1);
                 }
             } else {
-                long freeMem = Math.min(Math.max(buffer.capacity(), size), this.totalMemory - availableMemory);
+                long freeMem = Math.min(size, this.totalMemory - availableMemory);
                 this.nonPooledAvailableMemory += freeMem;
             }
             Condition moreMem = this.waiters.peekFirst();
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/BufferPoolTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/BufferPoolTest.java
index e6bb5d9264979..2c10284adf260 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/BufferPoolTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/BufferPoolTest.java
@@ -368,6 +368,27 @@ public void dedupeDeallocations() throws Exception {
         assertEquals(bufferPool.totalMemory(), bufferPool.availableMemory());
     }
 
+    @Test
+    public void testExpandBufferDeallocations() throws Exception {
+
+        int batchSize = 512;
+        BufferPool bufferPool = new BufferPool(2048, batchSize, metrics, time, metricGroup);
+
+        ByteBuffer bbuf1 = bufferPool.allocate(batchSize, 1);
+        ByteBuffer bbuf2 = bufferPool.allocate(batchSize, 1);
+
+        int expandSize = batchSize * 2;
+        // expand bbuf1 to 2x of the original size
+        bbuf1 = ByteBuffer.allocate(expandSize);
+
+        // return initially allocated size of bbuf1 to bufferpool
+        bufferPool.deallocate(bbuf1, batchSize);
+        assertEquals(bufferPool.totalMemory(), bufferPool.availableMemory() + batchSize);
+
+        bufferPool.deallocate(bbuf2, batchSize);
+        assertEquals(bufferPool.totalMemory(), bufferPool.availableMemory());
+    }
+
     public static class StressTestThread extends Thread {
         private final int iterations;
         private final BufferPool pool;

From 737c9759a208050f5b7bc8de9ba6b7f89883bf81 Mon Sep 17 00:00:00 2001
From: Zhanxiang Huang 
Date: Wed, 18 Jul 2018 16:20:49 -0700
Subject: [PATCH 1008/1071] [LI-HOTFIX] LIKAFKA-18349 - Reuse the same
 UpdateMetadataRequest object and struct to reduce controller memory usage
 TICKET = KAFKA-7186 LI_DESCRIPTION =

During controller failover and broker changes, it sends out UpdateMetadataRequest to all brokers in the cluster containing the states for all partitions and live brokers. The current implementation will instantiate the UpdateMetadataRequest object and its serialized form (Struct) for # of brokers times, which causes OOM if the memory exceeds the configure JVM heap size before GC kicks in. We have seen this issue in the production environment for multiple times.
For example, if we have 100 brokers in the cluster and each broker is the leader of 2k partitions, the extra memory usage introduced by controller trying to send out UpdateMetadataRequest is around:
```
 * <# of brokers> *  = 250B * 100 * 200k = 5GB
```
This patch avoids the recreation of UpdateMetadataReuqest and struct objects if the same builder object is used to build the request. This can significantly reduce memory footprint in Controller based on the above calculation.

RB=1364716
BUG=LIKAFKA-18349
G=Kafka-Code-Reviews
R=luwang,okaraman,mgharat
A=mgharat

EXIT_CRITERIA = TICKET [KAFKA-7186]
---
 checkstyle/suppressions.xml                     |  3 +++
 .../common/requests/UpdateMetadataRequest.java  | 17 +++++++++++++++--
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index 2e493efa5073a..3791b7354d00e 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -7,6 +7,9 @@
 
 
     
+    
+    
 
     
      partitionStates;
         private final List liveBrokers;
 
+        // LIKAFKA-18349 - Cache the UpdateMetadataRequest Objects to reduce memory usage
+        private final Map requestCache = new HashMap<>();
+
         public Builder(short version, int controllerId, int controllerEpoch, long brokerEpoch,
                        List partitionStates, List liveBrokers) {
             super(ApiKeys.UPDATE_METADATA, version, controllerId, controllerEpoch, brokerEpoch);
@@ -54,6 +57,8 @@ public Builder(short version, int controllerId, int controllerEpoch, long broker
 
         @Override
         public UpdateMetadataRequest build(short version) {
+            UpdateMetadataRequest updateMetadataRequest = requestCache.get(version);
+            if (updateMetadataRequest == null) {
             if (version < 3) {
                 for (UpdateMetadataBroker broker : liveBrokers) {
                     if (version == 0) {
@@ -88,7 +93,10 @@ public UpdateMetadataRequest build(short version) {
                 data.setUngroupedPartitionStates(partitionStates);
             }
 
-            return new UpdateMetadataRequest(data, version);
+            updateMetadataRequest = new UpdateMetadataRequest(data, version);
+            requestCache.put(version, updateMetadataRequest);
+            }
+            return updateMetadataRequest;
         }
 
         private static Map groupByTopic(List partitionStates) {
@@ -118,6 +126,8 @@ public String toString() {
     }
 
     private final UpdateMetadataRequestData data;
+    // LIKAFKA-18349 - Cache the UpdateMetadataRequest struct to reduce memory usage
+    private Struct struct = null;
 
     UpdateMetadataRequest(UpdateMetadataRequestData data, short version) {
         super(ApiKeys.UPDATE_METADATA, version);
@@ -206,7 +216,10 @@ public List liveBrokers() {
 
     @Override
     protected Struct toStruct() {
-        return data.toStruct(version());
+        if (struct == null) {
+            struct = data.toStruct(version());
+        }
+        return struct;
     }
 
     // Visible for testing

From 8c6e11cff063223078481f661c5b85028ac02b21 Mon Sep 17 00:00:00 2001
From: Zhanxiang Huang 
Date: Tue, 23 Oct 2018 15:35:33 -0700
Subject: [PATCH 1009/1071] [LI-HOTFIX] Protect UpdateMetadataRequest struct
 cache with a lock TICKET = LI_DESCRIPTION =
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This patch fixes a concurrency bug introduced by HOTFIX “LIKAFKA-18349 - Reuse the same UpdateMetadataRequest object and struct to reduce controller memory usage”

RB=1461329
G=Kafka-Code-Reviews
R=okaraman,jkoshy,mgharat
A=jkoshy

EXIT_CRITERIA = MANUAL ["Same as the EXIT_CRITERIA of LIKAFKA-18349 HOTFIX"]
---
 .../common/requests/UpdateMetadataRequest.java | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
index aa5820c16ca81..b8c6089a8c7eb 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
@@ -16,6 +16,8 @@
  */
 package org.apache.kafka.common.requests;
 
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 import org.apache.kafka.common.errors.UnsupportedVersionException;
 import org.apache.kafka.common.message.UpdateMetadataRequestData;
 import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataBroker;
@@ -44,6 +46,7 @@ public class UpdateMetadataRequest extends AbstractControlRequest {
     public static class Builder extends AbstractControlRequest.Builder {
         private final List partitionStates;
         private final List liveBrokers;
+        private Lock buildLock = new ReentrantLock();
 
         // LIKAFKA-18349 - Cache the UpdateMetadataRequest Objects to reduce memory usage
         private final Map requestCache = new HashMap<>();
@@ -57,6 +60,9 @@ public Builder(short version, int controllerId, int controllerEpoch, long broker
 
         @Override
         public UpdateMetadataRequest build(short version) {
+            // the following inner blocks are not indented in order to make hotfix cherry-picking easier
+            buildLock.lock();
+            try {
             UpdateMetadataRequest updateMetadataRequest = requestCache.get(version);
             if (updateMetadataRequest == null) {
             if (version < 3) {
@@ -97,6 +103,9 @@ public UpdateMetadataRequest build(short version) {
             requestCache.put(version, updateMetadataRequest);
             }
             return updateMetadataRequest;
+            } finally {
+                buildLock.unlock();
+            }
         }
 
         private static Map groupByTopic(List partitionStates) {
@@ -128,6 +137,7 @@ public String toString() {
     private final UpdateMetadataRequestData data;
     // LIKAFKA-18349 - Cache the UpdateMetadataRequest struct to reduce memory usage
     private Struct struct = null;
+    private Lock structLock = new ReentrantLock();
 
     UpdateMetadataRequest(UpdateMetadataRequestData data, short version) {
         super(ApiKeys.UPDATE_METADATA, version);
@@ -216,10 +226,16 @@ public List liveBrokers() {
 
     @Override
     protected Struct toStruct() {
+        // the following inner blocks are not indented in order to make hotfix cherry-picking easier
+        structLock.lock();
+        try {
         if (struct == null) {
-            struct = data.toStruct(version());
+        struct = data.toStruct(version());
         }
         return struct;
+        } finally {
+            structLock.unlock();
+        }
     }
 
     // Visible for testing

From 5eae6cd9662e54d020fe35b286c93a1567f39777 Mon Sep 17 00:00:00 2001
From: Xiongqi Wesley Wu 
Date: Fri, 2 Nov 2018 15:57:39 -0700
Subject: [PATCH 1010/1071] [LI-HOTFIX] LIKAFKA-20010 Disable/Enable
 auto-topic-creation at runtime TICKET = KAFKA-7650 LI_DESCRIPTION = enable
 dynamic broker config for auto-topic-creation.

There are two levels of dynamic configuration: cluster level and per broker level.

The configuration override order :
local config ==> (overrided by) dynamic cluster level configuration => (overrided by) dynamic per broker configuration

RB=1473779
BUG=LIKAFKA-20010
G=Kafka-Code-Reviews
R=zahuang
A=zahuang

EXIT_CRITERIA = TICKET [KAFKA-7650]
---
 core/src/main/scala/kafka/server/DynamicBrokerConfig.scala | 1 +
 core/src/main/scala/kafka/server/KafkaConfig.scala         | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
index 75081847e3e98..b510e70d73698 100755
--- a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
+++ b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
@@ -81,6 +81,7 @@ object DynamicBrokerConfig {
     DynamicLogConfig.ReconfigurableConfigs ++
     DynamicThreadPool.ReconfigurableConfigs ++
     Set(KafkaConfig.MetricReporterClassesProp) ++
+    Set(KafkaConfig.AutoCreateTopicsEnableProp) ++
     DynamicListenerConfig.ReconfigurableConfigs ++
     SocketServer.ReconfigurableConfigs
 
diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index bcdaa62d61ee1..f7344089907a4 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -1240,7 +1240,7 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO
   val replicaSelectorClassName = Option(getString(KafkaConfig.ReplicaSelectorClassProp))
 
   /** ********* Log Configuration ***********/
-  val autoCreateTopicsEnable = getBoolean(KafkaConfig.AutoCreateTopicsEnableProp)
+  def autoCreateTopicsEnable: java.lang.Boolean = getBoolean(KafkaConfig.AutoCreateTopicsEnableProp)
   val numPartitions = getInt(KafkaConfig.NumPartitionsProp)
   val logDirs = CoreUtils.parseCsvList(Option(getString(KafkaConfig.LogDirsProp)).getOrElse(getString(KafkaConfig.LogDirProp)))
   def logSegmentBytes = getInt(KafkaConfig.LogSegmentBytesProp)

From e3ca505e5cbe4182b10b8b8f21c1b882376a64c9 Mon Sep 17 00:00:00 2001
From: Nacho Solis 
Date: Wed, 12 Sep 2018 23:00:37 -0700
Subject: [PATCH 1011/1071] [LI-HOTFIX] Add note on LinkedIn Kafka Branch to
 README TICKET = LI_DESCRIPTION =

EXIT_CRITERIA = MANUAL ["NONE"]
---
 README.md | 43 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/README.md b/README.md
index b7081c42298b4..8d43191a03c2e 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,46 @@
+LinkedIn Branch of Apache Kafka
+=================
+
+This is the version of Kafka running at LinkedIn.
+
+Kafka was born at LinkedIn. We run thousands of brokers to deliver trillions of
+messages per day.  We run a slightly modified version of Apache Kafka trunk.
+This branch contains the LinkedIn Kafka release.
+
+This branch is made up of:
+
+* Apache Kafka trunk (upstream) up to some branch point, see *-li* branch name for base version, you'll be able to get the exact commit from git 
+* Cherry-picked commits from upstream after branch point
+* Patches that are on their way upstream but we have deployed internally in the meantime
+* Patches that are of no interest to upstream
+
+We are making this branch available for people interested. We will be
+documenting the changes in the near future with some more detailed explanations
+in the [LinkedIn Engineering Blog](https://engineering.linkedin.com/blog).
+
+If you are interested in learning more, we invite you to our [Streaming
+Meetup](https://www.meetup.com/Stream-Processing-Meetup-LinkedIn/) where we
+discuss streaming technologies like [Kafka](http://kafka.apache.org) and
+[Samza](http://samza.apache.org).
+
+You are encouraged to check out other Kafka projects from LinkedIn:
+
+* [Cruise Control](https://github.com/linkedin/cruise-control)
+* [Li-Apache-Kafka-Clients](https://github.com/linkedin/li-apache-kafka-clients)
+* [Burrow](https://github.com/linkedin/Burrow)
+* [Kafka Monitor](https://github.com/linkedin/kafka-monitor)
+
+### Contributing ###
+
+At this moment we are not accepting external contributions directly. Please
+contribute to [Apache Kafka](http://kafka.apache.org).
+
+For security issues with this branch please review 
+[LinkedIn Security
+Guidelines](https://www.linkedin.com/help/linkedin/answer/62924/security-vulnerabilities?lang=en).
+General Kafka issues should be communicated via the Kafka community.
+
+
 Apache Kafka
 =================
 See our [web site](https://kafka.apache.org) for details on the project.

From 79b66130e916395a8e30cc4dd8e7d601b98a632f Mon Sep 17 00:00:00 2001
From: Virupaksha Swamy 
Date: Thu, 28 Feb 2019 14:22:32 -0800
Subject: [PATCH 1012/1071] [LI-HOTFIX] Rename the heap dump files to contain
 .hprof extension. TICKET = LI_DESCRIPTION =

This is required for HotSpotDiagnostic Mbean to properly dump heap files in case of a failure scenario.

RB=1580053
G=Kafka-Code-Reviews
R=rrosenbl
A=rrosenbl

EXIT_CRITERIA = MANUAL [""]
---
 .../main/java/org/apache/kafka/common/utils/PoisonPill.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/utils/PoisonPill.java b/clients/src/main/java/org/apache/kafka/common/utils/PoisonPill.java
index b98853762ed60..383820d5cc989 100644
--- a/clients/src/main/java/org/apache/kafka/common/utils/PoisonPill.java
+++ b/clients/src/main/java/org/apache/kafka/common/utils/PoisonPill.java
@@ -93,8 +93,8 @@ public void run() {
         //(overwriting any previous such file). this attempts to guarantee there are
         //at most 2 (potentially large) dump files at any point in time.
 
-        File inProgress = new File(heapDumpFolder, "dump.inprogress");
-        File complete = new File(heapDumpFolder, "dump.complete");
+        File inProgress = new File(heapDumpFolder, "dump.inprogress.hprof");
+        File complete = new File(heapDumpFolder, "dump.complete.hprof");
         if (inProgress.exists() && !inProgress.delete()) {
             System.err.println("unable to delete existing dump file. will not proceed with dump");
             System.err.flush();

From 0d00e86070a5fcbdd41a334d228386ddc2d64572 Mon Sep 17 00:00:00 2001
From: Sean McCauliff 
Date: Mon, 29 Apr 2019 11:32:58 -0700
Subject: [PATCH 1013/1071] [LI-HOTFIX] Increase limits on non-commenting
 source statements (NCSS) style checks (#21) TICKET = LI_DESCRIPTION =

Change the non-commenting source statements (NCSS) style checks to allow the LinkedIn
branch to have more verbose methods, classes and files. This needs to be done so that
we can maintain our own changes outside of open source without needing to refactor
before open source refactors.  The disadvantage of this is that hot fixes submitted to upstream may require refactoring in upstream.

EXIT_CRITERIA = MANUAL ["We don't ever want to move this change upstream even if they would accept it.  Otherwise we may be required to do a large refactor in order to implement a hotfix.  "]
---
 checkstyle/checkstyle.xml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/checkstyle/checkstyle.xml b/checkstyle/checkstyle.xml
index 13cfdb82bd0a8..3cf229d51eb68 100644
--- a/checkstyle/checkstyle.xml
+++ b/checkstyle/checkstyle.xml
@@ -127,8 +127,15 @@
       
     
     
+      
       
-      
+      
+      
+      
+      
+      
     
     
       

From b780ee6a270a1b74da3ab06f21e1b76ab2bc14bb Mon Sep 17 00:00:00 2001
From: Ahmed Al-Mehdi 
Date: Fri, 9 Nov 2018 10:30:22 -0800
Subject: [PATCH 1014/1071] [LI-HOTFIX] Avoid decompressing a batch under
 certain conditions to improve performance (reduce CPU usage). TICKET =
 LI_DESCRIPTION = Avoid decompressing a batch sent in ProduceRequest under the
 following conditions: - Batch sent by producer is greater than V1 - Version
 of batch sent by producer matches the boker message format version

RB=1466111
G=Kafka-Code-Reviews
R=dolin,okaraman,jkoshy,rrakshe,nsolis
A=zahuang,mgharat

EXIT_CRITERIA = MANUAL [""]
---
 core/src/main/scala/kafka/log/Log.scala       |  3 +-
 core/src/main/scala/kafka/log/LogConfig.scala | 13 +++++
 .../main/scala/kafka/log/LogValidator.scala   | 27 +++++++--
 .../main/scala/kafka/server/KafkaConfig.scala |  5 ++
 .../main/scala/kafka/server/KafkaServer.scala |  1 +
 .../test/scala/unit/kafka/log/LogTest.scala   | 58 ++++++++++++++++++-
 6 files changed, 100 insertions(+), 7 deletions(-)

diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index 94c97f9e85e4b..f34160fccff2d 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -1091,7 +1091,8 @@ class Log(@volatile var dir: File,
               leaderEpoch,
               origin,
               interBrokerProtocolVersion,
-              brokerTopicStats)
+              brokerTopicStats,
+              config.producerBatchDecompressionEnable)
           } catch {
             case e: IOException =>
               throw new KafkaException(s"Error validating messages while appending to log $name", e)
diff --git a/core/src/main/scala/kafka/log/LogConfig.scala b/core/src/main/scala/kafka/log/LogConfig.scala
index 4f2671614bb06..ff4c83dfba21f 100755
--- a/core/src/main/scala/kafka/log/LogConfig.scala
+++ b/core/src/main/scala/kafka/log/LogConfig.scala
@@ -65,6 +65,7 @@ object Defaults {
   val FollowerReplicationThrottledReplicas = Collections.emptyList[String]()
   val MaxIdMapSnapshots = kafka.server.Defaults.MaxIdMapSnapshots
   val MessageDownConversionEnable = kafka.server.Defaults.MessageDownConversionEnable
+  val ProducerBatchDecompressionEnable = kafka.server.Defaults.ProducerBatchDecompressionEnable
 }
 
 case class LogConfig(props: java.util.Map[_, _], overriddenConfigs: Set[String] = Set.empty)
@@ -100,6 +101,7 @@ case class LogConfig(props: java.util.Map[_, _], overriddenConfigs: Set[String]
   val LeaderReplicationThrottledReplicas = getList(LogConfig.LeaderReplicationThrottledReplicasProp)
   val FollowerReplicationThrottledReplicas = getList(LogConfig.FollowerReplicationThrottledReplicasProp)
   val messageDownConversionEnable = getBoolean(LogConfig.MessageDownConversionEnableProp)
+  val producerBatchDecompressionEnable = getBoolean(LogConfig.ProducerBatchDecompressionEnableProp)
 
   def randomSegmentJitter: Long =
     if (segmentJitterMs == 0) 0 else Utils.abs(scala.util.Random.nextInt()) % math.min(segmentJitterMs, segmentMs)
@@ -172,6 +174,8 @@ object LogConfig {
   val MessageTimestampDifferenceMaxMsDoc = TopicConfig.MESSAGE_TIMESTAMP_DIFFERENCE_MAX_MS_DOC
   val MessageDownConversionEnableDoc = TopicConfig.MESSAGE_DOWNCONVERSION_ENABLE_DOC
 
+  val ProducerBatchDecompressionEnableProp = "producer.batch.decompression.enable"
+
   val LeaderReplicationThrottledReplicasDoc = "A list of replicas for which log replication should be throttled on " +
     "the leader side. The list should describe a set of replicas in the form " +
     "[PartitionId]:[BrokerId],[PartitionId]:[BrokerId]:... or alternatively the wildcard '*' can be used to throttle " +
@@ -181,6 +185,13 @@ object LogConfig {
     "[PartitionId]:[BrokerId],[PartitionId]:[BrokerId]:... or alternatively the wildcard '*' can be used to throttle " +
     "all replicas for this topic."
 
+  val ProducerBatchDecompressionEnableDoc = "This configuration controls whether a compressed batch sent by a producer " +
+    "will be decompressed to perform validation of individual records in the batch. The default policy is to decompress " +
+    "all batches.  When set to false, decompression will be skipped if the following three conditions are met : " +
+    "i) Batch sent by producer is greater than record format version V1  ii) Record format version of batch sent by producer " +
+    "is the same as the broker message format version. iii) The relative offsets of the records in the compressed batch are " +
+    "monotonically increasing by 1 starting from 0"
+
   private[log] val ServerDefaultHeaderName = "Server Default Property"
 
   // Package private for testing
@@ -291,6 +302,8 @@ object LogConfig {
         FollowerReplicationThrottledReplicasDoc, FollowerReplicationThrottledReplicasProp)
       .define(MessageDownConversionEnableProp, BOOLEAN, Defaults.MessageDownConversionEnable, LOW,
         MessageDownConversionEnableDoc, KafkaConfig.LogMessageDownConversionEnableProp)
+      .define(ProducerBatchDecompressionEnableProp, BOOLEAN, Defaults.ProducerBatchDecompressionEnable, LOW,
+        ProducerBatchDecompressionEnableDoc, KafkaConfig.ProducerBatchDecompressionEnableProp)
   }
 
   def apply(): LogConfig = LogConfig(new Properties())
diff --git a/core/src/main/scala/kafka/log/LogValidator.scala b/core/src/main/scala/kafka/log/LogValidator.scala
index b31d45abfaa86..274d67144691d 100644
--- a/core/src/main/scala/kafka/log/LogValidator.scala
+++ b/core/src/main/scala/kafka/log/LogValidator.scala
@@ -89,7 +89,8 @@ private[log] object LogValidator extends Logging {
                                                     partitionLeaderEpoch: Int,
                                                     origin: AppendOrigin,
                                                     interBrokerProtocolVersion: ApiVersion,
-                                                    brokerTopicStats: BrokerTopicStats): ValidationAndOffsetAssignResult = {
+                                                    brokerTopicStats: BrokerTopicStats,
+                                                    decompressionEnable: Boolean = true): ValidationAndOffsetAssignResult = {
     if (sourceCodec == NoCompressionCodec && targetCodec == NoCompressionCodec) {
       // check the magic value
       if (!records.hasMatchingMagic(magic))
@@ -101,7 +102,7 @@ private[log] object LogValidator extends Logging {
           partitionLeaderEpoch, origin, magic, brokerTopicStats)
     } else {
       validateMessagesAndAssignOffsetsCompressed(records, topicPartition, offsetCounter, time, now, sourceCodec, targetCodec, compactedTopic,
-        magic, timestampType, timestampDiffMaxMs, partitionLeaderEpoch, origin, interBrokerProtocolVersion, brokerTopicStats)
+        magic, timestampType, timestampDiffMaxMs, partitionLeaderEpoch, origin, interBrokerProtocolVersion, brokerTopicStats, decompressionEnable)
     }
   }
 
@@ -341,7 +342,8 @@ private[log] object LogValidator extends Logging {
                                                  partitionLeaderEpoch: Int,
                                                  origin: AppendOrigin,
                                                  interBrokerProtocolVersion: ApiVersion,
-                                                 brokerTopicStats: BrokerTopicStats): ValidationAndOffsetAssignResult = {
+                                                 brokerTopicStats: BrokerTopicStats,
+                                                 decompressionEnable: Boolean): ValidationAndOffsetAssignResult = {
 
     if (targetCodec == ZStdCompressionCodec && interBrokerProtocolVersion < KAFKA_2_1_IV0)
       throw new UnsupportedCompressionTypeException("Produce requests to inter.broker.protocol.version < 2.1 broker " +
@@ -360,6 +362,11 @@ private[log] object LogValidator extends Logging {
     // One exception though is that with format smaller than v2, if sourceCodec is noCompression, then each batch is actually
     // a single record so we'd need to special handle it by creating a single wrapper batch that includes all the records
     val firstBatch = getFirstBatchAndMaybeValidateNoMoreBatches(records, sourceCodec)
+    // Check is made in {@link LogValidator#validateBatch} that the relative offset(s) of the record(s) in the record
+    // version V2 batch is monotonically increasing by 1, which is one of the requirement to avoid decompression.
+    val decompressBatch = decompressionEnable ||
+      !records.hasMatchingMagic(toMagic) ||
+      (firstBatch.magic < RecordBatch.MAGIC_VALUE_V2)
 
     // No in place assignment situation 2 and 3: we only need to check for the first batch because:
     //  1. For most cases (compressed records, v2, for example), there's only one batch anyways.
@@ -376,6 +383,8 @@ private[log] object LogValidator extends Logging {
       validateBatch(topicPartition, firstBatch, batch, origin, toMagic, brokerTopicStats)
       uncompressedSizeInBytes += AbstractRecords.recordBatchHeaderSizeInBytes(toMagic, batch.compressionType())
 
+      // The following inner block are not indented to make the hotfix cherry-picking easier
+      if (decompressBatch || !inPlaceAssignment) {
       // if we are on version 2 and beyond, and we know we are going for in place assignment,
       // then we can optimize the iterator to skip key / value / headers since they would not be used at all
       val recordsIterator = if (inPlaceAssignment && firstBatch.magic >= RecordBatch.MAGIC_VALUE_V2)
@@ -411,6 +420,7 @@ private[log] object LogValidator extends Logging {
       } finally {
         recordsIterator.close()
       }
+      }
     }
 
     if (!inPlaceAssignment) {
@@ -428,12 +438,21 @@ private[log] object LogValidator extends Logging {
       // we can update the batch only and write the compressed payload as is;
       // again we assume only one record batch within the compressed set
       val batch = records.batches.iterator.next()
-      val lastOffset = offsetCounter.addAndGet(validatedRecords.size) - 1
+      val lastOffset = if (decompressBatch)
+        offsetCounter.addAndGet(validatedRecords.size) - 1
+      else {
+        // batch.countOrNull() will never be null as the following line is execteud
+        // for record format version V2.
+        offsetCounter.addAndGet(batch.countOrNull().longValue()) - 1
+      }
 
       batch.setLastOffset(lastOffset)
 
       if (timestampType == TimestampType.LOG_APPEND_TIME)
         maxTimestamp = now
+      else if (!decompressBatch)
+        maxTimestamp = batch.maxTimestamp
+
 
       if (toMagic >= RecordBatch.MAGIC_VALUE_V1)
         batch.setMaxTimestamp(timestampType, maxTimestamp)
diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index f7344089907a4..02775e5421f3b 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -62,6 +62,7 @@ object Defaults {
   val BackgroundThreads = 10
   val QueuedMaxRequests = 500
   val QueuedMaxRequestBytes = -1
+  val ProducerBatchDecompressionEnable = true
 
   /************* Authorizer Configuration ***********/
   val AuthorizerClassName = ""
@@ -290,6 +291,7 @@ object KafkaConfig {
   val RequestTimeoutMsProp = CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG
   val HeapDumpFolderProp = "heap.dump.folder"
   val HeapDumpTimeoutProp = "heap.dump.timeout"
+  val ProducerBatchDecompressionEnableProp = "producer.batch.decompression.enable"
   /************* Authorizer Configuration ***********/
   val AuthorizerClassNameProp = "authorizer.class.name"
   /** ********* Socket Server Configuration ***********/
@@ -532,6 +534,7 @@ object KafkaConfig {
   val RequestTimeoutMsDoc = CommonClientConfigs.REQUEST_TIMEOUT_MS_DOC
   val HeapDumpFolderDoc = "The Folder under which heap dumps will be written by the watchdog"
   val HeapDumpTimeoutDoc = "The max amount of time (in millis) to wait for heap dump to complete before halting regardless"
+  val ProducerBatchDecompressionEnableDoc = "Decompress batch sent by producer to perform verification of individual records inside the batch"
   /************* Authorizer Configuration ***********/
   val AuthorizerClassNameDoc = s"The fully qualified name of a class that implements s${classOf[Authorizer].getName}" +
   " interface, which is used by the broker for authorization. This config also supports authorizers that implement the deprecated" +
@@ -899,6 +902,7 @@ object KafkaConfig {
       .define(RequestTimeoutMsProp, INT, Defaults.RequestTimeoutMs, HIGH, RequestTimeoutMsDoc)
       .define(HeapDumpFolderProp, STRING, Defaults.HeapDumpFolder, LOW, HeapDumpFolderDoc)
       .define(HeapDumpTimeoutProp, LONG, Defaults.HeapDumpTimeout, LOW, HeapDumpTimeoutDoc)
+      .define(ProducerBatchDecompressionEnableProp, BOOLEAN, Defaults.ProducerBatchDecompressionEnable, LOW, ProducerBatchDecompressionEnableDoc)
 
       /************* Authorizer Configuration ***********/
       .define(AuthorizerClassNameProp, STRING, Defaults.AuthorizerClassName, LOW, AuthorizerClassNameDoc)
@@ -1197,6 +1201,7 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO
   val requestTimeoutMs = getInt(KafkaConfig.RequestTimeoutMsProp)
   val heapDumpFolder = new File(getString(KafkaConfig.HeapDumpFolderProp))
   val heapDumpTimeout = getLong(KafkaConfig.HeapDumpTimeoutProp)
+  val producerBatchDecompressionEnable = getBoolean(KafkaConfig.ProducerBatchDecompressionEnableProp)
 
   def getNumReplicaAlterLogDirsThreads: Int = {
     val numThreads: Integer = Option(getInt(KafkaConfig.NumReplicaAlterLogDirsThreadsProp)).getOrElse(logDirs.size)
diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala
index 8c47d20a7846f..ced8a9494c78d 100755
--- a/core/src/main/scala/kafka/server/KafkaServer.scala
+++ b/core/src/main/scala/kafka/server/KafkaServer.scala
@@ -82,6 +82,7 @@ object KafkaServer {
     logProps.put(LogConfig.MessageTimestampTypeProp, kafkaConfig.logMessageTimestampType.name)
     logProps.put(LogConfig.MessageTimestampDifferenceMaxMsProp, kafkaConfig.logMessageTimestampDifferenceMaxMs: java.lang.Long)
     logProps.put(LogConfig.MessageDownConversionEnableProp, kafkaConfig.logMessageDownConversionEnable: java.lang.Boolean)
+    logProps.put(LogConfig.ProducerBatchDecompressionEnableProp, kafkaConfig.producerBatchDecompressionEnable: java.lang.Boolean)
     logProps
   }
 
diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala
index 19067ab7e8591..fbad0ece1c6e3 100755
--- a/core/src/test/scala/unit/kafka/log/LogTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogTest.scala
@@ -32,7 +32,7 @@ import kafka.server.checkpoints.LeaderEpochCheckpointFile
 import kafka.server.epoch.{EpochEntry, LeaderEpochFileCache}
 import kafka.server.{BrokerTopicStats, FetchDataInfo, FetchHighWatermark, FetchIsolation, FetchLogEnd, FetchTxnCommitted, KafkaConfig, LogDirFailureChannel, LogOffsetMetadata}
 import kafka.utils._
-import org.apache.kafka.common.{KafkaException, TopicPartition}
+import org.apache.kafka.common.{InvalidRecordException, KafkaException, TopicPartition}
 import org.apache.kafka.common.errors._
 import org.apache.kafka.common.record.FileRecords.TimestampAndOffset
 import org.apache.kafka.common.record.MemoryRecords.RecordFilter
@@ -1773,6 +1773,58 @@ class LogTest {
     }
   }
 
+  /**
+    * This test generates a single record set that is a concatenation of many valid record sets,
+    * and appends them to a log.  Verifies decompression was not performed by checking the size
+    * of buffer used to validate batch header.
+    */
+  @Test
+  def testAvoidDecompression() {
+    val logConfig = LogTest.createLogConfig(segmentBytes = 5000, producerBatchDecompressionEnable = false)
+    val log = createLog(TestUtils.randomPartitionLogDir(tmpDir), logConfig, logStartOffset = 1000)
+    val values = (0 until 5).map(id => TestUtils.randomBytes(10)).toArray
+
+    // Build a single buffer with all record appended
+    val recordBuffer = ByteBuffer.allocate(5000)
+    val builder = MemoryRecords.builder(recordBuffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.GZIP, TimestampType.CREATE_TIME,
+      0, System.currentTimeMillis, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH,
+      RecordBatch.NO_SEQUENCE)
+    for(value <- values) {
+      builder.append(new SimpleRecord(RecordBatch.NO_TIMESTAMP, null, value))
+    }
+
+    val records = builder.build()
+    val logAppendInfo = log.appendAsLeader(records, leaderEpoch = 0)
+
+    assertEquals("Size of buffer used to validate batch should be equal to the size of record format version V2 batch header.",
+      DefaultRecordBatch.RECORD_BATCH_OVERHEAD, logAppendInfo.recordConversionStats.temporaryMemoryBytes)
+  }
+
+  /**
+    * This test generates a single record set that is a concatenation of many valid record sets,
+    * with non-monotonically increasing relative offset and appends them to a log.
+    */
+  @Test(expected = classOf[InvalidRecordException])
+  def testAppendBatchWithoutMonotonicallyIncreasingRelativeOffset() {
+    val logConfig = LogTest.createLogConfig(segmentBytes = 5000, producerBatchDecompressionEnable = false)
+    val log = createLog(TestUtils.randomPartitionLogDir(tmpDir), logConfig)
+    val values = (0 until 5).map(id => TestUtils.randomBytes(10)).toArray
+
+    // Build a single buffer with all record appended
+    val recordBuffer = ByteBuffer.allocate(5000)
+    val builder = MemoryRecords.builder(recordBuffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.GZIP, TimestampType.CREATE_TIME,
+      10, System.currentTimeMillis, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH,
+      RecordBatch.NO_SEQUENCE)
+    var count = 0
+    for(value <- values) {
+      builder.appendWithOffset(10 + (count * 2), new SimpleRecord(RecordBatch.NO_TIMESTAMP, null, value))
+      count = count + 1
+    }
+
+    val records = builder.build()
+    log.appendAsLeader(records, leaderEpoch = 0)
+  }
+
   /**
    * This test covers an odd case where we have a gap in the offsets that falls at the end of a log segment.
    * Specifically we create a log where the last message in the first segment has offset 0. If we
@@ -4454,7 +4506,8 @@ object LogTest {
                       indexIntervalBytes: Int = Defaults.IndexInterval,
                       segmentIndexBytes: Int = Defaults.MaxIndexSize,
                       messageFormatVersion: String = Defaults.MessageFormatVersion,
-                      fileDeleteDelayMs: Long = Defaults.FileDeleteDelayMs): LogConfig = {
+                      fileDeleteDelayMs: Long = Defaults.FileDeleteDelayMs,
+                      producerBatchDecompressionEnable: Boolean = Defaults.ProducerBatchDecompressionEnable): LogConfig = {
     val logProps = new Properties()
 
     logProps.put(LogConfig.SegmentMsProp, segmentMs: java.lang.Long)
@@ -4468,6 +4521,7 @@ object LogTest {
     logProps.put(LogConfig.SegmentIndexBytesProp, segmentIndexBytes: Integer)
     logProps.put(LogConfig.MessageFormatVersionProp, messageFormatVersion)
     logProps.put(LogConfig.FileDeleteDelayMsProp, fileDeleteDelayMs: java.lang.Long)
+    logProps.put(LogConfig.ProducerBatchDecompressionEnableProp, producerBatchDecompressionEnable: java.lang.Boolean)
     LogConfig(logProps)
   }
 

From 3fa60329b834730b23e316e68e0ef75e3938607b Mon Sep 17 00:00:00 2001
From: Jon Lee 
Date: Thu, 27 Sep 2018 18:02:48 -0700
Subject: [PATCH 1015/1071] [LI-HOTFIX] LIKAFKA-19419: Do not delete snapshot
 files during log recovery offset checkpoint when called for truncation TICKET
 = KAFKA-7452 LI_DESCRIPTION = In upstream, it is fixed by KAFKA-7557.
 However, we decide to keep this hotfix to provide further optimization since
 we are not using transactional producer in Linkedin currently. EXIT_CRITERIA
 = MANUAL ["when transactional producer is used"]

---
 core/src/main/scala/kafka/log/LogManager.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/scala/kafka/log/LogManager.scala b/core/src/main/scala/kafka/log/LogManager.scala
index 24e600319520a..7cbc79b9743e0 100755
--- a/core/src/main/scala/kafka/log/LogManager.scala
+++ b/core/src/main/scala/kafka/log/LogManager.scala
@@ -534,7 +534,7 @@ class LogManager(logDirs: Seq[File],
     }
 
     for ((dir, logs) <- affectedLogs.groupBy(_.dir.getParentFile)) {
-      checkpointRecoveryOffsetsAndCleanSnapshot(dir, logs)
+      checkpointRecoveryOffsetsAndCleanSnapshot(dir, ArrayBuffer.empty)
     }
   }
 

From f8bf556ddc5de5b5aa7a80b5464e3f60f8e66d8c Mon Sep 17 00:00:00 2001
From: Zhanxiang Huang 
Date: Tue, 22 Jan 2019 14:37:42 -0800
Subject: [PATCH 1016/1071] [LI-HOTFIX] Add deletePartitionReassignment() in
 KafkaZkClient TICKET = LI_DESCRIPTION =
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Upstream removes deletePartitionReassignment() in KafkaZkClient as a part of the fencing zk operations with controller epoch patch. However, cruise control depends on KafkaZkClient directly and relies on this API in some of its logics. This patch temporarily adds back the API in KafkaZkClient as a hotfix so that CC doesn’t break in compilation.

RB=1537116
R=rrosenbl,agencer
A=rrosenbl

EXIT_CRITERIA = MANUAL ["After CC no longer relies on KafkaZkClient"]
---
 core/src/main/scala/kafka/zk/KafkaZkClient.scala | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core/src/main/scala/kafka/zk/KafkaZkClient.scala b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
index f34ae3f97c5a0..fb80978f603e9 100644
--- a/core/src/main/scala/kafka/zk/KafkaZkClient.scala
+++ b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
@@ -900,6 +900,10 @@ class KafkaZkClient private[zk] (zooKeeperClient: ZooKeeperClient, isSecure: Boo
     createRecursive(ReassignPartitionsZNode.path, ReassignPartitionsZNode.encode(reassignment))
   }
 
+  def deletePartitionReassignment(): Unit = {
+    deletePath(ReassignPartitionsZNode.path, ZkVersion.MatchAnyVersion)
+  }
+
   /**
    * Deletes the partition reassignment znode.
    * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.

From 00a07cb1bc7f8668303258aa0e5f224e2a6bd3e4 Mon Sep 17 00:00:00 2001
From: Lincong Li 
Date: Mon, 15 Apr 2019 17:23:41 -0700
Subject: [PATCH 1017/1071] [LI-HOTFIX] Add logging when automatic topic
 creation happens in order to track which application might be relying on
 auto.topic.create.enable being true (#12) TICKET = LI_DESCRIPTION =

EXIT_CRITERIA = MANUAL [""]
---
 core/src/main/scala/kafka/server/KafkaApis.scala | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index da402ec4f8764..380d1cb510974 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -1057,10 +1057,11 @@ class KafkaApis(val requestChannel: RequestChannel,
     topicMetadata.headOption.getOrElse(createInternalTopic(topic))
   }
 
-  private def getTopicMetadata(allowAutoTopicCreation: Boolean, topics: Set[String], listenerName: ListenerName,
+  private def getTopicMetadata(allowAutoTopicCreation: Boolean, topics: Set[String], requestContext: RequestContext,
                                errorUnavailableEndpoints: Boolean,
                                errorUnavailableListeners: Boolean): Seq[MetadataResponse.TopicMetadata] = {
-    val topicResponses = metadataCache.getTopicMetadata(topics, listenerName,
+
+    val topicResponses = metadataCache.getTopicMetadata(topics, requestContext.listenerName,
         errorUnavailableEndpoints, errorUnavailableListeners)
     if (topics.isEmpty || topicResponses.size == topics.size) {
       topicResponses
@@ -1074,6 +1075,12 @@ class KafkaApis(val requestChannel: RequestChannel,
           else
             topicMetadata
         } else if (allowAutoTopicCreation && config.autoCreateTopicsEnable) {
+          val msg =  "Automatically creating topic: " + topic + " with " + config.numPartitions +
+                     " partitions and replication factor " + config.defaultReplicationFactor +
+                     " due to request from " + requestContext.principal + " at IP address " +
+            requestContext.clientAddress + " and header " + requestContext.header
+
+          info(msg);
           createTopic(topic, config.numPartitions, config.defaultReplicationFactor)
         } else {
           new MetadataResponse.TopicMetadata(Errors.UNKNOWN_TOPIC_OR_PARTITION, topic, false, util.Collections.emptyList())
@@ -1133,7 +1140,7 @@ class KafkaApis(val requestChannel: RequestChannel,
       if (authorizedTopics.isEmpty)
         Seq.empty[MetadataResponse.TopicMetadata]
       else
-        getTopicMetadata(metadataRequest.allowAutoTopicCreation, authorizedTopics, request.context.listenerName,
+        getTopicMetadata(metadataRequest.allowAutoTopicCreation, authorizedTopics, request.context,
           errorUnavailableEndpoints, errorUnavailableListeners)
 
     var clusterAuthorizedOperations = 0

From 60dad2e8ea45c0891ec8a09abd710b35cf3a7ddf Mon Sep 17 00:00:00 2001
From: Lincong Li 
Date: Sun, 24 Mar 2019 01:11:28 -0700
Subject: [PATCH 1018/1071] [LI-HOTFIX] LIKAFKA-21968: Add broker-side observer
 interface and NoOpObserver implementation (#6) TICKET = LI_DESCRIPTION = The
 observer interface lets us provide implementation which provides the usage
 accounting data unit for the C2S V3 service.

Reviewers: Radai Rosenblatt

EXIT_CRITERIA = MANUAL [""]
---
 .../main/scala/kafka/server/KafkaApis.scala   |  19 ++++
 .../main/scala/kafka/server/KafkaConfig.scala |  23 ++++
 .../main/scala/kafka/server/KafkaServer.scala |  17 ++-
 .../scala/kafka/server/NoOpObserver.scala     |  46 ++++++++
 .../main/scala/kafka/server/Observer.scala    | 105 ++++++++++++++++++
 .../unit/kafka/server/KafkaApisTest.scala     |   2 +
 .../unit/kafka/server/KafkaConfigTest.scala   |   4 +
 7 files changed, 214 insertions(+), 2 deletions(-)
 create mode 100644 core/src/main/scala/kafka/server/NoOpObserver.scala
 create mode 100644 core/src/main/scala/kafka/server/Observer.scala

diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 380d1cb510974..3f7bf04848fa5 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -99,6 +99,7 @@ class KafkaApis(val requestChannel: RequestChannel,
                 val metadataCache: MetadataCache,
                 val metrics: Metrics,
                 val authorizer: Option[Authorizer],
+                val observer: Observer,
                 val quotas: QuotaManagers,
                 val fetchManager: FetchManager,
                 brokerTopicStats: BrokerTopicStats,
@@ -566,6 +567,14 @@ class KafkaApis(val requestChannel: RequestChannel,
     if (authorizedRequestInfo.isEmpty)
       sendResponseCallback(Map.empty)
     else {
+
+      try
+        observer.observeProduceRequest(request.context, request.body[ProduceRequest])
+      catch {
+        case e: Exception => error(s"Observer failed to observe the produce request " +
+          s"${Observer.describeRequestAndResponse(request, null)}", e)
+      }
+
       val internalTopicsAllowed = request.header.clientId == AdminUtils.AdminClientId
 
       // call the replica manager to append messages to the replicas
@@ -2912,8 +2921,11 @@ class KafkaApis(val requestChannel: RequestChannel,
         val responseString =
           if (RequestChannel.isRequestLoggingEnabled) Some(response.toString(request.context.apiVersion))
           else None
+        observeRequestResponse(request, response)
+
         new RequestChannel.SendResponse(request, responseSend, responseString, onComplete)
       case None =>
+        observeRequestResponse(request, null)
         new RequestChannel.NoOpResponse(request)
     }
     sendResponse(response)
@@ -2935,4 +2947,11 @@ class KafkaApis(val requestChannel: RequestChannel,
     }
   }
 
+  private def observeRequestResponse(request: RequestChannel.Request, response: AbstractResponse): Unit = {
+    try {
+      observer.observe(request.context, request.body[AbstractRequest], response)
+    } catch {
+      case e: Exception => error(s"Observer failed to observe ${Observer.describeRequestAndResponse(request, response)}", e)
+    }
+  }
 }
diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index 02775e5421f3b..ce791ec1f6485 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -67,6 +67,10 @@ object Defaults {
   /************* Authorizer Configuration ***********/
   val AuthorizerClassName = ""
 
+  /** ********* Broker-side configuration ***********/
+  val ObserverClassName = "kafka.server.NoOpObserver"
+  val ObserverShutdownTimeoutMs = 2000
+
   /** ********* Socket Server Configuration ***********/
   val Port = 9092
   val HostName: String = new String("")
@@ -294,6 +298,11 @@ object KafkaConfig {
   val ProducerBatchDecompressionEnableProp = "producer.batch.decompression.enable"
   /************* Authorizer Configuration ***********/
   val AuthorizerClassNameProp = "authorizer.class.name"
+
+  /** ********* Broker-side observer Configuration ****************/
+  val ObserverClassNameProp = "observer.class.name"
+  val ObserverShutdownTimeoutMsProp = "observer.shutdown.timeout"
+
   /** ********* Socket Server Configuration ***********/
   val PortProp = "port"
   val HostNameProp = "host.name"
@@ -872,6 +881,12 @@ object KafkaConfig {
   val PasswordEncoderKeyLengthDoc =  "The key length used for encoding dynamically configured passwords."
   val PasswordEncoderIterationsDoc =  "The iteration count used for encoding dynamically configured passwords."
 
+  /** *********  Broker-side Observer Configuration *********/
+  val ObserverClassNameDoc = "The name of the observer class that is used to observe requests and/or response on broker."
+  val ObserverShutdownTimeoutMsDoc = "The maximum time of closing/shutting down an observer. This property can not be less than or equal to " +
+    "zero. When closing/shutting down an observer, most time is spent on flushing the observed stats. The reasonable timeout should be close to " +
+    "the time it takes to flush the stats."
+
   private val configDef = {
     import ConfigDef.Importance._
     import ConfigDef.Range._
@@ -907,6 +922,10 @@ object KafkaConfig {
       /************* Authorizer Configuration ***********/
       .define(AuthorizerClassNameProp, STRING, Defaults.AuthorizerClassName, LOW, AuthorizerClassNameDoc)
 
+      /************* Broker-side Observer Configuration ***********/
+      .define(ObserverClassNameProp, STRING, Defaults.ObserverClassName, MEDIUM, ObserverClassNameDoc)
+      .define(ObserverShutdownTimeoutMsProp, LONG, Defaults.ObserverShutdownTimeoutMs, atLeast(1), MEDIUM, ObserverShutdownTimeoutMsDoc)
+
       /** ********* Socket Server Configuration ***********/
       .define(PortProp, INT, Defaults.Port, HIGH, PortDoc)
       .define(HostNameProp, STRING, Defaults.HostName, HIGH, HostNameDoc)
@@ -1223,6 +1242,10 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO
     }
   }
 
+  /************* Broker-side Observer Configuration ********/
+  val ObserverClassName: String = getString(KafkaConfig.ObserverClassNameProp)
+  val ObserverShutdownTimeoutMs: Long = getLong(KafkaConfig.ObserverShutdownTimeoutMsProp)
+
   /** ********* Socket Server Configuration ***********/
   val hostName = getString(KafkaConfig.HostNameProp)
   val port = getInt(KafkaConfig.PortProp)
diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala
index ced8a9494c78d..5db43cf728db0 100755
--- a/core/src/main/scala/kafka/server/KafkaServer.scala
+++ b/core/src/main/scala/kafka/server/KafkaServer.scala
@@ -120,6 +120,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
   var controlPlaneRequestProcessor: KafkaApis = null
 
   var authorizer: Option[Authorizer] = None
+  var observer: Observer = null
   var socketServer: SocketServer = null
   var dataPlaneRequestHandlerPool: KafkaRequestHandlerPool = null
   var controlPlaneRequestHandlerPool: KafkaRequestHandlerPool = null
@@ -320,13 +321,22 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
             brokerInfo.broker.endPoints.map { ep => ep.toJava -> CompletableFuture.completedFuture[Void](null) }.toMap
         }
 
+        observer = try {
+          CoreUtils.createObject[Observer](config.ObserverClassName)
+        } catch {
+          case e: Exception =>
+            error(s"Creating observer instance from the given class name ${config.ObserverClassName} failed.", e)
+            new NoOpObserver
+        }
+        observer.configure(config.originals())
+
         val fetchManager = new FetchManager(Time.SYSTEM,
           new FetchSessionCache(config.maxIncrementalFetchSessionCacheSlots,
             KafkaServer.MIN_INCREMENTAL_FETCH_SESSION_EVICTION_MS))
 
         /* start processing requests */
         dataPlaneRequestProcessor = new KafkaApis(socketServer.dataPlaneRequestChannel, replicaManager, adminManager, groupCoordinator, transactionCoordinator,
-          kafkaController, zkClient, config.brokerId, config, metadataCache, metrics, authorizer, quotaManagers,
+          kafkaController, zkClient, config.brokerId, config, metadataCache, metrics, authorizer, observer, quotaManagers,
           fetchManager, brokerTopicStats, clusterId, time, tokenManager)
 
         dataPlaneRequestHandlerPool = new KafkaRequestHandlerPool(config.brokerId, socketServer.dataPlaneRequestChannel, dataPlaneRequestProcessor, time,
@@ -334,7 +344,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
 
         socketServer.controlPlaneRequestChannelOpt.foreach { controlPlaneRequestChannel =>
           controlPlaneRequestProcessor = new KafkaApis(controlPlaneRequestChannel, replicaManager, adminManager, groupCoordinator, transactionCoordinator,
-            kafkaController, zkClient, config.brokerId, config, metadataCache, metrics, authorizer, quotaManagers,
+            kafkaController, zkClient, config.brokerId, config, metadataCache, metrics, authorizer, observer, quotaManagers,
             fetchManager, brokerTopicStats, clusterId, time, tokenManager)
 
           controlPlaneRequestHandlerPool = new KafkaRequestHandlerPool(config.brokerId, socketServer.controlPlaneRequestChannelOpt.get, controlPlaneRequestProcessor, time,
@@ -643,6 +653,9 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
         if (controlPlaneRequestProcessor != null)
           CoreUtils.swallow(controlPlaneRequestProcessor.close(), this)
         CoreUtils.swallow(authorizer.foreach(_.close()), this)
+
+        CoreUtils.swallow(observer.close(config.ObserverShutdownTimeoutMs, TimeUnit.MILLISECONDS), this)
+
         if (adminManager != null)
           CoreUtils.swallow(adminManager.shutdown(), this)
 
diff --git a/core/src/main/scala/kafka/server/NoOpObserver.scala b/core/src/main/scala/kafka/server/NoOpObserver.scala
new file mode 100644
index 0000000000000..3fea54a7c8528
--- /dev/null
+++ b/core/src/main/scala/kafka/server/NoOpObserver.scala
@@ -0,0 +1,46 @@
+/**
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *    http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+
+package kafka.server
+
+import java.util.Map
+import java.util.concurrent.TimeUnit
+import org.apache.kafka.common.requests.{AbstractRequest, AbstractResponse, ProduceRequest, RequestContext}
+
+/**
+  * An observer implementation that has no operation and serves as a place holder.
+  */
+class NoOpObserver extends Observer {
+
+  def configure(configs: Map[String, _]): Unit = {}
+
+  /**
+    * Observe a request and its corresponding response.
+    */
+  def observe(requestContext: RequestContext, request: AbstractRequest, response: AbstractResponse): Unit = {}
+
+  /**
+    * Observe a produce request
+    */
+  def observeProduceRequest(requestContext: RequestContext, produceRequest: ProduceRequest): Unit = {}
+
+  /**
+    * Close the observer with timeout.
+    */
+  def close(timeout: Long, unit: TimeUnit): Unit = {}
+
+}
diff --git a/core/src/main/scala/kafka/server/Observer.scala b/core/src/main/scala/kafka/server/Observer.scala
new file mode 100644
index 0000000000000..624f609fbdd47
--- /dev/null
+++ b/core/src/main/scala/kafka/server/Observer.scala
@@ -0,0 +1,105 @@
+/**
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *    http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+
+package kafka.server
+
+import java.util.concurrent.TimeUnit
+import kafka.network.RequestChannel
+import org.apache.kafka.common.requests.{AbstractRequest, AbstractResponse, ProduceRequest, RequestContext}
+import org.apache.kafka.common.Configurable
+
+/**
+  * Top level interface that all pluggable observer must implement. Kafka will read the 'observer.class.name' config
+  * value at startup time, create an instance of the specificed class using the default constructor, and call its
+  * 'configure' method.
+  *
+  * From that point onwards, every pair of request and response will be routed to the 'record' method.
+  *
+  * If 'observer.class.name' has no value specified or the specified class does not exist, the NoOpObserver
+  * will be used as a place holder.
+  */
+trait Observer extends Configurable {
+
+  /**
+    * Observe a request and its corresponding response
+    *
+    * @param requestContext the context information about the request
+    * @param request  the request being observed for a various purpose(s)
+    * @param response the response to the request
+    */
+  def observe(requestContext: RequestContext, request: AbstractRequest, response: AbstractResponse): Unit
+
+  /**
+    * Observe a produce request. This method handles only the produce request since produce request is special in
+    * two ways. Firstly, if ACK is set to be 0, there is no produce response associated with the produce request.
+    * Secondly, the lifecycle of some inner fields in a ProduceRequest is shorter than the lifecycle of the produce
+    * request itself. That means in some situations, when observe is called on a produce request and
+    * response pair, some fields in the produce request has been null-ed already so that the produce request and
+    * response is not observable (or no useful information). Therefore this method exists for the purpose of allowing
+    * users to observe on the produce request before its corresponding response is created.
+    *
+    * @param requestContext the context information about the request
+    * @param produceRequest  the produce request being observed for a various purpose(s)
+    */
+  def observeProduceRequest(requestContext: RequestContext, produceRequest: ProduceRequest): Unit
+
+  /**
+    * Close the observer with timeout.
+    *
+    * @param timeout the maximum time to wait to close the observer.
+    * @param unit    the time unit.
+    */
+  def close(timeout: Long, unit: TimeUnit): Unit
+}
+
+object Observer {
+
+  /**
+    * Generates a description of the given request and response. It could be used mostly for debugging purpose.
+    *
+    * @param request  the request being described
+    * @param response the response to the request
+    */
+  def describeRequestAndResponse(request: RequestChannel.Request, response: AbstractResponse): String = {
+    var requestDesc = "Request"
+    var responseDesc = "Response"
+    try {
+      if (request == null) {
+        requestDesc += " null"
+      } else {
+        requestDesc += (" header: " + request.header)
+        requestDesc += (" from service with principal: " +
+          request.session.sanitizedUser +
+          " IP address: " + request.session.clientAddress)
+      }
+      requestDesc += " | " // Separate the response description from the request description
+
+      if (response == null) {
+        responseDesc += " null"
+      } else {
+        responseDesc += (if (response.errorCounts == null || response.errorCounts.size == 0) {
+          " with no error"
+        } else {
+          " with errors: " + response.errorCounts
+        })
+      }
+    } catch {
+      case e: Exception => return e.toString // If describing fails, return the exception message directly
+    }
+    requestDesc + responseDesc
+  }
+}
\ No newline at end of file
diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
index a3f58e9e6cad3..28f56c256203a 100644
--- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
@@ -76,6 +76,7 @@ class KafkaApisTest {
   private val brokerId = 1
   private val metadataCache = new MetadataCache(brokerId)
   private val authorizer: Option[Authorizer] = None
+  private val observer: Observer = EasyMock.createNiceMock(classOf[Observer])
   private val clientQuotaManager: ClientQuotaManager = EasyMock.createNiceMock(classOf[ClientQuotaManager])
   private val clientRequestQuotaManager: ClientRequestQuotaManager = EasyMock.createNiceMock(classOf[ClientRequestQuotaManager])
   private val replicaQuotaManager: ReplicationQuotaManager = EasyMock.createNiceMock(classOf[ReplicationQuotaManager])
@@ -108,6 +109,7 @@ class KafkaApisTest {
       metadataCache,
       metrics,
       authorizer,
+      observer,
       quotas,
       fetchManager,
       brokerTopicStats,
diff --git a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
index fee636094277e..2ba8b16687cfa 100755
--- a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
@@ -766,6 +766,10 @@ class KafkaConfigTest {
         case KafkaConfig.KafkaMetricsReporterClassesProp => // ignore
         case KafkaConfig.KafkaMetricsPollingIntervalSecondsProp => //ignore
 
+        // Broker-side observer configs
+        case KafkaConfig.ObserverClassNameProp => // ignore since even if the class name is invalid, a NoOpObserver class is used instead
+        case KafkaConfig.ObserverShutdownTimeoutMsProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_number", "-1", "0")
+
         case _ => assertPropertyInvalid(getBaseProperties(), name, "not_a_number", "-1")
       }
     })

From 5be24511f03fe6e68f5b4272c2895e4e5ccc59cf Mon Sep 17 00:00:00 2001
From: Jon Lee 
Date: Thu, 7 Mar 2019 12:34:20 -0800
Subject: [PATCH 1019/1071] [LI-HOTFIX] add Bintray support to LinkedIn Kafka
 Github TICKET = LI_DESCRIPTION = This contains changes for CI builds and
 publishing artifacts to the maven repo under the LinkedIn Bintray account.
 Travis will kick off a build and publish artifacts to bintray upon creating a
 tag in the "x.y.z.w" format.

EXIT_CRITERIA = MANUAL [""]
---
 .travis.yml       | 43 ++++++++++++++++++++-----------------------
 README.md         | 11 +++++++++--
 build.gradle      | 24 ++++++++++++++----------
 gradle.properties | 15 +++++++++++----
 4 files changed, 54 insertions(+), 39 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 8a22c9bc4cd4a..e5695ba13e0b0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,41 +14,38 @@ sudo: required
 dist: trusty
 language: java
 
-env:
-  - _DUCKTAPE_OPTIONS="--subset 0  --subsets 15"
-  - _DUCKTAPE_OPTIONS="--subset 1  --subsets 15"
-  - _DUCKTAPE_OPTIONS="--subset 2  --subsets 15"
-  - _DUCKTAPE_OPTIONS="--subset 3  --subsets 15"
-  - _DUCKTAPE_OPTIONS="--subset 4  --subsets 15"
-  - _DUCKTAPE_OPTIONS="--subset 5  --subsets 15"
-  - _DUCKTAPE_OPTIONS="--subset 6  --subsets 15"
-  - _DUCKTAPE_OPTIONS="--subset 7  --subsets 15"
-  - _DUCKTAPE_OPTIONS="--subset 8  --subsets 15"
-  - _DUCKTAPE_OPTIONS="--subset 9  --subsets 15"
-  - _DUCKTAPE_OPTIONS="--subset 10 --subsets 15"
-  - _DUCKTAPE_OPTIONS="--subset 11 --subsets 15"
-  - _DUCKTAPE_OPTIONS="--subset 12 --subsets 15"
-  - _DUCKTAPE_OPTIONS="--subset 13 --subsets 15"
-  - _DUCKTAPE_OPTIONS="--subset 14 --subsets 15"
-
 jdk:
   - oraclejdk8
-
+  
 before_install:
   - gradle wrapper
 
-script:
-  - ./gradlew rat
-  - ./gradlew systemTestLibs && /bin/bash ./tests/docker/run_tests.sh
+# By default, the install step will run ./gradlew assemble, which actually builds. Skip this step and do the actual
+# build during the build step.
+install: true
 
-services:
-  - docker
+# Excluded integration tests for now because Travis will hit build timeout (50 minutes) when including them.
+# Also excluded streams unitTest because they often fail with "pure virtual method called" error (KAFKA-3502).
+# TODO: re-enable these tests when the mentioned issues are resolved.
+script:
+  - ./gradlew clean compileJava compileScala compileTestJava compileTestScala checkstyleMain checkstyleTest findbugsMain unitTest -x :streams:unitTest rat --no-daemon -PxmlFindBugsReport=true -PtestLoggingEvents=started,passed,skipped,failed 
 
 before_cache:
   - rm -f  $HOME/.gradle/caches/modules-2/modules-2.lock
   - rm -fr $HOME/.gradle/caches/*/plugin-resolution/
+
 cache:
   directories:
     - "$HOME/.m2/repository"
     - "$HOME/.gradle/caches/"
     - "$HOME/.gradle/wrapper/"
+
+# This will triger publishing artifacts to bintray upon the creation of a new tag in the "x.y.z.w" format.
+deploy:
+  provider: script
+  script: ./gradlew -Pversion=$TRAVIS_TAG uploadArchivesAll
+  skip_cleanup: true
+  on:
+    tags: true
+    all_branches: true
+    condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$
diff --git a/README.md b/README.md
index 8d43191a03c2e..55c3212a03ec7 100644
--- a/README.md
+++ b/README.md
@@ -149,13 +149,20 @@ build directory (`${project_dir}/bin`) clashes with Kafka's scripts directory an
 to avoid known issues with this configuration.
 
 ### Publishing the jar for all version of Scala and for all projects to maven ###
-    ./gradlew uploadArchivesAll
+    ./gradlew -Pversion= uploadArchivesAll
 
-Please note for this to work you should create/update `${GRADLE_USER_HOME}/gradle.properties` (typically, `~/.gradle/gradle.properties`) and assign the following variables
+By default, this command will publish artifacts to a Bintray repository named "kafka" under an account specified by the BINTRAY_USER environment variable. The BINTRAY_KEY
+environment variable is used for the password for that account.
+
+If you want to override this to use a different maven repository, you should create/update `${GRADLE_USER_HOME}/gradle.properties` (typically, `~/.gradle/gradle.properties`)
+and assign the following variables
 
     mavenUrl=
     mavenUsername=
     mavenPassword=
+
+Signing is disabled by default. If you need signing, please set the following variables in `gradle.properties` as well:
+
     signing.keyId=
     signing.password=
     signing.secretKeyRingFile=
diff --git a/build.gradle b/build.gradle
index 6edbd02fe3ba9..cac618a443e69 100644
--- a/build.gradle
+++ b/build.gradle
@@ -54,7 +54,7 @@ allprojects {
   repositories {
     mavenCentral()
   }
-  
+
   apply plugin: 'idea'
   apply plugin: 'org.owasp.dependencycheck'
   apply plugin: 'com.github.ben-manes.versions'
@@ -100,9 +100,14 @@ ext {
   skipSigning = project.hasProperty('skipSigning') && skipSigning.toBoolean()
   shouldSign = !skipSigning && !version.endsWith("SNAPSHOT") && project.gradle.startParameter.taskNames.any { it.contains("upload") }
 
-  mavenUrl = project.hasProperty('mavenUrl') ? project.mavenUrl : ''
-  mavenUsername = project.hasProperty('mavenUsername') ? project.mavenUsername : ''
-  mavenPassword = project.hasProperty('mavenPassword') ? project.mavenPassword : ''
+  bintrayUsername = System.getenv('BINTRAY_USER')
+  bintrayKey=System.getenv('BINTRAY_KEY')
+  bintrayUrl = bintrayApiBaseUrl + '/maven/linkedin/' + bintrayRepo + '/' + bintrayPackage + '/;publish=1'
+
+  // By default, publish to Bintray.
+  mavenUrl = project.hasProperty('mavenUrl') ? project.mavenUrl : bintrayUrl
+  mavenUsername = project.hasProperty('mavenUsername') ? project.mavenUsername : bintrayUsername
+  mavenPassword = project.hasProperty('mavenPassword') ? project.mavenPassword : bintrayKey
 
   userShowStandardStreams = project.hasProperty("showStandardStreams") ? showStandardStreams : null
 
@@ -196,9 +201,9 @@ subprojects {
               afterEvaluate {
                   pom.artifactId = "${archivesBaseName}"
                   pom.project {
-                      name 'Apache Kafka'
+                      name 'LinkedIn fork of Apache Kafka'
                       packaging 'jar'
-                      url 'https://kafka.apache.org'
+                      url 'http://github.com/linkedin/kafka'
                       licenses {
                           license {
                               name 'The Apache Software License, Version 2.0'
@@ -533,8 +538,8 @@ def fineTuneEclipseClasspathFile(eclipse, project) {
       if (project.name.equals('core')) {
         cp.entries.findAll { it.kind == "src" && it.path.equals("src/test/scala") }*.excludes = ["integration/", "other/", "unit/"]
       }
-      /* 
-       * Set all eclipse build output to go to 'build_eclipse' directory. This is to ensure that gradle and eclipse use different 
+      /*
+       * Set all eclipse build output to go to 'build_eclipse' directory. This is to ensure that gradle and eclipse use different
        * build output directories, and also avoid using the eclpise default of 'bin' which clashes with some of our script directories.
        * https://discuss.gradle.org/t/eclipse-generated-files-should-be-put-in-the-same-place-as-the-gradle-generated-files/6986/2
        */
@@ -643,7 +648,6 @@ def pkgs = [
     'log4j-appender',
     'streams',
     'streams:examples',
-    'streams:streams-scala',
     'streams:test-utils',
     'tools'
 ] + connectPkgs
@@ -727,7 +731,7 @@ project(':core') {
     scoverage libs.scoveragePlugin
     scoverage libs.scoverageRuntime
   }
-  
+
   scoverage {
     reportDir = file("${rootProject.buildDir}/scoverage")
     highlighting = false
diff --git a/gradle.properties b/gradle.properties
index 8b42e427935dd..16e743ac59389 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -13,14 +13,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-group=org.apache.kafka
-# NOTE: When you change this version number, you should also make sure to update
-# the version numbers in
+group=com.linkedin.kafka
+
+# NOTE: publishing artifacts requires a property named "version" to be set explicitly. For example,
+#   ./gradlew -Pversion= uploadArchivesAll
+#
+# You should also make sure to update the version numbers in
 #  - docs/js/templateData.js
-#  - tests/kafkatest/__init__.py
+#  - tests/kafkatest/__init__.py,
 #  - tests/kafkatest/version.py (variable DEV_VERSION)
 #  - kafka-merge-pr.py
 version=2.4.2-SNAPSHOT
 scalaVersion=2.12.10
 task=build
 org.gradle.jvmargs=-Xmx1024m -Xss2m
+bintrayApiBaseUrl=https://api.bintray.com
+bintrayRepo=maven
+bintrayPackage=kafka
+skipSigning=true

From 0322d2512eef8f39b019288ffaeafb1f3a5711b1 Mon Sep 17 00:00:00 2001
From: Kyle Ambroff-Kao 
Date: Thu, 2 May 2019 18:31:28 -0700
Subject: [PATCH 1020/1071] [LI-HOTFIX] Duplicate metrics should not lead to
 failure (#24) TICKET = LI_DESCRIPTION =

* Duplicate metrics should not lead to failure

There have been many bugs that have crept in over the years that lead to
some sort of broker failure because sensors were not cleaned up properly
during some normal state transition.

KAFKA-8066 is a good example of this. Even after the fix in 6ca899e for
that bug, we are still seeing the same problem in production.

Rather than trying to find all possible code paths that can lead to
Selector#close not being invoked, this patch just makes duplicate
metrics log an error without throwing.

While duplicate metrics from multiple sensors can be the sign of another
bug, making it fatal seems like a mistake. A common failure we see is
that a replica fetcher thread does not shut down cleanly, leaving behind
some metrics as the thread dies (taking its NetworkClient with
it). Later, when the broker tries to start a replacement replica fetcher
thread in response to a LeaderAndIsr request from the controller, the
fetcher thread cannot start due to this sensor conflict.

Failing to start a replica fetcher in this case can lead to a data loss
situation, since we could end up with the cluster in a state where we
cannot safely restart any brokers to clear up this situation without
offline partitions.

Rather than continuing this trend, we should make sensor duplication
non-fatal. This behavior is configurable and is disabled by default.

EXIT_CRITERIA = MANUAL [""]
---
 .../kafka/clients/CommonClientConfigs.java    |  4 ++++
 .../clients/admin/AdminClientConfig.java      |  7 +++++++
 .../kafka/clients/admin/KafkaAdminClient.java |  1 +
 .../clients/consumer/ConsumerConfig.java      | 10 ++++++++++
 .../kafka/clients/consumer/KafkaConsumer.java |  1 +
 .../kafka/clients/producer/KafkaProducer.java |  1 +
 .../clients/producer/ProducerConfig.java      | 10 ++++++++++
 .../apache/kafka/common/metrics/Metrics.java  | 20 ++++++++++++++++---
 .../kafka/common/metrics/MetricsTest.java     | 18 +++++++++++++++++
 .../kafka/common/metrics/SensorTest.java      |  5 +++++
 .../main/scala/kafka/server/KafkaConfig.scala |  5 +++++
 .../main/scala/kafka/server/KafkaServer.scala |  1 +
 12 files changed, 80 insertions(+), 3 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java b/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java
index ecb3f78eff847..03b9b31716c3a 100644
--- a/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java
+++ b/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java
@@ -87,6 +87,10 @@ public class CommonClientConfigs {
     public static final String METRICS_RECORDING_LEVEL_CONFIG = "metrics.recording.level";
     public static final String METRICS_RECORDING_LEVEL_DOC = "The highest recording level for metrics.";
 
+    public static final String METRICS_REPLACE_ON_DUPLICATE_CONFIG = "metrics.replace.on.duplicate";
+    public static final String METRICS_REPLACE_ON_DUPLICATE_DOC = "If set to true, then multiple sensors registering the same metric will not throw, but will instead log an error. This makes sensor registration errors non fatal.";
+
+
     public static final String METRIC_REPORTER_CLASSES_CONFIG = "metric.reporters";
     public static final String METRIC_REPORTER_CLASSES_DOC = "A list of classes to use as metrics reporters. Implementing the org.apache.kafka.common.metrics.MetricsReporter interface allows plugging in classes that will be notified of new metric creation. The JmxReporter is always included to register JMX statistics.";
 
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/AdminClientConfig.java b/clients/src/main/java/org/apache/kafka/clients/admin/AdminClientConfig.java
index 107eb56d63aed..283d836887843 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/AdminClientConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/AdminClientConfig.java
@@ -100,11 +100,13 @@ public class AdminClientConfig extends AbstractConfig {
     private static final String METRICS_SAMPLE_WINDOW_MS_DOC = CommonClientConfigs.METRICS_SAMPLE_WINDOW_MS_DOC;
 
     public static final String METRICS_RECORDING_LEVEL_CONFIG = CommonClientConfigs.METRICS_RECORDING_LEVEL_CONFIG;
+    public static final String METRICS_REPLACE_ON_DUPLICATE_CONFIG = CommonClientConfigs.METRICS_REPLACE_ON_DUPLICATE_CONFIG;
 
     public static final String SECURITY_PROTOCOL_CONFIG = CommonClientConfigs.SECURITY_PROTOCOL_CONFIG;
     public static final String DEFAULT_SECURITY_PROTOCOL = CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL;
     private static final String SECURITY_PROTOCOL_DOC = CommonClientConfigs.SECURITY_PROTOCOL_DOC;
     private static final String METRICS_RECORDING_LEVEL_DOC = CommonClientConfigs.METRICS_RECORDING_LEVEL_DOC;
+    private static final String METRICS_REPLACE_ON_DUPLICATE_DOC = CommonClientConfigs.METRICS_REPLACE_ON_DUPLICATE_DOC;
 
     public static final String RETRIES_CONFIG = CommonClientConfigs.RETRIES_CONFIG;
 
@@ -180,6 +182,11 @@ public class AdminClientConfig extends AbstractConfig {
                                            ClientDnsLookup.RESOLVE_CANONICAL_BOOTSTRAP_SERVERS_ONLY.toString()),
                                         Importance.MEDIUM,
                                         CLIENT_DNS_LOOKUP_DOC)
+                                .define(METRICS_REPLACE_ON_DUPLICATE_CONFIG,
+                                        Type.BOOLEAN,
+                                        false,
+                                        Importance.LOW,
+                                        METRICS_REPLACE_ON_DUPLICATE_DOC)
                                 // security support
                                 .define(SECURITY_PROVIDERS_CONFIG,
                                         Type.STRING,
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
index 1da0e66a07193..26040db5acc20 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
@@ -422,6 +422,7 @@ static KafkaAdminClient createInternal(AdminClientConfig config, TimeoutProcesso
                 .tags(metricTags);
             reporters.add(new JmxReporter(JMX_PREFIX));
             metrics = new Metrics(metricConfig, reporters, time);
+            metrics.setReplaceOnDuplicateMetric(config.getBoolean(AdminClientConfig.METRICS_REPLACE_ON_DUPLICATE_CONFIG));
             String metricGrpPrefix = "admin-client";
             channelBuilder = ClientUtils.createChannelBuilder(config, time);
             selector = new Selector(config.getLong(AdminClientConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG),
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
index d50dfb65ab193..63eb9bcb37d40 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
@@ -200,6 +200,11 @@ public class ConsumerConfig extends AbstractConfig {
      */
     public static final String METRIC_REPORTER_CLASSES_CONFIG = CommonClientConfigs.METRIC_REPORTER_CLASSES_CONFIG;
 
+    /**
+     * metrics.replace.on.duplicate
+     */
+    public static final String METRIC_REPLACE_ON_DUPLICATE_CONFIG = CommonClientConfigs.METRICS_REPLACE_ON_DUPLICATE_CONFIG;
+
     /**
      * check.crcs
      */
@@ -439,6 +444,11 @@ public class ConsumerConfig extends AbstractConfig {
                                         new ConfigDef.NonNullValidator(),
                                         Importance.LOW,
                                         CommonClientConfigs.METRIC_REPORTER_CLASSES_DOC)
+                                .define(METRIC_REPLACE_ON_DUPLICATE_CONFIG,
+                                        Type.BOOLEAN,
+                                        false,
+                                        Importance.LOW,
+                                        CommonClientConfigs.METRICS_REPLACE_ON_DUPLICATE_DOC)
                                 .define(KEY_DESERIALIZER_CLASS_CONFIG,
                                         Type.CLASS,
                                         Importance.HIGH,
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index 28ac47c23be82..f59a773553987 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -701,6 +701,7 @@ else if (enableAutoCommit)
             this.defaultApiTimeoutMs = config.getInt(ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG);
             this.time = Time.SYSTEM;
             this.metrics = buildMetrics(config, time, clientId);
+            this.metrics.setReplaceOnDuplicateMetric(config.getBoolean(ConsumerConfig.METRIC_REPLACE_ON_DUPLICATE_CONFIG));
             this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG);
 
             // load interceptors and make sure they get clientId
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
index 8d0c224ea7172..1ab6ecf441384 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
@@ -353,6 +353,7 @@ public KafkaProducer(Properties properties, Serializer keySerializer, Seriali
                     Collections.singletonMap(ProducerConfig.CLIENT_ID_CONFIG, clientId));
             reporters.add(new JmxReporter(JMX_PREFIX));
             this.metrics = new Metrics(metricConfig, reporters, time);
+            this.metrics.setReplaceOnDuplicateMetric(config.getBoolean(ProducerConfig.METRICS_REPLACE_ON_DUPLICATE_CONFIG));
             this.partitioner = config.getConfiguredInstance(ProducerConfig.PARTITIONER_CLASS_CONFIG, Partitioner.class);
             long retryBackoffMs = config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG);
             if (keySerializer == null) {
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
index 96a14bca61c39..6ecf844d141b9 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
@@ -184,6 +184,11 @@ public class ProducerConfig extends AbstractConfig {
      */
     public static final String METRICS_RECORDING_LEVEL_CONFIG = CommonClientConfigs.METRICS_RECORDING_LEVEL_CONFIG;
 
+    /**
+     * metrics.replace.on.duplicate
+     */
+    public static final String METRICS_REPLACE_ON_DUPLICATE_CONFIG = CommonClientConfigs.METRICS_REPLACE_ON_DUPLICATE_CONFIG;
+
     /** metric.reporters */
     public static final String METRIC_REPORTER_CLASSES_CONFIG = CommonClientConfigs.METRIC_REPORTER_CLASSES_CONFIG;
 
@@ -319,6 +324,11 @@ public class ProducerConfig extends AbstractConfig {
                                         new ConfigDef.NonNullValidator(),
                                         Importance.LOW,
                                         CommonClientConfigs.METRIC_REPORTER_CLASSES_DOC)
+                                .define(METRICS_REPLACE_ON_DUPLICATE_CONFIG,
+                                        Type.BOOLEAN,
+                                        false,
+                                        Importance.LOW,
+                                        CommonClientConfigs.METRICS_REPLACE_ON_DUPLICATE_DOC)
                                 .define(MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION,
                                         Type.INT,
                                         5,
diff --git a/clients/src/main/java/org/apache/kafka/common/metrics/Metrics.java b/clients/src/main/java/org/apache/kafka/common/metrics/Metrics.java
index 3fc73e131e284..dd776cdc708e9 100644
--- a/clients/src/main/java/org/apache/kafka/common/metrics/Metrics.java
+++ b/clients/src/main/java/org/apache/kafka/common/metrics/Metrics.java
@@ -77,6 +77,8 @@ public class Metrics implements Closeable {
     private final ScheduledThreadPoolExecutor metricsScheduler;
     private static final Logger log = LoggerFactory.getLogger(Metrics.class);
 
+    private volatile boolean replaceOnDuplicate = false;
+
     /**
      * Create a metrics repository with no metric reporters and default configuration.
      * Expiration of Sensors is disabled.
@@ -533,7 +535,7 @@ public synchronized KafkaMetric removeMetric(MetricName metricName) {
                 try {
                     reporter.metricRemoval(metric);
                 } catch (Exception e) {
-                    log.error("Error when removing metric from " + reporter.getClass().getName(), e);
+                    log.error("Error when removing metric " + metricName + " from " + reporter.getClass().getName(), e);
                 }
             }
             log.trace("Removed metric named {}", metricName);
@@ -560,8 +562,16 @@ public synchronized void removeReporter(MetricsReporter reporter) {
 
     synchronized void registerMetric(KafkaMetric metric) {
         MetricName metricName = metric.metricName();
-        if (this.metrics.containsKey(metricName))
-            throw new IllegalArgumentException("A metric named '" + metricName + "' already exists, can't register another one.");
+
+        if (this.metrics.containsKey(metricName)) {
+            if (replaceOnDuplicate) {
+                log.error("The metric " + metricName + " is being replaced since it had already been registered. Please file a bug report.");
+                removeMetric(metricName);
+            } else {
+                throw new IllegalArgumentException("A metric named '" + metricName + "' already exists, can't register another one.");
+            }
+        }
+
         this.metrics.put(metricName, metric);
         for (MetricsReporter reporter : reporters) {
             try {
@@ -636,6 +646,10 @@ public MetricName metricInstance(MetricNameTemplate template, Map
Date: Fri, 6 Apr 2018 17:32:36 -0700
Subject: [PATCH 1021/1071] [LI-HOTFIX] LIKAFKA-16384: brokers should suggest
 the max ProduceRequest ApiVersion that supports the broker default configured
 message format version instead of always suggesting the maximum ApiVersion.
 TICKET = LI_DESCRIPTION =

RB=1275301
G=Kafka-Code-Reviews
A=tpalino,jkoshy,luwang

EXIT_CRITERIA = MANUAL [""]
---
 .../main/scala/kafka/server/KafkaApis.scala   | 38 +++++++++++++++++--
 1 file changed, 34 insertions(+), 4 deletions(-)

diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 3f7bf04848fa5..dc070e8c75e97 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -49,10 +49,11 @@ import org.apache.kafka.common.errors._
 import org.apache.kafka.common.internals.FatalExitError
 import org.apache.kafka.common.internals.Topic.{GROUP_METADATA_TOPIC_NAME, TRANSACTION_STATE_TOPIC_NAME, isInternal}
 import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic
-import org.apache.kafka.common.message.{AlterPartitionReassignmentsResponseData, CreateTopicsResponseData, DeleteGroupsResponseData, DeleteTopicsResponseData, DescribeGroupsResponseData, ExpireDelegationTokenResponseData, FindCoordinatorResponseData, HeartbeatResponseData, InitProducerIdResponseData, JoinGroupResponseData, LeaveGroupResponseData, ListGroupsResponseData, ListPartitionReassignmentsResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteResponseData, RenewDelegationTokenResponseData, SaslAuthenticateResponseData, SaslHandshakeResponseData, StopReplicaResponseData, SyncGroupResponseData, UpdateMetadataResponseData}
+import org.apache.kafka.common.message.{AlterPartitionReassignmentsResponseData, ApiVersionsResponseData, CreateTopicsResponseData, DeleteGroupsResponseData, DeleteTopicsResponseData, DescribeGroupsResponseData, ExpireDelegationTokenResponseData, FindCoordinatorResponseData, HeartbeatResponseData, InitProducerIdResponseData, JoinGroupResponseData, LeaveGroupResponseData, ListGroupsResponseData, ListPartitionReassignmentsResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteResponseData, RenewDelegationTokenResponseData, SaslAuthenticateResponseData, SaslHandshakeResponseData, StopReplicaResponseData, SyncGroupResponseData, UpdateMetadataResponseData}
 import org.apache.kafka.common.message.CreateTopicsResponseData.{CreatableTopicResult, CreatableTopicResultCollection}
 import org.apache.kafka.common.message.DeleteGroupsResponseData.{DeletableGroupResult, DeletableGroupResultCollection}
 import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.{ReassignablePartitionResponse, ReassignableTopicResponse}
+import org.apache.kafka.common.message.ApiVersionsResponseData.{ApiVersionsResponseKey, ApiVersionsResponseKeyCollection}
 import org.apache.kafka.common.message.DeleteTopicsResponseData.{DeletableTopicResult, DeletableTopicResultCollection}
 import org.apache.kafka.common.message.ElectLeadersResponseData.PartitionResult
 import org.apache.kafka.common.message.ElectLeadersResponseData.ReplicaElectionResult
@@ -1641,9 +1642,38 @@ class KafkaApis(val requestChannel: RequestChannel,
         apiVersionRequest.getErrorResponse(requestThrottleMs, Errors.UNSUPPORTED_VERSION.exception)
       else if (!apiVersionRequest.isValid)
         apiVersionRequest.getErrorResponse(requestThrottleMs, Errors.INVALID_REQUEST.exception)
-      else
-        ApiVersionsResponse.apiVersionsResponse(requestThrottleMs,
-          config.interBrokerProtocolVersion.recordVersion.value)
+      else {
+        /**
+          * HOTFIX LIKAFKA-16384: brokers should suggest the max ProduceRequest ApiVersion that supports the broker default
+          * configured message format version instead of always suggesting the maximum ApiVersion.
+          */
+        val maxProduceApiVersion: Short = config.logMessageFormatVersion.recordVersion match {
+          case RecordVersion.V0 => 1
+          case RecordVersion.V1 => 2
+          case RecordVersion.V2 => 8
+        }
+        val response = ApiVersionsResponse.apiVersionsResponse(requestThrottleMs, config.interBrokerProtocolVersion.recordVersion.value)
+        val apiVersions = response.data.apiKeys().asScala.toList.map { apiVersionResponseKey =>
+          if (apiVersionResponseKey.apiKey == ApiKeys.PRODUCE.id) {
+            new ApiVersionsResponseKey()
+              .setApiKey(ApiKeys.PRODUCE.id)
+              .setMinVersion(apiVersionResponseKey.minVersion())
+              .setMaxVersion(maxProduceApiVersion)
+          } else {
+            // Since the apiVersionResponseKey variable is a slice of the ImplicitLinkedHashCollection#Element object,
+            // we cannot insert the same object into the new list.
+            // Otherwise the prev and next pointers will be changed, and the original collection will be corrupted.
+            new ApiVersionsResponseKey()
+              .setApiKey(apiVersionResponseKey.apiKey())
+              .setMinVersion(apiVersionResponseKey.minVersion())
+              .setMaxVersion(apiVersionResponseKey.maxVersion())
+          }
+        }.asJava
+        new ApiVersionsResponse(new ApiVersionsResponseData()
+          .setApiKeys(new ApiVersionsResponseKeyCollection(apiVersions.iterator()))
+          .setErrorCode(Errors.NONE.code())
+          .setThrottleTimeMs(requestThrottleMs))
+      }
     }
     sendResponseMaybeThrottle(request, createResponseCallback)
   }

From 3cfa5d9d2f5efdd21417cde79a74c438019c9df0 Mon Sep 17 00:00:00 2001
From: kun du 
Date: Mon, 18 May 2020 14:37:04 -0700
Subject: [PATCH 1022/1071] [LI-HOTFIX] Add a metric to reflect broker fetch
 request failure rate. (#81)

LI_DESCRIPTION = The metric is helpful in detecting transient network issue.

EXIT_CRITERIA = MANUAL [""]
---
 .../kafka/server/AbstractFetcherManager.scala   | 17 +++++++++++++++++
 .../kafka/server/AbstractFetcherThread.scala    |  5 +++++
 .../server/AbstractFetcherThreadTest.scala      |  3 ++-
 3 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/kafka/server/AbstractFetcherManager.scala b/core/src/main/scala/kafka/server/AbstractFetcherManager.scala
index 2089858a50a74..fbb49c975b271 100755
--- a/core/src/main/scala/kafka/server/AbstractFetcherManager.scala
+++ b/core/src/main/scala/kafka/server/AbstractFetcherManager.scala
@@ -67,6 +67,23 @@ abstract class AbstractFetcherManager[T <: AbstractFetcherThread](val name: Stri
   Map("clientId" -> clientId)
   )
 
+  newGauge(
+    "FetchFailureRate", {
+      new Gauge[Double] {
+        // fetch failure rate sum across all fetchers/topics/partitions
+        def value: Double = {
+          val headRate: Double =
+            fetcherThreadMap.headOption.map(_._2.fetcherStats.requestFailureRate.oneMinuteRate).getOrElse(0)
+
+          fetcherThreadMap.foldLeft(headRate)((curSum, fetcherThreadMapEntry) => {
+            fetcherThreadMapEntry._2.fetcherStats.requestRate.oneMinuteRate + curSum
+          })
+        }
+      }
+    },
+    Map("clientId" -> clientId)
+  )
+
   val failedPartitionsCount = newGauge(
     "FailedPartitionsCount", {
       new Gauge[Int] {
diff --git a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
index 3747a0c8ddb8e..995e5beab02ae 100755
--- a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
+++ b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
@@ -305,6 +305,7 @@ abstract class AbstractFetcherThread(name: String,
       case t: Throwable =>
         if (isRunning) {
           warn(s"Error in response for fetch request $fetchRequest", t)
+          fetcherStats.requestFailureRate.mark()
           inLock(partitionMapLock) {
             partitionsWithError ++= partitionStates.partitionSet.asScala
             // there is an error occurred while fetching partitions, sleep a while
@@ -705,6 +706,7 @@ object AbstractFetcherThread {
 object FetcherMetrics {
   val ConsumerLag = "ConsumerLag"
   val RequestsPerSec = "RequestsPerSec"
+  val RequestFailuresPerSec = "RequestFailuresPerSec"
   val BytesPerSec = "BytesPerSec"
 }
 
@@ -769,10 +771,13 @@ class FetcherStats(metricId: ClientIdAndBroker) extends KafkaMetricsGroup {
 
   val requestRate = newMeter(FetcherMetrics.RequestsPerSec, "requests", TimeUnit.SECONDS, tags)
 
+  val requestFailureRate = newMeter(FetcherMetrics.RequestFailuresPerSec, "requestFailures", TimeUnit.SECONDS, tags)
+
   val byteRate = newMeter(FetcherMetrics.BytesPerSec, "bytes", TimeUnit.SECONDS, tags)
 
   def unregister(): Unit = {
     removeMetric(FetcherMetrics.RequestsPerSec, tags)
+    removeMetric(FetcherMetrics.RequestFailuresPerSec, tags)
     removeMetric(FetcherMetrics.BytesPerSec, tags)
   }
 
diff --git a/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala b/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala
index 55c38a1deab80..3a44e38b2ccf9 100644
--- a/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala
+++ b/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala
@@ -80,7 +80,8 @@ class AbstractFetcherThreadTest {
 
     // wait until all fetcher metrics are present
     TestUtils.waitUntilTrue(() =>
-      allMetricsNames == Set(FetcherMetrics.BytesPerSec, FetcherMetrics.RequestsPerSec, FetcherMetrics.ConsumerLag),
+      allMetricsNames == Set(FetcherMetrics.BytesPerSec, FetcherMetrics.RequestFailuresPerSec, FetcherMetrics.RequestsPerSec,
+                             FetcherMetrics.ConsumerLag),
       "Failed waiting for all fetcher metrics to be registered")
 
     fetcher.shutdown()

From 22d4522787b42d86c1e4c2a25731c9f77989a377 Mon Sep 17 00:00:00 2001
From: Kyle Ambroff-Kao 
Date: Wed, 29 Apr 2020 14:33:10 -0700
Subject: [PATCH 1023/1071] [LI-HOTFIX] Cherry-pick fix for KAFKA-9932

TICKET = LIKAFKA-20348
LI_DESCRIPTION = Speed up handling of LeaderAndISRRequest in some cases.

KAFKA-9932: Don't load configs from ZK when the log has already been loaded (#8582)

If a broker contains 8k replicas, we would previously issue 8k ZK calls to retrieve topic
configs when processing the first LeaderAndIsr request. That should translate to 0 after
these changes.

Conflicts:
  core/src/main/scala/kafka/cluster/Partition.scala
  core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
  core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala

EXIT_CRITERIA = MANUAL ["We can drop this when we rebase with Kafka 2.6"]
---
 .../main/scala/kafka/cluster/Partition.scala  | 12 ++++++----
 .../src/main/scala/kafka/log/LogManager.scala | 16 +++++++------
 .../unit/kafka/cluster/PartitionTest.scala    | 24 +++++++++----------
 .../scala/unit/kafka/log/LogManagerTest.scala | 24 +++++++++----------
 .../server/HighwatermarkPersistenceTest.scala |  6 ++---
 .../unit/kafka/server/LogOffsetTest.scala     |  4 ++--
 .../kafka/server/ReplicaManagerTest.scala     |  6 ++---
 7 files changed, 49 insertions(+), 43 deletions(-)

diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala
index fb8aa38466b6e..a07f6c2c3b73c 100755
--- a/core/src/main/scala/kafka/cluster/Partition.scala
+++ b/core/src/main/scala/kafka/cluster/Partition.scala
@@ -320,17 +320,21 @@ class Partition(val topicPartition: TopicPartition,
       LogConfig.fromProps(logManager.currentDefaultConfig.originals, props)
     }
 
-    logManager.initializingLog(topicPartition)
-    var maybeLog: Option[Log] = None
-    try {
-      val log = logManager.getOrCreateLog(topicPartition, fetchLogConfig(), isNew, isFutureReplica)
+    def updateHighWatermark(log: Log) = {
       val checkpointHighWatermark = offsetCheckpoints.fetch(log.dir.getParent, topicPartition).getOrElse {
         info(s"No checkpointed highwatermark is found for partition $topicPartition")
         0L
       }
       val initialHighWatermark = log.updateHighWatermark(checkpointHighWatermark)
       info(s"Log loaded for partition $topicPartition with initial high watermark $initialHighWatermark")
+    }
+
+    logManager.initializingLog(topicPartition)
+    var maybeLog: Option[Log] = None
+    try {
+      val log = logManager.getOrCreateLog(topicPartition, () => fetchLogConfig() : LogConfig, isNew, isFutureReplica)
       maybeLog = Some(log)
+      updateHighWatermark(log)
       log
     } finally {
       logManager.finishedInitializingLog(topicPartition, maybeLog, fetchLogConfig)
diff --git a/core/src/main/scala/kafka/log/LogManager.scala b/core/src/main/scala/kafka/log/LogManager.scala
index 7cbc79b9743e0..af97621841e68 100755
--- a/core/src/main/scala/kafka/log/LogManager.scala
+++ b/core/src/main/scala/kafka/log/LogManager.scala
@@ -696,16 +696,17 @@ class LogManager(logDirs: Seq[File],
 
   /**
    * Method to indicate that the log initialization for the partition passed in as argument is
-   * finished. This method should follow a call to [[kafka.log.LogManager#initializingLog]]
+   * finished. This method should follow a call to [[kafka.log.LogManager#initializingLog]].
+   *
+   * It will retrieve the topic configs a second time if they were updated while the
+   * relevant log was being loaded.
    */
   def finishedInitializingLog(topicPartition: TopicPartition,
                               maybeLog: Option[Log],
                               fetchLogConfig: () => LogConfig): Unit = {
-    if (partitionsInitializing(topicPartition)) {
+    val removedValue = partitionsInitializing.remove(topicPartition)
+    if (removedValue.contains(true))
       maybeLog.foreach(_.updateConfig(fetchLogConfig()))
-    }
-
-    partitionsInitializing -= topicPartition
   }
 
   /**
@@ -714,12 +715,12 @@ class LogManager(logDirs: Seq[File],
    * Otherwise throw KafkaStorageException
    *
    * @param topicPartition The partition whose log needs to be returned or created
-   * @param config The configuration of the log that should be applied for log creation
+   * @param loadConfig A function to retrieve the log config, this is only called if the log is created
    * @param isNew Whether the replica should have existed on the broker or not
    * @param isFuture True iff the future log of the specified partition should be returned or created
    * @throws KafkaStorageException if isNew=false, log is not found in the cache and there is offline log directory on the broker
    */
-  def getOrCreateLog(topicPartition: TopicPartition, config: LogConfig, isNew: Boolean = false, isFuture: Boolean = false): Log = {
+  def getOrCreateLog(topicPartition: TopicPartition, loadConfig: () => LogConfig, isNew: Boolean = false, isFuture: Boolean = false): Log = {
     logCreationOrDeletionLock synchronized {
       getLog(topicPartition, isFuture).getOrElse {
         // create the log if it has not already been created in another thread
@@ -756,6 +757,7 @@ class LogManager(logDirs: Seq[File],
           .getOrElse(Failure(new KafkaStorageException("No log directories available. Tried " + logDirs.map(_.getAbsolutePath).mkString(", "))))
           .get // If Failure, will throw
 
+        val config = loadConfig()
         val log = Log(
           dir = logDir,
           config = config,
diff --git a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
index 7f50ce1fd8b64..31f260d781c42 100644
--- a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
+++ b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
@@ -114,7 +114,7 @@ class PartitionTest {
   def testMakeLeaderUpdatesEpochCache(): Unit = {
     val leaderEpoch = 8
 
-    val log = logManager.getOrCreateLog(topicPartition, logConfig)
+    val log = logManager.getOrCreateLog(topicPartition, () => logConfig)
     log.appendAsLeader(MemoryRecords.withRecords(0L, CompressionType.NONE, 0,
       new SimpleRecord("k1".getBytes, "v1".getBytes),
       new SimpleRecord("k2".getBytes, "v2".getBytes)
@@ -140,7 +140,7 @@ class PartitionTest {
 
     val logConfig = LogConfig(createLogProperties(Map(
       LogConfig.MessageFormatVersionProp -> kafka.api.KAFKA_0_10_2_IV0.shortVersion)))
-    val log = logManager.getOrCreateLog(topicPartition, logConfig)
+    val log = logManager.getOrCreateLog(topicPartition, () => logConfig)
     log.appendAsLeader(TestUtils.records(List(
       new SimpleRecord("k1".getBytes, "v1".getBytes),
       new SimpleRecord("k2".getBytes, "v2".getBytes)),
@@ -642,7 +642,7 @@ class PartitionTest {
 
   private def setupPartitionWithMocks(leaderEpoch: Int,
                                       isLeader: Boolean,
-                                      log: Log = logManager.getOrCreateLog(topicPartition, logConfig)): Partition = {
+                                      log: Log = logManager.getOrCreateLog(topicPartition, () => logConfig)): Partition = {
     partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
 
     val controllerId = 0
@@ -952,7 +952,7 @@ class PartitionTest {
     val logConfig = LogConfig(new Properties)
 
     val topicPartitions = (0 until 5).map { i => new TopicPartition("test-topic", i) }
-    val logs = topicPartitions.map { tp => logManager.getOrCreateLog(tp, logConfig) }
+    val logs = topicPartitions.map { tp => logManager.getOrCreateLog(tp, () => logConfig) }
     val partitions = ListBuffer.empty[Partition]
 
     logs.foreach { log =>
@@ -1088,7 +1088,7 @@ class PartitionTest {
 
   @Test
   def testUpdateFollowerFetchState(): Unit = {
-    val log = logManager.getOrCreateLog(topicPartition, logConfig)
+    val log = logManager.getOrCreateLog(topicPartition, () => logConfig)
     seedLogData(log, numRecords = 6, leaderEpoch = 4)
 
     val controllerId = 0
@@ -1152,7 +1152,7 @@ class PartitionTest {
 
   @Test
   def testIsrExpansion(): Unit = {
-    val log = logManager.getOrCreateLog(topicPartition, logConfig)
+    val log = logManager.getOrCreateLog(topicPartition, () => logConfig)
     seedLogData(log, numRecords = 10, leaderEpoch = 4)
 
     val controllerId = 0
@@ -1219,7 +1219,7 @@ class PartitionTest {
 
   @Test
   def testIsrNotExpandedIfUpdateFails(): Unit = {
-    val log = logManager.getOrCreateLog(topicPartition, logConfig)
+    val log = logManager.getOrCreateLog(topicPartition, () => logConfig)
     seedLogData(log, numRecords = 10, leaderEpoch = 4)
 
     val controllerId = 0
@@ -1274,7 +1274,7 @@ class PartitionTest {
 
   @Test
   def testMaybeShrinkIsr(): Unit = {
-    val log = logManager.getOrCreateLog(topicPartition, logConfig)
+    val log = logManager.getOrCreateLog(topicPartition, () => logConfig)
     seedLogData(log, numRecords = 10, leaderEpoch = 4)
 
     val controllerId = 0
@@ -1331,7 +1331,7 @@ class PartitionTest {
 
   @Test
   def testShouldNotShrinkIsrIfPreviousFetchIsCaughtUp(): Unit = {
-    val log = logManager.getOrCreateLog(topicPartition, logConfig)
+    val log = logManager.getOrCreateLog(topicPartition, () => logConfig)
     seedLogData(log, numRecords = 10, leaderEpoch = 4)
 
     val controllerId = 0
@@ -1405,7 +1405,7 @@ class PartitionTest {
 
   @Test
   def testShouldNotShrinkIsrIfFollowerCaughtUpToLogEnd(): Unit = {
-    val log = logManager.getOrCreateLog(topicPartition, logConfig)
+    val log = logManager.getOrCreateLog(topicPartition, () => logConfig)
     seedLogData(log, numRecords = 10, leaderEpoch = 4)
 
     val controllerId = 0
@@ -1464,7 +1464,7 @@ class PartitionTest {
 
   @Test
   def testIsrNotShrunkIfUpdateFails(): Unit = {
-    val log = logManager.getOrCreateLog(topicPartition, logConfig)
+    val log = logManager.getOrCreateLog(topicPartition, () => logConfig)
     seedLogData(log, numRecords = 10, leaderEpoch = 4)
 
     val controllerId = 0
@@ -1516,7 +1516,7 @@ class PartitionTest {
 
   @Test
   def testUseCheckpointToInitializeHighWatermark(): Unit = {
-    val log = logManager.getOrCreateLog(topicPartition, logConfig)
+    val log = logManager.getOrCreateLog(topicPartition, () => logConfig)
     seedLogData(log, numRecords = 6, leaderEpoch = 5)
 
     when(offsetCheckpoints.fetch(logDir1.getAbsolutePath, topicPartition))
diff --git a/core/src/test/scala/unit/kafka/log/LogManagerTest.scala b/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
index a7c17f440fc2e..15139b5f22210 100755
--- a/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
@@ -74,7 +74,7 @@ class LogManagerTest {
    */
   @Test
   def testCreateLog(): Unit = {
-    val log = logManager.getOrCreateLog(new TopicPartition(name, 0), logConfig)
+    val log = logManager.getOrCreateLog(new TopicPartition(name, 0), () => logConfig)
     assertEquals(1, logManager.liveLogDirs.size)
 
     val logFile = new File(logDir, name + "-0")
@@ -97,7 +97,7 @@ class LogManagerTest {
     logManager = createLogManager(dirs)
     logManager.startup()
 
-    val log = logManager.getOrCreateLog(new TopicPartition(name, 0), logConfig, isNew = true)
+    val log = logManager.getOrCreateLog(new TopicPartition(name, 0), () => logConfig, isNew = true)
     val logFile = new File(logDir, name + "-0")
     assertTrue(logFile.exists)
     log.appendAsLeader(TestUtils.singletonRecords("test".getBytes()), leaderEpoch = 0)
@@ -131,7 +131,7 @@ class LogManagerTest {
 
     // Request creating a new log.
     // LogManager should try using all configured log directories until one succeeds.
-    logManager.getOrCreateLog(new TopicPartition(name, 0), logConfig, isNew = true)
+    logManager.getOrCreateLog(new TopicPartition(name, 0), () => logConfig, isNew = true)
 
     // Verify that half the directories were considered broken,
     assertEquals(dirs.length / 2, brokenDirs.size)
@@ -160,7 +160,7 @@ class LogManagerTest {
    */
   @Test
   def testCleanupExpiredSegments(): Unit = {
-    val log = logManager.getOrCreateLog(new TopicPartition(name, 0), logConfig)
+    val log = logManager.getOrCreateLog(new TopicPartition(name, 0), () => logConfig)
     var offset = 0L
     for(_ <- 0 until 200) {
       val set = TestUtils.singletonRecords("test".getBytes())
@@ -211,7 +211,7 @@ class LogManagerTest {
     logManager.startup()
 
     // create a log
-    val log = logManager.getOrCreateLog(new TopicPartition(name, 0), config)
+    val log = logManager.getOrCreateLog(new TopicPartition(name, 0), () => config)
     var offset = 0L
 
     // add a bunch of messages that should be larger than the retentionSize
@@ -265,7 +265,7 @@ class LogManagerTest {
   private def testDoesntCleanLogs(policy: String): Unit = {
     val logProps = new Properties()
     logProps.put(LogConfig.CleanupPolicyProp, policy)
-    val log = logManager.getOrCreateLog(new TopicPartition(name, 0), LogConfig.fromProps(logConfig.originals, logProps))
+    val log = logManager.getOrCreateLog(new TopicPartition(name, 0), () => LogConfig.fromProps(logConfig.originals, logProps))
     var offset = 0L
     for (_ <- 0 until 200) {
       val set = TestUtils.singletonRecords("test".getBytes(), key="test".getBytes())
@@ -294,7 +294,7 @@ class LogManagerTest {
 
     logManager = createLogManager()
     logManager.startup()
-    val log = logManager.getOrCreateLog(new TopicPartition(name, 0), config)
+    val log = logManager.getOrCreateLog(new TopicPartition(name, 0), () => config)
     val lastFlush = log.lastFlushTime
     for (_ <- 0 until 200) {
       val set = TestUtils.singletonRecords("test".getBytes())
@@ -318,7 +318,7 @@ class LogManagerTest {
 
     // verify that logs are always assigned to the least loaded partition
     for(partition <- 0 until 20) {
-      logManager.getOrCreateLog(new TopicPartition("test", partition), logConfig)
+      logManager.getOrCreateLog(new TopicPartition("test", partition), () => logConfig)
       assertEquals("We should have created the right number of logs", partition + 1, logManager.allLogs.size)
       val counts = logManager.allLogs.groupBy(_.dir.getParent).values.map(_.size)
       assertTrue("Load should balance evenly", counts.max <= counts.min + 1)
@@ -369,7 +369,7 @@ class LogManagerTest {
   }
 
   private def verifyCheckpointRecovery(topicPartitions: Seq[TopicPartition], logManager: LogManager, logDir: File): Unit = {
-    val logs = topicPartitions.map(logManager.getOrCreateLog(_, logConfig))
+    val logs = topicPartitions.map(logManager.getOrCreateLog(_, () => logConfig))
     logs.foreach { log =>
       for (_ <- 0 until 50)
         log.appendAsLeader(TestUtils.singletonRecords("test".getBytes()), leaderEpoch = 0)
@@ -395,7 +395,7 @@ class LogManagerTest {
 
   @Test
   def testFileReferencesAfterAsyncDelete(): Unit = {
-    val log = logManager.getOrCreateLog(new TopicPartition(name, 0), logConfig)
+    val log = logManager.getOrCreateLog(new TopicPartition(name, 0), () => logConfig)
     val activeSegment = log.activeSegment
     val logName = activeSegment.log.file.getName
     val indexName = activeSegment.offsetIndex.file.getName
@@ -432,7 +432,7 @@ class LogManagerTest {
   @Test
   def testCreateAndDeleteOverlyLongTopic(): Unit = {
     val invalidTopicName = String.join("", Collections.nCopies(253, "x"))
-    logManager.getOrCreateLog(new TopicPartition(invalidTopicName, 0), logConfig)
+    logManager.getOrCreateLog(new TopicPartition(invalidTopicName, 0), () => logConfig)
     logManager.asyncDelete(new TopicPartition(invalidTopicName, 0))
   }
 
@@ -445,7 +445,7 @@ class LogManagerTest {
       new TopicPartition("test-b", 0),
       new TopicPartition("test-b", 1))
 
-    val allLogs = tps.map(logManager.getOrCreateLog(_, logConfig))
+    val allLogs = tps.map(logManager.getOrCreateLog(_, () => logConfig))
     allLogs.foreach { log =>
       for (_ <- 0 until 50)
         log.appendAsLeader(TestUtils.singletonRecords("test".getBytes), leaderEpoch = 0)
diff --git a/core/src/test/scala/unit/kafka/server/HighwatermarkPersistenceTest.scala b/core/src/test/scala/unit/kafka/server/HighwatermarkPersistenceTest.scala
index 1efecedf9806e..e248f3b678ad2 100755
--- a/core/src/test/scala/unit/kafka/server/HighwatermarkPersistenceTest.scala
+++ b/core/src/test/scala/unit/kafka/server/HighwatermarkPersistenceTest.scala
@@ -75,7 +75,7 @@ class HighwatermarkPersistenceTest {
       val tp0 = new TopicPartition(topic, 0)
       val partition0 = replicaManager.createPartition(tp0)
       // create leader and follower replicas
-      val log0 = logManagers.head.getOrCreateLog(new TopicPartition(topic, 0), LogConfig())
+      val log0 = logManagers.head.getOrCreateLog(new TopicPartition(topic, 0), () => LogConfig())
       partition0.setLog(log0, isFutureLog = false)
 
       partition0.updateAssignmentAndIsr(
@@ -123,7 +123,7 @@ class HighwatermarkPersistenceTest {
       val t1p0 = new TopicPartition(topic1, 0)
       val topic1Partition0 = replicaManager.createPartition(t1p0)
       // create leader log
-      val topic1Log0 = logManagers.head.getOrCreateLog(t1p0, LogConfig())
+      val topic1Log0 = logManagers.head.getOrCreateLog(t1p0, () => LogConfig())
       // create a local replica for topic1
       topic1Partition0.setLog(topic1Log0, isFutureLog = false)
       replicaManager.checkpointHighWatermarks()
@@ -140,7 +140,7 @@ class HighwatermarkPersistenceTest {
       val t2p0 = new TopicPartition(topic2, 0)
       val topic2Partition0 = replicaManager.createPartition(t2p0)
       // create leader log
-      val topic2Log0 = logManagers.head.getOrCreateLog(t2p0, LogConfig())
+      val topic2Log0 = logManagers.head.getOrCreateLog(t2p0, () => LogConfig())
       // create a local replica for topic2
       topic2Partition0.setLog(topic2Log0, isFutureLog = false)
       replicaManager.checkpointHighWatermarks()
diff --git a/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala b/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala
index d88540d0a415d..abbb174e4665f 100755
--- a/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala
+++ b/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala
@@ -163,7 +163,7 @@ class LogOffsetTest extends BaseRequestTest {
     createTopic(topic, 3, 1)
 
     val logManager = server.getLogManager
-    val log = logManager.getOrCreateLog(topicPartition, logManager.initialDefaultConfig)
+    val log = logManager.getOrCreateLog(topicPartition, () => logManager.initialDefaultConfig)
 
     for (_ <- 0 until 20)
       log.appendAsLeader(TestUtils.singletonRecords(value = Integer.toString(42).getBytes()), leaderEpoch = 0)
@@ -193,7 +193,7 @@ class LogOffsetTest extends BaseRequestTest {
     createTopic(topic, 3, 1)
 
     val logManager = server.getLogManager
-    val log = logManager.getOrCreateLog(topicPartition, logManager.initialDefaultConfig)
+    val log = logManager.getOrCreateLog(topicPartition, () => logManager.initialDefaultConfig)
     for (_ <- 0 until 20)
       log.appendAsLeader(TestUtils.singletonRecords(value = Integer.toString(42).getBytes()), leaderEpoch = 0)
     log.flush()
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index d605eff6b7b29..c9be250f0c218 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -1373,9 +1373,9 @@ class ReplicaManagerTest {
     val topicPartitionObj = new TopicPartition(topic, topicPartition)
     val mockLogMgr: LogManager = EasyMock.createMock(classOf[LogManager])
     EasyMock.expect(mockLogMgr.liveLogDirs).andReturn(config.logDirs.map(new File(_).getAbsoluteFile)).anyTimes
-    EasyMock.expect(mockLogMgr.currentDefaultConfig).andReturn(LogConfig())
-    EasyMock.expect(mockLogMgr.getOrCreateLog(topicPartitionObj,
-      LogConfig(), isNew = false, isFuture = false)).andReturn(mockLog).anyTimes
+    EasyMock.expect(mockLogMgr.getOrCreateLog(EasyMock.eq(topicPartitionObj),
+      EasyMock.anyObject[() => LogConfig](), isNew = EasyMock.eq(false),
+      isFuture = EasyMock.eq(false))).andReturn(mockLog).anyTimes
     if (expectTruncation) {
       EasyMock.expect(mockLogMgr.truncateTo(Map(topicPartitionObj -> offsetFromLeader),
         isFuture = false)).once

From 3cba40cacc5229f27530da6ea7b3e13787a5dc65 Mon Sep 17 00:00:00 2001
From: "Zhanxiang (Patrick) Huang" 
Date: Wed, 17 Apr 2019 18:22:37 -0700
Subject: [PATCH 1024/1071] [LI-HOTFIX] Exclude topics being deleted from the
 offlinePartitionCount metric and clean up partitionState in
 PartitionStateMachine after topic deletion is done (#13) TICKET =
 LI_DESCRIPTION =

Exclude topics being deleted from the offlinePartitionCount metric and clean up partitionState in PartitionStateMachine after topic deletion is done

Currently the offlinePartitionCount metric also reports the partitions of the topic
that has already been queued for deletion, which creates noise for the alerting
system, especially for the cluster that has frequent topic deletion operation. This
patch adds a mechanism to exclude partitions already been queued for deletion from
the offlinePartitionCount metric and also remove the in-memory topicsWithDeletionStarted
in TopicDeletionManager since we no longer use it to update the metric.

This patch also addresses a potential memory pressure issue of not cleaning up the in-memory
partition states in PartitionStateMachine even after the topic has already been deleted.

EXIT_CRITERIA = MANUAL ["limiting this to LinkedIn branch only because we don't want to create offline partition noise for topics being deleting"]
---
 .../kafka/controller/ControllerContext.scala  | 13 ++++++++++-
 .../controller/TopicDeletionManager.scala     |  4 +++-
 .../ControllerIntegrationTest.scala           | 22 ++++++++++++++++++-
 3 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/core/src/main/scala/kafka/controller/ControllerContext.scala b/core/src/main/scala/kafka/controller/ControllerContext.scala
index 93b0b4d68dd97..e5f584304f690 100644
--- a/core/src/main/scala/kafka/controller/ControllerContext.scala
+++ b/core/src/main/scala/kafka/controller/ControllerContext.scala
@@ -288,6 +288,10 @@ class ControllerContext {
   def removeTopic(topic: String): Unit = {
     allTopics -= topic
     partitionAssignments.remove(topic)
+    partitionStates.foreach {
+      case (topicPartition, _) if topicPartition.topic == topic => partitionStates.remove(topicPartition)
+      case _ =>
+    }
     partitionLeadershipInfo.foreach {
       case (topicPartition, _) if topicPartition.topic == topic => partitionLeadershipInfo.remove(topicPartition)
       case _ =>
@@ -355,10 +359,17 @@ class ControllerContext {
     updatePartitionStateMetrics(partition, currentState, targetState)
   }
 
+  def excludeDeletingTopicFromOfflinePartitionCount(topic: String): Unit = {
+    if (isTopicQueuedUpForDeletion(topic)) {
+      offlinePartitionCount = offlinePartitionCount -
+        partitionsForTopic(topic).count(partition => partitionState(partition) == OfflinePartition)
+    }
+  }
+
   private def updatePartitionStateMetrics(partition: TopicPartition,
                                           currentState: PartitionState,
                                           targetState: PartitionState): Unit = {
-    if (!isTopicDeletionInProgress(partition.topic)) {
+    if (!isTopicQueuedUpForDeletion(partition.topic)) {
       if (currentState != OfflinePartition && targetState == OfflinePartition) {
         offlinePartitionCount = offlinePartitionCount + 1
       } else if (currentState == OfflinePartition && targetState != OfflinePartition) {
diff --git a/core/src/main/scala/kafka/controller/TopicDeletionManager.scala b/core/src/main/scala/kafka/controller/TopicDeletionManager.scala
index fa8cfdf547e5d..f5321258edcc0 100755
--- a/core/src/main/scala/kafka/controller/TopicDeletionManager.scala
+++ b/core/src/main/scala/kafka/controller/TopicDeletionManager.scala
@@ -139,7 +139,9 @@ class TopicDeletionManager(config: KafkaConfig,
    */
   def enqueueTopicsForDeletion(topics: Set[String]): Unit = {
     if (isDeleteTopicEnabled) {
-      controllerContext.queueTopicDeletion(topics)
+      val newTopicsToBeDeleted = topics -- controllerContext.topicsToBeDeleted
+      controllerContext.queueTopicDeletion(newTopicsToBeDeleted)
+      newTopicsToBeDeleted.foreach(controllerContext.excludeDeletingTopicFromOfflinePartitionCount)
       resumeDeletions()
     }
   }
diff --git a/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala b/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala
index 018a0bb8cb014..37eae0e3002f6 100644
--- a/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala
@@ -403,6 +403,24 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
       "failed to get expected partition state upon broker startup")
   }
 
+  @Test
+  def testTopicDeletionCleanUpPartitionState(): Unit = {
+    servers = makeServers(3, enableDeleteTopic = true)
+    val controllerId = TestUtils.waitUntilControllerElected(zkClient)
+    val controller = servers.find(broker => broker.config.brokerId == controllerId).get.kafkaController
+    val tp = new TopicPartition("t", 0)
+    val assignment = Map(tp.partition -> Seq(1, 0, 2))
+    TestUtils.createTopic(zkClient, tp.topic, partitionReplicaAssignment = assignment, servers = servers)
+
+    // Make sure the partition state has been populated
+    assertTrue(controller.controllerContext.partitionStates.contains(tp))
+    adminZkClient.deleteTopic(tp.topic())
+    TestUtils.verifyTopicDeletion(zkClient, tp.topic(), 1, servers)
+
+    // Make sure the partition state has been removed
+    assertTrue(!controller.controllerContext.partitionStates.contains(tp))
+  }
+
   @Test
   def testLeaderAndIsrWhenEntireIsrOfflineAndUncleanLeaderElectionDisabled(): Unit = {
     servers = makeServers(2)
@@ -684,7 +702,8 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
                           enableControlledShutdown: Boolean = true,
                           listeners : Option[String] = None,
                           listenerSecurityProtocolMap : Option[String] = None,
-                          controlPlaneListenerName : Option[String] = None) = {
+                          controlPlaneListenerName : Option[String] = None,
+                          enableDeleteTopic: Boolean = false) = {
     val configs = TestUtils.createBrokerConfigs(numConfigs, zkConnect, enableControlledShutdown = enableControlledShutdown)
     configs.foreach { config =>
       config.setProperty(KafkaConfig.AutoLeaderRebalanceEnableProp, autoLeaderRebalanceEnable.toString)
@@ -693,6 +712,7 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
       listeners.foreach(listener => config.setProperty(KafkaConfig.ListenersProp, listener))
       listenerSecurityProtocolMap.foreach(listenerMap => config.setProperty(KafkaConfig.ListenerSecurityProtocolMapProp, listenerMap))
       controlPlaneListenerName.foreach(controlPlaneListener => config.setProperty(KafkaConfig.ControlPlaneListenerNameProp, controlPlaneListener))
+      config.setProperty(KafkaConfig.DeleteTopicEnableProp, enableDeleteTopic.toString)
     }
     configs.map(config => TestUtils.createServer(KafkaConfig.fromProps(config)))
   }

From 697c0453237f2b9ff03d43b8e1b364bbb9cff4f0 Mon Sep 17 00:00:00 2001
From: Virupaksha Swamy 
Date: Thu, 25 Apr 2019 16:41:34 -0700
Subject: [PATCH 1025/1071] [LI-HOTFIX] Fix flush behavior

TICKET = KAFKA-8021

LI_DESCRIPTION = kafkaProducer#flush() api is expected to evaluate all record futures previously obtained via kafkaProducer#send(). The record futures can either be successful or may throw exception depending on the produce response by the broker but nonetheless, they should all be evaluated. This behavior is broken when a produce batch is split into smaller batches(because of RecordTooLargeException) and rescheduled to be sent to the broker. That is, it can so happen that after successfully executing #flush() api, there may exist few record futures which are not evaluated. This change checks if any of the produce batch failed with RecordTooLargeException and in such a case, retries flush operation of all incomplete batches. Note that, Although this change does not break the #flush() api contract, it has a behavior change in that more batches may get flushed than before. Rejected alternative: The other alternative was to chain the produce batch future. This approach fails to evaluate record futures as soon as their corresponding produce batch futures are evaluated. It also introduces a erroneous scenario where if one of the split batches are evaluated and the other split batch failed with an exception, we end up failing all record futures including the ones which belonged to the successful split batch.

EXIT_CRITERIA = TICKET [KAFKA-8021]
---
 .../producer/internals/RecordAccumulator.java | 38 ++++++++++----
 .../internals/RecordAccumulatorTest.java      | 51 +++++++++++++++++++
 2 files changed, 80 insertions(+), 9 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
index e5d8ec32f2c15..49a59f5be5e0c 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
@@ -38,6 +38,7 @@
 import org.apache.kafka.common.Node;
 import org.apache.kafka.common.PartitionInfo;
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.RecordBatchTooLargeException;
 import org.apache.kafka.common.errors.TimeoutException;
 import org.apache.kafka.common.errors.UnsupportedVersionException;
 import org.apache.kafka.common.header.Header;
@@ -685,6 +686,13 @@ boolean flushInProgress() {
         return flushesInProgress.get() > 0;
     }
 
+    /**
+     * This method should be used only for testing.
+     */
+    IncompleteBatches incompleteBatches() {
+        return incomplete;
+    }
+
     /* Visible for testing */
     Map> batches() {
         return Collections.unmodifiableMap(batches);
@@ -709,17 +717,29 @@ private boolean appendsInProgress() {
      */
     public void awaitFlushCompletion(long timeoutMs) throws InterruptedException {
         try {
+            boolean retry;
             Long expireMs = System.currentTimeMillis() + timeoutMs;
-            for (ProducerBatch batch : this.incomplete.copyAll()) {
-                Long currentMs = System.currentTimeMillis();
-                if (currentMs > expireMs) {
-                    throw new TimeoutException("Failed to flush accumulated records within" + timeoutMs + "milliseconds.");
-                }
-                boolean completed = batch.produceFuture.await(Math.max(expireMs - currentMs, 0), TimeUnit.MILLISECONDS);
-                if (!completed) {
-                    throw new TimeoutException("Failed to flush accumulated records within" + timeoutMs + "milliseconds.");
+            do {
+                retry = false;
+                for (ProducerBatch batch : this.incomplete.copyAll()) {
+                    Long currentMs = System.currentTimeMillis();
+                    if (currentMs > expireMs) {
+                        throw new TimeoutException("Failed to flush accumulated records within" + timeoutMs + "milliseconds.");
+                    }
+                    boolean completed = batch.produceFuture.await(Math.max(expireMs - currentMs, 0), TimeUnit.MILLISECONDS);
+                    if (!completed) {
+                        throw new TimeoutException("Failed to flush accumulated records within" + timeoutMs + "milliseconds.");
+                    }
+                    // If the produceFuture failed with RecordBatchTooLargeException, it means that the
+                    // batch was split into smaller batches and re-enqueued into the RecordAccumulator by Sender thread.
+                    // This if condition will make sure to retry and send all the split batches.
+                    // Note that, More records get sent to the broker than necessary because the retry mechanism
+                    // will also include all the newly added records via kafkaProducer.send() api.
+                    if (batch.produceFuture.error() instanceof RecordBatchTooLargeException) {
+                        retry = true;
+                    }
                 }
-            }
+            } while (retry);
         } finally {
             this.flushesInProgress.decrementAndGet();
         }
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java
index c1a63c92e5f23..1d76344c159a2 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java
@@ -16,6 +16,8 @@
  */
 package org.apache.kafka.clients.producer.internals;
 
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.kafka.clients.ApiVersions;
 import org.apache.kafka.clients.NodeApiVersions;
 import org.apache.kafka.clients.producer.Callback;
@@ -44,6 +46,7 @@
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.test.TestUtils;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Test;
 
 import java.nio.ByteBuffer;
@@ -415,6 +418,54 @@ public void run() {
         t.start();
     }
 
+    @Test
+    public void testSplitAwaitFlushComplete() throws Exception {
+        RecordAccumulator accum = createTestRecordAccumulator(1024, 10 * 1024, CompressionType.GZIP, 10);
+
+        // Create a big batch
+        byte[] value = new byte[256];
+        // Create a batch such that it fails with RecordBatchTooLargeException
+        accum.append(new TopicPartition(topic, 0), 0L, null, value, null, null, maxBlockTimeMs, false);
+        accum.append(new TopicPartition(topic, 0), 0L, null, value, null, null, maxBlockTimeMs, false);
+
+        CountDownLatch flushInProgress = new CountDownLatch(1);
+        Iterator incompleteBatches = accum.incompleteBatches().copyAll().iterator();
+
+        // Assert that there is only one batch
+        Assert.assertTrue(incompleteBatches.hasNext());
+        ProducerBatch producerBatch = incompleteBatches.next();
+        Assert.assertFalse(incompleteBatches.hasNext());
+
+        AtomicBoolean timedOut = new AtomicBoolean(false);
+        Thread thread = new Thread(() -> {
+            Assert.assertTrue(accum.hasIncomplete());
+            accum.beginFlush();
+            Assert.assertTrue(accum.flushInProgress());
+            try {
+                flushInProgress.countDown();
+                accum.awaitFlushCompletion(2000);
+            } catch (TimeoutException timeoutException) {
+              // Catch it and set the timedout variable
+              // This is the only valid path for this thread.
+                timedOut.set(true);
+            } catch (InterruptedException e) {
+            }
+        });
+        thread.start();
+        flushInProgress.await();
+        // Wait for 100ms to make sure that the flush is actually in progress
+        Thread.sleep(100);
+
+        // Split the big batch and re-enqueue
+        accum.splitAndReenqueue(producerBatch);
+        accum.deallocate(producerBatch);
+
+        thread.join();
+        // The thread would have failed with timeout exception because the child batches
+        // are not evaluated and it would have waited for 2seconds before the timeout.
+        Assert.assertTrue("The thread should have timed out", timedOut.get());
+    }
+
     @Test
     public void testAwaitFlushComplete() throws Exception {
         RecordAccumulator accum = createTestRecordAccumulator(

From 7b20d0322bfb7a88bd131363a70195c6e9c5e5a2 Mon Sep 17 00:00:00 2001
From: Abhishek Mendhekar 
Date: Wed, 12 Jun 2019 15:03:09 -0700
Subject: [PATCH 1026/1071] [LI-HOTFIX] LIKAFKA-24478: Reduce
 UpdateMetadataRequest toString() result (#27) TICKET = LI_DESCRIPTION = In
 large clusters with large metadata size the UpdateMetadataRequest::toString
 can generate really large strings. If there are n/w issue these strings are
 logged resulting in lots of these string to be generated and cause high
 memory usage.

EXIT_CRITERIA = MANUAL ["Occurs on large clusters only hence limiting this to LinkedIn branch only"]
---
 .../kafka/common/requests/UpdateMetadataRequest.java | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
index b8c6089a8c7eb..75e3542a3d4a1 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
@@ -123,13 +123,23 @@ private static Map groupByTopic(List
Date: Tue, 27 Aug 2019 13:49:15 -0700
Subject: [PATCH 1027/1071] [LI-HOTFIX] add prepare-commit-msg hook to help
 format commit message (#38)

TICKET =
LI_DESCRIPTION =
Automatically reformat commit messages via git hook:
  1) extract ticket number and reformat for cherry-picks
  2) provide a standard template for Hotfix patches.

Setup/Install the hook:
  .githooks/setup.sh
Remove the hook:
  .githooks/uninstall.sh

Once installed, the hook will be automatically invoked through the following usages:
  cherry-pick (please use -x):
    git cherry-pick -e  -x

  Hotfix:
    git commit ...
EXIT_CRITERIA = MANUAL [""]
---
 .githooks/prepare-commit-msg | 147 +++++++++++++++++++++++++++++++++++
 .githooks/setup.sh           |  17 ++++
 .githooks/uninstall.sh       |  17 ++++
 3 files changed, 181 insertions(+)
 create mode 100755 .githooks/prepare-commit-msg
 create mode 100755 .githooks/setup.sh
 create mode 100755 .githooks/uninstall.sh

diff --git a/.githooks/prepare-commit-msg b/.githooks/prepare-commit-msg
new file mode 100755
index 0000000000000..a091376b8c0d8
--- /dev/null
+++ b/.githooks/prepare-commit-msg
@@ -0,0 +1,147 @@
+#!/usr/bin/env python
+
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A hook script to prepare the commit log message.
+# Automatically Called by "git commit" (including git cherry-pick).
+# The hook's purpose is to edit the commit
+# message. If the hook fails with a non-zero status,
+# the commit is aborted. Users are allowed to further edit commit message
+# generated by this hook.
+
+# git examples to invoke the hook:
+#
+# Cherry-pick (please use -x):
+# git cherry-pick -e  -x
+#
+# Hotfix:
+# git commit ...
+
+import sys
+import re
+
+# 0: Hotfix 1: cherry-pick
+commit_type = 0
+keywords = ["[LI-HOTFIX]", "[LI-CHERRY-PICK]"]
+
+commit_msg_file = sys.argv[1]
+commit_source = None
+if (len(sys.argv) >= 3):
+	commit_source = sys.argv[2]
+
+SHORT_HASH_LEN = 8
+Original_Description_Key = "ORIGINAL_DESCRIPTION"
+LI_Description_Key = "LI_DESCRIPTION"
+LI_Description_Value = [""]
+Ticket_Key = "TICKET"
+Exit_Criteria_Key = "EXIT_CRITERIA"
+Exit_Criteria_Value = ["MANUAL [\"\"]"]
+Exit_Criteria_Help = ["# EXIT_CRITERIA =  \n", "# e.g., \n", \
+					  "# when the specified hash(s) is presented in the history, this commit is no longer needed:\n# EXIT_CRITERIA = HASH [,  ...] \n", \
+					  "# When the specified tickets are closed and there are patches with these tickets in the title in the commit history, this commit is no longer needed:\n# EXIT_CRITERIA = TICKET [,  ...]\n", \
+					  "# The exit criteria for this commit requires manual investigation:\n# EXIT_CRITERIA = MANUAL []\n"]
+
+def writeKeyValue(fh, key, value, multiline):
+	fh.write(key + " = ")
+	if (multiline):
+		fh.write("\n")
+	for line in value:
+		fh.write(line)
+	fh.write("\n")
+	return
+
+with open (commit_msg_file, "r") as fh:
+	oldMsg = fh.readlines()
+
+firstline = ""
+
+if (len(oldMsg) > 0):
+	firstline = oldMsg[0].lstrip()
+
+# skip already formatted commits
+for word in keywords:
+	if (firstline.startswith(word)):
+		sys.exit(0)
+
+actual_msg = []
+comment_msg = []
+for line in oldMsg[1:]:
+	if (line.startswith("#")):
+		comment_msg.append(line)
+	else:
+		actual_msg.append(line)
+
+for line in oldMsg:
+	cherry_pick_hash = re.search(r"\(cherry picked from commit ([^)]+)\)", line)
+	if (cherry_pick_hash is not None):
+		break
+ticket = ""
+if (cherry_pick_hash is not None):
+	hash = cherry_pick_hash.group(1)
+	print ("hash:" + hash)
+
+	commit_type = 1
+
+	match_ticket = re.search(r"(KAFKA-\d+)", firstline, re.I)
+
+	if (match_ticket is not None):
+		ticket = match_ticket.group(1)
+		print(ticket)
+
+	if (len(hash) > SHORT_HASH_LEN):
+		short_hash = hash[:SHORT_HASH_LEN]
+	else:
+		short_hash = hash
+	title = keywords[commit_type] + " [" + short_hash + "] " + firstline
+	Exit_Criteria_Value = "HASH [" + hash + "]"
+
+else:
+	if not firstline:
+		firstline = "\n"
+	title = keywords[commit_type] + " " + firstline
+
+# populate formatted commit message
+with open(commit_msg_file, 'w') as f:
+	f.write(title)
+	writeKeyValue(f, Ticket_Key, [ticket], False)
+
+	if (commit_type == 0 and len(actual_msg) > 0):
+		LI_Description_Value = actual_msg
+	writeKeyValue(f, LI_Description_Key, LI_Description_Value, True)
+
+	writeKeyValue(f, Exit_Criteria_Key, Exit_Criteria_Value, False)
+
+	if commit_source is None:
+		for line in Exit_Criteria_Help:
+			f.write(line)
+
+	# for cherry-pick, preserve the original commit message
+	if (commit_type == 1):
+		writeKeyValue(f, Original_Description_Key, actual_msg, True)
+
+	for line in comment_msg:
+		f.write(line)
+
+sys.exit(0)
+
+
+
+
+
+
+
+
diff --git a/.githooks/setup.sh b/.githooks/setup.sh
new file mode 100755
index 0000000000000..57cb8432b1805
--- /dev/null
+++ b/.githooks/setup.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cp $(dirname $0)/prepare-commit-msg $(dirname $0)/../.git/hooks/
diff --git a/.githooks/uninstall.sh b/.githooks/uninstall.sh
new file mode 100755
index 0000000000000..d99c67bb674b0
--- /dev/null
+++ b/.githooks/uninstall.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+rm $(dirname $0)/../.git/hooks/prepare-commit-msg

From a4c35c999d3634533c1716a15eb35c28a1541f98 Mon Sep 17 00:00:00 2001
From: Xiongqi Wu 
Date: Wed, 28 Aug 2019 16:18:29 -0700
Subject: [PATCH 1028/1071] [LI-HOTFIX] add back old parseAndValidateAddresses
 API for backward compatibility (#39)

TICKET =
LI_DESCRIPTION =
Some apps such as linkedin-kafka-clients and likafka-cruise-control still reply on this API:
parseAndValidateAddresses(List urls).
This patch adds back the function.

EXIT_CRITERIA = MANUAL ["when no APP makes direct use of this API"]
---
 .../main/java/org/apache/kafka/clients/ClientUtils.java   | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/clients/src/main/java/org/apache/kafka/clients/ClientUtils.java b/clients/src/main/java/org/apache/kafka/clients/ClientUtils.java
index 9b3df8284c4aa..599baeed508dd 100644
--- a/clients/src/main/java/org/apache/kafka/clients/ClientUtils.java
+++ b/clients/src/main/java/org/apache/kafka/clients/ClientUtils.java
@@ -47,6 +47,14 @@ public static List parseAndValidateAddresses(List url
         return parseAndValidateAddresses(urls, ClientDnsLookup.forConfig(clientDnsLookupConfig));
     }
 
+    /**
+     * Kafka does not use this function directly. However,
+     * some third-party applications still rely on this API to parse and validate addresses.
+     */
+    public static List parseAndValidateAddresses(List urls) {
+        return parseAndValidateAddresses(urls, ClientDnsLookup.DEFAULT);
+    }
+
     public static List parseAndValidateAddresses(List urls, ClientDnsLookup clientDnsLookup) {
         List addresses = new ArrayList<>();
         for (String url : urls) {

From bdaf67d2d2fe6b827552f4f66f069eef2717d039 Mon Sep 17 00:00:00 2001
From: Radai Rosenblatt 
Date: Fri, 6 Sep 2019 16:20:24 -0700
Subject: [PATCH 1029/1071] [LI-HOTFIX] embed versioning information in jars
 (#45)

TICKET = LIKAFKA-25730
LI_DESCRIPTION = embed version info into library jars to be able to query at runtime

EXIT_CRITERIA = never

Signed-off-by: Radai Rosenblatt 
---
 build.gradle | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/build.gradle b/build.gradle
index cac618a443e69..31a0377ed05a9 100644
--- a/build.gradle
+++ b/build.gradle
@@ -16,6 +16,8 @@
 import org.ajoberstar.grgit.Grgit
 
 import java.nio.charset.StandardCharsets
+import javax.xml.bind.DatatypeConverter
+import java.security.MessageDigest
 
 buildscript {
   repositories {
@@ -341,6 +343,39 @@ subprojects {
   }
 
   jar {
+
+    doFirst {
+      //version embedding at build time
+      //we create a file called __Versioning__[md5] and set its payload to contain versioning info
+      //
+      //the file name is not fixed, but contains an md5 hash of the project group name and version.
+      //its not fixed so that if multiple libraries are repackaged (think fat/shaded jar) the versioning files
+      //are unlikely to conflict or get overwritten. the md5 hash part is chosen such that builds are
+      //"repeatable" - re-running the same build results in the same output. this is nice for incremental builds.
+      //
+      //the value contains versioning information for both this module and the entire project it was built as part of.
+      //
+      //this information allows us, at runtime, to determine what libraries/versions exist on a given classpath
+
+      MessageDigest md = MessageDigest.getInstance("MD5")
+      md.update(String.valueOf(project.group).getBytes("UTF-8"))
+      md.update(String.valueOf(project.name).getBytes("UTF-8"))
+      md.update(String.valueOf(project.version).getBytes("UTF-8"))
+      byte[] digest = md.digest()
+      String hash = DatatypeConverter.printHexBinary(digest).toUpperCase()
+
+      String versioningFileName = "__Versioning__" + hash
+      String versioningPayload = "{" +
+          "\"format\":\"v1\"," +
+          "\"project\":\"" + rootProject.group + ":" + rootProject.name + ":" + rootProject.version + "\"," +
+          "\"module\":\"" + project.group + ":" + project.name + ":" + project.version + "\"" +
+          "}"
+
+      String outFolder = "${project.buildDir}/resources/main/META-INF/"
+      mkdir outFolder
+      file("${outFolder}/${versioningFileName}").text = versioningPayload
+    }
+
     from "$rootDir/LICENSE"
     from "$rootDir/NOTICE"
   }

From f20401f8dad020ba0345e78f673661fff2ad7764 Mon Sep 17 00:00:00 2001
From: Radai Rosenblatt 
Date: Mon, 9 Sep 2019 17:33:21 -0700
Subject: [PATCH 1030/1071] [LI-HOTFIX] embed versioning information in jars
 (v2) (#46)

TICKET = LIKAFKA-25730
LI_DESCRIPTION = embed version info into library jars to be able to query at runtime

EXIT_CRITERIA = never

Signed-off-by: Radai Rosenblatt 
---
 build.gradle | 45 ++++++++++++++++++++++++++++-----------------
 1 file changed, 28 insertions(+), 17 deletions(-)

diff --git a/build.gradle b/build.gradle
index 31a0377ed05a9..bad8595c59709 100644
--- a/build.gradle
+++ b/build.gradle
@@ -357,23 +357,34 @@ subprojects {
       //
       //this information allows us, at runtime, to determine what libraries/versions exist on a given classpath
 
-      MessageDigest md = MessageDigest.getInstance("MD5")
-      md.update(String.valueOf(project.group).getBytes("UTF-8"))
-      md.update(String.valueOf(project.name).getBytes("UTF-8"))
-      md.update(String.valueOf(project.version).getBytes("UTF-8"))
-      byte[] digest = md.digest()
-      String hash = DatatypeConverter.printHexBinary(digest).toUpperCase()
-
-      String versioningFileName = "__Versioning__" + hash
-      String versioningPayload = "{" +
-          "\"format\":\"v1\"," +
-          "\"project\":\"" + rootProject.group + ":" + rootProject.name + ":" + rootProject.version + "\"," +
-          "\"module\":\"" + project.group + ":" + project.name + ":" + project.version + "\"" +
-          "}"
-
-      String outFolder = "${project.buildDir}/resources/main/META-INF/"
-      mkdir outFolder
-      file("${outFolder}/${versioningFileName}").text = versioningPayload
+      String moduleVersion = String.valueOf(project.version)
+      String projectVersion = String.valueOf(rootProject.version)
+
+      if ("unspecified" != projectVersion && "unspecified" != moduleVersion) {
+        MessageDigest md = MessageDigest.getInstance("MD5")
+        md.update(String.valueOf(project.group).getBytes("UTF-8"))
+        md.update(String.valueOf(project.name).getBytes("UTF-8"))
+        md.update(moduleVersion.getBytes("UTF-8"))
+        byte[] digest = md.digest()
+        String hash = DatatypeConverter.printHexBinary(digest).toUpperCase()
+
+        String versioningFileName = "__Versioning__" + hash
+        String versioningPayload = "{" +
+            "\"format\":\"v1\"," +
+            "\"project\":\"" + rootProject.group + ":" + rootProject.name + ":" + projectVersion + "\"," +
+            "\"module\":\"" + project.group + ":" + project.name + ":" + moduleVersion + "\"" +
+            "}"
+
+        String outFolder = "${project.buildDir}/resources/main/META-INF/"
+        mkdir outFolder
+
+        fileTree(dir: outFolder, include:'__Versioning__*').each {
+          //remove any left-overs from previous builds
+          it.delete()
+        }
+
+        file("${outFolder}/${versioningFileName}").text = versioningPayload
+      }
     }
 
     from "$rootDir/LICENSE"

From 3d23b19add9ea97a99384f4efe39ffea306cb972 Mon Sep 17 00:00:00 2001
From: Alex Wang 
Date: Thu, 21 May 2020 15:52:19 -0700
Subject: [PATCH 1031/1071] [LI-HOTFIX] Enforce minimum replication factor on
 topic creation via CreateTopicPolicy TICKET = LI_DESCRIPTION = On new topic
 creation, the controller will use a custom policy called LiCreateTopicPolicy
 to determine if new topic partition assignment satisfies minimum replication
 factor. If fails the check, will override the user assignment by generating
 new assignment with sufficient RF and write back to ZooKeeper. The same
 policy will also be used by each broker to validate CreateTopicRequest's.

RB=1435237
G=Kafka-Code-Reviews
R=okaraman,jkoshy,zahuang
A=zahuang

EXIT_CRITERIA = MANUAL [""]
---
 .../kafka/controller/KafkaController.scala    | 84 ++++++++++++++++++-
 .../kafka/server/LiCreateTopicPolicy.scala    | 60 +++++++++++++
 .../main/scala/kafka/zk/KafkaZkClient.scala   |  9 ++
 .../ControllerIntegrationTest.scala           | 39 +++++++++
 .../controller/TopicsSatisfyMinRFTest.scala   | 74 ++++++++++++++++
 5 files changed, 263 insertions(+), 3 deletions(-)
 create mode 100644 core/src/main/scala/kafka/server/LiCreateTopicPolicy.scala
 create mode 100644 core/src/test/scala/unit/kafka/controller/TopicsSatisfyMinRFTest.scala

diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala
index 0723b16686d3b..ff5b65cb5f8f6 100644
--- a/core/src/main/scala/kafka/controller/KafkaController.scala
+++ b/core/src/main/scala/kafka/controller/KafkaController.scala
@@ -19,7 +19,7 @@ package kafka.controller
 import java.util.concurrent.TimeUnit
 
 import com.yammer.metrics.core.Gauge
-import kafka.admin.AdminOperationException
+import kafka.admin.{AdminOperationException, AdminUtils}
 import kafka.api._
 import kafka.common._
 import kafka.controller.KafkaController.{AlterReassignmentsCallback, ElectLeadersCallback, ListReassignmentsCallback}
@@ -33,12 +33,14 @@ import org.apache.kafka.common.ElectionType
 import org.apache.kafka.common.KafkaException
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.{BrokerNotAvailableException, ControllerMovedException, StaleBrokerEpochException}
+import org.apache.kafka.common.errors.PolicyViolationException
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.requests.{AbstractControlRequest, AbstractResponse, ApiError, LeaderAndIsrResponse}
 import org.apache.kafka.common.utils.Time
 import org.apache.zookeeper.KeeperException
 import org.apache.zookeeper.KeeperException.Code
+import org.apache.kafka.server.policy.CreateTopicPolicy
 
 import scala.collection.JavaConverters._
 import scala.collection.{Map, Seq, Set, immutable, mutable}
@@ -57,6 +59,36 @@ object KafkaController extends Logging {
   type ElectLeadersCallback = Map[TopicPartition, Either[ApiError, Int]] => Unit
   type ListReassignmentsCallback = Either[Map[TopicPartition, ReplicaAssignment], ApiError] => Unit
   type AlterReassignmentsCallback = Either[Map[TopicPartition, ApiError], ApiError] => Unit
+
+  def satisfiesLiCreateTopicPolicy(createTopicPolicy : Option[CreateTopicPolicy], zkClient : KafkaZkClient,
+    topic : String, partitionsAssignment : collection.Map[Int, ReplicaAssignment]): Boolean = {
+    try {
+      createTopicPolicy match {
+        case Some(policy) =>
+          if (policy.isInstanceOf[LiCreateTopicPolicy]) {
+            import scala.collection.JavaConverters._
+            val jPartitionAssignment = partitionsAssignment.map { case(partition, replicaAssignment) =>
+              (new Integer(partition), seqAsJavaListConverter(replicaAssignment.replicas.map{e => new Integer(e)}).asJava)
+            }
+            // Use min size of all replica lists as a stand in for replicationFactor. Generally replicas sizes should be
+            // the same, but minBy gets us the worst case.
+            val replicationFactor = partitionsAssignment.minBy(_._2.replicas.size)._1.toShort
+            policy.validate(new CreateTopicPolicy.RequestMetadata(topic, partitionsAssignment.size, replicationFactor,
+              jPartitionAssignment.asJava, new java.util.HashMap[String, String]()))
+          }
+          true
+        case None =>
+          true
+      }
+    } catch {
+      case e : PolicyViolationException => {
+        if (zkClient.getTopicPartitions(topic).isEmpty) {
+          info(e.getMessage)
+          false
+        } else true
+      }
+    }
+  }
 }
 
 class KafkaController(val config: KafkaConfig,
@@ -117,6 +149,9 @@ class KafkaController(val config: KafkaConfig,
   @volatile private var ineligibleTopicsToDeleteCount = 0
   @volatile private var ineligibleReplicasToDeleteCount = 0
 
+  private val createTopicPolicy =
+    Option(config.getConfiguredInstance(KafkaConfig.CreateTopicPolicyClassNameProp, classOf[CreateTopicPolicy]))
+
   /* single-thread scheduler to clean expired tokens */
   private val tokenCleanScheduler = new KafkaScheduler(threads = 1, threadNamePrefix = "delegation-token-cleaner")
 
@@ -784,7 +819,7 @@ class KafkaController(val config: KafkaConfig,
     info(s"Initialized broker epochs cache: ${controllerContext.liveBrokerIdAndEpochs}")
     controllerContext.allTopics = zkClient.getAllTopicsInCluster
     registerPartitionModificationsHandlers(controllerContext.allTopics.toSeq)
-    zkClient.getFullReplicaAssignmentForTopics(controllerContext.allTopics.toSet).foreach {
+    getReplicaAssignmentPolicyCompliant(controllerContext.allTopics.toSet).foreach {
       case (topicPartition, replicaAssignment) =>
         controllerContext.updatePartitionFullReplicaAssignment(topicPartition, replicaAssignment)
         if (replicaAssignment.isBeingReassigned)
@@ -1451,7 +1486,7 @@ class KafkaController(val config: KafkaConfig,
     controllerContext.allTopics = topics
 
     registerPartitionModificationsHandlers(newTopics.toSeq)
-    val addedPartitionReplicaAssignment = zkClient.getFullReplicaAssignmentForTopics(newTopics)
+    val addedPartitionReplicaAssignment = getReplicaAssignmentPolicyCompliant(newTopics)
     deletedTopics.foreach(controllerContext.removeTopic)
     addedPartitionReplicaAssignment.foreach {
       case (topicAndPartition, newReplicaAssignment) => controllerContext.updatePartitionFullReplicaAssignment(topicAndPartition, newReplicaAssignment)
@@ -1839,6 +1874,49 @@ class KafkaController(val config: KafkaConfig,
     onControllerResignation()
   }
 
+  // For any topics that fail to meet min RF requirement, generate new valid partition assignment and reset ZNode
+  private def fixTopicsFailingPolicy(topicsReplicaAssignment : Map[String, Map[Int, ReplicaAssignment]]) : Unit = {
+    if (topicsReplicaAssignment.isEmpty) return
+
+    val replicationFactor = config.defaultReplicationFactor
+    val brokers = controllerContext.liveOrShuttingDownBrokers.map { sb => kafka.admin.BrokerMetadata(sb.id, sb.rack) }.toSeq
+
+    topicsReplicaAssignment.foreach{
+      case(topic, partitionAssignment) => {
+        val numPartitions = partitionAssignment.size
+        val assignment = AdminUtils.assignReplicasToBrokers(brokers, numPartitions, replicationFactor)
+          .map{ case(partition, replicas) => (new TopicPartition(topic, partition), new ReplicaAssignment(replicas, Seq.empty[Int], Seq.empty[Int]))
+          }.toMap
+        zkClient.setTopicAssignment(topic, assignment, controllerContext.epochZkVersion)
+        info(s"Updated topic [$topic] with $assignment for replica assignment")
+      }
+    }
+  }
+
+  // Reset partition replica assignment for topics, if any, that fail replication factor check
+  private def getReplicaAssignmentPolicyCompliant(topics : immutable.Set[String]) : Map[TopicPartition, ReplicaAssignment] = {
+    val replicaAssignments = zkClient.getPartitionAssignmentForTopics(topics)
+    val (topicAssignmentSucceedingPolicy, topicAssignmentFailingPolicy) = replicaAssignments.partition(
+      assignment => KafkaController.satisfiesLiCreateTopicPolicy(createTopicPolicy, zkClient, assignment._1, assignment._2))
+
+    var retTopicAssignment = topicAssignmentSucceedingPolicy
+    val topicsFailingPolicy = topicAssignmentFailingPolicy.keySet
+    if (!topicAssignmentFailingPolicy.isEmpty) {
+      // Since fixTopicsFailingPolicy() will trigger PartitionModification event, need to temporarily unregister
+      // event handler
+      unregisterPartitionModificationsHandlers(topicsFailingPolicy.toSeq)
+      fixTopicsFailingPolicy(topicAssignmentFailingPolicy)
+      registerPartitionModificationsHandlers(topicsFailingPolicy.toSeq)
+
+      // the new partition assignments should be valid now
+      retTopicAssignment ++= zkClient.getPartitionAssignmentForTopics(topicsFailingPolicy.toSet)
+    }
+    retTopicAssignment.flatMap{ case(topic, partitionAssignment) =>
+      partitionAssignment.map{
+        case(partition, replicas)=> (new TopicPartition(topic, partition), replicas)
+      }
+    }
+  }
 
   override def process(event: ControllerEvent): Unit = {
     try {
diff --git a/core/src/main/scala/kafka/server/LiCreateTopicPolicy.scala b/core/src/main/scala/kafka/server/LiCreateTopicPolicy.scala
new file mode 100644
index 0000000000000..2553e9471931b
--- /dev/null
+++ b/core/src/main/scala/kafka/server/LiCreateTopicPolicy.scala
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package kafka.server
+
+import org.apache.kafka.common.errors.PolicyViolationException
+import org.apache.kafka.server.policy.CreateTopicPolicy
+
+class LiCreateTopicPolicy extends CreateTopicPolicy {
+  var minRf = 0
+
+  @throws[PolicyViolationException]
+  override def validate(requestMetadata: CreateTopicPolicy.RequestMetadata): Unit = {
+    val requestTopic = requestMetadata.topic
+    val requestRF = requestMetadata.replicationFactor()
+    import collection.JavaConverters._
+    val requestAssignment = requestMetadata.replicasAssignments().asScala
+
+    if (requestAssignment == null && requestRF == null)
+      throw new PolicyViolationException(s"Topic [$requestTopic] is missing both replica assignment and " +
+        s"replication factor.")
+
+    // In createTopics() in AdminManager, replicationFactor and replicasAssignments are not both set at same time. We
+    // follow the same rationale here and prioritize replicasAssignments over replicationFactor
+    if (requestAssignment != null) {
+      requestAssignment.foreach { case (p, assignment) =>
+          if (assignment.size() < minRf)
+            throw new PolicyViolationException(s"Topic [$requestTopic] fails RF requirement. Received RF for " +
+              s"[partition-$p]: ${assignment.size()}, min required RF: $minRf.")
+      }
+    } else if (requestRF < minRf) {
+      throw new PolicyViolationException(s"Topic [$requestTopic] fails RF requirement. " +
+        s"Received RF: ${requestMetadata.replicationFactor}, min required RF: $minRf.")
+    }
+  }
+
+  /**
+    * Configure this class with the given key-value pairs
+    */
+  override def configure(configs: java.util.Map[String, _]): Unit = {
+    minRf = configs.get(KafkaConfig.DefaultReplicationFactorProp).asInstanceOf[String].toInt
+  }
+
+  @throws[Exception]
+  override def close(): Unit = {
+  }
+}
diff --git a/core/src/main/scala/kafka/zk/KafkaZkClient.scala b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
index fb80978f603e9..8f0f1851eddfc 100644
--- a/core/src/main/scala/kafka/zk/KafkaZkClient.scala
+++ b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
@@ -673,6 +673,15 @@ class KafkaZkClient private[zk] (zooKeeperClient: ZooKeeperClient, isSecure: Boo
     }
   }
 
+  /**
+    * Gets all partitions in for a topic
+    * @param topic topic name
+    * @return all partitions for the topic
+    */
+  def getTopicPartitions(topic: String) : Seq[String] = {
+    getChildren(TopicPartitionsZNode.path(topic))
+  }
+
   /**
    * Gets the data and version at the given zk path
    * @param path zk node path
diff --git a/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala b/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala
index 37eae0e3002f6..78764f62e32b7 100644
--- a/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala
@@ -243,6 +243,33 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
       "failed to get expected partition state upon topic creation")
   }
 
+  /*
+  * Tests that controller will fix insufficient RF topic by assigning sufficient replicas
+  * */
+  @Test
+  def testTopicCreationWithFixingRF(): Unit = {
+    val topicRF1 = "test_topic_rf1"
+    val partition = 0
+    val defaultRF = 2
+    val serverConfigs = TestUtils.createBrokerConfigs(3, zkConnect, false)
+      .map(prop => {
+        prop.setProperty(KafkaConfig.DefaultReplicationFactorProp,defaultRF.toString)
+        prop.setProperty(KafkaConfig.CreateTopicPolicyClassNameProp, "kafka.server.LiCreateTopicPolicy")
+        prop
+      }).map(KafkaConfig.fromProps)
+    servers = serverConfigs.map(s => TestUtils.createServer(s))
+
+    val controllerId = TestUtils.waitUntilControllerElected(zkClient)
+    val assignment = Map(partition -> List(controllerId)) // topicRF1 original RF set to 1
+    TestUtils.createTopic(zkClient, topicRF1, partitionReplicaAssignment = assignment, servers = servers, new Properties())
+
+    waitForPartitionState(new TopicPartition(topicRF1, partition), firstControllerEpoch, zkClient.getAllBrokersInCluster.map(b => b.id),
+      LeaderAndIsr.initialLeaderEpoch, "failed to get expected partition state upon valid RF topic creation")
+
+    // Actual RF should be corrected to 2
+    assertEquals(defaultRF, zkClient.getReplicasForPartition(new TopicPartition(topicRF1, partition)).size)
+  }
+
   @Test
   def testTopicPartitionExpansion(): Unit = {
     servers = makeServers(1)
@@ -688,6 +715,18 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     }, message)
   }
 
+  private def waitForPartitionState(tp: TopicPartition,
+    controllerEpoch: Int,
+    leaders: Seq[Int],
+    leaderEpoch: Int,
+    message: String): Unit = {
+    TestUtils.waitUntilTrue(() => {
+      val leaderIsrAndControllerEpochMap = zkClient.getTopicPartitionStates(Seq(tp))
+      leaderIsrAndControllerEpochMap.contains(tp) && leaders.exists(
+        leader=> isExpectedPartitionState(leaderIsrAndControllerEpochMap(tp), controllerEpoch, leader, leaderEpoch))
+    }, message)
+  }
+
   private def isExpectedPartitionState(leaderIsrAndControllerEpoch: LeaderIsrAndControllerEpoch,
                                        controllerEpoch: Int,
                                        leader: Int,
diff --git a/core/src/test/scala/unit/kafka/controller/TopicsSatisfyMinRFTest.scala b/core/src/test/scala/unit/kafka/controller/TopicsSatisfyMinRFTest.scala
new file mode 100644
index 0000000000000..2de930612bfba
--- /dev/null
+++ b/core/src/test/scala/unit/kafka/controller/TopicsSatisfyMinRFTest.scala
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package unit.kafka.controller
+
+import kafka.controller.{KafkaController, ReplicaAssignment}
+import kafka.server.KafkaConfig
+import kafka.utils.TestUtils
+import kafka.zk.KafkaZkClient
+import org.apache.kafka.server.policy.CreateTopicPolicy
+import org.easymock.EasyMock
+import org.junit.{Before, Test}
+import org.junit.Assert.assertEquals
+import org.scalatest.junit.JUnitSuite
+
+class TopicsSatisfyMinRFTest extends JUnitSuite {
+  private var createTopicPolicy : Option[CreateTopicPolicy] = null
+  private var mockZkClient: KafkaZkClient = null
+  @Before
+  def setUp(): Unit = {
+    val config = TestUtils.createBrokerConfigs(1, "")
+      .map(prop => {
+        prop.setProperty(KafkaConfig.DefaultReplicationFactorProp,"2")  // Set min RF to be 2
+        prop.setProperty(KafkaConfig.CreateTopicPolicyClassNameProp, "kafka.server.LiCreateTopicPolicy")
+        prop
+      }).map(KafkaConfig.fromProps)
+    createTopicPolicy = Option(config.head.getConfiguredInstance(KafkaConfig.CreateTopicPolicyClassNameProp,
+      classOf[CreateTopicPolicy]))
+    mockZkClient = EasyMock.createMock(classOf[KafkaZkClient])
+  }
+
+  // Mock new topic with sufficient RF, should satisfy
+  @Test
+  def testNewTopicSucceed: Unit = {
+    val topicSucceed = "test_topic1"
+    assertEquals(true, KafkaController.satisfiesLiCreateTopicPolicy(createTopicPolicy, mockZkClient,
+      topicSucceed, Map(0 -> new ReplicaAssignment(Seq(0, 1), Seq.empty[Int], Seq.empty[Int]))))      // RF = 2
+  }
+
+  // Mock new topic with insufficient RF, should fail
+  @Test
+  def testNewTopicFail: Unit = {
+    val topicFail = "test_topic1"
+    EasyMock.expect(mockZkClient.getTopicPartitions(topicFail)).andReturn(Seq.empty)
+    EasyMock.replay(mockZkClient)
+    assertEquals(false, KafkaController.satisfiesLiCreateTopicPolicy(createTopicPolicy, mockZkClient,
+      topicFail, Map(0 -> new ReplicaAssignment(Seq(1), Seq.empty[Int], Seq.empty[Int]))))            // RF = 1
+    EasyMock.verify(mockZkClient)
+  }
+
+  // Mock existing topic with insufficient RF, because it's pre-existing, should still satisfy
+  @Test
+  def testExistingTopicSucceed: Unit = {
+    val topicExisting = "test_topic1"
+    EasyMock.expect(mockZkClient.getTopicPartitions(topicExisting)).andReturn(Seq("0"))  // Partition number = 1
+    EasyMock.replay(mockZkClient)
+    assertEquals(true, KafkaController.satisfiesLiCreateTopicPolicy(createTopicPolicy, mockZkClient,
+      topicExisting, Map(0 -> new ReplicaAssignment(Seq(1), Seq.empty[Int], Seq.empty[Int]))))        // RF = 1
+    EasyMock.verify(mockZkClient)
+  }
+}

From 8880b9407dd837f449ed91df05e10a8c6391771a Mon Sep 17 00:00:00 2001
From: Sean McCauliff 
Date: Wed, 2 Oct 2019 16:45:42 -0700
Subject: [PATCH 1032/1071] [LI-HOTFIX] Log the partition being made online due
 to unclean leader election. (#52)

[LI-HOTFIX] Log the partition being made online due to unclean leader election.
TICKET = KAFKA-8969
LI_DESCRIPTION = Log unclean leader election at  the warn level.  Add additional unit tests for unclean leader election.

EXIT_CRITERIA = TICKET [KAFKA-8969]
---
 .../scala/kafka/controller/Election.scala     | 12 ++++++--
 .../controller/PartitionStateMachine.scala    | 26 +++++++++-------
 ...artitionLeaderElectionAlgorithmsTest.scala | 22 ++++----------
 .../PartitionStateMachineTest.scala           | 30 +++++++++++++++----
 4 files changed, 55 insertions(+), 35 deletions(-)

diff --git a/core/src/main/scala/kafka/controller/Election.scala b/core/src/main/scala/kafka/controller/Election.scala
index dffa88841aac3..5a59143205bfb 100644
--- a/core/src/main/scala/kafka/controller/Election.scala
+++ b/core/src/main/scala/kafka/controller/Election.scala
@@ -17,13 +17,14 @@
 package kafka.controller
 
 import kafka.api.LeaderAndIsr
+import kafka.utils.Logging
 import org.apache.kafka.common.TopicPartition
 
 import scala.collection.Seq
 
 case class ElectionResult(topicPartition: TopicPartition, leaderAndIsr: Option[LeaderAndIsr], liveReplicas: Seq[Int])
 
-object Election {
+object Election extends Logging {
 
   private def leaderForOffline(partition: TopicPartition,
                                leaderAndIsrOpt: Option[LeaderAndIsr],
@@ -36,8 +37,13 @@ object Election {
       case Some(leaderAndIsr) =>
         val isr = leaderAndIsr.isr
         val leaderOpt = PartitionLeaderElectionAlgorithms.offlinePartitionLeaderElection(
-          assignment, isr, liveReplicas.toSet, uncleanLeaderElectionEnabled, controllerContext)
-        val newLeaderAndIsrOpt = leaderOpt.map { leader =>
+          assignment, isr, liveReplicas.toSet, uncleanLeaderElectionEnabled)
+        val newLeaderAndIsrOpt = leaderOpt.map { case (leader, uncleanElection) =>
+          if (uncleanElection) {
+            controllerContext.stats.uncleanLeaderElectionRate.mark()
+            warn(s"Unclean leader election. Partition $partition has been assigned leader $leader from deposed " +
+              s"leader ${leaderAndIsr.leader}.")
+          }
           val newIsr = if (isr.contains(leader)) isr.filter(replica => controllerContext.isReplicaOnline(replica, partition))
           else List(leader)
           leaderAndIsr.newLeaderAndIsr(leader, newIsr)
diff --git a/core/src/main/scala/kafka/controller/PartitionStateMachine.scala b/core/src/main/scala/kafka/controller/PartitionStateMachine.scala
index c2dc835fdc0e5..b1ecedfcd0684 100755
--- a/core/src/main/scala/kafka/controller/PartitionStateMachine.scala
+++ b/core/src/main/scala/kafka/controller/PartitionStateMachine.scala
@@ -516,16 +516,22 @@ class ZkPartitionStateMachine(config: KafkaConfig,
 }
 
 object PartitionLeaderElectionAlgorithms {
-  def offlinePartitionLeaderElection(assignment: Seq[Int], isr: Seq[Int], liveReplicas: Set[Int], uncleanLeaderElectionEnabled: Boolean, controllerContext: ControllerContext): Option[Int] = {
-    assignment.find(id => liveReplicas.contains(id) && isr.contains(id)).orElse {
-      if (uncleanLeaderElectionEnabled) {
-        val leaderOpt = assignment.find(liveReplicas.contains)
-        if (leaderOpt.isDefined)
-          controllerContext.stats.uncleanLeaderElectionRate.mark()
-        leaderOpt
-      } else {
-        None
-      }
+
+  /**
+   * @return Optionally, a tuple (replica, flag) where flag is a boolean indicating if unclean leader election was
+   *         used to replace the replica.
+   */
+  def offlinePartitionLeaderElection(assignment: Seq[Int], isr: Seq[Int], liveReplicas: Set[Int], uncleanLeaderElectionEnabled: Boolean): Option[(Int, Boolean)] = {
+    assignment.find(id => liveReplicas.contains(id) && isr.contains(id)) match {
+      case Some(replicaId) => Some(replicaId, false)
+      case None =>  if (uncleanLeaderElectionEnabled) {
+                     assignment.find(liveReplicas.contains) match {
+                        case Some(uncleanReplicaId) => Some(uncleanReplicaId, true)
+                        case None => None
+                      }
+                  } else {
+                    None
+                  }
     }
   }
 
diff --git a/core/src/test/scala/unit/kafka/controller/PartitionLeaderElectionAlgorithmsTest.scala b/core/src/test/scala/unit/kafka/controller/PartitionLeaderElectionAlgorithmsTest.scala
index 3fd419257c3bb..41bc8833ac88a 100644
--- a/core/src/test/scala/unit/kafka/controller/PartitionLeaderElectionAlgorithmsTest.scala
+++ b/core/src/test/scala/unit/kafka/controller/PartitionLeaderElectionAlgorithmsTest.scala
@@ -20,13 +20,6 @@ import org.junit.Assert._
 import org.junit.{Before, Test}
 
 class PartitionLeaderElectionAlgorithmsTest {
-  private var controllerContext: ControllerContext = null
-
-  @Before
-  def setUp(): Unit = {
-    controllerContext = new ControllerContext
-    controllerContext.stats.removeMetric("UncleanLeaderElectionsPerSec")
-  }
 
   @Test
   def testOfflinePartitionLeaderElection(): Unit = {
@@ -36,9 +29,8 @@ class PartitionLeaderElectionAlgorithmsTest {
     val leaderOpt = PartitionLeaderElectionAlgorithms.offlinePartitionLeaderElection(assignment,
       isr,
       liveReplicas,
-      uncleanLeaderElectionEnabled = false,
-      controllerContext)
-    assertEquals(Option(4), leaderOpt)
+      uncleanLeaderElectionEnabled = false)
+    assertEquals(Option(4, false), leaderOpt)
   }
 
   @Test
@@ -49,10 +41,8 @@ class PartitionLeaderElectionAlgorithmsTest {
     val leaderOpt = PartitionLeaderElectionAlgorithms.offlinePartitionLeaderElection(assignment,
       isr,
       liveReplicas,
-      uncleanLeaderElectionEnabled = false,
-      controllerContext)
+      uncleanLeaderElectionEnabled = false)
     assertEquals(None, leaderOpt)
-    assertEquals(0, controllerContext.stats.uncleanLeaderElectionRate.count())
   }
 
   @Test
@@ -63,10 +53,8 @@ class PartitionLeaderElectionAlgorithmsTest {
     val leaderOpt = PartitionLeaderElectionAlgorithms.offlinePartitionLeaderElection(assignment,
       isr,
       liveReplicas,
-      uncleanLeaderElectionEnabled = true,
-      controllerContext)
-    assertEquals(Option(4), leaderOpt)
-    assertEquals(1, controllerContext.stats.uncleanLeaderElectionRate.count())
+      uncleanLeaderElectionEnabled = true)
+    assertEquals(Option(4, true), leaderOpt)
   }
 
   @Test
diff --git a/core/src/test/scala/unit/kafka/controller/PartitionStateMachineTest.scala b/core/src/test/scala/unit/kafka/controller/PartitionStateMachineTest.scala
index d00e1bbdb1f65..ef496a146abdd 100644
--- a/core/src/test/scala/unit/kafka/controller/PartitionStateMachineTest.scala
+++ b/core/src/test/scala/unit/kafka/controller/PartitionStateMachineTest.scala
@@ -30,6 +30,7 @@ import org.easymock.EasyMock
 import org.junit.Assert._
 import org.junit.{Before, Test}
 import org.mockito.Mockito
+import scala.collection.JavaConverters._
 
 class PartitionStateMachineTest {
   private var controllerContext: ControllerContext = null
@@ -241,12 +242,21 @@ class PartitionStateMachineTest {
     assertEquals(OnlinePartition, partitionState(partition))
   }
 
+  def testOfflinePartitionToOnlinePartitionTransitionInIsr(): Unit = {
+    testOfflinePartitionToOnlinePartitionTransition(testUncleanLeaderElection = false)
+  }
+
   @Test
-  def testOfflinePartitionToOnlinePartitionTransition(): Unit = {
+  def testOfflinePartitionToOnlinePartitionTransitionUncleanLeaderElection(): Unit = {
+    testOfflinePartitionToOnlinePartitionTransition(testUncleanLeaderElection = true)
+  }
+
+  private[this] def testOfflinePartitionToOnlinePartitionTransition(testUncleanLeaderElection : Boolean) : Unit = {
     controllerContext.setLiveBrokerAndEpochs(Map(TestUtils.createBrokerAndEpoch(brokerId, "host", 0)))
     controllerContext.updatePartitionReplicaAssignment(partition, Seq(brokerId))
     controllerContext.putPartitionState(partition, OfflinePartition)
-    val leaderAndIsr = LeaderAndIsr(LeaderAndIsr.NoLeader, List(brokerId))
+    val leaderAndIsr = LeaderAndIsr(LeaderAndIsr.NoLeader, if (testUncleanLeaderElection) List.empty else List(brokerId))
+
     val leaderIsrAndControllerEpoch = LeaderIsrAndControllerEpoch(leaderAndIsr, controllerEpoch)
     controllerContext.partitionLeadershipInfo.put(partition, leaderIsrAndControllerEpoch)
 
@@ -256,9 +266,16 @@ class PartitionStateMachineTest {
       .andReturn(Seq(GetDataResponse(Code.OK, null, Some(partition),
         TopicPartitionStateZNode.encode(leaderIsrAndControllerEpoch), stat, ResponseMetadata(0, 0))))
 
-    EasyMock.expect(mockZkClient.getLogConfigs(Set.empty, config.originals()))
-      .andReturn((Map(partition.topic -> LogConfig()), Map.empty))
-    val leaderAndIsrAfterElection = leaderAndIsr.newLeader(brokerId)
+    if (testUncleanLeaderElection) {
+      val uleLogConfig = LogConfig(Map(LogConfig.UncleanLeaderElectionEnableProp -> "true").asJava)
+      EasyMock.expect(mockZkClient.getLogConfigs(Set("t"), config.originals()))
+        .andReturn((Map(partition.topic -> uleLogConfig), Map.empty))
+    } else {
+      EasyMock.expect(mockZkClient.getLogConfigs(Set.empty, config.originals()))
+        .andReturn((Map(partition.topic -> LogConfig()), Map.empty))
+    }
+    val leaderAndIsrAfterElection = leaderAndIsr.newLeaderAndIsr(brokerId, List(brokerId))
+
     val updatedLeaderAndIsr = leaderAndIsrAfterElection.withZkVersion(2)
     EasyMock.expect(mockZkClient.updateLeaderAndIsr(Map(partition -> leaderAndIsrAfterElection), controllerEpoch, controllerContext.epochZkVersion))
       .andReturn(UpdateLeaderAndIsrResult(Map(partition -> Right(updatedLeaderAndIsr)), Seq.empty))
@@ -267,6 +284,7 @@ class PartitionStateMachineTest {
     EasyMock.expect(mockControllerBrokerRequestBatch.sendRequestsToBrokers(controllerEpoch))
     EasyMock.replay(mockZkClient, mockControllerBrokerRequestBatch)
 
+    val numUnCleanLeaderElectionsBefore = controllerContext.stats.uncleanLeaderElectionRate.count()
     partitionStateMachine.handleStateChanges(
       partitions,
       OnlinePartition,
@@ -274,6 +292,8 @@ class PartitionStateMachineTest {
     )
     EasyMock.verify(mockZkClient, mockControllerBrokerRequestBatch)
     assertEquals(OnlinePartition, partitionState(partition))
+    if (testUncleanLeaderElection)
+      assertEquals(1, controllerContext.stats.uncleanLeaderElectionRate.count() - numUnCleanLeaderElectionsBefore)
   }
 
   @Test

From 9ae16ef6c47d828f1428496b445c75046ef95442 Mon Sep 17 00:00:00 2001
From: Virupaksha Swamy Uttanur Matada 
Date: Mon, 30 Sep 2019 12:24:31 -0700
Subject: [PATCH 1033/1071] [LI-HOTFIX] Implementation of memory pool based on
 weak references. (#49) LI_DESCRIPTION = * Implementation of memory pool based
 on weak references.

This will act as a NO-OP memory pool (current behavior) in all the clients and kafka broker except
when there is an opportunity to reuse buffer before it could have been garbage collected.
EXIT_CRITERIA =

Note: It is possible that if we allocate buffers in a strictly increasing order, we may accumulate certain keys over time with their weak references set to null. The idea is that allocation distribution is usually uniform and eventually an allocation for a smaller byte buffer would kick out these keys from the cache.
---
 .../kafka/common/memory/WeakMemoryPool.java   | 138 ++++++++++++++++++
 .../common/memory/WeakMemoryPoolTest.java     | 131 +++++++++++++++++
 2 files changed, 269 insertions(+)
 create mode 100644 clients/src/main/java/org/apache/kafka/common/memory/WeakMemoryPool.java
 create mode 100644 clients/src/test/java/org/apache/kafka/common/memory/WeakMemoryPoolTest.java

diff --git a/clients/src/main/java/org/apache/kafka/common/memory/WeakMemoryPool.java b/clients/src/main/java/org/apache/kafka/common/memory/WeakMemoryPool.java
new file mode 100644
index 0000000000000..48fb1b2194dd0
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/memory/WeakMemoryPool.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.common.memory;
+
+import java.lang.ref.WeakReference;
+import java.nio.ByteBuffer;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+
+
+/**
+ * An implementation of memory pool with weak references.
+ *
+ * Note: This will be a no-op for all the other clients.
+ *
+ * If and when the GC runs, the buffers will be reclaimed and no side effects will be seen.
+ */
+public class WeakMemoryPool implements MemoryPool {
+    private final NavigableMap>> cache;
+    private final Object lock = new Object();
+
+    public WeakMemoryPool() {
+        cache = new TreeMap<>();
+    }
+
+    /**
+     * Tries to acquire a ByteBuffer of the specified size
+     * @param sizeBytes size required
+     * @return a ByteBuffer (which later needs to be release()ed), or null if no memory available.
+     *         the buffer will be of the exact size requested, even if backed by a larger chunk of memory
+     */
+    @Override
+    public ByteBuffer tryAllocate(int sizeBytes) {
+        if (sizeBytes < 0) {
+            throw new IllegalArgumentException("The buffer size cannot be less than 0");
+        }
+        ByteBuffer buffer = null;
+        synchronized (lock) {
+            NavigableMap>> tailMap = cache.tailMap(sizeBytes, true);
+            LinkedList keysToDelete = new LinkedList<>();
+            for (Map.Entry>> entry : tailMap.entrySet()) {
+
+                LinkedList> queue = entry.getValue();
+                while (!queue.isEmpty() && buffer == null) {
+                    buffer = queue.pop().get();
+                }
+
+                if (queue.isEmpty()) {
+                    keysToDelete.add(entry.getKey());
+                }
+
+                // If buffer is non-null; break out of the loop!
+                if (buffer != null) {
+                    break;
+                }
+            }
+
+            // Remove all keys from the map which have zero queue lengths
+            for (Integer key : keysToDelete) {
+                cache.remove(key);
+            }
+        }
+
+        // If buffer is non-null, clear it before returning back to the user!
+        if (buffer != null) {
+            buffer.clear();
+            return buffer;
+        }
+
+        return ByteBuffer.allocate(sizeBytes);
+    }
+
+    /**
+     * Returns a previously allocated buffer to the pool.
+     * @param previouslyAllocated a buffer previously returned from tryAllocate()
+     */
+    @Override
+    public void release(ByteBuffer previouslyAllocated) {
+        if (previouslyAllocated == null) {
+            throw new IllegalArgumentException("the buffer to be released cannot be null!");
+        }
+
+        synchronized (lock) {
+            LinkedList> queue =
+                cache.computeIfAbsent(previouslyAllocated.capacity(), k -> new LinkedList<>());
+            queue.add(new WeakReference<>(previouslyAllocated));
+        }
+    }
+
+    /**
+     * Returns the total size of this pool
+     * @return total size, in bytes
+     */
+    @Override
+    public long size() {
+        return Long.MAX_VALUE;
+    }
+
+    /**
+     * Returns the amount of memory available for allocation by this pool.
+     * NOTE: result may be negative (pools may over allocate to avoid starvation issues)
+     * @return bytes available
+     */
+    @Override
+    public long availableMemory() {
+        return Long.MAX_VALUE;
+    }
+
+    /**
+     * Returns true if the pool cannot currently allocate any more buffers
+     * - meaning total outstanding buffers meets or exceeds pool size and
+     * some would need to be released before further allocations are possible.
+     *
+     * This is equivalent to availableMemory() <= 0
+     * @return true if out of memory
+     */
+    @Override
+    public boolean isOutOfMemory() {
+        return false;
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/common/memory/WeakMemoryPoolTest.java b/clients/src/test/java/org/apache/kafka/common/memory/WeakMemoryPoolTest.java
new file mode 100644
index 0000000000000..57be288e98c7c
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/memory/WeakMemoryPoolTest.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.common.memory;
+
+import java.nio.ByteBuffer;
+import org.junit.Assert;
+import org.junit.Test;
+
+
+public class WeakMemoryPoolTest {
+    private static final int FORTY_MEGABYTES = 40 * 1024 * 1024;
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testNegativeAllocation() {
+        WeakMemoryPool memoryPool = new WeakMemoryPool();
+        memoryPool.tryAllocate(-1);
+    }
+
+    @Test
+    public void testZeroAllocation() {
+        WeakMemoryPool memoryPool = new WeakMemoryPool();
+        memoryPool.tryAllocate(0);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testNullRelease() {
+        WeakMemoryPool memoryPool = new WeakMemoryPool();
+        memoryPool.release(null);
+    }
+
+    @Test
+    public void testAllocationMemorySize() {
+        WeakMemoryPool pool = new WeakMemoryPool();
+        long freeMemory = Runtime.getRuntime().freeMemory();
+        ByteBuffer buffer1 = pool.tryAllocate(FORTY_MEGABYTES + 1);
+        ByteBuffer buffer2 = pool.tryAllocate(FORTY_MEGABYTES + 2);
+        ByteBuffer buffer3 = pool.tryAllocate(FORTY_MEGABYTES + 3);
+        Assert.assertTrue(Runtime.getRuntime().freeMemory()
+            <= freeMemory - buffer1.capacity() - buffer2.capacity() - buffer3.capacity());
+
+        pool.release(buffer1);
+        ByteBuffer reuse1 = pool.tryAllocate(FORTY_MEGABYTES);
+        // Compare the references
+        Assert.assertTrue(reuse1 == buffer1);
+
+        pool.release(buffer2);
+        pool.release(buffer3);
+        ByteBuffer reuse3 = pool.tryAllocate(FORTY_MEGABYTES + 3);
+        ByteBuffer reuse2 = pool.tryAllocate(FORTY_MEGABYTES + 2);
+
+        Assert.assertTrue(reuse3 == buffer3);
+        Assert.assertTrue(reuse2 == buffer2);
+    }
+
+    @Test
+    public void testAllocation() {
+        WeakMemoryPool pool = new WeakMemoryPool();
+        ByteBuffer buffer1 = pool.tryAllocate(FORTY_MEGABYTES + 1);
+        ByteBuffer buffer2 = pool.tryAllocate(FORTY_MEGABYTES + 2);
+        ByteBuffer buffer3 = pool.tryAllocate(FORTY_MEGABYTES + 3);
+
+        pool.release(buffer1);
+        ByteBuffer reuse1 = pool.tryAllocate(FORTY_MEGABYTES);
+        // Compare the references
+        Assert.assertEquals(System.identityHashCode(reuse1), System.identityHashCode(buffer1));
+
+        pool.release(buffer2);
+        pool.release(buffer3);
+        ByteBuffer reuse3 = pool.tryAllocate(FORTY_MEGABYTES + 3);
+        ByteBuffer reuse2 = pool.tryAllocate(FORTY_MEGABYTES + 2);
+
+        Assert.assertEquals(System.identityHashCode(reuse3), System.identityHashCode(buffer3));
+        Assert.assertEquals(System.identityHashCode(reuse2), System.identityHashCode(buffer2));
+    }
+
+    @Test
+    public void testAllocationGC() {
+        // Clean all garbage before we begin!
+        System.gc();
+
+        WeakMemoryPool pool = new WeakMemoryPool();
+
+        ByteBuffer buffer1 = ByteBuffer.allocate(FORTY_MEGABYTES + 1);
+        ByteBuffer buffer2 = ByteBuffer.allocate(FORTY_MEGABYTES + 5);
+        ByteBuffer buffer3 = ByteBuffer.allocate(FORTY_MEGABYTES + 9);
+
+        // The byte buffers are not reachable from gc-roots
+        int identifier1 = System.identityHashCode(buffer1);
+        int identifier2 = System.identityHashCode(buffer2);
+        int identifier3 = System.identityHashCode(buffer3);
+
+        pool.release(buffer1);
+        pool.release(buffer2);
+        pool.release(buffer3);
+
+        Assert.assertEquals(identifier2, System.identityHashCode(pool.tryAllocate(FORTY_MEGABYTES + 3)));
+        Assert.assertEquals(identifier3, System.identityHashCode(pool.tryAllocate(FORTY_MEGABYTES + 7)));
+        Assert.assertEquals(identifier1, System.identityHashCode(pool.tryAllocate(FORTY_MEGABYTES + 0)));
+
+        pool.release(buffer1);
+        pool.release(buffer2);
+        pool.release(buffer3);
+
+        buffer1 = null;
+        buffer2 = null;
+        buffer3 = null;
+
+        // Reclaim all the objects
+        System.gc();
+
+        // Assert that the object is a different one!
+        Assert.assertNotEquals(identifier2, System.identityHashCode(pool.tryAllocate(FORTY_MEGABYTES + 3)));
+        Assert.assertNotEquals(identifier3, System.identityHashCode(pool.tryAllocate(FORTY_MEGABYTES + 7)));
+        Assert.assertNotEquals(identifier1, System.identityHashCode(pool.tryAllocate(FORTY_MEGABYTES + 0)));
+    }
+}

From d2f66fa8f3b6fdff468b031418ed240d5d6d04ec Mon Sep 17 00:00:00 2001
From: Virupaksha Swamy Uttanur Matada 
Date: Tue, 1 Oct 2019 16:05:43 -0700
Subject: [PATCH 1034/1071] [LI-HOTFIX] Preallocate maps with exact size
 instead of the default(which is 16) (#50) LI_DESCRIPTION = EXIT_CRITERIA =

---
 .../java/org/apache/kafka/common/protocol/types/Schema.java | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java
index 721b8c63bd76e..92144b37e04d1 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java
@@ -53,7 +53,7 @@ public Schema(Field... fs) {
      */
     public Schema(boolean tolerateMissingFieldsWithDefaults, Field... fs) {
         this.fields = new BoundField[fs.length];
-        this.fieldsByName = new HashMap<>();
+        this.fieldsByName = new HashMap<>(fs.length);
         this.tolerateMissingFieldsWithDefaults = tolerateMissingFieldsWithDefaults;
         for (int i = 0; i < this.fields.length; i++) {
             Field def = fs[i];
@@ -140,7 +140,7 @@ public int numFields() {
 
     /**
      * Get a field by its slot in the record array
-     * 
+     *
      * @param slot The slot at which this field sits
      * @return The field
      */
@@ -150,7 +150,7 @@ public BoundField get(int slot) {
 
     /**
      * Get a field by its name
-     * 
+     *
      * @param name The name of the field
      * @return The field
      */

From b844813687d33c05cb0d85fcbfd2c41962cfc13a Mon Sep 17 00:00:00 2001
From: Xiongqi Wu 
Date: Thu, 10 Oct 2019 11:41:47 -0700
Subject: [PATCH 1035/1071] =?UTF-8?q?[LI-HOTFIX]=20Make=20client-side=20au?=
 =?UTF-8?q?to.topic.creation=20configurable=20and=20def=E2=80=A6=20(#54)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* [LI-HOTFIX] Make client-side auto.topic.creation configurable and default to be false (#41)

TICKET =
LI_DESCRIPTION =
The client-side auto.topic.creation configuration property value has been hard-coded to be true. Making it configurable and default to false helps with disabling auto.topic.creation. This change is made as a hotfix so that the client HI can start enforcing clients to not use auto.topic.creation before the upstream changes are merged into the LI Kafka
EXIT_CRITERIA = 1. Until upstream supports producer side allow.auto.create.topics config 2. Both the
producer and consumer's default policy for auto create topic are set to false
---
 .../kafka/clients/consumer/ConsumerConfig.java     |  2 +-
 .../kafka/clients/producer/KafkaProducer.java      |  3 ++-
 .../kafka/clients/producer/ProducerConfig.java     | 11 ++++++++++-
 .../producer/internals/ProducerMetadata.java       | 14 +++++++++++---
 .../kafka/api/IntegrationTestHarness.scala         |  2 ++
 .../kafka/api/PlaintextProducerSendTest.scala      |  1 +
 .../kafka/tools/MirrorMakerIntegrationTest.scala   |  1 +
 .../kafka/admin/ConsumerGroupCommandTest.scala     |  3 ++-
 .../test/scala/unit/kafka/utils/TestUtils.scala    |  2 ++
 9 files changed, 32 insertions(+), 7 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
index 63eb9bcb37d40..ddfa30c9f6121 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
@@ -278,7 +278,7 @@ public class ConsumerConfig extends AbstractConfig {
             " subscribing to or assigning a topic. A topic being subscribed to will be automatically created only if the" +
             " broker allows for it using `auto.create.topics.enable` broker configuration. This configuration must" +
             " be set to `false` when using brokers older than 0.11.0";
-    public static final boolean DEFAULT_ALLOW_AUTO_CREATE_TOPICS = true;
+    public static final boolean DEFAULT_ALLOW_AUTO_CREATE_TOPICS = false;
 
     /**
      * security.providers
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
index 1ab6ecf441384..1be5f06b64a96 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
@@ -416,7 +416,8 @@ public KafkaProducer(Properties properties, Serializer keySerializer, Seriali
                         logContext,
                         clusterResourceListeners,
                         Time.SYSTEM,
-                        config.getLong(ProducerConfig.METADATA_TOPIC_EXPIRY_MS_CONFIG));
+                        config.getLong(ProducerConfig.METADATA_TOPIC_EXPIRY_MS_CONFIG),
+                        config.getBoolean(ProducerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG));
                 this.metadata.bootstrap(addresses);
             }
             this.errors = this.metrics.sensor("errors");
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
index 6ecf844d141b9..3a1f3db5a5843 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
@@ -257,6 +257,10 @@ public class ProducerConfig extends AbstractConfig {
      */
     public static final String SECURITY_PROVIDERS_CONFIG = SecurityConfig.SECURITY_PROVIDERS_CONFIG;
     private static final String SECURITY_PROVIDERS_DOC = SecurityConfig.SECURITY_PROVIDERS_DOC;
+    /** allow.auto.create.topics */
+    public static final String ALLOW_AUTO_CREATE_TOPICS_CONFIG = "allow.auto.create.topics";
+    private static final String ALLOW_AUTO_CREATE_TOPICS_DOC = "The client-side (producer) permission to allow auto-topic creation. Both the client-side and the broker-side should enable auto-topic creation in order for a topic to be automatically created";
+    public static final boolean DEFAULT_ALLOW_AUTO_CREATE_TOPICS = false;
 
     static {
         CONFIG = new ConfigDef().define(BOOTSTRAP_SERVERS_CONFIG, Type.LIST, Collections.emptyList(), new ConfigDef.NonNullValidator(), Importance.HIGH, CommonClientConfigs.BOOTSTRAP_SERVERS_DOC)
@@ -396,7 +400,12 @@ public class ProducerConfig extends AbstractConfig {
                                         null,
                                         new ConfigDef.NonEmptyString(),
                                         Importance.LOW,
-                                        TRANSACTIONAL_ID_DOC);
+                                        TRANSACTIONAL_ID_DOC)
+                                .define(ALLOW_AUTO_CREATE_TOPICS_CONFIG,
+                                        Type.BOOLEAN,
+                                        DEFAULT_ALLOW_AUTO_CREATE_TOPICS,
+                                        Importance.MEDIUM,
+                                        ALLOW_AUTO_CREATE_TOPICS_DOC);
     }
 
     @Override
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerMetadata.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerMetadata.java
index c21ae4d097867..8759be649e842 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerMetadata.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerMetadata.java
@@ -35,34 +35,42 @@ public class ProducerMetadata extends Metadata {
     private static final long TOPIC_EXPIRY_NEEDS_UPDATE = -1L;
     public static final long TOPIC_EXPIRY_MS = 5 * 60 * 1000;
 
+    private final boolean allowAutoTopicCreation;
     /* Topics with expiry time */
     private final Map topics = new HashMap<>();
     private final Logger log;
     private final Time time;
     private final long topicExpiryMs;
 
+    // LI-HOTFIX: this constructor should only be used for unit tests
+    // after the following hotfix changes:
+    // 1) add metadata.topic.expiry.ms config to KafkaProducer
+    // 2) Make client-side auto.topic.creation configurable and default to be false
     public ProducerMetadata(long refreshBackoffMs,
         long metadataExpireMs,
         LogContext logContext,
         ClusterResourceListeners clusterResourceListeners,
         Time time) {
-        this(refreshBackoffMs, metadataExpireMs, logContext, clusterResourceListeners, time, TOPIC_EXPIRY_MS);
+        this(refreshBackoffMs, metadataExpireMs, logContext, clusterResourceListeners, time, TOPIC_EXPIRY_MS, true);
     }
+
     public ProducerMetadata(long refreshBackoffMs,
                             long metadataExpireMs,
                             LogContext logContext,
                             ClusterResourceListeners clusterResourceListeners,
                             Time time,
-                            long topicExpiryMs) {
+                            long topicExpiryMs,
+                            boolean allowAutoTopicCreation) {
         super(refreshBackoffMs, metadataExpireMs, logContext, clusterResourceListeners);
         this.log = logContext.logger(ProducerMetadata.class);
         this.time = time;
         this.topicExpiryMs = topicExpiryMs;
+        this.allowAutoTopicCreation = allowAutoTopicCreation;
     }
 
     @Override
     public synchronized MetadataRequest.Builder newMetadataRequestBuilder() {
-        return new MetadataRequest.Builder(new ArrayList<>(topics.keySet()), true);
+        return new MetadataRequest.Builder(new ArrayList<>(topics.keySet()), allowAutoTopicCreation);
     }
 
     public synchronized void add(String topic) {
diff --git a/core/src/test/scala/integration/kafka/api/IntegrationTestHarness.scala b/core/src/test/scala/integration/kafka/api/IntegrationTestHarness.scala
index 557b278dfa70d..187766d7c2ecd 100644
--- a/core/src/test/scala/integration/kafka/api/IntegrationTestHarness.scala
+++ b/core/src/test/scala/integration/kafka/api/IntegrationTestHarness.scala
@@ -96,12 +96,14 @@ abstract class IntegrationTestHarness extends KafkaServerTestHarness {
     producerConfig.putIfAbsent(ProducerConfig.ACKS_CONFIG, "-1")
     producerConfig.putIfAbsent(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, classOf[ByteArraySerializer].getName)
     producerConfig.putIfAbsent(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, classOf[ByteArraySerializer].getName)
+    producerConfig.putIfAbsent(ProducerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, boolean2Boolean(true))
 
     consumerConfig.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList)
     consumerConfig.putIfAbsent(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
     consumerConfig.putIfAbsent(ConsumerConfig.GROUP_ID_CONFIG, "group")
     consumerConfig.putIfAbsent(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, classOf[ByteArrayDeserializer].getName)
     consumerConfig.putIfAbsent(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, classOf[ByteArrayDeserializer].getName)
+    consumerConfig.putIfAbsent(ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, boolean2Boolean(true))
 
     adminClientConfig.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList)
 
diff --git a/core/src/test/scala/integration/kafka/api/PlaintextProducerSendTest.scala b/core/src/test/scala/integration/kafka/api/PlaintextProducerSendTest.scala
index 1dda60c589a60..23824a3cfe3ed 100644
--- a/core/src/test/scala/integration/kafka/api/PlaintextProducerSendTest.scala
+++ b/core/src/test/scala/integration/kafka/api/PlaintextProducerSendTest.scala
@@ -36,6 +36,7 @@ class PlaintextProducerSendTest extends BaseProducerSendTest {
     producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList)
     producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
     producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
+    producerProps.put(ProducerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, boolean2Boolean(true))
     val producer = registerProducer(new KafkaProducer(producerProps))
     val record = new ProducerRecord[Array[Byte], Array[Byte]](topic, 0, "key".getBytes, "value".getBytes)
     producer.send(record)
diff --git a/core/src/test/scala/integration/kafka/tools/MirrorMakerIntegrationTest.scala b/core/src/test/scala/integration/kafka/tools/MirrorMakerIntegrationTest.scala
index 4384091ce8235..a5a6caffda2e2 100644
--- a/core/src/test/scala/integration/kafka/tools/MirrorMakerIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/tools/MirrorMakerIntegrationTest.scala
@@ -75,6 +75,7 @@ class MirrorMakerIntegrationTest extends KafkaServerTestHarness {
     producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList)
     producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, classOf[ByteArraySerializer])
     producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, classOf[ByteArraySerializer])
+    producerProps.put(ProducerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, boolean2Boolean(true))
     val producer = new MirrorMakerProducer(true, producerProps)
     MirrorMaker.producer = producer
     MirrorMaker.producer.send(new ProducerRecord(topic, msg.getBytes()))
diff --git a/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala b/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
index 853b2caa7c751..1fbdb0811c505 100644
--- a/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
@@ -26,7 +26,7 @@ import kafka.integration.KafkaServerTestHarness
 import kafka.server.KafkaConfig
 import kafka.utils.TestUtils
 import org.apache.kafka.clients.admin.AdminClientConfig
-import org.apache.kafka.clients.consumer.{KafkaConsumer, RangeAssignor}
+import org.apache.kafka.clients.consumer.{ConsumerConfig, KafkaConsumer, RangeAssignor}
 import org.apache.kafka.common.{PartitionInfo, TopicPartition}
 import org.apache.kafka.common.errors.WakeupException
 import org.apache.kafka.common.serialization.StringDeserializer
@@ -129,6 +129,7 @@ object ConsumerGroupCommandTest {
       props.put("group.id", groupId)
       props.put("key.deserializer", classOf[StringDeserializer].getName)
       props.put("value.deserializer", classOf[StringDeserializer].getName)
+      props.put(ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, boolean2Boolean(true))
     }
 
     def subscribe(): Unit
diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index c4c6466424273..00c40209682a8 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -608,6 +608,7 @@ object TestUtils extends Logging {
     producerProps.put(ProducerConfig.LINGER_MS_CONFIG, lingerMs.toString)
     producerProps.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize.toString)
     producerProps.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, compressionType)
+    producerProps.put(ProducerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, boolean2Boolean(true))
     producerProps ++= producerSecurityConfigs(securityProtocol, trustStoreFile, saslProperties)
     new KafkaProducer[K, V](producerProps, keySerializer, valueSerializer)
   }
@@ -649,6 +650,7 @@ object TestUtils extends Logging {
     consumerProps.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, enableAutoCommit.toString)
     consumerProps.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords.toString)
     consumerProps.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, if (readCommitted) "read_committed" else "read_uncommitted")
+    consumerProps.put(ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, boolean2Boolean(true))
     consumerProps ++= consumerSecurityConfigs(securityProtocol, trustStoreFile, saslProperties)
     new KafkaConsumer[K, V](consumerProps, keyDeserializer, valueDeserializer)
   }

From 6b93c274b9a188ee8f5bcb130dad72f92fb10c79 Mon Sep 17 00:00:00 2001
From: Xiongqi Wesley Wu 
Date: Tue, 13 Nov 2018 14:07:18 -0800
Subject: [PATCH 1036/1071] [LI-HOTFIX] Add dynamic maintenance broker config
 TICKET = KAFKA-8527 LI_DESCRIPTION = When a broker is masked as maintenance
 broker, Kafka will not assign partitions of new topics to the broker if
 partition assignment is done by kafka brokers. 1) add dynamic broker config
 to support maintenance brokers 2) add logic to update maintenance broker info
 in controller 3) support maintenance broker in Kafka API 4) support
 maintenance broker in AdminZkClient 5) add logic to rearrange replica
 assignment for new topics for maintenance brokers 5) test cases

RB=1484649
G=Kafka-Code-Reviews
R=agencer, amendhek, jkoshy, zahuang
A=zahuang

EXIT_CRITERIA = TICKET [KAFKA-8527]
---
 .../kafka/controller/KafkaController.scala    |  50 +++-
 .../scala/kafka/server/AdminManager.scala     |  14 +-
 .../kafka/server/DynamicBrokerConfig.scala    |  15 +-
 .../scala/kafka/server/DynamicConfig.scala    |  31 ++-
 .../main/scala/kafka/server/KafkaConfig.scala |   4 +
 .../main/scala/kafka/zk/AdminZkClient.scala   |  57 ++++-
 .../main/scala/kafka/zk/KafkaZkClient.scala   |  17 ++
 .../kafka/server/MaintenanceBrokerTest.scala  | 215 ++++++++++++++++++
 8 files changed, 388 insertions(+), 15 deletions(-)
 create mode 100644 core/src/test/scala/unit/kafka/server/MaintenanceBrokerTest.scala

diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala
index ff5b65cb5f8f6..f7fd9b3c22a05 100644
--- a/core/src/main/scala/kafka/controller/KafkaController.scala
+++ b/core/src/main/scala/kafka/controller/KafkaController.scala
@@ -101,6 +101,7 @@ class KafkaController(val config: KafkaConfig,
                       threadNamePrefix: Option[String] = None)
   extends ControllerEventProcessor with Logging with KafkaMetricsGroup {
 
+  val adminZkClient = new AdminZkClient(zkClient)
   this.logIdent = s"[Controller id=${config.brokerId}] "
 
   @volatile private var brokerInfo = initialBrokerInfo
@@ -218,6 +219,13 @@ class KafkaController(val config: KafkaConfig,
     }
   )
 
+    newGauge(
+    "MaintenanceBrokerCount",
+    new Gauge[Int] {
+      def value: Int = if (isActive) config.getMaintenanceBrokerList.size else 0
+    }
+  )
+
   /**
    * Returns true if this broker is the current controller.
    */
@@ -817,7 +825,9 @@ class KafkaController(val config: KafkaConfig,
     val curBrokerAndEpochs = zkClient.getAllBrokerAndEpochsInCluster
     controllerContext.setLiveBrokerAndEpochs(curBrokerAndEpochs)
     info(s"Initialized broker epochs cache: ${controllerContext.liveBrokerIdAndEpochs}")
+
     controllerContext.allTopics = zkClient.getAllTopicsInCluster
+    rearrangePartitionReplicaAssignmentForNewTopics(controllerContext.allTopics.toSet)
     registerPartitionModificationsHandlers(controllerContext.allTopics.toSeq)
     getReplicaAssignmentPolicyCompliant(controllerContext.allTopics.toSet).foreach {
       case (topicPartition, replicaAssignment) =>
@@ -903,6 +913,37 @@ class KafkaController(val config: KafkaConfig,
     }
   }
 
+  // Rearrange partition and replica assignment for new topics that get assigned to
+  // maintenance brokers that do not take new partitions
+  private def rearrangePartitionReplicaAssignmentForNewTopics(topics: Set[String]) {
+    try {
+      val noNewPartitionBrokerIds = config.getMaintenanceBrokerList
+      if (noNewPartitionBrokerIds.nonEmpty) {
+        val newTopics = zkClient.getPartitionNodeNonExistsTopics(topics.toSet)
+        val newTopicsToBeArranged = zkClient.getPartitionAssignmentForTopics(newTopics).filter {
+          case (_, partitionMap) =>
+            partitionMap.exists {
+              case (_, assignedReplicas) =>
+                assignedReplicas.replicas.intersect(noNewPartitionBrokerIds).nonEmpty
+            }
+        }
+        newTopicsToBeArranged.foreach {
+          case (topic, partitionMap) =>
+            val numPartitions = partitionMap.size
+            val numReplica = partitionMap.head._2.replicas.size
+            val brokers = controllerContext.liveOrShuttingDownBrokers.map { b => kafka.admin.BrokerMetadata(b.id, b.rack) }.toSeq
+
+            val replicaAssignment = adminZkClient.assignReplicasToAvailableBrokers(brokers, noNewPartitionBrokerIds.toSet, numPartitions, numReplica)
+            adminZkClient.writeTopicPartitionAssignment(topic, replicaAssignment.mapValues(ReplicaAssignment(_)).toMap, true)
+            info(s"Rearrange partition and replica assignment for topic [$topic]")
+        }
+      }
+    } catch {
+      case e =>
+        error("Error during rearranging partition and replica assignment for new topics for maintenance brokers :" + e.getMessage)
+    }
+  }
+
   private def moveReassignedPartitionLeaderIfRequired(topicPartition: TopicPartition,
                                                       newAssignment: ReplicaAssignment): Unit = {
     val reassignedReplicas = newAssignment.replicas
@@ -1484,6 +1525,7 @@ class KafkaController(val config: KafkaConfig,
     val newTopics = topics -- controllerContext.allTopics
     val deletedTopics = controllerContext.allTopics -- topics
     controllerContext.allTopics = topics
+    rearrangePartitionReplicaAssignmentForNewTopics(newTopics)
 
     registerPartitionModificationsHandlers(newTopics.toSeq)
     val addedPartitionReplicaAssignment = getReplicaAssignmentPolicyCompliant(newTopics)
@@ -1880,13 +1922,15 @@ class KafkaController(val config: KafkaConfig,
 
     val replicationFactor = config.defaultReplicationFactor
     val brokers = controllerContext.liveOrShuttingDownBrokers.map { sb => kafka.admin.BrokerMetadata(sb.id, sb.rack) }.toSeq
+    val noNewPartitionBrokerIds = config.getMaintenanceBrokerList.toSet
 
     topicsReplicaAssignment.foreach{
       case(topic, partitionAssignment) => {
         val numPartitions = partitionAssignment.size
-        val assignment = AdminUtils.assignReplicasToBrokers(brokers, numPartitions, replicationFactor)
-          .map{ case(partition, replicas) => (new TopicPartition(topic, partition), new ReplicaAssignment(replicas, Seq.empty[Int], Seq.empty[Int]))
-          }.toMap
+        val assignment = adminZkClient.assignReplicasToAvailableBrokers(brokers, noNewPartitionBrokerIds, numPartitions, replicationFactor)
+          .map{ case(partition, replicas) => {
+            (new TopicPartition(topic, partition), new ReplicaAssignment(replicas, Seq.empty[Int], Seq.empty[Int]))
+          }}.toMap
         zkClient.setTopicAssignment(topic, assignment, controllerContext.epochZkVersion)
         info(s"Updated topic [$topic] with $assignment for replica assignment")
       }
diff --git a/core/src/main/scala/kafka/server/AdminManager.scala b/core/src/main/scala/kafka/server/AdminManager.scala
index 346a013b4e346..1449a798fe167 100644
--- a/core/src/main/scala/kafka/server/AdminManager.scala
+++ b/core/src/main/scala/kafka/server/AdminManager.scala
@@ -18,7 +18,7 @@ package kafka.server
 
 import java.util.{Collections, Properties}
 
-import kafka.admin.{AdminOperationException, AdminUtils}
+import kafka.admin.AdminOperationException
 import kafka.common.TopicAlreadyMarkedForDeletionException
 import kafka.log.LogConfig
 import kafka.utils.Log4jController
@@ -111,8 +111,8 @@ class AdminManager(val config: KafkaConfig,
           defaultReplicationFactor else topic.replicationFactor
 
         val assignments = if (topic.assignments().isEmpty) {
-          AdminUtils.assignReplicasToBrokers(
-            brokers, resolvedNumPartitions, resolvedReplicationFactor)
+          adminZkClient.assignReplicasToAvailableBrokers(brokers, config.getMaintenanceBrokerList.toSet,
+            resolvedNumPartitions, resolvedReplicationFactor)
         } else {
           val assignments = new mutable.HashMap[Int, Seq[Int]]
           // Note: we don't check that replicaAssignment contains unknown brokers - unlike in add-partitions case,
@@ -307,7 +307,8 @@ class AdminManager(val config: KafkaConfig,
         }
 
         val updatedReplicaAssignment = adminZkClient.addPartitions(topic, existingAssignment, allBrokers,
-          newPartition.totalCount, newPartitionsAssignment, validateOnly = validateOnly)
+          newPartition.totalCount, newPartitionsAssignment, validateOnly = validateOnly,
+          noNewPartitionBrokerIds = config.getMaintenanceBrokerList.toSet)
         CreatePartitionsMetadata(topic, updatedReplicaAssignment, ApiError.NONE)
       } catch {
         case e: AdminOperationException =>
@@ -630,7 +631,10 @@ class AdminManager(val config: KafkaConfig,
   }
 
   private def configType(name: String, synonyms: List[String]): ConfigDef.Type = {
-    val configType = config.typeOf(name)
+    var configType = config.typeOf(name)
+    if (configType == null)
+      configType = DynamicConfig.Broker.typeOf(name)
+
     if (configType != null)
       configType
     else
diff --git a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
index b510e70d73698..bfb1efc73cdfb 100755
--- a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
+++ b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
@@ -130,7 +130,10 @@ object DynamicBrokerConfig {
     checkInvalidProps(securityConfigsWithoutListenerPrefix(props),
       "These security configs can be dynamically updated only per-listener using the listener prefix")
     validateConfigTypes(props)
-    if (!perBrokerConfig) {
+    if (perBrokerConfig) {
+      checkInvalidProps(clusterLevelConfigs(props),
+        "Cannot update these configs at per broker level, broker id must not be specified")
+    } else {
       checkInvalidProps(perBrokerConfigs(props),
         "Cannot update these configs at default cluster level, broker id must be specified")
     }
@@ -147,6 +150,11 @@ object DynamicBrokerConfig {
     configNames.intersect(PerBrokerConfigs) ++ configNames.filter(perBrokerListenerConfig)
   }
 
+  private def clusterLevelConfigs(props: Properties): Set[String] = {
+    val configNames = props.asScala.keySet
+    configNames.intersect(DynamicConfig.Broker.ClusterLevelConfigs)
+  }
+
   private def nonDynamicConfigs(props: Properties): Set[String] = {
     props.asScala.keySet.intersect(DynamicConfig.Broker.nonDynamicProps)
   }
@@ -324,6 +332,11 @@ class DynamicBrokerConfig(private val kafkaConfig: KafkaConfig) extends Logging
       }
   }
 
+  private[server] def getMaintenanceBrokerList: Seq[Int] = CoreUtils.inReadLock(lock) {
+    DynamicConfig.Broker.getMaintenanceBrokerListFromString(dynamicDefaultConfigs.getOrElse(DynamicConfig.Broker.MaintenanceBrokerListProp,
+      DynamicConfig.Broker.DefaultMaintenanceBrokerList).toString)
+  }
+
   private def maybeCreatePasswordEncoder(secret: Option[Password]): Option[PasswordEncoder] = {
    secret.map { secret =>
       new PasswordEncoder(secret,
diff --git a/core/src/main/scala/kafka/server/DynamicConfig.scala b/core/src/main/scala/kafka/server/DynamicConfig.scala
index e5974d3642dc8..e55cb492ecfc2 100644
--- a/core/src/main/scala/kafka/server/DynamicConfig.scala
+++ b/core/src/main/scala/kafka/server/DynamicConfig.scala
@@ -21,10 +21,11 @@ import java.util.Properties
 
 import kafka.log.LogConfig
 import kafka.security.CredentialProvider
-import org.apache.kafka.common.config.ConfigDef
+import org.apache.kafka.common.config.{ConfigDef, ConfigException}
 import org.apache.kafka.common.config.ConfigDef.Importance._
 import org.apache.kafka.common.config.ConfigDef.Range._
 import org.apache.kafka.common.config.ConfigDef.Type._
+import org.apache.kafka.common.config.ConfigDef.Validator
 
 import scala.collection.JavaConverters._
 
@@ -39,9 +40,11 @@ object DynamicConfig {
     val LeaderReplicationThrottledRateProp = "leader.replication.throttled.rate"
     val FollowerReplicationThrottledRateProp = "follower.replication.throttled.rate"
     val ReplicaAlterLogDirsIoMaxBytesPerSecondProp = "replica.alter.log.dirs.io.max.bytes.per.second"
+    val MaintenanceBrokerListProp = "maintenance.broker.list"
 
     //Defaults
     val DefaultReplicationThrottledRate = ReplicationQuotaManagerConfig.QuotaBytesPerSecondDefault
+    val DefaultMaintenanceBrokerList: String = ""
 
     //Documentation
     val LeaderReplicationThrottledRateDoc = "A long representing the upper bound (bytes/sec) on replication traffic for leaders enumerated in the " +
@@ -52,6 +55,7 @@ object DynamicConfig {
       s"limit be kept above 1MB/s for accurate behaviour."
     val ReplicaAlterLogDirsIoMaxBytesPerSecondDoc = "A long representing the upper bound (bytes/sec) on disk IO used for moving replica between log directories on the same broker. " +
       s"This property can be only set dynamically. It is suggested that the limit be kept above 1MB/s for accurate behaviour."
+    val MaintenanceBrokerListDoc = "A list containing maintenance broker Ids, separated by comma"
 
     //Definitions
     private val brokerConfigDef = new ConfigDef()
@@ -59,12 +63,37 @@ object DynamicConfig {
       .define(LeaderReplicationThrottledRateProp, LONG, DefaultReplicationThrottledRate, atLeast(0), MEDIUM, LeaderReplicationThrottledRateDoc)
       .define(FollowerReplicationThrottledRateProp, LONG, DefaultReplicationThrottledRate, atLeast(0), MEDIUM, FollowerReplicationThrottledRateDoc)
       .define(ReplicaAlterLogDirsIoMaxBytesPerSecondProp, LONG, DefaultReplicationThrottledRate, atLeast(0), MEDIUM, ReplicaAlterLogDirsIoMaxBytesPerSecondDoc)
+      .define(MaintenanceBrokerListProp, STRING, DefaultMaintenanceBrokerList, MaintenanceBrokerListValidator, MEDIUM, MaintenanceBrokerListDoc)
     DynamicBrokerConfig.addDynamicConfigs(brokerConfigDef)
     val nonDynamicProps = KafkaConfig.configNames.toSet -- brokerConfigDef.names.asScala
 
+    //cluster level only configs
+    val ClusterLevelConfigs = Set(MaintenanceBrokerListProp)
+    def typeOf(key: String): ConfigDef.Type = {
+      val configKey: ConfigDef.ConfigKey = brokerConfigDef.configKeys().get(key)
+      if (configKey == null)
+        null
+      else
+        configKey.`type`
+    }
+
     def names = brokerConfigDef.names
 
     def validate(props: Properties) = DynamicConfig.validate(brokerConfigDef, props, customPropsAllowed = true)
+
+    def getMaintenanceBrokerListFromString(brokerListStr: String): Seq[Int] = {
+      brokerListStr.split(",").map(_.trim).filter(_.nonEmpty).map(_.toInt)
+    }
+
+    object MaintenanceBrokerListValidator extends Validator {
+      override def ensureValid(name: String, value: Any): Unit = {
+        try {
+          getMaintenanceBrokerListFromString(value.toString)
+        } catch {
+          case e: NumberFormatException => throw new ConfigException(name, value.toString, e.getMessage)
+        }
+      }
+    }
   }
 
   object Client {
diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index 0e43792760a53..5b0208e04418f 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -1454,6 +1454,10 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO
     millis
   }
 
+  def getMaintenanceBrokerList: Seq[Int] = {
+    dynamicConfig.getMaintenanceBrokerList
+  }
+
   private def getMap(propName: String, propValue: String): Map[String, String] = {
     try {
       CoreUtils.parseCsvMap(propValue)
diff --git a/core/src/main/scala/kafka/zk/AdminZkClient.scala b/core/src/main/scala/kafka/zk/AdminZkClient.scala
index 63e2614aba93a..7c464fd50f60f 100644
--- a/core/src/main/scala/kafka/zk/AdminZkClient.scala
+++ b/core/src/main/scala/kafka/zk/AdminZkClient.scala
@@ -53,7 +53,8 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
                   topicConfig: Properties = new Properties,
                   rackAwareMode: RackAwareMode = RackAwareMode.Enforced): Unit = {
     val brokerMetadatas = getBrokerMetadatas(rackAwareMode)
-    val replicaAssignment = AdminUtils.assignReplicasToBrokers(brokerMetadatas, partitions, replicationFactor)
+    val noNewPartitionBrokerIds = getMaintenanceBrokerList()
+    val replicaAssignment = assignReplicasToAvailableBrokers(brokerMetadatas, noNewPartitionBrokerIds.toSet, partitions, replicationFactor)
     createTopicWithAssignment(topic, topicConfig, replicaAssignment)
   }
 
@@ -81,6 +82,16 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
     brokerMetadatas.sortBy(_.id)
   }
 
+  /**
+    * fetch maintenance broker list from zk
+    */
+  def getMaintenanceBrokerList(): Seq[Int] = {
+    val maintenanceBrokerConfig = fetchEntityConfig(ConfigType.Broker, ConfigEntityName.Default)
+      .getProperty(DynamicConfig.Broker.MaintenanceBrokerListProp, DynamicConfig.Broker.DefaultMaintenanceBrokerList)
+
+    DynamicConfig.Broker.getMaintenanceBrokerListFromString(maintenanceBrokerConfig)
+  }
+
   def createTopicWithAssignment(topic: String,
                                 config: Properties,
                                 partitionReplicaAssignment: Map[Int, Seq[Int]]): Unit = {
@@ -135,7 +146,8 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
     LogConfig.validate(config)
   }
 
-  private def writeTopicPartitionAssignment(topic: String, replicaAssignment: Map[Int, ReplicaAssignment], isUpdate: Boolean): Unit = {
+
+  def writeTopicPartitionAssignment(topic: String, replicaAssignment: Map[Int, ReplicaAssignment], isUpdate: Boolean): Unit = {
     try {
       val assignment = replicaAssignment.map { case (partitionId, replicas) => (new TopicPartition(topic,partitionId), replicas) }.toMap
 
@@ -169,6 +181,39 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
     }
   }
 
+  /**
+    * Assign replicas to brokers that take new partitions.
+    * If the number of replicationFactor is greater than the number of brokers that take new partitions,
+    * all brokers are used for assignment.
+    * @param brokerMetadatas
+    * @param noNewPartitionBrokerIds
+    * @param nPartitions
+    * @param replicationFactor
+    * @param fixedStartIndex
+    * @param startPartitionId
+    * @return
+    */
+  def assignReplicasToAvailableBrokers(brokerMetadatas: Seq[BrokerMetadata],
+    noNewPartitionBrokerIds: Set[Int],
+    nPartitions: Int,
+    replicationFactor: Int,
+    fixedStartIndex: Int = -1,
+    startPartitionId: Int = -1): Map[Int, Seq[Int]] = {
+
+    val availableBrokerMetadata = brokerMetadatas.filter {
+      brokerMetadata =>
+        if (noNewPartitionBrokerIds.contains(brokerMetadata.id)) false
+        else true
+    }
+
+    if (replicationFactor > availableBrokerMetadata.size) {
+      info(s"Using all brokers for replica assignment since replicationFactor[$replicationFactor] " +
+        s"is larger than the number of nonMaintenanceBroker[${availableBrokerMetadata.size}]")
+      AdminUtils.assignReplicasToBrokers(brokerMetadatas, nPartitions, replicationFactor, fixedStartIndex, startPartitionId)
+    } else
+      AdminUtils.assignReplicasToBrokers(availableBrokerMetadata, nPartitions, replicationFactor, fixedStartIndex, startPartitionId)
+  }
+
   /**
   * Add partitions to existing topic with optional replica assignment
   *
@@ -178,6 +223,7 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
   * @param numPartitions Number of partitions to be set
   * @param replicaAssignment Manual replica assignment, or none
   * @param validateOnly If true, validate the parameters without actually adding the partitions
+  * @param noNewPartitionBrokerIds Brokers that do not take new partitions
   * @return the updated replica assignment
   */
   def addPartitions(topic: String,
@@ -185,7 +231,8 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
                     allBrokers: Seq[BrokerMetadata],
                     numPartitions: Int = 1,
                     replicaAssignment: Option[Map[Int, Seq[Int]]] = None,
-                    validateOnly: Boolean = false): Map[Int, Seq[Int]] = {
+                    validateOnly: Boolean = false,
+                    noNewPartitionBrokerIds: Set[Int] = Set.empty[Int]): Map[Int, Seq[Int]] = {
     val existingAssignmentPartition0 = existingAssignment.getOrElse(0,
       throw new AdminOperationException(
         s"Unexpected existing replica assignment for topic '$topic', partition id 0 is missing. " +
@@ -205,8 +252,8 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
 
     val proposedAssignmentForNewPartitions = replicaAssignment.getOrElse {
       val startIndex = math.max(0, allBrokers.indexWhere(_.id >= existingAssignmentPartition0.head))
-      AdminUtils.assignReplicasToBrokers(allBrokers, partitionsToAdd, existingAssignmentPartition0.size,
-        startIndex, existingAssignment.size)
+      assignReplicasToAvailableBrokers(allBrokers, noNewPartitionBrokerIds, partitionsToAdd,
+        existingAssignmentPartition0.size, startIndex, existingAssignment.size)
     }
 
     val proposedAssignment = existingAssignment ++ proposedAssignmentForNewPartitions.map { case (tp, replicas) =>
diff --git a/core/src/main/scala/kafka/zk/KafkaZkClient.scala b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
index 8f0f1851eddfc..ee6a6b12dfe10 100644
--- a/core/src/main/scala/kafka/zk/KafkaZkClient.scala
+++ b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
@@ -622,6 +622,23 @@ class KafkaZkClient private[zk] (zooKeeperClient: ZooKeeperClient, isSecure: Boo
     }.toMap
   }
 
+  def getPartitionNodeNonExistsTopics(topics: Set[String]): Set[String] = {
+    val existsRequests = topics.map(topic => ExistsRequest(TopicPartitionsZNode.path(topic), ctx = Some(topic)))
+    val existsResponses = retryRequestsUntilConnected(existsRequests.toSeq)
+    val newTopics = scala.collection.mutable.Set.empty[String]
+
+    existsResponses.foreach {
+      existsResponse =>
+        val topic = existsResponse.ctx.get.asInstanceOf[String]
+        existsResponse.resultCode match {
+          case Code.OK =>
+          case Code.NONODE => newTopics.add(topic)
+          case _ => throw existsResponse.resultException.get
+        }
+    }
+    newTopics.toSet
+  }
+
   /**
    * Gets the partition numbers for the given topics
    * @param topics the topics whose partitions we wish to get.
diff --git a/core/src/test/scala/unit/kafka/server/MaintenanceBrokerTest.scala b/core/src/test/scala/unit/kafka/server/MaintenanceBrokerTest.scala
new file mode 100644
index 0000000000000..c97740136623b
--- /dev/null
+++ b/core/src/test/scala/unit/kafka/server/MaintenanceBrokerTest.scala
@@ -0,0 +1,215 @@
+/**
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  * http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+
+package kafka.server
+
+import java.util.{Optional, Properties}
+
+import kafka.server.KafkaConfig.fromProps
+import kafka.utils.CoreUtils._
+import kafka.utils.TestUtils
+import kafka.utils.TestUtils._
+import kafka.zk.ZooKeeperTestHarness
+import org.apache.kafka.clients.admin._
+import org.apache.kafka.common.network.ListenerName
+import org.apache.kafka.common.security.auth.SecurityProtocol
+
+import scala.collection.JavaConverters._
+import org.junit.Assert._
+import org.junit.{After, Test}
+
+import scala.collection.Map
+
+/**
+  * This is the main test which ensure maintenance broker work correctly.
+  */
+class MaintenanceBrokerTest extends ZooKeeperTestHarness {
+
+  var brokers: Seq[KafkaServer] = null
+
+  @After
+  override def tearDown() {
+    shutdownServers(brokers)
+    super.tearDown()
+  }
+
+  @Test
+  def testTopicCreatedByZkclientShouldHonorMaintenanceBrokers(): Unit = {
+
+    brokers = (0 to 2).map { id => createServer(fromProps(createBrokerConfig(id, zkConnect))) }
+
+    // create topic using zkclient
+    TestUtils.createTopic(zkClient, "topic1", 3, 2, brokers)
+    (0 to 2).foreach {
+      brokerId =>
+        assertTrue("topic1 should be in broker " + brokerId, !ensureTopicNotInBrokers("topic1", Set(brokerId)))
+    }
+
+    // setting broker 1 to not take new topic partitions
+    setMaintenanceBrokers(Seq(1))
+
+    TestUtils.createTopic(zkClient, "topic2", 3, 2, brokers)
+    assertTrue("topic2 should not be in broker 1", ensureTopicNotInBrokers("topic2", Set(1)))
+
+    // setting broker 1 and 2 to not take new topic partitions
+    setMaintenanceBrokers(Seq(1, 2))
+
+    TestUtils.createTopic(zkClient, "topic3", 3, 1, brokers)
+
+    assertTrue("topic3 should not be in broker 1 and 2", ensureTopicNotInBrokers("topic3", Set(1, 2)))
+    assertTrue("topic3 should in broker 0", !ensureTopicNotInBrokers("topic3", Set(0)))
+  }
+
+  @Test
+  def testTopicCreatedByAdminClientShouldHonorMaintenanceBrokers(): Unit = {
+
+    brokers = (0 to 2).map { id => createServer(fromProps(createBrokerConfig(id, zkConnect))) }
+
+    val brokerList = TestUtils.bootstrapServers(brokers, ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT))
+    val adminClientConfig = new Properties
+    adminClientConfig.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList)
+    val client = AdminClient.create(adminClientConfig)
+
+    // create topic using admin client
+    val future1 = client.createTopics(Seq("topic1").map(new NewTopic(_, 3, 2.toShort)).asJava,
+      new CreateTopicsOptions()).all()
+    future1.get()
+
+    (0 to 2).foreach {
+      brokerId =>
+        assertTrue("topic1 should be in broker " + brokerId, !ensureTopicNotInBrokers("topic1", Set(brokerId)))
+    }
+
+    TestUtils.waitUntilControllerElected(zkClient)
+
+    // setting broker 1 to not take new topic partitions
+    setMaintenanceBrokers(Seq(1))
+
+    val future2 = client.createTopics(Seq("topic2").map(new NewTopic(_, 3, 2.toShort)).asJava,
+      new CreateTopicsOptions()).all()
+    future2.get()
+
+    assertTrue("topic2 should not be in broker 1", ensureTopicNotInBrokers("topic2", Set(1)))
+
+    // setting broker 1 and 2 to not take new topic partitions
+    setMaintenanceBrokers(Seq(1, 2))
+
+    val future3 = client.createTopics(Seq("topic3").map(new NewTopic(_, 3, 1.toShort)).asJava,
+      new CreateTopicsOptions()).all()
+    future3.get()
+
+    assertTrue("topic3 should not be in broker 1 and 2", ensureTopicNotInBrokers("topic3", Set(1, 2)))
+    assertTrue("topic3 should be in broker 0", !ensureTopicNotInBrokers("topic3", Set(0)))
+
+    // create topic with #replicas > #non-maintenance brokers
+    val future4 = client.createTopics(Seq("topic4").map(new NewTopic(_, 3, 3.toShort)).asJava,
+      new CreateTopicsOptions()).all()
+    future4.get()
+
+    (0 to 2).foreach {
+      brokerId =>
+        assertTrue("topic4 should be in broker " + brokerId + " because #replicas > #non-maintenance brokers",
+          !ensureTopicNotInBrokers("topic4", Set(brokerId)))
+    }
+
+    // clear maintenance broker
+    setMaintenanceBrokers(Seq.empty[Int])
+
+    // create topic using admin client
+    val future5 = client.createTopics(Seq("topic5").map(new NewTopic(_, 3, 1.toShort)).asJava,
+      new CreateTopicsOptions()).all()
+    future5.get()
+
+    (0 to 2).foreach {
+      brokerId =>
+        assertTrue("topic5 should be in broker " + brokerId, !ensureTopicNotInBrokers("topic5", Set(brokerId)))
+    }
+
+    client.close()
+  }
+
+  @Test
+  def testAddPartitionByAdminClientShouldHonorMaintenanceBrokers(): Unit = {
+
+    brokers = (0 to 2).map { id => createServer(fromProps(createBrokerConfig(id, zkConnect))) }
+
+    val brokerList = TestUtils.bootstrapServers(brokers, ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT))
+    val adminClientConfig = new Properties
+    adminClientConfig.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList)
+    val client = AdminClient.create(adminClientConfig)
+
+    TestUtils.waitUntilControllerElected(zkClient)
+
+    // setting broker 1 to not take new topic partitions
+    setMaintenanceBrokers(Seq(1))
+
+    val future1 = client.createTopics(Seq("topic1").map(new NewTopic(_, 3, 2.toShort)).asJava,
+      new CreateTopicsOptions()).all()
+    future1.get()
+
+    assertTrue("topic1 should not be in broker 1", ensureTopicNotInBrokers("topic1", Set(1)))
+
+    val future2 = client.createPartitions(Map("topic1" -> NewPartitions.increaseTo(5)).asJava).all()
+    future2.get()
+
+    assertTrue("topic1 should not be in broker 1 after increasing partition count",
+      ensureTopicNotInBrokers("topic1", Set(1)))
+
+    client.close()
+  }
+
+  @Test
+  def testTopicCreatedInZkShouldBeRearrangedForMaintenanceBrokers(): Unit = {
+
+    brokers = (0 to 2).map { id => createServer(fromProps(createBrokerConfig(id, zkConnect))) }
+    TestUtils.waitUntilControllerElected(zkClient)
+
+    TestUtils.createTopic(zkClient, "topic1", Map(0 -> List(0, 1), 1 -> List(1, 2)), brokers)
+
+    // setting broker 1 to not take new topic partitions
+    setMaintenanceBrokers(Seq(1))
+
+    TestUtils.createTopic(zkClient, "topic2", Map(0 -> List(0, 1), 1 -> List(1, 2)), brokers)
+
+    assertTrue("new topic topic2 should be rearranged and not be in broker 1", ensureTopicNotInBrokers("topic2", Set(1)))
+    assertTrue("old topic topic1 should still in broker 1", !ensureTopicNotInBrokers("topic1", Set(1)))
+  }
+
+  def ensureTopicNotInBrokers(topic: String, brokerIds: Set[Int]): Boolean = {
+    val topicAssignment = zkClient.getReplicaAssignmentForTopics(Set(topic))
+    topicAssignment.map(_._2).flatten.toSet.intersect(brokerIds).isEmpty
+  }
+
+  def createBrokers(brokerIds: Seq[Int]): Unit = {
+    brokerIds.foreach { id =>
+      brokers = brokers :+ createServer(fromProps(createBrokerConfig(id, zkConnect)))
+    }
+  }
+
+  def setMaintenanceBrokers(brokerIds: Seq[Int]): Unit = {
+    var propstring = brokerIds.mkString(",")
+    adminZkClient.changeBrokerConfig(None,
+      propsWith((DynamicConfig.Broker.MaintenanceBrokerListProp, propstring)))
+
+    val controllerId = TestUtils.waitUntilControllerElected(zkClient)
+
+    TestUtils.waitUntilTrue(() => brokers(controllerId).config.getMaintenanceBrokerList == brokerIds,
+      s"wait until broker $propstring is masked as maintenance broker not taking new partitions", 5000)
+
+  }
+
+}

From f128b3db14663aaaa524f2aa34b0822385c8c865 Mon Sep 17 00:00:00 2001
From: Xiongqi Wu 
Date: Sat, 12 Oct 2019 10:34:13 -0700
Subject: [PATCH 1037/1071] [LI-HOTFIX] exclude stream artifacts and connect
 artifacts from travis ci publication (#56)

TICKET =
LI_DESCRIPTION =
Excluding stream artifacts and connect artifacts from travis ci publication to make travis ci
publication less flaky.
This patch also excludes one flaky unit test.

EXIT_CRITERIA = MANUAL [""]
---
 .travis.yml                                                     | 2 +-
 .../java/org/apache/kafka/common/memory/WeakMemoryPoolTest.java | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index e5695ba13e0b0..f31c79622f72d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -43,7 +43,7 @@ cache:
 # This will triger publishing artifacts to bintray upon the creation of a new tag in the "x.y.z.w" format.
 deploy:
   provider: script
-  script: ./gradlew -Pversion=$TRAVIS_TAG uploadArchivesAll
+  script: ./gradlew -Pversion=0.0.0.8 uploadArchivesAll -x :connect:api:compileJava -x :connect:api:processResources -x :connect:api:classes -x :connect:api:copyDependantLibs -x :connect:api:jar -x :connect:json:compileJava -x :connect:json:processResources -x :connect:json:classes -x :connect:json:copyDependantLibs -x :connect:json:jar -x :connect:api:javadoc -x :connect:api:javadocJar -x :connect:api:srcJar -x :connect:api:compileTestJava -x :connect:api:processTestResources -x :connect:api:testClasses -x :connect:api:testJar -x :connect:api:testSrcJar -x :connect:api:signArchives -x :connect:api:uploadArchives -x :connect:basic-auth-extension:compileJava -x :connect:basic-auth-extension:processResources -x :connect:basic-auth-extension:classes -x :connect:basic-auth-extension:copyDependantLibs -x :connect:basic-auth-extension:jar -x :connect:basic-auth-extension:javadoc -x :connect:basic-auth-extension:javadocJar -x :connect:basic-auth-extension:srcJar -x :connect:basic-auth-extension:compileTestJava -x :connect:basic-auth-extension:processTestResources -x :connect:basic-auth-extension:testClasses -x :connect:basic-auth-extension:testJar -x :connect:basic-auth-extension:testSrcJar -x :connect:basic-auth-extension:signArchives -x :connect:basic-auth-extension:uploadArchives -x :connect:file:compileJava -x :connect:file:processResources -x :connect:file:classes -x :connect:file:copyDependantLibs -x :connect:file:jar -x :connect:file:javadoc -x :connect:file:javadocJar -x :connect:file:srcJar -x :connect:file:compileTestJava -x :connect:file:processTestResources -x :connect:file:testClasses -x :connect:file:testJar -x :connect:file:testSrcJar -x :connect:file:signArchives -x :connect:file:uploadArchives -x :connect:json:javadoc -x :connect:json:javadocJar -x :connect:json:srcJar -x :connect:json:compileTestJava -x :connect:json:processTestResources -x :connect:json:testClasses -x :connect:json:testJar -x :connect:json:testSrcJar -x :connect:json:signArchives -x :connect:json:uploadArchives -x :connect:transforms:compileJava -x :connect:transforms:processResources -x :connect:transforms:classes -x :connect:transforms:copyDependantLibs -x :connect:transforms:jar -x :connect:runtime:compileJava -x :connect:runtime:processResources -x :connect:runtime:classes -x :connect:runtime:copyDependantLibs -x :connect:runtime:jar -x :connect:runtime:javadoc -x :connect:runtime:javadocJar -x :connect:runtime:srcJar -x :connect:runtime:compileTestJava -x :connect:runtime:processTestResources -x :connect:runtime:testClasses -x :connect:runtime:testJar -x :connect:runtime:testSrcJar -x :connect:runtime:signArchives -x :connect:runtime:uploadArchives -x :connect:transforms:javadoc -x :connect:transforms:javadocJar -x :connect:transforms:srcJar -x :connect:transforms:compileTestJava -x :connect:transforms:processTestResources -x :connect:transforms:testClasses -x :connect:transforms:testJar -x :connect:transforms:testSrcJar -x :connect:transforms:signArchives -x :connect:transforms:uploadArchives -x :streams:compileJava -x :streams:processResources -x :streams:classes -x :streams:copyDependantLibs -x :streams:jar -x :streams:javadoc -x :streams:javadocJar -x :streams:srcJar -x :streams:test-utils:compileJava -x :streams:test-utils:processResources -x :streams:test-utils:classes -x :streams:test-utils:copyDependantLibs -x :streams:test-utils:jar -x :streams:compileTestJava -x :streams:processTestResources -x :streams:testClasses -x :streams:testJar -x :streams:testSrcJar -x :streams:signArchives -x :streams:uploadArchives -x :streams:examples:compileJava -x :streams:examples:processResources -x :streams:examples:classes -x :streams:examples:copyDependantLibs -x :streams:examples:jar -x :streams:examples:javadoc -x :streams:examples:javadocJar -x :streams:examples:srcJar -x :streams:examples:compileTestJava -x :streams:examples:processTestResources -x :streams:examples:testClasses -x :streams:examples:testJar -x :streams:examples:testSrcJar -x :streams:examples:signArchives -x :streams:examples:uploadArchives -x :streams:test-utils:javadoc -x :streams:test-utils:javadocJar -x :streams:test-utils:srcJar -x :streams:test-utils:compileTestJava -x :streams:test-utils:processTestResources -x :streams:test-utils:testClasses -x :streams:test-utils:testJar -x :streams:test-utils:testSrcJar -x :streams:test-utils:signArchives -x :streams:test-utils:uploadArchives -x :streams:streams-scala:compileJava -x :streams:streams-scala:compileScala -x :streams:streams-scala:uploadArchives -x :streams:streams-scala:jar -x :streams:streams-scala:test -x :streams:streams-scala:srcJar -x :streams:streams-scala:docsJar -x :core:compileTestJava -x :core:compileTestScala -x :core:processTestResources -x :core:testClasses --no-daemon
   skip_cleanup: true
   on:
     tags: true
diff --git a/clients/src/test/java/org/apache/kafka/common/memory/WeakMemoryPoolTest.java b/clients/src/test/java/org/apache/kafka/common/memory/WeakMemoryPoolTest.java
index 57be288e98c7c..e1f89ea8f024e 100644
--- a/clients/src/test/java/org/apache/kafka/common/memory/WeakMemoryPoolTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/memory/WeakMemoryPoolTest.java
@@ -19,6 +19,7 @@
 
 import java.nio.ByteBuffer;
 import org.junit.Assert;
+import org.junit.Ignore;
 import org.junit.Test;
 
 
@@ -43,6 +44,7 @@ public void testNullRelease() {
         memoryPool.release(null);
     }
 
+    @Ignore("flaky test")
     @Test
     public void testAllocationMemorySize() {
         WeakMemoryPool pool = new WeakMemoryPool();

From d00ac0ec3f1928841d46d78bea99925d47eccc0e Mon Sep 17 00:00:00 2001
From: Xiongqi Wu 
Date: Sat, 12 Oct 2019 21:36:37 -0700
Subject: [PATCH 1038/1071] [LI-HOTFIX] fix hard coded travis tag (#57)

TICKET =
LI_DESCRIPTION =

EXIT_CRITERIA = MANUAL [" when this is combined with other travis ci fixes "]
---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index f31c79622f72d..c3999f1cb8daa 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -43,7 +43,7 @@ cache:
 # This will triger publishing artifacts to bintray upon the creation of a new tag in the "x.y.z.w" format.
 deploy:
   provider: script
-  script: ./gradlew -Pversion=0.0.0.8 uploadArchivesAll -x :connect:api:compileJava -x :connect:api:processResources -x :connect:api:classes -x :connect:api:copyDependantLibs -x :connect:api:jar -x :connect:json:compileJava -x :connect:json:processResources -x :connect:json:classes -x :connect:json:copyDependantLibs -x :connect:json:jar -x :connect:api:javadoc -x :connect:api:javadocJar -x :connect:api:srcJar -x :connect:api:compileTestJava -x :connect:api:processTestResources -x :connect:api:testClasses -x :connect:api:testJar -x :connect:api:testSrcJar -x :connect:api:signArchives -x :connect:api:uploadArchives -x :connect:basic-auth-extension:compileJava -x :connect:basic-auth-extension:processResources -x :connect:basic-auth-extension:classes -x :connect:basic-auth-extension:copyDependantLibs -x :connect:basic-auth-extension:jar -x :connect:basic-auth-extension:javadoc -x :connect:basic-auth-extension:javadocJar -x :connect:basic-auth-extension:srcJar -x :connect:basic-auth-extension:compileTestJava -x :connect:basic-auth-extension:processTestResources -x :connect:basic-auth-extension:testClasses -x :connect:basic-auth-extension:testJar -x :connect:basic-auth-extension:testSrcJar -x :connect:basic-auth-extension:signArchives -x :connect:basic-auth-extension:uploadArchives -x :connect:file:compileJava -x :connect:file:processResources -x :connect:file:classes -x :connect:file:copyDependantLibs -x :connect:file:jar -x :connect:file:javadoc -x :connect:file:javadocJar -x :connect:file:srcJar -x :connect:file:compileTestJava -x :connect:file:processTestResources -x :connect:file:testClasses -x :connect:file:testJar -x :connect:file:testSrcJar -x :connect:file:signArchives -x :connect:file:uploadArchives -x :connect:json:javadoc -x :connect:json:javadocJar -x :connect:json:srcJar -x :connect:json:compileTestJava -x :connect:json:processTestResources -x :connect:json:testClasses -x :connect:json:testJar -x :connect:json:testSrcJar -x :connect:json:signArchives -x :connect:json:uploadArchives -x :connect:transforms:compileJava -x :connect:transforms:processResources -x :connect:transforms:classes -x :connect:transforms:copyDependantLibs -x :connect:transforms:jar -x :connect:runtime:compileJava -x :connect:runtime:processResources -x :connect:runtime:classes -x :connect:runtime:copyDependantLibs -x :connect:runtime:jar -x :connect:runtime:javadoc -x :connect:runtime:javadocJar -x :connect:runtime:srcJar -x :connect:runtime:compileTestJava -x :connect:runtime:processTestResources -x :connect:runtime:testClasses -x :connect:runtime:testJar -x :connect:runtime:testSrcJar -x :connect:runtime:signArchives -x :connect:runtime:uploadArchives -x :connect:transforms:javadoc -x :connect:transforms:javadocJar -x :connect:transforms:srcJar -x :connect:transforms:compileTestJava -x :connect:transforms:processTestResources -x :connect:transforms:testClasses -x :connect:transforms:testJar -x :connect:transforms:testSrcJar -x :connect:transforms:signArchives -x :connect:transforms:uploadArchives -x :streams:compileJava -x :streams:processResources -x :streams:classes -x :streams:copyDependantLibs -x :streams:jar -x :streams:javadoc -x :streams:javadocJar -x :streams:srcJar -x :streams:test-utils:compileJava -x :streams:test-utils:processResources -x :streams:test-utils:classes -x :streams:test-utils:copyDependantLibs -x :streams:test-utils:jar -x :streams:compileTestJava -x :streams:processTestResources -x :streams:testClasses -x :streams:testJar -x :streams:testSrcJar -x :streams:signArchives -x :streams:uploadArchives -x :streams:examples:compileJava -x :streams:examples:processResources -x :streams:examples:classes -x :streams:examples:copyDependantLibs -x :streams:examples:jar -x :streams:examples:javadoc -x :streams:examples:javadocJar -x :streams:examples:srcJar -x :streams:examples:compileTestJava -x :streams:examples:processTestResources -x :streams:examples:testClasses -x :streams:examples:testJar -x :streams:examples:testSrcJar -x :streams:examples:signArchives -x :streams:examples:uploadArchives -x :streams:test-utils:javadoc -x :streams:test-utils:javadocJar -x :streams:test-utils:srcJar -x :streams:test-utils:compileTestJava -x :streams:test-utils:processTestResources -x :streams:test-utils:testClasses -x :streams:test-utils:testJar -x :streams:test-utils:testSrcJar -x :streams:test-utils:signArchives -x :streams:test-utils:uploadArchives -x :streams:streams-scala:compileJava -x :streams:streams-scala:compileScala -x :streams:streams-scala:uploadArchives -x :streams:streams-scala:jar -x :streams:streams-scala:test -x :streams:streams-scala:srcJar -x :streams:streams-scala:docsJar -x :core:compileTestJava -x :core:compileTestScala -x :core:processTestResources -x :core:testClasses --no-daemon
+  script: ./gradlew -Pversion=$TRAVIS_TAG uploadArchivesAll -x :connect:api:compileJava -x :connect:api:processResources -x :connect:api:classes -x :connect:api:copyDependantLibs -x :connect:api:jar -x :connect:json:compileJava -x :connect:json:processResources -x :connect:json:classes -x :connect:json:copyDependantLibs -x :connect:json:jar -x :connect:api:javadoc -x :connect:api:javadocJar -x :connect:api:srcJar -x :connect:api:compileTestJava -x :connect:api:processTestResources -x :connect:api:testClasses -x :connect:api:testJar -x :connect:api:testSrcJar -x :connect:api:signArchives -x :connect:api:uploadArchives -x :connect:basic-auth-extension:compileJava -x :connect:basic-auth-extension:processResources -x :connect:basic-auth-extension:classes -x :connect:basic-auth-extension:copyDependantLibs -x :connect:basic-auth-extension:jar -x :connect:basic-auth-extension:javadoc -x :connect:basic-auth-extension:javadocJar -x :connect:basic-auth-extension:srcJar -x :connect:basic-auth-extension:compileTestJava -x :connect:basic-auth-extension:processTestResources -x :connect:basic-auth-extension:testClasses -x :connect:basic-auth-extension:testJar -x :connect:basic-auth-extension:testSrcJar -x :connect:basic-auth-extension:signArchives -x :connect:basic-auth-extension:uploadArchives -x :connect:file:compileJava -x :connect:file:processResources -x :connect:file:classes -x :connect:file:copyDependantLibs -x :connect:file:jar -x :connect:file:javadoc -x :connect:file:javadocJar -x :connect:file:srcJar -x :connect:file:compileTestJava -x :connect:file:processTestResources -x :connect:file:testClasses -x :connect:file:testJar -x :connect:file:testSrcJar -x :connect:file:signArchives -x :connect:file:uploadArchives -x :connect:json:javadoc -x :connect:json:javadocJar -x :connect:json:srcJar -x :connect:json:compileTestJava -x :connect:json:processTestResources -x :connect:json:testClasses -x :connect:json:testJar -x :connect:json:testSrcJar -x :connect:json:signArchives -x :connect:json:uploadArchives -x :connect:transforms:compileJava -x :connect:transforms:processResources -x :connect:transforms:classes -x :connect:transforms:copyDependantLibs -x :connect:transforms:jar -x :connect:runtime:compileJava -x :connect:runtime:processResources -x :connect:runtime:classes -x :connect:runtime:copyDependantLibs -x :connect:runtime:jar -x :connect:runtime:javadoc -x :connect:runtime:javadocJar -x :connect:runtime:srcJar -x :connect:runtime:compileTestJava -x :connect:runtime:processTestResources -x :connect:runtime:testClasses -x :connect:runtime:testJar -x :connect:runtime:testSrcJar -x :connect:runtime:signArchives -x :connect:runtime:uploadArchives -x :connect:transforms:javadoc -x :connect:transforms:javadocJar -x :connect:transforms:srcJar -x :connect:transforms:compileTestJava -x :connect:transforms:processTestResources -x :connect:transforms:testClasses -x :connect:transforms:testJar -x :connect:transforms:testSrcJar -x :connect:transforms:signArchives -x :connect:transforms:uploadArchives -x :streams:compileJava -x :streams:processResources -x :streams:classes -x :streams:copyDependantLibs -x :streams:jar -x :streams:javadoc -x :streams:javadocJar -x :streams:srcJar -x :streams:test-utils:compileJava -x :streams:test-utils:processResources -x :streams:test-utils:classes -x :streams:test-utils:copyDependantLibs -x :streams:test-utils:jar -x :streams:compileTestJava -x :streams:processTestResources -x :streams:testClasses -x :streams:testJar -x :streams:testSrcJar -x :streams:signArchives -x :streams:uploadArchives -x :streams:examples:compileJava -x :streams:examples:processResources -x :streams:examples:classes -x :streams:examples:copyDependantLibs -x :streams:examples:jar -x :streams:examples:javadoc -x :streams:examples:javadocJar -x :streams:examples:srcJar -x :streams:examples:compileTestJava -x :streams:examples:processTestResources -x :streams:examples:testClasses -x :streams:examples:testJar -x :streams:examples:testSrcJar -x :streams:examples:signArchives -x :streams:examples:uploadArchives -x :streams:test-utils:javadoc -x :streams:test-utils:javadocJar -x :streams:test-utils:srcJar -x :streams:test-utils:compileTestJava -x :streams:test-utils:processTestResources -x :streams:test-utils:testClasses -x :streams:test-utils:testJar -x :streams:test-utils:testSrcJar -x :streams:test-utils:signArchives -x :streams:test-utils:uploadArchives -x :streams:streams-scala:compileJava -x :streams:streams-scala:compileScala -x :streams:streams-scala:uploadArchives -x :streams:streams-scala:jar -x :streams:streams-scala:test -x :streams:streams-scala:srcJar -x :streams:streams-scala:docsJar -x :core:compileTestJava -x :core:compileTestScala -x :core:processTestResources -x :core:testClasses --no-daemon
   skip_cleanup: true
   on:
     tags: true

From 10061b71267466d7e9810707e86119644482e028 Mon Sep 17 00:00:00 2001
From: Xiongqi Wu 
Date: Tue, 15 Oct 2019 11:51:28 -0700
Subject: [PATCH 1039/1071] [LI-HOTFIX] Do not throw exceptions when internal
 headers are set and the coordinated channel with the broker is using
 Message_v1(magic = 1) (#58)

TICKET =
LI_DESCRIPTION =
include record header key as part of exception.
EXIT_CRITERIA = MANUAL [""]
---
 .../common/record/MemoryRecordsBuilder.java   | 18 ++++-
 .../internals/RecordAccumulatorTest.java      | 37 +++++++++
 .../kafka/api/BaseProducerSendTest.scala      |  4 +-
 .../kafka/api/PlaintextProducerSendTest.scala |  1 -
 .../api/RecordHeaderProducerSendTest.scala    | 76 +++++++++++++++++++
 5 files changed, 132 insertions(+), 4 deletions(-)
 create mode 100644 core/src/test/scala/integration/kafka/api/RecordHeaderProducerSendTest.scala

diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java
index 95ee13ea1c53c..4319d199028e4 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java
@@ -39,6 +39,9 @@
  * This will release resources like compression buffers that can be relatively large (64 KB for LZ4).
  */
 public class MemoryRecordsBuilder implements AutoCloseable {
+    // TODO(viswamy): Revert this change once all brokers are on Message_v2
+    private static final String INTERNAL_HEADER_PREFIX = "_";
+
     private static final float COMPRESSION_RATE_ESTIMATION_FACTOR = 1.05f;
     private static final DataOutputStream CLOSED_STREAM = new DataOutputStream(new OutputStream() {
         @Override
@@ -403,6 +406,18 @@ private int writeLegacyCompressedWrapperHeader() {
         return writtenCompressed;
     }
 
+    // Validates and throws an exception if the headers contain non-internal record headers
+    private void validateHeaders(Header[] headers) {
+        if (magic < RecordBatch.MAGIC_VALUE_V2 && headers != null && headers.length > 0) {
+            for (Header header : headers) {
+                if (!header.key().startsWith(INTERNAL_HEADER_PREFIX)) {
+                    throw new IllegalArgumentException("Magic v" + magic + " does not support record headers. [Key = "
+                        + header.key() + "]");
+                }
+            }
+        }
+    }
+
     /**
      * Append a record and return its checksum for message format v0 and v1, or null for v2 and above.
      */
@@ -419,8 +434,7 @@ private Long appendWithOffset(long offset, boolean isControlRecord, long timesta
             if (timestamp < 0 && timestamp != RecordBatch.NO_TIMESTAMP)
                 throw new IllegalArgumentException("Invalid negative timestamp " + timestamp);
 
-            if (magic < RecordBatch.MAGIC_VALUE_V2 && headers != null && headers.length > 0)
-                throw new IllegalArgumentException("Magic v" + magic + " does not support record headers");
+            validateHeaders(headers);
 
             if (firstTimestamp == null)
                 firstTimestamp = timestamp;
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java
index 1d76344c159a2..9b01d6bc7cdaa 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java
@@ -30,6 +30,8 @@
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.errors.TimeoutException;
 import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.header.internals.RecordHeader;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.record.CompressionRatioEstimator;
@@ -780,6 +782,41 @@ CompressionType.NONE, lingerMs, retryBackoffMs, deliveryTimeoutMs, metrics, metr
         accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, 0, false);
     }
 
+    @Test
+    public void testRecordHeadersWithMagicV1() {
+        ByteBuffer buffer = ByteBuffer.allocate(4096);
+        MemoryRecordsBuilder builder = MemoryRecords.builder(
+            buffer,
+            (byte) 1, // Enforces that Message format v1 is used on the client
+            CompressionType.NONE,
+            TimestampType.CREATE_TIME,
+            0L
+        );
+        long now = System.currentTimeMillis();
+        ProducerBatch batch = new ProducerBatch(tp1, builder, now, true);
+        try {
+            batch.tryAppend(
+                1L,
+                new byte[10],
+                new byte[20],
+                new Header[] {new RecordHeader("key", new byte[20])},
+                null,
+                now
+            );
+            throw new IllegalStateException("The append of a record with non internal headers should have failed");
+        } catch (IllegalArgumentException exception) {
+            assertEquals(exception.getMessage(), "Magic v1 does not support record headers. [Key = key]");
+        }
+        batch.tryAppend(
+                1L,
+                new byte[10],
+                new byte[20],
+                new Header[] {new RecordHeader("_key", new byte[20])},
+                null,
+                now
+        );
+    }
+
     @Test
     public void testSplitAndReenqueue() throws ExecutionException, InterruptedException {
         long now = time.milliseconds();
diff --git a/core/src/test/scala/integration/kafka/api/BaseProducerSendTest.scala b/core/src/test/scala/integration/kafka/api/BaseProducerSendTest.scala
index c4979ec6a5d8b..b318f8bef88ae 100644
--- a/core/src/test/scala/integration/kafka/api/BaseProducerSendTest.scala
+++ b/core/src/test/scala/integration/kafka/api/BaseProducerSendTest.scala
@@ -42,8 +42,10 @@ import scala.concurrent.ExecutionException
 
 abstract class BaseProducerSendTest extends KafkaServerTestHarness {
 
+  def overrideConfigs() : Properties = {new Properties()}
+
   def generateConfigs = {
-    val overridingProps = new Properties()
+    val overridingProps = overrideConfigs()
     val numServers = 2
     overridingProps.put(KafkaConfig.NumPartitionsProp, 4.toString)
     TestUtils.createBrokerConfigs(numServers, zkConnect, false, interBrokerSecurityProtocol = Some(securityProtocol),
diff --git a/core/src/test/scala/integration/kafka/api/PlaintextProducerSendTest.scala b/core/src/test/scala/integration/kafka/api/PlaintextProducerSendTest.scala
index 23824a3cfe3ed..717154c7d2749 100644
--- a/core/src/test/scala/integration/kafka/api/PlaintextProducerSendTest.scala
+++ b/core/src/test/scala/integration/kafka/api/PlaintextProducerSendTest.scala
@@ -114,5 +114,4 @@ class PlaintextProducerSendTest extends BaseProducerSendTest {
       compressedProducer.close()
     }
   }
-
 }
diff --git a/core/src/test/scala/integration/kafka/api/RecordHeaderProducerSendTest.scala b/core/src/test/scala/integration/kafka/api/RecordHeaderProducerSendTest.scala
new file mode 100644
index 0000000000000..faeca6317cd5d
--- /dev/null
+++ b/core/src/test/scala/integration/kafka/api/RecordHeaderProducerSendTest.scala
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package integration.kafka.api
+
+import java.util.Properties
+
+import kafka.api.BaseProducerSendTest
+import kafka.server.KafkaConfig
+import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig, ProducerRecord}
+import org.apache.kafka.common.header.internals.RecordHeaders
+import org.junit.Test
+
+class RecordHeaderProducerSendTest extends BaseProducerSendTest {
+
+  @Test
+  def testRecordHeaders(): Unit = {
+    val producerProps = new Properties()
+    producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList)
+    producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
+    producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
+    producerProps.put(ProducerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, "true")
+    val producer = new KafkaProducer[String, String](producerProps)
+    var invalidRecordHeaders = new RecordHeaders()
+    invalidRecordHeaders.add("RecordHeaderKey", "RecordHeaderValue".getBytes)
+    // The record is invalid because it contain header with keys that doesn't start with a "_"
+    // Keys that do start with a "_" are internal to the clients and are dropped at the producer.
+    val invalidRecord = new ProducerRecord[String, String](
+      topic,
+      new Integer(0),
+      "RecordKey",
+      "RecordValue",
+      invalidRecordHeaders
+    )
+    try {
+      producer.send(invalidRecord).get
+      throw new IllegalStateException("The invalid record should have thrown an exception")
+    } catch {
+      // Ignore the exception because a non internal header was introduced into the producer record
+      case ignored: IllegalArgumentException =>
+    }
+
+    val validRecordHeaders = new RecordHeaders()
+    validRecordHeaders.add("_RecordHeaderKey", "RecordHeaderValue".getBytes)
+    val record = new ProducerRecord[String, String](
+      topic,
+      new Integer(0),
+      "RecordKey",
+      "RecordValue",
+      validRecordHeaders
+    )
+    producer.send(record).get
+    producer.close()
+  }
+
+  override def overrideConfigs(): Properties = {
+    val properties = new Properties()
+    properties.put(KafkaConfig.InterBrokerProtocolVersionProp, "0.10.2")
+    properties.put(KafkaConfig.LogMessageFormatVersionProp, "0.10.2")
+    properties
+  }
+}

From da0cee0835864b1c8516a6bba37d24465d9dba3a Mon Sep 17 00:00:00 2001
From: Virupaksha Swamy Uttanur Matada 
Date: Thu, 31 Oct 2019 13:48:45 -0700
Subject: [PATCH 1040/1071] [LI-HOTFIX] Add counters for the following: (#59)
 TICKET = LI_DESCRIPTION = 1. Total number of incoming messages 2. Total
 number of incoming bytes EXIT_CRITERIA = MANUAL [""]

---
 .../kafka/metrics/KafkaMetricsGroup.scala     |  3 ++
 .../kafka/server/KafkaRequestHandler.scala    | 50 ++++++++++++++++++-
 .../scala/kafka/server/ReplicaManager.scala   |  7 +++
 .../unit/kafka/metrics/MetricsTest.scala      |  3 +-
 4 files changed, 60 insertions(+), 3 deletions(-)

diff --git a/core/src/main/scala/kafka/metrics/KafkaMetricsGroup.scala b/core/src/main/scala/kafka/metrics/KafkaMetricsGroup.scala
index 03a4f7ccd4c03..a5f9911a2b077 100644
--- a/core/src/main/scala/kafka/metrics/KafkaMetricsGroup.scala
+++ b/core/src/main/scala/kafka/metrics/KafkaMetricsGroup.scala
@@ -71,6 +71,9 @@ trait KafkaMetricsGroup extends Logging {
   def newMeter(name: String, eventType: String, timeUnit: TimeUnit, tags: scala.collection.Map[String, String] = Map.empty) =
     Metrics.defaultRegistry().newMeter(metricName(name, tags), eventType, timeUnit)
 
+  def newCounter(name: String, tags: scala.collection.Map[String, String] = Map.empty) =
+    Metrics.defaultRegistry().newCounter(metricName(name, tags))
+
   def newHistogram(name: String, biased: Boolean = true, tags: scala.collection.Map[String, String] = Map.empty) =
     Metrics.defaultRegistry().newHistogram(metricName(name, tags), biased)
 
diff --git a/core/src/main/scala/kafka/server/KafkaRequestHandler.scala b/core/src/main/scala/kafka/server/KafkaRequestHandler.scala
index 639627fc022c9..d43b09256ab64 100755
--- a/core/src/main/scala/kafka/server/KafkaRequestHandler.scala
+++ b/core/src/main/scala/kafka/server/KafkaRequestHandler.scala
@@ -23,7 +23,7 @@ import kafka.metrics.KafkaMetricsGroup
 import java.util.concurrent.{CountDownLatch, TimeUnit}
 import java.util.concurrent.atomic.AtomicInteger
 
-import com.yammer.metrics.core.Meter
+import com.yammer.metrics.core.{Counter, Meter}
 import org.apache.kafka.common.internals.FatalExitError
 import org.apache.kafka.common.utils.{KafkaThread, Time}
 
@@ -176,6 +176,35 @@ class BrokerTopicMetrics(name: Option[String]) extends KafkaMetricsGroup {
       meter()
   }
 
+  case class CounterWrapper(metricType: String) {
+    @volatile private var lazyCounter: Counter = _
+    private val counterLock = new Object
+
+    def counter(): Counter = {
+      var counter = lazyCounter
+      if (counter == null) {
+        counterLock synchronized {
+          counter = lazyCounter
+          if (counter == null) {
+            counter = newCounter(metricType, tags)
+            lazyCounter = counter
+          }
+        }
+      }
+      counter
+    }
+
+    def close(): Unit = counterLock synchronized {
+      if (lazyCounter != null) {
+        removeMetric(metricType, tags)
+        lazyCounter = null
+      }
+    }
+
+    if (tags.isEmpty) // greedily initialize the general topic metrics
+    counter()
+  }
+
   // an internal map for "lazy initialization" of certain metrics
   private val metricTypeMap = new Pool[String, MeterWrapper]()
   metricTypeMap.putAll(Map(
@@ -199,13 +228,24 @@ class BrokerTopicMetrics(name: Option[String]) extends KafkaMetricsGroup {
     metricTypeMap.put(BrokerTopicStats.ReplicationBytesOutPerSec, MeterWrapper(BrokerTopicStats.ReplicationBytesOutPerSec, "bytes"))
   }
 
+  private val counterMetricTypeMap = new Pool[String, CounterWrapper]()
+  counterMetricTypeMap.putAll(Map(
+    BrokerTopicStats.MessagesInTotal-> CounterWrapper(BrokerTopicStats.MessagesInTotal),
+    BrokerTopicStats.BytesInTotal -> CounterWrapper(BrokerTopicStats.BytesInTotal)
+  ).asJava)
   // used for testing only
   def metricMap: Map[String, MeterWrapper] = metricTypeMap.toMap
 
+  def counterMetricMap: Map[String, CounterWrapper] = counterMetricTypeMap.toMap
+
   def messagesInRate = metricTypeMap.get(BrokerTopicStats.MessagesInPerSec).meter()
 
+  def messagesInTotal = counterMetricTypeMap.get(BrokerTopicStats.MessagesInTotal).counter()
+
   def bytesInRate = metricTypeMap.get(BrokerTopicStats.BytesInPerSec).meter()
 
+  def bytesInTotal = counterMetricTypeMap.get(BrokerTopicStats.BytesInTotal).counter()
+
   def bytesOutRate = metricTypeMap.get(BrokerTopicStats.BytesOutPerSec).meter()
 
   def bytesRejectedRate = metricTypeMap.get(BrokerTopicStats.BytesRejectedPerSec).meter()
@@ -244,12 +284,18 @@ class BrokerTopicMetrics(name: Option[String]) extends KafkaMetricsGroup {
       meter.close()
   }
 
-  def close(): Unit = metricTypeMap.values.foreach(_.close())
+  def close(): Unit = {
+    metricTypeMap.values.foreach(_.close())
+    removeMetric(BrokerTopicStats.MessagesInTotal, tags)
+    removeMetric(BrokerTopicStats.BytesInTotal, tags)
+  }
 }
 
 object BrokerTopicStats {
   val MessagesInPerSec = "MessagesInPerSec"
+  val MessagesInTotal = "MessagesInTotal"
   val BytesInPerSec = "BytesInPerSec"
+  val BytesInTotal = "BytesInTotal"
   val BytesOutPerSec = "BytesOutPerSec"
   val BytesRejectedPerSec = "BytesRejectedPerSec"
   val ReplicationBytesInPerSec = "ReplicationBytesInPerSec"
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index 0c1b97c72aaeb..e82485a56ada5 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -817,9 +817,16 @@ class ReplicaManager(val config: KafkaConfig,
 
           // update stats for successfully appended bytes and messages as bytesInRate and messageInRate
           brokerTopicStats.topicStats(topicPartition.topic).bytesInRate.mark(records.sizeInBytes)
+          brokerTopicStats.topicStats(topicPartition.topic).bytesInTotal.inc(records.sizeInBytes)
+
           brokerTopicStats.allTopicsStats.bytesInRate.mark(records.sizeInBytes)
+          brokerTopicStats.allTopicsStats.bytesInTotal.inc(records.sizeInBytes)
+
           brokerTopicStats.topicStats(topicPartition.topic).messagesInRate.mark(numAppendedMessages)
+          brokerTopicStats.topicStats(topicPartition.topic).messagesInTotal.inc(numAppendedMessages)
+
           brokerTopicStats.allTopicsStats.messagesInRate.mark(numAppendedMessages)
+          brokerTopicStats.allTopicsStats.messagesInTotal.inc(numAppendedMessages)
 
           trace(s"${records.sizeInBytes} written to log $topicPartition beginning at offset " +
             s"${info.firstOffset.getOrElse(-1)} and ending at offset ${info.lastOffset}")
diff --git a/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala b/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
index 587bf5e71e3ac..45d23c2a299c1 100644
--- a/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
+++ b/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
@@ -82,7 +82,8 @@ class MetricsTest extends KafkaServerTestHarness with Logging {
 
     // The broker metrics for all topics should be greedily registered
     assertTrue("General topic metrics don't exist", topicMetrics(None).nonEmpty)
-    assertEquals(servers.head.brokerTopicStats.allTopicsStats.metricMap.size, topicMetrics(None).size)
+    assertEquals(servers.head.brokerTopicStats.allTopicsStats.metricMap.size +
+      servers.head.brokerTopicStats.allTopicsStats.counterMetricMap.size, topicMetrics(None).size)
     // topic metrics should be lazily registered
     assertTrue("Topic metrics aren't lazily registered", topicMetricGroups(topic).isEmpty)
     TestUtils.generateAndProduceMessages(servers, topic, nMessages)

From eccd706a758d151ad5a497c7f0f133bc5b440969 Mon Sep 17 00:00:00 2001
From: kun du 
Date: Wed, 4 Sep 2019 14:17:58 -0700
Subject: [PATCH 1041/1071] [LI-HOTFIX] Add sensor to collect statistics of
 KafkaChannel memory allocation size.

TICKET =
LI_DESCRIPTION =
Add a percentile metrics to collect memory allocation size distribution, it helps
to determine if a smarter buffer pool make sense or not.
EXIT_CRITERIA = MANUAL [""]
---
 .../memory/GarbageCollectedMemoryPool.java      |  2 +-
 .../kafka/common/memory/SimpleMemoryPool.java   |  5 ++++-
 .../kafka/common/network/SelectorTest.java      |  6 +++++-
 .../kafka/common/network/SslSelectorTest.java   |  5 ++++-
 .../main/scala/kafka/network/SocketServer.scala | 17 +++++++++++++----
 5 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/memory/GarbageCollectedMemoryPool.java b/clients/src/main/java/org/apache/kafka/common/memory/GarbageCollectedMemoryPool.java
index 18f8ffe91c66e..bda419eddca5a 100644
--- a/clients/src/main/java/org/apache/kafka/common/memory/GarbageCollectedMemoryPool.java
+++ b/clients/src/main/java/org/apache/kafka/common/memory/GarbageCollectedMemoryPool.java
@@ -42,7 +42,7 @@ public class GarbageCollectedMemoryPool extends SimpleMemoryPool implements Auto
     private volatile boolean alive = true;
 
     public GarbageCollectedMemoryPool(long sizeBytes, int maxSingleAllocationSize, boolean strict, Sensor oomPeriodSensor) {
-        super(sizeBytes, maxSingleAllocationSize, strict, oomPeriodSensor);
+        super(sizeBytes, maxSingleAllocationSize, strict, oomPeriodSensor, null);
         this.alive = true;
         this.gcListenerThread = new Thread(gcListener, "memory pool GC listener");
         this.gcListenerThread.setDaemon(true); //so we dont need to worry about shutdown
diff --git a/clients/src/main/java/org/apache/kafka/common/memory/SimpleMemoryPool.java b/clients/src/main/java/org/apache/kafka/common/memory/SimpleMemoryPool.java
index f1ab8f71854e7..f6f20808cfffb 100644
--- a/clients/src/main/java/org/apache/kafka/common/memory/SimpleMemoryPool.java
+++ b/clients/src/main/java/org/apache/kafka/common/memory/SimpleMemoryPool.java
@@ -38,8 +38,9 @@ public class SimpleMemoryPool implements MemoryPool {
     protected final int maxSingleAllocationSize;
     protected final AtomicLong startOfNoMemPeriod = new AtomicLong(); //nanoseconds
     protected volatile Sensor oomTimeSensor;
+    protected volatile Sensor allocateSensor;
 
-    public SimpleMemoryPool(long sizeInBytes, int maxSingleAllocationBytes, boolean strict, Sensor oomPeriodSensor) {
+    public SimpleMemoryPool(long sizeInBytes, int maxSingleAllocationBytes, boolean strict, Sensor oomPeriodSensor, Sensor allocateSensor) {
         if (sizeInBytes <= 0 || maxSingleAllocationBytes <= 0 || maxSingleAllocationBytes > sizeInBytes)
             throw new IllegalArgumentException("must provide a positive size and max single allocation size smaller than size."
                 + "provided " + sizeInBytes + " and " + maxSingleAllocationBytes + " respectively");
@@ -48,6 +49,7 @@ public SimpleMemoryPool(long sizeInBytes, int maxSingleAllocationBytes, boolean
         this.availableMemory = new AtomicLong(sizeInBytes);
         this.maxSingleAllocationSize = maxSingleAllocationBytes;
         this.oomTimeSensor = oomPeriodSensor;
+        this.allocateSensor = allocateSensor;
     }
 
     @Override
@@ -111,6 +113,7 @@ public boolean isOutOfMemory() {
 
     //allows subclasses to do their own bookkeeping (and validation) _before_ memory is returned to client code.
     protected void bufferToBeReturned(ByteBuffer justAllocated) {
+        this.allocateSensor.record(justAllocated.capacity());
         log.trace("allocated buffer of size {} ", justAllocated.capacity());
     }
 
diff --git a/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java b/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java
index ae06836986f64..952d1ceebd9af 100644
--- a/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java
@@ -21,7 +21,9 @@
 import org.apache.kafka.common.memory.MemoryPool;
 import org.apache.kafka.common.memory.SimpleMemoryPool;
 import org.apache.kafka.common.metrics.KafkaMetric;
+import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
+import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.security.auth.SecurityProtocol;
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.common.utils.MockTime;
@@ -79,6 +81,7 @@ public class SelectorTest {
     protected Selector selector;
     protected ChannelBuilder channelBuilder;
     protected Metrics metrics;
+    protected Sensor sensor;
 
     @Before
     public void setUp() throws Exception {
@@ -90,6 +93,7 @@ public void setUp() throws Exception {
         this.channelBuilder.configure(configs);
         this.metrics = new Metrics();
         this.selector = new Selector(5000, this.metrics, time, "MetricGroup", channelBuilder, new LogContext());
+        this.sensor = metrics.sensor("infoSensor", new MetricConfig().recordLevel(Sensor.RecordingLevel.INFO), Sensor.RecordingLevel.INFO);
     }
 
     @After
@@ -534,7 +538,7 @@ private void verifyCloseOldestConnectionWithStagedReceives(int maxStagedReceives
     public void testMuteOnOOM() throws Exception {
         //clean up default selector, replace it with one that uses a finite mem pool
         selector.close();
-        MemoryPool pool = new SimpleMemoryPool(900, 900, false, null);
+        MemoryPool pool = new SimpleMemoryPool(900, 900, false, null, sensor);
         selector = new Selector(NetworkReceive.UNLIMITED, 5000, metrics, time, "MetricGroup",
             new HashMap(), true, false, channelBuilder, pool, new LogContext());
 
diff --git a/clients/src/test/java/org/apache/kafka/common/network/SslSelectorTest.java b/clients/src/test/java/org/apache/kafka/common/network/SslSelectorTest.java
index 19e7a8bf78af0..273d7a531baa4 100644
--- a/clients/src/test/java/org/apache/kafka/common/network/SslSelectorTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/network/SslSelectorTest.java
@@ -22,7 +22,9 @@
 import org.apache.kafka.common.config.SecurityConfig;
 import org.apache.kafka.common.memory.MemoryPool;
 import org.apache.kafka.common.memory.SimpleMemoryPool;
+import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
+import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.security.auth.SecurityProtocol;
 import org.apache.kafka.common.security.ssl.SslFactory;
 import org.apache.kafka.common.security.ssl.mock.TestKeyManagerFactory;
@@ -78,6 +80,7 @@ public void setUp() throws Exception {
         this.channelBuilder.configure(sslClientConfigs);
         this.metrics = new Metrics();
         this.selector = new Selector(5000, metrics, time, "MetricGroup", channelBuilder, new LogContext());
+        this.sensor = metrics.sensor("infoSensor", new MetricConfig().recordLevel(Sensor.RecordingLevel.INFO), Sensor.RecordingLevel.INFO);
     }
 
     @After
@@ -270,7 +273,7 @@ public void testRenegotiationFails() throws Exception {
     public void testMuteOnOOM() throws Exception {
         //clean up default selector, replace it with one that uses a finite mem pool
         selector.close();
-        MemoryPool pool = new SimpleMemoryPool(900, 900, false, null);
+        MemoryPool pool = new SimpleMemoryPool(900, 900, false, null, sensor);
         //the initial channel builder is for clients, we need a server one
         File trustStoreFile = File.createTempFile("truststore", ".jks");
         Map sslServerConfigs = TestSslUtils.createSslConfig(false, true, Mode.SERVER, trustStoreFile, "server");
diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala
index 6094082b64ab6..eb6fafd6a456b 100644
--- a/core/src/main/scala/kafka/network/SocketServer.scala
+++ b/core/src/main/scala/kafka/network/SocketServer.scala
@@ -40,7 +40,8 @@ import org.apache.kafka.common.config.ConfigException
 import org.apache.kafka.common.{Endpoint, KafkaException, Reconfigurable}
 import org.apache.kafka.common.memory.{MemoryPool, SimpleMemoryPool}
 import org.apache.kafka.common.metrics._
-import org.apache.kafka.common.metrics.stats.{CumulativeSum, Meter}
+import org.apache.kafka.common.metrics.stats.Percentiles.BucketSizing
+import org.apache.kafka.common.metrics.stats.{CumulativeSum, Max, Meter, Percentile, Percentiles}
 import org.apache.kafka.common.network.KafkaChannel.ChannelMuteEvent
 import org.apache.kafka.common.network.{ChannelBuilder, ChannelBuilders, KafkaChannel, ListenerName, ListenerReconfigurable, Selectable, Send, Selector => KSelector}
 import org.apache.kafka.common.protocol.ApiKeys
@@ -83,11 +84,19 @@ class SocketServer(val config: KafkaConfig,
   private val logContext = new LogContext(s"[SocketServer brokerId=${config.brokerId}] ")
   this.logIdent = logContext.logPrefix
 
-  private val memoryPoolSensor = metrics.sensor("MemoryPoolUtilization")
+  private val memoryPoolUsageSensor = metrics.sensor("MemoryPoolUtilization")
   private val memoryPoolDepletedPercentMetricName = metrics.metricName("MemoryPoolAvgDepletedPercent", MetricsGroup)
   private val memoryPoolDepletedTimeMetricName = metrics.metricName("MemoryPoolDepletedTimeTotal", MetricsGroup)
-  memoryPoolSensor.add(new Meter(TimeUnit.MILLISECONDS, memoryPoolDepletedPercentMetricName, memoryPoolDepletedTimeMetricName))
-  private val memoryPool = if (config.queuedMaxBytes > 0) new SimpleMemoryPool(config.queuedMaxBytes, config.socketRequestMaxBytes, false, memoryPoolSensor) else MemoryPool.NONE
+  memoryPoolUsageSensor.add(new Meter(TimeUnit.MILLISECONDS, memoryPoolDepletedPercentMetricName, memoryPoolDepletedTimeMetricName))
+  private val memoryPoolAllocationSensor = metrics.sensor("MemoryPoolAllocation")
+  private val memoryPoolMaxAllocateSizeMetricName = metrics.metricName("MemoryPoolMaxAllocateSize", "socket-server-metrics")
+  private val memoryPoolAllocateSizePercentilesMetricName = metrics.metricName("MemoryPoolAllocateSizePercentiles", "socket-server-metrics")
+  private val percentiles = (1 to 9).map( i => new Percentile(metrics.metricName("MemoryPoolAllocateSize%dPercentile".format(i * 10), "socket-server-metrics"), i * 10))
+  memoryPoolAllocationSensor.add(memoryPoolMaxAllocateSizeMetricName, new Max())
+  // At current stage, we do not know the max decrypted request size, temporarily set it to 10MB.
+  memoryPoolAllocationSensor.add(memoryPoolAllocateSizePercentilesMetricName, new Percentiles(400, 0.0, 10485760, BucketSizing.CONSTANT, percentiles:_*))
+  private val memoryPool = if (config.queuedMaxBytes > 0) new SimpleMemoryPool(config.queuedMaxBytes, config.socketRequestMaxBytes, false, memoryPoolUsageSensor, memoryPoolAllocationSensor)
+                           else MemoryPool.NONE
   // data-plane
   private val dataPlaneProcessors = new ConcurrentHashMap[Int, Processor]()
   private[network] val dataPlaneAcceptors = new ConcurrentHashMap[EndPoint, Acceptor]()

From 865303cc1587c91d0ca940852fca6145b1135010 Mon Sep 17 00:00:00 2001
From: kun du 
Date: Thu, 5 Sep 2019 20:42:27 -0700
Subject: [PATCH 1042/1071] [LI-HOTFIX] Change default value of config
 delivery.timeout.ms to MAX_INT.

TICKET =
LI_DESCRIPTION =
Change the delivery.timeout.ms to MAX_INT to preserve the same producer behaviour
after bumping the Kafka version to 2.*.
EXIT_CRITERIA = MANUAL [""]
---
 .../java/org/apache/kafka/clients/producer/ProducerConfig.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
index 3a1f3db5a5843..b8bf54012ce46 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
@@ -283,7 +283,7 @@ public class ProducerConfig extends AbstractConfig {
                                 .define(COMPRESSION_TYPE_CONFIG, Type.STRING, "none", Importance.HIGH, COMPRESSION_TYPE_DOC)
                                 .define(BATCH_SIZE_CONFIG, Type.INT, 16384, atLeast(0), Importance.MEDIUM, BATCH_SIZE_DOC)
                                 .define(LINGER_MS_CONFIG, Type.LONG, 0, atLeast(0), Importance.MEDIUM, LINGER_MS_DOC)
-                                .define(DELIVERY_TIMEOUT_MS_CONFIG, Type.INT, 120 * 1000, atLeast(0), Importance.MEDIUM, DELIVERY_TIMEOUT_MS_DOC)
+                                .define(DELIVERY_TIMEOUT_MS_CONFIG, Type.INT, Integer.MAX_VALUE, atLeast(0), Importance.MEDIUM, DELIVERY_TIMEOUT_MS_DOC)
                                 .define(CLIENT_ID_CONFIG, Type.STRING, "", Importance.MEDIUM, CommonClientConfigs.CLIENT_ID_DOC)
                                 .define(SEND_BUFFER_CONFIG, Type.INT, 128 * 1024, atLeast(CommonClientConfigs.SEND_BUFFER_LOWER_BOUND), Importance.MEDIUM, CommonClientConfigs.SEND_BUFFER_DOC)
                                 .define(RECEIVE_BUFFER_CONFIG, Type.INT, 32 * 1024, atLeast(CommonClientConfigs.RECEIVE_BUFFER_LOWER_BOUND), Importance.MEDIUM, CommonClientConfigs.RECEIVE_BUFFER_DOC)

From 5142212ce406d6fee5b1489a7eba657c408f2a2d Mon Sep 17 00:00:00 2001
From: kun du 
Date: Sun, 10 Nov 2019 22:48:33 -0800
Subject: [PATCH 1043/1071] [LI-HOTFIX] Add support to use BoringSSL for
 SSL/TLS.

TICKET = KAFKA-2561
LI_DESCRIPTION =
Make Kafka capable of using BoringSSL for encryption.
EXIT_CRITERIA = TICKET [KAFKA-2561]
---
 build.gradle                                  |  1 +
 checkstyle/import-control.xml                 |  1 +
 .../kafka/common/config/SslConfigs.java       |  8 +++
 .../common/network/SslTransportLayer.java     |  2 +-
 .../ssl/BoringSslContextProvider.java         | 43 ++++++++++++++
 .../ssl/SimpleSslContextProvider.java         | 43 ++++++++++++++
 .../security/ssl/SslContextProvider.java      | 37 ++++++++++++
 .../common/security/ssl/SslEngineBuilder.java | 18 +++---
 .../kafka/common/network/CertStores.java      | 16 ++---
 .../common/network/SslTransportLayerTest.java | 58 ++++++++++++-------
 .../SaslAuthenticatorFailureDelayTest.java    |  5 +-
 .../authenticator/SaslAuthenticatorTest.java  | 36 +++++++++---
 .../org/apache/kafka/test/TestSslUtils.java   | 19 ++++--
 .../main/scala/kafka/server/KafkaConfig.scala |  4 ++
 .../unit/kafka/server/KafkaConfigTest.scala   |  1 +
 gradle/dependencies.gradle                    |  8 ++-
 16 files changed, 243 insertions(+), 57 deletions(-)
 create mode 100644 clients/src/main/java/org/apache/kafka/common/security/ssl/BoringSslContextProvider.java
 create mode 100644 clients/src/main/java/org/apache/kafka/common/security/ssl/SimpleSslContextProvider.java
 create mode 100644 clients/src/main/java/org/apache/kafka/common/security/ssl/SslContextProvider.java

diff --git a/build.gradle b/build.gradle
index bad8595c59709..e869d364b4700 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1021,6 +1021,7 @@ project(':clients') {
 
     compileOnly libs.jacksonDatabind // for SASL/OAUTHBEARER bearer token parsing
     compileOnly libs.jacksonJDK8Datatypes
+    compile libs.conscrypt
 
     jacksonDatabindConfig libs.jacksonDatabind // to publish as provided scope dependency.
 
diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml
index 1b13be58ecf89..d9235396cbddc 100644
--- a/checkstyle/import-control.xml
+++ b/checkstyle/import-control.xml
@@ -54,6 +54,7 @@
     
     
     
+    
 
     
       
diff --git a/clients/src/main/java/org/apache/kafka/common/config/SslConfigs.java b/clients/src/main/java/org/apache/kafka/common/config/SslConfigs.java
index 6cd31be527492..47b7cf7643bff 100644
--- a/clients/src/main/java/org/apache/kafka/common/config/SslConfigs.java
+++ b/clients/src/main/java/org/apache/kafka/common/config/SslConfigs.java
@@ -17,6 +17,7 @@
 package org.apache.kafka.common.config;
 
 import org.apache.kafka.common.config.internals.BrokerSecurityConfigs;
+import org.apache.kafka.common.security.ssl.SimpleSslContextProvider;
 import org.apache.kafka.common.utils.Utils;
 
 import javax.net.ssl.KeyManagerFactory;
@@ -47,6 +48,11 @@ public class SslConfigs {
     public static final String DEFAULT_PRINCIPAL_BUILDER_CLASS =
             org.apache.kafka.common.security.auth.DefaultPrincipalBuilder.class.getName();
 
+    public static final String SSL_CONTEXT_PROVIDER_CLASS_CONFIG = "ssl.context.provider.class";
+    public static final String SSL_CONTEXT_PROVIDER_CLASS_DOC = "The fully qualified name of a class that implements the "
+            + "SslContextProvider interface, which is used to build SSLContext insides encrypted channels.";
+    public static final String DEFAULT_SSL_CONTEXT_PROVIDER_CLASS = SimpleSslContextProvider.class.getName();
+
     public static final String SSL_PROTOCOL_CONFIG = "ssl.protocol";
     public static final String SSL_PROTOCOL_DOC = "The SSL protocol used to generate the SSLContext. "
             + "Default setting is TLS, which is fine for most cases. "
@@ -123,6 +129,7 @@ public class SslConfigs {
 
     public static void addClientSslSupport(ConfigDef config) {
         config.define(SslConfigs.SSL_PROTOCOL_CONFIG, ConfigDef.Type.STRING, SslConfigs.DEFAULT_SSL_PROTOCOL, ConfigDef.Importance.MEDIUM, SslConfigs.SSL_PROTOCOL_DOC)
+                .define(SslConfigs.SSL_CONTEXT_PROVIDER_CLASS_CONFIG, ConfigDef.Type.STRING, SslConfigs.DEFAULT_SSL_CONTEXT_PROVIDER_CLASS, ConfigDef.Importance.MEDIUM, SslConfigs.SSL_CONTEXT_PROVIDER_CLASS_DOC)
                 .define(SslConfigs.SSL_PROVIDER_CONFIG, ConfigDef.Type.STRING, null, ConfigDef.Importance.MEDIUM, SslConfigs.SSL_PROVIDER_DOC)
                 .define(SslConfigs.SSL_CIPHER_SUITES_CONFIG, ConfigDef.Type.LIST, null, ConfigDef.Importance.LOW, SslConfigs.SSL_CIPHER_SUITES_DOC)
                 .define(SslConfigs.SSL_ENABLED_PROTOCOLS_CONFIG, ConfigDef.Type.LIST, SslConfigs.DEFAULT_SSL_ENABLED_PROTOCOLS, ConfigDef.Importance.MEDIUM, SslConfigs.SSL_ENABLED_PROTOCOLS_DOC)
@@ -152,6 +159,7 @@ public static void addClientSslSupport(ConfigDef config) {
             BrokerSecurityConfigs.SSL_CLIENT_AUTH_CONFIG,
             SslConfigs.SSL_PROTOCOL_CONFIG,
             SslConfigs.SSL_PROVIDER_CONFIG,
+            SslConfigs.SSL_CONTEXT_PROVIDER_CLASS_CONFIG,
             SslConfigs.SSL_CIPHER_SUITES_CONFIG,
             SslConfigs.SSL_ENABLED_PROTOCOLS_CONFIG,
             SslConfigs.SSL_KEYMANAGER_ALGORITHM_CONFIG,
diff --git a/clients/src/main/java/org/apache/kafka/common/network/SslTransportLayer.java b/clients/src/main/java/org/apache/kafka/common/network/SslTransportLayer.java
index 6ed2cea80632d..4ffcc6ce32fca 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/SslTransportLayer.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/SslTransportLayer.java
@@ -422,7 +422,7 @@ private void handshakeFinished() throws IOException {
                 key.interestOps(key.interestOps() & ~SelectionKey.OP_WRITE);
                 SSLSession session = sslEngine.getSession();
                 log.debug("SSL handshake completed successfully with peerHost '{}' peerPort {} peerPrincipal '{}' cipherSuite '{}'",
-                        session.getPeerHost(), session.getPeerPort(), peerPrincipal(), session.getCipherSuite());
+                            session.getPeerHost(), session.getPeerPort(), peerPrincipal(), session.getCipherSuite());
             }
 
             log.trace("SSLHandshake FINISHED channelId {}, appReadBuffer pos {}, netReadBuffer pos {}, netWriteBuffer pos {} ",
diff --git a/clients/src/main/java/org/apache/kafka/common/security/ssl/BoringSslContextProvider.java b/clients/src/main/java/org/apache/kafka/common/security/ssl/BoringSslContextProvider.java
new file mode 100644
index 0000000000000..0a41be81efeb0
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/security/ssl/BoringSslContextProvider.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.security.ssl;
+
+import java.security.NoSuchAlgorithmException;
+import java.security.Provider;
+import java.util.Map;
+import javax.net.ssl.SSLContext;
+import org.apache.kafka.common.config.SslConfigs;
+import org.conscrypt.Conscrypt;
+
+/**
+ * A BoringSslContext provider based on Conscrypt library.
+ */
+public class BoringSslContextProvider implements SslContextProvider {
+    private String protocol;
+    private Provider provider;
+
+    @Override
+    public void configure(Map configs) {
+        protocol = (String) configs.get(SslConfigs.SSL_PROTOCOL_CONFIG);
+        provider = Conscrypt.newProvider();
+    }
+
+    @Override
+    public SSLContext getSSLContext()  throws NoSuchAlgorithmException {
+        return SSLContext.getInstance(protocol, provider);
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/security/ssl/SimpleSslContextProvider.java b/clients/src/main/java/org/apache/kafka/common/security/ssl/SimpleSslContextProvider.java
new file mode 100644
index 0000000000000..90ba337e88e1f
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/security/ssl/SimpleSslContextProvider.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.security.ssl;
+
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.util.Map;
+import javax.net.ssl.SSLContext;
+import org.apache.kafka.common.config.SslConfigs;
+
+
+public class SimpleSslContextProvider implements SslContextProvider {
+    private String protocol;
+    private String provider;
+
+    @Override
+    public void configure(Map configs) {
+        this.protocol =  (String) configs.get(SslConfigs.SSL_PROTOCOL_CONFIG);
+        this.provider = (String) configs.get(SslConfigs.SSL_PROVIDER_CONFIG);
+    }
+
+    @Override
+    public SSLContext getSSLContext() throws NoSuchAlgorithmException, NoSuchProviderException {
+        if (provider == null) {
+            return SSLContext.getInstance(protocol);
+        }
+        return SSLContext.getInstance(protocol, provider);
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/security/ssl/SslContextProvider.java b/clients/src/main/java/org/apache/kafka/common/security/ssl/SslContextProvider.java
new file mode 100644
index 0000000000000..ff2e8fc954348
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/security/ssl/SslContextProvider.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.common.security.ssl;
+
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import javax.net.ssl.SSLContext;
+import org.apache.kafka.common.Configurable;
+
+/**
+ * A SSLContext provider interface to create SSLContext based on configs
+ */
+public interface SslContextProvider extends Configurable {
+
+    /**
+    * Create a SSLContext based on protocol and provider config.
+    * @return SSLContext
+    * @throws NoSuchAlgorithmException if protocol config is invalid.
+    * @throws NoSuchProviderException if provider config is invalid.
+    */
+    SSLContext getSSLContext() throws NoSuchAlgorithmException, NoSuchProviderException;
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/security/ssl/SslEngineBuilder.java b/clients/src/main/java/org/apache/kafka/common/security/ssl/SslEngineBuilder.java
index ae76690496b7c..b44298e25b990 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/ssl/SslEngineBuilder.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/ssl/SslEngineBuilder.java
@@ -50,8 +50,7 @@ public class SslEngineBuilder {
     private static final Logger log = LoggerFactory.getLogger(SslEngineBuilder.class);
 
     private final Map configs;
-    private final String protocol;
-    private final String provider;
+    private SslContextProvider sslContextProvider;
     private final String kmfAlgorithm;
     private final String tmfAlgorithm;
     private final SecurityStore keystore;
@@ -65,8 +64,13 @@ public class SslEngineBuilder {
     @SuppressWarnings("unchecked")
     SslEngineBuilder(Map configs) {
         this.configs = Collections.unmodifiableMap(configs);
-        this.protocol = (String) configs.get(SslConfigs.SSL_PROTOCOL_CONFIG);
-        this.provider = (String) configs.get(SslConfigs.SSL_PROVIDER_CONFIG);
+        try {
+            String sslContextProvider = (String) configs.get(SslConfigs.SSL_CONTEXT_PROVIDER_CLASS_CONFIG);
+            this.sslContextProvider = (SslContextProvider) Class.forName(sslContextProvider).newInstance();
+        } catch (Exception e) {
+            throw new KafkaException(e);
+        }
+        this.sslContextProvider.configure(configs);
         SecurityUtils.addConfiguredSecurityProviders(this.configs);
 
         List cipherSuitesList = (List) configs.get(SslConfigs.SSL_CIPHER_SUITES_CONFIG);
@@ -129,11 +133,7 @@ private static SecureRandom createSecureRandom(String key) {
 
     private SSLContext createSSLContext() {
         try {
-            SSLContext sslContext;
-            if (provider != null)
-                sslContext = SSLContext.getInstance(protocol, provider);
-            else
-                sslContext = SSLContext.getInstance(protocol);
+            SSLContext sslContext = sslContextProvider.getSSLContext();
 
             KeyManager[] keyManagers = null;
             if (keystore != null || kmfAlgorithm != null) {
diff --git a/clients/src/test/java/org/apache/kafka/common/network/CertStores.java b/clients/src/test/java/org/apache/kafka/common/network/CertStores.java
index 045e280880aea..d02bdfd91e582 100644
--- a/clients/src/test/java/org/apache/kafka/common/network/CertStores.java
+++ b/clients/src/test/java/org/apache/kafka/common/network/CertStores.java
@@ -41,23 +41,23 @@ public class CertStores {
 
     private final Map sslConfig;
 
-    public CertStores(boolean server, String hostName) throws Exception {
-        this(server, hostName, new TestSslUtils.CertificateBuilder());
+    public CertStores(boolean server, String hostName, TestSslUtils.SSLProvider provider) throws Exception {
+        this(server, hostName, new TestSslUtils.CertificateBuilder(), provider);
     }
 
-    public CertStores(boolean server, String commonName, String sanHostName) throws Exception {
-        this(server, commonName, new TestSslUtils.CertificateBuilder().sanDnsName(sanHostName));
+    public CertStores(boolean server, String commonName, String sanHostName, TestSslUtils.SSLProvider provider) throws Exception {
+        this(server, commonName, new TestSslUtils.CertificateBuilder().sanDnsName(sanHostName), provider);
     }
 
-    public CertStores(boolean server, String commonName, InetAddress hostAddress) throws Exception {
-        this(server, commonName, new TestSslUtils.CertificateBuilder().sanIpAddress(hostAddress));
+    public CertStores(boolean server, String commonName, InetAddress hostAddress, TestSslUtils.SSLProvider provider) throws Exception {
+        this(server, commonName, new TestSslUtils.CertificateBuilder().sanIpAddress(hostAddress), provider);
     }
 
-    private CertStores(boolean server, String commonName, TestSslUtils.CertificateBuilder certBuilder) throws Exception {
+    private CertStores(boolean server, String commonName, TestSslUtils.CertificateBuilder certBuilder, TestSslUtils.SSLProvider provider) throws Exception {
         String name = server ? "server" : "client";
         Mode mode = server ? Mode.SERVER : Mode.CLIENT;
         File truststoreFile = File.createTempFile(name + "TS", ".jks");
-        sslConfig = TestSslUtils.createSslConfig(!server, true, mode, truststoreFile, name, commonName, certBuilder);
+        sslConfig = TestSslUtils.createSslConfig(!server, true, mode, truststoreFile, name, commonName, certBuilder, provider);
     }
 
     public Map getTrustingConfig(CertStores truststoreConfig) {
diff --git a/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java b/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java
index d25fa6124681c..e7dd0d3956102 100644
--- a/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java
@@ -16,6 +16,8 @@
  */
 package org.apache.kafka.common.network;
 
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.config.SslConfigs;
@@ -30,12 +32,12 @@
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.test.TestCondition;
+import org.apache.kafka.test.TestSslUtils.SSLProvider;
 import org.apache.kafka.test.TestUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLParameters;
 import java.io.ByteArrayOutputStream;
@@ -51,6 +53,8 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -60,7 +64,13 @@
 /**
  * Tests for the SSL transport layer. These use a test harness that runs a simple socket server that echos back responses.
  */
+@RunWith(Parameterized.class)
 public class SslTransportLayerTest {
+    private final SSLProvider provider;
+
+    public SslTransportLayerTest(SSLProvider provider) {
+        this.provider = provider;
+    }
 
     private static final int BUFFER_SIZE = 4 * 1024;
     private static Time time = Time.SYSTEM;
@@ -76,8 +86,8 @@ public class SslTransportLayerTest {
     @Before
     public void setup() throws Exception {
         // Create certificates for use by client and server. Add server cert to client truststore and vice versa.
-        serverCertStores = new CertStores(true, "server",  "localhost");
-        clientCertStores = new CertStores(false, "client", "localhost");
+        serverCertStores = new CertStores(true, "server",  "localhost", provider);
+        clientCertStores = new CertStores(false, "client", "localhost", provider);
         sslServerConfigs = serverCertStores.getTrustingConfig(clientCertStores);
         sslClientConfigs = clientCertStores.getTrustingConfig(serverCertStores);
         this.channelBuilder = new SslChannelBuilder(Mode.CLIENT, null, false);
@@ -117,8 +127,8 @@ public void testValidEndpointIdentificationSanDns() throws Exception {
     @Test
     public void testValidEndpointIdentificationSanIp() throws Exception {
         String node = "0";
-        serverCertStores = new CertStores(true, "server", InetAddress.getByName("127.0.0.1"));
-        clientCertStores = new CertStores(false, "client", InetAddress.getByName("127.0.0.1"));
+        serverCertStores = new CertStores(true, "server", InetAddress.getByName("127.0.0.1"), provider);
+        clientCertStores = new CertStores(false, "client", InetAddress.getByName("127.0.0.1"), provider);
         sslServerConfigs = serverCertStores.getTrustingConfig(clientCertStores);
         sslClientConfigs = clientCertStores.getTrustingConfig(serverCertStores);
         server = createEchoServer(SecurityProtocol.SSL);
@@ -137,8 +147,8 @@ public void testValidEndpointIdentificationSanIp() throws Exception {
     @Test
     public void testValidEndpointIdentificationCN() throws Exception {
         String node = "0";
-        serverCertStores = new CertStores(true, "localhost");
-        clientCertStores = new CertStores(false, "localhost");
+        serverCertStores = new CertStores(true, "localhost", provider);
+        clientCertStores = new CertStores(false, "localhost", provider);
         sslServerConfigs = serverCertStores.getTrustingConfig(clientCertStores);
         sslClientConfigs = clientCertStores.getTrustingConfig(serverCertStores);
         server = createEchoServer(SecurityProtocol.SSL);
@@ -187,8 +197,8 @@ public void testClientEndpointNotValidated() throws Exception {
         String node = "0";
 
         // Create client certificate with an invalid hostname
-        clientCertStores = new CertStores(false, "non-existent.com");
-        serverCertStores = new CertStores(true, "localhost");
+        clientCertStores = new CertStores(false, "non-existent.com", provider);
+        serverCertStores = new CertStores(true, "localhost", provider);
         sslServerConfigs = serverCertStores.getTrustingConfig(clientCertStores);
         sslClientConfigs = clientCertStores.getTrustingConfig(serverCertStores);
 
@@ -222,8 +232,8 @@ protected TestSslTransportLayer newTransportLayer(String id, SelectionKey key, S
     @Test
     public void testInvalidEndpointIdentification() throws Exception {
         String node = "0";
-        serverCertStores = new CertStores(true, "server", "notahost");
-        clientCertStores = new CertStores(false, "client", "localhost");
+        serverCertStores = new CertStores(true, "server", "notahost", provider);
+        clientCertStores = new CertStores(false, "client", "localhost", provider);
         sslServerConfigs = serverCertStores.getTrustingConfig(clientCertStores);
         sslClientConfigs = clientCertStores.getTrustingConfig(serverCertStores);
         sslClientConfigs.put(SslConfigs.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG, "HTTPS");
@@ -242,8 +252,8 @@ public void testInvalidEndpointIdentification() throws Exception {
      */
     @Test
     public void testEndpointIdentificationDisabled() throws Exception {
-        serverCertStores = new CertStores(true, "server", "notahost");
-        clientCertStores = new CertStores(false, "client", "localhost");
+        serverCertStores = new CertStores(true, "server", "notahost", provider);
+        clientCertStores = new CertStores(false, "client", "localhost", provider);
         sslServerConfigs = serverCertStores.getTrustingConfig(clientCertStores);
         sslClientConfigs = clientCertStores.getTrustingConfig(serverCertStores);
 
@@ -543,13 +553,12 @@ public void testUnsupportedTLSVersion() throws Exception {
     @Test
     public void testUnsupportedCiphers() throws Exception {
         String node = "0";
-        SSLContext context = SSLContext.getInstance("TLSv1.2");
-        context.init(null, null, null);
-        String[] cipherSuites = context.getDefaultSSLParameters().getCipherSuites();
-        sslServerConfigs.put(SslConfigs.SSL_CIPHER_SUITES_CONFIG, Arrays.asList(cipherSuites[0]));
+        // Since Conscrypt does not support all the default JDK cipher suites, specifically pick two suites which are supported
+        // by both JDK and Conscrypt.
+        sslServerConfigs.put(SslConfigs.SSL_CIPHER_SUITES_CONFIG, Arrays.asList("TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA"));
         server = createEchoServer(SecurityProtocol.SSL);
 
-        sslClientConfigs.put(SslConfigs.SSL_CIPHER_SUITES_CONFIG, Arrays.asList(cipherSuites[1]));
+        sslClientConfigs.put(SslConfigs.SSL_CIPHER_SUITES_CONFIG, Arrays.asList("TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"));
         createSelector(sslClientConfigs);
         InetSocketAddress addr = new InetSocketAddress("localhost", server.port());
         selector.connect(node, addr, BUFFER_SIZE, BUFFER_SIZE);
@@ -950,7 +959,7 @@ public void testServerKeystoreDynamicUpdate() throws Exception {
         oldClientSelector.connect(oldNode, addr, BUFFER_SIZE, BUFFER_SIZE);
         NetworkTestUtils.checkClientConnection(selector, oldNode, 100, 10);
 
-        CertStores newServerCertStores = new CertStores(true, "server", "localhost");
+        CertStores newServerCertStores = new CertStores(true, "server", "localhost", provider);
         Map newKeystoreConfigs = newServerCertStores.keyStoreProps();
         assertTrue("SslChannelBuilder not reconfigurable", serverChannelBuilder instanceof ListenerReconfigurable);
         ListenerReconfigurable reconfigurableBuilder = (ListenerReconfigurable) serverChannelBuilder;
@@ -971,7 +980,7 @@ public void testServerKeystoreDynamicUpdate() throws Exception {
         // Verify that old client continues to work
         NetworkTestUtils.checkClientConnection(oldClientSelector, oldNode, 100, 10);
 
-        CertStores invalidCertStores = new CertStores(true, "server", "127.0.0.1");
+        CertStores invalidCertStores = new CertStores(true, "server", "127.0.0.1", provider);
         Map  invalidConfigs = invalidCertStores.getTrustingConfig(clientCertStores);
         verifyInvalidReconfigure(reconfigurableBuilder, invalidConfigs, "keystore with different SubjectAltName");
 
@@ -1010,7 +1019,7 @@ public void testServerTruststoreDynamicUpdate() throws Exception {
         oldClientSelector.connect(oldNode, addr, BUFFER_SIZE, BUFFER_SIZE);
         NetworkTestUtils.checkClientConnection(selector, oldNode, 100, 10);
 
-        CertStores newClientCertStores = new CertStores(true, "client", "localhost");
+        CertStores newClientCertStores = new CertStores(true, "client", "localhost", provider);
         sslClientConfigs = newClientCertStores.getTrustingConfig(serverCertStores);
         Map newTruststoreConfigs = newClientCertStores.trustStoreProps();
         assertTrue("SslChannelBuilder not reconfigurable", serverChannelBuilder instanceof ListenerReconfigurable);
@@ -1219,4 +1228,11 @@ int updateAndGet(int actualSize, boolean update) {
             }
         }
     }
+
+    @Parameterized.Parameters(name = "SSLProvider={0}")
+    public static Collection data() {
+        Collection p = new ArrayList<>();
+        p.add(new Object[]{SSLProvider.DEFAULT});
+        return p;
+    }
 }
diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorFailureDelayTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorFailureDelayTest.java
index 773fffbb287f5..0a128c9449476 100644
--- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorFailureDelayTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorFailureDelayTest.java
@@ -31,6 +31,7 @@
 import org.apache.kafka.common.security.TestSecurityConfig;
 import org.apache.kafka.common.security.auth.SecurityProtocol;
 import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.test.TestSslUtils;
 import org.apache.kafka.test.TestUtils;
 import org.junit.After;
 import org.junit.Assert;
@@ -81,8 +82,8 @@ public static Collection data() {
     @Before
     public void setup() throws Exception {
         LoginManager.closeAll();
-        serverCertStores = new CertStores(true, "localhost");
-        clientCertStores = new CertStores(false, "localhost");
+        serverCertStores = new CertStores(true, "localhost", TestSslUtils.SSLProvider.DEFAULT);
+        clientCertStores = new CertStores(false, "localhost", TestSslUtils.SSLProvider.DEFAULT);
         saslServerConfigs = serverCertStores.getTrustingConfig(clientCertStores);
         saslClientConfigs = clientCertStores.getTrustingConfig(serverCertStores);
         credentialCache = new CredentialCache();
diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
index dd3c61dc99bee..9bc574fe15e96 100644
--- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
@@ -25,6 +25,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Base64;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -120,10 +121,13 @@
 
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.test.TestSslUtils.SSLProvider;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 import static org.apache.kafka.common.protocol.ApiKeys.LIST_OFFSETS;
 import static org.junit.Assert.assertEquals;
@@ -135,7 +139,13 @@
 /**
  * Tests for the Sasl authenticator. These use a test harness that runs a simple socket server that echos back responses.
  */
+@RunWith(Parameterized.class)
 public class SaslAuthenticatorTest {
+    private final SSLProvider provider;
+
+    public SaslAuthenticatorTest(SSLProvider provider) {
+        this.provider = provider;
+    }
 
     private static final long CONNECTIONS_MAX_REAUTH_MS_VALUE = 100L;
     private static final int BUFFER_SIZE = 4 * 1024;
@@ -155,8 +165,8 @@ public class SaslAuthenticatorTest {
     public void setup() throws Exception {
         LoginManager.closeAll();
         time = Time.SYSTEM;
-        serverCertStores = new CertStores(true, "localhost");
-        clientCertStores = new CertStores(false, "localhost");
+        serverCertStores = new CertStores(true, "localhost", provider);
+        clientCertStores = new CertStores(false, "localhost", provider);
         saslServerConfigs = serverCertStores.getTrustingConfig(clientCertStores);
         saslClientConfigs = clientCertStores.getTrustingConfig(serverCertStores);
         credentialCache = new CredentialCache();
@@ -399,7 +409,7 @@ public void testMultipleServerMechanisms() throws Exception {
             selector.connect(node3, new InetSocketAddress("127.0.0.1", server.port()), BUFFER_SIZE, BUFFER_SIZE);
             NetworkTestUtils.checkClientConnection(selector, node3, 100, 10);
             server.verifyAuthenticationMetrics(3, 0);
-            
+
             /*
              * Now re-authenticate the connections. First we have to sleep long enough so
              * that the next write will cause re-authentication, which we expect to succeed.
@@ -412,7 +422,7 @@ public void testMultipleServerMechanisms() throws Exception {
 
             NetworkTestUtils.checkClientConnection(selector3, node3, 100, 10);
             server.verifyReauthenticationMetrics(2, 0);
-            
+
         } finally {
             if (selector2 != null)
                 selector2.close();
@@ -802,7 +812,7 @@ public void testSaslHandshakeRequestWithUnsupportedVersion() throws Exception {
         createClientConnection(SecurityProtocol.PLAINTEXT, node1);
         SaslHandshakeRequest request = buildSaslHandshakeRequest("PLAIN", ApiKeys.SASL_HANDSHAKE.latestVersion());
         RequestHeader header = new RequestHeader(ApiKeys.SASL_HANDSHAKE, Short.MAX_VALUE, "someclient", 2);
-        
+
         selector.send(request.toSend(node1, header));
         // This test uses a non-SASL PLAINTEXT client in order to do manual handshake.
         // So the channel is in READY state.
@@ -1483,7 +1493,7 @@ public void testValidSaslOauthBearerMechanism() throws Exception {
         server = createEchoServer(securityProtocol);
         createAndCheckClientConnection(securityProtocol, node);
     }
-    
+
     /**
      * Re-authentication must fail if principal changes
      */
@@ -1642,7 +1652,7 @@ public void testCannotReauthenticateAgainFasterThanOneSecond() throws Exception
                     e.getMessage().matches(
                             ".*\\<\\[" + expectedResponseTextRegex + "]>.*\\<\\[" + receivedResponseTextRegex + "]>"));
             server.verifyReauthenticationMetrics(1, 0); // unchanged
-        } finally { 
+        } finally {
             selector.close();
             selector = null;
         }
@@ -1684,7 +1694,7 @@ public void testRepeatedValidSaslPlainOverSsl() throws Exception {
         }
         server.verifyReauthenticationMetrics(desiredNumReauthentications, 0);
     }
-    
+
     /**
      * Tests OAUTHBEARER client channels without tokens for the server.
      */
@@ -2002,7 +2012,7 @@ private void createClientConnection(SecurityProtocol securityProtocol, String no
         InetSocketAddress addr = new InetSocketAddress("localhost", server.port());
         selector.connect(node, addr, BUFFER_SIZE, BUFFER_SIZE);
     }
-    
+
     private void checkClientConnection(String node) throws Exception {
         NetworkTestUtils.checkClientConnection(selector, node, 100, 10);
     }
@@ -2387,4 +2397,12 @@ protected SaslHandshakeRequest createSaslHandshakeRequest(short version) {
                 };
         }
     }
+
+    @Parameterized.Parameters(name = "SSLProvider={0}")
+    public static Collection data() {
+        Collection p = new ArrayList<>();
+        p.add(new Object[]{SSLProvider.DEFAULT});
+        p.add(new Object[]{SSLProvider.OPENSSL});
+        return p;
+    }
 }
diff --git a/clients/src/test/java/org/apache/kafka/test/TestSslUtils.java b/clients/src/test/java/org/apache/kafka/test/TestSslUtils.java
index 6dfceb28f09bd..bcdcb584e92bd 100644
--- a/clients/src/test/java/org/apache/kafka/test/TestSslUtils.java
+++ b/clients/src/test/java/org/apache/kafka/test/TestSslUtils.java
@@ -44,6 +44,8 @@
 import java.security.cert.X509Certificate;
 
 import org.apache.kafka.common.config.types.Password;
+import org.apache.kafka.common.security.ssl.BoringSslContextProvider;
+import org.apache.kafka.common.security.ssl.SimpleSslContextProvider;
 import org.bouncycastle.asn1.DEROctetString;
 import org.bouncycastle.asn1.x500.X500Name;
 import org.bouncycastle.asn1.x509.AlgorithmIdentifier;
@@ -71,6 +73,9 @@
 public class TestSslUtils {
 
     public static final String TRUST_STORE_PASSWORD = "TrustStorePassword";
+    public enum SSLProvider {
+        DEFAULT, OPENSSL
+    }
 
     /**
      * Create a self-signed X.509 Certificate.
@@ -153,7 +158,7 @@ public static  void createTrustStore(
     }
 
     private static Map createSslConfig(Mode mode, File keyStoreFile, Password password, Password keyPassword,
-                                                       File trustStoreFile, Password trustStorePassword) {
+                                                       File trustStoreFile, Password trustStorePassword, SSLProvider provider) {
         Map sslConfigs = new HashMap<>();
         sslConfigs.put(SslConfigs.SSL_PROTOCOL_CONFIG, "TLSv1.2"); // protocol to create SSLContext
 
@@ -173,6 +178,11 @@ private static Map createSslConfig(Mode mode, File keyStoreFile,
         List enabledProtocols  = new ArrayList<>();
         enabledProtocols.add("TLSv1.2");
         sslConfigs.put(SslConfigs.SSL_ENABLED_PROTOCOLS_CONFIG, enabledProtocols);
+        if (provider.equals(SSLProvider.OPENSSL)) {
+            sslConfigs.put(SslConfigs.SSL_CONTEXT_PROVIDER_CLASS_CONFIG, BoringSslContextProvider.class.getName());
+        } else {
+            sslConfigs.put(SslConfigs.SSL_CONTEXT_PROVIDER_CLASS_CONFIG, SimpleSslContextProvider.class.getName());
+        }
 
         return sslConfigs;
     }
@@ -180,6 +190,7 @@ private static Map createSslConfig(Mode mode, File keyStoreFile,
     public static Map createSslConfig(String keyManagerAlgorithm, String trustManagerAlgorithm) {
         Map sslConfigs = new HashMap<>();
         sslConfigs.put(SslConfigs.SSL_PROTOCOL_CONFIG, "TLSv1.2"); // protocol to create SSLContext
+        sslConfigs.put(SslConfigs.SSL_CONTEXT_PROVIDER_CLASS_CONFIG, SimpleSslContextProvider.class.getName());
 
         sslConfigs.put(SslConfigs.SSL_KEYMANAGER_ALGORITHM_CONFIG, keyManagerAlgorithm);
         sslConfigs.put(SslConfigs.SSL_TRUSTMANAGER_ALGORITHM_CONFIG, trustManagerAlgorithm);
@@ -199,11 +210,11 @@ public static  Map createSslConfig(boolean useClientCert, boolea
     public static  Map createSslConfig(boolean useClientCert, boolean trustStore,
             Mode mode, File trustStoreFile, String certAlias, String cn)
         throws IOException, GeneralSecurityException {
-        return createSslConfig(useClientCert, trustStore, mode, trustStoreFile, certAlias, cn, new CertificateBuilder());
+        return createSslConfig(useClientCert, trustStore, mode, trustStoreFile, certAlias, cn, new CertificateBuilder(), SSLProvider.DEFAULT);
     }
 
     public static  Map createSslConfig(boolean useClientCert, boolean trustStore,
-            Mode mode, File trustStoreFile, String certAlias, String cn, CertificateBuilder certBuilder)
+            Mode mode, File trustStoreFile, String certAlias, String cn, CertificateBuilder certBuilder, SSLProvider provider)
             throws IOException, GeneralSecurityException {
         Map certs = new HashMap<>();
         File keyStoreFile = null;
@@ -232,7 +243,7 @@ public static  Map createSslConfig(boolean useClientCert, boolea
             trustStoreFile.deleteOnExit();
         }
 
-        return createSslConfig(mode, keyStoreFile, password, password, trustStoreFile, trustStorePassword);
+        return createSslConfig(mode, keyStoreFile, password, password, trustStoreFile, trustStorePassword, provider);
     }
 
     public static class CertificateBuilder {
diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index 5b0208e04418f..e4b50379af065 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -230,6 +230,7 @@ object Defaults {
 
   /** ********* SSL configuration ***********/
   val SslProtocol = SslConfigs.DEFAULT_SSL_PROTOCOL
+  val SslContextProviderClass = SslConfigs.DEFAULT_SSL_CONTEXT_PROVIDER_CLASS
   val SslEnabledProtocols = SslConfigs.DEFAULT_SSL_ENABLED_PROTOCOLS
   val SslKeystoreType = SslConfigs.DEFAULT_SSL_KEYSTORE_TYPE
   val SslTruststoreType = SslConfigs.DEFAULT_SSL_TRUSTSTORE_TYPE
@@ -467,6 +468,7 @@ object KafkaConfig {
   /** ********* SSL Configuration ****************/
   val SslProtocolProp = SslConfigs.SSL_PROTOCOL_CONFIG
   val SslProviderProp = SslConfigs.SSL_PROVIDER_CONFIG
+  val SslContextProviderClassProp = SslConfigs.SSL_CONTEXT_PROVIDER_CLASS_CONFIG
   val SslCipherSuitesProp = SslConfigs.SSL_CIPHER_SUITES_CONFIG
   val SslEnabledProtocolsProp = SslConfigs.SSL_ENABLED_PROTOCOLS_CONFIG
   val SslKeystoreTypeProp = SslConfigs.SSL_KEYSTORE_TYPE_CONFIG
@@ -831,6 +833,7 @@ object KafkaConfig {
   /** ********* SSL Configuration ****************/
   val SslProtocolDoc = SslConfigs.SSL_PROTOCOL_DOC
   val SslProviderDoc = SslConfigs.SSL_PROVIDER_DOC
+  val SslContextProviderClassDoc = SslConfigs.SSL_CONTEXT_PROVIDER_CLASS_DOC
   val SslCipherSuitesDoc = SslConfigs.SSL_CIPHER_SUITES_DOC
   val SslEnabledProtocolsDoc = SslConfigs.SSL_ENABLED_PROTOCOLS_DOC
   val SslKeystoreTypeDoc = SslConfigs.SSL_KEYSTORE_TYPE_DOC
@@ -1097,6 +1100,7 @@ object KafkaConfig {
       .define(PrincipalBuilderClassProp, CLASS, null, MEDIUM, PrincipalBuilderClassDoc)
       .define(SslProtocolProp, STRING, Defaults.SslProtocol, MEDIUM, SslProtocolDoc)
       .define(SslProviderProp, STRING, null, MEDIUM, SslProviderDoc)
+      .define(SslContextProviderClassProp, STRING, Defaults.SslContextProviderClass, MEDIUM, SslContextProviderClassDoc)
       .define(SslEnabledProtocolsProp, LIST, Defaults.SslEnabledProtocols, MEDIUM, SslEnabledProtocolsDoc)
       .define(SslKeystoreTypeProp, STRING, Defaults.SslKeystoreType, MEDIUM, SslKeystoreTypeDoc)
       .define(SslKeystoreLocationProp, STRING, null, MEDIUM, SslKeystoreLocationDoc)
diff --git a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
index 2ba8b16687cfa..611dbe81aa2a9 100755
--- a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
@@ -710,6 +710,7 @@ class KafkaConfigTest {
         case KafkaConfig.ConnectionsMaxReauthMsProp =>
         case KafkaConfig.SslProtocolProp => // ignore string
         case KafkaConfig.SslProviderProp => // ignore string
+        case KafkaConfig.SslContextProviderClassProp => // ignore string
         case KafkaConfig.SslEnabledProtocolsProp =>
         case KafkaConfig.SslKeystoreTypeProp => // ignore string
         case KafkaConfig.SslKeystoreLocationProp => // ignore string
diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle
index da353006c7036..9e7bbc7259a52 100644
--- a/gradle/dependencies.gradle
+++ b/gradle/dependencies.gradle
@@ -20,7 +20,7 @@
 ext {
   versions = [:]
   libs = [:]
-  
+
   // Enabled by default when commands like `testAll` are invoked
   defaultScalaVersions = [ '2.11', '2.12', '2.13' ]
   // Available if -PscalaVersion is used. This is useful when we want to support a Scala version that has
@@ -115,7 +115,8 @@ versions += [
   spotbugsPlugin: "1.6.9",
   spotlessPlugin: "3.23.1",
   zookeeper: "3.5.8",
-  zstd: "1.4.3-1"
+  zstd: "1.4.3-1",
+  conscrypt: "2.1.0"
 ]
 
 libs += [
@@ -185,5 +186,6 @@ libs += [
   jfreechart: "jfreechart:jfreechart:$versions.jfreechart",
   mavenArtifact: "org.apache.maven:maven-artifact:$versions.mavenArtifact",
   zstd: "com.github.luben:zstd-jni:$versions.zstd",
-  httpclient: "org.apache.httpcomponents:httpclient:$versions.httpclient"
+  httpclient: "org.apache.httpcomponents:httpclient:$versions.httpclient",
+  conscrypt: "org.conscrypt:conscrypt-openjdk-uber:2.2.1"
 ]

From e42c7d99bf760ae730b6a3b78c4013e215853d09 Mon Sep 17 00:00:00 2001
From: Xiongqi Wu 
Date: Thu, 25 Apr 2019 14:42:23 -0700
Subject: [PATCH 1044/1071] [LI-HOTFIX] Cancel partition reassignment after
 topic deletion (#14) TICKET = KAFKA-8249 LI_DESCRIPTION =

kafka allows topic deletion to complete successfully when there are pending partition reassignments of the same topics. This leads several issues: 1) pending partition reassignment of the deleted topic never comes to completion after topic is deleted. 2) onPartitionReassignment -> updateAssignedReplicasForPartition will throw out IllegalStateException for non-existing node. This in turns causes controller not to resume topic deletion for online broker and not to register broker notification handler during onBrokerStartup.

The fix cleans up pending partition reassignment during topic deletion.

EXIT_CRITERIA = TICKET [KAFKA-8249]
---
 .../kafka/controller/KafkaController.scala    |  2 +-
 .../controller/TopicDeletionManager.scala     | 15 +++++++
 .../admin/ReassignPartitionsClusterTest.scala | 41 +++++++++++++++++++
 3 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala
index f7fd9b3c22a05..c7c6bc00a0b27 100644
--- a/core/src/main/scala/kafka/controller/KafkaController.scala
+++ b/core/src/main/scala/kafka/controller/KafkaController.scala
@@ -1063,7 +1063,7 @@ class KafkaController(val config: KafkaConfig,
    *
    * @param shouldRemoveReassignment Predicate indicating which partition reassignments should be removed
    */
-  private def maybeRemoveFromZkReassignment(shouldRemoveReassignment: (TopicPartition, Seq[Int]) => Boolean): Unit = {
+  private[controller] def maybeRemoveFromZkReassignment(shouldRemoveReassignment: (TopicPartition, Seq[Int]) => Boolean): Unit = {
     if (!zkClient.reassignPartitionsInProgress())
       return
 
diff --git a/core/src/main/scala/kafka/controller/TopicDeletionManager.scala b/core/src/main/scala/kafka/controller/TopicDeletionManager.scala
index f5321258edcc0..d33af05023c50 100755
--- a/core/src/main/scala/kafka/controller/TopicDeletionManager.scala
+++ b/core/src/main/scala/kafka/controller/TopicDeletionManager.scala
@@ -31,6 +31,7 @@ trait DeletionClient {
   def sendMetadataUpdate(partitions: Set[TopicPartition]): Unit
   def createDeleteTopicFlagPath(): Unit
   def getTopicDeletionFlag(): String
+  def removePartitionsFromReassignedPartitions(partitionsToBeRemoved: Set[TopicPartition]): Unit
 }
 
 class ControllerDeletionClient(controller: KafkaController, zkClient: KafkaZkClient) extends DeletionClient {
@@ -59,6 +60,10 @@ class ControllerDeletionClient(controller: KafkaController, zkClient: KafkaZkCli
   override def getTopicDeletionFlag(): String = {
     zkClient.getTopicDeletionFlag
   }
+
+  override def removePartitionsFromReassignedPartitions(partitionsToBeRemoved: Set[TopicPartition]): Unit = {
+    controller.maybeRemoveFromZkReassignment((tp, _) => partitionsToBeRemoved.contains(tp))
+  }
 }
 
 /**
@@ -258,6 +263,8 @@ class TopicDeletionManager(config: KafkaConfig,
   }
 
   private def completeDeleteTopic(topic: String): Unit = {
+    // abort pending partition reassignments for deleted topic
+    abortPartitionReassignmentForTopic(topic)
     // deregister partition change listener on the deleted topic. This is to prevent the partition change listener
     // firing before the new topic listener when a deleted topic gets auto created
     client.mutePartitionModifications(topic)
@@ -270,6 +277,14 @@ class TopicDeletionManager(config: KafkaConfig,
     controllerContext.removeTopic(topic)
   }
 
+  private def abortPartitionReassignmentForTopic(topic: String): Unit = {
+    val partitionsBeingReassignedForDeletedTopic =
+      controllerContext.partitionsBeingReassigned.filter(_.topic().equals(topic))
+    if (partitionsBeingReassignedForDeletedTopic.nonEmpty) {
+      client.removePartitionsFromReassignedPartitions(partitionsBeingReassignedForDeletedTopic)
+    }
+  }
+
   /**
    * Invoked with the list of topics to be deleted
    * It invokes onPartitionDeletion for all partitions of a topic.
diff --git a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
index 0b0788ffd7d43..5933127157973 100644
--- a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
@@ -1229,6 +1229,47 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     testCreatePartitions(topicName, false)
   }
 
+  @Test
+  def shouldFinishReassignmentAfterTopicDeletion() {
+
+    //Given partitions on 3 of 3 brokers
+    val brokers = Array(100, 101, 102)
+
+    // start servers with topic deletion enabled
+    servers = brokers.map(i => createBrokerConfig(i, zkConnect, enableDeleteTopic = true, enableControlledShutdown = false, logDirCount = 3))
+      .map(c => createServer(KafkaConfig.fromProps(c)))
+
+    createTopic(zkClient, topicName, Map(
+      0 -> Seq(100, 101)
+    ), servers = servers)
+
+    val topicPartition = new TopicPartition(topicName, 0)
+    //Given a small throttle to delay partition reassignment
+    val initialThrottle = Throttle(1)
+
+    val numMessages = 1000
+    val msgSize = 100 * 1000
+    produceMessages(topicName, numMessages, acks = 0, msgSize)
+
+    //Start rebalance which will move replica on 100 -> replica on 102
+    val newAssignment = generateAssignment(zkClient, Array(101, 102), generateAssignmentJson(topicName), true)._1
+
+    // Start reassignment
+    ReassignPartitionsCommand.executeAssignment(zkClient, None,
+      ReassignPartitionsCommand.formatAsReassignmentJson(newAssignment, Map.empty), initialThrottle)
+
+    // make sure all brokers have received partition reassignment request
+    TestUtils.waitUntilTrue(() => servers.forall(_.getLogManager().getLog(topicPartition).isDefined),
+      "reassignment for topic not started.")
+
+    // delete the topic
+    adminZkClient.deleteTopic(topicName)
+    TestUtils.verifyTopicDeletion(zkClient, topicName, 1, servers)
+
+    //Await completion
+    waitForZkReassignmentToComplete()
+  }
+
   /**
     * Asserts that a replica is being reassigned from the given replicas to the target replicas
     */

From 57daca48b6510a4d19c2540f4464404ed7a9b6d6 Mon Sep 17 00:00:00 2001
From: Adem Efe Gencer 
Date: Mon, 9 Dec 2019 13:13:32 -0800
Subject: [PATCH 1045/1071] [LI-HOTFIX] Remove
 KafkaZkClient#setOrCreatePartitionReassignment and add
 AdminZkClient#addPartitions (#60)

TICKET =
LI_DESCRIPTION =
1) Remove "KafkaZkClient#setOrCreatePartitionReassignment(reassignment: collection.Map[TopicPartition, Seq[Int]]): Unit" function, which has been temporarily added to preserve the dependency of Cruise Control on this API. Since Cruise Control no longer depends on this API (see https://github.com/linkedin/cruise-control/commit/df52a906f5f6f6b75f5fe69bca61473f9c6dc6d1#diff-ab474ca3213f2c014b31470c9e344f36L91), the corresponding EXIT CRITERIA is satisfied and we can remove this API.
2) Add "AdminZkClient#addPartitions(topic: String, existingAssignment: Map[Int, Seq[Int]], allBrokers: Seq[BrokerMetadata], numPartitions: Int = 1, replicaAssignment: Option[Map[Int, Seq[Int]]] = None, validateOnly: Boolean = false): Map[Int, Seq[Int]]". This method ensures that the API consistency is maintained with the upstream Kafka -- i.e. Originally this API exists in Apache Kafka 2.3, but has been removed from LinkedIn Kafka 2.3.

EXIT_CRITERIA = TICKET [KAFKA-8527]

* [LI-HOTFIX] Address the feedback.
TICKET =
LI_DESCRIPTION =

EXIT_CRITERIA = MANUAL [""]
---
 .../main/scala/kafka/zk/AdminZkClient.scala   | 32 ++++++++++++++++---
 1 file changed, 28 insertions(+), 4 deletions(-)

diff --git a/core/src/main/scala/kafka/zk/AdminZkClient.scala b/core/src/main/scala/kafka/zk/AdminZkClient.scala
index 7c464fd50f60f..f3a50ea483952 100644
--- a/core/src/main/scala/kafka/zk/AdminZkClient.scala
+++ b/core/src/main/scala/kafka/zk/AdminZkClient.scala
@@ -214,6 +214,30 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
       AdminUtils.assignReplicasToBrokers(availableBrokerMetadata, nPartitions, replicationFactor, fixedStartIndex, startPartitionId)
   }
 
+  /**
+   * Add partitions to existing topic with optional replica assignment and no constraints on whether brokers can be
+   * assigned new replicas.
+   *
+   * This method ensures that the API consistency is maintained with the upstream Kafka -- i.e. Originally this API
+   * exists in Apache Kafka 2.3, but has been removed from LinkedIn Kafka 2.3.
+   *
+   * @param topic Topic for adding partitions to
+   * @param existingAssignment A map from partition id to its assigned replicas
+   * @param allBrokers All brokers in the cluster
+   * @param numPartitions Number of partitions to be set
+   * @param replicaAssignment Manual replica assignment, or none
+   * @param validateOnly If true, validate the parameters without actually adding the partitions
+   * @return the updated replica assignment
+   */
+  def addPartitions(topic: String,
+                    existingAssignment: Map[Int, ReplicaAssignment],
+                    allBrokers: Seq[BrokerMetadata],
+                    numPartitions: Int = 1,
+                    replicaAssignment: Option[Map[Int, Seq[Int]]] = None,
+                    validateOnly: Boolean = false): Map[Int, Seq[Int]] = {
+    addPartitions(topic, existingAssignment, allBrokers, numPartitions, replicaAssignment, validateOnly, Set.empty[Int])
+  }
+
   /**
   * Add partitions to existing topic with optional replica assignment
   *
@@ -229,10 +253,10 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
   def addPartitions(topic: String,
                     existingAssignment: Map[Int, ReplicaAssignment],
                     allBrokers: Seq[BrokerMetadata],
-                    numPartitions: Int = 1,
-                    replicaAssignment: Option[Map[Int, Seq[Int]]] = None,
-                    validateOnly: Boolean = false,
-                    noNewPartitionBrokerIds: Set[Int] = Set.empty[Int]): Map[Int, Seq[Int]] = {
+                    numPartitions: Int,
+                    replicaAssignment: Option[Map[Int, Seq[Int]]],
+                    validateOnly: Boolean,
+                    noNewPartitionBrokerIds: Set[Int]): Map[Int, Seq[Int]] = {
     val existingAssignmentPartition0 = existingAssignment.getOrElse(0,
       throw new AdminOperationException(
         s"Unexpected existing replica assignment for topic '$topic', partition id 0 is missing. " +

From 5814807bada496e479ac47bcb5353ddfbc9fa233 Mon Sep 17 00:00:00 2001
From: Adem Efe Gencer 
Date: Thu, 19 Dec 2019 14:57:47 -0800
Subject: [PATCH 1046/1071] [LI-HOTFIX] Guard costly trace- and debug-level
 logging to avoid unnecessary resource use during index operations. (#64)

TICKET =
LI_DESCRIPTION = PR apache#6385 has added several debug and trace statements in AbstractIndex.scala, OffsetIndex.scala, and TimeIndex.scala. However, not guarding these logging statements yields waste of resources during index operations. This patch adds log guarding to avoid this resource waste.
EXIT_CRITERIA = MANUAL [""]
---
 .../main/scala/kafka/log/AbstractIndex.scala  |  8 +++---
 .../main/scala/kafka/log/OffsetIndex.scala    | 15 ++++++-----
 core/src/main/scala/kafka/log/TimeIndex.scala | 25 +++++++++++--------
 3 files changed, 29 insertions(+), 19 deletions(-)

diff --git a/core/src/main/scala/kafka/log/AbstractIndex.scala b/core/src/main/scala/kafka/log/AbstractIndex.scala
index 007ab07133072..d742a2dbfead1 100644
--- a/core/src/main/scala/kafka/log/AbstractIndex.scala
+++ b/core/src/main/scala/kafka/log/AbstractIndex.scala
@@ -175,7 +175,8 @@ abstract class AbstractIndex(@volatile var file: File, val baseOffset: Long, val
       val roundedNewSize = roundDownToExactMultiple(newSize, entrySize)
 
       if (_length == roundedNewSize) {
-        debug(s"Index ${file.getAbsolutePath} was not resized because it already has size $roundedNewSize")
+        if (isDebugEnabled)
+          debug(s"Index ${file.getAbsolutePath} was not resized because it already has size $roundedNewSize")
         false
       } else {
         val raf = new RandomAccessFile(file, "rw")
@@ -190,8 +191,9 @@ abstract class AbstractIndex(@volatile var file: File, val baseOffset: Long, val
           mmap = raf.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, roundedNewSize)
           _maxEntries = mmap.limit() / entrySize
           mmap.position(position)
-          debug(s"Resized ${file.getAbsolutePath} to $roundedNewSize, position is ${mmap.position()} " +
-            s"and limit is ${mmap.limit()}")
+          if (isDebugEnabled)
+            debug(s"Resized ${file.getAbsolutePath} to $roundedNewSize, position is ${mmap.position()} " +
+                  s"and limit is ${mmap.limit()}")
           true
         } finally {
           CoreUtils.swallow(raf.close(), AbstractIndex)
diff --git a/core/src/main/scala/kafka/log/OffsetIndex.scala b/core/src/main/scala/kafka/log/OffsetIndex.scala
index cd90799519fce..78ac79572fa7b 100755
--- a/core/src/main/scala/kafka/log/OffsetIndex.scala
+++ b/core/src/main/scala/kafka/log/OffsetIndex.scala
@@ -59,8 +59,9 @@ class OffsetIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writabl
   /* the last offset in the index */
   private[this] var _lastOffset = lastEntry.offset
 
-  debug(s"Loaded index file ${file.getAbsolutePath} with maxEntries = $maxEntries, " +
-    s"maxIndexSize = $maxIndexSize, entries = ${_entries}, lastOffset = ${_lastOffset}, file position = ${mmap.position()}")
+  if (isDebugEnabled)
+    debug(s"Loaded index file ${file.getAbsolutePath} with maxEntries = $maxEntries, " +
+      s"maxIndexSize = $maxIndexSize, entries = ${_entries}, lastOffset = ${_lastOffset}, file position = ${mmap.position()}")
 
   /**
    * The last entry in the index
@@ -136,13 +137,14 @@ class OffsetIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writabl
 
   /**
    * Append an entry for the given offset/location pair to the index. This entry must have a larger offset than all subsequent entries.
-   * @throws IndexOffsetOverflowException if the offset causes index offset to overflow
+   * @throws kafka.common.IndexOffsetOverflowException#IndexOffsetOverflowException if the offset causes index offset to overflow
    */
   def append(offset: Long, position: Int): Unit = {
     inLock(lock) {
       require(!isFull, "Attempt to append to a full index (size = " + _entries + ").")
       if (_entries == 0 || offset > _lastOffset) {
-        trace(s"Adding index entry $offset => $position to ${file.getAbsolutePath}")
+        if (isTraceEnabled)
+          trace(s"Adding index entry $offset => $position to ${file.getAbsolutePath}")
         mmap.putInt(relativeOffset(offset))
         mmap.putInt(position)
         _entries += 1
@@ -186,8 +188,9 @@ class OffsetIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writabl
       _entries = entries
       mmap.position(_entries * entrySize)
       _lastOffset = lastEntry.offset
-      debug(s"Truncated index ${file.getAbsolutePath} to $entries entries;" +
-        s" position is now ${mmap.position()} and last offset is now ${_lastOffset}")
+      if (isDebugEnabled)
+        debug(s"Truncated index ${file.getAbsolutePath} to $entries entries;" +
+              s" position is now ${mmap.position()} and last offset is now ${_lastOffset}")
     }
   }
 
diff --git a/core/src/main/scala/kafka/log/TimeIndex.scala b/core/src/main/scala/kafka/log/TimeIndex.scala
index 40e1dcf320e7a..79516ba0a04d8 100644
--- a/core/src/main/scala/kafka/log/TimeIndex.scala
+++ b/core/src/main/scala/kafka/log/TimeIndex.scala
@@ -58,8 +58,9 @@ class TimeIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writable:
 
   override def entrySize = 12
 
-  debug(s"Loaded index file ${file.getAbsolutePath} with maxEntries = $maxEntries, maxIndexSize = $maxIndexSize," +
-    s" entries = ${_entries}, lastOffset = ${_lastEntry}, file position = ${mmap.position()}")
+  if (isDebugEnabled)
+    debug(s"Loaded index file ${file.getAbsolutePath} with maxEntries = $maxEntries, maxIndexSize = $maxIndexSize," +
+          s" entries = ${_entries}, lastOffset = ${_lastEntry}, file position = ${mmap.position()}")
 
   // We override the full check to reserve the last time index entry slot for the on roll call.
   override def isFull: Boolean = entries >= maxEntries - 1
@@ -120,17 +121,20 @@ class TimeIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writable:
       // because that could happen in the following two scenarios:
       // 1. A log segment is closed.
       // 2. LogSegment.onBecomeInactiveSegment() is called when an active log segment is rolled.
-      if (_entries != 0 && offset < lastEntry.offset)
-        throw new InvalidOffsetException(s"Attempt to append an offset ($offset) to slot ${_entries} no larger than" +
-          s" the last offset appended (${lastEntry.offset}) to ${file.getAbsolutePath}.")
-      if (_entries != 0 && timestamp < lastEntry.timestamp)
-        throw new IllegalStateException(s"Attempt to append a timestamp ($timestamp) to slot ${_entries} no larger" +
-          s" than the last timestamp appended (${lastEntry.timestamp}) to ${file.getAbsolutePath}.")
+      if (_entries != 0) {
+        if (offset < lastEntry.offset)
+          throw new InvalidOffsetException(s"Attempt to append an offset ($offset) to slot ${_entries} no larger than" +
+                                           s" the last offset appended (${lastEntry.offset}) to ${file.getAbsolutePath}.")
+        if (timestamp < lastEntry.timestamp)
+          throw new IllegalStateException(s"Attempt to append a timestamp ($timestamp) to slot ${_entries} no larger" +
+                                          s" than the last timestamp appended (${lastEntry.timestamp}) to ${file.getAbsolutePath}.")
+      }
       // We only append to the time index when the timestamp is greater than the last inserted timestamp.
       // If all the messages are in message format v0, the timestamp will always be NoTimestamp. In that case, the time
       // index will be empty.
       if (timestamp > lastEntry.timestamp) {
-        trace(s"Adding index entry $timestamp => $offset to ${file.getAbsolutePath}.")
+        if (isTraceEnabled)
+          trace(s"Adding index entry $timestamp => $offset to ${file.getAbsolutePath}.")
         mmap.putLong(timestamp)
         mmap.putInt(relativeOffset(offset))
         _entries += 1
@@ -204,7 +208,8 @@ class TimeIndex(_file: File, baseOffset: Long, maxIndexSize: Int = -1, writable:
       _entries = entries
       mmap.position(_entries * entrySize)
       _lastEntry = lastEntryFromIndexFile
-      debug(s"Truncated index ${file.getAbsolutePath} to $entries entries; position is now ${mmap.position()} and last entry is now ${_lastEntry}")
+      if (isDebugEnabled)
+        debug(s"Truncated index ${file.getAbsolutePath} to $entries entries; position is now ${mmap.position()} and last entry is now ${_lastEntry}")
     }
   }
 

From 1f7bab6ca204918186896d97d09f0c6609337347 Mon Sep 17 00:00:00 2001
From: Xiongqi Wu 
Date: Thu, 9 Jan 2020 16:22:15 -0800
Subject: [PATCH 1047/1071] [LI-HOTFIX] fix DynamicConfig and
 DynamicBrokerConfig cross reference issue (#69)

TICKET =
LI_DESCRIPTION =
DynamicBrokerConfig holds a reference to DynamicConfig.Broker.ClusterLevelConfigs.
During DynamicConfig object initialization, DynamicBrokerConfig object is created if it was not created before.

kafka-configs.sh script (ConfigCommand.scala) tries to create DynamicConfig object directly, which in turns creates DynamicBrokerConfig object.
This fix makes sure that
when initializing DynamicBrokerConfig object, DynamicConfig.Broker.ClusterLevelConfigs should have already been defined.

EXIT_CRITERIA = MANUAL [""]
---
 core/src/main/scala/kafka/server/DynamicConfig.scala | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/kafka/server/DynamicConfig.scala b/core/src/main/scala/kafka/server/DynamicConfig.scala
index e55cb492ecfc2..695b1569fb4e0 100644
--- a/core/src/main/scala/kafka/server/DynamicConfig.scala
+++ b/core/src/main/scala/kafka/server/DynamicConfig.scala
@@ -57,6 +57,9 @@ object DynamicConfig {
       s"This property can be only set dynamically. It is suggested that the limit be kept above 1MB/s for accurate behaviour."
     val MaintenanceBrokerListDoc = "A list containing maintenance broker Ids, separated by comma"
 
+    //cluster level only configs
+    val ClusterLevelConfigs = Set(MaintenanceBrokerListProp)
+
     //Definitions
     private val brokerConfigDef = new ConfigDef()
       //round minimum value down, to make it easier for users.
@@ -67,8 +70,6 @@ object DynamicConfig {
     DynamicBrokerConfig.addDynamicConfigs(brokerConfigDef)
     val nonDynamicProps = KafkaConfig.configNames.toSet -- brokerConfigDef.names.asScala
 
-    //cluster level only configs
-    val ClusterLevelConfigs = Set(MaintenanceBrokerListProp)
     def typeOf(key: String): ConfigDef.Type = {
       val configKey: ConfigDef.ConfigKey = brokerConfigDef.configKeys().get(key)
       if (configKey == null)

From 051c36f465572fd16697635924c8320b23bf6c2d Mon Sep 17 00:00:00 2001
From: kun du 
Date: Thu, 16 Jan 2020 16:49:50 -0800
Subject: [PATCH 1048/1071] [LI-HOTFIX] Fix SSL channel close on fake buffer
 overflow/underflow (#62)

    TICKET =
    LI_DESCRIPTION = SslTransportLayer takes netBuffer/appBuffer full as buffer overflow and close the connection unnecessarily.

    EXIT_CRITERIA = MANUAL [""]
---
 .../kafka/common/network/SslTransportLayer.java  | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/network/SslTransportLayer.java b/clients/src/main/java/org/apache/kafka/common/network/SslTransportLayer.java
index 4ffcc6ce32fca..adcbcb22fc688 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/SslTransportLayer.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/SslTransportLayer.java
@@ -316,9 +316,9 @@ private void doHandshake() throws IOException {
                     netWriteBuffer.compact();
                     netWriteBuffer = Utils.ensureCapacity(netWriteBuffer, currentNetWriteBufferSize);
                     netWriteBuffer.flip();
-                    if (netWriteBuffer.limit() >= currentNetWriteBufferSize) {
+                    if (netWriteBuffer.limit() > currentNetWriteBufferSize) {
                         throw new IllegalStateException("Buffer overflow when available data size (" + netWriteBuffer.limit() +
-                                                        ") >= network buffer size (" + currentNetWriteBufferSize + ")");
+                                                        ") > network buffer size (" + currentNetWriteBufferSize + ")");
                     }
                 } else if (handshakeResult.getStatus() == Status.BUFFER_UNDERFLOW) {
                     throw new IllegalStateException("Should not have received BUFFER_UNDERFLOW during handshake WRAP.");
@@ -350,7 +350,7 @@ private void doHandshake() throws IOException {
                 if (handshakeResult.getStatus() == Status.BUFFER_UNDERFLOW) {
                     int currentNetReadBufferSize = netReadBufferSize();
                     netReadBuffer = Utils.ensureCapacity(netReadBuffer, currentNetReadBufferSize);
-                    if (netReadBuffer.position() >= currentNetReadBufferSize) {
+                    if (netReadBuffer.position() > currentNetReadBufferSize) {
                         throw new IllegalStateException("Buffer underflow when there is available data");
                     }
                 } else if (handshakeResult.getStatus() == Status.CLOSED) {
@@ -547,9 +547,9 @@ public int read(ByteBuffer dst) throws IOException {
                 } else if (unwrapResult.getStatus() == Status.BUFFER_OVERFLOW) {
                     int currentApplicationBufferSize = applicationBufferSize();
                     appReadBuffer = Utils.ensureCapacity(appReadBuffer, currentApplicationBufferSize);
-                    if (appReadBuffer.position() >= currentApplicationBufferSize) {
+                    if (appReadBuffer.position() > currentApplicationBufferSize) {
                         throw new IllegalStateException("Buffer overflow when available data size (" + appReadBuffer.position() +
-                                                        ") >= application buffer size (" + currentApplicationBufferSize + ")");
+                                                        ") > application buffer size (" + currentApplicationBufferSize + ")");
                     }
 
                     // appReadBuffer will extended upto currentApplicationBufferSize
@@ -562,7 +562,7 @@ public int read(ByteBuffer dst) throws IOException {
                 } else if (unwrapResult.getStatus() == Status.BUFFER_UNDERFLOW) {
                     int currentNetReadBufferSize = netReadBufferSize();
                     netReadBuffer = Utils.ensureCapacity(netReadBuffer, currentNetReadBufferSize);
-                    if (netReadBuffer.position() >= currentNetReadBufferSize) {
+                    if (netReadBuffer.position() > currentNetReadBufferSize) {
                         throw new IllegalStateException("Buffer underflow when available data size (" + netReadBuffer.position() +
                                                         ") > packet buffer size (" + currentNetReadBufferSize + ")");
                     }
@@ -667,8 +667,8 @@ public int write(ByteBuffer src) throws IOException {
             netWriteBuffer.compact();
             netWriteBuffer = Utils.ensureCapacity(netWriteBuffer, currentNetWriteBufferSize);
             netWriteBuffer.flip();
-            if (netWriteBuffer.limit() >= currentNetWriteBufferSize)
-                throw new IllegalStateException("SSL BUFFER_OVERFLOW when available data size (" + netWriteBuffer.limit() + ") >= network buffer size (" + currentNetWriteBufferSize + ")");
+            if (netWriteBuffer.limit() > currentNetWriteBufferSize)
+                throw new IllegalStateException("SSL BUFFER_OVERFLOW when available data size (" + netWriteBuffer.limit() + ") > network buffer size (" + currentNetWriteBufferSize + ")");
         } else if (wrapResult.getStatus() == Status.BUFFER_UNDERFLOW) {
             throw new IllegalStateException("SSL BUFFER_UNDERFLOW during write");
         } else if (wrapResult.getStatus() == Status.CLOSED) {

From e554e06551bafd3963fea5f2674fc0ed084bdc9b Mon Sep 17 00:00:00 2001
From: kun du 
Date: Fri, 31 Jan 2020 13:48:03 -0800
Subject: [PATCH 1049/1071] [LI-HOTFIX] Add OneAboveMinIsrCounter sensor. (#71)

TICKET =
LI_DESCRIPTION = Add the OneAboveMinIsrCounter metrics to help determining the good timing of broker rolling bounce.
EXIT_CRITERIA = MANUAL [""]
---
 core/src/main/scala/kafka/cluster/Partition.scala     | 9 +++++++++
 core/src/main/scala/kafka/server/ReplicaManager.scala | 6 ++++++
 2 files changed, 15 insertions(+)

diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala
index a07f6c2c3b73c..d9ca19a4b05bd 100755
--- a/core/src/main/scala/kafka/cluster/Partition.scala
+++ b/core/src/main/scala/kafka/cluster/Partition.scala
@@ -267,6 +267,15 @@ class Partition(val topicPartition: TopicPartition,
     leaderLogIfLocal.exists { inSyncReplicaIds.size == _.config.minInSyncReplicas }
   }
 
+  def isOneAboveMinIsr: Boolean = {
+    leaderLogIfLocal match {
+      case Some(leaderLog) =>
+        inSyncReplicaIds.size == leaderLog.config.minInSyncReplicas + 1
+      case None =>
+        false
+    }
+  }
+
   /**
     * Create the future replica if 1) the current replica is not in the given log directory and 2) the future replica
     * does not exist. This method assumes that the current replica has already been created.
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index e82485a56ada5..16ec06a3bcabf 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -267,6 +267,12 @@ class ReplicaManager(val config: KafkaConfig,
       def value = leaderPartitionsIterator.count(_.isAtMinIsr)
     }
   )
+  val oneAboveMinIsrPartitionCount = newGauge(
+    "OneAboveMinIsrPartitionCount",
+    new Gauge[Int] {
+      def value = leaderPartitionsIterator.count(_.isOneAboveMinIsr)
+    }
+  )
 
   val isrExpandRate: Meter = newMeter("IsrExpandsPerSec", "expands", TimeUnit.SECONDS)
   val isrShrinkRate: Meter = newMeter("IsrShrinksPerSec", "shrinks", TimeUnit.SECONDS)

From f44676e3b66c2d576c0a051ca99147e764d56966 Mon Sep 17 00:00:00 2001
From: kun du 
Date: Wed, 12 Feb 2020 10:49:01 -0800
Subject: [PATCH 1050/1071] [LI-HOTFIX] Implementation of recycling memory
 pool. (#72) TICKET = LI_DESCRIPTION = EXIT_CRITERIA = MANUAL [""]

---
 .../common/memory/RecyclingMemoryPool.java    | 114 ++++++++++++++++++
 .../memory/RecyclingMemoryPoolTest.java       | 110 +++++++++++++++++
 .../scala/kafka/network/SocketServer.scala    |  10 +-
 .../main/scala/kafka/server/KafkaConfig.scala |  10 ++
 .../unit/kafka/server/KafkaConfigTest.scala   |   2 +
 5 files changed, 241 insertions(+), 5 deletions(-)
 create mode 100644 clients/src/main/java/org/apache/kafka/common/memory/RecyclingMemoryPool.java
 create mode 100644 clients/src/test/java/org/apache/kafka/common/memory/RecyclingMemoryPoolTest.java

diff --git a/clients/src/main/java/org/apache/kafka/common/memory/RecyclingMemoryPool.java b/clients/src/main/java/org/apache/kafka/common/memory/RecyclingMemoryPool.java
new file mode 100644
index 0000000000000..971f0be72972c
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/memory/RecyclingMemoryPool.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.common.memory;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.LinkedBlockingQueue;
+import org.apache.kafka.common.metrics.Sensor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * An implementation of memory pool which recycles buffers of commonly used size.
+ * This memory pool is useful if most of the requested buffers' size are within close size range.
+ * In this case, instead of deallocate and reallocate the buffer, the memory pool will recycle the buffer for future use.
+ */
+public class RecyclingMemoryPool implements MemoryPool {
+    protected static final Logger log = LoggerFactory.getLogger(RecyclingMemoryPool.class);
+    protected final int cacheableBufferSizeUpperThreshold;
+    protected final int cacheableBufferSizeLowerThreshold;
+    protected final LinkedBlockingQueue bufferCache;
+    protected final Sensor requestSensor;
+
+    public RecyclingMemoryPool(int cacheableBufferSize, int bufferCacheCapacity, Sensor requestSensor) {
+        if (bufferCacheCapacity <= 0 || cacheableBufferSize <= 0) {
+            throw new IllegalArgumentException(String.format("Must provide a positive cacheable buffer size and buffer cache " +
+                    "capacity, provided %d and %d respectively.", cacheableBufferSize, bufferCacheCapacity));
+        }
+        this.bufferCache = new LinkedBlockingQueue<>(bufferCacheCapacity);
+        for (int i = 0; i < bufferCacheCapacity; i++) {
+            bufferCache.offer(ByteBuffer.allocate(cacheableBufferSize));
+        }
+        this.cacheableBufferSizeUpperThreshold = cacheableBufferSize;
+        this.cacheableBufferSizeLowerThreshold = cacheableBufferSize / 2;
+        this.requestSensor = requestSensor;
+    }
+
+    @Override
+    public ByteBuffer tryAllocate(int sizeBytes) {
+        if (sizeBytes < 1) {
+            throw new IllegalArgumentException("requested size " + sizeBytes + "<=0");
+        }
+
+        ByteBuffer allocated = null;
+        if (sizeBytes > cacheableBufferSizeLowerThreshold  && sizeBytes <= cacheableBufferSizeUpperThreshold) {
+            allocated = bufferCache.poll();
+        }
+        if (allocated != null) {
+            allocated.limit(sizeBytes);
+        } else {
+            allocated = ByteBuffer.allocate(sizeBytes);
+        }
+        bufferToBeAllocated(allocated);
+        return allocated;
+    }
+
+    @Override
+    public void release(ByteBuffer previouslyAllocated) {
+        if (previouslyAllocated == null) {
+            throw new IllegalArgumentException("provided null buffer");
+        }
+        if (previouslyAllocated.capacity() == cacheableBufferSizeUpperThreshold) {
+            previouslyAllocated.clear();
+            bufferCache.offer(previouslyAllocated);
+        } else {
+            bufferToBeReleased(previouslyAllocated);
+        }
+    }
+
+    //allows subclasses to do their own bookkeeping (and validation) _before_ memory is returned to client code.
+    protected void bufferToBeAllocated(ByteBuffer justAllocated) {
+        try {
+            this.requestSensor.record(justAllocated.limit());
+        } catch (Exception e) {
+            log.debug("failed to record size of allocated buffer");
+        }
+        log.trace("allocated buffer of size {}", justAllocated.capacity());
+    }
+
+    //allows subclasses to do their own bookkeeping (and validation) _before_ memory is marked as reclaimed.
+    protected void bufferToBeReleased(ByteBuffer justReleased) {
+        log.trace("released buffer of size {}", justReleased.capacity());
+    }
+
+    @Override
+    public long size() {
+        return Long.MAX_VALUE;
+    }
+
+    @Override
+    public long availableMemory() {
+        return Long.MAX_VALUE;
+    }
+
+    @Override
+    public boolean isOutOfMemory() {
+        return false;
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/common/memory/RecyclingMemoryPoolTest.java b/clients/src/test/java/org/apache/kafka/common/memory/RecyclingMemoryPoolTest.java
new file mode 100644
index 0000000000000..eec9e7e41668c
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/memory/RecyclingMemoryPoolTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.common.memory;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.kafka.common.metrics.Metrics;
+import org.apache.kafka.common.metrics.Sensor;
+import org.junit.Assert;
+import org.junit.Test;
+
+
+public class RecyclingMemoryPoolTest {
+    private static final int TWO_KILOBYTES = 2048;
+    private static final int CACHEABLE_BUFFER_SIZE = 1024;
+    private static final int BUFFER_CACHE_CAPACITY = 2;
+    private static final Sensor ALLOCATE_SENSOR = new Metrics().sensor("allocate_sensor");
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testNegativeAllocation() {
+        RecyclingMemoryPool memoryPool = new RecyclingMemoryPool(CACHEABLE_BUFFER_SIZE, BUFFER_CACHE_CAPACITY, ALLOCATE_SENSOR);
+        memoryPool.tryAllocate(-1);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testZeroAllocation() {
+        RecyclingMemoryPool memoryPool = new RecyclingMemoryPool(CACHEABLE_BUFFER_SIZE, BUFFER_CACHE_CAPACITY, ALLOCATE_SENSOR);
+        memoryPool.tryAllocate(0);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testNullRelease() {
+        RecyclingMemoryPool memoryPool = new RecyclingMemoryPool(CACHEABLE_BUFFER_SIZE, BUFFER_CACHE_CAPACITY, ALLOCATE_SENSOR);
+        memoryPool.release(null);
+    }
+
+    @Test
+    public void testAllocation() {
+        RecyclingMemoryPool memoryPool = new RecyclingMemoryPool(CACHEABLE_BUFFER_SIZE, BUFFER_CACHE_CAPACITY, ALLOCATE_SENSOR);
+        ByteBuffer buffer1 = memoryPool.tryAllocate(TWO_KILOBYTES);
+        ByteBuffer buffer2 = memoryPool.tryAllocate(CACHEABLE_BUFFER_SIZE);
+        ByteBuffer buffer3 = memoryPool.tryAllocate(CACHEABLE_BUFFER_SIZE * 2 / 3);
+        ByteBuffer buffer4 = memoryPool.tryAllocate(CACHEABLE_BUFFER_SIZE);
+
+        memoryPool.release(buffer1);
+        ByteBuffer reuse1 = memoryPool.tryAllocate(TWO_KILOBYTES);
+        // Compare the references
+        Assert.assertNotEquals(System.identityHashCode(reuse1), System.identityHashCode(buffer1));
+
+        memoryPool.release(buffer2);
+        memoryPool.release(buffer3);
+        memoryPool.release(buffer4);
+        ByteBuffer reuse2 = memoryPool.tryAllocate(CACHEABLE_BUFFER_SIZE);
+        ByteBuffer reuse3 = memoryPool.tryAllocate(CACHEABLE_BUFFER_SIZE * 2 / 3);
+        ByteBuffer reuse4 = memoryPool.tryAllocate(CACHEABLE_BUFFER_SIZE);
+
+        Assert.assertEquals(System.identityHashCode(reuse2), System.identityHashCode(buffer2));
+        Assert.assertEquals(System.identityHashCode(reuse3), System.identityHashCode(buffer3));
+        Assert.assertNotEquals(System.identityHashCode(reuse4), System.identityHashCode(buffer4));
+    }
+
+    @Test
+    public void testMultiThreadAllocation() {
+        RecyclingMemoryPool memoryPool = new RecyclingMemoryPool(CACHEABLE_BUFFER_SIZE, BUFFER_CACHE_CAPACITY, ALLOCATE_SENSOR);
+        AtomicReference error = new AtomicReference<>();
+        List processorThreads = new ArrayList<>(3);
+        for (int i = 0; i < 3; i++) {
+            processorThreads.add(new Thread(() -> {
+                try {
+                    ByteBuffer buffer = memoryPool.tryAllocate(CACHEABLE_BUFFER_SIZE);
+                    Thread.sleep(1000);
+                    memoryPool.release(buffer);
+                } catch (InterruptedException e) {
+                    error.compareAndSet(null, e);
+                }
+            }));
+        }
+        processorThreads.forEach(t -> {
+            t.setDaemon(true);
+            t.start();
+        });
+        processorThreads.forEach(t -> {
+            try {
+                t.join(30000);
+            } catch (InterruptedException e) {
+                error.compareAndSet(null, e);
+            }
+        });
+
+        Assert.assertNull(error.get());
+        Assert.assertEquals(memoryPool.bufferCache.size(), BUFFER_CACHE_CAPACITY);
+    }
+}
diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala
index eb6fafd6a456b..48d2a5000e337 100644
--- a/core/src/main/scala/kafka/network/SocketServer.scala
+++ b/core/src/main/scala/kafka/network/SocketServer.scala
@@ -38,7 +38,7 @@ import kafka.server.{BrokerReconfigurable, KafkaConfig}
 import kafka.utils._
 import org.apache.kafka.common.config.ConfigException
 import org.apache.kafka.common.{Endpoint, KafkaException, Reconfigurable}
-import org.apache.kafka.common.memory.{MemoryPool, SimpleMemoryPool}
+import org.apache.kafka.common.memory.{MemoryPool, RecyclingMemoryPool, SimpleMemoryPool}
 import org.apache.kafka.common.metrics._
 import org.apache.kafka.common.metrics.stats.Percentiles.BucketSizing
 import org.apache.kafka.common.metrics.stats.{CumulativeSum, Max, Meter, Percentile, Percentiles}
@@ -89,13 +89,13 @@ class SocketServer(val config: KafkaConfig,
   private val memoryPoolDepletedTimeMetricName = metrics.metricName("MemoryPoolDepletedTimeTotal", MetricsGroup)
   memoryPoolUsageSensor.add(new Meter(TimeUnit.MILLISECONDS, memoryPoolDepletedPercentMetricName, memoryPoolDepletedTimeMetricName))
   private val memoryPoolAllocationSensor = metrics.sensor("MemoryPoolAllocation")
-  private val memoryPoolMaxAllocateSizeMetricName = metrics.metricName("MemoryPoolMaxAllocateSize", "socket-server-metrics")
-  private val memoryPoolAllocateSizePercentilesMetricName = metrics.metricName("MemoryPoolAllocateSizePercentiles", "socket-server-metrics")
-  private val percentiles = (1 to 9).map( i => new Percentile(metrics.metricName("MemoryPoolAllocateSize%dPercentile".format(i * 10), "socket-server-metrics"), i * 10))
+  private val memoryPoolMaxAllocateSizeMetricName = metrics.metricName("MemoryPoolMaxAllocateSize", MetricsGroup)
   memoryPoolAllocationSensor.add(memoryPoolMaxAllocateSizeMetricName, new Max())
+  private val percentiles = (1 to 9).map( i => new Percentile(metrics.metricName("MemoryPoolAllocateSize%dPercentile".format(i * 10), MetricsGroup), i * 10))
   // At current stage, we do not know the max decrypted request size, temporarily set it to 10MB.
-  memoryPoolAllocationSensor.add(memoryPoolAllocateSizePercentilesMetricName, new Percentiles(400, 0.0, 10485760, BucketSizing.CONSTANT, percentiles:_*))
+  memoryPoolAllocationSensor.add(new Percentiles(400, 0.0, 10485760, BucketSizing.CONSTANT, percentiles:_*))
   private val memoryPool = if (config.queuedMaxBytes > 0) new SimpleMemoryPool(config.queuedMaxBytes, config.socketRequestMaxBytes, false, memoryPoolUsageSensor, memoryPoolAllocationSensor)
+                           else if (config.socketRequestCommonBytes > 0) new RecyclingMemoryPool(config.socketRequestCommonBytes, config.socketRequestBufferCacheSize, memoryPoolAllocationSensor)
                            else MemoryPool.NONE
   // data-plane
   private val dataPlaneProcessors = new ConcurrentHashMap[Int, Processor]()
diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index e4b50379af065..f3ba43b960890 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -82,6 +82,8 @@ object Defaults {
   val SocketSendBufferBytes: Int = 100 * 1024
   val SocketReceiveBufferBytes: Int = 100 * 1024
   val SocketRequestMaxBytes: Int = 100 * 1024 * 1024
+  val SocketRequestCommonBytes: Int = -1
+  val SocketRequestBufferCacheSize: Int = 0
   val RequestMaxLocalTimeMs = Long.MaxValue
   val MaxConnectionsPerIp: Int = Int.MaxValue
   val MaxConnectionsPerIpOverrides: String = ""
@@ -318,6 +320,8 @@ object KafkaConfig {
   val SocketReceiveBufferBytesProp = "socket.receive.buffer.bytes"
   val RequestMaxLocalTimeMsProp = "request.max.local.time.ms"
   val SocketRequestMaxBytesProp = "socket.request.max.bytes"
+  val SocketRequestCommonBytesProp = "socket.request.common.bytes"
+  val SocketRequestBufferCacheSizeProp = "socket.request.buffer.cache.size"
   val MaxConnectionsPerIpProp = "max.connections.per.ip"
   val MaxConnectionsPerIpOverridesProp = "max.connections.per.ip.overrides"
   val MaxConnectionsProp = "max.connections"
@@ -613,6 +617,8 @@ object KafkaConfig {
     "takes longer than this time the broker will kill itself as violating this timeout is a symptom of a more serious broker zombie state." +
     " It is useful to observe the RequestDequeuePollIntervalMs metric to find a suitable setting for this configuration."
   val SocketRequestMaxBytesDoc = "The maximum number of bytes in a socket request"
+  val SocketRequestCommonBytesDoc = "The common size in bytes of a socket request"
+  val SocketRequestBufferCacheSizeDoc = "The maximal number of cache slot recycling memory pool will keep"
   val MaxConnectionsPerIpDoc = "The maximum number of connections we allow from each ip address. This can be set to 0 if there are overrides " +
     s"configured using $MaxConnectionsPerIpOverridesProp property. New connections from the ip address are dropped if the limit is reached."
   val MaxConnectionsPerIpOverridesDoc = "A comma-separated list of per-ip or hostname overrides to the default maximum number of connections. " +
@@ -945,6 +951,8 @@ object KafkaConfig {
       .define(RequestMaxLocalTimeMsProp, LONG, Defaults.RequestMaxLocalTimeMs, atLeast(1), MEDIUM, RequestMaxLocalTimeMsDoc)
       .define(SocketReceiveBufferBytesProp, INT, Defaults.SocketReceiveBufferBytes, HIGH, SocketReceiveBufferBytesDoc)
       .define(SocketRequestMaxBytesProp, INT, Defaults.SocketRequestMaxBytes, atLeast(1), HIGH, SocketRequestMaxBytesDoc)
+      .define(SocketRequestCommonBytesProp, INT, Defaults.SocketRequestCommonBytes, MEDIUM, SocketRequestCommonBytesDoc)
+      .define(SocketRequestBufferCacheSizeProp, INT, Defaults.SocketRequestBufferCacheSize, atLeast(0), MEDIUM, SocketRequestBufferCacheSizeDoc)
       .define(MaxConnectionsPerIpProp, INT, Defaults.MaxConnectionsPerIp, atLeast(0), MEDIUM, MaxConnectionsPerIpDoc)
       .define(MaxConnectionsPerIpOverridesProp, STRING, Defaults.MaxConnectionsPerIpOverrides, MEDIUM, MaxConnectionsPerIpOverridesDoc)
       .define(MaxConnectionsProp, INT, Defaults.MaxConnections, atLeast(0), MEDIUM, MaxConnectionsDoc)
@@ -1263,6 +1271,8 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO
   val socketSendBufferBytes = getInt(KafkaConfig.SocketSendBufferBytesProp)
   val socketReceiveBufferBytes = getInt(KafkaConfig.SocketReceiveBufferBytesProp)
   val socketRequestMaxBytes = getInt(KafkaConfig.SocketRequestMaxBytesProp)
+  val socketRequestCommonBytes = getInt(KafkaConfig.SocketRequestCommonBytesProp)
+  val socketRequestBufferCacheSize = getInt(KafkaConfig.SocketRequestBufferCacheSizeProp)
   val requestMaxLocalTimeMs = getLong(KafkaConfig.RequestMaxLocalTimeMsProp)
   val maxConnectionsPerIp = getInt(KafkaConfig.MaxConnectionsPerIpProp)
   val maxConnectionsPerIpOverrides: Map[String, Int] =
diff --git a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
index 611dbe81aa2a9..afbb7dcba0b40 100755
--- a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
@@ -611,6 +611,8 @@ class KafkaConfigTest {
         case KafkaConfig.AdvertisedPortProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_number")
         case KafkaConfig.SocketSendBufferBytesProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_number")
         case KafkaConfig.SocketReceiveBufferBytesProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_number")
+        case KafkaConfig.SocketRequestCommonBytesProp =>
+        case KafkaConfig.SocketRequestBufferCacheSizeProp =>
         case KafkaConfig.MaxConnectionsPerIpOverridesProp =>
           assertPropertyInvalid(getBaseProperties(), name, "127.0.0.1:not_a_number")
         case KafkaConfig.ConnectionsMaxIdleMsProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_number")

From b0eca2ab0e0cb01c11f38b44ef0bb074016579fd Mon Sep 17 00:00:00 2001
From: Gardner Vickers 
Date: Wed, 25 Mar 2020 20:53:42 -0700
Subject: [PATCH 1051/1071] [LI-CHERRY-PICK] [8cf781e] MINOR: Improve
 performance of checkpointHighWatermarks, patch 1/2 (#6741) TICKET =
 LI_DESCRIPTION =

This PR works to improve high watermark checkpointing performance.

`ReplicaManager.checkpointHighWatermarks()` was found to be a major contributor to GC pressure, especially on Kafka clusters with high partition counts and low throughput.

Added a JMH benchmark for `checkpointHighWatermarks` which establishes a
performance baseline. The parameterized benchmark was run with 100, 1000 and
2000 topics.

Modified `ReplicaManager.checkpointHighWatermarks()` to avoid extra copies and cached
the Log parent directory Sting to avoid frequent allocations when calculating
`File.getParent()`.

A few clean-ups:
* Changed all usages of Log.dir.getParent to Log.parentDir and Log.dir.getParentFile to
Log.parentDirFile.
* Only expose public accessor for `Log.dir` (consistent with `Log.parentDir`)
* Removed unused parameters in `Partition.makeLeader`, `Partition.makeFollower` and `Partition.createLogIfNotExists`.

Benchmark results:

| Topic Count | Ops/ms | MB/sec allocated |
|-------------|---------|------------------|
| 100               | + 51%    |  - 91% |
| 1000             | + 143% |  - 49% |
| 2000            | + 149% |   - 50% |

Reviewers: Lucas Bradstreet . Ismael Juma 

Co-authored-by: Gardner Vickers 
Co-authored-by: Ismael Juma 

EXIT_CRITERIA = MANUAL [""]
---
 .gitignore                                    |   2 +
 build.gradle                                  |   6 +-
 checkstyle/import-control-jmh-benchmarks.xml  |   6 +
 checkstyle/import-control.xml                 |  11 --
 .../main/scala/kafka/cluster/Partition.scala  |  38 ++--
 core/src/main/scala/kafka/log/Log.scala       |  63 ++++---
 .../src/main/scala/kafka/log/LogCleaner.scala |   6 +-
 .../scala/kafka/log/LogCleanerManager.scala   |   6 +-
 .../src/main/scala/kafka/log/LogManager.scala |  38 ++--
 .../scala/kafka/server/ReplicaManager.scala   |  44 +++--
 .../unit/kafka/cluster/PartitionTest.scala    |  97 ++++------
 .../kafka/server/ReplicaManagerTest.scala     |  45 +++--
 gradle/spotbugs-exclude.xml                   |   5 +-
 .../UpdateFollowerFetchStateBenchmark.java    |   2 +-
 .../server/HighwatermarkCheckpointBench.java  | 174 ++++++++++++++++++
 15 files changed, 351 insertions(+), 192 deletions(-)
 create mode 100644 jmh-benchmarks/src/main/java/org/apache/kafka/jmh/server/HighwatermarkCheckpointBench.java

diff --git a/.gitignore b/.gitignore
index 4e717b9549772..ff7f9f07b5208 100644
--- a/.gitignore
+++ b/.gitignore
@@ -54,3 +54,5 @@ systest/
 clients/src/generated
 clients/src/generated-test
 streams/src/generated/
+jmh-benchmarks/generated
+jmh-benchmarks/src/main/generated
diff --git a/build.gradle b/build.gradle
index e869d364b4700..62e196d42635a 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1545,10 +1545,14 @@ project(':jmh-benchmarks') {
     compile project(':core')
     compile project(':clients')
     compile project(':streams')
+    compile project(':core')
+    compile project(':clients').sourceSets.test.output
+    compile project(':core').sourceSets.test.output
     compile libs.jmhCore
-    compile libs.mockitoCore
     annotationProcessor libs.jmhGeneratorAnnProcess
     compile libs.jmhCoreBenchmarks
+    compile libs.mockitoCore
+    compile libs.slf4jlog4j
   }
 
   jar {
diff --git a/checkstyle/import-control-jmh-benchmarks.xml b/checkstyle/import-control-jmh-benchmarks.xml
index 49bd8c786920d..270d47a926343 100644
--- a/checkstyle/import-control-jmh-benchmarks.xml
+++ b/checkstyle/import-control-jmh-benchmarks.xml
@@ -36,6 +36,12 @@
     
     
     
+    
+    
+    
+    
+    
+    
     
     
 
diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml
index d9235396cbddc..fe567794c23ce 100644
--- a/checkstyle/import-control.xml
+++ b/checkstyle/import-control.xml
@@ -281,17 +281,6 @@
     
   
 
-  
-    
-    
-    
-    
-    
-    
-    
-    
-  
-
   
     
     
diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala
index d9ca19a4b05bd..2d28aa12b69bc 100755
--- a/core/src/main/scala/kafka/cluster/Partition.scala
+++ b/core/src/main/scala/kafka/cluster/Partition.scala
@@ -20,7 +20,7 @@ import com.yammer.metrics.core.Gauge
 import java.util.concurrent.locks.ReentrantReadWriteLock
 import java.util.{Optional, Properties}
 
-import kafka.api.{ApiVersion, LeaderAndIsr, Request}
+import kafka.api.{ApiVersion, LeaderAndIsr}
 import kafka.common.UnexpectedAppendOffsetException
 import kafka.controller.KafkaController
 import kafka.log._
@@ -289,7 +289,7 @@ class Partition(val topicPartition: TopicPartition,
     // current replica and the existence of the future replica, no other thread can update the log directory of the
     // current replica or remove the future replica.
     inWriteLock(leaderIsrUpdateLock) {
-      val currentLogDir = localLogOrException.dir.getParent
+      val currentLogDir = localLogOrException.parentDir
       if (currentLogDir == logDir) {
         info(s"Current log directory $currentLogDir is same as requested log dir $logDir. " +
           s"Skipping future replica creation.")
@@ -297,40 +297,40 @@ class Partition(val topicPartition: TopicPartition,
       } else {
         futureLog match {
           case Some(partitionFutureLog) =>
-            val futureLogDir = partitionFutureLog.dir.getParent
+            val futureLogDir = partitionFutureLog.parentDir
             if (futureLogDir != logDir)
               throw new IllegalStateException(s"The future log dir $futureLogDir of $topicPartition is " +
                 s"different from the requested log dir $logDir")
             false
           case None =>
-            createLogIfNotExists(Request.FutureLocalReplicaId, isNew = false, isFutureReplica = true, highWatermarkCheckpoints)
+            createLogIfNotExists(isNew = false, isFutureReplica = true, highWatermarkCheckpoints)
             true
         }
       }
     }
   }
 
-  def createLogIfNotExists(replicaId: Int, isNew: Boolean, isFutureReplica: Boolean, offsetCheckpoints: OffsetCheckpoints): Unit = {
+  def createLogIfNotExists(isNew: Boolean, isFutureReplica: Boolean, offsetCheckpoints: OffsetCheckpoints): Unit = {
     isFutureReplica match {
       case true if futureLog.isEmpty =>
-        val log = createLog(replicaId, isNew, isFutureReplica, offsetCheckpoints)
+        val log = createLog(isNew, isFutureReplica, offsetCheckpoints)
         this.futureLog = Option(log)
       case false if log.isEmpty =>
-        val log = createLog(replicaId, isNew, isFutureReplica, offsetCheckpoints)
+        val log = createLog(isNew, isFutureReplica, offsetCheckpoints)
         this.log = Option(log)
       case _ => trace(s"${if (isFutureReplica) "Future Log" else "Log"} already exists.")
     }
   }
 
   // Visible for testing
-  private[cluster] def createLog(replicaId: Int, isNew: Boolean, isFutureReplica: Boolean, offsetCheckpoints: OffsetCheckpoints): Log = {
-    val fetchLogConfig = () => {
+  private[cluster] def createLog(isNew: Boolean, isFutureReplica: Boolean, offsetCheckpoints: OffsetCheckpoints): Log = {
+    def fetchLogConfig: LogConfig = {
       val props = stateStore.fetchTopicConfig()
       LogConfig.fromProps(logManager.currentDefaultConfig.originals, props)
     }
 
     def updateHighWatermark(log: Log) = {
-      val checkpointHighWatermark = offsetCheckpoints.fetch(log.dir.getParent, topicPartition).getOrElse {
+      val checkpointHighWatermark = offsetCheckpoints.fetch(log.parentDir, topicPartition).getOrElse {
         info(s"No checkpointed highwatermark is found for partition $topicPartition")
         0L
       }
@@ -341,12 +341,12 @@ class Partition(val topicPartition: TopicPartition,
     logManager.initializingLog(topicPartition)
     var maybeLog: Option[Log] = None
     try {
-      val log = logManager.getOrCreateLog(topicPartition, () => fetchLogConfig() : LogConfig, isNew, isFutureReplica)
+      val log = logManager.getOrCreateLog(topicPartition, () => fetchLogConfig, isNew, isFutureReplica)
       maybeLog = Some(log)
       updateHighWatermark(log)
       log
     } finally {
-      logManager.finishedInitializingLog(topicPartition, maybeLog, fetchLogConfig)
+      logManager.finishedInitializingLog(topicPartition, maybeLog, () => fetchLogConfig)
     }
   }
 
@@ -437,7 +437,7 @@ class Partition(val topicPartition: TopicPartition,
 
   def futureReplicaDirChanged(newDestinationDir: String): Boolean = {
     inReadLock(leaderIsrUpdateLock) {
-      futureLog.exists(_.dir.getParent != newDestinationDir)
+      futureLog.exists(_.parentDir != newDestinationDir)
     }
   }
 
@@ -502,9 +502,7 @@ class Partition(val topicPartition: TopicPartition,
    * from the time when this broker was the leader last time) and setting the new leader and ISR.
    * If the leader replica id does not change, return false to indicate the replica manager.
    */
-  def makeLeader(controllerId: Int,
-                 partitionState: LeaderAndIsrPartitionState,
-                 correlationId: Int,
+  def makeLeader(partitionState: LeaderAndIsrPartitionState,
                  highWatermarkCheckpoints: OffsetCheckpoints): Boolean = {
     val (leaderHWIncremented, isNewLeader) = inWriteLock(leaderIsrUpdateLock) {
       // record the epoch of the controller that made the leadership decision. This is useful while updating the isr
@@ -515,7 +513,7 @@ class Partition(val topicPartition: TopicPartition,
         assignment = partitionState.replicas.asScala.iterator.map(_.toInt).toSeq,
         isr = partitionState.isr.asScala.iterator.map(_.toInt).toSet
       )
-      createLogIfNotExists(localBrokerId, partitionState.isNew, isFutureReplica = false, highWatermarkCheckpoints)
+      createLogIfNotExists(partitionState.isNew, isFutureReplica = false, highWatermarkCheckpoints)
 
       val leaderLog = localLogOrException
       val leaderEpochStartOffset = leaderLog.logEndOffset
@@ -571,9 +569,7 @@ class Partition(val topicPartition: TopicPartition,
    *  greater (that is, no updates have been missed), return false to indicate to the
     * replica manager that state is already correct and the become-follower steps can be skipped
    */
-  def makeFollower(controllerId: Int,
-                   partitionState: LeaderAndIsrPartitionState,
-                   correlationId: Int,
+  def makeFollower(partitionState: LeaderAndIsrPartitionState,
                    highWatermarkCheckpoints: OffsetCheckpoints): Boolean = {
     inWriteLock(leaderIsrUpdateLock) {
       val newLeaderBrokerId = partitionState.leader
@@ -586,7 +582,7 @@ class Partition(val topicPartition: TopicPartition,
         assignment = partitionState.replicas.asScala.iterator.map(_.toInt).toSeq,
         isr = Set.empty[Int]
       )
-      createLogIfNotExists(localBrokerId, partitionState.isNew, isFutureReplica = false, highWatermarkCheckpoints)
+      createLogIfNotExists(partitionState.isNew, isFutureReplica = false, highWatermarkCheckpoints)
 
       leaderEpoch = partitionState.leaderEpoch
       leaderEpochStartOffsetOpt = None
diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index f34160fccff2d..00974403b756b 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -189,7 +189,7 @@ object RollParams {
  * New log segments are created according to a configurable policy that controls the size in bytes or time interval
  * for a given segment.
  *
- * @param dir The directory in which log segments are created.
+ * @param _dir The directory in which log segments are created.
  * @param config The log configuration settings
  * @param logStartOffset The earliest offset allowed to be exposed to kafka client.
  *                       The logStartOffset can be updated by :
@@ -210,7 +210,7 @@ object RollParams {
  * @param producerIdExpirationCheckIntervalMs How often to check for producer ids which need to be expired
  */
 @threadsafe
-class Log(@volatile var dir: File,
+class Log(@volatile private var _dir: File,
           @volatile var config: LogConfig,
           @volatile var logStartOffset: Long,
           @volatile var recoveryPoint: Long,
@@ -229,36 +229,17 @@ class Log(@volatile var dir: File,
 
   /* A lock that guards all modifications to the log */
   private val lock = new Object
+
   // The memory mapped buffer for index files of this log will be closed with either delete() or closeHandlers()
   // After memory mapped buffer is closed, no disk IO operation should be performed for this log
   @volatile private var isMemoryMappedBufferClosed = false
 
+  // Cache value of parent directory to avoid allocations in hot paths like ReplicaManager.checkpointHighWatermarks
+  @volatile private var _parentDir: String = dir.getParent
+
   /* last time it was flushed */
   private val lastFlushedTime = new AtomicLong(time.milliseconds)
 
-  def initFileSize: Int = {
-    if (config.preallocate)
-      config.segmentSize
-    else
-      0
-  }
-
-  def updateConfig(newConfig: LogConfig): Unit = {
-    val oldConfig = this.config
-    this.config = newConfig
-    val oldRecordVersion = oldConfig.messageFormatVersion.recordVersion
-    val newRecordVersion = newConfig.messageFormatVersion.recordVersion
-    if (newRecordVersion.precedes(oldRecordVersion))
-      warn(s"Record format version has been downgraded from $oldRecordVersion to $newRecordVersion.")
-    if (newRecordVersion.value != oldRecordVersion.value)
-      initializeLeaderEpochCache()
-  }
-
-  private def checkIfMemoryMappedBufferClosed(): Unit = {
-    if (isMemoryMappedBufferClosed)
-      throw new KafkaStorageException(s"The memory mapped buffer for log of $topicPartition is already closed")
-  }
-
   @volatile private var nextOffsetMetadata: LogOffsetMetadata = _
 
   /* The earliest offset which is part of an incomplete transaction. This is used to compute the
@@ -317,6 +298,35 @@ class Log(@volatile var dir: File,
       s"log end offset $logEndOffset in ${time.milliseconds() - startMs} ms")
   }
 
+  def dir: File = _dir
+
+  def parentDir: String = _parentDir
+
+  def parentDirFile: File = new File(_parentDir)
+
+  def initFileSize: Int = {
+    if (config.preallocate)
+      config.segmentSize
+    else
+      0
+  }
+
+  def updateConfig(newConfig: LogConfig): Unit = {
+    val oldConfig = this.config
+    this.config = newConfig
+    val oldRecordVersion = oldConfig.messageFormatVersion.recordVersion
+    val newRecordVersion = newConfig.messageFormatVersion.recordVersion
+    if (newRecordVersion.precedes(oldRecordVersion))
+      warn(s"Record format version has been downgraded from $oldRecordVersion to $newRecordVersion.")
+    if (newRecordVersion.value != oldRecordVersion.value)
+      initializeLeaderEpochCache()
+  }
+
+  private def checkIfMemoryMappedBufferClosed(): Unit = {
+    if (isMemoryMappedBufferClosed)
+      throw new KafkaStorageException(s"The memory mapped buffer for log of $topicPartition is already closed")
+  }
+
   def highWatermark: Long = highWatermarkMetadata.messageOffset
 
   /**
@@ -984,7 +994,8 @@ class Log(@volatile var dir: File,
         val renamedDir = new File(dir.getParent, name)
         Utils.atomicMoveWithFallback(dir.toPath, renamedDir.toPath)
         if (renamedDir != dir) {
-          dir = renamedDir
+          _dir = renamedDir
+          _parentDir = renamedDir.getParent
           logSegments.foreach(_.updateDir(renamedDir))
           producerStateManager.logDir = dir
           // re-initialize leader epoch cache so that LeaderEpochCheckpointFile.checkpoint can correctly reference
diff --git a/core/src/main/scala/kafka/log/LogCleaner.scala b/core/src/main/scala/kafka/log/LogCleaner.scala
index cf74520d3499e..27be737bc86e7 100644
--- a/core/src/main/scala/kafka/log/LogCleaner.scala
+++ b/core/src/main/scala/kafka/log/LogCleaner.scala
@@ -334,7 +334,7 @@ class LogCleaner(initialConfig: CleanerConfig,
       } catch {
         case e: LogCleaningException =>
           warn(s"Unexpected exception thrown when cleaning log ${e.log}. Marking its partition (${e.log.topicPartition}) as uncleanable", e)
-          cleanerManager.markPartitionUncleanable(e.log.dir.getParent, e.log.topicPartition)
+          cleanerManager.markPartitionUncleanable(e.log.parentDir, e.log.topicPartition)
 
           false
       }
@@ -384,11 +384,11 @@ class LogCleaner(initialConfig: CleanerConfig,
         case _: LogCleaningAbortedException => // task can be aborted, let it go.
         case _: KafkaStorageException => // partition is already offline. let it go.
         case e: IOException =>
-          val logDirectory = cleanable.log.dir.getParent
+          val logDirectory = cleanable.log.parentDir
           val msg = s"Failed to clean up log for ${cleanable.topicPartition} in dir ${logDirectory} due to IOException"
           logDirFailureChannel.maybeAddOfflineLogDir(logDirectory, msg, e)
       } finally {
-        cleanerManager.doneCleaning(cleanable.topicPartition, cleanable.log.dir.getParentFile, endOffset)
+        cleanerManager.doneCleaning(cleanable.topicPartition, cleanable.log.parentDirFile, endOffset)
       }
     }
 
diff --git a/core/src/main/scala/kafka/log/LogCleanerManager.scala b/core/src/main/scala/kafka/log/LogCleanerManager.scala
index c929a9d80c153..99287cf403fc0 100755
--- a/core/src/main/scala/kafka/log/LogCleanerManager.scala
+++ b/core/src/main/scala/kafka/log/LogCleanerManager.scala
@@ -194,7 +194,7 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
             val offsetsToClean = cleanableOffsets(log, lastCleanOffset, now)
             // update checkpoint for logs with invalid checkpointed offsets
             if (offsetsToClean.forceUpdateCheckpoint)
-              updateCheckpoints(log.dir.getParentFile(), Option(topicPartition, offsetsToClean.firstDirtyOffset))
+              updateCheckpoints(log.parentDirFile, Option(topicPartition, offsetsToClean.firstDirtyOffset))
             val compactionDelayMs = maxCompactionDelay(log, offsetsToClean.firstDirtyOffset, now)
             preCleanStats.updateMaxCompactionDelay(compactionDelayMs)
 
@@ -396,7 +396,7 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
           case Some(offset) =>
             // Remove this partition from the checkpoint file in the source log directory
             updateCheckpoints(sourceLogDir, None)
-            // Add offset for this partition to the checkpoint file in the source log directory
+            // Add offset for this partition to the checkpoint file in the destination log directory
             updateCheckpoints(destLogDir, Option(topicPartition, offset))
           case None =>
         }
@@ -495,7 +495,7 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
 
   private def isUncleanablePartition(log: Log, topicPartition: TopicPartition): Boolean = {
     inLock(lock) {
-      uncleanablePartitions.get(log.dir.getParent).exists(partitions => partitions.contains(topicPartition))
+      uncleanablePartitions.get(log.parentDir).exists(partitions => partitions.contains(topicPartition))
     }
   }
 }
diff --git a/core/src/main/scala/kafka/log/LogManager.scala b/core/src/main/scala/kafka/log/LogManager.scala
index af97621841e68..326feb99d911c 100755
--- a/core/src/main/scala/kafka/log/LogManager.scala
+++ b/core/src/main/scala/kafka/log/LogManager.scala
@@ -210,7 +210,7 @@ class LogManager(logDirs: Seq[File],
         cleaner.handleLogDirFailure(dir)
 
       val offlineCurrentTopicPartitions = currentLogs.collect {
-        case (tp, log) if log.dir.getParent == dir => tp
+        case (tp, log) if log.parentDir == dir => tp
       }
       offlineCurrentTopicPartitions.foreach { topicPartition => {
         val removedLog = currentLogs.remove(topicPartition)
@@ -221,7 +221,7 @@ class LogManager(logDirs: Seq[File],
       }}
 
       val offlineFutureTopicPartitions = futureLogs.collect {
-        case (tp, log) if log.dir.getParent == dir => tp
+        case (tp, log) if log.parentDir == dir => tp
       }
       offlineFutureTopicPartitions.foreach { topicPartition => {
         val removedLog = futureLogs.remove(topicPartition)
@@ -293,7 +293,7 @@ class LogManager(logDirs: Seq[File],
       }
       if (previous != null) {
         if (log.isFuture)
-          throw new IllegalStateException("Duplicate log directories found: %s, %s!".format(log.dir.getAbsolutePath, previous.dir.getAbsolutePath))
+          throw new IllegalStateException(s"Duplicate log directories found: ${log.dir.getAbsolutePath}, ${previous.dir.getAbsolutePath}")
         else
           throw new IllegalStateException(s"Duplicate log directories for $topicPartition are found in both ${log.dir.getAbsolutePath} " +
             s"and ${previous.dir.getAbsolutePath}. It is likely because log directory failure happened while broker was " +
@@ -523,7 +523,7 @@ class LogManager(logDirs: Seq[File],
           if (log.truncateTo(truncateOffset))
             affectedLogs += log
           if (needToStopCleaner && !isFuture)
-            cleaner.maybeTruncateCheckpoint(log.dir.getParentFile, topicPartition, log.activeSegment.baseOffset)
+            cleaner.maybeTruncateCheckpoint(log.parentDirFile, topicPartition, log.activeSegment.baseOffset)
         } finally {
           if (needToStopCleaner && !isFuture) {
             cleaner.resumeCleaning(Seq(topicPartition))
@@ -533,7 +533,7 @@ class LogManager(logDirs: Seq[File],
       }
     }
 
-    for ((dir, logs) <- affectedLogs.groupBy(_.dir.getParentFile)) {
+    for ((dir, logs) <- affectedLogs.groupBy(_.parentDirFile)) {
       checkpointRecoveryOffsetsAndCleanSnapshot(dir, ArrayBuffer.empty)
     }
   }
@@ -560,7 +560,7 @@ class LogManager(logDirs: Seq[File],
       try {
         log.truncateFullyAndStartAt(newOffset)
         if (cleaner != null && !isFuture) {
-          cleaner.maybeTruncateCheckpoint(log.dir.getParentFile, topicPartition, log.activeSegment.baseOffset)
+          cleaner.maybeTruncateCheckpoint(log.parentDirFile, topicPartition, log.activeSegment.baseOffset)
         }
       } finally {
         if (cleaner != null && !isFuture) {
@@ -568,7 +568,7 @@ class LogManager(logDirs: Seq[File],
           info(s"Compaction for partition $topicPartition is resumed")
         }
       }
-      checkpointRecoveryOffsetsAndCleanSnapshot(log.dir.getParentFile, Seq(log))
+      checkpointRecoveryOffsetsAndCleanSnapshot(log.parentDirFile, Seq(log))
     }
   }
 
@@ -642,8 +642,8 @@ class LogManager(logDirs: Seq[File],
   // The logDir should be an absolute path
   def maybeUpdatePreferredLogDir(topicPartition: TopicPartition, logDir: String): Unit = {
     // Do not cache the preferred log directory if either the current log or the future log for this partition exists in the specified logDir
-    if (!getLog(topicPartition).exists(_.dir.getParent == logDir) &&
-        !getLog(topicPartition, isFuture = true).exists(_.dir.getParent == logDir))
+    if (!getLog(topicPartition).exists(_.parentDir == logDir) &&
+        !getLog(topicPartition, isFuture = true).exists(_.parentDir == logDir))
       preferredLogDirs.put(topicPartition, logDir)
   }
 
@@ -733,7 +733,7 @@ class LogManager(logDirs: Seq[File],
           if (isFuture) {
             if (preferredLogDir == null)
               throw new IllegalStateException(s"Can not create the future log for $topicPartition without having a preferred log directory")
-            else if (getLog(topicPartition).get.dir.getParent == preferredLogDir)
+            else if (getLog(topicPartition).get.parentDir == preferredLogDir)
               throw new IllegalStateException(s"Can not create the future log for $topicPartition in the current log directory of this partition")
           }
 
@@ -829,7 +829,7 @@ class LogManager(logDirs: Seq[File],
             info(s"Deleted log for partition ${removedLog.topicPartition} in ${removedLog.dir.getAbsolutePath}.")
           } catch {
             case e: KafkaStorageException =>
-              error(s"Exception while deleting $removedLog in dir ${removedLog.dir.getParent}.", e)
+              error(s"Exception while deleting $removedLog in dir ${removedLog.parentDir}.", e)
           }
         }
       }
@@ -877,7 +877,7 @@ class LogManager(logDirs: Seq[File],
       futureLogs.remove(topicPartition)
       currentLogs.put(topicPartition, destLog)
       if (cleaner != null) {
-        cleaner.alterCheckpointDir(topicPartition, sourceLog.dir.getParentFile, destLog.dir.getParentFile)
+        cleaner.alterCheckpointDir(topicPartition, sourceLog.parentDirFile, destLog.parentDirFile)
         cleaner.resumeCleaning(Seq(topicPartition))
         info(s"Compaction for partition $topicPartition is resumed")
       }
@@ -887,8 +887,8 @@ class LogManager(logDirs: Seq[File],
         // Now that replica in source log directory has been successfully renamed for deletion.
         // Close the log, update checkpoint files, and enqueue this log to be deleted.
         sourceLog.close()
-        checkpointRecoveryOffsetsAndCleanSnapshot(sourceLog.dir.getParentFile, ArrayBuffer.empty)
-        checkpointLogStartOffsetsInDir(sourceLog.dir.getParentFile)
+        checkpointRecoveryOffsetsAndCleanSnapshot(sourceLog.parentDirFile, ArrayBuffer.empty)
+        checkpointLogStartOffsetsInDir(sourceLog.parentDirFile)
         addLogToBeDeleted(sourceLog)
       } catch {
         case e: KafkaStorageException =>
@@ -922,11 +922,11 @@ class LogManager(logDirs: Seq[File],
       //We need to wait until there is no more cleaning task on the log to be deleted before actually deleting it.
       if (cleaner != null && !isFuture) {
         cleaner.abortCleaning(topicPartition)
-        cleaner.updateCheckpoints(removedLog.dir.getParentFile)
+        cleaner.updateCheckpoints(removedLog.parentDirFile)
       }
       removedLog.renameDir(Log.logDeleteDirName(topicPartition))
-      checkpointRecoveryOffsetsAndCleanSnapshot(removedLog.dir.getParentFile, ArrayBuffer.empty)
-      checkpointLogStartOffsetsInDir(removedLog.dir.getParentFile)
+      checkpointRecoveryOffsetsAndCleanSnapshot(removedLog.parentDirFile, ArrayBuffer.empty)
+      checkpointLogStartOffsetsInDir(removedLog.parentDirFile)
       addLogToBeDeleted(removedLog)
       info(s"Log for partition ${removedLog.topicPartition} is renamed to ${removedLog.dir.getAbsolutePath} and is scheduled for deletion")
     } else if (offlineLogDirs.nonEmpty) {
@@ -945,7 +945,7 @@ class LogManager(logDirs: Seq[File],
       List(_liveLogDirs.peek())
     } else {
       // count the number of logs in each parent directory (including 0 for empty directories
-      val logCounts = allLogs.groupBy(_.dir.getParent).mapValues(_.size)
+      val logCounts = allLogs.groupBy(_.parentDir).mapValues(_.size)
       val zeros = _liveLogDirs.asScala.map(dir => (dir.getPath, 0)).toMap
       val dirCounts = (zeros ++ logCounts).toBuffer
 
@@ -1016,7 +1016,7 @@ class LogManager(logDirs: Seq[File],
    */
   private def logsByDir: Map[String, Map[TopicPartition, Log]] = {
     (this.currentLogs.toList ++ this.futureLogs.toList).toMap
-      .groupBy { case (_, log) => log.dir.getParent }
+      .groupBy { case (_, log) => log.parentDir }
   }
 
   // logDir should be an absolute path
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index 16ec06a3bcabf..a308c4ee2f0c4 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -512,7 +512,7 @@ class ReplicaManager(val config: KafkaConfig,
   }
 
   def getLogDir(topicPartition: TopicPartition): Option[String] = {
-    localLog(topicPartition).map(_.dir.getParent)
+    localLog(topicPartition).map(_.parentDir)
   }
 
   /**
@@ -692,7 +692,7 @@ class ReplicaManager(val config: KafkaConfig,
    *    are included. There may be future logs (which will replace the current logs of the partition in the future) on the broker after KIP-113 is implemented.
    */
   def describeLogDirs(partitions: Set[TopicPartition]): Map[String, LogDirInfo] = {
-    val logsByDir = logManager.allLogs.groupBy(log => log.dir.getParent)
+    val logsByDir = logManager.allLogs.groupBy(log => log.parentDir)
 
     config.logDirs.toSet.map { logDir: String =>
       val absolutePath = new File(logDir).getAbsolutePath
@@ -1325,7 +1325,7 @@ class ReplicaManager(val config: KafkaConfig,
               val leader = BrokerEndPoint(config.brokerId, "localhost", -1)
 
               // Add future replica to partition's map
-              partition.createLogIfNotExists(Request.FutureLocalReplicaId, isNew = false, isFutureReplica = true,
+              partition.createLogIfNotExists(isNew = false, isFutureReplica = true,
                 highWatermarkCheckpoints)
 
               // pause cleaning for partitions that are being moved and start ReplicaAlterDirThread to move
@@ -1391,7 +1391,7 @@ class ReplicaManager(val config: KafkaConfig,
       // Update the partition information to be the leader
       partitionStates.foreach { case (partition, partitionState) =>
         try {
-          if (partition.makeLeader(controllerId, partitionState, correlationId, highWatermarkCheckpoints)) {
+          if (partition.makeLeader(partitionState, highWatermarkCheckpoints)) {
             partitionsToMakeLeaders += partition
             stateChangeLogger.trace(s"Stopped fetchers as part of become-leader request from " +
               s"controller $controllerId epoch $controllerEpoch with correlation id $correlationId for partition ${partition.topicPartition} " +
@@ -1473,7 +1473,7 @@ class ReplicaManager(val config: KafkaConfig,
           metadataCache.getAliveBrokers.find(_.id == newLeaderBrokerId) match {
             // Only change partition state when the leader is available
             case Some(_) =>
-              if (partition.makeFollower(controllerId, partitionState, correlationId, highWatermarkCheckpoints))
+              if (partition.makeFollower(partitionState, highWatermarkCheckpoints))
                 partitionsToMakeFollower += partition
               else
                 stateChangeLogger.info(s"Skipped the become-follower state change after marking its partition as " +
@@ -1490,7 +1490,7 @@ class ReplicaManager(val config: KafkaConfig,
                 s"but cannot become follower since the new leader $newLeaderBrokerId is unavailable.")
               // Create the local replica even if the leader is unavailable. This is required to ensure that we include
               // the partition's high watermark in the checkpoint file (see KAFKA-1647)
-              partition.createLogIfNotExists(localBrokerId, isNew = partitionState.isNew, isFutureReplica = false,
+              partition.createLogIfNotExists(isNew = partitionState.isNew, isFutureReplica = false,
                 highWatermarkCheckpoints)
           }
         } catch {
@@ -1622,20 +1622,26 @@ class ReplicaManager(val config: KafkaConfig,
 
   // Flushes the highwatermark value for all partitions to the highwatermark file
   def checkpointHighWatermarks(): Unit = {
-    val localLogs = nonOfflinePartitionsIterator.flatMap { partition =>
-      val logsList: mutable.Set[Log] = mutable.Set()
-      partition.log.foreach(logsList.add)
-      partition.futureLog.foreach(logsList.add)
-      logsList
-    }.toBuffer
-    val logsByDir = localLogs.groupBy(_.dir.getParent)
-    for ((dir, logs) <- logsByDir) {
-      val hwms = logs.map(log => log.topicPartition -> log.highWatermark).toMap
+    def putHw(logDirToCheckpoints: mutable.AnyRefMap[String, mutable.AnyRefMap[TopicPartition, Long]],
+              log: Log): Unit = {
+      val checkpoints = logDirToCheckpoints.getOrElseUpdate(log.parentDir,
+        new mutable.AnyRefMap[TopicPartition, Long]())
+      checkpoints.put(log.topicPartition, log.highWatermark)
+    }
+
+    val logDirToHws = new mutable.AnyRefMap[String, mutable.AnyRefMap[TopicPartition, Long]](
+      allPartitions.size)
+    nonOfflinePartitionsIterator.foreach { partition =>
+      partition.log.foreach(putHw(logDirToHws, _))
+      partition.futureLog.foreach(putHw(logDirToHws, _))
+    }
+
+    for ((logDir, hws) <- logDirToHws) {
       try {
-        highWatermarkCheckpoints.get(dir).foreach(_.write(hwms))
+        highWatermarkCheckpoints.get(logDir).foreach(_.write(hws))
       } catch {
         case e: KafkaStorageException =>
-          error(s"Error while writing to highwatermark file in directory $dir", e)
+          error(s"Error while writing to highwatermark file in directory $logDir", e)
       }
     }
   }
@@ -1654,11 +1660,11 @@ class ReplicaManager(val config: KafkaConfig,
     info(s"Stopping serving replicas in dir $dir")
     replicaStateChangeLock synchronized {
       val newOfflinePartitions = nonOfflinePartitionsIterator.filter { partition =>
-        partition.log.exists { _.dir.getParent == dir }
+        partition.log.exists { _.parentDir == dir }
       }.map(_.topicPartition).toSet
 
       val partitionsWithOfflineFutureReplica = nonOfflinePartitionsIterator.filter { partition =>
-        partition.futureLog.exists { _.dir.getParent == dir }
+        partition.futureLog.exists { _.parentDir == dir }
       }.toSet
 
       replicaFetcherManager.removeFetcherForPartitions(newOfflinePartitions)
diff --git a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
index 31f260d781c42..800f672f659f4 100644
--- a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
+++ b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
@@ -169,7 +169,7 @@ class PartitionTest {
     val latch = new CountDownLatch(1)
 
     logManager.maybeUpdatePreferredLogDir(topicPartition, logDir1.getAbsolutePath)
-    partition.createLogIfNotExists(brokerId, isNew = true, isFutureReplica = false, offsetCheckpoints)
+    partition.createLogIfNotExists(isNew = true, isFutureReplica = false, offsetCheckpoints)
     logManager.maybeUpdatePreferredLogDir(topicPartition, logDir2.getAbsolutePath)
     partition.maybeCreateFutureReplica(logDir2.getAbsolutePath, offsetCheckpoints)
 
@@ -201,7 +201,7 @@ class PartitionTest {
   // active segment
   def testMaybeReplaceCurrentWithFutureReplicaDifferentBaseOffsets(): Unit = {
     logManager.maybeUpdatePreferredLogDir(topicPartition, logDir1.getAbsolutePath)
-    partition.createLogIfNotExists(brokerId, isNew = true, isFutureReplica = false, offsetCheckpoints)
+    partition.createLogIfNotExists(isNew = true, isFutureReplica = false, offsetCheckpoints)
     logManager.maybeUpdatePreferredLogDir(topicPartition, logDir2.getAbsolutePath)
     partition.maybeCreateFutureReplica(logDir2.getAbsolutePath, offsetCheckpoints)
 
@@ -472,7 +472,6 @@ class PartitionTest {
     val leader = brokerId
     val follower1 = brokerId + 1
     val follower2 = brokerId + 2
-    val controllerId = brokerId + 3
     val replicas = List(leader, follower1, follower2)
     val isr = List[Integer](leader, follower2).asJava
     val leaderEpoch = 8
@@ -493,7 +492,7 @@ class PartitionTest {
       .setIsNew(true)
 
     assertTrue("Expected first makeLeader() to return 'leader changed'",
-      partition.makeLeader(controllerId, leaderState, 0, offsetCheckpoints))
+      partition.makeLeader(leaderState, offsetCheckpoints))
     assertEquals("Current leader epoch", leaderEpoch, partition.getLeaderEpoch)
     assertEquals("ISR", Set[Integer](leader, follower2), partition.inSyncReplicaIds)
 
@@ -568,7 +567,7 @@ class PartitionTest {
       .setReplicas(replicas.map(Int.box).asJava)
       .setIsNew(false)
 
-    assertTrue(partition.makeFollower(controllerId, followerState, 1, offsetCheckpoints))
+    assertTrue(partition.makeFollower(followerState, offsetCheckpoints))
 
     // Back to leader, this resets the startLogOffset for this epoch (to 2), we're now in the fault condition
     val newLeaderState = new LeaderAndIsrPartitionState()
@@ -580,7 +579,7 @@ class PartitionTest {
       .setReplicas(replicas.map(Int.box).asJava)
       .setIsNew(false)
 
-    assertTrue(partition.makeLeader(controllerId, newLeaderState, 2, offsetCheckpoints))
+    assertTrue(partition.makeLeader(newLeaderState, offsetCheckpoints))
 
     // Try to get offsets as a client
     fetchOffsetsForTimestamp(ListOffsetRequest.LATEST_TIMESTAMP, Some(IsolationLevel.READ_UNCOMMITTED)) match {
@@ -643,34 +642,33 @@ class PartitionTest {
   private def setupPartitionWithMocks(leaderEpoch: Int,
                                       isLeader: Boolean,
                                       log: Log = logManager.getOrCreateLog(topicPartition, () => logConfig)): Partition = {
-    partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
 
-    val controllerId = 0
     val controllerEpoch = 0
     val replicas = List[Integer](brokerId, brokerId + 1).asJava
     val isr = replicas
 
     if (isLeader) {
       assertTrue("Expected become leader transition to succeed",
-        partition.makeLeader(controllerId, new LeaderAndIsrPartitionState()
+        partition.makeLeader(new LeaderAndIsrPartitionState()
           .setControllerEpoch(controllerEpoch)
           .setLeader(brokerId)
           .setLeaderEpoch(leaderEpoch)
           .setIsr(isr)
           .setZkVersion(1)
           .setReplicas(replicas)
-          .setIsNew(true), 0, offsetCheckpoints))
+          .setIsNew(true), offsetCheckpoints))
       assertEquals(leaderEpoch, partition.getLeaderEpoch)
     } else {
       assertTrue("Expected become follower transition to succeed",
-        partition.makeFollower(controllerId, new LeaderAndIsrPartitionState()
+        partition.makeFollower(new LeaderAndIsrPartitionState()
           .setControllerEpoch(controllerEpoch)
           .setLeader(brokerId + 1)
           .setLeaderEpoch(leaderEpoch)
           .setIsr(isr)
           .setZkVersion(1)
           .setReplicas(replicas)
-          .setIsNew(true), 0, offsetCheckpoints))
+          .setIsNew(true), offsetCheckpoints))
       assertEquals(leaderEpoch, partition.getLeaderEpoch)
       assertEquals(None, partition.leaderLogIfLocal)
     }
@@ -680,7 +678,7 @@ class PartitionTest {
 
   @Test
   def testAppendRecordsAsFollowerBelowLogStartOffset(): Unit = {
-    partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
     val log = partition.localLogOrException
 
     val initialLogStartOffset = 5L
@@ -730,7 +728,6 @@ class PartitionTest {
 
   @Test
   def testListOffsetIsolationLevels(): Unit = {
-    val controllerId = 0
     val controllerEpoch = 0
     val leaderEpoch = 5
     val replicas = List[Integer](brokerId, brokerId + 1).asJava
@@ -738,17 +735,17 @@ class PartitionTest {
 
     doNothing().when(delayedOperations).checkAndCompleteFetch()
 
-    partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
 
     assertTrue("Expected become leader transition to succeed",
-      partition.makeLeader(controllerId, new LeaderAndIsrPartitionState()
+      partition.makeLeader(new LeaderAndIsrPartitionState()
         .setControllerEpoch(controllerEpoch)
         .setLeader(brokerId)
         .setLeaderEpoch(leaderEpoch)
         .setIsr(isr)
         .setZkVersion(1)
         .setReplicas(replicas)
-        .setIsNew(true), 0, offsetCheckpoints))
+        .setIsNew(true), offsetCheckpoints))
     assertEquals(leaderEpoch, partition.getLeaderEpoch)
 
     val records = createTransactionalRecords(List(
@@ -818,7 +815,7 @@ class PartitionTest {
       .setZkVersion(1)
       .setReplicas(List[Integer](0, 1, 2, brokerId).asJava)
       .setIsNew(false)
-    partition.makeFollower(0, partitionState, 0, offsetCheckpoints)
+    partition.makeFollower(partitionState, offsetCheckpoints)
 
     // Request with same leader and epoch increases by only 1, do become-follower steps
     partitionState = new LeaderAndIsrPartitionState()
@@ -829,7 +826,7 @@ class PartitionTest {
       .setZkVersion(1)
       .setReplicas(List[Integer](0, 1, 2, brokerId).asJava)
       .setIsNew(false)
-    assertTrue(partition.makeFollower(0, partitionState, 2, offsetCheckpoints))
+    assertTrue(partition.makeFollower(partitionState, offsetCheckpoints))
 
     // Request with same leader and same epoch, skip become-follower steps
     partitionState = new LeaderAndIsrPartitionState()
@@ -839,7 +836,7 @@ class PartitionTest {
       .setIsr(List[Integer](0, 1, 2, brokerId).asJava)
       .setZkVersion(1)
       .setReplicas(List[Integer](0, 1, 2, brokerId).asJava)
-    assertFalse(partition.makeFollower(0, partitionState, 2, offsetCheckpoints))
+    assertFalse(partition.makeFollower(partitionState, offsetCheckpoints))
   }
 
   @Test
@@ -848,7 +845,6 @@ class PartitionTest {
     val leader = brokerId
     val follower1 = brokerId + 1
     val follower2 = brokerId + 2
-    val controllerId = brokerId + 3
     val replicas = List[Integer](leader, follower1, follower2).asJava
     val isr = List[Integer](leader, follower2).asJava
     val leaderEpoch = 8
@@ -869,7 +865,7 @@ class PartitionTest {
       .setReplicas(replicas)
       .setIsNew(true)
     assertTrue("Expected first makeLeader() to return 'leader changed'",
-               partition.makeLeader(controllerId, leaderState, 0, offsetCheckpoints))
+               partition.makeLeader(leaderState, offsetCheckpoints))
     assertEquals("Current leader epoch", leaderEpoch, partition.getLeaderEpoch)
     assertEquals("ISR", Set[Integer](leader, follower2), partition.inSyncReplicaIds)
 
@@ -905,7 +901,7 @@ class PartitionTest {
       .setZkVersion(1)
       .setReplicas(replicas)
       .setIsNew(false)
-    partition.makeFollower(controllerId, followerState, 1, offsetCheckpoints)
+    partition.makeFollower(followerState, offsetCheckpoints)
 
     val newLeaderState = new LeaderAndIsrPartitionState()
       .setControllerEpoch(controllerEpoch)
@@ -916,7 +912,7 @@ class PartitionTest {
       .setReplicas(replicas)
       .setIsNew(false)
     assertTrue("Expected makeLeader() to return 'leader changed' after makeFollower()",
-               partition.makeLeader(controllerEpoch, newLeaderState, 2, offsetCheckpoints))
+               partition.makeLeader(newLeaderState, offsetCheckpoints))
     val currentLeaderEpochStartOffset = partition.localLogOrException.logEndOffset
 
     // append records with the latest leader epoch
@@ -944,7 +940,6 @@ class PartitionTest {
    */
   @Test
   def testDelayedFetchAfterAppendRecords(): Unit = {
-    val controllerId = 0
     val controllerEpoch = 0
     val leaderEpoch = 5
     val replicaIds = List[Integer](brokerId, brokerId + 1).asJava
@@ -987,7 +982,7 @@ class PartitionTest {
         .setZkVersion(1)
         .setReplicas(replicaIds)
         .setIsNew(true)
-      partition.makeLeader(controllerId, leaderState, 0, offsetCheckpoints)
+      partition.makeLeader(leaderState, offsetCheckpoints)
       partitions += partition
     }
 
@@ -1044,8 +1039,7 @@ class PartitionTest {
   }
 
   def createTransactionalRecords(records: Iterable[SimpleRecord],
-                                 baseOffset: Long,
-                                 partitionLeaderEpoch: Int = 0): MemoryRecords = {
+                                 baseOffset: Long): MemoryRecords = {
     val producerId = 1L
     val producerEpoch = 0.toShort
     val baseSequence = 0
@@ -1067,7 +1061,6 @@ class PartitionTest {
     val leader = brokerId
     val follower1 = brokerId + 1
     val follower2 = brokerId + 2
-    val controllerId = brokerId + 3
     val replicas = List[Integer](leader, follower1, follower2).asJava
     val isr = List[Integer](leader).asJava
     val leaderEpoch = 8
@@ -1082,7 +1075,7 @@ class PartitionTest {
       .setZkVersion(1)
       .setReplicas(replicas)
       .setIsNew(true)
-    partition.makeLeader(controllerId, leaderState, 0, offsetCheckpoints)
+    partition.makeLeader(leaderState, offsetCheckpoints)
     assertTrue(partition.isAtMinIsr)
   }
 
@@ -1091,7 +1084,6 @@ class PartitionTest {
     val log = logManager.getOrCreateLog(topicPartition, () => logConfig)
     seedLogData(log, numRecords = 6, leaderEpoch = 4)
 
-    val controllerId = 0
     val controllerEpoch = 0
     val leaderEpoch = 5
     val remoteBrokerId = brokerId + 1
@@ -1100,12 +1092,11 @@ class PartitionTest {
 
     doNothing().when(delayedOperations).checkAndCompleteFetch()
 
-    partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
 
     val initializeTimeMs = time.milliseconds()
     assertTrue("Expected become leader transition to succeed",
       partition.makeLeader(
-        controllerId,
         new LeaderAndIsrPartitionState()
           .setControllerEpoch(controllerEpoch)
           .setLeader(brokerId)
@@ -1114,7 +1105,6 @@ class PartitionTest {
           .setZkVersion(1)
           .setReplicas(replicas)
           .setIsNew(true),
-        0,
         offsetCheckpoints))
 
     val remoteReplica = partition.getReplica(remoteBrokerId).get
@@ -1155,7 +1145,6 @@ class PartitionTest {
     val log = logManager.getOrCreateLog(topicPartition, () => logConfig)
     seedLogData(log, numRecords = 10, leaderEpoch = 4)
 
-    val controllerId = 0
     val controllerEpoch = 0
     val leaderEpoch = 5
     val remoteBrokerId = brokerId + 1
@@ -1164,11 +1153,10 @@ class PartitionTest {
 
     doNothing().when(delayedOperations).checkAndCompleteFetch()
 
-    partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
     assertTrue(
       "Expected become leader transition to succeed",
       partition.makeLeader(
-        controllerId,
         new LeaderAndIsrPartitionState()
           .setControllerEpoch(controllerEpoch)
           .setLeader(brokerId)
@@ -1177,7 +1165,6 @@ class PartitionTest {
           .setZkVersion(1)
           .setReplicas(replicas.map(Int.box).asJava)
           .setIsNew(true),
-        0,
         offsetCheckpoints)
     )
     assertEquals(Set(brokerId), partition.inSyncReplicaIds)
@@ -1222,7 +1209,6 @@ class PartitionTest {
     val log = logManager.getOrCreateLog(topicPartition, () => logConfig)
     seedLogData(log, numRecords = 10, leaderEpoch = 4)
 
-    val controllerId = 0
     val controllerEpoch = 0
     val leaderEpoch = 5
     val remoteBrokerId = brokerId + 1
@@ -1231,10 +1217,9 @@ class PartitionTest {
 
     doNothing().when(delayedOperations).checkAndCompleteFetch()
 
-    partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
     assertTrue("Expected become leader transition to succeed",
       partition.makeLeader(
-        controllerId,
         new LeaderAndIsrPartitionState()
           .setControllerEpoch(controllerEpoch)
           .setLeader(brokerId)
@@ -1243,7 +1228,6 @@ class PartitionTest {
           .setZkVersion(1)
           .setReplicas(replicas)
           .setIsNew(true),
-        0,
         offsetCheckpoints))
     assertEquals(Set(brokerId), partition.inSyncReplicaIds)
 
@@ -1277,7 +1261,6 @@ class PartitionTest {
     val log = logManager.getOrCreateLog(topicPartition, () => logConfig)
     seedLogData(log, numRecords = 10, leaderEpoch = 4)
 
-    val controllerId = 0
     val controllerEpoch = 0
     val leaderEpoch = 5
     val remoteBrokerId = brokerId + 1
@@ -1287,11 +1270,10 @@ class PartitionTest {
     doNothing().when(delayedOperations).checkAndCompleteFetch()
 
     val initializeTimeMs = time.milliseconds()
-    partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
     assertTrue(
       "Expected become leader transition to succeed",
       partition.makeLeader(
-        controllerId,
         new LeaderAndIsrPartitionState()
           .setControllerEpoch(controllerEpoch)
           .setLeader(brokerId)
@@ -1300,7 +1282,6 @@ class PartitionTest {
           .setZkVersion(1)
           .setReplicas(replicas.map(Int.box).asJava)
           .setIsNew(true),
-        0,
         offsetCheckpoints)
     )
     assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicaIds)
@@ -1334,7 +1315,6 @@ class PartitionTest {
     val log = logManager.getOrCreateLog(topicPartition, () => logConfig)
     seedLogData(log, numRecords = 10, leaderEpoch = 4)
 
-    val controllerId = 0
     val controllerEpoch = 0
     val leaderEpoch = 5
     val remoteBrokerId = brokerId + 1
@@ -1344,11 +1324,10 @@ class PartitionTest {
     doNothing().when(delayedOperations).checkAndCompleteFetch()
 
     val initializeTimeMs = time.milliseconds()
-    partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
     assertTrue(
       "Expected become leader transition to succeed",
       partition.makeLeader(
-        controllerId,
         new LeaderAndIsrPartitionState()
           .setControllerEpoch(controllerEpoch)
           .setLeader(brokerId)
@@ -1357,7 +1336,6 @@ class PartitionTest {
           .setZkVersion(1)
           .setReplicas(replicas.map(Int.box).asJava)
           .setIsNew(true),
-        0,
         offsetCheckpoints)
     )
     assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicaIds)
@@ -1408,7 +1386,6 @@ class PartitionTest {
     val log = logManager.getOrCreateLog(topicPartition, () => logConfig)
     seedLogData(log, numRecords = 10, leaderEpoch = 4)
 
-    val controllerId = 0
     val controllerEpoch = 0
     val leaderEpoch = 5
     val remoteBrokerId = brokerId + 1
@@ -1418,11 +1395,10 @@ class PartitionTest {
     doNothing().when(delayedOperations).checkAndCompleteFetch()
 
     val initializeTimeMs = time.milliseconds()
-    partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
     assertTrue(
       "Expected become leader transition to succeed",
       partition.makeLeader(
-        controllerId,
         new LeaderAndIsrPartitionState()
           .setControllerEpoch(controllerEpoch)
           .setLeader(brokerId)
@@ -1431,7 +1407,6 @@ class PartitionTest {
           .setZkVersion(1)
           .setReplicas(replicas.map(Int.box).asJava)
           .setIsNew(true),
-        0,
         offsetCheckpoints)
     )
     assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicaIds)
@@ -1467,7 +1442,6 @@ class PartitionTest {
     val log = logManager.getOrCreateLog(topicPartition, () => logConfig)
     seedLogData(log, numRecords = 10, leaderEpoch = 4)
 
-    val controllerId = 0
     val controllerEpoch = 0
     val leaderEpoch = 5
     val remoteBrokerId = brokerId + 1
@@ -1477,10 +1451,9 @@ class PartitionTest {
     doNothing().when(delayedOperations).checkAndCompleteFetch()
 
     val initializeTimeMs = time.milliseconds()
-    partition.createLogIfNotExists(brokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
     assertTrue("Expected become leader transition to succeed",
       partition.makeLeader(
-        controllerId,
         new LeaderAndIsrPartitionState()
           .setControllerEpoch(controllerEpoch)
           .setLeader(brokerId)
@@ -1489,7 +1462,6 @@ class PartitionTest {
           .setZkVersion(1)
           .setReplicas(replicas)
           .setIsNew(true),
-        0,
         offsetCheckpoints))
     assertEquals(Set(brokerId, remoteBrokerId), partition.inSyncReplicaIds)
     assertEquals(0L, partition.localLogOrException.highWatermark)
@@ -1522,7 +1494,6 @@ class PartitionTest {
     when(offsetCheckpoints.fetch(logDir1.getAbsolutePath, topicPartition))
       .thenReturn(Some(4L))
 
-    val controllerId = 0
     val controllerEpoch = 3
     val replicas = List[Integer](brokerId, brokerId + 1).asJava
     val leaderState = new LeaderAndIsrPartitionState()
@@ -1533,7 +1504,7 @@ class PartitionTest {
       .setZkVersion(1)
       .setReplicas(replicas)
       .setIsNew(false)
-    partition.makeLeader(controllerId, leaderState, 0, offsetCheckpoints)
+    partition.makeLeader(leaderState, offsetCheckpoints)
     assertEquals(4, partition.localLogOrException.highWatermark)
   }
 
@@ -1576,7 +1547,7 @@ class PartitionTest {
       metadataCache,
       spyLogManager)
 
-    partition.createLog(brokerId, isNew = true, isFutureReplica = false, offsetCheckpoints)
+    partition.createLog(isNew = true, isFutureReplica = false, offsetCheckpoints)
 
     // Validate that initializingLog and finishedInitializingLog was called
     verify(spyLogManager).initializingLog(ArgumentMatchers.eq(topicPartition))
@@ -1612,7 +1583,7 @@ class PartitionTest {
       metadataCache,
       spyLogManager)
 
-    partition.createLog(brokerId, isNew = true, isFutureReplica = false, offsetCheckpoints)
+    partition.createLog(isNew = true, isFutureReplica = false, offsetCheckpoints)
 
     // Validate that initializingLog and finishedInitializingLog was called
     verify(spyLogManager).initializingLog(ArgumentMatchers.eq(topicPartition))
@@ -1649,7 +1620,7 @@ class PartitionTest {
       metadataCache,
       spyLogManager)
 
-    partition.createLog(brokerId, isNew = true, isFutureReplica = false, offsetCheckpoints)
+    partition.createLog(isNew = true, isFutureReplica = false, offsetCheckpoints)
 
     // Validate that initializingLog and finishedInitializingLog was called
     verify(spyLogManager).initializingLog(ArgumentMatchers.eq(topicPartition))
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index c9be250f0c218..93011add60202 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -129,7 +129,7 @@ class ReplicaManagerTest {
       new MetadataCache(config.brokerId), new LogDirFailureChannel(config.logDirs.size))
     try {
       val partition = rm.createPartition(new TopicPartition(topic, 1))
-      partition.createLogIfNotExists(1, isNew = false, isFutureReplica = false,
+      partition.createLogIfNotExists(isNew = false, isFutureReplica = false,
         new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints))
       rm.checkpointHighWatermarks()
     } finally {
@@ -149,7 +149,7 @@ class ReplicaManagerTest {
       new MetadataCache(config.brokerId), new LogDirFailureChannel(config.logDirs.size))
     try {
       val partition = rm.createPartition(new TopicPartition(topic, 1))
-      partition.createLogIfNotExists(1, isNew = false, isFutureReplica = false,
+      partition.createLogIfNotExists(isNew = false, isFutureReplica = false,
         new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints))
       rm.checkpointHighWatermarks()
     } finally {
@@ -204,7 +204,7 @@ class ReplicaManagerTest {
       val brokerList = Seq[Integer](0, 1).asJava
 
       val partition = rm.createPartition(new TopicPartition(topic, 0))
-      partition.createLogIfNotExists(0, isNew = false, isFutureReplica = false,
+      partition.createLogIfNotExists(isNew = false, isFutureReplica = false,
         new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints))
       // Make this replica the leader.
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
@@ -262,7 +262,7 @@ class ReplicaManagerTest {
       val brokerList = Seq[Integer](0, 1).asJava
       val topicPartition = new TopicPartition(topic, 0)
       replicaManager.createPartition(topicPartition)
-        .createLogIfNotExists(0, isNew = false, isFutureReplica = false,
+        .createLogIfNotExists(isNew = false, isFutureReplica = false,
           new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
 
       def leaderAndIsrRequest(epoch: Int): LeaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
@@ -320,7 +320,7 @@ class ReplicaManagerTest {
       val brokerList = Seq[Integer](0, 1).asJava
 
       val partition = replicaManager.createPartition(new TopicPartition(topic, 0))
-      partition.createLogIfNotExists(0, isNew = false, isFutureReplica = false,
+      partition.createLogIfNotExists(isNew = false, isFutureReplica = false,
         new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
 
       // Make this replica the leader.
@@ -380,7 +380,7 @@ class ReplicaManagerTest {
       val brokerList = Seq[Integer](0, 1).asJava
 
       val partition = replicaManager.createPartition(new TopicPartition(topic, 0))
-      partition.createLogIfNotExists(0, isNew = false, isFutureReplica = false,
+      partition.createLogIfNotExists(isNew = false, isFutureReplica = false,
         new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
 
       // Make this replica the leader.
@@ -486,7 +486,7 @@ class ReplicaManagerTest {
     try {
       val brokerList = Seq[Integer](0, 1).asJava
       val partition = replicaManager.createPartition(new TopicPartition(topic, 0))
-      partition.createLogIfNotExists(0, isNew = false, isFutureReplica = false,
+      partition.createLogIfNotExists(isNew = false, isFutureReplica = false,
         new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
 
       // Make this replica the leader.
@@ -562,7 +562,7 @@ class ReplicaManagerTest {
       val brokerList = Seq[Integer](0, 1, 2).asJava
 
       val partition = rm.createPartition(new TopicPartition(topic, 0))
-      partition.createLogIfNotExists(0, isNew = false, isFutureReplica = false,
+      partition.createLogIfNotExists(isNew = false, isFutureReplica = false,
         new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints))
 
       // Make this replica the leader.
@@ -718,8 +718,8 @@ class ReplicaManagerTest {
       val tp0 = new TopicPartition(topic, 0)
       val tp1 = new TopicPartition(topic, 1)
       val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
-      replicaManager.createPartition(tp0).createLogIfNotExists(0, isNew = false, isFutureReplica = false, offsetCheckpoints)
-      replicaManager.createPartition(tp1).createLogIfNotExists(0, isNew = false, isFutureReplica = false, offsetCheckpoints)
+      replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
+      replicaManager.createPartition(tp1).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
       val partition0Replicas = Seq[Integer](0, 1).asJava
       val partition1Replicas = Seq[Integer](0, 2).asJava
       val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
@@ -832,10 +832,10 @@ class ReplicaManagerTest {
     val tp = new TopicPartition(topic, topicPartition)
     val partition = replicaManager.createPartition(tp)
     val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
-    partition.createLogIfNotExists(followerBrokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
-    partition.makeFollower(controllerId,
+    partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
+    partition.makeFollower(
       leaderAndIsrPartitionState(tp, leaderEpoch, leaderBrokerId, aliveBrokerIds),
-      correlationId, offsetCheckpoints)
+      offsetCheckpoints)
 
     // Make local partition a follower - because epoch increased by more than 1, truncation should
     // trigger even though leader does not change
@@ -858,7 +858,6 @@ class ReplicaManagerTest {
     val topicPartition = 0
     val followerBrokerId = 0
     val leaderBrokerId = 1
-    val controllerId = 0
     val leaderEpoch = 1
     val leaderEpochIncrement = 2
     val aliveBrokerIds = Seq[Integer] (followerBrokerId, leaderBrokerId)
@@ -873,11 +872,9 @@ class ReplicaManagerTest {
     val partition = replicaManager.createPartition(tp)
 
     val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
-    partition.createLogIfNotExists(leaderBrokerId, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
     partition.makeLeader(
-      controllerId,
       leaderAndIsrPartitionState(tp, leaderEpoch, leaderBrokerId, aliveBrokerIds),
-      correlationId,
       offsetCheckpoints
     )
 
@@ -1027,7 +1024,7 @@ class ReplicaManagerTest {
 
     val tp0 = new TopicPartition(topic, 0)
     val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
-    replicaManager.createPartition(tp0).createLogIfNotExists(0, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
     val partition0Replicas = Seq[Integer](0, 1).asJava
     val becomeFollowerRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
       Seq(new LeaderAndIsrPartitionState()
@@ -1066,7 +1063,7 @@ class ReplicaManagerTest {
 
     val tp0 = new TopicPartition(topic, 0)
     val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
-    replicaManager.createPartition(tp0).createLogIfNotExists(0, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
     val partition0Replicas = Seq[Integer](0, 1).asJava
 
     val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
@@ -1114,7 +1111,7 @@ class ReplicaManagerTest {
 
     val tp0 = new TopicPartition(topic, 0)
     val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
-    replicaManager.createPartition(tp0).createLogIfNotExists(0, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
     val partition0Replicas = Seq[Integer](0, 1).asJava
 
     val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
@@ -1162,7 +1159,7 @@ class ReplicaManagerTest {
 
     val tp0 = new TopicPartition(topic, 0)
     val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
-    replicaManager.createPartition(tp0).createLogIfNotExists(0, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
     val partition0Replicas = Seq[Integer](0, 1).asJava
 
     val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
@@ -1204,7 +1201,7 @@ class ReplicaManagerTest {
 
     val tp0 = new TopicPartition(topic, 0)
     val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
-    replicaManager.createPartition(tp0).createLogIfNotExists(0, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
     val partition0Replicas = Seq[Integer](0, 1).asJava
 
     val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
@@ -1243,7 +1240,7 @@ class ReplicaManagerTest {
 
     val tp0 = new TopicPartition(topic, 0)
     val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
-    replicaManager.createPartition(tp0).createLogIfNotExists(0, isNew = false, isFutureReplica = false, offsetCheckpoints)
+    replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
     val partition0Replicas = Seq[Integer](0, 1).asJava
 
     val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
@@ -1343,7 +1340,7 @@ class ReplicaManagerTest {
     val mockBrokerTopicStats = new BrokerTopicStats
     val mockLogDirFailureChannel = new LogDirFailureChannel(config.logDirs.size)
     val mockLog = new Log(
-      dir = new File(new File(config.logDirs.head), s"$topic-0"),
+      _dir = new File(new File(config.logDirs.head), s"$topic-0"),
       config = LogConfig(),
       logStartOffset = 0L,
       recoveryPoint = 0L,
diff --git a/gradle/spotbugs-exclude.xml b/gradle/spotbugs-exclude.xml
index 07ca2b450e753..e667243ef8abb 100644
--- a/gradle/spotbugs-exclude.xml
+++ b/gradle/spotbugs-exclude.xml
@@ -155,8 +155,11 @@ For a detailed description of spotbugs bug categories, see https://spotbugs.read
             
             
             
-            
             
+            
+            
+            
+            
         
     
 
diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/partition/UpdateFollowerFetchStateBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/partition/UpdateFollowerFetchStateBenchmark.java
index c3e074685b561..a686c3bcdc4d0 100644
--- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/partition/UpdateFollowerFetchStateBenchmark.java
+++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/partition/UpdateFollowerFetchStateBenchmark.java
@@ -119,7 +119,7 @@ public void setUp() {
                 ApiVersion$.MODULE$.latestVersion(), 0, Time.SYSTEM,
                 partitionStateStore, delayedOperations,
                 Mockito.mock(MetadataCache.class), logManager);
-        partition.makeLeader(0, partitionState, 0, offsetCheckpoints);
+        partition.makeLeader(partitionState, offsetCheckpoints);
     }
 
     // avoid mocked DelayedOperations to avoid mocked class affecting benchmark results
diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/server/HighwatermarkCheckpointBench.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/server/HighwatermarkCheckpointBench.java
new file mode 100644
index 0000000000000..9cb8ac6a97a9e
--- /dev/null
+++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/server/HighwatermarkCheckpointBench.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.jmh.server;
+
+import java.util.Properties;
+import kafka.cluster.Partition;
+import kafka.cluster.PartitionStateStore;
+import kafka.log.CleanerConfig;
+import kafka.log.LogConfig;
+import kafka.log.LogManager;
+import kafka.server.BrokerTopicStats;
+import kafka.server.KafkaConfig;
+import kafka.server.LogDirFailureChannel;
+import kafka.server.MetadataCache;
+import kafka.server.QuotaFactory;
+import kafka.server.ReplicaManager;
+import kafka.server.checkpoints.OffsetCheckpoints;
+import kafka.utils.KafkaScheduler;
+import kafka.utils.MockTime;
+import kafka.utils.Scheduler;
+import kafka.utils.TestUtils;
+import kafka.zk.KafkaZkClient;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.metrics.Metrics;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.mockito.Mockito;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import scala.Option;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+import scala.jdk.CollectionConverters;
+
+
+@Warmup(iterations = 5)
+@Measurement(iterations = 5)
+@Fork(3)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+public class HighwatermarkCheckpointBench {
+
+    @Param({"100", "1000", "2000"})
+    public int numTopics;
+
+    @Param({"3"})
+    public int numPartitions;
+
+    private final String topicName = "foo";
+
+    private Scheduler scheduler;
+
+    private Metrics metrics;
+
+    private MockTime time;
+
+    private KafkaConfig brokerProperties;
+
+    private ReplicaManager replicaManager;
+    private QuotaFactory.QuotaManagers quotaManagers;
+    private LogDirFailureChannel failureChannel;
+    private LogManager logManager;
+
+
+    @Setup(Level.Trial)
+    public void setup() {
+        this.scheduler = new KafkaScheduler(1, "scheduler-thread", true);
+        this.brokerProperties = KafkaConfig.fromProps(TestUtils.createBrokerConfig(
+                0, TestUtils.MockZkConnect(), true, true, 9092, Option.empty(), Option.empty(),
+                Option.empty(), true, false, 0, false, 0, false, 0, Option.empty(), 1, true, 1, (short) 1));
+        this.metrics = new Metrics();
+        this.time = new MockTime();
+        this.failureChannel = new LogDirFailureChannel(brokerProperties.logDirs().size());
+        final List files =
+            CollectionConverters.seqAsJavaList(brokerProperties.logDirs()).stream().map(File::new).collect(Collectors.toList());
+        this.logManager = TestUtils.createLogManager(CollectionConverters.asScalaBuffer(files),
+                LogConfig.apply(), CleanerConfig.apply(1, 4 * 1024 * 1024L, 0.9d,
+                        1024 * 1024, 32 * 1024 * 1024,
+                        Double.MAX_VALUE, 15 * 1000, true, "MD5"), time);
+        scheduler.startup();
+        final BrokerTopicStats brokerTopicStats = new BrokerTopicStats();
+        final MetadataCache metadataCache =
+                new MetadataCache(this.brokerProperties.brokerId());
+        this.quotaManagers =
+                QuotaFactory.instantiate(this.brokerProperties,
+                        this.metrics,
+                        this.time, "");
+        KafkaZkClient zkClient = new KafkaZkClient(null, false, Time.SYSTEM) {
+            @Override
+            public Properties getEntityConfigs(String rootEntityType, String sanitizedEntityName) {
+                return new Properties();
+            }
+        };
+        this.replicaManager = new ReplicaManager(
+                this.brokerProperties,
+                this.metrics,
+                this.time,
+                zkClient,
+                this.scheduler,
+                this.logManager,
+                new AtomicBoolean(false),
+                this.quotaManagers,
+                brokerTopicStats,
+                metadataCache,
+                this.failureChannel,
+                Option.empty());
+        replicaManager.startup();
+
+        List topicPartitions = new ArrayList<>();
+        for (int topicNum = 0; topicNum < numTopics; topicNum++) {
+            final String topicName = this.topicName + "-" + topicNum;
+            for (int partitionNum = 0; partitionNum < numPartitions; partitionNum++) {
+                topicPartitions.add(new TopicPartition(topicName, partitionNum));
+            }
+        }
+
+        PartitionStateStore partitionStateStore = Mockito.mock(PartitionStateStore.class);
+        Mockito.when(partitionStateStore.fetchTopicConfig()).thenReturn(new Properties());
+        OffsetCheckpoints checkpoints = (logDir, topicPartition) -> Option.apply(0L);
+        for (TopicPartition topicPartition : topicPartitions) {
+            final Partition partition = this.replicaManager.createPartition(topicPartition);
+            partition.createLogIfNotExists(true, false, checkpoints);
+        }
+
+        replicaManager.checkpointHighWatermarks();
+    }
+
+    @TearDown(Level.Trial)
+    public void tearDown() throws Exception {
+        this.replicaManager.shutdown(false);
+        this.metrics.close();
+        this.scheduler.shutdown();
+        this.quotaManagers.shutdown();
+        for (File dir : CollectionConverters.asJavaCollection(logManager.liveLogDirs())) {
+            Utils.delete(dir);
+        }
+    }
+
+
+    @Benchmark
+    @Threads(1)
+    public void measureCheckpointHighWatermarks() {
+        this.replicaManager.checkpointHighWatermarks();
+    }
+}

From 72bea504dd31d4f492a2213818d2574b87406ff2 Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Thu, 26 Mar 2020 04:26:51 -0700
Subject: [PATCH 1052/1071] [LI-CHERRY-PICK] [222726d] KAFKA-9373: Reduce
 shutdown time by avoiding unnecessary loading of indexes (#8346) TICKET =
 LI_DESCRIPTION =

KAFKA-7283 enabled lazy mmap on index files by initializing indices
on-demand rather than performing costly disk/memory operations when
creating all indices on broker startup. This helped reducing the startup
time of brokers. However, segment indices are still created on closing
segments, regardless of whether they need to be closed or not.

This is a cleaned up version of #7900, which was submitted by @efeg. It
eliminates unnecessary disk accesses and memory map operations while
deleting, renaming or closing offset and time indexes.

In a cluster with 31 brokers, where each broker has 13K to 20K segments,
@efeg and team observed up to 2 orders of magnitude faster LogManager
shutdown times - i.e. dropping the LogManager shutdown time of each
broker from 10s of seconds to 100s of milliseconds.

To avoid confusion between `renameTo` and `setFile`, I replaced the
latter with the more restricted updateParentDir` (it turns out that's
all we need).

Reviewers: Jun Rao , Andrew Choi 

Co-authored-by: Adem Efe Gencer 
Co-authored-by: Ismael Juma 

EXIT_CRITERIA = HASH [222726d]
---
 .../kafka/common/record/FileRecords.java      |   8 +-
 .../main/scala/kafka/log/AbstractIndex.scala  |  10 +-
 core/src/main/scala/kafka/log/LazyIndex.scala | 108 +++++++++++++++---
 core/src/main/scala/kafka/log/Log.scala       |   2 +-
 .../src/main/scala/kafka/log/LogSegment.scala |  34 +++---
 .../scala/kafka/log/TransactionIndex.scala    |  11 +-
 .../scala/unit/kafka/log/LogSegmentTest.scala |  17 +++
 .../server/HighwatermarkCheckpointBench.java  |  11 +-
 8 files changed, 152 insertions(+), 49 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java b/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java
index 4d770cbf845ab..46ac481e77388 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java
@@ -211,11 +211,11 @@ public void trim() throws IOException {
     }
 
     /**
-     * Update the file reference (to be used with caution since this does not reopen the file channel)
-     * @param file The new file to use
+     * Update the parent directory (to be used with caution since this does not reopen the file channel)
+     * @param parentDir The new parent directory
      */
-    public void setFile(File file) {
-        this.file = file;
+    public void updateParentDir(File parentDir) {
+        this.file = new File(parentDir, file.getName());
     }
 
     /**
diff --git a/core/src/main/scala/kafka/log/AbstractIndex.scala b/core/src/main/scala/kafka/log/AbstractIndex.scala
index d742a2dbfead1..9fabb6322e128 100644
--- a/core/src/main/scala/kafka/log/AbstractIndex.scala
+++ b/core/src/main/scala/kafka/log/AbstractIndex.scala
@@ -34,11 +34,11 @@ import scala.math.ceil
 /**
  * The abstract index class which holds entry format agnostic methods.
  *
- * @param file The index file
+ * @param _file The index file
  * @param baseOffset the base offset of the segment that this index is corresponding to.
  * @param maxIndexSize The maximum index size in bytes.
  */
-abstract class AbstractIndex(@volatile var file: File, val baseOffset: Long, val maxIndexSize: Int = -1,
+abstract class AbstractIndex(@volatile private var _file: File, val baseOffset: Long, val maxIndexSize: Int = -1,
                              val writable: Boolean) extends Closeable {
   import AbstractIndex._
 
@@ -155,12 +155,16 @@ abstract class AbstractIndex(@volatile var file: File, val baseOffset: Long, val
    */
   def isFull: Boolean = _entries >= _maxEntries
 
+  def file: File = _file
+
   def maxEntries: Int = _maxEntries
 
   def entries: Int = _entries
 
   def length: Long = _length
 
+  def updateParentDir(parentDir: File): Unit = _file = new File(parentDir, file.getName)
+
   /**
    * Reset the size of the memory map and the underneath file. This is used in two kinds of cases: (1) in
    * trimToValidSize() which is called at closing the segment or new segment being rolled; (2) at
@@ -209,7 +213,7 @@ abstract class AbstractIndex(@volatile var file: File, val baseOffset: Long, val
    */
   def renameTo(f: File): Unit = {
     try Utils.atomicMoveWithFallback(file.toPath, f.toPath)
-    finally file = f
+    finally _file = f
   }
 
   /**
diff --git a/core/src/main/scala/kafka/log/LazyIndex.scala b/core/src/main/scala/kafka/log/LazyIndex.scala
index 596fcaac52faa..a5a7c34a6e5b7 100644
--- a/core/src/main/scala/kafka/log/LazyIndex.scala
+++ b/core/src/main/scala/kafka/log/LazyIndex.scala
@@ -18,22 +18,32 @@
 package kafka.log
 
 import java.io.File
+import java.nio.file.{Files, NoSuchFileException}
 import java.util.concurrent.locks.ReentrantLock
 
 import LazyIndex._
 import kafka.utils.CoreUtils.inLock
 import kafka.utils.threadsafe
+import org.apache.kafka.common.utils.Utils
 
 /**
-  * A wrapper over an `AbstractIndex` instance that provides a mechanism to defer loading (i.e. memory mapping) the
-  * underlying index until it is accessed for the first time via the `get` method.
+  * A wrapper over an `AbstractIndex` instance that provides a mechanism to defer loading
+  * (i.e. memory mapping) the underlying index until it is accessed for the first time via the
+  * `get` method.
   *
-  * This is an important optimization with regards to broker start-up time if it has a large number of segments.
+  * In addition, this class exposes a number of methods (e.g. updateParentDir, renameTo, close,
+  * etc.) that provide the desired behavior without causing the index to be loaded. If the index
+  * had previously been loaded, the methods in this class simply delegate to the relevant method in
+  * the index.
   *
-  * Methods of this class are thread safe. Make sure to check `AbstractIndex` subclasses documentation
-  * to establish their thread safety.
+  * This is an important optimization with regards to broker start-up and shutdown time if it has a
+  * large number of segments.
   *
-  * @param loadIndex A function that takes a `File` pointing to an index and returns a loaded `AbstractIndex` instance.
+  * Methods of this class are thread safe. Make sure to check `AbstractIndex` subclasses
+  * documentation to establish their thread safety.
+  *
+  * @param loadIndex A function that takes a `File` pointing to an index and returns a loaded
+  *                  `AbstractIndex` instance.
   */
 @threadsafe
 class LazyIndex[T <: AbstractIndex] private (@volatile private var indexWrapper: IndexWrapper, loadIndex: File => T) {
@@ -42,12 +52,6 @@ class LazyIndex[T <: AbstractIndex] private (@volatile private var indexWrapper:
 
   def file: File = indexWrapper.file
 
-  def file_=(f: File): Unit = {
-    inLock(lock) {
-      indexWrapper.file = f
-    }
-  }
-
   def get: T = {
     indexWrapper match {
       case indexValue: IndexValue[T] => indexValue.index
@@ -64,6 +68,36 @@ class LazyIndex[T <: AbstractIndex] private (@volatile private var indexWrapper:
     }
   }
 
+  def updateParentDir(parentDir: File): Unit = {
+    inLock(lock) {
+      indexWrapper.updateParentDir(parentDir)
+    }
+  }
+
+  def renameTo(f: File): Unit = {
+    inLock(lock) {
+      indexWrapper.renameTo(f)
+    }
+  }
+
+  def deleteIfExists(): Boolean = {
+    inLock(lock) {
+      indexWrapper.deleteIfExists()
+    }
+  }
+
+  def close(): Unit = {
+    inLock(lock) {
+      indexWrapper.close()
+    }
+  }
+
+  def closeHandler(): Unit = {
+    inLock(lock) {
+      indexWrapper.closeHandler()
+    }
+  }
+
 }
 
 object LazyIndex {
@@ -75,15 +109,57 @@ object LazyIndex {
     new LazyIndex(new IndexFile(file), file => new TimeIndex(file, baseOffset, maxIndexSize, writable))
 
   private sealed trait IndexWrapper {
+
     def file: File
-    def file_=(f: File)
+
+    def updateParentDir(f: File): Unit
+
+    def renameTo(f: File): Unit
+
+    def deleteIfExists(): Boolean
+
+    def close(): Unit
+
+    def closeHandler(): Unit
+
   }
 
-  private class IndexFile(@volatile var file: File) extends IndexWrapper
+  private class IndexFile(@volatile private var _file: File) extends IndexWrapper {
+
+    def file: File = _file
+
+    def updateParentDir(parentDir: File): Unit = _file = new File(parentDir, file.getName)
+
+    def renameTo(f: File): Unit = {
+      try Utils.atomicMoveWithFallback(file.toPath, f.toPath)
+      catch {
+        case _: NoSuchFileException if !file.exists => ()
+      }
+      finally _file = f
+    }
+
+    def deleteIfExists(): Boolean = Files.deleteIfExists(file.toPath)
+
+    def close(): Unit = ()
+
+    def closeHandler(): Unit = ()
+
+  }
 
   private class IndexValue[T <: AbstractIndex](val index: T) extends IndexWrapper {
-    override def file: File = index.file
-    override def file_=(f: File): Unit = index.file = f
+
+    def file: File = index.file
+
+    def updateParentDir(parentDir: File): Unit = index.updateParentDir(parentDir)
+
+    def renameTo(f: File): Unit = index.renameTo(f)
+
+    def deleteIfExists(): Boolean = index.deleteIfExists()
+
+    def close(): Unit = index.close()
+
+    def closeHandler(): Unit = index.closeHandler()
+
   }
 
 }
diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index 00974403b756b..0d9f503e45019 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -996,7 +996,7 @@ class Log(@volatile private var _dir: File,
         if (renamedDir != dir) {
           _dir = renamedDir
           _parentDir = renamedDir.getParent
-          logSegments.foreach(_.updateDir(renamedDir))
+          logSegments.foreach(_.updateParentDir(renamedDir))
           producerStateManager.logDir = dir
           // re-initialize leader epoch cache so that LeaderEpochCheckpointFile.checkpoint can correctly reference
           // the checkpoint file in renamed log directory
diff --git a/core/src/main/scala/kafka/log/LogSegment.scala b/core/src/main/scala/kafka/log/LogSegment.scala
index 97fb45baabd0a..6d3beb7107617 100755
--- a/core/src/main/scala/kafka/log/LogSegment.scala
+++ b/core/src/main/scala/kafka/log/LogSegment.scala
@@ -480,21 +480,21 @@ class LogSegment private[log] (val log: FileRecords,
    * Update the directory reference for the log and indices in this segment. This would typically be called after a
    * directory is renamed.
    */
-  def updateDir(dir: File): Unit = {
-    log.setFile(new File(dir, log.file.getName))
-    lazyOffsetIndex.file = new File(dir, lazyOffsetIndex.file.getName)
-    lazyTimeIndex.file = new File(dir, lazyTimeIndex.file.getName)
-    txnIndex.file = new File(dir, txnIndex.file.getName)
+  def updateParentDir(dir: File): Unit = {
+    log.updateParentDir(dir)
+    lazyOffsetIndex.updateParentDir(dir)
+    lazyTimeIndex.updateParentDir(dir)
+    txnIndex.updateParentDir(dir)
   }
 
   /**
-   * Change the suffix for the index and log file for this log segment
+   * Change the suffix for the index and log files for this log segment
    * IOException from this method should be handled by the caller
    */
   def changeFileSuffixes(oldSuffix: String, newSuffix: String): Unit = {
     log.renameTo(new File(CoreUtils.replaceSuffix(log.file.getPath, oldSuffix, newSuffix)))
-    offsetIndex.renameTo(new File(CoreUtils.replaceSuffix(lazyOffsetIndex.file.getPath, oldSuffix, newSuffix)))
-    timeIndex.renameTo(new File(CoreUtils.replaceSuffix(lazyTimeIndex.file.getPath, oldSuffix, newSuffix)))
+    lazyOffsetIndex.renameTo(new File(CoreUtils.replaceSuffix(lazyOffsetIndex.file.getPath, oldSuffix, newSuffix)))
+    lazyTimeIndex.renameTo(new File(CoreUtils.replaceSuffix(lazyTimeIndex.file.getPath, oldSuffix, newSuffix)))
     txnIndex.renameTo(new File(CoreUtils.replaceSuffix(txnIndex.file.getPath, oldSuffix, newSuffix)))
   }
 
@@ -585,11 +585,11 @@ class LogSegment private[log] (val log: FileRecords,
    * Close this log segment
    */
   def close(): Unit = {
-    if (_maxTimestampSoFar.nonEmpty || _offsetOfMaxTimestampSoFar.nonEmpty) {
-      CoreUtils.swallow(timeIndex.maybeAppend(maxTimestampSoFar, offsetOfMaxTimestampSoFar, skipFullCheck = true), this)
-    }
-    CoreUtils.swallow(offsetIndex.close(), this)
-    CoreUtils.swallow(timeIndex.close(), this)
+    if (_maxTimestampSoFar.nonEmpty || _offsetOfMaxTimestampSoFar.nonEmpty)
+      CoreUtils.swallow(timeIndex.maybeAppend(maxTimestampSoFar, offsetOfMaxTimestampSoFar,
+        skipFullCheck = true), this)
+    CoreUtils.swallow(lazyOffsetIndex.close(), this)
+    CoreUtils.swallow(lazyTimeIndex.close(), this)
     CoreUtils.swallow(log.close(), this)
     CoreUtils.swallow(txnIndex.close(), this)
   }
@@ -598,8 +598,8 @@ class LogSegment private[log] (val log: FileRecords,
     * Close file handlers used by the log segment but don't write to disk. This is used when the disk may have failed
     */
   def closeHandlers(): Unit = {
-    CoreUtils.swallow(offsetIndex.closeHandler(), this)
-    CoreUtils.swallow(timeIndex.closeHandler(), this)
+    CoreUtils.swallow(lazyOffsetIndex.closeHandler(), this)
+    CoreUtils.swallow(lazyTimeIndex.closeHandler(), this)
     CoreUtils.swallow(log.closeHandlers(), this)
     CoreUtils.swallow(txnIndex.close(), this)
   }
@@ -622,8 +622,8 @@ class LogSegment private[log] (val log: FileRecords,
 
     CoreUtils.tryAll(Seq(
       () => delete(log.deleteIfExists _, "log", log.file, logIfMissing = true),
-      () => delete(offsetIndex.deleteIfExists _, "offset index", lazyOffsetIndex.file, logIfMissing = true),
-      () => delete(timeIndex.deleteIfExists _, "time index", lazyTimeIndex.file, logIfMissing = true),
+      () => delete(lazyOffsetIndex.deleteIfExists _, "offset index", lazyOffsetIndex.file, logIfMissing = true),
+      () => delete(lazyTimeIndex.deleteIfExists _, "time index", lazyTimeIndex.file, logIfMissing = true),
       () => delete(txnIndex.deleteIfExists _, "transaction index", txnIndex.file, logIfMissing = false)
     ))
   }
diff --git a/core/src/main/scala/kafka/log/TransactionIndex.scala b/core/src/main/scala/kafka/log/TransactionIndex.scala
index 696bc3a2a4f0c..9152bc41ab353 100644
--- a/core/src/main/scala/kafka/log/TransactionIndex.scala
+++ b/core/src/main/scala/kafka/log/TransactionIndex.scala
@@ -42,12 +42,13 @@ private[log] case class TxnIndexSearchResult(abortedTransactions: List[AbortedTx
  * order to find the start of the transactions.
  */
 @nonthreadsafe
-class TransactionIndex(val startOffset: Long, @volatile var file: File) extends Logging {
+class TransactionIndex(val startOffset: Long, @volatile private var _file: File) extends Logging {
+
   // note that the file is not created until we need it
   @volatile private var maybeChannel: Option[FileChannel] = None
   private var lastOffset: Option[Long] = None
 
-  if (file.exists)
+  if (_file.exists)
     openChannel()
 
   def append(abortedTxn: AbortedTxn): Unit = {
@@ -62,6 +63,10 @@ class TransactionIndex(val startOffset: Long, @volatile var file: File) extends
 
   def flush(): Unit = maybeChannel.foreach(_.force(true))
 
+  def file: File = _file
+
+  def updateParentDir(parentDir: File): Unit = _file = new File(parentDir, file.getName)
+
   /**
    * Delete this index.
    *
@@ -106,7 +111,7 @@ class TransactionIndex(val startOffset: Long, @volatile var file: File) extends
     try {
       if (file.exists)
         Utils.atomicMoveWithFallback(file.toPath, f.toPath)
-    } finally file = f
+    } finally _file = f
   }
 
   def truncateTo(offset: Long): Unit = {
diff --git a/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala b/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala
index 6cc6354ef2cf8..a29e7e5415c3a 100644
--- a/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala
@@ -145,6 +145,9 @@ class LogSegmentTest {
     val maxSegmentMs = 300000
     val time = new MockTime
     val seg = createSegment(0, time = time)
+    // Force load indexes before closing the segment
+    seg.timeIndex
+    seg.offsetIndex
     seg.close()
 
     val reopened = createSegment(0, time = time)
@@ -262,11 +265,25 @@ class LogSegmentTest {
     val seg = createSegment(40)
     val logFile = seg.log.file
     val indexFile = seg.lazyOffsetIndex.file
+    val timeIndexFile = seg.lazyTimeIndex.file
+    // Ensure that files for offset and time indices have not been created eagerly.
+    assertFalse(seg.lazyOffsetIndex.file.exists)
+    assertFalse(seg.lazyTimeIndex.file.exists)
     seg.changeFileSuffixes("", ".deleted")
+    // Ensure that attempt to change suffixes for non-existing offset and time indices does not create new files.
+    assertFalse(seg.lazyOffsetIndex.file.exists)
+    assertFalse(seg.lazyTimeIndex.file.exists)
+    // Ensure that file names are updated accordingly.
     assertEquals(logFile.getAbsolutePath + ".deleted", seg.log.file.getAbsolutePath)
     assertEquals(indexFile.getAbsolutePath + ".deleted", seg.lazyOffsetIndex.file.getAbsolutePath)
+    assertEquals(timeIndexFile.getAbsolutePath + ".deleted", seg.lazyTimeIndex.file.getAbsolutePath)
     assertTrue(seg.log.file.exists)
+    // Ensure lazy creation of offset index file upon accessing it.
+    seg.lazyOffsetIndex.get
     assertTrue(seg.lazyOffsetIndex.file.exists)
+    // Ensure lazy creation of time index file upon accessing it.
+    seg.lazyTimeIndex.get
+    assertTrue(seg.lazyTimeIndex.file.exists)
   }
 
   /**
diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/server/HighwatermarkCheckpointBench.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/server/HighwatermarkCheckpointBench.java
index 9cb8ac6a97a9e..98861dc1c7f19 100644
--- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/server/HighwatermarkCheckpointBench.java
+++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/server/HighwatermarkCheckpointBench.java
@@ -51,7 +51,6 @@
 import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
-import scala.Option;
 
 import java.io.File;
 import java.util.ArrayList;
@@ -59,8 +58,9 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
-import scala.jdk.CollectionConverters;
 
+import scala.collection.JavaConverters;
+import scala.Option;
 
 @Warmup(iterations = 5)
 @Measurement(iterations = 5)
@@ -91,6 +91,7 @@ public class HighwatermarkCheckpointBench {
     private LogManager logManager;
 
 
+    @SuppressWarnings("deprecation")
     @Setup(Level.Trial)
     public void setup() {
         this.scheduler = new KafkaScheduler(1, "scheduler-thread", true);
@@ -101,8 +102,8 @@ public void setup() {
         this.time = new MockTime();
         this.failureChannel = new LogDirFailureChannel(brokerProperties.logDirs().size());
         final List files =
-            CollectionConverters.seqAsJavaList(brokerProperties.logDirs()).stream().map(File::new).collect(Collectors.toList());
-        this.logManager = TestUtils.createLogManager(CollectionConverters.asScalaBuffer(files),
+            JavaConverters.seqAsJavaList(brokerProperties.logDirs()).stream().map(File::new).collect(Collectors.toList());
+        this.logManager = TestUtils.createLogManager(JavaConverters.asScalaBuffer(files),
                 LogConfig.apply(), CleanerConfig.apply(1, 4 * 1024 * 1024L, 0.9d,
                         1024 * 1024, 32 * 1024 * 1024,
                         Double.MAX_VALUE, 15 * 1000, true, "MD5"), time);
@@ -160,7 +161,7 @@ public void tearDown() throws Exception {
         this.metrics.close();
         this.scheduler.shutdown();
         this.quotaManagers.shutdown();
-        for (File dir : CollectionConverters.asJavaCollection(logManager.liveLogDirs())) {
+        for (File dir : JavaConverters.asJavaCollection(logManager.liveLogDirs())) {
             Utils.delete(dir);
         }
     }

From 01b343907e70e33fcd8ab21dadae52ad51a1915d Mon Sep 17 00:00:00 2001
From: Lucas Wang 
Date: Thu, 5 Mar 2020 11:36:30 -0800
Subject: [PATCH 1053/1071] [LI-HOTFIX] Use max broker epoch in controller
 requests TICKET = KAFKA-7186 LI_DESCERIPTION = Currently the controller is
 creating a separate copy of UpdateMetadataRequest for each broker in the
 cluster, which may cause OOMs upon controller startup. This patch is the
 first step toward making the UpdateMetadataRequests cacheable. We change the
 broker-specific epoch in control requests to a common maximum broker epoch
 seen in all brokers, such that all UpdateMetadataRequests have the same
 payload.

EXIT_CRITERIA = KAFKA-7186
---
 .../requests/AbstractControlRequest.java      |  6 +-
 .../common/requests/LeaderAndIsrRequest.java  | 11 ++-
 .../common/requests/StopReplicaRequest.java   | 11 ++-
 .../requests/UpdateMetadataRequest.java       | 11 ++-
 .../common/message/LeaderAndIsrRequest.json   | 18 +++--
 .../common/message/LeaderAndIsrResponse.json  |  8 +-
 .../common/message/StopReplicaRequest.json    | 12 ++-
 .../common/message/StopReplicaResponse.json   |  6 +-
 .../common/message/UpdateMetadataRequest.json | 12 ++-
 .../message/UpdateMetadataResponse.json       |  8 +-
 .../kafka/common/message/MessageTest.java     |  2 +-
 .../requests/LeaderAndIsrRequestTest.java     | 12 +--
 .../requests/LeaderAndIsrResponseTest.java    |  2 +-
 .../common/requests/RequestResponseTest.java  |  6 +-
 .../requests/StopReplicaRequestTest.java      |  6 +-
 .../requests/StopReplicaResponseTest.java     |  2 +-
 .../requests/UpdateMetadataRequestTest.java   |  8 +-
 .../src/main/scala/kafka/api/ApiVersion.scala | 14 +++-
 .../controller/ControllerChannelManager.scala | 28 ++++---
 .../kafka/controller/ControllerContext.scala  |  1 +
 .../main/scala/kafka/server/KafkaApis.scala   | 33 +++++----
 .../kafka/api/AuthorizerIntegrationTest.scala |  6 +-
 .../api/RecordHeaderProducerSendTest.scala    | 61 +++++++--------
 .../scala/unit/kafka/api/ApiVersionTest.scala |  3 +-
 .../ControllerChannelManagerTest.scala        | 16 ++--
 .../server/BrokerEpochIntegrationTest.scala   |  4 +-
 .../CacheableBrokerEpochIntegrationTest.scala | 74 +++++++++++++++++++
 .../unit/kafka/server/KafkaApisTest.scala     |  5 +-
 .../kafka/server/LeaderElectionTest.scala     |  5 +-
 .../unit/kafka/server/MetadataCacheTest.scala | 12 +--
 .../kafka/server/ReplicaManagerTest.scala     | 48 ++++++------
 .../unit/kafka/server/RequestQuotaTest.scala  |  6 +-
 .../kafka/server/ServerShutdownTest.scala     |  2 +-
 .../kafka/server/StopReplicaRequestTest.scala |  1 +
 34 files changed, 311 insertions(+), 149 deletions(-)
 create mode 100644 core/src/test/scala/unit/kafka/server/CacheableBrokerEpochIntegrationTest.scala

diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractControlRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractControlRequest.java
index dc4a1e21e8dd9..cf22160aad480 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractControlRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractControlRequest.java
@@ -27,12 +27,14 @@ public static abstract class Builder extends Abstract
         protected final int controllerId;
         protected final int controllerEpoch;
         protected final long brokerEpoch;
+        protected final long maxBrokerEpoch;
 
-        protected Builder(ApiKeys api, short version, int controllerId, int controllerEpoch, long brokerEpoch) {
+        protected Builder(ApiKeys api, short version, int controllerId, int controllerEpoch, long brokerEpoch, long maxBrokerEpoch) {
             super(api, version);
             this.controllerId = controllerId;
             this.controllerEpoch = controllerEpoch;
             this.brokerEpoch = brokerEpoch;
+            this.maxBrokerEpoch = maxBrokerEpoch;
         }
 
     }
@@ -47,4 +49,6 @@ protected AbstractControlRequest(ApiKeys api, short version) {
 
     public abstract long brokerEpoch();
 
+    public abstract long maxBrokerEpoch();
+
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
index 270069aef3c58..54d0e9762f3e2 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
@@ -45,9 +45,9 @@ public static class Builder extends AbstractControlRequest.Builder partitionStates;
         private final Collection liveLeaders;
 
-        public Builder(short version, int controllerId, int controllerEpoch, long brokerEpoch,
+        public Builder(short version, int controllerId, int controllerEpoch, long brokerEpoch, long maxBrokerEpoch,
                        List partitionStates, Collection liveLeaders) {
-            super(ApiKeys.LEADER_AND_ISR, version, controllerId, controllerEpoch, brokerEpoch);
+            super(ApiKeys.LEADER_AND_ISR, version, controllerId, controllerEpoch, brokerEpoch, maxBrokerEpoch);
             this.partitionStates = partitionStates;
             this.liveLeaders = liveLeaders;
         }
@@ -64,6 +64,7 @@ public LeaderAndIsrRequest build(short version) {
                 .setControllerId(controllerId)
                 .setControllerEpoch(controllerEpoch)
                 .setBrokerEpoch(brokerEpoch)
+                .setMaxBrokerEpoch(maxBrokerEpoch)
                 .setLiveLeaders(leaders);
 
             if (version >= 2) {
@@ -95,6 +96,7 @@ public String toString() {
                 .append(", controllerId=").append(controllerId)
                 .append(", controllerEpoch=").append(controllerEpoch)
                 .append(", brokerEpoch=").append(brokerEpoch)
+                .append(", maxBrokerEpoch=").append(maxBrokerEpoch)
                 .append(", partitionStates=").append(partitionStates)
                 .append(", liveLeaders=(").append(Utils.join(liveLeaders, ", ")).append(")")
                 .append(")");
@@ -164,6 +166,11 @@ public long brokerEpoch() {
         return data.brokerEpoch();
     }
 
+    @Override
+    public long maxBrokerEpoch() {
+        return data.maxBrokerEpoch();
+    }
+
     public Iterable partitionStates() {
         if (version() >= 2)
             return () -> new FlattenedIterator<>(data.topicStates().iterator(),
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
index 0f60e20f311ae..ea8046875b6bf 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
@@ -43,9 +43,9 @@ public static class Builder extends AbstractControlRequest.Builder partitions;
 
-        public Builder(short version, int controllerId, int controllerEpoch, long brokerEpoch, boolean deletePartitions,
+        public Builder(short version, int controllerId, int controllerEpoch, long brokerEpoch, long maxBrokerEpoch, boolean deletePartitions,
                        Collection partitions) {
-            super(ApiKeys.STOP_REPLICA, version, controllerId, controllerEpoch, brokerEpoch);
+            super(ApiKeys.STOP_REPLICA, version, controllerId, controllerEpoch, brokerEpoch, maxBrokerEpoch);
             this.deletePartitions = deletePartitions;
             this.partitions = partitions;
         }
@@ -55,6 +55,7 @@ public StopReplicaRequest build(short version) {
                 .setControllerId(controllerId)
                 .setControllerEpoch(controllerEpoch)
                 .setBrokerEpoch(brokerEpoch)
+                .setMaxBrokerEpoch(maxBrokerEpoch)
                 .setDeletePartitions(deletePartitions);
 
             if (version >= 1) {
@@ -85,6 +86,7 @@ public String toString() {
                 append(", controllerEpoch=").append(controllerEpoch).
                 append(", deletePartitions=").append(deletePartitions).
                 append(", brokerEpoch=").append(brokerEpoch).
+                append(", maxBrokerEpoch=").append(maxBrokerEpoch).
                 append(", partitions=").append(Utils.join(partitions, ",")).
                 append(")");
             return bld.toString();
@@ -156,6 +158,11 @@ public long brokerEpoch() {
         return data.brokerEpoch();
     }
 
+    @Override
+    public long maxBrokerEpoch() {
+        return data.maxBrokerEpoch();
+    }
+
     public static StopReplicaRequest parse(ByteBuffer buffer, short version) {
         return new StopReplicaRequest(ApiKeys.STOP_REPLICA.parseRequest(version, buffer), version);
     }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
index 75e3542a3d4a1..1ef4118dabb9a 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
@@ -51,9 +51,9 @@ public static class Builder extends AbstractControlRequest.Builder requestCache = new HashMap<>();
 
-        public Builder(short version, int controllerId, int controllerEpoch, long brokerEpoch,
+        public Builder(short version, int controllerId, int controllerEpoch, long brokerEpoch, long maxBrokerEpoch,
                        List partitionStates, List liveBrokers) {
-            super(ApiKeys.UPDATE_METADATA, version, controllerId, controllerEpoch, brokerEpoch);
+            super(ApiKeys.UPDATE_METADATA, version, controllerId, controllerEpoch, brokerEpoch, maxBrokerEpoch);
             this.partitionStates = partitionStates;
             this.liveBrokers = liveBrokers;
         }
@@ -90,6 +90,7 @@ public UpdateMetadataRequest build(short version) {
                     .setControllerId(controllerId)
                     .setControllerEpoch(controllerEpoch)
                     .setBrokerEpoch(brokerEpoch)
+                    .setMaxBrokerEpoch(maxBrokerEpoch)
                     .setLiveBrokers(liveBrokers);
 
             if (version >= 5) {
@@ -130,6 +131,7 @@ public String toString() {
                 append(", controllerId=").append(controllerId).
                 append(", controllerEpoch=").append(controllerEpoch).
                 append(", brokerEpoch=").append(brokerEpoch).
+                append(", maxBrokerEpoch=").append(maxBrokerEpoch).
                 append(", liveBrokers=").append(Utils.join(liveBrokers, ", ")).
                 append(")");
 
@@ -215,6 +217,11 @@ public long brokerEpoch() {
         return data.brokerEpoch();
     }
 
+    @Override
+    public long maxBrokerEpoch() {
+        return data.maxBrokerEpoch();
+    }
+
     @Override
     public UpdateMetadataResponse getErrorResponse(int throttleTimeMs, Throwable e) {
         UpdateMetadataResponseData data = new UpdateMetadataResponseData()
diff --git a/clients/src/main/resources/common/message/LeaderAndIsrRequest.json b/clients/src/main/resources/common/message/LeaderAndIsrRequest.json
index 852968801cee3..3a1f01e4790b0 100644
--- a/clients/src/main/resources/common/message/LeaderAndIsrRequest.json
+++ b/clients/src/main/resources/common/message/LeaderAndIsrRequest.json
@@ -21,18 +21,24 @@
   //
   // Version 2 adds broker epoch and reorganizes the partitions by topic.
   //
-  // Version 3 adds AddingReplicas and RemovingReplicas
-  "validVersions": "0-4",
-  "flexibleVersions": "4+",
+  // Version 3 adds the max broker epoch to make the UpdateMetadataRequest cacheable
+  //
+  // Version 4 adds AddingReplicas and RemovingReplicas
+  //
+  // Version 5 is the first flexible version.
+  "validVersions": "0-5",
+  "flexibleVersions": "5+",
   "fields": [
     { "name": "ControllerId", "type": "int32", "versions": "0+", "entityType": "brokerId",
       "about": "The current controller ID." },
     { "name": "ControllerEpoch", "type": "int32", "versions": "0+",
       "about": "The current controller epoch." },
-    { "name": "BrokerEpoch", "type": "int64", "versions": "2+", "ignorable": true, "default": "-1",
+    { "name": "BrokerEpoch", "type": "int64", "versions": "2", "ignorable": true, "default": "-1",
       "about": "The current broker epoch." },
     { "name": "UngroupedPartitionStates", "type": "[]LeaderAndIsrPartitionState", "versions": "0-1",
       "about": "The state of each partition, in a v0 or v1 message." },
+    { "name": "MaxBrokerEpoch", "type": "int64", "versions": "3+", "ignorable": true, "default": "-1",
+      "about": "The max broker epoch." },
     // In v0 or v1 requests, each partition is listed alongside its topic name.
     // In v2+ requests, partitions are organized by topic, so that each topic name
     // only needs to be listed once.
@@ -71,9 +77,9 @@
         "about": "The ZooKeeper version." },
       { "name": "Replicas", "type": "[]int32", "versions": "0+",
         "about": "The replica IDs." },
-      { "name": "AddingReplicas", "type": "[]int32", "versions": "3+", "ignorable": true,
+      { "name": "AddingReplicas", "type": "[]int32", "versions": "4+", "ignorable": true,
         "about": "The replica IDs that we are adding this partition to, or null if no replicas are being added." },
-      { "name": "RemovingReplicas", "type": "[]int32", "versions": "3+", "ignorable": true,
+      { "name": "RemovingReplicas", "type": "[]int32", "versions": "4+", "ignorable": true,
         "about": "The replica IDs that we are removing this partition from, or null if no replicas are being removed." },
       { "name": "IsNew", "type": "bool", "versions": "1+", "default": "false", "ignorable": true,
         "about": "Whether the replica should have existed on the broker or not." }
diff --git a/clients/src/main/resources/common/message/LeaderAndIsrResponse.json b/clients/src/main/resources/common/message/LeaderAndIsrResponse.json
index 10c3cd95e5fb2..073d89b190ee2 100644
--- a/clients/src/main/resources/common/message/LeaderAndIsrResponse.json
+++ b/clients/src/main/resources/common/message/LeaderAndIsrResponse.json
@@ -22,8 +22,12 @@
   // Version 2 is the same as version 1.
   //
   // Version 3 is the same as version 2.
-  "validVersions": "0-4",
-  "flexibleVersions": "4+",
+  //
+  // Version 4 is the same as version 3
+  //
+  // Version 5 is is the first flexible version
+  "validVersions": "0-5",
+  "flexibleVersions": "5+",
   "fields": [
     { "name": "ErrorCode", "type": "int16", "versions": "0+",
       "about": "The error code, or 0 if there was no error." },
diff --git a/clients/src/main/resources/common/message/StopReplicaRequest.json b/clients/src/main/resources/common/message/StopReplicaRequest.json
index 5c13705318a54..3d016b0fe181c 100644
--- a/clients/src/main/resources/common/message/StopReplicaRequest.json
+++ b/clients/src/main/resources/common/message/StopReplicaRequest.json
@@ -19,15 +19,21 @@
   "name": "StopReplicaRequest",
   // Version 1 adds the broker epoch and reorganizes the partitions to be stored
   // per topic.
-  "validVersions": "0-2",
-  "flexibleVersions": "2+",
+  //
+  // Version 2 adds the max broker epoch to make the UpdateMetadataRequest cacheable
+  //
+  // Version 3 is the first flexible version.
+  "validVersions": "0-3",
+  "flexibleVersions": "3+",
   "fields": [
     { "name": "ControllerId", "type": "int32", "versions": "0+", "entityType": "brokerId",
       "about": "The controller id." },
     { "name": "ControllerEpoch", "type": "int32", "versions": "0+",
       "about": "The controller epoch." },
-    { "name": "BrokerEpoch", "type": "int64", "versions": "1+", "default": "-1", "ignorable": true,
+    { "name": "BrokerEpoch", "type": "int64", "versions": "1", "default": "-1", "ignorable": true,
       "about": "The broker epoch." },
+    { "name": "MaxBrokerEpoch", "type": "int64", "versions": "2+", "default": "-1", "ignorable": true,
+      "about": "The max broker epoch." },
     { "name": "DeletePartitions", "type": "bool", "versions": "0+",
       "about": "Whether these partitions should be deleted." },
     { "name": "UngroupedPartitions", "type": "[]StopReplicaPartitionV0", "versions": "0",
diff --git a/clients/src/main/resources/common/message/StopReplicaResponse.json b/clients/src/main/resources/common/message/StopReplicaResponse.json
index d864e91a41b0e..6fb5128bab9e1 100644
--- a/clients/src/main/resources/common/message/StopReplicaResponse.json
+++ b/clients/src/main/resources/common/message/StopReplicaResponse.json
@@ -18,8 +18,10 @@
   "type": "response",
   "name": "StopReplicaResponse",
   // Version 1 is the same as version 0.
-  "validVersions": "0-2",
-  "flexibleVersions": "2+",
+  // Version 2 is the same as version 1.
+  // Version 3 is the first flexible version.
+  "validVersions": "0-3",
+  "flexibleVersions": "3+",
   "fields": [
     { "name": "ErrorCode", "type": "int16", "versions": "0+",
       "about": "The top-level error code, or 0 if there was no top-level error." },
diff --git a/clients/src/main/resources/common/message/UpdateMetadataRequest.json b/clients/src/main/resources/common/message/UpdateMetadataRequest.json
index 3c45f833a7cab..19ed06184b98d 100644
--- a/clients/src/main/resources/common/message/UpdateMetadataRequest.json
+++ b/clients/src/main/resources/common/message/UpdateMetadataRequest.json
@@ -26,17 +26,23 @@
   // Version 4 adds the offline replica list.
   //
   // Version 5 adds the broker epoch field and normalizes partitions by topic.
-  "validVersions": "0-6",
-  "flexibleVersions": "6+",
+  //
+  // Version 6 adds the max broker epoch to make the UpdateMetadataRequest cacheable
+  //
+  // Version 7 is the first flexible version.
+  "validVersions": "0-7",
+  "flexibleVersions": "7+",
   "fields": [
     { "name": "ControllerId", "type": "int32", "versions": "0+", "entityType": "brokerId",
       "about": "The controller id." },
     { "name": "ControllerEpoch", "type": "int32", "versions": "0+",
       "about": "The controller epoch." },
-    { "name": "BrokerEpoch", "type": "int64", "versions": "5+", "ignorable": true, "default": "-1",
+    { "name": "BrokerEpoch", "type": "int64", "versions": "5", "ignorable": true, "default": "-1",
       "about": "The broker epoch." },
     { "name": "UngroupedPartitionStates", "type": "[]UpdateMetadataPartitionState", "versions": "0-4",
       "about": "In older versions of this RPC, each partition that we would like to update." },
+    { "name": "MaxBrokerEpoch", "type": "int64", "versions": "6+", "ignorable": true, "default": "-1",
+      "about": "The max broker epoch." },
     { "name": "TopicStates", "type": "[]UpdateMetadataTopicState", "versions": "5+",
       "about": "In newer versions of this RPC, each topic that we would like to update.", "fields": [
       { "name": "TopicName", "type": "string", "versions": "5+", "entityType": "topicName",
diff --git a/clients/src/main/resources/common/message/UpdateMetadataResponse.json b/clients/src/main/resources/common/message/UpdateMetadataResponse.json
index aaebed04ab108..68d093f687951 100644
--- a/clients/src/main/resources/common/message/UpdateMetadataResponse.json
+++ b/clients/src/main/resources/common/message/UpdateMetadataResponse.json
@@ -17,9 +17,11 @@
   "apiKey": 6,
   "type": "response",
   "name": "UpdateMetadataResponse",
-  // Versions 1, 2, 3, 4, and 5 are the same as version 0
-  "validVersions": "0-6",
-  "flexibleVersions": "6+",
+  // Versions 1, 2, 3, 4, 5 and 6 are the same as version 0
+  //
+  // Version 7 is the first flexible version.
+  "validVersions": "0-7",
+  "flexibleVersions": "7+",
   "fields": [
       { "name": "ErrorCode", "type": "int16", "versions": "0+",
         "about": "The error code, or 0 if there was no error." }
diff --git a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
index c7dedf94cdbf7..d5662ada18e07 100644
--- a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
@@ -273,7 +273,7 @@ public void testLeaderAndIsrVersions() throws Exception {
             (short) 3,
             new LeaderAndIsrRequestData().setTopicStates(Collections.singletonList(partitionStateWithAddingRemovingReplicas)),
             new LeaderAndIsrRequestData().setTopicStates(Collections.singletonList(partitionStateNoAddingRemovingReplicas)));
-        testAllMessageRoundTripsFromVersion((short) 3, new LeaderAndIsrRequestData().setTopicStates(Collections.singletonList(partitionStateWithAddingRemovingReplicas)));
+        testAllMessageRoundTripsFromVersion((short) 4, new LeaderAndIsrRequestData().setTopicStates(Collections.singletonList(partitionStateWithAddingRemovingReplicas)));
     }
 
     @Test
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java
index 2235a8f643cf8..7a7475cc4790c 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java
@@ -50,7 +50,7 @@ public class LeaderAndIsrRequestTest {
     @Test
     public void testUnsupportedVersion() {
         LeaderAndIsrRequest.Builder builder = new LeaderAndIsrRequest.Builder(
-                (short) (LEADER_AND_ISR.latestVersion() + 1), 0, 0, 0,
+                (short) (LEADER_AND_ISR.latestVersion() + 1), 0, 0, 0L, 0L,
                 Collections.emptyList(), Collections.emptySet());
         assertThrows(UnsupportedVersionException.class, builder::build);
     }
@@ -58,7 +58,7 @@ public void testUnsupportedVersion() {
     @Test
     public void testGetErrorResponse() {
         for (short version = LEADER_AND_ISR.oldestVersion(); version < LEADER_AND_ISR.latestVersion(); version++) {
-            LeaderAndIsrRequest.Builder builder = new LeaderAndIsrRequest.Builder(version, 0, 0, 0,
+            LeaderAndIsrRequest.Builder builder = new LeaderAndIsrRequest.Builder(version, 0, 0, 0, 0,
                     Collections.emptyList(), Collections.emptySet());
             LeaderAndIsrRequest request = builder.build();
             LeaderAndIsrResponse response = request.getErrorResponse(0,
@@ -116,7 +116,7 @@ public void testVersionLogic() {
                 new Node(0, "host0", 9090),
                 new Node(1, "host1", 9091)
             );
-            LeaderAndIsrRequest request = new LeaderAndIsrRequest.Builder(version, 1, 2, 3, partitionStates,
+            LeaderAndIsrRequest request = new LeaderAndIsrRequest.Builder(version, 1, 2, 3, 3, partitionStates,
                 liveNodes).build();
 
             List liveLeaders = liveNodes.stream().map(n -> new LeaderAndIsrLiveLeader()
@@ -133,9 +133,9 @@ public void testVersionLogic() {
             LeaderAndIsrRequest deserializedRequest = new LeaderAndIsrRequest(new LeaderAndIsrRequestData(
                 new ByteBufferAccessor(byteBuffer), version), version);
 
-            // Adding/removing replicas is only supported from version 3, so the deserialized request won't have
+            // Adding/removing replicas is only supported from version 4, so the deserialized request won't have
             // them for earlier versions.
-            if (version < 3) {
+            if (version < 4) {
                 partitionStates.get(0)
                     .setAddingReplicas(emptyList())
                     .setRemovingReplicas(emptyList());
@@ -158,7 +158,7 @@ public void testTopicPartitionGroupingSizeReduction() {
                 .setTopicName(tp.topic())
                 .setPartitionIndex(tp.partition()));
         }
-        LeaderAndIsrRequest.Builder builder = new LeaderAndIsrRequest.Builder((short) 2, 0, 0, 0,
+        LeaderAndIsrRequest.Builder builder = new LeaderAndIsrRequest.Builder((short) 2, 0, 0, 0, 0,
             partitionStates, Collections.emptySet());
 
         LeaderAndIsrRequest v2 = builder.build((short) 2);
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java
index edb962b2c5805..4782a1a408741 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java
@@ -58,7 +58,7 @@ public void testErrorCountsFromGetErrorResponse() {
             .setReplicas(Collections.singletonList(10))
             .setIsNew(false));
         LeaderAndIsrRequest request = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion(),
-                15, 20, 0, partitionStates, Collections.emptySet()).build();
+                15, 20, 0, 0, partitionStates, Collections.emptySet()).build();
         LeaderAndIsrResponse response = request.getErrorResponse(0, Errors.CLUSTER_AUTHORIZATION_FAILED.exception());
         assertEquals(Collections.singletonMap(Errors.CLUSTER_AUTHORIZATION_FAILED, 2), response.errorCounts());
     }
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index 7e8410a7a2b08..0b2012e160585 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -1178,7 +1178,7 @@ private ProduceResponse createProduceResponseWithErrorMessage() {
 
     private StopReplicaRequest createStopReplicaRequest(int version, boolean deletePartitions) {
         Set partitions = Utils.mkSet(new TopicPartition("test", 0));
-        return new StopReplicaRequest.Builder((short) version, 0, 1, 0, deletePartitions, partitions).build();
+        return new StopReplicaRequest.Builder((short) version, 0, 1, 0, 0, deletePartitions, partitions).build();
     }
 
     private StopReplicaResponse createStopReplicaResponse() {
@@ -1265,7 +1265,7 @@ private LeaderAndIsrRequest createLeaderAndIsrRequest(int version) {
                 new Node(0, "test0", 1223),
                 new Node(1, "test1", 1223)
         );
-        return new LeaderAndIsrRequest.Builder((short) version, 1, 10, 0, partitionStates, leaders).build();
+        return new LeaderAndIsrRequest.Builder((short) version, 1, 10, 0, 0, partitionStates, leaders).build();
     }
 
     private LeaderAndIsrResponse createLeaderAndIsrResponse() {
@@ -1354,7 +1354,7 @@ private UpdateMetadataRequest createUpdateMetadataRequest(int version, String ra
                 .setEndpoints(endpoints2)
                 .setRack(rack)
         );
-        return new UpdateMetadataRequest.Builder((short) version, 1, 10, 0, partitionStates,
+        return new UpdateMetadataRequest.Builder((short) version, 1, 10, 0, 0, partitionStates,
             liveBrokers).build();
     }
 
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java
index 5bb4998456af4..ae1859ce1a73a 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java
@@ -38,7 +38,7 @@ public class StopReplicaRequestTest {
     public void testUnsupportedVersion() {
         StopReplicaRequest.Builder builder = new StopReplicaRequest.Builder(
                 (short) (STOP_REPLICA.latestVersion() + 1),
-                0, 0, 0L, false, Collections.emptyList());
+                0, 0, 0L, 0L, false, Collections.emptyList());
         assertThrows(UnsupportedVersionException.class, builder::build);
     }
 
@@ -46,7 +46,7 @@ public void testUnsupportedVersion() {
     public void testGetErrorResponse() {
         for (short version = STOP_REPLICA.oldestVersion(); version < STOP_REPLICA.latestVersion(); version++) {
             StopReplicaRequest.Builder builder = new StopReplicaRequest.Builder(version,
-                    0, 0, 0L, false, Collections.emptyList());
+                    0, 0, 0L, 0L, false, Collections.emptyList());
             StopReplicaRequest request = builder.build();
             StopReplicaResponse response = request.getErrorResponse(0,
                     new ClusterAuthorizationException("Not authorized"));
@@ -57,7 +57,7 @@ public void testGetErrorResponse() {
     @Test
     public void testStopReplicaRequestNormalization() {
         Set tps = TestUtils.generateRandomTopicPartitions(10, 10);
-        StopReplicaRequest.Builder builder = new StopReplicaRequest.Builder((short) 5, 0, 0, 0, false, tps);
+        StopReplicaRequest.Builder builder = new StopReplicaRequest.Builder((short) 5, 0, 0, 0, 0, false, tps);
         assertTrue(MessageTestUtil.messageSize(builder.build((short) 1).data(), (short) 1) <
             MessageTestUtil.messageSize(builder.build((short) 0).data(), (short) 0));
     }
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaResponseTest.java
index 1463a0bc41cc0..9f5c006785f86 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaResponseTest.java
@@ -36,7 +36,7 @@ public class StopReplicaResponseTest {
 
     @Test
     public void testErrorCountsFromGetErrorResponse() {
-        StopReplicaRequest request = new StopReplicaRequest.Builder(ApiKeys.STOP_REPLICA.latestVersion(), 15, 20, 0, false,
+        StopReplicaRequest request = new StopReplicaRequest.Builder(ApiKeys.STOP_REPLICA.latestVersion(), 15, 20, 0, 0, false,
                 Utils.mkSet(new TopicPartition("foo", 0), new TopicPartition("foo", 1))).build();
         StopReplicaResponse response = request.getErrorResponse(0, Errors.CLUSTER_AUTHORIZATION_FAILED.exception());
         assertEquals(Collections.singletonMap(Errors.CLUSTER_AUTHORIZATION_FAILED, 2), response.errorCounts());
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java
index fe688ce75cc9a..4758936bfd97c 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java
@@ -52,7 +52,7 @@ public class UpdateMetadataRequestTest {
     @Test
     public void testUnsupportedVersion() {
         UpdateMetadataRequest.Builder builder = new UpdateMetadataRequest.Builder(
-                (short) (UPDATE_METADATA.latestVersion() + 1), 0, 0, 0,
+                (short) (UPDATE_METADATA.latestVersion() + 1), 0, 0, 0, 0,
                 Collections.emptyList(), Collections.emptyList());
         assertThrows(UnsupportedVersionException.class, builder::build);
     }
@@ -61,7 +61,7 @@ public void testUnsupportedVersion() {
     public void testGetErrorResponse() {
         for (short version = UPDATE_METADATA.oldestVersion(); version < UPDATE_METADATA.latestVersion(); version++) {
             UpdateMetadataRequest.Builder builder = new UpdateMetadataRequest.Builder(
-                    version, 0, 0, 0, Collections.emptyList(), Collections.emptyList());
+                    version, 0, 0, 0, 0, Collections.emptyList(), Collections.emptyList());
             UpdateMetadataRequest request = builder.build();
             UpdateMetadataResponse response = request.getErrorResponse(0,
                     new ClusterAuthorizationException("Not authorized"));
@@ -148,7 +148,7 @@ public void testVersionLogic() {
                     ))
             );
 
-            UpdateMetadataRequest request = new UpdateMetadataRequest.Builder(version, 1, 2, 3,
+            UpdateMetadataRequest request = new UpdateMetadataRequest.Builder(version, 1, 2, 3, 3,
                 partitionStates, liveBrokers).build();
 
             assertEquals(new HashSet<>(partitionStates), iterableToSet(request.partitionStates()));
@@ -200,7 +200,7 @@ public void testTopicPartitionGroupingSizeReduction() {
                 .setTopicName(tp.topic())
                 .setPartitionIndex(tp.partition()));
         }
-        UpdateMetadataRequest.Builder builder = new UpdateMetadataRequest.Builder((short) 5, 0, 0, 0,
+        UpdateMetadataRequest.Builder builder = new UpdateMetadataRequest.Builder((short) 5, 0, 0, 0, 0,
                 partitionStates, Collections.emptyList());
 
         assertTrue(MessageTestUtil.messageSize(builder.build((short) 5).data(), (short) 5) <
diff --git a/core/src/main/scala/kafka/api/ApiVersion.scala b/core/src/main/scala/kafka/api/ApiVersion.scala
index 90ffb10347d8a..fb69cba087d89 100644
--- a/core/src/main/scala/kafka/api/ApiVersion.scala
+++ b/core/src/main/scala/kafka/api/ApiVersion.scala
@@ -91,6 +91,8 @@ object ApiVersion {
     KAFKA_2_3_IV0,
     // Add rack_id to FetchRequest, preferred_read_replica to FetchResponse, and replica_id to OffsetsForLeaderRequest
     KAFKA_2_3_IV1,
+    // Add cacheable broker epoch to UpdateMetadataRequest
+    KAFKA_2_3_IV2,
     // Add adding_replicas and removing_replicas fields to LeaderAndIsrRequest
     KAFKA_2_4_IV0,
     // Flexible version support in inter-broker APIs
@@ -320,20 +322,28 @@ case object KAFKA_2_3_IV1 extends DefaultApiVersion {
   val id: Int = 23
 }
 
+case object KAFKA_2_3_IV2 extends DefaultApiVersion {
+  val shortVersion: String = "2.3"
+  val subVersion = "IV2"
+  val recordVersion = RecordVersion.V2
+  val id: Int = 24
+}
+
 case object KAFKA_2_4_IV0 extends DefaultApiVersion {
   val shortVersion: String = "2.4"
   val subVersion = "IV0"
   val recordVersion = RecordVersion.V2
-  val id: Int = 24
+  val id: Int = 25
 }
 
 case object KAFKA_2_4_IV1 extends DefaultApiVersion {
   val shortVersion: String = "2.4"
   val subVersion = "IV1"
   val recordVersion = RecordVersion.V2
-  val id: Int = 25
+  val id: Int = 26
 }
 
+
 object ApiVersionValidator extends Validator {
 
   override def ensureValid(name: String, value: Any): Unit = {
diff --git a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
index 152743d197b0f..ab858617a5991 100755
--- a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
+++ b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
@@ -467,12 +467,14 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
 
   private def sendLeaderAndIsrRequest(controllerEpoch: Int, stateChangeLog: StateChangeLogger): Unit = {
     val leaderAndIsrRequestVersion: Short =
-      if (config.interBrokerProtocolVersion >= KAFKA_2_4_IV1) 4
-      else if (config.interBrokerProtocolVersion >= KAFKA_2_4_IV0) 3
+      if (config.interBrokerProtocolVersion >= KAFKA_2_4_IV1) 5
+      else if (config.interBrokerProtocolVersion >= KAFKA_2_4_IV0) 4
+      else if (config.interBrokerProtocolVersion >= KAFKA_2_3_IV2) 3
       else if (config.interBrokerProtocolVersion >= KAFKA_2_2_IV0) 2
       else if (config.interBrokerProtocolVersion >= KAFKA_1_0_IV0) 1
       else 0
 
+    val maxBrokerEpoch = controllerContext.maxBrokerEpoch
     leaderAndIsrRequestMap.filterKeys(controllerContext.liveOrShuttingDownBrokerIds.contains).foreach {
       case (broker, leaderAndIsrPartitionStates) =>
         if (stateChangeLog.isTraceEnabled) {
@@ -488,8 +490,9 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
           _.node(config.interBrokerListenerName)
         }
         val brokerEpoch = controllerContext.liveBrokerIdAndEpochs(broker)
+
         val leaderAndIsrRequestBuilder = new LeaderAndIsrRequest.Builder(leaderAndIsrRequestVersion, controllerId, controllerEpoch,
-          brokerEpoch, leaderAndIsrPartitionStates.values.toBuffer.asJava, leaders.asJava)
+          brokerEpoch, maxBrokerEpoch, leaderAndIsrPartitionStates.values.toBuffer.asJava, leaders.asJava)
         sendRequest(broker, leaderAndIsrRequestBuilder, (r: AbstractResponse) => sendEvent(LeaderAndIsrResponseReceived(r, broker)))
 
     }
@@ -504,7 +507,8 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
 
     val partitionStates = updateMetadataRequestPartitionInfoMap.values.toBuffer
     val updateMetadataRequestVersion: Short =
-      if (config.interBrokerProtocolVersion >= KAFKA_2_4_IV1) 6
+      if (config.interBrokerProtocolVersion >= KAFKA_2_4_IV1) 7
+      else if (config.interBrokerProtocolVersion >= KAFKA_2_3_IV2) 6
       else if (config.interBrokerProtocolVersion >= KAFKA_2_2_IV0) 5
       else if (config.interBrokerProtocolVersion >= KAFKA_1_0_IV0) 4
       else if (config.interBrokerProtocolVersion >= KAFKA_0_10_2_IV0) 3
@@ -538,19 +542,22 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
         .setRack(broker.rack.orNull)
     }.toBuffer
 
+    val maxBrokerEpoch = controllerContext.maxBrokerEpoch
     updateMetadataRequestBrokerSet.intersect(controllerContext.liveOrShuttingDownBrokerIds).foreach { broker =>
       val brokerEpoch = controllerContext.liveBrokerIdAndEpochs(broker)
       val updateMetadataRequest = new UpdateMetadataRequest.Builder(updateMetadataRequestVersion, controllerId, controllerEpoch,
-        brokerEpoch, partitionStates.asJava, liveBrokers.asJava)
+        brokerEpoch, maxBrokerEpoch, partitionStates.asJava, liveBrokers.asJava)
       sendRequest(broker, updateMetadataRequest)
     }
+
     updateMetadataRequestBrokerSet.clear()
     updateMetadataRequestPartitionInfoMap.clear()
   }
 
   private def sendStopReplicaRequests(controllerEpoch: Int): Unit = {
     val stopReplicaRequestVersion: Short =
-      if (config.interBrokerProtocolVersion >= KAFKA_2_4_IV1) 2
+      if (config.interBrokerProtocolVersion >= KAFKA_2_4_IV1) 3
+      else if (config.interBrokerProtocolVersion >= KAFKA_2_3_IV2) 2
       else if (config.interBrokerProtocolVersion >= KAFKA_2_2_IV0) 1
       else 0
 
@@ -564,26 +571,27 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
         sendEvent(TopicDeletionStopReplicaResponseReceived(brokerId, stopReplicaResponse.error, partitionErrorsForDeletingTopics))
     }
 
-    def createStopReplicaRequest(brokerEpoch: Long, requests: Seq[StopReplicaRequestInfo], deletePartitions: Boolean): StopReplicaRequest.Builder = {
+    def createStopReplicaRequest(brokerEpoch: Long, maxBrokerEpoch: Long, requests: Seq[StopReplicaRequestInfo], deletePartitions: Boolean): StopReplicaRequest.Builder = {
       val partitions = requests.map(_.replica.topicPartition).asJava
       new StopReplicaRequest.Builder(stopReplicaRequestVersion, controllerId, controllerEpoch,
-        brokerEpoch, deletePartitions, partitions)
+        brokerEpoch, maxBrokerEpoch, deletePartitions, partitions)
     }
 
+    val maxBrokerEpoch = controllerContext.maxBrokerEpoch
     stopReplicaRequestMap.filterKeys(controllerContext.liveOrShuttingDownBrokerIds.contains).foreach { case (brokerId, replicaInfoList) =>
       val (stopReplicaWithDelete, stopReplicaWithoutDelete) = replicaInfoList.partition(r => r.deletePartition)
       val brokerEpoch = controllerContext.liveBrokerIdAndEpochs(brokerId)
 
       if (stopReplicaWithDelete.nonEmpty) {
         debug(s"The stop replica request (delete = true) sent to broker $brokerId is ${stopReplicaWithDelete.mkString(",")}")
-        val stopReplicaRequest = createStopReplicaRequest(brokerEpoch, stopReplicaWithDelete, deletePartitions = true)
+        val stopReplicaRequest = createStopReplicaRequest(brokerEpoch, maxBrokerEpoch, stopReplicaWithDelete, deletePartitions = true)
         val callback = stopReplicaPartitionDeleteResponseCallback(brokerId) _
         sendRequest(brokerId, stopReplicaRequest, callback)
       }
 
       if (stopReplicaWithoutDelete.nonEmpty) {
         debug(s"The stop replica request (delete = false) sent to broker $brokerId is ${stopReplicaWithoutDelete.mkString(",")}")
-        val stopReplicaRequest = createStopReplicaRequest(brokerEpoch, stopReplicaWithoutDelete, deletePartitions = false)
+        val stopReplicaRequest = createStopReplicaRequest(brokerEpoch, maxBrokerEpoch, stopReplicaWithoutDelete, deletePartitions = false)
         sendRequest(brokerId, stopReplicaRequest)
       }
     }
diff --git a/core/src/main/scala/kafka/controller/ControllerContext.scala b/core/src/main/scala/kafka/controller/ControllerContext.scala
index e5f584304f690..85e27b82621aa 100644
--- a/core/src/main/scala/kafka/controller/ControllerContext.scala
+++ b/core/src/main/scala/kafka/controller/ControllerContext.scala
@@ -196,6 +196,7 @@ class ControllerContext {
   def liveOrShuttingDownBrokerIds: Set[Int] = liveBrokerEpochs.keySet
   def liveOrShuttingDownBrokers: Set[Broker] = liveBrokers
   def liveBrokerIdAndEpochs: Map[Int, Long] = liveBrokerEpochs
+  def maxBrokerEpoch: Long = liveBrokerEpochs.values.max
   def liveOrShuttingDownBroker(brokerId: Int): Option[Broker] = liveOrShuttingDownBrokers.find(_.id == brokerId)
 
   def partitionsOnBroker(brokerId: Int): Set[TopicPartition] = {
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index dc070e8c75e97..8d816baa76e2a 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -210,11 +210,12 @@ class KafkaApis(val requestChannel: RequestChannel,
     }
 
     authorizeClusterOperation(request, CLUSTER_ACTION)
-    if (isBrokerEpochStale(leaderAndIsrRequest.brokerEpoch())) {
+    if (isBrokerEpochStale(leaderAndIsrRequest.brokerEpoch(), leaderAndIsrRequest.maxBrokerEpoch())) {
       // When the broker restarts very quickly, it is possible for this broker to receive request intended
       // for its previous generation so the broker should skip the stale request.
-      info("Received LeaderAndIsr request with broker epoch " +
-        s"${leaderAndIsrRequest.brokerEpoch()} smaller than the current broker epoch ${controller.brokerEpoch}")
+      info("Received LeaderAndIsr request with stale broker epoch info " +
+        "(broker epoch:" + leaderAndIsrRequest.brokerEpoch() +"},max broker epoch:" + leaderAndIsrRequest.maxBrokerEpoch() + ") " +
+        "when the current broker epoch is " + controller.brokerEpoch)
       sendResponseExemptThrottle(request, leaderAndIsrRequest.getErrorResponse(0, Errors.STALE_BROKER_EPOCH.exception))
     } else {
       val response = replicaManager.becomeLeaderOrFollower(correlationId, leaderAndIsrRequest, onLeadershipChange)
@@ -228,11 +229,12 @@ class KafkaApis(val requestChannel: RequestChannel,
     // stop serving data to clients for the topic being deleted
     val stopReplicaRequest = request.body[StopReplicaRequest]
     authorizeClusterOperation(request, CLUSTER_ACTION)
-    if (isBrokerEpochStale(stopReplicaRequest.brokerEpoch())) {
+    if (isBrokerEpochStale(stopReplicaRequest.brokerEpoch(), stopReplicaRequest.maxBrokerEpoch())) {
       // When the broker restarts very quickly, it is possible for this broker to receive request intended
       // for its previous generation so the broker should skip the stale request.
-      info("Received stop replica request with broker epoch " +
-        s"${stopReplicaRequest.brokerEpoch()} smaller than the current broker epoch ${controller.brokerEpoch}")
+      info("Received stop replica request with stale broker epoch info " +
+        s"(broker epoch:${stopReplicaRequest.brokerEpoch()},max broker epoch:${stopReplicaRequest.maxBrokerEpoch()})" +
+        s"when the current broker epoch is ${controller.brokerEpoch}")
       sendResponseExemptThrottle(request, new StopReplicaResponse(new StopReplicaResponseData().setErrorCode(Errors.STALE_BROKER_EPOCH.code)))
     } else {
       val (result, error) = replicaManager.stopReplicas(stopReplicaRequest)
@@ -268,11 +270,12 @@ class KafkaApis(val requestChannel: RequestChannel,
     val updateMetadataRequest = request.body[UpdateMetadataRequest]
 
     authorizeClusterOperation(request, CLUSTER_ACTION)
-    if (isBrokerEpochStale(updateMetadataRequest.brokerEpoch)) {
+    if (isBrokerEpochStale(updateMetadataRequest.brokerEpoch, updateMetadataRequest.maxBrokerEpoch)) {
       // When the broker restarts very quickly, it is possible for this broker to receive request intended
       // for its previous generation so the broker should skip the stale request.
-      info("Received update metadata request with broker epoch " +
-        s"${updateMetadataRequest.brokerEpoch} smaller than the current broker epoch ${controller.brokerEpoch}")
+      info("Received update metadata request with stale broker epoch info " +
+        s"(broker epoch:${updateMetadataRequest.brokerEpoch},max broker epoch: ${updateMetadataRequest.maxBrokerEpoch}) " +
+        s"when the current broker epoch ${controller.brokerEpoch}")
       sendResponseExemptThrottle(request,
         new UpdateMetadataResponse(new UpdateMetadataResponseData().setErrorCode(Errors.STALE_BROKER_EPOCH.code)))
     } else {
@@ -2965,16 +2968,16 @@ class KafkaApis(val requestChannel: RequestChannel,
     requestChannel.sendResponse(response)
   }
 
-  private def isBrokerEpochStale(brokerEpochInRequest: Long): Boolean = {
-    // Broker epoch in LeaderAndIsr/UpdateMetadata/StopReplica request is unknown
-    // if the controller hasn't been upgraded to use KIP-380
-    if (brokerEpochInRequest == AbstractControlRequest.UNKNOWN_BROKER_EPOCH) false
-    else {
+  private def isBrokerEpochStale(brokerEpochInRequest: Long, maxBrokerEpochInRequest: Long): Boolean = {
+    if (maxBrokerEpochInRequest != AbstractControlRequest.UNKNOWN_BROKER_EPOCH)  {
+      maxBrokerEpochInRequest < controller.brokerEpoch
+    }
+    else if (brokerEpochInRequest != AbstractControlRequest.UNKNOWN_BROKER_EPOCH) {
       val curBrokerEpoch = controller.brokerEpoch
       if (brokerEpochInRequest < curBrokerEpoch) true
       else if (brokerEpochInRequest == curBrokerEpoch) false
       else throw new IllegalStateException(s"Epoch $brokerEpochInRequest larger than current broker epoch $curBrokerEpoch")
-    }
+    } else false
   }
 
   private def observeRequestResponse(request: RequestChannel.Request, response: AbstractResponse): Unit = {
diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
index 464e1ff22fa7c..b25e81d3a1364 100644
--- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
@@ -377,7 +377,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
         .setSecurityProtocol(securityProtocol.id)
         .setListener(ListenerName.forSecurityProtocol(securityProtocol).value)).asJava)).asJava
     val version = ApiKeys.UPDATE_METADATA.latestVersion
-    new requests.UpdateMetadataRequest.Builder(version, brokerId, Int.MaxValue, Long.MaxValue, partitionStates, brokers).build()
+    new requests.UpdateMetadataRequest.Builder(version, brokerId, Int.MaxValue, Long.MaxValue, Long.MaxValue, partitionStates, brokers).build()
   }
 
   private def createJoinGroupRequest = {
@@ -458,7 +458,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   ).build()
 
   private def leaderAndIsrRequest = {
-    new requests.LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, brokerId, Int.MaxValue, Long.MaxValue,
+    new requests.LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, brokerId, Int.MaxValue, Long.MaxValue, Long.MaxValue,
       Seq(new LeaderAndIsrPartitionState()
         .setTopicName(tp.topic)
         .setPartitionIndex(tp.partition)
@@ -472,7 +472,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
       Set(new Node(brokerId, "localhost", 0)).asJava).build()
   }
 
-  private def stopReplicaRequest = new StopReplicaRequest.Builder(ApiKeys.STOP_REPLICA.latestVersion, brokerId, Int.MaxValue, Long.MaxValue, true, Set(tp).asJava).build()
+  private def stopReplicaRequest = new StopReplicaRequest.Builder(ApiKeys.STOP_REPLICA.latestVersion, brokerId, Int.MaxValue, Long.MaxValue, Long.MaxValue, true, Set(tp).asJava).build()
 
   private def controlledShutdownRequest = new ControlledShutdownRequest.Builder(
       new ControlledShutdownRequestData()
diff --git a/core/src/test/scala/integration/kafka/api/RecordHeaderProducerSendTest.scala b/core/src/test/scala/integration/kafka/api/RecordHeaderProducerSendTest.scala
index faeca6317cd5d..6475d1620f418 100644
--- a/core/src/test/scala/integration/kafka/api/RecordHeaderProducerSendTest.scala
+++ b/core/src/test/scala/integration/kafka/api/RecordHeaderProducerSendTest.scala
@@ -26,7 +26,6 @@ import org.apache.kafka.common.header.internals.RecordHeaders
 import org.junit.Test
 
 class RecordHeaderProducerSendTest extends BaseProducerSendTest {
-
   @Test
   def testRecordHeaders(): Unit = {
     val producerProps = new Properties()
@@ -35,38 +34,42 @@ class RecordHeaderProducerSendTest extends BaseProducerSendTest {
     producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
     producerProps.put(ProducerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, "true")
     val producer = new KafkaProducer[String, String](producerProps)
-    var invalidRecordHeaders = new RecordHeaders()
-    invalidRecordHeaders.add("RecordHeaderKey", "RecordHeaderValue".getBytes)
-    // The record is invalid because it contain header with keys that doesn't start with a "_"
-    // Keys that do start with a "_" are internal to the clients and are dropped at the producer.
-    val invalidRecord = new ProducerRecord[String, String](
-      topic,
-      new Integer(0),
-      "RecordKey",
-      "RecordValue",
-      invalidRecordHeaders
-    )
     try {
-      producer.send(invalidRecord).get
-      throw new IllegalStateException("The invalid record should have thrown an exception")
-    } catch {
-      // Ignore the exception because a non internal header was introduced into the producer record
-      case ignored: IllegalArgumentException =>
+      var invalidRecordHeaders = new RecordHeaders()
+      invalidRecordHeaders.add("RecordHeaderKey", "RecordHeaderValue".getBytes)
+      // The record is invalid because it contain header with keys that doesn't start with a "_"
+      // Keys that do start with a "_" are internal to the clients and are dropped at the producer.
+      val invalidRecord = new ProducerRecord[String, String](
+        topic,
+        new Integer(0),
+        "RecordKey",
+        "RecordValue",
+        invalidRecordHeaders
+      )
+      try {
+        producer.send(invalidRecord).get
+        throw new IllegalStateException("The invalid record should have thrown an exception")
+      } catch {
+        // Ignore the exception because a non internal header was introduced into the producer record
+        case ignored: IllegalArgumentException =>
+      }
+  
+      val validRecordHeaders = new RecordHeaders()
+      validRecordHeaders.add("_RecordHeaderKey", "RecordHeaderValue".getBytes)
+      val record = new ProducerRecord[String, String](
+        topic,
+        new Integer(0),
+        "RecordKey",
+        "RecordValue",
+        validRecordHeaders
+      )
+      producer.send(record).get
+    } finally {
+      producer.close()
     }
-
-    val validRecordHeaders = new RecordHeaders()
-    validRecordHeaders.add("_RecordHeaderKey", "RecordHeaderValue".getBytes)
-    val record = new ProducerRecord[String, String](
-      topic,
-      new Integer(0),
-      "RecordKey",
-      "RecordValue",
-      validRecordHeaders
-    )
-    producer.send(record).get
-    producer.close()
   }
 
+
   override def overrideConfigs(): Properties = {
     val properties = new Properties()
     properties.put(KafkaConfig.InterBrokerProtocolVersionProp, "0.10.2")
diff --git a/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala b/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala
index 0519ea04c70e9..caff74ce2d666 100644
--- a/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala
+++ b/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala
@@ -89,9 +89,10 @@ class ApiVersionTest {
     assertEquals(KAFKA_2_2_IV0, ApiVersion("2.2-IV0"))
     assertEquals(KAFKA_2_2_IV1, ApiVersion("2.2-IV1"))
 
-    assertEquals(KAFKA_2_3_IV1, ApiVersion("2.3"))
+    assertEquals(KAFKA_2_3_IV2, ApiVersion("2.3"))
     assertEquals(KAFKA_2_3_IV0, ApiVersion("2.3-IV0"))
     assertEquals(KAFKA_2_3_IV1, ApiVersion("2.3-IV1"))
+    assertEquals(KAFKA_2_3_IV2, ApiVersion("2.3-IV2"))
 
     assertEquals(KAFKA_2_4_IV1, ApiVersion("2.4"))
     assertEquals(KAFKA_2_4_IV0, ApiVersion("2.4-IV0"))
diff --git a/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala b/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala
index d2869670897ef..100014358ce66 100644
--- a/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala
@@ -18,7 +18,7 @@ package kafka.controller
 
 import java.util.Properties
 
-import kafka.api.{ApiVersion, KAFKA_0_10_0_IV1, KAFKA_0_10_2_IV0, KAFKA_0_9_0, KAFKA_1_0_IV0, KAFKA_2_2_IV0, KAFKA_2_4_IV0, KAFKA_2_4_IV1, LeaderAndIsr}
+import kafka.api.{ApiVersion, KAFKA_0_10_0_IV1, KAFKA_0_10_2_IV0, KAFKA_0_9_0, KAFKA_1_0_IV0, KAFKA_2_2_IV0, KAFKA_2_3_IV2, KAFKA_2_4_IV0, KAFKA_2_4_IV1, LeaderAndIsr}
 import kafka.cluster.{Broker, EndPoint}
 import kafka.server.KafkaConfig
 import kafka.utils.TestUtils
@@ -157,8 +157,9 @@ class ControllerChannelManagerTest {
 
     for (apiVersion <- ApiVersion.allVersions) {
       val leaderAndIsrRequestVersion: Short =
-        if (apiVersion >= KAFKA_2_4_IV1) 4
-        else if (apiVersion >= KAFKA_2_4_IV0) 3
+        if (apiVersion >= KAFKA_2_4_IV1) 5
+        else if (apiVersion >= KAFKA_2_4_IV0) 4
+        else if (apiVersion >= KAFKA_2_3_IV2) 3
         else if (apiVersion >= KAFKA_2_2_IV0) 2
         else if (apiVersion >= KAFKA_1_0_IV0) 1
         else 0
@@ -327,7 +328,8 @@ class ControllerChannelManagerTest {
 
     for (apiVersion <- ApiVersion.allVersions) {
       val updateMetadataRequestVersion: Short =
-        if (apiVersion >= KAFKA_2_4_IV1) 6
+        if (apiVersion >= KAFKA_2_4_IV1) 7
+        else if (apiVersion >= KAFKA_2_3_IV2) 6
         else if (apiVersion >= KAFKA_2_2_IV0) 5
         else if (apiVersion >= KAFKA_1_0_IV0) 4
         else if (apiVersion >= KAFKA_0_10_2_IV0) 3
@@ -599,10 +601,12 @@ class ControllerChannelManagerTest {
     for (apiVersion <- ApiVersion.allVersions) {
       if (apiVersion < KAFKA_2_2_IV0)
         testStopReplicaFollowsInterBrokerProtocolVersion(apiVersion, 0.toShort)
-      else if (apiVersion < KAFKA_2_4_IV1)
+      else if (apiVersion < KAFKA_2_3_IV2)
         testStopReplicaFollowsInterBrokerProtocolVersion(apiVersion, 1.toShort)
-      else
+      else if (apiVersion < KAFKA_2_4_IV1)
         testStopReplicaFollowsInterBrokerProtocolVersion(apiVersion, 2.toShort)
+      else
+        testStopReplicaFollowsInterBrokerProtocolVersion(apiVersion, 3.toShort)
     }
   }
 
diff --git a/core/src/test/scala/unit/kafka/server/BrokerEpochIntegrationTest.scala b/core/src/test/scala/unit/kafka/server/BrokerEpochIntegrationTest.scala
index 23836dd66d8c7..dbeb70eb71a9c 100755
--- a/core/src/test/scala/unit/kafka/server/BrokerEpochIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/server/BrokerEpochIntegrationTest.scala
@@ -148,6 +148,7 @@ class BrokerEpochIntegrationTest extends ZooKeeperTestHarness {
         val requestBuilder = new LeaderAndIsrRequest.Builder(
           ApiKeys.LEADER_AND_ISR.latestVersion, controllerId, controllerEpoch,
           epochInRequest,
+          epochInRequest,
           partitionStates.asJava, nodes.toSet.asJava)
 
         if (isEpochInRequestStale) {
@@ -187,7 +188,7 @@ class BrokerEpochIntegrationTest extends ZooKeeperTestHarness {
         }.toBuffer
         val requestBuilder = new UpdateMetadataRequest.Builder(
           ApiKeys.UPDATE_METADATA.latestVersion, controllerId, controllerEpoch,
-          epochInRequest,
+          epochInRequest, epochInRequest,
           partitionStates.asJava, liveBrokers.asJava)
 
         if (isEpochInRequestStale) {
@@ -206,6 +207,7 @@ class BrokerEpochIntegrationTest extends ZooKeeperTestHarness {
         val requestBuilder = new StopReplicaRequest.Builder(
           ApiKeys.STOP_REPLICA.latestVersion, controllerId, controllerEpoch,
           epochInRequest, // Correct broker epoch
+          epochInRequest, // Correct broker epoch
           true, Set(tp).asJava)
 
         if (isEpochInRequestStale) {
diff --git a/core/src/test/scala/unit/kafka/server/CacheableBrokerEpochIntegrationTest.scala b/core/src/test/scala/unit/kafka/server/CacheableBrokerEpochIntegrationTest.scala
new file mode 100644
index 0000000000000..b8ce0dea8d95b
--- /dev/null
+++ b/core/src/test/scala/unit/kafka/server/CacheableBrokerEpochIntegrationTest.scala
@@ -0,0 +1,74 @@
+/**
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *    http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+
+package kafka.server
+
+import java.util.Properties
+
+import kafka.api.KAFKA_2_3_IV1
+import kafka.server.KafkaConfig
+import kafka.utils.TestUtils
+import kafka.zk.ZooKeeperTestHarness
+import org.apache.kafka.common.TopicPartition
+import org.junit.Test
+
+class CacheableBrokerEpochIntegrationTest extends ZooKeeperTestHarness {
+  @Test
+  def testNewControllerConfig(): Unit = {
+    testControlRequests(true)
+  }
+
+  @Test
+  def testOldControllerConfig(): Unit = {
+    testControlRequests(false)
+  }
+
+  def testControlRequests(controllerUseNewConfig: Boolean): Unit = {
+    val controllerId = 0
+    val controllerConfig: Properties =
+      if (controllerUseNewConfig) {
+        TestUtils.createBrokerConfig(controllerId, zkConnect)
+      } else {
+        val oldConfig = TestUtils.createBrokerConfig(controllerId, zkConnect)
+        oldConfig.put(KafkaConfig.InterBrokerProtocolVersionProp, KAFKA_2_3_IV1.toString)
+        oldConfig
+      }
+    val controller = TestUtils.createServer(KafkaConfig.fromProps(controllerConfig))
+
+    // Note that broker side logic does not depend on the InterBrokerProtocolVersion config
+    val brokerId = 1
+    val brokerConfig = TestUtils.createBrokerConfig(brokerId, zkConnect)
+    val broker = TestUtils.createServer(KafkaConfig.fromProps(brokerConfig))
+    val servers = Seq(controller, broker)
+
+    val tp = new TopicPartition("new-topic", 0)
+
+    try {
+      // Use topic creation to test the LeaderAndIsr and UpdateMetadata requests
+      TestUtils.createTopic(zkClient, tp.topic(), partitionReplicaAssignment = Map(0 -> Seq(brokerId, controllerId)),
+        servers = servers)
+      TestUtils.waitUntilLeaderIsKnown(Seq(broker), tp, 10000)
+      TestUtils.waitUntilMetadataIsPropagated(Seq(broker), tp.topic(), tp.partition())
+
+      // Use topic deletion to test StopReplica requests
+      adminZkClient.deleteTopic(tp.topic())
+      TestUtils.verifyTopicDeletion(zkClient, tp.topic(), 1, servers)
+    } finally {
+      TestUtils.shutdownServers(servers)
+    }
+  }
+}
diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
index 28f56c256203a..d822ace68f3b9 100644
--- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
@@ -332,6 +332,7 @@ class KafkaApisTest {
       controllerId,
       controllerEpoch,
       brokerEpoch,
+      brokerEpoch,
       true,
       Set(groupMetadataPartition, txnStatePartition).asJava))
 
@@ -888,7 +889,7 @@ class KafkaApisTest {
             .setListener(plaintextListener.value)).asJava)
     )
     val updateMetadataRequest = new UpdateMetadataRequest.Builder(ApiKeys.UPDATE_METADATA.latestVersion, 0,
-      0, 0, Seq.empty[UpdateMetadataPartitionState].asJava, brokers.asJava).build()
+      0, 0, 0, Seq.empty[UpdateMetadataPartitionState].asJava, brokers.asJava).build()
     metadataCache.updateMetadata(correlationId = 0, updateMetadataRequest)
     (plaintextListener, anotherListener)
   }
@@ -1004,7 +1005,7 @@ class KafkaApisTest {
         .setListener(plaintextListener.value)).asJava)
     val partitionStates = (0 until numPartitions).map(createPartitionState)
     val updateMetadataRequest = new UpdateMetadataRequest.Builder(ApiKeys.UPDATE_METADATA.latestVersion, 0,
-      0, 0, partitionStates.asJava, Seq(broker).asJava).build()
+      0, 0, 0, partitionStates.asJava, Seq(broker).asJava).build()
     metadataCache.updateMetadata(correlationId = 0, updateMetadataRequest)
   }
 }
diff --git a/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala b/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala
index 01423a39f61f0..8b97d65e4d1bb 100755
--- a/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala
+++ b/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala
@@ -17,6 +17,8 @@
 
 package kafka.server
 
+import java.util.concurrent.{CountDownLatch, TimeUnit}
+
 import org.apache.kafka.common.TopicPartition
 
 import scala.collection.JavaConverters._
@@ -28,6 +30,7 @@ import kafka.cluster.Broker
 import kafka.controller.{ControllerChannelManager, ControllerContext, StateChangeLogger}
 import kafka.utils.TestUtils._
 import kafka.zk.ZooKeeperTestHarness
+import org.apache.kafka.common.message.LeaderAndIsrRequestData
 import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.network.ListenerName
@@ -154,7 +157,7 @@ class LeaderElectionTest extends ZooKeeperTestHarness {
           .setIsNew(false)
       )
       val requestBuilder = new LeaderAndIsrRequest.Builder(
-        ApiKeys.LEADER_AND_ISR.latestVersion, controllerId, staleControllerEpoch,
+        ApiKeys.LEADER_AND_ISR.latestVersion, controllerId, staleControllerEpoch, servers(brokerId2).kafkaController.brokerEpoch, 
         servers(brokerId2).kafkaController.brokerEpoch, partitionStates.asJava, nodes.toSet.asJava)
 
       controllerChannelManager.sendRequest(brokerId2, requestBuilder, staleControllerEpochCallback)
diff --git a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala
index a42a86a5d03e5..0491dabb757ea 100644
--- a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala
+++ b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala
@@ -106,7 +106,7 @@ class MetadataCacheTest {
         .setReplicas(asList(2, 1, 3)))
 
     val version = ApiKeys.UPDATE_METADATA.latestVersion
-    val updateMetadataRequest = new UpdateMetadataRequest.Builder(version, controllerId, controllerEpoch, brokerEpoch,
+    val updateMetadataRequest = new UpdateMetadataRequest.Builder(version, controllerId, controllerEpoch, brokerEpoch, brokerEpoch,
       partitionStates.asJava, brokers.asJava).build()
     cache.updateMetadata(15, updateMetadataRequest)
 
@@ -251,7 +251,7 @@ class MetadataCacheTest {
       .setReplicas(asList(0)))
 
     val version = ApiKeys.UPDATE_METADATA.latestVersion
-    val updateMetadataRequest = new UpdateMetadataRequest.Builder(version, controllerId, controllerEpoch, brokerEpoch,
+    val updateMetadataRequest = new UpdateMetadataRequest.Builder(version, controllerId, controllerEpoch, brokerEpoch, brokerEpoch,
       partitionStates.asJava, brokers.asJava).build()
     cache.updateMetadata(15, updateMetadataRequest)
 
@@ -309,7 +309,7 @@ class MetadataCacheTest {
         .setReplicas(replicas))
 
     val version = ApiKeys.UPDATE_METADATA.latestVersion
-    val updateMetadataRequest = new UpdateMetadataRequest.Builder(version, controllerId, controllerEpoch, brokerEpoch,
+    val updateMetadataRequest = new UpdateMetadataRequest.Builder(version, controllerId, controllerEpoch, brokerEpoch, brokerEpoch,
       partitionStates.asJava, brokers.asJava).build()
     cache.updateMetadata(15, updateMetadataRequest)
 
@@ -383,7 +383,7 @@ class MetadataCacheTest {
       .setReplicas(replicas))
 
     val version = ApiKeys.UPDATE_METADATA.latestVersion
-    val updateMetadataRequest = new UpdateMetadataRequest.Builder(version, controllerId, controllerEpoch, brokerEpoch,
+    val updateMetadataRequest = new UpdateMetadataRequest.Builder(version, controllerId, controllerEpoch, brokerEpoch, brokerEpoch,
       partitionStates.asJava, brokers.asJava).build()
     cache.updateMetadata(15, updateMetadataRequest)
 
@@ -448,7 +448,7 @@ class MetadataCacheTest {
       .setZkVersion(3)
       .setReplicas(replicas))
     val version = ApiKeys.UPDATE_METADATA.latestVersion
-    val updateMetadataRequest = new UpdateMetadataRequest.Builder(version, 2, controllerEpoch, brokerEpoch, partitionStates.asJava,
+    val updateMetadataRequest = new UpdateMetadataRequest.Builder(version, 2, controllerEpoch, brokerEpoch, brokerEpoch, partitionStates.asJava,
       brokers.asJava).build()
     cache.updateMetadata(15, updateMetadataRequest)
 
@@ -490,7 +490,7 @@ class MetadataCacheTest {
         .setZkVersion(3)
         .setReplicas(replicas))
       val version = ApiKeys.UPDATE_METADATA.latestVersion
-      val updateMetadataRequest = new UpdateMetadataRequest.Builder(version, 2, controllerEpoch, brokerEpoch, partitionStates.asJava,
+      val updateMetadataRequest = new UpdateMetadataRequest.Builder(version, 2, controllerEpoch, brokerEpoch, brokerEpoch, partitionStates.asJava,
         brokers.asJava).build()
       cache.updateMetadata(15, updateMetadataRequest)
     }
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index 93011add60202..5730d90be6158 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -207,7 +207,7 @@ class ReplicaManagerTest {
       partition.createLogIfNotExists(isNew = false, isFutureReplica = false,
         new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints))
       // Make this replica the leader.
-      val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+      val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch, brokerEpoch,
         Seq(new LeaderAndIsrPartitionState()
           .setTopicName(topic)
           .setPartitionIndex(0)
@@ -229,7 +229,7 @@ class ReplicaManagerTest {
       }
 
       // Make this replica the follower
-      val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+      val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch, brokerEpoch,
         Seq(new LeaderAndIsrPartitionState()
           .setTopicName(topic)
           .setPartitionIndex(0)
@@ -265,7 +265,7 @@ class ReplicaManagerTest {
         .createLogIfNotExists(isNew = false, isFutureReplica = false,
           new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
 
-      def leaderAndIsrRequest(epoch: Int): LeaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+      def leaderAndIsrRequest(epoch: Int): LeaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch, brokerEpoch,
         Seq(new LeaderAndIsrPartitionState()
           .setTopicName(topic)
           .setPartitionIndex(0)
@@ -324,7 +324,7 @@ class ReplicaManagerTest {
         new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
 
       // Make this replica the leader.
-      val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+      val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch, brokerEpoch,
         Seq(new LeaderAndIsrPartitionState()
           .setTopicName(topic)
           .setPartitionIndex(0)
@@ -384,7 +384,7 @@ class ReplicaManagerTest {
         new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
 
       // Make this replica the leader.
-      val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+      val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch, brokerEpoch,
         Seq(new LeaderAndIsrPartitionState()
           .setTopicName(topic)
           .setPartitionIndex(0)
@@ -490,7 +490,7 @@ class ReplicaManagerTest {
         new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints))
 
       // Make this replica the leader.
-      val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+      val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch, brokerEpoch,
         Seq(new LeaderAndIsrPartitionState()
           .setTopicName(topic)
           .setPartitionIndex(0)
@@ -566,7 +566,7 @@ class ReplicaManagerTest {
         new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints))
 
       // Make this replica the leader.
-      val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+      val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch, brokerEpoch,
         Seq(new LeaderAndIsrPartitionState()
           .setTopicName(topic)
           .setPartitionIndex(0)
@@ -629,7 +629,7 @@ class ReplicaManagerTest {
         .setZkVersion(0)
         .setReplicas(replicas)
         .setIsNew(true)
-      val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+      val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch, brokerEpoch,
         Seq(leaderAndIsrPartitionState).asJava,
         Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
       val leaderAndIsrResponse = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest, (_, _) => ())
@@ -722,7 +722,7 @@ class ReplicaManagerTest {
       replicaManager.createPartition(tp1).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
       val partition0Replicas = Seq[Integer](0, 1).asJava
       val partition1Replicas = Seq[Integer](0, 2).asJava
-      val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+      val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch, brokerEpoch,
         Seq(
           new LeaderAndIsrPartitionState()
             .setTopicName(tp0.topic)
@@ -841,7 +841,7 @@ class ReplicaManagerTest {
     // trigger even though leader does not change
     leaderEpoch += leaderEpochIncrement
     val leaderAndIsrRequest0 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion,
-      controllerId, controllerEpoch, brokerEpoch,
+      controllerId, controllerEpoch, brokerEpoch, brokerEpoch,
       Seq(leaderAndIsrPartitionState(tp, leaderEpoch, leaderBrokerId, aliveBrokerIds)).asJava,
       Set(new Node(followerBrokerId, "host1", 0),
         new Node(leaderBrokerId, "host2", 1)).asJava).build()
@@ -910,7 +910,7 @@ class ReplicaManagerTest {
     replicaManager.createPartition(new TopicPartition(topic, 0))
 
     // Make this replica the follower
-    val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+    val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch, brokerEpoch,
       Seq(new LeaderAndIsrPartitionState()
         .setTopicName(topic)
         .setPartitionIndex(0)
@@ -959,7 +959,7 @@ class ReplicaManagerTest {
     replicaManager.createPartition(new TopicPartition(topic, 0))
 
     // Make this replica the follower
-    val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+    val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch, brokerEpoch,
       Seq(new LeaderAndIsrPartitionState()
         .setTopicName(topic)
         .setPartitionIndex(0)
@@ -1026,7 +1026,7 @@ class ReplicaManagerTest {
     val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)
     replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
     val partition0Replicas = Seq[Integer](0, 1).asJava
-    val becomeFollowerRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+    val becomeFollowerRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch, brokerEpoch,
       Seq(new LeaderAndIsrPartitionState()
         .setTopicName(tp0.topic)
         .setPartitionIndex(tp0.partition)
@@ -1066,7 +1066,7 @@ class ReplicaManagerTest {
     replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
     val partition0Replicas = Seq[Integer](0, 1).asJava
 
-    val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+    val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch, brokerEpoch,
       Seq(new LeaderAndIsrPartitionState()
         .setTopicName(tp0.topic)
         .setPartitionIndex(tp0.partition)
@@ -1086,7 +1086,7 @@ class ReplicaManagerTest {
     assertNull(fetchResult.get)
 
     // Become a follower and ensure that the delayed fetch returns immediately
-    val becomeFollowerRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+    val becomeFollowerRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch, brokerEpoch,
       Seq(new LeaderAndIsrPartitionState()
         .setTopicName(tp0.topic)
         .setPartitionIndex(tp0.partition)
@@ -1114,7 +1114,7 @@ class ReplicaManagerTest {
     replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
     val partition0Replicas = Seq[Integer](0, 1).asJava
 
-    val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+    val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch, brokerEpoch,
       Seq(new LeaderAndIsrPartitionState()
         .setTopicName(tp0.topic)
         .setPartitionIndex(tp0.partition)
@@ -1135,7 +1135,7 @@ class ReplicaManagerTest {
     assertNull(fetchResult.get)
 
     // Become a follower and ensure that the delayed fetch returns immediately
-    val becomeFollowerRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+    val becomeFollowerRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch, brokerEpoch,
       Seq(new LeaderAndIsrPartitionState()
         .setTopicName(tp0.topic)
         .setPartitionIndex(tp0.partition)
@@ -1162,7 +1162,7 @@ class ReplicaManagerTest {
     replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
     val partition0Replicas = Seq[Integer](0, 1).asJava
 
-    val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+    val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch, brokerEpoch,
       Seq(new LeaderAndIsrPartitionState()
         .setTopicName(tp0.topic)
         .setPartitionIndex(tp0.partition)
@@ -1204,7 +1204,7 @@ class ReplicaManagerTest {
     replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
     val partition0Replicas = Seq[Integer](0, 1).asJava
 
-    val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+    val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch, brokerEpoch,
       Seq(new LeaderAndIsrPartitionState()
         .setTopicName(tp0.topic)
         .setPartitionIndex(tp0.partition)
@@ -1243,7 +1243,7 @@ class ReplicaManagerTest {
     replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints)
     val partition0Replicas = Seq[Integer](0, 1).asJava
 
-    val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+    val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch, brokerEpoch,
       Seq(new LeaderAndIsrPartitionState()
         .setTopicName(tp0.topic)
         .setPartitionIndex(tp0.partition)
@@ -1619,7 +1619,7 @@ class ReplicaManagerTest {
       val partition1Replicas = Seq[Integer](1, 0).asJava
 
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion,
-        controllerId, 0, brokerEpoch,
+        controllerId, 0, brokerEpoch, brokerEpoch,
         Seq(
           new LeaderAndIsrPartitionState()
             .setTopicName(tp0.topic)
@@ -1649,7 +1649,7 @@ class ReplicaManagerTest {
 
       // make broker 0 the leader of partition 1 so broker 1 loses its leadership position
       val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, controllerId,
-        controllerEpoch, brokerEpoch,
+        controllerEpoch, brokerEpoch, brokerEpoch,
         Seq(
           new LeaderAndIsrPartitionState()
             .setTopicName(tp0.topic)
@@ -1707,7 +1707,7 @@ class ReplicaManagerTest {
       val partition1Replicas = Seq[Integer](1, 0).asJava
 
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion,
-        controllerId, 0, brokerEpoch,
+        controllerId, 0, brokerEpoch, brokerEpoch,
         Seq(
           new LeaderAndIsrPartitionState()
             .setTopicName(tp0.topic)
@@ -1737,7 +1737,7 @@ class ReplicaManagerTest {
 
       // make broker 0 the leader of partition 1 so broker 1 loses its leadership position
       val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, controllerId,
-        controllerEpoch, brokerEpoch,
+        controllerEpoch, brokerEpoch, brokerEpoch,
         Seq(
           new LeaderAndIsrPartitionState()
             .setTopicName(tp0.topic)
diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
index f34eae45b1e4d..79431bee503cd 100644
--- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
+++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
@@ -228,7 +228,7 @@ class RequestQuotaTest extends BaseRequestTest {
               0L, Optional.of[Integer](15))).asJava)
 
         case ApiKeys.LEADER_AND_ISR =>
-          new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, brokerId, Int.MaxValue, Long.MaxValue,
+          new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, brokerId, Int.MaxValue, Long.MaxValue, Long.MaxValue,
             Seq(new LeaderAndIsrPartitionState()
               .setTopicName(tp.topic)
               .setPartitionIndex(tp.partition)
@@ -242,7 +242,7 @@ class RequestQuotaTest extends BaseRequestTest {
             Set(new Node(brokerId, "localhost", 0)).asJava)
 
         case ApiKeys.STOP_REPLICA =>
-          new StopReplicaRequest.Builder(ApiKeys.STOP_REPLICA.latestVersion, brokerId, Int.MaxValue, Long.MaxValue, true, Set(tp).asJava)
+          new StopReplicaRequest.Builder(ApiKeys.STOP_REPLICA.latestVersion, brokerId, Int.MaxValue, Long.MaxValue, Long.MaxValue, true, Set(tp).asJava)
 
         case ApiKeys.UPDATE_METADATA =>
           val partitionState = Seq(new UpdateMetadataPartitionState()
@@ -262,7 +262,7 @@ class RequestQuotaTest extends BaseRequestTest {
               .setPort(0)
               .setSecurityProtocol(securityProtocol.id)
               .setListener(ListenerName.forSecurityProtocol(securityProtocol).value)).asJava)).asJava
-          new UpdateMetadataRequest.Builder(ApiKeys.UPDATE_METADATA.latestVersion, brokerId, Int.MaxValue, Long.MaxValue, partitionState, brokers)
+          new UpdateMetadataRequest.Builder(ApiKeys.UPDATE_METADATA.latestVersion, brokerId, Int.MaxValue, Long.MaxValue, Long.MaxValue, partitionState, brokers)
 
         case ApiKeys.CONTROLLED_SHUTDOWN =>
           new ControlledShutdownRequest.Builder(
diff --git a/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala b/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala
index 33e4566800380..75fae948d6fc1 100755
--- a/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala
@@ -233,7 +233,7 @@ class ServerShutdownTest extends ZooKeeperTestHarness {
 
       // Initiate a sendRequest and wait until connection is established and one byte is received by the peer
       val requestBuilder = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion,
-        controllerId, 1, 0L, Seq.empty.asJava, brokerAndEpochs.keys.map(_.node(listenerName)).toSet.asJava)
+        controllerId, 1, 0L, 0L, Seq.empty.asJava, brokerAndEpochs.keys.map(_.node(listenerName)).toSet.asJava)
       controllerChannelManager.sendRequest(1, requestBuilder)
       receiveFuture.get(10, TimeUnit.SECONDS)
 
diff --git a/core/src/test/scala/unit/kafka/server/StopReplicaRequestTest.scala b/core/src/test/scala/unit/kafka/server/StopReplicaRequestTest.scala
index b1b43d069c0d5..1af4bd8ddb449 100644
--- a/core/src/test/scala/unit/kafka/server/StopReplicaRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/StopReplicaRequestTest.scala
@@ -46,6 +46,7 @@ class StopReplicaRequestTest extends BaseRequestTest {
     for (_ <- 1 to 2) {
       val request1 = new StopReplicaRequest.Builder(1,
         server.config.brokerId, server.replicaManager.controllerEpoch, server.kafkaController.brokerEpoch,
+        server.kafkaController.brokerEpoch,
         true, Set(tp0, tp1).asJava).build()
       val response1 = connectAndSend(request1, ApiKeys.STOP_REPLICA, controllerSocketServer)
       val partitionErrors1 = StopReplicaResponse.parse(response1, request1.version).partitionErrors.asScala

From 8adc2d1e3f2d586ca3569e6521aef380304ba38c Mon Sep 17 00:00:00 2001
From: Lucas Wang 
Date: Fri, 20 Mar 2020 17:42:42 -0700
Subject: [PATCH 1054/1071] [LI-HOTFIX] Cache the UpdateMetadataRequest when
 the inter broker protocol version >= 2.3-IV2 (#75) TICKET = KAFKA-7186
 LI_DESCERIPTION = Cache the UpdateMetadataRequest when using the latest inter
 broker protocol version EXIT_CRITERIA = KAFKA-7186

---
 .../kafka/common/network/NetworkSend.java     | 16 ++++++++++++++
 .../kafka/common/requests/RequestUtils.java   |  7 ++++++
 .../requests/UpdateMetadataRequest.java       | 20 +++++++++++++++++
 .../controller/ControllerChannelManager.scala | 22 ++++++++++++++-----
 4 files changed, 60 insertions(+), 5 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/network/NetworkSend.java b/clients/src/main/java/org/apache/kafka/common/network/NetworkSend.java
index 94abc64bfcbee..791d5f5cec13c 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/NetworkSend.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/NetworkSend.java
@@ -27,6 +27,22 @@ public NetworkSend(String destination, ByteBuffer buffer) {
         super(destination, sizeBuffer(buffer.remaining()), buffer);
     }
 
+    public NetworkSend(String destination, ByteBuffer[] buffers) {
+        super(destination, sizeDelimit(buffers));
+    }
+
+    private static ByteBuffer[] sizeDelimit(ByteBuffer[] buffers) {
+        int totalRemaining = 0;
+        for (ByteBuffer byteBuffer: buffers) {
+            totalRemaining += byteBuffer.remaining();
+        }
+
+        ByteBuffer[] result = new ByteBuffer[buffers.length + 1];
+        result[0] = sizeBuffer(totalRemaining);
+        System.arraycopy(buffers, 0, result, 1, buffers.length);
+        return result;
+    }
+
     private static ByteBuffer sizeBuffer(int size) {
         ByteBuffer sizeBuffer = ByteBuffer.allocate(4);
         sizeBuffer.putInt(size);
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java
index c3dfaa13b996f..aae7a98fd42cd 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java
@@ -131,4 +131,11 @@ public static ByteBuffer serialize(Struct headerStruct, Struct bodyStruct) {
         buffer.rewind();
         return buffer;
     }
+
+    public static ByteBuffer serialize(Struct struct) {
+        ByteBuffer buffer = ByteBuffer.allocate(struct.sizeOf());
+        struct.writeTo(buffer);
+        buffer.rewind();
+        return buffer;
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
index 1ef4118dabb9a..6ab4d25f61f90 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
@@ -26,6 +26,8 @@
 import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataTopicState;
 import org.apache.kafka.common.message.UpdateMetadataResponseData;
 import org.apache.kafka.common.network.ListenerName;
+import org.apache.kafka.common.network.NetworkSend;
+import org.apache.kafka.common.network.Send;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.protocol.types.Struct;
@@ -42,6 +44,8 @@
 import static java.util.Collections.singletonList;
 
 public class UpdateMetadataRequest extends AbstractControlRequest {
+    private final ReentrantLock bodyBufferLock = new ReentrantLock();
+    private byte[] bodyBuffer;
 
     public static class Builder extends AbstractControlRequest.Builder {
         private final List partitionStates;
@@ -241,6 +245,22 @@ public List liveBrokers() {
         return data.liveBrokers();
     }
 
+    @Override
+    public Send toSend(String destination, RequestHeader header) {
+        // For UpdateMetadataRequest, the toSend method on the same object will be called many times, each time with a different destination
+        // value and a header containing a different correlation id.
+        ByteBuffer headerBuffer = serialize(header);
+        bodyBufferLock.lock();
+        try {
+            if (bodyBuffer == null) {
+                bodyBuffer = RequestUtils.serialize(toStruct()).array();
+            }
+        } finally {
+            bodyBufferLock.unlock();
+        }
+        return new NetworkSend(destination, new ByteBuffer[]{headerBuffer, ByteBuffer.wrap(bodyBuffer)});
+    }
+
     @Override
     protected Struct toStruct() {
         // the following inner blocks are not indented in order to make hotfix cherry-picking easier
diff --git a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
index ab858617a5991..a1dad5bcc48af 100755
--- a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
+++ b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
@@ -542,14 +542,26 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
         .setRack(broker.rack.orNull)
     }.toBuffer
 
-    val maxBrokerEpoch = controllerContext.maxBrokerEpoch
-    updateMetadataRequestBrokerSet.intersect(controllerContext.liveOrShuttingDownBrokerIds).foreach { broker =>
-      val brokerEpoch = controllerContext.liveBrokerIdAndEpochs(broker)
+    if (updateMetadataRequestVersion >= 6) {
+      // We should only create one copy UpdateMetadataRequest that should apply to all brokers.
+      // The goal is to reduce memory footprint on the controller.
+      val maxBrokerEpoch = controllerContext.maxBrokerEpoch
       val updateMetadataRequest = new UpdateMetadataRequest.Builder(updateMetadataRequestVersion, controllerId, controllerEpoch,
-        brokerEpoch, maxBrokerEpoch, partitionStates.asJava, liveBrokers.asJava)
-      sendRequest(broker, updateMetadataRequest)
+        AbstractControlRequest.UNKNOWN_BROKER_EPOCH, maxBrokerEpoch, partitionStates.asJava, liveBrokers.asJava)
+
+      updateMetadataRequestBrokerSet.intersect(controllerContext.liveOrShuttingDownBrokerIds).foreach { broker =>
+        sendRequest(broker, updateMetadataRequest)
+      }
+    } else {
+      updateMetadataRequestBrokerSet.intersect(controllerContext.liveOrShuttingDownBrokerIds).foreach { broker =>
+        val brokerEpoch = controllerContext.liveBrokerIdAndEpochs(broker)
+        val updateMetadataRequest = new UpdateMetadataRequest.Builder(updateMetadataRequestVersion, controllerId, controllerEpoch,
+          brokerEpoch, AbstractControlRequest.UNKNOWN_BROKER_EPOCH, partitionStates.asJava, liveBrokers.asJava)
+        sendRequest(broker, updateMetadataRequest)
+      }
     }
 
+
     updateMetadataRequestBrokerSet.clear()
     updateMetadataRequestPartitionInfoMap.clear()
   }

From 0fc7c849146788afa3763d9ff3f7937363189126 Mon Sep 17 00:00:00 2001
From: Xiongqi Wu 
Date: Thu, 10 Oct 2019 18:34:46 -0700
Subject: [PATCH 1055/1071] [LI-HOTFIX] Separate Kafka Controller Node from
 Non-Controller Node (#42)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

TICKET =
LI_DESCRIPTION =

This Patch separates Kafka Controller Node from non-controller (data) Node.
Summary of changes:
1)Add “preferred.controller” broker config (its type is boolean) to indicate whether a broker is used as a preferred controller node.
2)Add “allow.preferred.controller.fallback” broker config that determines whether a non-preferred controller node (data node) is allowed to become controller in the absence of preferred controller nodes. If “allow.preferred.controller.fallback” is set to “false”, a non-preferred controller node never tries to become controller, and only preferred controller nodes can be elected as controller. If “allow.preferred.controller.fallback” is set to “true”, a non-preferred controller node tries to elect itself as controller only in the absence of preferred controller node and a non-preferred active controller resigns as controller when it detects any live preferred controller.
3)Add ActivePreferredControllerCount metric to track whether the current broker is preferred controller node and is elected as Kafka controller
4)Add StandbyPreferredControllerCount metric to track whether the current broker is preferred controller node and is not elected as controller.

EXIT_CRITERIA = MANUAL [""]
---
 .../kafka/controller/ControllerContext.scala  |   8 +
 .../kafka/controller/ControllerState.scala    |   7 +-
 .../kafka/controller/KafkaController.scala    |  95 +++++++++-
 .../scala/kafka/server/AdminManager.scala     |   8 +-
 .../kafka/server/DynamicBrokerConfig.scala    |  23 ++-
 .../main/scala/kafka/server/KafkaConfig.scala |  16 ++
 .../main/scala/kafka/server/KafkaServer.scala |   5 +-
 .../main/scala/kafka/zk/KafkaZkClient.scala   |  15 ++
 core/src/main/scala/kafka/zk/ZkData.scala     |  10 +
 .../server/PreferredControllerTest.scala      | 173 ++++++++++++++++++
 10 files changed, 348 insertions(+), 12 deletions(-)
 create mode 100644 core/src/test/scala/unit/kafka/server/PreferredControllerTest.scala

diff --git a/core/src/main/scala/kafka/controller/ControllerContext.scala b/core/src/main/scala/kafka/controller/ControllerContext.scala
index 85e27b82621aa..3e1a5a9769e0c 100644
--- a/core/src/main/scala/kafka/controller/ControllerContext.scala
+++ b/core/src/main/scala/kafka/controller/ControllerContext.scala
@@ -99,6 +99,8 @@ class ControllerContext {
   val topicsWithDeletionStarted = mutable.Set.empty[String]
   val topicsIneligibleForDeletion = mutable.Set.empty[String]
 
+  @volatile var livePreferredControllerIds: Set[Int] = Set.empty
+
   private def clearTopicsState(): Unit = {
     allTopics = Set.empty
     partitionAssignments.clear()
@@ -191,6 +193,10 @@ class ControllerContext {
     liveBrokers += newMetadata
   }
 
+  def setLivePreferredControllerIds(preferredControllerIds: Set[Int]): Unit = {
+    livePreferredControllerIds = preferredControllerIds
+  }
+
   // getter
   def liveBrokerIds: Set[Int] = liveBrokerEpochs.keySet -- shuttingDownBrokerIds
   def liveOrShuttingDownBrokerIds: Set[Int] = liveBrokerEpochs.keySet
@@ -199,6 +205,8 @@ class ControllerContext {
   def maxBrokerEpoch: Long = liveBrokerEpochs.values.max
   def liveOrShuttingDownBroker(brokerId: Int): Option[Broker] = liveOrShuttingDownBrokers.find(_.id == brokerId)
 
+  def getLivePreferredControllerIds : Set[Int] = livePreferredControllerIds
+
   def partitionsOnBroker(brokerId: Int): Set[TopicPartition] = {
     partitionAssignments.flatMap {
       case (topic, topicReplicaAssignment) => topicReplicaAssignment.filter {
diff --git a/core/src/main/scala/kafka/controller/ControllerState.scala b/core/src/main/scala/kafka/controller/ControllerState.scala
index cf2137ae7b741..f734acbc6e307 100644
--- a/core/src/main/scala/kafka/controller/ControllerState.scala
+++ b/core/src/main/scala/kafka/controller/ControllerState.scala
@@ -108,7 +108,12 @@ object ControllerState {
     def value = 16
   }
 
+  case object PreferredControllerChange extends ControllerState {
+    def value = 17
+  }
+
   val values: Seq[ControllerState] = Seq(Idle, ControllerChange, BrokerChange, TopicChange, TopicDeletion,
     AlterPartitionReassignment, AutoLeaderBalance, ManualLeaderBalance, ControlledShutdown, IsrChange, LeaderAndIsrResponseReceived,
-    LogDirChange, ControllerShutdown, UncleanLeaderElectionEnable, TopicUncleanLeaderElectionEnable, ListPartitionReassignment, TopicDeletionFlagChange)
+    LogDirChange, ControllerShutdown, UncleanLeaderElectionEnable, TopicUncleanLeaderElectionEnable, ListPartitionReassignment,
+    TopicDeletionFlagChange, PreferredControllerChange)
 }
diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala
index c7c6bc00a0b27..e8b744227b968 100644
--- a/core/src/main/scala/kafka/controller/KafkaController.scala
+++ b/core/src/main/scala/kafka/controller/KafkaController.scala
@@ -131,6 +131,7 @@ class KafkaController(val config: KafkaConfig,
 
   private val controllerChangeHandler = new ControllerChangeHandler(eventManager)
   private val brokerChangeHandler = new BrokerChangeHandler(eventManager)
+  private val preferredControllerChangeHandler = new PreferredControllerChangeHandler(eventManager)
   private val brokerModificationsHandlers: mutable.Map[Int, BrokerModificationsHandler] = mutable.Map.empty
   private val topicChangeHandler = new TopicChangeHandler(eventManager)
   private val topicDeletionHandler = new TopicDeletionHandler(eventManager)
@@ -163,6 +164,20 @@ class KafkaController(val config: KafkaConfig,
     }
   )
 
+  newGauge(
+    "ActivePreferredControllerCount",
+    new Gauge[Int] {
+      def value = if (isActive && config.preferredController) 1 else 0
+    }
+  )
+
+  newGauge(
+    "StandbyPreferredControllerCount",
+    new Gauge[Int] {
+      def value = if (!isActive && config.preferredController) 1 else 0
+    }
+  )
+
   newGauge(
     "OfflinePartitionsCount",
     new Gauge[Int] {
@@ -235,6 +250,11 @@ class KafkaController(val config: KafkaConfig,
 
   def epoch: Int = controllerContext.epoch
 
+  /**
+    * @return brokers that don't accept new replicas, including maintenance brokers and preferred controller brokers
+    */
+  def partitionUnassignableBrokerIds: Seq[Int] = config.getMaintenanceBrokerList ++ controllerContext.getLivePreferredControllerIds
+
   /**
    * Invoked when the controller module of a Kafka server is started up. This does not assume that the current broker
    * is the controller. It merely registers the session expiration listener and starts the controller leader
@@ -297,6 +317,10 @@ class KafkaController(val config: KafkaConfig,
     }
   }
 
+  private[kafka] def enablePreferredControllerFallback(): Unit = {
+    eventManager.put(PreferredControllerChange)
+  }
+
   private def state: ControllerState = eventManager.state
 
   /**
@@ -917,14 +941,14 @@ class KafkaController(val config: KafkaConfig,
   // maintenance brokers that do not take new partitions
   private def rearrangePartitionReplicaAssignmentForNewTopics(topics: Set[String]) {
     try {
-      val noNewPartitionBrokerIds = config.getMaintenanceBrokerList
-      if (noNewPartitionBrokerIds.nonEmpty) {
+      val noNewPartitionBrokers = partitionUnassignableBrokerIds
+      if (noNewPartitionBrokers.nonEmpty) {
         val newTopics = zkClient.getPartitionNodeNonExistsTopics(topics.toSet)
         val newTopicsToBeArranged = zkClient.getPartitionAssignmentForTopics(newTopics).filter {
           case (_, partitionMap) =>
             partitionMap.exists {
               case (_, assignedReplicas) =>
-                assignedReplicas.replicas.intersect(noNewPartitionBrokerIds).nonEmpty
+                assignedReplicas.replicas.intersect(noNewPartitionBrokers).nonEmpty
             }
         }
         newTopicsToBeArranged.foreach {
@@ -933,7 +957,7 @@ class KafkaController(val config: KafkaConfig,
             val numReplica = partitionMap.head._2.replicas.size
             val brokers = controllerContext.liveOrShuttingDownBrokers.map { b => kafka.admin.BrokerMetadata(b.id, b.rack) }.toSeq
 
-            val replicaAssignment = adminZkClient.assignReplicasToAvailableBrokers(brokers, noNewPartitionBrokerIds.toSet, numPartitions, numReplica)
+            val replicaAssignment = adminZkClient.assignReplicasToAvailableBrokers(brokers, noNewPartitionBrokers.toSet, numPartitions, numReplica)
             adminZkClient.writeTopicPartitionAssignment(topic, replicaAssignment.mapValues(ReplicaAssignment(_)).toMap, true)
             info(s"Rearrange partition and replica assignment for topic [$topic]")
         }
@@ -1352,6 +1376,7 @@ class KafkaController(val config: KafkaConfig,
 
   private def processStartup(): Unit = {
     zkClient.registerZNodeChangeHandlerAndCheckExistence(controllerChangeHandler)
+    zkClient.registerZNodeChildChangeHandler(preferredControllerChangeHandler)
     elect()
   }
 
@@ -1422,6 +1447,10 @@ class KafkaController(val config: KafkaConfig,
   }
 
   private def elect(): Unit = {
+
+    // refresh preferred controller nodes
+    controllerContext.setLivePreferredControllerIds(zkClient.getPreferredControllerList.toSet)
+
     activeControllerId = zkClient.getControllerId.getOrElse(-1)
     /*
      * We can get here during the initial startup and the handleDeleted ZK callback. Because of the potential race condition,
@@ -1433,6 +1462,16 @@ class KafkaController(val config: KafkaConfig,
       return
     }
 
+    if (!config.preferredController) {
+      /*
+       * A broker that is not preferred controller only elects itself when there is no preferred controllers in cluster
+       * and allowPreferredControllerFallback is set to true.
+       */
+      if (!config.allowPreferredControllerFallback || controllerContext.getLivePreferredControllerIds.nonEmpty) {
+        return
+      }
+    }
+
     try {
       val (epoch, epochZkVersion) = zkClient.registerControllerAndIncrementControllerEpoch(config.brokerId)
       controllerContext.epoch = epoch
@@ -1504,6 +1543,33 @@ class KafkaController(val config: KafkaConfig,
     }
   }
 
+  /**
+    * PreferredControllerChange event should be handled by all brokers to perform:
+    * 1) update preferred controllers
+    * 2) non-preferred active controller resign
+    * 3) elect controller in the absence of preferred controllers. When the last preferred controller goes offline,
+    * ControllerChange event might be fired before the PreferredControllerChange event. When ControllerChange event
+    * is processed, a broker may still see the preferred controller under preferred controller znode and not to elect
+    * itself as controller. Therefore, when PreferredControllerChange event is processed,
+    * a controller election needs to be attempted.
+    */
+  private def processPreferredControllerChange(): Unit = {
+    // refresh the preferred controller list and reset zookeeper watch by reading the list
+    controllerContext.setLivePreferredControllerIds(zkClient.getPreferredControllerList.toSet)
+    if (isActive) {
+      if (!config.preferredController && controllerContext.getLivePreferredControllerIds.nonEmpty) {
+        // resign if a non-preferred active controller detects any live preferred controller
+        info(s"Resign when detecting preferred controllers.")
+        triggerControllerMove()
+      }
+    } else {
+      if (controllerContext.getLivePreferredControllerIds.isEmpty) {
+        // trigger election in the absence of preferred controllers
+        processReelect()
+      }
+    }
+  }
+
   private def processBrokerModification(brokerId: Int): Unit = {
     if (!isActive) return
     val newMetadataOpt = zkClient.getBroker(brokerId)
@@ -1907,6 +1973,9 @@ class KafkaController(val config: KafkaConfig,
   }
 
   private def processRegisterBrokerAndReelect(): Unit = {
+    if (config.preferredController) {
+      zkClient.registerPreferredControllerId(brokerInfo.broker.id)
+    }
     _brokerEpoch = zkClient.registerBroker(brokerInfo)
     processReelect()
   }
@@ -1922,12 +1991,12 @@ class KafkaController(val config: KafkaConfig,
 
     val replicationFactor = config.defaultReplicationFactor
     val brokers = controllerContext.liveOrShuttingDownBrokers.map { sb => kafka.admin.BrokerMetadata(sb.id, sb.rack) }.toSeq
-    val noNewPartitionBrokerIds = config.getMaintenanceBrokerList.toSet
+    val noNewPartitionBrokers = partitionUnassignableBrokerIds.toSet
 
     topicsReplicaAssignment.foreach{
       case(topic, partitionAssignment) => {
         val numPartitions = partitionAssignment.size
-        val assignment = adminZkClient.assignReplicasToAvailableBrokers(brokers, noNewPartitionBrokerIds, numPartitions, replicationFactor)
+        val assignment = adminZkClient.assignReplicasToAvailableBrokers(brokers, noNewPartitionBrokers, numPartitions, replicationFactor)
           .map{ case(partition, replicas) => {
             (new TopicPartition(topic, partition), new ReplicaAssignment(replicas, Seq.empty[Int], Seq.empty[Int]))
           }}.toMap
@@ -1986,6 +2055,8 @@ class KafkaController(val config: KafkaConfig,
           processTopicDeletionStopReplicaResponseReceived(replicaId, requestError, partitionErrors)
         case BrokerChange =>
           processBrokerChange()
+        case PreferredControllerChange =>
+          processPreferredControllerChange()
         case BrokerModifications(brokerId) =>
           processBrokerModification(brokerId)
         case ControllerChange =>
@@ -2049,6 +2120,14 @@ class BrokerChangeHandler(eventManager: ControllerEventManager) extends ZNodeChi
   }
 }
 
+class PreferredControllerChangeHandler(eventManager: ControllerEventManager) extends ZNodeChildChangeHandler {
+  override val path: String = PreferredControllersZNode.path
+
+  override def handleChildChange(): Unit = {
+    eventManager.put(PreferredControllerChange)
+  }
+}
+
 class BrokerModificationsHandler(eventManager: ControllerEventManager, brokerId: Int) extends ZNodeChangeHandler {
   override val path: String = BrokerIdZNode.path(brokerId)
 
@@ -2226,6 +2305,10 @@ case object BrokerChange extends ControllerEvent {
   override def state: ControllerState = ControllerState.BrokerChange
 }
 
+case object PreferredControllerChange extends ControllerEvent {
+  override def state: ControllerState = ControllerState.PreferredControllerChange
+}
+
 case class BrokerModifications(brokerId: Int) extends ControllerEvent {
   override def state: ControllerState = ControllerState.BrokerChange
 }
diff --git a/core/src/main/scala/kafka/server/AdminManager.scala b/core/src/main/scala/kafka/server/AdminManager.scala
index 1449a798fe167..8da9e3195cd38 100644
--- a/core/src/main/scala/kafka/server/AdminManager.scala
+++ b/core/src/main/scala/kafka/server/AdminManager.scala
@@ -20,6 +20,7 @@ import java.util.{Collections, Properties}
 
 import kafka.admin.AdminOperationException
 import kafka.common.TopicAlreadyMarkedForDeletionException
+import kafka.controller.KafkaController
 import kafka.log.LogConfig
 import kafka.utils.Log4jController
 import kafka.metrics.KafkaMetricsGroup
@@ -49,7 +50,8 @@ import scala.collection.JavaConverters._
 class AdminManager(val config: KafkaConfig,
                    val metrics: Metrics,
                    val metadataCache: MetadataCache,
-                   val zkClient: KafkaZkClient) extends Logging with KafkaMetricsGroup {
+                   val zkClient: KafkaZkClient,
+                   val controller: KafkaController) extends Logging with KafkaMetricsGroup {
 
   this.logIdent = "[Admin Manager on Broker " + config.brokerId + "]: "
 
@@ -111,7 +113,7 @@ class AdminManager(val config: KafkaConfig,
           defaultReplicationFactor else topic.replicationFactor
 
         val assignments = if (topic.assignments().isEmpty) {
-          adminZkClient.assignReplicasToAvailableBrokers(brokers, config.getMaintenanceBrokerList.toSet,
+          adminZkClient.assignReplicasToAvailableBrokers(brokers, controller.partitionUnassignableBrokerIds.toSet,
             resolvedNumPartitions, resolvedReplicationFactor)
         } else {
           val assignments = new mutable.HashMap[Int, Seq[Int]]
@@ -308,7 +310,7 @@ class AdminManager(val config: KafkaConfig,
 
         val updatedReplicaAssignment = adminZkClient.addPartitions(topic, existingAssignment, allBrokers,
           newPartition.totalCount, newPartitionsAssignment, validateOnly = validateOnly,
-          noNewPartitionBrokerIds = config.getMaintenanceBrokerList.toSet)
+          noNewPartitionBrokerIds = controller.partitionUnassignableBrokerIds.toSet)
         CreatePartitionsMetadata(topic, updatedReplicaAssignment, ApiError.NONE)
       } catch {
         case e: AdminOperationException =>
diff --git a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
index bfb1efc73cdfb..563fb87af6538 100755
--- a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
+++ b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
@@ -82,10 +82,13 @@ object DynamicBrokerConfig {
     DynamicThreadPool.ReconfigurableConfigs ++
     Set(KafkaConfig.MetricReporterClassesProp) ++
     Set(KafkaConfig.AutoCreateTopicsEnableProp) ++
+    Set(KafkaConfig.AllowPreferredControllerFallbackProp) ++
     DynamicListenerConfig.ReconfigurableConfigs ++
     SocketServer.ReconfigurableConfigs
 
   private val ClusterLevelListenerConfigs = Set(KafkaConfig.MaxConnectionsProp)
+  private val ClusterLevelConfigs = Set(KafkaConfig.AllowPreferredControllerFallbackProp) ++
+    DynamicConfig.Broker.ClusterLevelConfigs
   private val PerBrokerConfigs = DynamicSecurityConfigs  ++
     DynamicListenerConfig.ReconfigurableConfigs -- ClusterLevelListenerConfigs
   private val ListenerMechanismConfigs = Set(KafkaConfig.SaslJaasConfigProp)
@@ -152,7 +155,7 @@ object DynamicBrokerConfig {
 
   private def clusterLevelConfigs(props: Properties): Set[String] = {
     val configNames = props.asScala.keySet
-    configNames.intersect(DynamicConfig.Broker.ClusterLevelConfigs)
+    configNames.intersect(ClusterLevelConfigs)
   }
 
   private def nonDynamicConfigs(props: Properties): Set[String] = {
@@ -255,6 +258,7 @@ class DynamicBrokerConfig(private val kafkaConfig: KafkaConfig) extends Logging
     addBrokerReconfigurable(new DynamicLogConfig(kafkaServer.logManager, kafkaServer))
     addBrokerReconfigurable(new DynamicListenerConfig(kafkaServer))
     addBrokerReconfigurable(kafkaServer.socketServer)
+    addBrokerReconfigurable(new DynamicAllowPreferredControllerFallback(kafkaServer))
   }
 
   def addReconfigurable(reconfigurable: Reconfigurable): Unit = CoreUtils.inWriteLock(lock) {
@@ -669,6 +673,23 @@ object DynamicThreadPool {
     KafkaConfig.BackgroundThreadsProp)
 }
 
+class DynamicAllowPreferredControllerFallback(server: KafkaServer) extends BrokerReconfigurable {
+
+  override def reconfigurableConfigs: Set[String] = {
+    Set(KafkaConfig.AllowPreferredControllerFallbackProp)
+  }
+
+  override def validateReconfiguration(newConfig: KafkaConfig): Unit = {
+    // no additional validation is needed.
+  }
+
+  override def reconfigure(oldConfig: KafkaConfig, newConfig: KafkaConfig): Unit = {
+    if (newConfig.allowPreferredControllerFallback) {
+      server.kafkaController.enablePreferredControllerFallback
+    }
+  }
+}
+
 class DynamicThreadPool(server: KafkaServer) extends BrokerReconfigurable {
 
   override def reconfigurableConfigs: Set[String] = {
diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index f3ba43b960890..f82792474caf1 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -63,6 +63,8 @@ object Defaults {
   val QueuedMaxRequests = 500
   val QueuedMaxRequestBytes = -1
   val ProducerBatchDecompressionEnable = true
+  val PreferredController = false
+  val AllowPreferredControllerFallback = true
 
   /************* Authorizer Configuration ***********/
   val AuthorizerClassName = ""
@@ -300,6 +302,9 @@ object KafkaConfig {
   val HeapDumpFolderProp = "heap.dump.folder"
   val HeapDumpTimeoutProp = "heap.dump.timeout"
   val ProducerBatchDecompressionEnableProp = "producer.batch.decompression.enable"
+  val PreferredControllerProp = "preferred.controller"
+  val AllowPreferredControllerFallbackProp = "allow.preferred.controller.fallback"
+
   /************* Authorizer Configuration ***********/
   val AuthorizerClassNameProp = "authorizer.class.name"
 
@@ -552,6 +557,12 @@ object KafkaConfig {
   val HeapDumpFolderDoc = "The Folder under which heap dumps will be written by the watchdog"
   val HeapDumpTimeoutDoc = "The max amount of time (in millis) to wait for heap dump to complete before halting regardless"
   val ProducerBatchDecompressionEnableDoc = "Decompress batch sent by producer to perform verification of individual records inside the batch"
+  val PreferredControllerDoc = "Specifies whether the broker is a dedicated controller node. If set to true, the broker is a preferred controller node."
+  // Although AllowPreferredControllerFallback is expected to be configured dynamically at per cluster level, providing a static configuration entry
+  // here allows its value to be obtained without holding the dynamic broker configuration lock.
+  val AllowPreferredControllerFallbackDoc = "Specifies whether a non-preferred controller node (broker) is allowed to become the controller." +
+  " This configuration is expected to be configured at cluster level via dynamic broker configuration to provide a consistent configuration among all brokers." +
+  " If AllowPreferredControllerFallback is dynamically set to false and there is no preferred controllers, the non-preferred active controller does not resign."
   /************* Authorizer Configuration ***********/
   val AuthorizerClassNameDoc = s"The fully qualified name of a class that implements s${classOf[Authorizer].getName}" +
   " interface, which is used by the broker for authorization. This config also supports authorizers that implement the deprecated" +
@@ -930,6 +941,8 @@ object KafkaConfig {
       .define(HeapDumpFolderProp, STRING, Defaults.HeapDumpFolder, LOW, HeapDumpFolderDoc)
       .define(HeapDumpTimeoutProp, LONG, Defaults.HeapDumpTimeout, LOW, HeapDumpTimeoutDoc)
       .define(ProducerBatchDecompressionEnableProp, BOOLEAN, Defaults.ProducerBatchDecompressionEnable, LOW, ProducerBatchDecompressionEnableDoc)
+      .define(PreferredControllerProp, BOOLEAN, Defaults.PreferredController, HIGH, PreferredControllerDoc)
+      .define(AllowPreferredControllerFallbackProp, BOOLEAN, Defaults.AllowPreferredControllerFallback, HIGH, AllowPreferredControllerFallbackDoc)
 
       /************* Authorizer Configuration ***********/
       .define(AuthorizerClassNameProp, STRING, Defaults.AuthorizerClassName, LOW, AuthorizerClassNameDoc)
@@ -1238,6 +1251,9 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO
   val heapDumpTimeout = getLong(KafkaConfig.HeapDumpTimeoutProp)
   val producerBatchDecompressionEnable = getBoolean(KafkaConfig.ProducerBatchDecompressionEnableProp)
 
+  var preferredController = getBoolean(KafkaConfig.PreferredControllerProp)
+  def allowPreferredControllerFallback: Boolean = getBoolean(KafkaConfig.AllowPreferredControllerFallbackProp)
+
   def getNumReplicaAlterLogDirsThreads: Int = {
     val numThreads: Integer = Option(getInt(KafkaConfig.NumReplicaAlterLogDirsThreadsProp)).getOrElse(logDirs.size)
     numThreads
diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala
index 5bd021efc48cd..155ea5fde5330 100755
--- a/core/src/main/scala/kafka/server/KafkaServer.scala
+++ b/core/src/main/scala/kafka/server/KafkaServer.scala
@@ -280,6 +280,9 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
         replicaManager.startup()
 
         val brokerInfo = createBrokerInfo
+        if (config.preferredController) {
+          zkClient.registerPreferredControllerId(brokerInfo.broker.id)
+        }
         val brokerEpoch = zkClient.registerBroker(brokerInfo)
 
         healthCheckScheduler = new KafkaScheduler(threads = 1, threadNamePrefix = "kafka-healthcheck-scheduler-")
@@ -300,7 +303,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
         kafkaController = new KafkaController(config, zkClient, time, metrics, brokerInfo, brokerEpoch, tokenManager, threadNamePrefix)
         kafkaController.startup()
 
-        adminManager = new AdminManager(config, metrics, metadataCache, zkClient)
+        adminManager = new AdminManager(config, metrics, metadataCache, zkClient, kafkaController)
 
         /* start group coordinator */
         // Hardcode Time.SYSTEM for now as some Streams tests fail otherwise, it would be good to fix the underlying issue
diff --git a/core/src/main/scala/kafka/zk/KafkaZkClient.scala b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
index ee6a6b12dfe10..4f4eedc97a3c8 100644
--- a/core/src/main/scala/kafka/zk/KafkaZkClient.scala
+++ b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
@@ -95,6 +95,16 @@ class KafkaZkClient private[zk] (zooKeeperClient: ZooKeeperClient, isSecure: Boo
     stat.getCzxid
   }
 
+  /**
+    * Registers the preferred controller id in zookeeper.
+    * @param id controller id
+    */
+  def registerPreferredControllerId(id: Int): Unit = {
+    val path = PreferredControllerIdZNode.path(id)
+    val stat = checkedEphemeralCreate(path, null)
+    info(s"Registered preferred controller ${id} at path $path with czxid (preferred controller epoch): ${stat.getCzxid}")
+  }
+
   /**
    * Registers a given broker in zookeeper as the controller and increments controller epoch.
    * @param controllerId the id of the broker that is to be registered as the controller.
@@ -454,6 +464,11 @@ class KafkaZkClient private[zk] (zooKeeperClient: ZooKeeperClient, isSecure: Boo
    */
   def getSortedBrokerList: Seq[Int] = getChildren(BrokerIdsZNode.path).map(_.toInt).sorted
 
+  /**
+    * Gets the list of preferred controller Ids
+    */
+  def getPreferredControllerList: Seq[Int] = getChildren(PreferredControllersZNode.path).map(_.toInt)
+
   /**
    * Gets all topics in the cluster.
    * @return sequence of topics in the cluster.
diff --git a/core/src/main/scala/kafka/zk/ZkData.scala b/core/src/main/scala/kafka/zk/ZkData.scala
index 81deca92da116..601ea53a8ddc9 100644
--- a/core/src/main/scala/kafka/zk/ZkData.scala
+++ b/core/src/main/scala/kafka/zk/ZkData.scala
@@ -74,6 +74,15 @@ object BrokersZNode {
   def path = "/brokers"
 }
 
+object PreferredControllersZNode {
+  def path = s"${BrokersZNode.path}/preferred_controllers"
+  def encode: Array[Byte] = null
+}
+
+object PreferredControllerIdZNode {
+  def path(id: Int) = s"${BrokersZNode.path}/preferred_controllers/$id"
+}
+
 object BrokerIdsZNode {
   def path = s"${BrokersZNode.path}/ids"
   def encode: Array[Byte] = null
@@ -775,6 +784,7 @@ object ZkData {
   val PersistentZkPaths = Seq(
     ConsumerPathZNode.path, // old consumer path
     BrokerIdsZNode.path,
+    PreferredControllersZNode.path,
     TopicsZNode.path,
     ConfigEntityChangeNotificationZNode.path,
     DeleteTopicsZNode.path,
diff --git a/core/src/test/scala/unit/kafka/server/PreferredControllerTest.scala b/core/src/test/scala/unit/kafka/server/PreferredControllerTest.scala
new file mode 100644
index 0000000000000..3a130a22baafa
--- /dev/null
+++ b/core/src/test/scala/unit/kafka/server/PreferredControllerTest.scala
@@ -0,0 +1,173 @@
+/**
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  * http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+
+package kafka.server
+
+import java.util.Properties
+
+import kafka.server.KafkaConfig.fromProps
+import kafka.utils.CoreUtils._
+import kafka.utils.TestUtils
+import kafka.utils.TestUtils._
+import kafka.zk.ZooKeeperTestHarness
+import org.apache.kafka.clients.admin._
+import org.apache.kafka.common.network.ListenerName
+import org.apache.kafka.common.security.auth.SecurityProtocol
+import org.junit.Assert._
+import org.junit.{After, Test}
+import org.scalatest.Assertions.fail
+
+import scala.collection.JavaConverters._
+import scala.collection.Map
+
+class PreferredControllerTest extends ZooKeeperTestHarness {
+
+  var brokers: Seq[KafkaServer] = null
+
+  @After
+  override def tearDown() {
+    shutdownServers(brokers)
+    super.tearDown()
+  }
+
+  @Test
+  def testPartitionCreatedByAdminClientShouldNotBeAssignedToPreferredControllers(): Unit = {
+    val brokerConfigs = Seq((0, false), (1, true), (2, false))
+    createBrokersWithPreferredControllers(brokerConfigs, true)
+
+    val brokerList = TestUtils.bootstrapServers(brokers, ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT))
+    val adminClientConfig = new Properties
+    adminClientConfig.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList)
+    val client = AdminClient.create(adminClientConfig)
+
+    TestUtils.waitUntilControllerElected(zkClient)
+    // create topic using admin client
+    val future1 = client.createTopics(Seq("topic1").map(new NewTopic(_, 3, 2.toShort)).asJava,
+      new CreateTopicsOptions()).all()
+    future1.get()
+
+    assertTrue("topic1 should not be in broker 1", ensureTopicNotInBrokers("topic1", Set(1)))
+
+    val future2 = client.createPartitions(Map("topic1" -> NewPartitions.increaseTo(5)).asJava).all()
+    future2.get()
+
+    assertTrue("topic1 should not be in broker 1 after increasing partition count",
+      ensureTopicNotInBrokers("topic1", Set(1)))
+
+    client.close()
+  }
+
+  @Test
+  def testElectionWithoutPreferredControllersAndNoFallback(): Unit = {
+    val brokerConfigs = Seq((0, false), (1, false), (2, false))
+    createBrokersWithPreferredControllers(brokerConfigs, false)
+    // no broker can be elected as controller
+    ensureControllersInBrokers(Seq.empty, 5000L)
+  }
+
+  @Test
+  def testPreferredControllerElection(): Unit = {
+    val brokerConfigs = Seq((0, false), (1, true), (2, false))
+    createBrokersWithPreferredControllers(brokerConfigs, false)
+    // only broker 1 can be elected since it is the only preferred controller node
+    ensureControllersInBrokers(Seq(1))
+  }
+
+
+  @Test
+  def testNonPreferredControllerResignation(): Unit = {
+    val brokerConfigs = Seq((0, false), (1, true), (2, false))
+    createBrokersWithPreferredControllers(brokerConfigs, true)
+
+    // broker 1 should be elected since it is the only preferred controller node
+    ensureControllersInBrokers(Seq(1))
+    brokers(1).shutdown()
+
+    // broker 0 and broker 2 can become controller when broker 1 is offline
+    ensureControllersInBrokers(Seq(0, 2))
+    brokers(1).startup()
+    // broker 1 regains controllership
+    ensureControllersInBrokers(Seq(1))
+  }
+
+  @Test
+  def testDynamicAllowPreferredControllerFallback(): Unit = {
+    val brokerConfigs = Seq((0, false), (1, false), (2, false))
+    createBrokersWithPreferredControllers(brokerConfigs, false)
+
+    // non preferred controller nodes cannot be elected as the controller if fallback is not allowed
+    ensureControllersInBrokers(Seq.empty, 5000L)
+    setAllowPreferredControllerFallback(true)
+    // controller can be now elected among non preferred controller nodes
+    TestUtils.waitUntilControllerElected(zkClient)
+  }
+
+  @Test
+  def testCurrentControllerDoesNotResignWithoutPreferredControllersAndNoFallback(): Unit = {
+    val brokerConfigs = Seq((0, false), (1, false), (2, false))
+    createBrokersWithPreferredControllers(brokerConfigs, true)
+
+    val controllerId = TestUtils.waitUntilControllerElected(zkClient)
+
+    setAllowPreferredControllerFallback(false)
+
+    // current controller does not move
+    ensureControllersInBrokers(Seq(controllerId))
+  }
+
+  private def ensureControllersInBrokers(brokerIds: Seq[Int], timeout: Long = 15000L): Unit = {
+    val (controllerId, _) = TestUtils.computeUntilTrue(zkClient.getControllerId, waitTime = timeout) {
+      case controller =>
+        controller.isDefined && (brokerIds.isEmpty || brokerIds.contains(controller))
+    }
+    if (brokerIds.isEmpty) {
+      assertTrue("there should not be any controller", controllerId.isEmpty)
+    } else {
+      assertTrue(s"Controller should be elected in $brokerIds",
+        brokerIds.contains(controllerId.getOrElse(fail(s"Controller not elected after $timeout ms"))))
+    }
+  }
+
+  private def ensureTopicNotInBrokers(topic: String, brokerIds: Set[Int]): Boolean = {
+    val topicAssignment = zkClient.getReplicaAssignmentForTopics(Set(topic))
+    topicAssignment.map(_._2).flatten.toSet.intersect(brokerIds).isEmpty
+  }
+
+  /**
+    * @param brokerConfigs: a list of (brokerid, preferredController) configs
+    * @param allowFallback: "allow.preferred.controller.fallback" config
+    */
+  private def createBrokersWithPreferredControllers(brokerConfigs: Seq[(Int, Boolean)],  allowFallback: Boolean): Unit = {
+    brokers = brokerConfigs.map {
+      case (id, preferredController) =>
+        val props: Properties = createBrokerConfig(id, zkConnect)
+        props.put(KafkaConfig.PreferredControllerProp, preferredController.toString)
+        props.put(KafkaConfig.AllowPreferredControllerFallbackProp, allowFallback.toString)
+        createServer(fromProps(props))
+    }
+  }
+
+  private def setAllowPreferredControllerFallback(allowFallback: Boolean): Unit = {
+    adminZkClient.changeBrokerConfig(None,
+      propsWith((KafkaConfig.AllowPreferredControllerFallbackProp, allowFallback.toString)))
+
+    TestUtils.waitUntilTrue(() => {
+        brokers.forall(_.config.allowPreferredControllerFallback == allowFallback)
+      },
+      s"fail to set ${KafkaConfig.AllowPreferredControllerFallbackProp} to ${allowFallback}", 5000)
+  }
+}

From a1bd9410f85451b720cd22d11115fe11eba832d2 Mon Sep 17 00:00:00 2001
From: Radai Rosenblatt 
Date: Wed, 15 Apr 2020 13:12:52 -0700
Subject: [PATCH 1056/1071] [LI-HOTFIX] KAFKA-9855 - return cached Structs for
 Schemas with no fields (#77)

TICKET =
LI_DESCERIPTION = committing to our fork until upstream picks this up
EXIT_CRITERIA = when KAFKA-9855 is accepted upstream (specifically - https://github.com/apache/kafka/pull/8472)

Signed-off-by: radai-rosenblatt 
---
 .../org/apache/kafka/common/protocol/types/Schema.java    | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java
index 92144b37e04d1..8aa54fc3689c8 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java
@@ -25,10 +25,12 @@
  * The schema for a compound record definition
  */
 public class Schema extends Type {
+    private final static Object[] NO_VALUES = new Object[0];
 
     private final BoundField[] fields;
     private final Map fieldsByName;
     private final boolean tolerateMissingFieldsWithDefaults;
+    private final Struct cachedStruct;
 
     /**
      * Construct the schema with a given list of its field values
@@ -62,6 +64,9 @@ public Schema(boolean tolerateMissingFieldsWithDefaults, Field... fs) {
             this.fields[i] = new BoundField(def, this, i);
             this.fieldsByName.put(def.name, this.fields[i]);
         }
+        //6 schemas have no fields at the time of this writing (3 versions each of list_groups and api_versions)
+        //for such schemas there's no point in even creating a unique Struct object when deserializing.
+        this.cachedStruct = this.fields.length > 0 ? null : new Struct(this, NO_VALUES);
     }
 
     /**
@@ -90,6 +95,9 @@ public void write(ByteBuffer buffer, Object o) {
      */
     @Override
     public Struct read(ByteBuffer buffer) {
+        if (cachedStruct != null) {
+            return cachedStruct;
+        }
         Object[] objects = new Object[fields.length];
         for (int i = 0; i < fields.length; i++) {
             try {

From 092873519c0d75d904b552a413ad19b318c0ff4c Mon Sep 17 00:00:00 2001
From: Adem Efe Gencer 
Date: Tue, 28 Apr 2020 18:04:28 -0700
Subject: [PATCH 1057/1071] [LI-HOTFIX] Prevent ReplicaFetcherThread from
 throwing UnknownTopicOrPartitionException upon topic creation and deletion.
 (#80)

TICKET =
LI_DESCRIPTION =
When does UnknownTopicOrPartitionException typically occur?
* Upon a topic creation, a follower broker of a new partition starts replica fetcher before the prospective leader broker of the new partition receives the leadership information from the controller. Apache Kafka has a an open issue about this (see KAFKA-6221)
* Upon a topic deletion, a follower broker of a to-be-deleted partition starts replica fetcher after the leader broker of the to-be-deleted partition processes the deletion information from the controller.
* As expected, clusters with frequent topic creation and deletion report UnknownTopicOrPartitionException with relatively higher frequency.

What is the impact?
* Exception tracking systems identify the error logs with UnknownTopicOrPartitionException as an exception. This results in a lot of noise for a transient issue that is expected to recover by itself and a natural process in Kafka due to its asynchronous state propagation.

Why not move it to a lower than warn-level log?
* Despite typically being a transient issue, UnknownTopicOrPartitionException may also indicate real issues if it doesn't fix itself after a short period of time. To ensure detection of such scenarios, this PR sets the log level to warn.

EXIT_CRITERIA = TICKET [KAFKA-6221]
---
 core/src/main/scala/kafka/server/AbstractFetcherThread.scala | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
index 995e5beab02ae..c8d59f142ab89 100755
--- a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
+++ b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
@@ -389,6 +389,11 @@ abstract class AbstractFetcherThread(name: String,
                     "that the partition is being moved")
                   partitionsWithError += topicPartition
 
+                case Errors.UNKNOWN_TOPIC_OR_PARTITION =>
+                  warn(s"Remote broker does not host the partition $topicPartition, which could indicate " +
+                    "that the partition is being created or deleted.")
+                  partitionsWithError += topicPartition
+
                 case _ =>
                   error(s"Error for partition $topicPartition at offset ${currentFetchState.fetchOffset}",
                     partitionData.error.exception)

From a566ba29caf017f90484adb2a79d89c2e9faa2be Mon Sep 17 00:00:00 2001
From: Navina Ramesh 
Date: Wed, 20 May 2020 12:56:20 -0700
Subject: [PATCH 1058/1071] [LI-HOTFIX] KAFKA-10012 - Reducing memory overhead
 associated with strings in MetricName (#83)

TICKET =
LI_DESCRIPTION =
SelectorMetrics has a per-connection metrics, which means the number of
MetricName objects and the strings associated with it (such as group name and
description) grows with the number of connections in the client. This overhead
of duplicate string objects is amplified when there are multiple instances of
kafka clients within the same JVM. This patch address some of the memory
overhead by making metricGrpName a constant and introducing a new constant
perConnectionMetricGrpName. Additionally, the strings for metric name and
description in createMeter have been interned since there are about 8 types
of meter metric for every single client.
EXIT_CRITERIA = TICKET [KAFKA-10012]
---
 .../apache/kafka/common/network/Selector.java | 33 ++++++++++---------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selector.java b/clients/src/main/java/org/apache/kafka/common/network/Selector.java
index 2652906099d78..27c28f15a47d6 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/Selector.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/Selector.java
@@ -1030,6 +1030,8 @@ private class SelectorMetrics implements AutoCloseable {
         private final String metricGrpPrefix;
         private final Map metricTags;
         private final boolean metricsPerConnection;
+        private final String metricGrpName;
+        private final String perConnectionMetricGrpName;
 
         public final Sensor connectionClosed;
         public final Sensor connectionCreated;
@@ -1054,7 +1056,8 @@ public SelectorMetrics(Metrics metrics, String metricGrpPrefix, Map tag: metricTags.entrySet()) {
@@ -1143,10 +1146,12 @@ public SelectorMetrics(Metrics metrics, String metricGrpPrefix, Map metricTags,
                 SampledStat stat, String baseName, String descriptiveName) {
-            MetricName rateMetricName = metrics.metricName(baseName + "-rate", groupName,
-                            String.format("The number of %s per second", descriptiveName), metricTags);
-            MetricName totalMetricName = metrics.metricName(baseName + "-total", groupName,
-                            String.format("The total number of %s", descriptiveName), metricTags);
+            // Reason for using intern here: These strings are very repetitive and can lead to 10K+ identical objects in
+            // memory. They also low QPS (access). Hence, it is ok to intern() them.
+            MetricName rateMetricName = metrics.metricName((baseName + "-rate").intern(), groupName,
+                            String.format("The number of %s per second", descriptiveName).intern(), metricTags);
+            MetricName totalMetricName = metrics.metricName((baseName + "-total").intern(), groupName,
+                            String.format("The total number of %s", descriptiveName).intern(), metricTags);
             if (stat == null)
                 return new Meter(rateMetricName, totalMetricName);
             else
@@ -1180,29 +1185,27 @@ public void maybeRegisterConnectionMetrics(String connectionId) {
                 String nodeRequestName = "node-" + connectionId + ".bytes-sent";
                 Sensor nodeRequest = this.metrics.getSensor(nodeRequestName);
                 if (nodeRequest == null) {
-                    String metricGrpName = metricGrpPrefix + "-node-metrics";
-
                     Map tags = new LinkedHashMap<>(metricTags);
                     tags.put("node-id", "node-" + connectionId);
 
                     nodeRequest = sensor(nodeRequestName);
-                    nodeRequest.add(createMeter(metrics, metricGrpName, tags, "outgoing-byte", "outgoing bytes"));
-                    nodeRequest.add(createMeter(metrics, metricGrpName, tags, new WindowedCount(), "request", "requests sent"));
-                    MetricName metricName = metrics.metricName("request-size-avg", metricGrpName, "The average size of requests sent.", tags);
+                    nodeRequest.add(createMeter(metrics, perConnectionMetricGrpName, tags, "outgoing-byte", "outgoing bytes"));
+                    nodeRequest.add(createMeter(metrics, perConnectionMetricGrpName, tags, new WindowedCount(), "request", "requests sent"));
+                    MetricName metricName = metrics.metricName("request-size-avg", perConnectionMetricGrpName, "The average size of requests sent.", tags);
                     nodeRequest.add(metricName, new Avg());
-                    metricName = metrics.metricName("request-size-max", metricGrpName, "The maximum size of any request sent.", tags);
+                    metricName = metrics.metricName("request-size-max", perConnectionMetricGrpName, "The maximum size of any request sent.", tags);
                     nodeRequest.add(metricName, new Max());
 
                     String nodeResponseName = "node-" + connectionId + ".bytes-received";
                     Sensor nodeResponse = sensor(nodeResponseName);
-                    nodeResponse.add(createMeter(metrics, metricGrpName, tags, "incoming-byte", "incoming bytes"));
-                    nodeResponse.add(createMeter(metrics, metricGrpName, tags, new WindowedCount(), "response", "responses received"));
+                    nodeResponse.add(createMeter(metrics, perConnectionMetricGrpName, tags, "incoming-byte", "incoming bytes"));
+                    nodeResponse.add(createMeter(metrics, perConnectionMetricGrpName, tags, new WindowedCount(), "response", "responses received"));
 
                     String nodeTimeName = "node-" + connectionId + ".latency";
                     Sensor nodeRequestTime = sensor(nodeTimeName);
-                    metricName = metrics.metricName("request-latency-avg", metricGrpName, tags);
+                    metricName = metrics.metricName("request-latency-avg", perConnectionMetricGrpName, tags);
                     nodeRequestTime.add(metricName, new Avg());
-                    metricName = metrics.metricName("request-latency-max", metricGrpName, tags);
+                    metricName = metrics.metricName("request-latency-max", perConnectionMetricGrpName, tags);
                     nodeRequestTime.add(metricName, new Max());
                 }
             }

From fde1fbbd0ae0193c8ce64db51fc8db5b7ef15287 Mon Sep 17 00:00:00 2001
From: Alex Wang 
Date: Fri, 1 Mar 2019 15:53:04 -0800
Subject: [PATCH 1059/1071] [LI-HOTFIX] Fixed NPE in LiCreateTopicPolicy, setup
 test in AdminClientIntegrationTest. TICKET = LI_DESCRIPTION =

EXIT_CRITERIA = MANUAL [""]
---
 .../main/scala/kafka/server/LiCreateTopicPolicy.scala | 11 ++++++-----
 .../kafka/api/AdminClientIntegrationTest.scala        |  3 +++
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/core/src/main/scala/kafka/server/LiCreateTopicPolicy.scala b/core/src/main/scala/kafka/server/LiCreateTopicPolicy.scala
index 2553e9471931b..56c0f46e5da89 100644
--- a/core/src/main/scala/kafka/server/LiCreateTopicPolicy.scala
+++ b/core/src/main/scala/kafka/server/LiCreateTopicPolicy.scala
@@ -26,17 +26,18 @@ class LiCreateTopicPolicy extends CreateTopicPolicy {
   override def validate(requestMetadata: CreateTopicPolicy.RequestMetadata): Unit = {
     val requestTopic = requestMetadata.topic
     val requestRF = requestMetadata.replicationFactor()
-    import collection.JavaConverters._
-    val requestAssignment = requestMetadata.replicasAssignments().asScala
 
-    if (requestAssignment == null && requestRF == null)
+    // For scala 2.11.x, a scala object converted from a null java object via asScala is NOT null. This could lead to
+    // NPE, thus we check the java object directly. AdminClientIntegrationTest verifies the behavior.
+    if (requestMetadata.replicasAssignments() == null && requestRF == null)
       throw new PolicyViolationException(s"Topic [$requestTopic] is missing both replica assignment and " +
         s"replication factor.")
 
     // In createTopics() in AdminManager, replicationFactor and replicasAssignments are not both set at same time. We
     // follow the same rationale here and prioritize replicasAssignments over replicationFactor
-    if (requestAssignment != null) {
-      requestAssignment.foreach { case (p, assignment) =>
+    if (requestMetadata.replicasAssignments() != null) {
+      import collection.JavaConverters._
+      requestMetadata.replicasAssignments().asScala.foreach { case (p, assignment) =>
           if (assignment.size() < minRf)
             throw new PolicyViolationException(s"Topic [$requestTopic] fails RF requirement. Received RF for " +
               s"[partition-$p]: ${assignment.size()}, min required RF: $minRf.")
diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index c899719dacf25..b3afa1bc1265f 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -107,6 +107,9 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
       config.setProperty(KafkaConfig.GroupInitialRebalanceDelayMsProp, "0")
       config.setProperty(KafkaConfig.AutoLeaderRebalanceEnableProp, "false")
       config.setProperty(KafkaConfig.ControlledShutdownEnableProp, "false")
+      // Set up CreateTopicPolicy to be included in test.
+      config.setProperty(KafkaConfig.DefaultReplicationFactorProp, "1");
+      config.setProperty(KafkaConfig.CreateTopicPolicyClassNameProp, "kafka.server.LiCreateTopicPolicy")
       // We set this in order to test that we don't expose sensitive data via describe configs. This will already be
       // set for subclasses with security enabled and we don't want to overwrite it.
       if (!config.containsKey(KafkaConfig.SslTruststorePasswordProp))

From 0d9c52774c564616724c9b2d54152fa416bc9819 Mon Sep 17 00:00:00 2001
From: Lucas Wang 
Date: Mon, 15 Jun 2020 17:07:24 -0700
Subject: [PATCH 1060/1071] [LI-HOTFIX] passthrough performance improvement in
 Log.append TICKET = LI_DESCRIPTION = LIKAFKA-9737 Broker-side changes for
 pass-through solution

EXIT_CRITERIA = MANUAL ["If we no longer need pass-through support in Kafka. Only known user is KMM for venice; BrooklinMM has not enabled this everywhere."]
---
 core/src/main/scala/kafka/log/Log.scala       |  73 +++++++-
 .../test/scala/unit/kafka/log/LogTest.scala   | 164 ++++++++++++++++--
 2 files changed, 219 insertions(+), 18 deletions(-)

diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index 0d9f503e45019..3db6cc8e11579 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -1048,6 +1048,77 @@ class Log(@volatile private var _dir: File,
       leaderEpoch = -1)
   }
 
+  /**
+   *  Append records to the active segment of the log.
+   *
+   * This method accepts a MemoryRecords with a buffer that contains one or more valid MemoryRecords and calls
+   * appendSingleBatch for each individually. This allows multiple batches to be aggregated in a single
+   * record_set, but still be processed without forcing offset assignment.
+   *
+   * @param records The log records to append
+   * @param origin Declares the origin of the append which affects required validations
+   * @param interBrokerProtocolVersion Inter-broker message protocol version
+   * @param assignOffsets Should the log assign offsets to this message set or blindly apply what it is given
+   * @param leaderEpoch The partition's leader epoch which will be applied to messages when offsets are assigned on the leader
+   * @throws KafkaStorageException If the append fails due to an I/O error.
+   * @throws OffsetsOutOfOrderException If out of order offsets found in 'records'
+   * @throws UnexpectedAppendOffsetException If the first or last offset in append is less than next offset
+   * @return Information about the appended messages including the first and last offset.
+   */
+  private def append(records: MemoryRecords,
+                                origin: AppendOrigin,
+                                interBrokerProtocolVersion: ApiVersion,
+                                assignOffsets: Boolean,
+                                leaderEpoch: Int): LogAppendInfo = {
+    val batches = records.batches()
+
+    if (batches.asScala.isEmpty) {
+      return analyzeAndValidateRecords(records, origin)
+    }
+    val remainingBytes = records.buffer().duplicate()
+    batches.asScala.map(b => {
+      // Create the current record set using only the first batch
+      val batchSize = b.sizeInBytes()
+      val batchBuffer = remainingBytes.slice()
+      batchBuffer.limit(batchSize)
+      val batchRecords = MemoryRecords.readableRecords(batchBuffer)
+
+      // Advance the position in remaining bytes
+      remainingBytes.position(remainingBytes.position() + batchSize)
+
+      // Append the single record batch to the log
+      appendSingleBatch(batchRecords, origin, interBrokerProtocolVersion, assignOffsets, leaderEpoch)
+    }).reduceLeft((info1, info2) => {
+      // Don't assume that the max timestamp is always in the later batch
+      var maxTimestamp = info1.maxTimestamp
+      var offsetofMaxTimestamp = info1.offsetOfMaxTimestamp
+      if (info2.maxTimestamp > info1.maxTimestamp) {
+        maxTimestamp = info2.maxTimestamp
+        offsetofMaxTimestamp = info2.offsetOfMaxTimestamp
+      }
+      var combinedRecordConversionStats = info1.recordConversionStats
+      combinedRecordConversionStats.add(info2.recordConversionStats)
+
+      // Combine the LogAppendInfo to maintain an overall result to return
+      LogAppendInfo(
+        info1.firstOffset,
+        info2.lastOffset,
+        maxTimestamp,
+        offsetofMaxTimestamp,
+        info1.logAppendTime,
+        info1.logStartOffset,
+        combinedRecordConversionStats,
+        info1.sourceCodec,
+        info1.targetCodec,
+        info1.shallowCount + info2.shallowCount,
+        info1.validBytes + info2.validBytes,
+        info1.offsetsMonotonic && info2.offsetsMonotonic,
+        info1.lastOffsetOfFirstBatch,
+        info1.recordErrors ++ info2.recordErrors,
+        info1.errorMessage + info2.errorMessage
+      )
+    })
+  }
   /**
    * Append this message set to the active segment of the log, rolling over to a fresh segment if necessary.
    *
@@ -1064,7 +1135,7 @@ class Log(@volatile private var _dir: File,
    * @throws UnexpectedAppendOffsetException If the first or last offset in append is less than next offset
    * @return Information about the appended messages including the first and last offset.
    */
-  private def append(records: MemoryRecords,
+  private def appendSingleBatch(records: MemoryRecords,
                      origin: AppendOrigin,
                      interBrokerProtocolVersion: ApiVersion,
                      assignOffsets: Boolean,
diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala
index fbad0ece1c6e3..892997e09d088 100755
--- a/core/src/test/scala/unit/kafka/log/LogTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogTest.scala
@@ -52,7 +52,6 @@ import scala.collection.mutable.ListBuffer
 import org.scalatest.Assertions.{assertThrows, intercept, withClue}
 
 class LogTest {
-  var config: KafkaConfig = null
   val brokerTopicStats = new BrokerTopicStats
   val tmpDir = TestUtils.tempDir()
   val logDir = TestUtils.randomPartitionLogDir(tmpDir)
@@ -60,10 +59,7 @@ class LogTest {
   def metricsKeySet = Metrics.defaultRegistry.allMetrics.keySet.asScala
 
   @Before
-  def setUp(): Unit = {
-    val props = TestUtils.createBrokerConfig(0, "127.0.0.1:1", port = -1)
-    config = KafkaConfig.fromProps(props)
-  }
+  def setUp(): Unit = {}
 
   @After
   def tearDown(): Unit = {
@@ -1825,6 +1821,133 @@ class LogTest {
     log.appendAsLeader(records, leaderEpoch = 0)
   }
 
+  /**
+   * This test generates a single record set that is a concatenation of many valid record sets, and appends them to
+   * a log. It then checks to make sure we can read them all back.
+   */
+  @Test
+  def testAppendManyRecordSets() {
+    val msgFormatSet = Set(RecordBatch.MAGIC_VALUE_V0, RecordBatch.MAGIC_VALUE_V1, RecordBatch.MAGIC_VALUE_V2)
+    msgFormatSet.foreach { magic =>
+      appendManyRecordSets(createLog(TestUtils.randomPartitionLogDir(tmpDir), LogTest.createLogConfig(segmentBytes = 100)),
+        CompressionType.NONE, magic)
+      appendManyRecordSets(createLog(TestUtils.randomPartitionLogDir(tmpDir), LogTest.createLogConfig(segmentBytes = 100)),
+        CompressionType.GZIP, magic)
+      appendManyRecordSets(createLog(TestUtils.randomPartitionLogDir(tmpDir), LogTest.createLogConfig(segmentBytes = 100)),
+        CompressionType.SNAPPY, magic)
+      appendManyRecordSets(createLog(TestUtils.randomPartitionLogDir(tmpDir), LogTest.createLogConfig(segmentBytes = 100)),
+        CompressionType.LZ4, magic)
+    }
+  }
+
+  def readUncommitted(log: Log, startOffset: Long, maxLength: Int, maxOffset: Option[Long] = None): FetchDataInfo = {
+    log.read(startOffset,
+      maxLength = maxLength,
+      isolation = FetchLogEnd,
+      minOneMessage = false)
+  }
+
+  def appendManyRecordSets(log: Log,
+                           compressionType: CompressionType, magicValue: Byte) {
+    val values = (0 until 50).map(id => TestUtils.randomBytes(10)).toArray
+
+    // Build a single buffer with all record sets appended
+    val recordBuffer = ByteBuffer.allocate(5000)
+    for(value <- values) {
+      recordBuffer.put(
+        TestUtils.singletonRecords(
+          value = value, codec = compressionType, magicValue = magicValue).buffer())
+    }
+
+    // Close the buffer and create a MemoryRecords wrapping it
+    recordBuffer.flip()
+    recordBuffer.position(0)
+    val records = MemoryRecords.readableRecords(recordBuffer.slice())
+
+    log.appendAsLeader(records, leaderEpoch = 0)
+
+
+    for(i <- values.indices) {
+      val read = readUncommitted(log, i, 100, Some(i+1)).records.batches.iterator.next()
+      assertEquals("Offset read should match order appended.", i, read.lastOffset)
+      val actual = read.iterator.next()
+      assertNull("Key should be null", actual.key)
+      assertEquals("Values not equal", ByteBuffer.wrap(values(i)), actual.value)
+    }
+    assertEquals("Reading beyond the last message returns nothing.", 0,
+      readUncommitted(log, values.length, 100, None).records.batches.asScala.size)
+  }
+
+  /**
+   * This test makes sure that appending an empty record set does not fail. There should be no exception raised
+   */
+  @Test
+  def testAppendEmptyRecordSet() {
+    val logConfig = LogTest.createLogConfig(segmentBytes = 71)
+    val log = createLog(logDir, logConfig)
+    log.appendAsFollower(MemoryRecords.withRecords(CompressionType.NONE))
+    log.appendAsFollower(MemoryRecords.withRecords(CompressionType.GZIP))
+    log.appendAsFollower(MemoryRecords.withRecords(CompressionType.LZ4))
+    log.appendAsFollower(MemoryRecords.withRecords(CompressionType.SNAPPY))
+  }
+
+  /**
+   * This test generates a single record set that is a concatenation of many valid record sets, with one empty record
+   * set in the middle, and appends them to a log. It then checks to make sure we can read them all back.
+   */
+  @Test
+  def testAppendManyRecordSetsWithEmpty() {
+    val msgFormatSet = Set(RecordBatch.MAGIC_VALUE_V0, RecordBatch.MAGIC_VALUE_V1, RecordBatch.MAGIC_VALUE_V2)
+    msgFormatSet.foreach { magic =>
+      appendManyRecordSets(createLog(TestUtils.randomPartitionLogDir(tmpDir), LogTest.createLogConfig(segmentBytes = 100)),
+        CompressionType.NONE, magic)
+      appendManyRecordSets(createLog(TestUtils.randomPartitionLogDir(tmpDir), LogTest.createLogConfig(segmentBytes = 100)),
+        CompressionType.GZIP, magic)
+      appendManyRecordSets(createLog(TestUtils.randomPartitionLogDir(tmpDir), LogTest.createLogConfig(segmentBytes = 100)),
+        CompressionType.SNAPPY, magic)
+      appendManyRecordSets(createLog(TestUtils.randomPartitionLogDir(tmpDir), LogTest.createLogConfig(segmentBytes = 100)),
+        CompressionType.LZ4, magic)
+    }
+
+  }
+
+  def appendManyRecordSetsWithEmpty(log: Log, compressionType: CompressionType, magicValue: Byte): Unit = {
+    val values1 = (0 until 50 by 2).map(id => TestUtils.randomBytes(10)).toArray
+    val values2 = (50 until 100 by 2).map(id => TestUtils.randomBytes(10)).toArray
+
+    // Build a single buffer with all record sets appended
+    val recordBuffer = ByteBuffer.allocate(4000)
+    for (v <- values1) {
+      recordBuffer.put(
+        TestUtils.singletonRecords(value = v, codec = compressionType, magicValue = magicValue).buffer()
+      )
+    }
+    recordBuffer.put(MemoryRecords.withRecords(magicValue, CompressionType.NONE).buffer())
+    for (v <- values2) {
+      recordBuffer.put(
+        TestUtils.singletonRecords(value = v, codec = compressionType, magicValue = magicValue).buffer()
+      )
+    }
+
+    // Close the buffer and create a MemoryRecords wrapping it
+    recordBuffer.flip()
+    recordBuffer.position(0)
+    val records = MemoryRecords.readableRecords(recordBuffer.slice())
+
+    log.appendAsLeader(records, leaderEpoch = 0)
+
+    val values = values1 ++ values2
+    for (i <- values.indices) {
+      val read = readUncommitted(log, i, 100, Some(i + 1)).records.batches.iterator.next()
+      assertEquals("Offset read should match order appended.", i, read.lastOffset)
+      val actual = read.iterator.next()
+      assertNull("Key should be null", actual.key)
+      assertEquals("Values not equal", ByteBuffer.wrap(values(i)), actual.value)
+    }
+    assertEquals("Reading beyond the last message returns nothing.", 0,
+      readUncommitted(log, values.length, 100, None).records.batches.asScala.size)
+  }
+
   /**
    * This test covers an odd case where we have a gap in the offsets that falls at the end of a log segment.
    * Specifically we create a log where the last message in the first segment has offset 0. If we
@@ -2557,7 +2680,7 @@ class LogTest {
     buffer.flip()
     val memoryRecords = MemoryRecords.readableRecords(buffer)
 
-    assertThrows[OffsetsOutOfOrderException] {
+    assertThrows[UnexpectedAppendOffsetException] {
       log.appendAsFollower(memoryRecords)
     }
   }
@@ -2581,32 +2704,39 @@ class LogTest {
   }
 
   @Test
-  def testAppendEmptyLogBelowLogStartOffsetThrowsException(): Unit = {
+  def testAppendEmptyLogBelowLogStartOffsetThrowsException() {
     createEmptyLogs(logDir, 7)
     val log = createLog(logDir, LogConfig(), brokerTopicStats = brokerTopicStats)
     assertEquals(7L, log.logStartOffset)
     assertEquals(7L, log.logEndOffset)
 
-    val firstOffset = 4L
-    val magicVals = Seq(RecordBatch.MAGIC_VALUE_V0, RecordBatch.MAGIC_VALUE_V1, RecordBatch.MAGIC_VALUE_V2)
-    val compressionTypes = Seq(CompressionType.NONE, CompressionType.LZ4)
-    for (magic <- magicVals; compression <- compressionTypes) {
+    def testMain(magic: Byte, compression: CompressionType) {
+      val firstOffset = 4L
       val batch = TestUtils.records(List(new SimpleRecord("k1".getBytes, "v1".getBytes),
-                                         new SimpleRecord("k2".getBytes, "v2".getBytes),
-                                         new SimpleRecord("k3".getBytes, "v3".getBytes)),
-                                    magicValue = magic, codec = compression,
-                                    baseOffset = firstOffset)
+        new SimpleRecord("k2".getBytes, "v2".getBytes),
+        new SimpleRecord("k3".getBytes, "v3".getBytes)),
+        magicValue = magic, codec = compression,
+        baseOffset = firstOffset)
 
       withClue(s"Magic=$magic, compressionType=$compression") {
         val exception = intercept[UnexpectedAppendOffsetException] {
           log.appendAsFollower(records = batch)
         }
         assertEquals(s"Magic=$magic, compressionType=$compression, UnexpectedAppendOffsetException#firstOffset",
-                     firstOffset, exception.firstOffset)
+          firstOffset, exception.firstOffset)
         assertEquals(s"Magic=$magic, compressionType=$compression, UnexpectedAppendOffsetException#lastOffset",
-                     firstOffset + 2, exception.lastOffset)
+          firstOffset + 2, exception.lastOffset)
       }
     }
+
+    // For this test, we need a batch with multiple records. It appears that the above TestUtils.records() actually
+    // returns multiple batches with single record for magic values V0 and V1 when compression type is NONE. Exclude
+    // these combinations.
+    testMain(RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE)
+    val magicVals = Seq(RecordBatch.MAGIC_VALUE_V0, RecordBatch.MAGIC_VALUE_V1, RecordBatch.MAGIC_VALUE_V2)
+    for (magic <- magicVals) {
+      testMain(magic, CompressionType.LZ4)
+    }
   }
 
   @Test

From 0d8d0f7ce76bdb9a6ec82c2e7917f935e3d9fe7b Mon Sep 17 00:00:00 2001
From: Ke Hu 
Date: Thu, 28 Jun 2018 14:25:10 -0700
Subject: [PATCH 1061/1071] [LI-HOTFIX] Add recompressionRate sensor TICKET =
 LI_DESCRIPTION =

Add recompressionRate sensor

RB=1310804
G=Kafka-Code-Reviews
R=mgharat
A=jkoshy

EXIT_CRITERIA = MANUAL ["None"]
---
 core/src/main/scala/kafka/log/Log.scala          |  9 +++++++--
 core/src/main/scala/kafka/log/LogValidator.scala | 15 ++++++++++-----
 core/src/main/scala/kafka/server/KafkaApis.scala |  1 +
 .../main/scala/kafka/server/ReplicaManager.scala | 16 ++++++++++++++++
 .../kafka/server/AbstractFetcherThreadTest.scala |  3 ++-
 5 files changed, 36 insertions(+), 8 deletions(-)

diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index 3db6cc8e11579..f2466de358dec 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -101,7 +101,8 @@ case class LogAppendInfo(var firstOffset: Option[Long],
                          offsetsMonotonic: Boolean,
                          lastOffsetOfFirstBatch: Long,
                          recordErrors: List[RecordError] = List(),
-                         errorMessage: String = null) {
+                         errorMessage: String = null,
+                         var recompressedBatchCount: Long = 0) {
   /**
    * Get the first offset if it exists, else get the last offset of the first batch
    * For magic versions 2 and newer, this method will return first offset. For magic versions
@@ -1115,7 +1116,8 @@ class Log(@volatile private var _dir: File,
         info1.offsetsMonotonic && info2.offsetsMonotonic,
         info1.lastOffsetOfFirstBatch,
         info1.recordErrors ++ info2.recordErrors,
-        info1.errorMessage + info2.errorMessage
+        info1.errorMessage + info2.errorMessage,
+        info1.recompressedBatchCount + info2.recompressedBatchCount
       )
     })
   }
@@ -1184,6 +1186,9 @@ class Log(@volatile private var _dir: File,
           appendInfo.offsetOfMaxTimestamp = validateAndOffsetAssignResult.shallowOffsetOfMaxTimestamp
           appendInfo.lastOffset = offset.value - 1
           appendInfo.recordConversionStats = validateAndOffsetAssignResult.recordConversionStats
+          // update stats for compressed/decompressed batch count
+          if (validateAndOffsetAssignResult.recompressApplied)
+            appendInfo.recompressedBatchCount = 1
           if (config.messageTimestampType == TimestampType.LOG_APPEND_TIME)
             appendInfo.logAppendTime = now
 
diff --git a/core/src/main/scala/kafka/log/LogValidator.scala b/core/src/main/scala/kafka/log/LogValidator.scala
index 274d67144691d..a80b73c99fc9a 100644
--- a/core/src/main/scala/kafka/log/LogValidator.scala
+++ b/core/src/main/scala/kafka/log/LogValidator.scala
@@ -252,7 +252,8 @@ private[log] object LogValidator extends Logging {
       maxTimestamp = info.maxTimestamp,
       shallowOffsetOfMaxTimestamp = info.shallowOffsetOfMaxTimestamp,
       messageSizeMaybeChanged = true,
-      recordConversionStats = recordConversionStats)
+      recordConversionStats = recordConversionStats,
+      recompressApplied = false)
   }
 
   private def assignOffsetsNonCompressed(records: MemoryRecords,
@@ -319,7 +320,8 @@ private[log] object LogValidator extends Logging {
       maxTimestamp = maxTimestamp,
       shallowOffsetOfMaxTimestamp = offsetOfMaxTimestamp,
       messageSizeMaybeChanged = false,
-      recordConversionStats = RecordConversionStats.EMPTY)
+      recordConversionStats = RecordConversionStats.EMPTY,
+      recompressApplied = false)
   }
 
   /**
@@ -465,7 +467,8 @@ private[log] object LogValidator extends Logging {
         maxTimestamp = maxTimestamp,
         shallowOffsetOfMaxTimestamp = lastOffset,
         messageSizeMaybeChanged = false,
-        recordConversionStats = recordConversionStats)
+        recordConversionStats = recordConversionStats,
+        recompressApplied = false)
     }
   }
 
@@ -510,7 +513,8 @@ private[log] object LogValidator extends Logging {
       maxTimestamp = info.maxTimestamp,
       shallowOffsetOfMaxTimestamp = info.shallowOffsetOfMaxTimestamp,
       messageSizeMaybeChanged = true,
-      recordConversionStats = recordConversionStats)
+      recordConversionStats = recordConversionStats,
+      recompressApplied = true)
   }
 
   private def validateKey(record: Record, batchIndex: Int, topicPartition: TopicPartition, compactedTopic: Boolean, brokerTopicStats: BrokerTopicStats) {
@@ -550,6 +554,7 @@ private[log] object LogValidator extends Logging {
                                              maxTimestamp: Long,
                                              shallowOffsetOfMaxTimestamp: Long,
                                              messageSizeMaybeChanged: Boolean,
-                                             recordConversionStats: RecordConversionStats)
+                                             recordConversionStats: RecordConversionStats,
+                                             recompressApplied: Boolean)
 
 }
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 8d816baa76e2a..9e8655b6a1be7 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -697,6 +697,7 @@ class KafkaApis(val requestChannel: RequestChannel,
                 // as possible. With KIP-283, we have the ability to lazily down-convert in a chunked manner. The lazy, chunked
                 // down-conversion always guarantees that at least one batch of messages is down-converted and sent out to the
                 // client.
+                replicaManager.incrementRecompressionCount()
                 new FetchResponse.PartitionData[BaseRecords](partitionData.error, partitionData.highWatermark,
                   partitionData.lastStableOffset, partitionData.logStartOffset,
                   partitionData.preferredReadReplica, partitionData.abortedTransactions,
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index a308c4ee2f0c4..5e0c994c94f9c 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -217,6 +217,7 @@ class ReplicaManager(val config: KafkaConfig,
   private val lastIsrPropagationMs = new AtomicLong(System.currentTimeMillis())
 
   private var logDirFailureHandler: LogDirFailureHandler = null
+  val recompressedBatchCount: AtomicLong = new AtomicLong(0)
 
   private class LogDirFailureHandler(name: String, haltBrokerOnDirFailure: Boolean) extends ShutdownableThread(name) {
     override def doWork(): Unit = {
@@ -277,6 +278,13 @@ class ReplicaManager(val config: KafkaConfig,
   val isrExpandRate: Meter = newMeter("IsrExpandsPerSec", "expands", TimeUnit.SECONDS)
   val isrShrinkRate: Meter = newMeter("IsrShrinksPerSec", "shrinks", TimeUnit.SECONDS)
   val failedIsrUpdatesRate: Meter = newMeter("FailedIsrUpdatesPerSec", "failedUpdates", TimeUnit.SECONDS)
+  val recompressionCount = newGauge(
+    "recompressionCount",
+    new Gauge[Long] {
+      def value = recompressedBatchCount.get()
+    }
+  )
+
 
   def underReplicatedPartitionCount: Int = leaderPartitionsIterator.count(_.isUnderReplicated)
 
@@ -292,6 +300,10 @@ class ReplicaManager(val config: KafkaConfig,
     }
   }
 
+  def incrementRecompressionCount() {
+    recompressedBatchCount.incrementAndGet()
+  }
+
   /**
    * This function periodically runs to see if ISR needs to be propagated. It propagates ISR when:
    * 1. There is ISR change not propagated yet.
@@ -821,6 +833,9 @@ class ReplicaManager(val config: KafkaConfig,
           val info = partition.appendRecordsToLeader(records, origin, requiredAcks)
           val numAppendedMessages = info.numMessages
 
+          // update stats for compressed or decompressed batches on broker
+          recompressedBatchCount.addAndGet(info.recompressedBatchCount)
+
           // update stats for successfully appended bytes and messages as bytesInRate and messageInRate
           brokerTopicStats.topicStats(topicPartition.topic).bytesInRate.mark(records.sizeInBytes)
           brokerTopicStats.topicStats(topicPartition.topic).bytesInTotal.inc(records.sizeInBytes)
@@ -1696,6 +1711,7 @@ class ReplicaManager(val config: KafkaConfig,
     removeMetric("UnderReplicatedPartitions")
     removeMetric("UnderMinIsrPartitionCount")
     removeMetric("AtMinIsrPartitionCount")
+    removeMetric("recompressedBatch")
   }
 
   // High watermark do not need to be checkpointed only when under unit tests
diff --git a/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala b/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala
index 3a44e38b2ccf9..7958fee626eb6 100644
--- a/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala
+++ b/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala
@@ -882,7 +882,8 @@ class AbstractFetcherThreadTest {
         shallowCount = batches.size,
         validBytes = partitionData.records.sizeInBytes,
         offsetsMonotonic = true,
-        lastOffsetOfFirstBatch = batches.headOption.map(_.lastOffset).getOrElse(-1)))
+        lastOffsetOfFirstBatch = batches.headOption.map(_.lastOffset).getOrElse(-1),
+        recompressedBatchCount = 0))
     }
 
     override def truncate(topicPartition: TopicPartition, truncationState: OffsetTruncationState): Unit = {

From 6f4eef0276fbc45f2cee57c0c0238de5b9abccab Mon Sep 17 00:00:00 2001
From: Lucas Wang 
Date: Tue, 16 Jun 2020 05:46:23 -0700
Subject: [PATCH 1062/1071] [LI-HOTFIX] Fix broken travis tests

TICKET =
LI_DESCRIPTION =
1. Fix broken travis test by removing the "gradle wrapper" step
2. Always run :clients:test and :core:test for each PR
3. Cleaned up the step of uploading archives
EXIT_CRITERIA = MANUAL [""]
---
 .travis.yml | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index c3999f1cb8daa..4e9f19ce4f14c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,18 +17,13 @@ language: java
 jdk:
   - oraclejdk8
   
-before_install:
-  - gradle wrapper
-
 # By default, the install step will run ./gradlew assemble, which actually builds. Skip this step and do the actual
 # build during the build step.
 install: true
 
-# Excluded integration tests for now because Travis will hit build timeout (50 minutes) when including them.
-# Also excluded streams unitTest because they often fail with "pure virtual method called" error (KAFKA-3502).
-# TODO: re-enable these tests when the mentioned issues are resolved.
+# exclude the streams test and connect test
 script:
-  - ./gradlew clean compileJava compileScala compileTestJava compileTestScala checkstyleMain checkstyleTest findbugsMain unitTest -x :streams:unitTest rat --no-daemon -PxmlFindBugsReport=true -PtestLoggingEvents=started,passed,skipped,failed 
+  - ./gradlew cleanTest :clients:test :core:test --no-daemon -PxmlFindBugsReport=true -PtestLoggingEvents=started,passed,skipped,failed 
 
 before_cache:
   - rm -f  $HOME/.gradle/caches/modules-2/modules-2.lock
@@ -43,7 +38,7 @@ cache:
 # This will triger publishing artifacts to bintray upon the creation of a new tag in the "x.y.z.w" format.
 deploy:
   provider: script
-  script: ./gradlew -Pversion=$TRAVIS_TAG uploadArchivesAll -x :connect:api:compileJava -x :connect:api:processResources -x :connect:api:classes -x :connect:api:copyDependantLibs -x :connect:api:jar -x :connect:json:compileJava -x :connect:json:processResources -x :connect:json:classes -x :connect:json:copyDependantLibs -x :connect:json:jar -x :connect:api:javadoc -x :connect:api:javadocJar -x :connect:api:srcJar -x :connect:api:compileTestJava -x :connect:api:processTestResources -x :connect:api:testClasses -x :connect:api:testJar -x :connect:api:testSrcJar -x :connect:api:signArchives -x :connect:api:uploadArchives -x :connect:basic-auth-extension:compileJava -x :connect:basic-auth-extension:processResources -x :connect:basic-auth-extension:classes -x :connect:basic-auth-extension:copyDependantLibs -x :connect:basic-auth-extension:jar -x :connect:basic-auth-extension:javadoc -x :connect:basic-auth-extension:javadocJar -x :connect:basic-auth-extension:srcJar -x :connect:basic-auth-extension:compileTestJava -x :connect:basic-auth-extension:processTestResources -x :connect:basic-auth-extension:testClasses -x :connect:basic-auth-extension:testJar -x :connect:basic-auth-extension:testSrcJar -x :connect:basic-auth-extension:signArchives -x :connect:basic-auth-extension:uploadArchives -x :connect:file:compileJava -x :connect:file:processResources -x :connect:file:classes -x :connect:file:copyDependantLibs -x :connect:file:jar -x :connect:file:javadoc -x :connect:file:javadocJar -x :connect:file:srcJar -x :connect:file:compileTestJava -x :connect:file:processTestResources -x :connect:file:testClasses -x :connect:file:testJar -x :connect:file:testSrcJar -x :connect:file:signArchives -x :connect:file:uploadArchives -x :connect:json:javadoc -x :connect:json:javadocJar -x :connect:json:srcJar -x :connect:json:compileTestJava -x :connect:json:processTestResources -x :connect:json:testClasses -x :connect:json:testJar -x :connect:json:testSrcJar -x :connect:json:signArchives -x :connect:json:uploadArchives -x :connect:transforms:compileJava -x :connect:transforms:processResources -x :connect:transforms:classes -x :connect:transforms:copyDependantLibs -x :connect:transforms:jar -x :connect:runtime:compileJava -x :connect:runtime:processResources -x :connect:runtime:classes -x :connect:runtime:copyDependantLibs -x :connect:runtime:jar -x :connect:runtime:javadoc -x :connect:runtime:javadocJar -x :connect:runtime:srcJar -x :connect:runtime:compileTestJava -x :connect:runtime:processTestResources -x :connect:runtime:testClasses -x :connect:runtime:testJar -x :connect:runtime:testSrcJar -x :connect:runtime:signArchives -x :connect:runtime:uploadArchives -x :connect:transforms:javadoc -x :connect:transforms:javadocJar -x :connect:transforms:srcJar -x :connect:transforms:compileTestJava -x :connect:transforms:processTestResources -x :connect:transforms:testClasses -x :connect:transforms:testJar -x :connect:transforms:testSrcJar -x :connect:transforms:signArchives -x :connect:transforms:uploadArchives -x :streams:compileJava -x :streams:processResources -x :streams:classes -x :streams:copyDependantLibs -x :streams:jar -x :streams:javadoc -x :streams:javadocJar -x :streams:srcJar -x :streams:test-utils:compileJava -x :streams:test-utils:processResources -x :streams:test-utils:classes -x :streams:test-utils:copyDependantLibs -x :streams:test-utils:jar -x :streams:compileTestJava -x :streams:processTestResources -x :streams:testClasses -x :streams:testJar -x :streams:testSrcJar -x :streams:signArchives -x :streams:uploadArchives -x :streams:examples:compileJava -x :streams:examples:processResources -x :streams:examples:classes -x :streams:examples:copyDependantLibs -x :streams:examples:jar -x :streams:examples:javadoc -x :streams:examples:javadocJar -x :streams:examples:srcJar -x :streams:examples:compileTestJava -x :streams:examples:processTestResources -x :streams:examples:testClasses -x :streams:examples:testJar -x :streams:examples:testSrcJar -x :streams:examples:signArchives -x :streams:examples:uploadArchives -x :streams:test-utils:javadoc -x :streams:test-utils:javadocJar -x :streams:test-utils:srcJar -x :streams:test-utils:compileTestJava -x :streams:test-utils:processTestResources -x :streams:test-utils:testClasses -x :streams:test-utils:testJar -x :streams:test-utils:testSrcJar -x :streams:test-utils:signArchives -x :streams:test-utils:uploadArchives -x :streams:streams-scala:compileJava -x :streams:streams-scala:compileScala -x :streams:streams-scala:uploadArchives -x :streams:streams-scala:jar -x :streams:streams-scala:test -x :streams:streams-scala:srcJar -x :streams:streams-scala:docsJar -x :core:compileTestJava -x :core:compileTestScala -x :core:processTestResources -x :core:testClasses --no-daemon
+  script: ./gradlew -Pversion=$TRAVIS_TAG :clients:uploadArchives :core:uploadArchives --no-daemon
   skip_cleanup: true
   on:
     tags: true

From f37a97f15670520dab071b0ba3d4d9c41fd37a27 Mon Sep 17 00:00:00 2001
From: Lucas Wang 
Date: Tue, 16 Jun 2020 10:28:14 -0700
Subject: [PATCH 1063/1071] [LI-HOTFIX] Ignore the flaky test
 MetricsTest.testGeneralBrokerTopicMetricsAreGreedilyRegistered

TICKET = KAFKA-9786
LI_DESCRIPTION =
The test is flaky since gradle can use the same jvm to run multiple tests concurrently

EXIT_CRITERIA = KAFKA-9786
---
 core/src/test/scala/unit/kafka/metrics/MetricsTest.scala | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala b/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
index 45d23c2a299c1..3c2d30abb53e7 100644
--- a/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
+++ b/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
@@ -22,7 +22,7 @@ import java.util.Properties
 import javax.management.ObjectName
 import com.yammer.metrics.Metrics
 import com.yammer.metrics.core.MetricPredicate
-import org.junit.Test
+import org.junit.{Ignore, Test}
 import org.junit.Assert._
 import kafka.integration.KafkaServerTestHarness
 import kafka.server._
@@ -76,6 +76,7 @@ class MetricsTest extends KafkaServerTestHarness with Logging {
   }
 
   @Test
+  @Ignore // reenable once it's not prone to the failure identified in KAFKA-9786
   def testGeneralBrokerTopicMetricsAreGreedilyRegistered(): Unit = {
     val topic = "test-broker-topic-metric"
     createTopic(topic, 2, 1)

From d8d3c0695e1342ace9465e9fcbc22ab51531b66b Mon Sep 17 00:00:00 2001
From: Lucas Wang 
Date: Tue, 16 Jun 2020 12:21:49 -0700
Subject: [PATCH 1064/1071] [LI-HOTFIX] Ignore the flaky test
 ReplicaManagerTest.testFencedErrorCausedByBecomeLeader

TICKET = KAFKA-9750
LI_DESCRIPTION =

EXIT_CRITERIA = Once we upgrade to 2.5.0 which has the fix in KAFKA-9750
---
 core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index 5730d90be6158..78d4782a7d553 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -50,7 +50,7 @@ import org.apache.kafka.common.utils.Utils
 import org.apache.kafka.common.{Node, TopicPartition}
 import org.easymock.{Capture, EasyMock}
 import org.junit.Assert._
-import org.junit.{After, Before, Test}
+import org.junit.{After, Before, Ignore, Test}
 import org.mockito.Mockito
 
 import scala.collection.JavaConverters._
@@ -250,6 +250,7 @@ class ReplicaManagerTest {
   }
 
   @Test
+  @Ignore // reenable once we upgrade to 2.5.0 KAFKA-9750
   def testFencedErrorCausedByBecomeLeader(): Unit = {
     testFencedErrorCausedByBecomeLeader(0)
     testFencedErrorCausedByBecomeLeader(1)

From 7baa0f356ea0790ed6407148ca59e442f88c955a Mon Sep 17 00:00:00 2001
From: Lucas Wang 
Date: Thu, 18 Jun 2020 09:34:39 -0700
Subject: [PATCH 1065/1071] [LI-HOTFIX] Bump the netty-handler version to
 4.1.49.Final

TICKET =
LI_DESCERIPTION =
This patch is to resolve the following failure that occured during ELR
io.netty:netty-handler:4.1.48.Final: Vulnerability found and is blocked by oss-canary:
vulnerability: netty-handler is vulnerable to man-in-the-middle attacks. The library uses an SSLEngine that does not verify certificate hostnames when establishing connections with clients by default. This allows an attacker to potentially intercept and modify network traffic in a successful man-in-the-middle attack.

Upon investigation, the netty-handler version 4.1.48.Final is pulled by zookeeper version 3.5.8
+--- org.apache.zookeeper:zookeeper:3.5.8
|    +--- io.netty:netty-handler:4.1.48.Final

EXIT_CRITERIA = when the transitive dependency version of netty-handler is above 4.1.48.Final
---
 build.gradle               | 3 +++
 gradle/dependencies.gradle | 7 +++++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index 62e196d42635a..60840bf20b9f7 100644
--- a/build.gradle
+++ b/build.gradle
@@ -746,6 +746,9 @@ project(':core') {
       exclude module: 'log4j'
       exclude module: 'netty'
     }
+    // pin the netty to 4.1.49.Final since the transitive dependent version 4.1.48.Final is vulnerable
+    compile libs.nettyHandler
+    compile libs.nettyTransportNativeEpoll
     // ZooKeeperMain depends on commons-cli but declares the dependency as `provided`
     compile libs.commonsCli
 
diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle
index 9e7bbc7259a52..cfce86dca4498 100644
--- a/gradle/dependencies.gradle
+++ b/gradle/dependencies.gradle
@@ -116,7 +116,8 @@ versions += [
   spotlessPlugin: "3.23.1",
   zookeeper: "3.5.8",
   zstd: "1.4.3-1",
-  conscrypt: "2.1.0"
+  conscrypt: "2.1.0",
+  netty: "4.1.49.Final"
 ]
 
 libs += [
@@ -187,5 +188,7 @@ libs += [
   mavenArtifact: "org.apache.maven:maven-artifact:$versions.mavenArtifact",
   zstd: "com.github.luben:zstd-jni:$versions.zstd",
   httpclient: "org.apache.httpcomponents:httpclient:$versions.httpclient",
-  conscrypt: "org.conscrypt:conscrypt-openjdk-uber:2.2.1"
+  conscrypt: "org.conscrypt:conscrypt-openjdk-uber:2.2.1",
+  nettyHandler: "io.netty:netty-handler:$versions.netty",
+  nettyTransportNativeEpoll: "io.netty:netty-transport-native-epoll:$versions.netty"
 ]

From 0a69c9d483f5d11ce72e9548ba008a80d760240c Mon Sep 17 00:00:00 2001
From: Lincong Li 
Date: Thu, 18 Jun 2020 10:27:20 -0700
Subject: [PATCH 1066/1071] [LI-HOTFIX] Add a logline to include connection ID
 when describing a request and response pair (#88)

TICKET =
LI_DESCRIPTION =
The method Observer.describeRequestAndResponse is used to generate
request/response information to log when the observer throws an
exception. The generated request/response information will be logged
together with the exception thrown by the observer.
Logging the connection ID along with the exception will help us
understand how the value of connection ID plays a part in the exception
and help us root cause the problem.

EXIT_CRITERIA = Never since this is a linkedin internal change
---
 core/src/main/scala/kafka/server/Observer.scala | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/src/main/scala/kafka/server/Observer.scala b/core/src/main/scala/kafka/server/Observer.scala
index 624f609fbdd47..591fa05189ebb 100644
--- a/core/src/main/scala/kafka/server/Observer.scala
+++ b/core/src/main/scala/kafka/server/Observer.scala
@@ -82,6 +82,7 @@ object Observer {
         requestDesc += " null"
       } else {
         requestDesc += (" header: " + request.header)
+        requestDesc += (" connection ID: " + request.context.connectionId)
         requestDesc += (" from service with principal: " +
           request.session.sanitizedUser +
           " IP address: " + request.session.clientAddress)

From b99d81cf208acb4188055f86fad6b2ac84c92c21 Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Fri, 10 Jan 2020 11:23:24 +0000
Subject: [PATCH 1067/1071] [LI-CHERRY-PICK] [700a931ff5] KAFKA-9188; Fix flaky
 test
 SslAdminClientIntegrationTest.testSynchronousAuthorizerAclUpdatesBlockRequestThreads
 (#7918)

The test blocks requests threads while sending the ACL update requests and occasionally hits request timeout. Updated the test to tolerate timeouts and retry the request for that case. Added an additional check to verify that the requests threads are unblocked when the semaphore is released, ensuring that the timeout is not due to blocked threads.

Reviewers: Manikumar Reddy 
---
 .../api/SslAdminClientIntegrationTest.scala    | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala
index 071bd4c05acc1..f5fb42133e52a 100644
--- a/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SslAdminClientIntegrationTest.scala
@@ -197,8 +197,24 @@ class SslAdminClientIntegrationTest extends SaslSslAdminClientIntegrationTest {
 
     // Release the semaphore and verify that all requests complete
     testSemaphore.release(aclFutures.size)
+    waitForNoBlockedRequestThreads()
     assertNotNull(describeFuture.get(10, TimeUnit.SECONDS))
-    aclFutures.foreach(_.all().get())
+    // If any of the requests time out since we were blocking the threads earlier, retry the request.
+    val numTimedOut = aclFutures.count { future =>
+      try {
+        future.all().get()
+        false
+      } catch {
+        case e: ExecutionException =>
+          if (e.getCause.isInstanceOf[org.apache.kafka.common.errors.TimeoutException])
+            true
+          else
+            throw e.getCause
+      }
+    }
+    (0 until numTimedOut)
+      .map(_ => createAdminClient.createAcls(List(acl2).asJava))
+      .foreach(_.all().get(30, TimeUnit.SECONDS))
   }
 
   /**

From a3c90ea70ac38c2632171ade1b4ee814f0491eb6 Mon Sep 17 00:00:00 2001
From: Adem Efe Gencer 
Date: Fri, 26 Jun 2020 11:03:43 -0700
Subject: [PATCH 1068/1071] [LI-HOTFIX] Move info-level consumer logs on
 seeking to an offset to debug-level. (#89)

TICKET =
LI_DESCRIPTION =
Info-level logs cause a line to be logged for each partition seek. This leads to an excessive number of logs in Kafka consumer users with frequent seek operations.
Pre-2.3 versions of Kafka use debug-level logs for seek operations. This patch brings this behavior back.

EXIT_CRITERIA = TICKET [KAFKA-8883]
---
 .../org/apache/kafka/clients/consumer/KafkaConsumer.java    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index f59a773553987..7fd7d2c3bf52f 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -1564,7 +1564,7 @@ public void seek(TopicPartition partition, long offset) {
 
         acquireAndEnsureOpen();
         try {
-            log.info("Seeking to offset {} for partition {}", offset, partition);
+            log.debug("Seeking to offset {} for partition {}", offset, partition);
             SubscriptionState.FetchPosition newPosition = new SubscriptionState.FetchPosition(
                     offset,
                     Optional.empty(), // This will ensure we skip validation
@@ -1594,10 +1594,10 @@ public void seek(TopicPartition partition, OffsetAndMetadata offsetAndMetadata)
         acquireAndEnsureOpen();
         try {
             if (offsetAndMetadata.leaderEpoch().isPresent()) {
-                log.info("Seeking to offset {} for partition {} with epoch {}",
+                log.debug("Seeking to offset {} for partition {} with epoch {}",
                         offset, partition, offsetAndMetadata.leaderEpoch().get());
             } else {
-                log.info("Seeking to offset {} for partition {}", offset, partition);
+                log.debug("Seeking to offset {} for partition {}", offset, partition);
             }
             Metadata.LeaderAndEpoch currentLeaderAndEpoch = this.metadata.leaderAndEpoch(partition);
             SubscriptionState.FetchPosition newPosition = new SubscriptionState.FetchPosition(

From 39b373440508e14b3501f9c5877c5466163b2297 Mon Sep 17 00:00:00 2001
From: Sean McCauliff 
Date: Thu, 8 Oct 2020 16:18:23 -0700
Subject: [PATCH 1069/1071] [LI-HOTFIX] Enhance quota logging with bounds.
 (#93)

This adds additional information to the quota logs, the bounds which are configured
are also logged.  When enforcement occurs there is an existing log, this is changed
to info (from debug).  Finally, many of the quota logs contain usless information
(NaN) or (0.0) value entries.  Logging for these entries is skipped entirely.

TICKET =
LI_DESCRIPTION = LIKAFKA-32112,
EXIT_CRITERIA = MANUAL this is not going to merged with upstream since it change
levels and I assume the original changes to dump all the quota metrics is not in upstream
this too will probably be rejected from upstream.
---
 .../kafka/server/ClientQuotaManager.scala     | 26 +++++++++++++------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/core/src/main/scala/kafka/server/ClientQuotaManager.scala b/core/src/main/scala/kafka/server/ClientQuotaManager.scala
index b83469980762b..eda7ac3f49a72 100644
--- a/core/src/main/scala/kafka/server/ClientQuotaManager.scala
+++ b/core/src/main/scala/kafka/server/ClientQuotaManager.scala
@@ -214,17 +214,27 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig,
     }
   }
 
-  def logQuotaMetrics(): Unit = {
+  private def isQuotaMetric(metricName : MetricName) : Boolean = {
+    metricName.group().equals(quotaType.toString) &&
+    (metricName.name().equals("byte-rate") || metricName.name().equals("throttle-time") ||
+    metricName.name().equals("request-time")) &&
+    (metricName.tags().containsKey("client-id") || metricName.tags().containsKey("user"))
+  }
+
+  def logQuotaMetrics() : Unit = {
     import scala.collection.JavaConverters._
     val metricsMap = metrics.metrics().asScala;
     metricsMap.foreach {
       case (metricName: MetricName, kafkaMetric: KafkaMetric) =>
-        if (metricName.group().equals(quotaType.toString) &&
-          (metricName.name().equals("byte-rate") || metricName.name().equals("throttle-time") ||
-            metricName.name().equals("request-time")) &&
-          (metricName.tags().containsKey("client-id") || metricName.tags().containsKey("user"))) {
-          info("Metric name (" + metricName + ") has value (" + kafkaMetric.metricValue() + ")")
-        }
+       if (isQuotaMetric(metricName)) {
+         val quotaValueStr = kafkaMetric.metricValue().toString()
+         if (!quotaValueStr.equals("NaN") && !quotaValueStr.equals("0.0")) {
+           val metricConfig = kafkaMetric.config()
+           val quota = metricConfig.quota();
+           val quotaBoundsStr = if (quota == null) "not configured" else quota.bound().toString()
+           info("Metric name (" + metricName + ") has value (" + quotaValueStr + ") with bound (" + quotaBoundsStr + ")")
+         }
+       }
     }
   }
 
@@ -284,7 +294,7 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig,
         // Compute the delay
         val clientMetric = metrics.metrics().get(clientRateMetricName(clientSensors.metricTags))
         throttleTimeMs = throttleTime(clientMetric).toInt
-        debug("Quota violated for sensor (%s). Delay time: (%d)".format(clientSensors.quotaSensor.name(), throttleTimeMs))
+        info("Quota violated for sensor (%s). Delay time: (%d)".format(clientSensors.quotaSensor.name(), throttleTimeMs))
     }
     throttleTimeMs
   }

From 996c476976a574114c06633f40edceb8bae7a0aa Mon Sep 17 00:00:00 2001
From: Lincong Li 
Date: Wed, 14 Oct 2020 10:21:25 -0700
Subject: [PATCH 1070/1071] [LI-HOTFIX] Disable auto topic creation triggered
 by fetching metadata (#94)

[LI-HOTFIX] Disable auto topic creation triggered by fetching metadata for all topics

TICKET = KAFKA-10606
LI_DESCRIPTION = LIKAFKA-32857
EXIT_CRITERIA = Once we cherry-pick the fix for KAFKA-10606
---
 core/src/main/scala/kafka/server/KafkaApis.scala | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 9e8655b6a1be7..2e7b4d375890d 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -1153,9 +1153,13 @@ class KafkaApis(val requestChannel: RequestChannel,
     val topicMetadata =
       if (authorizedTopics.isEmpty)
         Seq.empty[MetadataResponse.TopicMetadata]
-      else
-        getTopicMetadata(metadataRequest.allowAutoTopicCreation, authorizedTopics, request.context,
+      else {
+        // If the metadata request is fetching metadata for all topics, auto topic creation is NOT allowed. This is an
+        // internal hotfix for issue KAFKA-10606
+        val allowAutoTopicCreation = (!metadataRequest.isAllTopics) && metadataRequest.allowAutoTopicCreation
+        getTopicMetadata(allowAutoTopicCreation, authorizedTopics, request.context,
           errorUnavailableEndpoints, errorUnavailableListeners)
+      }
 
     var clusterAuthorizedOperations = 0
 

From 616fac45b46c7238516f6cfc9ebe683b4b968e60 Mon Sep 17 00:00:00 2001
From: Ke Hu 
Date: Mon, 26 Oct 2020 14:28:55 -0700
Subject: [PATCH 1071/1071] [LI-HOTFIX] Reject stale metadata from different
 cluster

---
 .../org/apache/kafka/clients/Metadata.java    | 31 ++++++++++++++++
 .../apache/kafka/clients/NetworkClient.java   | 11 +++++-
 .../StaleClusterMetadataException.java        | 35 +++++++++++++++++++
 .../apache/kafka/clients/ClientUtilsTest.java |  2 +-
 .../clients/ClusterConnectionStatesTest.java  |  8 ++---
 .../consumer/internals/FetcherTest.java       | 22 ++++++------
 6 files changed, 92 insertions(+), 17 deletions(-)
 create mode 100644 clients/src/main/java/org/apache/kafka/clients/StaleClusterMetadataException.java

diff --git a/clients/src/main/java/org/apache/kafka/clients/Metadata.java b/clients/src/main/java/org/apache/kafka/clients/Metadata.java
index b6fdc0696915e..47e4141c7da28 100644
--- a/clients/src/main/java/org/apache/kafka/clients/Metadata.java
+++ b/clients/src/main/java/org/apache/kafka/clients/Metadata.java
@@ -236,6 +236,8 @@ public synchronized void update(int requestVersion, MetadataResponse response, l
         if (isClosed())
             throw new IllegalStateException("Update requested after metadata close");
 
+        validateCluster(response.clusterId());
+
         if (requestVersion == this.requestVersion)
             this.needUpdate = false;
         else
@@ -263,6 +265,35 @@ public synchronized void update(int requestVersion, MetadataResponse response, l
         log.debug("Updated cluster metadata updateVersion {} to {}", this.updateVersion, this.cache);
     }
 
+    private void validateCluster(String newClusterId) {
+        String previousClusterId = this.cache.cluster().clusterResource().clusterId();
+
+        if (previousClusterId != null && newClusterId != null && !previousClusterId.equals(newClusterId)) {
+            // kafka cluster id is unique.
+            // On client side, the cluster id in Metadata is only null during bootstrap, and client is
+            // expected to talk to the same cluster during its life cycle. Therefore if cluster id changes
+            // during metadata update, meaning this metadata update response is from a different cluster,
+            // client should reject this response and not update cached cluster to the wrong cluster
+
+            // Since removing brokers and adding to another cluster can be common operation,
+            // it might be more suitable to just throw an exception for update and fail this update operation
+            // instead of bringing down the client completely, so that the metadata can be updated later from
+            // other brokers in the same cluster.
+
+            // this code path will only get executed when all brokers in original cluster has been removed and
+            // one/some brokers have been added to another. If only some of the original brokers were removed/added
+            // to another cluster, the client should get updated metadata with valid brokers from other hosts.
+            // so we can just throw an exception and close the network client
+
+            log.error("Received metadata from a different cluster {}, current cluster {} has no valid brokers anymore,"
+                + "please reboot the producer/consumer", newClusterId, previousClusterId);
+
+            throw new StaleClusterMetadataException(
+                "Trying to access a different cluster " + newClusterId + ", previous connected cluster " + previousClusterId);
+
+        }
+    }
+
     private void maybeSetMetadataError(Cluster cluster) {
         clearRecoverableErrors();
         checkInvalidTopics(cluster);
diff --git a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
index d8af1aee6441f..5d4193cb1ce26 100644
--- a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
@@ -561,7 +561,16 @@ public List poll(long timeout, long now) {
         long updatedNow = this.time.milliseconds();
         List responses = new ArrayList<>();
         handleCompletedSends(responses, updatedNow);
-        handleCompletedReceives(responses, updatedNow);
+
+        try {
+            handleCompletedReceives(responses, updatedNow);
+        } catch (StaleClusterMetadataException e) {
+            // upon stale metadata exception from a different cluster, close the network client
+            // the producer/consumer will hit closedSelector exception and close
+            log.error("Received stale metadata from a different cluster, close the network client now");
+            this.close();
+        }
+
         handleDisconnections(responses, updatedNow);
         handleConnections();
         handleInitiateApiVersionRequests(updatedNow);
diff --git a/clients/src/main/java/org/apache/kafka/clients/StaleClusterMetadataException.java b/clients/src/main/java/org/apache/kafka/clients/StaleClusterMetadataException.java
new file mode 100644
index 0000000000000..6e012345b1738
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/StaleClusterMetadataException.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients;
+
+import org.apache.kafka.common.errors.InvalidMetadataException;
+
+/**
+ * Thrown when current metadata is from a stale cluster that has a different cluster id
+ * than original cluster.
+ *
+ * Note: this is not a public API.
+ */
+public class StaleClusterMetadataException extends InvalidMetadataException {
+    private static final long serialVersionUID = 1L;
+
+    public StaleClusterMetadataException() {}
+
+    public StaleClusterMetadataException(String message) {
+        super(message);
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/clients/ClientUtilsTest.java b/clients/src/test/java/org/apache/kafka/clients/ClientUtilsTest.java
index 5dd65ed86e3c1..572896f56bd11 100644
--- a/clients/src/test/java/org/apache/kafka/clients/ClientUtilsTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/ClientUtilsTest.java
@@ -107,7 +107,7 @@ public void testResolveDnsLookup() throws UnknownHostException {
 
     @Test
     public void testResolveDnsLookupAllIps() throws UnknownHostException {
-        assertEquals(2, ClientUtils.resolve("kafka.apache.org", ClientDnsLookup.USE_ALL_DNS_IPS).size());
+        assertTrue(ClientUtils.resolve("kafka.apache.org", ClientDnsLookup.USE_ALL_DNS_IPS).size() > 1);
     }
 
     private List checkWithoutLookup(String... url) {
diff --git a/clients/src/test/java/org/apache/kafka/clients/ClusterConnectionStatesTest.java b/clients/src/test/java/org/apache/kafka/clients/ClusterConnectionStatesTest.java
index 2a427cc5bad48..eb9d22fc1643f 100644
--- a/clients/src/test/java/org/apache/kafka/clients/ClusterConnectionStatesTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/ClusterConnectionStatesTest.java
@@ -256,7 +256,7 @@ public void testSingleIPWithUseAll() throws UnknownHostException {
 
     @Test
     public void testMultipleIPsWithDefault() throws UnknownHostException {
-        assertEquals(2, ClientUtils.resolve(hostTwoIps, ClientDnsLookup.USE_ALL_DNS_IPS).size());
+        assertTrue(ClientUtils.resolve(hostTwoIps, ClientDnsLookup.USE_ALL_DNS_IPS).size() > 1);
 
         connectionStates.connecting(nodeId1, time.milliseconds(), hostTwoIps, ClientDnsLookup.DEFAULT);
         InetAddress currAddress = connectionStates.currentAddress(nodeId1);
@@ -266,7 +266,7 @@ public void testMultipleIPsWithDefault() throws UnknownHostException {
 
     @Test
     public void testMultipleIPsWithUseAll() throws UnknownHostException {
-        assertEquals(2, ClientUtils.resolve(hostTwoIps, ClientDnsLookup.USE_ALL_DNS_IPS).size());
+        assertTrue(ClientUtils.resolve(hostTwoIps, ClientDnsLookup.USE_ALL_DNS_IPS).size() > 1);
 
         connectionStates.connecting(nodeId1, time.milliseconds(), hostTwoIps, ClientDnsLookup.USE_ALL_DNS_IPS);
         InetAddress addr1 = connectionStates.currentAddress(nodeId1);
@@ -276,12 +276,12 @@ public void testMultipleIPsWithUseAll() throws UnknownHostException {
 
         connectionStates.connecting(nodeId1, time.milliseconds(), hostTwoIps, ClientDnsLookup.USE_ALL_DNS_IPS);
         InetAddress addr3 = connectionStates.currentAddress(nodeId1);
-        assertSame(addr1, addr3);
+        assertNotSame(addr1, addr3);
     }
 
     @Test
     public void testHostResolveChange() throws UnknownHostException, ReflectiveOperationException {
-        assertEquals(2, ClientUtils.resolve(hostTwoIps, ClientDnsLookup.USE_ALL_DNS_IPS).size());
+        assertTrue(ClientUtils.resolve(hostTwoIps, ClientDnsLookup.USE_ALL_DNS_IPS).size() > 1);
 
         connectionStates.connecting(nodeId1, time.milliseconds(), hostTwoIps, ClientDnsLookup.DEFAULT);
         InetAddress addr1 = connectionStates.currentAddress(nodeId1);
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
index feb9a9d40c874..27c198f705ef7 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
@@ -1182,7 +1182,7 @@ public void testFetchUnknownLeaderEpoch() {
     public void testEpochSetInFetchRequest() {
         buildFetcher();
         subscriptions.assignFromUser(singleton(tp0));
-        MetadataResponse metadataResponse = TestUtils.metadataUpdateWith("dummy", 1,
+        MetadataResponse metadataResponse = TestUtils.metadataUpdateWith(null, 1,
                 Collections.emptyMap(), Collections.singletonMap(topicName, 4), tp -> 99);
         client.updateMetadata(metadataResponse);
 
@@ -2460,7 +2460,7 @@ public void testGetOffsetByTimeWithPartitionsRetryCouldTriggerMetadataUpdate() {
             Errors.LEADER_NOT_AVAILABLE, Errors.FENCED_LEADER_EPOCH, Errors.UNKNOWN_LEADER_EPOCH);
 
         final int newLeaderEpoch = 3;
-        MetadataResponse updatedMetadata = TestUtils.metadataUpdateWith("dummy", 3,
+        MetadataResponse updatedMetadata = TestUtils.metadataUpdateWith(null, 3,
             singletonMap(topicName, Errors.NONE), singletonMap(topicName, 4), tp -> newLeaderEpoch);
 
         Node originalLeader = initialUpdateResponse.cluster().leaderFor(tp1);
@@ -2560,7 +2560,7 @@ public void testGetOffsetsIncludesLeaderEpoch() {
         client.updateMetadata(initialUpdateResponse);
 
         // Metadata update with leader epochs
-        MetadataResponse metadataResponse = TestUtils.metadataUpdateWith("dummy", 1,
+        MetadataResponse metadataResponse = TestUtils.metadataUpdateWith(null, 1,
                 Collections.emptyMap(), Collections.singletonMap(topicName, 4), tp -> 99);
         client.updateMetadata(metadataResponse);
 
@@ -3541,7 +3541,7 @@ public void testSubscriptionPositionUpdatedWithEpoch() {
         // Initialize the epoch=1
         Map partitionCounts = new HashMap<>();
         partitionCounts.put(tp0.topic(), 4);
-        MetadataResponse metadataResponse = TestUtils.metadataUpdateWith("dummy", 1, Collections.emptyMap(), partitionCounts, tp -> 1);
+        MetadataResponse metadataResponse = TestUtils.metadataUpdateWith(null, 1, Collections.emptyMap(), partitionCounts, tp -> 1);
         metadata.update(metadataResponse, 0L);
 
         // Seek
@@ -3572,7 +3572,7 @@ public void testOffsetValidationAwaitsNodeApiVersion() {
 
         final int epochOne = 1;
 
-        metadata.update(TestUtils.metadataUpdateWith("dummy", 1,
+        metadata.update(TestUtils.metadataUpdateWith(null, 1,
                 Collections.emptyMap(), partitionCounts, tp -> epochOne), 0L);
 
         Node node = metadata.fetch().nodes().get(0);
@@ -3619,7 +3619,7 @@ public void testOffsetValidationSkippedForOldBroker() {
         final int epochTwo = 2;
 
         // Start with metadata, epoch=1
-        metadata.update(TestUtils.metadataUpdateWith("dummy", 1,
+        metadata.update(TestUtils.metadataUpdateWith(null, 1,
                 Collections.emptyMap(), partitionCounts, tp -> epochOne), 0L);
 
         // Offset validation requires OffsetForLeaderEpoch request v3 or higher
@@ -3633,7 +3633,7 @@ public void testOffsetValidationSkippedForOldBroker() {
         subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(0, Optional.of(epochOne), leaderAndEpoch));
 
         // Update metadata to epoch=2, enter validation
-        metadata.update(TestUtils.metadataUpdateWith("dummy", 1,
+        metadata.update(TestUtils.metadataUpdateWith(null, 1,
                 Collections.emptyMap(), partitionCounts, tp -> epochTwo), 0L);
         fetcher.validateOffsetsIfNeeded();
 
@@ -3651,7 +3651,7 @@ public void testOffsetValidationHandlesSeekWithInflightOffsetForLeaderRequest()
 
         final int epochOne = 1;
 
-        metadata.update(TestUtils.metadataUpdateWith("dummy", 1, Collections.emptyMap(), partitionCounts, tp -> epochOne), 0L);
+        metadata.update(TestUtils.metadataUpdateWith(null, 1, Collections.emptyMap(), partitionCounts, tp -> epochOne), 0L);
 
         // Offset validation requires OffsetForLeaderEpoch request v3 or higher
         Node node = metadata.fetch().nodes().get(0);
@@ -3693,7 +3693,7 @@ public void testOffsetValidationFencing() {
         final int epochThree = 3;
 
         // Start with metadata, epoch=1
-        metadata.update(TestUtils.metadataUpdateWith("dummy", 1, Collections.emptyMap(), partitionCounts, tp -> epochOne), 0L);
+        metadata.update(TestUtils.metadataUpdateWith(null, 1, Collections.emptyMap(), partitionCounts, tp -> epochOne), 0L);
 
         // Offset validation requires OffsetForLeaderEpoch request v3 or higher
         Node node = metadata.fetch().nodes().get(0);
@@ -3704,7 +3704,7 @@ public void testOffsetValidationFencing() {
         subscriptions.seekValidated(tp0, new SubscriptionState.FetchPosition(0, Optional.of(epochOne), leaderAndEpoch));
 
         // Update metadata to epoch=2, enter validation
-        metadata.update(TestUtils.metadataUpdateWith("dummy", 1, Collections.emptyMap(), partitionCounts, tp -> epochTwo), 0L);
+        metadata.update(TestUtils.metadataUpdateWith(null, 1, Collections.emptyMap(), partitionCounts, tp -> epochTwo), 0L);
         fetcher.validateOffsetsIfNeeded();
         assertTrue(subscriptions.awaitingValidation(tp0));
 
@@ -3762,7 +3762,7 @@ public void testTruncationDetected() {
         // Initialize the epoch=2
         Map partitionCounts = new HashMap<>();
         partitionCounts.put(tp0.topic(), 4);
-        MetadataResponse metadataResponse = TestUtils.metadataUpdateWith("dummy", 1, Collections.emptyMap(), partitionCounts, tp -> 2);
+        MetadataResponse metadataResponse = TestUtils.metadataUpdateWith(null, 1, Collections.emptyMap(), partitionCounts, tp -> 2);
         metadata.update(metadataResponse, 0L);
 
         // Offset validation requires OffsetForLeaderEpoch request v3 or higher